1 /* SPDX-License-Identifier: BSD-3-Clause
2 * Copyright(c) 2019 Intel Corporation
5 #include <rte_eal_memconfig.h>
9 #include <rte_malloc.h>
10 #include <rte_random.h>
11 #include <rte_rwlock.h>
12 #include <rte_tailq.h>
14 #include "rte_ipsec_sad.h"
17 * Rules are stored in three hash tables depending on key_type.
18 * Each rule will also be stored in SPI_ONLY table.
19 * for each data entry within this table last two bits are reserved to
20 * indicate presence of entries with the same SPI in DIP and DIP+SIP tables.
23 #define SAD_PREFIX "SAD_"
25 #define SAD_FORMAT SAD_PREFIX "%s"
27 #define DEFAULT_HASH_FUNC rte_jhash
28 #define MIN_HASH_ENTRIES 8U /* From rte_cuckoo_hash.h */
35 struct rte_ipsec_sad {
36 char name[RTE_IPSEC_SAD_NAMESIZE];
37 struct rte_hash *hash[RTE_IPSEC_SAD_KEY_TYPE_MASK];
38 /* Array to track number of more specific rules
39 * (spi_dip or spi_dip_sip). Used only in add/delete
42 __extension__ struct hash_cnt cnt_arr[];
45 TAILQ_HEAD(rte_ipsec_sad_list, rte_tailq_entry);
46 static struct rte_tailq_elem rte_ipsec_sad_tailq = {
47 .name = "RTE_IPSEC_SAD",
49 EAL_REGISTER_TAILQ(rte_ipsec_sad_tailq)
51 #define SET_BIT(ptr, bit) (void *)((uintptr_t)(ptr) | (uintptr_t)(bit))
52 #define CLEAR_BIT(ptr, bit) (void *)((uintptr_t)(ptr) & ~(uintptr_t)(bit))
53 #define GET_BIT(ptr, bit) (void *)((uintptr_t)(ptr) & (uintptr_t)(bit))
56 * @internal helper function
57 * Add a rule of type SPI_DIP or SPI_DIP_SIP.
58 * Inserts a rule into an appropriate hash table,
59 * updates the value for a given SPI in SPI_ONLY hash table
60 * reflecting presence of more specific rule type in two LSBs.
61 * Updates a counter that reflects the number of rules whith the same SPI.
64 add_specific(struct rte_ipsec_sad *sad, const void *key,
65 int key_type, void *sa)
70 /* Check if the key is present in the table.
71 * Need for further accaunting in cnt_arr
73 ret = rte_hash_lookup(sad->hash[key_type], key);
74 notexist = (ret == -ENOENT);
76 /* Add an SA to the corresponding table.*/
77 ret = rte_hash_add_key_data(sad->hash[key_type], key, sa);
81 /* Check if there is an entry in SPI only table with the same SPI */
82 ret = rte_hash_lookup_data(sad->hash[RTE_IPSEC_SAD_SPI_ONLY],
86 tmp_val = SET_BIT(tmp_val, key_type);
88 /* Add an entry into SPI only table */
89 ret = rte_hash_add_key_data(sad->hash[RTE_IPSEC_SAD_SPI_ONLY],
94 /* Update a counter for a given SPI */
95 ret = rte_hash_lookup(sad->hash[RTE_IPSEC_SAD_SPI_ONLY], key);
96 if (key_type == RTE_IPSEC_SAD_SPI_DIP)
97 sad->cnt_arr[ret].cnt_dip += notexist;
99 sad->cnt_arr[ret].cnt_dip_sip += notexist;
105 rte_ipsec_sad_add(struct rte_ipsec_sad *sad,
106 const union rte_ipsec_sad_key *key,
107 int key_type, void *sa)
112 if ((sad == NULL) || (key == NULL) || (sa == NULL) ||
113 /* sa must be 4 byte aligned */
114 (GET_BIT(sa, RTE_IPSEC_SAD_KEY_TYPE_MASK) != 0))
118 * Rules are stored in three hash tables depending on key_type.
119 * All rules will also have an entry in SPI_ONLY table, with entry
120 * value's two LSB's also indicating presence of rule with this SPI
124 case(RTE_IPSEC_SAD_SPI_ONLY):
125 ret = rte_hash_lookup_data(sad->hash[key_type],
128 tmp_val = SET_BIT(sa, GET_BIT(tmp_val,
129 RTE_IPSEC_SAD_KEY_TYPE_MASK));
132 ret = rte_hash_add_key_data(sad->hash[key_type],
135 case(RTE_IPSEC_SAD_SPI_DIP):
136 case(RTE_IPSEC_SAD_SPI_DIP_SIP):
137 return add_specific(sad, key, key_type, sa);
144 * @internal helper function
145 * Delete a rule of type SPI_DIP or SPI_DIP_SIP.
146 * Deletes an entry from an appropriate hash table and decrements
147 * an entry counter for given SPI.
148 * If entry to remove is the last one with given SPI within the table,
149 * then it will also update related entry in SPI_ONLY table.
150 * Removes an entry from SPI_ONLY hash table if there no rule left
151 * for this SPI in any table.
154 del_specific(struct rte_ipsec_sad *sad, const void *key, int key_type)
160 /* Remove an SA from the corresponding table.*/
161 ret = rte_hash_del_key(sad->hash[key_type], key);
165 /* Get an index of cnt_arr entry for a given SPI */
166 ret = rte_hash_lookup_data(sad->hash[RTE_IPSEC_SAD_SPI_ONLY],
170 cnt = (key_type == RTE_IPSEC_SAD_SPI_DIP) ?
171 &sad->cnt_arr[ret].cnt_dip :
172 &sad->cnt_arr[ret].cnt_dip_sip;
176 /* corresponding counter is 0, clear the bit indicating
177 * the presence of more specific rule for a given SPI.
179 tmp_val = CLEAR_BIT(tmp_val, key_type);
181 /* if there are no rules left with same SPI,
182 * remove an entry from SPI_only table
185 ret = rte_hash_del_key(sad->hash[RTE_IPSEC_SAD_SPI_ONLY], key);
187 ret = rte_hash_add_key_data(sad->hash[RTE_IPSEC_SAD_SPI_ONLY],
195 rte_ipsec_sad_del(struct rte_ipsec_sad *sad,
196 const union rte_ipsec_sad_key *key,
202 if ((sad == NULL) || (key == NULL))
205 case(RTE_IPSEC_SAD_SPI_ONLY):
206 ret = rte_hash_lookup_data(sad->hash[key_type],
210 if (GET_BIT(tmp_val, RTE_IPSEC_SAD_KEY_TYPE_MASK) == 0) {
211 ret = rte_hash_del_key(sad->hash[key_type],
213 ret = ret < 0 ? ret : 0;
215 tmp_val = GET_BIT(tmp_val,
216 RTE_IPSEC_SAD_KEY_TYPE_MASK);
217 ret = rte_hash_add_key_data(sad->hash[key_type],
221 case(RTE_IPSEC_SAD_SPI_DIP):
222 case(RTE_IPSEC_SAD_SPI_DIP_SIP):
223 return del_specific(sad, key, key_type);
229 struct rte_ipsec_sad *
230 rte_ipsec_sad_create(const char *name, const struct rte_ipsec_sad_conf *conf)
232 char hash_name[RTE_HASH_NAMESIZE];
233 char sad_name[RTE_IPSEC_SAD_NAMESIZE];
234 struct rte_tailq_entry *te;
235 struct rte_ipsec_sad_list *sad_list;
236 struct rte_ipsec_sad *sad, *tmp_sad = NULL;
237 struct rte_hash_parameters hash_params = {0};
241 RTE_BUILD_BUG_ON(RTE_IPSEC_SAD_KEY_TYPE_MASK != 3);
243 if ((name == NULL) || (conf == NULL) ||
244 ((conf->max_sa[RTE_IPSEC_SAD_SPI_ONLY] == 0) &&
245 (conf->max_sa[RTE_IPSEC_SAD_SPI_DIP] == 0) &&
246 (conf->max_sa[RTE_IPSEC_SAD_SPI_DIP_SIP] == 0))) {
251 ret = snprintf(sad_name, RTE_IPSEC_SAD_NAMESIZE, SAD_FORMAT, name);
252 if (ret < 0 || ret >= RTE_IPSEC_SAD_NAMESIZE) {
253 rte_errno = ENAMETOOLONG;
258 sa_sum = RTE_MAX(MIN_HASH_ENTRIES,
259 conf->max_sa[RTE_IPSEC_SAD_SPI_ONLY]) +
260 RTE_MAX(MIN_HASH_ENTRIES,
261 conf->max_sa[RTE_IPSEC_SAD_SPI_DIP]) +
262 RTE_MAX(MIN_HASH_ENTRIES,
263 conf->max_sa[RTE_IPSEC_SAD_SPI_DIP_SIP]);
264 sad = rte_zmalloc_socket(NULL, sizeof(*sad) +
265 (sizeof(struct hash_cnt) * sa_sum),
266 RTE_CACHE_LINE_SIZE, conf->socket_id);
271 memcpy(sad->name, sad_name, sizeof(sad_name));
273 hash_params.hash_func = DEFAULT_HASH_FUNC;
274 hash_params.hash_func_init_val = rte_rand();
275 hash_params.socket_id = conf->socket_id;
276 hash_params.name = hash_name;
277 if (conf->flags & RTE_IPSEC_SAD_FLAG_RW_CONCURRENCY)
278 hash_params.extra_flag = RTE_HASH_EXTRA_FLAGS_RW_CONCURRENCY;
280 /** Init hash[RTE_IPSEC_SAD_SPI_ONLY] for SPI only */
281 snprintf(hash_name, sizeof(hash_name), "sad_1_%p", sad);
282 hash_params.key_len = sizeof(((struct rte_ipsec_sadv4_key *)0)->spi);
283 hash_params.entries = sa_sum;
284 sad->hash[RTE_IPSEC_SAD_SPI_ONLY] = rte_hash_create(&hash_params);
285 if (sad->hash[RTE_IPSEC_SAD_SPI_ONLY] == NULL) {
286 rte_ipsec_sad_destroy(sad);
290 /** Init hash[RTE_IPSEC_SAD_SPI_DIP] for SPI + DIP */
291 snprintf(hash_name, sizeof(hash_name), "sad_2_%p", sad);
292 if (conf->flags & RTE_IPSEC_SAD_FLAG_IPV6)
293 hash_params.key_len +=
294 sizeof(((struct rte_ipsec_sadv6_key *)0)->dip);
296 hash_params.key_len +=
297 sizeof(((struct rte_ipsec_sadv4_key *)0)->dip);
298 hash_params.entries = RTE_MAX(MIN_HASH_ENTRIES,
299 conf->max_sa[RTE_IPSEC_SAD_SPI_DIP]);
300 sad->hash[RTE_IPSEC_SAD_SPI_DIP] = rte_hash_create(&hash_params);
301 if (sad->hash[RTE_IPSEC_SAD_SPI_DIP] == NULL) {
302 rte_ipsec_sad_destroy(sad);
306 /** Init hash[[RTE_IPSEC_SAD_SPI_DIP_SIP] for SPI + DIP + SIP */
307 snprintf(hash_name, sizeof(hash_name), "sad_3_%p", sad);
308 if (conf->flags & RTE_IPSEC_SAD_FLAG_IPV6)
309 hash_params.key_len +=
310 sizeof(((struct rte_ipsec_sadv6_key *)0)->sip);
312 hash_params.key_len +=
313 sizeof(((struct rte_ipsec_sadv4_key *)0)->sip);
314 hash_params.entries = RTE_MAX(MIN_HASH_ENTRIES,
315 conf->max_sa[RTE_IPSEC_SAD_SPI_DIP_SIP]);
316 sad->hash[RTE_IPSEC_SAD_SPI_DIP_SIP] = rte_hash_create(&hash_params);
317 if (sad->hash[RTE_IPSEC_SAD_SPI_DIP_SIP] == NULL) {
318 rte_ipsec_sad_destroy(sad);
322 sad_list = RTE_TAILQ_CAST(rte_ipsec_sad_tailq.head,
324 rte_mcfg_tailq_write_lock();
325 /* guarantee there's no existing */
326 TAILQ_FOREACH(te, sad_list, next) {
327 tmp_sad = (struct rte_ipsec_sad *)te->data;
328 if (strncmp(sad_name, tmp_sad->name,
329 RTE_IPSEC_SAD_NAMESIZE) == 0)
333 rte_mcfg_tailq_write_unlock();
335 rte_ipsec_sad_destroy(sad);
339 /* allocate tailq entry */
340 te = rte_zmalloc("IPSEC_SAD_TAILQ_ENTRY", sizeof(*te), 0);
342 rte_mcfg_tailq_write_unlock();
344 rte_ipsec_sad_destroy(sad);
348 te->data = (void *)sad;
349 TAILQ_INSERT_TAIL(sad_list, te, next);
350 rte_mcfg_tailq_write_unlock();
354 struct rte_ipsec_sad *
355 rte_ipsec_sad_find_existing(const char *name)
357 char sad_name[RTE_IPSEC_SAD_NAMESIZE];
358 struct rte_ipsec_sad *sad = NULL;
359 struct rte_tailq_entry *te;
360 struct rte_ipsec_sad_list *sad_list;
363 ret = snprintf(sad_name, RTE_IPSEC_SAD_NAMESIZE, SAD_FORMAT, name);
364 if (ret < 0 || ret >= RTE_IPSEC_SAD_NAMESIZE) {
365 rte_errno = ENAMETOOLONG;
369 sad_list = RTE_TAILQ_CAST(rte_ipsec_sad_tailq.head,
372 rte_mcfg_tailq_read_lock();
373 TAILQ_FOREACH(te, sad_list, next) {
374 sad = (struct rte_ipsec_sad *) te->data;
375 if (strncmp(sad_name, sad->name, RTE_IPSEC_SAD_NAMESIZE) == 0)
378 rte_mcfg_tailq_read_unlock();
389 rte_ipsec_sad_destroy(struct rte_ipsec_sad *sad)
391 struct rte_tailq_entry *te;
392 struct rte_ipsec_sad_list *sad_list;
397 sad_list = RTE_TAILQ_CAST(rte_ipsec_sad_tailq.head,
399 rte_mcfg_tailq_write_lock();
400 TAILQ_FOREACH(te, sad_list, next) {
401 if (te->data == (void *)sad)
405 TAILQ_REMOVE(sad_list, te, next);
407 rte_mcfg_tailq_write_unlock();
409 rte_hash_free(sad->hash[RTE_IPSEC_SAD_SPI_ONLY]);
410 rte_hash_free(sad->hash[RTE_IPSEC_SAD_SPI_DIP]);
411 rte_hash_free(sad->hash[RTE_IPSEC_SAD_SPI_DIP_SIP]);
418 * @internal helper function
419 * Lookup a batch of keys in three hash tables.
420 * First lookup key in SPI_ONLY table.
421 * If there is an entry for the corresponding SPI check its value.
422 * Two least significant bits of the value indicate
423 * the presence of more specific rule in other tables.
424 * Perform additional lookup in corresponding hash tables
425 * and update the value if lookup succeeded.
428 __ipsec_sad_lookup(const struct rte_ipsec_sad *sad,
429 const union rte_ipsec_sad_key *keys[], void *sa[], uint32_t n)
431 const void *keys_2[RTE_HASH_LOOKUP_BULK_MAX];
432 const void *keys_3[RTE_HASH_LOOKUP_BULK_MAX];
433 void *vals_2[RTE_HASH_LOOKUP_BULK_MAX] = {NULL};
434 void *vals_3[RTE_HASH_LOOKUP_BULK_MAX] = {NULL};
435 uint32_t idx_2[RTE_HASH_LOOKUP_BULK_MAX];
436 uint32_t idx_3[RTE_HASH_LOOKUP_BULK_MAX];
437 uint64_t mask_1, mask_2, mask_3;
438 uint64_t map, map_spec;
444 for (i = 0; i < n; i++)
448 * Lookup keys in SPI only hash table first.
450 rte_hash_lookup_bulk_data(sad->hash[RTE_IPSEC_SAD_SPI_ONLY],
451 (const void **)keys, n, &mask_1, sa);
452 for (map = mask_1; map; map &= (map - 1)) {
455 * if returned value indicates presence of a rule in other
456 * tables save a key for further lookup.
458 if ((uintptr_t)sa[i] & RTE_IPSEC_SAD_SPI_DIP_SIP) {
460 keys_3[n_3++] = keys[i];
462 if ((uintptr_t)sa[i] & RTE_IPSEC_SAD_SPI_DIP) {
464 keys_2[n_2++] = keys[i];
466 /* clear 2 LSB's which indicate the presence
467 * of more specific rules
469 sa[i] = CLEAR_BIT(sa[i], RTE_IPSEC_SAD_KEY_TYPE_MASK);
472 /* Lookup for more specific rules in SPI_DIP table */
474 rte_hash_lookup_bulk_data(sad->hash[RTE_IPSEC_SAD_SPI_DIP],
475 keys_2, n_2, &mask_2, vals_2);
476 for (map_spec = mask_2; map_spec; map_spec &= (map_spec - 1)) {
477 i = rte_bsf64(map_spec);
478 sa[idx_2[i]] = vals_2[i];
481 /* Lookup for more specific rules in SPI_DIP_SIP table */
483 rte_hash_lookup_bulk_data(sad->hash[RTE_IPSEC_SAD_SPI_DIP_SIP],
484 keys_3, n_3, &mask_3, vals_3);
485 for (map_spec = mask_3; map_spec; map_spec &= (map_spec - 1)) {
486 i = rte_bsf64(map_spec);
487 sa[idx_3[i]] = vals_3[i];
491 for (i = 0; i < n; i++)
492 found += (sa[i] != NULL);
498 rte_ipsec_sad_lookup(const struct rte_ipsec_sad *sad,
499 const union rte_ipsec_sad_key *keys[], void *sa[], uint32_t n)
504 if (unlikely((sad == NULL) || (keys == NULL) || (sa == NULL)))
508 num = RTE_MIN(n - i, (uint32_t)RTE_HASH_LOOKUP_BULK_MAX);
509 found += __ipsec_sad_lookup(sad,
510 &keys[i], &sa[i], num);