1 /* SPDX-License-Identifier: BSD-3-Clause
2 * Copyright(c) 2019 Intel Corporation
5 #include <rte_eal_memconfig.h>
9 #include <rte_malloc.h>
10 #include <rte_random.h>
11 #include <rte_rwlock.h>
12 #include <rte_tailq.h>
14 #include "rte_ipsec_sad.h"
17 * Rules are stored in three hash tables depending on key_type.
18 * Each rule will also be stored in SPI_ONLY table.
19 * for each data entry within this table last two bits are reserved to
20 * indicate presence of entries with the same SPI in DIP and DIP+SIP tables.
23 #define IPSEC_SAD_NAMESIZE 64
24 #define SAD_PREFIX "SAD_"
26 #define SAD_FORMAT SAD_PREFIX "%s"
28 #define DEFAULT_HASH_FUNC rte_jhash
29 #define MIN_HASH_ENTRIES 8U /* From rte_cuckoo_hash.h */
36 struct rte_ipsec_sad {
37 char name[IPSEC_SAD_NAMESIZE];
38 struct rte_hash *hash[RTE_IPSEC_SAD_KEY_TYPE_MASK];
39 /* Array to track number of more specific rules
40 * (spi_dip or spi_dip_sip). Used only in add/delete
43 __extension__ struct hash_cnt cnt_arr[];
46 TAILQ_HEAD(rte_ipsec_sad_list, rte_tailq_entry);
47 static struct rte_tailq_elem rte_ipsec_sad_tailq = {
48 .name = "RTE_IPSEC_SAD",
50 EAL_REGISTER_TAILQ(rte_ipsec_sad_tailq)
52 #define SET_BIT(ptr, bit) (void *)((uintptr_t)(ptr) | (uintptr_t)(bit))
53 #define CLEAR_BIT(ptr, bit) (void *)((uintptr_t)(ptr) & ~(uintptr_t)(bit))
54 #define GET_BIT(ptr, bit) (void *)((uintptr_t)(ptr) & (uintptr_t)(bit))
57 * @internal helper function
58 * Add a rule of type SPI_DIP or SPI_DIP_SIP.
59 * Inserts a rule into an appropriate hash table,
60 * updates the value for a given SPI in SPI_ONLY hash table
61 * reflecting presence of more specific rule type in two LSBs.
62 * Updates a counter that reflects the number of rules whith the same SPI.
65 add_specific(struct rte_ipsec_sad *sad, const void *key,
66 int key_type, void *sa)
71 /* Check if the key is present in the table.
72 * Need for further accaunting in cnt_arr
74 ret = rte_hash_lookup(sad->hash[key_type], key);
75 notexist = (ret == -ENOENT);
77 /* Add an SA to the corresponding table.*/
78 ret = rte_hash_add_key_data(sad->hash[key_type], key, sa);
82 /* Check if there is an entry in SPI only table with the same SPI */
83 ret = rte_hash_lookup_data(sad->hash[RTE_IPSEC_SAD_SPI_ONLY],
87 tmp_val = SET_BIT(tmp_val, key_type);
89 /* Add an entry into SPI only table */
90 ret = rte_hash_add_key_data(sad->hash[RTE_IPSEC_SAD_SPI_ONLY],
95 /* Update a counter for a given SPI */
96 ret = rte_hash_lookup(sad->hash[RTE_IPSEC_SAD_SPI_ONLY], key);
97 if (key_type == RTE_IPSEC_SAD_SPI_DIP)
98 sad->cnt_arr[ret].cnt_dip += notexist;
100 sad->cnt_arr[ret].cnt_dip_sip += notexist;
106 rte_ipsec_sad_add(struct rte_ipsec_sad *sad,
107 const union rte_ipsec_sad_key *key,
108 int key_type, void *sa)
113 if ((sad == NULL) || (key == NULL) || (sa == NULL) ||
114 /* sa must be 4 byte aligned */
115 (GET_BIT(sa, RTE_IPSEC_SAD_KEY_TYPE_MASK) != 0))
119 * Rules are stored in three hash tables depending on key_type.
120 * All rules will also have an entry in SPI_ONLY table, with entry
121 * value's two LSB's also indicating presence of rule with this SPI
125 case(RTE_IPSEC_SAD_SPI_ONLY):
126 ret = rte_hash_lookup_data(sad->hash[key_type],
129 tmp_val = SET_BIT(sa, GET_BIT(tmp_val,
130 RTE_IPSEC_SAD_KEY_TYPE_MASK));
133 ret = rte_hash_add_key_data(sad->hash[key_type],
136 case(RTE_IPSEC_SAD_SPI_DIP):
137 case(RTE_IPSEC_SAD_SPI_DIP_SIP):
138 return add_specific(sad, key, key_type, sa);
145 * @internal helper function
146 * Delete a rule of type SPI_DIP or SPI_DIP_SIP.
147 * Deletes an entry from an appropriate hash table and decrements
148 * an entry counter for given SPI.
149 * If entry to remove is the last one with given SPI within the table,
150 * then it will also update related entry in SPI_ONLY table.
151 * Removes an entry from SPI_ONLY hash table if there no rule left
152 * for this SPI in any table.
155 del_specific(struct rte_ipsec_sad *sad, const void *key, int key_type)
161 /* Remove an SA from the corresponding table.*/
162 ret = rte_hash_del_key(sad->hash[key_type], key);
166 /* Get an index of cnt_arr entry for a given SPI */
167 ret = rte_hash_lookup_data(sad->hash[RTE_IPSEC_SAD_SPI_ONLY],
171 cnt = (key_type == RTE_IPSEC_SAD_SPI_DIP) ?
172 &sad->cnt_arr[ret].cnt_dip :
173 &sad->cnt_arr[ret].cnt_dip_sip;
177 /* corresponding counter is 0, clear the bit indicating
178 * the presence of more specific rule for a given SPI.
180 tmp_val = CLEAR_BIT(tmp_val, key_type);
182 /* if there are no rules left with same SPI,
183 * remove an entry from SPI_only table
186 ret = rte_hash_del_key(sad->hash[RTE_IPSEC_SAD_SPI_ONLY], key);
188 ret = rte_hash_add_key_data(sad->hash[RTE_IPSEC_SAD_SPI_ONLY],
196 rte_ipsec_sad_del(struct rte_ipsec_sad *sad,
197 const union rte_ipsec_sad_key *key,
203 if ((sad == NULL) || (key == NULL))
206 case(RTE_IPSEC_SAD_SPI_ONLY):
207 ret = rte_hash_lookup_data(sad->hash[key_type],
211 if (GET_BIT(tmp_val, RTE_IPSEC_SAD_KEY_TYPE_MASK) == 0) {
212 ret = rte_hash_del_key(sad->hash[key_type],
214 ret = ret < 0 ? ret : 0;
216 tmp_val = GET_BIT(tmp_val,
217 RTE_IPSEC_SAD_KEY_TYPE_MASK);
218 ret = rte_hash_add_key_data(sad->hash[key_type],
222 case(RTE_IPSEC_SAD_SPI_DIP):
223 case(RTE_IPSEC_SAD_SPI_DIP_SIP):
224 return del_specific(sad, key, key_type);
230 struct rte_ipsec_sad *
231 rte_ipsec_sad_create(const char *name, const struct rte_ipsec_sad_conf *conf)
233 char hash_name[RTE_HASH_NAMESIZE];
234 char sad_name[IPSEC_SAD_NAMESIZE];
235 struct rte_tailq_entry *te;
236 struct rte_ipsec_sad_list *sad_list;
237 struct rte_ipsec_sad *sad, *tmp_sad = NULL;
238 struct rte_hash_parameters hash_params = {0};
242 RTE_BUILD_BUG_ON(RTE_IPSEC_SAD_KEY_TYPE_MASK != 3);
244 if ((name == NULL) || (conf == NULL) ||
245 ((conf->max_sa[RTE_IPSEC_SAD_SPI_ONLY] == 0) &&
246 (conf->max_sa[RTE_IPSEC_SAD_SPI_DIP] == 0) &&
247 (conf->max_sa[RTE_IPSEC_SAD_SPI_DIP_SIP] == 0))) {
252 ret = snprintf(sad_name, IPSEC_SAD_NAMESIZE, SAD_FORMAT, name);
253 if (ret < 0 || ret >= IPSEC_SAD_NAMESIZE) {
254 rte_errno = ENAMETOOLONG;
259 sa_sum = RTE_MAX(MIN_HASH_ENTRIES,
260 conf->max_sa[RTE_IPSEC_SAD_SPI_ONLY]) +
261 RTE_MAX(MIN_HASH_ENTRIES,
262 conf->max_sa[RTE_IPSEC_SAD_SPI_DIP]) +
263 RTE_MAX(MIN_HASH_ENTRIES,
264 conf->max_sa[RTE_IPSEC_SAD_SPI_DIP_SIP]);
265 sad = rte_zmalloc_socket(NULL, sizeof(*sad) +
266 (sizeof(struct hash_cnt) * sa_sum),
267 RTE_CACHE_LINE_SIZE, conf->socket_id);
272 memcpy(sad->name, sad_name, sizeof(sad_name));
274 hash_params.hash_func = DEFAULT_HASH_FUNC;
275 hash_params.hash_func_init_val = rte_rand();
276 hash_params.socket_id = conf->socket_id;
277 hash_params.name = hash_name;
278 if (conf->flags & RTE_IPSEC_SAD_FLAG_RW_CONCURRENCY)
279 hash_params.extra_flag = RTE_HASH_EXTRA_FLAGS_RW_CONCURRENCY;
281 /** Init hash[RTE_IPSEC_SAD_SPI_ONLY] for SPI only */
282 snprintf(hash_name, sizeof(hash_name), "sad_1_%p", sad);
283 hash_params.key_len = sizeof(((struct rte_ipsec_sadv4_key *)0)->spi);
284 hash_params.entries = sa_sum;
285 sad->hash[RTE_IPSEC_SAD_SPI_ONLY] = rte_hash_create(&hash_params);
286 if (sad->hash[RTE_IPSEC_SAD_SPI_ONLY] == NULL) {
287 rte_ipsec_sad_destroy(sad);
291 /** Init hash[RTE_IPSEC_SAD_SPI_DIP] for SPI + DIP */
292 snprintf(hash_name, sizeof(hash_name), "sad_2_%p", sad);
293 if (conf->flags & RTE_IPSEC_SAD_FLAG_IPV6)
294 hash_params.key_len +=
295 sizeof(((struct rte_ipsec_sadv6_key *)0)->dip);
297 hash_params.key_len +=
298 sizeof(((struct rte_ipsec_sadv4_key *)0)->dip);
299 hash_params.entries = RTE_MAX(MIN_HASH_ENTRIES,
300 conf->max_sa[RTE_IPSEC_SAD_SPI_DIP]);
301 sad->hash[RTE_IPSEC_SAD_SPI_DIP] = rte_hash_create(&hash_params);
302 if (sad->hash[RTE_IPSEC_SAD_SPI_DIP] == NULL) {
303 rte_ipsec_sad_destroy(sad);
307 /** Init hash[[RTE_IPSEC_SAD_SPI_DIP_SIP] for SPI + DIP + SIP */
308 snprintf(hash_name, sizeof(hash_name), "sad_3_%p", sad);
309 if (conf->flags & RTE_IPSEC_SAD_FLAG_IPV6)
310 hash_params.key_len +=
311 sizeof(((struct rte_ipsec_sadv6_key *)0)->sip);
313 hash_params.key_len +=
314 sizeof(((struct rte_ipsec_sadv4_key *)0)->sip);
315 hash_params.entries = RTE_MAX(MIN_HASH_ENTRIES,
316 conf->max_sa[RTE_IPSEC_SAD_SPI_DIP_SIP]);
317 sad->hash[RTE_IPSEC_SAD_SPI_DIP_SIP] = rte_hash_create(&hash_params);
318 if (sad->hash[RTE_IPSEC_SAD_SPI_DIP_SIP] == NULL) {
319 rte_ipsec_sad_destroy(sad);
323 sad_list = RTE_TAILQ_CAST(rte_ipsec_sad_tailq.head,
325 rte_mcfg_tailq_write_lock();
326 /* guarantee there's no existing */
327 TAILQ_FOREACH(te, sad_list, next) {
328 tmp_sad = (struct rte_ipsec_sad *)te->data;
329 if (strncmp(sad_name, tmp_sad->name, IPSEC_SAD_NAMESIZE) == 0)
333 rte_mcfg_tailq_write_unlock();
335 rte_ipsec_sad_destroy(sad);
339 /* allocate tailq entry */
340 te = rte_zmalloc("IPSEC_SAD_TAILQ_ENTRY", sizeof(*te), 0);
342 rte_mcfg_tailq_write_unlock();
344 rte_ipsec_sad_destroy(sad);
348 te->data = (void *)sad;
349 TAILQ_INSERT_TAIL(sad_list, te, next);
350 rte_mcfg_tailq_write_unlock();
354 struct rte_ipsec_sad *
355 rte_ipsec_sad_find_existing(const char *name)
357 char sad_name[IPSEC_SAD_NAMESIZE];
358 struct rte_ipsec_sad *sad = NULL;
359 struct rte_tailq_entry *te;
360 struct rte_ipsec_sad_list *sad_list;
363 ret = snprintf(sad_name, IPSEC_SAD_NAMESIZE, SAD_FORMAT, name);
364 if (ret < 0 || ret >= IPSEC_SAD_NAMESIZE) {
365 rte_errno = ENAMETOOLONG;
369 sad_list = RTE_TAILQ_CAST(rte_ipsec_sad_tailq.head,
372 rte_mcfg_tailq_read_lock();
373 TAILQ_FOREACH(te, sad_list, next) {
374 sad = (struct rte_ipsec_sad *) te->data;
375 if (strncmp(sad_name, sad->name, IPSEC_SAD_NAMESIZE) == 0)
378 rte_mcfg_tailq_read_unlock();
389 rte_ipsec_sad_destroy(struct rte_ipsec_sad *sad)
391 struct rte_tailq_entry *te;
392 struct rte_ipsec_sad_list *sad_list;
397 sad_list = RTE_TAILQ_CAST(rte_ipsec_sad_tailq.head,
399 rte_mcfg_tailq_write_lock();
400 TAILQ_FOREACH(te, sad_list, next) {
401 if (te->data == (void *)sad)
405 TAILQ_REMOVE(sad_list, te, next);
407 rte_mcfg_tailq_write_unlock();
409 rte_hash_free(sad->hash[RTE_IPSEC_SAD_SPI_ONLY]);
410 rte_hash_free(sad->hash[RTE_IPSEC_SAD_SPI_DIP]);
411 rte_hash_free(sad->hash[RTE_IPSEC_SAD_SPI_DIP_SIP]);
418 * @internal helper function
419 * Lookup a batch of keys in three hash tables.
420 * First lookup key in SPI_ONLY table.
421 * If there is an entry for the corresponding SPI check its value.
422 * Two least significant bits of the value indicate
423 * the presence of more specific rule in other tables.
424 * Perform additional lookup in corresponding hash tables
425 * and update the value if lookup succeeded.
428 __ipsec_sad_lookup(const struct rte_ipsec_sad *sad,
429 const union rte_ipsec_sad_key *keys[], void *sa[], uint32_t n)
431 const void *keys_2[RTE_HASH_LOOKUP_BULK_MAX];
432 const void *keys_3[RTE_HASH_LOOKUP_BULK_MAX];
433 void *vals_2[RTE_HASH_LOOKUP_BULK_MAX] = {NULL};
434 void *vals_3[RTE_HASH_LOOKUP_BULK_MAX] = {NULL};
435 uint32_t idx_2[RTE_HASH_LOOKUP_BULK_MAX];
436 uint32_t idx_3[RTE_HASH_LOOKUP_BULK_MAX];
437 uint64_t mask_1, mask_2, mask_3;
438 uint64_t map, map_spec;
444 for (i = 0; i < n; i++)
448 * Lookup keys in SPI only hash table first.
450 rte_hash_lookup_bulk_data(sad->hash[RTE_IPSEC_SAD_SPI_ONLY],
451 (const void **)keys, n, &mask_1, sa);
452 for (map = mask_1; map; map &= (map - 1)) {
455 * if returned value indicates presence of a rule in other
456 * tables save a key for further lookup.
458 if ((uintptr_t)sa[i] & RTE_IPSEC_SAD_SPI_DIP_SIP) {
460 keys_3[n_3++] = keys[i];
462 if ((uintptr_t)sa[i] & RTE_IPSEC_SAD_SPI_DIP) {
464 keys_2[n_2++] = keys[i];
466 /* clear 2 LSB's which indicate the presence
467 * of more specific rules
469 sa[i] = CLEAR_BIT(sa[i], RTE_IPSEC_SAD_KEY_TYPE_MASK);
472 /* Lookup for more specific rules in SPI_DIP table */
474 rte_hash_lookup_bulk_data(sad->hash[RTE_IPSEC_SAD_SPI_DIP],
475 keys_2, n_2, &mask_2, vals_2);
476 for (map_spec = mask_2; map_spec; map_spec &= (map_spec - 1)) {
477 i = rte_bsf64(map_spec);
478 sa[idx_2[i]] = vals_2[i];
481 /* Lookup for more specific rules in SPI_DIP_SIP table */
483 rte_hash_lookup_bulk_data(sad->hash[RTE_IPSEC_SAD_SPI_DIP_SIP],
484 keys_3, n_3, &mask_3, vals_3);
485 for (map_spec = mask_3; map_spec; map_spec &= (map_spec - 1)) {
486 i = rte_bsf64(map_spec);
487 sa[idx_3[i]] = vals_3[i];
491 for (i = 0; i < n; i++)
492 found += (sa[i] != NULL);
498 rte_ipsec_sad_lookup(const struct rte_ipsec_sad *sad,
499 const union rte_ipsec_sad_key *keys[], void *sa[], uint32_t n)
504 if (unlikely((sad == NULL) || (keys == NULL) || (sa == NULL)))
508 num = RTE_MIN(n - i, (uint32_t)RTE_HASH_LOOKUP_BULK_MAX);
509 found += __ipsec_sad_lookup(sad,
510 &keys[i], &sa[i], num);