lib/ipsec/ipsec_sad.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(c) 2019 Intel Corporation
   3  */
   4
   5 #include <string.h>
   6
   7 #include <rte_eal_memconfig.h>
   8 #include <rte_errno.h>
   9 #include <rte_hash.h>
  10 #include <rte_hash_crc.h>
  11 #include <rte_malloc.h>
  12 #include <rte_random.h>
  13 #include <rte_tailq.h>
  14
  15 #include "rte_ipsec_sad.h"
  16
  17 /*
  18  * Rules are stored in three hash tables depending on key_type.
  19  * Each rule will also be stored in SPI_ONLY table.
  20  * for each data entry within this table last two bits are reserved to
  21  * indicate presence of entries with the same SPI in DIP and DIP+SIP tables.
  22  */
  23
  24 #define SAD_PREFIX              "SAD_"
  25 /* "SAD_<name>" */
  26 #define SAD_FORMAT              SAD_PREFIX "%s"
  27
  28 #define DEFAULT_HASH_FUNC       rte_hash_crc
  29 #define MIN_HASH_ENTRIES        8U /* From rte_cuckoo_hash.h */
  30
  31 struct hash_cnt {
  32         uint32_t cnt_dip;
  33         uint32_t cnt_dip_sip;
  34 };
  35
  36 struct rte_ipsec_sad {
  37         char name[RTE_IPSEC_SAD_NAMESIZE];
  38         struct rte_hash *hash[RTE_IPSEC_SAD_KEY_TYPE_MASK];
  39         uint32_t keysize[RTE_IPSEC_SAD_KEY_TYPE_MASK];
  40         uint32_t init_val;
  41         /* Array to track number of more specific rules
  42          * (spi_dip or spi_dip_sip). Used only in add/delete
  43          * as a helper struct.
  44          */
  45         __extension__ struct hash_cnt cnt_arr[];
  46 };
  47
  48 TAILQ_HEAD(rte_ipsec_sad_list, rte_tailq_entry);
  49 static struct rte_tailq_elem rte_ipsec_sad_tailq = {
  50         .name = "RTE_IPSEC_SAD",
  51 };
  52 EAL_REGISTER_TAILQ(rte_ipsec_sad_tailq)
  53
  54 #define SET_BIT(ptr, bit)       (void *)((uintptr_t)(ptr) | (uintptr_t)(bit))
  55 #define CLEAR_BIT(ptr, bit)     (void *)((uintptr_t)(ptr) & ~(uintptr_t)(bit))
  56 #define GET_BIT(ptr, bit)       (void *)((uintptr_t)(ptr) & (uintptr_t)(bit))
  57
  58 /*
  59  * @internal helper function
  60  * Add a rule of type SPI_DIP or SPI_DIP_SIP.
  61  * Inserts a rule into an appropriate hash table,
  62  * updates the value for a given SPI in SPI_ONLY hash table
  63  * reflecting presence of more specific rule type in two LSBs.
  64  * Updates a counter that reflects the number of rules with the same SPI.
  65  */
  66 static inline int
  67 add_specific(struct rte_ipsec_sad *sad, const void *key,
  68                 int key_type, void *sa)
  69 {
  70         void *tmp_val;
  71         int ret, notexist;
  72
  73         /* Check if the key is present in the table.
  74          * Need for further accaunting in cnt_arr
  75          */
  76         ret = rte_hash_lookup_with_hash(sad->hash[key_type], key,
  77                 rte_hash_crc(key, sad->keysize[key_type], sad->init_val));
  78         notexist = (ret == -ENOENT);
  79
  80         /* Add an SA to the corresponding table.*/
  81         ret = rte_hash_add_key_with_hash_data(sad->hash[key_type], key,
  82                 rte_hash_crc(key, sad->keysize[key_type], sad->init_val), sa);
  83         if (ret != 0)
  84                 return ret;
  85
  86         /* Check if there is an entry in SPI only table with the same SPI */
  87         ret = rte_hash_lookup_with_hash_data(sad->hash[RTE_IPSEC_SAD_SPI_ONLY],
  88                 key, rte_hash_crc(key, sad->keysize[RTE_IPSEC_SAD_SPI_ONLY],
  89                 sad->init_val), &tmp_val);
  90         if (ret < 0)
  91                 tmp_val = NULL;
  92         tmp_val = SET_BIT(tmp_val, key_type);
  93
  94         /* Add an entry into SPI only table */
  95         ret = rte_hash_add_key_with_hash_data(
  96                 sad->hash[RTE_IPSEC_SAD_SPI_ONLY], key,
  97                 rte_hash_crc(key, sad->keysize[RTE_IPSEC_SAD_SPI_ONLY],
  98                 sad->init_val), tmp_val);
  99         if (ret != 0)
 100                 return ret;
 101
 102         /* Update a counter for a given SPI */
 103         ret = rte_hash_lookup_with_hash(sad->hash[RTE_IPSEC_SAD_SPI_ONLY], key,
 104                 rte_hash_crc(key, sad->keysize[RTE_IPSEC_SAD_SPI_ONLY],
 105                 sad->init_val));
 106         if (ret < 0)
 107                 return ret;
 108         if (key_type == RTE_IPSEC_SAD_SPI_DIP)
 109                 sad->cnt_arr[ret].cnt_dip += notexist;
 110         else
 111                 sad->cnt_arr[ret].cnt_dip_sip += notexist;
 112
 113         return 0;
 114 }
 115
 116 int
 117 rte_ipsec_sad_add(struct rte_ipsec_sad *sad,
 118                 const union rte_ipsec_sad_key *key,
 119                 int key_type, void *sa)
 120 {
 121         void *tmp_val;
 122         int ret;
 123
 124         if ((sad == NULL) || (key == NULL) || (sa == NULL) ||
 125                         /* sa must be 4 byte aligned */
 126                         (GET_BIT(sa, RTE_IPSEC_SAD_KEY_TYPE_MASK) != 0))
 127                 return -EINVAL;
 128
 129         /*
 130          * Rules are stored in three hash tables depending on key_type.
 131          * All rules will also have an entry in SPI_ONLY table, with entry
 132          * value's two LSB's also indicating presence of rule with this SPI
 133          * in other tables.
 134          */
 135         switch (key_type) {
 136         case(RTE_IPSEC_SAD_SPI_ONLY):
 137                 ret = rte_hash_lookup_with_hash_data(sad->hash[key_type],
 138                         key, rte_hash_crc(key, sad->keysize[key_type],
 139                         sad->init_val), &tmp_val);
 140                 if (ret >= 0)
 141                         tmp_val = SET_BIT(sa, GET_BIT(tmp_val,
 142                                 RTE_IPSEC_SAD_KEY_TYPE_MASK));
 143                 else
 144                         tmp_val = sa;
 145                 ret = rte_hash_add_key_with_hash_data(sad->hash[key_type],
 146                         key, rte_hash_crc(key, sad->keysize[key_type],
 147                         sad->init_val), tmp_val);
 148                 return ret;
 149         case(RTE_IPSEC_SAD_SPI_DIP):
 150         case(RTE_IPSEC_SAD_SPI_DIP_SIP):
 151                 return add_specific(sad, key, key_type, sa);
 152         default:
 153                 return -EINVAL;
 154         }
 155 }
 156
 157 /*
 158  * @internal helper function
 159  * Delete a rule of type SPI_DIP or SPI_DIP_SIP.
 160  * Deletes an entry from an appropriate hash table and decrements
 161  * an entry counter for given SPI.
 162  * If entry to remove is the last one with given SPI within the table,
 163  * then it will also update related entry in SPI_ONLY table.
 164  * Removes an entry from SPI_ONLY hash table if there no rule left
 165  * for this SPI in any table.
 166  */
 167 static inline int
 168 del_specific(struct rte_ipsec_sad *sad, const void *key, int key_type)
 169 {
 170         void *tmp_val;
 171         int ret;
 172         uint32_t *cnt;
 173
 174         /* Remove an SA from the corresponding table.*/
 175         ret = rte_hash_del_key_with_hash(sad->hash[key_type], key,
 176                 rte_hash_crc(key, sad->keysize[key_type], sad->init_val));
 177         if (ret < 0)
 178                 return ret;
 179
 180         /* Get an index of cnt_arr entry for a given SPI */
 181         ret = rte_hash_lookup_with_hash_data(sad->hash[RTE_IPSEC_SAD_SPI_ONLY],
 182                 key, rte_hash_crc(key, sad->keysize[RTE_IPSEC_SAD_SPI_ONLY],
 183                 sad->init_val), &tmp_val);
 184         if (ret < 0)
 185                 return ret;
 186         cnt = (key_type == RTE_IPSEC_SAD_SPI_DIP) ?
 187                         &sad->cnt_arr[ret].cnt_dip :
 188                         &sad->cnt_arr[ret].cnt_dip_sip;
 189         if (--(*cnt) != 0)
 190                 return 0;
 191
 192         /* corresponding counter is 0, clear the bit indicating
 193          * the presence of more specific rule for a given SPI.
 194          */
 195         tmp_val = CLEAR_BIT(tmp_val, key_type);
 196
 197         /* if there are no rules left with same SPI,
 198          * remove an entry from SPI_only table
 199          */
 200         if (tmp_val == NULL)
 201                 ret = rte_hash_del_key_with_hash(
 202                         sad->hash[RTE_IPSEC_SAD_SPI_ONLY], key,
 203                         rte_hash_crc(key, sad->keysize[RTE_IPSEC_SAD_SPI_ONLY],
 204                         sad->init_val));
 205         else
 206                 ret = rte_hash_add_key_with_hash_data(
 207                         sad->hash[RTE_IPSEC_SAD_SPI_ONLY], key,
 208                         rte_hash_crc(key, sad->keysize[RTE_IPSEC_SAD_SPI_ONLY],
 209                         sad->init_val), tmp_val);
 210         if (ret < 0)
 211                 return ret;
 212         return 0;
 213 }
 214
 215 int
 216 rte_ipsec_sad_del(struct rte_ipsec_sad *sad,
 217                 const union rte_ipsec_sad_key *key,
 218                 int key_type)
 219 {
 220         void *tmp_val;
 221         int ret;
 222
 223         if ((sad == NULL) || (key == NULL))
 224                 return -EINVAL;
 225         switch (key_type) {
 226         case(RTE_IPSEC_SAD_SPI_ONLY):
 227                 ret = rte_hash_lookup_with_hash_data(sad->hash[key_type],
 228                         key, rte_hash_crc(key, sad->keysize[key_type],
 229                         sad->init_val), &tmp_val);
 230                 if (ret < 0)
 231                         return ret;
 232                 if (GET_BIT(tmp_val, RTE_IPSEC_SAD_KEY_TYPE_MASK) == 0) {
 233                         ret = rte_hash_del_key_with_hash(sad->hash[key_type],
 234                                 key, rte_hash_crc(key, sad->keysize[key_type],
 235                                 sad->init_val));
 236                         ret = ret < 0 ? ret : 0;
 237                 } else {
 238                         tmp_val = GET_BIT(tmp_val,
 239                                 RTE_IPSEC_SAD_KEY_TYPE_MASK);
 240                         ret = rte_hash_add_key_with_hash_data(
 241                                 sad->hash[key_type], key,
 242                                 rte_hash_crc(key, sad->keysize[key_type],
 243                                 sad->init_val), tmp_val);
 244                 }
 245                 return ret;
 246         case(RTE_IPSEC_SAD_SPI_DIP):
 247         case(RTE_IPSEC_SAD_SPI_DIP_SIP):
 248                 return del_specific(sad, key, key_type);
 249         default:
 250                 return -EINVAL;
 251         }
 252 }
 253
 254 struct rte_ipsec_sad *
 255 rte_ipsec_sad_create(const char *name, const struct rte_ipsec_sad_conf *conf)
 256 {
 257         char hash_name[RTE_HASH_NAMESIZE];
 258         char sad_name[RTE_IPSEC_SAD_NAMESIZE];
 259         struct rte_tailq_entry *te;
 260         struct rte_ipsec_sad_list *sad_list;
 261         struct rte_ipsec_sad *sad, *tmp_sad = NULL;
 262         struct rte_hash_parameters hash_params = {0};
 263         int ret;
 264         uint32_t sa_sum;
 265
 266         RTE_BUILD_BUG_ON(RTE_IPSEC_SAD_KEY_TYPE_MASK != 3);
 267
 268         if ((name == NULL) || (conf == NULL) ||
 269                         ((conf->max_sa[RTE_IPSEC_SAD_SPI_ONLY] == 0) &&
 270                         (conf->max_sa[RTE_IPSEC_SAD_SPI_DIP] == 0) &&
 271                         (conf->max_sa[RTE_IPSEC_SAD_SPI_DIP_SIP] == 0))) {
 272                 rte_errno = EINVAL;
 273                 return NULL;
 274         }
 275
 276         ret = snprintf(sad_name, RTE_IPSEC_SAD_NAMESIZE, SAD_FORMAT, name);
 277         if (ret < 0 || ret >= RTE_IPSEC_SAD_NAMESIZE) {
 278                 rte_errno = ENAMETOOLONG;
 279                 return NULL;
 280         }
 281
 282         /** Init SAD*/
 283         sa_sum = RTE_MAX(MIN_HASH_ENTRIES,
 284                 conf->max_sa[RTE_IPSEC_SAD_SPI_ONLY]) +
 285                 RTE_MAX(MIN_HASH_ENTRIES,
 286                 conf->max_sa[RTE_IPSEC_SAD_SPI_DIP]) +
 287                 RTE_MAX(MIN_HASH_ENTRIES,
 288                 conf->max_sa[RTE_IPSEC_SAD_SPI_DIP_SIP]);
 289         sad = rte_zmalloc_socket(NULL, sizeof(*sad) +
 290                 (sizeof(struct hash_cnt) * sa_sum),
 291                 RTE_CACHE_LINE_SIZE, conf->socket_id);
 292         if (sad == NULL) {
 293                 rte_errno = ENOMEM;
 294                 return NULL;
 295         }
 296         memcpy(sad->name, sad_name, sizeof(sad_name));
 297
 298         hash_params.hash_func = DEFAULT_HASH_FUNC;
 299         hash_params.hash_func_init_val = rte_rand();
 300         sad->init_val = hash_params.hash_func_init_val;
 301         hash_params.socket_id = conf->socket_id;
 302         hash_params.name = hash_name;
 303         if (conf->flags & RTE_IPSEC_SAD_FLAG_RW_CONCURRENCY)
 304                 hash_params.extra_flag = RTE_HASH_EXTRA_FLAGS_RW_CONCURRENCY;
 305
 306         /** Init hash[RTE_IPSEC_SAD_SPI_ONLY] for SPI only */
 307         snprintf(hash_name, sizeof(hash_name), "sad_1_%p", sad);
 308         hash_params.key_len = sizeof(((struct rte_ipsec_sadv4_key *)0)->spi);
 309         sad->keysize[RTE_IPSEC_SAD_SPI_ONLY] = hash_params.key_len;
 310         hash_params.entries = sa_sum;
 311         sad->hash[RTE_IPSEC_SAD_SPI_ONLY] = rte_hash_create(&hash_params);
 312         if (sad->hash[RTE_IPSEC_SAD_SPI_ONLY] == NULL) {
 313                 rte_ipsec_sad_destroy(sad);
 314                 return NULL;
 315         }
 316
 317         /** Init hash[RTE_IPSEC_SAD_SPI_DIP] for SPI + DIP */
 318         snprintf(hash_name, sizeof(hash_name), "sad_2_%p", sad);
 319         if (conf->flags & RTE_IPSEC_SAD_FLAG_IPV6)
 320                 hash_params.key_len +=
 321                         sizeof(((struct rte_ipsec_sadv6_key *)0)->dip);
 322         else
 323                 hash_params.key_len +=
 324                         sizeof(((struct rte_ipsec_sadv4_key *)0)->dip);
 325         sad->keysize[RTE_IPSEC_SAD_SPI_DIP] = hash_params.key_len;
 326         hash_params.entries = RTE_MAX(MIN_HASH_ENTRIES,
 327                         conf->max_sa[RTE_IPSEC_SAD_SPI_DIP]);
 328         sad->hash[RTE_IPSEC_SAD_SPI_DIP] = rte_hash_create(&hash_params);
 329         if (sad->hash[RTE_IPSEC_SAD_SPI_DIP] == NULL) {
 330                 rte_ipsec_sad_destroy(sad);
 331                 return NULL;
 332         }
 333
 334         /** Init hash[[RTE_IPSEC_SAD_SPI_DIP_SIP] for SPI + DIP + SIP */
 335         snprintf(hash_name, sizeof(hash_name), "sad_3_%p", sad);
 336         if (conf->flags & RTE_IPSEC_SAD_FLAG_IPV6)
 337                 hash_params.key_len +=
 338                         sizeof(((struct rte_ipsec_sadv6_key *)0)->sip);
 339         else
 340                 hash_params.key_len +=
 341                         sizeof(((struct rte_ipsec_sadv4_key *)0)->sip);
 342         sad->keysize[RTE_IPSEC_SAD_SPI_DIP_SIP] = hash_params.key_len;
 343         hash_params.entries = RTE_MAX(MIN_HASH_ENTRIES,
 344                         conf->max_sa[RTE_IPSEC_SAD_SPI_DIP_SIP]);
 345         sad->hash[RTE_IPSEC_SAD_SPI_DIP_SIP] = rte_hash_create(&hash_params);
 346         if (sad->hash[RTE_IPSEC_SAD_SPI_DIP_SIP] == NULL) {
 347                 rte_ipsec_sad_destroy(sad);
 348                 return NULL;
 349         }
 350
 351         sad_list = RTE_TAILQ_CAST(rte_ipsec_sad_tailq.head,
 352                         rte_ipsec_sad_list);
 353         rte_mcfg_tailq_write_lock();
 354         /* guarantee there's no existing */
 355         TAILQ_FOREACH(te, sad_list, next) {
 356                 tmp_sad = (struct rte_ipsec_sad *)te->data;
 357                 if (strncmp(sad_name, tmp_sad->name,
 358                                 RTE_IPSEC_SAD_NAMESIZE) == 0)
 359                         break;
 360         }
 361         if (te != NULL) {
 362                 rte_mcfg_tailq_write_unlock();
 363                 rte_errno = EEXIST;
 364                 rte_ipsec_sad_destroy(sad);
 365                 return NULL;
 366         }
 367
 368         /* allocate tailq entry */
 369         te = rte_zmalloc("IPSEC_SAD_TAILQ_ENTRY", sizeof(*te), 0);
 370         if (te == NULL) {
 371                 rte_mcfg_tailq_write_unlock();
 372                 rte_errno = ENOMEM;
 373                 rte_ipsec_sad_destroy(sad);
 374                 return NULL;
 375         }
 376
 377         te->data = (void *)sad;
 378         TAILQ_INSERT_TAIL(sad_list, te, next);
 379         rte_mcfg_tailq_write_unlock();
 380         return sad;
 381 }
 382
 383 struct rte_ipsec_sad *
 384 rte_ipsec_sad_find_existing(const char *name)
 385 {
 386         char sad_name[RTE_IPSEC_SAD_NAMESIZE];
 387         struct rte_ipsec_sad *sad = NULL;
 388         struct rte_tailq_entry *te;
 389         struct rte_ipsec_sad_list *sad_list;
 390         int ret;
 391
 392         ret = snprintf(sad_name, RTE_IPSEC_SAD_NAMESIZE, SAD_FORMAT, name);
 393         if (ret < 0 || ret >= RTE_IPSEC_SAD_NAMESIZE) {
 394                 rte_errno = ENAMETOOLONG;
 395                 return NULL;
 396         }
 397
 398         sad_list = RTE_TAILQ_CAST(rte_ipsec_sad_tailq.head,
 399                 rte_ipsec_sad_list);
 400
 401         rte_mcfg_tailq_read_lock();
 402         TAILQ_FOREACH(te, sad_list, next) {
 403                 sad = (struct rte_ipsec_sad *) te->data;
 404                 if (strncmp(sad_name, sad->name, RTE_IPSEC_SAD_NAMESIZE) == 0)
 405                         break;
 406         }
 407         rte_mcfg_tailq_read_unlock();
 408
 409         if (te == NULL) {
 410                 rte_errno = ENOENT;
 411                 return NULL;
 412         }
 413
 414         return sad;
 415 }
 416
 417 void
 418 rte_ipsec_sad_destroy(struct rte_ipsec_sad *sad)
 419 {
 420         struct rte_tailq_entry *te;
 421         struct rte_ipsec_sad_list *sad_list;
 422
 423         if (sad == NULL)
 424                 return;
 425
 426         sad_list = RTE_TAILQ_CAST(rte_ipsec_sad_tailq.head,
 427                         rte_ipsec_sad_list);
 428         rte_mcfg_tailq_write_lock();
 429         TAILQ_FOREACH(te, sad_list, next) {
 430                 if (te->data == (void *)sad)
 431                         break;
 432         }
 433         if (te != NULL)
 434                 TAILQ_REMOVE(sad_list, te, next);
 435
 436         rte_mcfg_tailq_write_unlock();
 437
 438         rte_hash_free(sad->hash[RTE_IPSEC_SAD_SPI_ONLY]);
 439         rte_hash_free(sad->hash[RTE_IPSEC_SAD_SPI_DIP]);
 440         rte_hash_free(sad->hash[RTE_IPSEC_SAD_SPI_DIP_SIP]);
 441         rte_free(sad);
 442         rte_free(te);
 443 }
 444
 445 /*
 446  * @internal helper function
 447  * Lookup a batch of keys in three hash tables.
 448  * First lookup key in SPI_ONLY table.
 449  * If there is an entry for the corresponding SPI check its value.
 450  * Two least significant bits of the value indicate
 451  * the presence of more specific rule in other tables.
 452  * Perform additional lookup in corresponding hash tables
 453  * and update the value if lookup succeeded.
 454  */
 455 static int
 456 __ipsec_sad_lookup(const struct rte_ipsec_sad *sad,
 457                 const union rte_ipsec_sad_key *keys[], void *sa[], uint32_t n)
 458 {
 459         const void *keys_2[RTE_HASH_LOOKUP_BULK_MAX];
 460         const void *keys_3[RTE_HASH_LOOKUP_BULK_MAX];
 461         void *vals_2[RTE_HASH_LOOKUP_BULK_MAX] = {NULL};
 462         void *vals_3[RTE_HASH_LOOKUP_BULK_MAX] = {NULL};
 463         uint32_t idx_2[RTE_HASH_LOOKUP_BULK_MAX];
 464         uint32_t idx_3[RTE_HASH_LOOKUP_BULK_MAX];
 465         uint64_t mask_1, mask_2, mask_3;
 466         uint64_t map, map_spec;
 467         uint32_t n_2 = 0;
 468         uint32_t n_3 = 0;
 469         uint32_t i;
 470         int found = 0;
 471         hash_sig_t hash_sig[RTE_HASH_LOOKUP_BULK_MAX];
 472         hash_sig_t hash_sig_2[RTE_HASH_LOOKUP_BULK_MAX];
 473         hash_sig_t hash_sig_3[RTE_HASH_LOOKUP_BULK_MAX];
 474
 475         for (i = 0; i < n; i++) {
 476                 sa[i] = NULL;
 477                 hash_sig[i] = rte_hash_crc_4byte(keys[i]->v4.spi,
 478                         sad->init_val);
 479         }
 480
 481         /*
 482          * Lookup keys in SPI only hash table first.
 483          */
 484         rte_hash_lookup_with_hash_bulk_data(sad->hash[RTE_IPSEC_SAD_SPI_ONLY],
 485                 (const void **)keys, hash_sig, n, &mask_1, sa);
 486         for (map = mask_1; map; map &= (map - 1)) {
 487                 i = rte_bsf64(map);
 488                 /*
 489                  * if returned value indicates presence of a rule in other
 490                  * tables save a key for further lookup.
 491                  */
 492                 if ((uintptr_t)sa[i] & RTE_IPSEC_SAD_SPI_DIP_SIP) {
 493                         idx_3[n_3] = i;
 494                         hash_sig_3[n_3] = rte_hash_crc(keys[i],
 495                                 sad->keysize[RTE_IPSEC_SAD_SPI_DIP_SIP],
 496                                 sad->init_val);
 497                         keys_3[n_3++] = keys[i];
 498                 }
 499                 if ((uintptr_t)sa[i] & RTE_IPSEC_SAD_SPI_DIP) {
 500                         idx_2[n_2] = i;
 501                         hash_sig_2[n_2] = rte_hash_crc(keys[i],
 502                                 sad->keysize[RTE_IPSEC_SAD_SPI_DIP],
 503                                 sad->init_val);
 504                         keys_2[n_2++] = keys[i];
 505                 }
 506                 /* clear 2 LSB's which indicate the presence
 507                  * of more specific rules
 508                  */
 509                 sa[i] = CLEAR_BIT(sa[i], RTE_IPSEC_SAD_KEY_TYPE_MASK);
 510         }
 511
 512         /* Lookup for more specific rules in SPI_DIP table */
 513         if (n_2 != 0) {
 514                 rte_hash_lookup_with_hash_bulk_data(
 515                         sad->hash[RTE_IPSEC_SAD_SPI_DIP],
 516                         keys_2, hash_sig_2, n_2, &mask_2, vals_2);
 517                 for (map_spec = mask_2; map_spec; map_spec &= (map_spec - 1)) {
 518                         i = rte_bsf64(map_spec);
 519                         sa[idx_2[i]] = vals_2[i];
 520                 }
 521         }
 522         /* Lookup for more specific rules in SPI_DIP_SIP table */
 523         if (n_3 != 0) {
 524                 rte_hash_lookup_with_hash_bulk_data(
 525                         sad->hash[RTE_IPSEC_SAD_SPI_DIP_SIP],
 526                         keys_3, hash_sig_3, n_3, &mask_3, vals_3);
 527                 for (map_spec = mask_3; map_spec; map_spec &= (map_spec - 1)) {
 528                         i = rte_bsf64(map_spec);
 529                         sa[idx_3[i]] = vals_3[i];
 530                 }
 531         }
 532
 533         for (i = 0; i < n; i++)
 534                 found += (sa[i] != NULL);
 535
 536         return found;
 537 }
 538
 539 int
 540 rte_ipsec_sad_lookup(const struct rte_ipsec_sad *sad,
 541                 const union rte_ipsec_sad_key *keys[], void *sa[], uint32_t n)
 542 {
 543         uint32_t num, i = 0;
 544         int found = 0;
 545
 546         if (unlikely((sad == NULL) || (keys == NULL) || (sa == NULL)))
 547                 return -EINVAL;
 548
 549         do {
 550                 num = RTE_MIN(n - i, (uint32_t)RTE_HASH_LOOKUP_BULK_MAX);
 551                 found += __ipsec_sad_lookup(sad,
 552                         &keys[i], &sa[i], num);
 553                 i += num;
 554         } while (i != n);
 555
 556         return found;
 557 }