lib/librte_ipsec/ipsec_sad.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(c) 2019 Intel Corporation
   3  */
   4
   5 #include <string.h>
   6
   7 #include <rte_eal_memconfig.h>
   8 #include <rte_errno.h>
   9 #include <rte_hash.h>
  10 #include <rte_hash_crc.h>
  11 #include <rte_malloc.h>
  12 #include <rte_random.h>
  13 #include <rte_rwlock.h>
  14 #include <rte_tailq.h>
  15
  16 #include "rte_ipsec_sad.h"
  17
  18 /*
  19  * Rules are stored in three hash tables depending on key_type.
  20  * Each rule will also be stored in SPI_ONLY table.
  21  * for each data entry within this table last two bits are reserved to
  22  * indicate presence of entries with the same SPI in DIP and DIP+SIP tables.
  23  */
  24
  25 #define SAD_PREFIX              "SAD_"
  26 /* "SAD_<name>" */
  27 #define SAD_FORMAT              SAD_PREFIX "%s"
  28
  29 #define DEFAULT_HASH_FUNC       rte_hash_crc
  30 #define MIN_HASH_ENTRIES        8U /* From rte_cuckoo_hash.h */
  31
  32 struct hash_cnt {
  33         uint32_t cnt_dip;
  34         uint32_t cnt_dip_sip;
  35 };
  36
  37 struct rte_ipsec_sad {
  38         char name[RTE_IPSEC_SAD_NAMESIZE];
  39         struct rte_hash *hash[RTE_IPSEC_SAD_KEY_TYPE_MASK];
  40         uint32_t keysize[RTE_IPSEC_SAD_KEY_TYPE_MASK];
  41         uint32_t init_val;
  42         /* Array to track number of more specific rules
  43          * (spi_dip or spi_dip_sip). Used only in add/delete
  44          * as a helper struct.
  45          */
  46         __extension__ struct hash_cnt cnt_arr[];
  47 };
  48
  49 TAILQ_HEAD(rte_ipsec_sad_list, rte_tailq_entry);
  50 static struct rte_tailq_elem rte_ipsec_sad_tailq = {
  51         .name = "RTE_IPSEC_SAD",
  52 };
  53 EAL_REGISTER_TAILQ(rte_ipsec_sad_tailq)
  54
  55 #define SET_BIT(ptr, bit)       (void *)((uintptr_t)(ptr) | (uintptr_t)(bit))
  56 #define CLEAR_BIT(ptr, bit)     (void *)((uintptr_t)(ptr) & ~(uintptr_t)(bit))
  57 #define GET_BIT(ptr, bit)       (void *)((uintptr_t)(ptr) & (uintptr_t)(bit))
  58
  59 /*
  60  * @internal helper function
  61  * Add a rule of type SPI_DIP or SPI_DIP_SIP.
  62  * Inserts a rule into an appropriate hash table,
  63  * updates the value for a given SPI in SPI_ONLY hash table
  64  * reflecting presence of more specific rule type in two LSBs.
  65  * Updates a counter that reflects the number of rules whith the same SPI.
  66  */
  67 static inline int
  68 add_specific(struct rte_ipsec_sad *sad, const void *key,
  69                 int key_type, void *sa)
  70 {
  71         void *tmp_val;
  72         int ret, notexist;
  73
  74         /* Check if the key is present in the table.
  75          * Need for further accaunting in cnt_arr
  76          */
  77         ret = rte_hash_lookup_with_hash(sad->hash[key_type], key,
  78                 rte_hash_crc(key, sad->keysize[key_type], sad->init_val));
  79         notexist = (ret == -ENOENT);
  80
  81         /* Add an SA to the corresponding table.*/
  82         ret = rte_hash_add_key_with_hash_data(sad->hash[key_type], key,
  83                 rte_hash_crc(key, sad->keysize[key_type], sad->init_val), sa);
  84         if (ret != 0)
  85                 return ret;
  86
  87         /* Check if there is an entry in SPI only table with the same SPI */
  88         ret = rte_hash_lookup_with_hash_data(sad->hash[RTE_IPSEC_SAD_SPI_ONLY],
  89                 key, rte_hash_crc(key, sad->keysize[RTE_IPSEC_SAD_SPI_ONLY],
  90                 sad->init_val), &tmp_val);
  91         if (ret < 0)
  92                 tmp_val = NULL;
  93         tmp_val = SET_BIT(tmp_val, key_type);
  94
  95         /* Add an entry into SPI only table */
  96         ret = rte_hash_add_key_with_hash_data(
  97                 sad->hash[RTE_IPSEC_SAD_SPI_ONLY], key,
  98                 rte_hash_crc(key, sad->keysize[RTE_IPSEC_SAD_SPI_ONLY],
  99                 sad->init_val), tmp_val);
 100         if (ret != 0)
 101                 return ret;
 102
 103         /* Update a counter for a given SPI */
 104         ret = rte_hash_lookup_with_hash(sad->hash[RTE_IPSEC_SAD_SPI_ONLY], key,
 105                 rte_hash_crc(key, sad->keysize[RTE_IPSEC_SAD_SPI_ONLY],
 106                 sad->init_val));
 107         if (key_type == RTE_IPSEC_SAD_SPI_DIP)
 108                 sad->cnt_arr[ret].cnt_dip += notexist;
 109         else
 110                 sad->cnt_arr[ret].cnt_dip_sip += notexist;
 111
 112         return 0;
 113 }
 114
 115 int
 116 rte_ipsec_sad_add(struct rte_ipsec_sad *sad,
 117                 const union rte_ipsec_sad_key *key,
 118                 int key_type, void *sa)
 119 {
 120         void *tmp_val;
 121         int ret;
 122
 123         if ((sad == NULL) || (key == NULL) || (sa == NULL) ||
 124                         /* sa must be 4 byte aligned */
 125                         (GET_BIT(sa, RTE_IPSEC_SAD_KEY_TYPE_MASK) != 0))
 126                 return -EINVAL;
 127
 128         /*
 129          * Rules are stored in three hash tables depending on key_type.
 130          * All rules will also have an entry in SPI_ONLY table, with entry
 131          * value's two LSB's also indicating presence of rule with this SPI
 132          * in other tables.
 133          */
 134         switch (key_type) {
 135         case(RTE_IPSEC_SAD_SPI_ONLY):
 136                 ret = rte_hash_lookup_with_hash_data(sad->hash[key_type],
 137                         key, rte_hash_crc(key, sad->keysize[key_type],
 138                         sad->init_val), &tmp_val);
 139                 if (ret >= 0)
 140                         tmp_val = SET_BIT(sa, GET_BIT(tmp_val,
 141                                 RTE_IPSEC_SAD_KEY_TYPE_MASK));
 142                 else
 143                         tmp_val = sa;
 144                 ret = rte_hash_add_key_with_hash_data(sad->hash[key_type],
 145                         key, rte_hash_crc(key, sad->keysize[key_type],
 146                         sad->init_val), tmp_val);
 147                 return ret;
 148         case(RTE_IPSEC_SAD_SPI_DIP):
 149         case(RTE_IPSEC_SAD_SPI_DIP_SIP):
 150                 return add_specific(sad, key, key_type, sa);
 151         default:
 152                 return -EINVAL;
 153         }
 154 }
 155
 156 /*
 157  * @internal helper function
 158  * Delete a rule of type SPI_DIP or SPI_DIP_SIP.
 159  * Deletes an entry from an appropriate hash table and decrements
 160  * an entry counter for given SPI.
 161  * If entry to remove is the last one with given SPI within the table,
 162  * then it will also update related entry in SPI_ONLY table.
 163  * Removes an entry from SPI_ONLY hash table if there no rule left
 164  * for this SPI in any table.
 165  */
 166 static inline int
 167 del_specific(struct rte_ipsec_sad *sad, const void *key, int key_type)
 168 {
 169         void *tmp_val;
 170         int ret;
 171         uint32_t *cnt;
 172
 173         /* Remove an SA from the corresponding table.*/
 174         ret = rte_hash_del_key_with_hash(sad->hash[key_type], key,
 175                 rte_hash_crc(key, sad->keysize[key_type], sad->init_val));
 176         if (ret < 0)
 177                 return ret;
 178
 179         /* Get an index of cnt_arr entry for a given SPI */
 180         ret = rte_hash_lookup_with_hash_data(sad->hash[RTE_IPSEC_SAD_SPI_ONLY],
 181                 key, rte_hash_crc(key, sad->keysize[RTE_IPSEC_SAD_SPI_ONLY],
 182                 sad->init_val), &tmp_val);
 183         if (ret < 0)
 184                 return ret;
 185         cnt = (key_type == RTE_IPSEC_SAD_SPI_DIP) ?
 186                         &sad->cnt_arr[ret].cnt_dip :
 187                         &sad->cnt_arr[ret].cnt_dip_sip;
 188         if (--(*cnt) != 0)
 189                 return 0;
 190
 191         /* corresponding counter is 0, clear the bit indicating
 192          * the presence of more specific rule for a given SPI.
 193          */
 194         tmp_val = CLEAR_BIT(tmp_val, key_type);
 195
 196         /* if there are no rules left with same SPI,
 197          * remove an entry from SPI_only table
 198          */
 199         if (tmp_val == NULL)
 200                 ret = rte_hash_del_key_with_hash(
 201                         sad->hash[RTE_IPSEC_SAD_SPI_ONLY], key,
 202                         rte_hash_crc(key, sad->keysize[RTE_IPSEC_SAD_SPI_ONLY],
 203                         sad->init_val));
 204         else
 205                 ret = rte_hash_add_key_with_hash_data(
 206                         sad->hash[RTE_IPSEC_SAD_SPI_ONLY], key,
 207                         rte_hash_crc(key, sad->keysize[RTE_IPSEC_SAD_SPI_ONLY],
 208                         sad->init_val), tmp_val);
 209         if (ret < 0)
 210                 return ret;
 211         return 0;
 212 }
 213
 214 int
 215 rte_ipsec_sad_del(struct rte_ipsec_sad *sad,
 216                 const union rte_ipsec_sad_key *key,
 217                 int key_type)
 218 {
 219         void *tmp_val;
 220         int ret;
 221
 222         if ((sad == NULL) || (key == NULL))
 223                 return -EINVAL;
 224         switch (key_type) {
 225         case(RTE_IPSEC_SAD_SPI_ONLY):
 226                 ret = rte_hash_lookup_with_hash_data(sad->hash[key_type],
 227                         key, rte_hash_crc(key, sad->keysize[key_type],
 228                         sad->init_val), &tmp_val);
 229                 if (ret < 0)
 230                         return ret;
 231                 if (GET_BIT(tmp_val, RTE_IPSEC_SAD_KEY_TYPE_MASK) == 0) {
 232                         ret = rte_hash_del_key_with_hash(sad->hash[key_type],
 233                                 key, rte_hash_crc(key, sad->keysize[key_type],
 234                                 sad->init_val));
 235                         ret = ret < 0 ? ret : 0;
 236                 } else {
 237                         tmp_val = GET_BIT(tmp_val,
 238                                 RTE_IPSEC_SAD_KEY_TYPE_MASK);
 239                         ret = rte_hash_add_key_with_hash_data(
 240                                 sad->hash[key_type], key,
 241                                 rte_hash_crc(key, sad->keysize[key_type],
 242                                 sad->init_val), tmp_val);
 243                 }
 244                 return ret;
 245         case(RTE_IPSEC_SAD_SPI_DIP):
 246         case(RTE_IPSEC_SAD_SPI_DIP_SIP):
 247                 return del_specific(sad, key, key_type);
 248         default:
 249                 return -EINVAL;
 250         }
 251 }
 252
 253 struct rte_ipsec_sad *
 254 rte_ipsec_sad_create(const char *name, const struct rte_ipsec_sad_conf *conf)
 255 {
 256         char hash_name[RTE_HASH_NAMESIZE];
 257         char sad_name[RTE_IPSEC_SAD_NAMESIZE];
 258         struct rte_tailq_entry *te;
 259         struct rte_ipsec_sad_list *sad_list;
 260         struct rte_ipsec_sad *sad, *tmp_sad = NULL;
 261         struct rte_hash_parameters hash_params = {0};
 262         int ret;
 263         uint32_t sa_sum;
 264
 265         RTE_BUILD_BUG_ON(RTE_IPSEC_SAD_KEY_TYPE_MASK != 3);
 266
 267         if ((name == NULL) || (conf == NULL) ||
 268                         ((conf->max_sa[RTE_IPSEC_SAD_SPI_ONLY] == 0) &&
 269                         (conf->max_sa[RTE_IPSEC_SAD_SPI_DIP] == 0) &&
 270                         (conf->max_sa[RTE_IPSEC_SAD_SPI_DIP_SIP] == 0))) {
 271                 rte_errno = EINVAL;
 272                 return NULL;
 273         }
 274
 275         ret = snprintf(sad_name, RTE_IPSEC_SAD_NAMESIZE, SAD_FORMAT, name);
 276         if (ret < 0 || ret >= RTE_IPSEC_SAD_NAMESIZE) {
 277                 rte_errno = ENAMETOOLONG;
 278                 return NULL;
 279         }
 280
 281         /** Init SAD*/
 282         sa_sum = RTE_MAX(MIN_HASH_ENTRIES,
 283                 conf->max_sa[RTE_IPSEC_SAD_SPI_ONLY]) +
 284                 RTE_MAX(MIN_HASH_ENTRIES,
 285                 conf->max_sa[RTE_IPSEC_SAD_SPI_DIP]) +
 286                 RTE_MAX(MIN_HASH_ENTRIES,
 287                 conf->max_sa[RTE_IPSEC_SAD_SPI_DIP_SIP]);
 288         sad = rte_zmalloc_socket(NULL, sizeof(*sad) +
 289                 (sizeof(struct hash_cnt) * sa_sum),
 290                 RTE_CACHE_LINE_SIZE, conf->socket_id);
 291         if (sad == NULL) {
 292                 rte_errno = ENOMEM;
 293                 return NULL;
 294         }
 295         memcpy(sad->name, sad_name, sizeof(sad_name));
 296
 297         hash_params.hash_func = DEFAULT_HASH_FUNC;
 298         hash_params.hash_func_init_val = rte_rand();
 299         sad->init_val = hash_params.hash_func_init_val;
 300         hash_params.socket_id = conf->socket_id;
 301         hash_params.name = hash_name;
 302         if (conf->flags & RTE_IPSEC_SAD_FLAG_RW_CONCURRENCY)
 303                 hash_params.extra_flag = RTE_HASH_EXTRA_FLAGS_RW_CONCURRENCY;
 304
 305         /** Init hash[RTE_IPSEC_SAD_SPI_ONLY] for SPI only */
 306         snprintf(hash_name, sizeof(hash_name), "sad_1_%p", sad);
 307         hash_params.key_len = sizeof(((struct rte_ipsec_sadv4_key *)0)->spi);
 308         sad->keysize[RTE_IPSEC_SAD_SPI_ONLY] = hash_params.key_len;
 309         hash_params.entries = sa_sum;
 310         sad->hash[RTE_IPSEC_SAD_SPI_ONLY] = rte_hash_create(&hash_params);
 311         if (sad->hash[RTE_IPSEC_SAD_SPI_ONLY] == NULL) {
 312                 rte_ipsec_sad_destroy(sad);
 313                 return NULL;
 314         }
 315
 316         /** Init hash[RTE_IPSEC_SAD_SPI_DIP] for SPI + DIP */
 317         snprintf(hash_name, sizeof(hash_name), "sad_2_%p", sad);
 318         if (conf->flags & RTE_IPSEC_SAD_FLAG_IPV6)
 319                 hash_params.key_len +=
 320                         sizeof(((struct rte_ipsec_sadv6_key *)0)->dip);
 321         else
 322                 hash_params.key_len +=
 323                         sizeof(((struct rte_ipsec_sadv4_key *)0)->dip);
 324         sad->keysize[RTE_IPSEC_SAD_SPI_DIP] = hash_params.key_len;
 325         hash_params.entries = RTE_MAX(MIN_HASH_ENTRIES,
 326                         conf->max_sa[RTE_IPSEC_SAD_SPI_DIP]);
 327         sad->hash[RTE_IPSEC_SAD_SPI_DIP] = rte_hash_create(&hash_params);
 328         if (sad->hash[RTE_IPSEC_SAD_SPI_DIP] == NULL) {
 329                 rte_ipsec_sad_destroy(sad);
 330                 return NULL;
 331         }
 332
 333         /** Init hash[[RTE_IPSEC_SAD_SPI_DIP_SIP] for SPI + DIP + SIP */
 334         snprintf(hash_name, sizeof(hash_name), "sad_3_%p", sad);
 335         if (conf->flags & RTE_IPSEC_SAD_FLAG_IPV6)
 336                 hash_params.key_len +=
 337                         sizeof(((struct rte_ipsec_sadv6_key *)0)->sip);
 338         else
 339                 hash_params.key_len +=
 340                         sizeof(((struct rte_ipsec_sadv4_key *)0)->sip);
 341         sad->keysize[RTE_IPSEC_SAD_SPI_DIP_SIP] = hash_params.key_len;
 342         hash_params.entries = RTE_MAX(MIN_HASH_ENTRIES,
 343                         conf->max_sa[RTE_IPSEC_SAD_SPI_DIP_SIP]);
 344         sad->hash[RTE_IPSEC_SAD_SPI_DIP_SIP] = rte_hash_create(&hash_params);
 345         if (sad->hash[RTE_IPSEC_SAD_SPI_DIP_SIP] == NULL) {
 346                 rte_ipsec_sad_destroy(sad);
 347                 return NULL;
 348         }
 349
 350         sad_list = RTE_TAILQ_CAST(rte_ipsec_sad_tailq.head,
 351                         rte_ipsec_sad_list);
 352         rte_mcfg_tailq_write_lock();
 353         /* guarantee there's no existing */
 354         TAILQ_FOREACH(te, sad_list, next) {
 355                 tmp_sad = (struct rte_ipsec_sad *)te->data;
 356                 if (strncmp(sad_name, tmp_sad->name,
 357                                 RTE_IPSEC_SAD_NAMESIZE) == 0)
 358                         break;
 359         }
 360         if (te != NULL) {
 361                 rte_mcfg_tailq_write_unlock();
 362                 rte_errno = EEXIST;
 363                 rte_ipsec_sad_destroy(sad);
 364                 return NULL;
 365         }
 366
 367         /* allocate tailq entry */
 368         te = rte_zmalloc("IPSEC_SAD_TAILQ_ENTRY", sizeof(*te), 0);
 369         if (te == NULL) {
 370                 rte_mcfg_tailq_write_unlock();
 371                 rte_errno = ENOMEM;
 372                 rte_ipsec_sad_destroy(sad);
 373                 return NULL;
 374         }
 375
 376         te->data = (void *)sad;
 377         TAILQ_INSERT_TAIL(sad_list, te, next);
 378         rte_mcfg_tailq_write_unlock();
 379         return sad;
 380 }
 381
 382 struct rte_ipsec_sad *
 383 rte_ipsec_sad_find_existing(const char *name)
 384 {
 385         char sad_name[RTE_IPSEC_SAD_NAMESIZE];
 386         struct rte_ipsec_sad *sad = NULL;
 387         struct rte_tailq_entry *te;
 388         struct rte_ipsec_sad_list *sad_list;
 389         int ret;
 390
 391         ret = snprintf(sad_name, RTE_IPSEC_SAD_NAMESIZE, SAD_FORMAT, name);
 392         if (ret < 0 || ret >= RTE_IPSEC_SAD_NAMESIZE) {
 393                 rte_errno = ENAMETOOLONG;
 394                 return NULL;
 395         }
 396
 397         sad_list = RTE_TAILQ_CAST(rte_ipsec_sad_tailq.head,
 398                 rte_ipsec_sad_list);
 399
 400         rte_mcfg_tailq_read_lock();
 401         TAILQ_FOREACH(te, sad_list, next) {
 402                 sad = (struct rte_ipsec_sad *) te->data;
 403                 if (strncmp(sad_name, sad->name, RTE_IPSEC_SAD_NAMESIZE) == 0)
 404                         break;
 405         }
 406         rte_mcfg_tailq_read_unlock();
 407
 408         if (te == NULL) {
 409                 rte_errno = ENOENT;
 410                 return NULL;
 411         }
 412
 413         return sad;
 414 }
 415
 416 void
 417 rte_ipsec_sad_destroy(struct rte_ipsec_sad *sad)
 418 {
 419         struct rte_tailq_entry *te;
 420         struct rte_ipsec_sad_list *sad_list;
 421
 422         if (sad == NULL)
 423                 return;
 424
 425         sad_list = RTE_TAILQ_CAST(rte_ipsec_sad_tailq.head,
 426                         rte_ipsec_sad_list);
 427         rte_mcfg_tailq_write_lock();
 428         TAILQ_FOREACH(te, sad_list, next) {
 429                 if (te->data == (void *)sad)
 430                         break;
 431         }
 432         if (te != NULL)
 433                 TAILQ_REMOVE(sad_list, te, next);
 434
 435         rte_mcfg_tailq_write_unlock();
 436
 437         rte_hash_free(sad->hash[RTE_IPSEC_SAD_SPI_ONLY]);
 438         rte_hash_free(sad->hash[RTE_IPSEC_SAD_SPI_DIP]);
 439         rte_hash_free(sad->hash[RTE_IPSEC_SAD_SPI_DIP_SIP]);
 440         rte_free(sad);
 441         if (te != NULL)
 442                 rte_free(te);
 443 }
 444
 445 /*
 446  * @internal helper function
 447  * Lookup a batch of keys in three hash tables.
 448  * First lookup key in SPI_ONLY table.
 449  * If there is an entry for the corresponding SPI check its value.
 450  * Two least significant bits of the value indicate
 451  * the presence of more specific rule in other tables.
 452  * Perform additional lookup in corresponding hash tables
 453  * and update the value if lookup succeeded.
 454  */
 455 static int
 456 __ipsec_sad_lookup(const struct rte_ipsec_sad *sad,
 457                 const union rte_ipsec_sad_key *keys[], void *sa[], uint32_t n)
 458 {
 459         const void *keys_2[RTE_HASH_LOOKUP_BULK_MAX];
 460         const void *keys_3[RTE_HASH_LOOKUP_BULK_MAX];
 461         void *vals_2[RTE_HASH_LOOKUP_BULK_MAX] = {NULL};
 462         void *vals_3[RTE_HASH_LOOKUP_BULK_MAX] = {NULL};
 463         uint32_t idx_2[RTE_HASH_LOOKUP_BULK_MAX];
 464         uint32_t idx_3[RTE_HASH_LOOKUP_BULK_MAX];
 465         uint64_t mask_1, mask_2, mask_3;
 466         uint64_t map, map_spec;
 467         uint32_t n_2 = 0;
 468         uint32_t n_3 = 0;
 469         uint32_t i;
 470         int found = 0;
 471         hash_sig_t hash_sig[RTE_HASH_LOOKUP_BULK_MAX];
 472         hash_sig_t hash_sig_2[RTE_HASH_LOOKUP_BULK_MAX];
 473         hash_sig_t hash_sig_3[RTE_HASH_LOOKUP_BULK_MAX];
 474
 475         for (i = 0; i < n; i++) {
 476                 sa[i] = NULL;
 477                 hash_sig[i] = rte_hash_crc_4byte(keys[i]->v4.spi,
 478                         sad->init_val);
 479         }
 480
 481         /*
 482          * Lookup keys in SPI only hash table first.
 483          */
 484         rte_hash_lookup_with_hash_bulk_data(sad->hash[RTE_IPSEC_SAD_SPI_ONLY],
 485                 (const void **)keys, hash_sig, n, &mask_1, sa);
 486         for (map = mask_1; map; map &= (map - 1)) {
 487                 i = rte_bsf64(map);
 488                 /*
 489                  * if returned value indicates presence of a rule in other
 490                  * tables save a key for further lookup.
 491                  */
 492                 if ((uintptr_t)sa[i] & RTE_IPSEC_SAD_SPI_DIP_SIP) {
 493                         idx_3[n_3] = i;
 494                         hash_sig_3[n_3] = rte_hash_crc(keys[i],
 495                                 sad->keysize[RTE_IPSEC_SAD_SPI_DIP_SIP],
 496                                 sad->init_val);
 497                         keys_3[n_3++] = keys[i];
 498                 }
 499                 if ((uintptr_t)sa[i] & RTE_IPSEC_SAD_SPI_DIP) {
 500                         idx_2[n_2] = i;
 501                         hash_sig_2[n_2] = rte_hash_crc(keys[i],
 502                                 sad->keysize[RTE_IPSEC_SAD_SPI_DIP],
 503                                 sad->init_val);
 504                         keys_2[n_2++] = keys[i];
 505                 }
 506                 /* clear 2 LSB's which indicate the presence
 507                  * of more specific rules
 508                  */
 509                 sa[i] = CLEAR_BIT(sa[i], RTE_IPSEC_SAD_KEY_TYPE_MASK);
 510         }
 511
 512         /* Lookup for more specific rules in SPI_DIP table */
 513         if (n_2 != 0) {
 514                 rte_hash_lookup_with_hash_bulk_data(
 515                         sad->hash[RTE_IPSEC_SAD_SPI_DIP],
 516                         keys_2, hash_sig_2, n_2, &mask_2, vals_2);
 517                 for (map_spec = mask_2; map_spec; map_spec &= (map_spec - 1)) {
 518                         i = rte_bsf64(map_spec);
 519                         sa[idx_2[i]] = vals_2[i];
 520                 }
 521         }
 522         /* Lookup for more specific rules in SPI_DIP_SIP table */
 523         if (n_3 != 0) {
 524                 rte_hash_lookup_with_hash_bulk_data(
 525                         sad->hash[RTE_IPSEC_SAD_SPI_DIP_SIP],
 526                         keys_3, hash_sig_3, n_3, &mask_3, vals_3);
 527                 for (map_spec = mask_3; map_spec; map_spec &= (map_spec - 1)) {
 528                         i = rte_bsf64(map_spec);
 529                         sa[idx_3[i]] = vals_3[i];
 530                 }
 531         }
 532
 533         for (i = 0; i < n; i++)
 534                 found += (sa[i] != NULL);
 535
 536         return found;
 537 }
 538
 539 int
 540 rte_ipsec_sad_lookup(const struct rte_ipsec_sad *sad,
 541                 const union rte_ipsec_sad_key *keys[], void *sa[], uint32_t n)
 542 {
 543         uint32_t num, i = 0;
 544         int found = 0;
 545
 546         if (unlikely((sad == NULL) || (keys == NULL) || (sa == NULL)))
 547                 return -EINVAL;
 548
 549         do {
 550                 num = RTE_MIN(n - i, (uint32_t)RTE_HASH_LOOKUP_BULK_MAX);
 551                 found += __ipsec_sad_lookup(sad,
 552                         &keys[i], &sa[i], num);
 553                 i += num;
 554         } while (i != n);
 555
 556         return found;
 557 }