examples/ipsec-secgw/sa.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(c) 2016-2017 Intel Corporation
   3  */
   4
   5 /*
   6  * Security Associations
   7  */
   8 #include <sys/types.h>
   9 #include <netinet/in.h>
  10 #include <netinet/ip.h>
  11 #include <netinet/ip6.h>
  12
  13 #include <rte_memzone.h>
  14 #include <rte_crypto.h>
  15 #include <rte_security.h>
  16 #include <rte_cryptodev.h>
  17 #include <rte_byteorder.h>
  18 #include <rte_errno.h>
  19 #include <rte_ip.h>
  20 #include <rte_random.h>
  21 #include <rte_ethdev.h>
  22 #include <rte_malloc.h>
  23
  24 #include "ipsec.h"
  25 #include "esp.h"
  26 #include "parser.h"
  27
  28 #define IPDEFTTL 64
  29
  30 #define IP4_FULL_MASK (sizeof(((struct ip_addr *)NULL)->ip.ip4) * CHAR_BIT)
  31
  32 #define IP6_FULL_MASK (sizeof(((struct ip_addr *)NULL)->ip.ip6.ip6) * CHAR_BIT)
  33
  34 #define MBUF_NO_SEC_OFFLOAD(m) ((m->ol_flags & PKT_RX_SEC_OFFLOAD) == 0)
  35
  36 struct supported_cipher_algo {
  37         const char *keyword;
  38         enum rte_crypto_cipher_algorithm algo;
  39         uint16_t iv_len;
  40         uint16_t block_size;
  41         uint16_t key_len;
  42 };
  43
  44 struct supported_auth_algo {
  45         const char *keyword;
  46         enum rte_crypto_auth_algorithm algo;
  47         uint16_t digest_len;
  48         uint16_t key_len;
  49         uint8_t key_not_req;
  50 };
  51
  52 struct supported_aead_algo {
  53         const char *keyword;
  54         enum rte_crypto_aead_algorithm algo;
  55         uint16_t iv_len;
  56         uint16_t block_size;
  57         uint16_t digest_len;
  58         uint16_t key_len;
  59         uint8_t aad_len;
  60 };
  61
  62
  63 const struct supported_cipher_algo cipher_algos[] = {
  64         {
  65                 .keyword = "null",
  66                 .algo = RTE_CRYPTO_CIPHER_NULL,
  67                 .iv_len = 0,
  68                 .block_size = 4,
  69                 .key_len = 0
  70         },
  71         {
  72                 .keyword = "aes-128-cbc",
  73                 .algo = RTE_CRYPTO_CIPHER_AES_CBC,
  74                 .iv_len = 16,
  75                 .block_size = 16,
  76                 .key_len = 16
  77         },
  78         {
  79                 .keyword = "aes-256-cbc",
  80                 .algo = RTE_CRYPTO_CIPHER_AES_CBC,
  81                 .iv_len = 16,
  82                 .block_size = 16,
  83                 .key_len = 32
  84         },
  85         {
  86                 .keyword = "aes-128-ctr",
  87                 .algo = RTE_CRYPTO_CIPHER_AES_CTR,
  88                 .iv_len = 8,
  89                 .block_size = 4,
  90                 .key_len = 20
  91         },
  92         {
  93                 .keyword = "3des-cbc",
  94                 .algo = RTE_CRYPTO_CIPHER_3DES_CBC,
  95                 .iv_len = 8,
  96                 .block_size = 8,
  97                 .key_len = 24
  98         }
  99 };
 100
 101 const struct supported_auth_algo auth_algos[] = {
 102         {
 103                 .keyword = "null",
 104                 .algo = RTE_CRYPTO_AUTH_NULL,
 105                 .digest_len = 0,
 106                 .key_len = 0,
 107                 .key_not_req = 1
 108         },
 109         {
 110                 .keyword = "sha1-hmac",
 111                 .algo = RTE_CRYPTO_AUTH_SHA1_HMAC,
 112                 .digest_len = 12,
 113                 .key_len = 20
 114         },
 115         {
 116                 .keyword = "sha256-hmac",
 117                 .algo = RTE_CRYPTO_AUTH_SHA256_HMAC,
 118                 .digest_len = 16,
 119                 .key_len = 32
 120         }
 121 };
 122
 123 const struct supported_aead_algo aead_algos[] = {
 124         {
 125                 .keyword = "aes-128-gcm",
 126                 .algo = RTE_CRYPTO_AEAD_AES_GCM,
 127                 .iv_len = 8,
 128                 .block_size = 4,
 129                 .key_len = 20,
 130                 .digest_len = 16,
 131                 .aad_len = 8,
 132         }
 133 };
 134
 135 static struct ipsec_sa sa_out[IPSEC_SA_MAX_ENTRIES];
 136 static uint32_t nb_sa_out;
 137
 138 static struct ipsec_sa sa_in[IPSEC_SA_MAX_ENTRIES];
 139 static uint32_t nb_sa_in;
 140
 141 static const struct supported_cipher_algo *
 142 find_match_cipher_algo(const char *cipher_keyword)
 143 {
 144         size_t i;
 145
 146         for (i = 0; i < RTE_DIM(cipher_algos); i++) {
 147                 const struct supported_cipher_algo *algo =
 148                         &cipher_algos[i];
 149
 150                 if (strcmp(cipher_keyword, algo->keyword) == 0)
 151                         return algo;
 152         }
 153
 154         return NULL;
 155 }
 156
 157 static const struct supported_auth_algo *
 158 find_match_auth_algo(const char *auth_keyword)
 159 {
 160         size_t i;
 161
 162         for (i = 0; i < RTE_DIM(auth_algos); i++) {
 163                 const struct supported_auth_algo *algo =
 164                         &auth_algos[i];
 165
 166                 if (strcmp(auth_keyword, algo->keyword) == 0)
 167                         return algo;
 168         }
 169
 170         return NULL;
 171 }
 172
 173 static const struct supported_aead_algo *
 174 find_match_aead_algo(const char *aead_keyword)
 175 {
 176         size_t i;
 177
 178         for (i = 0; i < RTE_DIM(aead_algos); i++) {
 179                 const struct supported_aead_algo *algo =
 180                         &aead_algos[i];
 181
 182                 if (strcmp(aead_keyword, algo->keyword) == 0)
 183                         return algo;
 184         }
 185
 186         return NULL;
 187 }
 188
 189 /** parse_key_string
 190  *  parse x:x:x:x.... hex number key string into uint8_t *key
 191  *  return:
 192  *  > 0: number of bytes parsed
 193  *  0:   failed
 194  */
 195 static uint32_t
 196 parse_key_string(const char *key_str, uint8_t *key)
 197 {
 198         const char *pt_start = key_str, *pt_end = key_str;
 199         uint32_t nb_bytes = 0;
 200
 201         while (pt_end != NULL) {
 202                 char sub_str[3] = {0};
 203
 204                 pt_end = strchr(pt_start, ':');
 205
 206                 if (pt_end == NULL) {
 207                         if (strlen(pt_start) > 2)
 208                                 return 0;
 209                         strncpy(sub_str, pt_start, 2);
 210                 } else {
 211                         if (pt_end - pt_start > 2)
 212                                 return 0;
 213
 214                         strncpy(sub_str, pt_start, pt_end - pt_start);
 215                         pt_start = pt_end + 1;
 216                 }
 217
 218                 key[nb_bytes++] = strtol(sub_str, NULL, 16);
 219         }
 220
 221         return nb_bytes;
 222 }
 223
 224 void
 225 parse_sa_tokens(char **tokens, uint32_t n_tokens,
 226         struct parse_status *status)
 227 {
 228         struct ipsec_sa *rule = NULL;
 229         struct rte_ipsec_session *ips;
 230         uint32_t ti; /*token index*/
 231         uint32_t *ri /*rule index*/;
 232         uint32_t cipher_algo_p = 0;
 233         uint32_t auth_algo_p = 0;
 234         uint32_t aead_algo_p = 0;
 235         uint32_t src_p = 0;
 236         uint32_t dst_p = 0;
 237         uint32_t mode_p = 0;
 238         uint32_t type_p = 0;
 239         uint32_t portid_p = 0;
 240         uint32_t fallback_p = 0;
 241
 242         if (strcmp(tokens[0], "in") == 0) {
 243                 ri = &nb_sa_in;
 244
 245                 APP_CHECK(*ri <= IPSEC_SA_MAX_ENTRIES - 1, status,
 246                         "too many sa rules, abort insertion\n");
 247                 if (status->status < 0)
 248                         return;
 249
 250                 rule = &sa_in[*ri];
 251                 rule->direction = RTE_SECURITY_IPSEC_SA_DIR_INGRESS;
 252         } else {
 253                 ri = &nb_sa_out;
 254
 255                 APP_CHECK(*ri <= IPSEC_SA_MAX_ENTRIES - 1, status,
 256                         "too many sa rules, abort insertion\n");
 257                 if (status->status < 0)
 258                         return;
 259
 260                 rule = &sa_out[*ri];
 261                 rule->direction = RTE_SECURITY_IPSEC_SA_DIR_EGRESS;
 262         }
 263
 264         /* spi number */
 265         APP_CHECK_TOKEN_IS_NUM(tokens, 1, status);
 266         if (status->status < 0)
 267                 return;
 268         if (atoi(tokens[1]) == INVALID_SPI)
 269                 return;
 270         rule->spi = atoi(tokens[1]);
 271         ips = ipsec_get_primary_session(rule);
 272
 273         for (ti = 2; ti < n_tokens; ti++) {
 274                 if (strcmp(tokens[ti], "mode") == 0) {
 275                         APP_CHECK_PRESENCE(mode_p, tokens[ti], status);
 276                         if (status->status < 0)
 277                                 return;
 278
 279                         INCREMENT_TOKEN_INDEX(ti, n_tokens, status);
 280                         if (status->status < 0)
 281                                 return;
 282
 283                         if (strcmp(tokens[ti], "ipv4-tunnel") == 0)
 284                                 rule->flags = IP4_TUNNEL;
 285                         else if (strcmp(tokens[ti], "ipv6-tunnel") == 0)
 286                                 rule->flags = IP6_TUNNEL;
 287                         else if (strcmp(tokens[ti], "transport") == 0)
 288                                 rule->flags = TRANSPORT;
 289                         else {
 290                                 APP_CHECK(0, status, "unrecognized "
 291                                         "input \"%s\"", tokens[ti]);
 292                                 return;
 293                         }
 294
 295                         mode_p = 1;
 296                         continue;
 297                 }
 298
 299                 if (strcmp(tokens[ti], "cipher_algo") == 0) {
 300                         const struct supported_cipher_algo *algo;
 301                         uint32_t key_len;
 302
 303                         APP_CHECK_PRESENCE(cipher_algo_p, tokens[ti],
 304                                 status);
 305                         if (status->status < 0)
 306                                 return;
 307
 308                         INCREMENT_TOKEN_INDEX(ti, n_tokens, status);
 309                         if (status->status < 0)
 310                                 return;
 311
 312                         algo = find_match_cipher_algo(tokens[ti]);
 313
 314                         APP_CHECK(algo != NULL, status, "unrecognized "
 315                                 "input \"%s\"", tokens[ti]);
 316
 317                         rule->cipher_algo = algo->algo;
 318                         rule->block_size = algo->block_size;
 319                         rule->iv_len = algo->iv_len;
 320                         rule->cipher_key_len = algo->key_len;
 321
 322                         /* for NULL algorithm, no cipher key required */
 323                         if (rule->cipher_algo == RTE_CRYPTO_CIPHER_NULL) {
 324                                 cipher_algo_p = 1;
 325                                 continue;
 326                         }
 327
 328                         INCREMENT_TOKEN_INDEX(ti, n_tokens, status);
 329                         if (status->status < 0)
 330                                 return;
 331
 332                         APP_CHECK(strcmp(tokens[ti], "cipher_key") == 0,
 333                                 status, "unrecognized input \"%s\", "
 334                                 "expect \"cipher_key\"", tokens[ti]);
 335                         if (status->status < 0)
 336                                 return;
 337
 338                         INCREMENT_TOKEN_INDEX(ti, n_tokens, status);
 339                         if (status->status < 0)
 340                                 return;
 341
 342                         key_len = parse_key_string(tokens[ti],
 343                                 rule->cipher_key);
 344                         APP_CHECK(key_len == rule->cipher_key_len, status,
 345                                 "unrecognized input \"%s\"", tokens[ti]);
 346                         if (status->status < 0)
 347                                 return;
 348
 349                         if (algo->algo == RTE_CRYPTO_CIPHER_AES_CBC ||
 350                                 algo->algo == RTE_CRYPTO_CIPHER_3DES_CBC)
 351                                 rule->salt = (uint32_t)rte_rand();
 352
 353                         if (algo->algo == RTE_CRYPTO_CIPHER_AES_CTR) {
 354                                 key_len -= 4;
 355                                 rule->cipher_key_len = key_len;
 356                                 memcpy(&rule->salt,
 357                                         &rule->cipher_key[key_len], 4);
 358                         }
 359
 360                         cipher_algo_p = 1;
 361                         continue;
 362                 }
 363
 364                 if (strcmp(tokens[ti], "auth_algo") == 0) {
 365                         const struct supported_auth_algo *algo;
 366                         uint32_t key_len;
 367
 368                         APP_CHECK_PRESENCE(auth_algo_p, tokens[ti],
 369                                 status);
 370                         if (status->status < 0)
 371                                 return;
 372
 373                         INCREMENT_TOKEN_INDEX(ti, n_tokens, status);
 374                         if (status->status < 0)
 375                                 return;
 376
 377                         algo = find_match_auth_algo(tokens[ti]);
 378                         APP_CHECK(algo != NULL, status, "unrecognized "
 379                                 "input \"%s\"", tokens[ti]);
 380
 381                         rule->auth_algo = algo->algo;
 382                         rule->auth_key_len = algo->key_len;
 383                         rule->digest_len = algo->digest_len;
 384
 385                         /* NULL algorithm and combined algos do not
 386                          * require auth key
 387                          */
 388                         if (algo->key_not_req) {
 389                                 auth_algo_p = 1;
 390                                 continue;
 391                         }
 392
 393                         INCREMENT_TOKEN_INDEX(ti, n_tokens, status);
 394                         if (status->status < 0)
 395                                 return;
 396
 397                         APP_CHECK(strcmp(tokens[ti], "auth_key") == 0,
 398                                 status, "unrecognized input \"%s\", "
 399                                 "expect \"auth_key\"", tokens[ti]);
 400                         if (status->status < 0)
 401                                 return;
 402
 403                         INCREMENT_TOKEN_INDEX(ti, n_tokens, status);
 404                         if (status->status < 0)
 405                                 return;
 406
 407                         key_len = parse_key_string(tokens[ti],
 408                                 rule->auth_key);
 409                         APP_CHECK(key_len == rule->auth_key_len, status,
 410                                 "unrecognized input \"%s\"", tokens[ti]);
 411                         if (status->status < 0)
 412                                 return;
 413
 414                         auth_algo_p = 1;
 415                         continue;
 416                 }
 417
 418                 if (strcmp(tokens[ti], "aead_algo") == 0) {
 419                         const struct supported_aead_algo *algo;
 420                         uint32_t key_len;
 421
 422                         APP_CHECK_PRESENCE(aead_algo_p, tokens[ti],
 423                                 status);
 424                         if (status->status < 0)
 425                                 return;
 426
 427                         INCREMENT_TOKEN_INDEX(ti, n_tokens, status);
 428                         if (status->status < 0)
 429                                 return;
 430
 431                         algo = find_match_aead_algo(tokens[ti]);
 432
 433                         APP_CHECK(algo != NULL, status, "unrecognized "
 434                                 "input \"%s\"", tokens[ti]);
 435
 436                         rule->aead_algo = algo->algo;
 437                         rule->cipher_key_len = algo->key_len;
 438                         rule->digest_len = algo->digest_len;
 439                         rule->aad_len = algo->aad_len;
 440                         rule->block_size = algo->block_size;
 441                         rule->iv_len = algo->iv_len;
 442
 443                         INCREMENT_TOKEN_INDEX(ti, n_tokens, status);
 444                         if (status->status < 0)
 445                                 return;
 446
 447                         APP_CHECK(strcmp(tokens[ti], "aead_key") == 0,
 448                                 status, "unrecognized input \"%s\", "
 449                                 "expect \"aead_key\"", tokens[ti]);
 450                         if (status->status < 0)
 451                                 return;
 452
 453                         INCREMENT_TOKEN_INDEX(ti, n_tokens, status);
 454                         if (status->status < 0)
 455                                 return;
 456
 457                         key_len = parse_key_string(tokens[ti],
 458                                 rule->cipher_key);
 459                         APP_CHECK(key_len == rule->cipher_key_len, status,
 460                                 "unrecognized input \"%s\"", tokens[ti]);
 461                         if (status->status < 0)
 462                                 return;
 463
 464                         key_len -= 4;
 465                         rule->cipher_key_len = key_len;
 466                         memcpy(&rule->salt,
 467                                 &rule->cipher_key[key_len], 4);
 468
 469                         aead_algo_p = 1;
 470                         continue;
 471                 }
 472
 473                 if (strcmp(tokens[ti], "src") == 0) {
 474                         APP_CHECK_PRESENCE(src_p, tokens[ti], status);
 475                         if (status->status < 0)
 476                                 return;
 477
 478                         INCREMENT_TOKEN_INDEX(ti, n_tokens, status);
 479                         if (status->status < 0)
 480                                 return;
 481
 482                         if (IS_IP4_TUNNEL(rule->flags)) {
 483                                 struct in_addr ip;
 484
 485                                 APP_CHECK(parse_ipv4_addr(tokens[ti],
 486                                         &ip, NULL) == 0, status,
 487                                         "unrecognized input \"%s\", "
 488                                         "expect valid ipv4 addr",
 489                                         tokens[ti]);
 490                                 if (status->status < 0)
 491                                         return;
 492                                 rule->src.ip.ip4 = rte_bswap32(
 493                                         (uint32_t)ip.s_addr);
 494                         } else if (IS_IP6_TUNNEL(rule->flags)) {
 495                                 struct in6_addr ip;
 496
 497                                 APP_CHECK(parse_ipv6_addr(tokens[ti], &ip,
 498                                         NULL) == 0, status,
 499                                         "unrecognized input \"%s\", "
 500                                         "expect valid ipv6 addr",
 501                                         tokens[ti]);
 502                                 if (status->status < 0)
 503                                         return;
 504                                 memcpy(rule->src.ip.ip6.ip6_b,
 505                                         ip.s6_addr, 16);
 506                         } else if (IS_TRANSPORT(rule->flags)) {
 507                                 APP_CHECK(0, status, "unrecognized input "
 508                                         "\"%s\"", tokens[ti]);
 509                                 return;
 510                         }
 511
 512                         src_p = 1;
 513                         continue;
 514                 }
 515
 516                 if (strcmp(tokens[ti], "dst") == 0) {
 517                         APP_CHECK_PRESENCE(dst_p, tokens[ti], status);
 518                         if (status->status < 0)
 519                                 return;
 520
 521                         INCREMENT_TOKEN_INDEX(ti, n_tokens, status);
 522                         if (status->status < 0)
 523                                 return;
 524
 525                         if (IS_IP4_TUNNEL(rule->flags)) {
 526                                 struct in_addr ip;
 527
 528                                 APP_CHECK(parse_ipv4_addr(tokens[ti],
 529                                         &ip, NULL) == 0, status,
 530                                         "unrecognized input \"%s\", "
 531                                         "expect valid ipv4 addr",
 532                                         tokens[ti]);
 533                                 if (status->status < 0)
 534                                         return;
 535                                 rule->dst.ip.ip4 = rte_bswap32(
 536                                         (uint32_t)ip.s_addr);
 537                         } else if (IS_IP6_TUNNEL(rule->flags)) {
 538                                 struct in6_addr ip;
 539
 540                                 APP_CHECK(parse_ipv6_addr(tokens[ti], &ip,
 541                                         NULL) == 0, status,
 542                                         "unrecognized input \"%s\", "
 543                                         "expect valid ipv6 addr",
 544                                         tokens[ti]);
 545                                 if (status->status < 0)
 546                                         return;
 547                                 memcpy(rule->dst.ip.ip6.ip6_b, ip.s6_addr, 16);
 548                         } else if (IS_TRANSPORT(rule->flags)) {
 549                                 APP_CHECK(0, status, "unrecognized "
 550                                         "input \"%s\"", tokens[ti]);
 551                                 return;
 552                         }
 553
 554                         dst_p = 1;
 555                         continue;
 556                 }
 557
 558                 if (strcmp(tokens[ti], "type") == 0) {
 559                         APP_CHECK_PRESENCE(type_p, tokens[ti], status);
 560                         if (status->status < 0)
 561                                 return;
 562
 563                         INCREMENT_TOKEN_INDEX(ti, n_tokens, status);
 564                         if (status->status < 0)
 565                                 return;
 566
 567                         if (strcmp(tokens[ti], "inline-crypto-offload") == 0)
 568                                 ips->type =
 569                                         RTE_SECURITY_ACTION_TYPE_INLINE_CRYPTO;
 570                         else if (strcmp(tokens[ti],
 571                                         "inline-protocol-offload") == 0)
 572                                 ips->type =
 573                                 RTE_SECURITY_ACTION_TYPE_INLINE_PROTOCOL;
 574                         else if (strcmp(tokens[ti],
 575                                         "lookaside-protocol-offload") == 0)
 576                                 ips->type =
 577                                 RTE_SECURITY_ACTION_TYPE_LOOKASIDE_PROTOCOL;
 578                         else if (strcmp(tokens[ti], "no-offload") == 0)
 579                                 ips->type = RTE_SECURITY_ACTION_TYPE_NONE;
 580                         else {
 581                                 APP_CHECK(0, status, "Invalid input \"%s\"",
 582                                                 tokens[ti]);
 583                                 return;
 584                         }
 585
 586                         type_p = 1;
 587                         continue;
 588                 }
 589
 590                 if (strcmp(tokens[ti], "port_id") == 0) {
 591                         APP_CHECK_PRESENCE(portid_p, tokens[ti], status);
 592                         if (status->status < 0)
 593                                 return;
 594                         INCREMENT_TOKEN_INDEX(ti, n_tokens, status);
 595                         if (status->status < 0)
 596                                 return;
 597                         rule->portid = atoi(tokens[ti]);
 598                         if (status->status < 0)
 599                                 return;
 600                         portid_p = 1;
 601                         continue;
 602                 }
 603
 604                 if (strcmp(tokens[ti], "fallback") == 0) {
 605                         struct rte_ipsec_session *fb;
 606
 607                         APP_CHECK(app_sa_prm.enable, status, "Fallback session "
 608                                 "not allowed for legacy mode.");
 609                         if (status->status < 0)
 610                                 return;
 611                         APP_CHECK(ips->type ==
 612                                 RTE_SECURITY_ACTION_TYPE_INLINE_CRYPTO, status,
 613                                 "Fallback session allowed if primary session "
 614                                 "is of type inline-crypto-offload only.");
 615                         if (status->status < 0)
 616                                 return;
 617                         APP_CHECK(rule->direction ==
 618                                 RTE_SECURITY_IPSEC_SA_DIR_INGRESS, status,
 619                                 "Fallback session not allowed for egress "
 620                                 "rule");
 621                         if (status->status < 0)
 622                                 return;
 623                         APP_CHECK_PRESENCE(fallback_p, tokens[ti], status);
 624                         if (status->status < 0)
 625                                 return;
 626                         INCREMENT_TOKEN_INDEX(ti, n_tokens, status);
 627                         if (status->status < 0)
 628                                 return;
 629                         fb = ipsec_get_fallback_session(rule);
 630                         if (strcmp(tokens[ti], "lookaside-none") == 0) {
 631                                 fb->type = RTE_SECURITY_ACTION_TYPE_NONE;
 632                         } else {
 633                                 APP_CHECK(0, status, "unrecognized fallback "
 634                                         "type %s.", tokens[ti]);
 635                                 return;
 636                         }
 637
 638                         rule->fallback_sessions = 1;
 639                         fallback_p = 1;
 640                         continue;
 641                 }
 642
 643                 /* unrecognizeable input */
 644                 APP_CHECK(0, status, "unrecognized input \"%s\"",
 645                         tokens[ti]);
 646                 return;
 647         }
 648
 649         if (aead_algo_p) {
 650                 APP_CHECK(cipher_algo_p == 0, status,
 651                                 "AEAD used, no need for cipher options");
 652                 if (status->status < 0)
 653                         return;
 654
 655                 APP_CHECK(auth_algo_p == 0, status,
 656                                 "AEAD used, no need for auth options");
 657                 if (status->status < 0)
 658                         return;
 659         } else {
 660                 APP_CHECK(cipher_algo_p == 1, status, "missing cipher or AEAD options");
 661                 if (status->status < 0)
 662                         return;
 663
 664                 APP_CHECK(auth_algo_p == 1, status, "missing auth or AEAD options");
 665                 if (status->status < 0)
 666                         return;
 667         }
 668
 669         APP_CHECK(mode_p == 1, status, "missing mode option");
 670         if (status->status < 0)
 671                 return;
 672
 673         if ((ips->type != RTE_SECURITY_ACTION_TYPE_NONE) && (portid_p == 0))
 674                 printf("Missing portid option, falling back to non-offload\n");
 675
 676         if (!type_p || !portid_p) {
 677                 ips->type = RTE_SECURITY_ACTION_TYPE_NONE;
 678                 rule->portid = -1;
 679         }
 680
 681         *ri = *ri + 1;
 682 }
 683
 684 static void
 685 print_one_sa_rule(const struct ipsec_sa *sa, int inbound)
 686 {
 687         uint32_t i;
 688         uint8_t a, b, c, d;
 689         const struct rte_ipsec_session *ips;
 690         const struct rte_ipsec_session *fallback_ips;
 691
 692         printf("\tspi_%s(%3u):", inbound?"in":"out", sa->spi);
 693
 694         for (i = 0; i < RTE_DIM(cipher_algos); i++) {
 695                 if (cipher_algos[i].algo == sa->cipher_algo &&
 696                                 cipher_algos[i].key_len == sa->cipher_key_len) {
 697                         printf("%s ", cipher_algos[i].keyword);
 698                         break;
 699                 }
 700         }
 701
 702         for (i = 0; i < RTE_DIM(auth_algos); i++) {
 703                 if (auth_algos[i].algo == sa->auth_algo) {
 704                         printf("%s ", auth_algos[i].keyword);
 705                         break;
 706                 }
 707         }
 708
 709         for (i = 0; i < RTE_DIM(aead_algos); i++) {
 710                 if (aead_algos[i].algo == sa->aead_algo) {
 711                         printf("%s ", aead_algos[i].keyword);
 712                         break;
 713                 }
 714         }
 715
 716         printf("mode:");
 717
 718         switch (WITHOUT_TRANSPORT_VERSION(sa->flags)) {
 719         case IP4_TUNNEL:
 720                 printf("IP4Tunnel ");
 721                 uint32_t_to_char(sa->src.ip.ip4, &a, &b, &c, &d);
 722                 printf("%hhu.%hhu.%hhu.%hhu ", d, c, b, a);
 723                 uint32_t_to_char(sa->dst.ip.ip4, &a, &b, &c, &d);
 724                 printf("%hhu.%hhu.%hhu.%hhu", d, c, b, a);
 725                 break;
 726         case IP6_TUNNEL:
 727                 printf("IP6Tunnel ");
 728                 for (i = 0; i < 16; i++) {
 729                         if (i % 2 && i != 15)
 730                                 printf("%.2x:", sa->src.ip.ip6.ip6_b[i]);
 731                         else
 732                                 printf("%.2x", sa->src.ip.ip6.ip6_b[i]);
 733                 }
 734                 printf(" ");
 735                 for (i = 0; i < 16; i++) {
 736                         if (i % 2 && i != 15)
 737                                 printf("%.2x:", sa->dst.ip.ip6.ip6_b[i]);
 738                         else
 739                                 printf("%.2x", sa->dst.ip.ip6.ip6_b[i]);
 740                 }
 741                 break;
 742         case TRANSPORT:
 743                 printf("Transport ");
 744                 break;
 745         }
 746
 747         ips = &sa->sessions[IPSEC_SESSION_PRIMARY];
 748         printf(" type:");
 749         switch (ips->type) {
 750         case RTE_SECURITY_ACTION_TYPE_NONE:
 751                 printf("no-offload ");
 752                 break;
 753         case RTE_SECURITY_ACTION_TYPE_INLINE_CRYPTO:
 754                 printf("inline-crypto-offload ");
 755                 break;
 756         case RTE_SECURITY_ACTION_TYPE_INLINE_PROTOCOL:
 757                 printf("inline-protocol-offload ");
 758                 break;
 759         case RTE_SECURITY_ACTION_TYPE_LOOKASIDE_PROTOCOL:
 760                 printf("lookaside-protocol-offload ");
 761                 break;
 762         }
 763
 764         fallback_ips = &sa->sessions[IPSEC_SESSION_FALLBACK];
 765         if (fallback_ips != NULL && sa->fallback_sessions > 0) {
 766                 printf("inline fallback: ");
 767                 if (fallback_ips->type == RTE_SECURITY_ACTION_TYPE_NONE)
 768                         printf("lookaside-none");
 769                 else
 770                         printf("invalid");
 771         }
 772         printf("\n");
 773 }
 774
 775 struct sa_ctx {
 776         void *satbl; /* pointer to array of rte_ipsec_sa objects*/
 777         struct ipsec_sa sa[IPSEC_SA_MAX_ENTRIES];
 778         union {
 779                 struct {
 780                         struct rte_crypto_sym_xform a;
 781                         struct rte_crypto_sym_xform b;
 782                 };
 783         } xf[IPSEC_SA_MAX_ENTRIES];
 784 };
 785
 786 static struct sa_ctx *
 787 sa_create(const char *name, int32_t socket_id)
 788 {
 789         char s[PATH_MAX];
 790         struct sa_ctx *sa_ctx;
 791         uint32_t mz_size;
 792         const struct rte_memzone *mz;
 793
 794         snprintf(s, sizeof(s), "%s_%u", name, socket_id);
 795
 796         /* Create SA array table */
 797         printf("Creating SA context with %u maximum entries on socket %d\n",
 798                         IPSEC_SA_MAX_ENTRIES, socket_id);
 799
 800         mz_size = sizeof(struct sa_ctx);
 801         mz = rte_memzone_reserve(s, mz_size, socket_id,
 802                         RTE_MEMZONE_1GB | RTE_MEMZONE_SIZE_HINT_ONLY);
 803         if (mz == NULL) {
 804                 printf("Failed to allocate SA DB memory\n");
 805                 rte_errno = ENOMEM;
 806                 return NULL;
 807         }
 808
 809         sa_ctx = (struct sa_ctx *)mz->addr;
 810
 811         return sa_ctx;
 812 }
 813
 814 static int
 815 check_eth_dev_caps(uint16_t portid, uint32_t inbound)
 816 {
 817         struct rte_eth_dev_info dev_info;
 818         int retval;
 819
 820         retval = rte_eth_dev_info_get(portid, &dev_info);
 821         if (retval != 0) {
 822                 RTE_LOG(ERR, IPSEC,
 823                         "Error during getting device (port %u) info: %s\n",
 824                         portid, strerror(-retval));
 825
 826                 return retval;
 827         }
 828
 829         if (inbound) {
 830                 if ((dev_info.rx_offload_capa &
 831                                 DEV_RX_OFFLOAD_SECURITY) == 0) {
 832                         RTE_LOG(WARNING, PORT,
 833                                 "hardware RX IPSec offload is not supported\n");
 834                         return -EINVAL;
 835                 }
 836
 837         } else { /* outbound */
 838                 if ((dev_info.tx_offload_capa &
 839                                 DEV_TX_OFFLOAD_SECURITY) == 0) {
 840                         RTE_LOG(WARNING, PORT,
 841                                 "hardware TX IPSec offload is not supported\n");
 842                         return -EINVAL;
 843                 }
 844         }
 845         return 0;
 846 }
 847
 848 /*
 849  * Helper function, tries to determine next_proto for SPI
 850  * by searching though SP rules.
 851  */
 852 static int
 853 get_spi_proto(uint32_t spi, enum rte_security_ipsec_sa_direction dir,
 854                 struct ip_addr ip_addr[2], uint32_t mask[2])
 855 {
 856         int32_t rc4, rc6;
 857
 858         rc4 = sp4_spi_present(spi, dir == RTE_SECURITY_IPSEC_SA_DIR_INGRESS,
 859                                 ip_addr, mask);
 860         rc6 = sp6_spi_present(spi, dir == RTE_SECURITY_IPSEC_SA_DIR_INGRESS,
 861                                 ip_addr, mask);
 862
 863         if (rc4 >= 0) {
 864                 if (rc6 >= 0) {
 865                         RTE_LOG(ERR, IPSEC,
 866                                 "%s: SPI %u used simultaeously by "
 867                                 "IPv4(%d) and IPv6 (%d) SP rules\n",
 868                                 __func__, spi, rc4, rc6);
 869                         return -EINVAL;
 870                 } else
 871                         return IPPROTO_IPIP;
 872         } else if (rc6 < 0) {
 873                 RTE_LOG(ERR, IPSEC,
 874                         "%s: SPI %u is not used by any SP rule\n",
 875                         __func__, spi);
 876                 return -EINVAL;
 877         } else
 878                 return IPPROTO_IPV6;
 879 }
 880
 881 /*
 882  * Helper function for getting source and destination IP addresses
 883  * from SP. Needed for inline crypto transport mode, as addresses are not
 884  * provided in config file for that mode. It checks if SP for current SA exists,
 885  * and based on what type of protocol is returned, it stores appropriate
 886  * addresses got from SP into SA.
 887  */
 888 static int
 889 sa_add_address_inline_crypto(struct ipsec_sa *sa)
 890 {
 891         int protocol;
 892         struct ip_addr ip_addr[2];
 893         uint32_t mask[2];
 894
 895         protocol = get_spi_proto(sa->spi, sa->direction, ip_addr, mask);
 896         if (protocol < 0)
 897                 return protocol;
 898         else if (protocol == IPPROTO_IPIP) {
 899                 sa->flags |= IP4_TRANSPORT;
 900                 if (mask[0] == IP4_FULL_MASK &&
 901                                 mask[1] == IP4_FULL_MASK &&
 902                                 ip_addr[0].ip.ip4 != 0 &&
 903                                 ip_addr[1].ip.ip4 != 0) {
 904
 905                         sa->src.ip.ip4 = ip_addr[0].ip.ip4;
 906                         sa->dst.ip.ip4 = ip_addr[1].ip.ip4;
 907                 } else {
 908                         RTE_LOG(ERR, IPSEC,
 909                         "%s: No valid address or mask entry in"
 910                         " IPv4 SP rule for SPI %u\n",
 911                         __func__, sa->spi);
 912                         return -EINVAL;
 913                 }
 914         } else if (protocol == IPPROTO_IPV6) {
 915                 sa->flags |= IP6_TRANSPORT;
 916                 if (mask[0] == IP6_FULL_MASK &&
 917                                 mask[1] == IP6_FULL_MASK &&
 918                                 (ip_addr[0].ip.ip6.ip6[0] != 0 ||
 919                                 ip_addr[0].ip.ip6.ip6[1] != 0) &&
 920                                 (ip_addr[1].ip.ip6.ip6[0] != 0 ||
 921                                 ip_addr[1].ip.ip6.ip6[1] != 0)) {
 922
 923                         sa->src.ip.ip6 = ip_addr[0].ip.ip6;
 924                         sa->dst.ip.ip6 = ip_addr[1].ip.ip6;
 925                 } else {
 926                         RTE_LOG(ERR, IPSEC,
 927                         "%s: No valid address or mask entry in"
 928                         " IPv6 SP rule for SPI %u\n",
 929                         __func__, sa->spi);
 930                         return -EINVAL;
 931                 }
 932         }
 933         return 0;
 934 }
 935
 936 static int
 937 sa_add_rules(struct sa_ctx *sa_ctx, const struct ipsec_sa entries[],
 938                 uint32_t nb_entries, uint32_t inbound,
 939                 struct socket_ctx *skt_ctx)
 940 {
 941         struct ipsec_sa *sa;
 942         uint32_t i, idx;
 943         uint16_t iv_length, aad_length;
 944         int inline_status;
 945         int32_t rc;
 946         struct rte_ipsec_session *ips;
 947
 948         /* for ESN upper 32 bits of SQN also need to be part of AAD */
 949         aad_length = (app_sa_prm.enable_esn != 0) ? sizeof(uint32_t) : 0;
 950
 951         for (i = 0; i < nb_entries; i++) {
 952                 idx = SPI2IDX(entries[i].spi);
 953                 sa = &sa_ctx->sa[idx];
 954                 if (sa->spi != 0) {
 955                         printf("Index %u already in use by SPI %u\n",
 956                                         idx, sa->spi);
 957                         return -EINVAL;
 958                 }
 959                 *sa = entries[i];
 960                 sa->seq = 0;
 961                 ips = ipsec_get_primary_session(sa);
 962
 963                 if (ips->type == RTE_SECURITY_ACTION_TYPE_INLINE_PROTOCOL ||
 964                         ips->type == RTE_SECURITY_ACTION_TYPE_INLINE_CRYPTO) {
 965                         if (check_eth_dev_caps(sa->portid, inbound))
 966                                 return -EINVAL;
 967                 }
 968
 969
 970                 switch (WITHOUT_TRANSPORT_VERSION(sa->flags)) {
 971                 case IP4_TUNNEL:
 972                         sa->src.ip.ip4 = rte_cpu_to_be_32(sa->src.ip.ip4);
 973                         sa->dst.ip.ip4 = rte_cpu_to_be_32(sa->dst.ip.ip4);
 974                         break;
 975                 case TRANSPORT:
 976                         if (ips->type ==
 977                                 RTE_SECURITY_ACTION_TYPE_INLINE_CRYPTO) {
 978                                 inline_status =
 979                                         sa_add_address_inline_crypto(sa);
 980                                 if (inline_status < 0)
 981                                         return inline_status;
 982                         }
 983                         break;
 984                 }
 985
 986                 if (sa->aead_algo == RTE_CRYPTO_AEAD_AES_GCM) {
 987                         struct rte_ipsec_session *ips;
 988                         iv_length = 12;
 989
 990                         sa_ctx->xf[idx].a.type = RTE_CRYPTO_SYM_XFORM_AEAD;
 991                         sa_ctx->xf[idx].a.aead.algo = sa->aead_algo;
 992                         sa_ctx->xf[idx].a.aead.key.data = sa->cipher_key;
 993                         sa_ctx->xf[idx].a.aead.key.length =
 994                                 sa->cipher_key_len;
 995                         sa_ctx->xf[idx].a.aead.op = (inbound == 1) ?
 996                                 RTE_CRYPTO_AEAD_OP_DECRYPT :
 997                                 RTE_CRYPTO_AEAD_OP_ENCRYPT;
 998                         sa_ctx->xf[idx].a.next = NULL;
 999                         sa_ctx->xf[idx].a.aead.iv.offset = IV_OFFSET;
1000                         sa_ctx->xf[idx].a.aead.iv.length = iv_length;
1001                         sa_ctx->xf[idx].a.aead.aad_length =
1002                                 sa->aad_len + aad_length;
1003                         sa_ctx->xf[idx].a.aead.digest_length =
1004                                 sa->digest_len;
1005
1006                         sa->xforms = &sa_ctx->xf[idx].a;
1007
1008                         ips = ipsec_get_primary_session(sa);
1009                         if (ips->type ==
1010                                 RTE_SECURITY_ACTION_TYPE_INLINE_PROTOCOL ||
1011                                 ips->type ==
1012                                 RTE_SECURITY_ACTION_TYPE_INLINE_CRYPTO) {
1013                                 rc = create_inline_session(skt_ctx, sa, ips);
1014                                 if (rc != 0) {
1015                                         RTE_LOG(ERR, IPSEC_ESP,
1016                                                 "create_inline_session() failed\n");
1017                                         return -EINVAL;
1018                                 }
1019                         }
1020                         print_one_sa_rule(sa, inbound);
1021                 } else {
1022                         switch (sa->cipher_algo) {
1023                         case RTE_CRYPTO_CIPHER_NULL:
1024                         case RTE_CRYPTO_CIPHER_3DES_CBC:
1025                         case RTE_CRYPTO_CIPHER_AES_CBC:
1026                                 iv_length = sa->iv_len;
1027                                 break;
1028                         case RTE_CRYPTO_CIPHER_AES_CTR:
1029                                 iv_length = 16;
1030                                 break;
1031                         default:
1032                                 RTE_LOG(ERR, IPSEC_ESP,
1033                                                 "unsupported cipher algorithm %u\n",
1034                                                 sa->cipher_algo);
1035                                 return -EINVAL;
1036                         }
1037
1038                         if (inbound) {
1039                                 sa_ctx->xf[idx].b.type = RTE_CRYPTO_SYM_XFORM_CIPHER;
1040                                 sa_ctx->xf[idx].b.cipher.algo = sa->cipher_algo;
1041                                 sa_ctx->xf[idx].b.cipher.key.data = sa->cipher_key;
1042                                 sa_ctx->xf[idx].b.cipher.key.length =
1043                                         sa->cipher_key_len;
1044                                 sa_ctx->xf[idx].b.cipher.op =
1045                                         RTE_CRYPTO_CIPHER_OP_DECRYPT;
1046                                 sa_ctx->xf[idx].b.next = NULL;
1047                                 sa_ctx->xf[idx].b.cipher.iv.offset = IV_OFFSET;
1048                                 sa_ctx->xf[idx].b.cipher.iv.length = iv_length;
1049
1050                                 sa_ctx->xf[idx].a.type = RTE_CRYPTO_SYM_XFORM_AUTH;
1051                                 sa_ctx->xf[idx].a.auth.algo = sa->auth_algo;
1052                                 sa_ctx->xf[idx].a.auth.key.data = sa->auth_key;
1053                                 sa_ctx->xf[idx].a.auth.key.length =
1054                                         sa->auth_key_len;
1055                                 sa_ctx->xf[idx].a.auth.digest_length =
1056                                         sa->digest_len;
1057                                 sa_ctx->xf[idx].a.auth.op =
1058                                         RTE_CRYPTO_AUTH_OP_VERIFY;
1059                         } else { /* outbound */
1060                                 sa_ctx->xf[idx].a.type = RTE_CRYPTO_SYM_XFORM_CIPHER;
1061                                 sa_ctx->xf[idx].a.cipher.algo = sa->cipher_algo;
1062                                 sa_ctx->xf[idx].a.cipher.key.data = sa->cipher_key;
1063                                 sa_ctx->xf[idx].a.cipher.key.length =
1064                                         sa->cipher_key_len;
1065                                 sa_ctx->xf[idx].a.cipher.op =
1066                                         RTE_CRYPTO_CIPHER_OP_ENCRYPT;
1067                                 sa_ctx->xf[idx].a.next = NULL;
1068                                 sa_ctx->xf[idx].a.cipher.iv.offset = IV_OFFSET;
1069                                 sa_ctx->xf[idx].a.cipher.iv.length = iv_length;
1070
1071                                 sa_ctx->xf[idx].b.type = RTE_CRYPTO_SYM_XFORM_AUTH;
1072                                 sa_ctx->xf[idx].b.auth.algo = sa->auth_algo;
1073                                 sa_ctx->xf[idx].b.auth.key.data = sa->auth_key;
1074                                 sa_ctx->xf[idx].b.auth.key.length =
1075                                         sa->auth_key_len;
1076                                 sa_ctx->xf[idx].b.auth.digest_length =
1077                                         sa->digest_len;
1078                                 sa_ctx->xf[idx].b.auth.op =
1079                                         RTE_CRYPTO_AUTH_OP_GENERATE;
1080                         }
1081
1082                         sa_ctx->xf[idx].a.next = &sa_ctx->xf[idx].b;
1083                         sa_ctx->xf[idx].b.next = NULL;
1084                         sa->xforms = &sa_ctx->xf[idx].a;
1085
1086                         print_one_sa_rule(sa, inbound);
1087                 }
1088         }
1089
1090         return 0;
1091 }
1092
1093 static inline int
1094 sa_out_add_rules(struct sa_ctx *sa_ctx, const struct ipsec_sa entries[],
1095                 uint32_t nb_entries, struct socket_ctx *skt_ctx)
1096 {
1097         return sa_add_rules(sa_ctx, entries, nb_entries, 0, skt_ctx);
1098 }
1099
1100 static inline int
1101 sa_in_add_rules(struct sa_ctx *sa_ctx, const struct ipsec_sa entries[],
1102                 uint32_t nb_entries, struct socket_ctx *skt_ctx)
1103 {
1104         return sa_add_rules(sa_ctx, entries, nb_entries, 1, skt_ctx);
1105 }
1106
1107 /*
1108  * helper function, fills parameters that are identical for all SAs
1109  */
1110 static void
1111 fill_ipsec_app_sa_prm(struct rte_ipsec_sa_prm *prm,
1112         const struct app_sa_prm *app_prm)
1113 {
1114         memset(prm, 0, sizeof(*prm));
1115
1116         prm->flags = app_prm->flags;
1117         prm->ipsec_xform.options.esn = app_prm->enable_esn;
1118         prm->replay_win_sz = app_prm->window_size;
1119 }
1120
1121 static int
1122 fill_ipsec_sa_prm(struct rte_ipsec_sa_prm *prm, const struct ipsec_sa *ss,
1123         const struct rte_ipv4_hdr *v4, struct rte_ipv6_hdr *v6)
1124 {
1125         int32_t rc;
1126
1127         /*
1128          * Try to get SPI next proto by searching that SPI in SPD.
1129          * probably not the optimal way, but there seems nothing
1130          * better right now.
1131          */
1132         rc = get_spi_proto(ss->spi, ss->direction, NULL, NULL);
1133         if (rc < 0)
1134                 return rc;
1135
1136         fill_ipsec_app_sa_prm(prm, &app_sa_prm);
1137         prm->userdata = (uintptr_t)ss;
1138
1139         /* setup ipsec xform */
1140         prm->ipsec_xform.spi = ss->spi;
1141         prm->ipsec_xform.salt = ss->salt;
1142         prm->ipsec_xform.direction = ss->direction;
1143         prm->ipsec_xform.proto = RTE_SECURITY_IPSEC_SA_PROTO_ESP;
1144         prm->ipsec_xform.mode = (IS_TRANSPORT(ss->flags)) ?
1145                 RTE_SECURITY_IPSEC_SA_MODE_TRANSPORT :
1146                 RTE_SECURITY_IPSEC_SA_MODE_TUNNEL;
1147         prm->ipsec_xform.options.ecn = 1;
1148         prm->ipsec_xform.options.copy_dscp = 1;
1149
1150         if (IS_IP4_TUNNEL(ss->flags)) {
1151                 prm->ipsec_xform.tunnel.type = RTE_SECURITY_IPSEC_TUNNEL_IPV4;
1152                 prm->tun.hdr_len = sizeof(*v4);
1153                 prm->tun.next_proto = rc;
1154                 prm->tun.hdr = v4;
1155         } else if (IS_IP6_TUNNEL(ss->flags)) {
1156                 prm->ipsec_xform.tunnel.type = RTE_SECURITY_IPSEC_TUNNEL_IPV6;
1157                 prm->tun.hdr_len = sizeof(*v6);
1158                 prm->tun.next_proto = rc;
1159                 prm->tun.hdr = v6;
1160         } else {
1161                 /* transport mode */
1162                 prm->trs.proto = rc;
1163         }
1164
1165         /* setup crypto section */
1166         prm->crypto_xform = ss->xforms;
1167         return 0;
1168 }
1169
1170 static int
1171 fill_ipsec_session(struct rte_ipsec_session *ss, struct rte_ipsec_sa *sa)
1172 {
1173         int32_t rc = 0;
1174
1175         ss->sa = sa;
1176
1177         if (ss->type == RTE_SECURITY_ACTION_TYPE_INLINE_CRYPTO ||
1178                 ss->type == RTE_SECURITY_ACTION_TYPE_INLINE_PROTOCOL) {
1179                 if (ss->security.ses != NULL) {
1180                         rc = rte_ipsec_session_prepare(ss);
1181                         if (rc != 0)
1182                                 memset(ss, 0, sizeof(*ss));
1183                 }
1184         }
1185
1186         return rc;
1187 }
1188
1189 /*
1190  * Initialise related rte_ipsec_sa object.
1191  */
1192 static int
1193 ipsec_sa_init(struct ipsec_sa *lsa, struct rte_ipsec_sa *sa, uint32_t sa_size)
1194 {
1195         int rc;
1196         struct rte_ipsec_sa_prm prm;
1197         struct rte_ipsec_session *ips;
1198         struct rte_ipv4_hdr v4  = {
1199                 .version_ihl = IPVERSION << 4 |
1200                         sizeof(v4) / RTE_IPV4_IHL_MULTIPLIER,
1201                 .time_to_live = IPDEFTTL,
1202                 .next_proto_id = IPPROTO_ESP,
1203                 .src_addr = lsa->src.ip.ip4,
1204                 .dst_addr = lsa->dst.ip.ip4,
1205         };
1206         struct rte_ipv6_hdr v6 = {
1207                 .vtc_flow = htonl(IP6_VERSION << 28),
1208                 .proto = IPPROTO_ESP,
1209         };
1210
1211         if (IS_IP6_TUNNEL(lsa->flags)) {
1212                 memcpy(v6.src_addr, lsa->src.ip.ip6.ip6_b, sizeof(v6.src_addr));
1213                 memcpy(v6.dst_addr, lsa->dst.ip.ip6.ip6_b, sizeof(v6.dst_addr));
1214         }
1215
1216         rc = fill_ipsec_sa_prm(&prm, lsa, &v4, &v6);
1217         if (rc == 0)
1218                 rc = rte_ipsec_sa_init(sa, &prm, sa_size);
1219         if (rc < 0)
1220                 return rc;
1221
1222         /* init primary processing session */
1223         ips = ipsec_get_primary_session(lsa);
1224         rc = fill_ipsec_session(ips, sa);
1225         if (rc != 0)
1226                 return rc;
1227
1228         /* init inline fallback processing session */
1229         if (lsa->fallback_sessions == 1)
1230                 rc = fill_ipsec_session(ipsec_get_fallback_session(lsa), sa);
1231
1232         return rc;
1233 }
1234
1235 /*
1236  * Allocate space and init rte_ipsec_sa strcutures,
1237  * one per session.
1238  */
1239 static int
1240 ipsec_satbl_init(struct sa_ctx *ctx, const struct ipsec_sa *ent,
1241         uint32_t nb_ent, int32_t socket)
1242 {
1243         int32_t rc, sz;
1244         uint32_t i, idx;
1245         size_t tsz;
1246         struct rte_ipsec_sa *sa;
1247         struct ipsec_sa *lsa;
1248         struct rte_ipsec_sa_prm prm;
1249
1250         /* determine SA size */
1251         idx = SPI2IDX(ent[0].spi);
1252         fill_ipsec_sa_prm(&prm, ctx->sa + idx, NULL, NULL);
1253         sz = rte_ipsec_sa_size(&prm);
1254         if (sz < 0) {
1255                 RTE_LOG(ERR, IPSEC, "%s(%p, %u, %d): "
1256                         "failed to determine SA size, error code: %d\n",
1257                         __func__, ctx, nb_ent, socket, sz);
1258                 return sz;
1259         }
1260
1261         tsz = sz * nb_ent;
1262
1263         ctx->satbl = rte_zmalloc_socket(NULL, tsz, RTE_CACHE_LINE_SIZE, socket);
1264         if (ctx->satbl == NULL) {
1265                 RTE_LOG(ERR, IPSEC,
1266                         "%s(%p, %u, %d): failed to allocate %zu bytes\n",
1267                         __func__,  ctx, nb_ent, socket, tsz);
1268                 return -ENOMEM;
1269         }
1270
1271         rc = 0;
1272         for (i = 0; i != nb_ent && rc == 0; i++) {
1273
1274                 idx = SPI2IDX(ent[i].spi);
1275
1276                 sa = (struct rte_ipsec_sa *)((uintptr_t)ctx->satbl + sz * i);
1277                 lsa = ctx->sa + idx;
1278
1279                 rc = ipsec_sa_init(lsa, sa, sz);
1280         }
1281
1282         return rc;
1283 }
1284
1285 /*
1286  * Walk through all SA rules to find an SA with given SPI
1287  */
1288 int
1289 sa_spi_present(uint32_t spi, int inbound)
1290 {
1291         uint32_t i, num;
1292         const struct ipsec_sa *sar;
1293
1294         if (inbound != 0) {
1295                 sar = sa_in;
1296                 num = nb_sa_in;
1297         } else {
1298                 sar = sa_out;
1299                 num = nb_sa_out;
1300         }
1301
1302         for (i = 0; i != num; i++) {
1303                 if (sar[i].spi == spi)
1304                         return i;
1305         }
1306
1307         return -ENOENT;
1308 }
1309
1310 void
1311 sa_init(struct socket_ctx *ctx, int32_t socket_id)
1312 {
1313         int32_t rc;
1314         const char *name;
1315
1316         if (ctx == NULL)
1317                 rte_exit(EXIT_FAILURE, "NULL context.\n");
1318
1319         if (ctx->sa_in != NULL)
1320                 rte_exit(EXIT_FAILURE, "Inbound SA DB for socket %u already "
1321                                 "initialized\n", socket_id);
1322
1323         if (ctx->sa_out != NULL)
1324                 rte_exit(EXIT_FAILURE, "Outbound SA DB for socket %u already "
1325                                 "initialized\n", socket_id);
1326
1327         if (nb_sa_in > 0) {
1328                 name = "sa_in";
1329                 ctx->sa_in = sa_create(name, socket_id);
1330                 if (ctx->sa_in == NULL)
1331                         rte_exit(EXIT_FAILURE, "Error [%d] creating SA "
1332                                 "context %s in socket %d\n", rte_errno,
1333                                 name, socket_id);
1334
1335                 sa_in_add_rules(ctx->sa_in, sa_in, nb_sa_in, ctx);
1336
1337                 if (app_sa_prm.enable != 0) {
1338                         rc = ipsec_satbl_init(ctx->sa_in, sa_in, nb_sa_in,
1339                                 socket_id);
1340                         if (rc != 0)
1341                                 rte_exit(EXIT_FAILURE,
1342                                         "failed to init inbound SAs\n");
1343                 }
1344         } else
1345                 RTE_LOG(WARNING, IPSEC, "No SA Inbound rule specified\n");
1346
1347         if (nb_sa_out > 0) {
1348                 name = "sa_out";
1349                 ctx->sa_out = sa_create(name, socket_id);
1350                 if (ctx->sa_out == NULL)
1351                         rte_exit(EXIT_FAILURE, "Error [%d] creating SA "
1352                                 "context %s in socket %d\n", rte_errno,
1353                                 name, socket_id);
1354
1355                 sa_out_add_rules(ctx->sa_out, sa_out, nb_sa_out, ctx);
1356
1357                 if (app_sa_prm.enable != 0) {
1358                         rc = ipsec_satbl_init(ctx->sa_out, sa_out, nb_sa_out,
1359                                 socket_id);
1360                         if (rc != 0)
1361                                 rte_exit(EXIT_FAILURE,
1362                                         "failed to init outbound SAs\n");
1363                 }
1364         } else
1365                 RTE_LOG(WARNING, IPSEC, "No SA Outbound rule "
1366                         "specified\n");
1367 }
1368
1369 int
1370 inbound_sa_check(struct sa_ctx *sa_ctx, struct rte_mbuf *m, uint32_t sa_idx)
1371 {
1372         struct ipsec_mbuf_metadata *priv;
1373         struct ipsec_sa *sa;
1374
1375         priv = get_priv(m);
1376         sa = priv->sa;
1377         if (sa != NULL)
1378                 return (sa_ctx->sa[sa_idx].spi == sa->spi);
1379
1380         RTE_LOG(ERR, IPSEC, "SA not saved in private data\n");
1381         return 0;
1382 }
1383
1384 static inline void
1385 single_inbound_lookup(struct ipsec_sa *sadb, struct rte_mbuf *pkt,
1386                 void **sa_ret)
1387 {
1388         struct rte_esp_hdr *esp;
1389         struct ip *ip;
1390         uint32_t *src4_addr;
1391         uint8_t *src6_addr;
1392         struct ipsec_sa *sa;
1393         void *result_sa;
1394
1395         *sa_ret = NULL;
1396
1397         ip = rte_pktmbuf_mtod(pkt, struct ip *);
1398         esp = rte_pktmbuf_mtod_offset(pkt, struct rte_esp_hdr *, pkt->l3_len);
1399
1400         if (esp->spi == INVALID_SPI)
1401                 return;
1402
1403         result_sa = sa = &sadb[SPI2IDX(rte_be_to_cpu_32(esp->spi))];
1404         if (rte_be_to_cpu_32(esp->spi) != sa->spi)
1405                 return;
1406
1407         /*
1408          * Mark need for inline offload fallback on the LSB of SA pointer.
1409          * Thanks to packet grouping mechanism which ipsec_process is using
1410          * packets marked for fallback processing will form separate group.
1411          *
1412          * Because it is not safe to use SA pointer it is casted to generic
1413          * pointer to prevent from unintentional use. Use ipsec_mask_saptr
1414          * to get valid struct pointer.
1415          */
1416         if (MBUF_NO_SEC_OFFLOAD(pkt) && sa->fallback_sessions > 0) {
1417                 uintptr_t intsa = (uintptr_t)sa;
1418                 intsa |= IPSEC_SA_OFFLOAD_FALLBACK_FLAG;
1419                 result_sa = (void *)intsa;
1420         }
1421
1422         switch (WITHOUT_TRANSPORT_VERSION(sa->flags)) {
1423         case IP4_TUNNEL:
1424                 src4_addr = RTE_PTR_ADD(ip, offsetof(struct ip, ip_src));
1425                 if ((ip->ip_v == IPVERSION) &&
1426                                 (sa->src.ip.ip4 == *src4_addr) &&
1427                                 (sa->dst.ip.ip4 == *(src4_addr + 1)))
1428                         *sa_ret = result_sa;
1429                 break;
1430         case IP6_TUNNEL:
1431                 src6_addr = RTE_PTR_ADD(ip, offsetof(struct ip6_hdr, ip6_src));
1432                 if ((ip->ip_v == IP6_VERSION) &&
1433                                 !memcmp(&sa->src.ip.ip6.ip6, src6_addr, 16) &&
1434                                 !memcmp(&sa->dst.ip.ip6.ip6, src6_addr + 16, 16))
1435                         *sa_ret = result_sa;
1436                 break;
1437         case TRANSPORT:
1438                 *sa_ret = result_sa;
1439         }
1440 }
1441
1442 void
1443 inbound_sa_lookup(struct sa_ctx *sa_ctx, struct rte_mbuf *pkts[],
1444                 void *sa[], uint16_t nb_pkts)
1445 {
1446         uint32_t i;
1447
1448         for (i = 0; i < nb_pkts; i++)
1449                 single_inbound_lookup(sa_ctx->sa, pkts[i], &sa[i]);
1450 }
1451
1452 void
1453 outbound_sa_lookup(struct sa_ctx *sa_ctx, uint32_t sa_idx[],
1454                 void *sa[], uint16_t nb_pkts)
1455 {
1456         uint32_t i;
1457
1458         for (i = 0; i < nb_pkts; i++)
1459                 sa[i] = &sa_ctx->sa[sa_idx[i]];
1460 }
1461
1462 /*
1463  * Select HW offloads to be used.
1464  */
1465 int
1466 sa_check_offloads(uint16_t port_id, uint64_t *rx_offloads,
1467                 uint64_t *tx_offloads)
1468 {
1469         struct ipsec_sa *rule;
1470         uint32_t idx_sa;
1471         enum rte_security_session_action_type rule_type;
1472
1473         *rx_offloads = 0;
1474         *tx_offloads = 0;
1475
1476         /* Check for inbound rules that use offloads and use this port */
1477         for (idx_sa = 0; idx_sa < nb_sa_in; idx_sa++) {
1478                 rule = &sa_in[idx_sa];
1479                 rule_type = ipsec_get_action_type(rule);
1480                 if ((rule_type == RTE_SECURITY_ACTION_TYPE_INLINE_CRYPTO ||
1481                                 rule_type ==
1482                                 RTE_SECURITY_ACTION_TYPE_INLINE_PROTOCOL)
1483                                 && rule->portid == port_id)
1484                         *rx_offloads |= DEV_RX_OFFLOAD_SECURITY;
1485         }
1486
1487         /* Check for outbound rules that use offloads and use this port */
1488         for (idx_sa = 0; idx_sa < nb_sa_out; idx_sa++) {
1489                 rule = &sa_out[idx_sa];
1490                 rule_type = ipsec_get_action_type(rule);
1491                 if ((rule_type == RTE_SECURITY_ACTION_TYPE_INLINE_CRYPTO ||
1492                                 rule_type ==
1493                                 RTE_SECURITY_ACTION_TYPE_INLINE_PROTOCOL)
1494                                 && rule->portid == port_id)
1495                         *tx_offloads |= DEV_TX_OFFLOAD_SECURITY;
1496         }
1497         return 0;
1498 }