drivers/net/iavf/iavf_ipsec_crypto.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(c) 2020 Intel Corporation
   3  */
   4
   5 #include <rte_cryptodev.h>
   6 #include <rte_ethdev.h>
   7 #include <rte_security_driver.h>
   8 #include <rte_security.h>
   9
  10 #include "iavf.h"
  11 #include "iavf_rxtx.h"
  12 #include "iavf_log.h"
  13 #include "iavf_generic_flow.h"
  14
  15 #include "iavf_ipsec_crypto.h"
  16 #include "iavf_ipsec_crypto_capabilities.h"
  17
  18 /**
  19  * iAVF IPsec Crypto Security Context
  20  */
  21 struct iavf_security_ctx {
  22         struct iavf_adapter *adapter;
  23         int pkt_md_offset;
  24         struct rte_cryptodev_capabilities *crypto_capabilities;
  25 };
  26
  27 /**
  28  * iAVF IPsec Crypto Security Session Parameters
  29  */
  30 struct iavf_security_session {
  31         struct iavf_adapter *adapter;
  32
  33         enum rte_security_ipsec_sa_mode mode;
  34         enum rte_security_ipsec_tunnel_type type;
  35         enum rte_security_ipsec_sa_direction direction;
  36
  37         struct {
  38                 uint32_t spi; /* Security Parameter Index */
  39                 uint32_t hw_idx; /* SA Index in hardware table */
  40         } sa;
  41
  42         struct {
  43                 uint8_t enabled :1;
  44                 union {
  45                         uint64_t value;
  46                         struct {
  47                                 uint32_t hi;
  48                                 uint32_t low;
  49                         };
  50                 };
  51         } esn;
  52
  53         struct {
  54                 uint8_t enabled :1;
  55         } udp_encap;
  56
  57         size_t iv_sz;
  58         size_t icv_sz;
  59         size_t block_sz;
  60
  61         struct iavf_ipsec_crypto_pkt_metadata pkt_metadata_template;
  62 };
  63 /**
  64  *  IV Length field in IPsec Tx Desc uses the following encoding:
  65  *
  66  *  0B - 0
  67  *  4B - 1
  68  *  8B - 2
  69  *  16B - 3
  70  *
  71  * but we also need the IV Length for TSO to correctly calculate the total
  72  * header length so placing it in the upper 6-bits here for easier retrieval.
  73  */
  74 static inline uint8_t
  75 calc_ipsec_desc_iv_len_field(uint16_t iv_sz)
  76 {
  77         uint8_t iv_length = IAVF_IPSEC_IV_LEN_NONE;
  78
  79         switch (iv_sz) {
  80         case 4:
  81                 iv_length = IAVF_IPSEC_IV_LEN_DW;
  82                 break;
  83         case 8:
  84                 iv_length = IAVF_IPSEC_IV_LEN_DDW;
  85                 break;
  86         case 16:
  87                 iv_length = IAVF_IPSEC_IV_LEN_QDW;
  88                 break;
  89         }
  90
  91         return (iv_sz << 2) | iv_length;
  92 }
  93
  94 static unsigned int
  95 iavf_ipsec_crypto_session_size_get(void *device __rte_unused)
  96 {
  97         return sizeof(struct iavf_security_session);
  98 }
  99
 100 static const struct rte_cryptodev_symmetric_capability *
 101 get_capability(struct iavf_security_ctx *iavf_sctx,
 102         uint32_t algo, uint32_t type)
 103 {
 104         const struct rte_cryptodev_capabilities *capability;
 105         int i = 0;
 106
 107         capability = &iavf_sctx->crypto_capabilities[i];
 108
 109         while (capability->op != RTE_CRYPTO_OP_TYPE_UNDEFINED) {
 110                 if (capability->op == RTE_CRYPTO_OP_TYPE_SYMMETRIC &&
 111                         (uint32_t)capability->sym.xform_type == type &&
 112                         (uint32_t)capability->sym.cipher.algo == algo)
 113                         return &capability->sym;
 114                 /** try next capability */
 115                 capability = &iavf_crypto_capabilities[i++];
 116         }
 117
 118         return NULL;
 119 }
 120
 121 static const struct rte_cryptodev_symmetric_capability *
 122 get_auth_capability(struct iavf_security_ctx *iavf_sctx,
 123         enum rte_crypto_auth_algorithm algo)
 124 {
 125         return get_capability(iavf_sctx, algo, RTE_CRYPTO_SYM_XFORM_AUTH);
 126 }
 127
 128 static const struct rte_cryptodev_symmetric_capability *
 129 get_cipher_capability(struct iavf_security_ctx *iavf_sctx,
 130         enum rte_crypto_cipher_algorithm algo)
 131 {
 132         return get_capability(iavf_sctx, algo, RTE_CRYPTO_SYM_XFORM_CIPHER);
 133 }
 134 static const struct rte_cryptodev_symmetric_capability *
 135 get_aead_capability(struct iavf_security_ctx *iavf_sctx,
 136         enum rte_crypto_aead_algorithm algo)
 137 {
 138         return get_capability(iavf_sctx, algo, RTE_CRYPTO_SYM_XFORM_AEAD);
 139 }
 140
 141 static uint16_t
 142 get_cipher_blocksize(struct iavf_security_ctx *iavf_sctx,
 143         enum rte_crypto_cipher_algorithm algo)
 144 {
 145         const struct rte_cryptodev_symmetric_capability *capability;
 146
 147         capability = get_cipher_capability(iavf_sctx, algo);
 148         if (capability == NULL)
 149                 return 0;
 150
 151         return capability->cipher.block_size;
 152 }
 153
 154 static uint16_t
 155 get_aead_blocksize(struct iavf_security_ctx *iavf_sctx,
 156         enum rte_crypto_aead_algorithm algo)
 157 {
 158         const struct rte_cryptodev_symmetric_capability *capability;
 159
 160         capability = get_aead_capability(iavf_sctx, algo);
 161         if (capability == NULL)
 162                 return 0;
 163
 164         return capability->cipher.block_size;
 165 }
 166
 167 static uint16_t
 168 get_auth_blocksize(struct iavf_security_ctx *iavf_sctx,
 169         enum rte_crypto_auth_algorithm algo)
 170 {
 171         const struct rte_cryptodev_symmetric_capability *capability;
 172
 173         capability = get_auth_capability(iavf_sctx, algo);
 174         if (capability == NULL)
 175                 return 0;
 176
 177         return capability->auth.block_size;
 178 }
 179
 180 static uint8_t
 181 calc_context_desc_cipherblock_sz(size_t len)
 182 {
 183         switch (len) {
 184         case 8:
 185                 return 0x2;
 186         case 16:
 187                 return 0x3;
 188         default:
 189                 return 0x0;
 190         }
 191 }
 192
 193 static int
 194 valid_length(uint32_t len, uint32_t min, uint32_t max, uint32_t increment)
 195 {
 196         if (len < min || len > max)
 197                 return false;
 198
 199         if (increment == 0)
 200                 return true;
 201
 202         if ((len - min) % increment)
 203                 return false;
 204
 205         /* make sure it fits in the key array */
 206         if (len > VIRTCHNL_IPSEC_MAX_KEY_LEN)
 207                 return false;
 208
 209         return true;
 210 }
 211
 212 static int
 213 valid_auth_xform(struct iavf_security_ctx *iavf_sctx,
 214         struct rte_crypto_auth_xform *auth)
 215 {
 216         const struct rte_cryptodev_symmetric_capability *capability;
 217
 218         capability = get_auth_capability(iavf_sctx, auth->algo);
 219         if (capability == NULL)
 220                 return false;
 221
 222         /* verify key size */
 223         if (!valid_length(auth->key.length,
 224                 capability->auth.key_size.min,
 225                 capability->auth.key_size.max,
 226                 capability->aead.key_size.increment))
 227                 return false;
 228
 229         return true;
 230 }
 231
 232 static int
 233 valid_cipher_xform(struct iavf_security_ctx *iavf_sctx,
 234         struct rte_crypto_cipher_xform *cipher)
 235 {
 236         const struct rte_cryptodev_symmetric_capability *capability;
 237
 238         capability = get_cipher_capability(iavf_sctx, cipher->algo);
 239         if (capability == NULL)
 240                 return false;
 241
 242         /* verify key size */
 243         if (!valid_length(cipher->key.length,
 244                 capability->cipher.key_size.min,
 245                 capability->cipher.key_size.max,
 246                 capability->cipher.key_size.increment))
 247                 return false;
 248
 249         return true;
 250 }
 251
 252 static int
 253 valid_aead_xform(struct iavf_security_ctx *iavf_sctx,
 254         struct rte_crypto_aead_xform *aead)
 255 {
 256         const struct rte_cryptodev_symmetric_capability *capability;
 257
 258         capability = get_aead_capability(iavf_sctx, aead->algo);
 259         if (capability == NULL)
 260                 return false;
 261
 262         /* verify key size */
 263         if (!valid_length(aead->key.length,
 264                 capability->aead.key_size.min,
 265                 capability->aead.key_size.max,
 266                 capability->aead.key_size.increment))
 267                 return false;
 268
 269         return true;
 270 }
 271
 272 static int
 273 iavf_ipsec_crypto_session_validate_conf(struct iavf_security_ctx *iavf_sctx,
 274         struct rte_security_session_conf *conf)
 275 {
 276         /** validate security action/protocol selection */
 277         if (conf->action_type != RTE_SECURITY_ACTION_TYPE_INLINE_CRYPTO ||
 278                 conf->protocol != RTE_SECURITY_PROTOCOL_IPSEC) {
 279                 PMD_DRV_LOG(ERR, "Invalid action / protocol specified");
 280                 return -EINVAL;
 281         }
 282
 283         /** validate IPsec protocol selection */
 284         if (conf->ipsec.proto != RTE_SECURITY_IPSEC_SA_PROTO_ESP) {
 285                 PMD_DRV_LOG(ERR, "Invalid IPsec protocol specified");
 286                 return -EINVAL;
 287         }
 288
 289         /** validate selected options */
 290         if (conf->ipsec.options.copy_dscp ||
 291                 conf->ipsec.options.copy_flabel ||
 292                 conf->ipsec.options.copy_df ||
 293                 conf->ipsec.options.dec_ttl ||
 294                 conf->ipsec.options.ecn ||
 295                 conf->ipsec.options.stats) {
 296                 PMD_DRV_LOG(ERR, "Invalid IPsec option specified");
 297                 return -EINVAL;
 298         }
 299
 300         /**
 301          * Validate crypto xforms parameters.
 302          *
 303          * AEAD transforms can be used for either inbound/outbound IPsec SAs,
 304          * for non-AEAD crypto transforms we explicitly only support CIPHER/AUTH
 305          * for outbound and AUTH/CIPHER chained transforms for inbound IPsec.
 306          */
 307         if (conf->crypto_xform->type == RTE_CRYPTO_SYM_XFORM_AEAD) {
 308                 if (!valid_aead_xform(iavf_sctx, &conf->crypto_xform->aead)) {
 309                         PMD_DRV_LOG(ERR, "Invalid IPsec option specified");
 310                         return -EINVAL;
 311                 }
 312         } else if (conf->ipsec.direction == RTE_SECURITY_IPSEC_SA_DIR_EGRESS &&
 313                 conf->crypto_xform->type == RTE_CRYPTO_SYM_XFORM_CIPHER &&
 314                 conf->crypto_xform->next &&
 315                 conf->crypto_xform->next->type == RTE_CRYPTO_SYM_XFORM_AUTH) {
 316                 if (!valid_cipher_xform(iavf_sctx,
 317                                 &conf->crypto_xform->cipher)) {
 318                         PMD_DRV_LOG(ERR, "Invalid IPsec option specified");
 319                         return -EINVAL;
 320                 }
 321
 322                 if (!valid_auth_xform(iavf_sctx,
 323                                 &conf->crypto_xform->next->auth)) {
 324                         PMD_DRV_LOG(ERR, "Invalid IPsec option specified");
 325                         return -EINVAL;
 326                 }
 327         } else if (conf->ipsec.direction == RTE_SECURITY_IPSEC_SA_DIR_INGRESS &&
 328                 conf->crypto_xform->type == RTE_CRYPTO_SYM_XFORM_AUTH &&
 329                 conf->crypto_xform->next &&
 330                 conf->crypto_xform->next->type == RTE_CRYPTO_SYM_XFORM_CIPHER) {
 331                 if (!valid_auth_xform(iavf_sctx, &conf->crypto_xform->auth)) {
 332                         PMD_DRV_LOG(ERR, "Invalid IPsec option specified");
 333                         return -EINVAL;
 334                 }
 335
 336                 if (!valid_cipher_xform(iavf_sctx,
 337                                 &conf->crypto_xform->next->cipher)) {
 338                         PMD_DRV_LOG(ERR, "Invalid IPsec option specified");
 339                         return -EINVAL;
 340                 }
 341         }
 342
 343         return 0;
 344 }
 345
 346 static void
 347 sa_add_set_aead_params(struct virtchnl_ipsec_crypto_cfg_item *cfg,
 348         struct rte_crypto_aead_xform *aead, uint32_t salt)
 349 {
 350         cfg->crypto_type = VIRTCHNL_AEAD;
 351
 352         switch (aead->algo) {
 353         case RTE_CRYPTO_AEAD_AES_CCM:
 354                 cfg->algo_type = VIRTCHNL_AES_CCM; break;
 355         case RTE_CRYPTO_AEAD_AES_GCM:
 356                 cfg->algo_type = VIRTCHNL_AES_GCM; break;
 357         case RTE_CRYPTO_AEAD_CHACHA20_POLY1305:
 358                 cfg->algo_type = VIRTCHNL_CHACHA20_POLY1305; break;
 359         default:
 360                 PMD_DRV_LOG(ERR, "Invalid AEAD parameters");
 361                 break;
 362         }
 363
 364         cfg->key_len = aead->key.length;
 365         cfg->iv_len = sizeof(uint64_t); /* iv.length includes salt len */
 366         cfg->digest_len = aead->digest_length;
 367         cfg->salt = salt;
 368
 369         memcpy(cfg->key_data, aead->key.data, cfg->key_len);
 370 }
 371
 372 static void
 373 sa_add_set_cipher_params(struct virtchnl_ipsec_crypto_cfg_item *cfg,
 374         struct rte_crypto_cipher_xform *cipher, uint32_t salt)
 375 {
 376         cfg->crypto_type = VIRTCHNL_CIPHER;
 377
 378         switch (cipher->algo) {
 379         case RTE_CRYPTO_CIPHER_AES_CBC:
 380                 cfg->algo_type = VIRTCHNL_AES_CBC; break;
 381         case RTE_CRYPTO_CIPHER_3DES_CBC:
 382                 cfg->algo_type = VIRTCHNL_3DES_CBC; break;
 383         case RTE_CRYPTO_CIPHER_NULL:
 384                 cfg->algo_type = VIRTCHNL_CIPHER_NO_ALG; break;
 385         case RTE_CRYPTO_CIPHER_AES_CTR:
 386                 cfg->algo_type = VIRTCHNL_AES_CTR;
 387                 cfg->salt = salt;
 388                 break;
 389         default:
 390                 PMD_DRV_LOG(ERR, "Invalid cipher parameters");
 391                 break;
 392         }
 393
 394         cfg->key_len = cipher->key.length;
 395         cfg->iv_len = cipher->iv.length;
 396         cfg->salt = salt;
 397
 398         memcpy(cfg->key_data, cipher->key.data, cfg->key_len);
 399 }
 400
 401 static void
 402 sa_add_set_auth_params(struct virtchnl_ipsec_crypto_cfg_item *cfg,
 403         struct rte_crypto_auth_xform *auth, uint32_t salt)
 404 {
 405         cfg->crypto_type = VIRTCHNL_AUTH;
 406
 407         switch (auth->algo) {
 408         case RTE_CRYPTO_AUTH_NULL:
 409                 cfg->algo_type = VIRTCHNL_HASH_NO_ALG; break;
 410         case RTE_CRYPTO_AUTH_AES_CBC_MAC:
 411                 cfg->algo_type = VIRTCHNL_AES_CBC_MAC; break;
 412         case RTE_CRYPTO_AUTH_AES_CMAC:
 413                 cfg->algo_type = VIRTCHNL_AES_CMAC; break;
 414         case RTE_CRYPTO_AUTH_AES_XCBC_MAC:
 415                 cfg->algo_type = VIRTCHNL_AES_XCBC_MAC; break;
 416         case RTE_CRYPTO_AUTH_MD5_HMAC:
 417                 cfg->algo_type = VIRTCHNL_MD5_HMAC; break;
 418         case RTE_CRYPTO_AUTH_SHA1_HMAC:
 419                 cfg->algo_type = VIRTCHNL_SHA1_HMAC; break;
 420         case RTE_CRYPTO_AUTH_SHA224_HMAC:
 421                 cfg->algo_type = VIRTCHNL_SHA224_HMAC; break;
 422         case RTE_CRYPTO_AUTH_SHA256_HMAC:
 423                 cfg->algo_type = VIRTCHNL_SHA256_HMAC; break;
 424         case RTE_CRYPTO_AUTH_SHA384_HMAC:
 425                 cfg->algo_type = VIRTCHNL_SHA384_HMAC; break;
 426         case RTE_CRYPTO_AUTH_SHA512_HMAC:
 427                 cfg->algo_type = VIRTCHNL_SHA512_HMAC; break;
 428         case RTE_CRYPTO_AUTH_AES_GMAC:
 429                 cfg->algo_type = VIRTCHNL_AES_GMAC;
 430                 cfg->salt = salt;
 431                 break;
 432         default:
 433                 PMD_DRV_LOG(ERR, "Invalid auth parameters");
 434                 break;
 435         }
 436
 437         cfg->key_len = auth->key.length;
 438         /* special case for RTE_CRYPTO_AUTH_AES_GMAC */
 439         if (auth->algo == RTE_CRYPTO_AUTH_AES_GMAC)
 440                 cfg->iv_len = sizeof(uint64_t); /* iv.length includes salt */
 441         else
 442                 cfg->iv_len = auth->iv.length;
 443         cfg->digest_len = auth->digest_length;
 444
 445         memcpy(cfg->key_data, auth->key.data, cfg->key_len);
 446 }
 447
 448 /**
 449  * Send SA add virtual channel request to Inline IPsec driver.
 450  *
 451  * Inline IPsec driver expects SPI and destination IP address to be in host
 452  * order, but DPDK APIs are network order, therefore we need to do a htonl
 453  * conversion of these parameters.
 454  */
 455 static uint32_t
 456 iavf_ipsec_crypto_security_association_add(struct iavf_adapter *adapter,
 457         struct rte_security_session_conf *conf)
 458 {
 459         struct inline_ipsec_msg *request = NULL, *response = NULL;
 460         struct virtchnl_ipsec_sa_cfg *sa_cfg;
 461         size_t request_len, response_len;
 462
 463         int rc;
 464
 465         request_len = sizeof(struct inline_ipsec_msg) +
 466                         sizeof(struct virtchnl_ipsec_sa_cfg);
 467
 468         request = rte_malloc("iavf-sad-add-request", request_len, 0);
 469         if (request == NULL) {
 470                 rc = -ENOMEM;
 471                 goto update_cleanup;
 472         }
 473
 474         response_len = sizeof(struct inline_ipsec_msg) +
 475                         sizeof(struct virtchnl_ipsec_sa_cfg_resp);
 476         response = rte_malloc("iavf-sad-add-response", response_len, 0);
 477         if (response == NULL) {
 478                 rc = -ENOMEM;
 479                 goto update_cleanup;
 480         }
 481
 482         /* set msg header params */
 483         request->ipsec_opcode = INLINE_IPSEC_OP_SA_CREATE;
 484         request->req_id = (uint16_t)0xDEADBEEF;
 485
 486         /* set SA configuration params */
 487         sa_cfg = (struct virtchnl_ipsec_sa_cfg *)(request + 1);
 488
 489         sa_cfg->spi = conf->ipsec.spi;
 490         sa_cfg->virtchnl_protocol_type = VIRTCHNL_PROTO_ESP;
 491         sa_cfg->virtchnl_direction =
 492                 conf->ipsec.direction == RTE_SECURITY_IPSEC_SA_DIR_INGRESS ?
 493                         VIRTCHNL_DIR_INGRESS : VIRTCHNL_DIR_EGRESS;
 494
 495         if (conf->ipsec.options.esn) {
 496                 sa_cfg->esn_enabled = 1;
 497                 sa_cfg->esn_hi = conf->ipsec.esn.hi;
 498                 sa_cfg->esn_low = conf->ipsec.esn.low;
 499         }
 500
 501         if (conf->ipsec.options.udp_encap)
 502                 sa_cfg->udp_encap_enabled = 1;
 503
 504         /* Set outer IP params */
 505         if (conf->ipsec.tunnel.type == RTE_SECURITY_IPSEC_TUNNEL_IPV4) {
 506                 sa_cfg->virtchnl_ip_type = VIRTCHNL_IPV4;
 507
 508                 *((uint32_t *)sa_cfg->dst_addr) =
 509                         htonl(conf->ipsec.tunnel.ipv4.dst_ip.s_addr);
 510         } else {
 511                 uint32_t *v6_dst_addr =
 512                         (uint32_t *)conf->ipsec.tunnel.ipv6.dst_addr.s6_addr;
 513
 514                 sa_cfg->virtchnl_ip_type = VIRTCHNL_IPV6;
 515
 516                 ((uint32_t *)sa_cfg->dst_addr)[0] = htonl(v6_dst_addr[0]);
 517                 ((uint32_t *)sa_cfg->dst_addr)[1] = htonl(v6_dst_addr[1]);
 518                 ((uint32_t *)sa_cfg->dst_addr)[2] = htonl(v6_dst_addr[2]);
 519                 ((uint32_t *)sa_cfg->dst_addr)[3] = htonl(v6_dst_addr[3]);
 520         }
 521
 522         /* set crypto params */
 523         if (conf->crypto_xform->type == RTE_CRYPTO_SYM_XFORM_AEAD) {
 524                 sa_add_set_aead_params(&sa_cfg->crypto_cfg.items[0],
 525                         &conf->crypto_xform->aead, conf->ipsec.salt);
 526
 527         } else if (conf->crypto_xform->type == RTE_CRYPTO_SYM_XFORM_CIPHER) {
 528                 sa_add_set_cipher_params(&sa_cfg->crypto_cfg.items[0],
 529                         &conf->crypto_xform->cipher, conf->ipsec.salt);
 530                 sa_add_set_auth_params(&sa_cfg->crypto_cfg.items[1],
 531                         &conf->crypto_xform->next->auth, conf->ipsec.salt);
 532
 533         } else if (conf->crypto_xform->type == RTE_CRYPTO_SYM_XFORM_AUTH) {
 534                 sa_add_set_auth_params(&sa_cfg->crypto_cfg.items[0],
 535                         &conf->crypto_xform->auth, conf->ipsec.salt);
 536                 if (conf->crypto_xform->auth.algo != RTE_CRYPTO_AUTH_AES_GMAC)
 537                         sa_add_set_cipher_params(&sa_cfg->crypto_cfg.items[1],
 538                         &conf->crypto_xform->next->cipher, conf->ipsec.salt);
 539         }
 540
 541         /* send virtual channel request to add SA to hardware database */
 542         rc = iavf_ipsec_crypto_request(adapter,
 543                         (uint8_t *)request, request_len,
 544                         (uint8_t *)response, response_len);
 545         if (rc)
 546                 goto update_cleanup;
 547
 548         /* verify response id */
 549         if (response->ipsec_opcode != request->ipsec_opcode ||
 550                 response->req_id != request->req_id)
 551                 rc = -EFAULT;
 552         else
 553                 rc = response->ipsec_data.sa_cfg_resp->sa_handle;
 554 update_cleanup:
 555         rte_free(response);
 556         rte_free(request);
 557
 558         return rc;
 559 }
 560
 561 static void
 562 set_pkt_metadata_template(struct iavf_ipsec_crypto_pkt_metadata *template,
 563         struct iavf_security_session *sess)
 564 {
 565         template->sa_idx = sess->sa.hw_idx;
 566
 567         if (sess->udp_encap.enabled)
 568                 template->ol_flags = IAVF_IPSEC_CRYPTO_OL_FLAGS_NATT;
 569
 570         if (sess->esn.enabled)
 571                 template->ol_flags = IAVF_IPSEC_CRYPTO_OL_FLAGS_ESN;
 572
 573         template->len_iv = calc_ipsec_desc_iv_len_field(sess->iv_sz);
 574         template->ctx_desc_ipsec_params =
 575                         calc_context_desc_cipherblock_sz(sess->block_sz) |
 576                         ((uint8_t)(sess->icv_sz >> 2) << 3);
 577 }
 578
 579 static void
 580 set_session_parameter(struct iavf_security_ctx *iavf_sctx,
 581         struct iavf_security_session *sess,
 582         struct rte_security_session_conf *conf, uint32_t sa_idx)
 583 {
 584         sess->adapter = iavf_sctx->adapter;
 585
 586         sess->mode = conf->ipsec.mode;
 587         sess->direction = conf->ipsec.direction;
 588
 589         if (sess->mode == RTE_SECURITY_IPSEC_SA_MODE_TUNNEL)
 590                 sess->type = conf->ipsec.tunnel.type;
 591
 592         sess->sa.spi = conf->ipsec.spi;
 593         sess->sa.hw_idx = sa_idx;
 594
 595         if (conf->ipsec.options.esn) {
 596                 sess->esn.enabled = 1;
 597                 sess->esn.value = conf->ipsec.esn.value;
 598         }
 599
 600         if (conf->ipsec.options.udp_encap)
 601                 sess->udp_encap.enabled = 1;
 602
 603         if (conf->crypto_xform->type == RTE_CRYPTO_SYM_XFORM_AEAD) {
 604                 sess->block_sz = get_aead_blocksize(iavf_sctx,
 605                         conf->crypto_xform->aead.algo);
 606                 sess->iv_sz = sizeof(uint64_t); /* iv.length includes salt */
 607                 sess->icv_sz = conf->crypto_xform->aead.digest_length;
 608         } else if (conf->crypto_xform->type == RTE_CRYPTO_SYM_XFORM_CIPHER) {
 609                 sess->block_sz = get_cipher_blocksize(iavf_sctx,
 610                         conf->crypto_xform->cipher.algo);
 611                 sess->iv_sz = conf->crypto_xform->cipher.iv.length;
 612                 sess->icv_sz = conf->crypto_xform->next->auth.digest_length;
 613         } else if (conf->crypto_xform->type == RTE_CRYPTO_SYM_XFORM_AUTH) {
 614                 if (conf->crypto_xform->auth.algo == RTE_CRYPTO_AUTH_AES_GMAC) {
 615                         sess->block_sz = get_auth_blocksize(iavf_sctx,
 616                                 conf->crypto_xform->auth.algo);
 617                         sess->iv_sz = conf->crypto_xform->auth.iv.length;
 618                         sess->icv_sz = conf->crypto_xform->auth.digest_length;
 619                 } else {
 620                         sess->block_sz = get_cipher_blocksize(iavf_sctx,
 621                                 conf->crypto_xform->next->cipher.algo);
 622                         sess->iv_sz =
 623                                 conf->crypto_xform->next->cipher.iv.length;
 624                         sess->icv_sz = conf->crypto_xform->auth.digest_length;
 625                 }
 626         }
 627
 628         set_pkt_metadata_template(&sess->pkt_metadata_template, sess);
 629 }
 630
 631 /**
 632  * Create IPsec Security Association for inline IPsec Crypto offload.
 633  *
 634  * 1. validate session configuration parameters
 635  * 2. allocate session memory from mempool
 636  * 3. add SA to hardware database
 637  * 4. set session parameters
 638  * 5. create packet metadata template for datapath
 639  */
 640 static int
 641 iavf_ipsec_crypto_session_create(void *device,
 642                                  struct rte_security_session_conf *conf,
 643                                  struct rte_security_session *session,
 644                                  struct rte_mempool *mempool)
 645 {
 646         struct rte_eth_dev *ethdev = device;
 647         struct iavf_adapter *adapter =
 648                 IAVF_DEV_PRIVATE_TO_ADAPTER(ethdev->data->dev_private);
 649         struct iavf_security_ctx *iavf_sctx = adapter->security_ctx;
 650         struct iavf_security_session *iavf_session = NULL;
 651         int sa_idx;
 652         int ret = 0;
 653
 654         /* validate that all SA parameters are valid for device */
 655         ret = iavf_ipsec_crypto_session_validate_conf(iavf_sctx, conf);
 656         if (ret)
 657                 return ret;
 658
 659         /* allocate session context */
 660         if (rte_mempool_get(mempool, (void **)&iavf_session)) {
 661                 PMD_DRV_LOG(ERR, "Cannot get object from sess mempool");
 662                 return -ENOMEM;
 663         }
 664
 665         /* add SA to hardware database */
 666         sa_idx = iavf_ipsec_crypto_security_association_add(adapter, conf);
 667         if (sa_idx < 0) {
 668                 PMD_DRV_LOG(ERR,
 669                         "Failed to add SA (spi: %d, mode: %s, direction: %s)",
 670                         conf->ipsec.spi,
 671                         conf->ipsec.mode ==
 672                                 RTE_SECURITY_IPSEC_SA_MODE_TRANSPORT ?
 673                                 "transport" : "tunnel",
 674                         conf->ipsec.direction ==
 675                                 RTE_SECURITY_IPSEC_SA_DIR_INGRESS ?
 676                                 "inbound" : "outbound");
 677
 678                 rte_mempool_put(mempool, iavf_session);
 679                 return -EFAULT;
 680         }
 681
 682         /* save data plane required session parameters */
 683         set_session_parameter(iavf_sctx, iavf_session, conf, sa_idx);
 684
 685         /* save to security session private data */
 686         set_sec_session_private_data(session, iavf_session);
 687
 688         return 0;
 689 }
 690
 691 /**
 692  * Check if valid ipsec crypto action.
 693  * SPI must be non-zero and SPI in session must match SPI value
 694  * passed into function.
 695  *
 696  * returns: 0 if invalid session or SPI value equal zero
 697  * returns: 1 if valid
 698  */
 699 uint32_t
 700 iavf_ipsec_crypto_action_valid(struct rte_eth_dev *ethdev,
 701         const struct rte_security_session *session, uint32_t spi)
 702 {
 703         struct iavf_adapter *adapter =
 704                 IAVF_DEV_PRIVATE_TO_ADAPTER(ethdev->data->dev_private);
 705         struct iavf_security_session *sess = session->sess_private_data;
 706
 707         /* verify we have a valid session and that it belong to this adapter */
 708         if (unlikely(sess == NULL || sess->adapter != adapter))
 709                 return false;
 710
 711         /* SPI value must be non-zero */
 712         if (spi == 0)
 713                 return false;
 714         /* Session SPI must patch flow SPI*/
 715         else if (sess->sa.spi == spi) {
 716                 return true;
 717                 /**
 718                  * TODO: We should add a way of tracking valid hw SA indices to
 719                  * make validation less brittle
 720                  */
 721         }
 722
 723                 return true;
 724 }
 725
 726 /**
 727  * Send virtual channel security policy add request to IES driver.
 728  *
 729  * IES driver expects SPI and destination IP address to be in host
 730  * order, but DPDK APIs are network order, therefore we need to do a htonl
 731  * conversion of these parameters.
 732  */
 733 int
 734 iavf_ipsec_crypto_inbound_security_policy_add(struct iavf_adapter *adapter,
 735         uint32_t esp_spi,
 736         uint8_t is_v4,
 737         rte_be32_t v4_dst_addr,
 738         uint8_t *v6_dst_addr,
 739         uint8_t drop)
 740 {
 741         struct inline_ipsec_msg *request = NULL, *response = NULL;
 742         size_t request_len, response_len;
 743         int rc = 0;
 744
 745         request_len = sizeof(struct inline_ipsec_msg) +
 746                         sizeof(struct virtchnl_ipsec_sp_cfg);
 747         request = rte_malloc("iavf-inbound-security-policy-add-request",
 748                                 request_len, 0);
 749         if (request == NULL) {
 750                 rc = -ENOMEM;
 751                 goto update_cleanup;
 752         }
 753
 754         /* set msg header params */
 755         request->ipsec_opcode = INLINE_IPSEC_OP_SP_CREATE;
 756         request->req_id = (uint16_t)0xDEADBEEF;
 757
 758         /* ESP SPI */
 759         request->ipsec_data.sp_cfg->spi = htonl(esp_spi);
 760
 761         /* Destination IP  */
 762         if (is_v4) {
 763                 request->ipsec_data.sp_cfg->table_id =
 764                                 VIRTCHNL_IPSEC_INBOUND_SPD_TBL_IPV4;
 765                 request->ipsec_data.sp_cfg->dip[0] = htonl(v4_dst_addr);
 766         } else {
 767                 request->ipsec_data.sp_cfg->table_id =
 768                                 VIRTCHNL_IPSEC_INBOUND_SPD_TBL_IPV6;
 769                 request->ipsec_data.sp_cfg->dip[0] =
 770                                 htonl(((uint32_t *)v6_dst_addr)[0]);
 771                 request->ipsec_data.sp_cfg->dip[1] =
 772                                 htonl(((uint32_t *)v6_dst_addr)[1]);
 773                 request->ipsec_data.sp_cfg->dip[2] =
 774                                 htonl(((uint32_t *)v6_dst_addr)[2]);
 775                 request->ipsec_data.sp_cfg->dip[3] =
 776                                 htonl(((uint32_t *)v6_dst_addr)[3]);
 777         }
 778
 779         request->ipsec_data.sp_cfg->drop = drop;
 780
 781         /** Traffic Class/Congestion Domain currently not support */
 782         request->ipsec_data.sp_cfg->set_tc = 0;
 783         request->ipsec_data.sp_cfg->cgd = 0;
 784
 785         response_len = sizeof(struct inline_ipsec_msg) +
 786                         sizeof(struct virtchnl_ipsec_sp_cfg_resp);
 787         response = rte_malloc("iavf-inbound-security-policy-add-response",
 788                                 response_len, 0);
 789         if (response == NULL) {
 790                 rc = -ENOMEM;
 791                 goto update_cleanup;
 792         }
 793
 794         /* send virtual channel request to add SA to hardware database */
 795         rc = iavf_ipsec_crypto_request(adapter,
 796                         (uint8_t *)request, request_len,
 797                         (uint8_t *)response, response_len);
 798         if (rc)
 799                 goto update_cleanup;
 800
 801         /* verify response */
 802         if (response->ipsec_opcode != request->ipsec_opcode ||
 803                 response->req_id != request->req_id)
 804                 rc = -EFAULT;
 805         else
 806                 rc = response->ipsec_data.sp_cfg_resp->rule_id;
 807
 808 update_cleanup:
 809         rte_free(request);
 810         rte_free(response);
 811
 812         return rc;
 813 }
 814
 815 static uint32_t
 816 iavf_ipsec_crypto_sa_update_esn(struct iavf_adapter *adapter,
 817         struct iavf_security_session *sess)
 818 {
 819         struct inline_ipsec_msg *request = NULL, *response = NULL;
 820         size_t request_len, response_len;
 821         int rc = 0;
 822
 823         request_len = sizeof(struct inline_ipsec_msg) +
 824                         sizeof(struct virtchnl_ipsec_sa_update);
 825         request = rte_malloc("iavf-sa-update-request", request_len, 0);
 826         if (request == NULL) {
 827                 rc = -ENOMEM;
 828                 goto update_cleanup;
 829         }
 830
 831         response_len = sizeof(struct inline_ipsec_msg) +
 832                         sizeof(struct virtchnl_ipsec_resp);
 833         response = rte_malloc("iavf-sa-update-response", response_len, 0);
 834         if (response == NULL) {
 835                 rc = -ENOMEM;
 836                 goto update_cleanup;
 837         }
 838
 839         /* set msg header params */
 840         request->ipsec_opcode = INLINE_IPSEC_OP_SA_UPDATE;
 841         request->req_id = (uint16_t)0xDEADBEEF;
 842
 843         /* set request params */
 844         request->ipsec_data.sa_update->sa_index = sess->sa.hw_idx;
 845         request->ipsec_data.sa_update->esn_hi = sess->esn.hi;
 846
 847         /* send virtual channel request to add SA to hardware database */
 848         rc = iavf_ipsec_crypto_request(adapter,
 849                         (uint8_t *)request, request_len,
 850                         (uint8_t *)response, response_len);
 851         if (rc)
 852                 goto update_cleanup;
 853
 854         /* verify response */
 855         if (response->ipsec_opcode != request->ipsec_opcode ||
 856                 response->req_id != request->req_id)
 857                 rc = -EFAULT;
 858         else
 859                 rc = response->ipsec_data.ipsec_resp->resp;
 860
 861 update_cleanup:
 862         rte_free(request);
 863         rte_free(response);
 864
 865         return rc;
 866 }
 867
 868 static int
 869 iavf_ipsec_crypto_session_update(void *device,
 870                 struct rte_security_session *session,
 871                 struct rte_security_session_conf *conf)
 872 {
 873         struct iavf_adapter *adapter = NULL;
 874         struct iavf_security_session *iavf_sess = NULL;
 875         struct rte_eth_dev *eth_dev = (struct rte_eth_dev *)device;
 876         int rc = 0;
 877
 878         adapter = IAVF_DEV_PRIVATE_TO_ADAPTER(eth_dev->data->dev_private);
 879         iavf_sess = (struct iavf_security_session *)session->sess_private_data;
 880
 881         /* verify we have a valid session and that it belong to this adapter */
 882         if (unlikely(iavf_sess == NULL || iavf_sess->adapter != adapter))
 883                 return -EINVAL;
 884
 885         /* update esn hi 32-bits */
 886         if (iavf_sess->esn.enabled && conf->ipsec.options.esn) {
 887                 /**
 888                  * Update ESN in hardware for inbound SA. Store in
 889                  * iavf_security_session for outbound SA for use
 890                  * in *iavf_ipsec_crypto_pkt_metadata_set* function.
 891                  */
 892                 if (iavf_sess->direction == RTE_SECURITY_IPSEC_SA_DIR_INGRESS)
 893                         rc = iavf_ipsec_crypto_sa_update_esn(adapter,
 894                                         iavf_sess);
 895                 else
 896                         iavf_sess->esn.hi = conf->ipsec.esn.hi;
 897         }
 898
 899         return rc;
 900 }
 901
 902 static int
 903 iavf_ipsec_crypto_session_stats_get(void *device __rte_unused,
 904                 struct rte_security_session *session __rte_unused,
 905                 struct rte_security_stats *stats __rte_unused)
 906 {
 907         return -EOPNOTSUPP;
 908 }
 909
 910 int
 911 iavf_ipsec_crypto_security_policy_delete(struct iavf_adapter *adapter,
 912         uint8_t is_v4, uint32_t flow_id)
 913 {
 914         struct inline_ipsec_msg *request = NULL, *response = NULL;
 915         size_t request_len, response_len;
 916         int rc = 0;
 917
 918         request_len = sizeof(struct inline_ipsec_msg) +
 919                         sizeof(struct virtchnl_ipsec_sp_destroy);
 920         request = rte_malloc("iavf-sp-del-request", request_len, 0);
 921         if (request == NULL) {
 922                 rc = -ENOMEM;
 923                 goto update_cleanup;
 924         }
 925
 926         response_len = sizeof(struct inline_ipsec_msg) +
 927                         sizeof(struct virtchnl_ipsec_resp);
 928         response = rte_malloc("iavf-sp-del-response", response_len, 0);
 929         if (response == NULL) {
 930                 rc = -ENOMEM;
 931                 goto update_cleanup;
 932         }
 933
 934         /* set msg header params */
 935         request->ipsec_opcode = INLINE_IPSEC_OP_SP_DESTROY;
 936         request->req_id = (uint16_t)0xDEADBEEF;
 937
 938         /* set security policy params */
 939         request->ipsec_data.sp_destroy->table_id = is_v4 ?
 940                         VIRTCHNL_IPSEC_INBOUND_SPD_TBL_IPV4 :
 941                         VIRTCHNL_IPSEC_INBOUND_SPD_TBL_IPV6;
 942         request->ipsec_data.sp_destroy->rule_id = flow_id;
 943
 944         /* send virtual channel request to add SA to hardware database */
 945         rc = iavf_ipsec_crypto_request(adapter,
 946                         (uint8_t *)request, request_len,
 947                         (uint8_t *)response, response_len);
 948         if (rc)
 949                 goto update_cleanup;
 950
 951         /* verify response */
 952         if (response->ipsec_opcode != request->ipsec_opcode ||
 953                 response->req_id != request->req_id)
 954                 rc = -EFAULT;
 955         else
 956                 return response->ipsec_data.ipsec_status->status;
 957
 958 update_cleanup:
 959         rte_free(request);
 960         rte_free(response);
 961
 962         return rc;
 963 }
 964
 965 static uint32_t
 966 iavf_ipsec_crypto_sa_del(struct iavf_adapter *adapter,
 967         struct iavf_security_session *sess)
 968 {
 969         struct inline_ipsec_msg *request = NULL, *response = NULL;
 970         size_t request_len, response_len;
 971
 972         int rc = 0;
 973
 974         request_len = sizeof(struct inline_ipsec_msg) +
 975                         sizeof(struct virtchnl_ipsec_sa_destroy);
 976
 977         request = rte_malloc("iavf-sa-del-request", request_len, 0);
 978         if (request == NULL) {
 979                 rc = -ENOMEM;
 980                 goto update_cleanup;
 981         }
 982
 983         response_len = sizeof(struct inline_ipsec_msg) +
 984                         sizeof(struct virtchnl_ipsec_resp);
 985
 986         response = rte_malloc("iavf-sa-del-response", response_len, 0);
 987         if (response == NULL) {
 988                 rc = -ENOMEM;
 989                 goto update_cleanup;
 990         }
 991
 992         /* set msg header params */
 993         request->ipsec_opcode = INLINE_IPSEC_OP_SA_DESTROY;
 994         request->req_id = (uint16_t)0xDEADBEEF;
 995
 996         /**
 997          * SA delete supports deletion of 1-8 specified SA's or if the flag
 998          * field is zero, all SA's associated with VF will be deleted.
 999          */
1000         if (sess) {
1001                 request->ipsec_data.sa_destroy->flag = 0x1;
1002                 request->ipsec_data.sa_destroy->sa_index[0] = sess->sa.hw_idx;
1003         } else {
1004                 request->ipsec_data.sa_destroy->flag = 0x0;
1005         }
1006
1007         /* send virtual channel request to add SA to hardware database */
1008         rc = iavf_ipsec_crypto_request(adapter,
1009                         (uint8_t *)request, request_len,
1010                         (uint8_t *)response, response_len);
1011         if (rc)
1012                 goto update_cleanup;
1013
1014         /* verify response */
1015         if (response->ipsec_opcode != request->ipsec_opcode ||
1016                 response->req_id != request->req_id)
1017                 rc = -EFAULT;
1018
1019         /**
1020          * Delete status will be the same bitmask as sa_destroy request flag if
1021          * deletes successful
1022          */
1023         if (request->ipsec_data.sa_destroy->flag !=
1024                         response->ipsec_data.ipsec_status->status)
1025                 rc = -EFAULT;
1026
1027 update_cleanup:
1028         rte_free(response);
1029         rte_free(request);
1030
1031         return rc;
1032 }
1033
1034 static int
1035 iavf_ipsec_crypto_session_destroy(void *device,
1036                 struct rte_security_session *session)
1037 {
1038         struct iavf_adapter *adapter = NULL;
1039         struct iavf_security_session *iavf_sess = NULL;
1040         struct rte_eth_dev *eth_dev = (struct rte_eth_dev *)device;
1041         int ret;
1042
1043         adapter = IAVF_DEV_PRIVATE_TO_ADAPTER(eth_dev->data->dev_private);
1044         iavf_sess = (struct iavf_security_session *)session->sess_private_data;
1045
1046         /* verify we have a valid session and that it belong to this adapter */
1047         if (unlikely(iavf_sess == NULL || iavf_sess->adapter != adapter))
1048                 return -EINVAL;
1049
1050         ret = iavf_ipsec_crypto_sa_del(adapter, iavf_sess);
1051         rte_mempool_put(rte_mempool_from_obj(iavf_sess), (void *)iavf_sess);
1052         return ret;
1053 }
1054
1055 /**
1056  * Get ESP trailer from packet as well as calculate the total ESP trailer
1057  * length, which include padding, ESP trailer footer and the ICV
1058  */
1059 static inline struct rte_esp_tail *
1060 iavf_ipsec_crypto_get_esp_trailer(struct rte_mbuf *m,
1061         struct iavf_security_session *s, uint16_t *esp_trailer_length)
1062 {
1063         struct rte_esp_tail *esp_trailer;
1064
1065         uint16_t length = sizeof(struct rte_esp_tail) + s->icv_sz;
1066         uint16_t offset = 0;
1067
1068         /**
1069          * The ICV will not be present in TSO packets as this is appended by
1070          * hardware during segment generation
1071          */
1072         if (m->ol_flags & (RTE_MBUF_F_TX_TCP_SEG | RTE_MBUF_F_TX_UDP_SEG))
1073                 length -=  s->icv_sz;
1074
1075         *esp_trailer_length = length;
1076
1077         /**
1078          * Calculate offset in packet to ESP trailer header, this should be
1079          * total packet length less the size of the ESP trailer plus the ICV
1080          * length if it is present
1081          */
1082         offset = rte_pktmbuf_pkt_len(m) - length;
1083
1084         if (m->nb_segs > 1) {
1085                 /* find segment which esp trailer is located */
1086                 while (m->data_len < offset) {
1087                         offset -= m->data_len;
1088                         m = m->next;
1089                 }
1090         }
1091
1092         esp_trailer = rte_pktmbuf_mtod_offset(m, struct rte_esp_tail *, offset);
1093
1094         *esp_trailer_length += esp_trailer->pad_len;
1095
1096         return esp_trailer;
1097 }
1098
1099 static inline uint16_t
1100 iavf_ipsec_crypto_compute_l4_payload_length(struct rte_mbuf *m,
1101         struct iavf_security_session *s, uint16_t esp_tlen)
1102 {
1103         uint16_t ol2_len = m->l2_len;   /* MAC + VLAN */
1104         uint16_t ol3_len = 0;           /* ipv4/6 + ext hdrs */
1105         uint16_t ol4_len = 0;           /* UDP NATT */
1106         uint16_t l3_len = 0;            /* IPv4/6 + ext hdrs */
1107         uint16_t l4_len = 0;            /* TCP/UDP/STCP hdrs */
1108         uint16_t esp_hlen = sizeof(struct rte_esp_hdr) + s->iv_sz;
1109
1110         if (s->mode == RTE_SECURITY_IPSEC_SA_MODE_TUNNEL)
1111                 ol3_len = m->outer_l3_len;
1112                 /**<
1113                  * application provided l3len assumed to include length of
1114                  * ipv4/6 hdr + ext hdrs
1115                  */
1116
1117         if (s->udp_encap.enabled)
1118                 ol4_len = sizeof(struct rte_udp_hdr);
1119
1120         l3_len = m->l3_len;
1121         l4_len = m->l4_len;
1122
1123         return rte_pktmbuf_pkt_len(m) - (ol2_len + ol3_len + ol4_len +
1124                         esp_hlen + l3_len + l4_len + esp_tlen);
1125 }
1126
1127 static int
1128 iavf_ipsec_crypto_pkt_metadata_set(void *device,
1129                          struct rte_security_session *session,
1130                          struct rte_mbuf *m, void *params)
1131 {
1132         struct rte_eth_dev *ethdev = device;
1133         struct iavf_adapter *adapter =
1134                         IAVF_DEV_PRIVATE_TO_ADAPTER(ethdev->data->dev_private);
1135         struct iavf_security_ctx *iavf_sctx = adapter->security_ctx;
1136         struct iavf_security_session *iavf_sess = session->sess_private_data;
1137         struct iavf_ipsec_crypto_pkt_metadata *md;
1138         struct rte_esp_tail *esp_tail;
1139         uint64_t *sqn = params;
1140         uint16_t esp_trailer_length;
1141
1142         /* Check we have valid session and is associated with this device */
1143         if (unlikely(iavf_sess == NULL || iavf_sess->adapter != adapter))
1144                 return -EINVAL;
1145
1146         /* Get dynamic metadata location from mbuf */
1147         md = RTE_MBUF_DYNFIELD(m, iavf_sctx->pkt_md_offset,
1148                 struct iavf_ipsec_crypto_pkt_metadata *);
1149
1150         /* Set immutable metadata values from session template */
1151         memcpy(md, &iavf_sess->pkt_metadata_template,
1152                 sizeof(struct iavf_ipsec_crypto_pkt_metadata));
1153
1154         esp_tail = iavf_ipsec_crypto_get_esp_trailer(m, iavf_sess,
1155                         &esp_trailer_length);
1156
1157         /* Set per packet mutable metadata values */
1158         md->esp_trailer_len = esp_trailer_length;
1159         md->l4_payload_len = iavf_ipsec_crypto_compute_l4_payload_length(m,
1160                                 iavf_sess, esp_trailer_length);
1161         md->next_proto = esp_tail->next_proto;
1162
1163         /* If Extended SN in use set the upper 32-bits in metadata */
1164         if (iavf_sess->esn.enabled && sqn != NULL)
1165                 md->esn = (uint32_t)(*sqn >> 32);
1166
1167         return 0;
1168 }
1169
1170 static int
1171 iavf_ipsec_crypto_device_capabilities_get(struct iavf_adapter *adapter,
1172                 struct virtchnl_ipsec_cap *capability)
1173 {
1174         /* Perform pf-vf comms */
1175         struct inline_ipsec_msg *request = NULL, *response = NULL;
1176         size_t request_len, response_len;
1177         int rc;
1178
1179         request_len = sizeof(struct inline_ipsec_msg);
1180
1181         request = rte_malloc("iavf-device-capability-request", request_len, 0);
1182         if (request == NULL) {
1183                 rc = -ENOMEM;
1184                 goto update_cleanup;
1185         }
1186
1187         response_len = sizeof(struct inline_ipsec_msg) +
1188                         sizeof(struct virtchnl_ipsec_cap);
1189         response = rte_malloc("iavf-device-capability-response",
1190                         response_len, 0);
1191         if (response == NULL) {
1192                 rc = -ENOMEM;
1193                 goto update_cleanup;
1194         }
1195
1196         /* set msg header params */
1197         request->ipsec_opcode = INLINE_IPSEC_OP_GET_CAP;
1198         request->req_id = (uint16_t)0xDEADBEEF;
1199
1200         /* send virtual channel request to add SA to hardware database */
1201         rc = iavf_ipsec_crypto_request(adapter,
1202                         (uint8_t *)request, request_len,
1203                         (uint8_t *)response, response_len);
1204         if (rc)
1205                 goto update_cleanup;
1206
1207         /* verify response id */
1208         if (response->ipsec_opcode != request->ipsec_opcode ||
1209                 response->req_id != request->req_id){
1210                 rc = -EFAULT;
1211                 goto update_cleanup;
1212         }
1213         memcpy(capability, response->ipsec_data.ipsec_cap, sizeof(*capability));
1214
1215 update_cleanup:
1216         rte_free(response);
1217         rte_free(request);
1218
1219         return rc;
1220 }
1221
1222 enum rte_crypto_auth_algorithm auth_maptbl[] = {
1223         /* Hash Algorithm */
1224         [VIRTCHNL_HASH_NO_ALG] = RTE_CRYPTO_AUTH_NULL,
1225         [VIRTCHNL_AES_CBC_MAC] = RTE_CRYPTO_AUTH_AES_CBC_MAC,
1226         [VIRTCHNL_AES_CMAC] = RTE_CRYPTO_AUTH_AES_CMAC,
1227         [VIRTCHNL_AES_GMAC] = RTE_CRYPTO_AUTH_AES_GMAC,
1228         [VIRTCHNL_AES_XCBC_MAC] = RTE_CRYPTO_AUTH_AES_XCBC_MAC,
1229         [VIRTCHNL_MD5_HMAC] = RTE_CRYPTO_AUTH_MD5_HMAC,
1230         [VIRTCHNL_SHA1_HMAC] = RTE_CRYPTO_AUTH_SHA1_HMAC,
1231         [VIRTCHNL_SHA224_HMAC] = RTE_CRYPTO_AUTH_SHA224_HMAC,
1232         [VIRTCHNL_SHA256_HMAC] = RTE_CRYPTO_AUTH_SHA256_HMAC,
1233         [VIRTCHNL_SHA384_HMAC] = RTE_CRYPTO_AUTH_SHA384_HMAC,
1234         [VIRTCHNL_SHA512_HMAC] = RTE_CRYPTO_AUTH_SHA512_HMAC,
1235         [VIRTCHNL_SHA3_224_HMAC] = RTE_CRYPTO_AUTH_SHA3_224_HMAC,
1236         [VIRTCHNL_SHA3_256_HMAC] = RTE_CRYPTO_AUTH_SHA3_256_HMAC,
1237         [VIRTCHNL_SHA3_384_HMAC] = RTE_CRYPTO_AUTH_SHA3_384_HMAC,
1238         [VIRTCHNL_SHA3_512_HMAC] = RTE_CRYPTO_AUTH_SHA3_512_HMAC,
1239 };
1240
1241 static void
1242 update_auth_capabilities(struct rte_cryptodev_capabilities *scap,
1243                 struct virtchnl_algo_cap *acap)
1244 {
1245         struct rte_cryptodev_symmetric_capability *capability = &scap->sym;
1246
1247         scap->op = RTE_CRYPTO_OP_TYPE_SYMMETRIC;
1248
1249         capability->xform_type = RTE_CRYPTO_SYM_XFORM_AUTH;
1250
1251         capability->auth.algo = auth_maptbl[acap->algo_type];
1252         capability->auth.block_size = acap->block_size;
1253
1254         capability->auth.key_size.min = acap->min_key_size;
1255         capability->auth.key_size.max = acap->max_key_size;
1256         capability->auth.key_size.increment = acap->inc_key_size;
1257
1258         capability->auth.digest_size.min = acap->min_digest_size;
1259         capability->auth.digest_size.max = acap->max_digest_size;
1260         capability->auth.digest_size.increment = acap->inc_digest_size;
1261 }
1262
1263 enum rte_crypto_cipher_algorithm cipher_maptbl[] = {
1264         /* Cipher Algorithm */
1265         [VIRTCHNL_CIPHER_NO_ALG] = RTE_CRYPTO_CIPHER_NULL,
1266         [VIRTCHNL_3DES_CBC] = RTE_CRYPTO_CIPHER_3DES_CBC,
1267         [VIRTCHNL_AES_CBC] = RTE_CRYPTO_CIPHER_AES_CBC,
1268         [VIRTCHNL_AES_CTR] = RTE_CRYPTO_CIPHER_AES_CTR,
1269 };
1270
1271 static void
1272 update_cipher_capabilities(struct rte_cryptodev_capabilities *scap,
1273         struct virtchnl_algo_cap *acap)
1274 {
1275         struct rte_cryptodev_symmetric_capability *capability = &scap->sym;
1276
1277         scap->op = RTE_CRYPTO_OP_TYPE_SYMMETRIC;
1278
1279         capability->xform_type = RTE_CRYPTO_SYM_XFORM_CIPHER;
1280
1281         capability->cipher.algo = cipher_maptbl[acap->algo_type];
1282
1283         capability->cipher.block_size = acap->block_size;
1284
1285         capability->cipher.key_size.min = acap->min_key_size;
1286         capability->cipher.key_size.max = acap->max_key_size;
1287         capability->cipher.key_size.increment = acap->inc_key_size;
1288
1289         capability->cipher.iv_size.min = acap->min_iv_size;
1290         capability->cipher.iv_size.max = acap->max_iv_size;
1291         capability->cipher.iv_size.increment = acap->inc_iv_size;
1292 }
1293
1294 enum rte_crypto_aead_algorithm aead_maptbl[] = {
1295         /* AEAD Algorithm */
1296         [VIRTCHNL_AES_CCM] = RTE_CRYPTO_AEAD_AES_CCM,
1297         [VIRTCHNL_AES_GCM] = RTE_CRYPTO_AEAD_AES_GCM,
1298         [VIRTCHNL_CHACHA20_POLY1305] = RTE_CRYPTO_AEAD_CHACHA20_POLY1305,
1299 };
1300
1301 static void
1302 update_aead_capabilities(struct rte_cryptodev_capabilities *scap,
1303         struct virtchnl_algo_cap *acap)
1304 {
1305         struct rte_cryptodev_symmetric_capability *capability = &scap->sym;
1306
1307         scap->op = RTE_CRYPTO_OP_TYPE_SYMMETRIC;
1308
1309         capability->xform_type = RTE_CRYPTO_SYM_XFORM_AEAD;
1310
1311         capability->aead.algo = aead_maptbl[acap->algo_type];
1312
1313         capability->aead.block_size = acap->block_size;
1314
1315         capability->aead.key_size.min = acap->min_key_size;
1316         capability->aead.key_size.max = acap->max_key_size;
1317         capability->aead.key_size.increment = acap->inc_key_size;
1318
1319         capability->aead.aad_size.min = acap->min_aad_size;
1320         capability->aead.aad_size.max = acap->max_aad_size;
1321         capability->aead.aad_size.increment = acap->inc_aad_size;
1322
1323         capability->aead.iv_size.min = acap->min_iv_size;
1324         capability->aead.iv_size.max = acap->max_iv_size;
1325         capability->aead.iv_size.increment = acap->inc_iv_size;
1326
1327         capability->aead.digest_size.min = acap->min_digest_size;
1328         capability->aead.digest_size.max = acap->max_digest_size;
1329         capability->aead.digest_size.increment = acap->inc_digest_size;
1330 }
1331
1332 /**
1333  * Dynamically set crypto capabilities based on virtchannel IPsec
1334  * capabilities structure.
1335  */
1336 int
1337 iavf_ipsec_crypto_set_security_capabililites(struct iavf_security_ctx
1338                 *iavf_sctx, struct virtchnl_ipsec_cap *vch_cap)
1339 {
1340         struct rte_cryptodev_capabilities *capabilities;
1341         int i, j, number_of_capabilities = 0, ci = 0;
1342
1343         /* Count the total number of crypto algorithms supported */
1344         for (i = 0; i < VIRTCHNL_IPSEC_MAX_CRYPTO_CAP_NUM; i++)
1345                 number_of_capabilities += vch_cap->cap[i].algo_cap_num;
1346
1347         /**
1348          * Allocate cryptodev capabilities structure for
1349          * *number_of_capabilities* items plus one item to null terminate the
1350          * array
1351          */
1352         capabilities = rte_zmalloc("crypto_cap",
1353                 sizeof(struct rte_cryptodev_capabilities) *
1354                 (number_of_capabilities + 1), 0);
1355         if (!capabilities)
1356                 return -ENOMEM;
1357         capabilities[number_of_capabilities].op = RTE_CRYPTO_OP_TYPE_UNDEFINED;
1358
1359         /**
1360          * Iterate over each virtchnl crypto capability by crypto type and
1361          * algorithm.
1362          */
1363         for (i = 0; i < VIRTCHNL_IPSEC_MAX_CRYPTO_CAP_NUM; i++) {
1364                 for (j = 0; j < vch_cap->cap[i].algo_cap_num; j++, ci++) {
1365                         switch (vch_cap->cap[i].crypto_type) {
1366                         case VIRTCHNL_AUTH:
1367                                 update_auth_capabilities(&capabilities[ci],
1368                                         &vch_cap->cap[i].algo_cap_list[j]);
1369                                 break;
1370                         case VIRTCHNL_CIPHER:
1371                                 update_cipher_capabilities(&capabilities[ci],
1372                                         &vch_cap->cap[i].algo_cap_list[j]);
1373                                 break;
1374                         case VIRTCHNL_AEAD:
1375                                 update_aead_capabilities(&capabilities[ci],
1376                                         &vch_cap->cap[i].algo_cap_list[j]);
1377                                 break;
1378                         default:
1379                                 capabilities[ci].op =
1380                                                 RTE_CRYPTO_OP_TYPE_UNDEFINED;
1381                                 break;
1382                         }
1383                 }
1384         }
1385
1386         iavf_sctx->crypto_capabilities = capabilities;
1387         return 0;
1388 }
1389
1390 /**
1391  * Get security capabilities for device
1392  */
1393 static const struct rte_security_capability *
1394 iavf_ipsec_crypto_capabilities_get(void *device)
1395 {
1396         struct rte_eth_dev *eth_dev = (struct rte_eth_dev *)device;
1397         struct iavf_adapter *adapter =
1398                 IAVF_DEV_PRIVATE_TO_ADAPTER(eth_dev->data->dev_private);
1399         struct iavf_security_ctx *iavf_sctx = adapter->security_ctx;
1400         unsigned int i;
1401
1402         static struct rte_security_capability iavf_security_capabilities[] = {
1403                 { /* IPsec Inline Crypto ESP Tunnel Egress */
1404                         .action = RTE_SECURITY_ACTION_TYPE_INLINE_CRYPTO,
1405                         .protocol = RTE_SECURITY_PROTOCOL_IPSEC,
1406                         .ipsec = {
1407                                 .proto = RTE_SECURITY_IPSEC_SA_PROTO_ESP,
1408                                 .mode = RTE_SECURITY_IPSEC_SA_MODE_TUNNEL,
1409                                 .direction = RTE_SECURITY_IPSEC_SA_DIR_EGRESS,
1410                                 .options = { .udp_encap = 1,
1411                                                 .stats = 1, .esn = 1 },
1412                         },
1413                         .ol_flags = RTE_SECURITY_TX_OLOAD_NEED_MDATA
1414                 },
1415                 { /* IPsec Inline Crypto ESP Tunnel Ingress */
1416                         .action = RTE_SECURITY_ACTION_TYPE_INLINE_CRYPTO,
1417                         .protocol = RTE_SECURITY_PROTOCOL_IPSEC,
1418                         .ipsec = {
1419                                 .proto = RTE_SECURITY_IPSEC_SA_PROTO_ESP,
1420                                 .mode = RTE_SECURITY_IPSEC_SA_MODE_TUNNEL,
1421                                 .direction = RTE_SECURITY_IPSEC_SA_DIR_INGRESS,
1422                                 .options = { .udp_encap = 1,
1423                                                 .stats = 1, .esn = 1 },
1424                         },
1425                         .ol_flags = 0
1426                 },
1427                 { /* IPsec Inline Crypto ESP Transport Egress */
1428                         .action = RTE_SECURITY_ACTION_TYPE_INLINE_CRYPTO,
1429                         .protocol = RTE_SECURITY_PROTOCOL_IPSEC,
1430                         .ipsec = {
1431                                 .proto = RTE_SECURITY_IPSEC_SA_PROTO_ESP,
1432                                 .mode = RTE_SECURITY_IPSEC_SA_MODE_TRANSPORT,
1433                                 .direction = RTE_SECURITY_IPSEC_SA_DIR_EGRESS,
1434                                 .options = { .udp_encap = 1, .stats = 1,
1435                                                 .esn = 1 },
1436                         },
1437                         .ol_flags = RTE_SECURITY_TX_OLOAD_NEED_MDATA
1438                 },
1439                 { /* IPsec Inline Crypto ESP Transport Ingress */
1440                         .action = RTE_SECURITY_ACTION_TYPE_INLINE_CRYPTO,
1441                         .protocol = RTE_SECURITY_PROTOCOL_IPSEC,
1442                         .ipsec = {
1443                                 .proto = RTE_SECURITY_IPSEC_SA_PROTO_ESP,
1444                                 .mode = RTE_SECURITY_IPSEC_SA_MODE_TRANSPORT,
1445                                 .direction = RTE_SECURITY_IPSEC_SA_DIR_INGRESS,
1446                                 .options = { .udp_encap = 1, .stats = 1,
1447                                                 .esn = 1 }
1448                         },
1449                         .ol_flags = 0
1450                 },
1451                 {
1452                         .action = RTE_SECURITY_ACTION_TYPE_NONE
1453                 }
1454         };
1455
1456         /**
1457          * Update the security capabilities struct with the runtime discovered
1458          * crypto capabilities, except for last element of the array which is
1459          * the null termination
1460          */
1461         for (i = 0; i < ((sizeof(iavf_security_capabilities) /
1462                         sizeof(iavf_security_capabilities[0])) - 1); i++) {
1463                 iavf_security_capabilities[i].crypto_capabilities =
1464                         iavf_sctx->crypto_capabilities;
1465         }
1466
1467         return iavf_security_capabilities;
1468 }
1469
1470 static struct rte_security_ops iavf_ipsec_crypto_ops = {
1471         .session_get_size               = iavf_ipsec_crypto_session_size_get,
1472         .session_create                 = iavf_ipsec_crypto_session_create,
1473         .session_update                 = iavf_ipsec_crypto_session_update,
1474         .session_stats_get              = iavf_ipsec_crypto_session_stats_get,
1475         .session_destroy                = iavf_ipsec_crypto_session_destroy,
1476         .set_pkt_metadata               = iavf_ipsec_crypto_pkt_metadata_set,
1477         .get_userdata                   = NULL,
1478         .capabilities_get               = iavf_ipsec_crypto_capabilities_get,
1479 };
1480
1481 int
1482 iavf_security_ctx_create(struct iavf_adapter *adapter)
1483 {
1484         struct rte_security_ctx *sctx;
1485
1486         sctx = rte_malloc("security_ctx", sizeof(struct rte_security_ctx), 0);
1487         if (sctx == NULL)
1488                 return -ENOMEM;
1489
1490         sctx->device = adapter->vf.eth_dev;
1491         sctx->ops = &iavf_ipsec_crypto_ops;
1492         sctx->sess_cnt = 0;
1493
1494         adapter->vf.eth_dev->security_ctx = sctx;
1495
1496         if (adapter->security_ctx == NULL) {
1497                 adapter->security_ctx = rte_malloc("iavf_security_ctx",
1498                                 sizeof(struct iavf_security_ctx), 0);
1499                 if (adapter->security_ctx == NULL)
1500                         return -ENOMEM;
1501         }
1502
1503         return 0;
1504 }
1505
1506 int
1507 iavf_security_init(struct iavf_adapter *adapter)
1508 {
1509         struct iavf_security_ctx *iavf_sctx = adapter->security_ctx;
1510         struct rte_mbuf_dynfield pkt_md_dynfield = {
1511                 .name = "iavf_ipsec_crypto_pkt_metadata",
1512                 .size = sizeof(struct iavf_ipsec_crypto_pkt_metadata),
1513                 .align = __alignof__(struct iavf_ipsec_crypto_pkt_metadata)
1514         };
1515         struct virtchnl_ipsec_cap capabilities;
1516         int rc;
1517
1518         iavf_sctx->adapter = adapter;
1519
1520         iavf_sctx->pkt_md_offset = rte_mbuf_dynfield_register(&pkt_md_dynfield);
1521         if (iavf_sctx->pkt_md_offset < 0)
1522                 return iavf_sctx->pkt_md_offset;
1523
1524         /* Get device capabilities from Inline IPsec driver over PF-VF comms */
1525         rc = iavf_ipsec_crypto_device_capabilities_get(adapter, &capabilities);
1526         if (rc)
1527                 return rc;
1528
1529         return  iavf_ipsec_crypto_set_security_capabililites(iavf_sctx,
1530                         &capabilities);
1531 }
1532
1533 int
1534 iavf_security_get_pkt_md_offset(struct iavf_adapter *adapter)
1535 {
1536         struct iavf_security_ctx *iavf_sctx = adapter->security_ctx;
1537
1538         return iavf_sctx->pkt_md_offset;
1539 }
1540
1541 int
1542 iavf_security_ctx_destroy(struct iavf_adapter *adapter)
1543 {
1544         struct rte_security_ctx *sctx  = adapter->vf.eth_dev->security_ctx;
1545         struct iavf_security_ctx *iavf_sctx = adapter->security_ctx;
1546
1547         if (iavf_sctx == NULL)
1548                 return -ENODEV;
1549
1550         /* TODO: Add resources cleanup */
1551
1552         /* free and reset security data structures */
1553         rte_free(iavf_sctx);
1554         rte_free(sctx);
1555
1556         adapter->security_ctx = NULL;
1557         adapter->vf.eth_dev->security_ctx = NULL;
1558
1559         return 0;
1560 }
1561
1562 int
1563 iavf_ipsec_crypto_supported(struct iavf_adapter *adapter)
1564 {
1565         struct virtchnl_vf_resource *resources = adapter->vf.vf_res;
1566
1567         /** Capability check for IPsec Crypto */
1568         if (resources && (resources->vf_cap_flags &
1569                 VIRTCHNL_VF_OFFLOAD_INLINE_IPSEC_CRYPTO))
1570                 return true;
1571
1572         return false;
1573 }
1574
1575 #define IAVF_IPSEC_INSET_ESP (\
1576         IAVF_INSET_ESP_SPI)
1577
1578 #define IAVF_IPSEC_INSET_AH (\
1579         IAVF_INSET_AH_SPI)
1580
1581 #define IAVF_IPSEC_INSET_IPV4_NATT_ESP (\
1582         IAVF_INSET_IPV4_SRC | IAVF_INSET_IPV4_DST | \
1583         IAVF_INSET_ESP_SPI)
1584
1585 #define IAVF_IPSEC_INSET_IPV6_NATT_ESP (\
1586         IAVF_INSET_IPV6_SRC | IAVF_INSET_IPV6_DST | \
1587         IAVF_INSET_ESP_SPI)
1588
1589 enum iavf_ipsec_flow_pt_type {
1590         IAVF_PATTERN_ESP = 1,
1591         IAVF_PATTERN_AH,
1592         IAVF_PATTERN_UDP_ESP,
1593 };
1594 enum iavf_ipsec_flow_pt_ip_ver {
1595         IAVF_PATTERN_IPV4 = 1,
1596         IAVF_PATTERN_IPV6,
1597 };
1598
1599 #define IAVF_PATTERN(t, ipt) ((void *)((t) | ((ipt) << 4)))
1600 #define IAVF_PATTERN_TYPE(pt) ((pt) & 0x0F)
1601 #define IAVF_PATTERN_IP_V(pt) ((pt) >> 4)
1602
1603 static struct iavf_pattern_match_item iavf_ipsec_flow_pattern[] = {
1604         {iavf_pattern_eth_ipv4_esp,     IAVF_IPSEC_INSET_ESP,
1605                         IAVF_PATTERN(IAVF_PATTERN_ESP, IAVF_PATTERN_IPV4)},
1606         {iavf_pattern_eth_ipv6_esp,     IAVF_IPSEC_INSET_ESP,
1607                         IAVF_PATTERN(IAVF_PATTERN_ESP, IAVF_PATTERN_IPV6)},
1608         {iavf_pattern_eth_ipv4_ah,      IAVF_IPSEC_INSET_AH,
1609                         IAVF_PATTERN(IAVF_PATTERN_AH, IAVF_PATTERN_IPV4)},
1610         {iavf_pattern_eth_ipv6_ah,      IAVF_IPSEC_INSET_AH,
1611                         IAVF_PATTERN(IAVF_PATTERN_AH, IAVF_PATTERN_IPV6)},
1612         {iavf_pattern_eth_ipv4_udp_esp, IAVF_IPSEC_INSET_IPV4_NATT_ESP,
1613                         IAVF_PATTERN(IAVF_PATTERN_UDP_ESP, IAVF_PATTERN_IPV4)},
1614         {iavf_pattern_eth_ipv6_udp_esp, IAVF_IPSEC_INSET_IPV6_NATT_ESP,
1615                         IAVF_PATTERN(IAVF_PATTERN_UDP_ESP, IAVF_PATTERN_IPV6)},
1616 };
1617
1618 struct iavf_ipsec_flow_item {
1619         uint64_t id;
1620         uint8_t is_ipv4;
1621         uint32_t spi;
1622         struct rte_ether_hdr eth_hdr;
1623         union {
1624                 struct rte_ipv4_hdr ipv4_hdr;
1625                 struct rte_ipv6_hdr ipv6_hdr;
1626         };
1627         struct rte_udp_hdr udp_hdr;
1628 };
1629
1630 static void
1631 parse_eth_item(const struct rte_flow_item_eth *item,
1632                 struct rte_ether_hdr *eth)
1633 {
1634         memcpy(eth->src_addr.addr_bytes,
1635                         item->src.addr_bytes, sizeof(eth->src_addr));
1636         memcpy(eth->dst_addr.addr_bytes,
1637                         item->dst.addr_bytes, sizeof(eth->dst_addr));
1638 }
1639
1640 static void
1641 parse_ipv4_item(const struct rte_flow_item_ipv4 *item,
1642                 struct rte_ipv4_hdr *ipv4)
1643 {
1644         ipv4->src_addr = item->hdr.src_addr;
1645         ipv4->dst_addr = item->hdr.dst_addr;
1646 }
1647
1648 static void
1649 parse_ipv6_item(const struct rte_flow_item_ipv6 *item,
1650                 struct rte_ipv6_hdr *ipv6)
1651 {
1652         memcpy(ipv6->src_addr, item->hdr.src_addr, 16);
1653         memcpy(ipv6->dst_addr, item->hdr.dst_addr, 16);
1654 }
1655
1656 static void
1657 parse_udp_item(const struct rte_flow_item_udp *item, struct rte_udp_hdr *udp)
1658 {
1659         udp->dst_port = item->hdr.dst_port;
1660         udp->src_port = item->hdr.src_port;
1661 }
1662
1663 static int
1664 has_security_action(const struct rte_flow_action actions[],
1665         const void **session)
1666 {
1667         /* only {SECURITY; END} supported */
1668         if (actions[0].type == RTE_FLOW_ACTION_TYPE_SECURITY &&
1669                 actions[1].type == RTE_FLOW_ACTION_TYPE_END) {
1670                 *session = actions[0].conf;
1671                 return true;
1672         }
1673         return false;
1674 }
1675
1676 static struct iavf_ipsec_flow_item *
1677 iavf_ipsec_flow_item_parse(struct rte_eth_dev *ethdev,
1678                 const struct rte_flow_item pattern[],
1679                 const struct rte_flow_action actions[],
1680                 uint32_t type)
1681 {
1682         const void *session;
1683         struct iavf_ipsec_flow_item
1684                 *ipsec_flow = rte_malloc("security-flow-rule",
1685                 sizeof(struct iavf_ipsec_flow_item), 0);
1686         enum iavf_ipsec_flow_pt_type p_type = IAVF_PATTERN_TYPE(type);
1687         enum iavf_ipsec_flow_pt_ip_ver p_ip_type = IAVF_PATTERN_IP_V(type);
1688
1689         if (ipsec_flow == NULL)
1690                 return NULL;
1691
1692         ipsec_flow->is_ipv4 = (p_ip_type == IAVF_PATTERN_IPV4);
1693
1694         if (pattern[0].spec)
1695                 parse_eth_item((const struct rte_flow_item_eth *)
1696                                 pattern[0].spec, &ipsec_flow->eth_hdr);
1697
1698         switch (p_type) {
1699         case IAVF_PATTERN_ESP:
1700                 if (ipsec_flow->is_ipv4) {
1701                         parse_ipv4_item((const struct rte_flow_item_ipv4 *)
1702                                         pattern[1].spec,
1703                                         &ipsec_flow->ipv4_hdr);
1704                 } else {
1705                         parse_ipv6_item((const struct rte_flow_item_ipv6 *)
1706                                         pattern[1].spec,
1707                                         &ipsec_flow->ipv6_hdr);
1708                 }
1709                 ipsec_flow->spi =
1710                         ((const struct rte_flow_item_esp *)
1711                                         pattern[2].spec)->hdr.spi;
1712                 break;
1713         case IAVF_PATTERN_AH:
1714                 if (ipsec_flow->is_ipv4) {
1715                         parse_ipv4_item((const struct rte_flow_item_ipv4 *)
1716                                         pattern[1].spec,
1717                                         &ipsec_flow->ipv4_hdr);
1718                 } else {
1719                         parse_ipv6_item((const struct rte_flow_item_ipv6 *)
1720                                         pattern[1].spec,
1721                                         &ipsec_flow->ipv6_hdr);
1722                 }
1723                 ipsec_flow->spi =
1724                         ((const struct rte_flow_item_ah *)
1725                                         pattern[2].spec)->spi;
1726                 break;
1727         case IAVF_PATTERN_UDP_ESP:
1728                 if (ipsec_flow->is_ipv4) {
1729                         parse_ipv4_item((const struct rte_flow_item_ipv4 *)
1730                                         pattern[1].spec,
1731                                         &ipsec_flow->ipv4_hdr);
1732                 } else {
1733                         parse_ipv6_item((const struct rte_flow_item_ipv6 *)
1734                                         pattern[1].spec,
1735                                         &ipsec_flow->ipv6_hdr);
1736                 }
1737                 parse_udp_item((const struct rte_flow_item_udp *)
1738                                 pattern[2].spec,
1739                         &ipsec_flow->udp_hdr);
1740                 ipsec_flow->spi =
1741                         ((const struct rte_flow_item_esp *)
1742                                         pattern[3].spec)->hdr.spi;
1743                 break;
1744         default:
1745                 goto flow_cleanup;
1746         }
1747
1748         if (!has_security_action(actions, &session))
1749                 goto flow_cleanup;
1750
1751         if (!iavf_ipsec_crypto_action_valid(ethdev, session,
1752                         ipsec_flow->spi))
1753                 goto flow_cleanup;
1754
1755         return ipsec_flow;
1756
1757 flow_cleanup:
1758         rte_free(ipsec_flow);
1759         return NULL;
1760 }
1761
1762
1763 static struct iavf_flow_parser iavf_ipsec_flow_parser;
1764
1765 static int
1766 iavf_ipsec_flow_init(struct iavf_adapter *ad)
1767 {
1768         struct iavf_info *vf = IAVF_DEV_PRIVATE_TO_VF(ad);
1769         struct iavf_flow_parser *parser;
1770
1771         if (!vf->vf_res)
1772                 return -EINVAL;
1773
1774         if (vf->vf_res->vf_cap_flags & VIRTCHNL_VF_OFFLOAD_INLINE_IPSEC_CRYPTO)
1775                 parser = &iavf_ipsec_flow_parser;
1776         else
1777                 return -ENOTSUP;
1778
1779         return iavf_register_parser(parser, ad);
1780 }
1781
1782 static void
1783 iavf_ipsec_flow_uninit(struct iavf_adapter *ad)
1784 {
1785         iavf_unregister_parser(&iavf_ipsec_flow_parser, ad);
1786 }
1787
1788 static int
1789 iavf_ipsec_flow_create(struct iavf_adapter *ad,
1790                 struct rte_flow *flow,
1791                 void *meta,
1792                 struct rte_flow_error *error)
1793 {
1794         struct iavf_ipsec_flow_item *ipsec_flow = meta;
1795         if (!ipsec_flow) {
1796                 rte_flow_error_set(error, EINVAL,
1797                                 RTE_FLOW_ERROR_TYPE_HANDLE, NULL,
1798                                 "NULL rule.");
1799                 return -rte_errno;
1800         }
1801
1802         if (ipsec_flow->is_ipv4) {
1803                 ipsec_flow->id =
1804                         iavf_ipsec_crypto_inbound_security_policy_add(ad,
1805                         ipsec_flow->spi,
1806                         1,
1807                         ipsec_flow->ipv4_hdr.dst_addr,
1808                         NULL,
1809                         0);
1810         } else {
1811                 ipsec_flow->id =
1812                         iavf_ipsec_crypto_inbound_security_policy_add(ad,
1813                         ipsec_flow->spi,
1814                         0,
1815                         0,
1816                         ipsec_flow->ipv6_hdr.dst_addr,
1817                         0);
1818         }
1819
1820         if (ipsec_flow->id < 1) {
1821                 rte_flow_error_set(error, EINVAL,
1822                                 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
1823                                 "Failed to add SA.");
1824                 return -rte_errno;
1825         }
1826
1827         flow->rule = ipsec_flow;
1828
1829         return 0;
1830 }
1831
1832 static int
1833 iavf_ipsec_flow_destroy(struct iavf_adapter *ad,
1834                 struct rte_flow *flow,
1835                 struct rte_flow_error *error)
1836 {
1837         struct iavf_ipsec_flow_item *ipsec_flow = flow->rule;
1838         if (!ipsec_flow) {
1839                 rte_flow_error_set(error, EINVAL,
1840                                 RTE_FLOW_ERROR_TYPE_HANDLE, NULL,
1841                                 "NULL rule.");
1842                 return -rte_errno;
1843         }
1844
1845         iavf_ipsec_crypto_security_policy_delete(ad,
1846                         ipsec_flow->is_ipv4, ipsec_flow->id);
1847         rte_free(ipsec_flow);
1848         return 0;
1849 }
1850
1851 static struct iavf_flow_engine iavf_ipsec_flow_engine = {
1852         .init = iavf_ipsec_flow_init,
1853         .uninit = iavf_ipsec_flow_uninit,
1854         .create = iavf_ipsec_flow_create,
1855         .destroy = iavf_ipsec_flow_destroy,
1856         .type = IAVF_FLOW_ENGINE_IPSEC_CRYPTO,
1857 };
1858
1859 static int
1860 iavf_ipsec_flow_parse(struct iavf_adapter *ad,
1861                        struct iavf_pattern_match_item *array,
1862                        uint32_t array_len,
1863                        const struct rte_flow_item pattern[],
1864                        const struct rte_flow_action actions[],
1865                        void **meta,
1866                        struct rte_flow_error *error)
1867 {
1868         struct iavf_pattern_match_item *item = NULL;
1869         int ret = -1;
1870
1871         item = iavf_search_pattern_match_item(pattern, array, array_len, error);
1872         if (item && item->meta) {
1873                 uint32_t type = (uint64_t)(item->meta);
1874                 struct iavf_ipsec_flow_item *fi =
1875                                 iavf_ipsec_flow_item_parse(ad->vf.eth_dev,
1876                                                 pattern, actions, type);
1877                 if (fi && meta) {
1878                         *meta = fi;
1879                         ret = 0;
1880                 }
1881         }
1882         return ret;
1883 }
1884
1885 static struct iavf_flow_parser iavf_ipsec_flow_parser = {
1886         .engine = &iavf_ipsec_flow_engine,
1887         .array = iavf_ipsec_flow_pattern,
1888         .array_len = RTE_DIM(iavf_ipsec_flow_pattern),
1889         .parse_pattern_action = iavf_ipsec_flow_parse,
1890         .stage = IAVF_FLOW_STAGE_IPSEC_CRYPTO,
1891 };
1892
1893 RTE_INIT(iavf_ipsec_flow_engine_register)
1894 {
1895         iavf_register_flow_engine(&iavf_ipsec_flow_engine);
1896 }