drivers/net/iavf/iavf_ipsec_crypto.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(c) 2020 Intel Corporation
   3  */
   4
   5 #include <rte_cryptodev.h>
   6 #include <rte_ethdev.h>
   7 #include <rte_security_driver.h>
   8 #include <rte_security.h>
   9
  10 #include "iavf.h"
  11 #include "iavf_rxtx.h"
  12 #include "iavf_log.h"
  13 #include "iavf_generic_flow.h"
  14
  15 #include "iavf_ipsec_crypto.h"
  16 #include "iavf_ipsec_crypto_capabilities.h"
  17
  18 /**
  19  * iAVF IPsec Crypto Security Context
  20  */
  21 struct iavf_security_ctx {
  22         struct iavf_adapter *adapter;
  23         int pkt_md_offset;
  24         struct rte_cryptodev_capabilities *crypto_capabilities;
  25 };
  26
  27 /**
  28  * iAVF IPsec Crypto Security Session Parameters
  29  */
  30 struct iavf_security_session {
  31         struct iavf_adapter *adapter;
  32
  33         enum rte_security_ipsec_sa_mode mode;
  34         enum rte_security_ipsec_tunnel_type type;
  35         enum rte_security_ipsec_sa_direction direction;
  36
  37         struct {
  38                 uint32_t spi; /* Security Parameter Index */
  39                 uint32_t hw_idx; /* SA Index in hardware table */
  40         } sa;
  41
  42         struct {
  43                 uint8_t enabled :1;
  44                 union {
  45                         uint64_t value;
  46                         struct {
  47                                 uint32_t hi;
  48                                 uint32_t low;
  49                         };
  50                 };
  51         } esn;
  52
  53         struct {
  54                 uint8_t enabled :1;
  55         } udp_encap;
  56
  57         size_t iv_sz;
  58         size_t icv_sz;
  59         size_t block_sz;
  60
  61         struct iavf_ipsec_crypto_pkt_metadata pkt_metadata_template;
  62 };
  63 /**
  64  *  IV Length field in IPsec Tx Desc uses the following encoding:
  65  *
  66  *  0B - 0
  67  *  4B - 1
  68  *  8B - 2
  69  *  16B - 3
  70  *
  71  * but we also need the IV Length for TSO to correctly calculate the total
  72  * header length so placing it in the upper 6-bits here for easier reterival.
  73  */
  74 static inline uint8_t
  75 calc_ipsec_desc_iv_len_field(uint16_t iv_sz)
  76 {
  77         uint8_t iv_length = IAVF_IPSEC_IV_LEN_NONE;
  78
  79         switch (iv_sz) {
  80         case 4:
  81                 iv_length = IAVF_IPSEC_IV_LEN_DW;
  82                 break;
  83         case 8:
  84                 iv_length = IAVF_IPSEC_IV_LEN_DDW;
  85                 break;
  86         case 16:
  87                 iv_length = IAVF_IPSEC_IV_LEN_QDW;
  88                 break;
  89         }
  90
  91         return (iv_sz << 2) | iv_length;
  92 }
  93
  94 static unsigned int
  95 iavf_ipsec_crypto_session_size_get(void *device __rte_unused)
  96 {
  97         return sizeof(struct iavf_security_session);
  98 }
  99
 100 static const struct rte_cryptodev_symmetric_capability *
 101 get_capability(struct iavf_security_ctx *iavf_sctx,
 102         uint32_t algo, uint32_t type)
 103 {
 104         const struct rte_cryptodev_capabilities *capability;
 105         int i = 0;
 106
 107         capability = &iavf_sctx->crypto_capabilities[i];
 108
 109         while (capability->op != RTE_CRYPTO_OP_TYPE_UNDEFINED) {
 110                 if (capability->op == RTE_CRYPTO_OP_TYPE_SYMMETRIC &&
 111                         (uint32_t)capability->sym.xform_type == type &&
 112                         (uint32_t)capability->sym.cipher.algo == algo)
 113                         return &capability->sym;
 114                 /** try next capability */
 115                 capability = &iavf_crypto_capabilities[i++];
 116         }
 117
 118         return NULL;
 119 }
 120
 121 static const struct rte_cryptodev_symmetric_capability *
 122 get_auth_capability(struct iavf_security_ctx *iavf_sctx,
 123         enum rte_crypto_auth_algorithm algo)
 124 {
 125         return get_capability(iavf_sctx, algo, RTE_CRYPTO_SYM_XFORM_AUTH);
 126 }
 127
 128 static const struct rte_cryptodev_symmetric_capability *
 129 get_cipher_capability(struct iavf_security_ctx *iavf_sctx,
 130         enum rte_crypto_cipher_algorithm algo)
 131 {
 132         return get_capability(iavf_sctx, algo, RTE_CRYPTO_SYM_XFORM_CIPHER);
 133 }
 134 static const struct rte_cryptodev_symmetric_capability *
 135 get_aead_capability(struct iavf_security_ctx *iavf_sctx,
 136         enum rte_crypto_aead_algorithm algo)
 137 {
 138         return get_capability(iavf_sctx, algo, RTE_CRYPTO_SYM_XFORM_AEAD);
 139 }
 140
 141 static uint16_t
 142 get_cipher_blocksize(struct iavf_security_ctx *iavf_sctx,
 143         enum rte_crypto_cipher_algorithm algo)
 144 {
 145         const struct rte_cryptodev_symmetric_capability *capability;
 146
 147         capability = get_cipher_capability(iavf_sctx, algo);
 148         if (capability == NULL)
 149                 return 0;
 150
 151         return capability->cipher.block_size;
 152 }
 153
 154 static uint16_t
 155 get_aead_blocksize(struct iavf_security_ctx *iavf_sctx,
 156         enum rte_crypto_aead_algorithm algo)
 157 {
 158         const struct rte_cryptodev_symmetric_capability *capability;
 159
 160         capability = get_aead_capability(iavf_sctx, algo);
 161         if (capability == NULL)
 162                 return 0;
 163
 164         return capability->cipher.block_size;
 165 }
 166
 167 static uint16_t
 168 get_auth_blocksize(struct iavf_security_ctx *iavf_sctx,
 169         enum rte_crypto_auth_algorithm algo)
 170 {
 171         const struct rte_cryptodev_symmetric_capability *capability;
 172
 173         capability = get_auth_capability(iavf_sctx, algo);
 174         if (capability == NULL)
 175                 return 0;
 176
 177         return capability->auth.block_size;
 178 }
 179
 180 static uint8_t
 181 calc_context_desc_cipherblock_sz(size_t len)
 182 {
 183         switch (len) {
 184         case 8:
 185                 return 0x2;
 186         case 16:
 187                 return 0x3;
 188         default:
 189                 return 0x0;
 190         }
 191 }
 192
 193 static int
 194 valid_length(uint32_t len, uint32_t min, uint32_t max, uint32_t increment)
 195 {
 196         if (len < min || len > max)
 197                 return false;
 198
 199         if (increment == 0)
 200                 return true;
 201
 202         if ((len - min) % increment)
 203                 return false;
 204
 205         /* make sure it fits in the key array */
 206         if (len > VIRTCHNL_IPSEC_MAX_KEY_LEN)
 207                 return false;
 208
 209         return true;
 210 }
 211
 212 static int
 213 valid_auth_xform(struct iavf_security_ctx *iavf_sctx,
 214         struct rte_crypto_auth_xform *auth)
 215 {
 216         const struct rte_cryptodev_symmetric_capability *capability;
 217
 218         capability = get_auth_capability(iavf_sctx, auth->algo);
 219         if (capability == NULL)
 220                 return false;
 221
 222         /* verify key size */
 223         if (!valid_length(auth->key.length,
 224                 capability->auth.key_size.min,
 225                 capability->auth.key_size.max,
 226                 capability->aead.key_size.increment))
 227                 return false;
 228
 229         return true;
 230 }
 231
 232 static int
 233 valid_cipher_xform(struct iavf_security_ctx *iavf_sctx,
 234         struct rte_crypto_cipher_xform *cipher)
 235 {
 236         const struct rte_cryptodev_symmetric_capability *capability;
 237
 238         capability = get_cipher_capability(iavf_sctx, cipher->algo);
 239         if (capability == NULL)
 240                 return false;
 241
 242         /* verify key size */
 243         if (!valid_length(cipher->key.length,
 244                 capability->cipher.key_size.min,
 245                 capability->cipher.key_size.max,
 246                 capability->cipher.key_size.increment))
 247                 return false;
 248
 249         return true;
 250 }
 251
 252 static int
 253 valid_aead_xform(struct iavf_security_ctx *iavf_sctx,
 254         struct rte_crypto_aead_xform *aead)
 255 {
 256         const struct rte_cryptodev_symmetric_capability *capability;
 257
 258         capability = get_aead_capability(iavf_sctx, aead->algo);
 259         if (capability == NULL)
 260                 return false;
 261
 262         /* verify key size */
 263         if (!valid_length(aead->key.length,
 264                 capability->aead.key_size.min,
 265                 capability->aead.key_size.max,
 266                 capability->aead.key_size.increment))
 267                 return false;
 268
 269         return true;
 270 }
 271
 272 static int
 273 iavf_ipsec_crypto_session_validate_conf(struct iavf_security_ctx *iavf_sctx,
 274         struct rte_security_session_conf *conf)
 275 {
 276         /** validate security action/protocol selection */
 277         if (conf->action_type != RTE_SECURITY_ACTION_TYPE_INLINE_CRYPTO ||
 278                 conf->protocol != RTE_SECURITY_PROTOCOL_IPSEC) {
 279                 PMD_DRV_LOG(ERR, "Invalid action / protocol specified");
 280                 return -EINVAL;
 281         }
 282
 283         /** validate IPsec protocol selection */
 284         if (conf->ipsec.proto != RTE_SECURITY_IPSEC_SA_PROTO_ESP) {
 285                 PMD_DRV_LOG(ERR, "Invalid IPsec protocol specified");
 286                 return -EINVAL;
 287         }
 288
 289         /** validate selected options */
 290         if (conf->ipsec.options.copy_dscp ||
 291                 conf->ipsec.options.copy_flabel ||
 292                 conf->ipsec.options.copy_df ||
 293                 conf->ipsec.options.dec_ttl ||
 294                 conf->ipsec.options.ecn ||
 295                 conf->ipsec.options.stats) {
 296                 PMD_DRV_LOG(ERR, "Invalid IPsec option specified");
 297                 return -EINVAL;
 298         }
 299
 300         /**
 301          * Validate crypto xforms parameters.
 302          *
 303          * AEAD transforms can be used for either inbound/outbound IPsec SAs,
 304          * for non-AEAD crypto transforms we explicitly only support CIPHER/AUTH
 305          * for outbound and AUTH/CIPHER chained transforms for inbound IPsec.
 306          */
 307         if (conf->crypto_xform->type == RTE_CRYPTO_SYM_XFORM_AEAD) {
 308                 if (!valid_aead_xform(iavf_sctx, &conf->crypto_xform->aead)) {
 309                         PMD_DRV_LOG(ERR, "Invalid IPsec option specified");
 310                         return -EINVAL;
 311                 }
 312         } else if (conf->ipsec.direction == RTE_SECURITY_IPSEC_SA_DIR_EGRESS &&
 313                 conf->crypto_xform->type == RTE_CRYPTO_SYM_XFORM_CIPHER &&
 314                 conf->crypto_xform->next &&
 315                 conf->crypto_xform->next->type == RTE_CRYPTO_SYM_XFORM_AUTH) {
 316                 if (!valid_cipher_xform(iavf_sctx,
 317                                 &conf->crypto_xform->cipher)) {
 318                         PMD_DRV_LOG(ERR, "Invalid IPsec option specified");
 319                         return -EINVAL;
 320                 }
 321
 322                 if (!valid_auth_xform(iavf_sctx,
 323                                 &conf->crypto_xform->next->auth)) {
 324                         PMD_DRV_LOG(ERR, "Invalid IPsec option specified");
 325                         return -EINVAL;
 326                 }
 327         } else if (conf->ipsec.direction == RTE_SECURITY_IPSEC_SA_DIR_INGRESS &&
 328                 conf->crypto_xform->type == RTE_CRYPTO_SYM_XFORM_AUTH &&
 329                 conf->crypto_xform->next &&
 330                 conf->crypto_xform->next->type == RTE_CRYPTO_SYM_XFORM_CIPHER) {
 331                 if (!valid_auth_xform(iavf_sctx, &conf->crypto_xform->auth)) {
 332                         PMD_DRV_LOG(ERR, "Invalid IPsec option specified");
 333                         return -EINVAL;
 334                 }
 335
 336                 if (!valid_cipher_xform(iavf_sctx,
 337                                 &conf->crypto_xform->next->cipher)) {
 338                         PMD_DRV_LOG(ERR, "Invalid IPsec option specified");
 339                         return -EINVAL;
 340                 }
 341         }
 342
 343         return 0;
 344 }
 345
 346 static void
 347 sa_add_set_aead_params(struct virtchnl_ipsec_crypto_cfg_item *cfg,
 348         struct rte_crypto_aead_xform *aead, uint32_t salt)
 349 {
 350         cfg->crypto_type = VIRTCHNL_AEAD;
 351
 352         switch (aead->algo) {
 353         case RTE_CRYPTO_AEAD_AES_CCM:
 354                 cfg->algo_type = VIRTCHNL_AES_CCM; break;
 355         case RTE_CRYPTO_AEAD_AES_GCM:
 356                 cfg->algo_type = VIRTCHNL_AES_GCM; break;
 357         case RTE_CRYPTO_AEAD_CHACHA20_POLY1305:
 358                 cfg->algo_type = VIRTCHNL_CHACHA20_POLY1305; break;
 359         default:
 360                 PMD_DRV_LOG(ERR, "Invalid AEAD parameters");
 361                 break;
 362         }
 363
 364         cfg->key_len = aead->key.length;
 365         cfg->iv_len = sizeof(uint64_t); /* iv.length includes salt len */
 366         cfg->digest_len = aead->digest_length;
 367         cfg->salt = salt;
 368
 369         memcpy(cfg->key_data, aead->key.data, cfg->key_len);
 370 }
 371
 372 static void
 373 sa_add_set_cipher_params(struct virtchnl_ipsec_crypto_cfg_item *cfg,
 374         struct rte_crypto_cipher_xform *cipher, uint32_t salt)
 375 {
 376         cfg->crypto_type = VIRTCHNL_CIPHER;
 377
 378         switch (cipher->algo) {
 379         case RTE_CRYPTO_CIPHER_AES_CBC:
 380                 cfg->algo_type = VIRTCHNL_AES_CBC; break;
 381         case RTE_CRYPTO_CIPHER_3DES_CBC:
 382                 cfg->algo_type = VIRTCHNL_3DES_CBC; break;
 383         case RTE_CRYPTO_CIPHER_NULL:
 384                 cfg->algo_type = VIRTCHNL_CIPHER_NO_ALG; break;
 385         case RTE_CRYPTO_CIPHER_AES_CTR:
 386                 cfg->algo_type = VIRTCHNL_AES_CTR;
 387                 cfg->salt = salt;
 388                 break;
 389         default:
 390                 PMD_DRV_LOG(ERR, "Invalid cipher parameters");
 391                 break;
 392         }
 393
 394         cfg->key_len = cipher->key.length;
 395         cfg->iv_len = cipher->iv.length;
 396         cfg->salt = salt;
 397
 398         memcpy(cfg->key_data, cipher->key.data, cfg->key_len);
 399 }
 400
 401 static void
 402 sa_add_set_auth_params(struct virtchnl_ipsec_crypto_cfg_item *cfg,
 403         struct rte_crypto_auth_xform *auth, uint32_t salt)
 404 {
 405         cfg->crypto_type = VIRTCHNL_AUTH;
 406
 407         switch (auth->algo) {
 408         case RTE_CRYPTO_AUTH_NULL:
 409                 cfg->algo_type = VIRTCHNL_HASH_NO_ALG; break;
 410         case RTE_CRYPTO_AUTH_AES_CBC_MAC:
 411                 cfg->algo_type = VIRTCHNL_AES_CBC_MAC; break;
 412         case RTE_CRYPTO_AUTH_AES_CMAC:
 413                 cfg->algo_type = VIRTCHNL_AES_CMAC; break;
 414         case RTE_CRYPTO_AUTH_AES_XCBC_MAC:
 415                 cfg->algo_type = VIRTCHNL_AES_XCBC_MAC; break;
 416         case RTE_CRYPTO_AUTH_MD5_HMAC:
 417                 cfg->algo_type = VIRTCHNL_MD5_HMAC; break;
 418         case RTE_CRYPTO_AUTH_SHA1_HMAC:
 419                 cfg->algo_type = VIRTCHNL_SHA1_HMAC; break;
 420         case RTE_CRYPTO_AUTH_SHA224_HMAC:
 421                 cfg->algo_type = VIRTCHNL_SHA224_HMAC; break;
 422         case RTE_CRYPTO_AUTH_SHA256_HMAC:
 423                 cfg->algo_type = VIRTCHNL_SHA256_HMAC; break;
 424         case RTE_CRYPTO_AUTH_SHA384_HMAC:
 425                 cfg->algo_type = VIRTCHNL_SHA384_HMAC; break;
 426         case RTE_CRYPTO_AUTH_SHA512_HMAC:
 427                 cfg->algo_type = VIRTCHNL_SHA512_HMAC; break;
 428         case RTE_CRYPTO_AUTH_AES_GMAC:
 429                 cfg->algo_type = VIRTCHNL_AES_GMAC;
 430                 cfg->salt = salt;
 431                 break;
 432         default:
 433                 PMD_DRV_LOG(ERR, "Invalid auth parameters");
 434                 break;
 435         }
 436
 437         cfg->key_len = auth->key.length;
 438         /* special case for RTE_CRYPTO_AUTH_AES_GMAC */
 439         if (auth->algo == RTE_CRYPTO_AUTH_AES_GMAC)
 440                 cfg->iv_len = sizeof(uint64_t); /* iv.length includes salt */
 441         else
 442                 cfg->iv_len = auth->iv.length;
 443         cfg->digest_len = auth->digest_length;
 444
 445         memcpy(cfg->key_data, auth->key.data, cfg->key_len);
 446 }
 447
 448 /**
 449  * Send SA add virtual channel request to Inline IPsec driver.
 450  *
 451  * Inline IPsec driver expects SPI and destination IP adderss to be in host
 452  * order, but DPDK APIs are network order, therefore we need to do a htonl
 453  * conversion of these parameters.
 454  */
 455 static uint32_t
 456 iavf_ipsec_crypto_security_association_add(struct iavf_adapter *adapter,
 457         struct rte_security_session_conf *conf)
 458 {
 459         struct inline_ipsec_msg *request = NULL, *response = NULL;
 460         struct virtchnl_ipsec_sa_cfg *sa_cfg;
 461         size_t request_len, response_len;
 462
 463         int rc;
 464
 465         request_len = sizeof(struct inline_ipsec_msg) +
 466                         sizeof(struct virtchnl_ipsec_sa_cfg);
 467
 468         request = rte_malloc("iavf-sad-add-request", request_len, 0);
 469         if (request == NULL) {
 470                 rc = -ENOMEM;
 471                 goto update_cleanup;
 472         }
 473
 474         response_len = sizeof(struct inline_ipsec_msg) +
 475                         sizeof(struct virtchnl_ipsec_sa_cfg_resp);
 476         response = rte_malloc("iavf-sad-add-response", response_len, 0);
 477         if (response == NULL) {
 478                 rc = -ENOMEM;
 479                 goto update_cleanup;
 480         }
 481
 482         /* set msg header params */
 483         request->ipsec_opcode = INLINE_IPSEC_OP_SA_CREATE;
 484         request->req_id = (uint16_t)0xDEADBEEF;
 485
 486         /* set SA configuration params */
 487         sa_cfg = (struct virtchnl_ipsec_sa_cfg *)(request + 1);
 488
 489         sa_cfg->spi = conf->ipsec.spi;
 490         sa_cfg->virtchnl_protocol_type = VIRTCHNL_PROTO_ESP;
 491         sa_cfg->virtchnl_direction =
 492                 conf->ipsec.direction == RTE_SECURITY_IPSEC_SA_DIR_INGRESS ?
 493                         VIRTCHNL_DIR_INGRESS : VIRTCHNL_DIR_EGRESS;
 494
 495         if (conf->ipsec.options.esn) {
 496                 sa_cfg->esn_enabled = 1;
 497                 sa_cfg->esn_hi = conf->ipsec.esn.hi;
 498                 sa_cfg->esn_low = conf->ipsec.esn.low;
 499         }
 500
 501         if (conf->ipsec.options.udp_encap)
 502                 sa_cfg->udp_encap_enabled = 1;
 503
 504         /* Set outer IP params */
 505         if (conf->ipsec.tunnel.type == RTE_SECURITY_IPSEC_TUNNEL_IPV4) {
 506                 sa_cfg->virtchnl_ip_type = VIRTCHNL_IPV4;
 507
 508                 *((uint32_t *)sa_cfg->dst_addr) =
 509                         htonl(conf->ipsec.tunnel.ipv4.dst_ip.s_addr);
 510         } else {
 511                 uint32_t *v6_dst_addr =
 512                         (uint32_t *)conf->ipsec.tunnel.ipv6.dst_addr.s6_addr;
 513
 514                 sa_cfg->virtchnl_ip_type = VIRTCHNL_IPV6;
 515
 516                 ((uint32_t *)sa_cfg->dst_addr)[0] = htonl(v6_dst_addr[0]);
 517                 ((uint32_t *)sa_cfg->dst_addr)[1] = htonl(v6_dst_addr[1]);
 518                 ((uint32_t *)sa_cfg->dst_addr)[2] = htonl(v6_dst_addr[2]);
 519                 ((uint32_t *)sa_cfg->dst_addr)[3] = htonl(v6_dst_addr[3]);
 520         }
 521
 522         /* set crypto params */
 523         if (conf->crypto_xform->type == RTE_CRYPTO_SYM_XFORM_AEAD) {
 524                 sa_add_set_aead_params(&sa_cfg->crypto_cfg.items[0],
 525                         &conf->crypto_xform->aead, conf->ipsec.salt);
 526
 527         } else if (conf->crypto_xform->type == RTE_CRYPTO_SYM_XFORM_CIPHER) {
 528                 sa_add_set_cipher_params(&sa_cfg->crypto_cfg.items[0],
 529                         &conf->crypto_xform->cipher, conf->ipsec.salt);
 530                 sa_add_set_auth_params(&sa_cfg->crypto_cfg.items[1],
 531                         &conf->crypto_xform->next->auth, conf->ipsec.salt);
 532
 533         } else if (conf->crypto_xform->type == RTE_CRYPTO_SYM_XFORM_AUTH) {
 534                 sa_add_set_auth_params(&sa_cfg->crypto_cfg.items[0],
 535                         &conf->crypto_xform->auth, conf->ipsec.salt);
 536                 if (conf->crypto_xform->auth.algo != RTE_CRYPTO_AUTH_AES_GMAC)
 537                         sa_add_set_cipher_params(&sa_cfg->crypto_cfg.items[1],
 538                         &conf->crypto_xform->next->cipher, conf->ipsec.salt);
 539         }
 540
 541         /* send virtual channel request to add SA to hardware database */
 542         rc = iavf_ipsec_crypto_request(adapter,
 543                         (uint8_t *)request, request_len,
 544                         (uint8_t *)response, response_len);
 545         if (rc)
 546                 goto update_cleanup;
 547
 548         /* verify response id */
 549         if (response->ipsec_opcode != request->ipsec_opcode ||
 550                 response->req_id != request->req_id)
 551                 rc = -EFAULT;
 552         else
 553                 rc = response->ipsec_data.sa_cfg_resp->sa_handle;
 554 update_cleanup:
 555         rte_free(response);
 556         rte_free(request);
 557
 558         return rc;
 559 }
 560
 561 static void
 562 set_pkt_metadata_template(struct iavf_ipsec_crypto_pkt_metadata *template,
 563         struct iavf_security_session *sess)
 564 {
 565         template->sa_idx = sess->sa.hw_idx;
 566
 567         if (sess->udp_encap.enabled)
 568                 template->ol_flags = IAVF_IPSEC_CRYPTO_OL_FLAGS_NATT;
 569
 570         if (sess->esn.enabled)
 571                 template->ol_flags = IAVF_IPSEC_CRYPTO_OL_FLAGS_ESN;
 572
 573         template->len_iv = calc_ipsec_desc_iv_len_field(sess->iv_sz);
 574         template->ctx_desc_ipsec_params =
 575                         calc_context_desc_cipherblock_sz(sess->block_sz) |
 576                         ((uint8_t)(sess->icv_sz >> 2) << 3);
 577 }
 578
 579 static void
 580 set_session_parameter(struct iavf_security_ctx *iavf_sctx,
 581         struct iavf_security_session *sess,
 582         struct rte_security_session_conf *conf, uint32_t sa_idx)
 583 {
 584         sess->adapter = iavf_sctx->adapter;
 585
 586         sess->mode = conf->ipsec.mode;
 587         sess->direction = conf->ipsec.direction;
 588
 589         if (sess->mode == RTE_SECURITY_IPSEC_SA_MODE_TUNNEL)
 590                 sess->type = conf->ipsec.tunnel.type;
 591
 592         sess->sa.spi = conf->ipsec.spi;
 593         sess->sa.hw_idx = sa_idx;
 594
 595         if (conf->ipsec.options.esn) {
 596                 sess->esn.enabled = 1;
 597                 sess->esn.value = conf->ipsec.esn.value;
 598         }
 599
 600         if (conf->ipsec.options.udp_encap)
 601                 sess->udp_encap.enabled = 1;
 602
 603         if (conf->crypto_xform->type == RTE_CRYPTO_SYM_XFORM_AEAD) {
 604                 sess->block_sz = get_aead_blocksize(iavf_sctx,
 605                         conf->crypto_xform->aead.algo);
 606                 sess->iv_sz = sizeof(uint64_t); /* iv.length includes salt */
 607                 sess->icv_sz = conf->crypto_xform->aead.digest_length;
 608         } else if (conf->crypto_xform->type == RTE_CRYPTO_SYM_XFORM_CIPHER) {
 609                 sess->block_sz = get_cipher_blocksize(iavf_sctx,
 610                         conf->crypto_xform->cipher.algo);
 611                 sess->iv_sz = conf->crypto_xform->cipher.iv.length;
 612                 sess->icv_sz = conf->crypto_xform->next->auth.digest_length;
 613         } else if (conf->crypto_xform->type == RTE_CRYPTO_SYM_XFORM_AUTH) {
 614                 if (conf->crypto_xform->auth.algo == RTE_CRYPTO_AUTH_AES_GMAC) {
 615                         sess->block_sz = get_auth_blocksize(iavf_sctx,
 616                                 conf->crypto_xform->auth.algo);
 617                         sess->iv_sz = conf->crypto_xform->auth.iv.length;
 618                         sess->icv_sz = conf->crypto_xform->auth.digest_length;
 619                 } else {
 620                         sess->block_sz = get_cipher_blocksize(iavf_sctx,
 621                                 conf->crypto_xform->next->cipher.algo);
 622                         sess->iv_sz =
 623                                 conf->crypto_xform->next->cipher.iv.length;
 624                         sess->icv_sz = conf->crypto_xform->auth.digest_length;
 625                 }
 626         }
 627
 628         set_pkt_metadata_template(&sess->pkt_metadata_template, sess);
 629 }
 630
 631 /**
 632  * Create IPsec Security Association for inline IPsec Crypto offload.
 633  *
 634  * 1. validate session configuration parameters
 635  * 2. allocate session memory from mempool
 636  * 3. add SA to hardware database
 637  * 4. set session parameters
 638  * 5. create packet metadata template for datapath
 639  */
 640 static int
 641 iavf_ipsec_crypto_session_create(void *device,
 642                                  struct rte_security_session_conf *conf,
 643                                  struct rte_security_session *session,
 644                                  struct rte_mempool *mempool)
 645 {
 646         struct rte_eth_dev *ethdev = device;
 647         struct iavf_adapter *adapter =
 648                 IAVF_DEV_PRIVATE_TO_ADAPTER(ethdev->data->dev_private);
 649         struct iavf_security_ctx *iavf_sctx = adapter->security_ctx;
 650         struct iavf_security_session *iavf_session = NULL;
 651         int sa_idx;
 652         int ret = 0;
 653
 654         /* validate that all SA parameters are valid for device */
 655         ret = iavf_ipsec_crypto_session_validate_conf(iavf_sctx, conf);
 656         if (ret)
 657                 return ret;
 658
 659         /* allocate session context */
 660         if (rte_mempool_get(mempool, (void **)&iavf_session)) {
 661                 PMD_DRV_LOG(ERR, "Cannot get object from sess mempool");
 662                 return -ENOMEM;
 663         }
 664
 665         /* add SA to hardware database */
 666         sa_idx = iavf_ipsec_crypto_security_association_add(adapter, conf);
 667         if (sa_idx < 0) {
 668                 PMD_DRV_LOG(ERR,
 669                         "Failed to add SA (spi: %d, mode: %s, direction: %s)",
 670                         conf->ipsec.spi,
 671                         conf->ipsec.mode ==
 672                                 RTE_SECURITY_IPSEC_SA_MODE_TRANSPORT ?
 673                                 "transport" : "tunnel",
 674                         conf->ipsec.direction ==
 675                                 RTE_SECURITY_IPSEC_SA_DIR_INGRESS ?
 676                                 "inbound" : "outbound");
 677
 678                 rte_mempool_put(mempool, iavf_session);
 679                 return -EFAULT;
 680         }
 681
 682         /* save data plane required session parameters */
 683         set_session_parameter(iavf_sctx, iavf_session, conf, sa_idx);
 684
 685         /* save to security session private data */
 686         set_sec_session_private_data(session, iavf_session);
 687
 688         return 0;
 689 }
 690
 691 /**
 692  * Check if valid ipsec crypto action.
 693  * SPI must be non-zero and SPI in session must match SPI value
 694  * passed into function.
 695  *
 696  * returns: 0 if invalid session or SPI value equal zero
 697  * returns: 1 if valid
 698  */
 699 uint32_t
 700 iavf_ipsec_crypto_action_valid(struct rte_eth_dev *ethdev,
 701         const struct rte_security_session *session, uint32_t spi)
 702 {
 703         struct iavf_adapter *adapter =
 704                 IAVF_DEV_PRIVATE_TO_ADAPTER(ethdev->data->dev_private);
 705         struct iavf_security_session *sess = session->sess_private_data;
 706
 707         /* verify we have a valid session and that it belong to this adapter */
 708         if (unlikely(sess == NULL || sess->adapter != adapter))
 709                 return false;
 710
 711         /* SPI value must be non-zero */
 712         if (spi == 0)
 713                 return false;
 714         /* Session SPI must patch flow SPI*/
 715         else if (sess->sa.spi == spi) {
 716                 return true;
 717                 /**
 718                  * TODO: We should add a way of tracking valid hw SA indices to
 719                  * make validation less brittle
 720                  */
 721         }
 722
 723                 return true;
 724 }
 725
 726 /**
 727  * Send virtual channel security policy add request to IES driver.
 728  *
 729  * IES driver expects SPI and destination IP adderss to be in host
 730  * order, but DPDK APIs are network order, therefore we need to do a htonl
 731  * conversion of these parameters.
 732  */
 733 int
 734 iavf_ipsec_crypto_inbound_security_policy_add(struct iavf_adapter *adapter,
 735         uint32_t esp_spi,
 736         uint8_t is_v4,
 737         rte_be32_t v4_dst_addr,
 738         uint8_t *v6_dst_addr,
 739         uint8_t drop)
 740 {
 741         struct inline_ipsec_msg *request = NULL, *response = NULL;
 742         size_t request_len, response_len;
 743         int rc = 0;
 744
 745         request_len = sizeof(struct inline_ipsec_msg) +
 746                         sizeof(struct virtchnl_ipsec_sp_cfg);
 747         request = rte_malloc("iavf-inbound-security-policy-add-request",
 748                                 request_len, 0);
 749         if (request == NULL) {
 750                 rc = -ENOMEM;
 751                 goto update_cleanup;
 752         }
 753
 754         /* set msg header params */
 755         request->ipsec_opcode = INLINE_IPSEC_OP_SP_CREATE;
 756         request->req_id = (uint16_t)0xDEADBEEF;
 757
 758         /* ESP SPI */
 759         request->ipsec_data.sp_cfg->spi = htonl(esp_spi);
 760
 761         /* Destination IP  */
 762         if (is_v4) {
 763                 request->ipsec_data.sp_cfg->table_id =
 764                                 VIRTCHNL_IPSEC_INBOUND_SPD_TBL_IPV4;
 765                 request->ipsec_data.sp_cfg->dip[0] = htonl(v4_dst_addr);
 766         } else {
 767                 request->ipsec_data.sp_cfg->table_id =
 768                                 VIRTCHNL_IPSEC_INBOUND_SPD_TBL_IPV6;
 769                 request->ipsec_data.sp_cfg->dip[0] =
 770                                 htonl(((uint32_t *)v6_dst_addr)[0]);
 771                 request->ipsec_data.sp_cfg->dip[1] =
 772                                 htonl(((uint32_t *)v6_dst_addr)[1]);
 773                 request->ipsec_data.sp_cfg->dip[2] =
 774                                 htonl(((uint32_t *)v6_dst_addr)[2]);
 775                 request->ipsec_data.sp_cfg->dip[3] =
 776                                 htonl(((uint32_t *)v6_dst_addr)[3]);
 777         }
 778
 779         request->ipsec_data.sp_cfg->drop = drop;
 780
 781         /** Traffic Class/Congestion Domain currently not support */
 782         request->ipsec_data.sp_cfg->set_tc = 0;
 783         request->ipsec_data.sp_cfg->cgd = 0;
 784
 785         response_len = sizeof(struct inline_ipsec_msg) +
 786                         sizeof(struct virtchnl_ipsec_sp_cfg_resp);
 787         response = rte_malloc("iavf-inbound-security-policy-add-response",
 788                                 response_len, 0);
 789         if (response == NULL) {
 790                 rc = -ENOMEM;
 791                 goto update_cleanup;
 792         }
 793
 794         /* send virtual channel request to add SA to hardware database */
 795         rc = iavf_ipsec_crypto_request(adapter,
 796                         (uint8_t *)request, request_len,
 797                         (uint8_t *)response, response_len);
 798         if (rc)
 799                 goto update_cleanup;
 800
 801         /* verify response */
 802         if (response->ipsec_opcode != request->ipsec_opcode ||
 803                 response->req_id != request->req_id)
 804                 rc = -EFAULT;
 805         else
 806                 rc = response->ipsec_data.sp_cfg_resp->rule_id;
 807
 808 update_cleanup:
 809         rte_free(request);
 810         rte_free(response);
 811
 812         return rc;
 813 }
 814
 815 static uint32_t
 816 iavf_ipsec_crypto_sa_update_esn(struct iavf_adapter *adapter,
 817         struct iavf_security_session *sess)
 818 {
 819         struct inline_ipsec_msg *request = NULL, *response = NULL;
 820         size_t request_len, response_len;
 821         int rc = 0;
 822
 823         request_len = sizeof(struct inline_ipsec_msg) +
 824                         sizeof(struct virtchnl_ipsec_sa_update);
 825         request = rte_malloc("iavf-sa-update-request", request_len, 0);
 826         if (request == NULL) {
 827                 rc = -ENOMEM;
 828                 goto update_cleanup;
 829         }
 830
 831         response_len = sizeof(struct inline_ipsec_msg) +
 832                         sizeof(struct virtchnl_ipsec_resp);
 833         response = rte_malloc("iavf-sa-update-response", response_len, 0);
 834         if (response == NULL) {
 835                 rc = -ENOMEM;
 836                 goto update_cleanup;
 837         }
 838
 839         /* set msg header params */
 840         request->ipsec_opcode = INLINE_IPSEC_OP_SA_UPDATE;
 841         request->req_id = (uint16_t)0xDEADBEEF;
 842
 843         /* set request params */
 844         request->ipsec_data.sa_update->sa_index = sess->sa.hw_idx;
 845         request->ipsec_data.sa_update->esn_hi = sess->esn.hi;
 846
 847         /* send virtual channel request to add SA to hardware database */
 848         rc = iavf_ipsec_crypto_request(adapter,
 849                         (uint8_t *)request, request_len,
 850                         (uint8_t *)response, response_len);
 851         if (rc)
 852                 goto update_cleanup;
 853
 854         /* verify response */
 855         if (response->ipsec_opcode != request->ipsec_opcode ||
 856                 response->req_id != request->req_id)
 857                 rc = -EFAULT;
 858         else
 859                 rc = response->ipsec_data.ipsec_resp->resp;
 860
 861 update_cleanup:
 862         rte_free(request);
 863         rte_free(response);
 864
 865         return rc;
 866 }
 867
 868 static int
 869 iavf_ipsec_crypto_session_update(void *device,
 870                 struct rte_security_session *session,
 871                 struct rte_security_session_conf *conf)
 872 {
 873         struct iavf_adapter *adapter = NULL;
 874         struct iavf_security_session *iavf_sess = NULL;
 875         struct rte_eth_dev *eth_dev = (struct rte_eth_dev *)device;
 876         int rc = 0;
 877
 878         adapter = IAVF_DEV_PRIVATE_TO_ADAPTER(eth_dev->data->dev_private);
 879         iavf_sess = (struct iavf_security_session *)session->sess_private_data;
 880
 881         /* verify we have a valid session and that it belong to this adapter */
 882         if (unlikely(iavf_sess == NULL || iavf_sess->adapter != adapter))
 883                 return -EINVAL;
 884
 885         /* update esn hi 32-bits */
 886         if (iavf_sess->esn.enabled && conf->ipsec.options.esn) {
 887                 /**
 888                  * Update ESN in hardware for inbound SA. Store in
 889                  * iavf_security_session for outbound SA for use
 890                  * in *iavf_ipsec_crypto_pkt_metadata_set* function.
 891                  */
 892                 if (iavf_sess->direction == RTE_SECURITY_IPSEC_SA_DIR_INGRESS)
 893                         rc = iavf_ipsec_crypto_sa_update_esn(adapter,
 894                                         iavf_sess);
 895                 else
 896                         iavf_sess->esn.hi = conf->ipsec.esn.hi;
 897         }
 898
 899         return rc;
 900 }
 901
 902 static int
 903 iavf_ipsec_crypto_session_stats_get(void *device __rte_unused,
 904                 struct rte_security_session *session __rte_unused,
 905                 struct rte_security_stats *stats __rte_unused)
 906 {
 907         return -EOPNOTSUPP;
 908 }
 909
 910 int
 911 iavf_ipsec_crypto_security_policy_delete(struct iavf_adapter *adapter,
 912         uint8_t is_v4, uint32_t flow_id)
 913 {
 914         struct inline_ipsec_msg *request = NULL, *response = NULL;
 915         size_t request_len, response_len;
 916         int rc = 0;
 917
 918         request_len = sizeof(struct inline_ipsec_msg) +
 919                         sizeof(struct virtchnl_ipsec_sp_destroy);
 920         request = rte_malloc("iavf-sp-del-request", request_len, 0);
 921         if (request == NULL) {
 922                 rc = -ENOMEM;
 923                 goto update_cleanup;
 924         }
 925
 926         response_len = sizeof(struct inline_ipsec_msg) +
 927                         sizeof(struct virtchnl_ipsec_resp);
 928         response = rte_malloc("iavf-sp-del-response", response_len, 0);
 929         if (response == NULL) {
 930                 rc = -ENOMEM;
 931                 goto update_cleanup;
 932         }
 933
 934         /* set msg header params */
 935         request->ipsec_opcode = INLINE_IPSEC_OP_SP_DESTROY;
 936         request->req_id = (uint16_t)0xDEADBEEF;
 937
 938         /* set security policy params */
 939         request->ipsec_data.sp_destroy->table_id = is_v4 ?
 940                         VIRTCHNL_IPSEC_INBOUND_SPD_TBL_IPV4 :
 941                         VIRTCHNL_IPSEC_INBOUND_SPD_TBL_IPV6;
 942         request->ipsec_data.sp_destroy->rule_id = flow_id;
 943
 944         /* send virtual channel request to add SA to hardware database */
 945         rc = iavf_ipsec_crypto_request(adapter,
 946                         (uint8_t *)request, request_len,
 947                         (uint8_t *)response, response_len);
 948         if (rc)
 949                 goto update_cleanup;
 950
 951         /* verify response */
 952         if (response->ipsec_opcode != request->ipsec_opcode ||
 953                 response->req_id != request->req_id)
 954                 rc = -EFAULT;
 955         else
 956                 return response->ipsec_data.ipsec_status->status;
 957
 958 update_cleanup:
 959         rte_free(request);
 960         rte_free(response);
 961
 962         return rc;
 963 }
 964
 965 static uint32_t
 966 iavf_ipsec_crypto_sa_del(struct iavf_adapter *adapter,
 967         struct iavf_security_session *sess)
 968 {
 969         struct inline_ipsec_msg *request = NULL, *response = NULL;
 970         size_t request_len, response_len;
 971
 972         int rc = 0;
 973
 974         request_len = sizeof(struct inline_ipsec_msg) +
 975                         sizeof(struct virtchnl_ipsec_sa_destroy);
 976
 977         request = rte_malloc("iavf-sa-del-request", request_len, 0);
 978         if (request == NULL) {
 979                 rc = -ENOMEM;
 980                 goto update_cleanup;
 981         }
 982
 983         response_len = sizeof(struct inline_ipsec_msg) +
 984                         sizeof(struct virtchnl_ipsec_resp);
 985
 986         response = rte_malloc("iavf-sa-del-response", response_len, 0);
 987         if (response == NULL) {
 988                 rc = -ENOMEM;
 989                 goto update_cleanup;
 990         }
 991
 992         /* set msg header params */
 993         request->ipsec_opcode = INLINE_IPSEC_OP_SA_DESTROY;
 994         request->req_id = (uint16_t)0xDEADBEEF;
 995
 996         /**
 997          * SA delete supports deletetion of 1-8 specified SA's or if the flag
 998          * field is zero, all SA's associated with VF will be deleted.
 999          */
1000         if (sess) {
1001                 request->ipsec_data.sa_destroy->flag = 0x1;
1002                 request->ipsec_data.sa_destroy->sa_index[0] = sess->sa.hw_idx;
1003         } else {
1004                 request->ipsec_data.sa_destroy->flag = 0x0;
1005         }
1006
1007         /* send virtual channel request to add SA to hardware database */
1008         rc = iavf_ipsec_crypto_request(adapter,
1009                         (uint8_t *)request, request_len,
1010                         (uint8_t *)response, response_len);
1011         if (rc)
1012                 goto update_cleanup;
1013
1014         /* verify response */
1015         if (response->ipsec_opcode != request->ipsec_opcode ||
1016                 response->req_id != request->req_id)
1017                 rc = -EFAULT;
1018
1019         /**
1020          * Delete status will be the same bitmask as sa_destroy request flag if
1021          * deletes successful
1022          */
1023         if (request->ipsec_data.sa_destroy->flag !=
1024                         response->ipsec_data.ipsec_status->status)
1025                 rc = -EFAULT;
1026
1027 update_cleanup:
1028         rte_free(response);
1029         rte_free(request);
1030
1031         return rc;
1032 }
1033
1034 static int
1035 iavf_ipsec_crypto_session_destroy(void *device,
1036                 struct rte_security_session *session)
1037 {
1038         struct iavf_adapter *adapter = NULL;
1039         struct iavf_security_session *iavf_sess = NULL;
1040         struct rte_eth_dev *eth_dev = (struct rte_eth_dev *)device;
1041         int ret;
1042
1043         adapter = IAVF_DEV_PRIVATE_TO_ADAPTER(eth_dev->data->dev_private);
1044         iavf_sess = (struct iavf_security_session *)session->sess_private_data;
1045
1046         /* verify we have a valid session and that it belong to this adapter */
1047         if (unlikely(iavf_sess == NULL || iavf_sess->adapter != adapter))
1048                 return -EINVAL;
1049
1050         ret = iavf_ipsec_crypto_sa_del(adapter, iavf_sess);
1051         rte_mempool_put(rte_mempool_from_obj(iavf_sess), (void *)iavf_sess);
1052         return ret;
1053 }
1054
1055 /**
1056  * Get ESP trailer from packet as well as calculate the total ESP trailer
1057  * length, which include padding, ESP trailer footer and the ICV
1058  */
1059 static inline struct rte_esp_tail *
1060 iavf_ipsec_crypto_get_esp_trailer(struct rte_mbuf *m,
1061         struct iavf_security_session *s, uint16_t *esp_trailer_length)
1062 {
1063         struct rte_esp_tail *esp_trailer;
1064
1065         uint16_t length = sizeof(struct rte_esp_tail) + s->icv_sz;
1066         uint16_t offset = 0;
1067
1068         /**
1069          * The ICV will not be present in TSO packets as this is appended by
1070          * hardware during segment generation
1071          */
1072         if (m->ol_flags & (RTE_MBUF_F_TX_TCP_SEG | RTE_MBUF_F_TX_UDP_SEG))
1073                 length -=  s->icv_sz;
1074
1075         *esp_trailer_length = length;
1076
1077         /**
1078          * Calculate offset in packet to ESP trailer header, this should be
1079          * total packet length less the size of the ESP trailer plus the ICV
1080          * length if it is present
1081          */
1082         offset = rte_pktmbuf_pkt_len(m) - length;
1083
1084         if (m->nb_segs > 1) {
1085                 /* find segment which esp trailer is located */
1086                 while (m->data_len < offset) {
1087                         offset -= m->data_len;
1088                         m = m->next;
1089                 }
1090         }
1091
1092         esp_trailer = rte_pktmbuf_mtod_offset(m, struct rte_esp_tail *, offset);
1093
1094         *esp_trailer_length += esp_trailer->pad_len;
1095
1096         return esp_trailer;
1097 }
1098
1099 static inline uint16_t
1100 iavf_ipsec_crypto_compute_l4_payload_length(struct rte_mbuf *m,
1101         struct iavf_security_session *s, uint16_t esp_tlen)
1102 {
1103         uint16_t ol2_len = m->l2_len;   /* MAC + VLAN */
1104         uint16_t ol3_len = 0;           /* ipv4/6 + ext hdrs */
1105         uint16_t ol4_len = 0;           /* UDP NATT */
1106         uint16_t l3_len = 0;            /* IPv4/6 + ext hdrs */
1107         uint16_t l4_len = 0;            /* TCP/UDP/STCP hdrs */
1108         uint16_t esp_hlen = sizeof(struct rte_esp_hdr) + s->iv_sz;
1109
1110         if (s->mode == RTE_SECURITY_IPSEC_SA_MODE_TUNNEL)
1111                 ol3_len = m->outer_l3_len;
1112                 /**<
1113                  * application provided l3len assumed to include length of
1114                  * ipv4/6 hdr + ext hdrs
1115                  */
1116
1117         if (s->udp_encap.enabled)
1118                 ol4_len = sizeof(struct rte_udp_hdr);
1119
1120         l3_len = m->l3_len;
1121         l4_len = m->l4_len;
1122
1123         return rte_pktmbuf_pkt_len(m) - (ol2_len + ol3_len + ol4_len +
1124                         esp_hlen + l3_len + l4_len + esp_tlen);
1125 }
1126
1127 static int
1128 iavf_ipsec_crypto_pkt_metadata_set(void *device,
1129                          struct rte_security_session *session,
1130                          struct rte_mbuf *m, void *params)
1131 {
1132         struct rte_eth_dev *ethdev = device;
1133         struct iavf_adapter *adapter =
1134                         IAVF_DEV_PRIVATE_TO_ADAPTER(ethdev->data->dev_private);
1135         struct iavf_security_ctx *iavf_sctx = adapter->security_ctx;
1136         struct iavf_security_session *iavf_sess = session->sess_private_data;
1137         struct iavf_ipsec_crypto_pkt_metadata *md;
1138         struct rte_esp_tail *esp_tail;
1139         uint64_t *sqn = params;
1140         uint16_t esp_trailer_length;
1141
1142         /* Check we have valid session and is associated with this device */
1143         if (unlikely(iavf_sess == NULL || iavf_sess->adapter != adapter))
1144                 return -EINVAL;
1145
1146         /* Get dynamic metadata location from mbuf */
1147         md = RTE_MBUF_DYNFIELD(m, iavf_sctx->pkt_md_offset,
1148                 struct iavf_ipsec_crypto_pkt_metadata *);
1149
1150         /* Set immutatable metadata values from session template */
1151         memcpy(md, &iavf_sess->pkt_metadata_template,
1152                 sizeof(struct iavf_ipsec_crypto_pkt_metadata));
1153
1154         esp_tail = iavf_ipsec_crypto_get_esp_trailer(m, iavf_sess,
1155                         &esp_trailer_length);
1156
1157         /* Set per packet mutable metadata values */
1158         md->esp_trailer_len = esp_trailer_length;
1159         md->l4_payload_len = iavf_ipsec_crypto_compute_l4_payload_length(m,
1160                                 iavf_sess, esp_trailer_length);
1161         md->next_proto = esp_tail->next_proto;
1162
1163         /* If Extended SN in use set the upper 32-bits in metadata */
1164         if (iavf_sess->esn.enabled && sqn != NULL)
1165                 md->esn = (uint32_t)(*sqn >> 32);
1166
1167         return 0;
1168 }
1169
1170 static int
1171 iavf_ipsec_crypto_device_capabilities_get(struct iavf_adapter *adapter,
1172                 struct virtchnl_ipsec_cap *capability)
1173 {
1174         /* Perform pf-vf comms */
1175         struct inline_ipsec_msg *request = NULL, *response = NULL;
1176         size_t request_len, response_len;
1177         int rc;
1178
1179         request_len = sizeof(struct inline_ipsec_msg);
1180
1181         request = rte_malloc("iavf-device-capability-request", request_len, 0);
1182         if (request == NULL) {
1183                 rc = -ENOMEM;
1184                 goto update_cleanup;
1185         }
1186
1187         response_len = sizeof(struct inline_ipsec_msg) +
1188                         sizeof(struct virtchnl_ipsec_cap);
1189         response = rte_malloc("iavf-device-capability-response",
1190                         response_len, 0);
1191         if (response == NULL) {
1192                 rc = -ENOMEM;
1193                 goto update_cleanup;
1194         }
1195
1196         /* set msg header params */
1197         request->ipsec_opcode = INLINE_IPSEC_OP_GET_CAP;
1198         request->req_id = (uint16_t)0xDEADBEEF;
1199
1200         /* send virtual channel request to add SA to hardware database */
1201         rc = iavf_ipsec_crypto_request(adapter,
1202                         (uint8_t *)request, request_len,
1203                         (uint8_t *)response, response_len);
1204         if (rc)
1205                 goto update_cleanup;
1206
1207         /* verify response id */
1208         if (response->ipsec_opcode != request->ipsec_opcode ||
1209                 response->req_id != request->req_id){
1210                 rc = -EFAULT;
1211                 goto update_cleanup;
1212         }
1213         memcpy(capability, response->ipsec_data.ipsec_cap, sizeof(*capability));
1214
1215 update_cleanup:
1216         rte_free(response);
1217         rte_free(request);
1218
1219         return rc;
1220 }
1221
1222 enum rte_crypto_auth_algorithm auth_maptbl[] = {
1223         /* Hash Algorithm */
1224         [VIRTCHNL_HASH_NO_ALG] = RTE_CRYPTO_AUTH_NULL,
1225         [VIRTCHNL_AES_CBC_MAC] = RTE_CRYPTO_AUTH_AES_CBC_MAC,
1226         [VIRTCHNL_AES_CMAC] = RTE_CRYPTO_AUTH_AES_CMAC,
1227         [VIRTCHNL_AES_GMAC] = RTE_CRYPTO_AUTH_AES_GMAC,
1228         [VIRTCHNL_AES_XCBC_MAC] = RTE_CRYPTO_AUTH_AES_XCBC_MAC,
1229         [VIRTCHNL_MD5_HMAC] = RTE_CRYPTO_AUTH_MD5_HMAC,
1230         [VIRTCHNL_SHA1_HMAC] = RTE_CRYPTO_AUTH_SHA1_HMAC,
1231         [VIRTCHNL_SHA224_HMAC] = RTE_CRYPTO_AUTH_SHA224_HMAC,
1232         [VIRTCHNL_SHA256_HMAC] = RTE_CRYPTO_AUTH_SHA256_HMAC,
1233         [VIRTCHNL_SHA384_HMAC] = RTE_CRYPTO_AUTH_SHA384_HMAC,
1234         [VIRTCHNL_SHA512_HMAC] = RTE_CRYPTO_AUTH_SHA512_HMAC,
1235         [VIRTCHNL_SHA3_224_HMAC] = RTE_CRYPTO_AUTH_SHA3_224_HMAC,
1236         [VIRTCHNL_SHA3_256_HMAC] = RTE_CRYPTO_AUTH_SHA3_256_HMAC,
1237         [VIRTCHNL_SHA3_384_HMAC] = RTE_CRYPTO_AUTH_SHA3_384_HMAC,
1238         [VIRTCHNL_SHA3_512_HMAC] = RTE_CRYPTO_AUTH_SHA3_512_HMAC,
1239 };
1240
1241 static void
1242 update_auth_capabilities(struct rte_cryptodev_capabilities *scap,
1243                 struct virtchnl_algo_cap *acap)
1244 {
1245         struct rte_cryptodev_symmetric_capability *capability = &scap->sym;
1246
1247         scap->op = RTE_CRYPTO_OP_TYPE_SYMMETRIC;
1248
1249         capability->xform_type = RTE_CRYPTO_SYM_XFORM_AUTH;
1250
1251         capability->auth.algo = auth_maptbl[acap->algo_type];
1252         capability->auth.block_size = acap->block_size;
1253
1254         capability->auth.key_size.min = acap->min_key_size;
1255         capability->auth.key_size.max = acap->max_key_size;
1256         capability->auth.key_size.increment = acap->inc_key_size;
1257
1258         capability->auth.digest_size.min = acap->min_digest_size;
1259         capability->auth.digest_size.max = acap->max_digest_size;
1260         capability->auth.digest_size.increment = acap->inc_digest_size;
1261 }
1262
1263 enum rte_crypto_cipher_algorithm cipher_maptbl[] = {
1264         /* Cipher Algorithm */
1265         [VIRTCHNL_CIPHER_NO_ALG] = RTE_CRYPTO_CIPHER_NULL,
1266         [VIRTCHNL_3DES_CBC] = RTE_CRYPTO_CIPHER_3DES_CBC,
1267         [VIRTCHNL_AES_CBC] = RTE_CRYPTO_CIPHER_AES_CBC,
1268         [VIRTCHNL_AES_CTR] = RTE_CRYPTO_CIPHER_AES_CTR,
1269 };
1270
1271 static void
1272 update_cipher_capabilities(struct rte_cryptodev_capabilities *scap,
1273         struct virtchnl_algo_cap *acap)
1274 {
1275         struct rte_cryptodev_symmetric_capability *capability = &scap->sym;
1276
1277         scap->op = RTE_CRYPTO_OP_TYPE_SYMMETRIC;
1278
1279         capability->xform_type = RTE_CRYPTO_SYM_XFORM_CIPHER;
1280
1281         capability->cipher.algo = cipher_maptbl[acap->algo_type];
1282
1283         capability->cipher.block_size = acap->block_size;
1284
1285         capability->cipher.key_size.min = acap->min_key_size;
1286         capability->cipher.key_size.max = acap->max_key_size;
1287         capability->cipher.key_size.increment = acap->inc_key_size;
1288
1289         capability->cipher.iv_size.min = acap->min_iv_size;
1290         capability->cipher.iv_size.max = acap->max_iv_size;
1291         capability->cipher.iv_size.increment = acap->inc_iv_size;
1292 }
1293
1294 enum rte_crypto_aead_algorithm aead_maptbl[] = {
1295         /* AEAD Algorithm */
1296         [VIRTCHNL_AES_CCM] = RTE_CRYPTO_AEAD_AES_CCM,
1297         [VIRTCHNL_AES_GCM] = RTE_CRYPTO_AEAD_AES_GCM,
1298         [VIRTCHNL_CHACHA20_POLY1305] = RTE_CRYPTO_AEAD_CHACHA20_POLY1305,
1299 };
1300
1301 static void
1302 update_aead_capabilities(struct rte_cryptodev_capabilities *scap,
1303         struct virtchnl_algo_cap *acap)
1304 {
1305         struct rte_cryptodev_symmetric_capability *capability = &scap->sym;
1306
1307         scap->op = RTE_CRYPTO_OP_TYPE_SYMMETRIC;
1308
1309         capability->xform_type = RTE_CRYPTO_SYM_XFORM_AEAD;
1310
1311         capability->aead.algo = aead_maptbl[acap->algo_type];
1312
1313         capability->aead.block_size = acap->block_size;
1314
1315         capability->aead.key_size.min = acap->min_key_size;
1316         capability->aead.key_size.max = acap->max_key_size;
1317         capability->aead.key_size.increment = acap->inc_key_size;
1318
1319         capability->aead.aad_size.min = acap->min_aad_size;
1320         capability->aead.aad_size.max = acap->max_aad_size;
1321         capability->aead.aad_size.increment = acap->inc_aad_size;
1322
1323         capability->aead.iv_size.min = acap->min_iv_size;
1324         capability->aead.iv_size.max = acap->max_iv_size;
1325         capability->aead.iv_size.increment = acap->inc_iv_size;
1326
1327         capability->aead.digest_size.min = acap->min_digest_size;
1328         capability->aead.digest_size.max = acap->max_digest_size;
1329         capability->aead.digest_size.increment = acap->inc_digest_size;
1330 }
1331
1332 /**
1333  * Dynamically set crypto capabilities based on virtchannel IPsec
1334  * capabilities structure.
1335  */
1336 int
1337 iavf_ipsec_crypto_set_security_capabililites(struct iavf_security_ctx
1338                 *iavf_sctx, struct virtchnl_ipsec_cap *vch_cap)
1339 {
1340         struct rte_cryptodev_capabilities *capabilities;
1341         int i, j, number_of_capabilities = 0, ci = 0;
1342
1343         /* Count the total number of crypto algorithms supported */
1344         for (i = 0; i < VIRTCHNL_IPSEC_MAX_CRYPTO_CAP_NUM; i++)
1345                 number_of_capabilities += vch_cap->cap[i].algo_cap_num;
1346
1347         /**
1348          * Allocate cryptodev capabilities structure for
1349          * *number_of_capabilities* items plus one item to null terminate the
1350          * array
1351          */
1352         capabilities = rte_zmalloc("crypto_cap",
1353                 sizeof(struct rte_cryptodev_capabilities) *
1354                 (number_of_capabilities + 1), 0);
1355         capabilities[number_of_capabilities].op = RTE_CRYPTO_OP_TYPE_UNDEFINED;
1356
1357         /**
1358          * Iterate over each virtchl crypto capability by crypto type and
1359          * algorithm.
1360          */
1361         for (i = 0; i < VIRTCHNL_IPSEC_MAX_CRYPTO_CAP_NUM; i++) {
1362                 for (j = 0; j < vch_cap->cap[i].algo_cap_num; j++, ci++) {
1363                         switch (vch_cap->cap[i].crypto_type) {
1364                         case VIRTCHNL_AUTH:
1365                                 update_auth_capabilities(&capabilities[ci],
1366                                         &vch_cap->cap[i].algo_cap_list[j]);
1367                                 break;
1368                         case VIRTCHNL_CIPHER:
1369                                 update_cipher_capabilities(&capabilities[ci],
1370                                         &vch_cap->cap[i].algo_cap_list[j]);
1371                                 break;
1372                         case VIRTCHNL_AEAD:
1373                                 update_aead_capabilities(&capabilities[ci],
1374                                         &vch_cap->cap[i].algo_cap_list[j]);
1375                                 break;
1376                         default:
1377                                 capabilities[ci].op =
1378                                                 RTE_CRYPTO_OP_TYPE_UNDEFINED;
1379                                 break;
1380                         }
1381                 }
1382         }
1383
1384         iavf_sctx->crypto_capabilities = capabilities;
1385         return 0;
1386 }
1387
1388 /**
1389  * Get security capabilities for device
1390  */
1391 static const struct rte_security_capability *
1392 iavf_ipsec_crypto_capabilities_get(void *device)
1393 {
1394         struct rte_eth_dev *eth_dev = (struct rte_eth_dev *)device;
1395         struct iavf_adapter *adapter =
1396                 IAVF_DEV_PRIVATE_TO_ADAPTER(eth_dev->data->dev_private);
1397         struct iavf_security_ctx *iavf_sctx = adapter->security_ctx;
1398         unsigned int i;
1399
1400         static struct rte_security_capability iavf_security_capabilities[] = {
1401                 { /* IPsec Inline Crypto ESP Tunnel Egress */
1402                         .action = RTE_SECURITY_ACTION_TYPE_INLINE_CRYPTO,
1403                         .protocol = RTE_SECURITY_PROTOCOL_IPSEC,
1404                         .ipsec = {
1405                                 .proto = RTE_SECURITY_IPSEC_SA_PROTO_ESP,
1406                                 .mode = RTE_SECURITY_IPSEC_SA_MODE_TUNNEL,
1407                                 .direction = RTE_SECURITY_IPSEC_SA_DIR_EGRESS,
1408                                 .options = { .udp_encap = 1,
1409                                                 .stats = 1, .esn = 1 },
1410                         },
1411                         .ol_flags = RTE_SECURITY_TX_OLOAD_NEED_MDATA
1412                 },
1413                 { /* IPsec Inline Crypto ESP Tunnel Ingress */
1414                         .action = RTE_SECURITY_ACTION_TYPE_INLINE_CRYPTO,
1415                         .protocol = RTE_SECURITY_PROTOCOL_IPSEC,
1416                         .ipsec = {
1417                                 .proto = RTE_SECURITY_IPSEC_SA_PROTO_ESP,
1418                                 .mode = RTE_SECURITY_IPSEC_SA_MODE_TUNNEL,
1419                                 .direction = RTE_SECURITY_IPSEC_SA_DIR_INGRESS,
1420                                 .options = { .udp_encap = 1,
1421                                                 .stats = 1, .esn = 1 },
1422                         },
1423                         .ol_flags = 0
1424                 },
1425                 { /* IPsec Inline Crypto ESP Transport Egress */
1426                         .action = RTE_SECURITY_ACTION_TYPE_INLINE_CRYPTO,
1427                         .protocol = RTE_SECURITY_PROTOCOL_IPSEC,
1428                         .ipsec = {
1429                                 .proto = RTE_SECURITY_IPSEC_SA_PROTO_ESP,
1430                                 .mode = RTE_SECURITY_IPSEC_SA_MODE_TRANSPORT,
1431                                 .direction = RTE_SECURITY_IPSEC_SA_DIR_EGRESS,
1432                                 .options = { .udp_encap = 1, .stats = 1,
1433                                                 .esn = 1 },
1434                         },
1435                         .ol_flags = RTE_SECURITY_TX_OLOAD_NEED_MDATA
1436                 },
1437                 { /* IPsec Inline Crypto ESP Transport Ingress */
1438                         .action = RTE_SECURITY_ACTION_TYPE_INLINE_CRYPTO,
1439                         .protocol = RTE_SECURITY_PROTOCOL_IPSEC,
1440                         .ipsec = {
1441                                 .proto = RTE_SECURITY_IPSEC_SA_PROTO_ESP,
1442                                 .mode = RTE_SECURITY_IPSEC_SA_MODE_TRANSPORT,
1443                                 .direction = RTE_SECURITY_IPSEC_SA_DIR_INGRESS,
1444                                 .options = { .udp_encap = 1, .stats = 1,
1445                                                 .esn = 1 }
1446                         },
1447                         .ol_flags = 0
1448                 },
1449                 {
1450                         .action = RTE_SECURITY_ACTION_TYPE_NONE
1451                 }
1452         };
1453
1454         /**
1455          * Update the security capabilities struct with the runtime discovered
1456          * crypto capabilities, except for last element of the array which is
1457          * the null terminatation
1458          */
1459         for (i = 0; i < ((sizeof(iavf_security_capabilities) /
1460                         sizeof(iavf_security_capabilities[0])) - 1); i++) {
1461                 iavf_security_capabilities[i].crypto_capabilities =
1462                         iavf_sctx->crypto_capabilities;
1463         }
1464
1465         return iavf_security_capabilities;
1466 }
1467
1468 static struct rte_security_ops iavf_ipsec_crypto_ops = {
1469         .session_get_size               = iavf_ipsec_crypto_session_size_get,
1470         .session_create                 = iavf_ipsec_crypto_session_create,
1471         .session_update                 = iavf_ipsec_crypto_session_update,
1472         .session_stats_get              = iavf_ipsec_crypto_session_stats_get,
1473         .session_destroy                = iavf_ipsec_crypto_session_destroy,
1474         .set_pkt_metadata               = iavf_ipsec_crypto_pkt_metadata_set,
1475         .get_userdata                   = NULL,
1476         .capabilities_get               = iavf_ipsec_crypto_capabilities_get,
1477 };
1478
1479 int
1480 iavf_security_ctx_create(struct iavf_adapter *adapter)
1481 {
1482         struct rte_security_ctx *sctx;
1483
1484         sctx = rte_malloc("security_ctx", sizeof(struct rte_security_ctx), 0);
1485         if (sctx == NULL)
1486                 return -ENOMEM;
1487
1488         sctx->device = adapter->vf.eth_dev;
1489         sctx->ops = &iavf_ipsec_crypto_ops;
1490         sctx->sess_cnt = 0;
1491
1492         adapter->vf.eth_dev->security_ctx = sctx;
1493
1494         if (adapter->security_ctx == NULL) {
1495                 adapter->security_ctx = rte_malloc("iavf_security_ctx",
1496                                 sizeof(struct iavf_security_ctx), 0);
1497                 if (adapter->security_ctx == NULL)
1498                         return -ENOMEM;
1499         }
1500
1501         return 0;
1502 }
1503
1504 int
1505 iavf_security_init(struct iavf_adapter *adapter)
1506 {
1507         struct iavf_security_ctx *iavf_sctx = adapter->security_ctx;
1508         struct rte_mbuf_dynfield pkt_md_dynfield = {
1509                 .name = "iavf_ipsec_crypto_pkt_metadata",
1510                 .size = sizeof(struct iavf_ipsec_crypto_pkt_metadata),
1511                 .align = __alignof__(struct iavf_ipsec_crypto_pkt_metadata)
1512         };
1513         struct virtchnl_ipsec_cap capabilities;
1514         int rc;
1515
1516         iavf_sctx->adapter = adapter;
1517
1518         iavf_sctx->pkt_md_offset = rte_mbuf_dynfield_register(&pkt_md_dynfield);
1519         if (iavf_sctx->pkt_md_offset < 0)
1520                 return iavf_sctx->pkt_md_offset;
1521
1522         /* Get device capabilities from Inline IPsec driver over PF-VF comms */
1523         rc = iavf_ipsec_crypto_device_capabilities_get(adapter, &capabilities);
1524         if (rc)
1525                 return rc;
1526
1527         return  iavf_ipsec_crypto_set_security_capabililites(iavf_sctx,
1528                         &capabilities);
1529 }
1530
1531 int
1532 iavf_security_get_pkt_md_offset(struct iavf_adapter *adapter)
1533 {
1534         struct iavf_security_ctx *iavf_sctx = adapter->security_ctx;
1535
1536         return iavf_sctx->pkt_md_offset;
1537 }
1538
1539 int
1540 iavf_security_ctx_destroy(struct iavf_adapter *adapter)
1541 {
1542         struct rte_security_ctx *sctx  = adapter->vf.eth_dev->security_ctx;
1543         struct iavf_security_ctx *iavf_sctx = adapter->security_ctx;
1544
1545         if (iavf_sctx == NULL)
1546                 return -ENODEV;
1547
1548         /* TODO: Add resources cleanup */
1549
1550         /* free and reset security data structures */
1551         rte_free(iavf_sctx);
1552         rte_free(sctx);
1553
1554         iavf_sctx = NULL;
1555         sctx = NULL;
1556
1557         return 0;
1558 }
1559
1560 int
1561 iavf_ipsec_crypto_supported(struct iavf_adapter *adapter)
1562 {
1563         struct virtchnl_vf_resource *resources = adapter->vf.vf_res;
1564
1565         /** Capability check for IPsec Crypto */
1566         if (resources && (resources->vf_cap_flags &
1567                 VIRTCHNL_VF_OFFLOAD_INLINE_IPSEC_CRYPTO))
1568                 return true;
1569
1570         return false;
1571 }
1572
1573 #define IAVF_IPSEC_INSET_ESP (\
1574         IAVF_INSET_ESP_SPI)
1575
1576 #define IAVF_IPSEC_INSET_AH (\
1577         IAVF_INSET_AH_SPI)
1578
1579 #define IAVF_IPSEC_INSET_IPV4_NATT_ESP (\
1580         IAVF_INSET_IPV4_SRC | IAVF_INSET_IPV4_DST | \
1581         IAVF_INSET_ESP_SPI)
1582
1583 #define IAVF_IPSEC_INSET_IPV6_NATT_ESP (\
1584         IAVF_INSET_IPV6_SRC | IAVF_INSET_IPV6_DST | \
1585         IAVF_INSET_ESP_SPI)
1586
1587 enum iavf_ipsec_flow_pt_type {
1588         IAVF_PATTERN_ESP = 1,
1589         IAVF_PATTERN_AH,
1590         IAVF_PATTERN_UDP_ESP,
1591 };
1592 enum iavf_ipsec_flow_pt_ip_ver {
1593         IAVF_PATTERN_IPV4 = 1,
1594         IAVF_PATTERN_IPV6,
1595 };
1596
1597 #define IAVF_PATTERN(t, ipt) ((void *)((t) | ((ipt) << 4)))
1598 #define IAVF_PATTERN_TYPE(pt) ((pt) & 0x0F)
1599 #define IAVF_PATTERN_IP_V(pt) ((pt) >> 4)
1600
1601 static struct iavf_pattern_match_item iavf_ipsec_flow_pattern[] = {
1602         {iavf_pattern_eth_ipv4_esp,     IAVF_IPSEC_INSET_ESP,
1603                         IAVF_PATTERN(IAVF_PATTERN_ESP, IAVF_PATTERN_IPV4)},
1604         {iavf_pattern_eth_ipv6_esp,     IAVF_IPSEC_INSET_ESP,
1605                         IAVF_PATTERN(IAVF_PATTERN_ESP, IAVF_PATTERN_IPV6)},
1606         {iavf_pattern_eth_ipv4_ah,      IAVF_IPSEC_INSET_AH,
1607                         IAVF_PATTERN(IAVF_PATTERN_AH, IAVF_PATTERN_IPV4)},
1608         {iavf_pattern_eth_ipv6_ah,      IAVF_IPSEC_INSET_AH,
1609                         IAVF_PATTERN(IAVF_PATTERN_AH, IAVF_PATTERN_IPV6)},
1610         {iavf_pattern_eth_ipv4_udp_esp, IAVF_IPSEC_INSET_IPV4_NATT_ESP,
1611                         IAVF_PATTERN(IAVF_PATTERN_UDP_ESP, IAVF_PATTERN_IPV4)},
1612         {iavf_pattern_eth_ipv6_udp_esp, IAVF_IPSEC_INSET_IPV6_NATT_ESP,
1613                         IAVF_PATTERN(IAVF_PATTERN_UDP_ESP, IAVF_PATTERN_IPV6)},
1614 };
1615
1616 struct iavf_ipsec_flow_item {
1617         uint64_t id;
1618         uint8_t is_ipv4;
1619         uint32_t spi;
1620         struct rte_ether_hdr eth_hdr;
1621         union {
1622                 struct rte_ipv4_hdr ipv4_hdr;
1623                 struct rte_ipv6_hdr ipv6_hdr;
1624         };
1625         struct rte_udp_hdr udp_hdr;
1626 };
1627
1628 static void
1629 parse_eth_item(const struct rte_flow_item_eth *item,
1630                 struct rte_ether_hdr *eth)
1631 {
1632         memcpy(eth->src_addr.addr_bytes,
1633                         item->src.addr_bytes, sizeof(eth->src_addr));
1634         memcpy(eth->dst_addr.addr_bytes,
1635                         item->dst.addr_bytes, sizeof(eth->dst_addr));
1636 }
1637
1638 static void
1639 parse_ipv4_item(const struct rte_flow_item_ipv4 *item,
1640                 struct rte_ipv4_hdr *ipv4)
1641 {
1642         ipv4->src_addr = item->hdr.src_addr;
1643         ipv4->dst_addr = item->hdr.dst_addr;
1644 }
1645
1646 static void
1647 parse_ipv6_item(const struct rte_flow_item_ipv6 *item,
1648                 struct rte_ipv6_hdr *ipv6)
1649 {
1650         memcpy(ipv6->src_addr, item->hdr.src_addr, 16);
1651         memcpy(ipv6->dst_addr, item->hdr.dst_addr, 16);
1652 }
1653
1654 static void
1655 parse_udp_item(const struct rte_flow_item_udp *item, struct rte_udp_hdr *udp)
1656 {
1657         udp->dst_port = item->hdr.dst_port;
1658         udp->src_port = item->hdr.src_port;
1659 }
1660
1661 static int
1662 has_security_action(const struct rte_flow_action actions[],
1663         const void **session)
1664 {
1665         /* only {SECURITY; END} supported */
1666         if (actions[0].type == RTE_FLOW_ACTION_TYPE_SECURITY &&
1667                 actions[1].type == RTE_FLOW_ACTION_TYPE_END) {
1668                 *session = actions[0].conf;
1669                 return true;
1670         }
1671         return false;
1672 }
1673
1674 static struct iavf_ipsec_flow_item *
1675 iavf_ipsec_flow_item_parse(struct rte_eth_dev *ethdev,
1676                 const struct rte_flow_item pattern[],
1677                 const struct rte_flow_action actions[],
1678                 uint32_t type)
1679 {
1680         const void *session;
1681         struct iavf_ipsec_flow_item
1682                 *ipsec_flow = rte_malloc("security-flow-rule",
1683                 sizeof(struct iavf_ipsec_flow_item), 0);
1684         enum iavf_ipsec_flow_pt_type p_type = IAVF_PATTERN_TYPE(type);
1685         enum iavf_ipsec_flow_pt_ip_ver p_ip_type = IAVF_PATTERN_IP_V(type);
1686
1687         if (ipsec_flow == NULL)
1688                 return NULL;
1689
1690         ipsec_flow->is_ipv4 = (p_ip_type == IAVF_PATTERN_IPV4);
1691
1692         if (pattern[0].spec)
1693                 parse_eth_item((const struct rte_flow_item_eth *)
1694                                 pattern[0].spec, &ipsec_flow->eth_hdr);
1695
1696         switch (p_type) {
1697         case IAVF_PATTERN_ESP:
1698                 if (ipsec_flow->is_ipv4) {
1699                         parse_ipv4_item((const struct rte_flow_item_ipv4 *)
1700                                         pattern[1].spec,
1701                                         &ipsec_flow->ipv4_hdr);
1702                 } else {
1703                         parse_ipv6_item((const struct rte_flow_item_ipv6 *)
1704                                         pattern[1].spec,
1705                                         &ipsec_flow->ipv6_hdr);
1706                 }
1707                 ipsec_flow->spi =
1708                         ((const struct rte_flow_item_esp *)
1709                                         pattern[2].spec)->hdr.spi;
1710                 break;
1711         case IAVF_PATTERN_AH:
1712                 if (ipsec_flow->is_ipv4) {
1713                         parse_ipv4_item((const struct rte_flow_item_ipv4 *)
1714                                         pattern[1].spec,
1715                                         &ipsec_flow->ipv4_hdr);
1716                 } else {
1717                         parse_ipv6_item((const struct rte_flow_item_ipv6 *)
1718                                         pattern[1].spec,
1719                                         &ipsec_flow->ipv6_hdr);
1720                 }
1721                 ipsec_flow->spi =
1722                         ((const struct rte_flow_item_ah *)
1723                                         pattern[2].spec)->spi;
1724                 break;
1725         case IAVF_PATTERN_UDP_ESP:
1726                 if (ipsec_flow->is_ipv4) {
1727                         parse_ipv4_item((const struct rte_flow_item_ipv4 *)
1728                                         pattern[1].spec,
1729                                         &ipsec_flow->ipv4_hdr);
1730                 } else {
1731                         parse_ipv6_item((const struct rte_flow_item_ipv6 *)
1732                                         pattern[1].spec,
1733                                         &ipsec_flow->ipv6_hdr);
1734                 }
1735                 parse_udp_item((const struct rte_flow_item_udp *)
1736                                 pattern[2].spec,
1737                         &ipsec_flow->udp_hdr);
1738                 ipsec_flow->spi =
1739                         ((const struct rte_flow_item_esp *)
1740                                         pattern[3].spec)->hdr.spi;
1741                 break;
1742         default:
1743                 goto flow_cleanup;
1744         }
1745
1746         if (!has_security_action(actions, &session))
1747                 goto flow_cleanup;
1748
1749         if (!iavf_ipsec_crypto_action_valid(ethdev, session,
1750                         ipsec_flow->spi))
1751                 goto flow_cleanup;
1752
1753         return ipsec_flow;
1754
1755 flow_cleanup:
1756         rte_free(ipsec_flow);
1757         return NULL;
1758 }
1759
1760
1761 static struct iavf_flow_parser iavf_ipsec_flow_parser;
1762
1763 static int
1764 iavf_ipsec_flow_init(struct iavf_adapter *ad)
1765 {
1766         struct iavf_info *vf = IAVF_DEV_PRIVATE_TO_VF(ad);
1767         struct iavf_flow_parser *parser;
1768
1769         if (!vf->vf_res)
1770                 return -EINVAL;
1771
1772         if (vf->vf_res->vf_cap_flags & VIRTCHNL_VF_OFFLOAD_INLINE_IPSEC_CRYPTO)
1773                 parser = &iavf_ipsec_flow_parser;
1774         else
1775                 return -ENOTSUP;
1776
1777         return iavf_register_parser(parser, ad);
1778 }
1779
1780 static void
1781 iavf_ipsec_flow_uninit(struct iavf_adapter *ad)
1782 {
1783         iavf_unregister_parser(&iavf_ipsec_flow_parser, ad);
1784 }
1785
1786 static int
1787 iavf_ipsec_flow_create(struct iavf_adapter *ad,
1788                 struct rte_flow *flow,
1789                 void *meta,
1790                 struct rte_flow_error *error)
1791 {
1792         struct iavf_ipsec_flow_item *ipsec_flow = meta;
1793         if (!ipsec_flow) {
1794                 rte_flow_error_set(error, EINVAL,
1795                                 RTE_FLOW_ERROR_TYPE_HANDLE, NULL,
1796                                 "NULL rule.");
1797                 return -rte_errno;
1798         }
1799
1800         if (ipsec_flow->is_ipv4) {
1801                 ipsec_flow->id =
1802                         iavf_ipsec_crypto_inbound_security_policy_add(ad,
1803                         ipsec_flow->spi,
1804                         1,
1805                         ipsec_flow->ipv4_hdr.dst_addr,
1806                         NULL,
1807                         0);
1808         } else {
1809                 ipsec_flow->id =
1810                         iavf_ipsec_crypto_inbound_security_policy_add(ad,
1811                         ipsec_flow->spi,
1812                         0,
1813                         0,
1814                         ipsec_flow->ipv6_hdr.dst_addr,
1815                         0);
1816         }
1817
1818         if (ipsec_flow->id < 1) {
1819                 rte_flow_error_set(error, EINVAL,
1820                                 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
1821                                 "Failed to add SA.");
1822                 return -rte_errno;
1823         }
1824
1825         flow->rule = ipsec_flow;
1826
1827         return 0;
1828 }
1829
1830 static int
1831 iavf_ipsec_flow_destroy(struct iavf_adapter *ad,
1832                 struct rte_flow *flow,
1833                 struct rte_flow_error *error)
1834 {
1835         struct iavf_ipsec_flow_item *ipsec_flow = flow->rule;
1836         if (!ipsec_flow) {
1837                 rte_flow_error_set(error, EINVAL,
1838                                 RTE_FLOW_ERROR_TYPE_HANDLE, NULL,
1839                                 "NULL rule.");
1840                 return -rte_errno;
1841         }
1842
1843         iavf_ipsec_crypto_security_policy_delete(ad,
1844                         ipsec_flow->is_ipv4, ipsec_flow->id);
1845         rte_free(ipsec_flow);
1846         return 0;
1847 }
1848
1849 static struct iavf_flow_engine iavf_ipsec_flow_engine = {
1850         .init = iavf_ipsec_flow_init,
1851         .uninit = iavf_ipsec_flow_uninit,
1852         .create = iavf_ipsec_flow_create,
1853         .destroy = iavf_ipsec_flow_destroy,
1854         .type = IAVF_FLOW_ENGINE_IPSEC_CRYPTO,
1855 };
1856
1857 static int
1858 iavf_ipsec_flow_parse(struct iavf_adapter *ad,
1859                        struct iavf_pattern_match_item *array,
1860                        uint32_t array_len,
1861                        const struct rte_flow_item pattern[],
1862                        const struct rte_flow_action actions[],
1863                        void **meta,
1864                        struct rte_flow_error *error)
1865 {
1866         struct iavf_pattern_match_item *item = NULL;
1867         int ret = -1;
1868
1869         item = iavf_search_pattern_match_item(pattern, array, array_len, error);
1870         if (item && item->meta) {
1871                 uint32_t type = (uint64_t)(item->meta);
1872                 struct iavf_ipsec_flow_item *fi =
1873                                 iavf_ipsec_flow_item_parse(ad->vf.eth_dev,
1874                                                 pattern, actions, type);
1875                 if (fi && meta) {
1876                         *meta = fi;
1877                         ret = 0;
1878                 }
1879         }
1880         return ret;
1881 }
1882
1883 static struct iavf_flow_parser iavf_ipsec_flow_parser = {
1884         .engine = &iavf_ipsec_flow_engine,
1885         .array = iavf_ipsec_flow_pattern,
1886         .array_len = RTE_DIM(iavf_ipsec_flow_pattern),
1887         .parse_pattern_action = iavf_ipsec_flow_parse,
1888         .stage = IAVF_FLOW_STAGE_IPSEC_CRYPTO,
1889 };
1890
1891 RTE_INIT(iavf_ipsec_flow_engine_register)
1892 {
1893         iavf_register_flow_engine(&iavf_ipsec_flow_engine);
1894 }