drivers/net/iavf/iavf_ipsec_crypto.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(c) 2020 Intel Corporation
   3  */
   4
   5 #include <rte_cryptodev.h>
   6 #include <rte_ethdev.h>
   7 #include <rte_security_driver.h>
   8 #include <rte_security.h>
   9
  10 #include "iavf.h"
  11 #include "iavf_rxtx.h"
  12 #include "iavf_log.h"
  13 #include "iavf_generic_flow.h"
  14
  15 #include "iavf_ipsec_crypto.h"
  16 #include "iavf_ipsec_crypto_capabilities.h"
  17
  18 /**
  19  * iAVF IPsec Crypto Security Context
  20  */
  21 struct iavf_security_ctx {
  22         struct iavf_adapter *adapter;
  23         int pkt_md_offset;
  24         struct rte_cryptodev_capabilities *crypto_capabilities;
  25 };
  26
  27 /**
  28  * iAVF IPsec Crypto Security Session Parameters
  29  */
  30 struct iavf_security_session {
  31         struct iavf_adapter *adapter;
  32
  33         enum rte_security_ipsec_sa_mode mode;
  34         enum rte_security_ipsec_tunnel_type type;
  35         enum rte_security_ipsec_sa_direction direction;
  36
  37         struct {
  38                 uint32_t spi; /* Security Parameter Index */
  39                 uint32_t hw_idx; /* SA Index in hardware table */
  40         } sa;
  41
  42         struct {
  43                 uint8_t enabled :1;
  44                 union {
  45                         uint64_t value;
  46                         struct {
  47                                 uint32_t hi;
  48                                 uint32_t low;
  49                         };
  50                 };
  51         } esn;
  52
  53         struct {
  54                 uint8_t enabled :1;
  55         } udp_encap;
  56
  57         size_t iv_sz;
  58         size_t icv_sz;
  59         size_t block_sz;
  60
  61         struct iavf_ipsec_crypto_pkt_metadata pkt_metadata_template;
  62 };
  63 /**
  64  *  IV Length field in IPsec Tx Desc uses the following encoding:
  65  *
  66  *  0B - 0
  67  *  4B - 1
  68  *  8B - 2
  69  *  16B - 3
  70  *
  71  * but we also need the IV Length for TSO to correctly calculate the total
  72  * header length so placing it in the upper 6-bits here for easier retrieval.
  73  */
  74 static inline uint8_t
  75 calc_ipsec_desc_iv_len_field(uint16_t iv_sz)
  76 {
  77         uint8_t iv_length = IAVF_IPSEC_IV_LEN_NONE;
  78
  79         switch (iv_sz) {
  80         case 4:
  81                 iv_length = IAVF_IPSEC_IV_LEN_DW;
  82                 break;
  83         case 8:
  84                 iv_length = IAVF_IPSEC_IV_LEN_DDW;
  85                 break;
  86         case 16:
  87                 iv_length = IAVF_IPSEC_IV_LEN_QDW;
  88                 break;
  89         }
  90
  91         return (iv_sz << 2) | iv_length;
  92 }
  93
  94 static unsigned int
  95 iavf_ipsec_crypto_session_size_get(void *device __rte_unused)
  96 {
  97         return sizeof(struct iavf_security_session);
  98 }
  99
 100 static const struct rte_cryptodev_symmetric_capability *
 101 get_capability(struct iavf_security_ctx *iavf_sctx,
 102         uint32_t algo, uint32_t type)
 103 {
 104         const struct rte_cryptodev_capabilities *capability;
 105         int i = 0;
 106
 107         capability = &iavf_sctx->crypto_capabilities[i];
 108
 109         while (capability->op != RTE_CRYPTO_OP_TYPE_UNDEFINED) {
 110                 if (capability->op == RTE_CRYPTO_OP_TYPE_SYMMETRIC &&
 111                         (uint32_t)capability->sym.xform_type == type &&
 112                         (uint32_t)capability->sym.cipher.algo == algo)
 113                         return &capability->sym;
 114                 /** try next capability */
 115                 capability = &iavf_crypto_capabilities[i++];
 116         }
 117
 118         return NULL;
 119 }
 120
 121 static const struct rte_cryptodev_symmetric_capability *
 122 get_auth_capability(struct iavf_security_ctx *iavf_sctx,
 123         enum rte_crypto_auth_algorithm algo)
 124 {
 125         return get_capability(iavf_sctx, algo, RTE_CRYPTO_SYM_XFORM_AUTH);
 126 }
 127
 128 static const struct rte_cryptodev_symmetric_capability *
 129 get_cipher_capability(struct iavf_security_ctx *iavf_sctx,
 130         enum rte_crypto_cipher_algorithm algo)
 131 {
 132         return get_capability(iavf_sctx, algo, RTE_CRYPTO_SYM_XFORM_CIPHER);
 133 }
 134 static const struct rte_cryptodev_symmetric_capability *
 135 get_aead_capability(struct iavf_security_ctx *iavf_sctx,
 136         enum rte_crypto_aead_algorithm algo)
 137 {
 138         return get_capability(iavf_sctx, algo, RTE_CRYPTO_SYM_XFORM_AEAD);
 139 }
 140
 141 static uint16_t
 142 get_cipher_blocksize(struct iavf_security_ctx *iavf_sctx,
 143         enum rte_crypto_cipher_algorithm algo)
 144 {
 145         const struct rte_cryptodev_symmetric_capability *capability;
 146
 147         capability = get_cipher_capability(iavf_sctx, algo);
 148         if (capability == NULL)
 149                 return 0;
 150
 151         return capability->cipher.block_size;
 152 }
 153
 154 static uint16_t
 155 get_aead_blocksize(struct iavf_security_ctx *iavf_sctx,
 156         enum rte_crypto_aead_algorithm algo)
 157 {
 158         const struct rte_cryptodev_symmetric_capability *capability;
 159
 160         capability = get_aead_capability(iavf_sctx, algo);
 161         if (capability == NULL)
 162                 return 0;
 163
 164         return capability->cipher.block_size;
 165 }
 166
 167 static uint16_t
 168 get_auth_blocksize(struct iavf_security_ctx *iavf_sctx,
 169         enum rte_crypto_auth_algorithm algo)
 170 {
 171         const struct rte_cryptodev_symmetric_capability *capability;
 172
 173         capability = get_auth_capability(iavf_sctx, algo);
 174         if (capability == NULL)
 175                 return 0;
 176
 177         return capability->auth.block_size;
 178 }
 179
 180 static uint8_t
 181 calc_context_desc_cipherblock_sz(size_t len)
 182 {
 183         switch (len) {
 184         case 8:
 185                 return 0x2;
 186         case 16:
 187                 return 0x3;
 188         default:
 189                 return 0x0;
 190         }
 191 }
 192
 193 static int
 194 valid_length(uint32_t len, uint32_t min, uint32_t max, uint32_t increment)
 195 {
 196         if (len < min || len > max)
 197                 return false;
 198
 199         if (increment == 0)
 200                 return true;
 201
 202         if ((len - min) % increment)
 203                 return false;
 204
 205         /* make sure it fits in the key array */
 206         if (len > VIRTCHNL_IPSEC_MAX_KEY_LEN)
 207                 return false;
 208
 209         return true;
 210 }
 211
 212 static int
 213 valid_auth_xform(struct iavf_security_ctx *iavf_sctx,
 214         struct rte_crypto_auth_xform *auth)
 215 {
 216         const struct rte_cryptodev_symmetric_capability *capability;
 217
 218         capability = get_auth_capability(iavf_sctx, auth->algo);
 219         if (capability == NULL)
 220                 return false;
 221
 222         /* verify key size */
 223         if (!valid_length(auth->key.length,
 224                 capability->auth.key_size.min,
 225                 capability->auth.key_size.max,
 226                 capability->aead.key_size.increment))
 227                 return false;
 228
 229         return true;
 230 }
 231
 232 static int
 233 valid_cipher_xform(struct iavf_security_ctx *iavf_sctx,
 234         struct rte_crypto_cipher_xform *cipher)
 235 {
 236         const struct rte_cryptodev_symmetric_capability *capability;
 237
 238         capability = get_cipher_capability(iavf_sctx, cipher->algo);
 239         if (capability == NULL)
 240                 return false;
 241
 242         /* verify key size */
 243         if (!valid_length(cipher->key.length,
 244                 capability->cipher.key_size.min,
 245                 capability->cipher.key_size.max,
 246                 capability->cipher.key_size.increment))
 247                 return false;
 248
 249         return true;
 250 }
 251
 252 static int
 253 valid_aead_xform(struct iavf_security_ctx *iavf_sctx,
 254         struct rte_crypto_aead_xform *aead)
 255 {
 256         const struct rte_cryptodev_symmetric_capability *capability;
 257
 258         capability = get_aead_capability(iavf_sctx, aead->algo);
 259         if (capability == NULL)
 260                 return false;
 261
 262         /* verify key size */
 263         if (!valid_length(aead->key.length,
 264                 capability->aead.key_size.min,
 265                 capability->aead.key_size.max,
 266                 capability->aead.key_size.increment))
 267                 return false;
 268
 269         return true;
 270 }
 271
 272 static int
 273 iavf_ipsec_crypto_session_validate_conf(struct iavf_security_ctx *iavf_sctx,
 274         struct rte_security_session_conf *conf)
 275 {
 276         /** validate security action/protocol selection */
 277         if (conf->action_type != RTE_SECURITY_ACTION_TYPE_INLINE_CRYPTO ||
 278                 conf->protocol != RTE_SECURITY_PROTOCOL_IPSEC) {
 279                 PMD_DRV_LOG(ERR, "Invalid action / protocol specified");
 280                 return -EINVAL;
 281         }
 282
 283         /** validate IPsec protocol selection */
 284         if (conf->ipsec.proto != RTE_SECURITY_IPSEC_SA_PROTO_ESP) {
 285                 PMD_DRV_LOG(ERR, "Invalid IPsec protocol specified");
 286                 return -EINVAL;
 287         }
 288
 289         /** validate selected options */
 290         if (conf->ipsec.options.copy_dscp ||
 291                 conf->ipsec.options.copy_flabel ||
 292                 conf->ipsec.options.copy_df ||
 293                 conf->ipsec.options.dec_ttl ||
 294                 conf->ipsec.options.ecn ||
 295                 conf->ipsec.options.stats) {
 296                 PMD_DRV_LOG(ERR, "Invalid IPsec option specified");
 297                 return -EINVAL;
 298         }
 299
 300         /**
 301          * Validate crypto xforms parameters.
 302          *
 303          * AEAD transforms can be used for either inbound/outbound IPsec SAs,
 304          * for non-AEAD crypto transforms we explicitly only support CIPHER/AUTH
 305          * for outbound and AUTH/CIPHER chained transforms for inbound IPsec.
 306          */
 307         if (conf->crypto_xform->type == RTE_CRYPTO_SYM_XFORM_AEAD) {
 308                 if (!valid_aead_xform(iavf_sctx, &conf->crypto_xform->aead)) {
 309                         PMD_DRV_LOG(ERR, "Invalid IPsec option specified");
 310                         return -EINVAL;
 311                 }
 312         } else if (conf->ipsec.direction == RTE_SECURITY_IPSEC_SA_DIR_EGRESS &&
 313                 conf->crypto_xform->type == RTE_CRYPTO_SYM_XFORM_CIPHER &&
 314                 conf->crypto_xform->next &&
 315                 conf->crypto_xform->next->type == RTE_CRYPTO_SYM_XFORM_AUTH) {
 316                 if (!valid_cipher_xform(iavf_sctx,
 317                                 &conf->crypto_xform->cipher)) {
 318                         PMD_DRV_LOG(ERR, "Invalid IPsec option specified");
 319                         return -EINVAL;
 320                 }
 321
 322                 if (!valid_auth_xform(iavf_sctx,
 323                                 &conf->crypto_xform->next->auth)) {
 324                         PMD_DRV_LOG(ERR, "Invalid IPsec option specified");
 325                         return -EINVAL;
 326                 }
 327         } else if (conf->ipsec.direction == RTE_SECURITY_IPSEC_SA_DIR_INGRESS &&
 328                 conf->crypto_xform->type == RTE_CRYPTO_SYM_XFORM_AUTH &&
 329                 conf->crypto_xform->next &&
 330                 conf->crypto_xform->next->type == RTE_CRYPTO_SYM_XFORM_CIPHER) {
 331                 if (!valid_auth_xform(iavf_sctx, &conf->crypto_xform->auth)) {
 332                         PMD_DRV_LOG(ERR, "Invalid IPsec option specified");
 333                         return -EINVAL;
 334                 }
 335
 336                 if (!valid_cipher_xform(iavf_sctx,
 337                                 &conf->crypto_xform->next->cipher)) {
 338                         PMD_DRV_LOG(ERR, "Invalid IPsec option specified");
 339                         return -EINVAL;
 340                 }
 341         }
 342
 343         return 0;
 344 }
 345
 346 static void
 347 sa_add_set_aead_params(struct virtchnl_ipsec_crypto_cfg_item *cfg,
 348         struct rte_crypto_aead_xform *aead, uint32_t salt)
 349 {
 350         cfg->crypto_type = VIRTCHNL_AEAD;
 351
 352         switch (aead->algo) {
 353         case RTE_CRYPTO_AEAD_AES_CCM:
 354                 cfg->algo_type = VIRTCHNL_AES_CCM; break;
 355         case RTE_CRYPTO_AEAD_AES_GCM:
 356                 cfg->algo_type = VIRTCHNL_AES_GCM; break;
 357         case RTE_CRYPTO_AEAD_CHACHA20_POLY1305:
 358                 cfg->algo_type = VIRTCHNL_CHACHA20_POLY1305; break;
 359         default:
 360                 PMD_DRV_LOG(ERR, "Invalid AEAD parameters");
 361                 break;
 362         }
 363
 364         cfg->key_len = aead->key.length;
 365         cfg->iv_len = sizeof(uint64_t); /* iv.length includes salt len */
 366         cfg->digest_len = aead->digest_length;
 367         cfg->salt = salt;
 368
 369         memcpy(cfg->key_data, aead->key.data, cfg->key_len);
 370 }
 371
 372 static void
 373 sa_add_set_cipher_params(struct virtchnl_ipsec_crypto_cfg_item *cfg,
 374         struct rte_crypto_cipher_xform *cipher, uint32_t salt)
 375 {
 376         cfg->crypto_type = VIRTCHNL_CIPHER;
 377
 378         switch (cipher->algo) {
 379         case RTE_CRYPTO_CIPHER_AES_CBC:
 380                 cfg->algo_type = VIRTCHNL_AES_CBC; break;
 381         case RTE_CRYPTO_CIPHER_3DES_CBC:
 382                 cfg->algo_type = VIRTCHNL_3DES_CBC; break;
 383         case RTE_CRYPTO_CIPHER_NULL:
 384                 cfg->algo_type = VIRTCHNL_CIPHER_NO_ALG; break;
 385         case RTE_CRYPTO_CIPHER_AES_CTR:
 386                 cfg->algo_type = VIRTCHNL_AES_CTR;
 387                 cfg->salt = salt;
 388                 break;
 389         default:
 390                 PMD_DRV_LOG(ERR, "Invalid cipher parameters");
 391                 break;
 392         }
 393
 394         cfg->key_len = cipher->key.length;
 395         cfg->iv_len = cipher->iv.length;
 396         cfg->salt = salt;
 397
 398         memcpy(cfg->key_data, cipher->key.data, cfg->key_len);
 399 }
 400
 401 static void
 402 sa_add_set_auth_params(struct virtchnl_ipsec_crypto_cfg_item *cfg,
 403         struct rte_crypto_auth_xform *auth, uint32_t salt)
 404 {
 405         cfg->crypto_type = VIRTCHNL_AUTH;
 406
 407         switch (auth->algo) {
 408         case RTE_CRYPTO_AUTH_NULL:
 409                 cfg->algo_type = VIRTCHNL_HASH_NO_ALG; break;
 410         case RTE_CRYPTO_AUTH_AES_CBC_MAC:
 411                 cfg->algo_type = VIRTCHNL_AES_CBC_MAC; break;
 412         case RTE_CRYPTO_AUTH_AES_CMAC:
 413                 cfg->algo_type = VIRTCHNL_AES_CMAC; break;
 414         case RTE_CRYPTO_AUTH_AES_XCBC_MAC:
 415                 cfg->algo_type = VIRTCHNL_AES_XCBC_MAC; break;
 416         case RTE_CRYPTO_AUTH_MD5_HMAC:
 417                 cfg->algo_type = VIRTCHNL_MD5_HMAC; break;
 418         case RTE_CRYPTO_AUTH_SHA1_HMAC:
 419                 cfg->algo_type = VIRTCHNL_SHA1_HMAC; break;
 420         case RTE_CRYPTO_AUTH_SHA224_HMAC:
 421                 cfg->algo_type = VIRTCHNL_SHA224_HMAC; break;
 422         case RTE_CRYPTO_AUTH_SHA256_HMAC:
 423                 cfg->algo_type = VIRTCHNL_SHA256_HMAC; break;
 424         case RTE_CRYPTO_AUTH_SHA384_HMAC:
 425                 cfg->algo_type = VIRTCHNL_SHA384_HMAC; break;
 426         case RTE_CRYPTO_AUTH_SHA512_HMAC:
 427                 cfg->algo_type = VIRTCHNL_SHA512_HMAC; break;
 428         case RTE_CRYPTO_AUTH_AES_GMAC:
 429                 cfg->algo_type = VIRTCHNL_AES_GMAC;
 430                 cfg->salt = salt;
 431                 break;
 432         default:
 433                 PMD_DRV_LOG(ERR, "Invalid auth parameters");
 434                 break;
 435         }
 436
 437         cfg->key_len = auth->key.length;
 438         /* special case for RTE_CRYPTO_AUTH_AES_GMAC */
 439         if (auth->algo == RTE_CRYPTO_AUTH_AES_GMAC)
 440                 cfg->iv_len = sizeof(uint64_t); /* iv.length includes salt */
 441         else
 442                 cfg->iv_len = auth->iv.length;
 443         cfg->digest_len = auth->digest_length;
 444
 445         memcpy(cfg->key_data, auth->key.data, cfg->key_len);
 446 }
 447
 448 /**
 449  * Send SA add virtual channel request to Inline IPsec driver.
 450  *
 451  * Inline IPsec driver expects SPI and destination IP address to be in host
 452  * order, but DPDK APIs are network order, therefore we need to do a htonl
 453  * conversion of these parameters.
 454  */
 455 static uint32_t
 456 iavf_ipsec_crypto_security_association_add(struct iavf_adapter *adapter,
 457         struct rte_security_session_conf *conf)
 458 {
 459         struct inline_ipsec_msg *request = NULL, *response = NULL;
 460         struct virtchnl_ipsec_sa_cfg *sa_cfg;
 461         size_t request_len, response_len;
 462
 463         int rc;
 464
 465         request_len = sizeof(struct inline_ipsec_msg) +
 466                         sizeof(struct virtchnl_ipsec_sa_cfg);
 467
 468         request = rte_malloc("iavf-sad-add-request", request_len, 0);
 469         if (request == NULL) {
 470                 rc = -ENOMEM;
 471                 goto update_cleanup;
 472         }
 473
 474         response_len = sizeof(struct inline_ipsec_msg) +
 475                         sizeof(struct virtchnl_ipsec_sa_cfg_resp);
 476         response = rte_malloc("iavf-sad-add-response", response_len, 0);
 477         if (response == NULL) {
 478                 rc = -ENOMEM;
 479                 goto update_cleanup;
 480         }
 481
 482         /* set msg header params */
 483         request->ipsec_opcode = INLINE_IPSEC_OP_SA_CREATE;
 484         request->req_id = (uint16_t)0xDEADBEEF;
 485
 486         /* set SA configuration params */
 487         sa_cfg = (struct virtchnl_ipsec_sa_cfg *)(request + 1);
 488
 489         sa_cfg->spi = conf->ipsec.spi;
 490         sa_cfg->virtchnl_protocol_type = VIRTCHNL_PROTO_ESP;
 491         sa_cfg->virtchnl_direction =
 492                 conf->ipsec.direction == RTE_SECURITY_IPSEC_SA_DIR_INGRESS ?
 493                         VIRTCHNL_DIR_INGRESS : VIRTCHNL_DIR_EGRESS;
 494
 495         if (conf->ipsec.options.esn) {
 496                 sa_cfg->esn_enabled = 1;
 497                 sa_cfg->esn_hi = conf->ipsec.esn.hi;
 498                 sa_cfg->esn_low = conf->ipsec.esn.low;
 499         }
 500
 501         if (conf->ipsec.options.udp_encap)
 502                 sa_cfg->udp_encap_enabled = 1;
 503
 504         /* Set outer IP params */
 505         if (conf->ipsec.tunnel.type == RTE_SECURITY_IPSEC_TUNNEL_IPV4) {
 506                 sa_cfg->virtchnl_ip_type = VIRTCHNL_IPV4;
 507
 508                 *((uint32_t *)sa_cfg->dst_addr) =
 509                         htonl(conf->ipsec.tunnel.ipv4.dst_ip.s_addr);
 510         } else {
 511                 uint32_t *v6_dst_addr =
 512                         (uint32_t *)conf->ipsec.tunnel.ipv6.dst_addr.s6_addr;
 513
 514                 sa_cfg->virtchnl_ip_type = VIRTCHNL_IPV6;
 515
 516                 ((uint32_t *)sa_cfg->dst_addr)[0] = htonl(v6_dst_addr[0]);
 517                 ((uint32_t *)sa_cfg->dst_addr)[1] = htonl(v6_dst_addr[1]);
 518                 ((uint32_t *)sa_cfg->dst_addr)[2] = htonl(v6_dst_addr[2]);
 519                 ((uint32_t *)sa_cfg->dst_addr)[3] = htonl(v6_dst_addr[3]);
 520         }
 521
 522         /* set crypto params */
 523         if (conf->crypto_xform->type == RTE_CRYPTO_SYM_XFORM_AEAD) {
 524                 sa_add_set_aead_params(&sa_cfg->crypto_cfg.items[0],
 525                         &conf->crypto_xform->aead, conf->ipsec.salt);
 526
 527         } else if (conf->crypto_xform->type == RTE_CRYPTO_SYM_XFORM_CIPHER) {
 528                 sa_add_set_cipher_params(&sa_cfg->crypto_cfg.items[0],
 529                         &conf->crypto_xform->cipher, conf->ipsec.salt);
 530                 sa_add_set_auth_params(&sa_cfg->crypto_cfg.items[1],
 531                         &conf->crypto_xform->next->auth, conf->ipsec.salt);
 532
 533         } else if (conf->crypto_xform->type == RTE_CRYPTO_SYM_XFORM_AUTH) {
 534                 sa_add_set_auth_params(&sa_cfg->crypto_cfg.items[0],
 535                         &conf->crypto_xform->auth, conf->ipsec.salt);
 536                 if (conf->crypto_xform->auth.algo != RTE_CRYPTO_AUTH_AES_GMAC)
 537                         sa_add_set_cipher_params(&sa_cfg->crypto_cfg.items[1],
 538                         &conf->crypto_xform->next->cipher, conf->ipsec.salt);
 539         }
 540
 541         /* send virtual channel request to add SA to hardware database */
 542         rc = iavf_ipsec_crypto_request(adapter,
 543                         (uint8_t *)request, request_len,
 544                         (uint8_t *)response, response_len);
 545         if (rc)
 546                 goto update_cleanup;
 547
 548         /* verify response id */
 549         if (response->ipsec_opcode != request->ipsec_opcode ||
 550                 response->req_id != request->req_id)
 551                 rc = -EFAULT;
 552         else
 553                 rc = response->ipsec_data.sa_cfg_resp->sa_handle;
 554 update_cleanup:
 555         rte_free(response);
 556         rte_free(request);
 557
 558         return rc;
 559 }
 560
 561 static void
 562 set_pkt_metadata_template(struct iavf_ipsec_crypto_pkt_metadata *template,
 563         struct iavf_security_session *sess)
 564 {
 565         template->sa_idx = sess->sa.hw_idx;
 566
 567         if (sess->udp_encap.enabled)
 568                 template->ol_flags = IAVF_IPSEC_CRYPTO_OL_FLAGS_NATT;
 569
 570         if (sess->esn.enabled)
 571                 template->ol_flags = IAVF_IPSEC_CRYPTO_OL_FLAGS_ESN;
 572
 573         template->len_iv = calc_ipsec_desc_iv_len_field(sess->iv_sz);
 574         template->ctx_desc_ipsec_params =
 575                         calc_context_desc_cipherblock_sz(sess->block_sz) |
 576                         ((uint8_t)(sess->icv_sz >> 2) << 3);
 577 }
 578
 579 static void
 580 set_session_parameter(struct iavf_security_ctx *iavf_sctx,
 581         struct iavf_security_session *sess,
 582         struct rte_security_session_conf *conf, uint32_t sa_idx)
 583 {
 584         sess->adapter = iavf_sctx->adapter;
 585
 586         sess->mode = conf->ipsec.mode;
 587         sess->direction = conf->ipsec.direction;
 588
 589         if (sess->mode == RTE_SECURITY_IPSEC_SA_MODE_TUNNEL)
 590                 sess->type = conf->ipsec.tunnel.type;
 591
 592         sess->sa.spi = conf->ipsec.spi;
 593         sess->sa.hw_idx = sa_idx;
 594
 595         if (conf->ipsec.options.esn) {
 596                 sess->esn.enabled = 1;
 597                 sess->esn.value = conf->ipsec.esn.value;
 598         }
 599
 600         if (conf->ipsec.options.udp_encap)
 601                 sess->udp_encap.enabled = 1;
 602
 603         if (conf->crypto_xform->type == RTE_CRYPTO_SYM_XFORM_AEAD) {
 604                 sess->block_sz = get_aead_blocksize(iavf_sctx,
 605                         conf->crypto_xform->aead.algo);
 606                 sess->iv_sz = sizeof(uint64_t); /* iv.length includes salt */
 607                 sess->icv_sz = conf->crypto_xform->aead.digest_length;
 608         } else if (conf->crypto_xform->type == RTE_CRYPTO_SYM_XFORM_CIPHER) {
 609                 sess->block_sz = get_cipher_blocksize(iavf_sctx,
 610                         conf->crypto_xform->cipher.algo);
 611                 sess->iv_sz = conf->crypto_xform->cipher.iv.length;
 612                 sess->icv_sz = conf->crypto_xform->next->auth.digest_length;
 613         } else if (conf->crypto_xform->type == RTE_CRYPTO_SYM_XFORM_AUTH) {
 614                 if (conf->crypto_xform->auth.algo == RTE_CRYPTO_AUTH_AES_GMAC) {
 615                         sess->block_sz = get_auth_blocksize(iavf_sctx,
 616                                 conf->crypto_xform->auth.algo);
 617                         sess->iv_sz = conf->crypto_xform->auth.iv.length;
 618                         sess->icv_sz = conf->crypto_xform->auth.digest_length;
 619                 } else {
 620                         sess->block_sz = get_cipher_blocksize(iavf_sctx,
 621                                 conf->crypto_xform->next->cipher.algo);
 622                         sess->iv_sz =
 623                                 conf->crypto_xform->next->cipher.iv.length;
 624                         sess->icv_sz = conf->crypto_xform->auth.digest_length;
 625                 }
 626         }
 627
 628         set_pkt_metadata_template(&sess->pkt_metadata_template, sess);
 629 }
 630
 631 /**
 632  * Create IPsec Security Association for inline IPsec Crypto offload.
 633  *
 634  * 1. validate session configuration parameters
 635  * 2. allocate session memory from mempool
 636  * 3. add SA to hardware database
 637  * 4. set session parameters
 638  * 5. create packet metadata template for datapath
 639  */
 640 static int
 641 iavf_ipsec_crypto_session_create(void *device,
 642                                  struct rte_security_session_conf *conf,
 643                                  struct rte_security_session *session,
 644                                  struct rte_mempool *mempool)
 645 {
 646         struct rte_eth_dev *ethdev = device;
 647         struct iavf_adapter *adapter =
 648                 IAVF_DEV_PRIVATE_TO_ADAPTER(ethdev->data->dev_private);
 649         struct iavf_security_ctx *iavf_sctx = adapter->security_ctx;
 650         struct iavf_security_session *iavf_session = NULL;
 651         int sa_idx;
 652         int ret = 0;
 653
 654         /* validate that all SA parameters are valid for device */
 655         ret = iavf_ipsec_crypto_session_validate_conf(iavf_sctx, conf);
 656         if (ret)
 657                 return ret;
 658
 659         /* allocate session context */
 660         if (rte_mempool_get(mempool, (void **)&iavf_session)) {
 661                 PMD_DRV_LOG(ERR, "Cannot get object from sess mempool");
 662                 return -ENOMEM;
 663         }
 664
 665         /* add SA to hardware database */
 666         sa_idx = iavf_ipsec_crypto_security_association_add(adapter, conf);
 667         if (sa_idx < 0) {
 668                 PMD_DRV_LOG(ERR,
 669                         "Failed to add SA (spi: %d, mode: %s, direction: %s)",
 670                         conf->ipsec.spi,
 671                         conf->ipsec.mode ==
 672                                 RTE_SECURITY_IPSEC_SA_MODE_TRANSPORT ?
 673                                 "transport" : "tunnel",
 674                         conf->ipsec.direction ==
 675                                 RTE_SECURITY_IPSEC_SA_DIR_INGRESS ?
 676                                 "inbound" : "outbound");
 677
 678                 rte_mempool_put(mempool, iavf_session);
 679                 return -EFAULT;
 680         }
 681
 682         /* save data plane required session parameters */
 683         set_session_parameter(iavf_sctx, iavf_session, conf, sa_idx);
 684
 685         /* save to security session private data */
 686         set_sec_session_private_data(session, iavf_session);
 687
 688         return 0;
 689 }
 690
 691 /**
 692  * Check if valid ipsec crypto action.
 693  * SPI must be non-zero and SPI in session must match SPI value
 694  * passed into function.
 695  *
 696  * returns: 0 if invalid session or SPI value equal zero
 697  * returns: 1 if valid
 698  */
 699 uint32_t
 700 iavf_ipsec_crypto_action_valid(struct rte_eth_dev *ethdev,
 701         const struct rte_security_session *session, uint32_t spi)
 702 {
 703         struct iavf_adapter *adapter =
 704                 IAVF_DEV_PRIVATE_TO_ADAPTER(ethdev->data->dev_private);
 705         struct iavf_security_session *sess = session->sess_private_data;
 706
 707         /* verify we have a valid session and that it belong to this adapter */
 708         if (unlikely(sess == NULL || sess->adapter != adapter))
 709                 return false;
 710
 711         /* SPI value must be non-zero */
 712         if (spi == 0)
 713                 return false;
 714         /* Session SPI must patch flow SPI*/
 715         else if (sess->sa.spi == spi) {
 716                 return true;
 717                 /**
 718                  * TODO: We should add a way of tracking valid hw SA indices to
 719                  * make validation less brittle
 720                  */
 721         }
 722
 723                 return true;
 724 }
 725
 726 /**
 727  * Send virtual channel security policy add request to IES driver.
 728  *
 729  * IES driver expects SPI and destination IP address to be in host
 730  * order, but DPDK APIs are network order, therefore we need to do a htonl
 731  * conversion of these parameters.
 732  */
 733 int
 734 iavf_ipsec_crypto_inbound_security_policy_add(struct iavf_adapter *adapter,
 735         uint32_t esp_spi,
 736         uint8_t is_v4,
 737         rte_be32_t v4_dst_addr,
 738         uint8_t *v6_dst_addr,
 739         uint8_t drop,
 740         bool is_udp,
 741         uint16_t udp_port)
 742 {
 743         struct inline_ipsec_msg *request = NULL, *response = NULL;
 744         size_t request_len, response_len;
 745         int rc = 0;
 746
 747         request_len = sizeof(struct inline_ipsec_msg) +
 748                         sizeof(struct virtchnl_ipsec_sp_cfg);
 749         request = rte_malloc("iavf-inbound-security-policy-add-request",
 750                                 request_len, 0);
 751         if (request == NULL) {
 752                 rc = -ENOMEM;
 753                 goto update_cleanup;
 754         }
 755
 756         /* set msg header params */
 757         request->ipsec_opcode = INLINE_IPSEC_OP_SP_CREATE;
 758         request->req_id = (uint16_t)0xDEADBEEF;
 759
 760         /* ESP SPI */
 761         request->ipsec_data.sp_cfg->spi = htonl(esp_spi);
 762
 763         /* Destination IP  */
 764         if (is_v4) {
 765                 request->ipsec_data.sp_cfg->table_id =
 766                                 VIRTCHNL_IPSEC_INBOUND_SPD_TBL_IPV4;
 767                 request->ipsec_data.sp_cfg->dip[0] = htonl(v4_dst_addr);
 768         } else {
 769                 request->ipsec_data.sp_cfg->table_id =
 770                                 VIRTCHNL_IPSEC_INBOUND_SPD_TBL_IPV6;
 771                 request->ipsec_data.sp_cfg->dip[0] =
 772                                 htonl(((uint32_t *)v6_dst_addr)[0]);
 773                 request->ipsec_data.sp_cfg->dip[1] =
 774                                 htonl(((uint32_t *)v6_dst_addr)[1]);
 775                 request->ipsec_data.sp_cfg->dip[2] =
 776                                 htonl(((uint32_t *)v6_dst_addr)[2]);
 777                 request->ipsec_data.sp_cfg->dip[3] =
 778                                 htonl(((uint32_t *)v6_dst_addr)[3]);
 779         }
 780
 781         request->ipsec_data.sp_cfg->drop = drop;
 782
 783         /** Traffic Class/Congestion Domain currently not support */
 784         request->ipsec_data.sp_cfg->set_tc = 0;
 785         request->ipsec_data.sp_cfg->cgd = 0;
 786         request->ipsec_data.sp_cfg->is_udp = is_udp;
 787         request->ipsec_data.sp_cfg->udp_port = htons(udp_port);
 788
 789         response_len = sizeof(struct inline_ipsec_msg) +
 790                         sizeof(struct virtchnl_ipsec_sp_cfg_resp);
 791         response = rte_malloc("iavf-inbound-security-policy-add-response",
 792                                 response_len, 0);
 793         if (response == NULL) {
 794                 rc = -ENOMEM;
 795                 goto update_cleanup;
 796         }
 797
 798         /* send virtual channel request to add SA to hardware database */
 799         rc = iavf_ipsec_crypto_request(adapter,
 800                         (uint8_t *)request, request_len,
 801                         (uint8_t *)response, response_len);
 802         if (rc)
 803                 goto update_cleanup;
 804
 805         /* verify response */
 806         if (response->ipsec_opcode != request->ipsec_opcode ||
 807                 response->req_id != request->req_id)
 808                 rc = -EFAULT;
 809         else
 810                 rc = response->ipsec_data.sp_cfg_resp->rule_id;
 811
 812 update_cleanup:
 813         rte_free(request);
 814         rte_free(response);
 815
 816         return rc;
 817 }
 818
 819 static uint32_t
 820 iavf_ipsec_crypto_sa_update_esn(struct iavf_adapter *adapter,
 821         struct iavf_security_session *sess)
 822 {
 823         struct inline_ipsec_msg *request = NULL, *response = NULL;
 824         size_t request_len, response_len;
 825         int rc = 0;
 826
 827         request_len = sizeof(struct inline_ipsec_msg) +
 828                         sizeof(struct virtchnl_ipsec_sa_update);
 829         request = rte_malloc("iavf-sa-update-request", request_len, 0);
 830         if (request == NULL) {
 831                 rc = -ENOMEM;
 832                 goto update_cleanup;
 833         }
 834
 835         response_len = sizeof(struct inline_ipsec_msg) +
 836                         sizeof(struct virtchnl_ipsec_resp);
 837         response = rte_malloc("iavf-sa-update-response", response_len, 0);
 838         if (response == NULL) {
 839                 rc = -ENOMEM;
 840                 goto update_cleanup;
 841         }
 842
 843         /* set msg header params */
 844         request->ipsec_opcode = INLINE_IPSEC_OP_SA_UPDATE;
 845         request->req_id = (uint16_t)0xDEADBEEF;
 846
 847         /* set request params */
 848         request->ipsec_data.sa_update->sa_index = sess->sa.hw_idx;
 849         request->ipsec_data.sa_update->esn_hi = sess->esn.hi;
 850
 851         /* send virtual channel request to add SA to hardware database */
 852         rc = iavf_ipsec_crypto_request(adapter,
 853                         (uint8_t *)request, request_len,
 854                         (uint8_t *)response, response_len);
 855         if (rc)
 856                 goto update_cleanup;
 857
 858         /* verify response */
 859         if (response->ipsec_opcode != request->ipsec_opcode ||
 860                 response->req_id != request->req_id)
 861                 rc = -EFAULT;
 862         else
 863                 rc = response->ipsec_data.ipsec_resp->resp;
 864
 865 update_cleanup:
 866         rte_free(request);
 867         rte_free(response);
 868
 869         return rc;
 870 }
 871
 872 static int
 873 iavf_ipsec_crypto_session_update(void *device,
 874                 struct rte_security_session *session,
 875                 struct rte_security_session_conf *conf)
 876 {
 877         struct iavf_adapter *adapter = NULL;
 878         struct iavf_security_session *iavf_sess = NULL;
 879         struct rte_eth_dev *eth_dev = (struct rte_eth_dev *)device;
 880         int rc = 0;
 881
 882         adapter = IAVF_DEV_PRIVATE_TO_ADAPTER(eth_dev->data->dev_private);
 883         iavf_sess = (struct iavf_security_session *)session->sess_private_data;
 884
 885         /* verify we have a valid session and that it belong to this adapter */
 886         if (unlikely(iavf_sess == NULL || iavf_sess->adapter != adapter))
 887                 return -EINVAL;
 888
 889         /* update esn hi 32-bits */
 890         if (iavf_sess->esn.enabled && conf->ipsec.options.esn) {
 891                 /**
 892                  * Update ESN in hardware for inbound SA. Store in
 893                  * iavf_security_session for outbound SA for use
 894                  * in *iavf_ipsec_crypto_pkt_metadata_set* function.
 895                  */
 896                 if (iavf_sess->direction == RTE_SECURITY_IPSEC_SA_DIR_INGRESS)
 897                         rc = iavf_ipsec_crypto_sa_update_esn(adapter,
 898                                         iavf_sess);
 899                 else
 900                         iavf_sess->esn.hi = conf->ipsec.esn.hi;
 901         }
 902
 903         return rc;
 904 }
 905
 906 static int
 907 iavf_ipsec_crypto_session_stats_get(void *device __rte_unused,
 908                 struct rte_security_session *session __rte_unused,
 909                 struct rte_security_stats *stats __rte_unused)
 910 {
 911         return -EOPNOTSUPP;
 912 }
 913
 914 int
 915 iavf_ipsec_crypto_security_policy_delete(struct iavf_adapter *adapter,
 916         uint8_t is_v4, uint32_t flow_id)
 917 {
 918         struct inline_ipsec_msg *request = NULL, *response = NULL;
 919         size_t request_len, response_len;
 920         int rc = 0;
 921
 922         request_len = sizeof(struct inline_ipsec_msg) +
 923                         sizeof(struct virtchnl_ipsec_sp_destroy);
 924         request = rte_malloc("iavf-sp-del-request", request_len, 0);
 925         if (request == NULL) {
 926                 rc = -ENOMEM;
 927                 goto update_cleanup;
 928         }
 929
 930         response_len = sizeof(struct inline_ipsec_msg) +
 931                         sizeof(struct virtchnl_ipsec_resp);
 932         response = rte_malloc("iavf-sp-del-response", response_len, 0);
 933         if (response == NULL) {
 934                 rc = -ENOMEM;
 935                 goto update_cleanup;
 936         }
 937
 938         /* set msg header params */
 939         request->ipsec_opcode = INLINE_IPSEC_OP_SP_DESTROY;
 940         request->req_id = (uint16_t)0xDEADBEEF;
 941
 942         /* set security policy params */
 943         request->ipsec_data.sp_destroy->table_id = is_v4 ?
 944                         VIRTCHNL_IPSEC_INBOUND_SPD_TBL_IPV4 :
 945                         VIRTCHNL_IPSEC_INBOUND_SPD_TBL_IPV6;
 946         request->ipsec_data.sp_destroy->rule_id = flow_id;
 947
 948         /* send virtual channel request to add SA to hardware database */
 949         rc = iavf_ipsec_crypto_request(adapter,
 950                         (uint8_t *)request, request_len,
 951                         (uint8_t *)response, response_len);
 952         if (rc)
 953                 goto update_cleanup;
 954
 955         /* verify response */
 956         if (response->ipsec_opcode != request->ipsec_opcode ||
 957                 response->req_id != request->req_id)
 958                 rc = -EFAULT;
 959         else
 960                 return response->ipsec_data.ipsec_status->status;
 961
 962 update_cleanup:
 963         rte_free(request);
 964         rte_free(response);
 965
 966         return rc;
 967 }
 968
 969 static uint32_t
 970 iavf_ipsec_crypto_sa_del(struct iavf_adapter *adapter,
 971         struct iavf_security_session *sess)
 972 {
 973         struct inline_ipsec_msg *request = NULL, *response = NULL;
 974         size_t request_len, response_len;
 975
 976         int rc = 0;
 977
 978         request_len = sizeof(struct inline_ipsec_msg) +
 979                         sizeof(struct virtchnl_ipsec_sa_destroy);
 980
 981         request = rte_malloc("iavf-sa-del-request", request_len, 0);
 982         if (request == NULL) {
 983                 rc = -ENOMEM;
 984                 goto update_cleanup;
 985         }
 986
 987         response_len = sizeof(struct inline_ipsec_msg) +
 988                         sizeof(struct virtchnl_ipsec_resp);
 989
 990         response = rte_malloc("iavf-sa-del-response", response_len, 0);
 991         if (response == NULL) {
 992                 rc = -ENOMEM;
 993                 goto update_cleanup;
 994         }
 995
 996         /* set msg header params */
 997         request->ipsec_opcode = INLINE_IPSEC_OP_SA_DESTROY;
 998         request->req_id = (uint16_t)0xDEADBEEF;
 999
1000         /**
1001          * SA delete supports deletion of 1-8 specified SA's or if the flag
1002          * field is zero, all SA's associated with VF will be deleted.
1003          */
1004         if (sess) {
1005                 request->ipsec_data.sa_destroy->flag = 0x1;
1006                 request->ipsec_data.sa_destroy->sa_index[0] = sess->sa.hw_idx;
1007         } else {
1008                 request->ipsec_data.sa_destroy->flag = 0x0;
1009         }
1010
1011         /* send virtual channel request to add SA to hardware database */
1012         rc = iavf_ipsec_crypto_request(adapter,
1013                         (uint8_t *)request, request_len,
1014                         (uint8_t *)response, response_len);
1015         if (rc)
1016                 goto update_cleanup;
1017
1018         /* verify response */
1019         if (response->ipsec_opcode != request->ipsec_opcode ||
1020                 response->req_id != request->req_id)
1021                 rc = -EFAULT;
1022
1023         /**
1024          * Delete status will be the same bitmask as sa_destroy request flag if
1025          * deletes successful
1026          */
1027         if (request->ipsec_data.sa_destroy->flag !=
1028                         response->ipsec_data.ipsec_status->status)
1029                 rc = -EFAULT;
1030
1031 update_cleanup:
1032         rte_free(response);
1033         rte_free(request);
1034
1035         return rc;
1036 }
1037
1038 static int
1039 iavf_ipsec_crypto_session_destroy(void *device,
1040                 struct rte_security_session *session)
1041 {
1042         struct iavf_adapter *adapter = NULL;
1043         struct iavf_security_session *iavf_sess = NULL;
1044         struct rte_eth_dev *eth_dev = (struct rte_eth_dev *)device;
1045         int ret;
1046
1047         adapter = IAVF_DEV_PRIVATE_TO_ADAPTER(eth_dev->data->dev_private);
1048         iavf_sess = (struct iavf_security_session *)session->sess_private_data;
1049
1050         /* verify we have a valid session and that it belong to this adapter */
1051         if (unlikely(iavf_sess == NULL || iavf_sess->adapter != adapter))
1052                 return -EINVAL;
1053
1054         ret = iavf_ipsec_crypto_sa_del(adapter, iavf_sess);
1055         rte_mempool_put(rte_mempool_from_obj(iavf_sess), (void *)iavf_sess);
1056         return ret;
1057 }
1058
1059 /**
1060  * Get ESP trailer from packet as well as calculate the total ESP trailer
1061  * length, which include padding, ESP trailer footer and the ICV
1062  */
1063 static inline struct rte_esp_tail *
1064 iavf_ipsec_crypto_get_esp_trailer(struct rte_mbuf *m,
1065         struct iavf_security_session *s, uint16_t *esp_trailer_length)
1066 {
1067         struct rte_esp_tail *esp_trailer;
1068
1069         uint16_t length = sizeof(struct rte_esp_tail) + s->icv_sz;
1070         uint16_t offset = 0;
1071
1072         /**
1073          * The ICV will not be present in TSO packets as this is appended by
1074          * hardware during segment generation
1075          */
1076         if (m->ol_flags & (RTE_MBUF_F_TX_TCP_SEG | RTE_MBUF_F_TX_UDP_SEG))
1077                 length -=  s->icv_sz;
1078
1079         *esp_trailer_length = length;
1080
1081         /**
1082          * Calculate offset in packet to ESP trailer header, this should be
1083          * total packet length less the size of the ESP trailer plus the ICV
1084          * length if it is present
1085          */
1086         offset = rte_pktmbuf_pkt_len(m) - length;
1087
1088         if (m->nb_segs > 1) {
1089                 /* find segment which esp trailer is located */
1090                 while (m->data_len < offset) {
1091                         offset -= m->data_len;
1092                         m = m->next;
1093                 }
1094         }
1095
1096         esp_trailer = rte_pktmbuf_mtod_offset(m, struct rte_esp_tail *, offset);
1097
1098         *esp_trailer_length += esp_trailer->pad_len;
1099
1100         return esp_trailer;
1101 }
1102
1103 static inline uint16_t
1104 iavf_ipsec_crypto_compute_l4_payload_length(struct rte_mbuf *m,
1105         struct iavf_security_session *s, uint16_t esp_tlen)
1106 {
1107         uint16_t ol2_len = m->l2_len;   /* MAC + VLAN */
1108         uint16_t ol3_len = 0;           /* ipv4/6 + ext hdrs */
1109         uint16_t ol4_len = 0;           /* UDP NATT */
1110         uint16_t l3_len = 0;            /* IPv4/6 + ext hdrs */
1111         uint16_t l4_len = 0;            /* TCP/UDP/STCP hdrs */
1112         uint16_t esp_hlen = sizeof(struct rte_esp_hdr) + s->iv_sz;
1113
1114         if (s->mode == RTE_SECURITY_IPSEC_SA_MODE_TUNNEL)
1115                 ol3_len = m->outer_l3_len;
1116                 /**<
1117                  * application provided l3len assumed to include length of
1118                  * ipv4/6 hdr + ext hdrs
1119                  */
1120
1121         if (s->udp_encap.enabled)
1122                 ol4_len = sizeof(struct rte_udp_hdr);
1123
1124         l3_len = m->l3_len;
1125         l4_len = m->l4_len;
1126
1127         return rte_pktmbuf_pkt_len(m) - (ol2_len + ol3_len + ol4_len +
1128                         esp_hlen + l3_len + l4_len + esp_tlen);
1129 }
1130
1131 static int
1132 iavf_ipsec_crypto_pkt_metadata_set(void *device,
1133                          struct rte_security_session *session,
1134                          struct rte_mbuf *m, void *params)
1135 {
1136         struct rte_eth_dev *ethdev = device;
1137         struct iavf_adapter *adapter =
1138                         IAVF_DEV_PRIVATE_TO_ADAPTER(ethdev->data->dev_private);
1139         struct iavf_security_ctx *iavf_sctx = adapter->security_ctx;
1140         struct iavf_security_session *iavf_sess = session->sess_private_data;
1141         struct iavf_ipsec_crypto_pkt_metadata *md;
1142         struct rte_esp_tail *esp_tail;
1143         uint64_t *sqn = params;
1144         uint16_t esp_trailer_length;
1145
1146         /* Check we have valid session and is associated with this device */
1147         if (unlikely(iavf_sess == NULL || iavf_sess->adapter != adapter))
1148                 return -EINVAL;
1149
1150         /* Get dynamic metadata location from mbuf */
1151         md = RTE_MBUF_DYNFIELD(m, iavf_sctx->pkt_md_offset,
1152                 struct iavf_ipsec_crypto_pkt_metadata *);
1153
1154         /* Set immutable metadata values from session template */
1155         memcpy(md, &iavf_sess->pkt_metadata_template,
1156                 sizeof(struct iavf_ipsec_crypto_pkt_metadata));
1157
1158         esp_tail = iavf_ipsec_crypto_get_esp_trailer(m, iavf_sess,
1159                         &esp_trailer_length);
1160
1161         /* Set per packet mutable metadata values */
1162         md->esp_trailer_len = esp_trailer_length;
1163         md->l4_payload_len = iavf_ipsec_crypto_compute_l4_payload_length(m,
1164                                 iavf_sess, esp_trailer_length);
1165         md->next_proto = esp_tail->next_proto;
1166
1167         /* If Extended SN in use set the upper 32-bits in metadata */
1168         if (iavf_sess->esn.enabled && sqn != NULL)
1169                 md->esn = (uint32_t)(*sqn >> 32);
1170
1171         return 0;
1172 }
1173
1174 static int
1175 iavf_ipsec_crypto_device_capabilities_get(struct iavf_adapter *adapter,
1176                 struct virtchnl_ipsec_cap *capability)
1177 {
1178         /* Perform pf-vf comms */
1179         struct inline_ipsec_msg *request = NULL, *response = NULL;
1180         size_t request_len, response_len;
1181         int rc;
1182
1183         request_len = sizeof(struct inline_ipsec_msg);
1184
1185         request = rte_malloc("iavf-device-capability-request", request_len, 0);
1186         if (request == NULL) {
1187                 rc = -ENOMEM;
1188                 goto update_cleanup;
1189         }
1190
1191         response_len = sizeof(struct inline_ipsec_msg) +
1192                         sizeof(struct virtchnl_ipsec_cap);
1193         response = rte_malloc("iavf-device-capability-response",
1194                         response_len, 0);
1195         if (response == NULL) {
1196                 rc = -ENOMEM;
1197                 goto update_cleanup;
1198         }
1199
1200         /* set msg header params */
1201         request->ipsec_opcode = INLINE_IPSEC_OP_GET_CAP;
1202         request->req_id = (uint16_t)0xDEADBEEF;
1203
1204         /* send virtual channel request to add SA to hardware database */
1205         rc = iavf_ipsec_crypto_request(adapter,
1206                         (uint8_t *)request, request_len,
1207                         (uint8_t *)response, response_len);
1208         if (rc)
1209                 goto update_cleanup;
1210
1211         /* verify response id */
1212         if (response->ipsec_opcode != request->ipsec_opcode ||
1213                 response->req_id != request->req_id){
1214                 rc = -EFAULT;
1215                 goto update_cleanup;
1216         }
1217         memcpy(capability, response->ipsec_data.ipsec_cap, sizeof(*capability));
1218
1219 update_cleanup:
1220         rte_free(response);
1221         rte_free(request);
1222
1223         return rc;
1224 }
1225
1226 enum rte_crypto_auth_algorithm auth_maptbl[] = {
1227         /* Hash Algorithm */
1228         [VIRTCHNL_HASH_NO_ALG] = RTE_CRYPTO_AUTH_NULL,
1229         [VIRTCHNL_AES_CBC_MAC] = RTE_CRYPTO_AUTH_AES_CBC_MAC,
1230         [VIRTCHNL_AES_CMAC] = RTE_CRYPTO_AUTH_AES_CMAC,
1231         [VIRTCHNL_AES_GMAC] = RTE_CRYPTO_AUTH_AES_GMAC,
1232         [VIRTCHNL_AES_XCBC_MAC] = RTE_CRYPTO_AUTH_AES_XCBC_MAC,
1233         [VIRTCHNL_MD5_HMAC] = RTE_CRYPTO_AUTH_MD5_HMAC,
1234         [VIRTCHNL_SHA1_HMAC] = RTE_CRYPTO_AUTH_SHA1_HMAC,
1235         [VIRTCHNL_SHA224_HMAC] = RTE_CRYPTO_AUTH_SHA224_HMAC,
1236         [VIRTCHNL_SHA256_HMAC] = RTE_CRYPTO_AUTH_SHA256_HMAC,
1237         [VIRTCHNL_SHA384_HMAC] = RTE_CRYPTO_AUTH_SHA384_HMAC,
1238         [VIRTCHNL_SHA512_HMAC] = RTE_CRYPTO_AUTH_SHA512_HMAC,
1239         [VIRTCHNL_SHA3_224_HMAC] = RTE_CRYPTO_AUTH_SHA3_224_HMAC,
1240         [VIRTCHNL_SHA3_256_HMAC] = RTE_CRYPTO_AUTH_SHA3_256_HMAC,
1241         [VIRTCHNL_SHA3_384_HMAC] = RTE_CRYPTO_AUTH_SHA3_384_HMAC,
1242         [VIRTCHNL_SHA3_512_HMAC] = RTE_CRYPTO_AUTH_SHA3_512_HMAC,
1243 };
1244
1245 static void
1246 update_auth_capabilities(struct rte_cryptodev_capabilities *scap,
1247                 struct virtchnl_algo_cap *acap)
1248 {
1249         struct rte_cryptodev_symmetric_capability *capability = &scap->sym;
1250
1251         scap->op = RTE_CRYPTO_OP_TYPE_SYMMETRIC;
1252
1253         capability->xform_type = RTE_CRYPTO_SYM_XFORM_AUTH;
1254
1255         capability->auth.algo = auth_maptbl[acap->algo_type];
1256         capability->auth.block_size = acap->block_size;
1257
1258         capability->auth.key_size.min = acap->min_key_size;
1259         capability->auth.key_size.max = acap->max_key_size;
1260         capability->auth.key_size.increment = acap->inc_key_size;
1261
1262         capability->auth.digest_size.min = acap->min_digest_size;
1263         capability->auth.digest_size.max = acap->max_digest_size;
1264         capability->auth.digest_size.increment = acap->inc_digest_size;
1265 }
1266
1267 enum rte_crypto_cipher_algorithm cipher_maptbl[] = {
1268         /* Cipher Algorithm */
1269         [VIRTCHNL_CIPHER_NO_ALG] = RTE_CRYPTO_CIPHER_NULL,
1270         [VIRTCHNL_3DES_CBC] = RTE_CRYPTO_CIPHER_3DES_CBC,
1271         [VIRTCHNL_AES_CBC] = RTE_CRYPTO_CIPHER_AES_CBC,
1272         [VIRTCHNL_AES_CTR] = RTE_CRYPTO_CIPHER_AES_CTR,
1273 };
1274
1275 static void
1276 update_cipher_capabilities(struct rte_cryptodev_capabilities *scap,
1277         struct virtchnl_algo_cap *acap)
1278 {
1279         struct rte_cryptodev_symmetric_capability *capability = &scap->sym;
1280
1281         scap->op = RTE_CRYPTO_OP_TYPE_SYMMETRIC;
1282
1283         capability->xform_type = RTE_CRYPTO_SYM_XFORM_CIPHER;
1284
1285         capability->cipher.algo = cipher_maptbl[acap->algo_type];
1286
1287         capability->cipher.block_size = acap->block_size;
1288
1289         capability->cipher.key_size.min = acap->min_key_size;
1290         capability->cipher.key_size.max = acap->max_key_size;
1291         capability->cipher.key_size.increment = acap->inc_key_size;
1292
1293         capability->cipher.iv_size.min = acap->min_iv_size;
1294         capability->cipher.iv_size.max = acap->max_iv_size;
1295         capability->cipher.iv_size.increment = acap->inc_iv_size;
1296 }
1297
1298 enum rte_crypto_aead_algorithm aead_maptbl[] = {
1299         /* AEAD Algorithm */
1300         [VIRTCHNL_AES_CCM] = RTE_CRYPTO_AEAD_AES_CCM,
1301         [VIRTCHNL_AES_GCM] = RTE_CRYPTO_AEAD_AES_GCM,
1302         [VIRTCHNL_CHACHA20_POLY1305] = RTE_CRYPTO_AEAD_CHACHA20_POLY1305,
1303 };
1304
1305 static void
1306 update_aead_capabilities(struct rte_cryptodev_capabilities *scap,
1307         struct virtchnl_algo_cap *acap)
1308 {
1309         struct rte_cryptodev_symmetric_capability *capability = &scap->sym;
1310
1311         scap->op = RTE_CRYPTO_OP_TYPE_SYMMETRIC;
1312
1313         capability->xform_type = RTE_CRYPTO_SYM_XFORM_AEAD;
1314
1315         capability->aead.algo = aead_maptbl[acap->algo_type];
1316
1317         capability->aead.block_size = acap->block_size;
1318
1319         capability->aead.key_size.min = acap->min_key_size;
1320         capability->aead.key_size.max = acap->max_key_size;
1321         capability->aead.key_size.increment = acap->inc_key_size;
1322
1323         capability->aead.aad_size.min = acap->min_aad_size;
1324         capability->aead.aad_size.max = acap->max_aad_size;
1325         capability->aead.aad_size.increment = acap->inc_aad_size;
1326
1327         capability->aead.iv_size.min = acap->min_iv_size;
1328         capability->aead.iv_size.max = acap->max_iv_size;
1329         capability->aead.iv_size.increment = acap->inc_iv_size;
1330
1331         capability->aead.digest_size.min = acap->min_digest_size;
1332         capability->aead.digest_size.max = acap->max_digest_size;
1333         capability->aead.digest_size.increment = acap->inc_digest_size;
1334 }
1335
1336 /**
1337  * Dynamically set crypto capabilities based on virtchannel IPsec
1338  * capabilities structure.
1339  */
1340 int
1341 iavf_ipsec_crypto_set_security_capabililites(struct iavf_security_ctx
1342                 *iavf_sctx, struct virtchnl_ipsec_cap *vch_cap)
1343 {
1344         struct rte_cryptodev_capabilities *capabilities;
1345         int i, j, number_of_capabilities = 0, ci = 0;
1346
1347         /* Count the total number of crypto algorithms supported */
1348         for (i = 0; i < VIRTCHNL_IPSEC_MAX_CRYPTO_CAP_NUM; i++)
1349                 number_of_capabilities += vch_cap->cap[i].algo_cap_num;
1350
1351         /**
1352          * Allocate cryptodev capabilities structure for
1353          * *number_of_capabilities* items plus one item to null terminate the
1354          * array
1355          */
1356         capabilities = rte_zmalloc("crypto_cap",
1357                 sizeof(struct rte_cryptodev_capabilities) *
1358                 (number_of_capabilities + 1), 0);
1359         if (!capabilities)
1360                 return -ENOMEM;
1361         capabilities[number_of_capabilities].op = RTE_CRYPTO_OP_TYPE_UNDEFINED;
1362
1363         /**
1364          * Iterate over each virtchnl crypto capability by crypto type and
1365          * algorithm.
1366          */
1367         for (i = 0; i < VIRTCHNL_IPSEC_MAX_CRYPTO_CAP_NUM; i++) {
1368                 for (j = 0; j < vch_cap->cap[i].algo_cap_num; j++, ci++) {
1369                         switch (vch_cap->cap[i].crypto_type) {
1370                         case VIRTCHNL_AUTH:
1371                                 update_auth_capabilities(&capabilities[ci],
1372                                         &vch_cap->cap[i].algo_cap_list[j]);
1373                                 break;
1374                         case VIRTCHNL_CIPHER:
1375                                 update_cipher_capabilities(&capabilities[ci],
1376                                         &vch_cap->cap[i].algo_cap_list[j]);
1377                                 break;
1378                         case VIRTCHNL_AEAD:
1379                                 update_aead_capabilities(&capabilities[ci],
1380                                         &vch_cap->cap[i].algo_cap_list[j]);
1381                                 break;
1382                         default:
1383                                 capabilities[ci].op =
1384                                                 RTE_CRYPTO_OP_TYPE_UNDEFINED;
1385                                 break;
1386                         }
1387                 }
1388         }
1389
1390         iavf_sctx->crypto_capabilities = capabilities;
1391         return 0;
1392 }
1393
1394 /**
1395  * Get security capabilities for device
1396  */
1397 static const struct rte_security_capability *
1398 iavf_ipsec_crypto_capabilities_get(void *device)
1399 {
1400         struct rte_eth_dev *eth_dev = (struct rte_eth_dev *)device;
1401         struct iavf_adapter *adapter =
1402                 IAVF_DEV_PRIVATE_TO_ADAPTER(eth_dev->data->dev_private);
1403         struct iavf_security_ctx *iavf_sctx = adapter->security_ctx;
1404         unsigned int i;
1405
1406         static struct rte_security_capability iavf_security_capabilities[] = {
1407                 { /* IPsec Inline Crypto ESP Tunnel Egress */
1408                         .action = RTE_SECURITY_ACTION_TYPE_INLINE_CRYPTO,
1409                         .protocol = RTE_SECURITY_PROTOCOL_IPSEC,
1410                         .ipsec = {
1411                                 .proto = RTE_SECURITY_IPSEC_SA_PROTO_ESP,
1412                                 .mode = RTE_SECURITY_IPSEC_SA_MODE_TUNNEL,
1413                                 .direction = RTE_SECURITY_IPSEC_SA_DIR_EGRESS,
1414                                 .options = { .udp_encap = 1,
1415                                                 .stats = 1, .esn = 1 },
1416                         },
1417                         .ol_flags = RTE_SECURITY_TX_OLOAD_NEED_MDATA
1418                 },
1419                 { /* IPsec Inline Crypto ESP Tunnel Ingress */
1420                         .action = RTE_SECURITY_ACTION_TYPE_INLINE_CRYPTO,
1421                         .protocol = RTE_SECURITY_PROTOCOL_IPSEC,
1422                         .ipsec = {
1423                                 .proto = RTE_SECURITY_IPSEC_SA_PROTO_ESP,
1424                                 .mode = RTE_SECURITY_IPSEC_SA_MODE_TUNNEL,
1425                                 .direction = RTE_SECURITY_IPSEC_SA_DIR_INGRESS,
1426                                 .options = { .udp_encap = 1,
1427                                                 .stats = 1, .esn = 1 },
1428                         },
1429                         .ol_flags = 0
1430                 },
1431                 { /* IPsec Inline Crypto ESP Transport Egress */
1432                         .action = RTE_SECURITY_ACTION_TYPE_INLINE_CRYPTO,
1433                         .protocol = RTE_SECURITY_PROTOCOL_IPSEC,
1434                         .ipsec = {
1435                                 .proto = RTE_SECURITY_IPSEC_SA_PROTO_ESP,
1436                                 .mode = RTE_SECURITY_IPSEC_SA_MODE_TRANSPORT,
1437                                 .direction = RTE_SECURITY_IPSEC_SA_DIR_EGRESS,
1438                                 .options = { .udp_encap = 1, .stats = 1,
1439                                                 .esn = 1 },
1440                         },
1441                         .ol_flags = RTE_SECURITY_TX_OLOAD_NEED_MDATA
1442                 },
1443                 { /* IPsec Inline Crypto ESP Transport Ingress */
1444                         .action = RTE_SECURITY_ACTION_TYPE_INLINE_CRYPTO,
1445                         .protocol = RTE_SECURITY_PROTOCOL_IPSEC,
1446                         .ipsec = {
1447                                 .proto = RTE_SECURITY_IPSEC_SA_PROTO_ESP,
1448                                 .mode = RTE_SECURITY_IPSEC_SA_MODE_TRANSPORT,
1449                                 .direction = RTE_SECURITY_IPSEC_SA_DIR_INGRESS,
1450                                 .options = { .udp_encap = 1, .stats = 1,
1451                                                 .esn = 1 }
1452                         },
1453                         .ol_flags = 0
1454                 },
1455                 {
1456                         .action = RTE_SECURITY_ACTION_TYPE_NONE
1457                 }
1458         };
1459
1460         /**
1461          * Update the security capabilities struct with the runtime discovered
1462          * crypto capabilities, except for last element of the array which is
1463          * the null termination
1464          */
1465         for (i = 0; i < ((sizeof(iavf_security_capabilities) /
1466                         sizeof(iavf_security_capabilities[0])) - 1); i++) {
1467                 iavf_security_capabilities[i].crypto_capabilities =
1468                         iavf_sctx->crypto_capabilities;
1469         }
1470
1471         return iavf_security_capabilities;
1472 }
1473
1474 static struct rte_security_ops iavf_ipsec_crypto_ops = {
1475         .session_get_size               = iavf_ipsec_crypto_session_size_get,
1476         .session_create                 = iavf_ipsec_crypto_session_create,
1477         .session_update                 = iavf_ipsec_crypto_session_update,
1478         .session_stats_get              = iavf_ipsec_crypto_session_stats_get,
1479         .session_destroy                = iavf_ipsec_crypto_session_destroy,
1480         .set_pkt_metadata               = iavf_ipsec_crypto_pkt_metadata_set,
1481         .get_userdata                   = NULL,
1482         .capabilities_get               = iavf_ipsec_crypto_capabilities_get,
1483 };
1484
1485 int
1486 iavf_security_ctx_create(struct iavf_adapter *adapter)
1487 {
1488         struct rte_security_ctx *sctx;
1489
1490         sctx = rte_malloc("security_ctx", sizeof(struct rte_security_ctx), 0);
1491         if (sctx == NULL)
1492                 return -ENOMEM;
1493
1494         sctx->device = adapter->vf.eth_dev;
1495         sctx->ops = &iavf_ipsec_crypto_ops;
1496         sctx->sess_cnt = 0;
1497
1498         adapter->vf.eth_dev->security_ctx = sctx;
1499
1500         if (adapter->security_ctx == NULL) {
1501                 adapter->security_ctx = rte_malloc("iavf_security_ctx",
1502                                 sizeof(struct iavf_security_ctx), 0);
1503                 if (adapter->security_ctx == NULL)
1504                         return -ENOMEM;
1505         }
1506
1507         return 0;
1508 }
1509
1510 int
1511 iavf_security_init(struct iavf_adapter *adapter)
1512 {
1513         struct iavf_security_ctx *iavf_sctx = adapter->security_ctx;
1514         struct rte_mbuf_dynfield pkt_md_dynfield = {
1515                 .name = "iavf_ipsec_crypto_pkt_metadata",
1516                 .size = sizeof(struct iavf_ipsec_crypto_pkt_metadata),
1517                 .align = __alignof__(struct iavf_ipsec_crypto_pkt_metadata)
1518         };
1519         struct virtchnl_ipsec_cap capabilities;
1520         int rc;
1521
1522         iavf_sctx->adapter = adapter;
1523
1524         iavf_sctx->pkt_md_offset = rte_mbuf_dynfield_register(&pkt_md_dynfield);
1525         if (iavf_sctx->pkt_md_offset < 0)
1526                 return iavf_sctx->pkt_md_offset;
1527
1528         /* Get device capabilities from Inline IPsec driver over PF-VF comms */
1529         rc = iavf_ipsec_crypto_device_capabilities_get(adapter, &capabilities);
1530         if (rc)
1531                 return rc;
1532
1533         return  iavf_ipsec_crypto_set_security_capabililites(iavf_sctx,
1534                         &capabilities);
1535 }
1536
1537 int
1538 iavf_security_get_pkt_md_offset(struct iavf_adapter *adapter)
1539 {
1540         struct iavf_security_ctx *iavf_sctx = adapter->security_ctx;
1541
1542         return iavf_sctx->pkt_md_offset;
1543 }
1544
1545 int
1546 iavf_security_ctx_destroy(struct iavf_adapter *adapter)
1547 {
1548         struct rte_security_ctx *sctx  = adapter->vf.eth_dev->security_ctx;
1549         struct iavf_security_ctx *iavf_sctx = adapter->security_ctx;
1550
1551         if (iavf_sctx == NULL)
1552                 return -ENODEV;
1553
1554         /* TODO: Add resources cleanup */
1555
1556         /* free and reset security data structures */
1557         rte_free(iavf_sctx);
1558         rte_free(sctx);
1559
1560         adapter->security_ctx = NULL;
1561         adapter->vf.eth_dev->security_ctx = NULL;
1562
1563         return 0;
1564 }
1565
1566 int
1567 iavf_ipsec_crypto_supported(struct iavf_adapter *adapter)
1568 {
1569         struct virtchnl_vf_resource *resources = adapter->vf.vf_res;
1570
1571         /** Capability check for IPsec Crypto */
1572         if (resources && (resources->vf_cap_flags &
1573                 VIRTCHNL_VF_OFFLOAD_INLINE_IPSEC_CRYPTO))
1574                 return true;
1575
1576         return false;
1577 }
1578
1579 #define IAVF_IPSEC_INSET_ESP (\
1580         IAVF_INSET_ESP_SPI)
1581
1582 #define IAVF_IPSEC_INSET_AH (\
1583         IAVF_INSET_AH_SPI)
1584
1585 #define IAVF_IPSEC_INSET_IPV4_NATT_ESP (\
1586         IAVF_INSET_IPV4_SRC | IAVF_INSET_IPV4_DST | \
1587         IAVF_INSET_ESP_SPI)
1588
1589 #define IAVF_IPSEC_INSET_IPV6_NATT_ESP (\
1590         IAVF_INSET_IPV6_SRC | IAVF_INSET_IPV6_DST | \
1591         IAVF_INSET_ESP_SPI)
1592
1593 enum iavf_ipsec_flow_pt_type {
1594         IAVF_PATTERN_ESP = 1,
1595         IAVF_PATTERN_AH,
1596         IAVF_PATTERN_UDP_ESP,
1597 };
1598 enum iavf_ipsec_flow_pt_ip_ver {
1599         IAVF_PATTERN_IPV4 = 1,
1600         IAVF_PATTERN_IPV6,
1601 };
1602
1603 #define IAVF_PATTERN(t, ipt) ((void *)((t) | ((ipt) << 4)))
1604 #define IAVF_PATTERN_TYPE(pt) ((pt) & 0x0F)
1605 #define IAVF_PATTERN_IP_V(pt) ((pt) >> 4)
1606
1607 static struct iavf_pattern_match_item iavf_ipsec_flow_pattern[] = {
1608         {iavf_pattern_eth_ipv4_esp,     IAVF_IPSEC_INSET_ESP,
1609                         IAVF_PATTERN(IAVF_PATTERN_ESP, IAVF_PATTERN_IPV4)},
1610         {iavf_pattern_eth_ipv6_esp,     IAVF_IPSEC_INSET_ESP,
1611                         IAVF_PATTERN(IAVF_PATTERN_ESP, IAVF_PATTERN_IPV6)},
1612         {iavf_pattern_eth_ipv4_ah,      IAVF_IPSEC_INSET_AH,
1613                         IAVF_PATTERN(IAVF_PATTERN_AH, IAVF_PATTERN_IPV4)},
1614         {iavf_pattern_eth_ipv6_ah,      IAVF_IPSEC_INSET_AH,
1615                         IAVF_PATTERN(IAVF_PATTERN_AH, IAVF_PATTERN_IPV6)},
1616         {iavf_pattern_eth_ipv4_udp_esp, IAVF_IPSEC_INSET_IPV4_NATT_ESP,
1617                         IAVF_PATTERN(IAVF_PATTERN_UDP_ESP, IAVF_PATTERN_IPV4)},
1618         {iavf_pattern_eth_ipv6_udp_esp, IAVF_IPSEC_INSET_IPV6_NATT_ESP,
1619                         IAVF_PATTERN(IAVF_PATTERN_UDP_ESP, IAVF_PATTERN_IPV6)},
1620 };
1621
1622 struct iavf_ipsec_flow_item {
1623         uint64_t id;
1624         uint8_t is_ipv4;
1625         uint32_t spi;
1626         struct rte_ether_hdr eth_hdr;
1627         union {
1628                 struct rte_ipv4_hdr ipv4_hdr;
1629                 struct rte_ipv6_hdr ipv6_hdr;
1630         };
1631         struct rte_udp_hdr udp_hdr;
1632         uint8_t is_udp;
1633 };
1634
1635 static void
1636 parse_eth_item(const struct rte_flow_item_eth *item,
1637                 struct rte_ether_hdr *eth)
1638 {
1639         memcpy(eth->src_addr.addr_bytes,
1640                         item->src.addr_bytes, sizeof(eth->src_addr));
1641         memcpy(eth->dst_addr.addr_bytes,
1642                         item->dst.addr_bytes, sizeof(eth->dst_addr));
1643 }
1644
1645 static void
1646 parse_ipv4_item(const struct rte_flow_item_ipv4 *item,
1647                 struct rte_ipv4_hdr *ipv4)
1648 {
1649         ipv4->src_addr = item->hdr.src_addr;
1650         ipv4->dst_addr = item->hdr.dst_addr;
1651 }
1652
1653 static void
1654 parse_ipv6_item(const struct rte_flow_item_ipv6 *item,
1655                 struct rte_ipv6_hdr *ipv6)
1656 {
1657         memcpy(ipv6->src_addr, item->hdr.src_addr, 16);
1658         memcpy(ipv6->dst_addr, item->hdr.dst_addr, 16);
1659 }
1660
1661 static void
1662 parse_udp_item(const struct rte_flow_item_udp *item, struct rte_udp_hdr *udp)
1663 {
1664         udp->dst_port = item->hdr.dst_port;
1665         udp->src_port = item->hdr.src_port;
1666 }
1667
1668 static int
1669 has_security_action(const struct rte_flow_action actions[],
1670         const void **session)
1671 {
1672         /* only {SECURITY; END} supported */
1673         if (actions[0].type == RTE_FLOW_ACTION_TYPE_SECURITY &&
1674                 actions[1].type == RTE_FLOW_ACTION_TYPE_END) {
1675                 *session = actions[0].conf;
1676                 return true;
1677         }
1678         return false;
1679 }
1680
1681 static struct iavf_ipsec_flow_item *
1682 iavf_ipsec_flow_item_parse(struct rte_eth_dev *ethdev,
1683                 const struct rte_flow_item pattern[],
1684                 const struct rte_flow_action actions[],
1685                 uint32_t type)
1686 {
1687         const void *session;
1688         struct iavf_ipsec_flow_item
1689                 *ipsec_flow = rte_malloc("security-flow-rule",
1690                 sizeof(struct iavf_ipsec_flow_item), 0);
1691         enum iavf_ipsec_flow_pt_type p_type = IAVF_PATTERN_TYPE(type);
1692         enum iavf_ipsec_flow_pt_ip_ver p_ip_type = IAVF_PATTERN_IP_V(type);
1693
1694         if (ipsec_flow == NULL)
1695                 return NULL;
1696
1697         ipsec_flow->is_ipv4 = (p_ip_type == IAVF_PATTERN_IPV4);
1698
1699         if (pattern[0].spec)
1700                 parse_eth_item((const struct rte_flow_item_eth *)
1701                                 pattern[0].spec, &ipsec_flow->eth_hdr);
1702
1703         switch (p_type) {
1704         case IAVF_PATTERN_ESP:
1705                 if (ipsec_flow->is_ipv4) {
1706                         parse_ipv4_item((const struct rte_flow_item_ipv4 *)
1707                                         pattern[1].spec,
1708                                         &ipsec_flow->ipv4_hdr);
1709                 } else {
1710                         parse_ipv6_item((const struct rte_flow_item_ipv6 *)
1711                                         pattern[1].spec,
1712                                         &ipsec_flow->ipv6_hdr);
1713                 }
1714                 ipsec_flow->spi =
1715                         ((const struct rte_flow_item_esp *)
1716                                         pattern[2].spec)->hdr.spi;
1717                 break;
1718         case IAVF_PATTERN_AH:
1719                 if (ipsec_flow->is_ipv4) {
1720                         parse_ipv4_item((const struct rte_flow_item_ipv4 *)
1721                                         pattern[1].spec,
1722                                         &ipsec_flow->ipv4_hdr);
1723                 } else {
1724                         parse_ipv6_item((const struct rte_flow_item_ipv6 *)
1725                                         pattern[1].spec,
1726                                         &ipsec_flow->ipv6_hdr);
1727                 }
1728                 ipsec_flow->spi =
1729                         ((const struct rte_flow_item_ah *)
1730                                         pattern[2].spec)->spi;
1731                 break;
1732         case IAVF_PATTERN_UDP_ESP:
1733                 if (ipsec_flow->is_ipv4) {
1734                         parse_ipv4_item((const struct rte_flow_item_ipv4 *)
1735                                         pattern[1].spec,
1736                                         &ipsec_flow->ipv4_hdr);
1737                 } else {
1738                         parse_ipv6_item((const struct rte_flow_item_ipv6 *)
1739                                         pattern[1].spec,
1740                                         &ipsec_flow->ipv6_hdr);
1741                 }
1742                 parse_udp_item((const struct rte_flow_item_udp *)
1743                                 pattern[2].spec,
1744                         &ipsec_flow->udp_hdr);
1745                 ipsec_flow->is_udp = true;
1746                 ipsec_flow->spi =
1747                         ((const struct rte_flow_item_esp *)
1748                                         pattern[3].spec)->hdr.spi;
1749                 break;
1750         default:
1751                 goto flow_cleanup;
1752         }
1753
1754         if (!has_security_action(actions, &session))
1755                 goto flow_cleanup;
1756
1757         if (!iavf_ipsec_crypto_action_valid(ethdev, session,
1758                         ipsec_flow->spi))
1759                 goto flow_cleanup;
1760
1761         return ipsec_flow;
1762
1763 flow_cleanup:
1764         rte_free(ipsec_flow);
1765         return NULL;
1766 }
1767
1768
1769 static struct iavf_flow_parser iavf_ipsec_flow_parser;
1770
1771 static int
1772 iavf_ipsec_flow_init(struct iavf_adapter *ad)
1773 {
1774         struct iavf_info *vf = IAVF_DEV_PRIVATE_TO_VF(ad);
1775         struct iavf_flow_parser *parser;
1776
1777         if (!vf->vf_res)
1778                 return -EINVAL;
1779
1780         if (vf->vf_res->vf_cap_flags & VIRTCHNL_VF_OFFLOAD_INLINE_IPSEC_CRYPTO)
1781                 parser = &iavf_ipsec_flow_parser;
1782         else
1783                 return -ENOTSUP;
1784
1785         return iavf_register_parser(parser, ad);
1786 }
1787
1788 static void
1789 iavf_ipsec_flow_uninit(struct iavf_adapter *ad)
1790 {
1791         iavf_unregister_parser(&iavf_ipsec_flow_parser, ad);
1792 }
1793
1794 static int
1795 iavf_ipsec_flow_create(struct iavf_adapter *ad,
1796                 struct rte_flow *flow,
1797                 void *meta,
1798                 struct rte_flow_error *error)
1799 {
1800         struct iavf_ipsec_flow_item *ipsec_flow = meta;
1801         if (!ipsec_flow) {
1802                 rte_flow_error_set(error, EINVAL,
1803                                 RTE_FLOW_ERROR_TYPE_HANDLE, NULL,
1804                                 "NULL rule.");
1805                 return -rte_errno;
1806         }
1807
1808         if (ipsec_flow->is_ipv4) {
1809                 ipsec_flow->id =
1810                         iavf_ipsec_crypto_inbound_security_policy_add(ad,
1811                         ipsec_flow->spi,
1812                         1,
1813                         ipsec_flow->ipv4_hdr.dst_addr,
1814                         NULL,
1815                         0,
1816                         ipsec_flow->is_udp,
1817                         ipsec_flow->udp_hdr.dst_port);
1818         } else {
1819                 ipsec_flow->id =
1820                         iavf_ipsec_crypto_inbound_security_policy_add(ad,
1821                         ipsec_flow->spi,
1822                         0,
1823                         0,
1824                         ipsec_flow->ipv6_hdr.dst_addr,
1825                         0,
1826                         ipsec_flow->is_udp,
1827                         ipsec_flow->udp_hdr.dst_port);
1828         }
1829
1830         if (ipsec_flow->id < 1) {
1831                 rte_flow_error_set(error, EINVAL,
1832                                 RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
1833                                 "Failed to add SA.");
1834                 return -rte_errno;
1835         }
1836
1837         flow->rule = ipsec_flow;
1838
1839         return 0;
1840 }
1841
1842 static int
1843 iavf_ipsec_flow_destroy(struct iavf_adapter *ad,
1844                 struct rte_flow *flow,
1845                 struct rte_flow_error *error)
1846 {
1847         struct iavf_ipsec_flow_item *ipsec_flow = flow->rule;
1848         if (!ipsec_flow) {
1849                 rte_flow_error_set(error, EINVAL,
1850                                 RTE_FLOW_ERROR_TYPE_HANDLE, NULL,
1851                                 "NULL rule.");
1852                 return -rte_errno;
1853         }
1854
1855         iavf_ipsec_crypto_security_policy_delete(ad,
1856                         ipsec_flow->is_ipv4, ipsec_flow->id);
1857         rte_free(ipsec_flow);
1858         return 0;
1859 }
1860
1861 static struct iavf_flow_engine iavf_ipsec_flow_engine = {
1862         .init = iavf_ipsec_flow_init,
1863         .uninit = iavf_ipsec_flow_uninit,
1864         .create = iavf_ipsec_flow_create,
1865         .destroy = iavf_ipsec_flow_destroy,
1866         .type = IAVF_FLOW_ENGINE_IPSEC_CRYPTO,
1867 };
1868
1869 static int
1870 iavf_ipsec_flow_parse(struct iavf_adapter *ad,
1871                        struct iavf_pattern_match_item *array,
1872                        uint32_t array_len,
1873                        const struct rte_flow_item pattern[],
1874                        const struct rte_flow_action actions[],
1875                        void **meta,
1876                        struct rte_flow_error *error)
1877 {
1878         struct iavf_pattern_match_item *item = NULL;
1879         int ret = -1;
1880
1881         item = iavf_search_pattern_match_item(pattern, array, array_len, error);
1882         if (item && item->meta) {
1883                 uint32_t type = (uint64_t)(item->meta);
1884                 struct iavf_ipsec_flow_item *fi =
1885                                 iavf_ipsec_flow_item_parse(ad->vf.eth_dev,
1886                                                 pattern, actions, type);
1887                 if (fi && meta) {
1888                         *meta = fi;
1889                         ret = 0;
1890                 }
1891         }
1892         return ret;
1893 }
1894
1895 static struct iavf_flow_parser iavf_ipsec_flow_parser = {
1896         .engine = &iavf_ipsec_flow_engine,
1897         .array = iavf_ipsec_flow_pattern,
1898         .array_len = RTE_DIM(iavf_ipsec_flow_pattern),
1899         .parse_pattern_action = iavf_ipsec_flow_parse,
1900         .stage = IAVF_FLOW_STAGE_IPSEC_CRYPTO,
1901 };
1902
1903 RTE_INIT(iavf_ipsec_flow_engine_register)
1904 {
1905         iavf_register_flow_engine(&iavf_ipsec_flow_engine);
1906 }