examples/ipsec-secgw/ipsec.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(c) 2016-2017 Intel Corporation
   3  */
   4 #include <sys/types.h>
   5 #include <netinet/in.h>
   6 #include <netinet/ip.h>
   7
   8 #include <rte_branch_prediction.h>
   9 #include <rte_log.h>
  10 #include <rte_crypto.h>
  11 #include <rte_security.h>
  12 #include <rte_cryptodev.h>
  13 #include <rte_ethdev.h>
  14 #include <rte_mbuf.h>
  15 #include <rte_hash.h>
  16
  17 #include "ipsec.h"
  18 #include "esp.h"
  19
  20 static inline void
  21 set_ipsec_conf(struct ipsec_sa *sa, struct rte_security_ipsec_xform *ipsec)
  22 {
  23         if (ipsec->mode == RTE_SECURITY_IPSEC_SA_MODE_TUNNEL) {
  24                 struct rte_security_ipsec_tunnel_param *tunnel =
  25                                 &ipsec->tunnel;
  26                 if (IS_IP4_TUNNEL(sa->flags)) {
  27                         tunnel->type =
  28                                 RTE_SECURITY_IPSEC_TUNNEL_IPV4;
  29                         tunnel->ipv4.ttl = IPDEFTTL;
  30
  31                         memcpy((uint8_t *)&tunnel->ipv4.src_ip,
  32                                 (uint8_t *)&sa->src.ip.ip4, 4);
  33
  34                         memcpy((uint8_t *)&tunnel->ipv4.dst_ip,
  35                                 (uint8_t *)&sa->dst.ip.ip4, 4);
  36                 } else if (IS_IP6_TUNNEL(sa->flags)) {
  37                         tunnel->type =
  38                                 RTE_SECURITY_IPSEC_TUNNEL_IPV6;
  39                         tunnel->ipv6.hlimit = IPDEFTTL;
  40                         tunnel->ipv6.dscp = 0;
  41                         tunnel->ipv6.flabel = 0;
  42
  43                         memcpy((uint8_t *)&tunnel->ipv6.src_addr,
  44                                 (uint8_t *)&sa->src.ip.ip6.ip6_b, 16);
  45
  46                         memcpy((uint8_t *)&tunnel->ipv6.dst_addr,
  47                                 (uint8_t *)&sa->dst.ip.ip6.ip6_b, 16);
  48                 }
  49                 /* TODO support for Transport */
  50         }
  51         ipsec->esn_soft_limit = IPSEC_OFFLOAD_ESN_SOFTLIMIT;
  52 }
  53
  54 int
  55 create_lookaside_session(struct ipsec_ctx *ipsec_ctx, struct ipsec_sa *sa,
  56                 struct rte_ipsec_session *ips)
  57 {
  58         struct rte_cryptodev_info cdev_info;
  59         unsigned long cdev_id_qp = 0;
  60         int32_t ret = 0;
  61         struct cdev_key key = { 0 };
  62
  63         key.lcore_id = (uint8_t)rte_lcore_id();
  64
  65         key.cipher_algo = (uint8_t)sa->cipher_algo;
  66         key.auth_algo = (uint8_t)sa->auth_algo;
  67         key.aead_algo = (uint8_t)sa->aead_algo;
  68
  69         ret = rte_hash_lookup_data(ipsec_ctx->cdev_map, &key,
  70                         (void **)&cdev_id_qp);
  71         if (ret < 0) {
  72                 RTE_LOG(ERR, IPSEC,
  73                                 "No cryptodev: core %u, cipher_algo %u, "
  74                                 "auth_algo %u, aead_algo %u\n",
  75                                 key.lcore_id,
  76                                 key.cipher_algo,
  77                                 key.auth_algo,
  78                                 key.aead_algo);
  79                 return -1;
  80         }
  81
  82         RTE_LOG_DP(DEBUG, IPSEC, "Create session for SA spi %u on cryptodev "
  83                         "%u qp %u\n", sa->spi,
  84                         ipsec_ctx->tbl[cdev_id_qp].id,
  85                         ipsec_ctx->tbl[cdev_id_qp].qp);
  86
  87         if (ips->type != RTE_SECURITY_ACTION_TYPE_NONE) {
  88                 struct rte_security_session_conf sess_conf = {
  89                         .action_type = ips->type,
  90                         .protocol = RTE_SECURITY_PROTOCOL_IPSEC,
  91                         {.ipsec = {
  92                                 .spi = sa->spi,
  93                                 .salt = sa->salt,
  94                                 .options = { 0 },
  95                                 .direction = sa->direction,
  96                                 .proto = RTE_SECURITY_IPSEC_SA_PROTO_ESP,
  97                                 .mode = (IS_TUNNEL(sa->flags)) ?
  98                                         RTE_SECURITY_IPSEC_SA_MODE_TUNNEL :
  99                                         RTE_SECURITY_IPSEC_SA_MODE_TRANSPORT,
 100                         } },
 101                         .crypto_xform = sa->xforms,
 102                         .userdata = NULL,
 103
 104                 };
 105
 106                 if (ips->type == RTE_SECURITY_ACTION_TYPE_LOOKASIDE_PROTOCOL) {
 107                         struct rte_security_ctx *ctx = (struct rte_security_ctx *)
 108                                                         rte_cryptodev_get_sec_ctx(
 109                                                         ipsec_ctx->tbl[cdev_id_qp].id);
 110
 111                         /* Set IPsec parameters in conf */
 112                         set_ipsec_conf(sa, &(sess_conf.ipsec));
 113
 114                         ips->security.ses = rte_security_session_create(ctx,
 115                                         &sess_conf, ipsec_ctx->session_priv_pool);
 116                         if (ips->security.ses == NULL) {
 117                                 RTE_LOG(ERR, IPSEC,
 118                                 "SEC Session init failed: err: %d\n", ret);
 119                                 return -1;
 120                         }
 121                 } else {
 122                         RTE_LOG(ERR, IPSEC, "Inline not supported\n");
 123                         return -1;
 124                 }
 125         } else {
 126                 ips->crypto.ses = rte_cryptodev_sym_session_create(
 127                                 ipsec_ctx->session_pool);
 128                 rte_cryptodev_sym_session_init(ipsec_ctx->tbl[cdev_id_qp].id,
 129                                 ips->crypto.ses, sa->xforms,
 130                                 ipsec_ctx->session_priv_pool);
 131
 132                 rte_cryptodev_info_get(ipsec_ctx->tbl[cdev_id_qp].id,
 133                                 &cdev_info);
 134         }
 135
 136         sa->cdev_id_qp = cdev_id_qp;
 137
 138         return 0;
 139 }
 140
 141 int
 142 create_inline_session(struct socket_ctx *skt_ctx, struct ipsec_sa *sa,
 143                 struct rte_ipsec_session *ips)
 144 {
 145         int32_t ret = 0;
 146         struct rte_security_ctx *sec_ctx;
 147         struct rte_security_session_conf sess_conf = {
 148                 .action_type = ips->type,
 149                 .protocol = RTE_SECURITY_PROTOCOL_IPSEC,
 150                 {.ipsec = {
 151                         .spi = sa->spi,
 152                         .salt = sa->salt,
 153                         .options = { 0 },
 154                         .direction = sa->direction,
 155                         .proto = RTE_SECURITY_IPSEC_SA_PROTO_ESP,
 156                         .mode = (sa->flags == IP4_TUNNEL ||
 157                                         sa->flags == IP6_TUNNEL) ?
 158                                         RTE_SECURITY_IPSEC_SA_MODE_TUNNEL :
 159                                         RTE_SECURITY_IPSEC_SA_MODE_TRANSPORT,
 160                 } },
 161                 .crypto_xform = sa->xforms,
 162                 .userdata = NULL,
 163         };
 164
 165         RTE_LOG_DP(DEBUG, IPSEC, "Create session for SA spi %u on port %u\n",
 166                 sa->spi, sa->portid);
 167
 168         if (ips->type == RTE_SECURITY_ACTION_TYPE_INLINE_CRYPTO) {
 169                 struct rte_flow_error err;
 170                 const struct rte_security_capability *sec_cap;
 171                 int ret = 0;
 172
 173                 sec_ctx = (struct rte_security_ctx *)
 174                                         rte_eth_dev_get_sec_ctx(
 175                                         sa->portid);
 176                 if (sec_ctx == NULL) {
 177                         RTE_LOG(ERR, IPSEC,
 178                                 " rte_eth_dev_get_sec_ctx failed\n");
 179                         return -1;
 180                 }
 181
 182                 ips->security.ses = rte_security_session_create(sec_ctx,
 183                                 &sess_conf, skt_ctx->session_pool);
 184                 if (ips->security.ses == NULL) {
 185                         RTE_LOG(ERR, IPSEC,
 186                                 "SEC Session init failed: err: %d\n", ret);
 187                         return -1;
 188                 }
 189
 190                 sec_cap = rte_security_capabilities_get(sec_ctx);
 191
 192                 /* iterate until ESP tunnel*/
 193                 while (sec_cap->action != RTE_SECURITY_ACTION_TYPE_NONE) {
 194                         if (sec_cap->action == ips->type &&
 195                             sec_cap->protocol ==
 196                                 RTE_SECURITY_PROTOCOL_IPSEC &&
 197                             sec_cap->ipsec.mode ==
 198                                 RTE_SECURITY_IPSEC_SA_MODE_TUNNEL &&
 199                             sec_cap->ipsec.direction == sa->direction)
 200                                 break;
 201                         sec_cap++;
 202                 }
 203
 204                 if (sec_cap->action == RTE_SECURITY_ACTION_TYPE_NONE) {
 205                         RTE_LOG(ERR, IPSEC,
 206                                 "No suitable security capability found\n");
 207                         return -1;
 208                 }
 209
 210                 ips->security.ol_flags = sec_cap->ol_flags;
 211                 ips->security.ctx = sec_ctx;
 212                 sa->pattern[0].type = RTE_FLOW_ITEM_TYPE_ETH;
 213
 214                 if (IS_IP6(sa->flags)) {
 215                         sa->pattern[1].mask = &rte_flow_item_ipv6_mask;
 216                         sa->pattern[1].type = RTE_FLOW_ITEM_TYPE_IPV6;
 217                         sa->pattern[1].spec = &sa->ipv6_spec;
 218
 219                         memcpy(sa->ipv6_spec.hdr.dst_addr,
 220                                 sa->dst.ip.ip6.ip6_b, 16);
 221                         memcpy(sa->ipv6_spec.hdr.src_addr,
 222                                sa->src.ip.ip6.ip6_b, 16);
 223                 } else if (IS_IP4(sa->flags)) {
 224                         sa->pattern[1].mask = &rte_flow_item_ipv4_mask;
 225                         sa->pattern[1].type = RTE_FLOW_ITEM_TYPE_IPV4;
 226                         sa->pattern[1].spec = &sa->ipv4_spec;
 227
 228                         sa->ipv4_spec.hdr.dst_addr = sa->dst.ip.ip4;
 229                         sa->ipv4_spec.hdr.src_addr = sa->src.ip.ip4;
 230                 }
 231
 232                 sa->pattern[2].type = RTE_FLOW_ITEM_TYPE_ESP;
 233                 sa->pattern[2].spec = &sa->esp_spec;
 234                 sa->pattern[2].mask = &rte_flow_item_esp_mask;
 235                 sa->esp_spec.hdr.spi = rte_cpu_to_be_32(sa->spi);
 236
 237                 sa->pattern[3].type = RTE_FLOW_ITEM_TYPE_END;
 238
 239                 sa->action[0].type = RTE_FLOW_ACTION_TYPE_SECURITY;
 240                 sa->action[0].conf = ips->security.ses;
 241
 242                 sa->action[1].type = RTE_FLOW_ACTION_TYPE_END;
 243
 244                 sa->attr.egress = (sa->direction ==
 245                                 RTE_SECURITY_IPSEC_SA_DIR_EGRESS);
 246                 sa->attr.ingress = (sa->direction ==
 247                                 RTE_SECURITY_IPSEC_SA_DIR_INGRESS);
 248                 if (sa->attr.ingress) {
 249                         uint8_t rss_key[40];
 250                         struct rte_eth_rss_conf rss_conf = {
 251                                 .rss_key = rss_key,
 252                                 .rss_key_len = 40,
 253                         };
 254                         struct rte_eth_dev_info dev_info;
 255                         uint16_t queue[RTE_MAX_QUEUES_PER_PORT];
 256                         struct rte_flow_action_rss action_rss;
 257                         unsigned int i;
 258                         unsigned int j;
 259
 260                         ret = rte_eth_dev_info_get(sa->portid, &dev_info);
 261                         if (ret != 0) {
 262                                 RTE_LOG(ERR, IPSEC,
 263                                         "Error during getting device (port %u) info: %s\n",
 264                                         sa->portid, strerror(-ret));
 265                                 return ret;
 266                         }
 267
 268                         sa->action[2].type = RTE_FLOW_ACTION_TYPE_END;
 269                         /* Try RSS. */
 270                         sa->action[1].type = RTE_FLOW_ACTION_TYPE_RSS;
 271                         sa->action[1].conf = &action_rss;
 272                         ret = rte_eth_dev_rss_hash_conf_get(sa->portid,
 273                                         &rss_conf);
 274                         if (ret != 0) {
 275                                 RTE_LOG(ERR, IPSEC,
 276                                         "rte_eth_dev_rss_hash_conf_get:ret=%d\n",
 277                                         ret);
 278                                 return -1;
 279                         }
 280                         for (i = 0, j = 0; i < dev_info.nb_rx_queues; ++i)
 281                                 queue[j++] = i;
 282
 283                         action_rss = (struct rte_flow_action_rss){
 284                                         .types = rss_conf.rss_hf,
 285                                         .key_len = rss_conf.rss_key_len,
 286                                         .queue_num = j,
 287                                         .key = rss_key,
 288                                         .queue = queue,
 289                         };
 290                         ret = rte_flow_validate(sa->portid, &sa->attr,
 291                                                 sa->pattern, sa->action,
 292                                                 &err);
 293                         if (!ret)
 294                                 goto flow_create;
 295                         /* Try Queue. */
 296                         sa->action[1].type = RTE_FLOW_ACTION_TYPE_QUEUE;
 297                         sa->action[1].conf =
 298                                 &(struct rte_flow_action_queue){
 299                                 .index = 0,
 300                         };
 301                         ret = rte_flow_validate(sa->portid, &sa->attr,
 302                                                 sa->pattern, sa->action,
 303                                                 &err);
 304                         /* Try End. */
 305                         sa->action[1].type = RTE_FLOW_ACTION_TYPE_END;
 306                         sa->action[1].conf = NULL;
 307                         ret = rte_flow_validate(sa->portid, &sa->attr,
 308                                                 sa->pattern, sa->action,
 309                                                 &err);
 310                         if (ret)
 311                                 goto flow_create_failure;
 312                 } else if (sa->attr.egress &&
 313                                 (ips->security.ol_flags &
 314                                         RTE_SECURITY_TX_HW_TRAILER_OFFLOAD)) {
 315                         sa->action[1].type =
 316                                         RTE_FLOW_ACTION_TYPE_PASSTHRU;
 317                         sa->action[2].type =
 318                                         RTE_FLOW_ACTION_TYPE_END;
 319                 }
 320 flow_create:
 321                 sa->flow = rte_flow_create(sa->portid,
 322                                 &sa->attr, sa->pattern, sa->action, &err);
 323                 if (sa->flow == NULL) {
 324 flow_create_failure:
 325                         RTE_LOG(ERR, IPSEC,
 326                                 "Failed to create ipsec flow msg: %s\n",
 327                                 err.message);
 328                         return -1;
 329                 }
 330         } else if (ips->type == RTE_SECURITY_ACTION_TYPE_INLINE_PROTOCOL) {
 331                 const struct rte_security_capability *sec_cap;
 332
 333                 sec_ctx = (struct rte_security_ctx *)
 334                                 rte_eth_dev_get_sec_ctx(sa->portid);
 335
 336                 if (sec_ctx == NULL) {
 337                         RTE_LOG(ERR, IPSEC,
 338                                 "Ethernet device doesn't have security features registered\n");
 339                         return -1;
 340                 }
 341
 342                 /* Set IPsec parameters in conf */
 343                 set_ipsec_conf(sa, &(sess_conf.ipsec));
 344
 345                 /* Save SA as userdata for the security session. When
 346                  * the packet is received, this userdata will be
 347                  * retrieved using the metadata from the packet.
 348                  *
 349                  * The PMD is expected to set similar metadata for other
 350                  * operations, like rte_eth_event, which are tied to
 351                  * security session. In such cases, the userdata could
 352                  * be obtained to uniquely identify the security
 353                  * parameters denoted.
 354                  */
 355
 356                 sess_conf.userdata = (void *) sa;
 357
 358                 ips->security.ses = rte_security_session_create(sec_ctx,
 359                                         &sess_conf, skt_ctx->session_pool);
 360                 if (ips->security.ses == NULL) {
 361                         RTE_LOG(ERR, IPSEC,
 362                                 "SEC Session init failed: err: %d\n", ret);
 363                         return -1;
 364                 }
 365
 366                 sec_cap = rte_security_capabilities_get(sec_ctx);
 367                 if (sec_cap == NULL) {
 368                         RTE_LOG(ERR, IPSEC,
 369                                 "No capabilities registered\n");
 370                         return -1;
 371                 }
 372
 373                 /* iterate until ESP tunnel*/
 374                 while (sec_cap->action !=
 375                                 RTE_SECURITY_ACTION_TYPE_NONE) {
 376                         if (sec_cap->action == ips->type &&
 377                             sec_cap->protocol ==
 378                                 RTE_SECURITY_PROTOCOL_IPSEC &&
 379                             sec_cap->ipsec.mode ==
 380                                 sess_conf.ipsec.mode &&
 381                             sec_cap->ipsec.direction == sa->direction)
 382                                 break;
 383                         sec_cap++;
 384                 }
 385
 386                 if (sec_cap->action == RTE_SECURITY_ACTION_TYPE_NONE) {
 387                         RTE_LOG(ERR, IPSEC,
 388                                 "No suitable security capability found\n");
 389                         return -1;
 390                 }
 391
 392                 ips->security.ol_flags = sec_cap->ol_flags;
 393                 ips->security.ctx = sec_ctx;
 394         }
 395         sa->cdev_id_qp = 0;
 396
 397         return 0;
 398 }
 399
 400 /*
 401  * queue crypto-ops into PMD queue.
 402  */
 403 void
 404 enqueue_cop_burst(struct cdev_qp *cqp)
 405 {
 406         uint32_t i, len, ret;
 407
 408         len = cqp->len;
 409         ret = rte_cryptodev_enqueue_burst(cqp->id, cqp->qp, cqp->buf, len);
 410         if (ret < len) {
 411                 RTE_LOG_DP(DEBUG, IPSEC, "Cryptodev %u queue %u:"
 412                         " enqueued %u crypto ops out of %u\n",
 413                         cqp->id, cqp->qp, ret, len);
 414                         /* drop packets that we fail to enqueue */
 415                         for (i = ret; i < len; i++)
 416                                 rte_pktmbuf_free(cqp->buf[i]->sym->m_src);
 417         }
 418         cqp->in_flight += ret;
 419         cqp->len = 0;
 420 }
 421
 422 static inline void
 423 enqueue_cop(struct cdev_qp *cqp, struct rte_crypto_op *cop)
 424 {
 425         cqp->buf[cqp->len++] = cop;
 426
 427         if (cqp->len == MAX_PKT_BURST)
 428                 enqueue_cop_burst(cqp);
 429 }
 430
 431 static inline void
 432 ipsec_enqueue(ipsec_xform_fn xform_func, struct ipsec_ctx *ipsec_ctx,
 433                 struct rte_mbuf *pkts[], void *sas[],
 434                 uint16_t nb_pkts)
 435 {
 436         int32_t ret = 0, i;
 437         struct ipsec_mbuf_metadata *priv;
 438         struct rte_crypto_sym_op *sym_cop;
 439         struct ipsec_sa *sa;
 440         struct rte_ipsec_session *ips;
 441
 442         for (i = 0; i < nb_pkts; i++) {
 443                 if (unlikely(sas[i] == NULL)) {
 444                         rte_pktmbuf_free(pkts[i]);
 445                         continue;
 446                 }
 447
 448                 rte_prefetch0(sas[i]);
 449                 rte_prefetch0(pkts[i]);
 450
 451                 priv = get_priv(pkts[i]);
 452                 sa = ipsec_mask_saptr(sas[i]);
 453                 priv->sa = sa;
 454                 ips = ipsec_get_primary_session(sa);
 455
 456                 switch (ips->type) {
 457                 case RTE_SECURITY_ACTION_TYPE_LOOKASIDE_PROTOCOL:
 458                         priv->cop.type = RTE_CRYPTO_OP_TYPE_SYMMETRIC;
 459                         priv->cop.status = RTE_CRYPTO_OP_STATUS_NOT_PROCESSED;
 460
 461                         rte_prefetch0(&priv->sym_cop);
 462
 463                         if ((unlikely(ips->security.ses == NULL)) &&
 464                                 create_lookaside_session(ipsec_ctx, sa, ips)) {
 465                                 rte_pktmbuf_free(pkts[i]);
 466                                 continue;
 467                         }
 468
 469                         sym_cop = get_sym_cop(&priv->cop);
 470                         sym_cop->m_src = pkts[i];
 471
 472                         rte_security_attach_session(&priv->cop,
 473                                 ips->security.ses);
 474                         break;
 475                 case RTE_SECURITY_ACTION_TYPE_NONE:
 476
 477                         priv->cop.type = RTE_CRYPTO_OP_TYPE_SYMMETRIC;
 478                         priv->cop.status = RTE_CRYPTO_OP_STATUS_NOT_PROCESSED;
 479
 480                         rte_prefetch0(&priv->sym_cop);
 481
 482                         if ((unlikely(ips->crypto.ses == NULL)) &&
 483                                 create_lookaside_session(ipsec_ctx, sa, ips)) {
 484                                 rte_pktmbuf_free(pkts[i]);
 485                                 continue;
 486                         }
 487
 488                         rte_crypto_op_attach_sym_session(&priv->cop,
 489                                         ips->crypto.ses);
 490
 491                         ret = xform_func(pkts[i], sa, &priv->cop);
 492                         if (unlikely(ret)) {
 493                                 rte_pktmbuf_free(pkts[i]);
 494                                 continue;
 495                         }
 496                         break;
 497                 case RTE_SECURITY_ACTION_TYPE_INLINE_PROTOCOL:
 498                         RTE_ASSERT(ips->security.ses != NULL);
 499                         ipsec_ctx->ol_pkts[ipsec_ctx->ol_pkts_cnt++] = pkts[i];
 500                         if (ips->security.ol_flags &
 501                                 RTE_SECURITY_TX_OLOAD_NEED_MDATA)
 502                                 rte_security_set_pkt_metadata(
 503                                         ips->security.ctx, ips->security.ses,
 504                                         pkts[i], NULL);
 505                         continue;
 506                 case RTE_SECURITY_ACTION_TYPE_INLINE_CRYPTO:
 507                         RTE_ASSERT(ips->security.ses != NULL);
 508                         priv->cop.type = RTE_CRYPTO_OP_TYPE_SYMMETRIC;
 509                         priv->cop.status = RTE_CRYPTO_OP_STATUS_NOT_PROCESSED;
 510
 511                         rte_prefetch0(&priv->sym_cop);
 512                         rte_security_attach_session(&priv->cop,
 513                                         ips->security.ses);
 514
 515                         ret = xform_func(pkts[i], sa, &priv->cop);
 516                         if (unlikely(ret)) {
 517                                 rte_pktmbuf_free(pkts[i]);
 518                                 continue;
 519                         }
 520
 521                         ipsec_ctx->ol_pkts[ipsec_ctx->ol_pkts_cnt++] = pkts[i];
 522                         if (ips->security.ol_flags &
 523                                 RTE_SECURITY_TX_OLOAD_NEED_MDATA)
 524                                 rte_security_set_pkt_metadata(
 525                                         ips->security.ctx, ips->security.ses,
 526                                         pkts[i], NULL);
 527                         continue;
 528                 }
 529
 530                 RTE_ASSERT(sa->cdev_id_qp < ipsec_ctx->nb_qps);
 531                 enqueue_cop(&ipsec_ctx->tbl[sa->cdev_id_qp], &priv->cop);
 532         }
 533 }
 534
 535 static inline int32_t
 536 ipsec_inline_dequeue(ipsec_xform_fn xform_func, struct ipsec_ctx *ipsec_ctx,
 537               struct rte_mbuf *pkts[], uint16_t max_pkts)
 538 {
 539         int32_t nb_pkts, ret;
 540         struct ipsec_mbuf_metadata *priv;
 541         struct ipsec_sa *sa;
 542         struct rte_mbuf *pkt;
 543
 544         nb_pkts = 0;
 545         while (ipsec_ctx->ol_pkts_cnt > 0 && nb_pkts < max_pkts) {
 546                 pkt = ipsec_ctx->ol_pkts[--ipsec_ctx->ol_pkts_cnt];
 547                 rte_prefetch0(pkt);
 548                 priv = get_priv(pkt);
 549                 sa = priv->sa;
 550                 ret = xform_func(pkt, sa, &priv->cop);
 551                 if (unlikely(ret)) {
 552                         rte_pktmbuf_free(pkt);
 553                         continue;
 554                 }
 555                 pkts[nb_pkts++] = pkt;
 556         }
 557
 558         return nb_pkts;
 559 }
 560
 561 static inline int
 562 ipsec_dequeue(ipsec_xform_fn xform_func, struct ipsec_ctx *ipsec_ctx,
 563               struct rte_mbuf *pkts[], uint16_t max_pkts)
 564 {
 565         int32_t nb_pkts = 0, ret = 0, i, j, nb_cops;
 566         struct ipsec_mbuf_metadata *priv;
 567         struct rte_crypto_op *cops[max_pkts];
 568         struct ipsec_sa *sa;
 569         struct rte_mbuf *pkt;
 570
 571         for (i = 0; i < ipsec_ctx->nb_qps && nb_pkts < max_pkts; i++) {
 572                 struct cdev_qp *cqp;
 573
 574                 cqp = &ipsec_ctx->tbl[ipsec_ctx->last_qp++];
 575                 if (ipsec_ctx->last_qp == ipsec_ctx->nb_qps)
 576                         ipsec_ctx->last_qp %= ipsec_ctx->nb_qps;
 577
 578                 if (cqp->in_flight == 0)
 579                         continue;
 580
 581                 nb_cops = rte_cryptodev_dequeue_burst(cqp->id, cqp->qp,
 582                                 cops, max_pkts - nb_pkts);
 583
 584                 cqp->in_flight -= nb_cops;
 585
 586                 for (j = 0; j < nb_cops; j++) {
 587                         pkt = cops[j]->sym->m_src;
 588                         rte_prefetch0(pkt);
 589
 590                         priv = get_priv(pkt);
 591                         sa = priv->sa;
 592
 593                         RTE_ASSERT(sa != NULL);
 594
 595                         if (ipsec_get_action_type(sa) ==
 596                                 RTE_SECURITY_ACTION_TYPE_NONE) {
 597                                 ret = xform_func(pkt, sa, cops[j]);
 598                                 if (unlikely(ret)) {
 599                                         rte_pktmbuf_free(pkt);
 600                                         continue;
 601                                 }
 602                         } else if (ipsec_get_action_type(sa) ==
 603                                 RTE_SECURITY_ACTION_TYPE_LOOKASIDE_PROTOCOL) {
 604                                 if (cops[j]->status) {
 605                                         rte_pktmbuf_free(pkt);
 606                                         continue;
 607                                 }
 608                         }
 609                         pkts[nb_pkts++] = pkt;
 610                 }
 611         }
 612
 613         /* return packets */
 614         return nb_pkts;
 615 }
 616
 617 uint16_t
 618 ipsec_inbound(struct ipsec_ctx *ctx, struct rte_mbuf *pkts[],
 619                 uint16_t nb_pkts, uint16_t len)
 620 {
 621         void *sas[nb_pkts];
 622
 623         inbound_sa_lookup(ctx->sa_ctx, pkts, sas, nb_pkts);
 624
 625         ipsec_enqueue(esp_inbound, ctx, pkts, sas, nb_pkts);
 626
 627         return ipsec_inline_dequeue(esp_inbound_post, ctx, pkts, len);
 628 }
 629
 630 uint16_t
 631 ipsec_inbound_cqp_dequeue(struct ipsec_ctx *ctx, struct rte_mbuf *pkts[],
 632                 uint16_t len)
 633 {
 634         return ipsec_dequeue(esp_inbound_post, ctx, pkts, len);
 635 }
 636
 637 uint16_t
 638 ipsec_outbound(struct ipsec_ctx *ctx, struct rte_mbuf *pkts[],
 639                 uint32_t sa_idx[], uint16_t nb_pkts, uint16_t len)
 640 {
 641         void *sas[nb_pkts];
 642
 643         outbound_sa_lookup(ctx->sa_ctx, sa_idx, sas, nb_pkts);
 644
 645         ipsec_enqueue(esp_outbound, ctx, pkts, sas, nb_pkts);
 646
 647         return ipsec_inline_dequeue(esp_outbound_post, ctx, pkts, len);
 648 }
 649
 650 uint16_t
 651 ipsec_outbound_cqp_dequeue(struct ipsec_ctx *ctx, struct rte_mbuf *pkts[],
 652                 uint16_t len)
 653 {
 654         return ipsec_dequeue(esp_outbound_post, ctx, pkts, len);
 655 }