examples/ipsec-secgw/ipsec-secgw.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(c) 2016 Intel Corporation
   3  */
   4
   5 #include <stdbool.h>
   6 #include <stdio.h>
   7 #include <stdlib.h>
   8 #include <stdint.h>
   9 #include <inttypes.h>
  10 #include <sys/types.h>
  11 #include <netinet/in.h>
  12 #include <netinet/ip.h>
  13 #include <netinet/ip6.h>
  14 #include <string.h>
  15 #include <sys/queue.h>
  16 #include <stdarg.h>
  17 #include <errno.h>
  18 #include <signal.h>
  19 #include <getopt.h>
  20
  21 #include <rte_common.h>
  22 #include <rte_bitmap.h>
  23 #include <rte_byteorder.h>
  24 #include <rte_log.h>
  25 #include <rte_eal.h>
  26 #include <rte_launch.h>
  27 #include <rte_atomic.h>
  28 #include <rte_cycles.h>
  29 #include <rte_prefetch.h>
  30 #include <rte_lcore.h>
  31 #include <rte_per_lcore.h>
  32 #include <rte_branch_prediction.h>
  33 #include <rte_interrupts.h>
  34 #include <rte_random.h>
  35 #include <rte_debug.h>
  36 #include <rte_ether.h>
  37 #include <rte_ethdev.h>
  38 #include <rte_mempool.h>
  39 #include <rte_mbuf.h>
  40 #include <rte_acl.h>
  41 #include <rte_lpm.h>
  42 #include <rte_lpm6.h>
  43 #include <rte_hash.h>
  44 #include <rte_jhash.h>
  45 #include <rte_cryptodev.h>
  46 #include <rte_security.h>
  47 #include <rte_eventdev.h>
  48 #include <rte_ip.h>
  49 #include <rte_ip_frag.h>
  50
  51 #include "event_helper.h"
  52 #include "ipsec.h"
  53 #include "parser.h"
  54 #include "sad.h"
  55
  56 volatile bool force_quit;
  57
  58 #define RTE_LOGTYPE_IPSEC RTE_LOGTYPE_USER1
  59
  60 #define MAX_JUMBO_PKT_LEN  9600
  61
  62 #define MEMPOOL_CACHE_SIZE 256
  63
  64 #define NB_MBUF (32000)
  65
  66 #define CDEV_QUEUE_DESC 2048
  67 #define CDEV_MAP_ENTRIES 16384
  68 #define CDEV_MP_NB_OBJS 1024
  69 #define CDEV_MP_CACHE_SZ 64
  70 #define MAX_QUEUE_PAIRS 1
  71
  72 #define BURST_TX_DRAIN_US 100 /* TX drain every ~100us */
  73
  74 /* Configure how many packets ahead to prefetch, when reading packets */
  75 #define PREFETCH_OFFSET 3
  76
  77 #define MAX_RX_QUEUE_PER_LCORE 16
  78
  79 #define MAX_LCORE_PARAMS 1024
  80
  81 /*
  82  * Configurable number of RX/TX ring descriptors
  83  */
  84 #define IPSEC_SECGW_RX_DESC_DEFAULT 1024
  85 #define IPSEC_SECGW_TX_DESC_DEFAULT 1024
  86 static uint16_t nb_rxd = IPSEC_SECGW_RX_DESC_DEFAULT;
  87 static uint16_t nb_txd = IPSEC_SECGW_TX_DESC_DEFAULT;
  88
  89 #if RTE_BYTE_ORDER != RTE_LITTLE_ENDIAN
  90 #define __BYTES_TO_UINT64(a, b, c, d, e, f, g, h) \
  91         (((uint64_t)((a) & 0xff) << 56) | \
  92         ((uint64_t)((b) & 0xff) << 48) | \
  93         ((uint64_t)((c) & 0xff) << 40) | \
  94         ((uint64_t)((d) & 0xff) << 32) | \
  95         ((uint64_t)((e) & 0xff) << 24) | \
  96         ((uint64_t)((f) & 0xff) << 16) | \
  97         ((uint64_t)((g) & 0xff) << 8)  | \
  98         ((uint64_t)(h) & 0xff))
  99 #else
 100 #define __BYTES_TO_UINT64(a, b, c, d, e, f, g, h) \
 101         (((uint64_t)((h) & 0xff) << 56) | \
 102         ((uint64_t)((g) & 0xff) << 48) | \
 103         ((uint64_t)((f) & 0xff) << 40) | \
 104         ((uint64_t)((e) & 0xff) << 32) | \
 105         ((uint64_t)((d) & 0xff) << 24) | \
 106         ((uint64_t)((c) & 0xff) << 16) | \
 107         ((uint64_t)((b) & 0xff) << 8) | \
 108         ((uint64_t)(a) & 0xff))
 109 #endif
 110 #define ETHADDR(a, b, c, d, e, f) (__BYTES_TO_UINT64(a, b, c, d, e, f, 0, 0))
 111
 112 #define ETHADDR_TO_UINT64(addr) __BYTES_TO_UINT64( \
 113                 (addr)->addr_bytes[0], (addr)->addr_bytes[1], \
 114                 (addr)->addr_bytes[2], (addr)->addr_bytes[3], \
 115                 (addr)->addr_bytes[4], (addr)->addr_bytes[5], \
 116                 0, 0)
 117
 118 #define FRAG_TBL_BUCKET_ENTRIES 4
 119 #define MAX_FRAG_TTL_NS         (10LL * NS_PER_S)
 120
 121 #define MTU_TO_FRAMELEN(x)      ((x) + RTE_ETHER_HDR_LEN + RTE_ETHER_CRC_LEN)
 122
 123 /* port/source ethernet addr and destination ethernet addr */
 124 struct ethaddr_info {
 125         uint64_t src, dst;
 126 };
 127
 128 struct ethaddr_info ethaddr_tbl[RTE_MAX_ETHPORTS] = {
 129         { 0, ETHADDR(0x00, 0x16, 0x3e, 0x7e, 0x94, 0x9a) },
 130         { 0, ETHADDR(0x00, 0x16, 0x3e, 0x22, 0xa1, 0xd9) },
 131         { 0, ETHADDR(0x00, 0x16, 0x3e, 0x08, 0x69, 0x26) },
 132         { 0, ETHADDR(0x00, 0x16, 0x3e, 0x49, 0x9e, 0xdd) }
 133 };
 134
 135 struct flow_info flow_info_tbl[RTE_MAX_ETHPORTS];
 136
 137 #define CMD_LINE_OPT_CONFIG             "config"
 138 #define CMD_LINE_OPT_SINGLE_SA          "single-sa"
 139 #define CMD_LINE_OPT_CRYPTODEV_MASK     "cryptodev_mask"
 140 #define CMD_LINE_OPT_TRANSFER_MODE      "transfer-mode"
 141 #define CMD_LINE_OPT_SCHEDULE_TYPE      "event-schedule-type"
 142 #define CMD_LINE_OPT_RX_OFFLOAD         "rxoffload"
 143 #define CMD_LINE_OPT_TX_OFFLOAD         "txoffload"
 144 #define CMD_LINE_OPT_REASSEMBLE         "reassemble"
 145 #define CMD_LINE_OPT_MTU                "mtu"
 146 #define CMD_LINE_OPT_FRAG_TTL           "frag-ttl"
 147
 148 #define CMD_LINE_ARG_EVENT      "event"
 149 #define CMD_LINE_ARG_POLL       "poll"
 150 #define CMD_LINE_ARG_ORDERED    "ordered"
 151 #define CMD_LINE_ARG_ATOMIC     "atomic"
 152 #define CMD_LINE_ARG_PARALLEL   "parallel"
 153
 154 enum {
 155         /* long options mapped to a short option */
 156
 157         /* first long only option value must be >= 256, so that we won't
 158          * conflict with short options
 159          */
 160         CMD_LINE_OPT_MIN_NUM = 256,
 161         CMD_LINE_OPT_CONFIG_NUM,
 162         CMD_LINE_OPT_SINGLE_SA_NUM,
 163         CMD_LINE_OPT_CRYPTODEV_MASK_NUM,
 164         CMD_LINE_OPT_TRANSFER_MODE_NUM,
 165         CMD_LINE_OPT_SCHEDULE_TYPE_NUM,
 166         CMD_LINE_OPT_RX_OFFLOAD_NUM,
 167         CMD_LINE_OPT_TX_OFFLOAD_NUM,
 168         CMD_LINE_OPT_REASSEMBLE_NUM,
 169         CMD_LINE_OPT_MTU_NUM,
 170         CMD_LINE_OPT_FRAG_TTL_NUM,
 171 };
 172
 173 static const struct option lgopts[] = {
 174         {CMD_LINE_OPT_CONFIG, 1, 0, CMD_LINE_OPT_CONFIG_NUM},
 175         {CMD_LINE_OPT_SINGLE_SA, 1, 0, CMD_LINE_OPT_SINGLE_SA_NUM},
 176         {CMD_LINE_OPT_CRYPTODEV_MASK, 1, 0, CMD_LINE_OPT_CRYPTODEV_MASK_NUM},
 177         {CMD_LINE_OPT_TRANSFER_MODE, 1, 0, CMD_LINE_OPT_TRANSFER_MODE_NUM},
 178         {CMD_LINE_OPT_SCHEDULE_TYPE, 1, 0, CMD_LINE_OPT_SCHEDULE_TYPE_NUM},
 179         {CMD_LINE_OPT_RX_OFFLOAD, 1, 0, CMD_LINE_OPT_RX_OFFLOAD_NUM},
 180         {CMD_LINE_OPT_TX_OFFLOAD, 1, 0, CMD_LINE_OPT_TX_OFFLOAD_NUM},
 181         {CMD_LINE_OPT_REASSEMBLE, 1, 0, CMD_LINE_OPT_REASSEMBLE_NUM},
 182         {CMD_LINE_OPT_MTU, 1, 0, CMD_LINE_OPT_MTU_NUM},
 183         {CMD_LINE_OPT_FRAG_TTL, 1, 0, CMD_LINE_OPT_FRAG_TTL_NUM},
 184         {NULL, 0, 0, 0}
 185 };
 186
 187 uint32_t unprotected_port_mask;
 188 uint32_t single_sa_idx;
 189 /* mask of enabled ports */
 190 static uint32_t enabled_port_mask;
 191 static uint64_t enabled_cryptodev_mask = UINT64_MAX;
 192 static int32_t promiscuous_on = 1;
 193 static int32_t numa_on = 1; /**< NUMA is enabled by default. */
 194 static uint32_t nb_lcores;
 195 static uint32_t single_sa;
 196
 197 /*
 198  * RX/TX HW offload capabilities to enable/use on ethernet ports.
 199  * By default all capabilities are enabled.
 200  */
 201 static uint64_t dev_rx_offload = UINT64_MAX;
 202 static uint64_t dev_tx_offload = UINT64_MAX;
 203
 204 /*
 205  * global values that determine multi-seg policy
 206  */
 207 static uint32_t frag_tbl_sz;
 208 static uint32_t frame_buf_size = RTE_MBUF_DEFAULT_BUF_SIZE;
 209 static uint32_t mtu_size = RTE_ETHER_MTU;
 210 static uint64_t frag_ttl_ns = MAX_FRAG_TTL_NS;
 211
 212 /* application wide librte_ipsec/SA parameters */
 213 struct app_sa_prm app_sa_prm = {
 214                         .enable = 0,
 215                         .cache_sz = SA_CACHE_SZ
 216                 };
 217 static const char *cfgfile;
 218
 219 struct lcore_rx_queue {
 220         uint16_t port_id;
 221         uint8_t queue_id;
 222 } __rte_cache_aligned;
 223
 224 struct lcore_params {
 225         uint16_t port_id;
 226         uint8_t queue_id;
 227         uint8_t lcore_id;
 228 } __rte_cache_aligned;
 229
 230 static struct lcore_params lcore_params_array[MAX_LCORE_PARAMS];
 231
 232 static struct lcore_params *lcore_params;
 233 static uint16_t nb_lcore_params;
 234
 235 static struct rte_hash *cdev_map_in;
 236 static struct rte_hash *cdev_map_out;
 237
 238 struct buffer {
 239         uint16_t len;
 240         struct rte_mbuf *m_table[MAX_PKT_BURST] __rte_aligned(sizeof(void *));
 241 };
 242
 243 struct lcore_conf {
 244         uint16_t nb_rx_queue;
 245         struct lcore_rx_queue rx_queue_list[MAX_RX_QUEUE_PER_LCORE];
 246         uint16_t tx_queue_id[RTE_MAX_ETHPORTS];
 247         struct buffer tx_mbufs[RTE_MAX_ETHPORTS];
 248         struct ipsec_ctx inbound;
 249         struct ipsec_ctx outbound;
 250         struct rt_ctx *rt4_ctx;
 251         struct rt_ctx *rt6_ctx;
 252         struct {
 253                 struct rte_ip_frag_tbl *tbl;
 254                 struct rte_mempool *pool_dir;
 255                 struct rte_mempool *pool_indir;
 256                 struct rte_ip_frag_death_row dr;
 257         } frag;
 258 } __rte_cache_aligned;
 259
 260 static struct lcore_conf lcore_conf[RTE_MAX_LCORE];
 261
 262 static struct rte_eth_conf port_conf = {
 263         .rxmode = {
 264                 .mq_mode        = ETH_MQ_RX_RSS,
 265                 .max_rx_pkt_len = RTE_ETHER_MAX_LEN,
 266                 .split_hdr_size = 0,
 267                 .offloads = DEV_RX_OFFLOAD_CHECKSUM,
 268         },
 269         .rx_adv_conf = {
 270                 .rss_conf = {
 271                         .rss_key = NULL,
 272                         .rss_hf = ETH_RSS_IP | ETH_RSS_UDP |
 273                                 ETH_RSS_TCP | ETH_RSS_SCTP,
 274                 },
 275         },
 276         .txmode = {
 277                 .mq_mode = ETH_MQ_TX_NONE,
 278         },
 279 };
 280
 281 struct socket_ctx socket_ctx[NB_SOCKETS];
 282
 283 /*
 284  * Determine is multi-segment support required:
 285  *  - either frame buffer size is smaller then mtu
 286  *  - or reassmeble support is requested
 287  */
 288 static int
 289 multi_seg_required(void)
 290 {
 291         return (MTU_TO_FRAMELEN(mtu_size) + RTE_PKTMBUF_HEADROOM >
 292                 frame_buf_size || frag_tbl_sz != 0);
 293 }
 294
 295 static inline void
 296 adjust_ipv4_pktlen(struct rte_mbuf *m, const struct rte_ipv4_hdr *iph,
 297         uint32_t l2_len)
 298 {
 299         uint32_t plen, trim;
 300
 301         plen = rte_be_to_cpu_16(iph->total_length) + l2_len;
 302         if (plen < m->pkt_len) {
 303                 trim = m->pkt_len - plen;
 304                 rte_pktmbuf_trim(m, trim);
 305         }
 306 }
 307
 308 static inline void
 309 adjust_ipv6_pktlen(struct rte_mbuf *m, const struct rte_ipv6_hdr *iph,
 310         uint32_t l2_len)
 311 {
 312         uint32_t plen, trim;
 313
 314         plen = rte_be_to_cpu_16(iph->payload_len) + sizeof(*iph) + l2_len;
 315         if (plen < m->pkt_len) {
 316                 trim = m->pkt_len - plen;
 317                 rte_pktmbuf_trim(m, trim);
 318         }
 319 }
 320
 321 static inline void
 322 prepare_one_packet(struct rte_mbuf *pkt, struct ipsec_traffic *t)
 323 {
 324         const struct rte_ether_hdr *eth;
 325         const struct rte_ipv4_hdr *iph4;
 326         const struct rte_ipv6_hdr *iph6;
 327
 328         eth = rte_pktmbuf_mtod(pkt, const struct rte_ether_hdr *);
 329         if (eth->ether_type == rte_cpu_to_be_16(RTE_ETHER_TYPE_IPV4)) {
 330
 331                 iph4 = (const struct rte_ipv4_hdr *)rte_pktmbuf_adj(pkt,
 332                         RTE_ETHER_HDR_LEN);
 333                 adjust_ipv4_pktlen(pkt, iph4, 0);
 334
 335                 if (iph4->next_proto_id == IPPROTO_ESP)
 336                         t->ipsec.pkts[(t->ipsec.num)++] = pkt;
 337                 else {
 338                         t->ip4.data[t->ip4.num] = &iph4->next_proto_id;
 339                         t->ip4.pkts[(t->ip4.num)++] = pkt;
 340                 }
 341                 pkt->l2_len = 0;
 342                 pkt->l3_len = sizeof(*iph4);
 343                 pkt->packet_type |= RTE_PTYPE_L3_IPV4;
 344         } else if (eth->ether_type == rte_cpu_to_be_16(RTE_ETHER_TYPE_IPV6)) {
 345                 int next_proto;
 346                 size_t l3len, ext_len;
 347                 uint8_t *p;
 348
 349                 /* get protocol type */
 350                 iph6 = (const struct rte_ipv6_hdr *)rte_pktmbuf_adj(pkt,
 351                         RTE_ETHER_HDR_LEN);
 352                 adjust_ipv6_pktlen(pkt, iph6, 0);
 353
 354                 next_proto = iph6->proto;
 355
 356                 /* determine l3 header size up to ESP extension */
 357                 l3len = sizeof(struct ip6_hdr);
 358                 p = rte_pktmbuf_mtod(pkt, uint8_t *);
 359                 while (next_proto != IPPROTO_ESP && l3len < pkt->data_len &&
 360                         (next_proto = rte_ipv6_get_next_ext(p + l3len,
 361                                                 next_proto, &ext_len)) >= 0)
 362                         l3len += ext_len;
 363
 364                 /* drop packet when IPv6 header exceeds first segment length */
 365                 if (unlikely(l3len > pkt->data_len)) {
 366                         rte_pktmbuf_free(pkt);
 367                         return;
 368                 }
 369
 370                 if (next_proto == IPPROTO_ESP)
 371                         t->ipsec.pkts[(t->ipsec.num)++] = pkt;
 372                 else {
 373                         t->ip6.data[t->ip6.num] = &iph6->proto;
 374                         t->ip6.pkts[(t->ip6.num)++] = pkt;
 375                 }
 376                 pkt->l2_len = 0;
 377                 pkt->l3_len = l3len;
 378                 pkt->packet_type |= RTE_PTYPE_L3_IPV6;
 379         } else {
 380                 /* Unknown/Unsupported type, drop the packet */
 381                 RTE_LOG(ERR, IPSEC, "Unsupported packet type 0x%x\n",
 382                         rte_be_to_cpu_16(eth->ether_type));
 383                 rte_pktmbuf_free(pkt);
 384                 return;
 385         }
 386
 387         /* Check if the packet has been processed inline. For inline protocol
 388          * processed packets, the metadata in the mbuf can be used to identify
 389          * the security processing done on the packet. The metadata will be
 390          * used to retrieve the application registered userdata associated
 391          * with the security session.
 392          */
 393
 394         if (pkt->ol_flags & PKT_RX_SEC_OFFLOAD) {
 395                 struct ipsec_sa *sa;
 396                 struct ipsec_mbuf_metadata *priv;
 397                 struct rte_security_ctx *ctx = (struct rte_security_ctx *)
 398                                                 rte_eth_dev_get_sec_ctx(
 399                                                 pkt->port);
 400
 401                 /* Retrieve the userdata registered. Here, the userdata
 402                  * registered is the SA pointer.
 403                  */
 404
 405                 sa = (struct ipsec_sa *)
 406                                 rte_security_get_userdata(ctx, pkt->udata64);
 407
 408                 if (sa == NULL) {
 409                         /* userdata could not be retrieved */
 410                         return;
 411                 }
 412
 413                 /* Save SA as priv member in mbuf. This will be used in the
 414                  * IPsec selector(SP-SA) check.
 415                  */
 416
 417                 priv = get_priv(pkt);
 418                 priv->sa = sa;
 419         }
 420 }
 421
 422 static inline void
 423 prepare_traffic(struct rte_mbuf **pkts, struct ipsec_traffic *t,
 424                 uint16_t nb_pkts)
 425 {
 426         int32_t i;
 427
 428         t->ipsec.num = 0;
 429         t->ip4.num = 0;
 430         t->ip6.num = 0;
 431
 432         for (i = 0; i < (nb_pkts - PREFETCH_OFFSET); i++) {
 433                 rte_prefetch0(rte_pktmbuf_mtod(pkts[i + PREFETCH_OFFSET],
 434                                         void *));
 435                 prepare_one_packet(pkts[i], t);
 436         }
 437         /* Process left packets */
 438         for (; i < nb_pkts; i++)
 439                 prepare_one_packet(pkts[i], t);
 440 }
 441
 442 static inline void
 443 prepare_tx_pkt(struct rte_mbuf *pkt, uint16_t port,
 444                 const struct lcore_conf *qconf)
 445 {
 446         struct ip *ip;
 447         struct rte_ether_hdr *ethhdr;
 448
 449         ip = rte_pktmbuf_mtod(pkt, struct ip *);
 450
 451         ethhdr = (struct rte_ether_hdr *)
 452                 rte_pktmbuf_prepend(pkt, RTE_ETHER_HDR_LEN);
 453
 454         if (ip->ip_v == IPVERSION) {
 455                 pkt->ol_flags |= qconf->outbound.ipv4_offloads;
 456                 pkt->l3_len = sizeof(struct ip);
 457                 pkt->l2_len = RTE_ETHER_HDR_LEN;
 458
 459                 ip->ip_sum = 0;
 460
 461                 /* calculate IPv4 cksum in SW */
 462                 if ((pkt->ol_flags & PKT_TX_IP_CKSUM) == 0)
 463                         ip->ip_sum = rte_ipv4_cksum((struct rte_ipv4_hdr *)ip);
 464
 465                 ethhdr->ether_type = rte_cpu_to_be_16(RTE_ETHER_TYPE_IPV4);
 466         } else {
 467                 pkt->ol_flags |= qconf->outbound.ipv6_offloads;
 468                 pkt->l3_len = sizeof(struct ip6_hdr);
 469                 pkt->l2_len = RTE_ETHER_HDR_LEN;
 470
 471                 ethhdr->ether_type = rte_cpu_to_be_16(RTE_ETHER_TYPE_IPV6);
 472         }
 473
 474         memcpy(&ethhdr->s_addr, &ethaddr_tbl[port].src,
 475                         sizeof(struct rte_ether_addr));
 476         memcpy(&ethhdr->d_addr, &ethaddr_tbl[port].dst,
 477                         sizeof(struct rte_ether_addr));
 478 }
 479
 480 static inline void
 481 prepare_tx_burst(struct rte_mbuf *pkts[], uint16_t nb_pkts, uint16_t port,
 482                 const struct lcore_conf *qconf)
 483 {
 484         int32_t i;
 485         const int32_t prefetch_offset = 2;
 486
 487         for (i = 0; i < (nb_pkts - prefetch_offset); i++) {
 488                 rte_mbuf_prefetch_part2(pkts[i + prefetch_offset]);
 489                 prepare_tx_pkt(pkts[i], port, qconf);
 490         }
 491         /* Process left packets */
 492         for (; i < nb_pkts; i++)
 493                 prepare_tx_pkt(pkts[i], port, qconf);
 494 }
 495
 496 /* Send burst of packets on an output interface */
 497 static inline int32_t
 498 send_burst(struct lcore_conf *qconf, uint16_t n, uint16_t port)
 499 {
 500         struct rte_mbuf **m_table;
 501         int32_t ret;
 502         uint16_t queueid;
 503
 504         queueid = qconf->tx_queue_id[port];
 505         m_table = (struct rte_mbuf **)qconf->tx_mbufs[port].m_table;
 506
 507         prepare_tx_burst(m_table, n, port, qconf);
 508
 509         ret = rte_eth_tx_burst(port, queueid, m_table, n);
 510         if (unlikely(ret < n)) {
 511                 do {
 512                         rte_pktmbuf_free(m_table[ret]);
 513                 } while (++ret < n);
 514         }
 515
 516         return 0;
 517 }
 518
 519 /*
 520  * Helper function to fragment and queue for TX one packet.
 521  */
 522 static inline uint32_t
 523 send_fragment_packet(struct lcore_conf *qconf, struct rte_mbuf *m,
 524         uint16_t port, uint8_t proto)
 525 {
 526         struct buffer *tbl;
 527         uint32_t len, n;
 528         int32_t rc;
 529
 530         tbl =  qconf->tx_mbufs + port;
 531         len = tbl->len;
 532
 533         /* free space for new fragments */
 534         if (len + RTE_LIBRTE_IP_FRAG_MAX_FRAG >=  RTE_DIM(tbl->m_table)) {
 535                 send_burst(qconf, len, port);
 536                 len = 0;
 537         }
 538
 539         n = RTE_DIM(tbl->m_table) - len;
 540
 541         if (proto == IPPROTO_IP)
 542                 rc = rte_ipv4_fragment_packet(m, tbl->m_table + len,
 543                         n, mtu_size, qconf->frag.pool_dir,
 544                         qconf->frag.pool_indir);
 545         else
 546                 rc = rte_ipv6_fragment_packet(m, tbl->m_table + len,
 547                         n, mtu_size, qconf->frag.pool_dir,
 548                         qconf->frag.pool_indir);
 549
 550         if (rc >= 0)
 551                 len += rc;
 552         else
 553                 RTE_LOG(ERR, IPSEC,
 554                         "%s: failed to fragment packet with size %u, "
 555                         "error code: %d\n",
 556                         __func__, m->pkt_len, rte_errno);
 557
 558         rte_pktmbuf_free(m);
 559         return len;
 560 }
 561
 562 /* Enqueue a single packet, and send burst if queue is filled */
 563 static inline int32_t
 564 send_single_packet(struct rte_mbuf *m, uint16_t port, uint8_t proto)
 565 {
 566         uint32_t lcore_id;
 567         uint16_t len;
 568         struct lcore_conf *qconf;
 569
 570         lcore_id = rte_lcore_id();
 571
 572         qconf = &lcore_conf[lcore_id];
 573         len = qconf->tx_mbufs[port].len;
 574
 575         if (m->pkt_len <= mtu_size) {
 576                 qconf->tx_mbufs[port].m_table[len] = m;
 577                 len++;
 578
 579         /* need to fragment the packet */
 580         } else if (frag_tbl_sz > 0)
 581                 len = send_fragment_packet(qconf, m, port, proto);
 582         else
 583                 rte_pktmbuf_free(m);
 584
 585         /* enough pkts to be sent */
 586         if (unlikely(len == MAX_PKT_BURST)) {
 587                 send_burst(qconf, MAX_PKT_BURST, port);
 588                 len = 0;
 589         }
 590
 591         qconf->tx_mbufs[port].len = len;
 592         return 0;
 593 }
 594
 595 static inline void
 596 inbound_sp_sa(struct sp_ctx *sp, struct sa_ctx *sa, struct traffic_type *ip,
 597                 uint16_t lim)
 598 {
 599         struct rte_mbuf *m;
 600         uint32_t i, j, res, sa_idx;
 601
 602         if (ip->num == 0 || sp == NULL)
 603                 return;
 604
 605         rte_acl_classify((struct rte_acl_ctx *)sp, ip->data, ip->res,
 606                         ip->num, DEFAULT_MAX_CATEGORIES);
 607
 608         j = 0;
 609         for (i = 0; i < ip->num; i++) {
 610                 m = ip->pkts[i];
 611                 res = ip->res[i];
 612                 if (res == BYPASS) {
 613                         ip->pkts[j++] = m;
 614                         continue;
 615                 }
 616                 if (res == DISCARD) {
 617                         rte_pktmbuf_free(m);
 618                         continue;
 619                 }
 620
 621                 /* Only check SPI match for processed IPSec packets */
 622                 if (i < lim && ((m->ol_flags & PKT_RX_SEC_OFFLOAD) == 0)) {
 623                         rte_pktmbuf_free(m);
 624                         continue;
 625                 }
 626
 627                 sa_idx = res - 1;
 628                 if (!inbound_sa_check(sa, m, sa_idx)) {
 629                         rte_pktmbuf_free(m);
 630                         continue;
 631                 }
 632                 ip->pkts[j++] = m;
 633         }
 634         ip->num = j;
 635 }
 636
 637 static void
 638 split46_traffic(struct ipsec_traffic *trf, struct rte_mbuf *mb[], uint32_t num)
 639 {
 640         uint32_t i, n4, n6;
 641         struct ip *ip;
 642         struct rte_mbuf *m;
 643
 644         n4 = trf->ip4.num;
 645         n6 = trf->ip6.num;
 646
 647         for (i = 0; i < num; i++) {
 648
 649                 m = mb[i];
 650                 ip = rte_pktmbuf_mtod(m, struct ip *);
 651
 652                 if (ip->ip_v == IPVERSION) {
 653                         trf->ip4.pkts[n4] = m;
 654                         trf->ip4.data[n4] = rte_pktmbuf_mtod_offset(m,
 655                                         uint8_t *, offsetof(struct ip, ip_p));
 656                         n4++;
 657                 } else if (ip->ip_v == IP6_VERSION) {
 658                         trf->ip6.pkts[n6] = m;
 659                         trf->ip6.data[n6] = rte_pktmbuf_mtod_offset(m,
 660                                         uint8_t *,
 661                                         offsetof(struct ip6_hdr, ip6_nxt));
 662                         n6++;
 663                 } else
 664                         rte_pktmbuf_free(m);
 665         }
 666
 667         trf->ip4.num = n4;
 668         trf->ip6.num = n6;
 669 }
 670
 671
 672 static inline void
 673 process_pkts_inbound(struct ipsec_ctx *ipsec_ctx,
 674                 struct ipsec_traffic *traffic)
 675 {
 676         uint16_t nb_pkts_in, n_ip4, n_ip6;
 677
 678         n_ip4 = traffic->ip4.num;
 679         n_ip6 = traffic->ip6.num;
 680
 681         if (app_sa_prm.enable == 0) {
 682                 nb_pkts_in = ipsec_inbound(ipsec_ctx, traffic->ipsec.pkts,
 683                                 traffic->ipsec.num, MAX_PKT_BURST);
 684                 split46_traffic(traffic, traffic->ipsec.pkts, nb_pkts_in);
 685         } else {
 686                 inbound_sa_lookup(ipsec_ctx->sa_ctx, traffic->ipsec.pkts,
 687                         traffic->ipsec.saptr, traffic->ipsec.num);
 688                 ipsec_process(ipsec_ctx, traffic);
 689         }
 690
 691         inbound_sp_sa(ipsec_ctx->sp4_ctx, ipsec_ctx->sa_ctx, &traffic->ip4,
 692                         n_ip4);
 693
 694         inbound_sp_sa(ipsec_ctx->sp6_ctx, ipsec_ctx->sa_ctx, &traffic->ip6,
 695                         n_ip6);
 696 }
 697
 698 static inline void
 699 outbound_sp(struct sp_ctx *sp, struct traffic_type *ip,
 700                 struct traffic_type *ipsec)
 701 {
 702         struct rte_mbuf *m;
 703         uint32_t i, j, sa_idx;
 704
 705         if (ip->num == 0 || sp == NULL)
 706                 return;
 707
 708         rte_acl_classify((struct rte_acl_ctx *)sp, ip->data, ip->res,
 709                         ip->num, DEFAULT_MAX_CATEGORIES);
 710
 711         j = 0;
 712         for (i = 0; i < ip->num; i++) {
 713                 m = ip->pkts[i];
 714                 sa_idx = ip->res[i] - 1;
 715                 if (ip->res[i] == DISCARD)
 716                         rte_pktmbuf_free(m);
 717                 else if (ip->res[i] == BYPASS)
 718                         ip->pkts[j++] = m;
 719                 else {
 720                         ipsec->res[ipsec->num] = sa_idx;
 721                         ipsec->pkts[ipsec->num++] = m;
 722                 }
 723         }
 724         ip->num = j;
 725 }
 726
 727 static inline void
 728 process_pkts_outbound(struct ipsec_ctx *ipsec_ctx,
 729                 struct ipsec_traffic *traffic)
 730 {
 731         struct rte_mbuf *m;
 732         uint16_t idx, nb_pkts_out, i;
 733
 734         /* Drop any IPsec traffic from protected ports */
 735         for (i = 0; i < traffic->ipsec.num; i++)
 736                 rte_pktmbuf_free(traffic->ipsec.pkts[i]);
 737
 738         traffic->ipsec.num = 0;
 739
 740         outbound_sp(ipsec_ctx->sp4_ctx, &traffic->ip4, &traffic->ipsec);
 741
 742         outbound_sp(ipsec_ctx->sp6_ctx, &traffic->ip6, &traffic->ipsec);
 743
 744         if (app_sa_prm.enable == 0) {
 745
 746                 nb_pkts_out = ipsec_outbound(ipsec_ctx, traffic->ipsec.pkts,
 747                                 traffic->ipsec.res, traffic->ipsec.num,
 748                                 MAX_PKT_BURST);
 749
 750                 for (i = 0; i < nb_pkts_out; i++) {
 751                         m = traffic->ipsec.pkts[i];
 752                         struct ip *ip = rte_pktmbuf_mtod(m, struct ip *);
 753                         if (ip->ip_v == IPVERSION) {
 754                                 idx = traffic->ip4.num++;
 755                                 traffic->ip4.pkts[idx] = m;
 756                         } else {
 757                                 idx = traffic->ip6.num++;
 758                                 traffic->ip6.pkts[idx] = m;
 759                         }
 760                 }
 761         } else {
 762                 outbound_sa_lookup(ipsec_ctx->sa_ctx, traffic->ipsec.res,
 763                         traffic->ipsec.saptr, traffic->ipsec.num);
 764                 ipsec_process(ipsec_ctx, traffic);
 765         }
 766 }
 767
 768 static inline void
 769 process_pkts_inbound_nosp(struct ipsec_ctx *ipsec_ctx,
 770                 struct ipsec_traffic *traffic)
 771 {
 772         struct rte_mbuf *m;
 773         uint32_t nb_pkts_in, i, idx;
 774
 775         /* Drop any IPv4 traffic from unprotected ports */
 776         for (i = 0; i < traffic->ip4.num; i++)
 777                 rte_pktmbuf_free(traffic->ip4.pkts[i]);
 778
 779         traffic->ip4.num = 0;
 780
 781         /* Drop any IPv6 traffic from unprotected ports */
 782         for (i = 0; i < traffic->ip6.num; i++)
 783                 rte_pktmbuf_free(traffic->ip6.pkts[i]);
 784
 785         traffic->ip6.num = 0;
 786
 787         if (app_sa_prm.enable == 0) {
 788
 789                 nb_pkts_in = ipsec_inbound(ipsec_ctx, traffic->ipsec.pkts,
 790                                 traffic->ipsec.num, MAX_PKT_BURST);
 791
 792                 for (i = 0; i < nb_pkts_in; i++) {
 793                         m = traffic->ipsec.pkts[i];
 794                         struct ip *ip = rte_pktmbuf_mtod(m, struct ip *);
 795                         if (ip->ip_v == IPVERSION) {
 796                                 idx = traffic->ip4.num++;
 797                                 traffic->ip4.pkts[idx] = m;
 798                         } else {
 799                                 idx = traffic->ip6.num++;
 800                                 traffic->ip6.pkts[idx] = m;
 801                         }
 802                 }
 803         } else {
 804                 inbound_sa_lookup(ipsec_ctx->sa_ctx, traffic->ipsec.pkts,
 805                         traffic->ipsec.saptr, traffic->ipsec.num);
 806                 ipsec_process(ipsec_ctx, traffic);
 807         }
 808 }
 809
 810 static inline void
 811 process_pkts_outbound_nosp(struct ipsec_ctx *ipsec_ctx,
 812                 struct ipsec_traffic *traffic)
 813 {
 814         struct rte_mbuf *m;
 815         uint32_t nb_pkts_out, i, n;
 816         struct ip *ip;
 817
 818         /* Drop any IPsec traffic from protected ports */
 819         for (i = 0; i < traffic->ipsec.num; i++)
 820                 rte_pktmbuf_free(traffic->ipsec.pkts[i]);
 821
 822         n = 0;
 823
 824         for (i = 0; i < traffic->ip4.num; i++) {
 825                 traffic->ipsec.pkts[n] = traffic->ip4.pkts[i];
 826                 traffic->ipsec.res[n++] = single_sa_idx;
 827         }
 828
 829         for (i = 0; i < traffic->ip6.num; i++) {
 830                 traffic->ipsec.pkts[n] = traffic->ip6.pkts[i];
 831                 traffic->ipsec.res[n++] = single_sa_idx;
 832         }
 833
 834         traffic->ip4.num = 0;
 835         traffic->ip6.num = 0;
 836         traffic->ipsec.num = n;
 837
 838         if (app_sa_prm.enable == 0) {
 839
 840                 nb_pkts_out = ipsec_outbound(ipsec_ctx, traffic->ipsec.pkts,
 841                                 traffic->ipsec.res, traffic->ipsec.num,
 842                                 MAX_PKT_BURST);
 843
 844                 /* They all sue the same SA (ip4 or ip6 tunnel) */
 845                 m = traffic->ipsec.pkts[0];
 846                 ip = rte_pktmbuf_mtod(m, struct ip *);
 847                 if (ip->ip_v == IPVERSION) {
 848                         traffic->ip4.num = nb_pkts_out;
 849                         for (i = 0; i < nb_pkts_out; i++)
 850                                 traffic->ip4.pkts[i] = traffic->ipsec.pkts[i];
 851                 } else {
 852                         traffic->ip6.num = nb_pkts_out;
 853                         for (i = 0; i < nb_pkts_out; i++)
 854                                 traffic->ip6.pkts[i] = traffic->ipsec.pkts[i];
 855                 }
 856         } else {
 857                 outbound_sa_lookup(ipsec_ctx->sa_ctx, traffic->ipsec.res,
 858                         traffic->ipsec.saptr, traffic->ipsec.num);
 859                 ipsec_process(ipsec_ctx, traffic);
 860         }
 861 }
 862
 863 static inline int32_t
 864 get_hop_for_offload_pkt(struct rte_mbuf *pkt, int is_ipv6)
 865 {
 866         struct ipsec_mbuf_metadata *priv;
 867         struct ipsec_sa *sa;
 868
 869         priv = get_priv(pkt);
 870
 871         sa = priv->sa;
 872         if (unlikely(sa == NULL)) {
 873                 RTE_LOG(ERR, IPSEC, "SA not saved in private data\n");
 874                 goto fail;
 875         }
 876
 877         if (is_ipv6)
 878                 return sa->portid;
 879
 880         /* else */
 881         return (sa->portid | RTE_LPM_LOOKUP_SUCCESS);
 882
 883 fail:
 884         if (is_ipv6)
 885                 return -1;
 886
 887         /* else */
 888         return 0;
 889 }
 890
 891 static inline void
 892 route4_pkts(struct rt_ctx *rt_ctx, struct rte_mbuf *pkts[], uint8_t nb_pkts)
 893 {
 894         uint32_t hop[MAX_PKT_BURST * 2];
 895         uint32_t dst_ip[MAX_PKT_BURST * 2];
 896         int32_t pkt_hop = 0;
 897         uint16_t i, offset;
 898         uint16_t lpm_pkts = 0;
 899
 900         if (nb_pkts == 0)
 901                 return;
 902
 903         /* Need to do an LPM lookup for non-inline packets. Inline packets will
 904          * have port ID in the SA
 905          */
 906
 907         for (i = 0; i < nb_pkts; i++) {
 908                 if (!(pkts[i]->ol_flags & PKT_TX_SEC_OFFLOAD)) {
 909                         /* Security offload not enabled. So an LPM lookup is
 910                          * required to get the hop
 911                          */
 912                         offset = offsetof(struct ip, ip_dst);
 913                         dst_ip[lpm_pkts] = *rte_pktmbuf_mtod_offset(pkts[i],
 914                                         uint32_t *, offset);
 915                         dst_ip[lpm_pkts] = rte_be_to_cpu_32(dst_ip[lpm_pkts]);
 916                         lpm_pkts++;
 917                 }
 918         }
 919
 920         rte_lpm_lookup_bulk((struct rte_lpm *)rt_ctx, dst_ip, hop, lpm_pkts);
 921
 922         lpm_pkts = 0;
 923
 924         for (i = 0; i < nb_pkts; i++) {
 925                 if (pkts[i]->ol_flags & PKT_TX_SEC_OFFLOAD) {
 926                         /* Read hop from the SA */
 927                         pkt_hop = get_hop_for_offload_pkt(pkts[i], 0);
 928                 } else {
 929                         /* Need to use hop returned by lookup */
 930                         pkt_hop = hop[lpm_pkts++];
 931                 }
 932
 933                 if ((pkt_hop & RTE_LPM_LOOKUP_SUCCESS) == 0) {
 934                         rte_pktmbuf_free(pkts[i]);
 935                         continue;
 936                 }
 937                 send_single_packet(pkts[i], pkt_hop & 0xff, IPPROTO_IP);
 938         }
 939 }
 940
 941 static inline void
 942 route6_pkts(struct rt_ctx *rt_ctx, struct rte_mbuf *pkts[], uint8_t nb_pkts)
 943 {
 944         int32_t hop[MAX_PKT_BURST * 2];
 945         uint8_t dst_ip[MAX_PKT_BURST * 2][16];
 946         uint8_t *ip6_dst;
 947         int32_t pkt_hop = 0;
 948         uint16_t i, offset;
 949         uint16_t lpm_pkts = 0;
 950
 951         if (nb_pkts == 0)
 952                 return;
 953
 954         /* Need to do an LPM lookup for non-inline packets. Inline packets will
 955          * have port ID in the SA
 956          */
 957
 958         for (i = 0; i < nb_pkts; i++) {
 959                 if (!(pkts[i]->ol_flags & PKT_TX_SEC_OFFLOAD)) {
 960                         /* Security offload not enabled. So an LPM lookup is
 961                          * required to get the hop
 962                          */
 963                         offset = offsetof(struct ip6_hdr, ip6_dst);
 964                         ip6_dst = rte_pktmbuf_mtod_offset(pkts[i], uint8_t *,
 965                                         offset);
 966                         memcpy(&dst_ip[lpm_pkts][0], ip6_dst, 16);
 967                         lpm_pkts++;
 968                 }
 969         }
 970
 971         rte_lpm6_lookup_bulk_func((struct rte_lpm6 *)rt_ctx, dst_ip, hop,
 972                         lpm_pkts);
 973
 974         lpm_pkts = 0;
 975
 976         for (i = 0; i < nb_pkts; i++) {
 977                 if (pkts[i]->ol_flags & PKT_TX_SEC_OFFLOAD) {
 978                         /* Read hop from the SA */
 979                         pkt_hop = get_hop_for_offload_pkt(pkts[i], 1);
 980                 } else {
 981                         /* Need to use hop returned by lookup */
 982                         pkt_hop = hop[lpm_pkts++];
 983                 }
 984
 985                 if (pkt_hop == -1) {
 986                         rte_pktmbuf_free(pkts[i]);
 987                         continue;
 988                 }
 989                 send_single_packet(pkts[i], pkt_hop & 0xff, IPPROTO_IPV6);
 990         }
 991 }
 992
 993 static inline void
 994 process_pkts(struct lcore_conf *qconf, struct rte_mbuf **pkts,
 995                 uint8_t nb_pkts, uint16_t portid)
 996 {
 997         struct ipsec_traffic traffic;
 998
 999         prepare_traffic(pkts, &traffic, nb_pkts);
1000
1001         if (unlikely(single_sa)) {
1002                 if (is_unprotected_port(portid))
1003                         process_pkts_inbound_nosp(&qconf->inbound, &traffic);
1004                 else
1005                         process_pkts_outbound_nosp(&qconf->outbound, &traffic);
1006         } else {
1007                 if (is_unprotected_port(portid))
1008                         process_pkts_inbound(&qconf->inbound, &traffic);
1009                 else
1010                         process_pkts_outbound(&qconf->outbound, &traffic);
1011         }
1012
1013         route4_pkts(qconf->rt4_ctx, traffic.ip4.pkts, traffic.ip4.num);
1014         route6_pkts(qconf->rt6_ctx, traffic.ip6.pkts, traffic.ip6.num);
1015 }
1016
1017 static inline void
1018 drain_tx_buffers(struct lcore_conf *qconf)
1019 {
1020         struct buffer *buf;
1021         uint32_t portid;
1022
1023         for (portid = 0; portid < RTE_MAX_ETHPORTS; portid++) {
1024                 buf = &qconf->tx_mbufs[portid];
1025                 if (buf->len == 0)
1026                         continue;
1027                 send_burst(qconf, buf->len, portid);
1028                 buf->len = 0;
1029         }
1030 }
1031
1032 static inline void
1033 drain_crypto_buffers(struct lcore_conf *qconf)
1034 {
1035         uint32_t i;
1036         struct ipsec_ctx *ctx;
1037
1038         /* drain inbound buffers*/
1039         ctx = &qconf->inbound;
1040         for (i = 0; i != ctx->nb_qps; i++) {
1041                 if (ctx->tbl[i].len != 0)
1042                         enqueue_cop_burst(ctx->tbl  + i);
1043         }
1044
1045         /* drain outbound buffers*/
1046         ctx = &qconf->outbound;
1047         for (i = 0; i != ctx->nb_qps; i++) {
1048                 if (ctx->tbl[i].len != 0)
1049                         enqueue_cop_burst(ctx->tbl  + i);
1050         }
1051 }
1052
1053 static void
1054 drain_inbound_crypto_queues(const struct lcore_conf *qconf,
1055                 struct ipsec_ctx *ctx)
1056 {
1057         uint32_t n;
1058         struct ipsec_traffic trf;
1059
1060         if (app_sa_prm.enable == 0) {
1061
1062                 /* dequeue packets from crypto-queue */
1063                 n = ipsec_inbound_cqp_dequeue(ctx, trf.ipsec.pkts,
1064                         RTE_DIM(trf.ipsec.pkts));
1065
1066                 trf.ip4.num = 0;
1067                 trf.ip6.num = 0;
1068
1069                 /* split traffic by ipv4-ipv6 */
1070                 split46_traffic(&trf, trf.ipsec.pkts, n);
1071         } else
1072                 ipsec_cqp_process(ctx, &trf);
1073
1074         /* process ipv4 packets */
1075         if (trf.ip4.num != 0) {
1076                 inbound_sp_sa(ctx->sp4_ctx, ctx->sa_ctx, &trf.ip4, 0);
1077                 route4_pkts(qconf->rt4_ctx, trf.ip4.pkts, trf.ip4.num);
1078         }
1079
1080         /* process ipv6 packets */
1081         if (trf.ip6.num != 0) {
1082                 inbound_sp_sa(ctx->sp6_ctx, ctx->sa_ctx, &trf.ip6, 0);
1083                 route6_pkts(qconf->rt6_ctx, trf.ip6.pkts, trf.ip6.num);
1084         }
1085 }
1086
1087 static void
1088 drain_outbound_crypto_queues(const struct lcore_conf *qconf,
1089                 struct ipsec_ctx *ctx)
1090 {
1091         uint32_t n;
1092         struct ipsec_traffic trf;
1093
1094         if (app_sa_prm.enable == 0) {
1095
1096                 /* dequeue packets from crypto-queue */
1097                 n = ipsec_outbound_cqp_dequeue(ctx, trf.ipsec.pkts,
1098                         RTE_DIM(trf.ipsec.pkts));
1099
1100                 trf.ip4.num = 0;
1101                 trf.ip6.num = 0;
1102
1103                 /* split traffic by ipv4-ipv6 */
1104                 split46_traffic(&trf, trf.ipsec.pkts, n);
1105         } else
1106                 ipsec_cqp_process(ctx, &trf);
1107
1108         /* process ipv4 packets */
1109         if (trf.ip4.num != 0)
1110                 route4_pkts(qconf->rt4_ctx, trf.ip4.pkts, trf.ip4.num);
1111
1112         /* process ipv6 packets */
1113         if (trf.ip6.num != 0)
1114                 route6_pkts(qconf->rt6_ctx, trf.ip6.pkts, trf.ip6.num);
1115 }
1116
1117 /* main processing loop */
1118 void
1119 ipsec_poll_mode_worker(void)
1120 {
1121         struct rte_mbuf *pkts[MAX_PKT_BURST];
1122         uint32_t lcore_id;
1123         uint64_t prev_tsc, diff_tsc, cur_tsc;
1124         int32_t i, nb_rx;
1125         uint16_t portid;
1126         uint8_t queueid;
1127         struct lcore_conf *qconf;
1128         int32_t rc, socket_id;
1129         const uint64_t drain_tsc = (rte_get_tsc_hz() + US_PER_S - 1)
1130                         / US_PER_S * BURST_TX_DRAIN_US;
1131         struct lcore_rx_queue *rxql;
1132
1133         prev_tsc = 0;
1134         lcore_id = rte_lcore_id();
1135         qconf = &lcore_conf[lcore_id];
1136         rxql = qconf->rx_queue_list;
1137         socket_id = rte_lcore_to_socket_id(lcore_id);
1138
1139         qconf->rt4_ctx = socket_ctx[socket_id].rt_ip4;
1140         qconf->rt6_ctx = socket_ctx[socket_id].rt_ip6;
1141         qconf->inbound.sp4_ctx = socket_ctx[socket_id].sp_ip4_in;
1142         qconf->inbound.sp6_ctx = socket_ctx[socket_id].sp_ip6_in;
1143         qconf->inbound.sa_ctx = socket_ctx[socket_id].sa_in;
1144         qconf->inbound.cdev_map = cdev_map_in;
1145         qconf->inbound.session_pool = socket_ctx[socket_id].session_pool;
1146         qconf->inbound.session_priv_pool =
1147                         socket_ctx[socket_id].session_priv_pool;
1148         qconf->outbound.sp4_ctx = socket_ctx[socket_id].sp_ip4_out;
1149         qconf->outbound.sp6_ctx = socket_ctx[socket_id].sp_ip6_out;
1150         qconf->outbound.sa_ctx = socket_ctx[socket_id].sa_out;
1151         qconf->outbound.cdev_map = cdev_map_out;
1152         qconf->outbound.session_pool = socket_ctx[socket_id].session_pool;
1153         qconf->outbound.session_priv_pool =
1154                         socket_ctx[socket_id].session_priv_pool;
1155         qconf->frag.pool_dir = socket_ctx[socket_id].mbuf_pool;
1156         qconf->frag.pool_indir = socket_ctx[socket_id].mbuf_pool_indir;
1157
1158         rc = ipsec_sad_lcore_cache_init(app_sa_prm.cache_sz);
1159         if (rc != 0) {
1160                 RTE_LOG(ERR, IPSEC,
1161                         "SAD cache init on lcore %u, failed with code: %d\n",
1162                         lcore_id, rc);
1163                 return;
1164         }
1165
1166         if (qconf->nb_rx_queue == 0) {
1167                 RTE_LOG(DEBUG, IPSEC, "lcore %u has nothing to do\n",
1168                         lcore_id);
1169                 return;
1170         }
1171
1172         RTE_LOG(INFO, IPSEC, "entering main loop on lcore %u\n", lcore_id);
1173
1174         for (i = 0; i < qconf->nb_rx_queue; i++) {
1175                 portid = rxql[i].port_id;
1176                 queueid = rxql[i].queue_id;
1177                 RTE_LOG(INFO, IPSEC,
1178                         " -- lcoreid=%u portid=%u rxqueueid=%hhu\n",
1179                         lcore_id, portid, queueid);
1180         }
1181
1182         while (!force_quit) {
1183                 cur_tsc = rte_rdtsc();
1184
1185                 /* TX queue buffer drain */
1186                 diff_tsc = cur_tsc - prev_tsc;
1187
1188                 if (unlikely(diff_tsc > drain_tsc)) {
1189                         drain_tx_buffers(qconf);
1190                         drain_crypto_buffers(qconf);
1191                         prev_tsc = cur_tsc;
1192                 }
1193
1194                 for (i = 0; i < qconf->nb_rx_queue; ++i) {
1195
1196                         /* Read packets from RX queues */
1197                         portid = rxql[i].port_id;
1198                         queueid = rxql[i].queue_id;
1199                         nb_rx = rte_eth_rx_burst(portid, queueid,
1200                                         pkts, MAX_PKT_BURST);
1201
1202                         if (nb_rx > 0)
1203                                 process_pkts(qconf, pkts, nb_rx, portid);
1204
1205                         /* dequeue and process completed crypto-ops */
1206                         if (is_unprotected_port(portid))
1207                                 drain_inbound_crypto_queues(qconf,
1208                                         &qconf->inbound);
1209                         else
1210                                 drain_outbound_crypto_queues(qconf,
1211                                         &qconf->outbound);
1212                 }
1213         }
1214 }
1215
1216 static int32_t
1217 check_poll_mode_params(struct eh_conf *eh_conf)
1218 {
1219         uint8_t lcore;
1220         uint16_t portid;
1221         uint16_t i;
1222         int32_t socket_id;
1223
1224         if (!eh_conf)
1225                 return -EINVAL;
1226
1227         if (eh_conf->mode != EH_PKT_TRANSFER_MODE_POLL)
1228                 return 0;
1229
1230         if (lcore_params == NULL) {
1231                 printf("Error: No port/queue/core mappings\n");
1232                 return -1;
1233         }
1234
1235         for (i = 0; i < nb_lcore_params; ++i) {
1236                 lcore = lcore_params[i].lcore_id;
1237                 if (!rte_lcore_is_enabled(lcore)) {
1238                         printf("error: lcore %hhu is not enabled in "
1239                                 "lcore mask\n", lcore);
1240                         return -1;
1241                 }
1242                 socket_id = rte_lcore_to_socket_id(lcore);
1243                 if (socket_id != 0 && numa_on == 0) {
1244                         printf("warning: lcore %hhu is on socket %d "
1245                                 "with numa off\n",
1246                                 lcore, socket_id);
1247                 }
1248                 portid = lcore_params[i].port_id;
1249                 if ((enabled_port_mask & (1 << portid)) == 0) {
1250                         printf("port %u is not enabled in port mask\n", portid);
1251                         return -1;
1252                 }
1253                 if (!rte_eth_dev_is_valid_port(portid)) {
1254                         printf("port %u is not present on the board\n", portid);
1255                         return -1;
1256                 }
1257         }
1258         return 0;
1259 }
1260
1261 static uint8_t
1262 get_port_nb_rx_queues(const uint16_t port)
1263 {
1264         int32_t queue = -1;
1265         uint16_t i;
1266
1267         for (i = 0; i < nb_lcore_params; ++i) {
1268                 if (lcore_params[i].port_id == port &&
1269                                 lcore_params[i].queue_id > queue)
1270                         queue = lcore_params[i].queue_id;
1271         }
1272         return (uint8_t)(++queue);
1273 }
1274
1275 static int32_t
1276 init_lcore_rx_queues(void)
1277 {
1278         uint16_t i, nb_rx_queue;
1279         uint8_t lcore;
1280
1281         for (i = 0; i < nb_lcore_params; ++i) {
1282                 lcore = lcore_params[i].lcore_id;
1283                 nb_rx_queue = lcore_conf[lcore].nb_rx_queue;
1284                 if (nb_rx_queue >= MAX_RX_QUEUE_PER_LCORE) {
1285                         printf("error: too many queues (%u) for lcore: %u\n",
1286                                         nb_rx_queue + 1, lcore);
1287                         return -1;
1288                 }
1289                 lcore_conf[lcore].rx_queue_list[nb_rx_queue].port_id =
1290                         lcore_params[i].port_id;
1291                 lcore_conf[lcore].rx_queue_list[nb_rx_queue].queue_id =
1292                         lcore_params[i].queue_id;
1293                 lcore_conf[lcore].nb_rx_queue++;
1294         }
1295         return 0;
1296 }
1297
1298 /* display usage */
1299 static void
1300 print_usage(const char *prgname)
1301 {
1302         fprintf(stderr, "%s [EAL options] --"
1303                 " -p PORTMASK"
1304                 " [-P]"
1305                 " [-u PORTMASK]"
1306                 " [-j FRAMESIZE]"
1307                 " [-l]"
1308                 " [-w REPLAY_WINDOW_SIZE]"
1309                 " [-e]"
1310                 " [-a]"
1311                 " [-c]"
1312                 " -f CONFIG_FILE"
1313                 " --config (port,queue,lcore)[,(port,queue,lcore)]"
1314                 " [--single-sa SAIDX]"
1315                 " [--cryptodev_mask MASK]"
1316                 " [--transfer-mode MODE]"
1317                 " [--event-schedule-type TYPE]"
1318                 " [--" CMD_LINE_OPT_RX_OFFLOAD " RX_OFFLOAD_MASK]"
1319                 " [--" CMD_LINE_OPT_TX_OFFLOAD " TX_OFFLOAD_MASK]"
1320                 " [--" CMD_LINE_OPT_REASSEMBLE " REASSEMBLE_TABLE_SIZE]"
1321                 " [--" CMD_LINE_OPT_MTU " MTU]"
1322                 "\n\n"
1323                 "  -p PORTMASK: Hexadecimal bitmask of ports to configure\n"
1324                 "  -P : Enable promiscuous mode\n"
1325                 "  -u PORTMASK: Hexadecimal bitmask of unprotected ports\n"
1326                 "  -j FRAMESIZE: Data buffer size, minimum (and default)\n"
1327                 "     value: RTE_MBUF_DEFAULT_BUF_SIZE\n"
1328                 "  -l enables code-path that uses librte_ipsec\n"
1329                 "  -w REPLAY_WINDOW_SIZE specifies IPsec SQN replay window\n"
1330                 "     size for each SA\n"
1331                 "  -e enables ESN\n"
1332                 "  -a enables SA SQN atomic behaviour\n"
1333                 "  -c specifies inbound SAD cache size,\n"
1334                 "     zero value disables the cache (default value: 128)\n"
1335                 "  -f CONFIG_FILE: Configuration file\n"
1336                 "  --config (port,queue,lcore): Rx queue configuration. In poll\n"
1337                 "                               mode determines which queues from\n"
1338                 "                               which ports are mapped to which cores.\n"
1339                 "                               In event mode this option is not used\n"
1340                 "                               as packets are dynamically scheduled\n"
1341                 "                               to cores by HW.\n"
1342                 "  --single-sa SAIDX: In poll mode use single SA index for\n"
1343                 "                     outbound traffic, bypassing the SP\n"
1344                 "                     In event mode selects driver submode,\n"
1345                 "                     SA index value is ignored\n"
1346                 "  --cryptodev_mask MASK: Hexadecimal bitmask of the crypto\n"
1347                 "                         devices to configure\n"
1348                 "  --transfer-mode MODE\n"
1349                 "               \"poll\"  : Packet transfer via polling (default)\n"
1350                 "               \"event\" : Packet transfer via event device\n"
1351                 "  --event-schedule-type TYPE queue schedule type, used only when\n"
1352                 "                             transfer mode is set to event\n"
1353                 "               \"ordered\"  : Ordered (default)\n"
1354                 "               \"atomic\"   : Atomic\n"
1355                 "               \"parallel\" : Parallel\n"
1356                 "  --" CMD_LINE_OPT_RX_OFFLOAD
1357                 ": bitmask of the RX HW offload capabilities to enable/use\n"
1358                 "                         (DEV_RX_OFFLOAD_*)\n"
1359                 "  --" CMD_LINE_OPT_TX_OFFLOAD
1360                 ": bitmask of the TX HW offload capabilities to enable/use\n"
1361                 "                         (DEV_TX_OFFLOAD_*)\n"
1362                 "  --" CMD_LINE_OPT_REASSEMBLE " NUM"
1363                 ": max number of entries in reassemble(fragment) table\n"
1364                 "    (zero (default value) disables reassembly)\n"
1365                 "  --" CMD_LINE_OPT_MTU " MTU"
1366                 ": MTU value on all ports (default value: 1500)\n"
1367                 "    outgoing packets with bigger size will be fragmented\n"
1368                 "    incoming packets with bigger size will be discarded\n"
1369                 "  --" CMD_LINE_OPT_FRAG_TTL " FRAG_TTL_NS"
1370                 ": fragments lifetime in nanoseconds, default\n"
1371                 "    and maximum value is 10.000.000.000 ns (10 s)\n"
1372                 "\n",
1373                 prgname);
1374 }
1375
1376 static int
1377 parse_mask(const char *str, uint64_t *val)
1378 {
1379         char *end;
1380         unsigned long t;
1381
1382         errno = 0;
1383         t = strtoul(str, &end, 0);
1384         if (errno != 0 || end[0] != 0)
1385                 return -EINVAL;
1386
1387         *val = t;
1388         return 0;
1389 }
1390
1391 static int32_t
1392 parse_portmask(const char *portmask)
1393 {
1394         char *end = NULL;
1395         unsigned long pm;
1396
1397         /* parse hexadecimal string */
1398         pm = strtoul(portmask, &end, 16);
1399         if ((portmask[0] == '\0') || (end == NULL) || (*end != '\0'))
1400                 return -1;
1401
1402         if ((pm == 0) && errno)
1403                 return -1;
1404
1405         return pm;
1406 }
1407
1408 static int64_t
1409 parse_decimal(const char *str)
1410 {
1411         char *end = NULL;
1412         uint64_t num;
1413
1414         num = strtoull(str, &end, 10);
1415         if ((str[0] == '\0') || (end == NULL) || (*end != '\0')
1416                 || num > INT64_MAX)
1417                 return -1;
1418
1419         return num;
1420 }
1421
1422 static int32_t
1423 parse_config(const char *q_arg)
1424 {
1425         char s[256];
1426         const char *p, *p0 = q_arg;
1427         char *end;
1428         enum fieldnames {
1429                 FLD_PORT = 0,
1430                 FLD_QUEUE,
1431                 FLD_LCORE,
1432                 _NUM_FLD
1433         };
1434         unsigned long int_fld[_NUM_FLD];
1435         char *str_fld[_NUM_FLD];
1436         int32_t i;
1437         uint32_t size;
1438
1439         nb_lcore_params = 0;
1440
1441         while ((p = strchr(p0, '(')) != NULL) {
1442                 ++p;
1443                 p0 = strchr(p, ')');
1444                 if (p0 == NULL)
1445                         return -1;
1446
1447                 size = p0 - p;
1448                 if (size >= sizeof(s))
1449                         return -1;
1450
1451                 snprintf(s, sizeof(s), "%.*s", size, p);
1452                 if (rte_strsplit(s, sizeof(s), str_fld, _NUM_FLD, ',') !=
1453                                 _NUM_FLD)
1454                         return -1;
1455                 for (i = 0; i < _NUM_FLD; i++) {
1456                         errno = 0;
1457                         int_fld[i] = strtoul(str_fld[i], &end, 0);
1458                         if (errno != 0 || end == str_fld[i] || int_fld[i] > 255)
1459                                 return -1;
1460                 }
1461                 if (nb_lcore_params >= MAX_LCORE_PARAMS) {
1462                         printf("exceeded max number of lcore params: %hu\n",
1463                                 nb_lcore_params);
1464                         return -1;
1465                 }
1466                 lcore_params_array[nb_lcore_params].port_id =
1467                         (uint8_t)int_fld[FLD_PORT];
1468                 lcore_params_array[nb_lcore_params].queue_id =
1469                         (uint8_t)int_fld[FLD_QUEUE];
1470                 lcore_params_array[nb_lcore_params].lcore_id =
1471                         (uint8_t)int_fld[FLD_LCORE];
1472                 ++nb_lcore_params;
1473         }
1474         lcore_params = lcore_params_array;
1475         return 0;
1476 }
1477
1478 static void
1479 print_app_sa_prm(const struct app_sa_prm *prm)
1480 {
1481         printf("librte_ipsec usage: %s\n",
1482                 (prm->enable == 0) ? "disabled" : "enabled");
1483
1484         printf("replay window size: %u\n", prm->window_size);
1485         printf("ESN: %s\n", (prm->enable_esn == 0) ? "disabled" : "enabled");
1486         printf("SA flags: %#" PRIx64 "\n", prm->flags);
1487         printf("Frag TTL: %" PRIu64 " ns\n", frag_ttl_ns);
1488 }
1489
1490 static int
1491 parse_transfer_mode(struct eh_conf *conf, const char *optarg)
1492 {
1493         if (!strcmp(CMD_LINE_ARG_POLL, optarg))
1494                 conf->mode = EH_PKT_TRANSFER_MODE_POLL;
1495         else if (!strcmp(CMD_LINE_ARG_EVENT, optarg))
1496                 conf->mode = EH_PKT_TRANSFER_MODE_EVENT;
1497         else {
1498                 printf("Unsupported packet transfer mode\n");
1499                 return -EINVAL;
1500         }
1501
1502         return 0;
1503 }
1504
1505 static int
1506 parse_schedule_type(struct eh_conf *conf, const char *optarg)
1507 {
1508         struct eventmode_conf *em_conf = NULL;
1509
1510         /* Get eventmode conf */
1511         em_conf = conf->mode_params;
1512
1513         if (!strcmp(CMD_LINE_ARG_ORDERED, optarg))
1514                 em_conf->ext_params.sched_type = RTE_SCHED_TYPE_ORDERED;
1515         else if (!strcmp(CMD_LINE_ARG_ATOMIC, optarg))
1516                 em_conf->ext_params.sched_type = RTE_SCHED_TYPE_ATOMIC;
1517         else if (!strcmp(CMD_LINE_ARG_PARALLEL, optarg))
1518                 em_conf->ext_params.sched_type = RTE_SCHED_TYPE_PARALLEL;
1519         else {
1520                 printf("Unsupported queue schedule type\n");
1521                 return -EINVAL;
1522         }
1523
1524         return 0;
1525 }
1526
1527 static int32_t
1528 parse_args(int32_t argc, char **argv, struct eh_conf *eh_conf)
1529 {
1530         int opt;
1531         int64_t ret;
1532         char **argvopt;
1533         int32_t option_index;
1534         char *prgname = argv[0];
1535         int32_t f_present = 0;
1536
1537         argvopt = argv;
1538
1539         while ((opt = getopt_long(argc, argvopt, "aelp:Pu:f:j:w:c:",
1540                                 lgopts, &option_index)) != EOF) {
1541
1542                 switch (opt) {
1543                 case 'p':
1544                         enabled_port_mask = parse_portmask(optarg);
1545                         if (enabled_port_mask == 0) {
1546                                 printf("invalid portmask\n");
1547                                 print_usage(prgname);
1548                                 return -1;
1549                         }
1550                         break;
1551                 case 'P':
1552                         printf("Promiscuous mode selected\n");
1553                         promiscuous_on = 1;
1554                         break;
1555                 case 'u':
1556                         unprotected_port_mask = parse_portmask(optarg);
1557                         if (unprotected_port_mask == 0) {
1558                                 printf("invalid unprotected portmask\n");
1559                                 print_usage(prgname);
1560                                 return -1;
1561                         }
1562                         break;
1563                 case 'f':
1564                         if (f_present == 1) {
1565                                 printf("\"-f\" option present more than "
1566                                         "once!\n");
1567                                 print_usage(prgname);
1568                                 return -1;
1569                         }
1570                         cfgfile = optarg;
1571                         f_present = 1;
1572                         break;
1573                 case 'j':
1574                         ret = parse_decimal(optarg);
1575                         if (ret < RTE_MBUF_DEFAULT_BUF_SIZE ||
1576                                         ret > UINT16_MAX) {
1577                                 printf("Invalid frame buffer size value: %s\n",
1578                                         optarg);
1579                                 print_usage(prgname);
1580                                 return -1;
1581                         }
1582                         frame_buf_size = ret;
1583                         printf("Custom frame buffer size %u\n", frame_buf_size);
1584                         break;
1585                 case 'l':
1586                         app_sa_prm.enable = 1;
1587                         break;
1588                 case 'w':
1589                         app_sa_prm.window_size = parse_decimal(optarg);
1590                         break;
1591                 case 'e':
1592                         app_sa_prm.enable_esn = 1;
1593                         break;
1594                 case 'a':
1595                         app_sa_prm.enable = 1;
1596                         app_sa_prm.flags |= RTE_IPSEC_SAFLAG_SQN_ATOM;
1597                         break;
1598                 case 'c':
1599                         ret = parse_decimal(optarg);
1600                         if (ret < 0) {
1601                                 printf("Invalid SA cache size: %s\n", optarg);
1602                                 print_usage(prgname);
1603                                 return -1;
1604                         }
1605                         app_sa_prm.cache_sz = ret;
1606                         break;
1607                 case CMD_LINE_OPT_CONFIG_NUM:
1608                         ret = parse_config(optarg);
1609                         if (ret) {
1610                                 printf("Invalid config\n");
1611                                 print_usage(prgname);
1612                                 return -1;
1613                         }
1614                         break;
1615                 case CMD_LINE_OPT_SINGLE_SA_NUM:
1616                         ret = parse_decimal(optarg);
1617                         if (ret == -1 || ret > UINT32_MAX) {
1618                                 printf("Invalid argument[sa_idx]\n");
1619                                 print_usage(prgname);
1620                                 return -1;
1621                         }
1622
1623                         /* else */
1624                         single_sa = 1;
1625                         single_sa_idx = ret;
1626                         eh_conf->ipsec_mode = EH_IPSEC_MODE_TYPE_DRIVER;
1627                         printf("Configured with single SA index %u\n",
1628                                         single_sa_idx);
1629                         break;
1630                 case CMD_LINE_OPT_CRYPTODEV_MASK_NUM:
1631                         ret = parse_portmask(optarg);
1632                         if (ret == -1) {
1633                                 printf("Invalid argument[portmask]\n");
1634                                 print_usage(prgname);
1635                                 return -1;
1636                         }
1637
1638                         /* else */
1639                         enabled_cryptodev_mask = ret;
1640                         break;
1641
1642                 case CMD_LINE_OPT_TRANSFER_MODE_NUM:
1643                         ret = parse_transfer_mode(eh_conf, optarg);
1644                         if (ret < 0) {
1645                                 printf("Invalid packet transfer mode\n");
1646                                 print_usage(prgname);
1647                                 return -1;
1648                         }
1649                         break;
1650
1651                 case CMD_LINE_OPT_SCHEDULE_TYPE_NUM:
1652                         ret = parse_schedule_type(eh_conf, optarg);
1653                         if (ret < 0) {
1654                                 printf("Invalid queue schedule type\n");
1655                                 print_usage(prgname);
1656                                 return -1;
1657                         }
1658                         break;
1659
1660                 case CMD_LINE_OPT_RX_OFFLOAD_NUM:
1661                         ret = parse_mask(optarg, &dev_rx_offload);
1662                         if (ret != 0) {
1663                                 printf("Invalid argument for \'%s\': %s\n",
1664                                         CMD_LINE_OPT_RX_OFFLOAD, optarg);
1665                                 print_usage(prgname);
1666                                 return -1;
1667                         }
1668                         break;
1669                 case CMD_LINE_OPT_TX_OFFLOAD_NUM:
1670                         ret = parse_mask(optarg, &dev_tx_offload);
1671                         if (ret != 0) {
1672                                 printf("Invalid argument for \'%s\': %s\n",
1673                                         CMD_LINE_OPT_TX_OFFLOAD, optarg);
1674                                 print_usage(prgname);
1675                                 return -1;
1676                         }
1677                         break;
1678                 case CMD_LINE_OPT_REASSEMBLE_NUM:
1679                         ret = parse_decimal(optarg);
1680                         if (ret < 0 || ret > UINT32_MAX) {
1681                                 printf("Invalid argument for \'%s\': %s\n",
1682                                         CMD_LINE_OPT_REASSEMBLE, optarg);
1683                                 print_usage(prgname);
1684                                 return -1;
1685                         }
1686                         frag_tbl_sz = ret;
1687                         break;
1688                 case CMD_LINE_OPT_MTU_NUM:
1689                         ret = parse_decimal(optarg);
1690                         if (ret < 0 || ret > RTE_IPV4_MAX_PKT_LEN) {
1691                                 printf("Invalid argument for \'%s\': %s\n",
1692                                         CMD_LINE_OPT_MTU, optarg);
1693                                 print_usage(prgname);
1694                                 return -1;
1695                         }
1696                         mtu_size = ret;
1697                         break;
1698                 case CMD_LINE_OPT_FRAG_TTL_NUM:
1699                         ret = parse_decimal(optarg);
1700                         if (ret < 0 || ret > MAX_FRAG_TTL_NS) {
1701                                 printf("Invalid argument for \'%s\': %s\n",
1702                                         CMD_LINE_OPT_MTU, optarg);
1703                                 print_usage(prgname);
1704                                 return -1;
1705                         }
1706                         frag_ttl_ns = ret;
1707                         break;
1708                 default:
1709                         print_usage(prgname);
1710                         return -1;
1711                 }
1712         }
1713
1714         if (f_present == 0) {
1715                 printf("Mandatory option \"-f\" not present\n");
1716                 return -1;
1717         }
1718
1719         /* check do we need to enable multi-seg support */
1720         if (multi_seg_required()) {
1721                 /* legacy mode doesn't support multi-seg */
1722                 app_sa_prm.enable = 1;
1723                 printf("frame buf size: %u, mtu: %u, "
1724                         "number of reassemble entries: %u\n"
1725                         "multi-segment support is required\n",
1726                         frame_buf_size, mtu_size, frag_tbl_sz);
1727         }
1728
1729         print_app_sa_prm(&app_sa_prm);
1730
1731         if (optind >= 0)
1732                 argv[optind-1] = prgname;
1733
1734         ret = optind-1;
1735         optind = 1; /* reset getopt lib */
1736         return ret;
1737 }
1738
1739 static void
1740 print_ethaddr(const char *name, const struct rte_ether_addr *eth_addr)
1741 {
1742         char buf[RTE_ETHER_ADDR_FMT_SIZE];
1743         rte_ether_format_addr(buf, RTE_ETHER_ADDR_FMT_SIZE, eth_addr);
1744         printf("%s%s", name, buf);
1745 }
1746
1747 /*
1748  * Update destination ethaddr for the port.
1749  */
1750 int
1751 add_dst_ethaddr(uint16_t port, const struct rte_ether_addr *addr)
1752 {
1753         if (port >= RTE_DIM(ethaddr_tbl))
1754                 return -EINVAL;
1755
1756         ethaddr_tbl[port].dst = ETHADDR_TO_UINT64(addr);
1757         return 0;
1758 }
1759
1760 /* Check the link status of all ports in up to 9s, and print them finally */
1761 static void
1762 check_all_ports_link_status(uint32_t port_mask)
1763 {
1764 #define CHECK_INTERVAL 100 /* 100ms */
1765 #define MAX_CHECK_TIME 90 /* 9s (90 * 100ms) in total */
1766         uint16_t portid;
1767         uint8_t count, all_ports_up, print_flag = 0;
1768         struct rte_eth_link link;
1769         int ret;
1770
1771         printf("\nChecking link status");
1772         fflush(stdout);
1773         for (count = 0; count <= MAX_CHECK_TIME; count++) {
1774                 all_ports_up = 1;
1775                 RTE_ETH_FOREACH_DEV(portid) {
1776                         if ((port_mask & (1 << portid)) == 0)
1777                                 continue;
1778                         memset(&link, 0, sizeof(link));
1779                         ret = rte_eth_link_get_nowait(portid, &link);
1780                         if (ret < 0) {
1781                                 all_ports_up = 0;
1782                                 if (print_flag == 1)
1783                                         printf("Port %u link get failed: %s\n",
1784                                                 portid, rte_strerror(-ret));
1785                                 continue;
1786                         }
1787                         /* print link status if flag set */
1788                         if (print_flag == 1) {
1789                                 if (link.link_status)
1790                                         printf(
1791                                         "Port%d Link Up - speed %u Mbps -%s\n",
1792                                                 portid, link.link_speed,
1793                                 (link.link_duplex == ETH_LINK_FULL_DUPLEX) ?
1794                                         ("full-duplex") : ("half-duplex\n"));
1795                                 else
1796                                         printf("Port %d Link Down\n", portid);
1797                                 continue;
1798                         }
1799                         /* clear all_ports_up flag if any link down */
1800                         if (link.link_status == ETH_LINK_DOWN) {
1801                                 all_ports_up = 0;
1802                                 break;
1803                         }
1804                 }
1805                 /* after finally printing all link status, get out */
1806                 if (print_flag == 1)
1807                         break;
1808
1809                 if (all_ports_up == 0) {
1810                         printf(".");
1811                         fflush(stdout);
1812                         rte_delay_ms(CHECK_INTERVAL);
1813                 }
1814
1815                 /* set the print_flag if all ports up or timeout */
1816                 if (all_ports_up == 1 || count == (MAX_CHECK_TIME - 1)) {
1817                         print_flag = 1;
1818                         printf("done\n");
1819                 }
1820         }
1821 }
1822
1823 static int32_t
1824 add_mapping(struct rte_hash *map, const char *str, uint16_t cdev_id,
1825                 uint16_t qp, struct lcore_params *params,
1826                 struct ipsec_ctx *ipsec_ctx,
1827                 const struct rte_cryptodev_capabilities *cipher,
1828                 const struct rte_cryptodev_capabilities *auth,
1829                 const struct rte_cryptodev_capabilities *aead)
1830 {
1831         int32_t ret = 0;
1832         unsigned long i;
1833         struct cdev_key key = { 0 };
1834
1835         key.lcore_id = params->lcore_id;
1836         if (cipher)
1837                 key.cipher_algo = cipher->sym.cipher.algo;
1838         if (auth)
1839                 key.auth_algo = auth->sym.auth.algo;
1840         if (aead)
1841                 key.aead_algo = aead->sym.aead.algo;
1842
1843         ret = rte_hash_lookup(map, &key);
1844         if (ret != -ENOENT)
1845                 return 0;
1846
1847         for (i = 0; i < ipsec_ctx->nb_qps; i++)
1848                 if (ipsec_ctx->tbl[i].id == cdev_id)
1849                         break;
1850
1851         if (i == ipsec_ctx->nb_qps) {
1852                 if (ipsec_ctx->nb_qps == MAX_QP_PER_LCORE) {
1853                         printf("Maximum number of crypto devices assigned to "
1854                                 "a core, increase MAX_QP_PER_LCORE value\n");
1855                         return 0;
1856                 }
1857                 ipsec_ctx->tbl[i].id = cdev_id;
1858                 ipsec_ctx->tbl[i].qp = qp;
1859                 ipsec_ctx->nb_qps++;
1860                 printf("%s cdev mapping: lcore %u using cdev %u qp %u "
1861                                 "(cdev_id_qp %lu)\n", str, key.lcore_id,
1862                                 cdev_id, qp, i);
1863         }
1864
1865         ret = rte_hash_add_key_data(map, &key, (void *)i);
1866         if (ret < 0) {
1867                 printf("Faled to insert cdev mapping for (lcore %u, "
1868                                 "cdev %u, qp %u), errno %d\n",
1869                                 key.lcore_id, ipsec_ctx->tbl[i].id,
1870                                 ipsec_ctx->tbl[i].qp, ret);
1871                 return 0;
1872         }
1873
1874         return 1;
1875 }
1876
1877 static int32_t
1878 add_cdev_mapping(struct rte_cryptodev_info *dev_info, uint16_t cdev_id,
1879                 uint16_t qp, struct lcore_params *params)
1880 {
1881         int32_t ret = 0;
1882         const struct rte_cryptodev_capabilities *i, *j;
1883         struct rte_hash *map;
1884         struct lcore_conf *qconf;
1885         struct ipsec_ctx *ipsec_ctx;
1886         const char *str;
1887
1888         qconf = &lcore_conf[params->lcore_id];
1889
1890         if ((unprotected_port_mask & (1 << params->port_id)) == 0) {
1891                 map = cdev_map_out;
1892                 ipsec_ctx = &qconf->outbound;
1893                 str = "Outbound";
1894         } else {
1895                 map = cdev_map_in;
1896                 ipsec_ctx = &qconf->inbound;
1897                 str = "Inbound";
1898         }
1899
1900         /* Required cryptodevs with operation chainning */
1901         if (!(dev_info->feature_flags &
1902                                 RTE_CRYPTODEV_FF_SYM_OPERATION_CHAINING))
1903                 return ret;
1904
1905         for (i = dev_info->capabilities;
1906                         i->op != RTE_CRYPTO_OP_TYPE_UNDEFINED; i++) {
1907                 if (i->op != RTE_CRYPTO_OP_TYPE_SYMMETRIC)
1908                         continue;
1909
1910                 if (i->sym.xform_type == RTE_CRYPTO_SYM_XFORM_AEAD) {
1911                         ret |= add_mapping(map, str, cdev_id, qp, params,
1912                                         ipsec_ctx, NULL, NULL, i);
1913                         continue;
1914                 }
1915
1916                 if (i->sym.xform_type != RTE_CRYPTO_SYM_XFORM_CIPHER)
1917                         continue;
1918
1919                 for (j = dev_info->capabilities;
1920                                 j->op != RTE_CRYPTO_OP_TYPE_UNDEFINED; j++) {
1921                         if (j->op != RTE_CRYPTO_OP_TYPE_SYMMETRIC)
1922                                 continue;
1923
1924                         if (j->sym.xform_type != RTE_CRYPTO_SYM_XFORM_AUTH)
1925                                 continue;
1926
1927                         ret |= add_mapping(map, str, cdev_id, qp, params,
1928                                                 ipsec_ctx, i, j, NULL);
1929                 }
1930         }
1931
1932         return ret;
1933 }
1934
1935 /* Check if the device is enabled by cryptodev_mask */
1936 static int
1937 check_cryptodev_mask(uint8_t cdev_id)
1938 {
1939         if (enabled_cryptodev_mask & (1 << cdev_id))
1940                 return 0;
1941
1942         return -1;
1943 }
1944
1945 static int32_t
1946 cryptodevs_init(void)
1947 {
1948         struct rte_cryptodev_config dev_conf;
1949         struct rte_cryptodev_qp_conf qp_conf;
1950         uint16_t idx, max_nb_qps, qp, i;
1951         int16_t cdev_id;
1952         struct rte_hash_parameters params = { 0 };
1953
1954         const uint64_t mseg_flag = multi_seg_required() ?
1955                                 RTE_CRYPTODEV_FF_IN_PLACE_SGL : 0;
1956
1957         params.entries = CDEV_MAP_ENTRIES;
1958         params.key_len = sizeof(struct cdev_key);
1959         params.hash_func = rte_jhash;
1960         params.hash_func_init_val = 0;
1961         params.socket_id = rte_socket_id();
1962
1963         params.name = "cdev_map_in";
1964         cdev_map_in = rte_hash_create(&params);
1965         if (cdev_map_in == NULL)
1966                 rte_panic("Failed to create cdev_map hash table, errno = %d\n",
1967                                 rte_errno);
1968
1969         params.name = "cdev_map_out";
1970         cdev_map_out = rte_hash_create(&params);
1971         if (cdev_map_out == NULL)
1972                 rte_panic("Failed to create cdev_map hash table, errno = %d\n",
1973                                 rte_errno);
1974
1975         printf("lcore/cryptodev/qp mappings:\n");
1976
1977         idx = 0;
1978         for (cdev_id = 0; cdev_id < rte_cryptodev_count(); cdev_id++) {
1979                 struct rte_cryptodev_info cdev_info;
1980
1981                 if (check_cryptodev_mask((uint8_t)cdev_id))
1982                         continue;
1983
1984                 rte_cryptodev_info_get(cdev_id, &cdev_info);
1985
1986                 if ((mseg_flag & cdev_info.feature_flags) != mseg_flag)
1987                         rte_exit(EXIT_FAILURE,
1988                                 "Device %hd does not support \'%s\' feature\n",
1989                                 cdev_id,
1990                                 rte_cryptodev_get_feature_name(mseg_flag));
1991
1992                 if (nb_lcore_params > cdev_info.max_nb_queue_pairs)
1993                         max_nb_qps = cdev_info.max_nb_queue_pairs;
1994                 else
1995                         max_nb_qps = nb_lcore_params;
1996
1997                 qp = 0;
1998                 i = 0;
1999                 while (qp < max_nb_qps && i < nb_lcore_params) {
2000                         if (add_cdev_mapping(&cdev_info, cdev_id, qp,
2001                                                 &lcore_params[idx]))
2002                                 qp++;
2003                         idx++;
2004                         idx = idx % nb_lcore_params;
2005                         i++;
2006                 }
2007
2008                 if (qp == 0)
2009                         continue;
2010
2011                 dev_conf.socket_id = rte_cryptodev_socket_id(cdev_id);
2012                 dev_conf.nb_queue_pairs = qp;
2013                 dev_conf.ff_disable = RTE_CRYPTODEV_FF_ASYMMETRIC_CRYPTO;
2014
2015                 uint32_t dev_max_sess = cdev_info.sym.max_nb_sessions;
2016                 if (dev_max_sess != 0 && dev_max_sess < CDEV_MP_NB_OBJS)
2017                         rte_exit(EXIT_FAILURE,
2018                                 "Device does not support at least %u "
2019                                 "sessions", CDEV_MP_NB_OBJS);
2020
2021                 if (rte_cryptodev_configure(cdev_id, &dev_conf))
2022                         rte_panic("Failed to initialize cryptodev %u\n",
2023                                         cdev_id);
2024
2025                 qp_conf.nb_descriptors = CDEV_QUEUE_DESC;
2026                 qp_conf.mp_session =
2027                         socket_ctx[dev_conf.socket_id].session_pool;
2028                 qp_conf.mp_session_private =
2029                         socket_ctx[dev_conf.socket_id].session_priv_pool;
2030                 for (qp = 0; qp < dev_conf.nb_queue_pairs; qp++)
2031                         if (rte_cryptodev_queue_pair_setup(cdev_id, qp,
2032                                         &qp_conf, dev_conf.socket_id))
2033                                 rte_panic("Failed to setup queue %u for "
2034                                                 "cdev_id %u\n", 0, cdev_id);
2035
2036                 if (rte_cryptodev_start(cdev_id))
2037                         rte_panic("Failed to start cryptodev %u\n",
2038                                         cdev_id);
2039         }
2040
2041         printf("\n");
2042
2043         return 0;
2044 }
2045
2046 static void
2047 port_init(uint16_t portid, uint64_t req_rx_offloads, uint64_t req_tx_offloads)
2048 {
2049         uint32_t frame_size;
2050         struct rte_eth_dev_info dev_info;
2051         struct rte_eth_txconf *txconf;
2052         uint16_t nb_tx_queue, nb_rx_queue;
2053         uint16_t tx_queueid, rx_queueid, queue, lcore_id;
2054         int32_t ret, socket_id;
2055         struct lcore_conf *qconf;
2056         struct rte_ether_addr ethaddr;
2057         struct rte_eth_conf local_port_conf = port_conf;
2058
2059         ret = rte_eth_dev_info_get(portid, &dev_info);
2060         if (ret != 0)
2061                 rte_exit(EXIT_FAILURE,
2062                         "Error during getting device (port %u) info: %s\n",
2063                         portid, strerror(-ret));
2064
2065         /* limit allowed HW offloafs, as user requested */
2066         dev_info.rx_offload_capa &= dev_rx_offload;
2067         dev_info.tx_offload_capa &= dev_tx_offload;
2068
2069         printf("Configuring device port %u:\n", portid);
2070
2071         ret = rte_eth_macaddr_get(portid, &ethaddr);
2072         if (ret != 0)
2073                 rte_exit(EXIT_FAILURE,
2074                         "Error getting MAC address (port %u): %s\n",
2075                         portid, rte_strerror(-ret));
2076
2077         ethaddr_tbl[portid].src = ETHADDR_TO_UINT64(&ethaddr);
2078         print_ethaddr("Address: ", &ethaddr);
2079         printf("\n");
2080
2081         nb_rx_queue = get_port_nb_rx_queues(portid);
2082         nb_tx_queue = nb_lcores;
2083
2084         if (nb_rx_queue > dev_info.max_rx_queues)
2085                 rte_exit(EXIT_FAILURE, "Error: queue %u not available "
2086                                 "(max rx queue is %u)\n",
2087                                 nb_rx_queue, dev_info.max_rx_queues);
2088
2089         if (nb_tx_queue > dev_info.max_tx_queues)
2090                 rte_exit(EXIT_FAILURE, "Error: queue %u not available "
2091                                 "(max tx queue is %u)\n",
2092                                 nb_tx_queue, dev_info.max_tx_queues);
2093
2094         printf("Creating queues: nb_rx_queue=%d nb_tx_queue=%u...\n",
2095                         nb_rx_queue, nb_tx_queue);
2096
2097         frame_size = MTU_TO_FRAMELEN(mtu_size);
2098         if (frame_size > local_port_conf.rxmode.max_rx_pkt_len)
2099                 local_port_conf.rxmode.offloads |= DEV_RX_OFFLOAD_JUMBO_FRAME;
2100         local_port_conf.rxmode.max_rx_pkt_len = frame_size;
2101
2102         if (multi_seg_required()) {
2103                 local_port_conf.rxmode.offloads |= DEV_RX_OFFLOAD_SCATTER;
2104                 local_port_conf.txmode.offloads |= DEV_TX_OFFLOAD_MULTI_SEGS;
2105         }
2106
2107         local_port_conf.rxmode.offloads |= req_rx_offloads;
2108         local_port_conf.txmode.offloads |= req_tx_offloads;
2109
2110         /* Check that all required capabilities are supported */
2111         if ((local_port_conf.rxmode.offloads & dev_info.rx_offload_capa) !=
2112                         local_port_conf.rxmode.offloads)
2113                 rte_exit(EXIT_FAILURE,
2114                         "Error: port %u required RX offloads: 0x%" PRIx64
2115                         ", avaialbe RX offloads: 0x%" PRIx64 "\n",
2116                         portid, local_port_conf.rxmode.offloads,
2117                         dev_info.rx_offload_capa);
2118
2119         if ((local_port_conf.txmode.offloads & dev_info.tx_offload_capa) !=
2120                         local_port_conf.txmode.offloads)
2121                 rte_exit(EXIT_FAILURE,
2122                         "Error: port %u required TX offloads: 0x%" PRIx64
2123                         ", avaialbe TX offloads: 0x%" PRIx64 "\n",
2124                         portid, local_port_conf.txmode.offloads,
2125                         dev_info.tx_offload_capa);
2126
2127         if (dev_info.tx_offload_capa & DEV_TX_OFFLOAD_MBUF_FAST_FREE)
2128                 local_port_conf.txmode.offloads |=
2129                         DEV_TX_OFFLOAD_MBUF_FAST_FREE;
2130
2131         if (dev_info.tx_offload_capa & DEV_TX_OFFLOAD_IPV4_CKSUM)
2132                 local_port_conf.txmode.offloads |= DEV_TX_OFFLOAD_IPV4_CKSUM;
2133
2134         printf("port %u configurng rx_offloads=0x%" PRIx64
2135                 ", tx_offloads=0x%" PRIx64 "\n",
2136                 portid, local_port_conf.rxmode.offloads,
2137                 local_port_conf.txmode.offloads);
2138
2139         local_port_conf.rx_adv_conf.rss_conf.rss_hf &=
2140                 dev_info.flow_type_rss_offloads;
2141         if (local_port_conf.rx_adv_conf.rss_conf.rss_hf !=
2142                         port_conf.rx_adv_conf.rss_conf.rss_hf) {
2143                 printf("Port %u modified RSS hash function based on hardware support,"
2144                         "requested:%#"PRIx64" configured:%#"PRIx64"\n",
2145                         portid,
2146                         port_conf.rx_adv_conf.rss_conf.rss_hf,
2147                         local_port_conf.rx_adv_conf.rss_conf.rss_hf);
2148         }
2149
2150         ret = rte_eth_dev_configure(portid, nb_rx_queue, nb_tx_queue,
2151                         &local_port_conf);
2152         if (ret < 0)
2153                 rte_exit(EXIT_FAILURE, "Cannot configure device: "
2154                                 "err=%d, port=%d\n", ret, portid);
2155
2156         ret = rte_eth_dev_adjust_nb_rx_tx_desc(portid, &nb_rxd, &nb_txd);
2157         if (ret < 0)
2158                 rte_exit(EXIT_FAILURE, "Cannot adjust number of descriptors: "
2159                                 "err=%d, port=%d\n", ret, portid);
2160
2161         /* init one TX queue per lcore */
2162         tx_queueid = 0;
2163         for (lcore_id = 0; lcore_id < RTE_MAX_LCORE; lcore_id++) {
2164                 if (rte_lcore_is_enabled(lcore_id) == 0)
2165                         continue;
2166
2167                 if (numa_on)
2168                         socket_id = (uint8_t)rte_lcore_to_socket_id(lcore_id);
2169                 else
2170                         socket_id = 0;
2171
2172                 /* init TX queue */
2173                 printf("Setup txq=%u,%d,%d\n", lcore_id, tx_queueid, socket_id);
2174
2175                 txconf = &dev_info.default_txconf;
2176                 txconf->offloads = local_port_conf.txmode.offloads;
2177
2178                 ret = rte_eth_tx_queue_setup(portid, tx_queueid, nb_txd,
2179                                 socket_id, txconf);
2180                 if (ret < 0)
2181                         rte_exit(EXIT_FAILURE, "rte_eth_tx_queue_setup: "
2182                                         "err=%d, port=%d\n", ret, portid);
2183
2184                 qconf = &lcore_conf[lcore_id];
2185                 qconf->tx_queue_id[portid] = tx_queueid;
2186
2187                 /* Pre-populate pkt offloads based on capabilities */
2188                 qconf->outbound.ipv4_offloads = PKT_TX_IPV4;
2189                 qconf->outbound.ipv6_offloads = PKT_TX_IPV6;
2190                 if (local_port_conf.txmode.offloads & DEV_TX_OFFLOAD_IPV4_CKSUM)
2191                         qconf->outbound.ipv4_offloads |= PKT_TX_IP_CKSUM;
2192
2193                 tx_queueid++;
2194
2195                 /* init RX queues */
2196                 for (queue = 0; queue < qconf->nb_rx_queue; ++queue) {
2197                         struct rte_eth_rxconf rxq_conf;
2198
2199                         if (portid != qconf->rx_queue_list[queue].port_id)
2200                                 continue;
2201
2202                         rx_queueid = qconf->rx_queue_list[queue].queue_id;
2203
2204                         printf("Setup rxq=%d,%d,%d\n", portid, rx_queueid,
2205                                         socket_id);
2206
2207                         rxq_conf = dev_info.default_rxconf;
2208                         rxq_conf.offloads = local_port_conf.rxmode.offloads;
2209                         ret = rte_eth_rx_queue_setup(portid, rx_queueid,
2210                                         nb_rxd, socket_id, &rxq_conf,
2211                                         socket_ctx[socket_id].mbuf_pool);
2212                         if (ret < 0)
2213                                 rte_exit(EXIT_FAILURE,
2214                                         "rte_eth_rx_queue_setup: err=%d, "
2215                                         "port=%d\n", ret, portid);
2216                 }
2217         }
2218         printf("\n");
2219 }
2220
2221 static size_t
2222 max_session_size(void)
2223 {
2224         size_t max_sz, sz;
2225         void *sec_ctx;
2226         int16_t cdev_id, port_id, n;
2227
2228         max_sz = 0;
2229         n =  rte_cryptodev_count();
2230         for (cdev_id = 0; cdev_id != n; cdev_id++) {
2231                 sz = rte_cryptodev_sym_get_private_session_size(cdev_id);
2232                 if (sz > max_sz)
2233                         max_sz = sz;
2234                 /*
2235                  * If crypto device is security capable, need to check the
2236                  * size of security session as well.
2237                  */
2238
2239                 /* Get security context of the crypto device */
2240                 sec_ctx = rte_cryptodev_get_sec_ctx(cdev_id);
2241                 if (sec_ctx == NULL)
2242                         continue;
2243
2244                 /* Get size of security session */
2245                 sz = rte_security_session_get_size(sec_ctx);
2246                 if (sz > max_sz)
2247                         max_sz = sz;
2248         }
2249
2250         RTE_ETH_FOREACH_DEV(port_id) {
2251                 if ((enabled_port_mask & (1 << port_id)) == 0)
2252                         continue;
2253
2254                 sec_ctx = rte_eth_dev_get_sec_ctx(port_id);
2255                 if (sec_ctx == NULL)
2256                         continue;
2257
2258                 sz = rte_security_session_get_size(sec_ctx);
2259                 if (sz > max_sz)
2260                         max_sz = sz;
2261         }
2262
2263         return max_sz;
2264 }
2265
2266 static void
2267 session_pool_init(struct socket_ctx *ctx, int32_t socket_id, size_t sess_sz)
2268 {
2269         char mp_name[RTE_MEMPOOL_NAMESIZE];
2270         struct rte_mempool *sess_mp;
2271
2272         snprintf(mp_name, RTE_MEMPOOL_NAMESIZE,
2273                         "sess_mp_%u", socket_id);
2274         sess_mp = rte_cryptodev_sym_session_pool_create(
2275                         mp_name, CDEV_MP_NB_OBJS,
2276                         sess_sz, CDEV_MP_CACHE_SZ, 0,
2277                         socket_id);
2278         ctx->session_pool = sess_mp;
2279
2280         if (ctx->session_pool == NULL)
2281                 rte_exit(EXIT_FAILURE,
2282                         "Cannot init session pool on socket %d\n", socket_id);
2283         else
2284                 printf("Allocated session pool on socket %d\n", socket_id);
2285 }
2286
2287 static void
2288 session_priv_pool_init(struct socket_ctx *ctx, int32_t socket_id,
2289         size_t sess_sz)
2290 {
2291         char mp_name[RTE_MEMPOOL_NAMESIZE];
2292         struct rte_mempool *sess_mp;
2293
2294         snprintf(mp_name, RTE_MEMPOOL_NAMESIZE,
2295                         "sess_mp_priv_%u", socket_id);
2296         sess_mp = rte_mempool_create(mp_name,
2297                         CDEV_MP_NB_OBJS,
2298                         sess_sz,
2299                         CDEV_MP_CACHE_SZ,
2300                         0, NULL, NULL, NULL,
2301                         NULL, socket_id,
2302                         0);
2303         ctx->session_priv_pool = sess_mp;
2304
2305         if (ctx->session_priv_pool == NULL)
2306                 rte_exit(EXIT_FAILURE,
2307                         "Cannot init session priv pool on socket %d\n",
2308                         socket_id);
2309         else
2310                 printf("Allocated session priv pool on socket %d\n",
2311                         socket_id);
2312 }
2313
2314 static void
2315 pool_init(struct socket_ctx *ctx, int32_t socket_id, uint32_t nb_mbuf)
2316 {
2317         char s[64];
2318         int32_t ms;
2319
2320         snprintf(s, sizeof(s), "mbuf_pool_%d", socket_id);
2321         ctx->mbuf_pool = rte_pktmbuf_pool_create(s, nb_mbuf,
2322                         MEMPOOL_CACHE_SIZE, ipsec_metadata_size(),
2323                         frame_buf_size, socket_id);
2324
2325         /*
2326          * if multi-segment support is enabled, then create a pool
2327          * for indirect mbufs.
2328          */
2329         ms = multi_seg_required();
2330         if (ms != 0) {
2331                 snprintf(s, sizeof(s), "mbuf_pool_indir_%d", socket_id);
2332                 ctx->mbuf_pool_indir = rte_pktmbuf_pool_create(s, nb_mbuf,
2333                         MEMPOOL_CACHE_SIZE, 0, 0, socket_id);
2334         }
2335
2336         if (ctx->mbuf_pool == NULL || (ms != 0 && ctx->mbuf_pool_indir == NULL))
2337                 rte_exit(EXIT_FAILURE, "Cannot init mbuf pool on socket %d\n",
2338                                 socket_id);
2339         else
2340                 printf("Allocated mbuf pool on socket %d\n", socket_id);
2341 }
2342
2343 static inline int
2344 inline_ipsec_event_esn_overflow(struct rte_security_ctx *ctx, uint64_t md)
2345 {
2346         struct ipsec_sa *sa;
2347
2348         /* For inline protocol processing, the metadata in the event will
2349          * uniquely identify the security session which raised the event.
2350          * Application would then need the userdata it had registered with the
2351          * security session to process the event.
2352          */
2353
2354         sa = (struct ipsec_sa *)rte_security_get_userdata(ctx, md);
2355
2356         if (sa == NULL) {
2357                 /* userdata could not be retrieved */
2358                 return -1;
2359         }
2360
2361         /* Sequence number over flow. SA need to be re-established */
2362         RTE_SET_USED(sa);
2363         return 0;
2364 }
2365
2366 static int
2367 inline_ipsec_event_callback(uint16_t port_id, enum rte_eth_event_type type,
2368                  void *param, void *ret_param)
2369 {
2370         uint64_t md;
2371         struct rte_eth_event_ipsec_desc *event_desc = NULL;
2372         struct rte_security_ctx *ctx = (struct rte_security_ctx *)
2373                                         rte_eth_dev_get_sec_ctx(port_id);
2374
2375         RTE_SET_USED(param);
2376
2377         if (type != RTE_ETH_EVENT_IPSEC)
2378                 return -1;
2379
2380         event_desc = ret_param;
2381         if (event_desc == NULL) {
2382                 printf("Event descriptor not set\n");
2383                 return -1;
2384         }
2385
2386         md = event_desc->metadata;
2387
2388         if (event_desc->subtype == RTE_ETH_EVENT_IPSEC_ESN_OVERFLOW)
2389                 return inline_ipsec_event_esn_overflow(ctx, md);
2390         else if (event_desc->subtype >= RTE_ETH_EVENT_IPSEC_MAX) {
2391                 printf("Invalid IPsec event reported\n");
2392                 return -1;
2393         }
2394
2395         return -1;
2396 }
2397
2398 static uint16_t
2399 rx_callback(__rte_unused uint16_t port, __rte_unused uint16_t queue,
2400         struct rte_mbuf *pkt[], uint16_t nb_pkts,
2401         __rte_unused uint16_t max_pkts, void *user_param)
2402 {
2403         uint64_t tm;
2404         uint32_t i, k;
2405         struct lcore_conf *lc;
2406         struct rte_mbuf *mb;
2407         struct rte_ether_hdr *eth;
2408
2409         lc = user_param;
2410         k = 0;
2411         tm = 0;
2412
2413         for (i = 0; i != nb_pkts; i++) {
2414
2415                 mb = pkt[i];
2416                 eth = rte_pktmbuf_mtod(mb, struct rte_ether_hdr *);
2417                 if (eth->ether_type == rte_cpu_to_be_16(RTE_ETHER_TYPE_IPV4)) {
2418
2419                         struct rte_ipv4_hdr *iph;
2420
2421                         iph = (struct rte_ipv4_hdr *)(eth + 1);
2422                         if (rte_ipv4_frag_pkt_is_fragmented(iph)) {
2423
2424                                 mb->l2_len = sizeof(*eth);
2425                                 mb->l3_len = sizeof(*iph);
2426                                 tm = (tm != 0) ? tm : rte_rdtsc();
2427                                 mb = rte_ipv4_frag_reassemble_packet(
2428                                         lc->frag.tbl, &lc->frag.dr,
2429                                         mb, tm, iph);
2430
2431                                 if (mb != NULL) {
2432                                         /* fix ip cksum after reassemble. */
2433                                         iph = rte_pktmbuf_mtod_offset(mb,
2434                                                 struct rte_ipv4_hdr *,
2435                                                 mb->l2_len);
2436                                         iph->hdr_checksum = 0;
2437                                         iph->hdr_checksum = rte_ipv4_cksum(iph);
2438                                 }
2439                         }
2440                 } else if (eth->ether_type ==
2441                                 rte_cpu_to_be_16(RTE_ETHER_TYPE_IPV6)) {
2442
2443                         struct rte_ipv6_hdr *iph;
2444                         struct ipv6_extension_fragment *fh;
2445
2446                         iph = (struct rte_ipv6_hdr *)(eth + 1);
2447                         fh = rte_ipv6_frag_get_ipv6_fragment_header(iph);
2448                         if (fh != NULL) {
2449                                 mb->l2_len = sizeof(*eth);
2450                                 mb->l3_len = (uintptr_t)fh - (uintptr_t)iph +
2451                                         sizeof(*fh);
2452                                 tm = (tm != 0) ? tm : rte_rdtsc();
2453                                 mb = rte_ipv6_frag_reassemble_packet(
2454                                         lc->frag.tbl, &lc->frag.dr,
2455                                         mb, tm, iph, fh);
2456                                 if (mb != NULL)
2457                                         /* fix l3_len after reassemble. */
2458                                         mb->l3_len = mb->l3_len - sizeof(*fh);
2459                         }
2460                 }
2461
2462                 pkt[k] = mb;
2463                 k += (mb != NULL);
2464         }
2465
2466         /* some fragments were encountered, drain death row */
2467         if (tm != 0)
2468                 rte_ip_frag_free_death_row(&lc->frag.dr, 0);
2469
2470         return k;
2471 }
2472
2473
2474 static int
2475 reassemble_lcore_init(struct lcore_conf *lc, uint32_t cid)
2476 {
2477         int32_t sid;
2478         uint32_t i;
2479         uint64_t frag_cycles;
2480         const struct lcore_rx_queue *rxq;
2481         const struct rte_eth_rxtx_callback *cb;
2482
2483         /* create fragment table */
2484         sid = rte_lcore_to_socket_id(cid);
2485         frag_cycles = (rte_get_tsc_hz() + NS_PER_S - 1) /
2486                 NS_PER_S * frag_ttl_ns;
2487
2488         lc->frag.tbl = rte_ip_frag_table_create(frag_tbl_sz,
2489                 FRAG_TBL_BUCKET_ENTRIES, frag_tbl_sz, frag_cycles, sid);
2490         if (lc->frag.tbl == NULL) {
2491                 printf("%s(%u): failed to create fragment table of size: %u, "
2492                         "error code: %d\n",
2493                         __func__, cid, frag_tbl_sz, rte_errno);
2494                 return -ENOMEM;
2495         }
2496
2497         /* setup reassemble RX callbacks for all queues */
2498         for (i = 0; i != lc->nb_rx_queue; i++) {
2499
2500                 rxq = lc->rx_queue_list + i;
2501                 cb = rte_eth_add_rx_callback(rxq->port_id, rxq->queue_id,
2502                         rx_callback, lc);
2503                 if (cb == NULL) {
2504                         printf("%s(%u): failed to install RX callback for "
2505                                 "portid=%u, queueid=%u, error code: %d\n",
2506                                 __func__, cid,
2507                                 rxq->port_id, rxq->queue_id, rte_errno);
2508                         return -ENOMEM;
2509                 }
2510         }
2511
2512         return 0;
2513 }
2514
2515 static int
2516 reassemble_init(void)
2517 {
2518         int32_t rc;
2519         uint32_t i, lc;
2520
2521         rc = 0;
2522         for (i = 0; i != nb_lcore_params; i++) {
2523                 lc = lcore_params[i].lcore_id;
2524                 rc = reassemble_lcore_init(lcore_conf + lc, lc);
2525                 if (rc != 0)
2526                         break;
2527         }
2528
2529         return rc;
2530 }
2531
2532 static void
2533 create_default_ipsec_flow(uint16_t port_id, uint64_t rx_offloads)
2534 {
2535         struct rte_flow_action action[2];
2536         struct rte_flow_item pattern[2];
2537         struct rte_flow_attr attr = {0};
2538         struct rte_flow_error err;
2539         struct rte_flow *flow;
2540         int ret;
2541
2542         if (!(rx_offloads & DEV_RX_OFFLOAD_SECURITY))
2543                 return;
2544
2545         /* Add the default rte_flow to enable SECURITY for all ESP packets */
2546
2547         pattern[0].type = RTE_FLOW_ITEM_TYPE_ESP;
2548         pattern[0].spec = NULL;
2549         pattern[0].mask = NULL;
2550         pattern[0].last = NULL;
2551         pattern[1].type = RTE_FLOW_ITEM_TYPE_END;
2552
2553         action[0].type = RTE_FLOW_ACTION_TYPE_SECURITY;
2554         action[0].conf = NULL;
2555         action[1].type = RTE_FLOW_ACTION_TYPE_END;
2556         action[1].conf = NULL;
2557
2558         attr.ingress = 1;
2559
2560         ret = rte_flow_validate(port_id, &attr, pattern, action, &err);
2561         if (ret)
2562                 return;
2563
2564         flow = rte_flow_create(port_id, &attr, pattern, action, &err);
2565         if (flow == NULL)
2566                 return;
2567
2568         flow_info_tbl[port_id].rx_def_flow = flow;
2569         RTE_LOG(INFO, IPSEC,
2570                 "Created default flow enabling SECURITY for all ESP traffic on port %d\n",
2571                 port_id);
2572 }
2573
2574 static void
2575 signal_handler(int signum)
2576 {
2577         if (signum == SIGINT || signum == SIGTERM) {
2578                 printf("\n\nSignal %d received, preparing to exit...\n",
2579                                 signum);
2580                 force_quit = true;
2581         }
2582 }
2583
2584 static void
2585 ev_mode_sess_verify(struct ipsec_sa *sa, int nb_sa)
2586 {
2587         struct rte_ipsec_session *ips;
2588         int32_t i;
2589
2590         if (!sa || !nb_sa)
2591                 return;
2592
2593         for (i = 0; i < nb_sa; i++) {
2594                 ips = ipsec_get_primary_session(&sa[i]);
2595                 if (ips->type != RTE_SECURITY_ACTION_TYPE_INLINE_PROTOCOL)
2596                         rte_exit(EXIT_FAILURE, "Event mode supports only "
2597                                  "inline protocol sessions\n");
2598         }
2599
2600 }
2601
2602 static int32_t
2603 check_event_mode_params(struct eh_conf *eh_conf)
2604 {
2605         struct eventmode_conf *em_conf = NULL;
2606         struct lcore_params *params;
2607         uint16_t portid;
2608
2609         if (!eh_conf || !eh_conf->mode_params)
2610                 return -EINVAL;
2611
2612         /* Get eventmode conf */
2613         em_conf = eh_conf->mode_params;
2614
2615         if (eh_conf->mode == EH_PKT_TRANSFER_MODE_POLL &&
2616             em_conf->ext_params.sched_type != SCHED_TYPE_NOT_SET) {
2617                 printf("error: option --event-schedule-type applies only to "
2618                        "event mode\n");
2619                 return -EINVAL;
2620         }
2621
2622         if (eh_conf->mode != EH_PKT_TRANSFER_MODE_EVENT)
2623                 return 0;
2624
2625         /* Set schedule type to ORDERED if it wasn't explicitly set by user */
2626         if (em_conf->ext_params.sched_type == SCHED_TYPE_NOT_SET)
2627                 em_conf->ext_params.sched_type = RTE_SCHED_TYPE_ORDERED;
2628
2629         /*
2630          * Event mode currently supports only inline protocol sessions.
2631          * If there are other types of sessions configured then exit with
2632          * error.
2633          */
2634         ev_mode_sess_verify(sa_in, nb_sa_in);
2635         ev_mode_sess_verify(sa_out, nb_sa_out);
2636
2637
2638         /* Option --config does not apply to event mode */
2639         if (nb_lcore_params > 0) {
2640                 printf("error: option --config applies only to poll mode\n");
2641                 return -EINVAL;
2642         }
2643
2644         /*
2645          * In order to use the same port_init routine for both poll and event
2646          * modes initialize lcore_params with one queue for each eth port
2647          */
2648         lcore_params = lcore_params_array;
2649         RTE_ETH_FOREACH_DEV(portid) {
2650                 if ((enabled_port_mask & (1 << portid)) == 0)
2651                         continue;
2652
2653                 params = &lcore_params[nb_lcore_params++];
2654                 params->port_id = portid;
2655                 params->queue_id = 0;
2656                 params->lcore_id = rte_get_next_lcore(0, 0, 1);
2657         }
2658
2659         return 0;
2660 }
2661
2662 static void
2663 inline_sessions_free(struct sa_ctx *sa_ctx)
2664 {
2665         struct rte_ipsec_session *ips;
2666         struct ipsec_sa *sa;
2667         int32_t ret;
2668         uint32_t i;
2669
2670         if (!sa_ctx)
2671                 return;
2672
2673         for (i = 0; i < sa_ctx->nb_sa; i++) {
2674
2675                 sa = &sa_ctx->sa[i];
2676                 if (!sa->spi)
2677                         continue;
2678
2679                 ips = ipsec_get_primary_session(sa);
2680                 if (ips->type != RTE_SECURITY_ACTION_TYPE_INLINE_PROTOCOL &&
2681                     ips->type != RTE_SECURITY_ACTION_TYPE_INLINE_CRYPTO)
2682                         continue;
2683
2684                 if (!rte_eth_dev_is_valid_port(sa->portid))
2685                         continue;
2686
2687                 ret = rte_security_session_destroy(
2688                                 rte_eth_dev_get_sec_ctx(sa->portid),
2689                                 ips->security.ses);
2690                 if (ret)
2691                         RTE_LOG(ERR, IPSEC, "Failed to destroy security "
2692                                             "session type %d, spi %d\n",
2693                                             ips->type, sa->spi);
2694         }
2695 }
2696
2697 int32_t
2698 main(int32_t argc, char **argv)
2699 {
2700         int32_t ret;
2701         uint32_t lcore_id;
2702         uint32_t cdev_id;
2703         uint32_t i;
2704         uint8_t socket_id;
2705         uint16_t portid;
2706         uint64_t req_rx_offloads[RTE_MAX_ETHPORTS];
2707         uint64_t req_tx_offloads[RTE_MAX_ETHPORTS];
2708         struct eh_conf *eh_conf = NULL;
2709         size_t sess_sz;
2710
2711         /* init EAL */
2712         ret = rte_eal_init(argc, argv);
2713         if (ret < 0)
2714                 rte_exit(EXIT_FAILURE, "Invalid EAL parameters\n");
2715         argc -= ret;
2716         argv += ret;
2717
2718         force_quit = false;
2719         signal(SIGINT, signal_handler);
2720         signal(SIGTERM, signal_handler);
2721
2722         /* initialize event helper configuration */
2723         eh_conf = eh_conf_init();
2724         if (eh_conf == NULL)
2725                 rte_exit(EXIT_FAILURE, "Failed to init event helper config");
2726
2727         /* parse application arguments (after the EAL ones) */
2728         ret = parse_args(argc, argv, eh_conf);
2729         if (ret < 0)
2730                 rte_exit(EXIT_FAILURE, "Invalid parameters\n");
2731
2732         /* parse configuration file */
2733         if (parse_cfg_file(cfgfile) < 0) {
2734                 printf("parsing file \"%s\" failed\n",
2735                         optarg);
2736                 print_usage(argv[0]);
2737                 return -1;
2738         }
2739
2740         if ((unprotected_port_mask & enabled_port_mask) !=
2741                         unprotected_port_mask)
2742                 rte_exit(EXIT_FAILURE, "Invalid unprotected portmask 0x%x\n",
2743                                 unprotected_port_mask);
2744
2745         if (check_poll_mode_params(eh_conf) < 0)
2746                 rte_exit(EXIT_FAILURE, "check_poll_mode_params failed\n");
2747
2748         if (check_event_mode_params(eh_conf) < 0)
2749                 rte_exit(EXIT_FAILURE, "check_event_mode_params failed\n");
2750
2751         ret = init_lcore_rx_queues();
2752         if (ret < 0)
2753                 rte_exit(EXIT_FAILURE, "init_lcore_rx_queues failed\n");
2754
2755         nb_lcores = rte_lcore_count();
2756
2757         sess_sz = max_session_size();
2758
2759         for (lcore_id = 0; lcore_id < RTE_MAX_LCORE; lcore_id++) {
2760                 if (rte_lcore_is_enabled(lcore_id) == 0)
2761                         continue;
2762
2763                 if (numa_on)
2764                         socket_id = (uint8_t)rte_lcore_to_socket_id(lcore_id);
2765                 else
2766                         socket_id = 0;
2767
2768                 /* mbuf_pool is initialised by the pool_init() function*/
2769                 if (socket_ctx[socket_id].mbuf_pool)
2770                         continue;
2771
2772                 pool_init(&socket_ctx[socket_id], socket_id, NB_MBUF);
2773                 session_pool_init(&socket_ctx[socket_id], socket_id, sess_sz);
2774                 session_priv_pool_init(&socket_ctx[socket_id], socket_id,
2775                         sess_sz);
2776         }
2777
2778         RTE_ETH_FOREACH_DEV(portid) {
2779                 if ((enabled_port_mask & (1 << portid)) == 0)
2780                         continue;
2781
2782                 sa_check_offloads(portid, &req_rx_offloads[portid],
2783                                 &req_tx_offloads[portid]);
2784                 port_init(portid, req_rx_offloads[portid],
2785                                 req_tx_offloads[portid]);
2786         }
2787
2788         cryptodevs_init();
2789
2790         /*
2791          * Set the enabled port mask in helper config for use by helper
2792          * sub-system. This will be used while initializing devices using
2793          * helper sub-system.
2794          */
2795         eh_conf->eth_portmask = enabled_port_mask;
2796
2797         /* Initialize eventmode components */
2798         ret = eh_devs_init(eh_conf);
2799         if (ret < 0)
2800                 rte_exit(EXIT_FAILURE, "eh_devs_init failed, err=%d\n", ret);
2801
2802         /* start ports */
2803         RTE_ETH_FOREACH_DEV(portid) {
2804                 if ((enabled_port_mask & (1 << portid)) == 0)
2805                         continue;
2806
2807                 /* Create flow before starting the device */
2808                 create_default_ipsec_flow(portid, req_rx_offloads[portid]);
2809
2810                 ret = rte_eth_dev_start(portid);
2811                 if (ret < 0)
2812                         rte_exit(EXIT_FAILURE, "rte_eth_dev_start: "
2813                                         "err=%d, port=%d\n", ret, portid);
2814                 /*
2815                  * If enabled, put device in promiscuous mode.
2816                  * This allows IO forwarding mode to forward packets
2817                  * to itself through 2 cross-connected  ports of the
2818                  * target machine.
2819                  */
2820                 if (promiscuous_on) {
2821                         ret = rte_eth_promiscuous_enable(portid);
2822                         if (ret != 0)
2823                                 rte_exit(EXIT_FAILURE,
2824                                         "rte_eth_promiscuous_enable: err=%s, port=%d\n",
2825                                         rte_strerror(-ret), portid);
2826                 }
2827
2828                 rte_eth_dev_callback_register(portid,
2829                         RTE_ETH_EVENT_IPSEC, inline_ipsec_event_callback, NULL);
2830         }
2831
2832         /* fragment reassemble is enabled */
2833         if (frag_tbl_sz != 0) {
2834                 ret = reassemble_init();
2835                 if (ret != 0)
2836                         rte_exit(EXIT_FAILURE, "failed at reassemble init");
2837         }
2838
2839         /* Replicate each context per socket */
2840         for (i = 0; i < NB_SOCKETS && i < rte_socket_count(); i++) {
2841                 socket_id = rte_socket_id_by_idx(i);
2842                 if ((socket_ctx[socket_id].mbuf_pool != NULL) &&
2843                         (socket_ctx[socket_id].sa_in == NULL) &&
2844                         (socket_ctx[socket_id].sa_out == NULL)) {
2845                         sa_init(&socket_ctx[socket_id], socket_id);
2846                         sp4_init(&socket_ctx[socket_id], socket_id);
2847                         sp6_init(&socket_ctx[socket_id], socket_id);
2848                         rt_init(&socket_ctx[socket_id], socket_id);
2849                 }
2850         }
2851
2852         check_all_ports_link_status(enabled_port_mask);
2853
2854         /* launch per-lcore init on every lcore */
2855         rte_eal_mp_remote_launch(ipsec_launch_one_lcore, eh_conf, CALL_MASTER);
2856         RTE_LCORE_FOREACH_SLAVE(lcore_id) {
2857                 if (rte_eal_wait_lcore(lcore_id) < 0)
2858                         return -1;
2859         }
2860
2861         /* Uninitialize eventmode components */
2862         ret = eh_devs_uninit(eh_conf);
2863         if (ret < 0)
2864                 rte_exit(EXIT_FAILURE, "eh_devs_uninit failed, err=%d\n", ret);
2865
2866         /* Free eventmode configuration memory */
2867         eh_conf_uninit(eh_conf);
2868
2869         /* Destroy inline inbound and outbound sessions */
2870         for (i = 0; i < NB_SOCKETS && i < rte_socket_count(); i++) {
2871                 socket_id = rte_socket_id_by_idx(i);
2872                 inline_sessions_free(socket_ctx[socket_id].sa_in);
2873                 inline_sessions_free(socket_ctx[socket_id].sa_out);
2874         }
2875
2876         for (cdev_id = 0; cdev_id < rte_cryptodev_count(); cdev_id++) {
2877                 printf("Closing cryptodev %d...", cdev_id);
2878                 rte_cryptodev_stop(cdev_id);
2879                 rte_cryptodev_close(cdev_id);
2880                 printf(" Done\n");
2881         }
2882
2883         RTE_ETH_FOREACH_DEV(portid) {
2884                 if ((enabled_port_mask & (1 << portid)) == 0)
2885                         continue;
2886
2887                 printf("Closing port %d...", portid);
2888                 if (flow_info_tbl[portid].rx_def_flow) {
2889                         struct rte_flow_error err;
2890
2891                         ret = rte_flow_destroy(portid,
2892                                 flow_info_tbl[portid].rx_def_flow, &err);
2893                         if (ret)
2894                                 RTE_LOG(ERR, IPSEC, "Failed to destroy flow "
2895                                         " for port %u, err msg: %s\n", portid,
2896                                         err.message);
2897                 }
2898                 rte_eth_dev_stop(portid);
2899                 rte_eth_dev_close(portid);
2900                 printf(" Done\n");
2901         }
2902         printf("Bye...\n");
2903
2904         return 0;
2905 }