examples/l2fwd-crypto/main.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(c) 2015-2016 Intel Corporation
   3  */
   4
   5 #include <time.h>
   6 #include <stdio.h>
   7 #include <stdlib.h>
   8 #include <string.h>
   9 #include <stdint.h>
  10 #include <inttypes.h>
  11 #include <sys/types.h>
  12 #include <sys/queue.h>
  13 #include <netinet/in.h>
  14 #include <setjmp.h>
  15 #include <stdarg.h>
  16 #include <ctype.h>
  17 #include <errno.h>
  18 #include <getopt.h>
  19 #include <fcntl.h>
  20 #include <unistd.h>
  21
  22 #include <rte_string_fns.h>
  23 #include <rte_branch_prediction.h>
  24 #include <rte_common.h>
  25 #include <rte_cryptodev.h>
  26 #include <rte_cycles.h>
  27 #include <rte_debug.h>
  28 #include <rte_eal.h>
  29 #include <rte_ether.h>
  30 #include <rte_ethdev.h>
  31 #include <rte_interrupts.h>
  32 #include <rte_ip.h>
  33 #include <rte_launch.h>
  34 #include <rte_lcore.h>
  35 #include <rte_log.h>
  36 #include <rte_malloc.h>
  37 #include <rte_mbuf.h>
  38 #include <rte_memcpy.h>
  39 #include <rte_memory.h>
  40 #include <rte_mempool.h>
  41 #include <rte_per_lcore.h>
  42 #include <rte_prefetch.h>
  43 #include <rte_random.h>
  44 #include <rte_hexdump.h>
  45 #ifdef RTE_CRYPTO_SCHEDULER
  46 #include <rte_cryptodev_scheduler.h>
  47 #endif
  48
  49 enum cdev_type {
  50         CDEV_TYPE_ANY,
  51         CDEV_TYPE_HW,
  52         CDEV_TYPE_SW
  53 };
  54
  55 #define RTE_LOGTYPE_L2FWD RTE_LOGTYPE_USER1
  56
  57 #define NB_MBUF   8192
  58
  59 #define MAX_STR_LEN 32
  60 #define MAX_KEY_SIZE 128
  61 #define MAX_IV_SIZE 16
  62 #define MAX_AAD_SIZE 65535
  63 #define MAX_PKT_BURST 32
  64 #define BURST_TX_DRAIN_US 100 /* TX drain every ~100us */
  65 #define SESSION_POOL_CACHE_SIZE 0
  66
  67 #define MAXIMUM_IV_LENGTH       16
  68 #define IV_OFFSET               (sizeof(struct rte_crypto_op) + \
  69                                 sizeof(struct rte_crypto_sym_op))
  70
  71 /*
  72  * Configurable number of RX/TX ring descriptors
  73  */
  74 #define RTE_TEST_RX_DESC_DEFAULT 1024
  75 #define RTE_TEST_TX_DESC_DEFAULT 1024
  76
  77 static uint16_t nb_rxd = RTE_TEST_RX_DESC_DEFAULT;
  78 static uint16_t nb_txd = RTE_TEST_TX_DESC_DEFAULT;
  79
  80 /* ethernet addresses of ports */
  81 static struct rte_ether_addr l2fwd_ports_eth_addr[RTE_MAX_ETHPORTS];
  82
  83 /* mask of enabled ports */
  84 static uint64_t l2fwd_enabled_port_mask;
  85 static uint64_t l2fwd_enabled_crypto_mask;
  86
  87 /* list of enabled ports */
  88 static uint16_t l2fwd_dst_ports[RTE_MAX_ETHPORTS];
  89
  90
  91 struct pkt_buffer {
  92         unsigned len;
  93         struct rte_mbuf *buffer[MAX_PKT_BURST];
  94 };
  95
  96 struct op_buffer {
  97         unsigned len;
  98         struct rte_crypto_op *buffer[MAX_PKT_BURST];
  99 };
 100
 101 #define MAX_RX_QUEUE_PER_LCORE 16
 102 #define MAX_TX_QUEUE_PER_PORT 16
 103
 104 enum l2fwd_crypto_xform_chain {
 105         L2FWD_CRYPTO_CIPHER_HASH,
 106         L2FWD_CRYPTO_HASH_CIPHER,
 107         L2FWD_CRYPTO_CIPHER_ONLY,
 108         L2FWD_CRYPTO_HASH_ONLY,
 109         L2FWD_CRYPTO_AEAD
 110 };
 111
 112 struct l2fwd_key {
 113         uint8_t *data;
 114         uint32_t length;
 115         rte_iova_t phys_addr;
 116 };
 117
 118 struct l2fwd_iv {
 119         uint8_t *data;
 120         uint16_t length;
 121 };
 122
 123 /** l2fwd crypto application command line options */
 124 struct l2fwd_crypto_options {
 125         unsigned portmask;
 126         unsigned nb_ports_per_lcore;
 127         unsigned refresh_period;
 128         unsigned single_lcore:1;
 129
 130         enum cdev_type type;
 131         unsigned sessionless:1;
 132
 133         enum l2fwd_crypto_xform_chain xform_chain;
 134
 135         struct rte_crypto_sym_xform cipher_xform;
 136         unsigned ckey_param;
 137         int ckey_random_size;
 138         uint8_t cipher_key[MAX_KEY_SIZE];
 139
 140         struct l2fwd_iv cipher_iv;
 141         unsigned int cipher_iv_param;
 142         int cipher_iv_random_size;
 143
 144         struct rte_crypto_sym_xform auth_xform;
 145         uint8_t akey_param;
 146         int akey_random_size;
 147         uint8_t auth_key[MAX_KEY_SIZE];
 148
 149         struct l2fwd_iv auth_iv;
 150         unsigned int auth_iv_param;
 151         int auth_iv_random_size;
 152
 153         struct rte_crypto_sym_xform aead_xform;
 154         unsigned int aead_key_param;
 155         int aead_key_random_size;
 156         uint8_t aead_key[MAX_KEY_SIZE];
 157
 158         struct l2fwd_iv aead_iv;
 159         unsigned int aead_iv_param;
 160         int aead_iv_random_size;
 161
 162         struct l2fwd_key aad;
 163         unsigned aad_param;
 164         int aad_random_size;
 165
 166         int digest_size;
 167
 168         uint16_t block_size;
 169         char string_type[MAX_STR_LEN];
 170
 171         uint64_t cryptodev_mask;
 172
 173         unsigned int mac_updating;
 174 };
 175
 176 /** l2fwd crypto lcore params */
 177 struct l2fwd_crypto_params {
 178         uint8_t dev_id;
 179         uint8_t qp_id;
 180
 181         unsigned digest_length;
 182         unsigned block_size;
 183
 184         uint32_t cipher_dataunit_len;
 185
 186         struct l2fwd_iv cipher_iv;
 187         struct l2fwd_iv auth_iv;
 188         struct l2fwd_iv aead_iv;
 189         struct l2fwd_key aad;
 190         struct rte_cryptodev_sym_session *session;
 191
 192         uint8_t do_cipher;
 193         uint8_t do_hash;
 194         uint8_t do_aead;
 195         uint8_t hash_verify;
 196
 197         enum rte_crypto_cipher_algorithm cipher_algo;
 198         enum rte_crypto_auth_algorithm auth_algo;
 199         enum rte_crypto_aead_algorithm aead_algo;
 200 };
 201
 202 /** lcore configuration */
 203 struct lcore_queue_conf {
 204         unsigned nb_rx_ports;
 205         uint16_t rx_port_list[MAX_RX_QUEUE_PER_LCORE];
 206
 207         unsigned nb_crypto_devs;
 208         unsigned cryptodev_list[MAX_RX_QUEUE_PER_LCORE];
 209
 210         struct op_buffer op_buf[RTE_CRYPTO_MAX_DEVS];
 211         struct pkt_buffer pkt_buf[RTE_MAX_ETHPORTS];
 212 } __rte_cache_aligned;
 213
 214 struct lcore_queue_conf lcore_queue_conf[RTE_MAX_LCORE];
 215
 216 static struct rte_eth_conf port_conf = {
 217         .rxmode = {
 218                 .mq_mode = ETH_MQ_RX_NONE,
 219                 .max_rx_pkt_len = RTE_ETHER_MAX_LEN,
 220                 .split_hdr_size = 0,
 221         },
 222         .txmode = {
 223                 .mq_mode = ETH_MQ_TX_NONE,
 224         },
 225 };
 226
 227 struct rte_mempool *l2fwd_pktmbuf_pool;
 228 struct rte_mempool *l2fwd_crypto_op_pool;
 229 static struct {
 230         struct rte_mempool *sess_mp;
 231         struct rte_mempool *priv_mp;
 232 } session_pool_socket[RTE_MAX_NUMA_NODES];
 233
 234 /* Per-port statistics struct */
 235 struct l2fwd_port_statistics {
 236         uint64_t tx;
 237         uint64_t rx;
 238
 239         uint64_t crypto_enqueued;
 240         uint64_t crypto_dequeued;
 241
 242         uint64_t dropped;
 243 } __rte_cache_aligned;
 244
 245 struct l2fwd_crypto_statistics {
 246         uint64_t enqueued;
 247         uint64_t dequeued;
 248
 249         uint64_t errors;
 250 } __rte_cache_aligned;
 251
 252 struct l2fwd_port_statistics port_statistics[RTE_MAX_ETHPORTS];
 253 struct l2fwd_crypto_statistics crypto_statistics[RTE_CRYPTO_MAX_DEVS];
 254
 255 /* A tsc-based timer responsible for triggering statistics printout */
 256 #define TIMER_MILLISECOND 2000000ULL /* around 1ms at 2 Ghz */
 257 #define MAX_TIMER_PERIOD 86400UL /* 1 day max */
 258
 259 /* default period is 10 seconds */
 260 static int64_t timer_period = 10 * TIMER_MILLISECOND * 1000;
 261
 262 /* Print out statistics on packets dropped */
 263 static void
 264 print_stats(void)
 265 {
 266         uint64_t total_packets_dropped, total_packets_tx, total_packets_rx;
 267         uint64_t total_packets_enqueued, total_packets_dequeued,
 268                 total_packets_errors;
 269         uint16_t portid;
 270         uint64_t cdevid;
 271
 272         total_packets_dropped = 0;
 273         total_packets_tx = 0;
 274         total_packets_rx = 0;
 275         total_packets_enqueued = 0;
 276         total_packets_dequeued = 0;
 277         total_packets_errors = 0;
 278
 279         const char clr[] = { 27, '[', '2', 'J', '\0' };
 280         const char topLeft[] = { 27, '[', '1', ';', '1', 'H', '\0' };
 281
 282                 /* Clear screen and move to top left */
 283         printf("%s%s", clr, topLeft);
 284
 285         printf("\nPort statistics ====================================");
 286
 287         for (portid = 0; portid < RTE_MAX_ETHPORTS; portid++) {
 288                 /* skip disabled ports */
 289                 if ((l2fwd_enabled_port_mask & (1 << portid)) == 0)
 290                         continue;
 291                 printf("\nStatistics for port %u ------------------------------"
 292                            "\nPackets sent: %32"PRIu64
 293                            "\nPackets received: %28"PRIu64
 294                            "\nPackets dropped: %29"PRIu64,
 295                            portid,
 296                            port_statistics[portid].tx,
 297                            port_statistics[portid].rx,
 298                            port_statistics[portid].dropped);
 299
 300                 total_packets_dropped += port_statistics[portid].dropped;
 301                 total_packets_tx += port_statistics[portid].tx;
 302                 total_packets_rx += port_statistics[portid].rx;
 303         }
 304         printf("\nCrypto statistics ==================================");
 305
 306         for (cdevid = 0; cdevid < RTE_CRYPTO_MAX_DEVS; cdevid++) {
 307                 /* skip disabled ports */
 308                 if ((l2fwd_enabled_crypto_mask & (((uint64_t)1) << cdevid)) == 0)
 309                         continue;
 310                 printf("\nStatistics for cryptodev %"PRIu64
 311                                 " -------------------------"
 312                            "\nPackets enqueued: %28"PRIu64
 313                            "\nPackets dequeued: %28"PRIu64
 314                            "\nPackets errors: %30"PRIu64,
 315                            cdevid,
 316                            crypto_statistics[cdevid].enqueued,
 317                            crypto_statistics[cdevid].dequeued,
 318                            crypto_statistics[cdevid].errors);
 319
 320                 total_packets_enqueued += crypto_statistics[cdevid].enqueued;
 321                 total_packets_dequeued += crypto_statistics[cdevid].dequeued;
 322                 total_packets_errors += crypto_statistics[cdevid].errors;
 323         }
 324         printf("\nAggregate statistics ==============================="
 325                    "\nTotal packets received: %22"PRIu64
 326                    "\nTotal packets enqueued: %22"PRIu64
 327                    "\nTotal packets dequeued: %22"PRIu64
 328                    "\nTotal packets sent: %26"PRIu64
 329                    "\nTotal packets dropped: %23"PRIu64
 330                    "\nTotal packets crypto errors: %17"PRIu64,
 331                    total_packets_rx,
 332                    total_packets_enqueued,
 333                    total_packets_dequeued,
 334                    total_packets_tx,
 335                    total_packets_dropped,
 336                    total_packets_errors);
 337         printf("\n====================================================\n");
 338
 339         fflush(stdout);
 340 }
 341
 342 /* l2fwd_crypto_send_burst 8< */
 343 static int
 344 l2fwd_crypto_send_burst(struct lcore_queue_conf *qconf, unsigned n,
 345                 struct l2fwd_crypto_params *cparams)
 346 {
 347         struct rte_crypto_op **op_buffer;
 348         unsigned ret;
 349
 350         op_buffer = (struct rte_crypto_op **)
 351                         qconf->op_buf[cparams->dev_id].buffer;
 352
 353         ret = rte_cryptodev_enqueue_burst(cparams->dev_id,
 354                         cparams->qp_id, op_buffer, (uint16_t) n);
 355
 356         crypto_statistics[cparams->dev_id].enqueued += ret;
 357         if (unlikely(ret < n)) {
 358                 crypto_statistics[cparams->dev_id].errors += (n - ret);
 359                 do {
 360                         rte_pktmbuf_free(op_buffer[ret]->sym->m_src);
 361                         rte_crypto_op_free(op_buffer[ret]);
 362                 } while (++ret < n);
 363         }
 364
 365         return 0;
 366 }
 367 /* >8 End of l2fwd_crypto_send_burst. */
 368
 369 /* Crypto enqueue. 8< */
 370 static int
 371 l2fwd_crypto_enqueue(struct rte_crypto_op *op,
 372                 struct l2fwd_crypto_params *cparams)
 373 {
 374         unsigned lcore_id, len;
 375         struct lcore_queue_conf *qconf;
 376
 377         lcore_id = rte_lcore_id();
 378
 379         qconf = &lcore_queue_conf[lcore_id];
 380         len = qconf->op_buf[cparams->dev_id].len;
 381         qconf->op_buf[cparams->dev_id].buffer[len] = op;
 382         len++;
 383
 384         /* enough ops to be sent */
 385         if (len == MAX_PKT_BURST) {
 386                 l2fwd_crypto_send_burst(qconf, MAX_PKT_BURST, cparams);
 387                 len = 0;
 388         }
 389
 390         qconf->op_buf[cparams->dev_id].len = len;
 391         return 0;
 392 }
 393 /* >8 End of crypto enqueue. */
 394
 395 static int
 396 l2fwd_simple_crypto_enqueue(struct rte_mbuf *m,
 397                 struct rte_crypto_op *op,
 398                 struct l2fwd_crypto_params *cparams)
 399 {
 400         struct rte_ether_hdr *eth_hdr;
 401         struct rte_ipv4_hdr *ip_hdr;
 402
 403         uint32_t ipdata_offset, data_len;
 404         uint32_t pad_len = 0;
 405         char *padding;
 406
 407         eth_hdr = rte_pktmbuf_mtod(m, struct rte_ether_hdr *);
 408
 409         if (eth_hdr->ether_type != rte_cpu_to_be_16(RTE_ETHER_TYPE_IPV4))
 410                 return -1;
 411
 412         ipdata_offset = sizeof(struct rte_ether_hdr);
 413
 414         ip_hdr = (struct rte_ipv4_hdr *)(rte_pktmbuf_mtod(m, char *) +
 415                         ipdata_offset);
 416
 417         ipdata_offset += (ip_hdr->version_ihl & RTE_IPV4_HDR_IHL_MASK)
 418                         * RTE_IPV4_IHL_MULTIPLIER;
 419
 420
 421         /* Zero pad data to be crypto'd so it is block aligned */
 422         data_len  = rte_pktmbuf_data_len(m) - ipdata_offset;
 423
 424         if ((cparams->do_hash || cparams->do_aead) && cparams->hash_verify)
 425                 data_len -= cparams->digest_length;
 426
 427         if (cparams->do_cipher) {
 428                 /*
 429                  * Following algorithms are block cipher algorithms,
 430                  * and might need padding
 431                  */
 432                 switch (cparams->cipher_algo) {
 433                 case RTE_CRYPTO_CIPHER_AES_CBC:
 434                 case RTE_CRYPTO_CIPHER_AES_ECB:
 435                 case RTE_CRYPTO_CIPHER_DES_CBC:
 436                 case RTE_CRYPTO_CIPHER_3DES_CBC:
 437                 case RTE_CRYPTO_CIPHER_3DES_ECB:
 438                         if (data_len % cparams->block_size)
 439                                 pad_len = cparams->block_size -
 440                                         (data_len % cparams->block_size);
 441                         break;
 442                 case RTE_CRYPTO_CIPHER_AES_XTS:
 443                         if (cparams->cipher_dataunit_len != 0 &&
 444                             (data_len % cparams->cipher_dataunit_len))
 445                                 pad_len = cparams->cipher_dataunit_len -
 446                                         (data_len % cparams->cipher_dataunit_len);
 447                         break;
 448                 default:
 449                         pad_len = 0;
 450                 }
 451
 452                 if (pad_len) {
 453                         padding = rte_pktmbuf_append(m, pad_len);
 454                         if (unlikely(!padding))
 455                                 return -1;
 456
 457                         data_len += pad_len;
 458                         memset(padding, 0, pad_len);
 459                 }
 460         }
 461
 462         /* Set crypto operation data parameters */
 463         rte_crypto_op_attach_sym_session(op, cparams->session);
 464
 465         if (cparams->do_hash) {
 466                 if (cparams->auth_iv.length) {
 467                         uint8_t *iv_ptr = rte_crypto_op_ctod_offset(op,
 468                                                 uint8_t *,
 469                                                 IV_OFFSET +
 470                                                 cparams->cipher_iv.length);
 471                         /*
 472                          * Copy IV at the end of the crypto operation,
 473                          * after the cipher IV, if added
 474                          */
 475                         rte_memcpy(iv_ptr, cparams->auth_iv.data,
 476                                         cparams->auth_iv.length);
 477                 }
 478                 if (!cparams->hash_verify) {
 479                         /* Append space for digest to end of packet */
 480                         op->sym->auth.digest.data = (uint8_t *)rte_pktmbuf_append(m,
 481                                 cparams->digest_length);
 482                 } else {
 483                         op->sym->auth.digest.data = rte_pktmbuf_mtod(m,
 484                                 uint8_t *) + ipdata_offset + data_len;
 485                 }
 486
 487                 op->sym->auth.digest.phys_addr = rte_pktmbuf_iova_offset(m,
 488                                 rte_pktmbuf_pkt_len(m) - cparams->digest_length);
 489
 490                 /* For wireless algorithms, offset/length must be in bits */
 491                 if (cparams->auth_algo == RTE_CRYPTO_AUTH_SNOW3G_UIA2 ||
 492                                 cparams->auth_algo == RTE_CRYPTO_AUTH_KASUMI_F9 ||
 493                                 cparams->auth_algo == RTE_CRYPTO_AUTH_ZUC_EIA3) {
 494                         op->sym->auth.data.offset = ipdata_offset << 3;
 495                         op->sym->auth.data.length = data_len << 3;
 496                 } else {
 497                         op->sym->auth.data.offset = ipdata_offset;
 498                         op->sym->auth.data.length = data_len;
 499                 }
 500         }
 501
 502         if (cparams->do_cipher) {
 503                 uint8_t *iv_ptr = rte_crypto_op_ctod_offset(op, uint8_t *,
 504                                                         IV_OFFSET);
 505                 /* Copy IV at the end of the crypto operation */
 506                 rte_memcpy(iv_ptr, cparams->cipher_iv.data,
 507                                 cparams->cipher_iv.length);
 508
 509                 /* For wireless algorithms, offset/length must be in bits */
 510                 if (cparams->cipher_algo == RTE_CRYPTO_CIPHER_SNOW3G_UEA2 ||
 511                                 cparams->cipher_algo == RTE_CRYPTO_CIPHER_KASUMI_F8 ||
 512                                 cparams->cipher_algo == RTE_CRYPTO_CIPHER_ZUC_EEA3) {
 513                         op->sym->cipher.data.offset = ipdata_offset << 3;
 514                         op->sym->cipher.data.length = data_len << 3;
 515                 } else {
 516                         op->sym->cipher.data.offset = ipdata_offset;
 517                         op->sym->cipher.data.length = data_len;
 518                 }
 519         }
 520
 521         if (cparams->do_aead) {
 522                 uint8_t *iv_ptr = rte_crypto_op_ctod_offset(op, uint8_t *,
 523                                                         IV_OFFSET);
 524                 /* Copy IV at the end of the crypto operation */
 525                 /*
 526                  * If doing AES-CCM, nonce is copied one byte
 527                  * after the start of IV field
 528                  */
 529                 if (cparams->aead_algo == RTE_CRYPTO_AEAD_AES_CCM)
 530                         rte_memcpy(iv_ptr + 1, cparams->aead_iv.data,
 531                                         cparams->aead_iv.length);
 532                 else
 533                         rte_memcpy(iv_ptr, cparams->aead_iv.data,
 534                                         cparams->aead_iv.length);
 535
 536                 op->sym->aead.data.offset = ipdata_offset;
 537                 op->sym->aead.data.length = data_len;
 538
 539                 if (!cparams->hash_verify) {
 540                         /* Append space for digest to end of packet */
 541                         op->sym->aead.digest.data = (uint8_t *)rte_pktmbuf_append(m,
 542                                 cparams->digest_length);
 543                 } else {
 544                         op->sym->aead.digest.data = rte_pktmbuf_mtod(m,
 545                                 uint8_t *) + ipdata_offset + data_len;
 546                 }
 547
 548                 op->sym->aead.digest.phys_addr = rte_pktmbuf_iova_offset(m,
 549                                 rte_pktmbuf_pkt_len(m) - cparams->digest_length);
 550
 551                 if (cparams->aad.length) {
 552                         op->sym->aead.aad.data = cparams->aad.data;
 553                         op->sym->aead.aad.phys_addr = cparams->aad.phys_addr;
 554                 }
 555         }
 556
 557         op->sym->m_src = m;
 558
 559         return l2fwd_crypto_enqueue(op, cparams);
 560 }
 561
 562
 563 /* Send the burst of packets on an output interface */
 564 static int
 565 l2fwd_send_burst(struct lcore_queue_conf *qconf, unsigned n,
 566                 uint16_t port)
 567 {
 568         struct rte_mbuf **pkt_buffer;
 569         unsigned ret;
 570
 571         pkt_buffer = (struct rte_mbuf **)qconf->pkt_buf[port].buffer;
 572
 573         ret = rte_eth_tx_burst(port, 0, pkt_buffer, (uint16_t)n);
 574         port_statistics[port].tx += ret;
 575         if (unlikely(ret < n)) {
 576                 port_statistics[port].dropped += (n - ret);
 577                 do {
 578                         rte_pktmbuf_free(pkt_buffer[ret]);
 579                 } while (++ret < n);
 580         }
 581
 582         return 0;
 583 }
 584
 585 /* Enqueue packets for TX and prepare them to be sent. 8< */
 586 static int
 587 l2fwd_send_packet(struct rte_mbuf *m, uint16_t port)
 588 {
 589         unsigned lcore_id, len;
 590         struct lcore_queue_conf *qconf;
 591
 592         lcore_id = rte_lcore_id();
 593
 594         qconf = &lcore_queue_conf[lcore_id];
 595         len = qconf->pkt_buf[port].len;
 596         qconf->pkt_buf[port].buffer[len] = m;
 597         len++;
 598
 599         /* enough pkts to be sent */
 600         if (unlikely(len == MAX_PKT_BURST)) {
 601                 l2fwd_send_burst(qconf, MAX_PKT_BURST, port);
 602                 len = 0;
 603         }
 604
 605         qconf->pkt_buf[port].len = len;
 606         return 0;
 607 }
 608 /* >8 End of Enqueuing packets for TX. */
 609
 610 static void
 611 l2fwd_mac_updating(struct rte_mbuf *m, uint16_t dest_portid)
 612 {
 613         struct rte_ether_hdr *eth;
 614         void *tmp;
 615
 616         eth = rte_pktmbuf_mtod(m, struct rte_ether_hdr *);
 617
 618         /* 02:00:00:00:00:xx */
 619         tmp = &eth->dst_addr.addr_bytes[0];
 620         *((uint64_t *)tmp) = 0x000000000002 + ((uint64_t)dest_portid << 40);
 621
 622         /* src addr */
 623         rte_ether_addr_copy(&l2fwd_ports_eth_addr[dest_portid], &eth->src_addr);
 624 }
 625
 626 static void
 627 l2fwd_simple_forward(struct rte_mbuf *m, uint16_t portid,
 628                 struct l2fwd_crypto_options *options)
 629 {
 630         uint16_t dst_port;
 631         uint32_t pad_len;
 632         struct rte_ipv4_hdr *ip_hdr;
 633         uint32_t ipdata_offset = sizeof(struct rte_ether_hdr);
 634
 635         ip_hdr = (struct rte_ipv4_hdr *)(rte_pktmbuf_mtod(m, char *) +
 636                                          ipdata_offset);
 637         dst_port = l2fwd_dst_ports[portid];
 638
 639         if (options->mac_updating)
 640                 l2fwd_mac_updating(m, dst_port);
 641
 642         if (options->auth_xform.auth.op == RTE_CRYPTO_AUTH_OP_VERIFY)
 643                 rte_pktmbuf_trim(m, options->auth_xform.auth.digest_length);
 644
 645         if (options->cipher_xform.cipher.op == RTE_CRYPTO_CIPHER_OP_DECRYPT) {
 646                 pad_len = m->pkt_len - rte_be_to_cpu_16(ip_hdr->total_length) -
 647                           ipdata_offset;
 648                 rte_pktmbuf_trim(m, pad_len);
 649         }
 650
 651         l2fwd_send_packet(m, dst_port);
 652 }
 653
 654 /** Generate random key */
 655 static void
 656 generate_random_key(uint8_t *key, unsigned length)
 657 {
 658         int fd;
 659         int ret;
 660
 661         fd = open("/dev/urandom", O_RDONLY);
 662         if (fd < 0)
 663                 rte_exit(EXIT_FAILURE, "Failed to generate random key\n");
 664
 665         ret = read(fd, key, length);
 666         close(fd);
 667
 668         if (ret != (signed)length)
 669                 rte_exit(EXIT_FAILURE, "Failed to generate random key\n");
 670 }
 671
 672 /* Session is created and is later attached to the crypto operation. 8< */
 673 static struct rte_cryptodev_sym_session *
 674 initialize_crypto_session(struct l2fwd_crypto_options *options, uint8_t cdev_id)
 675 {
 676         struct rte_crypto_sym_xform *first_xform;
 677         struct rte_cryptodev_sym_session *session;
 678         int retval = rte_cryptodev_socket_id(cdev_id);
 679
 680         if (retval < 0)
 681                 return NULL;
 682
 683         uint8_t socket_id = (uint8_t) retval;
 684
 685         if (options->xform_chain == L2FWD_CRYPTO_AEAD) {
 686                 first_xform = &options->aead_xform;
 687         } else if (options->xform_chain == L2FWD_CRYPTO_CIPHER_HASH) {
 688                 first_xform = &options->cipher_xform;
 689                 first_xform->next = &options->auth_xform;
 690         } else if (options->xform_chain == L2FWD_CRYPTO_HASH_CIPHER) {
 691                 first_xform = &options->auth_xform;
 692                 first_xform->next = &options->cipher_xform;
 693         } else if (options->xform_chain == L2FWD_CRYPTO_CIPHER_ONLY) {
 694                 first_xform = &options->cipher_xform;
 695         } else {
 696                 first_xform = &options->auth_xform;
 697         }
 698
 699         session = rte_cryptodev_sym_session_create(
 700                         session_pool_socket[socket_id].sess_mp);
 701         if (session == NULL)
 702                 return NULL;
 703
 704         if (rte_cryptodev_sym_session_init(cdev_id, session,
 705                                 first_xform,
 706                                 session_pool_socket[socket_id].priv_mp) < 0)
 707                 return NULL;
 708
 709         return session;
 710 }
 711 /* >8 End of creation of session. */
 712
 713 static void
 714 l2fwd_crypto_options_print(struct l2fwd_crypto_options *options);
 715
 716 /* main processing loop */
 717 static void
 718 l2fwd_main_loop(struct l2fwd_crypto_options *options)
 719 {
 720         struct rte_mbuf *m, *pkts_burst[MAX_PKT_BURST];
 721         struct rte_crypto_op *ops_burst[MAX_PKT_BURST];
 722
 723         unsigned lcore_id = rte_lcore_id();
 724         uint64_t prev_tsc = 0, diff_tsc, cur_tsc, timer_tsc = 0;
 725         unsigned int i, j, nb_rx, len;
 726         uint16_t portid;
 727         struct lcore_queue_conf *qconf = &lcore_queue_conf[lcore_id];
 728         const uint64_t drain_tsc = (rte_get_tsc_hz() + US_PER_S - 1) /
 729                         US_PER_S * BURST_TX_DRAIN_US;
 730         struct l2fwd_crypto_params *cparams;
 731         struct l2fwd_crypto_params port_cparams[qconf->nb_crypto_devs];
 732         struct rte_cryptodev_sym_session *session;
 733
 734         if (qconf->nb_rx_ports == 0) {
 735                 RTE_LOG(INFO, L2FWD, "lcore %u has nothing to do\n", lcore_id);
 736                 return;
 737         }
 738
 739         RTE_LOG(INFO, L2FWD, "entering main loop on lcore %u\n", lcore_id);
 740
 741         for (i = 0; i < qconf->nb_rx_ports; i++) {
 742
 743                 portid = qconf->rx_port_list[i];
 744                 RTE_LOG(INFO, L2FWD, " -- lcoreid=%u portid=%u\n", lcore_id,
 745                         portid);
 746         }
 747
 748         for (i = 0; i < qconf->nb_crypto_devs; i++) {
 749                 port_cparams[i].do_cipher = 0;
 750                 port_cparams[i].do_hash = 0;
 751                 port_cparams[i].do_aead = 0;
 752
 753                 switch (options->xform_chain) {
 754                 case L2FWD_CRYPTO_AEAD:
 755                         port_cparams[i].do_aead = 1;
 756                         break;
 757                 case L2FWD_CRYPTO_CIPHER_HASH:
 758                 case L2FWD_CRYPTO_HASH_CIPHER:
 759                         port_cparams[i].do_cipher = 1;
 760                         port_cparams[i].do_hash = 1;
 761                         break;
 762                 case L2FWD_CRYPTO_HASH_ONLY:
 763                         port_cparams[i].do_hash = 1;
 764                         break;
 765                 case L2FWD_CRYPTO_CIPHER_ONLY:
 766                         port_cparams[i].do_cipher = 1;
 767                         break;
 768                 }
 769
 770                 port_cparams[i].dev_id = qconf->cryptodev_list[i];
 771                 port_cparams[i].qp_id = 0;
 772
 773                 port_cparams[i].block_size = options->block_size;
 774
 775                 if (port_cparams[i].do_hash) {
 776                         port_cparams[i].auth_iv.data = options->auth_iv.data;
 777                         port_cparams[i].auth_iv.length = options->auth_iv.length;
 778                         if (!options->auth_iv_param)
 779                                 generate_random_key(port_cparams[i].auth_iv.data,
 780                                                 port_cparams[i].auth_iv.length);
 781                         if (options->auth_xform.auth.op == RTE_CRYPTO_AUTH_OP_VERIFY)
 782                                 port_cparams[i].hash_verify = 1;
 783                         else
 784                                 port_cparams[i].hash_verify = 0;
 785
 786                         port_cparams[i].auth_algo = options->auth_xform.auth.algo;
 787                         port_cparams[i].digest_length =
 788                                         options->auth_xform.auth.digest_length;
 789                         /* Set IV parameters */
 790                         if (options->auth_iv.length) {
 791                                 options->auth_xform.auth.iv.offset =
 792                                         IV_OFFSET + options->cipher_iv.length;
 793                                 options->auth_xform.auth.iv.length =
 794                                         options->auth_iv.length;
 795                         }
 796                 }
 797
 798                 if (port_cparams[i].do_aead) {
 799                         port_cparams[i].aead_iv.data = options->aead_iv.data;
 800                         port_cparams[i].aead_iv.length = options->aead_iv.length;
 801                         if (!options->aead_iv_param)
 802                                 generate_random_key(port_cparams[i].aead_iv.data,
 803                                                 port_cparams[i].aead_iv.length);
 804                         port_cparams[i].aead_algo = options->aead_xform.aead.algo;
 805                         port_cparams[i].digest_length =
 806                                         options->aead_xform.aead.digest_length;
 807                         if (options->aead_xform.aead.aad_length) {
 808                                 port_cparams[i].aad.data = options->aad.data;
 809                                 port_cparams[i].aad.phys_addr = options->aad.phys_addr;
 810                                 port_cparams[i].aad.length = options->aad.length;
 811                                 if (!options->aad_param)
 812                                         generate_random_key(port_cparams[i].aad.data,
 813                                                 port_cparams[i].aad.length);
 814                                 /*
 815                                  * If doing AES-CCM, first 18 bytes has to be reserved,
 816                                  * and actual AAD should start from byte 18
 817                                  */
 818                                 if (port_cparams[i].aead_algo == RTE_CRYPTO_AEAD_AES_CCM)
 819                                         memmove(port_cparams[i].aad.data + 18,
 820                                                         port_cparams[i].aad.data,
 821                                                         port_cparams[i].aad.length);
 822
 823                         } else
 824                                 port_cparams[i].aad.length = 0;
 825
 826                         if (options->aead_xform.aead.op == RTE_CRYPTO_AEAD_OP_DECRYPT)
 827                                 port_cparams[i].hash_verify = 1;
 828                         else
 829                                 port_cparams[i].hash_verify = 0;
 830
 831                         /* Set IV parameters */
 832                         options->aead_xform.aead.iv.offset = IV_OFFSET;
 833                         options->aead_xform.aead.iv.length = options->aead_iv.length;
 834                 }
 835
 836                 if (port_cparams[i].do_cipher) {
 837                         port_cparams[i].cipher_iv.data = options->cipher_iv.data;
 838                         port_cparams[i].cipher_iv.length = options->cipher_iv.length;
 839                         if (!options->cipher_iv_param)
 840                                 generate_random_key(port_cparams[i].cipher_iv.data,
 841                                                 port_cparams[i].cipher_iv.length);
 842
 843                         port_cparams[i].cipher_algo = options->cipher_xform.cipher.algo;
 844                         port_cparams[i].cipher_dataunit_len =
 845                                 options->cipher_xform.cipher.dataunit_len;
 846                         /* Set IV parameters */
 847                         options->cipher_xform.cipher.iv.offset = IV_OFFSET;
 848                         options->cipher_xform.cipher.iv.length =
 849                                                 options->cipher_iv.length;
 850                 }
 851
 852                 session = initialize_crypto_session(options,
 853                                 port_cparams[i].dev_id);
 854                 if (session == NULL)
 855                         rte_exit(EXIT_FAILURE, "Failed to initialize crypto session\n");
 856
 857                 port_cparams[i].session = session;
 858
 859                 RTE_LOG(INFO, L2FWD, " -- lcoreid=%u cryptoid=%u\n", lcore_id,
 860                                 port_cparams[i].dev_id);
 861         }
 862
 863         l2fwd_crypto_options_print(options);
 864
 865         /*
 866          * Initialize previous tsc timestamp before the loop,
 867          * to avoid showing the port statistics immediately,
 868          * so user can see the crypto information.
 869          */
 870         prev_tsc = rte_rdtsc();
 871         while (1) {
 872
 873                 cur_tsc = rte_rdtsc();
 874
 875                 /*
 876                  * Crypto device/TX burst queue drain
 877                  */
 878                 diff_tsc = cur_tsc - prev_tsc;
 879                 if (unlikely(diff_tsc > drain_tsc)) {
 880                         /* Enqueue all crypto ops remaining in buffers */
 881                         for (i = 0; i < qconf->nb_crypto_devs; i++) {
 882                                 cparams = &port_cparams[i];
 883                                 len = qconf->op_buf[cparams->dev_id].len;
 884                                 l2fwd_crypto_send_burst(qconf, len, cparams);
 885                                 qconf->op_buf[cparams->dev_id].len = 0;
 886                         }
 887                         /* Transmit all packets remaining in buffers */
 888                         for (portid = 0; portid < RTE_MAX_ETHPORTS; portid++) {
 889                                 if (qconf->pkt_buf[portid].len == 0)
 890                                         continue;
 891                                 l2fwd_send_burst(&lcore_queue_conf[lcore_id],
 892                                                  qconf->pkt_buf[portid].len,
 893                                                  portid);
 894                                 qconf->pkt_buf[portid].len = 0;
 895                         }
 896
 897                         /* if timer is enabled */
 898                         if (timer_period > 0) {
 899
 900                                 /* advance the timer */
 901                                 timer_tsc += diff_tsc;
 902
 903                                 /* if timer has reached its timeout */
 904                                 if (unlikely(timer_tsc >=
 905                                                 (uint64_t)timer_period)) {
 906
 907                                         /* do this only on main core */
 908                                         if (lcore_id == rte_get_main_lcore()
 909                                                 && options->refresh_period) {
 910                                                 print_stats();
 911                                                 timer_tsc = 0;
 912                                         }
 913                                 }
 914                         }
 915
 916                         prev_tsc = cur_tsc;
 917                 }
 918
 919                 /*
 920                  * Read packet from RX queues
 921                  */
 922                 for (i = 0; i < qconf->nb_rx_ports; i++) {
 923                         portid = qconf->rx_port_list[i];
 924
 925                         cparams = &port_cparams[i];
 926
 927                         nb_rx = rte_eth_rx_burst(portid, 0,
 928                                                  pkts_burst, MAX_PKT_BURST);
 929
 930                         port_statistics[portid].rx += nb_rx;
 931
 932                         /* Allocate and fillcrypto operations. 8< */
 933                         if (nb_rx) {
 934                                 /*
 935                                  * If we can't allocate a crypto_ops, then drop
 936                                  * the rest of the burst and dequeue and
 937                                  * process the packets to free offload structs
 938                                  */
 939                                 if (rte_crypto_op_bulk_alloc(
 940                                                 l2fwd_crypto_op_pool,
 941                                                 RTE_CRYPTO_OP_TYPE_SYMMETRIC,
 942                                                 ops_burst, nb_rx) !=
 943                                                                 nb_rx) {
 944                                         for (j = 0; j < nb_rx; j++)
 945                                                 rte_pktmbuf_free(pkts_burst[j]);
 946
 947                                         nb_rx = 0;
 948                                 }
 949                                 /* >8 End of crypto operation allocated and filled. */
 950
 951                                 /* Enqueue packets from Crypto device*/
 952                                 for (j = 0; j < nb_rx; j++) {
 953                                         m = pkts_burst[j];
 954
 955                                         l2fwd_simple_crypto_enqueue(m,
 956                                                         ops_burst[j], cparams);
 957                                 }
 958                         }
 959
 960                         /* Dequeue packets from Crypto device. 8< */
 961                         do {
 962                                 nb_rx = rte_cryptodev_dequeue_burst(
 963                                                 cparams->dev_id, cparams->qp_id,
 964                                                 ops_burst, MAX_PKT_BURST);
 965
 966                                 crypto_statistics[cparams->dev_id].dequeued +=
 967                                                 nb_rx;
 968
 969                                 /* Forward crypto'd packets */
 970                                 for (j = 0; j < nb_rx; j++) {
 971                                         m = ops_burst[j]->sym->m_src;
 972
 973                                         rte_crypto_op_free(ops_burst[j]);
 974                                         l2fwd_simple_forward(m, portid,
 975                                                         options);
 976                                 }
 977                         } while (nb_rx == MAX_PKT_BURST);
 978                         /* >8 End of dequeue packets from crypto device. */
 979                 }
 980         }
 981 }
 982
 983 static int
 984 l2fwd_launch_one_lcore(void *arg)
 985 {
 986         l2fwd_main_loop((struct l2fwd_crypto_options *)arg);
 987         return 0;
 988 }
 989
 990 /* Display command line arguments usage */
 991 static void
 992 l2fwd_crypto_usage(const char *prgname)
 993 {
 994         printf("%s [EAL options] --\n"
 995                 "  -p PORTMASK: hexadecimal bitmask of ports to configure\n"
 996                 "  -q NQ: number of queue (=ports) per lcore (default is 1)\n"
 997                 "  -s manage all ports from single lcore\n"
 998                 "  -T PERIOD: statistics will be refreshed each PERIOD seconds"
 999                 " (0 to disable, 10 default, 86400 maximum)\n"
1000
1001                 "  --cdev_type HW / SW / ANY\n"
1002                 "  --chain HASH_CIPHER / CIPHER_HASH / CIPHER_ONLY /"
1003                 " HASH_ONLY / AEAD\n"
1004
1005                 "  --cipher_algo ALGO\n"
1006                 "  --cipher_op ENCRYPT / DECRYPT\n"
1007                 "  --cipher_key KEY (bytes separated with \":\")\n"
1008                 "  --cipher_key_random_size SIZE: size of cipher key when generated randomly\n"
1009                 "  --cipher_iv IV (bytes separated with \":\")\n"
1010                 "  --cipher_iv_random_size SIZE: size of cipher IV when generated randomly\n"
1011                 "  --cipher_dataunit_len SIZE: length of the algorithm data-unit\n"
1012
1013                 "  --auth_algo ALGO\n"
1014                 "  --auth_op GENERATE / VERIFY\n"
1015                 "  --auth_key KEY (bytes separated with \":\")\n"
1016                 "  --auth_key_random_size SIZE: size of auth key when generated randomly\n"
1017                 "  --auth_iv IV (bytes separated with \":\")\n"
1018                 "  --auth_iv_random_size SIZE: size of auth IV when generated randomly\n"
1019
1020                 "  --aead_algo ALGO\n"
1021                 "  --aead_op ENCRYPT / DECRYPT\n"
1022                 "  --aead_key KEY (bytes separated with \":\")\n"
1023                 "  --aead_key_random_size SIZE: size of AEAD key when generated randomly\n"
1024                 "  --aead_iv IV (bytes separated with \":\")\n"
1025                 "  --aead_iv_random_size SIZE: size of AEAD IV when generated randomly\n"
1026                 "  --aad AAD (bytes separated with \":\")\n"
1027                 "  --aad_random_size SIZE: size of AAD when generated randomly\n"
1028
1029                 "  --digest_size SIZE: size of digest to be generated/verified\n"
1030
1031                 "  --sessionless\n"
1032                 "  --cryptodev_mask MASK: hexadecimal bitmask of crypto devices to configure\n"
1033
1034                 "  --[no-]mac-updating: Enable or disable MAC addresses updating (enabled by default)\n"
1035                 "      When enabled:\n"
1036                 "       - The source MAC address is replaced by the TX port MAC address\n"
1037                 "       - The destination MAC address is replaced by 02:00:00:00:00:TX_PORT_ID\n",
1038                prgname);
1039 }
1040
1041 /** Parse crypto device type command line argument */
1042 static int
1043 parse_cryptodev_type(enum cdev_type *type, char *optarg)
1044 {
1045         if (strcmp("HW", optarg) == 0) {
1046                 *type = CDEV_TYPE_HW;
1047                 return 0;
1048         } else if (strcmp("SW", optarg) == 0) {
1049                 *type = CDEV_TYPE_SW;
1050                 return 0;
1051         } else if (strcmp("ANY", optarg) == 0) {
1052                 *type = CDEV_TYPE_ANY;
1053                 return 0;
1054         }
1055
1056         return -1;
1057 }
1058
1059 /** Parse crypto chain xform command line argument */
1060 static int
1061 parse_crypto_opt_chain(struct l2fwd_crypto_options *options, char *optarg)
1062 {
1063         if (strcmp("CIPHER_HASH", optarg) == 0) {
1064                 options->xform_chain = L2FWD_CRYPTO_CIPHER_HASH;
1065                 return 0;
1066         } else if (strcmp("HASH_CIPHER", optarg) == 0) {
1067                 options->xform_chain = L2FWD_CRYPTO_HASH_CIPHER;
1068                 return 0;
1069         } else if (strcmp("CIPHER_ONLY", optarg) == 0) {
1070                 options->xform_chain = L2FWD_CRYPTO_CIPHER_ONLY;
1071                 return 0;
1072         } else if (strcmp("HASH_ONLY", optarg) == 0) {
1073                 options->xform_chain = L2FWD_CRYPTO_HASH_ONLY;
1074                 return 0;
1075         } else if (strcmp("AEAD", optarg) == 0) {
1076                 options->xform_chain = L2FWD_CRYPTO_AEAD;
1077                 return 0;
1078         }
1079
1080         return -1;
1081 }
1082
1083 /** Parse crypto cipher algo option command line argument */
1084 static int
1085 parse_cipher_algo(enum rte_crypto_cipher_algorithm *algo, char *optarg)
1086 {
1087
1088         if (rte_cryptodev_get_cipher_algo_enum(algo, optarg) < 0) {
1089                 RTE_LOG(ERR, USER1, "Cipher algorithm specified "
1090                                 "not supported!\n");
1091                 return -1;
1092         }
1093
1094         return 0;
1095 }
1096
1097 /** Parse crypto cipher operation command line argument */
1098 static int
1099 parse_cipher_op(enum rte_crypto_cipher_operation *op, char *optarg)
1100 {
1101         if (strcmp("ENCRYPT", optarg) == 0) {
1102                 *op = RTE_CRYPTO_CIPHER_OP_ENCRYPT;
1103                 return 0;
1104         } else if (strcmp("DECRYPT", optarg) == 0) {
1105                 *op = RTE_CRYPTO_CIPHER_OP_DECRYPT;
1106                 return 0;
1107         }
1108
1109         printf("Cipher operation not supported!\n");
1110         return -1;
1111 }
1112
1113 /** Parse bytes from command line argument */
1114 static int
1115 parse_bytes(uint8_t *data, char *input_arg, uint16_t max_size)
1116 {
1117         unsigned byte_count;
1118         char *token;
1119
1120         errno = 0;
1121         for (byte_count = 0, token = strtok(input_arg, ":");
1122                         (byte_count < max_size) && (token != NULL);
1123                         token = strtok(NULL, ":")) {
1124
1125                 int number = (int)strtol(token, NULL, 16);
1126
1127                 if (errno == EINVAL || errno == ERANGE || number > 0xFF)
1128                         return -1;
1129
1130                 data[byte_count++] = (uint8_t)number;
1131         }
1132
1133         return byte_count;
1134 }
1135
1136 /** Parse size param*/
1137 static int
1138 parse_size(int *size, const char *q_arg)
1139 {
1140         char *end = NULL;
1141         unsigned long n;
1142
1143         /* parse hexadecimal string */
1144         n = strtoul(q_arg, &end, 10);
1145         if ((q_arg[0] == '\0') || (end == NULL) || (*end != '\0'))
1146                 n = 0;
1147
1148         if (n == 0) {
1149                 printf("invalid size\n");
1150                 return -1;
1151         }
1152
1153         *size = n;
1154         return 0;
1155 }
1156
1157 /** Parse crypto cipher operation command line argument */
1158 static int
1159 parse_auth_algo(enum rte_crypto_auth_algorithm *algo, char *optarg)
1160 {
1161         if (rte_cryptodev_get_auth_algo_enum(algo, optarg) < 0) {
1162                 RTE_LOG(ERR, USER1, "Authentication algorithm specified "
1163                                 "not supported!\n");
1164                 return -1;
1165         }
1166
1167         return 0;
1168 }
1169
1170 static int
1171 parse_auth_op(enum rte_crypto_auth_operation *op, char *optarg)
1172 {
1173         if (strcmp("VERIFY", optarg) == 0) {
1174                 *op = RTE_CRYPTO_AUTH_OP_VERIFY;
1175                 return 0;
1176         } else if (strcmp("GENERATE", optarg) == 0) {
1177                 *op = RTE_CRYPTO_AUTH_OP_GENERATE;
1178                 return 0;
1179         }
1180
1181         printf("Authentication operation specified not supported!\n");
1182         return -1;
1183 }
1184
1185 static int
1186 parse_aead_algo(enum rte_crypto_aead_algorithm *algo, char *optarg)
1187 {
1188         if (rte_cryptodev_get_aead_algo_enum(algo, optarg) < 0) {
1189                 RTE_LOG(ERR, USER1, "AEAD algorithm specified "
1190                                 "not supported!\n");
1191                 return -1;
1192         }
1193
1194         return 0;
1195 }
1196
1197 static int
1198 parse_aead_op(enum rte_crypto_aead_operation *op, char *optarg)
1199 {
1200         if (strcmp("ENCRYPT", optarg) == 0) {
1201                 *op = RTE_CRYPTO_AEAD_OP_ENCRYPT;
1202                 return 0;
1203         } else if (strcmp("DECRYPT", optarg) == 0) {
1204                 *op = RTE_CRYPTO_AEAD_OP_DECRYPT;
1205                 return 0;
1206         }
1207
1208         printf("AEAD operation specified not supported!\n");
1209         return -1;
1210 }
1211 static int
1212 parse_cryptodev_mask(struct l2fwd_crypto_options *options,
1213                 const char *q_arg)
1214 {
1215         char *end = NULL;
1216         uint64_t pm;
1217
1218         /* parse hexadecimal string */
1219         pm = strtoul(q_arg, &end, 16);
1220         if ((pm == '\0') || (end == NULL) || (*end != '\0'))
1221                 pm = 0;
1222
1223         options->cryptodev_mask = pm;
1224         if (options->cryptodev_mask == 0) {
1225                 printf("invalid cryptodev_mask specified\n");
1226                 return -1;
1227         }
1228
1229         return 0;
1230 }
1231
1232 /** Parse long options */
1233 static int
1234 l2fwd_crypto_parse_args_long_options(struct l2fwd_crypto_options *options,
1235                 struct option *lgopts, int option_index)
1236 {
1237         int retval;
1238         int val;
1239
1240         if (strcmp(lgopts[option_index].name, "cdev_type") == 0) {
1241                 retval = parse_cryptodev_type(&options->type, optarg);
1242                 if (retval == 0)
1243                         strlcpy(options->string_type, optarg, MAX_STR_LEN);
1244                 return retval;
1245         }
1246
1247         else if (strcmp(lgopts[option_index].name, "chain") == 0)
1248                 return parse_crypto_opt_chain(options, optarg);
1249
1250         /* Cipher options */
1251         else if (strcmp(lgopts[option_index].name, "cipher_algo") == 0)
1252                 return parse_cipher_algo(&options->cipher_xform.cipher.algo,
1253                                 optarg);
1254
1255         else if (strcmp(lgopts[option_index].name, "cipher_op") == 0)
1256                 return parse_cipher_op(&options->cipher_xform.cipher.op,
1257                                 optarg);
1258
1259         else if (strcmp(lgopts[option_index].name, "cipher_key") == 0) {
1260                 options->ckey_param = 1;
1261                 options->cipher_xform.cipher.key.length =
1262                         parse_bytes(options->cipher_key, optarg, MAX_KEY_SIZE);
1263                 if (options->cipher_xform.cipher.key.length > 0)
1264                         return 0;
1265                 else
1266                         return -1;
1267         }
1268
1269         else if (strcmp(lgopts[option_index].name, "cipher_dataunit_len") == 0) {
1270                 retval = parse_size(&val, optarg);
1271                 if (retval == 0 && val >= 0) {
1272                         options->cipher_xform.cipher.dataunit_len =
1273                                                                 (uint32_t)val;
1274                         return 0;
1275                 } else
1276                         return -1;
1277         }
1278
1279         else if (strcmp(lgopts[option_index].name, "cipher_key_random_size") == 0)
1280                 return parse_size(&options->ckey_random_size, optarg);
1281
1282         else if (strcmp(lgopts[option_index].name, "cipher_iv") == 0) {
1283                 options->cipher_iv_param = 1;
1284                 options->cipher_iv.length =
1285                         parse_bytes(options->cipher_iv.data, optarg, MAX_IV_SIZE);
1286                 if (options->cipher_iv.length > 0)
1287                         return 0;
1288                 else
1289                         return -1;
1290         }
1291
1292         else if (strcmp(lgopts[option_index].name, "cipher_iv_random_size") == 0)
1293                 return parse_size(&options->cipher_iv_random_size, optarg);
1294
1295         /* Authentication options */
1296         else if (strcmp(lgopts[option_index].name, "auth_algo") == 0) {
1297                 return parse_auth_algo(&options->auth_xform.auth.algo,
1298                                 optarg);
1299         }
1300
1301         else if (strcmp(lgopts[option_index].name, "auth_op") == 0)
1302                 return parse_auth_op(&options->auth_xform.auth.op,
1303                                 optarg);
1304
1305         else if (strcmp(lgopts[option_index].name, "auth_key") == 0) {
1306                 options->akey_param = 1;
1307                 options->auth_xform.auth.key.length =
1308                         parse_bytes(options->auth_key, optarg, MAX_KEY_SIZE);
1309                 if (options->auth_xform.auth.key.length > 0)
1310                         return 0;
1311                 else
1312                         return -1;
1313         }
1314
1315         else if (strcmp(lgopts[option_index].name, "auth_key_random_size") == 0) {
1316                 return parse_size(&options->akey_random_size, optarg);
1317         }
1318
1319         else if (strcmp(lgopts[option_index].name, "auth_iv") == 0) {
1320                 options->auth_iv_param = 1;
1321                 options->auth_iv.length =
1322                         parse_bytes(options->auth_iv.data, optarg, MAX_IV_SIZE);
1323                 if (options->auth_iv.length > 0)
1324                         return 0;
1325                 else
1326                         return -1;
1327         }
1328
1329         else if (strcmp(lgopts[option_index].name, "auth_iv_random_size") == 0)
1330                 return parse_size(&options->auth_iv_random_size, optarg);
1331
1332         /* AEAD options */
1333         else if (strcmp(lgopts[option_index].name, "aead_algo") == 0) {
1334                 return parse_aead_algo(&options->aead_xform.aead.algo,
1335                                 optarg);
1336         }
1337
1338         else if (strcmp(lgopts[option_index].name, "aead_op") == 0)
1339                 return parse_aead_op(&options->aead_xform.aead.op,
1340                                 optarg);
1341
1342         else if (strcmp(lgopts[option_index].name, "aead_key") == 0) {
1343                 options->aead_key_param = 1;
1344                 options->aead_xform.aead.key.length =
1345                         parse_bytes(options->aead_key, optarg, MAX_KEY_SIZE);
1346                 if (options->aead_xform.aead.key.length > 0)
1347                         return 0;
1348                 else
1349                         return -1;
1350         }
1351
1352         else if (strcmp(lgopts[option_index].name, "aead_key_random_size") == 0)
1353                 return parse_size(&options->aead_key_random_size, optarg);
1354
1355
1356         else if (strcmp(lgopts[option_index].name, "aead_iv") == 0) {
1357                 options->aead_iv_param = 1;
1358                 options->aead_iv.length =
1359                         parse_bytes(options->aead_iv.data, optarg, MAX_IV_SIZE);
1360                 if (options->aead_iv.length > 0)
1361                         return 0;
1362                 else
1363                         return -1;
1364         }
1365
1366         else if (strcmp(lgopts[option_index].name, "aead_iv_random_size") == 0)
1367                 return parse_size(&options->aead_iv_random_size, optarg);
1368
1369         else if (strcmp(lgopts[option_index].name, "aad") == 0) {
1370                 options->aad_param = 1;
1371                 options->aad.length =
1372                         parse_bytes(options->aad.data, optarg, MAX_AAD_SIZE);
1373                 if (options->aad.length > 0)
1374                         return 0;
1375                 else
1376                         return -1;
1377         }
1378
1379         else if (strcmp(lgopts[option_index].name, "aad_random_size") == 0) {
1380                 return parse_size(&options->aad_random_size, optarg);
1381         }
1382
1383         else if (strcmp(lgopts[option_index].name, "digest_size") == 0) {
1384                 return parse_size(&options->digest_size, optarg);
1385         }
1386
1387         else if (strcmp(lgopts[option_index].name, "sessionless") == 0) {
1388                 options->sessionless = 1;
1389                 return 0;
1390         }
1391
1392         else if (strcmp(lgopts[option_index].name, "cryptodev_mask") == 0)
1393                 return parse_cryptodev_mask(options, optarg);
1394
1395         else if (strcmp(lgopts[option_index].name, "mac-updating") == 0) {
1396                 options->mac_updating = 1;
1397                 return 0;
1398         }
1399
1400         else if (strcmp(lgopts[option_index].name, "no-mac-updating") == 0) {
1401                 options->mac_updating = 0;
1402                 return 0;
1403         }
1404
1405         return -1;
1406 }
1407
1408 /** Parse port mask */
1409 static int
1410 l2fwd_crypto_parse_portmask(struct l2fwd_crypto_options *options,
1411                 const char *q_arg)
1412 {
1413         char *end = NULL;
1414         unsigned long pm;
1415
1416         /* parse hexadecimal string */
1417         pm = strtoul(q_arg, &end, 16);
1418         if ((pm == '\0') || (end == NULL) || (*end != '\0'))
1419                 pm = 0;
1420
1421         options->portmask = pm;
1422         if (options->portmask == 0) {
1423                 printf("invalid portmask specified\n");
1424                 return -1;
1425         }
1426
1427         return pm;
1428 }
1429
1430 /** Parse number of queues */
1431 static int
1432 l2fwd_crypto_parse_nqueue(struct l2fwd_crypto_options *options,
1433                 const char *q_arg)
1434 {
1435         char *end = NULL;
1436         unsigned long n;
1437
1438         /* parse hexadecimal string */
1439         n = strtoul(q_arg, &end, 10);
1440         if ((q_arg[0] == '\0') || (end == NULL) || (*end != '\0'))
1441                 n = 0;
1442         else if (n >= MAX_RX_QUEUE_PER_LCORE)
1443                 n = 0;
1444
1445         options->nb_ports_per_lcore = n;
1446         if (options->nb_ports_per_lcore == 0) {
1447                 printf("invalid number of ports selected\n");
1448                 return -1;
1449         }
1450
1451         return 0;
1452 }
1453
1454 /** Parse timer period */
1455 static int
1456 l2fwd_crypto_parse_timer_period(struct l2fwd_crypto_options *options,
1457                 const char *q_arg)
1458 {
1459         char *end = NULL;
1460         unsigned long n;
1461
1462         /* parse number string */
1463         n = (unsigned)strtol(q_arg, &end, 10);
1464         if ((q_arg[0] == '\0') || (end == NULL) || (*end != '\0'))
1465                 n = 0;
1466
1467         if (n >= MAX_TIMER_PERIOD) {
1468                 printf("Warning refresh period specified %lu is greater than "
1469                                 "max value %lu! using max value",
1470                                 n, MAX_TIMER_PERIOD);
1471                 n = MAX_TIMER_PERIOD;
1472         }
1473
1474         options->refresh_period = n * 1000 * TIMER_MILLISECOND;
1475
1476         return 0;
1477 }
1478
1479 /** Generate default options for application */
1480 static void
1481 l2fwd_crypto_default_options(struct l2fwd_crypto_options *options)
1482 {
1483         options->portmask = 0xffffffff;
1484         options->nb_ports_per_lcore = 1;
1485         options->refresh_period = 10000;
1486         options->single_lcore = 0;
1487         options->sessionless = 0;
1488
1489         options->xform_chain = L2FWD_CRYPTO_CIPHER_HASH;
1490
1491         /* Cipher Data */
1492         options->cipher_xform.type = RTE_CRYPTO_SYM_XFORM_CIPHER;
1493         options->cipher_xform.next = NULL;
1494         options->ckey_param = 0;
1495         options->ckey_random_size = -1;
1496         options->cipher_xform.cipher.key.length = 0;
1497         options->cipher_iv_param = 0;
1498         options->cipher_iv_random_size = -1;
1499         options->cipher_iv.length = 0;
1500
1501         options->cipher_xform.cipher.algo = RTE_CRYPTO_CIPHER_AES_CBC;
1502         options->cipher_xform.cipher.op = RTE_CRYPTO_CIPHER_OP_ENCRYPT;
1503         options->cipher_xform.cipher.dataunit_len = 0;
1504
1505         /* Authentication Data */
1506         options->auth_xform.type = RTE_CRYPTO_SYM_XFORM_AUTH;
1507         options->auth_xform.next = NULL;
1508         options->akey_param = 0;
1509         options->akey_random_size = -1;
1510         options->auth_xform.auth.key.length = 0;
1511         options->auth_iv_param = 0;
1512         options->auth_iv_random_size = -1;
1513         options->auth_iv.length = 0;
1514
1515         options->auth_xform.auth.algo = RTE_CRYPTO_AUTH_SHA1_HMAC;
1516         options->auth_xform.auth.op = RTE_CRYPTO_AUTH_OP_GENERATE;
1517
1518         /* AEAD Data */
1519         options->aead_xform.type = RTE_CRYPTO_SYM_XFORM_AEAD;
1520         options->aead_xform.next = NULL;
1521         options->aead_key_param = 0;
1522         options->aead_key_random_size = -1;
1523         options->aead_xform.aead.key.length = 0;
1524         options->aead_iv_param = 0;
1525         options->aead_iv_random_size = -1;
1526         options->aead_iv.length = 0;
1527
1528         options->aead_xform.aead.algo = RTE_CRYPTO_AEAD_AES_GCM;
1529         options->aead_xform.aead.op = RTE_CRYPTO_AEAD_OP_ENCRYPT;
1530
1531         options->aad_param = 0;
1532         options->aad_random_size = -1;
1533         options->aad.length = 0;
1534
1535         options->digest_size = -1;
1536
1537         options->type = CDEV_TYPE_ANY;
1538         options->cryptodev_mask = UINT64_MAX;
1539
1540         options->mac_updating = 1;
1541 }
1542
1543 static void
1544 display_cipher_info(struct l2fwd_crypto_options *options)
1545 {
1546         printf("\n---- Cipher information ---\n");
1547         printf("Algorithm: %s\n",
1548                 rte_crypto_cipher_algorithm_strings[options->cipher_xform.cipher.algo]);
1549         rte_hexdump(stdout, "Cipher key:",
1550                         options->cipher_xform.cipher.key.data,
1551                         options->cipher_xform.cipher.key.length);
1552         rte_hexdump(stdout, "IV:", options->cipher_iv.data, options->cipher_iv.length);
1553 }
1554
1555 static void
1556 display_auth_info(struct l2fwd_crypto_options *options)
1557 {
1558         printf("\n---- Authentication information ---\n");
1559         printf("Algorithm: %s\n",
1560                 rte_crypto_auth_algorithm_strings[options->auth_xform.auth.algo]);
1561         rte_hexdump(stdout, "Auth key:",
1562                         options->auth_xform.auth.key.data,
1563                         options->auth_xform.auth.key.length);
1564         rte_hexdump(stdout, "IV:", options->auth_iv.data, options->auth_iv.length);
1565 }
1566
1567 static void
1568 display_aead_info(struct l2fwd_crypto_options *options)
1569 {
1570         printf("\n---- AEAD information ---\n");
1571         printf("Algorithm: %s\n",
1572                 rte_crypto_aead_algorithm_strings[options->aead_xform.aead.algo]);
1573         rte_hexdump(stdout, "AEAD key:",
1574                         options->aead_xform.aead.key.data,
1575                         options->aead_xform.aead.key.length);
1576         rte_hexdump(stdout, "IV:", options->aead_iv.data, options->aead_iv.length);
1577         rte_hexdump(stdout, "AAD:", options->aad.data, options->aad.length);
1578 }
1579
1580 static void
1581 l2fwd_crypto_options_print(struct l2fwd_crypto_options *options)
1582 {
1583         char string_cipher_op[MAX_STR_LEN];
1584         char string_auth_op[MAX_STR_LEN];
1585         char string_aead_op[MAX_STR_LEN];
1586
1587         if (options->cipher_xform.cipher.op == RTE_CRYPTO_CIPHER_OP_ENCRYPT)
1588                 strcpy(string_cipher_op, "Encrypt");
1589         else
1590                 strcpy(string_cipher_op, "Decrypt");
1591
1592         if (options->auth_xform.auth.op == RTE_CRYPTO_AUTH_OP_GENERATE)
1593                 strcpy(string_auth_op, "Auth generate");
1594         else
1595                 strcpy(string_auth_op, "Auth verify");
1596
1597         if (options->aead_xform.aead.op == RTE_CRYPTO_AEAD_OP_ENCRYPT)
1598                 strcpy(string_aead_op, "Authenticated encryption");
1599         else
1600                 strcpy(string_aead_op, "Authenticated decryption");
1601
1602
1603         printf("Options:-\nn");
1604         printf("portmask: %x\n", options->portmask);
1605         printf("ports per lcore: %u\n", options->nb_ports_per_lcore);
1606         printf("refresh period : %u\n", options->refresh_period);
1607         printf("single lcore mode: %s\n",
1608                         options->single_lcore ? "enabled" : "disabled");
1609         printf("stats_printing: %s\n",
1610                         options->refresh_period == 0 ? "disabled" : "enabled");
1611
1612         printf("sessionless crypto: %s\n",
1613                         options->sessionless ? "enabled" : "disabled");
1614
1615         if (options->ckey_param && (options->ckey_random_size != -1))
1616                 printf("Cipher key already parsed, ignoring size of random key\n");
1617
1618         if (options->akey_param && (options->akey_random_size != -1))
1619                 printf("Auth key already parsed, ignoring size of random key\n");
1620
1621         if (options->cipher_iv_param && (options->cipher_iv_random_size != -1))
1622                 printf("Cipher IV already parsed, ignoring size of random IV\n");
1623
1624         if (options->auth_iv_param && (options->auth_iv_random_size != -1))
1625                 printf("Auth IV already parsed, ignoring size of random IV\n");
1626
1627         if (options->aad_param && (options->aad_random_size != -1))
1628                 printf("AAD already parsed, ignoring size of random AAD\n");
1629
1630         printf("\nCrypto chain: ");
1631         switch (options->xform_chain) {
1632         case L2FWD_CRYPTO_AEAD:
1633                 printf("Input --> %s --> Output\n", string_aead_op);
1634                 display_aead_info(options);
1635                 break;
1636         case L2FWD_CRYPTO_CIPHER_HASH:
1637                 printf("Input --> %s --> %s --> Output\n",
1638                         string_cipher_op, string_auth_op);
1639                 display_cipher_info(options);
1640                 display_auth_info(options);
1641                 break;
1642         case L2FWD_CRYPTO_HASH_CIPHER:
1643                 printf("Input --> %s --> %s --> Output\n",
1644                         string_auth_op, string_cipher_op);
1645                 display_cipher_info(options);
1646                 display_auth_info(options);
1647                 break;
1648         case L2FWD_CRYPTO_HASH_ONLY:
1649                 printf("Input --> %s --> Output\n", string_auth_op);
1650                 display_auth_info(options);
1651                 break;
1652         case L2FWD_CRYPTO_CIPHER_ONLY:
1653                 printf("Input --> %s --> Output\n", string_cipher_op);
1654                 display_cipher_info(options);
1655                 break;
1656         }
1657 }
1658
1659 /* Parse the argument given in the command line of the application */
1660 static int
1661 l2fwd_crypto_parse_args(struct l2fwd_crypto_options *options,
1662                 int argc, char **argv)
1663 {
1664         int opt, retval, option_index;
1665         char **argvopt = argv, *prgname = argv[0];
1666
1667         static struct option lgopts[] = {
1668                         { "sessionless", no_argument, 0, 0 },
1669
1670                         { "cdev_type", required_argument, 0, 0 },
1671                         { "chain", required_argument, 0, 0 },
1672
1673                         { "cipher_algo", required_argument, 0, 0 },
1674                         { "cipher_op", required_argument, 0, 0 },
1675                         { "cipher_key", required_argument, 0, 0 },
1676                         { "cipher_key_random_size", required_argument, 0, 0 },
1677                         { "cipher_iv", required_argument, 0, 0 },
1678                         { "cipher_iv_random_size", required_argument, 0, 0 },
1679                         { "cipher_dataunit_len", required_argument, 0, 0},
1680
1681                         { "auth_algo", required_argument, 0, 0 },
1682                         { "auth_op", required_argument, 0, 0 },
1683                         { "auth_key", required_argument, 0, 0 },
1684                         { "auth_key_random_size", required_argument, 0, 0 },
1685                         { "auth_iv", required_argument, 0, 0 },
1686                         { "auth_iv_random_size", required_argument, 0, 0 },
1687
1688                         { "aead_algo", required_argument, 0, 0 },
1689                         { "aead_op", required_argument, 0, 0 },
1690                         { "aead_key", required_argument, 0, 0 },
1691                         { "aead_key_random_size", required_argument, 0, 0 },
1692                         { "aead_iv", required_argument, 0, 0 },
1693                         { "aead_iv_random_size", required_argument, 0, 0 },
1694
1695                         { "aad", required_argument, 0, 0 },
1696                         { "aad_random_size", required_argument, 0, 0 },
1697
1698                         { "digest_size", required_argument, 0, 0 },
1699
1700                         { "sessionless", no_argument, 0, 0 },
1701                         { "cryptodev_mask", required_argument, 0, 0},
1702
1703                         { "mac-updating", no_argument, 0, 0},
1704                         { "no-mac-updating", no_argument, 0, 0},
1705
1706                         { NULL, 0, 0, 0 }
1707         };
1708
1709         l2fwd_crypto_default_options(options);
1710
1711         while ((opt = getopt_long(argc, argvopt, "p:q:sT:", lgopts,
1712                         &option_index)) != EOF) {
1713                 switch (opt) {
1714                 /* long options */
1715                 case 0:
1716                         retval = l2fwd_crypto_parse_args_long_options(options,
1717                                         lgopts, option_index);
1718                         if (retval < 0) {
1719                                 l2fwd_crypto_usage(prgname);
1720                                 return -1;
1721                         }
1722                         break;
1723
1724                 /* portmask */
1725                 case 'p':
1726                         retval = l2fwd_crypto_parse_portmask(options, optarg);
1727                         if (retval < 0) {
1728                                 l2fwd_crypto_usage(prgname);
1729                                 return -1;
1730                         }
1731                         break;
1732
1733                 /* nqueue */
1734                 case 'q':
1735                         retval = l2fwd_crypto_parse_nqueue(options, optarg);
1736                         if (retval < 0) {
1737                                 l2fwd_crypto_usage(prgname);
1738                                 return -1;
1739                         }
1740                         break;
1741
1742                 /* single  */
1743                 case 's':
1744                         options->single_lcore = 1;
1745
1746                         break;
1747
1748                 /* timer period */
1749                 case 'T':
1750                         retval = l2fwd_crypto_parse_timer_period(options,
1751                                         optarg);
1752                         if (retval < 0) {
1753                                 l2fwd_crypto_usage(prgname);
1754                                 return -1;
1755                         }
1756                         break;
1757
1758                 default:
1759                         l2fwd_crypto_usage(prgname);
1760                         return -1;
1761                 }
1762         }
1763
1764
1765         if (optind >= 0)
1766                 argv[optind-1] = prgname;
1767
1768         retval = optind-1;
1769         optind = 1; /* reset getopt lib */
1770
1771         return retval;
1772 }
1773
1774 /* Check the link status of all ports in up to 9s, and print them finally */
1775 static void
1776 check_all_ports_link_status(uint32_t port_mask)
1777 {
1778 #define CHECK_INTERVAL 100 /* 100ms */
1779 #define MAX_CHECK_TIME 90 /* 9s (90 * 100ms) in total */
1780         uint16_t portid;
1781         uint8_t count, all_ports_up, print_flag = 0;
1782         struct rte_eth_link link;
1783         int ret;
1784         char link_status_text[RTE_ETH_LINK_MAX_STR_LEN];
1785
1786         printf("\nChecking link status");
1787         fflush(stdout);
1788         for (count = 0; count <= MAX_CHECK_TIME; count++) {
1789                 all_ports_up = 1;
1790                 RTE_ETH_FOREACH_DEV(portid) {
1791                         if ((port_mask & (1 << portid)) == 0)
1792                                 continue;
1793                         memset(&link, 0, sizeof(link));
1794                         ret = rte_eth_link_get_nowait(portid, &link);
1795                         if (ret < 0) {
1796                                 all_ports_up = 0;
1797                                 if (print_flag == 1)
1798                                         printf("Port %u link get failed: %s\n",
1799                                                 portid, rte_strerror(-ret));
1800                                 continue;
1801                         }
1802                         /* print link status if flag set */
1803                         if (print_flag == 1) {
1804                                 rte_eth_link_to_str(link_status_text,
1805                                         sizeof(link_status_text), &link);
1806                                 printf("Port %d %s\n", portid,
1807                                         link_status_text);
1808                                 continue;
1809                         }
1810                         /* clear all_ports_up flag if any link down */
1811                         if (link.link_status == ETH_LINK_DOWN) {
1812                                 all_ports_up = 0;
1813                                 break;
1814                         }
1815                 }
1816                 /* after finally printing all link status, get out */
1817                 if (print_flag == 1)
1818                         break;
1819
1820                 if (all_ports_up == 0) {
1821                         printf(".");
1822                         fflush(stdout);
1823                         rte_delay_ms(CHECK_INTERVAL);
1824                 }
1825
1826                 /* set the print_flag if all ports up or timeout */
1827                 if (all_ports_up == 1 || count == (MAX_CHECK_TIME - 1)) {
1828                         print_flag = 1;
1829                         printf("done\n");
1830                 }
1831         }
1832 }
1833
1834 /* Check if device has to be HW/SW or any */
1835 static int
1836 check_type(const struct l2fwd_crypto_options *options,
1837                 const struct rte_cryptodev_info *dev_info)
1838 {
1839         if (options->type == CDEV_TYPE_HW &&
1840                         (dev_info->feature_flags & RTE_CRYPTODEV_FF_HW_ACCELERATED))
1841                 return 0;
1842         if (options->type == CDEV_TYPE_SW &&
1843                         !(dev_info->feature_flags & RTE_CRYPTODEV_FF_HW_ACCELERATED))
1844                 return 0;
1845         if (options->type == CDEV_TYPE_ANY)
1846                 return 0;
1847
1848         return -1;
1849 }
1850
1851 static const struct rte_cryptodev_capabilities *
1852 check_device_support_cipher_algo(const struct l2fwd_crypto_options *options,
1853                 const struct rte_cryptodev_info *dev_info,
1854                 uint8_t cdev_id)
1855 {
1856         unsigned int i = 0;
1857         const struct rte_cryptodev_capabilities *cap = &dev_info->capabilities[0];
1858         enum rte_crypto_cipher_algorithm cap_cipher_algo;
1859         enum rte_crypto_cipher_algorithm opt_cipher_algo =
1860                                         options->cipher_xform.cipher.algo;
1861
1862         while (cap->op != RTE_CRYPTO_OP_TYPE_UNDEFINED) {
1863                 cap_cipher_algo = cap->sym.cipher.algo;
1864                 if (cap->sym.xform_type == RTE_CRYPTO_SYM_XFORM_CIPHER) {
1865                         if (cap_cipher_algo == opt_cipher_algo) {
1866                                 if (check_type(options, dev_info) == 0)
1867                                         break;
1868                         }
1869                 }
1870                 cap = &dev_info->capabilities[++i];
1871         }
1872
1873         if (cap->op == RTE_CRYPTO_OP_TYPE_UNDEFINED) {
1874                 printf("Algorithm %s not supported by cryptodev %u"
1875                         " or device not of preferred type (%s)\n",
1876                         rte_crypto_cipher_algorithm_strings[opt_cipher_algo],
1877                         cdev_id,
1878                         options->string_type);
1879                 return NULL;
1880         }
1881
1882         return cap;
1883 }
1884
1885 static const struct rte_cryptodev_capabilities *
1886 check_device_support_auth_algo(const struct l2fwd_crypto_options *options,
1887                 const struct rte_cryptodev_info *dev_info,
1888                 uint8_t cdev_id)
1889 {
1890         unsigned int i = 0;
1891         const struct rte_cryptodev_capabilities *cap = &dev_info->capabilities[0];
1892         enum rte_crypto_auth_algorithm cap_auth_algo;
1893         enum rte_crypto_auth_algorithm opt_auth_algo =
1894                                         options->auth_xform.auth.algo;
1895
1896         while (cap->op != RTE_CRYPTO_OP_TYPE_UNDEFINED) {
1897                 cap_auth_algo = cap->sym.auth.algo;
1898                 if (cap->sym.xform_type == RTE_CRYPTO_SYM_XFORM_AUTH) {
1899                         if (cap_auth_algo == opt_auth_algo) {
1900                                 if (check_type(options, dev_info) == 0)
1901                                         break;
1902                         }
1903                 }
1904                 cap = &dev_info->capabilities[++i];
1905         }
1906
1907         if (cap->op == RTE_CRYPTO_OP_TYPE_UNDEFINED) {
1908                 printf("Algorithm %s not supported by cryptodev %u"
1909                         " or device not of preferred type (%s)\n",
1910                         rte_crypto_auth_algorithm_strings[opt_auth_algo],
1911                         cdev_id,
1912                         options->string_type);
1913                 return NULL;
1914         }
1915
1916         return cap;
1917 }
1918
1919 static const struct rte_cryptodev_capabilities *
1920 check_device_support_aead_algo(const struct l2fwd_crypto_options *options,
1921                 const struct rte_cryptodev_info *dev_info,
1922                 uint8_t cdev_id)
1923 {
1924         unsigned int i = 0;
1925         const struct rte_cryptodev_capabilities *cap = &dev_info->capabilities[0];
1926         enum rte_crypto_aead_algorithm cap_aead_algo;
1927         enum rte_crypto_aead_algorithm opt_aead_algo =
1928                                         options->aead_xform.aead.algo;
1929
1930         while (cap->op != RTE_CRYPTO_OP_TYPE_UNDEFINED) {
1931                 cap_aead_algo = cap->sym.aead.algo;
1932                 if (cap->sym.xform_type == RTE_CRYPTO_SYM_XFORM_AEAD) {
1933                         if (cap_aead_algo == opt_aead_algo) {
1934                                 if (check_type(options, dev_info) == 0)
1935                                         break;
1936                         }
1937                 }
1938                 cap = &dev_info->capabilities[++i];
1939         }
1940
1941         if (cap->op == RTE_CRYPTO_OP_TYPE_UNDEFINED) {
1942                 printf("Algorithm %s not supported by cryptodev %u"
1943                         " or device not of preferred type (%s)\n",
1944                         rte_crypto_aead_algorithm_strings[opt_aead_algo],
1945                         cdev_id,
1946                         options->string_type);
1947                 return NULL;
1948         }
1949
1950         return cap;
1951 }
1952
1953 /* Check if the device is enabled by cryptodev_mask */
1954 static int
1955 check_cryptodev_mask(struct l2fwd_crypto_options *options,
1956                 uint8_t cdev_id)
1957 {
1958         if (options->cryptodev_mask & (1 << cdev_id))
1959                 return 0;
1960
1961         return -1;
1962 }
1963
1964 static inline int
1965 check_supported_size(uint16_t length, uint16_t min, uint16_t max,
1966                 uint16_t increment)
1967 {
1968         uint16_t supp_size;
1969
1970         /* Single value */
1971         if (increment == 0) {
1972                 if (length == min)
1973                         return 0;
1974                 else
1975                         return -1;
1976         }
1977
1978         /* Range of values */
1979         for (supp_size = min; supp_size <= max; supp_size += increment) {
1980                 if (length == supp_size)
1981                         return 0;
1982         }
1983
1984         return -1;
1985 }
1986
1987 static int
1988 check_iv_param(const struct rte_crypto_param_range *iv_range_size,
1989                 unsigned int iv_param, int iv_random_size,
1990                 uint16_t iv_length)
1991 {
1992         /*
1993          * Check if length of provided IV is supported
1994          * by the algorithm chosen.
1995          */
1996         if (iv_param) {
1997                 if (check_supported_size(iv_length,
1998                                 iv_range_size->min,
1999                                 iv_range_size->max,
2000                                 iv_range_size->increment)
2001                                         != 0)
2002                         return -1;
2003         /*
2004          * Check if length of IV to be randomly generated
2005          * is supported by the algorithm chosen.
2006          */
2007         } else if (iv_random_size != -1) {
2008                 if (check_supported_size(iv_random_size,
2009                                 iv_range_size->min,
2010                                 iv_range_size->max,
2011                                 iv_range_size->increment)
2012                                         != 0)
2013                         return -1;
2014         }
2015
2016         return 0;
2017 }
2018
2019 static int
2020 check_capabilities(struct l2fwd_crypto_options *options, uint8_t cdev_id)
2021 {
2022         struct rte_cryptodev_info dev_info;
2023         const struct rte_cryptodev_capabilities *cap;
2024
2025         rte_cryptodev_info_get(cdev_id, &dev_info);
2026
2027         /* Set AEAD parameters */
2028         if (options->xform_chain == L2FWD_CRYPTO_AEAD) {
2029                 /* Check if device supports AEAD algo */
2030                 cap = check_device_support_aead_algo(options, &dev_info,
2031                                                 cdev_id);
2032                 if (cap == NULL)
2033                         return -1;
2034
2035                 if (check_iv_param(&cap->sym.aead.iv_size,
2036                                 options->aead_iv_param,
2037                                 options->aead_iv_random_size,
2038                                 options->aead_iv.length) != 0) {
2039                         RTE_LOG(DEBUG, USER1,
2040                                 "Device %u does not support IV length\n",
2041                                 cdev_id);
2042                         return -1;
2043                 }
2044
2045                 /*
2046                  * Check if length of provided AEAD key is supported
2047                  * by the algorithm chosen.
2048                  */
2049                 if (options->aead_key_param) {
2050                         if (check_supported_size(
2051                                         options->aead_xform.aead.key.length,
2052                                         cap->sym.aead.key_size.min,
2053                                         cap->sym.aead.key_size.max,
2054                                         cap->sym.aead.key_size.increment)
2055                                                 != 0) {
2056                                 RTE_LOG(DEBUG, USER1,
2057                                         "Device %u does not support "
2058                                         "AEAD key length\n",
2059                                         cdev_id);
2060                                 return -1;
2061                         }
2062                 /*
2063                  * Check if length of the aead key to be randomly generated
2064                  * is supported by the algorithm chosen.
2065                  */
2066                 } else if (options->aead_key_random_size != -1) {
2067                         if (check_supported_size(options->aead_key_random_size,
2068                                         cap->sym.aead.key_size.min,
2069                                         cap->sym.aead.key_size.max,
2070                                         cap->sym.aead.key_size.increment)
2071                                                 != 0) {
2072                                 RTE_LOG(DEBUG, USER1,
2073                                         "Device %u does not support "
2074                                         "AEAD key length\n",
2075                                         cdev_id);
2076                                 return -1;
2077                         }
2078                 }
2079
2080
2081                 /*
2082                  * Check if length of provided AAD is supported
2083                  * by the algorithm chosen.
2084                  */
2085                 if (options->aad_param) {
2086                         if (check_supported_size(options->aad.length,
2087                                         cap->sym.aead.aad_size.min,
2088                                         cap->sym.aead.aad_size.max,
2089                                         cap->sym.aead.aad_size.increment)
2090                                                 != 0) {
2091                                 RTE_LOG(DEBUG, USER1,
2092                                         "Device %u does not support "
2093                                         "AAD length\n",
2094                                         cdev_id);
2095                                 return -1;
2096                         }
2097                 /*
2098                  * Check if length of AAD to be randomly generated
2099                  * is supported by the algorithm chosen.
2100                  */
2101                 } else if (options->aad_random_size != -1) {
2102                         if (check_supported_size(options->aad_random_size,
2103                                         cap->sym.aead.aad_size.min,
2104                                         cap->sym.aead.aad_size.max,
2105                                         cap->sym.aead.aad_size.increment)
2106                                                 != 0) {
2107                                 RTE_LOG(DEBUG, USER1,
2108                                         "Device %u does not support "
2109                                         "AAD length\n",
2110                                         cdev_id);
2111                                 return -1;
2112                         }
2113                 }
2114
2115                 /* Check if digest size is supported by the algorithm. */
2116                 if (options->digest_size != -1) {
2117                         if (check_supported_size(options->digest_size,
2118                                         cap->sym.aead.digest_size.min,
2119                                         cap->sym.aead.digest_size.max,
2120                                         cap->sym.aead.digest_size.increment)
2121                                                 != 0) {
2122                                 RTE_LOG(DEBUG, USER1,
2123                                         "Device %u does not support "
2124                                         "digest length\n",
2125                                         cdev_id);
2126                                 return -1;
2127                         }
2128                 }
2129         }
2130
2131         /* Set cipher parameters */
2132         if (options->xform_chain == L2FWD_CRYPTO_CIPHER_HASH ||
2133                         options->xform_chain == L2FWD_CRYPTO_HASH_CIPHER ||
2134                         options->xform_chain == L2FWD_CRYPTO_CIPHER_ONLY) {
2135
2136                 /* Check if device supports cipher algo. 8< */
2137                 cap = check_device_support_cipher_algo(options, &dev_info,
2138                                                 cdev_id);
2139                 if (cap == NULL)
2140                         return -1;
2141
2142                 if (check_iv_param(&cap->sym.cipher.iv_size,
2143                                 options->cipher_iv_param,
2144                                 options->cipher_iv_random_size,
2145                                 options->cipher_iv.length) != 0) {
2146                         RTE_LOG(DEBUG, USER1,
2147                                 "Device %u does not support IV length\n",
2148                                 cdev_id);
2149                         return -1;
2150                 }
2151                 /* >8 End of check if device supports cipher algo. */
2152
2153                 /* Check if capable cipher is supported. 8< */
2154
2155                 /*
2156                  * Check if length of provided cipher key is supported
2157                  * by the algorithm chosen.
2158                  */
2159                 if (options->ckey_param) {
2160                         if (check_supported_size(
2161                                         options->cipher_xform.cipher.key.length,
2162                                         cap->sym.cipher.key_size.min,
2163                                         cap->sym.cipher.key_size.max,
2164                                         cap->sym.cipher.key_size.increment)
2165                                                 != 0) {
2166                                 if (dev_info.feature_flags &
2167                                     RTE_CRYPTODEV_FF_CIPHER_WRAPPED_KEY) {
2168                                         RTE_LOG(DEBUG, USER1,
2169                                         "Key length does not match the device "
2170                                         "%u capability. Key may be wrapped\n",
2171                                         cdev_id);
2172                                 } else {
2173                                         RTE_LOG(DEBUG, USER1,
2174                                         "Key length does not match the device "
2175                                         "%u capability\n",
2176                                         cdev_id);
2177                                         return -1;
2178                                 }
2179                         }
2180
2181                 /*
2182                  * Check if length of the cipher key to be randomly generated
2183                  * is supported by the algorithm chosen.
2184                  */
2185                 } else if (options->ckey_random_size != -1) {
2186                         if (check_supported_size(options->ckey_random_size,
2187                                         cap->sym.cipher.key_size.min,
2188                                         cap->sym.cipher.key_size.max,
2189                                         cap->sym.cipher.key_size.increment)
2190                                                 != 0) {
2191                                 RTE_LOG(DEBUG, USER1,
2192                                         "Device %u does not support cipher "
2193                                         "key length\n",
2194                                         cdev_id);
2195                                 return -1;
2196                         }
2197                 }
2198
2199                 if (options->cipher_xform.cipher.dataunit_len > 0) {
2200                         if (!(dev_info.feature_flags &
2201                                 RTE_CRYPTODEV_FF_CIPHER_MULTIPLE_DATA_UNITS)) {
2202                                 RTE_LOG(DEBUG, USER1,
2203                                         "Device %u does not support "
2204                                         "cipher multiple data units\n",
2205                                         cdev_id);
2206                                 return -1;
2207                         }
2208                         if (cap->sym.cipher.dataunit_set != 0) {
2209                                 int ret = 0;
2210
2211                                 switch (options->cipher_xform.cipher.dataunit_len) {
2212                                 case 512:
2213                                         if (!(cap->sym.cipher.dataunit_set &
2214                                                 RTE_CRYPTO_CIPHER_DATA_UNIT_LEN_512_BYTES))
2215                                                 ret = -1;
2216                                         break;
2217                                 case 4096:
2218                                         if (!(cap->sym.cipher.dataunit_set &
2219                                                 RTE_CRYPTO_CIPHER_DATA_UNIT_LEN_4096_BYTES))
2220                                                 ret = -1;
2221                                         break;
2222                                 default:
2223                                         ret = -1;
2224                                 }
2225                                 if (ret == -1) {
2226                                         RTE_LOG(DEBUG, USER1,
2227                                                 "Device %u does not support "
2228                                                 "data-unit length %u\n",
2229                                                 cdev_id,
2230                                                 options->cipher_xform.cipher.dataunit_len);
2231                                         return -1;
2232                                 }
2233                         }
2234                 }
2235                 /* >8 End of checking if cipher is supported. */
2236         }
2237
2238         /* Set auth parameters */
2239         if (options->xform_chain == L2FWD_CRYPTO_CIPHER_HASH ||
2240                         options->xform_chain == L2FWD_CRYPTO_HASH_CIPHER ||
2241                         options->xform_chain == L2FWD_CRYPTO_HASH_ONLY) {
2242                 /* Check if device supports auth algo */
2243                 cap = check_device_support_auth_algo(options, &dev_info,
2244                                                 cdev_id);
2245                 if (cap == NULL)
2246                         return -1;
2247
2248                 if (check_iv_param(&cap->sym.auth.iv_size,
2249                                 options->auth_iv_param,
2250                                 options->auth_iv_random_size,
2251                                 options->auth_iv.length) != 0) {
2252                         RTE_LOG(DEBUG, USER1,
2253                                 "Device %u does not support IV length\n",
2254                                 cdev_id);
2255                         return -1;
2256                 }
2257                 /*
2258                  * Check if length of provided auth key is supported
2259                  * by the algorithm chosen.
2260                  */
2261                 if (options->akey_param) {
2262                         if (check_supported_size(
2263                                         options->auth_xform.auth.key.length,
2264                                         cap->sym.auth.key_size.min,
2265                                         cap->sym.auth.key_size.max,
2266                                         cap->sym.auth.key_size.increment)
2267                                                 != 0) {
2268                                 RTE_LOG(DEBUG, USER1,
2269                                         "Device %u does not support auth "
2270                                         "key length\n",
2271                                         cdev_id);
2272                                 return -1;
2273                         }
2274                 /*
2275                  * Check if length of the auth key to be randomly generated
2276                  * is supported by the algorithm chosen.
2277                  */
2278                 } else if (options->akey_random_size != -1) {
2279                         if (check_supported_size(options->akey_random_size,
2280                                         cap->sym.auth.key_size.min,
2281                                         cap->sym.auth.key_size.max,
2282                                         cap->sym.auth.key_size.increment)
2283                                                 != 0) {
2284                                 RTE_LOG(DEBUG, USER1,
2285                                         "Device %u does not support auth "
2286                                         "key length\n",
2287                                         cdev_id);
2288                                 return -1;
2289                         }
2290                 }
2291
2292                 /* Check if digest size is supported by the algorithm. */
2293                 if (options->digest_size != -1) {
2294                         if (check_supported_size(options->digest_size,
2295                                         cap->sym.auth.digest_size.min,
2296                                         cap->sym.auth.digest_size.max,
2297                                         cap->sym.auth.digest_size.increment)
2298                                                 != 0) {
2299                                 RTE_LOG(DEBUG, USER1,
2300                                         "Device %u does not support "
2301                                         "digest length\n",
2302                                         cdev_id);
2303                                 return -1;
2304                         }
2305                 }
2306         }
2307
2308         return 0;
2309 }
2310
2311 static int
2312 initialize_cryptodevs(struct l2fwd_crypto_options *options, unsigned nb_ports,
2313                 uint8_t *enabled_cdevs)
2314 {
2315         uint8_t cdev_id, cdev_count, enabled_cdev_count = 0;
2316         const struct rte_cryptodev_capabilities *cap;
2317         unsigned int sess_sz, max_sess_sz = 0;
2318         uint32_t sessions_needed = 0;
2319         int retval;
2320
2321         cdev_count = rte_cryptodev_count();
2322         if (cdev_count == 0) {
2323                 printf("No crypto devices available\n");
2324                 return -1;
2325         }
2326
2327         for (cdev_id = 0; cdev_id < cdev_count && enabled_cdev_count < nb_ports;
2328                         cdev_id++) {
2329                 if (check_cryptodev_mask(options, cdev_id) < 0)
2330                         continue;
2331
2332                 if (check_capabilities(options, cdev_id) < 0)
2333                         continue;
2334
2335                 sess_sz = rte_cryptodev_sym_get_private_session_size(cdev_id);
2336                 if (sess_sz > max_sess_sz)
2337                         max_sess_sz = sess_sz;
2338
2339                 l2fwd_enabled_crypto_mask |= (((uint64_t)1) << cdev_id);
2340
2341                 enabled_cdevs[cdev_id] = 1;
2342                 enabled_cdev_count++;
2343         }
2344
2345         for (cdev_id = 0; cdev_id < cdev_count; cdev_id++) {
2346                 struct rte_cryptodev_qp_conf qp_conf;
2347                 struct rte_cryptodev_info dev_info;
2348
2349                 if (enabled_cdevs[cdev_id] == 0)
2350                         continue;
2351
2352                 if (check_cryptodev_mask(options, cdev_id) < 0)
2353                         continue;
2354
2355                 if (check_capabilities(options, cdev_id) < 0)
2356                         continue;
2357
2358                 retval = rte_cryptodev_socket_id(cdev_id);
2359
2360                 if (retval < 0) {
2361                         printf("Invalid crypto device id used\n");
2362                         return -1;
2363                 }
2364
2365                 uint8_t socket_id = (uint8_t) retval;
2366
2367                 struct rte_cryptodev_config conf = {
2368                         .nb_queue_pairs = 1,
2369                         .socket_id = socket_id,
2370                         .ff_disable = RTE_CRYPTODEV_FF_SECURITY,
2371                 };
2372
2373                 rte_cryptodev_info_get(cdev_id, &dev_info);
2374
2375                 /*
2376                  * Two sessions objects are required for each session
2377                  * (one for the header, one for the private data)
2378                  */
2379                 if (!strcmp(dev_info.driver_name, "crypto_scheduler")) {
2380 #ifdef RTE_CRYPTO_SCHEDULER
2381                         uint32_t nb_workers =
2382                                 rte_cryptodev_scheduler_workers_get(cdev_id,
2383                                                                 NULL);
2384
2385                         sessions_needed = enabled_cdev_count * nb_workers;
2386 #endif
2387                 } else
2388                         sessions_needed = enabled_cdev_count;
2389
2390                 if (session_pool_socket[socket_id].priv_mp == NULL) {
2391                         char mp_name[RTE_MEMPOOL_NAMESIZE];
2392
2393                         snprintf(mp_name, RTE_MEMPOOL_NAMESIZE,
2394                                 "priv_sess_mp_%u", socket_id);
2395
2396                         session_pool_socket[socket_id].priv_mp =
2397                                         rte_mempool_create(mp_name,
2398                                                 sessions_needed,
2399                                                 max_sess_sz,
2400                                                 0, 0, NULL, NULL, NULL,
2401                                                 NULL, socket_id,
2402                                                 0);
2403
2404                         if (session_pool_socket[socket_id].priv_mp == NULL) {
2405                                 printf("Cannot create pool on socket %d\n",
2406                                         socket_id);
2407                                 return -ENOMEM;
2408                         }
2409
2410                         printf("Allocated pool \"%s\" on socket %d\n",
2411                                 mp_name, socket_id);
2412                 }
2413
2414                 if (session_pool_socket[socket_id].sess_mp == NULL) {
2415                         char mp_name[RTE_MEMPOOL_NAMESIZE];
2416                         snprintf(mp_name, RTE_MEMPOOL_NAMESIZE,
2417                                 "sess_mp_%u", socket_id);
2418
2419                         session_pool_socket[socket_id].sess_mp =
2420                                         rte_cryptodev_sym_session_pool_create(
2421                                                         mp_name,
2422                                                         sessions_needed,
2423                                                         0, 0, 0, socket_id);
2424
2425                         if (session_pool_socket[socket_id].sess_mp == NULL) {
2426                                 printf("Cannot create pool on socket %d\n",
2427                                         socket_id);
2428                                 return -ENOMEM;
2429                         }
2430
2431                         printf("Allocated pool \"%s\" on socket %d\n",
2432                                 mp_name, socket_id);
2433                 }
2434
2435                 /* Set AEAD parameters */
2436                 if (options->xform_chain == L2FWD_CRYPTO_AEAD) {
2437                         cap = check_device_support_aead_algo(options, &dev_info,
2438                                                         cdev_id);
2439
2440                         options->block_size = cap->sym.aead.block_size;
2441
2442                         /* Set IV if not provided from command line */
2443                         if (options->aead_iv_param == 0) {
2444                                 if (options->aead_iv_random_size != -1)
2445                                         options->aead_iv.length =
2446                                                 options->aead_iv_random_size;
2447                                 /* No size provided, use minimum size. */
2448                                 else
2449                                         options->aead_iv.length =
2450                                                 cap->sym.aead.iv_size.min;
2451                         }
2452
2453                         /* Set key if not provided from command line */
2454                         if (options->aead_key_param == 0) {
2455                                 if (options->aead_key_random_size != -1)
2456                                         options->aead_xform.aead.key.length =
2457                                                 options->aead_key_random_size;
2458                                 /* No size provided, use minimum size. */
2459                                 else
2460                                         options->aead_xform.aead.key.length =
2461                                                 cap->sym.aead.key_size.min;
2462
2463                                 generate_random_key(options->aead_key,
2464                                         options->aead_xform.aead.key.length);
2465                         }
2466
2467                         /* Set AAD if not provided from command line */
2468                         if (options->aad_param == 0) {
2469                                 if (options->aad_random_size != -1)
2470                                         options->aad.length =
2471                                                 options->aad_random_size;
2472                                 /* No size provided, use minimum size. */
2473                                 else
2474                                         options->aad.length =
2475                                                 cap->sym.auth.aad_size.min;
2476                         }
2477
2478                         options->aead_xform.aead.aad_length =
2479                                                 options->aad.length;
2480
2481                         /* Set digest size if not provided from command line */
2482                         if (options->digest_size != -1)
2483                                 options->aead_xform.aead.digest_length =
2484                                                         options->digest_size;
2485                                 /* No size provided, use minimum size. */
2486                         else
2487                                 options->aead_xform.aead.digest_length =
2488                                                 cap->sym.aead.digest_size.min;
2489                 }
2490
2491                 /* Set cipher parameters */
2492                 if (options->xform_chain == L2FWD_CRYPTO_CIPHER_HASH ||
2493                                 options->xform_chain == L2FWD_CRYPTO_HASH_CIPHER ||
2494                                 options->xform_chain == L2FWD_CRYPTO_CIPHER_ONLY) {
2495                         cap = check_device_support_cipher_algo(options, &dev_info,
2496                                                         cdev_id);
2497                         options->block_size = cap->sym.cipher.block_size;
2498
2499                         /* Set IV if not provided from command line */
2500                         if (options->cipher_iv_param == 0) {
2501                                 if (options->cipher_iv_random_size != -1)
2502                                         options->cipher_iv.length =
2503                                                 options->cipher_iv_random_size;
2504                                 /* No size provided, use minimum size. */
2505                                 else
2506                                         options->cipher_iv.length =
2507                                                 cap->sym.cipher.iv_size.min;
2508                         }
2509
2510                         /* Set key if not provided from command line */
2511                         if (options->ckey_param == 0) {
2512                                 if (options->ckey_random_size != -1)
2513                                         options->cipher_xform.cipher.key.length =
2514                                                 options->ckey_random_size;
2515                                 /* No size provided, use minimum size. */
2516                                 else
2517                                         options->cipher_xform.cipher.key.length =
2518                                                 cap->sym.cipher.key_size.min;
2519
2520                                 generate_random_key(options->cipher_key,
2521                                         options->cipher_xform.cipher.key.length);
2522                         }
2523                 }
2524
2525                 /* Set auth parameters */
2526                 if (options->xform_chain == L2FWD_CRYPTO_CIPHER_HASH ||
2527                                 options->xform_chain == L2FWD_CRYPTO_HASH_CIPHER ||
2528                                 options->xform_chain == L2FWD_CRYPTO_HASH_ONLY) {
2529                         cap = check_device_support_auth_algo(options, &dev_info,
2530                                                         cdev_id);
2531
2532                         /* Set IV if not provided from command line */
2533                         if (options->auth_iv_param == 0) {
2534                                 if (options->auth_iv_random_size != -1)
2535                                         options->auth_iv.length =
2536                                                 options->auth_iv_random_size;
2537                                 /* No size provided, use minimum size. */
2538                                 else
2539                                         options->auth_iv.length =
2540                                                 cap->sym.auth.iv_size.min;
2541                         }
2542
2543                         /* Set key if not provided from command line */
2544                         if (options->akey_param == 0) {
2545                                 if (options->akey_random_size != -1)
2546                                         options->auth_xform.auth.key.length =
2547                                                 options->akey_random_size;
2548                                 /* No size provided, use minimum size. */
2549                                 else
2550                                         options->auth_xform.auth.key.length =
2551                                                 cap->sym.auth.key_size.min;
2552
2553                                 generate_random_key(options->auth_key,
2554                                         options->auth_xform.auth.key.length);
2555                         }
2556
2557                         /* Set digest size if not provided from command line */
2558                         if (options->digest_size != -1)
2559                                 options->auth_xform.auth.digest_length =
2560                                                         options->digest_size;
2561                                 /* No size provided, use minimum size. */
2562                         else
2563                                 options->auth_xform.auth.digest_length =
2564                                                 cap->sym.auth.digest_size.min;
2565                 }
2566
2567                 retval = rte_cryptodev_configure(cdev_id, &conf);
2568                 if (retval < 0) {
2569                         printf("Failed to configure cryptodev %u", cdev_id);
2570                         return -1;
2571                 }
2572
2573                 qp_conf.nb_descriptors = 2048;
2574                 qp_conf.mp_session = session_pool_socket[socket_id].sess_mp;
2575                 qp_conf.mp_session_private =
2576                                 session_pool_socket[socket_id].priv_mp;
2577
2578                 retval = rte_cryptodev_queue_pair_setup(cdev_id, 0, &qp_conf,
2579                                 socket_id);
2580                 if (retval < 0) {
2581                         printf("Failed to setup queue pair %u on cryptodev %u",
2582                                         0, cdev_id);
2583                         return -1;
2584                 }
2585
2586                 retval = rte_cryptodev_start(cdev_id);
2587                 if (retval < 0) {
2588                         printf("Failed to start device %u: error %d\n",
2589                                         cdev_id, retval);
2590                         return -1;
2591                 }
2592         }
2593
2594         return enabled_cdev_count;
2595 }
2596
2597 static int
2598 initialize_ports(struct l2fwd_crypto_options *options)
2599 {
2600         uint16_t last_portid = 0, portid;
2601         unsigned enabled_portcount = 0;
2602         unsigned nb_ports = rte_eth_dev_count_avail();
2603
2604         if (nb_ports == 0) {
2605                 printf("No Ethernet ports - bye\n");
2606                 return -1;
2607         }
2608
2609         /* Reset l2fwd_dst_ports */
2610         for (portid = 0; portid < RTE_MAX_ETHPORTS; portid++)
2611                 l2fwd_dst_ports[portid] = 0;
2612
2613         RTE_ETH_FOREACH_DEV(portid) {
2614                 int retval;
2615                 struct rte_eth_dev_info dev_info;
2616                 struct rte_eth_rxconf rxq_conf;
2617                 struct rte_eth_txconf txq_conf;
2618                 struct rte_eth_conf local_port_conf = port_conf;
2619
2620                 /* Skip ports that are not enabled */
2621                 if ((options->portmask & (1 << portid)) == 0)
2622                         continue;
2623
2624                 /* init port */
2625                 printf("Initializing port %u... ", portid);
2626                 fflush(stdout);
2627
2628                 retval = rte_eth_dev_info_get(portid, &dev_info);
2629                 if (retval != 0) {
2630                         printf("Error during getting device (port %u) info: %s\n",
2631                                         portid, strerror(-retval));
2632                         return retval;
2633                 }
2634
2635                 if (dev_info.tx_offload_capa & DEV_TX_OFFLOAD_MBUF_FAST_FREE)
2636                         local_port_conf.txmode.offloads |=
2637                                 DEV_TX_OFFLOAD_MBUF_FAST_FREE;
2638                 retval = rte_eth_dev_configure(portid, 1, 1, &local_port_conf);
2639                 if (retval < 0) {
2640                         printf("Cannot configure device: err=%d, port=%u\n",
2641                                   retval, portid);
2642                         return -1;
2643                 }
2644
2645                 retval = rte_eth_dev_adjust_nb_rx_tx_desc(portid, &nb_rxd,
2646                                                           &nb_txd);
2647                 if (retval < 0) {
2648                         printf("Cannot adjust number of descriptors: err=%d, port=%u\n",
2649                                 retval, portid);
2650                         return -1;
2651                 }
2652
2653                 /* init one RX queue */
2654                 fflush(stdout);
2655                 rxq_conf = dev_info.default_rxconf;
2656                 rxq_conf.offloads = local_port_conf.rxmode.offloads;
2657                 retval = rte_eth_rx_queue_setup(portid, 0, nb_rxd,
2658                                              rte_eth_dev_socket_id(portid),
2659                                              &rxq_conf, l2fwd_pktmbuf_pool);
2660                 if (retval < 0) {
2661                         printf("rte_eth_rx_queue_setup:err=%d, port=%u\n",
2662                                         retval, portid);
2663                         return -1;
2664                 }
2665
2666                 /* init one TX queue on each port */
2667                 fflush(stdout);
2668                 txq_conf = dev_info.default_txconf;
2669                 txq_conf.offloads = local_port_conf.txmode.offloads;
2670                 retval = rte_eth_tx_queue_setup(portid, 0, nb_txd,
2671                                 rte_eth_dev_socket_id(portid),
2672                                 &txq_conf);
2673                 if (retval < 0) {
2674                         printf("rte_eth_tx_queue_setup:err=%d, port=%u\n",
2675                                 retval, portid);
2676
2677                         return -1;
2678                 }
2679
2680                 /* Start device */
2681                 retval = rte_eth_dev_start(portid);
2682                 if (retval < 0) {
2683                         printf("rte_eth_dev_start:err=%d, port=%u\n",
2684                                         retval, portid);
2685                         return -1;
2686                 }
2687
2688                 retval = rte_eth_promiscuous_enable(portid);
2689                 if (retval != 0) {
2690                         printf("rte_eth_promiscuous_enable:err=%s, port=%u\n",
2691                                 rte_strerror(-retval), portid);
2692                         return -1;
2693                 }
2694
2695                 retval = rte_eth_macaddr_get(portid,
2696                                              &l2fwd_ports_eth_addr[portid]);
2697                 if (retval < 0) {
2698                         printf("rte_eth_macaddr_get :err=%d, port=%u\n",
2699                                         retval, portid);
2700                         return -1;
2701                 }
2702
2703                 printf("Port %u, MAC address: " RTE_ETHER_ADDR_PRT_FMT "\n\n",
2704                         portid,
2705                         RTE_ETHER_ADDR_BYTES(&l2fwd_ports_eth_addr[portid]));
2706
2707                 /* initialize port stats */
2708                 memset(&port_statistics, 0, sizeof(port_statistics));
2709
2710                 /* Setup port forwarding table */
2711                 if (enabled_portcount % 2) {
2712                         l2fwd_dst_ports[portid] = last_portid;
2713                         l2fwd_dst_ports[last_portid] = portid;
2714                 } else {
2715                         last_portid = portid;
2716                 }
2717
2718                 l2fwd_enabled_port_mask |= (1 << portid);
2719                 enabled_portcount++;
2720         }
2721
2722         if (enabled_portcount == 1) {
2723                 l2fwd_dst_ports[last_portid] = last_portid;
2724         } else if (enabled_portcount % 2) {
2725                 printf("odd number of ports in portmask- bye\n");
2726                 return -1;
2727         }
2728
2729         check_all_ports_link_status(l2fwd_enabled_port_mask);
2730
2731         return enabled_portcount;
2732 }
2733
2734 static void
2735 reserve_key_memory(struct l2fwd_crypto_options *options)
2736 {
2737         options->cipher_xform.cipher.key.data = options->cipher_key;
2738
2739         options->auth_xform.auth.key.data = options->auth_key;
2740
2741         options->aead_xform.aead.key.data = options->aead_key;
2742
2743         options->cipher_iv.data = rte_malloc("cipher iv", MAX_KEY_SIZE, 0);
2744         if (options->cipher_iv.data == NULL)
2745                 rte_exit(EXIT_FAILURE, "Failed to allocate memory for cipher IV");
2746
2747         options->auth_iv.data = rte_malloc("auth iv", MAX_KEY_SIZE, 0);
2748         if (options->auth_iv.data == NULL)
2749                 rte_exit(EXIT_FAILURE, "Failed to allocate memory for auth IV");
2750
2751         options->aead_iv.data = rte_malloc("aead_iv", MAX_KEY_SIZE, 0);
2752         if (options->aead_iv.data == NULL)
2753                 rte_exit(EXIT_FAILURE, "Failed to allocate memory for AEAD iv");
2754
2755         options->aad.data = rte_malloc("aad", MAX_KEY_SIZE, 0);
2756         if (options->aad.data == NULL)
2757                 rte_exit(EXIT_FAILURE, "Failed to allocate memory for AAD");
2758         options->aad.phys_addr = rte_malloc_virt2iova(options->aad.data);
2759 }
2760
2761 int
2762 main(int argc, char **argv)
2763 {
2764         struct lcore_queue_conf *qconf = NULL;
2765         struct l2fwd_crypto_options options;
2766
2767         uint8_t nb_cryptodevs, cdev_id;
2768         uint16_t portid;
2769         unsigned lcore_id, rx_lcore_id = 0;
2770         int ret, enabled_cdevcount, enabled_portcount;
2771         uint8_t enabled_cdevs[RTE_CRYPTO_MAX_DEVS] = {0};
2772
2773         /* init EAL */
2774         ret = rte_eal_init(argc, argv);
2775         if (ret < 0)
2776                 rte_exit(EXIT_FAILURE, "Invalid EAL arguments\n");
2777         argc -= ret;
2778         argv += ret;
2779
2780         /* reserve memory for Cipher/Auth key and IV */
2781         reserve_key_memory(&options);
2782
2783         /* parse application arguments (after the EAL ones) */
2784         ret = l2fwd_crypto_parse_args(&options, argc, argv);
2785         if (ret < 0)
2786                 rte_exit(EXIT_FAILURE, "Invalid L2FWD-CRYPTO arguments\n");
2787
2788         printf("MAC updating %s\n",
2789                         options.mac_updating ? "enabled" : "disabled");
2790
2791         /* create the mbuf pool */
2792         l2fwd_pktmbuf_pool = rte_pktmbuf_pool_create("mbuf_pool", NB_MBUF, 512,
2793                         RTE_ALIGN(sizeof(struct rte_crypto_op),
2794                                 RTE_CACHE_LINE_SIZE),
2795                         RTE_MBUF_DEFAULT_BUF_SIZE, rte_socket_id());
2796         if (l2fwd_pktmbuf_pool == NULL)
2797                 rte_exit(EXIT_FAILURE, "Cannot create mbuf pool\n");
2798
2799         /* create crypto op pool */
2800         l2fwd_crypto_op_pool = rte_crypto_op_pool_create("crypto_op_pool",
2801                         RTE_CRYPTO_OP_TYPE_SYMMETRIC, NB_MBUF, 128, MAXIMUM_IV_LENGTH,
2802                         rte_socket_id());
2803         if (l2fwd_crypto_op_pool == NULL)
2804                 rte_exit(EXIT_FAILURE, "Cannot create crypto op pool\n");
2805
2806         /* Enable Ethernet ports */
2807         enabled_portcount = initialize_ports(&options);
2808         if (enabled_portcount < 1)
2809                 rte_exit(EXIT_FAILURE, "Failed to initial Ethernet ports\n");
2810
2811         /* Initialize the port/queue configuration of each logical core */
2812         RTE_ETH_FOREACH_DEV(portid) {
2813
2814                 /* skip ports that are not enabled */
2815                 if ((options.portmask & (1 << portid)) == 0)
2816                         continue;
2817
2818                 if (options.single_lcore && qconf == NULL) {
2819                         while (rte_lcore_is_enabled(rx_lcore_id) == 0) {
2820                                 rx_lcore_id++;
2821                                 if (rx_lcore_id >= RTE_MAX_LCORE)
2822                                         rte_exit(EXIT_FAILURE,
2823                                                         "Not enough cores\n");
2824                         }
2825                 } else if (!options.single_lcore) {
2826                         /* get the lcore_id for this port */
2827                         while (rte_lcore_is_enabled(rx_lcore_id) == 0 ||
2828                                lcore_queue_conf[rx_lcore_id].nb_rx_ports ==
2829                                options.nb_ports_per_lcore) {
2830                                 rx_lcore_id++;
2831                                 if (rx_lcore_id >= RTE_MAX_LCORE)
2832                                         rte_exit(EXIT_FAILURE,
2833                                                         "Not enough cores\n");
2834                         }
2835                 }
2836
2837                 /* Assigned a new logical core in the loop above. */
2838                 if (qconf != &lcore_queue_conf[rx_lcore_id])
2839                         qconf = &lcore_queue_conf[rx_lcore_id];
2840
2841                 qconf->rx_port_list[qconf->nb_rx_ports] = portid;
2842                 qconf->nb_rx_ports++;
2843
2844                 printf("Lcore %u: RX port %u\n", rx_lcore_id, portid);
2845         }
2846
2847         /* Enable Crypto devices */
2848         enabled_cdevcount = initialize_cryptodevs(&options, enabled_portcount,
2849                         enabled_cdevs);
2850         if (enabled_cdevcount < 0)
2851                 rte_exit(EXIT_FAILURE, "Failed to initialize crypto devices\n");
2852
2853         if (enabled_cdevcount < enabled_portcount)
2854                 rte_exit(EXIT_FAILURE, "Number of capable crypto devices (%d) "
2855                                 "has to be more or equal to number of ports (%d)\n",
2856                                 enabled_cdevcount, enabled_portcount);
2857
2858         nb_cryptodevs = rte_cryptodev_count();
2859
2860         /* Initialize the port/cryptodev configuration of each logical core */
2861         for (rx_lcore_id = 0, qconf = NULL, cdev_id = 0;
2862                         cdev_id < nb_cryptodevs && enabled_cdevcount;
2863                         cdev_id++) {
2864                 /* Crypto op not supported by crypto device */
2865                 if (!enabled_cdevs[cdev_id])
2866                         continue;
2867
2868                 if (options.single_lcore && qconf == NULL) {
2869                         while (rte_lcore_is_enabled(rx_lcore_id) == 0) {
2870                                 rx_lcore_id++;
2871                                 if (rx_lcore_id >= RTE_MAX_LCORE)
2872                                         rte_exit(EXIT_FAILURE,
2873                                                         "Not enough cores\n");
2874                         }
2875                 } else if (!options.single_lcore) {
2876                         /* get the lcore_id for this port */
2877                         while (rte_lcore_is_enabled(rx_lcore_id) == 0 ||
2878                                lcore_queue_conf[rx_lcore_id].nb_crypto_devs ==
2879                                options.nb_ports_per_lcore) {
2880                                 rx_lcore_id++;
2881                                 if (rx_lcore_id >= RTE_MAX_LCORE)
2882                                         rte_exit(EXIT_FAILURE,
2883                                                         "Not enough cores\n");
2884                         }
2885                 }
2886
2887                 /* Assigned a new logical core in the loop above. */
2888                 if (qconf != &lcore_queue_conf[rx_lcore_id])
2889                         qconf = &lcore_queue_conf[rx_lcore_id];
2890
2891                 qconf->cryptodev_list[qconf->nb_crypto_devs] = cdev_id;
2892                 qconf->nb_crypto_devs++;
2893
2894                 enabled_cdevcount--;
2895
2896                 printf("Lcore %u: cryptodev %u\n", rx_lcore_id,
2897                                 (unsigned)cdev_id);
2898         }
2899
2900         /* launch per-lcore init on every lcore */
2901         rte_eal_mp_remote_launch(l2fwd_launch_one_lcore, (void *)&options,
2902                         CALL_MAIN);
2903         RTE_LCORE_FOREACH_WORKER(lcore_id) {
2904                 if (rte_eal_wait_lcore(lcore_id) < 0)
2905                         return -1;
2906         }
2907
2908         /* clean up the EAL */
2909         rte_eal_cleanup();
2910
2911         return 0;
2912 }