examples/bbdev_app/main.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(c) 2017 Intel Corporation
   3  */
   4
   5 #include <stdio.h>
   6 #include <stdlib.h>
   7 #include <string.h>
   8 #include <stdint.h>
   9 #include <inttypes.h>
  10 #include <sys/types.h>
  11 #include <unistd.h>
  12 #include <sys/queue.h>
  13 #include <stdarg.h>
  14 #include <ctype.h>
  15 #include <errno.h>
  16 #include <math.h>
  17 #include <assert.h>
  18 #include <getopt.h>
  19 #include <signal.h>
  20
  21 #include <rte_common.h>
  22 #include <rte_eal.h>
  23 #include <rte_cycles.h>
  24 #include <rte_ether.h>
  25 #include <rte_ethdev.h>
  26 #include <rte_ip.h>
  27 #include <rte_lcore.h>
  28 #include <rte_malloc.h>
  29 #include <rte_mbuf.h>
  30 #include <rte_mbuf_dyn.h>
  31 #include <rte_memory.h>
  32 #include <rte_mempool.h>
  33 #include <rte_log.h>
  34 #include <rte_bbdev.h>
  35 #include <rte_bbdev_op.h>
  36
  37 /* LLR values - negative value for '1' bit */
  38 #define LLR_1_BIT 0x81
  39 #define LLR_0_BIT 0x7F
  40
  41 #define MAX_PKT_BURST 32
  42 #define NB_MBUF 8191
  43 #define MEMPOOL_CACHE_SIZE 256
  44
  45 /* Hardcoded K value */
  46 #define K 40
  47 #define NCB (3 * RTE_ALIGN_CEIL(K + 4, 32))
  48
  49 #define CRC_24B_LEN 3
  50
  51 /* Configurable number of RX/TX ring descriptors */
  52 #define RTE_TEST_RX_DESC_DEFAULT 128
  53 #define RTE_TEST_TX_DESC_DEFAULT 512
  54
  55 #define BBDEV_ASSERT(a) do { \
  56         if (!(a)) { \
  57                 usage(prgname); \
  58                 return -1; \
  59         } \
  60 } while (0)
  61
  62 static int input_dynfield_offset = -1;
  63
  64 static inline struct rte_mbuf **
  65 mbuf_input(struct rte_mbuf *mbuf)
  66 {
  67         return RTE_MBUF_DYNFIELD(mbuf,
  68                         input_dynfield_offset, struct rte_mbuf **);
  69 }
  70
  71 static const struct rte_eth_conf port_conf = {
  72         .rxmode = {
  73                 .mq_mode = ETH_MQ_RX_NONE,
  74                 .max_rx_pkt_len = RTE_ETHER_MAX_LEN,
  75                 .split_hdr_size = 0,
  76         },
  77         .txmode = {
  78                 .mq_mode = ETH_MQ_TX_NONE,
  79         },
  80 };
  81
  82 struct rte_bbdev_op_turbo_enc def_op_enc = {
  83         /* These values are arbitrarily put, and does not map to the real
  84          * values for the data received from ethdev ports
  85          */
  86         .rv_index = 0,
  87         .code_block_mode = 1,
  88         .cb_params = {
  89                 .k = K,
  90         },
  91         .op_flags = RTE_BBDEV_TURBO_CRC_24A_ATTACH
  92 };
  93
  94 struct rte_bbdev_op_turbo_dec def_op_dec = {
  95         /* These values are arbitrarily put, and does not map to the real
  96          * values for the data received from ethdev ports
  97          */
  98         .code_block_mode = 1,
  99         .cb_params = {
 100                 .k = K,
 101         },
 102         .rv_index = 0,
 103         .iter_max = 8,
 104         .iter_min = 4,
 105         .ext_scale = 15,
 106         .num_maps = 0,
 107         .op_flags = RTE_BBDEV_TURBO_NEG_LLR_1_BIT_IN
 108 };
 109
 110 struct app_config_params {
 111         /* Placeholders for app params */
 112         uint16_t port_id;
 113         uint16_t bbdev_id;
 114         uint64_t enc_core_mask;
 115         uint64_t dec_core_mask;
 116
 117         /* Values filled during init time */
 118         uint16_t enc_queue_ids[RTE_MAX_LCORE];
 119         uint16_t dec_queue_ids[RTE_MAX_LCORE];
 120         uint16_t num_enc_cores;
 121         uint16_t num_dec_cores;
 122 };
 123
 124 struct lcore_statistics {
 125         unsigned int enqueued;
 126         unsigned int dequeued;
 127         unsigned int rx_lost_packets;
 128         unsigned int enc_to_dec_lost_packets;
 129         unsigned int tx_lost_packets;
 130 } __rte_cache_aligned;
 131
 132 /** each lcore configuration */
 133 struct lcore_conf {
 134         uint64_t core_type;
 135
 136         unsigned int port_id;
 137         unsigned int rx_queue_id;
 138         unsigned int tx_queue_id;
 139
 140         unsigned int bbdev_id;
 141         unsigned int enc_queue_id;
 142         unsigned int dec_queue_id;
 143
 144         uint8_t llr_temp_buf[NCB];
 145
 146         struct rte_mempool *bbdev_dec_op_pool;
 147         struct rte_mempool *bbdev_enc_op_pool;
 148         struct rte_mempool *enc_out_pool;
 149         struct rte_ring *enc_to_dec_ring;
 150
 151         struct lcore_statistics *lcore_stats;
 152 } __rte_cache_aligned;
 153
 154 struct stats_lcore_params {
 155         struct lcore_conf *lconf;
 156         struct app_config_params *app_params;
 157 };
 158
 159
 160 static const struct app_config_params def_app_config = {
 161         .port_id = 0,
 162         .bbdev_id = 0,
 163         .enc_core_mask = 0x2,
 164         .dec_core_mask = 0x4,
 165         .num_enc_cores = 1,
 166         .num_dec_cores = 1,
 167 };
 168
 169 static uint16_t global_exit_flag;
 170
 171 /* display usage */
 172 static inline void
 173 usage(const char *prgname)
 174 {
 175         printf("%s [EAL options] "
 176                         "  --\n"
 177                         "  --enc_cores - number of encoding cores (default = 0x2)\n"
 178                         "  --dec_cores - number of decoding cores (default = 0x4)\n"
 179                         "  --port_id - Ethernet port ID (default = 0)\n"
 180                         "  --bbdev_id - BBDev ID (default = 0)\n"
 181                         "\n", prgname);
 182 }
 183
 184 /* parse core mask */
 185 static inline
 186 uint16_t bbdev_parse_mask(const char *mask)
 187 {
 188         char *end = NULL;
 189         unsigned long pm;
 190
 191         /* parse hexadecimal string */
 192         pm = strtoul(mask, &end, 16);
 193         if ((mask[0] == '\0') || (end == NULL) || (*end != '\0'))
 194                 return 0;
 195
 196         return pm;
 197 }
 198
 199 /* parse core mask */
 200 static inline
 201 uint16_t bbdev_parse_number(const char *mask)
 202 {
 203         char *end = NULL;
 204         unsigned long pm;
 205
 206         /* parse hexadecimal string */
 207         pm = strtoul(mask, &end, 10);
 208         if ((mask[0] == '\0') || (end == NULL) || (*end != '\0'))
 209                 return 0;
 210
 211         return pm;
 212 }
 213
 214 static int
 215 bbdev_parse_args(int argc, char **argv,
 216                 struct app_config_params *app_params)
 217 {
 218         int optind = 0;
 219         int opt;
 220         int opt_indx = 0;
 221         char *prgname = argv[0];
 222
 223         static struct option lgopts[] = {
 224                 { "enc_core_mask", required_argument, 0, 'e' },
 225                 { "dec_core_mask", required_argument, 0, 'd' },
 226                 { "port_id", required_argument, 0, 'p' },
 227                 { "bbdev_id", required_argument, 0, 'b' },
 228                 { NULL, 0, 0, 0 }
 229         };
 230
 231         BBDEV_ASSERT(argc != 0);
 232         BBDEV_ASSERT(argv != NULL);
 233         BBDEV_ASSERT(app_params != NULL);
 234
 235         while ((opt = getopt_long(argc, argv, "e:d:p:b:", lgopts, &opt_indx)) !=
 236                 EOF) {
 237                 switch (opt) {
 238                 case 'e':
 239                         app_params->enc_core_mask =
 240                                 bbdev_parse_mask(optarg);
 241                         if (app_params->enc_core_mask == 0) {
 242                                 usage(prgname);
 243                                 return -1;
 244                         }
 245                         app_params->num_enc_cores =
 246                                 __builtin_popcount(app_params->enc_core_mask);
 247                         break;
 248
 249                 case 'd':
 250                         app_params->dec_core_mask =
 251                                 bbdev_parse_mask(optarg);
 252                         if (app_params->dec_core_mask == 0) {
 253                                 usage(prgname);
 254                                 return -1;
 255                         }
 256                         app_params->num_dec_cores =
 257                                 __builtin_popcount(app_params->dec_core_mask);
 258                         break;
 259
 260                 case 'p':
 261                         app_params->port_id = bbdev_parse_number(optarg);
 262                         break;
 263
 264                 case 'b':
 265                         app_params->bbdev_id = bbdev_parse_number(optarg);
 266                         break;
 267
 268                 default:
 269                         usage(prgname);
 270                         return -1;
 271                 }
 272         }
 273         optind = 0;
 274         return optind;
 275 }
 276
 277 static void
 278 signal_handler(int signum)
 279 {
 280         printf("\nSignal %d received\n", signum);
 281         __atomic_store_n(&global_exit_flag, 1, __ATOMIC_RELAXED);
 282 }
 283
 284 static void
 285 print_mac(unsigned int portid, struct rte_ether_addr *bbdev_ports_eth_address)
 286 {
 287         printf("Port %u, MAC address: " RTE_ETHER_ADDR_PRT_FMT "\n\n",
 288                         (unsigned int) portid,
 289                         RTE_ETHER_ADDR_BYTES(bbdev_ports_eth_address));
 290 }
 291
 292 static inline void
 293 pktmbuf_free_bulk(struct rte_mbuf **mbufs, unsigned int nb_to_free)
 294 {
 295         unsigned int i;
 296         for (i = 0; i < nb_to_free; ++i)
 297                 rte_pktmbuf_free(mbufs[i]);
 298 }
 299
 300 static inline void
 301 pktmbuf_input_free_bulk(struct rte_mbuf **mbufs, unsigned int nb_to_free)
 302 {
 303         unsigned int i;
 304         for (i = 0; i < nb_to_free; ++i) {
 305                 struct rte_mbuf *rx_pkt = *mbuf_input(mbufs[i]);
 306                 rte_pktmbuf_free(rx_pkt);
 307                 rte_pktmbuf_free(mbufs[i]);
 308         }
 309 }
 310
 311 /* Check the link status of all ports in up to 9s, and print them finally */
 312 static int
 313 check_port_link_status(uint16_t port_id)
 314 {
 315 #define CHECK_INTERVAL 100 /* 100ms */
 316 #define MAX_CHECK_TIME 90 /* 9s (90 * 100ms) in total */
 317         uint8_t count;
 318         struct rte_eth_link link;
 319         int link_get_err = -EINVAL;
 320
 321         printf("\nChecking link status.");
 322         fflush(stdout);
 323
 324         for (count = 0; count <= MAX_CHECK_TIME &&
 325                         !__atomic_load_n(&global_exit_flag, __ATOMIC_RELAXED); count++) {
 326                 memset(&link, 0, sizeof(link));
 327                 link_get_err = rte_eth_link_get_nowait(port_id, &link);
 328
 329                 if (link_get_err >= 0 && link.link_status) {
 330                         const char *dp = (link.link_duplex ==
 331                                 ETH_LINK_FULL_DUPLEX) ?
 332                                 "full-duplex" : "half-duplex";
 333                         printf("\nPort %u Link Up - speed %s - %s\n",
 334                                 port_id,
 335                                 rte_eth_link_speed_to_str(link.link_speed),
 336                                 dp);
 337                         return 0;
 338                 }
 339                 printf(".");
 340                 fflush(stdout);
 341                 rte_delay_ms(CHECK_INTERVAL);
 342         }
 343
 344         if (link_get_err >= 0)
 345                 printf("\nPort %d Link Down\n", port_id);
 346         else
 347                 printf("\nGet link failed (port %d): %s\n", port_id,
 348                        rte_strerror(-link_get_err));
 349
 350         return 0;
 351 }
 352
 353 static inline void
 354 add_ether_hdr(struct rte_mbuf *pkt_src, struct rte_mbuf *pkt_dst)
 355 {
 356         struct rte_ether_hdr *eth_from;
 357         struct rte_ether_hdr *eth_to;
 358
 359         eth_from = rte_pktmbuf_mtod(pkt_src, struct rte_ether_hdr *);
 360         eth_to = rte_pktmbuf_mtod(pkt_dst, struct rte_ether_hdr *);
 361
 362         /* copy header */
 363         rte_memcpy(eth_to, eth_from, sizeof(struct rte_ether_hdr));
 364 }
 365
 366 static inline void
 367 add_awgn(struct rte_mbuf **mbufs, uint16_t num_pkts)
 368 {
 369         RTE_SET_USED(mbufs);
 370         RTE_SET_USED(num_pkts);
 371 }
 372
 373 /* Encoder output to Decoder input adapter. The Decoder accepts only soft input
 374  * so each bit of the encoder output must be translated into one byte of LLR. If
 375  * Sub-block Deinterleaver is bypassed, which is the case, the padding bytes
 376  * must additionally be insterted at the end of each sub-block.
 377  */
 378 static inline void
 379 transform_enc_out_dec_in(struct rte_mbuf **mbufs, uint8_t *temp_buf,
 380                 uint16_t num_pkts, uint16_t k)
 381 {
 382         uint16_t i, l, j;
 383         uint16_t start_bit_idx;
 384         uint16_t out_idx;
 385         uint16_t d = k + 4;
 386         uint16_t kpi = RTE_ALIGN_CEIL(d, 32);
 387         uint16_t nd = kpi - d;
 388         uint16_t ncb = 3 * kpi;
 389
 390         for (i = 0; i < num_pkts; ++i) {
 391                 uint16_t pkt_data_len = rte_pktmbuf_data_len(mbufs[i]) -
 392                                 sizeof(struct rte_ether_hdr);
 393
 394                 /* Resize the packet if needed */
 395                 if (pkt_data_len < ncb) {
 396                         char *data = rte_pktmbuf_append(mbufs[i],
 397                                         ncb - pkt_data_len);
 398                         if (data == NULL)
 399                                 printf(
 400                                         "Not enough space in decoder input packet");
 401                 }
 402
 403                 /* Translate each bit into 1 LLR byte. */
 404                 start_bit_idx = 0;
 405                 out_idx = 0;
 406                 for (j = 0; j < 3; ++j) {
 407                         for (l = start_bit_idx; l < start_bit_idx + d; ++l) {
 408                                 uint8_t *data = rte_pktmbuf_mtod_offset(
 409                                         mbufs[i], uint8_t *,
 410                                         sizeof(struct rte_ether_hdr) +
 411                                         (l >> 3));
 412                                 if (*data & (0x80 >> (l & 7)))
 413                                         temp_buf[out_idx] = LLR_1_BIT;
 414                                 else
 415                                         temp_buf[out_idx] = LLR_0_BIT;
 416                                 ++out_idx;
 417                         }
 418                         /* Padding bytes should be at the end of the sub-block.
 419                          */
 420                         memset(&temp_buf[out_idx], 0, nd);
 421                         out_idx += nd;
 422                         start_bit_idx += d;
 423                 }
 424
 425                 rte_memcpy(rte_pktmbuf_mtod_offset(mbufs[i], uint8_t *,
 426                                 sizeof(struct rte_ether_hdr)), temp_buf, ncb);
 427         }
 428 }
 429
 430 static inline void
 431 verify_data(struct rte_mbuf **mbufs, uint16_t num_pkts)
 432 {
 433         uint16_t i;
 434         for (i = 0; i < num_pkts; ++i) {
 435                 struct rte_mbuf *out = mbufs[i];
 436                 struct rte_mbuf *in = *mbuf_input(out);
 437
 438                 if (memcmp(rte_pktmbuf_mtod_offset(in, uint8_t *,
 439                                 sizeof(struct rte_ether_hdr)),
 440                                 rte_pktmbuf_mtod_offset(out, uint8_t *,
 441                                 sizeof(struct rte_ether_hdr)),
 442                                 K / 8 - CRC_24B_LEN))
 443                         printf("Input and output buffers are not equal!\n");
 444         }
 445 }
 446
 447 static int
 448 initialize_ports(struct app_config_params *app_params,
 449                 struct rte_mempool *ethdev_mbuf_mempool)
 450 {
 451         int ret;
 452         uint16_t port_id = app_params->port_id;
 453         uint16_t q;
 454         /* ethernet addresses of ports */
 455         struct rte_ether_addr bbdev_port_eth_addr;
 456
 457         /* initialize ports */
 458         printf("\nInitializing port %u...\n", app_params->port_id);
 459         ret = rte_eth_dev_configure(port_id, app_params->num_enc_cores,
 460                 app_params->num_dec_cores, &port_conf);
 461
 462         if (ret < 0) {
 463                 printf("Cannot configure device: err=%d, port=%u\n",
 464                         ret, port_id);
 465                 return -1;
 466         }
 467
 468         /* initialize RX queues for encoder */
 469         for (q = 0; q < app_params->num_enc_cores; q++) {
 470                 ret = rte_eth_rx_queue_setup(port_id, q,
 471                         RTE_TEST_RX_DESC_DEFAULT,
 472                         rte_eth_dev_socket_id(port_id),
 473                         NULL, ethdev_mbuf_mempool);
 474                 if (ret < 0) {
 475                         printf("rte_eth_rx_queue_setup: err=%d, queue=%u\n",
 476                                 ret, q);
 477                         return -1;
 478                 }
 479         }
 480         /* initialize TX queues for decoder */
 481         for (q = 0; q < app_params->num_dec_cores; q++) {
 482                 ret = rte_eth_tx_queue_setup(port_id, q,
 483                         RTE_TEST_TX_DESC_DEFAULT,
 484                         rte_eth_dev_socket_id(port_id), NULL);
 485                 if (ret < 0) {
 486                         printf("rte_eth_tx_queue_setup: err=%d, queue=%u\n",
 487                                 ret, q);
 488                         return -1;
 489                 }
 490         }
 491
 492         ret = rte_eth_promiscuous_enable(port_id);
 493         if (ret != 0) {
 494                 printf("Cannot enable promiscuous mode: err=%s, port=%u\n",
 495                         rte_strerror(-ret), port_id);
 496                 return ret;
 497         }
 498
 499         ret = rte_eth_macaddr_get(port_id, &bbdev_port_eth_addr);
 500         if (ret < 0) {
 501                 printf("rte_eth_macaddr_get: err=%d, queue=%u\n",
 502                         ret, q);
 503                 return -1;
 504         }
 505
 506         print_mac(port_id, &bbdev_port_eth_addr);
 507
 508         return 0;
 509 }
 510
 511 static void
 512 lcore_conf_init(struct app_config_params *app_params,
 513                 struct lcore_conf *lcore_conf,
 514                 struct rte_mempool **bbdev_op_pools,
 515                 struct rte_mempool *bbdev_mbuf_mempool,
 516                 struct rte_ring *enc_to_dec_ring,
 517                 struct lcore_statistics *lcore_stats)
 518 {
 519         unsigned int lcore_id;
 520         struct lcore_conf *lconf;
 521         uint16_t rx_queue_id = 0;
 522         uint16_t tx_queue_id = 0;
 523         uint16_t enc_q_id = 0;
 524         uint16_t dec_q_id = 0;
 525
 526         /* Configure lcores */
 527         for (lcore_id = 0; lcore_id < 8 * sizeof(uint64_t); ++lcore_id) {
 528                 lconf = &lcore_conf[lcore_id];
 529                 lconf->core_type = 0;
 530
 531                 if ((1ULL << lcore_id) & app_params->enc_core_mask) {
 532                         lconf->core_type |= (1 << RTE_BBDEV_OP_TURBO_ENC);
 533                         lconf->rx_queue_id = rx_queue_id++;
 534                         lconf->enc_queue_id =
 535                                         app_params->enc_queue_ids[enc_q_id++];
 536                 }
 537
 538                 if ((1ULL << lcore_id) & app_params->dec_core_mask) {
 539                         lconf->core_type |= (1 << RTE_BBDEV_OP_TURBO_DEC);
 540                         lconf->tx_queue_id = tx_queue_id++;
 541                         lconf->dec_queue_id =
 542                                         app_params->dec_queue_ids[dec_q_id++];
 543                 }
 544
 545                 lconf->bbdev_enc_op_pool =
 546                                 bbdev_op_pools[RTE_BBDEV_OP_TURBO_ENC];
 547                 lconf->bbdev_dec_op_pool =
 548                                 bbdev_op_pools[RTE_BBDEV_OP_TURBO_DEC];
 549                 lconf->bbdev_id = app_params->bbdev_id;
 550                 lconf->port_id = app_params->port_id;
 551                 lconf->enc_out_pool = bbdev_mbuf_mempool;
 552                 lconf->enc_to_dec_ring = enc_to_dec_ring;
 553                 lconf->lcore_stats = &lcore_stats[lcore_id];
 554         }
 555 }
 556
 557 static void
 558 print_lcore_stats(struct lcore_statistics *lstats, unsigned int lcore_id)
 559 {
 560         static const char *stats_border = "_______";
 561
 562         printf("\nLcore %d: %s enqueued count:\t\t%u\n",
 563                         lcore_id, stats_border, lstats->enqueued);
 564         printf("Lcore %d: %s dequeued count:\t\t%u\n",
 565                         lcore_id, stats_border, lstats->dequeued);
 566         printf("Lcore %d: %s RX lost packets count:\t\t%u\n",
 567                         lcore_id, stats_border, lstats->rx_lost_packets);
 568         printf("Lcore %d: %s encoder-to-decoder lost count:\t%u\n",
 569                         lcore_id, stats_border,
 570                         lstats->enc_to_dec_lost_packets);
 571         printf("Lcore %d: %s TX lost packets count:\t\t%u\n",
 572                         lcore_id, stats_border, lstats->tx_lost_packets);
 573 }
 574
 575 static void
 576 print_stats(struct stats_lcore_params *stats_lcore)
 577 {
 578         unsigned int l_id;
 579         unsigned int bbdev_id = stats_lcore->app_params->bbdev_id;
 580         unsigned int port_id = stats_lcore->app_params->port_id;
 581         int len, ret, i;
 582
 583         struct rte_eth_xstat *xstats;
 584         struct rte_eth_xstat_name *xstats_names;
 585         struct rte_bbdev_stats bbstats;
 586         static const char *stats_border = "_______";
 587
 588         const char clr[] = { 27, '[', '2', 'J', '\0' };
 589         const char topLeft[] = { 27, '[', '1', ';', '1', 'H', '\0' };
 590
 591         /* Clear screen and move to top left */
 592         printf("%s%s", clr, topLeft);
 593
 594         printf("PORT STATISTICS:\n================\n");
 595         len = rte_eth_xstats_get(port_id, NULL, 0);
 596         if (len < 0)
 597                 rte_exit(EXIT_FAILURE,
 598                                 "rte_eth_xstats_get(%u) failed: %d", port_id,
 599                                 len);
 600
 601         xstats = calloc(len, sizeof(*xstats));
 602         if (xstats == NULL)
 603                 rte_exit(EXIT_FAILURE,
 604                                 "Failed to calloc memory for xstats");
 605
 606         ret = rte_eth_xstats_get(port_id, xstats, len);
 607         if (ret < 0 || ret > len) {
 608                 free(xstats);
 609                 rte_exit(EXIT_FAILURE,
 610                                 "rte_eth_xstats_get(%u) len%i failed: %d",
 611                                 port_id, len, ret);
 612         }
 613
 614         xstats_names = calloc(len, sizeof(*xstats_names));
 615         if (xstats_names == NULL) {
 616                 free(xstats);
 617                 rte_exit(EXIT_FAILURE,
 618                                 "Failed to calloc memory for xstats_names");
 619         }
 620
 621         ret = rte_eth_xstats_get_names(port_id, xstats_names, len);
 622         if (ret < 0 || ret > len) {
 623                 free(xstats);
 624                 free(xstats_names);
 625                 rte_exit(EXIT_FAILURE,
 626                                 "rte_eth_xstats_get_names(%u) len%i failed: %d",
 627                                 port_id, len, ret);
 628         }
 629
 630         for (i = 0; i < len; i++) {
 631                 if (xstats[i].value > 0)
 632                         printf("Port %u: %s %s:\t\t%"PRIu64"\n",
 633                                         port_id, stats_border,
 634                                         xstats_names[i].name,
 635                                         xstats[i].value);
 636         }
 637
 638         ret = rte_bbdev_stats_get(bbdev_id, &bbstats);
 639         if (ret < 0) {
 640                 free(xstats);
 641                 free(xstats_names);
 642                 rte_exit(EXIT_FAILURE,
 643                                 "ERROR(%d): Failure to get BBDEV %u statistics\n",
 644                                 ret, bbdev_id);
 645         }
 646
 647         printf("\nBBDEV STATISTICS:\n=================\n");
 648         printf("BBDEV %u: %s enqueue count:\t\t%"PRIu64"\n",
 649                         bbdev_id, stats_border,
 650                         bbstats.enqueued_count);
 651         printf("BBDEV %u: %s dequeue count:\t\t%"PRIu64"\n",
 652                         bbdev_id, stats_border,
 653                         bbstats.dequeued_count);
 654         printf("BBDEV %u: %s enqueue error count:\t\t%"PRIu64"\n",
 655                         bbdev_id, stats_border,
 656                         bbstats.enqueue_err_count);
 657         printf("BBDEV %u: %s dequeue error count:\t\t%"PRIu64"\n\n",
 658                         bbdev_id, stats_border,
 659                         bbstats.dequeue_err_count);
 660
 661         printf("LCORE STATISTICS:\n=================\n");
 662         for (l_id = 0; l_id < RTE_MAX_LCORE; ++l_id) {
 663                 if (stats_lcore->lconf[l_id].core_type == 0)
 664                         continue;
 665                 print_lcore_stats(stats_lcore->lconf[l_id].lcore_stats, l_id);
 666         }
 667
 668         fflush(stdout);
 669
 670         free(xstats);
 671         free(xstats_names);
 672 }
 673
 674 static int
 675 stats_loop(void *arg)
 676 {
 677         struct stats_lcore_params *stats_lcore = arg;
 678
 679         while (!__atomic_load_n(&global_exit_flag, __ATOMIC_RELAXED)) {
 680                 print_stats(stats_lcore);
 681                 rte_delay_ms(500);
 682         }
 683
 684         return 0;
 685 }
 686
 687 static inline void
 688 run_encoding(struct lcore_conf *lcore_conf)
 689 {
 690         uint16_t i;
 691         uint16_t port_id, rx_queue_id;
 692         uint16_t bbdev_id, enc_queue_id;
 693         uint16_t nb_rx, nb_enq, nb_deq, nb_sent;
 694         struct rte_mbuf *rx_pkts_burst[MAX_PKT_BURST];
 695         struct rte_mbuf *enc_out_pkts[MAX_PKT_BURST];
 696         struct rte_bbdev_enc_op *bbdev_ops_burst[MAX_PKT_BURST];
 697         struct lcore_statistics *lcore_stats;
 698         struct rte_mempool *bbdev_op_pool, *enc_out_pool;
 699         struct rte_ring *enc_to_dec_ring;
 700         const int in_data_len = (def_op_enc.cb_params.k / 8) - CRC_24B_LEN;
 701
 702         lcore_stats = lcore_conf->lcore_stats;
 703         port_id = lcore_conf->port_id;
 704         rx_queue_id = lcore_conf->rx_queue_id;
 705         bbdev_id = lcore_conf->bbdev_id;
 706         enc_queue_id = lcore_conf->enc_queue_id;
 707         bbdev_op_pool = lcore_conf->bbdev_enc_op_pool;
 708         enc_out_pool = lcore_conf->enc_out_pool;
 709         enc_to_dec_ring = lcore_conf->enc_to_dec_ring;
 710
 711         /* Read packet from RX queues*/
 712         nb_rx = rte_eth_rx_burst(port_id, rx_queue_id, rx_pkts_burst,
 713                         MAX_PKT_BURST);
 714         if (!nb_rx)
 715                 return;
 716
 717         if (unlikely(rte_mempool_get_bulk(enc_out_pool, (void **)enc_out_pkts,
 718                         nb_rx) != 0)) {
 719                 pktmbuf_free_bulk(rx_pkts_burst, nb_rx);
 720                 lcore_stats->rx_lost_packets += nb_rx;
 721                 return;
 722         }
 723
 724         if (unlikely(rte_bbdev_enc_op_alloc_bulk(bbdev_op_pool, bbdev_ops_burst,
 725                         nb_rx) != 0)) {
 726                 pktmbuf_free_bulk(enc_out_pkts, nb_rx);
 727                 pktmbuf_free_bulk(rx_pkts_burst, nb_rx);
 728                 lcore_stats->rx_lost_packets += nb_rx;
 729                 return;
 730         }
 731
 732         for (i = 0; i < nb_rx; i++) {
 733                 char *data;
 734                 const uint16_t pkt_data_len =
 735                                 rte_pktmbuf_data_len(rx_pkts_burst[i]) -
 736                                 sizeof(struct rte_ether_hdr);
 737                 /* save input mbuf pointer for later comparison */
 738                 *mbuf_input(enc_out_pkts[i]) = rx_pkts_burst[i];
 739
 740                 /* copy ethernet header */
 741                 rte_pktmbuf_reset(enc_out_pkts[i]);
 742                 data = rte_pktmbuf_append(enc_out_pkts[i],
 743                                 sizeof(struct rte_ether_hdr));
 744                 if (data == NULL) {
 745                         printf(
 746                                 "Not enough space for ethernet header in encoder output mbuf\n");
 747                         continue;
 748                 }
 749                 add_ether_hdr(rx_pkts_burst[i], enc_out_pkts[i]);
 750
 751                 /* set op */
 752                 bbdev_ops_burst[i]->turbo_enc = def_op_enc;
 753
 754                 bbdev_ops_burst[i]->turbo_enc.input.data =
 755                                 rx_pkts_burst[i];
 756                 bbdev_ops_burst[i]->turbo_enc.input.offset =
 757                                 sizeof(struct rte_ether_hdr);
 758                 /* Encoder will attach the CRC24B, adjust the length */
 759                 bbdev_ops_burst[i]->turbo_enc.input.length = in_data_len;
 760
 761                 if (in_data_len < pkt_data_len)
 762                         rte_pktmbuf_trim(rx_pkts_burst[i], pkt_data_len -
 763                                         in_data_len);
 764                 else if (in_data_len > pkt_data_len) {
 765                         data = rte_pktmbuf_append(rx_pkts_burst[i],
 766                                         in_data_len - pkt_data_len);
 767                         if (data == NULL)
 768                                 printf(
 769                                         "Not enough storage in mbuf to perform the encoding\n");
 770                 }
 771
 772                 bbdev_ops_burst[i]->turbo_enc.output.data =
 773                                 enc_out_pkts[i];
 774                 bbdev_ops_burst[i]->turbo_enc.output.offset =
 775                                 sizeof(struct rte_ether_hdr);
 776         }
 777
 778         /* Enqueue packets on BBDevice */
 779         nb_enq = rte_bbdev_enqueue_enc_ops(bbdev_id, enc_queue_id,
 780                         bbdev_ops_burst, nb_rx);
 781         if (unlikely(nb_enq < nb_rx)) {
 782                 pktmbuf_input_free_bulk(&enc_out_pkts[nb_enq],
 783                                 nb_rx - nb_enq);
 784                 rte_bbdev_enc_op_free_bulk(&bbdev_ops_burst[nb_enq],
 785                                 nb_rx - nb_enq);
 786                 lcore_stats->rx_lost_packets += nb_rx - nb_enq;
 787
 788                 if (!nb_enq)
 789                         return;
 790         }
 791
 792         lcore_stats->enqueued += nb_enq;
 793
 794         /* Dequeue packets from bbdev device*/
 795         nb_deq = 0;
 796         do {
 797                 nb_deq += rte_bbdev_dequeue_enc_ops(bbdev_id, enc_queue_id,
 798                                 &bbdev_ops_burst[nb_deq], nb_enq - nb_deq);
 799         } while (unlikely(nb_deq < nb_enq));
 800
 801         lcore_stats->dequeued += nb_deq;
 802
 803         /* Generate and add AWGN */
 804         add_awgn(enc_out_pkts, nb_deq);
 805
 806         rte_bbdev_enc_op_free_bulk(bbdev_ops_burst, nb_deq);
 807
 808         /* Enqueue packets to encoder-to-decoder ring */
 809         nb_sent = rte_ring_enqueue_burst(enc_to_dec_ring, (void **)enc_out_pkts,
 810                         nb_deq, NULL);
 811         if (unlikely(nb_sent < nb_deq)) {
 812                 pktmbuf_input_free_bulk(&enc_out_pkts[nb_sent],
 813                                 nb_deq - nb_sent);
 814                 lcore_stats->enc_to_dec_lost_packets += nb_deq - nb_sent;
 815         }
 816 }
 817
 818 static void
 819 run_decoding(struct lcore_conf *lcore_conf)
 820 {
 821         uint16_t i;
 822         uint16_t port_id, tx_queue_id;
 823         uint16_t bbdev_id, bbdev_queue_id;
 824         uint16_t nb_recv, nb_enq, nb_deq, nb_tx;
 825         uint8_t *llr_temp_buf;
 826         struct rte_mbuf *recv_pkts_burst[MAX_PKT_BURST];
 827         struct rte_bbdev_dec_op *bbdev_ops_burst[MAX_PKT_BURST];
 828         struct lcore_statistics *lcore_stats;
 829         struct rte_mempool *bbdev_op_pool;
 830         struct rte_ring *enc_to_dec_ring;
 831
 832         lcore_stats = lcore_conf->lcore_stats;
 833         port_id = lcore_conf->port_id;
 834         tx_queue_id = lcore_conf->tx_queue_id;
 835         bbdev_id = lcore_conf->bbdev_id;
 836         bbdev_queue_id = lcore_conf->dec_queue_id;
 837         bbdev_op_pool = lcore_conf->bbdev_dec_op_pool;
 838         enc_to_dec_ring = lcore_conf->enc_to_dec_ring;
 839         llr_temp_buf = lcore_conf->llr_temp_buf;
 840
 841         /* Dequeue packets from the ring */
 842         nb_recv = rte_ring_dequeue_burst(enc_to_dec_ring,
 843                         (void **)recv_pkts_burst, MAX_PKT_BURST, NULL);
 844         if (!nb_recv)
 845                 return;
 846
 847         if (unlikely(rte_bbdev_dec_op_alloc_bulk(bbdev_op_pool, bbdev_ops_burst,
 848                         nb_recv) != 0)) {
 849                 pktmbuf_input_free_bulk(recv_pkts_burst, nb_recv);
 850                 lcore_stats->rx_lost_packets += nb_recv;
 851                 return;
 852         }
 853
 854         transform_enc_out_dec_in(recv_pkts_burst, llr_temp_buf, nb_recv,
 855                         def_op_dec.cb_params.k);
 856
 857         for (i = 0; i < nb_recv; i++) {
 858                 /* set op */
 859                 bbdev_ops_burst[i]->turbo_dec = def_op_dec;
 860
 861                 bbdev_ops_burst[i]->turbo_dec.input.data = recv_pkts_burst[i];
 862                 bbdev_ops_burst[i]->turbo_dec.input.offset =
 863                                 sizeof(struct rte_ether_hdr);
 864                 bbdev_ops_burst[i]->turbo_dec.input.length =
 865                                 rte_pktmbuf_data_len(recv_pkts_burst[i])
 866                                 - sizeof(struct rte_ether_hdr);
 867
 868                 bbdev_ops_burst[i]->turbo_dec.hard_output.data =
 869                                 recv_pkts_burst[i];
 870                 bbdev_ops_burst[i]->turbo_dec.hard_output.offset =
 871                                 sizeof(struct rte_ether_hdr);
 872         }
 873
 874         /* Enqueue packets on BBDevice */
 875         nb_enq = rte_bbdev_enqueue_dec_ops(bbdev_id, bbdev_queue_id,
 876                         bbdev_ops_burst, nb_recv);
 877         if (unlikely(nb_enq < nb_recv)) {
 878                 pktmbuf_input_free_bulk(&recv_pkts_burst[nb_enq],
 879                                 nb_recv - nb_enq);
 880                 rte_bbdev_dec_op_free_bulk(&bbdev_ops_burst[nb_enq],
 881                                 nb_recv - nb_enq);
 882                 lcore_stats->rx_lost_packets += nb_recv - nb_enq;
 883
 884                 if (!nb_enq)
 885                         return;
 886         }
 887
 888         lcore_stats->enqueued += nb_enq;
 889
 890         /* Dequeue packets from BBDevice */
 891         nb_deq = 0;
 892         do {
 893                 nb_deq += rte_bbdev_dequeue_dec_ops(bbdev_id, bbdev_queue_id,
 894                                 &bbdev_ops_burst[nb_deq], nb_enq - nb_deq);
 895         } while (unlikely(nb_deq < nb_enq));
 896
 897         lcore_stats->dequeued += nb_deq;
 898
 899         rte_bbdev_dec_op_free_bulk(bbdev_ops_burst, nb_deq);
 900
 901         verify_data(recv_pkts_burst, nb_deq);
 902
 903         /* Free the RX mbufs after verification */
 904         for (i = 0; i < nb_deq; ++i)
 905                 rte_pktmbuf_free(*mbuf_input(recv_pkts_burst[i]));
 906
 907         /* Transmit the packets */
 908         nb_tx = rte_eth_tx_burst(port_id, tx_queue_id, recv_pkts_burst, nb_deq);
 909         if (unlikely(nb_tx < nb_deq)) {
 910                 pktmbuf_input_free_bulk(&recv_pkts_burst[nb_tx],
 911                                 nb_deq - nb_tx);
 912                 lcore_stats->tx_lost_packets += nb_deq - nb_tx;
 913         }
 914 }
 915
 916 static int
 917 processing_loop(void *arg)
 918 {
 919         struct lcore_conf *lcore_conf = arg;
 920         const bool run_encoder = (lcore_conf->core_type &
 921                         (1 << RTE_BBDEV_OP_TURBO_ENC));
 922         const bool run_decoder = (lcore_conf->core_type &
 923                         (1 << RTE_BBDEV_OP_TURBO_DEC));
 924
 925         while (!__atomic_load_n(&global_exit_flag, __ATOMIC_RELAXED)) {
 926                 if (run_encoder)
 927                         run_encoding(lcore_conf);
 928                 if (run_decoder)
 929                         run_decoding(lcore_conf);
 930         }
 931
 932         return 0;
 933 }
 934
 935 static int
 936 prepare_bbdev_device(unsigned int dev_id, struct rte_bbdev_info *info,
 937                 struct app_config_params *app_params)
 938 {
 939         int ret;
 940         unsigned int q_id, dec_q_id, enc_q_id;
 941         struct rte_bbdev_queue_conf qconf = {0};
 942         uint16_t dec_qs_nb = app_params->num_dec_cores;
 943         uint16_t enc_qs_nb = app_params->num_enc_cores;
 944         uint16_t tot_qs = dec_qs_nb + enc_qs_nb;
 945
 946         ret = rte_bbdev_setup_queues(dev_id, tot_qs, info->socket_id);
 947         if (ret < 0)
 948                 rte_exit(EXIT_FAILURE,
 949                                 "ERROR(%d): BBDEV %u not configured properly\n",
 950                                 ret, dev_id);
 951
 952         /* setup device DEC queues */
 953         qconf.socket = info->socket_id;
 954         qconf.queue_size = info->drv.queue_size_lim;
 955         qconf.op_type = RTE_BBDEV_OP_TURBO_DEC;
 956
 957         for (q_id = 0, dec_q_id = 0; q_id < dec_qs_nb; q_id++) {
 958                 ret = rte_bbdev_queue_configure(dev_id, q_id, &qconf);
 959                 if (ret < 0)
 960                         rte_exit(EXIT_FAILURE,
 961                                         "ERROR(%d): BBDEV %u DEC queue %u not configured properly\n",
 962                                         ret, dev_id, q_id);
 963                 app_params->dec_queue_ids[dec_q_id++] = q_id;
 964         }
 965
 966         /* setup device ENC queues */
 967         qconf.op_type = RTE_BBDEV_OP_TURBO_ENC;
 968
 969         for (q_id = dec_qs_nb, enc_q_id = 0; q_id < tot_qs; q_id++) {
 970                 ret = rte_bbdev_queue_configure(dev_id, q_id, &qconf);
 971                 if (ret < 0)
 972                         rte_exit(EXIT_FAILURE,
 973                                         "ERROR(%d): BBDEV %u ENC queue %u not configured properly\n",
 974                                         ret, dev_id, q_id);
 975                 app_params->enc_queue_ids[enc_q_id++] = q_id;
 976         }
 977
 978         ret = rte_bbdev_start(dev_id);
 979
 980         if (ret != 0)
 981                 rte_exit(EXIT_FAILURE, "ERROR(%d): BBDEV %u not started\n",
 982                         ret, dev_id);
 983
 984         printf("BBdev %u started\n", dev_id);
 985
 986         return 0;
 987 }
 988
 989 static inline bool
 990 check_matching_capabilities(uint64_t mask, uint64_t required_mask)
 991 {
 992         return (mask & required_mask) == required_mask;
 993 }
 994
 995 static void
 996 enable_bbdev(struct app_config_params *app_params)
 997 {
 998         struct rte_bbdev_info dev_info;
 999         const struct rte_bbdev_op_cap *op_cap;
1000         uint16_t bbdev_id = app_params->bbdev_id;
1001         bool encoder_capable = false;
1002         bool decoder_capable = false;
1003
1004         rte_bbdev_info_get(bbdev_id, &dev_info);
1005         op_cap = dev_info.drv.capabilities;
1006
1007         while (op_cap->type != RTE_BBDEV_OP_NONE) {
1008                 if (op_cap->type == RTE_BBDEV_OP_TURBO_ENC) {
1009                         if (check_matching_capabilities(
1010                                         op_cap->cap.turbo_enc.capability_flags,
1011                                         def_op_enc.op_flags))
1012                                 encoder_capable = true;
1013                 }
1014
1015                 if (op_cap->type == RTE_BBDEV_OP_TURBO_DEC) {
1016                         if (check_matching_capabilities(
1017                                         op_cap->cap.turbo_dec.capability_flags,
1018                                         def_op_dec.op_flags))
1019                                 decoder_capable = true;
1020                 }
1021
1022                 op_cap++;
1023         }
1024
1025         if (encoder_capable == false)
1026                 rte_exit(EXIT_FAILURE,
1027                         "The specified BBDev %u doesn't have required encoder capabilities!\n",
1028                         bbdev_id);
1029         if (decoder_capable == false)
1030                 rte_exit(EXIT_FAILURE,
1031                         "The specified BBDev %u doesn't have required decoder capabilities!\n",
1032                         bbdev_id);
1033
1034         prepare_bbdev_device(bbdev_id, &dev_info, app_params);
1035 }
1036
1037 int
1038 main(int argc, char **argv)
1039 {
1040         int ret;
1041         unsigned int nb_bbdevs, flags, lcore_id;
1042         void *sigret;
1043         struct app_config_params app_params = def_app_config;
1044         struct rte_mempool *ethdev_mbuf_mempool, *bbdev_mbuf_mempool;
1045         struct rte_mempool *bbdev_op_pools[RTE_BBDEV_OP_TYPE_COUNT];
1046         struct lcore_conf lcore_conf[RTE_MAX_LCORE] = { {0} };
1047         struct lcore_statistics lcore_stats[RTE_MAX_LCORE] = { {0} };
1048         struct stats_lcore_params stats_lcore;
1049         struct rte_ring *enc_to_dec_ring;
1050         bool stats_thread_started = false;
1051         unsigned int main_lcore_id = rte_get_main_lcore();
1052
1053         static const struct rte_mbuf_dynfield input_dynfield_desc = {
1054                 .name = "example_bbdev_dynfield_input",
1055                 .size = sizeof(struct rte_mbuf *),
1056                 .align = __alignof__(struct rte_mbuf *),
1057         };
1058
1059         __atomic_store_n(&global_exit_flag, 0, __ATOMIC_RELAXED);
1060
1061         sigret = signal(SIGTERM, signal_handler);
1062         if (sigret == SIG_ERR)
1063                 rte_exit(EXIT_FAILURE, "signal(%d, ...) failed", SIGTERM);
1064
1065         sigret = signal(SIGINT, signal_handler);
1066         if (sigret == SIG_ERR)
1067                 rte_exit(EXIT_FAILURE, "signal(%d, ...) failed", SIGINT);
1068
1069         ret = rte_eal_init(argc, argv);
1070         if (ret < 0)
1071                 rte_exit(EXIT_FAILURE, "Invalid EAL arguments\n");
1072
1073         argc -= ret;
1074         argv += ret;
1075
1076         /* parse application arguments (after the EAL ones) */
1077         ret = bbdev_parse_args(argc, argv, &app_params);
1078         if (ret < 0)
1079                 rte_exit(EXIT_FAILURE, "Invalid BBDEV arguments\n");
1080
1081         /*create bbdev op pools*/
1082         bbdev_op_pools[RTE_BBDEV_OP_TURBO_DEC] =
1083                         rte_bbdev_op_pool_create("bbdev_op_pool_dec",
1084                         RTE_BBDEV_OP_TURBO_DEC, NB_MBUF, 128, rte_socket_id());
1085         bbdev_op_pools[RTE_BBDEV_OP_TURBO_ENC] =
1086                         rte_bbdev_op_pool_create("bbdev_op_pool_enc",
1087                         RTE_BBDEV_OP_TURBO_ENC, NB_MBUF, 128, rte_socket_id());
1088
1089         if ((bbdev_op_pools[RTE_BBDEV_OP_TURBO_DEC] == NULL) ||
1090                         (bbdev_op_pools[RTE_BBDEV_OP_TURBO_ENC] == NULL))
1091                 rte_exit(EXIT_FAILURE, "Cannot create bbdev op pools\n");
1092
1093         /* Create encoder to decoder ring */
1094         flags = (app_params.num_enc_cores == 1) ? RING_F_SP_ENQ : 0;
1095         if (app_params.num_dec_cores == 1)
1096                 flags |= RING_F_SC_DEQ;
1097
1098         enc_to_dec_ring = rte_ring_create("enc_to_dec_ring",
1099                 rte_align32pow2(NB_MBUF), rte_socket_id(), flags);
1100
1101         /* Get the number of available bbdev devices */
1102         nb_bbdevs = rte_bbdev_count();
1103         if (nb_bbdevs <= app_params.bbdev_id)
1104                 rte_exit(EXIT_FAILURE,
1105                                 "%u BBDevs detected, cannot use BBDev with ID %u!\n",
1106                                 nb_bbdevs, app_params.bbdev_id);
1107         printf("Number of bbdevs detected: %d\n", nb_bbdevs);
1108
1109         if (!rte_eth_dev_is_valid_port(app_params.port_id))
1110                 rte_exit(EXIT_FAILURE,
1111                                 "cannot use port with ID %u!\n",
1112                                 app_params.port_id);
1113
1114         /* create the mbuf mempool for ethdev pkts */
1115         ethdev_mbuf_mempool = rte_pktmbuf_pool_create("ethdev_mbuf_pool",
1116                         NB_MBUF, MEMPOOL_CACHE_SIZE, 0,
1117                         RTE_MBUF_DEFAULT_BUF_SIZE, rte_socket_id());
1118         if (ethdev_mbuf_mempool == NULL)
1119                 rte_exit(EXIT_FAILURE, "Cannot create ethdev mbuf mempool\n");
1120
1121         /* create the mbuf mempool for encoder output */
1122         bbdev_mbuf_mempool = rte_pktmbuf_pool_create("bbdev_mbuf_pool",
1123                         NB_MBUF, MEMPOOL_CACHE_SIZE, 0,
1124                         RTE_MBUF_DEFAULT_BUF_SIZE, rte_socket_id());
1125         if (bbdev_mbuf_mempool == NULL)
1126                 rte_exit(EXIT_FAILURE, "Cannot create ethdev mbuf mempool\n");
1127
1128         /* register mbuf field to store input pointer */
1129         input_dynfield_offset =
1130                 rte_mbuf_dynfield_register(&input_dynfield_desc);
1131         if (input_dynfield_offset < 0)
1132                 rte_exit(EXIT_FAILURE, "Cannot register mbuf field\n");
1133
1134         /* initialize ports */
1135         ret = initialize_ports(&app_params, ethdev_mbuf_mempool);
1136
1137         /* Check if all requested lcores are available */
1138         for (lcore_id = 0; lcore_id < 8 * sizeof(uint64_t); ++lcore_id)
1139                 if (((1ULL << lcore_id) & app_params.enc_core_mask) ||
1140                                 ((1ULL << lcore_id) & app_params.dec_core_mask))
1141                         if (!rte_lcore_is_enabled(lcore_id))
1142                                 rte_exit(EXIT_FAILURE,
1143                                                 "Requested lcore_id %u is not enabled!\n",
1144                                                 lcore_id);
1145
1146         /* Start ethernet port */
1147         ret = rte_eth_dev_start(app_params.port_id);
1148         if (ret < 0)
1149                 rte_exit(EXIT_FAILURE, "rte_eth_dev_start:err=%d, port=%u\n",
1150                                 ret, app_params.port_id);
1151
1152         ret = check_port_link_status(app_params.port_id);
1153         if (ret < 0)
1154                 exit(EXIT_FAILURE);
1155
1156         /* start BBDevice and save BBDev queue IDs */
1157         enable_bbdev(&app_params);
1158
1159         /* Initialize the port/queue configuration of each logical core */
1160         lcore_conf_init(&app_params, lcore_conf, bbdev_op_pools,
1161                         bbdev_mbuf_mempool, enc_to_dec_ring, lcore_stats);
1162
1163         stats_lcore.app_params = &app_params;
1164         stats_lcore.lconf = lcore_conf;
1165
1166         RTE_LCORE_FOREACH_WORKER(lcore_id) {
1167                 if (lcore_conf[lcore_id].core_type != 0)
1168                         /* launch per-lcore processing loop on worker lcores */
1169                         rte_eal_remote_launch(processing_loop,
1170                                         &lcore_conf[lcore_id], lcore_id);
1171                 else if (!stats_thread_started) {
1172                         /* launch statistics printing loop */
1173                         rte_eal_remote_launch(stats_loop, &stats_lcore,
1174                                         lcore_id);
1175                         stats_thread_started = true;
1176                 }
1177         }
1178
1179         if (!stats_thread_started &&
1180                         lcore_conf[main_lcore_id].core_type != 0)
1181                 rte_exit(EXIT_FAILURE,
1182                                 "Not enough lcores to run the statistics printing loop!");
1183         else if (lcore_conf[main_lcore_id].core_type != 0)
1184                 processing_loop(&lcore_conf[main_lcore_id]);
1185         else if (!stats_thread_started)
1186                 stats_loop(&stats_lcore);
1187
1188         RTE_LCORE_FOREACH_WORKER(lcore_id) {
1189                 ret |= rte_eal_wait_lcore(lcore_id);
1190         }
1191
1192         /* clean up the EAL */
1193         rte_eal_cleanup();
1194
1195         return ret;
1196 }