examples/bbdev_app/main.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(c) 2017 Intel Corporation
   3  */
   4
   5 #include <stdio.h>
   6 #include <stdlib.h>
   7 #include <string.h>
   8 #include <stdint.h>
   9 #include <inttypes.h>
  10 #include <sys/types.h>
  11 #include <unistd.h>
  12 #include <sys/queue.h>
  13 #include <stdarg.h>
  14 #include <ctype.h>
  15 #include <errno.h>
  16 #include <math.h>
  17 #include <assert.h>
  18 #include <getopt.h>
  19 #include <signal.h>
  20
  21 #include <rte_common.h>
  22 #include <rte_eal.h>
  23 #include <rte_cycles.h>
  24 #include <rte_ether.h>
  25 #include <rte_ethdev.h>
  26 #include <rte_ip.h>
  27 #include <rte_lcore.h>
  28 #include <rte_malloc.h>
  29 #include <rte_mbuf.h>
  30 #include <rte_mbuf_dyn.h>
  31 #include <rte_memory.h>
  32 #include <rte_mempool.h>
  33 #include <rte_log.h>
  34 #include <rte_bbdev.h>
  35 #include <rte_bbdev_op.h>
  36
  37 /* LLR values - negative value for '1' bit */
  38 #define LLR_1_BIT 0x81
  39 #define LLR_0_BIT 0x7F
  40
  41 #define MAX_PKT_BURST 32
  42 #define NB_MBUF 8191
  43 #define MEMPOOL_CACHE_SIZE 256
  44
  45 /* Hardcoded K value */
  46 #define K 40
  47 #define NCB (3 * RTE_ALIGN_CEIL(K + 4, 32))
  48
  49 #define CRC_24B_LEN 3
  50
  51 /* Configurable number of RX/TX ring descriptors */
  52 #define RTE_TEST_RX_DESC_DEFAULT 128
  53 #define RTE_TEST_TX_DESC_DEFAULT 512
  54
  55 #define BBDEV_ASSERT(a) do { \
  56         if (!(a)) { \
  57                 usage(prgname); \
  58                 return -1; \
  59         } \
  60 } while (0)
  61
  62 static int input_dynfield_offset = -1;
  63
  64 static inline struct rte_mbuf **
  65 mbuf_input(struct rte_mbuf *mbuf)
  66 {
  67         return RTE_MBUF_DYNFIELD(mbuf,
  68                         input_dynfield_offset, struct rte_mbuf **);
  69 }
  70
  71 static const struct rte_eth_conf port_conf = {
  72         .rxmode = {
  73                 .mq_mode = RTE_ETH_MQ_RX_NONE,
  74                 .split_hdr_size = 0,
  75         },
  76         .txmode = {
  77                 .mq_mode = RTE_ETH_MQ_TX_NONE,
  78         },
  79 };
  80
  81 struct rte_bbdev_op_turbo_enc def_op_enc = {
  82         /* These values are arbitrarily put, and does not map to the real
  83          * values for the data received from ethdev ports
  84          */
  85         .rv_index = 0,
  86         .code_block_mode = 1,
  87         .cb_params = {
  88                 .k = K,
  89         },
  90         .op_flags = RTE_BBDEV_TURBO_CRC_24A_ATTACH
  91 };
  92
  93 struct rte_bbdev_op_turbo_dec def_op_dec = {
  94         /* These values are arbitrarily put, and does not map to the real
  95          * values for the data received from ethdev ports
  96          */
  97         .code_block_mode = 1,
  98         .cb_params = {
  99                 .k = K,
 100         },
 101         .rv_index = 0,
 102         .iter_max = 8,
 103         .iter_min = 4,
 104         .ext_scale = 15,
 105         .num_maps = 0,
 106         .op_flags = RTE_BBDEV_TURBO_NEG_LLR_1_BIT_IN
 107 };
 108
 109 struct app_config_params {
 110         /* Placeholders for app params */
 111         uint16_t port_id;
 112         uint16_t bbdev_id;
 113         uint64_t enc_core_mask;
 114         uint64_t dec_core_mask;
 115
 116         /* Values filled during init time */
 117         uint16_t enc_queue_ids[RTE_MAX_LCORE];
 118         uint16_t dec_queue_ids[RTE_MAX_LCORE];
 119         uint16_t num_enc_cores;
 120         uint16_t num_dec_cores;
 121 };
 122
 123 struct lcore_statistics {
 124         unsigned int enqueued;
 125         unsigned int dequeued;
 126         unsigned int rx_lost_packets;
 127         unsigned int enc_to_dec_lost_packets;
 128         unsigned int tx_lost_packets;
 129 } __rte_cache_aligned;
 130
 131 /** each lcore configuration */
 132 struct lcore_conf {
 133         uint64_t core_type;
 134
 135         unsigned int port_id;
 136         unsigned int rx_queue_id;
 137         unsigned int tx_queue_id;
 138
 139         unsigned int bbdev_id;
 140         unsigned int enc_queue_id;
 141         unsigned int dec_queue_id;
 142
 143         uint8_t llr_temp_buf[NCB];
 144
 145         struct rte_mempool *bbdev_dec_op_pool;
 146         struct rte_mempool *bbdev_enc_op_pool;
 147         struct rte_mempool *enc_out_pool;
 148         struct rte_ring *enc_to_dec_ring;
 149
 150         struct lcore_statistics *lcore_stats;
 151 } __rte_cache_aligned;
 152
 153 struct stats_lcore_params {
 154         struct lcore_conf *lconf;
 155         struct app_config_params *app_params;
 156 };
 157
 158
 159 static const struct app_config_params def_app_config = {
 160         .port_id = 0,
 161         .bbdev_id = 0,
 162         .enc_core_mask = 0x2,
 163         .dec_core_mask = 0x4,
 164         .num_enc_cores = 1,
 165         .num_dec_cores = 1,
 166 };
 167
 168 static uint16_t global_exit_flag;
 169
 170 /* display usage */
 171 static inline void
 172 usage(const char *prgname)
 173 {
 174         printf("%s [EAL options] "
 175                         "  --\n"
 176                         "  --enc_cores - number of encoding cores (default = 0x2)\n"
 177                         "  --dec_cores - number of decoding cores (default = 0x4)\n"
 178                         "  --port_id - Ethernet port ID (default = 0)\n"
 179                         "  --bbdev_id - BBDev ID (default = 0)\n"
 180                         "\n", prgname);
 181 }
 182
 183 /* parse core mask */
 184 static inline
 185 uint16_t bbdev_parse_mask(const char *mask)
 186 {
 187         char *end = NULL;
 188         unsigned long pm;
 189
 190         /* parse hexadecimal string */
 191         pm = strtoul(mask, &end, 16);
 192         if ((mask[0] == '\0') || (end == NULL) || (*end != '\0'))
 193                 return 0;
 194
 195         return pm;
 196 }
 197
 198 /* parse core mask */
 199 static inline
 200 uint16_t bbdev_parse_number(const char *mask)
 201 {
 202         char *end = NULL;
 203         unsigned long pm;
 204
 205         /* parse hexadecimal string */
 206         pm = strtoul(mask, &end, 10);
 207         if ((mask[0] == '\0') || (end == NULL) || (*end != '\0'))
 208                 return 0;
 209
 210         return pm;
 211 }
 212
 213 static int
 214 bbdev_parse_args(int argc, char **argv,
 215                 struct app_config_params *app_params)
 216 {
 217         int optind = 0;
 218         int opt;
 219         int opt_indx = 0;
 220         char *prgname = argv[0];
 221
 222         static struct option lgopts[] = {
 223                 { "enc_core_mask", required_argument, 0, 'e' },
 224                 { "dec_core_mask", required_argument, 0, 'd' },
 225                 { "port_id", required_argument, 0, 'p' },
 226                 { "bbdev_id", required_argument, 0, 'b' },
 227                 { NULL, 0, 0, 0 }
 228         };
 229
 230         BBDEV_ASSERT(argc != 0);
 231         BBDEV_ASSERT(argv != NULL);
 232         BBDEV_ASSERT(app_params != NULL);
 233
 234         while ((opt = getopt_long(argc, argv, "e:d:p:b:", lgopts, &opt_indx)) !=
 235                 EOF) {
 236                 switch (opt) {
 237                 case 'e':
 238                         app_params->enc_core_mask =
 239                                 bbdev_parse_mask(optarg);
 240                         if (app_params->enc_core_mask == 0) {
 241                                 usage(prgname);
 242                                 return -1;
 243                         }
 244                         app_params->num_enc_cores =
 245                                 __builtin_popcount(app_params->enc_core_mask);
 246                         break;
 247
 248                 case 'd':
 249                         app_params->dec_core_mask =
 250                                 bbdev_parse_mask(optarg);
 251                         if (app_params->dec_core_mask == 0) {
 252                                 usage(prgname);
 253                                 return -1;
 254                         }
 255                         app_params->num_dec_cores =
 256                                 __builtin_popcount(app_params->dec_core_mask);
 257                         break;
 258
 259                 case 'p':
 260                         app_params->port_id = bbdev_parse_number(optarg);
 261                         break;
 262
 263                 case 'b':
 264                         app_params->bbdev_id = bbdev_parse_number(optarg);
 265                         break;
 266
 267                 default:
 268                         usage(prgname);
 269                         return -1;
 270                 }
 271         }
 272         optind = 0;
 273         return optind;
 274 }
 275
 276 static void
 277 signal_handler(int signum)
 278 {
 279         printf("\nSignal %d received\n", signum);
 280         __atomic_store_n(&global_exit_flag, 1, __ATOMIC_RELAXED);
 281 }
 282
 283 static void
 284 print_mac(unsigned int portid, struct rte_ether_addr *bbdev_ports_eth_address)
 285 {
 286         printf("Port %u, MAC address: " RTE_ETHER_ADDR_PRT_FMT "\n\n",
 287                         (unsigned int) portid,
 288                         RTE_ETHER_ADDR_BYTES(bbdev_ports_eth_address));
 289 }
 290
 291 static inline void
 292 pktmbuf_free_bulk(struct rte_mbuf **mbufs, unsigned int nb_to_free)
 293 {
 294         unsigned int i;
 295         for (i = 0; i < nb_to_free; ++i)
 296                 rte_pktmbuf_free(mbufs[i]);
 297 }
 298
 299 static inline void
 300 pktmbuf_input_free_bulk(struct rte_mbuf **mbufs, unsigned int nb_to_free)
 301 {
 302         unsigned int i;
 303         for (i = 0; i < nb_to_free; ++i) {
 304                 struct rte_mbuf *rx_pkt = *mbuf_input(mbufs[i]);
 305                 rte_pktmbuf_free(rx_pkt);
 306                 rte_pktmbuf_free(mbufs[i]);
 307         }
 308 }
 309
 310 /* Check the link status of all ports in up to 9s, and print them finally */
 311 static int
 312 check_port_link_status(uint16_t port_id)
 313 {
 314 #define CHECK_INTERVAL 100 /* 100ms */
 315 #define MAX_CHECK_TIME 90 /* 9s (90 * 100ms) in total */
 316         uint8_t count;
 317         struct rte_eth_link link;
 318         int link_get_err = -EINVAL;
 319
 320         printf("\nChecking link status.");
 321         fflush(stdout);
 322
 323         for (count = 0; count <= MAX_CHECK_TIME &&
 324                         !__atomic_load_n(&global_exit_flag, __ATOMIC_RELAXED); count++) {
 325                 memset(&link, 0, sizeof(link));
 326                 link_get_err = rte_eth_link_get_nowait(port_id, &link);
 327
 328                 if (link_get_err >= 0 && link.link_status) {
 329                         const char *dp = (link.link_duplex ==
 330                                 RTE_ETH_LINK_FULL_DUPLEX) ?
 331                                 "full-duplex" : "half-duplex";
 332                         printf("\nPort %u Link Up - speed %s - %s\n",
 333                                 port_id,
 334                                 rte_eth_link_speed_to_str(link.link_speed),
 335                                 dp);
 336                         return 0;
 337                 }
 338                 printf(".");
 339                 fflush(stdout);
 340                 rte_delay_ms(CHECK_INTERVAL);
 341         }
 342
 343         if (link_get_err >= 0)
 344                 printf("\nPort %d Link Down\n", port_id);
 345         else
 346                 printf("\nGet link failed (port %d): %s\n", port_id,
 347                        rte_strerror(-link_get_err));
 348
 349         return 0;
 350 }
 351
 352 static inline void
 353 add_ether_hdr(struct rte_mbuf *pkt_src, struct rte_mbuf *pkt_dst)
 354 {
 355         struct rte_ether_hdr *eth_from;
 356         struct rte_ether_hdr *eth_to;
 357
 358         eth_from = rte_pktmbuf_mtod(pkt_src, struct rte_ether_hdr *);
 359         eth_to = rte_pktmbuf_mtod(pkt_dst, struct rte_ether_hdr *);
 360
 361         /* copy header */
 362         rte_memcpy(eth_to, eth_from, sizeof(struct rte_ether_hdr));
 363 }
 364
 365 static inline void
 366 add_awgn(struct rte_mbuf **mbufs, uint16_t num_pkts)
 367 {
 368         RTE_SET_USED(mbufs);
 369         RTE_SET_USED(num_pkts);
 370 }
 371
 372 /* Encoder output to Decoder input adapter. The Decoder accepts only soft input
 373  * so each bit of the encoder output must be translated into one byte of LLR. If
 374  * Sub-block Deinterleaver is bypassed, which is the case, the padding bytes
 375  * must additionally be inserted at the end of each sub-block.
 376  */
 377 static inline void
 378 transform_enc_out_dec_in(struct rte_mbuf **mbufs, uint8_t *temp_buf,
 379                 uint16_t num_pkts, uint16_t k)
 380 {
 381         uint16_t i, l, j;
 382         uint16_t start_bit_idx;
 383         uint16_t out_idx;
 384         uint16_t d = k + 4;
 385         uint16_t kpi = RTE_ALIGN_CEIL(d, 32);
 386         uint16_t nd = kpi - d;
 387         uint16_t ncb = 3 * kpi;
 388
 389         for (i = 0; i < num_pkts; ++i) {
 390                 uint16_t pkt_data_len = rte_pktmbuf_data_len(mbufs[i]) -
 391                                 sizeof(struct rte_ether_hdr);
 392
 393                 /* Resize the packet if needed */
 394                 if (pkt_data_len < ncb) {
 395                         char *data = rte_pktmbuf_append(mbufs[i],
 396                                         ncb - pkt_data_len);
 397                         if (data == NULL)
 398                                 printf(
 399                                         "Not enough space in decoder input packet");
 400                 }
 401
 402                 /* Translate each bit into 1 LLR byte. */
 403                 start_bit_idx = 0;
 404                 out_idx = 0;
 405                 for (j = 0; j < 3; ++j) {
 406                         for (l = start_bit_idx; l < start_bit_idx + d; ++l) {
 407                                 uint8_t *data = rte_pktmbuf_mtod_offset(
 408                                         mbufs[i], uint8_t *,
 409                                         sizeof(struct rte_ether_hdr) +
 410                                         (l >> 3));
 411                                 if (*data & (0x80 >> (l & 7)))
 412                                         temp_buf[out_idx] = LLR_1_BIT;
 413                                 else
 414                                         temp_buf[out_idx] = LLR_0_BIT;
 415                                 ++out_idx;
 416                         }
 417                         /* Padding bytes should be at the end of the sub-block.
 418                          */
 419                         memset(&temp_buf[out_idx], 0, nd);
 420                         out_idx += nd;
 421                         start_bit_idx += d;
 422                 }
 423
 424                 rte_memcpy(rte_pktmbuf_mtod_offset(mbufs[i], uint8_t *,
 425                                 sizeof(struct rte_ether_hdr)), temp_buf, ncb);
 426         }
 427 }
 428
 429 static inline void
 430 verify_data(struct rte_mbuf **mbufs, uint16_t num_pkts)
 431 {
 432         uint16_t i;
 433         for (i = 0; i < num_pkts; ++i) {
 434                 struct rte_mbuf *out = mbufs[i];
 435                 struct rte_mbuf *in = *mbuf_input(out);
 436
 437                 if (memcmp(rte_pktmbuf_mtod_offset(in, uint8_t *,
 438                                 sizeof(struct rte_ether_hdr)),
 439                                 rte_pktmbuf_mtod_offset(out, uint8_t *,
 440                                 sizeof(struct rte_ether_hdr)),
 441                                 K / 8 - CRC_24B_LEN))
 442                         printf("Input and output buffers are not equal!\n");
 443         }
 444 }
 445
 446 static int
 447 initialize_ports(struct app_config_params *app_params,
 448                 struct rte_mempool *ethdev_mbuf_mempool)
 449 {
 450         int ret;
 451         uint16_t port_id = app_params->port_id;
 452         uint16_t q;
 453         /* ethernet addresses of ports */
 454         struct rte_ether_addr bbdev_port_eth_addr;
 455
 456         /* initialize ports */
 457         printf("\nInitializing port %u...\n", app_params->port_id);
 458         ret = rte_eth_dev_configure(port_id, app_params->num_enc_cores,
 459                 app_params->num_dec_cores, &port_conf);
 460
 461         if (ret < 0) {
 462                 printf("Cannot configure device: err=%d, port=%u\n",
 463                         ret, port_id);
 464                 return -1;
 465         }
 466
 467         /* initialize RX queues for encoder */
 468         for (q = 0; q < app_params->num_enc_cores; q++) {
 469                 ret = rte_eth_rx_queue_setup(port_id, q,
 470                         RTE_TEST_RX_DESC_DEFAULT,
 471                         rte_eth_dev_socket_id(port_id),
 472                         NULL, ethdev_mbuf_mempool);
 473                 if (ret < 0) {
 474                         printf("rte_eth_rx_queue_setup: err=%d, queue=%u\n",
 475                                 ret, q);
 476                         return -1;
 477                 }
 478         }
 479         /* initialize TX queues for decoder */
 480         for (q = 0; q < app_params->num_dec_cores; q++) {
 481                 ret = rte_eth_tx_queue_setup(port_id, q,
 482                         RTE_TEST_TX_DESC_DEFAULT,
 483                         rte_eth_dev_socket_id(port_id), NULL);
 484                 if (ret < 0) {
 485                         printf("rte_eth_tx_queue_setup: err=%d, queue=%u\n",
 486                                 ret, q);
 487                         return -1;
 488                 }
 489         }
 490
 491         ret = rte_eth_promiscuous_enable(port_id);
 492         if (ret != 0) {
 493                 printf("Cannot enable promiscuous mode: err=%s, port=%u\n",
 494                         rte_strerror(-ret), port_id);
 495                 return ret;
 496         }
 497
 498         ret = rte_eth_macaddr_get(port_id, &bbdev_port_eth_addr);
 499         if (ret < 0) {
 500                 printf("rte_eth_macaddr_get: err=%d, queue=%u\n",
 501                         ret, q);
 502                 return -1;
 503         }
 504
 505         print_mac(port_id, &bbdev_port_eth_addr);
 506
 507         return 0;
 508 }
 509
 510 static void
 511 lcore_conf_init(struct app_config_params *app_params,
 512                 struct lcore_conf *lcore_conf,
 513                 struct rte_mempool **bbdev_op_pools,
 514                 struct rte_mempool *bbdev_mbuf_mempool,
 515                 struct rte_ring *enc_to_dec_ring,
 516                 struct lcore_statistics *lcore_stats)
 517 {
 518         unsigned int lcore_id;
 519         struct lcore_conf *lconf;
 520         uint16_t rx_queue_id = 0;
 521         uint16_t tx_queue_id = 0;
 522         uint16_t enc_q_id = 0;
 523         uint16_t dec_q_id = 0;
 524
 525         /* Configure lcores */
 526         for (lcore_id = 0; lcore_id < 8 * sizeof(uint64_t); ++lcore_id) {
 527                 lconf = &lcore_conf[lcore_id];
 528                 lconf->core_type = 0;
 529
 530                 if ((1ULL << lcore_id) & app_params->enc_core_mask) {
 531                         lconf->core_type |= (1 << RTE_BBDEV_OP_TURBO_ENC);
 532                         lconf->rx_queue_id = rx_queue_id++;
 533                         lconf->enc_queue_id =
 534                                         app_params->enc_queue_ids[enc_q_id++];
 535                 }
 536
 537                 if ((1ULL << lcore_id) & app_params->dec_core_mask) {
 538                         lconf->core_type |= (1 << RTE_BBDEV_OP_TURBO_DEC);
 539                         lconf->tx_queue_id = tx_queue_id++;
 540                         lconf->dec_queue_id =
 541                                         app_params->dec_queue_ids[dec_q_id++];
 542                 }
 543
 544                 lconf->bbdev_enc_op_pool =
 545                                 bbdev_op_pools[RTE_BBDEV_OP_TURBO_ENC];
 546                 lconf->bbdev_dec_op_pool =
 547                                 bbdev_op_pools[RTE_BBDEV_OP_TURBO_DEC];
 548                 lconf->bbdev_id = app_params->bbdev_id;
 549                 lconf->port_id = app_params->port_id;
 550                 lconf->enc_out_pool = bbdev_mbuf_mempool;
 551                 lconf->enc_to_dec_ring = enc_to_dec_ring;
 552                 lconf->lcore_stats = &lcore_stats[lcore_id];
 553         }
 554 }
 555
 556 static void
 557 print_lcore_stats(struct lcore_statistics *lstats, unsigned int lcore_id)
 558 {
 559         static const char *stats_border = "_______";
 560
 561         printf("\nLcore %d: %s enqueued count:\t\t%u\n",
 562                         lcore_id, stats_border, lstats->enqueued);
 563         printf("Lcore %d: %s dequeued count:\t\t%u\n",
 564                         lcore_id, stats_border, lstats->dequeued);
 565         printf("Lcore %d: %s RX lost packets count:\t\t%u\n",
 566                         lcore_id, stats_border, lstats->rx_lost_packets);
 567         printf("Lcore %d: %s encoder-to-decoder lost count:\t%u\n",
 568                         lcore_id, stats_border,
 569                         lstats->enc_to_dec_lost_packets);
 570         printf("Lcore %d: %s TX lost packets count:\t\t%u\n",
 571                         lcore_id, stats_border, lstats->tx_lost_packets);
 572 }
 573
 574 static void
 575 print_stats(struct stats_lcore_params *stats_lcore)
 576 {
 577         unsigned int l_id;
 578         unsigned int bbdev_id = stats_lcore->app_params->bbdev_id;
 579         unsigned int port_id = stats_lcore->app_params->port_id;
 580         int len, ret, i;
 581
 582         struct rte_eth_xstat *xstats;
 583         struct rte_eth_xstat_name *xstats_names;
 584         struct rte_bbdev_stats bbstats;
 585         static const char *stats_border = "_______";
 586
 587         const char clr[] = { 27, '[', '2', 'J', '\0' };
 588         const char topLeft[] = { 27, '[', '1', ';', '1', 'H', '\0' };
 589
 590         /* Clear screen and move to top left */
 591         printf("%s%s", clr, topLeft);
 592
 593         printf("PORT STATISTICS:\n================\n");
 594         len = rte_eth_xstats_get(port_id, NULL, 0);
 595         if (len < 0)
 596                 rte_exit(EXIT_FAILURE,
 597                                 "rte_eth_xstats_get(%u) failed: %d", port_id,
 598                                 len);
 599
 600         xstats = calloc(len, sizeof(*xstats));
 601         if (xstats == NULL)
 602                 rte_exit(EXIT_FAILURE,
 603                                 "Failed to calloc memory for xstats");
 604
 605         ret = rte_eth_xstats_get(port_id, xstats, len);
 606         if (ret < 0 || ret > len) {
 607                 free(xstats);
 608                 rte_exit(EXIT_FAILURE,
 609                                 "rte_eth_xstats_get(%u) len%i failed: %d",
 610                                 port_id, len, ret);
 611         }
 612
 613         xstats_names = calloc(len, sizeof(*xstats_names));
 614         if (xstats_names == NULL) {
 615                 free(xstats);
 616                 rte_exit(EXIT_FAILURE,
 617                                 "Failed to calloc memory for xstats_names");
 618         }
 619
 620         ret = rte_eth_xstats_get_names(port_id, xstats_names, len);
 621         if (ret < 0 || ret > len) {
 622                 free(xstats);
 623                 free(xstats_names);
 624                 rte_exit(EXIT_FAILURE,
 625                                 "rte_eth_xstats_get_names(%u) len%i failed: %d",
 626                                 port_id, len, ret);
 627         }
 628
 629         for (i = 0; i < len; i++) {
 630                 if (xstats[i].value > 0)
 631                         printf("Port %u: %s %s:\t\t%"PRIu64"\n",
 632                                         port_id, stats_border,
 633                                         xstats_names[i].name,
 634                                         xstats[i].value);
 635         }
 636
 637         ret = rte_bbdev_stats_get(bbdev_id, &bbstats);
 638         if (ret < 0) {
 639                 free(xstats);
 640                 free(xstats_names);
 641                 rte_exit(EXIT_FAILURE,
 642                                 "ERROR(%d): Failure to get BBDEV %u statistics\n",
 643                                 ret, bbdev_id);
 644         }
 645
 646         printf("\nBBDEV STATISTICS:\n=================\n");
 647         printf("BBDEV %u: %s enqueue count:\t\t%"PRIu64"\n",
 648                         bbdev_id, stats_border,
 649                         bbstats.enqueued_count);
 650         printf("BBDEV %u: %s dequeue count:\t\t%"PRIu64"\n",
 651                         bbdev_id, stats_border,
 652                         bbstats.dequeued_count);
 653         printf("BBDEV %u: %s enqueue error count:\t\t%"PRIu64"\n",
 654                         bbdev_id, stats_border,
 655                         bbstats.enqueue_err_count);
 656         printf("BBDEV %u: %s dequeue error count:\t\t%"PRIu64"\n\n",
 657                         bbdev_id, stats_border,
 658                         bbstats.dequeue_err_count);
 659
 660         printf("LCORE STATISTICS:\n=================\n");
 661         for (l_id = 0; l_id < RTE_MAX_LCORE; ++l_id) {
 662                 if (stats_lcore->lconf[l_id].core_type == 0)
 663                         continue;
 664                 print_lcore_stats(stats_lcore->lconf[l_id].lcore_stats, l_id);
 665         }
 666
 667         fflush(stdout);
 668
 669         free(xstats);
 670         free(xstats_names);
 671 }
 672
 673 static int
 674 stats_loop(void *arg)
 675 {
 676         struct stats_lcore_params *stats_lcore = arg;
 677
 678         while (!__atomic_load_n(&global_exit_flag, __ATOMIC_RELAXED)) {
 679                 print_stats(stats_lcore);
 680                 rte_delay_ms(500);
 681         }
 682
 683         return 0;
 684 }
 685
 686 static inline void
 687 run_encoding(struct lcore_conf *lcore_conf)
 688 {
 689         uint16_t i;
 690         uint16_t port_id, rx_queue_id;
 691         uint16_t bbdev_id, enc_queue_id;
 692         uint16_t nb_rx, nb_enq, nb_deq, nb_sent;
 693         struct rte_mbuf *rx_pkts_burst[MAX_PKT_BURST];
 694         struct rte_mbuf *enc_out_pkts[MAX_PKT_BURST];
 695         struct rte_bbdev_enc_op *bbdev_ops_burst[MAX_PKT_BURST];
 696         struct lcore_statistics *lcore_stats;
 697         struct rte_mempool *bbdev_op_pool, *enc_out_pool;
 698         struct rte_ring *enc_to_dec_ring;
 699         const int in_data_len = (def_op_enc.cb_params.k / 8) - CRC_24B_LEN;
 700
 701         lcore_stats = lcore_conf->lcore_stats;
 702         port_id = lcore_conf->port_id;
 703         rx_queue_id = lcore_conf->rx_queue_id;
 704         bbdev_id = lcore_conf->bbdev_id;
 705         enc_queue_id = lcore_conf->enc_queue_id;
 706         bbdev_op_pool = lcore_conf->bbdev_enc_op_pool;
 707         enc_out_pool = lcore_conf->enc_out_pool;
 708         enc_to_dec_ring = lcore_conf->enc_to_dec_ring;
 709
 710         /* Read packet from RX queues*/
 711         nb_rx = rte_eth_rx_burst(port_id, rx_queue_id, rx_pkts_burst,
 712                         MAX_PKT_BURST);
 713         if (!nb_rx)
 714                 return;
 715
 716         if (unlikely(rte_mempool_get_bulk(enc_out_pool, (void **)enc_out_pkts,
 717                         nb_rx) != 0)) {
 718                 pktmbuf_free_bulk(rx_pkts_burst, nb_rx);
 719                 lcore_stats->rx_lost_packets += nb_rx;
 720                 return;
 721         }
 722
 723         if (unlikely(rte_bbdev_enc_op_alloc_bulk(bbdev_op_pool, bbdev_ops_burst,
 724                         nb_rx) != 0)) {
 725                 pktmbuf_free_bulk(enc_out_pkts, nb_rx);
 726                 pktmbuf_free_bulk(rx_pkts_burst, nb_rx);
 727                 lcore_stats->rx_lost_packets += nb_rx;
 728                 return;
 729         }
 730
 731         for (i = 0; i < nb_rx; i++) {
 732                 char *data;
 733                 const uint16_t pkt_data_len =
 734                                 rte_pktmbuf_data_len(rx_pkts_burst[i]) -
 735                                 sizeof(struct rte_ether_hdr);
 736                 /* save input mbuf pointer for later comparison */
 737                 *mbuf_input(enc_out_pkts[i]) = rx_pkts_burst[i];
 738
 739                 /* copy ethernet header */
 740                 rte_pktmbuf_reset(enc_out_pkts[i]);
 741                 data = rte_pktmbuf_append(enc_out_pkts[i],
 742                                 sizeof(struct rte_ether_hdr));
 743                 if (data == NULL) {
 744                         printf(
 745                                 "Not enough space for ethernet header in encoder output mbuf\n");
 746                         continue;
 747                 }
 748                 add_ether_hdr(rx_pkts_burst[i], enc_out_pkts[i]);
 749
 750                 /* set op */
 751                 bbdev_ops_burst[i]->turbo_enc = def_op_enc;
 752
 753                 bbdev_ops_burst[i]->turbo_enc.input.data =
 754                                 rx_pkts_burst[i];
 755                 bbdev_ops_burst[i]->turbo_enc.input.offset =
 756                                 sizeof(struct rte_ether_hdr);
 757                 /* Encoder will attach the CRC24B, adjust the length */
 758                 bbdev_ops_burst[i]->turbo_enc.input.length = in_data_len;
 759
 760                 if (in_data_len < pkt_data_len)
 761                         rte_pktmbuf_trim(rx_pkts_burst[i], pkt_data_len -
 762                                         in_data_len);
 763                 else if (in_data_len > pkt_data_len) {
 764                         data = rte_pktmbuf_append(rx_pkts_burst[i],
 765                                         in_data_len - pkt_data_len);
 766                         if (data == NULL)
 767                                 printf(
 768                                         "Not enough storage in mbuf to perform the encoding\n");
 769                 }
 770
 771                 bbdev_ops_burst[i]->turbo_enc.output.data =
 772                                 enc_out_pkts[i];
 773                 bbdev_ops_burst[i]->turbo_enc.output.offset =
 774                                 sizeof(struct rte_ether_hdr);
 775         }
 776
 777         /* Enqueue packets on BBDevice */
 778         nb_enq = rte_bbdev_enqueue_enc_ops(bbdev_id, enc_queue_id,
 779                         bbdev_ops_burst, nb_rx);
 780         if (unlikely(nb_enq < nb_rx)) {
 781                 pktmbuf_input_free_bulk(&enc_out_pkts[nb_enq],
 782                                 nb_rx - nb_enq);
 783                 rte_bbdev_enc_op_free_bulk(&bbdev_ops_burst[nb_enq],
 784                                 nb_rx - nb_enq);
 785                 lcore_stats->rx_lost_packets += nb_rx - nb_enq;
 786
 787                 if (!nb_enq)
 788                         return;
 789         }
 790
 791         lcore_stats->enqueued += nb_enq;
 792
 793         /* Dequeue packets from bbdev device*/
 794         nb_deq = 0;
 795         do {
 796                 nb_deq += rte_bbdev_dequeue_enc_ops(bbdev_id, enc_queue_id,
 797                                 &bbdev_ops_burst[nb_deq], nb_enq - nb_deq);
 798         } while (unlikely(nb_deq < nb_enq));
 799
 800         lcore_stats->dequeued += nb_deq;
 801
 802         /* Generate and add AWGN */
 803         add_awgn(enc_out_pkts, nb_deq);
 804
 805         rte_bbdev_enc_op_free_bulk(bbdev_ops_burst, nb_deq);
 806
 807         /* Enqueue packets to encoder-to-decoder ring */
 808         nb_sent = rte_ring_enqueue_burst(enc_to_dec_ring, (void **)enc_out_pkts,
 809                         nb_deq, NULL);
 810         if (unlikely(nb_sent < nb_deq)) {
 811                 pktmbuf_input_free_bulk(&enc_out_pkts[nb_sent],
 812                                 nb_deq - nb_sent);
 813                 lcore_stats->enc_to_dec_lost_packets += nb_deq - nb_sent;
 814         }
 815 }
 816
 817 static void
 818 run_decoding(struct lcore_conf *lcore_conf)
 819 {
 820         uint16_t i;
 821         uint16_t port_id, tx_queue_id;
 822         uint16_t bbdev_id, bbdev_queue_id;
 823         uint16_t nb_recv, nb_enq, nb_deq, nb_tx;
 824         uint8_t *llr_temp_buf;
 825         struct rte_mbuf *recv_pkts_burst[MAX_PKT_BURST];
 826         struct rte_bbdev_dec_op *bbdev_ops_burst[MAX_PKT_BURST];
 827         struct lcore_statistics *lcore_stats;
 828         struct rte_mempool *bbdev_op_pool;
 829         struct rte_ring *enc_to_dec_ring;
 830
 831         lcore_stats = lcore_conf->lcore_stats;
 832         port_id = lcore_conf->port_id;
 833         tx_queue_id = lcore_conf->tx_queue_id;
 834         bbdev_id = lcore_conf->bbdev_id;
 835         bbdev_queue_id = lcore_conf->dec_queue_id;
 836         bbdev_op_pool = lcore_conf->bbdev_dec_op_pool;
 837         enc_to_dec_ring = lcore_conf->enc_to_dec_ring;
 838         llr_temp_buf = lcore_conf->llr_temp_buf;
 839
 840         /* Dequeue packets from the ring */
 841         nb_recv = rte_ring_dequeue_burst(enc_to_dec_ring,
 842                         (void **)recv_pkts_burst, MAX_PKT_BURST, NULL);
 843         if (!nb_recv)
 844                 return;
 845
 846         if (unlikely(rte_bbdev_dec_op_alloc_bulk(bbdev_op_pool, bbdev_ops_burst,
 847                         nb_recv) != 0)) {
 848                 pktmbuf_input_free_bulk(recv_pkts_burst, nb_recv);
 849                 lcore_stats->rx_lost_packets += nb_recv;
 850                 return;
 851         }
 852
 853         transform_enc_out_dec_in(recv_pkts_burst, llr_temp_buf, nb_recv,
 854                         def_op_dec.cb_params.k);
 855
 856         for (i = 0; i < nb_recv; i++) {
 857                 /* set op */
 858                 bbdev_ops_burst[i]->turbo_dec = def_op_dec;
 859
 860                 bbdev_ops_burst[i]->turbo_dec.input.data = recv_pkts_burst[i];
 861                 bbdev_ops_burst[i]->turbo_dec.input.offset =
 862                                 sizeof(struct rte_ether_hdr);
 863                 bbdev_ops_burst[i]->turbo_dec.input.length =
 864                                 rte_pktmbuf_data_len(recv_pkts_burst[i])
 865                                 - sizeof(struct rte_ether_hdr);
 866
 867                 bbdev_ops_burst[i]->turbo_dec.hard_output.data =
 868                                 recv_pkts_burst[i];
 869                 bbdev_ops_burst[i]->turbo_dec.hard_output.offset =
 870                                 sizeof(struct rte_ether_hdr);
 871         }
 872
 873         /* Enqueue packets on BBDevice */
 874         nb_enq = rte_bbdev_enqueue_dec_ops(bbdev_id, bbdev_queue_id,
 875                         bbdev_ops_burst, nb_recv);
 876         if (unlikely(nb_enq < nb_recv)) {
 877                 pktmbuf_input_free_bulk(&recv_pkts_burst[nb_enq],
 878                                 nb_recv - nb_enq);
 879                 rte_bbdev_dec_op_free_bulk(&bbdev_ops_burst[nb_enq],
 880                                 nb_recv - nb_enq);
 881                 lcore_stats->rx_lost_packets += nb_recv - nb_enq;
 882
 883                 if (!nb_enq)
 884                         return;
 885         }
 886
 887         lcore_stats->enqueued += nb_enq;
 888
 889         /* Dequeue packets from BBDevice */
 890         nb_deq = 0;
 891         do {
 892                 nb_deq += rte_bbdev_dequeue_dec_ops(bbdev_id, bbdev_queue_id,
 893                                 &bbdev_ops_burst[nb_deq], nb_enq - nb_deq);
 894         } while (unlikely(nb_deq < nb_enq));
 895
 896         lcore_stats->dequeued += nb_deq;
 897
 898         rte_bbdev_dec_op_free_bulk(bbdev_ops_burst, nb_deq);
 899
 900         verify_data(recv_pkts_burst, nb_deq);
 901
 902         /* Free the RX mbufs after verification */
 903         for (i = 0; i < nb_deq; ++i)
 904                 rte_pktmbuf_free(*mbuf_input(recv_pkts_burst[i]));
 905
 906         /* Transmit the packets */
 907         nb_tx = rte_eth_tx_burst(port_id, tx_queue_id, recv_pkts_burst, nb_deq);
 908         if (unlikely(nb_tx < nb_deq)) {
 909                 pktmbuf_input_free_bulk(&recv_pkts_burst[nb_tx],
 910                                 nb_deq - nb_tx);
 911                 lcore_stats->tx_lost_packets += nb_deq - nb_tx;
 912         }
 913 }
 914
 915 static int
 916 processing_loop(void *arg)
 917 {
 918         struct lcore_conf *lcore_conf = arg;
 919         const bool run_encoder = (lcore_conf->core_type &
 920                         (1 << RTE_BBDEV_OP_TURBO_ENC));
 921         const bool run_decoder = (lcore_conf->core_type &
 922                         (1 << RTE_BBDEV_OP_TURBO_DEC));
 923
 924         while (!__atomic_load_n(&global_exit_flag, __ATOMIC_RELAXED)) {
 925                 if (run_encoder)
 926                         run_encoding(lcore_conf);
 927                 if (run_decoder)
 928                         run_decoding(lcore_conf);
 929         }
 930
 931         return 0;
 932 }
 933
 934 static int
 935 prepare_bbdev_device(unsigned int dev_id, struct rte_bbdev_info *info,
 936                 struct app_config_params *app_params)
 937 {
 938         int ret;
 939         unsigned int q_id, dec_q_id, enc_q_id;
 940         struct rte_bbdev_queue_conf qconf = {0};
 941         uint16_t dec_qs_nb = app_params->num_dec_cores;
 942         uint16_t enc_qs_nb = app_params->num_enc_cores;
 943         uint16_t tot_qs = dec_qs_nb + enc_qs_nb;
 944
 945         ret = rte_bbdev_setup_queues(dev_id, tot_qs, info->socket_id);
 946         if (ret < 0)
 947                 rte_exit(EXIT_FAILURE,
 948                                 "ERROR(%d): BBDEV %u not configured properly\n",
 949                                 ret, dev_id);
 950
 951         /* setup device DEC queues */
 952         qconf.socket = info->socket_id;
 953         qconf.queue_size = info->drv.queue_size_lim;
 954         qconf.op_type = RTE_BBDEV_OP_TURBO_DEC;
 955
 956         for (q_id = 0, dec_q_id = 0; q_id < dec_qs_nb; q_id++) {
 957                 ret = rte_bbdev_queue_configure(dev_id, q_id, &qconf);
 958                 if (ret < 0)
 959                         rte_exit(EXIT_FAILURE,
 960                                         "ERROR(%d): BBDEV %u DEC queue %u not configured properly\n",
 961                                         ret, dev_id, q_id);
 962                 app_params->dec_queue_ids[dec_q_id++] = q_id;
 963         }
 964
 965         /* setup device ENC queues */
 966         qconf.op_type = RTE_BBDEV_OP_TURBO_ENC;
 967
 968         for (q_id = dec_qs_nb, enc_q_id = 0; q_id < tot_qs; q_id++) {
 969                 ret = rte_bbdev_queue_configure(dev_id, q_id, &qconf);
 970                 if (ret < 0)
 971                         rte_exit(EXIT_FAILURE,
 972                                         "ERROR(%d): BBDEV %u ENC queue %u not configured properly\n",
 973                                         ret, dev_id, q_id);
 974                 app_params->enc_queue_ids[enc_q_id++] = q_id;
 975         }
 976
 977         ret = rte_bbdev_start(dev_id);
 978
 979         if (ret != 0)
 980                 rte_exit(EXIT_FAILURE, "ERROR(%d): BBDEV %u not started\n",
 981                         ret, dev_id);
 982
 983         printf("BBdev %u started\n", dev_id);
 984
 985         return 0;
 986 }
 987
 988 static inline bool
 989 check_matching_capabilities(uint64_t mask, uint64_t required_mask)
 990 {
 991         return (mask & required_mask) == required_mask;
 992 }
 993
 994 static void
 995 enable_bbdev(struct app_config_params *app_params)
 996 {
 997         struct rte_bbdev_info dev_info;
 998         const struct rte_bbdev_op_cap *op_cap;
 999         uint16_t bbdev_id = app_params->bbdev_id;
1000         bool encoder_capable = false;
1001         bool decoder_capable = false;
1002
1003         rte_bbdev_info_get(bbdev_id, &dev_info);
1004         op_cap = dev_info.drv.capabilities;
1005
1006         while (op_cap->type != RTE_BBDEV_OP_NONE) {
1007                 if (op_cap->type == RTE_BBDEV_OP_TURBO_ENC) {
1008                         if (check_matching_capabilities(
1009                                         op_cap->cap.turbo_enc.capability_flags,
1010                                         def_op_enc.op_flags))
1011                                 encoder_capable = true;
1012                 }
1013
1014                 if (op_cap->type == RTE_BBDEV_OP_TURBO_DEC) {
1015                         if (check_matching_capabilities(
1016                                         op_cap->cap.turbo_dec.capability_flags,
1017                                         def_op_dec.op_flags))
1018                                 decoder_capable = true;
1019                 }
1020
1021                 op_cap++;
1022         }
1023
1024         if (encoder_capable == false)
1025                 rte_exit(EXIT_FAILURE,
1026                         "The specified BBDev %u doesn't have required encoder capabilities!\n",
1027                         bbdev_id);
1028         if (decoder_capable == false)
1029                 rte_exit(EXIT_FAILURE,
1030                         "The specified BBDev %u doesn't have required decoder capabilities!\n",
1031                         bbdev_id);
1032
1033         prepare_bbdev_device(bbdev_id, &dev_info, app_params);
1034 }
1035
1036 int
1037 main(int argc, char **argv)
1038 {
1039         int ret;
1040         unsigned int nb_bbdevs, flags, lcore_id;
1041         void *sigret;
1042         struct app_config_params app_params = def_app_config;
1043         struct rte_mempool *ethdev_mbuf_mempool, *bbdev_mbuf_mempool;
1044         struct rte_mempool *bbdev_op_pools[RTE_BBDEV_OP_TYPE_COUNT];
1045         struct lcore_conf lcore_conf[RTE_MAX_LCORE] = { {0} };
1046         struct lcore_statistics lcore_stats[RTE_MAX_LCORE] = { {0} };
1047         struct stats_lcore_params stats_lcore;
1048         struct rte_ring *enc_to_dec_ring;
1049         bool stats_thread_started = false;
1050         unsigned int main_lcore_id = rte_get_main_lcore();
1051
1052         static const struct rte_mbuf_dynfield input_dynfield_desc = {
1053                 .name = "example_bbdev_dynfield_input",
1054                 .size = sizeof(struct rte_mbuf *),
1055                 .align = __alignof__(struct rte_mbuf *),
1056         };
1057
1058         __atomic_store_n(&global_exit_flag, 0, __ATOMIC_RELAXED);
1059
1060         sigret = signal(SIGTERM, signal_handler);
1061         if (sigret == SIG_ERR)
1062                 rte_exit(EXIT_FAILURE, "signal(%d, ...) failed", SIGTERM);
1063
1064         sigret = signal(SIGINT, signal_handler);
1065         if (sigret == SIG_ERR)
1066                 rte_exit(EXIT_FAILURE, "signal(%d, ...) failed", SIGINT);
1067
1068         ret = rte_eal_init(argc, argv);
1069         if (ret < 0)
1070                 rte_exit(EXIT_FAILURE, "Invalid EAL arguments\n");
1071
1072         argc -= ret;
1073         argv += ret;
1074
1075         /* parse application arguments (after the EAL ones) */
1076         ret = bbdev_parse_args(argc, argv, &app_params);
1077         if (ret < 0)
1078                 rte_exit(EXIT_FAILURE, "Invalid BBDEV arguments\n");
1079
1080         /*create bbdev op pools*/
1081         bbdev_op_pools[RTE_BBDEV_OP_TURBO_DEC] =
1082                         rte_bbdev_op_pool_create("bbdev_op_pool_dec",
1083                         RTE_BBDEV_OP_TURBO_DEC, NB_MBUF, 128, rte_socket_id());
1084         bbdev_op_pools[RTE_BBDEV_OP_TURBO_ENC] =
1085                         rte_bbdev_op_pool_create("bbdev_op_pool_enc",
1086                         RTE_BBDEV_OP_TURBO_ENC, NB_MBUF, 128, rte_socket_id());
1087
1088         if ((bbdev_op_pools[RTE_BBDEV_OP_TURBO_DEC] == NULL) ||
1089                         (bbdev_op_pools[RTE_BBDEV_OP_TURBO_ENC] == NULL))
1090                 rte_exit(EXIT_FAILURE, "Cannot create bbdev op pools\n");
1091
1092         /* Create encoder to decoder ring */
1093         flags = (app_params.num_enc_cores == 1) ? RING_F_SP_ENQ : 0;
1094         if (app_params.num_dec_cores == 1)
1095                 flags |= RING_F_SC_DEQ;
1096
1097         enc_to_dec_ring = rte_ring_create("enc_to_dec_ring",
1098                 rte_align32pow2(NB_MBUF), rte_socket_id(), flags);
1099
1100         /* Get the number of available bbdev devices */
1101         nb_bbdevs = rte_bbdev_count();
1102         if (nb_bbdevs <= app_params.bbdev_id)
1103                 rte_exit(EXIT_FAILURE,
1104                                 "%u BBDevs detected, cannot use BBDev with ID %u!\n",
1105                                 nb_bbdevs, app_params.bbdev_id);
1106         printf("Number of bbdevs detected: %d\n", nb_bbdevs);
1107
1108         if (!rte_eth_dev_is_valid_port(app_params.port_id))
1109                 rte_exit(EXIT_FAILURE,
1110                                 "cannot use port with ID %u!\n",
1111                                 app_params.port_id);
1112
1113         /* create the mbuf mempool for ethdev pkts */
1114         ethdev_mbuf_mempool = rte_pktmbuf_pool_create("ethdev_mbuf_pool",
1115                         NB_MBUF, MEMPOOL_CACHE_SIZE, 0,
1116                         RTE_MBUF_DEFAULT_BUF_SIZE, rte_socket_id());
1117         if (ethdev_mbuf_mempool == NULL)
1118                 rte_exit(EXIT_FAILURE, "Cannot create ethdev mbuf mempool\n");
1119
1120         /* create the mbuf mempool for encoder output */
1121         bbdev_mbuf_mempool = rte_pktmbuf_pool_create("bbdev_mbuf_pool",
1122                         NB_MBUF, MEMPOOL_CACHE_SIZE, 0,
1123                         RTE_MBUF_DEFAULT_BUF_SIZE, rte_socket_id());
1124         if (bbdev_mbuf_mempool == NULL)
1125                 rte_exit(EXIT_FAILURE, "Cannot create ethdev mbuf mempool\n");
1126
1127         /* register mbuf field to store input pointer */
1128         input_dynfield_offset =
1129                 rte_mbuf_dynfield_register(&input_dynfield_desc);
1130         if (input_dynfield_offset < 0)
1131                 rte_exit(EXIT_FAILURE, "Cannot register mbuf field\n");
1132
1133         /* initialize ports */
1134         ret = initialize_ports(&app_params, ethdev_mbuf_mempool);
1135
1136         /* Check if all requested lcores are available */
1137         for (lcore_id = 0; lcore_id < 8 * sizeof(uint64_t); ++lcore_id)
1138                 if (((1ULL << lcore_id) & app_params.enc_core_mask) ||
1139                                 ((1ULL << lcore_id) & app_params.dec_core_mask))
1140                         if (!rte_lcore_is_enabled(lcore_id))
1141                                 rte_exit(EXIT_FAILURE,
1142                                                 "Requested lcore_id %u is not enabled!\n",
1143                                                 lcore_id);
1144
1145         /* Start ethernet port */
1146         ret = rte_eth_dev_start(app_params.port_id);
1147         if (ret < 0)
1148                 rte_exit(EXIT_FAILURE, "rte_eth_dev_start:err=%d, port=%u\n",
1149                                 ret, app_params.port_id);
1150
1151         ret = check_port_link_status(app_params.port_id);
1152         if (ret < 0)
1153                 exit(EXIT_FAILURE);
1154
1155         /* start BBDevice and save BBDev queue IDs */
1156         enable_bbdev(&app_params);
1157
1158         /* Initialize the port/queue configuration of each logical core */
1159         lcore_conf_init(&app_params, lcore_conf, bbdev_op_pools,
1160                         bbdev_mbuf_mempool, enc_to_dec_ring, lcore_stats);
1161
1162         stats_lcore.app_params = &app_params;
1163         stats_lcore.lconf = lcore_conf;
1164
1165         RTE_LCORE_FOREACH_WORKER(lcore_id) {
1166                 if (lcore_conf[lcore_id].core_type != 0)
1167                         /* launch per-lcore processing loop on worker lcores */
1168                         rte_eal_remote_launch(processing_loop,
1169                                         &lcore_conf[lcore_id], lcore_id);
1170                 else if (!stats_thread_started) {
1171                         /* launch statistics printing loop */
1172                         rte_eal_remote_launch(stats_loop, &stats_lcore,
1173                                         lcore_id);
1174                         stats_thread_started = true;
1175                 }
1176         }
1177
1178         if (!stats_thread_started &&
1179                         lcore_conf[main_lcore_id].core_type != 0)
1180                 rte_exit(EXIT_FAILURE,
1181                                 "Not enough lcores to run the statistics printing loop!");
1182         else if (lcore_conf[main_lcore_id].core_type != 0)
1183                 processing_loop(&lcore_conf[main_lcore_id]);
1184         else if (!stats_thread_started)
1185                 stats_loop(&stats_lcore);
1186
1187         RTE_LCORE_FOREACH_WORKER(lcore_id) {
1188                 ret |= rte_eal_wait_lcore(lcore_id);
1189         }
1190
1191         /* clean up the EAL */
1192         rte_eal_cleanup();
1193
1194         return ret;
1195 }