examples/bbdev_app/main.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(c) 2017 Intel Corporation
   3  */
   4
   5 #include <stdio.h>
   6 #include <stdlib.h>
   7 #include <string.h>
   8 #include <stdint.h>
   9 #include <inttypes.h>
  10 #include <sys/types.h>
  11 #include <sys/unistd.h>
  12 #include <sys/queue.h>
  13 #include <stdarg.h>
  14 #include <ctype.h>
  15 #include <errno.h>
  16 #include <math.h>
  17 #include <assert.h>
  18 #include <getopt.h>
  19 #include <signal.h>
  20
  21 #include "rte_atomic.h"
  22 #include "rte_common.h"
  23 #include "rte_eal.h"
  24 #include "rte_cycles.h"
  25 #include "rte_ether.h"
  26 #include "rte_ethdev.h"
  27 #include "rte_ip.h"
  28 #include "rte_lcore.h"
  29 #include "rte_malloc.h"
  30 #include "rte_mbuf.h"
  31 #include "rte_memory.h"
  32 #include "rte_mempool.h"
  33 #include "rte_log.h"
  34 #include "rte_bbdev.h"
  35 #include "rte_bbdev_op.h"
  36
  37 /* LLR values - negative value for '1' bit */
  38 #define LLR_1_BIT 0x81
  39 #define LLR_0_BIT 0x7F
  40
  41 #define MAX_PKT_BURST 32
  42 #define NB_MBUF 8191
  43 #define MEMPOOL_CACHE_SIZE 256
  44
  45 /* Hardcoded K value */
  46 #define K 40
  47 #define NCB (3 * RTE_ALIGN_CEIL(K + 4, 32))
  48
  49 #define CRC_24B_LEN 3
  50
  51 /* Configurable number of RX/TX ring descriptors */
  52 #define RTE_TEST_RX_DESC_DEFAULT 128
  53 #define RTE_TEST_TX_DESC_DEFAULT 512
  54
  55 #define BBDEV_ASSERT(a) do { \
  56         if (!(a)) { \
  57                 usage(prgname); \
  58                 return -1; \
  59         } \
  60 } while (0)
  61
  62 static const struct rte_eth_conf port_conf = {
  63         .rxmode = {
  64                 .mq_mode = ETH_MQ_RX_NONE,
  65                 .max_rx_pkt_len = RTE_ETHER_MAX_LEN,
  66                 .split_hdr_size = 0,
  67         },
  68         .txmode = {
  69                 .mq_mode = ETH_MQ_TX_NONE,
  70         },
  71 };
  72
  73 struct rte_bbdev_op_turbo_enc def_op_enc = {
  74         /* These values are arbitrarily put, and does not map to the real
  75          * values for the data received from ethdev ports
  76          */
  77         .rv_index = 0,
  78         .code_block_mode = 1,
  79         .cb_params = {
  80                 .k = K,
  81         },
  82         .op_flags = RTE_BBDEV_TURBO_CRC_24A_ATTACH
  83 };
  84
  85 struct rte_bbdev_op_turbo_dec def_op_dec = {
  86         /* These values are arbitrarily put, and does not map to the real
  87          * values for the data received from ethdev ports
  88          */
  89         .code_block_mode = 1,
  90         .cb_params = {
  91                 .k = K,
  92         },
  93         .rv_index = 0,
  94         .iter_max = 8,
  95         .iter_min = 4,
  96         .ext_scale = 15,
  97         .num_maps = 0,
  98         .op_flags = RTE_BBDEV_TURBO_NEG_LLR_1_BIT_IN
  99 };
 100
 101 struct app_config_params {
 102         /* Placeholders for app params */
 103         uint16_t port_id;
 104         uint16_t bbdev_id;
 105         uint64_t enc_core_mask;
 106         uint64_t dec_core_mask;
 107
 108         /* Values filled during init time */
 109         uint16_t enc_queue_ids[RTE_MAX_LCORE];
 110         uint16_t dec_queue_ids[RTE_MAX_LCORE];
 111         uint16_t num_enc_cores;
 112         uint16_t num_dec_cores;
 113 };
 114
 115 struct lcore_statistics {
 116         unsigned int enqueued;
 117         unsigned int dequeued;
 118         unsigned int rx_lost_packets;
 119         unsigned int enc_to_dec_lost_packets;
 120         unsigned int tx_lost_packets;
 121 } __rte_cache_aligned;
 122
 123 /** each lcore configuration */
 124 struct lcore_conf {
 125         uint64_t core_type;
 126
 127         unsigned int port_id;
 128         unsigned int rx_queue_id;
 129         unsigned int tx_queue_id;
 130
 131         unsigned int bbdev_id;
 132         unsigned int enc_queue_id;
 133         unsigned int dec_queue_id;
 134
 135         uint8_t llr_temp_buf[NCB];
 136
 137         struct rte_mempool *bbdev_dec_op_pool;
 138         struct rte_mempool *bbdev_enc_op_pool;
 139         struct rte_mempool *enc_out_pool;
 140         struct rte_ring *enc_to_dec_ring;
 141
 142         struct lcore_statistics *lcore_stats;
 143 } __rte_cache_aligned;
 144
 145 struct stats_lcore_params {
 146         struct lcore_conf *lconf;
 147         struct app_config_params *app_params;
 148 };
 149
 150
 151 static const struct app_config_params def_app_config = {
 152         .port_id = 0,
 153         .bbdev_id = 0,
 154         .enc_core_mask = 0x2,
 155         .dec_core_mask = 0x4,
 156         .num_enc_cores = 1,
 157         .num_dec_cores = 1,
 158 };
 159
 160 static rte_atomic16_t global_exit_flag;
 161
 162 /* display usage */
 163 static inline void
 164 usage(const char *prgname)
 165 {
 166         printf("%s [EAL options] "
 167                         "  --\n"
 168                         "  --enc_cores - number of encoding cores (default = 0x2)\n"
 169                         "  --dec_cores - number of decoding cores (default = 0x4)\n"
 170                         "  --port_id - Ethernet port ID (default = 0)\n"
 171                         "  --bbdev_id - BBDev ID (default = 0)\n"
 172                         "\n", prgname);
 173 }
 174
 175 /* parse core mask */
 176 static inline
 177 uint16_t bbdev_parse_mask(const char *mask)
 178 {
 179         char *end = NULL;
 180         unsigned long pm;
 181
 182         /* parse hexadecimal string */
 183         pm = strtoul(mask, &end, 16);
 184         if ((mask[0] == '\0') || (end == NULL) || (*end != '\0'))
 185                 return 0;
 186
 187         return pm;
 188 }
 189
 190 /* parse core mask */
 191 static inline
 192 uint16_t bbdev_parse_number(const char *mask)
 193 {
 194         char *end = NULL;
 195         unsigned long pm;
 196
 197         /* parse hexadecimal string */
 198         pm = strtoul(mask, &end, 10);
 199         if ((mask[0] == '\0') || (end == NULL) || (*end != '\0'))
 200                 return 0;
 201
 202         return pm;
 203 }
 204
 205 static int
 206 bbdev_parse_args(int argc, char **argv,
 207                 struct app_config_params *app_params)
 208 {
 209         int optind = 0;
 210         int opt;
 211         int opt_indx = 0;
 212         char *prgname = argv[0];
 213
 214         static struct option lgopts[] = {
 215                 { "enc_core_mask", required_argument, 0, 'e' },
 216                 { "dec_core_mask", required_argument, 0, 'd' },
 217                 { "port_id", required_argument, 0, 'p' },
 218                 { "bbdev_id", required_argument, 0, 'b' },
 219                 { NULL, 0, 0, 0 }
 220         };
 221
 222         BBDEV_ASSERT(argc != 0);
 223         BBDEV_ASSERT(argv != NULL);
 224         BBDEV_ASSERT(app_params != NULL);
 225
 226         while ((opt = getopt_long(argc, argv, "e:d:p:b:", lgopts, &opt_indx)) !=
 227                 EOF) {
 228                 switch (opt) {
 229                 case 'e':
 230                         app_params->enc_core_mask =
 231                                 bbdev_parse_mask(optarg);
 232                         if (app_params->enc_core_mask == 0) {
 233                                 usage(prgname);
 234                                 return -1;
 235                         }
 236                         app_params->num_enc_cores =
 237                                 __builtin_popcount(app_params->enc_core_mask);
 238                         break;
 239
 240                 case 'd':
 241                         app_params->dec_core_mask =
 242                                 bbdev_parse_mask(optarg);
 243                         if (app_params->dec_core_mask == 0) {
 244                                 usage(prgname);
 245                                 return -1;
 246                         }
 247                         app_params->num_dec_cores =
 248                                 __builtin_popcount(app_params->dec_core_mask);
 249                         break;
 250
 251                 case 'p':
 252                         app_params->port_id = bbdev_parse_number(optarg);
 253                         break;
 254
 255                 case 'b':
 256                         app_params->bbdev_id = bbdev_parse_number(optarg);
 257                         break;
 258
 259                 default:
 260                         usage(prgname);
 261                         return -1;
 262                 }
 263         }
 264         optind = 0;
 265         return optind;
 266 }
 267
 268 static void
 269 signal_handler(int signum)
 270 {
 271         printf("\nSignal %d received\n", signum);
 272         rte_atomic16_set(&global_exit_flag, 1);
 273 }
 274
 275 static void
 276 print_mac(unsigned int portid, struct rte_ether_addr *bbdev_ports_eth_address)
 277 {
 278         printf("Port %u, MAC address: %02X:%02X:%02X:%02X:%02X:%02X\n\n",
 279                         (unsigned int) portid,
 280                         bbdev_ports_eth_address->addr_bytes[0],
 281                         bbdev_ports_eth_address->addr_bytes[1],
 282                         bbdev_ports_eth_address->addr_bytes[2],
 283                         bbdev_ports_eth_address->addr_bytes[3],
 284                         bbdev_ports_eth_address->addr_bytes[4],
 285                         bbdev_ports_eth_address->addr_bytes[5]);
 286 }
 287
 288 static inline void
 289 pktmbuf_free_bulk(struct rte_mbuf **mbufs, unsigned int nb_to_free)
 290 {
 291         unsigned int i;
 292         for (i = 0; i < nb_to_free; ++i)
 293                 rte_pktmbuf_free(mbufs[i]);
 294 }
 295
 296 static inline void
 297 pktmbuf_userdata_free_bulk(struct rte_mbuf **mbufs, unsigned int nb_to_free)
 298 {
 299         unsigned int i;
 300         for (i = 0; i < nb_to_free; ++i) {
 301                 struct rte_mbuf *rx_pkt = mbufs[i]->userdata;
 302                 rte_pktmbuf_free(rx_pkt);
 303                 rte_pktmbuf_free(mbufs[i]);
 304         }
 305 }
 306
 307 /* Check the link status of all ports in up to 9s, and print them finally */
 308 static int
 309 check_port_link_status(uint16_t port_id)
 310 {
 311 #define CHECK_INTERVAL 100 /* 100ms */
 312 #define MAX_CHECK_TIME 90 /* 9s (90 * 100ms) in total */
 313         uint8_t count;
 314         struct rte_eth_link link;
 315         int link_get_err = -EINVAL;
 316
 317         printf("\nChecking link status.");
 318         fflush(stdout);
 319
 320         for (count = 0; count <= MAX_CHECK_TIME &&
 321                         !rte_atomic16_read(&global_exit_flag); count++) {
 322                 memset(&link, 0, sizeof(link));
 323                 link_get_err = rte_eth_link_get_nowait(port_id, &link);
 324
 325                 if (link_get_err >= 0 && link.link_status) {
 326                         const char *dp = (link.link_duplex ==
 327                                 ETH_LINK_FULL_DUPLEX) ?
 328                                 "full-duplex" : "half-duplex";
 329                         printf("\nPort %u Link Up - speed %s - %s\n",
 330                                 port_id,
 331                                 rte_eth_link_speed_to_str(link.link_speed),
 332                                 dp);
 333                         return 0;
 334                 }
 335                 printf(".");
 336                 fflush(stdout);
 337                 rte_delay_ms(CHECK_INTERVAL);
 338         }
 339
 340         if (link_get_err >= 0)
 341                 printf("\nPort %d Link Down\n", port_id);
 342         else
 343                 printf("\nGet link failed (port %d): %s\n", port_id,
 344                        rte_strerror(-link_get_err));
 345
 346         return 0;
 347 }
 348
 349 static inline void
 350 add_ether_hdr(struct rte_mbuf *pkt_src, struct rte_mbuf *pkt_dst)
 351 {
 352         struct rte_ether_hdr *eth_from;
 353         struct rte_ether_hdr *eth_to;
 354
 355         eth_from = rte_pktmbuf_mtod(pkt_src, struct rte_ether_hdr *);
 356         eth_to = rte_pktmbuf_mtod(pkt_dst, struct rte_ether_hdr *);
 357
 358         /* copy header */
 359         rte_memcpy(eth_to, eth_from, sizeof(struct rte_ether_hdr));
 360 }
 361
 362 static inline void
 363 add_awgn(struct rte_mbuf **mbufs, uint16_t num_pkts)
 364 {
 365         RTE_SET_USED(mbufs);
 366         RTE_SET_USED(num_pkts);
 367 }
 368
 369 /* Encoder output to Decoder input adapter. The Decoder accepts only soft input
 370  * so each bit of the encoder output must be translated into one byte of LLR. If
 371  * Sub-block Deinterleaver is bypassed, which is the case, the padding bytes
 372  * must additionally be insterted at the end of each sub-block.
 373  */
 374 static inline void
 375 transform_enc_out_dec_in(struct rte_mbuf **mbufs, uint8_t *temp_buf,
 376                 uint16_t num_pkts, uint16_t k)
 377 {
 378         uint16_t i, l, j;
 379         uint16_t start_bit_idx;
 380         uint16_t out_idx;
 381         uint16_t d = k + 4;
 382         uint16_t kpi = RTE_ALIGN_CEIL(d, 32);
 383         uint16_t nd = kpi - d;
 384         uint16_t ncb = 3 * kpi;
 385
 386         for (i = 0; i < num_pkts; ++i) {
 387                 uint16_t pkt_data_len = rte_pktmbuf_data_len(mbufs[i]) -
 388                                 sizeof(struct rte_ether_hdr);
 389
 390                 /* Resize the packet if needed */
 391                 if (pkt_data_len < ncb) {
 392                         char *data = rte_pktmbuf_append(mbufs[i],
 393                                         ncb - pkt_data_len);
 394                         if (data == NULL)
 395                                 printf(
 396                                         "Not enough space in decoder input packet");
 397                 }
 398
 399                 /* Translate each bit into 1 LLR byte. */
 400                 start_bit_idx = 0;
 401                 out_idx = 0;
 402                 for (j = 0; j < 3; ++j) {
 403                         for (l = start_bit_idx; l < start_bit_idx + d; ++l) {
 404                                 uint8_t *data = rte_pktmbuf_mtod_offset(
 405                                         mbufs[i], uint8_t *,
 406                                         sizeof(struct rte_ether_hdr) +
 407                                         (l >> 3));
 408                                 if (*data & (0x80 >> (l & 7)))
 409                                         temp_buf[out_idx] = LLR_1_BIT;
 410                                 else
 411                                         temp_buf[out_idx] = LLR_0_BIT;
 412                                 ++out_idx;
 413                         }
 414                         /* Padding bytes should be at the end of the sub-block.
 415                          */
 416                         memset(&temp_buf[out_idx], 0, nd);
 417                         out_idx += nd;
 418                         start_bit_idx += d;
 419                 }
 420
 421                 rte_memcpy(rte_pktmbuf_mtod_offset(mbufs[i], uint8_t *,
 422                                 sizeof(struct rte_ether_hdr)), temp_buf, ncb);
 423         }
 424 }
 425
 426 static inline void
 427 verify_data(struct rte_mbuf **mbufs, uint16_t num_pkts)
 428 {
 429         uint16_t i;
 430         for (i = 0; i < num_pkts; ++i) {
 431                 struct rte_mbuf *out = mbufs[i];
 432                 struct rte_mbuf *in = out->userdata;
 433
 434                 if (memcmp(rte_pktmbuf_mtod_offset(in, uint8_t *,
 435                                 sizeof(struct rte_ether_hdr)),
 436                                 rte_pktmbuf_mtod_offset(out, uint8_t *,
 437                                 sizeof(struct rte_ether_hdr)),
 438                                 K / 8 - CRC_24B_LEN))
 439                         printf("Input and output buffers are not equal!\n");
 440         }
 441 }
 442
 443 static int
 444 initialize_ports(struct app_config_params *app_params,
 445                 struct rte_mempool *ethdev_mbuf_mempool)
 446 {
 447         int ret;
 448         uint16_t port_id = app_params->port_id;
 449         uint16_t q;
 450         /* ethernet addresses of ports */
 451         struct rte_ether_addr bbdev_port_eth_addr;
 452
 453         /* initialize ports */
 454         printf("\nInitializing port %u...\n", app_params->port_id);
 455         ret = rte_eth_dev_configure(port_id, app_params->num_enc_cores,
 456                 app_params->num_dec_cores, &port_conf);
 457
 458         if (ret < 0) {
 459                 printf("Cannot configure device: err=%d, port=%u\n",
 460                         ret, port_id);
 461                 return -1;
 462         }
 463
 464         /* initialize RX queues for encoder */
 465         for (q = 0; q < app_params->num_enc_cores; q++) {
 466                 ret = rte_eth_rx_queue_setup(port_id, q,
 467                         RTE_TEST_RX_DESC_DEFAULT,
 468                         rte_eth_dev_socket_id(port_id),
 469                         NULL, ethdev_mbuf_mempool);
 470                 if (ret < 0) {
 471                         printf("rte_eth_rx_queue_setup: err=%d, queue=%u\n",
 472                                 ret, q);
 473                         return -1;
 474                 }
 475         }
 476         /* initialize TX queues for decoder */
 477         for (q = 0; q < app_params->num_dec_cores; q++) {
 478                 ret = rte_eth_tx_queue_setup(port_id, q,
 479                         RTE_TEST_TX_DESC_DEFAULT,
 480                         rte_eth_dev_socket_id(port_id), NULL);
 481                 if (ret < 0) {
 482                         printf("rte_eth_tx_queue_setup: err=%d, queue=%u\n",
 483                                 ret, q);
 484                         return -1;
 485                 }
 486         }
 487
 488         ret = rte_eth_promiscuous_enable(port_id);
 489         if (ret != 0) {
 490                 printf("Cannot enable promiscuous mode: err=%s, port=%u\n",
 491                         rte_strerror(-ret), port_id);
 492                 return ret;
 493         }
 494
 495         ret = rte_eth_macaddr_get(port_id, &bbdev_port_eth_addr);
 496         if (ret < 0) {
 497                 printf("rte_eth_macaddr_get: err=%d, queue=%u\n",
 498                         ret, q);
 499                 return -1;
 500         }
 501
 502         print_mac(port_id, &bbdev_port_eth_addr);
 503
 504         return 0;
 505 }
 506
 507 static void
 508 lcore_conf_init(struct app_config_params *app_params,
 509                 struct lcore_conf *lcore_conf,
 510                 struct rte_mempool **bbdev_op_pools,
 511                 struct rte_mempool *bbdev_mbuf_mempool,
 512                 struct rte_ring *enc_to_dec_ring,
 513                 struct lcore_statistics *lcore_stats)
 514 {
 515         unsigned int lcore_id;
 516         struct lcore_conf *lconf;
 517         uint16_t rx_queue_id = 0;
 518         uint16_t tx_queue_id = 0;
 519         uint16_t enc_q_id = 0;
 520         uint16_t dec_q_id = 0;
 521
 522         /* Configure lcores */
 523         for (lcore_id = 0; lcore_id < 8 * sizeof(uint64_t); ++lcore_id) {
 524                 lconf = &lcore_conf[lcore_id];
 525                 lconf->core_type = 0;
 526
 527                 if ((1ULL << lcore_id) & app_params->enc_core_mask) {
 528                         lconf->core_type |= (1 << RTE_BBDEV_OP_TURBO_ENC);
 529                         lconf->rx_queue_id = rx_queue_id++;
 530                         lconf->enc_queue_id =
 531                                         app_params->enc_queue_ids[enc_q_id++];
 532                 }
 533
 534                 if ((1ULL << lcore_id) & app_params->dec_core_mask) {
 535                         lconf->core_type |= (1 << RTE_BBDEV_OP_TURBO_DEC);
 536                         lconf->tx_queue_id = tx_queue_id++;
 537                         lconf->dec_queue_id =
 538                                         app_params->dec_queue_ids[dec_q_id++];
 539                 }
 540
 541                 lconf->bbdev_enc_op_pool =
 542                                 bbdev_op_pools[RTE_BBDEV_OP_TURBO_ENC];
 543                 lconf->bbdev_dec_op_pool =
 544                                 bbdev_op_pools[RTE_BBDEV_OP_TURBO_DEC];
 545                 lconf->bbdev_id = app_params->bbdev_id;
 546                 lconf->port_id = app_params->port_id;
 547                 lconf->enc_out_pool = bbdev_mbuf_mempool;
 548                 lconf->enc_to_dec_ring = enc_to_dec_ring;
 549                 lconf->lcore_stats = &lcore_stats[lcore_id];
 550         }
 551 }
 552
 553 static void
 554 print_lcore_stats(struct lcore_statistics *lstats, unsigned int lcore_id)
 555 {
 556         static const char *stats_border = "_______";
 557
 558         printf("\nLcore %d: %s enqueued count:\t\t%u\n",
 559                         lcore_id, stats_border, lstats->enqueued);
 560         printf("Lcore %d: %s dequeued count:\t\t%u\n",
 561                         lcore_id, stats_border, lstats->dequeued);
 562         printf("Lcore %d: %s RX lost packets count:\t\t%u\n",
 563                         lcore_id, stats_border, lstats->rx_lost_packets);
 564         printf("Lcore %d: %s encoder-to-decoder lost count:\t%u\n",
 565                         lcore_id, stats_border,
 566                         lstats->enc_to_dec_lost_packets);
 567         printf("Lcore %d: %s TX lost packets count:\t\t%u\n",
 568                         lcore_id, stats_border, lstats->tx_lost_packets);
 569 }
 570
 571 static void
 572 print_stats(struct stats_lcore_params *stats_lcore)
 573 {
 574         unsigned int l_id;
 575         unsigned int bbdev_id = stats_lcore->app_params->bbdev_id;
 576         unsigned int port_id = stats_lcore->app_params->port_id;
 577         int len, ret, i;
 578
 579         struct rte_eth_xstat *xstats;
 580         struct rte_eth_xstat_name *xstats_names;
 581         struct rte_bbdev_stats bbstats;
 582         static const char *stats_border = "_______";
 583
 584         const char clr[] = { 27, '[', '2', 'J', '\0' };
 585         const char topLeft[] = { 27, '[', '1', ';', '1', 'H', '\0' };
 586
 587         /* Clear screen and move to top left */
 588         printf("%s%s", clr, topLeft);
 589
 590         printf("PORT STATISTICS:\n================\n");
 591         len = rte_eth_xstats_get(port_id, NULL, 0);
 592         if (len < 0)
 593                 rte_exit(EXIT_FAILURE,
 594                                 "rte_eth_xstats_get(%u) failed: %d", port_id,
 595                                 len);
 596
 597         xstats = calloc(len, sizeof(*xstats));
 598         if (xstats == NULL)
 599                 rte_exit(EXIT_FAILURE,
 600                                 "Failed to calloc memory for xstats");
 601
 602         ret = rte_eth_xstats_get(port_id, xstats, len);
 603         if (ret < 0 || ret > len) {
 604                 free(xstats);
 605                 rte_exit(EXIT_FAILURE,
 606                                 "rte_eth_xstats_get(%u) len%i failed: %d",
 607                                 port_id, len, ret);
 608         }
 609
 610         xstats_names = calloc(len, sizeof(*xstats_names));
 611         if (xstats_names == NULL) {
 612                 free(xstats);
 613                 rte_exit(EXIT_FAILURE,
 614                                 "Failed to calloc memory for xstats_names");
 615         }
 616
 617         ret = rte_eth_xstats_get_names(port_id, xstats_names, len);
 618         if (ret < 0 || ret > len) {
 619                 free(xstats);
 620                 free(xstats_names);
 621                 rte_exit(EXIT_FAILURE,
 622                                 "rte_eth_xstats_get_names(%u) len%i failed: %d",
 623                                 port_id, len, ret);
 624         }
 625
 626         for (i = 0; i < len; i++) {
 627                 if (xstats[i].value > 0)
 628                         printf("Port %u: %s %s:\t\t%"PRIu64"\n",
 629                                         port_id, stats_border,
 630                                         xstats_names[i].name,
 631                                         xstats[i].value);
 632         }
 633
 634         ret = rte_bbdev_stats_get(bbdev_id, &bbstats);
 635         if (ret < 0) {
 636                 free(xstats);
 637                 free(xstats_names);
 638                 rte_exit(EXIT_FAILURE,
 639                                 "ERROR(%d): Failure to get BBDEV %u statistics\n",
 640                                 ret, bbdev_id);
 641         }
 642
 643         printf("\nBBDEV STATISTICS:\n=================\n");
 644         printf("BBDEV %u: %s enqueue count:\t\t%"PRIu64"\n",
 645                         bbdev_id, stats_border,
 646                         bbstats.enqueued_count);
 647         printf("BBDEV %u: %s dequeue count:\t\t%"PRIu64"\n",
 648                         bbdev_id, stats_border,
 649                         bbstats.dequeued_count);
 650         printf("BBDEV %u: %s enqueue error count:\t\t%"PRIu64"\n",
 651                         bbdev_id, stats_border,
 652                         bbstats.enqueue_err_count);
 653         printf("BBDEV %u: %s dequeue error count:\t\t%"PRIu64"\n\n",
 654                         bbdev_id, stats_border,
 655                         bbstats.dequeue_err_count);
 656
 657         printf("LCORE STATISTICS:\n=================\n");
 658         for (l_id = 0; l_id < RTE_MAX_LCORE; ++l_id) {
 659                 if (stats_lcore->lconf[l_id].core_type == 0)
 660                         continue;
 661                 print_lcore_stats(stats_lcore->lconf[l_id].lcore_stats, l_id);
 662         }
 663
 664         fflush(stdout);
 665
 666         free(xstats);
 667         free(xstats_names);
 668 }
 669
 670 static int
 671 stats_loop(void *arg)
 672 {
 673         struct stats_lcore_params *stats_lcore = arg;
 674
 675         while (!rte_atomic16_read(&global_exit_flag)) {
 676                 print_stats(stats_lcore);
 677                 rte_delay_ms(500);
 678         }
 679
 680         return 0;
 681 }
 682
 683 static inline void
 684 run_encoding(struct lcore_conf *lcore_conf)
 685 {
 686         uint16_t i;
 687         uint16_t port_id, rx_queue_id;
 688         uint16_t bbdev_id, enc_queue_id;
 689         uint16_t nb_rx, nb_enq, nb_deq, nb_sent;
 690         struct rte_mbuf *rx_pkts_burst[MAX_PKT_BURST];
 691         struct rte_mbuf *enc_out_pkts[MAX_PKT_BURST];
 692         struct rte_bbdev_enc_op *bbdev_ops_burst[MAX_PKT_BURST];
 693         struct lcore_statistics *lcore_stats;
 694         struct rte_mempool *bbdev_op_pool, *enc_out_pool;
 695         struct rte_ring *enc_to_dec_ring;
 696         const int in_data_len = (def_op_enc.cb_params.k / 8) - CRC_24B_LEN;
 697
 698         lcore_stats = lcore_conf->lcore_stats;
 699         port_id = lcore_conf->port_id;
 700         rx_queue_id = lcore_conf->rx_queue_id;
 701         bbdev_id = lcore_conf->bbdev_id;
 702         enc_queue_id = lcore_conf->enc_queue_id;
 703         bbdev_op_pool = lcore_conf->bbdev_enc_op_pool;
 704         enc_out_pool = lcore_conf->enc_out_pool;
 705         enc_to_dec_ring = lcore_conf->enc_to_dec_ring;
 706
 707         /* Read packet from RX queues*/
 708         nb_rx = rte_eth_rx_burst(port_id, rx_queue_id, rx_pkts_burst,
 709                         MAX_PKT_BURST);
 710         if (!nb_rx)
 711                 return;
 712
 713         if (unlikely(rte_mempool_get_bulk(enc_out_pool, (void **)enc_out_pkts,
 714                         nb_rx) != 0)) {
 715                 pktmbuf_free_bulk(rx_pkts_burst, nb_rx);
 716                 lcore_stats->rx_lost_packets += nb_rx;
 717                 return;
 718         }
 719
 720         if (unlikely(rte_bbdev_enc_op_alloc_bulk(bbdev_op_pool, bbdev_ops_burst,
 721                         nb_rx) != 0)) {
 722                 pktmbuf_free_bulk(enc_out_pkts, nb_rx);
 723                 pktmbuf_free_bulk(rx_pkts_burst, nb_rx);
 724                 lcore_stats->rx_lost_packets += nb_rx;
 725                 return;
 726         }
 727
 728         for (i = 0; i < nb_rx; i++) {
 729                 char *data;
 730                 const uint16_t pkt_data_len =
 731                                 rte_pktmbuf_data_len(rx_pkts_burst[i]) -
 732                                 sizeof(struct rte_ether_hdr);
 733                 /* save input mbuf pointer for later comparison */
 734                 enc_out_pkts[i]->userdata = rx_pkts_burst[i];
 735
 736                 /* copy ethernet header */
 737                 rte_pktmbuf_reset(enc_out_pkts[i]);
 738                 data = rte_pktmbuf_append(enc_out_pkts[i],
 739                                 sizeof(struct rte_ether_hdr));
 740                 if (data == NULL) {
 741                         printf(
 742                                 "Not enough space for ethernet header in encoder output mbuf\n");
 743                         continue;
 744                 }
 745                 add_ether_hdr(rx_pkts_burst[i], enc_out_pkts[i]);
 746
 747                 /* set op */
 748                 bbdev_ops_burst[i]->turbo_enc = def_op_enc;
 749
 750                 bbdev_ops_burst[i]->turbo_enc.input.data =
 751                                 rx_pkts_burst[i];
 752                 bbdev_ops_burst[i]->turbo_enc.input.offset =
 753                                 sizeof(struct rte_ether_hdr);
 754                 /* Encoder will attach the CRC24B, adjust the length */
 755                 bbdev_ops_burst[i]->turbo_enc.input.length = in_data_len;
 756
 757                 if (in_data_len < pkt_data_len)
 758                         rte_pktmbuf_trim(rx_pkts_burst[i], pkt_data_len -
 759                                         in_data_len);
 760                 else if (in_data_len > pkt_data_len) {
 761                         data = rte_pktmbuf_append(rx_pkts_burst[i],
 762                                         in_data_len - pkt_data_len);
 763                         if (data == NULL)
 764                                 printf(
 765                                         "Not enough storage in mbuf to perform the encoding\n");
 766                 }
 767
 768                 bbdev_ops_burst[i]->turbo_enc.output.data =
 769                                 enc_out_pkts[i];
 770                 bbdev_ops_burst[i]->turbo_enc.output.offset =
 771                                 sizeof(struct rte_ether_hdr);
 772         }
 773
 774         /* Enqueue packets on BBDevice */
 775         nb_enq = rte_bbdev_enqueue_enc_ops(bbdev_id, enc_queue_id,
 776                         bbdev_ops_burst, nb_rx);
 777         if (unlikely(nb_enq < nb_rx)) {
 778                 pktmbuf_userdata_free_bulk(&enc_out_pkts[nb_enq],
 779                                 nb_rx - nb_enq);
 780                 rte_bbdev_enc_op_free_bulk(&bbdev_ops_burst[nb_enq],
 781                                 nb_rx - nb_enq);
 782                 lcore_stats->rx_lost_packets += nb_rx - nb_enq;
 783
 784                 if (!nb_enq)
 785                         return;
 786         }
 787
 788         lcore_stats->enqueued += nb_enq;
 789
 790         /* Dequeue packets from bbdev device*/
 791         nb_deq = 0;
 792         do {
 793                 nb_deq += rte_bbdev_dequeue_enc_ops(bbdev_id, enc_queue_id,
 794                                 &bbdev_ops_burst[nb_deq], nb_enq - nb_deq);
 795         } while (unlikely(nb_deq < nb_enq));
 796
 797         lcore_stats->dequeued += nb_deq;
 798
 799         /* Generate and add AWGN */
 800         add_awgn(enc_out_pkts, nb_deq);
 801
 802         rte_bbdev_enc_op_free_bulk(bbdev_ops_burst, nb_deq);
 803
 804         /* Enqueue packets to encoder-to-decoder ring */
 805         nb_sent = rte_ring_enqueue_burst(enc_to_dec_ring, (void **)enc_out_pkts,
 806                         nb_deq, NULL);
 807         if (unlikely(nb_sent < nb_deq)) {
 808                 pktmbuf_userdata_free_bulk(&enc_out_pkts[nb_sent],
 809                                 nb_deq - nb_sent);
 810                 lcore_stats->enc_to_dec_lost_packets += nb_deq - nb_sent;
 811         }
 812 }
 813
 814 static void
 815 run_decoding(struct lcore_conf *lcore_conf)
 816 {
 817         uint16_t i;
 818         uint16_t port_id, tx_queue_id;
 819         uint16_t bbdev_id, bbdev_queue_id;
 820         uint16_t nb_recv, nb_enq, nb_deq, nb_tx;
 821         uint8_t *llr_temp_buf;
 822         struct rte_mbuf *recv_pkts_burst[MAX_PKT_BURST];
 823         struct rte_bbdev_dec_op *bbdev_ops_burst[MAX_PKT_BURST];
 824         struct lcore_statistics *lcore_stats;
 825         struct rte_mempool *bbdev_op_pool;
 826         struct rte_ring *enc_to_dec_ring;
 827
 828         lcore_stats = lcore_conf->lcore_stats;
 829         port_id = lcore_conf->port_id;
 830         tx_queue_id = lcore_conf->tx_queue_id;
 831         bbdev_id = lcore_conf->bbdev_id;
 832         bbdev_queue_id = lcore_conf->dec_queue_id;
 833         bbdev_op_pool = lcore_conf->bbdev_dec_op_pool;
 834         enc_to_dec_ring = lcore_conf->enc_to_dec_ring;
 835         llr_temp_buf = lcore_conf->llr_temp_buf;
 836
 837         /* Dequeue packets from the ring */
 838         nb_recv = rte_ring_dequeue_burst(enc_to_dec_ring,
 839                         (void **)recv_pkts_burst, MAX_PKT_BURST, NULL);
 840         if (!nb_recv)
 841                 return;
 842
 843         if (unlikely(rte_bbdev_dec_op_alloc_bulk(bbdev_op_pool, bbdev_ops_burst,
 844                         nb_recv) != 0)) {
 845                 pktmbuf_userdata_free_bulk(recv_pkts_burst, nb_recv);
 846                 lcore_stats->rx_lost_packets += nb_recv;
 847                 return;
 848         }
 849
 850         transform_enc_out_dec_in(recv_pkts_burst, llr_temp_buf, nb_recv,
 851                         def_op_dec.cb_params.k);
 852
 853         for (i = 0; i < nb_recv; i++) {
 854                 /* set op */
 855                 bbdev_ops_burst[i]->turbo_dec = def_op_dec;
 856
 857                 bbdev_ops_burst[i]->turbo_dec.input.data = recv_pkts_burst[i];
 858                 bbdev_ops_burst[i]->turbo_dec.input.offset =
 859                                 sizeof(struct rte_ether_hdr);
 860                 bbdev_ops_burst[i]->turbo_dec.input.length =
 861                                 rte_pktmbuf_data_len(recv_pkts_burst[i])
 862                                 - sizeof(struct rte_ether_hdr);
 863
 864                 bbdev_ops_burst[i]->turbo_dec.hard_output.data =
 865                                 recv_pkts_burst[i];
 866                 bbdev_ops_burst[i]->turbo_dec.hard_output.offset =
 867                                 sizeof(struct rte_ether_hdr);
 868         }
 869
 870         /* Enqueue packets on BBDevice */
 871         nb_enq = rte_bbdev_enqueue_dec_ops(bbdev_id, bbdev_queue_id,
 872                         bbdev_ops_burst, nb_recv);
 873         if (unlikely(nb_enq < nb_recv)) {
 874                 pktmbuf_userdata_free_bulk(&recv_pkts_burst[nb_enq],
 875                                 nb_recv - nb_enq);
 876                 rte_bbdev_dec_op_free_bulk(&bbdev_ops_burst[nb_enq],
 877                                 nb_recv - nb_enq);
 878                 lcore_stats->rx_lost_packets += nb_recv - nb_enq;
 879
 880                 if (!nb_enq)
 881                         return;
 882         }
 883
 884         lcore_stats->enqueued += nb_enq;
 885
 886         /* Dequeue packets from BBDevice */
 887         nb_deq = 0;
 888         do {
 889                 nb_deq += rte_bbdev_dequeue_dec_ops(bbdev_id, bbdev_queue_id,
 890                                 &bbdev_ops_burst[nb_deq], nb_enq - nb_deq);
 891         } while (unlikely(nb_deq < nb_enq));
 892
 893         lcore_stats->dequeued += nb_deq;
 894
 895         rte_bbdev_dec_op_free_bulk(bbdev_ops_burst, nb_deq);
 896
 897         verify_data(recv_pkts_burst, nb_deq);
 898
 899         /* Free the RX mbufs after verification */
 900         for (i = 0; i < nb_deq; ++i)
 901                 rte_pktmbuf_free(recv_pkts_burst[i]->userdata);
 902
 903         /* Transmit the packets */
 904         nb_tx = rte_eth_tx_burst(port_id, tx_queue_id, recv_pkts_burst, nb_deq);
 905         if (unlikely(nb_tx < nb_deq)) {
 906                 pktmbuf_userdata_free_bulk(&recv_pkts_burst[nb_tx],
 907                                 nb_deq - nb_tx);
 908                 lcore_stats->tx_lost_packets += nb_deq - nb_tx;
 909         }
 910 }
 911
 912 static int
 913 processing_loop(void *arg)
 914 {
 915         struct lcore_conf *lcore_conf = arg;
 916         const bool run_encoder = (lcore_conf->core_type &
 917                         (1 << RTE_BBDEV_OP_TURBO_ENC));
 918         const bool run_decoder = (lcore_conf->core_type &
 919                         (1 << RTE_BBDEV_OP_TURBO_DEC));
 920
 921         while (!rte_atomic16_read(&global_exit_flag)) {
 922                 if (run_encoder)
 923                         run_encoding(lcore_conf);
 924                 if (run_decoder)
 925                         run_decoding(lcore_conf);
 926         }
 927
 928         return 0;
 929 }
 930
 931 static int
 932 prepare_bbdev_device(unsigned int dev_id, struct rte_bbdev_info *info,
 933                 struct app_config_params *app_params)
 934 {
 935         int ret;
 936         unsigned int q_id, dec_q_id, enc_q_id;
 937         struct rte_bbdev_queue_conf qconf = {0};
 938         uint16_t dec_qs_nb = app_params->num_dec_cores;
 939         uint16_t enc_qs_nb = app_params->num_enc_cores;
 940         uint16_t tot_qs = dec_qs_nb + enc_qs_nb;
 941
 942         ret = rte_bbdev_setup_queues(dev_id, tot_qs, info->socket_id);
 943         if (ret < 0)
 944                 rte_exit(EXIT_FAILURE,
 945                                 "ERROR(%d): BBDEV %u not configured properly\n",
 946                                 ret, dev_id);
 947
 948         /* setup device DEC queues */
 949         qconf.socket = info->socket_id;
 950         qconf.queue_size = info->drv.queue_size_lim;
 951         qconf.op_type = RTE_BBDEV_OP_TURBO_DEC;
 952
 953         for (q_id = 0, dec_q_id = 0; q_id < dec_qs_nb; q_id++) {
 954                 ret = rte_bbdev_queue_configure(dev_id, q_id, &qconf);
 955                 if (ret < 0)
 956                         rte_exit(EXIT_FAILURE,
 957                                         "ERROR(%d): BBDEV %u DEC queue %u not configured properly\n",
 958                                         ret, dev_id, q_id);
 959                 app_params->dec_queue_ids[dec_q_id++] = q_id;
 960         }
 961
 962         /* setup device ENC queues */
 963         qconf.op_type = RTE_BBDEV_OP_TURBO_ENC;
 964
 965         for (q_id = dec_qs_nb, enc_q_id = 0; q_id < tot_qs; q_id++) {
 966                 ret = rte_bbdev_queue_configure(dev_id, q_id, &qconf);
 967                 if (ret < 0)
 968                         rte_exit(EXIT_FAILURE,
 969                                         "ERROR(%d): BBDEV %u ENC queue %u not configured properly\n",
 970                                         ret, dev_id, q_id);
 971                 app_params->enc_queue_ids[enc_q_id++] = q_id;
 972         }
 973
 974         ret = rte_bbdev_start(dev_id);
 975
 976         if (ret != 0)
 977                 rte_exit(EXIT_FAILURE, "ERROR(%d): BBDEV %u not started\n",
 978                         ret, dev_id);
 979
 980         printf("BBdev %u started\n", dev_id);
 981
 982         return 0;
 983 }
 984
 985 static inline bool
 986 check_matching_capabilities(uint64_t mask, uint64_t required_mask)
 987 {
 988         return (mask & required_mask) == required_mask;
 989 }
 990
 991 static void
 992 enable_bbdev(struct app_config_params *app_params)
 993 {
 994         struct rte_bbdev_info dev_info;
 995         const struct rte_bbdev_op_cap *op_cap;
 996         uint16_t bbdev_id = app_params->bbdev_id;
 997         bool encoder_capable = false;
 998         bool decoder_capable = false;
 999
1000         rte_bbdev_info_get(bbdev_id, &dev_info);
1001         op_cap = dev_info.drv.capabilities;
1002
1003         while (op_cap->type != RTE_BBDEV_OP_NONE) {
1004                 if (op_cap->type == RTE_BBDEV_OP_TURBO_ENC) {
1005                         if (check_matching_capabilities(
1006                                         op_cap->cap.turbo_enc.capability_flags,
1007                                         def_op_enc.op_flags))
1008                                 encoder_capable = true;
1009                 }
1010
1011                 if (op_cap->type == RTE_BBDEV_OP_TURBO_DEC) {
1012                         if (check_matching_capabilities(
1013                                         op_cap->cap.turbo_dec.capability_flags,
1014                                         def_op_dec.op_flags))
1015                                 decoder_capable = true;
1016                 }
1017
1018                 op_cap++;
1019         }
1020
1021         if (encoder_capable == false)
1022                 rte_exit(EXIT_FAILURE,
1023                         "The specified BBDev %u doesn't have required encoder capabilities!\n",
1024                         bbdev_id);
1025         if (decoder_capable == false)
1026                 rte_exit(EXIT_FAILURE,
1027                         "The specified BBDev %u doesn't have required decoder capabilities!\n",
1028                         bbdev_id);
1029
1030         prepare_bbdev_device(bbdev_id, &dev_info, app_params);
1031 }
1032
1033 int
1034 main(int argc, char **argv)
1035 {
1036         int ret;
1037         unsigned int nb_bbdevs, flags, lcore_id;
1038         void *sigret;
1039         struct app_config_params app_params = def_app_config;
1040         struct rte_mempool *ethdev_mbuf_mempool, *bbdev_mbuf_mempool;
1041         struct rte_mempool *bbdev_op_pools[RTE_BBDEV_OP_TYPE_COUNT];
1042         struct lcore_conf lcore_conf[RTE_MAX_LCORE] = { {0} };
1043         struct lcore_statistics lcore_stats[RTE_MAX_LCORE] = { {0} };
1044         struct stats_lcore_params stats_lcore;
1045         struct rte_ring *enc_to_dec_ring;
1046         bool stats_thread_started = false;
1047         unsigned int master_lcore_id = rte_get_master_lcore();
1048
1049         rte_atomic16_init(&global_exit_flag);
1050
1051         sigret = signal(SIGTERM, signal_handler);
1052         if (sigret == SIG_ERR)
1053                 rte_exit(EXIT_FAILURE, "signal(%d, ...) failed", SIGTERM);
1054
1055         sigret = signal(SIGINT, signal_handler);
1056         if (sigret == SIG_ERR)
1057                 rte_exit(EXIT_FAILURE, "signal(%d, ...) failed", SIGINT);
1058
1059         ret = rte_eal_init(argc, argv);
1060         if (ret < 0)
1061                 rte_exit(EXIT_FAILURE, "Invalid EAL arguments\n");
1062
1063         argc -= ret;
1064         argv += ret;
1065
1066         /* parse application arguments (after the EAL ones) */
1067         ret = bbdev_parse_args(argc, argv, &app_params);
1068         if (ret < 0)
1069                 rte_exit(EXIT_FAILURE, "Invalid BBDEV arguments\n");
1070
1071         /*create bbdev op pools*/
1072         bbdev_op_pools[RTE_BBDEV_OP_TURBO_DEC] =
1073                         rte_bbdev_op_pool_create("bbdev_op_pool_dec",
1074                         RTE_BBDEV_OP_TURBO_DEC, NB_MBUF, 128, rte_socket_id());
1075         bbdev_op_pools[RTE_BBDEV_OP_TURBO_ENC] =
1076                         rte_bbdev_op_pool_create("bbdev_op_pool_enc",
1077                         RTE_BBDEV_OP_TURBO_ENC, NB_MBUF, 128, rte_socket_id());
1078
1079         if ((bbdev_op_pools[RTE_BBDEV_OP_TURBO_DEC] == NULL) ||
1080                         (bbdev_op_pools[RTE_BBDEV_OP_TURBO_ENC] == NULL))
1081                 rte_exit(EXIT_FAILURE, "Cannot create bbdev op pools\n");
1082
1083         /* Create encoder to decoder ring */
1084         flags = (app_params.num_enc_cores == 1) ? RING_F_SP_ENQ : 0;
1085         if (app_params.num_dec_cores == 1)
1086                 flags |= RING_F_SC_DEQ;
1087
1088         enc_to_dec_ring = rte_ring_create("enc_to_dec_ring",
1089                 rte_align32pow2(NB_MBUF), rte_socket_id(), flags);
1090
1091         /* Get the number of available bbdev devices */
1092         nb_bbdevs = rte_bbdev_count();
1093         if (nb_bbdevs <= app_params.bbdev_id)
1094                 rte_exit(EXIT_FAILURE,
1095                                 "%u BBDevs detected, cannot use BBDev with ID %u!\n",
1096                                 nb_bbdevs, app_params.bbdev_id);
1097         printf("Number of bbdevs detected: %d\n", nb_bbdevs);
1098
1099         if (!rte_eth_dev_is_valid_port(app_params.port_id))
1100                 rte_exit(EXIT_FAILURE,
1101                                 "cannot use port with ID %u!\n",
1102                                 app_params.port_id);
1103
1104         /* create the mbuf mempool for ethdev pkts */
1105         ethdev_mbuf_mempool = rte_pktmbuf_pool_create("ethdev_mbuf_pool",
1106                         NB_MBUF, MEMPOOL_CACHE_SIZE, 0,
1107                         RTE_MBUF_DEFAULT_BUF_SIZE, rte_socket_id());
1108         if (ethdev_mbuf_mempool == NULL)
1109                 rte_exit(EXIT_FAILURE, "Cannot create ethdev mbuf mempool\n");
1110
1111         /* create the mbuf mempool for encoder output */
1112         bbdev_mbuf_mempool = rte_pktmbuf_pool_create("bbdev_mbuf_pool",
1113                         NB_MBUF, MEMPOOL_CACHE_SIZE, 0,
1114                         RTE_MBUF_DEFAULT_BUF_SIZE, rte_socket_id());
1115         if (bbdev_mbuf_mempool == NULL)
1116                 rte_exit(EXIT_FAILURE, "Cannot create ethdev mbuf mempool\n");
1117
1118         /* initialize ports */
1119         ret = initialize_ports(&app_params, ethdev_mbuf_mempool);
1120
1121         /* Check if all requested lcores are available */
1122         for (lcore_id = 0; lcore_id < 8 * sizeof(uint64_t); ++lcore_id)
1123                 if (((1ULL << lcore_id) & app_params.enc_core_mask) ||
1124                                 ((1ULL << lcore_id) & app_params.dec_core_mask))
1125                         if (!rte_lcore_is_enabled(lcore_id))
1126                                 rte_exit(EXIT_FAILURE,
1127                                                 "Requested lcore_id %u is not enabled!\n",
1128                                                 lcore_id);
1129
1130         /* Start ethernet port */
1131         ret = rte_eth_dev_start(app_params.port_id);
1132         if (ret < 0)
1133                 rte_exit(EXIT_FAILURE, "rte_eth_dev_start:err=%d, port=%u\n",
1134                                 ret, app_params.port_id);
1135
1136         ret = check_port_link_status(app_params.port_id);
1137         if (ret < 0)
1138                 exit(EXIT_FAILURE);
1139
1140         /* start BBDevice and save BBDev queue IDs */
1141         enable_bbdev(&app_params);
1142
1143         /* Initialize the port/queue configuration of each logical core */
1144         lcore_conf_init(&app_params, lcore_conf, bbdev_op_pools,
1145                         bbdev_mbuf_mempool, enc_to_dec_ring, lcore_stats);
1146
1147         stats_lcore.app_params = &app_params;
1148         stats_lcore.lconf = lcore_conf;
1149
1150         RTE_LCORE_FOREACH_SLAVE(lcore_id) {
1151                 if (lcore_conf[lcore_id].core_type != 0)
1152                         /* launch per-lcore processing loop on slave lcores */
1153                         rte_eal_remote_launch(processing_loop,
1154                                         &lcore_conf[lcore_id], lcore_id);
1155                 else if (!stats_thread_started) {
1156                         /* launch statistics printing loop */
1157                         rte_eal_remote_launch(stats_loop, &stats_lcore,
1158                                         lcore_id);
1159                         stats_thread_started = true;
1160                 }
1161         }
1162
1163         if (!stats_thread_started &&
1164                         lcore_conf[master_lcore_id].core_type != 0)
1165                 rte_exit(EXIT_FAILURE,
1166                                 "Not enough lcores to run the statistics printing loop!");
1167         else if (lcore_conf[master_lcore_id].core_type != 0)
1168                 processing_loop(&lcore_conf[master_lcore_id]);
1169         else if (!stats_thread_started)
1170                 stats_loop(&stats_lcore);
1171
1172         RTE_LCORE_FOREACH_SLAVE(lcore_id) {
1173                 ret |= rte_eal_wait_lcore(lcore_id);
1174         }
1175
1176         return ret;
1177 }