1 /* SPDX-License-Identifier: BSD-3-Clause
2 * Copyright(c) 2017 Intel Corporation
10 #include <sys/types.h>
12 #include <sys/queue.h>
21 #include <rte_common.h>
23 #include <rte_cycles.h>
24 #include <rte_ether.h>
25 #include <rte_ethdev.h>
27 #include <rte_lcore.h>
28 #include <rte_malloc.h>
30 #include <rte_mbuf_dyn.h>
31 #include <rte_memory.h>
32 #include <rte_mempool.h>
34 #include <rte_bbdev.h>
35 #include <rte_bbdev_op.h>
37 /* LLR values - negative value for '1' bit */
38 #define LLR_1_BIT 0x81
39 #define LLR_0_BIT 0x7F
41 #define MAX_PKT_BURST 32
43 #define MEMPOOL_CACHE_SIZE 256
45 /* Hardcoded K value */
47 #define NCB (3 * RTE_ALIGN_CEIL(K + 4, 32))
51 /* Configurable number of RX/TX ring descriptors */
52 #define RTE_TEST_RX_DESC_DEFAULT 128
53 #define RTE_TEST_TX_DESC_DEFAULT 512
55 #define BBDEV_ASSERT(a) do { \
62 static int input_dynfield_offset = -1;
64 static inline struct rte_mbuf **
65 mbuf_input(struct rte_mbuf *mbuf)
67 return RTE_MBUF_DYNFIELD(mbuf,
68 input_dynfield_offset, struct rte_mbuf **);
71 static const struct rte_eth_conf port_conf = {
73 .mq_mode = ETH_MQ_RX_NONE,
74 .max_rx_pkt_len = RTE_ETHER_MAX_LEN,
78 .mq_mode = ETH_MQ_TX_NONE,
82 struct rte_bbdev_op_turbo_enc def_op_enc = {
83 /* These values are arbitrarily put, and does not map to the real
84 * values for the data received from ethdev ports
91 .op_flags = RTE_BBDEV_TURBO_CRC_24A_ATTACH
94 struct rte_bbdev_op_turbo_dec def_op_dec = {
95 /* These values are arbitrarily put, and does not map to the real
96 * values for the data received from ethdev ports
107 .op_flags = RTE_BBDEV_TURBO_NEG_LLR_1_BIT_IN
110 struct app_config_params {
111 /* Placeholders for app params */
114 uint64_t enc_core_mask;
115 uint64_t dec_core_mask;
117 /* Values filled during init time */
118 uint16_t enc_queue_ids[RTE_MAX_LCORE];
119 uint16_t dec_queue_ids[RTE_MAX_LCORE];
120 uint16_t num_enc_cores;
121 uint16_t num_dec_cores;
124 struct lcore_statistics {
125 unsigned int enqueued;
126 unsigned int dequeued;
127 unsigned int rx_lost_packets;
128 unsigned int enc_to_dec_lost_packets;
129 unsigned int tx_lost_packets;
130 } __rte_cache_aligned;
132 /** each lcore configuration */
136 unsigned int port_id;
137 unsigned int rx_queue_id;
138 unsigned int tx_queue_id;
140 unsigned int bbdev_id;
141 unsigned int enc_queue_id;
142 unsigned int dec_queue_id;
144 uint8_t llr_temp_buf[NCB];
146 struct rte_mempool *bbdev_dec_op_pool;
147 struct rte_mempool *bbdev_enc_op_pool;
148 struct rte_mempool *enc_out_pool;
149 struct rte_ring *enc_to_dec_ring;
151 struct lcore_statistics *lcore_stats;
152 } __rte_cache_aligned;
154 struct stats_lcore_params {
155 struct lcore_conf *lconf;
156 struct app_config_params *app_params;
160 static const struct app_config_params def_app_config = {
163 .enc_core_mask = 0x2,
164 .dec_core_mask = 0x4,
169 static uint16_t global_exit_flag;
173 usage(const char *prgname)
175 printf("%s [EAL options] "
177 " --enc_cores - number of encoding cores (default = 0x2)\n"
178 " --dec_cores - number of decoding cores (default = 0x4)\n"
179 " --port_id - Ethernet port ID (default = 0)\n"
180 " --bbdev_id - BBDev ID (default = 0)\n"
184 /* parse core mask */
186 uint16_t bbdev_parse_mask(const char *mask)
191 /* parse hexadecimal string */
192 pm = strtoul(mask, &end, 16);
193 if ((mask[0] == '\0') || (end == NULL) || (*end != '\0'))
199 /* parse core mask */
201 uint16_t bbdev_parse_number(const char *mask)
206 /* parse hexadecimal string */
207 pm = strtoul(mask, &end, 10);
208 if ((mask[0] == '\0') || (end == NULL) || (*end != '\0'))
215 bbdev_parse_args(int argc, char **argv,
216 struct app_config_params *app_params)
221 char *prgname = argv[0];
223 static struct option lgopts[] = {
224 { "enc_core_mask", required_argument, 0, 'e' },
225 { "dec_core_mask", required_argument, 0, 'd' },
226 { "port_id", required_argument, 0, 'p' },
227 { "bbdev_id", required_argument, 0, 'b' },
231 BBDEV_ASSERT(argc != 0);
232 BBDEV_ASSERT(argv != NULL);
233 BBDEV_ASSERT(app_params != NULL);
235 while ((opt = getopt_long(argc, argv, "e:d:p:b:", lgopts, &opt_indx)) !=
239 app_params->enc_core_mask =
240 bbdev_parse_mask(optarg);
241 if (app_params->enc_core_mask == 0) {
245 app_params->num_enc_cores =
246 __builtin_popcount(app_params->enc_core_mask);
250 app_params->dec_core_mask =
251 bbdev_parse_mask(optarg);
252 if (app_params->dec_core_mask == 0) {
256 app_params->num_dec_cores =
257 __builtin_popcount(app_params->dec_core_mask);
261 app_params->port_id = bbdev_parse_number(optarg);
265 app_params->bbdev_id = bbdev_parse_number(optarg);
278 signal_handler(int signum)
280 printf("\nSignal %d received\n", signum);
281 __atomic_store_n(&global_exit_flag, 1, __ATOMIC_RELAXED);
285 print_mac(unsigned int portid, struct rte_ether_addr *bbdev_ports_eth_address)
287 printf("Port %u, MAC address: " RTE_ETHER_ADDR_PRT_FMT "\n\n",
288 (unsigned int) portid,
289 RTE_ETHER_ADDR_BYTES(bbdev_ports_eth_address));
293 pktmbuf_free_bulk(struct rte_mbuf **mbufs, unsigned int nb_to_free)
296 for (i = 0; i < nb_to_free; ++i)
297 rte_pktmbuf_free(mbufs[i]);
301 pktmbuf_input_free_bulk(struct rte_mbuf **mbufs, unsigned int nb_to_free)
304 for (i = 0; i < nb_to_free; ++i) {
305 struct rte_mbuf *rx_pkt = *mbuf_input(mbufs[i]);
306 rte_pktmbuf_free(rx_pkt);
307 rte_pktmbuf_free(mbufs[i]);
311 /* Check the link status of all ports in up to 9s, and print them finally */
313 check_port_link_status(uint16_t port_id)
315 #define CHECK_INTERVAL 100 /* 100ms */
316 #define MAX_CHECK_TIME 90 /* 9s (90 * 100ms) in total */
318 struct rte_eth_link link;
319 int link_get_err = -EINVAL;
321 printf("\nChecking link status.");
324 for (count = 0; count <= MAX_CHECK_TIME &&
325 !__atomic_load_n(&global_exit_flag, __ATOMIC_RELAXED); count++) {
326 memset(&link, 0, sizeof(link));
327 link_get_err = rte_eth_link_get_nowait(port_id, &link);
329 if (link_get_err >= 0 && link.link_status) {
330 const char *dp = (link.link_duplex ==
331 ETH_LINK_FULL_DUPLEX) ?
332 "full-duplex" : "half-duplex";
333 printf("\nPort %u Link Up - speed %s - %s\n",
335 rte_eth_link_speed_to_str(link.link_speed),
341 rte_delay_ms(CHECK_INTERVAL);
344 if (link_get_err >= 0)
345 printf("\nPort %d Link Down\n", port_id);
347 printf("\nGet link failed (port %d): %s\n", port_id,
348 rte_strerror(-link_get_err));
354 add_ether_hdr(struct rte_mbuf *pkt_src, struct rte_mbuf *pkt_dst)
356 struct rte_ether_hdr *eth_from;
357 struct rte_ether_hdr *eth_to;
359 eth_from = rte_pktmbuf_mtod(pkt_src, struct rte_ether_hdr *);
360 eth_to = rte_pktmbuf_mtod(pkt_dst, struct rte_ether_hdr *);
363 rte_memcpy(eth_to, eth_from, sizeof(struct rte_ether_hdr));
367 add_awgn(struct rte_mbuf **mbufs, uint16_t num_pkts)
370 RTE_SET_USED(num_pkts);
373 /* Encoder output to Decoder input adapter. The Decoder accepts only soft input
374 * so each bit of the encoder output must be translated into one byte of LLR. If
375 * Sub-block Deinterleaver is bypassed, which is the case, the padding bytes
376 * must additionally be insterted at the end of each sub-block.
379 transform_enc_out_dec_in(struct rte_mbuf **mbufs, uint8_t *temp_buf,
380 uint16_t num_pkts, uint16_t k)
383 uint16_t start_bit_idx;
386 uint16_t kpi = RTE_ALIGN_CEIL(d, 32);
387 uint16_t nd = kpi - d;
388 uint16_t ncb = 3 * kpi;
390 for (i = 0; i < num_pkts; ++i) {
391 uint16_t pkt_data_len = rte_pktmbuf_data_len(mbufs[i]) -
392 sizeof(struct rte_ether_hdr);
394 /* Resize the packet if needed */
395 if (pkt_data_len < ncb) {
396 char *data = rte_pktmbuf_append(mbufs[i],
400 "Not enough space in decoder input packet");
403 /* Translate each bit into 1 LLR byte. */
406 for (j = 0; j < 3; ++j) {
407 for (l = start_bit_idx; l < start_bit_idx + d; ++l) {
408 uint8_t *data = rte_pktmbuf_mtod_offset(
410 sizeof(struct rte_ether_hdr) +
412 if (*data & (0x80 >> (l & 7)))
413 temp_buf[out_idx] = LLR_1_BIT;
415 temp_buf[out_idx] = LLR_0_BIT;
418 /* Padding bytes should be at the end of the sub-block.
420 memset(&temp_buf[out_idx], 0, nd);
425 rte_memcpy(rte_pktmbuf_mtod_offset(mbufs[i], uint8_t *,
426 sizeof(struct rte_ether_hdr)), temp_buf, ncb);
431 verify_data(struct rte_mbuf **mbufs, uint16_t num_pkts)
434 for (i = 0; i < num_pkts; ++i) {
435 struct rte_mbuf *out = mbufs[i];
436 struct rte_mbuf *in = *mbuf_input(out);
438 if (memcmp(rte_pktmbuf_mtod_offset(in, uint8_t *,
439 sizeof(struct rte_ether_hdr)),
440 rte_pktmbuf_mtod_offset(out, uint8_t *,
441 sizeof(struct rte_ether_hdr)),
442 K / 8 - CRC_24B_LEN))
443 printf("Input and output buffers are not equal!\n");
448 initialize_ports(struct app_config_params *app_params,
449 struct rte_mempool *ethdev_mbuf_mempool)
452 uint16_t port_id = app_params->port_id;
454 /* ethernet addresses of ports */
455 struct rte_ether_addr bbdev_port_eth_addr;
457 /* initialize ports */
458 printf("\nInitializing port %u...\n", app_params->port_id);
459 ret = rte_eth_dev_configure(port_id, app_params->num_enc_cores,
460 app_params->num_dec_cores, &port_conf);
463 printf("Cannot configure device: err=%d, port=%u\n",
468 /* initialize RX queues for encoder */
469 for (q = 0; q < app_params->num_enc_cores; q++) {
470 ret = rte_eth_rx_queue_setup(port_id, q,
471 RTE_TEST_RX_DESC_DEFAULT,
472 rte_eth_dev_socket_id(port_id),
473 NULL, ethdev_mbuf_mempool);
475 printf("rte_eth_rx_queue_setup: err=%d, queue=%u\n",
480 /* initialize TX queues for decoder */
481 for (q = 0; q < app_params->num_dec_cores; q++) {
482 ret = rte_eth_tx_queue_setup(port_id, q,
483 RTE_TEST_TX_DESC_DEFAULT,
484 rte_eth_dev_socket_id(port_id), NULL);
486 printf("rte_eth_tx_queue_setup: err=%d, queue=%u\n",
492 ret = rte_eth_promiscuous_enable(port_id);
494 printf("Cannot enable promiscuous mode: err=%s, port=%u\n",
495 rte_strerror(-ret), port_id);
499 ret = rte_eth_macaddr_get(port_id, &bbdev_port_eth_addr);
501 printf("rte_eth_macaddr_get: err=%d, queue=%u\n",
506 print_mac(port_id, &bbdev_port_eth_addr);
512 lcore_conf_init(struct app_config_params *app_params,
513 struct lcore_conf *lcore_conf,
514 struct rte_mempool **bbdev_op_pools,
515 struct rte_mempool *bbdev_mbuf_mempool,
516 struct rte_ring *enc_to_dec_ring,
517 struct lcore_statistics *lcore_stats)
519 unsigned int lcore_id;
520 struct lcore_conf *lconf;
521 uint16_t rx_queue_id = 0;
522 uint16_t tx_queue_id = 0;
523 uint16_t enc_q_id = 0;
524 uint16_t dec_q_id = 0;
526 /* Configure lcores */
527 for (lcore_id = 0; lcore_id < 8 * sizeof(uint64_t); ++lcore_id) {
528 lconf = &lcore_conf[lcore_id];
529 lconf->core_type = 0;
531 if ((1ULL << lcore_id) & app_params->enc_core_mask) {
532 lconf->core_type |= (1 << RTE_BBDEV_OP_TURBO_ENC);
533 lconf->rx_queue_id = rx_queue_id++;
534 lconf->enc_queue_id =
535 app_params->enc_queue_ids[enc_q_id++];
538 if ((1ULL << lcore_id) & app_params->dec_core_mask) {
539 lconf->core_type |= (1 << RTE_BBDEV_OP_TURBO_DEC);
540 lconf->tx_queue_id = tx_queue_id++;
541 lconf->dec_queue_id =
542 app_params->dec_queue_ids[dec_q_id++];
545 lconf->bbdev_enc_op_pool =
546 bbdev_op_pools[RTE_BBDEV_OP_TURBO_ENC];
547 lconf->bbdev_dec_op_pool =
548 bbdev_op_pools[RTE_BBDEV_OP_TURBO_DEC];
549 lconf->bbdev_id = app_params->bbdev_id;
550 lconf->port_id = app_params->port_id;
551 lconf->enc_out_pool = bbdev_mbuf_mempool;
552 lconf->enc_to_dec_ring = enc_to_dec_ring;
553 lconf->lcore_stats = &lcore_stats[lcore_id];
558 print_lcore_stats(struct lcore_statistics *lstats, unsigned int lcore_id)
560 static const char *stats_border = "_______";
562 printf("\nLcore %d: %s enqueued count:\t\t%u\n",
563 lcore_id, stats_border, lstats->enqueued);
564 printf("Lcore %d: %s dequeued count:\t\t%u\n",
565 lcore_id, stats_border, lstats->dequeued);
566 printf("Lcore %d: %s RX lost packets count:\t\t%u\n",
567 lcore_id, stats_border, lstats->rx_lost_packets);
568 printf("Lcore %d: %s encoder-to-decoder lost count:\t%u\n",
569 lcore_id, stats_border,
570 lstats->enc_to_dec_lost_packets);
571 printf("Lcore %d: %s TX lost packets count:\t\t%u\n",
572 lcore_id, stats_border, lstats->tx_lost_packets);
576 print_stats(struct stats_lcore_params *stats_lcore)
579 unsigned int bbdev_id = stats_lcore->app_params->bbdev_id;
580 unsigned int port_id = stats_lcore->app_params->port_id;
583 struct rte_eth_xstat *xstats;
584 struct rte_eth_xstat_name *xstats_names;
585 struct rte_bbdev_stats bbstats;
586 static const char *stats_border = "_______";
588 const char clr[] = { 27, '[', '2', 'J', '\0' };
589 const char topLeft[] = { 27, '[', '1', ';', '1', 'H', '\0' };
591 /* Clear screen and move to top left */
592 printf("%s%s", clr, topLeft);
594 printf("PORT STATISTICS:\n================\n");
595 len = rte_eth_xstats_get(port_id, NULL, 0);
597 rte_exit(EXIT_FAILURE,
598 "rte_eth_xstats_get(%u) failed: %d", port_id,
601 xstats = calloc(len, sizeof(*xstats));
603 rte_exit(EXIT_FAILURE,
604 "Failed to calloc memory for xstats");
606 ret = rte_eth_xstats_get(port_id, xstats, len);
607 if (ret < 0 || ret > len) {
609 rte_exit(EXIT_FAILURE,
610 "rte_eth_xstats_get(%u) len%i failed: %d",
614 xstats_names = calloc(len, sizeof(*xstats_names));
615 if (xstats_names == NULL) {
617 rte_exit(EXIT_FAILURE,
618 "Failed to calloc memory for xstats_names");
621 ret = rte_eth_xstats_get_names(port_id, xstats_names, len);
622 if (ret < 0 || ret > len) {
625 rte_exit(EXIT_FAILURE,
626 "rte_eth_xstats_get_names(%u) len%i failed: %d",
630 for (i = 0; i < len; i++) {
631 if (xstats[i].value > 0)
632 printf("Port %u: %s %s:\t\t%"PRIu64"\n",
633 port_id, stats_border,
634 xstats_names[i].name,
638 ret = rte_bbdev_stats_get(bbdev_id, &bbstats);
642 rte_exit(EXIT_FAILURE,
643 "ERROR(%d): Failure to get BBDEV %u statistics\n",
647 printf("\nBBDEV STATISTICS:\n=================\n");
648 printf("BBDEV %u: %s enqueue count:\t\t%"PRIu64"\n",
649 bbdev_id, stats_border,
650 bbstats.enqueued_count);
651 printf("BBDEV %u: %s dequeue count:\t\t%"PRIu64"\n",
652 bbdev_id, stats_border,
653 bbstats.dequeued_count);
654 printf("BBDEV %u: %s enqueue error count:\t\t%"PRIu64"\n",
655 bbdev_id, stats_border,
656 bbstats.enqueue_err_count);
657 printf("BBDEV %u: %s dequeue error count:\t\t%"PRIu64"\n\n",
658 bbdev_id, stats_border,
659 bbstats.dequeue_err_count);
661 printf("LCORE STATISTICS:\n=================\n");
662 for (l_id = 0; l_id < RTE_MAX_LCORE; ++l_id) {
663 if (stats_lcore->lconf[l_id].core_type == 0)
665 print_lcore_stats(stats_lcore->lconf[l_id].lcore_stats, l_id);
675 stats_loop(void *arg)
677 struct stats_lcore_params *stats_lcore = arg;
679 while (!__atomic_load_n(&global_exit_flag, __ATOMIC_RELAXED)) {
680 print_stats(stats_lcore);
688 run_encoding(struct lcore_conf *lcore_conf)
691 uint16_t port_id, rx_queue_id;
692 uint16_t bbdev_id, enc_queue_id;
693 uint16_t nb_rx, nb_enq, nb_deq, nb_sent;
694 struct rte_mbuf *rx_pkts_burst[MAX_PKT_BURST];
695 struct rte_mbuf *enc_out_pkts[MAX_PKT_BURST];
696 struct rte_bbdev_enc_op *bbdev_ops_burst[MAX_PKT_BURST];
697 struct lcore_statistics *lcore_stats;
698 struct rte_mempool *bbdev_op_pool, *enc_out_pool;
699 struct rte_ring *enc_to_dec_ring;
700 const int in_data_len = (def_op_enc.cb_params.k / 8) - CRC_24B_LEN;
702 lcore_stats = lcore_conf->lcore_stats;
703 port_id = lcore_conf->port_id;
704 rx_queue_id = lcore_conf->rx_queue_id;
705 bbdev_id = lcore_conf->bbdev_id;
706 enc_queue_id = lcore_conf->enc_queue_id;
707 bbdev_op_pool = lcore_conf->bbdev_enc_op_pool;
708 enc_out_pool = lcore_conf->enc_out_pool;
709 enc_to_dec_ring = lcore_conf->enc_to_dec_ring;
711 /* Read packet from RX queues*/
712 nb_rx = rte_eth_rx_burst(port_id, rx_queue_id, rx_pkts_burst,
717 if (unlikely(rte_mempool_get_bulk(enc_out_pool, (void **)enc_out_pkts,
719 pktmbuf_free_bulk(rx_pkts_burst, nb_rx);
720 lcore_stats->rx_lost_packets += nb_rx;
724 if (unlikely(rte_bbdev_enc_op_alloc_bulk(bbdev_op_pool, bbdev_ops_burst,
726 pktmbuf_free_bulk(enc_out_pkts, nb_rx);
727 pktmbuf_free_bulk(rx_pkts_burst, nb_rx);
728 lcore_stats->rx_lost_packets += nb_rx;
732 for (i = 0; i < nb_rx; i++) {
734 const uint16_t pkt_data_len =
735 rte_pktmbuf_data_len(rx_pkts_burst[i]) -
736 sizeof(struct rte_ether_hdr);
737 /* save input mbuf pointer for later comparison */
738 *mbuf_input(enc_out_pkts[i]) = rx_pkts_burst[i];
740 /* copy ethernet header */
741 rte_pktmbuf_reset(enc_out_pkts[i]);
742 data = rte_pktmbuf_append(enc_out_pkts[i],
743 sizeof(struct rte_ether_hdr));
746 "Not enough space for ethernet header in encoder output mbuf\n");
749 add_ether_hdr(rx_pkts_burst[i], enc_out_pkts[i]);
752 bbdev_ops_burst[i]->turbo_enc = def_op_enc;
754 bbdev_ops_burst[i]->turbo_enc.input.data =
756 bbdev_ops_burst[i]->turbo_enc.input.offset =
757 sizeof(struct rte_ether_hdr);
758 /* Encoder will attach the CRC24B, adjust the length */
759 bbdev_ops_burst[i]->turbo_enc.input.length = in_data_len;
761 if (in_data_len < pkt_data_len)
762 rte_pktmbuf_trim(rx_pkts_burst[i], pkt_data_len -
764 else if (in_data_len > pkt_data_len) {
765 data = rte_pktmbuf_append(rx_pkts_burst[i],
766 in_data_len - pkt_data_len);
769 "Not enough storage in mbuf to perform the encoding\n");
772 bbdev_ops_burst[i]->turbo_enc.output.data =
774 bbdev_ops_burst[i]->turbo_enc.output.offset =
775 sizeof(struct rte_ether_hdr);
778 /* Enqueue packets on BBDevice */
779 nb_enq = rte_bbdev_enqueue_enc_ops(bbdev_id, enc_queue_id,
780 bbdev_ops_burst, nb_rx);
781 if (unlikely(nb_enq < nb_rx)) {
782 pktmbuf_input_free_bulk(&enc_out_pkts[nb_enq],
784 rte_bbdev_enc_op_free_bulk(&bbdev_ops_burst[nb_enq],
786 lcore_stats->rx_lost_packets += nb_rx - nb_enq;
792 lcore_stats->enqueued += nb_enq;
794 /* Dequeue packets from bbdev device*/
797 nb_deq += rte_bbdev_dequeue_enc_ops(bbdev_id, enc_queue_id,
798 &bbdev_ops_burst[nb_deq], nb_enq - nb_deq);
799 } while (unlikely(nb_deq < nb_enq));
801 lcore_stats->dequeued += nb_deq;
803 /* Generate and add AWGN */
804 add_awgn(enc_out_pkts, nb_deq);
806 rte_bbdev_enc_op_free_bulk(bbdev_ops_burst, nb_deq);
808 /* Enqueue packets to encoder-to-decoder ring */
809 nb_sent = rte_ring_enqueue_burst(enc_to_dec_ring, (void **)enc_out_pkts,
811 if (unlikely(nb_sent < nb_deq)) {
812 pktmbuf_input_free_bulk(&enc_out_pkts[nb_sent],
814 lcore_stats->enc_to_dec_lost_packets += nb_deq - nb_sent;
819 run_decoding(struct lcore_conf *lcore_conf)
822 uint16_t port_id, tx_queue_id;
823 uint16_t bbdev_id, bbdev_queue_id;
824 uint16_t nb_recv, nb_enq, nb_deq, nb_tx;
825 uint8_t *llr_temp_buf;
826 struct rte_mbuf *recv_pkts_burst[MAX_PKT_BURST];
827 struct rte_bbdev_dec_op *bbdev_ops_burst[MAX_PKT_BURST];
828 struct lcore_statistics *lcore_stats;
829 struct rte_mempool *bbdev_op_pool;
830 struct rte_ring *enc_to_dec_ring;
832 lcore_stats = lcore_conf->lcore_stats;
833 port_id = lcore_conf->port_id;
834 tx_queue_id = lcore_conf->tx_queue_id;
835 bbdev_id = lcore_conf->bbdev_id;
836 bbdev_queue_id = lcore_conf->dec_queue_id;
837 bbdev_op_pool = lcore_conf->bbdev_dec_op_pool;
838 enc_to_dec_ring = lcore_conf->enc_to_dec_ring;
839 llr_temp_buf = lcore_conf->llr_temp_buf;
841 /* Dequeue packets from the ring */
842 nb_recv = rte_ring_dequeue_burst(enc_to_dec_ring,
843 (void **)recv_pkts_burst, MAX_PKT_BURST, NULL);
847 if (unlikely(rte_bbdev_dec_op_alloc_bulk(bbdev_op_pool, bbdev_ops_burst,
849 pktmbuf_input_free_bulk(recv_pkts_burst, nb_recv);
850 lcore_stats->rx_lost_packets += nb_recv;
854 transform_enc_out_dec_in(recv_pkts_burst, llr_temp_buf, nb_recv,
855 def_op_dec.cb_params.k);
857 for (i = 0; i < nb_recv; i++) {
859 bbdev_ops_burst[i]->turbo_dec = def_op_dec;
861 bbdev_ops_burst[i]->turbo_dec.input.data = recv_pkts_burst[i];
862 bbdev_ops_burst[i]->turbo_dec.input.offset =
863 sizeof(struct rte_ether_hdr);
864 bbdev_ops_burst[i]->turbo_dec.input.length =
865 rte_pktmbuf_data_len(recv_pkts_burst[i])
866 - sizeof(struct rte_ether_hdr);
868 bbdev_ops_burst[i]->turbo_dec.hard_output.data =
870 bbdev_ops_burst[i]->turbo_dec.hard_output.offset =
871 sizeof(struct rte_ether_hdr);
874 /* Enqueue packets on BBDevice */
875 nb_enq = rte_bbdev_enqueue_dec_ops(bbdev_id, bbdev_queue_id,
876 bbdev_ops_burst, nb_recv);
877 if (unlikely(nb_enq < nb_recv)) {
878 pktmbuf_input_free_bulk(&recv_pkts_burst[nb_enq],
880 rte_bbdev_dec_op_free_bulk(&bbdev_ops_burst[nb_enq],
882 lcore_stats->rx_lost_packets += nb_recv - nb_enq;
888 lcore_stats->enqueued += nb_enq;
890 /* Dequeue packets from BBDevice */
893 nb_deq += rte_bbdev_dequeue_dec_ops(bbdev_id, bbdev_queue_id,
894 &bbdev_ops_burst[nb_deq], nb_enq - nb_deq);
895 } while (unlikely(nb_deq < nb_enq));
897 lcore_stats->dequeued += nb_deq;
899 rte_bbdev_dec_op_free_bulk(bbdev_ops_burst, nb_deq);
901 verify_data(recv_pkts_burst, nb_deq);
903 /* Free the RX mbufs after verification */
904 for (i = 0; i < nb_deq; ++i)
905 rte_pktmbuf_free(*mbuf_input(recv_pkts_burst[i]));
907 /* Transmit the packets */
908 nb_tx = rte_eth_tx_burst(port_id, tx_queue_id, recv_pkts_burst, nb_deq);
909 if (unlikely(nb_tx < nb_deq)) {
910 pktmbuf_input_free_bulk(&recv_pkts_burst[nb_tx],
912 lcore_stats->tx_lost_packets += nb_deq - nb_tx;
917 processing_loop(void *arg)
919 struct lcore_conf *lcore_conf = arg;
920 const bool run_encoder = (lcore_conf->core_type &
921 (1 << RTE_BBDEV_OP_TURBO_ENC));
922 const bool run_decoder = (lcore_conf->core_type &
923 (1 << RTE_BBDEV_OP_TURBO_DEC));
925 while (!__atomic_load_n(&global_exit_flag, __ATOMIC_RELAXED)) {
927 run_encoding(lcore_conf);
929 run_decoding(lcore_conf);
936 prepare_bbdev_device(unsigned int dev_id, struct rte_bbdev_info *info,
937 struct app_config_params *app_params)
940 unsigned int q_id, dec_q_id, enc_q_id;
941 struct rte_bbdev_queue_conf qconf = {0};
942 uint16_t dec_qs_nb = app_params->num_dec_cores;
943 uint16_t enc_qs_nb = app_params->num_enc_cores;
944 uint16_t tot_qs = dec_qs_nb + enc_qs_nb;
946 ret = rte_bbdev_setup_queues(dev_id, tot_qs, info->socket_id);
948 rte_exit(EXIT_FAILURE,
949 "ERROR(%d): BBDEV %u not configured properly\n",
952 /* setup device DEC queues */
953 qconf.socket = info->socket_id;
954 qconf.queue_size = info->drv.queue_size_lim;
955 qconf.op_type = RTE_BBDEV_OP_TURBO_DEC;
957 for (q_id = 0, dec_q_id = 0; q_id < dec_qs_nb; q_id++) {
958 ret = rte_bbdev_queue_configure(dev_id, q_id, &qconf);
960 rte_exit(EXIT_FAILURE,
961 "ERROR(%d): BBDEV %u DEC queue %u not configured properly\n",
963 app_params->dec_queue_ids[dec_q_id++] = q_id;
966 /* setup device ENC queues */
967 qconf.op_type = RTE_BBDEV_OP_TURBO_ENC;
969 for (q_id = dec_qs_nb, enc_q_id = 0; q_id < tot_qs; q_id++) {
970 ret = rte_bbdev_queue_configure(dev_id, q_id, &qconf);
972 rte_exit(EXIT_FAILURE,
973 "ERROR(%d): BBDEV %u ENC queue %u not configured properly\n",
975 app_params->enc_queue_ids[enc_q_id++] = q_id;
978 ret = rte_bbdev_start(dev_id);
981 rte_exit(EXIT_FAILURE, "ERROR(%d): BBDEV %u not started\n",
984 printf("BBdev %u started\n", dev_id);
990 check_matching_capabilities(uint64_t mask, uint64_t required_mask)
992 return (mask & required_mask) == required_mask;
996 enable_bbdev(struct app_config_params *app_params)
998 struct rte_bbdev_info dev_info;
999 const struct rte_bbdev_op_cap *op_cap;
1000 uint16_t bbdev_id = app_params->bbdev_id;
1001 bool encoder_capable = false;
1002 bool decoder_capable = false;
1004 rte_bbdev_info_get(bbdev_id, &dev_info);
1005 op_cap = dev_info.drv.capabilities;
1007 while (op_cap->type != RTE_BBDEV_OP_NONE) {
1008 if (op_cap->type == RTE_BBDEV_OP_TURBO_ENC) {
1009 if (check_matching_capabilities(
1010 op_cap->cap.turbo_enc.capability_flags,
1011 def_op_enc.op_flags))
1012 encoder_capable = true;
1015 if (op_cap->type == RTE_BBDEV_OP_TURBO_DEC) {
1016 if (check_matching_capabilities(
1017 op_cap->cap.turbo_dec.capability_flags,
1018 def_op_dec.op_flags))
1019 decoder_capable = true;
1025 if (encoder_capable == false)
1026 rte_exit(EXIT_FAILURE,
1027 "The specified BBDev %u doesn't have required encoder capabilities!\n",
1029 if (decoder_capable == false)
1030 rte_exit(EXIT_FAILURE,
1031 "The specified BBDev %u doesn't have required decoder capabilities!\n",
1034 prepare_bbdev_device(bbdev_id, &dev_info, app_params);
1038 main(int argc, char **argv)
1041 unsigned int nb_bbdevs, flags, lcore_id;
1043 struct app_config_params app_params = def_app_config;
1044 struct rte_mempool *ethdev_mbuf_mempool, *bbdev_mbuf_mempool;
1045 struct rte_mempool *bbdev_op_pools[RTE_BBDEV_OP_TYPE_COUNT];
1046 struct lcore_conf lcore_conf[RTE_MAX_LCORE] = { {0} };
1047 struct lcore_statistics lcore_stats[RTE_MAX_LCORE] = { {0} };
1048 struct stats_lcore_params stats_lcore;
1049 struct rte_ring *enc_to_dec_ring;
1050 bool stats_thread_started = false;
1051 unsigned int main_lcore_id = rte_get_main_lcore();
1053 static const struct rte_mbuf_dynfield input_dynfield_desc = {
1054 .name = "example_bbdev_dynfield_input",
1055 .size = sizeof(struct rte_mbuf *),
1056 .align = __alignof__(struct rte_mbuf *),
1059 __atomic_store_n(&global_exit_flag, 0, __ATOMIC_RELAXED);
1061 sigret = signal(SIGTERM, signal_handler);
1062 if (sigret == SIG_ERR)
1063 rte_exit(EXIT_FAILURE, "signal(%d, ...) failed", SIGTERM);
1065 sigret = signal(SIGINT, signal_handler);
1066 if (sigret == SIG_ERR)
1067 rte_exit(EXIT_FAILURE, "signal(%d, ...) failed", SIGINT);
1069 ret = rte_eal_init(argc, argv);
1071 rte_exit(EXIT_FAILURE, "Invalid EAL arguments\n");
1076 /* parse application arguments (after the EAL ones) */
1077 ret = bbdev_parse_args(argc, argv, &app_params);
1079 rte_exit(EXIT_FAILURE, "Invalid BBDEV arguments\n");
1081 /*create bbdev op pools*/
1082 bbdev_op_pools[RTE_BBDEV_OP_TURBO_DEC] =
1083 rte_bbdev_op_pool_create("bbdev_op_pool_dec",
1084 RTE_BBDEV_OP_TURBO_DEC, NB_MBUF, 128, rte_socket_id());
1085 bbdev_op_pools[RTE_BBDEV_OP_TURBO_ENC] =
1086 rte_bbdev_op_pool_create("bbdev_op_pool_enc",
1087 RTE_BBDEV_OP_TURBO_ENC, NB_MBUF, 128, rte_socket_id());
1089 if ((bbdev_op_pools[RTE_BBDEV_OP_TURBO_DEC] == NULL) ||
1090 (bbdev_op_pools[RTE_BBDEV_OP_TURBO_ENC] == NULL))
1091 rte_exit(EXIT_FAILURE, "Cannot create bbdev op pools\n");
1093 /* Create encoder to decoder ring */
1094 flags = (app_params.num_enc_cores == 1) ? RING_F_SP_ENQ : 0;
1095 if (app_params.num_dec_cores == 1)
1096 flags |= RING_F_SC_DEQ;
1098 enc_to_dec_ring = rte_ring_create("enc_to_dec_ring",
1099 rte_align32pow2(NB_MBUF), rte_socket_id(), flags);
1101 /* Get the number of available bbdev devices */
1102 nb_bbdevs = rte_bbdev_count();
1103 if (nb_bbdevs <= app_params.bbdev_id)
1104 rte_exit(EXIT_FAILURE,
1105 "%u BBDevs detected, cannot use BBDev with ID %u!\n",
1106 nb_bbdevs, app_params.bbdev_id);
1107 printf("Number of bbdevs detected: %d\n", nb_bbdevs);
1109 if (!rte_eth_dev_is_valid_port(app_params.port_id))
1110 rte_exit(EXIT_FAILURE,
1111 "cannot use port with ID %u!\n",
1112 app_params.port_id);
1114 /* create the mbuf mempool for ethdev pkts */
1115 ethdev_mbuf_mempool = rte_pktmbuf_pool_create("ethdev_mbuf_pool",
1116 NB_MBUF, MEMPOOL_CACHE_SIZE, 0,
1117 RTE_MBUF_DEFAULT_BUF_SIZE, rte_socket_id());
1118 if (ethdev_mbuf_mempool == NULL)
1119 rte_exit(EXIT_FAILURE, "Cannot create ethdev mbuf mempool\n");
1121 /* create the mbuf mempool for encoder output */
1122 bbdev_mbuf_mempool = rte_pktmbuf_pool_create("bbdev_mbuf_pool",
1123 NB_MBUF, MEMPOOL_CACHE_SIZE, 0,
1124 RTE_MBUF_DEFAULT_BUF_SIZE, rte_socket_id());
1125 if (bbdev_mbuf_mempool == NULL)
1126 rte_exit(EXIT_FAILURE, "Cannot create ethdev mbuf mempool\n");
1128 /* register mbuf field to store input pointer */
1129 input_dynfield_offset =
1130 rte_mbuf_dynfield_register(&input_dynfield_desc);
1131 if (input_dynfield_offset < 0)
1132 rte_exit(EXIT_FAILURE, "Cannot register mbuf field\n");
1134 /* initialize ports */
1135 ret = initialize_ports(&app_params, ethdev_mbuf_mempool);
1137 /* Check if all requested lcores are available */
1138 for (lcore_id = 0; lcore_id < 8 * sizeof(uint64_t); ++lcore_id)
1139 if (((1ULL << lcore_id) & app_params.enc_core_mask) ||
1140 ((1ULL << lcore_id) & app_params.dec_core_mask))
1141 if (!rte_lcore_is_enabled(lcore_id))
1142 rte_exit(EXIT_FAILURE,
1143 "Requested lcore_id %u is not enabled!\n",
1146 /* Start ethernet port */
1147 ret = rte_eth_dev_start(app_params.port_id);
1149 rte_exit(EXIT_FAILURE, "rte_eth_dev_start:err=%d, port=%u\n",
1150 ret, app_params.port_id);
1152 ret = check_port_link_status(app_params.port_id);
1156 /* start BBDevice and save BBDev queue IDs */
1157 enable_bbdev(&app_params);
1159 /* Initialize the port/queue configuration of each logical core */
1160 lcore_conf_init(&app_params, lcore_conf, bbdev_op_pools,
1161 bbdev_mbuf_mempool, enc_to_dec_ring, lcore_stats);
1163 stats_lcore.app_params = &app_params;
1164 stats_lcore.lconf = lcore_conf;
1166 RTE_LCORE_FOREACH_WORKER(lcore_id) {
1167 if (lcore_conf[lcore_id].core_type != 0)
1168 /* launch per-lcore processing loop on worker lcores */
1169 rte_eal_remote_launch(processing_loop,
1170 &lcore_conf[lcore_id], lcore_id);
1171 else if (!stats_thread_started) {
1172 /* launch statistics printing loop */
1173 rte_eal_remote_launch(stats_loop, &stats_lcore,
1175 stats_thread_started = true;
1179 if (!stats_thread_started &&
1180 lcore_conf[main_lcore_id].core_type != 0)
1181 rte_exit(EXIT_FAILURE,
1182 "Not enough lcores to run the statistics printing loop!");
1183 else if (lcore_conf[main_lcore_id].core_type != 0)
1184 processing_loop(&lcore_conf[main_lcore_id]);
1185 else if (!stats_thread_started)
1186 stats_loop(&stats_lcore);
1188 RTE_LCORE_FOREACH_WORKER(lcore_id) {
1189 ret |= rte_eal_wait_lcore(lcore_id);
1192 /* clean up the EAL */