1 /* SPDX-License-Identifier: BSD-3-Clause
2 * Copyright(c) 2017 Intel Corporation
10 #include <sys/types.h>
12 #include <sys/queue.h>
21 #include <rte_atomic.h>
22 #include <rte_common.h>
24 #include <rte_cycles.h>
25 #include <rte_ether.h>
26 #include <rte_ethdev.h>
28 #include <rte_lcore.h>
29 #include <rte_malloc.h>
31 #include <rte_mbuf_dyn.h>
32 #include <rte_memory.h>
33 #include <rte_mempool.h>
35 #include <rte_bbdev.h>
36 #include <rte_bbdev_op.h>
38 /* LLR values - negative value for '1' bit */
39 #define LLR_1_BIT 0x81
40 #define LLR_0_BIT 0x7F
42 #define MAX_PKT_BURST 32
44 #define MEMPOOL_CACHE_SIZE 256
46 /* Hardcoded K value */
48 #define NCB (3 * RTE_ALIGN_CEIL(K + 4, 32))
52 /* Configurable number of RX/TX ring descriptors */
53 #define RTE_TEST_RX_DESC_DEFAULT 128
54 #define RTE_TEST_TX_DESC_DEFAULT 512
56 #define BBDEV_ASSERT(a) do { \
63 static int input_dynfield_offset = -1;
65 static inline struct rte_mbuf **
66 mbuf_input(struct rte_mbuf *mbuf)
68 return RTE_MBUF_DYNFIELD(mbuf,
69 input_dynfield_offset, struct rte_mbuf **);
72 static const struct rte_eth_conf port_conf = {
74 .mq_mode = ETH_MQ_RX_NONE,
75 .max_rx_pkt_len = RTE_ETHER_MAX_LEN,
79 .mq_mode = ETH_MQ_TX_NONE,
83 struct rte_bbdev_op_turbo_enc def_op_enc = {
84 /* These values are arbitrarily put, and does not map to the real
85 * values for the data received from ethdev ports
92 .op_flags = RTE_BBDEV_TURBO_CRC_24A_ATTACH
95 struct rte_bbdev_op_turbo_dec def_op_dec = {
96 /* These values are arbitrarily put, and does not map to the real
97 * values for the data received from ethdev ports
108 .op_flags = RTE_BBDEV_TURBO_NEG_LLR_1_BIT_IN
111 struct app_config_params {
112 /* Placeholders for app params */
115 uint64_t enc_core_mask;
116 uint64_t dec_core_mask;
118 /* Values filled during init time */
119 uint16_t enc_queue_ids[RTE_MAX_LCORE];
120 uint16_t dec_queue_ids[RTE_MAX_LCORE];
121 uint16_t num_enc_cores;
122 uint16_t num_dec_cores;
125 struct lcore_statistics {
126 unsigned int enqueued;
127 unsigned int dequeued;
128 unsigned int rx_lost_packets;
129 unsigned int enc_to_dec_lost_packets;
130 unsigned int tx_lost_packets;
131 } __rte_cache_aligned;
133 /** each lcore configuration */
137 unsigned int port_id;
138 unsigned int rx_queue_id;
139 unsigned int tx_queue_id;
141 unsigned int bbdev_id;
142 unsigned int enc_queue_id;
143 unsigned int dec_queue_id;
145 uint8_t llr_temp_buf[NCB];
147 struct rte_mempool *bbdev_dec_op_pool;
148 struct rte_mempool *bbdev_enc_op_pool;
149 struct rte_mempool *enc_out_pool;
150 struct rte_ring *enc_to_dec_ring;
152 struct lcore_statistics *lcore_stats;
153 } __rte_cache_aligned;
155 struct stats_lcore_params {
156 struct lcore_conf *lconf;
157 struct app_config_params *app_params;
161 static const struct app_config_params def_app_config = {
164 .enc_core_mask = 0x2,
165 .dec_core_mask = 0x4,
170 static rte_atomic16_t global_exit_flag;
174 usage(const char *prgname)
176 printf("%s [EAL options] "
178 " --enc_cores - number of encoding cores (default = 0x2)\n"
179 " --dec_cores - number of decoding cores (default = 0x4)\n"
180 " --port_id - Ethernet port ID (default = 0)\n"
181 " --bbdev_id - BBDev ID (default = 0)\n"
185 /* parse core mask */
187 uint16_t bbdev_parse_mask(const char *mask)
192 /* parse hexadecimal string */
193 pm = strtoul(mask, &end, 16);
194 if ((mask[0] == '\0') || (end == NULL) || (*end != '\0'))
200 /* parse core mask */
202 uint16_t bbdev_parse_number(const char *mask)
207 /* parse hexadecimal string */
208 pm = strtoul(mask, &end, 10);
209 if ((mask[0] == '\0') || (end == NULL) || (*end != '\0'))
216 bbdev_parse_args(int argc, char **argv,
217 struct app_config_params *app_params)
222 char *prgname = argv[0];
224 static struct option lgopts[] = {
225 { "enc_core_mask", required_argument, 0, 'e' },
226 { "dec_core_mask", required_argument, 0, 'd' },
227 { "port_id", required_argument, 0, 'p' },
228 { "bbdev_id", required_argument, 0, 'b' },
232 BBDEV_ASSERT(argc != 0);
233 BBDEV_ASSERT(argv != NULL);
234 BBDEV_ASSERT(app_params != NULL);
236 while ((opt = getopt_long(argc, argv, "e:d:p:b:", lgopts, &opt_indx)) !=
240 app_params->enc_core_mask =
241 bbdev_parse_mask(optarg);
242 if (app_params->enc_core_mask == 0) {
246 app_params->num_enc_cores =
247 __builtin_popcount(app_params->enc_core_mask);
251 app_params->dec_core_mask =
252 bbdev_parse_mask(optarg);
253 if (app_params->dec_core_mask == 0) {
257 app_params->num_dec_cores =
258 __builtin_popcount(app_params->dec_core_mask);
262 app_params->port_id = bbdev_parse_number(optarg);
266 app_params->bbdev_id = bbdev_parse_number(optarg);
279 signal_handler(int signum)
281 printf("\nSignal %d received\n", signum);
282 rte_atomic16_set(&global_exit_flag, 1);
286 print_mac(unsigned int portid, struct rte_ether_addr *bbdev_ports_eth_address)
288 printf("Port %u, MAC address: " RTE_ETHER_ADDR_PRT_FMT "\n\n",
289 (unsigned int) portid,
290 RTE_ETHER_ADDR_BYTES(bbdev_ports_eth_address));
294 pktmbuf_free_bulk(struct rte_mbuf **mbufs, unsigned int nb_to_free)
297 for (i = 0; i < nb_to_free; ++i)
298 rte_pktmbuf_free(mbufs[i]);
302 pktmbuf_input_free_bulk(struct rte_mbuf **mbufs, unsigned int nb_to_free)
305 for (i = 0; i < nb_to_free; ++i) {
306 struct rte_mbuf *rx_pkt = *mbuf_input(mbufs[i]);
307 rte_pktmbuf_free(rx_pkt);
308 rte_pktmbuf_free(mbufs[i]);
312 /* Check the link status of all ports in up to 9s, and print them finally */
314 check_port_link_status(uint16_t port_id)
316 #define CHECK_INTERVAL 100 /* 100ms */
317 #define MAX_CHECK_TIME 90 /* 9s (90 * 100ms) in total */
319 struct rte_eth_link link;
320 int link_get_err = -EINVAL;
322 printf("\nChecking link status.");
325 for (count = 0; count <= MAX_CHECK_TIME &&
326 !rte_atomic16_read(&global_exit_flag); count++) {
327 memset(&link, 0, sizeof(link));
328 link_get_err = rte_eth_link_get_nowait(port_id, &link);
330 if (link_get_err >= 0 && link.link_status) {
331 const char *dp = (link.link_duplex ==
332 ETH_LINK_FULL_DUPLEX) ?
333 "full-duplex" : "half-duplex";
334 printf("\nPort %u Link Up - speed %s - %s\n",
336 rte_eth_link_speed_to_str(link.link_speed),
342 rte_delay_ms(CHECK_INTERVAL);
345 if (link_get_err >= 0)
346 printf("\nPort %d Link Down\n", port_id);
348 printf("\nGet link failed (port %d): %s\n", port_id,
349 rte_strerror(-link_get_err));
355 add_ether_hdr(struct rte_mbuf *pkt_src, struct rte_mbuf *pkt_dst)
357 struct rte_ether_hdr *eth_from;
358 struct rte_ether_hdr *eth_to;
360 eth_from = rte_pktmbuf_mtod(pkt_src, struct rte_ether_hdr *);
361 eth_to = rte_pktmbuf_mtod(pkt_dst, struct rte_ether_hdr *);
364 rte_memcpy(eth_to, eth_from, sizeof(struct rte_ether_hdr));
368 add_awgn(struct rte_mbuf **mbufs, uint16_t num_pkts)
371 RTE_SET_USED(num_pkts);
374 /* Encoder output to Decoder input adapter. The Decoder accepts only soft input
375 * so each bit of the encoder output must be translated into one byte of LLR. If
376 * Sub-block Deinterleaver is bypassed, which is the case, the padding bytes
377 * must additionally be insterted at the end of each sub-block.
380 transform_enc_out_dec_in(struct rte_mbuf **mbufs, uint8_t *temp_buf,
381 uint16_t num_pkts, uint16_t k)
384 uint16_t start_bit_idx;
387 uint16_t kpi = RTE_ALIGN_CEIL(d, 32);
388 uint16_t nd = kpi - d;
389 uint16_t ncb = 3 * kpi;
391 for (i = 0; i < num_pkts; ++i) {
392 uint16_t pkt_data_len = rte_pktmbuf_data_len(mbufs[i]) -
393 sizeof(struct rte_ether_hdr);
395 /* Resize the packet if needed */
396 if (pkt_data_len < ncb) {
397 char *data = rte_pktmbuf_append(mbufs[i],
401 "Not enough space in decoder input packet");
404 /* Translate each bit into 1 LLR byte. */
407 for (j = 0; j < 3; ++j) {
408 for (l = start_bit_idx; l < start_bit_idx + d; ++l) {
409 uint8_t *data = rte_pktmbuf_mtod_offset(
411 sizeof(struct rte_ether_hdr) +
413 if (*data & (0x80 >> (l & 7)))
414 temp_buf[out_idx] = LLR_1_BIT;
416 temp_buf[out_idx] = LLR_0_BIT;
419 /* Padding bytes should be at the end of the sub-block.
421 memset(&temp_buf[out_idx], 0, nd);
426 rte_memcpy(rte_pktmbuf_mtod_offset(mbufs[i], uint8_t *,
427 sizeof(struct rte_ether_hdr)), temp_buf, ncb);
432 verify_data(struct rte_mbuf **mbufs, uint16_t num_pkts)
435 for (i = 0; i < num_pkts; ++i) {
436 struct rte_mbuf *out = mbufs[i];
437 struct rte_mbuf *in = *mbuf_input(out);
439 if (memcmp(rte_pktmbuf_mtod_offset(in, uint8_t *,
440 sizeof(struct rte_ether_hdr)),
441 rte_pktmbuf_mtod_offset(out, uint8_t *,
442 sizeof(struct rte_ether_hdr)),
443 K / 8 - CRC_24B_LEN))
444 printf("Input and output buffers are not equal!\n");
449 initialize_ports(struct app_config_params *app_params,
450 struct rte_mempool *ethdev_mbuf_mempool)
453 uint16_t port_id = app_params->port_id;
455 /* ethernet addresses of ports */
456 struct rte_ether_addr bbdev_port_eth_addr;
458 /* initialize ports */
459 printf("\nInitializing port %u...\n", app_params->port_id);
460 ret = rte_eth_dev_configure(port_id, app_params->num_enc_cores,
461 app_params->num_dec_cores, &port_conf);
464 printf("Cannot configure device: err=%d, port=%u\n",
469 /* initialize RX queues for encoder */
470 for (q = 0; q < app_params->num_enc_cores; q++) {
471 ret = rte_eth_rx_queue_setup(port_id, q,
472 RTE_TEST_RX_DESC_DEFAULT,
473 rte_eth_dev_socket_id(port_id),
474 NULL, ethdev_mbuf_mempool);
476 printf("rte_eth_rx_queue_setup: err=%d, queue=%u\n",
481 /* initialize TX queues for decoder */
482 for (q = 0; q < app_params->num_dec_cores; q++) {
483 ret = rte_eth_tx_queue_setup(port_id, q,
484 RTE_TEST_TX_DESC_DEFAULT,
485 rte_eth_dev_socket_id(port_id), NULL);
487 printf("rte_eth_tx_queue_setup: err=%d, queue=%u\n",
493 ret = rte_eth_promiscuous_enable(port_id);
495 printf("Cannot enable promiscuous mode: err=%s, port=%u\n",
496 rte_strerror(-ret), port_id);
500 ret = rte_eth_macaddr_get(port_id, &bbdev_port_eth_addr);
502 printf("rte_eth_macaddr_get: err=%d, queue=%u\n",
507 print_mac(port_id, &bbdev_port_eth_addr);
513 lcore_conf_init(struct app_config_params *app_params,
514 struct lcore_conf *lcore_conf,
515 struct rte_mempool **bbdev_op_pools,
516 struct rte_mempool *bbdev_mbuf_mempool,
517 struct rte_ring *enc_to_dec_ring,
518 struct lcore_statistics *lcore_stats)
520 unsigned int lcore_id;
521 struct lcore_conf *lconf;
522 uint16_t rx_queue_id = 0;
523 uint16_t tx_queue_id = 0;
524 uint16_t enc_q_id = 0;
525 uint16_t dec_q_id = 0;
527 /* Configure lcores */
528 for (lcore_id = 0; lcore_id < 8 * sizeof(uint64_t); ++lcore_id) {
529 lconf = &lcore_conf[lcore_id];
530 lconf->core_type = 0;
532 if ((1ULL << lcore_id) & app_params->enc_core_mask) {
533 lconf->core_type |= (1 << RTE_BBDEV_OP_TURBO_ENC);
534 lconf->rx_queue_id = rx_queue_id++;
535 lconf->enc_queue_id =
536 app_params->enc_queue_ids[enc_q_id++];
539 if ((1ULL << lcore_id) & app_params->dec_core_mask) {
540 lconf->core_type |= (1 << RTE_BBDEV_OP_TURBO_DEC);
541 lconf->tx_queue_id = tx_queue_id++;
542 lconf->dec_queue_id =
543 app_params->dec_queue_ids[dec_q_id++];
546 lconf->bbdev_enc_op_pool =
547 bbdev_op_pools[RTE_BBDEV_OP_TURBO_ENC];
548 lconf->bbdev_dec_op_pool =
549 bbdev_op_pools[RTE_BBDEV_OP_TURBO_DEC];
550 lconf->bbdev_id = app_params->bbdev_id;
551 lconf->port_id = app_params->port_id;
552 lconf->enc_out_pool = bbdev_mbuf_mempool;
553 lconf->enc_to_dec_ring = enc_to_dec_ring;
554 lconf->lcore_stats = &lcore_stats[lcore_id];
559 print_lcore_stats(struct lcore_statistics *lstats, unsigned int lcore_id)
561 static const char *stats_border = "_______";
563 printf("\nLcore %d: %s enqueued count:\t\t%u\n",
564 lcore_id, stats_border, lstats->enqueued);
565 printf("Lcore %d: %s dequeued count:\t\t%u\n",
566 lcore_id, stats_border, lstats->dequeued);
567 printf("Lcore %d: %s RX lost packets count:\t\t%u\n",
568 lcore_id, stats_border, lstats->rx_lost_packets);
569 printf("Lcore %d: %s encoder-to-decoder lost count:\t%u\n",
570 lcore_id, stats_border,
571 lstats->enc_to_dec_lost_packets);
572 printf("Lcore %d: %s TX lost packets count:\t\t%u\n",
573 lcore_id, stats_border, lstats->tx_lost_packets);
577 print_stats(struct stats_lcore_params *stats_lcore)
580 unsigned int bbdev_id = stats_lcore->app_params->bbdev_id;
581 unsigned int port_id = stats_lcore->app_params->port_id;
584 struct rte_eth_xstat *xstats;
585 struct rte_eth_xstat_name *xstats_names;
586 struct rte_bbdev_stats bbstats;
587 static const char *stats_border = "_______";
589 const char clr[] = { 27, '[', '2', 'J', '\0' };
590 const char topLeft[] = { 27, '[', '1', ';', '1', 'H', '\0' };
592 /* Clear screen and move to top left */
593 printf("%s%s", clr, topLeft);
595 printf("PORT STATISTICS:\n================\n");
596 len = rte_eth_xstats_get(port_id, NULL, 0);
598 rte_exit(EXIT_FAILURE,
599 "rte_eth_xstats_get(%u) failed: %d", port_id,
602 xstats = calloc(len, sizeof(*xstats));
604 rte_exit(EXIT_FAILURE,
605 "Failed to calloc memory for xstats");
607 ret = rte_eth_xstats_get(port_id, xstats, len);
608 if (ret < 0 || ret > len) {
610 rte_exit(EXIT_FAILURE,
611 "rte_eth_xstats_get(%u) len%i failed: %d",
615 xstats_names = calloc(len, sizeof(*xstats_names));
616 if (xstats_names == NULL) {
618 rte_exit(EXIT_FAILURE,
619 "Failed to calloc memory for xstats_names");
622 ret = rte_eth_xstats_get_names(port_id, xstats_names, len);
623 if (ret < 0 || ret > len) {
626 rte_exit(EXIT_FAILURE,
627 "rte_eth_xstats_get_names(%u) len%i failed: %d",
631 for (i = 0; i < len; i++) {
632 if (xstats[i].value > 0)
633 printf("Port %u: %s %s:\t\t%"PRIu64"\n",
634 port_id, stats_border,
635 xstats_names[i].name,
639 ret = rte_bbdev_stats_get(bbdev_id, &bbstats);
643 rte_exit(EXIT_FAILURE,
644 "ERROR(%d): Failure to get BBDEV %u statistics\n",
648 printf("\nBBDEV STATISTICS:\n=================\n");
649 printf("BBDEV %u: %s enqueue count:\t\t%"PRIu64"\n",
650 bbdev_id, stats_border,
651 bbstats.enqueued_count);
652 printf("BBDEV %u: %s dequeue count:\t\t%"PRIu64"\n",
653 bbdev_id, stats_border,
654 bbstats.dequeued_count);
655 printf("BBDEV %u: %s enqueue error count:\t\t%"PRIu64"\n",
656 bbdev_id, stats_border,
657 bbstats.enqueue_err_count);
658 printf("BBDEV %u: %s dequeue error count:\t\t%"PRIu64"\n\n",
659 bbdev_id, stats_border,
660 bbstats.dequeue_err_count);
662 printf("LCORE STATISTICS:\n=================\n");
663 for (l_id = 0; l_id < RTE_MAX_LCORE; ++l_id) {
664 if (stats_lcore->lconf[l_id].core_type == 0)
666 print_lcore_stats(stats_lcore->lconf[l_id].lcore_stats, l_id);
676 stats_loop(void *arg)
678 struct stats_lcore_params *stats_lcore = arg;
680 while (!rte_atomic16_read(&global_exit_flag)) {
681 print_stats(stats_lcore);
689 run_encoding(struct lcore_conf *lcore_conf)
692 uint16_t port_id, rx_queue_id;
693 uint16_t bbdev_id, enc_queue_id;
694 uint16_t nb_rx, nb_enq, nb_deq, nb_sent;
695 struct rte_mbuf *rx_pkts_burst[MAX_PKT_BURST];
696 struct rte_mbuf *enc_out_pkts[MAX_PKT_BURST];
697 struct rte_bbdev_enc_op *bbdev_ops_burst[MAX_PKT_BURST];
698 struct lcore_statistics *lcore_stats;
699 struct rte_mempool *bbdev_op_pool, *enc_out_pool;
700 struct rte_ring *enc_to_dec_ring;
701 const int in_data_len = (def_op_enc.cb_params.k / 8) - CRC_24B_LEN;
703 lcore_stats = lcore_conf->lcore_stats;
704 port_id = lcore_conf->port_id;
705 rx_queue_id = lcore_conf->rx_queue_id;
706 bbdev_id = lcore_conf->bbdev_id;
707 enc_queue_id = lcore_conf->enc_queue_id;
708 bbdev_op_pool = lcore_conf->bbdev_enc_op_pool;
709 enc_out_pool = lcore_conf->enc_out_pool;
710 enc_to_dec_ring = lcore_conf->enc_to_dec_ring;
712 /* Read packet from RX queues*/
713 nb_rx = rte_eth_rx_burst(port_id, rx_queue_id, rx_pkts_burst,
718 if (unlikely(rte_mempool_get_bulk(enc_out_pool, (void **)enc_out_pkts,
720 pktmbuf_free_bulk(rx_pkts_burst, nb_rx);
721 lcore_stats->rx_lost_packets += nb_rx;
725 if (unlikely(rte_bbdev_enc_op_alloc_bulk(bbdev_op_pool, bbdev_ops_burst,
727 pktmbuf_free_bulk(enc_out_pkts, nb_rx);
728 pktmbuf_free_bulk(rx_pkts_burst, nb_rx);
729 lcore_stats->rx_lost_packets += nb_rx;
733 for (i = 0; i < nb_rx; i++) {
735 const uint16_t pkt_data_len =
736 rte_pktmbuf_data_len(rx_pkts_burst[i]) -
737 sizeof(struct rte_ether_hdr);
738 /* save input mbuf pointer for later comparison */
739 *mbuf_input(enc_out_pkts[i]) = rx_pkts_burst[i];
741 /* copy ethernet header */
742 rte_pktmbuf_reset(enc_out_pkts[i]);
743 data = rte_pktmbuf_append(enc_out_pkts[i],
744 sizeof(struct rte_ether_hdr));
747 "Not enough space for ethernet header in encoder output mbuf\n");
750 add_ether_hdr(rx_pkts_burst[i], enc_out_pkts[i]);
753 bbdev_ops_burst[i]->turbo_enc = def_op_enc;
755 bbdev_ops_burst[i]->turbo_enc.input.data =
757 bbdev_ops_burst[i]->turbo_enc.input.offset =
758 sizeof(struct rte_ether_hdr);
759 /* Encoder will attach the CRC24B, adjust the length */
760 bbdev_ops_burst[i]->turbo_enc.input.length = in_data_len;
762 if (in_data_len < pkt_data_len)
763 rte_pktmbuf_trim(rx_pkts_burst[i], pkt_data_len -
765 else if (in_data_len > pkt_data_len) {
766 data = rte_pktmbuf_append(rx_pkts_burst[i],
767 in_data_len - pkt_data_len);
770 "Not enough storage in mbuf to perform the encoding\n");
773 bbdev_ops_burst[i]->turbo_enc.output.data =
775 bbdev_ops_burst[i]->turbo_enc.output.offset =
776 sizeof(struct rte_ether_hdr);
779 /* Enqueue packets on BBDevice */
780 nb_enq = rte_bbdev_enqueue_enc_ops(bbdev_id, enc_queue_id,
781 bbdev_ops_burst, nb_rx);
782 if (unlikely(nb_enq < nb_rx)) {
783 pktmbuf_input_free_bulk(&enc_out_pkts[nb_enq],
785 rte_bbdev_enc_op_free_bulk(&bbdev_ops_burst[nb_enq],
787 lcore_stats->rx_lost_packets += nb_rx - nb_enq;
793 lcore_stats->enqueued += nb_enq;
795 /* Dequeue packets from bbdev device*/
798 nb_deq += rte_bbdev_dequeue_enc_ops(bbdev_id, enc_queue_id,
799 &bbdev_ops_burst[nb_deq], nb_enq - nb_deq);
800 } while (unlikely(nb_deq < nb_enq));
802 lcore_stats->dequeued += nb_deq;
804 /* Generate and add AWGN */
805 add_awgn(enc_out_pkts, nb_deq);
807 rte_bbdev_enc_op_free_bulk(bbdev_ops_burst, nb_deq);
809 /* Enqueue packets to encoder-to-decoder ring */
810 nb_sent = rte_ring_enqueue_burst(enc_to_dec_ring, (void **)enc_out_pkts,
812 if (unlikely(nb_sent < nb_deq)) {
813 pktmbuf_input_free_bulk(&enc_out_pkts[nb_sent],
815 lcore_stats->enc_to_dec_lost_packets += nb_deq - nb_sent;
820 run_decoding(struct lcore_conf *lcore_conf)
823 uint16_t port_id, tx_queue_id;
824 uint16_t bbdev_id, bbdev_queue_id;
825 uint16_t nb_recv, nb_enq, nb_deq, nb_tx;
826 uint8_t *llr_temp_buf;
827 struct rte_mbuf *recv_pkts_burst[MAX_PKT_BURST];
828 struct rte_bbdev_dec_op *bbdev_ops_burst[MAX_PKT_BURST];
829 struct lcore_statistics *lcore_stats;
830 struct rte_mempool *bbdev_op_pool;
831 struct rte_ring *enc_to_dec_ring;
833 lcore_stats = lcore_conf->lcore_stats;
834 port_id = lcore_conf->port_id;
835 tx_queue_id = lcore_conf->tx_queue_id;
836 bbdev_id = lcore_conf->bbdev_id;
837 bbdev_queue_id = lcore_conf->dec_queue_id;
838 bbdev_op_pool = lcore_conf->bbdev_dec_op_pool;
839 enc_to_dec_ring = lcore_conf->enc_to_dec_ring;
840 llr_temp_buf = lcore_conf->llr_temp_buf;
842 /* Dequeue packets from the ring */
843 nb_recv = rte_ring_dequeue_burst(enc_to_dec_ring,
844 (void **)recv_pkts_burst, MAX_PKT_BURST, NULL);
848 if (unlikely(rte_bbdev_dec_op_alloc_bulk(bbdev_op_pool, bbdev_ops_burst,
850 pktmbuf_input_free_bulk(recv_pkts_burst, nb_recv);
851 lcore_stats->rx_lost_packets += nb_recv;
855 transform_enc_out_dec_in(recv_pkts_burst, llr_temp_buf, nb_recv,
856 def_op_dec.cb_params.k);
858 for (i = 0; i < nb_recv; i++) {
860 bbdev_ops_burst[i]->turbo_dec = def_op_dec;
862 bbdev_ops_burst[i]->turbo_dec.input.data = recv_pkts_burst[i];
863 bbdev_ops_burst[i]->turbo_dec.input.offset =
864 sizeof(struct rte_ether_hdr);
865 bbdev_ops_burst[i]->turbo_dec.input.length =
866 rte_pktmbuf_data_len(recv_pkts_burst[i])
867 - sizeof(struct rte_ether_hdr);
869 bbdev_ops_burst[i]->turbo_dec.hard_output.data =
871 bbdev_ops_burst[i]->turbo_dec.hard_output.offset =
872 sizeof(struct rte_ether_hdr);
875 /* Enqueue packets on BBDevice */
876 nb_enq = rte_bbdev_enqueue_dec_ops(bbdev_id, bbdev_queue_id,
877 bbdev_ops_burst, nb_recv);
878 if (unlikely(nb_enq < nb_recv)) {
879 pktmbuf_input_free_bulk(&recv_pkts_burst[nb_enq],
881 rte_bbdev_dec_op_free_bulk(&bbdev_ops_burst[nb_enq],
883 lcore_stats->rx_lost_packets += nb_recv - nb_enq;
889 lcore_stats->enqueued += nb_enq;
891 /* Dequeue packets from BBDevice */
894 nb_deq += rte_bbdev_dequeue_dec_ops(bbdev_id, bbdev_queue_id,
895 &bbdev_ops_burst[nb_deq], nb_enq - nb_deq);
896 } while (unlikely(nb_deq < nb_enq));
898 lcore_stats->dequeued += nb_deq;
900 rte_bbdev_dec_op_free_bulk(bbdev_ops_burst, nb_deq);
902 verify_data(recv_pkts_burst, nb_deq);
904 /* Free the RX mbufs after verification */
905 for (i = 0; i < nb_deq; ++i)
906 rte_pktmbuf_free(*mbuf_input(recv_pkts_burst[i]));
908 /* Transmit the packets */
909 nb_tx = rte_eth_tx_burst(port_id, tx_queue_id, recv_pkts_burst, nb_deq);
910 if (unlikely(nb_tx < nb_deq)) {
911 pktmbuf_input_free_bulk(&recv_pkts_burst[nb_tx],
913 lcore_stats->tx_lost_packets += nb_deq - nb_tx;
918 processing_loop(void *arg)
920 struct lcore_conf *lcore_conf = arg;
921 const bool run_encoder = (lcore_conf->core_type &
922 (1 << RTE_BBDEV_OP_TURBO_ENC));
923 const bool run_decoder = (lcore_conf->core_type &
924 (1 << RTE_BBDEV_OP_TURBO_DEC));
926 while (!rte_atomic16_read(&global_exit_flag)) {
928 run_encoding(lcore_conf);
930 run_decoding(lcore_conf);
937 prepare_bbdev_device(unsigned int dev_id, struct rte_bbdev_info *info,
938 struct app_config_params *app_params)
941 unsigned int q_id, dec_q_id, enc_q_id;
942 struct rte_bbdev_queue_conf qconf = {0};
943 uint16_t dec_qs_nb = app_params->num_dec_cores;
944 uint16_t enc_qs_nb = app_params->num_enc_cores;
945 uint16_t tot_qs = dec_qs_nb + enc_qs_nb;
947 ret = rte_bbdev_setup_queues(dev_id, tot_qs, info->socket_id);
949 rte_exit(EXIT_FAILURE,
950 "ERROR(%d): BBDEV %u not configured properly\n",
953 /* setup device DEC queues */
954 qconf.socket = info->socket_id;
955 qconf.queue_size = info->drv.queue_size_lim;
956 qconf.op_type = RTE_BBDEV_OP_TURBO_DEC;
958 for (q_id = 0, dec_q_id = 0; q_id < dec_qs_nb; q_id++) {
959 ret = rte_bbdev_queue_configure(dev_id, q_id, &qconf);
961 rte_exit(EXIT_FAILURE,
962 "ERROR(%d): BBDEV %u DEC queue %u not configured properly\n",
964 app_params->dec_queue_ids[dec_q_id++] = q_id;
967 /* setup device ENC queues */
968 qconf.op_type = RTE_BBDEV_OP_TURBO_ENC;
970 for (q_id = dec_qs_nb, enc_q_id = 0; q_id < tot_qs; q_id++) {
971 ret = rte_bbdev_queue_configure(dev_id, q_id, &qconf);
973 rte_exit(EXIT_FAILURE,
974 "ERROR(%d): BBDEV %u ENC queue %u not configured properly\n",
976 app_params->enc_queue_ids[enc_q_id++] = q_id;
979 ret = rte_bbdev_start(dev_id);
982 rte_exit(EXIT_FAILURE, "ERROR(%d): BBDEV %u not started\n",
985 printf("BBdev %u started\n", dev_id);
991 check_matching_capabilities(uint64_t mask, uint64_t required_mask)
993 return (mask & required_mask) == required_mask;
997 enable_bbdev(struct app_config_params *app_params)
999 struct rte_bbdev_info dev_info;
1000 const struct rte_bbdev_op_cap *op_cap;
1001 uint16_t bbdev_id = app_params->bbdev_id;
1002 bool encoder_capable = false;
1003 bool decoder_capable = false;
1005 rte_bbdev_info_get(bbdev_id, &dev_info);
1006 op_cap = dev_info.drv.capabilities;
1008 while (op_cap->type != RTE_BBDEV_OP_NONE) {
1009 if (op_cap->type == RTE_BBDEV_OP_TURBO_ENC) {
1010 if (check_matching_capabilities(
1011 op_cap->cap.turbo_enc.capability_flags,
1012 def_op_enc.op_flags))
1013 encoder_capable = true;
1016 if (op_cap->type == RTE_BBDEV_OP_TURBO_DEC) {
1017 if (check_matching_capabilities(
1018 op_cap->cap.turbo_dec.capability_flags,
1019 def_op_dec.op_flags))
1020 decoder_capable = true;
1026 if (encoder_capable == false)
1027 rte_exit(EXIT_FAILURE,
1028 "The specified BBDev %u doesn't have required encoder capabilities!\n",
1030 if (decoder_capable == false)
1031 rte_exit(EXIT_FAILURE,
1032 "The specified BBDev %u doesn't have required decoder capabilities!\n",
1035 prepare_bbdev_device(bbdev_id, &dev_info, app_params);
1039 main(int argc, char **argv)
1042 unsigned int nb_bbdevs, flags, lcore_id;
1044 struct app_config_params app_params = def_app_config;
1045 struct rte_mempool *ethdev_mbuf_mempool, *bbdev_mbuf_mempool;
1046 struct rte_mempool *bbdev_op_pools[RTE_BBDEV_OP_TYPE_COUNT];
1047 struct lcore_conf lcore_conf[RTE_MAX_LCORE] = { {0} };
1048 struct lcore_statistics lcore_stats[RTE_MAX_LCORE] = { {0} };
1049 struct stats_lcore_params stats_lcore;
1050 struct rte_ring *enc_to_dec_ring;
1051 bool stats_thread_started = false;
1052 unsigned int main_lcore_id = rte_get_main_lcore();
1054 static const struct rte_mbuf_dynfield input_dynfield_desc = {
1055 .name = "example_bbdev_dynfield_input",
1056 .size = sizeof(struct rte_mbuf *),
1057 .align = __alignof__(struct rte_mbuf *),
1060 rte_atomic16_init(&global_exit_flag);
1062 sigret = signal(SIGTERM, signal_handler);
1063 if (sigret == SIG_ERR)
1064 rte_exit(EXIT_FAILURE, "signal(%d, ...) failed", SIGTERM);
1066 sigret = signal(SIGINT, signal_handler);
1067 if (sigret == SIG_ERR)
1068 rte_exit(EXIT_FAILURE, "signal(%d, ...) failed", SIGINT);
1070 ret = rte_eal_init(argc, argv);
1072 rte_exit(EXIT_FAILURE, "Invalid EAL arguments\n");
1077 /* parse application arguments (after the EAL ones) */
1078 ret = bbdev_parse_args(argc, argv, &app_params);
1080 rte_exit(EXIT_FAILURE, "Invalid BBDEV arguments\n");
1082 /*create bbdev op pools*/
1083 bbdev_op_pools[RTE_BBDEV_OP_TURBO_DEC] =
1084 rte_bbdev_op_pool_create("bbdev_op_pool_dec",
1085 RTE_BBDEV_OP_TURBO_DEC, NB_MBUF, 128, rte_socket_id());
1086 bbdev_op_pools[RTE_BBDEV_OP_TURBO_ENC] =
1087 rte_bbdev_op_pool_create("bbdev_op_pool_enc",
1088 RTE_BBDEV_OP_TURBO_ENC, NB_MBUF, 128, rte_socket_id());
1090 if ((bbdev_op_pools[RTE_BBDEV_OP_TURBO_DEC] == NULL) ||
1091 (bbdev_op_pools[RTE_BBDEV_OP_TURBO_ENC] == NULL))
1092 rte_exit(EXIT_FAILURE, "Cannot create bbdev op pools\n");
1094 /* Create encoder to decoder ring */
1095 flags = (app_params.num_enc_cores == 1) ? RING_F_SP_ENQ : 0;
1096 if (app_params.num_dec_cores == 1)
1097 flags |= RING_F_SC_DEQ;
1099 enc_to_dec_ring = rte_ring_create("enc_to_dec_ring",
1100 rte_align32pow2(NB_MBUF), rte_socket_id(), flags);
1102 /* Get the number of available bbdev devices */
1103 nb_bbdevs = rte_bbdev_count();
1104 if (nb_bbdevs <= app_params.bbdev_id)
1105 rte_exit(EXIT_FAILURE,
1106 "%u BBDevs detected, cannot use BBDev with ID %u!\n",
1107 nb_bbdevs, app_params.bbdev_id);
1108 printf("Number of bbdevs detected: %d\n", nb_bbdevs);
1110 if (!rte_eth_dev_is_valid_port(app_params.port_id))
1111 rte_exit(EXIT_FAILURE,
1112 "cannot use port with ID %u!\n",
1113 app_params.port_id);
1115 /* create the mbuf mempool for ethdev pkts */
1116 ethdev_mbuf_mempool = rte_pktmbuf_pool_create("ethdev_mbuf_pool",
1117 NB_MBUF, MEMPOOL_CACHE_SIZE, 0,
1118 RTE_MBUF_DEFAULT_BUF_SIZE, rte_socket_id());
1119 if (ethdev_mbuf_mempool == NULL)
1120 rte_exit(EXIT_FAILURE, "Cannot create ethdev mbuf mempool\n");
1122 /* create the mbuf mempool for encoder output */
1123 bbdev_mbuf_mempool = rte_pktmbuf_pool_create("bbdev_mbuf_pool",
1124 NB_MBUF, MEMPOOL_CACHE_SIZE, 0,
1125 RTE_MBUF_DEFAULT_BUF_SIZE, rte_socket_id());
1126 if (bbdev_mbuf_mempool == NULL)
1127 rte_exit(EXIT_FAILURE, "Cannot create ethdev mbuf mempool\n");
1129 /* register mbuf field to store input pointer */
1130 input_dynfield_offset =
1131 rte_mbuf_dynfield_register(&input_dynfield_desc);
1132 if (input_dynfield_offset < 0)
1133 rte_exit(EXIT_FAILURE, "Cannot register mbuf field\n");
1135 /* initialize ports */
1136 ret = initialize_ports(&app_params, ethdev_mbuf_mempool);
1138 /* Check if all requested lcores are available */
1139 for (lcore_id = 0; lcore_id < 8 * sizeof(uint64_t); ++lcore_id)
1140 if (((1ULL << lcore_id) & app_params.enc_core_mask) ||
1141 ((1ULL << lcore_id) & app_params.dec_core_mask))
1142 if (!rte_lcore_is_enabled(lcore_id))
1143 rte_exit(EXIT_FAILURE,
1144 "Requested lcore_id %u is not enabled!\n",
1147 /* Start ethernet port */
1148 ret = rte_eth_dev_start(app_params.port_id);
1150 rte_exit(EXIT_FAILURE, "rte_eth_dev_start:err=%d, port=%u\n",
1151 ret, app_params.port_id);
1153 ret = check_port_link_status(app_params.port_id);
1157 /* start BBDevice and save BBDev queue IDs */
1158 enable_bbdev(&app_params);
1160 /* Initialize the port/queue configuration of each logical core */
1161 lcore_conf_init(&app_params, lcore_conf, bbdev_op_pools,
1162 bbdev_mbuf_mempool, enc_to_dec_ring, lcore_stats);
1164 stats_lcore.app_params = &app_params;
1165 stats_lcore.lconf = lcore_conf;
1167 RTE_LCORE_FOREACH_WORKER(lcore_id) {
1168 if (lcore_conf[lcore_id].core_type != 0)
1169 /* launch per-lcore processing loop on worker lcores */
1170 rte_eal_remote_launch(processing_loop,
1171 &lcore_conf[lcore_id], lcore_id);
1172 else if (!stats_thread_started) {
1173 /* launch statistics printing loop */
1174 rte_eal_remote_launch(stats_loop, &stats_lcore,
1176 stats_thread_started = true;
1180 if (!stats_thread_started &&
1181 lcore_conf[main_lcore_id].core_type != 0)
1182 rte_exit(EXIT_FAILURE,
1183 "Not enough lcores to run the statistics printing loop!");
1184 else if (lcore_conf[main_lcore_id].core_type != 0)
1185 processing_loop(&lcore_conf[main_lcore_id]);
1186 else if (!stats_thread_started)
1187 stats_loop(&stats_lcore);
1189 RTE_LCORE_FOREACH_WORKER(lcore_id) {
1190 ret |= rte_eal_wait_lcore(lcore_id);
1193 /* clean up the EAL */