eal: add device iterator interface
[dpdk.git] / examples / bbdev_app / main.c
1 /* SPDX-License-Identifier: BSD-3-Clause
2  * Copyright(c) 2017 Intel Corporation
3  */
4
5 #include <stdio.h>
6 #include <stdlib.h>
7 #include <string.h>
8 #include <stdint.h>
9 #include <inttypes.h>
10 #include <sys/types.h>
11 #include <sys/unistd.h>
12 #include <sys/queue.h>
13 #include <stdarg.h>
14 #include <ctype.h>
15 #include <errno.h>
16 #include <math.h>
17 #include <assert.h>
18 #include <getopt.h>
19 #include <signal.h>
20
21 #include "rte_atomic.h"
22 #include "rte_common.h"
23 #include "rte_eal.h"
24 #include "rte_cycles.h"
25 #include "rte_ether.h"
26 #include "rte_ethdev.h"
27 #include "rte_ip.h"
28 #include "rte_lcore.h"
29 #include "rte_malloc.h"
30 #include "rte_mbuf.h"
31 #include "rte_memory.h"
32 #include "rte_mempool.h"
33 #include "rte_log.h"
34 #include "rte_bbdev.h"
35 #include "rte_bbdev_op.h"
36
37 /* LLR values - negative value for '1' bit */
38 #define LLR_1_BIT 0x81
39 #define LLR_0_BIT 0x7F
40
41 #define MAX_PKT_BURST 32
42 #define NB_MBUF 8191
43 #define MEMPOOL_CACHE_SIZE 256
44
45 /* Hardcoded K value */
46 #define K 40
47 #define NCB (3 * RTE_ALIGN_CEIL(K + 4, 32))
48
49 #define CRC_24B_LEN 3
50
51 /* Configurable number of RX/TX ring descriptors */
52 #define RTE_TEST_RX_DESC_DEFAULT 128
53 #define RTE_TEST_TX_DESC_DEFAULT 512
54
55 #define BBDEV_ASSERT(a) do { \
56         if (!(a)) { \
57                 usage(prgname); \
58                 return -1; \
59         } \
60 } while (0)
61
62 static const struct rte_eth_conf port_conf = {
63         .rxmode = {
64                 .mq_mode = ETH_MQ_RX_NONE,
65                 .max_rx_pkt_len = ETHER_MAX_LEN,
66                 .split_hdr_size = 0,
67                 .offloads = DEV_RX_OFFLOAD_CRC_STRIP,
68         },
69         .txmode = {
70                 .mq_mode = ETH_MQ_TX_NONE,
71         },
72 };
73
74 struct rte_bbdev_op_turbo_enc def_op_enc = {
75         /* These values are arbitrarily put, and does not map to the real
76          * values for the data received from ethdev ports
77          */
78         .rv_index = 0,
79         .code_block_mode = 1,
80         .cb_params = {
81                 .k = K,
82         },
83         .op_flags = RTE_BBDEV_TURBO_CRC_24A_ATTACH
84 };
85
86 struct rte_bbdev_op_turbo_dec def_op_dec = {
87         /* These values are arbitrarily put, and does not map to the real
88          * values for the data received from ethdev ports
89          */
90         .code_block_mode = 1,
91         .cb_params = {
92                 .k = K,
93         },
94         .rv_index = 0,
95         .iter_max = 8,
96         .iter_min = 4,
97         .ext_scale = 15,
98         .num_maps = 0,
99         .op_flags = RTE_BBDEV_TURBO_NEG_LLR_1_BIT_IN
100 };
101
102 struct app_config_params {
103         /* Placeholders for app params */
104         uint16_t port_id;
105         uint16_t bbdev_id;
106         uint64_t enc_core_mask;
107         uint64_t dec_core_mask;
108
109         /* Values filled during init time */
110         uint16_t enc_queue_ids[RTE_MAX_LCORE];
111         uint16_t dec_queue_ids[RTE_MAX_LCORE];
112         uint16_t num_enc_cores;
113         uint16_t num_dec_cores;
114 };
115
116 struct lcore_statistics {
117         unsigned int enqueued;
118         unsigned int dequeued;
119         unsigned int rx_lost_packets;
120         unsigned int enc_to_dec_lost_packets;
121         unsigned int tx_lost_packets;
122 } __rte_cache_aligned;
123
124 /** each lcore configuration */
125 struct lcore_conf {
126         uint64_t core_type;
127
128         unsigned int port_id;
129         unsigned int rx_queue_id;
130         unsigned int tx_queue_id;
131
132         unsigned int bbdev_id;
133         unsigned int enc_queue_id;
134         unsigned int dec_queue_id;
135
136         uint8_t llr_temp_buf[NCB];
137
138         struct rte_mempool *bbdev_dec_op_pool;
139         struct rte_mempool *bbdev_enc_op_pool;
140         struct rte_mempool *enc_out_pool;
141         struct rte_ring *enc_to_dec_ring;
142
143         struct lcore_statistics *lcore_stats;
144 } __rte_cache_aligned;
145
146 struct stats_lcore_params {
147         struct lcore_conf *lconf;
148         struct app_config_params *app_params;
149 };
150
151
152 static const struct app_config_params def_app_config = {
153         .port_id = 0,
154         .bbdev_id = 0,
155         .enc_core_mask = 0x2,
156         .dec_core_mask = 0x4,
157         .num_enc_cores = 1,
158         .num_dec_cores = 1,
159 };
160
161 static rte_atomic16_t global_exit_flag;
162
163 /* display usage */
164 static inline void
165 usage(const char *prgname)
166 {
167         printf("%s [EAL options] "
168                         "  --\n"
169                         "  --enc_cores - number of encoding cores (default = 0x2)\n"
170                         "  --dec_cores - number of decoding cores (default = 0x4)\n"
171                         "  --port_id - Ethernet port ID (default = 0)\n"
172                         "  --bbdev_id - BBDev ID (default = 0)\n"
173                         "\n", prgname);
174 }
175
176 /* parse core mask */
177 static inline
178 uint16_t bbdev_parse_mask(const char *mask)
179 {
180         char *end = NULL;
181         unsigned long pm;
182
183         /* parse hexadecimal string */
184         pm = strtoul(mask, &end, 16);
185         if ((mask[0] == '\0') || (end == NULL) || (*end != '\0'))
186                 return 0;
187
188         return pm;
189 }
190
191 /* parse core mask */
192 static inline
193 uint16_t bbdev_parse_number(const char *mask)
194 {
195         char *end = NULL;
196         unsigned long pm;
197
198         /* parse hexadecimal string */
199         pm = strtoul(mask, &end, 10);
200         if ((mask[0] == '\0') || (end == NULL) || (*end != '\0'))
201                 return 0;
202
203         return pm;
204 }
205
206 static int
207 bbdev_parse_args(int argc, char **argv,
208                 struct app_config_params *app_params)
209 {
210         int optind = 0;
211         int opt;
212         int opt_indx = 0;
213         char *prgname = argv[0];
214
215         static struct option lgopts[] = {
216                 { "enc_core_mask", required_argument, 0, 'e' },
217                 { "dec_core_mask", required_argument, 0, 'd' },
218                 { "port_id", required_argument, 0, 'p' },
219                 { "bbdev_id", required_argument, 0, 'b' },
220                 { NULL, 0, 0, 0 }
221         };
222
223         BBDEV_ASSERT(argc != 0);
224         BBDEV_ASSERT(argv != NULL);
225         BBDEV_ASSERT(app_params != NULL);
226
227         while ((opt = getopt_long(argc, argv, "e:d:p:b:", lgopts, &opt_indx)) !=
228                 EOF) {
229                 switch (opt) {
230                 case 'e':
231                         app_params->enc_core_mask =
232                                 bbdev_parse_mask(optarg);
233                         if (app_params->enc_core_mask == 0) {
234                                 usage(prgname);
235                                 return -1;
236                         }
237                         app_params->num_enc_cores =
238                                 __builtin_popcount(app_params->enc_core_mask);
239                         break;
240
241                 case 'd':
242                         app_params->dec_core_mask =
243                                 bbdev_parse_mask(optarg);
244                         if (app_params->dec_core_mask == 0) {
245                                 usage(prgname);
246                                 return -1;
247                         }
248                         app_params->num_dec_cores =
249                                 __builtin_popcount(app_params->dec_core_mask);
250                         break;
251
252                 case 'p':
253                         app_params->port_id = bbdev_parse_number(optarg);
254                         break;
255
256                 case 'b':
257                         app_params->bbdev_id = bbdev_parse_number(optarg);
258                         break;
259
260                 default:
261                         usage(prgname);
262                         return -1;
263                 }
264         }
265         optind = 0;
266         return optind;
267 }
268
269 static void
270 signal_handler(int signum)
271 {
272         printf("\nSignal %d received\n", signum);
273         rte_atomic16_set(&global_exit_flag, 1);
274 }
275
276 static void
277 print_mac(unsigned int portid, struct ether_addr *bbdev_ports_eth_address)
278 {
279         printf("Port %u, MAC address: %02X:%02X:%02X:%02X:%02X:%02X\n\n",
280                         (unsigned int) portid,
281                         bbdev_ports_eth_address->addr_bytes[0],
282                         bbdev_ports_eth_address->addr_bytes[1],
283                         bbdev_ports_eth_address->addr_bytes[2],
284                         bbdev_ports_eth_address->addr_bytes[3],
285                         bbdev_ports_eth_address->addr_bytes[4],
286                         bbdev_ports_eth_address->addr_bytes[5]);
287 }
288
289 static inline void
290 pktmbuf_free_bulk(struct rte_mbuf **mbufs, unsigned int nb_to_free)
291 {
292         unsigned int i;
293         for (i = 0; i < nb_to_free; ++i)
294                 rte_pktmbuf_free(mbufs[i]);
295 }
296
297 static inline void
298 pktmbuf_userdata_free_bulk(struct rte_mbuf **mbufs, unsigned int nb_to_free)
299 {
300         unsigned int i;
301         for (i = 0; i < nb_to_free; ++i) {
302                 struct rte_mbuf *rx_pkt = mbufs[i]->userdata;
303                 rte_pktmbuf_free(rx_pkt);
304                 rte_pktmbuf_free(mbufs[i]);
305         }
306 }
307
308 /* Check the link status of all ports in up to 9s, and print them finally */
309 static int
310 check_port_link_status(uint16_t port_id)
311 {
312 #define CHECK_INTERVAL 100 /* 100ms */
313 #define MAX_CHECK_TIME 90 /* 9s (90 * 100ms) in total */
314         uint8_t count;
315         struct rte_eth_link link;
316
317         printf("\nChecking link status.");
318         fflush(stdout);
319
320         for (count = 0; count <= MAX_CHECK_TIME &&
321                         !rte_atomic16_read(&global_exit_flag); count++) {
322                 memset(&link, 0, sizeof(link));
323                 rte_eth_link_get_nowait(port_id, &link);
324
325                 if (link.link_status) {
326                         const char *dp = (link.link_duplex ==
327                                 ETH_LINK_FULL_DUPLEX) ?
328                                 "full-duplex" : "half-duplex";
329                         printf("\nPort %u Link Up - speed %u Mbps - %s\n",
330                                 port_id, link.link_speed, dp);
331                         return 0;
332                 }
333                 printf(".");
334                 fflush(stdout);
335                 rte_delay_ms(CHECK_INTERVAL);
336         }
337
338         printf("\nPort %d Link Down\n", port_id);
339         return 0;
340 }
341
342 static inline void
343 add_ether_hdr(struct rte_mbuf *pkt_src, struct rte_mbuf *pkt_dst)
344 {
345         struct ether_hdr *eth_from;
346         struct ether_hdr *eth_to;
347
348         eth_from = rte_pktmbuf_mtod(pkt_src, struct ether_hdr *);
349         eth_to = rte_pktmbuf_mtod(pkt_dst, struct ether_hdr *);
350
351         /* copy header */
352         rte_memcpy(eth_to, eth_from, sizeof(struct ether_hdr));
353 }
354
355 static inline void
356 add_awgn(struct rte_mbuf **mbufs, uint16_t num_pkts)
357 {
358         RTE_SET_USED(mbufs);
359         RTE_SET_USED(num_pkts);
360 }
361
362 /* Encoder output to Decoder input adapter. The Decoder accepts only soft input
363  * so each bit of the encoder output must be translated into one byte of LLR. If
364  * Sub-block Deinterleaver is bypassed, which is the case, the padding bytes
365  * must additionally be insterted at the end of each sub-block.
366  */
367 static inline void
368 transform_enc_out_dec_in(struct rte_mbuf **mbufs, uint8_t *temp_buf,
369                 uint16_t num_pkts, uint16_t k)
370 {
371         uint16_t i, l, j;
372         uint16_t start_bit_idx;
373         uint16_t out_idx;
374         uint16_t d = k + 4;
375         uint16_t kpi = RTE_ALIGN_CEIL(d, 32);
376         uint16_t nd = kpi - d;
377         uint16_t ncb = 3 * kpi;
378
379         for (i = 0; i < num_pkts; ++i) {
380                 uint16_t pkt_data_len = rte_pktmbuf_data_len(mbufs[i]) -
381                                 sizeof(struct ether_hdr);
382
383                 /* Resize the packet if needed */
384                 if (pkt_data_len < ncb) {
385                         char *data = rte_pktmbuf_append(mbufs[i],
386                                         ncb - pkt_data_len);
387                         if (data == NULL)
388                                 printf(
389                                         "Not enough space in decoder input packet");
390                 }
391
392                 /* Translate each bit into 1 LLR byte. */
393                 start_bit_idx = 0;
394                 out_idx = 0;
395                 for (j = 0; j < 3; ++j) {
396                         for (l = start_bit_idx; l < start_bit_idx + d; ++l) {
397                                 uint8_t *data = rte_pktmbuf_mtod_offset(
398                                         mbufs[i], uint8_t *,
399                                         sizeof(struct ether_hdr) + (l >> 3));
400                                 if (*data & (0x80 >> (l & 7)))
401                                         temp_buf[out_idx] = LLR_1_BIT;
402                                 else
403                                         temp_buf[out_idx] = LLR_0_BIT;
404                                 ++out_idx;
405                         }
406                         /* Padding bytes should be at the end of the sub-block.
407                          */
408                         memset(&temp_buf[out_idx], 0, nd);
409                         out_idx += nd;
410                         start_bit_idx += d;
411                 }
412
413                 rte_memcpy(rte_pktmbuf_mtod_offset(mbufs[i], uint8_t *,
414                                 sizeof(struct ether_hdr)), temp_buf, ncb);
415         }
416 }
417
418 static inline void
419 verify_data(struct rte_mbuf **mbufs, uint16_t num_pkts)
420 {
421         uint16_t i;
422         for (i = 0; i < num_pkts; ++i) {
423                 struct rte_mbuf *out = mbufs[i];
424                 struct rte_mbuf *in = out->userdata;
425
426                 if (memcmp(rte_pktmbuf_mtod_offset(in, uint8_t *,
427                                 sizeof(struct ether_hdr)),
428                                 rte_pktmbuf_mtod_offset(out, uint8_t *,
429                                 sizeof(struct ether_hdr)),
430                                 K / 8 - CRC_24B_LEN))
431                         printf("Input and output buffers are not equal!\n");
432         }
433 }
434
435 static int
436 initialize_ports(struct app_config_params *app_params,
437                 struct rte_mempool *ethdev_mbuf_mempool)
438 {
439         int ret;
440         uint16_t port_id = app_params->port_id;
441         uint16_t q;
442         /* ethernet addresses of ports */
443         struct ether_addr bbdev_port_eth_addr;
444
445         /* initialize ports */
446         printf("\nInitializing port %u...\n", app_params->port_id);
447         ret = rte_eth_dev_configure(port_id, app_params->num_enc_cores,
448                 app_params->num_dec_cores, &port_conf);
449
450         if (ret < 0) {
451                 printf("Cannot configure device: err=%d, port=%u\n",
452                         ret, port_id);
453                 return -1;
454         }
455
456         /* initialize RX queues for encoder */
457         for (q = 0; q < app_params->num_enc_cores; q++) {
458                 ret = rte_eth_rx_queue_setup(port_id, q,
459                         RTE_TEST_RX_DESC_DEFAULT,
460                         rte_eth_dev_socket_id(port_id),
461                         NULL, ethdev_mbuf_mempool);
462                 if (ret < 0) {
463                         printf("rte_eth_rx_queue_setup: err=%d, queue=%u\n",
464                                 ret, q);
465                         return -1;
466                 }
467         }
468         /* initialize TX queues for decoder */
469         for (q = 0; q < app_params->num_dec_cores; q++) {
470                 ret = rte_eth_tx_queue_setup(port_id, q,
471                         RTE_TEST_TX_DESC_DEFAULT,
472                         rte_eth_dev_socket_id(port_id), NULL);
473                 if (ret < 0) {
474                         printf("rte_eth_tx_queue_setup: err=%d, queue=%u\n",
475                                 ret, q);
476                         return -1;
477                 }
478         }
479
480         rte_eth_promiscuous_enable(port_id);
481
482         rte_eth_macaddr_get(port_id, &bbdev_port_eth_addr);
483         print_mac(port_id, &bbdev_port_eth_addr);
484
485         return 0;
486 }
487
488 static void
489 lcore_conf_init(struct app_config_params *app_params,
490                 struct lcore_conf *lcore_conf,
491                 struct rte_mempool **bbdev_op_pools,
492                 struct rte_mempool *bbdev_mbuf_mempool,
493                 struct rte_ring *enc_to_dec_ring,
494                 struct lcore_statistics *lcore_stats)
495 {
496         unsigned int lcore_id;
497         struct lcore_conf *lconf;
498         uint16_t rx_queue_id = 0;
499         uint16_t tx_queue_id = 0;
500         uint16_t enc_q_id = 0;
501         uint16_t dec_q_id = 0;
502
503         /* Configure lcores */
504         for (lcore_id = 0; lcore_id < 8 * sizeof(uint64_t); ++lcore_id) {
505                 lconf = &lcore_conf[lcore_id];
506                 lconf->core_type = 0;
507
508                 if ((1ULL << lcore_id) & app_params->enc_core_mask) {
509                         lconf->core_type |= (1 << RTE_BBDEV_OP_TURBO_ENC);
510                         lconf->rx_queue_id = rx_queue_id++;
511                         lconf->enc_queue_id =
512                                         app_params->enc_queue_ids[enc_q_id++];
513                 }
514
515                 if ((1ULL << lcore_id) & app_params->dec_core_mask) {
516                         lconf->core_type |= (1 << RTE_BBDEV_OP_TURBO_DEC);
517                         lconf->tx_queue_id = tx_queue_id++;
518                         lconf->dec_queue_id =
519                                         app_params->dec_queue_ids[dec_q_id++];
520                 }
521
522                 lconf->bbdev_enc_op_pool =
523                                 bbdev_op_pools[RTE_BBDEV_OP_TURBO_ENC];
524                 lconf->bbdev_dec_op_pool =
525                                 bbdev_op_pools[RTE_BBDEV_OP_TURBO_DEC];
526                 lconf->bbdev_id = app_params->bbdev_id;
527                 lconf->port_id = app_params->port_id;
528                 lconf->enc_out_pool = bbdev_mbuf_mempool;
529                 lconf->enc_to_dec_ring = enc_to_dec_ring;
530                 lconf->lcore_stats = &lcore_stats[lcore_id];
531         }
532 }
533
534 static void
535 print_lcore_stats(struct lcore_statistics *lstats, unsigned int lcore_id)
536 {
537         static const char *stats_border = "_______";
538
539         printf("\nLcore %d: %s enqueued count:\t\t%u\n",
540                         lcore_id, stats_border, lstats->enqueued);
541         printf("Lcore %d: %s dequeued count:\t\t%u\n",
542                         lcore_id, stats_border, lstats->dequeued);
543         printf("Lcore %d: %s RX lost packets count:\t\t%u\n",
544                         lcore_id, stats_border, lstats->rx_lost_packets);
545         printf("Lcore %d: %s encoder-to-decoder lost count:\t%u\n",
546                         lcore_id, stats_border,
547                         lstats->enc_to_dec_lost_packets);
548         printf("Lcore %d: %s TX lost packets count:\t\t%u\n",
549                         lcore_id, stats_border, lstats->tx_lost_packets);
550 }
551
552 static void
553 print_stats(struct stats_lcore_params *stats_lcore)
554 {
555         unsigned int l_id;
556         unsigned int bbdev_id = stats_lcore->app_params->bbdev_id;
557         unsigned int port_id = stats_lcore->app_params->port_id;
558         int len, ret, i;
559
560         struct rte_eth_xstat *xstats;
561         struct rte_eth_xstat_name *xstats_names;
562         struct rte_bbdev_stats bbstats;
563         static const char *stats_border = "_______";
564
565         const char clr[] = { 27, '[', '2', 'J', '\0' };
566         const char topLeft[] = { 27, '[', '1', ';', '1', 'H', '\0' };
567
568         /* Clear screen and move to top left */
569         printf("%s%s", clr, topLeft);
570
571         printf("PORT STATISTICS:\n================\n");
572         len = rte_eth_xstats_get(port_id, NULL, 0);
573         if (len < 0)
574                 rte_exit(EXIT_FAILURE,
575                                 "rte_eth_xstats_get(%u) failed: %d", port_id,
576                                 len);
577
578         xstats = calloc(len, sizeof(*xstats));
579         if (xstats == NULL)
580                 rte_exit(EXIT_FAILURE,
581                                 "Failed to calloc memory for xstats");
582
583         ret = rte_eth_xstats_get(port_id, xstats, len);
584         if (ret < 0 || ret > len) {
585                 free(xstats);
586                 rte_exit(EXIT_FAILURE,
587                                 "rte_eth_xstats_get(%u) len%i failed: %d",
588                                 port_id, len, ret);
589         }
590
591         xstats_names = calloc(len, sizeof(*xstats_names));
592         if (xstats_names == NULL) {
593                 free(xstats);
594                 rte_exit(EXIT_FAILURE,
595                                 "Failed to calloc memory for xstats_names");
596         }
597
598         ret = rte_eth_xstats_get_names(port_id, xstats_names, len);
599         if (ret < 0 || ret > len) {
600                 free(xstats);
601                 free(xstats_names);
602                 rte_exit(EXIT_FAILURE,
603                                 "rte_eth_xstats_get_names(%u) len%i failed: %d",
604                                 port_id, len, ret);
605         }
606
607         for (i = 0; i < len; i++) {
608                 if (xstats[i].value > 0)
609                         printf("Port %u: %s %s:\t\t%"PRIu64"\n",
610                                         port_id, stats_border,
611                                         xstats_names[i].name,
612                                         xstats[i].value);
613         }
614
615         ret = rte_bbdev_stats_get(bbdev_id, &bbstats);
616         if (ret < 0) {
617                 free(xstats);
618                 free(xstats_names);
619                 rte_exit(EXIT_FAILURE,
620                                 "ERROR(%d): Failure to get BBDEV %u statistics\n",
621                                 ret, bbdev_id);
622         }
623
624         printf("\nBBDEV STATISTICS:\n=================\n");
625         printf("BBDEV %u: %s enqueue count:\t\t%"PRIu64"\n",
626                         bbdev_id, stats_border,
627                         bbstats.enqueued_count);
628         printf("BBDEV %u: %s dequeue count:\t\t%"PRIu64"\n",
629                         bbdev_id, stats_border,
630                         bbstats.dequeued_count);
631         printf("BBDEV %u: %s enqueue error count:\t\t%"PRIu64"\n",
632                         bbdev_id, stats_border,
633                         bbstats.enqueue_err_count);
634         printf("BBDEV %u: %s dequeue error count:\t\t%"PRIu64"\n\n",
635                         bbdev_id, stats_border,
636                         bbstats.dequeue_err_count);
637
638         printf("LCORE STATISTICS:\n=================\n");
639         for (l_id = 0; l_id < RTE_MAX_LCORE; ++l_id) {
640                 if (stats_lcore->lconf[l_id].core_type == 0)
641                         continue;
642                 print_lcore_stats(stats_lcore->lconf[l_id].lcore_stats, l_id);
643         }
644
645         free(xstats);
646         free(xstats_names);
647 }
648
649 static int
650 stats_loop(void *arg)
651 {
652         struct stats_lcore_params *stats_lcore = arg;
653
654         while (!rte_atomic16_read(&global_exit_flag)) {
655                 print_stats(stats_lcore);
656                 rte_delay_ms(500);
657         }
658
659         return 0;
660 }
661
662 static inline void
663 run_encoding(struct lcore_conf *lcore_conf)
664 {
665         uint16_t i;
666         uint16_t port_id, rx_queue_id;
667         uint16_t bbdev_id, enc_queue_id;
668         uint16_t nb_rx, nb_enq, nb_deq, nb_sent;
669         struct rte_mbuf *rx_pkts_burst[MAX_PKT_BURST];
670         struct rte_mbuf *enc_out_pkts[MAX_PKT_BURST];
671         struct rte_bbdev_enc_op *bbdev_ops_burst[MAX_PKT_BURST];
672         struct lcore_statistics *lcore_stats;
673         struct rte_mempool *bbdev_op_pool, *enc_out_pool;
674         struct rte_ring *enc_to_dec_ring;
675         const int in_data_len = (def_op_enc.cb_params.k / 8) - CRC_24B_LEN;
676
677         lcore_stats = lcore_conf->lcore_stats;
678         port_id = lcore_conf->port_id;
679         rx_queue_id = lcore_conf->rx_queue_id;
680         bbdev_id = lcore_conf->bbdev_id;
681         enc_queue_id = lcore_conf->enc_queue_id;
682         bbdev_op_pool = lcore_conf->bbdev_enc_op_pool;
683         enc_out_pool = lcore_conf->enc_out_pool;
684         enc_to_dec_ring = lcore_conf->enc_to_dec_ring;
685
686         /* Read packet from RX queues*/
687         nb_rx = rte_eth_rx_burst(port_id, rx_queue_id, rx_pkts_burst,
688                         MAX_PKT_BURST);
689         if (!nb_rx)
690                 return;
691
692         if (unlikely(rte_mempool_get_bulk(enc_out_pool, (void **)enc_out_pkts,
693                         nb_rx) != 0)) {
694                 pktmbuf_free_bulk(rx_pkts_burst, nb_rx);
695                 lcore_stats->rx_lost_packets += nb_rx;
696                 return;
697         }
698
699         if (unlikely(rte_bbdev_enc_op_alloc_bulk(bbdev_op_pool, bbdev_ops_burst,
700                         nb_rx) != 0)) {
701                 pktmbuf_free_bulk(enc_out_pkts, nb_rx);
702                 pktmbuf_free_bulk(rx_pkts_burst, nb_rx);
703                 lcore_stats->rx_lost_packets += nb_rx;
704                 return;
705         }
706
707         for (i = 0; i < nb_rx; i++) {
708                 char *data;
709                 const uint16_t pkt_data_len =
710                                 rte_pktmbuf_data_len(rx_pkts_burst[i]) -
711                                 sizeof(struct ether_hdr);
712                 /* save input mbuf pointer for later comparison */
713                 enc_out_pkts[i]->userdata = rx_pkts_burst[i];
714
715                 /* copy ethernet header */
716                 rte_pktmbuf_reset(enc_out_pkts[i]);
717                 data = rte_pktmbuf_append(enc_out_pkts[i],
718                                 sizeof(struct ether_hdr));
719                 if (data == NULL) {
720                         printf(
721                                 "Not enough space for ethernet header in encoder output mbuf\n");
722                         continue;
723                 }
724                 add_ether_hdr(rx_pkts_burst[i], enc_out_pkts[i]);
725
726                 /* set op */
727                 bbdev_ops_burst[i]->turbo_enc = def_op_enc;
728
729                 bbdev_ops_burst[i]->turbo_enc.input.data =
730                                 rx_pkts_burst[i];
731                 bbdev_ops_burst[i]->turbo_enc.input.offset =
732                                 sizeof(struct ether_hdr);
733                 /* Encoder will attach the CRC24B, adjust the length */
734                 bbdev_ops_burst[i]->turbo_enc.input.length = in_data_len;
735
736                 if (in_data_len < pkt_data_len)
737                         rte_pktmbuf_trim(rx_pkts_burst[i], pkt_data_len -
738                                         in_data_len);
739                 else if (in_data_len > pkt_data_len) {
740                         data = rte_pktmbuf_append(rx_pkts_burst[i],
741                                         in_data_len - pkt_data_len);
742                         if (data == NULL)
743                                 printf(
744                                         "Not enough storage in mbuf to perform the encoding\n");
745                 }
746
747                 bbdev_ops_burst[i]->turbo_enc.output.data =
748                                 enc_out_pkts[i];
749                 bbdev_ops_burst[i]->turbo_enc.output.offset =
750                                 sizeof(struct ether_hdr);
751         }
752
753         /* Enqueue packets on BBDevice */
754         nb_enq = rte_bbdev_enqueue_enc_ops(bbdev_id, enc_queue_id,
755                         bbdev_ops_burst, nb_rx);
756         if (unlikely(nb_enq < nb_rx)) {
757                 pktmbuf_userdata_free_bulk(&enc_out_pkts[nb_enq],
758                                 nb_rx - nb_enq);
759                 rte_bbdev_enc_op_free_bulk(&bbdev_ops_burst[nb_enq],
760                                 nb_rx - nb_enq);
761                 lcore_stats->rx_lost_packets += nb_rx - nb_enq;
762
763                 if (!nb_enq)
764                         return;
765         }
766
767         lcore_stats->enqueued += nb_enq;
768
769         /* Dequeue packets from bbdev device*/
770         nb_deq = 0;
771         do {
772                 nb_deq += rte_bbdev_dequeue_enc_ops(bbdev_id, enc_queue_id,
773                                 &bbdev_ops_burst[nb_deq], nb_enq - nb_deq);
774         } while (unlikely(nb_deq < nb_enq));
775
776         lcore_stats->dequeued += nb_deq;
777
778         /* Generate and add AWGN */
779         add_awgn(enc_out_pkts, nb_deq);
780
781         rte_bbdev_enc_op_free_bulk(bbdev_ops_burst, nb_deq);
782
783         /* Enqueue packets to encoder-to-decoder ring */
784         nb_sent = rte_ring_enqueue_burst(enc_to_dec_ring, (void **)enc_out_pkts,
785                         nb_deq, NULL);
786         if (unlikely(nb_sent < nb_deq)) {
787                 pktmbuf_userdata_free_bulk(&enc_out_pkts[nb_sent],
788                                 nb_deq - nb_sent);
789                 lcore_stats->enc_to_dec_lost_packets += nb_deq - nb_sent;
790         }
791 }
792
793 static void
794 run_decoding(struct lcore_conf *lcore_conf)
795 {
796         uint16_t i;
797         uint16_t port_id, tx_queue_id;
798         uint16_t bbdev_id, bbdev_queue_id;
799         uint16_t nb_recv, nb_enq, nb_deq, nb_tx;
800         uint8_t *llr_temp_buf;
801         struct rte_mbuf *recv_pkts_burst[MAX_PKT_BURST];
802         struct rte_bbdev_dec_op *bbdev_ops_burst[MAX_PKT_BURST];
803         struct lcore_statistics *lcore_stats;
804         struct rte_mempool *bbdev_op_pool;
805         struct rte_ring *enc_to_dec_ring;
806
807         lcore_stats = lcore_conf->lcore_stats;
808         port_id = lcore_conf->port_id;
809         tx_queue_id = lcore_conf->tx_queue_id;
810         bbdev_id = lcore_conf->bbdev_id;
811         bbdev_queue_id = lcore_conf->dec_queue_id;
812         bbdev_op_pool = lcore_conf->bbdev_dec_op_pool;
813         enc_to_dec_ring = lcore_conf->enc_to_dec_ring;
814         llr_temp_buf = lcore_conf->llr_temp_buf;
815
816         /* Dequeue packets from the ring */
817         nb_recv = rte_ring_dequeue_burst(enc_to_dec_ring,
818                         (void **)recv_pkts_burst, MAX_PKT_BURST, NULL);
819         if (!nb_recv)
820                 return;
821
822         if (unlikely(rte_bbdev_dec_op_alloc_bulk(bbdev_op_pool, bbdev_ops_burst,
823                         nb_recv) != 0)) {
824                 pktmbuf_userdata_free_bulk(recv_pkts_burst, nb_recv);
825                 lcore_stats->rx_lost_packets += nb_recv;
826                 return;
827         }
828
829         transform_enc_out_dec_in(recv_pkts_burst, llr_temp_buf, nb_recv,
830                         def_op_dec.cb_params.k);
831
832         for (i = 0; i < nb_recv; i++) {
833                 /* set op */
834                 bbdev_ops_burst[i]->turbo_dec = def_op_dec;
835
836                 bbdev_ops_burst[i]->turbo_dec.input.data = recv_pkts_burst[i];
837                 bbdev_ops_burst[i]->turbo_dec.input.offset =
838                                 sizeof(struct ether_hdr);
839                 bbdev_ops_burst[i]->turbo_dec.input.length =
840                                 rte_pktmbuf_data_len(recv_pkts_burst[i])
841                                 - sizeof(struct ether_hdr);
842
843                 bbdev_ops_burst[i]->turbo_dec.hard_output.data =
844                                 recv_pkts_burst[i];
845                 bbdev_ops_burst[i]->turbo_dec.hard_output.offset =
846                                 sizeof(struct ether_hdr);
847         }
848
849         /* Enqueue packets on BBDevice */
850         nb_enq = rte_bbdev_enqueue_dec_ops(bbdev_id, bbdev_queue_id,
851                         bbdev_ops_burst, nb_recv);
852         if (unlikely(nb_enq < nb_recv)) {
853                 pktmbuf_userdata_free_bulk(&recv_pkts_burst[nb_enq],
854                                 nb_recv - nb_enq);
855                 rte_bbdev_dec_op_free_bulk(&bbdev_ops_burst[nb_enq],
856                                 nb_recv - nb_enq);
857                 lcore_stats->rx_lost_packets += nb_recv - nb_enq;
858
859                 if (!nb_enq)
860                         return;
861         }
862
863         lcore_stats->enqueued += nb_enq;
864
865         /* Dequeue packets from BBDevice */
866         nb_deq = 0;
867         do {
868                 nb_deq += rte_bbdev_dequeue_dec_ops(bbdev_id, bbdev_queue_id,
869                                 &bbdev_ops_burst[nb_deq], nb_enq - nb_deq);
870         } while (unlikely(nb_deq < nb_enq));
871
872         lcore_stats->dequeued += nb_deq;
873
874         rte_bbdev_dec_op_free_bulk(bbdev_ops_burst, nb_deq);
875
876         verify_data(recv_pkts_burst, nb_deq);
877
878         /* Free the RX mbufs after verification */
879         for (i = 0; i < nb_deq; ++i)
880                 rte_pktmbuf_free(recv_pkts_burst[i]->userdata);
881
882         /* Transmit the packets */
883         nb_tx = rte_eth_tx_burst(port_id, tx_queue_id, recv_pkts_burst, nb_deq);
884         if (unlikely(nb_tx < nb_deq)) {
885                 pktmbuf_userdata_free_bulk(&recv_pkts_burst[nb_tx],
886                                 nb_deq - nb_tx);
887                 lcore_stats->tx_lost_packets += nb_deq - nb_tx;
888         }
889 }
890
891 static int
892 processing_loop(void *arg)
893 {
894         struct lcore_conf *lcore_conf = arg;
895         const bool run_encoder = (lcore_conf->core_type &
896                         (1 << RTE_BBDEV_OP_TURBO_ENC));
897         const bool run_decoder = (lcore_conf->core_type &
898                         (1 << RTE_BBDEV_OP_TURBO_DEC));
899
900         while (!rte_atomic16_read(&global_exit_flag)) {
901                 if (run_encoder)
902                         run_encoding(lcore_conf);
903                 if (run_decoder)
904                         run_decoding(lcore_conf);
905         }
906
907         return 0;
908 }
909
910 static int
911 prepare_bbdev_device(unsigned int dev_id, struct rte_bbdev_info *info,
912                 struct app_config_params *app_params)
913 {
914         int ret;
915         unsigned int q_id, dec_q_id, enc_q_id;
916         struct rte_bbdev_queue_conf qconf = {0};
917         uint16_t dec_qs_nb = app_params->num_dec_cores;
918         uint16_t enc_qs_nb = app_params->num_enc_cores;
919         uint16_t tot_qs = dec_qs_nb + enc_qs_nb;
920
921         ret = rte_bbdev_setup_queues(dev_id, tot_qs, info->socket_id);
922         if (ret < 0)
923                 rte_exit(EXIT_FAILURE,
924                                 "ERROR(%d): BBDEV %u not configured properly\n",
925                                 ret, dev_id);
926
927         /* setup device DEC queues */
928         qconf.socket = info->socket_id;
929         qconf.queue_size = info->drv.queue_size_lim;
930         qconf.op_type = RTE_BBDEV_OP_TURBO_DEC;
931
932         for (q_id = 0, dec_q_id = 0; q_id < dec_qs_nb; q_id++) {
933                 ret = rte_bbdev_queue_configure(dev_id, q_id, &qconf);
934                 if (ret < 0)
935                         rte_exit(EXIT_FAILURE,
936                                         "ERROR(%d): BBDEV %u DEC queue %u not configured properly\n",
937                                         ret, dev_id, q_id);
938                 app_params->dec_queue_ids[dec_q_id++] = q_id;
939         }
940
941         /* setup device ENC queues */
942         qconf.op_type = RTE_BBDEV_OP_TURBO_ENC;
943
944         for (q_id = dec_qs_nb, enc_q_id = 0; q_id < tot_qs; q_id++) {
945                 ret = rte_bbdev_queue_configure(dev_id, q_id, &qconf);
946                 if (ret < 0)
947                         rte_exit(EXIT_FAILURE,
948                                         "ERROR(%d): BBDEV %u ENC queue %u not configured properly\n",
949                                         ret, dev_id, q_id);
950                 app_params->enc_queue_ids[enc_q_id++] = q_id;
951         }
952
953         ret = rte_bbdev_start(dev_id);
954
955         if (ret != 0)
956                 rte_exit(EXIT_FAILURE, "ERROR(%d): BBDEV %u not started\n",
957                         ret, dev_id);
958
959         printf("BBdev %u started\n", dev_id);
960
961         return 0;
962 }
963
964 static inline bool
965 check_matching_capabilities(uint64_t mask, uint64_t required_mask)
966 {
967         return (mask & required_mask) == required_mask;
968 }
969
970 static void
971 enable_bbdev(struct app_config_params *app_params)
972 {
973         struct rte_bbdev_info dev_info;
974         const struct rte_bbdev_op_cap *op_cap;
975         uint16_t bbdev_id = app_params->bbdev_id;
976         bool encoder_capable = false;
977         bool decoder_capable = false;
978
979         rte_bbdev_info_get(bbdev_id, &dev_info);
980         op_cap = dev_info.drv.capabilities;
981
982         while (op_cap->type != RTE_BBDEV_OP_NONE) {
983                 if (op_cap->type == RTE_BBDEV_OP_TURBO_ENC) {
984                         if (check_matching_capabilities(
985                                         op_cap->cap.turbo_enc.capability_flags,
986                                         def_op_enc.op_flags))
987                                 encoder_capable = true;
988                 }
989
990                 if (op_cap->type == RTE_BBDEV_OP_TURBO_DEC) {
991                         if (check_matching_capabilities(
992                                         op_cap->cap.turbo_dec.capability_flags,
993                                         def_op_dec.op_flags))
994                                 decoder_capable = true;
995                 }
996
997                 op_cap++;
998         }
999
1000         if (encoder_capable == false)
1001                 rte_exit(EXIT_FAILURE,
1002                         "The specified BBDev %u doesn't have required encoder capabilities!\n",
1003                         bbdev_id);
1004         if (decoder_capable == false)
1005                 rte_exit(EXIT_FAILURE,
1006                         "The specified BBDev %u doesn't have required decoder capabilities!\n",
1007                         bbdev_id);
1008
1009         prepare_bbdev_device(bbdev_id, &dev_info, app_params);
1010 }
1011
1012 int
1013 main(int argc, char **argv)
1014 {
1015         int ret;
1016         unsigned int nb_bbdevs, flags, lcore_id;
1017         void *sigret;
1018         struct app_config_params app_params = def_app_config;
1019         struct rte_mempool *ethdev_mbuf_mempool, *bbdev_mbuf_mempool;
1020         struct rte_mempool *bbdev_op_pools[RTE_BBDEV_OP_TYPE_COUNT];
1021         struct lcore_conf lcore_conf[RTE_MAX_LCORE] = { {0} };
1022         struct lcore_statistics lcore_stats[RTE_MAX_LCORE] = { {0} };
1023         struct stats_lcore_params stats_lcore;
1024         struct rte_ring *enc_to_dec_ring;
1025         bool stats_thread_started = false;
1026         unsigned int master_lcore_id = rte_get_master_lcore();
1027
1028         rte_atomic16_init(&global_exit_flag);
1029
1030         sigret = signal(SIGTERM, signal_handler);
1031         if (sigret == SIG_ERR)
1032                 rte_exit(EXIT_FAILURE, "signal(%d, ...) failed", SIGTERM);
1033
1034         sigret = signal(SIGINT, signal_handler);
1035         if (sigret == SIG_ERR)
1036                 rte_exit(EXIT_FAILURE, "signal(%d, ...) failed", SIGINT);
1037
1038         ret = rte_eal_init(argc, argv);
1039         if (ret < 0)
1040                 rte_exit(EXIT_FAILURE, "Invalid EAL arguments\n");
1041
1042         argc -= ret;
1043         argv += ret;
1044
1045         /* parse application arguments (after the EAL ones) */
1046         ret = bbdev_parse_args(argc, argv, &app_params);
1047         if (ret < 0)
1048                 rte_exit(EXIT_FAILURE, "Invalid BBDEV arguments\n");
1049
1050         /*create bbdev op pools*/
1051         bbdev_op_pools[RTE_BBDEV_OP_TURBO_DEC] =
1052                         rte_bbdev_op_pool_create("bbdev_op_pool_dec",
1053                         RTE_BBDEV_OP_TURBO_DEC, NB_MBUF, 128, rte_socket_id());
1054         bbdev_op_pools[RTE_BBDEV_OP_TURBO_ENC] =
1055                         rte_bbdev_op_pool_create("bbdev_op_pool_enc",
1056                         RTE_BBDEV_OP_TURBO_ENC, NB_MBUF, 128, rte_socket_id());
1057
1058         if ((bbdev_op_pools[RTE_BBDEV_OP_TURBO_DEC] == NULL) ||
1059                         (bbdev_op_pools[RTE_BBDEV_OP_TURBO_ENC] == NULL))
1060                 rte_exit(EXIT_FAILURE, "Cannot create bbdev op pools\n");
1061
1062         /* Create encoder to decoder ring */
1063         flags = (app_params.num_enc_cores == 1) ? RING_F_SP_ENQ : 0;
1064         if (app_params.num_dec_cores == 1)
1065                 flags |= RING_F_SC_DEQ;
1066
1067         enc_to_dec_ring = rte_ring_create("enc_to_dec_ring",
1068                 rte_align32pow2(NB_MBUF), rte_socket_id(), flags);
1069
1070         /* Get the number of available bbdev devices */
1071         nb_bbdevs = rte_bbdev_count();
1072         if (nb_bbdevs <= app_params.bbdev_id)
1073                 rte_exit(EXIT_FAILURE,
1074                                 "%u BBDevs detected, cannot use BBDev with ID %u!\n",
1075                                 nb_bbdevs, app_params.bbdev_id);
1076         printf("Number of bbdevs detected: %d\n", nb_bbdevs);
1077
1078         if (!rte_eth_dev_is_valid_port(app_params.port_id))
1079                 rte_exit(EXIT_FAILURE,
1080                                 "cannot use port with ID %u!\n",
1081                                 app_params.port_id);
1082
1083         /* create the mbuf mempool for ethdev pkts */
1084         ethdev_mbuf_mempool = rte_pktmbuf_pool_create("ethdev_mbuf_pool",
1085                         NB_MBUF, MEMPOOL_CACHE_SIZE, 0,
1086                         RTE_MBUF_DEFAULT_BUF_SIZE, rte_socket_id());
1087         if (ethdev_mbuf_mempool == NULL)
1088                 rte_exit(EXIT_FAILURE, "Cannot create ethdev mbuf mempool\n");
1089
1090         /* create the mbuf mempool for encoder output */
1091         bbdev_mbuf_mempool = rte_pktmbuf_pool_create("bbdev_mbuf_pool",
1092                         NB_MBUF, MEMPOOL_CACHE_SIZE, 0,
1093                         RTE_MBUF_DEFAULT_BUF_SIZE, rte_socket_id());
1094         if (bbdev_mbuf_mempool == NULL)
1095                 rte_exit(EXIT_FAILURE, "Cannot create ethdev mbuf mempool\n");
1096
1097         /* initialize ports */
1098         ret = initialize_ports(&app_params, ethdev_mbuf_mempool);
1099
1100         /* Check if all requested lcores are available */
1101         for (lcore_id = 0; lcore_id < 8 * sizeof(uint64_t); ++lcore_id)
1102                 if (((1ULL << lcore_id) & app_params.enc_core_mask) ||
1103                                 ((1ULL << lcore_id) & app_params.dec_core_mask))
1104                         if (!rte_lcore_is_enabled(lcore_id))
1105                                 rte_exit(EXIT_FAILURE,
1106                                                 "Requested lcore_id %u is not enabled!\n",
1107                                                 lcore_id);
1108
1109         /* Start ethernet port */
1110         ret = rte_eth_dev_start(app_params.port_id);
1111         if (ret < 0)
1112                 rte_exit(EXIT_FAILURE, "rte_eth_dev_start:err=%d, port=%u\n",
1113                                 ret, app_params.port_id);
1114
1115         ret = check_port_link_status(app_params.port_id);
1116         if (ret < 0)
1117                 exit(EXIT_FAILURE);
1118
1119         /* start BBDevice and save BBDev queue IDs */
1120         enable_bbdev(&app_params);
1121
1122         /* Initialize the port/queue configuration of each logical core */
1123         lcore_conf_init(&app_params, lcore_conf, bbdev_op_pools,
1124                         bbdev_mbuf_mempool, enc_to_dec_ring, lcore_stats);
1125
1126         stats_lcore.app_params = &app_params;
1127         stats_lcore.lconf = lcore_conf;
1128
1129         RTE_LCORE_FOREACH_SLAVE(lcore_id) {
1130                 if (lcore_conf[lcore_id].core_type != 0)
1131                         /* launch per-lcore processing loop on slave lcores */
1132                         rte_eal_remote_launch(processing_loop,
1133                                         &lcore_conf[lcore_id], lcore_id);
1134                 else if (!stats_thread_started) {
1135                         /* launch statistics printing loop */
1136                         rte_eal_remote_launch(stats_loop, &stats_lcore,
1137                                         lcore_id);
1138                         stats_thread_started = true;
1139                 }
1140         }
1141
1142         if (!stats_thread_started &&
1143                         lcore_conf[master_lcore_id].core_type != 0)
1144                 rte_exit(EXIT_FAILURE,
1145                                 "Not enough lcores to run the statistics printing loop!");
1146         else if (lcore_conf[master_lcore_id].core_type != 0)
1147                 processing_loop(&lcore_conf[master_lcore_id]);
1148         else if (!stats_thread_started)
1149                 stats_loop(&stats_lcore);
1150
1151         RTE_LCORE_FOREACH_SLAVE(lcore_id) {
1152                 ret |= rte_eal_wait_lcore(lcore_id);
1153         }
1154
1155         return ret;
1156 }