net/memif: fix Unix domain address length
[dpdk.git] / examples / bbdev_app / main.c
1 /* SPDX-License-Identifier: BSD-3-Clause
2  * Copyright(c) 2017 Intel Corporation
3  */
4
5 #include <stdio.h>
6 #include <stdlib.h>
7 #include <string.h>
8 #include <stdint.h>
9 #include <inttypes.h>
10 #include <sys/types.h>
11 #include <sys/unistd.h>
12 #include <sys/queue.h>
13 #include <stdarg.h>
14 #include <ctype.h>
15 #include <errno.h>
16 #include <math.h>
17 #include <assert.h>
18 #include <getopt.h>
19 #include <signal.h>
20
21 #include "rte_atomic.h"
22 #include "rte_common.h"
23 #include "rte_eal.h"
24 #include "rte_cycles.h"
25 #include "rte_ether.h"
26 #include "rte_ethdev.h"
27 #include "rte_ip.h"
28 #include "rte_lcore.h"
29 #include "rte_malloc.h"
30 #include "rte_mbuf.h"
31 #include "rte_memory.h"
32 #include "rte_mempool.h"
33 #include "rte_log.h"
34 #include "rte_bbdev.h"
35 #include "rte_bbdev_op.h"
36
37 /* LLR values - negative value for '1' bit */
38 #define LLR_1_BIT 0x81
39 #define LLR_0_BIT 0x7F
40
41 #define MAX_PKT_BURST 32
42 #define NB_MBUF 8191
43 #define MEMPOOL_CACHE_SIZE 256
44
45 /* Hardcoded K value */
46 #define K 40
47 #define NCB (3 * RTE_ALIGN_CEIL(K + 4, 32))
48
49 #define CRC_24B_LEN 3
50
51 /* Configurable number of RX/TX ring descriptors */
52 #define RTE_TEST_RX_DESC_DEFAULT 128
53 #define RTE_TEST_TX_DESC_DEFAULT 512
54
55 #define BBDEV_ASSERT(a) do { \
56         if (!(a)) { \
57                 usage(prgname); \
58                 return -1; \
59         } \
60 } while (0)
61
62 static const struct rte_eth_conf port_conf = {
63         .rxmode = {
64                 .mq_mode = ETH_MQ_RX_NONE,
65                 .max_rx_pkt_len = RTE_ETHER_MAX_LEN,
66                 .split_hdr_size = 0,
67         },
68         .txmode = {
69                 .mq_mode = ETH_MQ_TX_NONE,
70         },
71 };
72
73 struct rte_bbdev_op_turbo_enc def_op_enc = {
74         /* These values are arbitrarily put, and does not map to the real
75          * values for the data received from ethdev ports
76          */
77         .rv_index = 0,
78         .code_block_mode = 1,
79         .cb_params = {
80                 .k = K,
81         },
82         .op_flags = RTE_BBDEV_TURBO_CRC_24A_ATTACH
83 };
84
85 struct rte_bbdev_op_turbo_dec def_op_dec = {
86         /* These values are arbitrarily put, and does not map to the real
87          * values for the data received from ethdev ports
88          */
89         .code_block_mode = 1,
90         .cb_params = {
91                 .k = K,
92         },
93         .rv_index = 0,
94         .iter_max = 8,
95         .iter_min = 4,
96         .ext_scale = 15,
97         .num_maps = 0,
98         .op_flags = RTE_BBDEV_TURBO_NEG_LLR_1_BIT_IN
99 };
100
101 struct app_config_params {
102         /* Placeholders for app params */
103         uint16_t port_id;
104         uint16_t bbdev_id;
105         uint64_t enc_core_mask;
106         uint64_t dec_core_mask;
107
108         /* Values filled during init time */
109         uint16_t enc_queue_ids[RTE_MAX_LCORE];
110         uint16_t dec_queue_ids[RTE_MAX_LCORE];
111         uint16_t num_enc_cores;
112         uint16_t num_dec_cores;
113 };
114
115 struct lcore_statistics {
116         unsigned int enqueued;
117         unsigned int dequeued;
118         unsigned int rx_lost_packets;
119         unsigned int enc_to_dec_lost_packets;
120         unsigned int tx_lost_packets;
121 } __rte_cache_aligned;
122
123 /** each lcore configuration */
124 struct lcore_conf {
125         uint64_t core_type;
126
127         unsigned int port_id;
128         unsigned int rx_queue_id;
129         unsigned int tx_queue_id;
130
131         unsigned int bbdev_id;
132         unsigned int enc_queue_id;
133         unsigned int dec_queue_id;
134
135         uint8_t llr_temp_buf[NCB];
136
137         struct rte_mempool *bbdev_dec_op_pool;
138         struct rte_mempool *bbdev_enc_op_pool;
139         struct rte_mempool *enc_out_pool;
140         struct rte_ring *enc_to_dec_ring;
141
142         struct lcore_statistics *lcore_stats;
143 } __rte_cache_aligned;
144
145 struct stats_lcore_params {
146         struct lcore_conf *lconf;
147         struct app_config_params *app_params;
148 };
149
150
151 static const struct app_config_params def_app_config = {
152         .port_id = 0,
153         .bbdev_id = 0,
154         .enc_core_mask = 0x2,
155         .dec_core_mask = 0x4,
156         .num_enc_cores = 1,
157         .num_dec_cores = 1,
158 };
159
160 static rte_atomic16_t global_exit_flag;
161
162 /* display usage */
163 static inline void
164 usage(const char *prgname)
165 {
166         printf("%s [EAL options] "
167                         "  --\n"
168                         "  --enc_cores - number of encoding cores (default = 0x2)\n"
169                         "  --dec_cores - number of decoding cores (default = 0x4)\n"
170                         "  --port_id - Ethernet port ID (default = 0)\n"
171                         "  --bbdev_id - BBDev ID (default = 0)\n"
172                         "\n", prgname);
173 }
174
175 /* parse core mask */
176 static inline
177 uint16_t bbdev_parse_mask(const char *mask)
178 {
179         char *end = NULL;
180         unsigned long pm;
181
182         /* parse hexadecimal string */
183         pm = strtoul(mask, &end, 16);
184         if ((mask[0] == '\0') || (end == NULL) || (*end != '\0'))
185                 return 0;
186
187         return pm;
188 }
189
190 /* parse core mask */
191 static inline
192 uint16_t bbdev_parse_number(const char *mask)
193 {
194         char *end = NULL;
195         unsigned long pm;
196
197         /* parse hexadecimal string */
198         pm = strtoul(mask, &end, 10);
199         if ((mask[0] == '\0') || (end == NULL) || (*end != '\0'))
200                 return 0;
201
202         return pm;
203 }
204
205 static int
206 bbdev_parse_args(int argc, char **argv,
207                 struct app_config_params *app_params)
208 {
209         int optind = 0;
210         int opt;
211         int opt_indx = 0;
212         char *prgname = argv[0];
213
214         static struct option lgopts[] = {
215                 { "enc_core_mask", required_argument, 0, 'e' },
216                 { "dec_core_mask", required_argument, 0, 'd' },
217                 { "port_id", required_argument, 0, 'p' },
218                 { "bbdev_id", required_argument, 0, 'b' },
219                 { NULL, 0, 0, 0 }
220         };
221
222         BBDEV_ASSERT(argc != 0);
223         BBDEV_ASSERT(argv != NULL);
224         BBDEV_ASSERT(app_params != NULL);
225
226         while ((opt = getopt_long(argc, argv, "e:d:p:b:", lgopts, &opt_indx)) !=
227                 EOF) {
228                 switch (opt) {
229                 case 'e':
230                         app_params->enc_core_mask =
231                                 bbdev_parse_mask(optarg);
232                         if (app_params->enc_core_mask == 0) {
233                                 usage(prgname);
234                                 return -1;
235                         }
236                         app_params->num_enc_cores =
237                                 __builtin_popcount(app_params->enc_core_mask);
238                         break;
239
240                 case 'd':
241                         app_params->dec_core_mask =
242                                 bbdev_parse_mask(optarg);
243                         if (app_params->dec_core_mask == 0) {
244                                 usage(prgname);
245                                 return -1;
246                         }
247                         app_params->num_dec_cores =
248                                 __builtin_popcount(app_params->dec_core_mask);
249                         break;
250
251                 case 'p':
252                         app_params->port_id = bbdev_parse_number(optarg);
253                         break;
254
255                 case 'b':
256                         app_params->bbdev_id = bbdev_parse_number(optarg);
257                         break;
258
259                 default:
260                         usage(prgname);
261                         return -1;
262                 }
263         }
264         optind = 0;
265         return optind;
266 }
267
268 static void
269 signal_handler(int signum)
270 {
271         printf("\nSignal %d received\n", signum);
272         rte_atomic16_set(&global_exit_flag, 1);
273 }
274
275 static void
276 print_mac(unsigned int portid, struct rte_ether_addr *bbdev_ports_eth_address)
277 {
278         printf("Port %u, MAC address: %02X:%02X:%02X:%02X:%02X:%02X\n\n",
279                         (unsigned int) portid,
280                         bbdev_ports_eth_address->addr_bytes[0],
281                         bbdev_ports_eth_address->addr_bytes[1],
282                         bbdev_ports_eth_address->addr_bytes[2],
283                         bbdev_ports_eth_address->addr_bytes[3],
284                         bbdev_ports_eth_address->addr_bytes[4],
285                         bbdev_ports_eth_address->addr_bytes[5]);
286 }
287
288 static inline void
289 pktmbuf_free_bulk(struct rte_mbuf **mbufs, unsigned int nb_to_free)
290 {
291         unsigned int i;
292         for (i = 0; i < nb_to_free; ++i)
293                 rte_pktmbuf_free(mbufs[i]);
294 }
295
296 static inline void
297 pktmbuf_userdata_free_bulk(struct rte_mbuf **mbufs, unsigned int nb_to_free)
298 {
299         unsigned int i;
300         for (i = 0; i < nb_to_free; ++i) {
301                 struct rte_mbuf *rx_pkt = mbufs[i]->userdata;
302                 rte_pktmbuf_free(rx_pkt);
303                 rte_pktmbuf_free(mbufs[i]);
304         }
305 }
306
307 /* Check the link status of all ports in up to 9s, and print them finally */
308 static int
309 check_port_link_status(uint16_t port_id)
310 {
311 #define CHECK_INTERVAL 100 /* 100ms */
312 #define MAX_CHECK_TIME 90 /* 9s (90 * 100ms) in total */
313         uint8_t count;
314         struct rte_eth_link link;
315         int link_get_err = -EINVAL;
316
317         printf("\nChecking link status.");
318         fflush(stdout);
319
320         for (count = 0; count <= MAX_CHECK_TIME &&
321                         !rte_atomic16_read(&global_exit_flag); count++) {
322                 memset(&link, 0, sizeof(link));
323                 link_get_err = rte_eth_link_get_nowait(port_id, &link);
324
325                 if (link_get_err >= 0 && link.link_status) {
326                         const char *dp = (link.link_duplex ==
327                                 ETH_LINK_FULL_DUPLEX) ?
328                                 "full-duplex" : "half-duplex";
329                         printf("\nPort %u Link Up - speed %u Mbps - %s\n",
330                                 port_id, link.link_speed, dp);
331                         return 0;
332                 }
333                 printf(".");
334                 fflush(stdout);
335                 rte_delay_ms(CHECK_INTERVAL);
336         }
337
338         if (link_get_err >= 0)
339                 printf("\nPort %d Link Down\n", port_id);
340         else
341                 printf("\nGet link failed (port %d): %s\n", port_id,
342                        rte_strerror(-link_get_err));
343
344         return 0;
345 }
346
347 static inline void
348 add_ether_hdr(struct rte_mbuf *pkt_src, struct rte_mbuf *pkt_dst)
349 {
350         struct rte_ether_hdr *eth_from;
351         struct rte_ether_hdr *eth_to;
352
353         eth_from = rte_pktmbuf_mtod(pkt_src, struct rte_ether_hdr *);
354         eth_to = rte_pktmbuf_mtod(pkt_dst, struct rte_ether_hdr *);
355
356         /* copy header */
357         rte_memcpy(eth_to, eth_from, sizeof(struct rte_ether_hdr));
358 }
359
360 static inline void
361 add_awgn(struct rte_mbuf **mbufs, uint16_t num_pkts)
362 {
363         RTE_SET_USED(mbufs);
364         RTE_SET_USED(num_pkts);
365 }
366
367 /* Encoder output to Decoder input adapter. The Decoder accepts only soft input
368  * so each bit of the encoder output must be translated into one byte of LLR. If
369  * Sub-block Deinterleaver is bypassed, which is the case, the padding bytes
370  * must additionally be insterted at the end of each sub-block.
371  */
372 static inline void
373 transform_enc_out_dec_in(struct rte_mbuf **mbufs, uint8_t *temp_buf,
374                 uint16_t num_pkts, uint16_t k)
375 {
376         uint16_t i, l, j;
377         uint16_t start_bit_idx;
378         uint16_t out_idx;
379         uint16_t d = k + 4;
380         uint16_t kpi = RTE_ALIGN_CEIL(d, 32);
381         uint16_t nd = kpi - d;
382         uint16_t ncb = 3 * kpi;
383
384         for (i = 0; i < num_pkts; ++i) {
385                 uint16_t pkt_data_len = rte_pktmbuf_data_len(mbufs[i]) -
386                                 sizeof(struct rte_ether_hdr);
387
388                 /* Resize the packet if needed */
389                 if (pkt_data_len < ncb) {
390                         char *data = rte_pktmbuf_append(mbufs[i],
391                                         ncb - pkt_data_len);
392                         if (data == NULL)
393                                 printf(
394                                         "Not enough space in decoder input packet");
395                 }
396
397                 /* Translate each bit into 1 LLR byte. */
398                 start_bit_idx = 0;
399                 out_idx = 0;
400                 for (j = 0; j < 3; ++j) {
401                         for (l = start_bit_idx; l < start_bit_idx + d; ++l) {
402                                 uint8_t *data = rte_pktmbuf_mtod_offset(
403                                         mbufs[i], uint8_t *,
404                                         sizeof(struct rte_ether_hdr) +
405                                         (l >> 3));
406                                 if (*data & (0x80 >> (l & 7)))
407                                         temp_buf[out_idx] = LLR_1_BIT;
408                                 else
409                                         temp_buf[out_idx] = LLR_0_BIT;
410                                 ++out_idx;
411                         }
412                         /* Padding bytes should be at the end of the sub-block.
413                          */
414                         memset(&temp_buf[out_idx], 0, nd);
415                         out_idx += nd;
416                         start_bit_idx += d;
417                 }
418
419                 rte_memcpy(rte_pktmbuf_mtod_offset(mbufs[i], uint8_t *,
420                                 sizeof(struct rte_ether_hdr)), temp_buf, ncb);
421         }
422 }
423
424 static inline void
425 verify_data(struct rte_mbuf **mbufs, uint16_t num_pkts)
426 {
427         uint16_t i;
428         for (i = 0; i < num_pkts; ++i) {
429                 struct rte_mbuf *out = mbufs[i];
430                 struct rte_mbuf *in = out->userdata;
431
432                 if (memcmp(rte_pktmbuf_mtod_offset(in, uint8_t *,
433                                 sizeof(struct rte_ether_hdr)),
434                                 rte_pktmbuf_mtod_offset(out, uint8_t *,
435                                 sizeof(struct rte_ether_hdr)),
436                                 K / 8 - CRC_24B_LEN))
437                         printf("Input and output buffers are not equal!\n");
438         }
439 }
440
441 static int
442 initialize_ports(struct app_config_params *app_params,
443                 struct rte_mempool *ethdev_mbuf_mempool)
444 {
445         int ret;
446         uint16_t port_id = app_params->port_id;
447         uint16_t q;
448         /* ethernet addresses of ports */
449         struct rte_ether_addr bbdev_port_eth_addr;
450
451         /* initialize ports */
452         printf("\nInitializing port %u...\n", app_params->port_id);
453         ret = rte_eth_dev_configure(port_id, app_params->num_enc_cores,
454                 app_params->num_dec_cores, &port_conf);
455
456         if (ret < 0) {
457                 printf("Cannot configure device: err=%d, port=%u\n",
458                         ret, port_id);
459                 return -1;
460         }
461
462         /* initialize RX queues for encoder */
463         for (q = 0; q < app_params->num_enc_cores; q++) {
464                 ret = rte_eth_rx_queue_setup(port_id, q,
465                         RTE_TEST_RX_DESC_DEFAULT,
466                         rte_eth_dev_socket_id(port_id),
467                         NULL, ethdev_mbuf_mempool);
468                 if (ret < 0) {
469                         printf("rte_eth_rx_queue_setup: err=%d, queue=%u\n",
470                                 ret, q);
471                         return -1;
472                 }
473         }
474         /* initialize TX queues for decoder */
475         for (q = 0; q < app_params->num_dec_cores; q++) {
476                 ret = rte_eth_tx_queue_setup(port_id, q,
477                         RTE_TEST_TX_DESC_DEFAULT,
478                         rte_eth_dev_socket_id(port_id), NULL);
479                 if (ret < 0) {
480                         printf("rte_eth_tx_queue_setup: err=%d, queue=%u\n",
481                                 ret, q);
482                         return -1;
483                 }
484         }
485
486         ret = rte_eth_promiscuous_enable(port_id);
487         if (ret != 0) {
488                 printf("Cannot enable promiscuous mode: err=%s, port=%u\n",
489                         rte_strerror(-ret), port_id);
490                 return ret;
491         }
492
493         ret = rte_eth_macaddr_get(port_id, &bbdev_port_eth_addr);
494         if (ret < 0) {
495                 printf("rte_eth_macaddr_get: err=%d, queue=%u\n",
496                         ret, q);
497                 return -1;
498         }
499
500         print_mac(port_id, &bbdev_port_eth_addr);
501
502         return 0;
503 }
504
505 static void
506 lcore_conf_init(struct app_config_params *app_params,
507                 struct lcore_conf *lcore_conf,
508                 struct rte_mempool **bbdev_op_pools,
509                 struct rte_mempool *bbdev_mbuf_mempool,
510                 struct rte_ring *enc_to_dec_ring,
511                 struct lcore_statistics *lcore_stats)
512 {
513         unsigned int lcore_id;
514         struct lcore_conf *lconf;
515         uint16_t rx_queue_id = 0;
516         uint16_t tx_queue_id = 0;
517         uint16_t enc_q_id = 0;
518         uint16_t dec_q_id = 0;
519
520         /* Configure lcores */
521         for (lcore_id = 0; lcore_id < 8 * sizeof(uint64_t); ++lcore_id) {
522                 lconf = &lcore_conf[lcore_id];
523                 lconf->core_type = 0;
524
525                 if ((1ULL << lcore_id) & app_params->enc_core_mask) {
526                         lconf->core_type |= (1 << RTE_BBDEV_OP_TURBO_ENC);
527                         lconf->rx_queue_id = rx_queue_id++;
528                         lconf->enc_queue_id =
529                                         app_params->enc_queue_ids[enc_q_id++];
530                 }
531
532                 if ((1ULL << lcore_id) & app_params->dec_core_mask) {
533                         lconf->core_type |= (1 << RTE_BBDEV_OP_TURBO_DEC);
534                         lconf->tx_queue_id = tx_queue_id++;
535                         lconf->dec_queue_id =
536                                         app_params->dec_queue_ids[dec_q_id++];
537                 }
538
539                 lconf->bbdev_enc_op_pool =
540                                 bbdev_op_pools[RTE_BBDEV_OP_TURBO_ENC];
541                 lconf->bbdev_dec_op_pool =
542                                 bbdev_op_pools[RTE_BBDEV_OP_TURBO_DEC];
543                 lconf->bbdev_id = app_params->bbdev_id;
544                 lconf->port_id = app_params->port_id;
545                 lconf->enc_out_pool = bbdev_mbuf_mempool;
546                 lconf->enc_to_dec_ring = enc_to_dec_ring;
547                 lconf->lcore_stats = &lcore_stats[lcore_id];
548         }
549 }
550
551 static void
552 print_lcore_stats(struct lcore_statistics *lstats, unsigned int lcore_id)
553 {
554         static const char *stats_border = "_______";
555
556         printf("\nLcore %d: %s enqueued count:\t\t%u\n",
557                         lcore_id, stats_border, lstats->enqueued);
558         printf("Lcore %d: %s dequeued count:\t\t%u\n",
559                         lcore_id, stats_border, lstats->dequeued);
560         printf("Lcore %d: %s RX lost packets count:\t\t%u\n",
561                         lcore_id, stats_border, lstats->rx_lost_packets);
562         printf("Lcore %d: %s encoder-to-decoder lost count:\t%u\n",
563                         lcore_id, stats_border,
564                         lstats->enc_to_dec_lost_packets);
565         printf("Lcore %d: %s TX lost packets count:\t\t%u\n",
566                         lcore_id, stats_border, lstats->tx_lost_packets);
567 }
568
569 static void
570 print_stats(struct stats_lcore_params *stats_lcore)
571 {
572         unsigned int l_id;
573         unsigned int bbdev_id = stats_lcore->app_params->bbdev_id;
574         unsigned int port_id = stats_lcore->app_params->port_id;
575         int len, ret, i;
576
577         struct rte_eth_xstat *xstats;
578         struct rte_eth_xstat_name *xstats_names;
579         struct rte_bbdev_stats bbstats;
580         static const char *stats_border = "_______";
581
582         const char clr[] = { 27, '[', '2', 'J', '\0' };
583         const char topLeft[] = { 27, '[', '1', ';', '1', 'H', '\0' };
584
585         /* Clear screen and move to top left */
586         printf("%s%s", clr, topLeft);
587
588         printf("PORT STATISTICS:\n================\n");
589         len = rte_eth_xstats_get(port_id, NULL, 0);
590         if (len < 0)
591                 rte_exit(EXIT_FAILURE,
592                                 "rte_eth_xstats_get(%u) failed: %d", port_id,
593                                 len);
594
595         xstats = calloc(len, sizeof(*xstats));
596         if (xstats == NULL)
597                 rte_exit(EXIT_FAILURE,
598                                 "Failed to calloc memory for xstats");
599
600         ret = rte_eth_xstats_get(port_id, xstats, len);
601         if (ret < 0 || ret > len) {
602                 free(xstats);
603                 rte_exit(EXIT_FAILURE,
604                                 "rte_eth_xstats_get(%u) len%i failed: %d",
605                                 port_id, len, ret);
606         }
607
608         xstats_names = calloc(len, sizeof(*xstats_names));
609         if (xstats_names == NULL) {
610                 free(xstats);
611                 rte_exit(EXIT_FAILURE,
612                                 "Failed to calloc memory for xstats_names");
613         }
614
615         ret = rte_eth_xstats_get_names(port_id, xstats_names, len);
616         if (ret < 0 || ret > len) {
617                 free(xstats);
618                 free(xstats_names);
619                 rte_exit(EXIT_FAILURE,
620                                 "rte_eth_xstats_get_names(%u) len%i failed: %d",
621                                 port_id, len, ret);
622         }
623
624         for (i = 0; i < len; i++) {
625                 if (xstats[i].value > 0)
626                         printf("Port %u: %s %s:\t\t%"PRIu64"\n",
627                                         port_id, stats_border,
628                                         xstats_names[i].name,
629                                         xstats[i].value);
630         }
631
632         ret = rte_bbdev_stats_get(bbdev_id, &bbstats);
633         if (ret < 0) {
634                 free(xstats);
635                 free(xstats_names);
636                 rte_exit(EXIT_FAILURE,
637                                 "ERROR(%d): Failure to get BBDEV %u statistics\n",
638                                 ret, bbdev_id);
639         }
640
641         printf("\nBBDEV STATISTICS:\n=================\n");
642         printf("BBDEV %u: %s enqueue count:\t\t%"PRIu64"\n",
643                         bbdev_id, stats_border,
644                         bbstats.enqueued_count);
645         printf("BBDEV %u: %s dequeue count:\t\t%"PRIu64"\n",
646                         bbdev_id, stats_border,
647                         bbstats.dequeued_count);
648         printf("BBDEV %u: %s enqueue error count:\t\t%"PRIu64"\n",
649                         bbdev_id, stats_border,
650                         bbstats.enqueue_err_count);
651         printf("BBDEV %u: %s dequeue error count:\t\t%"PRIu64"\n\n",
652                         bbdev_id, stats_border,
653                         bbstats.dequeue_err_count);
654
655         printf("LCORE STATISTICS:\n=================\n");
656         for (l_id = 0; l_id < RTE_MAX_LCORE; ++l_id) {
657                 if (stats_lcore->lconf[l_id].core_type == 0)
658                         continue;
659                 print_lcore_stats(stats_lcore->lconf[l_id].lcore_stats, l_id);
660         }
661
662         free(xstats);
663         free(xstats_names);
664 }
665
666 static int
667 stats_loop(void *arg)
668 {
669         struct stats_lcore_params *stats_lcore = arg;
670
671         while (!rte_atomic16_read(&global_exit_flag)) {
672                 print_stats(stats_lcore);
673                 rte_delay_ms(500);
674         }
675
676         return 0;
677 }
678
679 static inline void
680 run_encoding(struct lcore_conf *lcore_conf)
681 {
682         uint16_t i;
683         uint16_t port_id, rx_queue_id;
684         uint16_t bbdev_id, enc_queue_id;
685         uint16_t nb_rx, nb_enq, nb_deq, nb_sent;
686         struct rte_mbuf *rx_pkts_burst[MAX_PKT_BURST];
687         struct rte_mbuf *enc_out_pkts[MAX_PKT_BURST];
688         struct rte_bbdev_enc_op *bbdev_ops_burst[MAX_PKT_BURST];
689         struct lcore_statistics *lcore_stats;
690         struct rte_mempool *bbdev_op_pool, *enc_out_pool;
691         struct rte_ring *enc_to_dec_ring;
692         const int in_data_len = (def_op_enc.cb_params.k / 8) - CRC_24B_LEN;
693
694         lcore_stats = lcore_conf->lcore_stats;
695         port_id = lcore_conf->port_id;
696         rx_queue_id = lcore_conf->rx_queue_id;
697         bbdev_id = lcore_conf->bbdev_id;
698         enc_queue_id = lcore_conf->enc_queue_id;
699         bbdev_op_pool = lcore_conf->bbdev_enc_op_pool;
700         enc_out_pool = lcore_conf->enc_out_pool;
701         enc_to_dec_ring = lcore_conf->enc_to_dec_ring;
702
703         /* Read packet from RX queues*/
704         nb_rx = rte_eth_rx_burst(port_id, rx_queue_id, rx_pkts_burst,
705                         MAX_PKT_BURST);
706         if (!nb_rx)
707                 return;
708
709         if (unlikely(rte_mempool_get_bulk(enc_out_pool, (void **)enc_out_pkts,
710                         nb_rx) != 0)) {
711                 pktmbuf_free_bulk(rx_pkts_burst, nb_rx);
712                 lcore_stats->rx_lost_packets += nb_rx;
713                 return;
714         }
715
716         if (unlikely(rte_bbdev_enc_op_alloc_bulk(bbdev_op_pool, bbdev_ops_burst,
717                         nb_rx) != 0)) {
718                 pktmbuf_free_bulk(enc_out_pkts, nb_rx);
719                 pktmbuf_free_bulk(rx_pkts_burst, nb_rx);
720                 lcore_stats->rx_lost_packets += nb_rx;
721                 return;
722         }
723
724         for (i = 0; i < nb_rx; i++) {
725                 char *data;
726                 const uint16_t pkt_data_len =
727                                 rte_pktmbuf_data_len(rx_pkts_burst[i]) -
728                                 sizeof(struct rte_ether_hdr);
729                 /* save input mbuf pointer for later comparison */
730                 enc_out_pkts[i]->userdata = rx_pkts_burst[i];
731
732                 /* copy ethernet header */
733                 rte_pktmbuf_reset(enc_out_pkts[i]);
734                 data = rte_pktmbuf_append(enc_out_pkts[i],
735                                 sizeof(struct rte_ether_hdr));
736                 if (data == NULL) {
737                         printf(
738                                 "Not enough space for ethernet header in encoder output mbuf\n");
739                         continue;
740                 }
741                 add_ether_hdr(rx_pkts_burst[i], enc_out_pkts[i]);
742
743                 /* set op */
744                 bbdev_ops_burst[i]->turbo_enc = def_op_enc;
745
746                 bbdev_ops_burst[i]->turbo_enc.input.data =
747                                 rx_pkts_burst[i];
748                 bbdev_ops_burst[i]->turbo_enc.input.offset =
749                                 sizeof(struct rte_ether_hdr);
750                 /* Encoder will attach the CRC24B, adjust the length */
751                 bbdev_ops_burst[i]->turbo_enc.input.length = in_data_len;
752
753                 if (in_data_len < pkt_data_len)
754                         rte_pktmbuf_trim(rx_pkts_burst[i], pkt_data_len -
755                                         in_data_len);
756                 else if (in_data_len > pkt_data_len) {
757                         data = rte_pktmbuf_append(rx_pkts_burst[i],
758                                         in_data_len - pkt_data_len);
759                         if (data == NULL)
760                                 printf(
761                                         "Not enough storage in mbuf to perform the encoding\n");
762                 }
763
764                 bbdev_ops_burst[i]->turbo_enc.output.data =
765                                 enc_out_pkts[i];
766                 bbdev_ops_burst[i]->turbo_enc.output.offset =
767                                 sizeof(struct rte_ether_hdr);
768         }
769
770         /* Enqueue packets on BBDevice */
771         nb_enq = rte_bbdev_enqueue_enc_ops(bbdev_id, enc_queue_id,
772                         bbdev_ops_burst, nb_rx);
773         if (unlikely(nb_enq < nb_rx)) {
774                 pktmbuf_userdata_free_bulk(&enc_out_pkts[nb_enq],
775                                 nb_rx - nb_enq);
776                 rte_bbdev_enc_op_free_bulk(&bbdev_ops_burst[nb_enq],
777                                 nb_rx - nb_enq);
778                 lcore_stats->rx_lost_packets += nb_rx - nb_enq;
779
780                 if (!nb_enq)
781                         return;
782         }
783
784         lcore_stats->enqueued += nb_enq;
785
786         /* Dequeue packets from bbdev device*/
787         nb_deq = 0;
788         do {
789                 nb_deq += rte_bbdev_dequeue_enc_ops(bbdev_id, enc_queue_id,
790                                 &bbdev_ops_burst[nb_deq], nb_enq - nb_deq);
791         } while (unlikely(nb_deq < nb_enq));
792
793         lcore_stats->dequeued += nb_deq;
794
795         /* Generate and add AWGN */
796         add_awgn(enc_out_pkts, nb_deq);
797
798         rte_bbdev_enc_op_free_bulk(bbdev_ops_burst, nb_deq);
799
800         /* Enqueue packets to encoder-to-decoder ring */
801         nb_sent = rte_ring_enqueue_burst(enc_to_dec_ring, (void **)enc_out_pkts,
802                         nb_deq, NULL);
803         if (unlikely(nb_sent < nb_deq)) {
804                 pktmbuf_userdata_free_bulk(&enc_out_pkts[nb_sent],
805                                 nb_deq - nb_sent);
806                 lcore_stats->enc_to_dec_lost_packets += nb_deq - nb_sent;
807         }
808 }
809
810 static void
811 run_decoding(struct lcore_conf *lcore_conf)
812 {
813         uint16_t i;
814         uint16_t port_id, tx_queue_id;
815         uint16_t bbdev_id, bbdev_queue_id;
816         uint16_t nb_recv, nb_enq, nb_deq, nb_tx;
817         uint8_t *llr_temp_buf;
818         struct rte_mbuf *recv_pkts_burst[MAX_PKT_BURST];
819         struct rte_bbdev_dec_op *bbdev_ops_burst[MAX_PKT_BURST];
820         struct lcore_statistics *lcore_stats;
821         struct rte_mempool *bbdev_op_pool;
822         struct rte_ring *enc_to_dec_ring;
823
824         lcore_stats = lcore_conf->lcore_stats;
825         port_id = lcore_conf->port_id;
826         tx_queue_id = lcore_conf->tx_queue_id;
827         bbdev_id = lcore_conf->bbdev_id;
828         bbdev_queue_id = lcore_conf->dec_queue_id;
829         bbdev_op_pool = lcore_conf->bbdev_dec_op_pool;
830         enc_to_dec_ring = lcore_conf->enc_to_dec_ring;
831         llr_temp_buf = lcore_conf->llr_temp_buf;
832
833         /* Dequeue packets from the ring */
834         nb_recv = rte_ring_dequeue_burst(enc_to_dec_ring,
835                         (void **)recv_pkts_burst, MAX_PKT_BURST, NULL);
836         if (!nb_recv)
837                 return;
838
839         if (unlikely(rte_bbdev_dec_op_alloc_bulk(bbdev_op_pool, bbdev_ops_burst,
840                         nb_recv) != 0)) {
841                 pktmbuf_userdata_free_bulk(recv_pkts_burst, nb_recv);
842                 lcore_stats->rx_lost_packets += nb_recv;
843                 return;
844         }
845
846         transform_enc_out_dec_in(recv_pkts_burst, llr_temp_buf, nb_recv,
847                         def_op_dec.cb_params.k);
848
849         for (i = 0; i < nb_recv; i++) {
850                 /* set op */
851                 bbdev_ops_burst[i]->turbo_dec = def_op_dec;
852
853                 bbdev_ops_burst[i]->turbo_dec.input.data = recv_pkts_burst[i];
854                 bbdev_ops_burst[i]->turbo_dec.input.offset =
855                                 sizeof(struct rte_ether_hdr);
856                 bbdev_ops_burst[i]->turbo_dec.input.length =
857                                 rte_pktmbuf_data_len(recv_pkts_burst[i])
858                                 - sizeof(struct rte_ether_hdr);
859
860                 bbdev_ops_burst[i]->turbo_dec.hard_output.data =
861                                 recv_pkts_burst[i];
862                 bbdev_ops_burst[i]->turbo_dec.hard_output.offset =
863                                 sizeof(struct rte_ether_hdr);
864         }
865
866         /* Enqueue packets on BBDevice */
867         nb_enq = rte_bbdev_enqueue_dec_ops(bbdev_id, bbdev_queue_id,
868                         bbdev_ops_burst, nb_recv);
869         if (unlikely(nb_enq < nb_recv)) {
870                 pktmbuf_userdata_free_bulk(&recv_pkts_burst[nb_enq],
871                                 nb_recv - nb_enq);
872                 rte_bbdev_dec_op_free_bulk(&bbdev_ops_burst[nb_enq],
873                                 nb_recv - nb_enq);
874                 lcore_stats->rx_lost_packets += nb_recv - nb_enq;
875
876                 if (!nb_enq)
877                         return;
878         }
879
880         lcore_stats->enqueued += nb_enq;
881
882         /* Dequeue packets from BBDevice */
883         nb_deq = 0;
884         do {
885                 nb_deq += rte_bbdev_dequeue_dec_ops(bbdev_id, bbdev_queue_id,
886                                 &bbdev_ops_burst[nb_deq], nb_enq - nb_deq);
887         } while (unlikely(nb_deq < nb_enq));
888
889         lcore_stats->dequeued += nb_deq;
890
891         rte_bbdev_dec_op_free_bulk(bbdev_ops_burst, nb_deq);
892
893         verify_data(recv_pkts_burst, nb_deq);
894
895         /* Free the RX mbufs after verification */
896         for (i = 0; i < nb_deq; ++i)
897                 rte_pktmbuf_free(recv_pkts_burst[i]->userdata);
898
899         /* Transmit the packets */
900         nb_tx = rte_eth_tx_burst(port_id, tx_queue_id, recv_pkts_burst, nb_deq);
901         if (unlikely(nb_tx < nb_deq)) {
902                 pktmbuf_userdata_free_bulk(&recv_pkts_burst[nb_tx],
903                                 nb_deq - nb_tx);
904                 lcore_stats->tx_lost_packets += nb_deq - nb_tx;
905         }
906 }
907
908 static int
909 processing_loop(void *arg)
910 {
911         struct lcore_conf *lcore_conf = arg;
912         const bool run_encoder = (lcore_conf->core_type &
913                         (1 << RTE_BBDEV_OP_TURBO_ENC));
914         const bool run_decoder = (lcore_conf->core_type &
915                         (1 << RTE_BBDEV_OP_TURBO_DEC));
916
917         while (!rte_atomic16_read(&global_exit_flag)) {
918                 if (run_encoder)
919                         run_encoding(lcore_conf);
920                 if (run_decoder)
921                         run_decoding(lcore_conf);
922         }
923
924         return 0;
925 }
926
927 static int
928 prepare_bbdev_device(unsigned int dev_id, struct rte_bbdev_info *info,
929                 struct app_config_params *app_params)
930 {
931         int ret;
932         unsigned int q_id, dec_q_id, enc_q_id;
933         struct rte_bbdev_queue_conf qconf = {0};
934         uint16_t dec_qs_nb = app_params->num_dec_cores;
935         uint16_t enc_qs_nb = app_params->num_enc_cores;
936         uint16_t tot_qs = dec_qs_nb + enc_qs_nb;
937
938         ret = rte_bbdev_setup_queues(dev_id, tot_qs, info->socket_id);
939         if (ret < 0)
940                 rte_exit(EXIT_FAILURE,
941                                 "ERROR(%d): BBDEV %u not configured properly\n",
942                                 ret, dev_id);
943
944         /* setup device DEC queues */
945         qconf.socket = info->socket_id;
946         qconf.queue_size = info->drv.queue_size_lim;
947         qconf.op_type = RTE_BBDEV_OP_TURBO_DEC;
948
949         for (q_id = 0, dec_q_id = 0; q_id < dec_qs_nb; q_id++) {
950                 ret = rte_bbdev_queue_configure(dev_id, q_id, &qconf);
951                 if (ret < 0)
952                         rte_exit(EXIT_FAILURE,
953                                         "ERROR(%d): BBDEV %u DEC queue %u not configured properly\n",
954                                         ret, dev_id, q_id);
955                 app_params->dec_queue_ids[dec_q_id++] = q_id;
956         }
957
958         /* setup device ENC queues */
959         qconf.op_type = RTE_BBDEV_OP_TURBO_ENC;
960
961         for (q_id = dec_qs_nb, enc_q_id = 0; q_id < tot_qs; q_id++) {
962                 ret = rte_bbdev_queue_configure(dev_id, q_id, &qconf);
963                 if (ret < 0)
964                         rte_exit(EXIT_FAILURE,
965                                         "ERROR(%d): BBDEV %u ENC queue %u not configured properly\n",
966                                         ret, dev_id, q_id);
967                 app_params->enc_queue_ids[enc_q_id++] = q_id;
968         }
969
970         ret = rte_bbdev_start(dev_id);
971
972         if (ret != 0)
973                 rte_exit(EXIT_FAILURE, "ERROR(%d): BBDEV %u not started\n",
974                         ret, dev_id);
975
976         printf("BBdev %u started\n", dev_id);
977
978         return 0;
979 }
980
981 static inline bool
982 check_matching_capabilities(uint64_t mask, uint64_t required_mask)
983 {
984         return (mask & required_mask) == required_mask;
985 }
986
987 static void
988 enable_bbdev(struct app_config_params *app_params)
989 {
990         struct rte_bbdev_info dev_info;
991         const struct rte_bbdev_op_cap *op_cap;
992         uint16_t bbdev_id = app_params->bbdev_id;
993         bool encoder_capable = false;
994         bool decoder_capable = false;
995
996         rte_bbdev_info_get(bbdev_id, &dev_info);
997         op_cap = dev_info.drv.capabilities;
998
999         while (op_cap->type != RTE_BBDEV_OP_NONE) {
1000                 if (op_cap->type == RTE_BBDEV_OP_TURBO_ENC) {
1001                         if (check_matching_capabilities(
1002                                         op_cap->cap.turbo_enc.capability_flags,
1003                                         def_op_enc.op_flags))
1004                                 encoder_capable = true;
1005                 }
1006
1007                 if (op_cap->type == RTE_BBDEV_OP_TURBO_DEC) {
1008                         if (check_matching_capabilities(
1009                                         op_cap->cap.turbo_dec.capability_flags,
1010                                         def_op_dec.op_flags))
1011                                 decoder_capable = true;
1012                 }
1013
1014                 op_cap++;
1015         }
1016
1017         if (encoder_capable == false)
1018                 rte_exit(EXIT_FAILURE,
1019                         "The specified BBDev %u doesn't have required encoder capabilities!\n",
1020                         bbdev_id);
1021         if (decoder_capable == false)
1022                 rte_exit(EXIT_FAILURE,
1023                         "The specified BBDev %u doesn't have required decoder capabilities!\n",
1024                         bbdev_id);
1025
1026         prepare_bbdev_device(bbdev_id, &dev_info, app_params);
1027 }
1028
1029 int
1030 main(int argc, char **argv)
1031 {
1032         int ret;
1033         unsigned int nb_bbdevs, flags, lcore_id;
1034         void *sigret;
1035         struct app_config_params app_params = def_app_config;
1036         struct rte_mempool *ethdev_mbuf_mempool, *bbdev_mbuf_mempool;
1037         struct rte_mempool *bbdev_op_pools[RTE_BBDEV_OP_TYPE_COUNT];
1038         struct lcore_conf lcore_conf[RTE_MAX_LCORE] = { {0} };
1039         struct lcore_statistics lcore_stats[RTE_MAX_LCORE] = { {0} };
1040         struct stats_lcore_params stats_lcore;
1041         struct rte_ring *enc_to_dec_ring;
1042         bool stats_thread_started = false;
1043         unsigned int master_lcore_id = rte_get_master_lcore();
1044
1045         rte_atomic16_init(&global_exit_flag);
1046
1047         sigret = signal(SIGTERM, signal_handler);
1048         if (sigret == SIG_ERR)
1049                 rte_exit(EXIT_FAILURE, "signal(%d, ...) failed", SIGTERM);
1050
1051         sigret = signal(SIGINT, signal_handler);
1052         if (sigret == SIG_ERR)
1053                 rte_exit(EXIT_FAILURE, "signal(%d, ...) failed", SIGINT);
1054
1055         ret = rte_eal_init(argc, argv);
1056         if (ret < 0)
1057                 rte_exit(EXIT_FAILURE, "Invalid EAL arguments\n");
1058
1059         argc -= ret;
1060         argv += ret;
1061
1062         /* parse application arguments (after the EAL ones) */
1063         ret = bbdev_parse_args(argc, argv, &app_params);
1064         if (ret < 0)
1065                 rte_exit(EXIT_FAILURE, "Invalid BBDEV arguments\n");
1066
1067         /*create bbdev op pools*/
1068         bbdev_op_pools[RTE_BBDEV_OP_TURBO_DEC] =
1069                         rte_bbdev_op_pool_create("bbdev_op_pool_dec",
1070                         RTE_BBDEV_OP_TURBO_DEC, NB_MBUF, 128, rte_socket_id());
1071         bbdev_op_pools[RTE_BBDEV_OP_TURBO_ENC] =
1072                         rte_bbdev_op_pool_create("bbdev_op_pool_enc",
1073                         RTE_BBDEV_OP_TURBO_ENC, NB_MBUF, 128, rte_socket_id());
1074
1075         if ((bbdev_op_pools[RTE_BBDEV_OP_TURBO_DEC] == NULL) ||
1076                         (bbdev_op_pools[RTE_BBDEV_OP_TURBO_ENC] == NULL))
1077                 rte_exit(EXIT_FAILURE, "Cannot create bbdev op pools\n");
1078
1079         /* Create encoder to decoder ring */
1080         flags = (app_params.num_enc_cores == 1) ? RING_F_SP_ENQ : 0;
1081         if (app_params.num_dec_cores == 1)
1082                 flags |= RING_F_SC_DEQ;
1083
1084         enc_to_dec_ring = rte_ring_create("enc_to_dec_ring",
1085                 rte_align32pow2(NB_MBUF), rte_socket_id(), flags);
1086
1087         /* Get the number of available bbdev devices */
1088         nb_bbdevs = rte_bbdev_count();
1089         if (nb_bbdevs <= app_params.bbdev_id)
1090                 rte_exit(EXIT_FAILURE,
1091                                 "%u BBDevs detected, cannot use BBDev with ID %u!\n",
1092                                 nb_bbdevs, app_params.bbdev_id);
1093         printf("Number of bbdevs detected: %d\n", nb_bbdevs);
1094
1095         if (!rte_eth_dev_is_valid_port(app_params.port_id))
1096                 rte_exit(EXIT_FAILURE,
1097                                 "cannot use port with ID %u!\n",
1098                                 app_params.port_id);
1099
1100         /* create the mbuf mempool for ethdev pkts */
1101         ethdev_mbuf_mempool = rte_pktmbuf_pool_create("ethdev_mbuf_pool",
1102                         NB_MBUF, MEMPOOL_CACHE_SIZE, 0,
1103                         RTE_MBUF_DEFAULT_BUF_SIZE, rte_socket_id());
1104         if (ethdev_mbuf_mempool == NULL)
1105                 rte_exit(EXIT_FAILURE, "Cannot create ethdev mbuf mempool\n");
1106
1107         /* create the mbuf mempool for encoder output */
1108         bbdev_mbuf_mempool = rte_pktmbuf_pool_create("bbdev_mbuf_pool",
1109                         NB_MBUF, MEMPOOL_CACHE_SIZE, 0,
1110                         RTE_MBUF_DEFAULT_BUF_SIZE, rte_socket_id());
1111         if (bbdev_mbuf_mempool == NULL)
1112                 rte_exit(EXIT_FAILURE, "Cannot create ethdev mbuf mempool\n");
1113
1114         /* initialize ports */
1115         ret = initialize_ports(&app_params, ethdev_mbuf_mempool);
1116
1117         /* Check if all requested lcores are available */
1118         for (lcore_id = 0; lcore_id < 8 * sizeof(uint64_t); ++lcore_id)
1119                 if (((1ULL << lcore_id) & app_params.enc_core_mask) ||
1120                                 ((1ULL << lcore_id) & app_params.dec_core_mask))
1121                         if (!rte_lcore_is_enabled(lcore_id))
1122                                 rte_exit(EXIT_FAILURE,
1123                                                 "Requested lcore_id %u is not enabled!\n",
1124                                                 lcore_id);
1125
1126         /* Start ethernet port */
1127         ret = rte_eth_dev_start(app_params.port_id);
1128         if (ret < 0)
1129                 rte_exit(EXIT_FAILURE, "rte_eth_dev_start:err=%d, port=%u\n",
1130                                 ret, app_params.port_id);
1131
1132         ret = check_port_link_status(app_params.port_id);
1133         if (ret < 0)
1134                 exit(EXIT_FAILURE);
1135
1136         /* start BBDevice and save BBDev queue IDs */
1137         enable_bbdev(&app_params);
1138
1139         /* Initialize the port/queue configuration of each logical core */
1140         lcore_conf_init(&app_params, lcore_conf, bbdev_op_pools,
1141                         bbdev_mbuf_mempool, enc_to_dec_ring, lcore_stats);
1142
1143         stats_lcore.app_params = &app_params;
1144         stats_lcore.lconf = lcore_conf;
1145
1146         RTE_LCORE_FOREACH_SLAVE(lcore_id) {
1147                 if (lcore_conf[lcore_id].core_type != 0)
1148                         /* launch per-lcore processing loop on slave lcores */
1149                         rte_eal_remote_launch(processing_loop,
1150                                         &lcore_conf[lcore_id], lcore_id);
1151                 else if (!stats_thread_started) {
1152                         /* launch statistics printing loop */
1153                         rte_eal_remote_launch(stats_loop, &stats_lcore,
1154                                         lcore_id);
1155                         stats_thread_started = true;
1156                 }
1157         }
1158
1159         if (!stats_thread_started &&
1160                         lcore_conf[master_lcore_id].core_type != 0)
1161                 rte_exit(EXIT_FAILURE,
1162                                 "Not enough lcores to run the statistics printing loop!");
1163         else if (lcore_conf[master_lcore_id].core_type != 0)
1164                 processing_loop(&lcore_conf[master_lcore_id]);
1165         else if (!stats_thread_started)
1166                 stats_loop(&stats_lcore);
1167
1168         RTE_LCORE_FOREACH_SLAVE(lcore_id) {
1169                 ret |= rte_eal_wait_lcore(lcore_id);
1170         }
1171
1172         return ret;
1173 }