1 /* SPDX-License-Identifier: BSD-3-Clause
2 * Copyright(c) 2019 Intel Corporation
12 #include <cmdline_parse_string.h>
13 #include <cmdline_socket.h>
15 #include <rte_common.h>
16 #include <rte_rawdev.h>
17 #include <rte_ethdev.h>
18 #include <rte_malloc.h>
19 #include <rte_lcore.h>
20 #include <rte_cycles.h>
21 #include <rte_pmd_ntb.h>
23 /* Per-port statistics struct */
24 struct ntb_port_statistics {
27 } __rte_cache_aligned;
28 /* Port 0: NTB dev, Port 1: ethdev when iofwd. */
29 struct ntb_port_statistics ntb_port_stats[2];
31 struct ntb_fwd_stream {
35 uint8_t tx_ntb; /* If ntb device is tx port. */
38 struct ntb_fwd_lcore_conf {
51 static const char *const fwd_mode_s[] = {
58 static enum ntb_fwd_mode fwd_mode = MAX_FWD_MODE;
60 static struct ntb_fwd_lcore_conf fwd_lcore_conf[RTE_MAX_LCORE];
61 static struct ntb_fwd_stream *fwd_streams;
63 static struct rte_mempool *mbuf_pool;
65 #define NTB_DRV_NAME_LEN 7
66 #define MEMPOOL_CACHE_SIZE 256
68 static uint8_t in_test;
69 static uint8_t interactive = 1;
70 static uint16_t eth_port_id = RTE_MAX_ETHPORTS;
71 static uint16_t dev_id;
73 /* Number of queues, default set as 1 */
74 static uint16_t num_queues = 1;
75 static uint16_t ntb_buf_size = RTE_MBUF_DEFAULT_BUF_SIZE;
77 /* Configurable number of descriptors */
78 #define NTB_DEFAULT_NUM_DESCS 1024
79 static uint16_t nb_desc = NTB_DEFAULT_NUM_DESCS;
81 static uint16_t tx_free_thresh;
83 #define NTB_MAX_PKT_BURST 32
84 #define NTB_DFLT_PKT_BURST 32
85 static uint16_t pkt_burst = NTB_DFLT_PKT_BURST;
87 #define BURST_TX_RETRIES 64
89 static struct rte_eth_conf eth_port_conf = {
91 .mq_mode = ETH_MQ_RX_RSS,
101 .mq_mode = ETH_MQ_TX_NONE,
105 /* *** Help command with introduction. *** */
106 struct cmd_help_result {
107 cmdline_fixed_string_t help;
111 cmd_help_parsed(__attribute__((unused)) void *parsed_result,
113 __attribute__((unused)) void *data)
118 "The following commands are currently available:\n\n"
121 " Quit the application.\n"
124 " Send [path] file. Only take effect in file-trans mode\n"
126 " Start transmissions.\n"
128 " Stop transmissions.\n"
129 " clear/show port stats :"
130 " Clear/show port stats.\n"
131 " set fwd file-trans/rxonly/txonly/iofwd :"
132 " Set packet forwarding mode.\n"
137 cmdline_parse_token_string_t cmd_help_help =
138 TOKEN_STRING_INITIALIZER(struct cmd_help_result, help, "help");
140 cmdline_parse_inst_t cmd_help = {
141 .f = cmd_help_parsed,
143 .help_str = "show help",
145 (void *)&cmd_help_help,
151 struct cmd_quit_result {
152 cmdline_fixed_string_t quit;
156 cmd_quit_parsed(__attribute__((unused)) void *parsed_result,
158 __attribute__((unused)) void *data)
160 struct ntb_fwd_lcore_conf *conf;
163 /* Stop transmission first. */
164 RTE_LCORE_FOREACH_SLAVE(lcore_id) {
165 conf = &fwd_lcore_conf[lcore_id];
167 if (!conf->nb_stream)
175 printf("\nWaiting for lcores to finish...\n");
176 rte_eal_mp_wait_lcore();
179 /* Stop traffic and Close port. */
180 rte_rawdev_stop(dev_id);
181 rte_rawdev_close(dev_id);
182 if (eth_port_id < RTE_MAX_ETHPORTS && fwd_mode == IOFWD) {
183 rte_eth_dev_stop(eth_port_id);
184 rte_eth_dev_close(eth_port_id);
190 cmdline_parse_token_string_t cmd_quit_quit =
191 TOKEN_STRING_INITIALIZER(struct cmd_quit_result, quit, "quit");
193 cmdline_parse_inst_t cmd_quit = {
194 .f = cmd_quit_parsed,
196 .help_str = "exit application",
198 (void *)&cmd_quit_quit,
203 /* *** SEND FILE PARAMETERS *** */
204 struct cmd_sendfile_result {
205 cmdline_fixed_string_t send_string;
210 cmd_sendfile_parsed(void *parsed_result,
211 __attribute__((unused)) struct cmdline *cl,
212 __attribute__((unused)) void *data)
214 struct cmd_sendfile_result *res = parsed_result;
215 struct rte_rawdev_buf *pkts_send[NTB_MAX_PKT_BURST];
216 struct rte_mbuf *mbuf_send[NTB_MAX_PKT_BURST];
217 uint64_t size, count, i, j, nb_burst;
218 uint16_t nb_tx, buf_size;
226 if (num_queues != 1) {
227 printf("File transmission only supports 1 queue.\n");
231 file = fopen(res->filepath, "r");
233 printf("Fail to open the file.\n");
237 if (fseek(file, 0, SEEK_END) < 0) {
238 printf("Fail to get file size.\n");
243 if (fseek(file, 0, SEEK_SET) < 0) {
244 printf("Fail to get file size.\n");
249 /* Tell remote about the file size. */
251 rte_rawdev_set_attr(dev_id, "spad_user_0", val);
253 rte_rawdev_set_attr(dev_id, "spad_user_1", val);
254 printf("Sending file, size is %"PRIu64"\n", size);
256 for (i = 0; i < NTB_MAX_PKT_BURST; i++)
257 pkts_send[i] = (struct rte_rawdev_buf *)
258 malloc(sizeof(struct rte_rawdev_buf));
260 buf_size = ntb_buf_size - RTE_PKTMBUF_HEADROOM;
261 count = (size + buf_size - 1) / buf_size;
262 nb_burst = (count + pkt_burst - 1) / pkt_burst;
264 for (i = 0; i < nb_burst; i++) {
265 val = RTE_MIN(count, pkt_burst);
266 if (rte_mempool_get_bulk(mbuf_pool, (void **)mbuf_send,
268 for (nb_pkt = 0; nb_pkt < val; nb_pkt++) {
269 mbuf_send[nb_pkt]->port = dev_id;
270 mbuf_send[nb_pkt]->data_len =
271 fread(rte_pktmbuf_mtod(mbuf_send[nb_pkt],
272 void *), 1, buf_size, file);
273 mbuf_send[nb_pkt]->pkt_len =
274 mbuf_send[nb_pkt]->data_len;
275 pkts_send[nb_pkt]->buf_addr = mbuf_send[nb_pkt];
278 for (nb_pkt = 0; nb_pkt < val; nb_pkt++) {
280 rte_mbuf_raw_alloc(mbuf_pool);
281 if (mbuf_send[nb_pkt] == NULL)
283 mbuf_send[nb_pkt]->port = dev_id;
284 mbuf_send[nb_pkt]->data_len =
285 fread(rte_pktmbuf_mtod(mbuf_send[nb_pkt],
286 void *), 1, buf_size, file);
287 mbuf_send[nb_pkt]->pkt_len =
288 mbuf_send[nb_pkt]->data_len;
289 pkts_send[nb_pkt]->buf_addr = mbuf_send[nb_pkt];
293 ret = rte_rawdev_enqueue_buffers(dev_id, pkts_send, nb_pkt,
296 printf("Enqueue failed with err %d\n", ret);
297 for (j = 0; j < nb_pkt; j++)
298 rte_pktmbuf_free(mbuf_send[j]);
302 while (nb_tx != nb_pkt && retry < BURST_TX_RETRIES) {
304 ret = rte_rawdev_enqueue_buffers(dev_id,
305 &pkts_send[nb_tx], nb_pkt - nb_tx,
308 printf("Enqueue failed with err %d\n", ret);
309 for (j = nb_tx; j < nb_pkt; j++)
310 rte_pktmbuf_free(mbuf_send[j]);
318 /* Clear register after file sending done. */
319 rte_rawdev_set_attr(dev_id, "spad_user_0", 0);
320 rte_rawdev_set_attr(dev_id, "spad_user_1", 0);
321 printf("Done sending file.\n");
324 for (i = 0; i < NTB_MAX_PKT_BURST; i++)
329 cmdline_parse_token_string_t cmd_send_file_send =
330 TOKEN_STRING_INITIALIZER(struct cmd_sendfile_result, send_string,
332 cmdline_parse_token_string_t cmd_send_file_filepath =
333 TOKEN_STRING_INITIALIZER(struct cmd_sendfile_result, filepath, NULL);
336 cmdline_parse_inst_t cmd_send_file = {
337 .f = cmd_sendfile_parsed,
339 .help_str = "send <file_path>",
341 (void *)&cmd_send_file_send,
342 (void *)&cmd_send_file_filepath,
347 #define RECV_FILE_LEN 30
349 start_polling_recv_file(void *param)
351 struct rte_rawdev_buf *pkts_recv[NTB_MAX_PKT_BURST];
352 struct ntb_fwd_lcore_conf *conf = param;
353 struct rte_mbuf *mbuf;
354 char filepath[RECV_FILE_LEN];
355 uint64_t val, size, file_len;
356 uint16_t nb_rx, i, file_no;
361 for (i = 0; i < NTB_MAX_PKT_BURST; i++)
362 pkts_recv[i] = (struct rte_rawdev_buf *)
363 malloc(sizeof(struct rte_rawdev_buf));
366 while (!conf->stopped) {
367 snprintf(filepath, RECV_FILE_LEN, "ntb_recv_file%d", file_no);
368 file = fopen(filepath, "w");
370 printf("Fail to open the file.\n");
374 rte_rawdev_get_attr(dev_id, "spad_user_0", &val);
376 rte_rawdev_get_attr(dev_id, "spad_user_1", &val);
385 nb_rx = NTB_MAX_PKT_BURST;
386 while (file_len < size && !conf->stopped) {
387 ret = rte_rawdev_dequeue_buffers(dev_id, pkts_recv,
388 pkt_burst, (void *)queue_id);
390 printf("Dequeue failed with err %d\n", ret);
395 ntb_port_stats[0].rx += nb_rx;
396 for (i = 0; i < nb_rx; i++) {
397 mbuf = pkts_recv[i]->buf_addr;
398 fwrite(rte_pktmbuf_mtod(mbuf, void *), 1,
399 mbuf->data_len, file);
400 file_len += mbuf->data_len;
401 rte_pktmbuf_free(mbuf);
402 pkts_recv[i]->buf_addr = NULL;
406 printf("Received file (size: %" PRIu64 ") from peer to %s.\n",
413 for (i = 0; i < NTB_MAX_PKT_BURST; i++)
419 start_iofwd_per_lcore(void *param)
421 struct rte_rawdev_buf *ntb_buf[NTB_MAX_PKT_BURST];
422 struct rte_mbuf *pkts_burst[NTB_MAX_PKT_BURST];
423 struct ntb_fwd_lcore_conf *conf = param;
424 struct ntb_fwd_stream fs;
425 uint16_t nb_rx, nb_tx;
428 for (i = 0; i < NTB_MAX_PKT_BURST; i++)
429 ntb_buf[i] = (struct rte_rawdev_buf *)
430 malloc(sizeof(struct rte_rawdev_buf));
432 while (!conf->stopped) {
433 for (i = 0; i < conf->nb_stream; i++) {
434 fs = fwd_streams[conf->stream_id + i];
436 nb_rx = rte_eth_rx_burst(fs.rx_port,
437 fs.qp_id, pkts_burst,
439 if (unlikely(nb_rx == 0))
441 for (j = 0; j < nb_rx; j++)
442 ntb_buf[j]->buf_addr = pkts_burst[j];
443 ret = rte_rawdev_enqueue_buffers(fs.tx_port,
445 (void *)(size_t)fs.qp_id);
447 printf("Enqueue failed with err %d\n",
449 for (j = 0; j < nb_rx; j++)
450 rte_pktmbuf_free(pkts_burst[j]);
454 ntb_port_stats[0].tx += nb_tx;
455 ntb_port_stats[1].rx += nb_rx;
457 ret = rte_rawdev_dequeue_buffers(fs.rx_port,
459 (void *)(size_t)fs.qp_id);
461 printf("Dequeue failed with err %d\n",
466 if (unlikely(nb_rx == 0))
468 for (j = 0; j < nb_rx; j++)
469 pkts_burst[j] = ntb_buf[j]->buf_addr;
470 nb_tx = rte_eth_tx_burst(fs.tx_port,
471 fs.qp_id, pkts_burst, nb_rx);
472 ntb_port_stats[1].tx += nb_tx;
473 ntb_port_stats[0].rx += nb_rx;
475 if (unlikely(nb_tx < nb_rx)) {
477 rte_pktmbuf_free(pkts_burst[nb_tx]);
478 } while (++nb_tx < nb_rx);
484 for (i = 0; i < NTB_MAX_PKT_BURST; i++)
491 start_rxonly_per_lcore(void *param)
493 struct rte_rawdev_buf *ntb_buf[NTB_MAX_PKT_BURST];
494 struct ntb_fwd_lcore_conf *conf = param;
495 struct ntb_fwd_stream fs;
499 for (i = 0; i < NTB_MAX_PKT_BURST; i++)
500 ntb_buf[i] = (struct rte_rawdev_buf *)
501 malloc(sizeof(struct rte_rawdev_buf));
503 while (!conf->stopped) {
504 for (i = 0; i < conf->nb_stream; i++) {
505 fs = fwd_streams[conf->stream_id + i];
506 ret = rte_rawdev_dequeue_buffers(fs.rx_port,
507 ntb_buf, pkt_burst, (void *)(size_t)fs.qp_id);
509 printf("Dequeue failed with err %d\n", ret);
513 if (unlikely(nb_rx == 0))
515 ntb_port_stats[0].rx += nb_rx;
517 for (j = 0; j < nb_rx; j++)
518 rte_pktmbuf_free(ntb_buf[j]->buf_addr);
523 for (i = 0; i < NTB_MAX_PKT_BURST; i++)
531 start_txonly_per_lcore(void *param)
533 struct rte_rawdev_buf *ntb_buf[NTB_MAX_PKT_BURST];
534 struct rte_mbuf *pkts_burst[NTB_MAX_PKT_BURST];
535 struct ntb_fwd_lcore_conf *conf = param;
536 struct ntb_fwd_stream fs;
537 uint16_t nb_pkt, nb_tx;
540 for (i = 0; i < NTB_MAX_PKT_BURST; i++)
541 ntb_buf[i] = (struct rte_rawdev_buf *)
542 malloc(sizeof(struct rte_rawdev_buf));
544 while (!conf->stopped) {
545 for (i = 0; i < conf->nb_stream; i++) {
546 fs = fwd_streams[conf->stream_id + i];
547 if (rte_mempool_get_bulk(mbuf_pool, (void **)pkts_burst,
549 for (nb_pkt = 0; nb_pkt < pkt_burst; nb_pkt++) {
550 pkts_burst[nb_pkt]->port = dev_id;
551 pkts_burst[nb_pkt]->data_len =
552 pkts_burst[nb_pkt]->buf_len -
553 RTE_PKTMBUF_HEADROOM;
554 pkts_burst[nb_pkt]->pkt_len =
555 pkts_burst[nb_pkt]->data_len;
556 ntb_buf[nb_pkt]->buf_addr =
560 for (nb_pkt = 0; nb_pkt < pkt_burst; nb_pkt++) {
562 rte_pktmbuf_alloc(mbuf_pool);
563 if (pkts_burst[nb_pkt] == NULL)
565 pkts_burst[nb_pkt]->port = dev_id;
566 pkts_burst[nb_pkt]->data_len =
567 pkts_burst[nb_pkt]->buf_len -
568 RTE_PKTMBUF_HEADROOM;
569 pkts_burst[nb_pkt]->pkt_len =
570 pkts_burst[nb_pkt]->data_len;
571 ntb_buf[nb_pkt]->buf_addr =
575 ret = rte_rawdev_enqueue_buffers(fs.tx_port, ntb_buf,
576 nb_pkt, (void *)(size_t)fs.qp_id);
578 printf("Enqueue failed with err %d\n", ret);
579 for (j = 0; j < nb_pkt; j++)
580 rte_pktmbuf_free(pkts_burst[j]);
584 ntb_port_stats[0].tx += nb_tx;
585 if (unlikely(nb_tx < nb_pkt)) {
587 rte_pktmbuf_free(pkts_burst[nb_tx]);
588 } while (++nb_tx < nb_pkt);
594 for (i = 0; i < NTB_MAX_PKT_BURST; i++)
601 ntb_fwd_config_setup(void)
605 /* Make sure iofwd has valid ethdev. */
606 if (fwd_mode == IOFWD && eth_port_id >= RTE_MAX_ETHPORTS) {
607 printf("No ethdev, cannot be in iofwd mode.");
611 if (fwd_mode == IOFWD) {
612 fwd_streams = rte_zmalloc("ntb_fwd: fwd_streams",
613 sizeof(struct ntb_fwd_stream) * num_queues * 2,
614 RTE_CACHE_LINE_SIZE);
615 for (i = 0; i < num_queues; i++) {
616 fwd_streams[i * 2].qp_id = i;
617 fwd_streams[i * 2].tx_port = dev_id;
618 fwd_streams[i * 2].rx_port = eth_port_id;
619 fwd_streams[i * 2].tx_ntb = 1;
621 fwd_streams[i * 2 + 1].qp_id = i;
622 fwd_streams[i * 2 + 1].tx_port = eth_port_id;
623 fwd_streams[i * 2 + 1].rx_port = dev_id;
624 fwd_streams[i * 2 + 1].tx_ntb = 0;
629 if (fwd_mode == RXONLY || fwd_mode == FILE_TRANS) {
630 /* Only support 1 queue in file-trans for in order. */
631 if (fwd_mode == FILE_TRANS)
634 fwd_streams = rte_zmalloc("ntb_fwd: fwd_streams",
635 sizeof(struct ntb_fwd_stream) * num_queues,
636 RTE_CACHE_LINE_SIZE);
637 for (i = 0; i < num_queues; i++) {
638 fwd_streams[i].qp_id = i;
639 fwd_streams[i].tx_port = RTE_MAX_ETHPORTS;
640 fwd_streams[i].rx_port = dev_id;
641 fwd_streams[i].tx_ntb = 0;
646 if (fwd_mode == TXONLY) {
647 fwd_streams = rte_zmalloc("ntb_fwd: fwd_streams",
648 sizeof(struct ntb_fwd_stream) * num_queues,
649 RTE_CACHE_LINE_SIZE);
650 for (i = 0; i < num_queues; i++) {
651 fwd_streams[i].qp_id = i;
652 fwd_streams[i].tx_port = dev_id;
653 fwd_streams[i].rx_port = RTE_MAX_ETHPORTS;
654 fwd_streams[i].tx_ntb = 1;
661 assign_stream_to_lcores(void)
663 struct ntb_fwd_lcore_conf *conf;
664 struct ntb_fwd_stream *fs;
665 uint16_t nb_streams, sm_per_lcore, sm_id, i;
667 uint8_t lcore_num, nb_extra;
669 lcore_num = rte_lcore_count();
670 /* Exclude master core */
673 nb_streams = (fwd_mode == IOFWD) ? num_queues * 2 : num_queues;
675 sm_per_lcore = nb_streams / lcore_num;
676 nb_extra = nb_streams % lcore_num;
680 RTE_LCORE_FOREACH_SLAVE(lcore_id) {
681 conf = &fwd_lcore_conf[lcore_id];
684 conf->nb_stream = sm_per_lcore + 1;
685 conf->stream_id = sm_id;
686 sm_id = sm_id + sm_per_lcore + 1;
688 conf->nb_stream = sm_per_lcore;
689 conf->stream_id = sm_id;
690 sm_id = sm_id + sm_per_lcore;
694 if (sm_id >= nb_streams)
698 /* Print packet forwading config. */
699 RTE_LCORE_FOREACH_SLAVE(lcore_id) {
700 conf = &fwd_lcore_conf[lcore_id];
702 if (!conf->nb_stream)
705 printf("Streams on Lcore %u :\n", lcore_id);
706 for (i = 0; i < conf->nb_stream; i++) {
707 fs = &fwd_streams[conf->stream_id + i];
708 if (fwd_mode == IOFWD)
709 printf(" + Stream %u : %s%u RX -> %s%u TX,"
710 " Q=%u\n", conf->stream_id + i,
711 fs->tx_ntb ? "Eth" : "NTB", fs->rx_port,
712 fs->tx_ntb ? "NTB" : "Eth", fs->tx_port,
714 if (fwd_mode == FILE_TRANS || fwd_mode == RXONLY)
715 printf(" + Stream %u : %s%u RX only\n",
716 conf->stream_id, "NTB", fs->rx_port);
717 if (fwd_mode == TXONLY)
718 printf(" + Stream %u : %s%u TX only\n",
719 conf->stream_id, "NTB", fs->tx_port);
727 struct ntb_fwd_lcore_conf *conf;
728 struct rte_eth_link eth_link;
732 ret = ntb_fwd_config_setup();
734 printf("Cannot start traffic. Please reset fwd mode.\n");
738 /* If using iofwd, checking ethdev link status first. */
739 if (fwd_mode == IOFWD) {
740 printf("Checking eth link status...\n");
741 /* Wait for eth link up at most 100 times. */
742 for (i = 0; i < 100; i++) {
743 ret = rte_eth_link_get(eth_port_id, ð_link);
745 printf("Link get failed with err %d\n", ret);
748 if (eth_link.link_status) {
749 printf("Eth%u Link Up. Speed %u Mbps - %s\n",
750 eth_port_id, eth_link.link_speed,
751 (eth_link.link_duplex ==
752 ETH_LINK_FULL_DUPLEX) ?
753 ("full-duplex") : ("half-duplex"));
757 if (!eth_link.link_status) {
758 printf("Eth%u link down. Cannot start traffic.\n",
764 assign_stream_to_lcores();
767 RTE_LCORE_FOREACH_SLAVE(lcore_id) {
768 conf = &fwd_lcore_conf[lcore_id];
770 if (!conf->nb_stream)
774 if (fwd_mode == FILE_TRANS)
775 rte_eal_remote_launch(start_polling_recv_file,
777 else if (fwd_mode == IOFWD)
778 rte_eal_remote_launch(start_iofwd_per_lcore,
780 else if (fwd_mode == RXONLY)
781 rte_eal_remote_launch(start_rxonly_per_lcore,
783 else if (fwd_mode == TXONLY)
784 rte_eal_remote_launch(start_txonly_per_lcore,
789 /* *** START FWD PARAMETERS *** */
790 struct cmd_start_result {
791 cmdline_fixed_string_t start;
795 cmd_start_parsed(__attribute__((unused)) void *parsed_result,
796 __attribute__((unused)) struct cmdline *cl,
797 __attribute__((unused)) void *data)
802 cmdline_parse_token_string_t cmd_start_start =
803 TOKEN_STRING_INITIALIZER(struct cmd_start_result, start, "start");
805 cmdline_parse_inst_t cmd_start = {
806 .f = cmd_start_parsed,
808 .help_str = "start pkt fwd between ntb and ethdev",
810 (void *)&cmd_start_start,
816 struct cmd_stop_result {
817 cmdline_fixed_string_t stop;
821 cmd_stop_parsed(__attribute__((unused)) void *parsed_result,
822 __attribute__((unused)) struct cmdline *cl,
823 __attribute__((unused)) void *data)
825 struct ntb_fwd_lcore_conf *conf;
828 RTE_LCORE_FOREACH_SLAVE(lcore_id) {
829 conf = &fwd_lcore_conf[lcore_id];
831 if (!conf->nb_stream)
839 printf("\nWaiting for lcores to finish...\n");
840 rte_eal_mp_wait_lcore();
845 cmdline_parse_token_string_t cmd_stop_stop =
846 TOKEN_STRING_INITIALIZER(struct cmd_stop_result, stop, "stop");
848 cmdline_parse_inst_t cmd_stop = {
849 .f = cmd_stop_parsed,
851 .help_str = "stop: Stop packet forwarding",
853 (void *)&cmd_stop_stop,
859 ntb_stats_clear(void)
864 /* Clear NTB dev stats */
865 nb_ids = rte_rawdev_xstats_names_get(dev_id, NULL, 0);
867 printf("Error: Cannot get count of xstats\n");
870 ids = malloc(sizeof(uint32_t) * nb_ids);
871 for (i = 0; i < nb_ids; i++)
873 rte_rawdev_xstats_reset(dev_id, ids, nb_ids);
874 printf("\n statistics for NTB port %d cleared\n", dev_id);
876 /* Clear Ethdev stats if have any */
877 if (fwd_mode == IOFWD && eth_port_id != RTE_MAX_ETHPORTS) {
878 rte_eth_stats_reset(eth_port_id);
879 printf("\n statistics for ETH port %d cleared\n", eth_port_id);
884 ntb_calculate_throughput(uint16_t port) {
885 uint64_t diff_pkts_rx, diff_pkts_tx, diff_cycles;
886 uint64_t mpps_rx, mpps_tx;
887 static uint64_t prev_pkts_rx[2];
888 static uint64_t prev_pkts_tx[2];
889 static uint64_t prev_cycles[2];
891 diff_cycles = prev_cycles[port];
892 prev_cycles[port] = rte_rdtsc();
894 diff_cycles = prev_cycles[port] - diff_cycles;
895 diff_pkts_rx = (ntb_port_stats[port].rx > prev_pkts_rx[port]) ?
896 (ntb_port_stats[port].rx - prev_pkts_rx[port]) : 0;
897 diff_pkts_tx = (ntb_port_stats[port].tx > prev_pkts_tx[port]) ?
898 (ntb_port_stats[port].tx - prev_pkts_tx[port]) : 0;
899 prev_pkts_rx[port] = ntb_port_stats[port].rx;
900 prev_pkts_tx[port] = ntb_port_stats[port].tx;
901 mpps_rx = diff_cycles > 0 ?
902 diff_pkts_rx * rte_get_tsc_hz() / diff_cycles : 0;
903 mpps_tx = diff_cycles > 0 ?
904 diff_pkts_tx * rte_get_tsc_hz() / diff_cycles : 0;
905 printf(" Throughput (since last show)\n");
906 printf(" Rx-pps: %12"PRIu64"\n Tx-pps: %12"PRIu64"\n",
912 ntb_stats_display(void)
914 struct rte_rawdev_xstats_name *xstats_names;
915 struct rte_eth_stats stats;
920 printf("###### statistics for NTB port %d #######\n", dev_id);
922 /* Get NTB dev stats and stats names */
923 nb_ids = rte_rawdev_xstats_names_get(dev_id, NULL, 0);
925 printf("Error: Cannot get count of xstats\n");
928 xstats_names = malloc(sizeof(struct rte_rawdev_xstats_name) * nb_ids);
929 if (xstats_names == NULL) {
930 printf("Cannot allocate memory for xstats lookup\n");
933 if (nb_ids != rte_rawdev_xstats_names_get(
934 dev_id, xstats_names, nb_ids)) {
935 printf("Error: Cannot get xstats lookup\n");
939 ids = malloc(sizeof(uint32_t) * nb_ids);
940 for (i = 0; i < nb_ids; i++)
942 values = malloc(sizeof(uint64_t) * nb_ids);
943 if (nb_ids != rte_rawdev_xstats_get(dev_id, ids, values, nb_ids)) {
944 printf("Error: Unable to get xstats\n");
951 /* Display NTB dev stats */
952 for (i = 0; i < nb_ids; i++)
953 printf(" %s: %"PRIu64"\n", xstats_names[i].name, values[i]);
954 ntb_calculate_throughput(0);
956 /* Get Ethdev stats if have any */
957 if (fwd_mode == IOFWD && eth_port_id != RTE_MAX_ETHPORTS) {
958 printf("###### statistics for ETH port %d ######\n",
960 rte_eth_stats_get(eth_port_id, &stats);
961 printf(" RX-packets: %"PRIu64"\n", stats.ipackets);
962 printf(" RX-bytes: %"PRIu64"\n", stats.ibytes);
963 printf(" RX-errors: %"PRIu64"\n", stats.ierrors);
964 printf(" RX-missed: %"PRIu64"\n", stats.imissed);
965 printf(" TX-packets: %"PRIu64"\n", stats.opackets);
966 printf(" TX-bytes: %"PRIu64"\n", stats.obytes);
967 printf(" TX-errors: %"PRIu64"\n", stats.oerrors);
968 ntb_calculate_throughput(1);
976 /* *** SHOW/CLEAR PORT STATS *** */
977 struct cmd_stats_result {
978 cmdline_fixed_string_t show;
979 cmdline_fixed_string_t port;
980 cmdline_fixed_string_t stats;
984 cmd_stats_parsed(void *parsed_result,
985 __attribute__((unused)) struct cmdline *cl,
986 __attribute__((unused)) void *data)
988 struct cmd_stats_result *res = parsed_result;
989 if (!strcmp(res->show, "clear"))
995 cmdline_parse_token_string_t cmd_stats_show =
996 TOKEN_STRING_INITIALIZER(struct cmd_stats_result, show, "show#clear");
997 cmdline_parse_token_string_t cmd_stats_port =
998 TOKEN_STRING_INITIALIZER(struct cmd_stats_result, port, "port");
999 cmdline_parse_token_string_t cmd_stats_stats =
1000 TOKEN_STRING_INITIALIZER(struct cmd_stats_result, stats, "stats");
1003 cmdline_parse_inst_t cmd_stats = {
1004 .f = cmd_stats_parsed,
1006 .help_str = "show|clear port stats",
1008 (void *)&cmd_stats_show,
1009 (void *)&cmd_stats_port,
1010 (void *)&cmd_stats_stats,
1015 /* *** SET FORWARDING MODE *** */
1016 struct cmd_set_fwd_mode_result {
1017 cmdline_fixed_string_t set;
1018 cmdline_fixed_string_t fwd;
1019 cmdline_fixed_string_t mode;
1023 cmd_set_fwd_mode_parsed(__attribute__((unused)) void *parsed_result,
1024 __attribute__((unused)) struct cmdline *cl,
1025 __attribute__((unused)) void *data)
1027 struct cmd_set_fwd_mode_result *res = parsed_result;
1031 printf("Please stop traffic first.\n");
1035 for (i = 0; i < MAX_FWD_MODE; i++) {
1036 if (!strcmp(res->mode, fwd_mode_s[i])) {
1041 printf("Invalid %s packet forwarding mode.\n", res->mode);
1044 cmdline_parse_token_string_t cmd_setfwd_set =
1045 TOKEN_STRING_INITIALIZER(struct cmd_set_fwd_mode_result, set, "set");
1046 cmdline_parse_token_string_t cmd_setfwd_fwd =
1047 TOKEN_STRING_INITIALIZER(struct cmd_set_fwd_mode_result, fwd, "fwd");
1048 cmdline_parse_token_string_t cmd_setfwd_mode =
1049 TOKEN_STRING_INITIALIZER(struct cmd_set_fwd_mode_result, mode,
1050 "file-trans#iofwd#txonly#rxonly");
1052 cmdline_parse_inst_t cmd_set_fwd_mode = {
1053 .f = cmd_set_fwd_mode_parsed,
1055 .help_str = "set forwarding mode as file-trans|rxonly|txonly|iofwd",
1057 (void *)&cmd_setfwd_set,
1058 (void *)&cmd_setfwd_fwd,
1059 (void *)&cmd_setfwd_mode,
1064 /* list of instructions */
1065 cmdline_parse_ctx_t main_ctx[] = {
1066 (cmdline_parse_inst_t *)&cmd_help,
1067 (cmdline_parse_inst_t *)&cmd_send_file,
1068 (cmdline_parse_inst_t *)&cmd_start,
1069 (cmdline_parse_inst_t *)&cmd_stop,
1070 (cmdline_parse_inst_t *)&cmd_stats,
1071 (cmdline_parse_inst_t *)&cmd_set_fwd_mode,
1072 (cmdline_parse_inst_t *)&cmd_quit,
1076 /* prompt function, called from main on MASTER lcore */
1082 cl = cmdline_stdin_new(main_ctx, "ntb> ");
1086 cmdline_interact(cl);
1087 cmdline_stdin_exit(cl);
1091 signal_handler(int signum)
1093 if (signum == SIGINT || signum == SIGTERM) {
1094 printf("\nSignal %d received, preparing to exit...\n", signum);
1095 signal(signum, SIG_DFL);
1096 kill(getpid(), signum);
1100 #define OPT_BUF_SIZE "buf-size"
1101 #define OPT_FWD_MODE "fwd-mode"
1102 #define OPT_NB_DESC "nb-desc"
1103 #define OPT_TXFREET "txfreet"
1104 #define OPT_BURST "burst"
1108 /* long options mapped to a short option */
1109 OPT_NO_ZERO_COPY_NUM = 1,
1118 static const char short_options[] =
1119 "i" /* interactive mode */
1122 static const struct option lgopts[] = {
1123 {OPT_BUF_SIZE, 1, NULL, OPT_BUF_SIZE_NUM },
1124 {OPT_FWD_MODE, 1, NULL, OPT_FWD_MODE_NUM },
1125 {OPT_NB_DESC, 1, NULL, OPT_NB_DESC_NUM },
1126 {OPT_TXFREET, 1, NULL, OPT_TXFREET_NUM },
1127 {OPT_BURST, 1, NULL, OPT_BURST_NUM },
1128 {OPT_QP, 1, NULL, OPT_QP_NUM },
1133 ntb_usage(const char *prgname)
1135 printf("%s [EAL options] -- [options]\n"
1136 "-i: run in interactive mode.\n"
1137 "-qp=N: set number of queues as N (N > 0, default: 1).\n"
1138 "--fwd-mode=N: set fwd mode (N: file-trans | rxonly | "
1139 "txonly | iofwd, default: file-trans)\n"
1140 "--buf-size=N: set mbuf dataroom size as N (0 < N < 65535,"
1141 " default: 2048).\n"
1142 "--nb-desc=N: set number of descriptors as N (%u <= N <= %u,"
1143 " default: 1024).\n"
1144 "--txfreet=N: set tx free thresh for NTB driver as N. (N >= 0)\n"
1145 "--burst=N: set pkt burst as N (0 < N <= %u default: 32).\n",
1146 prgname, NTB_MIN_DESC_SIZE, NTB_MAX_DESC_SIZE,
1151 ntb_parse_args(int argc, char **argv)
1153 char *prgname = argv[0], **argvopt = argv;
1154 int opt, opt_idx, n, i;
1156 while ((opt = getopt_long(argc, argvopt, short_options,
1157 lgopts, &opt_idx)) != EOF) {
1160 printf("Interactive-mode selected.\n");
1168 rte_exit(EXIT_FAILURE, "q must be > 0.\n");
1170 case OPT_BUF_SIZE_NUM:
1172 if (n > RTE_PKTMBUF_HEADROOM && n <= 0xFFFF)
1175 rte_exit(EXIT_FAILURE, "buf-size must be > "
1176 "%u and < 65536.\n",
1177 RTE_PKTMBUF_HEADROOM);
1179 case OPT_FWD_MODE_NUM:
1180 for (i = 0; i < MAX_FWD_MODE; i++) {
1181 if (!strcmp(optarg, fwd_mode_s[i])) {
1186 if (i == MAX_FWD_MODE)
1187 rte_exit(EXIT_FAILURE, "Unsupported mode. "
1188 "(Should be: file-trans | rxonly | txonly "
1191 case OPT_NB_DESC_NUM:
1193 if (n >= NTB_MIN_DESC_SIZE && n <= NTB_MAX_DESC_SIZE)
1196 rte_exit(EXIT_FAILURE, "nb-desc must be within"
1197 " [%u, %u].\n", NTB_MIN_DESC_SIZE,
1200 case OPT_TXFREET_NUM:
1205 rte_exit(EXIT_FAILURE, "txfreet must be"
1210 if (n > 0 && n <= NTB_MAX_PKT_BURST)
1213 rte_exit(EXIT_FAILURE, "burst must be within "
1214 "(0, %u].\n", NTB_MAX_PKT_BURST);
1219 rte_exit(EXIT_FAILURE,
1220 "Command line is incomplete or incorrect.\n");
1227 ntb_mempool_mz_free(__rte_unused struct rte_mempool_memhdr *memhdr,
1230 const struct rte_memzone *mz = opaque;
1231 rte_memzone_free(mz);
1234 static struct rte_mempool *
1235 ntb_mbuf_pool_create(uint16_t mbuf_seg_size, uint32_t nb_mbuf,
1236 struct ntb_dev_info ntb_info,
1237 struct ntb_dev_config *ntb_conf,
1238 unsigned int socket_id)
1240 size_t mz_len, total_elt_sz, max_mz_len, left_sz;
1241 struct rte_pktmbuf_pool_private mbp_priv;
1242 char pool_name[RTE_MEMPOOL_NAMESIZE];
1243 char mz_name[RTE_MEMZONE_NAMESIZE];
1244 const struct rte_memzone *mz;
1245 struct rte_mempool *mp;
1250 snprintf(pool_name, sizeof(pool_name), "ntb_mbuf_pool_%u", socket_id);
1251 mp = rte_mempool_create_empty(pool_name, nb_mbuf,
1252 (mbuf_seg_size + sizeof(struct rte_mbuf)),
1254 sizeof(struct rte_pktmbuf_pool_private),
1259 mbp_priv.mbuf_data_room_size = mbuf_seg_size;
1260 mbp_priv.mbuf_priv_size = 0;
1261 rte_pktmbuf_pool_init(mp, &mbp_priv);
1263 ntb_conf->mz_list = rte_zmalloc("ntb_memzone_list",
1264 sizeof(struct rte_memzone *) *
1265 ntb_info.mw_cnt, 0);
1266 if (ntb_conf->mz_list == NULL)
1269 /* Put ntb header on mw0. */
1270 if (ntb_info.mw_size[0] < ntb_info.ntb_hdr_size) {
1271 printf("mw0 (size: %" PRIu64 ") is not enough for ntb hdr"
1272 " (size: %u)\n", ntb_info.mw_size[0],
1273 ntb_info.ntb_hdr_size);
1277 total_elt_sz = mp->header_size + mp->elt_size + mp->trailer_size;
1278 left_sz = total_elt_sz * nb_mbuf;
1279 for (mz_id = 0; mz_id < ntb_info.mw_cnt; mz_id++) {
1280 /* If populated mbuf is enough, no need to reserve extra mz. */
1283 snprintf(mz_name, sizeof(mz_name), "ntb_mw_%d", mz_id);
1284 align = ntb_info.mw_size_align ? ntb_info.mw_size[mz_id] :
1285 RTE_CACHE_LINE_SIZE;
1286 /* Reserve ntb header space on memzone 0. */
1287 max_mz_len = mz_id ? ntb_info.mw_size[mz_id] :
1288 ntb_info.mw_size[mz_id] - ntb_info.ntb_hdr_size;
1289 mz_len = left_sz <= max_mz_len ? left_sz :
1290 (max_mz_len / total_elt_sz * total_elt_sz);
1293 mz = rte_memzone_reserve_aligned(mz_name, mz_len, socket_id,
1294 RTE_MEMZONE_IOVA_CONTIG, align);
1296 printf("Cannot allocate %" PRIu64 " aligned memzone"
1297 " %u\n", align, mz_id);
1302 /* Reserve ntb header space on memzone 0. */
1304 ret = rte_mempool_populate_iova(mp, mz->addr, mz->iova,
1305 mz->len, ntb_mempool_mz_free,
1306 (void *)(uintptr_t)mz);
1308 ret = rte_mempool_populate_iova(mp,
1309 (void *)((size_t)mz->addr +
1310 ntb_info.ntb_hdr_size),
1311 mz->iova + ntb_info.ntb_hdr_size,
1312 mz->len - ntb_info.ntb_hdr_size,
1313 ntb_mempool_mz_free,
1314 (void *)(uintptr_t)mz);
1316 rte_memzone_free(mz);
1317 rte_mempool_free(mp);
1321 ntb_conf->mz_list[mz_id] = mz;
1324 printf("mw space is not enough for mempool.\n");
1328 ntb_conf->mz_num = mz_id;
1329 rte_mempool_obj_iter(mp, rte_pktmbuf_init, NULL);
1333 rte_mempool_free(mp);
1338 main(int argc, char **argv)
1340 struct rte_eth_conf eth_pconf = eth_port_conf;
1341 struct rte_rawdev_info ntb_rawdev_conf;
1342 struct rte_rawdev_info ntb_rawdev_info;
1343 struct rte_eth_dev_info ethdev_info;
1344 struct rte_eth_rxconf eth_rx_conf;
1345 struct rte_eth_txconf eth_tx_conf;
1346 struct ntb_queue_conf ntb_q_conf;
1347 struct ntb_dev_config ntb_conf;
1348 struct ntb_dev_info ntb_info;
1349 uint64_t ntb_link_status;
1353 signal(SIGINT, signal_handler);
1354 signal(SIGTERM, signal_handler);
1356 ret = rte_eal_init(argc, argv);
1358 rte_exit(EXIT_FAILURE, "Error with EAL initialization.\n");
1360 if (rte_lcore_count() < 2)
1361 rte_exit(EXIT_FAILURE, "Need at least 2 cores\n");
1363 /* Find 1st ntb rawdev. */
1364 for (i = 0; i < RTE_RAWDEV_MAX_DEVS; i++)
1365 if (rte_rawdevs[i].driver_name &&
1366 (strncmp(rte_rawdevs[i].driver_name, "raw_ntb",
1367 NTB_DRV_NAME_LEN) == 0) && (rte_rawdevs[i].attached == 1))
1370 if (i == RTE_RAWDEV_MAX_DEVS)
1371 rte_exit(EXIT_FAILURE, "Cannot find any ntb device.\n");
1378 ntb_parse_args(argc, argv);
1380 rte_rawdev_set_attr(dev_id, NTB_QUEUE_SZ_NAME, nb_desc);
1381 printf("Set queue size as %u.\n", nb_desc);
1382 rte_rawdev_set_attr(dev_id, NTB_QUEUE_NUM_NAME, num_queues);
1383 printf("Set queue number as %u.\n", num_queues);
1384 ntb_rawdev_info.dev_private = (rte_rawdev_obj_t)(&ntb_info);
1385 rte_rawdev_info_get(dev_id, &ntb_rawdev_info);
1387 nb_mbuf = nb_desc * num_queues * 2 * 2 + rte_lcore_count() *
1389 mbuf_pool = ntb_mbuf_pool_create(ntb_buf_size, nb_mbuf, ntb_info,
1390 &ntb_conf, rte_socket_id());
1391 if (mbuf_pool == NULL)
1392 rte_exit(EXIT_FAILURE, "Cannot create mbuf pool.\n");
1394 ntb_conf.num_queues = num_queues;
1395 ntb_conf.queue_size = nb_desc;
1396 ntb_rawdev_conf.dev_private = (rte_rawdev_obj_t)(&ntb_conf);
1397 ret = rte_rawdev_configure(dev_id, &ntb_rawdev_conf);
1399 rte_exit(EXIT_FAILURE, "Can't config ntb dev: err=%d, "
1400 "port=%u\n", ret, dev_id);
1402 ntb_q_conf.tx_free_thresh = tx_free_thresh;
1403 ntb_q_conf.nb_desc = nb_desc;
1404 ntb_q_conf.rx_mp = mbuf_pool;
1405 for (i = 0; i < num_queues; i++) {
1406 /* Setup rawdev queue */
1407 ret = rte_rawdev_queue_setup(dev_id, i, &ntb_q_conf);
1409 rte_exit(EXIT_FAILURE,
1410 "Failed to setup ntb queue %u.\n", i);
1413 /* Waiting for peer dev up at most 100s.*/
1414 printf("Checking ntb link status...\n");
1415 for (i = 0; i < 1000; i++) {
1416 rte_rawdev_get_attr(dev_id, NTB_LINK_STATUS_NAME,
1418 if (ntb_link_status) {
1419 printf("Peer dev ready, ntb link up.\n");
1424 rte_rawdev_get_attr(dev_id, NTB_LINK_STATUS_NAME, &ntb_link_status);
1425 if (ntb_link_status == 0)
1426 printf("Expire 100s. Link is not up. Please restart app.\n");
1428 ret = rte_rawdev_start(dev_id);
1430 rte_exit(EXIT_FAILURE, "rte_rawdev_start: err=%d, port=%u\n",
1433 /* Find 1st ethdev */
1434 eth_port_id = rte_eth_find_next(0);
1436 if (eth_port_id < RTE_MAX_ETHPORTS) {
1437 rte_eth_dev_info_get(eth_port_id, ðdev_info);
1438 eth_pconf.rx_adv_conf.rss_conf.rss_hf &=
1439 ethdev_info.flow_type_rss_offloads;
1440 ret = rte_eth_dev_configure(eth_port_id, num_queues,
1441 num_queues, ð_pconf);
1443 rte_exit(EXIT_FAILURE, "Can't config ethdev: err=%d, "
1444 "port=%u\n", ret, eth_port_id);
1445 eth_rx_conf = ethdev_info.default_rxconf;
1446 eth_rx_conf.offloads = eth_pconf.rxmode.offloads;
1447 eth_tx_conf = ethdev_info.default_txconf;
1448 eth_tx_conf.offloads = eth_pconf.txmode.offloads;
1450 /* Setup ethdev queue if ethdev exists */
1451 for (i = 0; i < num_queues; i++) {
1452 ret = rte_eth_rx_queue_setup(eth_port_id, i, nb_desc,
1453 rte_eth_dev_socket_id(eth_port_id),
1454 ð_rx_conf, mbuf_pool);
1456 rte_exit(EXIT_FAILURE,
1457 "Failed to setup eth rxq %u.\n", i);
1458 ret = rte_eth_tx_queue_setup(eth_port_id, i, nb_desc,
1459 rte_eth_dev_socket_id(eth_port_id),
1462 rte_exit(EXIT_FAILURE,
1463 "Failed to setup eth txq %u.\n", i);
1466 ret = rte_eth_dev_start(eth_port_id);
1468 rte_exit(EXIT_FAILURE, "rte_eth_dev_start: err=%d, "
1469 "port=%u\n", ret, eth_port_id);
1472 /* initialize port stats */
1473 memset(&ntb_port_stats, 0, sizeof(ntb_port_stats));
1475 /* Set default fwd mode if user doesn't set it. */
1476 if (fwd_mode == MAX_FWD_MODE && eth_port_id < RTE_MAX_ETHPORTS) {
1477 printf("Set default fwd mode as iofwd.\n");
1480 if (fwd_mode == MAX_FWD_MODE) {
1481 printf("Set default fwd mode as file-trans.\n");
1482 fwd_mode = FILE_TRANS;