common/cnxk: allow building for generic arm64
[dpdk.git] / examples / ntb / ntb_fwd.c
1 /* SPDX-License-Identifier: BSD-3-Clause
2  * Copyright(c) 2019 Intel Corporation
3  */
4 #include <stdint.h>
5 #include <stdio.h>
6 #include <inttypes.h>
7 #include <unistd.h>
8 #include <signal.h>
9 #include <string.h>
10 #include <getopt.h>
11
12 #include <cmdline_parse_string.h>
13 #include <cmdline_socket.h>
14 #include <cmdline.h>
15 #include <rte_common.h>
16 #include <rte_rawdev.h>
17 #include <rte_ethdev.h>
18 #include <rte_malloc.h>
19 #include <rte_lcore.h>
20 #include <rte_cycles.h>
21 #include <rte_pmd_ntb.h>
22 #include <rte_mbuf_pool_ops.h>
23
24 /* Per-port statistics struct */
25 struct ntb_port_statistics {
26         uint64_t tx;
27         uint64_t rx;
28 } __rte_cache_aligned;
29 /* Port 0: NTB dev, Port 1: ethdev when iofwd. */
30 struct ntb_port_statistics ntb_port_stats[2];
31
32 struct ntb_fwd_stream {
33         uint16_t tx_port;
34         uint16_t rx_port;
35         uint16_t qp_id;
36         uint8_t tx_ntb;  /* If ntb device is tx port. */
37 };
38
39 struct ntb_fwd_lcore_conf {
40         uint16_t stream_id;
41         uint16_t nb_stream;
42         uint8_t stopped;
43 };
44
45 enum ntb_fwd_mode {
46         FILE_TRANS = 0,
47         RXONLY,
48         TXONLY,
49         IOFWD,
50         MAX_FWD_MODE,
51 };
52 static const char *const fwd_mode_s[] = {
53         "file-trans",
54         "rxonly",
55         "txonly",
56         "iofwd",
57         NULL,
58 };
59 static enum ntb_fwd_mode fwd_mode = MAX_FWD_MODE;
60
61 static struct ntb_fwd_lcore_conf fwd_lcore_conf[RTE_MAX_LCORE];
62 static struct ntb_fwd_stream *fwd_streams;
63
64 static struct rte_mempool *mbuf_pool;
65
66 #define NTB_DRV_NAME_LEN 7
67 #define MEMPOOL_CACHE_SIZE 256
68
69 static uint8_t in_test;
70 static uint8_t interactive = 1;
71 static uint16_t eth_port_id = RTE_MAX_ETHPORTS;
72 static uint16_t dev_id;
73
74 /* Number of queues, default set as 1 */
75 static uint16_t num_queues = 1;
76 static uint16_t ntb_buf_size = RTE_MBUF_DEFAULT_BUF_SIZE;
77
78 /* Configurable number of descriptors */
79 #define NTB_DEFAULT_NUM_DESCS 1024
80 static uint16_t nb_desc = NTB_DEFAULT_NUM_DESCS;
81
82 static uint16_t tx_free_thresh;
83
84 #define NTB_MAX_PKT_BURST 32
85 #define NTB_DFLT_PKT_BURST 32
86 static uint16_t pkt_burst = NTB_DFLT_PKT_BURST;
87
88 #define BURST_TX_RETRIES 64
89
90 static struct rte_eth_conf eth_port_conf = {
91         .rxmode = {
92                 .mq_mode = RTE_ETH_MQ_RX_RSS,
93                 .split_hdr_size = 0,
94         },
95         .rx_adv_conf = {
96                 .rss_conf = {
97                         .rss_key = NULL,
98                         .rss_hf = RTE_ETH_RSS_IP,
99                 },
100         },
101         .txmode = {
102                 .mq_mode = RTE_ETH_MQ_TX_NONE,
103         },
104 };
105
106 /* *** Help command with introduction. *** */
107 struct cmd_help_result {
108         cmdline_fixed_string_t help;
109 };
110
111 static void
112 cmd_help_parsed(__rte_unused void *parsed_result,
113                 struct cmdline *cl,
114                 __rte_unused void *data)
115 {
116         cmdline_printf(
117                 cl,
118                 "\n"
119                 "The following commands are currently available:\n\n"
120                 "Control:\n"
121                 "    quit                                      :"
122                 " Quit the application.\n"
123                 "\nTransmission:\n"
124                 "    send [path]                               :"
125                 " Send [path] file. Only take effect in file-trans mode\n"
126                 "    start                                     :"
127                 " Start transmissions.\n"
128                 "    stop                                      :"
129                 " Stop transmissions.\n"
130                 "    clear/show port stats                     :"
131                 " Clear/show port stats.\n"
132                 "    set fwd file-trans/rxonly/txonly/iofwd    :"
133                 " Set packet forwarding mode.\n"
134         );
135
136 }
137
138 cmdline_parse_token_string_t cmd_help_help =
139         TOKEN_STRING_INITIALIZER(struct cmd_help_result, help, "help");
140
141 cmdline_parse_inst_t cmd_help = {
142         .f = cmd_help_parsed,
143         .data = NULL,
144         .help_str = "show help",
145         .tokens = {
146                 (void *)&cmd_help_help,
147                 NULL,
148         },
149 };
150
151 /* *** QUIT *** */
152 struct cmd_quit_result {
153         cmdline_fixed_string_t quit;
154 };
155
156 static void
157 cmd_quit_parsed(__rte_unused void *parsed_result,
158                 struct cmdline *cl,
159                 __rte_unused void *data)
160 {
161         struct ntb_fwd_lcore_conf *conf;
162         uint32_t lcore_id;
163
164         /* Stop transmission first. */
165         RTE_LCORE_FOREACH_WORKER(lcore_id) {
166                 conf = &fwd_lcore_conf[lcore_id];
167
168                 if (!conf->nb_stream)
169                         continue;
170
171                 if (conf->stopped)
172                         continue;
173
174                 conf->stopped = 1;
175         }
176         printf("\nWaiting for lcores to finish...\n");
177         rte_eal_mp_wait_lcore();
178         in_test = 0;
179
180         /* Stop traffic and Close port. */
181         rte_rawdev_stop(dev_id);
182         rte_rawdev_close(dev_id);
183         if (eth_port_id < RTE_MAX_ETHPORTS && fwd_mode == IOFWD) {
184                 rte_eth_dev_stop(eth_port_id);
185                 rte_eth_dev_close(eth_port_id);
186         }
187
188         cmdline_quit(cl);
189 }
190
191 cmdline_parse_token_string_t cmd_quit_quit =
192                 TOKEN_STRING_INITIALIZER(struct cmd_quit_result, quit, "quit");
193
194 cmdline_parse_inst_t cmd_quit = {
195         .f = cmd_quit_parsed,
196         .data = NULL,
197         .help_str = "exit application",
198         .tokens = {
199                 (void *)&cmd_quit_quit,
200                 NULL,
201         },
202 };
203
204 /* *** SEND FILE PARAMETERS *** */
205 struct cmd_sendfile_result {
206         cmdline_fixed_string_t send_string;
207         char filepath[];
208 };
209
210 static void
211 cmd_sendfile_parsed(void *parsed_result,
212                     __rte_unused struct cmdline *cl,
213                     __rte_unused void *data)
214 {
215         struct cmd_sendfile_result *res = parsed_result;
216         struct rte_rawdev_buf *pkts_send[NTB_MAX_PKT_BURST];
217         struct rte_mbuf *mbuf_send[NTB_MAX_PKT_BURST];
218         uint64_t size, count, i, j, nb_burst;
219         uint16_t nb_tx, buf_size;
220         unsigned int nb_pkt;
221         size_t queue_id = 0;
222         uint16_t retry = 0;
223         uint32_t val;
224         FILE *file;
225         int ret;
226
227         if (num_queues != 1) {
228                 printf("File transmission only supports 1 queue.\n");
229                 num_queues = 1;
230         }
231
232         file = fopen(res->filepath, "r");
233         if (file == NULL) {
234                 printf("Fail to open the file.\n");
235                 return;
236         }
237
238         if (fseek(file, 0, SEEK_END) < 0) {
239                 printf("Fail to get file size.\n");
240                 fclose(file);
241                 return;
242         }
243         size = ftell(file);
244         if (fseek(file, 0, SEEK_SET) < 0) {
245                 printf("Fail to get file size.\n");
246                 fclose(file);
247                 return;
248         }
249
250         /* Tell remote about the file size. */
251         val = size >> 32;
252         rte_rawdev_set_attr(dev_id, "spad_user_0", val);
253         val = size;
254         rte_rawdev_set_attr(dev_id, "spad_user_1", val);
255         printf("Sending file, size is %"PRIu64"\n", size);
256
257         for (i = 0; i < NTB_MAX_PKT_BURST; i++)
258                 pkts_send[i] = (struct rte_rawdev_buf *)
259                                 malloc(sizeof(struct rte_rawdev_buf));
260
261         buf_size = ntb_buf_size - RTE_PKTMBUF_HEADROOM;
262         count = (size + buf_size - 1) / buf_size;
263         nb_burst = (count + pkt_burst - 1) / pkt_burst;
264
265         for (i = 0; i < nb_burst; i++) {
266                 val = RTE_MIN(count, pkt_burst);
267                 if (rte_mempool_get_bulk(mbuf_pool, (void **)mbuf_send,
268                                         val) == 0) {
269                         for (nb_pkt = 0; nb_pkt < val; nb_pkt++) {
270                                 mbuf_send[nb_pkt]->port = dev_id;
271                                 mbuf_send[nb_pkt]->data_len =
272                                 fread(rte_pktmbuf_mtod(mbuf_send[nb_pkt],
273                                         void *), 1, buf_size, file);
274                                 mbuf_send[nb_pkt]->pkt_len =
275                                         mbuf_send[nb_pkt]->data_len;
276                                 pkts_send[nb_pkt]->buf_addr = mbuf_send[nb_pkt];
277                         }
278                 } else {
279                         for (nb_pkt = 0; nb_pkt < val; nb_pkt++) {
280                                 mbuf_send[nb_pkt] =
281                                         rte_mbuf_raw_alloc(mbuf_pool);
282                                 if (mbuf_send[nb_pkt] == NULL)
283                                         break;
284                                 mbuf_send[nb_pkt]->port = dev_id;
285                                 mbuf_send[nb_pkt]->data_len =
286                                 fread(rte_pktmbuf_mtod(mbuf_send[nb_pkt],
287                                         void *), 1, buf_size, file);
288                                 mbuf_send[nb_pkt]->pkt_len =
289                                         mbuf_send[nb_pkt]->data_len;
290                                 pkts_send[nb_pkt]->buf_addr = mbuf_send[nb_pkt];
291                         }
292                 }
293
294                 ret = rte_rawdev_enqueue_buffers(dev_id, pkts_send, nb_pkt,
295                                                 (void *)queue_id);
296                 if (ret < 0) {
297                         printf("Enqueue failed with err %d\n", ret);
298                         for (j = 0; j < nb_pkt; j++)
299                                 rte_pktmbuf_free(mbuf_send[j]);
300                         goto clean;
301                 }
302                 nb_tx = ret;
303                 while (nb_tx != nb_pkt && retry < BURST_TX_RETRIES) {
304                         rte_delay_us(1);
305                         ret = rte_rawdev_enqueue_buffers(dev_id,
306                                         &pkts_send[nb_tx], nb_pkt - nb_tx,
307                                         (void *)queue_id);
308                         if (ret < 0) {
309                                 printf("Enqueue failed with err %d\n", ret);
310                                 for (j = nb_tx; j < nb_pkt; j++)
311                                         rte_pktmbuf_free(mbuf_send[j]);
312                                 goto clean;
313                         }
314                         nb_tx += ret;
315                 }
316                 count -= nb_pkt;
317         }
318
319         /* Clear register after file sending done. */
320         rte_rawdev_set_attr(dev_id, "spad_user_0", 0);
321         rte_rawdev_set_attr(dev_id, "spad_user_1", 0);
322         printf("Done sending file.\n");
323
324 clean:
325         for (i = 0; i < NTB_MAX_PKT_BURST; i++)
326                 free(pkts_send[i]);
327         fclose(file);
328 }
329
330 cmdline_parse_token_string_t cmd_send_file_send =
331         TOKEN_STRING_INITIALIZER(struct cmd_sendfile_result, send_string,
332                                  "send");
333 cmdline_parse_token_string_t cmd_send_file_filepath =
334         TOKEN_STRING_INITIALIZER(struct cmd_sendfile_result, filepath, NULL);
335
336
337 cmdline_parse_inst_t cmd_send_file = {
338         .f = cmd_sendfile_parsed,
339         .data = NULL,
340         .help_str = "send <file_path>",
341         .tokens = {
342                 (void *)&cmd_send_file_send,
343                 (void *)&cmd_send_file_filepath,
344                 NULL,
345         },
346 };
347
348 #define RECV_FILE_LEN 30
349 static int
350 start_polling_recv_file(void *param)
351 {
352         struct rte_rawdev_buf *pkts_recv[NTB_MAX_PKT_BURST];
353         struct ntb_fwd_lcore_conf *conf = param;
354         struct rte_mbuf *mbuf;
355         char filepath[RECV_FILE_LEN];
356         uint64_t val, size, file_len;
357         uint16_t nb_rx, i, file_no;
358         size_t queue_id = 0;
359         FILE *file;
360         int ret;
361
362         for (i = 0; i < NTB_MAX_PKT_BURST; i++)
363                 pkts_recv[i] = (struct rte_rawdev_buf *)
364                                 malloc(sizeof(struct rte_rawdev_buf));
365
366         file_no = 0;
367         while (!conf->stopped) {
368                 snprintf(filepath, RECV_FILE_LEN, "ntb_recv_file%d", file_no);
369                 file = fopen(filepath, "w");
370                 if (file == NULL) {
371                         printf("Fail to open the file.\n");
372                         return -EINVAL;
373                 }
374
375                 rte_rawdev_get_attr(dev_id, "spad_user_0", &val);
376                 size = val << 32;
377                 rte_rawdev_get_attr(dev_id, "spad_user_1", &val);
378                 size |= val;
379
380                 if (!size) {
381                         fclose(file);
382                         continue;
383                 }
384
385                 file_len = 0;
386                 nb_rx = NTB_MAX_PKT_BURST;
387                 while (file_len < size && !conf->stopped) {
388                         ret = rte_rawdev_dequeue_buffers(dev_id, pkts_recv,
389                                         pkt_burst, (void *)queue_id);
390                         if (ret < 0) {
391                                 printf("Dequeue failed with err %d\n", ret);
392                                 fclose(file);
393                                 goto clean;
394                         }
395                         nb_rx = ret;
396                         ntb_port_stats[0].rx += nb_rx;
397                         for (i = 0; i < nb_rx; i++) {
398                                 mbuf = pkts_recv[i]->buf_addr;
399                                 fwrite(rte_pktmbuf_mtod(mbuf, void *), 1,
400                                         mbuf->data_len, file);
401                                 file_len += mbuf->data_len;
402                                 rte_pktmbuf_free(mbuf);
403                                 pkts_recv[i]->buf_addr = NULL;
404                         }
405                 }
406
407                 printf("Received file (size: %" PRIu64 ") from peer to %s.\n",
408                         size, filepath);
409                 fclose(file);
410                 file_no++;
411         }
412
413 clean:
414         for (i = 0; i < NTB_MAX_PKT_BURST; i++)
415                 free(pkts_recv[i]);
416         return 0;
417 }
418
419 static int
420 start_iofwd_per_lcore(void *param)
421 {
422         struct rte_rawdev_buf *ntb_buf[NTB_MAX_PKT_BURST];
423         struct rte_mbuf *pkts_burst[NTB_MAX_PKT_BURST];
424         struct ntb_fwd_lcore_conf *conf = param;
425         struct ntb_fwd_stream fs;
426         uint16_t nb_rx, nb_tx;
427         int i, j, ret;
428
429         for (i = 0; i < NTB_MAX_PKT_BURST; i++)
430                 ntb_buf[i] = (struct rte_rawdev_buf *)
431                              malloc(sizeof(struct rte_rawdev_buf));
432
433         while (!conf->stopped) {
434                 for (i = 0; i < conf->nb_stream; i++) {
435                         fs = fwd_streams[conf->stream_id + i];
436                         if (fs.tx_ntb) {
437                                 nb_rx = rte_eth_rx_burst(fs.rx_port,
438                                                 fs.qp_id, pkts_burst,
439                                                 pkt_burst);
440                                 if (unlikely(nb_rx == 0))
441                                         continue;
442                                 for (j = 0; j < nb_rx; j++)
443                                         ntb_buf[j]->buf_addr = pkts_burst[j];
444                                 ret = rte_rawdev_enqueue_buffers(fs.tx_port,
445                                                 ntb_buf, nb_rx,
446                                                 (void *)(size_t)fs.qp_id);
447                                 if (ret < 0) {
448                                         printf("Enqueue failed with err %d\n",
449                                                 ret);
450                                         for (j = 0; j < nb_rx; j++)
451                                                 rte_pktmbuf_free(pkts_burst[j]);
452                                         goto clean;
453                                 }
454                                 nb_tx = ret;
455                                 ntb_port_stats[0].tx += nb_tx;
456                                 ntb_port_stats[1].rx += nb_rx;
457                         } else {
458                                 ret = rte_rawdev_dequeue_buffers(fs.rx_port,
459                                                 ntb_buf, pkt_burst,
460                                                 (void *)(size_t)fs.qp_id);
461                                 if (ret < 0) {
462                                         printf("Dequeue failed with err %d\n",
463                                                 ret);
464                                         goto clean;
465                                 }
466                                 nb_rx = ret;
467                                 if (unlikely(nb_rx == 0))
468                                         continue;
469                                 for (j = 0; j < nb_rx; j++)
470                                         pkts_burst[j] = ntb_buf[j]->buf_addr;
471                                 nb_tx = rte_eth_tx_burst(fs.tx_port,
472                                         fs.qp_id, pkts_burst, nb_rx);
473                                 ntb_port_stats[1].tx += nb_tx;
474                                 ntb_port_stats[0].rx += nb_rx;
475                         }
476                         if (unlikely(nb_tx < nb_rx)) {
477                                 do {
478                                         rte_pktmbuf_free(pkts_burst[nb_tx]);
479                                 } while (++nb_tx < nb_rx);
480                         }
481                 }
482         }
483
484 clean:
485         for (i = 0; i < NTB_MAX_PKT_BURST; i++)
486                 free(ntb_buf[i]);
487
488         return 0;
489 }
490
491 static int
492 start_rxonly_per_lcore(void *param)
493 {
494         struct rte_rawdev_buf *ntb_buf[NTB_MAX_PKT_BURST];
495         struct ntb_fwd_lcore_conf *conf = param;
496         struct ntb_fwd_stream fs;
497         uint16_t nb_rx;
498         int i, j, ret;
499
500         for (i = 0; i < NTB_MAX_PKT_BURST; i++)
501                 ntb_buf[i] = (struct rte_rawdev_buf *)
502                              malloc(sizeof(struct rte_rawdev_buf));
503
504         while (!conf->stopped) {
505                 for (i = 0; i < conf->nb_stream; i++) {
506                         fs = fwd_streams[conf->stream_id + i];
507                         ret = rte_rawdev_dequeue_buffers(fs.rx_port,
508                               ntb_buf, pkt_burst, (void *)(size_t)fs.qp_id);
509                         if (ret < 0) {
510                                 printf("Dequeue failed with err %d\n", ret);
511                                 goto clean;
512                         }
513                         nb_rx = ret;
514                         if (unlikely(nb_rx == 0))
515                                 continue;
516                         ntb_port_stats[0].rx += nb_rx;
517
518                         for (j = 0; j < nb_rx; j++)
519                                 rte_pktmbuf_free(ntb_buf[j]->buf_addr);
520                 }
521         }
522
523 clean:
524         for (i = 0; i < NTB_MAX_PKT_BURST; i++)
525                 free(ntb_buf[i]);
526
527         return 0;
528 }
529
530
531 static int
532 start_txonly_per_lcore(void *param)
533 {
534         struct rte_rawdev_buf *ntb_buf[NTB_MAX_PKT_BURST];
535         struct rte_mbuf *pkts_burst[NTB_MAX_PKT_BURST];
536         struct ntb_fwd_lcore_conf *conf = param;
537         struct ntb_fwd_stream fs;
538         uint16_t nb_pkt, nb_tx;
539         int i, j, ret;
540
541         for (i = 0; i < NTB_MAX_PKT_BURST; i++)
542                 ntb_buf[i] = (struct rte_rawdev_buf *)
543                              malloc(sizeof(struct rte_rawdev_buf));
544
545         while (!conf->stopped) {
546                 for (i = 0; i < conf->nb_stream; i++) {
547                         fs = fwd_streams[conf->stream_id + i];
548                         if (rte_mempool_get_bulk(mbuf_pool, (void **)pkts_burst,
549                                   pkt_burst) == 0) {
550                                 for (nb_pkt = 0; nb_pkt < pkt_burst; nb_pkt++) {
551                                         pkts_burst[nb_pkt]->port = dev_id;
552                                         pkts_burst[nb_pkt]->data_len =
553                                                 pkts_burst[nb_pkt]->buf_len -
554                                                 RTE_PKTMBUF_HEADROOM;
555                                         pkts_burst[nb_pkt]->pkt_len =
556                                                 pkts_burst[nb_pkt]->data_len;
557                                         ntb_buf[nb_pkt]->buf_addr =
558                                                 pkts_burst[nb_pkt];
559                                 }
560                         } else {
561                                 for (nb_pkt = 0; nb_pkt < pkt_burst; nb_pkt++) {
562                                         pkts_burst[nb_pkt] =
563                                                 rte_pktmbuf_alloc(mbuf_pool);
564                                         if (pkts_burst[nb_pkt] == NULL)
565                                                 break;
566                                         pkts_burst[nb_pkt]->port = dev_id;
567                                         pkts_burst[nb_pkt]->data_len =
568                                                 pkts_burst[nb_pkt]->buf_len -
569                                                 RTE_PKTMBUF_HEADROOM;
570                                         pkts_burst[nb_pkt]->pkt_len =
571                                                 pkts_burst[nb_pkt]->data_len;
572                                         ntb_buf[nb_pkt]->buf_addr =
573                                                 pkts_burst[nb_pkt];
574                                 }
575                         }
576                         ret = rte_rawdev_enqueue_buffers(fs.tx_port, ntb_buf,
577                                         nb_pkt, (void *)(size_t)fs.qp_id);
578                         if (ret < 0) {
579                                 printf("Enqueue failed with err %d\n", ret);
580                                 for (j = 0; j < nb_pkt; j++)
581                                         rte_pktmbuf_free(pkts_burst[j]);
582                                 goto clean;
583                         }
584                         nb_tx = ret;
585                         ntb_port_stats[0].tx += nb_tx;
586                         if (unlikely(nb_tx < nb_pkt)) {
587                                 do {
588                                         rte_pktmbuf_free(pkts_burst[nb_tx]);
589                                 } while (++nb_tx < nb_pkt);
590                         }
591                 }
592         }
593
594 clean:
595         for (i = 0; i < NTB_MAX_PKT_BURST; i++)
596                 free(ntb_buf[i]);
597
598         return 0;
599 }
600
601 static int
602 ntb_fwd_config_setup(void)
603 {
604         uint16_t i;
605
606         /* Make sure iofwd has valid ethdev. */
607         if (fwd_mode == IOFWD && eth_port_id >= RTE_MAX_ETHPORTS) {
608                 printf("No ethdev, cannot be in iofwd mode.");
609                 return -EINVAL;
610         }
611
612         if (fwd_mode == IOFWD) {
613                 fwd_streams = rte_zmalloc("ntb_fwd: fwd_streams",
614                         sizeof(struct ntb_fwd_stream) * num_queues * 2,
615                         RTE_CACHE_LINE_SIZE);
616                 for (i = 0; i < num_queues; i++) {
617                         fwd_streams[i * 2].qp_id = i;
618                         fwd_streams[i * 2].tx_port = dev_id;
619                         fwd_streams[i * 2].rx_port = eth_port_id;
620                         fwd_streams[i * 2].tx_ntb = 1;
621
622                         fwd_streams[i * 2 + 1].qp_id = i;
623                         fwd_streams[i * 2 + 1].tx_port = eth_port_id;
624                         fwd_streams[i * 2 + 1].rx_port = dev_id;
625                         fwd_streams[i * 2 + 1].tx_ntb = 0;
626                 }
627                 return 0;
628         }
629
630         if (fwd_mode == RXONLY || fwd_mode == FILE_TRANS) {
631                 /* Only support 1 queue in file-trans for in order. */
632                 if (fwd_mode == FILE_TRANS)
633                         num_queues = 1;
634
635                 fwd_streams = rte_zmalloc("ntb_fwd: fwd_streams",
636                                 sizeof(struct ntb_fwd_stream) * num_queues,
637                                 RTE_CACHE_LINE_SIZE);
638                 for (i = 0; i < num_queues; i++) {
639                         fwd_streams[i].qp_id = i;
640                         fwd_streams[i].tx_port = RTE_MAX_ETHPORTS;
641                         fwd_streams[i].rx_port = dev_id;
642                         fwd_streams[i].tx_ntb = 0;
643                 }
644                 return 0;
645         }
646
647         if (fwd_mode == TXONLY) {
648                 fwd_streams = rte_zmalloc("ntb_fwd: fwd_streams",
649                                 sizeof(struct ntb_fwd_stream) * num_queues,
650                                 RTE_CACHE_LINE_SIZE);
651                 for (i = 0; i < num_queues; i++) {
652                         fwd_streams[i].qp_id = i;
653                         fwd_streams[i].tx_port = dev_id;
654                         fwd_streams[i].rx_port = RTE_MAX_ETHPORTS;
655                         fwd_streams[i].tx_ntb = 1;
656                 }
657         }
658         return 0;
659 }
660
661 static void
662 assign_stream_to_lcores(void)
663 {
664         struct ntb_fwd_lcore_conf *conf;
665         struct ntb_fwd_stream *fs;
666         uint16_t nb_streams, sm_per_lcore, sm_id, i;
667         uint32_t lcore_id;
668         uint8_t lcore_num, nb_extra;
669
670         lcore_num = rte_lcore_count();
671         /* Exclude main core */
672         lcore_num--;
673
674         nb_streams = (fwd_mode == IOFWD) ? num_queues * 2 : num_queues;
675
676         sm_per_lcore = nb_streams / lcore_num;
677         nb_extra = nb_streams % lcore_num;
678         sm_id = 0;
679         i = 0;
680
681         RTE_LCORE_FOREACH_WORKER(lcore_id) {
682                 conf = &fwd_lcore_conf[lcore_id];
683
684                 if (i < nb_extra) {
685                         conf->nb_stream = sm_per_lcore + 1;
686                         conf->stream_id = sm_id;
687                         sm_id = sm_id + sm_per_lcore + 1;
688                 } else {
689                         conf->nb_stream = sm_per_lcore;
690                         conf->stream_id = sm_id;
691                         sm_id = sm_id + sm_per_lcore;
692                 }
693
694                 i++;
695                 if (sm_id >= nb_streams)
696                         break;
697         }
698
699         /* Print packet forwarding config. */
700         RTE_LCORE_FOREACH_WORKER(lcore_id) {
701                 conf = &fwd_lcore_conf[lcore_id];
702
703                 if (!conf->nb_stream)
704                         continue;
705
706                 printf("Streams on Lcore %u :\n", lcore_id);
707                 for (i = 0; i < conf->nb_stream; i++) {
708                         fs = &fwd_streams[conf->stream_id + i];
709                         if (fwd_mode == IOFWD)
710                                 printf(" + Stream %u : %s%u RX -> %s%u TX,"
711                                         " Q=%u\n", conf->stream_id + i,
712                                         fs->tx_ntb ? "Eth" : "NTB", fs->rx_port,
713                                         fs->tx_ntb ? "NTB" : "Eth", fs->tx_port,
714                                         fs->qp_id);
715                         if (fwd_mode == FILE_TRANS || fwd_mode == RXONLY)
716                                 printf(" + Stream %u : %s%u RX only\n",
717                                         conf->stream_id, "NTB", fs->rx_port);
718                         if (fwd_mode == TXONLY)
719                                 printf(" + Stream %u : %s%u TX only\n",
720                                         conf->stream_id, "NTB", fs->tx_port);
721                 }
722         }
723 }
724
725 static void
726 start_pkt_fwd(void)
727 {
728         struct ntb_fwd_lcore_conf *conf;
729         struct rte_eth_link eth_link;
730         uint32_t lcore_id;
731         int ret, i;
732         char link_status_text[RTE_ETH_LINK_MAX_STR_LEN];
733
734         ret = ntb_fwd_config_setup();
735         if (ret < 0) {
736                 printf("Cannot start traffic. Please reset fwd mode.\n");
737                 return;
738         }
739
740         /* If using iofwd, checking ethdev link status first. */
741         if (fwd_mode == IOFWD) {
742                 printf("Checking eth link status...\n");
743                 /* Wait for eth link up at most 100 times. */
744                 for (i = 0; i < 100; i++) {
745                         ret = rte_eth_link_get(eth_port_id, &eth_link);
746                         if (ret < 0) {
747                                 printf("Link get failed with err %d\n", ret);
748                                 return;
749                         }
750                         if (eth_link.link_status) {
751                                 rte_eth_link_to_str(link_status_text,
752                                         sizeof(link_status_text),
753                                         &eth_link);
754                                 printf("Eth%u %s\n", eth_port_id,
755                                        link_status_text);
756                                 break;
757                         }
758                 }
759                 if (!eth_link.link_status) {
760                         printf("Eth%u link down. Cannot start traffic.\n",
761                                 eth_port_id);
762                         return;
763                 }
764         }
765
766         assign_stream_to_lcores();
767         in_test = 1;
768
769         RTE_LCORE_FOREACH_WORKER(lcore_id) {
770                 conf = &fwd_lcore_conf[lcore_id];
771
772                 if (!conf->nb_stream)
773                         continue;
774
775                 conf->stopped = 0;
776                 if (fwd_mode == FILE_TRANS)
777                         rte_eal_remote_launch(start_polling_recv_file,
778                                               conf, lcore_id);
779                 else if (fwd_mode == IOFWD)
780                         rte_eal_remote_launch(start_iofwd_per_lcore,
781                                               conf, lcore_id);
782                 else if (fwd_mode == RXONLY)
783                         rte_eal_remote_launch(start_rxonly_per_lcore,
784                                               conf, lcore_id);
785                 else if (fwd_mode == TXONLY)
786                         rte_eal_remote_launch(start_txonly_per_lcore,
787                                               conf, lcore_id);
788         }
789 }
790
791 /* *** START FWD PARAMETERS *** */
792 struct cmd_start_result {
793         cmdline_fixed_string_t start;
794 };
795
796 static void
797 cmd_start_parsed(__rte_unused void *parsed_result,
798                             __rte_unused struct cmdline *cl,
799                             __rte_unused void *data)
800 {
801         start_pkt_fwd();
802 }
803
804 cmdline_parse_token_string_t cmd_start_start =
805                 TOKEN_STRING_INITIALIZER(struct cmd_start_result, start, "start");
806
807 cmdline_parse_inst_t cmd_start = {
808         .f = cmd_start_parsed,
809         .data = NULL,
810         .help_str = "start pkt fwd between ntb and ethdev",
811         .tokens = {
812                 (void *)&cmd_start_start,
813                 NULL,
814         },
815 };
816
817 /* *** STOP *** */
818 struct cmd_stop_result {
819         cmdline_fixed_string_t stop;
820 };
821
822 static void
823 cmd_stop_parsed(__rte_unused void *parsed_result,
824                 __rte_unused struct cmdline *cl,
825                 __rte_unused void *data)
826 {
827         struct ntb_fwd_lcore_conf *conf;
828         uint32_t lcore_id;
829
830         RTE_LCORE_FOREACH_WORKER(lcore_id) {
831                 conf = &fwd_lcore_conf[lcore_id];
832
833                 if (!conf->nb_stream)
834                         continue;
835
836                 if (conf->stopped)
837                         continue;
838
839                 conf->stopped = 1;
840         }
841         printf("\nWaiting for lcores to finish...\n");
842         rte_eal_mp_wait_lcore();
843         in_test = 0;
844         printf("\nDone.\n");
845 }
846
847 cmdline_parse_token_string_t cmd_stop_stop =
848                 TOKEN_STRING_INITIALIZER(struct cmd_stop_result, stop, "stop");
849
850 cmdline_parse_inst_t cmd_stop = {
851         .f = cmd_stop_parsed,
852         .data = NULL,
853         .help_str = "stop: Stop packet forwarding",
854         .tokens = {
855                 (void *)&cmd_stop_stop,
856                 NULL,
857         },
858 };
859
860 static void
861 ntb_stats_clear(void)
862 {
863         int nb_ids, i;
864         uint32_t *ids;
865
866         /* Clear NTB dev stats */
867         nb_ids = rte_rawdev_xstats_names_get(dev_id, NULL, 0);
868         if (nb_ids  < 0) {
869                 printf("Error: Cannot get count of xstats\n");
870                 return;
871         }
872         ids = malloc(sizeof(uint32_t) * nb_ids);
873         for (i = 0; i < nb_ids; i++)
874                 ids[i] = i;
875         rte_rawdev_xstats_reset(dev_id, ids, nb_ids);
876         printf("\n  statistics for NTB port %d cleared\n", dev_id);
877
878         /* Clear Ethdev stats if have any */
879         if (fwd_mode == IOFWD && eth_port_id != RTE_MAX_ETHPORTS) {
880                 rte_eth_stats_reset(eth_port_id);
881                 printf("\n  statistics for ETH port %d cleared\n", eth_port_id);
882         }
883 }
884
885 static inline void
886 ntb_calculate_throughput(uint16_t port) {
887         uint64_t diff_pkts_rx, diff_pkts_tx, diff_cycles;
888         uint64_t mpps_rx, mpps_tx;
889         static uint64_t prev_pkts_rx[2];
890         static uint64_t prev_pkts_tx[2];
891         static uint64_t prev_cycles[2];
892
893         diff_cycles = prev_cycles[port];
894         prev_cycles[port] = rte_rdtsc();
895         if (diff_cycles > 0)
896                 diff_cycles = prev_cycles[port] - diff_cycles;
897         diff_pkts_rx = (ntb_port_stats[port].rx > prev_pkts_rx[port]) ?
898                 (ntb_port_stats[port].rx - prev_pkts_rx[port]) : 0;
899         diff_pkts_tx = (ntb_port_stats[port].tx > prev_pkts_tx[port]) ?
900                 (ntb_port_stats[port].tx - prev_pkts_tx[port]) : 0;
901         prev_pkts_rx[port] = ntb_port_stats[port].rx;
902         prev_pkts_tx[port] = ntb_port_stats[port].tx;
903         mpps_rx = diff_cycles > 0 ?
904                 diff_pkts_rx * rte_get_tsc_hz() / diff_cycles : 0;
905         mpps_tx = diff_cycles > 0 ?
906                 diff_pkts_tx * rte_get_tsc_hz() / diff_cycles : 0;
907         printf("  Throughput (since last show)\n");
908         printf("  Rx-pps: %12"PRIu64"\n  Tx-pps: %12"PRIu64"\n",
909                         mpps_rx, mpps_tx);
910
911 }
912
913 static void
914 ntb_stats_display(void)
915 {
916         struct rte_rawdev_xstats_name *xstats_names;
917         struct rte_eth_stats stats;
918         uint64_t *values;
919         uint32_t *ids;
920         int nb_ids, i;
921
922         printf("###### statistics for NTB port %d #######\n", dev_id);
923
924         /* Get NTB dev stats and stats names */
925         nb_ids = rte_rawdev_xstats_names_get(dev_id, NULL, 0);
926         if (nb_ids  < 0) {
927                 printf("Error: Cannot get count of xstats\n");
928                 return;
929         }
930         xstats_names = malloc(sizeof(struct rte_rawdev_xstats_name) * nb_ids);
931         if (xstats_names == NULL) {
932                 printf("Cannot allocate memory for xstats lookup\n");
933                 return;
934         }
935         if (nb_ids != rte_rawdev_xstats_names_get(
936                         dev_id, xstats_names, nb_ids)) {
937                 printf("Error: Cannot get xstats lookup\n");
938                 free(xstats_names);
939                 return;
940         }
941         ids = malloc(sizeof(uint32_t) * nb_ids);
942         for (i = 0; i < nb_ids; i++)
943                 ids[i] = i;
944         values = malloc(sizeof(uint64_t) * nb_ids);
945         if (nb_ids != rte_rawdev_xstats_get(dev_id, ids, values, nb_ids)) {
946                 printf("Error: Unable to get xstats\n");
947                 free(xstats_names);
948                 free(values);
949                 free(ids);
950                 return;
951         }
952
953         /* Display NTB dev stats */
954         for (i = 0; i < nb_ids; i++)
955                 printf("  %s: %"PRIu64"\n", xstats_names[i].name, values[i]);
956         ntb_calculate_throughput(0);
957
958         /* Get Ethdev stats if have any */
959         if (fwd_mode == IOFWD && eth_port_id != RTE_MAX_ETHPORTS) {
960                 printf("###### statistics for ETH port %d ######\n",
961                         eth_port_id);
962                 rte_eth_stats_get(eth_port_id, &stats);
963                 printf("  RX-packets: %"PRIu64"\n", stats.ipackets);
964                 printf("  RX-bytes: %"PRIu64"\n", stats.ibytes);
965                 printf("  RX-errors: %"PRIu64"\n", stats.ierrors);
966                 printf("  RX-missed: %"PRIu64"\n", stats.imissed);
967                 printf("  TX-packets: %"PRIu64"\n", stats.opackets);
968                 printf("  TX-bytes: %"PRIu64"\n", stats.obytes);
969                 printf("  TX-errors: %"PRIu64"\n", stats.oerrors);
970                 ntb_calculate_throughput(1);
971         }
972
973         free(xstats_names);
974         free(values);
975         free(ids);
976 }
977
978 /* *** SHOW/CLEAR PORT STATS *** */
979 struct cmd_stats_result {
980         cmdline_fixed_string_t show;
981         cmdline_fixed_string_t port;
982         cmdline_fixed_string_t stats;
983 };
984
985 static void
986 cmd_stats_parsed(void *parsed_result,
987                  __rte_unused struct cmdline *cl,
988                  __rte_unused void *data)
989 {
990         struct cmd_stats_result *res = parsed_result;
991         if (!strcmp(res->show, "clear"))
992                 ntb_stats_clear();
993         else
994                 ntb_stats_display();
995 }
996
997 cmdline_parse_token_string_t cmd_stats_show =
998         TOKEN_STRING_INITIALIZER(struct cmd_stats_result, show, "show#clear");
999 cmdline_parse_token_string_t cmd_stats_port =
1000         TOKEN_STRING_INITIALIZER(struct cmd_stats_result, port, "port");
1001 cmdline_parse_token_string_t cmd_stats_stats =
1002         TOKEN_STRING_INITIALIZER(struct cmd_stats_result, stats, "stats");
1003
1004
1005 cmdline_parse_inst_t cmd_stats = {
1006         .f = cmd_stats_parsed,
1007         .data = NULL,
1008         .help_str = "show|clear port stats",
1009         .tokens = {
1010                 (void *)&cmd_stats_show,
1011                 (void *)&cmd_stats_port,
1012                 (void *)&cmd_stats_stats,
1013                 NULL,
1014         },
1015 };
1016
1017 /* *** SET FORWARDING MODE *** */
1018 struct cmd_set_fwd_mode_result {
1019         cmdline_fixed_string_t set;
1020         cmdline_fixed_string_t fwd;
1021         cmdline_fixed_string_t mode;
1022 };
1023
1024 static void
1025 cmd_set_fwd_mode_parsed(__rte_unused void *parsed_result,
1026                         __rte_unused struct cmdline *cl,
1027                         __rte_unused void *data)
1028 {
1029         struct cmd_set_fwd_mode_result *res = parsed_result;
1030         int i;
1031
1032         if (in_test) {
1033                 printf("Please stop traffic first.\n");
1034                 return;
1035         }
1036
1037         for (i = 0; i < MAX_FWD_MODE; i++) {
1038                 if (!strcmp(res->mode, fwd_mode_s[i])) {
1039                         fwd_mode = i;
1040                         return;
1041                 }
1042         }
1043         printf("Invalid %s packet forwarding mode.\n", res->mode);
1044 }
1045
1046 cmdline_parse_token_string_t cmd_setfwd_set =
1047         TOKEN_STRING_INITIALIZER(struct cmd_set_fwd_mode_result, set, "set");
1048 cmdline_parse_token_string_t cmd_setfwd_fwd =
1049         TOKEN_STRING_INITIALIZER(struct cmd_set_fwd_mode_result, fwd, "fwd");
1050 cmdline_parse_token_string_t cmd_setfwd_mode =
1051         TOKEN_STRING_INITIALIZER(struct cmd_set_fwd_mode_result, mode,
1052                                 "file-trans#iofwd#txonly#rxonly");
1053
1054 cmdline_parse_inst_t cmd_set_fwd_mode = {
1055         .f = cmd_set_fwd_mode_parsed,
1056         .data = NULL,
1057         .help_str = "set forwarding mode as file-trans|rxonly|txonly|iofwd",
1058         .tokens = {
1059                 (void *)&cmd_setfwd_set,
1060                 (void *)&cmd_setfwd_fwd,
1061                 (void *)&cmd_setfwd_mode,
1062                 NULL,
1063         },
1064 };
1065
1066 /* list of instructions */
1067 cmdline_parse_ctx_t main_ctx[] = {
1068         (cmdline_parse_inst_t *)&cmd_help,
1069         (cmdline_parse_inst_t *)&cmd_send_file,
1070         (cmdline_parse_inst_t *)&cmd_start,
1071         (cmdline_parse_inst_t *)&cmd_stop,
1072         (cmdline_parse_inst_t *)&cmd_stats,
1073         (cmdline_parse_inst_t *)&cmd_set_fwd_mode,
1074         (cmdline_parse_inst_t *)&cmd_quit,
1075         NULL,
1076 };
1077
1078 /* prompt function, called from main on MAIN lcore */
1079 static void
1080 prompt(void)
1081 {
1082         struct cmdline *cl;
1083
1084         cl = cmdline_stdin_new(main_ctx, "ntb> ");
1085         if (cl == NULL)
1086                 return;
1087
1088         cmdline_interact(cl);
1089         cmdline_stdin_exit(cl);
1090 }
1091
1092 static void
1093 signal_handler(int signum)
1094 {
1095         if (signum == SIGINT || signum == SIGTERM) {
1096                 printf("\nSignal %d received, preparing to exit...\n", signum);
1097                 signal(signum, SIG_DFL);
1098                 kill(getpid(), signum);
1099         }
1100 }
1101
1102 #define OPT_BUF_SIZE         "buf-size"
1103 #define OPT_FWD_MODE         "fwd-mode"
1104 #define OPT_NB_DESC          "nb-desc"
1105 #define OPT_TXFREET          "txfreet"
1106 #define OPT_BURST            "burst"
1107 #define OPT_QP               "qp"
1108
1109 enum {
1110         /* long options mapped to a short option */
1111         OPT_NO_ZERO_COPY_NUM = 1,
1112         OPT_BUF_SIZE_NUM,
1113         OPT_FWD_MODE_NUM,
1114         OPT_NB_DESC_NUM,
1115         OPT_TXFREET_NUM,
1116         OPT_BURST_NUM,
1117         OPT_QP_NUM,
1118 };
1119
1120 static const char short_options[] =
1121         "i" /* interactive mode */
1122         ;
1123
1124 static const struct option lgopts[] = {
1125         {OPT_BUF_SIZE,     1, NULL, OPT_BUF_SIZE_NUM     },
1126         {OPT_FWD_MODE,     1, NULL, OPT_FWD_MODE_NUM     },
1127         {OPT_NB_DESC,      1, NULL, OPT_NB_DESC_NUM      },
1128         {OPT_TXFREET,      1, NULL, OPT_TXFREET_NUM      },
1129         {OPT_BURST,        1, NULL, OPT_BURST_NUM        },
1130         {OPT_QP,           1, NULL, OPT_QP_NUM           },
1131         {0,                0, NULL, 0                    }
1132 };
1133
1134 static void
1135 ntb_usage(const char *prgname)
1136 {
1137         printf("%s [EAL options] -- [options]\n"
1138                "-i: run in interactive mode.\n"
1139                "-qp=N: set number of queues as N (N > 0, default: 1).\n"
1140                "--fwd-mode=N: set fwd mode (N: file-trans | rxonly | "
1141                "txonly | iofwd, default: file-trans)\n"
1142                "--buf-size=N: set mbuf dataroom size as N (0 < N < 65535,"
1143                " default: 2048).\n"
1144                "--nb-desc=N: set number of descriptors as N (%u <= N <= %u,"
1145                " default: 1024).\n"
1146                "--txfreet=N: set tx free thresh for NTB driver as N. (N >= 0)\n"
1147                "--burst=N: set pkt burst as N (0 < N <= %u default: 32).\n",
1148                prgname, NTB_MIN_DESC_SIZE, NTB_MAX_DESC_SIZE,
1149                NTB_MAX_PKT_BURST);
1150 }
1151
1152 static void
1153 ntb_parse_args(int argc, char **argv)
1154 {
1155         char *prgname = argv[0], **argvopt = argv;
1156         int opt, opt_idx, n, i;
1157
1158         while ((opt = getopt_long(argc, argvopt, short_options,
1159                                 lgopts, &opt_idx)) != EOF) {
1160                 switch (opt) {
1161                 case 'i':
1162                         printf("Interactive-mode selected.\n");
1163                         interactive = 1;
1164                         break;
1165                 case OPT_QP_NUM:
1166                         n = atoi(optarg);
1167                         if (n > 0)
1168                                 num_queues = n;
1169                         else
1170                                 rte_exit(EXIT_FAILURE, "q must be > 0.\n");
1171                         break;
1172                 case OPT_BUF_SIZE_NUM:
1173                         n = atoi(optarg);
1174                         if (n > RTE_PKTMBUF_HEADROOM && n <= 0xFFFF)
1175                                 ntb_buf_size = n;
1176                         else
1177                                 rte_exit(EXIT_FAILURE, "buf-size must be > "
1178                                         "%u and < 65536.\n",
1179                                         RTE_PKTMBUF_HEADROOM);
1180                         break;
1181                 case OPT_FWD_MODE_NUM:
1182                         for (i = 0; i < MAX_FWD_MODE; i++) {
1183                                 if (!strcmp(optarg, fwd_mode_s[i])) {
1184                                         fwd_mode = i;
1185                                         break;
1186                                 }
1187                         }
1188                         if (i == MAX_FWD_MODE)
1189                                 rte_exit(EXIT_FAILURE, "Unsupported mode. "
1190                                 "(Should be: file-trans | rxonly | txonly "
1191                                 "| iofwd)\n");
1192                         break;
1193                 case OPT_NB_DESC_NUM:
1194                         n = atoi(optarg);
1195                         if (n >= NTB_MIN_DESC_SIZE && n <= NTB_MAX_DESC_SIZE)
1196                                 nb_desc = n;
1197                         else
1198                                 rte_exit(EXIT_FAILURE, "nb-desc must be within"
1199                                         " [%u, %u].\n", NTB_MIN_DESC_SIZE,
1200                                         NTB_MAX_DESC_SIZE);
1201                         break;
1202                 case OPT_TXFREET_NUM:
1203                         n = atoi(optarg);
1204                         if (n >= 0)
1205                                 tx_free_thresh = n;
1206                         else
1207                                 rte_exit(EXIT_FAILURE, "txfreet must be"
1208                                         " >= 0\n");
1209                         break;
1210                 case OPT_BURST_NUM:
1211                         n = atoi(optarg);
1212                         if (n > 0 && n <= NTB_MAX_PKT_BURST)
1213                                 pkt_burst = n;
1214                         else
1215                                 rte_exit(EXIT_FAILURE, "burst must be within "
1216                                         "(0, %u].\n", NTB_MAX_PKT_BURST);
1217                         break;
1218
1219                 default:
1220                         ntb_usage(prgname);
1221                         rte_exit(EXIT_FAILURE,
1222                                  "Command line is incomplete or incorrect.\n");
1223                         break;
1224                 }
1225         }
1226 }
1227
1228 static void
1229 ntb_mempool_mz_free(__rte_unused struct rte_mempool_memhdr *memhdr,
1230                 void *opaque)
1231 {
1232         const struct rte_memzone *mz = opaque;
1233         rte_memzone_free(mz);
1234 }
1235
1236 static struct rte_mempool *
1237 ntb_mbuf_pool_create(uint16_t mbuf_seg_size, uint32_t nb_mbuf,
1238                      struct ntb_dev_info ntb_info,
1239                      struct ntb_dev_config *ntb_conf,
1240                      unsigned int socket_id)
1241 {
1242         size_t mz_len, total_elt_sz, max_mz_len, left_sz;
1243         struct rte_pktmbuf_pool_private mbp_priv;
1244         char pool_name[RTE_MEMPOOL_NAMESIZE];
1245         char mz_name[RTE_MEMZONE_NAMESIZE];
1246         const struct rte_memzone *mz;
1247         struct rte_mempool *mp;
1248         uint64_t align;
1249         uint32_t mz_id;
1250         int ret;
1251
1252         snprintf(pool_name, sizeof(pool_name), "ntb_mbuf_pool_%u", socket_id);
1253         mp = rte_mempool_create_empty(pool_name, nb_mbuf,
1254                                       (mbuf_seg_size + sizeof(struct rte_mbuf)),
1255                                       MEMPOOL_CACHE_SIZE,
1256                                       sizeof(struct rte_pktmbuf_pool_private),
1257                                       socket_id, 0);
1258         if (mp == NULL)
1259                 return NULL;
1260
1261         if (rte_mempool_set_ops_byname(mp, rte_mbuf_best_mempool_ops(), NULL)) {
1262                 printf("error setting mempool handler\n");
1263                 goto fail;
1264         }
1265
1266         memset(&mbp_priv, 0, sizeof(mbp_priv));
1267         mbp_priv.mbuf_data_room_size = mbuf_seg_size;
1268         mbp_priv.mbuf_priv_size = 0;
1269         rte_pktmbuf_pool_init(mp, &mbp_priv);
1270
1271         ntb_conf->mz_list = rte_zmalloc("ntb_memzone_list",
1272                                 sizeof(struct rte_memzone *) *
1273                                 ntb_info.mw_cnt, 0);
1274         if (ntb_conf->mz_list == NULL)
1275                 goto fail;
1276
1277         /* Put ntb header on mw0. */
1278         if (ntb_info.mw_size[0] < ntb_info.ntb_hdr_size) {
1279                 printf("mw0 (size: %" PRIu64 ") is not enough for ntb hdr"
1280                        " (size: %u)\n", ntb_info.mw_size[0],
1281                        ntb_info.ntb_hdr_size);
1282                 goto fail;
1283         }
1284
1285         total_elt_sz = mp->header_size + mp->elt_size + mp->trailer_size;
1286         left_sz = total_elt_sz * nb_mbuf;
1287         for (mz_id = 0; mz_id < ntb_info.mw_cnt; mz_id++) {
1288                 /* If populated mbuf is enough, no need to reserve extra mz. */
1289                 if (!left_sz)
1290                         break;
1291                 snprintf(mz_name, sizeof(mz_name), "ntb_mw_%d", mz_id);
1292                 align = ntb_info.mw_size_align ? ntb_info.mw_size[mz_id] :
1293                         RTE_CACHE_LINE_SIZE;
1294                 /* Reserve ntb header space on memzone 0. */
1295                 max_mz_len = mz_id ? ntb_info.mw_size[mz_id] :
1296                              ntb_info.mw_size[mz_id] - ntb_info.ntb_hdr_size;
1297                 mz_len = left_sz <= max_mz_len ? left_sz :
1298                         (max_mz_len / total_elt_sz * total_elt_sz);
1299                 if (!mz_len)
1300                         continue;
1301                 mz = rte_memzone_reserve_aligned(mz_name, mz_len, socket_id,
1302                                         RTE_MEMZONE_IOVA_CONTIG, align);
1303                 if (mz == NULL) {
1304                         printf("Cannot allocate %" PRIu64 " aligned memzone"
1305                                 " %u\n", align, mz_id);
1306                         goto fail;
1307                 }
1308                 left_sz -= mz_len;
1309
1310                 /* Reserve ntb header space on memzone 0. */
1311                 if (mz_id)
1312                         ret = rte_mempool_populate_iova(mp, mz->addr, mz->iova,
1313                                         mz->len, ntb_mempool_mz_free,
1314                                         (void *)(uintptr_t)mz);
1315                 else
1316                         ret = rte_mempool_populate_iova(mp,
1317                                         (void *)((size_t)mz->addr +
1318                                         ntb_info.ntb_hdr_size),
1319                                         mz->iova + ntb_info.ntb_hdr_size,
1320                                         mz->len - ntb_info.ntb_hdr_size,
1321                                         ntb_mempool_mz_free,
1322                                         (void *)(uintptr_t)mz);
1323                 if (ret <= 0) {
1324                         rte_memzone_free(mz);
1325                         rte_mempool_free(mp);
1326                         return NULL;
1327                 }
1328
1329                 ntb_conf->mz_list[mz_id] = mz;
1330         }
1331         if (left_sz) {
1332                 printf("mw space is not enough for mempool.\n");
1333                 goto fail;
1334         }
1335
1336         ntb_conf->mz_num = mz_id;
1337         rte_mempool_obj_iter(mp, rte_pktmbuf_init, NULL);
1338
1339         return mp;
1340 fail:
1341         rte_mempool_free(mp);
1342         return NULL;
1343 }
1344
1345 int
1346 main(int argc, char **argv)
1347 {
1348         struct rte_eth_conf eth_pconf = eth_port_conf;
1349         struct rte_rawdev_info ntb_rawdev_conf;
1350         struct rte_rawdev_info ntb_rawdev_info;
1351         struct rte_eth_dev_info ethdev_info;
1352         struct rte_eth_rxconf eth_rx_conf;
1353         struct rte_eth_txconf eth_tx_conf;
1354         struct ntb_queue_conf ntb_q_conf;
1355         struct ntb_dev_config ntb_conf;
1356         struct ntb_dev_info ntb_info;
1357         uint64_t ntb_link_status;
1358         uint32_t nb_mbuf;
1359         int ret, i;
1360
1361         signal(SIGINT, signal_handler);
1362         signal(SIGTERM, signal_handler);
1363
1364         ret = rte_eal_init(argc, argv);
1365         if (ret < 0)
1366                 rte_exit(EXIT_FAILURE, "Error with EAL initialization.\n");
1367
1368         if (rte_lcore_count() < 2)
1369                 rte_exit(EXIT_FAILURE, "Need at least 2 cores\n");
1370
1371         /* Find 1st ntb rawdev. */
1372         for (i = 0; i < RTE_RAWDEV_MAX_DEVS; i++)
1373                 if (rte_rawdevs[i].driver_name &&
1374                     (strncmp(rte_rawdevs[i].driver_name, "raw_ntb",
1375                     NTB_DRV_NAME_LEN) == 0) && (rte_rawdevs[i].attached == 1))
1376                         break;
1377
1378         if (i == RTE_RAWDEV_MAX_DEVS)
1379                 rte_exit(EXIT_FAILURE, "Cannot find any ntb device.\n");
1380
1381         dev_id = i;
1382
1383         argc -= ret;
1384         argv += ret;
1385
1386         ntb_parse_args(argc, argv);
1387
1388         rte_rawdev_set_attr(dev_id, NTB_QUEUE_SZ_NAME, nb_desc);
1389         printf("Set queue size as %u.\n", nb_desc);
1390         rte_rawdev_set_attr(dev_id, NTB_QUEUE_NUM_NAME, num_queues);
1391         printf("Set queue number as %u.\n", num_queues);
1392         ntb_rawdev_info.dev_private = (rte_rawdev_obj_t)(&ntb_info);
1393         rte_rawdev_info_get(dev_id, &ntb_rawdev_info, sizeof(ntb_info));
1394
1395         nb_mbuf = nb_desc * num_queues * 2 * 2 + rte_lcore_count() *
1396                   MEMPOOL_CACHE_SIZE;
1397         mbuf_pool = ntb_mbuf_pool_create(ntb_buf_size, nb_mbuf, ntb_info,
1398                                          &ntb_conf, rte_socket_id());
1399         if (mbuf_pool == NULL)
1400                 rte_exit(EXIT_FAILURE, "Cannot create mbuf pool.\n");
1401
1402         ntb_conf.num_queues = num_queues;
1403         ntb_conf.queue_size = nb_desc;
1404         ntb_rawdev_conf.dev_private = (rte_rawdev_obj_t)(&ntb_conf);
1405         ret = rte_rawdev_configure(dev_id, &ntb_rawdev_conf, sizeof(ntb_conf));
1406         if (ret)
1407                 rte_exit(EXIT_FAILURE, "Can't config ntb dev: err=%d, "
1408                         "port=%u\n", ret, dev_id);
1409
1410         ntb_q_conf.tx_free_thresh = tx_free_thresh;
1411         ntb_q_conf.nb_desc = nb_desc;
1412         ntb_q_conf.rx_mp = mbuf_pool;
1413         for (i = 0; i < num_queues; i++) {
1414                 /* Setup rawdev queue */
1415                 ret = rte_rawdev_queue_setup(dev_id, i, &ntb_q_conf,
1416                                 sizeof(ntb_q_conf));
1417                 if (ret < 0)
1418                         rte_exit(EXIT_FAILURE,
1419                                 "Failed to setup ntb queue %u.\n", i);
1420         }
1421
1422         /* Waiting for peer dev up at most 100s.*/
1423         printf("Checking ntb link status...\n");
1424         for (i = 0; i < 1000; i++) {
1425                 rte_rawdev_get_attr(dev_id, NTB_LINK_STATUS_NAME,
1426                                     &ntb_link_status);
1427                 if (ntb_link_status) {
1428                         printf("Peer dev ready, ntb link up.\n");
1429                         break;
1430                 }
1431                 rte_delay_ms(100);
1432         }
1433         rte_rawdev_get_attr(dev_id, NTB_LINK_STATUS_NAME, &ntb_link_status);
1434         if (ntb_link_status == 0)
1435                 printf("Expire 100s. Link is not up. Please restart app.\n");
1436
1437         ret = rte_rawdev_start(dev_id);
1438         if (ret < 0)
1439                 rte_exit(EXIT_FAILURE, "rte_rawdev_start: err=%d, port=%u\n",
1440                         ret, dev_id);
1441
1442         /* Find 1st ethdev */
1443         eth_port_id = rte_eth_find_next(0);
1444
1445         if (eth_port_id < RTE_MAX_ETHPORTS) {
1446                 rte_eth_dev_info_get(eth_port_id, &ethdev_info);
1447                 eth_pconf.rx_adv_conf.rss_conf.rss_hf &=
1448                                 ethdev_info.flow_type_rss_offloads;
1449                 ret = rte_eth_dev_configure(eth_port_id, num_queues,
1450                                             num_queues, &eth_pconf);
1451                 if (ret)
1452                         rte_exit(EXIT_FAILURE, "Can't config ethdev: err=%d, "
1453                                 "port=%u\n", ret, eth_port_id);
1454                 eth_rx_conf = ethdev_info.default_rxconf;
1455                 eth_rx_conf.offloads = eth_pconf.rxmode.offloads;
1456                 eth_tx_conf = ethdev_info.default_txconf;
1457                 eth_tx_conf.offloads = eth_pconf.txmode.offloads;
1458
1459                 /* Setup ethdev queue if ethdev exists */
1460                 for (i = 0; i < num_queues; i++) {
1461                         ret = rte_eth_rx_queue_setup(eth_port_id, i, nb_desc,
1462                                         rte_eth_dev_socket_id(eth_port_id),
1463                                         &eth_rx_conf, mbuf_pool);
1464                         if (ret < 0)
1465                                 rte_exit(EXIT_FAILURE,
1466                                         "Failed to setup eth rxq %u.\n", i);
1467                         ret = rte_eth_tx_queue_setup(eth_port_id, i, nb_desc,
1468                                         rte_eth_dev_socket_id(eth_port_id),
1469                                         &eth_tx_conf);
1470                         if (ret < 0)
1471                                 rte_exit(EXIT_FAILURE,
1472                                         "Failed to setup eth txq %u.\n", i);
1473                 }
1474
1475                 ret = rte_eth_dev_start(eth_port_id);
1476                 if (ret < 0)
1477                         rte_exit(EXIT_FAILURE, "rte_eth_dev_start: err=%d, "
1478                                 "port=%u\n", ret, eth_port_id);
1479         }
1480
1481         /* initialize port stats */
1482         memset(&ntb_port_stats, 0, sizeof(ntb_port_stats));
1483
1484         /* Set default fwd mode if user doesn't set it. */
1485         if (fwd_mode == MAX_FWD_MODE && eth_port_id < RTE_MAX_ETHPORTS) {
1486                 printf("Set default fwd mode as iofwd.\n");
1487                 fwd_mode = IOFWD;
1488         }
1489         if (fwd_mode == MAX_FWD_MODE) {
1490                 printf("Set default fwd mode as file-trans.\n");
1491                 fwd_mode = FILE_TRANS;
1492         }
1493
1494         if (interactive) {
1495                 sleep(1);
1496                 prompt();
1497         } else {
1498                 start_pkt_fwd();
1499         }
1500
1501         /* clean up the EAL */
1502         rte_eal_cleanup();
1503
1504         return 0;
1505 }