examples/ntb/ntb_fwd.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(c) 2019 Intel Corporation
   3  */
   4 #include <stdint.h>
   5 #include <stdio.h>
   6 #include <inttypes.h>
   7 #include <unistd.h>
   8 #include <signal.h>
   9 #include <string.h>
  10 #include <getopt.h>
  11
  12 #include <cmdline_parse_string.h>
  13 #include <cmdline_socket.h>
  14 #include <cmdline.h>
  15 #include <rte_common.h>
  16 #include <rte_rawdev.h>
  17 #include <rte_ethdev.h>
  18 #include <rte_malloc.h>
  19 #include <rte_lcore.h>
  20 #include <rte_cycles.h>
  21 #include <rte_pmd_ntb.h>
  22
  23 /* Per-port statistics struct */
  24 struct ntb_port_statistics {
  25         uint64_t tx;
  26         uint64_t rx;
  27 } __rte_cache_aligned;
  28 /* Port 0: NTB dev, Port 1: ethdev when iofwd. */
  29 struct ntb_port_statistics ntb_port_stats[2];
  30
  31 struct ntb_fwd_stream {
  32         uint16_t tx_port;
  33         uint16_t rx_port;
  34         uint16_t qp_id;
  35         uint8_t tx_ntb;  /* If ntb device is tx port. */
  36 };
  37
  38 struct ntb_fwd_lcore_conf {
  39         uint16_t stream_id;
  40         uint16_t nb_stream;
  41         uint8_t stopped;
  42 };
  43
  44 enum ntb_fwd_mode {
  45         FILE_TRANS = 0,
  46         RXONLY,
  47         TXONLY,
  48         IOFWD,
  49         MAX_FWD_MODE,
  50 };
  51 static const char *const fwd_mode_s[] = {
  52         "file-trans",
  53         "rxonly",
  54         "txonly",
  55         "iofwd",
  56         NULL,
  57 };
  58 static enum ntb_fwd_mode fwd_mode = MAX_FWD_MODE;
  59
  60 static struct ntb_fwd_lcore_conf fwd_lcore_conf[RTE_MAX_LCORE];
  61 static struct ntb_fwd_stream *fwd_streams;
  62
  63 static struct rte_mempool *mbuf_pool;
  64
  65 #define NTB_DRV_NAME_LEN 7
  66 #define MEMPOOL_CACHE_SIZE 256
  67
  68 static uint8_t in_test;
  69 static uint8_t interactive = 1;
  70 static uint16_t eth_port_id = RTE_MAX_ETHPORTS;
  71 static uint16_t dev_id;
  72
  73 /* Number of queues, default set as 1 */
  74 static uint16_t num_queues = 1;
  75 static uint16_t ntb_buf_size = RTE_MBUF_DEFAULT_BUF_SIZE;
  76
  77 /* Configurable number of descriptors */
  78 #define NTB_DEFAULT_NUM_DESCS 1024
  79 static uint16_t nb_desc = NTB_DEFAULT_NUM_DESCS;
  80
  81 static uint16_t tx_free_thresh;
  82
  83 #define NTB_MAX_PKT_BURST 32
  84 #define NTB_DFLT_PKT_BURST 32
  85 static uint16_t pkt_burst = NTB_DFLT_PKT_BURST;
  86
  87 #define BURST_TX_RETRIES 64
  88
  89 static struct rte_eth_conf eth_port_conf = {
  90         .rxmode = {
  91                 .mq_mode = ETH_MQ_RX_RSS,
  92                 .split_hdr_size = 0,
  93         },
  94         .rx_adv_conf = {
  95                 .rss_conf = {
  96                         .rss_key = NULL,
  97                         .rss_hf = ETH_RSS_IP,
  98                 },
  99         },
 100         .txmode = {
 101                 .mq_mode = ETH_MQ_TX_NONE,
 102         },
 103 };
 104
 105 /* *** Help command with introduction. *** */
 106 struct cmd_help_result {
 107         cmdline_fixed_string_t help;
 108 };
 109
 110 static void
 111 cmd_help_parsed(__attribute__((unused)) void *parsed_result,
 112                 struct cmdline *cl,
 113                 __attribute__((unused)) void *data)
 114 {
 115         cmdline_printf(
 116                 cl,
 117                 "\n"
 118                 "The following commands are currently available:\n\n"
 119                 "Control:\n"
 120                 "    quit                                      :"
 121                 " Quit the application.\n"
 122                 "\nTransmission:\n"
 123                 "    send [path]                               :"
 124                 " Send [path] file. Only take effect in file-trans mode\n"
 125                 "    start                                     :"
 126                 " Start transmissions.\n"
 127                 "    stop                                      :"
 128                 " Stop transmissions.\n"
 129                 "    clear/show port stats                     :"
 130                 " Clear/show port stats.\n"
 131                 "    set fwd file-trans/rxonly/txonly/iofwd    :"
 132                 " Set packet forwarding mode.\n"
 133         );
 134
 135 }
 136
 137 cmdline_parse_token_string_t cmd_help_help =
 138         TOKEN_STRING_INITIALIZER(struct cmd_help_result, help, "help");
 139
 140 cmdline_parse_inst_t cmd_help = {
 141         .f = cmd_help_parsed,
 142         .data = NULL,
 143         .help_str = "show help",
 144         .tokens = {
 145                 (void *)&cmd_help_help,
 146                 NULL,
 147         },
 148 };
 149
 150 /* *** QUIT *** */
 151 struct cmd_quit_result {
 152         cmdline_fixed_string_t quit;
 153 };
 154
 155 static void
 156 cmd_quit_parsed(__attribute__((unused)) void *parsed_result,
 157                 struct cmdline *cl,
 158                 __attribute__((unused)) void *data)
 159 {
 160         struct ntb_fwd_lcore_conf *conf;
 161         uint32_t lcore_id;
 162
 163         /* Stop transmission first. */
 164         RTE_LCORE_FOREACH_SLAVE(lcore_id) {
 165                 conf = &fwd_lcore_conf[lcore_id];
 166
 167                 if (!conf->nb_stream)
 168                         continue;
 169
 170                 if (conf->stopped)
 171                         continue;
 172
 173                 conf->stopped = 1;
 174         }
 175         printf("\nWaiting for lcores to finish...\n");
 176         rte_eal_mp_wait_lcore();
 177         in_test = 0;
 178
 179         /* Stop traffic and Close port. */
 180         rte_rawdev_stop(dev_id);
 181         rte_rawdev_close(dev_id);
 182         if (eth_port_id < RTE_MAX_ETHPORTS && fwd_mode == IOFWD) {
 183                 rte_eth_dev_stop(eth_port_id);
 184                 rte_eth_dev_close(eth_port_id);
 185         }
 186
 187         cmdline_quit(cl);
 188 }
 189
 190 cmdline_parse_token_string_t cmd_quit_quit =
 191                 TOKEN_STRING_INITIALIZER(struct cmd_quit_result, quit, "quit");
 192
 193 cmdline_parse_inst_t cmd_quit = {
 194         .f = cmd_quit_parsed,
 195         .data = NULL,
 196         .help_str = "exit application",
 197         .tokens = {
 198                 (void *)&cmd_quit_quit,
 199                 NULL,
 200         },
 201 };
 202
 203 /* *** SEND FILE PARAMETERS *** */
 204 struct cmd_sendfile_result {
 205         cmdline_fixed_string_t send_string;
 206         char filepath[];
 207 };
 208
 209 static void
 210 cmd_sendfile_parsed(void *parsed_result,
 211                     __attribute__((unused)) struct cmdline *cl,
 212                     __attribute__((unused)) void *data)
 213 {
 214         struct cmd_sendfile_result *res = parsed_result;
 215         struct rte_rawdev_buf *pkts_send[NTB_MAX_PKT_BURST];
 216         struct rte_mbuf *mbuf_send[NTB_MAX_PKT_BURST];
 217         uint64_t size, count, i, j, nb_burst;
 218         uint16_t nb_tx, buf_size;
 219         unsigned int nb_pkt;
 220         size_t queue_id = 0;
 221         uint16_t retry = 0;
 222         uint32_t val;
 223         FILE *file;
 224         int ret;
 225
 226         if (num_queues != 1) {
 227                 printf("File transmission only supports 1 queue.\n");
 228                 num_queues = 1;
 229         }
 230
 231         file = fopen(res->filepath, "r");
 232         if (file == NULL) {
 233                 printf("Fail to open the file.\n");
 234                 return;
 235         }
 236
 237         if (fseek(file, 0, SEEK_END) < 0) {
 238                 printf("Fail to get file size.\n");
 239                 fclose(file);
 240                 return;
 241         }
 242         size = ftell(file);
 243         if (fseek(file, 0, SEEK_SET) < 0) {
 244                 printf("Fail to get file size.\n");
 245                 fclose(file);
 246                 return;
 247         }
 248
 249         /* Tell remote about the file size. */
 250         val = size >> 32;
 251         rte_rawdev_set_attr(dev_id, "spad_user_0", val);
 252         val = size;
 253         rte_rawdev_set_attr(dev_id, "spad_user_1", val);
 254         printf("Sending file, size is %"PRIu64"\n", size);
 255
 256         for (i = 0; i < NTB_MAX_PKT_BURST; i++)
 257                 pkts_send[i] = (struct rte_rawdev_buf *)
 258                                 malloc(sizeof(struct rte_rawdev_buf));
 259
 260         buf_size = ntb_buf_size - RTE_PKTMBUF_HEADROOM;
 261         count = (size + buf_size - 1) / buf_size;
 262         nb_burst = (count + pkt_burst - 1) / pkt_burst;
 263
 264         for (i = 0; i < nb_burst; i++) {
 265                 val = RTE_MIN(count, pkt_burst);
 266                 if (rte_mempool_get_bulk(mbuf_pool, (void **)mbuf_send,
 267                                         val) == 0) {
 268                         for (nb_pkt = 0; nb_pkt < val; nb_pkt++) {
 269                                 mbuf_send[nb_pkt]->port = dev_id;
 270                                 mbuf_send[nb_pkt]->data_len =
 271                                 fread(rte_pktmbuf_mtod(mbuf_send[nb_pkt],
 272                                         void *), 1, buf_size, file);
 273                                 mbuf_send[nb_pkt]->pkt_len =
 274                                         mbuf_send[nb_pkt]->data_len;
 275                                 pkts_send[nb_pkt]->buf_addr = mbuf_send[nb_pkt];
 276                         }
 277                 } else {
 278                         for (nb_pkt = 0; nb_pkt < val; nb_pkt++) {
 279                                 mbuf_send[nb_pkt] =
 280                                         rte_mbuf_raw_alloc(mbuf_pool);
 281                                 if (mbuf_send[nb_pkt] == NULL)
 282                                         break;
 283                                 mbuf_send[nb_pkt]->port = dev_id;
 284                                 mbuf_send[nb_pkt]->data_len =
 285                                 fread(rte_pktmbuf_mtod(mbuf_send[nb_pkt],
 286                                         void *), 1, buf_size, file);
 287                                 mbuf_send[nb_pkt]->pkt_len =
 288                                         mbuf_send[nb_pkt]->data_len;
 289                                 pkts_send[nb_pkt]->buf_addr = mbuf_send[nb_pkt];
 290                         }
 291                 }
 292
 293                 ret = rte_rawdev_enqueue_buffers(dev_id, pkts_send, nb_pkt,
 294                                                 (void *)queue_id);
 295                 if (ret < 0) {
 296                         printf("Enqueue failed with err %d\n", ret);
 297                         for (j = 0; j < nb_pkt; j++)
 298                                 rte_pktmbuf_free(mbuf_send[j]);
 299                         goto clean;
 300                 }
 301                 nb_tx = ret;
 302                 while (nb_tx != nb_pkt && retry < BURST_TX_RETRIES) {
 303                         rte_delay_us(1);
 304                         ret = rte_rawdev_enqueue_buffers(dev_id,
 305                                         &pkts_send[nb_tx], nb_pkt - nb_tx,
 306                                         (void *)queue_id);
 307                         if (ret < 0) {
 308                                 printf("Enqueue failed with err %d\n", ret);
 309                                 for (j = nb_tx; j < nb_pkt; j++)
 310                                         rte_pktmbuf_free(mbuf_send[j]);
 311                                 goto clean;
 312                         }
 313                         nb_tx += ret;
 314                 }
 315                 count -= nb_pkt;
 316         }
 317
 318         /* Clear register after file sending done. */
 319         rte_rawdev_set_attr(dev_id, "spad_user_0", 0);
 320         rte_rawdev_set_attr(dev_id, "spad_user_1", 0);
 321         printf("Done sending file.\n");
 322
 323 clean:
 324         for (i = 0; i < NTB_MAX_PKT_BURST; i++)
 325                 free(pkts_send[i]);
 326         fclose(file);
 327 }
 328
 329 cmdline_parse_token_string_t cmd_send_file_send =
 330         TOKEN_STRING_INITIALIZER(struct cmd_sendfile_result, send_string,
 331                                  "send");
 332 cmdline_parse_token_string_t cmd_send_file_filepath =
 333         TOKEN_STRING_INITIALIZER(struct cmd_sendfile_result, filepath, NULL);
 334
 335
 336 cmdline_parse_inst_t cmd_send_file = {
 337         .f = cmd_sendfile_parsed,
 338         .data = NULL,
 339         .help_str = "send <file_path>",
 340         .tokens = {
 341                 (void *)&cmd_send_file_send,
 342                 (void *)&cmd_send_file_filepath,
 343                 NULL,
 344         },
 345 };
 346
 347 #define RECV_FILE_LEN 30
 348 static int
 349 start_polling_recv_file(void *param)
 350 {
 351         struct rte_rawdev_buf *pkts_recv[NTB_MAX_PKT_BURST];
 352         struct ntb_fwd_lcore_conf *conf = param;
 353         struct rte_mbuf *mbuf;
 354         char filepath[RECV_FILE_LEN];
 355         uint64_t val, size, file_len;
 356         uint16_t nb_rx, i, file_no;
 357         size_t queue_id = 0;
 358         FILE *file;
 359         int ret;
 360
 361         for (i = 0; i < NTB_MAX_PKT_BURST; i++)
 362                 pkts_recv[i] = (struct rte_rawdev_buf *)
 363                                 malloc(sizeof(struct rte_rawdev_buf));
 364
 365         file_no = 0;
 366         while (!conf->stopped) {
 367                 snprintf(filepath, RECV_FILE_LEN, "ntb_recv_file%d", file_no);
 368                 file = fopen(filepath, "w");
 369                 if (file == NULL) {
 370                         printf("Fail to open the file.\n");
 371                         return -EINVAL;
 372                 }
 373
 374                 rte_rawdev_get_attr(dev_id, "spad_user_0", &val);
 375                 size = val << 32;
 376                 rte_rawdev_get_attr(dev_id, "spad_user_1", &val);
 377                 size |= val;
 378
 379                 if (!size) {
 380                         fclose(file);
 381                         continue;
 382                 }
 383
 384                 file_len = 0;
 385                 nb_rx = NTB_MAX_PKT_BURST;
 386                 while (file_len < size && !conf->stopped) {
 387                         ret = rte_rawdev_dequeue_buffers(dev_id, pkts_recv,
 388                                         pkt_burst, (void *)queue_id);
 389                         if (ret < 0) {
 390                                 printf("Dequeue failed with err %d\n", ret);
 391                                 fclose(file);
 392                                 goto clean;
 393                         }
 394                         nb_rx = ret;
 395                         ntb_port_stats[0].rx += nb_rx;
 396                         for (i = 0; i < nb_rx; i++) {
 397                                 mbuf = pkts_recv[i]->buf_addr;
 398                                 fwrite(rte_pktmbuf_mtod(mbuf, void *), 1,
 399                                         mbuf->data_len, file);
 400                                 file_len += mbuf->data_len;
 401                                 rte_pktmbuf_free(mbuf);
 402                                 pkts_recv[i]->buf_addr = NULL;
 403                         }
 404                 }
 405
 406                 printf("Received file (size: %" PRIu64 ") from peer to %s.\n",
 407                         size, filepath);
 408                 fclose(file);
 409                 file_no++;
 410         }
 411
 412 clean:
 413         for (i = 0; i < NTB_MAX_PKT_BURST; i++)
 414                 free(pkts_recv[i]);
 415         return 0;
 416 }
 417
 418 static int
 419 start_iofwd_per_lcore(void *param)
 420 {
 421         struct rte_rawdev_buf *ntb_buf[NTB_MAX_PKT_BURST];
 422         struct rte_mbuf *pkts_burst[NTB_MAX_PKT_BURST];
 423         struct ntb_fwd_lcore_conf *conf = param;
 424         struct ntb_fwd_stream fs;
 425         uint16_t nb_rx, nb_tx;
 426         int i, j, ret;
 427
 428         for (i = 0; i < NTB_MAX_PKT_BURST; i++)
 429                 ntb_buf[i] = (struct rte_rawdev_buf *)
 430                              malloc(sizeof(struct rte_rawdev_buf));
 431
 432         while (!conf->stopped) {
 433                 for (i = 0; i < conf->nb_stream; i++) {
 434                         fs = fwd_streams[conf->stream_id + i];
 435                         if (fs.tx_ntb) {
 436                                 nb_rx = rte_eth_rx_burst(fs.rx_port,
 437                                                 fs.qp_id, pkts_burst,
 438                                                 pkt_burst);
 439                                 if (unlikely(nb_rx == 0))
 440                                         continue;
 441                                 for (j = 0; j < nb_rx; j++)
 442                                         ntb_buf[j]->buf_addr = pkts_burst[j];
 443                                 ret = rte_rawdev_enqueue_buffers(fs.tx_port,
 444                                                 ntb_buf, nb_rx,
 445                                                 (void *)(size_t)fs.qp_id);
 446                                 if (ret < 0) {
 447                                         printf("Enqueue failed with err %d\n",
 448                                                 ret);
 449                                         for (j = 0; j < nb_rx; j++)
 450                                                 rte_pktmbuf_free(pkts_burst[j]);
 451                                         goto clean;
 452                                 }
 453                                 nb_tx = ret;
 454                                 ntb_port_stats[0].tx += nb_tx;
 455                                 ntb_port_stats[1].rx += nb_rx;
 456                         } else {
 457                                 ret = rte_rawdev_dequeue_buffers(fs.rx_port,
 458                                                 ntb_buf, pkt_burst,
 459                                                 (void *)(size_t)fs.qp_id);
 460                                 if (ret < 0) {
 461                                         printf("Dequeue failed with err %d\n",
 462                                                 ret);
 463                                         goto clean;
 464                                 }
 465                                 nb_rx = ret;
 466                                 if (unlikely(nb_rx == 0))
 467                                         continue;
 468                                 for (j = 0; j < nb_rx; j++)
 469                                         pkts_burst[j] = ntb_buf[j]->buf_addr;
 470                                 nb_tx = rte_eth_tx_burst(fs.tx_port,
 471                                         fs.qp_id, pkts_burst, nb_rx);
 472                                 ntb_port_stats[1].tx += nb_tx;
 473                                 ntb_port_stats[0].rx += nb_rx;
 474                         }
 475                         if (unlikely(nb_tx < nb_rx)) {
 476                                 do {
 477                                         rte_pktmbuf_free(pkts_burst[nb_tx]);
 478                                 } while (++nb_tx < nb_rx);
 479                         }
 480                 }
 481         }
 482
 483 clean:
 484         for (i = 0; i < NTB_MAX_PKT_BURST; i++)
 485                 free(ntb_buf[i]);
 486
 487         return 0;
 488 }
 489
 490 static int
 491 start_rxonly_per_lcore(void *param)
 492 {
 493         struct rte_rawdev_buf *ntb_buf[NTB_MAX_PKT_BURST];
 494         struct ntb_fwd_lcore_conf *conf = param;
 495         struct ntb_fwd_stream fs;
 496         uint16_t nb_rx;
 497         int i, j, ret;
 498
 499         for (i = 0; i < NTB_MAX_PKT_BURST; i++)
 500                 ntb_buf[i] = (struct rte_rawdev_buf *)
 501                              malloc(sizeof(struct rte_rawdev_buf));
 502
 503         while (!conf->stopped) {
 504                 for (i = 0; i < conf->nb_stream; i++) {
 505                         fs = fwd_streams[conf->stream_id + i];
 506                         ret = rte_rawdev_dequeue_buffers(fs.rx_port,
 507                               ntb_buf, pkt_burst, (void *)(size_t)fs.qp_id);
 508                         if (ret < 0) {
 509                                 printf("Dequeue failed with err %d\n", ret);
 510                                 goto clean;
 511                         }
 512                         nb_rx = ret;
 513                         if (unlikely(nb_rx == 0))
 514                                 continue;
 515                         ntb_port_stats[0].rx += nb_rx;
 516
 517                         for (j = 0; j < nb_rx; j++)
 518                                 rte_pktmbuf_free(ntb_buf[j]->buf_addr);
 519                 }
 520         }
 521
 522 clean:
 523         for (i = 0; i < NTB_MAX_PKT_BURST; i++)
 524                 free(ntb_buf[i]);
 525
 526         return 0;
 527 }
 528
 529
 530 static int
 531 start_txonly_per_lcore(void *param)
 532 {
 533         struct rte_rawdev_buf *ntb_buf[NTB_MAX_PKT_BURST];
 534         struct rte_mbuf *pkts_burst[NTB_MAX_PKT_BURST];
 535         struct ntb_fwd_lcore_conf *conf = param;
 536         struct ntb_fwd_stream fs;
 537         uint16_t nb_pkt, nb_tx;
 538         int i, j, ret;
 539
 540         for (i = 0; i < NTB_MAX_PKT_BURST; i++)
 541                 ntb_buf[i] = (struct rte_rawdev_buf *)
 542                              malloc(sizeof(struct rte_rawdev_buf));
 543
 544         while (!conf->stopped) {
 545                 for (i = 0; i < conf->nb_stream; i++) {
 546                         fs = fwd_streams[conf->stream_id + i];
 547                         if (rte_mempool_get_bulk(mbuf_pool, (void **)pkts_burst,
 548                                   pkt_burst) == 0) {
 549                                 for (nb_pkt = 0; nb_pkt < pkt_burst; nb_pkt++) {
 550                                         pkts_burst[nb_pkt]->port = dev_id;
 551                                         pkts_burst[nb_pkt]->data_len =
 552                                                 pkts_burst[nb_pkt]->buf_len -
 553                                                 RTE_PKTMBUF_HEADROOM;
 554                                         pkts_burst[nb_pkt]->pkt_len =
 555                                                 pkts_burst[nb_pkt]->data_len;
 556                                         ntb_buf[nb_pkt]->buf_addr =
 557                                                 pkts_burst[nb_pkt];
 558                                 }
 559                         } else {
 560                                 for (nb_pkt = 0; nb_pkt < pkt_burst; nb_pkt++) {
 561                                         pkts_burst[nb_pkt] =
 562                                                 rte_pktmbuf_alloc(mbuf_pool);
 563                                         if (pkts_burst[nb_pkt] == NULL)
 564                                                 break;
 565                                         pkts_burst[nb_pkt]->port = dev_id;
 566                                         pkts_burst[nb_pkt]->data_len =
 567                                                 pkts_burst[nb_pkt]->buf_len -
 568                                                 RTE_PKTMBUF_HEADROOM;
 569                                         pkts_burst[nb_pkt]->pkt_len =
 570                                                 pkts_burst[nb_pkt]->data_len;
 571                                         ntb_buf[nb_pkt]->buf_addr =
 572                                                 pkts_burst[nb_pkt];
 573                                 }
 574                         }
 575                         ret = rte_rawdev_enqueue_buffers(fs.tx_port, ntb_buf,
 576                                         nb_pkt, (void *)(size_t)fs.qp_id);
 577                         if (ret < 0) {
 578                                 printf("Enqueue failed with err %d\n", ret);
 579                                 for (j = 0; j < nb_pkt; j++)
 580                                         rte_pktmbuf_free(pkts_burst[j]);
 581                                 goto clean;
 582                         }
 583                         nb_tx = ret;
 584                         ntb_port_stats[0].tx += nb_tx;
 585                         if (unlikely(nb_tx < nb_pkt)) {
 586                                 do {
 587                                         rte_pktmbuf_free(pkts_burst[nb_tx]);
 588                                 } while (++nb_tx < nb_pkt);
 589                         }
 590                 }
 591         }
 592
 593 clean:
 594         for (i = 0; i < NTB_MAX_PKT_BURST; i++)
 595                 free(ntb_buf[i]);
 596
 597         return 0;
 598 }
 599
 600 static int
 601 ntb_fwd_config_setup(void)
 602 {
 603         uint16_t i;
 604
 605         /* Make sure iofwd has valid ethdev. */
 606         if (fwd_mode == IOFWD && eth_port_id >= RTE_MAX_ETHPORTS) {
 607                 printf("No ethdev, cannot be in iofwd mode.");
 608                 return -EINVAL;
 609         }
 610
 611         if (fwd_mode == IOFWD) {
 612                 fwd_streams = rte_zmalloc("ntb_fwd: fwd_streams",
 613                         sizeof(struct ntb_fwd_stream) * num_queues * 2,
 614                         RTE_CACHE_LINE_SIZE);
 615                 for (i = 0; i < num_queues; i++) {
 616                         fwd_streams[i * 2].qp_id = i;
 617                         fwd_streams[i * 2].tx_port = dev_id;
 618                         fwd_streams[i * 2].rx_port = eth_port_id;
 619                         fwd_streams[i * 2].tx_ntb = 1;
 620
 621                         fwd_streams[i * 2 + 1].qp_id = i;
 622                         fwd_streams[i * 2 + 1].tx_port = eth_port_id;
 623                         fwd_streams[i * 2 + 1].rx_port = dev_id;
 624                         fwd_streams[i * 2 + 1].tx_ntb = 0;
 625                 }
 626                 return 0;
 627         }
 628
 629         if (fwd_mode == RXONLY || fwd_mode == FILE_TRANS) {
 630                 /* Only support 1 queue in file-trans for in order. */
 631                 if (fwd_mode == FILE_TRANS)
 632                         num_queues = 1;
 633
 634                 fwd_streams = rte_zmalloc("ntb_fwd: fwd_streams",
 635                                 sizeof(struct ntb_fwd_stream) * num_queues,
 636                                 RTE_CACHE_LINE_SIZE);
 637                 for (i = 0; i < num_queues; i++) {
 638                         fwd_streams[i].qp_id = i;
 639                         fwd_streams[i].tx_port = RTE_MAX_ETHPORTS;
 640                         fwd_streams[i].rx_port = dev_id;
 641                         fwd_streams[i].tx_ntb = 0;
 642                 }
 643                 return 0;
 644         }
 645
 646         if (fwd_mode == TXONLY) {
 647                 fwd_streams = rte_zmalloc("ntb_fwd: fwd_streams",
 648                                 sizeof(struct ntb_fwd_stream) * num_queues,
 649                                 RTE_CACHE_LINE_SIZE);
 650                 for (i = 0; i < num_queues; i++) {
 651                         fwd_streams[i].qp_id = i;
 652                         fwd_streams[i].tx_port = dev_id;
 653                         fwd_streams[i].rx_port = RTE_MAX_ETHPORTS;
 654                         fwd_streams[i].tx_ntb = 1;
 655                 }
 656         }
 657         return 0;
 658 }
 659
 660 static void
 661 assign_stream_to_lcores(void)
 662 {
 663         struct ntb_fwd_lcore_conf *conf;
 664         struct ntb_fwd_stream *fs;
 665         uint16_t nb_streams, sm_per_lcore, sm_id, i;
 666         uint32_t lcore_id;
 667         uint8_t lcore_num, nb_extra;
 668
 669         lcore_num = rte_lcore_count();
 670         /* Exclude master core */
 671         lcore_num--;
 672
 673         nb_streams = (fwd_mode == IOFWD) ? num_queues * 2 : num_queues;
 674
 675         sm_per_lcore = nb_streams / lcore_num;
 676         nb_extra = nb_streams % lcore_num;
 677         sm_id = 0;
 678         i = 0;
 679
 680         RTE_LCORE_FOREACH_SLAVE(lcore_id) {
 681                 conf = &fwd_lcore_conf[lcore_id];
 682
 683                 if (i < nb_extra) {
 684                         conf->nb_stream = sm_per_lcore + 1;
 685                         conf->stream_id = sm_id;
 686                         sm_id = sm_id + sm_per_lcore + 1;
 687                 } else {
 688                         conf->nb_stream = sm_per_lcore;
 689                         conf->stream_id = sm_id;
 690                         sm_id = sm_id + sm_per_lcore;
 691                 }
 692
 693                 i++;
 694                 if (sm_id >= nb_streams)
 695                         break;
 696         }
 697
 698         /* Print packet forwading config. */
 699         RTE_LCORE_FOREACH_SLAVE(lcore_id) {
 700                 conf = &fwd_lcore_conf[lcore_id];
 701
 702                 if (!conf->nb_stream)
 703                         continue;
 704
 705                 printf("Streams on Lcore %u :\n", lcore_id);
 706                 for (i = 0; i < conf->nb_stream; i++) {
 707                         fs = &fwd_streams[conf->stream_id + i];
 708                         if (fwd_mode == IOFWD)
 709                                 printf(" + Stream %u : %s%u RX -> %s%u TX,"
 710                                         " Q=%u\n", conf->stream_id + i,
 711                                         fs->tx_ntb ? "Eth" : "NTB", fs->rx_port,
 712                                         fs->tx_ntb ? "NTB" : "Eth", fs->tx_port,
 713                                         fs->qp_id);
 714                         if (fwd_mode == FILE_TRANS || fwd_mode == RXONLY)
 715                                 printf(" + Stream %u : %s%u RX only\n",
 716                                         conf->stream_id, "NTB", fs->rx_port);
 717                         if (fwd_mode == TXONLY)
 718                                 printf(" + Stream %u : %s%u TX only\n",
 719                                         conf->stream_id, "NTB", fs->tx_port);
 720                 }
 721         }
 722 }
 723
 724 static void
 725 start_pkt_fwd(void)
 726 {
 727         struct ntb_fwd_lcore_conf *conf;
 728         struct rte_eth_link eth_link;
 729         uint32_t lcore_id;
 730         int ret, i;
 731
 732         ret = ntb_fwd_config_setup();
 733         if (ret < 0) {
 734                 printf("Cannot start traffic. Please reset fwd mode.\n");
 735                 return;
 736         }
 737
 738         /* If using iofwd, checking ethdev link status first. */
 739         if (fwd_mode == IOFWD) {
 740                 printf("Checking eth link status...\n");
 741                 /* Wait for eth link up at most 100 times. */
 742                 for (i = 0; i < 100; i++) {
 743                         ret = rte_eth_link_get(eth_port_id, &eth_link);
 744                         if (ret < 0) {
 745                                 printf("Link get failed with err %d\n", ret);
 746                                 return;
 747                         }
 748                         if (eth_link.link_status) {
 749                                 printf("Eth%u Link Up. Speed %u Mbps - %s\n",
 750                                         eth_port_id, eth_link.link_speed,
 751                                         (eth_link.link_duplex ==
 752                                          ETH_LINK_FULL_DUPLEX) ?
 753                                         ("full-duplex") : ("half-duplex"));
 754                                 break;
 755                         }
 756                 }
 757                 if (!eth_link.link_status) {
 758                         printf("Eth%u link down. Cannot start traffic.\n",
 759                                 eth_port_id);
 760                         return;
 761                 }
 762         }
 763
 764         assign_stream_to_lcores();
 765         in_test = 1;
 766
 767         RTE_LCORE_FOREACH_SLAVE(lcore_id) {
 768                 conf = &fwd_lcore_conf[lcore_id];
 769
 770                 if (!conf->nb_stream)
 771                         continue;
 772
 773                 conf->stopped = 0;
 774                 if (fwd_mode == FILE_TRANS)
 775                         rte_eal_remote_launch(start_polling_recv_file,
 776                                               conf, lcore_id);
 777                 else if (fwd_mode == IOFWD)
 778                         rte_eal_remote_launch(start_iofwd_per_lcore,
 779                                               conf, lcore_id);
 780                 else if (fwd_mode == RXONLY)
 781                         rte_eal_remote_launch(start_rxonly_per_lcore,
 782                                               conf, lcore_id);
 783                 else if (fwd_mode == TXONLY)
 784                         rte_eal_remote_launch(start_txonly_per_lcore,
 785                                               conf, lcore_id);
 786         }
 787 }
 788
 789 /* *** START FWD PARAMETERS *** */
 790 struct cmd_start_result {
 791         cmdline_fixed_string_t start;
 792 };
 793
 794 static void
 795 cmd_start_parsed(__attribute__((unused)) void *parsed_result,
 796                             __attribute__((unused)) struct cmdline *cl,
 797                             __attribute__((unused)) void *data)
 798 {
 799         start_pkt_fwd();
 800 }
 801
 802 cmdline_parse_token_string_t cmd_start_start =
 803                 TOKEN_STRING_INITIALIZER(struct cmd_start_result, start, "start");
 804
 805 cmdline_parse_inst_t cmd_start = {
 806         .f = cmd_start_parsed,
 807         .data = NULL,
 808         .help_str = "start pkt fwd between ntb and ethdev",
 809         .tokens = {
 810                 (void *)&cmd_start_start,
 811                 NULL,
 812         },
 813 };
 814
 815 /* *** STOP *** */
 816 struct cmd_stop_result {
 817         cmdline_fixed_string_t stop;
 818 };
 819
 820 static void
 821 cmd_stop_parsed(__attribute__((unused)) void *parsed_result,
 822                 __attribute__((unused)) struct cmdline *cl,
 823                 __attribute__((unused)) void *data)
 824 {
 825         struct ntb_fwd_lcore_conf *conf;
 826         uint32_t lcore_id;
 827
 828         RTE_LCORE_FOREACH_SLAVE(lcore_id) {
 829                 conf = &fwd_lcore_conf[lcore_id];
 830
 831                 if (!conf->nb_stream)
 832                         continue;
 833
 834                 if (conf->stopped)
 835                         continue;
 836
 837                 conf->stopped = 1;
 838         }
 839         printf("\nWaiting for lcores to finish...\n");
 840         rte_eal_mp_wait_lcore();
 841         in_test = 0;
 842         printf("\nDone.\n");
 843 }
 844
 845 cmdline_parse_token_string_t cmd_stop_stop =
 846                 TOKEN_STRING_INITIALIZER(struct cmd_stop_result, stop, "stop");
 847
 848 cmdline_parse_inst_t cmd_stop = {
 849         .f = cmd_stop_parsed,
 850         .data = NULL,
 851         .help_str = "stop: Stop packet forwarding",
 852         .tokens = {
 853                 (void *)&cmd_stop_stop,
 854                 NULL,
 855         },
 856 };
 857
 858 static void
 859 ntb_stats_clear(void)
 860 {
 861         int nb_ids, i;
 862         uint32_t *ids;
 863
 864         /* Clear NTB dev stats */
 865         nb_ids = rte_rawdev_xstats_names_get(dev_id, NULL, 0);
 866         if (nb_ids  < 0) {
 867                 printf("Error: Cannot get count of xstats\n");
 868                 return;
 869         }
 870         ids = malloc(sizeof(uint32_t) * nb_ids);
 871         for (i = 0; i < nb_ids; i++)
 872                 ids[i] = i;
 873         rte_rawdev_xstats_reset(dev_id, ids, nb_ids);
 874         printf("\n  statistics for NTB port %d cleared\n", dev_id);
 875
 876         /* Clear Ethdev stats if have any */
 877         if (fwd_mode == IOFWD && eth_port_id != RTE_MAX_ETHPORTS) {
 878                 rte_eth_stats_reset(eth_port_id);
 879                 printf("\n  statistics for ETH port %d cleared\n", eth_port_id);
 880         }
 881 }
 882
 883 static inline void
 884 ntb_calculate_throughput(uint16_t port) {
 885         uint64_t diff_pkts_rx, diff_pkts_tx, diff_cycles;
 886         uint64_t mpps_rx, mpps_tx;
 887         static uint64_t prev_pkts_rx[2];
 888         static uint64_t prev_pkts_tx[2];
 889         static uint64_t prev_cycles[2];
 890
 891         diff_cycles = prev_cycles[port];
 892         prev_cycles[port] = rte_rdtsc();
 893         if (diff_cycles > 0)
 894                 diff_cycles = prev_cycles[port] - diff_cycles;
 895         diff_pkts_rx = (ntb_port_stats[port].rx > prev_pkts_rx[port]) ?
 896                 (ntb_port_stats[port].rx - prev_pkts_rx[port]) : 0;
 897         diff_pkts_tx = (ntb_port_stats[port].tx > prev_pkts_tx[port]) ?
 898                 (ntb_port_stats[port].tx - prev_pkts_tx[port]) : 0;
 899         prev_pkts_rx[port] = ntb_port_stats[port].rx;
 900         prev_pkts_tx[port] = ntb_port_stats[port].tx;
 901         mpps_rx = diff_cycles > 0 ?
 902                 diff_pkts_rx * rte_get_tsc_hz() / diff_cycles : 0;
 903         mpps_tx = diff_cycles > 0 ?
 904                 diff_pkts_tx * rte_get_tsc_hz() / diff_cycles : 0;
 905         printf("  Throughput (since last show)\n");
 906         printf("  Rx-pps: %12"PRIu64"\n  Tx-pps: %12"PRIu64"\n",
 907                         mpps_rx, mpps_tx);
 908
 909 }
 910
 911 static void
 912 ntb_stats_display(void)
 913 {
 914         struct rte_rawdev_xstats_name *xstats_names;
 915         struct rte_eth_stats stats;
 916         uint64_t *values;
 917         uint32_t *ids;
 918         int nb_ids, i;
 919
 920         printf("###### statistics for NTB port %d #######\n", dev_id);
 921
 922         /* Get NTB dev stats and stats names */
 923         nb_ids = rte_rawdev_xstats_names_get(dev_id, NULL, 0);
 924         if (nb_ids  < 0) {
 925                 printf("Error: Cannot get count of xstats\n");
 926                 return;
 927         }
 928         xstats_names = malloc(sizeof(struct rte_rawdev_xstats_name) * nb_ids);
 929         if (xstats_names == NULL) {
 930                 printf("Cannot allocate memory for xstats lookup\n");
 931                 return;
 932         }
 933         if (nb_ids != rte_rawdev_xstats_names_get(
 934                         dev_id, xstats_names, nb_ids)) {
 935                 printf("Error: Cannot get xstats lookup\n");
 936                 free(xstats_names);
 937                 return;
 938         }
 939         ids = malloc(sizeof(uint32_t) * nb_ids);
 940         for (i = 0; i < nb_ids; i++)
 941                 ids[i] = i;
 942         values = malloc(sizeof(uint64_t) * nb_ids);
 943         if (nb_ids != rte_rawdev_xstats_get(dev_id, ids, values, nb_ids)) {
 944                 printf("Error: Unable to get xstats\n");
 945                 free(xstats_names);
 946                 free(values);
 947                 free(ids);
 948                 return;
 949         }
 950
 951         /* Display NTB dev stats */
 952         for (i = 0; i < nb_ids; i++)
 953                 printf("  %s: %"PRIu64"\n", xstats_names[i].name, values[i]);
 954         ntb_calculate_throughput(0);
 955
 956         /* Get Ethdev stats if have any */
 957         if (fwd_mode == IOFWD && eth_port_id != RTE_MAX_ETHPORTS) {
 958                 printf("###### statistics for ETH port %d ######\n",
 959                         eth_port_id);
 960                 rte_eth_stats_get(eth_port_id, &stats);
 961                 printf("  RX-packets: %"PRIu64"\n", stats.ipackets);
 962                 printf("  RX-bytes: %"PRIu64"\n", stats.ibytes);
 963                 printf("  RX-errors: %"PRIu64"\n", stats.ierrors);
 964                 printf("  RX-missed: %"PRIu64"\n", stats.imissed);
 965                 printf("  TX-packets: %"PRIu64"\n", stats.opackets);
 966                 printf("  TX-bytes: %"PRIu64"\n", stats.obytes);
 967                 printf("  TX-errors: %"PRIu64"\n", stats.oerrors);
 968                 ntb_calculate_throughput(1);
 969         }
 970
 971         free(xstats_names);
 972         free(values);
 973         free(ids);
 974 }
 975
 976 /* *** SHOW/CLEAR PORT STATS *** */
 977 struct cmd_stats_result {
 978         cmdline_fixed_string_t show;
 979         cmdline_fixed_string_t port;
 980         cmdline_fixed_string_t stats;
 981 };
 982
 983 static void
 984 cmd_stats_parsed(void *parsed_result,
 985                  __attribute__((unused)) struct cmdline *cl,
 986                  __attribute__((unused)) void *data)
 987 {
 988         struct cmd_stats_result *res = parsed_result;
 989         if (!strcmp(res->show, "clear"))
 990                 ntb_stats_clear();
 991         else
 992                 ntb_stats_display();
 993 }
 994
 995 cmdline_parse_token_string_t cmd_stats_show =
 996         TOKEN_STRING_INITIALIZER(struct cmd_stats_result, show, "show#clear");
 997 cmdline_parse_token_string_t cmd_stats_port =
 998         TOKEN_STRING_INITIALIZER(struct cmd_stats_result, port, "port");
 999 cmdline_parse_token_string_t cmd_stats_stats =
1000         TOKEN_STRING_INITIALIZER(struct cmd_stats_result, stats, "stats");
1001
1002
1003 cmdline_parse_inst_t cmd_stats = {
1004         .f = cmd_stats_parsed,
1005         .data = NULL,
1006         .help_str = "show|clear port stats",
1007         .tokens = {
1008                 (void *)&cmd_stats_show,
1009                 (void *)&cmd_stats_port,
1010                 (void *)&cmd_stats_stats,
1011                 NULL,
1012         },
1013 };
1014
1015 /* *** SET FORWARDING MODE *** */
1016 struct cmd_set_fwd_mode_result {
1017         cmdline_fixed_string_t set;
1018         cmdline_fixed_string_t fwd;
1019         cmdline_fixed_string_t mode;
1020 };
1021
1022 static void
1023 cmd_set_fwd_mode_parsed(__attribute__((unused)) void *parsed_result,
1024                         __attribute__((unused)) struct cmdline *cl,
1025                         __attribute__((unused)) void *data)
1026 {
1027         struct cmd_set_fwd_mode_result *res = parsed_result;
1028         int i;
1029
1030         if (in_test) {
1031                 printf("Please stop traffic first.\n");
1032                 return;
1033         }
1034
1035         for (i = 0; i < MAX_FWD_MODE; i++) {
1036                 if (!strcmp(res->mode, fwd_mode_s[i])) {
1037                         fwd_mode = i;
1038                         return;
1039                 }
1040         }
1041         printf("Invalid %s packet forwarding mode.\n", res->mode);
1042 }
1043
1044 cmdline_parse_token_string_t cmd_setfwd_set =
1045         TOKEN_STRING_INITIALIZER(struct cmd_set_fwd_mode_result, set, "set");
1046 cmdline_parse_token_string_t cmd_setfwd_fwd =
1047         TOKEN_STRING_INITIALIZER(struct cmd_set_fwd_mode_result, fwd, "fwd");
1048 cmdline_parse_token_string_t cmd_setfwd_mode =
1049         TOKEN_STRING_INITIALIZER(struct cmd_set_fwd_mode_result, mode,
1050                                 "file-trans#iofwd#txonly#rxonly");
1051
1052 cmdline_parse_inst_t cmd_set_fwd_mode = {
1053         .f = cmd_set_fwd_mode_parsed,
1054         .data = NULL,
1055         .help_str = "set forwarding mode as file-trans|rxonly|txonly|iofwd",
1056         .tokens = {
1057                 (void *)&cmd_setfwd_set,
1058                 (void *)&cmd_setfwd_fwd,
1059                 (void *)&cmd_setfwd_mode,
1060                 NULL,
1061         },
1062 };
1063
1064 /* list of instructions */
1065 cmdline_parse_ctx_t main_ctx[] = {
1066         (cmdline_parse_inst_t *)&cmd_help,
1067         (cmdline_parse_inst_t *)&cmd_send_file,
1068         (cmdline_parse_inst_t *)&cmd_start,
1069         (cmdline_parse_inst_t *)&cmd_stop,
1070         (cmdline_parse_inst_t *)&cmd_stats,
1071         (cmdline_parse_inst_t *)&cmd_set_fwd_mode,
1072         (cmdline_parse_inst_t *)&cmd_quit,
1073         NULL,
1074 };
1075
1076 /* prompt function, called from main on MASTER lcore */
1077 static void
1078 prompt(void)
1079 {
1080         struct cmdline *cl;
1081
1082         cl = cmdline_stdin_new(main_ctx, "ntb> ");
1083         if (cl == NULL)
1084                 return;
1085
1086         cmdline_interact(cl);
1087         cmdline_stdin_exit(cl);
1088 }
1089
1090 static void
1091 signal_handler(int signum)
1092 {
1093         if (signum == SIGINT || signum == SIGTERM) {
1094                 printf("\nSignal %d received, preparing to exit...\n", signum);
1095                 signal(signum, SIG_DFL);
1096                 kill(getpid(), signum);
1097         }
1098 }
1099
1100 #define OPT_BUF_SIZE         "buf-size"
1101 #define OPT_FWD_MODE         "fwd-mode"
1102 #define OPT_NB_DESC          "nb-desc"
1103 #define OPT_TXFREET          "txfreet"
1104 #define OPT_BURST            "burst"
1105 #define OPT_QP               "qp"
1106
1107 enum {
1108         /* long options mapped to a short option */
1109         OPT_NO_ZERO_COPY_NUM = 1,
1110         OPT_BUF_SIZE_NUM,
1111         OPT_FWD_MODE_NUM,
1112         OPT_NB_DESC_NUM,
1113         OPT_TXFREET_NUM,
1114         OPT_BURST_NUM,
1115         OPT_QP_NUM,
1116 };
1117
1118 static const char short_options[] =
1119         "i" /* interactive mode */
1120         ;
1121
1122 static const struct option lgopts[] = {
1123         {OPT_BUF_SIZE,     1, NULL, OPT_BUF_SIZE_NUM     },
1124         {OPT_FWD_MODE,     1, NULL, OPT_FWD_MODE_NUM     },
1125         {OPT_NB_DESC,      1, NULL, OPT_NB_DESC_NUM      },
1126         {OPT_TXFREET,      1, NULL, OPT_TXFREET_NUM      },
1127         {OPT_BURST,        1, NULL, OPT_BURST_NUM        },
1128         {OPT_QP,           1, NULL, OPT_QP_NUM           },
1129         {0,                0, NULL, 0                    }
1130 };
1131
1132 static void
1133 ntb_usage(const char *prgname)
1134 {
1135         printf("%s [EAL options] -- [options]\n"
1136                "-i: run in interactive mode.\n"
1137                "-qp=N: set number of queues as N (N > 0, default: 1).\n"
1138                "--fwd-mode=N: set fwd mode (N: file-trans | rxonly | "
1139                "txonly | iofwd, default: file-trans)\n"
1140                "--buf-size=N: set mbuf dataroom size as N (0 < N < 65535,"
1141                " default: 2048).\n"
1142                "--nb-desc=N: set number of descriptors as N (%u <= N <= %u,"
1143                " default: 1024).\n"
1144                "--txfreet=N: set tx free thresh for NTB driver as N. (N >= 0)\n"
1145                "--burst=N: set pkt burst as N (0 < N <= %u default: 32).\n",
1146                prgname, NTB_MIN_DESC_SIZE, NTB_MAX_DESC_SIZE,
1147                NTB_MAX_PKT_BURST);
1148 }
1149
1150 static void
1151 ntb_parse_args(int argc, char **argv)
1152 {
1153         char *prgname = argv[0], **argvopt = argv;
1154         int opt, opt_idx, n, i;
1155
1156         while ((opt = getopt_long(argc, argvopt, short_options,
1157                                 lgopts, &opt_idx)) != EOF) {
1158                 switch (opt) {
1159                 case 'i':
1160                         printf("Interactive-mode selected.\n");
1161                         interactive = 1;
1162                         break;
1163                 case OPT_QP_NUM:
1164                         n = atoi(optarg);
1165                         if (n > 0)
1166                                 num_queues = n;
1167                         else
1168                                 rte_exit(EXIT_FAILURE, "q must be > 0.\n");
1169                         break;
1170                 case OPT_BUF_SIZE_NUM:
1171                         n = atoi(optarg);
1172                         if (n > RTE_PKTMBUF_HEADROOM && n <= 0xFFFF)
1173                                 ntb_buf_size = n;
1174                         else
1175                                 rte_exit(EXIT_FAILURE, "buf-size must be > "
1176                                         "%u and < 65536.\n",
1177                                         RTE_PKTMBUF_HEADROOM);
1178                         break;
1179                 case OPT_FWD_MODE_NUM:
1180                         for (i = 0; i < MAX_FWD_MODE; i++) {
1181                                 if (!strcmp(optarg, fwd_mode_s[i])) {
1182                                         fwd_mode = i;
1183                                         break;
1184                                 }
1185                         }
1186                         if (i == MAX_FWD_MODE)
1187                                 rte_exit(EXIT_FAILURE, "Unsupported mode. "
1188                                 "(Should be: file-trans | rxonly | txonly "
1189                                 "| iofwd)\n");
1190                         break;
1191                 case OPT_NB_DESC_NUM:
1192                         n = atoi(optarg);
1193                         if (n >= NTB_MIN_DESC_SIZE && n <= NTB_MAX_DESC_SIZE)
1194                                 nb_desc = n;
1195                         else
1196                                 rte_exit(EXIT_FAILURE, "nb-desc must be within"
1197                                         " [%u, %u].\n", NTB_MIN_DESC_SIZE,
1198                                         NTB_MAX_DESC_SIZE);
1199                         break;
1200                 case OPT_TXFREET_NUM:
1201                         n = atoi(optarg);
1202                         if (n >= 0)
1203                                 tx_free_thresh = n;
1204                         else
1205                                 rte_exit(EXIT_FAILURE, "txfreet must be"
1206                                         " >= 0\n");
1207                         break;
1208                 case OPT_BURST_NUM:
1209                         n = atoi(optarg);
1210                         if (n > 0 && n <= NTB_MAX_PKT_BURST)
1211                                 pkt_burst = n;
1212                         else
1213                                 rte_exit(EXIT_FAILURE, "burst must be within "
1214                                         "(0, %u].\n", NTB_MAX_PKT_BURST);
1215                         break;
1216
1217                 default:
1218                         ntb_usage(prgname);
1219                         rte_exit(EXIT_FAILURE,
1220                                  "Command line is incomplete or incorrect.\n");
1221                         break;
1222                 }
1223         }
1224 }
1225
1226 static void
1227 ntb_mempool_mz_free(__rte_unused struct rte_mempool_memhdr *memhdr,
1228                 void *opaque)
1229 {
1230         const struct rte_memzone *mz = opaque;
1231         rte_memzone_free(mz);
1232 }
1233
1234 static struct rte_mempool *
1235 ntb_mbuf_pool_create(uint16_t mbuf_seg_size, uint32_t nb_mbuf,
1236                      struct ntb_dev_info ntb_info,
1237                      struct ntb_dev_config *ntb_conf,
1238                      unsigned int socket_id)
1239 {
1240         size_t mz_len, total_elt_sz, max_mz_len, left_sz;
1241         struct rte_pktmbuf_pool_private mbp_priv;
1242         char pool_name[RTE_MEMPOOL_NAMESIZE];
1243         char mz_name[RTE_MEMZONE_NAMESIZE];
1244         const struct rte_memzone *mz;
1245         struct rte_mempool *mp;
1246         uint64_t align;
1247         uint32_t mz_id;
1248         int ret;
1249
1250         snprintf(pool_name, sizeof(pool_name), "ntb_mbuf_pool_%u", socket_id);
1251         mp = rte_mempool_create_empty(pool_name, nb_mbuf,
1252                                       (mbuf_seg_size + sizeof(struct rte_mbuf)),
1253                                       MEMPOOL_CACHE_SIZE,
1254                                       sizeof(struct rte_pktmbuf_pool_private),
1255                                       socket_id, 0);
1256         if (mp == NULL)
1257                 return NULL;
1258
1259         mbp_priv.mbuf_data_room_size = mbuf_seg_size;
1260         mbp_priv.mbuf_priv_size = 0;
1261         rte_pktmbuf_pool_init(mp, &mbp_priv);
1262
1263         ntb_conf->mz_list = rte_zmalloc("ntb_memzone_list",
1264                                 sizeof(struct rte_memzone *) *
1265                                 ntb_info.mw_cnt, 0);
1266         if (ntb_conf->mz_list == NULL)
1267                 goto fail;
1268
1269         /* Put ntb header on mw0. */
1270         if (ntb_info.mw_size[0] < ntb_info.ntb_hdr_size) {
1271                 printf("mw0 (size: %" PRIu64 ") is not enough for ntb hdr"
1272                        " (size: %u)\n", ntb_info.mw_size[0],
1273                        ntb_info.ntb_hdr_size);
1274                 goto fail;
1275         }
1276
1277         total_elt_sz = mp->header_size + mp->elt_size + mp->trailer_size;
1278         left_sz = total_elt_sz * nb_mbuf;
1279         for (mz_id = 0; mz_id < ntb_info.mw_cnt; mz_id++) {
1280                 /* If populated mbuf is enough, no need to reserve extra mz. */
1281                 if (!left_sz)
1282                         break;
1283                 snprintf(mz_name, sizeof(mz_name), "ntb_mw_%d", mz_id);
1284                 align = ntb_info.mw_size_align ? ntb_info.mw_size[mz_id] :
1285                         RTE_CACHE_LINE_SIZE;
1286                 /* Reserve ntb header space on memzone 0. */
1287                 max_mz_len = mz_id ? ntb_info.mw_size[mz_id] :
1288                              ntb_info.mw_size[mz_id] - ntb_info.ntb_hdr_size;
1289                 mz_len = left_sz <= max_mz_len ? left_sz :
1290                         (max_mz_len / total_elt_sz * total_elt_sz);
1291                 if (!mz_len)
1292                         continue;
1293                 mz = rte_memzone_reserve_aligned(mz_name, mz_len, socket_id,
1294                                         RTE_MEMZONE_IOVA_CONTIG, align);
1295                 if (mz == NULL) {
1296                         printf("Cannot allocate %" PRIu64 " aligned memzone"
1297                                 " %u\n", align, mz_id);
1298                         goto fail;
1299                 }
1300                 left_sz -= mz_len;
1301
1302                 /* Reserve ntb header space on memzone 0. */
1303                 if (mz_id)
1304                         ret = rte_mempool_populate_iova(mp, mz->addr, mz->iova,
1305                                         mz->len, ntb_mempool_mz_free,
1306                                         (void *)(uintptr_t)mz);
1307                 else
1308                         ret = rte_mempool_populate_iova(mp,
1309                                         (void *)((size_t)mz->addr +
1310                                         ntb_info.ntb_hdr_size),
1311                                         mz->iova + ntb_info.ntb_hdr_size,
1312                                         mz->len - ntb_info.ntb_hdr_size,
1313                                         ntb_mempool_mz_free,
1314                                         (void *)(uintptr_t)mz);
1315                 if (ret < 0) {
1316                         rte_memzone_free(mz);
1317                         rte_mempool_free(mp);
1318                         return NULL;
1319                 }
1320
1321                 ntb_conf->mz_list[mz_id] = mz;
1322         }
1323         if (left_sz) {
1324                 printf("mw space is not enough for mempool.\n");
1325                 goto fail;
1326         }
1327
1328         ntb_conf->mz_num = mz_id;
1329         rte_mempool_obj_iter(mp, rte_pktmbuf_init, NULL);
1330
1331         return mp;
1332 fail:
1333         rte_mempool_free(mp);
1334         return NULL;
1335 }
1336
1337 int
1338 main(int argc, char **argv)
1339 {
1340         struct rte_eth_conf eth_pconf = eth_port_conf;
1341         struct rte_rawdev_info ntb_rawdev_conf;
1342         struct rte_rawdev_info ntb_rawdev_info;
1343         struct rte_eth_dev_info ethdev_info;
1344         struct rte_eth_rxconf eth_rx_conf;
1345         struct rte_eth_txconf eth_tx_conf;
1346         struct ntb_queue_conf ntb_q_conf;
1347         struct ntb_dev_config ntb_conf;
1348         struct ntb_dev_info ntb_info;
1349         uint64_t ntb_link_status;
1350         uint32_t nb_mbuf;
1351         int ret, i;
1352
1353         signal(SIGINT, signal_handler);
1354         signal(SIGTERM, signal_handler);
1355
1356         ret = rte_eal_init(argc, argv);
1357         if (ret < 0)
1358                 rte_exit(EXIT_FAILURE, "Error with EAL initialization.\n");
1359
1360         if (rte_lcore_count() < 2)
1361                 rte_exit(EXIT_FAILURE, "Need at least 2 cores\n");
1362
1363         /* Find 1st ntb rawdev. */
1364         for (i = 0; i < RTE_RAWDEV_MAX_DEVS; i++)
1365                 if (rte_rawdevs[i].driver_name &&
1366                     (strncmp(rte_rawdevs[i].driver_name, "raw_ntb",
1367                     NTB_DRV_NAME_LEN) == 0) && (rte_rawdevs[i].attached == 1))
1368                         break;
1369
1370         if (i == RTE_RAWDEV_MAX_DEVS)
1371                 rte_exit(EXIT_FAILURE, "Cannot find any ntb device.\n");
1372
1373         dev_id = i;
1374
1375         argc -= ret;
1376         argv += ret;
1377
1378         ntb_parse_args(argc, argv);
1379
1380         rte_rawdev_set_attr(dev_id, NTB_QUEUE_SZ_NAME, nb_desc);
1381         printf("Set queue size as %u.\n", nb_desc);
1382         rte_rawdev_set_attr(dev_id, NTB_QUEUE_NUM_NAME, num_queues);
1383         printf("Set queue number as %u.\n", num_queues);
1384         ntb_rawdev_info.dev_private = (rte_rawdev_obj_t)(&ntb_info);
1385         rte_rawdev_info_get(dev_id, &ntb_rawdev_info);
1386
1387         nb_mbuf = nb_desc * num_queues * 2 * 2 + rte_lcore_count() *
1388                   MEMPOOL_CACHE_SIZE;
1389         mbuf_pool = ntb_mbuf_pool_create(ntb_buf_size, nb_mbuf, ntb_info,
1390                                          &ntb_conf, rte_socket_id());
1391         if (mbuf_pool == NULL)
1392                 rte_exit(EXIT_FAILURE, "Cannot create mbuf pool.\n");
1393
1394         ntb_conf.num_queues = num_queues;
1395         ntb_conf.queue_size = nb_desc;
1396         ntb_rawdev_conf.dev_private = (rte_rawdev_obj_t)(&ntb_conf);
1397         ret = rte_rawdev_configure(dev_id, &ntb_rawdev_conf);
1398         if (ret)
1399                 rte_exit(EXIT_FAILURE, "Can't config ntb dev: err=%d, "
1400                         "port=%u\n", ret, dev_id);
1401
1402         ntb_q_conf.tx_free_thresh = tx_free_thresh;
1403         ntb_q_conf.nb_desc = nb_desc;
1404         ntb_q_conf.rx_mp = mbuf_pool;
1405         for (i = 0; i < num_queues; i++) {
1406                 /* Setup rawdev queue */
1407                 ret = rte_rawdev_queue_setup(dev_id, i, &ntb_q_conf);
1408                 if (ret < 0)
1409                         rte_exit(EXIT_FAILURE,
1410                                 "Failed to setup ntb queue %u.\n", i);
1411         }
1412
1413         /* Waiting for peer dev up at most 100s.*/
1414         printf("Checking ntb link status...\n");
1415         for (i = 0; i < 1000; i++) {
1416                 rte_rawdev_get_attr(dev_id, NTB_LINK_STATUS_NAME,
1417                                     &ntb_link_status);
1418                 if (ntb_link_status) {
1419                         printf("Peer dev ready, ntb link up.\n");
1420                         break;
1421                 }
1422                 rte_delay_ms(100);
1423         }
1424         rte_rawdev_get_attr(dev_id, NTB_LINK_STATUS_NAME, &ntb_link_status);
1425         if (ntb_link_status == 0)
1426                 printf("Expire 100s. Link is not up. Please restart app.\n");
1427
1428         ret = rte_rawdev_start(dev_id);
1429         if (ret < 0)
1430                 rte_exit(EXIT_FAILURE, "rte_rawdev_start: err=%d, port=%u\n",
1431                         ret, dev_id);
1432
1433         /* Find 1st ethdev */
1434         eth_port_id = rte_eth_find_next(0);
1435
1436         if (eth_port_id < RTE_MAX_ETHPORTS) {
1437                 rte_eth_dev_info_get(eth_port_id, &ethdev_info);
1438                 eth_pconf.rx_adv_conf.rss_conf.rss_hf &=
1439                                 ethdev_info.flow_type_rss_offloads;
1440                 ret = rte_eth_dev_configure(eth_port_id, num_queues,
1441                                             num_queues, &eth_pconf);
1442                 if (ret)
1443                         rte_exit(EXIT_FAILURE, "Can't config ethdev: err=%d, "
1444                                 "port=%u\n", ret, eth_port_id);
1445                 eth_rx_conf = ethdev_info.default_rxconf;
1446                 eth_rx_conf.offloads = eth_pconf.rxmode.offloads;
1447                 eth_tx_conf = ethdev_info.default_txconf;
1448                 eth_tx_conf.offloads = eth_pconf.txmode.offloads;
1449
1450                 /* Setup ethdev queue if ethdev exists */
1451                 for (i = 0; i < num_queues; i++) {
1452                         ret = rte_eth_rx_queue_setup(eth_port_id, i, nb_desc,
1453                                         rte_eth_dev_socket_id(eth_port_id),
1454                                         &eth_rx_conf, mbuf_pool);
1455                         if (ret < 0)
1456                                 rte_exit(EXIT_FAILURE,
1457                                         "Failed to setup eth rxq %u.\n", i);
1458                         ret = rte_eth_tx_queue_setup(eth_port_id, i, nb_desc,
1459                                         rte_eth_dev_socket_id(eth_port_id),
1460                                         &eth_tx_conf);
1461                         if (ret < 0)
1462                                 rte_exit(EXIT_FAILURE,
1463                                         "Failed to setup eth txq %u.\n", i);
1464                 }
1465
1466                 ret = rte_eth_dev_start(eth_port_id);
1467                 if (ret < 0)
1468                         rte_exit(EXIT_FAILURE, "rte_eth_dev_start: err=%d, "
1469                                 "port=%u\n", ret, eth_port_id);
1470         }
1471
1472         /* initialize port stats */
1473         memset(&ntb_port_stats, 0, sizeof(ntb_port_stats));
1474
1475         /* Set default fwd mode if user doesn't set it. */
1476         if (fwd_mode == MAX_FWD_MODE && eth_port_id < RTE_MAX_ETHPORTS) {
1477                 printf("Set default fwd mode as iofwd.\n");
1478                 fwd_mode = IOFWD;
1479         }
1480         if (fwd_mode == MAX_FWD_MODE) {
1481                 printf("Set default fwd mode as file-trans.\n");
1482                 fwd_mode = FILE_TRANS;
1483         }
1484
1485         if (interactive) {
1486                 sleep(1);
1487                 prompt();
1488         } else {
1489                 start_pkt_fwd();
1490         }
1491
1492         return 0;
1493 }