examples/ntb/ntb_fwd.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(c) 2019 Intel Corporation
   3  */
   4 #include <stdint.h>
   5 #include <stdio.h>
   6 #include <inttypes.h>
   7 #include <unistd.h>
   8 #include <signal.h>
   9 #include <string.h>
  10 #include <getopt.h>
  11
  12 #include <cmdline_parse_string.h>
  13 #include <cmdline_socket.h>
  14 #include <cmdline.h>
  15 #include <rte_common.h>
  16 #include <rte_rawdev.h>
  17 #include <rte_ethdev.h>
  18 #include <rte_malloc.h>
  19 #include <rte_lcore.h>
  20 #include <rte_cycles.h>
  21 #include <rte_pmd_ntb.h>
  22 #include <rte_mbuf_pool_ops.h>
  23
  24 /* Per-port statistics struct */
  25 struct ntb_port_statistics {
  26         uint64_t tx;
  27         uint64_t rx;
  28 } __rte_cache_aligned;
  29 /* Port 0: NTB dev, Port 1: ethdev when iofwd. */
  30 struct ntb_port_statistics ntb_port_stats[2];
  31
  32 struct ntb_fwd_stream {
  33         uint16_t tx_port;
  34         uint16_t rx_port;
  35         uint16_t qp_id;
  36         uint8_t tx_ntb;  /* If ntb device is tx port. */
  37 };
  38
  39 struct ntb_fwd_lcore_conf {
  40         uint16_t stream_id;
  41         uint16_t nb_stream;
  42         uint8_t stopped;
  43 };
  44
  45 enum ntb_fwd_mode {
  46         FILE_TRANS = 0,
  47         RXONLY,
  48         TXONLY,
  49         IOFWD,
  50         MAX_FWD_MODE,
  51 };
  52 static const char *const fwd_mode_s[] = {
  53         "file-trans",
  54         "rxonly",
  55         "txonly",
  56         "iofwd",
  57         NULL,
  58 };
  59 static enum ntb_fwd_mode fwd_mode = MAX_FWD_MODE;
  60
  61 static struct ntb_fwd_lcore_conf fwd_lcore_conf[RTE_MAX_LCORE];
  62 static struct ntb_fwd_stream *fwd_streams;
  63
  64 static struct rte_mempool *mbuf_pool;
  65
  66 #define NTB_DRV_NAME_LEN 7
  67 #define MEMPOOL_CACHE_SIZE 256
  68
  69 static uint8_t in_test;
  70 static uint8_t interactive = 1;
  71 static uint16_t eth_port_id = RTE_MAX_ETHPORTS;
  72 static uint16_t dev_id;
  73
  74 /* Number of queues, default set as 1 */
  75 static uint16_t num_queues = 1;
  76 static uint16_t ntb_buf_size = RTE_MBUF_DEFAULT_BUF_SIZE;
  77
  78 /* Configurable number of descriptors */
  79 #define NTB_DEFAULT_NUM_DESCS 1024
  80 static uint16_t nb_desc = NTB_DEFAULT_NUM_DESCS;
  81
  82 static uint16_t tx_free_thresh;
  83
  84 #define NTB_MAX_PKT_BURST 32
  85 #define NTB_DFLT_PKT_BURST 32
  86 static uint16_t pkt_burst = NTB_DFLT_PKT_BURST;
  87
  88 #define BURST_TX_RETRIES 64
  89
  90 static struct rte_eth_conf eth_port_conf = {
  91         .rxmode = {
  92                 .mq_mode = ETH_MQ_RX_RSS,
  93                 .split_hdr_size = 0,
  94         },
  95         .rx_adv_conf = {
  96                 .rss_conf = {
  97                         .rss_key = NULL,
  98                         .rss_hf = ETH_RSS_IP,
  99                 },
 100         },
 101         .txmode = {
 102                 .mq_mode = ETH_MQ_TX_NONE,
 103         },
 104 };
 105
 106 /* *** Help command with introduction. *** */
 107 struct cmd_help_result {
 108         cmdline_fixed_string_t help;
 109 };
 110
 111 static void
 112 cmd_help_parsed(__rte_unused void *parsed_result,
 113                 struct cmdline *cl,
 114                 __rte_unused void *data)
 115 {
 116         cmdline_printf(
 117                 cl,
 118                 "\n"
 119                 "The following commands are currently available:\n\n"
 120                 "Control:\n"
 121                 "    quit                                      :"
 122                 " Quit the application.\n"
 123                 "\nTransmission:\n"
 124                 "    send [path]                               :"
 125                 " Send [path] file. Only take effect in file-trans mode\n"
 126                 "    start                                     :"
 127                 " Start transmissions.\n"
 128                 "    stop                                      :"
 129                 " Stop transmissions.\n"
 130                 "    clear/show port stats                     :"
 131                 " Clear/show port stats.\n"
 132                 "    set fwd file-trans/rxonly/txonly/iofwd    :"
 133                 " Set packet forwarding mode.\n"
 134         );
 135
 136 }
 137
 138 cmdline_parse_token_string_t cmd_help_help =
 139         TOKEN_STRING_INITIALIZER(struct cmd_help_result, help, "help");
 140
 141 cmdline_parse_inst_t cmd_help = {
 142         .f = cmd_help_parsed,
 143         .data = NULL,
 144         .help_str = "show help",
 145         .tokens = {
 146                 (void *)&cmd_help_help,
 147                 NULL,
 148         },
 149 };
 150
 151 /* *** QUIT *** */
 152 struct cmd_quit_result {
 153         cmdline_fixed_string_t quit;
 154 };
 155
 156 static void
 157 cmd_quit_parsed(__rte_unused void *parsed_result,
 158                 struct cmdline *cl,
 159                 __rte_unused void *data)
 160 {
 161         struct ntb_fwd_lcore_conf *conf;
 162         uint32_t lcore_id;
 163
 164         /* Stop transmission first. */
 165         RTE_LCORE_FOREACH_SLAVE(lcore_id) {
 166                 conf = &fwd_lcore_conf[lcore_id];
 167
 168                 if (!conf->nb_stream)
 169                         continue;
 170
 171                 if (conf->stopped)
 172                         continue;
 173
 174                 conf->stopped = 1;
 175         }
 176         printf("\nWaiting for lcores to finish...\n");
 177         rte_eal_mp_wait_lcore();
 178         in_test = 0;
 179
 180         /* Stop traffic and Close port. */
 181         rte_rawdev_stop(dev_id);
 182         rte_rawdev_close(dev_id);
 183         if (eth_port_id < RTE_MAX_ETHPORTS && fwd_mode == IOFWD) {
 184                 rte_eth_dev_stop(eth_port_id);
 185                 rte_eth_dev_close(eth_port_id);
 186         }
 187
 188         cmdline_quit(cl);
 189 }
 190
 191 cmdline_parse_token_string_t cmd_quit_quit =
 192                 TOKEN_STRING_INITIALIZER(struct cmd_quit_result, quit, "quit");
 193
 194 cmdline_parse_inst_t cmd_quit = {
 195         .f = cmd_quit_parsed,
 196         .data = NULL,
 197         .help_str = "exit application",
 198         .tokens = {
 199                 (void *)&cmd_quit_quit,
 200                 NULL,
 201         },
 202 };
 203
 204 /* *** SEND FILE PARAMETERS *** */
 205 struct cmd_sendfile_result {
 206         cmdline_fixed_string_t send_string;
 207         char filepath[];
 208 };
 209
 210 static void
 211 cmd_sendfile_parsed(void *parsed_result,
 212                     __rte_unused struct cmdline *cl,
 213                     __rte_unused void *data)
 214 {
 215         struct cmd_sendfile_result *res = parsed_result;
 216         struct rte_rawdev_buf *pkts_send[NTB_MAX_PKT_BURST];
 217         struct rte_mbuf *mbuf_send[NTB_MAX_PKT_BURST];
 218         uint64_t size, count, i, j, nb_burst;
 219         uint16_t nb_tx, buf_size;
 220         unsigned int nb_pkt;
 221         size_t queue_id = 0;
 222         uint16_t retry = 0;
 223         uint32_t val;
 224         FILE *file;
 225         int ret;
 226
 227         if (num_queues != 1) {
 228                 printf("File transmission only supports 1 queue.\n");
 229                 num_queues = 1;
 230         }
 231
 232         file = fopen(res->filepath, "r");
 233         if (file == NULL) {
 234                 printf("Fail to open the file.\n");
 235                 return;
 236         }
 237
 238         if (fseek(file, 0, SEEK_END) < 0) {
 239                 printf("Fail to get file size.\n");
 240                 fclose(file);
 241                 return;
 242         }
 243         size = ftell(file);
 244         if (fseek(file, 0, SEEK_SET) < 0) {
 245                 printf("Fail to get file size.\n");
 246                 fclose(file);
 247                 return;
 248         }
 249
 250         /* Tell remote about the file size. */
 251         val = size >> 32;
 252         rte_rawdev_set_attr(dev_id, "spad_user_0", val);
 253         val = size;
 254         rte_rawdev_set_attr(dev_id, "spad_user_1", val);
 255         printf("Sending file, size is %"PRIu64"\n", size);
 256
 257         for (i = 0; i < NTB_MAX_PKT_BURST; i++)
 258                 pkts_send[i] = (struct rte_rawdev_buf *)
 259                                 malloc(sizeof(struct rte_rawdev_buf));
 260
 261         buf_size = ntb_buf_size - RTE_PKTMBUF_HEADROOM;
 262         count = (size + buf_size - 1) / buf_size;
 263         nb_burst = (count + pkt_burst - 1) / pkt_burst;
 264
 265         for (i = 0; i < nb_burst; i++) {
 266                 val = RTE_MIN(count, pkt_burst);
 267                 if (rte_mempool_get_bulk(mbuf_pool, (void **)mbuf_send,
 268                                         val) == 0) {
 269                         for (nb_pkt = 0; nb_pkt < val; nb_pkt++) {
 270                                 mbuf_send[nb_pkt]->port = dev_id;
 271                                 mbuf_send[nb_pkt]->data_len =
 272                                 fread(rte_pktmbuf_mtod(mbuf_send[nb_pkt],
 273                                         void *), 1, buf_size, file);
 274                                 mbuf_send[nb_pkt]->pkt_len =
 275                                         mbuf_send[nb_pkt]->data_len;
 276                                 pkts_send[nb_pkt]->buf_addr = mbuf_send[nb_pkt];
 277                         }
 278                 } else {
 279                         for (nb_pkt = 0; nb_pkt < val; nb_pkt++) {
 280                                 mbuf_send[nb_pkt] =
 281                                         rte_mbuf_raw_alloc(mbuf_pool);
 282                                 if (mbuf_send[nb_pkt] == NULL)
 283                                         break;
 284                                 mbuf_send[nb_pkt]->port = dev_id;
 285                                 mbuf_send[nb_pkt]->data_len =
 286                                 fread(rte_pktmbuf_mtod(mbuf_send[nb_pkt],
 287                                         void *), 1, buf_size, file);
 288                                 mbuf_send[nb_pkt]->pkt_len =
 289                                         mbuf_send[nb_pkt]->data_len;
 290                                 pkts_send[nb_pkt]->buf_addr = mbuf_send[nb_pkt];
 291                         }
 292                 }
 293
 294                 ret = rte_rawdev_enqueue_buffers(dev_id, pkts_send, nb_pkt,
 295                                                 (void *)queue_id);
 296                 if (ret < 0) {
 297                         printf("Enqueue failed with err %d\n", ret);
 298                         for (j = 0; j < nb_pkt; j++)
 299                                 rte_pktmbuf_free(mbuf_send[j]);
 300                         goto clean;
 301                 }
 302                 nb_tx = ret;
 303                 while (nb_tx != nb_pkt && retry < BURST_TX_RETRIES) {
 304                         rte_delay_us(1);
 305                         ret = rte_rawdev_enqueue_buffers(dev_id,
 306                                         &pkts_send[nb_tx], nb_pkt - nb_tx,
 307                                         (void *)queue_id);
 308                         if (ret < 0) {
 309                                 printf("Enqueue failed with err %d\n", ret);
 310                                 for (j = nb_tx; j < nb_pkt; j++)
 311                                         rte_pktmbuf_free(mbuf_send[j]);
 312                                 goto clean;
 313                         }
 314                         nb_tx += ret;
 315                 }
 316                 count -= nb_pkt;
 317         }
 318
 319         /* Clear register after file sending done. */
 320         rte_rawdev_set_attr(dev_id, "spad_user_0", 0);
 321         rte_rawdev_set_attr(dev_id, "spad_user_1", 0);
 322         printf("Done sending file.\n");
 323
 324 clean:
 325         for (i = 0; i < NTB_MAX_PKT_BURST; i++)
 326                 free(pkts_send[i]);
 327         fclose(file);
 328 }
 329
 330 cmdline_parse_token_string_t cmd_send_file_send =
 331         TOKEN_STRING_INITIALIZER(struct cmd_sendfile_result, send_string,
 332                                  "send");
 333 cmdline_parse_token_string_t cmd_send_file_filepath =
 334         TOKEN_STRING_INITIALIZER(struct cmd_sendfile_result, filepath, NULL);
 335
 336
 337 cmdline_parse_inst_t cmd_send_file = {
 338         .f = cmd_sendfile_parsed,
 339         .data = NULL,
 340         .help_str = "send <file_path>",
 341         .tokens = {
 342                 (void *)&cmd_send_file_send,
 343                 (void *)&cmd_send_file_filepath,
 344                 NULL,
 345         },
 346 };
 347
 348 #define RECV_FILE_LEN 30
 349 static int
 350 start_polling_recv_file(void *param)
 351 {
 352         struct rte_rawdev_buf *pkts_recv[NTB_MAX_PKT_BURST];
 353         struct ntb_fwd_lcore_conf *conf = param;
 354         struct rte_mbuf *mbuf;
 355         char filepath[RECV_FILE_LEN];
 356         uint64_t val, size, file_len;
 357         uint16_t nb_rx, i, file_no;
 358         size_t queue_id = 0;
 359         FILE *file;
 360         int ret;
 361
 362         for (i = 0; i < NTB_MAX_PKT_BURST; i++)
 363                 pkts_recv[i] = (struct rte_rawdev_buf *)
 364                                 malloc(sizeof(struct rte_rawdev_buf));
 365
 366         file_no = 0;
 367         while (!conf->stopped) {
 368                 snprintf(filepath, RECV_FILE_LEN, "ntb_recv_file%d", file_no);
 369                 file = fopen(filepath, "w");
 370                 if (file == NULL) {
 371                         printf("Fail to open the file.\n");
 372                         return -EINVAL;
 373                 }
 374
 375                 rte_rawdev_get_attr(dev_id, "spad_user_0", &val);
 376                 size = val << 32;
 377                 rte_rawdev_get_attr(dev_id, "spad_user_1", &val);
 378                 size |= val;
 379
 380                 if (!size) {
 381                         fclose(file);
 382                         continue;
 383                 }
 384
 385                 file_len = 0;
 386                 nb_rx = NTB_MAX_PKT_BURST;
 387                 while (file_len < size && !conf->stopped) {
 388                         ret = rte_rawdev_dequeue_buffers(dev_id, pkts_recv,
 389                                         pkt_burst, (void *)queue_id);
 390                         if (ret < 0) {
 391                                 printf("Dequeue failed with err %d\n", ret);
 392                                 fclose(file);
 393                                 goto clean;
 394                         }
 395                         nb_rx = ret;
 396                         ntb_port_stats[0].rx += nb_rx;
 397                         for (i = 0; i < nb_rx; i++) {
 398                                 mbuf = pkts_recv[i]->buf_addr;
 399                                 fwrite(rte_pktmbuf_mtod(mbuf, void *), 1,
 400                                         mbuf->data_len, file);
 401                                 file_len += mbuf->data_len;
 402                                 rte_pktmbuf_free(mbuf);
 403                                 pkts_recv[i]->buf_addr = NULL;
 404                         }
 405                 }
 406
 407                 printf("Received file (size: %" PRIu64 ") from peer to %s.\n",
 408                         size, filepath);
 409                 fclose(file);
 410                 file_no++;
 411         }
 412
 413 clean:
 414         for (i = 0; i < NTB_MAX_PKT_BURST; i++)
 415                 free(pkts_recv[i]);
 416         return 0;
 417 }
 418
 419 static int
 420 start_iofwd_per_lcore(void *param)
 421 {
 422         struct rte_rawdev_buf *ntb_buf[NTB_MAX_PKT_BURST];
 423         struct rte_mbuf *pkts_burst[NTB_MAX_PKT_BURST];
 424         struct ntb_fwd_lcore_conf *conf = param;
 425         struct ntb_fwd_stream fs;
 426         uint16_t nb_rx, nb_tx;
 427         int i, j, ret;
 428
 429         for (i = 0; i < NTB_MAX_PKT_BURST; i++)
 430                 ntb_buf[i] = (struct rte_rawdev_buf *)
 431                              malloc(sizeof(struct rte_rawdev_buf));
 432
 433         while (!conf->stopped) {
 434                 for (i = 0; i < conf->nb_stream; i++) {
 435                         fs = fwd_streams[conf->stream_id + i];
 436                         if (fs.tx_ntb) {
 437                                 nb_rx = rte_eth_rx_burst(fs.rx_port,
 438                                                 fs.qp_id, pkts_burst,
 439                                                 pkt_burst);
 440                                 if (unlikely(nb_rx == 0))
 441                                         continue;
 442                                 for (j = 0; j < nb_rx; j++)
 443                                         ntb_buf[j]->buf_addr = pkts_burst[j];
 444                                 ret = rte_rawdev_enqueue_buffers(fs.tx_port,
 445                                                 ntb_buf, nb_rx,
 446                                                 (void *)(size_t)fs.qp_id);
 447                                 if (ret < 0) {
 448                                         printf("Enqueue failed with err %d\n",
 449                                                 ret);
 450                                         for (j = 0; j < nb_rx; j++)
 451                                                 rte_pktmbuf_free(pkts_burst[j]);
 452                                         goto clean;
 453                                 }
 454                                 nb_tx = ret;
 455                                 ntb_port_stats[0].tx += nb_tx;
 456                                 ntb_port_stats[1].rx += nb_rx;
 457                         } else {
 458                                 ret = rte_rawdev_dequeue_buffers(fs.rx_port,
 459                                                 ntb_buf, pkt_burst,
 460                                                 (void *)(size_t)fs.qp_id);
 461                                 if (ret < 0) {
 462                                         printf("Dequeue failed with err %d\n",
 463                                                 ret);
 464                                         goto clean;
 465                                 }
 466                                 nb_rx = ret;
 467                                 if (unlikely(nb_rx == 0))
 468                                         continue;
 469                                 for (j = 0; j < nb_rx; j++)
 470                                         pkts_burst[j] = ntb_buf[j]->buf_addr;
 471                                 nb_tx = rte_eth_tx_burst(fs.tx_port,
 472                                         fs.qp_id, pkts_burst, nb_rx);
 473                                 ntb_port_stats[1].tx += nb_tx;
 474                                 ntb_port_stats[0].rx += nb_rx;
 475                         }
 476                         if (unlikely(nb_tx < nb_rx)) {
 477                                 do {
 478                                         rte_pktmbuf_free(pkts_burst[nb_tx]);
 479                                 } while (++nb_tx < nb_rx);
 480                         }
 481                 }
 482         }
 483
 484 clean:
 485         for (i = 0; i < NTB_MAX_PKT_BURST; i++)
 486                 free(ntb_buf[i]);
 487
 488         return 0;
 489 }
 490
 491 static int
 492 start_rxonly_per_lcore(void *param)
 493 {
 494         struct rte_rawdev_buf *ntb_buf[NTB_MAX_PKT_BURST];
 495         struct ntb_fwd_lcore_conf *conf = param;
 496         struct ntb_fwd_stream fs;
 497         uint16_t nb_rx;
 498         int i, j, ret;
 499
 500         for (i = 0; i < NTB_MAX_PKT_BURST; i++)
 501                 ntb_buf[i] = (struct rte_rawdev_buf *)
 502                              malloc(sizeof(struct rte_rawdev_buf));
 503
 504         while (!conf->stopped) {
 505                 for (i = 0; i < conf->nb_stream; i++) {
 506                         fs = fwd_streams[conf->stream_id + i];
 507                         ret = rte_rawdev_dequeue_buffers(fs.rx_port,
 508                               ntb_buf, pkt_burst, (void *)(size_t)fs.qp_id);
 509                         if (ret < 0) {
 510                                 printf("Dequeue failed with err %d\n", ret);
 511                                 goto clean;
 512                         }
 513                         nb_rx = ret;
 514                         if (unlikely(nb_rx == 0))
 515                                 continue;
 516                         ntb_port_stats[0].rx += nb_rx;
 517
 518                         for (j = 0; j < nb_rx; j++)
 519                                 rte_pktmbuf_free(ntb_buf[j]->buf_addr);
 520                 }
 521         }
 522
 523 clean:
 524         for (i = 0; i < NTB_MAX_PKT_BURST; i++)
 525                 free(ntb_buf[i]);
 526
 527         return 0;
 528 }
 529
 530
 531 static int
 532 start_txonly_per_lcore(void *param)
 533 {
 534         struct rte_rawdev_buf *ntb_buf[NTB_MAX_PKT_BURST];
 535         struct rte_mbuf *pkts_burst[NTB_MAX_PKT_BURST];
 536         struct ntb_fwd_lcore_conf *conf = param;
 537         struct ntb_fwd_stream fs;
 538         uint16_t nb_pkt, nb_tx;
 539         int i, j, ret;
 540
 541         for (i = 0; i < NTB_MAX_PKT_BURST; i++)
 542                 ntb_buf[i] = (struct rte_rawdev_buf *)
 543                              malloc(sizeof(struct rte_rawdev_buf));
 544
 545         while (!conf->stopped) {
 546                 for (i = 0; i < conf->nb_stream; i++) {
 547                         fs = fwd_streams[conf->stream_id + i];
 548                         if (rte_mempool_get_bulk(mbuf_pool, (void **)pkts_burst,
 549                                   pkt_burst) == 0) {
 550                                 for (nb_pkt = 0; nb_pkt < pkt_burst; nb_pkt++) {
 551                                         pkts_burst[nb_pkt]->port = dev_id;
 552                                         pkts_burst[nb_pkt]->data_len =
 553                                                 pkts_burst[nb_pkt]->buf_len -
 554                                                 RTE_PKTMBUF_HEADROOM;
 555                                         pkts_burst[nb_pkt]->pkt_len =
 556                                                 pkts_burst[nb_pkt]->data_len;
 557                                         ntb_buf[nb_pkt]->buf_addr =
 558                                                 pkts_burst[nb_pkt];
 559                                 }
 560                         } else {
 561                                 for (nb_pkt = 0; nb_pkt < pkt_burst; nb_pkt++) {
 562                                         pkts_burst[nb_pkt] =
 563                                                 rte_pktmbuf_alloc(mbuf_pool);
 564                                         if (pkts_burst[nb_pkt] == NULL)
 565                                                 break;
 566                                         pkts_burst[nb_pkt]->port = dev_id;
 567                                         pkts_burst[nb_pkt]->data_len =
 568                                                 pkts_burst[nb_pkt]->buf_len -
 569                                                 RTE_PKTMBUF_HEADROOM;
 570                                         pkts_burst[nb_pkt]->pkt_len =
 571                                                 pkts_burst[nb_pkt]->data_len;
 572                                         ntb_buf[nb_pkt]->buf_addr =
 573                                                 pkts_burst[nb_pkt];
 574                                 }
 575                         }
 576                         ret = rte_rawdev_enqueue_buffers(fs.tx_port, ntb_buf,
 577                                         nb_pkt, (void *)(size_t)fs.qp_id);
 578                         if (ret < 0) {
 579                                 printf("Enqueue failed with err %d\n", ret);
 580                                 for (j = 0; j < nb_pkt; j++)
 581                                         rte_pktmbuf_free(pkts_burst[j]);
 582                                 goto clean;
 583                         }
 584                         nb_tx = ret;
 585                         ntb_port_stats[0].tx += nb_tx;
 586                         if (unlikely(nb_tx < nb_pkt)) {
 587                                 do {
 588                                         rte_pktmbuf_free(pkts_burst[nb_tx]);
 589                                 } while (++nb_tx < nb_pkt);
 590                         }
 591                 }
 592         }
 593
 594 clean:
 595         for (i = 0; i < NTB_MAX_PKT_BURST; i++)
 596                 free(ntb_buf[i]);
 597
 598         return 0;
 599 }
 600
 601 static int
 602 ntb_fwd_config_setup(void)
 603 {
 604         uint16_t i;
 605
 606         /* Make sure iofwd has valid ethdev. */
 607         if (fwd_mode == IOFWD && eth_port_id >= RTE_MAX_ETHPORTS) {
 608                 printf("No ethdev, cannot be in iofwd mode.");
 609                 return -EINVAL;
 610         }
 611
 612         if (fwd_mode == IOFWD) {
 613                 fwd_streams = rte_zmalloc("ntb_fwd: fwd_streams",
 614                         sizeof(struct ntb_fwd_stream) * num_queues * 2,
 615                         RTE_CACHE_LINE_SIZE);
 616                 for (i = 0; i < num_queues; i++) {
 617                         fwd_streams[i * 2].qp_id = i;
 618                         fwd_streams[i * 2].tx_port = dev_id;
 619                         fwd_streams[i * 2].rx_port = eth_port_id;
 620                         fwd_streams[i * 2].tx_ntb = 1;
 621
 622                         fwd_streams[i * 2 + 1].qp_id = i;
 623                         fwd_streams[i * 2 + 1].tx_port = eth_port_id;
 624                         fwd_streams[i * 2 + 1].rx_port = dev_id;
 625                         fwd_streams[i * 2 + 1].tx_ntb = 0;
 626                 }
 627                 return 0;
 628         }
 629
 630         if (fwd_mode == RXONLY || fwd_mode == FILE_TRANS) {
 631                 /* Only support 1 queue in file-trans for in order. */
 632                 if (fwd_mode == FILE_TRANS)
 633                         num_queues = 1;
 634
 635                 fwd_streams = rte_zmalloc("ntb_fwd: fwd_streams",
 636                                 sizeof(struct ntb_fwd_stream) * num_queues,
 637                                 RTE_CACHE_LINE_SIZE);
 638                 for (i = 0; i < num_queues; i++) {
 639                         fwd_streams[i].qp_id = i;
 640                         fwd_streams[i].tx_port = RTE_MAX_ETHPORTS;
 641                         fwd_streams[i].rx_port = dev_id;
 642                         fwd_streams[i].tx_ntb = 0;
 643                 }
 644                 return 0;
 645         }
 646
 647         if (fwd_mode == TXONLY) {
 648                 fwd_streams = rte_zmalloc("ntb_fwd: fwd_streams",
 649                                 sizeof(struct ntb_fwd_stream) * num_queues,
 650                                 RTE_CACHE_LINE_SIZE);
 651                 for (i = 0; i < num_queues; i++) {
 652                         fwd_streams[i].qp_id = i;
 653                         fwd_streams[i].tx_port = dev_id;
 654                         fwd_streams[i].rx_port = RTE_MAX_ETHPORTS;
 655                         fwd_streams[i].tx_ntb = 1;
 656                 }
 657         }
 658         return 0;
 659 }
 660
 661 static void
 662 assign_stream_to_lcores(void)
 663 {
 664         struct ntb_fwd_lcore_conf *conf;
 665         struct ntb_fwd_stream *fs;
 666         uint16_t nb_streams, sm_per_lcore, sm_id, i;
 667         uint32_t lcore_id;
 668         uint8_t lcore_num, nb_extra;
 669
 670         lcore_num = rte_lcore_count();
 671         /* Exclude master core */
 672         lcore_num--;
 673
 674         nb_streams = (fwd_mode == IOFWD) ? num_queues * 2 : num_queues;
 675
 676         sm_per_lcore = nb_streams / lcore_num;
 677         nb_extra = nb_streams % lcore_num;
 678         sm_id = 0;
 679         i = 0;
 680
 681         RTE_LCORE_FOREACH_SLAVE(lcore_id) {
 682                 conf = &fwd_lcore_conf[lcore_id];
 683
 684                 if (i < nb_extra) {
 685                         conf->nb_stream = sm_per_lcore + 1;
 686                         conf->stream_id = sm_id;
 687                         sm_id = sm_id + sm_per_lcore + 1;
 688                 } else {
 689                         conf->nb_stream = sm_per_lcore;
 690                         conf->stream_id = sm_id;
 691                         sm_id = sm_id + sm_per_lcore;
 692                 }
 693
 694                 i++;
 695                 if (sm_id >= nb_streams)
 696                         break;
 697         }
 698
 699         /* Print packet forwading config. */
 700         RTE_LCORE_FOREACH_SLAVE(lcore_id) {
 701                 conf = &fwd_lcore_conf[lcore_id];
 702
 703                 if (!conf->nb_stream)
 704                         continue;
 705
 706                 printf("Streams on Lcore %u :\n", lcore_id);
 707                 for (i = 0; i < conf->nb_stream; i++) {
 708                         fs = &fwd_streams[conf->stream_id + i];
 709                         if (fwd_mode == IOFWD)
 710                                 printf(" + Stream %u : %s%u RX -> %s%u TX,"
 711                                         " Q=%u\n", conf->stream_id + i,
 712                                         fs->tx_ntb ? "Eth" : "NTB", fs->rx_port,
 713                                         fs->tx_ntb ? "NTB" : "Eth", fs->tx_port,
 714                                         fs->qp_id);
 715                         if (fwd_mode == FILE_TRANS || fwd_mode == RXONLY)
 716                                 printf(" + Stream %u : %s%u RX only\n",
 717                                         conf->stream_id, "NTB", fs->rx_port);
 718                         if (fwd_mode == TXONLY)
 719                                 printf(" + Stream %u : %s%u TX only\n",
 720                                         conf->stream_id, "NTB", fs->tx_port);
 721                 }
 722         }
 723 }
 724
 725 static void
 726 start_pkt_fwd(void)
 727 {
 728         struct ntb_fwd_lcore_conf *conf;
 729         struct rte_eth_link eth_link;
 730         uint32_t lcore_id;
 731         int ret, i;
 732
 733         ret = ntb_fwd_config_setup();
 734         if (ret < 0) {
 735                 printf("Cannot start traffic. Please reset fwd mode.\n");
 736                 return;
 737         }
 738
 739         /* If using iofwd, checking ethdev link status first. */
 740         if (fwd_mode == IOFWD) {
 741                 printf("Checking eth link status...\n");
 742                 /* Wait for eth link up at most 100 times. */
 743                 for (i = 0; i < 100; i++) {
 744                         ret = rte_eth_link_get(eth_port_id, &eth_link);
 745                         if (ret < 0) {
 746                                 printf("Link get failed with err %d\n", ret);
 747                                 return;
 748                         }
 749                         if (eth_link.link_status) {
 750                                 printf("Eth%u Link Up. Speed %u Mbps - %s\n",
 751                                         eth_port_id, eth_link.link_speed,
 752                                         (eth_link.link_duplex ==
 753                                          ETH_LINK_FULL_DUPLEX) ?
 754                                         ("full-duplex") : ("half-duplex"));
 755                                 break;
 756                         }
 757                 }
 758                 if (!eth_link.link_status) {
 759                         printf("Eth%u link down. Cannot start traffic.\n",
 760                                 eth_port_id);
 761                         return;
 762                 }
 763         }
 764
 765         assign_stream_to_lcores();
 766         in_test = 1;
 767
 768         RTE_LCORE_FOREACH_SLAVE(lcore_id) {
 769                 conf = &fwd_lcore_conf[lcore_id];
 770
 771                 if (!conf->nb_stream)
 772                         continue;
 773
 774                 conf->stopped = 0;
 775                 if (fwd_mode == FILE_TRANS)
 776                         rte_eal_remote_launch(start_polling_recv_file,
 777                                               conf, lcore_id);
 778                 else if (fwd_mode == IOFWD)
 779                         rte_eal_remote_launch(start_iofwd_per_lcore,
 780                                               conf, lcore_id);
 781                 else if (fwd_mode == RXONLY)
 782                         rte_eal_remote_launch(start_rxonly_per_lcore,
 783                                               conf, lcore_id);
 784                 else if (fwd_mode == TXONLY)
 785                         rte_eal_remote_launch(start_txonly_per_lcore,
 786                                               conf, lcore_id);
 787         }
 788 }
 789
 790 /* *** START FWD PARAMETERS *** */
 791 struct cmd_start_result {
 792         cmdline_fixed_string_t start;
 793 };
 794
 795 static void
 796 cmd_start_parsed(__rte_unused void *parsed_result,
 797                             __rte_unused struct cmdline *cl,
 798                             __rte_unused void *data)
 799 {
 800         start_pkt_fwd();
 801 }
 802
 803 cmdline_parse_token_string_t cmd_start_start =
 804                 TOKEN_STRING_INITIALIZER(struct cmd_start_result, start, "start");
 805
 806 cmdline_parse_inst_t cmd_start = {
 807         .f = cmd_start_parsed,
 808         .data = NULL,
 809         .help_str = "start pkt fwd between ntb and ethdev",
 810         .tokens = {
 811                 (void *)&cmd_start_start,
 812                 NULL,
 813         },
 814 };
 815
 816 /* *** STOP *** */
 817 struct cmd_stop_result {
 818         cmdline_fixed_string_t stop;
 819 };
 820
 821 static void
 822 cmd_stop_parsed(__rte_unused void *parsed_result,
 823                 __rte_unused struct cmdline *cl,
 824                 __rte_unused void *data)
 825 {
 826         struct ntb_fwd_lcore_conf *conf;
 827         uint32_t lcore_id;
 828
 829         RTE_LCORE_FOREACH_SLAVE(lcore_id) {
 830                 conf = &fwd_lcore_conf[lcore_id];
 831
 832                 if (!conf->nb_stream)
 833                         continue;
 834
 835                 if (conf->stopped)
 836                         continue;
 837
 838                 conf->stopped = 1;
 839         }
 840         printf("\nWaiting for lcores to finish...\n");
 841         rte_eal_mp_wait_lcore();
 842         in_test = 0;
 843         printf("\nDone.\n");
 844 }
 845
 846 cmdline_parse_token_string_t cmd_stop_stop =
 847                 TOKEN_STRING_INITIALIZER(struct cmd_stop_result, stop, "stop");
 848
 849 cmdline_parse_inst_t cmd_stop = {
 850         .f = cmd_stop_parsed,
 851         .data = NULL,
 852         .help_str = "stop: Stop packet forwarding",
 853         .tokens = {
 854                 (void *)&cmd_stop_stop,
 855                 NULL,
 856         },
 857 };
 858
 859 static void
 860 ntb_stats_clear(void)
 861 {
 862         int nb_ids, i;
 863         uint32_t *ids;
 864
 865         /* Clear NTB dev stats */
 866         nb_ids = rte_rawdev_xstats_names_get(dev_id, NULL, 0);
 867         if (nb_ids  < 0) {
 868                 printf("Error: Cannot get count of xstats\n");
 869                 return;
 870         }
 871         ids = malloc(sizeof(uint32_t) * nb_ids);
 872         for (i = 0; i < nb_ids; i++)
 873                 ids[i] = i;
 874         rte_rawdev_xstats_reset(dev_id, ids, nb_ids);
 875         printf("\n  statistics for NTB port %d cleared\n", dev_id);
 876
 877         /* Clear Ethdev stats if have any */
 878         if (fwd_mode == IOFWD && eth_port_id != RTE_MAX_ETHPORTS) {
 879                 rte_eth_stats_reset(eth_port_id);
 880                 printf("\n  statistics for ETH port %d cleared\n", eth_port_id);
 881         }
 882 }
 883
 884 static inline void
 885 ntb_calculate_throughput(uint16_t port) {
 886         uint64_t diff_pkts_rx, diff_pkts_tx, diff_cycles;
 887         uint64_t mpps_rx, mpps_tx;
 888         static uint64_t prev_pkts_rx[2];
 889         static uint64_t prev_pkts_tx[2];
 890         static uint64_t prev_cycles[2];
 891
 892         diff_cycles = prev_cycles[port];
 893         prev_cycles[port] = rte_rdtsc();
 894         if (diff_cycles > 0)
 895                 diff_cycles = prev_cycles[port] - diff_cycles;
 896         diff_pkts_rx = (ntb_port_stats[port].rx > prev_pkts_rx[port]) ?
 897                 (ntb_port_stats[port].rx - prev_pkts_rx[port]) : 0;
 898         diff_pkts_tx = (ntb_port_stats[port].tx > prev_pkts_tx[port]) ?
 899                 (ntb_port_stats[port].tx - prev_pkts_tx[port]) : 0;
 900         prev_pkts_rx[port] = ntb_port_stats[port].rx;
 901         prev_pkts_tx[port] = ntb_port_stats[port].tx;
 902         mpps_rx = diff_cycles > 0 ?
 903                 diff_pkts_rx * rte_get_tsc_hz() / diff_cycles : 0;
 904         mpps_tx = diff_cycles > 0 ?
 905                 diff_pkts_tx * rte_get_tsc_hz() / diff_cycles : 0;
 906         printf("  Throughput (since last show)\n");
 907         printf("  Rx-pps: %12"PRIu64"\n  Tx-pps: %12"PRIu64"\n",
 908                         mpps_rx, mpps_tx);
 909
 910 }
 911
 912 static void
 913 ntb_stats_display(void)
 914 {
 915         struct rte_rawdev_xstats_name *xstats_names;
 916         struct rte_eth_stats stats;
 917         uint64_t *values;
 918         uint32_t *ids;
 919         int nb_ids, i;
 920
 921         printf("###### statistics for NTB port %d #######\n", dev_id);
 922
 923         /* Get NTB dev stats and stats names */
 924         nb_ids = rte_rawdev_xstats_names_get(dev_id, NULL, 0);
 925         if (nb_ids  < 0) {
 926                 printf("Error: Cannot get count of xstats\n");
 927                 return;
 928         }
 929         xstats_names = malloc(sizeof(struct rte_rawdev_xstats_name) * nb_ids);
 930         if (xstats_names == NULL) {
 931                 printf("Cannot allocate memory for xstats lookup\n");
 932                 return;
 933         }
 934         if (nb_ids != rte_rawdev_xstats_names_get(
 935                         dev_id, xstats_names, nb_ids)) {
 936                 printf("Error: Cannot get xstats lookup\n");
 937                 free(xstats_names);
 938                 return;
 939         }
 940         ids = malloc(sizeof(uint32_t) * nb_ids);
 941         for (i = 0; i < nb_ids; i++)
 942                 ids[i] = i;
 943         values = malloc(sizeof(uint64_t) * nb_ids);
 944         if (nb_ids != rte_rawdev_xstats_get(dev_id, ids, values, nb_ids)) {
 945                 printf("Error: Unable to get xstats\n");
 946                 free(xstats_names);
 947                 free(values);
 948                 free(ids);
 949                 return;
 950         }
 951
 952         /* Display NTB dev stats */
 953         for (i = 0; i < nb_ids; i++)
 954                 printf("  %s: %"PRIu64"\n", xstats_names[i].name, values[i]);
 955         ntb_calculate_throughput(0);
 956
 957         /* Get Ethdev stats if have any */
 958         if (fwd_mode == IOFWD && eth_port_id != RTE_MAX_ETHPORTS) {
 959                 printf("###### statistics for ETH port %d ######\n",
 960                         eth_port_id);
 961                 rte_eth_stats_get(eth_port_id, &stats);
 962                 printf("  RX-packets: %"PRIu64"\n", stats.ipackets);
 963                 printf("  RX-bytes: %"PRIu64"\n", stats.ibytes);
 964                 printf("  RX-errors: %"PRIu64"\n", stats.ierrors);
 965                 printf("  RX-missed: %"PRIu64"\n", stats.imissed);
 966                 printf("  TX-packets: %"PRIu64"\n", stats.opackets);
 967                 printf("  TX-bytes: %"PRIu64"\n", stats.obytes);
 968                 printf("  TX-errors: %"PRIu64"\n", stats.oerrors);
 969                 ntb_calculate_throughput(1);
 970         }
 971
 972         free(xstats_names);
 973         free(values);
 974         free(ids);
 975 }
 976
 977 /* *** SHOW/CLEAR PORT STATS *** */
 978 struct cmd_stats_result {
 979         cmdline_fixed_string_t show;
 980         cmdline_fixed_string_t port;
 981         cmdline_fixed_string_t stats;
 982 };
 983
 984 static void
 985 cmd_stats_parsed(void *parsed_result,
 986                  __rte_unused struct cmdline *cl,
 987                  __rte_unused void *data)
 988 {
 989         struct cmd_stats_result *res = parsed_result;
 990         if (!strcmp(res->show, "clear"))
 991                 ntb_stats_clear();
 992         else
 993                 ntb_stats_display();
 994 }
 995
 996 cmdline_parse_token_string_t cmd_stats_show =
 997         TOKEN_STRING_INITIALIZER(struct cmd_stats_result, show, "show#clear");
 998 cmdline_parse_token_string_t cmd_stats_port =
 999         TOKEN_STRING_INITIALIZER(struct cmd_stats_result, port, "port");
1000 cmdline_parse_token_string_t cmd_stats_stats =
1001         TOKEN_STRING_INITIALIZER(struct cmd_stats_result, stats, "stats");
1002
1003
1004 cmdline_parse_inst_t cmd_stats = {
1005         .f = cmd_stats_parsed,
1006         .data = NULL,
1007         .help_str = "show|clear port stats",
1008         .tokens = {
1009                 (void *)&cmd_stats_show,
1010                 (void *)&cmd_stats_port,
1011                 (void *)&cmd_stats_stats,
1012                 NULL,
1013         },
1014 };
1015
1016 /* *** SET FORWARDING MODE *** */
1017 struct cmd_set_fwd_mode_result {
1018         cmdline_fixed_string_t set;
1019         cmdline_fixed_string_t fwd;
1020         cmdline_fixed_string_t mode;
1021 };
1022
1023 static void
1024 cmd_set_fwd_mode_parsed(__rte_unused void *parsed_result,
1025                         __rte_unused struct cmdline *cl,
1026                         __rte_unused void *data)
1027 {
1028         struct cmd_set_fwd_mode_result *res = parsed_result;
1029         int i;
1030
1031         if (in_test) {
1032                 printf("Please stop traffic first.\n");
1033                 return;
1034         }
1035
1036         for (i = 0; i < MAX_FWD_MODE; i++) {
1037                 if (!strcmp(res->mode, fwd_mode_s[i])) {
1038                         fwd_mode = i;
1039                         return;
1040                 }
1041         }
1042         printf("Invalid %s packet forwarding mode.\n", res->mode);
1043 }
1044
1045 cmdline_parse_token_string_t cmd_setfwd_set =
1046         TOKEN_STRING_INITIALIZER(struct cmd_set_fwd_mode_result, set, "set");
1047 cmdline_parse_token_string_t cmd_setfwd_fwd =
1048         TOKEN_STRING_INITIALIZER(struct cmd_set_fwd_mode_result, fwd, "fwd");
1049 cmdline_parse_token_string_t cmd_setfwd_mode =
1050         TOKEN_STRING_INITIALIZER(struct cmd_set_fwd_mode_result, mode,
1051                                 "file-trans#iofwd#txonly#rxonly");
1052
1053 cmdline_parse_inst_t cmd_set_fwd_mode = {
1054         .f = cmd_set_fwd_mode_parsed,
1055         .data = NULL,
1056         .help_str = "set forwarding mode as file-trans|rxonly|txonly|iofwd",
1057         .tokens = {
1058                 (void *)&cmd_setfwd_set,
1059                 (void *)&cmd_setfwd_fwd,
1060                 (void *)&cmd_setfwd_mode,
1061                 NULL,
1062         },
1063 };
1064
1065 /* list of instructions */
1066 cmdline_parse_ctx_t main_ctx[] = {
1067         (cmdline_parse_inst_t *)&cmd_help,
1068         (cmdline_parse_inst_t *)&cmd_send_file,
1069         (cmdline_parse_inst_t *)&cmd_start,
1070         (cmdline_parse_inst_t *)&cmd_stop,
1071         (cmdline_parse_inst_t *)&cmd_stats,
1072         (cmdline_parse_inst_t *)&cmd_set_fwd_mode,
1073         (cmdline_parse_inst_t *)&cmd_quit,
1074         NULL,
1075 };
1076
1077 /* prompt function, called from main on MASTER lcore */
1078 static void
1079 prompt(void)
1080 {
1081         struct cmdline *cl;
1082
1083         cl = cmdline_stdin_new(main_ctx, "ntb> ");
1084         if (cl == NULL)
1085                 return;
1086
1087         cmdline_interact(cl);
1088         cmdline_stdin_exit(cl);
1089 }
1090
1091 static void
1092 signal_handler(int signum)
1093 {
1094         if (signum == SIGINT || signum == SIGTERM) {
1095                 printf("\nSignal %d received, preparing to exit...\n", signum);
1096                 signal(signum, SIG_DFL);
1097                 kill(getpid(), signum);
1098         }
1099 }
1100
1101 #define OPT_BUF_SIZE         "buf-size"
1102 #define OPT_FWD_MODE         "fwd-mode"
1103 #define OPT_NB_DESC          "nb-desc"
1104 #define OPT_TXFREET          "txfreet"
1105 #define OPT_BURST            "burst"
1106 #define OPT_QP               "qp"
1107
1108 enum {
1109         /* long options mapped to a short option */
1110         OPT_NO_ZERO_COPY_NUM = 1,
1111         OPT_BUF_SIZE_NUM,
1112         OPT_FWD_MODE_NUM,
1113         OPT_NB_DESC_NUM,
1114         OPT_TXFREET_NUM,
1115         OPT_BURST_NUM,
1116         OPT_QP_NUM,
1117 };
1118
1119 static const char short_options[] =
1120         "i" /* interactive mode */
1121         ;
1122
1123 static const struct option lgopts[] = {
1124         {OPT_BUF_SIZE,     1, NULL, OPT_BUF_SIZE_NUM     },
1125         {OPT_FWD_MODE,     1, NULL, OPT_FWD_MODE_NUM     },
1126         {OPT_NB_DESC,      1, NULL, OPT_NB_DESC_NUM      },
1127         {OPT_TXFREET,      1, NULL, OPT_TXFREET_NUM      },
1128         {OPT_BURST,        1, NULL, OPT_BURST_NUM        },
1129         {OPT_QP,           1, NULL, OPT_QP_NUM           },
1130         {0,                0, NULL, 0                    }
1131 };
1132
1133 static void
1134 ntb_usage(const char *prgname)
1135 {
1136         printf("%s [EAL options] -- [options]\n"
1137                "-i: run in interactive mode.\n"
1138                "-qp=N: set number of queues as N (N > 0, default: 1).\n"
1139                "--fwd-mode=N: set fwd mode (N: file-trans | rxonly | "
1140                "txonly | iofwd, default: file-trans)\n"
1141                "--buf-size=N: set mbuf dataroom size as N (0 < N < 65535,"
1142                " default: 2048).\n"
1143                "--nb-desc=N: set number of descriptors as N (%u <= N <= %u,"
1144                " default: 1024).\n"
1145                "--txfreet=N: set tx free thresh for NTB driver as N. (N >= 0)\n"
1146                "--burst=N: set pkt burst as N (0 < N <= %u default: 32).\n",
1147                prgname, NTB_MIN_DESC_SIZE, NTB_MAX_DESC_SIZE,
1148                NTB_MAX_PKT_BURST);
1149 }
1150
1151 static void
1152 ntb_parse_args(int argc, char **argv)
1153 {
1154         char *prgname = argv[0], **argvopt = argv;
1155         int opt, opt_idx, n, i;
1156
1157         while ((opt = getopt_long(argc, argvopt, short_options,
1158                                 lgopts, &opt_idx)) != EOF) {
1159                 switch (opt) {
1160                 case 'i':
1161                         printf("Interactive-mode selected.\n");
1162                         interactive = 1;
1163                         break;
1164                 case OPT_QP_NUM:
1165                         n = atoi(optarg);
1166                         if (n > 0)
1167                                 num_queues = n;
1168                         else
1169                                 rte_exit(EXIT_FAILURE, "q must be > 0.\n");
1170                         break;
1171                 case OPT_BUF_SIZE_NUM:
1172                         n = atoi(optarg);
1173                         if (n > RTE_PKTMBUF_HEADROOM && n <= 0xFFFF)
1174                                 ntb_buf_size = n;
1175                         else
1176                                 rte_exit(EXIT_FAILURE, "buf-size must be > "
1177                                         "%u and < 65536.\n",
1178                                         RTE_PKTMBUF_HEADROOM);
1179                         break;
1180                 case OPT_FWD_MODE_NUM:
1181                         for (i = 0; i < MAX_FWD_MODE; i++) {
1182                                 if (!strcmp(optarg, fwd_mode_s[i])) {
1183                                         fwd_mode = i;
1184                                         break;
1185                                 }
1186                         }
1187                         if (i == MAX_FWD_MODE)
1188                                 rte_exit(EXIT_FAILURE, "Unsupported mode. "
1189                                 "(Should be: file-trans | rxonly | txonly "
1190                                 "| iofwd)\n");
1191                         break;
1192                 case OPT_NB_DESC_NUM:
1193                         n = atoi(optarg);
1194                         if (n >= NTB_MIN_DESC_SIZE && n <= NTB_MAX_DESC_SIZE)
1195                                 nb_desc = n;
1196                         else
1197                                 rte_exit(EXIT_FAILURE, "nb-desc must be within"
1198                                         " [%u, %u].\n", NTB_MIN_DESC_SIZE,
1199                                         NTB_MAX_DESC_SIZE);
1200                         break;
1201                 case OPT_TXFREET_NUM:
1202                         n = atoi(optarg);
1203                         if (n >= 0)
1204                                 tx_free_thresh = n;
1205                         else
1206                                 rte_exit(EXIT_FAILURE, "txfreet must be"
1207                                         " >= 0\n");
1208                         break;
1209                 case OPT_BURST_NUM:
1210                         n = atoi(optarg);
1211                         if (n > 0 && n <= NTB_MAX_PKT_BURST)
1212                                 pkt_burst = n;
1213                         else
1214                                 rte_exit(EXIT_FAILURE, "burst must be within "
1215                                         "(0, %u].\n", NTB_MAX_PKT_BURST);
1216                         break;
1217
1218                 default:
1219                         ntb_usage(prgname);
1220                         rte_exit(EXIT_FAILURE,
1221                                  "Command line is incomplete or incorrect.\n");
1222                         break;
1223                 }
1224         }
1225 }
1226
1227 static void
1228 ntb_mempool_mz_free(__rte_unused struct rte_mempool_memhdr *memhdr,
1229                 void *opaque)
1230 {
1231         const struct rte_memzone *mz = opaque;
1232         rte_memzone_free(mz);
1233 }
1234
1235 static struct rte_mempool *
1236 ntb_mbuf_pool_create(uint16_t mbuf_seg_size, uint32_t nb_mbuf,
1237                      struct ntb_dev_info ntb_info,
1238                      struct ntb_dev_config *ntb_conf,
1239                      unsigned int socket_id)
1240 {
1241         size_t mz_len, total_elt_sz, max_mz_len, left_sz;
1242         struct rte_pktmbuf_pool_private mbp_priv;
1243         char pool_name[RTE_MEMPOOL_NAMESIZE];
1244         char mz_name[RTE_MEMZONE_NAMESIZE];
1245         const struct rte_memzone *mz;
1246         struct rte_mempool *mp;
1247         uint64_t align;
1248         uint32_t mz_id;
1249         int ret;
1250
1251         snprintf(pool_name, sizeof(pool_name), "ntb_mbuf_pool_%u", socket_id);
1252         mp = rte_mempool_create_empty(pool_name, nb_mbuf,
1253                                       (mbuf_seg_size + sizeof(struct rte_mbuf)),
1254                                       MEMPOOL_CACHE_SIZE,
1255                                       sizeof(struct rte_pktmbuf_pool_private),
1256                                       socket_id, 0);
1257         if (mp == NULL)
1258                 return NULL;
1259
1260         if (rte_mempool_set_ops_byname(mp, rte_mbuf_best_mempool_ops(), NULL)) {
1261                 printf("error setting mempool handler\n");
1262                 goto fail;
1263         }
1264
1265         memset(&mbp_priv, 0, sizeof(mbp_priv));
1266         mbp_priv.mbuf_data_room_size = mbuf_seg_size;
1267         mbp_priv.mbuf_priv_size = 0;
1268         rte_pktmbuf_pool_init(mp, &mbp_priv);
1269
1270         ntb_conf->mz_list = rte_zmalloc("ntb_memzone_list",
1271                                 sizeof(struct rte_memzone *) *
1272                                 ntb_info.mw_cnt, 0);
1273         if (ntb_conf->mz_list == NULL)
1274                 goto fail;
1275
1276         /* Put ntb header on mw0. */
1277         if (ntb_info.mw_size[0] < ntb_info.ntb_hdr_size) {
1278                 printf("mw0 (size: %" PRIu64 ") is not enough for ntb hdr"
1279                        " (size: %u)\n", ntb_info.mw_size[0],
1280                        ntb_info.ntb_hdr_size);
1281                 goto fail;
1282         }
1283
1284         total_elt_sz = mp->header_size + mp->elt_size + mp->trailer_size;
1285         left_sz = total_elt_sz * nb_mbuf;
1286         for (mz_id = 0; mz_id < ntb_info.mw_cnt; mz_id++) {
1287                 /* If populated mbuf is enough, no need to reserve extra mz. */
1288                 if (!left_sz)
1289                         break;
1290                 snprintf(mz_name, sizeof(mz_name), "ntb_mw_%d", mz_id);
1291                 align = ntb_info.mw_size_align ? ntb_info.mw_size[mz_id] :
1292                         RTE_CACHE_LINE_SIZE;
1293                 /* Reserve ntb header space on memzone 0. */
1294                 max_mz_len = mz_id ? ntb_info.mw_size[mz_id] :
1295                              ntb_info.mw_size[mz_id] - ntb_info.ntb_hdr_size;
1296                 mz_len = left_sz <= max_mz_len ? left_sz :
1297                         (max_mz_len / total_elt_sz * total_elt_sz);
1298                 if (!mz_len)
1299                         continue;
1300                 mz = rte_memzone_reserve_aligned(mz_name, mz_len, socket_id,
1301                                         RTE_MEMZONE_IOVA_CONTIG, align);
1302                 if (mz == NULL) {
1303                         printf("Cannot allocate %" PRIu64 " aligned memzone"
1304                                 " %u\n", align, mz_id);
1305                         goto fail;
1306                 }
1307                 left_sz -= mz_len;
1308
1309                 /* Reserve ntb header space on memzone 0. */
1310                 if (mz_id)
1311                         ret = rte_mempool_populate_iova(mp, mz->addr, mz->iova,
1312                                         mz->len, ntb_mempool_mz_free,
1313                                         (void *)(uintptr_t)mz);
1314                 else
1315                         ret = rte_mempool_populate_iova(mp,
1316                                         (void *)((size_t)mz->addr +
1317                                         ntb_info.ntb_hdr_size),
1318                                         mz->iova + ntb_info.ntb_hdr_size,
1319                                         mz->len - ntb_info.ntb_hdr_size,
1320                                         ntb_mempool_mz_free,
1321                                         (void *)(uintptr_t)mz);
1322                 if (ret <= 0) {
1323                         rte_memzone_free(mz);
1324                         rte_mempool_free(mp);
1325                         return NULL;
1326                 }
1327
1328                 ntb_conf->mz_list[mz_id] = mz;
1329         }
1330         if (left_sz) {
1331                 printf("mw space is not enough for mempool.\n");
1332                 goto fail;
1333         }
1334
1335         ntb_conf->mz_num = mz_id;
1336         rte_mempool_obj_iter(mp, rte_pktmbuf_init, NULL);
1337
1338         return mp;
1339 fail:
1340         rte_mempool_free(mp);
1341         return NULL;
1342 }
1343
1344 int
1345 main(int argc, char **argv)
1346 {
1347         struct rte_eth_conf eth_pconf = eth_port_conf;
1348         struct rte_rawdev_info ntb_rawdev_conf;
1349         struct rte_rawdev_info ntb_rawdev_info;
1350         struct rte_eth_dev_info ethdev_info;
1351         struct rte_eth_rxconf eth_rx_conf;
1352         struct rte_eth_txconf eth_tx_conf;
1353         struct ntb_queue_conf ntb_q_conf;
1354         struct ntb_dev_config ntb_conf;
1355         struct ntb_dev_info ntb_info;
1356         uint64_t ntb_link_status;
1357         uint32_t nb_mbuf;
1358         int ret, i;
1359
1360         signal(SIGINT, signal_handler);
1361         signal(SIGTERM, signal_handler);
1362
1363         ret = rte_eal_init(argc, argv);
1364         if (ret < 0)
1365                 rte_exit(EXIT_FAILURE, "Error with EAL initialization.\n");
1366
1367         if (rte_lcore_count() < 2)
1368                 rte_exit(EXIT_FAILURE, "Need at least 2 cores\n");
1369
1370         /* Find 1st ntb rawdev. */
1371         for (i = 0; i < RTE_RAWDEV_MAX_DEVS; i++)
1372                 if (rte_rawdevs[i].driver_name &&
1373                     (strncmp(rte_rawdevs[i].driver_name, "raw_ntb",
1374                     NTB_DRV_NAME_LEN) == 0) && (rte_rawdevs[i].attached == 1))
1375                         break;
1376
1377         if (i == RTE_RAWDEV_MAX_DEVS)
1378                 rte_exit(EXIT_FAILURE, "Cannot find any ntb device.\n");
1379
1380         dev_id = i;
1381
1382         argc -= ret;
1383         argv += ret;
1384
1385         ntb_parse_args(argc, argv);
1386
1387         rte_rawdev_set_attr(dev_id, NTB_QUEUE_SZ_NAME, nb_desc);
1388         printf("Set queue size as %u.\n", nb_desc);
1389         rte_rawdev_set_attr(dev_id, NTB_QUEUE_NUM_NAME, num_queues);
1390         printf("Set queue number as %u.\n", num_queues);
1391         ntb_rawdev_info.dev_private = (rte_rawdev_obj_t)(&ntb_info);
1392         rte_rawdev_info_get(dev_id, &ntb_rawdev_info);
1393
1394         nb_mbuf = nb_desc * num_queues * 2 * 2 + rte_lcore_count() *
1395                   MEMPOOL_CACHE_SIZE;
1396         mbuf_pool = ntb_mbuf_pool_create(ntb_buf_size, nb_mbuf, ntb_info,
1397                                          &ntb_conf, rte_socket_id());
1398         if (mbuf_pool == NULL)
1399                 rte_exit(EXIT_FAILURE, "Cannot create mbuf pool.\n");
1400
1401         ntb_conf.num_queues = num_queues;
1402         ntb_conf.queue_size = nb_desc;
1403         ntb_rawdev_conf.dev_private = (rte_rawdev_obj_t)(&ntb_conf);
1404         ret = rte_rawdev_configure(dev_id, &ntb_rawdev_conf);
1405         if (ret)
1406                 rte_exit(EXIT_FAILURE, "Can't config ntb dev: err=%d, "
1407                         "port=%u\n", ret, dev_id);
1408
1409         ntb_q_conf.tx_free_thresh = tx_free_thresh;
1410         ntb_q_conf.nb_desc = nb_desc;
1411         ntb_q_conf.rx_mp = mbuf_pool;
1412         for (i = 0; i < num_queues; i++) {
1413                 /* Setup rawdev queue */
1414                 ret = rte_rawdev_queue_setup(dev_id, i, &ntb_q_conf);
1415                 if (ret < 0)
1416                         rte_exit(EXIT_FAILURE,
1417                                 "Failed to setup ntb queue %u.\n", i);
1418         }
1419
1420         /* Waiting for peer dev up at most 100s.*/
1421         printf("Checking ntb link status...\n");
1422         for (i = 0; i < 1000; i++) {
1423                 rte_rawdev_get_attr(dev_id, NTB_LINK_STATUS_NAME,
1424                                     &ntb_link_status);
1425                 if (ntb_link_status) {
1426                         printf("Peer dev ready, ntb link up.\n");
1427                         break;
1428                 }
1429                 rte_delay_ms(100);
1430         }
1431         rte_rawdev_get_attr(dev_id, NTB_LINK_STATUS_NAME, &ntb_link_status);
1432         if (ntb_link_status == 0)
1433                 printf("Expire 100s. Link is not up. Please restart app.\n");
1434
1435         ret = rte_rawdev_start(dev_id);
1436         if (ret < 0)
1437                 rte_exit(EXIT_FAILURE, "rte_rawdev_start: err=%d, port=%u\n",
1438                         ret, dev_id);
1439
1440         /* Find 1st ethdev */
1441         eth_port_id = rte_eth_find_next(0);
1442
1443         if (eth_port_id < RTE_MAX_ETHPORTS) {
1444                 rte_eth_dev_info_get(eth_port_id, &ethdev_info);
1445                 eth_pconf.rx_adv_conf.rss_conf.rss_hf &=
1446                                 ethdev_info.flow_type_rss_offloads;
1447                 ret = rte_eth_dev_configure(eth_port_id, num_queues,
1448                                             num_queues, &eth_pconf);
1449                 if (ret)
1450                         rte_exit(EXIT_FAILURE, "Can't config ethdev: err=%d, "
1451                                 "port=%u\n", ret, eth_port_id);
1452                 eth_rx_conf = ethdev_info.default_rxconf;
1453                 eth_rx_conf.offloads = eth_pconf.rxmode.offloads;
1454                 eth_tx_conf = ethdev_info.default_txconf;
1455                 eth_tx_conf.offloads = eth_pconf.txmode.offloads;
1456
1457                 /* Setup ethdev queue if ethdev exists */
1458                 for (i = 0; i < num_queues; i++) {
1459                         ret = rte_eth_rx_queue_setup(eth_port_id, i, nb_desc,
1460                                         rte_eth_dev_socket_id(eth_port_id),
1461                                         &eth_rx_conf, mbuf_pool);
1462                         if (ret < 0)
1463                                 rte_exit(EXIT_FAILURE,
1464                                         "Failed to setup eth rxq %u.\n", i);
1465                         ret = rte_eth_tx_queue_setup(eth_port_id, i, nb_desc,
1466                                         rte_eth_dev_socket_id(eth_port_id),
1467                                         &eth_tx_conf);
1468                         if (ret < 0)
1469                                 rte_exit(EXIT_FAILURE,
1470                                         "Failed to setup eth txq %u.\n", i);
1471                 }
1472
1473                 ret = rte_eth_dev_start(eth_port_id);
1474                 if (ret < 0)
1475                         rte_exit(EXIT_FAILURE, "rte_eth_dev_start: err=%d, "
1476                                 "port=%u\n", ret, eth_port_id);
1477         }
1478
1479         /* initialize port stats */
1480         memset(&ntb_port_stats, 0, sizeof(ntb_port_stats));
1481
1482         /* Set default fwd mode if user doesn't set it. */
1483         if (fwd_mode == MAX_FWD_MODE && eth_port_id < RTE_MAX_ETHPORTS) {
1484                 printf("Set default fwd mode as iofwd.\n");
1485                 fwd_mode = IOFWD;
1486         }
1487         if (fwd_mode == MAX_FWD_MODE) {
1488                 printf("Set default fwd mode as file-trans.\n");
1489                 fwd_mode = FILE_TRANS;
1490         }
1491
1492         if (interactive) {
1493                 sleep(1);
1494                 prompt();
1495         } else {
1496                 start_pkt_fwd();
1497         }
1498
1499         return 0;
1500 }