examples/ntb/ntb_fwd.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(c) 2019 Intel Corporation
   3  */
   4 #include <stdint.h>
   5 #include <stdio.h>
   6 #include <inttypes.h>
   7 #include <unistd.h>
   8 #include <signal.h>
   9 #include <string.h>
  10 #include <getopt.h>
  11
  12 #include <cmdline_parse_string.h>
  13 #include <cmdline_socket.h>
  14 #include <cmdline.h>
  15 #include <rte_common.h>
  16 #include <rte_rawdev.h>
  17 #include <rte_ethdev.h>
  18 #include <rte_malloc.h>
  19 #include <rte_lcore.h>
  20 #include <rte_cycles.h>
  21 #include <rte_pmd_ntb.h>
  22
  23 /* Per-port statistics struct */
  24 struct ntb_port_statistics {
  25         uint64_t tx;
  26         uint64_t rx;
  27 } __rte_cache_aligned;
  28 /* Port 0: NTB dev, Port 1: ethdev when iofwd. */
  29 struct ntb_port_statistics ntb_port_stats[2];
  30
  31 struct ntb_fwd_stream {
  32         uint16_t tx_port;
  33         uint16_t rx_port;
  34         uint16_t qp_id;
  35         uint8_t tx_ntb;  /* If ntb device is tx port. */
  36 };
  37
  38 struct ntb_fwd_lcore_conf {
  39         uint16_t stream_id;
  40         uint16_t nb_stream;
  41         uint8_t stopped;
  42 };
  43
  44 enum ntb_fwd_mode {
  45         FILE_TRANS = 0,
  46         RXONLY,
  47         TXONLY,
  48         IOFWD,
  49         MAX_FWD_MODE,
  50 };
  51 static const char *const fwd_mode_s[] = {
  52         "file-trans",
  53         "rxonly",
  54         "txonly",
  55         "iofwd",
  56         NULL,
  57 };
  58 static enum ntb_fwd_mode fwd_mode = MAX_FWD_MODE;
  59
  60 static struct ntb_fwd_lcore_conf fwd_lcore_conf[RTE_MAX_LCORE];
  61 static struct ntb_fwd_stream *fwd_streams;
  62
  63 static struct rte_mempool *mbuf_pool;
  64
  65 #define NTB_DRV_NAME_LEN 7
  66 #define MEMPOOL_CACHE_SIZE 256
  67
  68 static uint8_t in_test;
  69 static uint8_t interactive = 1;
  70 static uint16_t eth_port_id = RTE_MAX_ETHPORTS;
  71 static uint16_t dev_id;
  72
  73 /* Number of queues, default set as 1 */
  74 static uint16_t num_queues = 1;
  75 static uint16_t ntb_buf_size = RTE_MBUF_DEFAULT_BUF_SIZE;
  76
  77 /* Configurable number of descriptors */
  78 #define NTB_DEFAULT_NUM_DESCS 1024
  79 static uint16_t nb_desc = NTB_DEFAULT_NUM_DESCS;
  80
  81 static uint16_t tx_free_thresh;
  82
  83 #define NTB_MAX_PKT_BURST 32
  84 #define NTB_DFLT_PKT_BURST 32
  85 static uint16_t pkt_burst = NTB_DFLT_PKT_BURST;
  86
  87 #define BURST_TX_RETRIES 64
  88
  89 static struct rte_eth_conf eth_port_conf = {
  90         .rxmode = {
  91                 .mq_mode = ETH_MQ_RX_RSS,
  92                 .split_hdr_size = 0,
  93         },
  94         .rx_adv_conf = {
  95                 .rss_conf = {
  96                         .rss_key = NULL,
  97                         .rss_hf = ETH_RSS_IP,
  98                 },
  99         },
 100         .txmode = {
 101                 .mq_mode = ETH_MQ_TX_NONE,
 102         },
 103 };
 104
 105 /* *** Help command with introduction. *** */
 106 struct cmd_help_result {
 107         cmdline_fixed_string_t help;
 108 };
 109
 110 static void
 111 cmd_help_parsed(__attribute__((unused)) void *parsed_result,
 112                 struct cmdline *cl,
 113                 __attribute__((unused)) void *data)
 114 {
 115         cmdline_printf(
 116                 cl,
 117                 "\n"
 118                 "The following commands are currently available:\n\n"
 119                 "Control:\n"
 120                 "    quit                                      :"
 121                 " Quit the application.\n"
 122                 "\nTransmission:\n"
 123                 "    send [path]                               :"
 124                 " Send [path] file. Only take effect in file-trans mode\n"
 125                 "    start                                     :"
 126                 " Start transmissions.\n"
 127                 "    stop                                      :"
 128                 " Stop transmissions.\n"
 129                 "    clear/show port stats                     :"
 130                 " Clear/show port stats.\n"
 131                 "    set fwd file-trans/rxonly/txonly/iofwd    :"
 132                 " Set packet forwarding mode.\n"
 133         );
 134
 135 }
 136
 137 cmdline_parse_token_string_t cmd_help_help =
 138         TOKEN_STRING_INITIALIZER(struct cmd_help_result, help, "help");
 139
 140 cmdline_parse_inst_t cmd_help = {
 141         .f = cmd_help_parsed,
 142         .data = NULL,
 143         .help_str = "show help",
 144         .tokens = {
 145                 (void *)&cmd_help_help,
 146                 NULL,
 147         },
 148 };
 149
 150 /* *** QUIT *** */
 151 struct cmd_quit_result {
 152         cmdline_fixed_string_t quit;
 153 };
 154
 155 static void
 156 cmd_quit_parsed(__attribute__((unused)) void *parsed_result,
 157                 struct cmdline *cl,
 158                 __attribute__((unused)) void *data)
 159 {
 160         struct ntb_fwd_lcore_conf *conf;
 161         uint32_t lcore_id;
 162
 163         /* Stop transmission first. */
 164         RTE_LCORE_FOREACH_SLAVE(lcore_id) {
 165                 conf = &fwd_lcore_conf[lcore_id];
 166
 167                 if (!conf->nb_stream)
 168                         continue;
 169
 170                 if (conf->stopped)
 171                         continue;
 172
 173                 conf->stopped = 1;
 174         }
 175         printf("\nWaiting for lcores to finish...\n");
 176         rte_eal_mp_wait_lcore();
 177         in_test = 0;
 178
 179         /* Stop traffic and Close port. */
 180         rte_rawdev_stop(dev_id);
 181         rte_rawdev_close(dev_id);
 182         if (eth_port_id < RTE_MAX_ETHPORTS && fwd_mode == IOFWD) {
 183                 rte_eth_dev_stop(eth_port_id);
 184                 rte_eth_dev_close(eth_port_id);
 185         }
 186
 187         cmdline_quit(cl);
 188 }
 189
 190 cmdline_parse_token_string_t cmd_quit_quit =
 191                 TOKEN_STRING_INITIALIZER(struct cmd_quit_result, quit, "quit");
 192
 193 cmdline_parse_inst_t cmd_quit = {
 194         .f = cmd_quit_parsed,
 195         .data = NULL,
 196         .help_str = "exit application",
 197         .tokens = {
 198                 (void *)&cmd_quit_quit,
 199                 NULL,
 200         },
 201 };
 202
 203 /* *** SEND FILE PARAMETERS *** */
 204 struct cmd_sendfile_result {
 205         cmdline_fixed_string_t send_string;
 206         char filepath[];
 207 };
 208
 209 static void
 210 cmd_sendfile_parsed(void *parsed_result,
 211                     __attribute__((unused)) struct cmdline *cl,
 212                     __attribute__((unused)) void *data)
 213 {
 214         struct cmd_sendfile_result *res = parsed_result;
 215         struct rte_rawdev_buf *pkts_send[NTB_MAX_PKT_BURST];
 216         struct rte_mbuf *mbuf_send[NTB_MAX_PKT_BURST];
 217         uint64_t size, count, i, nb_burst;
 218         uint16_t nb_tx, buf_size;
 219         unsigned int nb_pkt;
 220         size_t queue_id = 0;
 221         uint16_t retry = 0;
 222         uint32_t val;
 223         FILE *file;
 224
 225         if (num_queues != 1) {
 226                 printf("File transmission only supports 1 queue.\n");
 227                 num_queues = 1;
 228         }
 229
 230         file = fopen(res->filepath, "r");
 231         if (file == NULL) {
 232                 printf("Fail to open the file.\n");
 233                 return;
 234         }
 235
 236         if (fseek(file, 0, SEEK_END) < 0) {
 237                 printf("Fail to get file size.\n");
 238                 fclose(file);
 239                 return;
 240         }
 241         size = ftell(file);
 242         if (fseek(file, 0, SEEK_SET) < 0) {
 243                 printf("Fail to get file size.\n");
 244                 fclose(file);
 245                 return;
 246         }
 247
 248         /* Tell remote about the file size. */
 249         val = size >> 32;
 250         rte_rawdev_set_attr(dev_id, "spad_user_0", val);
 251         val = size;
 252         rte_rawdev_set_attr(dev_id, "spad_user_1", val);
 253         printf("Sending file, size is %"PRIu64"\n", size);
 254
 255         for (i = 0; i < NTB_MAX_PKT_BURST; i++)
 256                 pkts_send[i] = (struct rte_rawdev_buf *)
 257                                 malloc(sizeof(struct rte_rawdev_buf));
 258
 259         buf_size = ntb_buf_size - RTE_PKTMBUF_HEADROOM;
 260         count = (size + buf_size - 1) / buf_size;
 261         nb_burst = (count + pkt_burst - 1) / pkt_burst;
 262
 263         for (i = 0; i < nb_burst; i++) {
 264                 val = RTE_MIN(count, pkt_burst);
 265                 if (rte_mempool_get_bulk(mbuf_pool, (void **)mbuf_send,
 266                                         val) == 0) {
 267                         for (nb_pkt = 0; nb_pkt < val; nb_pkt++) {
 268                                 mbuf_send[nb_pkt]->port = dev_id;
 269                                 mbuf_send[nb_pkt]->data_len =
 270                                 fread(rte_pktmbuf_mtod(mbuf_send[nb_pkt],
 271                                         void *), 1, buf_size, file);
 272                                 mbuf_send[nb_pkt]->pkt_len =
 273                                         mbuf_send[nb_pkt]->data_len;
 274                                 pkts_send[nb_pkt]->buf_addr = mbuf_send[nb_pkt];
 275                         }
 276                 } else {
 277                         for (nb_pkt = 0; nb_pkt < val; nb_pkt++) {
 278                                 mbuf_send[nb_pkt] =
 279                                         rte_mbuf_raw_alloc(mbuf_pool);
 280                                 if (mbuf_send[nb_pkt] == NULL)
 281                                         break;
 282                                 mbuf_send[nb_pkt]->port = dev_id;
 283                                 mbuf_send[nb_pkt]->data_len =
 284                                 fread(rte_pktmbuf_mtod(mbuf_send[nb_pkt],
 285                                         void *), 1, buf_size, file);
 286                                 mbuf_send[nb_pkt]->pkt_len =
 287                                         mbuf_send[nb_pkt]->data_len;
 288                                 pkts_send[nb_pkt]->buf_addr = mbuf_send[nb_pkt];
 289                         }
 290                 }
 291
 292                 nb_tx = rte_rawdev_enqueue_buffers(dev_id, pkts_send, nb_pkt,
 293                                                    (void *)queue_id);
 294                 while (nb_tx != nb_pkt && retry < BURST_TX_RETRIES) {
 295                         rte_delay_us(1);
 296                         nb_tx += rte_rawdev_enqueue_buffers(dev_id,
 297                                 &pkts_send[nb_tx], nb_pkt - nb_tx,
 298                                 (void *)queue_id);
 299                 }
 300                 count -= nb_pkt;
 301         }
 302         /* Clear register after file sending done. */
 303         rte_rawdev_set_attr(dev_id, "spad_user_0", 0);
 304         rte_rawdev_set_attr(dev_id, "spad_user_1", 0);
 305         printf("Done sending file.\n");
 306
 307         for (i = 0; i < NTB_MAX_PKT_BURST; i++)
 308                 free(pkts_send[i]);
 309         fclose(file);
 310 }
 311
 312 cmdline_parse_token_string_t cmd_send_file_send =
 313         TOKEN_STRING_INITIALIZER(struct cmd_sendfile_result, send_string,
 314                                  "send");
 315 cmdline_parse_token_string_t cmd_send_file_filepath =
 316         TOKEN_STRING_INITIALIZER(struct cmd_sendfile_result, filepath, NULL);
 317
 318
 319 cmdline_parse_inst_t cmd_send_file = {
 320         .f = cmd_sendfile_parsed,
 321         .data = NULL,
 322         .help_str = "send <file_path>",
 323         .tokens = {
 324                 (void *)&cmd_send_file_send,
 325                 (void *)&cmd_send_file_filepath,
 326                 NULL,
 327         },
 328 };
 329
 330 #define RECV_FILE_LEN 30
 331 static int
 332 start_polling_recv_file(void *param)
 333 {
 334         struct rte_rawdev_buf *pkts_recv[NTB_MAX_PKT_BURST];
 335         struct ntb_fwd_lcore_conf *conf = param;
 336         struct rte_mbuf *mbuf;
 337         char filepath[RECV_FILE_LEN];
 338         uint64_t val, size, file_len;
 339         uint16_t nb_rx, i, file_no;
 340         size_t queue_id = 0;
 341         FILE *file;
 342
 343         for (i = 0; i < NTB_MAX_PKT_BURST; i++)
 344                 pkts_recv[i] = (struct rte_rawdev_buf *)
 345                                 malloc(sizeof(struct rte_rawdev_buf));
 346
 347         file_no = 0;
 348         while (!conf->stopped) {
 349                 snprintf(filepath, RECV_FILE_LEN, "ntb_recv_file%d", file_no);
 350                 file = fopen(filepath, "w");
 351                 if (file == NULL) {
 352                         printf("Fail to open the file.\n");
 353                         return -EINVAL;
 354                 }
 355
 356                 rte_rawdev_get_attr(dev_id, "spad_user_0", &val);
 357                 size = val << 32;
 358                 rte_rawdev_get_attr(dev_id, "spad_user_1", &val);
 359                 size |= val;
 360
 361                 if (!size) {
 362                         fclose(file);
 363                         continue;
 364                 }
 365
 366                 file_len = 0;
 367                 nb_rx = NTB_MAX_PKT_BURST;
 368                 while (file_len < size && !conf->stopped) {
 369                         nb_rx = rte_rawdev_dequeue_buffers(dev_id, pkts_recv,
 370                                                 pkt_burst, (void *)queue_id);
 371                         ntb_port_stats[0].rx += nb_rx;
 372                         for (i = 0; i < nb_rx; i++) {
 373                                 mbuf = pkts_recv[i]->buf_addr;
 374                                 fwrite(rte_pktmbuf_mtod(mbuf, void *), 1,
 375                                         mbuf->data_len, file);
 376                                 file_len += mbuf->data_len;
 377                                 rte_pktmbuf_free(mbuf);
 378                                 pkts_recv[i]->buf_addr = NULL;
 379                         }
 380                 }
 381
 382                 printf("Received file (size: %" PRIu64 ") from peer to %s.\n",
 383                         size, filepath);
 384                 fclose(file);
 385                 file_no++;
 386         }
 387
 388         for (i = 0; i < NTB_MAX_PKT_BURST; i++)
 389                 free(pkts_recv[i]);
 390         return 0;
 391 }
 392
 393 static int
 394 start_iofwd_per_lcore(void *param)
 395 {
 396         struct rte_rawdev_buf *ntb_buf[NTB_MAX_PKT_BURST];
 397         struct rte_mbuf *pkts_burst[NTB_MAX_PKT_BURST];
 398         struct ntb_fwd_lcore_conf *conf = param;
 399         struct ntb_fwd_stream fs;
 400         uint16_t nb_rx, nb_tx;
 401         int i, j;
 402
 403         for (i = 0; i < NTB_MAX_PKT_BURST; i++)
 404                 ntb_buf[i] = (struct rte_rawdev_buf *)
 405                              malloc(sizeof(struct rte_rawdev_buf));
 406
 407         while (!conf->stopped) {
 408                 for (i = 0; i < conf->nb_stream; i++) {
 409                         fs = fwd_streams[conf->stream_id + i];
 410                         if (fs.tx_ntb) {
 411                                 nb_rx = rte_eth_rx_burst(fs.rx_port,
 412                                                 fs.qp_id, pkts_burst,
 413                                                 pkt_burst);
 414                                 if (unlikely(nb_rx == 0))
 415                                         continue;
 416                                 for (j = 0; j < nb_rx; j++)
 417                                         ntb_buf[j]->buf_addr = pkts_burst[j];
 418                                 nb_tx =
 419                                 rte_rawdev_enqueue_buffers(fs.tx_port,
 420                                                 ntb_buf, nb_rx,
 421                                                 (void *)(size_t)fs.qp_id);
 422                                 ntb_port_stats[0].tx += nb_tx;
 423                                 ntb_port_stats[1].rx += nb_rx;
 424                         } else {
 425                                 nb_rx =
 426                                 rte_rawdev_dequeue_buffers(fs.rx_port,
 427                                                 ntb_buf, pkt_burst,
 428                                                 (void *)(size_t)fs.qp_id);
 429                                 if (unlikely(nb_rx == 0))
 430                                         continue;
 431                                 for (j = 0; j < nb_rx; j++)
 432                                         pkts_burst[j] = ntb_buf[j]->buf_addr;
 433                                 nb_tx = rte_eth_tx_burst(fs.tx_port,
 434                                         fs.qp_id, pkts_burst, nb_rx);
 435                                 ntb_port_stats[1].tx += nb_tx;
 436                                 ntb_port_stats[0].rx += nb_rx;
 437                         }
 438                         if (unlikely(nb_tx < nb_rx)) {
 439                                 do {
 440                                         rte_pktmbuf_free(pkts_burst[nb_tx]);
 441                                 } while (++nb_tx < nb_rx);
 442                         }
 443                 }
 444         }
 445
 446         for (i = 0; i < NTB_MAX_PKT_BURST; i++)
 447                 free(ntb_buf[i]);
 448
 449         return 0;
 450 }
 451
 452 static int
 453 start_rxonly_per_lcore(void *param)
 454 {
 455         struct rte_rawdev_buf *ntb_buf[NTB_MAX_PKT_BURST];
 456         struct ntb_fwd_lcore_conf *conf = param;
 457         struct ntb_fwd_stream fs;
 458         uint16_t nb_rx;
 459         int i, j;
 460
 461         for (i = 0; i < NTB_MAX_PKT_BURST; i++)
 462                 ntb_buf[i] = (struct rte_rawdev_buf *)
 463                              malloc(sizeof(struct rte_rawdev_buf));
 464
 465         while (!conf->stopped) {
 466                 for (i = 0; i < conf->nb_stream; i++) {
 467                         fs = fwd_streams[conf->stream_id + i];
 468                         nb_rx = rte_rawdev_dequeue_buffers(fs.rx_port,
 469                                 ntb_buf, pkt_burst, (void *)(size_t)fs.qp_id);
 470                         if (unlikely(nb_rx == 0))
 471                                 continue;
 472                         ntb_port_stats[0].rx += nb_rx;
 473
 474                         for (j = 0; j < nb_rx; j++)
 475                                 rte_pktmbuf_free(ntb_buf[j]->buf_addr);
 476                 }
 477         }
 478
 479         for (i = 0; i < NTB_MAX_PKT_BURST; i++)
 480                 free(ntb_buf[i]);
 481
 482         return 0;
 483 }
 484
 485
 486 static int
 487 start_txonly_per_lcore(void *param)
 488 {
 489         struct rte_rawdev_buf *ntb_buf[NTB_MAX_PKT_BURST];
 490         struct rte_mbuf *pkts_burst[NTB_MAX_PKT_BURST];
 491         struct ntb_fwd_lcore_conf *conf = param;
 492         struct ntb_fwd_stream fs;
 493         uint16_t nb_pkt, nb_tx;
 494         int i;
 495
 496         for (i = 0; i < NTB_MAX_PKT_BURST; i++)
 497                 ntb_buf[i] = (struct rte_rawdev_buf *)
 498                              malloc(sizeof(struct rte_rawdev_buf));
 499
 500         while (!conf->stopped) {
 501                 for (i = 0; i < conf->nb_stream; i++) {
 502                         fs = fwd_streams[conf->stream_id + i];
 503                         if (rte_mempool_get_bulk(mbuf_pool, (void **)pkts_burst,
 504                                   pkt_burst) == 0) {
 505                                 for (nb_pkt = 0; nb_pkt < pkt_burst; nb_pkt++) {
 506                                         pkts_burst[nb_pkt]->port = dev_id;
 507                                         pkts_burst[nb_pkt]->data_len =
 508                                                 pkts_burst[nb_pkt]->buf_len -
 509                                                 RTE_PKTMBUF_HEADROOM;
 510                                         pkts_burst[nb_pkt]->pkt_len =
 511                                                 pkts_burst[nb_pkt]->data_len;
 512                                         ntb_buf[nb_pkt]->buf_addr =
 513                                                 pkts_burst[nb_pkt];
 514                                 }
 515                         } else {
 516                                 for (nb_pkt = 0; nb_pkt < pkt_burst; nb_pkt++) {
 517                                         pkts_burst[nb_pkt] =
 518                                                 rte_pktmbuf_alloc(mbuf_pool);
 519                                         if (pkts_burst[nb_pkt] == NULL)
 520                                                 break;
 521                                         pkts_burst[nb_pkt]->port = dev_id;
 522                                         pkts_burst[nb_pkt]->data_len =
 523                                                 pkts_burst[nb_pkt]->buf_len -
 524                                                 RTE_PKTMBUF_HEADROOM;
 525                                         pkts_burst[nb_pkt]->pkt_len =
 526                                                 pkts_burst[nb_pkt]->data_len;
 527                                         ntb_buf[nb_pkt]->buf_addr =
 528                                                 pkts_burst[nb_pkt];
 529                                 }
 530                         }
 531                         nb_tx = rte_rawdev_enqueue_buffers(fs.tx_port,
 532                                 ntb_buf, nb_pkt, (void *)(size_t)fs.qp_id);
 533                         ntb_port_stats[0].tx += nb_tx;
 534                         if (unlikely(nb_tx < nb_pkt)) {
 535                                 do {
 536                                         rte_pktmbuf_free(
 537                                                 ntb_buf[nb_tx]->buf_addr);
 538                                 } while (++nb_tx < nb_pkt);
 539                         }
 540                 }
 541         }
 542
 543         for (i = 0; i < NTB_MAX_PKT_BURST; i++)
 544                 free(ntb_buf[i]);
 545
 546         return 0;
 547 }
 548
 549 static int
 550 ntb_fwd_config_setup(void)
 551 {
 552         uint16_t i;
 553
 554         /* Make sure iofwd has valid ethdev. */
 555         if (fwd_mode == IOFWD && eth_port_id >= RTE_MAX_ETHPORTS) {
 556                 printf("No ethdev, cannot be in iofwd mode.");
 557                 return -EINVAL;
 558         }
 559
 560         if (fwd_mode == IOFWD) {
 561                 fwd_streams = rte_zmalloc("ntb_fwd: fwd_streams",
 562                         sizeof(struct ntb_fwd_stream) * num_queues * 2,
 563                         RTE_CACHE_LINE_SIZE);
 564                 for (i = 0; i < num_queues; i++) {
 565                         fwd_streams[i * 2].qp_id = i;
 566                         fwd_streams[i * 2].tx_port = dev_id;
 567                         fwd_streams[i * 2].rx_port = eth_port_id;
 568                         fwd_streams[i * 2].tx_ntb = 1;
 569
 570                         fwd_streams[i * 2 + 1].qp_id = i;
 571                         fwd_streams[i * 2 + 1].tx_port = eth_port_id;
 572                         fwd_streams[i * 2 + 1].rx_port = dev_id;
 573                         fwd_streams[i * 2 + 1].tx_ntb = 0;
 574                 }
 575                 return 0;
 576         }
 577
 578         if (fwd_mode == RXONLY || fwd_mode == FILE_TRANS) {
 579                 /* Only support 1 queue in file-trans for in order. */
 580                 if (fwd_mode == FILE_TRANS)
 581                         num_queues = 1;
 582
 583                 fwd_streams = rte_zmalloc("ntb_fwd: fwd_streams",
 584                                 sizeof(struct ntb_fwd_stream) * num_queues,
 585                                 RTE_CACHE_LINE_SIZE);
 586                 for (i = 0; i < num_queues; i++) {
 587                         fwd_streams[i].qp_id = i;
 588                         fwd_streams[i].tx_port = RTE_MAX_ETHPORTS;
 589                         fwd_streams[i].rx_port = dev_id;
 590                         fwd_streams[i].tx_ntb = 0;
 591                 }
 592                 return 0;
 593         }
 594
 595         if (fwd_mode == TXONLY) {
 596                 fwd_streams = rte_zmalloc("ntb_fwd: fwd_streams",
 597                                 sizeof(struct ntb_fwd_stream) * num_queues,
 598                                 RTE_CACHE_LINE_SIZE);
 599                 for (i = 0; i < num_queues; i++) {
 600                         fwd_streams[i].qp_id = i;
 601                         fwd_streams[i].tx_port = dev_id;
 602                         fwd_streams[i].rx_port = RTE_MAX_ETHPORTS;
 603                         fwd_streams[i].tx_ntb = 1;
 604                 }
 605         }
 606         return 0;
 607 }
 608
 609 static void
 610 assign_stream_to_lcores(void)
 611 {
 612         struct ntb_fwd_lcore_conf *conf;
 613         struct ntb_fwd_stream *fs;
 614         uint16_t nb_streams, sm_per_lcore, sm_id, i;
 615         uint32_t lcore_id;
 616         uint8_t lcore_num, nb_extra;
 617
 618         lcore_num = rte_lcore_count();
 619         /* Exclude master core */
 620         lcore_num--;
 621
 622         nb_streams = (fwd_mode == IOFWD) ? num_queues * 2 : num_queues;
 623
 624         sm_per_lcore = nb_streams / lcore_num;
 625         nb_extra = nb_streams % lcore_num;
 626         sm_id = 0;
 627         i = 0;
 628
 629         RTE_LCORE_FOREACH_SLAVE(lcore_id) {
 630                 conf = &fwd_lcore_conf[lcore_id];
 631
 632                 if (i < nb_extra) {
 633                         conf->nb_stream = sm_per_lcore + 1;
 634                         conf->stream_id = sm_id;
 635                         sm_id = sm_id + sm_per_lcore + 1;
 636                 } else {
 637                         conf->nb_stream = sm_per_lcore;
 638                         conf->stream_id = sm_id;
 639                         sm_id = sm_id + sm_per_lcore;
 640                 }
 641
 642                 i++;
 643                 if (sm_id >= nb_streams)
 644                         break;
 645         }
 646
 647         /* Print packet forwading config. */
 648         RTE_LCORE_FOREACH_SLAVE(lcore_id) {
 649                 conf = &fwd_lcore_conf[lcore_id];
 650
 651                 if (!conf->nb_stream)
 652                         continue;
 653
 654                 printf("Streams on Lcore %u :\n", lcore_id);
 655                 for (i = 0; i < conf->nb_stream; i++) {
 656                         fs = &fwd_streams[conf->stream_id + i];
 657                         if (fwd_mode == IOFWD)
 658                                 printf(" + Stream %u : %s%u RX -> %s%u TX,"
 659                                         " Q=%u\n", conf->stream_id + i,
 660                                         fs->tx_ntb ? "Eth" : "NTB", fs->rx_port,
 661                                         fs->tx_ntb ? "NTB" : "Eth", fs->tx_port,
 662                                         fs->qp_id);
 663                         if (fwd_mode == FILE_TRANS || fwd_mode == RXONLY)
 664                                 printf(" + Stream %u : %s%u RX only\n",
 665                                         conf->stream_id, "NTB", fs->rx_port);
 666                         if (fwd_mode == TXONLY)
 667                                 printf(" + Stream %u : %s%u TX only\n",
 668                                         conf->stream_id, "NTB", fs->tx_port);
 669                 }
 670         }
 671 }
 672
 673 static void
 674 start_pkt_fwd(void)
 675 {
 676         struct ntb_fwd_lcore_conf *conf;
 677         struct rte_eth_link eth_link;
 678         uint32_t lcore_id;
 679         int ret, i;
 680
 681         ret = ntb_fwd_config_setup();
 682         if (ret < 0) {
 683                 printf("Cannot start traffic. Please reset fwd mode.\n");
 684                 return;
 685         }
 686
 687         /* If using iofwd, checking ethdev link status first. */
 688         if (fwd_mode == IOFWD) {
 689                 printf("Checking eth link status...\n");
 690                 /* Wait for eth link up at most 100 times. */
 691                 for (i = 0; i < 100; i++) {
 692                         rte_eth_link_get(eth_port_id, &eth_link);
 693                         if (eth_link.link_status) {
 694                                 printf("Eth%u Link Up. Speed %u Mbps - %s\n",
 695                                         eth_port_id, eth_link.link_speed,
 696                                         (eth_link.link_duplex ==
 697                                          ETH_LINK_FULL_DUPLEX) ?
 698                                         ("full-duplex") : ("half-duplex"));
 699                                 break;
 700                         }
 701                 }
 702                 if (!eth_link.link_status) {
 703                         printf("Eth%u link down. Cannot start traffic.\n",
 704                                 eth_port_id);
 705                         return;
 706                 }
 707         }
 708
 709         assign_stream_to_lcores();
 710         in_test = 1;
 711
 712         RTE_LCORE_FOREACH_SLAVE(lcore_id) {
 713                 conf = &fwd_lcore_conf[lcore_id];
 714
 715                 if (!conf->nb_stream)
 716                         continue;
 717
 718                 conf->stopped = 0;
 719                 if (fwd_mode == FILE_TRANS)
 720                         rte_eal_remote_launch(start_polling_recv_file,
 721                                               conf, lcore_id);
 722                 else if (fwd_mode == IOFWD)
 723                         rte_eal_remote_launch(start_iofwd_per_lcore,
 724                                               conf, lcore_id);
 725                 else if (fwd_mode == RXONLY)
 726                         rte_eal_remote_launch(start_rxonly_per_lcore,
 727                                               conf, lcore_id);
 728                 else if (fwd_mode == TXONLY)
 729                         rte_eal_remote_launch(start_txonly_per_lcore,
 730                                               conf, lcore_id);
 731         }
 732 }
 733
 734 /* *** START FWD PARAMETERS *** */
 735 struct cmd_start_result {
 736         cmdline_fixed_string_t start;
 737 };
 738
 739 static void
 740 cmd_start_parsed(__attribute__((unused)) void *parsed_result,
 741                             __attribute__((unused)) struct cmdline *cl,
 742                             __attribute__((unused)) void *data)
 743 {
 744         start_pkt_fwd();
 745 }
 746
 747 cmdline_parse_token_string_t cmd_start_start =
 748                 TOKEN_STRING_INITIALIZER(struct cmd_start_result, start, "start");
 749
 750 cmdline_parse_inst_t cmd_start = {
 751         .f = cmd_start_parsed,
 752         .data = NULL,
 753         .help_str = "start pkt fwd between ntb and ethdev",
 754         .tokens = {
 755                 (void *)&cmd_start_start,
 756                 NULL,
 757         },
 758 };
 759
 760 /* *** STOP *** */
 761 struct cmd_stop_result {
 762         cmdline_fixed_string_t stop;
 763 };
 764
 765 static void
 766 cmd_stop_parsed(__attribute__((unused)) void *parsed_result,
 767                 __attribute__((unused)) struct cmdline *cl,
 768                 __attribute__((unused)) void *data)
 769 {
 770         struct ntb_fwd_lcore_conf *conf;
 771         uint32_t lcore_id;
 772
 773         RTE_LCORE_FOREACH_SLAVE(lcore_id) {
 774                 conf = &fwd_lcore_conf[lcore_id];
 775
 776                 if (!conf->nb_stream)
 777                         continue;
 778
 779                 if (conf->stopped)
 780                         continue;
 781
 782                 conf->stopped = 1;
 783         }
 784         printf("\nWaiting for lcores to finish...\n");
 785         rte_eal_mp_wait_lcore();
 786         in_test = 0;
 787         printf("\nDone.\n");
 788 }
 789
 790 cmdline_parse_token_string_t cmd_stop_stop =
 791                 TOKEN_STRING_INITIALIZER(struct cmd_stop_result, stop, "stop");
 792
 793 cmdline_parse_inst_t cmd_stop = {
 794         .f = cmd_stop_parsed,
 795         .data = NULL,
 796         .help_str = "stop: Stop packet forwarding",
 797         .tokens = {
 798                 (void *)&cmd_stop_stop,
 799                 NULL,
 800         },
 801 };
 802
 803 static void
 804 ntb_stats_clear(void)
 805 {
 806         int nb_ids, i;
 807         uint32_t *ids;
 808
 809         /* Clear NTB dev stats */
 810         nb_ids = rte_rawdev_xstats_names_get(dev_id, NULL, 0);
 811         if (nb_ids  < 0) {
 812                 printf("Error: Cannot get count of xstats\n");
 813                 return;
 814         }
 815         ids = malloc(sizeof(uint32_t) * nb_ids);
 816         for (i = 0; i < nb_ids; i++)
 817                 ids[i] = i;
 818         rte_rawdev_xstats_reset(dev_id, ids, nb_ids);
 819         printf("\n  statistics for NTB port %d cleared\n", dev_id);
 820
 821         /* Clear Ethdev stats if have any */
 822         if (fwd_mode == IOFWD && eth_port_id != RTE_MAX_ETHPORTS) {
 823                 rte_eth_stats_reset(eth_port_id);
 824                 printf("\n  statistics for ETH port %d cleared\n", eth_port_id);
 825         }
 826 }
 827
 828 static inline void
 829 ntb_calculate_throughput(uint16_t port) {
 830         uint64_t diff_pkts_rx, diff_pkts_tx, diff_cycles;
 831         uint64_t mpps_rx, mpps_tx;
 832         static uint64_t prev_pkts_rx[2];
 833         static uint64_t prev_pkts_tx[2];
 834         static uint64_t prev_cycles[2];
 835
 836         diff_cycles = prev_cycles[port];
 837         prev_cycles[port] = rte_rdtsc();
 838         if (diff_cycles > 0)
 839                 diff_cycles = prev_cycles[port] - diff_cycles;
 840         diff_pkts_rx = (ntb_port_stats[port].rx > prev_pkts_rx[port]) ?
 841                 (ntb_port_stats[port].rx - prev_pkts_rx[port]) : 0;
 842         diff_pkts_tx = (ntb_port_stats[port].tx > prev_pkts_tx[port]) ?
 843                 (ntb_port_stats[port].tx - prev_pkts_tx[port]) : 0;
 844         prev_pkts_rx[port] = ntb_port_stats[port].rx;
 845         prev_pkts_tx[port] = ntb_port_stats[port].tx;
 846         mpps_rx = diff_cycles > 0 ?
 847                 diff_pkts_rx * rte_get_tsc_hz() / diff_cycles : 0;
 848         mpps_tx = diff_cycles > 0 ?
 849                 diff_pkts_tx * rte_get_tsc_hz() / diff_cycles : 0;
 850         printf("  Throughput (since last show)\n");
 851         printf("  Rx-pps: %12"PRIu64"\n  Tx-pps: %12"PRIu64"\n",
 852                         mpps_rx, mpps_tx);
 853
 854 }
 855
 856 static void
 857 ntb_stats_display(void)
 858 {
 859         struct rte_rawdev_xstats_name *xstats_names;
 860         struct rte_eth_stats stats;
 861         uint64_t *values;
 862         uint32_t *ids;
 863         int nb_ids, i;
 864
 865         printf("###### statistics for NTB port %d #######\n", dev_id);
 866
 867         /* Get NTB dev stats and stats names */
 868         nb_ids = rte_rawdev_xstats_names_get(dev_id, NULL, 0);
 869         if (nb_ids  < 0) {
 870                 printf("Error: Cannot get count of xstats\n");
 871                 return;
 872         }
 873         xstats_names = malloc(sizeof(struct rte_rawdev_xstats_name) * nb_ids);
 874         if (xstats_names == NULL) {
 875                 printf("Cannot allocate memory for xstats lookup\n");
 876                 return;
 877         }
 878         if (nb_ids != rte_rawdev_xstats_names_get(
 879                         dev_id, xstats_names, nb_ids)) {
 880                 printf("Error: Cannot get xstats lookup\n");
 881                 free(xstats_names);
 882                 return;
 883         }
 884         ids = malloc(sizeof(uint32_t) * nb_ids);
 885         for (i = 0; i < nb_ids; i++)
 886                 ids[i] = i;
 887         values = malloc(sizeof(uint64_t) * nb_ids);
 888         if (nb_ids != rte_rawdev_xstats_get(dev_id, ids, values, nb_ids)) {
 889                 printf("Error: Unable to get xstats\n");
 890                 free(xstats_names);
 891                 free(values);
 892                 free(ids);
 893                 return;
 894         }
 895
 896         /* Display NTB dev stats */
 897         for (i = 0; i < nb_ids; i++)
 898                 printf("  %s: %"PRIu64"\n", xstats_names[i].name, values[i]);
 899         ntb_calculate_throughput(0);
 900
 901         /* Get Ethdev stats if have any */
 902         if (fwd_mode == IOFWD && eth_port_id != RTE_MAX_ETHPORTS) {
 903                 printf("###### statistics for ETH port %d ######\n",
 904                         eth_port_id);
 905                 rte_eth_stats_get(eth_port_id, &stats);
 906                 printf("  RX-packets: %"PRIu64"\n", stats.ipackets);
 907                 printf("  RX-bytes: %"PRIu64"\n", stats.ibytes);
 908                 printf("  RX-errors: %"PRIu64"\n", stats.ierrors);
 909                 printf("  RX-missed: %"PRIu64"\n", stats.imissed);
 910                 printf("  TX-packets: %"PRIu64"\n", stats.opackets);
 911                 printf("  TX-bytes: %"PRIu64"\n", stats.obytes);
 912                 printf("  TX-errors: %"PRIu64"\n", stats.oerrors);
 913                 ntb_calculate_throughput(1);
 914         }
 915
 916         free(xstats_names);
 917         free(values);
 918         free(ids);
 919 }
 920
 921 /* *** SHOW/CLEAR PORT STATS *** */
 922 struct cmd_stats_result {
 923         cmdline_fixed_string_t show;
 924         cmdline_fixed_string_t port;
 925         cmdline_fixed_string_t stats;
 926 };
 927
 928 static void
 929 cmd_stats_parsed(void *parsed_result,
 930                  __attribute__((unused)) struct cmdline *cl,
 931                  __attribute__((unused)) void *data)
 932 {
 933         struct cmd_stats_result *res = parsed_result;
 934         if (!strcmp(res->show, "clear"))
 935                 ntb_stats_clear();
 936         else
 937                 ntb_stats_display();
 938 }
 939
 940 cmdline_parse_token_string_t cmd_stats_show =
 941         TOKEN_STRING_INITIALIZER(struct cmd_stats_result, show, "show#clear");
 942 cmdline_parse_token_string_t cmd_stats_port =
 943         TOKEN_STRING_INITIALIZER(struct cmd_stats_result, port, "port");
 944 cmdline_parse_token_string_t cmd_stats_stats =
 945         TOKEN_STRING_INITIALIZER(struct cmd_stats_result, stats, "stats");
 946
 947
 948 cmdline_parse_inst_t cmd_stats = {
 949         .f = cmd_stats_parsed,
 950         .data = NULL,
 951         .help_str = "show|clear port stats",
 952         .tokens = {
 953                 (void *)&cmd_stats_show,
 954                 (void *)&cmd_stats_port,
 955                 (void *)&cmd_stats_stats,
 956                 NULL,
 957         },
 958 };
 959
 960 /* *** SET FORWARDING MODE *** */
 961 struct cmd_set_fwd_mode_result {
 962         cmdline_fixed_string_t set;
 963         cmdline_fixed_string_t fwd;
 964         cmdline_fixed_string_t mode;
 965 };
 966
 967 static void
 968 cmd_set_fwd_mode_parsed(__attribute__((unused)) void *parsed_result,
 969                         __attribute__((unused)) struct cmdline *cl,
 970                         __attribute__((unused)) void *data)
 971 {
 972         struct cmd_set_fwd_mode_result *res = parsed_result;
 973         int i;
 974
 975         if (in_test) {
 976                 printf("Please stop traffic first.\n");
 977                 return;
 978         }
 979
 980         for (i = 0; i < MAX_FWD_MODE; i++) {
 981                 if (!strcmp(res->mode, fwd_mode_s[i])) {
 982                         fwd_mode = i;
 983                         return;
 984                 }
 985         }
 986         printf("Invalid %s packet forwarding mode.\n", res->mode);
 987 }
 988
 989 cmdline_parse_token_string_t cmd_setfwd_set =
 990         TOKEN_STRING_INITIALIZER(struct cmd_set_fwd_mode_result, set, "set");
 991 cmdline_parse_token_string_t cmd_setfwd_fwd =
 992         TOKEN_STRING_INITIALIZER(struct cmd_set_fwd_mode_result, fwd, "fwd");
 993 cmdline_parse_token_string_t cmd_setfwd_mode =
 994         TOKEN_STRING_INITIALIZER(struct cmd_set_fwd_mode_result, mode,
 995                                 "file-trans#iofwd#txonly#rxonly");
 996
 997 cmdline_parse_inst_t cmd_set_fwd_mode = {
 998         .f = cmd_set_fwd_mode_parsed,
 999         .data = NULL,
1000         .help_str = "set forwarding mode as file-trans|rxonly|txonly|iofwd",
1001         .tokens = {
1002                 (void *)&cmd_setfwd_set,
1003                 (void *)&cmd_setfwd_fwd,
1004                 (void *)&cmd_setfwd_mode,
1005                 NULL,
1006         },
1007 };
1008
1009 /* list of instructions */
1010 cmdline_parse_ctx_t main_ctx[] = {
1011         (cmdline_parse_inst_t *)&cmd_help,
1012         (cmdline_parse_inst_t *)&cmd_send_file,
1013         (cmdline_parse_inst_t *)&cmd_start,
1014         (cmdline_parse_inst_t *)&cmd_stop,
1015         (cmdline_parse_inst_t *)&cmd_stats,
1016         (cmdline_parse_inst_t *)&cmd_set_fwd_mode,
1017         (cmdline_parse_inst_t *)&cmd_quit,
1018         NULL,
1019 };
1020
1021 /* prompt function, called from main on MASTER lcore */
1022 static void
1023 prompt(void)
1024 {
1025         struct cmdline *cl;
1026
1027         cl = cmdline_stdin_new(main_ctx, "ntb> ");
1028         if (cl == NULL)
1029                 return;
1030
1031         cmdline_interact(cl);
1032         cmdline_stdin_exit(cl);
1033 }
1034
1035 static void
1036 signal_handler(int signum)
1037 {
1038         if (signum == SIGINT || signum == SIGTERM) {
1039                 printf("\nSignal %d received, preparing to exit...\n", signum);
1040                 signal(signum, SIG_DFL);
1041                 kill(getpid(), signum);
1042         }
1043 }
1044
1045 #define OPT_BUF_SIZE         "buf-size"
1046 #define OPT_FWD_MODE         "fwd-mode"
1047 #define OPT_NB_DESC          "nb-desc"
1048 #define OPT_TXFREET          "txfreet"
1049 #define OPT_BURST            "burst"
1050 #define OPT_QP               "qp"
1051
1052 enum {
1053         /* long options mapped to a short option */
1054         OPT_NO_ZERO_COPY_NUM = 1,
1055         OPT_BUF_SIZE_NUM,
1056         OPT_FWD_MODE_NUM,
1057         OPT_NB_DESC_NUM,
1058         OPT_TXFREET_NUM,
1059         OPT_BURST_NUM,
1060         OPT_QP_NUM,
1061 };
1062
1063 static const char short_options[] =
1064         "i" /* interactive mode */
1065         ;
1066
1067 static const struct option lgopts[] = {
1068         {OPT_BUF_SIZE,     1, NULL, OPT_BUF_SIZE_NUM     },
1069         {OPT_FWD_MODE,     1, NULL, OPT_FWD_MODE_NUM     },
1070         {OPT_NB_DESC,      1, NULL, OPT_NB_DESC_NUM      },
1071         {OPT_TXFREET,      1, NULL, OPT_TXFREET_NUM      },
1072         {OPT_BURST,        1, NULL, OPT_BURST_NUM        },
1073         {OPT_QP,           1, NULL, OPT_QP_NUM           },
1074         {0,                0, NULL, 0                    }
1075 };
1076
1077 static void
1078 ntb_usage(const char *prgname)
1079 {
1080         printf("%s [EAL options] -- [options]\n"
1081                "-i: run in interactive mode.\n"
1082                "-qp=N: set number of queues as N (N > 0, default: 1).\n"
1083                "--fwd-mode=N: set fwd mode (N: file-trans | rxonly | "
1084                "txonly | iofwd, default: file-trans)\n"
1085                "--buf-size=N: set mbuf dataroom size as N (0 < N < 65535,"
1086                " default: 2048).\n"
1087                "--nb-desc=N: set number of descriptors as N (%u <= N <= %u,"
1088                " default: 1024).\n"
1089                "--txfreet=N: set tx free thresh for NTB driver as N. (N >= 0)\n"
1090                "--burst=N: set pkt burst as N (0 < N <= %u default: 32).\n",
1091                prgname, NTB_MIN_DESC_SIZE, NTB_MAX_DESC_SIZE,
1092                NTB_MAX_PKT_BURST);
1093 }
1094
1095 static void
1096 ntb_parse_args(int argc, char **argv)
1097 {
1098         char *prgname = argv[0], **argvopt = argv;
1099         int opt, opt_idx, n, i;
1100
1101         while ((opt = getopt_long(argc, argvopt, short_options,
1102                                 lgopts, &opt_idx)) != EOF) {
1103                 switch (opt) {
1104                 case 'i':
1105                         printf("Interactive-mode selected.\n");
1106                         interactive = 1;
1107                         break;
1108                 case OPT_QP_NUM:
1109                         n = atoi(optarg);
1110                         if (n > 0)
1111                                 num_queues = n;
1112                         else
1113                                 rte_exit(EXIT_FAILURE, "q must be > 0.\n");
1114                         break;
1115                 case OPT_BUF_SIZE_NUM:
1116                         n = atoi(optarg);
1117                         if (n > RTE_PKTMBUF_HEADROOM && n <= 0xFFFF)
1118                                 ntb_buf_size = n;
1119                         else
1120                                 rte_exit(EXIT_FAILURE, "buf-size must be > "
1121                                         "%u and < 65536.\n",
1122                                         RTE_PKTMBUF_HEADROOM);
1123                         break;
1124                 case OPT_FWD_MODE_NUM:
1125                         for (i = 0; i < MAX_FWD_MODE; i++) {
1126                                 if (!strcmp(optarg, fwd_mode_s[i])) {
1127                                         fwd_mode = i;
1128                                         break;
1129                                 }
1130                         }
1131                         if (i == MAX_FWD_MODE)
1132                                 rte_exit(EXIT_FAILURE, "Unsupported mode. "
1133                                 "(Should be: file-trans | rxonly | txonly "
1134                                 "| iofwd)\n");
1135                         break;
1136                 case OPT_NB_DESC_NUM:
1137                         n = atoi(optarg);
1138                         if (n >= NTB_MIN_DESC_SIZE && n <= NTB_MAX_DESC_SIZE)
1139                                 nb_desc = n;
1140                         else
1141                                 rte_exit(EXIT_FAILURE, "nb-desc must be within"
1142                                         " [%u, %u].\n", NTB_MIN_DESC_SIZE,
1143                                         NTB_MAX_DESC_SIZE);
1144                         break;
1145                 case OPT_TXFREET_NUM:
1146                         n = atoi(optarg);
1147                         if (n >= 0)
1148                                 tx_free_thresh = n;
1149                         else
1150                                 rte_exit(EXIT_FAILURE, "txfreet must be"
1151                                         " >= 0\n");
1152                         break;
1153                 case OPT_BURST_NUM:
1154                         n = atoi(optarg);
1155                         if (n > 0 && n <= NTB_MAX_PKT_BURST)
1156                                 pkt_burst = n;
1157                         else
1158                                 rte_exit(EXIT_FAILURE, "burst must be within "
1159                                         "(0, %u].\n", NTB_MAX_PKT_BURST);
1160                         break;
1161
1162                 default:
1163                         ntb_usage(prgname);
1164                         rte_exit(EXIT_FAILURE,
1165                                  "Command line is incomplete or incorrect.\n");
1166                         break;
1167                 }
1168         }
1169 }
1170
1171 static void
1172 ntb_mempool_mz_free(__rte_unused struct rte_mempool_memhdr *memhdr,
1173                 void *opaque)
1174 {
1175         const struct rte_memzone *mz = opaque;
1176         rte_memzone_free(mz);
1177 }
1178
1179 static struct rte_mempool *
1180 ntb_mbuf_pool_create(uint16_t mbuf_seg_size, uint32_t nb_mbuf,
1181                      struct ntb_dev_info ntb_info,
1182                      struct ntb_dev_config *ntb_conf,
1183                      unsigned int socket_id)
1184 {
1185         size_t mz_len, total_elt_sz, max_mz_len, left_sz;
1186         struct rte_pktmbuf_pool_private mbp_priv;
1187         char pool_name[RTE_MEMPOOL_NAMESIZE];
1188         char mz_name[RTE_MEMZONE_NAMESIZE];
1189         const struct rte_memzone *mz;
1190         struct rte_mempool *mp;
1191         uint64_t align;
1192         uint32_t mz_id;
1193         int ret;
1194
1195         snprintf(pool_name, sizeof(pool_name), "ntb_mbuf_pool_%u", socket_id);
1196         mp = rte_mempool_create_empty(pool_name, nb_mbuf,
1197                                       (mbuf_seg_size + sizeof(struct rte_mbuf)),
1198                                       MEMPOOL_CACHE_SIZE,
1199                                       sizeof(struct rte_pktmbuf_pool_private),
1200                                       socket_id, 0);
1201         if (mp == NULL)
1202                 return NULL;
1203
1204         mbp_priv.mbuf_data_room_size = mbuf_seg_size;
1205         mbp_priv.mbuf_priv_size = 0;
1206         rte_pktmbuf_pool_init(mp, &mbp_priv);
1207
1208         ntb_conf->mz_list = rte_zmalloc("ntb_memzone_list",
1209                                 sizeof(struct rte_memzone *) *
1210                                 ntb_info.mw_cnt, 0);
1211         if (ntb_conf->mz_list == NULL)
1212                 goto fail;
1213
1214         /* Put ntb header on mw0. */
1215         if (ntb_info.mw_size[0] < ntb_info.ntb_hdr_size) {
1216                 printf("mw0 (size: %" PRIu64 ") is not enough for ntb hdr"
1217                        " (size: %u)\n", ntb_info.mw_size[0],
1218                        ntb_info.ntb_hdr_size);
1219                 goto fail;
1220         }
1221
1222         total_elt_sz = mp->header_size + mp->elt_size + mp->trailer_size;
1223         left_sz = total_elt_sz * nb_mbuf;
1224         for (mz_id = 0; mz_id < ntb_info.mw_cnt; mz_id++) {
1225                 /* If populated mbuf is enough, no need to reserve extra mz. */
1226                 if (!left_sz)
1227                         break;
1228                 snprintf(mz_name, sizeof(mz_name), "ntb_mw_%d", mz_id);
1229                 align = ntb_info.mw_size_align ? ntb_info.mw_size[mz_id] :
1230                         RTE_CACHE_LINE_SIZE;
1231                 /* Reserve ntb header space on memzone 0. */
1232                 max_mz_len = mz_id ? ntb_info.mw_size[mz_id] :
1233                              ntb_info.mw_size[mz_id] - ntb_info.ntb_hdr_size;
1234                 mz_len = left_sz <= max_mz_len ? left_sz :
1235                         (max_mz_len / total_elt_sz * total_elt_sz);
1236                 if (!mz_len)
1237                         continue;
1238                 mz = rte_memzone_reserve_aligned(mz_name, mz_len, socket_id,
1239                                         RTE_MEMZONE_IOVA_CONTIG, align);
1240                 if (mz == NULL) {
1241                         printf("Cannot allocate %" PRIu64 " aligned memzone"
1242                                 " %u\n", align, mz_id);
1243                         goto fail;
1244                 }
1245                 left_sz -= mz_len;
1246
1247                 /* Reserve ntb header space on memzone 0. */
1248                 if (mz_id)
1249                         ret = rte_mempool_populate_iova(mp, mz->addr, mz->iova,
1250                                         mz->len, ntb_mempool_mz_free,
1251                                         (void *)(uintptr_t)mz);
1252                 else
1253                         ret = rte_mempool_populate_iova(mp,
1254                                         (void *)((size_t)mz->addr +
1255                                         ntb_info.ntb_hdr_size),
1256                                         mz->iova + ntb_info.ntb_hdr_size,
1257                                         mz->len - ntb_info.ntb_hdr_size,
1258                                         ntb_mempool_mz_free,
1259                                         (void *)(uintptr_t)mz);
1260                 if (ret < 0) {
1261                         rte_memzone_free(mz);
1262                         rte_mempool_free(mp);
1263                         return NULL;
1264                 }
1265
1266                 ntb_conf->mz_list[mz_id] = mz;
1267         }
1268         if (left_sz) {
1269                 printf("mw space is not enough for mempool.\n");
1270                 goto fail;
1271         }
1272
1273         ntb_conf->mz_num = mz_id;
1274         rte_mempool_obj_iter(mp, rte_pktmbuf_init, NULL);
1275
1276         return mp;
1277 fail:
1278         rte_mempool_free(mp);
1279         return NULL;
1280 }
1281
1282 int
1283 main(int argc, char **argv)
1284 {
1285         struct rte_eth_conf eth_pconf = eth_port_conf;
1286         struct rte_rawdev_info ntb_rawdev_conf;
1287         struct rte_rawdev_info ntb_rawdev_info;
1288         struct rte_eth_dev_info ethdev_info;
1289         struct rte_eth_rxconf eth_rx_conf;
1290         struct rte_eth_txconf eth_tx_conf;
1291         struct ntb_queue_conf ntb_q_conf;
1292         struct ntb_dev_config ntb_conf;
1293         struct ntb_dev_info ntb_info;
1294         uint64_t ntb_link_status;
1295         uint32_t nb_mbuf;
1296         int ret, i;
1297
1298         signal(SIGINT, signal_handler);
1299         signal(SIGTERM, signal_handler);
1300
1301         ret = rte_eal_init(argc, argv);
1302         if (ret < 0)
1303                 rte_exit(EXIT_FAILURE, "Error with EAL initialization.\n");
1304
1305         if (rte_lcore_count() < 2)
1306                 rte_exit(EXIT_FAILURE, "Need at least 2 cores\n");
1307
1308         /* Find 1st ntb rawdev. */
1309         for (i = 0; i < RTE_RAWDEV_MAX_DEVS; i++)
1310                 if (rte_rawdevs[i].driver_name &&
1311                     (strncmp(rte_rawdevs[i].driver_name, "raw_ntb",
1312                     NTB_DRV_NAME_LEN) == 0) && (rte_rawdevs[i].attached == 1))
1313                         break;
1314
1315         if (i == RTE_RAWDEV_MAX_DEVS)
1316                 rte_exit(EXIT_FAILURE, "Cannot find any ntb device.\n");
1317
1318         dev_id = i;
1319
1320         argc -= ret;
1321         argv += ret;
1322
1323         ntb_parse_args(argc, argv);
1324
1325         rte_rawdev_set_attr(dev_id, NTB_QUEUE_SZ_NAME, nb_desc);
1326         printf("Set queue size as %u.\n", nb_desc);
1327         rte_rawdev_set_attr(dev_id, NTB_QUEUE_NUM_NAME, num_queues);
1328         printf("Set queue number as %u.\n", num_queues);
1329         ntb_rawdev_info.dev_private = (rte_rawdev_obj_t)(&ntb_info);
1330         rte_rawdev_info_get(dev_id, &ntb_rawdev_info);
1331
1332         nb_mbuf = nb_desc * num_queues * 2 * 2 + rte_lcore_count() *
1333                   MEMPOOL_CACHE_SIZE;
1334         mbuf_pool = ntb_mbuf_pool_create(ntb_buf_size, nb_mbuf, ntb_info,
1335                                          &ntb_conf, rte_socket_id());
1336         if (mbuf_pool == NULL)
1337                 rte_exit(EXIT_FAILURE, "Cannot create mbuf pool.\n");
1338
1339         ntb_conf.num_queues = num_queues;
1340         ntb_conf.queue_size = nb_desc;
1341         ntb_rawdev_conf.dev_private = (rte_rawdev_obj_t)(&ntb_conf);
1342         ret = rte_rawdev_configure(dev_id, &ntb_rawdev_conf);
1343         if (ret)
1344                 rte_exit(EXIT_FAILURE, "Can't config ntb dev: err=%d, "
1345                         "port=%u\n", ret, dev_id);
1346
1347         ntb_q_conf.tx_free_thresh = tx_free_thresh;
1348         ntb_q_conf.nb_desc = nb_desc;
1349         ntb_q_conf.rx_mp = mbuf_pool;
1350         for (i = 0; i < num_queues; i++) {
1351                 /* Setup rawdev queue */
1352                 ret = rte_rawdev_queue_setup(dev_id, i, &ntb_q_conf);
1353                 if (ret < 0)
1354                         rte_exit(EXIT_FAILURE,
1355                                 "Failed to setup ntb queue %u.\n", i);
1356         }
1357
1358         /* Waiting for peer dev up at most 100s.*/
1359         printf("Checking ntb link status...\n");
1360         for (i = 0; i < 1000; i++) {
1361                 rte_rawdev_get_attr(dev_id, NTB_LINK_STATUS_NAME,
1362                                     &ntb_link_status);
1363                 if (ntb_link_status) {
1364                         printf("Peer dev ready, ntb link up.\n");
1365                         break;
1366                 }
1367                 rte_delay_ms(100);
1368         }
1369         rte_rawdev_get_attr(dev_id, NTB_LINK_STATUS_NAME, &ntb_link_status);
1370         if (ntb_link_status == 0)
1371                 printf("Expire 100s. Link is not up. Please restart app.\n");
1372
1373         ret = rte_rawdev_start(dev_id);
1374         if (ret < 0)
1375                 rte_exit(EXIT_FAILURE, "rte_rawdev_start: err=%d, port=%u\n",
1376                         ret, dev_id);
1377
1378         /* Find 1st ethdev */
1379         eth_port_id = rte_eth_find_next(0);
1380
1381         if (eth_port_id < RTE_MAX_ETHPORTS) {
1382                 rte_eth_dev_info_get(eth_port_id, &ethdev_info);
1383                 eth_pconf.rx_adv_conf.rss_conf.rss_hf &=
1384                                 ethdev_info.flow_type_rss_offloads;
1385                 ret = rte_eth_dev_configure(eth_port_id, num_queues,
1386                                             num_queues, &eth_pconf);
1387                 if (ret)
1388                         rte_exit(EXIT_FAILURE, "Can't config ethdev: err=%d, "
1389                                 "port=%u\n", ret, eth_port_id);
1390                 eth_rx_conf = ethdev_info.default_rxconf;
1391                 eth_rx_conf.offloads = eth_pconf.rxmode.offloads;
1392                 eth_tx_conf = ethdev_info.default_txconf;
1393                 eth_tx_conf.offloads = eth_pconf.txmode.offloads;
1394
1395                 /* Setup ethdev queue if ethdev exists */
1396                 for (i = 0; i < num_queues; i++) {
1397                         ret = rte_eth_rx_queue_setup(eth_port_id, i, nb_desc,
1398                                         rte_eth_dev_socket_id(eth_port_id),
1399                                         &eth_rx_conf, mbuf_pool);
1400                         if (ret < 0)
1401                                 rte_exit(EXIT_FAILURE,
1402                                         "Failed to setup eth rxq %u.\n", i);
1403                         ret = rte_eth_tx_queue_setup(eth_port_id, i, nb_desc,
1404                                         rte_eth_dev_socket_id(eth_port_id),
1405                                         &eth_tx_conf);
1406                         if (ret < 0)
1407                                 rte_exit(EXIT_FAILURE,
1408                                         "Failed to setup eth txq %u.\n", i);
1409                 }
1410
1411                 ret = rte_eth_dev_start(eth_port_id);
1412                 if (ret < 0)
1413                         rte_exit(EXIT_FAILURE, "rte_eth_dev_start: err=%d, "
1414                                 "port=%u\n", ret, eth_port_id);
1415         }
1416
1417         /* initialize port stats */
1418         memset(&ntb_port_stats, 0, sizeof(ntb_port_stats));
1419
1420         /* Set default fwd mode if user doesn't set it. */
1421         if (fwd_mode == MAX_FWD_MODE && eth_port_id < RTE_MAX_ETHPORTS) {
1422                 printf("Set default fwd mode as iofwd.\n");
1423                 fwd_mode = IOFWD;
1424         }
1425         if (fwd_mode == MAX_FWD_MODE) {
1426                 printf("Set default fwd mode as file-trans.\n");
1427                 fwd_mode = FILE_TRANS;
1428         }
1429
1430         if (interactive) {
1431                 sleep(1);
1432                 prompt();
1433         } else {
1434                 start_pkt_fwd();
1435         }
1436
1437         return 0;
1438 }