examples/l3fwd/main.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(c) 2010-2016 Intel Corporation
   3  */
   4
   5 #include <stdio.h>
   6 #include <stdlib.h>
   7 #include <stdint.h>
   8 #include <inttypes.h>
   9 #include <sys/types.h>
  10 #include <string.h>
  11 #include <sys/queue.h>
  12 #include <stdarg.h>
  13 #include <errno.h>
  14 #include <getopt.h>
  15 #include <signal.h>
  16 #include <stdbool.h>
  17
  18 #include <rte_common.h>
  19 #include <rte_vect.h>
  20 #include <rte_byteorder.h>
  21 #include <rte_log.h>
  22 #include <rte_malloc.h>
  23 #include <rte_memory.h>
  24 #include <rte_memcpy.h>
  25 #include <rte_eal.h>
  26 #include <rte_launch.h>
  27 #include <rte_atomic.h>
  28 #include <rte_cycles.h>
  29 #include <rte_prefetch.h>
  30 #include <rte_lcore.h>
  31 #include <rte_per_lcore.h>
  32 #include <rte_branch_prediction.h>
  33 #include <rte_interrupts.h>
  34 #include <rte_random.h>
  35 #include <rte_debug.h>
  36 #include <rte_ether.h>
  37 #include <rte_mempool.h>
  38 #include <rte_mbuf.h>
  39 #include <rte_ip.h>
  40 #include <rte_tcp.h>
  41 #include <rte_udp.h>
  42 #include <rte_string_fns.h>
  43 #include <rte_cpuflags.h>
  44
  45 #include <cmdline_parse.h>
  46 #include <cmdline_parse_etheraddr.h>
  47
  48 #include "l3fwd.h"
  49 #include "l3fwd_event.h"
  50
  51 /*
  52  * Configurable number of RX/TX ring descriptors
  53  */
  54 #define RTE_TEST_RX_DESC_DEFAULT 1024
  55 #define RTE_TEST_TX_DESC_DEFAULT 1024
  56
  57 #define MAX_TX_QUEUE_PER_PORT RTE_MAX_ETHPORTS
  58 #define MAX_RX_QUEUE_PER_PORT 128
  59
  60 #define MAX_LCORE_PARAMS 1024
  61
  62 /* Static global variables used within this file. */
  63 static uint16_t nb_rxd = RTE_TEST_RX_DESC_DEFAULT;
  64 static uint16_t nb_txd = RTE_TEST_TX_DESC_DEFAULT;
  65
  66 /**< Ports set in promiscuous mode off by default. */
  67 static int promiscuous_on;
  68
  69 /* Select Longest-Prefix or Exact match. */
  70 static int l3fwd_lpm_on;
  71 static int l3fwd_em_on;
  72
  73 /* Global variables. */
  74
  75 static int numa_on = 1; /**< NUMA is enabled by default. */
  76 static int parse_ptype; /**< Parse packet type using rx callback, and */
  77                         /**< disabled by default */
  78 static int per_port_pool; /**< Use separate buffer pools per port; disabled */
  79                           /**< by default */
  80
  81 volatile bool force_quit;
  82
  83 /* ethernet addresses of ports */
  84 uint64_t dest_eth_addr[RTE_MAX_ETHPORTS];
  85 struct rte_ether_addr ports_eth_addr[RTE_MAX_ETHPORTS];
  86
  87 xmm_t val_eth[RTE_MAX_ETHPORTS];
  88
  89 /* mask of enabled ports */
  90 uint32_t enabled_port_mask;
  91
  92 /* Used only in exact match mode. */
  93 int ipv6; /**< ipv6 is false by default. */
  94 uint32_t hash_entry_number = HASH_ENTRY_NUMBER_DEFAULT;
  95
  96 struct lcore_conf lcore_conf[RTE_MAX_LCORE];
  97
  98 struct lcore_params {
  99         uint16_t port_id;
 100         uint8_t queue_id;
 101         uint8_t lcore_id;
 102 } __rte_cache_aligned;
 103
 104 static struct lcore_params lcore_params_array[MAX_LCORE_PARAMS];
 105 static struct lcore_params lcore_params_array_default[] = {
 106         {0, 0, 2},
 107         {0, 1, 2},
 108         {0, 2, 2},
 109         {1, 0, 2},
 110         {1, 1, 2},
 111         {1, 2, 2},
 112         {2, 0, 2},
 113         {3, 0, 3},
 114         {3, 1, 3},
 115 };
 116
 117 static struct lcore_params * lcore_params = lcore_params_array_default;
 118 static uint16_t nb_lcore_params = sizeof(lcore_params_array_default) /
 119                                 sizeof(lcore_params_array_default[0]);
 120
 121 static struct rte_eth_conf port_conf = {
 122         .rxmode = {
 123                 .mq_mode = ETH_MQ_RX_RSS,
 124                 .max_rx_pkt_len = RTE_ETHER_MAX_LEN,
 125                 .split_hdr_size = 0,
 126                 .offloads = DEV_RX_OFFLOAD_CHECKSUM,
 127         },
 128         .rx_adv_conf = {
 129                 .rss_conf = {
 130                         .rss_key = NULL,
 131                         .rss_hf = ETH_RSS_IP,
 132                 },
 133         },
 134         .txmode = {
 135                 .mq_mode = ETH_MQ_TX_NONE,
 136         },
 137 };
 138
 139 static struct rte_mempool *pktmbuf_pool[RTE_MAX_ETHPORTS][NB_SOCKETS];
 140 static uint8_t lkp_per_socket[NB_SOCKETS];
 141
 142 struct l3fwd_lkp_mode {
 143         void  (*setup)(int);
 144         int   (*check_ptype)(int);
 145         rte_rx_callback_fn cb_parse_ptype;
 146         int   (*main_loop)(void *);
 147         void* (*get_ipv4_lookup_struct)(int);
 148         void* (*get_ipv6_lookup_struct)(int);
 149 };
 150
 151 static struct l3fwd_lkp_mode l3fwd_lkp;
 152
 153 static struct l3fwd_lkp_mode l3fwd_em_lkp = {
 154         .setup                  = setup_hash,
 155         .check_ptype            = em_check_ptype,
 156         .cb_parse_ptype         = em_cb_parse_ptype,
 157         .main_loop              = em_main_loop,
 158         .get_ipv4_lookup_struct = em_get_ipv4_l3fwd_lookup_struct,
 159         .get_ipv6_lookup_struct = em_get_ipv6_l3fwd_lookup_struct,
 160 };
 161
 162 static struct l3fwd_lkp_mode l3fwd_lpm_lkp = {
 163         .setup                  = setup_lpm,
 164         .check_ptype            = lpm_check_ptype,
 165         .cb_parse_ptype         = lpm_cb_parse_ptype,
 166         .main_loop              = lpm_main_loop,
 167         .get_ipv4_lookup_struct = lpm_get_ipv4_l3fwd_lookup_struct,
 168         .get_ipv6_lookup_struct = lpm_get_ipv6_l3fwd_lookup_struct,
 169 };
 170
 171 /*
 172  * Setup lookup methods for forwarding.
 173  * Currently exact-match and longest-prefix-match
 174  * are supported ones.
 175  */
 176 static void
 177 setup_l3fwd_lookup_tables(void)
 178 {
 179         /* Setup HASH lookup functions. */
 180         if (l3fwd_em_on)
 181                 l3fwd_lkp = l3fwd_em_lkp;
 182         /* Setup LPM lookup functions. */
 183         else
 184                 l3fwd_lkp = l3fwd_lpm_lkp;
 185 }
 186
 187 static int
 188 check_lcore_params(void)
 189 {
 190         uint8_t queue, lcore;
 191         uint16_t i;
 192         int socketid;
 193
 194         for (i = 0; i < nb_lcore_params; ++i) {
 195                 queue = lcore_params[i].queue_id;
 196                 if (queue >= MAX_RX_QUEUE_PER_PORT) {
 197                         printf("invalid queue number: %hhu\n", queue);
 198                         return -1;
 199                 }
 200                 lcore = lcore_params[i].lcore_id;
 201                 if (!rte_lcore_is_enabled(lcore)) {
 202                         printf("error: lcore %hhu is not enabled in lcore mask\n", lcore);
 203                         return -1;
 204                 }
 205                 if ((socketid = rte_lcore_to_socket_id(lcore) != 0) &&
 206                         (numa_on == 0)) {
 207                         printf("warning: lcore %hhu is on socket %d with numa off \n",
 208                                 lcore, socketid);
 209                 }
 210         }
 211         return 0;
 212 }
 213
 214 static int
 215 check_port_config(void)
 216 {
 217         uint16_t portid;
 218         uint16_t i;
 219
 220         for (i = 0; i < nb_lcore_params; ++i) {
 221                 portid = lcore_params[i].port_id;
 222                 if ((enabled_port_mask & (1 << portid)) == 0) {
 223                         printf("port %u is not enabled in port mask\n", portid);
 224                         return -1;
 225                 }
 226                 if (!rte_eth_dev_is_valid_port(portid)) {
 227                         printf("port %u is not present on the board\n", portid);
 228                         return -1;
 229                 }
 230         }
 231         return 0;
 232 }
 233
 234 static uint8_t
 235 get_port_n_rx_queues(const uint16_t port)
 236 {
 237         int queue = -1;
 238         uint16_t i;
 239
 240         for (i = 0; i < nb_lcore_params; ++i) {
 241                 if (lcore_params[i].port_id == port) {
 242                         if (lcore_params[i].queue_id == queue+1)
 243                                 queue = lcore_params[i].queue_id;
 244                         else
 245                                 rte_exit(EXIT_FAILURE, "queue ids of the port %d must be"
 246                                                 " in sequence and must start with 0\n",
 247                                                 lcore_params[i].port_id);
 248                 }
 249         }
 250         return (uint8_t)(++queue);
 251 }
 252
 253 static int
 254 init_lcore_rx_queues(void)
 255 {
 256         uint16_t i, nb_rx_queue;
 257         uint8_t lcore;
 258
 259         for (i = 0; i < nb_lcore_params; ++i) {
 260                 lcore = lcore_params[i].lcore_id;
 261                 nb_rx_queue = lcore_conf[lcore].n_rx_queue;
 262                 if (nb_rx_queue >= MAX_RX_QUEUE_PER_LCORE) {
 263                         printf("error: too many queues (%u) for lcore: %u\n",
 264                                 (unsigned)nb_rx_queue + 1, (unsigned)lcore);
 265                         return -1;
 266                 } else {
 267                         lcore_conf[lcore].rx_queue_list[nb_rx_queue].port_id =
 268                                 lcore_params[i].port_id;
 269                         lcore_conf[lcore].rx_queue_list[nb_rx_queue].queue_id =
 270                                 lcore_params[i].queue_id;
 271                         lcore_conf[lcore].n_rx_queue++;
 272                 }
 273         }
 274         return 0;
 275 }
 276
 277 /* display usage */
 278 static void
 279 print_usage(const char *prgname)
 280 {
 281         fprintf(stderr, "%s [EAL options] --"
 282                 " -p PORTMASK"
 283                 " [-P]"
 284                 " [-E]"
 285                 " [-L]"
 286                 " --config (port,queue,lcore)[,(port,queue,lcore)]"
 287                 " [--eth-dest=X,MM:MM:MM:MM:MM:MM]"
 288                 " [--enable-jumbo [--max-pkt-len PKTLEN]]"
 289                 " [--no-numa]"
 290                 " [--hash-entry-num]"
 291                 " [--ipv6]"
 292                 " [--parse-ptype]"
 293                 " [--per-port-pool]"
 294                 " [--mode]"
 295                 " [--eventq-sched]\n\n"
 296
 297                 "  -p PORTMASK: Hexadecimal bitmask of ports to configure\n"
 298                 "  -P : Enable promiscuous mode\n"
 299                 "  -E : Enable exact match\n"
 300                 "  -L : Enable longest prefix match (default)\n"
 301                 "  --config (port,queue,lcore): Rx queue configuration\n"
 302                 "  --eth-dest=X,MM:MM:MM:MM:MM:MM: Ethernet destination for port X\n"
 303                 "  --enable-jumbo: Enable jumbo frames\n"
 304                 "  --max-pkt-len: Under the premise of enabling jumbo,\n"
 305                 "                 maximum packet length in decimal (64-9600)\n"
 306                 "  --no-numa: Disable numa awareness\n"
 307                 "  --hash-entry-num: Specify the hash entry number in hexadecimal to be setup\n"
 308                 "  --ipv6: Set if running ipv6 packets\n"
 309                 "  --parse-ptype: Set to use software to analyze packet type\n"
 310                 "  --per-port-pool: Use separate buffer pool per port\n"
 311                 "  --mode: Packet transfer mode for I/O, poll or eventdev\n"
 312                 "          Default mode = poll\n"
 313                 "  --eventq-sched: Event queue synchronization method\n"
 314                 "                  ordered, atomic or parallel.\n"
 315                 "                  Default: atomic\n"
 316                 "                  Valid only if --mode=eventdev\n"
 317                 "  --event-eth-rxqs: Number of ethernet RX queues per device.\n"
 318                 "                    Default: 1\n"
 319                 "                    Valid only if --mode=eventdev\n\n",
 320                 prgname);
 321 }
 322
 323 static int
 324 parse_max_pkt_len(const char *pktlen)
 325 {
 326         char *end = NULL;
 327         unsigned long len;
 328
 329         /* parse decimal string */
 330         len = strtoul(pktlen, &end, 10);
 331         if ((pktlen[0] == '\0') || (end == NULL) || (*end != '\0'))
 332                 return -1;
 333
 334         if (len == 0)
 335                 return -1;
 336
 337         return len;
 338 }
 339
 340 static int
 341 parse_portmask(const char *portmask)
 342 {
 343         char *end = NULL;
 344         unsigned long pm;
 345
 346         /* parse hexadecimal string */
 347         pm = strtoul(portmask, &end, 16);
 348         if ((portmask[0] == '\0') || (end == NULL) || (*end != '\0'))
 349                 return -1;
 350
 351         if (pm == 0)
 352                 return -1;
 353
 354         return pm;
 355 }
 356
 357 static int
 358 parse_hash_entry_number(const char *hash_entry_num)
 359 {
 360         char *end = NULL;
 361         unsigned long hash_en;
 362         /* parse hexadecimal string */
 363         hash_en = strtoul(hash_entry_num, &end, 16);
 364         if ((hash_entry_num[0] == '\0') || (end == NULL) || (*end != '\0'))
 365                 return -1;
 366
 367         if (hash_en == 0)
 368                 return -1;
 369
 370         return hash_en;
 371 }
 372
 373 static int
 374 parse_config(const char *q_arg)
 375 {
 376         char s[256];
 377         const char *p, *p0 = q_arg;
 378         char *end;
 379         enum fieldnames {
 380                 FLD_PORT = 0,
 381                 FLD_QUEUE,
 382                 FLD_LCORE,
 383                 _NUM_FLD
 384         };
 385         unsigned long int_fld[_NUM_FLD];
 386         char *str_fld[_NUM_FLD];
 387         int i;
 388         unsigned size;
 389
 390         nb_lcore_params = 0;
 391
 392         while ((p = strchr(p0,'(')) != NULL) {
 393                 ++p;
 394                 if((p0 = strchr(p,')')) == NULL)
 395                         return -1;
 396
 397                 size = p0 - p;
 398                 if(size >= sizeof(s))
 399                         return -1;
 400
 401                 snprintf(s, sizeof(s), "%.*s", size, p);
 402                 if (rte_strsplit(s, sizeof(s), str_fld, _NUM_FLD, ',') != _NUM_FLD)
 403                         return -1;
 404                 for (i = 0; i < _NUM_FLD; i++){
 405                         errno = 0;
 406                         int_fld[i] = strtoul(str_fld[i], &end, 0);
 407                         if (errno != 0 || end == str_fld[i] || int_fld[i] > 255)
 408                                 return -1;
 409                 }
 410                 if (nb_lcore_params >= MAX_LCORE_PARAMS) {
 411                         printf("exceeded max number of lcore params: %hu\n",
 412                                 nb_lcore_params);
 413                         return -1;
 414                 }
 415                 lcore_params_array[nb_lcore_params].port_id =
 416                         (uint8_t)int_fld[FLD_PORT];
 417                 lcore_params_array[nb_lcore_params].queue_id =
 418                         (uint8_t)int_fld[FLD_QUEUE];
 419                 lcore_params_array[nb_lcore_params].lcore_id =
 420                         (uint8_t)int_fld[FLD_LCORE];
 421                 ++nb_lcore_params;
 422         }
 423         lcore_params = lcore_params_array;
 424         return 0;
 425 }
 426
 427 static void
 428 parse_eth_dest(const char *optarg)
 429 {
 430         uint16_t portid;
 431         char *port_end;
 432         uint8_t c, *dest, peer_addr[6];
 433
 434         errno = 0;
 435         portid = strtoul(optarg, &port_end, 10);
 436         if (errno != 0 || port_end == optarg || *port_end++ != ',')
 437                 rte_exit(EXIT_FAILURE,
 438                 "Invalid eth-dest: %s", optarg);
 439         if (portid >= RTE_MAX_ETHPORTS)
 440                 rte_exit(EXIT_FAILURE,
 441                 "eth-dest: port %d >= RTE_MAX_ETHPORTS(%d)\n",
 442                 portid, RTE_MAX_ETHPORTS);
 443
 444         if (cmdline_parse_etheraddr(NULL, port_end,
 445                 &peer_addr, sizeof(peer_addr)) < 0)
 446                 rte_exit(EXIT_FAILURE,
 447                 "Invalid ethernet address: %s\n",
 448                 port_end);
 449         dest = (uint8_t *)&dest_eth_addr[portid];
 450         for (c = 0; c < 6; c++)
 451                 dest[c] = peer_addr[c];
 452         *(uint64_t *)(val_eth + portid) = dest_eth_addr[portid];
 453 }
 454
 455 static void
 456 parse_mode(const char *optarg)
 457 {
 458         struct l3fwd_event_resources *evt_rsrc = l3fwd_get_eventdev_rsrc();
 459
 460         if (!strcmp(optarg, "poll"))
 461                 evt_rsrc->enabled = false;
 462         else if (!strcmp(optarg, "eventdev"))
 463                 evt_rsrc->enabled = true;
 464 }
 465
 466 static void
 467 parse_eventq_sched(const char *optarg)
 468 {
 469         struct l3fwd_event_resources *evt_rsrc = l3fwd_get_eventdev_rsrc();
 470
 471         if (!strcmp(optarg, "ordered"))
 472                 evt_rsrc->sched_type = RTE_SCHED_TYPE_ORDERED;
 473         if (!strcmp(optarg, "atomic"))
 474                 evt_rsrc->sched_type = RTE_SCHED_TYPE_ATOMIC;
 475         if (!strcmp(optarg, "parallel"))
 476                 evt_rsrc->sched_type = RTE_SCHED_TYPE_PARALLEL;
 477 }
 478
 479 static void
 480 parse_event_eth_rx_queues(const char *eth_rx_queues)
 481 {
 482         struct l3fwd_event_resources *evt_rsrc = l3fwd_get_eventdev_rsrc();
 483         char *end = NULL;
 484         uint8_t num_eth_rx_queues;
 485
 486         /* parse decimal string */
 487         num_eth_rx_queues = strtoul(eth_rx_queues, &end, 10);
 488         if ((eth_rx_queues[0] == '\0') || (end == NULL) || (*end != '\0'))
 489                 return;
 490
 491         if (num_eth_rx_queues == 0)
 492                 return;
 493
 494         evt_rsrc->eth_rx_queues = num_eth_rx_queues;
 495 }
 496
 497 #define MAX_JUMBO_PKT_LEN  9600
 498 #define MEMPOOL_CACHE_SIZE 256
 499
 500 static const char short_options[] =
 501         "p:"  /* portmask */
 502         "P"   /* promiscuous */
 503         "L"   /* enable long prefix match */
 504         "E"   /* enable exact match */
 505         ;
 506
 507 #define CMD_LINE_OPT_CONFIG "config"
 508 #define CMD_LINE_OPT_ETH_DEST "eth-dest"
 509 #define CMD_LINE_OPT_NO_NUMA "no-numa"
 510 #define CMD_LINE_OPT_IPV6 "ipv6"
 511 #define CMD_LINE_OPT_ENABLE_JUMBO "enable-jumbo"
 512 #define CMD_LINE_OPT_HASH_ENTRY_NUM "hash-entry-num"
 513 #define CMD_LINE_OPT_PARSE_PTYPE "parse-ptype"
 514 #define CMD_LINE_OPT_PER_PORT_POOL "per-port-pool"
 515 #define CMD_LINE_OPT_MODE "mode"
 516 #define CMD_LINE_OPT_EVENTQ_SYNC "eventq-sched"
 517 #define CMD_LINE_OPT_EVENT_ETH_RX_QUEUES "event-eth-rxqs"
 518 enum {
 519         /* long options mapped to a short option */
 520
 521         /* first long only option value must be >= 256, so that we won't
 522          * conflict with short options */
 523         CMD_LINE_OPT_MIN_NUM = 256,
 524         CMD_LINE_OPT_CONFIG_NUM,
 525         CMD_LINE_OPT_ETH_DEST_NUM,
 526         CMD_LINE_OPT_NO_NUMA_NUM,
 527         CMD_LINE_OPT_IPV6_NUM,
 528         CMD_LINE_OPT_ENABLE_JUMBO_NUM,
 529         CMD_LINE_OPT_HASH_ENTRY_NUM_NUM,
 530         CMD_LINE_OPT_PARSE_PTYPE_NUM,
 531         CMD_LINE_OPT_PARSE_PER_PORT_POOL,
 532         CMD_LINE_OPT_MODE_NUM,
 533         CMD_LINE_OPT_EVENTQ_SYNC_NUM,
 534         CMD_LINE_OPT_EVENT_ETH_RX_QUEUES_NUM,
 535 };
 536
 537 static const struct option lgopts[] = {
 538         {CMD_LINE_OPT_CONFIG, 1, 0, CMD_LINE_OPT_CONFIG_NUM},
 539         {CMD_LINE_OPT_ETH_DEST, 1, 0, CMD_LINE_OPT_ETH_DEST_NUM},
 540         {CMD_LINE_OPT_NO_NUMA, 0, 0, CMD_LINE_OPT_NO_NUMA_NUM},
 541         {CMD_LINE_OPT_IPV6, 0, 0, CMD_LINE_OPT_IPV6_NUM},
 542         {CMD_LINE_OPT_ENABLE_JUMBO, 0, 0, CMD_LINE_OPT_ENABLE_JUMBO_NUM},
 543         {CMD_LINE_OPT_HASH_ENTRY_NUM, 1, 0, CMD_LINE_OPT_HASH_ENTRY_NUM_NUM},
 544         {CMD_LINE_OPT_PARSE_PTYPE, 0, 0, CMD_LINE_OPT_PARSE_PTYPE_NUM},
 545         {CMD_LINE_OPT_PER_PORT_POOL, 0, 0, CMD_LINE_OPT_PARSE_PER_PORT_POOL},
 546         {CMD_LINE_OPT_MODE, 1, 0, CMD_LINE_OPT_MODE_NUM},
 547         {CMD_LINE_OPT_EVENTQ_SYNC, 1, 0, CMD_LINE_OPT_EVENTQ_SYNC_NUM},
 548         {CMD_LINE_OPT_EVENT_ETH_RX_QUEUES, 1, 0,
 549                                         CMD_LINE_OPT_EVENT_ETH_RX_QUEUES_NUM},
 550         {NULL, 0, 0, 0}
 551 };
 552
 553 /*
 554  * This expression is used to calculate the number of mbufs needed
 555  * depending on user input, taking  into account memory for rx and
 556  * tx hardware rings, cache per lcore and mtable per port per lcore.
 557  * RTE_MAX is used to ensure that NB_MBUF never goes below a minimum
 558  * value of 8192
 559  */
 560 #define NB_MBUF(nports) RTE_MAX(        \
 561         (nports*nb_rx_queue*nb_rxd +            \
 562         nports*nb_lcores*MAX_PKT_BURST +        \
 563         nports*n_tx_queue*nb_txd +              \
 564         nb_lcores*MEMPOOL_CACHE_SIZE),          \
 565         (unsigned)8192)
 566
 567 /* Parse the argument given in the command line of the application */
 568 static int
 569 parse_args(int argc, char **argv)
 570 {
 571         int opt, ret;
 572         char **argvopt;
 573         int option_index;
 574         char *prgname = argv[0];
 575         uint8_t lcore_params = 0;
 576         uint8_t eventq_sched = 0;
 577         uint8_t eth_rx_q = 0;
 578         struct l3fwd_event_resources *evt_rsrc = l3fwd_get_eventdev_rsrc();
 579
 580         argvopt = argv;
 581
 582         /* Error or normal output strings. */
 583         while ((opt = getopt_long(argc, argvopt, short_options,
 584                                 lgopts, &option_index)) != EOF) {
 585
 586                 switch (opt) {
 587                 /* portmask */
 588                 case 'p':
 589                         enabled_port_mask = parse_portmask(optarg);
 590                         if (enabled_port_mask == 0) {
 591                                 fprintf(stderr, "Invalid portmask\n");
 592                                 print_usage(prgname);
 593                                 return -1;
 594                         }
 595                         break;
 596
 597                 case 'P':
 598                         promiscuous_on = 1;
 599                         break;
 600
 601                 case 'E':
 602                         l3fwd_em_on = 1;
 603                         break;
 604
 605                 case 'L':
 606                         l3fwd_lpm_on = 1;
 607                         break;
 608
 609                 /* long options */
 610                 case CMD_LINE_OPT_CONFIG_NUM:
 611                         ret = parse_config(optarg);
 612                         if (ret) {
 613                                 fprintf(stderr, "Invalid config\n");
 614                                 print_usage(prgname);
 615                                 return -1;
 616                         }
 617                         lcore_params = 1;
 618                         break;
 619
 620                 case CMD_LINE_OPT_ETH_DEST_NUM:
 621                         parse_eth_dest(optarg);
 622                         break;
 623
 624                 case CMD_LINE_OPT_NO_NUMA_NUM:
 625                         numa_on = 0;
 626                         break;
 627
 628                 case CMD_LINE_OPT_IPV6_NUM:
 629                         ipv6 = 1;
 630                         break;
 631
 632                 case CMD_LINE_OPT_ENABLE_JUMBO_NUM: {
 633                         const struct option lenopts = {
 634                                 "max-pkt-len", required_argument, 0, 0
 635                         };
 636
 637                         port_conf.rxmode.offloads |= DEV_RX_OFFLOAD_JUMBO_FRAME;
 638                         port_conf.txmode.offloads |= DEV_TX_OFFLOAD_MULTI_SEGS;
 639
 640                         /*
 641                          * if no max-pkt-len set, use the default
 642                          * value RTE_ETHER_MAX_LEN.
 643                          */
 644                         if (getopt_long(argc, argvopt, "",
 645                                         &lenopts, &option_index) == 0) {
 646                                 ret = parse_max_pkt_len(optarg);
 647                                 if (ret < 64 || ret > MAX_JUMBO_PKT_LEN) {
 648                                         fprintf(stderr,
 649                                                 "invalid maximum packet length\n");
 650                                         print_usage(prgname);
 651                                         return -1;
 652                                 }
 653                                 port_conf.rxmode.max_rx_pkt_len = ret;
 654                         }
 655                         break;
 656                 }
 657
 658                 case CMD_LINE_OPT_HASH_ENTRY_NUM_NUM:
 659                         ret = parse_hash_entry_number(optarg);
 660                         if ((ret > 0) && (ret <= L3FWD_HASH_ENTRIES)) {
 661                                 hash_entry_number = ret;
 662                         } else {
 663                                 fprintf(stderr, "invalid hash entry number\n");
 664                                 print_usage(prgname);
 665                                 return -1;
 666                         }
 667                         break;
 668
 669                 case CMD_LINE_OPT_PARSE_PTYPE_NUM:
 670                         printf("soft parse-ptype is enabled\n");
 671                         parse_ptype = 1;
 672                         break;
 673
 674                 case CMD_LINE_OPT_PARSE_PER_PORT_POOL:
 675                         printf("per port buffer pool is enabled\n");
 676                         per_port_pool = 1;
 677                         break;
 678
 679                 case CMD_LINE_OPT_MODE_NUM:
 680                         parse_mode(optarg);
 681                         break;
 682
 683                 case CMD_LINE_OPT_EVENTQ_SYNC_NUM:
 684                         parse_eventq_sched(optarg);
 685                         eventq_sched = 1;
 686                         break;
 687
 688                 case CMD_LINE_OPT_EVENT_ETH_RX_QUEUES_NUM:
 689                         parse_event_eth_rx_queues(optarg);
 690                         eth_rx_q = 1;
 691                         break;
 692
 693                 default:
 694                         print_usage(prgname);
 695                         return -1;
 696                 }
 697         }
 698
 699         /* If both LPM and EM are selected, return error. */
 700         if (l3fwd_lpm_on && l3fwd_em_on) {
 701                 fprintf(stderr, "LPM and EM are mutually exclusive, select only one\n");
 702                 return -1;
 703         }
 704
 705         if (evt_rsrc->enabled && lcore_params) {
 706                 fprintf(stderr, "lcore config is not valid when event mode is selected\n");
 707                 return -1;
 708         }
 709
 710         if (!evt_rsrc->enabled && eth_rx_q) {
 711                 fprintf(stderr, "eth_rx_queues is valid only when event mode is selected\n");
 712                 return -1;
 713         }
 714
 715         if (!evt_rsrc->enabled && eventq_sched) {
 716                 fprintf(stderr, "eventq_sched is valid only when event mode is selected\n");
 717                 return -1;
 718         }
 719
 720         /*
 721          * Nothing is selected, pick longest-prefix match
 722          * as default match.
 723          */
 724         if (!l3fwd_lpm_on && !l3fwd_em_on) {
 725                 fprintf(stderr, "LPM or EM none selected, default LPM on\n");
 726                 l3fwd_lpm_on = 1;
 727         }
 728
 729         /*
 730          * ipv6 and hash flags are valid only for
 731          * exact macth, reset them to default for
 732          * longest-prefix match.
 733          */
 734         if (l3fwd_lpm_on) {
 735                 ipv6 = 0;
 736                 hash_entry_number = HASH_ENTRY_NUMBER_DEFAULT;
 737         }
 738
 739         if (optind >= 0)
 740                 argv[optind-1] = prgname;
 741
 742         ret = optind-1;
 743         optind = 1; /* reset getopt lib */
 744         return ret;
 745 }
 746
 747 static void
 748 print_ethaddr(const char *name, const struct rte_ether_addr *eth_addr)
 749 {
 750         char buf[RTE_ETHER_ADDR_FMT_SIZE];
 751         rte_ether_format_addr(buf, RTE_ETHER_ADDR_FMT_SIZE, eth_addr);
 752         printf("%s%s", name, buf);
 753 }
 754
 755 static int
 756 init_mem(uint16_t portid, unsigned int nb_mbuf)
 757 {
 758         struct lcore_conf *qconf;
 759         int socketid;
 760         unsigned lcore_id;
 761         char s[64];
 762
 763         for (lcore_id = 0; lcore_id < RTE_MAX_LCORE; lcore_id++) {
 764                 if (rte_lcore_is_enabled(lcore_id) == 0)
 765                         continue;
 766
 767                 if (numa_on)
 768                         socketid = rte_lcore_to_socket_id(lcore_id);
 769                 else
 770                         socketid = 0;
 771
 772                 if (socketid >= NB_SOCKETS) {
 773                         rte_exit(EXIT_FAILURE,
 774                                 "Socket %d of lcore %u is out of range %d\n",
 775                                 socketid, lcore_id, NB_SOCKETS);
 776                 }
 777
 778                 if (pktmbuf_pool[portid][socketid] == NULL) {
 779                         snprintf(s, sizeof(s), "mbuf_pool_%d:%d",
 780                                  portid, socketid);
 781                         pktmbuf_pool[portid][socketid] =
 782                                 rte_pktmbuf_pool_create(s, nb_mbuf,
 783                                         MEMPOOL_CACHE_SIZE, 0,
 784                                         RTE_MBUF_DEFAULT_BUF_SIZE, socketid);
 785                         if (pktmbuf_pool[portid][socketid] == NULL)
 786                                 rte_exit(EXIT_FAILURE,
 787                                         "Cannot init mbuf pool on socket %d\n",
 788                                         socketid);
 789                         else
 790                                 printf("Allocated mbuf pool on socket %d\n",
 791                                         socketid);
 792
 793                         /* Setup either LPM or EM(f.e Hash). But, only once per
 794                          * available socket.
 795                          */
 796                         if (!lkp_per_socket[socketid]) {
 797                                 l3fwd_lkp.setup(socketid);
 798                                 lkp_per_socket[socketid] = 1;
 799                         }
 800                 }
 801                 qconf = &lcore_conf[lcore_id];
 802                 qconf->ipv4_lookup_struct =
 803                         l3fwd_lkp.get_ipv4_lookup_struct(socketid);
 804                 qconf->ipv6_lookup_struct =
 805                         l3fwd_lkp.get_ipv6_lookup_struct(socketid);
 806         }
 807         return 0;
 808 }
 809
 810 /* Check the link status of all ports in up to 9s, and print them finally */
 811 static void
 812 check_all_ports_link_status(uint32_t port_mask)
 813 {
 814 #define CHECK_INTERVAL 100 /* 100ms */
 815 #define MAX_CHECK_TIME 90 /* 9s (90 * 100ms) in total */
 816         uint16_t portid;
 817         uint8_t count, all_ports_up, print_flag = 0;
 818         struct rte_eth_link link;
 819         int ret;
 820
 821         printf("\nChecking link status");
 822         fflush(stdout);
 823         for (count = 0; count <= MAX_CHECK_TIME; count++) {
 824                 if (force_quit)
 825                         return;
 826                 all_ports_up = 1;
 827                 RTE_ETH_FOREACH_DEV(portid) {
 828                         if (force_quit)
 829                                 return;
 830                         if ((port_mask & (1 << portid)) == 0)
 831                                 continue;
 832                         memset(&link, 0, sizeof(link));
 833                         ret = rte_eth_link_get_nowait(portid, &link);
 834                         if (ret < 0) {
 835                                 all_ports_up = 0;
 836                                 if (print_flag == 1)
 837                                         printf("Port %u link get failed: %s\n",
 838                                                 portid, rte_strerror(-ret));
 839                                 continue;
 840                         }
 841                         /* print link status if flag set */
 842                         if (print_flag == 1) {
 843                                 if (link.link_status)
 844                                         printf(
 845                                         "Port%d Link Up. Speed %u Mbps -%s\n",
 846                                                 portid, link.link_speed,
 847                                 (link.link_duplex == ETH_LINK_FULL_DUPLEX) ?
 848                                         ("full-duplex") : ("half-duplex\n"));
 849                                 else
 850                                         printf("Port %d Link Down\n", portid);
 851                                 continue;
 852                         }
 853                         /* clear all_ports_up flag if any link down */
 854                         if (link.link_status == ETH_LINK_DOWN) {
 855                                 all_ports_up = 0;
 856                                 break;
 857                         }
 858                 }
 859                 /* after finally printing all link status, get out */
 860                 if (print_flag == 1)
 861                         break;
 862
 863                 if (all_ports_up == 0) {
 864                         printf(".");
 865                         fflush(stdout);
 866                         rte_delay_ms(CHECK_INTERVAL);
 867                 }
 868
 869                 /* set the print_flag if all ports up or timeout */
 870                 if (all_ports_up == 1 || count == (MAX_CHECK_TIME - 1)) {
 871                         print_flag = 1;
 872                         printf("done\n");
 873                 }
 874         }
 875 }
 876
 877 static void
 878 signal_handler(int signum)
 879 {
 880         if (signum == SIGINT || signum == SIGTERM) {
 881                 printf("\n\nSignal %d received, preparing to exit...\n",
 882                                 signum);
 883                 force_quit = true;
 884         }
 885 }
 886
 887 static int
 888 prepare_ptype_parser(uint16_t portid, uint16_t queueid)
 889 {
 890         if (parse_ptype) {
 891                 printf("Port %d: softly parse packet type info\n", portid);
 892                 if (rte_eth_add_rx_callback(portid, queueid,
 893                                             l3fwd_lkp.cb_parse_ptype,
 894                                             NULL))
 895                         return 1;
 896
 897                 printf("Failed to add rx callback: port=%d\n", portid);
 898                 return 0;
 899         }
 900
 901         if (l3fwd_lkp.check_ptype(portid))
 902                 return 1;
 903
 904         printf("port %d cannot parse packet type, please add --%s\n",
 905                portid, CMD_LINE_OPT_PARSE_PTYPE);
 906         return 0;
 907 }
 908
 909 int
 910 main(int argc, char **argv)
 911 {
 912         struct l3fwd_event_resources *evt_rsrc;
 913         struct lcore_conf *qconf;
 914         struct rte_eth_dev_info dev_info;
 915         struct rte_eth_txconf *txconf;
 916         int ret;
 917         unsigned nb_ports;
 918         uint16_t queueid, portid;
 919         unsigned lcore_id;
 920         uint32_t n_tx_queue, nb_lcores;
 921         uint8_t nb_rx_queue, queue, socketid;
 922
 923         /* init EAL */
 924         ret = rte_eal_init(argc, argv);
 925         if (ret < 0)
 926                 rte_exit(EXIT_FAILURE, "Invalid EAL parameters\n");
 927         argc -= ret;
 928         argv += ret;
 929
 930         force_quit = false;
 931         signal(SIGINT, signal_handler);
 932         signal(SIGTERM, signal_handler);
 933
 934         /* pre-init dst MACs for all ports to 02:00:00:00:00:xx */
 935         for (portid = 0; portid < RTE_MAX_ETHPORTS; portid++) {
 936                 dest_eth_addr[portid] =
 937                         RTE_ETHER_LOCAL_ADMIN_ADDR + ((uint64_t)portid << 40);
 938                 *(uint64_t *)(val_eth + portid) = dest_eth_addr[portid];
 939         }
 940
 941         evt_rsrc = l3fwd_get_eventdev_rsrc();
 942         RTE_SET_USED(evt_rsrc);
 943         /* parse application arguments (after the EAL ones) */
 944         ret = parse_args(argc, argv);
 945         if (ret < 0)
 946                 rte_exit(EXIT_FAILURE, "Invalid L3FWD parameters\n");
 947
 948         /* Configure eventdev parameters if user has requested */
 949         l3fwd_event_resource_setup();
 950
 951         if (check_lcore_params() < 0)
 952                 rte_exit(EXIT_FAILURE, "check_lcore_params failed\n");
 953
 954         ret = init_lcore_rx_queues();
 955         if (ret < 0)
 956                 rte_exit(EXIT_FAILURE, "init_lcore_rx_queues failed\n");
 957
 958         nb_ports = rte_eth_dev_count_avail();
 959
 960         if (check_port_config() < 0)
 961                 rte_exit(EXIT_FAILURE, "check_port_config failed\n");
 962
 963         nb_lcores = rte_lcore_count();
 964
 965         /* Setup function pointers for lookup method. */
 966         setup_l3fwd_lookup_tables();
 967
 968         /* initialize all ports */
 969         RTE_ETH_FOREACH_DEV(portid) {
 970                 struct rte_eth_conf local_port_conf = port_conf;
 971
 972                 /* skip ports that are not enabled */
 973                 if ((enabled_port_mask & (1 << portid)) == 0) {
 974                         printf("\nSkipping disabled port %d\n", portid);
 975                         continue;
 976                 }
 977
 978                 /* init port */
 979                 printf("Initializing port %d ... ", portid );
 980                 fflush(stdout);
 981
 982                 nb_rx_queue = get_port_n_rx_queues(portid);
 983                 n_tx_queue = nb_lcores;
 984                 if (n_tx_queue > MAX_TX_QUEUE_PER_PORT)
 985                         n_tx_queue = MAX_TX_QUEUE_PER_PORT;
 986                 printf("Creating queues: nb_rxq=%d nb_txq=%u... ",
 987                         nb_rx_queue, (unsigned)n_tx_queue );
 988
 989                 ret = rte_eth_dev_info_get(portid, &dev_info);
 990                 if (ret != 0)
 991                         rte_exit(EXIT_FAILURE,
 992                                 "Error during getting device (port %u) info: %s\n",
 993                                 portid, strerror(-ret));
 994
 995                 if (dev_info.tx_offload_capa & DEV_TX_OFFLOAD_MBUF_FAST_FREE)
 996                         local_port_conf.txmode.offloads |=
 997                                 DEV_TX_OFFLOAD_MBUF_FAST_FREE;
 998
 999                 local_port_conf.rx_adv_conf.rss_conf.rss_hf &=
1000                         dev_info.flow_type_rss_offloads;
1001                 if (local_port_conf.rx_adv_conf.rss_conf.rss_hf !=
1002                                 port_conf.rx_adv_conf.rss_conf.rss_hf) {
1003                         printf("Port %u modified RSS hash function based on hardware support,"
1004                                 "requested:%#"PRIx64" configured:%#"PRIx64"\n",
1005                                 portid,
1006                                 port_conf.rx_adv_conf.rss_conf.rss_hf,
1007                                 local_port_conf.rx_adv_conf.rss_conf.rss_hf);
1008                 }
1009
1010                 ret = rte_eth_dev_configure(portid, nb_rx_queue,
1011                                         (uint16_t)n_tx_queue, &local_port_conf);
1012                 if (ret < 0)
1013                         rte_exit(EXIT_FAILURE,
1014                                 "Cannot configure device: err=%d, port=%d\n",
1015                                 ret, portid);
1016
1017                 ret = rte_eth_dev_adjust_nb_rx_tx_desc(portid, &nb_rxd,
1018                                                        &nb_txd);
1019                 if (ret < 0)
1020                         rte_exit(EXIT_FAILURE,
1021                                  "Cannot adjust number of descriptors: err=%d, "
1022                                  "port=%d\n", ret, portid);
1023
1024                 ret = rte_eth_macaddr_get(portid, &ports_eth_addr[portid]);
1025                 if (ret < 0)
1026                         rte_exit(EXIT_FAILURE,
1027                                  "Cannot get MAC address: err=%d, port=%d\n",
1028                                  ret, portid);
1029
1030                 print_ethaddr(" Address:", &ports_eth_addr[portid]);
1031                 printf(", ");
1032                 print_ethaddr("Destination:",
1033                         (const struct rte_ether_addr *)&dest_eth_addr[portid]);
1034                 printf(", ");
1035
1036                 /*
1037                  * prepare src MACs for each port.
1038                  */
1039                 rte_ether_addr_copy(&ports_eth_addr[portid],
1040                         (struct rte_ether_addr *)(val_eth + portid) + 1);
1041
1042                 /* init memory */
1043                 if (!per_port_pool) {
1044                         /* portid = 0; this is *not* signifying the first port,
1045                          * rather, it signifies that portid is ignored.
1046                          */
1047                         ret = init_mem(0, NB_MBUF(nb_ports));
1048                 } else {
1049                         ret = init_mem(portid, NB_MBUF(1));
1050                 }
1051                 if (ret < 0)
1052                         rte_exit(EXIT_FAILURE, "init_mem failed\n");
1053
1054                 /* init one TX queue per couple (lcore,port) */
1055                 queueid = 0;
1056                 for (lcore_id = 0; lcore_id < RTE_MAX_LCORE; lcore_id++) {
1057                         if (rte_lcore_is_enabled(lcore_id) == 0)
1058                                 continue;
1059
1060                         if (numa_on)
1061                                 socketid =
1062                                 (uint8_t)rte_lcore_to_socket_id(lcore_id);
1063                         else
1064                                 socketid = 0;
1065
1066                         printf("txq=%u,%d,%d ", lcore_id, queueid, socketid);
1067                         fflush(stdout);
1068
1069                         txconf = &dev_info.default_txconf;
1070                         txconf->offloads = local_port_conf.txmode.offloads;
1071                         ret = rte_eth_tx_queue_setup(portid, queueid, nb_txd,
1072                                                      socketid, txconf);
1073                         if (ret < 0)
1074                                 rte_exit(EXIT_FAILURE,
1075                                         "rte_eth_tx_queue_setup: err=%d, "
1076                                         "port=%d\n", ret, portid);
1077
1078                         qconf = &lcore_conf[lcore_id];
1079                         qconf->tx_queue_id[portid] = queueid;
1080                         queueid++;
1081
1082                         qconf->tx_port_id[qconf->n_tx_port] = portid;
1083                         qconf->n_tx_port++;
1084                 }
1085                 printf("\n");
1086         }
1087
1088         for (lcore_id = 0; lcore_id < RTE_MAX_LCORE; lcore_id++) {
1089                 if (rte_lcore_is_enabled(lcore_id) == 0)
1090                         continue;
1091                 qconf = &lcore_conf[lcore_id];
1092                 printf("\nInitializing rx queues on lcore %u ... ", lcore_id );
1093                 fflush(stdout);
1094                 /* init RX queues */
1095                 for(queue = 0; queue < qconf->n_rx_queue; ++queue) {
1096                         struct rte_eth_rxconf rxq_conf;
1097
1098                         portid = qconf->rx_queue_list[queue].port_id;
1099                         queueid = qconf->rx_queue_list[queue].queue_id;
1100
1101                         if (numa_on)
1102                                 socketid =
1103                                 (uint8_t)rte_lcore_to_socket_id(lcore_id);
1104                         else
1105                                 socketid = 0;
1106
1107                         printf("rxq=%d,%d,%d ", portid, queueid, socketid);
1108                         fflush(stdout);
1109
1110                         ret = rte_eth_dev_info_get(portid, &dev_info);
1111                         if (ret != 0)
1112                                 rte_exit(EXIT_FAILURE,
1113                                         "Error during getting device (port %u) info: %s\n",
1114                                         portid, strerror(-ret));
1115
1116                         rxq_conf = dev_info.default_rxconf;
1117                         rxq_conf.offloads = port_conf.rxmode.offloads;
1118                         if (!per_port_pool)
1119                                 ret = rte_eth_rx_queue_setup(portid, queueid,
1120                                                 nb_rxd, socketid,
1121                                                 &rxq_conf,
1122                                                 pktmbuf_pool[0][socketid]);
1123                         else
1124                                 ret = rte_eth_rx_queue_setup(portid, queueid,
1125                                                 nb_rxd, socketid,
1126                                                 &rxq_conf,
1127                                                 pktmbuf_pool[portid][socketid]);
1128                         if (ret < 0)
1129                                 rte_exit(EXIT_FAILURE,
1130                                 "rte_eth_rx_queue_setup: err=%d, port=%d\n",
1131                                 ret, portid);
1132                 }
1133         }
1134
1135         printf("\n");
1136
1137         /* start ports */
1138         RTE_ETH_FOREACH_DEV(portid) {
1139                 if ((enabled_port_mask & (1 << portid)) == 0) {
1140                         continue;
1141                 }
1142                 /* Start device */
1143                 ret = rte_eth_dev_start(portid);
1144                 if (ret < 0)
1145                         rte_exit(EXIT_FAILURE,
1146                                 "rte_eth_dev_start: err=%d, port=%d\n",
1147                                 ret, portid);
1148
1149                 /*
1150                  * If enabled, put device in promiscuous mode.
1151                  * This allows IO forwarding mode to forward packets
1152                  * to itself through 2 cross-connected  ports of the
1153                  * target machine.
1154                  */
1155                 if (promiscuous_on) {
1156                         ret = rte_eth_promiscuous_enable(portid);
1157                         if (ret != 0)
1158                                 rte_exit(EXIT_FAILURE,
1159                                         "rte_eth_promiscuous_enable: err=%s, port=%u\n",
1160                                         rte_strerror(-ret), portid);
1161                 }
1162         }
1163
1164         printf("\n");
1165
1166         for (lcore_id = 0; lcore_id < RTE_MAX_LCORE; lcore_id++) {
1167                 if (rte_lcore_is_enabled(lcore_id) == 0)
1168                         continue;
1169                 qconf = &lcore_conf[lcore_id];
1170                 for (queue = 0; queue < qconf->n_rx_queue; ++queue) {
1171                         portid = qconf->rx_queue_list[queue].port_id;
1172                         queueid = qconf->rx_queue_list[queue].queue_id;
1173                         if (prepare_ptype_parser(portid, queueid) == 0)
1174                                 rte_exit(EXIT_FAILURE, "ptype check fails\n");
1175                 }
1176         }
1177
1178
1179         check_all_ports_link_status(enabled_port_mask);
1180
1181         ret = 0;
1182         /* launch per-lcore init on every lcore */
1183         rte_eal_mp_remote_launch(l3fwd_lkp.main_loop, NULL, CALL_MASTER);
1184         RTE_LCORE_FOREACH_SLAVE(lcore_id) {
1185                 if (rte_eal_wait_lcore(lcore_id) < 0) {
1186                         ret = -1;
1187                         break;
1188                 }
1189         }
1190
1191         /* stop ports */
1192         RTE_ETH_FOREACH_DEV(portid) {
1193                 if ((enabled_port_mask & (1 << portid)) == 0)
1194                         continue;
1195                 printf("Closing port %d...", portid);
1196                 rte_eth_dev_stop(portid);
1197                 rte_eth_dev_close(portid);
1198                 printf(" Done\n");
1199         }
1200         printf("Bye...\n");
1201
1202         return ret;
1203 }