examples/l3fwd/main.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(c) 2010-2021 Intel Corporation
   3  */
   4
   5 #include <stdio.h>
   6 #include <stdlib.h>
   7 #include <stdint.h>
   8 #include <inttypes.h>
   9 #include <sys/types.h>
  10 #include <string.h>
  11 #include <sys/queue.h>
  12 #include <stdarg.h>
  13 #include <errno.h>
  14 #include <getopt.h>
  15 #include <signal.h>
  16 #include <stdbool.h>
  17
  18 #include <rte_common.h>
  19 #include <rte_vect.h>
  20 #include <rte_byteorder.h>
  21 #include <rte_log.h>
  22 #include <rte_malloc.h>
  23 #include <rte_memory.h>
  24 #include <rte_memcpy.h>
  25 #include <rte_eal.h>
  26 #include <rte_launch.h>
  27 #include <rte_atomic.h>
  28 #include <rte_cycles.h>
  29 #include <rte_prefetch.h>
  30 #include <rte_lcore.h>
  31 #include <rte_per_lcore.h>
  32 #include <rte_branch_prediction.h>
  33 #include <rte_interrupts.h>
  34 #include <rte_random.h>
  35 #include <rte_debug.h>
  36 #include <rte_ether.h>
  37 #include <rte_mempool.h>
  38 #include <rte_mbuf.h>
  39 #include <rte_ip.h>
  40 #include <rte_tcp.h>
  41 #include <rte_udp.h>
  42 #include <rte_string_fns.h>
  43 #include <rte_cpuflags.h>
  44
  45 #include <cmdline_parse.h>
  46 #include <cmdline_parse_etheraddr.h>
  47
  48 #include "l3fwd.h"
  49 #include "l3fwd_event.h"
  50 #include "l3fwd_route.h"
  51
  52 #define MAX_TX_QUEUE_PER_PORT RTE_MAX_LCORE
  53 #define MAX_RX_QUEUE_PER_PORT 128
  54
  55 #define MAX_LCORE_PARAMS 1024
  56
  57 /* Static global variables used within this file. */
  58 static uint16_t nb_rxd = RTE_TEST_RX_DESC_DEFAULT;
  59 static uint16_t nb_txd = RTE_TEST_TX_DESC_DEFAULT;
  60
  61 /**< Ports set in promiscuous mode off by default. */
  62 static int promiscuous_on;
  63
  64 /* Select Longest-Prefix, Exact match or Forwarding Information Base. */
  65 enum L3FWD_LOOKUP_MODE {
  66         L3FWD_LOOKUP_DEFAULT,
  67         L3FWD_LOOKUP_LPM,
  68         L3FWD_LOOKUP_EM,
  69         L3FWD_LOOKUP_FIB
  70 };
  71 static enum L3FWD_LOOKUP_MODE lookup_mode;
  72
  73 /* Global variables. */
  74
  75 static int numa_on = 1; /**< NUMA is enabled by default. */
  76 static int parse_ptype; /**< Parse packet type using rx callback, and */
  77                         /**< disabled by default */
  78 static int per_port_pool; /**< Use separate buffer pools per port; disabled */
  79                           /**< by default */
  80
  81 volatile bool force_quit;
  82
  83 /* ethernet addresses of ports */
  84 uint64_t dest_eth_addr[RTE_MAX_ETHPORTS];
  85 struct rte_ether_addr ports_eth_addr[RTE_MAX_ETHPORTS];
  86
  87 xmm_t val_eth[RTE_MAX_ETHPORTS];
  88
  89 /* mask of enabled ports */
  90 uint32_t enabled_port_mask;
  91
  92 /* Used only in exact match mode. */
  93 int ipv6; /**< ipv6 is false by default. */
  94 uint32_t hash_entry_number = HASH_ENTRY_NUMBER_DEFAULT;
  95
  96 struct lcore_conf lcore_conf[RTE_MAX_LCORE];
  97
  98 struct lcore_params {
  99         uint16_t port_id;
 100         uint8_t queue_id;
 101         uint8_t lcore_id;
 102 } __rte_cache_aligned;
 103
 104 static struct lcore_params lcore_params_array[MAX_LCORE_PARAMS];
 105 static struct lcore_params lcore_params_array_default[] = {
 106         {0, 0, 2},
 107         {0, 1, 2},
 108         {0, 2, 2},
 109         {1, 0, 2},
 110         {1, 1, 2},
 111         {1, 2, 2},
 112         {2, 0, 2},
 113         {3, 0, 3},
 114         {3, 1, 3},
 115 };
 116
 117 static struct lcore_params * lcore_params = lcore_params_array_default;
 118 static uint16_t nb_lcore_params = sizeof(lcore_params_array_default) /
 119                                 sizeof(lcore_params_array_default[0]);
 120
 121 static struct rte_eth_conf port_conf = {
 122         .rxmode = {
 123                 .mq_mode = ETH_MQ_RX_RSS,
 124                 .max_rx_pkt_len = RTE_ETHER_MAX_LEN,
 125                 .split_hdr_size = 0,
 126                 .offloads = DEV_RX_OFFLOAD_CHECKSUM,
 127         },
 128         .rx_adv_conf = {
 129                 .rss_conf = {
 130                         .rss_key = NULL,
 131                         .rss_hf = ETH_RSS_IP,
 132                 },
 133         },
 134         .txmode = {
 135                 .mq_mode = ETH_MQ_TX_NONE,
 136         },
 137 };
 138
 139 static struct rte_mempool *pktmbuf_pool[RTE_MAX_ETHPORTS][NB_SOCKETS];
 140 static uint8_t lkp_per_socket[NB_SOCKETS];
 141
 142 struct l3fwd_lkp_mode {
 143         void  (*setup)(int);
 144         int   (*check_ptype)(int);
 145         rte_rx_callback_fn cb_parse_ptype;
 146         int   (*main_loop)(void *);
 147         void* (*get_ipv4_lookup_struct)(int);
 148         void* (*get_ipv6_lookup_struct)(int);
 149 };
 150
 151 static struct l3fwd_lkp_mode l3fwd_lkp;
 152
 153 static struct l3fwd_lkp_mode l3fwd_em_lkp = {
 154         .setup                  = setup_hash,
 155         .check_ptype            = em_check_ptype,
 156         .cb_parse_ptype         = em_cb_parse_ptype,
 157         .main_loop              = em_main_loop,
 158         .get_ipv4_lookup_struct = em_get_ipv4_l3fwd_lookup_struct,
 159         .get_ipv6_lookup_struct = em_get_ipv6_l3fwd_lookup_struct,
 160 };
 161
 162 static struct l3fwd_lkp_mode l3fwd_lpm_lkp = {
 163         .setup                  = setup_lpm,
 164         .check_ptype            = lpm_check_ptype,
 165         .cb_parse_ptype         = lpm_cb_parse_ptype,
 166         .main_loop              = lpm_main_loop,
 167         .get_ipv4_lookup_struct = lpm_get_ipv4_l3fwd_lookup_struct,
 168         .get_ipv6_lookup_struct = lpm_get_ipv6_l3fwd_lookup_struct,
 169 };
 170
 171 static struct l3fwd_lkp_mode l3fwd_fib_lkp = {
 172         .setup                  = setup_fib,
 173         .check_ptype            = lpm_check_ptype,
 174         .cb_parse_ptype         = lpm_cb_parse_ptype,
 175         .main_loop              = fib_main_loop,
 176         .get_ipv4_lookup_struct = fib_get_ipv4_l3fwd_lookup_struct,
 177         .get_ipv6_lookup_struct = fib_get_ipv6_l3fwd_lookup_struct,
 178 };
 179
 180 /*
 181  * 198.18.0.0/16 are set aside for RFC2544 benchmarking (RFC5735).
 182  * 198.18.{0-7}.0/24 = Port {0-7}
 183  */
 184 const struct ipv4_l3fwd_route ipv4_l3fwd_route_array[] = {
 185         {RTE_IPV4(198, 18, 0, 0), 24, 0},
 186         {RTE_IPV4(198, 18, 1, 0), 24, 1},
 187         {RTE_IPV4(198, 18, 2, 0), 24, 2},
 188         {RTE_IPV4(198, 18, 3, 0), 24, 3},
 189         {RTE_IPV4(198, 18, 4, 0), 24, 4},
 190         {RTE_IPV4(198, 18, 5, 0), 24, 5},
 191         {RTE_IPV4(198, 18, 6, 0), 24, 6},
 192         {RTE_IPV4(198, 18, 7, 0), 24, 7},
 193 };
 194
 195 /*
 196  * 2001:200::/48 is IANA reserved range for IPv6 benchmarking (RFC5180).
 197  * 2001:200:0:{0-7}::/64 = Port {0-7}
 198  */
 199 const struct ipv6_l3fwd_route ipv6_l3fwd_route_array[] = {
 200         {{32, 1, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, 64, 0},
 201         {{32, 1, 2, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0}, 64, 1},
 202         {{32, 1, 2, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0}, 64, 2},
 203         {{32, 1, 2, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0}, 64, 3},
 204         {{32, 1, 2, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0}, 64, 4},
 205         {{32, 1, 2, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0}, 64, 5},
 206         {{32, 1, 2, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0}, 64, 6},
 207         {{32, 1, 2, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0}, 64, 7},
 208 };
 209
 210 /*
 211  * Setup lookup methods for forwarding.
 212  * Currently exact-match, longest-prefix-match and forwarding information
 213  * base are the supported ones.
 214  */
 215 static void
 216 setup_l3fwd_lookup_tables(void)
 217 {
 218         /* Setup HASH lookup functions. */
 219         if (lookup_mode == L3FWD_LOOKUP_EM)
 220                 l3fwd_lkp = l3fwd_em_lkp;
 221         /* Setup FIB lookup functions. */
 222         else if (lookup_mode == L3FWD_LOOKUP_FIB)
 223                 l3fwd_lkp = l3fwd_fib_lkp;
 224         /* Setup LPM lookup functions. */
 225         else
 226                 l3fwd_lkp = l3fwd_lpm_lkp;
 227 }
 228
 229 static int
 230 check_lcore_params(void)
 231 {
 232         uint8_t queue, lcore;
 233         uint16_t i;
 234         int socketid;
 235
 236         for (i = 0; i < nb_lcore_params; ++i) {
 237                 queue = lcore_params[i].queue_id;
 238                 if (queue >= MAX_RX_QUEUE_PER_PORT) {
 239                         printf("invalid queue number: %hhu\n", queue);
 240                         return -1;
 241                 }
 242                 lcore = lcore_params[i].lcore_id;
 243                 if (!rte_lcore_is_enabled(lcore)) {
 244                         printf("error: lcore %hhu is not enabled in lcore mask\n", lcore);
 245                         return -1;
 246                 }
 247                 if ((socketid = rte_lcore_to_socket_id(lcore) != 0) &&
 248                         (numa_on == 0)) {
 249                         printf("warning: lcore %hhu is on socket %d with numa off \n",
 250                                 lcore, socketid);
 251                 }
 252         }
 253         return 0;
 254 }
 255
 256 static int
 257 check_port_config(void)
 258 {
 259         uint16_t portid;
 260         uint16_t i;
 261
 262         for (i = 0; i < nb_lcore_params; ++i) {
 263                 portid = lcore_params[i].port_id;
 264                 if ((enabled_port_mask & (1 << portid)) == 0) {
 265                         printf("port %u is not enabled in port mask\n", portid);
 266                         return -1;
 267                 }
 268                 if (!rte_eth_dev_is_valid_port(portid)) {
 269                         printf("port %u is not present on the board\n", portid);
 270                         return -1;
 271                 }
 272         }
 273         return 0;
 274 }
 275
 276 static uint8_t
 277 get_port_n_rx_queues(const uint16_t port)
 278 {
 279         int queue = -1;
 280         uint16_t i;
 281
 282         for (i = 0; i < nb_lcore_params; ++i) {
 283                 if (lcore_params[i].port_id == port) {
 284                         if (lcore_params[i].queue_id == queue+1)
 285                                 queue = lcore_params[i].queue_id;
 286                         else
 287                                 rte_exit(EXIT_FAILURE, "queue ids of the port %d must be"
 288                                                 " in sequence and must start with 0\n",
 289                                                 lcore_params[i].port_id);
 290                 }
 291         }
 292         return (uint8_t)(++queue);
 293 }
 294
 295 static int
 296 init_lcore_rx_queues(void)
 297 {
 298         uint16_t i, nb_rx_queue;
 299         uint8_t lcore;
 300
 301         for (i = 0; i < nb_lcore_params; ++i) {
 302                 lcore = lcore_params[i].lcore_id;
 303                 nb_rx_queue = lcore_conf[lcore].n_rx_queue;
 304                 if (nb_rx_queue >= MAX_RX_QUEUE_PER_LCORE) {
 305                         printf("error: too many queues (%u) for lcore: %u\n",
 306                                 (unsigned)nb_rx_queue + 1, (unsigned)lcore);
 307                         return -1;
 308                 } else {
 309                         lcore_conf[lcore].rx_queue_list[nb_rx_queue].port_id =
 310                                 lcore_params[i].port_id;
 311                         lcore_conf[lcore].rx_queue_list[nb_rx_queue].queue_id =
 312                                 lcore_params[i].queue_id;
 313                         lcore_conf[lcore].n_rx_queue++;
 314                 }
 315         }
 316         return 0;
 317 }
 318
 319 /* display usage */
 320 static void
 321 print_usage(const char *prgname)
 322 {
 323         fprintf(stderr, "%s [EAL options] --"
 324                 " -p PORTMASK"
 325                 " [-P]"
 326                 " [--lookup]"
 327                 " --config (port,queue,lcore)[,(port,queue,lcore)]"
 328                 " [--eth-dest=X,MM:MM:MM:MM:MM:MM]"
 329                 " [--enable-jumbo [--max-pkt-len PKTLEN]]"
 330                 " [--no-numa]"
 331                 " [--hash-entry-num]"
 332                 " [--ipv6]"
 333                 " [--parse-ptype]"
 334                 " [--per-port-pool]"
 335                 " [--mode]"
 336                 " [--eventq-sched]"
 337                 " [-E]"
 338                 " [-L]\n\n"
 339
 340                 "  -p PORTMASK: Hexadecimal bitmask of ports to configure\n"
 341                 "  -P : Enable promiscuous mode\n"
 342                 "  --lookup: Select the lookup method\n"
 343                 "            Default: lpm\n"
 344                 "            Accepted: em (Exact Match), lpm (Longest Prefix Match), fib (Forwarding Information Base)\n"
 345                 "  --config (port,queue,lcore): Rx queue configuration\n"
 346                 "  --eth-dest=X,MM:MM:MM:MM:MM:MM: Ethernet destination for port X\n"
 347                 "  --enable-jumbo: Enable jumbo frames\n"
 348                 "  --max-pkt-len: Under the premise of enabling jumbo,\n"
 349                 "                 maximum packet length in decimal (64-9600)\n"
 350                 "  --no-numa: Disable numa awareness\n"
 351                 "  --hash-entry-num: Specify the hash entry number in hexadecimal to be setup\n"
 352                 "  --ipv6: Set if running ipv6 packets\n"
 353                 "  --parse-ptype: Set to use software to analyze packet type\n"
 354                 "  --per-port-pool: Use separate buffer pool per port\n"
 355                 "  --mode: Packet transfer mode for I/O, poll or eventdev\n"
 356                 "          Default mode = poll\n"
 357                 "  --eventq-sched: Event queue synchronization method\n"
 358                 "                  ordered, atomic or parallel.\n"
 359                 "                  Default: atomic\n"
 360                 "                  Valid only if --mode=eventdev\n"
 361                 "  --event-eth-rxqs: Number of ethernet RX queues per device.\n"
 362                 "                    Default: 1\n"
 363                 "                    Valid only if --mode=eventdev\n"
 364                 "  -E : Enable exact match, legacy flag please use --lookup=em instead\n"
 365                 "  -L : Enable longest prefix match, legacy flag please use --lookup=lpm instead\n\n",
 366                 prgname);
 367 }
 368
 369 static int
 370 parse_max_pkt_len(const char *pktlen)
 371 {
 372         char *end = NULL;
 373         unsigned long len;
 374
 375         /* parse decimal string */
 376         len = strtoul(pktlen, &end, 10);
 377         if ((pktlen[0] == '\0') || (end == NULL) || (*end != '\0'))
 378                 return -1;
 379
 380         if (len == 0)
 381                 return -1;
 382
 383         return len;
 384 }
 385
 386 static int
 387 parse_portmask(const char *portmask)
 388 {
 389         char *end = NULL;
 390         unsigned long pm;
 391
 392         /* parse hexadecimal string */
 393         pm = strtoul(portmask, &end, 16);
 394         if ((portmask[0] == '\0') || (end == NULL) || (*end != '\0'))
 395                 return 0;
 396
 397         return pm;
 398 }
 399
 400 static int
 401 parse_hash_entry_number(const char *hash_entry_num)
 402 {
 403         char *end = NULL;
 404         unsigned long hash_en;
 405         /* parse hexadecimal string */
 406         hash_en = strtoul(hash_entry_num, &end, 16);
 407         if ((hash_entry_num[0] == '\0') || (end == NULL) || (*end != '\0'))
 408                 return -1;
 409
 410         if (hash_en == 0)
 411                 return -1;
 412
 413         return hash_en;
 414 }
 415
 416 static int
 417 parse_config(const char *q_arg)
 418 {
 419         char s[256];
 420         const char *p, *p0 = q_arg;
 421         char *end;
 422         enum fieldnames {
 423                 FLD_PORT = 0,
 424                 FLD_QUEUE,
 425                 FLD_LCORE,
 426                 _NUM_FLD
 427         };
 428         unsigned long int_fld[_NUM_FLD];
 429         char *str_fld[_NUM_FLD];
 430         int i;
 431         unsigned size;
 432
 433         nb_lcore_params = 0;
 434
 435         while ((p = strchr(p0,'(')) != NULL) {
 436                 ++p;
 437                 if((p0 = strchr(p,')')) == NULL)
 438                         return -1;
 439
 440                 size = p0 - p;
 441                 if(size >= sizeof(s))
 442                         return -1;
 443
 444                 snprintf(s, sizeof(s), "%.*s", size, p);
 445                 if (rte_strsplit(s, sizeof(s), str_fld, _NUM_FLD, ',') != _NUM_FLD)
 446                         return -1;
 447                 for (i = 0; i < _NUM_FLD; i++){
 448                         errno = 0;
 449                         int_fld[i] = strtoul(str_fld[i], &end, 0);
 450                         if (errno != 0 || end == str_fld[i] || int_fld[i] > 255)
 451                                 return -1;
 452                 }
 453                 if (nb_lcore_params >= MAX_LCORE_PARAMS) {
 454                         printf("exceeded max number of lcore params: %hu\n",
 455                                 nb_lcore_params);
 456                         return -1;
 457                 }
 458                 lcore_params_array[nb_lcore_params].port_id =
 459                         (uint8_t)int_fld[FLD_PORT];
 460                 lcore_params_array[nb_lcore_params].queue_id =
 461                         (uint8_t)int_fld[FLD_QUEUE];
 462                 lcore_params_array[nb_lcore_params].lcore_id =
 463                         (uint8_t)int_fld[FLD_LCORE];
 464                 ++nb_lcore_params;
 465         }
 466         lcore_params = lcore_params_array;
 467         return 0;
 468 }
 469
 470 static void
 471 parse_eth_dest(const char *optarg)
 472 {
 473         uint16_t portid;
 474         char *port_end;
 475         uint8_t c, *dest, peer_addr[6];
 476
 477         errno = 0;
 478         portid = strtoul(optarg, &port_end, 10);
 479         if (errno != 0 || port_end == optarg || *port_end++ != ',')
 480                 rte_exit(EXIT_FAILURE,
 481                 "Invalid eth-dest: %s", optarg);
 482         if (portid >= RTE_MAX_ETHPORTS)
 483                 rte_exit(EXIT_FAILURE,
 484                 "eth-dest: port %d >= RTE_MAX_ETHPORTS(%d)\n",
 485                 portid, RTE_MAX_ETHPORTS);
 486
 487         if (cmdline_parse_etheraddr(NULL, port_end,
 488                 &peer_addr, sizeof(peer_addr)) < 0)
 489                 rte_exit(EXIT_FAILURE,
 490                 "Invalid ethernet address: %s\n",
 491                 port_end);
 492         dest = (uint8_t *)&dest_eth_addr[portid];
 493         for (c = 0; c < 6; c++)
 494                 dest[c] = peer_addr[c];
 495         *(uint64_t *)(val_eth + portid) = dest_eth_addr[portid];
 496 }
 497
 498 static void
 499 parse_mode(const char *optarg)
 500 {
 501         struct l3fwd_event_resources *evt_rsrc = l3fwd_get_eventdev_rsrc();
 502
 503         if (!strcmp(optarg, "poll"))
 504                 evt_rsrc->enabled = false;
 505         else if (!strcmp(optarg, "eventdev"))
 506                 evt_rsrc->enabled = true;
 507 }
 508
 509 static void
 510 parse_eventq_sched(const char *optarg)
 511 {
 512         struct l3fwd_event_resources *evt_rsrc = l3fwd_get_eventdev_rsrc();
 513
 514         if (!strcmp(optarg, "ordered"))
 515                 evt_rsrc->sched_type = RTE_SCHED_TYPE_ORDERED;
 516         if (!strcmp(optarg, "atomic"))
 517                 evt_rsrc->sched_type = RTE_SCHED_TYPE_ATOMIC;
 518         if (!strcmp(optarg, "parallel"))
 519                 evt_rsrc->sched_type = RTE_SCHED_TYPE_PARALLEL;
 520 }
 521
 522 static void
 523 parse_event_eth_rx_queues(const char *eth_rx_queues)
 524 {
 525         struct l3fwd_event_resources *evt_rsrc = l3fwd_get_eventdev_rsrc();
 526         char *end = NULL;
 527         uint8_t num_eth_rx_queues;
 528
 529         /* parse decimal string */
 530         num_eth_rx_queues = strtoul(eth_rx_queues, &end, 10);
 531         if ((eth_rx_queues[0] == '\0') || (end == NULL) || (*end != '\0'))
 532                 return;
 533
 534         if (num_eth_rx_queues == 0)
 535                 return;
 536
 537         evt_rsrc->eth_rx_queues = num_eth_rx_queues;
 538 }
 539
 540 static int
 541 parse_lookup(const char *optarg)
 542 {
 543         if (!strcmp(optarg, "em"))
 544                 lookup_mode = L3FWD_LOOKUP_EM;
 545         else if (!strcmp(optarg, "lpm"))
 546                 lookup_mode = L3FWD_LOOKUP_LPM;
 547         else if (!strcmp(optarg, "fib"))
 548                 lookup_mode = L3FWD_LOOKUP_FIB;
 549         else {
 550                 fprintf(stderr, "Invalid lookup option! Accepted options: em, lpm, fib\n");
 551                 return -1;
 552         }
 553         return 0;
 554 }
 555
 556 #define MAX_JUMBO_PKT_LEN  9600
 557
 558 static const char short_options[] =
 559         "p:"  /* portmask */
 560         "P"   /* promiscuous */
 561         "L"   /* legacy enable long prefix match */
 562         "E"   /* legacy enable exact match */
 563         ;
 564
 565 #define CMD_LINE_OPT_CONFIG "config"
 566 #define CMD_LINE_OPT_ETH_DEST "eth-dest"
 567 #define CMD_LINE_OPT_NO_NUMA "no-numa"
 568 #define CMD_LINE_OPT_IPV6 "ipv6"
 569 #define CMD_LINE_OPT_ENABLE_JUMBO "enable-jumbo"
 570 #define CMD_LINE_OPT_HASH_ENTRY_NUM "hash-entry-num"
 571 #define CMD_LINE_OPT_PARSE_PTYPE "parse-ptype"
 572 #define CMD_LINE_OPT_PER_PORT_POOL "per-port-pool"
 573 #define CMD_LINE_OPT_MODE "mode"
 574 #define CMD_LINE_OPT_EVENTQ_SYNC "eventq-sched"
 575 #define CMD_LINE_OPT_EVENT_ETH_RX_QUEUES "event-eth-rxqs"
 576 #define CMD_LINE_OPT_LOOKUP "lookup"
 577 enum {
 578         /* long options mapped to a short option */
 579
 580         /* first long only option value must be >= 256, so that we won't
 581          * conflict with short options */
 582         CMD_LINE_OPT_MIN_NUM = 256,
 583         CMD_LINE_OPT_CONFIG_NUM,
 584         CMD_LINE_OPT_ETH_DEST_NUM,
 585         CMD_LINE_OPT_NO_NUMA_NUM,
 586         CMD_LINE_OPT_IPV6_NUM,
 587         CMD_LINE_OPT_ENABLE_JUMBO_NUM,
 588         CMD_LINE_OPT_HASH_ENTRY_NUM_NUM,
 589         CMD_LINE_OPT_PARSE_PTYPE_NUM,
 590         CMD_LINE_OPT_PARSE_PER_PORT_POOL,
 591         CMD_LINE_OPT_MODE_NUM,
 592         CMD_LINE_OPT_EVENTQ_SYNC_NUM,
 593         CMD_LINE_OPT_EVENT_ETH_RX_QUEUES_NUM,
 594         CMD_LINE_OPT_LOOKUP_NUM,
 595 };
 596
 597 static const struct option lgopts[] = {
 598         {CMD_LINE_OPT_CONFIG, 1, 0, CMD_LINE_OPT_CONFIG_NUM},
 599         {CMD_LINE_OPT_ETH_DEST, 1, 0, CMD_LINE_OPT_ETH_DEST_NUM},
 600         {CMD_LINE_OPT_NO_NUMA, 0, 0, CMD_LINE_OPT_NO_NUMA_NUM},
 601         {CMD_LINE_OPT_IPV6, 0, 0, CMD_LINE_OPT_IPV6_NUM},
 602         {CMD_LINE_OPT_ENABLE_JUMBO, 0, 0, CMD_LINE_OPT_ENABLE_JUMBO_NUM},
 603         {CMD_LINE_OPT_HASH_ENTRY_NUM, 1, 0, CMD_LINE_OPT_HASH_ENTRY_NUM_NUM},
 604         {CMD_LINE_OPT_PARSE_PTYPE, 0, 0, CMD_LINE_OPT_PARSE_PTYPE_NUM},
 605         {CMD_LINE_OPT_PER_PORT_POOL, 0, 0, CMD_LINE_OPT_PARSE_PER_PORT_POOL},
 606         {CMD_LINE_OPT_MODE, 1, 0, CMD_LINE_OPT_MODE_NUM},
 607         {CMD_LINE_OPT_EVENTQ_SYNC, 1, 0, CMD_LINE_OPT_EVENTQ_SYNC_NUM},
 608         {CMD_LINE_OPT_EVENT_ETH_RX_QUEUES, 1, 0,
 609                                         CMD_LINE_OPT_EVENT_ETH_RX_QUEUES_NUM},
 610         {CMD_LINE_OPT_LOOKUP, 1, 0, CMD_LINE_OPT_LOOKUP_NUM},
 611         {NULL, 0, 0, 0}
 612 };
 613
 614 /*
 615  * This expression is used to calculate the number of mbufs needed
 616  * depending on user input, taking  into account memory for rx and
 617  * tx hardware rings, cache per lcore and mtable per port per lcore.
 618  * RTE_MAX is used to ensure that NB_MBUF never goes below a minimum
 619  * value of 8192
 620  */
 621 #define NB_MBUF(nports) RTE_MAX(        \
 622         (nports*nb_rx_queue*nb_rxd +            \
 623         nports*nb_lcores*MAX_PKT_BURST +        \
 624         nports*n_tx_queue*nb_txd +              \
 625         nb_lcores*MEMPOOL_CACHE_SIZE),          \
 626         (unsigned)8192)
 627
 628 /* Parse the argument given in the command line of the application */
 629 static int
 630 parse_args(int argc, char **argv)
 631 {
 632         int opt, ret;
 633         char **argvopt;
 634         int option_index;
 635         char *prgname = argv[0];
 636         uint8_t lcore_params = 0;
 637         uint8_t eventq_sched = 0;
 638         uint8_t eth_rx_q = 0;
 639         struct l3fwd_event_resources *evt_rsrc = l3fwd_get_eventdev_rsrc();
 640
 641         argvopt = argv;
 642
 643         /* Error or normal output strings. */
 644         while ((opt = getopt_long(argc, argvopt, short_options,
 645                                 lgopts, &option_index)) != EOF) {
 646
 647                 switch (opt) {
 648                 /* portmask */
 649                 case 'p':
 650                         enabled_port_mask = parse_portmask(optarg);
 651                         if (enabled_port_mask == 0) {
 652                                 fprintf(stderr, "Invalid portmask\n");
 653                                 print_usage(prgname);
 654                                 return -1;
 655                         }
 656                         break;
 657
 658                 case 'P':
 659                         promiscuous_on = 1;
 660                         break;
 661
 662                 case 'E':
 663                         if (lookup_mode != L3FWD_LOOKUP_DEFAULT) {
 664                                 fprintf(stderr, "Only one lookup mode is allowed at a time!\n");
 665                                 return -1;
 666                         }
 667                         lookup_mode = L3FWD_LOOKUP_EM;
 668                         break;
 669
 670                 case 'L':
 671                         if (lookup_mode != L3FWD_LOOKUP_DEFAULT) {
 672                                 fprintf(stderr, "Only one lookup mode is allowed at a time!\n");
 673                                 return -1;
 674                         }
 675                         lookup_mode = L3FWD_LOOKUP_LPM;
 676                         break;
 677
 678                 /* long options */
 679                 case CMD_LINE_OPT_CONFIG_NUM:
 680                         ret = parse_config(optarg);
 681                         if (ret) {
 682                                 fprintf(stderr, "Invalid config\n");
 683                                 print_usage(prgname);
 684                                 return -1;
 685                         }
 686                         lcore_params = 1;
 687                         break;
 688
 689                 case CMD_LINE_OPT_ETH_DEST_NUM:
 690                         parse_eth_dest(optarg);
 691                         break;
 692
 693                 case CMD_LINE_OPT_NO_NUMA_NUM:
 694                         numa_on = 0;
 695                         break;
 696
 697                 case CMD_LINE_OPT_IPV6_NUM:
 698                         ipv6 = 1;
 699                         break;
 700
 701                 case CMD_LINE_OPT_ENABLE_JUMBO_NUM: {
 702                         const struct option lenopts = {
 703                                 "max-pkt-len", required_argument, 0, 0
 704                         };
 705
 706                         port_conf.rxmode.offloads |= DEV_RX_OFFLOAD_JUMBO_FRAME;
 707                         port_conf.txmode.offloads |= DEV_TX_OFFLOAD_MULTI_SEGS;
 708
 709                         /*
 710                          * if no max-pkt-len set, use the default
 711                          * value RTE_ETHER_MAX_LEN.
 712                          */
 713                         if (getopt_long(argc, argvopt, "",
 714                                         &lenopts, &option_index) == 0) {
 715                                 ret = parse_max_pkt_len(optarg);
 716                                 if (ret < 64 || ret > MAX_JUMBO_PKT_LEN) {
 717                                         fprintf(stderr,
 718                                                 "invalid maximum packet length\n");
 719                                         print_usage(prgname);
 720                                         return -1;
 721                                 }
 722                                 port_conf.rxmode.max_rx_pkt_len = ret;
 723                         }
 724                         break;
 725                 }
 726
 727                 case CMD_LINE_OPT_HASH_ENTRY_NUM_NUM:
 728                         ret = parse_hash_entry_number(optarg);
 729                         if ((ret > 0) && (ret <= L3FWD_HASH_ENTRIES)) {
 730                                 hash_entry_number = ret;
 731                         } else {
 732                                 fprintf(stderr, "invalid hash entry number\n");
 733                                 print_usage(prgname);
 734                                 return -1;
 735                         }
 736                         break;
 737
 738                 case CMD_LINE_OPT_PARSE_PTYPE_NUM:
 739                         printf("soft parse-ptype is enabled\n");
 740                         parse_ptype = 1;
 741                         break;
 742
 743                 case CMD_LINE_OPT_PARSE_PER_PORT_POOL:
 744                         printf("per port buffer pool is enabled\n");
 745                         per_port_pool = 1;
 746                         break;
 747
 748                 case CMD_LINE_OPT_MODE_NUM:
 749                         parse_mode(optarg);
 750                         break;
 751
 752                 case CMD_LINE_OPT_EVENTQ_SYNC_NUM:
 753                         parse_eventq_sched(optarg);
 754                         eventq_sched = 1;
 755                         break;
 756
 757                 case CMD_LINE_OPT_EVENT_ETH_RX_QUEUES_NUM:
 758                         parse_event_eth_rx_queues(optarg);
 759                         eth_rx_q = 1;
 760                         break;
 761
 762                 case CMD_LINE_OPT_LOOKUP_NUM:
 763                         if (lookup_mode != L3FWD_LOOKUP_DEFAULT) {
 764                                 fprintf(stderr, "Only one lookup mode is allowed at a time!\n");
 765                                 return -1;
 766                         }
 767                         ret = parse_lookup(optarg);
 768                         /*
 769                          * If parse_lookup was passed an invalid lookup type
 770                          * then return -1. Error log included within
 771                          * parse_lookup for simplicity.
 772                          */
 773                         if (ret)
 774                                 return -1;
 775                         break;
 776
 777                 default:
 778                         print_usage(prgname);
 779                         return -1;
 780                 }
 781         }
 782
 783         if (evt_rsrc->enabled && lcore_params) {
 784                 fprintf(stderr, "lcore config is not valid when event mode is selected\n");
 785                 return -1;
 786         }
 787
 788         if (!evt_rsrc->enabled && eth_rx_q) {
 789                 fprintf(stderr, "eth_rx_queues is valid only when event mode is selected\n");
 790                 return -1;
 791         }
 792
 793         if (!evt_rsrc->enabled && eventq_sched) {
 794                 fprintf(stderr, "eventq_sched is valid only when event mode is selected\n");
 795                 return -1;
 796         }
 797
 798         /*
 799          * Nothing is selected, pick longest-prefix match
 800          * as default match.
 801          */
 802         if (lookup_mode == L3FWD_LOOKUP_DEFAULT) {
 803                 fprintf(stderr, "Neither LPM, EM, or FIB selected, defaulting to LPM\n");
 804                 lookup_mode = L3FWD_LOOKUP_LPM;
 805         }
 806
 807         /*
 808          * ipv6 and hash flags are valid only for
 809          * exact match, reset them to default for
 810          * longest-prefix match.
 811          */
 812         if (lookup_mode == L3FWD_LOOKUP_LPM) {
 813                 ipv6 = 0;
 814                 hash_entry_number = HASH_ENTRY_NUMBER_DEFAULT;
 815         }
 816
 817         if (optind >= 0)
 818                 argv[optind-1] = prgname;
 819
 820         ret = optind-1;
 821         optind = 1; /* reset getopt lib */
 822         return ret;
 823 }
 824
 825 static void
 826 print_ethaddr(const char *name, const struct rte_ether_addr *eth_addr)
 827 {
 828         char buf[RTE_ETHER_ADDR_FMT_SIZE];
 829         rte_ether_format_addr(buf, RTE_ETHER_ADDR_FMT_SIZE, eth_addr);
 830         printf("%s%s", name, buf);
 831 }
 832
 833 int
 834 init_mem(uint16_t portid, unsigned int nb_mbuf)
 835 {
 836         struct lcore_conf *qconf;
 837         int socketid;
 838         unsigned lcore_id;
 839         char s[64];
 840
 841         for (lcore_id = 0; lcore_id < RTE_MAX_LCORE; lcore_id++) {
 842                 if (rte_lcore_is_enabled(lcore_id) == 0)
 843                         continue;
 844
 845                 if (numa_on)
 846                         socketid = rte_lcore_to_socket_id(lcore_id);
 847                 else
 848                         socketid = 0;
 849
 850                 if (socketid >= NB_SOCKETS) {
 851                         rte_exit(EXIT_FAILURE,
 852                                 "Socket %d of lcore %u is out of range %d\n",
 853                                 socketid, lcore_id, NB_SOCKETS);
 854                 }
 855
 856                 if (pktmbuf_pool[portid][socketid] == NULL) {
 857                         snprintf(s, sizeof(s), "mbuf_pool_%d:%d",
 858                                  portid, socketid);
 859                         pktmbuf_pool[portid][socketid] =
 860                                 rte_pktmbuf_pool_create(s, nb_mbuf,
 861                                         MEMPOOL_CACHE_SIZE, 0,
 862                                         RTE_MBUF_DEFAULT_BUF_SIZE, socketid);
 863                         if (pktmbuf_pool[portid][socketid] == NULL)
 864                                 rte_exit(EXIT_FAILURE,
 865                                         "Cannot init mbuf pool on socket %d\n",
 866                                         socketid);
 867                         else
 868                                 printf("Allocated mbuf pool on socket %d\n",
 869                                         socketid);
 870
 871                         /* Setup LPM, EM(f.e Hash) or FIB. But, only once per
 872                          * available socket.
 873                          */
 874                         if (!lkp_per_socket[socketid]) {
 875                                 l3fwd_lkp.setup(socketid);
 876                                 lkp_per_socket[socketid] = 1;
 877                         }
 878                 }
 879                 qconf = &lcore_conf[lcore_id];
 880                 qconf->ipv4_lookup_struct =
 881                         l3fwd_lkp.get_ipv4_lookup_struct(socketid);
 882                 qconf->ipv6_lookup_struct =
 883                         l3fwd_lkp.get_ipv6_lookup_struct(socketid);
 884         }
 885         return 0;
 886 }
 887
 888 /* Check the link status of all ports in up to 9s, and print them finally */
 889 static void
 890 check_all_ports_link_status(uint32_t port_mask)
 891 {
 892 #define CHECK_INTERVAL 100 /* 100ms */
 893 #define MAX_CHECK_TIME 90 /* 9s (90 * 100ms) in total */
 894         uint16_t portid;
 895         uint8_t count, all_ports_up, print_flag = 0;
 896         struct rte_eth_link link;
 897         int ret;
 898         char link_status_text[RTE_ETH_LINK_MAX_STR_LEN];
 899
 900         printf("\nChecking link status");
 901         fflush(stdout);
 902         for (count = 0; count <= MAX_CHECK_TIME; count++) {
 903                 if (force_quit)
 904                         return;
 905                 all_ports_up = 1;
 906                 RTE_ETH_FOREACH_DEV(portid) {
 907                         if (force_quit)
 908                                 return;
 909                         if ((port_mask & (1 << portid)) == 0)
 910                                 continue;
 911                         memset(&link, 0, sizeof(link));
 912                         ret = rte_eth_link_get_nowait(portid, &link);
 913                         if (ret < 0) {
 914                                 all_ports_up = 0;
 915                                 if (print_flag == 1)
 916                                         printf("Port %u link get failed: %s\n",
 917                                                 portid, rte_strerror(-ret));
 918                                 continue;
 919                         }
 920                         /* print link status if flag set */
 921                         if (print_flag == 1) {
 922                                 rte_eth_link_to_str(link_status_text,
 923                                         sizeof(link_status_text), &link);
 924                                 printf("Port %d %s\n", portid,
 925                                        link_status_text);
 926                                 continue;
 927                         }
 928                         /* clear all_ports_up flag if any link down */
 929                         if (link.link_status == ETH_LINK_DOWN) {
 930                                 all_ports_up = 0;
 931                                 break;
 932                         }
 933                 }
 934                 /* after finally printing all link status, get out */
 935                 if (print_flag == 1)
 936                         break;
 937
 938                 if (all_ports_up == 0) {
 939                         printf(".");
 940                         fflush(stdout);
 941                         rte_delay_ms(CHECK_INTERVAL);
 942                 }
 943
 944                 /* set the print_flag if all ports up or timeout */
 945                 if (all_ports_up == 1 || count == (MAX_CHECK_TIME - 1)) {
 946                         print_flag = 1;
 947                         printf("done\n");
 948                 }
 949         }
 950 }
 951
 952 static void
 953 signal_handler(int signum)
 954 {
 955         if (signum == SIGINT || signum == SIGTERM) {
 956                 printf("\n\nSignal %d received, preparing to exit...\n",
 957                                 signum);
 958                 force_quit = true;
 959         }
 960 }
 961
 962 static int
 963 prepare_ptype_parser(uint16_t portid, uint16_t queueid)
 964 {
 965         if (parse_ptype) {
 966                 printf("Port %d: softly parse packet type info\n", portid);
 967                 if (rte_eth_add_rx_callback(portid, queueid,
 968                                             l3fwd_lkp.cb_parse_ptype,
 969                                             NULL))
 970                         return 1;
 971
 972                 printf("Failed to add rx callback: port=%d\n", portid);
 973                 return 0;
 974         }
 975
 976         if (l3fwd_lkp.check_ptype(portid))
 977                 return 1;
 978
 979         printf("port %d cannot parse packet type, please add --%s\n",
 980                portid, CMD_LINE_OPT_PARSE_PTYPE);
 981         return 0;
 982 }
 983
 984 static void
 985 l3fwd_poll_resource_setup(void)
 986 {
 987         uint8_t nb_rx_queue, queue, socketid;
 988         struct rte_eth_dev_info dev_info;
 989         uint32_t n_tx_queue, nb_lcores;
 990         struct rte_eth_txconf *txconf;
 991         struct lcore_conf *qconf;
 992         uint16_t queueid, portid;
 993         unsigned int nb_ports;
 994         unsigned int lcore_id;
 995         int ret;
 996
 997         if (check_lcore_params() < 0)
 998                 rte_exit(EXIT_FAILURE, "check_lcore_params failed\n");
 999
1000         ret = init_lcore_rx_queues();
1001         if (ret < 0)
1002                 rte_exit(EXIT_FAILURE, "init_lcore_rx_queues failed\n");
1003
1004         nb_ports = rte_eth_dev_count_avail();
1005
1006         if (check_port_config() < 0)
1007                 rte_exit(EXIT_FAILURE, "check_port_config failed\n");
1008
1009         nb_lcores = rte_lcore_count();
1010
1011         /* initialize all ports */
1012         RTE_ETH_FOREACH_DEV(portid) {
1013                 struct rte_eth_conf local_port_conf = port_conf;
1014
1015                 /* skip ports that are not enabled */
1016                 if ((enabled_port_mask & (1 << portid)) == 0) {
1017                         printf("\nSkipping disabled port %d\n", portid);
1018                         continue;
1019                 }
1020
1021                 /* init port */
1022                 printf("Initializing port %d ... ", portid );
1023                 fflush(stdout);
1024
1025                 nb_rx_queue = get_port_n_rx_queues(portid);
1026                 n_tx_queue = nb_lcores;
1027                 if (n_tx_queue > MAX_TX_QUEUE_PER_PORT)
1028                         n_tx_queue = MAX_TX_QUEUE_PER_PORT;
1029                 printf("Creating queues: nb_rxq=%d nb_txq=%u... ",
1030                         nb_rx_queue, (unsigned)n_tx_queue );
1031
1032                 ret = rte_eth_dev_info_get(portid, &dev_info);
1033                 if (ret != 0)
1034                         rte_exit(EXIT_FAILURE,
1035                                 "Error during getting device (port %u) info: %s\n",
1036                                 portid, strerror(-ret));
1037
1038                 if (dev_info.tx_offload_capa & DEV_TX_OFFLOAD_MBUF_FAST_FREE)
1039                         local_port_conf.txmode.offloads |=
1040                                 DEV_TX_OFFLOAD_MBUF_FAST_FREE;
1041
1042                 local_port_conf.rx_adv_conf.rss_conf.rss_hf &=
1043                         dev_info.flow_type_rss_offloads;
1044                 if (local_port_conf.rx_adv_conf.rss_conf.rss_hf !=
1045                                 port_conf.rx_adv_conf.rss_conf.rss_hf) {
1046                         printf("Port %u modified RSS hash function based on hardware support,"
1047                                 "requested:%#"PRIx64" configured:%#"PRIx64"\n",
1048                                 portid,
1049                                 port_conf.rx_adv_conf.rss_conf.rss_hf,
1050                                 local_port_conf.rx_adv_conf.rss_conf.rss_hf);
1051                 }
1052
1053                 ret = rte_eth_dev_configure(portid, nb_rx_queue,
1054                                         (uint16_t)n_tx_queue, &local_port_conf);
1055                 if (ret < 0)
1056                         rte_exit(EXIT_FAILURE,
1057                                 "Cannot configure device: err=%d, port=%d\n",
1058                                 ret, portid);
1059
1060                 ret = rte_eth_dev_adjust_nb_rx_tx_desc(portid, &nb_rxd,
1061                                                        &nb_txd);
1062                 if (ret < 0)
1063                         rte_exit(EXIT_FAILURE,
1064                                  "Cannot adjust number of descriptors: err=%d, "
1065                                  "port=%d\n", ret, portid);
1066
1067                 ret = rte_eth_macaddr_get(portid, &ports_eth_addr[portid]);
1068                 if (ret < 0)
1069                         rte_exit(EXIT_FAILURE,
1070                                  "Cannot get MAC address: err=%d, port=%d\n",
1071                                  ret, portid);
1072
1073                 print_ethaddr(" Address:", &ports_eth_addr[portid]);
1074                 printf(", ");
1075                 print_ethaddr("Destination:",
1076                         (const struct rte_ether_addr *)&dest_eth_addr[portid]);
1077                 printf(", ");
1078
1079                 /*
1080                  * prepare src MACs for each port.
1081                  */
1082                 rte_ether_addr_copy(&ports_eth_addr[portid],
1083                         (struct rte_ether_addr *)(val_eth + portid) + 1);
1084
1085                 /* init memory */
1086                 if (!per_port_pool) {
1087                         /* portid = 0; this is *not* signifying the first port,
1088                          * rather, it signifies that portid is ignored.
1089                          */
1090                         ret = init_mem(0, NB_MBUF(nb_ports));
1091                 } else {
1092                         ret = init_mem(portid, NB_MBUF(1));
1093                 }
1094                 if (ret < 0)
1095                         rte_exit(EXIT_FAILURE, "init_mem failed\n");
1096
1097                 /* init one TX queue per couple (lcore,port) */
1098                 queueid = 0;
1099                 for (lcore_id = 0; lcore_id < RTE_MAX_LCORE; lcore_id++) {
1100                         if (rte_lcore_is_enabled(lcore_id) == 0)
1101                                 continue;
1102
1103                         if (numa_on)
1104                                 socketid =
1105                                 (uint8_t)rte_lcore_to_socket_id(lcore_id);
1106                         else
1107                                 socketid = 0;
1108
1109                         printf("txq=%u,%d,%d ", lcore_id, queueid, socketid);
1110                         fflush(stdout);
1111
1112                         txconf = &dev_info.default_txconf;
1113                         txconf->offloads = local_port_conf.txmode.offloads;
1114                         ret = rte_eth_tx_queue_setup(portid, queueid, nb_txd,
1115                                                      socketid, txconf);
1116                         if (ret < 0)
1117                                 rte_exit(EXIT_FAILURE,
1118                                         "rte_eth_tx_queue_setup: err=%d, "
1119                                         "port=%d\n", ret, portid);
1120
1121                         qconf = &lcore_conf[lcore_id];
1122                         qconf->tx_queue_id[portid] = queueid;
1123                         queueid++;
1124
1125                         qconf->tx_port_id[qconf->n_tx_port] = portid;
1126                         qconf->n_tx_port++;
1127                 }
1128                 printf("\n");
1129         }
1130
1131         for (lcore_id = 0; lcore_id < RTE_MAX_LCORE; lcore_id++) {
1132                 if (rte_lcore_is_enabled(lcore_id) == 0)
1133                         continue;
1134                 qconf = &lcore_conf[lcore_id];
1135                 printf("\nInitializing rx queues on lcore %u ... ", lcore_id );
1136                 fflush(stdout);
1137                 /* init RX queues */
1138                 for(queue = 0; queue < qconf->n_rx_queue; ++queue) {
1139                         struct rte_eth_rxconf rxq_conf;
1140
1141                         portid = qconf->rx_queue_list[queue].port_id;
1142                         queueid = qconf->rx_queue_list[queue].queue_id;
1143
1144                         if (numa_on)
1145                                 socketid =
1146                                 (uint8_t)rte_lcore_to_socket_id(lcore_id);
1147                         else
1148                                 socketid = 0;
1149
1150                         printf("rxq=%d,%d,%d ", portid, queueid, socketid);
1151                         fflush(stdout);
1152
1153                         ret = rte_eth_dev_info_get(portid, &dev_info);
1154                         if (ret != 0)
1155                                 rte_exit(EXIT_FAILURE,
1156                                         "Error during getting device (port %u) info: %s\n",
1157                                         portid, strerror(-ret));
1158
1159                         rxq_conf = dev_info.default_rxconf;
1160                         rxq_conf.offloads = port_conf.rxmode.offloads;
1161                         if (!per_port_pool)
1162                                 ret = rte_eth_rx_queue_setup(portid, queueid,
1163                                                 nb_rxd, socketid,
1164                                                 &rxq_conf,
1165                                                 pktmbuf_pool[0][socketid]);
1166                         else
1167                                 ret = rte_eth_rx_queue_setup(portid, queueid,
1168                                                 nb_rxd, socketid,
1169                                                 &rxq_conf,
1170                                                 pktmbuf_pool[portid][socketid]);
1171                         if (ret < 0)
1172                                 rte_exit(EXIT_FAILURE,
1173                                 "rte_eth_rx_queue_setup: err=%d, port=%d\n",
1174                                 ret, portid);
1175                 }
1176         }
1177 }
1178
1179 static inline int
1180 l3fwd_service_enable(uint32_t service_id)
1181 {
1182         uint8_t min_service_count = UINT8_MAX;
1183         uint32_t slcore_array[RTE_MAX_LCORE];
1184         unsigned int slcore = 0;
1185         uint8_t service_count;
1186         int32_t slcore_count;
1187
1188         if (!rte_service_lcore_count())
1189                 return -ENOENT;
1190
1191         slcore_count = rte_service_lcore_list(slcore_array, RTE_MAX_LCORE);
1192         if (slcore_count < 0)
1193                 return -ENOENT;
1194         /* Get the core which has least number of services running. */
1195         while (slcore_count--) {
1196                 /* Reset default mapping */
1197                 if (rte_service_map_lcore_set(service_id,
1198                                 slcore_array[slcore_count], 0) != 0)
1199                         return -ENOENT;
1200                 service_count = rte_service_lcore_count_services(
1201                                 slcore_array[slcore_count]);
1202                 if (service_count < min_service_count) {
1203                         slcore = slcore_array[slcore_count];
1204                         min_service_count = service_count;
1205                 }
1206         }
1207         if (rte_service_map_lcore_set(service_id, slcore, 1))
1208                 return -ENOENT;
1209         rte_service_lcore_start(slcore);
1210
1211         return 0;
1212 }
1213
1214 static void
1215 l3fwd_event_service_setup(void)
1216 {
1217         struct l3fwd_event_resources *evt_rsrc = l3fwd_get_eventdev_rsrc();
1218         struct rte_event_dev_info evdev_info;
1219         uint32_t service_id, caps;
1220         int ret, i;
1221
1222         rte_event_dev_info_get(evt_rsrc->event_d_id, &evdev_info);
1223         if (!(evdev_info.event_dev_cap & RTE_EVENT_DEV_CAP_DISTRIBUTED_SCHED)) {
1224                 ret = rte_event_dev_service_id_get(evt_rsrc->event_d_id,
1225                                 &service_id);
1226                 if (ret != -ESRCH && ret != 0)
1227                         rte_exit(EXIT_FAILURE,
1228                                  "Error in starting eventdev service\n");
1229                 l3fwd_service_enable(service_id);
1230         }
1231
1232         for (i = 0; i < evt_rsrc->rx_adptr.nb_rx_adptr; i++) {
1233                 ret = rte_event_eth_rx_adapter_caps_get(evt_rsrc->event_d_id,
1234                                 evt_rsrc->rx_adptr.rx_adptr[i], &caps);
1235                 if (ret < 0)
1236                         rte_exit(EXIT_FAILURE,
1237                                  "Failed to get Rx adapter[%d] caps\n",
1238                                  evt_rsrc->rx_adptr.rx_adptr[i]);
1239                 ret = rte_event_eth_rx_adapter_service_id_get(
1240                                 evt_rsrc->event_d_id,
1241                                 &service_id);
1242                 if (ret != -ESRCH && ret != 0)
1243                         rte_exit(EXIT_FAILURE,
1244                                  "Error in starting Rx adapter[%d] service\n",
1245                                  evt_rsrc->rx_adptr.rx_adptr[i]);
1246                 l3fwd_service_enable(service_id);
1247         }
1248
1249         for (i = 0; i < evt_rsrc->tx_adptr.nb_tx_adptr; i++) {
1250                 ret = rte_event_eth_tx_adapter_caps_get(evt_rsrc->event_d_id,
1251                                 evt_rsrc->tx_adptr.tx_adptr[i], &caps);
1252                 if (ret < 0)
1253                         rte_exit(EXIT_FAILURE,
1254                                  "Failed to get Rx adapter[%d] caps\n",
1255                                  evt_rsrc->tx_adptr.tx_adptr[i]);
1256                 ret = rte_event_eth_tx_adapter_service_id_get(
1257                                 evt_rsrc->event_d_id,
1258                                 &service_id);
1259                 if (ret != -ESRCH && ret != 0)
1260                         rte_exit(EXIT_FAILURE,
1261                                  "Error in starting Rx adapter[%d] service\n",
1262                                  evt_rsrc->tx_adptr.tx_adptr[i]);
1263                 l3fwd_service_enable(service_id);
1264         }
1265 }
1266
1267 int
1268 main(int argc, char **argv)
1269 {
1270         struct l3fwd_event_resources *evt_rsrc;
1271         struct lcore_conf *qconf;
1272         uint16_t queueid, portid;
1273         unsigned int lcore_id;
1274         uint8_t queue;
1275         int i, ret;
1276
1277         /* init EAL */
1278         ret = rte_eal_init(argc, argv);
1279         if (ret < 0)
1280                 rte_exit(EXIT_FAILURE, "Invalid EAL parameters\n");
1281         argc -= ret;
1282         argv += ret;
1283
1284         force_quit = false;
1285         signal(SIGINT, signal_handler);
1286         signal(SIGTERM, signal_handler);
1287
1288         /* pre-init dst MACs for all ports to 02:00:00:00:00:xx */
1289         for (portid = 0; portid < RTE_MAX_ETHPORTS; portid++) {
1290                 dest_eth_addr[portid] =
1291                         RTE_ETHER_LOCAL_ADMIN_ADDR + ((uint64_t)portid << 40);
1292                 *(uint64_t *)(val_eth + portid) = dest_eth_addr[portid];
1293         }
1294
1295         evt_rsrc = l3fwd_get_eventdev_rsrc();
1296         /* parse application arguments (after the EAL ones) */
1297         ret = parse_args(argc, argv);
1298         if (ret < 0)
1299                 rte_exit(EXIT_FAILURE, "Invalid L3FWD parameters\n");
1300
1301         /* Setup function pointers for lookup method. */
1302         setup_l3fwd_lookup_tables();
1303
1304         evt_rsrc->per_port_pool = per_port_pool;
1305         evt_rsrc->pkt_pool = pktmbuf_pool;
1306         evt_rsrc->port_mask = enabled_port_mask;
1307         /* Configure eventdev parameters if user has requested */
1308         if (evt_rsrc->enabled) {
1309                 l3fwd_event_resource_setup(&port_conf);
1310                 if (lookup_mode == L3FWD_LOOKUP_EM)
1311                         l3fwd_lkp.main_loop = evt_rsrc->ops.em_event_loop;
1312                 else if (lookup_mode == L3FWD_LOOKUP_FIB)
1313                         l3fwd_lkp.main_loop = evt_rsrc->ops.fib_event_loop;
1314                 else
1315                         l3fwd_lkp.main_loop = evt_rsrc->ops.lpm_event_loop;
1316                 l3fwd_event_service_setup();
1317         } else
1318                 l3fwd_poll_resource_setup();
1319
1320         /* start ports */
1321         RTE_ETH_FOREACH_DEV(portid) {
1322                 if ((enabled_port_mask & (1 << portid)) == 0) {
1323                         continue;
1324                 }
1325                 /* Start device */
1326                 ret = rte_eth_dev_start(portid);
1327                 if (ret < 0)
1328                         rte_exit(EXIT_FAILURE,
1329                                 "rte_eth_dev_start: err=%d, port=%d\n",
1330                                 ret, portid);
1331
1332                 /*
1333                  * If enabled, put device in promiscuous mode.
1334                  * This allows IO forwarding mode to forward packets
1335                  * to itself through 2 cross-connected  ports of the
1336                  * target machine.
1337                  */
1338                 if (promiscuous_on) {
1339                         ret = rte_eth_promiscuous_enable(portid);
1340                         if (ret != 0)
1341                                 rte_exit(EXIT_FAILURE,
1342                                         "rte_eth_promiscuous_enable: err=%s, port=%u\n",
1343                                         rte_strerror(-ret), portid);
1344                 }
1345         }
1346
1347         printf("\n");
1348
1349         for (lcore_id = 0; lcore_id < RTE_MAX_LCORE; lcore_id++) {
1350                 if (rte_lcore_is_enabled(lcore_id) == 0)
1351                         continue;
1352                 qconf = &lcore_conf[lcore_id];
1353                 for (queue = 0; queue < qconf->n_rx_queue; ++queue) {
1354                         portid = qconf->rx_queue_list[queue].port_id;
1355                         queueid = qconf->rx_queue_list[queue].queue_id;
1356                         if (prepare_ptype_parser(portid, queueid) == 0)
1357                                 rte_exit(EXIT_FAILURE, "ptype check fails\n");
1358                 }
1359         }
1360
1361         check_all_ports_link_status(enabled_port_mask);
1362
1363         ret = 0;
1364         /* launch per-lcore init on every lcore */
1365         rte_eal_mp_remote_launch(l3fwd_lkp.main_loop, NULL, CALL_MAIN);
1366         if (evt_rsrc->enabled) {
1367                 for (i = 0; i < evt_rsrc->rx_adptr.nb_rx_adptr; i++)
1368                         rte_event_eth_rx_adapter_stop(
1369                                         evt_rsrc->rx_adptr.rx_adptr[i]);
1370                 for (i = 0; i < evt_rsrc->tx_adptr.nb_tx_adptr; i++)
1371                         rte_event_eth_tx_adapter_stop(
1372                                         evt_rsrc->tx_adptr.tx_adptr[i]);
1373
1374                 RTE_ETH_FOREACH_DEV(portid) {
1375                         if ((enabled_port_mask & (1 << portid)) == 0)
1376                                 continue;
1377                         ret = rte_eth_dev_stop(portid);
1378                         if (ret != 0)
1379                                 printf("rte_eth_dev_stop: err=%d, port=%u\n",
1380                                        ret, portid);
1381                 }
1382
1383                 rte_eal_mp_wait_lcore();
1384                 RTE_ETH_FOREACH_DEV(portid) {
1385                         if ((enabled_port_mask & (1 << portid)) == 0)
1386                                 continue;
1387                         rte_eth_dev_close(portid);
1388                 }
1389
1390                 rte_event_dev_stop(evt_rsrc->event_d_id);
1391                 rte_event_dev_close(evt_rsrc->event_d_id);
1392
1393         } else {
1394                 rte_eal_mp_wait_lcore();
1395
1396                 RTE_ETH_FOREACH_DEV(portid) {
1397                         if ((enabled_port_mask & (1 << portid)) == 0)
1398                                 continue;
1399                         printf("Closing port %d...", portid);
1400                         ret = rte_eth_dev_stop(portid);
1401                         if (ret != 0)
1402                                 printf("rte_eth_dev_stop: err=%d, port=%u\n",
1403                                        ret, portid);
1404                         rte_eth_dev_close(portid);
1405                         printf(" Done\n");
1406                 }
1407         }
1408
1409         /* clean up the EAL */
1410         rte_eal_cleanup();
1411
1412         printf("Bye...\n");
1413
1414         return ret;
1415 }