examples/l3fwd-vf/main.c

   1 /*-
   2  *   BSD LICENSE
   3  *
   4  *   Copyright(c) 2010-2012 Intel Corporation. All rights reserved.
   5  *   All rights reserved.
   6  *
   7  *   Redistribution and use in source and binary forms, with or without
   8  *   modification, are permitted provided that the following conditions
   9  *   are met:
  10  *
  11  *     * Redistributions of source code must retain the above copyright
  12  *       notice, this list of conditions and the following disclaimer.
  13  *     * Redistributions in binary form must reproduce the above copyright
  14  *       notice, this list of conditions and the following disclaimer in
  15  *       the documentation and/or other materials provided with the
  16  *       distribution.
  17  *     * Neither the name of Intel Corporation nor the names of its
  18  *       contributors may be used to endorse or promote products derived
  19  *       from this software without specific prior written permission.
  20  *
  21  *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  22  *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  23  *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
  24  *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  25  *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  26  *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  27  *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  28  *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  29  *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  30  *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  31  *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  32  *
  33  */
  34
  35 #include <stdio.h>
  36 #include <stdlib.h>
  37 #include <stdint.h>
  38 #include <inttypes.h>
  39 #include <sys/types.h>
  40 #include <string.h>
  41 #include <sys/queue.h>
  42 #include <stdarg.h>
  43 #include <errno.h>
  44 #include <getopt.h>
  45
  46 #include <rte_common.h>
  47 #include <rte_byteorder.h>
  48 #include <rte_log.h>
  49 #include <rte_memory.h>
  50 #include <rte_memcpy.h>
  51 #include <rte_memzone.h>
  52 #include <rte_tailq.h>
  53 #include <rte_eal.h>
  54 #include <rte_per_lcore.h>
  55 #include <rte_launch.h>
  56 #include <rte_atomic.h>
  57 #include <rte_cycles.h>
  58 #include <rte_prefetch.h>
  59 #include <rte_lcore.h>
  60 #include <rte_per_lcore.h>
  61 #include <rte_branch_prediction.h>
  62 #include <rte_interrupts.h>
  63 #include <rte_pci.h>
  64 #include <rte_random.h>
  65 #include <rte_debug.h>
  66 #include <rte_ether.h>
  67 #include <rte_ethdev.h>
  68 #include <rte_ring.h>
  69 #include <rte_mempool.h>
  70 #include <rte_mbuf.h>
  71 #include <rte_ip.h>
  72 #include <rte_tcp.h>
  73 #include <rte_udp.h>
  74 #include <rte_string_fns.h>
  75
  76 #include "main.h"
  77
  78 #define APP_LOOKUP_EXACT_MATCH          0
  79 #define APP_LOOKUP_LPM                  1
  80 #define DO_RFC_1812_CHECKS
  81
  82 //#define APP_LOOKUP_METHOD             APP_LOOKUP_EXACT_MATCH
  83 #ifndef APP_LOOKUP_METHOD
  84 #define APP_LOOKUP_METHOD             APP_LOOKUP_LPM
  85 #endif
  86
  87 #if (APP_LOOKUP_METHOD == APP_LOOKUP_EXACT_MATCH)
  88 #include <rte_hash.h>
  89 #include <rte_hash_crc.h>
  90 #include <rte_jhash.h>
  91 #elif (APP_LOOKUP_METHOD == APP_LOOKUP_LPM)
  92 #include <rte_lpm.h>
  93 #else
  94 #error "APP_LOOKUP_METHOD set to incorrect value"
  95 #endif
  96
  97 #define RTE_LOGTYPE_L3FWD RTE_LOGTYPE_USER1
  98
  99 #define MAX_PORTS 32
 100
 101 #define MBUF_SIZE (2048 + sizeof(struct rte_mbuf) + RTE_PKTMBUF_HEADROOM)
 102 #define NB_MBUF   8192
 103
 104 /*
 105  * RX and TX Prefetch, Host, and Write-back threshold values should be
 106  * carefully set for optimal performance. Consult the network
 107  * controller's datasheet and supporting DPDK documentation for guidance
 108  * on how these parameters should be set.
 109  */
 110 #define RX_PTHRESH 8 /**< Default values of RX prefetch threshold reg. */
 111 #define RX_HTHRESH 8 /**< Default values of RX host threshold reg. */
 112 #define RX_WTHRESH 4 /**< Default values of RX write-back threshold reg. */
 113
 114 /*
 115  * These default values are optimized for use with the Intel(R) 82599 10 GbE
 116  * Controller and the DPDK ixgbe PMD. Consider using other values for other
 117  * network controllers and/or network drivers.
 118  */
 119 #define TX_PTHRESH 36 /**< Default values of TX prefetch threshold reg. */
 120 #define TX_HTHRESH 0  /**< Default values of TX host threshold reg. */
 121 #define TX_WTHRESH 0  /**< Default values of TX write-back threshold reg. */
 122
 123 #define MAX_PKT_BURST 32
 124 #define BURST_TX_DRAIN 200000ULL /* around 100us at 2 Ghz */
 125
 126 #define NB_SOCKETS 8
 127
 128 #define SOCKET0 0
 129
 130 /* Configure how many packets ahead to prefetch, when reading packets */
 131 #define PREFETCH_OFFSET 3
 132
 133 /*
 134  * Configurable number of RX/TX ring descriptors
 135  */
 136 #define RTE_TEST_RX_DESC_DEFAULT 128
 137 #define RTE_TEST_TX_DESC_DEFAULT 512
 138 static uint16_t nb_rxd = RTE_TEST_RX_DESC_DEFAULT;
 139 static uint16_t nb_txd = RTE_TEST_TX_DESC_DEFAULT;
 140
 141 /* ethernet addresses of ports */
 142 static struct ether_addr ports_eth_addr[MAX_PORTS];
 143
 144 /* mask of enabled ports */
 145 static uint32_t enabled_port_mask = 0;
 146 static int numa_on = 1; /**< NUMA is enabled by default. */
 147
 148 struct mbuf_table {
 149         uint16_t len;
 150         struct rte_mbuf *m_table[MAX_PKT_BURST];
 151 };
 152
 153 struct lcore_rx_queue {
 154         uint8_t port_id;
 155         uint8_t queue_id;
 156 } __rte_cache_aligned;
 157
 158 #define MAX_RX_QUEUE_PER_LCORE 16
 159 #define MAX_TX_QUEUE_PER_PORT 1
 160 #define MAX_RX_QUEUE_PER_PORT 1
 161
 162 #define MAX_LCORE_PARAMS 1024
 163 struct lcore_params {
 164         uint8_t port_id;
 165         uint8_t queue_id;
 166         uint8_t lcore_id;
 167 } __rte_cache_aligned;
 168
 169 static struct lcore_params lcore_params_array[MAX_LCORE_PARAMS];
 170 static struct lcore_params lcore_params_array_default[] = {
 171         {0, 0, 2},
 172         {0, 1, 2},
 173         {0, 2, 2},
 174         {1, 0, 2},
 175         {1, 1, 2},
 176         {1, 2, 2},
 177         {2, 0, 2},
 178         {3, 0, 3},
 179         {3, 1, 3},
 180 };
 181
 182 static struct lcore_params * lcore_params = lcore_params_array_default;
 183 static uint16_t nb_lcore_params = sizeof(lcore_params_array_default) /
 184                                 sizeof(lcore_params_array_default[0]);
 185
 186 static struct rte_eth_conf port_conf = {
 187         .rxmode = {
 188                 .split_hdr_size = 0,
 189                 .header_split   = 0, /**< Header Split disabled */
 190                 .hw_ip_checksum = 1, /**< IP checksum offload enabled */
 191                 .hw_vlan_filter = 0, /**< VLAN filtering disabled */
 192                 .jumbo_frame    = 0, /**< Jumbo Frame Support disabled */
 193                 .hw_strip_crc   = 1, /**< CRC stripped by hardware */
 194         },
 195         .rx_adv_conf = {
 196                 .rss_conf = {
 197                         .rss_key = NULL,
 198                         .rss_hf = ETH_RSS_IPV4,
 199                 },
 200         },
 201         .txmode = {
 202         },
 203 };
 204
 205 static const struct rte_eth_rxconf rx_conf = {
 206         .rx_thresh = {
 207                 .pthresh = RX_PTHRESH,
 208                 .hthresh = RX_HTHRESH,
 209                 .wthresh = RX_WTHRESH,
 210         },
 211 };
 212
 213 static const struct rte_eth_txconf tx_conf = {
 214         .tx_thresh = {
 215                 .pthresh = TX_PTHRESH,
 216                 .hthresh = TX_HTHRESH,
 217                 .wthresh = TX_WTHRESH,
 218         },
 219         .tx_free_thresh = 0, /* Use PMD default values */
 220         .tx_rs_thresh = 0, /* Use PMD default values */
 221 };
 222
 223 static struct rte_mempool * pktmbuf_pool[NB_SOCKETS];
 224
 225
 226 #if (APP_LOOKUP_METHOD == APP_LOOKUP_EXACT_MATCH)
 227 struct ipv4_5tuple {
 228         uint32_t ip_dst;
 229         uint32_t ip_src;
 230         uint16_t port_dst;
 231         uint16_t port_src;
 232         uint8_t proto;
 233 } __attribute__((__packed__));
 234
 235 struct l3fwd_route {
 236         struct ipv4_5tuple key;
 237         uint8_t if_out;
 238 };
 239
 240 static struct l3fwd_route l3fwd_route_array[] = {
 241         {{IPv4(100,10,0,1), IPv4(200,10,0,1), 101, 11, IPPROTO_TCP}, 0},
 242         {{IPv4(100,20,0,2), IPv4(200,20,0,2), 102, 12, IPPROTO_TCP}, 1},
 243         {{IPv4(100,30,0,3), IPv4(200,30,0,3), 103, 13, IPPROTO_TCP}, 2},
 244         {{IPv4(100,40,0,4), IPv4(200,40,0,4), 104, 14, IPPROTO_TCP}, 3},
 245 };
 246
 247 typedef struct rte_hash lookup_struct_t;
 248 static lookup_struct_t *l3fwd_lookup_struct[NB_SOCKETS];
 249
 250 #define L3FWD_HASH_ENTRIES      1024
 251 struct rte_hash_parameters l3fwd_hash_params = {
 252         .name = "l3fwd_hash_0",
 253         .entries = L3FWD_HASH_ENTRIES,
 254         .bucket_entries = 4,
 255         .key_len = sizeof(struct ipv4_5tuple),
 256         .hash_func = rte_hash_crc,
 257         .hash_func_init_val = 0,
 258         .socket_id = SOCKET0,
 259 };
 260
 261 #define L3FWD_NUM_ROUTES \
 262         (sizeof(l3fwd_route_array) / sizeof(l3fwd_route_array[0]))
 263
 264 static uint8_t l3fwd_out_if[L3FWD_HASH_ENTRIES] __rte_cache_aligned;
 265 #endif
 266
 267 #if (APP_LOOKUP_METHOD == APP_LOOKUP_LPM)
 268 struct l3fwd_route {
 269         uint32_t ip;
 270         uint8_t  depth;
 271         uint8_t  if_out;
 272 };
 273
 274 static struct l3fwd_route l3fwd_route_array[] = {
 275         {IPv4(1,1,1,0), 24, 0},
 276         {IPv4(2,1,1,0), 24, 1},
 277         {IPv4(3,1,1,0), 24, 2},
 278         {IPv4(4,1,1,0), 24, 3},
 279         {IPv4(5,1,1,0), 24, 4},
 280         {IPv4(6,1,1,0), 24, 5},
 281         {IPv4(7,1,1,0), 24, 6},
 282         {IPv4(8,1,1,0), 24, 7},
 283 };
 284
 285 #define L3FWD_NUM_ROUTES \
 286         (sizeof(l3fwd_route_array) / sizeof(l3fwd_route_array[0]))
 287
 288 #define L3FWD_LPM_MAX_RULES     1024
 289
 290 typedef struct rte_lpm lookup_struct_t;
 291 static lookup_struct_t *l3fwd_lookup_struct[NB_SOCKETS];
 292 #endif
 293
 294 struct lcore_conf {
 295         uint16_t n_rx_queue;
 296         struct lcore_rx_queue rx_queue_list[MAX_RX_QUEUE_PER_LCORE];
 297         uint16_t tx_queue_id;
 298         struct mbuf_table tx_mbufs[MAX_PORTS];
 299         lookup_struct_t * lookup_struct;
 300 } __rte_cache_aligned;
 301
 302 static struct lcore_conf lcore_conf[RTE_MAX_LCORE];
 303
 304 /* Send burst of packets on an output interface */
 305 static inline int
 306 send_burst(struct lcore_conf *qconf, uint16_t n, uint8_t port)
 307 {
 308         struct rte_mbuf **m_table;
 309         int ret;
 310         uint16_t queueid;
 311
 312         queueid = qconf->tx_queue_id;
 313         m_table = (struct rte_mbuf **)qconf->tx_mbufs[port].m_table;
 314
 315         ret = rte_eth_tx_burst(port, queueid, m_table, n);
 316         if (unlikely(ret < n)) {
 317                 do {
 318                         rte_pktmbuf_free(m_table[ret]);
 319                 } while (++ret < n);
 320         }
 321
 322         return 0;
 323 }
 324
 325 /* Enqueue a single packet, and send burst if queue is filled */
 326 static inline int
 327 send_single_packet(struct rte_mbuf *m, uint8_t port)
 328 {
 329         uint32_t lcore_id;
 330         uint16_t len;
 331         struct lcore_conf *qconf;
 332
 333         lcore_id = rte_lcore_id();
 334
 335         qconf = &lcore_conf[lcore_id];
 336         len = qconf->tx_mbufs[port].len;
 337         qconf->tx_mbufs[port].m_table[len] = m;
 338         len++;
 339
 340         /* enough pkts to be sent */
 341         if (unlikely(len == MAX_PKT_BURST)) {
 342                 send_burst(qconf, MAX_PKT_BURST, port);
 343                 len = 0;
 344         }
 345
 346         qconf->tx_mbufs[port].len = len;
 347         return 0;
 348 }
 349
 350 #ifdef DO_RFC_1812_CHECKS
 351 static inline int
 352 is_valid_ipv4_pkt(struct ipv4_hdr *pkt, uint32_t link_len)
 353 {
 354         /* From http://www.rfc-editor.org/rfc/rfc1812.txt section 5.2.2 */
 355         /*
 356          * 1. The packet length reported by the Link Layer must be large
 357          * enough to hold the minimum length legal IP datagram (20 bytes).
 358          */
 359         if (link_len < sizeof(struct ipv4_hdr))
 360                 return -1;
 361
 362         /* 2. The IP checksum must be correct. */
 363         /* this is checked in H/W */
 364
 365         /*
 366          * 3. The IP version number must be 4. If the version number is not 4
 367          * then the packet may be another version of IP, such as IPng or
 368          * ST-II.
 369          */
 370         if (((pkt->version_ihl) >> 4) != 4)
 371                 return -3;
 372         /*
 373          * 4. The IP header length field must be large enough to hold the
 374          * minimum length legal IP datagram (20 bytes = 5 words).
 375          */
 376         if ((pkt->version_ihl & 0xf) < 5)
 377                 return -4;
 378
 379         /*
 380          * 5. The IP total length field must be large enough to hold the IP
 381          * datagram header, whose length is specified in the IP header length
 382          * field.
 383          */
 384         if (rte_cpu_to_be_16(pkt->total_length) < sizeof(struct ipv4_hdr))
 385                 return -5;
 386
 387         return 0;
 388 }
 389 #endif
 390
 391 #if (APP_LOOKUP_METHOD == APP_LOOKUP_EXACT_MATCH)
 392 static void
 393 print_key(struct ipv4_5tuple key)
 394 {
 395         printf("IP dst = %08x, IP src = %08x, port dst = %d, port src = %d, proto = %d\n",
 396                (unsigned)key.ip_dst, (unsigned)key.ip_src, key.port_dst, key.port_src, key.proto);
 397 }
 398
 399 static inline uint8_t
 400 get_dst_port(struct ipv4_hdr *ipv4_hdr,  uint8_t portid, lookup_struct_t * l3fwd_lookup_struct)
 401 {
 402         struct ipv4_5tuple key;
 403         struct tcp_hdr *tcp;
 404         struct udp_hdr *udp;
 405         int ret = 0;
 406
 407         key.ip_dst = rte_be_to_cpu_32(ipv4_hdr->dst_addr);
 408         key.ip_src = rte_be_to_cpu_32(ipv4_hdr->src_addr);
 409         key.proto = ipv4_hdr->next_proto_id;
 410
 411         switch (ipv4_hdr->next_proto_id) {
 412         case IPPROTO_TCP:
 413                 tcp = (struct tcp_hdr *)((unsigned char *) ipv4_hdr +
 414                                         sizeof(struct ipv4_hdr));
 415                 key.port_dst = rte_be_to_cpu_16(tcp->dst_port);
 416                 key.port_src = rte_be_to_cpu_16(tcp->src_port);
 417                 break;
 418
 419         case IPPROTO_UDP:
 420                 udp = (struct udp_hdr *)((unsigned char *) ipv4_hdr +
 421                                         sizeof(struct ipv4_hdr));
 422                 key.port_dst = rte_be_to_cpu_16(udp->dst_port);
 423                 key.port_src = rte_be_to_cpu_16(udp->src_port);
 424                 break;
 425
 426         default:
 427                 key.port_dst = 0;
 428                 key.port_src = 0;
 429         }
 430
 431         /* Find destination port */
 432         ret = rte_hash_lookup(l3fwd_lookup_struct, (const void *)&key);
 433         return (uint8_t)((ret < 0)? portid : l3fwd_out_if[ret]);
 434 }
 435 #endif
 436
 437 #if (APP_LOOKUP_METHOD == APP_LOOKUP_LPM)
 438 static inline uint8_t
 439 get_dst_port(struct ipv4_hdr *ipv4_hdr,  uint8_t portid, lookup_struct_t * l3fwd_lookup_struct)
 440 {
 441         uint8_t next_hop;
 442
 443         return (uint8_t) ((rte_lpm_lookup(l3fwd_lookup_struct,
 444                         rte_be_to_cpu_32(ipv4_hdr->dst_addr), &next_hop) == 0)?
 445                         next_hop : portid);
 446 }
 447 #endif
 448
 449 static inline void
 450 l3fwd_simple_forward(struct rte_mbuf *m, uint8_t portid, lookup_struct_t * l3fwd_lookup_struct)
 451 {
 452         struct ether_hdr *eth_hdr;
 453         struct ipv4_hdr *ipv4_hdr;
 454         void *tmp;
 455         uint8_t dst_port;
 456
 457         eth_hdr = rte_pktmbuf_mtod(m, struct ether_hdr *);
 458
 459         ipv4_hdr = (struct ipv4_hdr *)(rte_pktmbuf_mtod(m, unsigned char *) +
 460                                 sizeof(struct ether_hdr));
 461
 462 #ifdef DO_RFC_1812_CHECKS
 463         /* Check to make sure the packet is valid (RFC1812) */
 464         if (is_valid_ipv4_pkt(ipv4_hdr, m->pkt.pkt_len) < 0) {
 465                 rte_pktmbuf_free(m);
 466                 return;
 467         }
 468 #endif
 469
 470         dst_port = get_dst_port(ipv4_hdr, portid, l3fwd_lookup_struct);
 471         if (dst_port >= MAX_PORTS || (enabled_port_mask & 1 << dst_port) == 0)
 472                 dst_port = portid;
 473
 474         /* 00:09:c0:00:00:xx */
 475         tmp = &eth_hdr->d_addr.addr_bytes[0];
 476         *((uint64_t *)tmp) = 0x000000c00900 + (dst_port << 24);
 477
 478 #ifdef DO_RFC_1812_CHECKS
 479         /* Update time to live and header checksum */
 480         --(ipv4_hdr->time_to_live);
 481         ++(ipv4_hdr->hdr_checksum);
 482 #endif
 483
 484         /* src addr */
 485         ether_addr_copy(&ports_eth_addr[dst_port], &eth_hdr->s_addr);
 486
 487         send_single_packet(m, dst_port);
 488
 489 }
 490
 491 /* main processing loop */
 492 static __attribute__((noreturn)) int
 493 main_loop(__attribute__((unused)) void *dummy)
 494 {
 495         struct rte_mbuf *pkts_burst[MAX_PKT_BURST];
 496         unsigned lcore_id;
 497         uint64_t prev_tsc = 0;
 498         uint64_t diff_tsc, cur_tsc;
 499         int i, j, nb_rx;
 500         uint8_t portid, queueid;
 501         struct lcore_conf *qconf;
 502
 503         lcore_id = rte_lcore_id();
 504         qconf = &lcore_conf[lcore_id];
 505
 506         if (qconf->n_rx_queue == 0) {
 507                 RTE_LOG(INFO, L3FWD, "lcore %u has nothing to do\n", lcore_id);
 508                 while(1);
 509         }
 510
 511         RTE_LOG(INFO, L3FWD, "entering main loop on lcore %u\n", lcore_id);
 512
 513         for (i = 0; i < qconf->n_rx_queue; i++) {
 514
 515                 portid = qconf->rx_queue_list[i].port_id;
 516                 queueid = qconf->rx_queue_list[i].queue_id;
 517                 RTE_LOG(INFO, L3FWD, " -- lcoreid=%u portid=%hhu rxqueueid=%hhu\n", lcore_id,
 518                         portid, queueid);
 519         }
 520
 521         while (1) {
 522
 523                 cur_tsc = rte_rdtsc();
 524
 525                 /*
 526                  * TX burst queue drain
 527                  */
 528                 diff_tsc = cur_tsc - prev_tsc;
 529                 if (unlikely(diff_tsc > BURST_TX_DRAIN)) {
 530
 531                         /*
 532                          * This could be optimized (use queueid instead of
 533                          * portid), but it is not called so often
 534                          */
 535                         for (portid = 0; portid < MAX_PORTS; portid++) {
 536                                 if (qconf->tx_mbufs[portid].len == 0)
 537                                         continue;
 538                                 send_burst(&lcore_conf[lcore_id],
 539                                         qconf->tx_mbufs[portid].len,
 540                                         portid);
 541                                 qconf->tx_mbufs[portid].len = 0;
 542                         }
 543
 544                         prev_tsc = cur_tsc;
 545                 }
 546
 547                 /*
 548                  * Read packet from RX queues
 549                  */
 550                 for (i = 0; i < qconf->n_rx_queue; ++i) {
 551
 552                         portid = qconf->rx_queue_list[i].port_id;
 553                         queueid = qconf->rx_queue_list[i].queue_id;
 554                         nb_rx = rte_eth_rx_burst(portid, queueid, pkts_burst, MAX_PKT_BURST);
 555
 556                         /* Prefetch first packets */
 557                         for (j = 0; j < PREFETCH_OFFSET && j < nb_rx; j++) {
 558                                 rte_prefetch0(rte_pktmbuf_mtod(
 559                                                 pkts_burst[j], void *));
 560                         }
 561
 562                         /* Prefetch and forward already prefetched packets */
 563                         for (j = 0; j < (nb_rx - PREFETCH_OFFSET); j++) {
 564                                 rte_prefetch0(rte_pktmbuf_mtod(pkts_burst[
 565                                                 j + PREFETCH_OFFSET], void *));
 566                                 l3fwd_simple_forward(pkts_burst[j], portid, qconf->lookup_struct);
 567                         }
 568
 569                         /* Forward remaining prefetched packets */
 570                         for (; j < nb_rx; j++) {
 571                                 l3fwd_simple_forward(pkts_burst[j], portid, qconf->lookup_struct);
 572                         }
 573                 }
 574         }
 575 }
 576
 577 static int
 578 check_lcore_params(void)
 579 {
 580         uint8_t queue, lcore;
 581         uint16_t i;
 582         int socketid;
 583
 584         for (i = 0; i < nb_lcore_params; ++i) {
 585                 queue = lcore_params[i].queue_id;
 586                 if (queue >= MAX_RX_QUEUE_PER_PORT) {
 587                         printf("invalid queue number: %hhu\n", queue);
 588                         return -1;
 589                 }
 590                 lcore = lcore_params[i].lcore_id;
 591                 if (!rte_lcore_is_enabled(lcore)) {
 592                         printf("error: lcore %hhu is not enabled in lcore mask\n", lcore);
 593                         return -1;
 594                 }
 595                 if ((socketid = rte_lcore_to_socket_id(lcore) != 0) &&
 596                         (numa_on == 0)) {
 597                         printf("warning: lcore %hhu is on socket %d with numa off \n",
 598                                 lcore, socketid);
 599                 }
 600         }
 601         return 0;
 602 }
 603
 604 static int
 605 check_port_config(const unsigned nb_ports)
 606 {
 607         unsigned portid;
 608         uint16_t i;
 609
 610         for (i = 0; i < nb_lcore_params; ++i) {
 611                 portid = lcore_params[i].port_id;
 612                 if ((enabled_port_mask & (1 << portid)) == 0) {
 613                         printf("port %u is not enabled in port mask\n", portid);
 614                         return -1;
 615                 }
 616                 if (portid >= nb_ports) {
 617                         printf("port %u is not present on the board\n", portid);
 618                         return -1;
 619                 }
 620         }
 621         return 0;
 622 }
 623
 624 static uint8_t
 625 get_port_n_rx_queues(const uint8_t port)
 626 {
 627         int queue = -1;
 628         uint16_t i;
 629
 630         for (i = 0; i < nb_lcore_params; ++i) {
 631                 if (lcore_params[i].port_id == port && lcore_params[i].queue_id > queue)
 632                         queue = lcore_params[i].queue_id;
 633         }
 634         return (uint8_t)(++queue);
 635 }
 636
 637 static int
 638 init_lcore_rx_queues(void)
 639 {
 640         uint16_t i, nb_rx_queue;
 641         uint8_t lcore;
 642
 643         for (i = 0; i < nb_lcore_params; ++i) {
 644                 lcore = lcore_params[i].lcore_id;
 645                 nb_rx_queue = lcore_conf[lcore].n_rx_queue;
 646                 if (nb_rx_queue >= MAX_RX_QUEUE_PER_LCORE) {
 647                         printf("error: too many queues (%u) for lcore: %u\n",
 648                                 (unsigned)nb_rx_queue + 1, (unsigned)lcore);
 649                         return -1;
 650                 } else {
 651                         lcore_conf[lcore].rx_queue_list[nb_rx_queue].port_id =
 652                                 lcore_params[i].port_id;
 653                         lcore_conf[lcore].rx_queue_list[nb_rx_queue].queue_id =
 654                                 lcore_params[i].queue_id;
 655                         lcore_conf[lcore].n_rx_queue++;
 656                 }
 657         }
 658         return 0;
 659 }
 660
 661 /* display usage */
 662 static void
 663 print_usage(const char *prgname)
 664 {
 665         printf ("%s [EAL options] -- -p PORTMASK"
 666                 "  [--config (port,queue,lcore)[,(port,queue,lcore]]\n"
 667                 "  -p PORTMASK: hexadecimal bitmask of ports to configure\n"
 668                 "  --config (port,queue,lcore): rx queues configuration\n"
 669                 "  --no-numa: optional, disable numa awareness\n",
 670                 prgname);
 671 }
 672
 673 static int
 674 parse_portmask(const char *portmask)
 675 {
 676         char *end = NULL;
 677         unsigned long pm;
 678
 679         /* parse hexadecimal string */
 680         pm = strtoul(portmask, &end, 16);
 681         if ((portmask[0] == '\0') || (end == NULL) || (*end != '\0'))
 682                 return -1;
 683
 684         if (pm == 0)
 685                 return -1;
 686
 687         return pm;
 688 }
 689
 690 static int
 691 parse_config(const char *q_arg)
 692 {
 693         char s[256];
 694         const char *p, *p0 = q_arg;
 695         char *end;
 696         enum fieldnames {
 697                 FLD_PORT = 0,
 698                 FLD_QUEUE,
 699                 FLD_LCORE,
 700                 _NUM_FLD
 701         };
 702         unsigned long int_fld[_NUM_FLD];
 703         char *str_fld[_NUM_FLD];
 704         int i;
 705         unsigned size;
 706
 707         nb_lcore_params = 0;
 708
 709         while ((p = strchr(p0,'(')) != NULL) {
 710                 ++p;
 711                 if((p0 = strchr(p,')')) == NULL)
 712                         return -1;
 713
 714                 size = p0 - p;
 715                 if(size >= sizeof(s))
 716                         return -1;
 717
 718                 rte_snprintf(s, sizeof(s), "%.*s", size, p);
 719                 if (rte_strsplit(s, sizeof(s), str_fld, _NUM_FLD, ',') != _NUM_FLD)
 720                         return -1;
 721                 for (i = 0; i < _NUM_FLD; i++){
 722                         errno = 0;
 723                         int_fld[i] = strtoul(str_fld[i], &end, 0);
 724                         if (errno != 0 || end == str_fld[i] || int_fld[i] > 255)
 725                                 return -1;
 726                 }
 727                 if (nb_lcore_params >= MAX_LCORE_PARAMS) {
 728                         printf("exceeded max number of lcore params: %hu\n",
 729                                 nb_lcore_params);
 730                         return -1;
 731                 }
 732                 lcore_params_array[nb_lcore_params].port_id = (uint8_t)int_fld[FLD_PORT];
 733                 lcore_params_array[nb_lcore_params].queue_id = (uint8_t)int_fld[FLD_QUEUE];
 734                 lcore_params_array[nb_lcore_params].lcore_id = (uint8_t)int_fld[FLD_LCORE];
 735                 ++nb_lcore_params;
 736         }
 737         lcore_params = lcore_params_array;
 738         return 0;
 739 }
 740
 741 /* Parse the argument given in the command line of the application */
 742 static int
 743 parse_args(int argc, char **argv)
 744 {
 745         int opt, ret;
 746         char **argvopt;
 747         int option_index;
 748         char *prgname = argv[0];
 749         static struct option lgopts[] = {
 750                 {"config", 1, 0, 0},
 751                 {"no-numa", 0, 0, 0},
 752                 {NULL, 0, 0, 0}
 753         };
 754
 755         argvopt = argv;
 756
 757         while ((opt = getopt_long(argc, argvopt, "p:",
 758                                 lgopts, &option_index)) != EOF) {
 759
 760                 switch (opt) {
 761                 /* portmask */
 762                 case 'p':
 763                         enabled_port_mask = parse_portmask(optarg);
 764                         if (enabled_port_mask == 0) {
 765                                 printf("invalid portmask\n");
 766                                 print_usage(prgname);
 767                                 return -1;
 768                         }
 769                         break;
 770
 771                 /* long options */
 772                 case 0:
 773                         if (!strcmp(lgopts[option_index].name, "config")) {
 774                                 ret = parse_config(optarg);
 775                                 if (ret) {
 776                                         printf("invalid config\n");
 777                                         print_usage(prgname);
 778                                         return -1;
 779                                 }
 780                         }
 781
 782                         if (!strcmp(lgopts[option_index].name, "no-numa")) {
 783                                 printf("numa is disabled \n");
 784                                 numa_on = 0;
 785                         }
 786                         break;
 787
 788                 default:
 789                         print_usage(prgname);
 790                         return -1;
 791                 }
 792         }
 793
 794         if (optind >= 0)
 795                 argv[optind-1] = prgname;
 796
 797         ret = optind-1;
 798         optind = 0; /* reset getopt lib */
 799         return ret;
 800 }
 801
 802 static void
 803 print_ethaddr(const char *name, const struct ether_addr *eth_addr)
 804 {
 805         printf ("%s%02X:%02X:%02X:%02X:%02X:%02X", name,
 806                 eth_addr->addr_bytes[0],
 807                 eth_addr->addr_bytes[1],
 808                 eth_addr->addr_bytes[2],
 809                 eth_addr->addr_bytes[3],
 810                 eth_addr->addr_bytes[4],
 811                 eth_addr->addr_bytes[5]);
 812 }
 813
 814 #if (APP_LOOKUP_METHOD == APP_LOOKUP_EXACT_MATCH)
 815 static void
 816 setup_hash(int socketid)
 817 {
 818         unsigned i;
 819         int ret;
 820         char s[64];
 821
 822         /* create  hashes */
 823         rte_snprintf(s, sizeof(s), "l3fwd_hash_%d", socketid);
 824         l3fwd_hash_params.name = s;
 825         l3fwd_hash_params.socket_id = socketid;
 826         l3fwd_lookup_struct[socketid] = rte_hash_create(&l3fwd_hash_params);
 827         if (l3fwd_lookup_struct[socketid] == NULL)
 828                 rte_exit(EXIT_FAILURE, "Unable to create the l3fwd hash on "
 829                                 "socket %d\n", socketid);
 830
 831         /* populate the hash */
 832         for (i = 0; i < L3FWD_NUM_ROUTES; i++) {
 833                 ret = rte_hash_add_key (l3fwd_lookup_struct[socketid],
 834                                 (void *) &l3fwd_route_array[i].key);
 835                 if (ret < 0) {
 836                         rte_exit(EXIT_FAILURE, "Unable to add entry %u to the"
 837                                 "l3fwd hash on socket %d\n", i, socketid);
 838                 }
 839                 l3fwd_out_if[ret] = l3fwd_route_array[i].if_out;
 840                 printf("Hash: Adding key\n");
 841                 print_key(l3fwd_route_array[i].key);
 842         }
 843 }
 844 #endif
 845
 846 #if (APP_LOOKUP_METHOD == APP_LOOKUP_LPM)
 847 static void
 848 setup_lpm(int socketid)
 849 {
 850         unsigned i;
 851         int ret;
 852         char s[64];
 853
 854         /* create the LPM table */
 855         rte_snprintf(s, sizeof(s), "L3FWD_LPM_%d", socketid);
 856         l3fwd_lookup_struct[socketid] = rte_lpm_create(s, socketid,
 857                                 L3FWD_LPM_MAX_RULES, RTE_LPM_MEMZONE);
 858         if (l3fwd_lookup_struct[socketid] == NULL)
 859                 rte_exit(EXIT_FAILURE, "Unable to create the l3fwd LPM table"
 860                                 " on socket %d\n", socketid);
 861
 862         /* populate the LPM table */
 863         for (i = 0; i < L3FWD_NUM_ROUTES; i++) {
 864                 ret = rte_lpm_add(l3fwd_lookup_struct[socketid],
 865                         l3fwd_route_array[i].ip,
 866                         l3fwd_route_array[i].depth,
 867                         l3fwd_route_array[i].if_out);
 868
 869                 if (ret < 0) {
 870                         rte_exit(EXIT_FAILURE, "Unable to add entry %u to the "
 871                                 "l3fwd LPM table on socket %d\n",
 872                                 i, socketid);
 873                 }
 874
 875                 printf("LPM: Adding route 0x%08x / %d (%d)\n",
 876                         (unsigned)l3fwd_route_array[i].ip,
 877                         l3fwd_route_array[i].depth,
 878                         l3fwd_route_array[i].if_out);
 879         }
 880 }
 881 #endif
 882
 883 static int
 884 init_mem(void)
 885 {
 886         struct lcore_conf *qconf;
 887         int socketid;
 888         unsigned lcore_id;
 889         char s[64];
 890
 891         for (lcore_id = 0; lcore_id < RTE_MAX_LCORE; lcore_id++) {
 892                 if (rte_lcore_is_enabled(lcore_id) == 0)
 893                         continue;
 894
 895                 if (numa_on)
 896                         socketid = rte_lcore_to_socket_id(lcore_id);
 897                 else
 898                         socketid = 0;
 899
 900                 if (socketid >= NB_SOCKETS) {
 901                         rte_exit(EXIT_FAILURE, "Socket %d of lcore %u is out of range %d\n",
 902                                 socketid, lcore_id, NB_SOCKETS);
 903                 }
 904                 if (pktmbuf_pool[socketid] == NULL) {
 905                         rte_snprintf(s, sizeof(s), "mbuf_pool_%d", socketid);
 906                         pktmbuf_pool[socketid] =
 907                                 rte_mempool_create(s, NB_MBUF, MBUF_SIZE, 32,
 908                                         sizeof(struct rte_pktmbuf_pool_private),
 909                                         rte_pktmbuf_pool_init, NULL,
 910                                         rte_pktmbuf_init, NULL,
 911                                         socketid, 0);
 912                         if (pktmbuf_pool[socketid] == NULL)
 913                                 rte_exit(EXIT_FAILURE, "Cannot init mbuf pool on socket %d\n", socketid);
 914                         else
 915                                 printf("Allocated mbuf pool on socket %d\n", socketid);
 916
 917 #if (APP_LOOKUP_METHOD == APP_LOOKUP_LPM)
 918                         setup_lpm(socketid);
 919 #else
 920                         setup_hash(socketid);
 921 #endif
 922                 }
 923                 qconf = &lcore_conf[lcore_id];
 924                 qconf->lookup_struct = l3fwd_lookup_struct[socketid];
 925         }
 926         return 0;
 927 }
 928
 929 int
 930 MAIN(int argc, char **argv)
 931 {
 932         struct lcore_conf *qconf;
 933         int ret;
 934         unsigned nb_ports;
 935         uint16_t queueid;
 936         unsigned lcore_id;
 937         uint8_t portid, nb_rx_queue, queue, socketid;
 938
 939         /* init EAL */
 940         ret = rte_eal_init(argc, argv);
 941         if (ret < 0)
 942                 rte_exit(EXIT_FAILURE, "Invalid EAL parameters\n");
 943         argc -= ret;
 944         argv += ret;
 945
 946         /* parse application arguments (after the EAL ones) */
 947         ret = parse_args(argc, argv);
 948         if (ret < 0)
 949                 rte_exit(EXIT_FAILURE, "Invalid L3FWD-VF parameters\n");
 950
 951         if (check_lcore_params() < 0)
 952                 rte_exit(EXIT_FAILURE, "check_lcore_params failed\n");
 953
 954         ret = init_lcore_rx_queues();
 955         if (ret < 0)
 956                 rte_exit(EXIT_FAILURE, "init_lcore_rx_queues failed\n");
 957
 958         ret = init_mem();
 959         if (ret < 0)
 960                 rte_exit(EXIT_FAILURE, "init_mem failed\n");
 961
 962         /* init driver */
 963 #ifdef RTE_LIBRTE_IGB_PMD
 964         if (rte_igb_pmd_init() < 0)
 965                 rte_exit(EXIT_FAILURE, "Cannot init igb pmd\n");
 966 #endif
 967 #ifdef RTE_LIBRTE_IXGBE_PMD
 968         if (rte_ixgbe_pmd_init() < 0)
 969                 rte_exit(EXIT_FAILURE, "Cannot init ixgbe pmd\n");
 970
 971         if (rte_ixgbevf_pmd_init() < 0)
 972                 rte_exit(EXIT_FAILURE, "Cannot init ixgbevf pmd\n");
 973 #endif
 974
 975         if (rte_eal_pci_probe() < 0)
 976                 rte_exit(EXIT_FAILURE, "Cannot probe PCI\n");
 977
 978         nb_ports = rte_eth_dev_count();
 979         if (nb_ports > MAX_PORTS)
 980                 nb_ports = MAX_PORTS;
 981
 982         if (check_port_config(nb_ports) < 0)
 983                 rte_exit(EXIT_FAILURE, "check_port_config failed\n");
 984
 985         /* initialize all ports */
 986         for (portid = 0; portid < nb_ports; portid++) {
 987                 /* skip ports that are not enabled */
 988                 if ((enabled_port_mask & (1 << portid)) == 0) {
 989                         printf("\nSkipping disabled port %d\n", portid);
 990                         continue;
 991                 }
 992
 993                 /* init port */
 994                 printf("Initializing port %d ... ", portid );
 995                 fflush(stdout);
 996
 997                 /* must always equal(=1) */
 998                 nb_rx_queue = get_port_n_rx_queues(portid);
 999
1000                 printf("Creating queues: nb_rxq=%d nb_txq=%u... ",
1001                         nb_rx_queue, (unsigned)1 );
1002                 ret = rte_eth_dev_configure(portid, nb_rx_queue, 1, &port_conf);
1003                 if (ret < 0)
1004                         rte_exit(EXIT_FAILURE, "Cannot configure device: err=%d, port=%d\n",
1005                                 ret, portid);
1006
1007                 rte_eth_macaddr_get(portid, &ports_eth_addr[portid]);
1008                 print_ethaddr(" Address:", &ports_eth_addr[portid]);
1009                 printf(", ");
1010
1011                 /* init one TX queue */
1012                 socketid = 0;
1013
1014                 printf("txq=%d,%d ", 0, socketid);
1015                 fflush(stdout);
1016                 ret = rte_eth_tx_queue_setup(portid, 0, nb_txd,
1017                                                  socketid, &tx_conf);
1018                 if (ret < 0)
1019                         rte_exit(EXIT_FAILURE, "rte_eth_tx_queue_setup: err=%d, "
1020                                 "port=%d\n", ret, portid);
1021
1022                 printf("\n");
1023         }
1024
1025         for (lcore_id = 0; lcore_id < RTE_MAX_LCORE; lcore_id++) {
1026                 if (rte_lcore_is_enabled(lcore_id) == 0)
1027                         continue;
1028                 qconf = &lcore_conf[lcore_id];
1029                 qconf->tx_queue_id = 0;
1030
1031                 printf("\nInitializing rx queues on lcore %u ... ", lcore_id );
1032                 fflush(stdout);
1033                 /* init RX queues */
1034                 for(queue = 0; queue < qconf->n_rx_queue; ++queue) {
1035                         portid = qconf->rx_queue_list[queue].port_id;
1036                         queueid = qconf->rx_queue_list[queue].queue_id;
1037
1038                         if (numa_on)
1039                                 socketid = (uint8_t)rte_lcore_to_socket_id(lcore_id);
1040                         else
1041                                 socketid = 0;
1042
1043                         printf("rxq=%d,%d,%d ", portid, queueid, socketid);
1044                         fflush(stdout);
1045
1046                         ret = rte_eth_rx_queue_setup(portid, queueid, nb_rxd,
1047                                                 socketid, &rx_conf, pktmbuf_pool[socketid]);
1048                         if (ret < 0)
1049                                 rte_exit(EXIT_FAILURE, "rte_eth_rx_queue_setup: err=%d,"
1050                                                 "port=%d\n", ret, portid);
1051                 }
1052         }
1053         printf("\n");
1054
1055         /* start ports */
1056         for (portid = 0; portid < nb_ports; portid++) {
1057                 if ((enabled_port_mask & (1 << portid)) == 0) {
1058                         continue;
1059                 }
1060                 /* Start device */
1061                 ret = rte_eth_dev_start(portid);
1062                 if (ret < 0)
1063                         rte_exit(EXIT_FAILURE, "rte_eth_dev_start: err=%d, port=%d\n",
1064                                 ret, portid);
1065
1066                 printf("done: Port %d\n", portid);
1067
1068         }
1069
1070         /* launch per-lcore init on every lcore */
1071         rte_eal_mp_remote_launch(main_loop, NULL, CALL_MASTER);
1072         RTE_LCORE_FOREACH_SLAVE(lcore_id) {
1073                 if (rte_eal_wait_lcore(lcore_id) < 0)
1074                         return -1;
1075         }
1076
1077         return 0;
1078 }