examples/l3fwd-power/main.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(c) 2010-2018 Intel Corporation
   3  */
   4
   5 #include <stdio.h>
   6 #include <stdlib.h>
   7 #include <stdint.h>
   8 #include <inttypes.h>
   9 #include <sys/types.h>
  10 #include <string.h>
  11 #include <sys/queue.h>
  12 #include <stdarg.h>
  13 #include <errno.h>
  14 #include <getopt.h>
  15 #include <unistd.h>
  16 #include <signal.h>
  17 #include <math.h>
  18
  19 #include <rte_common.h>
  20 #include <rte_byteorder.h>
  21 #include <rte_log.h>
  22 #include <rte_malloc.h>
  23 #include <rte_memory.h>
  24 #include <rte_memcpy.h>
  25 #include <rte_eal.h>
  26 #include <rte_launch.h>
  27 #include <rte_atomic.h>
  28 #include <rte_cycles.h>
  29 #include <rte_prefetch.h>
  30 #include <rte_lcore.h>
  31 #include <rte_per_lcore.h>
  32 #include <rte_branch_prediction.h>
  33 #include <rte_interrupts.h>
  34 #include <rte_random.h>
  35 #include <rte_debug.h>
  36 #include <rte_ether.h>
  37 #include <rte_ethdev.h>
  38 #include <rte_mempool.h>
  39 #include <rte_mbuf.h>
  40 #include <rte_ip.h>
  41 #include <rte_tcp.h>
  42 #include <rte_udp.h>
  43 #include <rte_string_fns.h>
  44 #include <rte_timer.h>
  45 #include <rte_power.h>
  46 #include <rte_spinlock.h>
  47 #include <rte_power_empty_poll.h>
  48 #include <rte_metrics.h>
  49 #include <rte_telemetry.h>
  50
  51 #include "perf_core.h"
  52 #include "main.h"
  53
  54 #define RTE_LOGTYPE_L3FWD_POWER RTE_LOGTYPE_USER1
  55
  56 #define MAX_PKT_BURST 32
  57
  58 #define MIN_ZERO_POLL_COUNT 10
  59
  60 /* 100 ms interval */
  61 #define TIMER_NUMBER_PER_SECOND           10
  62 /* (10ms) */
  63 #define INTERVALS_PER_SECOND             100
  64 /* 100000 us */
  65 #define SCALING_PERIOD                    (1000000/TIMER_NUMBER_PER_SECOND)
  66 #define SCALING_DOWN_TIME_RATIO_THRESHOLD 0.25
  67
  68 #define APP_LOOKUP_EXACT_MATCH          0
  69 #define APP_LOOKUP_LPM                  1
  70 #define DO_RFC_1812_CHECKS
  71
  72 #ifndef APP_LOOKUP_METHOD
  73 #define APP_LOOKUP_METHOD             APP_LOOKUP_LPM
  74 #endif
  75
  76 #if (APP_LOOKUP_METHOD == APP_LOOKUP_EXACT_MATCH)
  77 #include <rte_hash.h>
  78 #elif (APP_LOOKUP_METHOD == APP_LOOKUP_LPM)
  79 #include <rte_lpm.h>
  80 #else
  81 #error "APP_LOOKUP_METHOD set to incorrect value"
  82 #endif
  83
  84 #ifndef IPv6_BYTES
  85 #define IPv6_BYTES_FMT "%02x%02x:%02x%02x:%02x%02x:%02x%02x:"\
  86                        "%02x%02x:%02x%02x:%02x%02x:%02x%02x"
  87 #define IPv6_BYTES(addr) \
  88         addr[0],  addr[1], addr[2],  addr[3], \
  89         addr[4],  addr[5], addr[6],  addr[7], \
  90         addr[8],  addr[9], addr[10], addr[11],\
  91         addr[12], addr[13],addr[14], addr[15]
  92 #endif
  93
  94 #define MAX_JUMBO_PKT_LEN  9600
  95
  96 #define IPV6_ADDR_LEN 16
  97
  98 #define MEMPOOL_CACHE_SIZE 256
  99
 100 /*
 101  * This expression is used to calculate the number of mbufs needed depending on
 102  * user input, taking into account memory for rx and tx hardware rings, cache
 103  * per lcore and mtable per port per lcore. RTE_MAX is used to ensure that
 104  * NB_MBUF never goes below a minimum value of 8192.
 105  */
 106
 107 #define NB_MBUF RTE_MAX ( \
 108         (nb_ports*nb_rx_queue*nb_rxd + \
 109         nb_ports*nb_lcores*MAX_PKT_BURST + \
 110         nb_ports*n_tx_queue*nb_txd + \
 111         nb_lcores*MEMPOOL_CACHE_SIZE), \
 112         (unsigned)8192)
 113
 114 #define BURST_TX_DRAIN_US 100 /* TX drain every ~100us */
 115
 116 #define NB_SOCKETS 8
 117
 118 /* Configure how many packets ahead to prefetch, when reading packets */
 119 #define PREFETCH_OFFSET 3
 120
 121 /*
 122  * Configurable number of RX/TX ring descriptors
 123  */
 124 #define RTE_TEST_RX_DESC_DEFAULT 1024
 125 #define RTE_TEST_TX_DESC_DEFAULT 1024
 126
 127 /*
 128  * These two thresholds were decided on by running the training algorithm on
 129  * a 2.5GHz Xeon. These defaults can be overridden by supplying non-zero values
 130  * for the med_threshold and high_threshold parameters on the command line.
 131  */
 132 #define EMPTY_POLL_MED_THRESHOLD 350000UL
 133 #define EMPTY_POLL_HGH_THRESHOLD 580000UL
 134
 135 #define NUM_TELSTATS RTE_DIM(telstats_strings)
 136
 137 static uint16_t nb_rxd = RTE_TEST_RX_DESC_DEFAULT;
 138 static uint16_t nb_txd = RTE_TEST_TX_DESC_DEFAULT;
 139
 140 /* ethernet addresses of ports */
 141 static struct rte_ether_addr ports_eth_addr[RTE_MAX_ETHPORTS];
 142
 143 /* ethernet addresses of ports */
 144 static rte_spinlock_t locks[RTE_MAX_ETHPORTS];
 145
 146 /* mask of enabled ports */
 147 static uint32_t enabled_port_mask = 0;
 148 /* Ports set in promiscuous mode off by default. */
 149 static int promiscuous_on = 0;
 150 /* NUMA is enabled by default. */
 151 static int numa_on = 1;
 152 static bool empty_poll_stop;
 153 static bool empty_poll_train;
 154 volatile bool quit_signal;
 155 static struct  ep_params *ep_params;
 156 static struct  ep_policy policy;
 157 static long  ep_med_edpi, ep_hgh_edpi;
 158 /* timer to update telemetry every 500ms */
 159 static struct rte_timer telemetry_timer;
 160
 161 /* stats index returned by metrics lib */
 162 int telstats_index;
 163
 164 struct telstats_name {
 165         char name[RTE_ETH_XSTATS_NAME_SIZE];
 166 };
 167
 168 /* telemetry stats to be reported */
 169 const struct telstats_name telstats_strings[] = {
 170         {"empty_poll"},
 171         {"full_poll"},
 172         {"busy_percent"}
 173 };
 174
 175 /* core busyness in percentage */
 176 enum busy_rate {
 177         ZERO = 0,
 178         PARTIAL = 50,
 179         FULL = 100
 180 };
 181
 182 /* reference poll count to measure core busyness */
 183 #define DEFAULT_COUNT 10000
 184 /*
 185  * reference CYCLES to be used to
 186  * measure core busyness based on poll count
 187  */
 188 #define MIN_CYCLES  1500000ULL
 189 #define MAX_CYCLES 22000000ULL
 190
 191 /* (500ms) */
 192 #define TELEMETRY_INTERVALS_PER_SEC 2
 193
 194 static int parse_ptype; /**< Parse packet type using rx callback, and */
 195                         /**< disabled by default */
 196
 197 enum appmode {
 198         APP_MODE_DEFAULT = 0,
 199         APP_MODE_LEGACY,
 200         APP_MODE_EMPTY_POLL,
 201         APP_MODE_TELEMETRY
 202 };
 203
 204 enum appmode app_mode;
 205
 206 enum freq_scale_hint_t
 207 {
 208         FREQ_LOWER    =      -1,
 209         FREQ_CURRENT  =       0,
 210         FREQ_HIGHER   =       1,
 211         FREQ_HIGHEST  =       2
 212 };
 213
 214 struct lcore_rx_queue {
 215         uint16_t port_id;
 216         uint8_t queue_id;
 217         enum freq_scale_hint_t freq_up_hint;
 218         uint32_t zero_rx_packet_count;
 219         uint32_t idle_hint;
 220 } __rte_cache_aligned;
 221
 222 #define MAX_RX_QUEUE_PER_LCORE 16
 223 #define MAX_TX_QUEUE_PER_PORT RTE_MAX_ETHPORTS
 224 #define MAX_RX_QUEUE_PER_PORT 128
 225
 226 #define MAX_RX_QUEUE_INTERRUPT_PER_PORT 16
 227
 228
 229 struct lcore_params lcore_params_array[MAX_LCORE_PARAMS];
 230 static struct lcore_params lcore_params_array_default[] = {
 231         {0, 0, 2},
 232         {0, 1, 2},
 233         {0, 2, 2},
 234         {1, 0, 2},
 235         {1, 1, 2},
 236         {1, 2, 2},
 237         {2, 0, 2},
 238         {3, 0, 3},
 239         {3, 1, 3},
 240 };
 241
 242 struct lcore_params *lcore_params = lcore_params_array_default;
 243 uint16_t nb_lcore_params = RTE_DIM(lcore_params_array_default);
 244
 245 static struct rte_eth_conf port_conf = {
 246         .rxmode = {
 247                 .mq_mode        = ETH_MQ_RX_RSS,
 248                 .max_rx_pkt_len = RTE_ETHER_MAX_LEN,
 249                 .split_hdr_size = 0,
 250                 .offloads = DEV_RX_OFFLOAD_CHECKSUM,
 251         },
 252         .rx_adv_conf = {
 253                 .rss_conf = {
 254                         .rss_key = NULL,
 255                         .rss_hf = ETH_RSS_UDP,
 256                 },
 257         },
 258         .txmode = {
 259                 .mq_mode = ETH_MQ_TX_NONE,
 260         }
 261 };
 262
 263 static struct rte_mempool * pktmbuf_pool[NB_SOCKETS];
 264
 265
 266 #if (APP_LOOKUP_METHOD == APP_LOOKUP_EXACT_MATCH)
 267
 268 #ifdef RTE_ARCH_X86
 269 #include <rte_hash_crc.h>
 270 #define DEFAULT_HASH_FUNC       rte_hash_crc
 271 #else
 272 #include <rte_jhash.h>
 273 #define DEFAULT_HASH_FUNC       rte_jhash
 274 #endif
 275
 276 struct ipv4_5tuple {
 277         uint32_t ip_dst;
 278         uint32_t ip_src;
 279         uint16_t port_dst;
 280         uint16_t port_src;
 281         uint8_t  proto;
 282 } __rte_packed;
 283
 284 struct ipv6_5tuple {
 285         uint8_t  ip_dst[IPV6_ADDR_LEN];
 286         uint8_t  ip_src[IPV6_ADDR_LEN];
 287         uint16_t port_dst;
 288         uint16_t port_src;
 289         uint8_t  proto;
 290 } __rte_packed;
 291
 292 struct ipv4_l3fwd_route {
 293         struct ipv4_5tuple key;
 294         uint8_t if_out;
 295 };
 296
 297 struct ipv6_l3fwd_route {
 298         struct ipv6_5tuple key;
 299         uint8_t if_out;
 300 };
 301
 302 static struct ipv4_l3fwd_route ipv4_l3fwd_route_array[] = {
 303         {{RTE_IPV4(100,10,0,1), RTE_IPV4(200,10,0,1), 101, 11, IPPROTO_TCP}, 0},
 304         {{RTE_IPV4(100,20,0,2), RTE_IPV4(200,20,0,2), 102, 12, IPPROTO_TCP}, 1},
 305         {{RTE_IPV4(100,30,0,3), RTE_IPV4(200,30,0,3), 103, 13, IPPROTO_TCP}, 2},
 306         {{RTE_IPV4(100,40,0,4), RTE_IPV4(200,40,0,4), 104, 14, IPPROTO_TCP}, 3},
 307 };
 308
 309 static struct ipv6_l3fwd_route ipv6_l3fwd_route_array[] = {
 310         {
 311                 {
 312                         {0xfe, 0x80, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
 313                          0x02, 0x1b, 0x21, 0xff, 0xfe, 0x91, 0x38, 0x05},
 314                         {0xfe, 0x80, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
 315                          0x02, 0x1e, 0x67, 0xff, 0xfe, 0x0d, 0xb6, 0x0a},
 316                          1, 10, IPPROTO_UDP
 317                 }, 4
 318         },
 319 };
 320
 321 typedef struct rte_hash lookup_struct_t;
 322 static lookup_struct_t *ipv4_l3fwd_lookup_struct[NB_SOCKETS];
 323 static lookup_struct_t *ipv6_l3fwd_lookup_struct[NB_SOCKETS];
 324
 325 #define L3FWD_HASH_ENTRIES      1024
 326
 327 static uint16_t ipv4_l3fwd_out_if[L3FWD_HASH_ENTRIES] __rte_cache_aligned;
 328 static uint16_t ipv6_l3fwd_out_if[L3FWD_HASH_ENTRIES] __rte_cache_aligned;
 329 #endif
 330
 331 #if (APP_LOOKUP_METHOD == APP_LOOKUP_LPM)
 332 struct ipv4_l3fwd_route {
 333         uint32_t ip;
 334         uint8_t  depth;
 335         uint8_t  if_out;
 336 };
 337
 338 static struct ipv4_l3fwd_route ipv4_l3fwd_route_array[] = {
 339         {RTE_IPV4(1,1,1,0), 24, 0},
 340         {RTE_IPV4(2,1,1,0), 24, 1},
 341         {RTE_IPV4(3,1,1,0), 24, 2},
 342         {RTE_IPV4(4,1,1,0), 24, 3},
 343         {RTE_IPV4(5,1,1,0), 24, 4},
 344         {RTE_IPV4(6,1,1,0), 24, 5},
 345         {RTE_IPV4(7,1,1,0), 24, 6},
 346         {RTE_IPV4(8,1,1,0), 24, 7},
 347 };
 348
 349 #define IPV4_L3FWD_LPM_MAX_RULES     1024
 350
 351 typedef struct rte_lpm lookup_struct_t;
 352 static lookup_struct_t *ipv4_l3fwd_lookup_struct[NB_SOCKETS];
 353 #endif
 354
 355 struct lcore_conf {
 356         uint16_t n_rx_queue;
 357         struct lcore_rx_queue rx_queue_list[MAX_RX_QUEUE_PER_LCORE];
 358         uint16_t n_tx_port;
 359         uint16_t tx_port_id[RTE_MAX_ETHPORTS];
 360         uint16_t tx_queue_id[RTE_MAX_ETHPORTS];
 361         struct rte_eth_dev_tx_buffer *tx_buffer[RTE_MAX_ETHPORTS];
 362         lookup_struct_t * ipv4_lookup_struct;
 363         lookup_struct_t * ipv6_lookup_struct;
 364 } __rte_cache_aligned;
 365
 366 struct lcore_stats {
 367         /* total sleep time in ms since last frequency scaling down */
 368         uint32_t sleep_time;
 369         /* number of long sleep recently */
 370         uint32_t nb_long_sleep;
 371         /* freq. scaling up trend */
 372         uint32_t trend;
 373         /* total packet processed recently */
 374         uint64_t nb_rx_processed;
 375         /* total iterations looped recently */
 376         uint64_t nb_iteration_looped;
 377         /*
 378          * Represents empty and non empty polls
 379          * of rte_eth_rx_burst();
 380          * ep_nep[0] holds non empty polls
 381          * i.e. 0 < nb_rx <= MAX_BURST
 382          * ep_nep[1] holds empty polls.
 383          * i.e. nb_rx == 0
 384          */
 385         uint64_t ep_nep[2];
 386         /*
 387          * Represents full and empty+partial
 388          * polls of rte_eth_rx_burst();
 389          * ep_nep[0] holds empty+partial polls.
 390          * i.e. 0 <= nb_rx < MAX_BURST
 391          * ep_nep[1] holds full polls
 392          * i.e. nb_rx == MAX_BURST
 393          */
 394         uint64_t fp_nfp[2];
 395         enum busy_rate br;
 396         rte_spinlock_t telemetry_lock;
 397 } __rte_cache_aligned;
 398
 399 static struct lcore_conf lcore_conf[RTE_MAX_LCORE] __rte_cache_aligned;
 400 static struct lcore_stats stats[RTE_MAX_LCORE] __rte_cache_aligned;
 401 static struct rte_timer power_timers[RTE_MAX_LCORE];
 402
 403 static inline uint32_t power_idle_heuristic(uint32_t zero_rx_packet_count);
 404 static inline enum freq_scale_hint_t power_freq_scaleup_heuristic( \
 405                 unsigned int lcore_id, uint16_t port_id, uint16_t queue_id);
 406
 407
 408 /*
 409  * These defaults are using the max frequency index (1), a medium index (9)
 410  * and a typical low frequency index (14). These can be adjusted to use
 411  * different indexes using the relevant command line parameters.
 412  */
 413 static uint8_t  freq_tlb[] = {14, 9, 1};
 414
 415 static int is_done(void)
 416 {
 417         return quit_signal;
 418 }
 419
 420 /* exit signal handler */
 421 static void
 422 signal_exit_now(int sigtype)
 423 {
 424
 425         if (sigtype == SIGINT)
 426                 quit_signal = true;
 427
 428 }
 429
 430 /*  Freqency scale down timer callback */
 431 static void
 432 power_timer_cb(__rte_unused struct rte_timer *tim,
 433                           __rte_unused void *arg)
 434 {
 435         uint64_t hz;
 436         float sleep_time_ratio;
 437         unsigned lcore_id = rte_lcore_id();
 438
 439         /* accumulate total execution time in us when callback is invoked */
 440         sleep_time_ratio = (float)(stats[lcore_id].sleep_time) /
 441                                         (float)SCALING_PERIOD;
 442         /**
 443          * check whether need to scale down frequency a step if it sleep a lot.
 444          */
 445         if (sleep_time_ratio >= SCALING_DOWN_TIME_RATIO_THRESHOLD) {
 446                 if (rte_power_freq_down)
 447                         rte_power_freq_down(lcore_id);
 448         }
 449         else if ( (unsigned)(stats[lcore_id].nb_rx_processed /
 450                 stats[lcore_id].nb_iteration_looped) < MAX_PKT_BURST) {
 451                 /**
 452                  * scale down a step if average packet per iteration less
 453                  * than expectation.
 454                  */
 455                 if (rte_power_freq_down)
 456                         rte_power_freq_down(lcore_id);
 457         }
 458
 459         /**
 460          * initialize another timer according to current frequency to ensure
 461          * timer interval is relatively fixed.
 462          */
 463         hz = rte_get_timer_hz();
 464         rte_timer_reset(&power_timers[lcore_id], hz/TIMER_NUMBER_PER_SECOND,
 465                                 SINGLE, lcore_id, power_timer_cb, NULL);
 466
 467         stats[lcore_id].nb_rx_processed = 0;
 468         stats[lcore_id].nb_iteration_looped = 0;
 469
 470         stats[lcore_id].sleep_time = 0;
 471 }
 472
 473 /* Enqueue a single packet, and send burst if queue is filled */
 474 static inline int
 475 send_single_packet(struct rte_mbuf *m, uint16_t port)
 476 {
 477         uint32_t lcore_id;
 478         struct lcore_conf *qconf;
 479
 480         lcore_id = rte_lcore_id();
 481         qconf = &lcore_conf[lcore_id];
 482
 483         rte_eth_tx_buffer(port, qconf->tx_queue_id[port],
 484                         qconf->tx_buffer[port], m);
 485
 486         return 0;
 487 }
 488
 489 #ifdef DO_RFC_1812_CHECKS
 490 static inline int
 491 is_valid_ipv4_pkt(struct rte_ipv4_hdr *pkt, uint32_t link_len)
 492 {
 493         /* From http://www.rfc-editor.org/rfc/rfc1812.txt section 5.2.2 */
 494         /*
 495          * 1. The packet length reported by the Link Layer must be large
 496          * enough to hold the minimum length legal IP datagram (20 bytes).
 497          */
 498         if (link_len < sizeof(struct rte_ipv4_hdr))
 499                 return -1;
 500
 501         /* 2. The IP checksum must be correct. */
 502         /* this is checked in H/W */
 503
 504         /*
 505          * 3. The IP version number must be 4. If the version number is not 4
 506          * then the packet may be another version of IP, such as IPng or
 507          * ST-II.
 508          */
 509         if (((pkt->version_ihl) >> 4) != 4)
 510                 return -3;
 511         /*
 512          * 4. The IP header length field must be large enough to hold the
 513          * minimum length legal IP datagram (20 bytes = 5 words).
 514          */
 515         if ((pkt->version_ihl & 0xf) < 5)
 516                 return -4;
 517
 518         /*
 519          * 5. The IP total length field must be large enough to hold the IP
 520          * datagram header, whose length is specified in the IP header length
 521          * field.
 522          */
 523         if (rte_cpu_to_be_16(pkt->total_length) < sizeof(struct rte_ipv4_hdr))
 524                 return -5;
 525
 526         return 0;
 527 }
 528 #endif
 529
 530 #if (APP_LOOKUP_METHOD == APP_LOOKUP_EXACT_MATCH)
 531 static void
 532 print_ipv4_key(struct ipv4_5tuple key)
 533 {
 534         printf("IP dst = %08x, IP src = %08x, port dst = %d, port src = %d, "
 535                 "proto = %d\n", (unsigned)key.ip_dst, (unsigned)key.ip_src,
 536                                 key.port_dst, key.port_src, key.proto);
 537 }
 538 static void
 539 print_ipv6_key(struct ipv6_5tuple key)
 540 {
 541         printf( "IP dst = " IPv6_BYTES_FMT ", IP src = " IPv6_BYTES_FMT ", "
 542                 "port dst = %d, port src = %d, proto = %d\n",
 543                 IPv6_BYTES(key.ip_dst), IPv6_BYTES(key.ip_src),
 544                 key.port_dst, key.port_src, key.proto);
 545 }
 546
 547 static inline uint16_t
 548 get_ipv4_dst_port(struct rte_ipv4_hdr *ipv4_hdr, uint16_t portid,
 549                 lookup_struct_t * ipv4_l3fwd_lookup_struct)
 550 {
 551         struct ipv4_5tuple key;
 552         struct rte_tcp_hdr *tcp;
 553         struct rte_udp_hdr *udp;
 554         int ret = 0;
 555
 556         key.ip_dst = rte_be_to_cpu_32(ipv4_hdr->dst_addr);
 557         key.ip_src = rte_be_to_cpu_32(ipv4_hdr->src_addr);
 558         key.proto = ipv4_hdr->next_proto_id;
 559
 560         switch (ipv4_hdr->next_proto_id) {
 561         case IPPROTO_TCP:
 562                 tcp = (struct rte_tcp_hdr *)((unsigned char *)ipv4_hdr +
 563                                         sizeof(struct rte_ipv4_hdr));
 564                 key.port_dst = rte_be_to_cpu_16(tcp->dst_port);
 565                 key.port_src = rte_be_to_cpu_16(tcp->src_port);
 566                 break;
 567
 568         case IPPROTO_UDP:
 569                 udp = (struct rte_udp_hdr *)((unsigned char *)ipv4_hdr +
 570                                         sizeof(struct rte_ipv4_hdr));
 571                 key.port_dst = rte_be_to_cpu_16(udp->dst_port);
 572                 key.port_src = rte_be_to_cpu_16(udp->src_port);
 573                 break;
 574
 575         default:
 576                 key.port_dst = 0;
 577                 key.port_src = 0;
 578                 break;
 579         }
 580
 581         /* Find destination port */
 582         ret = rte_hash_lookup(ipv4_l3fwd_lookup_struct, (const void *)&key);
 583         return ((ret < 0) ? portid : ipv4_l3fwd_out_if[ret]);
 584 }
 585
 586 static inline uint16_t
 587 get_ipv6_dst_port(struct rte_ipv6_hdr *ipv6_hdr, uint16_t portid,
 588                         lookup_struct_t *ipv6_l3fwd_lookup_struct)
 589 {
 590         struct ipv6_5tuple key;
 591         struct rte_tcp_hdr *tcp;
 592         struct rte_udp_hdr *udp;
 593         int ret = 0;
 594
 595         memcpy(key.ip_dst, ipv6_hdr->dst_addr, IPV6_ADDR_LEN);
 596         memcpy(key.ip_src, ipv6_hdr->src_addr, IPV6_ADDR_LEN);
 597
 598         key.proto = ipv6_hdr->proto;
 599
 600         switch (ipv6_hdr->proto) {
 601         case IPPROTO_TCP:
 602                 tcp = (struct rte_tcp_hdr *)((unsigned char *) ipv6_hdr +
 603                                         sizeof(struct rte_ipv6_hdr));
 604                 key.port_dst = rte_be_to_cpu_16(tcp->dst_port);
 605                 key.port_src = rte_be_to_cpu_16(tcp->src_port);
 606                 break;
 607
 608         case IPPROTO_UDP:
 609                 udp = (struct rte_udp_hdr *)((unsigned char *) ipv6_hdr +
 610                                         sizeof(struct rte_ipv6_hdr));
 611                 key.port_dst = rte_be_to_cpu_16(udp->dst_port);
 612                 key.port_src = rte_be_to_cpu_16(udp->src_port);
 613                 break;
 614
 615         default:
 616                 key.port_dst = 0;
 617                 key.port_src = 0;
 618                 break;
 619         }
 620
 621         /* Find destination port */
 622         ret = rte_hash_lookup(ipv6_l3fwd_lookup_struct, (const void *)&key);
 623         return ((ret < 0) ? portid : ipv6_l3fwd_out_if[ret]);
 624 }
 625 #endif
 626
 627 #if (APP_LOOKUP_METHOD == APP_LOOKUP_LPM)
 628 static inline uint16_t
 629 get_ipv4_dst_port(struct rte_ipv4_hdr *ipv4_hdr, uint16_t portid,
 630                 lookup_struct_t *ipv4_l3fwd_lookup_struct)
 631 {
 632         uint32_t next_hop;
 633
 634         return ((rte_lpm_lookup(ipv4_l3fwd_lookup_struct,
 635                         rte_be_to_cpu_32(ipv4_hdr->dst_addr), &next_hop) == 0)?
 636                         next_hop : portid);
 637 }
 638 #endif
 639
 640 static inline void
 641 parse_ptype_one(struct rte_mbuf *m)
 642 {
 643         struct rte_ether_hdr *eth_hdr;
 644         uint32_t packet_type = RTE_PTYPE_UNKNOWN;
 645         uint16_t ether_type;
 646
 647         eth_hdr = rte_pktmbuf_mtod(m, struct rte_ether_hdr *);
 648         ether_type = eth_hdr->ether_type;
 649         if (ether_type == rte_cpu_to_be_16(RTE_ETHER_TYPE_IPV4))
 650                 packet_type |= RTE_PTYPE_L3_IPV4_EXT_UNKNOWN;
 651         else if (ether_type == rte_cpu_to_be_16(RTE_ETHER_TYPE_IPV6))
 652                 packet_type |= RTE_PTYPE_L3_IPV6_EXT_UNKNOWN;
 653
 654         m->packet_type = packet_type;
 655 }
 656
 657 static uint16_t
 658 cb_parse_ptype(uint16_t port __rte_unused, uint16_t queue __rte_unused,
 659                struct rte_mbuf *pkts[], uint16_t nb_pkts,
 660                uint16_t max_pkts __rte_unused,
 661                void *user_param __rte_unused)
 662 {
 663         unsigned int i;
 664
 665         for (i = 0; i < nb_pkts; ++i)
 666                 parse_ptype_one(pkts[i]);
 667
 668         return nb_pkts;
 669 }
 670
 671 static int
 672 add_cb_parse_ptype(uint16_t portid, uint16_t queueid)
 673 {
 674         printf("Port %d: softly parse packet type info\n", portid);
 675         if (rte_eth_add_rx_callback(portid, queueid, cb_parse_ptype, NULL))
 676                 return 0;
 677
 678         printf("Failed to add rx callback: port=%d\n", portid);
 679         return -1;
 680 }
 681
 682 static inline void
 683 l3fwd_simple_forward(struct rte_mbuf *m, uint16_t portid,
 684                                 struct lcore_conf *qconf)
 685 {
 686         struct rte_ether_hdr *eth_hdr;
 687         struct rte_ipv4_hdr *ipv4_hdr;
 688         void *d_addr_bytes;
 689         uint16_t dst_port;
 690
 691         eth_hdr = rte_pktmbuf_mtod(m, struct rte_ether_hdr *);
 692
 693         if (RTE_ETH_IS_IPV4_HDR(m->packet_type)) {
 694                 /* Handle IPv4 headers.*/
 695                 ipv4_hdr =
 696                         rte_pktmbuf_mtod_offset(m, struct rte_ipv4_hdr *,
 697                                                 sizeof(struct rte_ether_hdr));
 698
 699 #ifdef DO_RFC_1812_CHECKS
 700                 /* Check to make sure the packet is valid (RFC1812) */
 701                 if (is_valid_ipv4_pkt(ipv4_hdr, m->pkt_len) < 0) {
 702                         rte_pktmbuf_free(m);
 703                         return;
 704                 }
 705 #endif
 706
 707                 dst_port = get_ipv4_dst_port(ipv4_hdr, portid,
 708                                         qconf->ipv4_lookup_struct);
 709                 if (dst_port >= RTE_MAX_ETHPORTS ||
 710                                 (enabled_port_mask & 1 << dst_port) == 0)
 711                         dst_port = portid;
 712
 713                 /* 02:00:00:00:00:xx */
 714                 d_addr_bytes = &eth_hdr->d_addr.addr_bytes[0];
 715                 *((uint64_t *)d_addr_bytes) =
 716                         0x000000000002 + ((uint64_t)dst_port << 40);
 717
 718 #ifdef DO_RFC_1812_CHECKS
 719                 /* Update time to live and header checksum */
 720                 --(ipv4_hdr->time_to_live);
 721                 ++(ipv4_hdr->hdr_checksum);
 722 #endif
 723
 724                 /* src addr */
 725                 rte_ether_addr_copy(&ports_eth_addr[dst_port],
 726                                 &eth_hdr->s_addr);
 727
 728                 send_single_packet(m, dst_port);
 729         } else if (RTE_ETH_IS_IPV6_HDR(m->packet_type)) {
 730                 /* Handle IPv6 headers.*/
 731 #if (APP_LOOKUP_METHOD == APP_LOOKUP_EXACT_MATCH)
 732                 struct rte_ipv6_hdr *ipv6_hdr;
 733
 734                 ipv6_hdr =
 735                         rte_pktmbuf_mtod_offset(m, struct rte_ipv6_hdr *,
 736                                                 sizeof(struct rte_ether_hdr));
 737
 738                 dst_port = get_ipv6_dst_port(ipv6_hdr, portid,
 739                                         qconf->ipv6_lookup_struct);
 740
 741                 if (dst_port >= RTE_MAX_ETHPORTS ||
 742                                 (enabled_port_mask & 1 << dst_port) == 0)
 743                         dst_port = portid;
 744
 745                 /* 02:00:00:00:00:xx */
 746                 d_addr_bytes = &eth_hdr->d_addr.addr_bytes[0];
 747                 *((uint64_t *)d_addr_bytes) =
 748                         0x000000000002 + ((uint64_t)dst_port << 40);
 749
 750                 /* src addr */
 751                 rte_ether_addr_copy(&ports_eth_addr[dst_port],
 752                                 &eth_hdr->s_addr);
 753
 754                 send_single_packet(m, dst_port);
 755 #else
 756                 /* We don't currently handle IPv6 packets in LPM mode. */
 757                 rte_pktmbuf_free(m);
 758 #endif
 759         } else
 760                 rte_pktmbuf_free(m);
 761
 762 }
 763
 764 #define MINIMUM_SLEEP_TIME         1
 765 #define SUSPEND_THRESHOLD          300
 766
 767 static inline uint32_t
 768 power_idle_heuristic(uint32_t zero_rx_packet_count)
 769 {
 770         /* If zero count is less than 100,  sleep 1us */
 771         if (zero_rx_packet_count < SUSPEND_THRESHOLD)
 772                 return MINIMUM_SLEEP_TIME;
 773         /* If zero count is less than 1000, sleep 100 us which is the
 774                 minimum latency switching from C3/C6 to C0
 775         */
 776         else
 777                 return SUSPEND_THRESHOLD;
 778 }
 779
 780 static inline enum freq_scale_hint_t
 781 power_freq_scaleup_heuristic(unsigned lcore_id,
 782                              uint16_t port_id,
 783                              uint16_t queue_id)
 784 {
 785         uint32_t rxq_count = rte_eth_rx_queue_count(port_id, queue_id);
 786 /**
 787  * HW Rx queue size is 128 by default, Rx burst read at maximum 32 entries
 788  * per iteration
 789  */
 790 #define FREQ_GEAR1_RX_PACKET_THRESHOLD             MAX_PKT_BURST
 791 #define FREQ_GEAR2_RX_PACKET_THRESHOLD             (MAX_PKT_BURST*2)
 792 #define FREQ_GEAR3_RX_PACKET_THRESHOLD             (MAX_PKT_BURST*3)
 793 #define FREQ_UP_TREND1_ACC   1
 794 #define FREQ_UP_TREND2_ACC   100
 795 #define FREQ_UP_THRESHOLD    10000
 796
 797         if (likely(rxq_count > FREQ_GEAR3_RX_PACKET_THRESHOLD)) {
 798                 stats[lcore_id].trend = 0;
 799                 return FREQ_HIGHEST;
 800         } else if (likely(rxq_count > FREQ_GEAR2_RX_PACKET_THRESHOLD))
 801                 stats[lcore_id].trend += FREQ_UP_TREND2_ACC;
 802         else if (likely(rxq_count > FREQ_GEAR1_RX_PACKET_THRESHOLD))
 803                 stats[lcore_id].trend += FREQ_UP_TREND1_ACC;
 804
 805         if (likely(stats[lcore_id].trend > FREQ_UP_THRESHOLD)) {
 806                 stats[lcore_id].trend = 0;
 807                 return FREQ_HIGHER;
 808         }
 809
 810         return FREQ_CURRENT;
 811 }
 812
 813 /**
 814  * force polling thread sleep until one-shot rx interrupt triggers
 815  * @param port_id
 816  *  Port id.
 817  * @param queue_id
 818  *  Rx queue id.
 819  * @return
 820  *  0 on success
 821  */
 822 static int
 823 sleep_until_rx_interrupt(int num)
 824 {
 825         /*
 826          * we want to track when we are woken up by traffic so that we can go
 827          * back to sleep again without log spamming.
 828          */
 829         static bool timeout;
 830         struct rte_epoll_event event[num];
 831         int n, i;
 832         uint16_t port_id;
 833         uint8_t queue_id;
 834         void *data;
 835
 836         if (!timeout) {
 837                 RTE_LOG(INFO, L3FWD_POWER,
 838                                 "lcore %u sleeps until interrupt triggers\n",
 839                                 rte_lcore_id());
 840         }
 841
 842         n = rte_epoll_wait(RTE_EPOLL_PER_THREAD, event, num, 10);
 843         for (i = 0; i < n; i++) {
 844                 data = event[i].epdata.data;
 845                 port_id = ((uintptr_t)data) >> CHAR_BIT;
 846                 queue_id = ((uintptr_t)data) &
 847                         RTE_LEN2MASK(CHAR_BIT, uint8_t);
 848                 RTE_LOG(INFO, L3FWD_POWER,
 849                         "lcore %u is waked up from rx interrupt on"
 850                         " port %d queue %d\n",
 851                         rte_lcore_id(), port_id, queue_id);
 852         }
 853         timeout = n == 0;
 854
 855         return 0;
 856 }
 857
 858 static void turn_on_off_intr(struct lcore_conf *qconf, bool on)
 859 {
 860         int i;
 861         struct lcore_rx_queue *rx_queue;
 862         uint8_t queue_id;
 863         uint16_t port_id;
 864
 865         for (i = 0; i < qconf->n_rx_queue; ++i) {
 866                 rx_queue = &(qconf->rx_queue_list[i]);
 867                 port_id = rx_queue->port_id;
 868                 queue_id = rx_queue->queue_id;
 869
 870                 rte_spinlock_lock(&(locks[port_id]));
 871                 if (on)
 872                         rte_eth_dev_rx_intr_enable(port_id, queue_id);
 873                 else
 874                         rte_eth_dev_rx_intr_disable(port_id, queue_id);
 875                 rte_spinlock_unlock(&(locks[port_id]));
 876         }
 877 }
 878
 879 static int event_register(struct lcore_conf *qconf)
 880 {
 881         struct lcore_rx_queue *rx_queue;
 882         uint8_t queueid;
 883         uint16_t portid;
 884         uint32_t data;
 885         int ret;
 886         int i;
 887
 888         for (i = 0; i < qconf->n_rx_queue; ++i) {
 889                 rx_queue = &(qconf->rx_queue_list[i]);
 890                 portid = rx_queue->port_id;
 891                 queueid = rx_queue->queue_id;
 892                 data = portid << CHAR_BIT | queueid;
 893
 894                 ret = rte_eth_dev_rx_intr_ctl_q(portid, queueid,
 895                                                 RTE_EPOLL_PER_THREAD,
 896                                                 RTE_INTR_EVENT_ADD,
 897                                                 (void *)((uintptr_t)data));
 898                 if (ret)
 899                         return ret;
 900         }
 901
 902         return 0;
 903 }
 904 /* main processing loop */
 905 static int
 906 main_telemetry_loop(__rte_unused void *dummy)
 907 {
 908         struct rte_mbuf *pkts_burst[MAX_PKT_BURST];
 909         unsigned int lcore_id;
 910         uint64_t prev_tsc, diff_tsc, cur_tsc, prev_tel_tsc;
 911         int i, j, nb_rx;
 912         uint8_t queueid;
 913         uint16_t portid;
 914         struct lcore_conf *qconf;
 915         struct lcore_rx_queue *rx_queue;
 916         uint64_t ep_nep[2] = {0}, fp_nfp[2] = {0};
 917         uint64_t poll_count;
 918         enum busy_rate br;
 919
 920         const uint64_t drain_tsc = (rte_get_tsc_hz() + US_PER_S - 1) /
 921                                         US_PER_S * BURST_TX_DRAIN_US;
 922
 923         poll_count = 0;
 924         prev_tsc = 0;
 925         prev_tel_tsc = 0;
 926
 927         lcore_id = rte_lcore_id();
 928         qconf = &lcore_conf[lcore_id];
 929
 930         if (qconf->n_rx_queue == 0) {
 931                 RTE_LOG(INFO, L3FWD_POWER, "lcore %u has nothing to do\n",
 932                         lcore_id);
 933                 return 0;
 934         }
 935
 936         RTE_LOG(INFO, L3FWD_POWER, "entering main telemetry loop on lcore %u\n",
 937                 lcore_id);
 938
 939         for (i = 0; i < qconf->n_rx_queue; i++) {
 940                 portid = qconf->rx_queue_list[i].port_id;
 941                 queueid = qconf->rx_queue_list[i].queue_id;
 942                 RTE_LOG(INFO, L3FWD_POWER, " -- lcoreid=%u portid=%u "
 943                         "rxqueueid=%hhu\n", lcore_id, portid, queueid);
 944         }
 945
 946         while (!is_done()) {
 947
 948                 cur_tsc = rte_rdtsc();
 949                 /*
 950                  * TX burst queue drain
 951                  */
 952                 diff_tsc = cur_tsc - prev_tsc;
 953                 if (unlikely(diff_tsc > drain_tsc)) {
 954                         for (i = 0; i < qconf->n_tx_port; ++i) {
 955                                 portid = qconf->tx_port_id[i];
 956                                 rte_eth_tx_buffer_flush(portid,
 957                                                 qconf->tx_queue_id[portid],
 958                                                 qconf->tx_buffer[portid]);
 959                         }
 960                         prev_tsc = cur_tsc;
 961                 }
 962
 963                 /*
 964                  * Read packet from RX queues
 965                  */
 966                 for (i = 0; i < qconf->n_rx_queue; ++i) {
 967                         rx_queue = &(qconf->rx_queue_list[i]);
 968                         portid = rx_queue->port_id;
 969                         queueid = rx_queue->queue_id;
 970
 971                         nb_rx = rte_eth_rx_burst(portid, queueid, pkts_burst,
 972                                                                 MAX_PKT_BURST);
 973                         ep_nep[nb_rx == 0]++;
 974                         fp_nfp[nb_rx == MAX_PKT_BURST]++;
 975                         poll_count++;
 976                         if (unlikely(nb_rx == 0))
 977                                 continue;
 978
 979                         /* Prefetch first packets */
 980                         for (j = 0; j < PREFETCH_OFFSET && j < nb_rx; j++) {
 981                                 rte_prefetch0(rte_pktmbuf_mtod(
 982                                                 pkts_burst[j], void *));
 983                         }
 984
 985                         /* Prefetch and forward already prefetched packets */
 986                         for (j = 0; j < (nb_rx - PREFETCH_OFFSET); j++) {
 987                                 rte_prefetch0(rte_pktmbuf_mtod(pkts_burst[
 988                                                 j + PREFETCH_OFFSET], void *));
 989                                 l3fwd_simple_forward(pkts_burst[j], portid,
 990                                                                 qconf);
 991                         }
 992
 993                         /* Forward remaining prefetched packets */
 994                         for (; j < nb_rx; j++) {
 995                                 l3fwd_simple_forward(pkts_burst[j], portid,
 996                                                                 qconf);
 997                         }
 998                 }
 999                 if (unlikely(poll_count >= DEFAULT_COUNT)) {
1000                         diff_tsc = cur_tsc - prev_tel_tsc;
1001                         if (diff_tsc >= MAX_CYCLES) {
1002                                 br = FULL;
1003                         } else if (diff_tsc > MIN_CYCLES &&
1004                                         diff_tsc < MAX_CYCLES) {
1005                                 br = (diff_tsc * 100) / MAX_CYCLES;
1006                         } else {
1007                                 br = ZERO;
1008                         }
1009                         poll_count = 0;
1010                         prev_tel_tsc = cur_tsc;
1011                         /* update stats for telemetry */
1012                         rte_spinlock_lock(&stats[lcore_id].telemetry_lock);
1013                         stats[lcore_id].ep_nep[0] = ep_nep[0];
1014                         stats[lcore_id].ep_nep[1] = ep_nep[1];
1015                         stats[lcore_id].fp_nfp[0] = fp_nfp[0];
1016                         stats[lcore_id].fp_nfp[1] = fp_nfp[1];
1017                         stats[lcore_id].br = br;
1018                         rte_spinlock_unlock(&stats[lcore_id].telemetry_lock);
1019                 }
1020         }
1021
1022         return 0;
1023 }
1024 /* main processing loop */
1025 static int
1026 main_empty_poll_loop(__rte_unused void *dummy)
1027 {
1028         struct rte_mbuf *pkts_burst[MAX_PKT_BURST];
1029         unsigned int lcore_id;
1030         uint64_t prev_tsc, diff_tsc, cur_tsc;
1031         int i, j, nb_rx;
1032         uint8_t queueid;
1033         uint16_t portid;
1034         struct lcore_conf *qconf;
1035         struct lcore_rx_queue *rx_queue;
1036
1037         const uint64_t drain_tsc =
1038                 (rte_get_tsc_hz() + US_PER_S - 1) /
1039                 US_PER_S * BURST_TX_DRAIN_US;
1040
1041         prev_tsc = 0;
1042
1043         lcore_id = rte_lcore_id();
1044         qconf = &lcore_conf[lcore_id];
1045
1046         if (qconf->n_rx_queue == 0) {
1047                 RTE_LOG(INFO, L3FWD_POWER, "lcore %u has nothing to do\n",
1048                         lcore_id);
1049                 return 0;
1050         }
1051
1052         for (i = 0; i < qconf->n_rx_queue; i++) {
1053                 portid = qconf->rx_queue_list[i].port_id;
1054                 queueid = qconf->rx_queue_list[i].queue_id;
1055                 RTE_LOG(INFO, L3FWD_POWER, " -- lcoreid=%u portid=%u "
1056                                 "rxqueueid=%hhu\n", lcore_id, portid, queueid);
1057         }
1058
1059         while (!is_done()) {
1060                 stats[lcore_id].nb_iteration_looped++;
1061
1062                 cur_tsc = rte_rdtsc();
1063                 /*
1064                  * TX burst queue drain
1065                  */
1066                 diff_tsc = cur_tsc - prev_tsc;
1067                 if (unlikely(diff_tsc > drain_tsc)) {
1068                         for (i = 0; i < qconf->n_tx_port; ++i) {
1069                                 portid = qconf->tx_port_id[i];
1070                                 rte_eth_tx_buffer_flush(portid,
1071                                                 qconf->tx_queue_id[portid],
1072                                                 qconf->tx_buffer[portid]);
1073                         }
1074                         prev_tsc = cur_tsc;
1075                 }
1076
1077                 /*
1078                  * Read packet from RX queues
1079                  */
1080                 for (i = 0; i < qconf->n_rx_queue; ++i) {
1081                         rx_queue = &(qconf->rx_queue_list[i]);
1082                         rx_queue->idle_hint = 0;
1083                         portid = rx_queue->port_id;
1084                         queueid = rx_queue->queue_id;
1085
1086                         nb_rx = rte_eth_rx_burst(portid, queueid, pkts_burst,
1087                                         MAX_PKT_BURST);
1088
1089                         stats[lcore_id].nb_rx_processed += nb_rx;
1090
1091                         if (nb_rx == 0) {
1092
1093                                 rte_power_empty_poll_stat_update(lcore_id);
1094
1095                                 continue;
1096                         } else {
1097                                 rte_power_poll_stat_update(lcore_id, nb_rx);
1098                         }
1099
1100
1101                         /* Prefetch first packets */
1102                         for (j = 0; j < PREFETCH_OFFSET && j < nb_rx; j++) {
1103                                 rte_prefetch0(rte_pktmbuf_mtod(
1104                                                         pkts_burst[j], void *));
1105                         }
1106
1107                         /* Prefetch and forward already prefetched packets */
1108                         for (j = 0; j < (nb_rx - PREFETCH_OFFSET); j++) {
1109                                 rte_prefetch0(rte_pktmbuf_mtod(pkts_burst[
1110                                                         j + PREFETCH_OFFSET],
1111                                                         void *));
1112                                 l3fwd_simple_forward(pkts_burst[j], portid,
1113                                                 qconf);
1114                         }
1115
1116                         /* Forward remaining prefetched packets */
1117                         for (; j < nb_rx; j++) {
1118                                 l3fwd_simple_forward(pkts_burst[j], portid,
1119                                                 qconf);
1120                         }
1121
1122                 }
1123
1124         }
1125
1126         return 0;
1127 }
1128 /* main processing loop */
1129 static int
1130 main_legacy_loop(__rte_unused void *dummy)
1131 {
1132         struct rte_mbuf *pkts_burst[MAX_PKT_BURST];
1133         unsigned lcore_id;
1134         uint64_t prev_tsc, diff_tsc, cur_tsc, tim_res_tsc, hz;
1135         uint64_t prev_tsc_power = 0, cur_tsc_power, diff_tsc_power;
1136         int i, j, nb_rx;
1137         uint8_t queueid;
1138         uint16_t portid;
1139         struct lcore_conf *qconf;
1140         struct lcore_rx_queue *rx_queue;
1141         enum freq_scale_hint_t lcore_scaleup_hint;
1142         uint32_t lcore_rx_idle_count = 0;
1143         uint32_t lcore_idle_hint = 0;
1144         int intr_en = 0;
1145
1146         const uint64_t drain_tsc = (rte_get_tsc_hz() + US_PER_S - 1) / US_PER_S * BURST_TX_DRAIN_US;
1147
1148         prev_tsc = 0;
1149         hz = rte_get_timer_hz();
1150         tim_res_tsc = hz/TIMER_NUMBER_PER_SECOND;
1151
1152         lcore_id = rte_lcore_id();
1153         qconf = &lcore_conf[lcore_id];
1154
1155         if (qconf->n_rx_queue == 0) {
1156                 RTE_LOG(INFO, L3FWD_POWER, "lcore %u has nothing to do\n", lcore_id);
1157                 return 0;
1158         }
1159
1160         RTE_LOG(INFO, L3FWD_POWER, "entering main loop on lcore %u\n", lcore_id);
1161
1162         for (i = 0; i < qconf->n_rx_queue; i++) {
1163                 portid = qconf->rx_queue_list[i].port_id;
1164                 queueid = qconf->rx_queue_list[i].queue_id;
1165                 RTE_LOG(INFO, L3FWD_POWER, " -- lcoreid=%u portid=%u "
1166                         "rxqueueid=%hhu\n", lcore_id, portid, queueid);
1167         }
1168
1169         /* add into event wait list */
1170         if (event_register(qconf) == 0)
1171                 intr_en = 1;
1172         else
1173                 RTE_LOG(INFO, L3FWD_POWER, "RX interrupt won't enable.\n");
1174
1175         while (!is_done()) {
1176                 stats[lcore_id].nb_iteration_looped++;
1177
1178                 cur_tsc = rte_rdtsc();
1179                 cur_tsc_power = cur_tsc;
1180
1181                 /*
1182                  * TX burst queue drain
1183                  */
1184                 diff_tsc = cur_tsc - prev_tsc;
1185                 if (unlikely(diff_tsc > drain_tsc)) {
1186                         for (i = 0; i < qconf->n_tx_port; ++i) {
1187                                 portid = qconf->tx_port_id[i];
1188                                 rte_eth_tx_buffer_flush(portid,
1189                                                 qconf->tx_queue_id[portid],
1190                                                 qconf->tx_buffer[portid]);
1191                         }
1192                         prev_tsc = cur_tsc;
1193                 }
1194
1195                 diff_tsc_power = cur_tsc_power - prev_tsc_power;
1196                 if (diff_tsc_power > tim_res_tsc) {
1197                         rte_timer_manage();
1198                         prev_tsc_power = cur_tsc_power;
1199                 }
1200
1201 start_rx:
1202                 /*
1203                  * Read packet from RX queues
1204                  */
1205                 lcore_scaleup_hint = FREQ_CURRENT;
1206                 lcore_rx_idle_count = 0;
1207                 for (i = 0; i < qconf->n_rx_queue; ++i) {
1208                         rx_queue = &(qconf->rx_queue_list[i]);
1209                         rx_queue->idle_hint = 0;
1210                         portid = rx_queue->port_id;
1211                         queueid = rx_queue->queue_id;
1212
1213                         nb_rx = rte_eth_rx_burst(portid, queueid, pkts_burst,
1214                                                                 MAX_PKT_BURST);
1215
1216                         stats[lcore_id].nb_rx_processed += nb_rx;
1217                         if (unlikely(nb_rx == 0)) {
1218                                 /**
1219                                  * no packet received from rx queue, try to
1220                                  * sleep for a while forcing CPU enter deeper
1221                                  * C states.
1222                                  */
1223                                 rx_queue->zero_rx_packet_count++;
1224
1225                                 if (rx_queue->zero_rx_packet_count <=
1226                                                         MIN_ZERO_POLL_COUNT)
1227                                         continue;
1228
1229                                 rx_queue->idle_hint = power_idle_heuristic(\
1230                                         rx_queue->zero_rx_packet_count);
1231                                 lcore_rx_idle_count++;
1232                         } else {
1233                                 rx_queue->zero_rx_packet_count = 0;
1234
1235                                 /**
1236                                  * do not scale up frequency immediately as
1237                                  * user to kernel space communication is costly
1238                                  * which might impact packet I/O for received
1239                                  * packets.
1240                                  */
1241                                 rx_queue->freq_up_hint =
1242                                         power_freq_scaleup_heuristic(lcore_id,
1243                                                         portid, queueid);
1244                         }
1245
1246                         /* Prefetch first packets */
1247                         for (j = 0; j < PREFETCH_OFFSET && j < nb_rx; j++) {
1248                                 rte_prefetch0(rte_pktmbuf_mtod(
1249                                                 pkts_burst[j], void *));
1250                         }
1251
1252                         /* Prefetch and forward already prefetched packets */
1253                         for (j = 0; j < (nb_rx - PREFETCH_OFFSET); j++) {
1254                                 rte_prefetch0(rte_pktmbuf_mtod(pkts_burst[
1255                                                 j + PREFETCH_OFFSET], void *));
1256                                 l3fwd_simple_forward(pkts_burst[j], portid,
1257                                                                 qconf);
1258                         }
1259
1260                         /* Forward remaining prefetched packets */
1261                         for (; j < nb_rx; j++) {
1262                                 l3fwd_simple_forward(pkts_burst[j], portid,
1263                                                                 qconf);
1264                         }
1265                 }
1266
1267                 if (likely(lcore_rx_idle_count != qconf->n_rx_queue)) {
1268                         for (i = 1, lcore_scaleup_hint =
1269                                 qconf->rx_queue_list[0].freq_up_hint;
1270                                         i < qconf->n_rx_queue; ++i) {
1271                                 rx_queue = &(qconf->rx_queue_list[i]);
1272                                 if (rx_queue->freq_up_hint >
1273                                                 lcore_scaleup_hint)
1274                                         lcore_scaleup_hint =
1275                                                 rx_queue->freq_up_hint;
1276                         }
1277
1278                         if (lcore_scaleup_hint == FREQ_HIGHEST) {
1279                                 if (rte_power_freq_max)
1280                                         rte_power_freq_max(lcore_id);
1281                         } else if (lcore_scaleup_hint == FREQ_HIGHER) {
1282                                 if (rte_power_freq_up)
1283                                         rte_power_freq_up(lcore_id);
1284                         }
1285                 } else {
1286                         /**
1287                          * All Rx queues empty in recent consecutive polls,
1288                          * sleep in a conservative manner, meaning sleep as
1289                          * less as possible.
1290                          */
1291                         for (i = 1, lcore_idle_hint =
1292                                 qconf->rx_queue_list[0].idle_hint;
1293                                         i < qconf->n_rx_queue; ++i) {
1294                                 rx_queue = &(qconf->rx_queue_list[i]);
1295                                 if (rx_queue->idle_hint < lcore_idle_hint)
1296                                         lcore_idle_hint = rx_queue->idle_hint;
1297                         }
1298
1299                         if (lcore_idle_hint < SUSPEND_THRESHOLD)
1300                                 /**
1301                                  * execute "pause" instruction to avoid context
1302                                  * switch which generally take hundred of
1303                                  * microseconds for short sleep.
1304                                  */
1305                                 rte_delay_us(lcore_idle_hint);
1306                         else {
1307                                 /* suspend until rx interrupt triggers */
1308                                 if (intr_en) {
1309                                         turn_on_off_intr(qconf, 1);
1310                                         sleep_until_rx_interrupt(
1311                                                 qconf->n_rx_queue);
1312                                         turn_on_off_intr(qconf, 0);
1313                                         /**
1314                                          * start receiving packets immediately
1315                                          */
1316                                         if (likely(!is_done()))
1317                                                 goto start_rx;
1318                                 }
1319                         }
1320                         stats[lcore_id].sleep_time += lcore_idle_hint;
1321                 }
1322         }
1323
1324         return 0;
1325 }
1326
1327 static int
1328 check_lcore_params(void)
1329 {
1330         uint8_t queue, lcore;
1331         uint16_t i;
1332         int socketid;
1333
1334         for (i = 0; i < nb_lcore_params; ++i) {
1335                 queue = lcore_params[i].queue_id;
1336                 if (queue >= MAX_RX_QUEUE_PER_PORT) {
1337                         printf("invalid queue number: %hhu\n", queue);
1338                         return -1;
1339                 }
1340                 lcore = lcore_params[i].lcore_id;
1341                 if (!rte_lcore_is_enabled(lcore)) {
1342                         printf("error: lcore %hhu is not enabled in lcore "
1343                                                         "mask\n", lcore);
1344                         return -1;
1345                 }
1346                 if ((socketid = rte_lcore_to_socket_id(lcore) != 0) &&
1347                                                         (numa_on == 0)) {
1348                         printf("warning: lcore %hhu is on socket %d with numa "
1349                                                 "off\n", lcore, socketid);
1350                 }
1351                 if (app_mode == APP_MODE_TELEMETRY && lcore == rte_lcore_id()) {
1352                         printf("cannot enable master core %d in config for telemetry mode\n",
1353                                 rte_lcore_id());
1354                         return -1;
1355                 }
1356         }
1357         return 0;
1358 }
1359
1360 static int
1361 check_port_config(void)
1362 {
1363         unsigned portid;
1364         uint16_t i;
1365
1366         for (i = 0; i < nb_lcore_params; ++i) {
1367                 portid = lcore_params[i].port_id;
1368                 if ((enabled_port_mask & (1 << portid)) == 0) {
1369                         printf("port %u is not enabled in port mask\n",
1370                                                                 portid);
1371                         return -1;
1372                 }
1373                 if (!rte_eth_dev_is_valid_port(portid)) {
1374                         printf("port %u is not present on the board\n",
1375                                                                 portid);
1376                         return -1;
1377                 }
1378         }
1379         return 0;
1380 }
1381
1382 static uint8_t
1383 get_port_n_rx_queues(const uint16_t port)
1384 {
1385         int queue = -1;
1386         uint16_t i;
1387
1388         for (i = 0; i < nb_lcore_params; ++i) {
1389                 if (lcore_params[i].port_id == port &&
1390                                 lcore_params[i].queue_id > queue)
1391                         queue = lcore_params[i].queue_id;
1392         }
1393         return (uint8_t)(++queue);
1394 }
1395
1396 static int
1397 init_lcore_rx_queues(void)
1398 {
1399         uint16_t i, nb_rx_queue;
1400         uint8_t lcore;
1401
1402         for (i = 0; i < nb_lcore_params; ++i) {
1403                 lcore = lcore_params[i].lcore_id;
1404                 nb_rx_queue = lcore_conf[lcore].n_rx_queue;
1405                 if (nb_rx_queue >= MAX_RX_QUEUE_PER_LCORE) {
1406                         printf("error: too many queues (%u) for lcore: %u\n",
1407                                 (unsigned)nb_rx_queue + 1, (unsigned)lcore);
1408                         return -1;
1409                 } else {
1410                         lcore_conf[lcore].rx_queue_list[nb_rx_queue].port_id =
1411                                 lcore_params[i].port_id;
1412                         lcore_conf[lcore].rx_queue_list[nb_rx_queue].queue_id =
1413                                 lcore_params[i].queue_id;
1414                         lcore_conf[lcore].n_rx_queue++;
1415                 }
1416         }
1417         return 0;
1418 }
1419
1420 /* display usage */
1421 static void
1422 print_usage(const char *prgname)
1423 {
1424         printf ("%s [EAL options] -- -p PORTMASK -P"
1425                 "  [--config (port,queue,lcore)[,(port,queue,lcore]]"
1426                 "  [--high-perf-cores CORELIST"
1427                 "  [--perf-config (port,queue,hi_perf,lcore_index)[,(port,queue,hi_perf,lcore_index]]"
1428                 "  [--enable-jumbo [--max-pkt-len PKTLEN]]\n"
1429                 "  -p PORTMASK: hexadecimal bitmask of ports to configure\n"
1430                 "  -P : enable promiscuous mode\n"
1431                 "  --config (port,queue,lcore): rx queues configuration\n"
1432                 "  --high-perf-cores CORELIST: list of high performance cores\n"
1433                 "  --perf-config: similar as config, cores specified as indices"
1434                 " for bins containing high or regular performance cores\n"
1435                 "  --no-numa: optional, disable numa awareness\n"
1436                 "  --enable-jumbo: enable jumbo frame"
1437                 " which max packet len is PKTLEN in decimal (64-9600)\n"
1438                 "  --parse-ptype: parse packet type by software\n"
1439                 "  --legacy: use legacy interrupt-based scaling\n"
1440                 "  --empty-poll: enable empty poll detection"
1441                 " follow (training_flag, high_threshold, med_threshold)\n"
1442                 " --telemetry: enable telemetry mode, to update"
1443                 " empty polls, full polls, and core busyness to telemetry\n",
1444                 prgname);
1445 }
1446
1447 static int parse_max_pkt_len(const char *pktlen)
1448 {
1449         char *end = NULL;
1450         unsigned long len;
1451
1452         /* parse decimal string */
1453         len = strtoul(pktlen, &end, 10);
1454         if ((pktlen[0] == '\0') || (end == NULL) || (*end != '\0'))
1455                 return -1;
1456
1457         if (len == 0)
1458                 return -1;
1459
1460         return len;
1461 }
1462
1463 static int
1464 parse_portmask(const char *portmask)
1465 {
1466         char *end = NULL;
1467         unsigned long pm;
1468
1469         /* parse hexadecimal string */
1470         pm = strtoul(portmask, &end, 16);
1471         if ((portmask[0] == '\0') || (end == NULL) || (*end != '\0'))
1472                 return -1;
1473
1474         if (pm == 0)
1475                 return -1;
1476
1477         return pm;
1478 }
1479
1480 static int
1481 parse_config(const char *q_arg)
1482 {
1483         char s[256];
1484         const char *p, *p0 = q_arg;
1485         char *end;
1486         enum fieldnames {
1487                 FLD_PORT = 0,
1488                 FLD_QUEUE,
1489                 FLD_LCORE,
1490                 _NUM_FLD
1491         };
1492         unsigned long int_fld[_NUM_FLD];
1493         char *str_fld[_NUM_FLD];
1494         int i;
1495         unsigned size;
1496
1497         nb_lcore_params = 0;
1498
1499         while ((p = strchr(p0,'(')) != NULL) {
1500                 ++p;
1501                 if((p0 = strchr(p,')')) == NULL)
1502                         return -1;
1503
1504                 size = p0 - p;
1505                 if(size >= sizeof(s))
1506                         return -1;
1507
1508                 snprintf(s, sizeof(s), "%.*s", size, p);
1509                 if (rte_strsplit(s, sizeof(s), str_fld, _NUM_FLD, ',') !=
1510                                                                 _NUM_FLD)
1511                         return -1;
1512                 for (i = 0; i < _NUM_FLD; i++){
1513                         errno = 0;
1514                         int_fld[i] = strtoul(str_fld[i], &end, 0);
1515                         if (errno != 0 || end == str_fld[i] || int_fld[i] >
1516                                                                         255)
1517                                 return -1;
1518                 }
1519                 if (nb_lcore_params >= MAX_LCORE_PARAMS) {
1520                         printf("exceeded max number of lcore params: %hu\n",
1521                                 nb_lcore_params);
1522                         return -1;
1523                 }
1524                 lcore_params_array[nb_lcore_params].port_id =
1525                                 (uint8_t)int_fld[FLD_PORT];
1526                 lcore_params_array[nb_lcore_params].queue_id =
1527                                 (uint8_t)int_fld[FLD_QUEUE];
1528                 lcore_params_array[nb_lcore_params].lcore_id =
1529                                 (uint8_t)int_fld[FLD_LCORE];
1530                 ++nb_lcore_params;
1531         }
1532         lcore_params = lcore_params_array;
1533
1534         return 0;
1535 }
1536 static int
1537 parse_ep_config(const char *q_arg)
1538 {
1539         char s[256];
1540         const char *p = q_arg;
1541         char *end;
1542         int  num_arg;
1543
1544         char *str_fld[3];
1545
1546         int training_flag;
1547         int med_edpi;
1548         int hgh_edpi;
1549
1550         ep_med_edpi = EMPTY_POLL_MED_THRESHOLD;
1551         ep_hgh_edpi = EMPTY_POLL_MED_THRESHOLD;
1552
1553         strlcpy(s, p, sizeof(s));
1554
1555         num_arg = rte_strsplit(s, sizeof(s), str_fld, 3, ',');
1556
1557         empty_poll_train = false;
1558
1559         if (num_arg == 0)
1560                 return 0;
1561
1562         if (num_arg == 3) {
1563
1564                 training_flag = strtoul(str_fld[0], &end, 0);
1565                 med_edpi = strtoul(str_fld[1], &end, 0);
1566                 hgh_edpi = strtoul(str_fld[2], &end, 0);
1567
1568                 if (training_flag == 1)
1569                         empty_poll_train = true;
1570
1571                 if (med_edpi > 0)
1572                         ep_med_edpi = med_edpi;
1573
1574                 if (med_edpi > 0)
1575                         ep_hgh_edpi = hgh_edpi;
1576
1577         } else {
1578
1579                 return -1;
1580         }
1581
1582         return 0;
1583
1584 }
1585 #define CMD_LINE_OPT_PARSE_PTYPE "parse-ptype"
1586 #define CMD_LINE_OPT_LEGACY "legacy"
1587 #define CMD_LINE_OPT_EMPTY_POLL "empty-poll"
1588 #define CMD_LINE_OPT_TELEMETRY "telemetry"
1589
1590 /* Parse the argument given in the command line of the application */
1591 static int
1592 parse_args(int argc, char **argv)
1593 {
1594         int opt, ret;
1595         char **argvopt;
1596         int option_index;
1597         uint32_t limit;
1598         char *prgname = argv[0];
1599         static struct option lgopts[] = {
1600                 {"config", 1, 0, 0},
1601                 {"perf-config", 1, 0, 0},
1602                 {"high-perf-cores", 1, 0, 0},
1603                 {"no-numa", 0, 0, 0},
1604                 {"enable-jumbo", 0, 0, 0},
1605                 {CMD_LINE_OPT_EMPTY_POLL, 1, 0, 0},
1606                 {CMD_LINE_OPT_PARSE_PTYPE, 0, 0, 0},
1607                 {CMD_LINE_OPT_LEGACY, 0, 0, 0},
1608                 {CMD_LINE_OPT_TELEMETRY, 0, 0, 0},
1609                 {NULL, 0, 0, 0}
1610         };
1611
1612         argvopt = argv;
1613
1614         while ((opt = getopt_long(argc, argvopt, "p:l:m:h:P",
1615                                 lgopts, &option_index)) != EOF) {
1616
1617                 switch (opt) {
1618                 /* portmask */
1619                 case 'p':
1620                         enabled_port_mask = parse_portmask(optarg);
1621                         if (enabled_port_mask == 0) {
1622                                 printf("invalid portmask\n");
1623                                 print_usage(prgname);
1624                                 return -1;
1625                         }
1626                         break;
1627                 case 'P':
1628                         printf("Promiscuous mode selected\n");
1629                         promiscuous_on = 1;
1630                         break;
1631                 case 'l':
1632                         limit = parse_max_pkt_len(optarg);
1633                         freq_tlb[LOW] = limit;
1634                         break;
1635                 case 'm':
1636                         limit = parse_max_pkt_len(optarg);
1637                         freq_tlb[MED] = limit;
1638                         break;
1639                 case 'h':
1640                         limit = parse_max_pkt_len(optarg);
1641                         freq_tlb[HGH] = limit;
1642                         break;
1643                 /* long options */
1644                 case 0:
1645                         if (!strncmp(lgopts[option_index].name, "config", 6)) {
1646                                 ret = parse_config(optarg);
1647                                 if (ret) {
1648                                         printf("invalid config\n");
1649                                         print_usage(prgname);
1650                                         return -1;
1651                                 }
1652                         }
1653
1654                         if (!strncmp(lgopts[option_index].name,
1655                                         "perf-config", 11)) {
1656                                 ret = parse_perf_config(optarg);
1657                                 if (ret) {
1658                                         printf("invalid perf-config\n");
1659                                         print_usage(prgname);
1660                                         return -1;
1661                                 }
1662                         }
1663
1664                         if (!strncmp(lgopts[option_index].name,
1665                                         "high-perf-cores", 15)) {
1666                                 ret = parse_perf_core_list(optarg);
1667                                 if (ret) {
1668                                         printf("invalid high-perf-cores\n");
1669                                         print_usage(prgname);
1670                                         return -1;
1671                                 }
1672                         }
1673
1674                         if (!strncmp(lgopts[option_index].name,
1675                                                 "no-numa", 7)) {
1676                                 printf("numa is disabled \n");
1677                                 numa_on = 0;
1678                         }
1679
1680                         if (!strncmp(lgopts[option_index].name,
1681                                         CMD_LINE_OPT_LEGACY,
1682                                         sizeof(CMD_LINE_OPT_LEGACY))) {
1683                                 if (app_mode != APP_MODE_DEFAULT) {
1684                                         printf(" legacy mode is mutually exclusive with other modes\n");
1685                                         return -1;
1686                                 }
1687                                 app_mode = APP_MODE_LEGACY;
1688                                 printf("legacy mode is enabled\n");
1689                         }
1690
1691                         if (!strncmp(lgopts[option_index].name,
1692                                         CMD_LINE_OPT_EMPTY_POLL, 10)) {
1693                                 if (app_mode != APP_MODE_DEFAULT) {
1694                                         printf(" empty-poll mode is mutually exclusive with other modes\n");
1695                                         return -1;
1696                                 }
1697                                 app_mode = APP_MODE_EMPTY_POLL;
1698                                 ret = parse_ep_config(optarg);
1699
1700                                 if (ret) {
1701                                         printf("invalid empty poll config\n");
1702                                         print_usage(prgname);
1703                                         return -1;
1704                                 }
1705                                 printf("empty-poll is enabled\n");
1706                         }
1707
1708                         if (!strncmp(lgopts[option_index].name,
1709                                         CMD_LINE_OPT_TELEMETRY,
1710                                         sizeof(CMD_LINE_OPT_TELEMETRY))) {
1711                                 if (app_mode != APP_MODE_DEFAULT) {
1712                                         printf(" telemetry mode is mutually exclusive with other modes\n");
1713                                         return -1;
1714                                 }
1715                                 app_mode = APP_MODE_TELEMETRY;
1716                                 printf("telemetry mode is enabled\n");
1717                         }
1718
1719                         if (!strncmp(lgopts[option_index].name,
1720                                         "enable-jumbo", 12)) {
1721                                 struct option lenopts =
1722                                         {"max-pkt-len", required_argument, \
1723                                                                         0, 0};
1724
1725                                 printf("jumbo frame is enabled \n");
1726                                 port_conf.rxmode.offloads |=
1727                                                 DEV_RX_OFFLOAD_JUMBO_FRAME;
1728                                 port_conf.txmode.offloads |=
1729                                                 DEV_TX_OFFLOAD_MULTI_SEGS;
1730
1731                                 /**
1732                                  * if no max-pkt-len set, use the default value
1733                                  * RTE_ETHER_MAX_LEN
1734                                  */
1735                                 if (0 == getopt_long(argc, argvopt, "",
1736                                                 &lenopts, &option_index)) {
1737                                         ret = parse_max_pkt_len(optarg);
1738                                         if ((ret < 64) ||
1739                                                 (ret > MAX_JUMBO_PKT_LEN)){
1740                                                 printf("invalid packet "
1741                                                                 "length\n");
1742                                                 print_usage(prgname);
1743                                                 return -1;
1744                                         }
1745                                         port_conf.rxmode.max_rx_pkt_len = ret;
1746                                 }
1747                                 printf("set jumbo frame "
1748                                         "max packet length to %u\n",
1749                                 (unsigned int)port_conf.rxmode.max_rx_pkt_len);
1750                         }
1751
1752                         if (!strncmp(lgopts[option_index].name,
1753                                      CMD_LINE_OPT_PARSE_PTYPE,
1754                                      sizeof(CMD_LINE_OPT_PARSE_PTYPE))) {
1755                                 printf("soft parse-ptype is enabled\n");
1756                                 parse_ptype = 1;
1757                         }
1758
1759                         break;
1760
1761                 default:
1762                         print_usage(prgname);
1763                         return -1;
1764                 }
1765         }
1766
1767         if (optind >= 0)
1768                 argv[optind-1] = prgname;
1769
1770         ret = optind-1;
1771         optind = 1; /* reset getopt lib */
1772         return ret;
1773 }
1774
1775 static void
1776 print_ethaddr(const char *name, const struct rte_ether_addr *eth_addr)
1777 {
1778         char buf[RTE_ETHER_ADDR_FMT_SIZE];
1779         rte_ether_format_addr(buf, RTE_ETHER_ADDR_FMT_SIZE, eth_addr);
1780         printf("%s%s", name, buf);
1781 }
1782
1783 #if (APP_LOOKUP_METHOD == APP_LOOKUP_EXACT_MATCH)
1784 static void
1785 setup_hash(int socketid)
1786 {
1787         struct rte_hash_parameters ipv4_l3fwd_hash_params = {
1788                 .name = NULL,
1789                 .entries = L3FWD_HASH_ENTRIES,
1790                 .key_len = sizeof(struct ipv4_5tuple),
1791                 .hash_func = DEFAULT_HASH_FUNC,
1792                 .hash_func_init_val = 0,
1793         };
1794
1795         struct rte_hash_parameters ipv6_l3fwd_hash_params = {
1796                 .name = NULL,
1797                 .entries = L3FWD_HASH_ENTRIES,
1798                 .key_len = sizeof(struct ipv6_5tuple),
1799                 .hash_func = DEFAULT_HASH_FUNC,
1800                 .hash_func_init_val = 0,
1801         };
1802
1803         unsigned i;
1804         int ret;
1805         char s[64];
1806
1807         /* create ipv4 hash */
1808         snprintf(s, sizeof(s), "ipv4_l3fwd_hash_%d", socketid);
1809         ipv4_l3fwd_hash_params.name = s;
1810         ipv4_l3fwd_hash_params.socket_id = socketid;
1811         ipv4_l3fwd_lookup_struct[socketid] =
1812                 rte_hash_create(&ipv4_l3fwd_hash_params);
1813         if (ipv4_l3fwd_lookup_struct[socketid] == NULL)
1814                 rte_exit(EXIT_FAILURE, "Unable to create the l3fwd hash on "
1815                                 "socket %d\n", socketid);
1816
1817         /* create ipv6 hash */
1818         snprintf(s, sizeof(s), "ipv6_l3fwd_hash_%d", socketid);
1819         ipv6_l3fwd_hash_params.name = s;
1820         ipv6_l3fwd_hash_params.socket_id = socketid;
1821         ipv6_l3fwd_lookup_struct[socketid] =
1822                 rte_hash_create(&ipv6_l3fwd_hash_params);
1823         if (ipv6_l3fwd_lookup_struct[socketid] == NULL)
1824                 rte_exit(EXIT_FAILURE, "Unable to create the l3fwd hash on "
1825                                 "socket %d\n", socketid);
1826
1827
1828         /* populate the ipv4 hash */
1829         for (i = 0; i < RTE_DIM(ipv4_l3fwd_route_array); i++) {
1830                 ret = rte_hash_add_key (ipv4_l3fwd_lookup_struct[socketid],
1831                                 (void *) &ipv4_l3fwd_route_array[i].key);
1832                 if (ret < 0) {
1833                         rte_exit(EXIT_FAILURE, "Unable to add entry %u to the"
1834                                 "l3fwd hash on socket %d\n", i, socketid);
1835                 }
1836                 ipv4_l3fwd_out_if[ret] = ipv4_l3fwd_route_array[i].if_out;
1837                 printf("Hash: Adding key\n");
1838                 print_ipv4_key(ipv4_l3fwd_route_array[i].key);
1839         }
1840
1841         /* populate the ipv6 hash */
1842         for (i = 0; i < RTE_DIM(ipv6_l3fwd_route_array); i++) {
1843                 ret = rte_hash_add_key (ipv6_l3fwd_lookup_struct[socketid],
1844                                 (void *) &ipv6_l3fwd_route_array[i].key);
1845                 if (ret < 0) {
1846                         rte_exit(EXIT_FAILURE, "Unable to add entry %u to the"
1847                                 "l3fwd hash on socket %d\n", i, socketid);
1848                 }
1849                 ipv6_l3fwd_out_if[ret] = ipv6_l3fwd_route_array[i].if_out;
1850                 printf("Hash: Adding key\n");
1851                 print_ipv6_key(ipv6_l3fwd_route_array[i].key);
1852         }
1853 }
1854 #endif
1855
1856 #if (APP_LOOKUP_METHOD == APP_LOOKUP_LPM)
1857 static void
1858 setup_lpm(int socketid)
1859 {
1860         unsigned i;
1861         int ret;
1862         char s[64];
1863
1864         /* create the LPM table */
1865         struct rte_lpm_config lpm_ipv4_config;
1866
1867         lpm_ipv4_config.max_rules = IPV4_L3FWD_LPM_MAX_RULES;
1868         lpm_ipv4_config.number_tbl8s = 256;
1869         lpm_ipv4_config.flags = 0;
1870
1871         snprintf(s, sizeof(s), "IPV4_L3FWD_LPM_%d", socketid);
1872         ipv4_l3fwd_lookup_struct[socketid] =
1873                         rte_lpm_create(s, socketid, &lpm_ipv4_config);
1874         if (ipv4_l3fwd_lookup_struct[socketid] == NULL)
1875                 rte_exit(EXIT_FAILURE, "Unable to create the l3fwd LPM table"
1876                                 " on socket %d\n", socketid);
1877
1878         /* populate the LPM table */
1879         for (i = 0; i < RTE_DIM(ipv4_l3fwd_route_array); i++) {
1880                 ret = rte_lpm_add(ipv4_l3fwd_lookup_struct[socketid],
1881                         ipv4_l3fwd_route_array[i].ip,
1882                         ipv4_l3fwd_route_array[i].depth,
1883                         ipv4_l3fwd_route_array[i].if_out);
1884
1885                 if (ret < 0) {
1886                         rte_exit(EXIT_FAILURE, "Unable to add entry %u to the "
1887                                 "l3fwd LPM table on socket %d\n",
1888                                 i, socketid);
1889                 }
1890
1891                 printf("LPM: Adding route 0x%08x / %d (%d)\n",
1892                         (unsigned)ipv4_l3fwd_route_array[i].ip,
1893                         ipv4_l3fwd_route_array[i].depth,
1894                         ipv4_l3fwd_route_array[i].if_out);
1895         }
1896 }
1897 #endif
1898
1899 static int
1900 init_mem(unsigned nb_mbuf)
1901 {
1902         struct lcore_conf *qconf;
1903         int socketid;
1904         unsigned lcore_id;
1905         char s[64];
1906
1907         for (lcore_id = 0; lcore_id < RTE_MAX_LCORE; lcore_id++) {
1908                 if (rte_lcore_is_enabled(lcore_id) == 0)
1909                         continue;
1910
1911                 if (numa_on)
1912                         socketid = rte_lcore_to_socket_id(lcore_id);
1913                 else
1914                         socketid = 0;
1915
1916                 if (socketid >= NB_SOCKETS) {
1917                         rte_exit(EXIT_FAILURE, "Socket %d of lcore %u is "
1918                                         "out of range %d\n", socketid,
1919                                                 lcore_id, NB_SOCKETS);
1920                 }
1921                 if (pktmbuf_pool[socketid] == NULL) {
1922                         snprintf(s, sizeof(s), "mbuf_pool_%d", socketid);
1923                         pktmbuf_pool[socketid] =
1924                                 rte_pktmbuf_pool_create(s, nb_mbuf,
1925                                         MEMPOOL_CACHE_SIZE, 0,
1926                                         RTE_MBUF_DEFAULT_BUF_SIZE,
1927                                         socketid);
1928                         if (pktmbuf_pool[socketid] == NULL)
1929                                 rte_exit(EXIT_FAILURE,
1930                                         "Cannot init mbuf pool on socket %d\n",
1931                                                                 socketid);
1932                         else
1933                                 printf("Allocated mbuf pool on socket %d\n",
1934                                                                 socketid);
1935
1936 #if (APP_LOOKUP_METHOD == APP_LOOKUP_LPM)
1937                         setup_lpm(socketid);
1938 #else
1939                         setup_hash(socketid);
1940 #endif
1941                 }
1942                 qconf = &lcore_conf[lcore_id];
1943                 qconf->ipv4_lookup_struct = ipv4_l3fwd_lookup_struct[socketid];
1944 #if (APP_LOOKUP_METHOD == APP_LOOKUP_EXACT_MATCH)
1945                 qconf->ipv6_lookup_struct = ipv6_l3fwd_lookup_struct[socketid];
1946 #endif
1947         }
1948         return 0;
1949 }
1950
1951 /* Check the link status of all ports in up to 9s, and print them finally */
1952 static void
1953 check_all_ports_link_status(uint32_t port_mask)
1954 {
1955 #define CHECK_INTERVAL 100 /* 100ms */
1956 #define MAX_CHECK_TIME 90 /* 9s (90 * 100ms) in total */
1957         uint8_t count, all_ports_up, print_flag = 0;
1958         uint16_t portid;
1959         struct rte_eth_link link;
1960         int ret;
1961
1962         printf("\nChecking link status");
1963         fflush(stdout);
1964         for (count = 0; count <= MAX_CHECK_TIME; count++) {
1965                 all_ports_up = 1;
1966                 RTE_ETH_FOREACH_DEV(portid) {
1967                         if ((port_mask & (1 << portid)) == 0)
1968                                 continue;
1969                         memset(&link, 0, sizeof(link));
1970                         ret = rte_eth_link_get_nowait(portid, &link);
1971                         if (ret < 0) {
1972                                 all_ports_up = 0;
1973                                 if (print_flag == 1)
1974                                         printf("Port %u link get failed: %s\n",
1975                                                 portid, rte_strerror(-ret));
1976                                 continue;
1977                         }
1978                         /* print link status if flag set */
1979                         if (print_flag == 1) {
1980                                 if (link.link_status)
1981                                         printf("Port %d Link Up - speed %u "
1982                                                 "Mbps - %s\n", (uint8_t)portid,
1983                                                 (unsigned)link.link_speed,
1984                                 (link.link_duplex == ETH_LINK_FULL_DUPLEX) ?
1985                                         ("full-duplex") : ("half-duplex"));
1986                                 else
1987                                         printf("Port %d Link Down\n",
1988                                                 (uint8_t)portid);
1989                                 continue;
1990                         }
1991                         /* clear all_ports_up flag if any link down */
1992                         if (link.link_status == ETH_LINK_DOWN) {
1993                                 all_ports_up = 0;
1994                                 break;
1995                         }
1996                 }
1997                 /* after finally printing all link status, get out */
1998                 if (print_flag == 1)
1999                         break;
2000
2001                 if (all_ports_up == 0) {
2002                         printf(".");
2003                         fflush(stdout);
2004                         rte_delay_ms(CHECK_INTERVAL);
2005                 }
2006
2007                 /* set the print_flag if all ports up or timeout */
2008                 if (all_ports_up == 1 || count == (MAX_CHECK_TIME - 1)) {
2009                         print_flag = 1;
2010                         printf("done\n");
2011                 }
2012         }
2013 }
2014
2015 static int check_ptype(uint16_t portid)
2016 {
2017         int i, ret;
2018         int ptype_l3_ipv4 = 0;
2019 #if (APP_LOOKUP_METHOD == APP_LOOKUP_EXACT_MATCH)
2020         int ptype_l3_ipv6 = 0;
2021 #endif
2022         uint32_t ptype_mask = RTE_PTYPE_L3_MASK;
2023
2024         ret = rte_eth_dev_get_supported_ptypes(portid, ptype_mask, NULL, 0);
2025         if (ret <= 0)
2026                 return 0;
2027
2028         uint32_t ptypes[ret];
2029
2030         ret = rte_eth_dev_get_supported_ptypes(portid, ptype_mask, ptypes, ret);
2031         for (i = 0; i < ret; ++i) {
2032                 if (ptypes[i] & RTE_PTYPE_L3_IPV4)
2033                         ptype_l3_ipv4 = 1;
2034 #if (APP_LOOKUP_METHOD == APP_LOOKUP_EXACT_MATCH)
2035                 if (ptypes[i] & RTE_PTYPE_L3_IPV6)
2036                         ptype_l3_ipv6 = 1;
2037 #endif
2038         }
2039
2040         if (ptype_l3_ipv4 == 0)
2041                 printf("port %d cannot parse RTE_PTYPE_L3_IPV4\n", portid);
2042
2043 #if (APP_LOOKUP_METHOD == APP_LOOKUP_EXACT_MATCH)
2044         if (ptype_l3_ipv6 == 0)
2045                 printf("port %d cannot parse RTE_PTYPE_L3_IPV6\n", portid);
2046 #endif
2047
2048 #if (APP_LOOKUP_METHOD == APP_LOOKUP_LPM)
2049         if (ptype_l3_ipv4)
2050 #else /* APP_LOOKUP_EXACT_MATCH */
2051         if (ptype_l3_ipv4 && ptype_l3_ipv6)
2052 #endif
2053                 return 1;
2054
2055         return 0;
2056
2057 }
2058
2059 static int
2060 init_power_library(void)
2061 {
2062         enum power_management_env env;
2063         unsigned int lcore_id;
2064         int ret = 0;
2065
2066         RTE_LCORE_FOREACH(lcore_id) {
2067                 /* init power management library */
2068                 ret = rte_power_init(lcore_id);
2069                 if (ret) {
2070                         RTE_LOG(ERR, POWER,
2071                                 "Library initialization failed on core %u\n",
2072                                 lcore_id);
2073                         return ret;
2074                 }
2075                 /* we're not supporting the VM channel mode */
2076                 env = rte_power_get_env();
2077                 if (env != PM_ENV_ACPI_CPUFREQ &&
2078                                 env != PM_ENV_PSTATE_CPUFREQ) {
2079                         RTE_LOG(ERR, POWER,
2080                                 "Only ACPI and PSTATE mode are supported\n");
2081                         return -1;
2082                 }
2083         }
2084         return ret;
2085 }
2086
2087 static int
2088 deinit_power_library(void)
2089 {
2090         unsigned int lcore_id;
2091         int ret = 0;
2092
2093         RTE_LCORE_FOREACH(lcore_id) {
2094                 /* deinit power management library */
2095                 ret = rte_power_exit(lcore_id);
2096                 if (ret) {
2097                         RTE_LOG(ERR, POWER,
2098                                 "Library deinitialization failed on core %u\n",
2099                                 lcore_id);
2100                         return ret;
2101                 }
2102         }
2103         return ret;
2104 }
2105
2106 static void
2107 get_current_stat_values(uint64_t *values)
2108 {
2109         unsigned int lcore_id = rte_lcore_id();
2110         struct lcore_conf *qconf;
2111         uint64_t app_eps = 0, app_fps = 0, app_br = 0;
2112         uint64_t count = 0;
2113
2114         RTE_LCORE_FOREACH_SLAVE(lcore_id) {
2115                 qconf = &lcore_conf[lcore_id];
2116                 if (qconf->n_rx_queue == 0)
2117                         continue;
2118                 count++;
2119                 rte_spinlock_lock(&stats[lcore_id].telemetry_lock);
2120                 app_eps += stats[lcore_id].ep_nep[1];
2121                 app_fps += stats[lcore_id].fp_nfp[1];
2122                 app_br += stats[lcore_id].br;
2123                 rte_spinlock_unlock(&stats[lcore_id].telemetry_lock);
2124         }
2125
2126         if (count > 0) {
2127                 values[0] = app_eps/count;
2128                 values[1] = app_fps/count;
2129                 values[2] = app_br/count;
2130         } else
2131                 memset(values, 0, sizeof(uint64_t) * NUM_TELSTATS);
2132
2133 }
2134
2135 static void
2136 update_telemetry(__rte_unused struct rte_timer *tim,
2137                 __rte_unused void *arg)
2138 {
2139         int ret;
2140         uint64_t values[NUM_TELSTATS] = {0};
2141
2142         get_current_stat_values(values);
2143         ret = rte_metrics_update_values(RTE_METRICS_GLOBAL, telstats_index,
2144                                         values, RTE_DIM(values));
2145         if (ret < 0)
2146                 RTE_LOG(WARNING, POWER, "failed to update metrcis\n");
2147 }
2148
2149 static int
2150 handle_app_stats(const char *cmd __rte_unused,
2151                 const char *params __rte_unused,
2152                 struct rte_tel_data *d)
2153 {
2154         uint64_t values[NUM_TELSTATS] = {0};
2155         uint32_t i;
2156
2157         rte_tel_data_start_dict(d);
2158         get_current_stat_values(values);
2159         for (i = 0; i < NUM_TELSTATS; i++)
2160                 rte_tel_data_add_dict_u64(d, telstats_strings[i].name,
2161                                 values[i]);
2162         return 0;
2163 }
2164
2165 static void
2166 telemetry_setup_timer(void)
2167 {
2168         int lcore_id = rte_lcore_id();
2169         uint64_t hz = rte_get_timer_hz();
2170         uint64_t ticks;
2171
2172         ticks = hz / TELEMETRY_INTERVALS_PER_SEC;
2173         rte_timer_reset_sync(&telemetry_timer,
2174                         ticks,
2175                         PERIODICAL,
2176                         lcore_id,
2177                         update_telemetry,
2178                         NULL);
2179 }
2180 static void
2181 empty_poll_setup_timer(void)
2182 {
2183         int lcore_id = rte_lcore_id();
2184         uint64_t hz = rte_get_timer_hz();
2185
2186         struct  ep_params *ep_ptr = ep_params;
2187
2188         ep_ptr->interval_ticks = hz / INTERVALS_PER_SECOND;
2189
2190         rte_timer_reset_sync(&ep_ptr->timer0,
2191                         ep_ptr->interval_ticks,
2192                         PERIODICAL,
2193                         lcore_id,
2194                         rte_empty_poll_detection,
2195                         (void *)ep_ptr);
2196
2197 }
2198 static int
2199 launch_timer(unsigned int lcore_id)
2200 {
2201         int64_t prev_tsc = 0, cur_tsc, diff_tsc, cycles_10ms;
2202
2203         RTE_SET_USED(lcore_id);
2204
2205
2206         if (rte_get_master_lcore() != lcore_id) {
2207                 rte_panic("timer on lcore:%d which is not master core:%d\n",
2208                                 lcore_id,
2209                                 rte_get_master_lcore());
2210         }
2211
2212         RTE_LOG(INFO, POWER, "Bring up the Timer\n");
2213
2214         if (app_mode == APP_MODE_EMPTY_POLL)
2215                 empty_poll_setup_timer();
2216         else
2217                 telemetry_setup_timer();
2218
2219         cycles_10ms = rte_get_timer_hz() / 100;
2220
2221         while (!is_done()) {
2222                 cur_tsc = rte_rdtsc();
2223                 diff_tsc = cur_tsc - prev_tsc;
2224                 if (diff_tsc > cycles_10ms) {
2225                         rte_timer_manage();
2226                         prev_tsc = cur_tsc;
2227                         cycles_10ms = rte_get_timer_hz() / 100;
2228                 }
2229         }
2230
2231         RTE_LOG(INFO, POWER, "Timer_subsystem is done\n");
2232
2233         return 0;
2234 }
2235
2236
2237 int
2238 main(int argc, char **argv)
2239 {
2240         struct lcore_conf *qconf;
2241         struct rte_eth_dev_info dev_info;
2242         struct rte_eth_txconf *txconf;
2243         int ret;
2244         uint16_t nb_ports;
2245         uint16_t queueid;
2246         unsigned lcore_id;
2247         uint64_t hz;
2248         uint32_t n_tx_queue, nb_lcores;
2249         uint32_t dev_rxq_num, dev_txq_num;
2250         uint8_t nb_rx_queue, queue, socketid;
2251         uint16_t portid;
2252         const char *ptr_strings[NUM_TELSTATS];
2253
2254         /* catch SIGINT and restore cpufreq governor to ondemand */
2255         signal(SIGINT, signal_exit_now);
2256
2257         /* init EAL */
2258         ret = rte_eal_init(argc, argv);
2259         if (ret < 0)
2260                 rte_exit(EXIT_FAILURE, "Invalid EAL parameters\n");
2261         argc -= ret;
2262         argv += ret;
2263
2264         /* init RTE timer library to be used late */
2265         rte_timer_subsystem_init();
2266
2267         /* parse application arguments (after the EAL ones) */
2268         ret = parse_args(argc, argv);
2269         if (ret < 0)
2270                 rte_exit(EXIT_FAILURE, "Invalid L3FWD parameters\n");
2271
2272         if (app_mode == APP_MODE_DEFAULT)
2273                 app_mode = APP_MODE_LEGACY;
2274
2275         /* only legacy and empty poll mode rely on power library */
2276         if ((app_mode == APP_MODE_LEGACY || app_mode == APP_MODE_EMPTY_POLL) &&
2277                         init_power_library())
2278                 rte_exit(EXIT_FAILURE, "init_power_library failed\n");
2279
2280         if (update_lcore_params() < 0)
2281                 rte_exit(EXIT_FAILURE, "update_lcore_params failed\n");
2282
2283         if (check_lcore_params() < 0)
2284                 rte_exit(EXIT_FAILURE, "check_lcore_params failed\n");
2285
2286         ret = init_lcore_rx_queues();
2287         if (ret < 0)
2288                 rte_exit(EXIT_FAILURE, "init_lcore_rx_queues failed\n");
2289
2290         nb_ports = rte_eth_dev_count_avail();
2291
2292         if (check_port_config() < 0)
2293                 rte_exit(EXIT_FAILURE, "check_port_config failed\n");
2294
2295         nb_lcores = rte_lcore_count();
2296
2297         /* initialize all ports */
2298         RTE_ETH_FOREACH_DEV(portid) {
2299                 struct rte_eth_conf local_port_conf = port_conf;
2300                 /* not all app modes need interrupts */
2301                 bool need_intr = app_mode == APP_MODE_LEGACY;
2302
2303                 /* skip ports that are not enabled */
2304                 if ((enabled_port_mask & (1 << portid)) == 0) {
2305                         printf("\nSkipping disabled port %d\n", portid);
2306                         continue;
2307                 }
2308
2309                 /* init port */
2310                 printf("Initializing port %d ... ", portid );
2311                 fflush(stdout);
2312
2313                 ret = rte_eth_dev_info_get(portid, &dev_info);
2314                 if (ret != 0)
2315                         rte_exit(EXIT_FAILURE,
2316                                 "Error during getting device (port %u) info: %s\n",
2317                                 portid, strerror(-ret));
2318
2319                 dev_rxq_num = dev_info.max_rx_queues;
2320                 dev_txq_num = dev_info.max_tx_queues;
2321
2322                 nb_rx_queue = get_port_n_rx_queues(portid);
2323                 if (nb_rx_queue > dev_rxq_num)
2324                         rte_exit(EXIT_FAILURE,
2325                                 "Cannot configure not existed rxq: "
2326                                 "port=%d\n", portid);
2327
2328                 n_tx_queue = nb_lcores;
2329                 if (n_tx_queue > dev_txq_num)
2330                         n_tx_queue = dev_txq_num;
2331                 printf("Creating queues: nb_rxq=%d nb_txq=%u... ",
2332                         nb_rx_queue, (unsigned)n_tx_queue );
2333                 /* If number of Rx queue is 0, no need to enable Rx interrupt */
2334                 if (nb_rx_queue == 0)
2335                         need_intr = false;
2336
2337                 if (need_intr)
2338                         local_port_conf.intr_conf.rxq = 1;
2339
2340                 ret = rte_eth_dev_info_get(portid, &dev_info);
2341                 if (ret != 0)
2342                         rte_exit(EXIT_FAILURE,
2343                                 "Error during getting device (port %u) info: %s\n",
2344                                 portid, strerror(-ret));
2345
2346                 if (dev_info.tx_offload_capa & DEV_TX_OFFLOAD_MBUF_FAST_FREE)
2347                         local_port_conf.txmode.offloads |=
2348                                 DEV_TX_OFFLOAD_MBUF_FAST_FREE;
2349
2350                 local_port_conf.rx_adv_conf.rss_conf.rss_hf &=
2351                         dev_info.flow_type_rss_offloads;
2352                 if (local_port_conf.rx_adv_conf.rss_conf.rss_hf !=
2353                                 port_conf.rx_adv_conf.rss_conf.rss_hf) {
2354                         printf("Port %u modified RSS hash function based on hardware support,"
2355                                 "requested:%#"PRIx64" configured:%#"PRIx64"\n",
2356                                 portid,
2357                                 port_conf.rx_adv_conf.rss_conf.rss_hf,
2358                                 local_port_conf.rx_adv_conf.rss_conf.rss_hf);
2359                 }
2360
2361                 ret = rte_eth_dev_configure(portid, nb_rx_queue,
2362                                         (uint16_t)n_tx_queue, &local_port_conf);
2363                 if (ret < 0)
2364                         rte_exit(EXIT_FAILURE, "Cannot configure device: "
2365                                         "err=%d, port=%d\n", ret, portid);
2366
2367                 ret = rte_eth_dev_adjust_nb_rx_tx_desc(portid, &nb_rxd,
2368                                                        &nb_txd);
2369                 if (ret < 0)
2370                         rte_exit(EXIT_FAILURE,
2371                                  "Cannot adjust number of descriptors: err=%d, port=%d\n",
2372                                  ret, portid);
2373
2374                 ret = rte_eth_macaddr_get(portid, &ports_eth_addr[portid]);
2375                 if (ret < 0)
2376                         rte_exit(EXIT_FAILURE,
2377                                  "Cannot get MAC address: err=%d, port=%d\n",
2378                                  ret, portid);
2379
2380                 print_ethaddr(" Address:", &ports_eth_addr[portid]);
2381                 printf(", ");
2382
2383                 /* init memory */
2384                 ret = init_mem(NB_MBUF);
2385                 if (ret < 0)
2386                         rte_exit(EXIT_FAILURE, "init_mem failed\n");
2387
2388                 for (lcore_id = 0; lcore_id < RTE_MAX_LCORE; lcore_id++) {
2389                         if (rte_lcore_is_enabled(lcore_id) == 0)
2390                                 continue;
2391
2392                         /* Initialize TX buffers */
2393                         qconf = &lcore_conf[lcore_id];
2394                         qconf->tx_buffer[portid] = rte_zmalloc_socket("tx_buffer",
2395                                 RTE_ETH_TX_BUFFER_SIZE(MAX_PKT_BURST), 0,
2396                                 rte_eth_dev_socket_id(portid));
2397                         if (qconf->tx_buffer[portid] == NULL)
2398                                 rte_exit(EXIT_FAILURE, "Can't allocate tx buffer for port %u\n",
2399                                                  portid);
2400
2401                         rte_eth_tx_buffer_init(qconf->tx_buffer[portid], MAX_PKT_BURST);
2402                 }
2403
2404                 /* init one TX queue per couple (lcore,port) */
2405                 queueid = 0;
2406                 for (lcore_id = 0; lcore_id < RTE_MAX_LCORE; lcore_id++) {
2407                         if (rte_lcore_is_enabled(lcore_id) == 0)
2408                                 continue;
2409
2410                         if (queueid >= dev_txq_num)
2411                                 continue;
2412
2413                         if (numa_on)
2414                                 socketid = \
2415                                 (uint8_t)rte_lcore_to_socket_id(lcore_id);
2416                         else
2417                                 socketid = 0;
2418
2419                         printf("txq=%u,%d,%d ", lcore_id, queueid, socketid);
2420                         fflush(stdout);
2421
2422                         txconf = &dev_info.default_txconf;
2423                         txconf->offloads = local_port_conf.txmode.offloads;
2424                         ret = rte_eth_tx_queue_setup(portid, queueid, nb_txd,
2425                                                      socketid, txconf);
2426                         if (ret < 0)
2427                                 rte_exit(EXIT_FAILURE,
2428                                         "rte_eth_tx_queue_setup: err=%d, "
2429                                                 "port=%d\n", ret, portid);
2430
2431                         qconf = &lcore_conf[lcore_id];
2432                         qconf->tx_queue_id[portid] = queueid;
2433                         queueid++;
2434
2435                         qconf->tx_port_id[qconf->n_tx_port] = portid;
2436                         qconf->n_tx_port++;
2437                 }
2438                 printf("\n");
2439         }
2440
2441         for (lcore_id = 0; lcore_id < RTE_MAX_LCORE; lcore_id++) {
2442                 if (rte_lcore_is_enabled(lcore_id) == 0)
2443                         continue;
2444
2445                 if (app_mode == APP_MODE_LEGACY) {
2446                         /* init timer structures for each enabled lcore */
2447                         rte_timer_init(&power_timers[lcore_id]);
2448                         hz = rte_get_timer_hz();
2449                         rte_timer_reset(&power_timers[lcore_id],
2450                                         hz/TIMER_NUMBER_PER_SECOND,
2451                                         SINGLE, lcore_id,
2452                                         power_timer_cb, NULL);
2453                 }
2454                 qconf = &lcore_conf[lcore_id];
2455                 printf("\nInitializing rx queues on lcore %u ... ", lcore_id );
2456                 fflush(stdout);
2457                 /* init RX queues */
2458                 for(queue = 0; queue < qconf->n_rx_queue; ++queue) {
2459                         struct rte_eth_rxconf rxq_conf;
2460
2461                         portid = qconf->rx_queue_list[queue].port_id;
2462                         queueid = qconf->rx_queue_list[queue].queue_id;
2463
2464                         if (numa_on)
2465                                 socketid = \
2466                                 (uint8_t)rte_lcore_to_socket_id(lcore_id);
2467                         else
2468                                 socketid = 0;
2469
2470                         printf("rxq=%d,%d,%d ", portid, queueid, socketid);
2471                         fflush(stdout);
2472
2473                         ret = rte_eth_dev_info_get(portid, &dev_info);
2474                         if (ret != 0)
2475                                 rte_exit(EXIT_FAILURE,
2476                                         "Error during getting device (port %u) info: %s\n",
2477                                         portid, strerror(-ret));
2478
2479                         rxq_conf = dev_info.default_rxconf;
2480                         rxq_conf.offloads = port_conf.rxmode.offloads;
2481                         ret = rte_eth_rx_queue_setup(portid, queueid, nb_rxd,
2482                                 socketid, &rxq_conf,
2483                                 pktmbuf_pool[socketid]);
2484                         if (ret < 0)
2485                                 rte_exit(EXIT_FAILURE,
2486                                         "rte_eth_rx_queue_setup: err=%d, "
2487                                                 "port=%d\n", ret, portid);
2488
2489                         if (parse_ptype) {
2490                                 if (add_cb_parse_ptype(portid, queueid) < 0)
2491                                         rte_exit(EXIT_FAILURE,
2492                                                  "Fail to add ptype cb\n");
2493                         } else if (!check_ptype(portid))
2494                                 rte_exit(EXIT_FAILURE,
2495                                          "PMD can not provide needed ptypes\n");
2496                 }
2497         }
2498
2499         printf("\n");
2500
2501         /* start ports */
2502         RTE_ETH_FOREACH_DEV(portid) {
2503                 if ((enabled_port_mask & (1 << portid)) == 0) {
2504                         continue;
2505                 }
2506                 /* Start device */
2507                 ret = rte_eth_dev_start(portid);
2508                 if (ret < 0)
2509                         rte_exit(EXIT_FAILURE, "rte_eth_dev_start: err=%d, "
2510                                                 "port=%d\n", ret, portid);
2511                 /*
2512                  * If enabled, put device in promiscuous mode.
2513                  * This allows IO forwarding mode to forward packets
2514                  * to itself through 2 cross-connected  ports of the
2515                  * target machine.
2516                  */
2517                 if (promiscuous_on) {
2518                         ret = rte_eth_promiscuous_enable(portid);
2519                         if (ret != 0)
2520                                 rte_exit(EXIT_FAILURE,
2521                                         "rte_eth_promiscuous_enable: err=%s, port=%u\n",
2522                                         rte_strerror(-ret), portid);
2523                 }
2524                 /* initialize spinlock for each port */
2525                 rte_spinlock_init(&(locks[portid]));
2526         }
2527
2528         check_all_ports_link_status(enabled_port_mask);
2529
2530         if (app_mode == APP_MODE_EMPTY_POLL) {
2531
2532                 if (empty_poll_train) {
2533                         policy.state = TRAINING;
2534                 } else {
2535                         policy.state = MED_NORMAL;
2536                         policy.med_base_edpi = ep_med_edpi;
2537                         policy.hgh_base_edpi = ep_hgh_edpi;
2538                 }
2539
2540                 ret = rte_power_empty_poll_stat_init(&ep_params,
2541                                 freq_tlb,
2542                                 &policy);
2543                 if (ret < 0)
2544                         rte_exit(EXIT_FAILURE, "empty poll init failed");
2545         }
2546
2547
2548         /* launch per-lcore init on every lcore */
2549         if (app_mode == APP_MODE_LEGACY) {
2550                 rte_eal_mp_remote_launch(main_legacy_loop, NULL, CALL_MASTER);
2551         } else if (app_mode == APP_MODE_EMPTY_POLL) {
2552                 empty_poll_stop = false;
2553                 rte_eal_mp_remote_launch(main_empty_poll_loop, NULL,
2554                                 SKIP_MASTER);
2555         } else if (app_mode == APP_MODE_TELEMETRY) {
2556                 unsigned int i;
2557
2558                 /* Init metrics library */
2559                 rte_metrics_init(rte_socket_id());
2560                 /** Register stats with metrics library */
2561                 for (i = 0; i < NUM_TELSTATS; i++)
2562                         ptr_strings[i] = telstats_strings[i].name;
2563
2564                 ret = rte_metrics_reg_names(ptr_strings, NUM_TELSTATS);
2565                 if (ret >= 0)
2566                         telstats_index = ret;
2567                 else
2568                         rte_exit(EXIT_FAILURE, "failed to register metrics names");
2569
2570                 RTE_LCORE_FOREACH_SLAVE(lcore_id) {
2571                         rte_spinlock_init(&stats[lcore_id].telemetry_lock);
2572                 }
2573                 rte_timer_init(&telemetry_timer);
2574                 rte_telemetry_register_cmd("/l3fwd-power/stats",
2575                                 handle_app_stats,
2576                                 "Returns global power stats. Parameters: None");
2577                 rte_eal_mp_remote_launch(main_telemetry_loop, NULL,
2578                                                 SKIP_MASTER);
2579         }
2580
2581         if (app_mode == APP_MODE_EMPTY_POLL || app_mode == APP_MODE_TELEMETRY)
2582                 launch_timer(rte_lcore_id());
2583
2584         RTE_LCORE_FOREACH_SLAVE(lcore_id) {
2585                 if (rte_eal_wait_lcore(lcore_id) < 0)
2586                         return -1;
2587         }
2588
2589         RTE_ETH_FOREACH_DEV(portid)
2590         {
2591                 if ((enabled_port_mask & (1 << portid)) == 0)
2592                         continue;
2593
2594                 rte_eth_dev_stop(portid);
2595                 rte_eth_dev_close(portid);
2596         }
2597
2598         if (app_mode == APP_MODE_EMPTY_POLL)
2599                 rte_power_empty_poll_stat_free();
2600
2601         if ((app_mode == APP_MODE_LEGACY || app_mode == APP_MODE_EMPTY_POLL) &&
2602                         deinit_power_library())
2603                 rte_exit(EXIT_FAILURE, "deinit_power_library failed\n");
2604
2605         if (rte_eal_cleanup() < 0)
2606                 RTE_LOG(ERR, L3FWD_POWER, "EAL cleanup failed\n");
2607
2608         return 0;
2609 }