X-Git-Url: http://git.droids-corp.org/?a=blobdiff_plain;f=examples%2Fl3fwd-power%2Fmain.c;h=afa38c158c45c1e03bd997b0648470a6ddc6da78;hb=ce6b8c31548b4d71a986d9807cd06cf3a616d1ab;hp=e7ebe30f2f7e4a25e0b0fa70710541f77257d9ac;hpb=bec2f7df9c4509e42c266b164af3ac5998447ffb;p=dpdk.git diff --git a/examples/l3fwd-power/main.c b/examples/l3fwd-power/main.c index e7ebe30f2f..afa38c158c 100644 --- a/examples/l3fwd-power/main.c +++ b/examples/l3fwd-power/main.c @@ -1,34 +1,5 @@ -/*- - * BSD LICENSE - * - * Copyright(c) 2010-2016 Intel Corporation. All rights reserved. - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * - * * Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in - * the documentation and/or other materials provided with the - * distribution. - * * Neither the name of Intel Corporation nor the names of its - * contributors may be used to endorse or promote products derived - * from this software without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright(c) 2010-2018 Intel Corporation */ #include @@ -43,6 +14,7 @@ #include #include #include +#include #include #include @@ -50,9 +22,7 @@ #include #include #include -#include #include -#include #include #include #include @@ -61,12 +31,10 @@ #include #include #include -#include #include #include #include #include -#include #include #include #include @@ -75,8 +43,13 @@ #include #include #include -#include #include +#include +#include +#include + +#include "perf_core.h" +#include "main.h" #define RTE_LOGTYPE_L3FWD_POWER RTE_LOGTYPE_USER1 @@ -84,10 +57,10 @@ #define MIN_ZERO_POLL_COUNT 10 -/* around 100ms at 2 Ghz */ -#define TIMER_RESOLUTION_CYCLES 200000000ULL /* 100 ms interval */ #define TIMER_NUMBER_PER_SECOND 10 +/* (10ms) */ +#define INTERVALS_PER_SECOND 100 /* 100000 us */ #define SCALING_PERIOD (1000000/TIMER_NUMBER_PER_SECOND) #define SCALING_DOWN_TIME_RATIO_THRESHOLD 0.25 @@ -132,9 +105,9 @@ */ #define NB_MBUF RTE_MAX ( \ - (nb_ports*nb_rx_queue*RTE_TEST_RX_DESC_DEFAULT + \ + (nb_ports*nb_rx_queue*nb_rxd + \ nb_ports*nb_lcores*MAX_PKT_BURST + \ - nb_ports*n_tx_queue*RTE_TEST_TX_DESC_DEFAULT + \ + nb_ports*n_tx_queue*nb_txd + \ nb_lcores*MEMPOOL_CACHE_SIZE), \ (unsigned)8192) @@ -148,13 +121,24 @@ /* * Configurable number of RX/TX ring descriptors */ -#define RTE_TEST_RX_DESC_DEFAULT 128 -#define RTE_TEST_TX_DESC_DEFAULT 512 +#define RTE_TEST_RX_DESC_DEFAULT 1024 +#define RTE_TEST_TX_DESC_DEFAULT 1024 + +/* + * These two thresholds were decided on by running the training algorithm on + * a 2.5GHz Xeon. These defaults can be overridden by supplying non-zero values + * for the med_threshold and high_threshold parameters on the command line. + */ +#define EMPTY_POLL_MED_THRESHOLD 350000UL +#define EMPTY_POLL_HGH_THRESHOLD 580000UL + +#define NUM_TELSTATS RTE_DIM(telstats_strings) + static uint16_t nb_rxd = RTE_TEST_RX_DESC_DEFAULT; static uint16_t nb_txd = RTE_TEST_TX_DESC_DEFAULT; /* ethernet addresses of ports */ -static struct ether_addr ports_eth_addr[RTE_MAX_ETHPORTS]; +static struct rte_ether_addr ports_eth_addr[RTE_MAX_ETHPORTS]; /* ethernet addresses of ports */ static rte_spinlock_t locks[RTE_MAX_ETHPORTS]; @@ -165,6 +149,60 @@ static uint32_t enabled_port_mask = 0; static int promiscuous_on = 0; /* NUMA is enabled by default. */ static int numa_on = 1; +static bool empty_poll_stop; +static bool empty_poll_train; +volatile bool quit_signal; +static struct ep_params *ep_params; +static struct ep_policy policy; +static long ep_med_edpi, ep_hgh_edpi; +/* timer to update telemetry every 500ms */ +static struct rte_timer telemetry_timer; + +/* stats index returned by metrics lib */ +int telstats_index; + +struct telstats_name { + char name[RTE_ETH_XSTATS_NAME_SIZE]; +}; + +/* telemetry stats to be reported */ +const struct telstats_name telstats_strings[] = { + {"empty_poll"}, + {"full_poll"}, + {"busy_percent"} +}; + +/* core busyness in percentage */ +enum busy_rate { + ZERO = 0, + PARTIAL = 50, + FULL = 100 +}; + +/* reference poll count to measure core busyness */ +#define DEFAULT_COUNT 10000 +/* + * reference CYCLES to be used to + * measure core busyness based on poll count + */ +#define MIN_CYCLES 1500000ULL +#define MAX_CYCLES 22000000ULL + +/* (500ms) */ +#define TELEMETRY_INTERVALS_PER_SEC 2 + +static int parse_ptype; /**< Parse packet type using rx callback, and */ + /**< disabled by default */ + +enum appmode { + APP_MODE_DEFAULT = 0, + APP_MODE_LEGACY, + APP_MODE_EMPTY_POLL, + APP_MODE_TELEMETRY, + APP_MODE_INTERRUPT +}; + +enum appmode app_mode; enum freq_scale_hint_t { @@ -175,7 +213,7 @@ enum freq_scale_hint_t }; struct lcore_rx_queue { - uint8_t port_id; + uint16_t port_id; uint8_t queue_id; enum freq_scale_hint_t freq_up_hint; uint32_t zero_rx_packet_count; @@ -189,14 +227,7 @@ struct lcore_rx_queue { #define MAX_RX_QUEUE_INTERRUPT_PER_PORT 16 -#define MAX_LCORE_PARAMS 1024 -struct lcore_params { - uint8_t port_id; - uint8_t queue_id; - uint8_t lcore_id; -} __rte_cache_aligned; - -static struct lcore_params lcore_params_array[MAX_LCORE_PARAMS]; +struct lcore_params lcore_params_array[MAX_LCORE_PARAMS]; static struct lcore_params lcore_params_array_default[] = { {0, 0, 2}, {0, 1, 2}, @@ -209,20 +240,15 @@ static struct lcore_params lcore_params_array_default[] = { {3, 1, 3}, }; -static struct lcore_params * lcore_params = lcore_params_array_default; -static uint16_t nb_lcore_params = sizeof(lcore_params_array_default) / - sizeof(lcore_params_array_default[0]); +struct lcore_params *lcore_params = lcore_params_array_default; +uint16_t nb_lcore_params = RTE_DIM(lcore_params_array_default); static struct rte_eth_conf port_conf = { .rxmode = { .mq_mode = ETH_MQ_RX_RSS, - .max_rx_pkt_len = ETHER_MAX_LEN, + .max_rx_pkt_len = RTE_ETHER_MAX_LEN, .split_hdr_size = 0, - .header_split = 0, /**< Header Split disabled */ - .hw_ip_checksum = 1, /**< IP checksum offload enabled */ - .hw_vlan_filter = 0, /**< VLAN filtering disabled */ - .jumbo_frame = 0, /**< Jumbo Frame Support disabled */ - .hw_strip_crc = 0, /**< CRC stripped by hardware */ + .offloads = DEV_RX_OFFLOAD_CHECKSUM, }, .rx_adv_conf = { .rss_conf = { @@ -232,11 +258,7 @@ static struct rte_eth_conf port_conf = { }, .txmode = { .mq_mode = ETH_MQ_TX_NONE, - }, - .intr_conf = { - .lsc = 1, - .rxq = 1, - }, + } }; static struct rte_mempool * pktmbuf_pool[NB_SOCKETS]; @@ -244,7 +266,7 @@ static struct rte_mempool * pktmbuf_pool[NB_SOCKETS]; #if (APP_LOOKUP_METHOD == APP_LOOKUP_EXACT_MATCH) -#ifdef RTE_MACHINE_CPUFLAG_SSE4_2 +#ifdef RTE_ARCH_X86 #include #define DEFAULT_HASH_FUNC rte_hash_crc #else @@ -258,7 +280,7 @@ struct ipv4_5tuple { uint16_t port_dst; uint16_t port_src; uint8_t proto; -} __attribute__((__packed__)); +} __rte_packed; struct ipv6_5tuple { uint8_t ip_dst[IPV6_ADDR_LEN]; @@ -266,7 +288,7 @@ struct ipv6_5tuple { uint16_t port_dst; uint16_t port_src; uint8_t proto; -} __attribute__((__packed__)); +} __rte_packed; struct ipv4_l3fwd_route { struct ipv4_5tuple key; @@ -279,10 +301,10 @@ struct ipv6_l3fwd_route { }; static struct ipv4_l3fwd_route ipv4_l3fwd_route_array[] = { - {{IPv4(100,10,0,1), IPv4(200,10,0,1), 101, 11, IPPROTO_TCP}, 0}, - {{IPv4(100,20,0,2), IPv4(200,20,0,2), 102, 12, IPPROTO_TCP}, 1}, - {{IPv4(100,30,0,3), IPv4(200,30,0,3), 103, 13, IPPROTO_TCP}, 2}, - {{IPv4(100,40,0,4), IPv4(200,40,0,4), 104, 14, IPPROTO_TCP}, 3}, + {{RTE_IPV4(100,10,0,1), RTE_IPV4(200,10,0,1), 101, 11, IPPROTO_TCP}, 0}, + {{RTE_IPV4(100,20,0,2), RTE_IPV4(200,20,0,2), 102, 12, IPPROTO_TCP}, 1}, + {{RTE_IPV4(100,30,0,3), RTE_IPV4(200,30,0,3), 103, 13, IPPROTO_TCP}, 2}, + {{RTE_IPV4(100,40,0,4), RTE_IPV4(200,40,0,4), 104, 14, IPPROTO_TCP}, 3}, }; static struct ipv6_l3fwd_route ipv6_l3fwd_route_array[] = { @@ -303,14 +325,8 @@ static lookup_struct_t *ipv6_l3fwd_lookup_struct[NB_SOCKETS]; #define L3FWD_HASH_ENTRIES 1024 -#define IPV4_L3FWD_NUM_ROUTES \ - (sizeof(ipv4_l3fwd_route_array) / sizeof(ipv4_l3fwd_route_array[0])) - -#define IPV6_L3FWD_NUM_ROUTES \ - (sizeof(ipv6_l3fwd_route_array) / sizeof(ipv6_l3fwd_route_array[0])) - -static uint8_t ipv4_l3fwd_out_if[L3FWD_HASH_ENTRIES] __rte_cache_aligned; -static uint8_t ipv6_l3fwd_out_if[L3FWD_HASH_ENTRIES] __rte_cache_aligned; +static uint16_t ipv4_l3fwd_out_if[L3FWD_HASH_ENTRIES] __rte_cache_aligned; +static uint16_t ipv6_l3fwd_out_if[L3FWD_HASH_ENTRIES] __rte_cache_aligned; #endif #if (APP_LOOKUP_METHOD == APP_LOOKUP_LPM) @@ -321,19 +337,16 @@ struct ipv4_l3fwd_route { }; static struct ipv4_l3fwd_route ipv4_l3fwd_route_array[] = { - {IPv4(1,1,1,0), 24, 0}, - {IPv4(2,1,1,0), 24, 1}, - {IPv4(3,1,1,0), 24, 2}, - {IPv4(4,1,1,0), 24, 3}, - {IPv4(5,1,1,0), 24, 4}, - {IPv4(6,1,1,0), 24, 5}, - {IPv4(7,1,1,0), 24, 6}, - {IPv4(8,1,1,0), 24, 7}, + {RTE_IPV4(1,1,1,0), 24, 0}, + {RTE_IPV4(2,1,1,0), 24, 1}, + {RTE_IPV4(3,1,1,0), 24, 2}, + {RTE_IPV4(4,1,1,0), 24, 3}, + {RTE_IPV4(5,1,1,0), 24, 4}, + {RTE_IPV4(6,1,1,0), 24, 5}, + {RTE_IPV4(7,1,1,0), 24, 6}, + {RTE_IPV4(8,1,1,0), 24, 7}, }; -#define IPV4_L3FWD_NUM_ROUTES \ - (sizeof(ipv4_l3fwd_route_array) / sizeof(ipv4_l3fwd_route_array[0])) - #define IPV4_L3FWD_LPM_MAX_RULES 1024 typedef struct rte_lpm lookup_struct_t; @@ -362,7 +375,26 @@ struct lcore_stats { uint64_t nb_rx_processed; /* total iterations looped recently */ uint64_t nb_iteration_looped; - uint32_t padding[9]; + /* + * Represents empty and non empty polls + * of rte_eth_rx_burst(); + * ep_nep[0] holds non empty polls + * i.e. 0 < nb_rx <= MAX_BURST + * ep_nep[1] holds empty polls. + * i.e. nb_rx == 0 + */ + uint64_t ep_nep[2]; + /* + * Represents full and empty+partial + * polls of rte_eth_rx_burst(); + * ep_nep[0] holds empty+partial polls. + * i.e. 0 <= nb_rx < MAX_BURST + * ep_nep[1] holds full polls + * i.e. nb_rx == MAX_BURST + */ + uint64_t fp_nfp[2]; + enum busy_rate br; + rte_spinlock_t telemetry_lock; } __rte_cache_aligned; static struct lcore_conf lcore_conf[RTE_MAX_LCORE] __rte_cache_aligned; @@ -371,36 +403,35 @@ static struct rte_timer power_timers[RTE_MAX_LCORE]; static inline uint32_t power_idle_heuristic(uint32_t zero_rx_packet_count); static inline enum freq_scale_hint_t power_freq_scaleup_heuristic( \ - unsigned lcore_id, uint8_t port_id, uint16_t queue_id); + unsigned int lcore_id, uint16_t port_id, uint16_t queue_id); + + +/* + * These defaults are using the max frequency index (1), a medium index (9) + * and a typical low frequency index (14). These can be adjusted to use + * different indexes using the relevant command line parameters. + */ +static uint8_t freq_tlb[] = {14, 9, 1}; + +static int is_done(void) +{ + return quit_signal; +} /* exit signal handler */ static void signal_exit_now(int sigtype) { - unsigned lcore_id; - int ret; - - if (sigtype == SIGINT) { - for (lcore_id = 0; lcore_id < RTE_MAX_LCORE; lcore_id++) { - if (rte_lcore_is_enabled(lcore_id) == 0) - continue; - /* init power management library */ - ret = rte_power_exit(lcore_id); - if (ret) - rte_exit(EXIT_FAILURE, "Power management " - "library de-initialization failed on " - "core%u\n", lcore_id); - } - } + if (sigtype == SIGINT) + quit_signal = true; - rte_exit(EXIT_SUCCESS, "User forced exit\n"); } /* Freqency scale down timer callback */ static void -power_timer_cb(__attribute__((unused)) struct rte_timer *tim, - __attribute__((unused)) void *arg) +power_timer_cb(__rte_unused struct rte_timer *tim, + __rte_unused void *arg) { uint64_t hz; float sleep_time_ratio; @@ -442,7 +473,7 @@ power_timer_cb(__attribute__((unused)) struct rte_timer *tim, /* Enqueue a single packet, and send burst if queue is filled */ static inline int -send_single_packet(struct rte_mbuf *m, uint8_t port) +send_single_packet(struct rte_mbuf *m, uint16_t port) { uint32_t lcore_id; struct lcore_conf *qconf; @@ -458,14 +489,14 @@ send_single_packet(struct rte_mbuf *m, uint8_t port) #ifdef DO_RFC_1812_CHECKS static inline int -is_valid_ipv4_pkt(struct ipv4_hdr *pkt, uint32_t link_len) +is_valid_ipv4_pkt(struct rte_ipv4_hdr *pkt, uint32_t link_len) { /* From http://www.rfc-editor.org/rfc/rfc1812.txt section 5.2.2 */ /* * 1. The packet length reported by the Link Layer must be large * enough to hold the minimum length legal IP datagram (20 bytes). */ - if (link_len < sizeof(struct ipv4_hdr)) + if (link_len < sizeof(struct rte_ipv4_hdr)) return -1; /* 2. The IP checksum must be correct. */ @@ -490,7 +521,7 @@ is_valid_ipv4_pkt(struct ipv4_hdr *pkt, uint32_t link_len) * datagram header, whose length is specified in the IP header length * field. */ - if (rte_cpu_to_be_16(pkt->total_length) < sizeof(struct ipv4_hdr)) + if (rte_cpu_to_be_16(pkt->total_length) < sizeof(struct rte_ipv4_hdr)) return -5; return 0; @@ -514,13 +545,13 @@ print_ipv6_key(struct ipv6_5tuple key) key.port_dst, key.port_src, key.proto); } -static inline uint8_t -get_ipv4_dst_port(struct ipv4_hdr *ipv4_hdr, uint8_t portid, +static inline uint16_t +get_ipv4_dst_port(struct rte_ipv4_hdr *ipv4_hdr, uint16_t portid, lookup_struct_t * ipv4_l3fwd_lookup_struct) { struct ipv4_5tuple key; - struct tcp_hdr *tcp; - struct udp_hdr *udp; + struct rte_tcp_hdr *tcp; + struct rte_udp_hdr *udp; int ret = 0; key.ip_dst = rte_be_to_cpu_32(ipv4_hdr->dst_addr); @@ -529,15 +560,15 @@ get_ipv4_dst_port(struct ipv4_hdr *ipv4_hdr, uint8_t portid, switch (ipv4_hdr->next_proto_id) { case IPPROTO_TCP: - tcp = (struct tcp_hdr *)((unsigned char *)ipv4_hdr + - sizeof(struct ipv4_hdr)); + tcp = (struct rte_tcp_hdr *)((unsigned char *)ipv4_hdr + + sizeof(struct rte_ipv4_hdr)); key.port_dst = rte_be_to_cpu_16(tcp->dst_port); key.port_src = rte_be_to_cpu_16(tcp->src_port); break; case IPPROTO_UDP: - udp = (struct udp_hdr *)((unsigned char *)ipv4_hdr + - sizeof(struct ipv4_hdr)); + udp = (struct rte_udp_hdr *)((unsigned char *)ipv4_hdr + + sizeof(struct rte_ipv4_hdr)); key.port_dst = rte_be_to_cpu_16(udp->dst_port); key.port_src = rte_be_to_cpu_16(udp->src_port); break; @@ -550,16 +581,16 @@ get_ipv4_dst_port(struct ipv4_hdr *ipv4_hdr, uint8_t portid, /* Find destination port */ ret = rte_hash_lookup(ipv4_l3fwd_lookup_struct, (const void *)&key); - return (uint8_t)((ret < 0)? portid : ipv4_l3fwd_out_if[ret]); + return ((ret < 0) ? portid : ipv4_l3fwd_out_if[ret]); } -static inline uint8_t -get_ipv6_dst_port(struct ipv6_hdr *ipv6_hdr, uint8_t portid, +static inline uint16_t +get_ipv6_dst_port(struct rte_ipv6_hdr *ipv6_hdr, uint16_t portid, lookup_struct_t *ipv6_l3fwd_lookup_struct) { struct ipv6_5tuple key; - struct tcp_hdr *tcp; - struct udp_hdr *udp; + struct rte_tcp_hdr *tcp; + struct rte_udp_hdr *udp; int ret = 0; memcpy(key.ip_dst, ipv6_hdr->dst_addr, IPV6_ADDR_LEN); @@ -569,15 +600,15 @@ get_ipv6_dst_port(struct ipv6_hdr *ipv6_hdr, uint8_t portid, switch (ipv6_hdr->proto) { case IPPROTO_TCP: - tcp = (struct tcp_hdr *)((unsigned char *) ipv6_hdr + - sizeof(struct ipv6_hdr)); + tcp = (struct rte_tcp_hdr *)((unsigned char *) ipv6_hdr + + sizeof(struct rte_ipv6_hdr)); key.port_dst = rte_be_to_cpu_16(tcp->dst_port); key.port_src = rte_be_to_cpu_16(tcp->src_port); break; case IPPROTO_UDP: - udp = (struct udp_hdr *)((unsigned char *) ipv6_hdr + - sizeof(struct ipv6_hdr)); + udp = (struct rte_udp_hdr *)((unsigned char *) ipv6_hdr + + sizeof(struct rte_ipv6_hdr)); key.port_dst = rte_be_to_cpu_16(udp->dst_port); key.port_src = rte_be_to_cpu_16(udp->src_port); break; @@ -590,39 +621,81 @@ get_ipv6_dst_port(struct ipv6_hdr *ipv6_hdr, uint8_t portid, /* Find destination port */ ret = rte_hash_lookup(ipv6_l3fwd_lookup_struct, (const void *)&key); - return (uint8_t)((ret < 0)? portid : ipv6_l3fwd_out_if[ret]); + return ((ret < 0) ? portid : ipv6_l3fwd_out_if[ret]); } #endif #if (APP_LOOKUP_METHOD == APP_LOOKUP_LPM) -static inline uint8_t -get_ipv4_dst_port(struct ipv4_hdr *ipv4_hdr, uint8_t portid, +static inline uint16_t +get_ipv4_dst_port(struct rte_ipv4_hdr *ipv4_hdr, uint16_t portid, lookup_struct_t *ipv4_l3fwd_lookup_struct) { uint32_t next_hop; - return (uint8_t) ((rte_lpm_lookup(ipv4_l3fwd_lookup_struct, + return ((rte_lpm_lookup(ipv4_l3fwd_lookup_struct, rte_be_to_cpu_32(ipv4_hdr->dst_addr), &next_hop) == 0)? next_hop : portid); } #endif static inline void -l3fwd_simple_forward(struct rte_mbuf *m, uint8_t portid, +parse_ptype_one(struct rte_mbuf *m) +{ + struct rte_ether_hdr *eth_hdr; + uint32_t packet_type = RTE_PTYPE_UNKNOWN; + uint16_t ether_type; + + eth_hdr = rte_pktmbuf_mtod(m, struct rte_ether_hdr *); + ether_type = eth_hdr->ether_type; + if (ether_type == rte_cpu_to_be_16(RTE_ETHER_TYPE_IPV4)) + packet_type |= RTE_PTYPE_L3_IPV4_EXT_UNKNOWN; + else if (ether_type == rte_cpu_to_be_16(RTE_ETHER_TYPE_IPV6)) + packet_type |= RTE_PTYPE_L3_IPV6_EXT_UNKNOWN; + + m->packet_type = packet_type; +} + +static uint16_t +cb_parse_ptype(uint16_t port __rte_unused, uint16_t queue __rte_unused, + struct rte_mbuf *pkts[], uint16_t nb_pkts, + uint16_t max_pkts __rte_unused, + void *user_param __rte_unused) +{ + unsigned int i; + + for (i = 0; i < nb_pkts; ++i) + parse_ptype_one(pkts[i]); + + return nb_pkts; +} + +static int +add_cb_parse_ptype(uint16_t portid, uint16_t queueid) +{ + printf("Port %d: softly parse packet type info\n", portid); + if (rte_eth_add_rx_callback(portid, queueid, cb_parse_ptype, NULL)) + return 0; + + printf("Failed to add rx callback: port=%d\n", portid); + return -1; +} + +static inline void +l3fwd_simple_forward(struct rte_mbuf *m, uint16_t portid, struct lcore_conf *qconf) { - struct ether_hdr *eth_hdr; - struct ipv4_hdr *ipv4_hdr; + struct rte_ether_hdr *eth_hdr; + struct rte_ipv4_hdr *ipv4_hdr; void *d_addr_bytes; - uint8_t dst_port; + uint16_t dst_port; - eth_hdr = rte_pktmbuf_mtod(m, struct ether_hdr *); + eth_hdr = rte_pktmbuf_mtod(m, struct rte_ether_hdr *); if (RTE_ETH_IS_IPV4_HDR(m->packet_type)) { /* Handle IPv4 headers.*/ ipv4_hdr = - rte_pktmbuf_mtod_offset(m, struct ipv4_hdr *, - sizeof(struct ether_hdr)); + rte_pktmbuf_mtod_offset(m, struct rte_ipv4_hdr *, + sizeof(struct rte_ether_hdr)); #ifdef DO_RFC_1812_CHECKS /* Check to make sure the packet is valid (RFC1812) */ @@ -650,17 +723,18 @@ l3fwd_simple_forward(struct rte_mbuf *m, uint8_t portid, #endif /* src addr */ - ether_addr_copy(&ports_eth_addr[dst_port], ð_hdr->s_addr); + rte_ether_addr_copy(&ports_eth_addr[dst_port], + ð_hdr->s_addr); send_single_packet(m, dst_port); } else if (RTE_ETH_IS_IPV6_HDR(m->packet_type)) { /* Handle IPv6 headers.*/ #if (APP_LOOKUP_METHOD == APP_LOOKUP_EXACT_MATCH) - struct ipv6_hdr *ipv6_hdr; + struct rte_ipv6_hdr *ipv6_hdr; ipv6_hdr = - rte_pktmbuf_mtod_offset(m, struct ipv6_hdr *, - sizeof(struct ether_hdr)); + rte_pktmbuf_mtod_offset(m, struct rte_ipv6_hdr *, + sizeof(struct rte_ether_hdr)); dst_port = get_ipv6_dst_port(ipv6_hdr, portid, qconf->ipv6_lookup_struct); @@ -675,7 +749,8 @@ l3fwd_simple_forward(struct rte_mbuf *m, uint8_t portid, 0x000000000002 + ((uint64_t)dst_port << 40); /* src addr */ - ether_addr_copy(&ports_eth_addr[dst_port], ð_hdr->s_addr); + rte_ether_addr_copy(&ports_eth_addr[dst_port], + ð_hdr->s_addr); send_single_packet(m, dst_port); #else @@ -701,15 +776,14 @@ power_idle_heuristic(uint32_t zero_rx_packet_count) */ else return SUSPEND_THRESHOLD; - - return 0; } static inline enum freq_scale_hint_t power_freq_scaleup_heuristic(unsigned lcore_id, - uint8_t port_id, + uint16_t port_id, uint16_t queue_id) { + uint32_t rxq_count = rte_eth_rx_queue_count(port_id, queue_id); /** * HW Rx queue size is 128 by default, Rx burst read at maximum 32 entries * per iteration @@ -721,15 +795,12 @@ power_freq_scaleup_heuristic(unsigned lcore_id, #define FREQ_UP_TREND2_ACC 100 #define FREQ_UP_THRESHOLD 10000 - if (likely(rte_eth_rx_descriptor_done(port_id, queue_id, - FREQ_GEAR3_RX_PACKET_THRESHOLD) > 0)) { + if (likely(rxq_count > FREQ_GEAR3_RX_PACKET_THRESHOLD)) { stats[lcore_id].trend = 0; return FREQ_HIGHEST; - } else if (likely(rte_eth_rx_descriptor_done(port_id, queue_id, - FREQ_GEAR2_RX_PACKET_THRESHOLD) > 0)) + } else if (likely(rxq_count > FREQ_GEAR2_RX_PACKET_THRESHOLD)) stats[lcore_id].trend += FREQ_UP_TREND2_ACC; - else if (likely(rte_eth_rx_descriptor_done(port_id, queue_id, - FREQ_GEAR1_RX_PACKET_THRESHOLD) > 0)) + else if (likely(rxq_count > FREQ_GEAR1_RX_PACKET_THRESHOLD)) stats[lcore_id].trend += FREQ_UP_TREND1_ACC; if (likely(stats[lcore_id].trend > FREQ_UP_THRESHOLD)) { @@ -752,36 +823,45 @@ power_freq_scaleup_heuristic(unsigned lcore_id, static int sleep_until_rx_interrupt(int num) { + /* + * we want to track when we are woken up by traffic so that we can go + * back to sleep again without log spamming. + */ + static bool timeout; struct rte_epoll_event event[num]; int n, i; - uint8_t port_id, queue_id; + uint16_t port_id; + uint8_t queue_id; void *data; - RTE_LOG(INFO, L3FWD_POWER, - "lcore %u sleeps until interrupt triggers\n", - rte_lcore_id()); + if (!timeout) { + RTE_LOG(INFO, L3FWD_POWER, + "lcore %u sleeps until interrupt triggers\n", + rte_lcore_id()); + } - n = rte_epoll_wait(RTE_EPOLL_PER_THREAD, event, num, -1); + n = rte_epoll_wait(RTE_EPOLL_PER_THREAD, event, num, 10); for (i = 0; i < n; i++) { data = event[i].epdata.data; port_id = ((uintptr_t)data) >> CHAR_BIT; queue_id = ((uintptr_t)data) & RTE_LEN2MASK(CHAR_BIT, uint8_t); - rte_eth_dev_rx_intr_disable(port_id, queue_id); RTE_LOG(INFO, L3FWD_POWER, "lcore %u is waked up from rx interrupt on" " port %d queue %d\n", rte_lcore_id(), port_id, queue_id); } + timeout = n == 0; return 0; } -static void turn_on_intr(struct lcore_conf *qconf) +static void turn_on_off_intr(struct lcore_conf *qconf, bool on) { int i; struct lcore_rx_queue *rx_queue; - uint8_t port_id, queue_id; + uint8_t queue_id; + uint16_t port_id; for (i = 0; i < qconf->n_rx_queue; ++i) { rx_queue = &(qconf->rx_queue_list[i]); @@ -789,7 +869,10 @@ static void turn_on_intr(struct lcore_conf *qconf) queue_id = rx_queue->queue_id; rte_spinlock_lock(&(locks[port_id])); - rte_eth_dev_rx_intr_enable(port_id, queue_id); + if (on) + rte_eth_dev_rx_intr_enable(port_id, queue_id); + else + rte_eth_dev_rx_intr_disable(port_id, queue_id); rte_spinlock_unlock(&(locks[port_id])); } } @@ -797,7 +880,8 @@ static void turn_on_intr(struct lcore_conf *qconf) static int event_register(struct lcore_conf *qconf) { struct lcore_rx_queue *rx_queue; - uint8_t portid, queueid; + uint8_t queueid; + uint16_t portid; uint32_t data; int ret; int i; @@ -820,23 +904,22 @@ static int event_register(struct lcore_conf *qconf) } /* main processing loop */ -static int -main_loop(__attribute__((unused)) void *dummy) +static int main_intr_loop(__rte_unused void *dummy) { struct rte_mbuf *pkts_burst[MAX_PKT_BURST]; - unsigned lcore_id; + unsigned int lcore_id; uint64_t prev_tsc, diff_tsc, cur_tsc; - uint64_t prev_tsc_power = 0, cur_tsc_power, diff_tsc_power; int i, j, nb_rx; - uint8_t portid, queueid; + uint8_t queueid; + uint16_t portid; struct lcore_conf *qconf; struct lcore_rx_queue *rx_queue; - enum freq_scale_hint_t lcore_scaleup_hint; uint32_t lcore_rx_idle_count = 0; uint32_t lcore_idle_hint = 0; int intr_en = 0; - const uint64_t drain_tsc = (rte_get_tsc_hz() + US_PER_S - 1) / US_PER_S * BURST_TX_DRAIN_US; + const uint64_t drain_tsc = (rte_get_tsc_hz() + US_PER_S - 1) / + US_PER_S * BURST_TX_DRAIN_US; prev_tsc = 0; @@ -844,17 +927,20 @@ main_loop(__attribute__((unused)) void *dummy) qconf = &lcore_conf[lcore_id]; if (qconf->n_rx_queue == 0) { - RTE_LOG(INFO, L3FWD_POWER, "lcore %u has nothing to do\n", lcore_id); + RTE_LOG(INFO, L3FWD_POWER, "lcore %u has nothing to do\n", + lcore_id); return 0; } - RTE_LOG(INFO, L3FWD_POWER, "entering main loop on lcore %u\n", lcore_id); + RTE_LOG(INFO, L3FWD_POWER, "entering main interrupt loop on lcore %u\n", + lcore_id); for (i = 0; i < qconf->n_rx_queue; i++) { portid = qconf->rx_queue_list[i].port_id; queueid = qconf->rx_queue_list[i].queue_id; - RTE_LOG(INFO, L3FWD_POWER, " -- lcoreid=%u portid=%hhu " - "rxqueueid=%hhu\n", lcore_id, portid, queueid); + RTE_LOG(INFO, L3FWD_POWER, + " -- lcoreid=%u portid=%u rxqueueid=%hhu\n", + lcore_id, portid, queueid); } /* add into event wait list */ @@ -863,11 +949,10 @@ main_loop(__attribute__((unused)) void *dummy) else RTE_LOG(INFO, L3FWD_POWER, "RX interrupt won't enable.\n"); - while (1) { + while (!is_done()) { stats[lcore_id].nb_iteration_looped++; cur_tsc = rte_rdtsc(); - cur_tsc_power = cur_tsc; /* * TX burst queue drain @@ -883,17 +968,10 @@ main_loop(__attribute__((unused)) void *dummy) prev_tsc = cur_tsc; } - diff_tsc_power = cur_tsc_power - prev_tsc_power; - if (diff_tsc_power > TIMER_RESOLUTION_CYCLES) { - rte_timer_manage(); - prev_tsc_power = cur_tsc_power; - } - start_rx: /* * Read packet from RX queues */ - lcore_scaleup_hint = FREQ_CURRENT; lcore_rx_idle_count = 0; for (i = 0; i < qconf->n_rx_queue; ++i) { rx_queue = &(qconf->rx_queue_list[i]); @@ -902,7 +980,7 @@ start_rx: queueid = rx_queue->queue_id; nb_rx = rte_eth_rx_burst(portid, queueid, pkts_burst, - MAX_PKT_BURST); + MAX_PKT_BURST); stats[lcore_id].nb_rx_processed += nb_rx; if (unlikely(nb_rx == 0)) { @@ -914,24 +992,14 @@ start_rx: rx_queue->zero_rx_packet_count++; if (rx_queue->zero_rx_packet_count <= - MIN_ZERO_POLL_COUNT) + MIN_ZERO_POLL_COUNT) continue; - rx_queue->idle_hint = power_idle_heuristic(\ - rx_queue->zero_rx_packet_count); + rx_queue->idle_hint = power_idle_heuristic( + rx_queue->zero_rx_packet_count); lcore_rx_idle_count++; } else { rx_queue->zero_rx_packet_count = 0; - - /** - * do not scale up frequency immediately as - * user to kernel space communication is costly - * which might impact packet I/O for received - * packets. - */ - rx_queue->freq_up_hint = - power_freq_scaleup_heuristic(lcore_id, - portid, queueid); } /* Prefetch first packets */ @@ -942,45 +1010,28 @@ start_rx: /* Prefetch and forward already prefetched packets */ for (j = 0; j < (nb_rx - PREFETCH_OFFSET); j++) { - rte_prefetch0(rte_pktmbuf_mtod(pkts_burst[ - j + PREFETCH_OFFSET], void *)); - l3fwd_simple_forward(pkts_burst[j], portid, - qconf); + rte_prefetch0(rte_pktmbuf_mtod( + pkts_burst[j + PREFETCH_OFFSET], + void *)); + l3fwd_simple_forward( + pkts_burst[j], portid, qconf); } /* Forward remaining prefetched packets */ for (; j < nb_rx; j++) { - l3fwd_simple_forward(pkts_burst[j], portid, - qconf); + l3fwd_simple_forward( + pkts_burst[j], portid, qconf); } } - if (likely(lcore_rx_idle_count != qconf->n_rx_queue)) { - for (i = 1, lcore_scaleup_hint = - qconf->rx_queue_list[0].freq_up_hint; - i < qconf->n_rx_queue; ++i) { - rx_queue = &(qconf->rx_queue_list[i]); - if (rx_queue->freq_up_hint > - lcore_scaleup_hint) - lcore_scaleup_hint = - rx_queue->freq_up_hint; - } - - if (lcore_scaleup_hint == FREQ_HIGHEST) { - if (rte_power_freq_max) - rte_power_freq_max(lcore_id); - } else if (lcore_scaleup_hint == FREQ_HIGHER) { - if (rte_power_freq_up) - rte_power_freq_up(lcore_id); - } - } else { + if (unlikely(lcore_rx_idle_count == qconf->n_rx_queue)) { /** * All Rx queues empty in recent consecutive polls, * sleep in a conservative manner, meaning sleep as * less as possible. */ - for (i = 1, lcore_idle_hint = - qconf->rx_queue_list[0].idle_hint; + for (i = 1, + lcore_idle_hint = qconf->rx_queue_list[0].idle_hint; i < qconf->n_rx_queue; ++i) { rx_queue = &(qconf->rx_queue_list[i]); if (rx_queue->idle_hint < lcore_idle_hint) @@ -995,133 +1046,579 @@ start_rx: */ rte_delay_us(lcore_idle_hint); else { - /* suspend until rx interrupt trigges */ + /* suspend until rx interrupt triggers */ if (intr_en) { - turn_on_intr(qconf); + turn_on_off_intr(qconf, 1); sleep_until_rx_interrupt( - qconf->n_rx_queue); + qconf->n_rx_queue); + turn_on_off_intr(qconf, 0); + /** + * start receiving packets immediately + */ + if (likely(!is_done())) + goto start_rx; } - /* start receiving packets immediately */ - goto start_rx; } stats[lcore_id].sleep_time += lcore_idle_hint; } } -} - -static int -check_lcore_params(void) -{ - uint8_t queue, lcore; - uint16_t i; - int socketid; - for (i = 0; i < nb_lcore_params; ++i) { - queue = lcore_params[i].queue_id; - if (queue >= MAX_RX_QUEUE_PER_PORT) { - printf("invalid queue number: %hhu\n", queue); - return -1; - } - lcore = lcore_params[i].lcore_id; - if (!rte_lcore_is_enabled(lcore)) { - printf("error: lcore %hhu is not enabled in lcore " - "mask\n", lcore); - return -1; - } - if ((socketid = rte_lcore_to_socket_id(lcore) != 0) && - (numa_on == 0)) { - printf("warning: lcore %hhu is on socket %d with numa " - "off\n", lcore, socketid); - } - } return 0; } +/* main processing loop */ static int -check_port_config(const unsigned nb_ports) +main_telemetry_loop(__rte_unused void *dummy) { - unsigned portid; - uint16_t i; + struct rte_mbuf *pkts_burst[MAX_PKT_BURST]; + unsigned int lcore_id; + uint64_t prev_tsc, diff_tsc, cur_tsc, prev_tel_tsc; + int i, j, nb_rx; + uint8_t queueid; + uint16_t portid; + struct lcore_conf *qconf; + struct lcore_rx_queue *rx_queue; + uint64_t ep_nep[2] = {0}, fp_nfp[2] = {0}; + uint64_t poll_count; + enum busy_rate br; - for (i = 0; i < nb_lcore_params; ++i) { - portid = lcore_params[i].port_id; - if ((enabled_port_mask & (1 << portid)) == 0) { - printf("port %u is not enabled in port mask\n", - portid); - return -1; - } - if (portid >= nb_ports) { - printf("port %u is not present on the board\n", - portid); - return -1; - } - } - return 0; -} + const uint64_t drain_tsc = (rte_get_tsc_hz() + US_PER_S - 1) / + US_PER_S * BURST_TX_DRAIN_US; -static uint8_t -get_port_n_rx_queues(const uint8_t port) -{ - int queue = -1; - uint16_t i; + poll_count = 0; + prev_tsc = 0; + prev_tel_tsc = 0; - for (i = 0; i < nb_lcore_params; ++i) { - if (lcore_params[i].port_id == port && - lcore_params[i].queue_id > queue) - queue = lcore_params[i].queue_id; + lcore_id = rte_lcore_id(); + qconf = &lcore_conf[lcore_id]; + + if (qconf->n_rx_queue == 0) { + RTE_LOG(INFO, L3FWD_POWER, "lcore %u has nothing to do\n", + lcore_id); + return 0; } - return (uint8_t)(++queue); -} -static int -init_lcore_rx_queues(void) -{ - uint16_t i, nb_rx_queue; - uint8_t lcore; + RTE_LOG(INFO, L3FWD_POWER, "entering main telemetry loop on lcore %u\n", + lcore_id); - for (i = 0; i < nb_lcore_params; ++i) { - lcore = lcore_params[i].lcore_id; - nb_rx_queue = lcore_conf[lcore].n_rx_queue; - if (nb_rx_queue >= MAX_RX_QUEUE_PER_LCORE) { - printf("error: too many queues (%u) for lcore: %u\n", - (unsigned)nb_rx_queue + 1, (unsigned)lcore); - return -1; - } else { - lcore_conf[lcore].rx_queue_list[nb_rx_queue].port_id = - lcore_params[i].port_id; - lcore_conf[lcore].rx_queue_list[nb_rx_queue].queue_id = - lcore_params[i].queue_id; - lcore_conf[lcore].n_rx_queue++; - } + for (i = 0; i < qconf->n_rx_queue; i++) { + portid = qconf->rx_queue_list[i].port_id; + queueid = qconf->rx_queue_list[i].queue_id; + RTE_LOG(INFO, L3FWD_POWER, " -- lcoreid=%u portid=%u " + "rxqueueid=%hhu\n", lcore_id, portid, queueid); } - return 0; -} -/* display usage */ -static void -print_usage(const char *prgname) -{ - printf ("%s [EAL options] -- -p PORTMASK -P" - " [--config (port,queue,lcore)[,(port,queue,lcore]]" - " [--enable-jumbo [--max-pkt-len PKTLEN]]\n" - " -p PORTMASK: hexadecimal bitmask of ports to configure\n" - " -P : enable promiscuous mode\n" - " --config (port,queue,lcore): rx queues configuration\n" - " --no-numa: optional, disable numa awareness\n" - " --enable-jumbo: enable jumbo frame" - " which max packet len is PKTLEN in decimal (64-9600)\n", - prgname); -} + while (!is_done()) { -static int parse_max_pkt_len(const char *pktlen) -{ - char *end = NULL; - unsigned long len; + cur_tsc = rte_rdtsc(); + /* + * TX burst queue drain + */ + diff_tsc = cur_tsc - prev_tsc; + if (unlikely(diff_tsc > drain_tsc)) { + for (i = 0; i < qconf->n_tx_port; ++i) { + portid = qconf->tx_port_id[i]; + rte_eth_tx_buffer_flush(portid, + qconf->tx_queue_id[portid], + qconf->tx_buffer[portid]); + } + prev_tsc = cur_tsc; + } - /* parse decimal string */ - len = strtoul(pktlen, &end, 10); - if ((pktlen[0] == '\0') || (end == NULL) || (*end != '\0')) - return -1; + /* + * Read packet from RX queues + */ + for (i = 0; i < qconf->n_rx_queue; ++i) { + rx_queue = &(qconf->rx_queue_list[i]); + portid = rx_queue->port_id; + queueid = rx_queue->queue_id; + + nb_rx = rte_eth_rx_burst(portid, queueid, pkts_burst, + MAX_PKT_BURST); + ep_nep[nb_rx == 0]++; + fp_nfp[nb_rx == MAX_PKT_BURST]++; + poll_count++; + if (unlikely(nb_rx == 0)) + continue; + + /* Prefetch first packets */ + for (j = 0; j < PREFETCH_OFFSET && j < nb_rx; j++) { + rte_prefetch0(rte_pktmbuf_mtod( + pkts_burst[j], void *)); + } + + /* Prefetch and forward already prefetched packets */ + for (j = 0; j < (nb_rx - PREFETCH_OFFSET); j++) { + rte_prefetch0(rte_pktmbuf_mtod(pkts_burst[ + j + PREFETCH_OFFSET], void *)); + l3fwd_simple_forward(pkts_burst[j], portid, + qconf); + } + + /* Forward remaining prefetched packets */ + for (; j < nb_rx; j++) { + l3fwd_simple_forward(pkts_burst[j], portid, + qconf); + } + } + if (unlikely(poll_count >= DEFAULT_COUNT)) { + diff_tsc = cur_tsc - prev_tel_tsc; + if (diff_tsc >= MAX_CYCLES) { + br = FULL; + } else if (diff_tsc > MIN_CYCLES && + diff_tsc < MAX_CYCLES) { + br = (diff_tsc * 100) / MAX_CYCLES; + } else { + br = ZERO; + } + poll_count = 0; + prev_tel_tsc = cur_tsc; + /* update stats for telemetry */ + rte_spinlock_lock(&stats[lcore_id].telemetry_lock); + stats[lcore_id].ep_nep[0] = ep_nep[0]; + stats[lcore_id].ep_nep[1] = ep_nep[1]; + stats[lcore_id].fp_nfp[0] = fp_nfp[0]; + stats[lcore_id].fp_nfp[1] = fp_nfp[1]; + stats[lcore_id].br = br; + rte_spinlock_unlock(&stats[lcore_id].telemetry_lock); + } + } + + return 0; +} +/* main processing loop */ +static int +main_empty_poll_loop(__rte_unused void *dummy) +{ + struct rte_mbuf *pkts_burst[MAX_PKT_BURST]; + unsigned int lcore_id; + uint64_t prev_tsc, diff_tsc, cur_tsc; + int i, j, nb_rx; + uint8_t queueid; + uint16_t portid; + struct lcore_conf *qconf; + struct lcore_rx_queue *rx_queue; + + const uint64_t drain_tsc = + (rte_get_tsc_hz() + US_PER_S - 1) / + US_PER_S * BURST_TX_DRAIN_US; + + prev_tsc = 0; + + lcore_id = rte_lcore_id(); + qconf = &lcore_conf[lcore_id]; + + if (qconf->n_rx_queue == 0) { + RTE_LOG(INFO, L3FWD_POWER, "lcore %u has nothing to do\n", + lcore_id); + return 0; + } + + for (i = 0; i < qconf->n_rx_queue; i++) { + portid = qconf->rx_queue_list[i].port_id; + queueid = qconf->rx_queue_list[i].queue_id; + RTE_LOG(INFO, L3FWD_POWER, " -- lcoreid=%u portid=%u " + "rxqueueid=%hhu\n", lcore_id, portid, queueid); + } + + while (!is_done()) { + stats[lcore_id].nb_iteration_looped++; + + cur_tsc = rte_rdtsc(); + /* + * TX burst queue drain + */ + diff_tsc = cur_tsc - prev_tsc; + if (unlikely(diff_tsc > drain_tsc)) { + for (i = 0; i < qconf->n_tx_port; ++i) { + portid = qconf->tx_port_id[i]; + rte_eth_tx_buffer_flush(portid, + qconf->tx_queue_id[portid], + qconf->tx_buffer[portid]); + } + prev_tsc = cur_tsc; + } + + /* + * Read packet from RX queues + */ + for (i = 0; i < qconf->n_rx_queue; ++i) { + rx_queue = &(qconf->rx_queue_list[i]); + rx_queue->idle_hint = 0; + portid = rx_queue->port_id; + queueid = rx_queue->queue_id; + + nb_rx = rte_eth_rx_burst(portid, queueid, pkts_burst, + MAX_PKT_BURST); + + stats[lcore_id].nb_rx_processed += nb_rx; + + if (nb_rx == 0) { + + rte_power_empty_poll_stat_update(lcore_id); + + continue; + } else { + rte_power_poll_stat_update(lcore_id, nb_rx); + } + + + /* Prefetch first packets */ + for (j = 0; j < PREFETCH_OFFSET && j < nb_rx; j++) { + rte_prefetch0(rte_pktmbuf_mtod( + pkts_burst[j], void *)); + } + + /* Prefetch and forward already prefetched packets */ + for (j = 0; j < (nb_rx - PREFETCH_OFFSET); j++) { + rte_prefetch0(rte_pktmbuf_mtod(pkts_burst[ + j + PREFETCH_OFFSET], + void *)); + l3fwd_simple_forward(pkts_burst[j], portid, + qconf); + } + + /* Forward remaining prefetched packets */ + for (; j < nb_rx; j++) { + l3fwd_simple_forward(pkts_burst[j], portid, + qconf); + } + + } + + } + + return 0; +} +/* main processing loop */ +static int +main_legacy_loop(__rte_unused void *dummy) +{ + struct rte_mbuf *pkts_burst[MAX_PKT_BURST]; + unsigned lcore_id; + uint64_t prev_tsc, diff_tsc, cur_tsc, tim_res_tsc, hz; + uint64_t prev_tsc_power = 0, cur_tsc_power, diff_tsc_power; + int i, j, nb_rx; + uint8_t queueid; + uint16_t portid; + struct lcore_conf *qconf; + struct lcore_rx_queue *rx_queue; + enum freq_scale_hint_t lcore_scaleup_hint; + uint32_t lcore_rx_idle_count = 0; + uint32_t lcore_idle_hint = 0; + int intr_en = 0; + + const uint64_t drain_tsc = (rte_get_tsc_hz() + US_PER_S - 1) / US_PER_S * BURST_TX_DRAIN_US; + + prev_tsc = 0; + hz = rte_get_timer_hz(); + tim_res_tsc = hz/TIMER_NUMBER_PER_SECOND; + + lcore_id = rte_lcore_id(); + qconf = &lcore_conf[lcore_id]; + + if (qconf->n_rx_queue == 0) { + RTE_LOG(INFO, L3FWD_POWER, "lcore %u has nothing to do\n", lcore_id); + return 0; + } + + RTE_LOG(INFO, L3FWD_POWER, "entering main loop on lcore %u\n", lcore_id); + + for (i = 0; i < qconf->n_rx_queue; i++) { + portid = qconf->rx_queue_list[i].port_id; + queueid = qconf->rx_queue_list[i].queue_id; + RTE_LOG(INFO, L3FWD_POWER, " -- lcoreid=%u portid=%u " + "rxqueueid=%hhu\n", lcore_id, portid, queueid); + } + + /* add into event wait list */ + if (event_register(qconf) == 0) + intr_en = 1; + else + RTE_LOG(INFO, L3FWD_POWER, "RX interrupt won't enable.\n"); + + while (!is_done()) { + stats[lcore_id].nb_iteration_looped++; + + cur_tsc = rte_rdtsc(); + cur_tsc_power = cur_tsc; + + /* + * TX burst queue drain + */ + diff_tsc = cur_tsc - prev_tsc; + if (unlikely(diff_tsc > drain_tsc)) { + for (i = 0; i < qconf->n_tx_port; ++i) { + portid = qconf->tx_port_id[i]; + rte_eth_tx_buffer_flush(portid, + qconf->tx_queue_id[portid], + qconf->tx_buffer[portid]); + } + prev_tsc = cur_tsc; + } + + diff_tsc_power = cur_tsc_power - prev_tsc_power; + if (diff_tsc_power > tim_res_tsc) { + rte_timer_manage(); + prev_tsc_power = cur_tsc_power; + } + +start_rx: + /* + * Read packet from RX queues + */ + lcore_scaleup_hint = FREQ_CURRENT; + lcore_rx_idle_count = 0; + for (i = 0; i < qconf->n_rx_queue; ++i) { + rx_queue = &(qconf->rx_queue_list[i]); + rx_queue->idle_hint = 0; + portid = rx_queue->port_id; + queueid = rx_queue->queue_id; + + nb_rx = rte_eth_rx_burst(portid, queueid, pkts_burst, + MAX_PKT_BURST); + + stats[lcore_id].nb_rx_processed += nb_rx; + if (unlikely(nb_rx == 0)) { + /** + * no packet received from rx queue, try to + * sleep for a while forcing CPU enter deeper + * C states. + */ + rx_queue->zero_rx_packet_count++; + + if (rx_queue->zero_rx_packet_count <= + MIN_ZERO_POLL_COUNT) + continue; + + rx_queue->idle_hint = power_idle_heuristic(\ + rx_queue->zero_rx_packet_count); + lcore_rx_idle_count++; + } else { + rx_queue->zero_rx_packet_count = 0; + + /** + * do not scale up frequency immediately as + * user to kernel space communication is costly + * which might impact packet I/O for received + * packets. + */ + rx_queue->freq_up_hint = + power_freq_scaleup_heuristic(lcore_id, + portid, queueid); + } + + /* Prefetch first packets */ + for (j = 0; j < PREFETCH_OFFSET && j < nb_rx; j++) { + rte_prefetch0(rte_pktmbuf_mtod( + pkts_burst[j], void *)); + } + + /* Prefetch and forward already prefetched packets */ + for (j = 0; j < (nb_rx - PREFETCH_OFFSET); j++) { + rte_prefetch0(rte_pktmbuf_mtod(pkts_burst[ + j + PREFETCH_OFFSET], void *)); + l3fwd_simple_forward(pkts_burst[j], portid, + qconf); + } + + /* Forward remaining prefetched packets */ + for (; j < nb_rx; j++) { + l3fwd_simple_forward(pkts_burst[j], portid, + qconf); + } + } + + if (likely(lcore_rx_idle_count != qconf->n_rx_queue)) { + for (i = 1, lcore_scaleup_hint = + qconf->rx_queue_list[0].freq_up_hint; + i < qconf->n_rx_queue; ++i) { + rx_queue = &(qconf->rx_queue_list[i]); + if (rx_queue->freq_up_hint > + lcore_scaleup_hint) + lcore_scaleup_hint = + rx_queue->freq_up_hint; + } + + if (lcore_scaleup_hint == FREQ_HIGHEST) { + if (rte_power_freq_max) + rte_power_freq_max(lcore_id); + } else if (lcore_scaleup_hint == FREQ_HIGHER) { + if (rte_power_freq_up) + rte_power_freq_up(lcore_id); + } + } else { + /** + * All Rx queues empty in recent consecutive polls, + * sleep in a conservative manner, meaning sleep as + * less as possible. + */ + for (i = 1, lcore_idle_hint = + qconf->rx_queue_list[0].idle_hint; + i < qconf->n_rx_queue; ++i) { + rx_queue = &(qconf->rx_queue_list[i]); + if (rx_queue->idle_hint < lcore_idle_hint) + lcore_idle_hint = rx_queue->idle_hint; + } + + if (lcore_idle_hint < SUSPEND_THRESHOLD) + /** + * execute "pause" instruction to avoid context + * switch which generally take hundred of + * microseconds for short sleep. + */ + rte_delay_us(lcore_idle_hint); + else { + /* suspend until rx interrupt triggers */ + if (intr_en) { + turn_on_off_intr(qconf, 1); + sleep_until_rx_interrupt( + qconf->n_rx_queue); + turn_on_off_intr(qconf, 0); + /** + * start receiving packets immediately + */ + if (likely(!is_done())) + goto start_rx; + } + } + stats[lcore_id].sleep_time += lcore_idle_hint; + } + } + + return 0; +} + +static int +check_lcore_params(void) +{ + uint8_t queue, lcore; + uint16_t i; + int socketid; + + for (i = 0; i < nb_lcore_params; ++i) { + queue = lcore_params[i].queue_id; + if (queue >= MAX_RX_QUEUE_PER_PORT) { + printf("invalid queue number: %hhu\n", queue); + return -1; + } + lcore = lcore_params[i].lcore_id; + if (!rte_lcore_is_enabled(lcore)) { + printf("error: lcore %hhu is not enabled in lcore " + "mask\n", lcore); + return -1; + } + if ((socketid = rte_lcore_to_socket_id(lcore) != 0) && + (numa_on == 0)) { + printf("warning: lcore %hhu is on socket %d with numa " + "off\n", lcore, socketid); + } + if (app_mode == APP_MODE_TELEMETRY && lcore == rte_lcore_id()) { + printf("cannot enable master core %d in config for telemetry mode\n", + rte_lcore_id()); + return -1; + } + } + return 0; +} + +static int +check_port_config(void) +{ + unsigned portid; + uint16_t i; + + for (i = 0; i < nb_lcore_params; ++i) { + portid = lcore_params[i].port_id; + if ((enabled_port_mask & (1 << portid)) == 0) { + printf("port %u is not enabled in port mask\n", + portid); + return -1; + } + if (!rte_eth_dev_is_valid_port(portid)) { + printf("port %u is not present on the board\n", + portid); + return -1; + } + } + return 0; +} + +static uint8_t +get_port_n_rx_queues(const uint16_t port) +{ + int queue = -1; + uint16_t i; + + for (i = 0; i < nb_lcore_params; ++i) { + if (lcore_params[i].port_id == port && + lcore_params[i].queue_id > queue) + queue = lcore_params[i].queue_id; + } + return (uint8_t)(++queue); +} + +static int +init_lcore_rx_queues(void) +{ + uint16_t i, nb_rx_queue; + uint8_t lcore; + + for (i = 0; i < nb_lcore_params; ++i) { + lcore = lcore_params[i].lcore_id; + nb_rx_queue = lcore_conf[lcore].n_rx_queue; + if (nb_rx_queue >= MAX_RX_QUEUE_PER_LCORE) { + printf("error: too many queues (%u) for lcore: %u\n", + (unsigned)nb_rx_queue + 1, (unsigned)lcore); + return -1; + } else { + lcore_conf[lcore].rx_queue_list[nb_rx_queue].port_id = + lcore_params[i].port_id; + lcore_conf[lcore].rx_queue_list[nb_rx_queue].queue_id = + lcore_params[i].queue_id; + lcore_conf[lcore].n_rx_queue++; + } + } + return 0; +} + +/* display usage */ +static void +print_usage(const char *prgname) +{ + printf ("%s [EAL options] -- -p PORTMASK -P" + " [--config (port,queue,lcore)[,(port,queue,lcore]]" + " [--high-perf-cores CORELIST" + " [--perf-config (port,queue,hi_perf,lcore_index)[,(port,queue,hi_perf,lcore_index]]" + " [--enable-jumbo [--max-pkt-len PKTLEN]]\n" + " -p PORTMASK: hexadecimal bitmask of ports to configure\n" + " -P : enable promiscuous mode\n" + " --config (port,queue,lcore): rx queues configuration\n" + " --high-perf-cores CORELIST: list of high performance cores\n" + " --perf-config: similar as config, cores specified as indices" + " for bins containing high or regular performance cores\n" + " --no-numa: optional, disable numa awareness\n" + " --enable-jumbo: enable jumbo frame" + " which max packet len is PKTLEN in decimal (64-9600)\n" + " --parse-ptype: parse packet type by software\n" + " --legacy: use legacy interrupt-based scaling\n" + " --empty-poll: enable empty poll detection" + " follow (training_flag, high_threshold, med_threshold)\n" + " --telemetry: enable telemetry mode, to update" + " empty polls, full polls, and core busyness to telemetry\n" + " --interrupt-only: enable interrupt-only mode\n", + prgname); +} + +static int parse_max_pkt_len(const char *pktlen) +{ + char *end = NULL; + unsigned long len; + + /* parse decimal string */ + len = strtoul(pktlen, &end, 10); + if ((pktlen[0] == '\0') || (end == NULL) || (*end != '\0')) + return -1; if (len == 0) return -1; @@ -1138,10 +1635,7 @@ parse_portmask(const char *portmask) /* parse hexadecimal string */ pm = strtoul(portmask, &end, 16); if ((portmask[0] == '\0') || (end == NULL) || (*end != '\0')) - return -1; - - if (pm == 0) - return -1; + return 0; return pm; } @@ -1202,6 +1696,60 @@ parse_config(const char *q_arg) return 0; } +static int +parse_ep_config(const char *q_arg) +{ + char s[256]; + const char *p = q_arg; + char *end; + int num_arg; + + char *str_fld[3]; + + int training_flag; + int med_edpi; + int hgh_edpi; + + ep_med_edpi = EMPTY_POLL_MED_THRESHOLD; + ep_hgh_edpi = EMPTY_POLL_MED_THRESHOLD; + + strlcpy(s, p, sizeof(s)); + + num_arg = rte_strsplit(s, sizeof(s), str_fld, 3, ','); + + empty_poll_train = false; + + if (num_arg == 0) + return 0; + + if (num_arg == 3) { + + training_flag = strtoul(str_fld[0], &end, 0); + med_edpi = strtoul(str_fld[1], &end, 0); + hgh_edpi = strtoul(str_fld[2], &end, 0); + + if (training_flag == 1) + empty_poll_train = true; + + if (med_edpi > 0) + ep_med_edpi = med_edpi; + + if (med_edpi > 0) + ep_hgh_edpi = hgh_edpi; + + } else { + + return -1; + } + + return 0; + +} +#define CMD_LINE_OPT_PARSE_PTYPE "parse-ptype" +#define CMD_LINE_OPT_LEGACY "legacy" +#define CMD_LINE_OPT_EMPTY_POLL "empty-poll" +#define CMD_LINE_OPT_INTERRUPT_ONLY "interrupt-only" +#define CMD_LINE_OPT_TELEMETRY "telemetry" /* Parse the argument given in the command line of the application */ static int @@ -1210,17 +1758,25 @@ parse_args(int argc, char **argv) int opt, ret; char **argvopt; int option_index; + uint32_t limit; char *prgname = argv[0]; static struct option lgopts[] = { {"config", 1, 0, 0}, + {"perf-config", 1, 0, 0}, + {"high-perf-cores", 1, 0, 0}, {"no-numa", 0, 0, 0}, {"enable-jumbo", 0, 0, 0}, + {CMD_LINE_OPT_EMPTY_POLL, 1, 0, 0}, + {CMD_LINE_OPT_PARSE_PTYPE, 0, 0, 0}, + {CMD_LINE_OPT_LEGACY, 0, 0, 0}, + {CMD_LINE_OPT_TELEMETRY, 0, 0, 0}, + {CMD_LINE_OPT_INTERRUPT_ONLY, 0, 0, 0}, {NULL, 0, 0, 0} }; argvopt = argv; - while ((opt = getopt_long(argc, argvopt, "p:P", + while ((opt = getopt_long(argc, argvopt, "p:l:m:h:P", lgopts, &option_index)) != EOF) { switch (opt) { @@ -1237,7 +1793,18 @@ parse_args(int argc, char **argv) printf("Promiscuous mode selected\n"); promiscuous_on = 1; break; - + case 'l': + limit = parse_max_pkt_len(optarg); + freq_tlb[LOW] = limit; + break; + case 'm': + limit = parse_max_pkt_len(optarg); + freq_tlb[MED] = limit; + break; + case 'h': + limit = parse_max_pkt_len(optarg); + freq_tlb[HGH] = limit; + break; /* long options */ case 0: if (!strncmp(lgopts[option_index].name, "config", 6)) { @@ -1249,12 +1816,82 @@ parse_args(int argc, char **argv) } } + if (!strncmp(lgopts[option_index].name, + "perf-config", 11)) { + ret = parse_perf_config(optarg); + if (ret) { + printf("invalid perf-config\n"); + print_usage(prgname); + return -1; + } + } + + if (!strncmp(lgopts[option_index].name, + "high-perf-cores", 15)) { + ret = parse_perf_core_list(optarg); + if (ret) { + printf("invalid high-perf-cores\n"); + print_usage(prgname); + return -1; + } + } + if (!strncmp(lgopts[option_index].name, "no-numa", 7)) { printf("numa is disabled \n"); numa_on = 0; } + if (!strncmp(lgopts[option_index].name, + CMD_LINE_OPT_LEGACY, + sizeof(CMD_LINE_OPT_LEGACY))) { + if (app_mode != APP_MODE_DEFAULT) { + printf(" legacy mode is mutually exclusive with other modes\n"); + return -1; + } + app_mode = APP_MODE_LEGACY; + printf("legacy mode is enabled\n"); + } + + if (!strncmp(lgopts[option_index].name, + CMD_LINE_OPT_EMPTY_POLL, 10)) { + if (app_mode != APP_MODE_DEFAULT) { + printf(" empty-poll mode is mutually exclusive with other modes\n"); + return -1; + } + app_mode = APP_MODE_EMPTY_POLL; + ret = parse_ep_config(optarg); + + if (ret) { + printf("invalid empty poll config\n"); + print_usage(prgname); + return -1; + } + printf("empty-poll is enabled\n"); + } + + if (!strncmp(lgopts[option_index].name, + CMD_LINE_OPT_TELEMETRY, + sizeof(CMD_LINE_OPT_TELEMETRY))) { + if (app_mode != APP_MODE_DEFAULT) { + printf(" telemetry mode is mutually exclusive with other modes\n"); + return -1; + } + app_mode = APP_MODE_TELEMETRY; + printf("telemetry mode is enabled\n"); + } + + if (!strncmp(lgopts[option_index].name, + CMD_LINE_OPT_INTERRUPT_ONLY, + sizeof(CMD_LINE_OPT_INTERRUPT_ONLY))) { + if (app_mode != APP_MODE_DEFAULT) { + printf(" interrupt-only mode is mutually exclusive with other modes\n"); + return -1; + } + app_mode = APP_MODE_INTERRUPT; + printf("interrupt-only mode is enabled\n"); + } + if (!strncmp(lgopts[option_index].name, "enable-jumbo", 12)) { struct option lenopts = @@ -1262,11 +1899,14 @@ parse_args(int argc, char **argv) 0, 0}; printf("jumbo frame is enabled \n"); - port_conf.rxmode.jumbo_frame = 1; + port_conf.rxmode.offloads |= + DEV_RX_OFFLOAD_JUMBO_FRAME; + port_conf.txmode.offloads |= + DEV_TX_OFFLOAD_MULTI_SEGS; /** * if no max-pkt-len set, use the default value - * ETHER_MAX_LEN + * RTE_ETHER_MAX_LEN */ if (0 == getopt_long(argc, argvopt, "", &lenopts, &option_index)) { @@ -1285,6 +1925,13 @@ parse_args(int argc, char **argv) (unsigned int)port_conf.rxmode.max_rx_pkt_len); } + if (!strncmp(lgopts[option_index].name, + CMD_LINE_OPT_PARSE_PTYPE, + sizeof(CMD_LINE_OPT_PARSE_PTYPE))) { + printf("soft parse-ptype is enabled\n"); + parse_ptype = 1; + } + break; default: @@ -1297,15 +1944,15 @@ parse_args(int argc, char **argv) argv[optind-1] = prgname; ret = optind-1; - optind = 0; /* reset getopt lib */ + optind = 1; /* reset getopt lib */ return ret; } static void -print_ethaddr(const char *name, const struct ether_addr *eth_addr) +print_ethaddr(const char *name, const struct rte_ether_addr *eth_addr) { - char buf[ETHER_ADDR_FMT_SIZE]; - ether_format_addr(buf, ETHER_ADDR_FMT_SIZE, eth_addr); + char buf[RTE_ETHER_ADDR_FMT_SIZE]; + rte_ether_format_addr(buf, RTE_ETHER_ADDR_FMT_SIZE, eth_addr); printf("%s%s", name, buf); } @@ -1355,7 +2002,7 @@ setup_hash(int socketid) /* populate the ipv4 hash */ - for (i = 0; i < IPV4_L3FWD_NUM_ROUTES; i++) { + for (i = 0; i < RTE_DIM(ipv4_l3fwd_route_array); i++) { ret = rte_hash_add_key (ipv4_l3fwd_lookup_struct[socketid], (void *) &ipv4_l3fwd_route_array[i].key); if (ret < 0) { @@ -1368,7 +2015,7 @@ setup_hash(int socketid) } /* populate the ipv6 hash */ - for (i = 0; i < IPV6_L3FWD_NUM_ROUTES; i++) { + for (i = 0; i < RTE_DIM(ipv6_l3fwd_route_array); i++) { ret = rte_hash_add_key (ipv6_l3fwd_lookup_struct[socketid], (void *) &ipv6_l3fwd_route_array[i].key); if (ret < 0) { @@ -1405,7 +2052,7 @@ setup_lpm(int socketid) " on socket %d\n", socketid); /* populate the LPM table */ - for (i = 0; i < IPV4_L3FWD_NUM_ROUTES; i++) { + for (i = 0; i < RTE_DIM(ipv4_l3fwd_route_array); i++) { ret = rte_lpm_add(ipv4_l3fwd_lookup_struct[socketid], ipv4_l3fwd_route_array[i].ip, ipv4_l3fwd_route_array[i].depth, @@ -1479,22 +2126,31 @@ init_mem(unsigned nb_mbuf) /* Check the link status of all ports in up to 9s, and print them finally */ static void -check_all_ports_link_status(uint8_t port_num, uint32_t port_mask) +check_all_ports_link_status(uint32_t port_mask) { #define CHECK_INTERVAL 100 /* 100ms */ #define MAX_CHECK_TIME 90 /* 9s (90 * 100ms) in total */ - uint8_t portid, count, all_ports_up, print_flag = 0; + uint8_t count, all_ports_up, print_flag = 0; + uint16_t portid; struct rte_eth_link link; + int ret; printf("\nChecking link status"); fflush(stdout); for (count = 0; count <= MAX_CHECK_TIME; count++) { all_ports_up = 1; - for (portid = 0; portid < port_num; portid++) { + RTE_ETH_FOREACH_DEV(portid) { if ((port_mask & (1 << portid)) == 0) continue; memset(&link, 0, sizeof(link)); - rte_eth_link_get_nowait(portid, &link); + ret = rte_eth_link_get_nowait(portid, &link); + if (ret < 0) { + all_ports_up = 0; + if (print_flag == 1) + printf("Port %u link get failed: %s\n", + portid, rte_strerror(-ret)); + continue; + } /* print link status if flag set */ if (print_flag == 1) { if (link.link_status) @@ -1502,14 +2158,14 @@ check_all_ports_link_status(uint8_t port_num, uint32_t port_mask) "Mbps - %s\n", (uint8_t)portid, (unsigned)link.link_speed, (link.link_duplex == ETH_LINK_FULL_DUPLEX) ? - ("full-duplex") : ("half-duplex\n")); + ("full-duplex") : ("half-duplex")); else printf("Port %d Link Down\n", (uint8_t)portid); continue; } /* clear all_ports_up flag if any link down */ - if (link.link_status == 0) { + if (link.link_status == ETH_LINK_DOWN) { all_ports_up = 0; break; } @@ -1532,6 +2188,264 @@ check_all_ports_link_status(uint8_t port_num, uint32_t port_mask) } } +static int check_ptype(uint16_t portid) +{ + int i, ret; + int ptype_l3_ipv4 = 0; +#if (APP_LOOKUP_METHOD == APP_LOOKUP_EXACT_MATCH) + int ptype_l3_ipv6 = 0; +#endif + uint32_t ptype_mask = RTE_PTYPE_L3_MASK; + + ret = rte_eth_dev_get_supported_ptypes(portid, ptype_mask, NULL, 0); + if (ret <= 0) + return 0; + + uint32_t ptypes[ret]; + + ret = rte_eth_dev_get_supported_ptypes(portid, ptype_mask, ptypes, ret); + for (i = 0; i < ret; ++i) { + if (ptypes[i] & RTE_PTYPE_L3_IPV4) + ptype_l3_ipv4 = 1; +#if (APP_LOOKUP_METHOD == APP_LOOKUP_EXACT_MATCH) + if (ptypes[i] & RTE_PTYPE_L3_IPV6) + ptype_l3_ipv6 = 1; +#endif + } + + if (ptype_l3_ipv4 == 0) + printf("port %d cannot parse RTE_PTYPE_L3_IPV4\n", portid); + +#if (APP_LOOKUP_METHOD == APP_LOOKUP_EXACT_MATCH) + if (ptype_l3_ipv6 == 0) + printf("port %d cannot parse RTE_PTYPE_L3_IPV6\n", portid); +#endif + +#if (APP_LOOKUP_METHOD == APP_LOOKUP_LPM) + if (ptype_l3_ipv4) +#else /* APP_LOOKUP_EXACT_MATCH */ + if (ptype_l3_ipv4 && ptype_l3_ipv6) +#endif + return 1; + + return 0; + +} + +static int +init_power_library(void) +{ + enum power_management_env env; + unsigned int lcore_id; + int ret = 0; + + RTE_LCORE_FOREACH(lcore_id) { + /* init power management library */ + ret = rte_power_init(lcore_id); + if (ret) { + RTE_LOG(ERR, POWER, + "Library initialization failed on core %u\n", + lcore_id); + return ret; + } + /* we're not supporting the VM channel mode */ + env = rte_power_get_env(); + if (env != PM_ENV_ACPI_CPUFREQ && + env != PM_ENV_PSTATE_CPUFREQ) { + RTE_LOG(ERR, POWER, + "Only ACPI and PSTATE mode are supported\n"); + return -1; + } + } + return ret; +} + +static int +deinit_power_library(void) +{ + unsigned int lcore_id; + int ret = 0; + + RTE_LCORE_FOREACH(lcore_id) { + /* deinit power management library */ + ret = rte_power_exit(lcore_id); + if (ret) { + RTE_LOG(ERR, POWER, + "Library deinitialization failed on core %u\n", + lcore_id); + return ret; + } + } + return ret; +} + +static void +get_current_stat_values(uint64_t *values) +{ + unsigned int lcore_id = rte_lcore_id(); + struct lcore_conf *qconf; + uint64_t app_eps = 0, app_fps = 0, app_br = 0; + uint64_t count = 0; + + RTE_LCORE_FOREACH_SLAVE(lcore_id) { + qconf = &lcore_conf[lcore_id]; + if (qconf->n_rx_queue == 0) + continue; + count++; + rte_spinlock_lock(&stats[lcore_id].telemetry_lock); + app_eps += stats[lcore_id].ep_nep[1]; + app_fps += stats[lcore_id].fp_nfp[1]; + app_br += stats[lcore_id].br; + rte_spinlock_unlock(&stats[lcore_id].telemetry_lock); + } + + if (count > 0) { + values[0] = app_eps/count; + values[1] = app_fps/count; + values[2] = app_br/count; + } else + memset(values, 0, sizeof(uint64_t) * NUM_TELSTATS); + +} + +static void +update_telemetry(__rte_unused struct rte_timer *tim, + __rte_unused void *arg) +{ + int ret; + uint64_t values[NUM_TELSTATS] = {0}; + + get_current_stat_values(values); + ret = rte_metrics_update_values(RTE_METRICS_GLOBAL, telstats_index, + values, RTE_DIM(values)); + if (ret < 0) + RTE_LOG(WARNING, POWER, "failed to update metrcis\n"); +} + +static int +handle_app_stats(const char *cmd __rte_unused, + const char *params __rte_unused, + struct rte_tel_data *d) +{ + uint64_t values[NUM_TELSTATS] = {0}; + uint32_t i; + + rte_tel_data_start_dict(d); + get_current_stat_values(values); + for (i = 0; i < NUM_TELSTATS; i++) + rte_tel_data_add_dict_u64(d, telstats_strings[i].name, + values[i]); + return 0; +} + +static void +telemetry_setup_timer(void) +{ + int lcore_id = rte_lcore_id(); + uint64_t hz = rte_get_timer_hz(); + uint64_t ticks; + + ticks = hz / TELEMETRY_INTERVALS_PER_SEC; + rte_timer_reset_sync(&telemetry_timer, + ticks, + PERIODICAL, + lcore_id, + update_telemetry, + NULL); +} +static void +empty_poll_setup_timer(void) +{ + int lcore_id = rte_lcore_id(); + uint64_t hz = rte_get_timer_hz(); + + struct ep_params *ep_ptr = ep_params; + + ep_ptr->interval_ticks = hz / INTERVALS_PER_SECOND; + + rte_timer_reset_sync(&ep_ptr->timer0, + ep_ptr->interval_ticks, + PERIODICAL, + lcore_id, + rte_empty_poll_detection, + (void *)ep_ptr); + +} +static int +launch_timer(unsigned int lcore_id) +{ + int64_t prev_tsc = 0, cur_tsc, diff_tsc, cycles_10ms; + + RTE_SET_USED(lcore_id); + + + if (rte_get_master_lcore() != lcore_id) { + rte_panic("timer on lcore:%d which is not master core:%d\n", + lcore_id, + rte_get_master_lcore()); + } + + RTE_LOG(INFO, POWER, "Bring up the Timer\n"); + + if (app_mode == APP_MODE_EMPTY_POLL) + empty_poll_setup_timer(); + else + telemetry_setup_timer(); + + cycles_10ms = rte_get_timer_hz() / 100; + + while (!is_done()) { + cur_tsc = rte_rdtsc(); + diff_tsc = cur_tsc - prev_tsc; + if (diff_tsc > cycles_10ms) { + rte_timer_manage(); + prev_tsc = cur_tsc; + cycles_10ms = rte_get_timer_hz() / 100; + } + } + + RTE_LOG(INFO, POWER, "Timer_subsystem is done\n"); + + return 0; +} + +static int +autodetect_mode(void) +{ + RTE_LOG(NOTICE, L3FWD_POWER, "Operating mode not specified, probing frequency scaling support...\n"); + + /* + * Empty poll and telemetry modes have to be specifically requested to + * be enabled, but we can auto-detect between interrupt mode with or + * without frequency scaling. Both ACPI and pstate can be used. + */ + if (rte_power_check_env_supported(PM_ENV_ACPI_CPUFREQ)) + return APP_MODE_LEGACY; + if (rte_power_check_env_supported(PM_ENV_PSTATE_CPUFREQ)) + return APP_MODE_LEGACY; + + RTE_LOG(NOTICE, L3FWD_POWER, "Frequency scaling not supported, selecting interrupt-only mode\n"); + + return APP_MODE_INTERRUPT; +} + +static const char * +mode_to_str(enum appmode mode) +{ + switch (mode) { + case APP_MODE_LEGACY: + return "legacy"; + case APP_MODE_EMPTY_POLL: + return "empty poll"; + case APP_MODE_TELEMETRY: + return "telemetry"; + case APP_MODE_INTERRUPT: + return "interrupt-only"; + default: + return "invalid"; + } +} + int main(int argc, char **argv) { @@ -1539,14 +2453,15 @@ main(int argc, char **argv) struct rte_eth_dev_info dev_info; struct rte_eth_txconf *txconf; int ret; - unsigned nb_ports; + uint16_t nb_ports; uint16_t queueid; unsigned lcore_id; uint64_t hz; uint32_t n_tx_queue, nb_lcores; uint32_t dev_rxq_num, dev_txq_num; - uint8_t portid, nb_rx_queue, queue, socketid; - uint8_t nb_tx_port; + uint8_t nb_rx_queue, queue, socketid; + uint16_t portid; + const char *ptr_strings[NUM_TELSTATS]; /* catch SIGINT and restore cpufreq governor to ondemand */ signal(SIGINT, signal_exit_now); @@ -1566,6 +2481,20 @@ main(int argc, char **argv) if (ret < 0) rte_exit(EXIT_FAILURE, "Invalid L3FWD parameters\n"); + if (app_mode == APP_MODE_DEFAULT) + app_mode = autodetect_mode(); + + RTE_LOG(INFO, L3FWD_POWER, "Selected operation mode: %s\n", + mode_to_str(app_mode)); + + /* only legacy and empty poll mode rely on power library */ + if ((app_mode == APP_MODE_LEGACY || app_mode == APP_MODE_EMPTY_POLL) && + init_power_library()) + rte_exit(EXIT_FAILURE, "init_power_library failed\n"); + + if (update_lcore_params() < 0) + rte_exit(EXIT_FAILURE, "update_lcore_params failed\n"); + if (check_lcore_params() < 0) rte_exit(EXIT_FAILURE, "check_lcore_params failed\n"); @@ -1573,19 +2502,20 @@ main(int argc, char **argv) if (ret < 0) rte_exit(EXIT_FAILURE, "init_lcore_rx_queues failed\n"); + nb_ports = rte_eth_dev_count_avail(); - nb_ports = rte_eth_dev_count(); - if (nb_ports > RTE_MAX_ETHPORTS) - nb_ports = RTE_MAX_ETHPORTS; - - if (check_port_config(nb_ports) < 0) + if (check_port_config() < 0) rte_exit(EXIT_FAILURE, "check_port_config failed\n"); nb_lcores = rte_lcore_count(); - nb_tx_port = 0; /* initialize all ports */ - for (portid = 0; portid < nb_ports; portid++) { + RTE_ETH_FOREACH_DEV(portid) { + struct rte_eth_conf local_port_conf = port_conf; + /* not all app modes need interrupts */ + bool need_intr = app_mode == APP_MODE_LEGACY || + app_mode == APP_MODE_INTERRUPT; + /* skip ports that are not enabled */ if ((enabled_port_mask & (1 << portid)) == 0) { printf("\nSkipping disabled port %d\n", portid); @@ -1596,7 +2526,12 @@ main(int argc, char **argv) printf("Initializing port %d ... ", portid ); fflush(stdout); - rte_eth_dev_info_get(portid, &dev_info); + ret = rte_eth_dev_info_get(portid, &dev_info); + if (ret != 0) + rte_exit(EXIT_FAILURE, + "Error during getting device (port %u) info: %s\n", + portid, strerror(-ret)); + dev_rxq_num = dev_info.max_rx_queues; dev_txq_num = dev_info.max_tx_queues; @@ -1611,13 +2546,53 @@ main(int argc, char **argv) n_tx_queue = dev_txq_num; printf("Creating queues: nb_rxq=%d nb_txq=%u... ", nb_rx_queue, (unsigned)n_tx_queue ); + /* If number of Rx queue is 0, no need to enable Rx interrupt */ + if (nb_rx_queue == 0) + need_intr = false; + + if (need_intr) + local_port_conf.intr_conf.rxq = 1; + + ret = rte_eth_dev_info_get(portid, &dev_info); + if (ret != 0) + rte_exit(EXIT_FAILURE, + "Error during getting device (port %u) info: %s\n", + portid, strerror(-ret)); + + if (dev_info.tx_offload_capa & DEV_TX_OFFLOAD_MBUF_FAST_FREE) + local_port_conf.txmode.offloads |= + DEV_TX_OFFLOAD_MBUF_FAST_FREE; + + local_port_conf.rx_adv_conf.rss_conf.rss_hf &= + dev_info.flow_type_rss_offloads; + if (local_port_conf.rx_adv_conf.rss_conf.rss_hf != + port_conf.rx_adv_conf.rss_conf.rss_hf) { + printf("Port %u modified RSS hash function based on hardware support," + "requested:%#"PRIx64" configured:%#"PRIx64"\n", + portid, + port_conf.rx_adv_conf.rss_conf.rss_hf, + local_port_conf.rx_adv_conf.rss_conf.rss_hf); + } + ret = rte_eth_dev_configure(portid, nb_rx_queue, - (uint16_t)n_tx_queue, &port_conf); + (uint16_t)n_tx_queue, &local_port_conf); if (ret < 0) rte_exit(EXIT_FAILURE, "Cannot configure device: " "err=%d, port=%d\n", ret, portid); - rte_eth_macaddr_get(portid, &ports_eth_addr[portid]); + ret = rte_eth_dev_adjust_nb_rx_tx_desc(portid, &nb_rxd, + &nb_txd); + if (ret < 0) + rte_exit(EXIT_FAILURE, + "Cannot adjust number of descriptors: err=%d, port=%d\n", + ret, portid); + + ret = rte_eth_macaddr_get(portid, &ports_eth_addr[portid]); + if (ret < 0) + rte_exit(EXIT_FAILURE, + "Cannot get MAC address: err=%d, port=%d\n", + ret, portid); + print_ethaddr(" Address:", &ports_eth_addr[portid]); printf(", "); @@ -1637,7 +2612,7 @@ main(int argc, char **argv) rte_eth_dev_socket_id(portid)); if (qconf->tx_buffer[portid] == NULL) rte_exit(EXIT_FAILURE, "Can't allocate tx buffer for port %u\n", - (unsigned) portid); + portid); rte_eth_tx_buffer_init(qconf->tx_buffer[portid], MAX_PKT_BURST); } @@ -1660,10 +2635,8 @@ main(int argc, char **argv) printf("txq=%u,%d,%d ", lcore_id, queueid, socketid); fflush(stdout); - rte_eth_dev_info_get(portid, &dev_info); txconf = &dev_info.default_txconf; - if (port_conf.rxmode.jumbo_frame) - txconf->txq_flags = 0; + txconf->offloads = local_port_conf.txmode.offloads; ret = rte_eth_tx_queue_setup(portid, queueid, nb_txd, socketid, txconf); if (ret < 0) @@ -1675,36 +2648,32 @@ main(int argc, char **argv) qconf->tx_queue_id[portid] = queueid; queueid++; - qconf->n_tx_port = nb_tx_port; qconf->tx_port_id[qconf->n_tx_port] = portid; + qconf->n_tx_port++; } printf("\n"); - - nb_tx_port++; } for (lcore_id = 0; lcore_id < RTE_MAX_LCORE; lcore_id++) { if (rte_lcore_is_enabled(lcore_id) == 0) continue; - /* init power management library */ - ret = rte_power_init(lcore_id); - if (ret) - RTE_LOG(ERR, POWER, - "Library initialization failed on core %u\n", lcore_id); - - /* init timer structures for each enabled lcore */ - rte_timer_init(&power_timers[lcore_id]); - hz = rte_get_timer_hz(); - rte_timer_reset(&power_timers[lcore_id], - hz/TIMER_NUMBER_PER_SECOND, SINGLE, lcore_id, - power_timer_cb, NULL); - + if (app_mode == APP_MODE_LEGACY) { + /* init timer structures for each enabled lcore */ + rte_timer_init(&power_timers[lcore_id]); + hz = rte_get_timer_hz(); + rte_timer_reset(&power_timers[lcore_id], + hz/TIMER_NUMBER_PER_SECOND, + SINGLE, lcore_id, + power_timer_cb, NULL); + } qconf = &lcore_conf[lcore_id]; printf("\nInitializing rx queues on lcore %u ... ", lcore_id ); fflush(stdout); /* init RX queues */ for(queue = 0; queue < qconf->n_rx_queue; ++queue) { + struct rte_eth_rxconf rxq_conf; + portid = qconf->rx_queue_list[queue].port_id; queueid = qconf->rx_queue_list[queue].queue_id; @@ -1717,20 +2686,36 @@ main(int argc, char **argv) printf("rxq=%d,%d,%d ", portid, queueid, socketid); fflush(stdout); + ret = rte_eth_dev_info_get(portid, &dev_info); + if (ret != 0) + rte_exit(EXIT_FAILURE, + "Error during getting device (port %u) info: %s\n", + portid, strerror(-ret)); + + rxq_conf = dev_info.default_rxconf; + rxq_conf.offloads = port_conf.rxmode.offloads; ret = rte_eth_rx_queue_setup(portid, queueid, nb_rxd, - socketid, NULL, + socketid, &rxq_conf, pktmbuf_pool[socketid]); if (ret < 0) rte_exit(EXIT_FAILURE, "rte_eth_rx_queue_setup: err=%d, " "port=%d\n", ret, portid); + + if (parse_ptype) { + if (add_cb_parse_ptype(portid, queueid) < 0) + rte_exit(EXIT_FAILURE, + "Fail to add ptype cb\n"); + } else if (!check_ptype(portid)) + rte_exit(EXIT_FAILURE, + "PMD can not provide needed ptypes\n"); } } printf("\n"); /* start ports */ - for (portid = 0; portid < nb_ports; portid++) { + RTE_ETH_FOREACH_DEV(portid) { if ((enabled_port_mask & (1 << portid)) == 0) { continue; } @@ -1745,20 +2730,98 @@ main(int argc, char **argv) * to itself through 2 cross-connected ports of the * target machine. */ - if (promiscuous_on) - rte_eth_promiscuous_enable(portid); + if (promiscuous_on) { + ret = rte_eth_promiscuous_enable(portid); + if (ret != 0) + rte_exit(EXIT_FAILURE, + "rte_eth_promiscuous_enable: err=%s, port=%u\n", + rte_strerror(-ret), portid); + } /* initialize spinlock for each port */ rte_spinlock_init(&(locks[portid])); } - check_all_ports_link_status((uint8_t)nb_ports, enabled_port_mask); + check_all_ports_link_status(enabled_port_mask); + + if (app_mode == APP_MODE_EMPTY_POLL) { + + if (empty_poll_train) { + policy.state = TRAINING; + } else { + policy.state = MED_NORMAL; + policy.med_base_edpi = ep_med_edpi; + policy.hgh_base_edpi = ep_hgh_edpi; + } + + ret = rte_power_empty_poll_stat_init(&ep_params, + freq_tlb, + &policy); + if (ret < 0) + rte_exit(EXIT_FAILURE, "empty poll init failed"); + } + /* launch per-lcore init on every lcore */ - rte_eal_mp_remote_launch(main_loop, NULL, CALL_MASTER); + if (app_mode == APP_MODE_LEGACY) { + rte_eal_mp_remote_launch(main_legacy_loop, NULL, CALL_MASTER); + } else if (app_mode == APP_MODE_EMPTY_POLL) { + empty_poll_stop = false; + rte_eal_mp_remote_launch(main_empty_poll_loop, NULL, + SKIP_MASTER); + } else if (app_mode == APP_MODE_TELEMETRY) { + unsigned int i; + + /* Init metrics library */ + rte_metrics_init(rte_socket_id()); + /** Register stats with metrics library */ + for (i = 0; i < NUM_TELSTATS; i++) + ptr_strings[i] = telstats_strings[i].name; + + ret = rte_metrics_reg_names(ptr_strings, NUM_TELSTATS); + if (ret >= 0) + telstats_index = ret; + else + rte_exit(EXIT_FAILURE, "failed to register metrics names"); + + RTE_LCORE_FOREACH_SLAVE(lcore_id) { + rte_spinlock_init(&stats[lcore_id].telemetry_lock); + } + rte_timer_init(&telemetry_timer); + rte_telemetry_register_cmd("/l3fwd-power/stats", + handle_app_stats, + "Returns global power stats. Parameters: None"); + rte_eal_mp_remote_launch(main_telemetry_loop, NULL, + SKIP_MASTER); + } else if (app_mode == APP_MODE_INTERRUPT) { + rte_eal_mp_remote_launch(main_intr_loop, NULL, CALL_MASTER); + } + + if (app_mode == APP_MODE_EMPTY_POLL || app_mode == APP_MODE_TELEMETRY) + launch_timer(rte_lcore_id()); + RTE_LCORE_FOREACH_SLAVE(lcore_id) { if (rte_eal_wait_lcore(lcore_id) < 0) return -1; } + RTE_ETH_FOREACH_DEV(portid) + { + if ((enabled_port_mask & (1 << portid)) == 0) + continue; + + rte_eth_dev_stop(portid); + rte_eth_dev_close(portid); + } + + if (app_mode == APP_MODE_EMPTY_POLL) + rte_power_empty_poll_stat_free(); + + if ((app_mode == APP_MODE_LEGACY || app_mode == APP_MODE_EMPTY_POLL) && + deinit_power_library()) + rte_exit(EXIT_FAILURE, "deinit_power_library failed\n"); + + if (rte_eal_cleanup() < 0) + RTE_LOG(ERR, L3FWD_POWER, "EAL cleanup failed\n"); + return 0; }