X-Git-Url: http://git.droids-corp.org/?a=blobdiff_plain;f=examples%2Fl3fwd-power%2Fmain.c;h=aa7b8db44ae878c48d50163a05d59190cfb9d89d;hb=fa71c328d1d7cf11531a7688fe679ef61f81ec7d;hp=7a95605c48f261449c6d536f3bef027a753e5141;hpb=018faf21d670e06d51beb52e1c3c12192a762339;p=dpdk.git diff --git a/examples/l3fwd-power/main.c b/examples/l3fwd-power/main.c index 7a95605c48..aa7b8db44a 100644 --- a/examples/l3fwd-power/main.c +++ b/examples/l3fwd-power/main.c @@ -46,6 +46,8 @@ #include #include #include +#include +#include #include "perf_core.h" #include "main.h" @@ -131,7 +133,7 @@ #define EMPTY_POLL_MED_THRESHOLD 350000UL #define EMPTY_POLL_HGH_THRESHOLD 580000UL - +#define NUM_TELSTATS RTE_DIM(telstats_strings) static uint16_t nb_rxd = RTE_TEST_RX_DESC_DEFAULT; static uint16_t nb_txd = RTE_TEST_TX_DESC_DEFAULT; @@ -184,8 +186,8 @@ enum busy_rate { * reference CYCLES to be used to * measure core busyness based on poll count */ -#define MIN_CYCLES 1500000ULL -#define MAX_CYCLES 2500000ULL +#define MIN_CYCLES 1500000ULL +#define MAX_CYCLES 22000000ULL /* (500ms) */ #define TELEMETRY_INTERVALS_PER_SEC 2 @@ -194,13 +196,19 @@ static int parse_ptype; /**< Parse packet type using rx callback, and */ /**< disabled by default */ enum appmode { - APP_MODE_LEGACY = 0, + APP_MODE_DEFAULT = 0, + APP_MODE_LEGACY, APP_MODE_EMPTY_POLL, - APP_MODE_TELEMETRY + APP_MODE_TELEMETRY, + APP_MODE_INTERRUPT, + APP_MODE_PMD_MGMT }; enum appmode app_mode; +static enum rte_power_pmd_mgmt_type pmgmt_type; +bool baseline_enabled; + enum freq_scale_hint_t { FREQ_LOWER = -1, @@ -238,8 +246,7 @@ static struct lcore_params lcore_params_array_default[] = { }; struct lcore_params *lcore_params = lcore_params_array_default; -uint16_t nb_lcore_params = sizeof(lcore_params_array_default) / - sizeof(lcore_params_array_default[0]); +uint16_t nb_lcore_params = RTE_DIM(lcore_params_array_default); static struct rte_eth_conf port_conf = { .rxmode = { @@ -256,10 +263,7 @@ static struct rte_eth_conf port_conf = { }, .txmode = { .mq_mode = ETH_MQ_TX_NONE, - }, - .intr_conf = { - .rxq = 1, - }, + } }; static struct rte_mempool * pktmbuf_pool[NB_SOCKETS]; @@ -281,7 +285,7 @@ struct ipv4_5tuple { uint16_t port_dst; uint16_t port_src; uint8_t proto; -} __attribute__((__packed__)); +} __rte_packed; struct ipv6_5tuple { uint8_t ip_dst[IPV6_ADDR_LEN]; @@ -289,7 +293,7 @@ struct ipv6_5tuple { uint16_t port_dst; uint16_t port_src; uint8_t proto; -} __attribute__((__packed__)); +} __rte_packed; struct ipv4_l3fwd_route { struct ipv4_5tuple key; @@ -326,12 +330,6 @@ static lookup_struct_t *ipv6_l3fwd_lookup_struct[NB_SOCKETS]; #define L3FWD_HASH_ENTRIES 1024 -#define IPV4_L3FWD_NUM_ROUTES \ - (sizeof(ipv4_l3fwd_route_array) / sizeof(ipv4_l3fwd_route_array[0])) - -#define IPV6_L3FWD_NUM_ROUTES \ - (sizeof(ipv6_l3fwd_route_array) / sizeof(ipv6_l3fwd_route_array[0])) - static uint16_t ipv4_l3fwd_out_if[L3FWD_HASH_ENTRIES] __rte_cache_aligned; static uint16_t ipv6_l3fwd_out_if[L3FWD_HASH_ENTRIES] __rte_cache_aligned; #endif @@ -354,9 +352,6 @@ static struct ipv4_l3fwd_route ipv4_l3fwd_route_array[] = { {RTE_IPV4(8,1,1,0), 24, 7}, }; -#define IPV4_L3FWD_NUM_ROUTES \ - (sizeof(ipv4_l3fwd_route_array) / sizeof(ipv4_l3fwd_route_array[0])) - #define IPV4_L3FWD_LPM_MAX_RULES 1024 typedef struct rte_lpm lookup_struct_t; @@ -432,47 +427,16 @@ static int is_done(void) static void signal_exit_now(int sigtype) { - unsigned lcore_id; - unsigned int portid; - int ret; - if (sigtype == SIGINT) { - if (app_mode == APP_MODE_EMPTY_POLL || - app_mode == APP_MODE_TELEMETRY) - quit_signal = true; + if (sigtype == SIGINT) + quit_signal = true; - - for (lcore_id = 0; lcore_id < RTE_MAX_LCORE; lcore_id++) { - if (rte_lcore_is_enabled(lcore_id) == 0) - continue; - - /* init power management library */ - ret = rte_power_exit(lcore_id); - if (ret) - rte_exit(EXIT_FAILURE, "Power management " - "library de-initialization failed on " - "core%u\n", lcore_id); - } - - if (app_mode != APP_MODE_EMPTY_POLL) { - RTE_ETH_FOREACH_DEV(portid) { - if ((enabled_port_mask & (1 << portid)) == 0) - continue; - - rte_eth_dev_stop(portid); - rte_eth_dev_close(portid); - } - } - } - - if (app_mode != APP_MODE_EMPTY_POLL) - rte_exit(EXIT_SUCCESS, "User forced exit\n"); } /* Freqency scale down timer callback */ static void -power_timer_cb(__attribute__((unused)) struct rte_timer *tim, - __attribute__((unused)) void *arg) +power_timer_cb(__rte_unused struct rte_timer *tim, + __rte_unused void *arg) { uint64_t hz; float sleep_time_ratio; @@ -862,35 +826,45 @@ power_freq_scaleup_heuristic(unsigned lcore_id, * 0 on success */ static int -sleep_until_rx_interrupt(int num) +sleep_until_rx_interrupt(int num, int lcore) { + /* + * we want to track when we are woken up by traffic so that we can go + * back to sleep again without log spamming. Avoid cache line sharing + * to prevent threads stepping on each others' toes. + */ + static struct { + bool wakeup; + } __rte_cache_aligned status[RTE_MAX_LCORE]; struct rte_epoll_event event[num]; int n, i; uint16_t port_id; uint8_t queue_id; void *data; - RTE_LOG(INFO, L3FWD_POWER, - "lcore %u sleeps until interrupt triggers\n", - rte_lcore_id()); + if (status[lcore].wakeup) { + RTE_LOG(INFO, L3FWD_POWER, + "lcore %u sleeps until interrupt triggers\n", + rte_lcore_id()); + } - n = rte_epoll_wait(RTE_EPOLL_PER_THREAD, event, num, -1); + n = rte_epoll_wait(RTE_EPOLL_PER_THREAD, event, num, 10); for (i = 0; i < n; i++) { data = event[i].epdata.data; port_id = ((uintptr_t)data) >> CHAR_BIT; queue_id = ((uintptr_t)data) & RTE_LEN2MASK(CHAR_BIT, uint8_t); - rte_eth_dev_rx_intr_disable(port_id, queue_id); RTE_LOG(INFO, L3FWD_POWER, "lcore %u is waked up from rx interrupt on" " port %d queue %d\n", rte_lcore_id(), port_id, queue_id); } + status[lcore].wakeup = n != 0; return 0; } -static void turn_on_intr(struct lcore_conf *qconf) +static void turn_on_off_intr(struct lcore_conf *qconf, bool on) { int i; struct lcore_rx_queue *rx_queue; @@ -903,7 +877,10 @@ static void turn_on_intr(struct lcore_conf *qconf) queue_id = rx_queue->queue_id; rte_spinlock_lock(&(locks[port_id])); - rte_eth_dev_rx_intr_enable(port_id, queue_id); + if (on) + rte_eth_dev_rx_intr_enable(port_id, queue_id); + else + rte_eth_dev_rx_intr_disable(port_id, queue_id); rte_spinlock_unlock(&(locks[port_id])); } } @@ -933,9 +910,175 @@ static int event_register(struct lcore_conf *qconf) return 0; } + +/* Main processing loop. 8< */ +static int main_intr_loop(__rte_unused void *dummy) +{ + struct rte_mbuf *pkts_burst[MAX_PKT_BURST]; + unsigned int lcore_id; + uint64_t prev_tsc, diff_tsc, cur_tsc; + int i, j, nb_rx; + uint8_t queueid; + uint16_t portid; + struct lcore_conf *qconf; + struct lcore_rx_queue *rx_queue; + uint32_t lcore_rx_idle_count = 0; + uint32_t lcore_idle_hint = 0; + int intr_en = 0; + + const uint64_t drain_tsc = (rte_get_tsc_hz() + US_PER_S - 1) / + US_PER_S * BURST_TX_DRAIN_US; + + prev_tsc = 0; + + lcore_id = rte_lcore_id(); + qconf = &lcore_conf[lcore_id]; + + if (qconf->n_rx_queue == 0) { + RTE_LOG(INFO, L3FWD_POWER, "lcore %u has nothing to do\n", + lcore_id); + return 0; + } + + RTE_LOG(INFO, L3FWD_POWER, "entering main interrupt loop on lcore %u\n", + lcore_id); + + for (i = 0; i < qconf->n_rx_queue; i++) { + portid = qconf->rx_queue_list[i].port_id; + queueid = qconf->rx_queue_list[i].queue_id; + RTE_LOG(INFO, L3FWD_POWER, + " -- lcoreid=%u portid=%u rxqueueid=%hhu\n", + lcore_id, portid, queueid); + } + + /* add into event wait list */ + if (event_register(qconf) == 0) + intr_en = 1; + else + RTE_LOG(INFO, L3FWD_POWER, "RX interrupt won't enable.\n"); + + while (!is_done()) { + stats[lcore_id].nb_iteration_looped++; + + cur_tsc = rte_rdtsc(); + + /* + * TX burst queue drain + */ + diff_tsc = cur_tsc - prev_tsc; + if (unlikely(diff_tsc > drain_tsc)) { + for (i = 0; i < qconf->n_tx_port; ++i) { + portid = qconf->tx_port_id[i]; + rte_eth_tx_buffer_flush(portid, + qconf->tx_queue_id[portid], + qconf->tx_buffer[portid]); + } + prev_tsc = cur_tsc; + } + +start_rx: + /* + * Read packet from RX queues + */ + lcore_rx_idle_count = 0; + for (i = 0; i < qconf->n_rx_queue; ++i) { + rx_queue = &(qconf->rx_queue_list[i]); + rx_queue->idle_hint = 0; + portid = rx_queue->port_id; + queueid = rx_queue->queue_id; + + nb_rx = rte_eth_rx_burst(portid, queueid, pkts_burst, + MAX_PKT_BURST); + + stats[lcore_id].nb_rx_processed += nb_rx; + if (unlikely(nb_rx == 0)) { + /** + * no packet received from rx queue, try to + * sleep for a while forcing CPU enter deeper + * C states. + */ + rx_queue->zero_rx_packet_count++; + + if (rx_queue->zero_rx_packet_count <= + MIN_ZERO_POLL_COUNT) + continue; + + rx_queue->idle_hint = power_idle_heuristic( + rx_queue->zero_rx_packet_count); + lcore_rx_idle_count++; + } else { + rx_queue->zero_rx_packet_count = 0; + } + + /* Prefetch first packets */ + for (j = 0; j < PREFETCH_OFFSET && j < nb_rx; j++) { + rte_prefetch0(rte_pktmbuf_mtod( + pkts_burst[j], void *)); + } + + /* Prefetch and forward already prefetched packets */ + for (j = 0; j < (nb_rx - PREFETCH_OFFSET); j++) { + rte_prefetch0(rte_pktmbuf_mtod( + pkts_burst[j + PREFETCH_OFFSET], + void *)); + l3fwd_simple_forward( + pkts_burst[j], portid, qconf); + } + + /* Forward remaining prefetched packets */ + for (; j < nb_rx; j++) { + l3fwd_simple_forward( + pkts_burst[j], portid, qconf); + } + } + + if (unlikely(lcore_rx_idle_count == qconf->n_rx_queue)) { + /** + * All Rx queues empty in recent consecutive polls, + * sleep in a conservative manner, meaning sleep as + * less as possible. + */ + for (i = 1, + lcore_idle_hint = qconf->rx_queue_list[0].idle_hint; + i < qconf->n_rx_queue; ++i) { + rx_queue = &(qconf->rx_queue_list[i]); + if (rx_queue->idle_hint < lcore_idle_hint) + lcore_idle_hint = rx_queue->idle_hint; + } + + if (lcore_idle_hint < SUSPEND_THRESHOLD) + /** + * execute "pause" instruction to avoid context + * switch which generally take hundred of + * microseconds for short sleep. + */ + rte_delay_us(lcore_idle_hint); + else { + /* suspend until rx interrupt triggers */ + if (intr_en) { + turn_on_off_intr(qconf, 1); + sleep_until_rx_interrupt( + qconf->n_rx_queue, + lcore_id); + turn_on_off_intr(qconf, 0); + /** + * start receiving packets immediately + */ + if (likely(!is_done())) + goto start_rx; + } + } + stats[lcore_id].sleep_time += lcore_idle_hint; + } + } + + return 0; +} +/* >8 End of main processing loop. */ + /* main processing loop */ static int -main_telemetry_loop(__attribute__((unused)) void *dummy) +main_telemetry_loop(__rte_unused void *dummy) { struct rte_mbuf *pkts_burst[MAX_PKT_BURST]; unsigned int lcore_id; @@ -1034,7 +1177,7 @@ main_telemetry_loop(__attribute__((unused)) void *dummy) br = FULL; } else if (diff_tsc > MIN_CYCLES && diff_tsc < MAX_CYCLES) { - br = PARTIAL; + br = (diff_tsc * 100) / MAX_CYCLES; } else { br = ZERO; } @@ -1055,7 +1198,7 @@ main_telemetry_loop(__attribute__((unused)) void *dummy) } /* main processing loop */ static int -main_empty_poll_loop(__attribute__((unused)) void *dummy) +main_empty_poll_loop(__rte_unused void *dummy) { struct rte_mbuf *pkts_burst[MAX_PKT_BURST]; unsigned int lcore_id; @@ -1159,7 +1302,7 @@ main_empty_poll_loop(__attribute__((unused)) void *dummy) } /* main processing loop */ static int -main_loop(__attribute__((unused)) void *dummy) +main_legacy_loop(__rte_unused void *dummy) { struct rte_mbuf *pkts_burst[MAX_PKT_BURST]; unsigned lcore_id; @@ -1204,7 +1347,7 @@ main_loop(__attribute__((unused)) void *dummy) else RTE_LOG(INFO, L3FWD_POWER, "RX interrupt won't enable.\n"); - while (1) { + while (!is_done()) { stats[lcore_id].nb_iteration_looped++; cur_tsc = rte_rdtsc(); @@ -1336,20 +1479,25 @@ start_rx: */ rte_delay_us(lcore_idle_hint); else { - /* suspend until rx interrupt trigges */ + /* suspend until rx interrupt triggers */ if (intr_en) { - turn_on_intr(qconf); + turn_on_off_intr(qconf, 1); sleep_until_rx_interrupt( - qconf->n_rx_queue); + qconf->n_rx_queue, + lcore_id); + turn_on_off_intr(qconf, 0); /** * start receiving packets immediately */ - goto start_rx; + if (likely(!is_done())) + goto start_rx; } } stats[lcore_id].sleep_time += lcore_idle_hint; } } + + return 0; } static int @@ -1377,7 +1525,7 @@ check_lcore_params(void) "off\n", lcore, socketid); } if (app_mode == APP_MODE_TELEMETRY && lcore == rte_lcore_id()) { - printf("cannot enable master core %d in config for telemetry mode\n", + printf("cannot enable main core %d in config for telemetry mode\n", rte_lcore_id()); return -1; } @@ -1464,10 +1612,14 @@ print_usage(const char *prgname) " --enable-jumbo: enable jumbo frame" " which max packet len is PKTLEN in decimal (64-9600)\n" " --parse-ptype: parse packet type by software\n" + " --legacy: use legacy interrupt-based scaling\n" " --empty-poll: enable empty poll detection" " follow (training_flag, high_threshold, med_threshold)\n" " --telemetry: enable telemetry mode, to update" - " empty polls, full polls, and core busyness to telemetry\n", + " empty polls, full polls, and core busyness to telemetry\n" + " --interrupt-only: enable interrupt-only mode\n" + " --pmd-mgmt MODE: enable PMD power management mode. " + "Currently supported modes: baseline, monitor, pause, scale\n", prgname); } @@ -1496,10 +1648,7 @@ parse_portmask(const char *portmask) /* parse hexadecimal string */ pm = strtoul(portmask, &end, 16); if ((portmask[0] == '\0') || (end == NULL) || (*end != '\0')) - return -1; - - if (pm == 0) - return -1; + return 0; return pm; } @@ -1560,6 +1709,37 @@ parse_config(const char *q_arg) return 0; } + +static int +parse_pmd_mgmt_config(const char *name) +{ +#define PMD_MGMT_MONITOR "monitor" +#define PMD_MGMT_PAUSE "pause" +#define PMD_MGMT_SCALE "scale" +#define PMD_MGMT_BASELINE "baseline" + + if (strncmp(PMD_MGMT_MONITOR, name, sizeof(PMD_MGMT_MONITOR)) == 0) { + pmgmt_type = RTE_POWER_MGMT_TYPE_MONITOR; + return 0; + } + + if (strncmp(PMD_MGMT_PAUSE, name, sizeof(PMD_MGMT_PAUSE)) == 0) { + pmgmt_type = RTE_POWER_MGMT_TYPE_PAUSE; + return 0; + } + + if (strncmp(PMD_MGMT_SCALE, name, sizeof(PMD_MGMT_SCALE)) == 0) { + pmgmt_type = RTE_POWER_MGMT_TYPE_SCALE; + return 0; + } + if (strncmp(PMD_MGMT_BASELINE, name, sizeof(PMD_MGMT_BASELINE)) == 0) { + baseline_enabled = true; + return 0; + } + /* unknown PMD power management mode */ + return -1; +} + static int parse_ep_config(const char *q_arg) { @@ -1575,7 +1755,7 @@ parse_ep_config(const char *q_arg) int hgh_edpi; ep_med_edpi = EMPTY_POLL_MED_THRESHOLD; - ep_hgh_edpi = EMPTY_POLL_MED_THRESHOLD; + ep_hgh_edpi = EMPTY_POLL_HGH_THRESHOLD; strlcpy(s, p, sizeof(s)); @@ -1598,7 +1778,7 @@ parse_ep_config(const char *q_arg) if (med_edpi > 0) ep_med_edpi = med_edpi; - if (med_edpi > 0) + if (hgh_edpi > 0) ep_hgh_edpi = hgh_edpi; } else { @@ -1610,7 +1790,11 @@ parse_ep_config(const char *q_arg) } #define CMD_LINE_OPT_PARSE_PTYPE "parse-ptype" +#define CMD_LINE_OPT_LEGACY "legacy" +#define CMD_LINE_OPT_EMPTY_POLL "empty-poll" +#define CMD_LINE_OPT_INTERRUPT_ONLY "interrupt-only" #define CMD_LINE_OPT_TELEMETRY "telemetry" +#define CMD_LINE_OPT_PMD_MGMT "pmd-mgmt" /* Parse the argument given in the command line of the application */ static int @@ -1627,9 +1811,12 @@ parse_args(int argc, char **argv) {"high-perf-cores", 1, 0, 0}, {"no-numa", 0, 0, 0}, {"enable-jumbo", 0, 0, 0}, - {"empty-poll", 1, 0, 0}, + {CMD_LINE_OPT_EMPTY_POLL, 1, 0, 0}, {CMD_LINE_OPT_PARSE_PTYPE, 0, 0, 0}, + {CMD_LINE_OPT_LEGACY, 0, 0, 0}, {CMD_LINE_OPT_TELEMETRY, 0, 0, 0}, + {CMD_LINE_OPT_INTERRUPT_ONLY, 0, 0, 0}, + {CMD_LINE_OPT_PMD_MGMT, 1, 0, 0}, {NULL, 0, 0, 0} }; @@ -1702,9 +1889,20 @@ parse_args(int argc, char **argv) } if (!strncmp(lgopts[option_index].name, - "empty-poll", 10)) { - if (app_mode == APP_MODE_TELEMETRY) { - printf(" empty-poll cannot be enabled as telemetry mode is enabled\n"); + CMD_LINE_OPT_LEGACY, + sizeof(CMD_LINE_OPT_LEGACY))) { + if (app_mode != APP_MODE_DEFAULT) { + printf(" legacy mode is mutually exclusive with other modes\n"); + return -1; + } + app_mode = APP_MODE_LEGACY; + printf("legacy mode is enabled\n"); + } + + if (!strncmp(lgopts[option_index].name, + CMD_LINE_OPT_EMPTY_POLL, 10)) { + if (app_mode != APP_MODE_DEFAULT) { + printf(" empty-poll mode is mutually exclusive with other modes\n"); return -1; } app_mode = APP_MODE_EMPTY_POLL; @@ -1721,14 +1919,40 @@ parse_args(int argc, char **argv) if (!strncmp(lgopts[option_index].name, CMD_LINE_OPT_TELEMETRY, sizeof(CMD_LINE_OPT_TELEMETRY))) { - if (app_mode == APP_MODE_EMPTY_POLL) { - printf("telemetry mode cannot be enabled as empty poll mode is enabled\n"); + if (app_mode != APP_MODE_DEFAULT) { + printf(" telemetry mode is mutually exclusive with other modes\n"); return -1; } app_mode = APP_MODE_TELEMETRY; printf("telemetry mode is enabled\n"); } + if (!strncmp(lgopts[option_index].name, + CMD_LINE_OPT_PMD_MGMT, + sizeof(CMD_LINE_OPT_PMD_MGMT))) { + if (app_mode != APP_MODE_DEFAULT) { + printf(" power mgmt mode is mutually exclusive with other modes\n"); + return -1; + } + if (parse_pmd_mgmt_config(optarg) < 0) { + printf(" Invalid PMD power management mode: %s\n", + optarg); + return -1; + } + app_mode = APP_MODE_PMD_MGMT; + printf("PMD power mgmt mode is enabled\n"); + } + if (!strncmp(lgopts[option_index].name, + CMD_LINE_OPT_INTERRUPT_ONLY, + sizeof(CMD_LINE_OPT_INTERRUPT_ONLY))) { + if (app_mode != APP_MODE_DEFAULT) { + printf(" interrupt-only mode is mutually exclusive with other modes\n"); + return -1; + } + app_mode = APP_MODE_INTERRUPT; + printf("interrupt-only mode is enabled\n"); + } + if (!strncmp(lgopts[option_index].name, "enable-jumbo", 12)) { struct option lenopts = @@ -1839,7 +2063,7 @@ setup_hash(int socketid) /* populate the ipv4 hash */ - for (i = 0; i < IPV4_L3FWD_NUM_ROUTES; i++) { + for (i = 0; i < RTE_DIM(ipv4_l3fwd_route_array); i++) { ret = rte_hash_add_key (ipv4_l3fwd_lookup_struct[socketid], (void *) &ipv4_l3fwd_route_array[i].key); if (ret < 0) { @@ -1852,7 +2076,7 @@ setup_hash(int socketid) } /* populate the ipv6 hash */ - for (i = 0; i < IPV6_L3FWD_NUM_ROUTES; i++) { + for (i = 0; i < RTE_DIM(ipv6_l3fwd_route_array); i++) { ret = rte_hash_add_key (ipv6_l3fwd_lookup_struct[socketid], (void *) &ipv6_l3fwd_route_array[i].key); if (ret < 0) { @@ -1889,7 +2113,7 @@ setup_lpm(int socketid) " on socket %d\n", socketid); /* populate the LPM table */ - for (i = 0; i < IPV4_L3FWD_NUM_ROUTES; i++) { + for (i = 0; i < RTE_DIM(ipv4_l3fwd_route_array); i++) { ret = rte_lpm_add(ipv4_l3fwd_lookup_struct[socketid], ipv4_l3fwd_route_array[i].ip, ipv4_l3fwd_route_array[i].depth, @@ -1970,6 +2194,8 @@ check_all_ports_link_status(uint32_t port_mask) uint8_t count, all_ports_up, print_flag = 0; uint16_t portid; struct rte_eth_link link; + int ret; + char link_status_text[RTE_ETH_LINK_MAX_STR_LEN]; printf("\nChecking link status"); fflush(stdout); @@ -1979,18 +2205,20 @@ check_all_ports_link_status(uint32_t port_mask) if ((port_mask & (1 << portid)) == 0) continue; memset(&link, 0, sizeof(link)); - rte_eth_link_get_nowait(portid, &link); + ret = rte_eth_link_get_nowait(portid, &link); + if (ret < 0) { + all_ports_up = 0; + if (print_flag == 1) + printf("Port %u link get failed: %s\n", + portid, rte_strerror(-ret)); + continue; + } /* print link status if flag set */ if (print_flag == 1) { - if (link.link_status) - printf("Port %d Link Up - speed %u " - "Mbps - %s\n", (uint8_t)portid, - (unsigned)link.link_speed, - (link.link_duplex == ETH_LINK_FULL_DUPLEX) ? - ("full-duplex") : ("half-duplex\n")); - else - printf("Port %d Link Down\n", - (uint8_t)portid); + rte_eth_link_to_str(link_status_text, + sizeof(link_status_text), &link); + printf("Port %d %s\n", portid, + link_status_text); continue; } /* clear all_ports_up flag if any link down */ @@ -2064,31 +2292,59 @@ static int check_ptype(uint16_t portid) static int init_power_library(void) { - int ret = 0, lcore_id; - for (lcore_id = 0; lcore_id < RTE_MAX_LCORE; lcore_id++) { - if (rte_lcore_is_enabled(lcore_id)) { - /* init power management library */ - ret = rte_power_init(lcore_id); - if (ret) - RTE_LOG(ERR, POWER, + enum power_management_env env; + unsigned int lcore_id; + int ret = 0; + + RTE_LCORE_FOREACH(lcore_id) { + /* init power management library */ + ret = rte_power_init(lcore_id); + if (ret) { + RTE_LOG(ERR, POWER, "Library initialization failed on core %u\n", lcore_id); + return ret; + } + /* we're not supporting the VM channel mode */ + env = rte_power_get_env(); + if (env != PM_ENV_ACPI_CPUFREQ && + env != PM_ENV_PSTATE_CPUFREQ) { + RTE_LOG(ERR, POWER, + "Only ACPI and PSTATE mode are supported\n"); + return -1; } } return ret; } + +static int +deinit_power_library(void) +{ + unsigned int lcore_id; + int ret = 0; + + RTE_LCORE_FOREACH(lcore_id) { + /* deinit power management library */ + ret = rte_power_exit(lcore_id); + if (ret) { + RTE_LOG(ERR, POWER, + "Library deinitialization failed on core %u\n", + lcore_id); + return ret; + } + } + return ret; +} + static void -update_telemetry(__attribute__((unused)) struct rte_timer *tim, - __attribute__((unused)) void *arg) +get_current_stat_values(uint64_t *values) { unsigned int lcore_id = rte_lcore_id(); struct lcore_conf *qconf; uint64_t app_eps = 0, app_fps = 0, app_br = 0; - uint64_t values[3] = {0}; - int ret; uint64_t count = 0; - RTE_LCORE_FOREACH_SLAVE(lcore_id) { + RTE_LCORE_FOREACH_WORKER(lcore_id) { qconf = &lcore_conf[lcore_id]; if (qconf->n_rx_queue == 0) continue; @@ -2104,17 +2360,41 @@ update_telemetry(__attribute__((unused)) struct rte_timer *tim, values[0] = app_eps/count; values[1] = app_fps/count; values[2] = app_br/count; - } else { - values[0] = 0; - values[1] = 0; - values[2] = 0; - } + } else + memset(values, 0, sizeof(uint64_t) * NUM_TELSTATS); + +} +static void +update_telemetry(__rte_unused struct rte_timer *tim, + __rte_unused void *arg) +{ + int ret; + uint64_t values[NUM_TELSTATS] = {0}; + + get_current_stat_values(values); ret = rte_metrics_update_values(RTE_METRICS_GLOBAL, telstats_index, values, RTE_DIM(values)); if (ret < 0) RTE_LOG(WARNING, POWER, "failed to update metrcis\n"); } + +static int +handle_app_stats(const char *cmd __rte_unused, + const char *params __rte_unused, + struct rte_tel_data *d) +{ + uint64_t values[NUM_TELSTATS] = {0}; + uint32_t i; + + rte_tel_data_start_dict(d); + get_current_stat_values(values); + for (i = 0; i < NUM_TELSTATS; i++) + rte_tel_data_add_dict_u64(d, telstats_strings[i].name, + values[i]); + return 0; +} + static void telemetry_setup_timer(void) { @@ -2156,10 +2436,10 @@ launch_timer(unsigned int lcore_id) RTE_SET_USED(lcore_id); - if (rte_get_master_lcore() != lcore_id) { - rte_panic("timer on lcore:%d which is not master core:%d\n", + if (rte_get_main_lcore() != lcore_id) { + rte_panic("timer on lcore:%d which is not main core:%d\n", lcore_id, - rte_get_master_lcore()); + rte_get_main_lcore()); } RTE_LOG(INFO, POWER, "Bring up the Timer\n"); @@ -2186,7 +2466,46 @@ launch_timer(unsigned int lcore_id) return 0; } +static int +autodetect_mode(void) +{ + RTE_LOG(NOTICE, L3FWD_POWER, "Operating mode not specified, probing frequency scaling support...\n"); + + /* + * Empty poll and telemetry modes have to be specifically requested to + * be enabled, but we can auto-detect between interrupt mode with or + * without frequency scaling. Both ACPI and pstate can be used. + */ + if (rte_power_check_env_supported(PM_ENV_ACPI_CPUFREQ)) + return APP_MODE_LEGACY; + if (rte_power_check_env_supported(PM_ENV_PSTATE_CPUFREQ)) + return APP_MODE_LEGACY; + + RTE_LOG(NOTICE, L3FWD_POWER, "Frequency scaling not supported, selecting interrupt-only mode\n"); + return APP_MODE_INTERRUPT; +} + +static const char * +mode_to_str(enum appmode mode) +{ + switch (mode) { + case APP_MODE_LEGACY: + return "legacy"; + case APP_MODE_EMPTY_POLL: + return "empty poll"; + case APP_MODE_TELEMETRY: + return "telemetry"; + case APP_MODE_INTERRUPT: + return "interrupt-only"; + case APP_MODE_PMD_MGMT: + return "pmd mgmt"; + default: + return "invalid"; + } +} + +/* Power library initialized in the main routine. 8< */ int main(int argc, char **argv) { @@ -2202,8 +2521,7 @@ main(int argc, char **argv) uint32_t dev_rxq_num, dev_txq_num; uint8_t nb_rx_queue, queue, socketid; uint16_t portid; - uint8_t num_telstats = RTE_DIM(telstats_strings); - const char *ptr_strings[num_telstats]; + const char *ptr_strings[NUM_TELSTATS]; /* catch SIGINT and restore cpufreq governor to ondemand */ signal(SIGINT, signal_exit_now); @@ -2218,13 +2536,24 @@ main(int argc, char **argv) /* init RTE timer library to be used late */ rte_timer_subsystem_init(); + /* if we're running pmd-mgmt mode, don't default to baseline mode */ + baseline_enabled = false; + /* parse application arguments (after the EAL ones) */ ret = parse_args(argc, argv); if (ret < 0) rte_exit(EXIT_FAILURE, "Invalid L3FWD parameters\n"); - if (init_power_library()) - RTE_LOG(ERR, L3FWD_POWER, "init_power_library failed\n"); + if (app_mode == APP_MODE_DEFAULT) + app_mode = autodetect_mode(); + + RTE_LOG(INFO, L3FWD_POWER, "Selected operation mode: %s\n", + mode_to_str(app_mode)); + + /* only legacy and empty poll mode rely on power library */ + if ((app_mode == APP_MODE_LEGACY || app_mode == APP_MODE_EMPTY_POLL) && + init_power_library()) + rte_exit(EXIT_FAILURE, "init_power_library failed\n"); if (update_lcore_params() < 0) rte_exit(EXIT_FAILURE, "update_lcore_params failed\n"); @@ -2246,6 +2575,9 @@ main(int argc, char **argv) /* initialize all ports */ RTE_ETH_FOREACH_DEV(portid) { struct rte_eth_conf local_port_conf = port_conf; + /* not all app modes need interrupts */ + bool need_intr = app_mode == APP_MODE_LEGACY || + app_mode == APP_MODE_INTERRUPT; /* skip ports that are not enabled */ if ((enabled_port_mask & (1 << portid)) == 0) { @@ -2257,7 +2589,12 @@ main(int argc, char **argv) printf("Initializing port %d ... ", portid ); fflush(stdout); - rte_eth_dev_info_get(portid, &dev_info); + ret = rte_eth_dev_info_get(portid, &dev_info); + if (ret != 0) + rte_exit(EXIT_FAILURE, + "Error during getting device (port %u) info: %s\n", + portid, strerror(-ret)); + dev_rxq_num = dev_info.max_rx_queues; dev_txq_num = dev_info.max_tx_queues; @@ -2274,8 +2611,17 @@ main(int argc, char **argv) nb_rx_queue, (unsigned)n_tx_queue ); /* If number of Rx queue is 0, no need to enable Rx interrupt */ if (nb_rx_queue == 0) - local_port_conf.intr_conf.rxq = 0; - rte_eth_dev_info_get(portid, &dev_info); + need_intr = false; + + if (need_intr) + local_port_conf.intr_conf.rxq = 1; + + ret = rte_eth_dev_info_get(portid, &dev_info); + if (ret != 0) + rte_exit(EXIT_FAILURE, + "Error during getting device (port %u) info: %s\n", + portid, strerror(-ret)); + if (dev_info.tx_offload_capa & DEV_TX_OFFLOAD_MBUF_FAST_FREE) local_port_conf.txmode.offloads |= DEV_TX_OFFLOAD_MBUF_FAST_FREE; @@ -2304,7 +2650,12 @@ main(int argc, char **argv) "Cannot adjust number of descriptors: err=%d, port=%d\n", ret, portid); - rte_eth_macaddr_get(portid, &ports_eth_addr[portid]); + ret = rte_eth_macaddr_get(portid, &ports_eth_addr[portid]); + if (ret < 0) + rte_exit(EXIT_FAILURE, + "Cannot get MAC address: err=%d, port=%d\n", + ret, portid); + print_ethaddr(" Address:", &ports_eth_addr[portid]); printf(", "); @@ -2382,16 +2733,13 @@ main(int argc, char **argv) qconf = &lcore_conf[lcore_id]; printf("\nInitializing rx queues on lcore %u ... ", lcore_id ); fflush(stdout); + /* init RX queues */ for(queue = 0; queue < qconf->n_rx_queue; ++queue) { struct rte_eth_rxconf rxq_conf; - struct rte_eth_dev *dev; - struct rte_eth_conf *conf; portid = qconf->rx_queue_list[queue].port_id; queueid = qconf->rx_queue_list[queue].queue_id; - dev = &rte_eth_devices[portid]; - conf = &dev->data->dev_conf; if (numa_on) socketid = \ @@ -2402,9 +2750,14 @@ main(int argc, char **argv) printf("rxq=%d,%d,%d ", portid, queueid, socketid); fflush(stdout); - rte_eth_dev_info_get(portid, &dev_info); + ret = rte_eth_dev_info_get(portid, &dev_info); + if (ret != 0) + rte_exit(EXIT_FAILURE, + "Error during getting device (port %u) info: %s\n", + portid, strerror(-ret)); + rxq_conf = dev_info.default_rxconf; - rxq_conf.offloads = conf->rxmode.offloads; + rxq_conf.offloads = port_conf.rxmode.offloads; ret = rte_eth_rx_queue_setup(portid, queueid, nb_rxd, socketid, &rxq_conf, pktmbuf_pool[socketid]); @@ -2417,11 +2770,20 @@ main(int argc, char **argv) if (add_cb_parse_ptype(portid, queueid) < 0) rte_exit(EXIT_FAILURE, "Fail to add ptype cb\n"); - } else if (!check_ptype(portid)) - rte_exit(EXIT_FAILURE, - "PMD can not provide needed ptypes\n"); + } + + if (app_mode == APP_MODE_PMD_MGMT && !baseline_enabled) { + ret = rte_power_ethdev_pmgmt_queue_enable( + lcore_id, portid, queueid, + pmgmt_type); + if (ret < 0) + rte_exit(EXIT_FAILURE, + "rte_power_ethdev_pmgmt_queue_enable: err=%d, port=%d\n", + ret, portid); + } } } + /* >8 End of power library initialization. */ printf("\n"); @@ -2441,10 +2803,20 @@ main(int argc, char **argv) * to itself through 2 cross-connected ports of the * target machine. */ - if (promiscuous_on) - rte_eth_promiscuous_enable(portid); + if (promiscuous_on) { + ret = rte_eth_promiscuous_enable(portid); + if (ret != 0) + rte_exit(EXIT_FAILURE, + "rte_eth_promiscuous_enable: err=%s, port=%u\n", + rte_strerror(-ret), portid); + } /* initialize spinlock for each port */ rte_spinlock_init(&(locks[portid])); + + if (!parse_ptype) + if (!check_ptype(portid)) + rte_exit(EXIT_FAILURE, + "PMD can not provide needed ptypes\n"); } check_all_ports_link_status(enabled_port_mask); @@ -2469,44 +2841,87 @@ main(int argc, char **argv) /* launch per-lcore init on every lcore */ if (app_mode == APP_MODE_LEGACY) { - rte_eal_mp_remote_launch(main_loop, NULL, CALL_MASTER); + rte_eal_mp_remote_launch(main_legacy_loop, NULL, CALL_MAIN); } else if (app_mode == APP_MODE_EMPTY_POLL) { empty_poll_stop = false; rte_eal_mp_remote_launch(main_empty_poll_loop, NULL, - SKIP_MASTER); - } else { + SKIP_MAIN); + } else if (app_mode == APP_MODE_TELEMETRY) { unsigned int i; /* Init metrics library */ rte_metrics_init(rte_socket_id()); /** Register stats with metrics library */ - for (i = 0; i < num_telstats; i++) + for (i = 0; i < NUM_TELSTATS; i++) ptr_strings[i] = telstats_strings[i].name; - ret = rte_metrics_reg_names(ptr_strings, num_telstats); + ret = rte_metrics_reg_names(ptr_strings, NUM_TELSTATS); if (ret >= 0) telstats_index = ret; else rte_exit(EXIT_FAILURE, "failed to register metrics names"); - RTE_LCORE_FOREACH_SLAVE(lcore_id) { + RTE_LCORE_FOREACH_WORKER(lcore_id) { rte_spinlock_init(&stats[lcore_id].telemetry_lock); } rte_timer_init(&telemetry_timer); + rte_telemetry_register_cmd("/l3fwd-power/stats", + handle_app_stats, + "Returns global power stats. Parameters: None"); rte_eal_mp_remote_launch(main_telemetry_loop, NULL, - SKIP_MASTER); + SKIP_MAIN); + } else if (app_mode == APP_MODE_INTERRUPT) { + rte_eal_mp_remote_launch(main_intr_loop, NULL, CALL_MAIN); + } else if (app_mode == APP_MODE_PMD_MGMT) { + /* reuse telemetry loop for PMD power management mode */ + rte_eal_mp_remote_launch(main_telemetry_loop, NULL, CALL_MAIN); } if (app_mode == APP_MODE_EMPTY_POLL || app_mode == APP_MODE_TELEMETRY) launch_timer(rte_lcore_id()); - RTE_LCORE_FOREACH_SLAVE(lcore_id) { + RTE_LCORE_FOREACH_WORKER(lcore_id) { if (rte_eal_wait_lcore(lcore_id) < 0) return -1; } + if (app_mode == APP_MODE_PMD_MGMT) { + for (lcore_id = 0; lcore_id < RTE_MAX_LCORE; lcore_id++) { + if (rte_lcore_is_enabled(lcore_id) == 0) + continue; + qconf = &lcore_conf[lcore_id]; + for (queue = 0; queue < qconf->n_rx_queue; ++queue) { + portid = qconf->rx_queue_list[queue].port_id; + queueid = qconf->rx_queue_list[queue].queue_id; + + rte_power_ethdev_pmgmt_queue_disable(lcore_id, + portid, queueid); + } + } + } + + RTE_ETH_FOREACH_DEV(portid) + { + if ((enabled_port_mask & (1 << portid)) == 0) + continue; + + ret = rte_eth_dev_stop(portid); + if (ret != 0) + RTE_LOG(ERR, L3FWD_POWER, "rte_eth_dev_stop: err=%d, port=%u\n", + ret, portid); + + rte_eth_dev_close(portid); + } + if (app_mode == APP_MODE_EMPTY_POLL) rte_power_empty_poll_stat_free(); + if ((app_mode == APP_MODE_LEGACY || app_mode == APP_MODE_EMPTY_POLL) && + deinit_power_library()) + rte_exit(EXIT_FAILURE, "deinit_power_library failed\n"); + + if (rte_eal_cleanup() < 0) + RTE_LOG(ERR, L3FWD_POWER, "EAL cleanup failed\n"); + return 0; }