X-Git-Url: http://git.droids-corp.org/?a=blobdiff_plain;f=examples%2Fioat%2Fioatfwd.c;h=7255ff3c9e5e0d978ca9d321889cb84eed69d4a7;hb=e0b0e55c8f1515c2be6d932be977d926ab7ffa28;hp=2a4da948aab0ddd448d17a89f448a08803f0681a;hpb=ae368c96ee53380532eadfd0035c655e8b9a8d2c;p=dpdk.git diff --git a/examples/ioat/ioatfwd.c b/examples/ioat/ioatfwd.c index 2a4da948aa..7255ff3c9e 100644 --- a/examples/ioat/ioatfwd.c +++ b/examples/ioat/ioatfwd.c @@ -48,6 +48,23 @@ struct rxtx_transmission_config { uint16_t nb_lcores; }; +/* per-port statistics struct */ +struct ioat_port_statistics { + uint64_t rx[RTE_MAX_ETHPORTS]; + uint64_t tx[RTE_MAX_ETHPORTS]; + uint64_t tx_dropped[RTE_MAX_ETHPORTS]; + uint64_t copy_dropped[RTE_MAX_ETHPORTS]; +}; +struct ioat_port_statistics port_statistics; + +struct total_statistics { + uint64_t total_packets_dropped; + uint64_t total_packets_tx; + uint64_t total_packets_rx; + uint64_t total_successful_enqueues; + uint64_t total_failed_enqueues; +}; + typedef enum copy_mode_t { #define COPY_MODE_SW "sw" COPY_MODE_SW_NUM, @@ -89,6 +106,207 @@ static struct rte_ether_addr ioat_ports_eth_addr[RTE_MAX_ETHPORTS]; static struct rte_eth_dev_tx_buffer *tx_buffer[RTE_MAX_ETHPORTS]; struct rte_mempool *ioat_pktmbuf_pool; +/* Print out statistics for one port. */ +static void +print_port_stats(uint16_t port_id) +{ + printf("\nStatistics for port %u ------------------------------" + "\nPackets sent: %34"PRIu64 + "\nPackets received: %30"PRIu64 + "\nPackets dropped on tx: %25"PRIu64 + "\nPackets dropped on copy: %23"PRIu64, + port_id, + port_statistics.tx[port_id], + port_statistics.rx[port_id], + port_statistics.tx_dropped[port_id], + port_statistics.copy_dropped[port_id]); +} + +/* Print out statistics for one IOAT rawdev device. */ +static void +print_rawdev_stats(uint32_t dev_id, uint64_t *xstats, + unsigned int *ids_xstats, uint16_t nb_xstats, + struct rte_rawdev_xstats_name *names_xstats) +{ + uint16_t i; + + printf("\nIOAT channel %u", dev_id); + for (i = 0; i < nb_xstats; i++) + printf("\n\t %s: %*"PRIu64, + names_xstats[ids_xstats[i]].name, + (int)(37 - strlen(names_xstats[ids_xstats[i]].name)), + xstats[i]); +} + +static void +print_total_stats(struct total_statistics *ts) +{ + printf("\nAggregate statistics ===============================" + "\nTotal packets Tx: %24"PRIu64" [pps]" + "\nTotal packets Rx: %24"PRIu64" [pps]" + "\nTotal packets dropped: %19"PRIu64" [pps]", + ts->total_packets_tx, + ts->total_packets_rx, + ts->total_packets_dropped); + + if (copy_mode == COPY_MODE_IOAT_NUM) { + printf("\nTotal IOAT successful enqueues: %8"PRIu64" [enq/s]" + "\nTotal IOAT failed enqueues: %12"PRIu64" [enq/s]", + ts->total_successful_enqueues, + ts->total_failed_enqueues); + } + + printf("\n====================================================\n"); +} + +/* Print out statistics on packets dropped. */ +static void +print_stats(char *prgname) +{ + struct total_statistics ts, delta_ts; + uint32_t i, port_id, dev_id; + struct rte_rawdev_xstats_name *names_xstats; + uint64_t *xstats; + unsigned int *ids_xstats, nb_xstats; + char status_string[120]; /* to print at the top of the output */ + int status_strlen; + int ret; + + const char clr[] = { 27, '[', '2', 'J', '\0' }; + const char topLeft[] = { 27, '[', '1', ';', '1', 'H', '\0' }; + + status_strlen = snprintf(status_string, sizeof(status_string), + "%s, ", prgname); + status_strlen += snprintf(status_string + status_strlen, + sizeof(status_string) - status_strlen, + "Worker Threads = %d, ", + rte_lcore_count() > 2 ? 2 : 1); + status_strlen += snprintf(status_string + status_strlen, + sizeof(status_string) - status_strlen, + "Copy Mode = %s,\n", copy_mode == COPY_MODE_SW_NUM ? + COPY_MODE_SW : COPY_MODE_IOAT); + status_strlen += snprintf(status_string + status_strlen, + sizeof(status_string) - status_strlen, + "Updating MAC = %s, ", mac_updating ? + "enabled" : "disabled"); + status_strlen += snprintf(status_string + status_strlen, + sizeof(status_string) - status_strlen, + "Rx Queues = %d, ", nb_queues); + status_strlen += snprintf(status_string + status_strlen, + sizeof(status_string) - status_strlen, + "Ring Size = %d\n", ring_size); + + /* Allocate memory for xstats names and values */ + ret = rte_rawdev_xstats_names_get( + cfg.ports[0].ioat_ids[0], NULL, 0); + if (ret < 0) + return; + nb_xstats = (unsigned int)ret; + + names_xstats = malloc(sizeof(*names_xstats) * nb_xstats); + if (names_xstats == NULL) { + rte_exit(EXIT_FAILURE, + "Error allocating xstat names memory\n"); + } + rte_rawdev_xstats_names_get(cfg.ports[0].ioat_ids[0], + names_xstats, nb_xstats); + + ids_xstats = malloc(sizeof(*ids_xstats) * 2); + if (ids_xstats == NULL) { + rte_exit(EXIT_FAILURE, + "Error allocating xstat ids_xstats memory\n"); + } + + xstats = malloc(sizeof(*xstats) * 2); + if (xstats == NULL) { + rte_exit(EXIT_FAILURE, + "Error allocating xstat memory\n"); + } + + /* Get failed/successful enqueues stats index */ + ids_xstats[0] = ids_xstats[1] = nb_xstats; + for (i = 0; i < nb_xstats; i++) { + if (!strcmp(names_xstats[i].name, "failed_enqueues")) + ids_xstats[0] = i; + else if (!strcmp(names_xstats[i].name, "successful_enqueues")) + ids_xstats[1] = i; + if (ids_xstats[0] < nb_xstats && ids_xstats[1] < nb_xstats) + break; + } + if (ids_xstats[0] == nb_xstats || ids_xstats[1] == nb_xstats) { + rte_exit(EXIT_FAILURE, + "Error getting failed/successful enqueues stats index\n"); + } + + memset(&ts, 0, sizeof(struct total_statistics)); + + while (!force_quit) { + /* Sleep for 1 second each round - init sleep allows reading + * messages from app startup. + */ + sleep(1); + + /* Clear screen and move to top left */ + printf("%s%s", clr, topLeft); + + memset(&delta_ts, 0, sizeof(struct total_statistics)); + + printf("%s", status_string); + + for (i = 0; i < cfg.nb_ports; i++) { + port_id = cfg.ports[i].rxtx_port; + print_port_stats(port_id); + + delta_ts.total_packets_dropped += + port_statistics.tx_dropped[port_id] + + port_statistics.copy_dropped[port_id]; + delta_ts.total_packets_tx += + port_statistics.tx[port_id]; + delta_ts.total_packets_rx += + port_statistics.rx[port_id]; + + if (copy_mode == COPY_MODE_IOAT_NUM) { + uint32_t j; + + for (j = 0; j < cfg.ports[i].nb_queues; j++) { + dev_id = cfg.ports[i].ioat_ids[j]; + rte_rawdev_xstats_get(dev_id, + ids_xstats, xstats, 2); + + print_rawdev_stats(dev_id, xstats, + ids_xstats, 2, names_xstats); + + delta_ts.total_failed_enqueues += + xstats[ids_xstats[0]]; + delta_ts.total_successful_enqueues += + xstats[ids_xstats[1]]; + } + } + } + + delta_ts.total_packets_tx -= ts.total_packets_tx; + delta_ts.total_packets_rx -= ts.total_packets_rx; + delta_ts.total_packets_dropped -= ts.total_packets_dropped; + delta_ts.total_failed_enqueues -= ts.total_failed_enqueues; + delta_ts.total_successful_enqueues -= + ts.total_successful_enqueues; + + printf("\n"); + print_total_stats(&delta_ts); + + ts.total_packets_tx += delta_ts.total_packets_tx; + ts.total_packets_rx += delta_ts.total_packets_rx; + ts.total_packets_dropped += delta_ts.total_packets_dropped; + ts.total_failed_enqueues += delta_ts.total_failed_enqueues; + ts.total_successful_enqueues += + delta_ts.total_successful_enqueues; + } + + free(names_xstats); + free(xstats); + free(ids_xstats); +} + static void update_mac_addrs(struct rte_mbuf *m, uint32_t dest_portid) { @@ -121,6 +339,49 @@ pktmbuf_sw_copy(struct rte_mbuf *src, struct rte_mbuf *dst) rte_pktmbuf_mtod(src, char *), src->data_len); } +static uint32_t +ioat_enqueue_packets(struct rte_mbuf **pkts, + uint32_t nb_rx, uint16_t dev_id) +{ + int ret; + uint32_t i; + struct rte_mbuf *pkts_copy[MAX_PKT_BURST]; + + const uint64_t addr_offset = RTE_PTR_DIFF(pkts[0]->buf_addr, + &pkts[0]->rearm_data); + + ret = rte_mempool_get_bulk(ioat_pktmbuf_pool, + (void *)pkts_copy, nb_rx); + + if (unlikely(ret < 0)) + rte_exit(EXIT_FAILURE, "Unable to allocate memory.\n"); + + for (i = 0; i < nb_rx; i++) { + /* Perform data copy */ + ret = rte_ioat_enqueue_copy(dev_id, + pkts[i]->buf_iova + - addr_offset, + pkts_copy[i]->buf_iova + - addr_offset, + rte_pktmbuf_data_len(pkts[i]) + + addr_offset, + (uintptr_t)pkts[i], + (uintptr_t)pkts_copy[i], + 0 /* nofence */); + + if (ret != 1) + break; + } + + ret = i; + /* Free any not enqueued packets. */ + rte_mempool_put_bulk(ioat_pktmbuf_pool, (void *)&pkts[i], nb_rx - i); + rte_mempool_put_bulk(ioat_pktmbuf_pool, (void *)&pkts_copy[i], + nb_rx - i); + + return ret; +} + /* Receive packets on one port and enqueue to IOAT rawdev or rte_ring. */ static void ioat_rx_port(struct rxtx_port_config *rx_config) @@ -136,8 +397,14 @@ ioat_rx_port(struct rxtx_port_config *rx_config) if (nb_rx == 0) continue; + port_statistics.rx[rx_config->rxtx_port] += nb_rx; + if (copy_mode == COPY_MODE_IOAT_NUM) { /* Perform packet hardware copy */ + nb_enq = ioat_enqueue_packets(pkts_burst, + nb_rx, rx_config->ioat_ids[i]); + if (nb_enq > 0) + rte_ioat_do_copies(rx_config->ioat_ids[i]); } else { /* Perform packet software copy, free source packets */ int ret; @@ -166,6 +433,9 @@ ioat_rx_port(struct rxtx_port_config *rx_config) (void *)&pkts_burst_copy[nb_enq], nb_rx - nb_enq); } + + port_statistics.copy_dropped[rx_config->rxtx_port] += + (nb_rx - nb_enq); } } @@ -174,11 +444,15 @@ static void ioat_tx_port(struct rxtx_port_config *tx_config) { uint32_t i, j, nb_dq = 0; + struct rte_mbuf *mbufs_src[MAX_PKT_BURST]; struct rte_mbuf *mbufs_dst[MAX_PKT_BURST]; for (i = 0; i < tx_config->nb_queues; i++) { if (copy_mode == COPY_MODE_IOAT_NUM) { /* Deque the mbufs from IOAT device. */ + nb_dq = rte_ioat_completed_copies( + tx_config->ioat_ids[i], MAX_PKT_BURST, + (void *)mbufs_src, (void *)mbufs_dst); } else { /* Deque the mbufs from rx_to_tx_ring. */ nb_dq = rte_ring_dequeue_burst( @@ -186,9 +460,13 @@ ioat_tx_port(struct rxtx_port_config *tx_config) MAX_PKT_BURST, NULL); } - if (nb_dq == 0) + if ((int32_t) nb_dq <= 0) return; + if (copy_mode == COPY_MODE_IOAT_NUM) + rte_mempool_put_bulk(ioat_pktmbuf_pool, + (void *)mbufs_src, nb_dq); + /* Update macs if enabled */ if (mac_updating) { for (j = 0; j < nb_dq; j++) @@ -200,6 +478,8 @@ ioat_tx_port(struct rxtx_port_config *tx_config) tx_config->rxtx_port, 0, (void *)mbufs_dst, nb_dq); + port_statistics.tx[tx_config->rxtx_port] += nb_tx; + /* Free any unsent packets. */ if (unlikely(nb_tx < nb_dq)) rte_mempool_put_bulk(ioat_pktmbuf_pool, @@ -208,6 +488,36 @@ ioat_tx_port(struct rxtx_port_config *tx_config) } } +/* Main rx processing loop for IOAT rawdev. */ +static void +rx_main_loop(void) +{ + uint16_t i; + uint16_t nb_ports = cfg.nb_ports; + + RTE_LOG(INFO, IOAT, "Entering main rx loop for copy on lcore %u\n", + rte_lcore_id()); + + while (!force_quit) + for (i = 0; i < nb_ports; i++) + ioat_rx_port(&cfg.ports[i]); +} + +/* Main tx processing loop for hardware copy. */ +static void +tx_main_loop(void) +{ + uint16_t i; + uint16_t nb_ports = cfg.nb_ports; + + RTE_LOG(INFO, IOAT, "Entering main tx loop for copy on lcore %u\n", + rte_lcore_id()); + + while (!force_quit) + for (i = 0; i < nb_ports; i++) + ioat_tx_port(&cfg.ports[i]); +} + /* Main rx and tx loop if only one slave lcore available */ static void rxtx_main_loop(void) @@ -232,9 +542,19 @@ static void start_forwarding_cores(void) RTE_LOG(INFO, IOAT, "Entering %s on lcore %u\n", __func__, rte_lcore_id()); - lcore_id = rte_get_next_lcore(lcore_id, true, true); - rte_eal_remote_launch((lcore_function_t *)rxtx_main_loop, - NULL, lcore_id); + if (cfg.nb_lcores == 1) { + lcore_id = rte_get_next_lcore(lcore_id, true, true); + rte_eal_remote_launch((lcore_function_t *)rxtx_main_loop, + NULL, lcore_id); + } else if (cfg.nb_lcores > 1) { + lcore_id = rte_get_next_lcore(lcore_id, true, true); + rte_eal_remote_launch((lcore_function_t *)rx_main_loop, + NULL, lcore_id); + + lcore_id = rte_get_next_lcore(lcore_id, true, true); + rte_eal_remote_launch((lcore_function_t *)tx_main_loop, NULL, + lcore_id); + } } /* Display usage */ @@ -377,7 +697,7 @@ check_link_status(uint32_t port_mask) { uint16_t portid; struct rte_eth_link link; - int retval = 0; + int ret, link_status = 0; printf("\nChecking link status\n"); RTE_ETH_FOREACH_DEV(portid) { @@ -385,7 +705,12 @@ check_link_status(uint32_t port_mask) continue; memset(&link, 0, sizeof(link)); - rte_eth_link_get(portid, &link); + ret = rte_eth_link_get(portid, &link); + if (ret < 0) { + printf("Port %u link get failed: err=%d\n", + portid, ret); + continue; + } /* Print link status */ if (link.link_status) { @@ -394,11 +719,58 @@ check_link_status(uint32_t port_mask) portid, link.link_speed, (link.link_duplex == ETH_LINK_FULL_DUPLEX) ? ("full-duplex") : ("half-duplex\n")); - retval = 1; + link_status = 1; } else printf("Port %d Link Down\n", portid); } - return retval; + return link_status; +} + +static void +configure_rawdev_queue(uint32_t dev_id) +{ + struct rte_ioat_rawdev_config dev_config = { .ring_size = ring_size }; + struct rte_rawdev_info info = { .dev_private = &dev_config }; + + if (rte_rawdev_configure(dev_id, &info) != 0) { + rte_exit(EXIT_FAILURE, + "Error with rte_rawdev_configure()\n"); + } + if (rte_rawdev_start(dev_id) != 0) { + rte_exit(EXIT_FAILURE, + "Error with rte_rawdev_start()\n"); + } +} + +static void +assign_rawdevs(void) +{ + uint16_t nb_rawdev = 0, rdev_id = 0; + uint32_t i, j; + + for (i = 0; i < cfg.nb_ports; i++) { + for (j = 0; j < cfg.ports[i].nb_queues; j++) { + struct rte_rawdev_info rdev_info = { 0 }; + + do { + if (rdev_id == rte_rawdev_count()) + goto end; + rte_rawdev_info_get(rdev_id++, &rdev_info); + } while (rdev_info.driver_name == NULL || + strcmp(rdev_info.driver_name, + IOAT_PMD_RAWDEV_NAME_STR) != 0); + + cfg.ports[i].ioat_ids[j] = rdev_id - 1; + configure_rawdev_queue(cfg.ports[i].ioat_ids[j]); + ++nb_rawdev; + } + } +end: + if (nb_rawdev < cfg.nb_ports * cfg.ports[0].nb_queues) + rte_exit(EXIT_FAILURE, + "Not enough IOAT rawdevs (%u) for all queues (%u).\n", + nb_rawdev, cfg.nb_ports * cfg.ports[0].nb_queues); + RTE_LOG(INFO, IOAT, "Number of used rawdevs: %u.\n", nb_rawdev); } static void @@ -457,7 +829,11 @@ port_init(uint16_t portid, struct rte_mempool *mbuf_pool, uint16_t nb_queues) /* Init port */ printf("Initializing port %u... ", portid); fflush(stdout); - rte_eth_dev_info_get(portid, &dev_info); + ret = rte_eth_dev_info_get(portid, &dev_info); + if (ret < 0) + rte_exit(EXIT_FAILURE, "Cannot get device info: %s, port=%u\n", + rte_strerror(-ret), portid); + local_port_conf.rx_adv_conf.rss_conf.rss_hf &= dev_info.flow_type_rss_offloads; if (dev_info.tx_offload_capa & DEV_TX_OFFLOAD_MBUF_FAST_FREE) @@ -512,6 +888,14 @@ port_init(uint16_t portid, struct rte_mempool *mbuf_pool, uint16_t nb_queues) rte_eth_tx_buffer_init(tx_buffer[portid], MAX_PKT_BURST); + ret = rte_eth_tx_buffer_set_err_callback(tx_buffer[portid], + rte_eth_tx_buffer_count_callback, + &port_statistics.tx_dropped[portid]); + if (ret < 0) + rte_exit(EXIT_FAILURE, + "Cannot set error callback for tx buffer on port %u\n", + portid); + /* Start device */ ret = rte_eth_dev_start(portid); if (ret < 0) @@ -588,6 +972,9 @@ main(int argc, char **argv) RTE_ETH_FOREACH_DEV(portid) port_init(portid, ioat_pktmbuf_pool, nb_queues); + /* Initialize port xstats */ + memset(&port_statistics, 0, sizeof(port_statistics)); + while (!check_link_status(ioat_enabled_port_mask) && !force_quit) sleep(1); @@ -597,18 +984,31 @@ main(int argc, char **argv) rte_exit(EXIT_FAILURE, "There should be at least one slave lcore.\n"); - assign_rings(); + if (copy_mode == COPY_MODE_IOAT_NUM) + assign_rawdevs(); + else /* copy_mode == COPY_MODE_SW_NUM */ + assign_rings(); start_forwarding_cores(); + /* master core prints stats while other cores forward */ + print_stats(argv[0]); /* force_quit is true when we get here */ rte_eal_mp_wait_lcore(); + uint32_t j; for (i = 0; i < cfg.nb_ports; i++) { printf("Closing port %d\n", cfg.ports[i].rxtx_port); rte_eth_dev_stop(cfg.ports[i].rxtx_port); rte_eth_dev_close(cfg.ports[i].rxtx_port); - rte_ring_free(cfg.ports[i].rx_to_tx_ring); + if (copy_mode == COPY_MODE_IOAT_NUM) { + for (j = 0; j < cfg.ports[i].nb_queues; j++) { + printf("Stopping rawdev %d\n", + cfg.ports[i].ioat_ids[j]); + rte_rawdev_stop(cfg.ports[i].ioat_ids[j]); + } + } else /* copy_mode == COPY_MODE_SW_NUM */ + rte_ring_free(cfg.ports[i].rx_to_tx_ring); } printf("Bye...\n");