+/* Print out statistics for one port. */
+static void
+print_port_stats(uint16_t port_id)
+{
+ printf("\nStatistics for port %u ------------------------------"
+ "\nPackets sent: %34"PRIu64
+ "\nPackets received: %30"PRIu64
+ "\nPackets dropped on tx: %25"PRIu64
+ "\nPackets dropped on copy: %23"PRIu64,
+ port_id,
+ port_statistics.tx[port_id],
+ port_statistics.rx[port_id],
+ port_statistics.tx_dropped[port_id],
+ port_statistics.copy_dropped[port_id]);
+}
+
+/* Print out statistics for one IOAT rawdev device. */
+static void
+print_rawdev_stats(uint32_t dev_id, uint64_t *xstats,
+ unsigned int *ids_xstats, uint16_t nb_xstats,
+ struct rte_rawdev_xstats_name *names_xstats)
+{
+ uint16_t i;
+
+ printf("\nIOAT channel %u", dev_id);
+ for (i = 0; i < nb_xstats; i++)
+ printf("\n\t %s: %*"PRIu64,
+ names_xstats[ids_xstats[i]].name,
+ (int)(37 - strlen(names_xstats[ids_xstats[i]].name)),
+ xstats[i]);
+}
+
+static void
+print_total_stats(struct total_statistics *ts)
+{
+ printf("\nAggregate statistics ==============================="
+ "\nTotal packets Tx: %24"PRIu64" [pps]"
+ "\nTotal packets Rx: %24"PRIu64" [pps]"
+ "\nTotal packets dropped: %19"PRIu64" [pps]",
+ ts->total_packets_tx,
+ ts->total_packets_rx,
+ ts->total_packets_dropped);
+
+ if (copy_mode == COPY_MODE_IOAT_NUM) {
+ printf("\nTotal IOAT successful enqueues: %8"PRIu64" [enq/s]"
+ "\nTotal IOAT failed enqueues: %12"PRIu64" [enq/s]",
+ ts->total_successful_enqueues,
+ ts->total_failed_enqueues);
+ }
+
+ printf("\n====================================================\n");
+}
+
+/* Print out statistics on packets dropped. */
+static void
+print_stats(char *prgname)
+{
+ struct total_statistics ts, delta_ts;
+ uint32_t i, port_id, dev_id;
+ struct rte_rawdev_xstats_name *names_xstats;
+ uint64_t *xstats;
+ unsigned int *ids_xstats, nb_xstats;
+ char status_string[120]; /* to print at the top of the output */
+ int status_strlen;
+
+
+ const char clr[] = { 27, '[', '2', 'J', '\0' };
+ const char topLeft[] = { 27, '[', '1', ';', '1', 'H', '\0' };
+
+ status_strlen = snprintf(status_string, sizeof(status_string),
+ "%s, ", prgname);
+ status_strlen += snprintf(status_string + status_strlen,
+ sizeof(status_string) - status_strlen,
+ "Worker Threads = %d, ",
+ rte_lcore_count() > 2 ? 2 : 1);
+ status_strlen += snprintf(status_string + status_strlen,
+ sizeof(status_string) - status_strlen,
+ "Copy Mode = %s,\n", copy_mode == COPY_MODE_SW_NUM ?
+ COPY_MODE_SW : COPY_MODE_IOAT);
+ status_strlen += snprintf(status_string + status_strlen,
+ sizeof(status_string) - status_strlen,
+ "Updating MAC = %s, ", mac_updating ?
+ "enabled" : "disabled");
+ status_strlen += snprintf(status_string + status_strlen,
+ sizeof(status_string) - status_strlen,
+ "Rx Queues = %d, ", nb_queues);
+ status_strlen += snprintf(status_string + status_strlen,
+ sizeof(status_string) - status_strlen,
+ "Ring Size = %d\n", ring_size);
+
+ /* Allocate memory for xstats names and values */
+ nb_xstats = rte_rawdev_xstats_names_get(
+ cfg.ports[0].ioat_ids[0], NULL, 0);
+
+ names_xstats = malloc(sizeof(*names_xstats) * nb_xstats);
+ if (names_xstats == NULL) {
+ rte_exit(EXIT_FAILURE,
+ "Error allocating xstat names memory\n");
+ }
+ rte_rawdev_xstats_names_get(cfg.ports[0].ioat_ids[0],
+ names_xstats, nb_xstats);
+
+ ids_xstats = malloc(sizeof(*ids_xstats) * 2);
+ if (ids_xstats == NULL) {
+ rte_exit(EXIT_FAILURE,
+ "Error allocating xstat ids_xstats memory\n");
+ }
+
+ xstats = malloc(sizeof(*xstats) * 2);
+ if (xstats == NULL) {
+ rte_exit(EXIT_FAILURE,
+ "Error allocating xstat memory\n");
+ }
+
+ /* Get failed/successful enqueues stats index */
+ ids_xstats[0] = ids_xstats[1] = nb_xstats;
+ for (i = 0; i < nb_xstats; i++) {
+ if (!strcmp(names_xstats[i].name, "failed_enqueues"))
+ ids_xstats[0] = i;
+ else if (!strcmp(names_xstats[i].name, "successful_enqueues"))
+ ids_xstats[1] = i;
+ if (ids_xstats[0] < nb_xstats && ids_xstats[1] < nb_xstats)
+ break;
+ }
+ if (ids_xstats[0] == nb_xstats || ids_xstats[1] == nb_xstats) {
+ rte_exit(EXIT_FAILURE,
+ "Error getting failed/successful enqueues stats index\n");
+ }
+
+ memset(&ts, 0, sizeof(struct total_statistics));
+
+ while (!force_quit) {
+ /* Sleep for 1 second each round - init sleep allows reading
+ * messages from app startup.
+ */
+ sleep(1);
+
+ /* Clear screen and move to top left */
+ printf("%s%s", clr, topLeft);
+
+ memset(&delta_ts, 0, sizeof(struct total_statistics));
+
+ printf("%s", status_string);
+
+ for (i = 0; i < cfg.nb_ports; i++) {
+ port_id = cfg.ports[i].rxtx_port;
+ print_port_stats(port_id);
+
+ delta_ts.total_packets_dropped +=
+ port_statistics.tx_dropped[port_id]
+ + port_statistics.copy_dropped[port_id];
+ delta_ts.total_packets_tx +=
+ port_statistics.tx[port_id];
+ delta_ts.total_packets_rx +=
+ port_statistics.rx[port_id];
+
+ if (copy_mode == COPY_MODE_IOAT_NUM) {
+ uint32_t j;
+
+ for (j = 0; j < cfg.ports[i].nb_queues; j++) {
+ dev_id = cfg.ports[i].ioat_ids[j];
+ rte_rawdev_xstats_get(dev_id,
+ ids_xstats, xstats, 2);
+
+ print_rawdev_stats(dev_id, xstats,
+ ids_xstats, 2, names_xstats);
+
+ delta_ts.total_failed_enqueues +=
+ xstats[ids_xstats[0]];
+ delta_ts.total_successful_enqueues +=
+ xstats[ids_xstats[1]];
+ }
+ }
+ }
+
+ delta_ts.total_packets_tx -= ts.total_packets_tx;
+ delta_ts.total_packets_rx -= ts.total_packets_rx;
+ delta_ts.total_packets_dropped -= ts.total_packets_dropped;
+ delta_ts.total_failed_enqueues -= ts.total_failed_enqueues;
+ delta_ts.total_successful_enqueues -=
+ ts.total_successful_enqueues;
+
+ printf("\n");
+ print_total_stats(&delta_ts);
+
+ ts.total_packets_tx += delta_ts.total_packets_tx;
+ ts.total_packets_rx += delta_ts.total_packets_rx;
+ ts.total_packets_dropped += delta_ts.total_packets_dropped;
+ ts.total_failed_enqueues += delta_ts.total_failed_enqueues;
+ ts.total_successful_enqueues +=
+ delta_ts.total_successful_enqueues;
+ }
+
+ free(names_xstats);
+ free(xstats);
+ free(ids_xstats);
+}
+
+static void
+update_mac_addrs(struct rte_mbuf *m, uint32_t dest_portid)
+{
+ struct rte_ether_hdr *eth;
+ void *tmp;
+
+ eth = rte_pktmbuf_mtod(m, struct rte_ether_hdr *);
+
+ /* 02:00:00:00:00:xx - overwriting 2 bytes of source address but
+ * it's acceptable cause it gets overwritten by rte_ether_addr_copy
+ */
+ tmp = ð->d_addr.addr_bytes[0];
+ *((uint64_t *)tmp) = 0x000000000002 + ((uint64_t)dest_portid << 40);
+
+ /* src addr */
+ rte_ether_addr_copy(&ioat_ports_eth_addr[dest_portid], ð->s_addr);
+}
+
+static inline void
+pktmbuf_sw_copy(struct rte_mbuf *src, struct rte_mbuf *dst)
+{
+ /* Copy packet metadata */
+ rte_memcpy(&dst->rearm_data,
+ &src->rearm_data,
+ offsetof(struct rte_mbuf, cacheline1)
+ - offsetof(struct rte_mbuf, rearm_data));
+
+ /* Copy packet data */
+ rte_memcpy(rte_pktmbuf_mtod(dst, char *),
+ rte_pktmbuf_mtod(src, char *), src->data_len);
+}
+
+static uint32_t
+ioat_enqueue_packets(struct rte_mbuf **pkts,
+ uint32_t nb_rx, uint16_t dev_id)
+{
+ int ret;
+ uint32_t i;
+ struct rte_mbuf *pkts_copy[MAX_PKT_BURST];
+
+ const uint64_t addr_offset = RTE_PTR_DIFF(pkts[0]->buf_addr,
+ &pkts[0]->rearm_data);
+
+ ret = rte_mempool_get_bulk(ioat_pktmbuf_pool,
+ (void *)pkts_copy, nb_rx);
+
+ if (unlikely(ret < 0))
+ rte_exit(EXIT_FAILURE, "Unable to allocate memory.\n");
+
+ for (i = 0; i < nb_rx; i++) {
+ /* Perform data copy */
+ ret = rte_ioat_enqueue_copy(dev_id,
+ pkts[i]->buf_iova
+ - addr_offset,
+ pkts_copy[i]->buf_iova
+ - addr_offset,
+ rte_pktmbuf_data_len(pkts[i])
+ + addr_offset,
+ (uintptr_t)pkts[i],
+ (uintptr_t)pkts_copy[i],
+ 0 /* nofence */);
+
+ if (ret != 1)
+ break;
+ }
+
+ ret = i;
+ /* Free any not enqueued packets. */
+ rte_mempool_put_bulk(ioat_pktmbuf_pool, (void *)&pkts[i], nb_rx - i);
+ rte_mempool_put_bulk(ioat_pktmbuf_pool, (void *)&pkts_copy[i],
+ nb_rx - i);
+
+ return ret;
+}
+
+/* Receive packets on one port and enqueue to IOAT rawdev or rte_ring. */
+static void
+ioat_rx_port(struct rxtx_port_config *rx_config)
+{
+ uint32_t nb_rx, nb_enq, i, j;
+ struct rte_mbuf *pkts_burst[MAX_PKT_BURST];
+
+ for (i = 0; i < rx_config->nb_queues; i++) {
+
+ nb_rx = rte_eth_rx_burst(rx_config->rxtx_port, i,
+ pkts_burst, MAX_PKT_BURST);
+
+ if (nb_rx == 0)
+ continue;
+
+ port_statistics.rx[rx_config->rxtx_port] += nb_rx;
+
+ if (copy_mode == COPY_MODE_IOAT_NUM) {
+ /* Perform packet hardware copy */
+ nb_enq = ioat_enqueue_packets(pkts_burst,
+ nb_rx, rx_config->ioat_ids[i]);
+ if (nb_enq > 0)
+ rte_ioat_do_copies(rx_config->ioat_ids[i]);
+ } else {
+ /* Perform packet software copy, free source packets */
+ int ret;
+ struct rte_mbuf *pkts_burst_copy[MAX_PKT_BURST];
+
+ ret = rte_mempool_get_bulk(ioat_pktmbuf_pool,
+ (void *)pkts_burst_copy, nb_rx);
+
+ if (unlikely(ret < 0))
+ rte_exit(EXIT_FAILURE,
+ "Unable to allocate memory.\n");
+
+ for (j = 0; j < nb_rx; j++)
+ pktmbuf_sw_copy(pkts_burst[j],
+ pkts_burst_copy[j]);
+
+ rte_mempool_put_bulk(ioat_pktmbuf_pool,
+ (void *)pkts_burst, nb_rx);
+
+ nb_enq = rte_ring_enqueue_burst(
+ rx_config->rx_to_tx_ring,
+ (void *)pkts_burst_copy, nb_rx, NULL);
+
+ /* Free any not enqueued packets. */
+ rte_mempool_put_bulk(ioat_pktmbuf_pool,
+ (void *)&pkts_burst_copy[nb_enq],
+ nb_rx - nb_enq);
+ }
+
+ port_statistics.copy_dropped[rx_config->rxtx_port] +=
+ (nb_rx - nb_enq);
+ }
+}
+
+/* Transmit packets from IOAT rawdev/rte_ring for one port. */
+static void
+ioat_tx_port(struct rxtx_port_config *tx_config)
+{
+ uint32_t i, j, nb_dq = 0;
+ struct rte_mbuf *mbufs_src[MAX_PKT_BURST];
+ struct rte_mbuf *mbufs_dst[MAX_PKT_BURST];
+
+ for (i = 0; i < tx_config->nb_queues; i++) {
+ if (copy_mode == COPY_MODE_IOAT_NUM) {
+ /* Deque the mbufs from IOAT device. */
+ nb_dq = rte_ioat_completed_copies(
+ tx_config->ioat_ids[i], MAX_PKT_BURST,
+ (void *)mbufs_src, (void *)mbufs_dst);
+ } else {
+ /* Deque the mbufs from rx_to_tx_ring. */
+ nb_dq = rte_ring_dequeue_burst(
+ tx_config->rx_to_tx_ring, (void *)mbufs_dst,
+ MAX_PKT_BURST, NULL);
+ }
+
+ if (nb_dq == 0)
+ return;
+
+ if (copy_mode == COPY_MODE_IOAT_NUM)
+ rte_mempool_put_bulk(ioat_pktmbuf_pool,
+ (void *)mbufs_src, nb_dq);
+
+ /* Update macs if enabled */
+ if (mac_updating) {
+ for (j = 0; j < nb_dq; j++)
+ update_mac_addrs(mbufs_dst[j],
+ tx_config->rxtx_port);
+ }
+
+ const uint16_t nb_tx = rte_eth_tx_burst(
+ tx_config->rxtx_port, 0,
+ (void *)mbufs_dst, nb_dq);
+
+ port_statistics.tx[tx_config->rxtx_port] += nb_tx;
+
+ /* Free any unsent packets. */
+ if (unlikely(nb_tx < nb_dq))
+ rte_mempool_put_bulk(ioat_pktmbuf_pool,
+ (void *)&mbufs_dst[nb_tx],
+ nb_dq - nb_tx);
+ }
+}
+
+/* Main rx processing loop for IOAT rawdev. */
+static void
+rx_main_loop(void)
+{
+ uint16_t i;
+ uint16_t nb_ports = cfg.nb_ports;
+
+ RTE_LOG(INFO, IOAT, "Entering main rx loop for copy on lcore %u\n",
+ rte_lcore_id());
+
+ while (!force_quit)
+ for (i = 0; i < nb_ports; i++)
+ ioat_rx_port(&cfg.ports[i]);
+}
+
+/* Main tx processing loop for hardware copy. */
+static void
+tx_main_loop(void)
+{
+ uint16_t i;
+ uint16_t nb_ports = cfg.nb_ports;
+
+ RTE_LOG(INFO, IOAT, "Entering main tx loop for copy on lcore %u\n",
+ rte_lcore_id());
+
+ while (!force_quit)
+ for (i = 0; i < nb_ports; i++)
+ ioat_tx_port(&cfg.ports[i]);
+}
+
+/* Main rx and tx loop if only one slave lcore available */
+static void
+rxtx_main_loop(void)
+{
+ uint16_t i;
+ uint16_t nb_ports = cfg.nb_ports;
+
+ RTE_LOG(INFO, IOAT, "Entering main rx and tx loop for copy on"
+ " lcore %u\n", rte_lcore_id());
+
+ while (!force_quit)
+ for (i = 0; i < nb_ports; i++) {
+ ioat_rx_port(&cfg.ports[i]);
+ ioat_tx_port(&cfg.ports[i]);
+ }
+}
+
+static void start_forwarding_cores(void)
+{
+ uint32_t lcore_id = rte_lcore_id();
+
+ RTE_LOG(INFO, IOAT, "Entering %s on lcore %u\n",
+ __func__, rte_lcore_id());
+
+ if (cfg.nb_lcores == 1) {
+ lcore_id = rte_get_next_lcore(lcore_id, true, true);
+ rte_eal_remote_launch((lcore_function_t *)rxtx_main_loop,
+ NULL, lcore_id);
+ } else if (cfg.nb_lcores > 1) {
+ lcore_id = rte_get_next_lcore(lcore_id, true, true);
+ rte_eal_remote_launch((lcore_function_t *)rx_main_loop,
+ NULL, lcore_id);
+
+ lcore_id = rte_get_next_lcore(lcore_id, true, true);
+ rte_eal_remote_launch((lcore_function_t *)tx_main_loop, NULL,
+ lcore_id);
+ }
+}
+