net/enic: add the simple version of Tx handler

Add a much-simplified handler that works when all offloads are
disabled, except mbuf fast free. When compared against the default
handler, under ideal conditions, cycles per packet drop by 60+%.
The driver tries to use the simple handler first.

The idea of using specialized/simplified handlers is from the Intel
and Mellanox drivers.

Signed-off-by: Hyong Youb Kim <hyonkim@cisco.com>
Reviewed-by: John Daley <johndale@cisco.com>

diff --git a/drivers/net/enic/enic.h b/drivers/net/enic/enic.h
index af790fc..a40ea79 100644 (file)
--- a/drivers/net/enic/enic.h
+++ b/drivers/net/enic/enic.h
@@ -318,6 +318,8 @@ uint16_t enic_dummy_recv_pkts(void *rx_queue,
                              uint16_t nb_pkts);
 uint16_t enic_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts,
                        uint16_t nb_pkts);
                              uint16_t nb_pkts);
 uint16_t enic_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts,
                        uint16_t nb_pkts);

+uint16_t enic_simple_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts,
+                              uint16_t nb_pkts);

 uint16_t enic_prep_pkts(void *tx_queue, struct rte_mbuf **tx_pkts,
                        uint16_t nb_pkts);
 int enic_set_mtu(struct enic *enic, uint16_t new_mtu);
 uint16_t enic_prep_pkts(void *tx_queue, struct rte_mbuf **tx_pkts,
                        uint16_t nb_pkts);
 int enic_set_mtu(struct enic *enic, uint16_t new_mtu);

diff --git a/drivers/net/enic/enic_compat.h b/drivers/net/enic/enic_compat.h
index c0af1ed..ceb1b09 100644 (file)
--- a/drivers/net/enic/enic_compat.h
+++ b/drivers/net/enic/enic_compat.h
@@ -56,6 +56,11 @@
 #define dev_debug(x, args...) dev_printk(DEBUG, args)
 
 extern int enicpmd_logtype_flow;
 #define dev_debug(x, args...) dev_printk(DEBUG, args)
 
 extern int enicpmd_logtype_flow;

+extern int enicpmd_logtype_init;
+
+#define PMD_INIT_LOG(level, fmt, args...) \
+       rte_log(RTE_LOG_ ## level, enicpmd_logtype_init, \
+               "%s" fmt "\n", __func__, ##args)

 
 #define __le16 u16
 #define __le32 u32
 
 #define __le16 u16
 #define __le32 u32

diff --git a/drivers/net/enic/enic_ethdev.c b/drivers/net/enic/enic_ethdev.c
index ef18f88..15d9371 100644 (file)
--- a/drivers/net/enic/enic_ethdev.c
+++ b/drivers/net/enic/enic_ethdev.c
@@ -24,10 +24,6 @@
 int enicpmd_logtype_init;
 int enicpmd_logtype_flow;
 
 int enicpmd_logtype_init;
 int enicpmd_logtype_flow;
 

-#define PMD_INIT_LOG(level, fmt, args...) \
-       rte_log(RTE_LOG_ ## level, enicpmd_logtype_init, \
-               "%s" fmt "\n", __func__, ##args)
-

 #define ENICPMD_FUNC_TRACE() PMD_INIT_LOG(DEBUG, " >>")
 
 /*
 #define ENICPMD_FUNC_TRACE() PMD_INIT_LOG(DEBUG, " >>")
 
 /*

diff --git a/drivers/net/enic/enic_main.c b/drivers/net/enic/enic_main.c
index c03ec24..e89105d 100644 (file)
--- a/drivers/net/enic/enic_main.c
+++ b/drivers/net/enic/enic_main.c
@@ -493,6 +493,27 @@ static void enic_rxq_intr_deinit(struct enic *enic)
        }
 }
 
        }
 }
 

+static void enic_prep_wq_for_simple_tx(struct enic *enic, uint16_t queue_idx)
+{
+       struct wq_enet_desc *desc;
+       struct vnic_wq *wq;
+       unsigned int i;
+
+       /*
+        * Fill WQ descriptor fields that never change. Every descriptor is
+        * one packet, so set EOP. Also set CQ_ENTRY every ENIC_WQ_CQ_THRESH
+        * descriptors (i.e. request one completion update every 32 packets).
+        */
+       wq = &enic->wq[queue_idx];
+       desc = (struct wq_enet_desc *)wq->ring.descs;
+       for (i = 0; i < wq->ring.desc_count; i++, desc++) {
+               desc->header_length_flags = 1 << WQ_ENET_FLAGS_EOP_SHIFT;
+               if (i % ENIC_WQ_CQ_THRESH == ENIC_WQ_CQ_THRESH - 1)
+                       desc->header_length_flags |=
+                               (1 << WQ_ENET_FLAGS_CQ_ENTRY_SHIFT);
+       }
+}
+

 int enic_enable(struct enic *enic)
 {
        unsigned int index;
 int enic_enable(struct enic *enic)
 {
        unsigned int index;


@@ -535,6 +556,21 @@ int enic_enable(struct enic *enic)
                }
        }
 
                }
        }
 

+       /*
+        * Use the simple TX handler if possible. All offloads must be disabled
+        * except mbuf fast free.
+        */
+       if ((eth_dev->data->dev_conf.txmode.offloads &
+            ~DEV_TX_OFFLOAD_MBUF_FAST_FREE) == 0) {
+               PMD_INIT_LOG(DEBUG, " use the simple tx handler");
+               eth_dev->tx_pkt_burst = &enic_simple_xmit_pkts;
+               for (index = 0; index < enic->wq_count; index++)
+                       enic_prep_wq_for_simple_tx(enic, index);
+       } else {
+               PMD_INIT_LOG(DEBUG, " use the default tx handler");
+               eth_dev->tx_pkt_burst = &enic_xmit_pkts;
+       }
+

        for (index = 0; index < enic->wq_count; index++)
                enic_start_wq(enic, index);
        for (index = 0; index < enic->rq_count; index++)
        for (index = 0; index < enic->wq_count; index++)
                enic_start_wq(enic, index);
        for (index = 0; index < enic->rq_count; index++)

diff --git a/drivers/net/enic/enic_rxtx.c b/drivers/net/enic/enic_rxtx.c
index 7cddb53..7dec486 100644 (file)
--- a/drivers/net/enic/enic_rxtx.c
+++ b/drivers/net/enic/enic_rxtx.c
@@ -741,4 +741,81 @@ uint16_t enic_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts,
        return index;
 }
 
        return index;
 }
 

+static void enqueue_simple_pkts(struct rte_mbuf **pkts,
+                               struct wq_enet_desc *desc,
+                               uint16_t n,
+                               struct enic *enic)
+{
+       struct rte_mbuf *p;
+
+       while (n) {
+               n--;
+               p = *pkts++;
+               desc->address = p->buf_iova + p->data_off;
+               desc->length = p->pkt_len;
+               /*
+                * The app should not send oversized
+                * packets. tx_pkt_prepare includes a check as
+                * well. But some apps ignore the device max size and
+                * tx_pkt_prepare. Oversized packets cause WQ errrors
+                * and the NIC ends up disabling the whole WQ. So
+                * truncate packets..
+                */
+               if (unlikely(p->pkt_len > ENIC_TX_MAX_PKT_SIZE)) {
+                       desc->length = ENIC_TX_MAX_PKT_SIZE;
+                       rte_atomic64_inc(&enic->soft_stats.tx_oversized);
+               }
+               desc++;
+       }
+}
+
+uint16_t enic_simple_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts,
+                              uint16_t nb_pkts)
+{
+       unsigned int head_idx, desc_count;
+       struct wq_enet_desc *desc;
+       struct vnic_wq *wq;
+       struct enic *enic;
+       uint16_t rem, n;
+
+       wq = (struct vnic_wq *)tx_queue;
+       enic = vnic_dev_priv(wq->vdev);
+       enic_cleanup_wq(enic, wq);
+       /* Will enqueue this many packets in this call */
+       nb_pkts = RTE_MIN(nb_pkts, wq->ring.desc_avail);
+       if (nb_pkts == 0)
+               return 0;
+
+       head_idx = wq->head_idx;
+       desc_count = wq->ring.desc_count;
+
+       /* Descriptors until the end of the ring */
+       n = desc_count - head_idx;
+       n = RTE_MIN(nb_pkts, n);

 
 

+       /* Save mbuf pointers to free later */
+       memcpy(wq->bufs + head_idx, tx_pkts, sizeof(struct rte_mbuf *) * n);
+
+       /* Enqueue until the ring end */
+       rem = nb_pkts - n;
+       desc = ((struct wq_enet_desc *)wq->ring.descs) + head_idx;
+       enqueue_simple_pkts(tx_pkts, desc, n, enic);
+
+       /* Wrap to the start of the ring */
+       if (rem) {
+               tx_pkts += n;
+               memcpy(wq->bufs, tx_pkts, sizeof(struct rte_mbuf *) * rem);
+               desc = (struct wq_enet_desc *)wq->ring.descs;
+               enqueue_simple_pkts(tx_pkts, desc, rem, enic);
+       }
+       rte_wmb();
+
+       /* Update head_idx and desc_avail */
+       wq->ring.desc_avail -= nb_pkts;
+       head_idx += nb_pkts;
+       if (head_idx >= desc_count)
+               head_idx -= desc_count;
+       wq->head_idx = head_idx;
+       iowrite32_relaxed(head_idx, &wq->ctrl->posted_index);
+       return nb_pkts;
+}