i40e: add vector scatter Rx

To support the multiple segments in one packets when the received pkts exceed
 one buffer size.

Signed-off-by: Zhe Tao <zhe.tao@intel.com>
Acked-by: Cunming Liang <cunming.liang@intel.com>

diff --git a/drivers/net/i40e/i40e_rxtx.c b/drivers/net/i40e/i40e_rxtx.c
index 8fac220..d37cbde 100644 (file)
--- a/drivers/net/i40e/i40e_rxtx.c
+++ b/drivers/net/i40e/i40e_rxtx.c
@@ -3074,6 +3074,15 @@ i40e_recv_pkts_vec(
        return 0;
 }
 
+uint16_t __attribute__((weak))
+i40e_recv_scattered_pkts_vec(
+       void __rte_unused *rx_queue,
+       struct rte_mbuf __rte_unused **rx_pkts,
+       uint16_t __rte_unused nb_pkts)
+{
+       return 0;
+}
+
 int __attribute__((weak))
 i40e_rxq_vec_setup(struct i40e_rx_queue __rte_unused *rxq)
 {

diff --git a/drivers/net/i40e/i40e_rxtx.h b/drivers/net/i40e/i40e_rxtx.h
index 6dea402..98d6e90 100644 (file)
--- a/drivers/net/i40e/i40e_rxtx.h
+++ b/drivers/net/i40e/i40e_rxtx.h
@@ -231,6 +231,9 @@ int i40e_dev_rx_descriptor_done(void *rx_queue, uint16_t offset);
 
 uint16_t i40e_recv_pkts_vec(void *rx_queue, struct rte_mbuf **rx_pkts,
                            uint16_t nb_pkts);
+uint16_t i40e_recv_scattered_pkts_vec(void *rx_queue,
+                                     struct rte_mbuf **rx_pkts,
+                                     uint16_t nb_pkts);
 int i40e_rxq_vec_setup(struct i40e_rx_queue *rxq);
 int i40e_txq_vec_setup(struct i40e_tx_queue *txq);
 void i40e_rx_queue_release_mbufs_vec(struct i40e_rx_queue *rxq);

diff --git a/drivers/net/i40e/i40e_rxtx_vec.c b/drivers/net/i40e/i40e_rxtx_vec.c
index b97e372..f746209 100644 (file)
--- a/drivers/net/i40e/i40e_rxtx_vec.c
+++ b/drivers/net/i40e/i40e_rxtx_vec.c
@@ -220,9 +220,7 @@ desc_pktlen_align(__m128i descs[4])
        *((uint16_t *)&descs[3]+7) = vol.e[3];
 }
 
- /* vPMD receive routine, now only accept (nb_pkts == RTE_I40E_VPMD_RX_BURST)
- * in one loop
- *
+ /*
  * Notice:
  * - nb_pkts < RTE_I40E_DESCS_PER_LOOP, just return no packet
  * - nb_pkts > RTE_I40E_VPMD_RX_BURST, only scan RTE_I40E_VPMD_RX_BURST
@@ -430,13 +428,11 @@ _recv_raw_pkts_vec(struct i40e_rx_queue *rxq, struct rte_mbuf **rx_pkts,
        return nb_pkts_recd;
 }
 
- /* vPMD receive routine, now only accept (nb_pkts == RTE_IXGBE_VPMD_RX_BURST)
- * in one loop
- *
+ /*
  * Notice:
- * - nb_pkts < RTE_I40E_VPMD_RX_BURST, just return no packet
- * - nb_pkts > RTE_I40E_VPMD_RX_BURST, only scan RTE_IXGBE_VPMD_RX_BURST
- *   numbers of DD bit
+ * - nb_pkts < RTE_I40E_DESCS_PER_LOOP, just return no packet
+ * - nb_pkts > RTE_I40E_VPMD_RX_BURST, only scan RTE_I40E_VPMD_RX_BURST
+ *   numbers of DD bits
  */
 uint16_t
 i40e_recv_pkts_vec(void *rx_queue, struct rte_mbuf **rx_pkts,
@@ -445,6 +441,110 @@ i40e_recv_pkts_vec(void *rx_queue, struct rte_mbuf **rx_pkts,
        return _recv_raw_pkts_vec(rx_queue, rx_pkts, nb_pkts, NULL);
 }
 
+static inline uint16_t
+reassemble_packets(struct i40e_rx_queue *rxq, struct rte_mbuf **rx_bufs,
+                  uint16_t nb_bufs, uint8_t *split_flags)
+{
+       struct rte_mbuf *pkts[RTE_I40E_VPMD_RX_BURST]; /*finished pkts*/
+       struct rte_mbuf *start = rxq->pkt_first_seg;
+       struct rte_mbuf *end =  rxq->pkt_last_seg;
+       unsigned pkt_idx, buf_idx;
+
+       for (buf_idx = 0, pkt_idx = 0; buf_idx < nb_bufs; buf_idx++) {
+               if (end != NULL) {
+                       /* processing a split packet */
+                       end->next = rx_bufs[buf_idx];
+                       rx_bufs[buf_idx]->data_len += rxq->crc_len;
+
+                       start->nb_segs++;
+                       start->pkt_len += rx_bufs[buf_idx]->data_len;
+                       end = end->next;
+
+                       if (!split_flags[buf_idx]) {
+                               /* it's the last packet of the set */
+                               start->hash = end->hash;
+                               start->ol_flags = end->ol_flags;
+                               /* we need to strip crc for the whole packet */
+                               start->pkt_len -= rxq->crc_len;
+                               if (end->data_len > rxq->crc_len) {
+                                       end->data_len -= rxq->crc_len;
+                               } else {
+                                       /* free up last mbuf */
+                                       struct rte_mbuf *secondlast = start;
+
+                                       while (secondlast->next != end)
+                                               secondlast = secondlast->next;
+                                       secondlast->data_len -= (rxq->crc_len -
+                                                       end->data_len);
+                                       secondlast->next = NULL;
+                                       rte_pktmbuf_free_seg(end);
+                                       end = secondlast;
+                               }
+                               pkts[pkt_idx++] = start;
+                               start = end = NULL;
+                       }
+               } else {
+                       /* not processing a split packet */
+                       if (!split_flags[buf_idx]) {
+                               /* not a split packet, save and skip */
+                               pkts[pkt_idx++] = rx_bufs[buf_idx];
+                               continue;
+                       }
+                       end = start = rx_bufs[buf_idx];
+                       rx_bufs[buf_idx]->data_len += rxq->crc_len;
+                       rx_bufs[buf_idx]->pkt_len += rxq->crc_len;
+               }
+       }
+
+       /* save the partial packet for next time */
+       rxq->pkt_first_seg = start;
+       rxq->pkt_last_seg = end;
+       memcpy(rx_bufs, pkts, pkt_idx * (sizeof(*pkts)));
+       return pkt_idx;
+}
+
+ /* vPMD receive routine that reassembles scattered packets
+ * Notice:
+ * - nb_pkts < RTE_I40E_DESCS_PER_LOOP, just return no packet
+ * - nb_pkts > RTE_I40E_VPMD_RX_BURST, only scan RTE_I40E_VPMD_RX_BURST
+ *   numbers of DD bits
+ */
+uint16_t
+i40e_recv_scattered_pkts_vec(void *rx_queue, struct rte_mbuf **rx_pkts,
+                            uint16_t nb_pkts)
+{
+
+       struct i40e_rx_queue *rxq = rx_queue;
+       uint8_t split_flags[RTE_I40E_VPMD_RX_BURST] = {0};
+
+       /* get some new buffers */
+       uint16_t nb_bufs = _recv_raw_pkts_vec(rxq, rx_pkts, nb_pkts,
+                       split_flags);
+       if (nb_bufs == 0)
+               return 0;
+
+       /* happy day case, full burst + no packets to be joined */
+       const uint64_t *split_fl64 = (uint64_t *)split_flags;
+
+       if (rxq->pkt_first_seg == NULL &&
+                       split_fl64[0] == 0 && split_fl64[1] == 0 &&
+                       split_fl64[2] == 0 && split_fl64[3] == 0)
+               return nb_bufs;
+
+       /* reassemble any packets that need reassembly*/
+       unsigned i = 0;
+
+       if (rxq->pkt_first_seg == NULL) {
+               /* find the first split flag, and only reassemble then*/
+               while (i < nb_bufs && !split_flags[i])
+                       i++;
+               if (i == nb_bufs)
+                       return nb_bufs;
+       }
+       return i + reassemble_packets(rxq, &rx_pkts[i], nb_bufs - i,
+               &split_flags[i]);
+}
+
 static inline void
 vtx1(volatile struct i40e_tx_desc *txdp,
                struct rte_mbuf *pkt, uint64_t flags)