mbuf: add raw allocation function

[dpdk.git] / drivers / net / mlx5 / mlx5_rxtx.c

diff --git a/drivers/net/mlx5/mlx5_rxtx.c b/drivers/net/mlx5/mlx5_rxtx.c
index 0128139..1832a21 100644 (file)
--- a/drivers/net/mlx5/mlx5_rxtx.c
+++ b/drivers/net/mlx5/mlx5_rxtx.c
@@ -120,6 +120,10 @@ txq_complete(struct txq *txq)
                struct rte_mbuf *tmp = elt->buf;
                struct txq_elt *elt_next = &(*txq->elts)[elts_free_next];
 
+#ifndef NDEBUG
+               /* Poisoning. */
+               memset(elt, 0x66, sizeof(*elt));
+#endif
                RTE_MBUF_PREFETCH_TO_FREE(elt_next->buf);
                /* Faster than rte_pktmbuf_free(). */
                do {
@@ -333,6 +337,36 @@ txq_mp2mr_iter(const struct rte_mempool *mp, void *arg)
        txq_mp2mr(txq, mp);
 }
 
+/**
+ * Insert VLAN using mbuf headroom space.
+ *
+ * @param buf
+ *   Buffer for VLAN insertion.
+ *
+ * @return
+ *   0 on success, errno value on failure.
+ */
+static inline int
+insert_vlan_sw(struct rte_mbuf *buf)
+{
+       uintptr_t addr;
+       uint32_t vlan;
+       uint16_t head_room_len = rte_pktmbuf_headroom(buf);
+
+       if (head_room_len < 4)
+               return EINVAL;
+
+       addr = rte_pktmbuf_mtod(buf, uintptr_t);
+       vlan = htonl(0x81000000 | buf->vlan_tci);
+       memmove((void *)(addr - 4), (void *)addr, 12);
+       memcpy((void *)(addr + 8), &vlan, sizeof(vlan));
+
+       SET_DATA_OFF(buf, head_room_len - 4);
+       DATA_LEN(buf) += 4;
+
+       return 0;
+}
+
 #if MLX5_PMD_SGE_WR_N > 1
 
 /**
@@ -534,6 +568,9 @@ mlx5_tx_burst(void *dpdk_txq, struct rte_mbuf **pkts, uint16_t pkts_n)
                unsigned int sent_size = 0;
 #endif
                uint32_t send_flags = 0;
+#ifdef HAVE_VERBS_VLAN_INSERTION
+               int insert_vlan = 0;
+#endif /* HAVE_VERBS_VLAN_INSERTION */
 
                if (i + 1 < max)
                        rte_prefetch0(buf_next);
@@ -554,6 +591,18 @@ mlx5_tx_burst(void *dpdk_txq, struct rte_mbuf **pkts, uint16_t pkts_n)
                        if (RTE_ETH_IS_TUNNEL_PKT(buf->packet_type))
                                send_flags |= IBV_EXP_QP_BURST_TUNNEL;
                }
+               if (buf->ol_flags & PKT_TX_VLAN_PKT) {
+#ifdef HAVE_VERBS_VLAN_INSERTION
+                       if (!txq->priv->mps)
+                               insert_vlan = 1;
+                       else
+#endif /* HAVE_VERBS_VLAN_INSERTION */
+                       {
+                               err = insert_vlan_sw(buf);
+                               if (unlikely(err))
+                                       goto stop;
+                       }
+               }
                if (likely(segs == 1)) {
                        uintptr_t addr;
                        uint32_t length;
@@ -577,13 +626,23 @@ mlx5_tx_burst(void *dpdk_txq, struct rte_mbuf **pkts, uint16_t pkts_n)
                        }
                        /* Put packet into send queue. */
 #if MLX5_PMD_MAX_INLINE > 0
-                       if (length <= txq->max_inline)
-                               err = txq->send_pending_inline
-                                       (txq->qp,
-                                        (void *)addr,
-                                        length,
-                                        send_flags);
-                       else
+                       if (length <= txq->max_inline) {
+#ifdef HAVE_VERBS_VLAN_INSERTION
+                               if (insert_vlan)
+                                       err = txq->send_pending_inline_vlan
+                                               (txq->qp,
+                                                (void *)addr,
+                                                length,
+                                                send_flags,
+                                                &buf->vlan_tci);
+                               else
+#endif /* HAVE_VERBS_VLAN_INSERTION */
+                                       err = txq->send_pending_inline
+                                               (txq->qp,
+                                                (void *)addr,
+                                                length,
+                                                send_flags);
+                       } else
 #endif
                        {
                                /* Retrieve Memory Region key for this
@@ -597,12 +656,23 @@ mlx5_tx_burst(void *dpdk_txq, struct rte_mbuf **pkts, uint16_t pkts_n)
                                        elt->buf = NULL;
                                        goto stop;
                                }
-                               err = txq->send_pending
-                                       (txq->qp,
-                                        addr,
-                                        length,
-                                        lkey,
-                                        send_flags);
+#ifdef HAVE_VERBS_VLAN_INSERTION
+                               if (insert_vlan)
+                                       err = txq->send_pending_vlan
+                                               (txq->qp,
+                                                addr,
+                                                length,
+                                                lkey,
+                                                send_flags,
+                                                &buf->vlan_tci);
+                               else
+#endif /* HAVE_VERBS_VLAN_INSERTION */
+                                       err = txq->send_pending
+                                               (txq->qp,
+                                                addr,
+                                                length,
+                                                lkey,
+                                                send_flags);
                        }
                        if (unlikely(err))
                                goto stop;
@@ -619,11 +689,21 @@ mlx5_tx_burst(void *dpdk_txq, struct rte_mbuf **pkts, uint16_t pkts_n)
                        if (ret.length == (unsigned int)-1)
                                goto stop;
                        /* Put SG list into send queue. */
-                       err = txq->send_pending_sg_list
-                               (txq->qp,
-                                sges,
-                                ret.num,
-                                send_flags);
+#ifdef HAVE_VERBS_VLAN_INSERTION
+                       if (insert_vlan)
+                               err = txq->send_pending_sg_list_vlan
+                                       (txq->qp,
+                                        sges,
+                                        ret.num,
+                                        send_flags,
+                                        &buf->vlan_tci);
+                       else
+#endif /* HAVE_VERBS_VLAN_INSERTION */
+                               err = txq->send_pending_sg_list
+                                       (txq->qp,
+                                        sges,
+                                        ret.num,
+                                        send_flags);
                        if (unlikely(err))
                                goto stop;
 #ifdef MLX5_PMD_SOFT_COUNTERS
@@ -830,7 +910,8 @@ mlx5_rx_burst_sp(void *dpdk_rxq, struct rte_mbuf **pkts, uint16_t pkts_n)
                }
                if (ret == 0)
                        break;
-               len = ret;
+               assert(ret >= (rxq->crc_present << 2));
+               len = ret - (rxq->crc_present << 2);
                pkt_buf_len = len;
                /*
                 * Replace spent segments with new ones, concatenate and
@@ -848,7 +929,7 @@ mlx5_rx_burst_sp(void *dpdk_rxq, struct rte_mbuf **pkts, uint16_t pkts_n)
                         * cacheline while allocating rep.
                         */
                        rte_prefetch0(seg);
-                       rep = __rte_mbuf_raw_alloc(rxq->mp);
+                       rep = rte_mbuf_raw_alloc(rxq->mp);
                        if (unlikely(rep == NULL)) {
                                /*
                                 * Unable to allocate a replacement mbuf,
@@ -1042,8 +1123,9 @@ mlx5_rx_burst(void *dpdk_rxq, struct rte_mbuf **pkts, uint16_t pkts_n)
                }
                if (ret == 0)
                        break;
-               len = ret;
-               rep = __rte_mbuf_raw_alloc(rxq->mp);
+               assert(ret >= (rxq->crc_present << 2));
+               len = ret - (rxq->crc_present << 2);
+               rep = rte_mbuf_raw_alloc(rxq->mp);
                if (unlikely(rep == NULL)) {
                        /*
                         * Unable to allocate a replacement mbuf,