net/mlx5: remove Rx queues indexes correlation

[dpdk.git] / drivers / net / mlx5 / mlx5_rxtx_vec_sse.h

diff --git a/drivers/net/mlx5/mlx5_rxtx_vec_sse.h b/drivers/net/mlx5/mlx5_rxtx_vec_sse.h
index e0f95f9..c49c52c 100644 (file)
--- a/drivers/net/mlx5/mlx5_rxtx_vec_sse.h
+++ b/drivers/net/mlx5/mlx5_rxtx_vec_sse.h
@@ -104,6 +104,8 @@ txq_scatter_v(struct mlx5_txq_data *txq, struct rte_mbuf **pkts,
                sizeof(struct mlx5_wqe) / MLX5_WQE_DWORD_SIZE;
        unsigned int n;
        volatile struct mlx5_wqe *wqe = NULL;
+       bool metadata_ol =
+               txq->offloads & DEV_TX_OFFLOAD_MATCH_METADATA ? true : false;
 
        assert(elts_n > pkts_n);
        mlx5_tx_complete(txq);
@@ -125,6 +127,9 @@ txq_scatter_v(struct mlx5_txq_data *txq, struct rte_mbuf **pkts,
                uint16_t max_wqe;
                __m128i *t_wqe, *dseg;
                __m128i ctrl;
+               rte_be32_t metadata =
+                       metadata_ol && (buf->ol_flags & PKT_TX_METADATA) ?
+                       buf->tx_metadata : 0;
 
                assert(segs_n);
                max_elts = elts_n - (elts_head - txq->elts_tail);
@@ -165,9 +170,9 @@ txq_scatter_v(struct mlx5_txq_data *txq, struct rte_mbuf **pkts,
                _mm_store_si128(t_wqe, ctrl);
                /* Fill ESEG in the header. */
                _mm_store_si128(t_wqe + 1,
-                               _mm_set_epi16(0, 0, 0, 0,
-                                             rte_cpu_to_be_16(len), cs_flags,
-                                             0, 0));
+                               _mm_set_epi32(0, metadata,
+                                             (rte_cpu_to_be_16(len) << 16) |
+                                             cs_flags, 0));
                txq->wqe_ci = wqe_ci;
        }
        if (!n)
@@ -344,8 +349,11 @@ rxq_copy_mbuf_v(struct mlx5_rxq_data *rxq, struct rte_mbuf **pkts, uint16_t n)
  * @param elts
  *   Pointer to SW ring to be filled. The first mbuf has to be pre-built from
  *   the title completion descriptor to be copied to the rest of mbufs.
+ *
+ * @return
+ *   Number of mini-CQEs successfully decompressed.
  */
-static inline void
+static inline uint16_t
 rxq_cq_decompress_v(struct mlx5_rxq_data *rxq, volatile struct mlx5_cqe *cq,
                    struct rte_mbuf **elts)
 {
@@ -369,16 +377,16 @@ rxq_cq_decompress_v(struct mlx5_rxq_data *rxq, volatile struct mlx5_cqe *cq,
                            -1, -1, -1, -1  /* skip packet_type */);
        /* Restore the compressed count. Must be 16 bits. */
        const uint16_t mcqe_n = t_pkt->data_len +
-                               (rxq->crc_present * ETHER_CRC_LEN);
+                               (rxq->crc_present * RTE_ETHER_CRC_LEN);
        const __m128i rearm =
                _mm_loadu_si128((__m128i *)&t_pkt->rearm_data);
        const __m128i rxdf =
                _mm_loadu_si128((__m128i *)&t_pkt->rx_descriptor_fields1);
        const __m128i crc_adj =
                _mm_set_epi16(0, 0, 0,
-                             rxq->crc_present * ETHER_CRC_LEN,
+                             rxq->crc_present * RTE_ETHER_CRC_LEN,
                              0,
-                             rxq->crc_present * ETHER_CRC_LEN,
+                             rxq->crc_present * RTE_ETHER_CRC_LEN,
                              0, 0);
        const uint32_t flow_tag = t_pkt->hash.fdir.hi;
 #ifdef MLX5_PMD_SOFT_COUNTERS
@@ -481,6 +489,7 @@ rxq_cq_decompress_v(struct mlx5_rxq_data *rxq, volatile struct mlx5_cqe *cq,
        rxq->stats.ibytes += rcvd_byte;
 #endif
        rxq->cq_ci += mcqe_n;
+       return mcqe_n;
 }
 
 /**
@@ -694,9 +703,9 @@ rxq_burst_v(struct mlx5_rxq_data *rxq, struct rte_mbuf **pkts, uint16_t pkts_n,
        const __m128i ones = _mm_cmpeq_epi32(zero, zero);
        const __m128i crc_adj =
                _mm_set_epi16(0, 0, 0, 0, 0,
-                             rxq->crc_present * ETHER_CRC_LEN,
+                             rxq->crc_present * RTE_ETHER_CRC_LEN,
                              0,
-                             rxq->crc_present * ETHER_CRC_LEN);
+                             rxq->crc_present * RTE_ETHER_CRC_LEN);
        const __m128i flow_mark_adj = _mm_set_epi32(rxq->mark * (-1), 0, 0, 0);
 
        assert(rxq->sges_n == 0);
@@ -707,23 +716,16 @@ rxq_burst_v(struct mlx5_rxq_data *rxq, struct rte_mbuf **pkts, uint16_t pkts_n,
        rte_prefetch0(cq + 2);
        rte_prefetch0(cq + 3);
        pkts_n = RTE_MIN(pkts_n, MLX5_VPMD_RX_MAX_BURST);
-       /*
-        * Order of indexes:
-        *   rq_ci >= cq_ci >= rq_pi
-        * Definition of indexes:
-        *   rq_ci - cq_ci := # of buffers owned by HW (posted).
-        *   cq_ci - rq_pi := # of buffers not returned to app (decompressed).
-        *   N - (rq_ci - rq_pi) := # of buffers consumed (to be replenished).
-        */
        repl_n = q_n - (rxq->rq_ci - rxq->rq_pi);
-       if (repl_n >= MLX5_VPMD_RXQ_RPLNSH_THRESH(q_n))
+       if (repl_n >= rxq->rq_repl_thresh)
                mlx5_rx_replenish_bulk_mbuf(rxq, repl_n);
        /* See if there're unreturned mbufs from compressed CQE. */
-       rcvd_pkt = rxq->cq_ci - rxq->rq_pi;
+       rcvd_pkt = rxq->decompressed;
        if (rcvd_pkt > 0) {
                rcvd_pkt = RTE_MIN(rcvd_pkt, pkts_n);
                rxq_copy_mbuf_v(rxq, pkts, rcvd_pkt);
                rxq->rq_pi += rcvd_pkt;
+               rxq->decompressed -= rcvd_pkt;
                pkts += rcvd_pkt;
        }
        elts_idx = rxq->rq_pi & q_mask;
@@ -732,10 +734,11 @@ rxq_burst_v(struct mlx5_rxq_data *rxq, struct rte_mbuf **pkts, uint16_t pkts_n,
        pkts_n = RTE_ALIGN_FLOOR(pkts_n - rcvd_pkt, MLX5_VPMD_DESCS_PER_LOOP);
        /* Not to cross queue end. */
        pkts_n = RTE_MIN(pkts_n, q_n - elts_idx);
+       pkts_n = RTE_MIN(pkts_n, q_n - cq_idx);
        if (!pkts_n)
                return rcvd_pkt;
        /* At this point, there shouldn't be any remained packets. */
-       assert(rxq->rq_pi == rxq->cq_ci);
+       assert(rxq->decompressed == 0);
        /*
         * A. load first Qword (8bytes) in one loop.
         * B. copy 4 mbuf pointers from elts ring to returing pkts.
@@ -948,15 +951,17 @@ rxq_burst_v(struct mlx5_rxq_data *rxq, struct rte_mbuf **pkts, uint16_t pkts_n,
        /* Decompress the last CQE if compressed. */
        if (comp_idx < MLX5_VPMD_DESCS_PER_LOOP && comp_idx == n) {
                assert(comp_idx == (nocmp_n % MLX5_VPMD_DESCS_PER_LOOP));
-               rxq_cq_decompress_v(rxq, &cq[nocmp_n], &elts[nocmp_n]);
+               rxq->decompressed = rxq_cq_decompress_v(rxq, &cq[nocmp_n],
+                                                       &elts[nocmp_n]);
                /* Return more packets if needed. */
                if (nocmp_n < pkts_n) {
-                       uint16_t n = rxq->cq_ci - rxq->rq_pi;
+                       uint16_t n = rxq->decompressed;
 
                        n = RTE_MIN(n, pkts_n - nocmp_n);
                        rxq_copy_mbuf_v(rxq, &pkts[nocmp_n], n);
                        rxq->rq_pi += n;
                        rcvd_pkt += n;
+                       rxq->decompressed -= n;
                }
        }
        rte_compiler_barrier();