net/mlx5: fix vectorized mini-CQE prefetching

[dpdk.git] / drivers / net / mlx5 / mlx5_rxtx_vec_altivec.h

diff --git a/drivers/net/mlx5/mlx5_rxtx_vec_altivec.h b/drivers/net/mlx5/mlx5_rxtx_vec_altivec.h
index 45ff8e6..cb4ce1a 100644 (file)
--- a/drivers/net/mlx5/mlx5_rxtx_vec_altivec.h
+++ b/drivers/net/mlx5/mlx5_rxtx_vec_altivec.h
@@ -10,7 +10,7 @@
 #include <string.h>
 #include <stdlib.h>
 
-#include <altivec.h>
+#include <rte_altivec.h>
 
 #include <rte_mbuf.h>
 #include <rte_mempool.h>
@@ -158,7 +158,6 @@ rxq_cq_decompress_v(struct mlx5_rxq_data *rxq, volatile struct mlx5_cqe *cq,
                for (i = 0; i < MLX5_VPMD_DESCS_PER_LOOP; ++i)
                        if (likely(pos + i < mcqe_n))
                                rte_prefetch0((void *)(cq + pos + i));
-
                /* A.1 load mCQEs into a 128bit register. */
                mcqe1 = (vector unsigned char)vec_vsx_ld(0,
                        (signed int const *)&mcq[pos % 8]);
@@ -264,14 +263,15 @@ rxq_cq_decompress_v(struct mlx5_rxq_data *rxq, volatile struct mlx5_cqe *cq,
                        elts[pos + 2]->hash.fdir.hi = flow_tag;
                        elts[pos + 3]->hash.fdir.hi = flow_tag;
                }
-               if (!!rxq->flow_meta_mask) {
+               if (rxq->dynf_meta) {
                        int32_t offs = rxq->flow_meta_offset;
                        const uint32_t meta =
                                *RTE_MBUF_DYNFIELD(t_pkt, offs, uint32_t *);
 
                        /* Check if title packet has valid metadata. */
                        if (meta) {
-                               MLX5_ASSERT(t_pkt->ol_flags & offs);
+                               MLX5_ASSERT(t_pkt->ol_flags &
+                                           rxq->flow_meta_mask);
                                *RTE_MBUF_DYNFIELD(elts[pos], offs,
                                                        uint32_t *) = meta;
                                *RTE_MBUF_DYNFIELD(elts[pos + 1], offs,
@@ -286,6 +286,8 @@ rxq_cq_decompress_v(struct mlx5_rxq_data *rxq, volatile struct mlx5_cqe *cq,
                pos += MLX5_VPMD_DESCS_PER_LOOP;
                /* Move to next CQE and invalidate consumed CQEs. */
                if (!(pos & 0x7) && pos < mcqe_n) {
+                       if (pos + 8 < mcqe_n)
+                               rte_prefetch0((void *)(cq + pos + 8));
                        mcq = (void *)&(cq + pos)->pkt_info;
                        for (i = 0; i < 8; ++i)
                                cq[inv++].op_own = MLX5_CQE_INVALIDATE;
@@ -563,13 +565,15 @@ rxq_cq_to_ptype_oflags_v(struct mlx5_rxq_data *rxq,
  * @param[out] err
  *   Pointer to a flag. Set non-zero value if pkts array has at least one error
  *   packet to handle.
+ * @param[out] no_cq
+ *  Pointer to a boolean. Set true if no new CQE seen.
  *
  * @return
  *   Number of packets received including errors (<= pkts_n).
  */
 static inline uint16_t
 rxq_burst_v(struct mlx5_rxq_data *rxq, struct rte_mbuf **pkts, uint16_t pkts_n,
-           uint64_t *err)
+           uint64_t *err, bool *no_cq)
 {
        const uint16_t q_n = 1 << rxq->cqe_n;
        const uint16_t q_mask = q_n - 1;
@@ -662,8 +666,10 @@ rxq_burst_v(struct mlx5_rxq_data *rxq, struct rte_mbuf **pkts, uint16_t pkts_n,
        /* Not to cross queue end. */
        pkts_n = RTE_MIN(pkts_n, q_n - elts_idx);
        pkts_n = RTE_MIN(pkts_n, q_n - cq_idx);
-       if (!pkts_n)
+       if (!pkts_n) {
+               *no_cq = !rcvd_pkt;
                return rcvd_pkt;
+       }
        /* At this point, there shouldn't be any remaining packets. */
        MLX5_ASSERT(rxq->decompressed == 0);
 
@@ -1019,16 +1025,34 @@ rxq_burst_v(struct mlx5_rxq_data *rxq, struct rte_mbuf **pkts, uint16_t pkts_n,
                /* D.5 fill in mbuf - rearm_data and packet_type. */
                rxq_cq_to_ptype_oflags_v(rxq, cqes, opcode, &pkts[pos]);
                if (rxq->hw_timestamp) {
-                       pkts[pos]->timestamp =
-                               rte_be_to_cpu_64(cq[pos].timestamp);
-                       pkts[pos + 1]->timestamp =
-                               rte_be_to_cpu_64(cq[pos + p1].timestamp);
-                       pkts[pos + 2]->timestamp =
-                               rte_be_to_cpu_64(cq[pos + p2].timestamp);
-                       pkts[pos + 3]->timestamp =
-                               rte_be_to_cpu_64(cq[pos + p3].timestamp);
+                       if (rxq->rt_timestamp) {
+                               struct mlx5_dev_ctx_shared *sh = rxq->sh;
+                               uint64_t ts;
+
+                               ts = rte_be_to_cpu_64(cq[pos].timestamp);
+                               pkts[pos]->timestamp =
+                                       mlx5_txpp_convert_rx_ts(sh, ts);
+                               ts = rte_be_to_cpu_64(cq[pos + p1].timestamp);
+                               pkts[pos + 1]->timestamp =
+                                       mlx5_txpp_convert_rx_ts(sh, ts);
+                               ts = rte_be_to_cpu_64(cq[pos + p2].timestamp);
+                               pkts[pos + 2]->timestamp =
+                                       mlx5_txpp_convert_rx_ts(sh, ts);
+                               ts = rte_be_to_cpu_64(cq[pos + p3].timestamp);
+                               pkts[pos + 3]->timestamp =
+                                       mlx5_txpp_convert_rx_ts(sh, ts);
+                       } else {
+                               pkts[pos]->timestamp = rte_be_to_cpu_64
+                                               (cq[pos].timestamp);
+                               pkts[pos + 1]->timestamp = rte_be_to_cpu_64
+                                               (cq[pos + p1].timestamp);
+                               pkts[pos + 2]->timestamp = rte_be_to_cpu_64
+                                               (cq[pos + p2].timestamp);
+                               pkts[pos + 3]->timestamp = rte_be_to_cpu_64
+                                               (cq[pos + p3].timestamp);
+                       }
                }
-               if (!!rxq->flow_meta_mask) {
+               if (rxq->dynf_meta) {
                        uint64_t flag = rxq->flow_meta_mask;
                        int32_t offs = rxq->flow_meta_offset;
                        uint32_t metadata;
@@ -1078,8 +1102,10 @@ rxq_burst_v(struct mlx5_rxq_data *rxq, struct rte_mbuf **pkts, uint16_t pkts_n,
                        break;
        }
        /* If no new CQE seen, return without updating cq_db. */
-       if (unlikely(!nocmp_n && comp_idx == MLX5_VPMD_DESCS_PER_LOOP))
+       if (unlikely(!nocmp_n && comp_idx == MLX5_VPMD_DESCS_PER_LOOP)) {
+               *no_cq = true;
                return rcvd_pkt;
+       }
        /* Update the consumer indexes for non-compressed CQEs. */
        MLX5_ASSERT(nocmp_n <= pkts_n);
        rxq->cq_ci += nocmp_n;
@@ -1107,6 +1133,7 @@ rxq_burst_v(struct mlx5_rxq_data *rxq, struct rte_mbuf **pkts, uint16_t pkts_n,
        }
        rte_compiler_barrier();
        *rxq->cq_db = rte_cpu_to_be_32(rxq->cq_ci);
+       *no_cq = !rcvd_pkt;
        return rcvd_pkt;
 }