net/virtio: fix memory leak when reinitializing device

[dpdk.git] / drivers / net / mlx5 / mlx5_rxtx_vec_sse.h

diff --git a/drivers/net/mlx5/mlx5_rxtx_vec_sse.h b/drivers/net/mlx5/mlx5_rxtx_vec_sse.h
index f256811..df66c2f 100644 (file)
--- a/drivers/net/mlx5/mlx5_rxtx_vec_sse.h
+++ b/drivers/net/mlx5/mlx5_rxtx_vec_sse.h
@@ -135,6 +135,8 @@ txq_scatter_v(struct mlx5_txq_data *txq, struct rte_mbuf **pkts,
 
        assert(elts_n > pkts_n);
        mlx5_tx_complete(txq);
+       /* A CQE slot must always be available. */
+       assert((1u << txq->cqe_n) - (txq->cq_pi - txq->cq_ci));
        if (unlikely(!pkts_n))
                return 0;
        for (n = 0; n < pkts_n; ++n) {
@@ -206,7 +208,9 @@ txq_scatter_v(struct mlx5_txq_data *txq, struct rte_mbuf **pkts,
                wqe->ctrl[2] = rte_cpu_to_be_32(8);
                wqe->ctrl[3] = txq->elts_head;
                txq->elts_comp = 0;
+#ifndef NDEBUG
                ++txq->cq_pi;
+#endif
        }
 #ifdef MLX5_PMD_SOFT_COUNTERS
        txq->stats.opackets += n;
@@ -219,7 +223,7 @@ txq_scatter_v(struct mlx5_txq_data *txq, struct rte_mbuf **pkts,
  * Send burst of packets with Enhanced MPW. If it encounters a multi-seg packet,
  * it returns to make it processed by txq_scatter_v(). All the packets in
  * the pkts list should be single segment packets having same offload flags.
- * This must be checked by txq_check_multiseg() and txq_calc_offload().
+ * This must be checked by txq_count_contig_single_seg() and txq_calc_offload().
  *
  * @param txq
  *   Pointer to TX queue structure.
@@ -268,6 +272,8 @@ txq_burst_v(struct mlx5_txq_data *txq, struct rte_mbuf **pkts, uint16_t pkts_n,
        assert(elts_n > pkts_n);
        mlx5_tx_complete(txq);
        max_elts = (elts_n - (elts_head - txq->elts_tail));
+       /* A CQE slot must always be available. */
+       assert((1u << txq->cqe_n) - (txq->cq_pi - txq->cq_ci));
        max_wqe = (1u << txq->wqe_n) - (txq->wqe_ci - txq->wqe_pi);
        pkts_n = RTE_MIN((unsigned int)RTE_MIN(pkts_n, max_wqe), max_elts);
        assert(pkts_n <= MLX5_DSEG_MAX - nb_dword_in_hdr);
@@ -307,7 +313,9 @@ txq_burst_v(struct mlx5_txq_data *txq, struct rte_mbuf **pkts, uint16_t pkts_n,
        } else {
                /* Request a completion. */
                txq->elts_comp = 0;
+#ifndef NDEBUG
                ++txq->cq_pi;
+#endif
                comp_req = 8;
        }
        /* Fill CTRL in the header. */
@@ -576,7 +584,7 @@ rxq_cq_to_ptype_oflags_v(struct mlx5_rxq_data *rxq, __m128i cqes[4],
                        _mm_set_epi32(0xffffff00, 0xffffff00,
                                      0xffffff00, 0xffffff00);
                const __m128i fdir_flags = _mm_set1_epi32(PKT_RX_FDIR);
-               const __m128i fdir_id_flags = _mm_set1_epi32(PKT_RX_FDIR_ID);
+               __m128i fdir_id_flags = _mm_set1_epi32(PKT_RX_FDIR_ID);
                __m128i flow_tag, invalid_mask;
 
                flow_tag = _mm_and_si128(pinfo, pinfo_ft_mask);
@@ -586,7 +594,7 @@ rxq_cq_to_ptype_oflags_v(struct mlx5_rxq_data *rxq, __m128i cqes[4],
                                        _mm_andnot_si128(invalid_mask,
                                                         fdir_flags));
                /* Mask out invalid entries. */
-               flow_tag = _mm_andnot_si128(invalid_mask, flow_tag);
+               fdir_id_flags = _mm_andnot_si128(invalid_mask, fdir_id_flags);
                /* Check if flow tag MLX5_FLOW_MARK_DEFAULT. */
                ol_flags = _mm_or_si128(ol_flags,
                                        _mm_andnot_si128(
@@ -654,12 +662,16 @@ rxq_cq_to_ptype_oflags_v(struct mlx5_rxq_data *rxq, __m128i cqes[4],
  *   Array to store received packets.
  * @param pkts_n
  *   Maximum number of packets in array.
+ * @param[out] err
+ *   Pointer to a flag. Set non-zero value if pkts array has at least one error
+ *   packet to handle.
  *
  * @return
  *   Number of packets received including errors (<= pkts_n).
  */
 static inline uint16_t
-rxq_burst_v(struct mlx5_rxq_data *rxq, struct rte_mbuf **pkts, uint16_t pkts_n)
+rxq_burst_v(struct mlx5_rxq_data *rxq, struct rte_mbuf **pkts, uint16_t pkts_n,
+           uint64_t *err)
 {
        const uint16_t q_n = 1 << rxq->cqe_n;
        const uint16_t q_mask = q_n - 1;
@@ -821,7 +833,7 @@ rxq_burst_v(struct mlx5_rxq_data *rxq, struct rte_mbuf **pkts, uint16_t pkts_n)
                /* B.2 copy mbuf pointers. */
                _mm_storeu_si128((__m128i *)&pkts[pos], mbp1);
                _mm_storeu_si128((__m128i *)&pkts[pos + 2], mbp2);
-               rte_compiler_barrier();
+               rte_cio_rmb();
                /* C.1 load remained CQE data and extract necessary fields. */
                cqe_tmp2 = _mm_load_si128((__m128i *)&cq[pos + p3]);
                cqe_tmp1 = _mm_load_si128((__m128i *)&cq[pos + p2]);
@@ -921,7 +933,7 @@ rxq_burst_v(struct mlx5_rxq_data *rxq, struct rte_mbuf **pkts, uint16_t pkts_n)
                opcode = _mm_packs_epi32(opcode, zero);
                opcode = _mm_andnot_si128(invalid_mask, opcode);
                /* D.4 mark if any error is set */
-               rxq->pending_err |= !!_mm_cvtsi128_si64(opcode);
+               *err |= _mm_cvtsi128_si64(opcode);
                /* D.5 fill in mbuf - rearm_data and packet_type. */
                rxq_cq_to_ptype_oflags_v(rxq, cqes, opcode, &pkts[pos]);
                if (rxq->hw_timestamp) {