git.droids-corp.org / dpdk.git / blobdiff
? search:
summary | shortlog | log | commit | commitdiff | tree
raw | inline | side by side
eal: introduce macro for always inline
[dpdk.git] / drivers / net / mlx5 / mlx5_rxtx.c
diff --git a/drivers/net/mlx5/mlx5_rxtx.c b/drivers/net/mlx5/mlx5_rxtx.c
index 6254228..53b5c68 100644 (file)
--- a/drivers/net/mlx5/mlx5_rxtx.c
+++ b/drivers/net/mlx5/mlx5_rxtx.c
@@ -69,34 +69,28 @@
 #include "mlx5_defs.h"
 #include "mlx5_prm.h"
 
-static inline int
+static __rte_always_inline int
 check_cqe(volatile struct mlx5_cqe *cqe,
-         unsigned int cqes_n, const uint16_t ci)
-         __attribute__((always_inline));
+         unsigned int cqes_n, const uint16_t ci);
 
-static inline void
-txq_complete(struct txq *txq) __attribute__((always_inline));
+static __rte_always_inline void
+txq_complete(struct txq *txq);
 
-static inline uint32_t
-txq_mp2mr(struct txq *txq, struct rte_mempool *mp)
-       __attribute__((always_inline));
+static __rte_always_inline uint32_t
+txq_mp2mr(struct txq *txq, struct rte_mempool *mp);
 
-static inline void
-mlx5_tx_dbrec(struct txq *txq, volatile struct mlx5_wqe *wqe)
-       __attribute__((always_inline));
+static __rte_always_inline void
+mlx5_tx_dbrec(struct txq *txq, volatile struct mlx5_wqe *wqe);
 
-static inline uint32_t
-rxq_cq_to_pkt_type(volatile struct mlx5_cqe *cqe)
-       __attribute__((always_inline));
+static __rte_always_inline uint32_t
+rxq_cq_to_pkt_type(volatile struct mlx5_cqe *cqe);
 
-static inline int
+static __rte_always_inline int
 mlx5_rx_poll_len(struct rxq *rxq, volatile struct mlx5_cqe *cqe,
-                uint16_t cqe_cnt, uint32_t *rss_hash)
-                __attribute__((always_inline));
+                uint16_t cqe_cnt, uint32_t *rss_hash);
 
-static inline uint32_t
-rxq_cq_to_ol_flags(struct rxq *rxq, volatile struct mlx5_cqe *cqe)
-                  __attribute__((always_inline));
+static __rte_always_inline uint32_t
+rxq_cq_to_ol_flags(struct rxq *rxq, volatile struct mlx5_cqe *cqe);
 
 #ifndef NDEBUG
 
@@ -504,6 +498,7 @@ mlx5_tx_burst(void *dpdk_txq, struct rte_mbuf **pkts, uint16_t pkts_n)
        uint16_t max_wqe;
        unsigned int comp;
        volatile struct mlx5_wqe_v *wqe = NULL;
+       volatile struct mlx5_wqe_ctrl *last_wqe = NULL;
        unsigned int segs_n = 0;
        struct rte_mbuf *buf = NULL;
        uint8_t *raw;
@@ -524,6 +519,7 @@ mlx5_tx_burst(void *dpdk_txq, struct rte_mbuf **pkts, uint16_t pkts_n)
                volatile rte_v128u32_t *dseg = NULL;
                uint32_t length;
                unsigned int ds = 0;
+               unsigned int sg = 0; /* counter of additional segs attached. */
                uintptr_t addr;
                uint64_t naddr;
                uint16_t pkt_inline_sz = MLX5_WQE_DWORD_SIZE + 2;
@@ -536,7 +532,7 @@ mlx5_tx_burst(void *dpdk_txq, struct rte_mbuf **pkts, uint16_t pkts_n)
 #endif
 
                /* first_seg */
-               buf = *(pkts++);
+               buf = *pkts;
                segs_n = buf->nb_segs;
                /*
                 * Make sure there is enough room to store this packet and
@@ -547,15 +543,13 @@ mlx5_tx_burst(void *dpdk_txq, struct rte_mbuf **pkts, uint16_t pkts_n)
                        break;
                max -= segs_n;
                --segs_n;
-               if (!segs_n)
-                       --pkts_n;
                if (unlikely(--max_wqe == 0))
                        break;
                wqe = (volatile struct mlx5_wqe_v *)
                        tx_mlx5_wqe(txq, txq->wqe_ci);
                rte_prefetch0(tx_mlx5_wqe(txq, txq->wqe_ci + 1));
-               if (pkts_n > 1)
-                       rte_prefetch0(*pkts);
+               if (pkts_n - i > 1)
+                       rte_prefetch0(*(pkts + 1));
                addr = rte_pktmbuf_mtod(buf, uintptr_t);
                length = DATA_LEN(buf);
                ehdr = (((uint8_t *)addr)[1] << 8) |
@@ -567,14 +561,10 @@ mlx5_tx_burst(void *dpdk_txq, struct rte_mbuf **pkts, uint16_t pkts_n)
                        break;
                /* Update element. */
                (*txq->elts)[elts_head] = buf;
-               elts_head = (elts_head + 1) & (elts_n - 1);
                /* Prefetch next buffer data. */
-               if (pkts_n > 1) {
-                       volatile void *pkt_addr;
-
-                       pkt_addr = rte_pktmbuf_mtod(*pkts, volatile void *);
-                       rte_prefetch0(pkt_addr);
-               }
+               if (pkts_n - i > 1)
+                       rte_prefetch0(
+                           rte_pktmbuf_mtod(*(pkts + 1), volatile void *));
                /* Should we enable HW CKSUM offload */
                if (buf->ol_flags &
                    (PKT_TX_IP_CKSUM | PKT_TX_TCP_CKSUM | PKT_TX_UDP_CKSUM)) {
@@ -677,10 +667,6 @@ mlx5_tx_burst(void *dpdk_txq, struct rte_mbuf **pkts, uint16_t pkts_n)
                                        };
                                        ds = 1;
                                        total_length = 0;
-                                       pkts--;
-                                       pkts_n++;
-                                       elts_head = (elts_head - 1) &
-                                                   (elts_n - 1);
                                        k++;
                                        goto next_wqe;
                                }
@@ -813,15 +799,17 @@ next_seg:
                        naddr,
                        naddr >> 32,
                };
-               (*txq->elts)[elts_head] = buf;
                elts_head = (elts_head + 1) & (elts_n - 1);
-               ++j;
-               --segs_n;
-               if (segs_n)
+               (*txq->elts)[elts_head] = buf;
+               ++sg;
+               /* Advance counter only if all segs are successfully posted. */
+               if (sg < segs_n)
                        goto next_seg;
                else
-                       --pkts_n;
+                       j += sg;
 next_pkt:
+               elts_head = (elts_head + 1) & (elts_n - 1);
+               ++pkts;
                ++i;
                /* Initialize known and common part of the WQE structure. */
                if (tso) {
@@ -853,24 +841,24 @@ next_pkt:
                }
 next_wqe:
                txq->wqe_ci += (ds + 3) / 4;
+               /* Save the last successful WQE for completion request */
+               last_wqe = (volatile struct mlx5_wqe_ctrl *)wqe;
 #ifdef MLX5_PMD_SOFT_COUNTERS
                /* Increment sent bytes counter. */
                txq->stats.obytes += total_length;
 #endif
-       } while (pkts_n);
+       } while (i < pkts_n);
        /* Take a shortcut if nothing must be sent. */
        if (unlikely((i + k) == 0))
                return 0;
+       txq->elts_head = (txq->elts_head + i + j) & (elts_n - 1);
        /* Check whether completion threshold has been reached. */
        comp = txq->elts_comp + i + j + k;
        if (comp >= MLX5_TX_COMP_THRESH) {
-               volatile struct mlx5_wqe_ctrl *w =
-                       (volatile struct mlx5_wqe_ctrl *)wqe;
-
                /* Request completion on last WQE. */
-               w->ctrl2 = htonl(8);
+               last_wqe->ctrl2 = htonl(8);
                /* Save elts_head in unused "immediate" field of WQE. */
-               w->ctrl3 = elts_head;
+               last_wqe->ctrl3 = txq->elts_head;
                txq->elts_comp = 0;
        } else {
                txq->elts_comp = comp;
@@ -880,8 +868,7 @@ next_wqe:
        txq->stats.opackets += i;
 #endif
        /* Ring QP doorbell. */
-       mlx5_tx_dbrec(txq, (volatile struct mlx5_wqe *)wqe);
-       txq->elts_head = elts_head;
+       mlx5_tx_dbrec(txq, (volatile struct mlx5_wqe *)last_wqe);
        return i;
 }
 
DPDK repo used for reviews