net/mlx5: support hardware TSO

[dpdk.git] / drivers / net / mlx5 / mlx5_rxtx.c

diff --git a/drivers/net/mlx5/mlx5_rxtx.c b/drivers/net/mlx5/mlx5_rxtx.c
index 8399a91..7ef6c8d 100644 (file)
--- a/drivers/net/mlx5/mlx5_rxtx.c
+++ b/drivers/net/mlx5/mlx5_rxtx.c
@@ -441,6 +441,7 @@ mlx5_tx_burst(void *dpdk_txq, struct rte_mbuf **pkts, uint16_t pkts_n)
        const unsigned int elts_n = 1 << txq->elts_n;
        unsigned int i = 0;
        unsigned int j = 0;
+       unsigned int k = 0;
        unsigned int max;
        uint16_t max_wqe;
        unsigned int comp;
@@ -468,8 +469,10 @@ mlx5_tx_burst(void *dpdk_txq, struct rte_mbuf **pkts, uint16_t pkts_n)
                uintptr_t addr;
                uint64_t naddr;
                uint16_t pkt_inline_sz = MLX5_WQE_DWORD_SIZE + 2;
+               uint16_t tso_header_sz = 0;
                uint16_t ehdr;
                uint8_t cs_flags = 0;
+               uint64_t tso = 0;
 #ifdef MLX5_PMD_SOFT_COUNTERS
                uint32_t total_length = 0;
 #endif
@@ -541,14 +544,74 @@ mlx5_tx_burst(void *dpdk_txq, struct rte_mbuf **pkts, uint16_t pkts_n)
                        length -= pkt_inline_sz;
                        addr += pkt_inline_sz;
                }
+               if (txq->tso_en) {
+                       tso = buf->ol_flags & PKT_TX_TCP_SEG;
+                       if (tso) {
+                               uintptr_t end = (uintptr_t)
+                                               (((uintptr_t)txq->wqes) +
+                                               (1 << txq->wqe_n) *
+                                               MLX5_WQE_SIZE);
+                               unsigned int copy_b;
+                               uint8_t vlan_sz = (buf->ol_flags &
+                                                 PKT_TX_VLAN_PKT) ? 4 : 0;
+
+                               tso_header_sz = buf->l2_len + vlan_sz +
+                                               buf->l3_len + buf->l4_len;
+
+                               if (unlikely(tso_header_sz >
+                                            MLX5_MAX_TSO_HEADER))
+                                       break;
+                               copy_b = tso_header_sz - pkt_inline_sz;
+                               /* First seg must contain all headers. */
+                               assert(copy_b <= length);
+                               raw += MLX5_WQE_DWORD_SIZE;
+                               if (copy_b &&
+                                  ((end - (uintptr_t)raw) > copy_b)) {
+                                       uint16_t n = (MLX5_WQE_DS(copy_b) -
+                                                     1 + 3) / 4;
+
+                                       if (unlikely(max_wqe < n))
+                                               break;
+                                       max_wqe -= n;
+                                       rte_memcpy((void *)raw,
+                                                  (void *)addr, copy_b);
+                                       addr += copy_b;
+                                       length -= copy_b;
+                                       pkt_inline_sz += copy_b;
+                                       /*
+                                        * Another DWORD will be added
+                                        * in the inline part.
+                                        */
+                                       raw += MLX5_WQE_DS(copy_b) *
+                                              MLX5_WQE_DWORD_SIZE -
+                                              MLX5_WQE_DWORD_SIZE;
+                               } else {
+                                       /* NOP WQE. */
+                                       wqe->ctrl = (rte_v128u32_t){
+                                                    htonl(txq->wqe_ci << 8),
+                                                    htonl(txq->qp_num_8s | 1),
+                                                    0,
+                                                    0,
+                                       };
+                                       ds = 1;
+                                       total_length = 0;
+                                       pkts--;
+                                       pkts_n++;
+                                       elts_head = (elts_head - 1) &
+                                                   (elts_n - 1);
+                                       k++;
+                                       goto next_wqe;
+                               }
+                       }
+               }
                /* Inline if enough room. */
-               if (txq->max_inline) {
+               if (txq->inline_en || tso) {
                        uintptr_t end = (uintptr_t)
                                (((uintptr_t)txq->wqes) +
                                 (1 << txq->wqe_n) * MLX5_WQE_SIZE);
                        unsigned int max_inline = txq->max_inline *
                                                  RTE_CACHE_LINE_SIZE -
-                                                 MLX5_WQE_DWORD_SIZE;
+                                                 (pkt_inline_sz - 2);
                        uintptr_t addr_end = (addr + max_inline) &
                                             ~(RTE_CACHE_LINE_SIZE - 1);
                        unsigned int copy_b = (addr_end > addr) ?
@@ -567,6 +630,18 @@ mlx5_tx_burst(void *dpdk_txq, struct rte_mbuf **pkts, uint16_t pkts_n)
                                if (unlikely(max_wqe < n))
                                        break;
                                max_wqe -= n;
+                               if (tso) {
+                                       uint32_t inl =
+                                               htonl(copy_b | MLX5_INLINE_SEG);
+
+                                       pkt_inline_sz =
+                                               MLX5_WQE_DS(tso_header_sz) *
+                                               MLX5_WQE_DWORD_SIZE;
+                                       rte_memcpy((void *)raw,
+                                                  (void *)&inl, sizeof(inl));
+                                       raw += sizeof(inl);
+                                       pkt_inline_sz += sizeof(inl);
+                               }
                                rte_memcpy((void *)raw, (void *)addr, copy_b);
                                addr += copy_b;
                                length -= copy_b;
@@ -667,18 +742,34 @@ next_seg:
 next_pkt:
                ++i;
                /* Initialize known and common part of the WQE structure. */
-               wqe->ctrl = (rte_v128u32_t){
-                       htonl((txq->wqe_ci << 8) | MLX5_OPCODE_SEND),
-                       htonl(txq->qp_num_8s | ds),
-                       0,
-                       0,
-               };
-               wqe->eseg = (rte_v128u32_t){
-                       0,
-                       cs_flags,
-                       0,
-                       (ehdr << 16) | htons(pkt_inline_sz),
-               };
+               if (tso) {
+                       wqe->ctrl = (rte_v128u32_t){
+                               htonl((txq->wqe_ci << 8) | MLX5_OPCODE_TSO),
+                               htonl(txq->qp_num_8s | ds),
+                               0,
+                               0,
+                       };
+                       wqe->eseg = (rte_v128u32_t){
+                               0,
+                               cs_flags | (htons(buf->tso_segsz) << 16),
+                               0,
+                               (ehdr << 16) | htons(tso_header_sz),
+                       };
+               } else {
+                       wqe->ctrl = (rte_v128u32_t){
+                               htonl((txq->wqe_ci << 8) | MLX5_OPCODE_SEND),
+                               htonl(txq->qp_num_8s | ds),
+                               0,
+                               0,
+                       };
+                       wqe->eseg = (rte_v128u32_t){
+                               0,
+                               cs_flags,
+                               0,
+                               (ehdr << 16) | htons(pkt_inline_sz),
+                       };
+               }
+next_wqe:
                txq->wqe_ci += (ds + 3) / 4;
 #ifdef MLX5_PMD_SOFT_COUNTERS
                /* Increment sent bytes counter. */
@@ -686,10 +777,10 @@ next_pkt:
 #endif
        } while (pkts_n);
        /* Take a shortcut if nothing must be sent. */
-       if (unlikely(i == 0))
+       if (unlikely((i + k) == 0))
                return 0;
        /* Check whether completion threshold has been reached. */
-       comp = txq->elts_comp + i + j;
+       comp = txq->elts_comp + i + j + k;
        if (comp >= MLX5_TX_COMP_THRESH) {
                volatile struct mlx5_wqe_ctrl *w =
                        (volatile struct mlx5_wqe_ctrl *)wqe;