net/mlx5: add VLAN push/pop DR commands to glue
[dpdk.git] / drivers / net / mlx4 / mlx4_rxtx.c
1 /* SPDX-License-Identifier: BSD-3-Clause
2  * Copyright 2017 6WIND S.A.
3  * Copyright 2017 Mellanox Technologies, Ltd
4  */
5
6 /**
7  * @file
8  * Data plane functions for mlx4 driver.
9  */
10
11 #include <assert.h>
12 #include <stdbool.h>
13 #include <stdint.h>
14 #include <string.h>
15
16 /* Verbs headers do not support -pedantic. */
17 #ifdef PEDANTIC
18 #pragma GCC diagnostic ignored "-Wpedantic"
19 #endif
20 #include <infiniband/verbs.h>
21 #ifdef PEDANTIC
22 #pragma GCC diagnostic error "-Wpedantic"
23 #endif
24
25 #include <rte_branch_prediction.h>
26 #include <rte_common.h>
27 #include <rte_io.h>
28 #include <rte_mbuf.h>
29 #include <rte_mempool.h>
30 #include <rte_prefetch.h>
31
32 #include "mlx4.h"
33 #include "mlx4_prm.h"
34 #include "mlx4_rxtx.h"
35 #include "mlx4_utils.h"
36
37 /**
38  * Pointer-value pair structure used in tx_post_send for saving the first
39  * DWORD (32 byte) of a TXBB.
40  */
41 struct pv {
42         union {
43                 volatile struct mlx4_wqe_data_seg *dseg;
44                 volatile uint32_t *dst;
45         };
46         uint32_t val;
47 };
48
49 /** A helper structure for TSO packet handling. */
50 struct tso_info {
51         /** Pointer to the array of saved first DWORD (32 byte) of a TXBB. */
52         struct pv *pv;
53         /** Current entry in the pv array. */
54         int pv_counter;
55         /** Total size of the WQE including padding. */
56         uint32_t wqe_size;
57         /** Size of TSO header to prepend to each packet to send. */
58         uint16_t tso_header_size;
59         /** Total size of the TSO segment in the WQE. */
60         uint16_t wqe_tso_seg_size;
61         /** Raw WQE size in units of 16 Bytes and without padding. */
62         uint8_t fence_size;
63 };
64
65 /** A table to translate Rx completion flags to packet type. */
66 uint32_t mlx4_ptype_table[0x100] __rte_cache_aligned = {
67         /*
68          * The index to the array should have:
69          *  bit[7] - MLX4_CQE_L2_TUNNEL
70          *  bit[6] - MLX4_CQE_L2_TUNNEL_IPV4
71          *  bit[5] - MLX4_CQE_STATUS_UDP
72          *  bit[4] - MLX4_CQE_STATUS_TCP
73          *  bit[3] - MLX4_CQE_STATUS_IPV4OPT
74          *  bit[2] - MLX4_CQE_STATUS_IPV6
75          *  bit[1] - MLX4_CQE_STATUS_IPF
76          *  bit[0] - MLX4_CQE_STATUS_IPV4
77          * giving a total of up to 256 entries.
78          */
79         /* L2 */
80         [0x00] = RTE_PTYPE_L2_ETHER,
81         /* L3 */
82         [0x01] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
83                      RTE_PTYPE_L4_NONFRAG,
84         [0x02] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
85                      RTE_PTYPE_L4_FRAG,
86         [0x03] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
87                      RTE_PTYPE_L4_FRAG,
88         [0x04] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
89                      RTE_PTYPE_L4_NONFRAG,
90         [0x06] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
91                      RTE_PTYPE_L4_FRAG,
92         [0x08] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT |
93                      RTE_PTYPE_L4_NONFRAG,
94         [0x09] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT |
95                      RTE_PTYPE_L4_NONFRAG,
96         [0x0a] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT |
97                      RTE_PTYPE_L4_FRAG,
98         [0x0b] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT |
99                      RTE_PTYPE_L4_FRAG,
100         /* TCP */
101         [0x11] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
102                      RTE_PTYPE_L4_TCP,
103         [0x14] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
104                      RTE_PTYPE_L4_TCP,
105         [0x16] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
106                      RTE_PTYPE_L4_FRAG,
107         [0x18] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT |
108                      RTE_PTYPE_L4_TCP,
109         [0x19] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT |
110                      RTE_PTYPE_L4_TCP,
111         /* UDP */
112         [0x21] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
113                      RTE_PTYPE_L4_UDP,
114         [0x24] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
115                      RTE_PTYPE_L4_UDP,
116         [0x26] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
117                      RTE_PTYPE_L4_FRAG,
118         [0x28] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT |
119                      RTE_PTYPE_L4_UDP,
120         [0x29] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT |
121                      RTE_PTYPE_L4_UDP,
122         /* Tunneled - L3 IPV6 */
123         [0x80] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN,
124         [0x81] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
125                      RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
126                      RTE_PTYPE_INNER_L4_NONFRAG,
127         [0x82] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
128                      RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
129                      RTE_PTYPE_INNER_L4_FRAG,
130         [0x83] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
131                      RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
132                      RTE_PTYPE_INNER_L4_FRAG,
133         [0x84] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
134                      RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
135                      RTE_PTYPE_INNER_L4_NONFRAG,
136         [0x86] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
137                      RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
138                      RTE_PTYPE_INNER_L4_FRAG,
139         [0x88] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
140                      RTE_PTYPE_INNER_L3_IPV4_EXT |
141                      RTE_PTYPE_INNER_L4_NONFRAG,
142         [0x89] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
143                      RTE_PTYPE_INNER_L3_IPV4_EXT |
144                      RTE_PTYPE_INNER_L4_NONFRAG,
145         [0x8a] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
146                      RTE_PTYPE_INNER_L3_IPV4_EXT |
147                      RTE_PTYPE_INNER_L4_FRAG,
148         [0x8b] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
149                      RTE_PTYPE_INNER_L3_IPV4_EXT |
150                      RTE_PTYPE_INNER_L4_FRAG,
151         /* Tunneled - L3 IPV6, TCP */
152         [0x91] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
153                      RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
154                      RTE_PTYPE_INNER_L4_TCP,
155         [0x94] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
156                      RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
157                      RTE_PTYPE_INNER_L4_TCP,
158         [0x96] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
159                      RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
160                      RTE_PTYPE_INNER_L4_FRAG,
161         [0x98] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
162                      RTE_PTYPE_INNER_L3_IPV4_EXT | RTE_PTYPE_INNER_L4_TCP,
163         [0x99] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
164                      RTE_PTYPE_INNER_L3_IPV4_EXT | RTE_PTYPE_INNER_L4_TCP,
165         /* Tunneled - L3 IPV6, UDP */
166         [0xa1] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
167                      RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
168                      RTE_PTYPE_INNER_L4_UDP,
169         [0xa4] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
170                      RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
171                      RTE_PTYPE_INNER_L4_UDP,
172         [0xa6] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
173                      RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
174                      RTE_PTYPE_INNER_L4_FRAG,
175         [0xa8] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
176                      RTE_PTYPE_INNER_L3_IPV4_EXT |
177                      RTE_PTYPE_INNER_L4_UDP,
178         [0xa9] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
179                      RTE_PTYPE_INNER_L3_IPV4_EXT |
180                      RTE_PTYPE_INNER_L4_UDP,
181         /* Tunneled - L3 IPV4 */
182         [0xc0] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN,
183         [0xc1] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
184                      RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
185                      RTE_PTYPE_INNER_L4_NONFRAG,
186         [0xc2] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
187                      RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
188                      RTE_PTYPE_INNER_L4_FRAG,
189         [0xc3] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
190                      RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
191                      RTE_PTYPE_INNER_L4_FRAG,
192         [0xc4] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
193                      RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
194                      RTE_PTYPE_INNER_L4_NONFRAG,
195         [0xc6] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
196                      RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
197                      RTE_PTYPE_INNER_L4_FRAG,
198         [0xc8] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
199                      RTE_PTYPE_INNER_L3_IPV4_EXT |
200                      RTE_PTYPE_INNER_L4_NONFRAG,
201         [0xc9] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
202                      RTE_PTYPE_INNER_L3_IPV4_EXT |
203                      RTE_PTYPE_INNER_L4_NONFRAG,
204         [0xca] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
205                      RTE_PTYPE_INNER_L3_IPV4_EXT |
206                      RTE_PTYPE_INNER_L4_FRAG,
207         [0xcb] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
208                      RTE_PTYPE_INNER_L3_IPV4_EXT |
209                      RTE_PTYPE_INNER_L4_FRAG,
210         /* Tunneled - L3 IPV4, TCP */
211         [0xd1] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
212                      RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
213                      RTE_PTYPE_INNER_L4_TCP,
214         [0xd4] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
215                      RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
216                      RTE_PTYPE_INNER_L4_TCP,
217         [0xd6] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
218                      RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
219                      RTE_PTYPE_INNER_L4_FRAG,
220         [0xd8] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
221                      RTE_PTYPE_INNER_L3_IPV4_EXT |
222                      RTE_PTYPE_INNER_L4_TCP,
223         [0xd9] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
224                      RTE_PTYPE_INNER_L3_IPV4_EXT |
225                      RTE_PTYPE_INNER_L4_TCP,
226         /* Tunneled - L3 IPV4, UDP */
227         [0xe1] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
228                      RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
229                      RTE_PTYPE_INNER_L4_UDP,
230         [0xe4] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
231                      RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
232                      RTE_PTYPE_INNER_L4_UDP,
233         [0xe6] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
234                      RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
235                      RTE_PTYPE_INNER_L4_FRAG,
236         [0xe8] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
237                      RTE_PTYPE_INNER_L3_IPV4_EXT |
238                      RTE_PTYPE_INNER_L4_UDP,
239         [0xe9] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
240                      RTE_PTYPE_INNER_L3_IPV4_EXT |
241                      RTE_PTYPE_INNER_L4_UDP,
242 };
243
244 /**
245  * Stamp TXBB burst so it won't be reused by the HW.
246  *
247  * Routine is used when freeing WQE used by the chip or when failing
248  * building an WQ entry has failed leaving partial information on the queue.
249  *
250  * @param sq
251  *   Pointer to the SQ structure.
252  * @param start
253  *   Pointer to the first TXBB to stamp.
254  * @param end
255  *   Pointer to the followed end TXBB to stamp.
256  *
257  * @return
258  *   Stamping burst size in byte units.
259  */
260 static uint32_t
261 mlx4_txq_stamp_freed_wqe(struct mlx4_sq *sq, volatile uint32_t *start,
262                          volatile uint32_t *end)
263 {
264         uint32_t stamp = sq->stamp;
265         int32_t size = (intptr_t)end - (intptr_t)start;
266
267         assert(start != end);
268         /* Hold SQ ring wrap around. */
269         if (size < 0) {
270                 size = (int32_t)sq->size + size;
271                 do {
272                         *start = stamp;
273                         start += MLX4_SQ_STAMP_DWORDS;
274                 } while (start != (volatile uint32_t *)sq->eob);
275                 start = (volatile uint32_t *)sq->buf;
276                 /* Flip invalid stamping ownership. */
277                 stamp ^= RTE_BE32(1u << MLX4_SQ_OWNER_BIT);
278                 sq->stamp = stamp;
279                 if (start == end)
280                         return size;
281         }
282         do {
283                 *start = stamp;
284                 start += MLX4_SQ_STAMP_DWORDS;
285         } while (start != end);
286         return (uint32_t)size;
287 }
288
289 /**
290  * Manage Tx completions.
291  *
292  * When sending a burst, mlx4_tx_burst() posts several WRs.
293  * To improve performance, a completion event is only required once every
294  * MLX4_PMD_TX_PER_COMP_REQ sends. Doing so discards completion information
295  * for other WRs, but this information would not be used anyway.
296  *
297  * @param txq
298  *   Pointer to Tx queue structure.
299  * @param elts_m
300  *   Tx elements number mask.
301  * @param sq
302  *   Pointer to the SQ structure.
303  */
304 static void
305 mlx4_txq_complete(struct txq *txq, const unsigned int elts_m,
306                   struct mlx4_sq *sq)
307 {
308         unsigned int elts_tail = txq->elts_tail;
309         struct mlx4_cq *cq = &txq->mcq;
310         volatile struct mlx4_cqe *cqe;
311         uint32_t completed;
312         uint32_t cons_index = cq->cons_index;
313         volatile uint32_t *first_txbb;
314
315         /*
316          * Traverse over all CQ entries reported and handle each WQ entry
317          * reported by them.
318          */
319         do {
320                 cqe = (volatile struct mlx4_cqe *)mlx4_get_cqe(cq, cons_index);
321                 if (unlikely(!!(cqe->owner_sr_opcode & MLX4_CQE_OWNER_MASK) ^
322                     !!(cons_index & cq->cqe_cnt)))
323                         break;
324 #ifndef NDEBUG
325                 /*
326                  * Make sure we read the CQE after we read the ownership bit.
327                  */
328                 rte_io_rmb();
329                 if (unlikely((cqe->owner_sr_opcode & MLX4_CQE_OPCODE_MASK) ==
330                              MLX4_CQE_OPCODE_ERROR)) {
331                         volatile struct mlx4_err_cqe *cqe_err =
332                                 (volatile struct mlx4_err_cqe *)cqe;
333                         ERROR("%p CQE error - vendor syndrome: 0x%x"
334                               " syndrome: 0x%x\n",
335                               (void *)txq, cqe_err->vendor_err,
336                               cqe_err->syndrome);
337                         break;
338                 }
339 #endif /* NDEBUG */
340                 cons_index++;
341         } while (1);
342         completed = (cons_index - cq->cons_index) * txq->elts_comp_cd_init;
343         if (unlikely(!completed))
344                 return;
345         /* First stamping address is the end of the last one. */
346         first_txbb = (&(*txq->elts)[elts_tail & elts_m])->eocb;
347         elts_tail += completed;
348         /* The new tail element holds the end address. */
349         sq->remain_size += mlx4_txq_stamp_freed_wqe(sq, first_txbb,
350                 (&(*txq->elts)[elts_tail & elts_m])->eocb);
351         /* Update CQ consumer index. */
352         cq->cons_index = cons_index;
353         *cq->set_ci_db = rte_cpu_to_be_32(cons_index & MLX4_CQ_DB_CI_MASK);
354         txq->elts_tail = elts_tail;
355 }
356
357 /**
358  * Write Tx data segment to the SQ.
359  *
360  * @param dseg
361  *   Pointer to data segment in SQ.
362  * @param lkey
363  *   Memory region lkey.
364  * @param addr
365  *   Data address.
366  * @param byte_count
367  *   Big endian bytes count of the data to send.
368  */
369 static inline void
370 mlx4_fill_tx_data_seg(volatile struct mlx4_wqe_data_seg *dseg,
371                        uint32_t lkey, uintptr_t addr, rte_be32_t  byte_count)
372 {
373         dseg->addr = rte_cpu_to_be_64(addr);
374         dseg->lkey = lkey;
375 #if RTE_CACHE_LINE_SIZE < 64
376         /*
377          * Need a barrier here before writing the byte_count
378          * fields to make sure that all the data is visible
379          * before the byte_count field is set.
380          * Otherwise, if the segment begins a new cacheline,
381          * the HCA prefetcher could grab the 64-byte chunk and
382          * get a valid (!= 0xffffffff) byte count but stale
383          * data, and end up sending the wrong data.
384          */
385         rte_io_wmb();
386 #endif /* RTE_CACHE_LINE_SIZE */
387         dseg->byte_count = byte_count;
388 }
389
390 /**
391  * Obtain and calculate TSO information needed for assembling a TSO WQE.
392  *
393  * @param buf
394  *   Pointer to the first packet mbuf.
395  * @param txq
396  *   Pointer to Tx queue structure.
397  * @param tinfo
398  *   Pointer to a structure to fill the info with.
399  *
400  * @return
401  *   0 on success, negative value upon error.
402  */
403 static inline int
404 mlx4_tx_burst_tso_get_params(struct rte_mbuf *buf,
405                              struct txq *txq,
406                              struct tso_info *tinfo)
407 {
408         struct mlx4_sq *sq = &txq->msq;
409         const uint8_t tunneled = txq->priv->hw_csum_l2tun &&
410                                  (buf->ol_flags & PKT_TX_TUNNEL_MASK);
411
412         tinfo->tso_header_size = buf->l2_len + buf->l3_len + buf->l4_len;
413         if (tunneled)
414                 tinfo->tso_header_size +=
415                                 buf->outer_l2_len + buf->outer_l3_len;
416         if (unlikely(buf->tso_segsz == 0 ||
417                      tinfo->tso_header_size == 0 ||
418                      tinfo->tso_header_size > MLX4_MAX_TSO_HEADER ||
419                      tinfo->tso_header_size > buf->data_len))
420                 return -EINVAL;
421         /*
422          * Calculate the WQE TSO segment size
423          * Note:
424          * 1. An LSO segment must be padded such that the subsequent data
425          *    segment is 16-byte aligned.
426          * 2. The start address of the TSO segment is always 16 Bytes aligned.
427          */
428         tinfo->wqe_tso_seg_size = RTE_ALIGN(sizeof(struct mlx4_wqe_lso_seg) +
429                                             tinfo->tso_header_size,
430                                             sizeof(struct mlx4_wqe_data_seg));
431         tinfo->fence_size = ((sizeof(struct mlx4_wqe_ctrl_seg) +
432                              tinfo->wqe_tso_seg_size) >> MLX4_SEG_SHIFT) +
433                              buf->nb_segs;
434         tinfo->wqe_size =
435                 RTE_ALIGN((uint32_t)(tinfo->fence_size << MLX4_SEG_SHIFT),
436                           MLX4_TXBB_SIZE);
437         /* Validate WQE size and WQE space in the send queue. */
438         if (sq->remain_size < tinfo->wqe_size ||
439             tinfo->wqe_size > MLX4_MAX_WQE_SIZE)
440                 return -ENOMEM;
441         /* Init pv. */
442         tinfo->pv = (struct pv *)txq->bounce_buf;
443         tinfo->pv_counter = 0;
444         return 0;
445 }
446
447 /**
448  * Fill the TSO WQE data segments with info on buffers to transmit .
449  *
450  * @param buf
451  *   Pointer to the first packet mbuf.
452  * @param txq
453  *   Pointer to Tx queue structure.
454  * @param tinfo
455  *   Pointer to TSO info to use.
456  * @param dseg
457  *   Pointer to the first data segment in the TSO WQE.
458  * @param ctrl
459  *   Pointer to the control segment in the TSO WQE.
460  *
461  * @return
462  *   0 on success, negative value upon error.
463  */
464 static inline volatile struct mlx4_wqe_ctrl_seg *
465 mlx4_tx_burst_fill_tso_dsegs(struct rte_mbuf *buf,
466                              struct txq *txq,
467                              struct tso_info *tinfo,
468                              volatile struct mlx4_wqe_data_seg *dseg,
469                              volatile struct mlx4_wqe_ctrl_seg *ctrl)
470 {
471         uint32_t lkey;
472         int nb_segs = buf->nb_segs;
473         int nb_segs_txbb;
474         struct mlx4_sq *sq = &txq->msq;
475         struct rte_mbuf *sbuf = buf;
476         struct pv *pv = tinfo->pv;
477         int *pv_counter = &tinfo->pv_counter;
478         volatile struct mlx4_wqe_ctrl_seg *ctrl_next =
479                         (volatile struct mlx4_wqe_ctrl_seg *)
480                                 ((volatile uint8_t *)ctrl + tinfo->wqe_size);
481         uint16_t data_len = sbuf->data_len - tinfo->tso_header_size;
482         uintptr_t data_addr = rte_pktmbuf_mtod_offset(sbuf, uintptr_t,
483                                                       tinfo->tso_header_size);
484
485         do {
486                 /* how many dseg entries do we have in the current TXBB ? */
487                 nb_segs_txbb = (MLX4_TXBB_SIZE -
488                                 ((uintptr_t)dseg & (MLX4_TXBB_SIZE - 1))) >>
489                                MLX4_SEG_SHIFT;
490                 switch (nb_segs_txbb) {
491 #ifndef NDEBUG
492                 default:
493                         /* Should never happen. */
494                         rte_panic("%p: Invalid number of SGEs(%d) for a TXBB",
495                         (void *)txq, nb_segs_txbb);
496                         /* rte_panic never returns. */
497                         break;
498 #endif /* NDEBUG */
499                 case 4:
500                         /* Memory region key for this memory pool. */
501                         lkey = mlx4_tx_mb2mr(txq, sbuf);
502                         if (unlikely(lkey == (uint32_t)-1))
503                                 goto err;
504                         dseg->addr = rte_cpu_to_be_64(data_addr);
505                         dseg->lkey = lkey;
506                         /*
507                          * This data segment starts at the beginning of a new
508                          * TXBB, so we need to postpone its byte_count writing
509                          * for later.
510                          */
511                         pv[*pv_counter].dseg = dseg;
512                         /*
513                          * Zero length segment is treated as inline segment
514                          * with zero data.
515                          */
516                         pv[(*pv_counter)++].val =
517                                 rte_cpu_to_be_32(data_len ?
518                                                  data_len :
519                                                  0x80000000);
520                         if (--nb_segs == 0)
521                                 return ctrl_next;
522                         /* Prepare next buf info */
523                         sbuf = sbuf->next;
524                         dseg++;
525                         data_len = sbuf->data_len;
526                         data_addr = rte_pktmbuf_mtod(sbuf, uintptr_t);
527                         /* fallthrough */
528                 case 3:
529                         lkey = mlx4_tx_mb2mr(txq, sbuf);
530                         if (unlikely(lkey == (uint32_t)-1))
531                                 goto err;
532                         mlx4_fill_tx_data_seg(dseg, lkey, data_addr,
533                                         rte_cpu_to_be_32(data_len ?
534                                                          data_len :
535                                                          0x80000000));
536                         if (--nb_segs == 0)
537                                 return ctrl_next;
538                         /* Prepare next buf info */
539                         sbuf = sbuf->next;
540                         dseg++;
541                         data_len = sbuf->data_len;
542                         data_addr = rte_pktmbuf_mtod(sbuf, uintptr_t);
543                         /* fallthrough */
544                 case 2:
545                         lkey = mlx4_tx_mb2mr(txq, sbuf);
546                         if (unlikely(lkey == (uint32_t)-1))
547                                 goto err;
548                         mlx4_fill_tx_data_seg(dseg, lkey, data_addr,
549                                         rte_cpu_to_be_32(data_len ?
550                                                          data_len :
551                                                          0x80000000));
552                         if (--nb_segs == 0)
553                                 return ctrl_next;
554                         /* Prepare next buf info */
555                         sbuf = sbuf->next;
556                         dseg++;
557                         data_len = sbuf->data_len;
558                         data_addr = rte_pktmbuf_mtod(sbuf, uintptr_t);
559                         /* fallthrough */
560                 case 1:
561                         lkey = mlx4_tx_mb2mr(txq, sbuf);
562                         if (unlikely(lkey == (uint32_t)-1))
563                                 goto err;
564                         mlx4_fill_tx_data_seg(dseg, lkey, data_addr,
565                                         rte_cpu_to_be_32(data_len ?
566                                                          data_len :
567                                                          0x80000000));
568                         if (--nb_segs == 0)
569                                 return ctrl_next;
570                         /* Prepare next buf info */
571                         sbuf = sbuf->next;
572                         dseg++;
573                         data_len = sbuf->data_len;
574                         data_addr = rte_pktmbuf_mtod(sbuf, uintptr_t);
575                         /* fallthrough */
576                 }
577                 /* Wrap dseg if it points at the end of the queue. */
578                 if ((volatile uint8_t *)dseg >= sq->eob)
579                         dseg = (volatile struct mlx4_wqe_data_seg *)
580                                         ((volatile uint8_t *)dseg - sq->size);
581         } while (true);
582 err:
583         return NULL;
584 }
585
586 /**
587  * Fill the packet's l2, l3 and l4 headers to the WQE.
588  *
589  * This will be used as the header for each TSO segment that is transmitted.
590  *
591  * @param buf
592  *   Pointer to the first packet mbuf.
593  * @param txq
594  *   Pointer to Tx queue structure.
595  * @param tinfo
596  *   Pointer to TSO info to use.
597  * @param ctrl
598  *   Pointer to the control segment in the TSO WQE.
599  *
600  * @return
601  *   0 on success, negative value upon error.
602  */
603 static inline volatile struct mlx4_wqe_data_seg *
604 mlx4_tx_burst_fill_tso_hdr(struct rte_mbuf *buf,
605                            struct txq *txq,
606                            struct tso_info *tinfo,
607                            volatile struct mlx4_wqe_ctrl_seg *ctrl)
608 {
609         volatile struct mlx4_wqe_lso_seg *tseg =
610                 (volatile struct mlx4_wqe_lso_seg *)(ctrl + 1);
611         struct mlx4_sq *sq = &txq->msq;
612         struct pv *pv = tinfo->pv;
613         int *pv_counter = &tinfo->pv_counter;
614         int remain_size = tinfo->tso_header_size;
615         char *from = rte_pktmbuf_mtod(buf, char *);
616         uint16_t txbb_avail_space;
617         /* Union to overcome volatile constraints when copying TSO header. */
618         union {
619                 volatile uint8_t *vto;
620                 uint8_t *to;
621         } thdr = { .vto = (volatile uint8_t *)tseg->header, };
622
623         /*
624          * TSO data always starts at offset 20 from the beginning of the TXBB
625          * (16 byte ctrl + 4byte TSO desc). Since each TXBB is 64Byte aligned
626          * we can write the first 44 TSO header bytes without worry for TxQ
627          * wrapping or overwriting the first TXBB 32bit word.
628          */
629         txbb_avail_space = MLX4_TXBB_SIZE -
630                            (sizeof(struct mlx4_wqe_ctrl_seg) +
631                             sizeof(struct mlx4_wqe_lso_seg));
632         while (remain_size >= (int)(txbb_avail_space + sizeof(uint32_t))) {
633                 /* Copy to end of txbb. */
634                 rte_memcpy(thdr.to, from, txbb_avail_space);
635                 from += txbb_avail_space;
636                 thdr.to += txbb_avail_space;
637                 /* New TXBB, Check for TxQ wrap. */
638                 if (thdr.to >= sq->eob)
639                         thdr.vto = sq->buf;
640                 /* New TXBB, stash the first 32bits for later use. */
641                 pv[*pv_counter].dst = (volatile uint32_t *)thdr.to;
642                 pv[(*pv_counter)++].val = *(uint32_t *)from,
643                 from += sizeof(uint32_t);
644                 thdr.to += sizeof(uint32_t);
645                 remain_size -= txbb_avail_space + sizeof(uint32_t);
646                 /* Avail space in new TXBB is TXBB size - 4 */
647                 txbb_avail_space = MLX4_TXBB_SIZE - sizeof(uint32_t);
648         }
649         if (remain_size > txbb_avail_space) {
650                 rte_memcpy(thdr.to, from, txbb_avail_space);
651                 from += txbb_avail_space;
652                 thdr.to += txbb_avail_space;
653                 remain_size -= txbb_avail_space;
654                 /* New TXBB, Check for TxQ wrap. */
655                 if (thdr.to >= sq->eob)
656                         thdr.vto = sq->buf;
657                 pv[*pv_counter].dst = (volatile uint32_t *)thdr.to;
658                 rte_memcpy(&pv[*pv_counter].val, from, remain_size);
659                 (*pv_counter)++;
660         } else if (remain_size) {
661                 rte_memcpy(thdr.to, from, remain_size);
662         }
663         tseg->mss_hdr_size = rte_cpu_to_be_32((buf->tso_segsz << 16) |
664                                               tinfo->tso_header_size);
665         /* Calculate data segment location */
666         return (volatile struct mlx4_wqe_data_seg *)
667                                 ((uintptr_t)tseg + tinfo->wqe_tso_seg_size);
668 }
669
670 /**
671  * Write data segments and header for TSO uni/multi segment packet.
672  *
673  * @param buf
674  *   Pointer to the first packet mbuf.
675  * @param txq
676  *   Pointer to Tx queue structure.
677  * @param ctrl
678  *   Pointer to the WQE control segment.
679  *
680  * @return
681  *   Pointer to the next WQE control segment on success, NULL otherwise.
682  */
683 static volatile struct mlx4_wqe_ctrl_seg *
684 mlx4_tx_burst_tso(struct rte_mbuf *buf, struct txq *txq,
685                   volatile struct mlx4_wqe_ctrl_seg *ctrl)
686 {
687         volatile struct mlx4_wqe_data_seg *dseg;
688         volatile struct mlx4_wqe_ctrl_seg *ctrl_next;
689         struct mlx4_sq *sq = &txq->msq;
690         struct tso_info tinfo;
691         struct pv *pv;
692         int pv_counter;
693         int ret;
694
695         ret = mlx4_tx_burst_tso_get_params(buf, txq, &tinfo);
696         if (unlikely(ret))
697                 goto error;
698         dseg = mlx4_tx_burst_fill_tso_hdr(buf, txq, &tinfo, ctrl);
699         if (unlikely(dseg == NULL))
700                 goto error;
701         if ((uintptr_t)dseg >= (uintptr_t)sq->eob)
702                 dseg = (volatile struct mlx4_wqe_data_seg *)
703                                         ((uintptr_t)dseg - sq->size);
704         ctrl_next = mlx4_tx_burst_fill_tso_dsegs(buf, txq, &tinfo, dseg, ctrl);
705         if (unlikely(ctrl_next == NULL))
706                 goto error;
707         /* Write the first DWORD of each TXBB save earlier. */
708         if (likely(tinfo.pv_counter)) {
709                 pv = tinfo.pv;
710                 pv_counter = tinfo.pv_counter;
711                 /* Need a barrier here before writing the first TXBB word. */
712                 rte_io_wmb();
713                 do {
714                         --pv_counter;
715                         *pv[pv_counter].dst = pv[pv_counter].val;
716                 } while (pv_counter > 0);
717         }
718         ctrl->fence_size = tinfo.fence_size;
719         sq->remain_size -= tinfo.wqe_size;
720         return ctrl_next;
721 error:
722         txq->stats.odropped++;
723         return NULL;
724 }
725
726 /**
727  * Write data segments of multi-segment packet.
728  *
729  * @param buf
730  *   Pointer to the first packet mbuf.
731  * @param txq
732  *   Pointer to Tx queue structure.
733  * @param ctrl
734  *   Pointer to the WQE control segment.
735  *
736  * @return
737  *   Pointer to the next WQE control segment on success, NULL otherwise.
738  */
739 static volatile struct mlx4_wqe_ctrl_seg *
740 mlx4_tx_burst_segs(struct rte_mbuf *buf, struct txq *txq,
741                    volatile struct mlx4_wqe_ctrl_seg *ctrl)
742 {
743         struct pv *pv = (struct pv *)txq->bounce_buf;
744         struct mlx4_sq *sq = &txq->msq;
745         struct rte_mbuf *sbuf = buf;
746         uint32_t lkey;
747         int pv_counter = 0;
748         int nb_segs = buf->nb_segs;
749         uint32_t wqe_size;
750         volatile struct mlx4_wqe_data_seg *dseg =
751                 (volatile struct mlx4_wqe_data_seg *)(ctrl + 1);
752
753         ctrl->fence_size = 1 + nb_segs;
754         wqe_size = RTE_ALIGN((uint32_t)(ctrl->fence_size << MLX4_SEG_SHIFT),
755                              MLX4_TXBB_SIZE);
756         /* Validate WQE size and WQE space in the send queue. */
757         if (sq->remain_size < wqe_size ||
758             wqe_size > MLX4_MAX_WQE_SIZE)
759                 return NULL;
760         /*
761          * Fill the data segments with buffer information.
762          * First WQE TXBB head segment is always control segment,
763          * so jump to tail TXBB data segments code for the first
764          * WQE data segments filling.
765          */
766         goto txbb_tail_segs;
767 txbb_head_seg:
768         /* Memory region key (big endian) for this memory pool. */
769         lkey = mlx4_tx_mb2mr(txq, sbuf);
770         if (unlikely(lkey == (uint32_t)-1)) {
771                 DEBUG("%p: unable to get MP <-> MR association",
772                       (void *)txq);
773                 return NULL;
774         }
775         /* Handle WQE wraparound. */
776         if (dseg >=
777                 (volatile struct mlx4_wqe_data_seg *)sq->eob)
778                 dseg = (volatile struct mlx4_wqe_data_seg *)
779                         sq->buf;
780         dseg->addr = rte_cpu_to_be_64(rte_pktmbuf_mtod(sbuf, uintptr_t));
781         dseg->lkey = lkey;
782         /*
783          * This data segment starts at the beginning of a new
784          * TXBB, so we need to postpone its byte_count writing
785          * for later.
786          */
787         pv[pv_counter].dseg = dseg;
788         /*
789          * Zero length segment is treated as inline segment
790          * with zero data.
791          */
792         pv[pv_counter++].val = rte_cpu_to_be_32(sbuf->data_len ?
793                                                 sbuf->data_len : 0x80000000);
794         sbuf = sbuf->next;
795         dseg++;
796         nb_segs--;
797 txbb_tail_segs:
798         /* Jump to default if there are more than two segments remaining. */
799         switch (nb_segs) {
800         default:
801                 lkey = mlx4_tx_mb2mr(txq, sbuf);
802                 if (unlikely(lkey == (uint32_t)-1)) {
803                         DEBUG("%p: unable to get MP <-> MR association",
804                               (void *)txq);
805                         return NULL;
806                 }
807                 mlx4_fill_tx_data_seg(dseg, lkey,
808                                       rte_pktmbuf_mtod(sbuf, uintptr_t),
809                                       rte_cpu_to_be_32(sbuf->data_len ?
810                                                        sbuf->data_len :
811                                                        0x80000000));
812                 sbuf = sbuf->next;
813                 dseg++;
814                 nb_segs--;
815                 /* fallthrough */
816         case 2:
817                 lkey = mlx4_tx_mb2mr(txq, sbuf);
818                 if (unlikely(lkey == (uint32_t)-1)) {
819                         DEBUG("%p: unable to get MP <-> MR association",
820                               (void *)txq);
821                         return NULL;
822                 }
823                 mlx4_fill_tx_data_seg(dseg, lkey,
824                                       rte_pktmbuf_mtod(sbuf, uintptr_t),
825                                       rte_cpu_to_be_32(sbuf->data_len ?
826                                                        sbuf->data_len :
827                                                        0x80000000));
828                 sbuf = sbuf->next;
829                 dseg++;
830                 nb_segs--;
831                 /* fallthrough */
832         case 1:
833                 lkey = mlx4_tx_mb2mr(txq, sbuf);
834                 if (unlikely(lkey == (uint32_t)-1)) {
835                         DEBUG("%p: unable to get MP <-> MR association",
836                               (void *)txq);
837                         return NULL;
838                 }
839                 mlx4_fill_tx_data_seg(dseg, lkey,
840                                       rte_pktmbuf_mtod(sbuf, uintptr_t),
841                                       rte_cpu_to_be_32(sbuf->data_len ?
842                                                        sbuf->data_len :
843                                                        0x80000000));
844                 nb_segs--;
845                 if (nb_segs) {
846                         sbuf = sbuf->next;
847                         dseg++;
848                         goto txbb_head_seg;
849                 }
850                 /* fallthrough */
851         case 0:
852                 break;
853         }
854         /* Write the first DWORD of each TXBB save earlier. */
855         if (pv_counter) {
856                 /* Need a barrier here before writing the byte_count. */
857                 rte_io_wmb();
858                 for (--pv_counter; pv_counter  >= 0; pv_counter--)
859                         pv[pv_counter].dseg->byte_count = pv[pv_counter].val;
860         }
861         sq->remain_size -= wqe_size;
862         /* Align next WQE address to the next TXBB. */
863         return (volatile struct mlx4_wqe_ctrl_seg *)
864                 ((volatile uint8_t *)ctrl + wqe_size);
865 }
866
867 /**
868  * DPDK callback for Tx.
869  *
870  * @param dpdk_txq
871  *   Generic pointer to Tx queue structure.
872  * @param[in] pkts
873  *   Packets to transmit.
874  * @param pkts_n
875  *   Number of packets in array.
876  *
877  * @return
878  *   Number of packets successfully transmitted (<= pkts_n).
879  */
880 uint16_t
881 mlx4_tx_burst(void *dpdk_txq, struct rte_mbuf **pkts, uint16_t pkts_n)
882 {
883         struct txq *txq = (struct txq *)dpdk_txq;
884         unsigned int elts_head = txq->elts_head;
885         const unsigned int elts_n = txq->elts_n;
886         const unsigned int elts_m = elts_n - 1;
887         unsigned int bytes_sent = 0;
888         unsigned int i;
889         unsigned int max = elts_head - txq->elts_tail;
890         struct mlx4_sq *sq = &txq->msq;
891         volatile struct mlx4_wqe_ctrl_seg *ctrl;
892         struct txq_elt *elt;
893
894         assert(txq->elts_comp_cd != 0);
895         if (likely(max >= txq->elts_comp_cd_init))
896                 mlx4_txq_complete(txq, elts_m, sq);
897         max = elts_n - max;
898         assert(max >= 1);
899         assert(max <= elts_n);
900         /* Always leave one free entry in the ring. */
901         --max;
902         if (max > pkts_n)
903                 max = pkts_n;
904         elt = &(*txq->elts)[elts_head & elts_m];
905         /* First Tx burst element saves the next WQE control segment. */
906         ctrl = elt->wqe;
907         for (i = 0; (i != max); ++i) {
908                 struct rte_mbuf *buf = pkts[i];
909                 struct txq_elt *elt_next = &(*txq->elts)[++elts_head & elts_m];
910                 uint32_t owner_opcode = sq->owner_opcode;
911                 volatile struct mlx4_wqe_data_seg *dseg =
912                                 (volatile struct mlx4_wqe_data_seg *)(ctrl + 1);
913                 volatile struct mlx4_wqe_ctrl_seg *ctrl_next;
914                 union {
915                         uint32_t flags;
916                         uint16_t flags16[2];
917                 } srcrb;
918                 uint32_t lkey;
919                 bool tso = txq->priv->tso && (buf->ol_flags & PKT_TX_TCP_SEG);
920
921                 /* Clean up old buffer. */
922                 if (likely(elt->buf != NULL)) {
923                         struct rte_mbuf *tmp = elt->buf;
924
925 #ifndef NDEBUG
926                         /* Poisoning. */
927                         memset(&elt->buf, 0x66, sizeof(struct rte_mbuf *));
928 #endif
929                         /* Faster than rte_pktmbuf_free(). */
930                         do {
931                                 struct rte_mbuf *next = tmp->next;
932
933                                 rte_pktmbuf_free_seg(tmp);
934                                 tmp = next;
935                         } while (tmp != NULL);
936                 }
937                 RTE_MBUF_PREFETCH_TO_FREE(elt_next->buf);
938                 if (tso) {
939                         /* Change opcode to TSO */
940                         owner_opcode &= ~MLX4_OPCODE_CONFIG_CMD;
941                         owner_opcode |= MLX4_OPCODE_LSO | MLX4_WQE_CTRL_RR;
942                         ctrl_next = mlx4_tx_burst_tso(buf, txq, ctrl);
943                         if (!ctrl_next) {
944                                 elt->buf = NULL;
945                                 break;
946                         }
947                 } else if (buf->nb_segs == 1) {
948                         /* Validate WQE space in the send queue. */
949                         if (sq->remain_size < MLX4_TXBB_SIZE) {
950                                 elt->buf = NULL;
951                                 break;
952                         }
953                         lkey = mlx4_tx_mb2mr(txq, buf);
954                         if (unlikely(lkey == (uint32_t)-1)) {
955                                 /* MR does not exist. */
956                                 DEBUG("%p: unable to get MP <-> MR association",
957                                       (void *)txq);
958                                 elt->buf = NULL;
959                                 break;
960                         }
961                         mlx4_fill_tx_data_seg(dseg++, lkey,
962                                               rte_pktmbuf_mtod(buf, uintptr_t),
963                                               rte_cpu_to_be_32(buf->data_len));
964                         /* Set WQE size in 16-byte units. */
965                         ctrl->fence_size = 0x2;
966                         sq->remain_size -= MLX4_TXBB_SIZE;
967                         /* Align next WQE address to the next TXBB. */
968                         ctrl_next = ctrl + 0x4;
969                 } else {
970                         ctrl_next = mlx4_tx_burst_segs(buf, txq, ctrl);
971                         if (!ctrl_next) {
972                                 elt->buf = NULL;
973                                 break;
974                         }
975                 }
976                 /* Hold SQ ring wrap around. */
977                 if ((volatile uint8_t *)ctrl_next >= sq->eob) {
978                         ctrl_next = (volatile struct mlx4_wqe_ctrl_seg *)
979                                 ((volatile uint8_t *)ctrl_next - sq->size);
980                         /* Flip HW valid ownership. */
981                         sq->owner_opcode ^= 1u << MLX4_SQ_OWNER_BIT;
982                 }
983                 /*
984                  * For raw Ethernet, the SOLICIT flag is used to indicate
985                  * that no ICRC should be calculated.
986                  */
987                 if (--txq->elts_comp_cd == 0) {
988                         /* Save the completion burst end address. */
989                         elt_next->eocb = (volatile uint32_t *)ctrl_next;
990                         txq->elts_comp_cd = txq->elts_comp_cd_init;
991                         srcrb.flags = RTE_BE32(MLX4_WQE_CTRL_SOLICIT |
992                                                MLX4_WQE_CTRL_CQ_UPDATE);
993                 } else {
994                         srcrb.flags = RTE_BE32(MLX4_WQE_CTRL_SOLICIT);
995                 }
996                 /* Enable HW checksum offload if requested */
997                 if (txq->csum &&
998                     (buf->ol_flags &
999                      (PKT_TX_IP_CKSUM | PKT_TX_TCP_CKSUM | PKT_TX_UDP_CKSUM))) {
1000                         const uint64_t is_tunneled = (buf->ol_flags &
1001                                                       (PKT_TX_TUNNEL_GRE |
1002                                                        PKT_TX_TUNNEL_VXLAN));
1003
1004                         if (is_tunneled && txq->csum_l2tun) {
1005                                 owner_opcode |= MLX4_WQE_CTRL_IIP_HDR_CSUM |
1006                                                 MLX4_WQE_CTRL_IL4_HDR_CSUM;
1007                                 if (buf->ol_flags & PKT_TX_OUTER_IP_CKSUM)
1008                                         srcrb.flags |=
1009                                             RTE_BE32(MLX4_WQE_CTRL_IP_HDR_CSUM);
1010                         } else {
1011                                 srcrb.flags |=
1012                                         RTE_BE32(MLX4_WQE_CTRL_IP_HDR_CSUM |
1013                                                 MLX4_WQE_CTRL_TCP_UDP_CSUM);
1014                         }
1015                 }
1016                 if (txq->lb) {
1017                         /*
1018                          * Copy destination MAC address to the WQE, this allows
1019                          * loopback in eSwitch, so that VFs and PF can
1020                          * communicate with each other.
1021                          */
1022                         srcrb.flags16[0] = *(rte_pktmbuf_mtod(buf, uint16_t *));
1023                         ctrl->imm = *(rte_pktmbuf_mtod_offset(buf, uint32_t *,
1024                                               sizeof(uint16_t)));
1025                 } else {
1026                         ctrl->imm = 0;
1027                 }
1028                 ctrl->srcrb_flags = srcrb.flags;
1029                 /*
1030                  * Make sure descriptor is fully written before
1031                  * setting ownership bit (because HW can start
1032                  * executing as soon as we do).
1033                  */
1034                 rte_io_wmb();
1035                 ctrl->owner_opcode = rte_cpu_to_be_32(owner_opcode);
1036                 elt->buf = buf;
1037                 bytes_sent += buf->pkt_len;
1038                 ctrl = ctrl_next;
1039                 elt = elt_next;
1040         }
1041         /* Take a shortcut if nothing must be sent. */
1042         if (unlikely(i == 0))
1043                 return 0;
1044         /* Save WQE address of the next Tx burst element. */
1045         elt->wqe = ctrl;
1046         /* Increment send statistics counters. */
1047         txq->stats.opackets += i;
1048         txq->stats.obytes += bytes_sent;
1049         /* Make sure that descriptors are written before doorbell record. */
1050         rte_wmb();
1051         /* Ring QP doorbell. */
1052         rte_write32(txq->msq.doorbell_qpn, MLX4_TX_BFREG(txq));
1053         txq->elts_head += i;
1054         return i;
1055 }
1056
1057 /**
1058  * Translate Rx completion flags to packet type.
1059  *
1060  * @param[in] cqe
1061  *   Pointer to CQE.
1062  *
1063  * @return
1064  *   Packet type for struct rte_mbuf.
1065  */
1066 static inline uint32_t
1067 rxq_cq_to_pkt_type(volatile struct mlx4_cqe *cqe,
1068                    uint32_t l2tun_offload)
1069 {
1070         uint8_t idx = 0;
1071         uint32_t pinfo = rte_be_to_cpu_32(cqe->vlan_my_qpn);
1072         uint32_t status = rte_be_to_cpu_32(cqe->status);
1073
1074         /*
1075          * The index to the array should have:
1076          *  bit[7] - MLX4_CQE_L2_TUNNEL
1077          *  bit[6] - MLX4_CQE_L2_TUNNEL_IPV4
1078          */
1079         if (l2tun_offload && (pinfo & MLX4_CQE_L2_TUNNEL))
1080                 idx |= ((pinfo & MLX4_CQE_L2_TUNNEL) >> 20) |
1081                        ((pinfo & MLX4_CQE_L2_TUNNEL_IPV4) >> 19);
1082         /*
1083          * The index to the array should have:
1084          *  bit[5] - MLX4_CQE_STATUS_UDP
1085          *  bit[4] - MLX4_CQE_STATUS_TCP
1086          *  bit[3] - MLX4_CQE_STATUS_IPV4OPT
1087          *  bit[2] - MLX4_CQE_STATUS_IPV6
1088          *  bit[1] - MLX4_CQE_STATUS_IPF
1089          *  bit[0] - MLX4_CQE_STATUS_IPV4
1090          * giving a total of up to 256 entries.
1091          */
1092         idx |= ((status & MLX4_CQE_STATUS_PTYPE_MASK) >> 22);
1093         if (status & MLX4_CQE_STATUS_IPV6)
1094                 idx |= ((status & MLX4_CQE_STATUS_IPV6F) >> 11);
1095         return mlx4_ptype_table[idx];
1096 }
1097
1098 /**
1099  * Translate Rx completion flags to offload flags.
1100  *
1101  * @param flags
1102  *   Rx completion flags returned by mlx4_cqe_flags().
1103  * @param csum
1104  *   Whether Rx checksums are enabled.
1105  * @param csum_l2tun
1106  *   Whether Rx L2 tunnel checksums are enabled.
1107  *
1108  * @return
1109  *   Offload flags (ol_flags) in mbuf format.
1110  */
1111 static inline uint32_t
1112 rxq_cq_to_ol_flags(uint32_t flags, int csum, int csum_l2tun)
1113 {
1114         uint32_t ol_flags = 0;
1115
1116         if (csum)
1117                 ol_flags |=
1118                         mlx4_transpose(flags,
1119                                        MLX4_CQE_STATUS_IP_HDR_CSUM_OK,
1120                                        PKT_RX_IP_CKSUM_GOOD) |
1121                         mlx4_transpose(flags,
1122                                        MLX4_CQE_STATUS_TCP_UDP_CSUM_OK,
1123                                        PKT_RX_L4_CKSUM_GOOD);
1124         if ((flags & MLX4_CQE_L2_TUNNEL) && csum_l2tun)
1125                 ol_flags |=
1126                         mlx4_transpose(flags,
1127                                        MLX4_CQE_L2_TUNNEL_IPOK,
1128                                        PKT_RX_IP_CKSUM_GOOD) |
1129                         mlx4_transpose(flags,
1130                                        MLX4_CQE_L2_TUNNEL_L4_CSUM,
1131                                        PKT_RX_L4_CKSUM_GOOD);
1132         return ol_flags;
1133 }
1134
1135 /**
1136  * Extract checksum information from CQE flags.
1137  *
1138  * @param cqe
1139  *   Pointer to CQE structure.
1140  * @param csum
1141  *   Whether Rx checksums are enabled.
1142  * @param csum_l2tun
1143  *   Whether Rx L2 tunnel checksums are enabled.
1144  *
1145  * @return
1146  *   CQE checksum information.
1147  */
1148 static inline uint32_t
1149 mlx4_cqe_flags(volatile struct mlx4_cqe *cqe, int csum, int csum_l2tun)
1150 {
1151         uint32_t flags = 0;
1152
1153         /*
1154          * The relevant bits are in different locations on their
1155          * CQE fields therefore we can join them in one 32bit
1156          * variable.
1157          */
1158         if (csum)
1159                 flags = (rte_be_to_cpu_32(cqe->status) &
1160                          MLX4_CQE_STATUS_IPV4_CSUM_OK);
1161         if (csum_l2tun)
1162                 flags |= (rte_be_to_cpu_32(cqe->vlan_my_qpn) &
1163                           (MLX4_CQE_L2_TUNNEL |
1164                            MLX4_CQE_L2_TUNNEL_IPOK |
1165                            MLX4_CQE_L2_TUNNEL_L4_CSUM |
1166                            MLX4_CQE_L2_TUNNEL_IPV4));
1167         return flags;
1168 }
1169
1170 /**
1171  * Poll one CQE from CQ.
1172  *
1173  * @param rxq
1174  *   Pointer to the receive queue structure.
1175  * @param[out] out
1176  *   Just polled CQE.
1177  *
1178  * @return
1179  *   Number of bytes of the CQE, 0 in case there is no completion.
1180  */
1181 static unsigned int
1182 mlx4_cq_poll_one(struct rxq *rxq, volatile struct mlx4_cqe **out)
1183 {
1184         int ret = 0;
1185         volatile struct mlx4_cqe *cqe = NULL;
1186         struct mlx4_cq *cq = &rxq->mcq;
1187
1188         cqe = (volatile struct mlx4_cqe *)mlx4_get_cqe(cq, cq->cons_index);
1189         if (!!(cqe->owner_sr_opcode & MLX4_CQE_OWNER_MASK) ^
1190             !!(cq->cons_index & cq->cqe_cnt))
1191                 goto out;
1192         /*
1193          * Make sure we read CQ entry contents after we've checked the
1194          * ownership bit.
1195          */
1196         rte_rmb();
1197         assert(!(cqe->owner_sr_opcode & MLX4_CQE_IS_SEND_MASK));
1198         assert((cqe->owner_sr_opcode & MLX4_CQE_OPCODE_MASK) !=
1199                MLX4_CQE_OPCODE_ERROR);
1200         ret = rte_be_to_cpu_32(cqe->byte_cnt);
1201         ++cq->cons_index;
1202 out:
1203         *out = cqe;
1204         return ret;
1205 }
1206
1207 /**
1208  * DPDK callback for Rx with scattered packets support.
1209  *
1210  * @param dpdk_rxq
1211  *   Generic pointer to Rx queue structure.
1212  * @param[out] pkts
1213  *   Array to store received packets.
1214  * @param pkts_n
1215  *   Maximum number of packets in array.
1216  *
1217  * @return
1218  *   Number of packets successfully received (<= pkts_n).
1219  */
1220 uint16_t
1221 mlx4_rx_burst(void *dpdk_rxq, struct rte_mbuf **pkts, uint16_t pkts_n)
1222 {
1223         struct rxq *rxq = dpdk_rxq;
1224         const uint32_t wr_cnt = (1 << rxq->elts_n) - 1;
1225         const uint16_t sges_n = rxq->sges_n;
1226         struct rte_mbuf *pkt = NULL;
1227         struct rte_mbuf *seg = NULL;
1228         unsigned int i = 0;
1229         uint32_t rq_ci = rxq->rq_ci << sges_n;
1230         int len = 0;
1231
1232         while (pkts_n) {
1233                 volatile struct mlx4_cqe *cqe;
1234                 uint32_t idx = rq_ci & wr_cnt;
1235                 struct rte_mbuf *rep = (*rxq->elts)[idx];
1236                 volatile struct mlx4_wqe_data_seg *scat = &(*rxq->wqes)[idx];
1237
1238                 /* Update the 'next' pointer of the previous segment. */
1239                 if (pkt)
1240                         seg->next = rep;
1241                 seg = rep;
1242                 rte_prefetch0(seg);
1243                 rte_prefetch0(scat);
1244                 rep = rte_mbuf_raw_alloc(rxq->mp);
1245                 if (unlikely(rep == NULL)) {
1246                         ++rxq->stats.rx_nombuf;
1247                         if (!pkt) {
1248                                 /*
1249                                  * No buffers before we even started,
1250                                  * bail out silently.
1251                                  */
1252                                 break;
1253                         }
1254                         while (pkt != seg) {
1255                                 assert(pkt != (*rxq->elts)[idx]);
1256                                 rep = pkt->next;
1257                                 pkt->next = NULL;
1258                                 pkt->nb_segs = 1;
1259                                 rte_mbuf_raw_free(pkt);
1260                                 pkt = rep;
1261                         }
1262                         break;
1263                 }
1264                 if (!pkt) {
1265                         /* Looking for the new packet. */
1266                         len = mlx4_cq_poll_one(rxq, &cqe);
1267                         if (!len) {
1268                                 rte_mbuf_raw_free(rep);
1269                                 break;
1270                         }
1271                         if (unlikely(len < 0)) {
1272                                 /* Rx error, packet is likely too large. */
1273                                 rte_mbuf_raw_free(rep);
1274                                 ++rxq->stats.idropped;
1275                                 goto skip;
1276                         }
1277                         pkt = seg;
1278                         assert(len >= (rxq->crc_present << 2));
1279                         /* Update packet information. */
1280                         pkt->packet_type =
1281                                 rxq_cq_to_pkt_type(cqe, rxq->l2tun_offload);
1282                         pkt->ol_flags = PKT_RX_RSS_HASH;
1283                         pkt->hash.rss = cqe->immed_rss_invalid;
1284                         if (rxq->crc_present)
1285                                 len -= RTE_ETHER_CRC_LEN;
1286                         pkt->pkt_len = len;
1287                         if (rxq->csum | rxq->csum_l2tun) {
1288                                 uint32_t flags =
1289                                         mlx4_cqe_flags(cqe,
1290                                                        rxq->csum,
1291                                                        rxq->csum_l2tun);
1292
1293                                 pkt->ol_flags =
1294                                         rxq_cq_to_ol_flags(flags,
1295                                                            rxq->csum,
1296                                                            rxq->csum_l2tun);
1297                         }
1298                 }
1299                 rep->nb_segs = 1;
1300                 rep->port = rxq->port_id;
1301                 rep->data_len = seg->data_len;
1302                 rep->data_off = seg->data_off;
1303                 (*rxq->elts)[idx] = rep;
1304                 /*
1305                  * Fill NIC descriptor with the new buffer. The lkey and size
1306                  * of the buffers are already known, only the buffer address
1307                  * changes.
1308                  */
1309                 scat->addr = rte_cpu_to_be_64(rte_pktmbuf_mtod(rep, uintptr_t));
1310                 /* If there's only one MR, no need to replace LKey in WQE. */
1311                 if (unlikely(mlx4_mr_btree_len(&rxq->mr_ctrl.cache_bh) > 1))
1312                         scat->lkey = mlx4_rx_mb2mr(rxq, rep);
1313                 if (len > seg->data_len) {
1314                         len -= seg->data_len;
1315                         ++pkt->nb_segs;
1316                         ++rq_ci;
1317                         continue;
1318                 }
1319                 /* The last segment. */
1320                 seg->data_len = len;
1321                 /* Increment bytes counter. */
1322                 rxq->stats.ibytes += pkt->pkt_len;
1323                 /* Return packet. */
1324                 *(pkts++) = pkt;
1325                 pkt = NULL;
1326                 --pkts_n;
1327                 ++i;
1328 skip:
1329                 /* Align consumer index to the next stride. */
1330                 rq_ci >>= sges_n;
1331                 ++rq_ci;
1332                 rq_ci <<= sges_n;
1333         }
1334         if (unlikely(i == 0 && (rq_ci >> sges_n) == rxq->rq_ci))
1335                 return 0;
1336         /* Update the consumer index. */
1337         rxq->rq_ci = rq_ci >> sges_n;
1338         rte_wmb();
1339         *rxq->rq_db = rte_cpu_to_be_32(rxq->rq_ci);
1340         *rxq->mcq.set_ci_db =
1341                 rte_cpu_to_be_32(rxq->mcq.cons_index & MLX4_CQ_DB_CI_MASK);
1342         /* Increment packets counter. */
1343         rxq->stats.ipackets += i;
1344         return i;
1345 }
1346
1347 /**
1348  * Dummy DPDK callback for Tx.
1349  *
1350  * This function is used to temporarily replace the real callback during
1351  * unsafe control operations on the queue, or in case of error.
1352  *
1353  * @param dpdk_txq
1354  *   Generic pointer to Tx queue structure.
1355  * @param[in] pkts
1356  *   Packets to transmit.
1357  * @param pkts_n
1358  *   Number of packets in array.
1359  *
1360  * @return
1361  *   Number of packets successfully transmitted (<= pkts_n).
1362  */
1363 uint16_t
1364 mlx4_tx_burst_removed(void *dpdk_txq, struct rte_mbuf **pkts, uint16_t pkts_n)
1365 {
1366         (void)dpdk_txq;
1367         (void)pkts;
1368         (void)pkts_n;
1369         rte_mb();
1370         return 0;
1371 }
1372
1373 /**
1374  * Dummy DPDK callback for Rx.
1375  *
1376  * This function is used to temporarily replace the real callback during
1377  * unsafe control operations on the queue, or in case of error.
1378  *
1379  * @param dpdk_rxq
1380  *   Generic pointer to Rx queue structure.
1381  * @param[out] pkts
1382  *   Array to store received packets.
1383  * @param pkts_n
1384  *   Maximum number of packets in array.
1385  *
1386  * @return
1387  *   Number of packets successfully received (<= pkts_n).
1388  */
1389 uint16_t
1390 mlx4_rx_burst_removed(void *dpdk_rxq, struct rte_mbuf **pkts, uint16_t pkts_n)
1391 {
1392         (void)dpdk_rxq;
1393         (void)pkts;
1394         (void)pkts_n;
1395         rte_mb();
1396         return 0;
1397 }