drivers/net/mlx4/mlx4_rxtx.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright 2017 6WIND S.A.
   3  * Copyright 2017 Mellanox
   4  */
   5
   6 /**
   7  * @file
   8  * Data plane functions for mlx4 driver.
   9  */
  10
  11 #include <assert.h>
  12 #include <stdint.h>
  13 #include <string.h>
  14
  15 /* Verbs headers do not support -pedantic. */
  16 #ifdef PEDANTIC
  17 #pragma GCC diagnostic ignored "-Wpedantic"
  18 #endif
  19 #include <infiniband/verbs.h>
  20 #ifdef PEDANTIC
  21 #pragma GCC diagnostic error "-Wpedantic"
  22 #endif
  23
  24 #include <rte_branch_prediction.h>
  25 #include <rte_common.h>
  26 #include <rte_io.h>
  27 #include <rte_mbuf.h>
  28 #include <rte_mempool.h>
  29 #include <rte_prefetch.h>
  30
  31 #include "mlx4.h"
  32 #include "mlx4_prm.h"
  33 #include "mlx4_rxtx.h"
  34 #include "mlx4_utils.h"
  35
  36 /**
  37  * Pointer-value pair structure used in tx_post_send for saving the first
  38  * DWORD (32 byte) of a TXBB.
  39  */
  40 struct pv {
  41         volatile struct mlx4_wqe_data_seg *dseg;
  42         uint32_t val;
  43 };
  44
  45 /** A table to translate Rx completion flags to packet type. */
  46 uint32_t mlx4_ptype_table[0x100] __rte_cache_aligned = {
  47         /*
  48          * The index to the array should have:
  49          *  bit[7] - MLX4_CQE_L2_TUNNEL
  50          *  bit[6] - MLX4_CQE_L2_TUNNEL_IPV4
  51          *  bit[5] - MLX4_CQE_STATUS_UDP
  52          *  bit[4] - MLX4_CQE_STATUS_TCP
  53          *  bit[3] - MLX4_CQE_STATUS_IPV4OPT
  54          *  bit[2] - MLX4_CQE_STATUS_IPV6
  55          *  bit[1] - MLX4_CQE_STATUS_IPV4F
  56          *  bit[0] - MLX4_CQE_STATUS_IPV4
  57          * giving a total of up to 256 entries.
  58          */
  59         [0x00] = RTE_PTYPE_L2_ETHER,
  60         [0x01] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN,
  61         [0x02] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
  62                      RTE_PTYPE_L4_FRAG,
  63         [0x03] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
  64                      RTE_PTYPE_L4_FRAG,
  65         [0x04] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN,
  66         [0x09] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT,
  67         [0x0a] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT |
  68                      RTE_PTYPE_L4_FRAG,
  69         [0x11] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
  70                      RTE_PTYPE_L4_TCP,
  71         [0x12] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
  72                      RTE_PTYPE_L4_TCP,
  73         [0x14] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
  74                      RTE_PTYPE_L4_TCP,
  75         [0x18] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT |
  76                      RTE_PTYPE_L4_TCP,
  77         [0x19] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT |
  78                      RTE_PTYPE_L4_TCP,
  79         [0x1a] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT |
  80                      RTE_PTYPE_L4_TCP,
  81         [0x21] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
  82                      RTE_PTYPE_L4_UDP,
  83         [0x22] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
  84                      RTE_PTYPE_L4_UDP,
  85         [0x24] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
  86                      RTE_PTYPE_L4_UDP,
  87         [0x28] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT |
  88                      RTE_PTYPE_L4_UDP,
  89         [0x29] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT |
  90                      RTE_PTYPE_L4_UDP,
  91         [0x2a] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT |
  92                      RTE_PTYPE_L4_UDP,
  93         /* Tunneled - L3 IPV6 */
  94         [0x80] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN,
  95         [0x81] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
  96                      RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN,
  97         [0x82] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
  98                      RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
  99                      RTE_PTYPE_INNER_L4_FRAG,
 100         [0x83] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
 101                      RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
 102                      RTE_PTYPE_INNER_L4_FRAG,
 103         [0x84] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
 104                      RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN,
 105         [0x88] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
 106                      RTE_PTYPE_INNER_L3_IPV4_EXT,
 107         [0x89] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
 108                      RTE_PTYPE_INNER_L3_IPV4_EXT,
 109         [0x8a] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
 110                      RTE_PTYPE_INNER_L3_IPV4_EXT | RTE_PTYPE_INNER_L4_FRAG,
 111         /* Tunneled - L3 IPV6, TCP */
 112         [0x91] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
 113                      RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
 114                      RTE_PTYPE_INNER_L4_TCP,
 115         [0x92] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
 116                      RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
 117                      RTE_PTYPE_INNER_L4_FRAG |
 118                      RTE_PTYPE_INNER_L4_TCP,
 119         [0x93] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
 120                      RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
 121                      RTE_PTYPE_INNER_L4_FRAG |
 122                      RTE_PTYPE_INNER_L4_TCP,
 123         [0x94] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
 124                      RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
 125                      RTE_PTYPE_INNER_L4_TCP,
 126         [0x98] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
 127                      RTE_PTYPE_INNER_L3_IPV4_EXT |
 128                      RTE_PTYPE_INNER_L4_TCP,
 129         [0x99] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
 130                      RTE_PTYPE_INNER_L3_IPV4_EXT |
 131                      RTE_PTYPE_INNER_L4_TCP,
 132         [0x9a] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
 133                      RTE_PTYPE_INNER_L3_IPV4_EXT | RTE_PTYPE_INNER_L4_FRAG |
 134                      RTE_PTYPE_INNER_L4_TCP,
 135         /* Tunneled - L3 IPV6, UDP */
 136         [0xa1] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
 137                      RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
 138                      RTE_PTYPE_INNER_L4_UDP,
 139         [0xa2] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
 140                      RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
 141                      RTE_PTYPE_INNER_L4_FRAG |
 142                      RTE_PTYPE_INNER_L4_UDP,
 143         [0xa3] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
 144                      RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
 145                      RTE_PTYPE_INNER_L4_FRAG |
 146                      RTE_PTYPE_INNER_L4_UDP,
 147         [0xa4] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
 148                      RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
 149                      RTE_PTYPE_INNER_L4_UDP,
 150         [0xa8] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
 151                      RTE_PTYPE_INNER_L3_IPV4_EXT |
 152                      RTE_PTYPE_INNER_L4_UDP,
 153         [0xa9] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
 154                      RTE_PTYPE_INNER_L3_IPV4_EXT |
 155                      RTE_PTYPE_INNER_L4_UDP,
 156         [0xaa] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
 157                      RTE_PTYPE_INNER_L3_IPV4_EXT | RTE_PTYPE_INNER_L4_FRAG |
 158                      RTE_PTYPE_INNER_L4_UDP,
 159         /* Tunneled - L3 IPV4 */
 160         [0xc0] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN,
 161         [0xc1] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
 162                      RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN,
 163         [0xc2] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
 164                      RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
 165                      RTE_PTYPE_INNER_L4_FRAG,
 166         [0xc3] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
 167                      RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
 168                      RTE_PTYPE_INNER_L4_FRAG,
 169         [0xc4] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
 170                      RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN,
 171         [0xc8] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
 172                      RTE_PTYPE_INNER_L3_IPV4_EXT,
 173         [0xc9] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
 174                      RTE_PTYPE_INNER_L3_IPV4_EXT,
 175         [0xca] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
 176                      RTE_PTYPE_INNER_L3_IPV4_EXT |
 177                      RTE_PTYPE_INNER_L4_FRAG,
 178         /* Tunneled - L3 IPV4, TCP */
 179         [0xd0] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
 180                      RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
 181                      RTE_PTYPE_INNER_L4_TCP,
 182         [0xd1] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
 183                      RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
 184                      RTE_PTYPE_INNER_L4_TCP,
 185         [0xd2] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
 186                      RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
 187                      RTE_PTYPE_INNER_L4_FRAG |
 188                      RTE_PTYPE_INNER_L4_TCP,
 189         [0xd3] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
 190                      RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
 191                      RTE_PTYPE_INNER_L4_FRAG |
 192                      RTE_PTYPE_INNER_L4_TCP,
 193         [0xd4] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
 194                      RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
 195                      RTE_PTYPE_INNER_L4_TCP,
 196         [0xd8] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
 197                      RTE_PTYPE_INNER_L3_IPV4_EXT |
 198                      RTE_PTYPE_INNER_L4_TCP,
 199         [0xd9] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
 200                      RTE_PTYPE_INNER_L3_IPV4_EXT |
 201                      RTE_PTYPE_INNER_L4_TCP,
 202         [0xda] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
 203                      RTE_PTYPE_INNER_L3_IPV4_EXT | RTE_PTYPE_INNER_L4_FRAG |
 204                      RTE_PTYPE_INNER_L4_TCP,
 205         /* Tunneled - L3 IPV4, UDP */
 206         [0xe0] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
 207                      RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
 208                      RTE_PTYPE_INNER_L4_UDP,
 209         [0xe1] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
 210                      RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
 211                      RTE_PTYPE_INNER_L4_UDP,
 212         [0xe2] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
 213                      RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
 214                      RTE_PTYPE_INNER_L4_FRAG |
 215                      RTE_PTYPE_INNER_L4_UDP,
 216         [0xe3] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
 217                      RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
 218                      RTE_PTYPE_INNER_L4_FRAG |
 219                      RTE_PTYPE_INNER_L4_UDP,
 220         [0xe4] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
 221                      RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
 222                      RTE_PTYPE_INNER_L4_UDP,
 223         [0xe8] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
 224                      RTE_PTYPE_INNER_L3_IPV4_EXT | RTE_PTYPE_INNER_L4_UDP,
 225         [0xe9] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
 226                      RTE_PTYPE_INNER_L3_IPV4_EXT | RTE_PTYPE_INNER_L4_UDP,
 227         [0xea] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
 228                      RTE_PTYPE_INNER_L3_IPV4_EXT | RTE_PTYPE_INNER_L4_FRAG |
 229                      RTE_PTYPE_INNER_L4_UDP,
 230 };
 231
 232 /**
 233  * Stamp TXBB burst so it won't be reused by the HW.
 234  *
 235  * Routine is used when freeing WQE used by the chip or when failing
 236  * building an WQ entry has failed leaving partial information on the queue.
 237  *
 238  * @param sq
 239  *   Pointer to the SQ structure.
 240  * @param start
 241  *   Pointer to the first TXBB to stamp.
 242  * @param end
 243  *   Pointer to the followed end TXBB to stamp.
 244  *
 245  * @return
 246  *   Stamping burst size in byte units.
 247  */
 248 static uint32_t
 249 mlx4_txq_stamp_freed_wqe(struct mlx4_sq *sq, volatile uint32_t *start,
 250                          volatile uint32_t *end)
 251 {
 252         uint32_t stamp = sq->stamp;
 253         int32_t size = (intptr_t)end - (intptr_t)start;
 254
 255         assert(start != end);
 256         /* Hold SQ ring wrap around. */
 257         if (size < 0) {
 258                 size = (int32_t)sq->size + size;
 259                 do {
 260                         *start = stamp;
 261                         start += MLX4_SQ_STAMP_DWORDS;
 262                 } while (start != (volatile uint32_t *)sq->eob);
 263                 start = (volatile uint32_t *)sq->buf;
 264                 /* Flip invalid stamping ownership. */
 265                 stamp ^= RTE_BE32(0x1 << MLX4_SQ_OWNER_BIT);
 266                 sq->stamp = stamp;
 267                 if (start == end)
 268                         return size;
 269         }
 270         do {
 271                 *start = stamp;
 272                 start += MLX4_SQ_STAMP_DWORDS;
 273         } while (start != end);
 274         return (uint32_t)size;
 275 }
 276
 277 /**
 278  * Manage Tx completions.
 279  *
 280  * When sending a burst, mlx4_tx_burst() posts several WRs.
 281  * To improve performance, a completion event is only required once every
 282  * MLX4_PMD_TX_PER_COMP_REQ sends. Doing so discards completion information
 283  * for other WRs, but this information would not be used anyway.
 284  *
 285  * @param txq
 286  *   Pointer to Tx queue structure.
 287  * @param elts_m
 288  *   Tx elements number mask.
 289  * @param sq
 290  *   Pointer to the SQ structure.
 291  */
 292 static void
 293 mlx4_txq_complete(struct txq *txq, const unsigned int elts_m,
 294                   struct mlx4_sq *sq)
 295 {
 296         unsigned int elts_tail = txq->elts_tail;
 297         struct mlx4_cq *cq = &txq->mcq;
 298         volatile struct mlx4_cqe *cqe;
 299         uint32_t completed;
 300         uint32_t cons_index = cq->cons_index;
 301         volatile uint32_t *first_txbb;
 302
 303         /*
 304          * Traverse over all CQ entries reported and handle each WQ entry
 305          * reported by them.
 306          */
 307         do {
 308                 cqe = (volatile struct mlx4_cqe *)mlx4_get_cqe(cq, cons_index);
 309                 if (unlikely(!!(cqe->owner_sr_opcode & MLX4_CQE_OWNER_MASK) ^
 310                     !!(cons_index & cq->cqe_cnt)))
 311                         break;
 312 #ifndef NDEBUG
 313                 /*
 314                  * Make sure we read the CQE after we read the ownership bit.
 315                  */
 316                 rte_io_rmb();
 317                 if (unlikely((cqe->owner_sr_opcode & MLX4_CQE_OPCODE_MASK) ==
 318                              MLX4_CQE_OPCODE_ERROR)) {
 319                         volatile struct mlx4_err_cqe *cqe_err =
 320                                 (volatile struct mlx4_err_cqe *)cqe;
 321                         ERROR("%p CQE error - vendor syndrome: 0x%x"
 322                               " syndrome: 0x%x\n",
 323                               (void *)txq, cqe_err->vendor_err,
 324                               cqe_err->syndrome);
 325                         break;
 326                 }
 327 #endif /* NDEBUG */
 328                 cons_index++;
 329         } while (1);
 330         completed = (cons_index - cq->cons_index) * txq->elts_comp_cd_init;
 331         if (unlikely(!completed))
 332                 return;
 333         /* First stamping address is the end of the last one. */
 334         first_txbb = (&(*txq->elts)[elts_tail & elts_m])->eocb;
 335         elts_tail += completed;
 336         /* The new tail element holds the end address. */
 337         sq->remain_size += mlx4_txq_stamp_freed_wqe(sq, first_txbb,
 338                 (&(*txq->elts)[elts_tail & elts_m])->eocb);
 339         /* Update CQ consumer index. */
 340         cq->cons_index = cons_index;
 341         *cq->set_ci_db = rte_cpu_to_be_32(cons_index & MLX4_CQ_DB_CI_MASK);
 342         txq->elts_tail = elts_tail;
 343 }
 344
 345 /**
 346  * Get memory pool (MP) from mbuf. If mbuf is indirect, the pool from which
 347  * the cloned mbuf is allocated is returned instead.
 348  *
 349  * @param buf
 350  *   Pointer to mbuf.
 351  *
 352  * @return
 353  *   Memory pool where data is located for given mbuf.
 354  */
 355 static struct rte_mempool *
 356 mlx4_txq_mb2mp(struct rte_mbuf *buf)
 357 {
 358         if (unlikely(RTE_MBUF_INDIRECT(buf)))
 359                 return rte_mbuf_from_indirect(buf)->pool;
 360         return buf->pool;
 361 }
 362
 363 /**
 364  * Write Tx data segment to the SQ.
 365  *
 366  * @param dseg
 367  *   Pointer to data segment in SQ.
 368  * @param lkey
 369  *   Memory region lkey.
 370  * @param addr
 371  *   Data address.
 372  * @param byte_count
 373  *   Big endian bytes count of the data to send.
 374  */
 375 static inline void
 376 mlx4_fill_tx_data_seg(volatile struct mlx4_wqe_data_seg *dseg,
 377                        uint32_t lkey, uintptr_t addr, rte_be32_t  byte_count)
 378 {
 379         dseg->addr = rte_cpu_to_be_64(addr);
 380         dseg->lkey = rte_cpu_to_be_32(lkey);
 381 #if RTE_CACHE_LINE_SIZE < 64
 382         /*
 383          * Need a barrier here before writing the byte_count
 384          * fields to make sure that all the data is visible
 385          * before the byte_count field is set.
 386          * Otherwise, if the segment begins a new cacheline,
 387          * the HCA prefetcher could grab the 64-byte chunk and
 388          * get a valid (!= 0xffffffff) byte count but stale
 389          * data, and end up sending the wrong data.
 390          */
 391         rte_io_wmb();
 392 #endif /* RTE_CACHE_LINE_SIZE */
 393         dseg->byte_count = byte_count;
 394 }
 395
 396 /**
 397  * Write data segments of multi-segment packet.
 398  *
 399  * @param buf
 400  *   Pointer to the first packet mbuf.
 401  * @param txq
 402  *   Pointer to Tx queue structure.
 403  * @param ctrl
 404  *   Pointer to the WQE control segment.
 405  *
 406  * @return
 407  *   Pointer to the next WQE control segment on success, NULL otherwise.
 408  */
 409 static volatile struct mlx4_wqe_ctrl_seg *
 410 mlx4_tx_burst_segs(struct rte_mbuf *buf, struct txq *txq,
 411                    volatile struct mlx4_wqe_ctrl_seg *ctrl)
 412 {
 413         struct pv *pv = (struct pv *)txq->bounce_buf;
 414         struct mlx4_sq *sq = &txq->msq;
 415         struct rte_mbuf *sbuf = buf;
 416         uint32_t lkey;
 417         int pv_counter = 0;
 418         int nb_segs = buf->nb_segs;
 419         uint32_t wqe_size;
 420         volatile struct mlx4_wqe_data_seg *dseg =
 421                 (volatile struct mlx4_wqe_data_seg *)(ctrl + 1);
 422
 423         ctrl->fence_size = 1 + nb_segs;
 424         wqe_size = RTE_ALIGN((uint32_t)(ctrl->fence_size << MLX4_SEG_SHIFT),
 425                              MLX4_TXBB_SIZE);
 426         /* Validate WQE size and WQE space in the send queue. */
 427         if (sq->remain_size < wqe_size ||
 428             wqe_size > MLX4_MAX_WQE_SIZE)
 429                 return NULL;
 430         /*
 431          * Fill the data segments with buffer information.
 432          * First WQE TXBB head segment is always control segment,
 433          * so jump to tail TXBB data segments code for the first
 434          * WQE data segments filling.
 435          */
 436         goto txbb_tail_segs;
 437 txbb_head_seg:
 438         /* Memory region key (big endian) for this memory pool. */
 439         lkey = mlx4_txq_mp2mr(txq, mlx4_txq_mb2mp(sbuf));
 440         if (unlikely(lkey == (uint32_t)-1)) {
 441                 DEBUG("%p: unable to get MP <-> MR association",
 442                       (void *)txq);
 443                 return NULL;
 444         }
 445         /* Handle WQE wraparound. */
 446         if (dseg >=
 447                 (volatile struct mlx4_wqe_data_seg *)sq->eob)
 448                 dseg = (volatile struct mlx4_wqe_data_seg *)
 449                         sq->buf;
 450         dseg->addr = rte_cpu_to_be_64(rte_pktmbuf_mtod(sbuf, uintptr_t));
 451         dseg->lkey = rte_cpu_to_be_32(lkey);
 452         /*
 453          * This data segment starts at the beginning of a new
 454          * TXBB, so we need to postpone its byte_count writing
 455          * for later.
 456          */
 457         pv[pv_counter].dseg = dseg;
 458         /*
 459          * Zero length segment is treated as inline segment
 460          * with zero data.
 461          */
 462         pv[pv_counter++].val = rte_cpu_to_be_32(sbuf->data_len ?
 463                                                 sbuf->data_len : 0x80000000);
 464         sbuf = sbuf->next;
 465         dseg++;
 466         nb_segs--;
 467 txbb_tail_segs:
 468         /* Jump to default if there are more than two segments remaining. */
 469         switch (nb_segs) {
 470         default:
 471                 lkey = mlx4_txq_mp2mr(txq, mlx4_txq_mb2mp(sbuf));
 472                 if (unlikely(lkey == (uint32_t)-1)) {
 473                         DEBUG("%p: unable to get MP <-> MR association",
 474                               (void *)txq);
 475                         return NULL;
 476                 }
 477                 mlx4_fill_tx_data_seg(dseg, lkey,
 478                                       rte_pktmbuf_mtod(sbuf, uintptr_t),
 479                                       rte_cpu_to_be_32(sbuf->data_len ?
 480                                                        sbuf->data_len :
 481                                                        0x80000000));
 482                 sbuf = sbuf->next;
 483                 dseg++;
 484                 nb_segs--;
 485                 /* fallthrough */
 486         case 2:
 487                 lkey = mlx4_txq_mp2mr(txq, mlx4_txq_mb2mp(sbuf));
 488                 if (unlikely(lkey == (uint32_t)-1)) {
 489                         DEBUG("%p: unable to get MP <-> MR association",
 490                               (void *)txq);
 491                         return NULL;
 492                 }
 493                 mlx4_fill_tx_data_seg(dseg, lkey,
 494                                       rte_pktmbuf_mtod(sbuf, uintptr_t),
 495                                       rte_cpu_to_be_32(sbuf->data_len ?
 496                                                        sbuf->data_len :
 497                                                        0x80000000));
 498                 sbuf = sbuf->next;
 499                 dseg++;
 500                 nb_segs--;
 501                 /* fallthrough */
 502         case 1:
 503                 lkey = mlx4_txq_mp2mr(txq, mlx4_txq_mb2mp(sbuf));
 504                 if (unlikely(lkey == (uint32_t)-1)) {
 505                         DEBUG("%p: unable to get MP <-> MR association",
 506                               (void *)txq);
 507                         return NULL;
 508                 }
 509                 mlx4_fill_tx_data_seg(dseg, lkey,
 510                                       rte_pktmbuf_mtod(sbuf, uintptr_t),
 511                                       rte_cpu_to_be_32(sbuf->data_len ?
 512                                                        sbuf->data_len :
 513                                                        0x80000000));
 514                 nb_segs--;
 515                 if (nb_segs) {
 516                         sbuf = sbuf->next;
 517                         dseg++;
 518                         goto txbb_head_seg;
 519                 }
 520                 /* fallthrough */
 521         case 0:
 522                 break;
 523         }
 524         /* Write the first DWORD of each TXBB save earlier. */
 525         if (pv_counter) {
 526                 /* Need a barrier here before writing the byte_count. */
 527                 rte_io_wmb();
 528                 for (--pv_counter; pv_counter  >= 0; pv_counter--)
 529                         pv[pv_counter].dseg->byte_count = pv[pv_counter].val;
 530         }
 531         sq->remain_size -= wqe_size;
 532         /* Align next WQE address to the next TXBB. */
 533         return (volatile struct mlx4_wqe_ctrl_seg *)
 534                 ((volatile uint8_t *)ctrl + wqe_size);
 535 }
 536
 537 /**
 538  * DPDK callback for Tx.
 539  *
 540  * @param dpdk_txq
 541  *   Generic pointer to Tx queue structure.
 542  * @param[in] pkts
 543  *   Packets to transmit.
 544  * @param pkts_n
 545  *   Number of packets in array.
 546  *
 547  * @return
 548  *   Number of packets successfully transmitted (<= pkts_n).
 549  */
 550 uint16_t
 551 mlx4_tx_burst(void *dpdk_txq, struct rte_mbuf **pkts, uint16_t pkts_n)
 552 {
 553         struct txq *txq = (struct txq *)dpdk_txq;
 554         unsigned int elts_head = txq->elts_head;
 555         const unsigned int elts_n = txq->elts_n;
 556         const unsigned int elts_m = elts_n - 1;
 557         unsigned int bytes_sent = 0;
 558         unsigned int i;
 559         unsigned int max = elts_head - txq->elts_tail;
 560         struct mlx4_sq *sq = &txq->msq;
 561         volatile struct mlx4_wqe_ctrl_seg *ctrl;
 562         struct txq_elt *elt;
 563
 564         assert(txq->elts_comp_cd != 0);
 565         if (likely(max >= txq->elts_comp_cd_init))
 566                 mlx4_txq_complete(txq, elts_m, sq);
 567         max = elts_n - max;
 568         assert(max >= 1);
 569         assert(max <= elts_n);
 570         /* Always leave one free entry in the ring. */
 571         --max;
 572         if (max > pkts_n)
 573                 max = pkts_n;
 574         elt = &(*txq->elts)[elts_head & elts_m];
 575         /* First Tx burst element saves the next WQE control segment. */
 576         ctrl = elt->wqe;
 577         for (i = 0; (i != max); ++i) {
 578                 struct rte_mbuf *buf = pkts[i];
 579                 struct txq_elt *elt_next = &(*txq->elts)[++elts_head & elts_m];
 580                 uint32_t owner_opcode = sq->owner_opcode;
 581                 volatile struct mlx4_wqe_data_seg *dseg =
 582                                 (volatile struct mlx4_wqe_data_seg *)(ctrl + 1);
 583                 volatile struct mlx4_wqe_ctrl_seg *ctrl_next;
 584                 union {
 585                         uint32_t flags;
 586                         uint16_t flags16[2];
 587                 } srcrb;
 588                 uint32_t lkey;
 589
 590                 /* Clean up old buffer. */
 591                 if (likely(elt->buf != NULL)) {
 592                         struct rte_mbuf *tmp = elt->buf;
 593
 594 #ifndef NDEBUG
 595                         /* Poisoning. */
 596                         memset(&elt->buf, 0x66, sizeof(struct rte_mbuf *));
 597 #endif
 598                         /* Faster than rte_pktmbuf_free(). */
 599                         do {
 600                                 struct rte_mbuf *next = tmp->next;
 601
 602                                 rte_pktmbuf_free_seg(tmp);
 603                                 tmp = next;
 604                         } while (tmp != NULL);
 605                 }
 606                 RTE_MBUF_PREFETCH_TO_FREE(elt_next->buf);
 607                 if (buf->nb_segs == 1) {
 608                         /* Validate WQE space in the send queue. */
 609                         if (sq->remain_size < MLX4_TXBB_SIZE) {
 610                                 elt->buf = NULL;
 611                                 break;
 612                         }
 613                         lkey = mlx4_txq_mp2mr(txq, mlx4_txq_mb2mp(buf));
 614                         if (unlikely(lkey == (uint32_t)-1)) {
 615                                 /* MR does not exist. */
 616                                 DEBUG("%p: unable to get MP <-> MR association",
 617                                       (void *)txq);
 618                                 elt->buf = NULL;
 619                                 break;
 620                         }
 621                         mlx4_fill_tx_data_seg(dseg++, lkey,
 622                                               rte_pktmbuf_mtod(buf, uintptr_t),
 623                                               rte_cpu_to_be_32(buf->data_len));
 624                         /* Set WQE size in 16-byte units. */
 625                         ctrl->fence_size = 0x2;
 626                         sq->remain_size -= MLX4_TXBB_SIZE;
 627                         /* Align next WQE address to the next TXBB. */
 628                         ctrl_next = ctrl + 0x4;
 629                 } else {
 630                         ctrl_next = mlx4_tx_burst_segs(buf, txq, ctrl);
 631                         if (!ctrl_next) {
 632                                 elt->buf = NULL;
 633                                 break;
 634                         }
 635                 }
 636                 /* Hold SQ ring wrap around. */
 637                 if ((volatile uint8_t *)ctrl_next >= sq->eob) {
 638                         ctrl_next = (volatile struct mlx4_wqe_ctrl_seg *)
 639                                 ((volatile uint8_t *)ctrl_next - sq->size);
 640                         /* Flip HW valid ownership. */
 641                         sq->owner_opcode ^= 0x1 << MLX4_SQ_OWNER_BIT;
 642                 }
 643                 /*
 644                  * For raw Ethernet, the SOLICIT flag is used to indicate
 645                  * that no ICRC should be calculated.
 646                  */
 647                 if (--txq->elts_comp_cd == 0) {
 648                         /* Save the completion burst end address. */
 649                         elt_next->eocb = (volatile uint32_t *)ctrl_next;
 650                         txq->elts_comp_cd = txq->elts_comp_cd_init;
 651                         srcrb.flags = RTE_BE32(MLX4_WQE_CTRL_SOLICIT |
 652                                                MLX4_WQE_CTRL_CQ_UPDATE);
 653                 } else {
 654                         srcrb.flags = RTE_BE32(MLX4_WQE_CTRL_SOLICIT);
 655                 }
 656                 /* Enable HW checksum offload if requested */
 657                 if (txq->csum &&
 658                     (buf->ol_flags &
 659                      (PKT_TX_IP_CKSUM | PKT_TX_TCP_CKSUM | PKT_TX_UDP_CKSUM))) {
 660                         const uint64_t is_tunneled = (buf->ol_flags &
 661                                                       (PKT_TX_TUNNEL_GRE |
 662                                                        PKT_TX_TUNNEL_VXLAN));
 663
 664                         if (is_tunneled && txq->csum_l2tun) {
 665                                 owner_opcode |= MLX4_WQE_CTRL_IIP_HDR_CSUM |
 666                                                 MLX4_WQE_CTRL_IL4_HDR_CSUM;
 667                                 if (buf->ol_flags & PKT_TX_OUTER_IP_CKSUM)
 668                                         srcrb.flags |=
 669                                             RTE_BE32(MLX4_WQE_CTRL_IP_HDR_CSUM);
 670                         } else {
 671                                 srcrb.flags |=
 672                                         RTE_BE32(MLX4_WQE_CTRL_IP_HDR_CSUM |
 673                                                 MLX4_WQE_CTRL_TCP_UDP_CSUM);
 674                         }
 675                 }
 676                 if (txq->lb) {
 677                         /*
 678                          * Copy destination MAC address to the WQE, this allows
 679                          * loopback in eSwitch, so that VFs and PF can
 680                          * communicate with each other.
 681                          */
 682                         srcrb.flags16[0] = *(rte_pktmbuf_mtod(buf, uint16_t *));
 683                         ctrl->imm = *(rte_pktmbuf_mtod_offset(buf, uint32_t *,
 684                                               sizeof(uint16_t)));
 685                 } else {
 686                         ctrl->imm = 0;
 687                 }
 688                 ctrl->srcrb_flags = srcrb.flags;
 689                 /*
 690                  * Make sure descriptor is fully written before
 691                  * setting ownership bit (because HW can start
 692                  * executing as soon as we do).
 693                  */
 694                 rte_io_wmb();
 695                 ctrl->owner_opcode = rte_cpu_to_be_32(owner_opcode);
 696                 elt->buf = buf;
 697                 bytes_sent += buf->pkt_len;
 698                 ctrl = ctrl_next;
 699                 elt = elt_next;
 700         }
 701         /* Take a shortcut if nothing must be sent. */
 702         if (unlikely(i == 0))
 703                 return 0;
 704         /* Save WQE address of the next Tx burst element. */
 705         elt->wqe = ctrl;
 706         /* Increment send statistics counters. */
 707         txq->stats.opackets += i;
 708         txq->stats.obytes += bytes_sent;
 709         /* Make sure that descriptors are written before doorbell record. */
 710         rte_wmb();
 711         /* Ring QP doorbell. */
 712         rte_write32(txq->msq.doorbell_qpn, txq->msq.db);
 713         txq->elts_head += i;
 714         return i;
 715 }
 716
 717 /**
 718  * Translate Rx completion flags to packet type.
 719  *
 720  * @param[in] cqe
 721  *   Pointer to CQE.
 722  *
 723  * @return
 724  *   Packet type for struct rte_mbuf.
 725  */
 726 static inline uint32_t
 727 rxq_cq_to_pkt_type(volatile struct mlx4_cqe *cqe,
 728                    uint32_t l2tun_offload)
 729 {
 730         uint8_t idx = 0;
 731         uint32_t pinfo = rte_be_to_cpu_32(cqe->vlan_my_qpn);
 732         uint32_t status = rte_be_to_cpu_32(cqe->status);
 733
 734         /*
 735          * The index to the array should have:
 736          *  bit[7] - MLX4_CQE_L2_TUNNEL
 737          *  bit[6] - MLX4_CQE_L2_TUNNEL_IPV4
 738          */
 739         if (l2tun_offload && (pinfo & MLX4_CQE_L2_TUNNEL))
 740                 idx |= ((pinfo & MLX4_CQE_L2_TUNNEL) >> 20) |
 741                        ((pinfo & MLX4_CQE_L2_TUNNEL_IPV4) >> 19);
 742         /*
 743          * The index to the array should have:
 744          *  bit[5] - MLX4_CQE_STATUS_UDP
 745          *  bit[4] - MLX4_CQE_STATUS_TCP
 746          *  bit[3] - MLX4_CQE_STATUS_IPV4OPT
 747          *  bit[2] - MLX4_CQE_STATUS_IPV6
 748          *  bit[1] - MLX4_CQE_STATUS_IPV4F
 749          *  bit[0] - MLX4_CQE_STATUS_IPV4
 750          * giving a total of up to 256 entries.
 751          */
 752         idx |= ((status & MLX4_CQE_STATUS_PTYPE_MASK) >> 22);
 753         return mlx4_ptype_table[idx];
 754 }
 755
 756 /**
 757  * Translate Rx completion flags to offload flags.
 758  *
 759  * @param flags
 760  *   Rx completion flags returned by mlx4_cqe_flags().
 761  * @param csum
 762  *   Whether Rx checksums are enabled.
 763  * @param csum_l2tun
 764  *   Whether Rx L2 tunnel checksums are enabled.
 765  *
 766  * @return
 767  *   Offload flags (ol_flags) in mbuf format.
 768  */
 769 static inline uint32_t
 770 rxq_cq_to_ol_flags(uint32_t flags, int csum, int csum_l2tun)
 771 {
 772         uint32_t ol_flags = 0;
 773
 774         if (csum)
 775                 ol_flags |=
 776                         mlx4_transpose(flags,
 777                                        MLX4_CQE_STATUS_IP_HDR_CSUM_OK,
 778                                        PKT_RX_IP_CKSUM_GOOD) |
 779                         mlx4_transpose(flags,
 780                                        MLX4_CQE_STATUS_TCP_UDP_CSUM_OK,
 781                                        PKT_RX_L4_CKSUM_GOOD);
 782         if ((flags & MLX4_CQE_L2_TUNNEL) && csum_l2tun)
 783                 ol_flags |=
 784                         mlx4_transpose(flags,
 785                                        MLX4_CQE_L2_TUNNEL_IPOK,
 786                                        PKT_RX_IP_CKSUM_GOOD) |
 787                         mlx4_transpose(flags,
 788                                        MLX4_CQE_L2_TUNNEL_L4_CSUM,
 789                                        PKT_RX_L4_CKSUM_GOOD);
 790         return ol_flags;
 791 }
 792
 793 /**
 794  * Extract checksum information from CQE flags.
 795  *
 796  * @param cqe
 797  *   Pointer to CQE structure.
 798  * @param csum
 799  *   Whether Rx checksums are enabled.
 800  * @param csum_l2tun
 801  *   Whether Rx L2 tunnel checksums are enabled.
 802  *
 803  * @return
 804  *   CQE checksum information.
 805  */
 806 static inline uint32_t
 807 mlx4_cqe_flags(volatile struct mlx4_cqe *cqe, int csum, int csum_l2tun)
 808 {
 809         uint32_t flags = 0;
 810
 811         /*
 812          * The relevant bits are in different locations on their
 813          * CQE fields therefore we can join them in one 32bit
 814          * variable.
 815          */
 816         if (csum)
 817                 flags = (rte_be_to_cpu_32(cqe->status) &
 818                          MLX4_CQE_STATUS_IPV4_CSUM_OK);
 819         if (csum_l2tun)
 820                 flags |= (rte_be_to_cpu_32(cqe->vlan_my_qpn) &
 821                           (MLX4_CQE_L2_TUNNEL |
 822                            MLX4_CQE_L2_TUNNEL_IPOK |
 823                            MLX4_CQE_L2_TUNNEL_L4_CSUM |
 824                            MLX4_CQE_L2_TUNNEL_IPV4));
 825         return flags;
 826 }
 827
 828 /**
 829  * Poll one CQE from CQ.
 830  *
 831  * @param rxq
 832  *   Pointer to the receive queue structure.
 833  * @param[out] out
 834  *   Just polled CQE.
 835  *
 836  * @return
 837  *   Number of bytes of the CQE, 0 in case there is no completion.
 838  */
 839 static unsigned int
 840 mlx4_cq_poll_one(struct rxq *rxq, volatile struct mlx4_cqe **out)
 841 {
 842         int ret = 0;
 843         volatile struct mlx4_cqe *cqe = NULL;
 844         struct mlx4_cq *cq = &rxq->mcq;
 845
 846         cqe = (volatile struct mlx4_cqe *)mlx4_get_cqe(cq, cq->cons_index);
 847         if (!!(cqe->owner_sr_opcode & MLX4_CQE_OWNER_MASK) ^
 848             !!(cq->cons_index & cq->cqe_cnt))
 849                 goto out;
 850         /*
 851          * Make sure we read CQ entry contents after we've checked the
 852          * ownership bit.
 853          */
 854         rte_rmb();
 855         assert(!(cqe->owner_sr_opcode & MLX4_CQE_IS_SEND_MASK));
 856         assert((cqe->owner_sr_opcode & MLX4_CQE_OPCODE_MASK) !=
 857                MLX4_CQE_OPCODE_ERROR);
 858         ret = rte_be_to_cpu_32(cqe->byte_cnt);
 859         ++cq->cons_index;
 860 out:
 861         *out = cqe;
 862         return ret;
 863 }
 864
 865 /**
 866  * DPDK callback for Rx with scattered packets support.
 867  *
 868  * @param dpdk_rxq
 869  *   Generic pointer to Rx queue structure.
 870  * @param[out] pkts
 871  *   Array to store received packets.
 872  * @param pkts_n
 873  *   Maximum number of packets in array.
 874  *
 875  * @return
 876  *   Number of packets successfully received (<= pkts_n).
 877  */
 878 uint16_t
 879 mlx4_rx_burst(void *dpdk_rxq, struct rte_mbuf **pkts, uint16_t pkts_n)
 880 {
 881         struct rxq *rxq = dpdk_rxq;
 882         const uint32_t wr_cnt = (1 << rxq->elts_n) - 1;
 883         const uint16_t sges_n = rxq->sges_n;
 884         struct rte_mbuf *pkt = NULL;
 885         struct rte_mbuf *seg = NULL;
 886         unsigned int i = 0;
 887         uint32_t rq_ci = rxq->rq_ci << sges_n;
 888         int len = 0;
 889
 890         while (pkts_n) {
 891                 volatile struct mlx4_cqe *cqe;
 892                 uint32_t idx = rq_ci & wr_cnt;
 893                 struct rte_mbuf *rep = (*rxq->elts)[idx];
 894                 volatile struct mlx4_wqe_data_seg *scat = &(*rxq->wqes)[idx];
 895
 896                 /* Update the 'next' pointer of the previous segment. */
 897                 if (pkt)
 898                         seg->next = rep;
 899                 seg = rep;
 900                 rte_prefetch0(seg);
 901                 rte_prefetch0(scat);
 902                 rep = rte_mbuf_raw_alloc(rxq->mp);
 903                 if (unlikely(rep == NULL)) {
 904                         ++rxq->stats.rx_nombuf;
 905                         if (!pkt) {
 906                                 /*
 907                                  * No buffers before we even started,
 908                                  * bail out silently.
 909                                  */
 910                                 break;
 911                         }
 912                         while (pkt != seg) {
 913                                 assert(pkt != (*rxq->elts)[idx]);
 914                                 rep = pkt->next;
 915                                 pkt->next = NULL;
 916                                 pkt->nb_segs = 1;
 917                                 rte_mbuf_raw_free(pkt);
 918                                 pkt = rep;
 919                         }
 920                         break;
 921                 }
 922                 if (!pkt) {
 923                         /* Looking for the new packet. */
 924                         len = mlx4_cq_poll_one(rxq, &cqe);
 925                         if (!len) {
 926                                 rte_mbuf_raw_free(rep);
 927                                 break;
 928                         }
 929                         if (unlikely(len < 0)) {
 930                                 /* Rx error, packet is likely too large. */
 931                                 rte_mbuf_raw_free(rep);
 932                                 ++rxq->stats.idropped;
 933                                 goto skip;
 934                         }
 935                         pkt = seg;
 936                         /* Update packet information. */
 937                         pkt->packet_type =
 938                                 rxq_cq_to_pkt_type(cqe, rxq->l2tun_offload);
 939                         pkt->ol_flags = PKT_RX_RSS_HASH;
 940                         pkt->hash.rss = cqe->immed_rss_invalid;
 941                         pkt->pkt_len = len;
 942                         if (rxq->csum | rxq->csum_l2tun) {
 943                                 uint32_t flags =
 944                                         mlx4_cqe_flags(cqe,
 945                                                        rxq->csum,
 946                                                        rxq->csum_l2tun);
 947
 948                                 pkt->ol_flags =
 949                                         rxq_cq_to_ol_flags(flags,
 950                                                            rxq->csum,
 951                                                            rxq->csum_l2tun);
 952                         }
 953                 }
 954                 rep->nb_segs = 1;
 955                 rep->port = rxq->port_id;
 956                 rep->data_len = seg->data_len;
 957                 rep->data_off = seg->data_off;
 958                 (*rxq->elts)[idx] = rep;
 959                 /*
 960                  * Fill NIC descriptor with the new buffer. The lkey and size
 961                  * of the buffers are already known, only the buffer address
 962                  * changes.
 963                  */
 964                 scat->addr = rte_cpu_to_be_64(rte_pktmbuf_mtod(rep, uintptr_t));
 965                 if (len > seg->data_len) {
 966                         len -= seg->data_len;
 967                         ++pkt->nb_segs;
 968                         ++rq_ci;
 969                         continue;
 970                 }
 971                 /* The last segment. */
 972                 seg->data_len = len;
 973                 /* Increment bytes counter. */
 974                 rxq->stats.ibytes += pkt->pkt_len;
 975                 /* Return packet. */
 976                 *(pkts++) = pkt;
 977                 pkt = NULL;
 978                 --pkts_n;
 979                 ++i;
 980 skip:
 981                 /* Align consumer index to the next stride. */
 982                 rq_ci >>= sges_n;
 983                 ++rq_ci;
 984                 rq_ci <<= sges_n;
 985         }
 986         if (unlikely(i == 0 && (rq_ci >> sges_n) == rxq->rq_ci))
 987                 return 0;
 988         /* Update the consumer index. */
 989         rxq->rq_ci = rq_ci >> sges_n;
 990         rte_wmb();
 991         *rxq->rq_db = rte_cpu_to_be_32(rxq->rq_ci);
 992         *rxq->mcq.set_ci_db =
 993                 rte_cpu_to_be_32(rxq->mcq.cons_index & MLX4_CQ_DB_CI_MASK);
 994         /* Increment packets counter. */
 995         rxq->stats.ipackets += i;
 996         return i;
 997 }
 998
 999 /**
1000  * Dummy DPDK callback for Tx.
1001  *
1002  * This function is used to temporarily replace the real callback during
1003  * unsafe control operations on the queue, or in case of error.
1004  *
1005  * @param dpdk_txq
1006  *   Generic pointer to Tx queue structure.
1007  * @param[in] pkts
1008  *   Packets to transmit.
1009  * @param pkts_n
1010  *   Number of packets in array.
1011  *
1012  * @return
1013  *   Number of packets successfully transmitted (<= pkts_n).
1014  */
1015 uint16_t
1016 mlx4_tx_burst_removed(void *dpdk_txq, struct rte_mbuf **pkts, uint16_t pkts_n)
1017 {
1018         (void)dpdk_txq;
1019         (void)pkts;
1020         (void)pkts_n;
1021         return 0;
1022 }
1023
1024 /**
1025  * Dummy DPDK callback for Rx.
1026  *
1027  * This function is used to temporarily replace the real callback during
1028  * unsafe control operations on the queue, or in case of error.
1029  *
1030  * @param dpdk_rxq
1031  *   Generic pointer to Rx queue structure.
1032  * @param[out] pkts
1033  *   Array to store received packets.
1034  * @param pkts_n
1035  *   Maximum number of packets in array.
1036  *
1037  * @return
1038  *   Number of packets successfully received (<= pkts_n).
1039  */
1040 uint16_t
1041 mlx4_rx_burst_removed(void *dpdk_rxq, struct rte_mbuf **pkts, uint16_t pkts_n)
1042 {
1043         (void)dpdk_rxq;
1044         (void)pkts;
1045         (void)pkts_n;
1046         return 0;
1047 }