drivers/net/vmxnet3/vmxnet3_rxtx.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(c) 2010-2015 Intel Corporation
   3  */
   4
   5 #include <sys/queue.h>
   6
   7 #include <stdio.h>
   8 #include <stdlib.h>
   9 #include <string.h>
  10 #include <errno.h>
  11 #include <stdint.h>
  12 #include <stdarg.h>
  13 #include <unistd.h>
  14 #include <inttypes.h>
  15
  16 #include <rte_byteorder.h>
  17 #include <rte_common.h>
  18 #include <rte_cycles.h>
  19 #include <rte_log.h>
  20 #include <rte_debug.h>
  21 #include <rte_interrupts.h>
  22 #include <rte_pci.h>
  23 #include <rte_memory.h>
  24 #include <rte_memzone.h>
  25 #include <rte_launch.h>
  26 #include <rte_eal.h>
  27 #include <rte_per_lcore.h>
  28 #include <rte_lcore.h>
  29 #include <rte_atomic.h>
  30 #include <rte_branch_prediction.h>
  31 #include <rte_mempool.h>
  32 #include <rte_malloc.h>
  33 #include <rte_mbuf.h>
  34 #include <rte_ether.h>
  35 #include <ethdev_driver.h>
  36 #include <rte_prefetch.h>
  37 #include <rte_ip.h>
  38 #include <rte_udp.h>
  39 #include <rte_tcp.h>
  40 #include <rte_sctp.h>
  41 #include <rte_string_fns.h>
  42 #include <rte_errno.h>
  43 #include <rte_net.h>
  44
  45 #include "base/vmxnet3_defs.h"
  46 #include "vmxnet3_ring.h"
  47
  48 #include "vmxnet3_logs.h"
  49 #include "vmxnet3_ethdev.h"
  50
  51 #define VMXNET3_TX_OFFLOAD_MASK (RTE_MBUF_F_TX_VLAN | \
  52                 RTE_MBUF_F_TX_IPV6 |     \
  53                 RTE_MBUF_F_TX_IPV4 |     \
  54                 RTE_MBUF_F_TX_L4_MASK |  \
  55                 RTE_MBUF_F_TX_TCP_SEG)
  56
  57 #define VMXNET3_TX_OFFLOAD_NOTSUP_MASK  \
  58         (RTE_MBUF_F_TX_OFFLOAD_MASK ^ VMXNET3_TX_OFFLOAD_MASK)
  59
  60 static const uint32_t rxprod_reg[2] = {VMXNET3_REG_RXPROD, VMXNET3_REG_RXPROD2};
  61
  62 static int vmxnet3_post_rx_bufs(vmxnet3_rx_queue_t*, uint8_t);
  63 static void vmxnet3_tq_tx_complete(vmxnet3_tx_queue_t *);
  64 #ifdef RTE_LIBRTE_VMXNET3_DEBUG_DRIVER_NOT_USED
  65 static void vmxnet3_rxq_dump(struct vmxnet3_rx_queue *);
  66 static void vmxnet3_txq_dump(struct vmxnet3_tx_queue *);
  67 #endif
  68
  69 #ifdef RTE_LIBRTE_VMXNET3_DEBUG_DRIVER_NOT_USED
  70 static void
  71 vmxnet3_rxq_dump(struct vmxnet3_rx_queue *rxq)
  72 {
  73         uint32_t avail = 0;
  74
  75         if (rxq == NULL)
  76                 return;
  77
  78         PMD_RX_LOG(DEBUG,
  79                    "RXQ: cmd0 base : %p cmd1 base : %p comp ring base : %p.",
  80                    rxq->cmd_ring[0].base, rxq->cmd_ring[1].base, rxq->comp_ring.base);
  81         PMD_RX_LOG(DEBUG,
  82                    "RXQ: cmd0 basePA : 0x%lx cmd1 basePA : 0x%lx comp ring basePA : 0x%lx.",
  83                    (unsigned long)rxq->cmd_ring[0].basePA,
  84                    (unsigned long)rxq->cmd_ring[1].basePA,
  85                    (unsigned long)rxq->comp_ring.basePA);
  86
  87         avail = vmxnet3_cmd_ring_desc_avail(&rxq->cmd_ring[0]);
  88         PMD_RX_LOG(DEBUG,
  89                    "RXQ:cmd0: size=%u; free=%u; next2proc=%u; queued=%u",
  90                    (uint32_t)rxq->cmd_ring[0].size, avail,
  91                    rxq->comp_ring.next2proc,
  92                    rxq->cmd_ring[0].size - avail);
  93
  94         avail = vmxnet3_cmd_ring_desc_avail(&rxq->cmd_ring[1]);
  95         PMD_RX_LOG(DEBUG, "RXQ:cmd1 size=%u; free=%u; next2proc=%u; queued=%u",
  96                    (uint32_t)rxq->cmd_ring[1].size, avail, rxq->comp_ring.next2proc,
  97                    rxq->cmd_ring[1].size - avail);
  98
  99 }
 100
 101 static void
 102 vmxnet3_txq_dump(struct vmxnet3_tx_queue *txq)
 103 {
 104         uint32_t avail = 0;
 105
 106         if (txq == NULL)
 107                 return;
 108
 109         PMD_TX_LOG(DEBUG, "TXQ: cmd base : %p comp ring base : %p data ring base : %p.",
 110                    txq->cmd_ring.base, txq->comp_ring.base, txq->data_ring.base);
 111         PMD_TX_LOG(DEBUG, "TXQ: cmd basePA : 0x%lx comp ring basePA : 0x%lx data ring basePA : 0x%lx.",
 112                    (unsigned long)txq->cmd_ring.basePA,
 113                    (unsigned long)txq->comp_ring.basePA,
 114                    (unsigned long)txq->data_ring.basePA);
 115
 116         avail = vmxnet3_cmd_ring_desc_avail(&txq->cmd_ring);
 117         PMD_TX_LOG(DEBUG, "TXQ: size=%u; free=%u; next2proc=%u; queued=%u",
 118                    (uint32_t)txq->cmd_ring.size, avail,
 119                    txq->comp_ring.next2proc, txq->cmd_ring.size - avail);
 120 }
 121 #endif
 122
 123 static void
 124 vmxnet3_tx_cmd_ring_release_mbufs(vmxnet3_cmd_ring_t *ring)
 125 {
 126         while (ring->next2comp != ring->next2fill) {
 127                 /* No need to worry about desc ownership, device is quiesced by now. */
 128                 vmxnet3_buf_info_t *buf_info = ring->buf_info + ring->next2comp;
 129
 130                 if (buf_info->m) {
 131                         rte_pktmbuf_free(buf_info->m);
 132                         buf_info->m = NULL;
 133                         buf_info->bufPA = 0;
 134                         buf_info->len = 0;
 135                 }
 136                 vmxnet3_cmd_ring_adv_next2comp(ring);
 137         }
 138 }
 139
 140 static void
 141 vmxnet3_rx_cmd_ring_release_mbufs(vmxnet3_cmd_ring_t *ring)
 142 {
 143         uint32_t i;
 144
 145         for (i = 0; i < ring->size; i++) {
 146                 /* No need to worry about desc ownership, device is quiesced by now. */
 147                 vmxnet3_buf_info_t *buf_info = &ring->buf_info[i];
 148
 149                 if (buf_info->m) {
 150                         rte_pktmbuf_free_seg(buf_info->m);
 151                         buf_info->m = NULL;
 152                         buf_info->bufPA = 0;
 153                         buf_info->len = 0;
 154                 }
 155                 vmxnet3_cmd_ring_adv_next2comp(ring);
 156         }
 157 }
 158
 159 static void
 160 vmxnet3_cmd_ring_release(vmxnet3_cmd_ring_t *ring)
 161 {
 162         rte_free(ring->buf_info);
 163         ring->buf_info = NULL;
 164 }
 165
 166 void
 167 vmxnet3_dev_tx_queue_release(struct rte_eth_dev *dev, uint16_t qid)
 168 {
 169         vmxnet3_tx_queue_t *tq = dev->data->tx_queues[qid];
 170
 171         if (tq != NULL) {
 172                 /* Release mbufs */
 173                 vmxnet3_tx_cmd_ring_release_mbufs(&tq->cmd_ring);
 174                 /* Release the cmd_ring */
 175                 vmxnet3_cmd_ring_release(&tq->cmd_ring);
 176                 /* Release the memzone */
 177                 rte_memzone_free(tq->mz);
 178                 /* Release the queue */
 179                 rte_free(tq);
 180         }
 181 }
 182
 183 void
 184 vmxnet3_dev_rx_queue_release(struct rte_eth_dev *dev, uint16_t qid)
 185 {
 186         int i;
 187         vmxnet3_rx_queue_t *rq = dev->data->rx_queues[qid];
 188
 189         if (rq != NULL) {
 190                 /* Release mbufs */
 191                 for (i = 0; i < VMXNET3_RX_CMDRING_SIZE; i++)
 192                         vmxnet3_rx_cmd_ring_release_mbufs(&rq->cmd_ring[i]);
 193
 194                 /* Release both the cmd_rings */
 195                 for (i = 0; i < VMXNET3_RX_CMDRING_SIZE; i++)
 196                         vmxnet3_cmd_ring_release(&rq->cmd_ring[i]);
 197
 198                 /* Release the memzone */
 199                 rte_memzone_free(rq->mz);
 200
 201                 /* Release the queue */
 202                 rte_free(rq);
 203         }
 204 }
 205
 206 static void
 207 vmxnet3_dev_tx_queue_reset(void *txq)
 208 {
 209         vmxnet3_tx_queue_t *tq = txq;
 210         struct vmxnet3_cmd_ring *ring = &tq->cmd_ring;
 211         struct vmxnet3_comp_ring *comp_ring = &tq->comp_ring;
 212         struct vmxnet3_data_ring *data_ring = &tq->data_ring;
 213         int size;
 214
 215         if (tq != NULL) {
 216                 /* Release the cmd_ring mbufs */
 217                 vmxnet3_tx_cmd_ring_release_mbufs(&tq->cmd_ring);
 218         }
 219
 220         /* Tx vmxnet rings structure initialization*/
 221         ring->next2fill = 0;
 222         ring->next2comp = 0;
 223         ring->gen = VMXNET3_INIT_GEN;
 224         comp_ring->next2proc = 0;
 225         comp_ring->gen = VMXNET3_INIT_GEN;
 226
 227         size = sizeof(struct Vmxnet3_TxDesc) * ring->size;
 228         size += sizeof(struct Vmxnet3_TxCompDesc) * comp_ring->size;
 229         size += tq->txdata_desc_size * data_ring->size;
 230
 231         memset(ring->base, 0, size);
 232 }
 233
 234 static void
 235 vmxnet3_dev_rx_queue_reset(void *rxq)
 236 {
 237         int i;
 238         vmxnet3_rx_queue_t *rq = rxq;
 239         struct vmxnet3_hw *hw = rq->hw;
 240         struct vmxnet3_cmd_ring *ring0, *ring1;
 241         struct vmxnet3_comp_ring *comp_ring;
 242         struct vmxnet3_rx_data_ring *data_ring = &rq->data_ring;
 243         int size;
 244
 245         /* Release both the cmd_rings mbufs */
 246         for (i = 0; i < VMXNET3_RX_CMDRING_SIZE; i++)
 247                 vmxnet3_rx_cmd_ring_release_mbufs(&rq->cmd_ring[i]);
 248
 249         ring0 = &rq->cmd_ring[0];
 250         ring1 = &rq->cmd_ring[1];
 251         comp_ring = &rq->comp_ring;
 252
 253         /* Rx vmxnet rings structure initialization */
 254         ring0->next2fill = 0;
 255         ring1->next2fill = 0;
 256         ring0->next2comp = 0;
 257         ring1->next2comp = 0;
 258         ring0->gen = VMXNET3_INIT_GEN;
 259         ring1->gen = VMXNET3_INIT_GEN;
 260         comp_ring->next2proc = 0;
 261         comp_ring->gen = VMXNET3_INIT_GEN;
 262
 263         size = sizeof(struct Vmxnet3_RxDesc) * (ring0->size + ring1->size);
 264         size += sizeof(struct Vmxnet3_RxCompDesc) * comp_ring->size;
 265         if (VMXNET3_VERSION_GE_3(hw) && rq->data_desc_size)
 266                 size += rq->data_desc_size * data_ring->size;
 267
 268         memset(ring0->base, 0, size);
 269 }
 270
 271 void
 272 vmxnet3_dev_clear_queues(struct rte_eth_dev *dev)
 273 {
 274         unsigned i;
 275
 276         PMD_INIT_FUNC_TRACE();
 277
 278         for (i = 0; i < dev->data->nb_tx_queues; i++) {
 279                 struct vmxnet3_tx_queue *txq = dev->data->tx_queues[i];
 280
 281                 if (txq != NULL) {
 282                         txq->stopped = TRUE;
 283                         vmxnet3_dev_tx_queue_reset(txq);
 284                 }
 285         }
 286
 287         for (i = 0; i < dev->data->nb_rx_queues; i++) {
 288                 struct vmxnet3_rx_queue *rxq = dev->data->rx_queues[i];
 289
 290                 if (rxq != NULL) {
 291                         rxq->stopped = TRUE;
 292                         vmxnet3_dev_rx_queue_reset(rxq);
 293                 }
 294         }
 295 }
 296
 297 static int
 298 vmxnet3_unmap_pkt(uint16_t eop_idx, vmxnet3_tx_queue_t *txq)
 299 {
 300         int completed = 0;
 301         struct rte_mbuf *mbuf;
 302
 303         /* Release cmd_ring descriptor and free mbuf */
 304         RTE_ASSERT(txq->cmd_ring.base[eop_idx].txd.eop == 1);
 305
 306         mbuf = txq->cmd_ring.buf_info[eop_idx].m;
 307         if (mbuf == NULL)
 308                 rte_panic("EOP desc does not point to a valid mbuf");
 309         rte_pktmbuf_free(mbuf);
 310
 311         txq->cmd_ring.buf_info[eop_idx].m = NULL;
 312
 313         while (txq->cmd_ring.next2comp != eop_idx) {
 314                 /* no out-of-order completion */
 315                 RTE_ASSERT(txq->cmd_ring.base[txq->cmd_ring.next2comp].txd.cq == 0);
 316                 vmxnet3_cmd_ring_adv_next2comp(&txq->cmd_ring);
 317                 completed++;
 318         }
 319
 320         /* Mark the txd for which tcd was generated as completed */
 321         vmxnet3_cmd_ring_adv_next2comp(&txq->cmd_ring);
 322
 323         return completed + 1;
 324 }
 325
 326 static void
 327 vmxnet3_tq_tx_complete(vmxnet3_tx_queue_t *txq)
 328 {
 329         int completed = 0;
 330         vmxnet3_comp_ring_t *comp_ring = &txq->comp_ring;
 331         struct Vmxnet3_TxCompDesc *tcd = (struct Vmxnet3_TxCompDesc *)
 332                 (comp_ring->base + comp_ring->next2proc);
 333
 334         while (tcd->gen == comp_ring->gen) {
 335                 completed += vmxnet3_unmap_pkt(tcd->txdIdx, txq);
 336
 337                 vmxnet3_comp_ring_adv_next2proc(comp_ring);
 338                 tcd = (struct Vmxnet3_TxCompDesc *)(comp_ring->base +
 339                                                     comp_ring->next2proc);
 340         }
 341
 342         PMD_TX_LOG(DEBUG, "Processed %d tx comps & command descs.", completed);
 343
 344         /* To avoid compiler warnings when not in DEBUG mode. */
 345         RTE_SET_USED(completed);
 346 }
 347
 348 uint16_t
 349 vmxnet3_prep_pkts(__rte_unused void *tx_queue, struct rte_mbuf **tx_pkts,
 350         uint16_t nb_pkts)
 351 {
 352         int32_t ret;
 353         uint32_t i;
 354         uint64_t ol_flags;
 355         struct rte_mbuf *m;
 356
 357         for (i = 0; i != nb_pkts; i++) {
 358                 m = tx_pkts[i];
 359                 ol_flags = m->ol_flags;
 360
 361                 /* Non-TSO packet cannot occupy more than
 362                  * VMXNET3_MAX_TXD_PER_PKT TX descriptors.
 363                  */
 364                 if ((ol_flags & RTE_MBUF_F_TX_TCP_SEG) == 0 &&
 365                                 m->nb_segs > VMXNET3_MAX_TXD_PER_PKT) {
 366                         rte_errno = EINVAL;
 367                         return i;
 368                 }
 369
 370                 /* check that only supported TX offloads are requested. */
 371                 if ((ol_flags & VMXNET3_TX_OFFLOAD_NOTSUP_MASK) != 0 ||
 372                                 (ol_flags & RTE_MBUF_F_TX_L4_MASK) ==
 373                                 RTE_MBUF_F_TX_SCTP_CKSUM) {
 374                         rte_errno = ENOTSUP;
 375                         return i;
 376                 }
 377
 378 #ifdef RTE_LIBRTE_ETHDEV_DEBUG
 379                 ret = rte_validate_tx_offload(m);
 380                 if (ret != 0) {
 381                         rte_errno = -ret;
 382                         return i;
 383                 }
 384 #endif
 385                 ret = rte_net_intel_cksum_prepare(m);
 386                 if (ret != 0) {
 387                         rte_errno = -ret;
 388                         return i;
 389                 }
 390         }
 391
 392         return i;
 393 }
 394
 395 uint16_t
 396 vmxnet3_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts,
 397                   uint16_t nb_pkts)
 398 {
 399         uint16_t nb_tx;
 400         vmxnet3_tx_queue_t *txq = tx_queue;
 401         struct vmxnet3_hw *hw = txq->hw;
 402         Vmxnet3_TxQueueCtrl *txq_ctrl = &txq->shared->ctrl;
 403         uint32_t deferred = rte_le_to_cpu_32(txq_ctrl->txNumDeferred);
 404
 405         if (unlikely(txq->stopped)) {
 406                 PMD_TX_LOG(DEBUG, "Tx queue is stopped.");
 407                 return 0;
 408         }
 409
 410         /* Free up the comp_descriptors aggressively */
 411         vmxnet3_tq_tx_complete(txq);
 412
 413         nb_tx = 0;
 414         while (nb_tx < nb_pkts) {
 415                 Vmxnet3_GenericDesc *gdesc;
 416                 vmxnet3_buf_info_t *tbi;
 417                 uint32_t first2fill, avail, dw2;
 418                 struct rte_mbuf *txm = tx_pkts[nb_tx];
 419                 struct rte_mbuf *m_seg = txm;
 420                 int copy_size = 0;
 421                 bool tso = (txm->ol_flags & RTE_MBUF_F_TX_TCP_SEG) != 0;
 422                 /* # of descriptors needed for a packet. */
 423                 unsigned count = txm->nb_segs;
 424
 425                 avail = vmxnet3_cmd_ring_desc_avail(&txq->cmd_ring);
 426                 if (count > avail) {
 427                         /* Is command ring full? */
 428                         if (unlikely(avail == 0)) {
 429                                 PMD_TX_LOG(DEBUG, "No free ring descriptors");
 430                                 txq->stats.tx_ring_full++;
 431                                 txq->stats.drop_total += (nb_pkts - nb_tx);
 432                                 break;
 433                         }
 434
 435                         /* Command ring is not full but cannot handle the
 436                          * multi-segmented packet. Let's try the next packet
 437                          * in this case.
 438                          */
 439                         PMD_TX_LOG(DEBUG, "Running out of ring descriptors "
 440                                    "(avail %d needed %d)", avail, count);
 441                         txq->stats.drop_total++;
 442                         if (tso)
 443                                 txq->stats.drop_tso++;
 444                         rte_pktmbuf_free(txm);
 445                         nb_tx++;
 446                         continue;
 447                 }
 448
 449                 /* Drop non-TSO packet that is excessively fragmented */
 450                 if (unlikely(!tso && count > VMXNET3_MAX_TXD_PER_PKT)) {
 451                         PMD_TX_LOG(ERR, "Non-TSO packet cannot occupy more than %d tx "
 452                                    "descriptors. Packet dropped.", VMXNET3_MAX_TXD_PER_PKT);
 453                         txq->stats.drop_too_many_segs++;
 454                         txq->stats.drop_total++;
 455                         rte_pktmbuf_free(txm);
 456                         nb_tx++;
 457                         continue;
 458                 }
 459
 460                 if (txm->nb_segs == 1 &&
 461                     rte_pktmbuf_pkt_len(txm) <= txq->txdata_desc_size) {
 462                         struct Vmxnet3_TxDataDesc *tdd;
 463
 464                         /* Skip empty packets */
 465                         if (unlikely(rte_pktmbuf_pkt_len(txm) == 0)) {
 466                                 txq->stats.drop_total++;
 467                                 rte_pktmbuf_free(txm);
 468                                 nb_tx++;
 469                                 continue;
 470                         }
 471
 472                         tdd = (struct Vmxnet3_TxDataDesc *)
 473                                 ((uint8 *)txq->data_ring.base +
 474                                  txq->cmd_ring.next2fill *
 475                                  txq->txdata_desc_size);
 476                         copy_size = rte_pktmbuf_pkt_len(txm);
 477                         rte_memcpy(tdd->data, rte_pktmbuf_mtod(txm, char *), copy_size);
 478                 }
 479
 480                 /* use the previous gen bit for the SOP desc */
 481                 dw2 = (txq->cmd_ring.gen ^ 0x1) << VMXNET3_TXD_GEN_SHIFT;
 482                 first2fill = txq->cmd_ring.next2fill;
 483                 do {
 484                         /* Remember the transmit buffer for cleanup */
 485                         tbi = txq->cmd_ring.buf_info + txq->cmd_ring.next2fill;
 486
 487                         /* NB: the following assumes that VMXNET3 maximum
 488                          * transmit buffer size (16K) is greater than
 489                          * maximum size of mbuf segment size.
 490                          */
 491                         gdesc = txq->cmd_ring.base + txq->cmd_ring.next2fill;
 492
 493                         /* Skip empty segments */
 494                         if (unlikely(m_seg->data_len == 0))
 495                                 continue;
 496
 497                         if (copy_size) {
 498                                 uint64 offset =
 499                                         (uint64)txq->cmd_ring.next2fill *
 500                                                         txq->txdata_desc_size;
 501                                 gdesc->txd.addr =
 502                                         rte_cpu_to_le_64(txq->data_ring.basePA +
 503                                                          offset);
 504                         } else {
 505                                 gdesc->txd.addr = rte_mbuf_data_iova(m_seg);
 506                         }
 507
 508                         gdesc->dword[2] = dw2 | m_seg->data_len;
 509                         gdesc->dword[3] = 0;
 510
 511                         /* move to the next2fill descriptor */
 512                         vmxnet3_cmd_ring_adv_next2fill(&txq->cmd_ring);
 513
 514                         /* use the right gen for non-SOP desc */
 515                         dw2 = txq->cmd_ring.gen << VMXNET3_TXD_GEN_SHIFT;
 516                 } while ((m_seg = m_seg->next) != NULL);
 517
 518                 /* set the last buf_info for the pkt */
 519                 tbi->m = txm;
 520                 /* Update the EOP descriptor */
 521                 gdesc->dword[3] |= VMXNET3_TXD_EOP | VMXNET3_TXD_CQ;
 522
 523                 /* Add VLAN tag if present */
 524                 gdesc = txq->cmd_ring.base + first2fill;
 525                 if (txm->ol_flags & RTE_MBUF_F_TX_VLAN) {
 526                         gdesc->txd.ti = 1;
 527                         gdesc->txd.tci = txm->vlan_tci;
 528                 }
 529
 530                 if (tso) {
 531                         uint16_t mss = txm->tso_segsz;
 532
 533                         RTE_ASSERT(mss > 0);
 534
 535                         gdesc->txd.hlen = txm->l2_len + txm->l3_len + txm->l4_len;
 536                         gdesc->txd.om = VMXNET3_OM_TSO;
 537                         gdesc->txd.msscof = mss;
 538
 539                         deferred += (rte_pktmbuf_pkt_len(txm) - gdesc->txd.hlen + mss - 1) / mss;
 540                 } else if (txm->ol_flags & RTE_MBUF_F_TX_L4_MASK) {
 541                         gdesc->txd.om = VMXNET3_OM_CSUM;
 542                         gdesc->txd.hlen = txm->l2_len + txm->l3_len;
 543
 544                         switch (txm->ol_flags & RTE_MBUF_F_TX_L4_MASK) {
 545                         case RTE_MBUF_F_TX_TCP_CKSUM:
 546                                 gdesc->txd.msscof = gdesc->txd.hlen +
 547                                         offsetof(struct rte_tcp_hdr, cksum);
 548                                 break;
 549                         case RTE_MBUF_F_TX_UDP_CKSUM:
 550                                 gdesc->txd.msscof = gdesc->txd.hlen +
 551                                         offsetof(struct rte_udp_hdr,
 552                                                 dgram_cksum);
 553                                 break;
 554                         default:
 555                                 PMD_TX_LOG(WARNING, "requested cksum offload not supported %#llx",
 556                                            txm->ol_flags & RTE_MBUF_F_TX_L4_MASK);
 557                                 abort();
 558                         }
 559                         deferred++;
 560                 } else {
 561                         gdesc->txd.hlen = 0;
 562                         gdesc->txd.om = VMXNET3_OM_NONE;
 563                         gdesc->txd.msscof = 0;
 564                         deferred++;
 565                 }
 566
 567                 /* flip the GEN bit on the SOP */
 568                 rte_compiler_barrier();
 569                 gdesc->dword[2] ^= VMXNET3_TXD_GEN;
 570
 571                 txq_ctrl->txNumDeferred = rte_cpu_to_le_32(deferred);
 572                 nb_tx++;
 573         }
 574
 575         PMD_TX_LOG(DEBUG, "vmxnet3 txThreshold: %u", rte_le_to_cpu_32(txq_ctrl->txThreshold));
 576
 577         if (deferred >= rte_le_to_cpu_32(txq_ctrl->txThreshold)) {
 578                 txq_ctrl->txNumDeferred = 0;
 579                 /* Notify vSwitch that packets are available. */
 580                 VMXNET3_WRITE_BAR0_REG(hw, (VMXNET3_REG_TXPROD + txq->queue_id * VMXNET3_REG_ALIGN),
 581                                        txq->cmd_ring.next2fill);
 582         }
 583
 584         return nb_tx;
 585 }
 586
 587 static inline void
 588 vmxnet3_renew_desc(vmxnet3_rx_queue_t *rxq, uint8_t ring_id,
 589                    struct rte_mbuf *mbuf)
 590 {
 591         uint32_t val;
 592         struct vmxnet3_cmd_ring *ring = &rxq->cmd_ring[ring_id];
 593         struct Vmxnet3_RxDesc *rxd =
 594                 (struct Vmxnet3_RxDesc *)(ring->base + ring->next2fill);
 595         vmxnet3_buf_info_t *buf_info = &ring->buf_info[ring->next2fill];
 596
 597         if (ring_id == 0) {
 598                 /* Usually: One HEAD type buf per packet
 599                  * val = (ring->next2fill % rxq->hw->bufs_per_pkt) ?
 600                  * VMXNET3_RXD_BTYPE_BODY : VMXNET3_RXD_BTYPE_HEAD;
 601                  */
 602
 603                 /* We use single packet buffer so all heads here */
 604                 val = VMXNET3_RXD_BTYPE_HEAD;
 605         } else {
 606                 /* All BODY type buffers for 2nd ring */
 607                 val = VMXNET3_RXD_BTYPE_BODY;
 608         }
 609
 610         /*
 611          * Load mbuf pointer into buf_info[ring_size]
 612          * buf_info structure is equivalent to cookie for virtio-virtqueue
 613          */
 614         buf_info->m = mbuf;
 615         buf_info->len = (uint16_t)(mbuf->buf_len - RTE_PKTMBUF_HEADROOM);
 616         buf_info->bufPA = rte_mbuf_data_iova_default(mbuf);
 617
 618         /* Load Rx Descriptor with the buffer's GPA */
 619         rxd->addr = buf_info->bufPA;
 620
 621         /* After this point rxd->addr MUST not be NULL */
 622         rxd->btype = val;
 623         rxd->len = buf_info->len;
 624         /* Flip gen bit at the end to change ownership */
 625         rxd->gen = ring->gen;
 626
 627         vmxnet3_cmd_ring_adv_next2fill(ring);
 628 }
 629 /*
 630  *  Allocates mbufs and clusters. Post rx descriptors with buffer details
 631  *  so that device can receive packets in those buffers.
 632  *  Ring layout:
 633  *      Among the two rings, 1st ring contains buffers of type 0 and type 1.
 634  *      bufs_per_pkt is set such that for non-LRO cases all the buffers required
 635  *      by a frame will fit in 1st ring (1st buf of type0 and rest of type1).
 636  *      2nd ring contains buffers of type 1 alone. Second ring mostly be used
 637  *      only for LRO.
 638  */
 639 static int
 640 vmxnet3_post_rx_bufs(vmxnet3_rx_queue_t *rxq, uint8_t ring_id)
 641 {
 642         int err = 0;
 643         uint32_t i = 0;
 644         struct vmxnet3_cmd_ring *ring = &rxq->cmd_ring[ring_id];
 645
 646         while (vmxnet3_cmd_ring_desc_avail(ring) > 0) {
 647                 struct rte_mbuf *mbuf;
 648
 649                 /* Allocate blank mbuf for the current Rx Descriptor */
 650                 mbuf = rte_mbuf_raw_alloc(rxq->mp);
 651                 if (unlikely(mbuf == NULL)) {
 652                         PMD_RX_LOG(ERR, "Error allocating mbuf");
 653                         rxq->stats.rx_buf_alloc_failure++;
 654                         err = ENOMEM;
 655                         break;
 656                 }
 657
 658                 vmxnet3_renew_desc(rxq, ring_id, mbuf);
 659                 i++;
 660         }
 661
 662         /* Return error only if no buffers are posted at present */
 663         if (vmxnet3_cmd_ring_desc_avail(ring) >= (ring->size - 1))
 664                 return -err;
 665         else
 666                 return i;
 667 }
 668
 669 /* MSS not provided by vmxnet3, guess one with available information */
 670 static uint16_t
 671 vmxnet3_guess_mss(struct vmxnet3_hw *hw, const Vmxnet3_RxCompDesc *rcd,
 672                 struct rte_mbuf *rxm)
 673 {
 674         uint32_t hlen, slen;
 675         struct rte_ipv4_hdr *ipv4_hdr;
 676         struct rte_ipv6_hdr *ipv6_hdr;
 677         struct rte_tcp_hdr *tcp_hdr;
 678         char *ptr;
 679         uint8_t segs;
 680
 681         RTE_ASSERT(rcd->tcp);
 682
 683         ptr = rte_pktmbuf_mtod(rxm, char *);
 684         slen = rte_pktmbuf_data_len(rxm);
 685         hlen = sizeof(struct rte_ether_hdr);
 686
 687         if (rcd->v4) {
 688                 if (unlikely(slen < hlen + sizeof(struct rte_ipv4_hdr)))
 689                         return hw->mtu - sizeof(struct rte_ipv4_hdr)
 690                                         - sizeof(struct rte_tcp_hdr);
 691
 692                 ipv4_hdr = (struct rte_ipv4_hdr *)(ptr + hlen);
 693                 hlen += rte_ipv4_hdr_len(ipv4_hdr);
 694         } else if (rcd->v6) {
 695                 if (unlikely(slen < hlen + sizeof(struct rte_ipv6_hdr)))
 696                         return hw->mtu - sizeof(struct rte_ipv6_hdr) -
 697                                         sizeof(struct rte_tcp_hdr);
 698
 699                 ipv6_hdr = (struct rte_ipv6_hdr *)(ptr + hlen);
 700                 hlen += sizeof(struct rte_ipv6_hdr);
 701                 if (unlikely(ipv6_hdr->proto != IPPROTO_TCP)) {
 702                         int frag;
 703
 704                         rte_net_skip_ip6_ext(ipv6_hdr->proto, rxm,
 705                                         &hlen, &frag);
 706                 }
 707         }
 708
 709         if (unlikely(slen < hlen + sizeof(struct rte_tcp_hdr)))
 710                 return hw->mtu - hlen - sizeof(struct rte_tcp_hdr) +
 711                                 sizeof(struct rte_ether_hdr);
 712
 713         tcp_hdr = (struct rte_tcp_hdr *)(ptr + hlen);
 714         hlen += (tcp_hdr->data_off & 0xf0) >> 2;
 715
 716         segs = *vmxnet3_segs_dynfield(rxm);
 717         if (segs > 1)
 718                 return (rte_pktmbuf_pkt_len(rxm) - hlen + segs - 1) / segs;
 719         else
 720                 return hw->mtu - hlen + sizeof(struct rte_ether_hdr);
 721 }
 722
 723 /* Receive side checksum and other offloads */
 724 static inline void
 725 vmxnet3_rx_offload(struct vmxnet3_hw *hw, const Vmxnet3_RxCompDesc *rcd,
 726                 struct rte_mbuf *rxm, const uint8_t sop)
 727 {
 728         uint64_t ol_flags = rxm->ol_flags;
 729         uint32_t packet_type = rxm->packet_type;
 730
 731         /* Offloads set in sop */
 732         if (sop) {
 733                 /* Set packet type */
 734                 packet_type |= RTE_PTYPE_L2_ETHER;
 735
 736                 /* Check large packet receive */
 737                 if (VMXNET3_VERSION_GE_2(hw) &&
 738                     rcd->type == VMXNET3_CDTYPE_RXCOMP_LRO) {
 739                         const Vmxnet3_RxCompDescExt *rcde =
 740                                         (const Vmxnet3_RxCompDescExt *)rcd;
 741
 742                         rxm->tso_segsz = rcde->mss;
 743                         *vmxnet3_segs_dynfield(rxm) = rcde->segCnt;
 744                         ol_flags |= RTE_MBUF_F_RX_LRO;
 745                 }
 746         } else { /* Offloads set in eop */
 747                 /* Check for RSS */
 748                 if (rcd->rssType != VMXNET3_RCD_RSS_TYPE_NONE) {
 749                         ol_flags |= RTE_MBUF_F_RX_RSS_HASH;
 750                         rxm->hash.rss = rcd->rssHash;
 751                 }
 752
 753                 /* Check for hardware stripped VLAN tag */
 754                 if (rcd->ts) {
 755                         ol_flags |= (RTE_MBUF_F_RX_VLAN | RTE_MBUF_F_RX_VLAN_STRIPPED);
 756                         rxm->vlan_tci = rte_le_to_cpu_16((uint16_t)rcd->tci);
 757                 }
 758
 759                 /* Check packet type, checksum errors, etc. */
 760                 if (rcd->cnc) {
 761                         ol_flags |= RTE_MBUF_F_RX_L4_CKSUM_UNKNOWN;
 762
 763                         if (rcd->v4) {
 764                                 packet_type |= RTE_PTYPE_L3_IPV4_EXT_UNKNOWN;
 765                                 if (rcd->tcp)
 766                                         packet_type |= RTE_PTYPE_L4_TCP;
 767                                 else if (rcd->udp)
 768                                         packet_type |= RTE_PTYPE_L4_UDP;
 769                         } else if (rcd->v6) {
 770                                 packet_type |= RTE_PTYPE_L3_IPV6_EXT_UNKNOWN;
 771                                 if (rcd->tcp)
 772                                         packet_type |= RTE_PTYPE_L4_TCP;
 773                                 else if (rcd->udp)
 774                                         packet_type |= RTE_PTYPE_L4_UDP;
 775                         } else {
 776                                 packet_type |= RTE_PTYPE_UNKNOWN;
 777                         }
 778
 779                 } else {
 780                         if (rcd->v4) {
 781                                 packet_type |= RTE_PTYPE_L3_IPV4_EXT_UNKNOWN;
 782
 783                                 if (rcd->ipc)
 784                                         ol_flags |= RTE_MBUF_F_RX_IP_CKSUM_GOOD;
 785                                 else
 786                                         ol_flags |= RTE_MBUF_F_RX_IP_CKSUM_BAD;
 787
 788                                 if (rcd->tuc) {
 789                                         ol_flags |= RTE_MBUF_F_RX_L4_CKSUM_GOOD;
 790                                         if (rcd->tcp)
 791                                                 packet_type |= RTE_PTYPE_L4_TCP;
 792                                         else
 793                                                 packet_type |= RTE_PTYPE_L4_UDP;
 794                                 } else {
 795                                         if (rcd->tcp) {
 796                                                 packet_type |= RTE_PTYPE_L4_TCP;
 797                                                 ol_flags |= RTE_MBUF_F_RX_L4_CKSUM_BAD;
 798                                         } else if (rcd->udp) {
 799                                                 packet_type |= RTE_PTYPE_L4_UDP;
 800                                                 ol_flags |= RTE_MBUF_F_RX_L4_CKSUM_BAD;
 801                                         }
 802                                 }
 803                         } else if (rcd->v6) {
 804                                 packet_type |= RTE_PTYPE_L3_IPV6_EXT_UNKNOWN;
 805
 806                                 if (rcd->tuc) {
 807                                         ol_flags |= RTE_MBUF_F_RX_L4_CKSUM_GOOD;
 808                                         if (rcd->tcp)
 809                                                 packet_type |= RTE_PTYPE_L4_TCP;
 810                                         else
 811                                                 packet_type |= RTE_PTYPE_L4_UDP;
 812                                 } else {
 813                                         if (rcd->tcp) {
 814                                                 packet_type |= RTE_PTYPE_L4_TCP;
 815                                                 ol_flags |= RTE_MBUF_F_RX_L4_CKSUM_BAD;
 816                                         } else if (rcd->udp) {
 817                                                 packet_type |= RTE_PTYPE_L4_UDP;
 818                                                 ol_flags |= RTE_MBUF_F_RX_L4_CKSUM_BAD;
 819                                         }
 820                                 }
 821                         } else {
 822                                 packet_type |= RTE_PTYPE_UNKNOWN;
 823                         }
 824
 825                         /* Old variants of vmxnet3 do not provide MSS */
 826                         if ((ol_flags & RTE_MBUF_F_RX_LRO) && rxm->tso_segsz == 0)
 827                                 rxm->tso_segsz = vmxnet3_guess_mss(hw,
 828                                                 rcd, rxm);
 829                 }
 830         }
 831
 832         rxm->ol_flags = ol_flags;
 833         rxm->packet_type = packet_type;
 834 }
 835
 836 /*
 837  * Process the Rx Completion Ring of given vmxnet3_rx_queue
 838  * for nb_pkts burst and return the number of packets received
 839  */
 840 uint16_t
 841 vmxnet3_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts, uint16_t nb_pkts)
 842 {
 843         uint16_t nb_rx;
 844         uint32_t nb_rxd, idx;
 845         uint8_t ring_idx;
 846         vmxnet3_rx_queue_t *rxq;
 847         Vmxnet3_RxCompDesc *rcd;
 848         vmxnet3_buf_info_t *rbi;
 849         Vmxnet3_RxDesc *rxd;
 850         struct rte_mbuf *rxm = NULL;
 851         struct vmxnet3_hw *hw;
 852
 853         nb_rx = 0;
 854         ring_idx = 0;
 855         nb_rxd = 0;
 856         idx = 0;
 857
 858         rxq = rx_queue;
 859         hw = rxq->hw;
 860
 861         rcd = &rxq->comp_ring.base[rxq->comp_ring.next2proc].rcd;
 862
 863         if (unlikely(rxq->stopped)) {
 864                 PMD_RX_LOG(DEBUG, "Rx queue is stopped.");
 865                 return 0;
 866         }
 867
 868         while (rcd->gen == rxq->comp_ring.gen) {
 869                 struct rte_mbuf *newm;
 870
 871                 if (nb_rx >= nb_pkts)
 872                         break;
 873
 874                 newm = rte_mbuf_raw_alloc(rxq->mp);
 875                 if (unlikely(newm == NULL)) {
 876                         PMD_RX_LOG(ERR, "Error allocating mbuf");
 877                         rxq->stats.rx_buf_alloc_failure++;
 878                         break;
 879                 }
 880
 881                 idx = rcd->rxdIdx;
 882                 ring_idx = vmxnet3_get_ring_idx(hw, rcd->rqID);
 883                 rxd = (Vmxnet3_RxDesc *)rxq->cmd_ring[ring_idx].base + idx;
 884                 RTE_SET_USED(rxd); /* used only for assert when enabled */
 885                 rbi = rxq->cmd_ring[ring_idx].buf_info + idx;
 886
 887                 PMD_RX_LOG(DEBUG, "rxd idx: %d ring idx: %d.", idx, ring_idx);
 888
 889                 RTE_ASSERT(rcd->len <= rxd->len);
 890                 RTE_ASSERT(rbi->m);
 891
 892                 /* Get the packet buffer pointer from buf_info */
 893                 rxm = rbi->m;
 894
 895                 /* Clear descriptor associated buf_info to be reused */
 896                 rbi->m = NULL;
 897                 rbi->bufPA = 0;
 898
 899                 /* Update the index that we received a packet */
 900                 rxq->cmd_ring[ring_idx].next2comp = idx;
 901
 902                 /* For RCD with EOP set, check if there is frame error */
 903                 if (unlikely(rcd->eop && rcd->err)) {
 904                         rxq->stats.drop_total++;
 905                         rxq->stats.drop_err++;
 906
 907                         if (!rcd->fcs) {
 908                                 rxq->stats.drop_fcs++;
 909                                 PMD_RX_LOG(ERR, "Recv packet dropped due to frame err.");
 910                         }
 911                         PMD_RX_LOG(ERR, "Error in received packet rcd#:%d rxd:%d",
 912                                    (int)(rcd - (struct Vmxnet3_RxCompDesc *)
 913                                          rxq->comp_ring.base), rcd->rxdIdx);
 914                         rte_pktmbuf_free_seg(rxm);
 915                         if (rxq->start_seg) {
 916                                 struct rte_mbuf *start = rxq->start_seg;
 917
 918                                 rxq->start_seg = NULL;
 919                                 rte_pktmbuf_free(start);
 920                         }
 921                         goto rcd_done;
 922                 }
 923
 924                 /* Initialize newly received packet buffer */
 925                 rxm->port = rxq->port_id;
 926                 rxm->nb_segs = 1;
 927                 rxm->next = NULL;
 928                 rxm->pkt_len = (uint16_t)rcd->len;
 929                 rxm->data_len = (uint16_t)rcd->len;
 930                 rxm->data_off = RTE_PKTMBUF_HEADROOM;
 931                 rxm->ol_flags = 0;
 932                 rxm->vlan_tci = 0;
 933                 rxm->packet_type = 0;
 934
 935                 /*
 936                  * If this is the first buffer of the received packet,
 937                  * set the pointer to the first mbuf of the packet
 938                  * Otherwise, update the total length and the number of segments
 939                  * of the current scattered packet, and update the pointer to
 940                  * the last mbuf of the current packet.
 941                  */
 942                 if (rcd->sop) {
 943                         RTE_ASSERT(rxd->btype == VMXNET3_RXD_BTYPE_HEAD);
 944
 945                         if (unlikely(rcd->len == 0)) {
 946                                 RTE_ASSERT(rcd->eop);
 947
 948                                 PMD_RX_LOG(DEBUG,
 949                                            "Rx buf was skipped. rxring[%d][%d])",
 950                                            ring_idx, idx);
 951                                 rte_pktmbuf_free_seg(rxm);
 952                                 goto rcd_done;
 953                         }
 954
 955                         if (vmxnet3_rx_data_ring(hw, rcd->rqID)) {
 956                                 uint8_t *rdd = rxq->data_ring.base +
 957                                         idx * rxq->data_desc_size;
 958
 959                                 RTE_ASSERT(VMXNET3_VERSION_GE_3(hw));
 960                                 rte_memcpy(rte_pktmbuf_mtod(rxm, char *),
 961                                            rdd, rcd->len);
 962                         }
 963
 964                         rxq->start_seg = rxm;
 965                         rxq->last_seg = rxm;
 966                         vmxnet3_rx_offload(hw, rcd, rxm, 1);
 967                 } else {
 968                         struct rte_mbuf *start = rxq->start_seg;
 969
 970                         RTE_ASSERT(rxd->btype == VMXNET3_RXD_BTYPE_BODY);
 971
 972                         if (likely(start && rxm->data_len > 0)) {
 973                                 start->pkt_len += rxm->data_len;
 974                                 start->nb_segs++;
 975
 976                                 rxq->last_seg->next = rxm;
 977                                 rxq->last_seg = rxm;
 978                         } else {
 979                                 PMD_RX_LOG(ERR, "Error received empty or out of order frame.");
 980                                 rxq->stats.drop_total++;
 981                                 rxq->stats.drop_err++;
 982
 983                                 rte_pktmbuf_free_seg(rxm);
 984                         }
 985                 }
 986
 987                 if (rcd->eop) {
 988                         struct rte_mbuf *start = rxq->start_seg;
 989
 990                         vmxnet3_rx_offload(hw, rcd, start, 0);
 991                         rx_pkts[nb_rx++] = start;
 992                         rxq->start_seg = NULL;
 993                 }
 994
 995 rcd_done:
 996                 rxq->cmd_ring[ring_idx].next2comp = idx;
 997                 VMXNET3_INC_RING_IDX_ONLY(rxq->cmd_ring[ring_idx].next2comp,
 998                                           rxq->cmd_ring[ring_idx].size);
 999
1000                 /* It's time to renew descriptors */
1001                 vmxnet3_renew_desc(rxq, ring_idx, newm);
1002                 if (unlikely(rxq->shared->ctrl.updateRxProd)) {
1003                         VMXNET3_WRITE_BAR0_REG(hw, rxprod_reg[ring_idx] + (rxq->queue_id * VMXNET3_REG_ALIGN),
1004                                                rxq->cmd_ring[ring_idx].next2fill);
1005                 }
1006
1007                 /* Advance to the next descriptor in comp_ring */
1008                 vmxnet3_comp_ring_adv_next2proc(&rxq->comp_ring);
1009
1010                 rcd = &rxq->comp_ring.base[rxq->comp_ring.next2proc].rcd;
1011                 nb_rxd++;
1012                 if (nb_rxd > rxq->cmd_ring[0].size) {
1013                         PMD_RX_LOG(ERR, "Used up quota of receiving packets,"
1014                                    " relinquish control.");
1015                         break;
1016                 }
1017         }
1018
1019         if (unlikely(nb_rxd == 0)) {
1020                 uint32_t avail;
1021                 for (ring_idx = 0; ring_idx < VMXNET3_RX_CMDRING_SIZE; ring_idx++) {
1022                         avail = vmxnet3_cmd_ring_desc_avail(&rxq->cmd_ring[ring_idx]);
1023                         if (unlikely(avail > 0)) {
1024                                 /* try to alloc new buf and renew descriptors */
1025                                 vmxnet3_post_rx_bufs(rxq, ring_idx);
1026                         }
1027                 }
1028                 if (unlikely(rxq->shared->ctrl.updateRxProd)) {
1029                         for (ring_idx = 0; ring_idx < VMXNET3_RX_CMDRING_SIZE; ring_idx++) {
1030                                 VMXNET3_WRITE_BAR0_REG(hw, rxprod_reg[ring_idx] + (rxq->queue_id * VMXNET3_REG_ALIGN),
1031                                                        rxq->cmd_ring[ring_idx].next2fill);
1032                         }
1033                 }
1034         }
1035
1036         return nb_rx;
1037 }
1038
1039 uint32_t
1040 vmxnet3_dev_rx_queue_count(void *rx_queue)
1041 {
1042         const vmxnet3_rx_queue_t *rxq;
1043         const Vmxnet3_RxCompDesc *rcd;
1044         uint32_t idx, nb_rxd = 0;
1045         uint8_t gen;
1046
1047         rxq = rx_queue;
1048         if (unlikely(rxq->stopped)) {
1049                 PMD_RX_LOG(DEBUG, "Rx queue is stopped.");
1050                 return 0;
1051         }
1052
1053         gen = rxq->comp_ring.gen;
1054         idx = rxq->comp_ring.next2proc;
1055         rcd = &rxq->comp_ring.base[idx].rcd;
1056         while (rcd->gen == gen) {
1057                 if (rcd->eop)
1058                         ++nb_rxd;
1059                 if (++idx == rxq->comp_ring.size) {
1060                         idx = 0;
1061                         gen ^= 1;
1062                 }
1063                 rcd = &rxq->comp_ring.base[idx].rcd;
1064         }
1065
1066         return nb_rxd;
1067 }
1068
1069 int
1070 vmxnet3_dev_tx_queue_setup(struct rte_eth_dev *dev,
1071                            uint16_t queue_idx,
1072                            uint16_t nb_desc,
1073                            unsigned int socket_id,
1074                            const struct rte_eth_txconf *tx_conf __rte_unused)
1075 {
1076         struct vmxnet3_hw *hw = dev->data->dev_private;
1077         const struct rte_memzone *mz;
1078         struct vmxnet3_tx_queue *txq;
1079         struct vmxnet3_cmd_ring *ring;
1080         struct vmxnet3_comp_ring *comp_ring;
1081         struct vmxnet3_data_ring *data_ring;
1082         int size;
1083
1084         PMD_INIT_FUNC_TRACE();
1085
1086         txq = rte_zmalloc("ethdev_tx_queue", sizeof(struct vmxnet3_tx_queue),
1087                           RTE_CACHE_LINE_SIZE);
1088         if (txq == NULL) {
1089                 PMD_INIT_LOG(ERR, "Can not allocate tx queue structure");
1090                 return -ENOMEM;
1091         }
1092
1093         txq->queue_id = queue_idx;
1094         txq->port_id = dev->data->port_id;
1095         txq->shared = NULL; /* set in vmxnet3_setup_driver_shared() */
1096         txq->hw = hw;
1097         txq->qid = queue_idx;
1098         txq->stopped = TRUE;
1099         txq->txdata_desc_size = hw->txdata_desc_size;
1100
1101         ring = &txq->cmd_ring;
1102         comp_ring = &txq->comp_ring;
1103         data_ring = &txq->data_ring;
1104
1105         /* Tx vmxnet ring length should be between 512-4096 */
1106         if (nb_desc < VMXNET3_DEF_TX_RING_SIZE) {
1107                 PMD_INIT_LOG(ERR, "VMXNET3 Tx Ring Size Min: %u",
1108                              VMXNET3_DEF_TX_RING_SIZE);
1109                 return -EINVAL;
1110         } else if (nb_desc > VMXNET3_TX_RING_MAX_SIZE) {
1111                 PMD_INIT_LOG(ERR, "VMXNET3 Tx Ring Size Max: %u",
1112                              VMXNET3_TX_RING_MAX_SIZE);
1113                 return -EINVAL;
1114         } else {
1115                 ring->size = nb_desc;
1116                 ring->size &= ~VMXNET3_RING_SIZE_MASK;
1117         }
1118         comp_ring->size = data_ring->size = ring->size;
1119
1120         /* Tx vmxnet rings structure initialization*/
1121         ring->next2fill = 0;
1122         ring->next2comp = 0;
1123         ring->gen = VMXNET3_INIT_GEN;
1124         comp_ring->next2proc = 0;
1125         comp_ring->gen = VMXNET3_INIT_GEN;
1126
1127         size = sizeof(struct Vmxnet3_TxDesc) * ring->size;
1128         size += sizeof(struct Vmxnet3_TxCompDesc) * comp_ring->size;
1129         size += txq->txdata_desc_size * data_ring->size;
1130
1131         mz = rte_eth_dma_zone_reserve(dev, "txdesc", queue_idx, size,
1132                                       VMXNET3_RING_BA_ALIGN, socket_id);
1133         if (mz == NULL) {
1134                 PMD_INIT_LOG(ERR, "ERROR: Creating queue descriptors zone");
1135                 return -ENOMEM;
1136         }
1137         txq->mz = mz;
1138         memset(mz->addr, 0, mz->len);
1139
1140         /* cmd_ring initialization */
1141         ring->base = mz->addr;
1142         ring->basePA = mz->iova;
1143
1144         /* comp_ring initialization */
1145         comp_ring->base = ring->base + ring->size;
1146         comp_ring->basePA = ring->basePA +
1147                 (sizeof(struct Vmxnet3_TxDesc) * ring->size);
1148
1149         /* data_ring initialization */
1150         data_ring->base = (Vmxnet3_TxDataDesc *)(comp_ring->base + comp_ring->size);
1151         data_ring->basePA = comp_ring->basePA +
1152                         (sizeof(struct Vmxnet3_TxCompDesc) * comp_ring->size);
1153
1154         /* cmd_ring0 buf_info allocation */
1155         ring->buf_info = rte_zmalloc("tx_ring_buf_info",
1156                                      ring->size * sizeof(vmxnet3_buf_info_t), RTE_CACHE_LINE_SIZE);
1157         if (ring->buf_info == NULL) {
1158                 PMD_INIT_LOG(ERR, "ERROR: Creating tx_buf_info structure");
1159                 return -ENOMEM;
1160         }
1161
1162         /* Update the data portion with txq */
1163         dev->data->tx_queues[queue_idx] = txq;
1164
1165         return 0;
1166 }
1167
1168 int
1169 vmxnet3_dev_rx_queue_setup(struct rte_eth_dev *dev,
1170                            uint16_t queue_idx,
1171                            uint16_t nb_desc,
1172                            unsigned int socket_id,
1173                            __rte_unused const struct rte_eth_rxconf *rx_conf,
1174                            struct rte_mempool *mp)
1175 {
1176         const struct rte_memzone *mz;
1177         struct vmxnet3_rx_queue *rxq;
1178         struct vmxnet3_hw *hw = dev->data->dev_private;
1179         struct vmxnet3_cmd_ring *ring0, *ring1, *ring;
1180         struct vmxnet3_comp_ring *comp_ring;
1181         struct vmxnet3_rx_data_ring *data_ring;
1182         int size;
1183         uint8_t i;
1184         char mem_name[32];
1185
1186         PMD_INIT_FUNC_TRACE();
1187
1188         rxq = rte_zmalloc("ethdev_rx_queue", sizeof(struct vmxnet3_rx_queue),
1189                           RTE_CACHE_LINE_SIZE);
1190         if (rxq == NULL) {
1191                 PMD_INIT_LOG(ERR, "Can not allocate rx queue structure");
1192                 return -ENOMEM;
1193         }
1194
1195         rxq->mp = mp;
1196         rxq->queue_id = queue_idx;
1197         rxq->port_id = dev->data->port_id;
1198         rxq->shared = NULL; /* set in vmxnet3_setup_driver_shared() */
1199         rxq->hw = hw;
1200         rxq->qid1 = queue_idx;
1201         rxq->qid2 = queue_idx + hw->num_rx_queues;
1202         rxq->data_ring_qid = queue_idx + 2 * hw->num_rx_queues;
1203         rxq->data_desc_size = hw->rxdata_desc_size;
1204         rxq->stopped = TRUE;
1205
1206         ring0 = &rxq->cmd_ring[0];
1207         ring1 = &rxq->cmd_ring[1];
1208         comp_ring = &rxq->comp_ring;
1209         data_ring = &rxq->data_ring;
1210
1211         /* Rx vmxnet rings length should be between 256-4096 */
1212         if (nb_desc < VMXNET3_DEF_RX_RING_SIZE) {
1213                 PMD_INIT_LOG(ERR, "VMXNET3 Rx Ring Size Min: 256");
1214                 return -EINVAL;
1215         } else if (nb_desc > VMXNET3_RX_RING_MAX_SIZE) {
1216                 PMD_INIT_LOG(ERR, "VMXNET3 Rx Ring Size Max: 4096");
1217                 return -EINVAL;
1218         } else {
1219                 ring0->size = nb_desc;
1220                 ring0->size &= ~VMXNET3_RING_SIZE_MASK;
1221                 ring1->size = ring0->size;
1222         }
1223
1224         comp_ring->size = ring0->size + ring1->size;
1225         data_ring->size = ring0->size;
1226
1227         /* Rx vmxnet rings structure initialization */
1228         ring0->next2fill = 0;
1229         ring1->next2fill = 0;
1230         ring0->next2comp = 0;
1231         ring1->next2comp = 0;
1232         ring0->gen = VMXNET3_INIT_GEN;
1233         ring1->gen = VMXNET3_INIT_GEN;
1234         comp_ring->next2proc = 0;
1235         comp_ring->gen = VMXNET3_INIT_GEN;
1236
1237         size = sizeof(struct Vmxnet3_RxDesc) * (ring0->size + ring1->size);
1238         size += sizeof(struct Vmxnet3_RxCompDesc) * comp_ring->size;
1239         if (VMXNET3_VERSION_GE_3(hw) && rxq->data_desc_size)
1240                 size += rxq->data_desc_size * data_ring->size;
1241
1242         mz = rte_eth_dma_zone_reserve(dev, "rxdesc", queue_idx, size,
1243                                       VMXNET3_RING_BA_ALIGN, socket_id);
1244         if (mz == NULL) {
1245                 PMD_INIT_LOG(ERR, "ERROR: Creating queue descriptors zone");
1246                 return -ENOMEM;
1247         }
1248         rxq->mz = mz;
1249         memset(mz->addr, 0, mz->len);
1250
1251         /* cmd_ring0 initialization */
1252         ring0->base = mz->addr;
1253         ring0->basePA = mz->iova;
1254
1255         /* cmd_ring1 initialization */
1256         ring1->base = ring0->base + ring0->size;
1257         ring1->basePA = ring0->basePA + sizeof(struct Vmxnet3_RxDesc) * ring0->size;
1258
1259         /* comp_ring initialization */
1260         comp_ring->base = ring1->base + ring1->size;
1261         comp_ring->basePA = ring1->basePA + sizeof(struct Vmxnet3_RxDesc) *
1262                 ring1->size;
1263
1264         /* data_ring initialization */
1265         if (VMXNET3_VERSION_GE_3(hw) && rxq->data_desc_size) {
1266                 data_ring->base =
1267                         (uint8_t *)(comp_ring->base + comp_ring->size);
1268                 data_ring->basePA = comp_ring->basePA +
1269                         sizeof(struct Vmxnet3_RxCompDesc) * comp_ring->size;
1270         }
1271
1272         /* cmd_ring0-cmd_ring1 buf_info allocation */
1273         for (i = 0; i < VMXNET3_RX_CMDRING_SIZE; i++) {
1274
1275                 ring = &rxq->cmd_ring[i];
1276                 ring->rid = i;
1277                 snprintf(mem_name, sizeof(mem_name), "rx_ring_%d_buf_info", i);
1278
1279                 ring->buf_info = rte_zmalloc(mem_name,
1280                                              ring->size * sizeof(vmxnet3_buf_info_t),
1281                                              RTE_CACHE_LINE_SIZE);
1282                 if (ring->buf_info == NULL) {
1283                         PMD_INIT_LOG(ERR, "ERROR: Creating rx_buf_info structure");
1284                         return -ENOMEM;
1285                 }
1286         }
1287
1288         /* Update the data portion with rxq */
1289         dev->data->rx_queues[queue_idx] = rxq;
1290
1291         return 0;
1292 }
1293
1294 /*
1295  * Initializes Receive Unit
1296  * Load mbufs in rx queue in advance
1297  */
1298 int
1299 vmxnet3_dev_rxtx_init(struct rte_eth_dev *dev)
1300 {
1301         struct vmxnet3_hw *hw = dev->data->dev_private;
1302
1303         int i, ret;
1304         uint8_t j;
1305
1306         PMD_INIT_FUNC_TRACE();
1307
1308         for (i = 0; i < hw->num_rx_queues; i++) {
1309                 vmxnet3_rx_queue_t *rxq = dev->data->rx_queues[i];
1310
1311                 for (j = 0; j < VMXNET3_RX_CMDRING_SIZE; j++) {
1312                         /* Passing 0 as alloc_num will allocate full ring */
1313                         ret = vmxnet3_post_rx_bufs(rxq, j);
1314                         if (ret <= 0) {
1315                                 PMD_INIT_LOG(ERR,
1316                                              "ERROR: Posting Rxq: %d buffers ring: %d",
1317                                              i, j);
1318                                 return -ret;
1319                         }
1320                         /*
1321                          * Updating device with the index:next2fill to fill the
1322                          * mbufs for coming packets.
1323                          */
1324                         if (unlikely(rxq->shared->ctrl.updateRxProd)) {
1325                                 VMXNET3_WRITE_BAR0_REG(hw, rxprod_reg[j] + (rxq->queue_id * VMXNET3_REG_ALIGN),
1326                                                        rxq->cmd_ring[j].next2fill);
1327                         }
1328                 }
1329                 rxq->stopped = FALSE;
1330                 rxq->start_seg = NULL;
1331         }
1332
1333         for (i = 0; i < dev->data->nb_tx_queues; i++) {
1334                 struct vmxnet3_tx_queue *txq = dev->data->tx_queues[i];
1335
1336                 txq->stopped = FALSE;
1337         }
1338
1339         return 0;
1340 }
1341
1342 static uint8_t rss_intel_key[40] = {
1343         0x6D, 0x5A, 0x56, 0xDA, 0x25, 0x5B, 0x0E, 0xC2,
1344         0x41, 0x67, 0x25, 0x3D, 0x43, 0xA3, 0x8F, 0xB0,
1345         0xD0, 0xCA, 0x2B, 0xCB, 0xAE, 0x7B, 0x30, 0xB4,
1346         0x77, 0xCB, 0x2D, 0xA3, 0x80, 0x30, 0xF2, 0x0C,
1347         0x6A, 0x42, 0xB7, 0x3B, 0xBE, 0xAC, 0x01, 0xFA,
1348 };
1349
1350 /*
1351  * Additional RSS configurations based on vmxnet v4+ APIs
1352  */
1353 int
1354 vmxnet3_v4_rss_configure(struct rte_eth_dev *dev)
1355 {
1356         struct vmxnet3_hw *hw = dev->data->dev_private;
1357         Vmxnet3_DriverShared *shared = hw->shared;
1358         Vmxnet3_CmdInfo *cmdInfo = &shared->cu.cmdInfo;
1359         struct rte_eth_rss_conf *port_rss_conf;
1360         uint64_t rss_hf;
1361         uint32_t ret;
1362
1363         PMD_INIT_FUNC_TRACE();
1364
1365         cmdInfo->setRSSFields = 0;
1366         port_rss_conf = &dev->data->dev_conf.rx_adv_conf.rss_conf;
1367
1368         if ((port_rss_conf->rss_hf & VMXNET3_MANDATORY_V4_RSS) !=
1369             VMXNET3_MANDATORY_V4_RSS) {
1370                 PMD_INIT_LOG(WARNING, "RSS: IPv4/6 TCP is required for vmxnet3 v4 RSS,"
1371                              "automatically setting it");
1372                 port_rss_conf->rss_hf |= VMXNET3_MANDATORY_V4_RSS;
1373         }
1374
1375         rss_hf = port_rss_conf->rss_hf &
1376                 (VMXNET3_V4_RSS_MASK | VMXNET3_RSS_OFFLOAD_ALL);
1377
1378         if (rss_hf & RTE_ETH_RSS_NONFRAG_IPV4_TCP)
1379                 cmdInfo->setRSSFields |= VMXNET3_RSS_FIELDS_TCPIP4;
1380         if (rss_hf & RTE_ETH_RSS_NONFRAG_IPV6_TCP)
1381                 cmdInfo->setRSSFields |= VMXNET3_RSS_FIELDS_TCPIP6;
1382         if (rss_hf & RTE_ETH_RSS_NONFRAG_IPV4_UDP)
1383                 cmdInfo->setRSSFields |= VMXNET3_RSS_FIELDS_UDPIP4;
1384         if (rss_hf & RTE_ETH_RSS_NONFRAG_IPV6_UDP)
1385                 cmdInfo->setRSSFields |= VMXNET3_RSS_FIELDS_UDPIP6;
1386
1387         VMXNET3_WRITE_BAR1_REG(hw, VMXNET3_REG_CMD,
1388                                VMXNET3_CMD_SET_RSS_FIELDS);
1389         ret = VMXNET3_READ_BAR1_REG(hw, VMXNET3_REG_CMD);
1390
1391         if (ret != VMXNET3_SUCCESS) {
1392                 PMD_DRV_LOG(ERR, "Set RSS fields (v4) failed: %d", ret);
1393         }
1394
1395         return ret;
1396 }
1397
1398 /*
1399  * Configure RSS feature
1400  */
1401 int
1402 vmxnet3_rss_configure(struct rte_eth_dev *dev)
1403 {
1404         struct vmxnet3_hw *hw = dev->data->dev_private;
1405         struct VMXNET3_RSSConf *dev_rss_conf;
1406         struct rte_eth_rss_conf *port_rss_conf;
1407         uint64_t rss_hf;
1408         uint8_t i, j;
1409
1410         PMD_INIT_FUNC_TRACE();
1411
1412         dev_rss_conf = hw->rss_conf;
1413         port_rss_conf = &dev->data->dev_conf.rx_adv_conf.rss_conf;
1414
1415         /* loading hashFunc */
1416         dev_rss_conf->hashFunc = VMXNET3_RSS_HASH_FUNC_TOEPLITZ;
1417         /* loading hashKeySize */
1418         dev_rss_conf->hashKeySize = VMXNET3_RSS_MAX_KEY_SIZE;
1419         /* loading indTableSize: Must not exceed VMXNET3_RSS_MAX_IND_TABLE_SIZE (128)*/
1420         dev_rss_conf->indTableSize = (uint16_t)((MAX_RX_QUEUES(hw)) * 4);
1421
1422         if (port_rss_conf->rss_key == NULL) {
1423                 /* Default hash key */
1424                 port_rss_conf->rss_key = rss_intel_key;
1425         }
1426
1427         /* loading hashKey */
1428         memcpy(&dev_rss_conf->hashKey[0], port_rss_conf->rss_key,
1429                dev_rss_conf->hashKeySize);
1430
1431         /* loading indTable */
1432         for (i = 0, j = 0; i < dev_rss_conf->indTableSize; i++, j++) {
1433                 if (j == dev->data->nb_rx_queues)
1434                         j = 0;
1435                 dev_rss_conf->indTable[i] = j;
1436         }
1437
1438         /* loading hashType */
1439         dev_rss_conf->hashType = 0;
1440         rss_hf = port_rss_conf->rss_hf & VMXNET3_RSS_OFFLOAD_ALL;
1441         if (rss_hf & RTE_ETH_RSS_IPV4)
1442                 dev_rss_conf->hashType |= VMXNET3_RSS_HASH_TYPE_IPV4;
1443         if (rss_hf & RTE_ETH_RSS_NONFRAG_IPV4_TCP)
1444                 dev_rss_conf->hashType |= VMXNET3_RSS_HASH_TYPE_TCP_IPV4;
1445         if (rss_hf & RTE_ETH_RSS_IPV6)
1446                 dev_rss_conf->hashType |= VMXNET3_RSS_HASH_TYPE_IPV6;
1447         if (rss_hf & RTE_ETH_RSS_NONFRAG_IPV6_TCP)
1448                 dev_rss_conf->hashType |= VMXNET3_RSS_HASH_TYPE_TCP_IPV6;
1449
1450         return VMXNET3_SUCCESS;
1451 }