drivers/net/vmxnet3/vmxnet3_rxtx.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(c) 2010-2015 Intel Corporation
   3  */
   4
   5 #include <sys/queue.h>
   6
   7 #include <stdio.h>
   8 #include <stdlib.h>
   9 #include <string.h>
  10 #include <errno.h>
  11 #include <stdint.h>
  12 #include <stdarg.h>
  13 #include <unistd.h>
  14 #include <inttypes.h>
  15
  16 #include <rte_byteorder.h>
  17 #include <rte_common.h>
  18 #include <rte_cycles.h>
  19 #include <rte_log.h>
  20 #include <rte_debug.h>
  21 #include <rte_interrupts.h>
  22 #include <rte_pci.h>
  23 #include <rte_memory.h>
  24 #include <rte_memzone.h>
  25 #include <rte_launch.h>
  26 #include <rte_eal.h>
  27 #include <rte_per_lcore.h>
  28 #include <rte_lcore.h>
  29 #include <rte_atomic.h>
  30 #include <rte_branch_prediction.h>
  31 #include <rte_mempool.h>
  32 #include <rte_malloc.h>
  33 #include <rte_mbuf.h>
  34 #include <rte_ether.h>
  35 #include <rte_ethdev_driver.h>
  36 #include <rte_prefetch.h>
  37 #include <rte_ip.h>
  38 #include <rte_udp.h>
  39 #include <rte_tcp.h>
  40 #include <rte_sctp.h>
  41 #include <rte_string_fns.h>
  42 #include <rte_errno.h>
  43 #include <rte_net.h>
  44
  45 #include "base/vmxnet3_defs.h"
  46 #include "vmxnet3_ring.h"
  47
  48 #include "vmxnet3_logs.h"
  49 #include "vmxnet3_ethdev.h"
  50
  51 #define VMXNET3_TX_OFFLOAD_MASK ( \
  52                 PKT_TX_VLAN_PKT | \
  53                 PKT_TX_IPV6 |     \
  54                 PKT_TX_IPV4 |     \
  55                 PKT_TX_L4_MASK |  \
  56                 PKT_TX_TCP_SEG)
  57
  58 #define VMXNET3_TX_OFFLOAD_NOTSUP_MASK  \
  59         (PKT_TX_OFFLOAD_MASK ^ VMXNET3_TX_OFFLOAD_MASK)
  60
  61 static const uint32_t rxprod_reg[2] = {VMXNET3_REG_RXPROD, VMXNET3_REG_RXPROD2};
  62
  63 static int vmxnet3_post_rx_bufs(vmxnet3_rx_queue_t*, uint8_t);
  64 static void vmxnet3_tq_tx_complete(vmxnet3_tx_queue_t *);
  65 #ifdef RTE_LIBRTE_VMXNET3_DEBUG_DRIVER_NOT_USED
  66 static void vmxnet3_rxq_dump(struct vmxnet3_rx_queue *);
  67 static void vmxnet3_txq_dump(struct vmxnet3_tx_queue *);
  68 #endif
  69
  70 #ifdef RTE_LIBRTE_VMXNET3_DEBUG_DRIVER_NOT_USED
  71 static void
  72 vmxnet3_rxq_dump(struct vmxnet3_rx_queue *rxq)
  73 {
  74         uint32_t avail = 0;
  75
  76         if (rxq == NULL)
  77                 return;
  78
  79         PMD_RX_LOG(DEBUG,
  80                    "RXQ: cmd0 base : %p cmd1 base : %p comp ring base : %p.",
  81                    rxq->cmd_ring[0].base, rxq->cmd_ring[1].base, rxq->comp_ring.base);
  82         PMD_RX_LOG(DEBUG,
  83                    "RXQ: cmd0 basePA : 0x%lx cmd1 basePA : 0x%lx comp ring basePA : 0x%lx.",
  84                    (unsigned long)rxq->cmd_ring[0].basePA,
  85                    (unsigned long)rxq->cmd_ring[1].basePA,
  86                    (unsigned long)rxq->comp_ring.basePA);
  87
  88         avail = vmxnet3_cmd_ring_desc_avail(&rxq->cmd_ring[0]);
  89         PMD_RX_LOG(DEBUG,
  90                    "RXQ:cmd0: size=%u; free=%u; next2proc=%u; queued=%u",
  91                    (uint32_t)rxq->cmd_ring[0].size, avail,
  92                    rxq->comp_ring.next2proc,
  93                    rxq->cmd_ring[0].size - avail);
  94
  95         avail = vmxnet3_cmd_ring_desc_avail(&rxq->cmd_ring[1]);
  96         PMD_RX_LOG(DEBUG, "RXQ:cmd1 size=%u; free=%u; next2proc=%u; queued=%u",
  97                    (uint32_t)rxq->cmd_ring[1].size, avail, rxq->comp_ring.next2proc,
  98                    rxq->cmd_ring[1].size - avail);
  99
 100 }
 101
 102 static void
 103 vmxnet3_txq_dump(struct vmxnet3_tx_queue *txq)
 104 {
 105         uint32_t avail = 0;
 106
 107         if (txq == NULL)
 108                 return;
 109
 110         PMD_TX_LOG(DEBUG, "TXQ: cmd base : %p comp ring base : %p data ring base : %p.",
 111                    txq->cmd_ring.base, txq->comp_ring.base, txq->data_ring.base);
 112         PMD_TX_LOG(DEBUG, "TXQ: cmd basePA : 0x%lx comp ring basePA : 0x%lx data ring basePA : 0x%lx.",
 113                    (unsigned long)txq->cmd_ring.basePA,
 114                    (unsigned long)txq->comp_ring.basePA,
 115                    (unsigned long)txq->data_ring.basePA);
 116
 117         avail = vmxnet3_cmd_ring_desc_avail(&txq->cmd_ring);
 118         PMD_TX_LOG(DEBUG, "TXQ: size=%u; free=%u; next2proc=%u; queued=%u",
 119                    (uint32_t)txq->cmd_ring.size, avail,
 120                    txq->comp_ring.next2proc, txq->cmd_ring.size - avail);
 121 }
 122 #endif
 123
 124 static void
 125 vmxnet3_tx_cmd_ring_release_mbufs(vmxnet3_cmd_ring_t *ring)
 126 {
 127         while (ring->next2comp != ring->next2fill) {
 128                 /* No need to worry about desc ownership, device is quiesced by now. */
 129                 vmxnet3_buf_info_t *buf_info = ring->buf_info + ring->next2comp;
 130
 131                 if (buf_info->m) {
 132                         rte_pktmbuf_free(buf_info->m);
 133                         buf_info->m = NULL;
 134                         buf_info->bufPA = 0;
 135                         buf_info->len = 0;
 136                 }
 137                 vmxnet3_cmd_ring_adv_next2comp(ring);
 138         }
 139 }
 140
 141 static void
 142 vmxnet3_rx_cmd_ring_release_mbufs(vmxnet3_cmd_ring_t *ring)
 143 {
 144         uint32_t i;
 145
 146         for (i = 0; i < ring->size; i++) {
 147                 /* No need to worry about desc ownership, device is quiesced by now. */
 148                 vmxnet3_buf_info_t *buf_info = &ring->buf_info[i];
 149
 150                 if (buf_info->m) {
 151                         rte_pktmbuf_free_seg(buf_info->m);
 152                         buf_info->m = NULL;
 153                         buf_info->bufPA = 0;
 154                         buf_info->len = 0;
 155                 }
 156                 vmxnet3_cmd_ring_adv_next2comp(ring);
 157         }
 158 }
 159
 160 static void
 161 vmxnet3_cmd_ring_release(vmxnet3_cmd_ring_t *ring)
 162 {
 163         rte_free(ring->buf_info);
 164         ring->buf_info = NULL;
 165 }
 166
 167 void
 168 vmxnet3_dev_tx_queue_release(void *txq)
 169 {
 170         vmxnet3_tx_queue_t *tq = txq;
 171
 172         if (tq != NULL) {
 173                 /* Release mbufs */
 174                 vmxnet3_tx_cmd_ring_release_mbufs(&tq->cmd_ring);
 175                 /* Release the cmd_ring */
 176                 vmxnet3_cmd_ring_release(&tq->cmd_ring);
 177                 /* Release the memzone */
 178                 rte_memzone_free(tq->mz);
 179                 /* Release the queue */
 180                 rte_free(tq);
 181         }
 182 }
 183
 184 void
 185 vmxnet3_dev_rx_queue_release(void *rxq)
 186 {
 187         int i;
 188         vmxnet3_rx_queue_t *rq = rxq;
 189
 190         if (rq != NULL) {
 191                 /* Release mbufs */
 192                 for (i = 0; i < VMXNET3_RX_CMDRING_SIZE; i++)
 193                         vmxnet3_rx_cmd_ring_release_mbufs(&rq->cmd_ring[i]);
 194
 195                 /* Release both the cmd_rings */
 196                 for (i = 0; i < VMXNET3_RX_CMDRING_SIZE; i++)
 197                         vmxnet3_cmd_ring_release(&rq->cmd_ring[i]);
 198
 199                 /* Release the memzone */
 200                 rte_memzone_free(rq->mz);
 201
 202                 /* Release the queue */
 203                 rte_free(rq);
 204         }
 205 }
 206
 207 static void
 208 vmxnet3_dev_tx_queue_reset(void *txq)
 209 {
 210         vmxnet3_tx_queue_t *tq = txq;
 211         struct vmxnet3_cmd_ring *ring = &tq->cmd_ring;
 212         struct vmxnet3_comp_ring *comp_ring = &tq->comp_ring;
 213         struct vmxnet3_data_ring *data_ring = &tq->data_ring;
 214         int size;
 215
 216         if (tq != NULL) {
 217                 /* Release the cmd_ring mbufs */
 218                 vmxnet3_tx_cmd_ring_release_mbufs(&tq->cmd_ring);
 219         }
 220
 221         /* Tx vmxnet rings structure initialization*/
 222         ring->next2fill = 0;
 223         ring->next2comp = 0;
 224         ring->gen = VMXNET3_INIT_GEN;
 225         comp_ring->next2proc = 0;
 226         comp_ring->gen = VMXNET3_INIT_GEN;
 227
 228         size = sizeof(struct Vmxnet3_TxDesc) * ring->size;
 229         size += sizeof(struct Vmxnet3_TxCompDesc) * comp_ring->size;
 230         size += tq->txdata_desc_size * data_ring->size;
 231
 232         memset(ring->base, 0, size);
 233 }
 234
 235 static void
 236 vmxnet3_dev_rx_queue_reset(void *rxq)
 237 {
 238         int i;
 239         vmxnet3_rx_queue_t *rq = rxq;
 240         struct vmxnet3_hw *hw = rq->hw;
 241         struct vmxnet3_cmd_ring *ring0, *ring1;
 242         struct vmxnet3_comp_ring *comp_ring;
 243         struct vmxnet3_rx_data_ring *data_ring = &rq->data_ring;
 244         int size;
 245
 246         /* Release both the cmd_rings mbufs */
 247         for (i = 0; i < VMXNET3_RX_CMDRING_SIZE; i++)
 248                 vmxnet3_rx_cmd_ring_release_mbufs(&rq->cmd_ring[i]);
 249
 250         ring0 = &rq->cmd_ring[0];
 251         ring1 = &rq->cmd_ring[1];
 252         comp_ring = &rq->comp_ring;
 253
 254         /* Rx vmxnet rings structure initialization */
 255         ring0->next2fill = 0;
 256         ring1->next2fill = 0;
 257         ring0->next2comp = 0;
 258         ring1->next2comp = 0;
 259         ring0->gen = VMXNET3_INIT_GEN;
 260         ring1->gen = VMXNET3_INIT_GEN;
 261         comp_ring->next2proc = 0;
 262         comp_ring->gen = VMXNET3_INIT_GEN;
 263
 264         size = sizeof(struct Vmxnet3_RxDesc) * (ring0->size + ring1->size);
 265         size += sizeof(struct Vmxnet3_RxCompDesc) * comp_ring->size;
 266         if (VMXNET3_VERSION_GE_3(hw) && rq->data_desc_size)
 267                 size += rq->data_desc_size * data_ring->size;
 268
 269         memset(ring0->base, 0, size);
 270 }
 271
 272 void
 273 vmxnet3_dev_clear_queues(struct rte_eth_dev *dev)
 274 {
 275         unsigned i;
 276
 277         PMD_INIT_FUNC_TRACE();
 278
 279         for (i = 0; i < dev->data->nb_tx_queues; i++) {
 280                 struct vmxnet3_tx_queue *txq = dev->data->tx_queues[i];
 281
 282                 if (txq != NULL) {
 283                         txq->stopped = TRUE;
 284                         vmxnet3_dev_tx_queue_reset(txq);
 285                 }
 286         }
 287
 288         for (i = 0; i < dev->data->nb_rx_queues; i++) {
 289                 struct vmxnet3_rx_queue *rxq = dev->data->rx_queues[i];
 290
 291                 if (rxq != NULL) {
 292                         rxq->stopped = TRUE;
 293                         vmxnet3_dev_rx_queue_reset(rxq);
 294                 }
 295         }
 296 }
 297
 298 static int
 299 vmxnet3_unmap_pkt(uint16_t eop_idx, vmxnet3_tx_queue_t *txq)
 300 {
 301         int completed = 0;
 302         struct rte_mbuf *mbuf;
 303
 304         /* Release cmd_ring descriptor and free mbuf */
 305         RTE_ASSERT(txq->cmd_ring.base[eop_idx].txd.eop == 1);
 306
 307         mbuf = txq->cmd_ring.buf_info[eop_idx].m;
 308         if (mbuf == NULL)
 309                 rte_panic("EOP desc does not point to a valid mbuf");
 310         rte_pktmbuf_free(mbuf);
 311
 312         txq->cmd_ring.buf_info[eop_idx].m = NULL;
 313
 314         while (txq->cmd_ring.next2comp != eop_idx) {
 315                 /* no out-of-order completion */
 316                 RTE_ASSERT(txq->cmd_ring.base[txq->cmd_ring.next2comp].txd.cq == 0);
 317                 vmxnet3_cmd_ring_adv_next2comp(&txq->cmd_ring);
 318                 completed++;
 319         }
 320
 321         /* Mark the txd for which tcd was generated as completed */
 322         vmxnet3_cmd_ring_adv_next2comp(&txq->cmd_ring);
 323
 324         return completed + 1;
 325 }
 326
 327 static void
 328 vmxnet3_tq_tx_complete(vmxnet3_tx_queue_t *txq)
 329 {
 330         int completed = 0;
 331         vmxnet3_comp_ring_t *comp_ring = &txq->comp_ring;
 332         struct Vmxnet3_TxCompDesc *tcd = (struct Vmxnet3_TxCompDesc *)
 333                 (comp_ring->base + comp_ring->next2proc);
 334
 335         while (tcd->gen == comp_ring->gen) {
 336                 completed += vmxnet3_unmap_pkt(tcd->txdIdx, txq);
 337
 338                 vmxnet3_comp_ring_adv_next2proc(comp_ring);
 339                 tcd = (struct Vmxnet3_TxCompDesc *)(comp_ring->base +
 340                                                     comp_ring->next2proc);
 341         }
 342
 343         PMD_TX_LOG(DEBUG, "Processed %d tx comps & command descs.", completed);
 344 }
 345
 346 uint16_t
 347 vmxnet3_prep_pkts(__rte_unused void *tx_queue, struct rte_mbuf **tx_pkts,
 348         uint16_t nb_pkts)
 349 {
 350         int32_t ret;
 351         uint32_t i;
 352         uint64_t ol_flags;
 353         struct rte_mbuf *m;
 354
 355         for (i = 0; i != nb_pkts; i++) {
 356                 m = tx_pkts[i];
 357                 ol_flags = m->ol_flags;
 358
 359                 /* Non-TSO packet cannot occupy more than
 360                  * VMXNET3_MAX_TXD_PER_PKT TX descriptors.
 361                  */
 362                 if ((ol_flags & PKT_TX_TCP_SEG) == 0 &&
 363                                 m->nb_segs > VMXNET3_MAX_TXD_PER_PKT) {
 364                         rte_errno = -EINVAL;
 365                         return i;
 366                 }
 367
 368                 /* check that only supported TX offloads are requested. */
 369                 if ((ol_flags & VMXNET3_TX_OFFLOAD_NOTSUP_MASK) != 0 ||
 370                                 (ol_flags & PKT_TX_L4_MASK) ==
 371                                 PKT_TX_SCTP_CKSUM) {
 372                         rte_errno = -ENOTSUP;
 373                         return i;
 374                 }
 375
 376 #ifdef RTE_LIBRTE_ETHDEV_DEBUG
 377                 ret = rte_validate_tx_offload(m);
 378                 if (ret != 0) {
 379                         rte_errno = ret;
 380                         return i;
 381                 }
 382 #endif
 383                 ret = rte_net_intel_cksum_prepare(m);
 384                 if (ret != 0) {
 385                         rte_errno = ret;
 386                         return i;
 387                 }
 388         }
 389
 390         return i;
 391 }
 392
 393 uint16_t
 394 vmxnet3_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts,
 395                   uint16_t nb_pkts)
 396 {
 397         uint16_t nb_tx;
 398         vmxnet3_tx_queue_t *txq = tx_queue;
 399         struct vmxnet3_hw *hw = txq->hw;
 400         Vmxnet3_TxQueueCtrl *txq_ctrl = &txq->shared->ctrl;
 401         uint32_t deferred = rte_le_to_cpu_32(txq_ctrl->txNumDeferred);
 402
 403         if (unlikely(txq->stopped)) {
 404                 PMD_TX_LOG(DEBUG, "Tx queue is stopped.");
 405                 return 0;
 406         }
 407
 408         /* Free up the comp_descriptors aggressively */
 409         vmxnet3_tq_tx_complete(txq);
 410
 411         nb_tx = 0;
 412         while (nb_tx < nb_pkts) {
 413                 Vmxnet3_GenericDesc *gdesc;
 414                 vmxnet3_buf_info_t *tbi;
 415                 uint32_t first2fill, avail, dw2;
 416                 struct rte_mbuf *txm = tx_pkts[nb_tx];
 417                 struct rte_mbuf *m_seg = txm;
 418                 int copy_size = 0;
 419                 bool tso = (txm->ol_flags & PKT_TX_TCP_SEG) != 0;
 420                 /* # of descriptors needed for a packet. */
 421                 unsigned count = txm->nb_segs;
 422
 423                 avail = vmxnet3_cmd_ring_desc_avail(&txq->cmd_ring);
 424                 if (count > avail) {
 425                         /* Is command ring full? */
 426                         if (unlikely(avail == 0)) {
 427                                 PMD_TX_LOG(DEBUG, "No free ring descriptors");
 428                                 txq->stats.tx_ring_full++;
 429                                 txq->stats.drop_total += (nb_pkts - nb_tx);
 430                                 break;
 431                         }
 432
 433                         /* Command ring is not full but cannot handle the
 434                          * multi-segmented packet. Let's try the next packet
 435                          * in this case.
 436                          */
 437                         PMD_TX_LOG(DEBUG, "Running out of ring descriptors "
 438                                    "(avail %d needed %d)", avail, count);
 439                         txq->stats.drop_total++;
 440                         if (tso)
 441                                 txq->stats.drop_tso++;
 442                         rte_pktmbuf_free(txm);
 443                         nb_tx++;
 444                         continue;
 445                 }
 446
 447                 /* Drop non-TSO packet that is excessively fragmented */
 448                 if (unlikely(!tso && count > VMXNET3_MAX_TXD_PER_PKT)) {
 449                         PMD_TX_LOG(ERR, "Non-TSO packet cannot occupy more than %d tx "
 450                                    "descriptors. Packet dropped.", VMXNET3_MAX_TXD_PER_PKT);
 451                         txq->stats.drop_too_many_segs++;
 452                         txq->stats.drop_total++;
 453                         rte_pktmbuf_free(txm);
 454                         nb_tx++;
 455                         continue;
 456                 }
 457
 458                 if (txm->nb_segs == 1 &&
 459                     rte_pktmbuf_pkt_len(txm) <= txq->txdata_desc_size) {
 460                         struct Vmxnet3_TxDataDesc *tdd;
 461
 462                         /* Skip empty packets */
 463                         if (unlikely(rte_pktmbuf_pkt_len(txm) == 0)) {
 464                                 txq->stats.drop_total++;
 465                                 rte_pktmbuf_free(txm);
 466                                 nb_tx++;
 467                                 continue;
 468                         }
 469
 470                         tdd = (struct Vmxnet3_TxDataDesc *)
 471                                 ((uint8 *)txq->data_ring.base +
 472                                  txq->cmd_ring.next2fill *
 473                                  txq->txdata_desc_size);
 474                         copy_size = rte_pktmbuf_pkt_len(txm);
 475                         rte_memcpy(tdd->data, rte_pktmbuf_mtod(txm, char *), copy_size);
 476                 }
 477
 478                 /* use the previous gen bit for the SOP desc */
 479                 dw2 = (txq->cmd_ring.gen ^ 0x1) << VMXNET3_TXD_GEN_SHIFT;
 480                 first2fill = txq->cmd_ring.next2fill;
 481                 do {
 482                         /* Remember the transmit buffer for cleanup */
 483                         tbi = txq->cmd_ring.buf_info + txq->cmd_ring.next2fill;
 484
 485                         /* NB: the following assumes that VMXNET3 maximum
 486                          * transmit buffer size (16K) is greater than
 487                          * maximum size of mbuf segment size.
 488                          */
 489                         gdesc = txq->cmd_ring.base + txq->cmd_ring.next2fill;
 490
 491                         /* Skip empty segments */
 492                         if (unlikely(m_seg->data_len == 0))
 493                                 continue;
 494
 495                         if (copy_size) {
 496                                 uint64 offset =
 497                                         (uint64)txq->cmd_ring.next2fill *
 498                                                         txq->txdata_desc_size;
 499                                 gdesc->txd.addr =
 500                                         rte_cpu_to_le_64(txq->data_ring.basePA +
 501                                                          offset);
 502                         } else {
 503                                 gdesc->txd.addr = rte_mbuf_data_iova(m_seg);
 504                         }
 505
 506                         gdesc->dword[2] = dw2 | m_seg->data_len;
 507                         gdesc->dword[3] = 0;
 508
 509                         /* move to the next2fill descriptor */
 510                         vmxnet3_cmd_ring_adv_next2fill(&txq->cmd_ring);
 511
 512                         /* use the right gen for non-SOP desc */
 513                         dw2 = txq->cmd_ring.gen << VMXNET3_TXD_GEN_SHIFT;
 514                 } while ((m_seg = m_seg->next) != NULL);
 515
 516                 /* set the last buf_info for the pkt */
 517                 tbi->m = txm;
 518                 /* Update the EOP descriptor */
 519                 gdesc->dword[3] |= VMXNET3_TXD_EOP | VMXNET3_TXD_CQ;
 520
 521                 /* Add VLAN tag if present */
 522                 gdesc = txq->cmd_ring.base + first2fill;
 523                 if (txm->ol_flags & PKT_TX_VLAN_PKT) {
 524                         gdesc->txd.ti = 1;
 525                         gdesc->txd.tci = txm->vlan_tci;
 526                 }
 527
 528                 if (tso) {
 529                         uint16_t mss = txm->tso_segsz;
 530
 531                         RTE_ASSERT(mss > 0);
 532
 533                         gdesc->txd.hlen = txm->l2_len + txm->l3_len + txm->l4_len;
 534                         gdesc->txd.om = VMXNET3_OM_TSO;
 535                         gdesc->txd.msscof = mss;
 536
 537                         deferred += (rte_pktmbuf_pkt_len(txm) - gdesc->txd.hlen + mss - 1) / mss;
 538                 } else if (txm->ol_flags & PKT_TX_L4_MASK) {
 539                         gdesc->txd.om = VMXNET3_OM_CSUM;
 540                         gdesc->txd.hlen = txm->l2_len + txm->l3_len;
 541
 542                         switch (txm->ol_flags & PKT_TX_L4_MASK) {
 543                         case PKT_TX_TCP_CKSUM:
 544                                 gdesc->txd.msscof = gdesc->txd.hlen +
 545                                         offsetof(struct rte_tcp_hdr, cksum);
 546                                 break;
 547                         case PKT_TX_UDP_CKSUM:
 548                                 gdesc->txd.msscof = gdesc->txd.hlen +
 549                                         offsetof(struct rte_udp_hdr,
 550                                                 dgram_cksum);
 551                                 break;
 552                         default:
 553                                 PMD_TX_LOG(WARNING, "requested cksum offload not supported %#llx",
 554                                            txm->ol_flags & PKT_TX_L4_MASK);
 555                                 abort();
 556                         }
 557                         deferred++;
 558                 } else {
 559                         gdesc->txd.hlen = 0;
 560                         gdesc->txd.om = VMXNET3_OM_NONE;
 561                         gdesc->txd.msscof = 0;
 562                         deferred++;
 563                 }
 564
 565                 /* flip the GEN bit on the SOP */
 566                 rte_compiler_barrier();
 567                 gdesc->dword[2] ^= VMXNET3_TXD_GEN;
 568
 569                 txq_ctrl->txNumDeferred = rte_cpu_to_le_32(deferred);
 570                 nb_tx++;
 571         }
 572
 573         PMD_TX_LOG(DEBUG, "vmxnet3 txThreshold: %u", rte_le_to_cpu_32(txq_ctrl->txThreshold));
 574
 575         if (deferred >= rte_le_to_cpu_32(txq_ctrl->txThreshold)) {
 576                 txq_ctrl->txNumDeferred = 0;
 577                 /* Notify vSwitch that packets are available. */
 578                 VMXNET3_WRITE_BAR0_REG(hw, (VMXNET3_REG_TXPROD + txq->queue_id * VMXNET3_REG_ALIGN),
 579                                        txq->cmd_ring.next2fill);
 580         }
 581
 582         return nb_tx;
 583 }
 584
 585 static inline void
 586 vmxnet3_renew_desc(vmxnet3_rx_queue_t *rxq, uint8_t ring_id,
 587                    struct rte_mbuf *mbuf)
 588 {
 589         uint32_t val;
 590         struct vmxnet3_cmd_ring *ring = &rxq->cmd_ring[ring_id];
 591         struct Vmxnet3_RxDesc *rxd =
 592                 (struct Vmxnet3_RxDesc *)(ring->base + ring->next2fill);
 593         vmxnet3_buf_info_t *buf_info = &ring->buf_info[ring->next2fill];
 594
 595         if (ring_id == 0) {
 596                 /* Usually: One HEAD type buf per packet
 597                  * val = (ring->next2fill % rxq->hw->bufs_per_pkt) ?
 598                  * VMXNET3_RXD_BTYPE_BODY : VMXNET3_RXD_BTYPE_HEAD;
 599                  */
 600
 601                 /* We use single packet buffer so all heads here */
 602                 val = VMXNET3_RXD_BTYPE_HEAD;
 603         } else {
 604                 /* All BODY type buffers for 2nd ring */
 605                 val = VMXNET3_RXD_BTYPE_BODY;
 606         }
 607
 608         /*
 609          * Load mbuf pointer into buf_info[ring_size]
 610          * buf_info structure is equivalent to cookie for virtio-virtqueue
 611          */
 612         buf_info->m = mbuf;
 613         buf_info->len = (uint16_t)(mbuf->buf_len - RTE_PKTMBUF_HEADROOM);
 614         buf_info->bufPA = rte_mbuf_data_iova_default(mbuf);
 615
 616         /* Load Rx Descriptor with the buffer's GPA */
 617         rxd->addr = buf_info->bufPA;
 618
 619         /* After this point rxd->addr MUST not be NULL */
 620         rxd->btype = val;
 621         rxd->len = buf_info->len;
 622         /* Flip gen bit at the end to change ownership */
 623         rxd->gen = ring->gen;
 624
 625         vmxnet3_cmd_ring_adv_next2fill(ring);
 626 }
 627 /*
 628  *  Allocates mbufs and clusters. Post rx descriptors with buffer details
 629  *  so that device can receive packets in those buffers.
 630  *  Ring layout:
 631  *      Among the two rings, 1st ring contains buffers of type 0 and type 1.
 632  *      bufs_per_pkt is set such that for non-LRO cases all the buffers required
 633  *      by a frame will fit in 1st ring (1st buf of type0 and rest of type1).
 634  *      2nd ring contains buffers of type 1 alone. Second ring mostly be used
 635  *      only for LRO.
 636  */
 637 static int
 638 vmxnet3_post_rx_bufs(vmxnet3_rx_queue_t *rxq, uint8_t ring_id)
 639 {
 640         int err = 0;
 641         uint32_t i = 0;
 642         struct vmxnet3_cmd_ring *ring = &rxq->cmd_ring[ring_id];
 643
 644         while (vmxnet3_cmd_ring_desc_avail(ring) > 0) {
 645                 struct rte_mbuf *mbuf;
 646
 647                 /* Allocate blank mbuf for the current Rx Descriptor */
 648                 mbuf = rte_mbuf_raw_alloc(rxq->mp);
 649                 if (unlikely(mbuf == NULL)) {
 650                         PMD_RX_LOG(ERR, "Error allocating mbuf");
 651                         rxq->stats.rx_buf_alloc_failure++;
 652                         err = ENOMEM;
 653                         break;
 654                 }
 655
 656                 vmxnet3_renew_desc(rxq, ring_id, mbuf);
 657                 i++;
 658         }
 659
 660         /* Return error only if no buffers are posted at present */
 661         if (vmxnet3_cmd_ring_desc_avail(ring) >= (ring->size - 1))
 662                 return -err;
 663         else
 664                 return i;
 665 }
 666
 667 /* MSS not provided by vmxnet3, guess one with available information */
 668 static uint16_t
 669 vmxnet3_guess_mss(struct vmxnet3_hw *hw, const Vmxnet3_RxCompDesc *rcd,
 670                 struct rte_mbuf *rxm)
 671 {
 672         uint32_t hlen, slen;
 673         struct rte_ipv4_hdr *ipv4_hdr;
 674         struct rte_ipv6_hdr *ipv6_hdr;
 675         struct rte_tcp_hdr *tcp_hdr;
 676         char *ptr;
 677
 678         RTE_ASSERT(rcd->tcp);
 679
 680         ptr = rte_pktmbuf_mtod(rxm, char *);
 681         slen = rte_pktmbuf_data_len(rxm);
 682         hlen = sizeof(struct rte_ether_hdr);
 683
 684         if (rcd->v4) {
 685                 if (unlikely(slen < hlen + sizeof(struct rte_ipv4_hdr)))
 686                         return hw->mtu - sizeof(struct rte_ipv4_hdr)
 687                                         - sizeof(struct rte_tcp_hdr);
 688
 689                 ipv4_hdr = (struct rte_ipv4_hdr *)(ptr + hlen);
 690                 hlen += (ipv4_hdr->version_ihl & RTE_IPV4_HDR_IHL_MASK) *
 691                                 RTE_IPV4_IHL_MULTIPLIER;
 692         } else if (rcd->v6) {
 693                 if (unlikely(slen < hlen + sizeof(struct rte_ipv6_hdr)))
 694                         return hw->mtu - sizeof(struct rte_ipv6_hdr) -
 695                                         sizeof(struct rte_tcp_hdr);
 696
 697                 ipv6_hdr = (struct rte_ipv6_hdr *)(ptr + hlen);
 698                 hlen += sizeof(struct rte_ipv6_hdr);
 699                 if (unlikely(ipv6_hdr->proto != IPPROTO_TCP)) {
 700                         int frag;
 701
 702                         rte_net_skip_ip6_ext(ipv6_hdr->proto, rxm,
 703                                         &hlen, &frag);
 704                 }
 705         }
 706
 707         if (unlikely(slen < hlen + sizeof(struct rte_tcp_hdr)))
 708                 return hw->mtu - hlen - sizeof(struct rte_tcp_hdr) +
 709                                 sizeof(struct rte_ether_hdr);
 710
 711         tcp_hdr = (struct rte_tcp_hdr *)(ptr + hlen);
 712         hlen += (tcp_hdr->data_off & 0xf0) >> 2;
 713
 714         if (rxm->udata64 > 1)
 715                 return (rte_pktmbuf_pkt_len(rxm) - hlen +
 716                                 rxm->udata64 - 1) / rxm->udata64;
 717         else
 718                 return hw->mtu - hlen + sizeof(struct rte_ether_hdr);
 719 }
 720
 721 /* Receive side checksum and other offloads */
 722 static inline void
 723 vmxnet3_rx_offload(struct vmxnet3_hw *hw, const Vmxnet3_RxCompDesc *rcd,
 724                 struct rte_mbuf *rxm, const uint8_t sop)
 725 {
 726         uint64_t ol_flags = rxm->ol_flags;
 727         uint32_t packet_type = rxm->packet_type;
 728
 729         /* Offloads set in sop */
 730         if (sop) {
 731                 /* Set packet type */
 732                 packet_type |= RTE_PTYPE_L2_ETHER;
 733
 734                 /* Check large packet receive */
 735                 if (VMXNET3_VERSION_GE_2(hw) &&
 736                     rcd->type == VMXNET3_CDTYPE_RXCOMP_LRO) {
 737                         const Vmxnet3_RxCompDescExt *rcde =
 738                                         (const Vmxnet3_RxCompDescExt *)rcd;
 739
 740                         rxm->tso_segsz = rcde->mss;
 741                         rxm->udata64 = rcde->segCnt;
 742                         ol_flags |= PKT_RX_LRO;
 743                 }
 744         } else { /* Offloads set in eop */
 745                 /* Check for RSS */
 746                 if (rcd->rssType != VMXNET3_RCD_RSS_TYPE_NONE) {
 747                         ol_flags |= PKT_RX_RSS_HASH;
 748                         rxm->hash.rss = rcd->rssHash;
 749                 }
 750
 751                 /* Check for hardware stripped VLAN tag */
 752                 if (rcd->ts) {
 753                         ol_flags |= (PKT_RX_VLAN | PKT_RX_VLAN_STRIPPED);
 754                         rxm->vlan_tci = rte_le_to_cpu_16((uint16_t)rcd->tci);
 755                 }
 756
 757                 /* Check packet type, checksum errors, etc. */
 758                 if (rcd->cnc) {
 759                         ol_flags |= PKT_RX_L4_CKSUM_UNKNOWN;
 760                 } else {
 761                         if (rcd->v4) {
 762                                 packet_type |= RTE_PTYPE_L3_IPV4_EXT_UNKNOWN;
 763
 764                                 if (rcd->ipc)
 765                                         ol_flags |= PKT_RX_IP_CKSUM_GOOD;
 766                                 else
 767                                         ol_flags |= PKT_RX_IP_CKSUM_BAD;
 768
 769                                 if (rcd->tuc) {
 770                                         ol_flags |= PKT_RX_L4_CKSUM_GOOD;
 771                                         if (rcd->tcp)
 772                                                 packet_type |= RTE_PTYPE_L4_TCP;
 773                                         else
 774                                                 packet_type |= RTE_PTYPE_L4_UDP;
 775                                 } else {
 776                                         if (rcd->tcp) {
 777                                                 packet_type |= RTE_PTYPE_L4_TCP;
 778                                                 ol_flags |= PKT_RX_L4_CKSUM_BAD;
 779                                         } else if (rcd->udp) {
 780                                                 packet_type |= RTE_PTYPE_L4_UDP;
 781                                                 ol_flags |= PKT_RX_L4_CKSUM_BAD;
 782                                         }
 783                                 }
 784                         } else if (rcd->v6) {
 785                                 packet_type |= RTE_PTYPE_L3_IPV6_EXT_UNKNOWN;
 786
 787                                 if (rcd->tuc) {
 788                                         ol_flags |= PKT_RX_L4_CKSUM_GOOD;
 789                                         if (rcd->tcp)
 790                                                 packet_type |= RTE_PTYPE_L4_TCP;
 791                                         else
 792                                                 packet_type |= RTE_PTYPE_L4_UDP;
 793                                 } else {
 794                                         if (rcd->tcp) {
 795                                                 packet_type |= RTE_PTYPE_L4_TCP;
 796                                                 ol_flags |= PKT_RX_L4_CKSUM_BAD;
 797                                         } else if (rcd->udp) {
 798                                                 packet_type |= RTE_PTYPE_L4_UDP;
 799                                                 ol_flags |= PKT_RX_L4_CKSUM_BAD;
 800                                         }
 801                                 }
 802                         } else {
 803                                 packet_type |= RTE_PTYPE_UNKNOWN;
 804                         }
 805
 806                         /* Old variants of vmxnet3 do not provide MSS */
 807                         if ((ol_flags & PKT_RX_LRO) && rxm->tso_segsz == 0)
 808                                 rxm->tso_segsz = vmxnet3_guess_mss(hw,
 809                                                 rcd, rxm);
 810                 }
 811         }
 812
 813         rxm->ol_flags = ol_flags;
 814         rxm->packet_type = packet_type;
 815 }
 816
 817 /*
 818  * Process the Rx Completion Ring of given vmxnet3_rx_queue
 819  * for nb_pkts burst and return the number of packets received
 820  */
 821 uint16_t
 822 vmxnet3_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts, uint16_t nb_pkts)
 823 {
 824         uint16_t nb_rx;
 825         uint32_t nb_rxd, idx;
 826         uint8_t ring_idx;
 827         vmxnet3_rx_queue_t *rxq;
 828         Vmxnet3_RxCompDesc *rcd;
 829         vmxnet3_buf_info_t *rbi;
 830         Vmxnet3_RxDesc *rxd;
 831         struct rte_mbuf *rxm = NULL;
 832         struct vmxnet3_hw *hw;
 833
 834         nb_rx = 0;
 835         ring_idx = 0;
 836         nb_rxd = 0;
 837         idx = 0;
 838
 839         rxq = rx_queue;
 840         hw = rxq->hw;
 841
 842         rcd = &rxq->comp_ring.base[rxq->comp_ring.next2proc].rcd;
 843
 844         if (unlikely(rxq->stopped)) {
 845                 PMD_RX_LOG(DEBUG, "Rx queue is stopped.");
 846                 return 0;
 847         }
 848
 849         while (rcd->gen == rxq->comp_ring.gen) {
 850                 struct rte_mbuf *newm;
 851
 852                 if (nb_rx >= nb_pkts)
 853                         break;
 854
 855                 newm = rte_mbuf_raw_alloc(rxq->mp);
 856                 if (unlikely(newm == NULL)) {
 857                         PMD_RX_LOG(ERR, "Error allocating mbuf");
 858                         rxq->stats.rx_buf_alloc_failure++;
 859                         break;
 860                 }
 861
 862                 idx = rcd->rxdIdx;
 863                 ring_idx = vmxnet3_get_ring_idx(hw, rcd->rqID);
 864                 rxd = (Vmxnet3_RxDesc *)rxq->cmd_ring[ring_idx].base + idx;
 865                 RTE_SET_USED(rxd); /* used only for assert when enabled */
 866                 rbi = rxq->cmd_ring[ring_idx].buf_info + idx;
 867
 868                 PMD_RX_LOG(DEBUG, "rxd idx: %d ring idx: %d.", idx, ring_idx);
 869
 870                 RTE_ASSERT(rcd->len <= rxd->len);
 871                 RTE_ASSERT(rbi->m);
 872
 873                 /* Get the packet buffer pointer from buf_info */
 874                 rxm = rbi->m;
 875
 876                 /* Clear descriptor associated buf_info to be reused */
 877                 rbi->m = NULL;
 878                 rbi->bufPA = 0;
 879
 880                 /* Update the index that we received a packet */
 881                 rxq->cmd_ring[ring_idx].next2comp = idx;
 882
 883                 /* For RCD with EOP set, check if there is frame error */
 884                 if (unlikely(rcd->eop && rcd->err)) {
 885                         rxq->stats.drop_total++;
 886                         rxq->stats.drop_err++;
 887
 888                         if (!rcd->fcs) {
 889                                 rxq->stats.drop_fcs++;
 890                                 PMD_RX_LOG(ERR, "Recv packet dropped due to frame err.");
 891                         }
 892                         PMD_RX_LOG(ERR, "Error in received packet rcd#:%d rxd:%d",
 893                                    (int)(rcd - (struct Vmxnet3_RxCompDesc *)
 894                                          rxq->comp_ring.base), rcd->rxdIdx);
 895                         rte_pktmbuf_free_seg(rxm);
 896                         if (rxq->start_seg) {
 897                                 struct rte_mbuf *start = rxq->start_seg;
 898
 899                                 rxq->start_seg = NULL;
 900                                 rte_pktmbuf_free(start);
 901                         }
 902                         goto rcd_done;
 903                 }
 904
 905                 /* Initialize newly received packet buffer */
 906                 rxm->port = rxq->port_id;
 907                 rxm->nb_segs = 1;
 908                 rxm->next = NULL;
 909                 rxm->pkt_len = (uint16_t)rcd->len;
 910                 rxm->data_len = (uint16_t)rcd->len;
 911                 rxm->data_off = RTE_PKTMBUF_HEADROOM;
 912                 rxm->ol_flags = 0;
 913                 rxm->vlan_tci = 0;
 914                 rxm->packet_type = 0;
 915
 916                 /*
 917                  * If this is the first buffer of the received packet,
 918                  * set the pointer to the first mbuf of the packet
 919                  * Otherwise, update the total length and the number of segments
 920                  * of the current scattered packet, and update the pointer to
 921                  * the last mbuf of the current packet.
 922                  */
 923                 if (rcd->sop) {
 924                         RTE_ASSERT(rxd->btype == VMXNET3_RXD_BTYPE_HEAD);
 925
 926                         if (unlikely(rcd->len == 0)) {
 927                                 RTE_ASSERT(rcd->eop);
 928
 929                                 PMD_RX_LOG(DEBUG,
 930                                            "Rx buf was skipped. rxring[%d][%d])",
 931                                            ring_idx, idx);
 932                                 rte_pktmbuf_free_seg(rxm);
 933                                 goto rcd_done;
 934                         }
 935
 936                         if (vmxnet3_rx_data_ring(hw, rcd->rqID)) {
 937                                 uint8_t *rdd = rxq->data_ring.base +
 938                                         idx * rxq->data_desc_size;
 939
 940                                 RTE_ASSERT(VMXNET3_VERSION_GE_3(hw));
 941                                 rte_memcpy(rte_pktmbuf_mtod(rxm, char *),
 942                                            rdd, rcd->len);
 943                         }
 944
 945                         rxq->start_seg = rxm;
 946                         rxq->last_seg = rxm;
 947                         vmxnet3_rx_offload(hw, rcd, rxm, 1);
 948                 } else {
 949                         struct rte_mbuf *start = rxq->start_seg;
 950
 951                         RTE_ASSERT(rxd->btype == VMXNET3_RXD_BTYPE_BODY);
 952
 953                         if (rxm->data_len) {
 954                                 start->pkt_len += rxm->data_len;
 955                                 start->nb_segs++;
 956
 957                                 rxq->last_seg->next = rxm;
 958                                 rxq->last_seg = rxm;
 959                         } else {
 960                                 rte_pktmbuf_free_seg(rxm);
 961                         }
 962                 }
 963
 964                 if (rcd->eop) {
 965                         struct rte_mbuf *start = rxq->start_seg;
 966
 967                         vmxnet3_rx_offload(hw, rcd, start, 0);
 968                         rx_pkts[nb_rx++] = start;
 969                         rxq->start_seg = NULL;
 970                 }
 971
 972 rcd_done:
 973                 rxq->cmd_ring[ring_idx].next2comp = idx;
 974                 VMXNET3_INC_RING_IDX_ONLY(rxq->cmd_ring[ring_idx].next2comp,
 975                                           rxq->cmd_ring[ring_idx].size);
 976
 977                 /* It's time to renew descriptors */
 978                 vmxnet3_renew_desc(rxq, ring_idx, newm);
 979                 if (unlikely(rxq->shared->ctrl.updateRxProd)) {
 980                         VMXNET3_WRITE_BAR0_REG(hw, rxprod_reg[ring_idx] + (rxq->queue_id * VMXNET3_REG_ALIGN),
 981                                                rxq->cmd_ring[ring_idx].next2fill);
 982                 }
 983
 984                 /* Advance to the next descriptor in comp_ring */
 985                 vmxnet3_comp_ring_adv_next2proc(&rxq->comp_ring);
 986
 987                 rcd = &rxq->comp_ring.base[rxq->comp_ring.next2proc].rcd;
 988                 nb_rxd++;
 989                 if (nb_rxd > rxq->cmd_ring[0].size) {
 990                         PMD_RX_LOG(ERR, "Used up quota of receiving packets,"
 991                                    " relinquish control.");
 992                         break;
 993                 }
 994         }
 995
 996         if (unlikely(nb_rxd == 0)) {
 997                 uint32_t avail;
 998                 for (ring_idx = 0; ring_idx < VMXNET3_RX_CMDRING_SIZE; ring_idx++) {
 999                         avail = vmxnet3_cmd_ring_desc_avail(&rxq->cmd_ring[ring_idx]);
1000                         if (unlikely(avail > 0)) {
1001                                 /* try to alloc new buf and renew descriptors */
1002                                 vmxnet3_post_rx_bufs(rxq, ring_idx);
1003                         }
1004                 }
1005                 if (unlikely(rxq->shared->ctrl.updateRxProd)) {
1006                         for (ring_idx = 0; ring_idx < VMXNET3_RX_CMDRING_SIZE; ring_idx++) {
1007                                 VMXNET3_WRITE_BAR0_REG(hw, rxprod_reg[ring_idx] + (rxq->queue_id * VMXNET3_REG_ALIGN),
1008                                                        rxq->cmd_ring[ring_idx].next2fill);
1009                         }
1010                 }
1011         }
1012
1013         return nb_rx;
1014 }
1015
1016 int
1017 vmxnet3_dev_tx_queue_setup(struct rte_eth_dev *dev,
1018                            uint16_t queue_idx,
1019                            uint16_t nb_desc,
1020                            unsigned int socket_id,
1021                            const struct rte_eth_txconf *tx_conf __rte_unused)
1022 {
1023         struct vmxnet3_hw *hw = dev->data->dev_private;
1024         const struct rte_memzone *mz;
1025         struct vmxnet3_tx_queue *txq;
1026         struct vmxnet3_cmd_ring *ring;
1027         struct vmxnet3_comp_ring *comp_ring;
1028         struct vmxnet3_data_ring *data_ring;
1029         int size;
1030
1031         PMD_INIT_FUNC_TRACE();
1032
1033         txq = rte_zmalloc("ethdev_tx_queue", sizeof(struct vmxnet3_tx_queue),
1034                           RTE_CACHE_LINE_SIZE);
1035         if (txq == NULL) {
1036                 PMD_INIT_LOG(ERR, "Can not allocate tx queue structure");
1037                 return -ENOMEM;
1038         }
1039
1040         txq->queue_id = queue_idx;
1041         txq->port_id = dev->data->port_id;
1042         txq->shared = NULL; /* set in vmxnet3_setup_driver_shared() */
1043         txq->hw = hw;
1044         txq->qid = queue_idx;
1045         txq->stopped = TRUE;
1046         txq->txdata_desc_size = hw->txdata_desc_size;
1047
1048         ring = &txq->cmd_ring;
1049         comp_ring = &txq->comp_ring;
1050         data_ring = &txq->data_ring;
1051
1052         /* Tx vmxnet ring length should be between 512-4096 */
1053         if (nb_desc < VMXNET3_DEF_TX_RING_SIZE) {
1054                 PMD_INIT_LOG(ERR, "VMXNET3 Tx Ring Size Min: %u",
1055                              VMXNET3_DEF_TX_RING_SIZE);
1056                 return -EINVAL;
1057         } else if (nb_desc > VMXNET3_TX_RING_MAX_SIZE) {
1058                 PMD_INIT_LOG(ERR, "VMXNET3 Tx Ring Size Max: %u",
1059                              VMXNET3_TX_RING_MAX_SIZE);
1060                 return -EINVAL;
1061         } else {
1062                 ring->size = nb_desc;
1063                 ring->size &= ~VMXNET3_RING_SIZE_MASK;
1064         }
1065         comp_ring->size = data_ring->size = ring->size;
1066
1067         /* Tx vmxnet rings structure initialization*/
1068         ring->next2fill = 0;
1069         ring->next2comp = 0;
1070         ring->gen = VMXNET3_INIT_GEN;
1071         comp_ring->next2proc = 0;
1072         comp_ring->gen = VMXNET3_INIT_GEN;
1073
1074         size = sizeof(struct Vmxnet3_TxDesc) * ring->size;
1075         size += sizeof(struct Vmxnet3_TxCompDesc) * comp_ring->size;
1076         size += txq->txdata_desc_size * data_ring->size;
1077
1078         mz = rte_eth_dma_zone_reserve(dev, "txdesc", queue_idx, size,
1079                                       VMXNET3_RING_BA_ALIGN, socket_id);
1080         if (mz == NULL) {
1081                 PMD_INIT_LOG(ERR, "ERROR: Creating queue descriptors zone");
1082                 return -ENOMEM;
1083         }
1084         txq->mz = mz;
1085         memset(mz->addr, 0, mz->len);
1086
1087         /* cmd_ring initialization */
1088         ring->base = mz->addr;
1089         ring->basePA = mz->iova;
1090
1091         /* comp_ring initialization */
1092         comp_ring->base = ring->base + ring->size;
1093         comp_ring->basePA = ring->basePA +
1094                 (sizeof(struct Vmxnet3_TxDesc) * ring->size);
1095
1096         /* data_ring initialization */
1097         data_ring->base = (Vmxnet3_TxDataDesc *)(comp_ring->base + comp_ring->size);
1098         data_ring->basePA = comp_ring->basePA +
1099                         (sizeof(struct Vmxnet3_TxCompDesc) * comp_ring->size);
1100
1101         /* cmd_ring0 buf_info allocation */
1102         ring->buf_info = rte_zmalloc("tx_ring_buf_info",
1103                                      ring->size * sizeof(vmxnet3_buf_info_t), RTE_CACHE_LINE_SIZE);
1104         if (ring->buf_info == NULL) {
1105                 PMD_INIT_LOG(ERR, "ERROR: Creating tx_buf_info structure");
1106                 return -ENOMEM;
1107         }
1108
1109         /* Update the data portion with txq */
1110         dev->data->tx_queues[queue_idx] = txq;
1111
1112         return 0;
1113 }
1114
1115 int
1116 vmxnet3_dev_rx_queue_setup(struct rte_eth_dev *dev,
1117                            uint16_t queue_idx,
1118                            uint16_t nb_desc,
1119                            unsigned int socket_id,
1120                            __rte_unused const struct rte_eth_rxconf *rx_conf,
1121                            struct rte_mempool *mp)
1122 {
1123         const struct rte_memzone *mz;
1124         struct vmxnet3_rx_queue *rxq;
1125         struct vmxnet3_hw *hw = dev->data->dev_private;
1126         struct vmxnet3_cmd_ring *ring0, *ring1, *ring;
1127         struct vmxnet3_comp_ring *comp_ring;
1128         struct vmxnet3_rx_data_ring *data_ring;
1129         int size;
1130         uint8_t i;
1131         char mem_name[32];
1132
1133         PMD_INIT_FUNC_TRACE();
1134
1135         rxq = rte_zmalloc("ethdev_rx_queue", sizeof(struct vmxnet3_rx_queue),
1136                           RTE_CACHE_LINE_SIZE);
1137         if (rxq == NULL) {
1138                 PMD_INIT_LOG(ERR, "Can not allocate rx queue structure");
1139                 return -ENOMEM;
1140         }
1141
1142         rxq->mp = mp;
1143         rxq->queue_id = queue_idx;
1144         rxq->port_id = dev->data->port_id;
1145         rxq->shared = NULL; /* set in vmxnet3_setup_driver_shared() */
1146         rxq->hw = hw;
1147         rxq->qid1 = queue_idx;
1148         rxq->qid2 = queue_idx + hw->num_rx_queues;
1149         rxq->data_ring_qid = queue_idx + 2 * hw->num_rx_queues;
1150         rxq->data_desc_size = hw->rxdata_desc_size;
1151         rxq->stopped = TRUE;
1152
1153         ring0 = &rxq->cmd_ring[0];
1154         ring1 = &rxq->cmd_ring[1];
1155         comp_ring = &rxq->comp_ring;
1156         data_ring = &rxq->data_ring;
1157
1158         /* Rx vmxnet rings length should be between 256-4096 */
1159         if (nb_desc < VMXNET3_DEF_RX_RING_SIZE) {
1160                 PMD_INIT_LOG(ERR, "VMXNET3 Rx Ring Size Min: 256");
1161                 return -EINVAL;
1162         } else if (nb_desc > VMXNET3_RX_RING_MAX_SIZE) {
1163                 PMD_INIT_LOG(ERR, "VMXNET3 Rx Ring Size Max: 4096");
1164                 return -EINVAL;
1165         } else {
1166                 ring0->size = nb_desc;
1167                 ring0->size &= ~VMXNET3_RING_SIZE_MASK;
1168                 ring1->size = ring0->size;
1169         }
1170
1171         comp_ring->size = ring0->size + ring1->size;
1172         data_ring->size = ring0->size;
1173
1174         /* Rx vmxnet rings structure initialization */
1175         ring0->next2fill = 0;
1176         ring1->next2fill = 0;
1177         ring0->next2comp = 0;
1178         ring1->next2comp = 0;
1179         ring0->gen = VMXNET3_INIT_GEN;
1180         ring1->gen = VMXNET3_INIT_GEN;
1181         comp_ring->next2proc = 0;
1182         comp_ring->gen = VMXNET3_INIT_GEN;
1183
1184         size = sizeof(struct Vmxnet3_RxDesc) * (ring0->size + ring1->size);
1185         size += sizeof(struct Vmxnet3_RxCompDesc) * comp_ring->size;
1186         if (VMXNET3_VERSION_GE_3(hw) && rxq->data_desc_size)
1187                 size += rxq->data_desc_size * data_ring->size;
1188
1189         mz = rte_eth_dma_zone_reserve(dev, "rxdesc", queue_idx, size,
1190                                       VMXNET3_RING_BA_ALIGN, socket_id);
1191         if (mz == NULL) {
1192                 PMD_INIT_LOG(ERR, "ERROR: Creating queue descriptors zone");
1193                 return -ENOMEM;
1194         }
1195         rxq->mz = mz;
1196         memset(mz->addr, 0, mz->len);
1197
1198         /* cmd_ring0 initialization */
1199         ring0->base = mz->addr;
1200         ring0->basePA = mz->iova;
1201
1202         /* cmd_ring1 initialization */
1203         ring1->base = ring0->base + ring0->size;
1204         ring1->basePA = ring0->basePA + sizeof(struct Vmxnet3_RxDesc) * ring0->size;
1205
1206         /* comp_ring initialization */
1207         comp_ring->base = ring1->base + ring1->size;
1208         comp_ring->basePA = ring1->basePA + sizeof(struct Vmxnet3_RxDesc) *
1209                 ring1->size;
1210
1211         /* data_ring initialization */
1212         if (VMXNET3_VERSION_GE_3(hw) && rxq->data_desc_size) {
1213                 data_ring->base =
1214                         (uint8_t *)(comp_ring->base + comp_ring->size);
1215                 data_ring->basePA = comp_ring->basePA +
1216                         sizeof(struct Vmxnet3_RxCompDesc) * comp_ring->size;
1217         }
1218
1219         /* cmd_ring0-cmd_ring1 buf_info allocation */
1220         for (i = 0; i < VMXNET3_RX_CMDRING_SIZE; i++) {
1221
1222                 ring = &rxq->cmd_ring[i];
1223                 ring->rid = i;
1224                 snprintf(mem_name, sizeof(mem_name), "rx_ring_%d_buf_info", i);
1225
1226                 ring->buf_info = rte_zmalloc(mem_name,
1227                                              ring->size * sizeof(vmxnet3_buf_info_t),
1228                                              RTE_CACHE_LINE_SIZE);
1229                 if (ring->buf_info == NULL) {
1230                         PMD_INIT_LOG(ERR, "ERROR: Creating rx_buf_info structure");
1231                         return -ENOMEM;
1232                 }
1233         }
1234
1235         /* Update the data portion with rxq */
1236         dev->data->rx_queues[queue_idx] = rxq;
1237
1238         return 0;
1239 }
1240
1241 /*
1242  * Initializes Receive Unit
1243  * Load mbufs in rx queue in advance
1244  */
1245 int
1246 vmxnet3_dev_rxtx_init(struct rte_eth_dev *dev)
1247 {
1248         struct vmxnet3_hw *hw = dev->data->dev_private;
1249
1250         int i, ret;
1251         uint8_t j;
1252
1253         PMD_INIT_FUNC_TRACE();
1254
1255         for (i = 0; i < hw->num_rx_queues; i++) {
1256                 vmxnet3_rx_queue_t *rxq = dev->data->rx_queues[i];
1257
1258                 for (j = 0; j < VMXNET3_RX_CMDRING_SIZE; j++) {
1259                         /* Passing 0 as alloc_num will allocate full ring */
1260                         ret = vmxnet3_post_rx_bufs(rxq, j);
1261                         if (ret <= 0) {
1262                                 PMD_INIT_LOG(ERR,
1263                                              "ERROR: Posting Rxq: %d buffers ring: %d",
1264                                              i, j);
1265                                 return -ret;
1266                         }
1267                         /*
1268                          * Updating device with the index:next2fill to fill the
1269                          * mbufs for coming packets.
1270                          */
1271                         if (unlikely(rxq->shared->ctrl.updateRxProd)) {
1272                                 VMXNET3_WRITE_BAR0_REG(hw, rxprod_reg[j] + (rxq->queue_id * VMXNET3_REG_ALIGN),
1273                                                        rxq->cmd_ring[j].next2fill);
1274                         }
1275                 }
1276                 rxq->stopped = FALSE;
1277                 rxq->start_seg = NULL;
1278         }
1279
1280         for (i = 0; i < dev->data->nb_tx_queues; i++) {
1281                 struct vmxnet3_tx_queue *txq = dev->data->tx_queues[i];
1282
1283                 txq->stopped = FALSE;
1284         }
1285
1286         return 0;
1287 }
1288
1289 static uint8_t rss_intel_key[40] = {
1290         0x6D, 0x5A, 0x56, 0xDA, 0x25, 0x5B, 0x0E, 0xC2,
1291         0x41, 0x67, 0x25, 0x3D, 0x43, 0xA3, 0x8F, 0xB0,
1292         0xD0, 0xCA, 0x2B, 0xCB, 0xAE, 0x7B, 0x30, 0xB4,
1293         0x77, 0xCB, 0x2D, 0xA3, 0x80, 0x30, 0xF2, 0x0C,
1294         0x6A, 0x42, 0xB7, 0x3B, 0xBE, 0xAC, 0x01, 0xFA,
1295 };
1296
1297 /*
1298  * Additional RSS configurations based on vmxnet v4+ APIs
1299  */
1300 int
1301 vmxnet3_v4_rss_configure(struct rte_eth_dev *dev)
1302 {
1303         struct vmxnet3_hw *hw = dev->data->dev_private;
1304         Vmxnet3_DriverShared *shared = hw->shared;
1305         Vmxnet3_CmdInfo *cmdInfo = &shared->cu.cmdInfo;
1306         struct rte_eth_rss_conf *port_rss_conf;
1307         uint64_t rss_hf;
1308         uint32_t ret;
1309
1310         PMD_INIT_FUNC_TRACE();
1311
1312         cmdInfo->setRSSFields = 0;
1313         port_rss_conf = &dev->data->dev_conf.rx_adv_conf.rss_conf;
1314         rss_hf = port_rss_conf->rss_hf &
1315                 (VMXNET3_V4_RSS_MASK | VMXNET3_RSS_OFFLOAD_ALL);
1316
1317         if (rss_hf & ETH_RSS_NONFRAG_IPV4_TCP)
1318                 cmdInfo->setRSSFields |= VMXNET3_RSS_FIELDS_TCPIP4;
1319         if (rss_hf & ETH_RSS_NONFRAG_IPV6_TCP)
1320                 cmdInfo->setRSSFields |= VMXNET3_RSS_FIELDS_TCPIP6;
1321         if (rss_hf & ETH_RSS_NONFRAG_IPV4_UDP)
1322                 cmdInfo->setRSSFields |= VMXNET3_RSS_FIELDS_UDPIP4;
1323         if (rss_hf & ETH_RSS_NONFRAG_IPV6_UDP)
1324                 cmdInfo->setRSSFields |= VMXNET3_RSS_FIELDS_UDPIP6;
1325
1326         VMXNET3_WRITE_BAR1_REG(hw, VMXNET3_REG_CMD,
1327                                VMXNET3_CMD_SET_RSS_FIELDS);
1328         ret = VMXNET3_READ_BAR1_REG(hw, VMXNET3_REG_CMD);
1329
1330         if (ret != VMXNET3_SUCCESS) {
1331                 PMD_DRV_LOG(ERR, "Set RSS fields (v4) failed: %d", ret);
1332         }
1333
1334         return ret;
1335 }
1336
1337 /*
1338  * Configure RSS feature
1339  */
1340 int
1341 vmxnet3_rss_configure(struct rte_eth_dev *dev)
1342 {
1343         struct vmxnet3_hw *hw = dev->data->dev_private;
1344         struct VMXNET3_RSSConf *dev_rss_conf;
1345         struct rte_eth_rss_conf *port_rss_conf;
1346         uint64_t rss_hf;
1347         uint8_t i, j;
1348
1349         PMD_INIT_FUNC_TRACE();
1350
1351         dev_rss_conf = hw->rss_conf;
1352         port_rss_conf = &dev->data->dev_conf.rx_adv_conf.rss_conf;
1353
1354         /* loading hashFunc */
1355         dev_rss_conf->hashFunc = VMXNET3_RSS_HASH_FUNC_TOEPLITZ;
1356         /* loading hashKeySize */
1357         dev_rss_conf->hashKeySize = VMXNET3_RSS_MAX_KEY_SIZE;
1358         /* loading indTableSize: Must not exceed VMXNET3_RSS_MAX_IND_TABLE_SIZE (128)*/
1359         dev_rss_conf->indTableSize = (uint16_t)(hw->num_rx_queues * 4);
1360
1361         if (port_rss_conf->rss_key == NULL) {
1362                 /* Default hash key */
1363                 port_rss_conf->rss_key = rss_intel_key;
1364         }
1365
1366         /* loading hashKey */
1367         memcpy(&dev_rss_conf->hashKey[0], port_rss_conf->rss_key,
1368                dev_rss_conf->hashKeySize);
1369
1370         /* loading indTable */
1371         for (i = 0, j = 0; i < dev_rss_conf->indTableSize; i++, j++) {
1372                 if (j == dev->data->nb_rx_queues)
1373                         j = 0;
1374                 dev_rss_conf->indTable[i] = j;
1375         }
1376
1377         /* loading hashType */
1378         dev_rss_conf->hashType = 0;
1379         rss_hf = port_rss_conf->rss_hf & VMXNET3_RSS_OFFLOAD_ALL;
1380         if (rss_hf & ETH_RSS_IPV4)
1381                 dev_rss_conf->hashType |= VMXNET3_RSS_HASH_TYPE_IPV4;
1382         if (rss_hf & ETH_RSS_NONFRAG_IPV4_TCP)
1383                 dev_rss_conf->hashType |= VMXNET3_RSS_HASH_TYPE_TCP_IPV4;
1384         if (rss_hf & ETH_RSS_IPV6)
1385                 dev_rss_conf->hashType |= VMXNET3_RSS_HASH_TYPE_IPV6;
1386         if (rss_hf & ETH_RSS_NONFRAG_IPV6_TCP)
1387                 dev_rss_conf->hashType |= VMXNET3_RSS_HASH_TYPE_TCP_IPV6;
1388
1389         return VMXNET3_SUCCESS;
1390 }