drivers/net/vmxnet3/vmxnet3_rxtx.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(c) 2010-2015 Intel Corporation
   3  */
   4
   5 #include <sys/queue.h>
   6
   7 #include <stdio.h>
   8 #include <stdlib.h>
   9 #include <string.h>
  10 #include <errno.h>
  11 #include <stdint.h>
  12 #include <stdarg.h>
  13 #include <unistd.h>
  14 #include <inttypes.h>
  15
  16 #include <rte_byteorder.h>
  17 #include <rte_common.h>
  18 #include <rte_cycles.h>
  19 #include <rte_log.h>
  20 #include <rte_debug.h>
  21 #include <rte_interrupts.h>
  22 #include <rte_pci.h>
  23 #include <rte_memory.h>
  24 #include <rte_memzone.h>
  25 #include <rte_launch.h>
  26 #include <rte_eal.h>
  27 #include <rte_per_lcore.h>
  28 #include <rte_lcore.h>
  29 #include <rte_atomic.h>
  30 #include <rte_branch_prediction.h>
  31 #include <rte_mempool.h>
  32 #include <rte_malloc.h>
  33 #include <rte_mbuf.h>
  34 #include <rte_ether.h>
  35 #include <ethdev_driver.h>
  36 #include <rte_prefetch.h>
  37 #include <rte_ip.h>
  38 #include <rte_udp.h>
  39 #include <rte_tcp.h>
  40 #include <rte_sctp.h>
  41 #include <rte_string_fns.h>
  42 #include <rte_errno.h>
  43 #include <rte_net.h>
  44
  45 #include "base/vmxnet3_defs.h"
  46 #include "vmxnet3_ring.h"
  47
  48 #include "vmxnet3_logs.h"
  49 #include "vmxnet3_ethdev.h"
  50
  51 #define VMXNET3_TX_OFFLOAD_MASK ( \
  52                 PKT_TX_VLAN_PKT | \
  53                 PKT_TX_IPV6 |     \
  54                 PKT_TX_IPV4 |     \
  55                 PKT_TX_L4_MASK |  \
  56                 PKT_TX_TCP_SEG)
  57
  58 #define VMXNET3_TX_OFFLOAD_NOTSUP_MASK  \
  59         (PKT_TX_OFFLOAD_MASK ^ VMXNET3_TX_OFFLOAD_MASK)
  60
  61 static const uint32_t rxprod_reg[2] = {VMXNET3_REG_RXPROD, VMXNET3_REG_RXPROD2};
  62
  63 static int vmxnet3_post_rx_bufs(vmxnet3_rx_queue_t*, uint8_t);
  64 static void vmxnet3_tq_tx_complete(vmxnet3_tx_queue_t *);
  65 #ifdef RTE_LIBRTE_VMXNET3_DEBUG_DRIVER_NOT_USED
  66 static void vmxnet3_rxq_dump(struct vmxnet3_rx_queue *);
  67 static void vmxnet3_txq_dump(struct vmxnet3_tx_queue *);
  68 #endif
  69
  70 #ifdef RTE_LIBRTE_VMXNET3_DEBUG_DRIVER_NOT_USED
  71 static void
  72 vmxnet3_rxq_dump(struct vmxnet3_rx_queue *rxq)
  73 {
  74         uint32_t avail = 0;
  75
  76         if (rxq == NULL)
  77                 return;
  78
  79         PMD_RX_LOG(DEBUG,
  80                    "RXQ: cmd0 base : %p cmd1 base : %p comp ring base : %p.",
  81                    rxq->cmd_ring[0].base, rxq->cmd_ring[1].base, rxq->comp_ring.base);
  82         PMD_RX_LOG(DEBUG,
  83                    "RXQ: cmd0 basePA : 0x%lx cmd1 basePA : 0x%lx comp ring basePA : 0x%lx.",
  84                    (unsigned long)rxq->cmd_ring[0].basePA,
  85                    (unsigned long)rxq->cmd_ring[1].basePA,
  86                    (unsigned long)rxq->comp_ring.basePA);
  87
  88         avail = vmxnet3_cmd_ring_desc_avail(&rxq->cmd_ring[0]);
  89         PMD_RX_LOG(DEBUG,
  90                    "RXQ:cmd0: size=%u; free=%u; next2proc=%u; queued=%u",
  91                    (uint32_t)rxq->cmd_ring[0].size, avail,
  92                    rxq->comp_ring.next2proc,
  93                    rxq->cmd_ring[0].size - avail);
  94
  95         avail = vmxnet3_cmd_ring_desc_avail(&rxq->cmd_ring[1]);
  96         PMD_RX_LOG(DEBUG, "RXQ:cmd1 size=%u; free=%u; next2proc=%u; queued=%u",
  97                    (uint32_t)rxq->cmd_ring[1].size, avail, rxq->comp_ring.next2proc,
  98                    rxq->cmd_ring[1].size - avail);
  99
 100 }
 101
 102 static void
 103 vmxnet3_txq_dump(struct vmxnet3_tx_queue *txq)
 104 {
 105         uint32_t avail = 0;
 106
 107         if (txq == NULL)
 108                 return;
 109
 110         PMD_TX_LOG(DEBUG, "TXQ: cmd base : %p comp ring base : %p data ring base : %p.",
 111                    txq->cmd_ring.base, txq->comp_ring.base, txq->data_ring.base);
 112         PMD_TX_LOG(DEBUG, "TXQ: cmd basePA : 0x%lx comp ring basePA : 0x%lx data ring basePA : 0x%lx.",
 113                    (unsigned long)txq->cmd_ring.basePA,
 114                    (unsigned long)txq->comp_ring.basePA,
 115                    (unsigned long)txq->data_ring.basePA);
 116
 117         avail = vmxnet3_cmd_ring_desc_avail(&txq->cmd_ring);
 118         PMD_TX_LOG(DEBUG, "TXQ: size=%u; free=%u; next2proc=%u; queued=%u",
 119                    (uint32_t)txq->cmd_ring.size, avail,
 120                    txq->comp_ring.next2proc, txq->cmd_ring.size - avail);
 121 }
 122 #endif
 123
 124 static void
 125 vmxnet3_tx_cmd_ring_release_mbufs(vmxnet3_cmd_ring_t *ring)
 126 {
 127         while (ring->next2comp != ring->next2fill) {
 128                 /* No need to worry about desc ownership, device is quiesced by now. */
 129                 vmxnet3_buf_info_t *buf_info = ring->buf_info + ring->next2comp;
 130
 131                 if (buf_info->m) {
 132                         rte_pktmbuf_free(buf_info->m);
 133                         buf_info->m = NULL;
 134                         buf_info->bufPA = 0;
 135                         buf_info->len = 0;
 136                 }
 137                 vmxnet3_cmd_ring_adv_next2comp(ring);
 138         }
 139 }
 140
 141 static void
 142 vmxnet3_rx_cmd_ring_release_mbufs(vmxnet3_cmd_ring_t *ring)
 143 {
 144         uint32_t i;
 145
 146         for (i = 0; i < ring->size; i++) {
 147                 /* No need to worry about desc ownership, device is quiesced by now. */
 148                 vmxnet3_buf_info_t *buf_info = &ring->buf_info[i];
 149
 150                 if (buf_info->m) {
 151                         rte_pktmbuf_free_seg(buf_info->m);
 152                         buf_info->m = NULL;
 153                         buf_info->bufPA = 0;
 154                         buf_info->len = 0;
 155                 }
 156                 vmxnet3_cmd_ring_adv_next2comp(ring);
 157         }
 158 }
 159
 160 static void
 161 vmxnet3_cmd_ring_release(vmxnet3_cmd_ring_t *ring)
 162 {
 163         rte_free(ring->buf_info);
 164         ring->buf_info = NULL;
 165 }
 166
 167 void
 168 vmxnet3_dev_tx_queue_release(void *txq)
 169 {
 170         vmxnet3_tx_queue_t *tq = txq;
 171
 172         if (tq != NULL) {
 173                 /* Release mbufs */
 174                 vmxnet3_tx_cmd_ring_release_mbufs(&tq->cmd_ring);
 175                 /* Release the cmd_ring */
 176                 vmxnet3_cmd_ring_release(&tq->cmd_ring);
 177                 /* Release the memzone */
 178                 rte_memzone_free(tq->mz);
 179                 /* Release the queue */
 180                 rte_free(tq);
 181         }
 182 }
 183
 184 void
 185 vmxnet3_dev_rx_queue_release(void *rxq)
 186 {
 187         int i;
 188         vmxnet3_rx_queue_t *rq = rxq;
 189
 190         if (rq != NULL) {
 191                 /* Release mbufs */
 192                 for (i = 0; i < VMXNET3_RX_CMDRING_SIZE; i++)
 193                         vmxnet3_rx_cmd_ring_release_mbufs(&rq->cmd_ring[i]);
 194
 195                 /* Release both the cmd_rings */
 196                 for (i = 0; i < VMXNET3_RX_CMDRING_SIZE; i++)
 197                         vmxnet3_cmd_ring_release(&rq->cmd_ring[i]);
 198
 199                 /* Release the memzone */
 200                 rte_memzone_free(rq->mz);
 201
 202                 /* Release the queue */
 203                 rte_free(rq);
 204         }
 205 }
 206
 207 static void
 208 vmxnet3_dev_tx_queue_reset(void *txq)
 209 {
 210         vmxnet3_tx_queue_t *tq = txq;
 211         struct vmxnet3_cmd_ring *ring = &tq->cmd_ring;
 212         struct vmxnet3_comp_ring *comp_ring = &tq->comp_ring;
 213         struct vmxnet3_data_ring *data_ring = &tq->data_ring;
 214         int size;
 215
 216         if (tq != NULL) {
 217                 /* Release the cmd_ring mbufs */
 218                 vmxnet3_tx_cmd_ring_release_mbufs(&tq->cmd_ring);
 219         }
 220
 221         /* Tx vmxnet rings structure initialization*/
 222         ring->next2fill = 0;
 223         ring->next2comp = 0;
 224         ring->gen = VMXNET3_INIT_GEN;
 225         comp_ring->next2proc = 0;
 226         comp_ring->gen = VMXNET3_INIT_GEN;
 227
 228         size = sizeof(struct Vmxnet3_TxDesc) * ring->size;
 229         size += sizeof(struct Vmxnet3_TxCompDesc) * comp_ring->size;
 230         size += tq->txdata_desc_size * data_ring->size;
 231
 232         memset(ring->base, 0, size);
 233 }
 234
 235 static void
 236 vmxnet3_dev_rx_queue_reset(void *rxq)
 237 {
 238         int i;
 239         vmxnet3_rx_queue_t *rq = rxq;
 240         struct vmxnet3_hw *hw = rq->hw;
 241         struct vmxnet3_cmd_ring *ring0, *ring1;
 242         struct vmxnet3_comp_ring *comp_ring;
 243         struct vmxnet3_rx_data_ring *data_ring = &rq->data_ring;
 244         int size;
 245
 246         /* Release both the cmd_rings mbufs */
 247         for (i = 0; i < VMXNET3_RX_CMDRING_SIZE; i++)
 248                 vmxnet3_rx_cmd_ring_release_mbufs(&rq->cmd_ring[i]);
 249
 250         ring0 = &rq->cmd_ring[0];
 251         ring1 = &rq->cmd_ring[1];
 252         comp_ring = &rq->comp_ring;
 253
 254         /* Rx vmxnet rings structure initialization */
 255         ring0->next2fill = 0;
 256         ring1->next2fill = 0;
 257         ring0->next2comp = 0;
 258         ring1->next2comp = 0;
 259         ring0->gen = VMXNET3_INIT_GEN;
 260         ring1->gen = VMXNET3_INIT_GEN;
 261         comp_ring->next2proc = 0;
 262         comp_ring->gen = VMXNET3_INIT_GEN;
 263
 264         size = sizeof(struct Vmxnet3_RxDesc) * (ring0->size + ring1->size);
 265         size += sizeof(struct Vmxnet3_RxCompDesc) * comp_ring->size;
 266         if (VMXNET3_VERSION_GE_3(hw) && rq->data_desc_size)
 267                 size += rq->data_desc_size * data_ring->size;
 268
 269         memset(ring0->base, 0, size);
 270 }
 271
 272 void
 273 vmxnet3_dev_clear_queues(struct rte_eth_dev *dev)
 274 {
 275         unsigned i;
 276
 277         PMD_INIT_FUNC_TRACE();
 278
 279         for (i = 0; i < dev->data->nb_tx_queues; i++) {
 280                 struct vmxnet3_tx_queue *txq = dev->data->tx_queues[i];
 281
 282                 if (txq != NULL) {
 283                         txq->stopped = TRUE;
 284                         vmxnet3_dev_tx_queue_reset(txq);
 285                 }
 286         }
 287
 288         for (i = 0; i < dev->data->nb_rx_queues; i++) {
 289                 struct vmxnet3_rx_queue *rxq = dev->data->rx_queues[i];
 290
 291                 if (rxq != NULL) {
 292                         rxq->stopped = TRUE;
 293                         vmxnet3_dev_rx_queue_reset(rxq);
 294                 }
 295         }
 296 }
 297
 298 static int
 299 vmxnet3_unmap_pkt(uint16_t eop_idx, vmxnet3_tx_queue_t *txq)
 300 {
 301         int completed = 0;
 302         struct rte_mbuf *mbuf;
 303
 304         /* Release cmd_ring descriptor and free mbuf */
 305         RTE_ASSERT(txq->cmd_ring.base[eop_idx].txd.eop == 1);
 306
 307         mbuf = txq->cmd_ring.buf_info[eop_idx].m;
 308         if (mbuf == NULL)
 309                 rte_panic("EOP desc does not point to a valid mbuf");
 310         rte_pktmbuf_free(mbuf);
 311
 312         txq->cmd_ring.buf_info[eop_idx].m = NULL;
 313
 314         while (txq->cmd_ring.next2comp != eop_idx) {
 315                 /* no out-of-order completion */
 316                 RTE_ASSERT(txq->cmd_ring.base[txq->cmd_ring.next2comp].txd.cq == 0);
 317                 vmxnet3_cmd_ring_adv_next2comp(&txq->cmd_ring);
 318                 completed++;
 319         }
 320
 321         /* Mark the txd for which tcd was generated as completed */
 322         vmxnet3_cmd_ring_adv_next2comp(&txq->cmd_ring);
 323
 324         return completed + 1;
 325 }
 326
 327 static void
 328 vmxnet3_tq_tx_complete(vmxnet3_tx_queue_t *txq)
 329 {
 330         int completed = 0;
 331         vmxnet3_comp_ring_t *comp_ring = &txq->comp_ring;
 332         struct Vmxnet3_TxCompDesc *tcd = (struct Vmxnet3_TxCompDesc *)
 333                 (comp_ring->base + comp_ring->next2proc);
 334
 335         while (tcd->gen == comp_ring->gen) {
 336                 completed += vmxnet3_unmap_pkt(tcd->txdIdx, txq);
 337
 338                 vmxnet3_comp_ring_adv_next2proc(comp_ring);
 339                 tcd = (struct Vmxnet3_TxCompDesc *)(comp_ring->base +
 340                                                     comp_ring->next2proc);
 341         }
 342
 343         PMD_TX_LOG(DEBUG, "Processed %d tx comps & command descs.", completed);
 344 }
 345
 346 uint16_t
 347 vmxnet3_prep_pkts(__rte_unused void *tx_queue, struct rte_mbuf **tx_pkts,
 348         uint16_t nb_pkts)
 349 {
 350         int32_t ret;
 351         uint32_t i;
 352         uint64_t ol_flags;
 353         struct rte_mbuf *m;
 354
 355         for (i = 0; i != nb_pkts; i++) {
 356                 m = tx_pkts[i];
 357                 ol_flags = m->ol_flags;
 358
 359                 /* Non-TSO packet cannot occupy more than
 360                  * VMXNET3_MAX_TXD_PER_PKT TX descriptors.
 361                  */
 362                 if ((ol_flags & PKT_TX_TCP_SEG) == 0 &&
 363                                 m->nb_segs > VMXNET3_MAX_TXD_PER_PKT) {
 364                         rte_errno = EINVAL;
 365                         return i;
 366                 }
 367
 368                 /* check that only supported TX offloads are requested. */
 369                 if ((ol_flags & VMXNET3_TX_OFFLOAD_NOTSUP_MASK) != 0 ||
 370                                 (ol_flags & PKT_TX_L4_MASK) ==
 371                                 PKT_TX_SCTP_CKSUM) {
 372                         rte_errno = ENOTSUP;
 373                         return i;
 374                 }
 375
 376 #ifdef RTE_LIBRTE_ETHDEV_DEBUG
 377                 ret = rte_validate_tx_offload(m);
 378                 if (ret != 0) {
 379                         rte_errno = -ret;
 380                         return i;
 381                 }
 382 #endif
 383                 ret = rte_net_intel_cksum_prepare(m);
 384                 if (ret != 0) {
 385                         rte_errno = -ret;
 386                         return i;
 387                 }
 388         }
 389
 390         return i;
 391 }
 392
 393 uint16_t
 394 vmxnet3_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts,
 395                   uint16_t nb_pkts)
 396 {
 397         uint16_t nb_tx;
 398         vmxnet3_tx_queue_t *txq = tx_queue;
 399         struct vmxnet3_hw *hw = txq->hw;
 400         Vmxnet3_TxQueueCtrl *txq_ctrl = &txq->shared->ctrl;
 401         uint32_t deferred = rte_le_to_cpu_32(txq_ctrl->txNumDeferred);
 402
 403         if (unlikely(txq->stopped)) {
 404                 PMD_TX_LOG(DEBUG, "Tx queue is stopped.");
 405                 return 0;
 406         }
 407
 408         /* Free up the comp_descriptors aggressively */
 409         vmxnet3_tq_tx_complete(txq);
 410
 411         nb_tx = 0;
 412         while (nb_tx < nb_pkts) {
 413                 Vmxnet3_GenericDesc *gdesc;
 414                 vmxnet3_buf_info_t *tbi;
 415                 uint32_t first2fill, avail, dw2;
 416                 struct rte_mbuf *txm = tx_pkts[nb_tx];
 417                 struct rte_mbuf *m_seg = txm;
 418                 int copy_size = 0;
 419                 bool tso = (txm->ol_flags & PKT_TX_TCP_SEG) != 0;
 420                 /* # of descriptors needed for a packet. */
 421                 unsigned count = txm->nb_segs;
 422
 423                 avail = vmxnet3_cmd_ring_desc_avail(&txq->cmd_ring);
 424                 if (count > avail) {
 425                         /* Is command ring full? */
 426                         if (unlikely(avail == 0)) {
 427                                 PMD_TX_LOG(DEBUG, "No free ring descriptors");
 428                                 txq->stats.tx_ring_full++;
 429                                 txq->stats.drop_total += (nb_pkts - nb_tx);
 430                                 break;
 431                         }
 432
 433                         /* Command ring is not full but cannot handle the
 434                          * multi-segmented packet. Let's try the next packet
 435                          * in this case.
 436                          */
 437                         PMD_TX_LOG(DEBUG, "Running out of ring descriptors "
 438                                    "(avail %d needed %d)", avail, count);
 439                         txq->stats.drop_total++;
 440                         if (tso)
 441                                 txq->stats.drop_tso++;
 442                         rte_pktmbuf_free(txm);
 443                         nb_tx++;
 444                         continue;
 445                 }
 446
 447                 /* Drop non-TSO packet that is excessively fragmented */
 448                 if (unlikely(!tso && count > VMXNET3_MAX_TXD_PER_PKT)) {
 449                         PMD_TX_LOG(ERR, "Non-TSO packet cannot occupy more than %d tx "
 450                                    "descriptors. Packet dropped.", VMXNET3_MAX_TXD_PER_PKT);
 451                         txq->stats.drop_too_many_segs++;
 452                         txq->stats.drop_total++;
 453                         rte_pktmbuf_free(txm);
 454                         nb_tx++;
 455                         continue;
 456                 }
 457
 458                 if (txm->nb_segs == 1 &&
 459                     rte_pktmbuf_pkt_len(txm) <= txq->txdata_desc_size) {
 460                         struct Vmxnet3_TxDataDesc *tdd;
 461
 462                         /* Skip empty packets */
 463                         if (unlikely(rte_pktmbuf_pkt_len(txm) == 0)) {
 464                                 txq->stats.drop_total++;
 465                                 rte_pktmbuf_free(txm);
 466                                 nb_tx++;
 467                                 continue;
 468                         }
 469
 470                         tdd = (struct Vmxnet3_TxDataDesc *)
 471                                 ((uint8 *)txq->data_ring.base +
 472                                  txq->cmd_ring.next2fill *
 473                                  txq->txdata_desc_size);
 474                         copy_size = rte_pktmbuf_pkt_len(txm);
 475                         rte_memcpy(tdd->data, rte_pktmbuf_mtod(txm, char *), copy_size);
 476                 }
 477
 478                 /* use the previous gen bit for the SOP desc */
 479                 dw2 = (txq->cmd_ring.gen ^ 0x1) << VMXNET3_TXD_GEN_SHIFT;
 480                 first2fill = txq->cmd_ring.next2fill;
 481                 do {
 482                         /* Remember the transmit buffer for cleanup */
 483                         tbi = txq->cmd_ring.buf_info + txq->cmd_ring.next2fill;
 484
 485                         /* NB: the following assumes that VMXNET3 maximum
 486                          * transmit buffer size (16K) is greater than
 487                          * maximum size of mbuf segment size.
 488                          */
 489                         gdesc = txq->cmd_ring.base + txq->cmd_ring.next2fill;
 490
 491                         /* Skip empty segments */
 492                         if (unlikely(m_seg->data_len == 0))
 493                                 continue;
 494
 495                         if (copy_size) {
 496                                 uint64 offset =
 497                                         (uint64)txq->cmd_ring.next2fill *
 498                                                         txq->txdata_desc_size;
 499                                 gdesc->txd.addr =
 500                                         rte_cpu_to_le_64(txq->data_ring.basePA +
 501                                                          offset);
 502                         } else {
 503                                 gdesc->txd.addr = rte_mbuf_data_iova(m_seg);
 504                         }
 505
 506                         gdesc->dword[2] = dw2 | m_seg->data_len;
 507                         gdesc->dword[3] = 0;
 508
 509                         /* move to the next2fill descriptor */
 510                         vmxnet3_cmd_ring_adv_next2fill(&txq->cmd_ring);
 511
 512                         /* use the right gen for non-SOP desc */
 513                         dw2 = txq->cmd_ring.gen << VMXNET3_TXD_GEN_SHIFT;
 514                 } while ((m_seg = m_seg->next) != NULL);
 515
 516                 /* set the last buf_info for the pkt */
 517                 tbi->m = txm;
 518                 /* Update the EOP descriptor */
 519                 gdesc->dword[3] |= VMXNET3_TXD_EOP | VMXNET3_TXD_CQ;
 520
 521                 /* Add VLAN tag if present */
 522                 gdesc = txq->cmd_ring.base + first2fill;
 523                 if (txm->ol_flags & PKT_TX_VLAN_PKT) {
 524                         gdesc->txd.ti = 1;
 525                         gdesc->txd.tci = txm->vlan_tci;
 526                 }
 527
 528                 if (tso) {
 529                         uint16_t mss = txm->tso_segsz;
 530
 531                         RTE_ASSERT(mss > 0);
 532
 533                         gdesc->txd.hlen = txm->l2_len + txm->l3_len + txm->l4_len;
 534                         gdesc->txd.om = VMXNET3_OM_TSO;
 535                         gdesc->txd.msscof = mss;
 536
 537                         deferred += (rte_pktmbuf_pkt_len(txm) - gdesc->txd.hlen + mss - 1) / mss;
 538                 } else if (txm->ol_flags & PKT_TX_L4_MASK) {
 539                         gdesc->txd.om = VMXNET3_OM_CSUM;
 540                         gdesc->txd.hlen = txm->l2_len + txm->l3_len;
 541
 542                         switch (txm->ol_flags & PKT_TX_L4_MASK) {
 543                         case PKT_TX_TCP_CKSUM:
 544                                 gdesc->txd.msscof = gdesc->txd.hlen +
 545                                         offsetof(struct rte_tcp_hdr, cksum);
 546                                 break;
 547                         case PKT_TX_UDP_CKSUM:
 548                                 gdesc->txd.msscof = gdesc->txd.hlen +
 549                                         offsetof(struct rte_udp_hdr,
 550                                                 dgram_cksum);
 551                                 break;
 552                         default:
 553                                 PMD_TX_LOG(WARNING, "requested cksum offload not supported %#llx",
 554                                            txm->ol_flags & PKT_TX_L4_MASK);
 555                                 abort();
 556                         }
 557                         deferred++;
 558                 } else {
 559                         gdesc->txd.hlen = 0;
 560                         gdesc->txd.om = VMXNET3_OM_NONE;
 561                         gdesc->txd.msscof = 0;
 562                         deferred++;
 563                 }
 564
 565                 /* flip the GEN bit on the SOP */
 566                 rte_compiler_barrier();
 567                 gdesc->dword[2] ^= VMXNET3_TXD_GEN;
 568
 569                 txq_ctrl->txNumDeferred = rte_cpu_to_le_32(deferred);
 570                 nb_tx++;
 571         }
 572
 573         PMD_TX_LOG(DEBUG, "vmxnet3 txThreshold: %u", rte_le_to_cpu_32(txq_ctrl->txThreshold));
 574
 575         if (deferred >= rte_le_to_cpu_32(txq_ctrl->txThreshold)) {
 576                 txq_ctrl->txNumDeferred = 0;
 577                 /* Notify vSwitch that packets are available. */
 578                 VMXNET3_WRITE_BAR0_REG(hw, (VMXNET3_REG_TXPROD + txq->queue_id * VMXNET3_REG_ALIGN),
 579                                        txq->cmd_ring.next2fill);
 580         }
 581
 582         return nb_tx;
 583 }
 584
 585 static inline void
 586 vmxnet3_renew_desc(vmxnet3_rx_queue_t *rxq, uint8_t ring_id,
 587                    struct rte_mbuf *mbuf)
 588 {
 589         uint32_t val;
 590         struct vmxnet3_cmd_ring *ring = &rxq->cmd_ring[ring_id];
 591         struct Vmxnet3_RxDesc *rxd =
 592                 (struct Vmxnet3_RxDesc *)(ring->base + ring->next2fill);
 593         vmxnet3_buf_info_t *buf_info = &ring->buf_info[ring->next2fill];
 594
 595         if (ring_id == 0) {
 596                 /* Usually: One HEAD type buf per packet
 597                  * val = (ring->next2fill % rxq->hw->bufs_per_pkt) ?
 598                  * VMXNET3_RXD_BTYPE_BODY : VMXNET3_RXD_BTYPE_HEAD;
 599                  */
 600
 601                 /* We use single packet buffer so all heads here */
 602                 val = VMXNET3_RXD_BTYPE_HEAD;
 603         } else {
 604                 /* All BODY type buffers for 2nd ring */
 605                 val = VMXNET3_RXD_BTYPE_BODY;
 606         }
 607
 608         /*
 609          * Load mbuf pointer into buf_info[ring_size]
 610          * buf_info structure is equivalent to cookie for virtio-virtqueue
 611          */
 612         buf_info->m = mbuf;
 613         buf_info->len = (uint16_t)(mbuf->buf_len - RTE_PKTMBUF_HEADROOM);
 614         buf_info->bufPA = rte_mbuf_data_iova_default(mbuf);
 615
 616         /* Load Rx Descriptor with the buffer's GPA */
 617         rxd->addr = buf_info->bufPA;
 618
 619         /* After this point rxd->addr MUST not be NULL */
 620         rxd->btype = val;
 621         rxd->len = buf_info->len;
 622         /* Flip gen bit at the end to change ownership */
 623         rxd->gen = ring->gen;
 624
 625         vmxnet3_cmd_ring_adv_next2fill(ring);
 626 }
 627 /*
 628  *  Allocates mbufs and clusters. Post rx descriptors with buffer details
 629  *  so that device can receive packets in those buffers.
 630  *  Ring layout:
 631  *      Among the two rings, 1st ring contains buffers of type 0 and type 1.
 632  *      bufs_per_pkt is set such that for non-LRO cases all the buffers required
 633  *      by a frame will fit in 1st ring (1st buf of type0 and rest of type1).
 634  *      2nd ring contains buffers of type 1 alone. Second ring mostly be used
 635  *      only for LRO.
 636  */
 637 static int
 638 vmxnet3_post_rx_bufs(vmxnet3_rx_queue_t *rxq, uint8_t ring_id)
 639 {
 640         int err = 0;
 641         uint32_t i = 0;
 642         struct vmxnet3_cmd_ring *ring = &rxq->cmd_ring[ring_id];
 643
 644         while (vmxnet3_cmd_ring_desc_avail(ring) > 0) {
 645                 struct rte_mbuf *mbuf;
 646
 647                 /* Allocate blank mbuf for the current Rx Descriptor */
 648                 mbuf = rte_mbuf_raw_alloc(rxq->mp);
 649                 if (unlikely(mbuf == NULL)) {
 650                         PMD_RX_LOG(ERR, "Error allocating mbuf");
 651                         rxq->stats.rx_buf_alloc_failure++;
 652                         err = ENOMEM;
 653                         break;
 654                 }
 655
 656                 vmxnet3_renew_desc(rxq, ring_id, mbuf);
 657                 i++;
 658         }
 659
 660         /* Return error only if no buffers are posted at present */
 661         if (vmxnet3_cmd_ring_desc_avail(ring) >= (ring->size - 1))
 662                 return -err;
 663         else
 664                 return i;
 665 }
 666
 667 /* MSS not provided by vmxnet3, guess one with available information */
 668 static uint16_t
 669 vmxnet3_guess_mss(struct vmxnet3_hw *hw, const Vmxnet3_RxCompDesc *rcd,
 670                 struct rte_mbuf *rxm)
 671 {
 672         uint32_t hlen, slen;
 673         struct rte_ipv4_hdr *ipv4_hdr;
 674         struct rte_ipv6_hdr *ipv6_hdr;
 675         struct rte_tcp_hdr *tcp_hdr;
 676         char *ptr;
 677         uint8_t segs;
 678
 679         RTE_ASSERT(rcd->tcp);
 680
 681         ptr = rte_pktmbuf_mtod(rxm, char *);
 682         slen = rte_pktmbuf_data_len(rxm);
 683         hlen = sizeof(struct rte_ether_hdr);
 684
 685         if (rcd->v4) {
 686                 if (unlikely(slen < hlen + sizeof(struct rte_ipv4_hdr)))
 687                         return hw->mtu - sizeof(struct rte_ipv4_hdr)
 688                                         - sizeof(struct rte_tcp_hdr);
 689
 690                 ipv4_hdr = (struct rte_ipv4_hdr *)(ptr + hlen);
 691                 hlen += rte_ipv4_hdr_len(ipv4_hdr);
 692         } else if (rcd->v6) {
 693                 if (unlikely(slen < hlen + sizeof(struct rte_ipv6_hdr)))
 694                         return hw->mtu - sizeof(struct rte_ipv6_hdr) -
 695                                         sizeof(struct rte_tcp_hdr);
 696
 697                 ipv6_hdr = (struct rte_ipv6_hdr *)(ptr + hlen);
 698                 hlen += sizeof(struct rte_ipv6_hdr);
 699                 if (unlikely(ipv6_hdr->proto != IPPROTO_TCP)) {
 700                         int frag;
 701
 702                         rte_net_skip_ip6_ext(ipv6_hdr->proto, rxm,
 703                                         &hlen, &frag);
 704                 }
 705         }
 706
 707         if (unlikely(slen < hlen + sizeof(struct rte_tcp_hdr)))
 708                 return hw->mtu - hlen - sizeof(struct rte_tcp_hdr) +
 709                                 sizeof(struct rte_ether_hdr);
 710
 711         tcp_hdr = (struct rte_tcp_hdr *)(ptr + hlen);
 712         hlen += (tcp_hdr->data_off & 0xf0) >> 2;
 713
 714         segs = *vmxnet3_segs_dynfield(rxm);
 715         if (segs > 1)
 716                 return (rte_pktmbuf_pkt_len(rxm) - hlen + segs - 1) / segs;
 717         else
 718                 return hw->mtu - hlen + sizeof(struct rte_ether_hdr);
 719 }
 720
 721 /* Receive side checksum and other offloads */
 722 static inline void
 723 vmxnet3_rx_offload(struct vmxnet3_hw *hw, const Vmxnet3_RxCompDesc *rcd,
 724                 struct rte_mbuf *rxm, const uint8_t sop)
 725 {
 726         uint64_t ol_flags = rxm->ol_flags;
 727         uint32_t packet_type = rxm->packet_type;
 728
 729         /* Offloads set in sop */
 730         if (sop) {
 731                 /* Set packet type */
 732                 packet_type |= RTE_PTYPE_L2_ETHER;
 733
 734                 /* Check large packet receive */
 735                 if (VMXNET3_VERSION_GE_2(hw) &&
 736                     rcd->type == VMXNET3_CDTYPE_RXCOMP_LRO) {
 737                         const Vmxnet3_RxCompDescExt *rcde =
 738                                         (const Vmxnet3_RxCompDescExt *)rcd;
 739
 740                         rxm->tso_segsz = rcde->mss;
 741                         *vmxnet3_segs_dynfield(rxm) = rcde->segCnt;
 742                         ol_flags |= PKT_RX_LRO;
 743                 }
 744         } else { /* Offloads set in eop */
 745                 /* Check for RSS */
 746                 if (rcd->rssType != VMXNET3_RCD_RSS_TYPE_NONE) {
 747                         ol_flags |= PKT_RX_RSS_HASH;
 748                         rxm->hash.rss = rcd->rssHash;
 749                 }
 750
 751                 /* Check for hardware stripped VLAN tag */
 752                 if (rcd->ts) {
 753                         ol_flags |= (PKT_RX_VLAN | PKT_RX_VLAN_STRIPPED);
 754                         rxm->vlan_tci = rte_le_to_cpu_16((uint16_t)rcd->tci);
 755                 }
 756
 757                 /* Check packet type, checksum errors, etc. */
 758                 if (rcd->cnc) {
 759                         ol_flags |= PKT_RX_L4_CKSUM_UNKNOWN;
 760                 } else {
 761                         if (rcd->v4) {
 762                                 packet_type |= RTE_PTYPE_L3_IPV4_EXT_UNKNOWN;
 763
 764                                 if (rcd->ipc)
 765                                         ol_flags |= PKT_RX_IP_CKSUM_GOOD;
 766                                 else
 767                                         ol_flags |= PKT_RX_IP_CKSUM_BAD;
 768
 769                                 if (rcd->tuc) {
 770                                         ol_flags |= PKT_RX_L4_CKSUM_GOOD;
 771                                         if (rcd->tcp)
 772                                                 packet_type |= RTE_PTYPE_L4_TCP;
 773                                         else
 774                                                 packet_type |= RTE_PTYPE_L4_UDP;
 775                                 } else {
 776                                         if (rcd->tcp) {
 777                                                 packet_type |= RTE_PTYPE_L4_TCP;
 778                                                 ol_flags |= PKT_RX_L4_CKSUM_BAD;
 779                                         } else if (rcd->udp) {
 780                                                 packet_type |= RTE_PTYPE_L4_UDP;
 781                                                 ol_flags |= PKT_RX_L4_CKSUM_BAD;
 782                                         }
 783                                 }
 784                         } else if (rcd->v6) {
 785                                 packet_type |= RTE_PTYPE_L3_IPV6_EXT_UNKNOWN;
 786
 787                                 if (rcd->tuc) {
 788                                         ol_flags |= PKT_RX_L4_CKSUM_GOOD;
 789                                         if (rcd->tcp)
 790                                                 packet_type |= RTE_PTYPE_L4_TCP;
 791                                         else
 792                                                 packet_type |= RTE_PTYPE_L4_UDP;
 793                                 } else {
 794                                         if (rcd->tcp) {
 795                                                 packet_type |= RTE_PTYPE_L4_TCP;
 796                                                 ol_flags |= PKT_RX_L4_CKSUM_BAD;
 797                                         } else if (rcd->udp) {
 798                                                 packet_type |= RTE_PTYPE_L4_UDP;
 799                                                 ol_flags |= PKT_RX_L4_CKSUM_BAD;
 800                                         }
 801                                 }
 802                         } else {
 803                                 packet_type |= RTE_PTYPE_UNKNOWN;
 804                         }
 805
 806                         /* Old variants of vmxnet3 do not provide MSS */
 807                         if ((ol_flags & PKT_RX_LRO) && rxm->tso_segsz == 0)
 808                                 rxm->tso_segsz = vmxnet3_guess_mss(hw,
 809                                                 rcd, rxm);
 810                 }
 811         }
 812
 813         rxm->ol_flags = ol_flags;
 814         rxm->packet_type = packet_type;
 815 }
 816
 817 /*
 818  * Process the Rx Completion Ring of given vmxnet3_rx_queue
 819  * for nb_pkts burst and return the number of packets received
 820  */
 821 uint16_t
 822 vmxnet3_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts, uint16_t nb_pkts)
 823 {
 824         uint16_t nb_rx;
 825         uint32_t nb_rxd, idx;
 826         uint8_t ring_idx;
 827         vmxnet3_rx_queue_t *rxq;
 828         Vmxnet3_RxCompDesc *rcd;
 829         vmxnet3_buf_info_t *rbi;
 830         Vmxnet3_RxDesc *rxd;
 831         struct rte_mbuf *rxm = NULL;
 832         struct vmxnet3_hw *hw;
 833
 834         nb_rx = 0;
 835         ring_idx = 0;
 836         nb_rxd = 0;
 837         idx = 0;
 838
 839         rxq = rx_queue;
 840         hw = rxq->hw;
 841
 842         rcd = &rxq->comp_ring.base[rxq->comp_ring.next2proc].rcd;
 843
 844         if (unlikely(rxq->stopped)) {
 845                 PMD_RX_LOG(DEBUG, "Rx queue is stopped.");
 846                 return 0;
 847         }
 848
 849         while (rcd->gen == rxq->comp_ring.gen) {
 850                 struct rte_mbuf *newm;
 851
 852                 if (nb_rx >= nb_pkts)
 853                         break;
 854
 855                 newm = rte_mbuf_raw_alloc(rxq->mp);
 856                 if (unlikely(newm == NULL)) {
 857                         PMD_RX_LOG(ERR, "Error allocating mbuf");
 858                         rxq->stats.rx_buf_alloc_failure++;
 859                         break;
 860                 }
 861
 862                 idx = rcd->rxdIdx;
 863                 ring_idx = vmxnet3_get_ring_idx(hw, rcd->rqID);
 864                 rxd = (Vmxnet3_RxDesc *)rxq->cmd_ring[ring_idx].base + idx;
 865                 RTE_SET_USED(rxd); /* used only for assert when enabled */
 866                 rbi = rxq->cmd_ring[ring_idx].buf_info + idx;
 867
 868                 PMD_RX_LOG(DEBUG, "rxd idx: %d ring idx: %d.", idx, ring_idx);
 869
 870                 RTE_ASSERT(rcd->len <= rxd->len);
 871                 RTE_ASSERT(rbi->m);
 872
 873                 /* Get the packet buffer pointer from buf_info */
 874                 rxm = rbi->m;
 875
 876                 /* Clear descriptor associated buf_info to be reused */
 877                 rbi->m = NULL;
 878                 rbi->bufPA = 0;
 879
 880                 /* Update the index that we received a packet */
 881                 rxq->cmd_ring[ring_idx].next2comp = idx;
 882
 883                 /* For RCD with EOP set, check if there is frame error */
 884                 if (unlikely(rcd->eop && rcd->err)) {
 885                         rxq->stats.drop_total++;
 886                         rxq->stats.drop_err++;
 887
 888                         if (!rcd->fcs) {
 889                                 rxq->stats.drop_fcs++;
 890                                 PMD_RX_LOG(ERR, "Recv packet dropped due to frame err.");
 891                         }
 892                         PMD_RX_LOG(ERR, "Error in received packet rcd#:%d rxd:%d",
 893                                    (int)(rcd - (struct Vmxnet3_RxCompDesc *)
 894                                          rxq->comp_ring.base), rcd->rxdIdx);
 895                         rte_pktmbuf_free_seg(rxm);
 896                         if (rxq->start_seg) {
 897                                 struct rte_mbuf *start = rxq->start_seg;
 898
 899                                 rxq->start_seg = NULL;
 900                                 rte_pktmbuf_free(start);
 901                         }
 902                         goto rcd_done;
 903                 }
 904
 905                 /* Initialize newly received packet buffer */
 906                 rxm->port = rxq->port_id;
 907                 rxm->nb_segs = 1;
 908                 rxm->next = NULL;
 909                 rxm->pkt_len = (uint16_t)rcd->len;
 910                 rxm->data_len = (uint16_t)rcd->len;
 911                 rxm->data_off = RTE_PKTMBUF_HEADROOM;
 912                 rxm->ol_flags = 0;
 913                 rxm->vlan_tci = 0;
 914                 rxm->packet_type = 0;
 915
 916                 /*
 917                  * If this is the first buffer of the received packet,
 918                  * set the pointer to the first mbuf of the packet
 919                  * Otherwise, update the total length and the number of segments
 920                  * of the current scattered packet, and update the pointer to
 921                  * the last mbuf of the current packet.
 922                  */
 923                 if (rcd->sop) {
 924                         RTE_ASSERT(rxd->btype == VMXNET3_RXD_BTYPE_HEAD);
 925
 926                         if (unlikely(rcd->len == 0)) {
 927                                 RTE_ASSERT(rcd->eop);
 928
 929                                 PMD_RX_LOG(DEBUG,
 930                                            "Rx buf was skipped. rxring[%d][%d])",
 931                                            ring_idx, idx);
 932                                 rte_pktmbuf_free_seg(rxm);
 933                                 goto rcd_done;
 934                         }
 935
 936                         if (vmxnet3_rx_data_ring(hw, rcd->rqID)) {
 937                                 uint8_t *rdd = rxq->data_ring.base +
 938                                         idx * rxq->data_desc_size;
 939
 940                                 RTE_ASSERT(VMXNET3_VERSION_GE_3(hw));
 941                                 rte_memcpy(rte_pktmbuf_mtod(rxm, char *),
 942                                            rdd, rcd->len);
 943                         }
 944
 945                         rxq->start_seg = rxm;
 946                         rxq->last_seg = rxm;
 947                         vmxnet3_rx_offload(hw, rcd, rxm, 1);
 948                 } else {
 949                         struct rte_mbuf *start = rxq->start_seg;
 950
 951                         RTE_ASSERT(rxd->btype == VMXNET3_RXD_BTYPE_BODY);
 952
 953                         if (likely(start && rxm->data_len > 0)) {
 954                                 start->pkt_len += rxm->data_len;
 955                                 start->nb_segs++;
 956
 957                                 rxq->last_seg->next = rxm;
 958                                 rxq->last_seg = rxm;
 959                         } else {
 960                                 PMD_RX_LOG(ERR, "Error received empty or out of order frame.");
 961                                 rxq->stats.drop_total++;
 962                                 rxq->stats.drop_err++;
 963
 964                                 rte_pktmbuf_free_seg(rxm);
 965                         }
 966                 }
 967
 968                 if (rcd->eop) {
 969                         struct rte_mbuf *start = rxq->start_seg;
 970
 971                         vmxnet3_rx_offload(hw, rcd, start, 0);
 972                         rx_pkts[nb_rx++] = start;
 973                         rxq->start_seg = NULL;
 974                 }
 975
 976 rcd_done:
 977                 rxq->cmd_ring[ring_idx].next2comp = idx;
 978                 VMXNET3_INC_RING_IDX_ONLY(rxq->cmd_ring[ring_idx].next2comp,
 979                                           rxq->cmd_ring[ring_idx].size);
 980
 981                 /* It's time to renew descriptors */
 982                 vmxnet3_renew_desc(rxq, ring_idx, newm);
 983                 if (unlikely(rxq->shared->ctrl.updateRxProd)) {
 984                         VMXNET3_WRITE_BAR0_REG(hw, rxprod_reg[ring_idx] + (rxq->queue_id * VMXNET3_REG_ALIGN),
 985                                                rxq->cmd_ring[ring_idx].next2fill);
 986                 }
 987
 988                 /* Advance to the next descriptor in comp_ring */
 989                 vmxnet3_comp_ring_adv_next2proc(&rxq->comp_ring);
 990
 991                 rcd = &rxq->comp_ring.base[rxq->comp_ring.next2proc].rcd;
 992                 nb_rxd++;
 993                 if (nb_rxd > rxq->cmd_ring[0].size) {
 994                         PMD_RX_LOG(ERR, "Used up quota of receiving packets,"
 995                                    " relinquish control.");
 996                         break;
 997                 }
 998         }
 999
1000         if (unlikely(nb_rxd == 0)) {
1001                 uint32_t avail;
1002                 for (ring_idx = 0; ring_idx < VMXNET3_RX_CMDRING_SIZE; ring_idx++) {
1003                         avail = vmxnet3_cmd_ring_desc_avail(&rxq->cmd_ring[ring_idx]);
1004                         if (unlikely(avail > 0)) {
1005                                 /* try to alloc new buf and renew descriptors */
1006                                 vmxnet3_post_rx_bufs(rxq, ring_idx);
1007                         }
1008                 }
1009                 if (unlikely(rxq->shared->ctrl.updateRxProd)) {
1010                         for (ring_idx = 0; ring_idx < VMXNET3_RX_CMDRING_SIZE; ring_idx++) {
1011                                 VMXNET3_WRITE_BAR0_REG(hw, rxprod_reg[ring_idx] + (rxq->queue_id * VMXNET3_REG_ALIGN),
1012                                                        rxq->cmd_ring[ring_idx].next2fill);
1013                         }
1014                 }
1015         }
1016
1017         return nb_rx;
1018 }
1019
1020 int
1021 vmxnet3_dev_tx_queue_setup(struct rte_eth_dev *dev,
1022                            uint16_t queue_idx,
1023                            uint16_t nb_desc,
1024                            unsigned int socket_id,
1025                            const struct rte_eth_txconf *tx_conf __rte_unused)
1026 {
1027         struct vmxnet3_hw *hw = dev->data->dev_private;
1028         const struct rte_memzone *mz;
1029         struct vmxnet3_tx_queue *txq;
1030         struct vmxnet3_cmd_ring *ring;
1031         struct vmxnet3_comp_ring *comp_ring;
1032         struct vmxnet3_data_ring *data_ring;
1033         int size;
1034
1035         PMD_INIT_FUNC_TRACE();
1036
1037         txq = rte_zmalloc("ethdev_tx_queue", sizeof(struct vmxnet3_tx_queue),
1038                           RTE_CACHE_LINE_SIZE);
1039         if (txq == NULL) {
1040                 PMD_INIT_LOG(ERR, "Can not allocate tx queue structure");
1041                 return -ENOMEM;
1042         }
1043
1044         txq->queue_id = queue_idx;
1045         txq->port_id = dev->data->port_id;
1046         txq->shared = NULL; /* set in vmxnet3_setup_driver_shared() */
1047         txq->hw = hw;
1048         txq->qid = queue_idx;
1049         txq->stopped = TRUE;
1050         txq->txdata_desc_size = hw->txdata_desc_size;
1051
1052         ring = &txq->cmd_ring;
1053         comp_ring = &txq->comp_ring;
1054         data_ring = &txq->data_ring;
1055
1056         /* Tx vmxnet ring length should be between 512-4096 */
1057         if (nb_desc < VMXNET3_DEF_TX_RING_SIZE) {
1058                 PMD_INIT_LOG(ERR, "VMXNET3 Tx Ring Size Min: %u",
1059                              VMXNET3_DEF_TX_RING_SIZE);
1060                 return -EINVAL;
1061         } else if (nb_desc > VMXNET3_TX_RING_MAX_SIZE) {
1062                 PMD_INIT_LOG(ERR, "VMXNET3 Tx Ring Size Max: %u",
1063                              VMXNET3_TX_RING_MAX_SIZE);
1064                 return -EINVAL;
1065         } else {
1066                 ring->size = nb_desc;
1067                 ring->size &= ~VMXNET3_RING_SIZE_MASK;
1068         }
1069         comp_ring->size = data_ring->size = ring->size;
1070
1071         /* Tx vmxnet rings structure initialization*/
1072         ring->next2fill = 0;
1073         ring->next2comp = 0;
1074         ring->gen = VMXNET3_INIT_GEN;
1075         comp_ring->next2proc = 0;
1076         comp_ring->gen = VMXNET3_INIT_GEN;
1077
1078         size = sizeof(struct Vmxnet3_TxDesc) * ring->size;
1079         size += sizeof(struct Vmxnet3_TxCompDesc) * comp_ring->size;
1080         size += txq->txdata_desc_size * data_ring->size;
1081
1082         mz = rte_eth_dma_zone_reserve(dev, "txdesc", queue_idx, size,
1083                                       VMXNET3_RING_BA_ALIGN, socket_id);
1084         if (mz == NULL) {
1085                 PMD_INIT_LOG(ERR, "ERROR: Creating queue descriptors zone");
1086                 return -ENOMEM;
1087         }
1088         txq->mz = mz;
1089         memset(mz->addr, 0, mz->len);
1090
1091         /* cmd_ring initialization */
1092         ring->base = mz->addr;
1093         ring->basePA = mz->iova;
1094
1095         /* comp_ring initialization */
1096         comp_ring->base = ring->base + ring->size;
1097         comp_ring->basePA = ring->basePA +
1098                 (sizeof(struct Vmxnet3_TxDesc) * ring->size);
1099
1100         /* data_ring initialization */
1101         data_ring->base = (Vmxnet3_TxDataDesc *)(comp_ring->base + comp_ring->size);
1102         data_ring->basePA = comp_ring->basePA +
1103                         (sizeof(struct Vmxnet3_TxCompDesc) * comp_ring->size);
1104
1105         /* cmd_ring0 buf_info allocation */
1106         ring->buf_info = rte_zmalloc("tx_ring_buf_info",
1107                                      ring->size * sizeof(vmxnet3_buf_info_t), RTE_CACHE_LINE_SIZE);
1108         if (ring->buf_info == NULL) {
1109                 PMD_INIT_LOG(ERR, "ERROR: Creating tx_buf_info structure");
1110                 return -ENOMEM;
1111         }
1112
1113         /* Update the data portion with txq */
1114         dev->data->tx_queues[queue_idx] = txq;
1115
1116         return 0;
1117 }
1118
1119 int
1120 vmxnet3_dev_rx_queue_setup(struct rte_eth_dev *dev,
1121                            uint16_t queue_idx,
1122                            uint16_t nb_desc,
1123                            unsigned int socket_id,
1124                            __rte_unused const struct rte_eth_rxconf *rx_conf,
1125                            struct rte_mempool *mp)
1126 {
1127         const struct rte_memzone *mz;
1128         struct vmxnet3_rx_queue *rxq;
1129         struct vmxnet3_hw *hw = dev->data->dev_private;
1130         struct vmxnet3_cmd_ring *ring0, *ring1, *ring;
1131         struct vmxnet3_comp_ring *comp_ring;
1132         struct vmxnet3_rx_data_ring *data_ring;
1133         int size;
1134         uint8_t i;
1135         char mem_name[32];
1136
1137         PMD_INIT_FUNC_TRACE();
1138
1139         rxq = rte_zmalloc("ethdev_rx_queue", sizeof(struct vmxnet3_rx_queue),
1140                           RTE_CACHE_LINE_SIZE);
1141         if (rxq == NULL) {
1142                 PMD_INIT_LOG(ERR, "Can not allocate rx queue structure");
1143                 return -ENOMEM;
1144         }
1145
1146         rxq->mp = mp;
1147         rxq->queue_id = queue_idx;
1148         rxq->port_id = dev->data->port_id;
1149         rxq->shared = NULL; /* set in vmxnet3_setup_driver_shared() */
1150         rxq->hw = hw;
1151         rxq->qid1 = queue_idx;
1152         rxq->qid2 = queue_idx + hw->num_rx_queues;
1153         rxq->data_ring_qid = queue_idx + 2 * hw->num_rx_queues;
1154         rxq->data_desc_size = hw->rxdata_desc_size;
1155         rxq->stopped = TRUE;
1156
1157         ring0 = &rxq->cmd_ring[0];
1158         ring1 = &rxq->cmd_ring[1];
1159         comp_ring = &rxq->comp_ring;
1160         data_ring = &rxq->data_ring;
1161
1162         /* Rx vmxnet rings length should be between 256-4096 */
1163         if (nb_desc < VMXNET3_DEF_RX_RING_SIZE) {
1164                 PMD_INIT_LOG(ERR, "VMXNET3 Rx Ring Size Min: 256");
1165                 return -EINVAL;
1166         } else if (nb_desc > VMXNET3_RX_RING_MAX_SIZE) {
1167                 PMD_INIT_LOG(ERR, "VMXNET3 Rx Ring Size Max: 4096");
1168                 return -EINVAL;
1169         } else {
1170                 ring0->size = nb_desc;
1171                 ring0->size &= ~VMXNET3_RING_SIZE_MASK;
1172                 ring1->size = ring0->size;
1173         }
1174
1175         comp_ring->size = ring0->size + ring1->size;
1176         data_ring->size = ring0->size;
1177
1178         /* Rx vmxnet rings structure initialization */
1179         ring0->next2fill = 0;
1180         ring1->next2fill = 0;
1181         ring0->next2comp = 0;
1182         ring1->next2comp = 0;
1183         ring0->gen = VMXNET3_INIT_GEN;
1184         ring1->gen = VMXNET3_INIT_GEN;
1185         comp_ring->next2proc = 0;
1186         comp_ring->gen = VMXNET3_INIT_GEN;
1187
1188         size = sizeof(struct Vmxnet3_RxDesc) * (ring0->size + ring1->size);
1189         size += sizeof(struct Vmxnet3_RxCompDesc) * comp_ring->size;
1190         if (VMXNET3_VERSION_GE_3(hw) && rxq->data_desc_size)
1191                 size += rxq->data_desc_size * data_ring->size;
1192
1193         mz = rte_eth_dma_zone_reserve(dev, "rxdesc", queue_idx, size,
1194                                       VMXNET3_RING_BA_ALIGN, socket_id);
1195         if (mz == NULL) {
1196                 PMD_INIT_LOG(ERR, "ERROR: Creating queue descriptors zone");
1197                 return -ENOMEM;
1198         }
1199         rxq->mz = mz;
1200         memset(mz->addr, 0, mz->len);
1201
1202         /* cmd_ring0 initialization */
1203         ring0->base = mz->addr;
1204         ring0->basePA = mz->iova;
1205
1206         /* cmd_ring1 initialization */
1207         ring1->base = ring0->base + ring0->size;
1208         ring1->basePA = ring0->basePA + sizeof(struct Vmxnet3_RxDesc) * ring0->size;
1209
1210         /* comp_ring initialization */
1211         comp_ring->base = ring1->base + ring1->size;
1212         comp_ring->basePA = ring1->basePA + sizeof(struct Vmxnet3_RxDesc) *
1213                 ring1->size;
1214
1215         /* data_ring initialization */
1216         if (VMXNET3_VERSION_GE_3(hw) && rxq->data_desc_size) {
1217                 data_ring->base =
1218                         (uint8_t *)(comp_ring->base + comp_ring->size);
1219                 data_ring->basePA = comp_ring->basePA +
1220                         sizeof(struct Vmxnet3_RxCompDesc) * comp_ring->size;
1221         }
1222
1223         /* cmd_ring0-cmd_ring1 buf_info allocation */
1224         for (i = 0; i < VMXNET3_RX_CMDRING_SIZE; i++) {
1225
1226                 ring = &rxq->cmd_ring[i];
1227                 ring->rid = i;
1228                 snprintf(mem_name, sizeof(mem_name), "rx_ring_%d_buf_info", i);
1229
1230                 ring->buf_info = rte_zmalloc(mem_name,
1231                                              ring->size * sizeof(vmxnet3_buf_info_t),
1232                                              RTE_CACHE_LINE_SIZE);
1233                 if (ring->buf_info == NULL) {
1234                         PMD_INIT_LOG(ERR, "ERROR: Creating rx_buf_info structure");
1235                         return -ENOMEM;
1236                 }
1237         }
1238
1239         /* Update the data portion with rxq */
1240         dev->data->rx_queues[queue_idx] = rxq;
1241
1242         return 0;
1243 }
1244
1245 /*
1246  * Initializes Receive Unit
1247  * Load mbufs in rx queue in advance
1248  */
1249 int
1250 vmxnet3_dev_rxtx_init(struct rte_eth_dev *dev)
1251 {
1252         struct vmxnet3_hw *hw = dev->data->dev_private;
1253
1254         int i, ret;
1255         uint8_t j;
1256
1257         PMD_INIT_FUNC_TRACE();
1258
1259         for (i = 0; i < hw->num_rx_queues; i++) {
1260                 vmxnet3_rx_queue_t *rxq = dev->data->rx_queues[i];
1261
1262                 for (j = 0; j < VMXNET3_RX_CMDRING_SIZE; j++) {
1263                         /* Passing 0 as alloc_num will allocate full ring */
1264                         ret = vmxnet3_post_rx_bufs(rxq, j);
1265                         if (ret <= 0) {
1266                                 PMD_INIT_LOG(ERR,
1267                                              "ERROR: Posting Rxq: %d buffers ring: %d",
1268                                              i, j);
1269                                 return -ret;
1270                         }
1271                         /*
1272                          * Updating device with the index:next2fill to fill the
1273                          * mbufs for coming packets.
1274                          */
1275                         if (unlikely(rxq->shared->ctrl.updateRxProd)) {
1276                                 VMXNET3_WRITE_BAR0_REG(hw, rxprod_reg[j] + (rxq->queue_id * VMXNET3_REG_ALIGN),
1277                                                        rxq->cmd_ring[j].next2fill);
1278                         }
1279                 }
1280                 rxq->stopped = FALSE;
1281                 rxq->start_seg = NULL;
1282         }
1283
1284         for (i = 0; i < dev->data->nb_tx_queues; i++) {
1285                 struct vmxnet3_tx_queue *txq = dev->data->tx_queues[i];
1286
1287                 txq->stopped = FALSE;
1288         }
1289
1290         return 0;
1291 }
1292
1293 static uint8_t rss_intel_key[40] = {
1294         0x6D, 0x5A, 0x56, 0xDA, 0x25, 0x5B, 0x0E, 0xC2,
1295         0x41, 0x67, 0x25, 0x3D, 0x43, 0xA3, 0x8F, 0xB0,
1296         0xD0, 0xCA, 0x2B, 0xCB, 0xAE, 0x7B, 0x30, 0xB4,
1297         0x77, 0xCB, 0x2D, 0xA3, 0x80, 0x30, 0xF2, 0x0C,
1298         0x6A, 0x42, 0xB7, 0x3B, 0xBE, 0xAC, 0x01, 0xFA,
1299 };
1300
1301 /*
1302  * Additional RSS configurations based on vmxnet v4+ APIs
1303  */
1304 int
1305 vmxnet3_v4_rss_configure(struct rte_eth_dev *dev)
1306 {
1307         struct vmxnet3_hw *hw = dev->data->dev_private;
1308         Vmxnet3_DriverShared *shared = hw->shared;
1309         Vmxnet3_CmdInfo *cmdInfo = &shared->cu.cmdInfo;
1310         struct rte_eth_rss_conf *port_rss_conf;
1311         uint64_t rss_hf;
1312         uint32_t ret;
1313
1314         PMD_INIT_FUNC_TRACE();
1315
1316         cmdInfo->setRSSFields = 0;
1317         port_rss_conf = &dev->data->dev_conf.rx_adv_conf.rss_conf;
1318
1319         if ((port_rss_conf->rss_hf & VMXNET3_MANDATORY_V4_RSS) !=
1320             VMXNET3_MANDATORY_V4_RSS) {
1321                 PMD_INIT_LOG(WARNING, "RSS: IPv4/6 TCP is required for vmxnet3 v4 RSS,"
1322                              "automatically setting it");
1323                 port_rss_conf->rss_hf |= VMXNET3_MANDATORY_V4_RSS;
1324         }
1325
1326         rss_hf = port_rss_conf->rss_hf &
1327                 (VMXNET3_V4_RSS_MASK | VMXNET3_RSS_OFFLOAD_ALL);
1328
1329         if (rss_hf & ETH_RSS_NONFRAG_IPV4_TCP)
1330                 cmdInfo->setRSSFields |= VMXNET3_RSS_FIELDS_TCPIP4;
1331         if (rss_hf & ETH_RSS_NONFRAG_IPV6_TCP)
1332                 cmdInfo->setRSSFields |= VMXNET3_RSS_FIELDS_TCPIP6;
1333         if (rss_hf & ETH_RSS_NONFRAG_IPV4_UDP)
1334                 cmdInfo->setRSSFields |= VMXNET3_RSS_FIELDS_UDPIP4;
1335         if (rss_hf & ETH_RSS_NONFRAG_IPV6_UDP)
1336                 cmdInfo->setRSSFields |= VMXNET3_RSS_FIELDS_UDPIP6;
1337
1338         VMXNET3_WRITE_BAR1_REG(hw, VMXNET3_REG_CMD,
1339                                VMXNET3_CMD_SET_RSS_FIELDS);
1340         ret = VMXNET3_READ_BAR1_REG(hw, VMXNET3_REG_CMD);
1341
1342         if (ret != VMXNET3_SUCCESS) {
1343                 PMD_DRV_LOG(ERR, "Set RSS fields (v4) failed: %d", ret);
1344         }
1345
1346         return ret;
1347 }
1348
1349 /*
1350  * Configure RSS feature
1351  */
1352 int
1353 vmxnet3_rss_configure(struct rte_eth_dev *dev)
1354 {
1355         struct vmxnet3_hw *hw = dev->data->dev_private;
1356         struct VMXNET3_RSSConf *dev_rss_conf;
1357         struct rte_eth_rss_conf *port_rss_conf;
1358         uint64_t rss_hf;
1359         uint8_t i, j;
1360
1361         PMD_INIT_FUNC_TRACE();
1362
1363         dev_rss_conf = hw->rss_conf;
1364         port_rss_conf = &dev->data->dev_conf.rx_adv_conf.rss_conf;
1365
1366         /* loading hashFunc */
1367         dev_rss_conf->hashFunc = VMXNET3_RSS_HASH_FUNC_TOEPLITZ;
1368         /* loading hashKeySize */
1369         dev_rss_conf->hashKeySize = VMXNET3_RSS_MAX_KEY_SIZE;
1370         /* loading indTableSize: Must not exceed VMXNET3_RSS_MAX_IND_TABLE_SIZE (128)*/
1371         dev_rss_conf->indTableSize = (uint16_t)(hw->num_rx_queues * 4);
1372
1373         if (port_rss_conf->rss_key == NULL) {
1374                 /* Default hash key */
1375                 port_rss_conf->rss_key = rss_intel_key;
1376         }
1377
1378         /* loading hashKey */
1379         memcpy(&dev_rss_conf->hashKey[0], port_rss_conf->rss_key,
1380                dev_rss_conf->hashKeySize);
1381
1382         /* loading indTable */
1383         for (i = 0, j = 0; i < dev_rss_conf->indTableSize; i++, j++) {
1384                 if (j == dev->data->nb_rx_queues)
1385                         j = 0;
1386                 dev_rss_conf->indTable[i] = j;
1387         }
1388
1389         /* loading hashType */
1390         dev_rss_conf->hashType = 0;
1391         rss_hf = port_rss_conf->rss_hf & VMXNET3_RSS_OFFLOAD_ALL;
1392         if (rss_hf & ETH_RSS_IPV4)
1393                 dev_rss_conf->hashType |= VMXNET3_RSS_HASH_TYPE_IPV4;
1394         if (rss_hf & ETH_RSS_NONFRAG_IPV4_TCP)
1395                 dev_rss_conf->hashType |= VMXNET3_RSS_HASH_TYPE_TCP_IPV4;
1396         if (rss_hf & ETH_RSS_IPV6)
1397                 dev_rss_conf->hashType |= VMXNET3_RSS_HASH_TYPE_IPV6;
1398         if (rss_hf & ETH_RSS_NONFRAG_IPV6_TCP)
1399                 dev_rss_conf->hashType |= VMXNET3_RSS_HASH_TYPE_TCP_IPV6;
1400
1401         return VMXNET3_SUCCESS;
1402 }