lib/librte_pmd_vmxnet3/vmxnet3_rxtx.c

   1 /*-
   2  *   BSD LICENSE
   3  *
   4  *   Copyright(c) 2010-2014 Intel Corporation. All rights reserved.
   5  *   All rights reserved.
   6  *
   7  *   Redistribution and use in source and binary forms, with or without
   8  *   modification, are permitted provided that the following conditions
   9  *   are met:
  10  *
  11  *     * Redistributions of source code must retain the above copyright
  12  *       notice, this list of conditions and the following disclaimer.
  13  *     * Redistributions in binary form must reproduce the above copyright
  14  *       notice, this list of conditions and the following disclaimer in
  15  *       the documentation and/or other materials provided with the
  16  *       distribution.
  17  *     * Neither the name of Intel Corporation nor the names of its
  18  *       contributors may be used to endorse or promote products derived
  19  *       from this software without specific prior written permission.
  20  *
  21  *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  22  *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  23  *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
  24  *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  25  *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  26  *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  27  *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  28  *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  29  *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  30  *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  31  *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  32  */
  33
  34 #include <sys/queue.h>
  35
  36 #include <stdio.h>
  37 #include <stdlib.h>
  38 #include <string.h>
  39 #include <errno.h>
  40 #include <stdint.h>
  41 #include <stdarg.h>
  42 #include <unistd.h>
  43 #include <inttypes.h>
  44
  45 #include <rte_byteorder.h>
  46 #include <rte_common.h>
  47 #include <rte_cycles.h>
  48 #include <rte_log.h>
  49 #include <rte_debug.h>
  50 #include <rte_interrupts.h>
  51 #include <rte_pci.h>
  52 #include <rte_memory.h>
  53 #include <rte_memzone.h>
  54 #include <rte_launch.h>
  55 #include <rte_tailq.h>
  56 #include <rte_eal.h>
  57 #include <rte_per_lcore.h>
  58 #include <rte_lcore.h>
  59 #include <rte_atomic.h>
  60 #include <rte_branch_prediction.h>
  61 #include <rte_ring.h>
  62 #include <rte_mempool.h>
  63 #include <rte_malloc.h>
  64 #include <rte_mbuf.h>
  65 #include <rte_ether.h>
  66 #include <rte_ethdev.h>
  67 #include <rte_prefetch.h>
  68 #include <rte_udp.h>
  69 #include <rte_tcp.h>
  70 #include <rte_sctp.h>
  71 #include <rte_string_fns.h>
  72 #include <rte_errno.h>
  73
  74 #include "vmxnet3/vmxnet3_defs.h"
  75 #include "vmxnet3_ring.h"
  76
  77 #include "vmxnet3_logs.h"
  78 #include "vmxnet3_ethdev.h"
  79
  80
  81 #define RTE_MBUF_DATA_DMA_ADDR(mb) \
  82         (uint64_t) ((mb)->buf_physaddr + (mb)->data_off)
  83
  84 #define RTE_MBUF_DATA_DMA_ADDR_DEFAULT(mb) \
  85         (uint64_t) ((mb)->buf_physaddr + RTE_PKTMBUF_HEADROOM)
  86
  87 static uint32_t rxprod_reg[2] = {VMXNET3_REG_RXPROD, VMXNET3_REG_RXPROD2};
  88
  89 static inline int vmxnet3_post_rx_bufs(vmxnet3_rx_queue_t* , uint8_t);
  90 static inline void vmxnet3_tq_tx_complete(vmxnet3_tx_queue_t *);
  91 #ifdef RTE_LIBRTE_VMXNET3_DEBUG_DRIVER_NOT_USED
  92 static void vmxnet3_rxq_dump(struct vmxnet3_rx_queue *);
  93 static void vmxnet3_txq_dump(struct vmxnet3_tx_queue *);
  94 #endif
  95
  96 static inline struct rte_mbuf *
  97 rte_rxmbuf_alloc(struct rte_mempool *mp)
  98 {
  99         struct rte_mbuf *m;
 100
 101         m = __rte_mbuf_raw_alloc(mp);
 102         __rte_mbuf_sanity_check_raw(m, 0);
 103         return m;
 104 }
 105
 106 #ifdef RTE_LIBRTE_VMXNET3_DEBUG_DRIVER_NOT_USED
 107 static void
 108 vmxnet3_rxq_dump(struct vmxnet3_rx_queue *rxq)
 109 {
 110         uint32_t avail = 0;
 111
 112         if (rxq == NULL)
 113                 return;
 114
 115         PMD_RX_LOG(DEBUG,
 116                    "RXQ: cmd0 base : 0x%p cmd1 base : 0x%p comp ring base : 0x%p.",
 117                    rxq->cmd_ring[0].base, rxq->cmd_ring[1].base, rxq->comp_ring.base);
 118         PMD_RX_LOG(DEBUG,
 119                    "RXQ: cmd0 basePA : 0x%lx cmd1 basePA : 0x%lx comp ring basePA : 0x%lx.",
 120                    (unsigned long)rxq->cmd_ring[0].basePA,
 121                    (unsigned long)rxq->cmd_ring[1].basePA,
 122                    (unsigned long)rxq->comp_ring.basePA);
 123
 124         avail = vmxnet3_cmd_ring_desc_avail(&rxq->cmd_ring[0]);
 125         PMD_RX_LOG(DEBUG,
 126                    "RXQ:cmd0: size=%u; free=%u; next2proc=%u; queued=%u",
 127                    (uint32_t)rxq->cmd_ring[0].size, avail,
 128                    rxq->comp_ring.next2proc,
 129                    rxq->cmd_ring[0].size - avail);
 130
 131         avail = vmxnet3_cmd_ring_desc_avail(&rxq->cmd_ring[1]);
 132         PMD_RX_LOG(DEBUG, "RXQ:cmd1 size=%u; free=%u; next2proc=%u; queued=%u",
 133                    (uint32_t)rxq->cmd_ring[1].size, avail, rxq->comp_ring.next2proc,
 134                    rxq->cmd_ring[1].size - avail);
 135
 136 }
 137
 138 static void
 139 vmxnet3_txq_dump(struct vmxnet3_tx_queue *txq)
 140 {
 141         uint32_t avail = 0;
 142
 143         if (txq == NULL)
 144                 return;
 145
 146         PMD_TX_LOG(DEBUG, "TXQ: cmd base : 0x%p comp ring base : 0x%p.",
 147                    txq->cmd_ring.base, txq->comp_ring.base);
 148         PMD_TX_LOG(DEBUG, "TXQ: cmd basePA : 0x%lx comp ring basePA : 0x%lx.",
 149                    (unsigned long)txq->cmd_ring.basePA,
 150                    (unsigned long)txq->comp_ring.basePA);
 151
 152         avail = vmxnet3_cmd_ring_desc_avail(&txq->cmd_ring);
 153         PMD_TX_LOG(DEBUG, "TXQ: size=%u; free=%u; next2proc=%u; queued=%u",
 154                    (uint32_t)txq->cmd_ring.size, avail,
 155                    txq->comp_ring.next2proc, txq->cmd_ring.size - avail);
 156 }
 157 #endif
 158
 159 static inline void
 160 vmxnet3_cmd_ring_release_mbufs(vmxnet3_cmd_ring_t *ring)
 161 {
 162         while (ring->next2comp != ring->next2fill) {
 163                 /* No need to worry about tx desc ownership, device is quiesced by now. */
 164                 vmxnet3_buf_info_t *buf_info = ring->buf_info + ring->next2comp;
 165
 166                 if (buf_info->m) {
 167                         rte_pktmbuf_free(buf_info->m);
 168                         buf_info->m = NULL;
 169                         buf_info->bufPA = 0;
 170                         buf_info->len = 0;
 171                 }
 172                 vmxnet3_cmd_ring_adv_next2comp(ring);
 173         }
 174 }
 175
 176 static void
 177 vmxnet3_cmd_ring_release(vmxnet3_cmd_ring_t *ring)
 178 {
 179         vmxnet3_cmd_ring_release_mbufs(ring);
 180         rte_free(ring->buf_info);
 181         ring->buf_info = NULL;
 182 }
 183
 184
 185 void
 186 vmxnet3_dev_tx_queue_release(void *txq)
 187 {
 188         vmxnet3_tx_queue_t *tq = txq;
 189
 190         if (tq != NULL) {
 191                 /* Release the cmd_ring */
 192                 vmxnet3_cmd_ring_release(&tq->cmd_ring);
 193         }
 194 }
 195
 196 void
 197 vmxnet3_dev_rx_queue_release(void *rxq)
 198 {
 199         int i;
 200         vmxnet3_rx_queue_t *rq = rxq;
 201
 202         if (rq != NULL) {
 203                 /* Release both the cmd_rings */
 204                 for (i = 0; i < VMXNET3_RX_CMDRING_SIZE; i++)
 205                         vmxnet3_cmd_ring_release(&rq->cmd_ring[i]);
 206         }
 207 }
 208
 209 static void
 210 vmxnet3_dev_tx_queue_reset(void *txq)
 211 {
 212         vmxnet3_tx_queue_t *tq = txq;
 213         struct vmxnet3_cmd_ring *ring = &tq->cmd_ring;
 214         struct vmxnet3_comp_ring *comp_ring = &tq->comp_ring;
 215         int size;
 216
 217         if (tq != NULL) {
 218                 /* Release the cmd_ring mbufs */
 219                 vmxnet3_cmd_ring_release_mbufs(&tq->cmd_ring);
 220         }
 221
 222         /* Tx vmxnet rings structure initialization*/
 223         ring->next2fill = 0;
 224         ring->next2comp = 0;
 225         ring->gen = VMXNET3_INIT_GEN;
 226         comp_ring->next2proc = 0;
 227         comp_ring->gen = VMXNET3_INIT_GEN;
 228
 229         size = sizeof(struct Vmxnet3_TxDesc) * ring->size;
 230         size += sizeof(struct Vmxnet3_TxCompDesc) * comp_ring->size;
 231
 232         memset(ring->base, 0, size);
 233 }
 234
 235 static void
 236 vmxnet3_dev_rx_queue_reset(void *rxq)
 237 {
 238         int i;
 239         vmxnet3_rx_queue_t *rq = rxq;
 240         struct vmxnet3_cmd_ring *ring0, *ring1;
 241         struct vmxnet3_comp_ring *comp_ring;
 242         int size;
 243
 244         if (rq != NULL) {
 245                 /* Release both the cmd_rings mbufs */
 246                 for (i = 0; i < VMXNET3_RX_CMDRING_SIZE; i++)
 247                         vmxnet3_cmd_ring_release_mbufs(&rq->cmd_ring[i]);
 248         }
 249
 250         ring0 = &rq->cmd_ring[0];
 251         ring1 = &rq->cmd_ring[1];
 252         comp_ring = &rq->comp_ring;
 253
 254         /* Rx vmxnet rings structure initialization */
 255         ring0->next2fill = 0;
 256         ring1->next2fill = 0;
 257         ring0->next2comp = 0;
 258         ring1->next2comp = 0;
 259         ring0->gen = VMXNET3_INIT_GEN;
 260         ring1->gen = VMXNET3_INIT_GEN;
 261         comp_ring->next2proc = 0;
 262         comp_ring->gen = VMXNET3_INIT_GEN;
 263
 264         size = sizeof(struct Vmxnet3_RxDesc) * (ring0->size + ring1->size);
 265         size += sizeof(struct Vmxnet3_RxCompDesc) * comp_ring->size;
 266
 267         memset(ring0->base, 0, size);
 268 }
 269
 270 void
 271 vmxnet3_dev_clear_queues(struct rte_eth_dev *dev)
 272 {
 273         unsigned i;
 274
 275         PMD_INIT_FUNC_TRACE();
 276
 277         for (i = 0; i < dev->data->nb_tx_queues; i++) {
 278                 struct vmxnet3_tx_queue *txq = dev->data->tx_queues[i];
 279
 280                 if (txq != NULL) {
 281                         txq->stopped = TRUE;
 282                         vmxnet3_dev_tx_queue_reset(txq);
 283                 }
 284         }
 285
 286         for (i = 0; i < dev->data->nb_rx_queues; i++) {
 287                 struct vmxnet3_rx_queue *rxq = dev->data->rx_queues[i];
 288
 289                 if (rxq != NULL) {
 290                         rxq->stopped = TRUE;
 291                         vmxnet3_dev_rx_queue_reset(rxq);
 292                 }
 293         }
 294 }
 295
 296 static inline void
 297 vmxnet3_tq_tx_complete(vmxnet3_tx_queue_t *txq)
 298 {
 299         int completed = 0;
 300         struct rte_mbuf *mbuf;
 301         vmxnet3_comp_ring_t *comp_ring = &txq->comp_ring;
 302         struct Vmxnet3_TxCompDesc *tcd = (struct Vmxnet3_TxCompDesc *)
 303                 (comp_ring->base + comp_ring->next2proc);
 304
 305         while (tcd->gen == comp_ring->gen) {
 306
 307                 /* Release cmd_ring descriptor and free mbuf */
 308 #ifdef RTE_LIBRTE_VMXNET3_DEBUG_DRIVER
 309                 VMXNET3_ASSERT(txq->cmd_ring.base[tcd->txdIdx].txd.eop == 1);
 310 #endif
 311                 mbuf = txq->cmd_ring.buf_info[tcd->txdIdx].m;
 312                 if (unlikely(mbuf == NULL))
 313                         rte_panic("EOP desc does not point to a valid mbuf");
 314                 else
 315                         rte_pktmbuf_free(mbuf);
 316
 317
 318                 txq->cmd_ring.buf_info[tcd->txdIdx].m = NULL;
 319                 /* Mark the txd for which tcd was generated as completed */
 320                 vmxnet3_cmd_ring_adv_next2comp(&txq->cmd_ring);
 321
 322                 vmxnet3_comp_ring_adv_next2proc(comp_ring);
 323                 tcd = (struct Vmxnet3_TxCompDesc *)(comp_ring->base +
 324                                                     comp_ring->next2proc);
 325                 completed++;
 326         }
 327
 328         PMD_TX_LOG(DEBUG, "Processed %d tx comps & command descs.", completed);
 329 }
 330
 331 uint16_t
 332 vmxnet3_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts,
 333                   uint16_t nb_pkts)
 334 {
 335         uint16_t nb_tx;
 336         Vmxnet3_TxDesc *txd = NULL;
 337         vmxnet3_buf_info_t *tbi = NULL;
 338         struct vmxnet3_hw *hw;
 339         struct rte_mbuf *txm;
 340         vmxnet3_tx_queue_t *txq = tx_queue;
 341
 342         hw = txq->hw;
 343
 344         if (txq->stopped) {
 345                 PMD_TX_LOG(DEBUG, "Tx queue is stopped.");
 346                 return 0;
 347         }
 348
 349         /* Free up the comp_descriptors aggressively */
 350         vmxnet3_tq_tx_complete(txq);
 351
 352         nb_tx = 0;
 353         while (nb_tx < nb_pkts) {
 354
 355                 if (vmxnet3_cmd_ring_desc_avail(&txq->cmd_ring)) {
 356
 357                         txm = tx_pkts[nb_tx];
 358                         /* Don't support scatter packets yet, free them if met */
 359                         if (txm->nb_segs != 1) {
 360                                 PMD_TX_LOG(DEBUG, "Don't support scatter packets yet, drop!");
 361                                 rte_pktmbuf_free(tx_pkts[nb_tx]);
 362                                 txq->stats.drop_total++;
 363
 364                                 nb_tx++;
 365                                 continue;
 366                         }
 367
 368                         /* Needs to minus ether header len */
 369                         if (txm->data_len > (hw->cur_mtu + ETHER_HDR_LEN)) {
 370                                 PMD_TX_LOG(DEBUG, "Packet data_len higher than MTU");
 371                                 rte_pktmbuf_free(tx_pkts[nb_tx]);
 372                                 txq->stats.drop_total++;
 373
 374                                 nb_tx++;
 375                                 continue;
 376                         }
 377
 378                         txd = (Vmxnet3_TxDesc *)(txq->cmd_ring.base + txq->cmd_ring.next2fill);
 379
 380                         /* Fill the tx descriptor */
 381                         tbi = txq->cmd_ring.buf_info + txq->cmd_ring.next2fill;
 382                         tbi->bufPA = RTE_MBUF_DATA_DMA_ADDR(txm);
 383                         txd->addr = tbi->bufPA;
 384                         txd->len = txm->data_len;
 385
 386                         /* Mark the last descriptor as End of Packet. */
 387                         txd->cq = 1;
 388                         txd->eop = 1;
 389
 390                         /* Add VLAN tag if requested */
 391                         if (txm->ol_flags & PKT_TX_VLAN_PKT) {
 392                                 txd->ti = 1;
 393                                 txd->tci = rte_cpu_to_le_16(txm->vlan_tci);
 394                         }
 395
 396                         /* Record current mbuf for freeing it later in tx complete */
 397 #ifdef RTE_LIBRTE_VMXNET3_DEBUG_DRIVER
 398                         VMXNET3_ASSERT(txm);
 399 #endif
 400                         tbi->m = txm;
 401
 402                         /* Set the offloading mode to default */
 403                         txd->hlen = 0;
 404                         txd->om = VMXNET3_OM_NONE;
 405                         txd->msscof = 0;
 406
 407                         /* finally flip the GEN bit of the SOP desc  */
 408                         txd->gen = txq->cmd_ring.gen;
 409                         txq->shared->ctrl.txNumDeferred++;
 410
 411                         /* move to the next2fill descriptor */
 412                         vmxnet3_cmd_ring_adv_next2fill(&txq->cmd_ring);
 413                         nb_tx++;
 414
 415                 } else {
 416                         PMD_TX_LOG(DEBUG, "No free tx cmd desc(s)");
 417                         txq->stats.drop_total += (nb_pkts - nb_tx);
 418                         break;
 419                 }
 420         }
 421
 422         PMD_TX_LOG(DEBUG, "vmxnet3 txThreshold: %u", txq->shared->ctrl.txThreshold);
 423
 424         if (txq->shared->ctrl.txNumDeferred >= txq->shared->ctrl.txThreshold) {
 425
 426                 txq->shared->ctrl.txNumDeferred = 0;
 427                 /* Notify vSwitch that packets are available. */
 428                 VMXNET3_WRITE_BAR0_REG(hw, (VMXNET3_REG_TXPROD + txq->queue_id * VMXNET3_REG_ALIGN),
 429                                        txq->cmd_ring.next2fill);
 430         }
 431
 432         return nb_tx;
 433 }
 434
 435 /*
 436  *  Allocates mbufs and clusters. Post rx descriptors with buffer details
 437  *  so that device can receive packets in those buffers.
 438  *      Ring layout:
 439  *      Among the two rings, 1st ring contains buffers of type 0 and type1.
 440  *      bufs_per_pkt is set such that for non-LRO cases all the buffers required
 441  *      by a frame will fit in 1st ring (1st buf of type0 and rest of type1).
 442  *      2nd ring contains buffers of type 1 alone. Second ring mostly be used
 443  *      only for LRO.
 444  *
 445  */
 446 static inline int
 447 vmxnet3_post_rx_bufs(vmxnet3_rx_queue_t *rxq, uint8_t ring_id)
 448 {
 449         int err = 0;
 450         uint32_t i = 0, val = 0;
 451         struct vmxnet3_cmd_ring *ring = &rxq->cmd_ring[ring_id];
 452
 453         while (vmxnet3_cmd_ring_desc_avail(ring) > 0) {
 454                 struct Vmxnet3_RxDesc *rxd;
 455                 struct rte_mbuf *mbuf;
 456                 vmxnet3_buf_info_t *buf_info = &ring->buf_info[ring->next2fill];
 457
 458                 rxd = (struct Vmxnet3_RxDesc *)(ring->base + ring->next2fill);
 459
 460                 if (ring->rid == 0) {
 461                         /* Usually: One HEAD type buf per packet
 462                          * val = (ring->next2fill % rxq->hw->bufs_per_pkt) ?
 463                          * VMXNET3_RXD_BTYPE_BODY : VMXNET3_RXD_BTYPE_HEAD;
 464                          */
 465
 466                         /* We use single packet buffer so all heads here */
 467                         val = VMXNET3_RXD_BTYPE_HEAD;
 468                 } else {
 469                         /* All BODY type buffers for 2nd ring; which won't be used at all by ESXi */
 470                         val = VMXNET3_RXD_BTYPE_BODY;
 471                 }
 472
 473                 /* Allocate blank mbuf for the current Rx Descriptor */
 474                 mbuf = rte_rxmbuf_alloc(rxq->mp);
 475                 if (mbuf == NULL) {
 476                         PMD_RX_LOG(ERR, "Error allocating mbuf in %s", __func__);
 477                         rxq->stats.rx_buf_alloc_failure++;
 478                         err = ENOMEM;
 479                         break;
 480                 }
 481
 482                 /*
 483                  * Load mbuf pointer into buf_info[ring_size]
 484                  * buf_info structure is equivalent to cookie for virtio-virtqueue
 485                  */
 486                 buf_info->m = mbuf;
 487                 buf_info->len = (uint16_t)(mbuf->buf_len -
 488                                            RTE_PKTMBUF_HEADROOM);
 489                 buf_info->bufPA = RTE_MBUF_DATA_DMA_ADDR_DEFAULT(mbuf);
 490
 491                 /* Load Rx Descriptor with the buffer's GPA */
 492                 rxd->addr = buf_info->bufPA;
 493
 494                 /* After this point rxd->addr MUST not be NULL */
 495                 rxd->btype = val;
 496                 rxd->len = buf_info->len;
 497                 /* Flip gen bit at the end to change ownership */
 498                 rxd->gen = ring->gen;
 499
 500                 vmxnet3_cmd_ring_adv_next2fill(ring);
 501                 i++;
 502         }
 503
 504         /* Return error only if no buffers are posted at present */
 505         if (vmxnet3_cmd_ring_desc_avail(ring) >= (ring->size - 1))
 506                 return -err;
 507         else
 508                 return i;
 509 }
 510
 511 /*
 512  * Process the Rx Completion Ring of given vmxnet3_rx_queue
 513  * for nb_pkts burst and return the number of packets received
 514  */
 515 uint16_t
 516 vmxnet3_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts, uint16_t nb_pkts)
 517 {
 518         uint16_t nb_rx;
 519         uint32_t nb_rxd, idx;
 520         uint8_t ring_idx;
 521         vmxnet3_rx_queue_t *rxq;
 522         Vmxnet3_RxCompDesc *rcd;
 523         vmxnet3_buf_info_t *rbi;
 524         Vmxnet3_RxDesc *rxd;
 525         struct rte_mbuf *rxm = NULL;
 526         struct vmxnet3_hw *hw;
 527
 528         nb_rx = 0;
 529         ring_idx = 0;
 530         nb_rxd = 0;
 531         idx = 0;
 532
 533         rxq = rx_queue;
 534         hw = rxq->hw;
 535
 536         rcd = &rxq->comp_ring.base[rxq->comp_ring.next2proc].rcd;
 537
 538         if (rxq->stopped) {
 539                 PMD_RX_LOG(DEBUG, "Rx queue is stopped.");
 540                 return 0;
 541         }
 542
 543         while (rcd->gen == rxq->comp_ring.gen) {
 544
 545                 if (nb_rx >= nb_pkts)
 546                         break;
 547                 idx = rcd->rxdIdx;
 548                 ring_idx = (uint8_t)((rcd->rqID == rxq->qid1) ? 0 : 1);
 549                 rxd = (Vmxnet3_RxDesc *)rxq->cmd_ring[ring_idx].base + idx;
 550                 rbi = rxq->cmd_ring[ring_idx].buf_info + idx;
 551
 552                 if (rcd->sop != 1 || rcd->eop != 1) {
 553                         rte_pktmbuf_free_seg(rbi->m);
 554
 555                         PMD_RX_LOG(DEBUG, "Packet spread across multiple buffers\n)");
 556                         goto rcd_done;
 557
 558                 } else {
 559
 560                         PMD_RX_LOG(DEBUG, "rxd idx: %d ring idx: %d.", idx, ring_idx);
 561
 562 #ifdef RTE_LIBRTE_VMXNET3_DEBUG_DRIVER
 563                         VMXNET3_ASSERT(rcd->len <= rxd->len);
 564                         VMXNET3_ASSERT(rbi->m);
 565 #endif
 566                         if (rcd->len == 0) {
 567                                 PMD_RX_LOG(DEBUG, "Rx buf was skipped. rxring[%d][%d]\n)",
 568                                            ring_idx, idx);
 569 #ifdef RTE_LIBRTE_VMXNET3_DEBUG_DRIVER
 570                                 VMXNET3_ASSERT(rcd->sop && rcd->eop);
 571 #endif
 572                                 rte_pktmbuf_free_seg(rbi->m);
 573
 574                                 goto rcd_done;
 575                         }
 576
 577                         /* Assuming a packet is coming in a single packet buffer */
 578                         if (rxd->btype != VMXNET3_RXD_BTYPE_HEAD) {
 579                                 PMD_RX_LOG(DEBUG,
 580                                            "Alert : Misbehaving device, incorrect "
 581                                            " buffer type used. iPacket dropped.");
 582                                 rte_pktmbuf_free_seg(rbi->m);
 583                                 goto rcd_done;
 584                         }
 585 #ifdef RTE_LIBRTE_VMXNET3_DEBUG_DRIVER
 586                         VMXNET3_ASSERT(rxd->btype == VMXNET3_RXD_BTYPE_HEAD);
 587 #endif
 588                         /* Get the packet buffer pointer from buf_info */
 589                         rxm = rbi->m;
 590
 591                         /* Clear descriptor associated buf_info to be reused */
 592                         rbi->m = NULL;
 593                         rbi->bufPA = 0;
 594
 595                         /* Update the index that we received a packet */
 596                         rxq->cmd_ring[ring_idx].next2comp = idx;
 597
 598                         /* For RCD with EOP set, check if there is frame error */
 599                         if (rcd->err) {
 600                                 rxq->stats.drop_total++;
 601                                 rxq->stats.drop_err++;
 602
 603                                 if (!rcd->fcs) {
 604                                         rxq->stats.drop_fcs++;
 605                                         PMD_RX_LOG(ERR, "Recv packet dropped due to frame err.");
 606                                 }
 607                                 PMD_RX_LOG(ERR, "Error in received packet rcd#:%d rxd:%d",
 608                                            (int)(rcd - (struct Vmxnet3_RxCompDesc *)
 609                                                  rxq->comp_ring.base), rcd->rxdIdx);
 610                                 rte_pktmbuf_free_seg(rxm);
 611
 612                                 goto rcd_done;
 613                         }
 614
 615                         /* Check for hardware stripped VLAN tag */
 616                         if (rcd->ts) {
 617                                 PMD_RX_LOG(ERR, "Received packet with vlan ID: %d.",
 618                                            rcd->tci);
 619                                 rxm->ol_flags = PKT_RX_VLAN_PKT;
 620 #ifdef RTE_LIBRTE_VMXNET3_DEBUG_DRIVER
 621                                 VMXNET3_ASSERT(rxm &&
 622                                                rte_pktmbuf_mtod(rxm, void *));
 623 #endif
 624                                 /* Copy vlan tag in packet buffer */
 625                                 rxm->vlan_tci = rte_le_to_cpu_16((uint16_t)rcd->tci);
 626                         } else {
 627                                 rxm->ol_flags = 0;
 628                                 rxm->vlan_tci = 0;
 629                         }
 630
 631                         /* Initialize newly received packet buffer */
 632                         rxm->port = rxq->port_id;
 633                         rxm->nb_segs = 1;
 634                         rxm->next = NULL;
 635                         rxm->pkt_len = (uint16_t)rcd->len;
 636                         rxm->data_len = (uint16_t)rcd->len;
 637                         rxm->port = rxq->port_id;
 638                         rxm->data_off = RTE_PKTMBUF_HEADROOM;
 639
 640                         rx_pkts[nb_rx++] = rxm;
 641 rcd_done:
 642                         rxq->cmd_ring[ring_idx].next2comp = idx;
 643                         VMXNET3_INC_RING_IDX_ONLY(rxq->cmd_ring[ring_idx].next2comp, rxq->cmd_ring[ring_idx].size);
 644
 645                         /* It's time to allocate some new buf and renew descriptors */
 646                         vmxnet3_post_rx_bufs(rxq, ring_idx);
 647                         if (unlikely(rxq->shared->ctrl.updateRxProd)) {
 648                                 VMXNET3_WRITE_BAR0_REG(hw, rxprod_reg[ring_idx] + (rxq->queue_id * VMXNET3_REG_ALIGN),
 649                                                        rxq->cmd_ring[ring_idx].next2fill);
 650                         }
 651
 652                         /* Advance to the next descriptor in comp_ring */
 653                         vmxnet3_comp_ring_adv_next2proc(&rxq->comp_ring);
 654
 655                         rcd = &rxq->comp_ring.base[rxq->comp_ring.next2proc].rcd;
 656                         nb_rxd++;
 657                         if (nb_rxd > rxq->cmd_ring[0].size) {
 658                                 PMD_RX_LOG(ERR,
 659                                            "Used up quota of receiving packets,"
 660                                            " relinquish control.");
 661                                 break;
 662                         }
 663                 }
 664         }
 665
 666         return nb_rx;
 667 }
 668
 669 /*
 670  * Create memzone for device rings. malloc can't be used as the physical address is
 671  * needed. If the memzone is already created, then this function returns a ptr
 672  * to the old one.
 673  */
 674 static const struct rte_memzone *
 675 ring_dma_zone_reserve(struct rte_eth_dev *dev, const char *ring_name,
 676                       uint16_t queue_id, uint32_t ring_size, int socket_id)
 677 {
 678         char z_name[RTE_MEMZONE_NAMESIZE];
 679         const struct rte_memzone *mz;
 680
 681         snprintf(z_name, sizeof(z_name), "%s_%s_%d_%d",
 682                         dev->driver->pci_drv.name, ring_name,
 683                         dev->data->port_id, queue_id);
 684
 685         mz = rte_memzone_lookup(z_name);
 686         if (mz)
 687                 return mz;
 688
 689         return rte_memzone_reserve_aligned(z_name, ring_size,
 690                         socket_id, 0, VMXNET3_RING_BA_ALIGN);
 691 }
 692
 693 int
 694 vmxnet3_dev_tx_queue_setup(struct rte_eth_dev *dev,
 695                            uint16_t queue_idx,
 696                            uint16_t nb_desc,
 697                            unsigned int socket_id,
 698                            __attribute__((unused)) const struct rte_eth_txconf *tx_conf)
 699 {
 700         struct vmxnet3_hw     *hw = dev->data->dev_private;
 701         const struct rte_memzone *mz;
 702         struct vmxnet3_tx_queue *txq;
 703         struct vmxnet3_cmd_ring *ring;
 704         struct vmxnet3_comp_ring *comp_ring;
 705         int size;
 706
 707         PMD_INIT_FUNC_TRACE();
 708
 709         if ((tx_conf->txq_flags & ETH_TXQ_FLAGS_NOMULTSEGS) !=
 710             ETH_TXQ_FLAGS_NOMULTSEGS) {
 711                 PMD_INIT_LOG(ERR, "TX Multi segment not support yet");
 712                 return -EINVAL;
 713         }
 714
 715         if ((tx_conf->txq_flags & ETH_TXQ_FLAGS_NOOFFLOADS) !=
 716             ETH_TXQ_FLAGS_NOOFFLOADS) {
 717                 PMD_INIT_LOG(ERR, "TX not support offload function yet");
 718                 return -EINVAL;
 719         }
 720
 721         txq = rte_zmalloc("ethdev_tx_queue", sizeof(struct vmxnet3_tx_queue), CACHE_LINE_SIZE);
 722         if (txq == NULL) {
 723                 PMD_INIT_LOG(ERR, "Can not allocate tx queue structure");
 724                 return -ENOMEM;
 725         }
 726
 727         txq->queue_id = queue_idx;
 728         txq->port_id = dev->data->port_id;
 729         txq->shared = &hw->tqd_start[queue_idx];
 730         txq->hw = hw;
 731         txq->qid = queue_idx;
 732         txq->stopped = TRUE;
 733
 734         ring = &txq->cmd_ring;
 735         comp_ring = &txq->comp_ring;
 736
 737         /* Tx vmxnet ring length should be between 512-4096 */
 738         if (nb_desc < VMXNET3_DEF_TX_RING_SIZE) {
 739                 PMD_INIT_LOG(ERR, "VMXNET3 Tx Ring Size Min: %u",
 740                              VMXNET3_DEF_TX_RING_SIZE);
 741                 return -EINVAL;
 742         } else if (nb_desc > VMXNET3_TX_RING_MAX_SIZE) {
 743                 PMD_INIT_LOG(ERR, "VMXNET3 Tx Ring Size Max: %u",
 744                              VMXNET3_TX_RING_MAX_SIZE);
 745                 return -EINVAL;
 746         } else {
 747                 ring->size = nb_desc;
 748                 ring->size &= ~VMXNET3_RING_SIZE_MASK;
 749         }
 750         comp_ring->size = ring->size;
 751
 752         /* Tx vmxnet rings structure initialization*/
 753         ring->next2fill = 0;
 754         ring->next2comp = 0;
 755         ring->gen = VMXNET3_INIT_GEN;
 756         comp_ring->next2proc = 0;
 757         comp_ring->gen = VMXNET3_INIT_GEN;
 758
 759         size = sizeof(struct Vmxnet3_TxDesc) * ring->size;
 760         size += sizeof(struct Vmxnet3_TxCompDesc) * comp_ring->size;
 761
 762         mz = ring_dma_zone_reserve(dev, "txdesc", queue_idx, size, socket_id);
 763         if (mz == NULL) {
 764                 PMD_INIT_LOG(ERR, "ERROR: Creating queue descriptors zone");
 765                 return -ENOMEM;
 766         }
 767         memset(mz->addr, 0, mz->len);
 768
 769         /* cmd_ring initialization */
 770         ring->base = mz->addr;
 771         ring->basePA = mz->phys_addr;
 772
 773         /* comp_ring initialization */
 774         comp_ring->base = ring->base + ring->size;
 775         comp_ring->basePA = ring->basePA +
 776                 (sizeof(struct Vmxnet3_TxDesc) * ring->size);
 777
 778         /* cmd_ring0 buf_info allocation */
 779         ring->buf_info = rte_zmalloc("tx_ring_buf_info",
 780                                      ring->size * sizeof(vmxnet3_buf_info_t), CACHE_LINE_SIZE);
 781         if (ring->buf_info == NULL) {
 782                 PMD_INIT_LOG(ERR, "ERROR: Creating tx_buf_info structure");
 783                 return -ENOMEM;
 784         }
 785
 786         /* Update the data portion with txq */
 787         dev->data->tx_queues[queue_idx] = txq;
 788
 789         return 0;
 790 }
 791
 792 int
 793 vmxnet3_dev_rx_queue_setup(struct rte_eth_dev *dev,
 794                            uint16_t queue_idx,
 795                            uint16_t nb_desc,
 796                            unsigned int socket_id,
 797                            __attribute__((unused)) const struct rte_eth_rxconf *rx_conf,
 798                            struct rte_mempool *mp)
 799 {
 800         const struct rte_memzone *mz;
 801         struct vmxnet3_rx_queue *rxq;
 802         struct vmxnet3_hw     *hw = dev->data->dev_private;
 803         struct vmxnet3_cmd_ring *ring0, *ring1, *ring;
 804         struct vmxnet3_comp_ring *comp_ring;
 805         int size;
 806         uint8_t i;
 807         char mem_name[32];
 808         uint16_t buf_size;
 809         struct rte_pktmbuf_pool_private *mbp_priv;
 810
 811         PMD_INIT_FUNC_TRACE();
 812
 813         mbp_priv = (struct rte_pktmbuf_pool_private *)
 814                 rte_mempool_get_priv(mp);
 815         buf_size = (uint16_t) (mbp_priv->mbuf_data_room_size -
 816                                RTE_PKTMBUF_HEADROOM);
 817
 818         if (dev->data->dev_conf.rxmode.max_rx_pkt_len > buf_size) {
 819                 PMD_INIT_LOG(ERR, "buf_size = %u, max_pkt_len = %u, "
 820                              "VMXNET3 don't support scatter packets yet",
 821                              buf_size, dev->data->dev_conf.rxmode.max_rx_pkt_len);
 822                 return -EINVAL;
 823         }
 824
 825         rxq = rte_zmalloc("ethdev_rx_queue", sizeof(struct vmxnet3_rx_queue), CACHE_LINE_SIZE);
 826         if (rxq == NULL) {
 827                 PMD_INIT_LOG(ERR, "Can not allocate rx queue structure");
 828                 return -ENOMEM;
 829         }
 830
 831         rxq->mp = mp;
 832         rxq->queue_id = queue_idx;
 833         rxq->port_id = dev->data->port_id;
 834         rxq->shared = &hw->rqd_start[queue_idx];
 835         rxq->hw = hw;
 836         rxq->qid1 = queue_idx;
 837         rxq->qid2 = queue_idx + hw->num_rx_queues;
 838         rxq->stopped = TRUE;
 839
 840         ring0 = &rxq->cmd_ring[0];
 841         ring1 = &rxq->cmd_ring[1];
 842         comp_ring = &rxq->comp_ring;
 843
 844         /* Rx vmxnet rings length should be between 256-4096 */
 845         if (nb_desc < VMXNET3_DEF_RX_RING_SIZE) {
 846                 PMD_INIT_LOG(ERR, "VMXNET3 Rx Ring Size Min: 256");
 847                 return -EINVAL;
 848         } else if (nb_desc > VMXNET3_RX_RING_MAX_SIZE) {
 849                 PMD_INIT_LOG(ERR, "VMXNET3 Rx Ring Size Max: 4096");
 850                 return -EINVAL;
 851         } else {
 852                 ring0->size = nb_desc;
 853                 ring0->size &= ~VMXNET3_RING_SIZE_MASK;
 854                 ring1->size = ring0->size;
 855         }
 856
 857         comp_ring->size = ring0->size + ring1->size;
 858
 859         /* Rx vmxnet rings structure initialization */
 860         ring0->next2fill = 0;
 861         ring1->next2fill = 0;
 862         ring0->next2comp = 0;
 863         ring1->next2comp = 0;
 864         ring0->gen = VMXNET3_INIT_GEN;
 865         ring1->gen = VMXNET3_INIT_GEN;
 866         comp_ring->next2proc = 0;
 867         comp_ring->gen = VMXNET3_INIT_GEN;
 868
 869         size = sizeof(struct Vmxnet3_RxDesc) * (ring0->size + ring1->size);
 870         size += sizeof(struct Vmxnet3_RxCompDesc) * comp_ring->size;
 871
 872         mz = ring_dma_zone_reserve(dev, "rxdesc", queue_idx, size, socket_id);
 873         if (mz == NULL) {
 874                 PMD_INIT_LOG(ERR, "ERROR: Creating queue descriptors zone");
 875                 return -ENOMEM;
 876         }
 877         memset(mz->addr, 0, mz->len);
 878
 879         /* cmd_ring0 initialization */
 880         ring0->base = mz->addr;
 881         ring0->basePA = mz->phys_addr;
 882
 883         /* cmd_ring1 initialization */
 884         ring1->base = ring0->base + ring0->size;
 885         ring1->basePA = ring0->basePA + sizeof(struct Vmxnet3_RxDesc) * ring0->size;
 886
 887         /* comp_ring initialization */
 888         comp_ring->base = ring1->base +  ring1->size;
 889         comp_ring->basePA = ring1->basePA + sizeof(struct Vmxnet3_RxDesc) *
 890                 ring1->size;
 891
 892         /* cmd_ring0-cmd_ring1 buf_info allocation */
 893         for (i = 0; i < VMXNET3_RX_CMDRING_SIZE; i++) {
 894
 895                 ring = &rxq->cmd_ring[i];
 896                 ring->rid = i;
 897                 snprintf(mem_name, sizeof(mem_name), "rx_ring_%d_buf_info", i);
 898
 899                 ring->buf_info = rte_zmalloc(mem_name, ring->size * sizeof(vmxnet3_buf_info_t), CACHE_LINE_SIZE);
 900                 if (ring->buf_info == NULL) {
 901                         PMD_INIT_LOG(ERR, "ERROR: Creating rx_buf_info structure");
 902                         return -ENOMEM;
 903                 }
 904         }
 905
 906         /* Update the data portion with rxq */
 907         dev->data->rx_queues[queue_idx] = rxq;
 908
 909         return 0;
 910 }
 911
 912 /*
 913  * Initializes Receive Unit
 914  * Load mbufs in rx queue in advance
 915  */
 916 int
 917 vmxnet3_dev_rxtx_init(struct rte_eth_dev *dev)
 918 {
 919         struct vmxnet3_hw *hw = dev->data->dev_private;
 920
 921         int i, ret;
 922         uint8_t j;
 923
 924         PMD_INIT_FUNC_TRACE();
 925
 926         for (i = 0; i < hw->num_rx_queues; i++) {
 927                 vmxnet3_rx_queue_t *rxq = dev->data->rx_queues[i];
 928
 929                 for (j = 0; j < VMXNET3_RX_CMDRING_SIZE; j++) {
 930                         /* Passing 0 as alloc_num will allocate full ring */
 931                         ret = vmxnet3_post_rx_bufs(rxq, j);
 932                         if (ret <= 0) {
 933                                 PMD_INIT_LOG(ERR, "ERROR: Posting Rxq: %d buffers ring: %d", i, j);
 934                                 return -ret;
 935                         }
 936                         /* Updating device with the index:next2fill to fill the mbufs for coming packets */
 937                         if (unlikely(rxq->shared->ctrl.updateRxProd)) {
 938                                 VMXNET3_WRITE_BAR0_REG(hw, rxprod_reg[j] + (rxq->queue_id * VMXNET3_REG_ALIGN),
 939                                                        rxq->cmd_ring[j].next2fill);
 940                         }
 941                 }
 942                 rxq->stopped = FALSE;
 943         }
 944
 945         for (i = 0; i < dev->data->nb_tx_queues; i++) {
 946                 struct vmxnet3_tx_queue *txq = dev->data->tx_queues[i];
 947
 948                 txq->stopped = FALSE;
 949         }
 950
 951         return 0;
 952 }
 953
 954 static uint8_t rss_intel_key[40] = {
 955         0x6D, 0x5A, 0x56, 0xDA, 0x25, 0x5B, 0x0E, 0xC2,
 956         0x41, 0x67, 0x25, 0x3D, 0x43, 0xA3, 0x8F, 0xB0,
 957         0xD0, 0xCA, 0x2B, 0xCB, 0xAE, 0x7B, 0x30, 0xB4,
 958         0x77, 0xCB, 0x2D, 0xA3, 0x80, 0x30, 0xF2, 0x0C,
 959         0x6A, 0x42, 0xB7, 0x3B, 0xBE, 0xAC, 0x01, 0xFA,
 960 };
 961
 962 /*
 963  * Configure RSS feature
 964  */
 965 int
 966 vmxnet3_rss_configure(struct rte_eth_dev *dev)
 967 {
 968 #define VMXNET3_RSS_OFFLOAD_ALL ( \
 969                 ETH_RSS_IPV4 | \
 970                 ETH_RSS_IPV4_TCP | \
 971                 ETH_RSS_IPV6 | \
 972                 ETH_RSS_IPV6_TCP)
 973
 974         struct vmxnet3_hw *hw = dev->data->dev_private;
 975         struct VMXNET3_RSSConf *dev_rss_conf;
 976         struct rte_eth_rss_conf *port_rss_conf;
 977         uint64_t rss_hf;
 978         uint8_t i, j;
 979
 980         PMD_INIT_FUNC_TRACE();
 981
 982         dev_rss_conf = hw->rss_conf;
 983         port_rss_conf = &dev->data->dev_conf.rx_adv_conf.rss_conf;
 984
 985         /* loading hashFunc */
 986         dev_rss_conf->hashFunc = VMXNET3_RSS_HASH_FUNC_TOEPLITZ;
 987         /* loading hashKeySize */
 988         dev_rss_conf->hashKeySize = VMXNET3_RSS_MAX_KEY_SIZE;
 989         /* loading indTableSize : Must not exceed VMXNET3_RSS_MAX_IND_TABLE_SIZE (128)*/
 990         dev_rss_conf->indTableSize = (uint16_t)(hw->num_rx_queues * 4);
 991
 992         if (port_rss_conf->rss_key == NULL) {
 993                 /* Default hash key */
 994                 port_rss_conf->rss_key = rss_intel_key;
 995         }
 996
 997         /* loading hashKey */
 998         memcpy(&dev_rss_conf->hashKey[0], port_rss_conf->rss_key, dev_rss_conf->hashKeySize);
 999
1000         /* loading indTable */
1001         for (i = 0, j = 0; i < dev_rss_conf->indTableSize; i++, j++) {
1002                 if (j == dev->data->nb_rx_queues)
1003                         j = 0;
1004                 dev_rss_conf->indTable[i] = j;
1005         }
1006
1007         /* loading hashType */
1008         dev_rss_conf->hashType = 0;
1009         rss_hf = port_rss_conf->rss_hf & VMXNET3_RSS_OFFLOAD_ALL;
1010         if (rss_hf & ETH_RSS_IPV4)
1011                 dev_rss_conf->hashType |= VMXNET3_RSS_HASH_TYPE_IPV4;
1012         if (rss_hf & ETH_RSS_IPV4_TCP)
1013                 dev_rss_conf->hashType |= VMXNET3_RSS_HASH_TYPE_TCP_IPV4;
1014         if (rss_hf & ETH_RSS_IPV6)
1015                 dev_rss_conf->hashType |= VMXNET3_RSS_HASH_TYPE_IPV6;
1016         if (rss_hf & ETH_RSS_IPV6_TCP)
1017                 dev_rss_conf->hashType |= VMXNET3_RSS_HASH_TYPE_TCP_IPV6;
1018
1019         return VMXNET3_SUCCESS;
1020 }
1021
1022 /*
1023  * Configure VLAN Filter feature
1024  */
1025 int
1026 vmxnet3_vlan_configure(struct rte_eth_dev *dev)
1027 {
1028         uint8_t i;
1029         struct vmxnet3_hw *hw = dev->data->dev_private;
1030         uint32_t *vf_table = hw->shared->devRead.rxFilterConf.vfTable;
1031
1032         PMD_INIT_FUNC_TRACE();
1033
1034         /* Verify if this tag is already set */
1035         for (i = 0; i < VMXNET3_VFT_SIZE; i++) {
1036                 /* Filter all vlan tags out by default */
1037                 vf_table[i] = 0;
1038                 /* To-Do: Provide another routine in dev_ops for user config */
1039
1040                 PMD_INIT_LOG(DEBUG, "Registering VLAN portid: %"PRIu8" tag %u",
1041                                         dev->data->port_id, vf_table[i]);
1042         }
1043
1044         return VMXNET3_SUCCESS;
1045 }