4 * Copyright(c) 2010-2014 Intel Corporation. All rights reserved.
7 * Redistribution and use in source and binary forms, with or without
8 * modification, are permitted provided that the following conditions
11 * * Redistributions of source code must retain the above copyright
12 * notice, this list of conditions and the following disclaimer.
13 * * Redistributions in binary form must reproduce the above copyright
14 * notice, this list of conditions and the following disclaimer in
15 * the documentation and/or other materials provided with the
17 * * Neither the name of Intel Corporation nor the names of its
18 * contributors may be used to endorse or promote products derived
19 * from this software without specific prior written permission.
21 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
22 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
23 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
24 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
25 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
26 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
27 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
28 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
29 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
30 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
31 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
34 #include <sys/queue.h>
45 #include <rte_byteorder.h>
46 #include <rte_common.h>
47 #include <rte_cycles.h>
49 #include <rte_debug.h>
50 #include <rte_interrupts.h>
52 #include <rte_memory.h>
53 #include <rte_memzone.h>
54 #include <rte_launch.h>
55 #include <rte_tailq.h>
57 #include <rte_per_lcore.h>
58 #include <rte_lcore.h>
59 #include <rte_atomic.h>
60 #include <rte_branch_prediction.h>
62 #include <rte_mempool.h>
63 #include <rte_malloc.h>
65 #include <rte_ether.h>
66 #include <rte_ethdev.h>
67 #include <rte_prefetch.h>
71 #include <rte_string_fns.h>
72 #include <rte_errno.h>
74 #include "vmxnet3/vmxnet3_defs.h"
75 #include "vmxnet3_ring.h"
77 #include "vmxnet3_logs.h"
78 #include "vmxnet3_ethdev.h"
81 #define RTE_MBUF_DATA_DMA_ADDR(mb) \
82 (uint64_t) ((mb)->buf_physaddr + (uint64_t)((char *)((mb)->pkt.data) - \
83 (char *)(mb)->buf_addr))
85 #define RTE_MBUF_DATA_DMA_ADDR_DEFAULT(mb) \
86 (uint64_t) ((mb)->buf_physaddr + RTE_PKTMBUF_HEADROOM)
88 static uint32_t rxprod_reg[2] = {VMXNET3_REG_RXPROD, VMXNET3_REG_RXPROD2};
90 static inline int vmxnet3_post_rx_bufs(vmxnet3_rx_queue_t* , uint8_t);
91 static inline void vmxnet3_tq_tx_complete(vmxnet3_tx_queue_t *);
92 #ifdef RTE_LIBRTE_VMXNET3_DEBUG_DRIVER_NOT_USED
93 static void vmxnet3_rxq_dump(struct vmxnet3_rx_queue *);
94 static void vmxnet3_txq_dump(struct vmxnet3_tx_queue *);
97 static inline struct rte_mbuf *
98 rte_rxmbuf_alloc(struct rte_mempool *mp)
102 m = __rte_mbuf_raw_alloc(mp);
103 __rte_mbuf_sanity_check_raw(m, RTE_MBUF_PKT, 0);
107 #ifdef RTE_LIBRTE_VMXNET3_DEBUG_DRIVER_NOT_USED
109 vmxnet3_rxq_dump(struct vmxnet3_rx_queue *rxq)
115 PMD_RX_LOG(DEBUG, "RXQ: cmd0 base : 0x%p cmd1 base : 0x%p comp ring base : 0x%p.\n",
116 rxq->cmd_ring[0].base, rxq->cmd_ring[1].base, rxq->comp_ring.base);
117 PMD_RX_LOG(DEBUG, "RXQ: cmd0 basePA : 0x%lx cmd1 basePA : 0x%lx comp ring basePA : 0x%lx.\n",
118 (unsigned long)rxq->cmd_ring[0].basePA, (unsigned long)rxq->cmd_ring[1].basePA,
119 (unsigned long)rxq->comp_ring.basePA);
121 avail = vmxnet3_cmd_ring_desc_avail(&rxq->cmd_ring[0]);
122 PMD_RX_LOG(DEBUG, "RXQ:cmd0: size=%u; free=%u; next2proc=%u; queued=%u\n",
123 (uint32_t)rxq->cmd_ring[0].size, avail, rxq->comp_ring.next2proc,
124 rxq->cmd_ring[0].size - avail);
126 avail = vmxnet3_cmd_ring_desc_avail(&rxq->cmd_ring[1]);
127 PMD_RX_LOG(DEBUG, "RXQ:cmd1 size=%u; free=%u; next2proc=%u; queued=%u\n",
128 (uint32_t)rxq->cmd_ring[1].size, avail, rxq->comp_ring.next2proc,
129 rxq->cmd_ring[1].size - avail);
134 vmxnet3_txq_dump(struct vmxnet3_tx_queue *txq)
140 PMD_TX_LOG(DEBUG, "TXQ: cmd base : 0x%p comp ring base : 0x%p.\n",
141 txq->cmd_ring.base, txq->comp_ring.base);
142 PMD_TX_LOG(DEBUG, "TXQ: cmd basePA : 0x%lx comp ring basePA : 0x%lx.\n",
143 (unsigned long)txq->cmd_ring.basePA, (unsigned long)txq->comp_ring.basePA);
145 avail = vmxnet3_cmd_ring_desc_avail(&txq->cmd_ring);
146 PMD_TX_LOG(DEBUG, "TXQ: size=%u; free=%u; next2proc=%u; queued=%u\n",
147 (uint32_t)txq->cmd_ring.size, avail,
148 txq->comp_ring.next2proc, txq->cmd_ring.size - avail);
153 vmxnet3_cmd_ring_release(vmxnet3_cmd_ring_t *ring)
155 while (ring->next2comp != ring->next2fill) {
156 /* No need to worry about tx desc ownership, device is quiesced by now. */
157 vmxnet3_buf_info_t *buf_info = ring->buf_info + ring->next2comp;
159 rte_pktmbuf_free(buf_info->m);
164 vmxnet3_cmd_ring_adv_next2comp(ring);
166 rte_free(ring->buf_info);
170 vmxnet3_dev_tx_queue_release(void *txq)
172 vmxnet3_tx_queue_t *tq = txq;
174 /* Release the cmd_ring */
175 vmxnet3_cmd_ring_release(&tq->cmd_ring);
180 vmxnet3_dev_rx_queue_release(void *rxq)
183 vmxnet3_rx_queue_t *rq = rxq;
185 /* Release both the cmd_rings */
186 for (i = 0; i < VMXNET3_RX_CMDRING_SIZE; i++)
187 vmxnet3_cmd_ring_release(&rq->cmd_ring[i]);
192 vmxnet3_dev_clear_queues(struct rte_eth_dev *dev)
196 PMD_INIT_FUNC_TRACE();
198 for (i = 0; i < dev->data->nb_tx_queues; i++) {
199 struct vmxnet3_tx_queue *txq = dev->data->tx_queues[i];
202 vmxnet3_dev_tx_queue_release(txq);
206 for (i = 0; i < dev->data->nb_rx_queues; i++) {
207 struct vmxnet3_rx_queue *rxq = dev->data->rx_queues[i];
210 vmxnet3_dev_rx_queue_release(rxq);
216 vmxnet3_tq_tx_complete(vmxnet3_tx_queue_t *txq)
219 struct rte_mbuf *mbuf;
220 vmxnet3_comp_ring_t *comp_ring = &txq->comp_ring;
221 struct Vmxnet3_TxCompDesc *tcd = (struct Vmxnet3_TxCompDesc *)
222 (comp_ring->base + comp_ring->next2proc);
224 while (tcd->gen == comp_ring->gen) {
226 /* Release cmd_ring descriptor and free mbuf */
227 #ifdef RTE_LIBRTE_VMXNET3_DEBUG_DRIVER
228 VMXNET3_ASSERT(txq->cmd_ring.base[tcd->txdIdx].txd.eop == 1);
230 mbuf = txq->cmd_ring.buf_info[tcd->txdIdx].m;
231 if (unlikely(mbuf == NULL))
232 rte_panic("EOP desc does not point to a valid mbuf");
234 rte_pktmbuf_free(mbuf);
237 txq->cmd_ring.buf_info[tcd->txdIdx].m = NULL;
238 /* Mark the txd for which tcd was generated as completed */
239 vmxnet3_cmd_ring_adv_next2comp(&txq->cmd_ring);
241 vmxnet3_comp_ring_adv_next2proc(comp_ring);
242 tcd = (struct Vmxnet3_TxCompDesc *)(comp_ring->base +
243 comp_ring->next2proc);
247 PMD_TX_LOG(DEBUG, "Processed %d tx comps & command descs.\n", completed);
251 vmxnet3_xmit_pkts( void *tx_queue, struct rte_mbuf **tx_pkts,
255 Vmxnet3_TxDesc *txd = NULL;
256 vmxnet3_buf_info_t *tbi = NULL;
257 struct vmxnet3_hw *hw;
258 struct rte_mbuf *txm;
259 vmxnet3_tx_queue_t *txq = tx_queue;
264 PMD_TX_LOG(DEBUG, "Tx queue is stopped.\n");
268 /* Free up the comp_descriptors aggressively */
269 vmxnet3_tq_tx_complete(txq);
272 while(nb_tx < nb_pkts) {
274 if(vmxnet3_cmd_ring_desc_avail(&txq->cmd_ring)) {
276 txm = tx_pkts[nb_tx];
277 /* Don't support scatter packets yet, free them if met */
278 if (txm->pkt.nb_segs != 1) {
279 PMD_TX_LOG(DEBUG, "Don't support scatter packets yet, drop!\n");
280 rte_pktmbuf_free(tx_pkts[nb_tx]);
281 txq->stats.drop_total++;
287 /* Needs to minus ether header len */
288 if(txm->pkt.data_len > (hw->cur_mtu + ETHER_HDR_LEN)) {
289 PMD_TX_LOG(DEBUG, "Packet data_len higher than MTU\n");
290 rte_pktmbuf_free(tx_pkts[nb_tx]);
291 txq->stats.drop_total++;
297 txd = (Vmxnet3_TxDesc *)(txq->cmd_ring.base + txq->cmd_ring.next2fill);
299 /* Fill the tx descriptor */
300 tbi = txq->cmd_ring.buf_info + txq->cmd_ring.next2fill;
301 tbi->bufPA = RTE_MBUF_DATA_DMA_ADDR(txm);
302 txd->addr = tbi->bufPA;
303 txd->len = txm->pkt.data_len;
305 /* Mark the last descriptor as End of Packet. */
309 /* Record current mbuf for freeing it later in tx complete */
310 #ifdef RTE_LIBRTE_VMXNET3_DEBUG_DRIVER
315 /* Set the offloading mode to default */
317 txd->om = VMXNET3_OM_NONE;
320 /* finally flip the GEN bit of the SOP desc */
321 txd->gen = txq->cmd_ring.gen;
322 txq->shared->ctrl.txNumDeferred++;
324 /* move to the next2fill descriptor */
325 vmxnet3_cmd_ring_adv_next2fill(&txq->cmd_ring);
329 PMD_TX_LOG(DEBUG, "No free tx cmd desc(s)\n");
330 txq->stats.drop_total += (nb_pkts - nb_tx);
335 PMD_TX_LOG(DEBUG, "vmxnet3 txThreshold: %u", txq->shared->ctrl.txThreshold);
337 if (txq->shared->ctrl.txNumDeferred >= txq->shared->ctrl.txThreshold) {
339 txq->shared->ctrl.txNumDeferred = 0;
340 /* Notify vSwitch that packets are available. */
341 VMXNET3_WRITE_BAR0_REG(hw, (VMXNET3_REG_TXPROD + txq->queue_id * VMXNET3_REG_ALIGN),
342 txq->cmd_ring.next2fill);
349 * Allocates mbufs and clusters. Post rx descriptors with buffer details
350 * so that device can receive packets in those buffers.
352 * Among the two rings, 1st ring contains buffers of type 0 and type1.
353 * bufs_per_pkt is set such that for non-LRO cases all the buffers required
354 * by a frame will fit in 1st ring (1st buf of type0 and rest of type1).
355 * 2nd ring contains buffers of type 1 alone. Second ring mostly be used
360 vmxnet3_post_rx_bufs(vmxnet3_rx_queue_t* rxq, uint8_t ring_id)
363 uint32_t i = 0, val = 0;
364 struct vmxnet3_cmd_ring *ring = &rxq->cmd_ring[ring_id];
366 while (vmxnet3_cmd_ring_desc_avail(ring) > 0) {
368 struct Vmxnet3_RxDesc *rxd;
369 struct rte_mbuf *mbuf;
370 vmxnet3_buf_info_t *buf_info = &ring->buf_info[ring->next2fill];
371 rxd = (struct Vmxnet3_RxDesc *)(ring->base + ring->next2fill);
373 if (ring->rid == 0) {
374 /* Usually: One HEAD type buf per packet
375 * val = (ring->next2fill % rxq->hw->bufs_per_pkt) ?
376 * VMXNET3_RXD_BTYPE_BODY : VMXNET3_RXD_BTYPE_HEAD;
379 /* We use single packet buffer so all heads here */
380 val = VMXNET3_RXD_BTYPE_HEAD;
382 /* All BODY type buffers for 2nd ring; which won't be used at all by ESXi */
383 val = VMXNET3_RXD_BTYPE_BODY;
386 /* Allocate blank mbuf for the current Rx Descriptor */
387 mbuf = rte_rxmbuf_alloc(rxq->mp);
389 PMD_RX_LOG(ERR, "Error allocating mbuf in %s\n", __func__);
390 rxq->stats.rx_buf_alloc_failure++;
396 * Load mbuf pointer into buf_info[ring_size]
397 * buf_info structure is equivalent to cookie for virtio-virtqueue
400 buf_info->len = (uint16_t)(mbuf->buf_len -
401 RTE_PKTMBUF_HEADROOM);
402 buf_info->bufPA = RTE_MBUF_DATA_DMA_ADDR_DEFAULT(mbuf);
404 /* Load Rx Descriptor with the buffer's GPA */
405 rxd->addr = buf_info->bufPA;
407 /* After this point rxd->addr MUST not be NULL */
409 rxd->len = buf_info->len;
410 /* Flip gen bit at the end to change ownership */
411 rxd->gen = ring->gen;
413 vmxnet3_cmd_ring_adv_next2fill(ring);
417 /* Return error only if no buffers are posted at present */
418 if (vmxnet3_cmd_ring_desc_avail(ring) >= (ring->size -1))
425 * Process the Rx Completion Ring of given vmxnet3_rx_queue
426 * for nb_pkts burst and return the number of packets received
429 vmxnet3_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts, uint16_t nb_pkts)
432 uint32_t nb_rxd, idx;
434 vmxnet3_rx_queue_t *rxq;
435 Vmxnet3_RxCompDesc *rcd;
436 vmxnet3_buf_info_t *rbi;
438 struct rte_mbuf *rxm = NULL;
439 struct vmxnet3_hw *hw;
449 rcd = &rxq->comp_ring.base[rxq->comp_ring.next2proc].rcd;
452 PMD_RX_LOG(DEBUG, "Rx queue is stopped.\n");
456 while (rcd->gen == rxq->comp_ring.gen) {
461 ring_idx = (uint8_t)((rcd->rqID == rxq->qid1) ? 0 : 1);
462 rxd = (Vmxnet3_RxDesc *)rxq->cmd_ring[ring_idx].base + idx;
463 rbi = rxq->cmd_ring[ring_idx].buf_info + idx;
465 if(rcd->sop !=1 || rcd->eop != 1) {
466 rte_pktmbuf_free_seg(rbi->m);
468 PMD_RX_LOG(DEBUG, "Packet spread across multiple buffers\n)");
473 PMD_RX_LOG(DEBUG, "rxd idx: %d ring idx: %d.\n", idx, ring_idx);
475 #ifdef RTE_LIBRTE_VMXNET3_DEBUG_DRIVER
476 VMXNET3_ASSERT(rcd->len <= rxd->len);
477 VMXNET3_ASSERT(rbi->m);
480 PMD_RX_LOG(DEBUG, "Rx buf was skipped. rxring[%d][%d]\n)",
482 #ifdef RTE_LIBRTE_VMXNET3_DEBUG_DRIVER
483 VMXNET3_ASSERT(rcd->sop && rcd->eop);
485 rte_pktmbuf_free_seg(rbi->m);
490 /* Assuming a packet is coming in a single packet buffer */
491 if (rxd->btype != VMXNET3_RXD_BTYPE_HEAD) {
492 PMD_RX_LOG(DEBUG, "Alert : Misbehaving device, incorrect "
493 " buffer type used. iPacket dropped.\n");
494 rte_pktmbuf_free_seg(rbi->m);
497 #ifdef RTE_LIBRTE_VMXNET3_DEBUG_DRIVER
498 VMXNET3_ASSERT(rxd->btype == VMXNET3_RXD_BTYPE_HEAD);
500 /* Get the packet buffer pointer from buf_info */
503 /* Clear descriptor associated buf_info to be reused */
507 /* Update the index that we received a packet */
508 rxq->cmd_ring[ring_idx].next2comp = idx;
510 /* For RCD with EOP set, check if there is frame error */
512 rxq->stats.drop_total++;
513 rxq->stats.drop_err++;
516 rxq->stats.drop_fcs++;
517 PMD_RX_LOG(ERR, "Recv packet dropped due to frame err.\n");
519 PMD_RX_LOG(ERR, "Error in received packet rcd#:%d rxd:%d\n",
520 (int)(rcd - (struct Vmxnet3_RxCompDesc *)
521 rxq->comp_ring.base), rcd->rxdIdx);
522 rte_pktmbuf_free_seg(rxm);
527 /* Check for hardware stripped VLAN tag */
530 PMD_RX_LOG(ERR, "Received packet with vlan ID: %d.\n",
532 rxm->ol_flags = PKT_RX_VLAN_PKT;
534 #ifdef RTE_LIBRTE_VMXNET3_DEBUG_DRIVER
535 VMXNET3_ASSERT(rxm &&
536 rte_pktmbuf_mtod(rxm, void *));
538 //Copy vlan tag in packet buffer
539 rxm->pkt.vlan_macip.f.vlan_tci =
540 rte_le_to_cpu_16((uint16_t)rcd->tci);
545 /* Initialize newly received packet buffer */
546 rxm->pkt.in_port = rxq->port_id;
547 rxm->pkt.nb_segs = 1;
548 rxm->pkt.next = NULL;
549 rxm->pkt.pkt_len = (uint16_t)rcd->len;
550 rxm->pkt.data_len = (uint16_t)rcd->len;
551 rxm->pkt.in_port = rxq->port_id;
552 rxm->pkt.vlan_macip.f.vlan_tci = 0;
553 rxm->pkt.data = (char *)rxm->buf_addr + RTE_PKTMBUF_HEADROOM;
555 rx_pkts[nb_rx++] = rxm;
558 rxq->cmd_ring[ring_idx].next2comp = idx;
559 VMXNET3_INC_RING_IDX_ONLY(rxq->cmd_ring[ring_idx].next2comp, rxq->cmd_ring[ring_idx].size);
561 /* It's time to allocate some new buf and renew descriptors */
562 vmxnet3_post_rx_bufs(rxq, ring_idx);
563 if (unlikely(rxq->shared->ctrl.updateRxProd)) {
564 VMXNET3_WRITE_BAR0_REG(hw, rxprod_reg[ring_idx] + (rxq->queue_id * VMXNET3_REG_ALIGN),
565 rxq->cmd_ring[ring_idx].next2fill);
568 /* Advance to the next descriptor in comp_ring */
569 vmxnet3_comp_ring_adv_next2proc(&rxq->comp_ring);
571 rcd = &rxq->comp_ring.base[rxq->comp_ring.next2proc].rcd;
573 if (nb_rxd > rxq->cmd_ring[0].size) {
574 PMD_RX_LOG(ERR, "Used up quota of receiving packets,"
575 " relinquish control.\n");
585 * Create memzone for device rings. malloc can't be used as the physical address is
586 * needed. If the memzone is already created, then this function returns a ptr
589 static const struct rte_memzone *
590 ring_dma_zone_reserve(struct rte_eth_dev *dev, const char *ring_name,
591 uint16_t queue_id, uint32_t ring_size, int socket_id)
593 char z_name[RTE_MEMZONE_NAMESIZE];
594 const struct rte_memzone *mz;
596 snprintf(z_name, sizeof(z_name), "%s_%s_%d_%d",
597 dev->driver->pci_drv.name, ring_name,
598 dev->data->port_id, queue_id);
600 mz = rte_memzone_lookup(z_name);
604 return rte_memzone_reserve_aligned(z_name, ring_size,
605 socket_id, 0, VMXNET3_RING_BA_ALIGN);
609 vmxnet3_dev_tx_queue_setup(struct rte_eth_dev *dev,
612 unsigned int socket_id,
613 __attribute__((unused)) const struct rte_eth_txconf *tx_conf)
615 const struct rte_memzone *mz;
616 struct vmxnet3_tx_queue *txq;
617 struct vmxnet3_hw *hw;
618 struct vmxnet3_cmd_ring *ring;
619 struct vmxnet3_comp_ring *comp_ring;
622 PMD_INIT_FUNC_TRACE();
623 hw = VMXNET3_DEV_PRIVATE_TO_HW(dev->data->dev_private);
625 if ((tx_conf->txq_flags & ETH_TXQ_FLAGS_NOMULTSEGS) !=
626 ETH_TXQ_FLAGS_NOMULTSEGS) {
627 PMD_INIT_LOG(ERR, "TX Multi segment not support yet\n");
631 if ((tx_conf->txq_flags & ETH_TXQ_FLAGS_NOOFFLOADS) !=
632 ETH_TXQ_FLAGS_NOOFFLOADS) {
633 PMD_INIT_LOG(ERR, "TX not support offload function yet\n");
637 txq = rte_zmalloc("ethdev_tx_queue", sizeof(struct vmxnet3_tx_queue), CACHE_LINE_SIZE);
639 PMD_INIT_LOG(ERR, "Can not allocate tx queue structure\n");
643 txq->queue_id = queue_idx;
644 txq->port_id = dev->data->port_id;
645 txq->shared = &hw->tqd_start[queue_idx];
647 txq->qid = queue_idx;
650 ring = &txq->cmd_ring;
651 comp_ring = &txq->comp_ring;
653 /* Tx vmxnet ring length should be between 512-4096 */
654 if (nb_desc < VMXNET3_DEF_TX_RING_SIZE) {
655 PMD_INIT_LOG(ERR, "VMXNET3 Tx Ring Size Min: %u\n",
656 VMXNET3_DEF_TX_RING_SIZE);
658 } else if (nb_desc > VMXNET3_TX_RING_MAX_SIZE) {
659 PMD_INIT_LOG(ERR, "VMXNET3 Tx Ring Size Max: %u\n",
660 VMXNET3_TX_RING_MAX_SIZE);
663 ring->size = nb_desc;
664 ring->size &= ~VMXNET3_RING_SIZE_MASK;
666 comp_ring->size = ring->size;
668 /* Tx vmxnet rings structure initialization*/
671 ring->gen = VMXNET3_INIT_GEN;
672 comp_ring->next2proc = 0;
673 comp_ring->gen = VMXNET3_INIT_GEN;
675 size = sizeof(struct Vmxnet3_TxDesc) * ring->size;
676 size += sizeof(struct Vmxnet3_TxCompDesc) * comp_ring->size;
678 mz = ring_dma_zone_reserve(dev, "txdesc", queue_idx, size, socket_id);
680 PMD_INIT_LOG(ERR, "ERROR: Creating queue descriptors zone\n");
683 memset(mz->addr, 0, mz->len);
685 /* cmd_ring initialization */
686 ring->base = mz->addr;
687 ring->basePA = mz->phys_addr;
689 /* comp_ring initialization */
690 comp_ring->base = ring->base + ring->size;
691 comp_ring->basePA = ring->basePA +
692 (sizeof(struct Vmxnet3_TxDesc) * ring->size);
694 /* cmd_ring0 buf_info allocation */
695 ring->buf_info = rte_zmalloc("tx_ring_buf_info",
696 ring->size * sizeof(vmxnet3_buf_info_t), CACHE_LINE_SIZE);
697 if (ring->buf_info == NULL) {
698 PMD_INIT_LOG(ERR, "ERROR: Creating tx_buf_info structure\n");
702 /* Update the data portion with txq */
703 dev->data->tx_queues[queue_idx] = txq;
709 vmxnet3_dev_rx_queue_setup(struct rte_eth_dev *dev,
712 unsigned int socket_id,
713 __attribute__((unused)) const struct rte_eth_rxconf *rx_conf,
714 struct rte_mempool *mp)
716 const struct rte_memzone *mz;
717 struct vmxnet3_rx_queue *rxq;
718 struct vmxnet3_hw *hw;
719 struct vmxnet3_cmd_ring *ring0, *ring1, *ring;
720 struct vmxnet3_comp_ring *comp_ring;
725 struct rte_pktmbuf_pool_private *mbp_priv;
727 PMD_INIT_FUNC_TRACE();
728 hw = VMXNET3_DEV_PRIVATE_TO_HW(dev->data->dev_private);
730 mbp_priv = (struct rte_pktmbuf_pool_private *)
731 rte_mempool_get_priv(mp);
732 buf_size = (uint16_t) (mbp_priv->mbuf_data_room_size -
733 RTE_PKTMBUF_HEADROOM);
735 if (dev->data->dev_conf.rxmode.max_rx_pkt_len > buf_size) {
736 PMD_INIT_LOG(ERR, "buf_size = %u, max_pkt_len = %u, "
737 "VMXNET3 don't support scatter packets yet\n",
738 buf_size, dev->data->dev_conf.rxmode.max_rx_pkt_len);
742 rxq = rte_zmalloc("ethdev_rx_queue", sizeof(struct vmxnet3_rx_queue), CACHE_LINE_SIZE);
744 PMD_INIT_LOG(ERR, "Can not allocate rx queue structure\n");
749 rxq->queue_id = queue_idx;
750 rxq->port_id = dev->data->port_id;
751 rxq->shared = &hw->rqd_start[queue_idx];
753 rxq->qid1 = queue_idx;
754 rxq->qid2 = queue_idx + hw->num_rx_queues;
757 ring0 = &rxq->cmd_ring[0];
758 ring1 = &rxq->cmd_ring[1];
759 comp_ring = &rxq->comp_ring;
761 /* Rx vmxnet rings length should be between 256-4096 */
762 if(nb_desc < VMXNET3_DEF_RX_RING_SIZE) {
763 PMD_INIT_LOG(ERR, "VMXNET3 Rx Ring Size Min: 256\n");
765 } else if(nb_desc > VMXNET3_RX_RING_MAX_SIZE) {
766 PMD_INIT_LOG(ERR, "VMXNET3 Rx Ring Size Max: 4096\n");
769 ring0->size = nb_desc;
770 ring0->size &= ~VMXNET3_RING_SIZE_MASK;
771 ring1->size = ring0->size;
774 comp_ring->size = ring0->size + ring1->size;
776 /* Rx vmxnet rings structure initialization */
777 ring0->next2fill = 0;
778 ring1->next2fill = 0;
779 ring0->next2comp = 0;
780 ring1->next2comp = 0;
781 ring0->gen = VMXNET3_INIT_GEN;
782 ring1->gen = VMXNET3_INIT_GEN;
783 comp_ring->next2proc = 0;
784 comp_ring->gen = VMXNET3_INIT_GEN;
786 size = sizeof(struct Vmxnet3_RxDesc) * (ring0->size + ring1->size);
787 size += sizeof(struct Vmxnet3_RxCompDesc) * comp_ring->size;
789 mz = ring_dma_zone_reserve(dev, "rxdesc", queue_idx, size, socket_id);
791 PMD_INIT_LOG(ERR, "ERROR: Creating queue descriptors zone\n");
794 memset(mz->addr, 0, mz->len);
796 /* cmd_ring0 initialization */
797 ring0->base = mz->addr;
798 ring0->basePA = mz->phys_addr;
800 /* cmd_ring1 initialization */
801 ring1->base = ring0->base + ring0->size;
802 ring1->basePA = ring0->basePA + sizeof(struct Vmxnet3_RxDesc) * ring0->size;
804 /* comp_ring initialization */
805 comp_ring->base = ring1->base + ring1->size;
806 comp_ring->basePA = ring1->basePA + sizeof(struct Vmxnet3_RxDesc) *
809 /* cmd_ring0-cmd_ring1 buf_info allocation */
810 for(i = 0; i < VMXNET3_RX_CMDRING_SIZE; i++) {
812 ring = &rxq->cmd_ring[i];
814 snprintf(mem_name, sizeof(mem_name), "rx_ring_%d_buf_info", i);
816 ring->buf_info = rte_zmalloc(mem_name, ring->size * sizeof(vmxnet3_buf_info_t), CACHE_LINE_SIZE);
817 if (ring->buf_info == NULL) {
818 PMD_INIT_LOG(ERR, "ERROR: Creating rx_buf_info structure\n");
823 /* Update the data portion with rxq */
824 dev->data->rx_queues[queue_idx] = rxq;
830 * Initializes Receive Unit
831 * Load mbufs in rx queue in advance
834 vmxnet3_dev_rxtx_init(struct rte_eth_dev *dev)
836 struct vmxnet3_hw *hw;
840 PMD_INIT_FUNC_TRACE();
841 hw = VMXNET3_DEV_PRIVATE_TO_HW(dev->data->dev_private);
843 for (i = 0; i < hw->num_rx_queues; i++) {
845 vmxnet3_rx_queue_t *rxq = dev->data->rx_queues[i];
846 for(j = 0;j < VMXNET3_RX_CMDRING_SIZE;j++) {
847 /* Passing 0 as alloc_num will allocate full ring */
848 ret = vmxnet3_post_rx_bufs(rxq, j);
850 PMD_INIT_LOG(ERR, "ERROR: Posting Rxq: %d buffers ring: %d\n", i, j);
853 /* Updating device with the index:next2fill to fill the mbufs for coming packets */
854 if (unlikely(rxq->shared->ctrl.updateRxProd)) {
855 VMXNET3_WRITE_BAR0_REG(hw, rxprod_reg[j] + (rxq->queue_id * VMXNET3_REG_ALIGN),
856 rxq->cmd_ring[j].next2fill);
859 rxq->stopped = FALSE;
862 for (i = 0; i < dev->data->nb_tx_queues; i++) {
863 struct vmxnet3_tx_queue *txq = dev->data->tx_queues[i];
864 txq->stopped = FALSE;
870 static uint8_t rss_intel_key[40] = {
871 0x6D, 0x5A, 0x56, 0xDA, 0x25, 0x5B, 0x0E, 0xC2,
872 0x41, 0x67, 0x25, 0x3D, 0x43, 0xA3, 0x8F, 0xB0,
873 0xD0, 0xCA, 0x2B, 0xCB, 0xAE, 0x7B, 0x30, 0xB4,
874 0x77, 0xCB, 0x2D, 0xA3, 0x80, 0x30, 0xF2, 0x0C,
875 0x6A, 0x42, 0xB7, 0x3B, 0xBE, 0xAC, 0x01, 0xFA,
879 * Configure RSS feature
882 vmxnet3_rss_configure(struct rte_eth_dev *dev)
884 #define VMXNET3_RSS_OFFLOAD_ALL ( \
890 struct vmxnet3_hw *hw;
891 struct VMXNET3_RSSConf *dev_rss_conf;
892 struct rte_eth_rss_conf *port_rss_conf;
896 PMD_INIT_FUNC_TRACE();
897 hw = VMXNET3_DEV_PRIVATE_TO_HW(dev->data->dev_private);
898 dev_rss_conf = hw->rss_conf;
899 port_rss_conf = &dev->data->dev_conf.rx_adv_conf.rss_conf;
901 /* loading hashFunc */
902 dev_rss_conf->hashFunc = VMXNET3_RSS_HASH_FUNC_TOEPLITZ;
903 /* loading hashKeySize */
904 dev_rss_conf->hashKeySize = VMXNET3_RSS_MAX_KEY_SIZE;
905 /* loading indTableSize : Must not exceed VMXNET3_RSS_MAX_IND_TABLE_SIZE (128)*/
906 dev_rss_conf->indTableSize = (uint16_t)(hw->num_rx_queues * 4);
908 if (port_rss_conf->rss_key == NULL) {
909 /* Default hash key */
910 port_rss_conf->rss_key = rss_intel_key;
913 /* loading hashKey */
914 memcpy(&dev_rss_conf->hashKey[0], port_rss_conf->rss_key, dev_rss_conf->hashKeySize);
916 /* loading indTable */
917 for (i = 0, j = 0; i < dev_rss_conf->indTableSize; i++, j++) {
918 if (j == dev->data->nb_rx_queues)
920 dev_rss_conf->indTable[i] = j;
923 /* loading hashType */
924 dev_rss_conf->hashType = 0;
925 rss_hf = port_rss_conf->rss_hf & VMXNET3_RSS_OFFLOAD_ALL;
926 if (rss_hf & ETH_RSS_IPV4)
927 dev_rss_conf->hashType |= VMXNET3_RSS_HASH_TYPE_IPV4;
928 if (rss_hf & ETH_RSS_IPV4_TCP)
929 dev_rss_conf->hashType |= VMXNET3_RSS_HASH_TYPE_TCP_IPV4;
930 if (rss_hf & ETH_RSS_IPV6)
931 dev_rss_conf->hashType |= VMXNET3_RSS_HASH_TYPE_IPV6;
932 if (rss_hf & ETH_RSS_IPV6_TCP)
933 dev_rss_conf->hashType |= VMXNET3_RSS_HASH_TYPE_TCP_IPV6;
935 return VMXNET3_SUCCESS;
939 * Configure VLAN Filter feature
942 vmxnet3_vlan_configure(struct rte_eth_dev *dev)
945 struct vmxnet3_hw *hw = VMXNET3_DEV_PRIVATE_TO_HW(dev->data->dev_private);
946 uint32_t *vf_table = hw->shared->devRead.rxFilterConf.vfTable;
948 PMD_INIT_FUNC_TRACE();
950 /* Verify if this tag is already set */
951 for (i = 0; i < VMXNET3_VFT_SIZE; i++) {
952 /* Filter all vlan tags out by default */
954 /* To-Do: Provide another routine in dev_ops for user config */
956 PMD_INIT_LOG(DEBUG, "Registering VLAN portid: %"PRIu8" tag %u\n",
957 dev->data->port_id, vf_table[i]);
960 return VMXNET3_SUCCESS;