4 * Copyright(c) 2010-2015 Intel Corporation. All rights reserved.
7 * Redistribution and use in source and binary forms, with or without
8 * modification, are permitted provided that the following conditions
11 * * Redistributions of source code must retain the above copyright
12 * notice, this list of conditions and the following disclaimer.
13 * * Redistributions in binary form must reproduce the above copyright
14 * notice, this list of conditions and the following disclaimer in
15 * the documentation and/or other materials provided with the
17 * * Neither the name of Intel Corporation nor the names of its
18 * contributors may be used to endorse or promote products derived
19 * from this software without specific prior written permission.
21 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
22 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
23 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
24 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
25 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
26 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
27 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
28 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
29 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
30 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
31 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
34 #include <sys/queue.h>
45 #include <rte_byteorder.h>
46 #include <rte_common.h>
47 #include <rte_cycles.h>
49 #include <rte_debug.h>
50 #include <rte_interrupts.h>
52 #include <rte_memory.h>
53 #include <rte_memzone.h>
54 #include <rte_launch.h>
56 #include <rte_per_lcore.h>
57 #include <rte_lcore.h>
58 #include <rte_atomic.h>
59 #include <rte_branch_prediction.h>
60 #include <rte_mempool.h>
61 #include <rte_malloc.h>
63 #include <rte_ether.h>
64 #include <rte_ethdev.h>
65 #include <rte_prefetch.h>
70 #include <rte_string_fns.h>
71 #include <rte_errno.h>
74 #include "base/vmxnet3_defs.h"
75 #include "vmxnet3_ring.h"
77 #include "vmxnet3_logs.h"
78 #include "vmxnet3_ethdev.h"
80 #define VMXNET3_TX_OFFLOAD_MASK ( \
85 #define VMXNET3_TX_OFFLOAD_NOTSUP_MASK \
86 (PKT_TX_OFFLOAD_MASK ^ VMXNET3_TX_OFFLOAD_MASK)
88 static const uint32_t rxprod_reg[2] = {VMXNET3_REG_RXPROD, VMXNET3_REG_RXPROD2};
90 static int vmxnet3_post_rx_bufs(vmxnet3_rx_queue_t*, uint8_t);
91 static void vmxnet3_tq_tx_complete(vmxnet3_tx_queue_t *);
92 #ifdef RTE_LIBRTE_VMXNET3_DEBUG_DRIVER_NOT_USED
93 static void vmxnet3_rxq_dump(struct vmxnet3_rx_queue *);
94 static void vmxnet3_txq_dump(struct vmxnet3_tx_queue *);
97 #ifdef RTE_LIBRTE_VMXNET3_DEBUG_DRIVER_NOT_USED
99 vmxnet3_rxq_dump(struct vmxnet3_rx_queue *rxq)
107 "RXQ: cmd0 base : %p cmd1 base : %p comp ring base : %p.",
108 rxq->cmd_ring[0].base, rxq->cmd_ring[1].base, rxq->comp_ring.base);
110 "RXQ: cmd0 basePA : 0x%lx cmd1 basePA : 0x%lx comp ring basePA : 0x%lx.",
111 (unsigned long)rxq->cmd_ring[0].basePA,
112 (unsigned long)rxq->cmd_ring[1].basePA,
113 (unsigned long)rxq->comp_ring.basePA);
115 avail = vmxnet3_cmd_ring_desc_avail(&rxq->cmd_ring[0]);
117 "RXQ:cmd0: size=%u; free=%u; next2proc=%u; queued=%u",
118 (uint32_t)rxq->cmd_ring[0].size, avail,
119 rxq->comp_ring.next2proc,
120 rxq->cmd_ring[0].size - avail);
122 avail = vmxnet3_cmd_ring_desc_avail(&rxq->cmd_ring[1]);
123 PMD_RX_LOG(DEBUG, "RXQ:cmd1 size=%u; free=%u; next2proc=%u; queued=%u",
124 (uint32_t)rxq->cmd_ring[1].size, avail, rxq->comp_ring.next2proc,
125 rxq->cmd_ring[1].size - avail);
130 vmxnet3_txq_dump(struct vmxnet3_tx_queue *txq)
137 PMD_TX_LOG(DEBUG, "TXQ: cmd base : %p comp ring base : %p data ring base : %p.",
138 txq->cmd_ring.base, txq->comp_ring.base, txq->data_ring.base);
139 PMD_TX_LOG(DEBUG, "TXQ: cmd basePA : 0x%lx comp ring basePA : 0x%lx data ring basePA : 0x%lx.",
140 (unsigned long)txq->cmd_ring.basePA,
141 (unsigned long)txq->comp_ring.basePA,
142 (unsigned long)txq->data_ring.basePA);
144 avail = vmxnet3_cmd_ring_desc_avail(&txq->cmd_ring);
145 PMD_TX_LOG(DEBUG, "TXQ: size=%u; free=%u; next2proc=%u; queued=%u",
146 (uint32_t)txq->cmd_ring.size, avail,
147 txq->comp_ring.next2proc, txq->cmd_ring.size - avail);
152 vmxnet3_tx_cmd_ring_release_mbufs(vmxnet3_cmd_ring_t *ring)
154 while (ring->next2comp != ring->next2fill) {
155 /* No need to worry about desc ownership, device is quiesced by now. */
156 vmxnet3_buf_info_t *buf_info = ring->buf_info + ring->next2comp;
159 rte_pktmbuf_free(buf_info->m);
164 vmxnet3_cmd_ring_adv_next2comp(ring);
169 vmxnet3_rx_cmd_ring_release_mbufs(vmxnet3_cmd_ring_t *ring)
173 for (i = 0; i < ring->size; i++) {
174 /* No need to worry about desc ownership, device is quiesced by now. */
175 vmxnet3_buf_info_t *buf_info = &ring->buf_info[i];
178 rte_pktmbuf_free_seg(buf_info->m);
183 vmxnet3_cmd_ring_adv_next2comp(ring);
188 vmxnet3_cmd_ring_release(vmxnet3_cmd_ring_t *ring)
190 rte_free(ring->buf_info);
191 ring->buf_info = NULL;
195 vmxnet3_dev_tx_queue_release(void *txq)
197 vmxnet3_tx_queue_t *tq = txq;
201 vmxnet3_tx_cmd_ring_release_mbufs(&tq->cmd_ring);
202 /* Release the cmd_ring */
203 vmxnet3_cmd_ring_release(&tq->cmd_ring);
204 /* Release the memzone */
205 rte_memzone_free(tq->mz);
210 vmxnet3_dev_rx_queue_release(void *rxq)
213 vmxnet3_rx_queue_t *rq = rxq;
217 for (i = 0; i < VMXNET3_RX_CMDRING_SIZE; i++)
218 vmxnet3_rx_cmd_ring_release_mbufs(&rq->cmd_ring[i]);
220 /* Release both the cmd_rings */
221 for (i = 0; i < VMXNET3_RX_CMDRING_SIZE; i++)
222 vmxnet3_cmd_ring_release(&rq->cmd_ring[i]);
224 /* Release the memzone */
225 rte_memzone_free(rq->mz);
230 vmxnet3_dev_tx_queue_reset(void *txq)
232 vmxnet3_tx_queue_t *tq = txq;
233 struct vmxnet3_cmd_ring *ring = &tq->cmd_ring;
234 struct vmxnet3_comp_ring *comp_ring = &tq->comp_ring;
235 struct vmxnet3_data_ring *data_ring = &tq->data_ring;
239 /* Release the cmd_ring mbufs */
240 vmxnet3_tx_cmd_ring_release_mbufs(&tq->cmd_ring);
243 /* Tx vmxnet rings structure initialization*/
246 ring->gen = VMXNET3_INIT_GEN;
247 comp_ring->next2proc = 0;
248 comp_ring->gen = VMXNET3_INIT_GEN;
250 size = sizeof(struct Vmxnet3_TxDesc) * ring->size;
251 size += sizeof(struct Vmxnet3_TxCompDesc) * comp_ring->size;
252 size += tq->txdata_desc_size * data_ring->size;
254 memset(ring->base, 0, size);
258 vmxnet3_dev_rx_queue_reset(void *rxq)
261 vmxnet3_rx_queue_t *rq = rxq;
262 struct vmxnet3_hw *hw = rq->hw;
263 struct vmxnet3_cmd_ring *ring0, *ring1;
264 struct vmxnet3_comp_ring *comp_ring;
265 struct vmxnet3_rx_data_ring *data_ring = &rq->data_ring;
268 /* Release both the cmd_rings mbufs */
269 for (i = 0; i < VMXNET3_RX_CMDRING_SIZE; i++)
270 vmxnet3_rx_cmd_ring_release_mbufs(&rq->cmd_ring[i]);
272 ring0 = &rq->cmd_ring[0];
273 ring1 = &rq->cmd_ring[1];
274 comp_ring = &rq->comp_ring;
276 /* Rx vmxnet rings structure initialization */
277 ring0->next2fill = 0;
278 ring1->next2fill = 0;
279 ring0->next2comp = 0;
280 ring1->next2comp = 0;
281 ring0->gen = VMXNET3_INIT_GEN;
282 ring1->gen = VMXNET3_INIT_GEN;
283 comp_ring->next2proc = 0;
284 comp_ring->gen = VMXNET3_INIT_GEN;
286 size = sizeof(struct Vmxnet3_RxDesc) * (ring0->size + ring1->size);
287 size += sizeof(struct Vmxnet3_RxCompDesc) * comp_ring->size;
288 if (VMXNET3_VERSION_GE_3(hw) && rq->data_desc_size)
289 size += rq->data_desc_size * data_ring->size;
291 memset(ring0->base, 0, size);
295 vmxnet3_dev_clear_queues(struct rte_eth_dev *dev)
299 PMD_INIT_FUNC_TRACE();
301 for (i = 0; i < dev->data->nb_tx_queues; i++) {
302 struct vmxnet3_tx_queue *txq = dev->data->tx_queues[i];
306 vmxnet3_dev_tx_queue_reset(txq);
310 for (i = 0; i < dev->data->nb_rx_queues; i++) {
311 struct vmxnet3_rx_queue *rxq = dev->data->rx_queues[i];
315 vmxnet3_dev_rx_queue_reset(rxq);
321 vmxnet3_unmap_pkt(uint16_t eop_idx, vmxnet3_tx_queue_t *txq)
324 struct rte_mbuf *mbuf;
326 /* Release cmd_ring descriptor and free mbuf */
327 RTE_ASSERT(txq->cmd_ring.base[eop_idx].txd.eop == 1);
329 mbuf = txq->cmd_ring.buf_info[eop_idx].m;
331 rte_panic("EOP desc does not point to a valid mbuf");
332 rte_pktmbuf_free(mbuf);
334 txq->cmd_ring.buf_info[eop_idx].m = NULL;
336 while (txq->cmd_ring.next2comp != eop_idx) {
337 /* no out-of-order completion */
338 RTE_ASSERT(txq->cmd_ring.base[txq->cmd_ring.next2comp].txd.cq == 0);
339 vmxnet3_cmd_ring_adv_next2comp(&txq->cmd_ring);
343 /* Mark the txd for which tcd was generated as completed */
344 vmxnet3_cmd_ring_adv_next2comp(&txq->cmd_ring);
346 return completed + 1;
350 vmxnet3_tq_tx_complete(vmxnet3_tx_queue_t *txq)
353 vmxnet3_comp_ring_t *comp_ring = &txq->comp_ring;
354 struct Vmxnet3_TxCompDesc *tcd = (struct Vmxnet3_TxCompDesc *)
355 (comp_ring->base + comp_ring->next2proc);
357 while (tcd->gen == comp_ring->gen) {
358 completed += vmxnet3_unmap_pkt(tcd->txdIdx, txq);
360 vmxnet3_comp_ring_adv_next2proc(comp_ring);
361 tcd = (struct Vmxnet3_TxCompDesc *)(comp_ring->base +
362 comp_ring->next2proc);
365 PMD_TX_LOG(DEBUG, "Processed %d tx comps & command descs.", completed);
369 vmxnet3_prep_pkts(__rte_unused void *tx_queue, struct rte_mbuf **tx_pkts,
377 for (i = 0; i != nb_pkts; i++) {
379 ol_flags = m->ol_flags;
381 /* Non-TSO packet cannot occupy more than
382 * VMXNET3_MAX_TXD_PER_PKT TX descriptors.
384 if ((ol_flags & PKT_TX_TCP_SEG) == 0 &&
385 m->nb_segs > VMXNET3_MAX_TXD_PER_PKT) {
390 /* check that only supported TX offloads are requested. */
391 if ((ol_flags & VMXNET3_TX_OFFLOAD_NOTSUP_MASK) != 0 ||
392 (ol_flags & PKT_TX_L4_MASK) ==
394 rte_errno = -ENOTSUP;
398 #ifdef RTE_LIBRTE_ETHDEV_DEBUG
399 ret = rte_validate_tx_offload(m);
405 ret = rte_net_intel_cksum_prepare(m);
416 vmxnet3_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts,
420 vmxnet3_tx_queue_t *txq = tx_queue;
421 struct vmxnet3_hw *hw = txq->hw;
422 Vmxnet3_TxQueueCtrl *txq_ctrl = &txq->shared->ctrl;
423 uint32_t deferred = rte_le_to_cpu_32(txq_ctrl->txNumDeferred);
425 if (unlikely(txq->stopped)) {
426 PMD_TX_LOG(DEBUG, "Tx queue is stopped.");
430 /* Free up the comp_descriptors aggressively */
431 vmxnet3_tq_tx_complete(txq);
434 while (nb_tx < nb_pkts) {
435 Vmxnet3_GenericDesc *gdesc;
436 vmxnet3_buf_info_t *tbi;
437 uint32_t first2fill, avail, dw2;
438 struct rte_mbuf *txm = tx_pkts[nb_tx];
439 struct rte_mbuf *m_seg = txm;
441 bool tso = (txm->ol_flags & PKT_TX_TCP_SEG) != 0;
442 /* # of descriptors needed for a packet. */
443 unsigned count = txm->nb_segs;
445 avail = vmxnet3_cmd_ring_desc_avail(&txq->cmd_ring);
447 /* Is command ring full? */
448 if (unlikely(avail == 0)) {
449 PMD_TX_LOG(DEBUG, "No free ring descriptors");
450 txq->stats.tx_ring_full++;
451 txq->stats.drop_total += (nb_pkts - nb_tx);
455 /* Command ring is not full but cannot handle the
456 * multi-segmented packet. Let's try the next packet
459 PMD_TX_LOG(DEBUG, "Running out of ring descriptors "
460 "(avail %d needed %d)", avail, count);
461 txq->stats.drop_total++;
463 txq->stats.drop_tso++;
464 rte_pktmbuf_free(txm);
469 /* Drop non-TSO packet that is excessively fragmented */
470 if (unlikely(!tso && count > VMXNET3_MAX_TXD_PER_PKT)) {
471 PMD_TX_LOG(ERR, "Non-TSO packet cannot occupy more than %d tx "
472 "descriptors. Packet dropped.", VMXNET3_MAX_TXD_PER_PKT);
473 txq->stats.drop_too_many_segs++;
474 txq->stats.drop_total++;
475 rte_pktmbuf_free(txm);
480 if (txm->nb_segs == 1 &&
481 rte_pktmbuf_pkt_len(txm) <= txq->txdata_desc_size) {
482 struct Vmxnet3_TxDataDesc *tdd;
484 tdd = (struct Vmxnet3_TxDataDesc *)
485 ((uint8 *)txq->data_ring.base +
486 txq->cmd_ring.next2fill *
487 txq->txdata_desc_size);
488 copy_size = rte_pktmbuf_pkt_len(txm);
489 rte_memcpy(tdd->data, rte_pktmbuf_mtod(txm, char *), copy_size);
492 /* use the previous gen bit for the SOP desc */
493 dw2 = (txq->cmd_ring.gen ^ 0x1) << VMXNET3_TXD_GEN_SHIFT;
494 first2fill = txq->cmd_ring.next2fill;
496 /* Remember the transmit buffer for cleanup */
497 tbi = txq->cmd_ring.buf_info + txq->cmd_ring.next2fill;
499 /* NB: the following assumes that VMXNET3 maximum
500 * transmit buffer size (16K) is greater than
501 * maximum size of mbuf segment size.
503 gdesc = txq->cmd_ring.base + txq->cmd_ring.next2fill;
506 (uint64)txq->cmd_ring.next2fill *
507 txq->txdata_desc_size;
509 rte_cpu_to_le_64(txq->data_ring.basePA +
512 gdesc->txd.addr = rte_mbuf_data_dma_addr(m_seg);
515 gdesc->dword[2] = dw2 | m_seg->data_len;
518 /* move to the next2fill descriptor */
519 vmxnet3_cmd_ring_adv_next2fill(&txq->cmd_ring);
521 /* use the right gen for non-SOP desc */
522 dw2 = txq->cmd_ring.gen << VMXNET3_TXD_GEN_SHIFT;
523 } while ((m_seg = m_seg->next) != NULL);
525 /* set the last buf_info for the pkt */
527 /* Update the EOP descriptor */
528 gdesc->dword[3] |= VMXNET3_TXD_EOP | VMXNET3_TXD_CQ;
530 /* Add VLAN tag if present */
531 gdesc = txq->cmd_ring.base + first2fill;
532 if (txm->ol_flags & PKT_TX_VLAN_PKT) {
534 gdesc->txd.tci = txm->vlan_tci;
538 uint16_t mss = txm->tso_segsz;
542 gdesc->txd.hlen = txm->l2_len + txm->l3_len + txm->l4_len;
543 gdesc->txd.om = VMXNET3_OM_TSO;
544 gdesc->txd.msscof = mss;
546 deferred += (rte_pktmbuf_pkt_len(txm) - gdesc->txd.hlen + mss - 1) / mss;
547 } else if (txm->ol_flags & PKT_TX_L4_MASK) {
548 gdesc->txd.om = VMXNET3_OM_CSUM;
549 gdesc->txd.hlen = txm->l2_len + txm->l3_len;
551 switch (txm->ol_flags & PKT_TX_L4_MASK) {
552 case PKT_TX_TCP_CKSUM:
553 gdesc->txd.msscof = gdesc->txd.hlen + offsetof(struct tcp_hdr, cksum);
555 case PKT_TX_UDP_CKSUM:
556 gdesc->txd.msscof = gdesc->txd.hlen + offsetof(struct udp_hdr, dgram_cksum);
559 PMD_TX_LOG(WARNING, "requested cksum offload not supported %#llx",
560 txm->ol_flags & PKT_TX_L4_MASK);
566 gdesc->txd.om = VMXNET3_OM_NONE;
567 gdesc->txd.msscof = 0;
571 /* flip the GEN bit on the SOP */
572 rte_compiler_barrier();
573 gdesc->dword[2] ^= VMXNET3_TXD_GEN;
575 txq_ctrl->txNumDeferred = rte_cpu_to_le_32(deferred);
579 PMD_TX_LOG(DEBUG, "vmxnet3 txThreshold: %u", rte_le_to_cpu_32(txq_ctrl->txThreshold));
581 if (deferred >= rte_le_to_cpu_32(txq_ctrl->txThreshold)) {
582 txq_ctrl->txNumDeferred = 0;
583 /* Notify vSwitch that packets are available. */
584 VMXNET3_WRITE_BAR0_REG(hw, (VMXNET3_REG_TXPROD + txq->queue_id * VMXNET3_REG_ALIGN),
585 txq->cmd_ring.next2fill);
592 vmxnet3_renew_desc(vmxnet3_rx_queue_t *rxq, uint8_t ring_id,
593 struct rte_mbuf *mbuf)
596 struct vmxnet3_cmd_ring *ring = &rxq->cmd_ring[ring_id];
597 struct Vmxnet3_RxDesc *rxd =
598 (struct Vmxnet3_RxDesc *)(ring->base + ring->next2fill);
599 vmxnet3_buf_info_t *buf_info = &ring->buf_info[ring->next2fill];
602 /* Usually: One HEAD type buf per packet
603 * val = (ring->next2fill % rxq->hw->bufs_per_pkt) ?
604 * VMXNET3_RXD_BTYPE_BODY : VMXNET3_RXD_BTYPE_HEAD;
607 /* We use single packet buffer so all heads here */
608 val = VMXNET3_RXD_BTYPE_HEAD;
610 /* All BODY type buffers for 2nd ring */
611 val = VMXNET3_RXD_BTYPE_BODY;
615 * Load mbuf pointer into buf_info[ring_size]
616 * buf_info structure is equivalent to cookie for virtio-virtqueue
619 buf_info->len = (uint16_t)(mbuf->buf_len - RTE_PKTMBUF_HEADROOM);
620 buf_info->bufPA = rte_mbuf_data_dma_addr_default(mbuf);
622 /* Load Rx Descriptor with the buffer's GPA */
623 rxd->addr = buf_info->bufPA;
625 /* After this point rxd->addr MUST not be NULL */
627 rxd->len = buf_info->len;
628 /* Flip gen bit at the end to change ownership */
629 rxd->gen = ring->gen;
631 vmxnet3_cmd_ring_adv_next2fill(ring);
634 * Allocates mbufs and clusters. Post rx descriptors with buffer details
635 * so that device can receive packets in those buffers.
637 * Among the two rings, 1st ring contains buffers of type 0 and type 1.
638 * bufs_per_pkt is set such that for non-LRO cases all the buffers required
639 * by a frame will fit in 1st ring (1st buf of type0 and rest of type1).
640 * 2nd ring contains buffers of type 1 alone. Second ring mostly be used
644 vmxnet3_post_rx_bufs(vmxnet3_rx_queue_t *rxq, uint8_t ring_id)
648 struct vmxnet3_cmd_ring *ring = &rxq->cmd_ring[ring_id];
650 while (vmxnet3_cmd_ring_desc_avail(ring) > 0) {
651 struct rte_mbuf *mbuf;
653 /* Allocate blank mbuf for the current Rx Descriptor */
654 mbuf = rte_mbuf_raw_alloc(rxq->mp);
655 if (unlikely(mbuf == NULL)) {
656 PMD_RX_LOG(ERR, "Error allocating mbuf");
657 rxq->stats.rx_buf_alloc_failure++;
662 vmxnet3_renew_desc(rxq, ring_id, mbuf);
666 /* Return error only if no buffers are posted at present */
667 if (vmxnet3_cmd_ring_desc_avail(ring) >= (ring->size - 1))
674 /* Receive side checksum and other offloads */
676 vmxnet3_rx_offload(const Vmxnet3_RxCompDesc *rcd, struct rte_mbuf *rxm)
679 if (rcd->rssType != VMXNET3_RCD_RSS_TYPE_NONE) {
680 rxm->ol_flags |= PKT_RX_RSS_HASH;
681 rxm->hash.rss = rcd->rssHash;
684 /* Check packet type, checksum errors, etc. Only support IPv4 for now. */
686 struct ether_hdr *eth = rte_pktmbuf_mtod(rxm, struct ether_hdr *);
687 struct ipv4_hdr *ip = (struct ipv4_hdr *)(eth + 1);
689 if (((ip->version_ihl & 0xf) << 2) > (int)sizeof(struct ipv4_hdr))
690 rxm->packet_type = RTE_PTYPE_L3_IPV4_EXT;
692 rxm->packet_type = RTE_PTYPE_L3_IPV4;
696 rxm->ol_flags |= PKT_RX_IP_CKSUM_BAD;
698 if ((rcd->tcp || rcd->udp) && !rcd->tuc)
699 rxm->ol_flags |= PKT_RX_L4_CKSUM_BAD;
705 * Process the Rx Completion Ring of given vmxnet3_rx_queue
706 * for nb_pkts burst and return the number of packets received
709 vmxnet3_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts, uint16_t nb_pkts)
712 uint32_t nb_rxd, idx;
714 vmxnet3_rx_queue_t *rxq;
715 Vmxnet3_RxCompDesc *rcd;
716 vmxnet3_buf_info_t *rbi;
718 struct rte_mbuf *rxm = NULL;
719 struct vmxnet3_hw *hw;
729 rcd = &rxq->comp_ring.base[rxq->comp_ring.next2proc].rcd;
731 if (unlikely(rxq->stopped)) {
732 PMD_RX_LOG(DEBUG, "Rx queue is stopped.");
736 while (rcd->gen == rxq->comp_ring.gen) {
737 struct rte_mbuf *newm;
739 if (nb_rx >= nb_pkts)
742 newm = rte_mbuf_raw_alloc(rxq->mp);
743 if (unlikely(newm == NULL)) {
744 PMD_RX_LOG(ERR, "Error allocating mbuf");
745 rxq->stats.rx_buf_alloc_failure++;
750 ring_idx = vmxnet3_get_ring_idx(hw, rcd->rqID);
751 rxd = (Vmxnet3_RxDesc *)rxq->cmd_ring[ring_idx].base + idx;
752 RTE_SET_USED(rxd); /* used only for assert when enabled */
753 rbi = rxq->cmd_ring[ring_idx].buf_info + idx;
755 PMD_RX_LOG(DEBUG, "rxd idx: %d ring idx: %d.", idx, ring_idx);
757 RTE_ASSERT(rcd->len <= rxd->len);
760 /* Get the packet buffer pointer from buf_info */
763 /* Clear descriptor associated buf_info to be reused */
767 /* Update the index that we received a packet */
768 rxq->cmd_ring[ring_idx].next2comp = idx;
770 /* For RCD with EOP set, check if there is frame error */
771 if (unlikely(rcd->eop && rcd->err)) {
772 rxq->stats.drop_total++;
773 rxq->stats.drop_err++;
776 rxq->stats.drop_fcs++;
777 PMD_RX_LOG(ERR, "Recv packet dropped due to frame err.");
779 PMD_RX_LOG(ERR, "Error in received packet rcd#:%d rxd:%d",
780 (int)(rcd - (struct Vmxnet3_RxCompDesc *)
781 rxq->comp_ring.base), rcd->rxdIdx);
782 rte_pktmbuf_free_seg(rxm);
783 if (rxq->start_seg) {
784 struct rte_mbuf *start = rxq->start_seg;
786 rxq->start_seg = NULL;
787 rte_pktmbuf_free(start);
792 /* Initialize newly received packet buffer */
793 rxm->port = rxq->port_id;
796 rxm->pkt_len = (uint16_t)rcd->len;
797 rxm->data_len = (uint16_t)rcd->len;
798 rxm->data_off = RTE_PKTMBUF_HEADROOM;
803 * If this is the first buffer of the received packet,
804 * set the pointer to the first mbuf of the packet
805 * Otherwise, update the total length and the number of segments
806 * of the current scattered packet, and update the pointer to
807 * the last mbuf of the current packet.
810 RTE_ASSERT(rxd->btype == VMXNET3_RXD_BTYPE_HEAD);
812 if (unlikely(rcd->len == 0)) {
813 RTE_ASSERT(rcd->eop);
816 "Rx buf was skipped. rxring[%d][%d])",
818 rte_pktmbuf_free_seg(rxm);
822 if (vmxnet3_rx_data_ring(hw, rcd->rqID)) {
823 uint8_t *rdd = rxq->data_ring.base +
824 idx * rxq->data_desc_size;
826 RTE_ASSERT(VMXNET3_VERSION_GE_3(hw));
827 rte_memcpy(rte_pktmbuf_mtod(rxm, char *),
831 rxq->start_seg = rxm;
832 vmxnet3_rx_offload(rcd, rxm);
834 struct rte_mbuf *start = rxq->start_seg;
836 RTE_ASSERT(rxd->btype == VMXNET3_RXD_BTYPE_BODY);
838 start->pkt_len += rxm->data_len;
841 rxq->last_seg->next = rxm;
846 struct rte_mbuf *start = rxq->start_seg;
848 /* Check for hardware stripped VLAN tag */
850 start->ol_flags |= (PKT_RX_VLAN |
851 PKT_RX_VLAN_STRIPPED);
852 start->vlan_tci = rte_le_to_cpu_16((uint16_t)rcd->tci);
855 rx_pkts[nb_rx++] = start;
856 rxq->start_seg = NULL;
860 rxq->cmd_ring[ring_idx].next2comp = idx;
861 VMXNET3_INC_RING_IDX_ONLY(rxq->cmd_ring[ring_idx].next2comp,
862 rxq->cmd_ring[ring_idx].size);
864 /* It's time to renew descriptors */
865 vmxnet3_renew_desc(rxq, ring_idx, newm);
866 if (unlikely(rxq->shared->ctrl.updateRxProd)) {
867 VMXNET3_WRITE_BAR0_REG(hw, rxprod_reg[ring_idx] + (rxq->queue_id * VMXNET3_REG_ALIGN),
868 rxq->cmd_ring[ring_idx].next2fill);
871 /* Advance to the next descriptor in comp_ring */
872 vmxnet3_comp_ring_adv_next2proc(&rxq->comp_ring);
874 rcd = &rxq->comp_ring.base[rxq->comp_ring.next2proc].rcd;
876 if (nb_rxd > rxq->cmd_ring[0].size) {
877 PMD_RX_LOG(ERR, "Used up quota of receiving packets,"
878 " relinquish control.");
883 if (unlikely(nb_rxd == 0)) {
885 for (ring_idx = 0; ring_idx < VMXNET3_RX_CMDRING_SIZE; ring_idx++) {
886 avail = vmxnet3_cmd_ring_desc_avail(&rxq->cmd_ring[ring_idx]);
887 if (unlikely(avail > 0)) {
888 /* try to alloc new buf and renew descriptors */
889 vmxnet3_post_rx_bufs(rxq, ring_idx);
892 if (unlikely(rxq->shared->ctrl.updateRxProd)) {
893 for (ring_idx = 0; ring_idx < VMXNET3_RX_CMDRING_SIZE; ring_idx++) {
894 VMXNET3_WRITE_BAR0_REG(hw, rxprod_reg[ring_idx] + (rxq->queue_id * VMXNET3_REG_ALIGN),
895 rxq->cmd_ring[ring_idx].next2fill);
904 vmxnet3_dev_tx_queue_setup(struct rte_eth_dev *dev,
907 unsigned int socket_id,
908 const struct rte_eth_txconf *tx_conf)
910 struct vmxnet3_hw *hw = dev->data->dev_private;
911 const struct rte_memzone *mz;
912 struct vmxnet3_tx_queue *txq;
913 struct vmxnet3_cmd_ring *ring;
914 struct vmxnet3_comp_ring *comp_ring;
915 struct vmxnet3_data_ring *data_ring;
918 PMD_INIT_FUNC_TRACE();
920 if ((tx_conf->txq_flags & ETH_TXQ_FLAGS_NOXSUMSCTP) !=
921 ETH_TXQ_FLAGS_NOXSUMSCTP) {
922 PMD_INIT_LOG(ERR, "SCTP checksum offload not supported");
926 txq = rte_zmalloc("ethdev_tx_queue", sizeof(struct vmxnet3_tx_queue),
927 RTE_CACHE_LINE_SIZE);
929 PMD_INIT_LOG(ERR, "Can not allocate tx queue structure");
933 txq->queue_id = queue_idx;
934 txq->port_id = dev->data->port_id;
935 txq->shared = &hw->tqd_start[queue_idx];
937 txq->qid = queue_idx;
939 txq->txdata_desc_size = hw->txdata_desc_size;
941 ring = &txq->cmd_ring;
942 comp_ring = &txq->comp_ring;
943 data_ring = &txq->data_ring;
945 /* Tx vmxnet ring length should be between 512-4096 */
946 if (nb_desc < VMXNET3_DEF_TX_RING_SIZE) {
947 PMD_INIT_LOG(ERR, "VMXNET3 Tx Ring Size Min: %u",
948 VMXNET3_DEF_TX_RING_SIZE);
950 } else if (nb_desc > VMXNET3_TX_RING_MAX_SIZE) {
951 PMD_INIT_LOG(ERR, "VMXNET3 Tx Ring Size Max: %u",
952 VMXNET3_TX_RING_MAX_SIZE);
955 ring->size = nb_desc;
956 ring->size &= ~VMXNET3_RING_SIZE_MASK;
958 comp_ring->size = data_ring->size = ring->size;
960 /* Tx vmxnet rings structure initialization*/
963 ring->gen = VMXNET3_INIT_GEN;
964 comp_ring->next2proc = 0;
965 comp_ring->gen = VMXNET3_INIT_GEN;
967 size = sizeof(struct Vmxnet3_TxDesc) * ring->size;
968 size += sizeof(struct Vmxnet3_TxCompDesc) * comp_ring->size;
969 size += txq->txdata_desc_size * data_ring->size;
971 mz = rte_eth_dma_zone_reserve(dev, "txdesc", queue_idx, size,
972 VMXNET3_RING_BA_ALIGN, socket_id);
974 PMD_INIT_LOG(ERR, "ERROR: Creating queue descriptors zone");
978 memset(mz->addr, 0, mz->len);
980 /* cmd_ring initialization */
981 ring->base = mz->addr;
982 ring->basePA = mz->iova;
984 /* comp_ring initialization */
985 comp_ring->base = ring->base + ring->size;
986 comp_ring->basePA = ring->basePA +
987 (sizeof(struct Vmxnet3_TxDesc) * ring->size);
989 /* data_ring initialization */
990 data_ring->base = (Vmxnet3_TxDataDesc *)(comp_ring->base + comp_ring->size);
991 data_ring->basePA = comp_ring->basePA +
992 (sizeof(struct Vmxnet3_TxCompDesc) * comp_ring->size);
994 /* cmd_ring0 buf_info allocation */
995 ring->buf_info = rte_zmalloc("tx_ring_buf_info",
996 ring->size * sizeof(vmxnet3_buf_info_t), RTE_CACHE_LINE_SIZE);
997 if (ring->buf_info == NULL) {
998 PMD_INIT_LOG(ERR, "ERROR: Creating tx_buf_info structure");
1002 /* Update the data portion with txq */
1003 dev->data->tx_queues[queue_idx] = txq;
1009 vmxnet3_dev_rx_queue_setup(struct rte_eth_dev *dev,
1012 unsigned int socket_id,
1013 __rte_unused const struct rte_eth_rxconf *rx_conf,
1014 struct rte_mempool *mp)
1016 const struct rte_memzone *mz;
1017 struct vmxnet3_rx_queue *rxq;
1018 struct vmxnet3_hw *hw = dev->data->dev_private;
1019 struct vmxnet3_cmd_ring *ring0, *ring1, *ring;
1020 struct vmxnet3_comp_ring *comp_ring;
1021 struct vmxnet3_rx_data_ring *data_ring;
1026 PMD_INIT_FUNC_TRACE();
1028 rxq = rte_zmalloc("ethdev_rx_queue", sizeof(struct vmxnet3_rx_queue),
1029 RTE_CACHE_LINE_SIZE);
1031 PMD_INIT_LOG(ERR, "Can not allocate rx queue structure");
1036 rxq->queue_id = queue_idx;
1037 rxq->port_id = dev->data->port_id;
1038 rxq->shared = &hw->rqd_start[queue_idx];
1040 rxq->qid1 = queue_idx;
1041 rxq->qid2 = queue_idx + hw->num_rx_queues;
1042 rxq->data_ring_qid = queue_idx + 2 * hw->num_rx_queues;
1043 rxq->data_desc_size = hw->rxdata_desc_size;
1044 rxq->stopped = TRUE;
1046 ring0 = &rxq->cmd_ring[0];
1047 ring1 = &rxq->cmd_ring[1];
1048 comp_ring = &rxq->comp_ring;
1049 data_ring = &rxq->data_ring;
1051 /* Rx vmxnet rings length should be between 256-4096 */
1052 if (nb_desc < VMXNET3_DEF_RX_RING_SIZE) {
1053 PMD_INIT_LOG(ERR, "VMXNET3 Rx Ring Size Min: 256");
1055 } else if (nb_desc > VMXNET3_RX_RING_MAX_SIZE) {
1056 PMD_INIT_LOG(ERR, "VMXNET3 Rx Ring Size Max: 4096");
1059 ring0->size = nb_desc;
1060 ring0->size &= ~VMXNET3_RING_SIZE_MASK;
1061 ring1->size = ring0->size;
1064 comp_ring->size = ring0->size + ring1->size;
1065 data_ring->size = ring0->size;
1067 /* Rx vmxnet rings structure initialization */
1068 ring0->next2fill = 0;
1069 ring1->next2fill = 0;
1070 ring0->next2comp = 0;
1071 ring1->next2comp = 0;
1072 ring0->gen = VMXNET3_INIT_GEN;
1073 ring1->gen = VMXNET3_INIT_GEN;
1074 comp_ring->next2proc = 0;
1075 comp_ring->gen = VMXNET3_INIT_GEN;
1077 size = sizeof(struct Vmxnet3_RxDesc) * (ring0->size + ring1->size);
1078 size += sizeof(struct Vmxnet3_RxCompDesc) * comp_ring->size;
1079 if (VMXNET3_VERSION_GE_3(hw) && rxq->data_desc_size)
1080 size += rxq->data_desc_size * data_ring->size;
1082 mz = rte_eth_dma_zone_reserve(dev, "rxdesc", queue_idx, size,
1083 VMXNET3_RING_BA_ALIGN, socket_id);
1085 PMD_INIT_LOG(ERR, "ERROR: Creating queue descriptors zone");
1089 memset(mz->addr, 0, mz->len);
1091 /* cmd_ring0 initialization */
1092 ring0->base = mz->addr;
1093 ring0->basePA = mz->iova;
1095 /* cmd_ring1 initialization */
1096 ring1->base = ring0->base + ring0->size;
1097 ring1->basePA = ring0->basePA + sizeof(struct Vmxnet3_RxDesc) * ring0->size;
1099 /* comp_ring initialization */
1100 comp_ring->base = ring1->base + ring1->size;
1101 comp_ring->basePA = ring1->basePA + sizeof(struct Vmxnet3_RxDesc) *
1104 /* data_ring initialization */
1105 if (VMXNET3_VERSION_GE_3(hw) && rxq->data_desc_size) {
1107 (uint8_t *)(comp_ring->base + comp_ring->size);
1108 data_ring->basePA = comp_ring->basePA +
1109 sizeof(struct Vmxnet3_RxCompDesc) * comp_ring->size;
1112 /* cmd_ring0-cmd_ring1 buf_info allocation */
1113 for (i = 0; i < VMXNET3_RX_CMDRING_SIZE; i++) {
1115 ring = &rxq->cmd_ring[i];
1117 snprintf(mem_name, sizeof(mem_name), "rx_ring_%d_buf_info", i);
1119 ring->buf_info = rte_zmalloc(mem_name,
1120 ring->size * sizeof(vmxnet3_buf_info_t),
1121 RTE_CACHE_LINE_SIZE);
1122 if (ring->buf_info == NULL) {
1123 PMD_INIT_LOG(ERR, "ERROR: Creating rx_buf_info structure");
1128 /* Update the data portion with rxq */
1129 dev->data->rx_queues[queue_idx] = rxq;
1135 * Initializes Receive Unit
1136 * Load mbufs in rx queue in advance
1139 vmxnet3_dev_rxtx_init(struct rte_eth_dev *dev)
1141 struct vmxnet3_hw *hw = dev->data->dev_private;
1146 PMD_INIT_FUNC_TRACE();
1148 for (i = 0; i < hw->num_rx_queues; i++) {
1149 vmxnet3_rx_queue_t *rxq = dev->data->rx_queues[i];
1151 for (j = 0; j < VMXNET3_RX_CMDRING_SIZE; j++) {
1152 /* Passing 0 as alloc_num will allocate full ring */
1153 ret = vmxnet3_post_rx_bufs(rxq, j);
1156 "ERROR: Posting Rxq: %d buffers ring: %d",
1161 * Updating device with the index:next2fill to fill the
1162 * mbufs for coming packets.
1164 if (unlikely(rxq->shared->ctrl.updateRxProd)) {
1165 VMXNET3_WRITE_BAR0_REG(hw, rxprod_reg[j] + (rxq->queue_id * VMXNET3_REG_ALIGN),
1166 rxq->cmd_ring[j].next2fill);
1169 rxq->stopped = FALSE;
1170 rxq->start_seg = NULL;
1173 for (i = 0; i < dev->data->nb_tx_queues; i++) {
1174 struct vmxnet3_tx_queue *txq = dev->data->tx_queues[i];
1176 txq->stopped = FALSE;
1182 static uint8_t rss_intel_key[40] = {
1183 0x6D, 0x5A, 0x56, 0xDA, 0x25, 0x5B, 0x0E, 0xC2,
1184 0x41, 0x67, 0x25, 0x3D, 0x43, 0xA3, 0x8F, 0xB0,
1185 0xD0, 0xCA, 0x2B, 0xCB, 0xAE, 0x7B, 0x30, 0xB4,
1186 0x77, 0xCB, 0x2D, 0xA3, 0x80, 0x30, 0xF2, 0x0C,
1187 0x6A, 0x42, 0xB7, 0x3B, 0xBE, 0xAC, 0x01, 0xFA,
1191 * Configure RSS feature
1194 vmxnet3_rss_configure(struct rte_eth_dev *dev)
1196 struct vmxnet3_hw *hw = dev->data->dev_private;
1197 struct VMXNET3_RSSConf *dev_rss_conf;
1198 struct rte_eth_rss_conf *port_rss_conf;
1202 PMD_INIT_FUNC_TRACE();
1204 dev_rss_conf = hw->rss_conf;
1205 port_rss_conf = &dev->data->dev_conf.rx_adv_conf.rss_conf;
1207 /* loading hashFunc */
1208 dev_rss_conf->hashFunc = VMXNET3_RSS_HASH_FUNC_TOEPLITZ;
1209 /* loading hashKeySize */
1210 dev_rss_conf->hashKeySize = VMXNET3_RSS_MAX_KEY_SIZE;
1211 /* loading indTableSize: Must not exceed VMXNET3_RSS_MAX_IND_TABLE_SIZE (128)*/
1212 dev_rss_conf->indTableSize = (uint16_t)(hw->num_rx_queues * 4);
1214 if (port_rss_conf->rss_key == NULL) {
1215 /* Default hash key */
1216 port_rss_conf->rss_key = rss_intel_key;
1219 /* loading hashKey */
1220 memcpy(&dev_rss_conf->hashKey[0], port_rss_conf->rss_key,
1221 dev_rss_conf->hashKeySize);
1223 /* loading indTable */
1224 for (i = 0, j = 0; i < dev_rss_conf->indTableSize; i++, j++) {
1225 if (j == dev->data->nb_rx_queues)
1227 dev_rss_conf->indTable[i] = j;
1230 /* loading hashType */
1231 dev_rss_conf->hashType = 0;
1232 rss_hf = port_rss_conf->rss_hf & VMXNET3_RSS_OFFLOAD_ALL;
1233 if (rss_hf & ETH_RSS_IPV4)
1234 dev_rss_conf->hashType |= VMXNET3_RSS_HASH_TYPE_IPV4;
1235 if (rss_hf & ETH_RSS_NONFRAG_IPV4_TCP)
1236 dev_rss_conf->hashType |= VMXNET3_RSS_HASH_TYPE_TCP_IPV4;
1237 if (rss_hf & ETH_RSS_IPV6)
1238 dev_rss_conf->hashType |= VMXNET3_RSS_HASH_TYPE_IPV6;
1239 if (rss_hf & ETH_RSS_NONFRAG_IPV6_TCP)
1240 dev_rss_conf->hashType |= VMXNET3_RSS_HASH_TYPE_TCP_IPV6;
1242 return VMXNET3_SUCCESS;