X-Git-Url: http://git.droids-corp.org/?a=blobdiff_plain;f=lib%2Flibrte_pmd_vmxnet3%2Fvmxnet3_rxtx.c;h=8425f32ece009af9c23bb5aae90cdb1023c503a7;hb=fdf20fa7bee9df9037116318a87080e1eb7e757e;hp=08d7330d00ab8e24295a57a60d40d91772fe5a70;hpb=186e9cbeba1992ec617123a80a0e2b676938b368;p=dpdk.git diff --git a/lib/librte_pmd_vmxnet3/vmxnet3_rxtx.c b/lib/librte_pmd_vmxnet3/vmxnet3_rxtx.c index 08d7330d00..8425f32ece 100644 --- a/lib/librte_pmd_vmxnet3/vmxnet3_rxtx.c +++ b/lib/librte_pmd_vmxnet3/vmxnet3_rxtx.c @@ -65,6 +65,7 @@ #include #include #include +#include #include #include #include @@ -77,10 +78,8 @@ #include "vmxnet3_logs.h" #include "vmxnet3_ethdev.h" - #define RTE_MBUF_DATA_DMA_ADDR(mb) \ - (uint64_t) ((mb)->buf_physaddr + (uint64_t)((char *)((mb)->pkt.data) - \ - (char *)(mb)->buf_addr)) + (uint64_t) ((mb)->buf_physaddr + (mb)->data_off) #define RTE_MBUF_DATA_DMA_ADDR_DEFAULT(mb) \ (uint64_t) ((mb)->buf_physaddr + RTE_PKTMBUF_HEADROOM) @@ -100,7 +99,7 @@ rte_rxmbuf_alloc(struct rte_mempool *mp) struct rte_mbuf *m; m = __rte_mbuf_raw_alloc(mp); - __rte_mbuf_sanity_check_raw(m, RTE_MBUF_PKT, 0); + __rte_mbuf_sanity_check_raw(m, 0); return m; } @@ -114,23 +113,23 @@ vmxnet3_rxq_dump(struct vmxnet3_rx_queue *rxq) return; PMD_RX_LOG(DEBUG, - "RXQ: cmd0 base : 0x%p cmd1 base : 0x%p comp ring base : 0x%p.\n", + "RXQ: cmd0 base : 0x%p cmd1 base : 0x%p comp ring base : 0x%p.", rxq->cmd_ring[0].base, rxq->cmd_ring[1].base, rxq->comp_ring.base); PMD_RX_LOG(DEBUG, - "RXQ: cmd0 basePA : 0x%lx cmd1 basePA : 0x%lx comp ring basePA : 0x%lx.\n", + "RXQ: cmd0 basePA : 0x%lx cmd1 basePA : 0x%lx comp ring basePA : 0x%lx.", (unsigned long)rxq->cmd_ring[0].basePA, (unsigned long)rxq->cmd_ring[1].basePA, (unsigned long)rxq->comp_ring.basePA); avail = vmxnet3_cmd_ring_desc_avail(&rxq->cmd_ring[0]); PMD_RX_LOG(DEBUG, - "RXQ:cmd0: size=%u; free=%u; next2proc=%u; queued=%u\n", + "RXQ:cmd0: size=%u; free=%u; next2proc=%u; queued=%u", (uint32_t)rxq->cmd_ring[0].size, avail, rxq->comp_ring.next2proc, rxq->cmd_ring[0].size - avail); avail = vmxnet3_cmd_ring_desc_avail(&rxq->cmd_ring[1]); - PMD_RX_LOG(DEBUG, "RXQ:cmd1 size=%u; free=%u; next2proc=%u; queued=%u\n", + PMD_RX_LOG(DEBUG, "RXQ:cmd1 size=%u; free=%u; next2proc=%u; queued=%u", (uint32_t)rxq->cmd_ring[1].size, avail, rxq->comp_ring.next2proc, rxq->cmd_ring[1].size - avail); @@ -144,21 +143,22 @@ vmxnet3_txq_dump(struct vmxnet3_tx_queue *txq) if (txq == NULL) return; - PMD_TX_LOG(DEBUG, "TXQ: cmd base : 0x%p comp ring base : 0x%p.\n", - txq->cmd_ring.base, txq->comp_ring.base); - PMD_TX_LOG(DEBUG, "TXQ: cmd basePA : 0x%lx comp ring basePA : 0x%lx.\n", + PMD_TX_LOG(DEBUG, "TXQ: cmd base : 0x%p comp ring base : 0x%p data ring base : 0x%p.", + txq->cmd_ring.base, txq->comp_ring.base, txq->data_ring.base); + PMD_TX_LOG(DEBUG, "TXQ: cmd basePA : 0x%lx comp ring basePA : 0x%lx data ring basePA : 0x%lx.", (unsigned long)txq->cmd_ring.basePA, - (unsigned long)txq->comp_ring.basePA); + (unsigned long)txq->comp_ring.basePA, + (unsigned long)txq->data_ring.basePA); avail = vmxnet3_cmd_ring_desc_avail(&txq->cmd_ring); - PMD_TX_LOG(DEBUG, "TXQ: size=%u; free=%u; next2proc=%u; queued=%u\n", + PMD_TX_LOG(DEBUG, "TXQ: size=%u; free=%u; next2proc=%u; queued=%u", (uint32_t)txq->cmd_ring.size, avail, txq->comp_ring.next2proc, txq->cmd_ring.size - avail); } #endif static inline void -vmxnet3_cmd_ring_release(vmxnet3_cmd_ring_t *ring) +vmxnet3_cmd_ring_release_mbufs(vmxnet3_cmd_ring_t *ring) { while (ring->next2comp != ring->next2fill) { /* No need to worry about tx desc ownership, device is quiesced by now. */ @@ -172,15 +172,23 @@ vmxnet3_cmd_ring_release(vmxnet3_cmd_ring_t *ring) } vmxnet3_cmd_ring_adv_next2comp(ring); } +} + +static void +vmxnet3_cmd_ring_release(vmxnet3_cmd_ring_t *ring) +{ + vmxnet3_cmd_ring_release_mbufs(ring); rte_free(ring->buf_info); + ring->buf_info = NULL; } + void vmxnet3_dev_tx_queue_release(void *txq) { vmxnet3_tx_queue_t *tq = txq; - if (txq != NULL) { + if (tq != NULL) { /* Release the cmd_ring */ vmxnet3_cmd_ring_release(&tq->cmd_ring); } @@ -192,13 +200,76 @@ vmxnet3_dev_rx_queue_release(void *rxq) int i; vmxnet3_rx_queue_t *rq = rxq; - if (rxq != NULL) { + if (rq != NULL) { /* Release both the cmd_rings */ for (i = 0; i < VMXNET3_RX_CMDRING_SIZE; i++) vmxnet3_cmd_ring_release(&rq->cmd_ring[i]); } } +static void +vmxnet3_dev_tx_queue_reset(void *txq) +{ + vmxnet3_tx_queue_t *tq = txq; + struct vmxnet3_cmd_ring *ring = &tq->cmd_ring; + struct vmxnet3_comp_ring *comp_ring = &tq->comp_ring; + struct vmxnet3_data_ring *data_ring = &tq->data_ring; + int size; + + if (tq != NULL) { + /* Release the cmd_ring mbufs */ + vmxnet3_cmd_ring_release_mbufs(&tq->cmd_ring); + } + + /* Tx vmxnet rings structure initialization*/ + ring->next2fill = 0; + ring->next2comp = 0; + ring->gen = VMXNET3_INIT_GEN; + comp_ring->next2proc = 0; + comp_ring->gen = VMXNET3_INIT_GEN; + + size = sizeof(struct Vmxnet3_TxDesc) * ring->size; + size += sizeof(struct Vmxnet3_TxCompDesc) * comp_ring->size; + size += sizeof(struct Vmxnet3_TxDataDesc) * data_ring->size; + + memset(ring->base, 0, size); +} + +static void +vmxnet3_dev_rx_queue_reset(void *rxq) +{ + int i; + vmxnet3_rx_queue_t *rq = rxq; + struct vmxnet3_cmd_ring *ring0, *ring1; + struct vmxnet3_comp_ring *comp_ring; + int size; + + if (rq != NULL) { + /* Release both the cmd_rings mbufs */ + for (i = 0; i < VMXNET3_RX_CMDRING_SIZE; i++) + vmxnet3_cmd_ring_release_mbufs(&rq->cmd_ring[i]); + } + + ring0 = &rq->cmd_ring[0]; + ring1 = &rq->cmd_ring[1]; + comp_ring = &rq->comp_ring; + + /* Rx vmxnet rings structure initialization */ + ring0->next2fill = 0; + ring1->next2fill = 0; + ring0->next2comp = 0; + ring1->next2comp = 0; + ring0->gen = VMXNET3_INIT_GEN; + ring1->gen = VMXNET3_INIT_GEN; + comp_ring->next2proc = 0; + comp_ring->gen = VMXNET3_INIT_GEN; + + size = sizeof(struct Vmxnet3_RxDesc) * (ring0->size + ring1->size); + size += sizeof(struct Vmxnet3_RxCompDesc) * comp_ring->size; + + memset(ring0->base, 0, size); +} + void vmxnet3_dev_clear_queues(struct rte_eth_dev *dev) { @@ -211,7 +282,7 @@ vmxnet3_dev_clear_queues(struct rte_eth_dev *dev) if (txq != NULL) { txq->stopped = TRUE; - vmxnet3_dev_tx_queue_release(txq); + vmxnet3_dev_tx_queue_reset(txq); } } @@ -220,7 +291,7 @@ vmxnet3_dev_clear_queues(struct rte_eth_dev *dev) if (rxq != NULL) { rxq->stopped = TRUE; - vmxnet3_dev_rx_queue_release(rxq); + vmxnet3_dev_rx_queue_reset(rxq); } } } @@ -257,7 +328,7 @@ vmxnet3_tq_tx_complete(vmxnet3_tx_queue_t *txq) completed++; } - PMD_TX_LOG(DEBUG, "Processed %d tx comps & command descs.\n", completed); + PMD_TX_LOG(DEBUG, "Processed %d tx comps & command descs.", completed); } uint16_t @@ -273,8 +344,8 @@ vmxnet3_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts, hw = txq->hw; - if (txq->stopped) { - PMD_TX_LOG(DEBUG, "Tx queue is stopped.\n"); + if (unlikely(txq->stopped)) { + PMD_TX_LOG(DEBUG, "Tx queue is stopped."); return 0; } @@ -285,11 +356,12 @@ vmxnet3_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts, while (nb_tx < nb_pkts) { if (vmxnet3_cmd_ring_desc_avail(&txq->cmd_ring)) { + int copy_size = 0; txm = tx_pkts[nb_tx]; /* Don't support scatter packets yet, free them if met */ - if (txm->pkt.nb_segs != 1) { - PMD_TX_LOG(DEBUG, "Don't support scatter packets yet, drop!\n"); + if (txm->nb_segs != 1) { + PMD_TX_LOG(DEBUG, "Don't support scatter packets yet, drop!"); rte_pktmbuf_free(tx_pkts[nb_tx]); txq->stats.drop_total++; @@ -298,8 +370,8 @@ vmxnet3_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts, } /* Needs to minus ether header len */ - if (txm->pkt.data_len > (hw->cur_mtu + ETHER_HDR_LEN)) { - PMD_TX_LOG(DEBUG, "Packet data_len higher than MTU\n"); + if (txm->data_len > (hw->cur_mtu + ETHER_HDR_LEN)) { + PMD_TX_LOG(DEBUG, "Packet data_len higher than MTU"); rte_pktmbuf_free(tx_pkts[nb_tx]); txq->stats.drop_total++; @@ -308,17 +380,35 @@ vmxnet3_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts, } txd = (Vmxnet3_TxDesc *)(txq->cmd_ring.base + txq->cmd_ring.next2fill); + if (rte_pktmbuf_pkt_len(txm) <= VMXNET3_HDR_COPY_SIZE) { + struct Vmxnet3_TxDataDesc *tdd; + + tdd = txq->data_ring.base + txq->cmd_ring.next2fill; + copy_size = rte_pktmbuf_pkt_len(txm); + rte_memcpy(tdd->data, rte_pktmbuf_mtod(txm, char *), copy_size); + } /* Fill the tx descriptor */ tbi = txq->cmd_ring.buf_info + txq->cmd_ring.next2fill; tbi->bufPA = RTE_MBUF_DATA_DMA_ADDR(txm); - txd->addr = tbi->bufPA; - txd->len = txm->pkt.data_len; + if (copy_size) + txd->addr = rte_cpu_to_le_64(txq->data_ring.basePA + + txq->cmd_ring.next2fill * + sizeof(struct Vmxnet3_TxDataDesc)); + else + txd->addr = tbi->bufPA; + txd->len = txm->data_len; /* Mark the last descriptor as End of Packet. */ txd->cq = 1; txd->eop = 1; + /* Add VLAN tag if requested */ + if (txm->ol_flags & PKT_TX_VLAN_PKT) { + txd->ti = 1; + txd->tci = rte_cpu_to_le_16(txm->vlan_tci); + } + /* Record current mbuf for freeing it later in tx complete */ #ifdef RTE_LIBRTE_VMXNET3_DEBUG_DRIVER VMXNET3_ASSERT(txm); @@ -339,7 +429,7 @@ vmxnet3_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts, nb_tx++; } else { - PMD_TX_LOG(DEBUG, "No free tx cmd desc(s)\n"); + PMD_TX_LOG(DEBUG, "No free tx cmd desc(s)"); txq->stats.drop_total += (nb_pkts - nb_tx); break; } @@ -376,6 +466,19 @@ vmxnet3_post_rx_bufs(vmxnet3_rx_queue_t *rxq, uint8_t ring_id) uint32_t i = 0, val = 0; struct vmxnet3_cmd_ring *ring = &rxq->cmd_ring[ring_id]; + if (ring_id == 0) { + /* Usually: One HEAD type buf per packet + * val = (ring->next2fill % rxq->hw->bufs_per_pkt) ? + * VMXNET3_RXD_BTYPE_BODY : VMXNET3_RXD_BTYPE_HEAD; + */ + + /* We use single packet buffer so all heads here */ + val = VMXNET3_RXD_BTYPE_HEAD; + } else { + /* All BODY type buffers for 2nd ring */ + val = VMXNET3_RXD_BTYPE_BODY; + } + while (vmxnet3_cmd_ring_desc_avail(ring) > 0) { struct Vmxnet3_RxDesc *rxd; struct rte_mbuf *mbuf; @@ -383,23 +486,10 @@ vmxnet3_post_rx_bufs(vmxnet3_rx_queue_t *rxq, uint8_t ring_id) rxd = (struct Vmxnet3_RxDesc *)(ring->base + ring->next2fill); - if (ring->rid == 0) { - /* Usually: One HEAD type buf per packet - * val = (ring->next2fill % rxq->hw->bufs_per_pkt) ? - * VMXNET3_RXD_BTYPE_BODY : VMXNET3_RXD_BTYPE_HEAD; - */ - - /* We use single packet buffer so all heads here */ - val = VMXNET3_RXD_BTYPE_HEAD; - } else { - /* All BODY type buffers for 2nd ring; which won't be used at all by ESXi */ - val = VMXNET3_RXD_BTYPE_BODY; - } - /* Allocate blank mbuf for the current Rx Descriptor */ mbuf = rte_rxmbuf_alloc(rxq->mp); - if (mbuf == NULL) { - PMD_RX_LOG(ERR, "Error allocating mbuf in %s\n", __func__); + if (unlikely(mbuf == NULL)) { + PMD_RX_LOG(ERR, "Error allocating mbuf in %s", __func__); rxq->stats.rx_buf_alloc_failure++; err = ENOMEM; break; @@ -461,135 +551,140 @@ vmxnet3_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts, uint16_t nb_pkts) rcd = &rxq->comp_ring.base[rxq->comp_ring.next2proc].rcd; - if (rxq->stopped) { - PMD_RX_LOG(DEBUG, "Rx queue is stopped.\n"); + if (unlikely(rxq->stopped)) { + PMD_RX_LOG(DEBUG, "Rx queue is stopped."); return 0; } while (rcd->gen == rxq->comp_ring.gen) { - if (nb_rx >= nb_pkts) break; + idx = rcd->rxdIdx; ring_idx = (uint8_t)((rcd->rqID == rxq->qid1) ? 0 : 1); rxd = (Vmxnet3_RxDesc *)rxq->cmd_ring[ring_idx].base + idx; rbi = rxq->cmd_ring[ring_idx].buf_info + idx; - if (rcd->sop != 1 || rcd->eop != 1) { + if (unlikely(rcd->sop != 1 || rcd->eop != 1)) { rte_pktmbuf_free_seg(rbi->m); - PMD_RX_LOG(DEBUG, "Packet spread across multiple buffers\n)"); goto rcd_done; + } - } else { - - PMD_RX_LOG(DEBUG, "rxd idx: %d ring idx: %d.\n", idx, ring_idx); + PMD_RX_LOG(DEBUG, "rxd idx: %d ring idx: %d.", idx, ring_idx); #ifdef RTE_LIBRTE_VMXNET3_DEBUG_DRIVER - VMXNET3_ASSERT(rcd->len <= rxd->len); - VMXNET3_ASSERT(rbi->m); + VMXNET3_ASSERT(rcd->len <= rxd->len); + VMXNET3_ASSERT(rbi->m); #endif - if (rcd->len == 0) { - PMD_RX_LOG(DEBUG, "Rx buf was skipped. rxring[%d][%d]\n)", - ring_idx, idx); + if (unlikely(rcd->len == 0)) { + PMD_RX_LOG(DEBUG, "Rx buf was skipped. rxring[%d][%d]\n)", + ring_idx, idx); #ifdef RTE_LIBRTE_VMXNET3_DEBUG_DRIVER - VMXNET3_ASSERT(rcd->sop && rcd->eop); + VMXNET3_ASSERT(rcd->sop && rcd->eop); #endif - rte_pktmbuf_free_seg(rbi->m); - - goto rcd_done; - } + rte_pktmbuf_free_seg(rbi->m); + goto rcd_done; + } - /* Assuming a packet is coming in a single packet buffer */ - if (rxd->btype != VMXNET3_RXD_BTYPE_HEAD) { - PMD_RX_LOG(DEBUG, - "Alert : Misbehaving device, incorrect " - " buffer type used. iPacket dropped.\n"); - rte_pktmbuf_free_seg(rbi->m); - goto rcd_done; - } + /* Assuming a packet is coming in a single packet buffer */ + if (unlikely(rxd->btype != VMXNET3_RXD_BTYPE_HEAD)) { + PMD_RX_LOG(DEBUG, + "Alert : Misbehaving device, incorrect " + " buffer type used. iPacket dropped."); + rte_pktmbuf_free_seg(rbi->m); + goto rcd_done; + } #ifdef RTE_LIBRTE_VMXNET3_DEBUG_DRIVER - VMXNET3_ASSERT(rxd->btype == VMXNET3_RXD_BTYPE_HEAD); + VMXNET3_ASSERT(rxd->btype == VMXNET3_RXD_BTYPE_HEAD); #endif - /* Get the packet buffer pointer from buf_info */ - rxm = rbi->m; - - /* Clear descriptor associated buf_info to be reused */ - rbi->m = NULL; - rbi->bufPA = 0; - - /* Update the index that we received a packet */ - rxq->cmd_ring[ring_idx].next2comp = idx; - - /* For RCD with EOP set, check if there is frame error */ - if (rcd->err) { - rxq->stats.drop_total++; - rxq->stats.drop_err++; - - if (!rcd->fcs) { - rxq->stats.drop_fcs++; - PMD_RX_LOG(ERR, "Recv packet dropped due to frame err.\n"); - } - PMD_RX_LOG(ERR, "Error in received packet rcd#:%d rxd:%d\n", - (int)(rcd - (struct Vmxnet3_RxCompDesc *) - rxq->comp_ring.base), rcd->rxdIdx); - rte_pktmbuf_free_seg(rxm); - - goto rcd_done; - } + /* Get the packet buffer pointer from buf_info */ + rxm = rbi->m; - /* Check for hardware stripped VLAN tag */ - if (rcd->ts) { + /* Clear descriptor associated buf_info to be reused */ + rbi->m = NULL; + rbi->bufPA = 0; - PMD_RX_LOG(ERR, "Received packet with vlan ID: %d.\n", - rcd->tci); - rxm->ol_flags = PKT_RX_VLAN_PKT; + /* Update the index that we received a packet */ + rxq->cmd_ring[ring_idx].next2comp = idx; -#ifdef RTE_LIBRTE_VMXNET3_DEBUG_DRIVER - VMXNET3_ASSERT(rxm && - rte_pktmbuf_mtod(rxm, void *)); -#endif - /* Copy vlan tag in packet buffer */ - rxm->pkt.vlan_macip.f.vlan_tci = - rte_le_to_cpu_16((uint16_t)rcd->tci); + /* For RCD with EOP set, check if there is frame error */ + if (unlikely(rcd->err)) { + rxq->stats.drop_total++; + rxq->stats.drop_err++; - } else - rxm->ol_flags = 0; + if (!rcd->fcs) { + rxq->stats.drop_fcs++; + PMD_RX_LOG(ERR, "Recv packet dropped due to frame err."); + } + PMD_RX_LOG(ERR, "Error in received packet rcd#:%d rxd:%d", + (int)(rcd - (struct Vmxnet3_RxCompDesc *) + rxq->comp_ring.base), rcd->rxdIdx); + rte_pktmbuf_free_seg(rxm); + goto rcd_done; + } - /* Initialize newly received packet buffer */ - rxm->pkt.in_port = rxq->port_id; - rxm->pkt.nb_segs = 1; - rxm->pkt.next = NULL; - rxm->pkt.pkt_len = (uint16_t)rcd->len; - rxm->pkt.data_len = (uint16_t)rcd->len; - rxm->pkt.in_port = rxq->port_id; - rxm->pkt.vlan_macip.f.vlan_tci = 0; - rxm->pkt.data = (char *)rxm->buf_addr + RTE_PKTMBUF_HEADROOM; + /* Check for hardware stripped VLAN tag */ + if (rcd->ts) { + PMD_RX_LOG(DEBUG, "Received packet with vlan ID: %d.", + rcd->tci); + rxm->ol_flags = PKT_RX_VLAN_PKT; + /* Copy vlan tag in packet buffer */ + rxm->vlan_tci = rte_le_to_cpu_16((uint16_t)rcd->tci); + } else { + rxm->ol_flags = 0; + rxm->vlan_tci = 0; + } - rx_pkts[nb_rx++] = rxm; + /* Initialize newly received packet buffer */ + rxm->port = rxq->port_id; + rxm->nb_segs = 1; + rxm->next = NULL; + rxm->pkt_len = (uint16_t)rcd->len; + rxm->data_len = (uint16_t)rcd->len; + rxm->data_off = RTE_PKTMBUF_HEADROOM; + + /* Check packet type, checksum errors, etc. Only support IPv4 for now. */ + if (rcd->v4) { + struct ether_hdr *eth = rte_pktmbuf_mtod(rxm, struct ether_hdr *); + struct ipv4_hdr *ip = (struct ipv4_hdr *)(eth + 1); + + if (((ip->version_ihl & 0xf) << 2) > (int)sizeof(struct ipv4_hdr)) + rxm->ol_flags |= PKT_RX_IPV4_HDR_EXT; + else + rxm->ol_flags |= PKT_RX_IPV4_HDR; + + if (!rcd->cnc) { + if (!rcd->ipc) + rxm->ol_flags |= PKT_RX_IP_CKSUM_BAD; + + if ((rcd->tcp || rcd->udp) && !rcd->tuc) + rxm->ol_flags |= PKT_RX_L4_CKSUM_BAD; + } + } + rx_pkts[nb_rx++] = rxm; rcd_done: - rxq->cmd_ring[ring_idx].next2comp = idx; - VMXNET3_INC_RING_IDX_ONLY(rxq->cmd_ring[ring_idx].next2comp, rxq->cmd_ring[ring_idx].size); - - /* It's time to allocate some new buf and renew descriptors */ - vmxnet3_post_rx_bufs(rxq, ring_idx); - if (unlikely(rxq->shared->ctrl.updateRxProd)) { - VMXNET3_WRITE_BAR0_REG(hw, rxprod_reg[ring_idx] + (rxq->queue_id * VMXNET3_REG_ALIGN), - rxq->cmd_ring[ring_idx].next2fill); - } + rxq->cmd_ring[ring_idx].next2comp = idx; + VMXNET3_INC_RING_IDX_ONLY(rxq->cmd_ring[ring_idx].next2comp, rxq->cmd_ring[ring_idx].size); + + /* It's time to allocate some new buf and renew descriptors */ + vmxnet3_post_rx_bufs(rxq, ring_idx); + if (unlikely(rxq->shared->ctrl.updateRxProd)) { + VMXNET3_WRITE_BAR0_REG(hw, rxprod_reg[ring_idx] + (rxq->queue_id * VMXNET3_REG_ALIGN), + rxq->cmd_ring[ring_idx].next2fill); + } - /* Advance to the next descriptor in comp_ring */ - vmxnet3_comp_ring_adv_next2proc(&rxq->comp_ring); + /* Advance to the next descriptor in comp_ring */ + vmxnet3_comp_ring_adv_next2proc(&rxq->comp_ring); - rcd = &rxq->comp_ring.base[rxq->comp_ring.next2proc].rcd; - nb_rxd++; - if (nb_rxd > rxq->cmd_ring[0].size) { - PMD_RX_LOG(ERR, - "Used up quota of receiving packets," - " relinquish control.\n"); - break; - } + rcd = &rxq->comp_ring.base[rxq->comp_ring.next2proc].rcd; + nb_rxd++; + if (nb_rxd > rxq->cmd_ring[0].size) { + PMD_RX_LOG(ERR, + "Used up quota of receiving packets," + " relinquish control."); + break; } } @@ -627,31 +722,31 @@ vmxnet3_dev_tx_queue_setup(struct rte_eth_dev *dev, unsigned int socket_id, __attribute__((unused)) const struct rte_eth_txconf *tx_conf) { + struct vmxnet3_hw *hw = dev->data->dev_private; const struct rte_memzone *mz; struct vmxnet3_tx_queue *txq; - struct vmxnet3_hw *hw; struct vmxnet3_cmd_ring *ring; struct vmxnet3_comp_ring *comp_ring; + struct vmxnet3_data_ring *data_ring; int size; PMD_INIT_FUNC_TRACE(); - hw = VMXNET3_DEV_PRIVATE_TO_HW(dev->data->dev_private); if ((tx_conf->txq_flags & ETH_TXQ_FLAGS_NOMULTSEGS) != ETH_TXQ_FLAGS_NOMULTSEGS) { - PMD_INIT_LOG(ERR, "TX Multi segment not support yet\n"); + PMD_INIT_LOG(ERR, "TX Multi segment not support yet"); return -EINVAL; } if ((tx_conf->txq_flags & ETH_TXQ_FLAGS_NOOFFLOADS) != ETH_TXQ_FLAGS_NOOFFLOADS) { - PMD_INIT_LOG(ERR, "TX not support offload function yet\n"); + PMD_INIT_LOG(ERR, "TX not support offload function yet"); return -EINVAL; } - txq = rte_zmalloc("ethdev_tx_queue", sizeof(struct vmxnet3_tx_queue), CACHE_LINE_SIZE); + txq = rte_zmalloc("ethdev_tx_queue", sizeof(struct vmxnet3_tx_queue), RTE_CACHE_LINE_SIZE); if (txq == NULL) { - PMD_INIT_LOG(ERR, "Can not allocate tx queue structure\n"); + PMD_INIT_LOG(ERR, "Can not allocate tx queue structure"); return -ENOMEM; } @@ -664,21 +759,22 @@ vmxnet3_dev_tx_queue_setup(struct rte_eth_dev *dev, ring = &txq->cmd_ring; comp_ring = &txq->comp_ring; + data_ring = &txq->data_ring; /* Tx vmxnet ring length should be between 512-4096 */ if (nb_desc < VMXNET3_DEF_TX_RING_SIZE) { - PMD_INIT_LOG(ERR, "VMXNET3 Tx Ring Size Min: %u\n", + PMD_INIT_LOG(ERR, "VMXNET3 Tx Ring Size Min: %u", VMXNET3_DEF_TX_RING_SIZE); return -EINVAL; } else if (nb_desc > VMXNET3_TX_RING_MAX_SIZE) { - PMD_INIT_LOG(ERR, "VMXNET3 Tx Ring Size Max: %u\n", + PMD_INIT_LOG(ERR, "VMXNET3 Tx Ring Size Max: %u", VMXNET3_TX_RING_MAX_SIZE); return -EINVAL; } else { ring->size = nb_desc; ring->size &= ~VMXNET3_RING_SIZE_MASK; } - comp_ring->size = ring->size; + comp_ring->size = data_ring->size = ring->size; /* Tx vmxnet rings structure initialization*/ ring->next2fill = 0; @@ -689,10 +785,11 @@ vmxnet3_dev_tx_queue_setup(struct rte_eth_dev *dev, size = sizeof(struct Vmxnet3_TxDesc) * ring->size; size += sizeof(struct Vmxnet3_TxCompDesc) * comp_ring->size; + size += sizeof(struct Vmxnet3_TxDataDesc) * data_ring->size; mz = ring_dma_zone_reserve(dev, "txdesc", queue_idx, size, socket_id); if (mz == NULL) { - PMD_INIT_LOG(ERR, "ERROR: Creating queue descriptors zone\n"); + PMD_INIT_LOG(ERR, "ERROR: Creating queue descriptors zone"); return -ENOMEM; } memset(mz->addr, 0, mz->len); @@ -706,11 +803,16 @@ vmxnet3_dev_tx_queue_setup(struct rte_eth_dev *dev, comp_ring->basePA = ring->basePA + (sizeof(struct Vmxnet3_TxDesc) * ring->size); + /* data_ring initialization */ + data_ring->base = (Vmxnet3_TxDataDesc *)(comp_ring->base + comp_ring->size); + data_ring->basePA = comp_ring->basePA + + (sizeof(struct Vmxnet3_TxCompDesc) * comp_ring->size); + /* cmd_ring0 buf_info allocation */ ring->buf_info = rte_zmalloc("tx_ring_buf_info", - ring->size * sizeof(vmxnet3_buf_info_t), CACHE_LINE_SIZE); + ring->size * sizeof(vmxnet3_buf_info_t), RTE_CACHE_LINE_SIZE); if (ring->buf_info == NULL) { - PMD_INIT_LOG(ERR, "ERROR: Creating tx_buf_info structure\n"); + PMD_INIT_LOG(ERR, "ERROR: Creating tx_buf_info structure"); return -ENOMEM; } @@ -730,7 +832,7 @@ vmxnet3_dev_rx_queue_setup(struct rte_eth_dev *dev, { const struct rte_memzone *mz; struct vmxnet3_rx_queue *rxq; - struct vmxnet3_hw *hw; + struct vmxnet3_hw *hw = dev->data->dev_private; struct vmxnet3_cmd_ring *ring0, *ring1, *ring; struct vmxnet3_comp_ring *comp_ring; int size; @@ -740,7 +842,6 @@ vmxnet3_dev_rx_queue_setup(struct rte_eth_dev *dev, struct rte_pktmbuf_pool_private *mbp_priv; PMD_INIT_FUNC_TRACE(); - hw = VMXNET3_DEV_PRIVATE_TO_HW(dev->data->dev_private); mbp_priv = (struct rte_pktmbuf_pool_private *) rte_mempool_get_priv(mp); @@ -749,14 +850,14 @@ vmxnet3_dev_rx_queue_setup(struct rte_eth_dev *dev, if (dev->data->dev_conf.rxmode.max_rx_pkt_len > buf_size) { PMD_INIT_LOG(ERR, "buf_size = %u, max_pkt_len = %u, " - "VMXNET3 don't support scatter packets yet\n", + "VMXNET3 don't support scatter packets yet", buf_size, dev->data->dev_conf.rxmode.max_rx_pkt_len); return -EINVAL; } - rxq = rte_zmalloc("ethdev_rx_queue", sizeof(struct vmxnet3_rx_queue), CACHE_LINE_SIZE); + rxq = rte_zmalloc("ethdev_rx_queue", sizeof(struct vmxnet3_rx_queue), RTE_CACHE_LINE_SIZE); if (rxq == NULL) { - PMD_INIT_LOG(ERR, "Can not allocate rx queue structure\n"); + PMD_INIT_LOG(ERR, "Can not allocate rx queue structure"); return -ENOMEM; } @@ -775,10 +876,10 @@ vmxnet3_dev_rx_queue_setup(struct rte_eth_dev *dev, /* Rx vmxnet rings length should be between 256-4096 */ if (nb_desc < VMXNET3_DEF_RX_RING_SIZE) { - PMD_INIT_LOG(ERR, "VMXNET3 Rx Ring Size Min: 256\n"); + PMD_INIT_LOG(ERR, "VMXNET3 Rx Ring Size Min: 256"); return -EINVAL; } else if (nb_desc > VMXNET3_RX_RING_MAX_SIZE) { - PMD_INIT_LOG(ERR, "VMXNET3 Rx Ring Size Max: 4096\n"); + PMD_INIT_LOG(ERR, "VMXNET3 Rx Ring Size Max: 4096"); return -EINVAL; } else { ring0->size = nb_desc; @@ -803,7 +904,7 @@ vmxnet3_dev_rx_queue_setup(struct rte_eth_dev *dev, mz = ring_dma_zone_reserve(dev, "rxdesc", queue_idx, size, socket_id); if (mz == NULL) { - PMD_INIT_LOG(ERR, "ERROR: Creating queue descriptors zone\n"); + PMD_INIT_LOG(ERR, "ERROR: Creating queue descriptors zone"); return -ENOMEM; } memset(mz->addr, 0, mz->len); @@ -817,7 +918,7 @@ vmxnet3_dev_rx_queue_setup(struct rte_eth_dev *dev, ring1->basePA = ring0->basePA + sizeof(struct Vmxnet3_RxDesc) * ring0->size; /* comp_ring initialization */ - comp_ring->base = ring1->base + ring1->size; + comp_ring->base = ring1->base + ring1->size; comp_ring->basePA = ring1->basePA + sizeof(struct Vmxnet3_RxDesc) * ring1->size; @@ -828,9 +929,9 @@ vmxnet3_dev_rx_queue_setup(struct rte_eth_dev *dev, ring->rid = i; snprintf(mem_name, sizeof(mem_name), "rx_ring_%d_buf_info", i); - ring->buf_info = rte_zmalloc(mem_name, ring->size * sizeof(vmxnet3_buf_info_t), CACHE_LINE_SIZE); + ring->buf_info = rte_zmalloc(mem_name, ring->size * sizeof(vmxnet3_buf_info_t), RTE_CACHE_LINE_SIZE); if (ring->buf_info == NULL) { - PMD_INIT_LOG(ERR, "ERROR: Creating rx_buf_info structure\n"); + PMD_INIT_LOG(ERR, "ERROR: Creating rx_buf_info structure"); return -ENOMEM; } } @@ -848,12 +949,12 @@ vmxnet3_dev_rx_queue_setup(struct rte_eth_dev *dev, int vmxnet3_dev_rxtx_init(struct rte_eth_dev *dev) { - struct vmxnet3_hw *hw; + struct vmxnet3_hw *hw = dev->data->dev_private; + int i, ret; uint8_t j; PMD_INIT_FUNC_TRACE(); - hw = VMXNET3_DEV_PRIVATE_TO_HW(dev->data->dev_private); for (i = 0; i < hw->num_rx_queues; i++) { vmxnet3_rx_queue_t *rxq = dev->data->rx_queues[i]; @@ -862,7 +963,7 @@ vmxnet3_dev_rxtx_init(struct rte_eth_dev *dev) /* Passing 0 as alloc_num will allocate full ring */ ret = vmxnet3_post_rx_bufs(rxq, j); if (ret <= 0) { - PMD_INIT_LOG(ERR, "ERROR: Posting Rxq: %d buffers ring: %d\n", i, j); + PMD_INIT_LOG(ERR, "ERROR: Posting Rxq: %d buffers ring: %d", i, j); return -ret; } /* Updating device with the index:next2fill to fill the mbufs for coming packets */ @@ -903,14 +1004,14 @@ vmxnet3_rss_configure(struct rte_eth_dev *dev) ETH_RSS_IPV6 | \ ETH_RSS_IPV6_TCP) - struct vmxnet3_hw *hw; + struct vmxnet3_hw *hw = dev->data->dev_private; struct VMXNET3_RSSConf *dev_rss_conf; struct rte_eth_rss_conf *port_rss_conf; uint64_t rss_hf; uint8_t i, j; PMD_INIT_FUNC_TRACE(); - hw = VMXNET3_DEV_PRIVATE_TO_HW(dev->data->dev_private); + dev_rss_conf = hw->rss_conf; port_rss_conf = &dev->data->dev_conf.rx_adv_conf.rss_conf; @@ -958,7 +1059,7 @@ int vmxnet3_vlan_configure(struct rte_eth_dev *dev) { uint8_t i; - struct vmxnet3_hw *hw = VMXNET3_DEV_PRIVATE_TO_HW(dev->data->dev_private); + struct vmxnet3_hw *hw = dev->data->dev_private; uint32_t *vf_table = hw->shared->devRead.rxFilterConf.vfTable; PMD_INIT_FUNC_TRACE(); @@ -969,7 +1070,7 @@ vmxnet3_vlan_configure(struct rte_eth_dev *dev) vf_table[i] = 0; /* To-Do: Provide another routine in dev_ops for user config */ - PMD_INIT_LOG(DEBUG, "Registering VLAN portid: %"PRIu8" tag %u\n", + PMD_INIT_LOG(DEBUG, "Registering VLAN portid: %"PRIu8" tag %u", dev->data->port_id, vf_table[i]); }