+ nicvf_netdev_qidx(nic, qidx));
+ nicvf_dev_tx_queue_release(
+ dev->data->tx_queues[nicvf_netdev_qidx(nic, qidx)]);
+ dev->data->tx_queues[nicvf_netdev_qidx(nic, qidx)] = NULL;
+ }
+
+ /* Allocating tx queue data structure */
+ txq = rte_zmalloc_socket("ethdev TX queue", sizeof(struct nicvf_txq),
+ RTE_CACHE_LINE_SIZE, nic->node);
+ if (txq == NULL) {
+ PMD_INIT_LOG(ERR, "Failed to allocate txq=%d",
+ nicvf_netdev_qidx(nic, qidx));
+ return -ENOMEM;
+ }
+
+ txq->nic = nic;
+ txq->queue_id = qidx;
+ txq->tx_free_thresh = tx_free_thresh;
+ txq->sq_head = nicvf_qset_base(nic, qidx) + NIC_QSET_SQ_0_7_HEAD;
+ txq->sq_door = nicvf_qset_base(nic, qidx) + NIC_QSET_SQ_0_7_DOOR;
+ offloads = tx_conf->offloads | dev->data->dev_conf.txmode.offloads;
+ txq->offloads = offloads;
+
+ is_single_pool = !!(offloads & DEV_TX_OFFLOAD_MBUF_FAST_FREE);
+
+ /* Choose optimum free threshold value for multipool case */
+ if (!is_single_pool) {
+ txq->tx_free_thresh = (uint16_t)
+ (tx_conf->tx_free_thresh == NICVF_DEFAULT_TX_FREE_THRESH ?
+ NICVF_TX_FREE_MPOOL_THRESH :
+ tx_conf->tx_free_thresh);
+ txq->pool_free = nicvf_multi_pool_free_xmited_buffers;
+ } else {
+ txq->pool_free = nicvf_single_pool_free_xmited_buffers;
+ }
+
+ /* Allocate software ring */
+ txq->txbuffs = rte_zmalloc_socket("txq->txbuffs",
+ nb_desc * sizeof(struct rte_mbuf *),
+ RTE_CACHE_LINE_SIZE, nic->node);
+
+ if (txq->txbuffs == NULL) {
+ nicvf_dev_tx_queue_release(txq);
+ return -ENOMEM;
+ }
+
+ if (nicvf_qset_sq_alloc(dev, nic, txq, qidx, nb_desc)) {
+ PMD_INIT_LOG(ERR, "Failed to allocate mem for sq %d", qidx);
+ nicvf_dev_tx_queue_release(txq);
+ return -ENOMEM;
+ }
+
+ nicvf_tx_queue_reset(txq);
+
+ PMD_INIT_LOG(DEBUG, "[%d] txq=%p nb_desc=%d desc=%p"
+ " phys=0x%" PRIx64 " offloads=0x%" PRIx64,
+ nicvf_netdev_qidx(nic, qidx), txq, nb_desc, txq->desc,
+ txq->phys, txq->offloads);
+
+ dev->data->tx_queues[nicvf_netdev_qidx(nic, qidx)] = txq;
+ dev->data->tx_queue_state[nicvf_netdev_qidx(nic, qidx)] =
+ RTE_ETH_QUEUE_STATE_STOPPED;
+ return 0;
+}
+
+static inline void
+nicvf_rx_queue_release_mbufs(struct rte_eth_dev *dev, struct nicvf_rxq *rxq)
+{
+ uint32_t rxq_cnt;
+ uint32_t nb_pkts, released_pkts = 0;
+ uint32_t refill_cnt = 0;
+ struct rte_mbuf *rx_pkts[NICVF_MAX_RX_FREE_THRESH];
+
+ if (dev->rx_pkt_burst == NULL)
+ return;
+
+ while ((rxq_cnt = nicvf_dev_rx_queue_count(dev,
+ nicvf_netdev_qidx(rxq->nic, rxq->queue_id)))) {
+ nb_pkts = dev->rx_pkt_burst(rxq, rx_pkts,
+ NICVF_MAX_RX_FREE_THRESH);
+ PMD_DRV_LOG(INFO, "nb_pkts=%d rxq_cnt=%d", nb_pkts, rxq_cnt);
+ while (nb_pkts) {
+ rte_pktmbuf_free_seg(rx_pkts[--nb_pkts]);
+ released_pkts++;
+ }
+ }
+
+
+ refill_cnt += nicvf_dev_rbdr_refill(dev,
+ nicvf_netdev_qidx(rxq->nic, rxq->queue_id));
+
+ PMD_DRV_LOG(INFO, "free_cnt=%d refill_cnt=%d",
+ released_pkts, refill_cnt);
+}
+
+static void
+nicvf_rx_queue_reset(struct nicvf_rxq *rxq)
+{
+ rxq->head = 0;
+ rxq->available_space = 0;
+ rxq->recv_buffers = 0;
+}
+
+static inline int
+nicvf_vf_start_rx_queue(struct rte_eth_dev *dev, struct nicvf *nic,
+ uint16_t qidx)
+{
+ struct nicvf_rxq *rxq;
+ int ret;
+
+ assert(qidx < MAX_RCV_QUEUES_PER_QS);
+
+ if (dev->data->rx_queue_state[nicvf_netdev_qidx(nic, qidx)] ==
+ RTE_ETH_QUEUE_STATE_STARTED)
+ return 0;
+
+ /* Update rbdr pointer to all rxq */
+ rxq = dev->data->rx_queues[nicvf_netdev_qidx(nic, qidx)];
+ rxq->shared_rbdr = nic->rbdr;
+
+ ret = nicvf_qset_rq_config(nic, qidx, rxq);
+ if (ret) {
+ PMD_INIT_LOG(ERR, "Failed to configure rq VF%d %d %d",
+ nic->vf_id, qidx, ret);
+ goto config_rq_error;
+ }
+ ret = nicvf_qset_cq_config(nic, qidx, rxq);
+ if (ret) {
+ PMD_INIT_LOG(ERR, "Failed to configure cq VF%d %d %d",
+ nic->vf_id, qidx, ret);
+ goto config_cq_error;
+ }
+
+ dev->data->rx_queue_state[nicvf_netdev_qidx(nic, qidx)] =
+ RTE_ETH_QUEUE_STATE_STARTED;
+ return 0;
+
+config_cq_error:
+ nicvf_qset_cq_reclaim(nic, qidx);
+config_rq_error:
+ nicvf_qset_rq_reclaim(nic, qidx);
+ return ret;
+}
+
+static inline int
+nicvf_vf_stop_rx_queue(struct rte_eth_dev *dev, struct nicvf *nic,
+ uint16_t qidx)
+{
+ struct nicvf_rxq *rxq;
+ int ret, other_error;
+
+ if (dev->data->rx_queue_state[nicvf_netdev_qidx(nic, qidx)] ==
+ RTE_ETH_QUEUE_STATE_STOPPED)
+ return 0;
+
+ ret = nicvf_qset_rq_reclaim(nic, qidx);
+ if (ret)
+ PMD_INIT_LOG(ERR, "Failed to reclaim rq VF%d %d %d",
+ nic->vf_id, qidx, ret);
+
+ other_error = ret;
+ rxq = dev->data->rx_queues[nicvf_netdev_qidx(nic, qidx)];
+ nicvf_rx_queue_release_mbufs(dev, rxq);
+ nicvf_rx_queue_reset(rxq);
+
+ ret = nicvf_qset_cq_reclaim(nic, qidx);
+ if (ret)
+ PMD_INIT_LOG(ERR, "Failed to reclaim cq VF%d %d %d",
+ nic->vf_id, qidx, ret);
+
+ other_error |= ret;
+ dev->data->rx_queue_state[nicvf_netdev_qidx(nic, qidx)] =
+ RTE_ETH_QUEUE_STATE_STOPPED;
+ return other_error;
+}
+
+static void
+nicvf_dev_rx_queue_release(void *rx_queue)
+{
+ PMD_INIT_FUNC_TRACE();
+
+ rte_free(rx_queue);
+}
+
+static int
+nicvf_dev_rx_queue_start(struct rte_eth_dev *dev, uint16_t qidx)
+{
+ struct nicvf *nic = nicvf_pmd_priv(dev);
+ int ret;
+
+ if (qidx >= MAX_RCV_QUEUES_PER_QS)
+ nic = nic->snicvf[(qidx / MAX_RCV_QUEUES_PER_QS - 1)];
+
+ qidx = qidx % MAX_RCV_QUEUES_PER_QS;
+
+ ret = nicvf_vf_start_rx_queue(dev, nic, qidx);
+ if (ret)
+ return ret;
+
+ ret = nicvf_configure_cpi(dev);
+ if (ret)
+ return ret;
+
+ return nicvf_configure_rss_reta(dev);
+}
+
+static int
+nicvf_dev_rx_queue_stop(struct rte_eth_dev *dev, uint16_t qidx)
+{
+ int ret;
+ struct nicvf *nic = nicvf_pmd_priv(dev);
+
+ if (qidx >= MAX_SND_QUEUES_PER_QS)
+ nic = nic->snicvf[(qidx / MAX_SND_QUEUES_PER_QS - 1)];
+
+ qidx = qidx % MAX_RCV_QUEUES_PER_QS;
+
+ ret = nicvf_vf_stop_rx_queue(dev, nic, qidx);
+ ret |= nicvf_configure_cpi(dev);
+ ret |= nicvf_configure_rss_reta(dev);
+ return ret;
+}
+
+static int
+nicvf_dev_tx_queue_start(struct rte_eth_dev *dev, uint16_t qidx)
+{
+ struct nicvf *nic = nicvf_pmd_priv(dev);
+
+ if (qidx >= MAX_SND_QUEUES_PER_QS)
+ nic = nic->snicvf[(qidx / MAX_SND_QUEUES_PER_QS - 1)];
+
+ qidx = qidx % MAX_SND_QUEUES_PER_QS;
+
+ return nicvf_vf_start_tx_queue(dev, nic, qidx);
+}
+
+static int
+nicvf_dev_tx_queue_stop(struct rte_eth_dev *dev, uint16_t qidx)
+{
+ struct nicvf *nic = nicvf_pmd_priv(dev);
+
+ if (qidx >= MAX_SND_QUEUES_PER_QS)
+ nic = nic->snicvf[(qidx / MAX_SND_QUEUES_PER_QS - 1)];
+
+ qidx = qidx % MAX_SND_QUEUES_PER_QS;
+
+ return nicvf_vf_stop_tx_queue(dev, nic, qidx);
+}
+
+static inline void
+nicvf_rxq_mbuf_setup(struct nicvf_rxq *rxq)
+{
+ uintptr_t p;
+ struct rte_mbuf mb_def;
+ struct nicvf *nic = rxq->nic;
+
+ RTE_BUILD_BUG_ON(sizeof(union mbuf_initializer) != 8);
+ RTE_BUILD_BUG_ON(offsetof(struct rte_mbuf, data_off) % 8 != 0);
+ RTE_BUILD_BUG_ON(offsetof(struct rte_mbuf, refcnt) -
+ offsetof(struct rte_mbuf, data_off) != 2);
+ RTE_BUILD_BUG_ON(offsetof(struct rte_mbuf, nb_segs) -
+ offsetof(struct rte_mbuf, data_off) != 4);
+ RTE_BUILD_BUG_ON(offsetof(struct rte_mbuf, port) -
+ offsetof(struct rte_mbuf, data_off) != 6);
+ RTE_BUILD_BUG_ON(offsetof(struct nicvf_rxq, rxq_fastpath_data_end) -
+ offsetof(struct nicvf_rxq,
+ rxq_fastpath_data_start) > 128);
+ mb_def.nb_segs = 1;
+ mb_def.data_off = RTE_PKTMBUF_HEADROOM + (nic->skip_bytes);
+ mb_def.port = rxq->port_id;
+ rte_mbuf_refcnt_set(&mb_def, 1);
+
+ /* Prevent compiler reordering: rearm_data covers previous fields */
+ rte_compiler_barrier();
+ p = (uintptr_t)&mb_def.rearm_data;
+ rxq->mbuf_initializer.value = *(uint64_t *)p;
+}
+
+static int
+nicvf_dev_rx_queue_setup(struct rte_eth_dev *dev, uint16_t qidx,
+ uint16_t nb_desc, unsigned int socket_id,
+ const struct rte_eth_rxconf *rx_conf,
+ struct rte_mempool *mp)
+{
+ uint16_t rx_free_thresh;
+ struct nicvf_rxq *rxq;
+ struct nicvf *nic = nicvf_pmd_priv(dev);
+ uint64_t offloads;
+ uint32_t buffsz;
+ struct rte_pktmbuf_pool_private *mbp_priv;
+
+ PMD_INIT_FUNC_TRACE();
+
+ /* First skip check */
+ mbp_priv = rte_mempool_get_priv(mp);
+ buffsz = mbp_priv->mbuf_data_room_size - RTE_PKTMBUF_HEADROOM;
+ if (buffsz < (uint32_t)(nic->skip_bytes)) {
+ PMD_INIT_LOG(ERR, "First skip is more than configured buffer size");
+ return -EINVAL;
+ }
+
+ if (qidx >= MAX_RCV_QUEUES_PER_QS)
+ nic = nic->snicvf[qidx / MAX_RCV_QUEUES_PER_QS - 1];
+
+ qidx = qidx % MAX_RCV_QUEUES_PER_QS;
+
+ /* Socket id check */
+ if (socket_id != (unsigned int)SOCKET_ID_ANY && socket_id != nic->node)
+ PMD_DRV_LOG(WARNING, "socket_id expected %d, configured %d",
+ socket_id, nic->node);
+
+ /* Mempool memory must be contiguous, so must be one memory segment*/
+ if (mp->nb_mem_chunks != 1) {
+ PMD_INIT_LOG(ERR, "Non-contiguous mempool, add more huge pages");
+ return -EINVAL;
+ }
+
+ /* Mempool memory must be physically contiguous */
+ if (mp->flags & MEMPOOL_F_NO_IOVA_CONTIG) {
+ PMD_INIT_LOG(ERR, "Mempool memory must be physically contiguous");
+ return -EINVAL;
+ }
+
+ /* Rx deferred start is not supported */
+ if (rx_conf->rx_deferred_start) {
+ PMD_INIT_LOG(ERR, "Rx deferred start not supported");
+ return -EINVAL;
+ }
+
+ /* Roundup nb_desc to available qsize and validate max number of desc */
+ nb_desc = nicvf_qsize_cq_roundup(nb_desc);
+ if (nb_desc == 0) {
+ PMD_INIT_LOG(ERR, "Value nb_desc beyond available hw cq qsize");
+ return -EINVAL;
+ }
+
+
+ /* Check rx_free_thresh upper bound */
+ rx_free_thresh = (uint16_t)((rx_conf->rx_free_thresh) ?
+ rx_conf->rx_free_thresh :
+ NICVF_DEFAULT_RX_FREE_THRESH);
+ if (rx_free_thresh > NICVF_MAX_RX_FREE_THRESH ||
+ rx_free_thresh >= nb_desc * .75) {
+ PMD_INIT_LOG(ERR, "rx_free_thresh greater than expected %d",
+ rx_free_thresh);
+ return -EINVAL;
+ }
+
+ /* Free memory prior to re-allocation if needed */
+ if (dev->data->rx_queues[nicvf_netdev_qidx(nic, qidx)] != NULL) {
+ PMD_RX_LOG(DEBUG, "Freeing memory prior to re-allocation %d",
+ nicvf_netdev_qidx(nic, qidx));
+ nicvf_dev_rx_queue_release(
+ dev->data->rx_queues[nicvf_netdev_qidx(nic, qidx)]);
+ dev->data->rx_queues[nicvf_netdev_qidx(nic, qidx)] = NULL;
+ }
+
+ /* Allocate rxq memory */
+ rxq = rte_zmalloc_socket("ethdev rx queue", sizeof(struct nicvf_rxq),
+ RTE_CACHE_LINE_SIZE, nic->node);
+ if (rxq == NULL) {
+ PMD_INIT_LOG(ERR, "Failed to allocate rxq=%d",
+ nicvf_netdev_qidx(nic, qidx));
+ return -ENOMEM;
+ }
+
+ rxq->nic = nic;
+ rxq->pool = mp;
+ rxq->queue_id = qidx;
+ rxq->port_id = dev->data->port_id;
+ rxq->rx_free_thresh = rx_free_thresh;
+ rxq->rx_drop_en = rx_conf->rx_drop_en;
+ rxq->cq_status = nicvf_qset_base(nic, qidx) + NIC_QSET_CQ_0_7_STATUS;
+ rxq->cq_door = nicvf_qset_base(nic, qidx) + NIC_QSET_CQ_0_7_DOOR;
+ rxq->precharge_cnt = 0;
+
+ if (nicvf_hw_cap(nic) & NICVF_CAP_CQE_RX2)
+ rxq->rbptr_offset = NICVF_CQE_RX2_RBPTR_WORD;
+ else
+ rxq->rbptr_offset = NICVF_CQE_RBPTR_WORD;
+
+ nicvf_rxq_mbuf_setup(rxq);
+
+ /* Alloc completion queue */
+ if (nicvf_qset_cq_alloc(dev, nic, rxq, rxq->queue_id, nb_desc)) {
+ PMD_INIT_LOG(ERR, "failed to allocate cq %u", rxq->queue_id);
+ nicvf_dev_rx_queue_release(rxq);
+ return -ENOMEM;
+ }
+
+ nicvf_rx_queue_reset(rxq);
+
+ offloads = rx_conf->offloads | dev->data->dev_conf.rxmode.offloads;
+ PMD_INIT_LOG(DEBUG, "[%d] rxq=%p pool=%s nb_desc=(%d/%d)"
+ " phy=0x%" PRIx64 " offloads=0x%" PRIx64,
+ nicvf_netdev_qidx(nic, qidx), rxq, mp->name, nb_desc,
+ rte_mempool_avail_count(mp), rxq->phys, offloads);
+
+ dev->data->rx_queues[nicvf_netdev_qidx(nic, qidx)] = rxq;
+ dev->data->rx_queue_state[nicvf_netdev_qidx(nic, qidx)] =
+ RTE_ETH_QUEUE_STATE_STOPPED;
+ return 0;
+}
+
+static int
+nicvf_dev_info_get(struct rte_eth_dev *dev, struct rte_eth_dev_info *dev_info)
+{
+ struct nicvf *nic = nicvf_pmd_priv(dev);
+ struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev);
+
+ PMD_INIT_FUNC_TRACE();
+
+ /* Autonegotiation may be disabled */
+ dev_info->speed_capa = ETH_LINK_SPEED_FIXED;
+ dev_info->speed_capa |= ETH_LINK_SPEED_10M | ETH_LINK_SPEED_100M |
+ ETH_LINK_SPEED_1G | ETH_LINK_SPEED_10G;
+ if (nicvf_hw_version(nic) != PCI_SUB_DEVICE_ID_CN81XX_NICVF)
+ dev_info->speed_capa |= ETH_LINK_SPEED_40G;
+
+ dev_info->min_rx_bufsize = RTE_ETHER_MIN_MTU;
+ dev_info->max_rx_pktlen = NIC_HW_MAX_MTU + RTE_ETHER_HDR_LEN;
+ dev_info->max_rx_queues =
+ (uint16_t)MAX_RCV_QUEUES_PER_QS * (MAX_SQS_PER_VF + 1);
+ dev_info->max_tx_queues =
+ (uint16_t)MAX_SND_QUEUES_PER_QS * (MAX_SQS_PER_VF + 1);
+ dev_info->max_mac_addrs = 1;
+ dev_info->max_vfs = pci_dev->max_vfs;
+
+ dev_info->rx_offload_capa = NICVF_RX_OFFLOAD_CAPA;
+ dev_info->tx_offload_capa = NICVF_TX_OFFLOAD_CAPA;
+ dev_info->rx_queue_offload_capa = NICVF_RX_OFFLOAD_CAPA;
+ dev_info->tx_queue_offload_capa = NICVF_TX_OFFLOAD_CAPA;
+
+ dev_info->reta_size = nic->rss_info.rss_size;
+ dev_info->hash_key_size = RSS_HASH_KEY_BYTE_SIZE;
+ dev_info->flow_type_rss_offloads = NICVF_RSS_OFFLOAD_PASS1;
+ if (nicvf_hw_cap(nic) & NICVF_CAP_TUNNEL_PARSING)
+ dev_info->flow_type_rss_offloads |= NICVF_RSS_OFFLOAD_TUNNEL;
+
+ dev_info->default_rxconf = (struct rte_eth_rxconf) {
+ .rx_free_thresh = NICVF_DEFAULT_RX_FREE_THRESH,
+ .rx_drop_en = 0,
+ };
+
+ dev_info->default_txconf = (struct rte_eth_txconf) {
+ .tx_free_thresh = NICVF_DEFAULT_TX_FREE_THRESH,
+ .offloads = DEV_TX_OFFLOAD_MBUF_FAST_FREE |
+ DEV_TX_OFFLOAD_OUTER_IPV4_CKSUM |
+ DEV_TX_OFFLOAD_UDP_CKSUM |
+ DEV_TX_OFFLOAD_TCP_CKSUM,
+ };
+
+ return 0;
+}
+
+static nicvf_iova_addr_t
+rbdr_rte_mempool_get(void *dev, void *opaque)
+{
+ uint16_t qidx;
+ uintptr_t mbuf;
+ struct nicvf_rxq *rxq;
+ struct rte_eth_dev *eth_dev = (struct rte_eth_dev *)dev;
+ struct nicvf *nic = (struct nicvf *)opaque;
+ uint16_t rx_start, rx_end;
+
+ /* Get queue ranges for this VF */
+ nicvf_rx_range(eth_dev, nic, &rx_start, &rx_end);
+
+ for (qidx = rx_start; qidx <= rx_end; qidx++) {
+ rxq = eth_dev->data->rx_queues[qidx];
+ /* Maintain equal buffer count across all pools */
+ if (rxq->precharge_cnt >= rxq->qlen_mask)
+ continue;
+ rxq->precharge_cnt++;
+ mbuf = (uintptr_t)rte_pktmbuf_alloc(rxq->pool);
+ if (mbuf)
+ return nicvf_mbuff_virt2phy(mbuf, rxq->mbuf_phys_off);
+ }
+ return 0;
+}
+
+static int
+nicvf_vf_start(struct rte_eth_dev *dev, struct nicvf *nic, uint32_t rbdrsz)
+{
+ int ret;
+ uint16_t qidx, data_off;
+ uint32_t total_rxq_desc, nb_rbdr_desc, exp_buffs;
+ uint64_t mbuf_phys_off = 0;
+ struct nicvf_rxq *rxq;
+ struct rte_mbuf *mbuf;
+ uint16_t rx_start, rx_end;
+ uint16_t tx_start, tx_end;
+ int mask;
+
+ PMD_INIT_FUNC_TRACE();
+
+ /* Userspace process exited without proper shutdown in last run */
+ if (nicvf_qset_rbdr_active(nic, 0))
+ nicvf_vf_stop(dev, nic, false);
+
+ /* Get queue ranges for this VF */
+ nicvf_rx_range(dev, nic, &rx_start, &rx_end);
+
+ /*
+ * Thunderx nicvf PMD can support more than one pool per port only when
+ * 1) Data payload size is same across all the pools in given port
+ * AND
+ * 2) All mbuffs in the pools are from the same hugepage
+ * AND
+ * 3) Mbuff metadata size is same across all the pools in given port
+ *
+ * This is to support existing application that uses multiple pool/port.
+ * But, the purpose of using multipool for QoS will not be addressed.
+ *
+ */
+
+ /* Validate mempool attributes */
+ for (qidx = rx_start; qidx <= rx_end; qidx++) {
+ rxq = dev->data->rx_queues[qidx];
+ rxq->mbuf_phys_off = nicvf_mempool_phy_offset(rxq->pool);
+ mbuf = rte_pktmbuf_alloc(rxq->pool);
+ if (mbuf == NULL) {
+ PMD_INIT_LOG(ERR, "Failed allocate mbuf VF%d qid=%d "
+ "pool=%s",
+ nic->vf_id, qidx, rxq->pool->name);
+ return -ENOMEM;
+ }
+ data_off = nicvf_mbuff_meta_length(mbuf);
+ data_off += RTE_PKTMBUF_HEADROOM;
+ rte_pktmbuf_free(mbuf);
+
+ if (data_off % RTE_CACHE_LINE_SIZE) {
+ PMD_INIT_LOG(ERR, "%s: unaligned data_off=%d delta=%d",
+ rxq->pool->name, data_off,
+ data_off % RTE_CACHE_LINE_SIZE);
+ return -EINVAL;
+ }
+ rxq->mbuf_phys_off -= data_off;
+ rxq->mbuf_phys_off -= nic->skip_bytes;
+
+ if (mbuf_phys_off == 0)
+ mbuf_phys_off = rxq->mbuf_phys_off;
+ if (mbuf_phys_off != rxq->mbuf_phys_off) {
+ PMD_INIT_LOG(ERR, "pool params not same,%s VF%d %"
+ PRIx64, rxq->pool->name, nic->vf_id,
+ mbuf_phys_off);
+ return -EINVAL;
+ }
+ }
+
+ /* Check the level of buffers in the pool */
+ total_rxq_desc = 0;
+ for (qidx = rx_start; qidx <= rx_end; qidx++) {
+ rxq = dev->data->rx_queues[qidx];
+ /* Count total numbers of rxq descs */
+ total_rxq_desc += rxq->qlen_mask + 1;
+ exp_buffs = RTE_MEMPOOL_CACHE_MAX_SIZE + rxq->rx_free_thresh;
+ exp_buffs *= dev->data->nb_rx_queues;
+ if (rte_mempool_avail_count(rxq->pool) < exp_buffs) {
+ PMD_INIT_LOG(ERR, "Buff shortage in pool=%s (%d/%d)",
+ rxq->pool->name,
+ rte_mempool_avail_count(rxq->pool),
+ exp_buffs);
+ return -ENOENT;
+ }
+ }
+
+ /* Check RBDR desc overflow */
+ ret = nicvf_qsize_rbdr_roundup(total_rxq_desc);
+ if (ret == 0) {
+ PMD_INIT_LOG(ERR, "Reached RBDR desc limit, reduce nr desc "
+ "VF%d", nic->vf_id);
+ return -ENOMEM;
+ }
+
+ /* Enable qset */
+ ret = nicvf_qset_config(nic);
+ if (ret) {
+ PMD_INIT_LOG(ERR, "Failed to enable qset %d VF%d", ret,
+ nic->vf_id);
+ return ret;
+ }
+
+ /* Allocate RBDR and RBDR ring desc */
+ nb_rbdr_desc = nicvf_qsize_rbdr_roundup(total_rxq_desc);
+ ret = nicvf_qset_rbdr_alloc(dev, nic, nb_rbdr_desc, rbdrsz);
+ if (ret) {
+ PMD_INIT_LOG(ERR, "Failed to allocate memory for rbdr alloc "
+ "VF%d", nic->vf_id);
+ goto qset_reclaim;
+ }
+
+ /* Enable and configure RBDR registers */
+ ret = nicvf_qset_rbdr_config(nic, 0);
+ if (ret) {
+ PMD_INIT_LOG(ERR, "Failed to configure rbdr %d VF%d", ret,
+ nic->vf_id);
+ goto qset_rbdr_free;
+ }
+
+ /* Fill rte_mempool buffers in RBDR pool and precharge it */
+ ret = nicvf_qset_rbdr_precharge(dev, nic, 0, rbdr_rte_mempool_get,
+ total_rxq_desc);
+ if (ret) {
+ PMD_INIT_LOG(ERR, "Failed to fill rbdr %d VF%d", ret,
+ nic->vf_id);
+ goto qset_rbdr_reclaim;
+ }
+
+ PMD_DRV_LOG(INFO, "Filled %d out of %d entries in RBDR VF%d",
+ nic->rbdr->tail, nb_rbdr_desc, nic->vf_id);
+
+ /* Configure VLAN Strip */
+ mask = ETH_VLAN_STRIP_MASK | ETH_VLAN_FILTER_MASK |
+ ETH_VLAN_EXTEND_MASK;
+ ret = nicvf_vlan_offload_config(dev, mask);
+
+ /* Based on the packet type(IPv4 or IPv6), the nicvf HW aligns L3 data
+ * to the 64bit memory address.
+ * The alignment creates a hole in mbuf(between the end of headroom and
+ * packet data start). The new revision of the HW provides an option to
+ * disable the L3 alignment feature and make mbuf layout looks
+ * more like other NICs. For better application compatibility, disabling
+ * l3 alignment feature on the hardware revisions it supports
+ */
+ nicvf_apad_config(nic, false);
+
+ /* Get queue ranges for this VF */
+ nicvf_tx_range(dev, nic, &tx_start, &tx_end);
+
+ /* Configure TX queues */
+ for (qidx = tx_start; qidx <= tx_end; qidx++) {
+ ret = nicvf_vf_start_tx_queue(dev, nic,
+ qidx % MAX_SND_QUEUES_PER_QS);
+ if (ret)
+ goto start_txq_error;
+ }
+
+ /* Configure RX queues */
+ for (qidx = rx_start; qidx <= rx_end; qidx++) {
+ ret = nicvf_vf_start_rx_queue(dev, nic,
+ qidx % MAX_RCV_QUEUES_PER_QS);
+ if (ret)
+ goto start_rxq_error;
+ }
+
+ if (!nic->sqs_mode) {
+ /* Configure CPI algorithm */
+ ret = nicvf_configure_cpi(dev);
+ if (ret)
+ goto start_txq_error;
+
+ ret = nicvf_mbox_get_rss_size(nic);
+ if (ret) {
+ PMD_INIT_LOG(ERR, "Failed to get rss table size");
+ goto qset_rss_error;
+ }
+
+ /* Configure RSS */
+ ret = nicvf_configure_rss(dev);
+ if (ret)
+ goto qset_rss_error;
+ }
+
+ /* Done; Let PF make the BGX's RX and TX switches to ON position */
+ nicvf_mbox_cfg_done(nic);
+ return 0;
+
+qset_rss_error:
+ nicvf_rss_term(nic);
+start_rxq_error:
+ for (qidx = rx_start; qidx <= rx_end; qidx++)
+ nicvf_vf_stop_rx_queue(dev, nic, qidx % MAX_RCV_QUEUES_PER_QS);
+start_txq_error:
+ for (qidx = tx_start; qidx <= tx_end; qidx++)
+ nicvf_vf_stop_tx_queue(dev, nic, qidx % MAX_SND_QUEUES_PER_QS);
+qset_rbdr_reclaim:
+ nicvf_qset_rbdr_reclaim(nic, 0);
+ nicvf_rbdr_release_mbufs(dev, nic);
+qset_rbdr_free:
+ if (nic->rbdr) {
+ rte_free(nic->rbdr);
+ nic->rbdr = NULL;
+ }
+qset_reclaim:
+ nicvf_qset_reclaim(nic);
+ return ret;
+}
+
+static int
+nicvf_dev_start(struct rte_eth_dev *dev)
+{
+ uint16_t qidx;
+ int ret;
+ size_t i;
+ struct nicvf *nic = nicvf_pmd_priv(dev);
+ struct rte_eth_rxmode *rx_conf = &dev->data->dev_conf.rxmode;
+ uint16_t mtu;
+ uint32_t buffsz = 0, rbdrsz = 0;
+ struct rte_pktmbuf_pool_private *mbp_priv;
+ struct nicvf_rxq *rxq;
+
+ PMD_INIT_FUNC_TRACE();
+
+ /* This function must be called for a primary device */
+ assert_primary(nic);
+
+ /* Validate RBDR buff size */
+ for (qidx = 0; qidx < dev->data->nb_rx_queues; qidx++) {
+ rxq = dev->data->rx_queues[qidx];
+ mbp_priv = rte_mempool_get_priv(rxq->pool);
+ buffsz = mbp_priv->mbuf_data_room_size - RTE_PKTMBUF_HEADROOM;
+ if (buffsz % 128) {
+ PMD_INIT_LOG(ERR, "rxbuf size must be multiply of 128");
+ return -EINVAL;
+ }
+ if (rbdrsz == 0)
+ rbdrsz = buffsz;
+ if (rbdrsz != buffsz) {
+ PMD_INIT_LOG(ERR, "buffsz not same, qidx=%d (%d/%d)",
+ qidx, rbdrsz, buffsz);
+ return -EINVAL;
+ }