X-Git-Url: http://git.droids-corp.org/?a=blobdiff_plain;f=drivers%2Fnet%2Fthunderx%2Fnicvf_ethdev.c;h=eb2c11da53046cb1bc2af2b07aabfc83c7cf6d7c;hb=35b2d13fd6fdcbd191f2a30d74648faeb1186c65;hp=08f65b3364ddad8431fe671d4b161fcaf8d83618;hpb=43362c6a76476f6807390ac51a6d87e6dedada6e;p=dpdk.git diff --git a/drivers/net/thunderx/nicvf_ethdev.c b/drivers/net/thunderx/nicvf_ethdev.c index 08f65b3364..eb2c11da53 100644 --- a/drivers/net/thunderx/nicvf_ethdev.c +++ b/drivers/net/thunderx/nicvf_ethdev.c @@ -1,33 +1,5 @@ -/* - * BSD LICENSE - * - * Copyright (C) Cavium networks Ltd. 2016. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * - * * Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in - * the documentation and/or other materials provided with the - * distribution. - * * Neither the name of Cavium networks nor the names of its - * contributors may be used to endorse or promote products derived - * from this software without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright(c) 2016 Cavium, Inc */ #include @@ -41,10 +13,8 @@ #include #include #include -#include #include -#include #include #include #include @@ -53,7 +23,8 @@ #include #include #include -#include +#include +#include #include #include #include @@ -61,92 +32,181 @@ #include #include #include +#include #include +#include +#include #include "base/nicvf_plat.h" #include "nicvf_ethdev.h" - +#include "nicvf_rxtx.h" +#include "nicvf_svf.h" #include "nicvf_logs.h" -static inline int -nicvf_atomic_write_link_status(struct rte_eth_dev *dev, - struct rte_eth_link *link) +int nicvf_logtype_mbox; +int nicvf_logtype_init; +int nicvf_logtype_driver; + +static void nicvf_dev_stop(struct rte_eth_dev *dev); +static void nicvf_dev_stop_cleanup(struct rte_eth_dev *dev, bool cleanup); +static void nicvf_vf_stop(struct rte_eth_dev *dev, struct nicvf *nic, + bool cleanup); +static int nicvf_vlan_offload_config(struct rte_eth_dev *dev, int mask); +static int nicvf_vlan_offload_set(struct rte_eth_dev *dev, int mask); + +RTE_INIT(nicvf_init_log) { - struct rte_eth_link *dst = &dev->data->dev_link; - struct rte_eth_link *src = link; + nicvf_logtype_mbox = rte_log_register("pmd.net.thunderx.mbox"); + if (nicvf_logtype_mbox >= 0) + rte_log_set_level(nicvf_logtype_mbox, RTE_LOG_NOTICE); - if (rte_atomic64_cmpset((uint64_t *)dst, *(uint64_t *)dst, - *(uint64_t *)src) == 0) - return -1; + nicvf_logtype_init = rte_log_register("pmd.net.thunderx.init"); + if (nicvf_logtype_init >= 0) + rte_log_set_level(nicvf_logtype_init, RTE_LOG_NOTICE); - return 0; + nicvf_logtype_driver = rte_log_register("pmd.net.thunderx.driver"); + if (nicvf_logtype_driver >= 0) + rte_log_set_level(nicvf_logtype_driver, RTE_LOG_NOTICE); } -static inline void -nicvf_set_eth_link_status(struct nicvf *nic, struct rte_eth_link *link) +static void +nicvf_link_status_update(struct nicvf *nic, + struct rte_eth_link *link) { - link->link_status = nic->link_up; - link->link_duplex = ETH_LINK_AUTONEG; + memset(link, 0, sizeof(*link)); + + link->link_status = nic->link_up ? ETH_LINK_UP : ETH_LINK_DOWN; + if (nic->duplex == NICVF_HALF_DUPLEX) link->link_duplex = ETH_LINK_HALF_DUPLEX; else if (nic->duplex == NICVF_FULL_DUPLEX) link->link_duplex = ETH_LINK_FULL_DUPLEX; link->link_speed = nic->speed; - link->link_autoneg = ETH_LINK_SPEED_AUTONEG; + link->link_autoneg = ETH_LINK_AUTONEG; } static void nicvf_interrupt(void *arg) { - struct nicvf *nic = arg; + struct rte_eth_dev *dev = arg; + struct nicvf *nic = nicvf_pmd_priv(dev); + struct rte_eth_link link; if (nicvf_reg_poll_interrupts(nic) == NIC_MBOX_MSG_BGX_LINK_CHANGE) { - if (nic->eth_dev->data->dev_conf.intr_conf.lsc) - nicvf_set_eth_link_status(nic, - &nic->eth_dev->data->dev_link); - _rte_eth_dev_callback_process(nic->eth_dev, - RTE_ETH_EVENT_INTR_LSC); + if (dev->data->dev_conf.intr_conf.lsc) { + nicvf_link_status_update(nic, &link); + rte_eth_linkstatus_set(dev, &link); + + _rte_eth_dev_callback_process(dev, + RTE_ETH_EVENT_INTR_LSC, + NULL); + } } rte_eal_alarm_set(NICVF_INTR_POLL_INTERVAL_MS * 1000, - nicvf_interrupt, nic); + nicvf_interrupt, dev); +} + +static void +nicvf_vf_interrupt(void *arg) +{ + struct nicvf *nic = arg; + + nicvf_reg_poll_interrupts(nic); + + rte_eal_alarm_set(NICVF_INTR_POLL_INTERVAL_MS * 1000, + nicvf_vf_interrupt, nic); } static int -nicvf_periodic_alarm_start(struct nicvf *nic) +nicvf_periodic_alarm_start(void (fn)(void *), void *arg) { - return rte_eal_alarm_set(NICVF_INTR_POLL_INTERVAL_MS * 1000, - nicvf_interrupt, nic); + return rte_eal_alarm_set(NICVF_INTR_POLL_INTERVAL_MS * 1000, fn, arg); } static int -nicvf_periodic_alarm_stop(struct nicvf *nic) +nicvf_periodic_alarm_stop(void (fn)(void *), void *arg) { - return rte_eal_alarm_cancel(nicvf_interrupt, nic); + return rte_eal_alarm_cancel(fn, arg); } /* * Return 0 means link status changed, -1 means not changed */ static int -nicvf_dev_link_update(struct rte_eth_dev *dev, - int wait_to_complete __rte_unused) +nicvf_dev_link_update(struct rte_eth_dev *dev, int wait_to_complete) { +#define CHECK_INTERVAL 100 /* 100ms */ +#define MAX_CHECK_TIME 90 /* 9s (90 * 100ms) in total */ struct rte_eth_link link; struct nicvf *nic = nicvf_pmd_priv(dev); + int i; PMD_INIT_FUNC_TRACE(); - memset(&link, 0, sizeof(link)); - nicvf_set_eth_link_status(nic, &link); - return nicvf_atomic_write_link_status(dev, &link); + if (wait_to_complete) { + /* rte_eth_link_get() might need to wait up to 9 seconds */ + for (i = 0; i < MAX_CHECK_TIME; i++) { + nicvf_link_status_update(nic, &link); + if (link.link_status == ETH_LINK_UP) + break; + rte_delay_ms(CHECK_INTERVAL); + } + } else { + nicvf_link_status_update(nic, &link); + } + + return rte_eth_linkstatus_set(dev, &link); } static int -nicvf_dev_get_reg_length(struct rte_eth_dev *dev __rte_unused) +nicvf_dev_set_mtu(struct rte_eth_dev *dev, uint16_t mtu) { - return nicvf_reg_get_count(); + struct nicvf *nic = nicvf_pmd_priv(dev); + uint32_t buffsz, frame_size = mtu + NIC_HW_L2_OVERHEAD; + size_t i; + struct rte_eth_rxmode *rxmode = &dev->data->dev_conf.rxmode; + + PMD_INIT_FUNC_TRACE(); + + if (frame_size > NIC_HW_MAX_FRS) + return -EINVAL; + + if (frame_size < NIC_HW_MIN_FRS) + return -EINVAL; + + buffsz = dev->data->min_rx_buf_size - RTE_PKTMBUF_HEADROOM; + + /* + * Refuse mtu that requires the support of scattered packets + * when this feature has not been enabled before. + */ + if (dev->data->dev_started && !dev->data->scattered_rx && + (frame_size + 2 * VLAN_TAG_SIZE > buffsz)) + return -EINVAL; + + /* check * >= max_frame */ + if (dev->data->scattered_rx && + (frame_size + 2 * VLAN_TAG_SIZE > buffsz * NIC_HW_MAX_SEGS)) + return -EINVAL; + + if (frame_size > RTE_ETHER_MAX_LEN) + rxmode->offloads |= DEV_RX_OFFLOAD_JUMBO_FRAME; + else + rxmode->offloads &= ~DEV_RX_OFFLOAD_JUMBO_FRAME; + + if (nicvf_mbox_update_hw_max_frs(nic, mtu)) + return -EINVAL; + + /* Update max_rx_pkt_len */ + rxmode->max_rx_pkt_len = mtu + RTE_ETHER_HDR_LEN; + nic->mtu = mtu; + + for (i = 0; i < nic->sqs_count; i++) + nic->snicvf[i]->mtu = mtu; + + return 0; } static int @@ -155,8 +215,11 @@ nicvf_dev_get_regs(struct rte_eth_dev *dev, struct rte_dev_reg_info *regs) uint64_t *data = regs->data; struct nicvf *nic = nicvf_pmd_priv(dev); - if (data == NULL) - return -EINVAL; + if (data == NULL) { + regs->length = nicvf_reg_get_count(); + regs->width = THUNDERX_REG_BYTES; + return 0; + } /* Support only full register dump */ if ((regs->length == 0) || @@ -168,6 +231,181 @@ nicvf_dev_get_regs(struct rte_eth_dev *dev, struct rte_dev_reg_info *regs) return -ENOTSUP; } +static int +nicvf_dev_stats_get(struct rte_eth_dev *dev, struct rte_eth_stats *stats) +{ + uint16_t qidx; + struct nicvf_hw_rx_qstats rx_qstats; + struct nicvf_hw_tx_qstats tx_qstats; + struct nicvf_hw_stats port_stats; + struct nicvf *nic = nicvf_pmd_priv(dev); + uint16_t rx_start, rx_end; + uint16_t tx_start, tx_end; + size_t i; + + /* RX queue indices for the first VF */ + nicvf_rx_range(dev, nic, &rx_start, &rx_end); + + /* Reading per RX ring stats */ + for (qidx = rx_start; qidx <= rx_end; qidx++) { + if (qidx >= RTE_ETHDEV_QUEUE_STAT_CNTRS) + break; + + nicvf_hw_get_rx_qstats(nic, &rx_qstats, qidx); + stats->q_ibytes[qidx] = rx_qstats.q_rx_bytes; + stats->q_ipackets[qidx] = rx_qstats.q_rx_packets; + } + + /* TX queue indices for the first VF */ + nicvf_tx_range(dev, nic, &tx_start, &tx_end); + + /* Reading per TX ring stats */ + for (qidx = tx_start; qidx <= tx_end; qidx++) { + if (qidx >= RTE_ETHDEV_QUEUE_STAT_CNTRS) + break; + + nicvf_hw_get_tx_qstats(nic, &tx_qstats, qidx); + stats->q_obytes[qidx] = tx_qstats.q_tx_bytes; + stats->q_opackets[qidx] = tx_qstats.q_tx_packets; + } + + for (i = 0; i < nic->sqs_count; i++) { + struct nicvf *snic = nic->snicvf[i]; + + if (snic == NULL) + break; + + /* RX queue indices for a secondary VF */ + nicvf_rx_range(dev, snic, &rx_start, &rx_end); + + /* Reading per RX ring stats */ + for (qidx = rx_start; qidx <= rx_end; qidx++) { + if (qidx >= RTE_ETHDEV_QUEUE_STAT_CNTRS) + break; + + nicvf_hw_get_rx_qstats(snic, &rx_qstats, + qidx % MAX_RCV_QUEUES_PER_QS); + stats->q_ibytes[qidx] = rx_qstats.q_rx_bytes; + stats->q_ipackets[qidx] = rx_qstats.q_rx_packets; + } + + /* TX queue indices for a secondary VF */ + nicvf_tx_range(dev, snic, &tx_start, &tx_end); + /* Reading per TX ring stats */ + for (qidx = tx_start; qidx <= tx_end; qidx++) { + if (qidx >= RTE_ETHDEV_QUEUE_STAT_CNTRS) + break; + + nicvf_hw_get_tx_qstats(snic, &tx_qstats, + qidx % MAX_SND_QUEUES_PER_QS); + stats->q_obytes[qidx] = tx_qstats.q_tx_bytes; + stats->q_opackets[qidx] = tx_qstats.q_tx_packets; + } + } + + nicvf_hw_get_stats(nic, &port_stats); + stats->ibytes = port_stats.rx_bytes; + stats->ipackets = port_stats.rx_ucast_frames; + stats->ipackets += port_stats.rx_bcast_frames; + stats->ipackets += port_stats.rx_mcast_frames; + stats->ierrors = port_stats.rx_l2_errors; + stats->imissed = port_stats.rx_drop_red; + stats->imissed += port_stats.rx_drop_overrun; + stats->imissed += port_stats.rx_drop_bcast; + stats->imissed += port_stats.rx_drop_mcast; + stats->imissed += port_stats.rx_drop_l3_bcast; + stats->imissed += port_stats.rx_drop_l3_mcast; + + stats->obytes = port_stats.tx_bytes_ok; + stats->opackets = port_stats.tx_ucast_frames_ok; + stats->opackets += port_stats.tx_bcast_frames_ok; + stats->opackets += port_stats.tx_mcast_frames_ok; + stats->oerrors = port_stats.tx_drops; + + return 0; +} + +static const uint32_t * +nicvf_dev_supported_ptypes_get(struct rte_eth_dev *dev) +{ + size_t copied; + static uint32_t ptypes[32]; + struct nicvf *nic = nicvf_pmd_priv(dev); + static const uint32_t ptypes_common[] = { + RTE_PTYPE_L3_IPV4, + RTE_PTYPE_L3_IPV4_EXT, + RTE_PTYPE_L3_IPV6, + RTE_PTYPE_L3_IPV6_EXT, + RTE_PTYPE_L4_TCP, + RTE_PTYPE_L4_UDP, + RTE_PTYPE_L4_FRAG, + }; + static const uint32_t ptypes_tunnel[] = { + RTE_PTYPE_TUNNEL_GRE, + RTE_PTYPE_TUNNEL_GENEVE, + RTE_PTYPE_TUNNEL_VXLAN, + RTE_PTYPE_TUNNEL_NVGRE, + }; + static const uint32_t ptypes_end = RTE_PTYPE_UNKNOWN; + + copied = sizeof(ptypes_common); + memcpy(ptypes, ptypes_common, copied); + if (nicvf_hw_cap(nic) & NICVF_CAP_TUNNEL_PARSING) { + memcpy((char *)ptypes + copied, ptypes_tunnel, + sizeof(ptypes_tunnel)); + copied += sizeof(ptypes_tunnel); + } + + memcpy((char *)ptypes + copied, &ptypes_end, sizeof(ptypes_end)); + + /* All Ptypes are supported in all Rx functions. */ + return ptypes; +} + +static void +nicvf_dev_stats_reset(struct rte_eth_dev *dev) +{ + int i; + uint16_t rxqs = 0, txqs = 0; + struct nicvf *nic = nicvf_pmd_priv(dev); + uint16_t rx_start, rx_end; + uint16_t tx_start, tx_end; + + /* Reset all primary nic counters */ + nicvf_rx_range(dev, nic, &rx_start, &rx_end); + for (i = rx_start; i <= rx_end; i++) + rxqs |= (0x3 << (i * 2)); + + nicvf_tx_range(dev, nic, &tx_start, &tx_end); + for (i = tx_start; i <= tx_end; i++) + txqs |= (0x3 << (i * 2)); + + nicvf_mbox_reset_stat_counters(nic, 0x3FFF, 0x1F, rxqs, txqs); + + /* Reset secondary nic queue counters */ + for (i = 0; i < nic->sqs_count; i++) { + struct nicvf *snic = nic->snicvf[i]; + if (snic == NULL) + break; + + nicvf_rx_range(dev, snic, &rx_start, &rx_end); + for (i = rx_start; i <= rx_end; i++) + rxqs |= (0x3 << ((i % MAX_CMP_QUEUES_PER_QS) * 2)); + + nicvf_tx_range(dev, snic, &tx_start, &tx_end); + for (i = tx_start; i <= tx_end; i++) + txqs |= (0x3 << ((i % MAX_SND_QUEUES_PER_QS) * 2)); + + nicvf_mbox_reset_stat_counters(snic, 0, 0, rxqs, txqs); + } +} + +/* Promiscuous mode enabled by default in LMAC to VF 1:1 map configuration */ +static void +nicvf_dev_promisc_enable(struct rte_eth_dev *dev __rte_unused) +{ +} + static inline uint64_t nicvf_rss_ethdev_to_nic(struct nicvf *nic, uint64_t ethdev_rss) { @@ -337,14 +575,15 @@ nicvf_dev_rss_hash_update(struct rte_eth_dev *dev, } static int -nicvf_qset_cq_alloc(struct nicvf *nic, struct nicvf_rxq *rxq, uint16_t qidx, - uint32_t desc_cnt) +nicvf_qset_cq_alloc(struct rte_eth_dev *dev, struct nicvf *nic, + struct nicvf_rxq *rxq, uint16_t qidx, uint32_t desc_cnt) { const struct rte_memzone *rz; - uint32_t ring_size = desc_cnt * sizeof(union cq_entry_t); + uint32_t ring_size = CMP_QUEUE_SZ_MAX * sizeof(union cq_entry_t); - rz = rte_eth_dma_zone_reserve(nic->eth_dev, "cq_ring", qidx, ring_size, - NICVF_CQ_BASE_ALIGN_BYTES, nic->node); + rz = rte_eth_dma_zone_reserve(dev, "cq_ring", + nicvf_netdev_qidx(nic, qidx), ring_size, + NICVF_CQ_BASE_ALIGN_BYTES, nic->node); if (rz == NULL) { PMD_INIT_LOG(ERR, "Failed to allocate mem for cq hw ring"); return -ENOMEM; @@ -352,7 +591,7 @@ nicvf_qset_cq_alloc(struct nicvf *nic, struct nicvf_rxq *rxq, uint16_t qidx, memset(rz->addr, 0, ring_size); - rxq->phys = rz->phys_addr; + rxq->phys = rz->iova; rxq->desc = rz->addr; rxq->qlen_mask = desc_cnt - 1; @@ -360,14 +599,15 @@ nicvf_qset_cq_alloc(struct nicvf *nic, struct nicvf_rxq *rxq, uint16_t qidx, } static int -nicvf_qset_sq_alloc(struct nicvf *nic, struct nicvf_txq *sq, uint16_t qidx, - uint32_t desc_cnt) +nicvf_qset_sq_alloc(struct rte_eth_dev *dev, struct nicvf *nic, + struct nicvf_txq *sq, uint16_t qidx, uint32_t desc_cnt) { const struct rte_memzone *rz; - uint32_t ring_size = desc_cnt * sizeof(union sq_entry_t); + uint32_t ring_size = SND_QUEUE_SZ_MAX * sizeof(union sq_entry_t); - rz = rte_eth_dma_zone_reserve(nic->eth_dev, "sq", qidx, ring_size, - NICVF_SQ_BASE_ALIGN_BYTES, nic->node); + rz = rte_eth_dma_zone_reserve(dev, "sq", + nicvf_netdev_qidx(nic, qidx), ring_size, + NICVF_SQ_BASE_ALIGN_BYTES, nic->node); if (rz == NULL) { PMD_INIT_LOG(ERR, "Failed allocate mem for sq hw ring"); return -ENOMEM; @@ -375,13 +615,96 @@ nicvf_qset_sq_alloc(struct nicvf *nic, struct nicvf_txq *sq, uint16_t qidx, memset(rz->addr, 0, ring_size); - sq->phys = rz->phys_addr; + sq->phys = rz->iova; sq->desc = rz->addr; sq->qlen_mask = desc_cnt - 1; return 0; } +static int +nicvf_qset_rbdr_alloc(struct rte_eth_dev *dev, struct nicvf *nic, + uint32_t desc_cnt, uint32_t buffsz) +{ + struct nicvf_rbdr *rbdr; + const struct rte_memzone *rz; + uint32_t ring_size; + + assert(nic->rbdr == NULL); + rbdr = rte_zmalloc_socket("rbdr", sizeof(struct nicvf_rbdr), + RTE_CACHE_LINE_SIZE, nic->node); + if (rbdr == NULL) { + PMD_INIT_LOG(ERR, "Failed to allocate mem for rbdr"); + return -ENOMEM; + } + + ring_size = sizeof(struct rbdr_entry_t) * RBDR_QUEUE_SZ_MAX; + rz = rte_eth_dma_zone_reserve(dev, "rbdr", + nicvf_netdev_qidx(nic, 0), ring_size, + NICVF_RBDR_BASE_ALIGN_BYTES, nic->node); + if (rz == NULL) { + PMD_INIT_LOG(ERR, "Failed to allocate mem for rbdr desc ring"); + return -ENOMEM; + } + + memset(rz->addr, 0, ring_size); + + rbdr->phys = rz->iova; + rbdr->tail = 0; + rbdr->next_tail = 0; + rbdr->desc = rz->addr; + rbdr->buffsz = buffsz; + rbdr->qlen_mask = desc_cnt - 1; + rbdr->rbdr_status = + nicvf_qset_base(nic, 0) + NIC_QSET_RBDR_0_1_STATUS0; + rbdr->rbdr_door = + nicvf_qset_base(nic, 0) + NIC_QSET_RBDR_0_1_DOOR; + + nic->rbdr = rbdr; + return 0; +} + +static void +nicvf_rbdr_release_mbuf(struct rte_eth_dev *dev, struct nicvf *nic, + nicvf_iova_addr_t phy) +{ + uint16_t qidx; + void *obj; + struct nicvf_rxq *rxq; + uint16_t rx_start, rx_end; + + /* Get queue ranges for this VF */ + nicvf_rx_range(dev, nic, &rx_start, &rx_end); + + for (qidx = rx_start; qidx <= rx_end; qidx++) { + rxq = dev->data->rx_queues[qidx]; + if (rxq->precharge_cnt) { + obj = (void *)nicvf_mbuff_phy2virt(phy, + rxq->mbuf_phys_off); + rte_mempool_put(rxq->pool, obj); + rxq->precharge_cnt--; + break; + } + } +} + +static inline void +nicvf_rbdr_release_mbufs(struct rte_eth_dev *dev, struct nicvf *nic) +{ + uint32_t qlen_mask, head; + struct rbdr_entry_t *entry; + struct nicvf_rbdr *rbdr = nic->rbdr; + + qlen_mask = rbdr->qlen_mask; + head = rbdr->head; + while (head != rbdr->tail) { + entry = rbdr->desc + head; + nicvf_rbdr_release_mbuf(dev, nic, entry->full_addr); + head++; + head = head & qlen_mask; + } +} + static inline void nicvf_tx_queue_release_mbufs(struct nicvf_txq *txq) { @@ -410,6 +733,135 @@ nicvf_tx_queue_reset(struct nicvf_txq *txq) txq->xmit_bufs = 0; } +static inline int +nicvf_vf_start_tx_queue(struct rte_eth_dev *dev, struct nicvf *nic, + uint16_t qidx) +{ + struct nicvf_txq *txq; + int ret; + + assert(qidx < MAX_SND_QUEUES_PER_QS); + + if (dev->data->tx_queue_state[nicvf_netdev_qidx(nic, qidx)] == + RTE_ETH_QUEUE_STATE_STARTED) + return 0; + + txq = dev->data->tx_queues[nicvf_netdev_qidx(nic, qidx)]; + txq->pool = NULL; + ret = nicvf_qset_sq_config(nic, qidx, txq); + if (ret) { + PMD_INIT_LOG(ERR, "Failed to configure sq VF%d %d %d", + nic->vf_id, qidx, ret); + goto config_sq_error; + } + + dev->data->tx_queue_state[nicvf_netdev_qidx(nic, qidx)] = + RTE_ETH_QUEUE_STATE_STARTED; + return ret; + +config_sq_error: + nicvf_qset_sq_reclaim(nic, qidx); + return ret; +} + +static inline int +nicvf_vf_stop_tx_queue(struct rte_eth_dev *dev, struct nicvf *nic, + uint16_t qidx) +{ + struct nicvf_txq *txq; + int ret; + + assert(qidx < MAX_SND_QUEUES_PER_QS); + + if (dev->data->tx_queue_state[nicvf_netdev_qidx(nic, qidx)] == + RTE_ETH_QUEUE_STATE_STOPPED) + return 0; + + ret = nicvf_qset_sq_reclaim(nic, qidx); + if (ret) + PMD_INIT_LOG(ERR, "Failed to reclaim sq VF%d %d %d", + nic->vf_id, qidx, ret); + + txq = dev->data->tx_queues[nicvf_netdev_qidx(nic, qidx)]; + nicvf_tx_queue_release_mbufs(txq); + nicvf_tx_queue_reset(txq); + + dev->data->tx_queue_state[nicvf_netdev_qidx(nic, qidx)] = + RTE_ETH_QUEUE_STATE_STOPPED; + return ret; +} + +static inline int +nicvf_configure_cpi(struct rte_eth_dev *dev) +{ + struct nicvf *nic = nicvf_pmd_priv(dev); + uint16_t qidx, qcnt; + int ret; + + /* Count started rx queues */ + for (qidx = qcnt = 0; qidx < dev->data->nb_rx_queues; qidx++) + if (dev->data->rx_queue_state[qidx] == + RTE_ETH_QUEUE_STATE_STARTED) + qcnt++; + + nic->cpi_alg = CPI_ALG_NONE; + ret = nicvf_mbox_config_cpi(nic, qcnt); + if (ret) + PMD_INIT_LOG(ERR, "Failed to configure CPI %d", ret); + + return ret; +} + +static inline int +nicvf_configure_rss(struct rte_eth_dev *dev) +{ + struct nicvf *nic = nicvf_pmd_priv(dev); + uint64_t rsshf; + int ret = -EINVAL; + + rsshf = nicvf_rss_ethdev_to_nic(nic, + dev->data->dev_conf.rx_adv_conf.rss_conf.rss_hf); + PMD_DRV_LOG(INFO, "mode=%d rx_queues=%d loopback=%d rsshf=0x%" PRIx64, + dev->data->dev_conf.rxmode.mq_mode, + dev->data->nb_rx_queues, + dev->data->dev_conf.lpbk_mode, rsshf); + + if (dev->data->dev_conf.rxmode.mq_mode == ETH_MQ_RX_NONE) + ret = nicvf_rss_term(nic); + else if (dev->data->dev_conf.rxmode.mq_mode == ETH_MQ_RX_RSS) + ret = nicvf_rss_config(nic, dev->data->nb_rx_queues, rsshf); + if (ret) + PMD_INIT_LOG(ERR, "Failed to configure RSS %d", ret); + + return ret; +} + +static int +nicvf_configure_rss_reta(struct rte_eth_dev *dev) +{ + struct nicvf *nic = nicvf_pmd_priv(dev); + unsigned int idx, qmap_size; + uint8_t qmap[RTE_MAX_QUEUES_PER_PORT]; + uint8_t default_reta[NIC_MAX_RSS_IDR_TBL_SIZE]; + + if (nic->cpi_alg != CPI_ALG_NONE) + return -EINVAL; + + /* Prepare queue map */ + for (idx = 0, qmap_size = 0; idx < dev->data->nb_rx_queues; idx++) { + if (dev->data->rx_queue_state[idx] == + RTE_ETH_QUEUE_STATE_STARTED) + qmap[qmap_size++] = idx; + } + + /* Update default RSS RETA */ + for (idx = 0; idx < NIC_MAX_RSS_IDR_TBL_SIZE; idx++) + default_reta[idx] = qmap[idx % qmap_size]; + + return nicvf_rss_reta_update(nic, default_reta, + NIC_MAX_RSS_IDR_TBL_SIZE); +} + static void nicvf_dev_tx_queue_release(void *sq) { @@ -428,18 +880,79 @@ nicvf_dev_tx_queue_release(void *sq) } } +static void +nicvf_set_tx_function(struct rte_eth_dev *dev) +{ + struct nicvf_txq *txq = NULL; + size_t i; + bool multiseg = false; + + for (i = 0; i < dev->data->nb_tx_queues; i++) { + txq = dev->data->tx_queues[i]; + if (txq->offloads & DEV_TX_OFFLOAD_MULTI_SEGS) { + multiseg = true; + break; + } + } + + /* Use a simple Tx queue (no offloads, no multi segs) if possible */ + if (multiseg) { + PMD_DRV_LOG(DEBUG, "Using multi-segment tx callback"); + dev->tx_pkt_burst = nicvf_xmit_pkts_multiseg; + } else { + PMD_DRV_LOG(DEBUG, "Using single-segment tx callback"); + dev->tx_pkt_burst = nicvf_xmit_pkts; + } + + if (!txq) + return; + + if (txq->pool_free == nicvf_single_pool_free_xmited_buffers) + PMD_DRV_LOG(DEBUG, "Using single-mempool tx free method"); + else + PMD_DRV_LOG(DEBUG, "Using multi-mempool tx free method"); +} + +static void +nicvf_set_rx_function(struct rte_eth_dev *dev) +{ + struct nicvf *nic = nicvf_pmd_priv(dev); + + const eth_rx_burst_t rx_burst_func[2][2][2] = { + /* [NORMAL/SCATTER] [CKSUM/NO_CKSUM] [VLAN_STRIP/NO_VLAN_STRIP] */ + [0][0][0] = nicvf_recv_pkts_no_offload, + [0][0][1] = nicvf_recv_pkts_vlan_strip, + [0][1][0] = nicvf_recv_pkts_cksum, + [0][1][1] = nicvf_recv_pkts_cksum_vlan_strip, + [1][0][0] = nicvf_recv_pkts_multiseg_no_offload, + [1][0][1] = nicvf_recv_pkts_multiseg_vlan_strip, + [1][1][0] = nicvf_recv_pkts_multiseg_cksum, + [1][1][1] = nicvf_recv_pkts_multiseg_cksum_vlan_strip, + }; + + dev->rx_pkt_burst = + rx_burst_func[dev->data->scattered_rx] + [nic->offload_cksum][nic->vlan_strip]; +} + static int nicvf_dev_tx_queue_setup(struct rte_eth_dev *dev, uint16_t qidx, uint16_t nb_desc, unsigned int socket_id, const struct rte_eth_txconf *tx_conf) { uint16_t tx_free_thresh; - uint8_t is_single_pool; + bool is_single_pool; struct nicvf_txq *txq; struct nicvf *nic = nicvf_pmd_priv(dev); + uint64_t offloads; PMD_INIT_FUNC_TRACE(); + if (qidx >= MAX_SND_QUEUES_PER_QS) + nic = nic->snicvf[qidx / MAX_SND_QUEUES_PER_QS - 1]; + + qidx = qidx % MAX_SND_QUEUES_PER_QS; + /* Socket id check */ if (socket_id != (unsigned int)SOCKET_ID_ANY && socket_id != nic->node) PMD_DRV_LOG(WARNING, "socket_id expected %d, configured %d", @@ -474,29 +987,32 @@ nicvf_dev_tx_queue_setup(struct rte_eth_dev *dev, uint16_t qidx, } /* Free memory prior to re-allocation if needed. */ - if (dev->data->tx_queues[qidx] != NULL) { + if (dev->data->tx_queues[nicvf_netdev_qidx(nic, qidx)] != NULL) { PMD_TX_LOG(DEBUG, "Freeing memory prior to re-allocation %d", - qidx); - nicvf_dev_tx_queue_release(dev->data->tx_queues[qidx]); - dev->data->tx_queues[qidx] = NULL; + nicvf_netdev_qidx(nic, qidx)); + nicvf_dev_tx_queue_release( + dev->data->tx_queues[nicvf_netdev_qidx(nic, qidx)]); + dev->data->tx_queues[nicvf_netdev_qidx(nic, qidx)] = NULL; } /* Allocating tx queue data structure */ txq = rte_zmalloc_socket("ethdev TX queue", sizeof(struct nicvf_txq), RTE_CACHE_LINE_SIZE, nic->node); if (txq == NULL) { - PMD_INIT_LOG(ERR, "Failed to allocate txq=%d", qidx); + PMD_INIT_LOG(ERR, "Failed to allocate txq=%d", + nicvf_netdev_qidx(nic, qidx)); return -ENOMEM; } txq->nic = nic; txq->queue_id = qidx; txq->tx_free_thresh = tx_free_thresh; - txq->txq_flags = tx_conf->txq_flags; txq->sq_head = nicvf_qset_base(nic, qidx) + NIC_QSET_SQ_0_7_HEAD; txq->sq_door = nicvf_qset_base(nic, qidx) + NIC_QSET_SQ_0_7_DOOR; - is_single_pool = (txq->txq_flags & ETH_TXQ_FLAGS_NOREFCOUNT && - txq->txq_flags & ETH_TXQ_FLAGS_NOMULTMEMP); + offloads = tx_conf->offloads | dev->data->dev_conf.txmode.offloads; + txq->offloads = offloads; + + is_single_pool = !!(offloads & DEV_TX_OFFLOAD_MBUF_FAST_FREE); /* Choose optimum free threshold value for multipool case */ if (!is_single_pool) { @@ -504,6 +1020,9 @@ nicvf_dev_tx_queue_setup(struct rte_eth_dev *dev, uint16_t qidx, (tx_conf->tx_free_thresh == NICVF_DEFAULT_TX_FREE_THRESH ? NICVF_TX_FREE_MPOOL_THRESH : tx_conf->tx_free_thresh); + txq->pool_free = nicvf_multi_pool_free_xmited_buffers; + } else { + txq->pool_free = nicvf_single_pool_free_xmited_buffers; } /* Allocate software ring */ @@ -516,7 +1035,7 @@ nicvf_dev_tx_queue_setup(struct rte_eth_dev *dev, uint16_t qidx, return -ENOMEM; } - if (nicvf_qset_sq_alloc(nic, txq, qidx, nb_desc)) { + if (nicvf_qset_sq_alloc(dev, nic, txq, qidx, nb_desc)) { PMD_INIT_LOG(ERR, "Failed to allocate mem for sq %d", qidx); nicvf_dev_tx_queue_release(txq); return -ENOMEM; @@ -524,14 +1043,47 @@ nicvf_dev_tx_queue_setup(struct rte_eth_dev *dev, uint16_t qidx, nicvf_tx_queue_reset(txq); - PMD_TX_LOG(DEBUG, "[%d] txq=%p nb_desc=%d desc=%p phys=0x%" PRIx64, - qidx, txq, nb_desc, txq->desc, txq->phys); + PMD_INIT_LOG(DEBUG, "[%d] txq=%p nb_desc=%d desc=%p" + " phys=0x%" PRIx64 " offloads=0x%" PRIx64, + nicvf_netdev_qidx(nic, qidx), txq, nb_desc, txq->desc, + txq->phys, txq->offloads); - dev->data->tx_queues[qidx] = txq; - dev->data->tx_queue_state[qidx] = RTE_ETH_QUEUE_STATE_STOPPED; + dev->data->tx_queues[nicvf_netdev_qidx(nic, qidx)] = txq; + dev->data->tx_queue_state[nicvf_netdev_qidx(nic, qidx)] = + RTE_ETH_QUEUE_STATE_STOPPED; return 0; } +static inline void +nicvf_rx_queue_release_mbufs(struct rte_eth_dev *dev, struct nicvf_rxq *rxq) +{ + uint32_t rxq_cnt; + uint32_t nb_pkts, released_pkts = 0; + uint32_t refill_cnt = 0; + struct rte_mbuf *rx_pkts[NICVF_MAX_RX_FREE_THRESH]; + + if (dev->rx_pkt_burst == NULL) + return; + + while ((rxq_cnt = nicvf_dev_rx_queue_count(dev, + nicvf_netdev_qidx(rxq->nic, rxq->queue_id)))) { + nb_pkts = dev->rx_pkt_burst(rxq, rx_pkts, + NICVF_MAX_RX_FREE_THRESH); + PMD_DRV_LOG(INFO, "nb_pkts=%d rxq_cnt=%d", nb_pkts, rxq_cnt); + while (nb_pkts) { + rte_pktmbuf_free_seg(rx_pkts[--nb_pkts]); + released_pkts++; + } + } + + + refill_cnt += nicvf_dev_rbdr_refill(dev, + nicvf_netdev_qidx(rxq->nic, rxq->queue_id)); + + PMD_DRV_LOG(INFO, "free_cnt=%d refill_cnt=%d", + released_pkts, refill_cnt); +} + static void nicvf_rx_queue_reset(struct nicvf_rxq *rxq) { @@ -540,37 +1092,223 @@ nicvf_rx_queue_reset(struct nicvf_rxq *rxq) rxq->recv_buffers = 0; } -static void -nicvf_dev_rx_queue_release(void *rx_queue) +static inline int +nicvf_vf_start_rx_queue(struct rte_eth_dev *dev, struct nicvf *nic, + uint16_t qidx) { - struct nicvf_rxq *rxq = rx_queue; + struct nicvf_rxq *rxq; + int ret; - PMD_INIT_FUNC_TRACE(); + assert(qidx < MAX_RCV_QUEUES_PER_QS); + + if (dev->data->rx_queue_state[nicvf_netdev_qidx(nic, qidx)] == + RTE_ETH_QUEUE_STATE_STARTED) + return 0; + + /* Update rbdr pointer to all rxq */ + rxq = dev->data->rx_queues[nicvf_netdev_qidx(nic, qidx)]; + rxq->shared_rbdr = nic->rbdr; + + ret = nicvf_qset_rq_config(nic, qidx, rxq); + if (ret) { + PMD_INIT_LOG(ERR, "Failed to configure rq VF%d %d %d", + nic->vf_id, qidx, ret); + goto config_rq_error; + } + ret = nicvf_qset_cq_config(nic, qidx, rxq); + if (ret) { + PMD_INIT_LOG(ERR, "Failed to configure cq VF%d %d %d", + nic->vf_id, qidx, ret); + goto config_cq_error; + } + + dev->data->rx_queue_state[nicvf_netdev_qidx(nic, qidx)] = + RTE_ETH_QUEUE_STATE_STARTED; + return 0; - if (rxq) - rte_free(rxq); +config_cq_error: + nicvf_qset_cq_reclaim(nic, qidx); +config_rq_error: + nicvf_qset_rq_reclaim(nic, qidx); + return ret; } -static int -nicvf_dev_rx_queue_setup(struct rte_eth_dev *dev, uint16_t qidx, - uint16_t nb_desc, unsigned int socket_id, - const struct rte_eth_rxconf *rx_conf, - struct rte_mempool *mp) +static inline int +nicvf_vf_stop_rx_queue(struct rte_eth_dev *dev, struct nicvf *nic, + uint16_t qidx) { - uint16_t rx_free_thresh; struct nicvf_rxq *rxq; - struct nicvf *nic = nicvf_pmd_priv(dev); + int ret, other_error; - PMD_INIT_FUNC_TRACE(); + if (dev->data->rx_queue_state[nicvf_netdev_qidx(nic, qidx)] == + RTE_ETH_QUEUE_STATE_STOPPED) + return 0; - /* Socket id check */ - if (socket_id != (unsigned int)SOCKET_ID_ANY && socket_id != nic->node) - PMD_DRV_LOG(WARNING, "socket_id expected %d, configured %d", - socket_id, nic->node); + ret = nicvf_qset_rq_reclaim(nic, qidx); + if (ret) + PMD_INIT_LOG(ERR, "Failed to reclaim rq VF%d %d %d", + nic->vf_id, qidx, ret); - /* Mempool memory should be contiguous */ + other_error = ret; + rxq = dev->data->rx_queues[nicvf_netdev_qidx(nic, qidx)]; + nicvf_rx_queue_release_mbufs(dev, rxq); + nicvf_rx_queue_reset(rxq); + + ret = nicvf_qset_cq_reclaim(nic, qidx); + if (ret) + PMD_INIT_LOG(ERR, "Failed to reclaim cq VF%d %d %d", + nic->vf_id, qidx, ret); + + other_error |= ret; + dev->data->rx_queue_state[nicvf_netdev_qidx(nic, qidx)] = + RTE_ETH_QUEUE_STATE_STOPPED; + return other_error; +} + +static void +nicvf_dev_rx_queue_release(void *rx_queue) +{ + PMD_INIT_FUNC_TRACE(); + + rte_free(rx_queue); +} + +static int +nicvf_dev_rx_queue_start(struct rte_eth_dev *dev, uint16_t qidx) +{ + struct nicvf *nic = nicvf_pmd_priv(dev); + int ret; + + if (qidx >= MAX_RCV_QUEUES_PER_QS) + nic = nic->snicvf[(qidx / MAX_RCV_QUEUES_PER_QS - 1)]; + + qidx = qidx % MAX_RCV_QUEUES_PER_QS; + + ret = nicvf_vf_start_rx_queue(dev, nic, qidx); + if (ret) + return ret; + + ret = nicvf_configure_cpi(dev); + if (ret) + return ret; + + return nicvf_configure_rss_reta(dev); +} + +static int +nicvf_dev_rx_queue_stop(struct rte_eth_dev *dev, uint16_t qidx) +{ + int ret; + struct nicvf *nic = nicvf_pmd_priv(dev); + + if (qidx >= MAX_SND_QUEUES_PER_QS) + nic = nic->snicvf[(qidx / MAX_SND_QUEUES_PER_QS - 1)]; + + qidx = qidx % MAX_RCV_QUEUES_PER_QS; + + ret = nicvf_vf_stop_rx_queue(dev, nic, qidx); + ret |= nicvf_configure_cpi(dev); + ret |= nicvf_configure_rss_reta(dev); + return ret; +} + +static int +nicvf_dev_tx_queue_start(struct rte_eth_dev *dev, uint16_t qidx) +{ + struct nicvf *nic = nicvf_pmd_priv(dev); + + if (qidx >= MAX_SND_QUEUES_PER_QS) + nic = nic->snicvf[(qidx / MAX_SND_QUEUES_PER_QS - 1)]; + + qidx = qidx % MAX_SND_QUEUES_PER_QS; + + return nicvf_vf_start_tx_queue(dev, nic, qidx); +} + +static int +nicvf_dev_tx_queue_stop(struct rte_eth_dev *dev, uint16_t qidx) +{ + struct nicvf *nic = nicvf_pmd_priv(dev); + + if (qidx >= MAX_SND_QUEUES_PER_QS) + nic = nic->snicvf[(qidx / MAX_SND_QUEUES_PER_QS - 1)]; + + qidx = qidx % MAX_SND_QUEUES_PER_QS; + + return nicvf_vf_stop_tx_queue(dev, nic, qidx); +} + +static inline void +nicvf_rxq_mbuf_setup(struct nicvf_rxq *rxq) +{ + uintptr_t p; + struct rte_mbuf mb_def; + struct nicvf *nic = rxq->nic; + + RTE_BUILD_BUG_ON(sizeof(union mbuf_initializer) != 8); + RTE_BUILD_BUG_ON(offsetof(struct rte_mbuf, data_off) % 8 != 0); + RTE_BUILD_BUG_ON(offsetof(struct rte_mbuf, refcnt) - + offsetof(struct rte_mbuf, data_off) != 2); + RTE_BUILD_BUG_ON(offsetof(struct rte_mbuf, nb_segs) - + offsetof(struct rte_mbuf, data_off) != 4); + RTE_BUILD_BUG_ON(offsetof(struct rte_mbuf, port) - + offsetof(struct rte_mbuf, data_off) != 6); + RTE_BUILD_BUG_ON(offsetof(struct nicvf_rxq, rxq_fastpath_data_end) - + offsetof(struct nicvf_rxq, + rxq_fastpath_data_start) > 128); + mb_def.nb_segs = 1; + mb_def.data_off = RTE_PKTMBUF_HEADROOM + (nic->skip_bytes); + mb_def.port = rxq->port_id; + rte_mbuf_refcnt_set(&mb_def, 1); + + /* Prevent compiler reordering: rearm_data covers previous fields */ + rte_compiler_barrier(); + p = (uintptr_t)&mb_def.rearm_data; + rxq->mbuf_initializer.value = *(uint64_t *)p; +} + +static int +nicvf_dev_rx_queue_setup(struct rte_eth_dev *dev, uint16_t qidx, + uint16_t nb_desc, unsigned int socket_id, + const struct rte_eth_rxconf *rx_conf, + struct rte_mempool *mp) +{ + uint16_t rx_free_thresh; + struct nicvf_rxq *rxq; + struct nicvf *nic = nicvf_pmd_priv(dev); + uint64_t offloads; + uint32_t buffsz; + struct rte_pktmbuf_pool_private *mbp_priv; + + PMD_INIT_FUNC_TRACE(); + + /* First skip check */ + mbp_priv = rte_mempool_get_priv(mp); + buffsz = mbp_priv->mbuf_data_room_size - RTE_PKTMBUF_HEADROOM; + if (buffsz < (uint32_t)(nic->skip_bytes)) { + PMD_INIT_LOG(ERR, "First skip is more than configured buffer size"); + return -EINVAL; + } + + if (qidx >= MAX_RCV_QUEUES_PER_QS) + nic = nic->snicvf[qidx / MAX_RCV_QUEUES_PER_QS - 1]; + + qidx = qidx % MAX_RCV_QUEUES_PER_QS; + + /* Socket id check */ + if (socket_id != (unsigned int)SOCKET_ID_ANY && socket_id != nic->node) + PMD_DRV_LOG(WARNING, "socket_id expected %d, configured %d", + socket_id, nic->node); + + /* Mempool memory must be contiguous, so must be one memory segment*/ if (mp->nb_mem_chunks != 1) { - PMD_INIT_LOG(ERR, "Non contiguous mempool, check huge page sz"); + PMD_INIT_LOG(ERR, "Non-contiguous mempool, add more huge pages"); + return -EINVAL; + } + + /* Mempool memory must be physically contiguous */ + if (mp->flags & MEMPOOL_F_NO_IOVA_CONTIG) { + PMD_INIT_LOG(ERR, "Mempool memory must be physically contiguous"); return -EINVAL; } @@ -587,6 +1325,7 @@ nicvf_dev_rx_queue_setup(struct rte_eth_dev *dev, uint16_t qidx, return -EINVAL; } + /* Check rx_free_thresh upper bound */ rx_free_thresh = (uint16_t)((rx_conf->rx_free_thresh) ? rx_conf->rx_free_thresh : @@ -599,18 +1338,20 @@ nicvf_dev_rx_queue_setup(struct rte_eth_dev *dev, uint16_t qidx, } /* Free memory prior to re-allocation if needed */ - if (dev->data->rx_queues[qidx] != NULL) { + if (dev->data->rx_queues[nicvf_netdev_qidx(nic, qidx)] != NULL) { PMD_RX_LOG(DEBUG, "Freeing memory prior to re-allocation %d", - qidx); - nicvf_dev_rx_queue_release(dev->data->rx_queues[qidx]); - dev->data->rx_queues[qidx] = NULL; + nicvf_netdev_qidx(nic, qidx)); + nicvf_dev_rx_queue_release( + dev->data->rx_queues[nicvf_netdev_qidx(nic, qidx)]); + dev->data->rx_queues[nicvf_netdev_qidx(nic, qidx)] = NULL; } /* Allocate rxq memory */ rxq = rte_zmalloc_socket("ethdev rx queue", sizeof(struct nicvf_rxq), RTE_CACHE_LINE_SIZE, nic->node); if (rxq == NULL) { - PMD_INIT_LOG(ERR, "Failed to allocate rxq=%d", qidx); + PMD_INIT_LOG(ERR, "Failed to allocate rxq=%d", + nicvf_netdev_qidx(nic, qidx)); return -ENOMEM; } @@ -623,10 +1364,16 @@ nicvf_dev_rx_queue_setup(struct rte_eth_dev *dev, uint16_t qidx, rxq->cq_status = nicvf_qset_base(nic, qidx) + NIC_QSET_CQ_0_7_STATUS; rxq->cq_door = nicvf_qset_base(nic, qidx) + NIC_QSET_CQ_0_7_DOOR; rxq->precharge_cnt = 0; - rxq->rbptr_offset = NICVF_CQE_RBPTR_WORD; + + if (nicvf_hw_cap(nic) & NICVF_CAP_CQE_RX2) + rxq->rbptr_offset = NICVF_CQE_RX2_RBPTR_WORD; + else + rxq->rbptr_offset = NICVF_CQE_RBPTR_WORD; + + nicvf_rxq_mbuf_setup(rxq); /* Alloc completion queue */ - if (nicvf_qset_cq_alloc(nic, rxq, rxq->queue_id, nb_desc)) { + if (nicvf_qset_cq_alloc(dev, nic, rxq, rxq->queue_id, nb_desc)) { PMD_INIT_LOG(ERR, "failed to allocate cq %u", rxq->queue_id); nicvf_dev_rx_queue_release(rxq); return -ENOMEM; @@ -634,12 +1381,15 @@ nicvf_dev_rx_queue_setup(struct rte_eth_dev *dev, uint16_t qidx, nicvf_rx_queue_reset(rxq); - PMD_RX_LOG(DEBUG, "[%d] rxq=%p pool=%s nb_desc=(%d/%d) phy=%" PRIx64, - qidx, rxq, mp->name, nb_desc, - rte_mempool_count(mp), rxq->phys); + offloads = rx_conf->offloads | dev->data->dev_conf.rxmode.offloads; + PMD_INIT_LOG(DEBUG, "[%d] rxq=%p pool=%s nb_desc=(%d/%d)" + " phy=0x%" PRIx64 " offloads=0x%" PRIx64, + nicvf_netdev_qidx(nic, qidx), rxq, mp->name, nb_desc, + rte_mempool_avail_count(mp), rxq->phys, offloads); - dev->data->rx_queues[qidx] = rxq; - dev->data->rx_queue_state[qidx] = RTE_ETH_QUEUE_STATE_STOPPED; + dev->data->rx_queues[nicvf_netdev_qidx(nic, qidx)] = rxq; + dev->data->rx_queue_state[nicvf_netdev_qidx(nic, qidx)] = + RTE_ETH_QUEUE_STATE_STOPPED; return 0; } @@ -647,23 +1397,30 @@ static void nicvf_dev_info_get(struct rte_eth_dev *dev, struct rte_eth_dev_info *dev_info) { struct nicvf *nic = nicvf_pmd_priv(dev); + struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev); PMD_INIT_FUNC_TRACE(); - dev_info->min_rx_bufsize = ETHER_MIN_MTU; - dev_info->max_rx_pktlen = NIC_HW_MAX_FRS; - dev_info->max_rx_queues = (uint16_t)MAX_RCV_QUEUES_PER_QS; - dev_info->max_tx_queues = (uint16_t)MAX_SND_QUEUES_PER_QS; + /* Autonegotiation may be disabled */ + dev_info->speed_capa = ETH_LINK_SPEED_FIXED; + dev_info->speed_capa |= ETH_LINK_SPEED_10M | ETH_LINK_SPEED_100M | + ETH_LINK_SPEED_1G | ETH_LINK_SPEED_10G; + if (nicvf_hw_version(nic) != PCI_SUB_DEVICE_ID_CN81XX_NICVF) + dev_info->speed_capa |= ETH_LINK_SPEED_40G; + + dev_info->min_rx_bufsize = RTE_ETHER_MIN_MTU; + dev_info->max_rx_pktlen = NIC_HW_MAX_MTU + RTE_ETHER_HDR_LEN; + dev_info->max_rx_queues = + (uint16_t)MAX_RCV_QUEUES_PER_QS * (MAX_SQS_PER_VF + 1); + dev_info->max_tx_queues = + (uint16_t)MAX_SND_QUEUES_PER_QS * (MAX_SQS_PER_VF + 1); dev_info->max_mac_addrs = 1; - dev_info->max_vfs = dev->pci_dev->max_vfs; + dev_info->max_vfs = pci_dev->max_vfs; - dev_info->rx_offload_capa = DEV_RX_OFFLOAD_VLAN_STRIP; - dev_info->tx_offload_capa = - DEV_TX_OFFLOAD_IPV4_CKSUM | - DEV_TX_OFFLOAD_UDP_CKSUM | - DEV_TX_OFFLOAD_TCP_CKSUM | - DEV_TX_OFFLOAD_TCP_TSO | - DEV_TX_OFFLOAD_OUTER_IPV4_CKSUM; + dev_info->rx_offload_capa = NICVF_RX_OFFLOAD_CAPA; + dev_info->tx_offload_capa = NICVF_TX_OFFLOAD_CAPA; + dev_info->rx_queue_offload_capa = NICVF_RX_OFFLOAD_CAPA; + dev_info->tx_queue_offload_capa = NICVF_TX_OFFLOAD_CAPA; dev_info->reta_size = nic->rss_info.rss_size; dev_info->hash_key_size = RSS_HASH_KEY_BYTE_SIZE; @@ -678,22 +1435,478 @@ nicvf_dev_info_get(struct rte_eth_dev *dev, struct rte_eth_dev_info *dev_info) dev_info->default_txconf = (struct rte_eth_txconf) { .tx_free_thresh = NICVF_DEFAULT_TX_FREE_THRESH, - .txq_flags = - ETH_TXQ_FLAGS_NOMULTSEGS | - ETH_TXQ_FLAGS_NOREFCOUNT | - ETH_TXQ_FLAGS_NOMULTMEMP | - ETH_TXQ_FLAGS_NOVLANOFFL | - ETH_TXQ_FLAGS_NOXSUMSCTP, + .offloads = DEV_TX_OFFLOAD_MBUF_FAST_FREE | + DEV_TX_OFFLOAD_OUTER_IPV4_CKSUM | + DEV_TX_OFFLOAD_UDP_CKSUM | + DEV_TX_OFFLOAD_TCP_CKSUM, }; } +static nicvf_iova_addr_t +rbdr_rte_mempool_get(void *dev, void *opaque) +{ + uint16_t qidx; + uintptr_t mbuf; + struct nicvf_rxq *rxq; + struct rte_eth_dev *eth_dev = (struct rte_eth_dev *)dev; + struct nicvf *nic = (struct nicvf *)opaque; + uint16_t rx_start, rx_end; + + /* Get queue ranges for this VF */ + nicvf_rx_range(eth_dev, nic, &rx_start, &rx_end); + + for (qidx = rx_start; qidx <= rx_end; qidx++) { + rxq = eth_dev->data->rx_queues[qidx]; + /* Maintain equal buffer count across all pools */ + if (rxq->precharge_cnt >= rxq->qlen_mask) + continue; + rxq->precharge_cnt++; + mbuf = (uintptr_t)rte_pktmbuf_alloc(rxq->pool); + if (mbuf) + return nicvf_mbuff_virt2phy(mbuf, rxq->mbuf_phys_off); + } + return 0; +} + +static int +nicvf_vf_start(struct rte_eth_dev *dev, struct nicvf *nic, uint32_t rbdrsz) +{ + int ret; + uint16_t qidx, data_off; + uint32_t total_rxq_desc, nb_rbdr_desc, exp_buffs; + uint64_t mbuf_phys_off = 0; + struct nicvf_rxq *rxq; + struct rte_mbuf *mbuf; + uint16_t rx_start, rx_end; + uint16_t tx_start, tx_end; + int mask; + + PMD_INIT_FUNC_TRACE(); + + /* Userspace process exited without proper shutdown in last run */ + if (nicvf_qset_rbdr_active(nic, 0)) + nicvf_vf_stop(dev, nic, false); + + /* Get queue ranges for this VF */ + nicvf_rx_range(dev, nic, &rx_start, &rx_end); + + /* + * Thunderx nicvf PMD can support more than one pool per port only when + * 1) Data payload size is same across all the pools in given port + * AND + * 2) All mbuffs in the pools are from the same hugepage + * AND + * 3) Mbuff metadata size is same across all the pools in given port + * + * This is to support existing application that uses multiple pool/port. + * But, the purpose of using multipool for QoS will not be addressed. + * + */ + + /* Validate mempool attributes */ + for (qidx = rx_start; qidx <= rx_end; qidx++) { + rxq = dev->data->rx_queues[qidx]; + rxq->mbuf_phys_off = nicvf_mempool_phy_offset(rxq->pool); + mbuf = rte_pktmbuf_alloc(rxq->pool); + if (mbuf == NULL) { + PMD_INIT_LOG(ERR, "Failed allocate mbuf VF%d qid=%d " + "pool=%s", + nic->vf_id, qidx, rxq->pool->name); + return -ENOMEM; + } + data_off = nicvf_mbuff_meta_length(mbuf); + data_off += RTE_PKTMBUF_HEADROOM; + rte_pktmbuf_free(mbuf); + + if (data_off % RTE_CACHE_LINE_SIZE) { + PMD_INIT_LOG(ERR, "%s: unaligned data_off=%d delta=%d", + rxq->pool->name, data_off, + data_off % RTE_CACHE_LINE_SIZE); + return -EINVAL; + } + rxq->mbuf_phys_off -= data_off; + rxq->mbuf_phys_off -= nic->skip_bytes; + + if (mbuf_phys_off == 0) + mbuf_phys_off = rxq->mbuf_phys_off; + if (mbuf_phys_off != rxq->mbuf_phys_off) { + PMD_INIT_LOG(ERR, "pool params not same,%s VF%d %" + PRIx64, rxq->pool->name, nic->vf_id, + mbuf_phys_off); + return -EINVAL; + } + } + + /* Check the level of buffers in the pool */ + total_rxq_desc = 0; + for (qidx = rx_start; qidx <= rx_end; qidx++) { + rxq = dev->data->rx_queues[qidx]; + /* Count total numbers of rxq descs */ + total_rxq_desc += rxq->qlen_mask + 1; + exp_buffs = RTE_MEMPOOL_CACHE_MAX_SIZE + rxq->rx_free_thresh; + exp_buffs *= dev->data->nb_rx_queues; + if (rte_mempool_avail_count(rxq->pool) < exp_buffs) { + PMD_INIT_LOG(ERR, "Buff shortage in pool=%s (%d/%d)", + rxq->pool->name, + rte_mempool_avail_count(rxq->pool), + exp_buffs); + return -ENOENT; + } + } + + /* Check RBDR desc overflow */ + ret = nicvf_qsize_rbdr_roundup(total_rxq_desc); + if (ret == 0) { + PMD_INIT_LOG(ERR, "Reached RBDR desc limit, reduce nr desc " + "VF%d", nic->vf_id); + return -ENOMEM; + } + + /* Enable qset */ + ret = nicvf_qset_config(nic); + if (ret) { + PMD_INIT_LOG(ERR, "Failed to enable qset %d VF%d", ret, + nic->vf_id); + return ret; + } + + /* Allocate RBDR and RBDR ring desc */ + nb_rbdr_desc = nicvf_qsize_rbdr_roundup(total_rxq_desc); + ret = nicvf_qset_rbdr_alloc(dev, nic, nb_rbdr_desc, rbdrsz); + if (ret) { + PMD_INIT_LOG(ERR, "Failed to allocate memory for rbdr alloc " + "VF%d", nic->vf_id); + goto qset_reclaim; + } + + /* Enable and configure RBDR registers */ + ret = nicvf_qset_rbdr_config(nic, 0); + if (ret) { + PMD_INIT_LOG(ERR, "Failed to configure rbdr %d VF%d", ret, + nic->vf_id); + goto qset_rbdr_free; + } + + /* Fill rte_mempool buffers in RBDR pool and precharge it */ + ret = nicvf_qset_rbdr_precharge(dev, nic, 0, rbdr_rte_mempool_get, + total_rxq_desc); + if (ret) { + PMD_INIT_LOG(ERR, "Failed to fill rbdr %d VF%d", ret, + nic->vf_id); + goto qset_rbdr_reclaim; + } + + PMD_DRV_LOG(INFO, "Filled %d out of %d entries in RBDR VF%d", + nic->rbdr->tail, nb_rbdr_desc, nic->vf_id); + + /* Configure VLAN Strip */ + mask = ETH_VLAN_STRIP_MASK | ETH_VLAN_FILTER_MASK | + ETH_VLAN_EXTEND_MASK; + ret = nicvf_vlan_offload_config(dev, mask); + + /* Based on the packet type(IPv4 or IPv6), the nicvf HW aligns L3 data + * to the 64bit memory address. + * The alignment creates a hole in mbuf(between the end of headroom and + * packet data start). The new revision of the HW provides an option to + * disable the L3 alignment feature and make mbuf layout looks + * more like other NICs. For better application compatibility, disabling + * l3 alignment feature on the hardware revisions it supports + */ + nicvf_apad_config(nic, false); + + /* Get queue ranges for this VF */ + nicvf_tx_range(dev, nic, &tx_start, &tx_end); + + /* Configure TX queues */ + for (qidx = tx_start; qidx <= tx_end; qidx++) { + ret = nicvf_vf_start_tx_queue(dev, nic, + qidx % MAX_SND_QUEUES_PER_QS); + if (ret) + goto start_txq_error; + } + + /* Configure RX queues */ + for (qidx = rx_start; qidx <= rx_end; qidx++) { + ret = nicvf_vf_start_rx_queue(dev, nic, + qidx % MAX_RCV_QUEUES_PER_QS); + if (ret) + goto start_rxq_error; + } + + if (!nic->sqs_mode) { + /* Configure CPI algorithm */ + ret = nicvf_configure_cpi(dev); + if (ret) + goto start_txq_error; + + ret = nicvf_mbox_get_rss_size(nic); + if (ret) { + PMD_INIT_LOG(ERR, "Failed to get rss table size"); + goto qset_rss_error; + } + + /* Configure RSS */ + ret = nicvf_configure_rss(dev); + if (ret) + goto qset_rss_error; + } + + /* Done; Let PF make the BGX's RX and TX switches to ON position */ + nicvf_mbox_cfg_done(nic); + return 0; + +qset_rss_error: + nicvf_rss_term(nic); +start_rxq_error: + for (qidx = rx_start; qidx <= rx_end; qidx++) + nicvf_vf_stop_rx_queue(dev, nic, qidx % MAX_RCV_QUEUES_PER_QS); +start_txq_error: + for (qidx = tx_start; qidx <= tx_end; qidx++) + nicvf_vf_stop_tx_queue(dev, nic, qidx % MAX_SND_QUEUES_PER_QS); +qset_rbdr_reclaim: + nicvf_qset_rbdr_reclaim(nic, 0); + nicvf_rbdr_release_mbufs(dev, nic); +qset_rbdr_free: + if (nic->rbdr) { + rte_free(nic->rbdr); + nic->rbdr = NULL; + } +qset_reclaim: + nicvf_qset_reclaim(nic); + return ret; +} + +static int +nicvf_dev_start(struct rte_eth_dev *dev) +{ + uint16_t qidx; + int ret; + size_t i; + struct nicvf *nic = nicvf_pmd_priv(dev); + struct rte_eth_rxmode *rx_conf = &dev->data->dev_conf.rxmode; + uint16_t mtu; + uint32_t buffsz = 0, rbdrsz = 0; + struct rte_pktmbuf_pool_private *mbp_priv; + struct nicvf_rxq *rxq; + + PMD_INIT_FUNC_TRACE(); + + /* This function must be called for a primary device */ + assert_primary(nic); + + /* Validate RBDR buff size */ + for (qidx = 0; qidx < dev->data->nb_rx_queues; qidx++) { + rxq = dev->data->rx_queues[qidx]; + mbp_priv = rte_mempool_get_priv(rxq->pool); + buffsz = mbp_priv->mbuf_data_room_size - RTE_PKTMBUF_HEADROOM; + if (buffsz % 128) { + PMD_INIT_LOG(ERR, "rxbuf size must be multiply of 128"); + return -EINVAL; + } + if (rbdrsz == 0) + rbdrsz = buffsz; + if (rbdrsz != buffsz) { + PMD_INIT_LOG(ERR, "buffsz not same, qidx=%d (%d/%d)", + qidx, rbdrsz, buffsz); + return -EINVAL; + } + } + + /* Configure loopback */ + ret = nicvf_loopback_config(nic, dev->data->dev_conf.lpbk_mode); + if (ret) { + PMD_INIT_LOG(ERR, "Failed to configure loopback %d", ret); + return ret; + } + + /* Reset all statistics counters attached to this port */ + ret = nicvf_mbox_reset_stat_counters(nic, 0x3FFF, 0x1F, 0xFFFF, 0xFFFF); + if (ret) { + PMD_INIT_LOG(ERR, "Failed to reset stat counters %d", ret); + return ret; + } + + /* Setup scatter mode if needed by jumbo */ + if (dev->data->dev_conf.rxmode.max_rx_pkt_len + + 2 * VLAN_TAG_SIZE > buffsz) + dev->data->scattered_rx = 1; + if ((rx_conf->offloads & DEV_RX_OFFLOAD_SCATTER) != 0) + dev->data->scattered_rx = 1; + + /* Setup MTU based on max_rx_pkt_len or default */ + mtu = dev->data->dev_conf.rxmode.offloads & DEV_RX_OFFLOAD_JUMBO_FRAME ? + dev->data->dev_conf.rxmode.max_rx_pkt_len + - RTE_ETHER_HDR_LEN : RTE_ETHER_MTU; + + if (nicvf_dev_set_mtu(dev, mtu)) { + PMD_INIT_LOG(ERR, "Failed to set default mtu size"); + return -EBUSY; + } + + ret = nicvf_vf_start(dev, nic, rbdrsz); + if (ret != 0) + return ret; + + for (i = 0; i < nic->sqs_count; i++) { + assert(nic->snicvf[i]); + + ret = nicvf_vf_start(dev, nic->snicvf[i], rbdrsz); + if (ret != 0) + return ret; + } + + /* Configure callbacks based on offloads */ + nicvf_set_tx_function(dev); + nicvf_set_rx_function(dev); + + return 0; +} + +static void +nicvf_dev_stop_cleanup(struct rte_eth_dev *dev, bool cleanup) +{ + size_t i; + int ret; + struct nicvf *nic = nicvf_pmd_priv(dev); + + PMD_INIT_FUNC_TRACE(); + + /* Teardown secondary vf first */ + for (i = 0; i < nic->sqs_count; i++) { + if (!nic->snicvf[i]) + continue; + + nicvf_vf_stop(dev, nic->snicvf[i], cleanup); + } + + /* Stop the primary VF now */ + nicvf_vf_stop(dev, nic, cleanup); + + /* Disable loopback */ + ret = nicvf_loopback_config(nic, 0); + if (ret) + PMD_INIT_LOG(ERR, "Failed to disable loopback %d", ret); + + /* Reclaim CPI configuration */ + ret = nicvf_mbox_config_cpi(nic, 0); + if (ret) + PMD_INIT_LOG(ERR, "Failed to reclaim CPI config %d", ret); +} + +static void +nicvf_dev_stop(struct rte_eth_dev *dev) +{ + PMD_INIT_FUNC_TRACE(); + + nicvf_dev_stop_cleanup(dev, false); +} + +static void +nicvf_vf_stop(struct rte_eth_dev *dev, struct nicvf *nic, bool cleanup) +{ + int ret; + uint16_t qidx; + uint16_t tx_start, tx_end; + uint16_t rx_start, rx_end; + + PMD_INIT_FUNC_TRACE(); + + if (cleanup) { + /* Let PF make the BGX's RX and TX switches to OFF position */ + nicvf_mbox_shutdown(nic); + } + + /* Disable VLAN Strip */ + nicvf_vlan_hw_strip(nic, 0); + + /* Get queue ranges for this VF */ + nicvf_tx_range(dev, nic, &tx_start, &tx_end); + + for (qidx = tx_start; qidx <= tx_end; qidx++) + nicvf_vf_stop_tx_queue(dev, nic, qidx % MAX_SND_QUEUES_PER_QS); + + /* Get queue ranges for this VF */ + nicvf_rx_range(dev, nic, &rx_start, &rx_end); + + /* Reclaim rq */ + for (qidx = rx_start; qidx <= rx_end; qidx++) + nicvf_vf_stop_rx_queue(dev, nic, qidx % MAX_RCV_QUEUES_PER_QS); + + /* Reclaim RBDR */ + ret = nicvf_qset_rbdr_reclaim(nic, 0); + if (ret) + PMD_INIT_LOG(ERR, "Failed to reclaim RBDR %d", ret); + + /* Move all charged buffers in RBDR back to pool */ + if (nic->rbdr != NULL) + nicvf_rbdr_release_mbufs(dev, nic); + + /* Disable qset */ + ret = nicvf_qset_reclaim(nic); + if (ret) + PMD_INIT_LOG(ERR, "Failed to disable qset %d", ret); + + /* Disable all interrupts */ + nicvf_disable_all_interrupts(nic); + + /* Free RBDR SW structure */ + if (nic->rbdr) { + rte_free(nic->rbdr); + nic->rbdr = NULL; + } +} + +static void +nicvf_dev_close(struct rte_eth_dev *dev) +{ + size_t i; + struct nicvf *nic = nicvf_pmd_priv(dev); + + PMD_INIT_FUNC_TRACE(); + + nicvf_dev_stop_cleanup(dev, true); + nicvf_periodic_alarm_stop(nicvf_interrupt, dev); + + for (i = 0; i < nic->sqs_count; i++) { + if (!nic->snicvf[i]) + continue; + + nicvf_periodic_alarm_stop(nicvf_vf_interrupt, nic->snicvf[i]); + } +} + +static int +nicvf_request_sqs(struct nicvf *nic) +{ + size_t i; + + assert_primary(nic); + assert(nic->sqs_count > 0); + assert(nic->sqs_count <= MAX_SQS_PER_VF); + + /* Set no of Rx/Tx queues in each of the SQsets */ + for (i = 0; i < nic->sqs_count; i++) { + if (nicvf_svf_empty()) + rte_panic("Cannot assign sufficient number of " + "secondary queues to primary VF%" PRIu8 "\n", + nic->vf_id); + + nic->snicvf[i] = nicvf_svf_pop(); + nic->snicvf[i]->sqs_id = i; + } + + return nicvf_mbox_request_sqs(nic); +} + static int nicvf_dev_configure(struct rte_eth_dev *dev) { - struct rte_eth_conf *conf = &dev->data->dev_conf; + struct rte_eth_dev_data *data = dev->data; + struct rte_eth_conf *conf = &data->dev_conf; struct rte_eth_rxmode *rxmode = &conf->rxmode; struct rte_eth_txmode *txmode = &conf->txmode; struct nicvf *nic = nicvf_pmd_priv(dev); + uint8_t cqcount; PMD_INIT_FUNC_TRACE(); @@ -713,36 +1926,11 @@ nicvf_dev_configure(struct rte_eth_dev *dev) return -EINVAL; } - if (!rxmode->hw_strip_crc) { - PMD_INIT_LOG(NOTICE, "Can't disable hw crc strip"); - rxmode->hw_strip_crc = 1; - } - - if (rxmode->hw_ip_checksum) { - PMD_INIT_LOG(NOTICE, "Rxcksum not supported"); - rxmode->hw_ip_checksum = 0; - } - if (rxmode->split_hdr_size) { PMD_INIT_LOG(INFO, "Rxmode does not support split header"); return -EINVAL; } - if (rxmode->hw_vlan_filter) { - PMD_INIT_LOG(INFO, "VLAN filter not supported"); - return -EINVAL; - } - - if (rxmode->hw_vlan_extend) { - PMD_INIT_LOG(INFO, "VLAN extended not supported"); - return -EINVAL; - } - - if (rxmode->enable_lro) { - PMD_INIT_LOG(INFO, "LRO not supported"); - return -EINVAL; - } - if (conf->link_speeds & ETH_LINK_SPEED_FIXED) { PMD_INIT_LOG(INFO, "Setting link speed/duplex not supported"); return -EINVAL; @@ -758,6 +1946,29 @@ nicvf_dev_configure(struct rte_eth_dev *dev) return -EINVAL; } + assert_primary(nic); + NICVF_STATIC_ASSERT(MAX_RCV_QUEUES_PER_QS == MAX_SND_QUEUES_PER_QS); + cqcount = RTE_MAX(data->nb_tx_queues, data->nb_rx_queues); + if (cqcount > MAX_RCV_QUEUES_PER_QS) { + nic->sqs_count = RTE_ALIGN_CEIL(cqcount, MAX_RCV_QUEUES_PER_QS); + nic->sqs_count = (nic->sqs_count / MAX_RCV_QUEUES_PER_QS) - 1; + } else { + nic->sqs_count = 0; + } + + assert(nic->sqs_count <= MAX_SQS_PER_VF); + + if (nic->sqs_count > 0) { + if (nicvf_request_sqs(nic)) { + rte_panic("Cannot assign sufficient number of " + "secondary queues to PORT%d VF%" PRIu8 "\n", + dev->data->port_id, nic->vf_id); + } + } + + if (rxmode->offloads & DEV_RX_OFFLOAD_CHECKSUM) + nic->offload_cksum = 1; + PMD_INIT_LOG(DEBUG, "Configured ethdev port%d hwcap=0x%" PRIx64, dev->data->port_id, nicvf_hw_cap(nic)); @@ -767,20 +1978,110 @@ nicvf_dev_configure(struct rte_eth_dev *dev) /* Initialize and register driver with DPDK Application */ static const struct eth_dev_ops nicvf_eth_dev_ops = { .dev_configure = nicvf_dev_configure, + .dev_start = nicvf_dev_start, + .dev_stop = nicvf_dev_stop, .link_update = nicvf_dev_link_update, + .dev_close = nicvf_dev_close, + .stats_get = nicvf_dev_stats_get, + .stats_reset = nicvf_dev_stats_reset, + .promiscuous_enable = nicvf_dev_promisc_enable, .dev_infos_get = nicvf_dev_info_get, + .dev_supported_ptypes_get = nicvf_dev_supported_ptypes_get, + .mtu_set = nicvf_dev_set_mtu, + .vlan_offload_set = nicvf_vlan_offload_set, .reta_update = nicvf_dev_reta_update, .reta_query = nicvf_dev_reta_query, .rss_hash_update = nicvf_dev_rss_hash_update, .rss_hash_conf_get = nicvf_dev_rss_hash_conf_get, + .rx_queue_start = nicvf_dev_rx_queue_start, + .rx_queue_stop = nicvf_dev_rx_queue_stop, + .tx_queue_start = nicvf_dev_tx_queue_start, + .tx_queue_stop = nicvf_dev_tx_queue_stop, .rx_queue_setup = nicvf_dev_rx_queue_setup, .rx_queue_release = nicvf_dev_rx_queue_release, + .rx_queue_count = nicvf_dev_rx_queue_count, .tx_queue_setup = nicvf_dev_tx_queue_setup, .tx_queue_release = nicvf_dev_tx_queue_release, - .get_reg_length = nicvf_dev_get_reg_length, .get_reg = nicvf_dev_get_regs, }; +static int +nicvf_vlan_offload_config(struct rte_eth_dev *dev, int mask) +{ + struct rte_eth_rxmode *rxmode; + struct nicvf *nic = nicvf_pmd_priv(dev); + rxmode = &dev->data->dev_conf.rxmode; + if (mask & ETH_VLAN_STRIP_MASK) { + if (rxmode->offloads & DEV_RX_OFFLOAD_VLAN_STRIP) + nicvf_vlan_hw_strip(nic, true); + else + nicvf_vlan_hw_strip(nic, false); + } + + return 0; +} + +static int +nicvf_vlan_offload_set(struct rte_eth_dev *dev, int mask) +{ + nicvf_vlan_offload_config(dev, mask); + + return 0; +} + +static inline int +nicvf_set_first_skip(struct rte_eth_dev *dev) +{ + int bytes_to_skip = 0; + int ret = 0; + unsigned int i; + struct rte_kvargs *kvlist; + static const char *const skip[] = { + SKIP_DATA_BYTES, + NULL}; + struct nicvf *nic = nicvf_pmd_priv(dev); + + if (!dev->device->devargs) { + nicvf_first_skip_config(nic, 0); + return ret; + } + + kvlist = rte_kvargs_parse(dev->device->devargs->args, skip); + if (!kvlist) + return -EINVAL; + + if (kvlist->count == 0) + goto exit; + + for (i = 0; i != kvlist->count; ++i) { + const struct rte_kvargs_pair *pair = &kvlist->pairs[i]; + + if (!strcmp(pair->key, SKIP_DATA_BYTES)) + bytes_to_skip = atoi(pair->value); + } + + /*128 bytes amounts to one cache line*/ + if (bytes_to_skip >= 0 && bytes_to_skip < 128) { + if (!(bytes_to_skip % 8)) { + nicvf_first_skip_config(nic, (bytes_to_skip / 8)); + nic->skip_bytes = bytes_to_skip; + goto kvlist_free; + } else { + PMD_INIT_LOG(ERR, "skip_data_bytes should be multiple of 8"); + ret = -EINVAL; + goto exit; + } + } else { + PMD_INIT_LOG(ERR, "skip_data_bytes should be less than 128"); + ret = -EINVAL; + goto exit; + } +exit: + nicvf_first_skip_config(nic, 0); +kvlist_free: + rte_kvargs_free(kvlist); + return ret; +} static int nicvf_eth_dev_init(struct rte_eth_dev *eth_dev) { @@ -792,14 +2093,27 @@ nicvf_eth_dev_init(struct rte_eth_dev *eth_dev) eth_dev->dev_ops = &nicvf_eth_dev_ops; - pci_dev = eth_dev->pci_dev; + /* For secondary processes, the primary has done all the work */ + if (rte_eal_process_type() != RTE_PROC_PRIMARY) { + if (nic) { + /* Setup callbacks for secondary process */ + nicvf_set_tx_function(eth_dev); + nicvf_set_rx_function(eth_dev); + return 0; + } else { + /* If nic == NULL than it is secondary function + * so ethdev need to be released by caller */ + return ENOTSUP; + } + } + + pci_dev = RTE_ETH_DEV_TO_PCI(eth_dev); rte_eth_copy_pci_info(eth_dev, pci_dev); nic->device_id = pci_dev->id.device_id; nic->vendor_id = pci_dev->id.vendor_id; nic->subsystem_device_id = pci_dev->id.subsystem_device_id; nic->subsystem_vendor_id = pci_dev->id.subsystem_vendor_id; - nic->eth_dev = eth_dev; PMD_INIT_LOG(DEBUG, "nicvf: device (%x:%x) %u:%u:%u:%u", pci_dev->id.vendor_id, pci_dev->id.device_id, @@ -815,7 +2129,7 @@ nicvf_eth_dev_init(struct rte_eth_dev *eth_dev) nicvf_disable_all_interrupts(nic); - ret = nicvf_periodic_alarm_start(nic); + ret = nicvf_periodic_alarm_start(nicvf_interrupt, eth_dev); if (ret) { PMD_INIT_LOG(ERR, "Failed to start period alarm"); goto fail; @@ -835,23 +2149,41 @@ nicvf_eth_dev_init(struct rte_eth_dev *eth_dev) ); } + ret = nicvf_base_init(nic); + if (ret) { + PMD_INIT_LOG(ERR, "Failed to execute nicvf_base_init"); + goto malloc_fail; + } + if (nic->sqs_mode) { - PMD_INIT_LOG(INFO, "Unsupported SQS VF detected, Detaching..."); - /* Detach port by returning Positive error number */ - ret = ENOTSUP; - goto alarm_fail; + /* Push nic to stack of secondary vfs */ + nicvf_svf_push(nic); + + /* Steal nic pointer from the device for further reuse */ + eth_dev->data->dev_private = NULL; + + nicvf_periodic_alarm_stop(nicvf_interrupt, eth_dev); + ret = nicvf_periodic_alarm_start(nicvf_vf_interrupt, nic); + if (ret) { + PMD_INIT_LOG(ERR, "Failed to start period alarm"); + goto fail; + } + + /* Detach port by returning positive error number */ + return ENOTSUP; } - eth_dev->data->mac_addrs = rte_zmalloc("mac_addr", ETHER_ADDR_LEN, 0); + eth_dev->data->mac_addrs = rte_zmalloc("mac_addr", + RTE_ETHER_ADDR_LEN, 0); if (eth_dev->data->mac_addrs == NULL) { PMD_INIT_LOG(ERR, "Failed to allocate memory for mac addr"); ret = -ENOMEM; goto alarm_fail; } - if (is_zero_ether_addr((struct ether_addr *)nic->mac_addr)) - eth_random_addr(&nic->mac_addr[0]); + if (rte_is_zero_ether_addr((struct rte_ether_addr *)nic->mac_addr)) + rte_eth_random_addr(&nic->mac_addr[0]); - ether_addr_copy((struct ether_addr *)nic->mac_addr, + rte_ether_addr_copy((struct rte_ether_addr *)nic->mac_addr, ð_dev->data->mac_addrs[0]); ret = nicvf_mbox_set_mac_addr(nic, nic->mac_addr); @@ -860,18 +2192,11 @@ nicvf_eth_dev_init(struct rte_eth_dev *eth_dev) goto malloc_fail; } - ret = nicvf_base_init(nic); - if (ret) { - PMD_INIT_LOG(ERR, "Failed to execute nicvf_base_init"); - goto malloc_fail; - } - - ret = nicvf_mbox_get_rss_size(nic); + ret = nicvf_set_first_skip(eth_dev); if (ret) { - PMD_INIT_LOG(ERR, "Failed to get rss table size"); + PMD_INIT_LOG(ERR, "Failed to configure first skip"); goto malloc_fail; } - PMD_INIT_LOG(INFO, "Port %d (%x:%x) mac=%02x:%02x:%02x:%02x:%02x:%02x", eth_dev->data->port_id, nic->vendor_id, nic->device_id, nic->mac_addr[0], nic->mac_addr[1], nic->mac_addr[2], @@ -882,7 +2207,7 @@ nicvf_eth_dev_init(struct rte_eth_dev *eth_dev) malloc_fail: rte_free(eth_dev->data->mac_addrs); alarm_fail: - nicvf_periodic_alarm_stop(nic); + nicvf_periodic_alarm_stop(nicvf_interrupt, eth_dev); fail: return ret; } @@ -891,47 +2216,57 @@ static const struct rte_pci_id pci_id_nicvf_map[] = { { .class_id = RTE_CLASS_ANY_ID, .vendor_id = PCI_VENDOR_ID_CAVIUM, - .device_id = PCI_DEVICE_ID_THUNDERX_PASS1_NICVF, + .device_id = PCI_DEVICE_ID_THUNDERX_CN88XX_PASS1_NICVF, .subsystem_vendor_id = PCI_VENDOR_ID_CAVIUM, - .subsystem_device_id = PCI_SUB_DEVICE_ID_THUNDERX_PASS1_NICVF, + .subsystem_device_id = PCI_SUB_DEVICE_ID_CN88XX_PASS1_NICVF, }, { .class_id = RTE_CLASS_ANY_ID, .vendor_id = PCI_VENDOR_ID_CAVIUM, - .device_id = PCI_DEVICE_ID_THUNDERX_PASS2_NICVF, + .device_id = PCI_DEVICE_ID_THUNDERX_NICVF, .subsystem_vendor_id = PCI_VENDOR_ID_CAVIUM, - .subsystem_device_id = PCI_SUB_DEVICE_ID_THUNDERX_PASS2_NICVF, + .subsystem_device_id = PCI_SUB_DEVICE_ID_CN88XX_PASS2_NICVF, }, { - .vendor_id = 0, + .class_id = RTE_CLASS_ANY_ID, + .vendor_id = PCI_VENDOR_ID_CAVIUM, + .device_id = PCI_DEVICE_ID_THUNDERX_NICVF, + .subsystem_vendor_id = PCI_VENDOR_ID_CAVIUM, + .subsystem_device_id = PCI_SUB_DEVICE_ID_CN81XX_NICVF, }, -}; - -static struct eth_driver rte_nicvf_pmd = { - .pci_drv = { - .name = "rte_nicvf_pmd", - .id_table = pci_id_nicvf_map, - .drv_flags = RTE_PCI_DRV_NEED_MAPPING | RTE_PCI_DRV_INTR_LSC, + { + .class_id = RTE_CLASS_ANY_ID, + .vendor_id = PCI_VENDOR_ID_CAVIUM, + .device_id = PCI_DEVICE_ID_THUNDERX_NICVF, + .subsystem_vendor_id = PCI_VENDOR_ID_CAVIUM, + .subsystem_device_id = PCI_SUB_DEVICE_ID_CN83XX_NICVF, + }, + { + .vendor_id = 0, }, - .eth_dev_init = nicvf_eth_dev_init, - .dev_private_size = sizeof(struct nicvf), }; -static int -rte_nicvf_pmd_init(const char *name __rte_unused, const char *para __rte_unused) +static int nicvf_eth_pci_probe(struct rte_pci_driver *pci_drv __rte_unused, + struct rte_pci_device *pci_dev) { - PMD_INIT_FUNC_TRACE(); - PMD_INIT_LOG(INFO, "librte_pmd_thunderx nicvf version %s", - THUNDERX_NICVF_PMD_VERSION); + return rte_eth_dev_pci_generic_probe(pci_dev, sizeof(struct nicvf), + nicvf_eth_dev_init); +} - rte_eth_driver_register(&rte_nicvf_pmd); - return 0; +static int nicvf_eth_pci_remove(struct rte_pci_device *pci_dev) +{ + return rte_eth_dev_pci_generic_remove(pci_dev, NULL); } -static struct rte_driver rte_nicvf_driver = { - .name = "nicvf_driver", - .type = PMD_PDEV, - .init = rte_nicvf_pmd_init, +static struct rte_pci_driver rte_nicvf_pmd = { + .id_table = pci_id_nicvf_map, + .drv_flags = RTE_PCI_DRV_NEED_MAPPING | RTE_PCI_DRV_KEEP_MAPPED_RES | + RTE_PCI_DRV_INTR_LSC, + .probe = nicvf_eth_pci_probe, + .remove = nicvf_eth_pci_remove, }; -PMD_REGISTER_DRIVER(rte_nicvf_driver); +RTE_PMD_REGISTER_PCI(net_thunderx, rte_nicvf_pmd); +RTE_PMD_REGISTER_PCI_TABLE(net_thunderx, pci_id_nicvf_map); +RTE_PMD_REGISTER_KMOD_DEP(net_thunderx, "* igb_uio | uio_pci_generic | vfio-pci"); +RTE_PMD_REGISTER_PARAM_STRING(net_thunderx, SKIP_DATA_BYTES "=");