From: Vlad Zolotarov Date: Thu, 12 Mar 2015 21:17:32 +0000 (+0200) Subject: ixgbe: unify Rx setup X-Git-Tag: spdx-start~9432 X-Git-Url: http://git.droids-corp.org/?a=commitdiff_plain;h=01fa1d6215fa7cd6b5303ac9296381b75b9226de;p=dpdk.git ixgbe: unify Rx setup - Set the callback in a single function that is called from ixgbe_dev_rx_init() for a primary process and from eth_ixgbe_dev_init() for a secondary processes. This is instead of multiple, hard to track places. - Added ixgbe_hw.rx_bulk_alloc_allowed - see ixgbe_hw.rx_vec_allowed description below. - Added ixgbe_hw.rx_vec_allowed: like with Bulk Allocation, Vector Rx is enabled or disabled on a per-port level. All queues have to meet the appropriate preconditions and if any of them doesn't - the feature has to be disabled. Therefore ixgbe_hw.rx_vec_allowed will be updated during each queues configuration (rte_eth_rx_queue_setup()) and then used in rte_eth_dev_start() to configure the appropriate callbacks. The same happens with ixgbe_hw.rx_vec_allowed in a Bulk Allocation context. - Bugs fixed: - Vector scattered packets callback was called regardless the appropriate preconditions: - Vector Rx specific preconditions. - Bulk Allocation preconditions. - Vector Rx was enabled/disabled according to the last queue setting and not based on all queues setting (which may be different for each queue). Signed-off-by: Vlad Zolotarov Acked-by: Konstantin Ananyev --- diff --git a/lib/librte_pmd_ixgbe/ixgbe/ixgbe_type.h b/lib/librte_pmd_ixgbe/ixgbe/ixgbe_type.h index c67d462fc2..9a66370a06 100644 --- a/lib/librte_pmd_ixgbe/ixgbe/ixgbe_type.h +++ b/lib/librte_pmd_ixgbe/ixgbe/ixgbe_type.h @@ -3657,6 +3657,8 @@ struct ixgbe_hw { bool force_full_reset; bool allow_unsupported_sfp; bool wol_enabled; + bool rx_bulk_alloc_allowed; + bool rx_vec_allowed; }; #define ixgbe_call_func(hw, func, params, error) \ diff --git a/lib/librte_pmd_ixgbe/ixgbe_ethdev.c b/lib/librte_pmd_ixgbe/ixgbe_ethdev.c index e4edb01a72..92d75dbe65 100644 --- a/lib/librte_pmd_ixgbe/ixgbe_ethdev.c +++ b/lib/librte_pmd_ixgbe/ixgbe_ethdev.c @@ -756,8 +756,8 @@ eth_ixgbe_dev_init(struct rte_eth_dev *eth_dev) "Using default TX function."); } - if (eth_dev->data->scattered_rx) - eth_dev->rx_pkt_burst = ixgbe_recv_scattered_pkts; + ixgbe_set_rx_function(eth_dev); + return 0; } pci_dev = eth_dev->pci_dev; @@ -1429,12 +1429,21 @@ ixgbe_dev_configure(struct rte_eth_dev *dev) { struct ixgbe_interrupt *intr = IXGBE_DEV_PRIVATE_TO_INTR(dev->data->dev_private); + struct ixgbe_hw *hw = + IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); PMD_INIT_FUNC_TRACE(); /* set flag to update link status after init */ intr->flags |= IXGBE_FLAG_NEED_LINK_UPDATE; + /* + * Initialize to TRUE. If any of Rx queues doesn't meet the bulk + * allocation or vector Rx preconditions we will reset it. + */ + hw->rx_bulk_alloc_allowed = true; + hw->rx_vec_allowed = true; + return 0; } diff --git a/lib/librte_pmd_ixgbe/ixgbe_rxtx.c b/lib/librte_pmd_ixgbe/ixgbe_rxtx.c index 92be61e749..42f0aa5e7c 100644 --- a/lib/librte_pmd_ixgbe/ixgbe_rxtx.c +++ b/lib/librte_pmd_ixgbe/ixgbe_rxtx.c @@ -2075,12 +2075,12 @@ check_rx_burst_bulk_alloc_preconditions(__rte_unused struct ixgbe_rx_queue *rxq) /* Reset dynamic ixgbe_rx_queue fields back to defaults */ static void -ixgbe_reset_rx_queue(struct ixgbe_rx_queue *rxq) +ixgbe_reset_rx_queue(struct ixgbe_hw *hw, struct ixgbe_rx_queue *rxq) { static const union ixgbe_adv_rx_desc zeroed_desc = { .read = { .pkt_addr = 0}}; unsigned i; - uint16_t len; + uint16_t len = rxq->nb_rx_desc; /* * By default, the Rx queue setup function allocates enough memory for @@ -2092,14 +2092,9 @@ ixgbe_reset_rx_queue(struct ixgbe_rx_queue *rxq) * constraints here to see if we need to zero out memory after the end * of the H/W descriptor ring. */ -#ifdef RTE_LIBRTE_IXGBE_RX_ALLOW_BULK_ALLOC - if (check_rx_burst_bulk_alloc_preconditions(rxq) == 0) + if (hw->rx_bulk_alloc_allowed) /* zero out extra memory */ - len = (uint16_t)(rxq->nb_rx_desc + RTE_PMD_IXGBE_RX_MAX_BURST); - else -#endif - /* do not zero out extra memory */ - len = rxq->nb_rx_desc; + len += RTE_PMD_IXGBE_RX_MAX_BURST; /* * Zero out HW ring memory. Zero out extra memory at the end of @@ -2141,7 +2136,6 @@ ixgbe_dev_rx_queue_setup(struct rte_eth_dev *dev, const struct rte_memzone *rz; struct ixgbe_rx_queue *rxq; struct ixgbe_hw *hw; - int use_def_burst_func = 1; uint16_t len; PMD_INIT_FUNC_TRACE(); @@ -2223,16 +2217,28 @@ ixgbe_dev_rx_queue_setup(struct rte_eth_dev *dev, #endif rxq->rx_ring = (union ixgbe_adv_rx_desc *) rz->addr; + /* + * Certain constraints must be met in order to use the bulk buffer + * allocation Rx burst function. If any of Rx queues doesn't meet them + * the feature should be disabled for the whole port. + */ + if (check_rx_burst_bulk_alloc_preconditions(rxq)) { + PMD_INIT_LOG(DEBUG, "queue[%d] doesn't meet Rx Bulk Alloc " + "preconditions - canceling the feature for " + "the whole port[%d]", + rxq->queue_id, rxq->port_id); + hw->rx_bulk_alloc_allowed = false; + } + /* * Allocate software ring. Allow for space at the end of the * S/W ring to make sure look-ahead logic in bulk alloc Rx burst * function does not access an invalid memory region. */ -#ifdef RTE_LIBRTE_IXGBE_RX_ALLOW_BULK_ALLOC - len = (uint16_t)(nb_desc + RTE_PMD_IXGBE_RX_MAX_BURST); -#else len = nb_desc; -#endif + if (hw->rx_bulk_alloc_allowed) + len += RTE_PMD_IXGBE_RX_MAX_BURST; + rxq->sw_ring = rte_zmalloc_socket("rxq->sw_ring", sizeof(struct ixgbe_rx_entry) * len, RTE_CACHE_LINE_SIZE, socket_id); @@ -2243,42 +2249,18 @@ ixgbe_dev_rx_queue_setup(struct rte_eth_dev *dev, PMD_INIT_LOG(DEBUG, "sw_ring=%p hw_ring=%p dma_addr=0x%"PRIx64, rxq->sw_ring, rxq->rx_ring, rxq->rx_ring_phys_addr); - /* - * Certain constraints must be met in order to use the bulk buffer - * allocation Rx burst function. - */ - use_def_burst_func = check_rx_burst_bulk_alloc_preconditions(rxq); + if (!rte_is_power_of_2(nb_desc)) { + PMD_INIT_LOG(DEBUG, "queue[%d] doesn't meet Vector Rx " + "preconditions - canceling the feature for " + "the whole port[%d]", + rxq->queue_id, rxq->port_id); + hw->rx_vec_allowed = false; + } else + ixgbe_rxq_vec_setup(rxq); -#ifdef RTE_IXGBE_INC_VECTOR - ixgbe_rxq_vec_setup(rxq); -#endif - /* Check if pre-conditions are satisfied, and no Scattered Rx */ - if (!use_def_burst_func && !dev->data->scattered_rx) { -#ifdef RTE_LIBRTE_IXGBE_RX_ALLOW_BULK_ALLOC - PMD_INIT_LOG(DEBUG, "Rx Burst Bulk Alloc Preconditions are " - "satisfied. Rx Burst Bulk Alloc function will be " - "used on port=%d, queue=%d.", - rxq->port_id, rxq->queue_id); - dev->rx_pkt_burst = ixgbe_recv_pkts_bulk_alloc; -#ifdef RTE_IXGBE_INC_VECTOR - if (!ixgbe_rx_vec_condition_check(dev) && - (rte_is_power_of_2(nb_desc))) { - PMD_INIT_LOG(INFO, "Vector rx enabled, please make " - "sure RX burst size no less than 32."); - dev->rx_pkt_burst = ixgbe_recv_pkts_vec; - } -#endif -#endif - } else { - PMD_INIT_LOG(DEBUG, "Rx Burst Bulk Alloc Preconditions " - "are not satisfied, Scattered Rx is requested, " - "or RTE_LIBRTE_IXGBE_RX_ALLOW_BULK_ALLOC is not " - "enabled (port=%d, queue=%d).", - rxq->port_id, rxq->queue_id); - } dev->data->rx_queues[queue_idx] = rxq; - ixgbe_reset_rx_queue(rxq); + ixgbe_reset_rx_queue(hw, rxq); return 0; } @@ -2332,6 +2314,7 @@ void ixgbe_dev_clear_queues(struct rte_eth_dev *dev) { unsigned i; + struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); PMD_INIT_FUNC_TRACE(); @@ -2347,7 +2330,7 @@ ixgbe_dev_clear_queues(struct rte_eth_dev *dev) struct ixgbe_rx_queue *rxq = dev->data->rx_queues[i]; if (rxq != NULL) { ixgbe_rx_queue_release_mbufs(rxq); - ixgbe_reset_rx_queue(rxq); + ixgbe_reset_rx_queue(hw, rxq); } } } @@ -3509,6 +3492,74 @@ ixgbe_dev_mq_tx_configure(struct rte_eth_dev *dev) return 0; } +void ixgbe_set_rx_function(struct rte_eth_dev *dev) +{ + struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private); + + /* + * In order to allow Vector Rx there are a few configuration + * conditions to be met and Rx Bulk Allocation should be allowed. + */ + if (ixgbe_rx_vec_dev_conf_condition_check(dev) || + !hw->rx_bulk_alloc_allowed) { + PMD_INIT_LOG(DEBUG, "Port[%d] doesn't meet Vector Rx " + "preconditions or RTE_IXGBE_INC_VECTOR is " + "not enabled", + dev->data->port_id); + + hw->rx_vec_allowed = false; + } + + if (dev->data->scattered_rx) { + /* + * Set the non-LRO scattered callback: there are Vector and + * single allocation versions. + */ + if (hw->rx_vec_allowed) { + PMD_INIT_LOG(DEBUG, "Using Vector Scattered Rx " + "callback (port=%d).", + dev->data->port_id); + + dev->rx_pkt_burst = ixgbe_recv_scattered_pkts_vec; + } else { + PMD_INIT_LOG(DEBUG, "Using Regualr (non-vector) " + "Scattered Rx callback " + "(port=%d).", + dev->data->port_id); + + dev->rx_pkt_burst = ixgbe_recv_scattered_pkts; + } + /* + * Below we set "simple" callbacks according to port/queues parameters. + * If parameters allow we are going to choose between the following + * callbacks: + * - Vector + * - Bulk Allocation + * - Single buffer allocation (the simplest one) + */ + } else if (hw->rx_vec_allowed) { + PMD_INIT_LOG(INFO, "Vector rx enabled, please make sure RX " + "burst size no less than 32."); + + dev->rx_pkt_burst = ixgbe_recv_pkts_vec; + } else if (hw->rx_bulk_alloc_allowed) { + PMD_INIT_LOG(DEBUG, "Rx Burst Bulk Alloc Preconditions are " + "satisfied. Rx Burst Bulk Alloc function " + "will be used on port=%d.", + dev->data->port_id); + + dev->rx_pkt_burst = ixgbe_recv_pkts_bulk_alloc; + } else { + PMD_INIT_LOG(DEBUG, "Rx Burst Bulk Alloc Preconditions are not " + "satisfied, or Scattered Rx is requested, " + "or RTE_LIBRTE_IXGBE_RX_ALLOW_BULK_ALLOC " + "is not enabled (port=%d).", + dev->data->port_id); + + dev->rx_pkt_burst = ixgbe_recv_pkts; + } +} + /* * Initializes Receive Unit. */ @@ -3644,23 +3695,17 @@ ixgbe_dev_rx_init(struct rte_eth_dev *dev) buf_size = (uint16_t) ((srrctl & IXGBE_SRRCTL_BSIZEPKT_MASK) << IXGBE_SRRCTL_BSIZEPKT_SHIFT); - if (dev->data->dev_conf.rxmode.enable_scatter || - /* It adds dual VLAN length for supporting dual VLAN */ - (dev->data->dev_conf.rxmode.max_rx_pkt_len + - 2 * IXGBE_VLAN_TAG_SIZE) > buf_size){ - if (!dev->data->scattered_rx) - PMD_INIT_LOG(DEBUG, "forcing scatter mode"); + /* It adds dual VLAN length for supporting dual VLAN */ + if (dev->data->dev_conf.rxmode.max_rx_pkt_len + + 2 * IXGBE_VLAN_TAG_SIZE > buf_size) dev->data->scattered_rx = 1; -#ifdef RTE_IXGBE_INC_VECTOR - if (rte_is_power_of_2(rxq->nb_rx_desc)) - dev->rx_pkt_burst = - ixgbe_recv_scattered_pkts_vec; - else -#endif - dev->rx_pkt_burst = ixgbe_recv_scattered_pkts; - } } + if (dev->data->dev_conf.rxmode.enable_scatter) + dev->data->scattered_rx = 1; + + ixgbe_set_rx_function(dev); + /* * Device configured with multiple RX queues. */ @@ -3936,7 +3981,7 @@ ixgbe_dev_rx_queue_stop(struct rte_eth_dev *dev, uint16_t rx_queue_id) rte_delay_us(RTE_IXGBE_WAIT_100_US); ixgbe_rx_queue_release_mbufs(rxq); - ixgbe_reset_rx_queue(rxq); + ixgbe_reset_rx_queue(hw, rxq); } else return -1; @@ -4292,3 +4337,34 @@ ixgbevf_dev_rxtx_start(struct rte_eth_dev *dev) } } + +/* Stubs needed for linkage when CONFIG_RTE_IXGBE_INC_VECTOR is set to 'n' */ +int __attribute__((weak)) +ixgbe_rx_vec_dev_conf_condition_check(struct rte_eth_dev __rte_unused *dev) +{ + return -1; +} + +uint16_t __attribute__((weak)) +ixgbe_recv_pkts_vec( + void __rte_unused *rx_queue, + struct rte_mbuf __rte_unused **rx_pkts, + uint16_t __rte_unused nb_pkts) +{ + return 0; +} + +uint16_t __attribute__((weak)) +ixgbe_recv_scattered_pkts_vec( + void __rte_unused *rx_queue, + struct rte_mbuf __rte_unused **rx_pkts, + uint16_t __rte_unused nb_pkts) +{ + return 0; +} + +int __attribute__((weak)) +ixgbe_rxq_vec_setup(struct ixgbe_rx_queue __rte_unused *rxq) +{ + return -1; +} diff --git a/lib/librte_pmd_ixgbe/ixgbe_rxtx.h b/lib/librte_pmd_ixgbe/ixgbe_rxtx.h index 4cc1d6c839..3937cf670b 100644 --- a/lib/librte_pmd_ixgbe/ixgbe_rxtx.h +++ b/lib/librte_pmd_ixgbe/ixgbe_rxtx.h @@ -255,16 +255,34 @@ struct ixgbe_txq_ops { */ void ixgbe_set_tx_function(struct rte_eth_dev *dev, struct ixgbe_tx_queue *txq); -#ifdef RTE_IXGBE_INC_VECTOR +/** + * Sets the rx_pkt_burst callback in the ixgbe rte_eth_dev instance. + * + * Sets the callback based on the device parameters: + * - ixgbe_hw.rx_bulk_alloc_allowed + * - rte_eth_dev_data.scattered_rx + * - rte_eth_dev_data.lro + * - conditions checked in ixgbe_rx_vec_condition_check() + * + * This means that the parameters above have to be configured prior to calling + * to this function. + * + * @dev rte_eth_dev handle + */ +void ixgbe_set_rx_function(struct rte_eth_dev *dev); + uint16_t ixgbe_recv_pkts_vec(void *rx_queue, struct rte_mbuf **rx_pkts, uint16_t nb_pkts); uint16_t ixgbe_recv_scattered_pkts_vec(void *rx_queue, struct rte_mbuf **rx_pkts, uint16_t nb_pkts); +int ixgbe_rx_vec_dev_conf_condition_check(struct rte_eth_dev *dev); +int ixgbe_rxq_vec_setup(struct ixgbe_rx_queue *rxq); + +#ifdef RTE_IXGBE_INC_VECTOR + uint16_t ixgbe_xmit_pkts_vec(void *tx_queue, struct rte_mbuf **tx_pkts, uint16_t nb_pkts); int ixgbe_txq_vec_setup(struct ixgbe_tx_queue *txq); -int ixgbe_rxq_vec_setup(struct ixgbe_rx_queue *rxq); -int ixgbe_rx_vec_condition_check(struct rte_eth_dev *dev); -#endif -#endif +#endif /* RTE_IXGBE_INC_VECTOR */ +#endif /* _IXGBE_RXTX_H_ */ diff --git a/lib/librte_pmd_ixgbe/ixgbe_rxtx_vec.c b/lib/librte_pmd_ixgbe/ixgbe_rxtx_vec.c index 9d8fa8d378..7ac6b612bf 100644 --- a/lib/librte_pmd_ixgbe/ixgbe_rxtx_vec.c +++ b/lib/librte_pmd_ixgbe/ixgbe_rxtx_vec.c @@ -760,7 +760,7 @@ int ixgbe_txq_vec_setup(struct ixgbe_tx_queue *txq) return 0; } -int ixgbe_rx_vec_condition_check(struct rte_eth_dev *dev) +int ixgbe_rx_vec_dev_conf_condition_check(struct rte_eth_dev *dev) { #ifndef RTE_LIBRTE_IEEE1588 struct rte_eth_rxmode *rxmode = &dev->data->dev_conf.rxmode;