ixgbe: unify Rx setup
authorVlad Zolotarov <vladz@cloudius-systems.com>
Thu, 12 Mar 2015 21:17:32 +0000 (23:17 +0200)
committerThomas Monjalon <thomas.monjalon@6wind.com>
Mon, 16 Mar 2015 23:46:01 +0000 (00:46 +0100)
   - Set the callback in a single function that is called from
     ixgbe_dev_rx_init() for a primary process and from eth_ixgbe_dev_init()
     for a secondary processes. This is instead of multiple, hard to track places.
   - Added ixgbe_hw.rx_bulk_alloc_allowed - see ixgbe_hw.rx_vec_allowed description below.
   - Added ixgbe_hw.rx_vec_allowed: like with Bulk Allocation, Vector Rx is
     enabled or disabled on a per-port level. All queues have to meet the appropriate
     preconditions and if any of them doesn't - the feature has to be disabled.
     Therefore ixgbe_hw.rx_vec_allowed will be updated during each queues configuration
     (rte_eth_rx_queue_setup()) and then used in rte_eth_dev_start() to configure the
     appropriate callbacks. The same happens with ixgbe_hw.rx_vec_allowed in a Bulk Allocation
     context.
   - Bugs fixed:
      - Vector scattered packets callback was called regardless the appropriate
        preconditions:
         - Vector Rx specific preconditions.
         - Bulk Allocation preconditions.
      - Vector Rx was enabled/disabled according to the last queue setting and not
        based on all queues setting (which may be different for each queue).

Signed-off-by: Vlad Zolotarov <vladz@cloudius-systems.com>
Acked-by: Konstantin Ananyev <konstantin.ananyev@intel.com>
lib/librte_pmd_ixgbe/ixgbe/ixgbe_type.h
lib/librte_pmd_ixgbe/ixgbe_ethdev.c
lib/librte_pmd_ixgbe/ixgbe_rxtx.c
lib/librte_pmd_ixgbe/ixgbe_rxtx.h
lib/librte_pmd_ixgbe/ixgbe_rxtx_vec.c

index c67d462..9a66370 100644 (file)
@@ -3657,6 +3657,8 @@ struct ixgbe_hw {
        bool force_full_reset;
        bool allow_unsupported_sfp;
        bool wol_enabled;
+       bool rx_bulk_alloc_allowed;
+       bool rx_vec_allowed;
 };
 
 #define ixgbe_call_func(hw, func, params, error) \
index e4edb01..92d75db 100644 (file)
@@ -756,8 +756,8 @@ eth_ixgbe_dev_init(struct rte_eth_dev *eth_dev)
                                           "Using default TX function.");
                }
 
-               if (eth_dev->data->scattered_rx)
-                       eth_dev->rx_pkt_burst = ixgbe_recv_scattered_pkts;
+               ixgbe_set_rx_function(eth_dev);
+
                return 0;
        }
        pci_dev = eth_dev->pci_dev;
@@ -1429,12 +1429,21 @@ ixgbe_dev_configure(struct rte_eth_dev *dev)
 {
        struct ixgbe_interrupt *intr =
                IXGBE_DEV_PRIVATE_TO_INTR(dev->data->dev_private);
+       struct ixgbe_hw *hw =
+               IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 
        PMD_INIT_FUNC_TRACE();
 
        /* set flag to update link status after init */
        intr->flags |= IXGBE_FLAG_NEED_LINK_UPDATE;
 
+       /*
+        * Initialize to TRUE. If any of Rx queues doesn't meet the bulk
+        * allocation or vector Rx preconditions we will reset it.
+        */
+       hw->rx_bulk_alloc_allowed = true;
+       hw->rx_vec_allowed = true;
+
        return 0;
 }
 
index 92be61e..42f0aa5 100644 (file)
@@ -2075,12 +2075,12 @@ check_rx_burst_bulk_alloc_preconditions(__rte_unused struct ixgbe_rx_queue *rxq)
 
 /* Reset dynamic ixgbe_rx_queue fields back to defaults */
 static void
-ixgbe_reset_rx_queue(struct ixgbe_rx_queue *rxq)
+ixgbe_reset_rx_queue(struct ixgbe_hw *hw, struct ixgbe_rx_queue *rxq)
 {
        static const union ixgbe_adv_rx_desc zeroed_desc = { .read = {
                        .pkt_addr = 0}};
        unsigned i;
-       uint16_t len;
+       uint16_t len = rxq->nb_rx_desc;
 
        /*
         * By default, the Rx queue setup function allocates enough memory for
@@ -2092,14 +2092,9 @@ ixgbe_reset_rx_queue(struct ixgbe_rx_queue *rxq)
         * constraints here to see if we need to zero out memory after the end
         * of the H/W descriptor ring.
         */
-#ifdef RTE_LIBRTE_IXGBE_RX_ALLOW_BULK_ALLOC
-       if (check_rx_burst_bulk_alloc_preconditions(rxq) == 0)
+       if (hw->rx_bulk_alloc_allowed)
                /* zero out extra memory */
-               len = (uint16_t)(rxq->nb_rx_desc + RTE_PMD_IXGBE_RX_MAX_BURST);
-       else
-#endif
-               /* do not zero out extra memory */
-               len = rxq->nb_rx_desc;
+               len += RTE_PMD_IXGBE_RX_MAX_BURST;
 
        /*
         * Zero out HW ring memory. Zero out extra memory at the end of
@@ -2141,7 +2136,6 @@ ixgbe_dev_rx_queue_setup(struct rte_eth_dev *dev,
        const struct rte_memzone *rz;
        struct ixgbe_rx_queue *rxq;
        struct ixgbe_hw     *hw;
-       int use_def_burst_func = 1;
        uint16_t len;
 
        PMD_INIT_FUNC_TRACE();
@@ -2223,16 +2217,28 @@ ixgbe_dev_rx_queue_setup(struct rte_eth_dev *dev,
 #endif
        rxq->rx_ring = (union ixgbe_adv_rx_desc *) rz->addr;
 
+       /*
+        * Certain constraints must be met in order to use the bulk buffer
+        * allocation Rx burst function. If any of Rx queues doesn't meet them
+        * the feature should be disabled for the whole port.
+        */
+       if (check_rx_burst_bulk_alloc_preconditions(rxq)) {
+               PMD_INIT_LOG(DEBUG, "queue[%d] doesn't meet Rx Bulk Alloc "
+                                   "preconditions - canceling the feature for "
+                                   "the whole port[%d]",
+                            rxq->queue_id, rxq->port_id);
+               hw->rx_bulk_alloc_allowed = false;
+       }
+
        /*
         * Allocate software ring. Allow for space at the end of the
         * S/W ring to make sure look-ahead logic in bulk alloc Rx burst
         * function does not access an invalid memory region.
         */
-#ifdef RTE_LIBRTE_IXGBE_RX_ALLOW_BULK_ALLOC
-       len = (uint16_t)(nb_desc + RTE_PMD_IXGBE_RX_MAX_BURST);
-#else
        len = nb_desc;
-#endif
+       if (hw->rx_bulk_alloc_allowed)
+               len += RTE_PMD_IXGBE_RX_MAX_BURST;
+
        rxq->sw_ring = rte_zmalloc_socket("rxq->sw_ring",
                                          sizeof(struct ixgbe_rx_entry) * len,
                                          RTE_CACHE_LINE_SIZE, socket_id);
@@ -2243,42 +2249,18 @@ ixgbe_dev_rx_queue_setup(struct rte_eth_dev *dev,
        PMD_INIT_LOG(DEBUG, "sw_ring=%p hw_ring=%p dma_addr=0x%"PRIx64,
                     rxq->sw_ring, rxq->rx_ring, rxq->rx_ring_phys_addr);
 
-       /*
-        * Certain constraints must be met in order to use the bulk buffer
-        * allocation Rx burst function.
-        */
-       use_def_burst_func = check_rx_burst_bulk_alloc_preconditions(rxq);
+       if (!rte_is_power_of_2(nb_desc)) {
+               PMD_INIT_LOG(DEBUG, "queue[%d] doesn't meet Vector Rx "
+                                   "preconditions - canceling the feature for "
+                                   "the whole port[%d]",
+                            rxq->queue_id, rxq->port_id);
+               hw->rx_vec_allowed = false;
+       } else
+               ixgbe_rxq_vec_setup(rxq);
 
-#ifdef RTE_IXGBE_INC_VECTOR
-       ixgbe_rxq_vec_setup(rxq);
-#endif
-       /* Check if pre-conditions are satisfied, and no Scattered Rx */
-       if (!use_def_burst_func && !dev->data->scattered_rx) {
-#ifdef RTE_LIBRTE_IXGBE_RX_ALLOW_BULK_ALLOC
-               PMD_INIT_LOG(DEBUG, "Rx Burst Bulk Alloc Preconditions are "
-                            "satisfied. Rx Burst Bulk Alloc function will be "
-                            "used on port=%d, queue=%d.",
-                            rxq->port_id, rxq->queue_id);
-               dev->rx_pkt_burst = ixgbe_recv_pkts_bulk_alloc;
-#ifdef RTE_IXGBE_INC_VECTOR
-               if (!ixgbe_rx_vec_condition_check(dev) &&
-                   (rte_is_power_of_2(nb_desc))) {
-                       PMD_INIT_LOG(INFO, "Vector rx enabled, please make "
-                                    "sure RX burst size no less than 32.");
-                       dev->rx_pkt_burst = ixgbe_recv_pkts_vec;
-               }
-#endif
-#endif
-       } else {
-               PMD_INIT_LOG(DEBUG, "Rx Burst Bulk Alloc Preconditions "
-                            "are not satisfied, Scattered Rx is requested, "
-                            "or RTE_LIBRTE_IXGBE_RX_ALLOW_BULK_ALLOC is not "
-                            "enabled (port=%d, queue=%d).",
-                            rxq->port_id, rxq->queue_id);
-       }
        dev->data->rx_queues[queue_idx] = rxq;
 
-       ixgbe_reset_rx_queue(rxq);
+       ixgbe_reset_rx_queue(hw, rxq);
 
        return 0;
 }
@@ -2332,6 +2314,7 @@ void
 ixgbe_dev_clear_queues(struct rte_eth_dev *dev)
 {
        unsigned i;
+       struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 
        PMD_INIT_FUNC_TRACE();
 
@@ -2347,7 +2330,7 @@ ixgbe_dev_clear_queues(struct rte_eth_dev *dev)
                struct ixgbe_rx_queue *rxq = dev->data->rx_queues[i];
                if (rxq != NULL) {
                        ixgbe_rx_queue_release_mbufs(rxq);
-                       ixgbe_reset_rx_queue(rxq);
+                       ixgbe_reset_rx_queue(hw, rxq);
                }
        }
 }
@@ -3509,6 +3492,74 @@ ixgbe_dev_mq_tx_configure(struct rte_eth_dev *dev)
        return 0;
 }
 
+void ixgbe_set_rx_function(struct rte_eth_dev *dev)
+{
+       struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
+
+       /*
+        * In order to allow Vector Rx there are a few configuration
+        * conditions to be met and Rx Bulk Allocation should be allowed.
+        */
+       if (ixgbe_rx_vec_dev_conf_condition_check(dev) ||
+           !hw->rx_bulk_alloc_allowed) {
+               PMD_INIT_LOG(DEBUG, "Port[%d] doesn't meet Vector Rx "
+                                   "preconditions or RTE_IXGBE_INC_VECTOR is "
+                                   "not enabled",
+                            dev->data->port_id);
+
+               hw->rx_vec_allowed = false;
+       }
+
+       if (dev->data->scattered_rx) {
+               /*
+                * Set the non-LRO scattered callback: there are Vector and
+                * single allocation versions.
+                */
+               if (hw->rx_vec_allowed) {
+                       PMD_INIT_LOG(DEBUG, "Using Vector Scattered Rx "
+                                           "callback (port=%d).",
+                                    dev->data->port_id);
+
+                       dev->rx_pkt_burst = ixgbe_recv_scattered_pkts_vec;
+               } else {
+                       PMD_INIT_LOG(DEBUG, "Using Regualr (non-vector) "
+                                           "Scattered Rx callback "
+                                           "(port=%d).",
+                                    dev->data->port_id);
+
+                       dev->rx_pkt_burst = ixgbe_recv_scattered_pkts;
+               }
+       /*
+        * Below we set "simple" callbacks according to port/queues parameters.
+        * If parameters allow we are going to choose between the following
+        * callbacks:
+        *    - Vector
+        *    - Bulk Allocation
+        *    - Single buffer allocation (the simplest one)
+        */
+       } else if (hw->rx_vec_allowed) {
+               PMD_INIT_LOG(INFO, "Vector rx enabled, please make sure RX "
+                                  "burst size no less than 32.");
+
+               dev->rx_pkt_burst = ixgbe_recv_pkts_vec;
+       } else if (hw->rx_bulk_alloc_allowed) {
+               PMD_INIT_LOG(DEBUG, "Rx Burst Bulk Alloc Preconditions are "
+                                   "satisfied. Rx Burst Bulk Alloc function "
+                                   "will be used on port=%d.",
+                            dev->data->port_id);
+
+               dev->rx_pkt_burst = ixgbe_recv_pkts_bulk_alloc;
+       } else {
+               PMD_INIT_LOG(DEBUG, "Rx Burst Bulk Alloc Preconditions are not "
+                                   "satisfied, or Scattered Rx is requested, "
+                                   "or RTE_LIBRTE_IXGBE_RX_ALLOW_BULK_ALLOC "
+                                   "is not enabled (port=%d).",
+                            dev->data->port_id);
+
+               dev->rx_pkt_burst = ixgbe_recv_pkts;
+       }
+}
+
 /*
  * Initializes Receive Unit.
  */
@@ -3644,23 +3695,17 @@ ixgbe_dev_rx_init(struct rte_eth_dev *dev)
                buf_size = (uint16_t) ((srrctl & IXGBE_SRRCTL_BSIZEPKT_MASK) <<
                                       IXGBE_SRRCTL_BSIZEPKT_SHIFT);
 
-               if (dev->data->dev_conf.rxmode.enable_scatter ||
-                   /* It adds dual VLAN length for supporting dual VLAN */
-                   (dev->data->dev_conf.rxmode.max_rx_pkt_len +
-                               2 * IXGBE_VLAN_TAG_SIZE) > buf_size){
-                       if (!dev->data->scattered_rx)
-                               PMD_INIT_LOG(DEBUG, "forcing scatter mode");
+               /* It adds dual VLAN length for supporting dual VLAN */
+               if (dev->data->dev_conf.rxmode.max_rx_pkt_len +
+                                           2 * IXGBE_VLAN_TAG_SIZE > buf_size)
                        dev->data->scattered_rx = 1;
-#ifdef RTE_IXGBE_INC_VECTOR
-                       if (rte_is_power_of_2(rxq->nb_rx_desc))
-                               dev->rx_pkt_burst =
-                                       ixgbe_recv_scattered_pkts_vec;
-                       else
-#endif
-                               dev->rx_pkt_burst = ixgbe_recv_scattered_pkts;
-               }
        }
 
+       if (dev->data->dev_conf.rxmode.enable_scatter)
+               dev->data->scattered_rx = 1;
+
+       ixgbe_set_rx_function(dev);
+
        /*
         * Device configured with multiple RX queues.
         */
@@ -3936,7 +3981,7 @@ ixgbe_dev_rx_queue_stop(struct rte_eth_dev *dev, uint16_t rx_queue_id)
                rte_delay_us(RTE_IXGBE_WAIT_100_US);
 
                ixgbe_rx_queue_release_mbufs(rxq);
-               ixgbe_reset_rx_queue(rxq);
+               ixgbe_reset_rx_queue(hw, rxq);
        } else
                return -1;
 
@@ -4292,3 +4337,34 @@ ixgbevf_dev_rxtx_start(struct rte_eth_dev *dev)
 
        }
 }
+
+/* Stubs needed for linkage when CONFIG_RTE_IXGBE_INC_VECTOR is set to 'n' */
+int __attribute__((weak))
+ixgbe_rx_vec_dev_conf_condition_check(struct rte_eth_dev __rte_unused *dev)
+{
+       return -1;
+}
+
+uint16_t __attribute__((weak))
+ixgbe_recv_pkts_vec(
+       void __rte_unused *rx_queue,
+       struct rte_mbuf __rte_unused **rx_pkts,
+       uint16_t __rte_unused nb_pkts)
+{
+       return 0;
+}
+
+uint16_t __attribute__((weak))
+ixgbe_recv_scattered_pkts_vec(
+       void __rte_unused *rx_queue,
+       struct rte_mbuf __rte_unused **rx_pkts,
+       uint16_t __rte_unused nb_pkts)
+{
+       return 0;
+}
+
+int __attribute__((weak))
+ixgbe_rxq_vec_setup(struct ixgbe_rx_queue __rte_unused *rxq)
+{
+       return -1;
+}
index 4cc1d6c..3937cf6 100644 (file)
@@ -255,16 +255,34 @@ struct ixgbe_txq_ops {
  */
 void ixgbe_set_tx_function(struct rte_eth_dev *dev, struct ixgbe_tx_queue *txq);
 
-#ifdef RTE_IXGBE_INC_VECTOR
+/**
+ * Sets the rx_pkt_burst callback in the ixgbe rte_eth_dev instance.
+ *
+ * Sets the callback based on the device parameters:
+ *  - ixgbe_hw.rx_bulk_alloc_allowed
+ *  - rte_eth_dev_data.scattered_rx
+ *  - rte_eth_dev_data.lro
+ *  - conditions checked in ixgbe_rx_vec_condition_check()
+ *
+ *  This means that the parameters above have to be configured prior to calling
+ *  to this function.
+ *
+ * @dev rte_eth_dev handle
+ */
+void ixgbe_set_rx_function(struct rte_eth_dev *dev);
+
 uint16_t ixgbe_recv_pkts_vec(void *rx_queue, struct rte_mbuf **rx_pkts,
                uint16_t nb_pkts);
 uint16_t ixgbe_recv_scattered_pkts_vec(void *rx_queue,
                struct rte_mbuf **rx_pkts, uint16_t nb_pkts);
+int ixgbe_rx_vec_dev_conf_condition_check(struct rte_eth_dev *dev);
+int ixgbe_rxq_vec_setup(struct ixgbe_rx_queue *rxq);
+
+#ifdef RTE_IXGBE_INC_VECTOR
+
 uint16_t ixgbe_xmit_pkts_vec(void *tx_queue, struct rte_mbuf **tx_pkts,
                uint16_t nb_pkts);
 int ixgbe_txq_vec_setup(struct ixgbe_tx_queue *txq);
-int ixgbe_rxq_vec_setup(struct ixgbe_rx_queue *rxq);
-int ixgbe_rx_vec_condition_check(struct rte_eth_dev *dev);
-#endif
 
-#endif
+#endif /* RTE_IXGBE_INC_VECTOR */
+#endif /* _IXGBE_RXTX_H_ */
index 9d8fa8d..7ac6b61 100644 (file)
@@ -760,7 +760,7 @@ int ixgbe_txq_vec_setup(struct ixgbe_tx_queue *txq)
        return 0;
 }
 
-int ixgbe_rx_vec_condition_check(struct rte_eth_dev *dev)
+int ixgbe_rx_vec_dev_conf_condition_check(struct rte_eth_dev *dev)
 {
 #ifndef RTE_LIBRTE_IEEE1588
        struct rte_eth_rxmode *rxmode = &dev->data->dev_conf.rxmode;