From c87d435a4d79739c0cec2ed280b94b41cb908af7 Mon Sep 17 00:00:00 2001 From: Konstantin Ananyev Date: Wed, 13 Oct 2021 14:37:01 +0100 Subject: [PATCH] ethdev: copy fast-path API into separate structure Copy public function pointers (rx_pkt_burst(), etc.) and related pointers to internal data from rte_eth_dev structure into a separate flat array. That array will remain in a public header. The intention here is to make rte_eth_dev and related structures internal. That should allow future possible changes to core eth_dev structures to be transparent to the user and help to avoid ABI/API breakages. The plan is to keep minimal part of data from rte_eth_dev public, so we still can use inline functions for fast-path calls (like rte_eth_rx_burst(), etc.) to avoid/minimize slowdown. The whole idea beyond this new schema: 1. PMDs keep to setup fast-path function pointers and related data inside rte_eth_dev struct in the same way they did it before. 2. Inside rte_eth_dev_start() and inside rte_eth_dev_probing_finish() (for secondary process) we call eth_dev_fp_ops_setup, which copies these function and data pointers into rte_eth_fp_ops[port_id]. 3. Inside rte_eth_dev_stop() and inside rte_eth_dev_release_port() we call eth_dev_fp_ops_reset(), which resets rte_eth_fp_ops[port_id] into some dummy values. 4. fast-path ethdev API (rte_eth_rx_burst(), etc.) will use that new flat array to call PMD specific functions. That approach should allow us to make rte_eth_devices[] private without introducing regression and help to avoid changes in drivers code. Signed-off-by: Konstantin Ananyev Reviewed-by: Andrew Rybchenko Reviewed-by: Ferruh Yigit Tested-by: Feifei Wang --- lib/ethdev/ethdev_private.c | 52 +++++++++++++++++++++++++++++++ lib/ethdev/ethdev_private.h | 7 +++++ lib/ethdev/rte_ethdev.c | 27 +++++++++++++++++ lib/ethdev/rte_ethdev_core.h | 59 ++++++++++++++++++++++++++++++++++++ 4 files changed, 145 insertions(+) diff --git a/lib/ethdev/ethdev_private.c b/lib/ethdev/ethdev_private.c index 012cf73ca2..d810c3a1d4 100644 --- a/lib/ethdev/ethdev_private.c +++ b/lib/ethdev/ethdev_private.c @@ -174,3 +174,55 @@ done: RTE_LOG(ERR, EAL, "wrong representor format: %s\n", str); return str == NULL ? -1 : 0; } + +static uint16_t +dummy_eth_rx_burst(__rte_unused void *rxq, + __rte_unused struct rte_mbuf **rx_pkts, + __rte_unused uint16_t nb_pkts) +{ + RTE_ETHDEV_LOG(ERR, "rx_pkt_burst for not ready port\n"); + rte_errno = ENOTSUP; + return 0; +} + +static uint16_t +dummy_eth_tx_burst(__rte_unused void *txq, + __rte_unused struct rte_mbuf **tx_pkts, + __rte_unused uint16_t nb_pkts) +{ + RTE_ETHDEV_LOG(ERR, "tx_pkt_burst for not ready port\n"); + rte_errno = ENOTSUP; + return 0; +} + +void +eth_dev_fp_ops_reset(struct rte_eth_fp_ops *fpo) +{ + static void *dummy_data[RTE_MAX_QUEUES_PER_PORT]; + static const struct rte_eth_fp_ops dummy_ops = { + .rx_pkt_burst = dummy_eth_rx_burst, + .tx_pkt_burst = dummy_eth_tx_burst, + .rxq = {.data = dummy_data, .clbk = dummy_data,}, + .txq = {.data = dummy_data, .clbk = dummy_data,}, + }; + + *fpo = dummy_ops; +} + +void +eth_dev_fp_ops_setup(struct rte_eth_fp_ops *fpo, + const struct rte_eth_dev *dev) +{ + fpo->rx_pkt_burst = dev->rx_pkt_burst; + fpo->tx_pkt_burst = dev->tx_pkt_burst; + fpo->tx_pkt_prepare = dev->tx_pkt_prepare; + fpo->rx_queue_count = dev->rx_queue_count; + fpo->rx_descriptor_status = dev->rx_descriptor_status; + fpo->tx_descriptor_status = dev->tx_descriptor_status; + + fpo->rxq.data = dev->data->rx_queues; + fpo->rxq.clbk = (void **)(uintptr_t)dev->post_rx_burst_cbs; + + fpo->txq.data = dev->data->tx_queues; + fpo->txq.clbk = (void **)(uintptr_t)dev->pre_tx_burst_cbs; +} diff --git a/lib/ethdev/ethdev_private.h b/lib/ethdev/ethdev_private.h index 3724429577..5721be7bdc 100644 --- a/lib/ethdev/ethdev_private.h +++ b/lib/ethdev/ethdev_private.h @@ -26,4 +26,11 @@ eth_find_device(const struct rte_eth_dev *_start, rte_eth_cmp_t cmp, /* Parse devargs value for representor parameter. */ int rte_eth_devargs_parse_representor_ports(char *str, void *data); +/* reset eth fast-path API to dummy values */ +void eth_dev_fp_ops_reset(struct rte_eth_fp_ops *fpo); + +/* setup eth fast-path API to ethdev values */ +void eth_dev_fp_ops_setup(struct rte_eth_fp_ops *fpo, + const struct rte_eth_dev *dev); + #endif /* _ETH_PRIVATE_H_ */ diff --git a/lib/ethdev/rte_ethdev.c b/lib/ethdev/rte_ethdev.c index e446f3b3f8..178f5b88b7 100644 --- a/lib/ethdev/rte_ethdev.c +++ b/lib/ethdev/rte_ethdev.c @@ -44,6 +44,9 @@ static const char *MZ_RTE_ETH_DEV_DATA = "rte_eth_dev_data"; struct rte_eth_dev rte_eth_devices[RTE_MAX_ETHPORTS]; +/* public fast-path API */ +struct rte_eth_fp_ops rte_eth_fp_ops[RTE_MAX_ETHPORTS]; + /* spinlock for eth device callbacks */ static rte_spinlock_t eth_dev_cb_lock = RTE_SPINLOCK_INITIALIZER; @@ -579,6 +582,8 @@ rte_eth_dev_release_port(struct rte_eth_dev *eth_dev) rte_eth_dev_callback_process(eth_dev, RTE_ETH_EVENT_DESTROY, NULL); + eth_dev_fp_ops_reset(rte_eth_fp_ops + eth_dev->data->port_id); + rte_spinlock_lock(ð_dev_shared_data->ownership_lock); eth_dev->state = RTE_ETH_DEV_UNUSED; @@ -1792,6 +1797,9 @@ rte_eth_dev_start(uint16_t port_id) (*dev->dev_ops->link_update)(dev, 0); } + /* expose selection of PMD fast-path functions */ + eth_dev_fp_ops_setup(rte_eth_fp_ops + port_id, dev); + rte_ethdev_trace_start(port_id); return 0; } @@ -1814,6 +1822,9 @@ rte_eth_dev_stop(uint16_t port_id) return 0; } + /* point fast-path functions to dummy ones */ + eth_dev_fp_ops_reset(rte_eth_fp_ops + port_id); + dev->data->dev_started = 0; ret = (*dev->dev_ops->dev_stop)(dev); rte_ethdev_trace_stop(port_id, ret); @@ -4477,6 +4488,14 @@ int rte_eth_set_queue_rate_limit(uint16_t port_id, uint16_t queue_idx, queue_idx, tx_rate)); } +RTE_INIT(eth_dev_init_fp_ops) +{ + uint32_t i; + + for (i = 0; i != RTE_DIM(rte_eth_fp_ops); i++) + eth_dev_fp_ops_reset(rte_eth_fp_ops + i); +} + RTE_INIT(eth_dev_init_cb_lists) { uint16_t i; @@ -4645,6 +4664,14 @@ rte_eth_dev_probing_finish(struct rte_eth_dev *dev) if (dev == NULL) return; + /* + * for secondary process, at that point we expect device + * to be already 'usable', so shared data and all function pointers + * for fast-path devops have to be setup properly inside rte_eth_dev. + */ + if (rte_eal_process_type() == RTE_PROC_SECONDARY) + eth_dev_fp_ops_setup(rte_eth_fp_ops + dev->data->port_id, dev); + rte_eth_dev_callback_process(dev, RTE_ETH_EVENT_NEW, NULL); dev->state = RTE_ETH_DEV_ATTACHED; diff --git a/lib/ethdev/rte_ethdev_core.h b/lib/ethdev/rte_ethdev_core.h index af824ef890..df7168ca4b 100644 --- a/lib/ethdev/rte_ethdev_core.h +++ b/lib/ethdev/rte_ethdev_core.h @@ -50,6 +50,65 @@ typedef int (*eth_rx_descriptor_status_t)(void *rxq, uint16_t offset); typedef int (*eth_tx_descriptor_status_t)(void *txq, uint16_t offset); /**< @internal Check the status of a Tx descriptor */ +/** + * @internal + * Structure used to hold opaque pointers to internal ethdev Rx/Tx + * queues data. + * The main purpose to expose these pointers at all - allow compiler + * to fetch this data for fast-path ethdev inline functions in advance. + */ +struct rte_ethdev_qdata { + /** points to array of internal queue data pointers */ + void **data; + /** points to array of queue callback data pointers */ + void **clbk; +}; + +/** + * @internal + * fast-path ethdev functions and related data are hold in a flat array. + * One entry per ethdev. + * On 64-bit systems contents of this structure occupy exactly two 64B lines. + * On 32-bit systems contents of this structure fits into one 64B line. + */ +struct rte_eth_fp_ops { + + /**@{*/ + /** + * Rx fast-path functions and related data. + * 64-bit systems: occupies first 64B line + */ + /** PMD receive function. */ + eth_rx_burst_t rx_pkt_burst; + /** Get the number of used RX descriptors. */ + eth_rx_queue_count_t rx_queue_count; + /** Check the status of a Rx descriptor. */ + eth_rx_descriptor_status_t rx_descriptor_status; + /** Rx queues data. */ + struct rte_ethdev_qdata rxq; + uintptr_t reserved1[3]; + /**@}*/ + + /**@{*/ + /** + * Tx fast-path functions and related data. + * 64-bit systems: occupies second 64B line + */ + /** PMD transmit function. */ + eth_tx_burst_t tx_pkt_burst; + /** PMD transmit prepare function. */ + eth_tx_prep_t tx_pkt_prepare; + /** Check the status of a Tx descriptor. */ + eth_tx_descriptor_status_t tx_descriptor_status; + /** Tx queues data. */ + struct rte_ethdev_qdata txq; + uintptr_t reserved2[3]; + /**@}*/ + +} __rte_cache_aligned; + +extern struct rte_eth_fp_ops rte_eth_fp_ops[RTE_MAX_ETHPORTS]; + /** * @internal -- 2.20.1