From af4f09f28294fac762ff413fbf14b48c42c128fd Mon Sep 17 00:00:00 2001 From: =?utf8?q?N=C3=A9lio=20Laranjeiro?= Date: Mon, 5 Mar 2018 13:21:04 +0100 Subject: [PATCH] net/mlx5: prefix all functions with mlx5 This change removes the need to distinguish unlocked priv_*() functions which are therefore renamed using a mlx5_*() prefix for consistency. At the same time, all functions from mlx5 uses a pointer to the ETH device instead of the one to the PMD private data. Signed-off-by: Nelio Laranjeiro Acked-by: Adrien Mazarguil --- drivers/net/mlx5/mlx5.c | 104 +++++----- drivers/net/mlx5/mlx5.h | 90 ++++---- drivers/net/mlx5/mlx5_ethdev.c | 288 +++++++++++--------------- drivers/net/mlx5/mlx5_flow.c | 342 +++++++++++++++---------------- drivers/net/mlx5/mlx5_mac.c | 8 +- drivers/net/mlx5/mlx5_mr.c | 86 +++----- drivers/net/mlx5/mlx5_rss.c | 107 +++------- drivers/net/mlx5/mlx5_rxq.c | 285 +++++++++++++------------- drivers/net/mlx5/mlx5_rxtx.c | 10 +- drivers/net/mlx5/mlx5_rxtx.h | 115 +++++------ drivers/net/mlx5/mlx5_rxtx_vec.c | 25 +-- drivers/net/mlx5/mlx5_socket.c | 29 +-- drivers/net/mlx5/mlx5_stats.c | 135 +++++------- drivers/net/mlx5/mlx5_trigger.c | 118 +++++------ drivers/net/mlx5/mlx5_txq.c | 134 ++++++------ drivers/net/mlx5/mlx5_vlan.c | 61 ++---- 16 files changed, 868 insertions(+), 1069 deletions(-) diff --git a/drivers/net/mlx5/mlx5.c b/drivers/net/mlx5/mlx5.c index 2fdec80efd..15b77224cf 100644 --- a/drivers/net/mlx5/mlx5.c +++ b/drivers/net/mlx5/mlx5.c @@ -169,8 +169,8 @@ mlx5_dev_close(struct rte_eth_dev *dev) (void *)dev, ((priv->ctx != NULL) ? priv->ctx->device->name : "")); /* In case mlx5_dev_stop() has not been called. */ - priv_dev_interrupt_handler_uninstall(priv, dev); - priv_dev_traffic_disable(priv, dev); + mlx5_dev_interrupt_handler_uninstall(dev); + mlx5_traffic_disable(dev); /* Prevent crashes when queues are still in use. */ dev->rx_pkt_burst = removed_rx_burst; dev->tx_pkt_burst = removed_tx_burst; @@ -178,7 +178,7 @@ mlx5_dev_close(struct rte_eth_dev *dev) /* XXX race condition if mlx5_rx_burst() is still running. */ usleep(1000); for (i = 0; (i != priv->rxqs_n); ++i) - mlx5_priv_rxq_release(priv, i); + mlx5_rxq_release(dev, i); priv->rxqs_n = 0; priv->rxqs = NULL; } @@ -186,7 +186,7 @@ mlx5_dev_close(struct rte_eth_dev *dev) /* XXX race condition if mlx5_tx_burst() is still running. */ usleep(1000); for (i = 0; (i != priv->txqs_n); ++i) - mlx5_priv_txq_release(priv, i); + mlx5_txq_release(dev, i); priv->txqs_n = 0; priv->txqs = NULL; } @@ -201,31 +201,31 @@ mlx5_dev_close(struct rte_eth_dev *dev) if (priv->reta_idx != NULL) rte_free(priv->reta_idx); if (priv->primary_socket) - priv_socket_uninit(priv); - ret = mlx5_priv_hrxq_ibv_verify(priv); + mlx5_socket_uninit(dev); + ret = mlx5_hrxq_ibv_verify(dev); if (ret) - WARN("%p: some Hash Rx queue still remain", (void *)priv); - ret = mlx5_priv_ind_table_ibv_verify(priv); + WARN("%p: some Hash Rx queue still remain", (void *)dev); + ret = mlx5_ind_table_ibv_verify(dev); if (ret) - WARN("%p: some Indirection table still remain", (void *)priv); - ret = mlx5_priv_rxq_ibv_verify(priv); + WARN("%p: some Indirection table still remain", (void *)dev); + ret = mlx5_rxq_ibv_verify(dev); if (ret) - WARN("%p: some Verbs Rx queue still remain", (void *)priv); - ret = mlx5_priv_rxq_verify(priv); + WARN("%p: some Verbs Rx queue still remain", (void *)dev); + ret = mlx5_rxq_verify(dev); if (ret) - WARN("%p: some Rx Queues still remain", (void *)priv); - ret = mlx5_priv_txq_ibv_verify(priv); + WARN("%p: some Rx Queues still remain", (void *)dev); + ret = mlx5_txq_ibv_verify(dev); if (ret) - WARN("%p: some Verbs Tx queue still remain", (void *)priv); - ret = mlx5_priv_txq_verify(priv); + WARN("%p: some Verbs Tx queue still remain", (void *)dev); + ret = mlx5_txq_verify(dev); if (ret) - WARN("%p: some Tx Queues still remain", (void *)priv); - ret = priv_flow_verify(priv); + WARN("%p: some Tx Queues still remain", (void *)dev); + ret = mlx5_flow_verify(dev); if (ret) - WARN("%p: some flows still remain", (void *)priv); - ret = priv_mr_verify(priv); + WARN("%p: some flows still remain", (void *)dev); + ret = mlx5_mr_verify(dev); if (ret) - WARN("%p: some Memory Region still remain", (void *)priv); + WARN("%p: some Memory Region still remain", (void *)dev); memset(priv, 0, sizeof(*priv)); } @@ -466,15 +466,16 @@ static void *uar_base; /** * Reserve UAR address space for primary process. * - * @param[in] priv - * Pointer to private structure. + * @param[in] dev + * Pointer to Ethernet device. * * @return * 0 on success, errno value on failure. */ static int -priv_uar_init_primary(struct priv *priv) +mlx5_uar_init_primary(struct rte_eth_dev *dev) { + struct priv *priv = dev->data->dev_private; void *addr = (void *)0; int i; const struct rte_mem_config *mcfg; @@ -516,15 +517,16 @@ priv_uar_init_primary(struct priv *priv) * Reserve UAR address space for secondary process, align with * primary process. * - * @param[in] priv - * Pointer to private structure. + * @param[in] dev + * Pointer to Ethernet device. * * @return * 0 on success, errno value on failure. */ static int -priv_uar_init_secondary(struct priv *priv) +mlx5_uar_init_secondary(struct rte_eth_dev *dev) { + struct priv *priv = dev->data->dev_private; void *addr; int ret; @@ -690,7 +692,7 @@ mlx5_pci_probe(struct rte_pci_driver *pci_drv __rte_unused, struct ibv_port_attr port_attr; struct ibv_pd *pd = NULL; struct priv *priv = NULL; - struct rte_eth_dev *eth_dev; + struct rte_eth_dev *eth_dev = NULL; struct ibv_device_attr_ex device_attr_ex; struct ether_addr mac; struct ibv_device_attr_ex device_attr; @@ -721,20 +723,19 @@ mlx5_pci_probe(struct rte_pci_driver *pci_drv __rte_unused, } eth_dev->device = &pci_dev->device; eth_dev->dev_ops = &mlx5_dev_sec_ops; - priv = eth_dev->data->dev_private; - err = priv_uar_init_secondary(priv); + err = mlx5_uar_init_secondary(eth_dev); if (err < 0) { err = -err; goto error; } /* Receive command fd from primary process */ - err = priv_socket_connect(priv); + err = mlx5_socket_connect(eth_dev); if (err < 0) { err = -err; goto error; } /* Remap UAR for Tx queues. */ - err = priv_tx_uar_remap(priv, err); + err = mlx5_tx_uar_remap(eth_dev, err); if (err) goto error; /* @@ -743,9 +744,9 @@ mlx5_pci_probe(struct rte_pci_driver *pci_drv __rte_unused, * secondary process. */ eth_dev->rx_pkt_burst = - priv_select_rx_function(priv, eth_dev); + mlx5_select_rx_function(eth_dev); eth_dev->tx_pkt_burst = - priv_select_tx_function(priv, eth_dev); + mlx5_select_tx_function(eth_dev); continue; } DEBUG("using port %u (%08" PRIx32 ")", port, test); @@ -859,11 +860,23 @@ mlx5_pci_probe(struct rte_pci_driver *pci_drv __rte_unused, WARN("Rx CQE compression isn't supported"); config.cqe_comp = 0; } - err = priv_uar_init_primary(priv); + eth_dev = rte_eth_dev_allocate(name); + if (eth_dev == NULL) { + ERROR("can not allocate rte ethdev"); + err = ENOMEM; + goto port_error; + } + eth_dev->data->dev_private = priv; + priv->dev = eth_dev; + eth_dev->data->mac_addrs = priv->mac; + eth_dev->device = &pci_dev->device; + rte_eth_copy_pci_info(eth_dev, pci_dev); + eth_dev->device->driver = &mlx5_driver.driver; + err = mlx5_uar_init_primary(eth_dev); if (err) goto port_error; /* Configure the first MAC address by default. */ - if (priv_get_mac(priv, &mac.addr_bytes)) { + if (mlx5_get_mac(eth_dev, &mac.addr_bytes)) { ERROR("cannot get MAC address, is mlx5_en loaded?" " (errno: %s)", strerror(errno)); err = ENODEV; @@ -878,7 +891,7 @@ mlx5_pci_probe(struct rte_pci_driver *pci_drv __rte_unused, { char ifname[IF_NAMESIZE]; - if (priv_get_ifname(priv, &ifname) == 0) + if (mlx5_get_ifname(eth_dev, &ifname) == 0) DEBUG("port %u ifname is \"%s\"", priv->port, ifname); else @@ -886,25 +899,13 @@ mlx5_pci_probe(struct rte_pci_driver *pci_drv __rte_unused, } #endif /* Get actual MTU if possible. */ - priv_get_mtu(priv, &priv->mtu); + mlx5_get_mtu(eth_dev, &priv->mtu); DEBUG("port %u MTU is %u", priv->port, priv->mtu); - eth_dev = rte_eth_dev_allocate(name); - if (eth_dev == NULL) { - ERROR("can not allocate rte ethdev"); - err = ENOMEM; - goto port_error; - } - eth_dev->data->dev_private = priv; - eth_dev->data->mac_addrs = priv->mac; - eth_dev->device = &pci_dev->device; - rte_eth_copy_pci_info(eth_dev, pci_dev); - eth_dev->device->driver = &mlx5_driver.driver; /* * Initialize burst functions to prevent crashes before link-up. */ eth_dev->rx_pkt_burst = removed_rx_burst; eth_dev->tx_pkt_burst = removed_tx_burst; - priv->dev = eth_dev; eth_dev->dev_ops = &mlx5_dev_ops; /* Register MAC address. */ claim_zero(mlx5_mac_addr_add(eth_dev, &mac, 0, 0)); @@ -919,10 +920,9 @@ mlx5_pci_probe(struct rte_pci_driver *pci_drv __rte_unused, mlx5_glue->dv_set_context_attr(ctx, MLX5DV_CTX_ATTR_BUF_ALLOCATORS, (void *)((uintptr_t)&alctr)); - /* Bring Ethernet device up. */ DEBUG("forcing Ethernet interface up"); - priv_set_flags(priv, ~IFF_UP, IFF_UP); + mlx5_set_flags(eth_dev, ~IFF_UP, IFF_UP); /* Store device configuration on private structure. */ priv->config = config; continue; diff --git a/drivers/net/mlx5/mlx5.h b/drivers/net/mlx5/mlx5.h index 8e021544ce..2cb463b622 100644 --- a/drivers/net/mlx5/mlx5.h +++ b/drivers/net/mlx5/mlx5.h @@ -163,18 +163,16 @@ int mlx5_getenv_int(const char *); /* mlx5_ethdev.c */ -struct priv *mlx5_get_priv(struct rte_eth_dev *dev); -int mlx5_is_secondary(void); -int priv_get_ifname(const struct priv *priv, char (*ifname)[IF_NAMESIZE]); -int priv_ifreq(const struct priv *priv, int req, struct ifreq *ifr); -int priv_get_mtu(struct priv *priv, uint16_t *mtu); -int priv_set_flags(struct priv *priv, unsigned int keep, unsigned int flags); +int mlx5_get_ifname(const struct rte_eth_dev *dev, char (*ifname)[IF_NAMESIZE]); +int mlx5_ifreq(const struct rte_eth_dev *dev, int req, struct ifreq *ifr); +int mlx5_get_mtu(struct rte_eth_dev *dev, uint16_t *mtu); +int mlx5_set_flags(struct rte_eth_dev *dev, unsigned int keep, + unsigned int flags); int mlx5_dev_configure(struct rte_eth_dev *dev); void mlx5_dev_infos_get(struct rte_eth_dev *dev, struct rte_eth_dev_info *info); const uint32_t *mlx5_dev_supported_ptypes_get(struct rte_eth_dev *dev); -int priv_link_update(struct priv *priv, int wait_to_complete); -int priv_force_link_status_change(struct priv *priv, int status); int mlx5_link_update(struct rte_eth_dev *dev, int wait_to_complete); +int mlx5_force_link_status_change(struct rte_eth_dev *dev, int status); int mlx5_dev_set_mtu(struct rte_eth_dev *dev, uint16_t mtu); int mlx5_dev_get_flow_ctrl(struct rte_eth_dev *dev, struct rte_eth_fc_conf *fc_conf); @@ -183,22 +181,18 @@ int mlx5_dev_set_flow_ctrl(struct rte_eth_dev *dev, int mlx5_ibv_device_to_pci_addr(const struct ibv_device *device, struct rte_pci_addr *pci_addr); void mlx5_dev_link_status_handler(void *arg); -void mlx5_dev_interrupt_handler(void *cb_arg); -void priv_dev_interrupt_handler_uninstall(struct priv *priv, - struct rte_eth_dev *dev); -void priv_dev_interrupt_handler_install(struct priv *priv, - struct rte_eth_dev *dev); +void mlx5_dev_interrupt_handler(void *arg); +void mlx5_dev_interrupt_handler_uninstall(struct rte_eth_dev *dev); +void mlx5_dev_interrupt_handler_install(struct rte_eth_dev *dev); int mlx5_set_link_down(struct rte_eth_dev *dev); int mlx5_set_link_up(struct rte_eth_dev *dev); -eth_tx_burst_t priv_select_tx_function(struct priv *priv, - struct rte_eth_dev *dev); -eth_rx_burst_t priv_select_rx_function(struct priv *priv, - struct rte_eth_dev *dev); int mlx5_is_removed(struct rte_eth_dev *dev); +eth_tx_burst_t mlx5_select_tx_function(struct rte_eth_dev *dev); +eth_rx_burst_t mlx5_select_rx_function(struct rte_eth_dev *dev); /* mlx5_mac.c */ -int priv_get_mac(struct priv *priv, uint8_t (*mac)[ETHER_ADDR_LEN]); +int mlx5_get_mac(struct rte_eth_dev *dev, uint8_t (*mac)[ETHER_ADDR_LEN]); void mlx5_mac_addr_remove(struct rte_eth_dev *dev, uint32_t index); int mlx5_mac_addr_add(struct rte_eth_dev *dev, struct ether_addr *mac, uint32_t index, uint32_t vmdq); @@ -210,7 +204,7 @@ int mlx5_rss_hash_update(struct rte_eth_dev *dev, struct rte_eth_rss_conf *rss_conf); int mlx5_rss_hash_conf_get(struct rte_eth_dev *dev, struct rte_eth_rss_conf *rss_conf); -int priv_rss_reta_index_resize(struct priv *priv, unsigned int reta_size); +int mlx5_rss_reta_index_resize(struct rte_eth_dev *dev, unsigned int reta_size); int mlx5_dev_rss_reta_query(struct rte_eth_dev *dev, struct rte_eth_rss_reta_entry64 *reta_conf, uint16_t reta_size); @@ -227,13 +221,13 @@ void mlx5_allmulticast_disable(struct rte_eth_dev *dev); /* mlx5_stats.c */ -void priv_xstats_init(struct priv *priv); +void mlx5_xstats_init(struct rte_eth_dev *dev); int mlx5_stats_get(struct rte_eth_dev *dev, struct rte_eth_stats *stats); void mlx5_stats_reset(struct rte_eth_dev *dev); -int mlx5_xstats_get(struct rte_eth_dev *dev, - struct rte_eth_xstat *stats, unsigned int n); +int mlx5_xstats_get(struct rte_eth_dev *dev, struct rte_eth_xstat *stats, + unsigned int n); void mlx5_xstats_reset(struct rte_eth_dev *dev); -int mlx5_xstats_get_names(struct rte_eth_dev *dev, +int mlx5_xstats_get_names(struct rte_eth_dev *dev __rte_unused, struct rte_eth_xstat_name *xstats_names, unsigned int n); @@ -247,9 +241,8 @@ int mlx5_vlan_offload_set(struct rte_eth_dev *dev, int mask); int mlx5_dev_start(struct rte_eth_dev *dev); void mlx5_dev_stop(struct rte_eth_dev *dev); -int priv_dev_traffic_enable(struct priv *priv, struct rte_eth_dev *dev); -int priv_dev_traffic_disable(struct priv *priv, struct rte_eth_dev *dev); -int priv_dev_traffic_restart(struct priv *priv, struct rte_eth_dev *dev); +int mlx5_traffic_enable(struct rte_eth_dev *dev); +int mlx5_traffic_disable(struct rte_eth_dev *dev); int mlx5_traffic_restart(struct rte_eth_dev *dev); /* mlx5_flow.c */ @@ -259,21 +252,6 @@ int mlx5_flow_validate(struct rte_eth_dev *dev, const struct rte_flow_item items[], const struct rte_flow_action actions[], struct rte_flow_error *error); -void priv_flow_flush(struct priv *priv, struct mlx5_flows *list); -int priv_flow_create_drop_queue(struct priv *priv); -void priv_flow_stop(struct priv *priv, struct mlx5_flows *list); -int priv_flow_start(struct priv *priv, struct mlx5_flows *list); -int priv_flow_verify(struct priv *priv); -int priv_flow_create_drop_queue(struct priv *priv); -void priv_flow_delete_drop_queue(struct priv *priv); -int mlx5_ctrl_flow_vlan(struct rte_eth_dev *dev, - struct rte_flow_item_eth *eth_spec, - struct rte_flow_item_eth *eth_mask, - struct rte_flow_item_vlan *vlan_spec, - struct rte_flow_item_vlan *vlan_mask); -int mlx5_ctrl_flow(struct rte_eth_dev *dev, - struct rte_flow_item_eth *eth_spec, - struct rte_flow_item_eth *eth_mask); struct rte_flow *mlx5_flow_create(struct rte_eth_dev *dev, const struct rte_flow_attr *attr, const struct rte_flow_item items[], @@ -281,6 +259,7 @@ struct rte_flow *mlx5_flow_create(struct rte_eth_dev *dev, struct rte_flow_error *error); int mlx5_flow_destroy(struct rte_eth_dev *dev, struct rte_flow *flow, struct rte_flow_error *error); +void mlx5_flow_list_flush(struct rte_eth_dev *dev, struct mlx5_flows *list); int mlx5_flow_flush(struct rte_eth_dev *dev, struct rte_flow_error *error); int mlx5_flow_query(struct rte_eth_dev *dev, struct rte_flow *flow, enum rte_flow_action_type action, void *data, @@ -291,19 +270,32 @@ int mlx5_dev_filter_ctrl(struct rte_eth_dev *dev, enum rte_filter_type filter_type, enum rte_filter_op filter_op, void *arg); +int mlx5_flow_start(struct rte_eth_dev *dev, struct mlx5_flows *list); +void mlx5_flow_stop(struct rte_eth_dev *dev, struct mlx5_flows *list); +int mlx5_flow_verify(struct rte_eth_dev *dev); +int mlx5_ctrl_flow_vlan(struct rte_eth_dev *dev, + struct rte_flow_item_eth *eth_spec, + struct rte_flow_item_eth *eth_mask, + struct rte_flow_item_vlan *vlan_spec, + struct rte_flow_item_vlan *vlan_mask); +int mlx5_ctrl_flow(struct rte_eth_dev *dev, + struct rte_flow_item_eth *eth_spec, + struct rte_flow_item_eth *eth_mask); +int mlx5_flow_create_drop_queue(struct rte_eth_dev *dev); +void mlx5_flow_delete_drop_queue(struct rte_eth_dev *dev); /* mlx5_socket.c */ -int priv_socket_init(struct priv *priv); -int priv_socket_uninit(struct priv *priv); -void priv_socket_handle(struct priv *priv); -int priv_socket_connect(struct priv *priv); +int mlx5_socket_init(struct rte_eth_dev *priv); +int mlx5_socket_uninit(struct rte_eth_dev *priv); +void mlx5_socket_handle(struct rte_eth_dev *priv); +int mlx5_socket_connect(struct rte_eth_dev *priv); /* mlx5_mr.c */ -struct mlx5_mr *priv_mr_new(struct priv *priv, struct rte_mempool *mp); -struct mlx5_mr *priv_mr_get(struct priv *priv, struct rte_mempool *mp); -int priv_mr_release(struct priv *priv, struct mlx5_mr *mr); -int priv_mr_verify(struct priv *priv); +struct mlx5_mr *mlx5_mr_new(struct rte_eth_dev *dev, struct rte_mempool *mp); +struct mlx5_mr *mlx5_mr_get(struct rte_eth_dev *dev, struct rte_mempool *mp); +int mlx5_mr_release(struct mlx5_mr *mr); +int mlx5_mr_verify(struct rte_eth_dev *dev); #endif /* RTE_PMD_MLX5_H_ */ diff --git a/drivers/net/mlx5/mlx5_ethdev.c b/drivers/net/mlx5/mlx5_ethdev.c index f0defc69d1..1fde3d842e 100644 --- a/drivers/net/mlx5/mlx5_ethdev.c +++ b/drivers/net/mlx5/mlx5_ethdev.c @@ -95,8 +95,8 @@ struct ethtool_link_settings { /** * Get interface name from private structure. * - * @param[in] priv - * Pointer to private structure. + * @param[in] dev + * Pointer to Ethernet device. * @param[out] ifname * Interface name output buffer. * @@ -104,8 +104,9 @@ struct ethtool_link_settings { * 0 on success, -1 on failure and errno is set. */ int -priv_get_ifname(const struct priv *priv, char (*ifname)[IF_NAMESIZE]) +mlx5_get_ifname(const struct rte_eth_dev *dev, char (*ifname)[IF_NAMESIZE]) { + struct priv *priv = dev->data->dev_private; DIR *dir; struct dirent *dent; unsigned int dev_type = 0; @@ -176,8 +177,8 @@ try_dev_id: /** * Perform ifreq ioctl() on associated Ethernet device. * - * @param[in] priv - * Pointer to private structure. + * @param[in] dev + * Pointer to Ethernet device. * @param req * Request number to pass to ioctl(). * @param[out] ifr @@ -187,14 +188,14 @@ try_dev_id: * 0 on success, -1 on failure and errno is set. */ int -priv_ifreq(const struct priv *priv, int req, struct ifreq *ifr) +mlx5_ifreq(const struct rte_eth_dev *dev, int req, struct ifreq *ifr) { int sock = socket(PF_INET, SOCK_DGRAM, IPPROTO_IP); int ret = -1; if (sock == -1) return ret; - if (priv_get_ifname(priv, &ifr->ifr_name) == 0) + if (mlx5_get_ifname(dev, &ifr->ifr_name) == 0) ret = ioctl(sock, req, ifr); close(sock); return ret; @@ -203,8 +204,8 @@ priv_ifreq(const struct priv *priv, int req, struct ifreq *ifr) /** * Get device MTU. * - * @param priv - * Pointer to private structure. + * @param dev + * Pointer to Ethernet device. * @param[out] mtu * MTU value output buffer. * @@ -212,10 +213,10 @@ priv_ifreq(const struct priv *priv, int req, struct ifreq *ifr) * 0 on success, -1 on failure and errno is set. */ int -priv_get_mtu(struct priv *priv, uint16_t *mtu) +mlx5_get_mtu(struct rte_eth_dev *dev, uint16_t *mtu) { struct ifreq request; - int ret = priv_ifreq(priv, SIOCGIFMTU, &request); + int ret = mlx5_ifreq(dev, SIOCGIFMTU, &request); if (ret) return ret; @@ -226,8 +227,8 @@ priv_get_mtu(struct priv *priv, uint16_t *mtu) /** * Set device MTU. * - * @param priv - * Pointer to private structure. + * @param dev + * Pointer to Ethernet device. * @param mtu * MTU value to set. * @@ -235,18 +236,18 @@ priv_get_mtu(struct priv *priv, uint16_t *mtu) * 0 on success, -1 on failure and errno is set. */ static int -priv_set_mtu(struct priv *priv, uint16_t mtu) +mlx5_set_mtu(struct rte_eth_dev *dev, uint16_t mtu) { struct ifreq request = { .ifr_mtu = mtu, }; - return priv_ifreq(priv, SIOCSIFMTU, &request); + return mlx5_ifreq(dev, SIOCSIFMTU, &request); } /** * Set device flags. * - * @param priv - * Pointer to private structure. + * @param dev + * Pointer to Ethernet device. * @param keep * Bitmask for flags that must remain untouched. * @param flags @@ -256,16 +257,16 @@ priv_set_mtu(struct priv *priv, uint16_t mtu) * 0 on success, -1 on failure and errno is set. */ int -priv_set_flags(struct priv *priv, unsigned int keep, unsigned int flags) +mlx5_set_flags(struct rte_eth_dev *dev, unsigned int keep, unsigned int flags) { struct ifreq request; - int ret = priv_ifreq(priv, SIOCGIFFLAGS, &request); + int ret = mlx5_ifreq(dev, SIOCGIFFLAGS, &request); if (ret) return ret; request.ifr_flags &= keep; request.ifr_flags |= flags & ~keep; - return priv_ifreq(priv, SIOCSIFFLAGS, &request); + return mlx5_ifreq(dev, SIOCSIFFLAGS, &request); } /** @@ -288,11 +289,11 @@ mlx5_dev_configure(struct rte_eth_dev *dev) unsigned int reta_idx_n; const uint8_t use_app_rss_key = !!dev->data->dev_conf.rx_adv_conf.rss_conf.rss_key; - uint64_t supp_tx_offloads = mlx5_priv_get_tx_port_offloads(priv); + uint64_t supp_tx_offloads = mlx5_get_tx_port_offloads(dev); uint64_t tx_offloads = dev->data->dev_conf.txmode.offloads; uint64_t supp_rx_offloads = - (mlx5_priv_get_rx_port_offloads(priv) | - mlx5_priv_get_rx_queue_offloads(priv)); + (mlx5_get_rx_port_offloads() | + mlx5_get_rx_queue_offloads(dev)); uint64_t rx_offloads = dev->data->dev_conf.rxmode.offloads; if ((tx_offloads & supp_tx_offloads) != tx_offloads) { @@ -349,7 +350,7 @@ mlx5_dev_configure(struct rte_eth_dev *dev) reta_idx_n = (1 << log2above((rxqs_n & (rxqs_n - 1)) ? priv->config.ind_table_max_size : rxqs_n)); - if (priv_rss_reta_index_resize(priv, reta_idx_n)) + if (mlx5_rss_reta_index_resize(dev, reta_idx_n)) return ENOMEM; /* When the number of RX queues is not a power of two, the remaining * table entries are padded with reused WQs and hashes are not spread @@ -395,12 +396,11 @@ mlx5_dev_infos_get(struct rte_eth_dev *dev, struct rte_eth_dev_info *info) info->max_rx_queues = max; info->max_tx_queues = max; info->max_mac_addrs = RTE_DIM(priv->mac); - info->rx_queue_offload_capa = - mlx5_priv_get_rx_queue_offloads(priv); - info->rx_offload_capa = (mlx5_priv_get_rx_port_offloads(priv) | + info->rx_queue_offload_capa = mlx5_get_rx_queue_offloads(dev); + info->rx_offload_capa = (mlx5_get_rx_port_offloads() | info->rx_queue_offload_capa); - info->tx_offload_capa = mlx5_priv_get_tx_port_offloads(priv); - if (priv_get_ifname(priv, &ifname) == 0) + info->tx_offload_capa = mlx5_get_tx_port_offloads(dev); + if (mlx5_get_ifname(dev, &ifname) == 0) info->if_index = if_nametoindex(ifname); info->reta_size = priv->reta_idx_n ? priv->reta_idx_n : config->ind_table_max_size; @@ -465,7 +465,7 @@ mlx5_link_update_unlocked_gset(struct rte_eth_dev *dev) struct rte_eth_link dev_link; int link_speed = 0; - if (priv_ifreq(priv, SIOCGIFFLAGS, &ifr)) { + if (mlx5_ifreq(dev, SIOCGIFFLAGS, &ifr)) { WARN("ioctl(SIOCGIFFLAGS) failed: %s", strerror(errno)); return -1; } @@ -473,7 +473,7 @@ mlx5_link_update_unlocked_gset(struct rte_eth_dev *dev) dev_link.link_status = ((ifr.ifr_flags & IFF_UP) && (ifr.ifr_flags & IFF_RUNNING)); ifr.ifr_data = (void *)&edata; - if (priv_ifreq(priv, SIOCETHTOOL, &ifr)) { + if (mlx5_ifreq(dev, SIOCETHTOOL, &ifr)) { WARN("ioctl(SIOCETHTOOL, ETHTOOL_GSET) failed: %s", strerror(errno)); return -1; @@ -527,7 +527,7 @@ mlx5_link_update_unlocked_gs(struct rte_eth_dev *dev) struct rte_eth_link dev_link; uint64_t sc; - if (priv_ifreq(priv, SIOCGIFFLAGS, &ifr)) { + if (mlx5_ifreq(dev, SIOCGIFFLAGS, &ifr)) { WARN("ioctl(SIOCGIFFLAGS) failed: %s", strerror(errno)); return -1; } @@ -535,7 +535,7 @@ mlx5_link_update_unlocked_gs(struct rte_eth_dev *dev) dev_link.link_status = ((ifr.ifr_flags & IFF_UP) && (ifr.ifr_flags & IFF_RUNNING)); ifr.ifr_data = (void *)&gcmd; - if (priv_ifreq(priv, SIOCETHTOOL, &ifr)) { + if (mlx5_ifreq(dev, SIOCETHTOOL, &ifr)) { DEBUG("ioctl(SIOCETHTOOL, ETHTOOL_GLINKSETTINGS) failed: %s", strerror(errno)); return -1; @@ -549,7 +549,7 @@ mlx5_link_update_unlocked_gs(struct rte_eth_dev *dev) *ecmd = gcmd; ifr.ifr_data = (void *)ecmd; - if (priv_ifreq(priv, SIOCETHTOOL, &ifr)) { + if (mlx5_ifreq(dev, SIOCETHTOOL, &ifr)) { DEBUG("ioctl(SIOCETHTOOL, ETHTOOL_GLINKSETTINGS) failed: %s", strerror(errno)); return -1; @@ -608,90 +608,50 @@ mlx5_link_update_unlocked_gs(struct rte_eth_dev *dev) /** * Enable receiving and transmitting traffic. * - * @param priv - * Pointer to private structure. + * @param dev + * Pointer to Ethernet device. */ static void -priv_link_start(struct priv *priv) +mlx5_link_start(struct rte_eth_dev *dev) { - struct rte_eth_dev *dev = priv->dev; + struct priv *priv = dev->data->dev_private; int err; - dev->tx_pkt_burst = priv_select_tx_function(priv, dev); - dev->rx_pkt_burst = priv_select_rx_function(priv, dev); - err = priv_dev_traffic_enable(priv, dev); + dev->tx_pkt_burst = mlx5_select_tx_function(dev); + dev->rx_pkt_burst = mlx5_select_rx_function(dev); + err = mlx5_traffic_enable(dev); if (err) ERROR("%p: error occurred while configuring control flows: %s", - (void *)priv, strerror(err)); - err = priv_flow_start(priv, &priv->flows); + (void *)dev, strerror(err)); + err = mlx5_flow_start(dev, &priv->flows); if (err) ERROR("%p: error occurred while configuring flows: %s", - (void *)priv, strerror(err)); + (void *)dev, strerror(err)); } /** * Disable receiving and transmitting traffic. * - * @param priv - * Pointer to private structure. + * @param dev + * Pointer to Ethernet device. */ static void -priv_link_stop(struct priv *priv) +mlx5_link_stop(struct rte_eth_dev *dev) { - struct rte_eth_dev *dev = priv->dev; + struct priv *priv = dev->data->dev_private; - priv_flow_stop(priv, &priv->flows); - priv_dev_traffic_disable(priv, dev); + mlx5_flow_stop(dev, &priv->flows); + mlx5_traffic_disable(dev); dev->rx_pkt_burst = removed_rx_burst; dev->tx_pkt_burst = removed_tx_burst; } -/** - * Retrieve physical link information and update rx/tx_pkt_burst callbacks - * accordingly. - * - * @param priv - * Pointer to private structure. - * @param wait_to_complete - * Wait for request completion (ignored). - */ -int -priv_link_update(struct priv *priv, int wait_to_complete __rte_unused) -{ - struct rte_eth_dev *dev = priv->dev; - struct utsname utsname; - int ver[3]; - int ret; - struct rte_eth_link dev_link = dev->data->dev_link; - - if (uname(&utsname) == -1 || - sscanf(utsname.release, "%d.%d.%d", - &ver[0], &ver[1], &ver[2]) != 3 || - KERNEL_VERSION(ver[0], ver[1], ver[2]) < KERNEL_VERSION(4, 9, 0)) - ret = mlx5_link_update_unlocked_gset(dev); - else - ret = mlx5_link_update_unlocked_gs(dev); - /* If lsc interrupt is disabled, should always be ready for traffic. */ - if (!dev->data->dev_conf.intr_conf.lsc) { - priv_link_start(priv); - return ret; - } - /* Re-select burst callbacks only if link status has been changed. */ - if (!ret && dev_link.link_status != dev->data->dev_link.link_status) { - if (dev->data->dev_link.link_status == ETH_LINK_UP) - priv_link_start(priv); - else - priv_link_stop(priv); - } - return ret; -} - /** * Querying the link status till it changes to the desired state. * Number of query attempts is bounded by MLX5_MAX_LINK_QUERY_ATTEMPTS. * - * @param priv - * Pointer to private structure. + * @param dev + * Pointer to Ethernet device. * @param status * Link desired status. * @@ -699,13 +659,13 @@ priv_link_update(struct priv *priv, int wait_to_complete __rte_unused) * 0 on success, negative errno value on failure. */ int -priv_force_link_status_change(struct priv *priv, int status) +mlx5_force_link_status_change(struct rte_eth_dev *dev, int status) { int try = 0; while (try < MLX5_MAX_LINK_QUERY_ATTEMPTS) { - priv_link_update(priv, 0); - if (priv->dev->data->dev_link.link_status == status) + mlx5_link_update(dev, 0); + if (dev->data->dev_link.link_status == status) return 0; try++; sleep(1); @@ -727,10 +687,30 @@ priv_force_link_status_change(struct priv *priv, int status) int mlx5_link_update(struct rte_eth_dev *dev, int wait_to_complete __rte_unused) { - struct priv *priv = dev->data->dev_private; + struct utsname utsname; + int ver[3]; int ret; + struct rte_eth_link dev_link = dev->data->dev_link; - ret = priv_link_update(priv, wait_to_complete); + if (uname(&utsname) == -1 || + sscanf(utsname.release, "%d.%d.%d", + &ver[0], &ver[1], &ver[2]) != 3 || + KERNEL_VERSION(ver[0], ver[1], ver[2]) < KERNEL_VERSION(4, 9, 0)) + ret = mlx5_link_update_unlocked_gset(dev); + else + ret = mlx5_link_update_unlocked_gs(dev); + /* If lsc interrupt is disabled, should always be ready for traffic. */ + if (!dev->data->dev_conf.intr_conf.lsc) { + mlx5_link_start(dev); + return ret; + } + /* Re-select burst callbacks only if link status has been changed. */ + if (!ret && dev_link.link_status != dev->data->dev_link.link_status) { + if (dev->data->dev_link.link_status == ETH_LINK_UP) + mlx5_link_start(dev); + else + mlx5_link_stop(dev); + } return ret; } @@ -752,14 +732,14 @@ mlx5_dev_set_mtu(struct rte_eth_dev *dev, uint16_t mtu) uint16_t kern_mtu; int ret = 0; - ret = priv_get_mtu(priv, &kern_mtu); + ret = mlx5_get_mtu(dev, &kern_mtu); if (ret) goto out; /* Set kernel interface MTU first. */ - ret = priv_set_mtu(priv, mtu); + ret = mlx5_set_mtu(dev, mtu); if (ret) goto out; - ret = priv_get_mtu(priv, &kern_mtu); + ret = mlx5_get_mtu(dev, &kern_mtu); if (ret) goto out; if (kern_mtu == mtu) { @@ -789,7 +769,6 @@ out: int mlx5_dev_get_flow_ctrl(struct rte_eth_dev *dev, struct rte_eth_fc_conf *fc_conf) { - struct priv *priv = dev->data->dev_private; struct ifreq ifr; struct ethtool_pauseparam ethpause = { .cmd = ETHTOOL_GPAUSEPARAM @@ -797,10 +776,9 @@ mlx5_dev_get_flow_ctrl(struct rte_eth_dev *dev, struct rte_eth_fc_conf *fc_conf) int ret; ifr.ifr_data = (void *)ðpause; - if (priv_ifreq(priv, SIOCETHTOOL, &ifr)) { + if (mlx5_ifreq(dev, SIOCETHTOOL, &ifr)) { ret = errno; - WARN("ioctl(SIOCETHTOOL, ETHTOOL_GPAUSEPARAM)" - " failed: %s", + WARN("ioctl(SIOCETHTOOL, ETHTOOL_GPAUSEPARAM) failed: %s", strerror(ret)); goto out; } @@ -833,7 +811,6 @@ out: int mlx5_dev_set_flow_ctrl(struct rte_eth_dev *dev, struct rte_eth_fc_conf *fc_conf) { - struct priv *priv = dev->data->dev_private; struct ifreq ifr; struct ethtool_pauseparam ethpause = { .cmd = ETHTOOL_SPAUSEPARAM @@ -853,7 +830,7 @@ mlx5_dev_set_flow_ctrl(struct rte_eth_dev *dev, struct rte_eth_fc_conf *fc_conf) ethpause.tx_pause = 1; else ethpause.tx_pause = 0; - if (priv_ifreq(priv, SIOCETHTOOL, &ifr)) { + if (mlx5_ifreq(dev, SIOCETHTOOL, &ifr)) { ret = errno; WARN("ioctl(SIOCETHTOOL, ETHTOOL_SPAUSEPARAM)" " failed: %s", @@ -919,18 +896,19 @@ mlx5_ibv_device_to_pci_addr(const struct ibv_device *device, /** * Update the link status. * - * @param priv - * Pointer to private structure. + * @param dev + * Pointer to Ethernet device. * * @return * Zero if the callback process can be called immediately. */ static int -priv_link_status_update(struct priv *priv) +mlx5_link_status_update(struct rte_eth_dev *dev) { - struct rte_eth_link *link = &priv->dev->data->dev_link; + struct priv *priv = dev->data->dev_private; + struct rte_eth_link *link = &dev->data->dev_link; - priv_link_update(priv, 0); + mlx5_link_update(dev, 0); if (((link->link_speed == 0) && link->link_status) || ((link->link_speed != 0) && !link->link_status)) { /* @@ -955,8 +933,8 @@ priv_link_status_update(struct priv *priv) /** * Device status handler. * - * @param priv - * Pointer to private structure. + * @param dev + * Pointer to Ethernet device. * @param events * Pointer to event flags holder. * @@ -964,8 +942,9 @@ priv_link_status_update(struct priv *priv) * Events bitmap of callback process which can be called immediately. */ static uint32_t -priv_dev_status_handler(struct priv *priv) +mlx5_dev_status_handler(struct rte_eth_dev *dev) { + struct priv *priv = dev->data->dev_private; struct ibv_async_event event; uint32_t ret = 0; @@ -975,10 +954,10 @@ priv_dev_status_handler(struct priv *priv) break; if ((event.event_type == IBV_EVENT_PORT_ACTIVE || event.event_type == IBV_EVENT_PORT_ERR) && - (priv->dev->data->dev_conf.intr_conf.lsc == 1)) + (dev->data->dev_conf.intr_conf.lsc == 1)) ret |= (1 << RTE_ETH_EVENT_INTR_LSC); else if (event.event_type == IBV_EVENT_DEVICE_FATAL && - priv->dev->data->dev_conf.intr_conf.rmv == 1) + dev->data->dev_conf.intr_conf.rmv == 1) ret |= (1 << RTE_ETH_EVENT_INTR_RMV); else DEBUG("event type %d on port %d not handled", @@ -986,7 +965,7 @@ priv_dev_status_handler(struct priv *priv) mlx5_glue->ack_async_event(&event); } if (ret & (1 << RTE_ETH_EVENT_INTR_LSC)) - if (priv_link_status_update(priv)) + if (mlx5_link_status_update(dev)) ret &= ~(1 << RTE_ETH_EVENT_INTR_LSC); return ret; } @@ -1005,7 +984,7 @@ mlx5_dev_link_status_handler(void *arg) int ret; priv->pending_alarm = 0; - ret = priv_link_status_update(priv); + ret = mlx5_link_status_update(dev); if (!ret) _rte_eth_dev_callback_process(dev, RTE_ETH_EVENT_INTR_LSC, NULL); } @@ -1022,10 +1001,9 @@ void mlx5_dev_interrupt_handler(void *cb_arg) { struct rte_eth_dev *dev = cb_arg; - struct priv *priv = dev->data->dev_private; uint32_t events; - events = priv_dev_status_handler(priv); + events = mlx5_dev_status_handler(dev); if (events & (1 << RTE_ETH_EVENT_INTR_LSC)) _rte_eth_dev_callback_process(dev, RTE_ETH_EVENT_INTR_LSC, NULL); if (events & (1 << RTE_ETH_EVENT_INTR_RMV)) @@ -1042,22 +1020,21 @@ static void mlx5_dev_handler_socket(void *cb_arg) { struct rte_eth_dev *dev = cb_arg; - struct priv *priv = dev->data->dev_private; - priv_socket_handle(priv); + mlx5_socket_handle(dev); } /** * Uninstall interrupt handler. * - * @param priv - * Pointer to private structure. * @param dev - * Pointer to the rte_eth_dev structure. + * Pointer to Ethernet device. */ void -priv_dev_interrupt_handler_uninstall(struct priv *priv, struct rte_eth_dev *dev) +mlx5_dev_interrupt_handler_uninstall(struct rte_eth_dev *dev) { + struct priv *priv = dev->data->dev_private; + if (dev->data->dev_conf.intr_conf.lsc || dev->data->dev_conf.intr_conf.rmv) rte_intr_callback_unregister(&priv->intr_handle, @@ -1078,14 +1055,13 @@ priv_dev_interrupt_handler_uninstall(struct priv *priv, struct rte_eth_dev *dev) /** * Install interrupt handler. * - * @param priv - * Pointer to private structure. * @param dev - * Pointer to the rte_eth_dev structure. + * Pointer to Ethernet device. */ void -priv_dev_interrupt_handler_install(struct priv *priv, struct rte_eth_dev *dev) +mlx5_dev_interrupt_handler_install(struct rte_eth_dev *dev) { + struct priv *priv = dev->data->dev_private; int rc, flags; assert(priv->ctx->async_fd > 0); @@ -1103,7 +1079,7 @@ priv_dev_interrupt_handler_install(struct priv *priv, struct rte_eth_dev *dev) rte_intr_callback_register(&priv->intr_handle, mlx5_dev_interrupt_handler, dev); } - rc = priv_socket_init(priv); + rc = mlx5_socket_init(dev); if (!rc && priv->primary_socket) { priv->intr_handle_socket.fd = priv->primary_socket; priv->intr_handle_socket.type = RTE_INTR_HANDLE_EXT; @@ -1112,23 +1088,6 @@ priv_dev_interrupt_handler_install(struct priv *priv, struct rte_eth_dev *dev) } } -/** - * Change the link state (UP / DOWN). - * - * @param priv - * Pointer to private data structure. - * @param up - * Nonzero for link up, otherwise link down. - * - * @return - * 0 on success, errno value on failure. - */ -static int -priv_dev_set_link(struct priv *priv, int up) -{ - return priv_set_flags(priv, ~IFF_UP, up ? IFF_UP : ~IFF_UP); -} - /** * DPDK callback to bring the link DOWN. * @@ -1141,11 +1100,7 @@ priv_dev_set_link(struct priv *priv, int up) int mlx5_set_link_down(struct rte_eth_dev *dev) { - struct priv *priv = dev->data->dev_private; - int err; - - err = priv_dev_set_link(priv, 0); - return err; + return mlx5_set_flags(dev, ~IFF_UP, ~IFF_UP); } /** @@ -1160,27 +1115,22 @@ mlx5_set_link_down(struct rte_eth_dev *dev) int mlx5_set_link_up(struct rte_eth_dev *dev) { - struct priv *priv = dev->data->dev_private; - int err; - - err = priv_dev_set_link(priv, 1); - return err; + return mlx5_set_flags(dev, ~IFF_UP, IFF_UP); } /** * Configure the TX function to use. * - * @param priv - * Pointer to private data structure. * @param dev - * Pointer to rte_eth_dev structure. + * Pointer to private data structure. * * @return * Pointer to selected Tx burst function. */ eth_tx_burst_t -priv_select_tx_function(struct priv *priv, struct rte_eth_dev *dev) +mlx5_select_tx_function(struct rte_eth_dev *dev) { + struct priv *priv = dev->data->dev_private; eth_tx_burst_t tx_pkt_burst = mlx5_tx_burst; struct mlx5_dev_config *config = &priv->config; uint64_t tx_offloads = dev->data->dev_conf.txmode.offloads; @@ -1194,8 +1144,8 @@ priv_select_tx_function(struct priv *priv, struct rte_eth_dev *dev) if (vlan_insert || tso) return tx_pkt_burst; if (config->mps == MLX5_MPW_ENHANCED) { - if (priv_check_vec_tx_support(priv, dev) > 0) { - if (priv_check_raw_vec_tx_support(priv, dev) > 0) + if (mlx5_check_vec_tx_support(dev) > 0) { + if (mlx5_check_raw_vec_tx_support(dev) > 0) tx_pkt_burst = mlx5_tx_burst_raw_vec; else tx_pkt_burst = mlx5_tx_burst_vec; @@ -1217,21 +1167,19 @@ priv_select_tx_function(struct priv *priv, struct rte_eth_dev *dev) /** * Configure the RX function to use. * - * @param priv - * Pointer to private data structure. * @param dev - * Pointer to rte_eth_dev structure. + * Pointer to private data structure. * * @return * Pointer to selected Rx burst function. */ eth_rx_burst_t -priv_select_rx_function(struct priv *priv, __rte_unused struct rte_eth_dev *dev) +mlx5_select_rx_function(struct rte_eth_dev *dev) { eth_rx_burst_t rx_pkt_burst = mlx5_rx_burst; - assert(priv != NULL); - if (priv_check_vec_rx_support(priv) > 0) { + assert(dev != NULL); + if (mlx5_check_vec_rx_support(dev) > 0) { rx_pkt_burst = mlx5_rx_burst_vec; DEBUG("selected RX vectorized function"); } diff --git a/drivers/net/mlx5/mlx5_flow.c b/drivers/net/mlx5/mlx5_flow.c index 2af0ea9def..9ffcd0308d 100644 --- a/drivers/net/mlx5/mlx5_flow.c +++ b/drivers/net/mlx5/mlx5_flow.c @@ -89,7 +89,7 @@ static int mlx5_flow_create_flag_mark(struct mlx5_flow_parse *parser, uint32_t mark_id); static int -mlx5_flow_create_count(struct priv *priv, struct mlx5_flow_parse *parser); +mlx5_flow_create_count(struct rte_eth_dev *dev, struct mlx5_flow_parse *parser); /* Hash RX queue types. */ enum hash_rxq_type { @@ -515,8 +515,6 @@ mlx5_flow_item_validate(const struct rte_flow_item *item, * Copy the RSS configuration from the user ones, of the rss_conf is null, * uses the driver one. * - * @param priv - * Pointer to private structure. * @param parser * Internal parser structure. * @param rss_conf @@ -526,13 +524,12 @@ mlx5_flow_item_validate(const struct rte_flow_item *item, * 0 on success, errno value on failure. */ static int -priv_flow_convert_rss_conf(struct priv *priv __rte_unused, - struct mlx5_flow_parse *parser, +mlx5_flow_convert_rss_conf(struct mlx5_flow_parse *parser, const struct rte_eth_rss_conf *rss_conf) { /* * This function is also called at the beginning of - * priv_flow_convert_actions() to initialize the parser with the + * mlx5_flow_convert_actions() to initialize the parser with the * device default RSS configuration. */ if (rss_conf) { @@ -554,23 +551,17 @@ priv_flow_convert_rss_conf(struct priv *priv __rte_unused, /** * Extract attribute to the parser. * - * @param priv - * Pointer to private structure. * @param[in] attr * Flow rule attributes. * @param[out] error * Perform verbose error reporting if not NULL. - * @param[in, out] parser - * Internal parser structure. * * @return * 0 on success, a negative errno value otherwise and rte_errno is set. */ static int -priv_flow_convert_attributes(struct priv *priv __rte_unused, - const struct rte_flow_attr *attr, - struct rte_flow_error *error, - struct mlx5_flow_parse *parser __rte_unused) +mlx5_flow_convert_attributes(const struct rte_flow_attr *attr, + struct rte_flow_error *error) { if (attr->group) { rte_flow_error_set(error, ENOTSUP, @@ -606,8 +597,8 @@ priv_flow_convert_attributes(struct priv *priv __rte_unused, /** * Extract actions request to the parser. * - * @param priv - * Pointer to private structure. + * @param dev + * Pointer to Ethernet device. * @param[in] actions * Associated actions (list terminated by the END action). * @param[out] error @@ -619,16 +610,18 @@ priv_flow_convert_attributes(struct priv *priv __rte_unused, * 0 on success, a negative errno value otherwise and rte_errno is set. */ static int -priv_flow_convert_actions(struct priv *priv, +mlx5_flow_convert_actions(struct rte_eth_dev *dev, const struct rte_flow_action actions[], struct rte_flow_error *error, struct mlx5_flow_parse *parser) { + struct priv *priv = dev->data->dev_private; + /* * Add default RSS configuration necessary for Verbs to create QP even * if no RSS is necessary. */ - priv_flow_convert_rss_conf(priv, parser, + mlx5_flow_convert_rss_conf(parser, (const struct rte_eth_rss_conf *) &priv->rss_conf); for (; actions->type != RTE_FLOW_ACTION_TYPE_END; ++actions) { @@ -708,8 +701,7 @@ priv_flow_convert_actions(struct priv *priv, for (n = 0; n < rss->num; ++n) parser->queues[n] = rss->queue[n]; parser->queues_n = rss->num; - if (priv_flow_convert_rss_conf(priv, parser, - rss->rss_conf)) { + if (mlx5_flow_convert_rss_conf(parser, rss->rss_conf)) { rte_flow_error_set(error, EINVAL, RTE_FLOW_ERROR_TYPE_ACTION, actions, @@ -763,8 +755,6 @@ exit_action_not_supported: /** * Validate items. * - * @param priv - * Pointer to private structure. * @param[in] items * Pattern specification (list terminated by the END pattern item). * @param[out] error @@ -776,8 +766,7 @@ exit_action_not_supported: * 0 on success, a negative errno value otherwise and rte_errno is set. */ static int -priv_flow_convert_items_validate(struct priv *priv __rte_unused, - const struct rte_flow_item items[], +mlx5_flow_convert_items_validate(const struct rte_flow_item items[], struct rte_flow_error *error, struct mlx5_flow_parse *parser) { @@ -854,8 +843,6 @@ exit_item_not_supported: /** * Allocate memory space to store verbs flow attributes. * - * @param priv - * Pointer to private structure. * @param[in] priority * Flow priority. * @param[in] size @@ -867,8 +854,7 @@ exit_item_not_supported: * A verbs flow attribute on success, NULL otherwise. */ static struct ibv_flow_attr * -priv_flow_convert_allocate(struct priv *priv __rte_unused, - unsigned int priority, +mlx5_flow_convert_allocate(unsigned int priority, unsigned int size, struct rte_flow_error *error) { @@ -889,14 +875,11 @@ priv_flow_convert_allocate(struct priv *priv __rte_unused, /** * Finalise verbs flow attributes. * - * @param priv - * Pointer to private structure. * @param[in, out] parser * Internal parser structure. */ static void -priv_flow_convert_finalise(struct priv *priv __rte_unused, - struct mlx5_flow_parse *parser) +mlx5_flow_convert_finalise(struct mlx5_flow_parse *parser) { const unsigned int ipv4 = hash_rxq_init[parser->layer].ip_version == MLX5_IPV4; @@ -1015,8 +998,8 @@ fill: /** * Validate and convert a flow supported by the NIC. * - * @param priv - * Pointer to private structure. + * @param dev + * Pointer to Ethernet device. * @param[in] attr * Flow rule attributes. * @param[in] pattern @@ -1032,7 +1015,7 @@ fill: * 0 on success, a negative errno value otherwise and rte_errno is set. */ static int -priv_flow_convert(struct priv *priv, +mlx5_flow_convert(struct rte_eth_dev *dev, const struct rte_flow_attr *attr, const struct rte_flow_item items[], const struct rte_flow_action actions[], @@ -1049,16 +1032,16 @@ priv_flow_convert(struct priv *priv, .layer = HASH_RXQ_ETH, .mark_id = MLX5_FLOW_MARK_DEFAULT, }; - ret = priv_flow_convert_attributes(priv, attr, error, parser); + ret = mlx5_flow_convert_attributes(attr, error); if (ret) return ret; - ret = priv_flow_convert_actions(priv, actions, error, parser); + ret = mlx5_flow_convert_actions(dev, actions, error, parser); if (ret) return ret; - ret = priv_flow_convert_items_validate(priv, items, error, parser); + ret = mlx5_flow_convert_items_validate(items, error, parser); if (ret) return ret; - priv_flow_convert_finalise(priv, parser); + mlx5_flow_convert_finalise(parser); /* * Second step. * Allocate the memory space to store verbs specifications. @@ -1070,8 +1053,7 @@ priv_flow_convert(struct priv *priv, unsigned int offset = parser->queue[HASH_RXQ_ETH].offset; parser->queue[HASH_RXQ_ETH].ibv_attr = - priv_flow_convert_allocate(priv, priority, - offset, error); + mlx5_flow_convert_allocate(priority, offset, error); if (!parser->queue[HASH_RXQ_ETH].ibv_attr) return ENOMEM; parser->queue[HASH_RXQ_ETH].offset = @@ -1089,7 +1071,7 @@ priv_flow_convert(struct priv *priv, continue; offset = parser->queue[i].offset; parser->queue[i].ibv_attr = - priv_flow_convert_allocate(priv, priority, + mlx5_flow_convert_allocate(priority, offset, error); if (!parser->queue[i].ibv_attr) goto exit_enomem; @@ -1117,7 +1099,7 @@ priv_flow_convert(struct priv *priv, if (parser->mark) mlx5_flow_create_flag_mark(parser, parser->mark_id); if (parser->count && parser->create) { - mlx5_flow_create_count(priv, parser); + mlx5_flow_create_count(dev, parser); if (!parser->cs) goto exit_count_error; } @@ -1126,7 +1108,7 @@ priv_flow_convert(struct priv *priv, * configuration. */ if (!parser->drop) { - priv_flow_convert_finalise(priv, parser); + mlx5_flow_convert_finalise(parser); } else { parser->queue[HASH_RXQ_ETH].ibv_attr->priority = attr->priority + @@ -1578,8 +1560,8 @@ mlx5_flow_create_flag_mark(struct mlx5_flow_parse *parser, uint32_t mark_id) /** * Convert count action to Verbs specification. * - * @param priv - * Pointer to private structure. + * @param dev + * Pointer to Ethernet device. * @param parser * Pointer to MLX5 flow parser structure. * @@ -1587,10 +1569,11 @@ mlx5_flow_create_flag_mark(struct mlx5_flow_parse *parser, uint32_t mark_id) * 0 on success, errno value on failure. */ static int -mlx5_flow_create_count(struct priv *priv __rte_unused, +mlx5_flow_create_count(struct rte_eth_dev *dev __rte_unused, struct mlx5_flow_parse *parser __rte_unused) { #ifdef HAVE_IBV_DEVICE_COUNTERS_SET_SUPPORT + struct priv *priv = dev->data->dev_private; unsigned int size = sizeof(struct ibv_flow_spec_counter_action); struct ibv_counter_set_init_attr init_attr = {0}; struct ibv_flow_spec_counter_action counter = { @@ -1612,8 +1595,8 @@ mlx5_flow_create_count(struct priv *priv __rte_unused, /** * Complete flow rule creation with a drop queue. * - * @param priv - * Pointer to private structure. + * @param dev + * Pointer to Ethernet device. * @param parser * Internal parser structure. * @param flow @@ -1625,11 +1608,12 @@ mlx5_flow_create_count(struct priv *priv __rte_unused, * 0 on success, errno value on failure. */ static int -priv_flow_create_action_queue_drop(struct priv *priv, +mlx5_flow_create_action_queue_drop(struct rte_eth_dev *dev, struct mlx5_flow_parse *parser, struct rte_flow *flow, struct rte_flow_error *error) { + struct priv *priv = dev->data->dev_private; struct ibv_flow_spec_action_drop *drop; unsigned int size = sizeof(struct ibv_flow_spec_action_drop); int err = 0; @@ -1684,8 +1668,8 @@ error: /** * Create hash Rx queues when RSS is enabled. * - * @param priv - * Pointer to private structure. + * @param dev + * Pointer to Ethernet device. * @param parser * Internal parser structure. * @param flow @@ -1697,11 +1681,12 @@ error: * 0 on success, a errno value otherwise and rte_errno is set. */ static int -priv_flow_create_action_queue_rss(struct priv *priv, +mlx5_flow_create_action_queue_rss(struct rte_eth_dev *dev, struct mlx5_flow_parse *parser, struct rte_flow *flow, struct rte_flow_error *error) { + struct priv *priv = dev->data->dev_private; unsigned int i; for (i = 0; i != hash_rxq_init_n; ++i) { @@ -1715,21 +1700,21 @@ priv_flow_create_action_queue_rss(struct priv *priv, if (!priv->dev->data->dev_started) continue; flow->frxq[i].hrxq = - mlx5_priv_hrxq_get(priv, - parser->rss_conf.rss_key, - parser->rss_conf.rss_key_len, - hash_fields, - parser->queues, - parser->queues_n); + mlx5_hrxq_get(dev, + parser->rss_conf.rss_key, + parser->rss_conf.rss_key_len, + hash_fields, + parser->queues, + parser->queues_n); if (flow->frxq[i].hrxq) continue; flow->frxq[i].hrxq = - mlx5_priv_hrxq_new(priv, - parser->rss_conf.rss_key, - parser->rss_conf.rss_key_len, - hash_fields, - parser->queues, - parser->queues_n); + mlx5_hrxq_new(dev, + parser->rss_conf.rss_key, + parser->rss_conf.rss_key_len, + hash_fields, + parser->queues, + parser->queues_n); if (!flow->frxq[i].hrxq) { rte_flow_error_set(error, ENOMEM, RTE_FLOW_ERROR_TYPE_HANDLE, @@ -1743,8 +1728,8 @@ priv_flow_create_action_queue_rss(struct priv *priv, /** * Complete flow rule creation. * - * @param priv - * Pointer to private structure. + * @param dev + * Pointer to Ethernet device. * @param parser * Internal parser structure. * @param flow @@ -1756,11 +1741,12 @@ priv_flow_create_action_queue_rss(struct priv *priv, * 0 on success, a errno value otherwise and rte_errno is set. */ static int -priv_flow_create_action_queue(struct priv *priv, +mlx5_flow_create_action_queue(struct rte_eth_dev *dev, struct mlx5_flow_parse *parser, struct rte_flow *flow, struct rte_flow_error *error) { + struct priv *priv = dev->data->dev_private; int err = 0; unsigned int i; unsigned int flows_n = 0; @@ -1768,7 +1754,7 @@ priv_flow_create_action_queue(struct priv *priv, assert(priv->pd); assert(priv->ctx); assert(!parser->drop); - err = priv_flow_create_action_queue_rss(priv, parser, flow, error); + err = mlx5_flow_create_action_queue_rss(dev, parser, flow, error); if (err) goto error; if (parser->count) @@ -1815,7 +1801,7 @@ error: claim_zero(mlx5_glue->destroy_flow(ibv_flow)); } if (flow->frxq[i].hrxq) - mlx5_priv_hrxq_release(priv, flow->frxq[i].hrxq); + mlx5_hrxq_release(dev, flow->frxq[i].hrxq); if (flow->frxq[i].ibv_attr) rte_free(flow->frxq[i].ibv_attr); } @@ -1830,8 +1816,8 @@ error: /** * Convert a flow. * - * @param priv - * Pointer to private structure. + * @param dev + * Pointer to Ethernet device. * @param list * Pointer to a TAILQ flow list. * @param[in] attr @@ -1847,19 +1833,19 @@ error: * A flow on success, NULL otherwise. */ static struct rte_flow * -priv_flow_create(struct priv *priv, - struct mlx5_flows *list, - const struct rte_flow_attr *attr, - const struct rte_flow_item items[], - const struct rte_flow_action actions[], - struct rte_flow_error *error) +mlx5_flow_list_create(struct rte_eth_dev *dev, + struct mlx5_flows *list, + const struct rte_flow_attr *attr, + const struct rte_flow_item items[], + const struct rte_flow_action actions[], + struct rte_flow_error *error) { struct mlx5_flow_parse parser = { .create = 1, }; struct rte_flow *flow = NULL; unsigned int i; int err; - err = priv_flow_convert(priv, attr, items, actions, error, &parser); + err = mlx5_flow_convert(dev, attr, items, actions, error, &parser); if (err) goto exit; flow = rte_calloc(__func__, 1, @@ -1883,10 +1869,10 @@ priv_flow_create(struct priv *priv, memcpy(flow->rss_key, parser.rss_key, parser.rss_conf.rss_key_len); /* finalise the flow. */ if (parser.drop) - err = priv_flow_create_action_queue_drop(priv, &parser, flow, + err = mlx5_flow_create_action_queue_drop(dev, &parser, flow, error); else - err = priv_flow_create_action_queue(priv, &parser, flow, error); + err = mlx5_flow_create_action_queue(dev, &parser, flow, error); if (err) goto exit; TAILQ_INSERT_TAIL(list, flow, next); @@ -1915,11 +1901,10 @@ mlx5_flow_validate(struct rte_eth_dev *dev, const struct rte_flow_action actions[], struct rte_flow_error *error) { - struct priv *priv = dev->data->dev_private; int ret; struct mlx5_flow_parse parser = { .create = 0, }; - ret = priv_flow_convert(priv, attr, items, actions, error, &parser); + ret = mlx5_flow_convert(dev, attr, items, actions, error, &parser); return ret; } @@ -1937,28 +1922,26 @@ mlx5_flow_create(struct rte_eth_dev *dev, struct rte_flow_error *error) { struct priv *priv = dev->data->dev_private; - struct rte_flow *flow; - flow = priv_flow_create(priv, &priv->flows, attr, items, actions, - error); - return flow; + return mlx5_flow_list_create(dev, &priv->flows, attr, items, actions, + error); } /** - * Destroy a flow. + * Destroy a flow in a list. * - * @param priv - * Pointer to private structure. + * @param dev + * Pointer to Ethernet device. * @param list * Pointer to a TAILQ flow list. * @param[in] flow * Flow to destroy. */ static void -priv_flow_destroy(struct priv *priv, - struct mlx5_flows *list, - struct rte_flow *flow) +mlx5_flow_list_destroy(struct rte_eth_dev *dev, struct mlx5_flows *list, + struct rte_flow *flow) { + struct priv *priv = dev->data->dev_private; unsigned int i; if (flow->drop || !flow->mark) @@ -2006,7 +1989,7 @@ free: claim_zero(mlx5_glue->destroy_flow (frxq->ibv_flow)); if (frxq->hrxq) - mlx5_priv_hrxq_release(priv, frxq->hrxq); + mlx5_hrxq_release(dev, frxq->hrxq); if (frxq->ibv_attr) rte_free(frxq->ibv_attr); } @@ -2023,34 +2006,35 @@ free: /** * Destroy all flows. * - * @param priv - * Pointer to private structure. + * @param dev + * Pointer to Ethernet device. * @param list * Pointer to a TAILQ flow list. */ void -priv_flow_flush(struct priv *priv, struct mlx5_flows *list) +mlx5_flow_list_flush(struct rte_eth_dev *dev, struct mlx5_flows *list) { while (!TAILQ_EMPTY(list)) { struct rte_flow *flow; flow = TAILQ_FIRST(list); - priv_flow_destroy(priv, list, flow); + mlx5_flow_list_destroy(dev, list, flow); } } /** * Create drop queue. * - * @param priv - * Pointer to private structure. + * @param dev + * Pointer to Ethernet device. * * @return * 0 on success. */ int -priv_flow_create_drop_queue(struct priv *priv) +mlx5_flow_create_drop_queue(struct rte_eth_dev *dev) { + struct priv *priv = dev->data->dev_private; struct mlx5_hrxq_drop *fdq = NULL; assert(priv->pd); @@ -2131,12 +2115,13 @@ error: /** * Delete drop queue. * - * @param priv - * Pointer to private structure. + * @param dev + * Pointer to Ethernet device. */ void -priv_flow_delete_drop_queue(struct priv *priv) +mlx5_flow_delete_drop_queue(struct rte_eth_dev *dev) { + struct priv *priv = dev->data->dev_private; struct mlx5_hrxq_drop *fdq = priv->flow_drop_queue; if (!fdq) @@ -2156,14 +2141,15 @@ priv_flow_delete_drop_queue(struct priv *priv) /** * Remove all flows. * - * @param priv - * Pointer to private structure. + * @param dev + * Pointer to Ethernet device. * @param list * Pointer to a TAILQ flow list. */ void -priv_flow_stop(struct priv *priv, struct mlx5_flows *list) +mlx5_flow_stop(struct rte_eth_dev *dev, struct mlx5_flows *list) { + struct priv *priv = dev->data->dev_private; struct rte_flow *flow; TAILQ_FOREACH_REVERSE(flow, list, mlx5_flows, next) { @@ -2206,7 +2192,7 @@ priv_flow_stop(struct priv *priv, struct mlx5_flows *list) claim_zero(mlx5_glue->destroy_flow (flow->frxq[i].ibv_flow)); flow->frxq[i].ibv_flow = NULL; - mlx5_priv_hrxq_release(priv, flow->frxq[i].hrxq); + mlx5_hrxq_release(dev, flow->frxq[i].hrxq); flow->frxq[i].hrxq = NULL; } DEBUG("Flow %p removed", (void *)flow); @@ -2216,8 +2202,8 @@ priv_flow_stop(struct priv *priv, struct mlx5_flows *list) /** * Add all flows. * - * @param priv - * Pointer to private structure. + * @param dev + * Pointer to Ethernet device. * @param list * Pointer to a TAILQ flow list. * @@ -2225,8 +2211,9 @@ priv_flow_stop(struct priv *priv, struct mlx5_flows *list) * 0 on success, a errno value otherwise and rte_errno is set. */ int -priv_flow_start(struct priv *priv, struct mlx5_flows *list) +mlx5_flow_start(struct rte_eth_dev *dev, struct mlx5_flows *list) { + struct priv *priv = dev->data->dev_private; struct rte_flow *flow; TAILQ_FOREACH(flow, list, next) { @@ -2251,19 +2238,19 @@ priv_flow_start(struct priv *priv, struct mlx5_flows *list) if (!flow->frxq[i].ibv_attr) continue; flow->frxq[i].hrxq = - mlx5_priv_hrxq_get(priv, flow->rss_conf.rss_key, - flow->rss_conf.rss_key_len, - hash_rxq_init[i].hash_fields, - (*flow->queues), - flow->queues_n); + mlx5_hrxq_get(dev, flow->rss_conf.rss_key, + flow->rss_conf.rss_key_len, + hash_rxq_init[i].hash_fields, + (*flow->queues), + flow->queues_n); if (flow->frxq[i].hrxq) goto flow_create; flow->frxq[i].hrxq = - mlx5_priv_hrxq_new(priv, flow->rss_conf.rss_key, - flow->rss_conf.rss_key_len, - hash_rxq_init[i].hash_fields, - (*flow->queues), - flow->queues_n); + mlx5_hrxq_new(dev, flow->rss_conf.rss_key, + flow->rss_conf.rss_key_len, + hash_rxq_init[i].hash_fields, + (*flow->queues), + flow->queues_n); if (!flow->frxq[i].hrxq) { DEBUG("Flow %p cannot be applied", (void *)flow); @@ -2293,19 +2280,20 @@ flow_create: /** * Verify the flow list is empty * - * @param priv - * Pointer to private structure. + * @param dev + * Pointer to Ethernet device. * * @return the number of flows not released. */ int -priv_flow_verify(struct priv *priv) +mlx5_flow_verify(struct rte_eth_dev *dev) { + struct priv *priv = dev->data->dev_private; struct rte_flow *flow; int ret = 0; TAILQ_FOREACH(flow, &priv->flows, next) { - DEBUG("%p: flow %p still referenced", (void *)priv, + DEBUG("%p: flow %p still referenced", (void *)dev, (void *)flow); ++ret; } @@ -2386,8 +2374,8 @@ mlx5_ctrl_flow_vlan(struct rte_eth_dev *dev, action_rss.local.rss_conf = &priv->rss_conf; action_rss.local.num = priv->reta_idx_n; actions[0].conf = (const void *)&action_rss.rss; - flow = priv_flow_create(priv, &priv->ctrl_flows, &attr, items, actions, - &error); + flow = mlx5_flow_list_create(dev, &priv->ctrl_flows, &attr, items, + actions, &error); if (!flow) return rte_errno; return 0; @@ -2427,7 +2415,7 @@ mlx5_flow_destroy(struct rte_eth_dev *dev, { struct priv *priv = dev->data->dev_private; - priv_flow_destroy(priv, &priv->flows, flow); + mlx5_flow_list_destroy(dev, &priv->flows, flow); return 0; } @@ -2443,7 +2431,7 @@ mlx5_flow_flush(struct rte_eth_dev *dev, { struct priv *priv = dev->data->dev_private; - priv_flow_flush(priv, &priv->flows); + mlx5_flow_list_flush(dev, &priv->flows); return 0; } @@ -2460,7 +2448,7 @@ mlx5_flow_flush(struct rte_eth_dev *dev, * 0 on success, a errno value otherwise and rte_errno is set. */ static int -priv_flow_query_count(struct ibv_counter_set *cs, +mlx5_flow_query_count(struct ibv_counter_set *cs, struct mlx5_flow_counter_stats *counter_stats, struct rte_flow_query_count *query_count, struct rte_flow_error *error) @@ -2510,7 +2498,7 @@ mlx5_flow_query(struct rte_eth_dev *dev __rte_unused, int res = EINVAL; if (flow->cs) { - res = priv_flow_query_count(flow->cs, + res = mlx5_flow_query_count(flow->cs, &flow->counter_stats, (struct rte_flow_query_count *)data, error); @@ -2555,8 +2543,8 @@ mlx5_flow_isolate(struct rte_eth_dev *dev, /** * Convert a flow director filter to a generic flow. * - * @param priv - * Private structure. + * @param dev + * Pointer to Ethernet device. * @param fdir_filter * Flow director filter to add. * @param attributes @@ -2566,10 +2554,11 @@ mlx5_flow_isolate(struct rte_eth_dev *dev, * 0 on success, errno value on error. */ static int -priv_fdir_filter_convert(struct priv *priv, +mlx5_fdir_filter_convert(struct rte_eth_dev *dev, const struct rte_eth_fdir_filter *fdir_filter, struct mlx5_fdir *attributes) { + struct priv *priv = dev->data->dev_private; const struct rte_eth_fdir_input *input = &fdir_filter->input; /* Validate queue number. */ @@ -2741,8 +2730,8 @@ priv_fdir_filter_convert(struct priv *priv, /** * Add new flow director filter and store it in list. * - * @param priv - * Private structure. + * @param dev + * Pointer to Ethernet device. * @param fdir_filter * Flow director filter to add. * @@ -2750,9 +2739,10 @@ priv_fdir_filter_convert(struct priv *priv, * 0 on success, errno value on failure. */ static int -priv_fdir_filter_add(struct priv *priv, +mlx5_fdir_filter_add(struct rte_eth_dev *dev, const struct rte_eth_fdir_filter *fdir_filter) { + struct priv *priv = dev->data->dev_private; struct mlx5_fdir attributes = { .attr.group = 0, .l2_mask = { @@ -2768,19 +2758,16 @@ priv_fdir_filter_add(struct priv *priv, struct rte_flow *flow; int ret; - ret = priv_fdir_filter_convert(priv, fdir_filter, &attributes); + ret = mlx5_fdir_filter_convert(dev, fdir_filter, &attributes); if (ret) return -ret; - ret = priv_flow_convert(priv, &attributes.attr, attributes.items, + ret = mlx5_flow_convert(dev, &attributes.attr, attributes.items, attributes.actions, &error, &parser); if (ret) return -ret; - flow = priv_flow_create(priv, - &priv->flows, - &attributes.attr, - attributes.items, - attributes.actions, - &error); + flow = mlx5_flow_list_create(dev, &priv->flows, &attributes.attr, + attributes.items, attributes.actions, + &error); if (flow) { DEBUG("FDIR created %p", (void *)flow); return 0; @@ -2791,8 +2778,8 @@ priv_fdir_filter_add(struct priv *priv, /** * Delete specific filter. * - * @param priv - * Private structure. + * @param dev + * Pointer to Ethernet device. * @param fdir_filter * Filter to be deleted. * @@ -2800,9 +2787,10 @@ priv_fdir_filter_add(struct priv *priv, * 0 on success, errno value on failure. */ static int -priv_fdir_filter_delete(struct priv *priv, +mlx5_fdir_filter_delete(struct rte_eth_dev *dev, const struct rte_eth_fdir_filter *fdir_filter) { + struct priv *priv = dev->data->dev_private; struct mlx5_fdir attributes = { .attr.group = 0, }; @@ -2815,10 +2803,10 @@ priv_fdir_filter_delete(struct priv *priv, unsigned int i; int ret; - ret = priv_fdir_filter_convert(priv, fdir_filter, &attributes); + ret = mlx5_fdir_filter_convert(dev, fdir_filter, &attributes); if (ret) return -ret; - ret = priv_flow_convert(priv, &attributes.attr, attributes.items, + ret = mlx5_flow_convert(dev, &attributes.attr, attributes.items, attributes.actions, &error, &parser); if (ret) goto exit; @@ -2876,7 +2864,7 @@ wrong_flow: continue; } if (flow) - priv_flow_destroy(priv, &priv->flows, flow); + mlx5_flow_list_destroy(dev, &priv->flows, flow); exit: for (i = 0; i != hash_rxq_init_n; ++i) { if (parser.queue[i].ibv_attr) @@ -2888,8 +2876,8 @@ exit: /** * Update queue for specific filter. * - * @param priv - * Private structure. + * @param dev + * Pointer to Ethernet device. * @param fdir_filter * Filter to be updated. * @@ -2897,41 +2885,44 @@ exit: * 0 on success, errno value on failure. */ static int -priv_fdir_filter_update(struct priv *priv, +mlx5_fdir_filter_update(struct rte_eth_dev *dev, const struct rte_eth_fdir_filter *fdir_filter) { int ret; - ret = priv_fdir_filter_delete(priv, fdir_filter); + ret = mlx5_fdir_filter_delete(dev, fdir_filter); if (ret) return ret; - ret = priv_fdir_filter_add(priv, fdir_filter); + ret = mlx5_fdir_filter_add(dev, fdir_filter); return ret; } /** * Flush all filters. * - * @param priv - * Private structure. + * @param dev + * Pointer to Ethernet device. */ static void -priv_fdir_filter_flush(struct priv *priv) +mlx5_fdir_filter_flush(struct rte_eth_dev *dev) { - priv_flow_flush(priv, &priv->flows); + struct priv *priv = dev->data->dev_private; + + mlx5_flow_list_flush(dev, &priv->flows); } /** * Get flow director information. * - * @param priv - * Private structure. + * @param dev + * Pointer to Ethernet device. * @param[out] fdir_info * Resulting flow director information. */ static void -priv_fdir_info_get(struct priv *priv, struct rte_eth_fdir_info *fdir_info) +mlx5_fdir_info_get(struct rte_eth_dev *dev, struct rte_eth_fdir_info *fdir_info) { + struct priv *priv = dev->data->dev_private; struct rte_eth_fdir_masks *mask = &priv->dev->data->dev_conf.fdir_conf.mask; @@ -2949,8 +2940,8 @@ priv_fdir_info_get(struct priv *priv, struct rte_eth_fdir_info *fdir_info) /** * Deal with flow director operations. * - * @param priv - * Pointer to private structure. + * @param dev + * Pointer to Ethernet device. * @param filter_op * Operation to perform. * @param arg @@ -2960,8 +2951,10 @@ priv_fdir_info_get(struct priv *priv, struct rte_eth_fdir_info *fdir_info) * 0 on success, errno value on failure. */ static int -priv_fdir_ctrl_func(struct priv *priv, enum rte_filter_op filter_op, void *arg) +mlx5_fdir_ctrl_func(struct rte_eth_dev *dev, enum rte_filter_op filter_op, + void *arg) { + struct priv *priv = dev->data->dev_private; enum rte_fdir_mode fdir_mode = priv->dev->data->dev_conf.fdir_conf.mode; int ret = 0; @@ -2971,27 +2964,27 @@ priv_fdir_ctrl_func(struct priv *priv, enum rte_filter_op filter_op, void *arg) if (fdir_mode != RTE_FDIR_MODE_PERFECT && fdir_mode != RTE_FDIR_MODE_PERFECT_MAC_VLAN) { ERROR("%p: flow director mode %d not supported", - (void *)priv, fdir_mode); + (void *)dev, fdir_mode); return EINVAL; } switch (filter_op) { case RTE_ETH_FILTER_ADD: - ret = priv_fdir_filter_add(priv, arg); + ret = mlx5_fdir_filter_add(dev, arg); break; case RTE_ETH_FILTER_UPDATE: - ret = priv_fdir_filter_update(priv, arg); + ret = mlx5_fdir_filter_update(dev, arg); break; case RTE_ETH_FILTER_DELETE: - ret = priv_fdir_filter_delete(priv, arg); + ret = mlx5_fdir_filter_delete(dev, arg); break; case RTE_ETH_FILTER_FLUSH: - priv_fdir_filter_flush(priv); + mlx5_fdir_filter_flush(dev); break; case RTE_ETH_FILTER_INFO: - priv_fdir_info_get(priv, arg); + mlx5_fdir_info_get(dev, arg); break; default: - DEBUG("%p: unknown operation %u", (void *)priv, + DEBUG("%p: unknown operation %u", (void *)dev, filter_op); ret = EINVAL; break; @@ -3021,7 +3014,6 @@ mlx5_dev_filter_ctrl(struct rte_eth_dev *dev, void *arg) { int ret = EINVAL; - struct priv *priv = dev->data->dev_private; switch (filter_type) { case RTE_ETH_FILTER_GENERIC: @@ -3030,7 +3022,7 @@ mlx5_dev_filter_ctrl(struct rte_eth_dev *dev, *(const void **)arg = &mlx5_flow_ops; return 0; case RTE_ETH_FILTER_FDIR: - ret = priv_fdir_ctrl_func(priv, filter_op, arg); + ret = mlx5_fdir_ctrl_func(dev, filter_op, arg); break; default: ERROR("%p: filter type (%d) not supported", diff --git a/drivers/net/mlx5/mlx5_mac.c b/drivers/net/mlx5/mlx5_mac.c index a529dfeac7..91c977bc56 100644 --- a/drivers/net/mlx5/mlx5_mac.c +++ b/drivers/net/mlx5/mlx5_mac.c @@ -35,8 +35,8 @@ /** * Get MAC address by querying netdevice. * - * @param[in] priv - * struct priv for the requested device. + * @param[in] dev + * Pointer to Ethernet device. * @param[out] mac * MAC address output buffer. * @@ -44,11 +44,11 @@ * 0 on success, -1 on failure and errno is set. */ int -priv_get_mac(struct priv *priv, uint8_t (*mac)[ETHER_ADDR_LEN]) +mlx5_get_mac(struct rte_eth_dev *dev, uint8_t (*mac)[ETHER_ADDR_LEN]) { struct ifreq request; - if (priv_ifreq(priv, SIOCGIFHWADDR, &request)) + if (mlx5_ifreq(dev, SIOCGIFHWADDR, &request)) return -1; memcpy(mac, request.ifr_hwaddr.sa_data, ETHER_ADDR_LEN); return 0; diff --git a/drivers/net/mlx5/mlx5_mr.c b/drivers/net/mlx5/mlx5_mr.c index 3b7b6d1401..fe60dd1329 100644 --- a/drivers/net/mlx5/mlx5_mr.c +++ b/drivers/net/mlx5/mlx5_mr.c @@ -83,10 +83,6 @@ mlx5_check_mempool(struct rte_mempool *mp, uintptr_t *start, * Register a Memory Region (MR) <-> Memory Pool (MP) association in * txq->mp2mr[]. If mp2mr[] is full, remove an entry first. * - * This function should only be called by txq_mp2mr(). - * - * @param priv - * Pointer to private structure. * @param txq * Pointer to TX queue structure. * @param[in] mp @@ -98,29 +94,35 @@ mlx5_check_mempool(struct rte_mempool *mp, uintptr_t *start, * mr on success, NULL on failure. */ struct mlx5_mr * -priv_txq_mp2mr_reg(struct priv *priv, struct mlx5_txq_data *txq, - struct rte_mempool *mp, unsigned int idx) +mlx5_txq_mp2mr_reg(struct mlx5_txq_data *txq, struct rte_mempool *mp, + unsigned int idx) { struct mlx5_txq_ctrl *txq_ctrl = container_of(txq, struct mlx5_txq_ctrl, txq); + struct rte_eth_dev *dev; struct mlx5_mr *mr; + rte_spinlock_lock(&txq_ctrl->priv->mr_lock); /* Add a new entry, register MR first. */ DEBUG("%p: discovered new memory pool \"%s\" (%p)", (void *)txq_ctrl, mp->name, (void *)mp); - mr = priv_mr_get(priv, mp); + dev = txq_ctrl->priv->dev; + mr = mlx5_mr_get(dev, mp); if (mr == NULL) { if (rte_eal_process_type() != RTE_PROC_PRIMARY) { - DEBUG("Using unregistered mempool 0x%p(%s) in secondary process," - " please create mempool before rte_eth_dev_start()", + DEBUG("Using unregistered mempool 0x%p(%s) in " + "secondary process, please create mempool before " + " rte_eth_dev_start()", (void *)mp, mp->name); + rte_spinlock_unlock(&txq_ctrl->priv->mr_lock); return NULL; } - mr = priv_mr_new(priv, mp); + mr = mlx5_mr_new(dev, mp); } if (unlikely(mr == NULL)) { DEBUG("%p: unable to configure MR, ibv_reg_mr() failed.", (void *)txq_ctrl); + rte_spinlock_unlock(&txq_ctrl->priv->mr_lock); return NULL; } if (unlikely(idx == RTE_DIM(txq->mp2mr))) { @@ -128,7 +130,7 @@ priv_txq_mp2mr_reg(struct priv *priv, struct mlx5_txq_data *txq, DEBUG("%p: MR <-> MP table full, dropping oldest entry.", (void *)txq_ctrl); --idx; - priv_mr_release(priv, txq->mp2mr[0]); + mlx5_mr_release(txq->mp2mr[0]); memmove(&txq->mp2mr[0], &txq->mp2mr[1], (sizeof(txq->mp2mr) - sizeof(txq->mp2mr[0]))); } @@ -137,35 +139,6 @@ priv_txq_mp2mr_reg(struct priv *priv, struct mlx5_txq_data *txq, DEBUG("%p: new MR lkey for MP \"%s\" (%p): 0x%08" PRIu32, (void *)txq_ctrl, mp->name, (void *)mp, txq_ctrl->txq.mp2mr[idx]->lkey); - return mr; -} - -/** - * Register a Memory Region (MR) <-> Memory Pool (MP) association in - * txq->mp2mr[]. If mp2mr[] is full, remove an entry first. - * - * This function should only be called by txq_mp2mr(). - * - * @param txq - * Pointer to TX queue structure. - * @param[in] mp - * Memory Pool for which a Memory Region lkey must be returned. - * @param idx - * Index of the next available entry. - * - * @return - * mr on success, NULL on failure. - */ -struct mlx5_mr* -mlx5_txq_mp2mr_reg(struct mlx5_txq_data *txq, struct rte_mempool *mp, - unsigned int idx) -{ - struct mlx5_txq_ctrl *txq_ctrl = - container_of(txq, struct mlx5_txq_ctrl, txq); - struct mlx5_mr *mr; - - rte_spinlock_lock(&txq_ctrl->priv->mr_lock); - mr = priv_txq_mp2mr_reg(txq_ctrl->priv, txq, mp, idx); rte_spinlock_unlock(&txq_ctrl->priv->mr_lock); return mr; } @@ -225,20 +198,20 @@ mlx5_mp2mr_iter(struct rte_mempool *mp, void *arg) if (rte_mempool_obj_iter(mp, txq_mp2mr_mbuf_check, &data) == 0 || data.ret == -1) return; - mr = priv_mr_get(priv, mp); + mr = mlx5_mr_get(priv->dev, mp); if (mr) { - priv_mr_release(priv, mr); + mlx5_mr_release(mr); return; } - priv_mr_new(priv, mp); + mlx5_mr_new(priv->dev, mp); } /** * Register a new memory region from the mempool and store it in the memory * region list. * - * @param priv - * Pointer to private structure. + * @param dev + * Pointer to Ethernet device. * @param mp * Pointer to the memory pool to register. * @@ -246,8 +219,9 @@ mlx5_mp2mr_iter(struct rte_mempool *mp, void *arg) * The memory region on success. */ struct mlx5_mr * -priv_mr_new(struct priv *priv, struct rte_mempool *mp) +mlx5_mr_new(struct rte_eth_dev *dev, struct rte_mempool *mp) { + struct priv *priv = dev->data->dev_private; const struct rte_memseg *ms = rte_eal_get_physmem_layout(); uintptr_t start; uintptr_t end; @@ -289,7 +263,7 @@ priv_mr_new(struct priv *priv, struct rte_mempool *mp) mr->mp = mp; mr->lkey = rte_cpu_to_be_32(mr->mr->lkey); rte_atomic32_inc(&mr->refcnt); - DEBUG("%p: new Memory Region %p refcnt: %d", (void *)priv, + DEBUG("%p: new Memory Region %p refcnt: %d", (void *)dev, (void *)mr, rte_atomic32_read(&mr->refcnt)); LIST_INSERT_HEAD(&priv->mr, mr, next); return mr; @@ -298,8 +272,8 @@ priv_mr_new(struct priv *priv, struct rte_mempool *mp) /** * Search the memory region object in the memory region list. * - * @param priv - * Pointer to private structure. + * @param dev + * Pointer to Ethernet device. * @param mp * Pointer to the memory pool to register. * @@ -307,8 +281,9 @@ priv_mr_new(struct priv *priv, struct rte_mempool *mp) * The memory region on success. */ struct mlx5_mr * -priv_mr_get(struct priv *priv, struct rte_mempool *mp) +mlx5_mr_get(struct rte_eth_dev *dev, struct rte_mempool *mp) { + struct priv *priv = dev->data->dev_private; struct mlx5_mr *mr; assert(mp); @@ -335,7 +310,7 @@ priv_mr_get(struct priv *priv, struct rte_mempool *mp) * 0 on success, errno on failure. */ int -priv_mr_release(struct priv *priv __rte_unused, struct mlx5_mr *mr) +mlx5_mr_release(struct mlx5_mr *mr) { assert(mr); DEBUG("Memory Region %p refcnt: %d", @@ -352,20 +327,21 @@ priv_mr_release(struct priv *priv __rte_unused, struct mlx5_mr *mr) /** * Verify the flow list is empty * - * @param priv - * Pointer to private structure. + * @param dev + * Pointer to Ethernet device. * * @return * The number of object not released. */ int -priv_mr_verify(struct priv *priv) +mlx5_mr_verify(struct rte_eth_dev *dev) { + struct priv *priv = dev->data->dev_private; int ret = 0; struct mlx5_mr *mr; LIST_FOREACH(mr, &priv->mr, next) { - DEBUG("%p: mr %p still referenced", (void *)priv, + DEBUG("%p: mr %p still referenced", (void *)dev, (void *)mr); ++ret; } diff --git a/drivers/net/mlx5/mlx5_rss.c b/drivers/net/mlx5/mlx5_rss.c index 4b88215fba..a654a5a7d4 100644 --- a/drivers/net/mlx5/mlx5_rss.c +++ b/drivers/net/mlx5/mlx5_rss.c @@ -96,8 +96,8 @@ mlx5_rss_hash_conf_get(struct rte_eth_dev *dev, /** * Allocate/reallocate RETA index table. * - * @param priv - * Pointer to private structure. + * @param dev + * Pointer to Ethernet device. * @praram reta_size * The size of the array to allocate. * @@ -105,8 +105,9 @@ mlx5_rss_hash_conf_get(struct rte_eth_dev *dev, * 0 on success, errno value on failure. */ int -priv_rss_reta_index_resize(struct priv *priv, unsigned int reta_size) +mlx5_rss_reta_index_resize(struct rte_eth_dev *dev, unsigned int reta_size) { + struct priv *priv = dev->data->dev_private; void *mem; unsigned int old_size = priv->reta_idx_n; @@ -127,28 +128,29 @@ priv_rss_reta_index_resize(struct priv *priv, unsigned int reta_size) } /** - * Query RETA table. + * DPDK callback to get the RETA indirection table. * - * @param priv - * Pointer to private structure. - * @param[in, out] reta_conf - * Pointer to the first RETA configuration structure. + * @param dev + * Pointer to Ethernet device structure. + * @param reta_conf + * Pointer to RETA configuration structure array. * @param reta_size - * Number of entries. + * Size of the RETA table. * * @return - * 0 on success, errno value on failure. + * 0 on success, negative errno value on failure. */ -static int -priv_dev_rss_reta_query(struct priv *priv, +int +mlx5_dev_rss_reta_query(struct rte_eth_dev *dev, struct rte_eth_rss_reta_entry64 *reta_conf, - unsigned int reta_size) + uint16_t reta_size) { + struct priv *priv = dev->data->dev_private; unsigned int idx; unsigned int i; if (!reta_size || reta_size > priv->reta_idx_n) - return EINVAL; + return -EINVAL; /* Fill each entry of the table even if its bit is not set. */ for (idx = 0, i = 0; (i != reta_size); ++i) { idx = i / RTE_RETA_GROUP_SIZE; @@ -159,31 +161,32 @@ priv_dev_rss_reta_query(struct priv *priv, } /** - * Update RETA table. + * DPDK callback to update the RETA indirection table. * - * @param priv - * Pointer to private structure. - * @param[in] reta_conf - * Pointer to the first RETA configuration structure. + * @param dev + * Pointer to Ethernet device structure. + * @param reta_conf + * Pointer to RETA configuration structure array. * @param reta_size - * Number of entries. + * Size of the RETA table. * * @return - * 0 on success, errno value on failure. + * 0 on success, negative errno value on failure. */ -static int -priv_dev_rss_reta_update(struct priv *priv, +int +mlx5_dev_rss_reta_update(struct rte_eth_dev *dev, struct rte_eth_rss_reta_entry64 *reta_conf, - unsigned int reta_size) + uint16_t reta_size) { + int ret; + struct priv *priv = dev->data->dev_private; unsigned int idx; unsigned int i; unsigned int pos; - int ret; if (!reta_size) - return EINVAL; - ret = priv_rss_reta_index_resize(priv, reta_size); + return -EINVAL; + ret = mlx5_rss_reta_index_resize(dev, reta_size); if (ret) return ret; for (idx = 0, i = 0; (i != reta_size); ++i) { @@ -194,56 +197,6 @@ priv_dev_rss_reta_update(struct priv *priv, assert(reta_conf[idx].reta[pos] < priv->rxqs_n); (*priv->reta_idx)[i] = reta_conf[idx].reta[pos]; } - return 0; -} - -/** - * DPDK callback to get the RETA indirection table. - * - * @param dev - * Pointer to Ethernet device structure. - * @param reta_conf - * Pointer to RETA configuration structure array. - * @param reta_size - * Size of the RETA table. - * - * @return - * 0 on success, negative errno value on failure. - */ -int -mlx5_dev_rss_reta_query(struct rte_eth_dev *dev, - struct rte_eth_rss_reta_entry64 *reta_conf, - uint16_t reta_size) -{ - int ret; - struct priv *priv = dev->data->dev_private; - - ret = priv_dev_rss_reta_query(priv, reta_conf, reta_size); - return -ret; -} - -/** - * DPDK callback to update the RETA indirection table. - * - * @param dev - * Pointer to Ethernet device structure. - * @param reta_conf - * Pointer to RETA configuration structure array. - * @param reta_size - * Size of the RETA table. - * - * @return - * 0 on success, negative errno value on failure. - */ -int -mlx5_dev_rss_reta_update(struct rte_eth_dev *dev, - struct rte_eth_rss_reta_entry64 *reta_conf, - uint16_t reta_size) -{ - int ret; - struct priv *priv = dev->data->dev_private; - - ret = priv_dev_rss_reta_update(priv, reta_conf, reta_size); if (dev->data->dev_started) { mlx5_dev_stop(dev); mlx5_dev_start(dev); diff --git a/drivers/net/mlx5/mlx5_rxq.c b/drivers/net/mlx5/mlx5_rxq.c index de3335cb92..a3b08a1a3b 100644 --- a/drivers/net/mlx5/mlx5_rxq.c +++ b/drivers/net/mlx5/mlx5_rxq.c @@ -97,7 +97,7 @@ rxq_alloc_elts(struct mlx5_rxq_ctrl *rxq_ctrl) (*rxq_ctrl->rxq.elts)[i] = buf; } /* If Rx vector is activated. */ - if (rxq_check_vec_support(&rxq_ctrl->rxq) > 0) { + if (mlx5_rxq_check_vec_support(&rxq_ctrl->rxq) > 0) { struct mlx5_rxq_data *rxq = &rxq_ctrl->rxq; struct rte_mbuf *mbuf_init = &rxq->fake_mbuf; int j; @@ -156,7 +156,7 @@ rxq_free_elts(struct mlx5_rxq_ctrl *rxq_ctrl) * Some mbuf in the Ring belongs to the application. They cannot be * freed. */ - if (rxq_check_vec_support(rxq) > 0) { + if (mlx5_rxq_check_vec_support(rxq) > 0) { for (i = 0; i < used; ++i) (*rxq->elts)[(rxq->rq_ci + i) & q_mask] = NULL; rxq->rq_pi = rxq->rq_ci; @@ -181,22 +181,23 @@ mlx5_rxq_cleanup(struct mlx5_rxq_ctrl *rxq_ctrl) { DEBUG("cleaning up %p", (void *)rxq_ctrl); if (rxq_ctrl->ibv) - mlx5_priv_rxq_ibv_release(rxq_ctrl->priv, rxq_ctrl->ibv); + mlx5_rxq_ibv_release(rxq_ctrl->ibv); memset(rxq_ctrl, 0, sizeof(*rxq_ctrl)); } /** * Returns the per-queue supported offloads. * - * @param priv - * Pointer to private structure. + * @param dev + * Pointer to Ethernet device. * * @return * Supported Rx offloads. */ uint64_t -mlx5_priv_get_rx_queue_offloads(struct priv *priv) +mlx5_get_rx_queue_offloads(struct rte_eth_dev *dev) { + struct priv *priv = dev->data->dev_private; struct mlx5_dev_config *config = &priv->config; uint64_t offloads = (DEV_RX_OFFLOAD_SCATTER | DEV_RX_OFFLOAD_TIMESTAMP | @@ -217,13 +218,11 @@ mlx5_priv_get_rx_queue_offloads(struct priv *priv) /** * Returns the per-port supported offloads. * - * @param priv - * Pointer to private structure. * @return * Supported Rx offloads. */ uint64_t -mlx5_priv_get_rx_port_offloads(struct priv *priv __rte_unused) +mlx5_get_rx_port_offloads(void) { uint64_t offloads = DEV_RX_OFFLOAD_VLAN_FILTER; @@ -233,8 +232,8 @@ mlx5_priv_get_rx_port_offloads(struct priv *priv __rte_unused) /** * Checks if the per-queue offload configuration is valid. * - * @param priv - * Pointer to private structure. + * @param dev + * Pointer to Ethernet device. * @param offloads * Per-queue offloads configuration. * @@ -242,12 +241,11 @@ mlx5_priv_get_rx_port_offloads(struct priv *priv __rte_unused) * 1 if the configuration is valid, 0 otherwise. */ static int -priv_is_rx_queue_offloads_allowed(struct priv *priv, uint64_t offloads) +mlx5_is_rx_queue_offloads_allowed(struct rte_eth_dev *dev, uint64_t offloads) { - uint64_t port_offloads = priv->dev->data->dev_conf.rxmode.offloads; - uint64_t queue_supp_offloads = - mlx5_priv_get_rx_queue_offloads(priv); - uint64_t port_supp_offloads = mlx5_priv_get_rx_port_offloads(priv); + uint64_t port_offloads = dev->data->dev_conf.rxmode.offloads; + uint64_t queue_supp_offloads = mlx5_get_rx_queue_offloads(dev); + uint64_t port_supp_offloads = mlx5_get_rx_port_offloads(); if ((offloads & (queue_supp_offloads | port_supp_offloads)) != offloads) @@ -299,24 +297,24 @@ mlx5_rx_queue_setup(struct rte_eth_dev *dev, uint16_t idx, uint16_t desc, (void *)dev, idx, priv->rxqs_n); return -EOVERFLOW; } - if (!priv_is_rx_queue_offloads_allowed(priv, conf->offloads)) { + if (!mlx5_is_rx_queue_offloads_allowed(dev, conf->offloads)) { ret = ENOTSUP; ERROR("%p: Rx queue offloads 0x%" PRIx64 " don't match port " "offloads 0x%" PRIx64 " or supported offloads 0x%" PRIx64, (void *)dev, conf->offloads, dev->data->dev_conf.rxmode.offloads, - (mlx5_priv_get_rx_port_offloads(priv) | - mlx5_priv_get_rx_queue_offloads(priv))); + (mlx5_get_rx_port_offloads() | + mlx5_get_rx_queue_offloads(dev))); goto out; } - if (!mlx5_priv_rxq_releasable(priv, idx)) { + if (!mlx5_rxq_releasable(dev, idx)) { ret = EBUSY; ERROR("%p: unable to release queue index %u", (void *)dev, idx); goto out; } - mlx5_priv_rxq_release(priv, idx); - rxq_ctrl = mlx5_priv_rxq_new(priv, idx, desc, socket, conf, mp); + mlx5_rxq_release(dev, idx); + rxq_ctrl = mlx5_rxq_new(dev, idx, desc, socket, conf, mp); if (!rxq_ctrl) { ERROR("%p: unable to allocate queue index %u", (void *)dev, idx); @@ -347,24 +345,25 @@ mlx5_rx_queue_release(void *dpdk_rxq) return; rxq_ctrl = container_of(rxq, struct mlx5_rxq_ctrl, rxq); priv = rxq_ctrl->priv; - if (!mlx5_priv_rxq_releasable(priv, rxq_ctrl->rxq.stats.idx)) + if (!mlx5_rxq_releasable(priv->dev, rxq_ctrl->rxq.stats.idx)) rte_panic("Rx queue %p is still used by a flow and cannot be" " removed\n", (void *)rxq_ctrl); - mlx5_priv_rxq_release(priv, rxq_ctrl->rxq.stats.idx); + mlx5_rxq_release(priv->dev, rxq_ctrl->rxq.stats.idx); } /** * Allocate queue vector and fill epoll fd list for Rx interrupts. * - * @param priv - * Pointer to private structure. + * @param dev + * Pointer to Ethernet device. * * @return * 0 on success, negative on failure. */ int -priv_rx_intr_vec_enable(struct priv *priv) +mlx5_rx_intr_vec_enable(struct rte_eth_dev *dev) { + struct priv *priv = dev->data->dev_private; unsigned int i; unsigned int rxqs_n = priv->rxqs_n; unsigned int n = RTE_MIN(rxqs_n, (uint32_t)RTE_MAX_RXTX_INTR_VEC_ID); @@ -373,7 +372,7 @@ priv_rx_intr_vec_enable(struct priv *priv) if (!priv->dev->data->dev_conf.intr_conf.rxq) return 0; - priv_rx_intr_vec_disable(priv); + mlx5_rx_intr_vec_disable(dev); intr_handle->intr_vec = malloc(n * sizeof(intr_handle->intr_vec[0])); if (intr_handle->intr_vec == NULL) { ERROR("failed to allocate memory for interrupt vector," @@ -383,7 +382,7 @@ priv_rx_intr_vec_enable(struct priv *priv) intr_handle->type = RTE_INTR_HANDLE_EXT; for (i = 0; i != n; ++i) { /* This rxq ibv must not be released in this function. */ - struct mlx5_rxq_ibv *rxq_ibv = mlx5_priv_rxq_ibv_get(priv, i); + struct mlx5_rxq_ibv *rxq_ibv = mlx5_rxq_ibv_get(dev, i); int fd; int flags; int rc; @@ -400,7 +399,7 @@ priv_rx_intr_vec_enable(struct priv *priv) ERROR("too many Rx queues for interrupt vector size" " (%d), Rx interrupts cannot be enabled", RTE_MAX_RXTX_INTR_VEC_ID); - priv_rx_intr_vec_disable(priv); + mlx5_rx_intr_vec_disable(dev); return -1; } fd = rxq_ibv->channel->fd; @@ -409,7 +408,7 @@ priv_rx_intr_vec_enable(struct priv *priv) if (rc < 0) { ERROR("failed to make Rx interrupt file descriptor" " %d non-blocking for queue index %d", fd, i); - priv_rx_intr_vec_disable(priv); + mlx5_rx_intr_vec_disable(dev); return -1; } intr_handle->intr_vec[i] = RTE_INTR_VEC_RXTX_OFFSET + count; @@ -417,7 +416,7 @@ priv_rx_intr_vec_enable(struct priv *priv) count++; } if (!count) - priv_rx_intr_vec_disable(priv); + mlx5_rx_intr_vec_disable(dev); else intr_handle->nb_efd = count; return 0; @@ -426,12 +425,13 @@ priv_rx_intr_vec_enable(struct priv *priv) /** * Clean up Rx interrupts handler. * - * @param priv - * Pointer to private structure. + * @param dev + * Pointer to Ethernet device. */ void -priv_rx_intr_vec_disable(struct priv *priv) +mlx5_rx_intr_vec_disable(struct rte_eth_dev *dev) { + struct priv *priv = dev->data->dev_private; struct rte_intr_handle *intr_handle = priv->dev->intr_handle; unsigned int i; unsigned int rxqs_n = priv->rxqs_n; @@ -454,7 +454,7 @@ priv_rx_intr_vec_disable(struct priv *priv) */ rxq_data = (*priv->rxqs)[i]; rxq_ctrl = container_of(rxq_data, struct mlx5_rxq_ctrl, rxq); - mlx5_priv_rxq_ibv_release(priv, rxq_ctrl->ibv); + mlx5_rxq_ibv_release(rxq_ctrl->ibv); } free: rte_intr_free_epoll_fd(intr_handle); @@ -516,13 +516,13 @@ mlx5_rx_intr_enable(struct rte_eth_dev *dev, uint16_t rx_queue_id) if (rxq_ctrl->irq) { struct mlx5_rxq_ibv *rxq_ibv; - rxq_ibv = mlx5_priv_rxq_ibv_get(priv, rx_queue_id); + rxq_ibv = mlx5_rxq_ibv_get(dev, rx_queue_id); if (!rxq_ibv) { ret = EINVAL; goto exit; } mlx5_arm_cq(rxq_data, rxq_data->cq_arm_sn); - mlx5_priv_rxq_ibv_release(priv, rxq_ibv); + mlx5_rxq_ibv_release(rxq_ibv); } exit: if (ret) @@ -560,7 +560,7 @@ mlx5_rx_intr_disable(struct rte_eth_dev *dev, uint16_t rx_queue_id) rxq_ctrl = container_of(rxq_data, struct mlx5_rxq_ctrl, rxq); if (!rxq_ctrl->irq) goto exit; - rxq_ibv = mlx5_priv_rxq_ibv_get(priv, rx_queue_id); + rxq_ibv = mlx5_rxq_ibv_get(dev, rx_queue_id); if (!rxq_ibv) { ret = EINVAL; goto exit; @@ -574,7 +574,7 @@ mlx5_rx_intr_disable(struct rte_eth_dev *dev, uint16_t rx_queue_id) mlx5_glue->ack_cq_events(rxq_ibv->cq, 1); exit: if (rxq_ibv) - mlx5_priv_rxq_ibv_release(priv, rxq_ibv); + mlx5_rxq_ibv_release(rxq_ibv); if (ret) WARN("unable to disable interrupt on rx queue %d", rx_queue_id); @@ -584,8 +584,8 @@ exit: /** * Create the Rx queue Verbs object. * - * @param priv - * Pointer to private structure. + * @param dev + * Pointer to Ethernet device. * @param idx * Queue index in DPDK Rx queue array * @@ -593,8 +593,9 @@ exit: * The Verbs object initialised if it can be created. */ struct mlx5_rxq_ibv * -mlx5_priv_rxq_ibv_new(struct priv *priv, uint16_t idx) +mlx5_rxq_ibv_new(struct rte_eth_dev *dev, uint16_t idx) { + struct priv *priv = dev->data->dev_private; struct mlx5_rxq_data *rxq_data = (*priv->rxqs)[idx]; struct mlx5_rxq_ctrl *rxq_ctrl = container_of(rxq_data, struct mlx5_rxq_ctrl, rxq); @@ -629,9 +630,9 @@ mlx5_priv_rxq_ibv_new(struct priv *priv, uint16_t idx) } tmpl->rxq_ctrl = rxq_ctrl; /* Use the entire RX mempool as the memory region. */ - tmpl->mr = priv_mr_get(priv, rxq_data->mp); + tmpl->mr = mlx5_mr_get(dev, rxq_data->mp); if (!tmpl->mr) { - tmpl->mr = priv_mr_new(priv, rxq_data->mp); + tmpl->mr = mlx5_mr_new(dev, rxq_data->mp); if (!tmpl->mr) { ERROR("%p: MR creation failure", (void *)rxq_ctrl); goto error; @@ -661,7 +662,7 @@ mlx5_priv_rxq_ibv_new(struct priv *priv, uint16_t idx) * For vectorized Rx, it must not be doubled in order to * make cq_ci and rq_ci aligned. */ - if (rxq_check_vec_support(rxq_data) < 0) + if (mlx5_rxq_check_vec_support(rxq_data) < 0) attr.cq.ibv.cqe *= 2; } else if (config->cqe_comp && rxq_data->hw_timestamp) { DEBUG("Rx CQE compression is disabled for HW timestamp"); @@ -781,7 +782,7 @@ mlx5_priv_rxq_ibv_new(struct priv *priv, uint16_t idx) *rxq_data->rq_db = rte_cpu_to_be_32(rxq_data->rq_ci); DEBUG("%p: rxq updated with %p", (void *)rxq_ctrl, (void *)&tmpl); rte_atomic32_inc(&tmpl->refcnt); - DEBUG("%p: Verbs Rx queue %p: refcnt %d", (void *)priv, + DEBUG("%p: Verbs Rx queue %p: refcnt %d", (void *)dev, (void *)tmpl, rte_atomic32_read(&tmpl->refcnt)); LIST_INSERT_HEAD(&priv->rxqsibv, tmpl, next); priv->verbs_alloc_ctx.type = MLX5_VERBS_ALLOC_TYPE_NONE; @@ -794,7 +795,7 @@ error: if (tmpl->channel) claim_zero(mlx5_glue->destroy_comp_channel(tmpl->channel)); if (tmpl->mr) - priv_mr_release(priv, tmpl->mr); + mlx5_mr_release(tmpl->mr); priv->verbs_alloc_ctx.type = MLX5_VERBS_ALLOC_TYPE_NONE; return NULL; } @@ -802,8 +803,8 @@ error: /** * Get an Rx queue Verbs object. * - * @param priv - * Pointer to private structure. + * @param dev + * Pointer to Ethernet device. * @param idx * Queue index in DPDK Rx queue array * @@ -811,8 +812,9 @@ error: * The Verbs object if it exists. */ struct mlx5_rxq_ibv * -mlx5_priv_rxq_ibv_get(struct priv *priv, uint16_t idx) +mlx5_rxq_ibv_get(struct rte_eth_dev *dev, uint16_t idx) { + struct priv *priv = dev->data->dev_private; struct mlx5_rxq_data *rxq_data = (*priv->rxqs)[idx]; struct mlx5_rxq_ctrl *rxq_ctrl; @@ -822,9 +824,9 @@ mlx5_priv_rxq_ibv_get(struct priv *priv, uint16_t idx) return NULL; rxq_ctrl = container_of(rxq_data, struct mlx5_rxq_ctrl, rxq); if (rxq_ctrl->ibv) { - priv_mr_get(priv, rxq_data->mp); + mlx5_mr_get(dev, rxq_data->mp); rte_atomic32_inc(&rxq_ctrl->ibv->refcnt); - DEBUG("%p: Verbs Rx queue %p: refcnt %d", (void *)priv, + DEBUG("%p: Verbs Rx queue %p: refcnt %d", (void *)dev, (void *)rxq_ctrl->ibv, rte_atomic32_read(&rxq_ctrl->ibv->refcnt)); } @@ -834,8 +836,6 @@ mlx5_priv_rxq_ibv_get(struct priv *priv, uint16_t idx) /** * Release an Rx verbs queue object. * - * @param priv - * Pointer to private structure. * @param rxq_ibv * Verbs Rx queue object. * @@ -843,7 +843,7 @@ mlx5_priv_rxq_ibv_get(struct priv *priv, uint16_t idx) * 0 on success, errno value on failure. */ int -mlx5_priv_rxq_ibv_release(struct priv *priv, struct mlx5_rxq_ibv *rxq_ibv) +mlx5_rxq_ibv_release(struct mlx5_rxq_ibv *rxq_ibv) { int ret; @@ -851,10 +851,10 @@ mlx5_priv_rxq_ibv_release(struct priv *priv, struct mlx5_rxq_ibv *rxq_ibv) assert(rxq_ibv->wq); assert(rxq_ibv->cq); assert(rxq_ibv->mr); - ret = priv_mr_release(priv, rxq_ibv->mr); + ret = mlx5_mr_release(rxq_ibv->mr); if (!ret) rxq_ibv->mr = NULL; - DEBUG("%p: Verbs Rx queue %p: refcnt %d", (void *)priv, + DEBUG("Verbs Rx queue %p: refcnt %d", (void *)rxq_ibv, rte_atomic32_read(&rxq_ibv->refcnt)); if (rte_atomic32_dec_and_test(&rxq_ibv->refcnt)) { rxq_free_elts(rxq_ibv->rxq_ctrl); @@ -873,20 +873,21 @@ mlx5_priv_rxq_ibv_release(struct priv *priv, struct mlx5_rxq_ibv *rxq_ibv) /** * Verify the Verbs Rx queue list is empty * - * @param priv - * Pointer to private structure. + * @param dev + * Pointer to Ethernet device. * * @return * The number of object not released. */ int -mlx5_priv_rxq_ibv_verify(struct priv *priv) +mlx5_rxq_ibv_verify(struct rte_eth_dev *dev) { + struct priv *priv = dev->data->dev_private; int ret = 0; struct mlx5_rxq_ibv *rxq_ibv; LIST_FOREACH(rxq_ibv, &priv->rxqsibv, next) { - DEBUG("%p: Verbs Rx queue %p still referenced", (void *)priv, + DEBUG("%p: Verbs Rx queue %p still referenced", (void *)dev, (void *)rxq_ibv); ++ret; } @@ -896,14 +897,11 @@ mlx5_priv_rxq_ibv_verify(struct priv *priv) /** * Return true if a single reference exists on the object. * - * @param priv - * Pointer to private structure. * @param rxq_ibv * Verbs Rx queue object. */ int -mlx5_priv_rxq_ibv_releasable(struct priv *priv __rte_unused, - struct mlx5_rxq_ibv *rxq_ibv) +mlx5_rxq_ibv_releasable(struct mlx5_rxq_ibv *rxq_ibv) { assert(rxq_ibv); return (rte_atomic32_read(&rxq_ibv->refcnt) == 1); @@ -912,8 +910,8 @@ mlx5_priv_rxq_ibv_releasable(struct priv *priv __rte_unused, /** * Create a DPDK Rx queue. * - * @param priv - * Pointer to private structure. + * @param dev + * Pointer to Ethernet device. * @param idx * TX queue index. * @param desc @@ -925,11 +923,11 @@ mlx5_priv_rxq_ibv_releasable(struct priv *priv __rte_unused, * A DPDK queue object on success. */ struct mlx5_rxq_ctrl * -mlx5_priv_rxq_new(struct priv *priv, uint16_t idx, uint16_t desc, - unsigned int socket, const struct rte_eth_rxconf *conf, - struct rte_mempool *mp) +mlx5_rxq_new(struct rte_eth_dev *dev, uint16_t idx, uint16_t desc, + unsigned int socket, const struct rte_eth_rxconf *conf, + struct rte_mempool *mp) { - struct rte_eth_dev *dev = priv->dev; + struct priv *priv = dev->data->dev_private; struct mlx5_rxq_ctrl *tmpl; unsigned int mb_len = rte_pktmbuf_data_room_size(mp); struct mlx5_dev_config *config = &priv->config; @@ -1029,7 +1027,7 @@ mlx5_priv_rxq_new(struct priv *priv, uint16_t idx, uint16_t desc, tmpl->rxq.elts = (struct rte_mbuf *(*)[1 << tmpl->rxq.elts_n])(tmpl + 1); rte_atomic32_inc(&tmpl->refcnt); - DEBUG("%p: Rx queue %p: refcnt %d", (void *)priv, + DEBUG("%p: Rx queue %p: refcnt %d", (void *)dev, (void *)tmpl, rte_atomic32_read(&tmpl->refcnt)); LIST_INSERT_HEAD(&priv->rxqsctrl, tmpl, next); return tmpl; @@ -1041,8 +1039,8 @@ error: /** * Get a Rx queue. * - * @param priv - * Pointer to private structure. + * @param dev + * Pointer to Ethernet device. * @param idx * TX queue index. * @@ -1050,17 +1048,18 @@ error: * A pointer to the queue if it exists. */ struct mlx5_rxq_ctrl * -mlx5_priv_rxq_get(struct priv *priv, uint16_t idx) +mlx5_rxq_get(struct rte_eth_dev *dev, uint16_t idx) { + struct priv *priv = dev->data->dev_private; struct mlx5_rxq_ctrl *rxq_ctrl = NULL; if ((*priv->rxqs)[idx]) { rxq_ctrl = container_of((*priv->rxqs)[idx], struct mlx5_rxq_ctrl, rxq); - mlx5_priv_rxq_ibv_get(priv, idx); + mlx5_rxq_ibv_get(dev, idx); rte_atomic32_inc(&rxq_ctrl->refcnt); - DEBUG("%p: Rx queue %p: refcnt %d", (void *)priv, + DEBUG("%p: Rx queue %p: refcnt %d", (void *)dev, (void *)rxq_ctrl, rte_atomic32_read(&rxq_ctrl->refcnt)); } return rxq_ctrl; @@ -1069,8 +1068,8 @@ mlx5_priv_rxq_get(struct priv *priv, uint16_t idx) /** * Release a Rx queue. * - * @param priv - * Pointer to private structure. + * @param dev + * Pointer to Ethernet device. * @param idx * TX queue index. * @@ -1078,8 +1077,9 @@ mlx5_priv_rxq_get(struct priv *priv, uint16_t idx) * 0 on success, errno value on failure. */ int -mlx5_priv_rxq_release(struct priv *priv, uint16_t idx) +mlx5_rxq_release(struct rte_eth_dev *dev, uint16_t idx) { + struct priv *priv = dev->data->dev_private; struct mlx5_rxq_ctrl *rxq_ctrl; if (!(*priv->rxqs)[idx]) @@ -1089,11 +1089,11 @@ mlx5_priv_rxq_release(struct priv *priv, uint16_t idx) if (rxq_ctrl->ibv) { int ret; - ret = mlx5_priv_rxq_ibv_release(rxq_ctrl->priv, rxq_ctrl->ibv); + ret = mlx5_rxq_ibv_release(rxq_ctrl->ibv); if (!ret) rxq_ctrl->ibv = NULL; } - DEBUG("%p: Rx queue %p: refcnt %d", (void *)priv, + DEBUG("%p: Rx queue %p: refcnt %d", (void *)dev, (void *)rxq_ctrl, rte_atomic32_read(&rxq_ctrl->refcnt)); if (rte_atomic32_dec_and_test(&rxq_ctrl->refcnt)) { LIST_REMOVE(rxq_ctrl, next); @@ -1107,8 +1107,8 @@ mlx5_priv_rxq_release(struct priv *priv, uint16_t idx) /** * Verify if the queue can be released. * - * @param priv - * Pointer to private structure. + * @param dev + * Pointer to Ethernet device. * @param idx * TX queue index. * @@ -1116,8 +1116,9 @@ mlx5_priv_rxq_release(struct priv *priv, uint16_t idx) * 1 if the queue can be released. */ int -mlx5_priv_rxq_releasable(struct priv *priv, uint16_t idx) +mlx5_rxq_releasable(struct rte_eth_dev *dev, uint16_t idx) { + struct priv *priv = dev->data->dev_private; struct mlx5_rxq_ctrl *rxq_ctrl; if (!(*priv->rxqs)[idx]) @@ -1129,20 +1130,21 @@ mlx5_priv_rxq_releasable(struct priv *priv, uint16_t idx) /** * Verify the Rx Queue list is empty * - * @param priv - * Pointer to private structure. + * @param dev + * Pointer to Ethernet device. * * @return * The number of object not released. */ int -mlx5_priv_rxq_verify(struct priv *priv) +mlx5_rxq_verify(struct rte_eth_dev *dev) { + struct priv *priv = dev->data->dev_private; struct mlx5_rxq_ctrl *rxq_ctrl; int ret = 0; LIST_FOREACH(rxq_ctrl, &priv->rxqsctrl, next) { - DEBUG("%p: Rx Queue %p still referenced", (void *)priv, + DEBUG("%p: Rx Queue %p still referenced", (void *)dev, (void *)rxq_ctrl); ++ret; } @@ -1152,8 +1154,8 @@ mlx5_priv_rxq_verify(struct priv *priv) /** * Create an indirection table. * - * @param priv - * Pointer to private structure. + * @param dev + * Pointer to Ethernet device. * @param queues * Queues entering in the indirection table. * @param queues_n @@ -1163,9 +1165,10 @@ mlx5_priv_rxq_verify(struct priv *priv) * A new indirection table. */ struct mlx5_ind_table_ibv * -mlx5_priv_ind_table_ibv_new(struct priv *priv, uint16_t queues[], - uint16_t queues_n) +mlx5_ind_table_ibv_new(struct rte_eth_dev *dev, uint16_t queues[], + uint16_t queues_n) { + struct priv *priv = dev->data->dev_private; struct mlx5_ind_table_ibv *ind_tbl; const unsigned int wq_n = rte_is_power_of_2(queues_n) ? log2above(queues_n) : @@ -1179,8 +1182,7 @@ mlx5_priv_ind_table_ibv_new(struct priv *priv, uint16_t queues[], if (!ind_tbl) return NULL; for (i = 0; i != queues_n; ++i) { - struct mlx5_rxq_ctrl *rxq = - mlx5_priv_rxq_get(priv, queues[i]); + struct mlx5_rxq_ctrl *rxq = mlx5_rxq_get(dev, queues[i]); if (!rxq) goto error; @@ -1202,20 +1204,20 @@ mlx5_priv_ind_table_ibv_new(struct priv *priv, uint16_t queues[], goto error; rte_atomic32_inc(&ind_tbl->refcnt); LIST_INSERT_HEAD(&priv->ind_tbls, ind_tbl, next); - DEBUG("%p: Indirection table %p: refcnt %d", (void *)priv, + DEBUG("%p: Indirection table %p: refcnt %d", (void *)dev, (void *)ind_tbl, rte_atomic32_read(&ind_tbl->refcnt)); return ind_tbl; error: rte_free(ind_tbl); - DEBUG("%p cannot create indirection table", (void *)priv); + DEBUG("%p cannot create indirection table", (void *)dev); return NULL; } /** * Get an indirection table. * - * @param priv - * Pointer to private structure. + * @param dev + * Pointer to Ethernet device. * @param queues * Queues entering in the indirection table. * @param queues_n @@ -1225,9 +1227,10 @@ error: * An indirection table if found. */ struct mlx5_ind_table_ibv * -mlx5_priv_ind_table_ibv_get(struct priv *priv, uint16_t queues[], - uint16_t queues_n) +mlx5_ind_table_ibv_get(struct rte_eth_dev *dev, uint16_t queues[], + uint16_t queues_n) { + struct priv *priv = dev->data->dev_private; struct mlx5_ind_table_ibv *ind_tbl; LIST_FOREACH(ind_tbl, &priv->ind_tbls, next) { @@ -1241,10 +1244,10 @@ mlx5_priv_ind_table_ibv_get(struct priv *priv, uint16_t queues[], unsigned int i; rte_atomic32_inc(&ind_tbl->refcnt); - DEBUG("%p: Indirection table %p: refcnt %d", (void *)priv, + DEBUG("%p: Indirection table %p: refcnt %d", (void *)dev, (void *)ind_tbl, rte_atomic32_read(&ind_tbl->refcnt)); for (i = 0; i != ind_tbl->queues_n; ++i) - mlx5_priv_rxq_get(priv, ind_tbl->queues[i]); + mlx5_rxq_get(dev, ind_tbl->queues[i]); } return ind_tbl; } @@ -1252,8 +1255,8 @@ mlx5_priv_ind_table_ibv_get(struct priv *priv, uint16_t queues[], /** * Release an indirection table. * - * @param priv - * Pointer to private structure. + * @param dev + * Pointer to Ethernet device. * @param ind_table * Indirection table to release. * @@ -1261,18 +1264,18 @@ mlx5_priv_ind_table_ibv_get(struct priv *priv, uint16_t queues[], * 0 on success, errno value on failure. */ int -mlx5_priv_ind_table_ibv_release(struct priv *priv, - struct mlx5_ind_table_ibv *ind_tbl) +mlx5_ind_table_ibv_release(struct rte_eth_dev *dev, + struct mlx5_ind_table_ibv *ind_tbl) { unsigned int i; - DEBUG("%p: Indirection table %p: refcnt %d", (void *)priv, + DEBUG("%p: Indirection table %p: refcnt %d", (void *)dev, (void *)ind_tbl, rte_atomic32_read(&ind_tbl->refcnt)); if (rte_atomic32_dec_and_test(&ind_tbl->refcnt)) claim_zero(mlx5_glue->destroy_rwq_ind_table (ind_tbl->ind_table)); for (i = 0; i != ind_tbl->queues_n; ++i) - claim_nonzero(mlx5_priv_rxq_release(priv, ind_tbl->queues[i])); + claim_nonzero(mlx5_rxq_release(dev, ind_tbl->queues[i])); if (!rte_atomic32_read(&ind_tbl->refcnt)) { LIST_REMOVE(ind_tbl, next); rte_free(ind_tbl); @@ -1284,21 +1287,22 @@ mlx5_priv_ind_table_ibv_release(struct priv *priv, /** * Verify the Rx Queue list is empty * - * @param priv - * Pointer to private structure. + * @param dev + * Pointer to Ethernet device. * * @return * The number of object not released. */ int -mlx5_priv_ind_table_ibv_verify(struct priv *priv) +mlx5_ind_table_ibv_verify(struct rte_eth_dev *dev) { + struct priv *priv = dev->data->dev_private; struct mlx5_ind_table_ibv *ind_tbl; int ret = 0; LIST_FOREACH(ind_tbl, &priv->ind_tbls, next) { DEBUG("%p: Verbs indirection table %p still referenced", - (void *)priv, (void *)ind_tbl); + (void *)dev, (void *)ind_tbl); ++ret; } return ret; @@ -1307,8 +1311,8 @@ mlx5_priv_ind_table_ibv_verify(struct priv *priv) /** * Create an Rx Hash queue. * - * @param priv - * Pointer to private structure. + * @param dev + * Pointer to Ethernet device. * @param rss_key * RSS key for the Rx hash queue. * @param rss_key_len @@ -1325,17 +1329,18 @@ mlx5_priv_ind_table_ibv_verify(struct priv *priv) * An hash Rx queue on success. */ struct mlx5_hrxq * -mlx5_priv_hrxq_new(struct priv *priv, uint8_t *rss_key, uint8_t rss_key_len, - uint64_t hash_fields, uint16_t queues[], uint16_t queues_n) +mlx5_hrxq_new(struct rte_eth_dev *dev, uint8_t *rss_key, uint8_t rss_key_len, + uint64_t hash_fields, uint16_t queues[], uint16_t queues_n) { + struct priv *priv = dev->data->dev_private; struct mlx5_hrxq *hrxq; struct mlx5_ind_table_ibv *ind_tbl; struct ibv_qp *qp; queues_n = hash_fields ? queues_n : 1; - ind_tbl = mlx5_priv_ind_table_ibv_get(priv, queues, queues_n); + ind_tbl = mlx5_ind_table_ibv_get(dev, queues, queues_n); if (!ind_tbl) - ind_tbl = mlx5_priv_ind_table_ibv_new(priv, queues, queues_n); + ind_tbl = mlx5_ind_table_ibv_new(dev, queues, queues_n); if (!ind_tbl) return NULL; qp = mlx5_glue->create_qp_ex @@ -1367,11 +1372,11 @@ mlx5_priv_hrxq_new(struct priv *priv, uint8_t *rss_key, uint8_t rss_key_len, memcpy(hrxq->rss_key, rss_key, rss_key_len); rte_atomic32_inc(&hrxq->refcnt); LIST_INSERT_HEAD(&priv->hrxqs, hrxq, next); - DEBUG("%p: Hash Rx queue %p: refcnt %d", (void *)priv, + DEBUG("%p: Hash Rx queue %p: refcnt %d", (void *)dev, (void *)hrxq, rte_atomic32_read(&hrxq->refcnt)); return hrxq; error: - mlx5_priv_ind_table_ibv_release(priv, ind_tbl); + mlx5_ind_table_ibv_release(dev, ind_tbl); if (qp) claim_zero(mlx5_glue->destroy_qp(qp)); return NULL; @@ -1380,8 +1385,8 @@ error: /** * Get an Rx Hash queue. * - * @param priv - * Pointer to private structure. + * @param dev + * Pointer to Ethernet device. * @param rss_conf * RSS configuration for the Rx hash queue. * @param queues @@ -1394,9 +1399,10 @@ error: * An hash Rx queue on success. */ struct mlx5_hrxq * -mlx5_priv_hrxq_get(struct priv *priv, uint8_t *rss_key, uint8_t rss_key_len, - uint64_t hash_fields, uint16_t queues[], uint16_t queues_n) +mlx5_hrxq_get(struct rte_eth_dev *dev, uint8_t *rss_key, uint8_t rss_key_len, + uint64_t hash_fields, uint16_t queues[], uint16_t queues_n) { + struct priv *priv = dev->data->dev_private; struct mlx5_hrxq *hrxq; queues_n = hash_fields ? queues_n : 1; @@ -1409,15 +1415,15 @@ mlx5_priv_hrxq_get(struct priv *priv, uint8_t *rss_key, uint8_t rss_key_len, continue; if (hrxq->hash_fields != hash_fields) continue; - ind_tbl = mlx5_priv_ind_table_ibv_get(priv, queues, queues_n); + ind_tbl = mlx5_ind_table_ibv_get(dev, queues, queues_n); if (!ind_tbl) continue; if (ind_tbl != hrxq->ind_table) { - mlx5_priv_ind_table_ibv_release(priv, ind_tbl); + mlx5_ind_table_ibv_release(dev, ind_tbl); continue; } rte_atomic32_inc(&hrxq->refcnt); - DEBUG("%p: Hash Rx queue %p: refcnt %d", (void *)priv, + DEBUG("%p: Hash Rx queue %p: refcnt %d", (void *)dev, (void *)hrxq, rte_atomic32_read(&hrxq->refcnt)); return hrxq; } @@ -1427,8 +1433,8 @@ mlx5_priv_hrxq_get(struct priv *priv, uint8_t *rss_key, uint8_t rss_key_len, /** * Release the hash Rx queue. * - * @param priv - * Pointer to private structure. + * @param dev + * Pointer to Ethernet device. * @param hrxq * Pointer to Hash Rx queue to release. * @@ -1436,39 +1442,40 @@ mlx5_priv_hrxq_get(struct priv *priv, uint8_t *rss_key, uint8_t rss_key_len, * 0 on success, errno value on failure. */ int -mlx5_priv_hrxq_release(struct priv *priv, struct mlx5_hrxq *hrxq) +mlx5_hrxq_release(struct rte_eth_dev *dev, struct mlx5_hrxq *hrxq) { - DEBUG("%p: Hash Rx queue %p: refcnt %d", (void *)priv, + DEBUG("%p: Hash Rx queue %p: refcnt %d", (void *)dev, (void *)hrxq, rte_atomic32_read(&hrxq->refcnt)); if (rte_atomic32_dec_and_test(&hrxq->refcnt)) { claim_zero(mlx5_glue->destroy_qp(hrxq->qp)); - mlx5_priv_ind_table_ibv_release(priv, hrxq->ind_table); + mlx5_ind_table_ibv_release(dev, hrxq->ind_table); LIST_REMOVE(hrxq, next); rte_free(hrxq); return 0; } - claim_nonzero(mlx5_priv_ind_table_ibv_release(priv, hrxq->ind_table)); + claim_nonzero(mlx5_ind_table_ibv_release(dev, hrxq->ind_table)); return EBUSY; } /** * Verify the Rx Queue list is empty * - * @param priv - * Pointer to private structure. + * @param dev + * Pointer to Ethernet device. * * @return * The number of object not released. */ int -mlx5_priv_hrxq_ibv_verify(struct priv *priv) +mlx5_hrxq_ibv_verify(struct rte_eth_dev *dev) { + struct priv *priv = dev->data->dev_private; struct mlx5_hrxq *hrxq; int ret = 0; LIST_FOREACH(hrxq, &priv->hrxqs, next) { DEBUG("%p: Verbs Hash Rx queue %p still referenced", - (void *)priv, (void *)hrxq); + (void *)dev, (void *)hrxq); ++ret; } return ret; diff --git a/drivers/net/mlx5/mlx5_rxtx.c b/drivers/net/mlx5/mlx5_rxtx.c index 93d794ede7..11dd1b84e8 100644 --- a/drivers/net/mlx5/mlx5_rxtx.c +++ b/drivers/net/mlx5/mlx5_rxtx.c @@ -1962,27 +1962,25 @@ mlx5_rx_burst_vec(void *dpdk_txq __rte_unused, } int __attribute__((weak)) -priv_check_raw_vec_tx_support(struct priv *priv __rte_unused, - struct rte_eth_dev *dev __rte_unused) +mlx5_check_raw_vec_tx_support(struct rte_eth_dev *dev __rte_unused) { return -ENOTSUP; } int __attribute__((weak)) -priv_check_vec_tx_support(struct priv *priv __rte_unused, - struct rte_eth_dev *dev __rte_unused) +mlx5_check_vec_tx_support(struct rte_eth_dev *dev __rte_unused) { return -ENOTSUP; } int __attribute__((weak)) -rxq_check_vec_support(struct mlx5_rxq_data *rxq __rte_unused) +mlx5_rxq_check_vec_support(struct mlx5_rxq_data *rxq __rte_unused) { return -ENOTSUP; } int __attribute__((weak)) -priv_check_vec_rx_support(struct priv *priv __rte_unused) +mlx5_check_vec_rx_support(struct rte_eth_dev *dev __rte_unused) { return -ENOTSUP; } diff --git a/drivers/net/mlx5/mlx5_rxtx.h b/drivers/net/mlx5/mlx5_rxtx.h index d0ec9a2146..17a6072e20 100644 --- a/drivers/net/mlx5/mlx5_rxtx.h +++ b/drivers/net/mlx5/mlx5_rxtx.h @@ -215,67 +215,64 @@ int mlx5_rx_queue_setup(struct rte_eth_dev *dev, uint16_t idx, uint16_t desc, unsigned int socket, const struct rte_eth_rxconf *conf, struct rte_mempool *mp); void mlx5_rx_queue_release(void *dpdk_rxq); -int priv_rx_intr_vec_enable(struct priv *priv); -void priv_rx_intr_vec_disable(struct priv *priv); +int mlx5_rx_intr_vec_enable(struct rte_eth_dev *dev); +void mlx5_rx_intr_vec_disable(struct rte_eth_dev *dev); int mlx5_rx_intr_enable(struct rte_eth_dev *dev, uint16_t rx_queue_id); int mlx5_rx_intr_disable(struct rte_eth_dev *dev, uint16_t rx_queue_id); -struct mlx5_rxq_ibv *mlx5_priv_rxq_ibv_new(struct priv *priv, uint16_t idx); -struct mlx5_rxq_ibv *mlx5_priv_rxq_ibv_get(struct priv *priv, uint16_t idx); -int mlx5_priv_rxq_ibv_release(struct priv *priv, struct mlx5_rxq_ibv *rxq_ibv); -int mlx5_priv_rxq_ibv_releasable(struct priv *priv, - struct mlx5_rxq_ibv *rxq_ibv); -int mlx5_priv_rxq_ibv_verify(struct priv *priv); -struct mlx5_rxq_ctrl *mlx5_priv_rxq_new(struct priv *priv, uint16_t idx, - uint16_t desc, - unsigned int socket, - const struct rte_eth_rxconf *conf, - struct rte_mempool *mp); -struct mlx5_rxq_ctrl *mlx5_priv_rxq_get(struct priv *priv, uint16_t idx); -int mlx5_priv_rxq_release(struct priv *priv, uint16_t idx); -int mlx5_priv_rxq_releasable(struct priv *priv, uint16_t idx); -int mlx5_priv_rxq_verify(struct priv *priv); +struct mlx5_rxq_ibv *mlx5_rxq_ibv_new(struct rte_eth_dev *dev, uint16_t idx); +struct mlx5_rxq_ibv *mlx5_rxq_ibv_get(struct rte_eth_dev *dev, uint16_t idx); +int mlx5_rxq_ibv_release(struct mlx5_rxq_ibv *rxq_ibv); +int mlx5_rxq_ibv_releasable(struct mlx5_rxq_ibv *rxq_ibv); +int mlx5_rxq_ibv_verify(struct rte_eth_dev *dev); +struct mlx5_rxq_ctrl *mlx5_rxq_new(struct rte_eth_dev *dev, uint16_t idx, + uint16_t desc, unsigned int socket, + const struct rte_eth_rxconf *conf, + struct rte_mempool *mp); +struct mlx5_rxq_ctrl *mlx5_rxq_get(struct rte_eth_dev *dev, uint16_t idx); +int mlx5_rxq_release(struct rte_eth_dev *dev, uint16_t idx); +int mlx5_rxq_releasable(struct rte_eth_dev *dev, uint16_t idx); +int mlx5_rxq_verify(struct rte_eth_dev *dev); int rxq_alloc_elts(struct mlx5_rxq_ctrl *rxq_ctrl); -struct mlx5_ind_table_ibv *mlx5_priv_ind_table_ibv_new(struct priv *priv, - uint16_t queues[], - uint16_t queues_n); -struct mlx5_ind_table_ibv *mlx5_priv_ind_table_ibv_get(struct priv *priv, - uint16_t queues[], - uint16_t queues_n); -int mlx5_priv_ind_table_ibv_release(struct priv *priv, - struct mlx5_ind_table_ibv *ind_tbl); -int mlx5_priv_ind_table_ibv_verify(struct priv *priv); -struct mlx5_hrxq *mlx5_priv_hrxq_new(struct priv *priv, uint8_t *rss_key, - uint8_t rss_key_len, uint64_t hash_fields, - uint16_t queues[], uint16_t queues_n); -struct mlx5_hrxq *mlx5_priv_hrxq_get(struct priv *priv, uint8_t *rss_key, - uint8_t rss_key_len, uint64_t hash_fields, - uint16_t queues[], uint16_t queues_n); -int mlx5_priv_hrxq_release(struct priv *priv, struct mlx5_hrxq *hrxq); -int mlx5_priv_hrxq_ibv_verify(struct priv *priv); -uint64_t mlx5_priv_get_rx_port_offloads(struct priv *priv); -uint64_t mlx5_priv_get_rx_queue_offloads(struct priv *priv); +struct mlx5_ind_table_ibv *mlx5_ind_table_ibv_new(struct rte_eth_dev *dev, + uint16_t queues[], + uint16_t queues_n); +struct mlx5_ind_table_ibv *mlx5_ind_table_ibv_get(struct rte_eth_dev *dev, + uint16_t queues[], + uint16_t queues_n); +int mlx5_ind_table_ibv_release(struct rte_eth_dev *dev, + struct mlx5_ind_table_ibv *ind_tbl); +int mlx5_ind_table_ibv_verify(struct rte_eth_dev *dev); +struct mlx5_hrxq *mlx5_hrxq_new(struct rte_eth_dev *dev, uint8_t *rss_key, + uint8_t rss_key_len, uint64_t hash_fields, + uint16_t queues[], uint16_t queues_n); +struct mlx5_hrxq *mlx5_hrxq_get(struct rte_eth_dev *dev, uint8_t *rss_key, + uint8_t rss_key_len, uint64_t hash_fields, + uint16_t queues[], uint16_t queues_n); +int mlx5_hrxq_release(struct rte_eth_dev *dev, struct mlx5_hrxq *hxrq); +int mlx5_hrxq_ibv_verify(struct rte_eth_dev *dev); +uint64_t mlx5_get_rx_port_offloads(void); +uint64_t mlx5_get_rx_queue_offloads(struct rte_eth_dev *dev); /* mlx5_txq.c */ int mlx5_tx_queue_setup(struct rte_eth_dev *dev, uint16_t idx, uint16_t desc, unsigned int socket, const struct rte_eth_txconf *conf); void mlx5_tx_queue_release(void *dpdk_txq); -int priv_tx_uar_remap(struct priv *priv, int fd); -struct mlx5_txq_ibv *mlx5_priv_txq_ibv_new(struct priv *priv, uint16_t idx); -struct mlx5_txq_ibv *mlx5_priv_txq_ibv_get(struct priv *priv, uint16_t idx); -int mlx5_priv_txq_ibv_release(struct priv *priv, struct mlx5_txq_ibv *txq_ibv); -int mlx5_priv_txq_ibv_releasable(struct priv *priv, - struct mlx5_txq_ibv *txq_ibv); -int mlx5_priv_txq_ibv_verify(struct priv *priv); -struct mlx5_txq_ctrl *mlx5_priv_txq_new(struct priv *priv, uint16_t idx, - uint16_t desc, unsigned int socket, - const struct rte_eth_txconf *conf); -struct mlx5_txq_ctrl *mlx5_priv_txq_get(struct priv *priv, uint16_t idx); -int mlx5_priv_txq_release(struct priv *priv, uint16_t idx); -int mlx5_priv_txq_releasable(struct priv *priv, uint16_t idx); -int mlx5_priv_txq_verify(struct priv *priv); +int mlx5_tx_uar_remap(struct rte_eth_dev *dev, int fd); +struct mlx5_txq_ibv *mlx5_txq_ibv_new(struct rte_eth_dev *dev, uint16_t idx); +struct mlx5_txq_ibv *mlx5_txq_ibv_get(struct rte_eth_dev *dev, uint16_t idx); +int mlx5_txq_ibv_release(struct mlx5_txq_ibv *txq_ibv); +int mlx5_txq_ibv_releasable(struct mlx5_txq_ibv *txq_ibv); +int mlx5_txq_ibv_verify(struct rte_eth_dev *dev); +struct mlx5_txq_ctrl *mlx5_txq_new(struct rte_eth_dev *dev, uint16_t idx, + uint16_t desc, unsigned int socket, + const struct rte_eth_txconf *conf); +struct mlx5_txq_ctrl *mlx5_txq_get(struct rte_eth_dev *dev, uint16_t idx); +int mlx5_txq_release(struct rte_eth_dev *dev, uint16_t idx); +int mlx5_txq_releasable(struct rte_eth_dev *dev, uint16_t idx); +int mlx5_txq_verify(struct rte_eth_dev *dev); void txq_alloc_elts(struct mlx5_txq_ctrl *txq_ctrl); -uint64_t mlx5_priv_get_tx_port_offloads(struct priv *priv); +uint64_t mlx5_get_tx_port_offloads(struct rte_eth_dev *dev); /* mlx5_rxtx.c */ @@ -299,26 +296,22 @@ int mlx5_rx_descriptor_status(void *rx_queue, uint16_t offset); int mlx5_tx_descriptor_status(void *tx_queue, uint16_t offset); /* Vectorized version of mlx5_rxtx.c */ - -int priv_check_raw_vec_tx_support(struct priv *priv, struct rte_eth_dev *dev); -int priv_check_vec_tx_support(struct priv *priv, struct rte_eth_dev *dev); -int rxq_check_vec_support(struct mlx5_rxq_data *rxq); -int priv_check_vec_rx_support(struct priv *priv); +int mlx5_check_raw_vec_tx_support(struct rte_eth_dev *dev); +int mlx5_check_vec_tx_support(struct rte_eth_dev *dev); +int mlx5_rxq_check_vec_support(struct mlx5_rxq_data *rxq_data); +int mlx5_check_vec_rx_support(struct rte_eth_dev *dev); uint16_t mlx5_tx_burst_raw_vec(void *dpdk_txq, struct rte_mbuf **pkts, uint16_t pkts_n); uint16_t mlx5_tx_burst_vec(void *dpdk_txq, struct rte_mbuf **pkts, uint16_t pkts_n); -uint16_t mlx5_rx_burst_vec(void *dpdk_rxq, struct rte_mbuf **pkts, +uint16_t mlx5_rx_burst_vec(void *dpdk_txq, struct rte_mbuf **pkts, uint16_t pkts_n); /* mlx5_mr.c */ void mlx5_mp2mr_iter(struct rte_mempool *mp, void *arg); -struct mlx5_mr *priv_txq_mp2mr_reg(struct priv *priv, struct mlx5_txq_data *txq, - struct rte_mempool *mp, unsigned int idx); struct mlx5_mr *mlx5_txq_mp2mr_reg(struct mlx5_txq_data *txq, - struct rte_mempool *mp, - unsigned int idx); + struct rte_mempool *mp, unsigned int idx); #ifndef NDEBUG /** diff --git a/drivers/net/mlx5/mlx5_rxtx_vec.c b/drivers/net/mlx5/mlx5_rxtx_vec.c index b66c2916f5..257d7b11c6 100644 --- a/drivers/net/mlx5/mlx5_rxtx_vec.c +++ b/drivers/net/mlx5/mlx5_rxtx_vec.c @@ -223,17 +223,14 @@ mlx5_rx_burst_vec(void *dpdk_rxq, struct rte_mbuf **pkts, uint16_t pkts_n) /** * Check Tx queue flags are set for raw vectorized Tx. * - * @param priv - * Pointer to private structure. * @param dev - * Pointer to rte_eth_dev structure. + * Pointer to Ethernet device. * * @return * 1 if supported, negative errno value if not. */ int __attribute__((cold)) -priv_check_raw_vec_tx_support(__rte_unused struct priv *priv, - struct rte_eth_dev *dev) +mlx5_check_raw_vec_tx_support(struct rte_eth_dev *dev) { uint64_t offloads = dev->data->dev_conf.txmode.offloads; @@ -246,17 +243,16 @@ priv_check_raw_vec_tx_support(__rte_unused struct priv *priv, /** * Check a device can support vectorized TX. * - * @param priv - * Pointer to private structure. * @param dev - * Pointer to rte_eth_dev structure. + * Pointer to Ethernet device. * * @return * 1 if supported, negative errno value if not. */ int __attribute__((cold)) -priv_check_vec_tx_support(struct priv *priv, struct rte_eth_dev *dev) +mlx5_check_vec_tx_support(struct rte_eth_dev *dev) { + struct priv *priv = dev->data->dev_private; uint64_t offloads = dev->data->dev_conf.txmode.offloads; if (!priv->config.tx_vec_en || @@ -277,7 +273,7 @@ priv_check_vec_tx_support(struct priv *priv, struct rte_eth_dev *dev) * 1 if supported, negative errno value if not. */ int __attribute__((cold)) -rxq_check_vec_support(struct mlx5_rxq_data *rxq) +mlx5_rxq_check_vec_support(struct mlx5_rxq_data *rxq) { struct mlx5_rxq_ctrl *ctrl = container_of(rxq, struct mlx5_rxq_ctrl, rxq); @@ -290,15 +286,16 @@ rxq_check_vec_support(struct mlx5_rxq_data *rxq) /** * Check a device can support vectorized RX. * - * @param priv - * Pointer to private structure. + * @param dev + * Pointer to Ethernet device. * * @return * 1 if supported, negative errno value if not. */ int __attribute__((cold)) -priv_check_vec_rx_support(struct priv *priv) +mlx5_check_vec_rx_support(struct rte_eth_dev *dev) { + struct priv *priv = dev->data->dev_private; uint16_t i; if (!priv->config.rx_vec_en) @@ -309,7 +306,7 @@ priv_check_vec_rx_support(struct priv *priv) if (!rxq) continue; - if (rxq_check_vec_support(rxq) < 0) + if (mlx5_rxq_check_vec_support(rxq) < 0) break; } if (i != priv->rxqs_n) diff --git a/drivers/net/mlx5/mlx5_socket.c b/drivers/net/mlx5/mlx5_socket.c index 61c1a4a502..b8f610df30 100644 --- a/drivers/net/mlx5/mlx5_socket.c +++ b/drivers/net/mlx5/mlx5_socket.c @@ -18,15 +18,16 @@ /** * Initialise the socket to communicate with the secondary process * - * @param[in] priv - * Pointer to private structure. + * @param[in] dev + * Pointer to Ethernet device. * * @return * 0 on success, errno value on failure. */ int -priv_socket_init(struct priv *priv) +mlx5_socket_init(struct rte_eth_dev *dev) { + struct priv *priv = dev->data->dev_private; struct sockaddr_un sun = { .sun_family = AF_UNIX, }; @@ -79,15 +80,17 @@ out: /** * Un-Initialise the socket to communicate with the secondary process * - * @param[in] priv - * Pointer to private structure. + * @param[in] dev + * Pointer to Ethernet device. * * @return * 0 on success, errno value on failure. */ int -priv_socket_uninit(struct priv *priv) +mlx5_socket_uninit(struct rte_eth_dev *dev) { + struct priv *priv = dev->data->dev_private; + MKSTR(path, "/var/tmp/%s_%d", MLX5_DRIVER_NAME, priv->primary_socket); claim_zero(close(priv->primary_socket)); priv->primary_socket = 0; @@ -98,12 +101,13 @@ priv_socket_uninit(struct priv *priv) /** * Handle socket interrupts. * - * @param priv - * Pointer to private structure. + * @param dev + * Pointer to Ethernet device. */ void -priv_socket_handle(struct priv *priv) +mlx5_socket_handle(struct rte_eth_dev *dev) { + struct priv *priv = dev->data->dev_private; int conn_sock; int ret = 0; struct cmsghdr *cmsg = NULL; @@ -179,15 +183,16 @@ out: /** * Connect to the primary process. * - * @param[in] priv - * Pointer to private structure. + * @param[in] dev + * Pointer to Ethernet structure. * * @return * fd on success, negative errno value on failure. */ int -priv_socket_connect(struct priv *priv) +mlx5_socket_connect(struct rte_eth_dev *dev) { + struct priv *priv = dev->data->dev_private; struct sockaddr_un sun = { .sun_family = AF_UNIX, }; diff --git a/drivers/net/mlx5/mlx5_stats.c b/drivers/net/mlx5/mlx5_stats.c index 39be1865a8..0febed878a 100644 --- a/drivers/net/mlx5/mlx5_stats.c +++ b/drivers/net/mlx5/mlx5_stats.c @@ -122,8 +122,8 @@ static const unsigned int xstats_n = RTE_DIM(mlx5_counters_init); /** * Read device counters table. * - * @param priv - * Pointer to private structure. + * @param dev + * Pointer to Ethernet device. * @param[out] stats * Counters table output buffer. * @@ -131,8 +131,9 @@ static const unsigned int xstats_n = RTE_DIM(mlx5_counters_init); * 0 on success and stats is filled, negative on error. */ static int -priv_read_dev_counters(struct priv *priv, uint64_t *stats) +mlx5_read_dev_counters(struct rte_eth_dev *dev, uint64_t *stats) { + struct priv *priv = dev->data->dev_private; struct mlx5_xstats_ctrl *xstats_ctrl = &priv->xstats_ctrl; unsigned int i; struct ifreq ifr; @@ -143,7 +144,7 @@ priv_read_dev_counters(struct priv *priv, uint64_t *stats) et_stats->cmd = ETHTOOL_GSTATS; et_stats->n_stats = xstats_ctrl->stats_n; ifr.ifr_data = (caddr_t)et_stats; - if (priv_ifreq(priv, SIOCETHTOOL, &ifr) != 0) { + if (mlx5_ifreq(dev, SIOCETHTOOL, &ifr) != 0) { WARN("unable to read statistic values from device"); return -1; } @@ -173,20 +174,20 @@ priv_read_dev_counters(struct priv *priv, uint64_t *stats) /** * Query the number of statistics provided by ETHTOOL. * - * @param priv - * Pointer to private structure. + * @param dev + * Pointer to Ethernet device. * * @return * Number of statistics on success, -1 on error. */ static int -priv_ethtool_get_stats_n(struct priv *priv) { +mlx5_ethtool_get_stats_n(struct rte_eth_dev *dev) { struct ethtool_drvinfo drvinfo; struct ifreq ifr; drvinfo.cmd = ETHTOOL_GDRVINFO; ifr.ifr_data = (caddr_t)&drvinfo; - if (priv_ifreq(priv, SIOCETHTOOL, &ifr) != 0) { + if (mlx5_ifreq(dev, SIOCETHTOOL, &ifr) != 0) { WARN("unable to query number of statistics"); return -1; } @@ -196,12 +197,13 @@ priv_ethtool_get_stats_n(struct priv *priv) { /** * Init the structures to read device counters. * - * @param priv - * Pointer to private structure. + * @param dev + * Pointer to Ethernet device. */ void -priv_xstats_init(struct priv *priv) +mlx5_xstats_init(struct rte_eth_dev *dev) { + struct priv *priv = dev->data->dev_private; struct mlx5_xstats_ctrl *xstats_ctrl = &priv->xstats_ctrl; unsigned int i; unsigned int j; @@ -210,7 +212,7 @@ priv_xstats_init(struct priv *priv) unsigned int dev_stats_n; unsigned int str_sz; - dev_stats_n = priv_ethtool_get_stats_n(priv); + dev_stats_n = mlx5_ethtool_get_stats_n(dev); if (dev_stats_n < 1) { WARN("no extended statistics available"); return; @@ -229,7 +231,7 @@ priv_xstats_init(struct priv *priv) strings->string_set = ETH_SS_STATS; strings->len = dev_stats_n; ifr.ifr_data = (caddr_t)strings; - if (priv_ifreq(priv, SIOCETHTOOL, &ifr) != 0) { + if (mlx5_ifreq(dev, SIOCETHTOOL, &ifr) != 0) { WARN("unable to get statistic names"); goto free; } @@ -258,60 +260,54 @@ priv_xstats_init(struct priv *priv) } /* Copy to base at first time. */ assert(xstats_n <= MLX5_MAX_XSTATS); - priv_read_dev_counters(priv, xstats_ctrl->base); + mlx5_read_dev_counters(dev, xstats_ctrl->base); free: rte_free(strings); } /** - * Get device extended statistics. + * DPDK callback to get extended device statistics. * - * @param priv - * Pointer to private structure. + * @param dev + * Pointer to Ethernet device. * @param[out] stats * Pointer to rte extended stats table. + * @param n + * The size of the stats table. * * @return * Number of extended stats on success and stats is filled, * negative on error. */ -static int -priv_xstats_get(struct priv *priv, struct rte_eth_xstat *stats) +int +mlx5_xstats_get(struct rte_eth_dev *dev, struct rte_eth_xstat *stats, + unsigned int n) { - struct mlx5_xstats_ctrl *xstats_ctrl = &priv->xstats_ctrl; + struct priv *priv = dev->data->dev_private; unsigned int i; - unsigned int n = xstats_n; uint64_t counters[n]; + int ret = 0; - if (priv_read_dev_counters(priv, counters) < 0) - return -1; - for (i = 0; i != xstats_n; ++i) { - stats[i].id = i; - stats[i].value = (counters[i] - xstats_ctrl->base[i]); + if (n >= xstats_n && stats) { + struct mlx5_xstats_ctrl *xstats_ctrl = &priv->xstats_ctrl; + int stats_n; + + stats_n = mlx5_ethtool_get_stats_n(dev); + if (stats_n < 0) + return -1; + if (xstats_ctrl->stats_n != stats_n) + mlx5_xstats_init(dev); + ret = mlx5_read_dev_counters(dev, counters); + if (ret) + return ret; + for (i = 0; i != xstats_n; ++i) { + stats[i].id = i; + stats[i].value = (counters[i] - xstats_ctrl->base[i]); + } } return n; } -/** - * Reset device extended statistics. - * - * @param priv - * Pointer to private structure. - */ -static void -priv_xstats_reset(struct priv *priv) -{ - struct mlx5_xstats_ctrl *xstats_ctrl = &priv->xstats_ctrl; - unsigned int i; - unsigned int n = xstats_n; - uint64_t counters[n]; - - if (priv_read_dev_counters(priv, counters) < 0) - return; - for (i = 0; i != n; ++i) - xstats_ctrl->base[i] = counters[i]; -} - /** * DPDK callback to get device statistics. * @@ -408,41 +404,6 @@ mlx5_stats_reset(struct rte_eth_dev *dev) #endif } -/** - * DPDK callback to get extended device statistics. - * - * @param dev - * Pointer to Ethernet device structure. - * @param[out] stats - * Stats table output buffer. - * @param n - * The size of the stats table. - * - * @return - * Number of xstats on success, negative on failure. - */ -int -mlx5_xstats_get(struct rte_eth_dev *dev, - struct rte_eth_xstat *stats, unsigned int n) -{ - struct priv *priv = dev->data->dev_private; - int ret = xstats_n; - - if (n >= xstats_n && stats) { - struct mlx5_xstats_ctrl *xstats_ctrl = &priv->xstats_ctrl; - int stats_n; - - stats_n = priv_ethtool_get_stats_n(priv); - if (stats_n < 0) { - return -1; - } - if (xstats_ctrl->stats_n != stats_n) - priv_xstats_init(priv); - ret = priv_xstats_get(priv, stats); - } - return ret; -} - /** * DPDK callback to clear device extended statistics. * @@ -455,13 +416,19 @@ mlx5_xstats_reset(struct rte_eth_dev *dev) struct priv *priv = dev->data->dev_private; struct mlx5_xstats_ctrl *xstats_ctrl = &priv->xstats_ctrl; int stats_n; + unsigned int i; + unsigned int n = xstats_n; + uint64_t counters[n]; - stats_n = priv_ethtool_get_stats_n(priv); + stats_n = mlx5_ethtool_get_stats_n(dev); if (stats_n < 0) return; if (xstats_ctrl->stats_n != stats_n) - priv_xstats_init(priv); - priv_xstats_reset(priv); + mlx5_xstats_init(dev); + if (mlx5_read_dev_counters(dev, counters) < 0) + return; + for (i = 0; i != n; ++i) + xstats_ctrl->base[i] = counters[i]; } /** diff --git a/drivers/net/mlx5/mlx5_trigger.c b/drivers/net/mlx5/mlx5_trigger.c index 3ce93910d0..07226b8641 100644 --- a/drivers/net/mlx5/mlx5_trigger.c +++ b/drivers/net/mlx5/mlx5_trigger.c @@ -21,12 +21,13 @@ * Pointer to Ethernet device structure. */ static void -priv_txq_stop(struct priv *priv) +mlx5_txq_stop(struct rte_eth_dev *dev) { + struct priv *priv = dev->data->dev_private; unsigned int i; for (i = 0; i != priv->txqs_n; ++i) - mlx5_priv_txq_release(priv, i); + mlx5_txq_release(dev, i); } /** @@ -39,8 +40,9 @@ priv_txq_stop(struct priv *priv) * 0 on success, errno on error. */ static int -priv_txq_start(struct priv *priv) +mlx5_txq_start(struct rte_eth_dev *dev) { + struct priv *priv = dev->data->dev_private; unsigned int i; int ret = 0; @@ -48,28 +50,28 @@ priv_txq_start(struct priv *priv) for (i = 0; i != priv->txqs_n; ++i) { unsigned int idx = 0; struct mlx5_mr *mr; - struct mlx5_txq_ctrl *txq_ctrl = mlx5_priv_txq_get(priv, i); + struct mlx5_txq_ctrl *txq_ctrl = mlx5_txq_get(dev, i); if (!txq_ctrl) continue; LIST_FOREACH(mr, &priv->mr, next) { - priv_txq_mp2mr_reg(priv, &txq_ctrl->txq, mr->mp, idx++); + mlx5_txq_mp2mr_reg(&txq_ctrl->txq, mr->mp, idx++); if (idx == MLX5_PMD_TX_MP_CACHE) break; } txq_alloc_elts(txq_ctrl); - txq_ctrl->ibv = mlx5_priv_txq_ibv_new(priv, i); + txq_ctrl->ibv = mlx5_txq_ibv_new(dev, i); if (!txq_ctrl->ibv) { ret = ENOMEM; goto error; } } - ret = priv_tx_uar_remap(priv, priv->ctx->cmd_fd); + ret = mlx5_tx_uar_remap(dev, priv->ctx->cmd_fd); if (ret) goto error; return ret; error: - priv_txq_stop(priv); + mlx5_txq_stop(dev); return ret; } @@ -80,12 +82,13 @@ error: * Pointer to Ethernet device structure. */ static void -priv_rxq_stop(struct priv *priv) +mlx5_rxq_stop(struct rte_eth_dev *dev) { + struct priv *priv = dev->data->dev_private; unsigned int i; for (i = 0; i != priv->rxqs_n; ++i) - mlx5_priv_rxq_release(priv, i); + mlx5_rxq_release(dev, i); } /** @@ -98,20 +101,21 @@ priv_rxq_stop(struct priv *priv) * 0 on success, errno on error. */ static int -priv_rxq_start(struct priv *priv) +mlx5_rxq_start(struct rte_eth_dev *dev) { + struct priv *priv = dev->data->dev_private; unsigned int i; int ret = 0; for (i = 0; i != priv->rxqs_n; ++i) { - struct mlx5_rxq_ctrl *rxq_ctrl = mlx5_priv_rxq_get(priv, i); + struct mlx5_rxq_ctrl *rxq_ctrl = mlx5_rxq_get(dev, i); if (!rxq_ctrl) continue; ret = rxq_alloc_elts(rxq_ctrl); if (ret) goto error; - rxq_ctrl->ibv = mlx5_priv_rxq_ibv_new(priv, i); + rxq_ctrl->ibv = mlx5_rxq_ibv_new(dev, i); if (!rxq_ctrl->ibv) { ret = ENOMEM; goto error; @@ -119,7 +123,7 @@ priv_rxq_start(struct priv *priv) } return -ret; error: - priv_rxq_stop(priv); + mlx5_rxq_stop(dev); return -ret; } @@ -142,7 +146,7 @@ mlx5_dev_start(struct rte_eth_dev *dev) int err; dev->data->dev_started = 1; - err = priv_flow_create_drop_queue(priv); + err = mlx5_flow_create_drop_queue(dev); if (err) { ERROR("%p: Drop queue allocation failed: %s", (void *)dev, strerror(err)); @@ -150,46 +154,46 @@ mlx5_dev_start(struct rte_eth_dev *dev) } DEBUG("%p: allocating and configuring hash RX queues", (void *)dev); rte_mempool_walk(mlx5_mp2mr_iter, priv); - err = priv_txq_start(priv); + err = mlx5_txq_start(dev); if (err) { ERROR("%p: TXQ allocation failed: %s", (void *)dev, strerror(err)); goto error; } - err = priv_rxq_start(priv); + err = mlx5_rxq_start(dev); if (err) { ERROR("%p: RXQ allocation failed: %s", (void *)dev, strerror(err)); goto error; } - err = priv_rx_intr_vec_enable(priv); + err = mlx5_rx_intr_vec_enable(dev); if (err) { ERROR("%p: RX interrupt vector creation failed", (void *)priv); goto error; } - priv_xstats_init(priv); + mlx5_xstats_init(dev); /* Update link status and Tx/Rx callbacks for the first time. */ memset(&dev->data->dev_link, 0, sizeof(struct rte_eth_link)); INFO("Forcing port %u link to be up", dev->data->port_id); - err = priv_force_link_status_change(priv, ETH_LINK_UP); + err = mlx5_force_link_status_change(dev, ETH_LINK_UP); if (err) { DEBUG("Failed to set port %u link to be up", dev->data->port_id); goto error; } - priv_dev_interrupt_handler_install(priv, dev); + mlx5_dev_interrupt_handler_install(dev); return 0; error: /* Rollback. */ dev->data->dev_started = 0; for (mr = LIST_FIRST(&priv->mr); mr; mr = LIST_FIRST(&priv->mr)) - priv_mr_release(priv, mr); - priv_flow_stop(priv, &priv->flows); - priv_dev_traffic_disable(priv, dev); - priv_txq_stop(priv); - priv_rxq_stop(priv); - priv_flow_delete_drop_queue(priv); + mlx5_mr_release(mr); + mlx5_flow_stop(dev, &priv->flows); + mlx5_traffic_disable(dev); + mlx5_txq_stop(dev); + mlx5_rxq_stop(dev); + mlx5_flow_delete_drop_queue(dev); return err; } @@ -214,21 +218,21 @@ mlx5_dev_stop(struct rte_eth_dev *dev) rte_wmb(); usleep(1000 * priv->rxqs_n); DEBUG("%p: cleaning up and destroying hash RX queues", (void *)dev); - priv_flow_stop(priv, &priv->flows); - priv_dev_traffic_disable(priv, dev); - priv_rx_intr_vec_disable(priv); - priv_dev_interrupt_handler_uninstall(priv, dev); - priv_txq_stop(priv); - priv_rxq_stop(priv); + mlx5_flow_stop(dev, &priv->flows); + mlx5_traffic_disable(dev); + mlx5_rx_intr_vec_disable(dev); + mlx5_dev_interrupt_handler_uninstall(dev); + mlx5_txq_stop(dev); + mlx5_rxq_stop(dev); for (mr = LIST_FIRST(&priv->mr); mr; mr = LIST_FIRST(&priv->mr)) - priv_mr_release(priv, mr); - priv_flow_delete_drop_queue(priv); + mlx5_mr_release(mr); + mlx5_flow_delete_drop_queue(dev); } /** * Enable traffic flows configured by control plane * - * @param priv + * @param dev * Pointer to Ethernet device private data. * @param dev * Pointer to Ethernet device structure. @@ -237,8 +241,9 @@ mlx5_dev_stop(struct rte_eth_dev *dev) * 0 on success. */ int -priv_dev_traffic_enable(struct priv *priv, struct rte_eth_dev *dev) +mlx5_traffic_enable(struct rte_eth_dev *dev) { + struct priv *priv = dev->data->dev_private; struct rte_flow_item_eth bcast = { .dst.addr_bytes = "\xff\xff\xff\xff\xff\xff", }; @@ -356,40 +361,18 @@ error: /** * Disable traffic flows configured by control plane * - * @param priv - * Pointer to Ethernet device private data. * @param dev - * Pointer to Ethernet device structure. - * - * @return - * 0 on success. - */ -int -priv_dev_traffic_disable(struct priv *priv, - struct rte_eth_dev *dev __rte_unused) -{ - priv_flow_flush(priv, &priv->ctrl_flows); - return 0; -} - -/** - * Restart traffic flows configured by control plane - * - * @param priv * Pointer to Ethernet device private data. - * @param dev - * Pointer to Ethernet device structure. * * @return * 0 on success. */ int -priv_dev_traffic_restart(struct priv *priv, struct rte_eth_dev *dev) +mlx5_traffic_disable(struct rte_eth_dev *dev) { - if (dev->data->dev_started) { - priv_dev_traffic_disable(priv, dev); - priv_dev_traffic_enable(priv, dev); - } + struct priv *priv = dev->data->dev_private; + + mlx5_flow_list_flush(dev, &priv->ctrl_flows); return 0; } @@ -397,7 +380,7 @@ priv_dev_traffic_restart(struct priv *priv, struct rte_eth_dev *dev) * Restart traffic flows configured by control plane * * @param dev - * Pointer to Ethernet device structure. + * Pointer to Ethernet device private data. * * @return * 0 on success. @@ -405,8 +388,9 @@ priv_dev_traffic_restart(struct priv *priv, struct rte_eth_dev *dev) int mlx5_traffic_restart(struct rte_eth_dev *dev) { - struct priv *priv = dev->data->dev_private; - - priv_dev_traffic_restart(priv, dev); + if (dev->data->dev_started) { + mlx5_traffic_disable(dev); + mlx5_traffic_enable(dev); + } return 0; } diff --git a/drivers/net/mlx5/mlx5_txq.c b/drivers/net/mlx5/mlx5_txq.c index 47ee959901..54ed972d71 100644 --- a/drivers/net/mlx5/mlx5_txq.c +++ b/drivers/net/mlx5/mlx5_txq.c @@ -91,15 +91,16 @@ txq_free_elts(struct mlx5_txq_ctrl *txq_ctrl) /** * Returns the per-port supported offloads. * - * @param priv - * Pointer to private structure. + * @param dev + * Pointer to Ethernet device. * * @return * Supported Tx offloads. */ uint64_t -mlx5_priv_get_tx_port_offloads(struct priv *priv) +mlx5_get_tx_port_offloads(struct rte_eth_dev *dev) { + struct priv *priv = dev->data->dev_private; uint64_t offloads = (DEV_TX_OFFLOAD_MULTI_SEGS | DEV_TX_OFFLOAD_VLAN_INSERT); struct mlx5_dev_config *config = &priv->config; @@ -123,8 +124,8 @@ mlx5_priv_get_tx_port_offloads(struct priv *priv) /** * Checks if the per-queue offload configuration is valid. * - * @param priv - * Pointer to private structure. + * @param dev + * Pointer to Ethernet device. * @param offloads * Per-queue offloads configuration. * @@ -132,10 +133,10 @@ mlx5_priv_get_tx_port_offloads(struct priv *priv) * 1 if the configuration is valid, 0 otherwise. */ static int -priv_is_tx_queue_offloads_allowed(struct priv *priv, uint64_t offloads) +mlx5_is_tx_queue_offloads_allowed(struct rte_eth_dev *dev, uint64_t offloads) { - uint64_t port_offloads = priv->dev->data->dev_conf.txmode.offloads; - uint64_t port_supp_offloads = mlx5_priv_get_tx_port_offloads(priv); + uint64_t port_offloads = dev->data->dev_conf.txmode.offloads; + uint64_t port_supp_offloads = mlx5_get_tx_port_offloads(dev); /* There are no Tx offloads which are per queue. */ if ((offloads & port_supp_offloads) != offloads) @@ -177,13 +178,13 @@ mlx5_tx_queue_setup(struct rte_eth_dev *dev, uint16_t idx, uint16_t desc, * use the old API. */ if (!!(conf->txq_flags & ETH_TXQ_FLAGS_IGNORE) && - !priv_is_tx_queue_offloads_allowed(priv, conf->offloads)) { + !mlx5_is_tx_queue_offloads_allowed(dev, conf->offloads)) { ret = ENOTSUP; ERROR("%p: Tx queue offloads 0x%" PRIx64 " don't match port " "offloads 0x%" PRIx64 " or supported offloads 0x%" PRIx64, (void *)dev, conf->offloads, dev->data->dev_conf.txmode.offloads, - mlx5_priv_get_tx_port_offloads(priv)); + mlx5_get_tx_port_offloads(dev)); goto out; } if (desc <= MLX5_TX_COMP_THRESH) { @@ -206,14 +207,14 @@ mlx5_tx_queue_setup(struct rte_eth_dev *dev, uint16_t idx, uint16_t desc, (void *)dev, idx, priv->txqs_n); return -EOVERFLOW; } - if (!mlx5_priv_txq_releasable(priv, idx)) { + if (!mlx5_txq_releasable(dev, idx)) { ret = EBUSY; ERROR("%p: unable to release queue index %u", (void *)dev, idx); goto out; } - mlx5_priv_txq_release(priv, idx); - txq_ctrl = mlx5_priv_txq_new(priv, idx, desc, socket, conf); + mlx5_txq_release(dev, idx); + txq_ctrl = mlx5_txq_new(dev, idx, desc, socket, conf); if (!txq_ctrl) { ERROR("%p: unable to allocate queue index %u", (void *)dev, idx); @@ -249,7 +250,7 @@ mlx5_tx_queue_release(void *dpdk_txq) if ((*priv->txqs)[i] == txq) { DEBUG("%p: removing TX queue %p from list", (void *)priv->dev, (void *)txq_ctrl); - mlx5_priv_txq_release(priv, i); + mlx5_txq_release(priv->dev, i); break; } } @@ -260,8 +261,8 @@ mlx5_tx_queue_release(void *dpdk_txq) * Both primary and secondary process do mmap to make UAR address * aligned. * - * @param[in] priv - * Pointer to private structure. + * @param[in] dev + * Pointer to Ethernet device. * @param fd * Verbs file descriptor to map UAR pages. * @@ -269,8 +270,9 @@ mlx5_tx_queue_release(void *dpdk_txq) * 0 on success, errno value on failure. */ int -priv_tx_uar_remap(struct priv *priv, int fd) +mlx5_tx_uar_remap(struct rte_eth_dev *dev, int fd) { + struct priv *priv = dev->data->dev_private; unsigned int i, j; uintptr_t pages[priv->txqs_n]; unsigned int pages_n = 0; @@ -356,8 +358,8 @@ is_empw_burst_func(eth_tx_burst_t tx_pkt_burst) /** * Create the Tx queue Verbs object. * - * @param priv - * Pointer to private structure. + * @param dev + * Pointer to Ethernet device. * @param idx * Queue index in DPDK Rx queue array * @@ -365,8 +367,9 @@ is_empw_burst_func(eth_tx_burst_t tx_pkt_burst) * The Verbs object initialised if it can be created. */ struct mlx5_txq_ibv * -mlx5_priv_txq_ibv_new(struct priv *priv, uint16_t idx) +mlx5_txq_ibv_new(struct rte_eth_dev *dev, uint16_t idx) { + struct priv *priv = dev->data->dev_private; struct mlx5_txq_data *txq_data = (*priv->txqs)[idx]; struct mlx5_txq_ctrl *txq_ctrl = container_of(txq_data, struct mlx5_txq_ctrl, txq); @@ -383,7 +386,7 @@ mlx5_priv_txq_ibv_new(struct priv *priv, uint16_t idx) struct mlx5dv_cq cq_info; struct mlx5dv_obj obj; const int desc = 1 << txq_data->elts_n; - eth_tx_burst_t tx_pkt_burst = priv_select_tx_function(priv, priv->dev); + eth_tx_burst_t tx_pkt_burst = mlx5_select_tx_function(dev); int ret = 0; assert(txq_data); @@ -517,7 +520,7 @@ mlx5_priv_txq_ibv_new(struct priv *priv, uint16_t idx) ERROR("Failed to retrieve UAR info, invalid libmlx5.so version"); goto error; } - DEBUG("%p: Verbs Tx queue %p: refcnt %d", (void *)priv, + DEBUG("%p: Verbs Tx queue %p: refcnt %d", (void *)dev, (void *)txq_ibv, rte_atomic32_read(&txq_ibv->refcnt)); LIST_INSERT_HEAD(&priv->txqsibv, txq_ibv, next); priv->verbs_alloc_ctx.type = MLX5_VERBS_ALLOC_TYPE_NONE; @@ -534,8 +537,8 @@ error: /** * Get an Tx queue Verbs object. * - * @param priv - * Pointer to private structure. + * @param dev + * Pointer to Ethernet device. * @param idx * Queue index in DPDK Rx queue array * @@ -543,8 +546,9 @@ error: * The Verbs object if it exists. */ struct mlx5_txq_ibv * -mlx5_priv_txq_ibv_get(struct priv *priv, uint16_t idx) +mlx5_txq_ibv_get(struct rte_eth_dev *dev, uint16_t idx) { + struct priv *priv = dev->data->dev_private; struct mlx5_txq_ctrl *txq_ctrl; if (idx >= priv->txqs_n) @@ -554,7 +558,7 @@ mlx5_priv_txq_ibv_get(struct priv *priv, uint16_t idx) txq_ctrl = container_of((*priv->txqs)[idx], struct mlx5_txq_ctrl, txq); if (txq_ctrl->ibv) { rte_atomic32_inc(&txq_ctrl->ibv->refcnt); - DEBUG("%p: Verbs Tx queue %p: refcnt %d", (void *)priv, + DEBUG("%p: Verbs Tx queue %p: refcnt %d", (void *)dev, (void *)txq_ctrl->ibv, rte_atomic32_read(&txq_ctrl->ibv->refcnt)); } @@ -564,8 +568,6 @@ mlx5_priv_txq_ibv_get(struct priv *priv, uint16_t idx) /** * Release an Tx verbs queue object. * - * @param priv - * Pointer to private structure. * @param txq_ibv * Verbs Tx queue object. * @@ -573,11 +575,10 @@ mlx5_priv_txq_ibv_get(struct priv *priv, uint16_t idx) * 0 on success, errno on failure. */ int -mlx5_priv_txq_ibv_release(struct priv *priv __rte_unused, - struct mlx5_txq_ibv *txq_ibv) +mlx5_txq_ibv_release(struct mlx5_txq_ibv *txq_ibv) { assert(txq_ibv); - DEBUG("%p: Verbs Tx queue %p: refcnt %d", (void *)priv, + DEBUG("Verbs Tx queue %p: refcnt %d", (void *)txq_ibv, rte_atomic32_read(&txq_ibv->refcnt)); if (rte_atomic32_dec_and_test(&txq_ibv->refcnt)) { claim_zero(mlx5_glue->destroy_qp(txq_ibv->qp)); @@ -592,14 +593,11 @@ mlx5_priv_txq_ibv_release(struct priv *priv __rte_unused, /** * Return true if a single reference exists on the object. * - * @param priv - * Pointer to private structure. * @param txq_ibv * Verbs Tx queue object. */ int -mlx5_priv_txq_ibv_releasable(struct priv *priv __rte_unused, - struct mlx5_txq_ibv *txq_ibv) +mlx5_txq_ibv_releasable(struct mlx5_txq_ibv *txq_ibv) { assert(txq_ibv); return (rte_atomic32_read(&txq_ibv->refcnt) == 1); @@ -608,20 +606,21 @@ mlx5_priv_txq_ibv_releasable(struct priv *priv __rte_unused, /** * Verify the Verbs Tx queue list is empty * - * @param priv - * Pointer to private structure. + * @param dev + * Pointer to Ethernet device. * * @return * The number of object not released. */ int -mlx5_priv_txq_ibv_verify(struct priv *priv) +mlx5_txq_ibv_verify(struct rte_eth_dev *dev) { + struct priv *priv = dev->data->dev_private; int ret = 0; struct mlx5_txq_ibv *txq_ibv; LIST_FOREACH(txq_ibv, &priv->txqsibv, next) { - DEBUG("%p: Verbs Tx queue %p still referenced", (void *)priv, + DEBUG("%p: Verbs Tx queue %p still referenced", (void *)dev, (void *)txq_ibv); ++ret; } @@ -645,7 +644,8 @@ txq_set_params(struct mlx5_txq_ctrl *txq_ctrl) unsigned int txq_inline; unsigned int txqs_inline; unsigned int inline_max_packet_sz; - eth_tx_burst_t tx_pkt_burst = priv_select_tx_function(priv, priv->dev); + eth_tx_burst_t tx_pkt_burst = + mlx5_select_tx_function(txq_ctrl->priv->dev); int is_empw_func = is_empw_burst_func(tx_pkt_burst); int tso = !!(txq_ctrl->txq.offloads & DEV_TX_OFFLOAD_TCP_TSO); @@ -731,8 +731,8 @@ txq_set_params(struct mlx5_txq_ctrl *txq_ctrl) /** * Create a DPDK Tx queue. * - * @param priv - * Pointer to private structure. + * @param dev + * Pointer to Ethernet device. * @param idx * TX queue index. * @param desc @@ -746,10 +746,10 @@ txq_set_params(struct mlx5_txq_ctrl *txq_ctrl) * A DPDK queue object on success. */ struct mlx5_txq_ctrl * -mlx5_priv_txq_new(struct priv *priv, uint16_t idx, uint16_t desc, - unsigned int socket, - const struct rte_eth_txconf *conf) +mlx5_txq_new(struct rte_eth_dev *dev, uint16_t idx, uint16_t desc, + unsigned int socket, const struct rte_eth_txconf *conf) { + struct priv *priv = dev->data->dev_private; struct mlx5_txq_ctrl *tmpl; tmpl = rte_calloc_socket("TXQ", 1, @@ -773,7 +773,7 @@ mlx5_priv_txq_new(struct priv *priv, uint16_t idx, uint16_t desc, (struct rte_mbuf *(*)[1 << tmpl->txq.elts_n])(tmpl + 1); tmpl->txq.stats.idx = idx; rte_atomic32_inc(&tmpl->refcnt); - DEBUG("%p: Tx queue %p: refcnt %d", (void *)priv, + DEBUG("%p: Tx queue %p: refcnt %d", (void *)dev, (void *)tmpl, rte_atomic32_read(&tmpl->refcnt)); LIST_INSERT_HEAD(&priv->txqsctrl, tmpl, next); return tmpl; @@ -782,8 +782,8 @@ mlx5_priv_txq_new(struct priv *priv, uint16_t idx, uint16_t desc, /** * Get a Tx queue. * - * @param priv - * Pointer to private structure. + * @param dev + * Pointer to Ethernet device. * @param idx * TX queue index. * @@ -791,8 +791,9 @@ mlx5_priv_txq_new(struct priv *priv, uint16_t idx, uint16_t desc, * A pointer to the queue if it exists. */ struct mlx5_txq_ctrl * -mlx5_priv_txq_get(struct priv *priv, uint16_t idx) +mlx5_txq_get(struct rte_eth_dev *dev, uint16_t idx) { + struct priv *priv = dev->data->dev_private; struct mlx5_txq_ctrl *ctrl = NULL; if ((*priv->txqs)[idx]) { @@ -800,15 +801,15 @@ mlx5_priv_txq_get(struct priv *priv, uint16_t idx) txq); unsigned int i; - mlx5_priv_txq_ibv_get(priv, idx); + mlx5_txq_ibv_get(dev, idx); for (i = 0; i != MLX5_PMD_TX_MP_CACHE; ++i) { if (ctrl->txq.mp2mr[i]) claim_nonzero - (priv_mr_get(priv, + (mlx5_mr_get(dev, ctrl->txq.mp2mr[i]->mp)); } rte_atomic32_inc(&ctrl->refcnt); - DEBUG("%p: Tx queue %p: refcnt %d", (void *)priv, + DEBUG("%p: Tx queue %p: refcnt %d", (void *)dev, (void *)ctrl, rte_atomic32_read(&ctrl->refcnt)); } return ctrl; @@ -817,8 +818,8 @@ mlx5_priv_txq_get(struct priv *priv, uint16_t idx) /** * Release a Tx queue. * - * @param priv - * Pointer to private structure. + * @param dev + * Pointer to Ethernet device. * @param idx * TX queue index. * @@ -826,8 +827,9 @@ mlx5_priv_txq_get(struct priv *priv, uint16_t idx) * 0 on success, errno on failure. */ int -mlx5_priv_txq_release(struct priv *priv, uint16_t idx) +mlx5_txq_release(struct rte_eth_dev *dev, uint16_t idx) { + struct priv *priv = dev->data->dev_private; unsigned int i; struct mlx5_txq_ctrl *txq; size_t page_size = sysconf(_SC_PAGESIZE); @@ -835,18 +837,18 @@ mlx5_priv_txq_release(struct priv *priv, uint16_t idx) if (!(*priv->txqs)[idx]) return 0; txq = container_of((*priv->txqs)[idx], struct mlx5_txq_ctrl, txq); - DEBUG("%p: Tx queue %p: refcnt %d", (void *)priv, + DEBUG("%p: Tx queue %p: refcnt %d", (void *)dev, (void *)txq, rte_atomic32_read(&txq->refcnt)); if (txq->ibv) { int ret; - ret = mlx5_priv_txq_ibv_release(priv, txq->ibv); + ret = mlx5_txq_ibv_release(txq->ibv); if (!ret) txq->ibv = NULL; } for (i = 0; i != MLX5_PMD_TX_MP_CACHE; ++i) { if (txq->txq.mp2mr[i]) { - priv_mr_release(priv, txq->txq.mp2mr[i]); + mlx5_mr_release(txq->txq.mp2mr[i]); txq->txq.mp2mr[i] = NULL; } } @@ -866,8 +868,8 @@ mlx5_priv_txq_release(struct priv *priv, uint16_t idx) /** * Verify if the queue can be released. * - * @param priv - * Pointer to private structure. + * @param dev + * Pointer to Ethernet device. * @param idx * TX queue index. * @@ -875,8 +877,9 @@ mlx5_priv_txq_release(struct priv *priv, uint16_t idx) * 1 if the queue can be released. */ int -mlx5_priv_txq_releasable(struct priv *priv, uint16_t idx) +mlx5_txq_releasable(struct rte_eth_dev *dev, uint16_t idx) { + struct priv *priv = dev->data->dev_private; struct mlx5_txq_ctrl *txq; if (!(*priv->txqs)[idx]) @@ -888,20 +891,21 @@ mlx5_priv_txq_releasable(struct priv *priv, uint16_t idx) /** * Verify the Tx Queue list is empty * - * @param priv - * Pointer to private structure. + * @param dev + * Pointer to Ethernet device. * * @return * The number of object not released. */ int -mlx5_priv_txq_verify(struct priv *priv) +mlx5_txq_verify(struct rte_eth_dev *dev) { + struct priv *priv = dev->data->dev_private; struct mlx5_txq_ctrl *txq; int ret = 0; LIST_FOREACH(txq, &priv->txqsctrl, next) { - DEBUG("%p: Tx Queue %p still referenced", (void *)priv, + DEBUG("%p: Tx Queue %p still referenced", (void *)dev, (void *)txq); ++ret; } diff --git a/drivers/net/mlx5/mlx5_vlan.c b/drivers/net/mlx5/mlx5_vlan.c index 184ae2f4e6..3df962a904 100644 --- a/drivers/net/mlx5/mlx5_vlan.c +++ b/drivers/net/mlx5/mlx5_vlan.c @@ -79,25 +79,26 @@ mlx5_vlan_filter_set(struct rte_eth_dev *dev, uint16_t vlan_id, int on) ++priv->vlan_filter_n; } if (dev->data->dev_started) - priv_dev_traffic_restart(priv, dev); + mlx5_traffic_restart(dev); out: return ret; } /** - * Set/reset VLAN stripping for a specific queue. + * Callback to set/reset VLAN stripping for a specific queue. * - * @param priv - * Pointer to private structure. - * @param idx + * @param dev + * Pointer to Ethernet device structure. + * @param queue * RX queue index. * @param on * Enable/disable VLAN stripping. */ -static void -priv_vlan_strip_queue_set(struct priv *priv, uint16_t idx, int on) +void +mlx5_vlan_strip_queue_set(struct rte_eth_dev *dev, uint16_t queue, int on) { - struct mlx5_rxq_data *rxq = (*priv->rxqs)[idx]; + struct priv *priv = dev->data->dev_private; + struct mlx5_rxq_data *rxq = (*priv->rxqs)[queue]; struct mlx5_rxq_ctrl *rxq_ctrl = container_of(rxq, struct mlx5_rxq_ctrl, rxq); struct ibv_wq_attr mod; @@ -106,8 +107,18 @@ priv_vlan_strip_queue_set(struct priv *priv, uint16_t idx, int on) 0; int err; + /* Validate hw support */ + if (!priv->config.hw_vlan_strip) { + ERROR("VLAN stripping is not supported"); + return; + } + /* Validate queue number */ + if (queue >= priv->rxqs_n) { + ERROR("VLAN stripping, invalid queue number %d", queue); + return; + } DEBUG("set VLAN offloads 0x%x for port %d queue %d", - vlan_offloads, rxq->port_id, idx); + vlan_offloads, rxq->port_id, queue); if (!rxq_ctrl->ibv) { /* Update related bits in RX queue. */ rxq->vlan_strip = !!on; @@ -121,41 +132,13 @@ priv_vlan_strip_queue_set(struct priv *priv, uint16_t idx, int on) err = mlx5_glue->modify_wq(rxq_ctrl->ibv->wq, &mod); if (err) { ERROR("%p: failed to modified stripping mode: %s", - (void *)priv, strerror(err)); + (void *)dev, strerror(err)); return; } /* Update related bits in RX queue. */ rxq->vlan_strip = !!on; } -/** - * Callback to set/reset VLAN stripping for a specific queue. - * - * @param dev - * Pointer to Ethernet device structure. - * @param queue - * RX queue index. - * @param on - * Enable/disable VLAN stripping. - */ -void -mlx5_vlan_strip_queue_set(struct rte_eth_dev *dev, uint16_t queue, int on) -{ - struct priv *priv = dev->data->dev_private; - - /* Validate hw support */ - if (!priv->config.hw_vlan_strip) { - ERROR("VLAN stripping is not supported"); - return; - } - /* Validate queue number */ - if (queue >= priv->rxqs_n) { - ERROR("VLAN stripping, invalid queue number %d", queue); - return; - } - priv_vlan_strip_queue_set(priv, queue, on); -} - /** * Callback to set/reset VLAN offloads for a port. * @@ -180,7 +163,7 @@ mlx5_vlan_offload_set(struct rte_eth_dev *dev, int mask) } /* Run on every RX queue and set/reset VLAN stripping. */ for (i = 0; (i != priv->rxqs_n); i++) - priv_vlan_strip_queue_set(priv, i, hw_vlan_strip); + mlx5_vlan_strip_queue_set(dev, i, hw_vlan_strip); } return 0; } -- 2.20.1