X-Git-Url: http://git.droids-corp.org/?a=blobdiff_plain;f=drivers%2Fnet%2Fmlx5%2Fmlx5_ethdev.c;h=1fde3d842e33379326fd9d023cad2d9126756262;hb=af4f09f28294fac762ff413fbf14b48c42c128fd;hp=318bc9db2ba549fa2015134b57cd43d0615561cf;hpb=87ec44ce1651f82effa1e308cb0c70307908d154;p=dpdk.git diff --git a/drivers/net/mlx5/mlx5_ethdev.c b/drivers/net/mlx5/mlx5_ethdev.c index 318bc9db2b..1fde3d842e 100644 --- a/drivers/net/mlx5/mlx5_ethdev.c +++ b/drivers/net/mlx5/mlx5_ethdev.c @@ -1,40 +1,13 @@ -/*- - * BSD LICENSE - * - * Copyright 2015 6WIND S.A. - * Copyright 2015 Mellanox. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * - * * Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in - * the documentation and/or other materials provided with the - * distribution. - * * Neither the name of 6WIND S.A. nor the names of its - * contributors may be used to endorse or promote products derived - * from this software without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright 2015 6WIND S.A. + * Copyright 2015 Mellanox. */ #define _GNU_SOURCE #include #include +#include #include #include #include @@ -55,7 +28,8 @@ #include #include -#include +#include +#include #include #include #include @@ -63,6 +37,7 @@ #include #include "mlx5.h" +#include "mlx5_glue.h" #include "mlx5_rxtx.h" #include "mlx5_utils.h" @@ -117,38 +92,11 @@ struct ethtool_link_settings { #define ETHTOOL_LINK_MODE_100000baseLR4_ER4_Full_BIT 39 #endif -/** - * Return private structure associated with an Ethernet device. - * - * @param dev - * Pointer to Ethernet device structure. - * - * @return - * Pointer to private structure. - */ -struct priv * -mlx5_get_priv(struct rte_eth_dev *dev) -{ - return dev->data->dev_private; -} - -/** - * Check if running as a secondary process. - * - * @return - * Nonzero if running as a secondary process. - */ -inline int -mlx5_is_secondary(void) -{ - return rte_eal_process_type() == RTE_PROC_SECONDARY; -} - /** * Get interface name from private structure. * - * @param[in] priv - * Pointer to private structure. + * @param[in] dev + * Pointer to Ethernet device. * @param[out] ifname * Interface name output buffer. * @@ -156,8 +104,9 @@ mlx5_is_secondary(void) * 0 on success, -1 on failure and errno is set. */ int -priv_get_ifname(const struct priv *priv, char (*ifname)[IF_NAMESIZE]) +mlx5_get_ifname(const struct rte_eth_dev *dev, char (*ifname)[IF_NAMESIZE]) { + struct priv *priv = dev->data->dev_private; DIR *dir; struct dirent *dent; unsigned int dev_type = 0; @@ -225,186 +174,11 @@ try_dev_id: return 0; } -/** - * Check if the counter is located on ib counters file. - * - * @param[in] cntr - * Counter name. - * - * @return - * 1 if counter is located on ib counters file , 0 otherwise. - */ -int -priv_is_ib_cntr(const char *cntr) -{ - if (!strcmp(cntr, "out_of_buffer")) - return 1; - return 0; -} - -/** - * Read from sysfs entry. - * - * @param[in] priv - * Pointer to private structure. - * @param[in] entry - * Entry name relative to sysfs path. - * @param[out] buf - * Data output buffer. - * @param size - * Buffer size. - * - * @return - * 0 on success, -1 on failure and errno is set. - */ -static int -priv_sysfs_read(const struct priv *priv, const char *entry, - char *buf, size_t size) -{ - char ifname[IF_NAMESIZE]; - FILE *file; - int ret; - int err; - - if (priv_get_ifname(priv, &ifname)) - return -1; - - if (priv_is_ib_cntr(entry)) { - MKSTR(path, "%s/ports/1/hw_counters/%s", - priv->ibdev_path, entry); - file = fopen(path, "rb"); - } else { - MKSTR(path, "%s/device/net/%s/%s", - priv->ibdev_path, ifname, entry); - file = fopen(path, "rb"); - } - if (file == NULL) - return -1; - ret = fread(buf, 1, size, file); - err = errno; - if (((size_t)ret < size) && (ferror(file))) - ret = -1; - else - ret = size; - fclose(file); - errno = err; - return ret; -} - -/** - * Write to sysfs entry. - * - * @param[in] priv - * Pointer to private structure. - * @param[in] entry - * Entry name relative to sysfs path. - * @param[in] buf - * Data buffer. - * @param size - * Buffer size. - * - * @return - * 0 on success, -1 on failure and errno is set. - */ -static int -priv_sysfs_write(const struct priv *priv, const char *entry, - char *buf, size_t size) -{ - char ifname[IF_NAMESIZE]; - FILE *file; - int ret; - int err; - - if (priv_get_ifname(priv, &ifname)) - return -1; - - MKSTR(path, "%s/device/net/%s/%s", priv->ibdev_path, ifname, entry); - - file = fopen(path, "wb"); - if (file == NULL) - return -1; - ret = fwrite(buf, 1, size, file); - err = errno; - if (((size_t)ret < size) || (ferror(file))) - ret = -1; - else - ret = size; - fclose(file); - errno = err; - return ret; -} - -/** - * Get unsigned long sysfs property. - * - * @param priv - * Pointer to private structure. - * @param[in] name - * Entry name relative to sysfs path. - * @param[out] value - * Value output buffer. - * - * @return - * 0 on success, -1 on failure and errno is set. - */ -static int -priv_get_sysfs_ulong(struct priv *priv, const char *name, unsigned long *value) -{ - int ret; - unsigned long value_ret; - char value_str[32]; - - ret = priv_sysfs_read(priv, name, value_str, (sizeof(value_str) - 1)); - if (ret == -1) { - DEBUG("cannot read %s value from sysfs: %s", - name, strerror(errno)); - return -1; - } - value_str[ret] = '\0'; - errno = 0; - value_ret = strtoul(value_str, NULL, 0); - if (errno) { - DEBUG("invalid %s value `%s': %s", name, value_str, - strerror(errno)); - return -1; - } - *value = value_ret; - return 0; -} - -/** - * Set unsigned long sysfs property. - * - * @param priv - * Pointer to private structure. - * @param[in] name - * Entry name relative to sysfs path. - * @param value - * Value to set. - * - * @return - * 0 on success, -1 on failure and errno is set. - */ -static int -priv_set_sysfs_ulong(struct priv *priv, const char *name, unsigned long value) -{ - int ret; - MKSTR(value_str, "%lu", value); - - ret = priv_sysfs_write(priv, name, value_str, (sizeof(value_str) - 1)); - if (ret == -1) { - DEBUG("cannot write %s `%s' (%lu) to sysfs: %s", - name, value_str, value, strerror(errno)); - return -1; - } - return 0; -} - /** * Perform ifreq ioctl() on associated Ethernet device. * - * @param[in] priv - * Pointer to private structure. + * @param[in] dev + * Pointer to Ethernet device. * @param req * Request number to pass to ioctl(). * @param[out] ifr @@ -414,56 +188,24 @@ priv_set_sysfs_ulong(struct priv *priv, const char *name, unsigned long value) * 0 on success, -1 on failure and errno is set. */ int -priv_ifreq(const struct priv *priv, int req, struct ifreq *ifr) +mlx5_ifreq(const struct rte_eth_dev *dev, int req, struct ifreq *ifr) { int sock = socket(PF_INET, SOCK_DGRAM, IPPROTO_IP); int ret = -1; if (sock == -1) return ret; - if (priv_get_ifname(priv, &ifr->ifr_name) == 0) + if (mlx5_get_ifname(dev, &ifr->ifr_name) == 0) ret = ioctl(sock, req, ifr); close(sock); return ret; } -/** - * Return the number of active VFs for the current device. - * - * @param[in] priv - * Pointer to private structure. - * @param[out] num_vfs - * Number of active VFs. - * - * @return - * 0 on success, -1 on failure and errno is set. - */ -int -priv_get_num_vfs(struct priv *priv, uint16_t *num_vfs) -{ - /* The sysfs entry name depends on the operating system. */ - const char **name = (const char *[]){ - "device/sriov_numvfs", - "device/mlx5_num_vfs", - NULL, - }; - int ret; - - do { - unsigned long ulong_num_vfs; - - ret = priv_get_sysfs_ulong(priv, *name, &ulong_num_vfs); - if (!ret) - *num_vfs = ulong_num_vfs; - } while (*(++name) && ret); - return ret; -} - /** * Get device MTU. * - * @param priv - * Pointer to private structure. + * @param dev + * Pointer to Ethernet device. * @param[out] mtu * MTU value output buffer. * @@ -471,45 +213,22 @@ priv_get_num_vfs(struct priv *priv, uint16_t *num_vfs) * 0 on success, -1 on failure and errno is set. */ int -priv_get_mtu(struct priv *priv, uint16_t *mtu) -{ - unsigned long ulong_mtu; - - if (priv_get_sysfs_ulong(priv, "mtu", &ulong_mtu) == -1) - return -1; - *mtu = ulong_mtu; - return 0; -} - -/** - * Read device counter from sysfs. - * - * @param priv - * Pointer to private structure. - * @param name - * Counter name. - * @param[out] cntr - * Counter output buffer. - * - * @return - * 0 on success, -1 on failure and errno is set. - */ -int -priv_get_cntr_sysfs(struct priv *priv, const char *name, uint64_t *cntr) +mlx5_get_mtu(struct rte_eth_dev *dev, uint16_t *mtu) { - unsigned long ulong_ctr; + struct ifreq request; + int ret = mlx5_ifreq(dev, SIOCGIFMTU, &request); - if (priv_get_sysfs_ulong(priv, name, &ulong_ctr) == -1) - return -1; - *cntr = ulong_ctr; + if (ret) + return ret; + *mtu = request.ifr_mtu; return 0; } /** * Set device MTU. * - * @param priv - * Pointer to private structure. + * @param dev + * Pointer to Ethernet device. * @param mtu * MTU value to set. * @@ -517,24 +236,18 @@ priv_get_cntr_sysfs(struct priv *priv, const char *name, uint64_t *cntr) * 0 on success, -1 on failure and errno is set. */ static int -priv_set_mtu(struct priv *priv, uint16_t mtu) +mlx5_set_mtu(struct rte_eth_dev *dev, uint16_t mtu) { - uint16_t new_mtu; + struct ifreq request = { .ifr_mtu = mtu, }; - if (priv_set_sysfs_ulong(priv, "mtu", mtu) || - priv_get_mtu(priv, &new_mtu)) - return -1; - if (new_mtu == mtu) - return 0; - errno = EINVAL; - return -1; + return mlx5_ifreq(dev, SIOCSIFMTU, &request); } /** * Set device flags. * - * @param priv - * Pointer to private structure. + * @param dev + * Pointer to Ethernet device. * @param keep * Bitmask for flags that must remain untouched. * @param flags @@ -544,30 +257,29 @@ priv_set_mtu(struct priv *priv, uint16_t mtu) * 0 on success, -1 on failure and errno is set. */ int -priv_set_flags(struct priv *priv, unsigned int keep, unsigned int flags) +mlx5_set_flags(struct rte_eth_dev *dev, unsigned int keep, unsigned int flags) { - unsigned long tmp; + struct ifreq request; + int ret = mlx5_ifreq(dev, SIOCGIFFLAGS, &request); - if (priv_get_sysfs_ulong(priv, "flags", &tmp) == -1) - return -1; - tmp &= keep; - tmp |= (flags & (~keep)); - return priv_set_sysfs_ulong(priv, "flags", tmp); + if (ret) + return ret; + request.ifr_flags &= keep; + request.ifr_flags |= flags & ~keep; + return mlx5_ifreq(dev, SIOCSIFFLAGS, &request); } /** - * Ethernet device configuration. - * - * Prepare the driver for a given number of TX and RX queues. + * DPDK callback for Ethernet device configuration. * * @param dev * Pointer to Ethernet device structure. * * @return - * 0 on success, errno value on failure. + * 0 on success, negative errno value on failure. */ -static int -dev_configure(struct rte_eth_dev *dev) +int +mlx5_dev_configure(struct rte_eth_dev *dev) { struct priv *priv = dev->data->dev_private; unsigned int rxqs_n = dev->data->nb_rx_queues; @@ -575,8 +287,47 @@ dev_configure(struct rte_eth_dev *dev) unsigned int i; unsigned int j; unsigned int reta_idx_n; - - priv->rss_hf = dev->data->dev_conf.rx_adv_conf.rss_conf.rss_hf; + const uint8_t use_app_rss_key = + !!dev->data->dev_conf.rx_adv_conf.rss_conf.rss_key; + uint64_t supp_tx_offloads = mlx5_get_tx_port_offloads(dev); + uint64_t tx_offloads = dev->data->dev_conf.txmode.offloads; + uint64_t supp_rx_offloads = + (mlx5_get_rx_port_offloads() | + mlx5_get_rx_queue_offloads(dev)); + uint64_t rx_offloads = dev->data->dev_conf.rxmode.offloads; + + if ((tx_offloads & supp_tx_offloads) != tx_offloads) { + ERROR("Some Tx offloads are not supported " + "requested 0x%" PRIx64 " supported 0x%" PRIx64, + tx_offloads, supp_tx_offloads); + return ENOTSUP; + } + if ((rx_offloads & supp_rx_offloads) != rx_offloads) { + ERROR("Some Rx offloads are not supported " + "requested 0x%" PRIx64 " supported 0x%" PRIx64, + rx_offloads, supp_rx_offloads); + return ENOTSUP; + } + if (use_app_rss_key && + (dev->data->dev_conf.rx_adv_conf.rss_conf.rss_key_len != + rss_hash_default_key_len)) { + /* MLX5 RSS only support 40bytes key. */ + return EINVAL; + } + priv->rss_conf.rss_key = + rte_realloc(priv->rss_conf.rss_key, + rss_hash_default_key_len, 0); + if (!priv->rss_conf.rss_key) { + ERROR("cannot allocate RSS hash key memory (%u)", rxqs_n); + return ENOMEM; + } + memcpy(priv->rss_conf.rss_key, + use_app_rss_key ? + dev->data->dev_conf.rx_adv_conf.rss_conf.rss_key : + rss_hash_default_key, + rss_hash_default_key_len); + priv->rss_conf.rss_key_len = rss_hash_default_key_len; + priv->rss_conf.rss_hf = dev->data->dev_conf.rx_adv_conf.rss_conf.rss_hf; priv->rxqs = (void *)dev->data->rx_queues; priv->txqs = (void *)dev->data->tx_queues; if (txqs_n != priv->txqs_n) { @@ -584,7 +335,7 @@ dev_configure(struct rte_eth_dev *dev) (void *)dev, priv->txqs_n, txqs_n); priv->txqs_n = txqs_n; } - if (rxqs_n > priv->ind_table_max_size) { + if (rxqs_n > priv->config.ind_table_max_size) { ERROR("cannot handle this many RX queues (%u)", rxqs_n); return EINVAL; } @@ -597,9 +348,9 @@ dev_configure(struct rte_eth_dev *dev) * maximum indirection table size for better balancing. * The result is always rounded to the next power of two. */ reta_idx_n = (1 << log2above((rxqs_n & (rxqs_n - 1)) ? - priv->ind_table_max_size : + priv->config.ind_table_max_size : rxqs_n)); - if (priv_rss_reta_index_resize(priv, reta_idx_n)) + if (mlx5_rss_reta_index_resize(dev, reta_idx_n)) return ENOMEM; /* When the number of RX queues is not a power of two, the remaining * table entries are padded with reused WQs and hashes are not spread @@ -610,31 +361,7 @@ dev_configure(struct rte_eth_dev *dev) j = 0; } return 0; -} -/** - * DPDK callback for Ethernet device configuration. - * - * @param dev - * Pointer to Ethernet device structure. - * - * @return - * 0 on success, negative errno value on failure. - */ -int -mlx5_dev_configure(struct rte_eth_dev *dev) -{ - struct priv *priv = dev->data->dev_private; - int ret; - - if (mlx5_is_secondary()) - return -E_RTE_SECONDARY; - - priv_lock(priv); - ret = dev_configure(dev); - assert(ret >= 0); - priv_unlock(priv); - return -ret; } /** @@ -648,13 +375,12 @@ mlx5_dev_configure(struct rte_eth_dev *dev) void mlx5_dev_infos_get(struct rte_eth_dev *dev, struct rte_eth_dev_info *info) { - struct priv *priv = mlx5_get_priv(dev); + struct priv *priv = dev->data->dev_private; + struct mlx5_dev_config *config = &priv->config; unsigned int max; char ifname[IF_NAMESIZE]; info->pci_dev = RTE_ETH_DEV_TO_PCI(dev); - - priv_lock(priv); /* FIXME: we should ask the device for these values. */ info->min_rx_bufsize = 32; info->max_rx_pktlen = 65536; @@ -670,37 +396,28 @@ mlx5_dev_infos_get(struct rte_eth_dev *dev, struct rte_eth_dev_info *info) info->max_rx_queues = max; info->max_tx_queues = max; info->max_mac_addrs = RTE_DIM(priv->mac); - info->rx_offload_capa = - (priv->hw_csum ? - (DEV_RX_OFFLOAD_IPV4_CKSUM | - DEV_RX_OFFLOAD_UDP_CKSUM | - DEV_RX_OFFLOAD_TCP_CKSUM) : - 0) | - (priv->hw_vlan_strip ? DEV_RX_OFFLOAD_VLAN_STRIP : 0); - if (!priv->mps) - info->tx_offload_capa = DEV_TX_OFFLOAD_VLAN_INSERT; - if (priv->hw_csum) - info->tx_offload_capa |= - (DEV_TX_OFFLOAD_IPV4_CKSUM | - DEV_TX_OFFLOAD_UDP_CKSUM | - DEV_TX_OFFLOAD_TCP_CKSUM); - if (priv->tso) - info->tx_offload_capa |= DEV_TX_OFFLOAD_TCP_TSO; - if (priv->tunnel_en) - info->tx_offload_capa |= (DEV_TX_OFFLOAD_OUTER_IPV4_CKSUM | - DEV_TX_OFFLOAD_VXLAN_TNL_TSO | - DEV_TX_OFFLOAD_GRE_TNL_TSO); - if (priv_get_ifname(priv, &ifname) == 0) + info->rx_queue_offload_capa = mlx5_get_rx_queue_offloads(dev); + info->rx_offload_capa = (mlx5_get_rx_port_offloads() | + info->rx_queue_offload_capa); + info->tx_offload_capa = mlx5_get_tx_port_offloads(dev); + if (mlx5_get_ifname(dev, &ifname) == 0) info->if_index = if_nametoindex(ifname); info->reta_size = priv->reta_idx_n ? - priv->reta_idx_n : priv->ind_table_max_size; - info->hash_key_size = ((*priv->rss_conf) ? - (*priv->rss_conf)[0]->rss_key_len : - 0); + priv->reta_idx_n : config->ind_table_max_size; + info->hash_key_size = priv->rss_conf.rss_key_len; info->speed_capa = priv->link_speed_capa; - priv_unlock(priv); + info->flow_type_rss_offloads = ~MLX5_RSS_HF_MASK; } +/** + * Get supported packet types. + * + * @param dev + * Pointer to Ethernet device structure. + * + * @return + * A pointer to the supported Packet types array. + */ const uint32_t * mlx5_dev_supported_ptypes_get(struct rte_eth_dev *dev) { @@ -733,13 +450,14 @@ mlx5_dev_supported_ptypes_get(struct rte_eth_dev *dev) * * @param dev * Pointer to Ethernet device structure. - * @param wait_to_complete - * Wait for request completion (ignored). + * + * @return + * 0 on success, -1 on error. */ static int -mlx5_link_update_unlocked_gset(struct rte_eth_dev *dev, int wait_to_complete) +mlx5_link_update_unlocked_gset(struct rte_eth_dev *dev) { - struct priv *priv = mlx5_get_priv(dev); + struct priv *priv = dev->data->dev_private; struct ethtool_cmd edata = { .cmd = ETHTOOL_GSET /* Deprecated since Linux v4.5. */ }; @@ -747,10 +465,7 @@ mlx5_link_update_unlocked_gset(struct rte_eth_dev *dev, int wait_to_complete) struct rte_eth_link dev_link; int link_speed = 0; - /* priv_lock() is not taken to allow concurrent calls. */ - - (void)wait_to_complete; - if (priv_ifreq(priv, SIOCGIFFLAGS, &ifr)) { + if (mlx5_ifreq(dev, SIOCGIFFLAGS, &ifr)) { WARN("ioctl(SIOCGIFFLAGS) failed: %s", strerror(errno)); return -1; } @@ -758,7 +473,7 @@ mlx5_link_update_unlocked_gset(struct rte_eth_dev *dev, int wait_to_complete) dev_link.link_status = ((ifr.ifr_flags & IFF_UP) && (ifr.ifr_flags & IFF_RUNNING)); ifr.ifr_data = (void *)&edata; - if (priv_ifreq(priv, SIOCETHTOOL, &ifr)) { + if (mlx5_ifreq(dev, SIOCETHTOOL, &ifr)) { WARN("ioctl(SIOCETHTOOL, ETHTOOL_GSET) failed: %s", strerror(errno)); return -1; @@ -799,20 +514,20 @@ mlx5_link_update_unlocked_gset(struct rte_eth_dev *dev, int wait_to_complete) * * @param dev * Pointer to Ethernet device structure. - * @param wait_to_complete - * Wait for request completion (ignored). + * + * @return + * 0 on success, -1 on error. */ static int -mlx5_link_update_unlocked_gs(struct rte_eth_dev *dev, int wait_to_complete) +mlx5_link_update_unlocked_gs(struct rte_eth_dev *dev) { - struct priv *priv = mlx5_get_priv(dev); + struct priv *priv = dev->data->dev_private; struct ethtool_link_settings gcmd = { .cmd = ETHTOOL_GLINKSETTINGS }; struct ifreq ifr; struct rte_eth_link dev_link; uint64_t sc; - (void)wait_to_complete; - if (priv_ifreq(priv, SIOCGIFFLAGS, &ifr)) { + if (mlx5_ifreq(dev, SIOCGIFFLAGS, &ifr)) { WARN("ioctl(SIOCGIFFLAGS) failed: %s", strerror(errno)); return -1; } @@ -820,7 +535,7 @@ mlx5_link_update_unlocked_gs(struct rte_eth_dev *dev, int wait_to_complete) dev_link.link_status = ((ifr.ifr_flags & IFF_UP) && (ifr.ifr_flags & IFF_RUNNING)); ifr.ifr_data = (void *)&gcmd; - if (priv_ifreq(priv, SIOCETHTOOL, &ifr)) { + if (mlx5_ifreq(dev, SIOCETHTOOL, &ifr)) { DEBUG("ioctl(SIOCETHTOOL, ETHTOOL_GLINKSETTINGS) failed: %s", strerror(errno)); return -1; @@ -834,7 +549,7 @@ mlx5_link_update_unlocked_gs(struct rte_eth_dev *dev, int wait_to_complete) *ecmd = gcmd; ifr.ifr_data = (void *)ecmd; - if (priv_ifreq(priv, SIOCETHTOOL, &ifr)) { + if (mlx5_ifreq(dev, SIOCETHTOOL, &ifr)) { DEBUG("ioctl(SIOCETHTOOL, ETHTOOL_GLINKSETTINGS) failed: %s", strerror(errno)); return -1; @@ -843,39 +558,39 @@ mlx5_link_update_unlocked_gs(struct rte_eth_dev *dev, int wait_to_complete) sc = ecmd->link_mode_masks[0] | ((uint64_t)ecmd->link_mode_masks[1] << 32); priv->link_speed_capa = 0; - if (sc & ETHTOOL_LINK_MODE_Autoneg_BIT) + if (sc & MLX5_BITSHIFT(ETHTOOL_LINK_MODE_Autoneg_BIT)) priv->link_speed_capa |= ETH_LINK_SPEED_AUTONEG; - if (sc & (ETHTOOL_LINK_MODE_1000baseT_Full_BIT | - ETHTOOL_LINK_MODE_1000baseKX_Full_BIT)) + if (sc & (MLX5_BITSHIFT(ETHTOOL_LINK_MODE_1000baseT_Full_BIT) | + MLX5_BITSHIFT(ETHTOOL_LINK_MODE_1000baseKX_Full_BIT))) priv->link_speed_capa |= ETH_LINK_SPEED_1G; - if (sc & (ETHTOOL_LINK_MODE_10000baseKX4_Full_BIT | - ETHTOOL_LINK_MODE_10000baseKR_Full_BIT | - ETHTOOL_LINK_MODE_10000baseR_FEC_BIT)) + if (sc & (MLX5_BITSHIFT(ETHTOOL_LINK_MODE_10000baseKX4_Full_BIT) | + MLX5_BITSHIFT(ETHTOOL_LINK_MODE_10000baseKR_Full_BIT) | + MLX5_BITSHIFT(ETHTOOL_LINK_MODE_10000baseR_FEC_BIT))) priv->link_speed_capa |= ETH_LINK_SPEED_10G; - if (sc & (ETHTOOL_LINK_MODE_20000baseMLD2_Full_BIT | - ETHTOOL_LINK_MODE_20000baseKR2_Full_BIT)) + if (sc & (MLX5_BITSHIFT(ETHTOOL_LINK_MODE_20000baseMLD2_Full_BIT) | + MLX5_BITSHIFT(ETHTOOL_LINK_MODE_20000baseKR2_Full_BIT))) priv->link_speed_capa |= ETH_LINK_SPEED_20G; - if (sc & (ETHTOOL_LINK_MODE_40000baseKR4_Full_BIT | - ETHTOOL_LINK_MODE_40000baseCR4_Full_BIT | - ETHTOOL_LINK_MODE_40000baseSR4_Full_BIT | - ETHTOOL_LINK_MODE_40000baseLR4_Full_BIT)) + if (sc & (MLX5_BITSHIFT(ETHTOOL_LINK_MODE_40000baseKR4_Full_BIT) | + MLX5_BITSHIFT(ETHTOOL_LINK_MODE_40000baseCR4_Full_BIT) | + MLX5_BITSHIFT(ETHTOOL_LINK_MODE_40000baseSR4_Full_BIT) | + MLX5_BITSHIFT(ETHTOOL_LINK_MODE_40000baseLR4_Full_BIT))) priv->link_speed_capa |= ETH_LINK_SPEED_40G; - if (sc & (ETHTOOL_LINK_MODE_56000baseKR4_Full_BIT | - ETHTOOL_LINK_MODE_56000baseCR4_Full_BIT | - ETHTOOL_LINK_MODE_56000baseSR4_Full_BIT | - ETHTOOL_LINK_MODE_56000baseLR4_Full_BIT)) + if (sc & (MLX5_BITSHIFT(ETHTOOL_LINK_MODE_56000baseKR4_Full_BIT) | + MLX5_BITSHIFT(ETHTOOL_LINK_MODE_56000baseCR4_Full_BIT) | + MLX5_BITSHIFT(ETHTOOL_LINK_MODE_56000baseSR4_Full_BIT) | + MLX5_BITSHIFT(ETHTOOL_LINK_MODE_56000baseLR4_Full_BIT))) priv->link_speed_capa |= ETH_LINK_SPEED_56G; - if (sc & (ETHTOOL_LINK_MODE_25000baseCR_Full_BIT | - ETHTOOL_LINK_MODE_25000baseKR_Full_BIT | - ETHTOOL_LINK_MODE_25000baseSR_Full_BIT)) + if (sc & (MLX5_BITSHIFT(ETHTOOL_LINK_MODE_25000baseCR_Full_BIT) | + MLX5_BITSHIFT(ETHTOOL_LINK_MODE_25000baseKR_Full_BIT) | + MLX5_BITSHIFT(ETHTOOL_LINK_MODE_25000baseSR_Full_BIT))) priv->link_speed_capa |= ETH_LINK_SPEED_25G; - if (sc & (ETHTOOL_LINK_MODE_50000baseCR2_Full_BIT | - ETHTOOL_LINK_MODE_50000baseKR2_Full_BIT)) + if (sc & (MLX5_BITSHIFT(ETHTOOL_LINK_MODE_50000baseCR2_Full_BIT) | + MLX5_BITSHIFT(ETHTOOL_LINK_MODE_50000baseKR2_Full_BIT))) priv->link_speed_capa |= ETH_LINK_SPEED_50G; - if (sc & (ETHTOOL_LINK_MODE_100000baseKR4_Full_BIT | - ETHTOOL_LINK_MODE_100000baseSR4_Full_BIT | - ETHTOOL_LINK_MODE_100000baseCR4_Full_BIT | - ETHTOOL_LINK_MODE_100000baseLR4_ER4_Full_BIT)) + if (sc & (MLX5_BITSHIFT(ETHTOOL_LINK_MODE_100000baseKR4_Full_BIT) | + MLX5_BITSHIFT(ETHTOOL_LINK_MODE_100000baseSR4_Full_BIT) | + MLX5_BITSHIFT(ETHTOOL_LINK_MODE_100000baseCR4_Full_BIT) | + MLX5_BITSHIFT(ETHTOOL_LINK_MODE_100000baseLR4_ER4_Full_BIT))) priv->link_speed_capa |= ETH_LINK_SPEED_100G; dev_link.link_duplex = ((ecmd->duplex == DUPLEX_HALF) ? ETH_LINK_HALF_DUPLEX : ETH_LINK_FULL_DUPLEX); @@ -890,6 +605,74 @@ mlx5_link_update_unlocked_gs(struct rte_eth_dev *dev, int wait_to_complete) return -1; } +/** + * Enable receiving and transmitting traffic. + * + * @param dev + * Pointer to Ethernet device. + */ +static void +mlx5_link_start(struct rte_eth_dev *dev) +{ + struct priv *priv = dev->data->dev_private; + int err; + + dev->tx_pkt_burst = mlx5_select_tx_function(dev); + dev->rx_pkt_burst = mlx5_select_rx_function(dev); + err = mlx5_traffic_enable(dev); + if (err) + ERROR("%p: error occurred while configuring control flows: %s", + (void *)dev, strerror(err)); + err = mlx5_flow_start(dev, &priv->flows); + if (err) + ERROR("%p: error occurred while configuring flows: %s", + (void *)dev, strerror(err)); +} + +/** + * Disable receiving and transmitting traffic. + * + * @param dev + * Pointer to Ethernet device. + */ +static void +mlx5_link_stop(struct rte_eth_dev *dev) +{ + struct priv *priv = dev->data->dev_private; + + mlx5_flow_stop(dev, &priv->flows); + mlx5_traffic_disable(dev); + dev->rx_pkt_burst = removed_rx_burst; + dev->tx_pkt_burst = removed_tx_burst; +} + +/** + * Querying the link status till it changes to the desired state. + * Number of query attempts is bounded by MLX5_MAX_LINK_QUERY_ATTEMPTS. + * + * @param dev + * Pointer to Ethernet device. + * @param status + * Link desired status. + * + * @return + * 0 on success, negative errno value on failure. + */ +int +mlx5_force_link_status_change(struct rte_eth_dev *dev, int status) +{ + int try = 0; + + while (try < MLX5_MAX_LINK_QUERY_ATTEMPTS) { + mlx5_link_update(dev, 0); + if (dev->data->dev_link.link_status == status) + return 0; + try++; + sleep(1); + } + return -EAGAIN; +} + /** * DPDK callback to retrieve physical link information. * @@ -897,19 +680,38 @@ mlx5_link_update_unlocked_gs(struct rte_eth_dev *dev, int wait_to_complete) * Pointer to Ethernet device structure. * @param wait_to_complete * Wait for request completion (ignored). + * + * @return + * 0 on success, -1 on error. */ int -mlx5_link_update(struct rte_eth_dev *dev, int wait_to_complete) +mlx5_link_update(struct rte_eth_dev *dev, int wait_to_complete __rte_unused) { struct utsname utsname; int ver[3]; + int ret; + struct rte_eth_link dev_link = dev->data->dev_link; if (uname(&utsname) == -1 || sscanf(utsname.release, "%d.%d.%d", &ver[0], &ver[1], &ver[2]) != 3 || KERNEL_VERSION(ver[0], ver[1], ver[2]) < KERNEL_VERSION(4, 9, 0)) - return mlx5_link_update_unlocked_gset(dev, wait_to_complete); - return mlx5_link_update_unlocked_gs(dev, wait_to_complete); + ret = mlx5_link_update_unlocked_gset(dev); + else + ret = mlx5_link_update_unlocked_gs(dev); + /* If lsc interrupt is disabled, should always be ready for traffic. */ + if (!dev->data->dev_conf.intr_conf.lsc) { + mlx5_link_start(dev); + return ret; + } + /* Re-select burst callbacks only if link status has been changed. */ + if (!ret && dev_link.link_status != dev->data->dev_link.link_status) { + if (dev->data->dev_link.link_status == ETH_LINK_UP) + mlx5_link_start(dev); + else + mlx5_link_stop(dev); + } + return ret; } /** @@ -930,31 +732,25 @@ mlx5_dev_set_mtu(struct rte_eth_dev *dev, uint16_t mtu) uint16_t kern_mtu; int ret = 0; - if (mlx5_is_secondary()) - return -E_RTE_SECONDARY; - - priv_lock(priv); - ret = priv_get_mtu(priv, &kern_mtu); + ret = mlx5_get_mtu(dev, &kern_mtu); if (ret) goto out; /* Set kernel interface MTU first. */ - ret = priv_set_mtu(priv, mtu); + ret = mlx5_set_mtu(dev, mtu); if (ret) goto out; - ret = priv_get_mtu(priv, &kern_mtu); + ret = mlx5_get_mtu(dev, &kern_mtu); if (ret) goto out; if (kern_mtu == mtu) { priv->mtu = mtu; DEBUG("adapter port %u MTU set to %u", priv->port, mtu); } - priv_unlock(priv); return 0; out: ret = errno; WARN("cannot set port %u MTU to %u: %s", priv->port, mtu, strerror(ret)); - priv_unlock(priv); assert(ret >= 0); return -ret; } @@ -973,26 +769,19 @@ out: int mlx5_dev_get_flow_ctrl(struct rte_eth_dev *dev, struct rte_eth_fc_conf *fc_conf) { - struct priv *priv = dev->data->dev_private; struct ifreq ifr; struct ethtool_pauseparam ethpause = { .cmd = ETHTOOL_GPAUSEPARAM }; int ret; - if (mlx5_is_secondary()) - return -E_RTE_SECONDARY; - ifr.ifr_data = (void *)ðpause; - priv_lock(priv); - if (priv_ifreq(priv, SIOCETHTOOL, &ifr)) { + if (mlx5_ifreq(dev, SIOCETHTOOL, &ifr)) { ret = errno; - WARN("ioctl(SIOCETHTOOL, ETHTOOL_GPAUSEPARAM)" - " failed: %s", + WARN("ioctl(SIOCETHTOOL, ETHTOOL_GPAUSEPARAM) failed: %s", strerror(ret)); goto out; } - fc_conf->autoneg = ethpause.autoneg; if (ethpause.rx_pause && ethpause.tx_pause) fc_conf->mode = RTE_FC_FULL; @@ -1003,9 +792,7 @@ mlx5_dev_get_flow_ctrl(struct rte_eth_dev *dev, struct rte_eth_fc_conf *fc_conf) else fc_conf->mode = RTE_FC_NONE; ret = 0; - out: - priv_unlock(priv); assert(ret >= 0); return -ret; } @@ -1024,16 +811,12 @@ out: int mlx5_dev_set_flow_ctrl(struct rte_eth_dev *dev, struct rte_eth_fc_conf *fc_conf) { - struct priv *priv = dev->data->dev_private; struct ifreq ifr; struct ethtool_pauseparam ethpause = { .cmd = ETHTOOL_SPAUSEPARAM }; int ret; - if (mlx5_is_secondary()) - return -E_RTE_SECONDARY; - ifr.ifr_data = (void *)ðpause; ethpause.autoneg = fc_conf->autoneg; if (((fc_conf->mode & RTE_FC_FULL) == RTE_FC_FULL) || @@ -1047,9 +830,7 @@ mlx5_dev_set_flow_ctrl(struct rte_eth_dev *dev, struct rte_eth_fc_conf *fc_conf) ethpause.tx_pause = 1; else ethpause.tx_pause = 0; - - priv_lock(priv); - if (priv_ifreq(priv, SIOCETHTOOL, &ifr)) { + if (mlx5_ifreq(dev, SIOCETHTOOL, &ifr)) { ret = errno; WARN("ioctl(SIOCETHTOOL, ETHTOOL_SPAUSEPARAM)" " failed: %s", @@ -1057,9 +838,7 @@ mlx5_dev_set_flow_ctrl(struct rte_eth_dev *dev, struct rte_eth_fc_conf *fc_conf) goto out; } ret = 0; - out: - priv_unlock(priv); assert(ret >= 0); return -ret; } @@ -1117,18 +896,19 @@ mlx5_ibv_device_to_pci_addr(const struct ibv_device *device, /** * Update the link status. * - * @param priv - * Pointer to private structure. + * @param dev + * Pointer to Ethernet device. * * @return * Zero if the callback process can be called immediately. */ static int -priv_link_status_update(struct priv *priv) +mlx5_link_status_update(struct rte_eth_dev *dev) { - struct rte_eth_link *link = &priv->dev->data->dev_link; + struct priv *priv = dev->data->dev_private; + struct rte_eth_link *link = &dev->data->dev_link; - mlx5_link_update(priv->dev, 0); + mlx5_link_update(dev, 0); if (((link->link_speed == 0) && link->link_status) || ((link->link_speed != 0) && !link->link_status)) { /* @@ -1153,8 +933,8 @@ priv_link_status_update(struct priv *priv) /** * Device status handler. * - * @param priv - * Pointer to private structure. + * @param dev + * Pointer to Ethernet device. * @param events * Pointer to event flags holder. * @@ -1162,29 +942,30 @@ priv_link_status_update(struct priv *priv) * Events bitmap of callback process which can be called immediately. */ static uint32_t -priv_dev_status_handler(struct priv *priv) +mlx5_dev_status_handler(struct rte_eth_dev *dev) { + struct priv *priv = dev->data->dev_private; struct ibv_async_event event; uint32_t ret = 0; /* Read all message and acknowledge them. */ for (;;) { - if (ibv_get_async_event(priv->ctx, &event)) + if (mlx5_glue->get_async_event(priv->ctx, &event)) break; if ((event.event_type == IBV_EVENT_PORT_ACTIVE || event.event_type == IBV_EVENT_PORT_ERR) && - (priv->dev->data->dev_conf.intr_conf.lsc == 1)) + (dev->data->dev_conf.intr_conf.lsc == 1)) ret |= (1 << RTE_ETH_EVENT_INTR_LSC); else if (event.event_type == IBV_EVENT_DEVICE_FATAL && - priv->dev->data->dev_conf.intr_conf.rmv == 1) + dev->data->dev_conf.intr_conf.rmv == 1) ret |= (1 << RTE_ETH_EVENT_INTR_RMV); else DEBUG("event type %d on port %d not handled", event.event_type, event.element.port_num); - ibv_ack_async_event(&event); + mlx5_glue->ack_async_event(&event); } if (ret & (1 << RTE_ETH_EVENT_INTR_LSC)) - if (priv_link_status_update(priv)) + if (mlx5_link_status_update(dev)) ret &= ~(1 << RTE_ETH_EVENT_INTR_LSC); return ret; } @@ -1202,14 +983,10 @@ mlx5_dev_link_status_handler(void *arg) struct priv *priv = dev->data->dev_private; int ret; - priv_lock(priv); - assert(priv->pending_alarm == 1); priv->pending_alarm = 0; - ret = priv_link_status_update(priv); - priv_unlock(priv); + ret = mlx5_link_status_update(dev); if (!ret) - _rte_eth_dev_callback_process(dev, RTE_ETH_EVENT_INTR_LSC, NULL, - NULL); + _rte_eth_dev_callback_process(dev, RTE_ETH_EVENT_INTR_LSC, NULL); } /** @@ -1224,18 +1001,13 @@ void mlx5_dev_interrupt_handler(void *cb_arg) { struct rte_eth_dev *dev = cb_arg; - struct priv *priv = dev->data->dev_private; uint32_t events; - priv_lock(priv); - events = priv_dev_status_handler(priv); - priv_unlock(priv); + events = mlx5_dev_status_handler(dev); if (events & (1 << RTE_ETH_EVENT_INTR_LSC)) - _rte_eth_dev_callback_process(dev, RTE_ETH_EVENT_INTR_LSC, NULL, - NULL); + _rte_eth_dev_callback_process(dev, RTE_ETH_EVENT_INTR_LSC, NULL); if (events & (1 << RTE_ETH_EVENT_INTR_RMV)) - _rte_eth_dev_callback_process(dev, RTE_ETH_EVENT_INTR_RMV, NULL, - NULL); + _rte_eth_dev_callback_process(dev, RTE_ETH_EVENT_INTR_RMV, NULL); } /** @@ -1248,24 +1020,21 @@ static void mlx5_dev_handler_socket(void *cb_arg) { struct rte_eth_dev *dev = cb_arg; - struct priv *priv = dev->data->dev_private; - priv_lock(priv); - priv_socket_handle(priv); - priv_unlock(priv); + mlx5_socket_handle(dev); } /** * Uninstall interrupt handler. * - * @param priv - * Pointer to private structure. * @param dev - * Pointer to the rte_eth_dev structure. + * Pointer to Ethernet device. */ void -priv_dev_interrupt_handler_uninstall(struct priv *priv, struct rte_eth_dev *dev) +mlx5_dev_interrupt_handler_uninstall(struct rte_eth_dev *dev) { + struct priv *priv = dev->data->dev_private; + if (dev->data->dev_conf.intr_conf.lsc || dev->data->dev_conf.intr_conf.rmv) rte_intr_callback_unregister(&priv->intr_handle, @@ -1273,9 +1042,10 @@ priv_dev_interrupt_handler_uninstall(struct priv *priv, struct rte_eth_dev *dev) if (priv->primary_socket) rte_intr_callback_unregister(&priv->intr_handle_socket, mlx5_dev_handler_socket, dev); - if (priv->pending_alarm) + if (priv->pending_alarm) { + priv->pending_alarm = 0; rte_eal_alarm_cancel(mlx5_dev_link_status_handler, dev); - priv->pending_alarm = 0; + } priv->intr_handle.fd = 0; priv->intr_handle.type = RTE_INTR_HANDLE_UNKNOWN; priv->intr_handle_socket.fd = 0; @@ -1285,17 +1055,15 @@ priv_dev_interrupt_handler_uninstall(struct priv *priv, struct rte_eth_dev *dev) /** * Install interrupt handler. * - * @param priv - * Pointer to private structure. * @param dev - * Pointer to the rte_eth_dev structure. + * Pointer to Ethernet device. */ void -priv_dev_interrupt_handler_install(struct priv *priv, struct rte_eth_dev *dev) +mlx5_dev_interrupt_handler_install(struct rte_eth_dev *dev) { + struct priv *priv = dev->data->dev_private; int rc, flags; - assert(!mlx5_is_secondary()); assert(priv->ctx->async_fd > 0); flags = fcntl(priv->ctx->async_fd, F_GETFL); rc = fcntl(priv->ctx->async_fd, F_SETFL, flags | O_NONBLOCK); @@ -1311,8 +1079,7 @@ priv_dev_interrupt_handler_install(struct priv *priv, struct rte_eth_dev *dev) rte_intr_callback_register(&priv->intr_handle, mlx5_dev_interrupt_handler, dev); } - - rc = priv_socket_init(priv); + rc = mlx5_socket_init(dev); if (!rc && priv->primary_socket) { priv->intr_handle_socket.fd = priv->primary_socket; priv->intr_handle_socket.type = RTE_INTR_HANDLE_EXT; @@ -1321,40 +1088,6 @@ priv_dev_interrupt_handler_install(struct priv *priv, struct rte_eth_dev *dev) } } -/** - * Change the link state (UP / DOWN). - * - * @param priv - * Pointer to private data structure. - * @param dev - * Pointer to rte_eth_dev structure. - * @param up - * Nonzero for link up, otherwise link down. - * - * @return - * 0 on success, errno value on failure. - */ -static int -priv_dev_set_link(struct priv *priv, struct rte_eth_dev *dev, int up) -{ - int err; - - if (up) { - err = priv_set_flags(priv, ~IFF_UP, IFF_UP); - if (err) - return err; - priv_dev_select_tx_function(priv, dev); - priv_dev_select_rx_function(priv, dev); - } else { - err = priv_set_flags(priv, ~IFF_UP, ~IFF_UP); - if (err) - return err; - dev->rx_pkt_burst = removed_rx_burst; - dev->tx_pkt_burst = removed_tx_burst; - } - return 0; -} - /** * DPDK callback to bring the link DOWN. * @@ -1367,13 +1100,7 @@ priv_dev_set_link(struct priv *priv, struct rte_eth_dev *dev, int up) int mlx5_set_link_down(struct rte_eth_dev *dev) { - struct priv *priv = dev->data->dev_private; - int err; - - priv_lock(priv); - err = priv_dev_set_link(priv, dev, 0); - priv_unlock(priv); - return err; + return mlx5_set_flags(dev, ~IFF_UP, ~IFF_UP); } /** @@ -1388,67 +1115,93 @@ mlx5_set_link_down(struct rte_eth_dev *dev) int mlx5_set_link_up(struct rte_eth_dev *dev) { - struct priv *priv = dev->data->dev_private; - int err; - - priv_lock(priv); - err = priv_dev_set_link(priv, dev, 1); - priv_unlock(priv); - return err; + return mlx5_set_flags(dev, ~IFF_UP, IFF_UP); } /** * Configure the TX function to use. * - * @param priv - * Pointer to private data structure. * @param dev - * Pointer to rte_eth_dev structure. + * Pointer to private data structure. + * + * @return + * Pointer to selected Tx burst function. */ -void -priv_dev_select_tx_function(struct priv *priv, struct rte_eth_dev *dev) +eth_tx_burst_t +mlx5_select_tx_function(struct rte_eth_dev *dev) { + struct priv *priv = dev->data->dev_private; + eth_tx_burst_t tx_pkt_burst = mlx5_tx_burst; + struct mlx5_dev_config *config = &priv->config; + uint64_t tx_offloads = dev->data->dev_conf.txmode.offloads; + int tso = !!(tx_offloads & (DEV_TX_OFFLOAD_TCP_TSO | + DEV_TX_OFFLOAD_VXLAN_TNL_TSO | + DEV_TX_OFFLOAD_GRE_TNL_TSO)); + int vlan_insert = !!(tx_offloads & DEV_TX_OFFLOAD_VLAN_INSERT); + assert(priv != NULL); - assert(dev != NULL); - dev->tx_pkt_burst = mlx5_tx_burst; /* Select appropriate TX function. */ - if (priv->mps == MLX5_MPW_ENHANCED) { - if (priv_check_vec_tx_support(priv) > 0) { - if (priv_check_raw_vec_tx_support(priv) > 0) - dev->tx_pkt_burst = mlx5_tx_burst_raw_vec; + if (vlan_insert || tso) + return tx_pkt_burst; + if (config->mps == MLX5_MPW_ENHANCED) { + if (mlx5_check_vec_tx_support(dev) > 0) { + if (mlx5_check_raw_vec_tx_support(dev) > 0) + tx_pkt_burst = mlx5_tx_burst_raw_vec; else - dev->tx_pkt_burst = mlx5_tx_burst_vec; + tx_pkt_burst = mlx5_tx_burst_vec; DEBUG("selected Enhanced MPW TX vectorized function"); } else { - dev->tx_pkt_burst = mlx5_tx_burst_empw; + tx_pkt_burst = mlx5_tx_burst_empw; DEBUG("selected Enhanced MPW TX function"); } - } else if (priv->mps && priv->txq_inline) { - dev->tx_pkt_burst = mlx5_tx_burst_mpw_inline; + } else if (config->mps && (config->txq_inline > 0)) { + tx_pkt_burst = mlx5_tx_burst_mpw_inline; DEBUG("selected MPW inline TX function"); - } else if (priv->mps) { - dev->tx_pkt_burst = mlx5_tx_burst_mpw; + } else if (config->mps) { + tx_pkt_burst = mlx5_tx_burst_mpw; DEBUG("selected MPW TX function"); } + return tx_pkt_burst; } /** * Configure the RX function to use. * - * @param priv - * Pointer to private data structure. * @param dev - * Pointer to rte_eth_dev structure. + * Pointer to private data structure. + * + * @return + * Pointer to selected Rx burst function. */ -void -priv_dev_select_rx_function(struct priv *priv, struct rte_eth_dev *dev) +eth_rx_burst_t +mlx5_select_rx_function(struct rte_eth_dev *dev) { - assert(priv != NULL); + eth_rx_burst_t rx_pkt_burst = mlx5_rx_burst; + assert(dev != NULL); - if (priv_check_vec_rx_support(priv) > 0) { - dev->rx_pkt_burst = mlx5_rx_burst_vec; + if (mlx5_check_vec_rx_support(dev) > 0) { + rx_pkt_burst = mlx5_rx_burst_vec; DEBUG("selected RX vectorized function"); - } else { - dev->rx_pkt_burst = mlx5_rx_burst; } + return rx_pkt_burst; +} + +/** + * Check if mlx5 device was removed. + * + * @param dev + * Pointer to Ethernet device structure. + * + * @return + * 1 when device is removed, otherwise 0. + */ +int +mlx5_is_removed(struct rte_eth_dev *dev) +{ + struct ibv_device_attr device_attr; + struct priv *priv = dev->data->dev_private; + + if (mlx5_glue->query_device(priv->ctx, &device_attr) == EIO) + return 1; + return 0; }