drivers/net/mlx5/mlx5_ethdev.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright 2015 6WIND S.A.
   3  * Copyright 2015 Mellanox Technologies, Ltd
   4  */
   5
   6 #define _GNU_SOURCE
   7
   8 #include <stddef.h>
   9 #include <assert.h>
  10 #include <inttypes.h>
  11 #include <unistd.h>
  12 #include <stdint.h>
  13 #include <stdio.h>
  14 #include <string.h>
  15 #include <stdlib.h>
  16 #include <errno.h>
  17 #include <dirent.h>
  18 #include <net/if.h>
  19 #include <sys/ioctl.h>
  20 #include <sys/socket.h>
  21 #include <netinet/in.h>
  22 #include <linux/ethtool.h>
  23 #include <linux/sockios.h>
  24 #include <fcntl.h>
  25 #include <stdalign.h>
  26 #include <sys/un.h>
  27 #include <time.h>
  28
  29 #include <rte_atomic.h>
  30 #include <rte_ethdev_driver.h>
  31 #include <rte_bus_pci.h>
  32 #include <rte_mbuf.h>
  33 #include <rte_common.h>
  34 #include <rte_interrupts.h>
  35 #include <rte_malloc.h>
  36 #include <rte_string_fns.h>
  37 #include <rte_rwlock.h>
  38
  39 #include "mlx5.h"
  40 #include "mlx5_glue.h"
  41 #include "mlx5_rxtx.h"
  42 #include "mlx5_utils.h"
  43
  44 /* Add defines in case the running kernel is not the same as user headers. */
  45 #ifndef ETHTOOL_GLINKSETTINGS
  46 struct ethtool_link_settings {
  47         uint32_t cmd;
  48         uint32_t speed;
  49         uint8_t duplex;
  50         uint8_t port;
  51         uint8_t phy_address;
  52         uint8_t autoneg;
  53         uint8_t mdio_support;
  54         uint8_t eth_to_mdix;
  55         uint8_t eth_tp_mdix_ctrl;
  56         int8_t link_mode_masks_nwords;
  57         uint32_t reserved[8];
  58         uint32_t link_mode_masks[];
  59 };
  60
  61 #define ETHTOOL_GLINKSETTINGS 0x0000004c
  62 #define ETHTOOL_LINK_MODE_1000baseT_Full_BIT 5
  63 #define ETHTOOL_LINK_MODE_Autoneg_BIT 6
  64 #define ETHTOOL_LINK_MODE_1000baseKX_Full_BIT 17
  65 #define ETHTOOL_LINK_MODE_10000baseKX4_Full_BIT 18
  66 #define ETHTOOL_LINK_MODE_10000baseKR_Full_BIT 19
  67 #define ETHTOOL_LINK_MODE_10000baseR_FEC_BIT 20
  68 #define ETHTOOL_LINK_MODE_20000baseMLD2_Full_BIT 21
  69 #define ETHTOOL_LINK_MODE_20000baseKR2_Full_BIT 22
  70 #define ETHTOOL_LINK_MODE_40000baseKR4_Full_BIT 23
  71 #define ETHTOOL_LINK_MODE_40000baseCR4_Full_BIT 24
  72 #define ETHTOOL_LINK_MODE_40000baseSR4_Full_BIT 25
  73 #define ETHTOOL_LINK_MODE_40000baseLR4_Full_BIT 26
  74 #define ETHTOOL_LINK_MODE_56000baseKR4_Full_BIT 27
  75 #define ETHTOOL_LINK_MODE_56000baseCR4_Full_BIT 28
  76 #define ETHTOOL_LINK_MODE_56000baseSR4_Full_BIT 29
  77 #define ETHTOOL_LINK_MODE_56000baseLR4_Full_BIT 30
  78 #endif
  79 #ifndef HAVE_ETHTOOL_LINK_MODE_25G
  80 #define ETHTOOL_LINK_MODE_25000baseCR_Full_BIT 31
  81 #define ETHTOOL_LINK_MODE_25000baseKR_Full_BIT 32
  82 #define ETHTOOL_LINK_MODE_25000baseSR_Full_BIT 33
  83 #endif
  84 #ifndef HAVE_ETHTOOL_LINK_MODE_50G
  85 #define ETHTOOL_LINK_MODE_50000baseCR2_Full_BIT 34
  86 #define ETHTOOL_LINK_MODE_50000baseKR2_Full_BIT 35
  87 #endif
  88 #ifndef HAVE_ETHTOOL_LINK_MODE_100G
  89 #define ETHTOOL_LINK_MODE_100000baseKR4_Full_BIT 36
  90 #define ETHTOOL_LINK_MODE_100000baseSR4_Full_BIT 37
  91 #define ETHTOOL_LINK_MODE_100000baseCR4_Full_BIT 38
  92 #define ETHTOOL_LINK_MODE_100000baseLR4_ER4_Full_BIT 39
  93 #endif
  94
  95 /**
  96  * Get master interface name from private structure.
  97  *
  98  * @param[in] dev
  99  *   Pointer to Ethernet device.
 100  * @param[out] ifname
 101  *   Interface name output buffer.
 102  *
 103  * @return
 104  *   0 on success, a negative errno value otherwise and rte_errno is set.
 105  */
 106 int
 107 mlx5_get_master_ifname(const struct rte_eth_dev *dev,
 108                        char (*ifname)[IF_NAMESIZE])
 109 {
 110         struct priv *priv = dev->data->dev_private;
 111         DIR *dir;
 112         struct dirent *dent;
 113         unsigned int dev_type = 0;
 114         unsigned int dev_port_prev = ~0u;
 115         char match[IF_NAMESIZE] = "";
 116
 117         {
 118                 MKSTR(path, "%s/device/net", priv->ibdev_path);
 119
 120                 dir = opendir(path);
 121                 if (dir == NULL) {
 122                         rte_errno = errno;
 123                         return -rte_errno;
 124                 }
 125         }
 126         while ((dent = readdir(dir)) != NULL) {
 127                 char *name = dent->d_name;
 128                 FILE *file;
 129                 unsigned int dev_port;
 130                 int r;
 131
 132                 if ((name[0] == '.') &&
 133                     ((name[1] == '\0') ||
 134                      ((name[1] == '.') && (name[2] == '\0'))))
 135                         continue;
 136
 137                 MKSTR(path, "%s/device/net/%s/%s",
 138                       priv->ibdev_path, name,
 139                       (dev_type ? "dev_id" : "dev_port"));
 140
 141                 file = fopen(path, "rb");
 142                 if (file == NULL) {
 143                         if (errno != ENOENT)
 144                                 continue;
 145                         /*
 146                          * Switch to dev_id when dev_port does not exist as
 147                          * is the case with Linux kernel versions < 3.15.
 148                          */
 149 try_dev_id:
 150                         match[0] = '\0';
 151                         if (dev_type)
 152                                 break;
 153                         dev_type = 1;
 154                         dev_port_prev = ~0u;
 155                         rewinddir(dir);
 156                         continue;
 157                 }
 158                 r = fscanf(file, (dev_type ? "%x" : "%u"), &dev_port);
 159                 fclose(file);
 160                 if (r != 1)
 161                         continue;
 162                 /*
 163                  * Switch to dev_id when dev_port returns the same value for
 164                  * all ports. May happen when using a MOFED release older than
 165                  * 3.0 with a Linux kernel >= 3.15.
 166                  */
 167                 if (dev_port == dev_port_prev)
 168                         goto try_dev_id;
 169                 dev_port_prev = dev_port;
 170                 if (dev_port == 0)
 171                         strlcpy(match, name, sizeof(match));
 172         }
 173         closedir(dir);
 174         if (match[0] == '\0') {
 175                 rte_errno = ENOENT;
 176                 return -rte_errno;
 177         }
 178         strncpy(*ifname, match, sizeof(*ifname));
 179         return 0;
 180 }
 181
 182 /**
 183  * Get interface name from private structure.
 184  *
 185  * This is a port representor-aware version of mlx5_get_master_ifname().
 186  *
 187  * @param[in] dev
 188  *   Pointer to Ethernet device.
 189  * @param[out] ifname
 190  *   Interface name output buffer.
 191  *
 192  * @return
 193  *   0 on success, a negative errno value otherwise and rte_errno is set.
 194  */
 195 int
 196 mlx5_get_ifname(const struct rte_eth_dev *dev, char (*ifname)[IF_NAMESIZE])
 197 {
 198         struct priv *priv = dev->data->dev_private;
 199         unsigned int ifindex =
 200                 priv->nl_socket_rdma >= 0 ?
 201                 mlx5_nl_ifindex(priv->nl_socket_rdma, priv->ibdev_name) : 0;
 202
 203         if (!ifindex) {
 204                 if (!priv->representor)
 205                         return mlx5_get_master_ifname(dev, ifname);
 206                 rte_errno = ENXIO;
 207                 return -rte_errno;
 208         }
 209         if (if_indextoname(ifindex, &(*ifname)[0]))
 210                 return 0;
 211         rte_errno = errno;
 212         return -rte_errno;
 213 }
 214
 215 /**
 216  * Get the interface index from device name.
 217  *
 218  * @param[in] dev
 219  *   Pointer to Ethernet device.
 220  *
 221  * @return
 222  *   Interface index on success, a negative errno value otherwise and
 223  *   rte_errno is set.
 224  */
 225 int
 226 mlx5_ifindex(const struct rte_eth_dev *dev)
 227 {
 228         char ifname[IF_NAMESIZE];
 229         unsigned int ret;
 230
 231         ret = mlx5_get_ifname(dev, &ifname);
 232         if (ret)
 233                 return ret;
 234         ret = if_nametoindex(ifname);
 235         if (ret == 0) {
 236                 rte_errno = errno;
 237                 return -rte_errno;
 238         }
 239         return ret;
 240 }
 241
 242 /**
 243  * Perform ifreq ioctl() on associated Ethernet device.
 244  *
 245  * @param[in] dev
 246  *   Pointer to Ethernet device.
 247  * @param req
 248  *   Request number to pass to ioctl().
 249  * @param[out] ifr
 250  *   Interface request structure output buffer.
 251  * @param master
 252  *   When device is a port representor, perform request on master device
 253  *   instead.
 254  *
 255  * @return
 256  *   0 on success, a negative errno value otherwise and rte_errno is set.
 257  */
 258 int
 259 mlx5_ifreq(const struct rte_eth_dev *dev, int req, struct ifreq *ifr,
 260            int master)
 261 {
 262         int sock = socket(PF_INET, SOCK_DGRAM, IPPROTO_IP);
 263         int ret = 0;
 264
 265         if (sock == -1) {
 266                 rte_errno = errno;
 267                 return -rte_errno;
 268         }
 269         if (master)
 270                 ret = mlx5_get_master_ifname(dev, &ifr->ifr_name);
 271         else
 272                 ret = mlx5_get_ifname(dev, &ifr->ifr_name);
 273         if (ret)
 274                 goto error;
 275         ret = ioctl(sock, req, ifr);
 276         if (ret == -1) {
 277                 rte_errno = errno;
 278                 goto error;
 279         }
 280         close(sock);
 281         return 0;
 282 error:
 283         close(sock);
 284         return -rte_errno;
 285 }
 286
 287 /**
 288  * Get device MTU.
 289  *
 290  * @param dev
 291  *   Pointer to Ethernet device.
 292  * @param[out] mtu
 293  *   MTU value output buffer.
 294  *
 295  * @return
 296  *   0 on success, a negative errno value otherwise and rte_errno is set.
 297  */
 298 int
 299 mlx5_get_mtu(struct rte_eth_dev *dev, uint16_t *mtu)
 300 {
 301         struct ifreq request;
 302         int ret = mlx5_ifreq(dev, SIOCGIFMTU, &request, 0);
 303
 304         if (ret)
 305                 return ret;
 306         *mtu = request.ifr_mtu;
 307         return 0;
 308 }
 309
 310 /**
 311  * Set device MTU.
 312  *
 313  * @param dev
 314  *   Pointer to Ethernet device.
 315  * @param mtu
 316  *   MTU value to set.
 317  *
 318  * @return
 319  *   0 on success, a negative errno value otherwise and rte_errno is set.
 320  */
 321 static int
 322 mlx5_set_mtu(struct rte_eth_dev *dev, uint16_t mtu)
 323 {
 324         struct ifreq request = { .ifr_mtu = mtu, };
 325
 326         return mlx5_ifreq(dev, SIOCSIFMTU, &request, 0);
 327 }
 328
 329 /**
 330  * Set device flags.
 331  *
 332  * @param dev
 333  *   Pointer to Ethernet device.
 334  * @param keep
 335  *   Bitmask for flags that must remain untouched.
 336  * @param flags
 337  *   Bitmask for flags to modify.
 338  *
 339  * @return
 340  *   0 on success, a negative errno value otherwise and rte_errno is set.
 341  */
 342 int
 343 mlx5_set_flags(struct rte_eth_dev *dev, unsigned int keep, unsigned int flags)
 344 {
 345         struct ifreq request;
 346         int ret = mlx5_ifreq(dev, SIOCGIFFLAGS, &request, 0);
 347
 348         if (ret)
 349                 return ret;
 350         request.ifr_flags &= keep;
 351         request.ifr_flags |= flags & ~keep;
 352         return mlx5_ifreq(dev, SIOCSIFFLAGS, &request, 0);
 353 }
 354
 355 /**
 356  * DPDK callback for Ethernet device configuration.
 357  *
 358  * @param dev
 359  *   Pointer to Ethernet device structure.
 360  *
 361  * @return
 362  *   0 on success, a negative errno value otherwise and rte_errno is set.
 363  */
 364 int
 365 mlx5_dev_configure(struct rte_eth_dev *dev)
 366 {
 367         struct priv *priv = dev->data->dev_private;
 368         unsigned int rxqs_n = dev->data->nb_rx_queues;
 369         unsigned int txqs_n = dev->data->nb_tx_queues;
 370         unsigned int i;
 371         unsigned int j;
 372         unsigned int reta_idx_n;
 373         const uint8_t use_app_rss_key =
 374                 !!dev->data->dev_conf.rx_adv_conf.rss_conf.rss_key;
 375         int ret = 0;
 376
 377         if (use_app_rss_key &&
 378             (dev->data->dev_conf.rx_adv_conf.rss_conf.rss_key_len !=
 379              rss_hash_default_key_len)) {
 380                 DRV_LOG(ERR, "port %u RSS key len must be %zu Bytes long",
 381                         dev->data->port_id, rss_hash_default_key_len);
 382                 rte_errno = EINVAL;
 383                 return -rte_errno;
 384         }
 385         priv->rss_conf.rss_key =
 386                 rte_realloc(priv->rss_conf.rss_key,
 387                             rss_hash_default_key_len, 0);
 388         if (!priv->rss_conf.rss_key) {
 389                 DRV_LOG(ERR, "port %u cannot allocate RSS hash key memory (%u)",
 390                         dev->data->port_id, rxqs_n);
 391                 rte_errno = ENOMEM;
 392                 return -rte_errno;
 393         }
 394         memcpy(priv->rss_conf.rss_key,
 395                use_app_rss_key ?
 396                dev->data->dev_conf.rx_adv_conf.rss_conf.rss_key :
 397                rss_hash_default_key,
 398                rss_hash_default_key_len);
 399         priv->rss_conf.rss_key_len = rss_hash_default_key_len;
 400         priv->rss_conf.rss_hf = dev->data->dev_conf.rx_adv_conf.rss_conf.rss_hf;
 401         priv->rxqs = (void *)dev->data->rx_queues;
 402         priv->txqs = (void *)dev->data->tx_queues;
 403         if (txqs_n != priv->txqs_n) {
 404                 DRV_LOG(INFO, "port %u Tx queues number update: %u -> %u",
 405                         dev->data->port_id, priv->txqs_n, txqs_n);
 406                 priv->txqs_n = txqs_n;
 407         }
 408         if (rxqs_n > priv->config.ind_table_max_size) {
 409                 DRV_LOG(ERR, "port %u cannot handle this many Rx queues (%u)",
 410                         dev->data->port_id, rxqs_n);
 411                 rte_errno = EINVAL;
 412                 return -rte_errno;
 413         }
 414         if (rxqs_n == priv->rxqs_n)
 415                 return 0;
 416         DRV_LOG(INFO, "port %u Rx queues number update: %u -> %u",
 417                 dev->data->port_id, priv->rxqs_n, rxqs_n);
 418         priv->rxqs_n = rxqs_n;
 419         /* If the requested number of RX queues is not a power of two, use the
 420          * maximum indirection table size for better balancing.
 421          * The result is always rounded to the next power of two. */
 422         reta_idx_n = (1 << log2above((rxqs_n & (rxqs_n - 1)) ?
 423                                      priv->config.ind_table_max_size :
 424                                      rxqs_n));
 425         ret = mlx5_rss_reta_index_resize(dev, reta_idx_n);
 426         if (ret)
 427                 return ret;
 428         /* When the number of RX queues is not a power of two, the remaining
 429          * table entries are padded with reused WQs and hashes are not spread
 430          * uniformly. */
 431         for (i = 0, j = 0; (i != reta_idx_n); ++i) {
 432                 (*priv->reta_idx)[i] = j;
 433                 if (++j == rxqs_n)
 434                         j = 0;
 435         }
 436         return 0;
 437 }
 438
 439 /**
 440  * Sets default tuning parameters.
 441  *
 442  * @param dev
 443  *   Pointer to Ethernet device.
 444  * @param[out] info
 445  *   Info structure output buffer.
 446  */
 447 static void
 448 mlx5_set_default_params(struct rte_eth_dev *dev, struct rte_eth_dev_info *info)
 449 {
 450         struct priv *priv = dev->data->dev_private;
 451
 452         /* Minimum CPU utilization. */
 453         info->default_rxportconf.ring_size = 256;
 454         info->default_txportconf.ring_size = 256;
 455         info->default_rxportconf.burst_size = 64;
 456         info->default_txportconf.burst_size = 64;
 457         if (priv->link_speed_capa & ETH_LINK_SPEED_100G) {
 458                 info->default_rxportconf.nb_queues = 16;
 459                 info->default_txportconf.nb_queues = 16;
 460                 if (dev->data->nb_rx_queues > 2 ||
 461                     dev->data->nb_tx_queues > 2) {
 462                         /* Max Throughput. */
 463                         info->default_rxportconf.ring_size = 2048;
 464                         info->default_txportconf.ring_size = 2048;
 465                 }
 466         } else {
 467                 info->default_rxportconf.nb_queues = 8;
 468                 info->default_txportconf.nb_queues = 8;
 469                 if (dev->data->nb_rx_queues > 2 ||
 470                     dev->data->nb_tx_queues > 2) {
 471                         /* Max Throughput. */
 472                         info->default_rxportconf.ring_size = 4096;
 473                         info->default_txportconf.ring_size = 4096;
 474                 }
 475         }
 476 }
 477
 478 /**
 479  * DPDK callback to get information about the device.
 480  *
 481  * @param dev
 482  *   Pointer to Ethernet device structure.
 483  * @param[out] info
 484  *   Info structure output buffer.
 485  */
 486 void
 487 mlx5_dev_infos_get(struct rte_eth_dev *dev, struct rte_eth_dev_info *info)
 488 {
 489         struct priv *priv = dev->data->dev_private;
 490         struct mlx5_dev_config *config = &priv->config;
 491         unsigned int max;
 492         char ifname[IF_NAMESIZE];
 493
 494         /* FIXME: we should ask the device for these values. */
 495         info->min_rx_bufsize = 32;
 496         info->max_rx_pktlen = 65536;
 497         /*
 498          * Since we need one CQ per QP, the limit is the minimum number
 499          * between the two values.
 500          */
 501         max = RTE_MIN(priv->device_attr.orig_attr.max_cq,
 502                       priv->device_attr.orig_attr.max_qp);
 503         /* If max >= 65535 then max = 0, max_rx_queues is uint16_t. */
 504         if (max >= 65535)
 505                 max = 65535;
 506         info->max_rx_queues = max;
 507         info->max_tx_queues = max;
 508         info->max_mac_addrs = MLX5_MAX_UC_MAC_ADDRESSES;
 509         info->rx_queue_offload_capa = mlx5_get_rx_queue_offloads(dev);
 510         info->rx_offload_capa = (mlx5_get_rx_port_offloads() |
 511                                  info->rx_queue_offload_capa);
 512         info->tx_offload_capa = mlx5_get_tx_port_offloads(dev);
 513         if (mlx5_get_ifname(dev, &ifname) == 0)
 514                 info->if_index = if_nametoindex(ifname);
 515         info->reta_size = priv->reta_idx_n ?
 516                 priv->reta_idx_n : config->ind_table_max_size;
 517         info->hash_key_size = rss_hash_default_key_len;
 518         info->speed_capa = priv->link_speed_capa;
 519         info->flow_type_rss_offloads = ~MLX5_RSS_HF_MASK;
 520         mlx5_set_default_params(dev, info);
 521         info->switch_info.name = dev->data->name;
 522         info->switch_info.domain_id = priv->domain_id;
 523         info->switch_info.port_id = priv->representor_id;
 524         if (priv->representor) {
 525                 unsigned int i = mlx5_dev_to_port_id(dev->device, NULL, 0);
 526                 uint16_t port_id[i];
 527
 528                 i = RTE_MIN(mlx5_dev_to_port_id(dev->device, port_id, i), i);
 529                 while (i--) {
 530                         struct priv *opriv =
 531                                 rte_eth_devices[port_id[i]].data->dev_private;
 532
 533                         if (!opriv ||
 534                             opriv->representor ||
 535                             opriv->domain_id != priv->domain_id)
 536                                 continue;
 537                         /*
 538                          * Override switch name with that of the master
 539                          * device.
 540                          */
 541                         info->switch_info.name = opriv->dev_data->name;
 542                         break;
 543                 }
 544         }
 545 }
 546
 547 /**
 548  * Get supported packet types.
 549  *
 550  * @param dev
 551  *   Pointer to Ethernet device structure.
 552  *
 553  * @return
 554  *   A pointer to the supported Packet types array.
 555  */
 556 const uint32_t *
 557 mlx5_dev_supported_ptypes_get(struct rte_eth_dev *dev)
 558 {
 559         static const uint32_t ptypes[] = {
 560                 /* refers to rxq_cq_to_pkt_type() */
 561                 RTE_PTYPE_L2_ETHER,
 562                 RTE_PTYPE_L3_IPV4_EXT_UNKNOWN,
 563                 RTE_PTYPE_L3_IPV6_EXT_UNKNOWN,
 564                 RTE_PTYPE_L4_NONFRAG,
 565                 RTE_PTYPE_L4_FRAG,
 566                 RTE_PTYPE_L4_TCP,
 567                 RTE_PTYPE_L4_UDP,
 568                 RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN,
 569                 RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN,
 570                 RTE_PTYPE_INNER_L4_NONFRAG,
 571                 RTE_PTYPE_INNER_L4_FRAG,
 572                 RTE_PTYPE_INNER_L4_TCP,
 573                 RTE_PTYPE_INNER_L4_UDP,
 574                 RTE_PTYPE_UNKNOWN
 575         };
 576
 577         if (dev->rx_pkt_burst == mlx5_rx_burst ||
 578             dev->rx_pkt_burst == mlx5_rx_burst_mprq ||
 579             dev->rx_pkt_burst == mlx5_rx_burst_vec)
 580                 return ptypes;
 581         return NULL;
 582 }
 583
 584 /**
 585  * DPDK callback to retrieve physical link information.
 586  *
 587  * @param dev
 588  *   Pointer to Ethernet device structure.
 589  * @param[out] link
 590  *   Storage for current link status.
 591  *
 592  * @return
 593  *   0 on success, a negative errno value otherwise and rte_errno is set.
 594  */
 595 static int
 596 mlx5_link_update_unlocked_gset(struct rte_eth_dev *dev,
 597                                struct rte_eth_link *link)
 598 {
 599         struct priv *priv = dev->data->dev_private;
 600         struct ethtool_cmd edata = {
 601                 .cmd = ETHTOOL_GSET /* Deprecated since Linux v4.5. */
 602         };
 603         struct ifreq ifr;
 604         struct rte_eth_link dev_link;
 605         int link_speed = 0;
 606         int ret;
 607
 608         ret = mlx5_ifreq(dev, SIOCGIFFLAGS, &ifr, 1);
 609         if (ret) {
 610                 DRV_LOG(WARNING, "port %u ioctl(SIOCGIFFLAGS) failed: %s",
 611                         dev->data->port_id, strerror(rte_errno));
 612                 return ret;
 613         }
 614         memset(&dev_link, 0, sizeof(dev_link));
 615         dev_link.link_status = ((ifr.ifr_flags & IFF_UP) &&
 616                                 (ifr.ifr_flags & IFF_RUNNING));
 617         ifr.ifr_data = (void *)&edata;
 618         ret = mlx5_ifreq(dev, SIOCETHTOOL, &ifr, 1);
 619         if (ret) {
 620                 DRV_LOG(WARNING,
 621                         "port %u ioctl(SIOCETHTOOL, ETHTOOL_GSET) failed: %s",
 622                         dev->data->port_id, strerror(rte_errno));
 623                 return ret;
 624         }
 625         link_speed = ethtool_cmd_speed(&edata);
 626         if (link_speed == -1)
 627                 dev_link.link_speed = ETH_SPEED_NUM_NONE;
 628         else
 629                 dev_link.link_speed = link_speed;
 630         priv->link_speed_capa = 0;
 631         if (edata.supported & SUPPORTED_Autoneg)
 632                 priv->link_speed_capa |= ETH_LINK_SPEED_AUTONEG;
 633         if (edata.supported & (SUPPORTED_1000baseT_Full |
 634                                SUPPORTED_1000baseKX_Full))
 635                 priv->link_speed_capa |= ETH_LINK_SPEED_1G;
 636         if (edata.supported & SUPPORTED_10000baseKR_Full)
 637                 priv->link_speed_capa |= ETH_LINK_SPEED_10G;
 638         if (edata.supported & (SUPPORTED_40000baseKR4_Full |
 639                                SUPPORTED_40000baseCR4_Full |
 640                                SUPPORTED_40000baseSR4_Full |
 641                                SUPPORTED_40000baseLR4_Full))
 642                 priv->link_speed_capa |= ETH_LINK_SPEED_40G;
 643         dev_link.link_duplex = ((edata.duplex == DUPLEX_HALF) ?
 644                                 ETH_LINK_HALF_DUPLEX : ETH_LINK_FULL_DUPLEX);
 645         dev_link.link_autoneg = !(dev->data->dev_conf.link_speeds &
 646                         ETH_LINK_SPEED_FIXED);
 647         if ((dev_link.link_speed && !dev_link.link_status) ||
 648             (!dev_link.link_speed && dev_link.link_status)) {
 649                 rte_errno = EAGAIN;
 650                 return -rte_errno;
 651         }
 652         *link = dev_link;
 653         return 0;
 654 }
 655
 656 /**
 657  * Retrieve physical link information (unlocked version using new ioctl).
 658  *
 659  * @param dev
 660  *   Pointer to Ethernet device structure.
 661  * @param[out] link
 662  *   Storage for current link status.
 663  *
 664  * @return
 665  *   0 on success, a negative errno value otherwise and rte_errno is set.
 666  */
 667 static int
 668 mlx5_link_update_unlocked_gs(struct rte_eth_dev *dev,
 669                              struct rte_eth_link *link)
 670
 671 {
 672         struct priv *priv = dev->data->dev_private;
 673         struct ethtool_link_settings gcmd = { .cmd = ETHTOOL_GLINKSETTINGS };
 674         struct ifreq ifr;
 675         struct rte_eth_link dev_link;
 676         uint64_t sc;
 677         int ret;
 678
 679         ret = mlx5_ifreq(dev, SIOCGIFFLAGS, &ifr, 1);
 680         if (ret) {
 681                 DRV_LOG(WARNING, "port %u ioctl(SIOCGIFFLAGS) failed: %s",
 682                         dev->data->port_id, strerror(rte_errno));
 683                 return ret;
 684         }
 685         memset(&dev_link, 0, sizeof(dev_link));
 686         dev_link.link_status = ((ifr.ifr_flags & IFF_UP) &&
 687                                 (ifr.ifr_flags & IFF_RUNNING));
 688         ifr.ifr_data = (void *)&gcmd;
 689         ret = mlx5_ifreq(dev, SIOCETHTOOL, &ifr, 1);
 690         if (ret) {
 691                 DRV_LOG(DEBUG,
 692                         "port %u ioctl(SIOCETHTOOL, ETHTOOL_GLINKSETTINGS)"
 693                         " failed: %s",
 694                         dev->data->port_id, strerror(rte_errno));
 695                 return ret;
 696         }
 697         gcmd.link_mode_masks_nwords = -gcmd.link_mode_masks_nwords;
 698
 699         alignas(struct ethtool_link_settings)
 700         uint8_t data[offsetof(struct ethtool_link_settings, link_mode_masks) +
 701                      sizeof(uint32_t) * gcmd.link_mode_masks_nwords * 3];
 702         struct ethtool_link_settings *ecmd = (void *)data;
 703
 704         *ecmd = gcmd;
 705         ifr.ifr_data = (void *)ecmd;
 706         ret = mlx5_ifreq(dev, SIOCETHTOOL, &ifr, 1);
 707         if (ret) {
 708                 DRV_LOG(DEBUG,
 709                         "port %u ioctl(SIOCETHTOOL, ETHTOOL_GLINKSETTINGS)"
 710                         " failed: %s",
 711                         dev->data->port_id, strerror(rte_errno));
 712                 return ret;
 713         }
 714         dev_link.link_speed = ecmd->speed;
 715         sc = ecmd->link_mode_masks[0] |
 716                 ((uint64_t)ecmd->link_mode_masks[1] << 32);
 717         priv->link_speed_capa = 0;
 718         if (sc & MLX5_BITSHIFT(ETHTOOL_LINK_MODE_Autoneg_BIT))
 719                 priv->link_speed_capa |= ETH_LINK_SPEED_AUTONEG;
 720         if (sc & (MLX5_BITSHIFT(ETHTOOL_LINK_MODE_1000baseT_Full_BIT) |
 721                   MLX5_BITSHIFT(ETHTOOL_LINK_MODE_1000baseKX_Full_BIT)))
 722                 priv->link_speed_capa |= ETH_LINK_SPEED_1G;
 723         if (sc & (MLX5_BITSHIFT(ETHTOOL_LINK_MODE_10000baseKX4_Full_BIT) |
 724                   MLX5_BITSHIFT(ETHTOOL_LINK_MODE_10000baseKR_Full_BIT) |
 725                   MLX5_BITSHIFT(ETHTOOL_LINK_MODE_10000baseR_FEC_BIT)))
 726                 priv->link_speed_capa |= ETH_LINK_SPEED_10G;
 727         if (sc & (MLX5_BITSHIFT(ETHTOOL_LINK_MODE_20000baseMLD2_Full_BIT) |
 728                   MLX5_BITSHIFT(ETHTOOL_LINK_MODE_20000baseKR2_Full_BIT)))
 729                 priv->link_speed_capa |= ETH_LINK_SPEED_20G;
 730         if (sc & (MLX5_BITSHIFT(ETHTOOL_LINK_MODE_40000baseKR4_Full_BIT) |
 731                   MLX5_BITSHIFT(ETHTOOL_LINK_MODE_40000baseCR4_Full_BIT) |
 732                   MLX5_BITSHIFT(ETHTOOL_LINK_MODE_40000baseSR4_Full_BIT) |
 733                   MLX5_BITSHIFT(ETHTOOL_LINK_MODE_40000baseLR4_Full_BIT)))
 734                 priv->link_speed_capa |= ETH_LINK_SPEED_40G;
 735         if (sc & (MLX5_BITSHIFT(ETHTOOL_LINK_MODE_56000baseKR4_Full_BIT) |
 736                   MLX5_BITSHIFT(ETHTOOL_LINK_MODE_56000baseCR4_Full_BIT) |
 737                   MLX5_BITSHIFT(ETHTOOL_LINK_MODE_56000baseSR4_Full_BIT) |
 738                   MLX5_BITSHIFT(ETHTOOL_LINK_MODE_56000baseLR4_Full_BIT)))
 739                 priv->link_speed_capa |= ETH_LINK_SPEED_56G;
 740         if (sc & (MLX5_BITSHIFT(ETHTOOL_LINK_MODE_25000baseCR_Full_BIT) |
 741                   MLX5_BITSHIFT(ETHTOOL_LINK_MODE_25000baseKR_Full_BIT) |
 742                   MLX5_BITSHIFT(ETHTOOL_LINK_MODE_25000baseSR_Full_BIT)))
 743                 priv->link_speed_capa |= ETH_LINK_SPEED_25G;
 744         if (sc & (MLX5_BITSHIFT(ETHTOOL_LINK_MODE_50000baseCR2_Full_BIT) |
 745                   MLX5_BITSHIFT(ETHTOOL_LINK_MODE_50000baseKR2_Full_BIT)))
 746                 priv->link_speed_capa |= ETH_LINK_SPEED_50G;
 747         if (sc & (MLX5_BITSHIFT(ETHTOOL_LINK_MODE_100000baseKR4_Full_BIT) |
 748                   MLX5_BITSHIFT(ETHTOOL_LINK_MODE_100000baseSR4_Full_BIT) |
 749                   MLX5_BITSHIFT(ETHTOOL_LINK_MODE_100000baseCR4_Full_BIT) |
 750                   MLX5_BITSHIFT(ETHTOOL_LINK_MODE_100000baseLR4_ER4_Full_BIT)))
 751                 priv->link_speed_capa |= ETH_LINK_SPEED_100G;
 752         dev_link.link_duplex = ((ecmd->duplex == DUPLEX_HALF) ?
 753                                 ETH_LINK_HALF_DUPLEX : ETH_LINK_FULL_DUPLEX);
 754         dev_link.link_autoneg = !(dev->data->dev_conf.link_speeds &
 755                                   ETH_LINK_SPEED_FIXED);
 756         if ((dev_link.link_speed && !dev_link.link_status) ||
 757             (!dev_link.link_speed && dev_link.link_status)) {
 758                 rte_errno = EAGAIN;
 759                 return -rte_errno;
 760         }
 761         *link = dev_link;
 762         return 0;
 763 }
 764
 765 /**
 766  * DPDK callback to retrieve physical link information.
 767  *
 768  * @param dev
 769  *   Pointer to Ethernet device structure.
 770  * @param wait_to_complete
 771  *   Wait for request completion.
 772  *
 773  * @return
 774  *   0 if link status was not updated, positive if it was, a negative errno
 775  *   value otherwise and rte_errno is set.
 776  */
 777 int
 778 mlx5_link_update(struct rte_eth_dev *dev, int wait_to_complete)
 779 {
 780         int ret;
 781         struct rte_eth_link dev_link;
 782         time_t start_time = time(NULL);
 783
 784         do {
 785                 ret = mlx5_link_update_unlocked_gs(dev, &dev_link);
 786                 if (ret)
 787                         ret = mlx5_link_update_unlocked_gset(dev, &dev_link);
 788                 if (ret == 0)
 789                         break;
 790                 /* Handle wait to complete situation. */
 791                 if (wait_to_complete && ret == -EAGAIN) {
 792                         if (abs((int)difftime(time(NULL), start_time)) <
 793                             MLX5_LINK_STATUS_TIMEOUT) {
 794                                 usleep(0);
 795                                 continue;
 796                         } else {
 797                                 rte_errno = EBUSY;
 798                                 return -rte_errno;
 799                         }
 800                 } else if (ret < 0) {
 801                         return ret;
 802                 }
 803         } while (wait_to_complete);
 804         ret = !!memcmp(&dev->data->dev_link, &dev_link,
 805                        sizeof(struct rte_eth_link));
 806         dev->data->dev_link = dev_link;
 807         return ret;
 808 }
 809
 810 /**
 811  * DPDK callback to change the MTU.
 812  *
 813  * @param dev
 814  *   Pointer to Ethernet device structure.
 815  * @param in_mtu
 816  *   New MTU.
 817  *
 818  * @return
 819  *   0 on success, a negative errno value otherwise and rte_errno is set.
 820  */
 821 int
 822 mlx5_dev_set_mtu(struct rte_eth_dev *dev, uint16_t mtu)
 823 {
 824         struct priv *priv = dev->data->dev_private;
 825         uint16_t kern_mtu = 0;
 826         int ret;
 827
 828         ret = mlx5_get_mtu(dev, &kern_mtu);
 829         if (ret)
 830                 return ret;
 831         /* Set kernel interface MTU first. */
 832         ret = mlx5_set_mtu(dev, mtu);
 833         if (ret)
 834                 return ret;
 835         ret = mlx5_get_mtu(dev, &kern_mtu);
 836         if (ret)
 837                 return ret;
 838         if (kern_mtu == mtu) {
 839                 priv->mtu = mtu;
 840                 DRV_LOG(DEBUG, "port %u adapter MTU set to %u",
 841                         dev->data->port_id, mtu);
 842                 return 0;
 843         }
 844         rte_errno = EAGAIN;
 845         return -rte_errno;
 846 }
 847
 848 /**
 849  * DPDK callback to get flow control status.
 850  *
 851  * @param dev
 852  *   Pointer to Ethernet device structure.
 853  * @param[out] fc_conf
 854  *   Flow control output buffer.
 855  *
 856  * @return
 857  *   0 on success, a negative errno value otherwise and rte_errno is set.
 858  */
 859 int
 860 mlx5_dev_get_flow_ctrl(struct rte_eth_dev *dev, struct rte_eth_fc_conf *fc_conf)
 861 {
 862         struct ifreq ifr;
 863         struct ethtool_pauseparam ethpause = {
 864                 .cmd = ETHTOOL_GPAUSEPARAM
 865         };
 866         int ret;
 867
 868         ifr.ifr_data = (void *)&ethpause;
 869         ret = mlx5_ifreq(dev, SIOCETHTOOL, &ifr, 1);
 870         if (ret) {
 871                 DRV_LOG(WARNING,
 872                         "port %u ioctl(SIOCETHTOOL, ETHTOOL_GPAUSEPARAM) failed:"
 873                         " %s",
 874                         dev->data->port_id, strerror(rte_errno));
 875                 return ret;
 876         }
 877         fc_conf->autoneg = ethpause.autoneg;
 878         if (ethpause.rx_pause && ethpause.tx_pause)
 879                 fc_conf->mode = RTE_FC_FULL;
 880         else if (ethpause.rx_pause)
 881                 fc_conf->mode = RTE_FC_RX_PAUSE;
 882         else if (ethpause.tx_pause)
 883                 fc_conf->mode = RTE_FC_TX_PAUSE;
 884         else
 885                 fc_conf->mode = RTE_FC_NONE;
 886         return 0;
 887 }
 888
 889 /**
 890  * DPDK callback to modify flow control parameters.
 891  *
 892  * @param dev
 893  *   Pointer to Ethernet device structure.
 894  * @param[in] fc_conf
 895  *   Flow control parameters.
 896  *
 897  * @return
 898  *   0 on success, a negative errno value otherwise and rte_errno is set.
 899  */
 900 int
 901 mlx5_dev_set_flow_ctrl(struct rte_eth_dev *dev, struct rte_eth_fc_conf *fc_conf)
 902 {
 903         struct ifreq ifr;
 904         struct ethtool_pauseparam ethpause = {
 905                 .cmd = ETHTOOL_SPAUSEPARAM
 906         };
 907         int ret;
 908
 909         ifr.ifr_data = (void *)&ethpause;
 910         ethpause.autoneg = fc_conf->autoneg;
 911         if (((fc_conf->mode & RTE_FC_FULL) == RTE_FC_FULL) ||
 912             (fc_conf->mode & RTE_FC_RX_PAUSE))
 913                 ethpause.rx_pause = 1;
 914         else
 915                 ethpause.rx_pause = 0;
 916
 917         if (((fc_conf->mode & RTE_FC_FULL) == RTE_FC_FULL) ||
 918             (fc_conf->mode & RTE_FC_TX_PAUSE))
 919                 ethpause.tx_pause = 1;
 920         else
 921                 ethpause.tx_pause = 0;
 922         ret = mlx5_ifreq(dev, SIOCETHTOOL, &ifr, 0);
 923         if (ret) {
 924                 DRV_LOG(WARNING,
 925                         "port %u ioctl(SIOCETHTOOL, ETHTOOL_SPAUSEPARAM)"
 926                         " failed: %s",
 927                         dev->data->port_id, strerror(rte_errno));
 928                 return ret;
 929         }
 930         return 0;
 931 }
 932
 933 /**
 934  * Get PCI information from struct ibv_device.
 935  *
 936  * @param device
 937  *   Pointer to Ethernet device structure.
 938  * @param[out] pci_addr
 939  *   PCI bus address output buffer.
 940  *
 941  * @return
 942  *   0 on success, a negative errno value otherwise and rte_errno is set.
 943  */
 944 int
 945 mlx5_ibv_device_to_pci_addr(const struct ibv_device *device,
 946                             struct rte_pci_addr *pci_addr)
 947 {
 948         FILE *file;
 949         char line[32];
 950         MKSTR(path, "%s/device/uevent", device->ibdev_path);
 951
 952         file = fopen(path, "rb");
 953         if (file == NULL) {
 954                 rte_errno = errno;
 955                 return -rte_errno;
 956         }
 957         while (fgets(line, sizeof(line), file) == line) {
 958                 size_t len = strlen(line);
 959                 int ret;
 960
 961                 /* Truncate long lines. */
 962                 if (len == (sizeof(line) - 1))
 963                         while (line[(len - 1)] != '\n') {
 964                                 ret = fgetc(file);
 965                                 if (ret == EOF)
 966                                         break;
 967                                 line[(len - 1)] = ret;
 968                         }
 969                 /* Extract information. */
 970                 if (sscanf(line,
 971                            "PCI_SLOT_NAME="
 972                            "%" SCNx32 ":%" SCNx8 ":%" SCNx8 ".%" SCNx8 "\n",
 973                            &pci_addr->domain,
 974                            &pci_addr->bus,
 975                            &pci_addr->devid,
 976                            &pci_addr->function) == 4) {
 977                         ret = 0;
 978                         break;
 979                 }
 980         }
 981         fclose(file);
 982         return 0;
 983 }
 984
 985 /**
 986  * Device status handler.
 987  *
 988  * @param dev
 989  *   Pointer to Ethernet device.
 990  * @param events
 991  *   Pointer to event flags holder.
 992  *
 993  * @return
 994  *   Events bitmap of callback process which can be called immediately.
 995  */
 996 static uint32_t
 997 mlx5_dev_status_handler(struct rte_eth_dev *dev)
 998 {
 999         struct priv *priv = dev->data->dev_private;
1000         struct ibv_async_event event;
1001         uint32_t ret = 0;
1002
1003         if (mlx5_link_update(dev, 0) == -EAGAIN) {
1004                 usleep(0);
1005                 return 0;
1006         }
1007         /* Read all message and acknowledge them. */
1008         for (;;) {
1009                 if (mlx5_glue->get_async_event(priv->ctx, &event))
1010                         break;
1011                 if ((event.event_type == IBV_EVENT_PORT_ACTIVE ||
1012                         event.event_type == IBV_EVENT_PORT_ERR) &&
1013                         (dev->data->dev_conf.intr_conf.lsc == 1))
1014                         ret |= (1 << RTE_ETH_EVENT_INTR_LSC);
1015                 else if (event.event_type == IBV_EVENT_DEVICE_FATAL &&
1016                         dev->data->dev_conf.intr_conf.rmv == 1)
1017                         ret |= (1 << RTE_ETH_EVENT_INTR_RMV);
1018                 else
1019                         DRV_LOG(DEBUG,
1020                                 "port %u event type %d on not handled",
1021                                 dev->data->port_id, event.event_type);
1022                 mlx5_glue->ack_async_event(&event);
1023         }
1024         return ret;
1025 }
1026
1027 /**
1028  * Handle interrupts from the NIC.
1029  *
1030  * @param[in] intr_handle
1031  *   Interrupt handler.
1032  * @param cb_arg
1033  *   Callback argument.
1034  */
1035 void
1036 mlx5_dev_interrupt_handler(void *cb_arg)
1037 {
1038         struct rte_eth_dev *dev = cb_arg;
1039         uint32_t events;
1040
1041         events = mlx5_dev_status_handler(dev);
1042         if (events & (1 << RTE_ETH_EVENT_INTR_LSC))
1043                 _rte_eth_dev_callback_process(dev, RTE_ETH_EVENT_INTR_LSC, NULL);
1044         if (events & (1 << RTE_ETH_EVENT_INTR_RMV))
1045                 _rte_eth_dev_callback_process(dev, RTE_ETH_EVENT_INTR_RMV, NULL);
1046 }
1047
1048 /**
1049  * Handle interrupts from the socket.
1050  *
1051  * @param cb_arg
1052  *   Callback argument.
1053  */
1054 static void
1055 mlx5_dev_handler_socket(void *cb_arg)
1056 {
1057         struct rte_eth_dev *dev = cb_arg;
1058
1059         mlx5_socket_handle(dev);
1060 }
1061
1062 /**
1063  * Uninstall interrupt handler.
1064  *
1065  * @param dev
1066  *   Pointer to Ethernet device.
1067  */
1068 void
1069 mlx5_dev_interrupt_handler_uninstall(struct rte_eth_dev *dev)
1070 {
1071         struct priv *priv = dev->data->dev_private;
1072
1073         if (dev->data->dev_conf.intr_conf.lsc ||
1074             dev->data->dev_conf.intr_conf.rmv)
1075                 rte_intr_callback_unregister(&priv->intr_handle,
1076                                              mlx5_dev_interrupt_handler, dev);
1077         if (priv->primary_socket)
1078                 rte_intr_callback_unregister(&priv->intr_handle_socket,
1079                                              mlx5_dev_handler_socket, dev);
1080         priv->intr_handle.fd = 0;
1081         priv->intr_handle.type = RTE_INTR_HANDLE_UNKNOWN;
1082         priv->intr_handle_socket.fd = 0;
1083         priv->intr_handle_socket.type = RTE_INTR_HANDLE_UNKNOWN;
1084 }
1085
1086 /**
1087  * Install interrupt handler.
1088  *
1089  * @param dev
1090  *   Pointer to Ethernet device.
1091  */
1092 void
1093 mlx5_dev_interrupt_handler_install(struct rte_eth_dev *dev)
1094 {
1095         struct priv *priv = dev->data->dev_private;
1096         int ret;
1097         int flags;
1098
1099         assert(priv->ctx->async_fd > 0);
1100         flags = fcntl(priv->ctx->async_fd, F_GETFL);
1101         ret = fcntl(priv->ctx->async_fd, F_SETFL, flags | O_NONBLOCK);
1102         if (ret) {
1103                 DRV_LOG(INFO,
1104                         "port %u failed to change file descriptor async event"
1105                         " queue",
1106                         dev->data->port_id);
1107                 dev->data->dev_conf.intr_conf.lsc = 0;
1108                 dev->data->dev_conf.intr_conf.rmv = 0;
1109         }
1110         if (dev->data->dev_conf.intr_conf.lsc ||
1111             dev->data->dev_conf.intr_conf.rmv) {
1112                 priv->intr_handle.fd = priv->ctx->async_fd;
1113                 priv->intr_handle.type = RTE_INTR_HANDLE_EXT;
1114                 rte_intr_callback_register(&priv->intr_handle,
1115                                            mlx5_dev_interrupt_handler, dev);
1116         }
1117         ret = mlx5_socket_init(dev);
1118         if (ret)
1119                 DRV_LOG(ERR, "port %u cannot initialise socket: %s",
1120                         dev->data->port_id, strerror(rte_errno));
1121         else if (priv->primary_socket) {
1122                 priv->intr_handle_socket.fd = priv->primary_socket;
1123                 priv->intr_handle_socket.type = RTE_INTR_HANDLE_EXT;
1124                 rte_intr_callback_register(&priv->intr_handle_socket,
1125                                            mlx5_dev_handler_socket, dev);
1126         }
1127 }
1128
1129 /**
1130  * DPDK callback to bring the link DOWN.
1131  *
1132  * @param dev
1133  *   Pointer to Ethernet device structure.
1134  *
1135  * @return
1136  *   0 on success, a negative errno value otherwise and rte_errno is set.
1137  */
1138 int
1139 mlx5_set_link_down(struct rte_eth_dev *dev)
1140 {
1141         return mlx5_set_flags(dev, ~IFF_UP, ~IFF_UP);
1142 }
1143
1144 /**
1145  * DPDK callback to bring the link UP.
1146  *
1147  * @param dev
1148  *   Pointer to Ethernet device structure.
1149  *
1150  * @return
1151  *   0 on success, a negative errno value otherwise and rte_errno is set.
1152  */
1153 int
1154 mlx5_set_link_up(struct rte_eth_dev *dev)
1155 {
1156         return mlx5_set_flags(dev, ~IFF_UP, IFF_UP);
1157 }
1158
1159 /**
1160  * Configure the TX function to use.
1161  *
1162  * @param dev
1163  *   Pointer to private data structure.
1164  *
1165  * @return
1166  *   Pointer to selected Tx burst function.
1167  */
1168 eth_tx_burst_t
1169 mlx5_select_tx_function(struct rte_eth_dev *dev)
1170 {
1171         struct priv *priv = dev->data->dev_private;
1172         eth_tx_burst_t tx_pkt_burst = mlx5_tx_burst;
1173         struct mlx5_dev_config *config = &priv->config;
1174         uint64_t tx_offloads = dev->data->dev_conf.txmode.offloads;
1175         int tso = !!(tx_offloads & (DEV_TX_OFFLOAD_TCP_TSO |
1176                                     DEV_TX_OFFLOAD_VXLAN_TNL_TSO |
1177                                     DEV_TX_OFFLOAD_GRE_TNL_TSO |
1178                                     DEV_TX_OFFLOAD_IP_TNL_TSO |
1179                                     DEV_TX_OFFLOAD_UDP_TNL_TSO));
1180         int swp = !!(tx_offloads & (DEV_TX_OFFLOAD_IP_TNL_TSO |
1181                                     DEV_TX_OFFLOAD_UDP_TNL_TSO |
1182                                     DEV_TX_OFFLOAD_OUTER_IPV4_CKSUM));
1183         int vlan_insert = !!(tx_offloads & DEV_TX_OFFLOAD_VLAN_INSERT);
1184
1185         assert(priv != NULL);
1186         /* Select appropriate TX function. */
1187         if (vlan_insert || tso || swp)
1188                 return tx_pkt_burst;
1189         if (config->mps == MLX5_MPW_ENHANCED) {
1190                 if (mlx5_check_vec_tx_support(dev) > 0) {
1191                         if (mlx5_check_raw_vec_tx_support(dev) > 0)
1192                                 tx_pkt_burst = mlx5_tx_burst_raw_vec;
1193                         else
1194                                 tx_pkt_burst = mlx5_tx_burst_vec;
1195                         DRV_LOG(DEBUG,
1196                                 "port %u selected enhanced MPW Tx vectorized"
1197                                 " function",
1198                                 dev->data->port_id);
1199                 } else {
1200                         tx_pkt_burst = mlx5_tx_burst_empw;
1201                         DRV_LOG(DEBUG,
1202                                 "port %u selected enhanced MPW Tx function",
1203                                 dev->data->port_id);
1204                 }
1205         } else if (config->mps && (config->txq_inline > 0)) {
1206                 tx_pkt_burst = mlx5_tx_burst_mpw_inline;
1207                 DRV_LOG(DEBUG, "port %u selected MPW inline Tx function",
1208                         dev->data->port_id);
1209         } else if (config->mps) {
1210                 tx_pkt_burst = mlx5_tx_burst_mpw;
1211                 DRV_LOG(DEBUG, "port %u selected MPW Tx function",
1212                         dev->data->port_id);
1213         }
1214         return tx_pkt_burst;
1215 }
1216
1217 /**
1218  * Configure the RX function to use.
1219  *
1220  * @param dev
1221  *   Pointer to private data structure.
1222  *
1223  * @return
1224  *   Pointer to selected Rx burst function.
1225  */
1226 eth_rx_burst_t
1227 mlx5_select_rx_function(struct rte_eth_dev *dev)
1228 {
1229         eth_rx_burst_t rx_pkt_burst = mlx5_rx_burst;
1230
1231         assert(dev != NULL);
1232         if (mlx5_check_vec_rx_support(dev) > 0) {
1233                 rx_pkt_burst = mlx5_rx_burst_vec;
1234                 DRV_LOG(DEBUG, "port %u selected Rx vectorized function",
1235                         dev->data->port_id);
1236         } else if (mlx5_mprq_enabled(dev)) {
1237                 rx_pkt_burst = mlx5_rx_burst_mprq;
1238         }
1239         return rx_pkt_burst;
1240 }
1241
1242 /**
1243  * Check if mlx5 device was removed.
1244  *
1245  * @param dev
1246  *   Pointer to Ethernet device structure.
1247  *
1248  * @return
1249  *   1 when device is removed, otherwise 0.
1250  */
1251 int
1252 mlx5_is_removed(struct rte_eth_dev *dev)
1253 {
1254         struct ibv_device_attr device_attr;
1255         struct priv *priv = dev->data->dev_private;
1256
1257         if (mlx5_glue->query_device(priv->ctx, &device_attr) == EIO)
1258                 return 1;
1259         return 0;
1260 }
1261
1262 /**
1263  * Get port ID list of mlx5 instances sharing a common device.
1264  *
1265  * @param[in] dev
1266  *   Device to look for.
1267  * @param[out] port_list
1268  *   Result buffer for collected port IDs.
1269  * @param port_list_n
1270  *   Maximum number of entries in result buffer. If 0, @p port_list can be
1271  *   NULL.
1272  *
1273  * @return
1274  *   Number of matching instances regardless of the @p port_list_n
1275  *   parameter, 0 if none were found.
1276  */
1277 unsigned int
1278 mlx5_dev_to_port_id(const struct rte_device *dev, uint16_t *port_list,
1279                     unsigned int port_list_n)
1280 {
1281         uint16_t id;
1282         unsigned int n = 0;
1283
1284         RTE_ETH_FOREACH_DEV(id) {
1285                 struct rte_eth_dev *ldev = &rte_eth_devices[id];
1286
1287                 if (!ldev->device ||
1288                     !ldev->device->driver ||
1289                     strcmp(ldev->device->driver->name, MLX5_DRIVER_NAME) ||
1290                     ldev->device != dev)
1291                         continue;
1292                 if (n < port_list_n)
1293                         port_list[n] = id;
1294                 n++;
1295         }
1296         return n;
1297 }