drivers/net/mlx5/mlx5_ethdev.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright 2015 6WIND S.A.
   3  * Copyright 2015 Mellanox Technologies, Ltd
   4  */
   5
   6 #include <stddef.h>
   7 #include <assert.h>
   8 #include <inttypes.h>
   9 #include <unistd.h>
  10 #include <stdint.h>
  11 #include <stdio.h>
  12 #include <string.h>
  13 #include <stdlib.h>
  14 #include <errno.h>
  15 #include <dirent.h>
  16 #include <net/if.h>
  17 #include <sys/ioctl.h>
  18 #include <sys/socket.h>
  19 #include <netinet/in.h>
  20 #include <linux/ethtool.h>
  21 #include <linux/sockios.h>
  22 #include <fcntl.h>
  23 #include <stdalign.h>
  24 #include <sys/un.h>
  25 #include <time.h>
  26
  27 #include <rte_atomic.h>
  28 #include <rte_ethdev_driver.h>
  29 #include <rte_bus_pci.h>
  30 #include <rte_mbuf.h>
  31 #include <rte_common.h>
  32 #include <rte_interrupts.h>
  33 #include <rte_malloc.h>
  34 #include <rte_string_fns.h>
  35 #include <rte_rwlock.h>
  36
  37 #include "mlx5.h"
  38 #include "mlx5_glue.h"
  39 #include "mlx5_rxtx.h"
  40 #include "mlx5_utils.h"
  41
  42 /* Supported speed values found in /usr/include/linux/ethtool.h */
  43 #ifndef HAVE_SUPPORTED_40000baseKR4_Full
  44 #define SUPPORTED_40000baseKR4_Full (1 << 23)
  45 #endif
  46 #ifndef HAVE_SUPPORTED_40000baseCR4_Full
  47 #define SUPPORTED_40000baseCR4_Full (1 << 24)
  48 #endif
  49 #ifndef HAVE_SUPPORTED_40000baseSR4_Full
  50 #define SUPPORTED_40000baseSR4_Full (1 << 25)
  51 #endif
  52 #ifndef HAVE_SUPPORTED_40000baseLR4_Full
  53 #define SUPPORTED_40000baseLR4_Full (1 << 26)
  54 #endif
  55 #ifndef HAVE_SUPPORTED_56000baseKR4_Full
  56 #define SUPPORTED_56000baseKR4_Full (1 << 27)
  57 #endif
  58 #ifndef HAVE_SUPPORTED_56000baseCR4_Full
  59 #define SUPPORTED_56000baseCR4_Full (1 << 28)
  60 #endif
  61 #ifndef HAVE_SUPPORTED_56000baseSR4_Full
  62 #define SUPPORTED_56000baseSR4_Full (1 << 29)
  63 #endif
  64 #ifndef HAVE_SUPPORTED_56000baseLR4_Full
  65 #define SUPPORTED_56000baseLR4_Full (1 << 30)
  66 #endif
  67
  68 /* Add defines in case the running kernel is not the same as user headers. */
  69 #ifndef ETHTOOL_GLINKSETTINGS
  70 struct ethtool_link_settings {
  71         uint32_t cmd;
  72         uint32_t speed;
  73         uint8_t duplex;
  74         uint8_t port;
  75         uint8_t phy_address;
  76         uint8_t autoneg;
  77         uint8_t mdio_support;
  78         uint8_t eth_to_mdix;
  79         uint8_t eth_tp_mdix_ctrl;
  80         int8_t link_mode_masks_nwords;
  81         uint32_t reserved[8];
  82         uint32_t link_mode_masks[];
  83 };
  84
  85 #define ETHTOOL_GLINKSETTINGS 0x0000004c
  86 #define ETHTOOL_LINK_MODE_1000baseT_Full_BIT 5
  87 #define ETHTOOL_LINK_MODE_Autoneg_BIT 6
  88 #define ETHTOOL_LINK_MODE_1000baseKX_Full_BIT 17
  89 #define ETHTOOL_LINK_MODE_10000baseKX4_Full_BIT 18
  90 #define ETHTOOL_LINK_MODE_10000baseKR_Full_BIT 19
  91 #define ETHTOOL_LINK_MODE_10000baseR_FEC_BIT 20
  92 #define ETHTOOL_LINK_MODE_20000baseMLD2_Full_BIT 21
  93 #define ETHTOOL_LINK_MODE_20000baseKR2_Full_BIT 22
  94 #define ETHTOOL_LINK_MODE_40000baseKR4_Full_BIT 23
  95 #define ETHTOOL_LINK_MODE_40000baseCR4_Full_BIT 24
  96 #define ETHTOOL_LINK_MODE_40000baseSR4_Full_BIT 25
  97 #define ETHTOOL_LINK_MODE_40000baseLR4_Full_BIT 26
  98 #define ETHTOOL_LINK_MODE_56000baseKR4_Full_BIT 27
  99 #define ETHTOOL_LINK_MODE_56000baseCR4_Full_BIT 28
 100 #define ETHTOOL_LINK_MODE_56000baseSR4_Full_BIT 29
 101 #define ETHTOOL_LINK_MODE_56000baseLR4_Full_BIT 30
 102 #endif
 103 #ifndef HAVE_ETHTOOL_LINK_MODE_25G
 104 #define ETHTOOL_LINK_MODE_25000baseCR_Full_BIT 31
 105 #define ETHTOOL_LINK_MODE_25000baseKR_Full_BIT 32
 106 #define ETHTOOL_LINK_MODE_25000baseSR_Full_BIT 33
 107 #endif
 108 #ifndef HAVE_ETHTOOL_LINK_MODE_50G
 109 #define ETHTOOL_LINK_MODE_50000baseCR2_Full_BIT 34
 110 #define ETHTOOL_LINK_MODE_50000baseKR2_Full_BIT 35
 111 #endif
 112 #ifndef HAVE_ETHTOOL_LINK_MODE_100G
 113 #define ETHTOOL_LINK_MODE_100000baseKR4_Full_BIT 36
 114 #define ETHTOOL_LINK_MODE_100000baseSR4_Full_BIT 37
 115 #define ETHTOOL_LINK_MODE_100000baseCR4_Full_BIT 38
 116 #define ETHTOOL_LINK_MODE_100000baseLR4_ER4_Full_BIT 39
 117 #endif
 118
 119 /**
 120  * Get master interface name from private structure.
 121  *
 122  * @param[in] dev
 123  *   Pointer to Ethernet device.
 124  * @param[out] ifname
 125  *   Interface name output buffer.
 126  *
 127  * @return
 128  *   0 on success, a negative errno value otherwise and rte_errno is set.
 129  */
 130 static int
 131 mlx5_get_master_ifname(const struct rte_eth_dev *dev,
 132                        char (*ifname)[IF_NAMESIZE])
 133 {
 134         struct mlx5_priv *priv = dev->data->dev_private;
 135         DIR *dir;
 136         struct dirent *dent;
 137         unsigned int dev_type = 0;
 138         unsigned int dev_port_prev = ~0u;
 139         char match[IF_NAMESIZE] = "";
 140
 141         {
 142                 MKSTR(path, "%s/device/net", priv->ibdev_path);
 143
 144                 dir = opendir(path);
 145                 if (dir == NULL) {
 146                         rte_errno = errno;
 147                         return -rte_errno;
 148                 }
 149         }
 150         while ((dent = readdir(dir)) != NULL) {
 151                 char *name = dent->d_name;
 152                 FILE *file;
 153                 unsigned int dev_port;
 154                 int r;
 155
 156                 if ((name[0] == '.') &&
 157                     ((name[1] == '\0') ||
 158                      ((name[1] == '.') && (name[2] == '\0'))))
 159                         continue;
 160
 161                 MKSTR(path, "%s/device/net/%s/%s",
 162                       priv->ibdev_path, name,
 163                       (dev_type ? "dev_id" : "dev_port"));
 164
 165                 file = fopen(path, "rb");
 166                 if (file == NULL) {
 167                         if (errno != ENOENT)
 168                                 continue;
 169                         /*
 170                          * Switch to dev_id when dev_port does not exist as
 171                          * is the case with Linux kernel versions < 3.15.
 172                          */
 173 try_dev_id:
 174                         match[0] = '\0';
 175                         if (dev_type)
 176                                 break;
 177                         dev_type = 1;
 178                         dev_port_prev = ~0u;
 179                         rewinddir(dir);
 180                         continue;
 181                 }
 182                 r = fscanf(file, (dev_type ? "%x" : "%u"), &dev_port);
 183                 fclose(file);
 184                 if (r != 1)
 185                         continue;
 186                 /*
 187                  * Switch to dev_id when dev_port returns the same value for
 188                  * all ports. May happen when using a MOFED release older than
 189                  * 3.0 with a Linux kernel >= 3.15.
 190                  */
 191                 if (dev_port == dev_port_prev)
 192                         goto try_dev_id;
 193                 dev_port_prev = dev_port;
 194                 if (dev_port == 0)
 195                         strlcpy(match, name, sizeof(match));
 196         }
 197         closedir(dir);
 198         if (match[0] == '\0') {
 199                 rte_errno = ENOENT;
 200                 return -rte_errno;
 201         }
 202         strncpy(*ifname, match, sizeof(*ifname));
 203         return 0;
 204 }
 205
 206 /**
 207  * Get interface name from private structure.
 208  *
 209  * This is a port representor-aware version of mlx5_get_master_ifname().
 210  *
 211  * @param[in] dev
 212  *   Pointer to Ethernet device.
 213  * @param[out] ifname
 214  *   Interface name output buffer.
 215  *
 216  * @return
 217  *   0 on success, a negative errno value otherwise and rte_errno is set.
 218  */
 219 int
 220 mlx5_get_ifname(const struct rte_eth_dev *dev, char (*ifname)[IF_NAMESIZE])
 221 {
 222         struct mlx5_priv *priv = dev->data->dev_private;
 223         unsigned int ifindex =
 224                 priv->nl_socket_rdma >= 0 ?
 225                 mlx5_nl_ifindex(priv->nl_socket_rdma, priv->ibdev_name, 1) : 0;
 226
 227         if (!ifindex) {
 228                 if (!priv->representor)
 229                         return mlx5_get_master_ifname(dev, ifname);
 230                 rte_errno = ENXIO;
 231                 return -rte_errno;
 232         }
 233         if (if_indextoname(ifindex, &(*ifname)[0]))
 234                 return 0;
 235         rte_errno = errno;
 236         return -rte_errno;
 237 }
 238
 239 /**
 240  * Get the interface index from device name.
 241  *
 242  * @param[in] dev
 243  *   Pointer to Ethernet device.
 244  *
 245  * @return
 246  *   Nonzero interface index on success, zero otherwise and rte_errno is set.
 247  */
 248 unsigned int
 249 mlx5_ifindex(const struct rte_eth_dev *dev)
 250 {
 251         char ifname[IF_NAMESIZE];
 252         unsigned int ifindex;
 253
 254         if (mlx5_get_ifname(dev, &ifname))
 255                 return 0;
 256         ifindex = if_nametoindex(ifname);
 257         if (!ifindex)
 258                 rte_errno = errno;
 259         return ifindex;
 260 }
 261
 262 /**
 263  * Perform ifreq ioctl() on associated Ethernet device.
 264  *
 265  * @param[in] dev
 266  *   Pointer to Ethernet device.
 267  * @param req
 268  *   Request number to pass to ioctl().
 269  * @param[out] ifr
 270  *   Interface request structure output buffer.
 271  *
 272  * @return
 273  *   0 on success, a negative errno value otherwise and rte_errno is set.
 274  */
 275 int
 276 mlx5_ifreq(const struct rte_eth_dev *dev, int req, struct ifreq *ifr)
 277 {
 278         int sock = socket(PF_INET, SOCK_DGRAM, IPPROTO_IP);
 279         int ret = 0;
 280
 281         if (sock == -1) {
 282                 rte_errno = errno;
 283                 return -rte_errno;
 284         }
 285         ret = mlx5_get_ifname(dev, &ifr->ifr_name);
 286         if (ret)
 287                 goto error;
 288         ret = ioctl(sock, req, ifr);
 289         if (ret == -1) {
 290                 rte_errno = errno;
 291                 goto error;
 292         }
 293         close(sock);
 294         return 0;
 295 error:
 296         close(sock);
 297         return -rte_errno;
 298 }
 299
 300 /**
 301  * Get device MTU.
 302  *
 303  * @param dev
 304  *   Pointer to Ethernet device.
 305  * @param[out] mtu
 306  *   MTU value output buffer.
 307  *
 308  * @return
 309  *   0 on success, a negative errno value otherwise and rte_errno is set.
 310  */
 311 int
 312 mlx5_get_mtu(struct rte_eth_dev *dev, uint16_t *mtu)
 313 {
 314         struct ifreq request;
 315         int ret = mlx5_ifreq(dev, SIOCGIFMTU, &request);
 316
 317         if (ret)
 318                 return ret;
 319         *mtu = request.ifr_mtu;
 320         return 0;
 321 }
 322
 323 /**
 324  * Set device MTU.
 325  *
 326  * @param dev
 327  *   Pointer to Ethernet device.
 328  * @param mtu
 329  *   MTU value to set.
 330  *
 331  * @return
 332  *   0 on success, a negative errno value otherwise and rte_errno is set.
 333  */
 334 static int
 335 mlx5_set_mtu(struct rte_eth_dev *dev, uint16_t mtu)
 336 {
 337         struct ifreq request = { .ifr_mtu = mtu, };
 338
 339         return mlx5_ifreq(dev, SIOCSIFMTU, &request);
 340 }
 341
 342 /**
 343  * Set device flags.
 344  *
 345  * @param dev
 346  *   Pointer to Ethernet device.
 347  * @param keep
 348  *   Bitmask for flags that must remain untouched.
 349  * @param flags
 350  *   Bitmask for flags to modify.
 351  *
 352  * @return
 353  *   0 on success, a negative errno value otherwise and rte_errno is set.
 354  */
 355 int
 356 mlx5_set_flags(struct rte_eth_dev *dev, unsigned int keep, unsigned int flags)
 357 {
 358         struct ifreq request;
 359         int ret = mlx5_ifreq(dev, SIOCGIFFLAGS, &request);
 360
 361         if (ret)
 362                 return ret;
 363         request.ifr_flags &= keep;
 364         request.ifr_flags |= flags & ~keep;
 365         return mlx5_ifreq(dev, SIOCSIFFLAGS, &request);
 366 }
 367
 368 /**
 369  * DPDK callback for Ethernet device configuration.
 370  *
 371  * @param dev
 372  *   Pointer to Ethernet device structure.
 373  *
 374  * @return
 375  *   0 on success, a negative errno value otherwise and rte_errno is set.
 376  */
 377 int
 378 mlx5_dev_configure(struct rte_eth_dev *dev)
 379 {
 380         struct mlx5_priv *priv = dev->data->dev_private;
 381         unsigned int rxqs_n = dev->data->nb_rx_queues;
 382         unsigned int txqs_n = dev->data->nb_tx_queues;
 383         unsigned int i;
 384         unsigned int j;
 385         unsigned int reta_idx_n;
 386         const uint8_t use_app_rss_key =
 387                 !!dev->data->dev_conf.rx_adv_conf.rss_conf.rss_key;
 388         int ret = 0;
 389
 390         if (use_app_rss_key &&
 391             (dev->data->dev_conf.rx_adv_conf.rss_conf.rss_key_len !=
 392              MLX5_RSS_HASH_KEY_LEN)) {
 393                 DRV_LOG(ERR, "port %u RSS key len must be %s Bytes long",
 394                         dev->data->port_id, RTE_STR(MLX5_RSS_HASH_KEY_LEN));
 395                 rte_errno = EINVAL;
 396                 return -rte_errno;
 397         }
 398         priv->rss_conf.rss_key =
 399                 rte_realloc(priv->rss_conf.rss_key,
 400                             MLX5_RSS_HASH_KEY_LEN, 0);
 401         if (!priv->rss_conf.rss_key) {
 402                 DRV_LOG(ERR, "port %u cannot allocate RSS hash key memory (%u)",
 403                         dev->data->port_id, rxqs_n);
 404                 rte_errno = ENOMEM;
 405                 return -rte_errno;
 406         }
 407         memcpy(priv->rss_conf.rss_key,
 408                use_app_rss_key ?
 409                dev->data->dev_conf.rx_adv_conf.rss_conf.rss_key :
 410                rss_hash_default_key,
 411                MLX5_RSS_HASH_KEY_LEN);
 412         priv->rss_conf.rss_key_len = MLX5_RSS_HASH_KEY_LEN;
 413         priv->rss_conf.rss_hf = dev->data->dev_conf.rx_adv_conf.rss_conf.rss_hf;
 414         priv->rxqs = (void *)dev->data->rx_queues;
 415         priv->txqs = (void *)dev->data->tx_queues;
 416         if (txqs_n != priv->txqs_n) {
 417                 DRV_LOG(INFO, "port %u Tx queues number update: %u -> %u",
 418                         dev->data->port_id, priv->txqs_n, txqs_n);
 419                 priv->txqs_n = txqs_n;
 420         }
 421         if (rxqs_n > priv->config.ind_table_max_size) {
 422                 DRV_LOG(ERR, "port %u cannot handle this many Rx queues (%u)",
 423                         dev->data->port_id, rxqs_n);
 424                 rte_errno = EINVAL;
 425                 return -rte_errno;
 426         }
 427         if (rxqs_n == priv->rxqs_n)
 428                 return 0;
 429         DRV_LOG(INFO, "port %u Rx queues number update: %u -> %u",
 430                 dev->data->port_id, priv->rxqs_n, rxqs_n);
 431         priv->rxqs_n = rxqs_n;
 432         /* If the requested number of RX queues is not a power of two, use the
 433          * maximum indirection table size for better balancing.
 434          * The result is always rounded to the next power of two. */
 435         reta_idx_n = (1 << log2above((rxqs_n & (rxqs_n - 1)) ?
 436                                      priv->config.ind_table_max_size :
 437                                      rxqs_n));
 438         ret = mlx5_rss_reta_index_resize(dev, reta_idx_n);
 439         if (ret)
 440                 return ret;
 441         /* When the number of RX queues is not a power of two, the remaining
 442          * table entries are padded with reused WQs and hashes are not spread
 443          * uniformly. */
 444         for (i = 0, j = 0; (i != reta_idx_n); ++i) {
 445                 (*priv->reta_idx)[i] = j;
 446                 if (++j == rxqs_n)
 447                         j = 0;
 448         }
 449         return 0;
 450 }
 451
 452 /**
 453  * Sets default tuning parameters.
 454  *
 455  * @param dev
 456  *   Pointer to Ethernet device.
 457  * @param[out] info
 458  *   Info structure output buffer.
 459  */
 460 static void
 461 mlx5_set_default_params(struct rte_eth_dev *dev, struct rte_eth_dev_info *info)
 462 {
 463         struct mlx5_priv *priv = dev->data->dev_private;
 464
 465         /* Minimum CPU utilization. */
 466         info->default_rxportconf.ring_size = 256;
 467         info->default_txportconf.ring_size = 256;
 468         info->default_rxportconf.burst_size = 64;
 469         info->default_txportconf.burst_size = 64;
 470         if (priv->link_speed_capa & ETH_LINK_SPEED_100G) {
 471                 info->default_rxportconf.nb_queues = 16;
 472                 info->default_txportconf.nb_queues = 16;
 473                 if (dev->data->nb_rx_queues > 2 ||
 474                     dev->data->nb_tx_queues > 2) {
 475                         /* Max Throughput. */
 476                         info->default_rxportconf.ring_size = 2048;
 477                         info->default_txportconf.ring_size = 2048;
 478                 }
 479         } else {
 480                 info->default_rxportconf.nb_queues = 8;
 481                 info->default_txportconf.nb_queues = 8;
 482                 if (dev->data->nb_rx_queues > 2 ||
 483                     dev->data->nb_tx_queues > 2) {
 484                         /* Max Throughput. */
 485                         info->default_rxportconf.ring_size = 4096;
 486                         info->default_txportconf.ring_size = 4096;
 487                 }
 488         }
 489 }
 490
 491 /**
 492  * DPDK callback to get information about the device.
 493  *
 494  * @param dev
 495  *   Pointer to Ethernet device structure.
 496  * @param[out] info
 497  *   Info structure output buffer.
 498  */
 499 void
 500 mlx5_dev_infos_get(struct rte_eth_dev *dev, struct rte_eth_dev_info *info)
 501 {
 502         struct mlx5_priv *priv = dev->data->dev_private;
 503         struct mlx5_dev_config *config = &priv->config;
 504         unsigned int max;
 505         char ifname[IF_NAMESIZE];
 506
 507         /* FIXME: we should ask the device for these values. */
 508         info->min_rx_bufsize = 32;
 509         info->max_rx_pktlen = 65536;
 510         /*
 511          * Since we need one CQ per QP, the limit is the minimum number
 512          * between the two values.
 513          */
 514         max = RTE_MIN(priv->device_attr.orig_attr.max_cq,
 515                       priv->device_attr.orig_attr.max_qp);
 516         /* If max >= 65535 then max = 0, max_rx_queues is uint16_t. */
 517         if (max >= 65535)
 518                 max = 65535;
 519         info->max_rx_queues = max;
 520         info->max_tx_queues = max;
 521         info->max_mac_addrs = MLX5_MAX_UC_MAC_ADDRESSES;
 522         info->rx_queue_offload_capa = mlx5_get_rx_queue_offloads(dev);
 523         info->rx_offload_capa = (mlx5_get_rx_port_offloads() |
 524                                  info->rx_queue_offload_capa);
 525         info->tx_offload_capa = mlx5_get_tx_port_offloads(dev);
 526         if (mlx5_get_ifname(dev, &ifname) == 0)
 527                 info->if_index = if_nametoindex(ifname);
 528         info->reta_size = priv->reta_idx_n ?
 529                 priv->reta_idx_n : config->ind_table_max_size;
 530         info->hash_key_size = MLX5_RSS_HASH_KEY_LEN;
 531         info->speed_capa = priv->link_speed_capa;
 532         info->flow_type_rss_offloads = ~MLX5_RSS_HF_MASK;
 533         mlx5_set_default_params(dev, info);
 534         info->switch_info.name = dev->data->name;
 535         info->switch_info.domain_id = priv->domain_id;
 536         info->switch_info.port_id = priv->representor_id;
 537         if (priv->representor) {
 538                 unsigned int i = mlx5_dev_to_port_id(dev->device, NULL, 0);
 539                 uint16_t port_id[i];
 540
 541                 i = RTE_MIN(mlx5_dev_to_port_id(dev->device, port_id, i), i);
 542                 while (i--) {
 543                         struct mlx5_priv *opriv =
 544                                 rte_eth_devices[port_id[i]].data->dev_private;
 545
 546                         if (!opriv ||
 547                             opriv->representor ||
 548                             opriv->domain_id != priv->domain_id)
 549                                 continue;
 550                         /*
 551                          * Override switch name with that of the master
 552                          * device.
 553                          */
 554                         info->switch_info.name = opriv->dev_data->name;
 555                         break;
 556                 }
 557         }
 558 }
 559
 560 /**
 561  * Get firmware version of a device.
 562  *
 563  * @param dev
 564  *   Ethernet device port.
 565  * @param fw_ver
 566  *   String output allocated by caller.
 567  * @param fw_size
 568  *   Size of the output string, including terminating null byte.
 569  *
 570  * @return
 571  *   0 on success, or the size of the non truncated string if too big.
 572  */
 573 int mlx5_fw_version_get(struct rte_eth_dev *dev, char *fw_ver, size_t fw_size)
 574 {
 575         struct mlx5_priv *priv = dev->data->dev_private;
 576         struct ibv_device_attr *attr = &priv->device_attr.orig_attr;
 577         size_t size = strnlen(attr->fw_ver, sizeof(attr->fw_ver)) + 1;
 578
 579         if (fw_size < size)
 580                 return size;
 581         if (fw_ver != NULL)
 582                 strlcpy(fw_ver, attr->fw_ver, fw_size);
 583         return 0;
 584 }
 585
 586 /**
 587  * Get supported packet types.
 588  *
 589  * @param dev
 590  *   Pointer to Ethernet device structure.
 591  *
 592  * @return
 593  *   A pointer to the supported Packet types array.
 594  */
 595 const uint32_t *
 596 mlx5_dev_supported_ptypes_get(struct rte_eth_dev *dev)
 597 {
 598         static const uint32_t ptypes[] = {
 599                 /* refers to rxq_cq_to_pkt_type() */
 600                 RTE_PTYPE_L2_ETHER,
 601                 RTE_PTYPE_L3_IPV4_EXT_UNKNOWN,
 602                 RTE_PTYPE_L3_IPV6_EXT_UNKNOWN,
 603                 RTE_PTYPE_L4_NONFRAG,
 604                 RTE_PTYPE_L4_FRAG,
 605                 RTE_PTYPE_L4_TCP,
 606                 RTE_PTYPE_L4_UDP,
 607                 RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN,
 608                 RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN,
 609                 RTE_PTYPE_INNER_L4_NONFRAG,
 610                 RTE_PTYPE_INNER_L4_FRAG,
 611                 RTE_PTYPE_INNER_L4_TCP,
 612                 RTE_PTYPE_INNER_L4_UDP,
 613                 RTE_PTYPE_UNKNOWN
 614         };
 615
 616         if (dev->rx_pkt_burst == mlx5_rx_burst ||
 617             dev->rx_pkt_burst == mlx5_rx_burst_mprq ||
 618             dev->rx_pkt_burst == mlx5_rx_burst_vec)
 619                 return ptypes;
 620         return NULL;
 621 }
 622
 623 /**
 624  * DPDK callback to retrieve physical link information.
 625  *
 626  * @param dev
 627  *   Pointer to Ethernet device structure.
 628  * @param[out] link
 629  *   Storage for current link status.
 630  *
 631  * @return
 632  *   0 on success, a negative errno value otherwise and rte_errno is set.
 633  */
 634 static int
 635 mlx5_link_update_unlocked_gset(struct rte_eth_dev *dev,
 636                                struct rte_eth_link *link)
 637 {
 638         struct mlx5_priv *priv = dev->data->dev_private;
 639         struct ethtool_cmd edata = {
 640                 .cmd = ETHTOOL_GSET /* Deprecated since Linux v4.5. */
 641         };
 642         struct ifreq ifr;
 643         struct rte_eth_link dev_link;
 644         int link_speed = 0;
 645         int ret;
 646
 647         ret = mlx5_ifreq(dev, SIOCGIFFLAGS, &ifr);
 648         if (ret) {
 649                 DRV_LOG(WARNING, "port %u ioctl(SIOCGIFFLAGS) failed: %s",
 650                         dev->data->port_id, strerror(rte_errno));
 651                 return ret;
 652         }
 653         dev_link = (struct rte_eth_link) {
 654                 .link_status = ((ifr.ifr_flags & IFF_UP) &&
 655                                 (ifr.ifr_flags & IFF_RUNNING)),
 656         };
 657         ifr = (struct ifreq) {
 658                 .ifr_data = (void *)&edata,
 659         };
 660         ret = mlx5_ifreq(dev, SIOCETHTOOL, &ifr);
 661         if (ret) {
 662                 DRV_LOG(WARNING,
 663                         "port %u ioctl(SIOCETHTOOL, ETHTOOL_GSET) failed: %s",
 664                         dev->data->port_id, strerror(rte_errno));
 665                 return ret;
 666         }
 667         link_speed = ethtool_cmd_speed(&edata);
 668         if (link_speed == -1)
 669                 dev_link.link_speed = ETH_SPEED_NUM_NONE;
 670         else
 671                 dev_link.link_speed = link_speed;
 672         priv->link_speed_capa = 0;
 673         if (edata.supported & SUPPORTED_Autoneg)
 674                 priv->link_speed_capa |= ETH_LINK_SPEED_AUTONEG;
 675         if (edata.supported & (SUPPORTED_1000baseT_Full |
 676                                SUPPORTED_1000baseKX_Full))
 677                 priv->link_speed_capa |= ETH_LINK_SPEED_1G;
 678         if (edata.supported & SUPPORTED_10000baseKR_Full)
 679                 priv->link_speed_capa |= ETH_LINK_SPEED_10G;
 680         if (edata.supported & (SUPPORTED_40000baseKR4_Full |
 681                                SUPPORTED_40000baseCR4_Full |
 682                                SUPPORTED_40000baseSR4_Full |
 683                                SUPPORTED_40000baseLR4_Full))
 684                 priv->link_speed_capa |= ETH_LINK_SPEED_40G;
 685         dev_link.link_duplex = ((edata.duplex == DUPLEX_HALF) ?
 686                                 ETH_LINK_HALF_DUPLEX : ETH_LINK_FULL_DUPLEX);
 687         dev_link.link_autoneg = !(dev->data->dev_conf.link_speeds &
 688                         ETH_LINK_SPEED_FIXED);
 689         if (((dev_link.link_speed && !dev_link.link_status) ||
 690              (!dev_link.link_speed && dev_link.link_status))) {
 691                 rte_errno = EAGAIN;
 692                 return -rte_errno;
 693         }
 694         *link = dev_link;
 695         return 0;
 696 }
 697
 698 /**
 699  * Retrieve physical link information (unlocked version using new ioctl).
 700  *
 701  * @param dev
 702  *   Pointer to Ethernet device structure.
 703  * @param[out] link
 704  *   Storage for current link status.
 705  *
 706  * @return
 707  *   0 on success, a negative errno value otherwise and rte_errno is set.
 708  */
 709 static int
 710 mlx5_link_update_unlocked_gs(struct rte_eth_dev *dev,
 711                              struct rte_eth_link *link)
 712
 713 {
 714         struct mlx5_priv *priv = dev->data->dev_private;
 715         struct ethtool_link_settings gcmd = { .cmd = ETHTOOL_GLINKSETTINGS };
 716         struct ifreq ifr;
 717         struct rte_eth_link dev_link;
 718         uint64_t sc;
 719         int ret;
 720
 721         ret = mlx5_ifreq(dev, SIOCGIFFLAGS, &ifr);
 722         if (ret) {
 723                 DRV_LOG(WARNING, "port %u ioctl(SIOCGIFFLAGS) failed: %s",
 724                         dev->data->port_id, strerror(rte_errno));
 725                 return ret;
 726         }
 727         dev_link = (struct rte_eth_link) {
 728                 .link_status = ((ifr.ifr_flags & IFF_UP) &&
 729                                 (ifr.ifr_flags & IFF_RUNNING)),
 730         };
 731         ifr = (struct ifreq) {
 732                 .ifr_data = (void *)&gcmd,
 733         };
 734         ret = mlx5_ifreq(dev, SIOCETHTOOL, &ifr);
 735         if (ret) {
 736                 DRV_LOG(DEBUG,
 737                         "port %u ioctl(SIOCETHTOOL, ETHTOOL_GLINKSETTINGS)"
 738                         " failed: %s",
 739                         dev->data->port_id, strerror(rte_errno));
 740                 return ret;
 741         }
 742         gcmd.link_mode_masks_nwords = -gcmd.link_mode_masks_nwords;
 743
 744         alignas(struct ethtool_link_settings)
 745         uint8_t data[offsetof(struct ethtool_link_settings, link_mode_masks) +
 746                      sizeof(uint32_t) * gcmd.link_mode_masks_nwords * 3];
 747         struct ethtool_link_settings *ecmd = (void *)data;
 748
 749         *ecmd = gcmd;
 750         ifr.ifr_data = (void *)ecmd;
 751         ret = mlx5_ifreq(dev, SIOCETHTOOL, &ifr);
 752         if (ret) {
 753                 DRV_LOG(DEBUG,
 754                         "port %u ioctl(SIOCETHTOOL, ETHTOOL_GLINKSETTINGS)"
 755                         " failed: %s",
 756                         dev->data->port_id, strerror(rte_errno));
 757                 return ret;
 758         }
 759         dev_link.link_speed = ecmd->speed;
 760         sc = ecmd->link_mode_masks[0] |
 761                 ((uint64_t)ecmd->link_mode_masks[1] << 32);
 762         priv->link_speed_capa = 0;
 763         if (sc & MLX5_BITSHIFT(ETHTOOL_LINK_MODE_Autoneg_BIT))
 764                 priv->link_speed_capa |= ETH_LINK_SPEED_AUTONEG;
 765         if (sc & (MLX5_BITSHIFT(ETHTOOL_LINK_MODE_1000baseT_Full_BIT) |
 766                   MLX5_BITSHIFT(ETHTOOL_LINK_MODE_1000baseKX_Full_BIT)))
 767                 priv->link_speed_capa |= ETH_LINK_SPEED_1G;
 768         if (sc & (MLX5_BITSHIFT(ETHTOOL_LINK_MODE_10000baseKX4_Full_BIT) |
 769                   MLX5_BITSHIFT(ETHTOOL_LINK_MODE_10000baseKR_Full_BIT) |
 770                   MLX5_BITSHIFT(ETHTOOL_LINK_MODE_10000baseR_FEC_BIT)))
 771                 priv->link_speed_capa |= ETH_LINK_SPEED_10G;
 772         if (sc & (MLX5_BITSHIFT(ETHTOOL_LINK_MODE_20000baseMLD2_Full_BIT) |
 773                   MLX5_BITSHIFT(ETHTOOL_LINK_MODE_20000baseKR2_Full_BIT)))
 774                 priv->link_speed_capa |= ETH_LINK_SPEED_20G;
 775         if (sc & (MLX5_BITSHIFT(ETHTOOL_LINK_MODE_40000baseKR4_Full_BIT) |
 776                   MLX5_BITSHIFT(ETHTOOL_LINK_MODE_40000baseCR4_Full_BIT) |
 777                   MLX5_BITSHIFT(ETHTOOL_LINK_MODE_40000baseSR4_Full_BIT) |
 778                   MLX5_BITSHIFT(ETHTOOL_LINK_MODE_40000baseLR4_Full_BIT)))
 779                 priv->link_speed_capa |= ETH_LINK_SPEED_40G;
 780         if (sc & (MLX5_BITSHIFT(ETHTOOL_LINK_MODE_56000baseKR4_Full_BIT) |
 781                   MLX5_BITSHIFT(ETHTOOL_LINK_MODE_56000baseCR4_Full_BIT) |
 782                   MLX5_BITSHIFT(ETHTOOL_LINK_MODE_56000baseSR4_Full_BIT) |
 783                   MLX5_BITSHIFT(ETHTOOL_LINK_MODE_56000baseLR4_Full_BIT)))
 784                 priv->link_speed_capa |= ETH_LINK_SPEED_56G;
 785         if (sc & (MLX5_BITSHIFT(ETHTOOL_LINK_MODE_25000baseCR_Full_BIT) |
 786                   MLX5_BITSHIFT(ETHTOOL_LINK_MODE_25000baseKR_Full_BIT) |
 787                   MLX5_BITSHIFT(ETHTOOL_LINK_MODE_25000baseSR_Full_BIT)))
 788                 priv->link_speed_capa |= ETH_LINK_SPEED_25G;
 789         if (sc & (MLX5_BITSHIFT(ETHTOOL_LINK_MODE_50000baseCR2_Full_BIT) |
 790                   MLX5_BITSHIFT(ETHTOOL_LINK_MODE_50000baseKR2_Full_BIT)))
 791                 priv->link_speed_capa |= ETH_LINK_SPEED_50G;
 792         if (sc & (MLX5_BITSHIFT(ETHTOOL_LINK_MODE_100000baseKR4_Full_BIT) |
 793                   MLX5_BITSHIFT(ETHTOOL_LINK_MODE_100000baseSR4_Full_BIT) |
 794                   MLX5_BITSHIFT(ETHTOOL_LINK_MODE_100000baseCR4_Full_BIT) |
 795                   MLX5_BITSHIFT(ETHTOOL_LINK_MODE_100000baseLR4_ER4_Full_BIT)))
 796                 priv->link_speed_capa |= ETH_LINK_SPEED_100G;
 797         dev_link.link_duplex = ((ecmd->duplex == DUPLEX_HALF) ?
 798                                 ETH_LINK_HALF_DUPLEX : ETH_LINK_FULL_DUPLEX);
 799         dev_link.link_autoneg = !(dev->data->dev_conf.link_speeds &
 800                                   ETH_LINK_SPEED_FIXED);
 801         if (((dev_link.link_speed && !dev_link.link_status) ||
 802              (!dev_link.link_speed && dev_link.link_status))) {
 803                 rte_errno = EAGAIN;
 804                 return -rte_errno;
 805         }
 806         *link = dev_link;
 807         return 0;
 808 }
 809
 810 /**
 811  * DPDK callback to retrieve physical link information.
 812  *
 813  * @param dev
 814  *   Pointer to Ethernet device structure.
 815  * @param wait_to_complete
 816  *   Wait for request completion.
 817  *
 818  * @return
 819  *   0 if link status was not updated, positive if it was, a negative errno
 820  *   value otherwise and rte_errno is set.
 821  */
 822 int
 823 mlx5_link_update(struct rte_eth_dev *dev, int wait_to_complete)
 824 {
 825         int ret;
 826         struct rte_eth_link dev_link;
 827         time_t start_time = time(NULL);
 828
 829         do {
 830                 ret = mlx5_link_update_unlocked_gs(dev, &dev_link);
 831                 if (ret)
 832                         ret = mlx5_link_update_unlocked_gset(dev, &dev_link);
 833                 if (ret == 0)
 834                         break;
 835                 /* Handle wait to complete situation. */
 836                 if (wait_to_complete && ret == -EAGAIN) {
 837                         if (abs((int)difftime(time(NULL), start_time)) <
 838                             MLX5_LINK_STATUS_TIMEOUT) {
 839                                 usleep(0);
 840                                 continue;
 841                         } else {
 842                                 rte_errno = EBUSY;
 843                                 return -rte_errno;
 844                         }
 845                 } else if (ret < 0) {
 846                         return ret;
 847                 }
 848         } while (wait_to_complete);
 849         ret = !!memcmp(&dev->data->dev_link, &dev_link,
 850                        sizeof(struct rte_eth_link));
 851         dev->data->dev_link = dev_link;
 852         return ret;
 853 }
 854
 855 /**
 856  * DPDK callback to change the MTU.
 857  *
 858  * @param dev
 859  *   Pointer to Ethernet device structure.
 860  * @param in_mtu
 861  *   New MTU.
 862  *
 863  * @return
 864  *   0 on success, a negative errno value otherwise and rte_errno is set.
 865  */
 866 int
 867 mlx5_dev_set_mtu(struct rte_eth_dev *dev, uint16_t mtu)
 868 {
 869         struct mlx5_priv *priv = dev->data->dev_private;
 870         uint16_t kern_mtu = 0;
 871         int ret;
 872
 873         ret = mlx5_get_mtu(dev, &kern_mtu);
 874         if (ret)
 875                 return ret;
 876         /* Set kernel interface MTU first. */
 877         ret = mlx5_set_mtu(dev, mtu);
 878         if (ret)
 879                 return ret;
 880         ret = mlx5_get_mtu(dev, &kern_mtu);
 881         if (ret)
 882                 return ret;
 883         if (kern_mtu == mtu) {
 884                 priv->mtu = mtu;
 885                 DRV_LOG(DEBUG, "port %u adapter MTU set to %u",
 886                         dev->data->port_id, mtu);
 887                 return 0;
 888         }
 889         rte_errno = EAGAIN;
 890         return -rte_errno;
 891 }
 892
 893 /**
 894  * DPDK callback to get flow control status.
 895  *
 896  * @param dev
 897  *   Pointer to Ethernet device structure.
 898  * @param[out] fc_conf
 899  *   Flow control output buffer.
 900  *
 901  * @return
 902  *   0 on success, a negative errno value otherwise and rte_errno is set.
 903  */
 904 int
 905 mlx5_dev_get_flow_ctrl(struct rte_eth_dev *dev, struct rte_eth_fc_conf *fc_conf)
 906 {
 907         struct ifreq ifr;
 908         struct ethtool_pauseparam ethpause = {
 909                 .cmd = ETHTOOL_GPAUSEPARAM
 910         };
 911         int ret;
 912
 913         ifr.ifr_data = (void *)&ethpause;
 914         ret = mlx5_ifreq(dev, SIOCETHTOOL, &ifr);
 915         if (ret) {
 916                 DRV_LOG(WARNING,
 917                         "port %u ioctl(SIOCETHTOOL, ETHTOOL_GPAUSEPARAM) failed:"
 918                         " %s",
 919                         dev->data->port_id, strerror(rte_errno));
 920                 return ret;
 921         }
 922         fc_conf->autoneg = ethpause.autoneg;
 923         if (ethpause.rx_pause && ethpause.tx_pause)
 924                 fc_conf->mode = RTE_FC_FULL;
 925         else if (ethpause.rx_pause)
 926                 fc_conf->mode = RTE_FC_RX_PAUSE;
 927         else if (ethpause.tx_pause)
 928                 fc_conf->mode = RTE_FC_TX_PAUSE;
 929         else
 930                 fc_conf->mode = RTE_FC_NONE;
 931         return 0;
 932 }
 933
 934 /**
 935  * DPDK callback to modify flow control parameters.
 936  *
 937  * @param dev
 938  *   Pointer to Ethernet device structure.
 939  * @param[in] fc_conf
 940  *   Flow control parameters.
 941  *
 942  * @return
 943  *   0 on success, a negative errno value otherwise and rte_errno is set.
 944  */
 945 int
 946 mlx5_dev_set_flow_ctrl(struct rte_eth_dev *dev, struct rte_eth_fc_conf *fc_conf)
 947 {
 948         struct ifreq ifr;
 949         struct ethtool_pauseparam ethpause = {
 950                 .cmd = ETHTOOL_SPAUSEPARAM
 951         };
 952         int ret;
 953
 954         ifr.ifr_data = (void *)&ethpause;
 955         ethpause.autoneg = fc_conf->autoneg;
 956         if (((fc_conf->mode & RTE_FC_FULL) == RTE_FC_FULL) ||
 957             (fc_conf->mode & RTE_FC_RX_PAUSE))
 958                 ethpause.rx_pause = 1;
 959         else
 960                 ethpause.rx_pause = 0;
 961
 962         if (((fc_conf->mode & RTE_FC_FULL) == RTE_FC_FULL) ||
 963             (fc_conf->mode & RTE_FC_TX_PAUSE))
 964                 ethpause.tx_pause = 1;
 965         else
 966                 ethpause.tx_pause = 0;
 967         ret = mlx5_ifreq(dev, SIOCETHTOOL, &ifr);
 968         if (ret) {
 969                 DRV_LOG(WARNING,
 970                         "port %u ioctl(SIOCETHTOOL, ETHTOOL_SPAUSEPARAM)"
 971                         " failed: %s",
 972                         dev->data->port_id, strerror(rte_errno));
 973                 return ret;
 974         }
 975         return 0;
 976 }
 977
 978 /**
 979  * Get PCI information from struct ibv_device.
 980  *
 981  * @param device
 982  *   Pointer to Ethernet device structure.
 983  * @param[out] pci_addr
 984  *   PCI bus address output buffer.
 985  *
 986  * @return
 987  *   0 on success, a negative errno value otherwise and rte_errno is set.
 988  */
 989 int
 990 mlx5_ibv_device_to_pci_addr(const struct ibv_device *device,
 991                             struct rte_pci_addr *pci_addr)
 992 {
 993         FILE *file;
 994         char line[32];
 995         MKSTR(path, "%s/device/uevent", device->ibdev_path);
 996
 997         file = fopen(path, "rb");
 998         if (file == NULL) {
 999                 rte_errno = errno;
1000                 return -rte_errno;
1001         }
1002         while (fgets(line, sizeof(line), file) == line) {
1003                 size_t len = strlen(line);
1004                 int ret;
1005
1006                 /* Truncate long lines. */
1007                 if (len == (sizeof(line) - 1))
1008                         while (line[(len - 1)] != '\n') {
1009                                 ret = fgetc(file);
1010                                 if (ret == EOF)
1011                                         break;
1012                                 line[(len - 1)] = ret;
1013                         }
1014                 /* Extract information. */
1015                 if (sscanf(line,
1016                            "PCI_SLOT_NAME="
1017                            "%" SCNx32 ":%" SCNx8 ":%" SCNx8 ".%" SCNx8 "\n",
1018                            &pci_addr->domain,
1019                            &pci_addr->bus,
1020                            &pci_addr->devid,
1021                            &pci_addr->function) == 4) {
1022                         ret = 0;
1023                         break;
1024                 }
1025         }
1026         fclose(file);
1027         return 0;
1028 }
1029
1030 /**
1031  * Device status handler.
1032  *
1033  * @param dev
1034  *   Pointer to Ethernet device.
1035  * @param events
1036  *   Pointer to event flags holder.
1037  *
1038  * @return
1039  *   Events bitmap of callback process which can be called immediately.
1040  */
1041 static uint32_t
1042 mlx5_dev_status_handler(struct rte_eth_dev *dev)
1043 {
1044         struct mlx5_priv *priv = dev->data->dev_private;
1045         struct ibv_async_event event;
1046         uint32_t ret = 0;
1047
1048         if (mlx5_link_update(dev, 0) == -EAGAIN) {
1049                 usleep(0);
1050                 return 0;
1051         }
1052         /* Read all message and acknowledge them. */
1053         for (;;) {
1054                 if (mlx5_glue->get_async_event(priv->ctx, &event))
1055                         break;
1056                 if ((event.event_type == IBV_EVENT_PORT_ACTIVE ||
1057                         event.event_type == IBV_EVENT_PORT_ERR) &&
1058                         (dev->data->dev_conf.intr_conf.lsc == 1))
1059                         ret |= (1 << RTE_ETH_EVENT_INTR_LSC);
1060                 else if (event.event_type == IBV_EVENT_DEVICE_FATAL &&
1061                         dev->data->dev_conf.intr_conf.rmv == 1)
1062                         ret |= (1 << RTE_ETH_EVENT_INTR_RMV);
1063                 else
1064                         DRV_LOG(DEBUG,
1065                                 "port %u event type %d on not handled",
1066                                 dev->data->port_id, event.event_type);
1067                 mlx5_glue->ack_async_event(&event);
1068         }
1069         return ret;
1070 }
1071
1072 /**
1073  * Handle interrupts from the NIC.
1074  *
1075  * @param[in] intr_handle
1076  *   Interrupt handler.
1077  * @param cb_arg
1078  *   Callback argument.
1079  */
1080 void
1081 mlx5_dev_interrupt_handler(void *cb_arg)
1082 {
1083         struct rte_eth_dev *dev = cb_arg;
1084         uint32_t events;
1085
1086         events = mlx5_dev_status_handler(dev);
1087         if (events & (1 << RTE_ETH_EVENT_INTR_LSC))
1088                 _rte_eth_dev_callback_process(dev, RTE_ETH_EVENT_INTR_LSC, NULL);
1089         if (events & (1 << RTE_ETH_EVENT_INTR_RMV))
1090                 _rte_eth_dev_callback_process(dev, RTE_ETH_EVENT_INTR_RMV, NULL);
1091 }
1092
1093 /**
1094  * Handle interrupts from the socket.
1095  *
1096  * @param cb_arg
1097  *   Callback argument.
1098  */
1099 static void
1100 mlx5_dev_handler_socket(void *cb_arg)
1101 {
1102         struct rte_eth_dev *dev = cb_arg;
1103
1104         mlx5_socket_handle(dev);
1105 }
1106
1107 /**
1108  * Uninstall interrupt handler.
1109  *
1110  * @param dev
1111  *   Pointer to Ethernet device.
1112  */
1113 void
1114 mlx5_dev_interrupt_handler_uninstall(struct rte_eth_dev *dev)
1115 {
1116         struct mlx5_priv *priv = dev->data->dev_private;
1117
1118         if (dev->data->dev_conf.intr_conf.lsc ||
1119             dev->data->dev_conf.intr_conf.rmv)
1120                 rte_intr_callback_unregister(&priv->intr_handle,
1121                                              mlx5_dev_interrupt_handler, dev);
1122         if (priv->primary_socket)
1123                 rte_intr_callback_unregister(&priv->intr_handle_socket,
1124                                              mlx5_dev_handler_socket, dev);
1125         priv->intr_handle.fd = 0;
1126         priv->intr_handle.type = RTE_INTR_HANDLE_UNKNOWN;
1127         priv->intr_handle_socket.fd = 0;
1128         priv->intr_handle_socket.type = RTE_INTR_HANDLE_UNKNOWN;
1129 }
1130
1131 /**
1132  * Install interrupt handler.
1133  *
1134  * @param dev
1135  *   Pointer to Ethernet device.
1136  */
1137 void
1138 mlx5_dev_interrupt_handler_install(struct rte_eth_dev *dev)
1139 {
1140         struct mlx5_priv *priv = dev->data->dev_private;
1141         int ret;
1142         int flags;
1143
1144         assert(priv->ctx->async_fd > 0);
1145         flags = fcntl(priv->ctx->async_fd, F_GETFL);
1146         ret = fcntl(priv->ctx->async_fd, F_SETFL, flags | O_NONBLOCK);
1147         if (ret) {
1148                 DRV_LOG(INFO,
1149                         "port %u failed to change file descriptor async event"
1150                         " queue",
1151                         dev->data->port_id);
1152                 dev->data->dev_conf.intr_conf.lsc = 0;
1153                 dev->data->dev_conf.intr_conf.rmv = 0;
1154         }
1155         if (dev->data->dev_conf.intr_conf.lsc ||
1156             dev->data->dev_conf.intr_conf.rmv) {
1157                 priv->intr_handle.fd = priv->ctx->async_fd;
1158                 priv->intr_handle.type = RTE_INTR_HANDLE_EXT;
1159                 rte_intr_callback_register(&priv->intr_handle,
1160                                            mlx5_dev_interrupt_handler, dev);
1161         }
1162         ret = mlx5_socket_init(dev);
1163         if (ret)
1164                 DRV_LOG(ERR, "port %u cannot initialise socket: %s",
1165                         dev->data->port_id, strerror(rte_errno));
1166         else if (priv->primary_socket) {
1167                 priv->intr_handle_socket.fd = priv->primary_socket;
1168                 priv->intr_handle_socket.type = RTE_INTR_HANDLE_EXT;
1169                 rte_intr_callback_register(&priv->intr_handle_socket,
1170                                            mlx5_dev_handler_socket, dev);
1171         }
1172 }
1173
1174 /**
1175  * DPDK callback to bring the link DOWN.
1176  *
1177  * @param dev
1178  *   Pointer to Ethernet device structure.
1179  *
1180  * @return
1181  *   0 on success, a negative errno value otherwise and rte_errno is set.
1182  */
1183 int
1184 mlx5_set_link_down(struct rte_eth_dev *dev)
1185 {
1186         return mlx5_set_flags(dev, ~IFF_UP, ~IFF_UP);
1187 }
1188
1189 /**
1190  * DPDK callback to bring the link UP.
1191  *
1192  * @param dev
1193  *   Pointer to Ethernet device structure.
1194  *
1195  * @return
1196  *   0 on success, a negative errno value otherwise and rte_errno is set.
1197  */
1198 int
1199 mlx5_set_link_up(struct rte_eth_dev *dev)
1200 {
1201         return mlx5_set_flags(dev, ~IFF_UP, IFF_UP);
1202 }
1203
1204 /**
1205  * Configure the TX function to use.
1206  *
1207  * @param dev
1208  *   Pointer to private data structure.
1209  *
1210  * @return
1211  *   Pointer to selected Tx burst function.
1212  */
1213 eth_tx_burst_t
1214 mlx5_select_tx_function(struct rte_eth_dev *dev)
1215 {
1216         struct mlx5_priv *priv = dev->data->dev_private;
1217         eth_tx_burst_t tx_pkt_burst = mlx5_tx_burst;
1218         struct mlx5_dev_config *config = &priv->config;
1219         uint64_t tx_offloads = dev->data->dev_conf.txmode.offloads;
1220         int tso = !!(tx_offloads & (DEV_TX_OFFLOAD_TCP_TSO |
1221                                     DEV_TX_OFFLOAD_VXLAN_TNL_TSO |
1222                                     DEV_TX_OFFLOAD_GRE_TNL_TSO |
1223                                     DEV_TX_OFFLOAD_IP_TNL_TSO |
1224                                     DEV_TX_OFFLOAD_UDP_TNL_TSO));
1225         int swp = !!(tx_offloads & (DEV_TX_OFFLOAD_IP_TNL_TSO |
1226                                     DEV_TX_OFFLOAD_UDP_TNL_TSO |
1227                                     DEV_TX_OFFLOAD_OUTER_IPV4_CKSUM));
1228         int vlan_insert = !!(tx_offloads & DEV_TX_OFFLOAD_VLAN_INSERT);
1229
1230         assert(priv != NULL);
1231         /* Select appropriate TX function. */
1232         if (vlan_insert || tso || swp)
1233                 return tx_pkt_burst;
1234         if (config->mps == MLX5_MPW_ENHANCED) {
1235                 if (mlx5_check_vec_tx_support(dev) > 0) {
1236                         if (mlx5_check_raw_vec_tx_support(dev) > 0)
1237                                 tx_pkt_burst = mlx5_tx_burst_raw_vec;
1238                         else
1239                                 tx_pkt_burst = mlx5_tx_burst_vec;
1240                         DRV_LOG(DEBUG,
1241                                 "port %u selected enhanced MPW Tx vectorized"
1242                                 " function",
1243                                 dev->data->port_id);
1244                 } else {
1245                         tx_pkt_burst = mlx5_tx_burst_empw;
1246                         DRV_LOG(DEBUG,
1247                                 "port %u selected enhanced MPW Tx function",
1248                                 dev->data->port_id);
1249                 }
1250         } else if (config->mps && (config->txq_inline > 0)) {
1251                 tx_pkt_burst = mlx5_tx_burst_mpw_inline;
1252                 DRV_LOG(DEBUG, "port %u selected MPW inline Tx function",
1253                         dev->data->port_id);
1254         } else if (config->mps) {
1255                 tx_pkt_burst = mlx5_tx_burst_mpw;
1256                 DRV_LOG(DEBUG, "port %u selected MPW Tx function",
1257                         dev->data->port_id);
1258         }
1259         return tx_pkt_burst;
1260 }
1261
1262 /**
1263  * Configure the RX function to use.
1264  *
1265  * @param dev
1266  *   Pointer to private data structure.
1267  *
1268  * @return
1269  *   Pointer to selected Rx burst function.
1270  */
1271 eth_rx_burst_t
1272 mlx5_select_rx_function(struct rte_eth_dev *dev)
1273 {
1274         eth_rx_burst_t rx_pkt_burst = mlx5_rx_burst;
1275
1276         assert(dev != NULL);
1277         if (mlx5_check_vec_rx_support(dev) > 0) {
1278                 rx_pkt_burst = mlx5_rx_burst_vec;
1279                 DRV_LOG(DEBUG, "port %u selected Rx vectorized function",
1280                         dev->data->port_id);
1281         } else if (mlx5_mprq_enabled(dev)) {
1282                 rx_pkt_burst = mlx5_rx_burst_mprq;
1283         }
1284         return rx_pkt_burst;
1285 }
1286
1287 /**
1288  * Check if mlx5 device was removed.
1289  *
1290  * @param dev
1291  *   Pointer to Ethernet device structure.
1292  *
1293  * @return
1294  *   1 when device is removed, otherwise 0.
1295  */
1296 int
1297 mlx5_is_removed(struct rte_eth_dev *dev)
1298 {
1299         struct ibv_device_attr device_attr;
1300         struct mlx5_priv *priv = dev->data->dev_private;
1301
1302         if (mlx5_glue->query_device(priv->ctx, &device_attr) == EIO)
1303                 return 1;
1304         return 0;
1305 }
1306
1307 /**
1308  * Get port ID list of mlx5 instances sharing a common device.
1309  *
1310  * @param[in] dev
1311  *   Device to look for.
1312  * @param[out] port_list
1313  *   Result buffer for collected port IDs.
1314  * @param port_list_n
1315  *   Maximum number of entries in result buffer. If 0, @p port_list can be
1316  *   NULL.
1317  *
1318  * @return
1319  *   Number of matching instances regardless of the @p port_list_n
1320  *   parameter, 0 if none were found.
1321  */
1322 unsigned int
1323 mlx5_dev_to_port_id(const struct rte_device *dev, uint16_t *port_list,
1324                     unsigned int port_list_n)
1325 {
1326         uint16_t id;
1327         unsigned int n = 0;
1328
1329         RTE_ETH_FOREACH_DEV(id) {
1330                 struct rte_eth_dev *ldev = &rte_eth_devices[id];
1331
1332                 if (ldev->device != dev)
1333                         continue;
1334                 if (n < port_list_n)
1335                         port_list[n] = id;
1336                 n++;
1337         }
1338         return n;
1339 }
1340
1341 /**
1342  * Get switch information associated with network interface.
1343  *
1344  * @param ifindex
1345  *   Network interface index.
1346  * @param[out] info
1347  *   Switch information object, populated in case of success.
1348  *
1349  * @return
1350  *   0 on success, a negative errno value otherwise and rte_errno is set.
1351  */
1352 int
1353 mlx5_sysfs_switch_info(unsigned int ifindex, struct mlx5_switch_info *info)
1354 {
1355         char ifname[IF_NAMESIZE];
1356         char port_name[IF_NAMESIZE];
1357         FILE *file;
1358         struct mlx5_switch_info data = {
1359                 .master = 0,
1360                 .representor = 0,
1361                 .port_name_new = 0,
1362                 .port_name = 0,
1363                 .switch_id = 0,
1364         };
1365         DIR *dir;
1366         bool port_name_set = false;
1367         bool port_switch_id_set = false;
1368         bool device_dir = false;
1369         char c;
1370         int ret;
1371
1372         if (!if_indextoname(ifindex, ifname)) {
1373                 rte_errno = errno;
1374                 return -rte_errno;
1375         }
1376
1377         MKSTR(phys_port_name, "/sys/class/net/%s/phys_port_name",
1378               ifname);
1379         MKSTR(phys_switch_id, "/sys/class/net/%s/phys_switch_id",
1380               ifname);
1381         MKSTR(pci_device, "/sys/class/net/%s/device",
1382               ifname);
1383
1384         file = fopen(phys_port_name, "rb");
1385         if (file != NULL) {
1386                 ret = fscanf(file, "%s", port_name);
1387                 fclose(file);
1388                 if (ret == 1)
1389                         port_name_set = mlx5_translate_port_name(port_name,
1390                                                                  &data);
1391         }
1392         file = fopen(phys_switch_id, "rb");
1393         if (file == NULL) {
1394                 rte_errno = errno;
1395                 return -rte_errno;
1396         }
1397         port_switch_id_set =
1398                 fscanf(file, "%" SCNx64 "%c", &data.switch_id, &c) == 2 &&
1399                 c == '\n';
1400         fclose(file);
1401         dir = opendir(pci_device);
1402         if (dir != NULL) {
1403                 closedir(dir);
1404                 device_dir = true;
1405         }
1406         data.master = port_switch_id_set && (!port_name_set || device_dir);
1407         data.representor = port_switch_id_set && port_name_set && !device_dir;
1408         *info = data;
1409         assert(!(data.master && data.representor));
1410         if (data.master && data.representor) {
1411                 DRV_LOG(ERR, "ifindex %u device is recognized as master"
1412                              " and as representor", ifindex);
1413                 rte_errno = ENODEV;
1414                 return -rte_errno;
1415         }
1416         return 0;
1417 }
1418
1419 /**
1420  * Extract port name, as a number, from sysfs or netlink information.
1421  *
1422  * @param[in] port_name_in
1423  *   String representing the port name.
1424  * @param[out] port_info_out
1425  *   Port information, including port name as a number.
1426  *
1427  * @return
1428  *   true on success, false otherwise.
1429  */
1430 bool
1431 mlx5_translate_port_name(const char *port_name_in,
1432                          struct mlx5_switch_info *port_info_out)
1433 {
1434         char pf_c1, pf_c2, vf_c1, vf_c2;
1435         char *end;
1436         int32_t pf_num;
1437         bool port_name_set = false;
1438
1439         /*
1440          * Check for port-name as a string of the form pf0vf0
1441          * (support kernel ver >= 5.0)
1442          */
1443         port_name_set = (sscanf(port_name_in, "%c%c%d%c%c%d", &pf_c1, &pf_c2,
1444                                 &pf_num, &vf_c1, &vf_c2,
1445                                 &port_info_out->port_name) == 6);
1446         if (port_name_set) {
1447                 port_info_out->port_name_new = 1;
1448         } else {
1449                 /* Check for port-name as a number (support kernel ver < 5.0 */
1450                 errno = 0;
1451                 port_info_out->port_name = strtol(port_name_in, &end, 0);
1452                 if (!errno &&
1453                     (size_t)(end - port_name_in) == strlen(port_name_in))
1454                         port_name_set = true;
1455         }
1456         return port_name_set;
1457 }