drivers/net/mlx5/linux/mlx5_ethdev_os.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright 2015 6WIND S.A.
   3  * Copyright 2015 Mellanox Technologies, Ltd
   4  */
   5
   6 #include <stddef.h>
   7 #include <inttypes.h>
   8 #include <unistd.h>
   9 #include <stdbool.h>
  10 #include <stdint.h>
  11 #include <stdio.h>
  12 #include <string.h>
  13 #include <stdlib.h>
  14 #include <errno.h>
  15 #include <dirent.h>
  16 #include <net/if.h>
  17 #include <sys/ioctl.h>
  18 #include <sys/socket.h>
  19 #include <netinet/in.h>
  20 #include <linux/ethtool.h>
  21 #include <linux/sockios.h>
  22 #include <fcntl.h>
  23 #include <stdalign.h>
  24 #include <sys/un.h>
  25 #include <time.h>
  26
  27 #include <ethdev_driver.h>
  28 #include <rte_bus_pci.h>
  29 #include <rte_mbuf.h>
  30 #include <rte_common.h>
  31 #include <rte_interrupts.h>
  32 #include <rte_malloc.h>
  33 #include <rte_string_fns.h>
  34 #include <rte_rwlock.h>
  35 #include <rte_cycles.h>
  36
  37 #include <mlx5_glue.h>
  38 #include <mlx5_devx_cmds.h>
  39 #include <mlx5_common.h>
  40 #include <mlx5_malloc.h>
  41
  42 #include "mlx5.h"
  43 #include "mlx5_rxtx.h"
  44 #include "mlx5_utils.h"
  45
  46 /* Supported speed values found in /usr/include/linux/ethtool.h */
  47 #ifndef HAVE_SUPPORTED_40000baseKR4_Full
  48 #define SUPPORTED_40000baseKR4_Full (1 << 23)
  49 #endif
  50 #ifndef HAVE_SUPPORTED_40000baseCR4_Full
  51 #define SUPPORTED_40000baseCR4_Full (1 << 24)
  52 #endif
  53 #ifndef HAVE_SUPPORTED_40000baseSR4_Full
  54 #define SUPPORTED_40000baseSR4_Full (1 << 25)
  55 #endif
  56 #ifndef HAVE_SUPPORTED_40000baseLR4_Full
  57 #define SUPPORTED_40000baseLR4_Full (1 << 26)
  58 #endif
  59 #ifndef HAVE_SUPPORTED_56000baseKR4_Full
  60 #define SUPPORTED_56000baseKR4_Full (1 << 27)
  61 #endif
  62 #ifndef HAVE_SUPPORTED_56000baseCR4_Full
  63 #define SUPPORTED_56000baseCR4_Full (1 << 28)
  64 #endif
  65 #ifndef HAVE_SUPPORTED_56000baseSR4_Full
  66 #define SUPPORTED_56000baseSR4_Full (1 << 29)
  67 #endif
  68 #ifndef HAVE_SUPPORTED_56000baseLR4_Full
  69 #define SUPPORTED_56000baseLR4_Full (1 << 30)
  70 #endif
  71
  72 /* Add defines in case the running kernel is not the same as user headers. */
  73 #ifndef ETHTOOL_GLINKSETTINGS
  74 struct ethtool_link_settings {
  75         uint32_t cmd;
  76         uint32_t speed;
  77         uint8_t duplex;
  78         uint8_t port;
  79         uint8_t phy_address;
  80         uint8_t autoneg;
  81         uint8_t mdio_support;
  82         uint8_t eth_to_mdix;
  83         uint8_t eth_tp_mdix_ctrl;
  84         int8_t link_mode_masks_nwords;
  85         uint32_t reserved[8];
  86         uint32_t link_mode_masks[];
  87 };
  88
  89 /* The kernel values can be found in /include/uapi/linux/ethtool.h */
  90 #define ETHTOOL_GLINKSETTINGS 0x0000004c
  91 #define ETHTOOL_LINK_MODE_1000baseT_Full_BIT 5
  92 #define ETHTOOL_LINK_MODE_Autoneg_BIT 6
  93 #define ETHTOOL_LINK_MODE_1000baseKX_Full_BIT 17
  94 #define ETHTOOL_LINK_MODE_10000baseKX4_Full_BIT 18
  95 #define ETHTOOL_LINK_MODE_10000baseKR_Full_BIT 19
  96 #define ETHTOOL_LINK_MODE_10000baseR_FEC_BIT 20
  97 #define ETHTOOL_LINK_MODE_20000baseMLD2_Full_BIT 21
  98 #define ETHTOOL_LINK_MODE_20000baseKR2_Full_BIT 22
  99 #define ETHTOOL_LINK_MODE_40000baseKR4_Full_BIT 23
 100 #define ETHTOOL_LINK_MODE_40000baseCR4_Full_BIT 24
 101 #define ETHTOOL_LINK_MODE_40000baseSR4_Full_BIT 25
 102 #define ETHTOOL_LINK_MODE_40000baseLR4_Full_BIT 26
 103 #define ETHTOOL_LINK_MODE_56000baseKR4_Full_BIT 27
 104 #define ETHTOOL_LINK_MODE_56000baseCR4_Full_BIT 28
 105 #define ETHTOOL_LINK_MODE_56000baseSR4_Full_BIT 29
 106 #define ETHTOOL_LINK_MODE_56000baseLR4_Full_BIT 30
 107 #endif
 108 #ifndef HAVE_ETHTOOL_LINK_MODE_25G
 109 #define ETHTOOL_LINK_MODE_25000baseCR_Full_BIT 31
 110 #define ETHTOOL_LINK_MODE_25000baseKR_Full_BIT 32
 111 #define ETHTOOL_LINK_MODE_25000baseSR_Full_BIT 33
 112 #endif
 113 #ifndef HAVE_ETHTOOL_LINK_MODE_50G
 114 #define ETHTOOL_LINK_MODE_50000baseCR2_Full_BIT 34
 115 #define ETHTOOL_LINK_MODE_50000baseKR2_Full_BIT 35
 116 #endif
 117 #ifndef HAVE_ETHTOOL_LINK_MODE_100G
 118 #define ETHTOOL_LINK_MODE_100000baseKR4_Full_BIT 36
 119 #define ETHTOOL_LINK_MODE_100000baseSR4_Full_BIT 37
 120 #define ETHTOOL_LINK_MODE_100000baseCR4_Full_BIT 38
 121 #define ETHTOOL_LINK_MODE_100000baseLR4_ER4_Full_BIT 39
 122 #endif
 123 #ifndef HAVE_ETHTOOL_LINK_MODE_200G
 124 #define ETHTOOL_LINK_MODE_200000baseKR4_Full_BIT 62
 125 #define ETHTOOL_LINK_MODE_200000baseSR4_Full_BIT 63
 126 #define ETHTOOL_LINK_MODE_200000baseLR4_ER4_FR4_Full_BIT 0 /* 64 - 64 */
 127 #define ETHTOOL_LINK_MODE_200000baseDR4_Full_BIT 1 /* 65 - 64 */
 128 #define ETHTOOL_LINK_MODE_200000baseCR4_Full_BIT 2 /* 66 - 64 */
 129 #endif
 130
 131
 132 /**
 133  * Get interface name from private structure.
 134  *
 135  * This is a port representor-aware version of mlx5_get_ifname_sysfs().
 136  *
 137  * @param[in] dev
 138  *   Pointer to Ethernet device.
 139  * @param[out] ifname
 140  *   Interface name output buffer.
 141  *
 142  * @return
 143  *   0 on success, a negative errno value otherwise and rte_errno is set.
 144  */
 145 int
 146 mlx5_get_ifname(const struct rte_eth_dev *dev, char (*ifname)[MLX5_NAMESIZE])
 147 {
 148         struct mlx5_priv *priv = dev->data->dev_private;
 149         unsigned int ifindex;
 150
 151         MLX5_ASSERT(priv);
 152         MLX5_ASSERT(priv->sh);
 153         if (priv->master && priv->sh->bond.ifindex > 0) {
 154                 memcpy(ifname, priv->sh->bond.ifname, MLX5_NAMESIZE);
 155                 return 0;
 156         }
 157         ifindex = mlx5_ifindex(dev);
 158         if (!ifindex) {
 159                 if (!priv->representor)
 160                         return mlx5_get_ifname_sysfs(priv->sh->ibdev_path,
 161                                                      *ifname);
 162                 rte_errno = ENXIO;
 163                 return -rte_errno;
 164         }
 165         if (if_indextoname(ifindex, &(*ifname)[0]))
 166                 return 0;
 167         rte_errno = errno;
 168         return -rte_errno;
 169 }
 170
 171 /**
 172  * Perform ifreq ioctl() on associated netdev ifname.
 173  *
 174  * @param[in] ifname
 175  *   Pointer to netdev name.
 176  * @param req
 177  *   Request number to pass to ioctl().
 178  * @param[out] ifr
 179  *   Interface request structure output buffer.
 180  *
 181  * @return
 182  *   0 on success, a negative errno value otherwise and rte_errno is set.
 183  */
 184 static int
 185 mlx5_ifreq_by_ifname(const char *ifname, int req, struct ifreq *ifr)
 186 {
 187         int sock = socket(PF_INET, SOCK_DGRAM, IPPROTO_IP);
 188         int ret = 0;
 189
 190         if (sock == -1) {
 191                 rte_errno = errno;
 192                 return -rte_errno;
 193         }
 194         rte_strscpy(ifr->ifr_name, ifname, sizeof(ifr->ifr_name));
 195         ret = ioctl(sock, req, ifr);
 196         if (ret == -1) {
 197                 rte_errno = errno;
 198                 goto error;
 199         }
 200         close(sock);
 201         return 0;
 202 error:
 203         close(sock);
 204         return -rte_errno;
 205 }
 206
 207 /**
 208  * Perform ifreq ioctl() on associated Ethernet device.
 209  *
 210  * @param[in] dev
 211  *   Pointer to Ethernet device.
 212  * @param req
 213  *   Request number to pass to ioctl().
 214  * @param[out] ifr
 215  *   Interface request structure output buffer.
 216  *
 217  * @return
 218  *   0 on success, a negative errno value otherwise and rte_errno is set.
 219  */
 220 static int
 221 mlx5_ifreq(const struct rte_eth_dev *dev, int req, struct ifreq *ifr)
 222 {
 223         char ifname[sizeof(ifr->ifr_name)];
 224         int ret;
 225
 226         ret = mlx5_get_ifname(dev, &ifname);
 227         if (ret)
 228                 return -rte_errno;
 229         return mlx5_ifreq_by_ifname(ifname, req, ifr);
 230 }
 231
 232 /**
 233  * Get device MTU.
 234  *
 235  * @param dev
 236  *   Pointer to Ethernet device.
 237  * @param[out] mtu
 238  *   MTU value output buffer.
 239  *
 240  * @return
 241  *   0 on success, a negative errno value otherwise and rte_errno is set.
 242  */
 243 int
 244 mlx5_get_mtu(struct rte_eth_dev *dev, uint16_t *mtu)
 245 {
 246         struct ifreq request;
 247         int ret = mlx5_ifreq(dev, SIOCGIFMTU, &request);
 248
 249         if (ret)
 250                 return ret;
 251         *mtu = request.ifr_mtu;
 252         return 0;
 253 }
 254
 255 /**
 256  * Set device MTU.
 257  *
 258  * @param dev
 259  *   Pointer to Ethernet device.
 260  * @param mtu
 261  *   MTU value to set.
 262  *
 263  * @return
 264  *   0 on success, a negative errno value otherwise and rte_errno is set.
 265  */
 266 int
 267 mlx5_set_mtu(struct rte_eth_dev *dev, uint16_t mtu)
 268 {
 269         struct ifreq request = { .ifr_mtu = mtu, };
 270
 271         return mlx5_ifreq(dev, SIOCSIFMTU, &request);
 272 }
 273
 274 /**
 275  * Set device flags.
 276  *
 277  * @param dev
 278  *   Pointer to Ethernet device.
 279  * @param keep
 280  *   Bitmask for flags that must remain untouched.
 281  * @param flags
 282  *   Bitmask for flags to modify.
 283  *
 284  * @return
 285  *   0 on success, a negative errno value otherwise and rte_errno is set.
 286  */
 287 static int
 288 mlx5_set_flags(struct rte_eth_dev *dev, unsigned int keep, unsigned int flags)
 289 {
 290         struct ifreq request;
 291         int ret = mlx5_ifreq(dev, SIOCGIFFLAGS, &request);
 292
 293         if (ret)
 294                 return ret;
 295         request.ifr_flags &= keep;
 296         request.ifr_flags |= flags & ~keep;
 297         return mlx5_ifreq(dev, SIOCSIFFLAGS, &request);
 298 }
 299
 300 /**
 301  * Get device current raw clock counter
 302  *
 303  * @param dev
 304  *   Pointer to Ethernet device structure.
 305  * @param[out] time
 306  *   Current raw clock counter of the device.
 307  *
 308  * @return
 309  *   0 if the clock has correctly been read
 310  *   The value of errno in case of error
 311  */
 312 int
 313 mlx5_read_clock(struct rte_eth_dev *dev, uint64_t *clock)
 314 {
 315         struct mlx5_priv *priv = dev->data->dev_private;
 316         struct ibv_context *ctx = priv->sh->ctx;
 317         struct ibv_values_ex values;
 318         int err = 0;
 319
 320         values.comp_mask = IBV_VALUES_MASK_RAW_CLOCK;
 321         err = mlx5_glue->query_rt_values_ex(ctx, &values);
 322         if (err != 0) {
 323                 DRV_LOG(WARNING, "Could not query the clock !");
 324                 return err;
 325         }
 326         *clock = values.raw_clock.tv_nsec;
 327         return 0;
 328 }
 329
 330 /**
 331  * Retrieve the master device for representor in the same switch domain.
 332  *
 333  * @param dev
 334  *   Pointer to representor Ethernet device structure.
 335  *
 336  * @return
 337  *   Master device structure  on success, NULL otherwise.
 338  */
 339 static struct rte_eth_dev *
 340 mlx5_find_master_dev(struct rte_eth_dev *dev)
 341 {
 342         struct mlx5_priv *priv;
 343         uint16_t port_id;
 344         uint16_t domain_id;
 345
 346         priv = dev->data->dev_private;
 347         domain_id = priv->domain_id;
 348         MLX5_ASSERT(priv->representor);
 349         MLX5_ETH_FOREACH_DEV(port_id, dev->device) {
 350                 struct mlx5_priv *opriv =
 351                         rte_eth_devices[port_id].data->dev_private;
 352                 if (opriv &&
 353                     opriv->master &&
 354                     opriv->domain_id == domain_id &&
 355                     opriv->sh == priv->sh)
 356                         return &rte_eth_devices[port_id];
 357         }
 358         return NULL;
 359 }
 360
 361 /**
 362  * DPDK callback to retrieve physical link information.
 363  *
 364  * @param dev
 365  *   Pointer to Ethernet device structure.
 366  * @param[out] link
 367  *   Storage for current link status.
 368  *
 369  * @return
 370  *   0 on success, a negative errno value otherwise and rte_errno is set.
 371  */
 372 static int
 373 mlx5_link_update_unlocked_gset(struct rte_eth_dev *dev,
 374                                struct rte_eth_link *link)
 375 {
 376         struct mlx5_priv *priv = dev->data->dev_private;
 377         struct ethtool_cmd edata = {
 378                 .cmd = ETHTOOL_GSET /* Deprecated since Linux v4.5. */
 379         };
 380         struct ifreq ifr;
 381         struct rte_eth_link dev_link;
 382         int link_speed = 0;
 383         int ret;
 384
 385         ret = mlx5_ifreq(dev, SIOCGIFFLAGS, &ifr);
 386         if (ret) {
 387                 DRV_LOG(WARNING, "port %u ioctl(SIOCGIFFLAGS) failed: %s",
 388                         dev->data->port_id, strerror(rte_errno));
 389                 return ret;
 390         }
 391         dev_link = (struct rte_eth_link) {
 392                 .link_status = ((ifr.ifr_flags & IFF_UP) &&
 393                                 (ifr.ifr_flags & IFF_RUNNING)),
 394         };
 395         ifr = (struct ifreq) {
 396                 .ifr_data = (void *)&edata,
 397         };
 398         ret = mlx5_ifreq(dev, SIOCETHTOOL, &ifr);
 399         if (ret) {
 400                 if (ret == -ENOTSUP && priv->representor) {
 401                         struct rte_eth_dev *master;
 402
 403                         /*
 404                          * For representors we can try to inherit link
 405                          * settings from the master device. Actually
 406                          * link settings do not make a lot of sense
 407                          * for representors due to missing physical
 408                          * link. The old kernel drivers supported
 409                          * emulated settings query for representors,
 410                          * the new ones do not, so we have to add
 411                          * this code for compatibility issues.
 412                          */
 413                         master = mlx5_find_master_dev(dev);
 414                         if (master) {
 415                                 ifr = (struct ifreq) {
 416                                         .ifr_data = (void *)&edata,
 417                                 };
 418                                 ret = mlx5_ifreq(master, SIOCETHTOOL, &ifr);
 419                         }
 420                 }
 421                 if (ret) {
 422                         DRV_LOG(WARNING,
 423                                 "port %u ioctl(SIOCETHTOOL,"
 424                                 " ETHTOOL_GSET) failed: %s",
 425                                 dev->data->port_id, strerror(rte_errno));
 426                         return ret;
 427                 }
 428         }
 429         link_speed = ethtool_cmd_speed(&edata);
 430         if (link_speed == -1)
 431                 dev_link.link_speed = ETH_SPEED_NUM_UNKNOWN;
 432         else
 433                 dev_link.link_speed = link_speed;
 434         priv->link_speed_capa = 0;
 435         if (edata.supported & (SUPPORTED_1000baseT_Full |
 436                                SUPPORTED_1000baseKX_Full))
 437                 priv->link_speed_capa |= ETH_LINK_SPEED_1G;
 438         if (edata.supported & SUPPORTED_10000baseKR_Full)
 439                 priv->link_speed_capa |= ETH_LINK_SPEED_10G;
 440         if (edata.supported & (SUPPORTED_40000baseKR4_Full |
 441                                SUPPORTED_40000baseCR4_Full |
 442                                SUPPORTED_40000baseSR4_Full |
 443                                SUPPORTED_40000baseLR4_Full))
 444                 priv->link_speed_capa |= ETH_LINK_SPEED_40G;
 445         dev_link.link_duplex = ((edata.duplex == DUPLEX_HALF) ?
 446                                 ETH_LINK_HALF_DUPLEX : ETH_LINK_FULL_DUPLEX);
 447         dev_link.link_autoneg = !(dev->data->dev_conf.link_speeds &
 448                         ETH_LINK_SPEED_FIXED);
 449         *link = dev_link;
 450         return 0;
 451 }
 452
 453 /**
 454  * Retrieve physical link information (unlocked version using new ioctl).
 455  *
 456  * @param dev
 457  *   Pointer to Ethernet device structure.
 458  * @param[out] link
 459  *   Storage for current link status.
 460  *
 461  * @return
 462  *   0 on success, a negative errno value otherwise and rte_errno is set.
 463  */
 464 static int
 465 mlx5_link_update_unlocked_gs(struct rte_eth_dev *dev,
 466                              struct rte_eth_link *link)
 467
 468 {
 469         struct mlx5_priv *priv = dev->data->dev_private;
 470         struct ethtool_link_settings gcmd = { .cmd = ETHTOOL_GLINKSETTINGS };
 471         struct ifreq ifr;
 472         struct rte_eth_link dev_link;
 473         struct rte_eth_dev *master = NULL;
 474         uint64_t sc;
 475         int ret;
 476
 477         ret = mlx5_ifreq(dev, SIOCGIFFLAGS, &ifr);
 478         if (ret) {
 479                 DRV_LOG(WARNING, "port %u ioctl(SIOCGIFFLAGS) failed: %s",
 480                         dev->data->port_id, strerror(rte_errno));
 481                 return ret;
 482         }
 483         dev_link = (struct rte_eth_link) {
 484                 .link_status = ((ifr.ifr_flags & IFF_UP) &&
 485                                 (ifr.ifr_flags & IFF_RUNNING)),
 486         };
 487         ifr = (struct ifreq) {
 488                 .ifr_data = (void *)&gcmd,
 489         };
 490         ret = mlx5_ifreq(dev, SIOCETHTOOL, &ifr);
 491         if (ret) {
 492                 if (ret == -ENOTSUP && priv->representor) {
 493                         /*
 494                          * For representors we can try to inherit link
 495                          * settings from the master device. Actually
 496                          * link settings do not make a lot of sense
 497                          * for representors due to missing physical
 498                          * link. The old kernel drivers supported
 499                          * emulated settings query for representors,
 500                          * the new ones do not, so we have to add
 501                          * this code for compatibility issues.
 502                          */
 503                         master = mlx5_find_master_dev(dev);
 504                         if (master) {
 505                                 ifr = (struct ifreq) {
 506                                         .ifr_data = (void *)&gcmd,
 507                                 };
 508                                 ret = mlx5_ifreq(master, SIOCETHTOOL, &ifr);
 509                         }
 510                 }
 511                 if (ret) {
 512                         DRV_LOG(DEBUG,
 513                                 "port %u ioctl(SIOCETHTOOL,"
 514                                 " ETHTOOL_GLINKSETTINGS) failed: %s",
 515                                 dev->data->port_id, strerror(rte_errno));
 516                         return ret;
 517                 }
 518         }
 519         gcmd.link_mode_masks_nwords = -gcmd.link_mode_masks_nwords;
 520
 521         alignas(struct ethtool_link_settings)
 522         uint8_t data[offsetof(struct ethtool_link_settings, link_mode_masks) +
 523                      sizeof(uint32_t) * gcmd.link_mode_masks_nwords * 3];
 524         struct ethtool_link_settings *ecmd = (void *)data;
 525
 526         *ecmd = gcmd;
 527         ifr.ifr_data = (void *)ecmd;
 528         ret = mlx5_ifreq(master ? master : dev, SIOCETHTOOL, &ifr);
 529         if (ret) {
 530                 DRV_LOG(DEBUG,
 531                         "port %u ioctl(SIOCETHTOOL,"
 532                         "ETHTOOL_GLINKSETTINGS) failed: %s",
 533                         dev->data->port_id, strerror(rte_errno));
 534                 return ret;
 535         }
 536         dev_link.link_speed = (ecmd->speed == UINT32_MAX) ?
 537                                 ETH_SPEED_NUM_UNKNOWN : ecmd->speed;
 538         sc = ecmd->link_mode_masks[0] |
 539                 ((uint64_t)ecmd->link_mode_masks[1] << 32);
 540         priv->link_speed_capa = 0;
 541         if (sc & (MLX5_BITSHIFT(ETHTOOL_LINK_MODE_1000baseT_Full_BIT) |
 542                   MLX5_BITSHIFT(ETHTOOL_LINK_MODE_1000baseKX_Full_BIT)))
 543                 priv->link_speed_capa |= ETH_LINK_SPEED_1G;
 544         if (sc & (MLX5_BITSHIFT(ETHTOOL_LINK_MODE_10000baseKX4_Full_BIT) |
 545                   MLX5_BITSHIFT(ETHTOOL_LINK_MODE_10000baseKR_Full_BIT) |
 546                   MLX5_BITSHIFT(ETHTOOL_LINK_MODE_10000baseR_FEC_BIT)))
 547                 priv->link_speed_capa |= ETH_LINK_SPEED_10G;
 548         if (sc & (MLX5_BITSHIFT(ETHTOOL_LINK_MODE_20000baseMLD2_Full_BIT) |
 549                   MLX5_BITSHIFT(ETHTOOL_LINK_MODE_20000baseKR2_Full_BIT)))
 550                 priv->link_speed_capa |= ETH_LINK_SPEED_20G;
 551         if (sc & (MLX5_BITSHIFT(ETHTOOL_LINK_MODE_40000baseKR4_Full_BIT) |
 552                   MLX5_BITSHIFT(ETHTOOL_LINK_MODE_40000baseCR4_Full_BIT) |
 553                   MLX5_BITSHIFT(ETHTOOL_LINK_MODE_40000baseSR4_Full_BIT) |
 554                   MLX5_BITSHIFT(ETHTOOL_LINK_MODE_40000baseLR4_Full_BIT)))
 555                 priv->link_speed_capa |= ETH_LINK_SPEED_40G;
 556         if (sc & (MLX5_BITSHIFT(ETHTOOL_LINK_MODE_56000baseKR4_Full_BIT) |
 557                   MLX5_BITSHIFT(ETHTOOL_LINK_MODE_56000baseCR4_Full_BIT) |
 558                   MLX5_BITSHIFT(ETHTOOL_LINK_MODE_56000baseSR4_Full_BIT) |
 559                   MLX5_BITSHIFT(ETHTOOL_LINK_MODE_56000baseLR4_Full_BIT)))
 560                 priv->link_speed_capa |= ETH_LINK_SPEED_56G;
 561         if (sc & (MLX5_BITSHIFT(ETHTOOL_LINK_MODE_25000baseCR_Full_BIT) |
 562                   MLX5_BITSHIFT(ETHTOOL_LINK_MODE_25000baseKR_Full_BIT) |
 563                   MLX5_BITSHIFT(ETHTOOL_LINK_MODE_25000baseSR_Full_BIT)))
 564                 priv->link_speed_capa |= ETH_LINK_SPEED_25G;
 565         if (sc & (MLX5_BITSHIFT(ETHTOOL_LINK_MODE_50000baseCR2_Full_BIT) |
 566                   MLX5_BITSHIFT(ETHTOOL_LINK_MODE_50000baseKR2_Full_BIT)))
 567                 priv->link_speed_capa |= ETH_LINK_SPEED_50G;
 568         if (sc & (MLX5_BITSHIFT(ETHTOOL_LINK_MODE_100000baseKR4_Full_BIT) |
 569                   MLX5_BITSHIFT(ETHTOOL_LINK_MODE_100000baseSR4_Full_BIT) |
 570                   MLX5_BITSHIFT(ETHTOOL_LINK_MODE_100000baseCR4_Full_BIT) |
 571                   MLX5_BITSHIFT(ETHTOOL_LINK_MODE_100000baseLR4_ER4_Full_BIT)))
 572                 priv->link_speed_capa |= ETH_LINK_SPEED_100G;
 573         if (sc & (MLX5_BITSHIFT(ETHTOOL_LINK_MODE_200000baseKR4_Full_BIT) |
 574                   MLX5_BITSHIFT(ETHTOOL_LINK_MODE_200000baseSR4_Full_BIT)))
 575                 priv->link_speed_capa |= ETH_LINK_SPEED_200G;
 576
 577         sc = ecmd->link_mode_masks[2] |
 578                 ((uint64_t)ecmd->link_mode_masks[3] << 32);
 579         if (sc & (MLX5_BITSHIFT(ETHTOOL_LINK_MODE_200000baseCR4_Full_BIT) |
 580                   MLX5_BITSHIFT
 581                        (ETHTOOL_LINK_MODE_200000baseLR4_ER4_FR4_Full_BIT) |
 582                   MLX5_BITSHIFT(ETHTOOL_LINK_MODE_200000baseDR4_Full_BIT)))
 583                 priv->link_speed_capa |= ETH_LINK_SPEED_200G;
 584         dev_link.link_duplex = ((ecmd->duplex == DUPLEX_HALF) ?
 585                                 ETH_LINK_HALF_DUPLEX : ETH_LINK_FULL_DUPLEX);
 586         dev_link.link_autoneg = !(dev->data->dev_conf.link_speeds &
 587                                   ETH_LINK_SPEED_FIXED);
 588         *link = dev_link;
 589         return 0;
 590 }
 591
 592 /**
 593  * DPDK callback to retrieve physical link information.
 594  *
 595  * @param dev
 596  *   Pointer to Ethernet device structure.
 597  * @param wait_to_complete
 598  *   Wait for request completion.
 599  *
 600  * @return
 601  *   0 if link status was not updated, positive if it was, a negative errno
 602  *   value otherwise and rte_errno is set.
 603  */
 604 int
 605 mlx5_link_update(struct rte_eth_dev *dev, int wait_to_complete)
 606 {
 607         int ret;
 608         struct rte_eth_link dev_link;
 609         time_t start_time = time(NULL);
 610         int retry = MLX5_GET_LINK_STATUS_RETRY_COUNT;
 611
 612         do {
 613                 ret = mlx5_link_update_unlocked_gs(dev, &dev_link);
 614                 if (ret == -ENOTSUP)
 615                         ret = mlx5_link_update_unlocked_gset(dev, &dev_link);
 616                 if (ret == 0)
 617                         break;
 618                 /* Handle wait to complete situation. */
 619                 if ((wait_to_complete || retry) && ret == -EAGAIN) {
 620                         if (abs((int)difftime(time(NULL), start_time)) <
 621                             MLX5_LINK_STATUS_TIMEOUT) {
 622                                 usleep(0);
 623                                 continue;
 624                         } else {
 625                                 rte_errno = EBUSY;
 626                                 return -rte_errno;
 627                         }
 628                 } else if (ret < 0) {
 629                         return ret;
 630                 }
 631         } while (wait_to_complete || retry-- > 0);
 632         ret = !!memcmp(&dev->data->dev_link, &dev_link,
 633                        sizeof(struct rte_eth_link));
 634         dev->data->dev_link = dev_link;
 635         return ret;
 636 }
 637
 638 /**
 639  * DPDK callback to get flow control status.
 640  *
 641  * @param dev
 642  *   Pointer to Ethernet device structure.
 643  * @param[out] fc_conf
 644  *   Flow control output buffer.
 645  *
 646  * @return
 647  *   0 on success, a negative errno value otherwise and rte_errno is set.
 648  */
 649 int
 650 mlx5_dev_get_flow_ctrl(struct rte_eth_dev *dev, struct rte_eth_fc_conf *fc_conf)
 651 {
 652         struct ifreq ifr;
 653         struct ethtool_pauseparam ethpause = {
 654                 .cmd = ETHTOOL_GPAUSEPARAM
 655         };
 656         int ret;
 657
 658         ifr.ifr_data = (void *)&ethpause;
 659         ret = mlx5_ifreq(dev, SIOCETHTOOL, &ifr);
 660         if (ret) {
 661                 DRV_LOG(WARNING,
 662                         "port %u ioctl(SIOCETHTOOL, ETHTOOL_GPAUSEPARAM) failed:"
 663                         " %s",
 664                         dev->data->port_id, strerror(rte_errno));
 665                 return ret;
 666         }
 667         fc_conf->autoneg = ethpause.autoneg;
 668         if (ethpause.rx_pause && ethpause.tx_pause)
 669                 fc_conf->mode = RTE_FC_FULL;
 670         else if (ethpause.rx_pause)
 671                 fc_conf->mode = RTE_FC_RX_PAUSE;
 672         else if (ethpause.tx_pause)
 673                 fc_conf->mode = RTE_FC_TX_PAUSE;
 674         else
 675                 fc_conf->mode = RTE_FC_NONE;
 676         return 0;
 677 }
 678
 679 /**
 680  * DPDK callback to modify flow control parameters.
 681  *
 682  * @param dev
 683  *   Pointer to Ethernet device structure.
 684  * @param[in] fc_conf
 685  *   Flow control parameters.
 686  *
 687  * @return
 688  *   0 on success, a negative errno value otherwise and rte_errno is set.
 689  */
 690 int
 691 mlx5_dev_set_flow_ctrl(struct rte_eth_dev *dev, struct rte_eth_fc_conf *fc_conf)
 692 {
 693         struct ifreq ifr;
 694         struct ethtool_pauseparam ethpause = {
 695                 .cmd = ETHTOOL_SPAUSEPARAM
 696         };
 697         int ret;
 698
 699         ifr.ifr_data = (void *)&ethpause;
 700         ethpause.autoneg = fc_conf->autoneg;
 701         if (((fc_conf->mode & RTE_FC_FULL) == RTE_FC_FULL) ||
 702             (fc_conf->mode & RTE_FC_RX_PAUSE))
 703                 ethpause.rx_pause = 1;
 704         else
 705                 ethpause.rx_pause = 0;
 706
 707         if (((fc_conf->mode & RTE_FC_FULL) == RTE_FC_FULL) ||
 708             (fc_conf->mode & RTE_FC_TX_PAUSE))
 709                 ethpause.tx_pause = 1;
 710         else
 711                 ethpause.tx_pause = 0;
 712         ret = mlx5_ifreq(dev, SIOCETHTOOL, &ifr);
 713         if (ret) {
 714                 DRV_LOG(WARNING,
 715                         "port %u ioctl(SIOCETHTOOL, ETHTOOL_SPAUSEPARAM)"
 716                         " failed: %s",
 717                         dev->data->port_id, strerror(rte_errno));
 718                 return ret;
 719         }
 720         return 0;
 721 }
 722
 723 /**
 724  * Handle asynchronous removal event for entire multiport device.
 725  *
 726  * @param sh
 727  *   Infiniband device shared context.
 728  */
 729 static void
 730 mlx5_dev_interrupt_device_fatal(struct mlx5_dev_ctx_shared *sh)
 731 {
 732         uint32_t i;
 733
 734         for (i = 0; i < sh->max_port; ++i) {
 735                 struct rte_eth_dev *dev;
 736
 737                 if (sh->port[i].ih_port_id >= RTE_MAX_ETHPORTS) {
 738                         /*
 739                          * Or not existing port either no
 740                          * handler installed for this port.
 741                          */
 742                         continue;
 743                 }
 744                 dev = &rte_eth_devices[sh->port[i].ih_port_id];
 745                 MLX5_ASSERT(dev);
 746                 if (dev->data->dev_conf.intr_conf.rmv)
 747                         rte_eth_dev_callback_process
 748                                 (dev, RTE_ETH_EVENT_INTR_RMV, NULL);
 749         }
 750 }
 751
 752 /**
 753  * Handle shared asynchronous events the NIC (removal event
 754  * and link status change). Supports multiport IB device.
 755  *
 756  * @param cb_arg
 757  *   Callback argument.
 758  */
 759 void
 760 mlx5_dev_interrupt_handler(void *cb_arg)
 761 {
 762         struct mlx5_dev_ctx_shared *sh = cb_arg;
 763         struct ibv_async_event event;
 764
 765         /* Read all message from the IB device and acknowledge them. */
 766         for (;;) {
 767                 struct rte_eth_dev *dev;
 768                 uint32_t tmp;
 769
 770                 if (mlx5_glue->get_async_event(sh->ctx, &event))
 771                         break;
 772                 /* Retrieve and check IB port index. */
 773                 tmp = (uint32_t)event.element.port_num;
 774                 if (!tmp && event.event_type == IBV_EVENT_DEVICE_FATAL) {
 775                         /*
 776                          * The DEVICE_FATAL event is called once for
 777                          * entire device without port specifying.
 778                          * We should notify all existing ports.
 779                          */
 780                         mlx5_glue->ack_async_event(&event);
 781                         mlx5_dev_interrupt_device_fatal(sh);
 782                         continue;
 783                 }
 784                 MLX5_ASSERT(tmp && (tmp <= sh->max_port));
 785                 if (!tmp) {
 786                         /* Unsupported device level event. */
 787                         mlx5_glue->ack_async_event(&event);
 788                         DRV_LOG(DEBUG,
 789                                 "unsupported common event (type %d)",
 790                                 event.event_type);
 791                         continue;
 792                 }
 793                 if (tmp > sh->max_port) {
 794                         /* Invalid IB port index. */
 795                         mlx5_glue->ack_async_event(&event);
 796                         DRV_LOG(DEBUG,
 797                                 "cannot handle an event (type %d)"
 798                                 "due to invalid IB port index (%u)",
 799                                 event.event_type, tmp);
 800                         continue;
 801                 }
 802                 if (sh->port[tmp - 1].ih_port_id >= RTE_MAX_ETHPORTS) {
 803                         /* No handler installed. */
 804                         mlx5_glue->ack_async_event(&event);
 805                         DRV_LOG(DEBUG,
 806                                 "cannot handle an event (type %d)"
 807                                 "due to no handler installed for port %u",
 808                                 event.event_type, tmp);
 809                         continue;
 810                 }
 811                 /* Retrieve ethernet device descriptor. */
 812                 tmp = sh->port[tmp - 1].ih_port_id;
 813                 dev = &rte_eth_devices[tmp];
 814                 MLX5_ASSERT(dev);
 815                 if ((event.event_type == IBV_EVENT_PORT_ACTIVE ||
 816                      event.event_type == IBV_EVENT_PORT_ERR) &&
 817                         dev->data->dev_conf.intr_conf.lsc) {
 818                         mlx5_glue->ack_async_event(&event);
 819                         if (mlx5_link_update(dev, 0) == -EAGAIN) {
 820                                 usleep(0);
 821                                 continue;
 822                         }
 823                         rte_eth_dev_callback_process
 824                                 (dev, RTE_ETH_EVENT_INTR_LSC, NULL);
 825                         continue;
 826                 }
 827                 DRV_LOG(DEBUG,
 828                         "port %u cannot handle an unknown event (type %d)",
 829                         dev->data->port_id, event.event_type);
 830                 mlx5_glue->ack_async_event(&event);
 831         }
 832 }
 833
 834 /*
 835  * Unregister callback handler safely. The handler may be active
 836  * while we are trying to unregister it, in this case code -EAGAIN
 837  * is returned by rte_intr_callback_unregister(). This routine checks
 838  * the return code and tries to unregister handler again.
 839  *
 840  * @param handle
 841  *   interrupt handle
 842  * @param cb_fn
 843  *   pointer to callback routine
 844  * @cb_arg
 845  *   opaque callback parameter
 846  */
 847 void
 848 mlx5_intr_callback_unregister(const struct rte_intr_handle *handle,
 849                               rte_intr_callback_fn cb_fn, void *cb_arg)
 850 {
 851         /*
 852          * Try to reduce timeout management overhead by not calling
 853          * the timer related routines on the first iteration. If the
 854          * unregistering succeeds on first call there will be no
 855          * timer calls at all.
 856          */
 857         uint64_t twait = 0;
 858         uint64_t start = 0;
 859
 860         do {
 861                 int ret;
 862
 863                 ret = rte_intr_callback_unregister(handle, cb_fn, cb_arg);
 864                 if (ret >= 0)
 865                         return;
 866                 if (ret != -EAGAIN) {
 867                         DRV_LOG(INFO, "failed to unregister interrupt"
 868                                       " handler (error: %d)", ret);
 869                         MLX5_ASSERT(false);
 870                         return;
 871                 }
 872                 if (twait) {
 873                         struct timespec onems;
 874
 875                         /* Wait one millisecond and try again. */
 876                         onems.tv_sec = 0;
 877                         onems.tv_nsec = NS_PER_S / MS_PER_S;
 878                         nanosleep(&onems, 0);
 879                         /* Check whether one second elapsed. */
 880                         if ((rte_get_timer_cycles() - start) <= twait)
 881                                 continue;
 882                 } else {
 883                         /*
 884                          * We get the amount of timer ticks for one second.
 885                          * If this amount elapsed it means we spent one
 886                          * second in waiting. This branch is executed once
 887                          * on first iteration.
 888                          */
 889                         twait = rte_get_timer_hz();
 890                         MLX5_ASSERT(twait);
 891                 }
 892                 /*
 893                  * Timeout elapsed, show message (once a second) and retry.
 894                  * We have no other acceptable option here, if we ignore
 895                  * the unregistering return code the handler will not
 896                  * be unregistered, fd will be closed and we may get the
 897                  * crush. Hanging and messaging in the loop seems not to be
 898                  * the worst choice.
 899                  */
 900                 DRV_LOG(INFO, "Retrying to unregister interrupt handler");
 901                 start = rte_get_timer_cycles();
 902         } while (true);
 903 }
 904
 905 /**
 906  * Handle DEVX interrupts from the NIC.
 907  * This function is probably called from the DPDK host thread.
 908  *
 909  * @param cb_arg
 910  *   Callback argument.
 911  */
 912 void
 913 mlx5_dev_interrupt_handler_devx(void *cb_arg)
 914 {
 915 #ifndef HAVE_IBV_DEVX_ASYNC
 916         (void)cb_arg;
 917         return;
 918 #else
 919         struct mlx5_dev_ctx_shared *sh = cb_arg;
 920         union {
 921                 struct mlx5dv_devx_async_cmd_hdr cmd_resp;
 922                 uint8_t buf[MLX5_ST_SZ_BYTES(query_flow_counter_out) +
 923                             MLX5_ST_SZ_BYTES(traffic_counter) +
 924                             sizeof(struct mlx5dv_devx_async_cmd_hdr)];
 925         } out;
 926         uint8_t *buf = out.buf + sizeof(out.cmd_resp);
 927
 928         while (!mlx5_glue->devx_get_async_cmd_comp(sh->devx_comp,
 929                                                    &out.cmd_resp,
 930                                                    sizeof(out.buf)))
 931                 mlx5_flow_async_pool_query_handle
 932                         (sh, (uint64_t)out.cmd_resp.wr_id,
 933                          mlx5_devx_get_out_command_status(buf));
 934 #endif /* HAVE_IBV_DEVX_ASYNC */
 935 }
 936
 937 /**
 938  * DPDK callback to bring the link DOWN.
 939  *
 940  * @param dev
 941  *   Pointer to Ethernet device structure.
 942  *
 943  * @return
 944  *   0 on success, a negative errno value otherwise and rte_errno is set.
 945  */
 946 int
 947 mlx5_set_link_down(struct rte_eth_dev *dev)
 948 {
 949         return mlx5_set_flags(dev, ~IFF_UP, ~IFF_UP);
 950 }
 951
 952 /**
 953  * DPDK callback to bring the link UP.
 954  *
 955  * @param dev
 956  *   Pointer to Ethernet device structure.
 957  *
 958  * @return
 959  *   0 on success, a negative errno value otherwise and rte_errno is set.
 960  */
 961 int
 962 mlx5_set_link_up(struct rte_eth_dev *dev)
 963 {
 964         return mlx5_set_flags(dev, ~IFF_UP, IFF_UP);
 965 }
 966
 967 /**
 968  * Check if mlx5 device was removed.
 969  *
 970  * @param dev
 971  *   Pointer to Ethernet device structure.
 972  *
 973  * @return
 974  *   1 when device is removed, otherwise 0.
 975  */
 976 int
 977 mlx5_is_removed(struct rte_eth_dev *dev)
 978 {
 979         struct ibv_device_attr device_attr;
 980         struct mlx5_priv *priv = dev->data->dev_private;
 981
 982         if (mlx5_glue->query_device(priv->sh->ctx, &device_attr) == EIO)
 983                 return 1;
 984         return 0;
 985 }
 986
 987 /**
 988  * Analyze gathered port parameters via sysfs to recognize master
 989  * and representor devices for E-Switch configuration.
 990  *
 991  * @param[in] device_dir
 992  *   flag of presence of "device" directory under port device key.
 993  * @param[inout] switch_info
 994  *   Port information, including port name as a number and port name
 995  *   type if recognized
 996  *
 997  * @return
 998  *   master and representor flags are set in switch_info according to
 999  *   recognized parameters (if any).
1000  */
1001 static void
1002 mlx5_sysfs_check_switch_info(bool device_dir,
1003                              struct mlx5_switch_info *switch_info)
1004 {
1005         switch (switch_info->name_type) {
1006         case MLX5_PHYS_PORT_NAME_TYPE_UNKNOWN:
1007                 /*
1008                  * Name is not recognized, assume the master,
1009                  * check the device directory presence.
1010                  */
1011                 switch_info->master = device_dir;
1012                 break;
1013         case MLX5_PHYS_PORT_NAME_TYPE_NOTSET:
1014                 /*
1015                  * Name is not set, this assumes the legacy naming
1016                  * schema for master, just check if there is
1017                  * a device directory.
1018                  */
1019                 switch_info->master = device_dir;
1020                 break;
1021         case MLX5_PHYS_PORT_NAME_TYPE_UPLINK:
1022                 /* New uplink naming schema recognized. */
1023                 switch_info->master = 1;
1024                 break;
1025         case MLX5_PHYS_PORT_NAME_TYPE_LEGACY:
1026                 /* Legacy representors naming schema. */
1027                 switch_info->representor = !device_dir;
1028                 break;
1029         case MLX5_PHYS_PORT_NAME_TYPE_PFHPF:
1030                 /* Fallthrough */
1031         case MLX5_PHYS_PORT_NAME_TYPE_PFVF:
1032                 /* Fallthrough */
1033         case MLX5_PHYS_PORT_NAME_TYPE_PFSF:
1034                 /* New representors naming schema. */
1035                 switch_info->representor = 1;
1036                 break;
1037         default:
1038                 switch_info->master = device_dir;
1039                 break;
1040         }
1041 }
1042
1043 /**
1044  * Get switch information associated with network interface.
1045  *
1046  * @param ifindex
1047  *   Network interface index.
1048  * @param[out] info
1049  *   Switch information object, populated in case of success.
1050  *
1051  * @return
1052  *   0 on success, a negative errno value otherwise and rte_errno is set.
1053  */
1054 int
1055 mlx5_sysfs_switch_info(unsigned int ifindex, struct mlx5_switch_info *info)
1056 {
1057         char ifname[IF_NAMESIZE];
1058         char port_name[IF_NAMESIZE];
1059         FILE *file;
1060         struct mlx5_switch_info data = {
1061                 .master = 0,
1062                 .representor = 0,
1063                 .name_type = MLX5_PHYS_PORT_NAME_TYPE_NOTSET,
1064                 .port_name = 0,
1065                 .switch_id = 0,
1066         };
1067         DIR *dir;
1068         bool port_switch_id_set = false;
1069         bool device_dir = false;
1070         char c;
1071         int ret;
1072
1073         if (!if_indextoname(ifindex, ifname)) {
1074                 rte_errno = errno;
1075                 return -rte_errno;
1076         }
1077
1078         MKSTR(phys_port_name, "/sys/class/net/%s/phys_port_name",
1079               ifname);
1080         MKSTR(phys_switch_id, "/sys/class/net/%s/phys_switch_id",
1081               ifname);
1082         MKSTR(pci_device, "/sys/class/net/%s/device",
1083               ifname);
1084
1085         file = fopen(phys_port_name, "rb");
1086         if (file != NULL) {
1087                 ret = fscanf(file, "%" RTE_STR(IF_NAMESIZE) "s", port_name);
1088                 fclose(file);
1089                 if (ret == 1)
1090                         mlx5_translate_port_name(port_name, &data);
1091         }
1092         file = fopen(phys_switch_id, "rb");
1093         if (file == NULL) {
1094                 rte_errno = errno;
1095                 return -rte_errno;
1096         }
1097         port_switch_id_set =
1098                 fscanf(file, "%" SCNx64 "%c", &data.switch_id, &c) == 2 &&
1099                 c == '\n';
1100         fclose(file);
1101         dir = opendir(pci_device);
1102         if (dir != NULL) {
1103                 closedir(dir);
1104                 device_dir = true;
1105         }
1106         if (port_switch_id_set) {
1107                 /* We have some E-Switch configuration. */
1108                 mlx5_sysfs_check_switch_info(device_dir, &data);
1109         }
1110         *info = data;
1111         MLX5_ASSERT(!(data.master && data.representor));
1112         if (data.master && data.representor) {
1113                 DRV_LOG(ERR, "ifindex %u device is recognized as master"
1114                              " and as representor", ifindex);
1115                 rte_errno = ENODEV;
1116                 return -rte_errno;
1117         }
1118         return 0;
1119 }
1120
1121 /**
1122  * Get bond information associated with network interface.
1123  *
1124  * @param pf_ifindex
1125  *   Network interface index of bond slave interface
1126  * @param[out] ifindex
1127  *   Pointer to bond ifindex.
1128  * @param[out] ifname
1129  *   Pointer to bond ifname.
1130  *
1131  * @return
1132  *   0 on success, a negative errno value otherwise and rte_errno is set.
1133  */
1134 int
1135 mlx5_sysfs_bond_info(unsigned int pf_ifindex, unsigned int *ifindex,
1136                      char *ifname)
1137 {
1138         char name[IF_NAMESIZE];
1139         FILE *file;
1140         unsigned int index;
1141         int ret;
1142
1143         if (!if_indextoname(pf_ifindex, name) || !strlen(name)) {
1144                 rte_errno = errno;
1145                 return -rte_errno;
1146         }
1147         MKSTR(bond_if, "/sys/class/net/%s/master/ifindex", name);
1148         /* read bond ifindex */
1149         file = fopen(bond_if, "rb");
1150         if (file == NULL) {
1151                 rte_errno = errno;
1152                 return -rte_errno;
1153         }
1154         ret = fscanf(file, "%u", &index);
1155         fclose(file);
1156         if (ret <= 0) {
1157                 rte_errno = errno;
1158                 return -rte_errno;
1159         }
1160         if (ifindex)
1161                 *ifindex = index;
1162
1163         /* read bond device name from symbol link */
1164         if (ifname) {
1165                 if (!if_indextoname(index, ifname)) {
1166                         rte_errno = errno;
1167                         return -rte_errno;
1168                 }
1169         }
1170         return 0;
1171 }
1172
1173 /**
1174  * DPDK callback to retrieve plug-in module EEPROM information (type and size).
1175  *
1176  * @param dev
1177  *   Pointer to Ethernet device structure.
1178  * @param[out] modinfo
1179  *   Storage for plug-in module EEPROM information.
1180  *
1181  * @return
1182  *   0 on success, a negative errno value otherwise and rte_errno is set.
1183  */
1184 int
1185 mlx5_get_module_info(struct rte_eth_dev *dev,
1186                      struct rte_eth_dev_module_info *modinfo)
1187 {
1188         struct ethtool_modinfo info = {
1189                 .cmd = ETHTOOL_GMODULEINFO,
1190         };
1191         struct ifreq ifr = (struct ifreq) {
1192                 .ifr_data = (void *)&info,
1193         };
1194         int ret = 0;
1195
1196         if (!dev) {
1197                 DRV_LOG(WARNING, "missing argument, cannot get module info");
1198                 rte_errno = EINVAL;
1199                 return -rte_errno;
1200         }
1201         ret = mlx5_ifreq(dev, SIOCETHTOOL, &ifr);
1202         if (ret) {
1203                 DRV_LOG(WARNING, "port %u ioctl(SIOCETHTOOL) failed: %s",
1204                         dev->data->port_id, strerror(rte_errno));
1205                 return ret;
1206         }
1207         modinfo->type = info.type;
1208         modinfo->eeprom_len = info.eeprom_len;
1209         return ret;
1210 }
1211
1212 /**
1213  * DPDK callback to retrieve plug-in module EEPROM data.
1214  *
1215  * @param dev
1216  *   Pointer to Ethernet device structure.
1217  * @param[out] info
1218  *   Storage for plug-in module EEPROM data.
1219  *
1220  * @return
1221  *   0 on success, a negative errno value otherwise and rte_errno is set.
1222  */
1223 int mlx5_get_module_eeprom(struct rte_eth_dev *dev,
1224                            struct rte_dev_eeprom_info *info)
1225 {
1226         struct ethtool_eeprom *eeprom;
1227         struct ifreq ifr;
1228         int ret = 0;
1229
1230         if (!dev) {
1231                 DRV_LOG(WARNING, "missing argument, cannot get module eeprom");
1232                 rte_errno = EINVAL;
1233                 return -rte_errno;
1234         }
1235         eeprom = mlx5_malloc(MLX5_MEM_ZERO,
1236                              (sizeof(struct ethtool_eeprom) + info->length), 0,
1237                              SOCKET_ID_ANY);
1238         if (!eeprom) {
1239                 DRV_LOG(WARNING, "port %u cannot allocate memory for "
1240                         "eeprom data", dev->data->port_id);
1241                 rte_errno = ENOMEM;
1242                 return -rte_errno;
1243         }
1244         eeprom->cmd = ETHTOOL_GMODULEEEPROM;
1245         eeprom->offset = info->offset;
1246         eeprom->len = info->length;
1247         ifr = (struct ifreq) {
1248                 .ifr_data = (void *)eeprom,
1249         };
1250         ret = mlx5_ifreq(dev, SIOCETHTOOL, &ifr);
1251         if (ret)
1252                 DRV_LOG(WARNING, "port %u ioctl(SIOCETHTOOL) failed: %s",
1253                         dev->data->port_id, strerror(rte_errno));
1254         else
1255                 rte_memcpy(info->data, eeprom->data, info->length);
1256         mlx5_free(eeprom);
1257         return ret;
1258 }
1259
1260 /**
1261  * Read device counters table.
1262  *
1263  * @param dev
1264  *   Pointer to Ethernet device.
1265  * @param[in] pf
1266  *   PF index in case of bonding device, -1 otherwise
1267  * @param[out] stats
1268  *   Counters table output buffer.
1269  *
1270  * @return
1271  *   0 on success and stats is filled, negative errno value otherwise and
1272  *   rte_errno is set.
1273  */
1274 static int
1275 _mlx5_os_read_dev_counters(struct rte_eth_dev *dev, int pf, uint64_t *stats)
1276 {
1277         struct mlx5_priv *priv = dev->data->dev_private;
1278         struct mlx5_xstats_ctrl *xstats_ctrl = &priv->xstats_ctrl;
1279         unsigned int i;
1280         struct ifreq ifr;
1281         unsigned int stats_sz = xstats_ctrl->stats_n * sizeof(uint64_t);
1282         unsigned char et_stat_buf[sizeof(struct ethtool_stats) + stats_sz];
1283         struct ethtool_stats *et_stats = (struct ethtool_stats *)et_stat_buf;
1284         int ret;
1285
1286         et_stats->cmd = ETHTOOL_GSTATS;
1287         et_stats->n_stats = xstats_ctrl->stats_n;
1288         ifr.ifr_data = (caddr_t)et_stats;
1289         if (pf >= 0)
1290                 ret = mlx5_ifreq_by_ifname(priv->sh->bond.ports[pf].ifname,
1291                                            SIOCETHTOOL, &ifr);
1292         else
1293                 ret = mlx5_ifreq(dev, SIOCETHTOOL, &ifr);
1294         if (ret) {
1295                 DRV_LOG(WARNING,
1296                         "port %u unable to read statistic values from device",
1297                         dev->data->port_id);
1298                 return ret;
1299         }
1300         for (i = 0; i != xstats_ctrl->mlx5_stats_n; ++i) {
1301                 if (xstats_ctrl->info[i].dev)
1302                         continue;
1303                 stats[i] += (uint64_t)
1304                             et_stats->data[xstats_ctrl->dev_table_idx[i]];
1305         }
1306         return 0;
1307 }
1308
1309 /**
1310  * Read device counters.
1311  *
1312  * @param dev
1313  *   Pointer to Ethernet device.
1314  * @param[out] stats
1315  *   Counters table output buffer.
1316  *
1317  * @return
1318  *   0 on success and stats is filled, negative errno value otherwise and
1319  *   rte_errno is set.
1320  */
1321 int
1322 mlx5_os_read_dev_counters(struct rte_eth_dev *dev, uint64_t *stats)
1323 {
1324         struct mlx5_priv *priv = dev->data->dev_private;
1325         struct mlx5_xstats_ctrl *xstats_ctrl = &priv->xstats_ctrl;
1326         int ret = 0, i;
1327
1328         memset(stats, 0, sizeof(*stats) * xstats_ctrl->mlx5_stats_n);
1329         /* Read ifreq counters. */
1330         if (priv->master && priv->pf_bond >= 0) {
1331                 /* Sum xstats from bonding device member ports. */
1332                 for (i = 0; i < priv->sh->bond.n_port; i++) {
1333                         ret = _mlx5_os_read_dev_counters(dev, i, stats);
1334                         if (ret)
1335                                 return ret;
1336                 }
1337         } else {
1338                 ret = _mlx5_os_read_dev_counters(dev, -1, stats);
1339         }
1340         /* Read IB counters. */
1341         for (i = 0; i != xstats_ctrl->mlx5_stats_n; ++i) {
1342                 if (!xstats_ctrl->info[i].dev)
1343                         continue;
1344                 ret = mlx5_os_read_dev_stat(priv, xstats_ctrl->info[i].ctr_name,
1345                                             &stats[i]);
1346                 /* return last xstats counter if fail to read. */
1347                 if (ret != 0)
1348                         xstats_ctrl->xstats[i] = stats[i];
1349                 else
1350                         stats[i] = xstats_ctrl->xstats[i];
1351         }
1352         return ret;
1353 }
1354
1355 /**
1356  * Query the number of statistics provided by ETHTOOL.
1357  *
1358  * @param dev
1359  *   Pointer to Ethernet device.
1360  *
1361  * @return
1362  *   Number of statistics on success, negative errno value otherwise and
1363  *   rte_errno is set.
1364  */
1365 int
1366 mlx5_os_get_stats_n(struct rte_eth_dev *dev)
1367 {
1368         struct mlx5_priv *priv = dev->data->dev_private;
1369         struct ethtool_drvinfo drvinfo;
1370         struct ifreq ifr;
1371         int ret;
1372
1373         drvinfo.cmd = ETHTOOL_GDRVINFO;
1374         ifr.ifr_data = (caddr_t)&drvinfo;
1375         if (priv->master && priv->pf_bond >= 0)
1376                 /* Bonding PF. */
1377                 ret = mlx5_ifreq_by_ifname(priv->sh->bond.ports[0].ifname,
1378                                            SIOCETHTOOL, &ifr);
1379         else
1380                 ret = mlx5_ifreq(dev, SIOCETHTOOL, &ifr);
1381         if (ret) {
1382                 DRV_LOG(WARNING, "port %u unable to query number of statistics",
1383                         dev->data->port_id);
1384                 return ret;
1385         }
1386         return drvinfo.n_stats;
1387 }
1388
1389 static const struct mlx5_counter_ctrl mlx5_counters_init[] = {
1390         {
1391                 .dpdk_name = "rx_unicast_bytes",
1392                 .ctr_name = "rx_vport_unicast_bytes",
1393         },
1394         {
1395                 .dpdk_name = "rx_multicast_bytes",
1396                 .ctr_name = "rx_vport_multicast_bytes",
1397         },
1398         {
1399                 .dpdk_name = "rx_broadcast_bytes",
1400                 .ctr_name = "rx_vport_broadcast_bytes",
1401         },
1402         {
1403                 .dpdk_name = "rx_unicast_packets",
1404                 .ctr_name = "rx_vport_unicast_packets",
1405         },
1406         {
1407                 .dpdk_name = "rx_multicast_packets",
1408                 .ctr_name = "rx_vport_multicast_packets",
1409         },
1410         {
1411                 .dpdk_name = "rx_broadcast_packets",
1412                 .ctr_name = "rx_vport_broadcast_packets",
1413         },
1414         {
1415                 .dpdk_name = "tx_unicast_bytes",
1416                 .ctr_name = "tx_vport_unicast_bytes",
1417         },
1418         {
1419                 .dpdk_name = "tx_multicast_bytes",
1420                 .ctr_name = "tx_vport_multicast_bytes",
1421         },
1422         {
1423                 .dpdk_name = "tx_broadcast_bytes",
1424                 .ctr_name = "tx_vport_broadcast_bytes",
1425         },
1426         {
1427                 .dpdk_name = "tx_unicast_packets",
1428                 .ctr_name = "tx_vport_unicast_packets",
1429         },
1430         {
1431                 .dpdk_name = "tx_multicast_packets",
1432                 .ctr_name = "tx_vport_multicast_packets",
1433         },
1434         {
1435                 .dpdk_name = "tx_broadcast_packets",
1436                 .ctr_name = "tx_vport_broadcast_packets",
1437         },
1438         {
1439                 .dpdk_name = "rx_wqe_errors",
1440                 .ctr_name = "rx_wqe_err",
1441         },
1442         {
1443                 .dpdk_name = "rx_phy_crc_errors",
1444                 .ctr_name = "rx_crc_errors_phy",
1445         },
1446         {
1447                 .dpdk_name = "rx_phy_in_range_len_errors",
1448                 .ctr_name = "rx_in_range_len_errors_phy",
1449         },
1450         {
1451                 .dpdk_name = "rx_phy_symbol_errors",
1452                 .ctr_name = "rx_symbol_err_phy",
1453         },
1454         {
1455                 .dpdk_name = "tx_phy_errors",
1456                 .ctr_name = "tx_errors_phy",
1457         },
1458         {
1459                 .dpdk_name = "rx_out_of_buffer",
1460                 .ctr_name = "out_of_buffer",
1461                 .dev = 1,
1462         },
1463         {
1464                 .dpdk_name = "tx_phy_packets",
1465                 .ctr_name = "tx_packets_phy",
1466         },
1467         {
1468                 .dpdk_name = "rx_phy_packets",
1469                 .ctr_name = "rx_packets_phy",
1470         },
1471         {
1472                 .dpdk_name = "tx_phy_discard_packets",
1473                 .ctr_name = "tx_discards_phy",
1474         },
1475         {
1476                 .dpdk_name = "rx_phy_discard_packets",
1477                 .ctr_name = "rx_discards_phy",
1478         },
1479         {
1480                 .dpdk_name = "tx_phy_bytes",
1481                 .ctr_name = "tx_bytes_phy",
1482         },
1483         {
1484                 .dpdk_name = "rx_phy_bytes",
1485                 .ctr_name = "rx_bytes_phy",
1486         },
1487         /* Representor only */
1488         {
1489                 .dpdk_name = "rx_vport_packets",
1490                 .ctr_name = "vport_rx_packets",
1491         },
1492         {
1493                 .dpdk_name = "rx_vport_bytes",
1494                 .ctr_name = "vport_rx_bytes",
1495         },
1496         {
1497                 .dpdk_name = "tx_vport_packets",
1498                 .ctr_name = "vport_tx_packets",
1499         },
1500         {
1501                 .dpdk_name = "tx_vport_bytes",
1502                 .ctr_name = "vport_tx_bytes",
1503         },
1504 };
1505
1506 static const unsigned int xstats_n = RTE_DIM(mlx5_counters_init);
1507
1508 /**
1509  * Init the structures to read device counters.
1510  *
1511  * @param dev
1512  *   Pointer to Ethernet device.
1513  */
1514 void
1515 mlx5_os_stats_init(struct rte_eth_dev *dev)
1516 {
1517         struct mlx5_priv *priv = dev->data->dev_private;
1518         struct mlx5_xstats_ctrl *xstats_ctrl = &priv->xstats_ctrl;
1519         struct mlx5_stats_ctrl *stats_ctrl = &priv->stats_ctrl;
1520         unsigned int i;
1521         unsigned int j;
1522         struct ifreq ifr;
1523         struct ethtool_gstrings *strings = NULL;
1524         unsigned int dev_stats_n;
1525         unsigned int str_sz;
1526         int ret;
1527
1528         /* So that it won't aggregate for each init. */
1529         xstats_ctrl->mlx5_stats_n = 0;
1530         ret = mlx5_os_get_stats_n(dev);
1531         if (ret < 0) {
1532                 DRV_LOG(WARNING, "port %u no extended statistics available",
1533                         dev->data->port_id);
1534                 return;
1535         }
1536         dev_stats_n = ret;
1537         /* Allocate memory to grab stat names and values. */
1538         str_sz = dev_stats_n * ETH_GSTRING_LEN;
1539         strings = (struct ethtool_gstrings *)
1540                   mlx5_malloc(0, str_sz + sizeof(struct ethtool_gstrings), 0,
1541                               SOCKET_ID_ANY);
1542         if (!strings) {
1543                 DRV_LOG(WARNING, "port %u unable to allocate memory for xstats",
1544                      dev->data->port_id);
1545                 return;
1546         }
1547         strings->cmd = ETHTOOL_GSTRINGS;
1548         strings->string_set = ETH_SS_STATS;
1549         strings->len = dev_stats_n;
1550         ifr.ifr_data = (caddr_t)strings;
1551         if (priv->master && priv->pf_bond >= 0)
1552                 /* Bonding master. */
1553                 ret = mlx5_ifreq_by_ifname(priv->sh->bond.ports[0].ifname,
1554                                            SIOCETHTOOL, &ifr);
1555         else
1556                 ret = mlx5_ifreq(dev, SIOCETHTOOL, &ifr);
1557         if (ret) {
1558                 DRV_LOG(WARNING, "port %u unable to get statistic names",
1559                         dev->data->port_id);
1560                 goto free;
1561         }
1562         for (i = 0; i != dev_stats_n; ++i) {
1563                 const char *curr_string = (const char *)
1564                         &strings->data[i * ETH_GSTRING_LEN];
1565
1566                 for (j = 0; j != xstats_n; ++j) {
1567                         if (!strcmp(mlx5_counters_init[j].ctr_name,
1568                                     curr_string)) {
1569                                 unsigned int idx = xstats_ctrl->mlx5_stats_n++;
1570
1571                                 xstats_ctrl->dev_table_idx[idx] = i;
1572                                 xstats_ctrl->info[idx] = mlx5_counters_init[j];
1573                                 break;
1574                         }
1575                 }
1576         }
1577         /* Add dev counters. */
1578         for (i = 0; i != xstats_n; ++i) {
1579                 if (mlx5_counters_init[i].dev) {
1580                         unsigned int idx = xstats_ctrl->mlx5_stats_n++;
1581
1582                         xstats_ctrl->info[idx] = mlx5_counters_init[i];
1583                         xstats_ctrl->hw_stats[idx] = 0;
1584                 }
1585         }
1586         MLX5_ASSERT(xstats_ctrl->mlx5_stats_n <= MLX5_MAX_XSTATS);
1587         xstats_ctrl->stats_n = dev_stats_n;
1588         /* Copy to base at first time. */
1589         ret = mlx5_os_read_dev_counters(dev, xstats_ctrl->base);
1590         if (ret)
1591                 DRV_LOG(ERR, "port %u cannot read device counters: %s",
1592                         dev->data->port_id, strerror(rte_errno));
1593         mlx5_os_read_dev_stat(priv, "out_of_buffer", &stats_ctrl->imissed_base);
1594         stats_ctrl->imissed = 0;
1595 free:
1596         mlx5_free(strings);
1597 }
1598
1599 /**
1600  * Get MAC address by querying netdevice.
1601  *
1602  * @param[in] dev
1603  *   Pointer to Ethernet device.
1604  * @param[out] mac
1605  *   MAC address output buffer.
1606  *
1607  * @return
1608  *   0 on success, a negative errno value otherwise and rte_errno is set.
1609  */
1610 int
1611 mlx5_get_mac(struct rte_eth_dev *dev, uint8_t (*mac)[RTE_ETHER_ADDR_LEN])
1612 {
1613         struct ifreq request;
1614         int ret;
1615
1616         ret = mlx5_ifreq(dev, SIOCGIFHWADDR, &request);
1617         if (ret)
1618                 return ret;
1619         memcpy(mac, request.ifr_hwaddr.sa_data, RTE_ETHER_ADDR_LEN);
1620         return 0;
1621 }
1622