drivers/net/mlx5/linux/mlx5_ethdev_os.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright 2015 6WIND S.A.
   3  * Copyright 2015 Mellanox Technologies, Ltd
   4  */
   5
   6 #include <stddef.h>
   7 #include <inttypes.h>
   8 #include <unistd.h>
   9 #include <stdbool.h>
  10 #include <stdint.h>
  11 #include <stdio.h>
  12 #include <string.h>
  13 #include <stdlib.h>
  14 #include <errno.h>
  15 #include <dirent.h>
  16 #include <net/if.h>
  17 #include <sys/ioctl.h>
  18 #include <sys/socket.h>
  19 #include <netinet/in.h>
  20 #include <linux/ethtool.h>
  21 #include <linux/sockios.h>
  22 #include <fcntl.h>
  23 #include <stdalign.h>
  24 #include <sys/un.h>
  25 #include <time.h>
  26
  27 #include <rte_atomic.h>
  28 #include <rte_ethdev_driver.h>
  29 #include <rte_bus_pci.h>
  30 #include <rte_mbuf.h>
  31 #include <rte_common.h>
  32 #include <rte_interrupts.h>
  33 #include <rte_malloc.h>
  34 #include <rte_string_fns.h>
  35 #include <rte_rwlock.h>
  36 #include <rte_cycles.h>
  37
  38 #include <mlx5_glue.h>
  39 #include <mlx5_devx_cmds.h>
  40 #include <mlx5_common.h>
  41 #include <mlx5_malloc.h>
  42
  43 #include "mlx5.h"
  44 #include "mlx5_rxtx.h"
  45 #include "mlx5_utils.h"
  46
  47 /* Supported speed values found in /usr/include/linux/ethtool.h */
  48 #ifndef HAVE_SUPPORTED_40000baseKR4_Full
  49 #define SUPPORTED_40000baseKR4_Full (1 << 23)
  50 #endif
  51 #ifndef HAVE_SUPPORTED_40000baseCR4_Full
  52 #define SUPPORTED_40000baseCR4_Full (1 << 24)
  53 #endif
  54 #ifndef HAVE_SUPPORTED_40000baseSR4_Full
  55 #define SUPPORTED_40000baseSR4_Full (1 << 25)
  56 #endif
  57 #ifndef HAVE_SUPPORTED_40000baseLR4_Full
  58 #define SUPPORTED_40000baseLR4_Full (1 << 26)
  59 #endif
  60 #ifndef HAVE_SUPPORTED_56000baseKR4_Full
  61 #define SUPPORTED_56000baseKR4_Full (1 << 27)
  62 #endif
  63 #ifndef HAVE_SUPPORTED_56000baseCR4_Full
  64 #define SUPPORTED_56000baseCR4_Full (1 << 28)
  65 #endif
  66 #ifndef HAVE_SUPPORTED_56000baseSR4_Full
  67 #define SUPPORTED_56000baseSR4_Full (1 << 29)
  68 #endif
  69 #ifndef HAVE_SUPPORTED_56000baseLR4_Full
  70 #define SUPPORTED_56000baseLR4_Full (1 << 30)
  71 #endif
  72
  73 /* Add defines in case the running kernel is not the same as user headers. */
  74 #ifndef ETHTOOL_GLINKSETTINGS
  75 struct ethtool_link_settings {
  76         uint32_t cmd;
  77         uint32_t speed;
  78         uint8_t duplex;
  79         uint8_t port;
  80         uint8_t phy_address;
  81         uint8_t autoneg;
  82         uint8_t mdio_support;
  83         uint8_t eth_to_mdix;
  84         uint8_t eth_tp_mdix_ctrl;
  85         int8_t link_mode_masks_nwords;
  86         uint32_t reserved[8];
  87         uint32_t link_mode_masks[];
  88 };
  89
  90 /* The kernel values can be found in /include/uapi/linux/ethtool.h */
  91 #define ETHTOOL_GLINKSETTINGS 0x0000004c
  92 #define ETHTOOL_LINK_MODE_1000baseT_Full_BIT 5
  93 #define ETHTOOL_LINK_MODE_Autoneg_BIT 6
  94 #define ETHTOOL_LINK_MODE_1000baseKX_Full_BIT 17
  95 #define ETHTOOL_LINK_MODE_10000baseKX4_Full_BIT 18
  96 #define ETHTOOL_LINK_MODE_10000baseKR_Full_BIT 19
  97 #define ETHTOOL_LINK_MODE_10000baseR_FEC_BIT 20
  98 #define ETHTOOL_LINK_MODE_20000baseMLD2_Full_BIT 21
  99 #define ETHTOOL_LINK_MODE_20000baseKR2_Full_BIT 22
 100 #define ETHTOOL_LINK_MODE_40000baseKR4_Full_BIT 23
 101 #define ETHTOOL_LINK_MODE_40000baseCR4_Full_BIT 24
 102 #define ETHTOOL_LINK_MODE_40000baseSR4_Full_BIT 25
 103 #define ETHTOOL_LINK_MODE_40000baseLR4_Full_BIT 26
 104 #define ETHTOOL_LINK_MODE_56000baseKR4_Full_BIT 27
 105 #define ETHTOOL_LINK_MODE_56000baseCR4_Full_BIT 28
 106 #define ETHTOOL_LINK_MODE_56000baseSR4_Full_BIT 29
 107 #define ETHTOOL_LINK_MODE_56000baseLR4_Full_BIT 30
 108 #endif
 109 #ifndef HAVE_ETHTOOL_LINK_MODE_25G
 110 #define ETHTOOL_LINK_MODE_25000baseCR_Full_BIT 31
 111 #define ETHTOOL_LINK_MODE_25000baseKR_Full_BIT 32
 112 #define ETHTOOL_LINK_MODE_25000baseSR_Full_BIT 33
 113 #endif
 114 #ifndef HAVE_ETHTOOL_LINK_MODE_50G
 115 #define ETHTOOL_LINK_MODE_50000baseCR2_Full_BIT 34
 116 #define ETHTOOL_LINK_MODE_50000baseKR2_Full_BIT 35
 117 #endif
 118 #ifndef HAVE_ETHTOOL_LINK_MODE_100G
 119 #define ETHTOOL_LINK_MODE_100000baseKR4_Full_BIT 36
 120 #define ETHTOOL_LINK_MODE_100000baseSR4_Full_BIT 37
 121 #define ETHTOOL_LINK_MODE_100000baseCR4_Full_BIT 38
 122 #define ETHTOOL_LINK_MODE_100000baseLR4_ER4_Full_BIT 39
 123 #endif
 124 #ifndef HAVE_ETHTOOL_LINK_MODE_200G
 125 #define ETHTOOL_LINK_MODE_200000baseKR4_Full_BIT 62
 126 #define ETHTOOL_LINK_MODE_200000baseSR4_Full_BIT 63
 127 #define ETHTOOL_LINK_MODE_200000baseLR4_ER4_FR4_Full_BIT 0 /* 64 - 64 */
 128 #define ETHTOOL_LINK_MODE_200000baseDR4_Full_BIT 1 /* 65 - 64 */
 129 #define ETHTOOL_LINK_MODE_200000baseCR4_Full_BIT 2 /* 66 - 64 */
 130 #endif
 131
 132
 133 /**
 134  * Get interface name from private structure.
 135  *
 136  * This is a port representor-aware version of mlx5_get_ifname_sysfs().
 137  *
 138  * @param[in] dev
 139  *   Pointer to Ethernet device.
 140  * @param[out] ifname
 141  *   Interface name output buffer.
 142  *
 143  * @return
 144  *   0 on success, a negative errno value otherwise and rte_errno is set.
 145  */
 146 int
 147 mlx5_get_ifname(const struct rte_eth_dev *dev, char (*ifname)[IF_NAMESIZE])
 148 {
 149         struct mlx5_priv *priv = dev->data->dev_private;
 150         unsigned int ifindex;
 151
 152         MLX5_ASSERT(priv);
 153         MLX5_ASSERT(priv->sh);
 154         if (priv->bond_ifindex > 0) {
 155                 memcpy(ifname, priv->bond_name, IF_NAMESIZE);
 156                 return 0;
 157         }
 158         ifindex = mlx5_ifindex(dev);
 159         if (!ifindex) {
 160                 if (!priv->representor)
 161                         return mlx5_get_ifname_sysfs(priv->sh->ibdev_path,
 162                                                      *ifname);
 163                 rte_errno = ENXIO;
 164                 return -rte_errno;
 165         }
 166         if (if_indextoname(ifindex, &(*ifname)[0]))
 167                 return 0;
 168         rte_errno = errno;
 169         return -rte_errno;
 170 }
 171
 172 /**
 173  * Perform ifreq ioctl() on associated Ethernet device.
 174  *
 175  * @param[in] dev
 176  *   Pointer to Ethernet device.
 177  * @param req
 178  *   Request number to pass to ioctl().
 179  * @param[out] ifr
 180  *   Interface request structure output buffer.
 181  *
 182  * @return
 183  *   0 on success, a negative errno value otherwise and rte_errno is set.
 184  */
 185 static int
 186 mlx5_ifreq(const struct rte_eth_dev *dev, int req, struct ifreq *ifr)
 187 {
 188         int sock = socket(PF_INET, SOCK_DGRAM, IPPROTO_IP);
 189         int ret = 0;
 190
 191         if (sock == -1) {
 192                 rte_errno = errno;
 193                 return -rte_errno;
 194         }
 195         ret = mlx5_get_ifname(dev, &ifr->ifr_name);
 196         if (ret)
 197                 goto error;
 198         ret = ioctl(sock, req, ifr);
 199         if (ret == -1) {
 200                 rte_errno = errno;
 201                 goto error;
 202         }
 203         close(sock);
 204         return 0;
 205 error:
 206         close(sock);
 207         return -rte_errno;
 208 }
 209
 210 /**
 211  * Get device MTU.
 212  *
 213  * @param dev
 214  *   Pointer to Ethernet device.
 215  * @param[out] mtu
 216  *   MTU value output buffer.
 217  *
 218  * @return
 219  *   0 on success, a negative errno value otherwise and rte_errno is set.
 220  */
 221 int
 222 mlx5_get_mtu(struct rte_eth_dev *dev, uint16_t *mtu)
 223 {
 224         struct ifreq request;
 225         int ret = mlx5_ifreq(dev, SIOCGIFMTU, &request);
 226
 227         if (ret)
 228                 return ret;
 229         *mtu = request.ifr_mtu;
 230         return 0;
 231 }
 232
 233 /**
 234  * Set device MTU.
 235  *
 236  * @param dev
 237  *   Pointer to Ethernet device.
 238  * @param mtu
 239  *   MTU value to set.
 240  *
 241  * @return
 242  *   0 on success, a negative errno value otherwise and rte_errno is set.
 243  */
 244 int
 245 mlx5_set_mtu(struct rte_eth_dev *dev, uint16_t mtu)
 246 {
 247         struct ifreq request = { .ifr_mtu = mtu, };
 248
 249         return mlx5_ifreq(dev, SIOCSIFMTU, &request);
 250 }
 251
 252 /**
 253  * Set device flags.
 254  *
 255  * @param dev
 256  *   Pointer to Ethernet device.
 257  * @param keep
 258  *   Bitmask for flags that must remain untouched.
 259  * @param flags
 260  *   Bitmask for flags to modify.
 261  *
 262  * @return
 263  *   0 on success, a negative errno value otherwise and rte_errno is set.
 264  */
 265 static int
 266 mlx5_set_flags(struct rte_eth_dev *dev, unsigned int keep, unsigned int flags)
 267 {
 268         struct ifreq request;
 269         int ret = mlx5_ifreq(dev, SIOCGIFFLAGS, &request);
 270
 271         if (ret)
 272                 return ret;
 273         request.ifr_flags &= keep;
 274         request.ifr_flags |= flags & ~keep;
 275         return mlx5_ifreq(dev, SIOCSIFFLAGS, &request);
 276 }
 277
 278 /**
 279  * Get device current raw clock counter
 280  *
 281  * @param dev
 282  *   Pointer to Ethernet device structure.
 283  * @param[out] time
 284  *   Current raw clock counter of the device.
 285  *
 286  * @return
 287  *   0 if the clock has correctly been read
 288  *   The value of errno in case of error
 289  */
 290 int
 291 mlx5_read_clock(struct rte_eth_dev *dev, uint64_t *clock)
 292 {
 293         struct mlx5_priv *priv = dev->data->dev_private;
 294         struct ibv_context *ctx = priv->sh->ctx;
 295         struct ibv_values_ex values;
 296         int err = 0;
 297
 298         values.comp_mask = IBV_VALUES_MASK_RAW_CLOCK;
 299         err = mlx5_glue->query_rt_values_ex(ctx, &values);
 300         if (err != 0) {
 301                 DRV_LOG(WARNING, "Could not query the clock !");
 302                 return err;
 303         }
 304         *clock = values.raw_clock.tv_nsec;
 305         return 0;
 306 }
 307
 308 /**
 309  * Retrieve the master device for representor in the same switch domain.
 310  *
 311  * @param dev
 312  *   Pointer to representor Ethernet device structure.
 313  *
 314  * @return
 315  *   Master device structure  on success, NULL otherwise.
 316  */
 317 static struct rte_eth_dev *
 318 mlx5_find_master_dev(struct rte_eth_dev *dev)
 319 {
 320         struct mlx5_priv *priv;
 321         uint16_t port_id;
 322         uint16_t domain_id;
 323
 324         priv = dev->data->dev_private;
 325         domain_id = priv->domain_id;
 326         MLX5_ASSERT(priv->representor);
 327         MLX5_ETH_FOREACH_DEV(port_id, priv->pci_dev) {
 328                 struct mlx5_priv *opriv =
 329                         rte_eth_devices[port_id].data->dev_private;
 330                 if (opriv &&
 331                     opriv->master &&
 332                     opriv->domain_id == domain_id &&
 333                     opriv->sh == priv->sh)
 334                         return &rte_eth_devices[port_id];
 335         }
 336         return NULL;
 337 }
 338
 339 /**
 340  * DPDK callback to retrieve physical link information.
 341  *
 342  * @param dev
 343  *   Pointer to Ethernet device structure.
 344  * @param[out] link
 345  *   Storage for current link status.
 346  *
 347  * @return
 348  *   0 on success, a negative errno value otherwise and rte_errno is set.
 349  */
 350 static int
 351 mlx5_link_update_unlocked_gset(struct rte_eth_dev *dev,
 352                                struct rte_eth_link *link)
 353 {
 354         struct mlx5_priv *priv = dev->data->dev_private;
 355         struct ethtool_cmd edata = {
 356                 .cmd = ETHTOOL_GSET /* Deprecated since Linux v4.5. */
 357         };
 358         struct ifreq ifr;
 359         struct rte_eth_link dev_link;
 360         int link_speed = 0;
 361         int ret;
 362
 363         ret = mlx5_ifreq(dev, SIOCGIFFLAGS, &ifr);
 364         if (ret) {
 365                 DRV_LOG(WARNING, "port %u ioctl(SIOCGIFFLAGS) failed: %s",
 366                         dev->data->port_id, strerror(rte_errno));
 367                 return ret;
 368         }
 369         dev_link = (struct rte_eth_link) {
 370                 .link_status = ((ifr.ifr_flags & IFF_UP) &&
 371                                 (ifr.ifr_flags & IFF_RUNNING)),
 372         };
 373         ifr = (struct ifreq) {
 374                 .ifr_data = (void *)&edata,
 375         };
 376         ret = mlx5_ifreq(dev, SIOCETHTOOL, &ifr);
 377         if (ret) {
 378                 if (ret == -ENOTSUP && priv->representor) {
 379                         struct rte_eth_dev *master;
 380
 381                         /*
 382                          * For representors we can try to inherit link
 383                          * settings from the master device. Actually
 384                          * link settings do not make a lot of sense
 385                          * for representors due to missing physical
 386                          * link. The old kernel drivers supported
 387                          * emulated settings query for representors,
 388                          * the new ones do not, so we have to add
 389                          * this code for compatibility issues.
 390                          */
 391                         master = mlx5_find_master_dev(dev);
 392                         if (master) {
 393                                 ifr = (struct ifreq) {
 394                                         .ifr_data = (void *)&edata,
 395                                 };
 396                                 ret = mlx5_ifreq(master, SIOCETHTOOL, &ifr);
 397                         }
 398                 }
 399                 if (ret) {
 400                         DRV_LOG(WARNING,
 401                                 "port %u ioctl(SIOCETHTOOL,"
 402                                 " ETHTOOL_GSET) failed: %s",
 403                                 dev->data->port_id, strerror(rte_errno));
 404                         return ret;
 405                 }
 406         }
 407         link_speed = ethtool_cmd_speed(&edata);
 408         if (link_speed == -1)
 409                 dev_link.link_speed = ETH_SPEED_NUM_NONE;
 410         else
 411                 dev_link.link_speed = link_speed;
 412         priv->link_speed_capa = 0;
 413         if (edata.supported & SUPPORTED_Autoneg)
 414                 priv->link_speed_capa |= ETH_LINK_SPEED_AUTONEG;
 415         if (edata.supported & (SUPPORTED_1000baseT_Full |
 416                                SUPPORTED_1000baseKX_Full))
 417                 priv->link_speed_capa |= ETH_LINK_SPEED_1G;
 418         if (edata.supported & SUPPORTED_10000baseKR_Full)
 419                 priv->link_speed_capa |= ETH_LINK_SPEED_10G;
 420         if (edata.supported & (SUPPORTED_40000baseKR4_Full |
 421                                SUPPORTED_40000baseCR4_Full |
 422                                SUPPORTED_40000baseSR4_Full |
 423                                SUPPORTED_40000baseLR4_Full))
 424                 priv->link_speed_capa |= ETH_LINK_SPEED_40G;
 425         dev_link.link_duplex = ((edata.duplex == DUPLEX_HALF) ?
 426                                 ETH_LINK_HALF_DUPLEX : ETH_LINK_FULL_DUPLEX);
 427         dev_link.link_autoneg = !(dev->data->dev_conf.link_speeds &
 428                         ETH_LINK_SPEED_FIXED);
 429         if (((dev_link.link_speed && !dev_link.link_status) ||
 430              (!dev_link.link_speed && dev_link.link_status))) {
 431                 rte_errno = EAGAIN;
 432                 return -rte_errno;
 433         }
 434         *link = dev_link;
 435         return 0;
 436 }
 437
 438 /**
 439  * Retrieve physical link information (unlocked version using new ioctl).
 440  *
 441  * @param dev
 442  *   Pointer to Ethernet device structure.
 443  * @param[out] link
 444  *   Storage for current link status.
 445  *
 446  * @return
 447  *   0 on success, a negative errno value otherwise and rte_errno is set.
 448  */
 449 static int
 450 mlx5_link_update_unlocked_gs(struct rte_eth_dev *dev,
 451                              struct rte_eth_link *link)
 452
 453 {
 454         struct mlx5_priv *priv = dev->data->dev_private;
 455         struct ethtool_link_settings gcmd = { .cmd = ETHTOOL_GLINKSETTINGS };
 456         struct ifreq ifr;
 457         struct rte_eth_link dev_link;
 458         struct rte_eth_dev *master = NULL;
 459         uint64_t sc;
 460         int ret;
 461
 462         ret = mlx5_ifreq(dev, SIOCGIFFLAGS, &ifr);
 463         if (ret) {
 464                 DRV_LOG(WARNING, "port %u ioctl(SIOCGIFFLAGS) failed: %s",
 465                         dev->data->port_id, strerror(rte_errno));
 466                 return ret;
 467         }
 468         dev_link = (struct rte_eth_link) {
 469                 .link_status = ((ifr.ifr_flags & IFF_UP) &&
 470                                 (ifr.ifr_flags & IFF_RUNNING)),
 471         };
 472         ifr = (struct ifreq) {
 473                 .ifr_data = (void *)&gcmd,
 474         };
 475         ret = mlx5_ifreq(dev, SIOCETHTOOL, &ifr);
 476         if (ret) {
 477                 if (ret == -ENOTSUP && priv->representor) {
 478                         /*
 479                          * For representors we can try to inherit link
 480                          * settings from the master device. Actually
 481                          * link settings do not make a lot of sense
 482                          * for representors due to missing physical
 483                          * link. The old kernel drivers supported
 484                          * emulated settings query for representors,
 485                          * the new ones do not, so we have to add
 486                          * this code for compatibility issues.
 487                          */
 488                         master = mlx5_find_master_dev(dev);
 489                         if (master) {
 490                                 ifr = (struct ifreq) {
 491                                         .ifr_data = (void *)&gcmd,
 492                                 };
 493                                 ret = mlx5_ifreq(master, SIOCETHTOOL, &ifr);
 494                         }
 495                 }
 496                 if (ret) {
 497                         DRV_LOG(DEBUG,
 498                                 "port %u ioctl(SIOCETHTOOL,"
 499                                 " ETHTOOL_GLINKSETTINGS) failed: %s",
 500                                 dev->data->port_id, strerror(rte_errno));
 501                         return ret;
 502                 }
 503         }
 504         gcmd.link_mode_masks_nwords = -gcmd.link_mode_masks_nwords;
 505
 506         alignas(struct ethtool_link_settings)
 507         uint8_t data[offsetof(struct ethtool_link_settings, link_mode_masks) +
 508                      sizeof(uint32_t) * gcmd.link_mode_masks_nwords * 3];
 509         struct ethtool_link_settings *ecmd = (void *)data;
 510
 511         *ecmd = gcmd;
 512         ifr.ifr_data = (void *)ecmd;
 513         ret = mlx5_ifreq(master ? master : dev, SIOCETHTOOL, &ifr);
 514         if (ret) {
 515                 DRV_LOG(DEBUG,
 516                         "port %u ioctl(SIOCETHTOOL,"
 517                         "ETHTOOL_GLINKSETTINGS) failed: %s",
 518                         dev->data->port_id, strerror(rte_errno));
 519                 return ret;
 520         }
 521         dev_link.link_speed = (ecmd->speed == UINT32_MAX) ? ETH_SPEED_NUM_NONE :
 522                                                             ecmd->speed;
 523         sc = ecmd->link_mode_masks[0] |
 524                 ((uint64_t)ecmd->link_mode_masks[1] << 32);
 525         priv->link_speed_capa = 0;
 526         if (sc & MLX5_BITSHIFT(ETHTOOL_LINK_MODE_Autoneg_BIT))
 527                 priv->link_speed_capa |= ETH_LINK_SPEED_AUTONEG;
 528         if (sc & (MLX5_BITSHIFT(ETHTOOL_LINK_MODE_1000baseT_Full_BIT) |
 529                   MLX5_BITSHIFT(ETHTOOL_LINK_MODE_1000baseKX_Full_BIT)))
 530                 priv->link_speed_capa |= ETH_LINK_SPEED_1G;
 531         if (sc & (MLX5_BITSHIFT(ETHTOOL_LINK_MODE_10000baseKX4_Full_BIT) |
 532                   MLX5_BITSHIFT(ETHTOOL_LINK_MODE_10000baseKR_Full_BIT) |
 533                   MLX5_BITSHIFT(ETHTOOL_LINK_MODE_10000baseR_FEC_BIT)))
 534                 priv->link_speed_capa |= ETH_LINK_SPEED_10G;
 535         if (sc & (MLX5_BITSHIFT(ETHTOOL_LINK_MODE_20000baseMLD2_Full_BIT) |
 536                   MLX5_BITSHIFT(ETHTOOL_LINK_MODE_20000baseKR2_Full_BIT)))
 537                 priv->link_speed_capa |= ETH_LINK_SPEED_20G;
 538         if (sc & (MLX5_BITSHIFT(ETHTOOL_LINK_MODE_40000baseKR4_Full_BIT) |
 539                   MLX5_BITSHIFT(ETHTOOL_LINK_MODE_40000baseCR4_Full_BIT) |
 540                   MLX5_BITSHIFT(ETHTOOL_LINK_MODE_40000baseSR4_Full_BIT) |
 541                   MLX5_BITSHIFT(ETHTOOL_LINK_MODE_40000baseLR4_Full_BIT)))
 542                 priv->link_speed_capa |= ETH_LINK_SPEED_40G;
 543         if (sc & (MLX5_BITSHIFT(ETHTOOL_LINK_MODE_56000baseKR4_Full_BIT) |
 544                   MLX5_BITSHIFT(ETHTOOL_LINK_MODE_56000baseCR4_Full_BIT) |
 545                   MLX5_BITSHIFT(ETHTOOL_LINK_MODE_56000baseSR4_Full_BIT) |
 546                   MLX5_BITSHIFT(ETHTOOL_LINK_MODE_56000baseLR4_Full_BIT)))
 547                 priv->link_speed_capa |= ETH_LINK_SPEED_56G;
 548         if (sc & (MLX5_BITSHIFT(ETHTOOL_LINK_MODE_25000baseCR_Full_BIT) |
 549                   MLX5_BITSHIFT(ETHTOOL_LINK_MODE_25000baseKR_Full_BIT) |
 550                   MLX5_BITSHIFT(ETHTOOL_LINK_MODE_25000baseSR_Full_BIT)))
 551                 priv->link_speed_capa |= ETH_LINK_SPEED_25G;
 552         if (sc & (MLX5_BITSHIFT(ETHTOOL_LINK_MODE_50000baseCR2_Full_BIT) |
 553                   MLX5_BITSHIFT(ETHTOOL_LINK_MODE_50000baseKR2_Full_BIT)))
 554                 priv->link_speed_capa |= ETH_LINK_SPEED_50G;
 555         if (sc & (MLX5_BITSHIFT(ETHTOOL_LINK_MODE_100000baseKR4_Full_BIT) |
 556                   MLX5_BITSHIFT(ETHTOOL_LINK_MODE_100000baseSR4_Full_BIT) |
 557                   MLX5_BITSHIFT(ETHTOOL_LINK_MODE_100000baseCR4_Full_BIT) |
 558                   MLX5_BITSHIFT(ETHTOOL_LINK_MODE_100000baseLR4_ER4_Full_BIT)))
 559                 priv->link_speed_capa |= ETH_LINK_SPEED_100G;
 560         if (sc & (MLX5_BITSHIFT(ETHTOOL_LINK_MODE_200000baseKR4_Full_BIT) |
 561                   MLX5_BITSHIFT(ETHTOOL_LINK_MODE_200000baseSR4_Full_BIT)))
 562                 priv->link_speed_capa |= ETH_LINK_SPEED_200G;
 563
 564         sc = ecmd->link_mode_masks[2] |
 565                 ((uint64_t)ecmd->link_mode_masks[3] << 32);
 566         if (sc & (MLX5_BITSHIFT(ETHTOOL_LINK_MODE_200000baseCR4_Full_BIT) |
 567                   MLX5_BITSHIFT
 568                        (ETHTOOL_LINK_MODE_200000baseLR4_ER4_FR4_Full_BIT) |
 569                   MLX5_BITSHIFT(ETHTOOL_LINK_MODE_200000baseDR4_Full_BIT)))
 570                 priv->link_speed_capa |= ETH_LINK_SPEED_200G;
 571         dev_link.link_duplex = ((ecmd->duplex == DUPLEX_HALF) ?
 572                                 ETH_LINK_HALF_DUPLEX : ETH_LINK_FULL_DUPLEX);
 573         dev_link.link_autoneg = !(dev->data->dev_conf.link_speeds &
 574                                   ETH_LINK_SPEED_FIXED);
 575         if (((dev_link.link_speed && !dev_link.link_status) ||
 576              (!dev_link.link_speed && dev_link.link_status))) {
 577                 rte_errno = EAGAIN;
 578                 return -rte_errno;
 579         }
 580         *link = dev_link;
 581         return 0;
 582 }
 583
 584 /**
 585  * DPDK callback to retrieve physical link information.
 586  *
 587  * @param dev
 588  *   Pointer to Ethernet device structure.
 589  * @param wait_to_complete
 590  *   Wait for request completion.
 591  *
 592  * @return
 593  *   0 if link status was not updated, positive if it was, a negative errno
 594  *   value otherwise and rte_errno is set.
 595  */
 596 int
 597 mlx5_link_update(struct rte_eth_dev *dev, int wait_to_complete)
 598 {
 599         int ret;
 600         struct rte_eth_link dev_link;
 601         time_t start_time = time(NULL);
 602         int retry = MLX5_GET_LINK_STATUS_RETRY_COUNT;
 603
 604         do {
 605                 ret = mlx5_link_update_unlocked_gs(dev, &dev_link);
 606                 if (ret == -ENOTSUP)
 607                         ret = mlx5_link_update_unlocked_gset(dev, &dev_link);
 608                 if (ret == 0)
 609                         break;
 610                 /* Handle wait to complete situation. */
 611                 if ((wait_to_complete || retry) && ret == -EAGAIN) {
 612                         if (abs((int)difftime(time(NULL), start_time)) <
 613                             MLX5_LINK_STATUS_TIMEOUT) {
 614                                 usleep(0);
 615                                 continue;
 616                         } else {
 617                                 rte_errno = EBUSY;
 618                                 return -rte_errno;
 619                         }
 620                 } else if (ret < 0) {
 621                         return ret;
 622                 }
 623         } while (wait_to_complete || retry-- > 0);
 624         ret = !!memcmp(&dev->data->dev_link, &dev_link,
 625                        sizeof(struct rte_eth_link));
 626         dev->data->dev_link = dev_link;
 627         return ret;
 628 }
 629
 630 /**
 631  * DPDK callback to get flow control status.
 632  *
 633  * @param dev
 634  *   Pointer to Ethernet device structure.
 635  * @param[out] fc_conf
 636  *   Flow control output buffer.
 637  *
 638  * @return
 639  *   0 on success, a negative errno value otherwise and rte_errno is set.
 640  */
 641 int
 642 mlx5_dev_get_flow_ctrl(struct rte_eth_dev *dev, struct rte_eth_fc_conf *fc_conf)
 643 {
 644         struct ifreq ifr;
 645         struct ethtool_pauseparam ethpause = {
 646                 .cmd = ETHTOOL_GPAUSEPARAM
 647         };
 648         int ret;
 649
 650         ifr.ifr_data = (void *)&ethpause;
 651         ret = mlx5_ifreq(dev, SIOCETHTOOL, &ifr);
 652         if (ret) {
 653                 DRV_LOG(WARNING,
 654                         "port %u ioctl(SIOCETHTOOL, ETHTOOL_GPAUSEPARAM) failed:"
 655                         " %s",
 656                         dev->data->port_id, strerror(rte_errno));
 657                 return ret;
 658         }
 659         fc_conf->autoneg = ethpause.autoneg;
 660         if (ethpause.rx_pause && ethpause.tx_pause)
 661                 fc_conf->mode = RTE_FC_FULL;
 662         else if (ethpause.rx_pause)
 663                 fc_conf->mode = RTE_FC_RX_PAUSE;
 664         else if (ethpause.tx_pause)
 665                 fc_conf->mode = RTE_FC_TX_PAUSE;
 666         else
 667                 fc_conf->mode = RTE_FC_NONE;
 668         return 0;
 669 }
 670
 671 /**
 672  * DPDK callback to modify flow control parameters.
 673  *
 674  * @param dev
 675  *   Pointer to Ethernet device structure.
 676  * @param[in] fc_conf
 677  *   Flow control parameters.
 678  *
 679  * @return
 680  *   0 on success, a negative errno value otherwise and rte_errno is set.
 681  */
 682 int
 683 mlx5_dev_set_flow_ctrl(struct rte_eth_dev *dev, struct rte_eth_fc_conf *fc_conf)
 684 {
 685         struct ifreq ifr;
 686         struct ethtool_pauseparam ethpause = {
 687                 .cmd = ETHTOOL_SPAUSEPARAM
 688         };
 689         int ret;
 690
 691         ifr.ifr_data = (void *)&ethpause;
 692         ethpause.autoneg = fc_conf->autoneg;
 693         if (((fc_conf->mode & RTE_FC_FULL) == RTE_FC_FULL) ||
 694             (fc_conf->mode & RTE_FC_RX_PAUSE))
 695                 ethpause.rx_pause = 1;
 696         else
 697                 ethpause.rx_pause = 0;
 698
 699         if (((fc_conf->mode & RTE_FC_FULL) == RTE_FC_FULL) ||
 700             (fc_conf->mode & RTE_FC_TX_PAUSE))
 701                 ethpause.tx_pause = 1;
 702         else
 703                 ethpause.tx_pause = 0;
 704         ret = mlx5_ifreq(dev, SIOCETHTOOL, &ifr);
 705         if (ret) {
 706                 DRV_LOG(WARNING,
 707                         "port %u ioctl(SIOCETHTOOL, ETHTOOL_SPAUSEPARAM)"
 708                         " failed: %s",
 709                         dev->data->port_id, strerror(rte_errno));
 710                 return ret;
 711         }
 712         return 0;
 713 }
 714
 715 /**
 716  * Handle asynchronous removal event for entire multiport device.
 717  *
 718  * @param sh
 719  *   Infiniband device shared context.
 720  */
 721 static void
 722 mlx5_dev_interrupt_device_fatal(struct mlx5_dev_ctx_shared *sh)
 723 {
 724         uint32_t i;
 725
 726         for (i = 0; i < sh->max_port; ++i) {
 727                 struct rte_eth_dev *dev;
 728
 729                 if (sh->port[i].ih_port_id >= RTE_MAX_ETHPORTS) {
 730                         /*
 731                          * Or not existing port either no
 732                          * handler installed for this port.
 733                          */
 734                         continue;
 735                 }
 736                 dev = &rte_eth_devices[sh->port[i].ih_port_id];
 737                 MLX5_ASSERT(dev);
 738                 if (dev->data->dev_conf.intr_conf.rmv)
 739                         rte_eth_dev_callback_process
 740                                 (dev, RTE_ETH_EVENT_INTR_RMV, NULL);
 741         }
 742 }
 743
 744 /**
 745  * Handle shared asynchronous events the NIC (removal event
 746  * and link status change). Supports multiport IB device.
 747  *
 748  * @param cb_arg
 749  *   Callback argument.
 750  */
 751 void
 752 mlx5_dev_interrupt_handler(void *cb_arg)
 753 {
 754         struct mlx5_dev_ctx_shared *sh = cb_arg;
 755         struct ibv_async_event event;
 756
 757         /* Read all message from the IB device and acknowledge them. */
 758         for (;;) {
 759                 struct rte_eth_dev *dev;
 760                 uint32_t tmp;
 761
 762                 if (mlx5_glue->get_async_event(sh->ctx, &event))
 763                         break;
 764                 /* Retrieve and check IB port index. */
 765                 tmp = (uint32_t)event.element.port_num;
 766                 if (!tmp && event.event_type == IBV_EVENT_DEVICE_FATAL) {
 767                         /*
 768                          * The DEVICE_FATAL event is called once for
 769                          * entire device without port specifying.
 770                          * We should notify all existing ports.
 771                          */
 772                         mlx5_glue->ack_async_event(&event);
 773                         mlx5_dev_interrupt_device_fatal(sh);
 774                         continue;
 775                 }
 776                 MLX5_ASSERT(tmp && (tmp <= sh->max_port));
 777                 if (!tmp) {
 778                         /* Unsupported device level event. */
 779                         mlx5_glue->ack_async_event(&event);
 780                         DRV_LOG(DEBUG,
 781                                 "unsupported common event (type %d)",
 782                                 event.event_type);
 783                         continue;
 784                 }
 785                 if (tmp > sh->max_port) {
 786                         /* Invalid IB port index. */
 787                         mlx5_glue->ack_async_event(&event);
 788                         DRV_LOG(DEBUG,
 789                                 "cannot handle an event (type %d)"
 790                                 "due to invalid IB port index (%u)",
 791                                 event.event_type, tmp);
 792                         continue;
 793                 }
 794                 if (sh->port[tmp - 1].ih_port_id >= RTE_MAX_ETHPORTS) {
 795                         /* No handler installed. */
 796                         mlx5_glue->ack_async_event(&event);
 797                         DRV_LOG(DEBUG,
 798                                 "cannot handle an event (type %d)"
 799                                 "due to no handler installed for port %u",
 800                                 event.event_type, tmp);
 801                         continue;
 802                 }
 803                 /* Retrieve ethernet device descriptor. */
 804                 tmp = sh->port[tmp - 1].ih_port_id;
 805                 dev = &rte_eth_devices[tmp];
 806                 MLX5_ASSERT(dev);
 807                 if ((event.event_type == IBV_EVENT_PORT_ACTIVE ||
 808                      event.event_type == IBV_EVENT_PORT_ERR) &&
 809                         dev->data->dev_conf.intr_conf.lsc) {
 810                         mlx5_glue->ack_async_event(&event);
 811                         if (mlx5_link_update(dev, 0) == -EAGAIN) {
 812                                 usleep(0);
 813                                 continue;
 814                         }
 815                         rte_eth_dev_callback_process
 816                                 (dev, RTE_ETH_EVENT_INTR_LSC, NULL);
 817                         continue;
 818                 }
 819                 DRV_LOG(DEBUG,
 820                         "port %u cannot handle an unknown event (type %d)",
 821                         dev->data->port_id, event.event_type);
 822                 mlx5_glue->ack_async_event(&event);
 823         }
 824 }
 825
 826 /*
 827  * Unregister callback handler safely. The handler may be active
 828  * while we are trying to unregister it, in this case code -EAGAIN
 829  * is returned by rte_intr_callback_unregister(). This routine checks
 830  * the return code and tries to unregister handler again.
 831  *
 832  * @param handle
 833  *   interrupt handle
 834  * @param cb_fn
 835  *   pointer to callback routine
 836  * @cb_arg
 837  *   opaque callback parameter
 838  */
 839 void
 840 mlx5_intr_callback_unregister(const struct rte_intr_handle *handle,
 841                               rte_intr_callback_fn cb_fn, void *cb_arg)
 842 {
 843         /*
 844          * Try to reduce timeout management overhead by not calling
 845          * the timer related routines on the first iteration. If the
 846          * unregistering succeeds on first call there will be no
 847          * timer calls at all.
 848          */
 849         uint64_t twait = 0;
 850         uint64_t start = 0;
 851
 852         do {
 853                 int ret;
 854
 855                 ret = rte_intr_callback_unregister(handle, cb_fn, cb_arg);
 856                 if (ret >= 0)
 857                         return;
 858                 if (ret != -EAGAIN) {
 859                         DRV_LOG(INFO, "failed to unregister interrupt"
 860                                       " handler (error: %d)", ret);
 861                         MLX5_ASSERT(false);
 862                         return;
 863                 }
 864                 if (twait) {
 865                         struct timespec onems;
 866
 867                         /* Wait one millisecond and try again. */
 868                         onems.tv_sec = 0;
 869                         onems.tv_nsec = NS_PER_S / MS_PER_S;
 870                         nanosleep(&onems, 0);
 871                         /* Check whether one second elapsed. */
 872                         if ((rte_get_timer_cycles() - start) <= twait)
 873                                 continue;
 874                 } else {
 875                         /*
 876                          * We get the amount of timer ticks for one second.
 877                          * If this amount elapsed it means we spent one
 878                          * second in waiting. This branch is executed once
 879                          * on first iteration.
 880                          */
 881                         twait = rte_get_timer_hz();
 882                         MLX5_ASSERT(twait);
 883                 }
 884                 /*
 885                  * Timeout elapsed, show message (once a second) and retry.
 886                  * We have no other acceptable option here, if we ignore
 887                  * the unregistering return code the handler will not
 888                  * be unregistered, fd will be closed and we may get the
 889                  * crush. Hanging and messaging in the loop seems not to be
 890                  * the worst choice.
 891                  */
 892                 DRV_LOG(INFO, "Retrying to unregister interrupt handler");
 893                 start = rte_get_timer_cycles();
 894         } while (true);
 895 }
 896
 897 /**
 898  * Handle DEVX interrupts from the NIC.
 899  * This function is probably called from the DPDK host thread.
 900  *
 901  * @param cb_arg
 902  *   Callback argument.
 903  */
 904 void
 905 mlx5_dev_interrupt_handler_devx(void *cb_arg)
 906 {
 907 #ifndef HAVE_IBV_DEVX_ASYNC
 908         (void)cb_arg;
 909         return;
 910 #else
 911         struct mlx5_dev_ctx_shared *sh = cb_arg;
 912         union {
 913                 struct mlx5dv_devx_async_cmd_hdr cmd_resp;
 914                 uint8_t buf[MLX5_ST_SZ_BYTES(query_flow_counter_out) +
 915                             MLX5_ST_SZ_BYTES(traffic_counter) +
 916                             sizeof(struct mlx5dv_devx_async_cmd_hdr)];
 917         } out;
 918         uint8_t *buf = out.buf + sizeof(out.cmd_resp);
 919
 920         while (!mlx5_glue->devx_get_async_cmd_comp(sh->devx_comp,
 921                                                    &out.cmd_resp,
 922                                                    sizeof(out.buf)))
 923                 mlx5_flow_async_pool_query_handle
 924                         (sh, (uint64_t)out.cmd_resp.wr_id,
 925                          mlx5_devx_get_out_command_status(buf));
 926 #endif /* HAVE_IBV_DEVX_ASYNC */
 927 }
 928
 929 /**
 930  * DPDK callback to bring the link DOWN.
 931  *
 932  * @param dev
 933  *   Pointer to Ethernet device structure.
 934  *
 935  * @return
 936  *   0 on success, a negative errno value otherwise and rte_errno is set.
 937  */
 938 int
 939 mlx5_set_link_down(struct rte_eth_dev *dev)
 940 {
 941         return mlx5_set_flags(dev, ~IFF_UP, ~IFF_UP);
 942 }
 943
 944 /**
 945  * DPDK callback to bring the link UP.
 946  *
 947  * @param dev
 948  *   Pointer to Ethernet device structure.
 949  *
 950  * @return
 951  *   0 on success, a negative errno value otherwise and rte_errno is set.
 952  */
 953 int
 954 mlx5_set_link_up(struct rte_eth_dev *dev)
 955 {
 956         return mlx5_set_flags(dev, ~IFF_UP, IFF_UP);
 957 }
 958
 959 /**
 960  * Check if mlx5 device was removed.
 961  *
 962  * @param dev
 963  *   Pointer to Ethernet device structure.
 964  *
 965  * @return
 966  *   1 when device is removed, otherwise 0.
 967  */
 968 int
 969 mlx5_is_removed(struct rte_eth_dev *dev)
 970 {
 971         struct ibv_device_attr device_attr;
 972         struct mlx5_priv *priv = dev->data->dev_private;
 973
 974         if (mlx5_glue->query_device(priv->sh->ctx, &device_attr) == EIO)
 975                 return 1;
 976         return 0;
 977 }
 978
 979 /**
 980  * Analyze gathered port parameters via sysfs to recognize master
 981  * and representor devices for E-Switch configuration.
 982  *
 983  * @param[in] device_dir
 984  *   flag of presence of "device" directory under port device key.
 985  * @param[inout] switch_info
 986  *   Port information, including port name as a number and port name
 987  *   type if recognized
 988  *
 989  * @return
 990  *   master and representor flags are set in switch_info according to
 991  *   recognized parameters (if any).
 992  */
 993 static void
 994 mlx5_sysfs_check_switch_info(bool device_dir,
 995                              struct mlx5_switch_info *switch_info)
 996 {
 997         switch (switch_info->name_type) {
 998         case MLX5_PHYS_PORT_NAME_TYPE_UNKNOWN:
 999                 /*
1000                  * Name is not recognized, assume the master,
1001                  * check the device directory presence.
1002                  */
1003                 switch_info->master = device_dir;
1004                 break;
1005         case MLX5_PHYS_PORT_NAME_TYPE_NOTSET:
1006                 /*
1007                  * Name is not set, this assumes the legacy naming
1008                  * schema for master, just check if there is
1009                  * a device directory.
1010                  */
1011                 switch_info->master = device_dir;
1012                 break;
1013         case MLX5_PHYS_PORT_NAME_TYPE_UPLINK:
1014                 /* New uplink naming schema recognized. */
1015                 switch_info->master = 1;
1016                 break;
1017         case MLX5_PHYS_PORT_NAME_TYPE_LEGACY:
1018                 /* Legacy representors naming schema. */
1019                 switch_info->representor = !device_dir;
1020                 break;
1021         case MLX5_PHYS_PORT_NAME_TYPE_PFHPF:
1022                 /* Fallthrough */
1023         case MLX5_PHYS_PORT_NAME_TYPE_PFVF:
1024                 /* New representors naming schema. */
1025                 switch_info->representor = 1;
1026                 break;
1027         }
1028 }
1029
1030 /**
1031  * Get switch information associated with network interface.
1032  *
1033  * @param ifindex
1034  *   Network interface index.
1035  * @param[out] info
1036  *   Switch information object, populated in case of success.
1037  *
1038  * @return
1039  *   0 on success, a negative errno value otherwise and rte_errno is set.
1040  */
1041 int
1042 mlx5_sysfs_switch_info(unsigned int ifindex, struct mlx5_switch_info *info)
1043 {
1044         char ifname[IF_NAMESIZE];
1045         char port_name[IF_NAMESIZE];
1046         FILE *file;
1047         struct mlx5_switch_info data = {
1048                 .master = 0,
1049                 .representor = 0,
1050                 .name_type = MLX5_PHYS_PORT_NAME_TYPE_NOTSET,
1051                 .port_name = 0,
1052                 .switch_id = 0,
1053         };
1054         DIR *dir;
1055         bool port_switch_id_set = false;
1056         bool device_dir = false;
1057         char c;
1058         int ret;
1059
1060         if (!if_indextoname(ifindex, ifname)) {
1061                 rte_errno = errno;
1062                 return -rte_errno;
1063         }
1064
1065         MKSTR(phys_port_name, "/sys/class/net/%s/phys_port_name",
1066               ifname);
1067         MKSTR(phys_switch_id, "/sys/class/net/%s/phys_switch_id",
1068               ifname);
1069         MKSTR(pci_device, "/sys/class/net/%s/device",
1070               ifname);
1071
1072         file = fopen(phys_port_name, "rb");
1073         if (file != NULL) {
1074                 ret = fscanf(file, "%" RTE_STR(IF_NAMESIZE) "s", port_name);
1075                 fclose(file);
1076                 if (ret == 1)
1077                         mlx5_translate_port_name(port_name, &data);
1078         }
1079         file = fopen(phys_switch_id, "rb");
1080         if (file == NULL) {
1081                 rte_errno = errno;
1082                 return -rte_errno;
1083         }
1084         port_switch_id_set =
1085                 fscanf(file, "%" SCNx64 "%c", &data.switch_id, &c) == 2 &&
1086                 c == '\n';
1087         fclose(file);
1088         dir = opendir(pci_device);
1089         if (dir != NULL) {
1090                 closedir(dir);
1091                 device_dir = true;
1092         }
1093         if (port_switch_id_set) {
1094                 /* We have some E-Switch configuration. */
1095                 mlx5_sysfs_check_switch_info(device_dir, &data);
1096         }
1097         *info = data;
1098         MLX5_ASSERT(!(data.master && data.representor));
1099         if (data.master && data.representor) {
1100                 DRV_LOG(ERR, "ifindex %u device is recognized as master"
1101                              " and as representor", ifindex);
1102                 rte_errno = ENODEV;
1103                 return -rte_errno;
1104         }
1105         return 0;
1106 }
1107
1108 /**
1109  * Get bond information associated with network interface.
1110  *
1111  * @param pf_ifindex
1112  *   Network interface index of bond slave interface
1113  * @param[out] ifindex
1114  *   Pointer to bond ifindex.
1115  * @param[out] ifname
1116  *   Pointer to bond ifname.
1117  *
1118  * @return
1119  *   0 on success, a negative errno value otherwise and rte_errno is set.
1120  */
1121 int
1122 mlx5_sysfs_bond_info(unsigned int pf_ifindex, unsigned int *ifindex,
1123                      char *ifname)
1124 {
1125         char name[IF_NAMESIZE];
1126         FILE *file;
1127         unsigned int index;
1128         int ret;
1129
1130         if (!if_indextoname(pf_ifindex, name) || !strlen(name)) {
1131                 rte_errno = errno;
1132                 return -rte_errno;
1133         }
1134         MKSTR(bond_if, "/sys/class/net/%s/master/ifindex", name);
1135         /* read bond ifindex */
1136         file = fopen(bond_if, "rb");
1137         if (file == NULL) {
1138                 rte_errno = errno;
1139                 return -rte_errno;
1140         }
1141         ret = fscanf(file, "%u", &index);
1142         fclose(file);
1143         if (ret <= 0) {
1144                 rte_errno = errno;
1145                 return -rte_errno;
1146         }
1147         if (ifindex)
1148                 *ifindex = index;
1149
1150         /* read bond device name from symbol link */
1151         if (ifname) {
1152                 if (!if_indextoname(index, ifname)) {
1153                         rte_errno = errno;
1154                         return -rte_errno;
1155                 }
1156         }
1157         return 0;
1158 }
1159
1160 /**
1161  * DPDK callback to retrieve plug-in module EEPROM information (type and size).
1162  *
1163  * @param dev
1164  *   Pointer to Ethernet device structure.
1165  * @param[out] modinfo
1166  *   Storage for plug-in module EEPROM information.
1167  *
1168  * @return
1169  *   0 on success, a negative errno value otherwise and rte_errno is set.
1170  */
1171 int
1172 mlx5_get_module_info(struct rte_eth_dev *dev,
1173                      struct rte_eth_dev_module_info *modinfo)
1174 {
1175         struct ethtool_modinfo info = {
1176                 .cmd = ETHTOOL_GMODULEINFO,
1177         };
1178         struct ifreq ifr = (struct ifreq) {
1179                 .ifr_data = (void *)&info,
1180         };
1181         int ret = 0;
1182
1183         if (!dev || !modinfo) {
1184                 DRV_LOG(WARNING, "missing argument, cannot get module info");
1185                 rte_errno = EINVAL;
1186                 return -rte_errno;
1187         }
1188         ret = mlx5_ifreq(dev, SIOCETHTOOL, &ifr);
1189         if (ret) {
1190                 DRV_LOG(WARNING, "port %u ioctl(SIOCETHTOOL) failed: %s",
1191                         dev->data->port_id, strerror(rte_errno));
1192                 return ret;
1193         }
1194         modinfo->type = info.type;
1195         modinfo->eeprom_len = info.eeprom_len;
1196         return ret;
1197 }
1198
1199 /**
1200  * DPDK callback to retrieve plug-in module EEPROM data.
1201  *
1202  * @param dev
1203  *   Pointer to Ethernet device structure.
1204  * @param[out] info
1205  *   Storage for plug-in module EEPROM data.
1206  *
1207  * @return
1208  *   0 on success, a negative errno value otherwise and rte_errno is set.
1209  */
1210 int mlx5_get_module_eeprom(struct rte_eth_dev *dev,
1211                            struct rte_dev_eeprom_info *info)
1212 {
1213         struct ethtool_eeprom *eeprom;
1214         struct ifreq ifr;
1215         int ret = 0;
1216
1217         if (!dev || !info) {
1218                 DRV_LOG(WARNING, "missing argument, cannot get module eeprom");
1219                 rte_errno = EINVAL;
1220                 return -rte_errno;
1221         }
1222         eeprom = mlx5_malloc(MLX5_MEM_ZERO,
1223                              (sizeof(struct ethtool_eeprom) + info->length), 0,
1224                              SOCKET_ID_ANY);
1225         if (!eeprom) {
1226                 DRV_LOG(WARNING, "port %u cannot allocate memory for "
1227                         "eeprom data", dev->data->port_id);
1228                 rte_errno = ENOMEM;
1229                 return -rte_errno;
1230         }
1231         eeprom->cmd = ETHTOOL_GMODULEEEPROM;
1232         eeprom->offset = info->offset;
1233         eeprom->len = info->length;
1234         ifr = (struct ifreq) {
1235                 .ifr_data = (void *)eeprom,
1236         };
1237         ret = mlx5_ifreq(dev, SIOCETHTOOL, &ifr);
1238         if (ret)
1239                 DRV_LOG(WARNING, "port %u ioctl(SIOCETHTOOL) failed: %s",
1240                         dev->data->port_id, strerror(rte_errno));
1241         else
1242                 rte_memcpy(info->data, eeprom->data, info->length);
1243         mlx5_free(eeprom);
1244         return ret;
1245 }
1246
1247 /**
1248  * Read device counters table.
1249  *
1250  * @param dev
1251  *   Pointer to Ethernet device.
1252  * @param[out] stats
1253  *   Counters table output buffer.
1254  *
1255  * @return
1256  *   0 on success and stats is filled, negative errno value otherwise and
1257  *   rte_errno is set.
1258  */
1259 int
1260 mlx5_os_read_dev_counters(struct rte_eth_dev *dev, uint64_t *stats)
1261 {
1262         struct mlx5_priv *priv = dev->data->dev_private;
1263         struct mlx5_xstats_ctrl *xstats_ctrl = &priv->xstats_ctrl;
1264         unsigned int i;
1265         struct ifreq ifr;
1266         unsigned int stats_sz = xstats_ctrl->stats_n * sizeof(uint64_t);
1267         unsigned char et_stat_buf[sizeof(struct ethtool_stats) + stats_sz];
1268         struct ethtool_stats *et_stats = (struct ethtool_stats *)et_stat_buf;
1269         int ret;
1270
1271         et_stats->cmd = ETHTOOL_GSTATS;
1272         et_stats->n_stats = xstats_ctrl->stats_n;
1273         ifr.ifr_data = (caddr_t)et_stats;
1274         ret = mlx5_ifreq(dev, SIOCETHTOOL, &ifr);
1275         if (ret) {
1276                 DRV_LOG(WARNING,
1277                         "port %u unable to read statistic values from device",
1278                         dev->data->port_id);
1279                 return ret;
1280         }
1281         for (i = 0; i != xstats_ctrl->mlx5_stats_n; ++i) {
1282                 if (xstats_ctrl->info[i].dev) {
1283                         ret = mlx5_os_read_dev_stat(priv,
1284                                             xstats_ctrl->info[i].ctr_name,
1285                                             &stats[i]);
1286                         /* return last xstats counter if fail to read. */
1287                         if (ret == 0)
1288                                 xstats_ctrl->xstats[i] = stats[i];
1289                         else
1290                                 stats[i] = xstats_ctrl->xstats[i];
1291                 } else {
1292                         stats[i] = (uint64_t)
1293                                 et_stats->data[xstats_ctrl->dev_table_idx[i]];
1294                 }
1295         }
1296         return 0;
1297 }
1298
1299 /**
1300  * Query the number of statistics provided by ETHTOOL.
1301  *
1302  * @param dev
1303  *   Pointer to Ethernet device.
1304  *
1305  * @return
1306  *   Number of statistics on success, negative errno value otherwise and
1307  *   rte_errno is set.
1308  */
1309 int
1310 mlx5_os_get_stats_n(struct rte_eth_dev *dev)
1311 {
1312         struct ethtool_drvinfo drvinfo;
1313         struct ifreq ifr;
1314         int ret;
1315
1316         drvinfo.cmd = ETHTOOL_GDRVINFO;
1317         ifr.ifr_data = (caddr_t)&drvinfo;
1318         ret = mlx5_ifreq(dev, SIOCETHTOOL, &ifr);
1319         if (ret) {
1320                 DRV_LOG(WARNING, "port %u unable to query number of statistics",
1321                         dev->data->port_id);
1322                 return ret;
1323         }
1324         return drvinfo.n_stats;
1325 }
1326
1327 static const struct mlx5_counter_ctrl mlx5_counters_init[] = {
1328         {
1329                 .dpdk_name = "rx_port_unicast_bytes",
1330                 .ctr_name = "rx_vport_unicast_bytes",
1331         },
1332         {
1333                 .dpdk_name = "rx_port_multicast_bytes",
1334                 .ctr_name = "rx_vport_multicast_bytes",
1335         },
1336         {
1337                 .dpdk_name = "rx_port_broadcast_bytes",
1338                 .ctr_name = "rx_vport_broadcast_bytes",
1339         },
1340         {
1341                 .dpdk_name = "rx_port_unicast_packets",
1342                 .ctr_name = "rx_vport_unicast_packets",
1343         },
1344         {
1345                 .dpdk_name = "rx_port_multicast_packets",
1346                 .ctr_name = "rx_vport_multicast_packets",
1347         },
1348         {
1349                 .dpdk_name = "rx_port_broadcast_packets",
1350                 .ctr_name = "rx_vport_broadcast_packets",
1351         },
1352         {
1353                 .dpdk_name = "tx_port_unicast_bytes",
1354                 .ctr_name = "tx_vport_unicast_bytes",
1355         },
1356         {
1357                 .dpdk_name = "tx_port_multicast_bytes",
1358                 .ctr_name = "tx_vport_multicast_bytes",
1359         },
1360         {
1361                 .dpdk_name = "tx_port_broadcast_bytes",
1362                 .ctr_name = "tx_vport_broadcast_bytes",
1363         },
1364         {
1365                 .dpdk_name = "tx_port_unicast_packets",
1366                 .ctr_name = "tx_vport_unicast_packets",
1367         },
1368         {
1369                 .dpdk_name = "tx_port_multicast_packets",
1370                 .ctr_name = "tx_vport_multicast_packets",
1371         },
1372         {
1373                 .dpdk_name = "tx_port_broadcast_packets",
1374                 .ctr_name = "tx_vport_broadcast_packets",
1375         },
1376         {
1377                 .dpdk_name = "rx_wqe_err",
1378                 .ctr_name = "rx_wqe_err",
1379         },
1380         {
1381                 .dpdk_name = "rx_crc_errors_phy",
1382                 .ctr_name = "rx_crc_errors_phy",
1383         },
1384         {
1385                 .dpdk_name = "rx_in_range_len_errors_phy",
1386                 .ctr_name = "rx_in_range_len_errors_phy",
1387         },
1388         {
1389                 .dpdk_name = "rx_symbol_err_phy",
1390                 .ctr_name = "rx_symbol_err_phy",
1391         },
1392         {
1393                 .dpdk_name = "tx_errors_phy",
1394                 .ctr_name = "tx_errors_phy",
1395         },
1396         {
1397                 .dpdk_name = "rx_out_of_buffer",
1398                 .ctr_name = "out_of_buffer",
1399                 .dev = 1,
1400         },
1401         {
1402                 .dpdk_name = "tx_packets_phy",
1403                 .ctr_name = "tx_packets_phy",
1404         },
1405         {
1406                 .dpdk_name = "rx_packets_phy",
1407                 .ctr_name = "rx_packets_phy",
1408         },
1409         {
1410                 .dpdk_name = "tx_discards_phy",
1411                 .ctr_name = "tx_discards_phy",
1412         },
1413         {
1414                 .dpdk_name = "rx_discards_phy",
1415                 .ctr_name = "rx_discards_phy",
1416         },
1417         {
1418                 .dpdk_name = "tx_bytes_phy",
1419                 .ctr_name = "tx_bytes_phy",
1420         },
1421         {
1422                 .dpdk_name = "rx_bytes_phy",
1423                 .ctr_name = "rx_bytes_phy",
1424         },
1425         /* Representor only */
1426         {
1427                 .dpdk_name = "rx_packets",
1428                 .ctr_name = "vport_rx_packets",
1429         },
1430         {
1431                 .dpdk_name = "rx_bytes",
1432                 .ctr_name = "vport_rx_bytes",
1433         },
1434         {
1435                 .dpdk_name = "tx_packets",
1436                 .ctr_name = "vport_tx_packets",
1437         },
1438         {
1439                 .dpdk_name = "tx_bytes",
1440                 .ctr_name = "vport_tx_bytes",
1441         },
1442 };
1443
1444 static const unsigned int xstats_n = RTE_DIM(mlx5_counters_init);
1445
1446 /**
1447  * Init the structures to read device counters.
1448  *
1449  * @param dev
1450  *   Pointer to Ethernet device.
1451  */
1452 void
1453 mlx5_os_stats_init(struct rte_eth_dev *dev)
1454 {
1455         struct mlx5_priv *priv = dev->data->dev_private;
1456         struct mlx5_xstats_ctrl *xstats_ctrl = &priv->xstats_ctrl;
1457         struct mlx5_stats_ctrl *stats_ctrl = &priv->stats_ctrl;
1458         unsigned int i;
1459         unsigned int j;
1460         struct ifreq ifr;
1461         struct ethtool_gstrings *strings = NULL;
1462         unsigned int dev_stats_n;
1463         unsigned int str_sz;
1464         int ret;
1465
1466         /* So that it won't aggregate for each init. */
1467         xstats_ctrl->mlx5_stats_n = 0;
1468         ret = mlx5_os_get_stats_n(dev);
1469         if (ret < 0) {
1470                 DRV_LOG(WARNING, "port %u no extended statistics available",
1471                         dev->data->port_id);
1472                 return;
1473         }
1474         dev_stats_n = ret;
1475         /* Allocate memory to grab stat names and values. */
1476         str_sz = dev_stats_n * ETH_GSTRING_LEN;
1477         strings = (struct ethtool_gstrings *)
1478                   mlx5_malloc(0, str_sz + sizeof(struct ethtool_gstrings), 0,
1479                               SOCKET_ID_ANY);
1480         if (!strings) {
1481                 DRV_LOG(WARNING, "port %u unable to allocate memory for xstats",
1482                      dev->data->port_id);
1483                 return;
1484         }
1485         strings->cmd = ETHTOOL_GSTRINGS;
1486         strings->string_set = ETH_SS_STATS;
1487         strings->len = dev_stats_n;
1488         ifr.ifr_data = (caddr_t)strings;
1489         ret = mlx5_ifreq(dev, SIOCETHTOOL, &ifr);
1490         if (ret) {
1491                 DRV_LOG(WARNING, "port %u unable to get statistic names",
1492                         dev->data->port_id);
1493                 goto free;
1494         }
1495         for (i = 0; i != dev_stats_n; ++i) {
1496                 const char *curr_string = (const char *)
1497                         &strings->data[i * ETH_GSTRING_LEN];
1498
1499                 for (j = 0; j != xstats_n; ++j) {
1500                         if (!strcmp(mlx5_counters_init[j].ctr_name,
1501                                     curr_string)) {
1502                                 unsigned int idx = xstats_ctrl->mlx5_stats_n++;
1503
1504                                 xstats_ctrl->dev_table_idx[idx] = i;
1505                                 xstats_ctrl->info[idx] = mlx5_counters_init[j];
1506                                 break;
1507                         }
1508                 }
1509         }
1510         /* Add dev counters. */
1511         for (i = 0; i != xstats_n; ++i) {
1512                 if (mlx5_counters_init[i].dev) {
1513                         unsigned int idx = xstats_ctrl->mlx5_stats_n++;
1514
1515                         xstats_ctrl->info[idx] = mlx5_counters_init[i];
1516                         xstats_ctrl->hw_stats[idx] = 0;
1517                 }
1518         }
1519         MLX5_ASSERT(xstats_ctrl->mlx5_stats_n <= MLX5_MAX_XSTATS);
1520         xstats_ctrl->stats_n = dev_stats_n;
1521         /* Copy to base at first time. */
1522         ret = mlx5_os_read_dev_counters(dev, xstats_ctrl->base);
1523         if (ret)
1524                 DRV_LOG(ERR, "port %u cannot read device counters: %s",
1525                         dev->data->port_id, strerror(rte_errno));
1526         mlx5_os_read_dev_stat(priv, "out_of_buffer", &stats_ctrl->imissed_base);
1527         stats_ctrl->imissed = 0;
1528 free:
1529         mlx5_free(strings);
1530 }
1531
1532 /**
1533  * Get MAC address by querying netdevice.
1534  *
1535  * @param[in] dev
1536  *   Pointer to Ethernet device.
1537  * @param[out] mac
1538  *   MAC address output buffer.
1539  *
1540  * @return
1541  *   0 on success, a negative errno value otherwise and rte_errno is set.
1542  */
1543 int
1544 mlx5_get_mac(struct rte_eth_dev *dev, uint8_t (*mac)[RTE_ETHER_ADDR_LEN])
1545 {
1546         struct ifreq request;
1547         int ret;
1548
1549         ret = mlx5_ifreq(dev, SIOCGIFHWADDR, &request);
1550         if (ret)
1551                 return ret;
1552         memcpy(mac, request.ifr_hwaddr.sa_data, RTE_ETHER_ADDR_LEN);
1553         return 0;
1554 }
1555