drivers/net/mlx5/linux/mlx5_ethdev_os.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright 2015 6WIND S.A.
   3  * Copyright 2015 Mellanox Technologies, Ltd
   4  */
   5
   6 #include <stddef.h>
   7 #include <inttypes.h>
   8 #include <unistd.h>
   9 #include <stdbool.h>
  10 #include <stdint.h>
  11 #include <stdio.h>
  12 #include <string.h>
  13 #include <stdlib.h>
  14 #include <errno.h>
  15 #include <dirent.h>
  16 #include <net/if.h>
  17 #include <sys/ioctl.h>
  18 #include <sys/socket.h>
  19 #include <netinet/in.h>
  20 #include <linux/ethtool.h>
  21 #include <linux/sockios.h>
  22 #include <fcntl.h>
  23 #include <stdalign.h>
  24 #include <sys/un.h>
  25 #include <time.h>
  26
  27 #include <rte_ethdev_driver.h>
  28 #include <rte_bus_pci.h>
  29 #include <rte_mbuf.h>
  30 #include <rte_common.h>
  31 #include <rte_interrupts.h>
  32 #include <rte_malloc.h>
  33 #include <rte_string_fns.h>
  34 #include <rte_rwlock.h>
  35 #include <rte_cycles.h>
  36
  37 #include <mlx5_glue.h>
  38 #include <mlx5_devx_cmds.h>
  39 #include <mlx5_common.h>
  40 #include <mlx5_malloc.h>
  41
  42 #include "mlx5.h"
  43 #include "mlx5_rxtx.h"
  44 #include "mlx5_utils.h"
  45
  46 /* Supported speed values found in /usr/include/linux/ethtool.h */
  47 #ifndef HAVE_SUPPORTED_40000baseKR4_Full
  48 #define SUPPORTED_40000baseKR4_Full (1 << 23)
  49 #endif
  50 #ifndef HAVE_SUPPORTED_40000baseCR4_Full
  51 #define SUPPORTED_40000baseCR4_Full (1 << 24)
  52 #endif
  53 #ifndef HAVE_SUPPORTED_40000baseSR4_Full
  54 #define SUPPORTED_40000baseSR4_Full (1 << 25)
  55 #endif
  56 #ifndef HAVE_SUPPORTED_40000baseLR4_Full
  57 #define SUPPORTED_40000baseLR4_Full (1 << 26)
  58 #endif
  59 #ifndef HAVE_SUPPORTED_56000baseKR4_Full
  60 #define SUPPORTED_56000baseKR4_Full (1 << 27)
  61 #endif
  62 #ifndef HAVE_SUPPORTED_56000baseCR4_Full
  63 #define SUPPORTED_56000baseCR4_Full (1 << 28)
  64 #endif
  65 #ifndef HAVE_SUPPORTED_56000baseSR4_Full
  66 #define SUPPORTED_56000baseSR4_Full (1 << 29)
  67 #endif
  68 #ifndef HAVE_SUPPORTED_56000baseLR4_Full
  69 #define SUPPORTED_56000baseLR4_Full (1 << 30)
  70 #endif
  71
  72 /* Add defines in case the running kernel is not the same as user headers. */
  73 #ifndef ETHTOOL_GLINKSETTINGS
  74 struct ethtool_link_settings {
  75         uint32_t cmd;
  76         uint32_t speed;
  77         uint8_t duplex;
  78         uint8_t port;
  79         uint8_t phy_address;
  80         uint8_t autoneg;
  81         uint8_t mdio_support;
  82         uint8_t eth_to_mdix;
  83         uint8_t eth_tp_mdix_ctrl;
  84         int8_t link_mode_masks_nwords;
  85         uint32_t reserved[8];
  86         uint32_t link_mode_masks[];
  87 };
  88
  89 /* The kernel values can be found in /include/uapi/linux/ethtool.h */
  90 #define ETHTOOL_GLINKSETTINGS 0x0000004c
  91 #define ETHTOOL_LINK_MODE_1000baseT_Full_BIT 5
  92 #define ETHTOOL_LINK_MODE_Autoneg_BIT 6
  93 #define ETHTOOL_LINK_MODE_1000baseKX_Full_BIT 17
  94 #define ETHTOOL_LINK_MODE_10000baseKX4_Full_BIT 18
  95 #define ETHTOOL_LINK_MODE_10000baseKR_Full_BIT 19
  96 #define ETHTOOL_LINK_MODE_10000baseR_FEC_BIT 20
  97 #define ETHTOOL_LINK_MODE_20000baseMLD2_Full_BIT 21
  98 #define ETHTOOL_LINK_MODE_20000baseKR2_Full_BIT 22
  99 #define ETHTOOL_LINK_MODE_40000baseKR4_Full_BIT 23
 100 #define ETHTOOL_LINK_MODE_40000baseCR4_Full_BIT 24
 101 #define ETHTOOL_LINK_MODE_40000baseSR4_Full_BIT 25
 102 #define ETHTOOL_LINK_MODE_40000baseLR4_Full_BIT 26
 103 #define ETHTOOL_LINK_MODE_56000baseKR4_Full_BIT 27
 104 #define ETHTOOL_LINK_MODE_56000baseCR4_Full_BIT 28
 105 #define ETHTOOL_LINK_MODE_56000baseSR4_Full_BIT 29
 106 #define ETHTOOL_LINK_MODE_56000baseLR4_Full_BIT 30
 107 #endif
 108 #ifndef HAVE_ETHTOOL_LINK_MODE_25G
 109 #define ETHTOOL_LINK_MODE_25000baseCR_Full_BIT 31
 110 #define ETHTOOL_LINK_MODE_25000baseKR_Full_BIT 32
 111 #define ETHTOOL_LINK_MODE_25000baseSR_Full_BIT 33
 112 #endif
 113 #ifndef HAVE_ETHTOOL_LINK_MODE_50G
 114 #define ETHTOOL_LINK_MODE_50000baseCR2_Full_BIT 34
 115 #define ETHTOOL_LINK_MODE_50000baseKR2_Full_BIT 35
 116 #endif
 117 #ifndef HAVE_ETHTOOL_LINK_MODE_100G
 118 #define ETHTOOL_LINK_MODE_100000baseKR4_Full_BIT 36
 119 #define ETHTOOL_LINK_MODE_100000baseSR4_Full_BIT 37
 120 #define ETHTOOL_LINK_MODE_100000baseCR4_Full_BIT 38
 121 #define ETHTOOL_LINK_MODE_100000baseLR4_ER4_Full_BIT 39
 122 #endif
 123 #ifndef HAVE_ETHTOOL_LINK_MODE_200G
 124 #define ETHTOOL_LINK_MODE_200000baseKR4_Full_BIT 62
 125 #define ETHTOOL_LINK_MODE_200000baseSR4_Full_BIT 63
 126 #define ETHTOOL_LINK_MODE_200000baseLR4_ER4_FR4_Full_BIT 0 /* 64 - 64 */
 127 #define ETHTOOL_LINK_MODE_200000baseDR4_Full_BIT 1 /* 65 - 64 */
 128 #define ETHTOOL_LINK_MODE_200000baseCR4_Full_BIT 2 /* 66 - 64 */
 129 #endif
 130
 131
 132 /**
 133  * Get interface name from private structure.
 134  *
 135  * This is a port representor-aware version of mlx5_get_ifname_sysfs().
 136  *
 137  * @param[in] dev
 138  *   Pointer to Ethernet device.
 139  * @param[out] ifname
 140  *   Interface name output buffer.
 141  *
 142  * @return
 143  *   0 on success, a negative errno value otherwise and rte_errno is set.
 144  */
 145 int
 146 mlx5_get_ifname(const struct rte_eth_dev *dev, char (*ifname)[IF_NAMESIZE])
 147 {
 148         struct mlx5_priv *priv = dev->data->dev_private;
 149         unsigned int ifindex;
 150
 151         MLX5_ASSERT(priv);
 152         MLX5_ASSERT(priv->sh);
 153         if (priv->bond_ifindex > 0) {
 154                 memcpy(ifname, priv->bond_name, IF_NAMESIZE);
 155                 return 0;
 156         }
 157         ifindex = mlx5_ifindex(dev);
 158         if (!ifindex) {
 159                 if (!priv->representor)
 160                         return mlx5_get_ifname_sysfs(priv->sh->ibdev_path,
 161                                                      *ifname);
 162                 rte_errno = ENXIO;
 163                 return -rte_errno;
 164         }
 165         if (if_indextoname(ifindex, &(*ifname)[0]))
 166                 return 0;
 167         rte_errno = errno;
 168         return -rte_errno;
 169 }
 170
 171 /**
 172  * Perform ifreq ioctl() on associated Ethernet device.
 173  *
 174  * @param[in] dev
 175  *   Pointer to Ethernet device.
 176  * @param req
 177  *   Request number to pass to ioctl().
 178  * @param[out] ifr
 179  *   Interface request structure output buffer.
 180  *
 181  * @return
 182  *   0 on success, a negative errno value otherwise and rte_errno is set.
 183  */
 184 static int
 185 mlx5_ifreq(const struct rte_eth_dev *dev, int req, struct ifreq *ifr)
 186 {
 187         int sock = socket(PF_INET, SOCK_DGRAM, IPPROTO_IP);
 188         int ret = 0;
 189
 190         if (sock == -1) {
 191                 rte_errno = errno;
 192                 return -rte_errno;
 193         }
 194         ret = mlx5_get_ifname(dev, &ifr->ifr_name);
 195         if (ret)
 196                 goto error;
 197         ret = ioctl(sock, req, ifr);
 198         if (ret == -1) {
 199                 rte_errno = errno;
 200                 goto error;
 201         }
 202         close(sock);
 203         return 0;
 204 error:
 205         close(sock);
 206         return -rte_errno;
 207 }
 208
 209 /**
 210  * Get device MTU.
 211  *
 212  * @param dev
 213  *   Pointer to Ethernet device.
 214  * @param[out] mtu
 215  *   MTU value output buffer.
 216  *
 217  * @return
 218  *   0 on success, a negative errno value otherwise and rte_errno is set.
 219  */
 220 int
 221 mlx5_get_mtu(struct rte_eth_dev *dev, uint16_t *mtu)
 222 {
 223         struct ifreq request;
 224         int ret = mlx5_ifreq(dev, SIOCGIFMTU, &request);
 225
 226         if (ret)
 227                 return ret;
 228         *mtu = request.ifr_mtu;
 229         return 0;
 230 }
 231
 232 /**
 233  * Set device MTU.
 234  *
 235  * @param dev
 236  *   Pointer to Ethernet device.
 237  * @param mtu
 238  *   MTU value to set.
 239  *
 240  * @return
 241  *   0 on success, a negative errno value otherwise and rte_errno is set.
 242  */
 243 int
 244 mlx5_set_mtu(struct rte_eth_dev *dev, uint16_t mtu)
 245 {
 246         struct ifreq request = { .ifr_mtu = mtu, };
 247
 248         return mlx5_ifreq(dev, SIOCSIFMTU, &request);
 249 }
 250
 251 /**
 252  * Set device flags.
 253  *
 254  * @param dev
 255  *   Pointer to Ethernet device.
 256  * @param keep
 257  *   Bitmask for flags that must remain untouched.
 258  * @param flags
 259  *   Bitmask for flags to modify.
 260  *
 261  * @return
 262  *   0 on success, a negative errno value otherwise and rte_errno is set.
 263  */
 264 static int
 265 mlx5_set_flags(struct rte_eth_dev *dev, unsigned int keep, unsigned int flags)
 266 {
 267         struct ifreq request;
 268         int ret = mlx5_ifreq(dev, SIOCGIFFLAGS, &request);
 269
 270         if (ret)
 271                 return ret;
 272         request.ifr_flags &= keep;
 273         request.ifr_flags |= flags & ~keep;
 274         return mlx5_ifreq(dev, SIOCSIFFLAGS, &request);
 275 }
 276
 277 /**
 278  * Get device current raw clock counter
 279  *
 280  * @param dev
 281  *   Pointer to Ethernet device structure.
 282  * @param[out] time
 283  *   Current raw clock counter of the device.
 284  *
 285  * @return
 286  *   0 if the clock has correctly been read
 287  *   The value of errno in case of error
 288  */
 289 int
 290 mlx5_read_clock(struct rte_eth_dev *dev, uint64_t *clock)
 291 {
 292         struct mlx5_priv *priv = dev->data->dev_private;
 293         struct ibv_context *ctx = priv->sh->ctx;
 294         struct ibv_values_ex values;
 295         int err = 0;
 296
 297         values.comp_mask = IBV_VALUES_MASK_RAW_CLOCK;
 298         err = mlx5_glue->query_rt_values_ex(ctx, &values);
 299         if (err != 0) {
 300                 DRV_LOG(WARNING, "Could not query the clock !");
 301                 return err;
 302         }
 303         *clock = values.raw_clock.tv_nsec;
 304         return 0;
 305 }
 306
 307 /**
 308  * Retrieve the master device for representor in the same switch domain.
 309  *
 310  * @param dev
 311  *   Pointer to representor Ethernet device structure.
 312  *
 313  * @return
 314  *   Master device structure  on success, NULL otherwise.
 315  */
 316 static struct rte_eth_dev *
 317 mlx5_find_master_dev(struct rte_eth_dev *dev)
 318 {
 319         struct mlx5_priv *priv;
 320         uint16_t port_id;
 321         uint16_t domain_id;
 322
 323         priv = dev->data->dev_private;
 324         domain_id = priv->domain_id;
 325         MLX5_ASSERT(priv->representor);
 326         MLX5_ETH_FOREACH_DEV(port_id, priv->pci_dev) {
 327                 struct mlx5_priv *opriv =
 328                         rte_eth_devices[port_id].data->dev_private;
 329                 if (opriv &&
 330                     opriv->master &&
 331                     opriv->domain_id == domain_id &&
 332                     opriv->sh == priv->sh)
 333                         return &rte_eth_devices[port_id];
 334         }
 335         return NULL;
 336 }
 337
 338 /**
 339  * DPDK callback to retrieve physical link information.
 340  *
 341  * @param dev
 342  *   Pointer to Ethernet device structure.
 343  * @param[out] link
 344  *   Storage for current link status.
 345  *
 346  * @return
 347  *   0 on success, a negative errno value otherwise and rte_errno is set.
 348  */
 349 static int
 350 mlx5_link_update_unlocked_gset(struct rte_eth_dev *dev,
 351                                struct rte_eth_link *link)
 352 {
 353         struct mlx5_priv *priv = dev->data->dev_private;
 354         struct ethtool_cmd edata = {
 355                 .cmd = ETHTOOL_GSET /* Deprecated since Linux v4.5. */
 356         };
 357         struct ifreq ifr;
 358         struct rte_eth_link dev_link;
 359         int link_speed = 0;
 360         int ret;
 361
 362         ret = mlx5_ifreq(dev, SIOCGIFFLAGS, &ifr);
 363         if (ret) {
 364                 DRV_LOG(WARNING, "port %u ioctl(SIOCGIFFLAGS) failed: %s",
 365                         dev->data->port_id, strerror(rte_errno));
 366                 return ret;
 367         }
 368         dev_link = (struct rte_eth_link) {
 369                 .link_status = ((ifr.ifr_flags & IFF_UP) &&
 370                                 (ifr.ifr_flags & IFF_RUNNING)),
 371         };
 372         ifr = (struct ifreq) {
 373                 .ifr_data = (void *)&edata,
 374         };
 375         ret = mlx5_ifreq(dev, SIOCETHTOOL, &ifr);
 376         if (ret) {
 377                 if (ret == -ENOTSUP && priv->representor) {
 378                         struct rte_eth_dev *master;
 379
 380                         /*
 381                          * For representors we can try to inherit link
 382                          * settings from the master device. Actually
 383                          * link settings do not make a lot of sense
 384                          * for representors due to missing physical
 385                          * link. The old kernel drivers supported
 386                          * emulated settings query for representors,
 387                          * the new ones do not, so we have to add
 388                          * this code for compatibility issues.
 389                          */
 390                         master = mlx5_find_master_dev(dev);
 391                         if (master) {
 392                                 ifr = (struct ifreq) {
 393                                         .ifr_data = (void *)&edata,
 394                                 };
 395                                 ret = mlx5_ifreq(master, SIOCETHTOOL, &ifr);
 396                         }
 397                 }
 398                 if (ret) {
 399                         DRV_LOG(WARNING,
 400                                 "port %u ioctl(SIOCETHTOOL,"
 401                                 " ETHTOOL_GSET) failed: %s",
 402                                 dev->data->port_id, strerror(rte_errno));
 403                         return ret;
 404                 }
 405         }
 406         link_speed = ethtool_cmd_speed(&edata);
 407         if (link_speed == -1)
 408                 dev_link.link_speed = ETH_SPEED_NUM_NONE;
 409         else
 410                 dev_link.link_speed = link_speed;
 411         priv->link_speed_capa = 0;
 412         if (edata.supported & SUPPORTED_Autoneg)
 413                 priv->link_speed_capa |= ETH_LINK_SPEED_AUTONEG;
 414         if (edata.supported & (SUPPORTED_1000baseT_Full |
 415                                SUPPORTED_1000baseKX_Full))
 416                 priv->link_speed_capa |= ETH_LINK_SPEED_1G;
 417         if (edata.supported & SUPPORTED_10000baseKR_Full)
 418                 priv->link_speed_capa |= ETH_LINK_SPEED_10G;
 419         if (edata.supported & (SUPPORTED_40000baseKR4_Full |
 420                                SUPPORTED_40000baseCR4_Full |
 421                                SUPPORTED_40000baseSR4_Full |
 422                                SUPPORTED_40000baseLR4_Full))
 423                 priv->link_speed_capa |= ETH_LINK_SPEED_40G;
 424         dev_link.link_duplex = ((edata.duplex == DUPLEX_HALF) ?
 425                                 ETH_LINK_HALF_DUPLEX : ETH_LINK_FULL_DUPLEX);
 426         dev_link.link_autoneg = !(dev->data->dev_conf.link_speeds &
 427                         ETH_LINK_SPEED_FIXED);
 428         if (((dev_link.link_speed && !dev_link.link_status) ||
 429              (!dev_link.link_speed && dev_link.link_status))) {
 430                 rte_errno = EAGAIN;
 431                 return -rte_errno;
 432         }
 433         *link = dev_link;
 434         return 0;
 435 }
 436
 437 /**
 438  * Retrieve physical link information (unlocked version using new ioctl).
 439  *
 440  * @param dev
 441  *   Pointer to Ethernet device structure.
 442  * @param[out] link
 443  *   Storage for current link status.
 444  *
 445  * @return
 446  *   0 on success, a negative errno value otherwise and rte_errno is set.
 447  */
 448 static int
 449 mlx5_link_update_unlocked_gs(struct rte_eth_dev *dev,
 450                              struct rte_eth_link *link)
 451
 452 {
 453         struct mlx5_priv *priv = dev->data->dev_private;
 454         struct ethtool_link_settings gcmd = { .cmd = ETHTOOL_GLINKSETTINGS };
 455         struct ifreq ifr;
 456         struct rte_eth_link dev_link;
 457         struct rte_eth_dev *master = NULL;
 458         uint64_t sc;
 459         int ret;
 460
 461         ret = mlx5_ifreq(dev, SIOCGIFFLAGS, &ifr);
 462         if (ret) {
 463                 DRV_LOG(WARNING, "port %u ioctl(SIOCGIFFLAGS) failed: %s",
 464                         dev->data->port_id, strerror(rte_errno));
 465                 return ret;
 466         }
 467         dev_link = (struct rte_eth_link) {
 468                 .link_status = ((ifr.ifr_flags & IFF_UP) &&
 469                                 (ifr.ifr_flags & IFF_RUNNING)),
 470         };
 471         ifr = (struct ifreq) {
 472                 .ifr_data = (void *)&gcmd,
 473         };
 474         ret = mlx5_ifreq(dev, SIOCETHTOOL, &ifr);
 475         if (ret) {
 476                 if (ret == -ENOTSUP && priv->representor) {
 477                         /*
 478                          * For representors we can try to inherit link
 479                          * settings from the master device. Actually
 480                          * link settings do not make a lot of sense
 481                          * for representors due to missing physical
 482                          * link. The old kernel drivers supported
 483                          * emulated settings query for representors,
 484                          * the new ones do not, so we have to add
 485                          * this code for compatibility issues.
 486                          */
 487                         master = mlx5_find_master_dev(dev);
 488                         if (master) {
 489                                 ifr = (struct ifreq) {
 490                                         .ifr_data = (void *)&gcmd,
 491                                 };
 492                                 ret = mlx5_ifreq(master, SIOCETHTOOL, &ifr);
 493                         }
 494                 }
 495                 if (ret) {
 496                         DRV_LOG(DEBUG,
 497                                 "port %u ioctl(SIOCETHTOOL,"
 498                                 " ETHTOOL_GLINKSETTINGS) failed: %s",
 499                                 dev->data->port_id, strerror(rte_errno));
 500                         return ret;
 501                 }
 502         }
 503         gcmd.link_mode_masks_nwords = -gcmd.link_mode_masks_nwords;
 504
 505         alignas(struct ethtool_link_settings)
 506         uint8_t data[offsetof(struct ethtool_link_settings, link_mode_masks) +
 507                      sizeof(uint32_t) * gcmd.link_mode_masks_nwords * 3];
 508         struct ethtool_link_settings *ecmd = (void *)data;
 509
 510         *ecmd = gcmd;
 511         ifr.ifr_data = (void *)ecmd;
 512         ret = mlx5_ifreq(master ? master : dev, SIOCETHTOOL, &ifr);
 513         if (ret) {
 514                 DRV_LOG(DEBUG,
 515                         "port %u ioctl(SIOCETHTOOL,"
 516                         "ETHTOOL_GLINKSETTINGS) failed: %s",
 517                         dev->data->port_id, strerror(rte_errno));
 518                 return ret;
 519         }
 520         dev_link.link_speed = (ecmd->speed == UINT32_MAX) ? ETH_SPEED_NUM_NONE :
 521                                                             ecmd->speed;
 522         sc = ecmd->link_mode_masks[0] |
 523                 ((uint64_t)ecmd->link_mode_masks[1] << 32);
 524         priv->link_speed_capa = 0;
 525         if (sc & MLX5_BITSHIFT(ETHTOOL_LINK_MODE_Autoneg_BIT))
 526                 priv->link_speed_capa |= ETH_LINK_SPEED_AUTONEG;
 527         if (sc & (MLX5_BITSHIFT(ETHTOOL_LINK_MODE_1000baseT_Full_BIT) |
 528                   MLX5_BITSHIFT(ETHTOOL_LINK_MODE_1000baseKX_Full_BIT)))
 529                 priv->link_speed_capa |= ETH_LINK_SPEED_1G;
 530         if (sc & (MLX5_BITSHIFT(ETHTOOL_LINK_MODE_10000baseKX4_Full_BIT) |
 531                   MLX5_BITSHIFT(ETHTOOL_LINK_MODE_10000baseKR_Full_BIT) |
 532                   MLX5_BITSHIFT(ETHTOOL_LINK_MODE_10000baseR_FEC_BIT)))
 533                 priv->link_speed_capa |= ETH_LINK_SPEED_10G;
 534         if (sc & (MLX5_BITSHIFT(ETHTOOL_LINK_MODE_20000baseMLD2_Full_BIT) |
 535                   MLX5_BITSHIFT(ETHTOOL_LINK_MODE_20000baseKR2_Full_BIT)))
 536                 priv->link_speed_capa |= ETH_LINK_SPEED_20G;
 537         if (sc & (MLX5_BITSHIFT(ETHTOOL_LINK_MODE_40000baseKR4_Full_BIT) |
 538                   MLX5_BITSHIFT(ETHTOOL_LINK_MODE_40000baseCR4_Full_BIT) |
 539                   MLX5_BITSHIFT(ETHTOOL_LINK_MODE_40000baseSR4_Full_BIT) |
 540                   MLX5_BITSHIFT(ETHTOOL_LINK_MODE_40000baseLR4_Full_BIT)))
 541                 priv->link_speed_capa |= ETH_LINK_SPEED_40G;
 542         if (sc & (MLX5_BITSHIFT(ETHTOOL_LINK_MODE_56000baseKR4_Full_BIT) |
 543                   MLX5_BITSHIFT(ETHTOOL_LINK_MODE_56000baseCR4_Full_BIT) |
 544                   MLX5_BITSHIFT(ETHTOOL_LINK_MODE_56000baseSR4_Full_BIT) |
 545                   MLX5_BITSHIFT(ETHTOOL_LINK_MODE_56000baseLR4_Full_BIT)))
 546                 priv->link_speed_capa |= ETH_LINK_SPEED_56G;
 547         if (sc & (MLX5_BITSHIFT(ETHTOOL_LINK_MODE_25000baseCR_Full_BIT) |
 548                   MLX5_BITSHIFT(ETHTOOL_LINK_MODE_25000baseKR_Full_BIT) |
 549                   MLX5_BITSHIFT(ETHTOOL_LINK_MODE_25000baseSR_Full_BIT)))
 550                 priv->link_speed_capa |= ETH_LINK_SPEED_25G;
 551         if (sc & (MLX5_BITSHIFT(ETHTOOL_LINK_MODE_50000baseCR2_Full_BIT) |
 552                   MLX5_BITSHIFT(ETHTOOL_LINK_MODE_50000baseKR2_Full_BIT)))
 553                 priv->link_speed_capa |= ETH_LINK_SPEED_50G;
 554         if (sc & (MLX5_BITSHIFT(ETHTOOL_LINK_MODE_100000baseKR4_Full_BIT) |
 555                   MLX5_BITSHIFT(ETHTOOL_LINK_MODE_100000baseSR4_Full_BIT) |
 556                   MLX5_BITSHIFT(ETHTOOL_LINK_MODE_100000baseCR4_Full_BIT) |
 557                   MLX5_BITSHIFT(ETHTOOL_LINK_MODE_100000baseLR4_ER4_Full_BIT)))
 558                 priv->link_speed_capa |= ETH_LINK_SPEED_100G;
 559         if (sc & (MLX5_BITSHIFT(ETHTOOL_LINK_MODE_200000baseKR4_Full_BIT) |
 560                   MLX5_BITSHIFT(ETHTOOL_LINK_MODE_200000baseSR4_Full_BIT)))
 561                 priv->link_speed_capa |= ETH_LINK_SPEED_200G;
 562
 563         sc = ecmd->link_mode_masks[2] |
 564                 ((uint64_t)ecmd->link_mode_masks[3] << 32);
 565         if (sc & (MLX5_BITSHIFT(ETHTOOL_LINK_MODE_200000baseCR4_Full_BIT) |
 566                   MLX5_BITSHIFT
 567                        (ETHTOOL_LINK_MODE_200000baseLR4_ER4_FR4_Full_BIT) |
 568                   MLX5_BITSHIFT(ETHTOOL_LINK_MODE_200000baseDR4_Full_BIT)))
 569                 priv->link_speed_capa |= ETH_LINK_SPEED_200G;
 570         dev_link.link_duplex = ((ecmd->duplex == DUPLEX_HALF) ?
 571                                 ETH_LINK_HALF_DUPLEX : ETH_LINK_FULL_DUPLEX);
 572         dev_link.link_autoneg = !(dev->data->dev_conf.link_speeds &
 573                                   ETH_LINK_SPEED_FIXED);
 574         if (((dev_link.link_speed && !dev_link.link_status) ||
 575              (!dev_link.link_speed && dev_link.link_status))) {
 576                 rte_errno = EAGAIN;
 577                 return -rte_errno;
 578         }
 579         *link = dev_link;
 580         return 0;
 581 }
 582
 583 /**
 584  * DPDK callback to retrieve physical link information.
 585  *
 586  * @param dev
 587  *   Pointer to Ethernet device structure.
 588  * @param wait_to_complete
 589  *   Wait for request completion.
 590  *
 591  * @return
 592  *   0 if link status was not updated, positive if it was, a negative errno
 593  *   value otherwise and rte_errno is set.
 594  */
 595 int
 596 mlx5_link_update(struct rte_eth_dev *dev, int wait_to_complete)
 597 {
 598         int ret;
 599         struct rte_eth_link dev_link;
 600         time_t start_time = time(NULL);
 601         int retry = MLX5_GET_LINK_STATUS_RETRY_COUNT;
 602
 603         do {
 604                 ret = mlx5_link_update_unlocked_gs(dev, &dev_link);
 605                 if (ret == -ENOTSUP)
 606                         ret = mlx5_link_update_unlocked_gset(dev, &dev_link);
 607                 if (ret == 0)
 608                         break;
 609                 /* Handle wait to complete situation. */
 610                 if ((wait_to_complete || retry) && ret == -EAGAIN) {
 611                         if (abs((int)difftime(time(NULL), start_time)) <
 612                             MLX5_LINK_STATUS_TIMEOUT) {
 613                                 usleep(0);
 614                                 continue;
 615                         } else {
 616                                 rte_errno = EBUSY;
 617                                 return -rte_errno;
 618                         }
 619                 } else if (ret < 0) {
 620                         return ret;
 621                 }
 622         } while (wait_to_complete || retry-- > 0);
 623         ret = !!memcmp(&dev->data->dev_link, &dev_link,
 624                        sizeof(struct rte_eth_link));
 625         dev->data->dev_link = dev_link;
 626         return ret;
 627 }
 628
 629 /**
 630  * DPDK callback to get flow control status.
 631  *
 632  * @param dev
 633  *   Pointer to Ethernet device structure.
 634  * @param[out] fc_conf
 635  *   Flow control output buffer.
 636  *
 637  * @return
 638  *   0 on success, a negative errno value otherwise and rte_errno is set.
 639  */
 640 int
 641 mlx5_dev_get_flow_ctrl(struct rte_eth_dev *dev, struct rte_eth_fc_conf *fc_conf)
 642 {
 643         struct ifreq ifr;
 644         struct ethtool_pauseparam ethpause = {
 645                 .cmd = ETHTOOL_GPAUSEPARAM
 646         };
 647         int ret;
 648
 649         ifr.ifr_data = (void *)&ethpause;
 650         ret = mlx5_ifreq(dev, SIOCETHTOOL, &ifr);
 651         if (ret) {
 652                 DRV_LOG(WARNING,
 653                         "port %u ioctl(SIOCETHTOOL, ETHTOOL_GPAUSEPARAM) failed:"
 654                         " %s",
 655                         dev->data->port_id, strerror(rte_errno));
 656                 return ret;
 657         }
 658         fc_conf->autoneg = ethpause.autoneg;
 659         if (ethpause.rx_pause && ethpause.tx_pause)
 660                 fc_conf->mode = RTE_FC_FULL;
 661         else if (ethpause.rx_pause)
 662                 fc_conf->mode = RTE_FC_RX_PAUSE;
 663         else if (ethpause.tx_pause)
 664                 fc_conf->mode = RTE_FC_TX_PAUSE;
 665         else
 666                 fc_conf->mode = RTE_FC_NONE;
 667         return 0;
 668 }
 669
 670 /**
 671  * DPDK callback to modify flow control parameters.
 672  *
 673  * @param dev
 674  *   Pointer to Ethernet device structure.
 675  * @param[in] fc_conf
 676  *   Flow control parameters.
 677  *
 678  * @return
 679  *   0 on success, a negative errno value otherwise and rte_errno is set.
 680  */
 681 int
 682 mlx5_dev_set_flow_ctrl(struct rte_eth_dev *dev, struct rte_eth_fc_conf *fc_conf)
 683 {
 684         struct ifreq ifr;
 685         struct ethtool_pauseparam ethpause = {
 686                 .cmd = ETHTOOL_SPAUSEPARAM
 687         };
 688         int ret;
 689
 690         ifr.ifr_data = (void *)&ethpause;
 691         ethpause.autoneg = fc_conf->autoneg;
 692         if (((fc_conf->mode & RTE_FC_FULL) == RTE_FC_FULL) ||
 693             (fc_conf->mode & RTE_FC_RX_PAUSE))
 694                 ethpause.rx_pause = 1;
 695         else
 696                 ethpause.rx_pause = 0;
 697
 698         if (((fc_conf->mode & RTE_FC_FULL) == RTE_FC_FULL) ||
 699             (fc_conf->mode & RTE_FC_TX_PAUSE))
 700                 ethpause.tx_pause = 1;
 701         else
 702                 ethpause.tx_pause = 0;
 703         ret = mlx5_ifreq(dev, SIOCETHTOOL, &ifr);
 704         if (ret) {
 705                 DRV_LOG(WARNING,
 706                         "port %u ioctl(SIOCETHTOOL, ETHTOOL_SPAUSEPARAM)"
 707                         " failed: %s",
 708                         dev->data->port_id, strerror(rte_errno));
 709                 return ret;
 710         }
 711         return 0;
 712 }
 713
 714 /**
 715  * Handle asynchronous removal event for entire multiport device.
 716  *
 717  * @param sh
 718  *   Infiniband device shared context.
 719  */
 720 static void
 721 mlx5_dev_interrupt_device_fatal(struct mlx5_dev_ctx_shared *sh)
 722 {
 723         uint32_t i;
 724
 725         for (i = 0; i < sh->max_port; ++i) {
 726                 struct rte_eth_dev *dev;
 727
 728                 if (sh->port[i].ih_port_id >= RTE_MAX_ETHPORTS) {
 729                         /*
 730                          * Or not existing port either no
 731                          * handler installed for this port.
 732                          */
 733                         continue;
 734                 }
 735                 dev = &rte_eth_devices[sh->port[i].ih_port_id];
 736                 MLX5_ASSERT(dev);
 737                 if (dev->data->dev_conf.intr_conf.rmv)
 738                         rte_eth_dev_callback_process
 739                                 (dev, RTE_ETH_EVENT_INTR_RMV, NULL);
 740         }
 741 }
 742
 743 /**
 744  * Handle shared asynchronous events the NIC (removal event
 745  * and link status change). Supports multiport IB device.
 746  *
 747  * @param cb_arg
 748  *   Callback argument.
 749  */
 750 void
 751 mlx5_dev_interrupt_handler(void *cb_arg)
 752 {
 753         struct mlx5_dev_ctx_shared *sh = cb_arg;
 754         struct ibv_async_event event;
 755
 756         /* Read all message from the IB device and acknowledge them. */
 757         for (;;) {
 758                 struct rte_eth_dev *dev;
 759                 uint32_t tmp;
 760
 761                 if (mlx5_glue->get_async_event(sh->ctx, &event))
 762                         break;
 763                 /* Retrieve and check IB port index. */
 764                 tmp = (uint32_t)event.element.port_num;
 765                 if (!tmp && event.event_type == IBV_EVENT_DEVICE_FATAL) {
 766                         /*
 767                          * The DEVICE_FATAL event is called once for
 768                          * entire device without port specifying.
 769                          * We should notify all existing ports.
 770                          */
 771                         mlx5_glue->ack_async_event(&event);
 772                         mlx5_dev_interrupt_device_fatal(sh);
 773                         continue;
 774                 }
 775                 MLX5_ASSERT(tmp && (tmp <= sh->max_port));
 776                 if (!tmp) {
 777                         /* Unsupported device level event. */
 778                         mlx5_glue->ack_async_event(&event);
 779                         DRV_LOG(DEBUG,
 780                                 "unsupported common event (type %d)",
 781                                 event.event_type);
 782                         continue;
 783                 }
 784                 if (tmp > sh->max_port) {
 785                         /* Invalid IB port index. */
 786                         mlx5_glue->ack_async_event(&event);
 787                         DRV_LOG(DEBUG,
 788                                 "cannot handle an event (type %d)"
 789                                 "due to invalid IB port index (%u)",
 790                                 event.event_type, tmp);
 791                         continue;
 792                 }
 793                 if (sh->port[tmp - 1].ih_port_id >= RTE_MAX_ETHPORTS) {
 794                         /* No handler installed. */
 795                         mlx5_glue->ack_async_event(&event);
 796                         DRV_LOG(DEBUG,
 797                                 "cannot handle an event (type %d)"
 798                                 "due to no handler installed for port %u",
 799                                 event.event_type, tmp);
 800                         continue;
 801                 }
 802                 /* Retrieve ethernet device descriptor. */
 803                 tmp = sh->port[tmp - 1].ih_port_id;
 804                 dev = &rte_eth_devices[tmp];
 805                 MLX5_ASSERT(dev);
 806                 if ((event.event_type == IBV_EVENT_PORT_ACTIVE ||
 807                      event.event_type == IBV_EVENT_PORT_ERR) &&
 808                         dev->data->dev_conf.intr_conf.lsc) {
 809                         mlx5_glue->ack_async_event(&event);
 810                         if (mlx5_link_update(dev, 0) == -EAGAIN) {
 811                                 usleep(0);
 812                                 continue;
 813                         }
 814                         rte_eth_dev_callback_process
 815                                 (dev, RTE_ETH_EVENT_INTR_LSC, NULL);
 816                         continue;
 817                 }
 818                 DRV_LOG(DEBUG,
 819                         "port %u cannot handle an unknown event (type %d)",
 820                         dev->data->port_id, event.event_type);
 821                 mlx5_glue->ack_async_event(&event);
 822         }
 823 }
 824
 825 /*
 826  * Unregister callback handler safely. The handler may be active
 827  * while we are trying to unregister it, in this case code -EAGAIN
 828  * is returned by rte_intr_callback_unregister(). This routine checks
 829  * the return code and tries to unregister handler again.
 830  *
 831  * @param handle
 832  *   interrupt handle
 833  * @param cb_fn
 834  *   pointer to callback routine
 835  * @cb_arg
 836  *   opaque callback parameter
 837  */
 838 void
 839 mlx5_intr_callback_unregister(const struct rte_intr_handle *handle,
 840                               rte_intr_callback_fn cb_fn, void *cb_arg)
 841 {
 842         /*
 843          * Try to reduce timeout management overhead by not calling
 844          * the timer related routines on the first iteration. If the
 845          * unregistering succeeds on first call there will be no
 846          * timer calls at all.
 847          */
 848         uint64_t twait = 0;
 849         uint64_t start = 0;
 850
 851         do {
 852                 int ret;
 853
 854                 ret = rte_intr_callback_unregister(handle, cb_fn, cb_arg);
 855                 if (ret >= 0)
 856                         return;
 857                 if (ret != -EAGAIN) {
 858                         DRV_LOG(INFO, "failed to unregister interrupt"
 859                                       " handler (error: %d)", ret);
 860                         MLX5_ASSERT(false);
 861                         return;
 862                 }
 863                 if (twait) {
 864                         struct timespec onems;
 865
 866                         /* Wait one millisecond and try again. */
 867                         onems.tv_sec = 0;
 868                         onems.tv_nsec = NS_PER_S / MS_PER_S;
 869                         nanosleep(&onems, 0);
 870                         /* Check whether one second elapsed. */
 871                         if ((rte_get_timer_cycles() - start) <= twait)
 872                                 continue;
 873                 } else {
 874                         /*
 875                          * We get the amount of timer ticks for one second.
 876                          * If this amount elapsed it means we spent one
 877                          * second in waiting. This branch is executed once
 878                          * on first iteration.
 879                          */
 880                         twait = rte_get_timer_hz();
 881                         MLX5_ASSERT(twait);
 882                 }
 883                 /*
 884                  * Timeout elapsed, show message (once a second) and retry.
 885                  * We have no other acceptable option here, if we ignore
 886                  * the unregistering return code the handler will not
 887                  * be unregistered, fd will be closed and we may get the
 888                  * crush. Hanging and messaging in the loop seems not to be
 889                  * the worst choice.
 890                  */
 891                 DRV_LOG(INFO, "Retrying to unregister interrupt handler");
 892                 start = rte_get_timer_cycles();
 893         } while (true);
 894 }
 895
 896 /**
 897  * Handle DEVX interrupts from the NIC.
 898  * This function is probably called from the DPDK host thread.
 899  *
 900  * @param cb_arg
 901  *   Callback argument.
 902  */
 903 void
 904 mlx5_dev_interrupt_handler_devx(void *cb_arg)
 905 {
 906 #ifndef HAVE_IBV_DEVX_ASYNC
 907         (void)cb_arg;
 908         return;
 909 #else
 910         struct mlx5_dev_ctx_shared *sh = cb_arg;
 911         union {
 912                 struct mlx5dv_devx_async_cmd_hdr cmd_resp;
 913                 uint8_t buf[MLX5_ST_SZ_BYTES(query_flow_counter_out) +
 914                             MLX5_ST_SZ_BYTES(traffic_counter) +
 915                             sizeof(struct mlx5dv_devx_async_cmd_hdr)];
 916         } out;
 917         uint8_t *buf = out.buf + sizeof(out.cmd_resp);
 918
 919         while (!mlx5_glue->devx_get_async_cmd_comp(sh->devx_comp,
 920                                                    &out.cmd_resp,
 921                                                    sizeof(out.buf)))
 922                 mlx5_flow_async_pool_query_handle
 923                         (sh, (uint64_t)out.cmd_resp.wr_id,
 924                          mlx5_devx_get_out_command_status(buf));
 925 #endif /* HAVE_IBV_DEVX_ASYNC */
 926 }
 927
 928 /**
 929  * DPDK callback to bring the link DOWN.
 930  *
 931  * @param dev
 932  *   Pointer to Ethernet device structure.
 933  *
 934  * @return
 935  *   0 on success, a negative errno value otherwise and rte_errno is set.
 936  */
 937 int
 938 mlx5_set_link_down(struct rte_eth_dev *dev)
 939 {
 940         return mlx5_set_flags(dev, ~IFF_UP, ~IFF_UP);
 941 }
 942
 943 /**
 944  * DPDK callback to bring the link UP.
 945  *
 946  * @param dev
 947  *   Pointer to Ethernet device structure.
 948  *
 949  * @return
 950  *   0 on success, a negative errno value otherwise and rte_errno is set.
 951  */
 952 int
 953 mlx5_set_link_up(struct rte_eth_dev *dev)
 954 {
 955         return mlx5_set_flags(dev, ~IFF_UP, IFF_UP);
 956 }
 957
 958 /**
 959  * Check if mlx5 device was removed.
 960  *
 961  * @param dev
 962  *   Pointer to Ethernet device structure.
 963  *
 964  * @return
 965  *   1 when device is removed, otherwise 0.
 966  */
 967 int
 968 mlx5_is_removed(struct rte_eth_dev *dev)
 969 {
 970         struct ibv_device_attr device_attr;
 971         struct mlx5_priv *priv = dev->data->dev_private;
 972
 973         if (mlx5_glue->query_device(priv->sh->ctx, &device_attr) == EIO)
 974                 return 1;
 975         return 0;
 976 }
 977
 978 /**
 979  * Analyze gathered port parameters via sysfs to recognize master
 980  * and representor devices for E-Switch configuration.
 981  *
 982  * @param[in] device_dir
 983  *   flag of presence of "device" directory under port device key.
 984  * @param[inout] switch_info
 985  *   Port information, including port name as a number and port name
 986  *   type if recognized
 987  *
 988  * @return
 989  *   master and representor flags are set in switch_info according to
 990  *   recognized parameters (if any).
 991  */
 992 static void
 993 mlx5_sysfs_check_switch_info(bool device_dir,
 994                              struct mlx5_switch_info *switch_info)
 995 {
 996         switch (switch_info->name_type) {
 997         case MLX5_PHYS_PORT_NAME_TYPE_UNKNOWN:
 998                 /*
 999                  * Name is not recognized, assume the master,
1000                  * check the device directory presence.
1001                  */
1002                 switch_info->master = device_dir;
1003                 break;
1004         case MLX5_PHYS_PORT_NAME_TYPE_NOTSET:
1005                 /*
1006                  * Name is not set, this assumes the legacy naming
1007                  * schema for master, just check if there is
1008                  * a device directory.
1009                  */
1010                 switch_info->master = device_dir;
1011                 break;
1012         case MLX5_PHYS_PORT_NAME_TYPE_UPLINK:
1013                 /* New uplink naming schema recognized. */
1014                 switch_info->master = 1;
1015                 break;
1016         case MLX5_PHYS_PORT_NAME_TYPE_LEGACY:
1017                 /* Legacy representors naming schema. */
1018                 switch_info->representor = !device_dir;
1019                 break;
1020         case MLX5_PHYS_PORT_NAME_TYPE_PFHPF:
1021                 /* Fallthrough */
1022         case MLX5_PHYS_PORT_NAME_TYPE_PFVF:
1023                 /* New representors naming schema. */
1024                 switch_info->representor = 1;
1025                 break;
1026         }
1027 }
1028
1029 /**
1030  * Get switch information associated with network interface.
1031  *
1032  * @param ifindex
1033  *   Network interface index.
1034  * @param[out] info
1035  *   Switch information object, populated in case of success.
1036  *
1037  * @return
1038  *   0 on success, a negative errno value otherwise and rte_errno is set.
1039  */
1040 int
1041 mlx5_sysfs_switch_info(unsigned int ifindex, struct mlx5_switch_info *info)
1042 {
1043         char ifname[IF_NAMESIZE];
1044         char port_name[IF_NAMESIZE];
1045         FILE *file;
1046         struct mlx5_switch_info data = {
1047                 .master = 0,
1048                 .representor = 0,
1049                 .name_type = MLX5_PHYS_PORT_NAME_TYPE_NOTSET,
1050                 .port_name = 0,
1051                 .switch_id = 0,
1052         };
1053         DIR *dir;
1054         bool port_switch_id_set = false;
1055         bool device_dir = false;
1056         char c;
1057         int ret;
1058
1059         if (!if_indextoname(ifindex, ifname)) {
1060                 rte_errno = errno;
1061                 return -rte_errno;
1062         }
1063
1064         MKSTR(phys_port_name, "/sys/class/net/%s/phys_port_name",
1065               ifname);
1066         MKSTR(phys_switch_id, "/sys/class/net/%s/phys_switch_id",
1067               ifname);
1068         MKSTR(pci_device, "/sys/class/net/%s/device",
1069               ifname);
1070
1071         file = fopen(phys_port_name, "rb");
1072         if (file != NULL) {
1073                 ret = fscanf(file, "%" RTE_STR(IF_NAMESIZE) "s", port_name);
1074                 fclose(file);
1075                 if (ret == 1)
1076                         mlx5_translate_port_name(port_name, &data);
1077         }
1078         file = fopen(phys_switch_id, "rb");
1079         if (file == NULL) {
1080                 rte_errno = errno;
1081                 return -rte_errno;
1082         }
1083         port_switch_id_set =
1084                 fscanf(file, "%" SCNx64 "%c", &data.switch_id, &c) == 2 &&
1085                 c == '\n';
1086         fclose(file);
1087         dir = opendir(pci_device);
1088         if (dir != NULL) {
1089                 closedir(dir);
1090                 device_dir = true;
1091         }
1092         if (port_switch_id_set) {
1093                 /* We have some E-Switch configuration. */
1094                 mlx5_sysfs_check_switch_info(device_dir, &data);
1095         }
1096         *info = data;
1097         MLX5_ASSERT(!(data.master && data.representor));
1098         if (data.master && data.representor) {
1099                 DRV_LOG(ERR, "ifindex %u device is recognized as master"
1100                              " and as representor", ifindex);
1101                 rte_errno = ENODEV;
1102                 return -rte_errno;
1103         }
1104         return 0;
1105 }
1106
1107 /**
1108  * Get bond information associated with network interface.
1109  *
1110  * @param pf_ifindex
1111  *   Network interface index of bond slave interface
1112  * @param[out] ifindex
1113  *   Pointer to bond ifindex.
1114  * @param[out] ifname
1115  *   Pointer to bond ifname.
1116  *
1117  * @return
1118  *   0 on success, a negative errno value otherwise and rte_errno is set.
1119  */
1120 int
1121 mlx5_sysfs_bond_info(unsigned int pf_ifindex, unsigned int *ifindex,
1122                      char *ifname)
1123 {
1124         char name[IF_NAMESIZE];
1125         FILE *file;
1126         unsigned int index;
1127         int ret;
1128
1129         if (!if_indextoname(pf_ifindex, name) || !strlen(name)) {
1130                 rte_errno = errno;
1131                 return -rte_errno;
1132         }
1133         MKSTR(bond_if, "/sys/class/net/%s/master/ifindex", name);
1134         /* read bond ifindex */
1135         file = fopen(bond_if, "rb");
1136         if (file == NULL) {
1137                 rte_errno = errno;
1138                 return -rte_errno;
1139         }
1140         ret = fscanf(file, "%u", &index);
1141         fclose(file);
1142         if (ret <= 0) {
1143                 rte_errno = errno;
1144                 return -rte_errno;
1145         }
1146         if (ifindex)
1147                 *ifindex = index;
1148
1149         /* read bond device name from symbol link */
1150         if (ifname) {
1151                 if (!if_indextoname(index, ifname)) {
1152                         rte_errno = errno;
1153                         return -rte_errno;
1154                 }
1155         }
1156         return 0;
1157 }
1158
1159 /**
1160  * DPDK callback to retrieve plug-in module EEPROM information (type and size).
1161  *
1162  * @param dev
1163  *   Pointer to Ethernet device structure.
1164  * @param[out] modinfo
1165  *   Storage for plug-in module EEPROM information.
1166  *
1167  * @return
1168  *   0 on success, a negative errno value otherwise and rte_errno is set.
1169  */
1170 int
1171 mlx5_get_module_info(struct rte_eth_dev *dev,
1172                      struct rte_eth_dev_module_info *modinfo)
1173 {
1174         struct ethtool_modinfo info = {
1175                 .cmd = ETHTOOL_GMODULEINFO,
1176         };
1177         struct ifreq ifr = (struct ifreq) {
1178                 .ifr_data = (void *)&info,
1179         };
1180         int ret = 0;
1181
1182         if (!dev || !modinfo) {
1183                 DRV_LOG(WARNING, "missing argument, cannot get module info");
1184                 rte_errno = EINVAL;
1185                 return -rte_errno;
1186         }
1187         ret = mlx5_ifreq(dev, SIOCETHTOOL, &ifr);
1188         if (ret) {
1189                 DRV_LOG(WARNING, "port %u ioctl(SIOCETHTOOL) failed: %s",
1190                         dev->data->port_id, strerror(rte_errno));
1191                 return ret;
1192         }
1193         modinfo->type = info.type;
1194         modinfo->eeprom_len = info.eeprom_len;
1195         return ret;
1196 }
1197
1198 /**
1199  * DPDK callback to retrieve plug-in module EEPROM data.
1200  *
1201  * @param dev
1202  *   Pointer to Ethernet device structure.
1203  * @param[out] info
1204  *   Storage for plug-in module EEPROM data.
1205  *
1206  * @return
1207  *   0 on success, a negative errno value otherwise and rte_errno is set.
1208  */
1209 int mlx5_get_module_eeprom(struct rte_eth_dev *dev,
1210                            struct rte_dev_eeprom_info *info)
1211 {
1212         struct ethtool_eeprom *eeprom;
1213         struct ifreq ifr;
1214         int ret = 0;
1215
1216         if (!dev || !info) {
1217                 DRV_LOG(WARNING, "missing argument, cannot get module eeprom");
1218                 rte_errno = EINVAL;
1219                 return -rte_errno;
1220         }
1221         eeprom = mlx5_malloc(MLX5_MEM_ZERO,
1222                              (sizeof(struct ethtool_eeprom) + info->length), 0,
1223                              SOCKET_ID_ANY);
1224         if (!eeprom) {
1225                 DRV_LOG(WARNING, "port %u cannot allocate memory for "
1226                         "eeprom data", dev->data->port_id);
1227                 rte_errno = ENOMEM;
1228                 return -rte_errno;
1229         }
1230         eeprom->cmd = ETHTOOL_GMODULEEEPROM;
1231         eeprom->offset = info->offset;
1232         eeprom->len = info->length;
1233         ifr = (struct ifreq) {
1234                 .ifr_data = (void *)eeprom,
1235         };
1236         ret = mlx5_ifreq(dev, SIOCETHTOOL, &ifr);
1237         if (ret)
1238                 DRV_LOG(WARNING, "port %u ioctl(SIOCETHTOOL) failed: %s",
1239                         dev->data->port_id, strerror(rte_errno));
1240         else
1241                 rte_memcpy(info->data, eeprom->data, info->length);
1242         mlx5_free(eeprom);
1243         return ret;
1244 }
1245
1246 /**
1247  * Read device counters table.
1248  *
1249  * @param dev
1250  *   Pointer to Ethernet device.
1251  * @param[out] stats
1252  *   Counters table output buffer.
1253  *
1254  * @return
1255  *   0 on success and stats is filled, negative errno value otherwise and
1256  *   rte_errno is set.
1257  */
1258 int
1259 mlx5_os_read_dev_counters(struct rte_eth_dev *dev, uint64_t *stats)
1260 {
1261         struct mlx5_priv *priv = dev->data->dev_private;
1262         struct mlx5_xstats_ctrl *xstats_ctrl = &priv->xstats_ctrl;
1263         unsigned int i;
1264         struct ifreq ifr;
1265         unsigned int stats_sz = xstats_ctrl->stats_n * sizeof(uint64_t);
1266         unsigned char et_stat_buf[sizeof(struct ethtool_stats) + stats_sz];
1267         struct ethtool_stats *et_stats = (struct ethtool_stats *)et_stat_buf;
1268         int ret;
1269
1270         et_stats->cmd = ETHTOOL_GSTATS;
1271         et_stats->n_stats = xstats_ctrl->stats_n;
1272         ifr.ifr_data = (caddr_t)et_stats;
1273         ret = mlx5_ifreq(dev, SIOCETHTOOL, &ifr);
1274         if (ret) {
1275                 DRV_LOG(WARNING,
1276                         "port %u unable to read statistic values from device",
1277                         dev->data->port_id);
1278                 return ret;
1279         }
1280         for (i = 0; i != xstats_ctrl->mlx5_stats_n; ++i) {
1281                 if (xstats_ctrl->info[i].dev) {
1282                         ret = mlx5_os_read_dev_stat(priv,
1283                                             xstats_ctrl->info[i].ctr_name,
1284                                             &stats[i]);
1285                         /* return last xstats counter if fail to read. */
1286                         if (ret == 0)
1287                                 xstats_ctrl->xstats[i] = stats[i];
1288                         else
1289                                 stats[i] = xstats_ctrl->xstats[i];
1290                 } else {
1291                         stats[i] = (uint64_t)
1292                                 et_stats->data[xstats_ctrl->dev_table_idx[i]];
1293                 }
1294         }
1295         return 0;
1296 }
1297
1298 /**
1299  * Query the number of statistics provided by ETHTOOL.
1300  *
1301  * @param dev
1302  *   Pointer to Ethernet device.
1303  *
1304  * @return
1305  *   Number of statistics on success, negative errno value otherwise and
1306  *   rte_errno is set.
1307  */
1308 int
1309 mlx5_os_get_stats_n(struct rte_eth_dev *dev)
1310 {
1311         struct ethtool_drvinfo drvinfo;
1312         struct ifreq ifr;
1313         int ret;
1314
1315         drvinfo.cmd = ETHTOOL_GDRVINFO;
1316         ifr.ifr_data = (caddr_t)&drvinfo;
1317         ret = mlx5_ifreq(dev, SIOCETHTOOL, &ifr);
1318         if (ret) {
1319                 DRV_LOG(WARNING, "port %u unable to query number of statistics",
1320                         dev->data->port_id);
1321                 return ret;
1322         }
1323         return drvinfo.n_stats;
1324 }
1325
1326 static const struct mlx5_counter_ctrl mlx5_counters_init[] = {
1327         {
1328                 .dpdk_name = "rx_port_unicast_bytes",
1329                 .ctr_name = "rx_vport_unicast_bytes",
1330         },
1331         {
1332                 .dpdk_name = "rx_port_multicast_bytes",
1333                 .ctr_name = "rx_vport_multicast_bytes",
1334         },
1335         {
1336                 .dpdk_name = "rx_port_broadcast_bytes",
1337                 .ctr_name = "rx_vport_broadcast_bytes",
1338         },
1339         {
1340                 .dpdk_name = "rx_port_unicast_packets",
1341                 .ctr_name = "rx_vport_unicast_packets",
1342         },
1343         {
1344                 .dpdk_name = "rx_port_multicast_packets",
1345                 .ctr_name = "rx_vport_multicast_packets",
1346         },
1347         {
1348                 .dpdk_name = "rx_port_broadcast_packets",
1349                 .ctr_name = "rx_vport_broadcast_packets",
1350         },
1351         {
1352                 .dpdk_name = "tx_port_unicast_bytes",
1353                 .ctr_name = "tx_vport_unicast_bytes",
1354         },
1355         {
1356                 .dpdk_name = "tx_port_multicast_bytes",
1357                 .ctr_name = "tx_vport_multicast_bytes",
1358         },
1359         {
1360                 .dpdk_name = "tx_port_broadcast_bytes",
1361                 .ctr_name = "tx_vport_broadcast_bytes",
1362         },
1363         {
1364                 .dpdk_name = "tx_port_unicast_packets",
1365                 .ctr_name = "tx_vport_unicast_packets",
1366         },
1367         {
1368                 .dpdk_name = "tx_port_multicast_packets",
1369                 .ctr_name = "tx_vport_multicast_packets",
1370         },
1371         {
1372                 .dpdk_name = "tx_port_broadcast_packets",
1373                 .ctr_name = "tx_vport_broadcast_packets",
1374         },
1375         {
1376                 .dpdk_name = "rx_wqe_err",
1377                 .ctr_name = "rx_wqe_err",
1378         },
1379         {
1380                 .dpdk_name = "rx_crc_errors_phy",
1381                 .ctr_name = "rx_crc_errors_phy",
1382         },
1383         {
1384                 .dpdk_name = "rx_in_range_len_errors_phy",
1385                 .ctr_name = "rx_in_range_len_errors_phy",
1386         },
1387         {
1388                 .dpdk_name = "rx_symbol_err_phy",
1389                 .ctr_name = "rx_symbol_err_phy",
1390         },
1391         {
1392                 .dpdk_name = "tx_errors_phy",
1393                 .ctr_name = "tx_errors_phy",
1394         },
1395         {
1396                 .dpdk_name = "rx_out_of_buffer",
1397                 .ctr_name = "out_of_buffer",
1398                 .dev = 1,
1399         },
1400         {
1401                 .dpdk_name = "tx_packets_phy",
1402                 .ctr_name = "tx_packets_phy",
1403         },
1404         {
1405                 .dpdk_name = "rx_packets_phy",
1406                 .ctr_name = "rx_packets_phy",
1407         },
1408         {
1409                 .dpdk_name = "tx_discards_phy",
1410                 .ctr_name = "tx_discards_phy",
1411         },
1412         {
1413                 .dpdk_name = "rx_discards_phy",
1414                 .ctr_name = "rx_discards_phy",
1415         },
1416         {
1417                 .dpdk_name = "tx_bytes_phy",
1418                 .ctr_name = "tx_bytes_phy",
1419         },
1420         {
1421                 .dpdk_name = "rx_bytes_phy",
1422                 .ctr_name = "rx_bytes_phy",
1423         },
1424         /* Representor only */
1425         {
1426                 .dpdk_name = "rx_packets",
1427                 .ctr_name = "vport_rx_packets",
1428         },
1429         {
1430                 .dpdk_name = "rx_bytes",
1431                 .ctr_name = "vport_rx_bytes",
1432         },
1433         {
1434                 .dpdk_name = "tx_packets",
1435                 .ctr_name = "vport_tx_packets",
1436         },
1437         {
1438                 .dpdk_name = "tx_bytes",
1439                 .ctr_name = "vport_tx_bytes",
1440         },
1441 };
1442
1443 static const unsigned int xstats_n = RTE_DIM(mlx5_counters_init);
1444
1445 /**
1446  * Init the structures to read device counters.
1447  *
1448  * @param dev
1449  *   Pointer to Ethernet device.
1450  */
1451 void
1452 mlx5_os_stats_init(struct rte_eth_dev *dev)
1453 {
1454         struct mlx5_priv *priv = dev->data->dev_private;
1455         struct mlx5_xstats_ctrl *xstats_ctrl = &priv->xstats_ctrl;
1456         struct mlx5_stats_ctrl *stats_ctrl = &priv->stats_ctrl;
1457         unsigned int i;
1458         unsigned int j;
1459         struct ifreq ifr;
1460         struct ethtool_gstrings *strings = NULL;
1461         unsigned int dev_stats_n;
1462         unsigned int str_sz;
1463         int ret;
1464
1465         /* So that it won't aggregate for each init. */
1466         xstats_ctrl->mlx5_stats_n = 0;
1467         ret = mlx5_os_get_stats_n(dev);
1468         if (ret < 0) {
1469                 DRV_LOG(WARNING, "port %u no extended statistics available",
1470                         dev->data->port_id);
1471                 return;
1472         }
1473         dev_stats_n = ret;
1474         /* Allocate memory to grab stat names and values. */
1475         str_sz = dev_stats_n * ETH_GSTRING_LEN;
1476         strings = (struct ethtool_gstrings *)
1477                   mlx5_malloc(0, str_sz + sizeof(struct ethtool_gstrings), 0,
1478                               SOCKET_ID_ANY);
1479         if (!strings) {
1480                 DRV_LOG(WARNING, "port %u unable to allocate memory for xstats",
1481                      dev->data->port_id);
1482                 return;
1483         }
1484         strings->cmd = ETHTOOL_GSTRINGS;
1485         strings->string_set = ETH_SS_STATS;
1486         strings->len = dev_stats_n;
1487         ifr.ifr_data = (caddr_t)strings;
1488         ret = mlx5_ifreq(dev, SIOCETHTOOL, &ifr);
1489         if (ret) {
1490                 DRV_LOG(WARNING, "port %u unable to get statistic names",
1491                         dev->data->port_id);
1492                 goto free;
1493         }
1494         for (i = 0; i != dev_stats_n; ++i) {
1495                 const char *curr_string = (const char *)
1496                         &strings->data[i * ETH_GSTRING_LEN];
1497
1498                 for (j = 0; j != xstats_n; ++j) {
1499                         if (!strcmp(mlx5_counters_init[j].ctr_name,
1500                                     curr_string)) {
1501                                 unsigned int idx = xstats_ctrl->mlx5_stats_n++;
1502
1503                                 xstats_ctrl->dev_table_idx[idx] = i;
1504                                 xstats_ctrl->info[idx] = mlx5_counters_init[j];
1505                                 break;
1506                         }
1507                 }
1508         }
1509         /* Add dev counters. */
1510         for (i = 0; i != xstats_n; ++i) {
1511                 if (mlx5_counters_init[i].dev) {
1512                         unsigned int idx = xstats_ctrl->mlx5_stats_n++;
1513
1514                         xstats_ctrl->info[idx] = mlx5_counters_init[i];
1515                         xstats_ctrl->hw_stats[idx] = 0;
1516                 }
1517         }
1518         MLX5_ASSERT(xstats_ctrl->mlx5_stats_n <= MLX5_MAX_XSTATS);
1519         xstats_ctrl->stats_n = dev_stats_n;
1520         /* Copy to base at first time. */
1521         ret = mlx5_os_read_dev_counters(dev, xstats_ctrl->base);
1522         if (ret)
1523                 DRV_LOG(ERR, "port %u cannot read device counters: %s",
1524                         dev->data->port_id, strerror(rte_errno));
1525         mlx5_os_read_dev_stat(priv, "out_of_buffer", &stats_ctrl->imissed_base);
1526         stats_ctrl->imissed = 0;
1527 free:
1528         mlx5_free(strings);
1529 }
1530
1531 /**
1532  * Get MAC address by querying netdevice.
1533  *
1534  * @param[in] dev
1535  *   Pointer to Ethernet device.
1536  * @param[out] mac
1537  *   MAC address output buffer.
1538  *
1539  * @return
1540  *   0 on success, a negative errno value otherwise and rte_errno is set.
1541  */
1542 int
1543 mlx5_get_mac(struct rte_eth_dev *dev, uint8_t (*mac)[RTE_ETHER_ADDR_LEN])
1544 {
1545         struct ifreq request;
1546         int ret;
1547
1548         ret = mlx5_ifreq(dev, SIOCGIFHWADDR, &request);
1549         if (ret)
1550                 return ret;
1551         memcpy(mac, request.ifr_hwaddr.sa_data, RTE_ETHER_ADDR_LEN);
1552         return 0;
1553 }
1554