drivers/net/mlx5/linux/mlx5_ethdev_os.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright 2015 6WIND S.A.
   3  * Copyright 2015 Mellanox Technologies, Ltd
   4  */
   5
   6 #include <stddef.h>
   7 #include <inttypes.h>
   8 #include <unistd.h>
   9 #include <stdbool.h>
  10 #include <stdint.h>
  11 #include <stdio.h>
  12 #include <string.h>
  13 #include <stdlib.h>
  14 #include <errno.h>
  15 #include <dirent.h>
  16 #include <net/if.h>
  17 #include <sys/ioctl.h>
  18 #include <sys/socket.h>
  19 #include <netinet/in.h>
  20 #include <linux/ethtool.h>
  21 #include <linux/sockios.h>
  22 #include <fcntl.h>
  23 #include <stdalign.h>
  24 #include <sys/un.h>
  25 #include <time.h>
  26
  27 #include <rte_ethdev_driver.h>
  28 #include <rte_bus_pci.h>
  29 #include <rte_mbuf.h>
  30 #include <rte_common.h>
  31 #include <rte_interrupts.h>
  32 #include <rte_malloc.h>
  33 #include <rte_string_fns.h>
  34 #include <rte_rwlock.h>
  35 #include <rte_cycles.h>
  36
  37 #include <mlx5_glue.h>
  38 #include <mlx5_devx_cmds.h>
  39 #include <mlx5_common.h>
  40 #include <mlx5_malloc.h>
  41
  42 #include "mlx5.h"
  43 #include "mlx5_rxtx.h"
  44 #include "mlx5_utils.h"
  45
  46 /* Supported speed values found in /usr/include/linux/ethtool.h */
  47 #ifndef HAVE_SUPPORTED_40000baseKR4_Full
  48 #define SUPPORTED_40000baseKR4_Full (1 << 23)
  49 #endif
  50 #ifndef HAVE_SUPPORTED_40000baseCR4_Full
  51 #define SUPPORTED_40000baseCR4_Full (1 << 24)
  52 #endif
  53 #ifndef HAVE_SUPPORTED_40000baseSR4_Full
  54 #define SUPPORTED_40000baseSR4_Full (1 << 25)
  55 #endif
  56 #ifndef HAVE_SUPPORTED_40000baseLR4_Full
  57 #define SUPPORTED_40000baseLR4_Full (1 << 26)
  58 #endif
  59 #ifndef HAVE_SUPPORTED_56000baseKR4_Full
  60 #define SUPPORTED_56000baseKR4_Full (1 << 27)
  61 #endif
  62 #ifndef HAVE_SUPPORTED_56000baseCR4_Full
  63 #define SUPPORTED_56000baseCR4_Full (1 << 28)
  64 #endif
  65 #ifndef HAVE_SUPPORTED_56000baseSR4_Full
  66 #define SUPPORTED_56000baseSR4_Full (1 << 29)
  67 #endif
  68 #ifndef HAVE_SUPPORTED_56000baseLR4_Full
  69 #define SUPPORTED_56000baseLR4_Full (1 << 30)
  70 #endif
  71
  72 /* Add defines in case the running kernel is not the same as user headers. */
  73 #ifndef ETHTOOL_GLINKSETTINGS
  74 struct ethtool_link_settings {
  75         uint32_t cmd;
  76         uint32_t speed;
  77         uint8_t duplex;
  78         uint8_t port;
  79         uint8_t phy_address;
  80         uint8_t autoneg;
  81         uint8_t mdio_support;
  82         uint8_t eth_to_mdix;
  83         uint8_t eth_tp_mdix_ctrl;
  84         int8_t link_mode_masks_nwords;
  85         uint32_t reserved[8];
  86         uint32_t link_mode_masks[];
  87 };
  88
  89 /* The kernel values can be found in /include/uapi/linux/ethtool.h */
  90 #define ETHTOOL_GLINKSETTINGS 0x0000004c
  91 #define ETHTOOL_LINK_MODE_1000baseT_Full_BIT 5
  92 #define ETHTOOL_LINK_MODE_Autoneg_BIT 6
  93 #define ETHTOOL_LINK_MODE_1000baseKX_Full_BIT 17
  94 #define ETHTOOL_LINK_MODE_10000baseKX4_Full_BIT 18
  95 #define ETHTOOL_LINK_MODE_10000baseKR_Full_BIT 19
  96 #define ETHTOOL_LINK_MODE_10000baseR_FEC_BIT 20
  97 #define ETHTOOL_LINK_MODE_20000baseMLD2_Full_BIT 21
  98 #define ETHTOOL_LINK_MODE_20000baseKR2_Full_BIT 22
  99 #define ETHTOOL_LINK_MODE_40000baseKR4_Full_BIT 23
 100 #define ETHTOOL_LINK_MODE_40000baseCR4_Full_BIT 24
 101 #define ETHTOOL_LINK_MODE_40000baseSR4_Full_BIT 25
 102 #define ETHTOOL_LINK_MODE_40000baseLR4_Full_BIT 26
 103 #define ETHTOOL_LINK_MODE_56000baseKR4_Full_BIT 27
 104 #define ETHTOOL_LINK_MODE_56000baseCR4_Full_BIT 28
 105 #define ETHTOOL_LINK_MODE_56000baseSR4_Full_BIT 29
 106 #define ETHTOOL_LINK_MODE_56000baseLR4_Full_BIT 30
 107 #endif
 108 #ifndef HAVE_ETHTOOL_LINK_MODE_25G
 109 #define ETHTOOL_LINK_MODE_25000baseCR_Full_BIT 31
 110 #define ETHTOOL_LINK_MODE_25000baseKR_Full_BIT 32
 111 #define ETHTOOL_LINK_MODE_25000baseSR_Full_BIT 33
 112 #endif
 113 #ifndef HAVE_ETHTOOL_LINK_MODE_50G
 114 #define ETHTOOL_LINK_MODE_50000baseCR2_Full_BIT 34
 115 #define ETHTOOL_LINK_MODE_50000baseKR2_Full_BIT 35
 116 #endif
 117 #ifndef HAVE_ETHTOOL_LINK_MODE_100G
 118 #define ETHTOOL_LINK_MODE_100000baseKR4_Full_BIT 36
 119 #define ETHTOOL_LINK_MODE_100000baseSR4_Full_BIT 37
 120 #define ETHTOOL_LINK_MODE_100000baseCR4_Full_BIT 38
 121 #define ETHTOOL_LINK_MODE_100000baseLR4_ER4_Full_BIT 39
 122 #endif
 123 #ifndef HAVE_ETHTOOL_LINK_MODE_200G
 124 #define ETHTOOL_LINK_MODE_200000baseKR4_Full_BIT 62
 125 #define ETHTOOL_LINK_MODE_200000baseSR4_Full_BIT 63
 126 #define ETHTOOL_LINK_MODE_200000baseLR4_ER4_FR4_Full_BIT 0 /* 64 - 64 */
 127 #define ETHTOOL_LINK_MODE_200000baseDR4_Full_BIT 1 /* 65 - 64 */
 128 #define ETHTOOL_LINK_MODE_200000baseCR4_Full_BIT 2 /* 66 - 64 */
 129 #endif
 130
 131
 132 /**
 133  * Get interface name from private structure.
 134  *
 135  * This is a port representor-aware version of mlx5_get_ifname_sysfs().
 136  *
 137  * @param[in] dev
 138  *   Pointer to Ethernet device.
 139  * @param[out] ifname
 140  *   Interface name output buffer.
 141  *
 142  * @return
 143  *   0 on success, a negative errno value otherwise and rte_errno is set.
 144  */
 145 int
 146 mlx5_get_ifname(const struct rte_eth_dev *dev, char (*ifname)[IF_NAMESIZE])
 147 {
 148         struct mlx5_priv *priv = dev->data->dev_private;
 149         unsigned int ifindex;
 150
 151         MLX5_ASSERT(priv);
 152         MLX5_ASSERT(priv->sh);
 153         if (priv->bond_ifindex > 0) {
 154                 memcpy(ifname, priv->bond_name, IF_NAMESIZE);
 155                 return 0;
 156         }
 157         ifindex = mlx5_ifindex(dev);
 158         if (!ifindex) {
 159                 if (!priv->representor)
 160                         return mlx5_get_ifname_sysfs(priv->sh->ibdev_path,
 161                                                      *ifname);
 162                 rte_errno = ENXIO;
 163                 return -rte_errno;
 164         }
 165         if (if_indextoname(ifindex, &(*ifname)[0]))
 166                 return 0;
 167         rte_errno = errno;
 168         return -rte_errno;
 169 }
 170
 171 /**
 172  * Perform ifreq ioctl() on associated Ethernet device.
 173  *
 174  * @param[in] dev
 175  *   Pointer to Ethernet device.
 176  * @param req
 177  *   Request number to pass to ioctl().
 178  * @param[out] ifr
 179  *   Interface request structure output buffer.
 180  *
 181  * @return
 182  *   0 on success, a negative errno value otherwise and rte_errno is set.
 183  */
 184 static int
 185 mlx5_ifreq(const struct rte_eth_dev *dev, int req, struct ifreq *ifr)
 186 {
 187         int sock = socket(PF_INET, SOCK_DGRAM, IPPROTO_IP);
 188         int ret = 0;
 189
 190         if (sock == -1) {
 191                 rte_errno = errno;
 192                 return -rte_errno;
 193         }
 194         ret = mlx5_get_ifname(dev, &ifr->ifr_name);
 195         if (ret)
 196                 goto error;
 197         ret = ioctl(sock, req, ifr);
 198         if (ret == -1) {
 199                 rte_errno = errno;
 200                 goto error;
 201         }
 202         close(sock);
 203         return 0;
 204 error:
 205         close(sock);
 206         return -rte_errno;
 207 }
 208
 209 /**
 210  * Get device MTU.
 211  *
 212  * @param dev
 213  *   Pointer to Ethernet device.
 214  * @param[out] mtu
 215  *   MTU value output buffer.
 216  *
 217  * @return
 218  *   0 on success, a negative errno value otherwise and rte_errno is set.
 219  */
 220 int
 221 mlx5_get_mtu(struct rte_eth_dev *dev, uint16_t *mtu)
 222 {
 223         struct ifreq request;
 224         int ret = mlx5_ifreq(dev, SIOCGIFMTU, &request);
 225
 226         if (ret)
 227                 return ret;
 228         *mtu = request.ifr_mtu;
 229         return 0;
 230 }
 231
 232 /**
 233  * Set device MTU.
 234  *
 235  * @param dev
 236  *   Pointer to Ethernet device.
 237  * @param mtu
 238  *   MTU value to set.
 239  *
 240  * @return
 241  *   0 on success, a negative errno value otherwise and rte_errno is set.
 242  */
 243 int
 244 mlx5_set_mtu(struct rte_eth_dev *dev, uint16_t mtu)
 245 {
 246         struct ifreq request = { .ifr_mtu = mtu, };
 247
 248         return mlx5_ifreq(dev, SIOCSIFMTU, &request);
 249 }
 250
 251 /**
 252  * Set device flags.
 253  *
 254  * @param dev
 255  *   Pointer to Ethernet device.
 256  * @param keep
 257  *   Bitmask for flags that must remain untouched.
 258  * @param flags
 259  *   Bitmask for flags to modify.
 260  *
 261  * @return
 262  *   0 on success, a negative errno value otherwise and rte_errno is set.
 263  */
 264 static int
 265 mlx5_set_flags(struct rte_eth_dev *dev, unsigned int keep, unsigned int flags)
 266 {
 267         struct ifreq request;
 268         int ret = mlx5_ifreq(dev, SIOCGIFFLAGS, &request);
 269
 270         if (ret)
 271                 return ret;
 272         request.ifr_flags &= keep;
 273         request.ifr_flags |= flags & ~keep;
 274         return mlx5_ifreq(dev, SIOCSIFFLAGS, &request);
 275 }
 276
 277 /**
 278  * Get device current raw clock counter
 279  *
 280  * @param dev
 281  *   Pointer to Ethernet device structure.
 282  * @param[out] time
 283  *   Current raw clock counter of the device.
 284  *
 285  * @return
 286  *   0 if the clock has correctly been read
 287  *   The value of errno in case of error
 288  */
 289 int
 290 mlx5_read_clock(struct rte_eth_dev *dev, uint64_t *clock)
 291 {
 292         struct mlx5_priv *priv = dev->data->dev_private;
 293         struct ibv_context *ctx = priv->sh->ctx;
 294         struct ibv_values_ex values;
 295         int err = 0;
 296
 297         values.comp_mask = IBV_VALUES_MASK_RAW_CLOCK;
 298         err = mlx5_glue->query_rt_values_ex(ctx, &values);
 299         if (err != 0) {
 300                 DRV_LOG(WARNING, "Could not query the clock !");
 301                 return err;
 302         }
 303         *clock = values.raw_clock.tv_nsec;
 304         return 0;
 305 }
 306
 307 /**
 308  * Retrieve the master device for representor in the same switch domain.
 309  *
 310  * @param dev
 311  *   Pointer to representor Ethernet device structure.
 312  *
 313  * @return
 314  *   Master device structure  on success, NULL otherwise.
 315  */
 316 static struct rte_eth_dev *
 317 mlx5_find_master_dev(struct rte_eth_dev *dev)
 318 {
 319         struct mlx5_priv *priv;
 320         uint16_t port_id;
 321         uint16_t domain_id;
 322
 323         priv = dev->data->dev_private;
 324         domain_id = priv->domain_id;
 325         MLX5_ASSERT(priv->representor);
 326         MLX5_ETH_FOREACH_DEV(port_id, priv->pci_dev) {
 327                 struct mlx5_priv *opriv =
 328                         rte_eth_devices[port_id].data->dev_private;
 329                 if (opriv &&
 330                     opriv->master &&
 331                     opriv->domain_id == domain_id &&
 332                     opriv->sh == priv->sh)
 333                         return &rte_eth_devices[port_id];
 334         }
 335         return NULL;
 336 }
 337
 338 /**
 339  * DPDK callback to retrieve physical link information.
 340  *
 341  * @param dev
 342  *   Pointer to Ethernet device structure.
 343  * @param[out] link
 344  *   Storage for current link status.
 345  *
 346  * @return
 347  *   0 on success, a negative errno value otherwise and rte_errno is set.
 348  */
 349 static int
 350 mlx5_link_update_unlocked_gset(struct rte_eth_dev *dev,
 351                                struct rte_eth_link *link)
 352 {
 353         struct mlx5_priv *priv = dev->data->dev_private;
 354         struct ethtool_cmd edata = {
 355                 .cmd = ETHTOOL_GSET /* Deprecated since Linux v4.5. */
 356         };
 357         struct ifreq ifr;
 358         struct rte_eth_link dev_link;
 359         int link_speed = 0;
 360         int ret;
 361
 362         ret = mlx5_ifreq(dev, SIOCGIFFLAGS, &ifr);
 363         if (ret) {
 364                 DRV_LOG(WARNING, "port %u ioctl(SIOCGIFFLAGS) failed: %s",
 365                         dev->data->port_id, strerror(rte_errno));
 366                 return ret;
 367         }
 368         dev_link = (struct rte_eth_link) {
 369                 .link_status = ((ifr.ifr_flags & IFF_UP) &&
 370                                 (ifr.ifr_flags & IFF_RUNNING)),
 371         };
 372         ifr = (struct ifreq) {
 373                 .ifr_data = (void *)&edata,
 374         };
 375         ret = mlx5_ifreq(dev, SIOCETHTOOL, &ifr);
 376         if (ret) {
 377                 if (ret == -ENOTSUP && priv->representor) {
 378                         struct rte_eth_dev *master;
 379
 380                         /*
 381                          * For representors we can try to inherit link
 382                          * settings from the master device. Actually
 383                          * link settings do not make a lot of sense
 384                          * for representors due to missing physical
 385                          * link. The old kernel drivers supported
 386                          * emulated settings query for representors,
 387                          * the new ones do not, so we have to add
 388                          * this code for compatibility issues.
 389                          */
 390                         master = mlx5_find_master_dev(dev);
 391                         if (master) {
 392                                 ifr = (struct ifreq) {
 393                                         .ifr_data = (void *)&edata,
 394                                 };
 395                                 ret = mlx5_ifreq(master, SIOCETHTOOL, &ifr);
 396                         }
 397                 }
 398                 if (ret) {
 399                         DRV_LOG(WARNING,
 400                                 "port %u ioctl(SIOCETHTOOL,"
 401                                 " ETHTOOL_GSET) failed: %s",
 402                                 dev->data->port_id, strerror(rte_errno));
 403                         return ret;
 404                 }
 405         }
 406         link_speed = ethtool_cmd_speed(&edata);
 407         if (link_speed == -1)
 408                 dev_link.link_speed = ETH_SPEED_NUM_UNKNOWN;
 409         else
 410                 dev_link.link_speed = link_speed;
 411         priv->link_speed_capa = 0;
 412         if (edata.supported & SUPPORTED_Autoneg)
 413                 priv->link_speed_capa |= ETH_LINK_SPEED_AUTONEG;
 414         if (edata.supported & (SUPPORTED_1000baseT_Full |
 415                                SUPPORTED_1000baseKX_Full))
 416                 priv->link_speed_capa |= ETH_LINK_SPEED_1G;
 417         if (edata.supported & SUPPORTED_10000baseKR_Full)
 418                 priv->link_speed_capa |= ETH_LINK_SPEED_10G;
 419         if (edata.supported & (SUPPORTED_40000baseKR4_Full |
 420                                SUPPORTED_40000baseCR4_Full |
 421                                SUPPORTED_40000baseSR4_Full |
 422                                SUPPORTED_40000baseLR4_Full))
 423                 priv->link_speed_capa |= ETH_LINK_SPEED_40G;
 424         dev_link.link_duplex = ((edata.duplex == DUPLEX_HALF) ?
 425                                 ETH_LINK_HALF_DUPLEX : ETH_LINK_FULL_DUPLEX);
 426         dev_link.link_autoneg = !(dev->data->dev_conf.link_speeds &
 427                         ETH_LINK_SPEED_FIXED);
 428         *link = dev_link;
 429         return 0;
 430 }
 431
 432 /**
 433  * Retrieve physical link information (unlocked version using new ioctl).
 434  *
 435  * @param dev
 436  *   Pointer to Ethernet device structure.
 437  * @param[out] link
 438  *   Storage for current link status.
 439  *
 440  * @return
 441  *   0 on success, a negative errno value otherwise and rte_errno is set.
 442  */
 443 static int
 444 mlx5_link_update_unlocked_gs(struct rte_eth_dev *dev,
 445                              struct rte_eth_link *link)
 446
 447 {
 448         struct mlx5_priv *priv = dev->data->dev_private;
 449         struct ethtool_link_settings gcmd = { .cmd = ETHTOOL_GLINKSETTINGS };
 450         struct ifreq ifr;
 451         struct rte_eth_link dev_link;
 452         struct rte_eth_dev *master = NULL;
 453         uint64_t sc;
 454         int ret;
 455
 456         ret = mlx5_ifreq(dev, SIOCGIFFLAGS, &ifr);
 457         if (ret) {
 458                 DRV_LOG(WARNING, "port %u ioctl(SIOCGIFFLAGS) failed: %s",
 459                         dev->data->port_id, strerror(rte_errno));
 460                 return ret;
 461         }
 462         dev_link = (struct rte_eth_link) {
 463                 .link_status = ((ifr.ifr_flags & IFF_UP) &&
 464                                 (ifr.ifr_flags & IFF_RUNNING)),
 465         };
 466         ifr = (struct ifreq) {
 467                 .ifr_data = (void *)&gcmd,
 468         };
 469         ret = mlx5_ifreq(dev, SIOCETHTOOL, &ifr);
 470         if (ret) {
 471                 if (ret == -ENOTSUP && priv->representor) {
 472                         /*
 473                          * For representors we can try to inherit link
 474                          * settings from the master device. Actually
 475                          * link settings do not make a lot of sense
 476                          * for representors due to missing physical
 477                          * link. The old kernel drivers supported
 478                          * emulated settings query for representors,
 479                          * the new ones do not, so we have to add
 480                          * this code for compatibility issues.
 481                          */
 482                         master = mlx5_find_master_dev(dev);
 483                         if (master) {
 484                                 ifr = (struct ifreq) {
 485                                         .ifr_data = (void *)&gcmd,
 486                                 };
 487                                 ret = mlx5_ifreq(master, SIOCETHTOOL, &ifr);
 488                         }
 489                 }
 490                 if (ret) {
 491                         DRV_LOG(DEBUG,
 492                                 "port %u ioctl(SIOCETHTOOL,"
 493                                 " ETHTOOL_GLINKSETTINGS) failed: %s",
 494                                 dev->data->port_id, strerror(rte_errno));
 495                         return ret;
 496                 }
 497         }
 498         gcmd.link_mode_masks_nwords = -gcmd.link_mode_masks_nwords;
 499
 500         alignas(struct ethtool_link_settings)
 501         uint8_t data[offsetof(struct ethtool_link_settings, link_mode_masks) +
 502                      sizeof(uint32_t) * gcmd.link_mode_masks_nwords * 3];
 503         struct ethtool_link_settings *ecmd = (void *)data;
 504
 505         *ecmd = gcmd;
 506         ifr.ifr_data = (void *)ecmd;
 507         ret = mlx5_ifreq(master ? master : dev, SIOCETHTOOL, &ifr);
 508         if (ret) {
 509                 DRV_LOG(DEBUG,
 510                         "port %u ioctl(SIOCETHTOOL,"
 511                         "ETHTOOL_GLINKSETTINGS) failed: %s",
 512                         dev->data->port_id, strerror(rte_errno));
 513                 return ret;
 514         }
 515         dev_link.link_speed = (ecmd->speed == UINT32_MAX) ?
 516                                 ETH_SPEED_NUM_UNKNOWN : ecmd->speed;
 517         sc = ecmd->link_mode_masks[0] |
 518                 ((uint64_t)ecmd->link_mode_masks[1] << 32);
 519         priv->link_speed_capa = 0;
 520         if (sc & MLX5_BITSHIFT(ETHTOOL_LINK_MODE_Autoneg_BIT))
 521                 priv->link_speed_capa |= ETH_LINK_SPEED_AUTONEG;
 522         if (sc & (MLX5_BITSHIFT(ETHTOOL_LINK_MODE_1000baseT_Full_BIT) |
 523                   MLX5_BITSHIFT(ETHTOOL_LINK_MODE_1000baseKX_Full_BIT)))
 524                 priv->link_speed_capa |= ETH_LINK_SPEED_1G;
 525         if (sc & (MLX5_BITSHIFT(ETHTOOL_LINK_MODE_10000baseKX4_Full_BIT) |
 526                   MLX5_BITSHIFT(ETHTOOL_LINK_MODE_10000baseKR_Full_BIT) |
 527                   MLX5_BITSHIFT(ETHTOOL_LINK_MODE_10000baseR_FEC_BIT)))
 528                 priv->link_speed_capa |= ETH_LINK_SPEED_10G;
 529         if (sc & (MLX5_BITSHIFT(ETHTOOL_LINK_MODE_20000baseMLD2_Full_BIT) |
 530                   MLX5_BITSHIFT(ETHTOOL_LINK_MODE_20000baseKR2_Full_BIT)))
 531                 priv->link_speed_capa |= ETH_LINK_SPEED_20G;
 532         if (sc & (MLX5_BITSHIFT(ETHTOOL_LINK_MODE_40000baseKR4_Full_BIT) |
 533                   MLX5_BITSHIFT(ETHTOOL_LINK_MODE_40000baseCR4_Full_BIT) |
 534                   MLX5_BITSHIFT(ETHTOOL_LINK_MODE_40000baseSR4_Full_BIT) |
 535                   MLX5_BITSHIFT(ETHTOOL_LINK_MODE_40000baseLR4_Full_BIT)))
 536                 priv->link_speed_capa |= ETH_LINK_SPEED_40G;
 537         if (sc & (MLX5_BITSHIFT(ETHTOOL_LINK_MODE_56000baseKR4_Full_BIT) |
 538                   MLX5_BITSHIFT(ETHTOOL_LINK_MODE_56000baseCR4_Full_BIT) |
 539                   MLX5_BITSHIFT(ETHTOOL_LINK_MODE_56000baseSR4_Full_BIT) |
 540                   MLX5_BITSHIFT(ETHTOOL_LINK_MODE_56000baseLR4_Full_BIT)))
 541                 priv->link_speed_capa |= ETH_LINK_SPEED_56G;
 542         if (sc & (MLX5_BITSHIFT(ETHTOOL_LINK_MODE_25000baseCR_Full_BIT) |
 543                   MLX5_BITSHIFT(ETHTOOL_LINK_MODE_25000baseKR_Full_BIT) |
 544                   MLX5_BITSHIFT(ETHTOOL_LINK_MODE_25000baseSR_Full_BIT)))
 545                 priv->link_speed_capa |= ETH_LINK_SPEED_25G;
 546         if (sc & (MLX5_BITSHIFT(ETHTOOL_LINK_MODE_50000baseCR2_Full_BIT) |
 547                   MLX5_BITSHIFT(ETHTOOL_LINK_MODE_50000baseKR2_Full_BIT)))
 548                 priv->link_speed_capa |= ETH_LINK_SPEED_50G;
 549         if (sc & (MLX5_BITSHIFT(ETHTOOL_LINK_MODE_100000baseKR4_Full_BIT) |
 550                   MLX5_BITSHIFT(ETHTOOL_LINK_MODE_100000baseSR4_Full_BIT) |
 551                   MLX5_BITSHIFT(ETHTOOL_LINK_MODE_100000baseCR4_Full_BIT) |
 552                   MLX5_BITSHIFT(ETHTOOL_LINK_MODE_100000baseLR4_ER4_Full_BIT)))
 553                 priv->link_speed_capa |= ETH_LINK_SPEED_100G;
 554         if (sc & (MLX5_BITSHIFT(ETHTOOL_LINK_MODE_200000baseKR4_Full_BIT) |
 555                   MLX5_BITSHIFT(ETHTOOL_LINK_MODE_200000baseSR4_Full_BIT)))
 556                 priv->link_speed_capa |= ETH_LINK_SPEED_200G;
 557
 558         sc = ecmd->link_mode_masks[2] |
 559                 ((uint64_t)ecmd->link_mode_masks[3] << 32);
 560         if (sc & (MLX5_BITSHIFT(ETHTOOL_LINK_MODE_200000baseCR4_Full_BIT) |
 561                   MLX5_BITSHIFT
 562                        (ETHTOOL_LINK_MODE_200000baseLR4_ER4_FR4_Full_BIT) |
 563                   MLX5_BITSHIFT(ETHTOOL_LINK_MODE_200000baseDR4_Full_BIT)))
 564                 priv->link_speed_capa |= ETH_LINK_SPEED_200G;
 565         dev_link.link_duplex = ((ecmd->duplex == DUPLEX_HALF) ?
 566                                 ETH_LINK_HALF_DUPLEX : ETH_LINK_FULL_DUPLEX);
 567         dev_link.link_autoneg = !(dev->data->dev_conf.link_speeds &
 568                                   ETH_LINK_SPEED_FIXED);
 569         *link = dev_link;
 570         return 0;
 571 }
 572
 573 /**
 574  * DPDK callback to retrieve physical link information.
 575  *
 576  * @param dev
 577  *   Pointer to Ethernet device structure.
 578  * @param wait_to_complete
 579  *   Wait for request completion.
 580  *
 581  * @return
 582  *   0 if link status was not updated, positive if it was, a negative errno
 583  *   value otherwise and rte_errno is set.
 584  */
 585 int
 586 mlx5_link_update(struct rte_eth_dev *dev, int wait_to_complete)
 587 {
 588         int ret;
 589         struct rte_eth_link dev_link;
 590         time_t start_time = time(NULL);
 591         int retry = MLX5_GET_LINK_STATUS_RETRY_COUNT;
 592
 593         do {
 594                 ret = mlx5_link_update_unlocked_gs(dev, &dev_link);
 595                 if (ret == -ENOTSUP)
 596                         ret = mlx5_link_update_unlocked_gset(dev, &dev_link);
 597                 if (ret == 0)
 598                         break;
 599                 /* Handle wait to complete situation. */
 600                 if ((wait_to_complete || retry) && ret == -EAGAIN) {
 601                         if (abs((int)difftime(time(NULL), start_time)) <
 602                             MLX5_LINK_STATUS_TIMEOUT) {
 603                                 usleep(0);
 604                                 continue;
 605                         } else {
 606                                 rte_errno = EBUSY;
 607                                 return -rte_errno;
 608                         }
 609                 } else if (ret < 0) {
 610                         return ret;
 611                 }
 612         } while (wait_to_complete || retry-- > 0);
 613         ret = !!memcmp(&dev->data->dev_link, &dev_link,
 614                        sizeof(struct rte_eth_link));
 615         dev->data->dev_link = dev_link;
 616         return ret;
 617 }
 618
 619 /**
 620  * DPDK callback to get flow control status.
 621  *
 622  * @param dev
 623  *   Pointer to Ethernet device structure.
 624  * @param[out] fc_conf
 625  *   Flow control output buffer.
 626  *
 627  * @return
 628  *   0 on success, a negative errno value otherwise and rte_errno is set.
 629  */
 630 int
 631 mlx5_dev_get_flow_ctrl(struct rte_eth_dev *dev, struct rte_eth_fc_conf *fc_conf)
 632 {
 633         struct ifreq ifr;
 634         struct ethtool_pauseparam ethpause = {
 635                 .cmd = ETHTOOL_GPAUSEPARAM
 636         };
 637         int ret;
 638
 639         ifr.ifr_data = (void *)&ethpause;
 640         ret = mlx5_ifreq(dev, SIOCETHTOOL, &ifr);
 641         if (ret) {
 642                 DRV_LOG(WARNING,
 643                         "port %u ioctl(SIOCETHTOOL, ETHTOOL_GPAUSEPARAM) failed:"
 644                         " %s",
 645                         dev->data->port_id, strerror(rte_errno));
 646                 return ret;
 647         }
 648         fc_conf->autoneg = ethpause.autoneg;
 649         if (ethpause.rx_pause && ethpause.tx_pause)
 650                 fc_conf->mode = RTE_FC_FULL;
 651         else if (ethpause.rx_pause)
 652                 fc_conf->mode = RTE_FC_RX_PAUSE;
 653         else if (ethpause.tx_pause)
 654                 fc_conf->mode = RTE_FC_TX_PAUSE;
 655         else
 656                 fc_conf->mode = RTE_FC_NONE;
 657         return 0;
 658 }
 659
 660 /**
 661  * DPDK callback to modify flow control parameters.
 662  *
 663  * @param dev
 664  *   Pointer to Ethernet device structure.
 665  * @param[in] fc_conf
 666  *   Flow control parameters.
 667  *
 668  * @return
 669  *   0 on success, a negative errno value otherwise and rte_errno is set.
 670  */
 671 int
 672 mlx5_dev_set_flow_ctrl(struct rte_eth_dev *dev, struct rte_eth_fc_conf *fc_conf)
 673 {
 674         struct ifreq ifr;
 675         struct ethtool_pauseparam ethpause = {
 676                 .cmd = ETHTOOL_SPAUSEPARAM
 677         };
 678         int ret;
 679
 680         ifr.ifr_data = (void *)&ethpause;
 681         ethpause.autoneg = fc_conf->autoneg;
 682         if (((fc_conf->mode & RTE_FC_FULL) == RTE_FC_FULL) ||
 683             (fc_conf->mode & RTE_FC_RX_PAUSE))
 684                 ethpause.rx_pause = 1;
 685         else
 686                 ethpause.rx_pause = 0;
 687
 688         if (((fc_conf->mode & RTE_FC_FULL) == RTE_FC_FULL) ||
 689             (fc_conf->mode & RTE_FC_TX_PAUSE))
 690                 ethpause.tx_pause = 1;
 691         else
 692                 ethpause.tx_pause = 0;
 693         ret = mlx5_ifreq(dev, SIOCETHTOOL, &ifr);
 694         if (ret) {
 695                 DRV_LOG(WARNING,
 696                         "port %u ioctl(SIOCETHTOOL, ETHTOOL_SPAUSEPARAM)"
 697                         " failed: %s",
 698                         dev->data->port_id, strerror(rte_errno));
 699                 return ret;
 700         }
 701         return 0;
 702 }
 703
 704 /**
 705  * Handle asynchronous removal event for entire multiport device.
 706  *
 707  * @param sh
 708  *   Infiniband device shared context.
 709  */
 710 static void
 711 mlx5_dev_interrupt_device_fatal(struct mlx5_dev_ctx_shared *sh)
 712 {
 713         uint32_t i;
 714
 715         for (i = 0; i < sh->max_port; ++i) {
 716                 struct rte_eth_dev *dev;
 717
 718                 if (sh->port[i].ih_port_id >= RTE_MAX_ETHPORTS) {
 719                         /*
 720                          * Or not existing port either no
 721                          * handler installed for this port.
 722                          */
 723                         continue;
 724                 }
 725                 dev = &rte_eth_devices[sh->port[i].ih_port_id];
 726                 MLX5_ASSERT(dev);
 727                 if (dev->data->dev_conf.intr_conf.rmv)
 728                         rte_eth_dev_callback_process
 729                                 (dev, RTE_ETH_EVENT_INTR_RMV, NULL);
 730         }
 731 }
 732
 733 /**
 734  * Handle shared asynchronous events the NIC (removal event
 735  * and link status change). Supports multiport IB device.
 736  *
 737  * @param cb_arg
 738  *   Callback argument.
 739  */
 740 void
 741 mlx5_dev_interrupt_handler(void *cb_arg)
 742 {
 743         struct mlx5_dev_ctx_shared *sh = cb_arg;
 744         struct ibv_async_event event;
 745
 746         /* Read all message from the IB device and acknowledge them. */
 747         for (;;) {
 748                 struct rte_eth_dev *dev;
 749                 uint32_t tmp;
 750
 751                 if (mlx5_glue->get_async_event(sh->ctx, &event))
 752                         break;
 753                 /* Retrieve and check IB port index. */
 754                 tmp = (uint32_t)event.element.port_num;
 755                 if (!tmp && event.event_type == IBV_EVENT_DEVICE_FATAL) {
 756                         /*
 757                          * The DEVICE_FATAL event is called once for
 758                          * entire device without port specifying.
 759                          * We should notify all existing ports.
 760                          */
 761                         mlx5_glue->ack_async_event(&event);
 762                         mlx5_dev_interrupt_device_fatal(sh);
 763                         continue;
 764                 }
 765                 MLX5_ASSERT(tmp && (tmp <= sh->max_port));
 766                 if (!tmp) {
 767                         /* Unsupported device level event. */
 768                         mlx5_glue->ack_async_event(&event);
 769                         DRV_LOG(DEBUG,
 770                                 "unsupported common event (type %d)",
 771                                 event.event_type);
 772                         continue;
 773                 }
 774                 if (tmp > sh->max_port) {
 775                         /* Invalid IB port index. */
 776                         mlx5_glue->ack_async_event(&event);
 777                         DRV_LOG(DEBUG,
 778                                 "cannot handle an event (type %d)"
 779                                 "due to invalid IB port index (%u)",
 780                                 event.event_type, tmp);
 781                         continue;
 782                 }
 783                 if (sh->port[tmp - 1].ih_port_id >= RTE_MAX_ETHPORTS) {
 784                         /* No handler installed. */
 785                         mlx5_glue->ack_async_event(&event);
 786                         DRV_LOG(DEBUG,
 787                                 "cannot handle an event (type %d)"
 788                                 "due to no handler installed for port %u",
 789                                 event.event_type, tmp);
 790                         continue;
 791                 }
 792                 /* Retrieve ethernet device descriptor. */
 793                 tmp = sh->port[tmp - 1].ih_port_id;
 794                 dev = &rte_eth_devices[tmp];
 795                 MLX5_ASSERT(dev);
 796                 if ((event.event_type == IBV_EVENT_PORT_ACTIVE ||
 797                      event.event_type == IBV_EVENT_PORT_ERR) &&
 798                         dev->data->dev_conf.intr_conf.lsc) {
 799                         mlx5_glue->ack_async_event(&event);
 800                         if (mlx5_link_update(dev, 0) == -EAGAIN) {
 801                                 usleep(0);
 802                                 continue;
 803                         }
 804                         rte_eth_dev_callback_process
 805                                 (dev, RTE_ETH_EVENT_INTR_LSC, NULL);
 806                         continue;
 807                 }
 808                 DRV_LOG(DEBUG,
 809                         "port %u cannot handle an unknown event (type %d)",
 810                         dev->data->port_id, event.event_type);
 811                 mlx5_glue->ack_async_event(&event);
 812         }
 813 }
 814
 815 /*
 816  * Unregister callback handler safely. The handler may be active
 817  * while we are trying to unregister it, in this case code -EAGAIN
 818  * is returned by rte_intr_callback_unregister(). This routine checks
 819  * the return code and tries to unregister handler again.
 820  *
 821  * @param handle
 822  *   interrupt handle
 823  * @param cb_fn
 824  *   pointer to callback routine
 825  * @cb_arg
 826  *   opaque callback parameter
 827  */
 828 void
 829 mlx5_intr_callback_unregister(const struct rte_intr_handle *handle,
 830                               rte_intr_callback_fn cb_fn, void *cb_arg)
 831 {
 832         /*
 833          * Try to reduce timeout management overhead by not calling
 834          * the timer related routines on the first iteration. If the
 835          * unregistering succeeds on first call there will be no
 836          * timer calls at all.
 837          */
 838         uint64_t twait = 0;
 839         uint64_t start = 0;
 840
 841         do {
 842                 int ret;
 843
 844                 ret = rte_intr_callback_unregister(handle, cb_fn, cb_arg);
 845                 if (ret >= 0)
 846                         return;
 847                 if (ret != -EAGAIN) {
 848                         DRV_LOG(INFO, "failed to unregister interrupt"
 849                                       " handler (error: %d)", ret);
 850                         MLX5_ASSERT(false);
 851                         return;
 852                 }
 853                 if (twait) {
 854                         struct timespec onems;
 855
 856                         /* Wait one millisecond and try again. */
 857                         onems.tv_sec = 0;
 858                         onems.tv_nsec = NS_PER_S / MS_PER_S;
 859                         nanosleep(&onems, 0);
 860                         /* Check whether one second elapsed. */
 861                         if ((rte_get_timer_cycles() - start) <= twait)
 862                                 continue;
 863                 } else {
 864                         /*
 865                          * We get the amount of timer ticks for one second.
 866                          * If this amount elapsed it means we spent one
 867                          * second in waiting. This branch is executed once
 868                          * on first iteration.
 869                          */
 870                         twait = rte_get_timer_hz();
 871                         MLX5_ASSERT(twait);
 872                 }
 873                 /*
 874                  * Timeout elapsed, show message (once a second) and retry.
 875                  * We have no other acceptable option here, if we ignore
 876                  * the unregistering return code the handler will not
 877                  * be unregistered, fd will be closed and we may get the
 878                  * crush. Hanging and messaging in the loop seems not to be
 879                  * the worst choice.
 880                  */
 881                 DRV_LOG(INFO, "Retrying to unregister interrupt handler");
 882                 start = rte_get_timer_cycles();
 883         } while (true);
 884 }
 885
 886 /**
 887  * Handle DEVX interrupts from the NIC.
 888  * This function is probably called from the DPDK host thread.
 889  *
 890  * @param cb_arg
 891  *   Callback argument.
 892  */
 893 void
 894 mlx5_dev_interrupt_handler_devx(void *cb_arg)
 895 {
 896 #ifndef HAVE_IBV_DEVX_ASYNC
 897         (void)cb_arg;
 898         return;
 899 #else
 900         struct mlx5_dev_ctx_shared *sh = cb_arg;
 901         union {
 902                 struct mlx5dv_devx_async_cmd_hdr cmd_resp;
 903                 uint8_t buf[MLX5_ST_SZ_BYTES(query_flow_counter_out) +
 904                             MLX5_ST_SZ_BYTES(traffic_counter) +
 905                             sizeof(struct mlx5dv_devx_async_cmd_hdr)];
 906         } out;
 907         uint8_t *buf = out.buf + sizeof(out.cmd_resp);
 908
 909         while (!mlx5_glue->devx_get_async_cmd_comp(sh->devx_comp,
 910                                                    &out.cmd_resp,
 911                                                    sizeof(out.buf)))
 912                 mlx5_flow_async_pool_query_handle
 913                         (sh, (uint64_t)out.cmd_resp.wr_id,
 914                          mlx5_devx_get_out_command_status(buf));
 915 #endif /* HAVE_IBV_DEVX_ASYNC */
 916 }
 917
 918 /**
 919  * DPDK callback to bring the link DOWN.
 920  *
 921  * @param dev
 922  *   Pointer to Ethernet device structure.
 923  *
 924  * @return
 925  *   0 on success, a negative errno value otherwise and rte_errno is set.
 926  */
 927 int
 928 mlx5_set_link_down(struct rte_eth_dev *dev)
 929 {
 930         return mlx5_set_flags(dev, ~IFF_UP, ~IFF_UP);
 931 }
 932
 933 /**
 934  * DPDK callback to bring the link UP.
 935  *
 936  * @param dev
 937  *   Pointer to Ethernet device structure.
 938  *
 939  * @return
 940  *   0 on success, a negative errno value otherwise and rte_errno is set.
 941  */
 942 int
 943 mlx5_set_link_up(struct rte_eth_dev *dev)
 944 {
 945         return mlx5_set_flags(dev, ~IFF_UP, IFF_UP);
 946 }
 947
 948 /**
 949  * Check if mlx5 device was removed.
 950  *
 951  * @param dev
 952  *   Pointer to Ethernet device structure.
 953  *
 954  * @return
 955  *   1 when device is removed, otherwise 0.
 956  */
 957 int
 958 mlx5_is_removed(struct rte_eth_dev *dev)
 959 {
 960         struct ibv_device_attr device_attr;
 961         struct mlx5_priv *priv = dev->data->dev_private;
 962
 963         if (mlx5_glue->query_device(priv->sh->ctx, &device_attr) == EIO)
 964                 return 1;
 965         return 0;
 966 }
 967
 968 /**
 969  * Analyze gathered port parameters via sysfs to recognize master
 970  * and representor devices for E-Switch configuration.
 971  *
 972  * @param[in] device_dir
 973  *   flag of presence of "device" directory under port device key.
 974  * @param[inout] switch_info
 975  *   Port information, including port name as a number and port name
 976  *   type if recognized
 977  *
 978  * @return
 979  *   master and representor flags are set in switch_info according to
 980  *   recognized parameters (if any).
 981  */
 982 static void
 983 mlx5_sysfs_check_switch_info(bool device_dir,
 984                              struct mlx5_switch_info *switch_info)
 985 {
 986         switch (switch_info->name_type) {
 987         case MLX5_PHYS_PORT_NAME_TYPE_UNKNOWN:
 988                 /*
 989                  * Name is not recognized, assume the master,
 990                  * check the device directory presence.
 991                  */
 992                 switch_info->master = device_dir;
 993                 break;
 994         case MLX5_PHYS_PORT_NAME_TYPE_NOTSET:
 995                 /*
 996                  * Name is not set, this assumes the legacy naming
 997                  * schema for master, just check if there is
 998                  * a device directory.
 999                  */
1000                 switch_info->master = device_dir;
1001                 break;
1002         case MLX5_PHYS_PORT_NAME_TYPE_UPLINK:
1003                 /* New uplink naming schema recognized. */
1004                 switch_info->master = 1;
1005                 break;
1006         case MLX5_PHYS_PORT_NAME_TYPE_LEGACY:
1007                 /* Legacy representors naming schema. */
1008                 switch_info->representor = !device_dir;
1009                 break;
1010         case MLX5_PHYS_PORT_NAME_TYPE_PFHPF:
1011                 /* Fallthrough */
1012         case MLX5_PHYS_PORT_NAME_TYPE_PFVF:
1013                 /* New representors naming schema. */
1014                 switch_info->representor = 1;
1015                 break;
1016         }
1017 }
1018
1019 /**
1020  * Get switch information associated with network interface.
1021  *
1022  * @param ifindex
1023  *   Network interface index.
1024  * @param[out] info
1025  *   Switch information object, populated in case of success.
1026  *
1027  * @return
1028  *   0 on success, a negative errno value otherwise and rte_errno is set.
1029  */
1030 int
1031 mlx5_sysfs_switch_info(unsigned int ifindex, struct mlx5_switch_info *info)
1032 {
1033         char ifname[IF_NAMESIZE];
1034         char port_name[IF_NAMESIZE];
1035         FILE *file;
1036         struct mlx5_switch_info data = {
1037                 .master = 0,
1038                 .representor = 0,
1039                 .name_type = MLX5_PHYS_PORT_NAME_TYPE_NOTSET,
1040                 .port_name = 0,
1041                 .switch_id = 0,
1042         };
1043         DIR *dir;
1044         bool port_switch_id_set = false;
1045         bool device_dir = false;
1046         char c;
1047         int ret;
1048
1049         if (!if_indextoname(ifindex, ifname)) {
1050                 rte_errno = errno;
1051                 return -rte_errno;
1052         }
1053
1054         MKSTR(phys_port_name, "/sys/class/net/%s/phys_port_name",
1055               ifname);
1056         MKSTR(phys_switch_id, "/sys/class/net/%s/phys_switch_id",
1057               ifname);
1058         MKSTR(pci_device, "/sys/class/net/%s/device",
1059               ifname);
1060
1061         file = fopen(phys_port_name, "rb");
1062         if (file != NULL) {
1063                 ret = fscanf(file, "%" RTE_STR(IF_NAMESIZE) "s", port_name);
1064                 fclose(file);
1065                 if (ret == 1)
1066                         mlx5_translate_port_name(port_name, &data);
1067         }
1068         file = fopen(phys_switch_id, "rb");
1069         if (file == NULL) {
1070                 rte_errno = errno;
1071                 return -rte_errno;
1072         }
1073         port_switch_id_set =
1074                 fscanf(file, "%" SCNx64 "%c", &data.switch_id, &c) == 2 &&
1075                 c == '\n';
1076         fclose(file);
1077         dir = opendir(pci_device);
1078         if (dir != NULL) {
1079                 closedir(dir);
1080                 device_dir = true;
1081         }
1082         if (port_switch_id_set) {
1083                 /* We have some E-Switch configuration. */
1084                 mlx5_sysfs_check_switch_info(device_dir, &data);
1085         }
1086         *info = data;
1087         MLX5_ASSERT(!(data.master && data.representor));
1088         if (data.master && data.representor) {
1089                 DRV_LOG(ERR, "ifindex %u device is recognized as master"
1090                              " and as representor", ifindex);
1091                 rte_errno = ENODEV;
1092                 return -rte_errno;
1093         }
1094         return 0;
1095 }
1096
1097 /**
1098  * Get bond information associated with network interface.
1099  *
1100  * @param pf_ifindex
1101  *   Network interface index of bond slave interface
1102  * @param[out] ifindex
1103  *   Pointer to bond ifindex.
1104  * @param[out] ifname
1105  *   Pointer to bond ifname.
1106  *
1107  * @return
1108  *   0 on success, a negative errno value otherwise and rte_errno is set.
1109  */
1110 int
1111 mlx5_sysfs_bond_info(unsigned int pf_ifindex, unsigned int *ifindex,
1112                      char *ifname)
1113 {
1114         char name[IF_NAMESIZE];
1115         FILE *file;
1116         unsigned int index;
1117         int ret;
1118
1119         if (!if_indextoname(pf_ifindex, name) || !strlen(name)) {
1120                 rte_errno = errno;
1121                 return -rte_errno;
1122         }
1123         MKSTR(bond_if, "/sys/class/net/%s/master/ifindex", name);
1124         /* read bond ifindex */
1125         file = fopen(bond_if, "rb");
1126         if (file == NULL) {
1127                 rte_errno = errno;
1128                 return -rte_errno;
1129         }
1130         ret = fscanf(file, "%u", &index);
1131         fclose(file);
1132         if (ret <= 0) {
1133                 rte_errno = errno;
1134                 return -rte_errno;
1135         }
1136         if (ifindex)
1137                 *ifindex = index;
1138
1139         /* read bond device name from symbol link */
1140         if (ifname) {
1141                 if (!if_indextoname(index, ifname)) {
1142                         rte_errno = errno;
1143                         return -rte_errno;
1144                 }
1145         }
1146         return 0;
1147 }
1148
1149 /**
1150  * DPDK callback to retrieve plug-in module EEPROM information (type and size).
1151  *
1152  * @param dev
1153  *   Pointer to Ethernet device structure.
1154  * @param[out] modinfo
1155  *   Storage for plug-in module EEPROM information.
1156  *
1157  * @return
1158  *   0 on success, a negative errno value otherwise and rte_errno is set.
1159  */
1160 int
1161 mlx5_get_module_info(struct rte_eth_dev *dev,
1162                      struct rte_eth_dev_module_info *modinfo)
1163 {
1164         struct ethtool_modinfo info = {
1165                 .cmd = ETHTOOL_GMODULEINFO,
1166         };
1167         struct ifreq ifr = (struct ifreq) {
1168                 .ifr_data = (void *)&info,
1169         };
1170         int ret = 0;
1171
1172         if (!dev || !modinfo) {
1173                 DRV_LOG(WARNING, "missing argument, cannot get module info");
1174                 rte_errno = EINVAL;
1175                 return -rte_errno;
1176         }
1177         ret = mlx5_ifreq(dev, SIOCETHTOOL, &ifr);
1178         if (ret) {
1179                 DRV_LOG(WARNING, "port %u ioctl(SIOCETHTOOL) failed: %s",
1180                         dev->data->port_id, strerror(rte_errno));
1181                 return ret;
1182         }
1183         modinfo->type = info.type;
1184         modinfo->eeprom_len = info.eeprom_len;
1185         return ret;
1186 }
1187
1188 /**
1189  * DPDK callback to retrieve plug-in module EEPROM data.
1190  *
1191  * @param dev
1192  *   Pointer to Ethernet device structure.
1193  * @param[out] info
1194  *   Storage for plug-in module EEPROM data.
1195  *
1196  * @return
1197  *   0 on success, a negative errno value otherwise and rte_errno is set.
1198  */
1199 int mlx5_get_module_eeprom(struct rte_eth_dev *dev,
1200                            struct rte_dev_eeprom_info *info)
1201 {
1202         struct ethtool_eeprom *eeprom;
1203         struct ifreq ifr;
1204         int ret = 0;
1205
1206         if (!dev || !info) {
1207                 DRV_LOG(WARNING, "missing argument, cannot get module eeprom");
1208                 rte_errno = EINVAL;
1209                 return -rte_errno;
1210         }
1211         eeprom = mlx5_malloc(MLX5_MEM_ZERO,
1212                              (sizeof(struct ethtool_eeprom) + info->length), 0,
1213                              SOCKET_ID_ANY);
1214         if (!eeprom) {
1215                 DRV_LOG(WARNING, "port %u cannot allocate memory for "
1216                         "eeprom data", dev->data->port_id);
1217                 rte_errno = ENOMEM;
1218                 return -rte_errno;
1219         }
1220         eeprom->cmd = ETHTOOL_GMODULEEEPROM;
1221         eeprom->offset = info->offset;
1222         eeprom->len = info->length;
1223         ifr = (struct ifreq) {
1224                 .ifr_data = (void *)eeprom,
1225         };
1226         ret = mlx5_ifreq(dev, SIOCETHTOOL, &ifr);
1227         if (ret)
1228                 DRV_LOG(WARNING, "port %u ioctl(SIOCETHTOOL) failed: %s",
1229                         dev->data->port_id, strerror(rte_errno));
1230         else
1231                 rte_memcpy(info->data, eeprom->data, info->length);
1232         mlx5_free(eeprom);
1233         return ret;
1234 }
1235
1236 /**
1237  * Read device counters table.
1238  *
1239  * @param dev
1240  *   Pointer to Ethernet device.
1241  * @param[out] stats
1242  *   Counters table output buffer.
1243  *
1244  * @return
1245  *   0 on success and stats is filled, negative errno value otherwise and
1246  *   rte_errno is set.
1247  */
1248 int
1249 mlx5_os_read_dev_counters(struct rte_eth_dev *dev, uint64_t *stats)
1250 {
1251         struct mlx5_priv *priv = dev->data->dev_private;
1252         struct mlx5_xstats_ctrl *xstats_ctrl = &priv->xstats_ctrl;
1253         unsigned int i;
1254         struct ifreq ifr;
1255         unsigned int stats_sz = xstats_ctrl->stats_n * sizeof(uint64_t);
1256         unsigned char et_stat_buf[sizeof(struct ethtool_stats) + stats_sz];
1257         struct ethtool_stats *et_stats = (struct ethtool_stats *)et_stat_buf;
1258         int ret;
1259
1260         et_stats->cmd = ETHTOOL_GSTATS;
1261         et_stats->n_stats = xstats_ctrl->stats_n;
1262         ifr.ifr_data = (caddr_t)et_stats;
1263         ret = mlx5_ifreq(dev, SIOCETHTOOL, &ifr);
1264         if (ret) {
1265                 DRV_LOG(WARNING,
1266                         "port %u unable to read statistic values from device",
1267                         dev->data->port_id);
1268                 return ret;
1269         }
1270         for (i = 0; i != xstats_ctrl->mlx5_stats_n; ++i) {
1271                 if (xstats_ctrl->info[i].dev) {
1272                         ret = mlx5_os_read_dev_stat(priv,
1273                                             xstats_ctrl->info[i].ctr_name,
1274                                             &stats[i]);
1275                         /* return last xstats counter if fail to read. */
1276                         if (ret == 0)
1277                                 xstats_ctrl->xstats[i] = stats[i];
1278                         else
1279                                 stats[i] = xstats_ctrl->xstats[i];
1280                 } else {
1281                         stats[i] = (uint64_t)
1282                                 et_stats->data[xstats_ctrl->dev_table_idx[i]];
1283                 }
1284         }
1285         return 0;
1286 }
1287
1288 /**
1289  * Query the number of statistics provided by ETHTOOL.
1290  *
1291  * @param dev
1292  *   Pointer to Ethernet device.
1293  *
1294  * @return
1295  *   Number of statistics on success, negative errno value otherwise and
1296  *   rte_errno is set.
1297  */
1298 int
1299 mlx5_os_get_stats_n(struct rte_eth_dev *dev)
1300 {
1301         struct ethtool_drvinfo drvinfo;
1302         struct ifreq ifr;
1303         int ret;
1304
1305         drvinfo.cmd = ETHTOOL_GDRVINFO;
1306         ifr.ifr_data = (caddr_t)&drvinfo;
1307         ret = mlx5_ifreq(dev, SIOCETHTOOL, &ifr);
1308         if (ret) {
1309                 DRV_LOG(WARNING, "port %u unable to query number of statistics",
1310                         dev->data->port_id);
1311                 return ret;
1312         }
1313         return drvinfo.n_stats;
1314 }
1315
1316 static const struct mlx5_counter_ctrl mlx5_counters_init[] = {
1317         {
1318                 .dpdk_name = "rx_port_unicast_bytes",
1319                 .ctr_name = "rx_vport_unicast_bytes",
1320         },
1321         {
1322                 .dpdk_name = "rx_port_multicast_bytes",
1323                 .ctr_name = "rx_vport_multicast_bytes",
1324         },
1325         {
1326                 .dpdk_name = "rx_port_broadcast_bytes",
1327                 .ctr_name = "rx_vport_broadcast_bytes",
1328         },
1329         {
1330                 .dpdk_name = "rx_port_unicast_packets",
1331                 .ctr_name = "rx_vport_unicast_packets",
1332         },
1333         {
1334                 .dpdk_name = "rx_port_multicast_packets",
1335                 .ctr_name = "rx_vport_multicast_packets",
1336         },
1337         {
1338                 .dpdk_name = "rx_port_broadcast_packets",
1339                 .ctr_name = "rx_vport_broadcast_packets",
1340         },
1341         {
1342                 .dpdk_name = "tx_port_unicast_bytes",
1343                 .ctr_name = "tx_vport_unicast_bytes",
1344         },
1345         {
1346                 .dpdk_name = "tx_port_multicast_bytes",
1347                 .ctr_name = "tx_vport_multicast_bytes",
1348         },
1349         {
1350                 .dpdk_name = "tx_port_broadcast_bytes",
1351                 .ctr_name = "tx_vport_broadcast_bytes",
1352         },
1353         {
1354                 .dpdk_name = "tx_port_unicast_packets",
1355                 .ctr_name = "tx_vport_unicast_packets",
1356         },
1357         {
1358                 .dpdk_name = "tx_port_multicast_packets",
1359                 .ctr_name = "tx_vport_multicast_packets",
1360         },
1361         {
1362                 .dpdk_name = "tx_port_broadcast_packets",
1363                 .ctr_name = "tx_vport_broadcast_packets",
1364         },
1365         {
1366                 .dpdk_name = "rx_wqe_err",
1367                 .ctr_name = "rx_wqe_err",
1368         },
1369         {
1370                 .dpdk_name = "rx_crc_errors_phy",
1371                 .ctr_name = "rx_crc_errors_phy",
1372         },
1373         {
1374                 .dpdk_name = "rx_in_range_len_errors_phy",
1375                 .ctr_name = "rx_in_range_len_errors_phy",
1376         },
1377         {
1378                 .dpdk_name = "rx_symbol_err_phy",
1379                 .ctr_name = "rx_symbol_err_phy",
1380         },
1381         {
1382                 .dpdk_name = "tx_errors_phy",
1383                 .ctr_name = "tx_errors_phy",
1384         },
1385         {
1386                 .dpdk_name = "rx_out_of_buffer",
1387                 .ctr_name = "out_of_buffer",
1388                 .dev = 1,
1389         },
1390         {
1391                 .dpdk_name = "tx_packets_phy",
1392                 .ctr_name = "tx_packets_phy",
1393         },
1394         {
1395                 .dpdk_name = "rx_packets_phy",
1396                 .ctr_name = "rx_packets_phy",
1397         },
1398         {
1399                 .dpdk_name = "tx_discards_phy",
1400                 .ctr_name = "tx_discards_phy",
1401         },
1402         {
1403                 .dpdk_name = "rx_discards_phy",
1404                 .ctr_name = "rx_discards_phy",
1405         },
1406         {
1407                 .dpdk_name = "tx_bytes_phy",
1408                 .ctr_name = "tx_bytes_phy",
1409         },
1410         {
1411                 .dpdk_name = "rx_bytes_phy",
1412                 .ctr_name = "rx_bytes_phy",
1413         },
1414         /* Representor only */
1415         {
1416                 .dpdk_name = "rx_packets",
1417                 .ctr_name = "vport_rx_packets",
1418         },
1419         {
1420                 .dpdk_name = "rx_bytes",
1421                 .ctr_name = "vport_rx_bytes",
1422         },
1423         {
1424                 .dpdk_name = "tx_packets",
1425                 .ctr_name = "vport_tx_packets",
1426         },
1427         {
1428                 .dpdk_name = "tx_bytes",
1429                 .ctr_name = "vport_tx_bytes",
1430         },
1431 };
1432
1433 static const unsigned int xstats_n = RTE_DIM(mlx5_counters_init);
1434
1435 /**
1436  * Init the structures to read device counters.
1437  *
1438  * @param dev
1439  *   Pointer to Ethernet device.
1440  */
1441 void
1442 mlx5_os_stats_init(struct rte_eth_dev *dev)
1443 {
1444         struct mlx5_priv *priv = dev->data->dev_private;
1445         struct mlx5_xstats_ctrl *xstats_ctrl = &priv->xstats_ctrl;
1446         struct mlx5_stats_ctrl *stats_ctrl = &priv->stats_ctrl;
1447         unsigned int i;
1448         unsigned int j;
1449         struct ifreq ifr;
1450         struct ethtool_gstrings *strings = NULL;
1451         unsigned int dev_stats_n;
1452         unsigned int str_sz;
1453         int ret;
1454
1455         /* So that it won't aggregate for each init. */
1456         xstats_ctrl->mlx5_stats_n = 0;
1457         ret = mlx5_os_get_stats_n(dev);
1458         if (ret < 0) {
1459                 DRV_LOG(WARNING, "port %u no extended statistics available",
1460                         dev->data->port_id);
1461                 return;
1462         }
1463         dev_stats_n = ret;
1464         /* Allocate memory to grab stat names and values. */
1465         str_sz = dev_stats_n * ETH_GSTRING_LEN;
1466         strings = (struct ethtool_gstrings *)
1467                   mlx5_malloc(0, str_sz + sizeof(struct ethtool_gstrings), 0,
1468                               SOCKET_ID_ANY);
1469         if (!strings) {
1470                 DRV_LOG(WARNING, "port %u unable to allocate memory for xstats",
1471                      dev->data->port_id);
1472                 return;
1473         }
1474         strings->cmd = ETHTOOL_GSTRINGS;
1475         strings->string_set = ETH_SS_STATS;
1476         strings->len = dev_stats_n;
1477         ifr.ifr_data = (caddr_t)strings;
1478         ret = mlx5_ifreq(dev, SIOCETHTOOL, &ifr);
1479         if (ret) {
1480                 DRV_LOG(WARNING, "port %u unable to get statistic names",
1481                         dev->data->port_id);
1482                 goto free;
1483         }
1484         for (i = 0; i != dev_stats_n; ++i) {
1485                 const char *curr_string = (const char *)
1486                         &strings->data[i * ETH_GSTRING_LEN];
1487
1488                 for (j = 0; j != xstats_n; ++j) {
1489                         if (!strcmp(mlx5_counters_init[j].ctr_name,
1490                                     curr_string)) {
1491                                 unsigned int idx = xstats_ctrl->mlx5_stats_n++;
1492
1493                                 xstats_ctrl->dev_table_idx[idx] = i;
1494                                 xstats_ctrl->info[idx] = mlx5_counters_init[j];
1495                                 break;
1496                         }
1497                 }
1498         }
1499         /* Add dev counters. */
1500         for (i = 0; i != xstats_n; ++i) {
1501                 if (mlx5_counters_init[i].dev) {
1502                         unsigned int idx = xstats_ctrl->mlx5_stats_n++;
1503
1504                         xstats_ctrl->info[idx] = mlx5_counters_init[i];
1505                         xstats_ctrl->hw_stats[idx] = 0;
1506                 }
1507         }
1508         MLX5_ASSERT(xstats_ctrl->mlx5_stats_n <= MLX5_MAX_XSTATS);
1509         xstats_ctrl->stats_n = dev_stats_n;
1510         /* Copy to base at first time. */
1511         ret = mlx5_os_read_dev_counters(dev, xstats_ctrl->base);
1512         if (ret)
1513                 DRV_LOG(ERR, "port %u cannot read device counters: %s",
1514                         dev->data->port_id, strerror(rte_errno));
1515         mlx5_os_read_dev_stat(priv, "out_of_buffer", &stats_ctrl->imissed_base);
1516         stats_ctrl->imissed = 0;
1517 free:
1518         mlx5_free(strings);
1519 }
1520
1521 /**
1522  * Get MAC address by querying netdevice.
1523  *
1524  * @param[in] dev
1525  *   Pointer to Ethernet device.
1526  * @param[out] mac
1527  *   MAC address output buffer.
1528  *
1529  * @return
1530  *   0 on success, a negative errno value otherwise and rte_errno is set.
1531  */
1532 int
1533 mlx5_get_mac(struct rte_eth_dev *dev, uint8_t (*mac)[RTE_ETHER_ADDR_LEN])
1534 {
1535         struct ifreq request;
1536         int ret;
1537
1538         ret = mlx5_ifreq(dev, SIOCGIFHWADDR, &request);
1539         if (ret)
1540                 return ret;
1541         memcpy(mac, request.ifr_hwaddr.sa_data, RTE_ETHER_ADDR_LEN);
1542         return 0;
1543 }
1544