drivers/net/mlx5/linux/mlx5_ethdev_os.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright 2015 6WIND S.A.
   3  * Copyright 2015 Mellanox Technologies, Ltd
   4  */
   5
   6 #include <stddef.h>
   7 #include <inttypes.h>
   8 #include <unistd.h>
   9 #include <stdbool.h>
  10 #include <stdint.h>
  11 #include <stdio.h>
  12 #include <string.h>
  13 #include <stdlib.h>
  14 #include <errno.h>
  15 #include <dirent.h>
  16 #include <net/if.h>
  17 #include <sys/ioctl.h>
  18 #include <sys/socket.h>
  19 #include <netinet/in.h>
  20 #include <linux/ethtool.h>
  21 #include <linux/sockios.h>
  22 #include <fcntl.h>
  23 #include <stdalign.h>
  24 #include <sys/un.h>
  25 #include <time.h>
  26
  27 #include <ethdev_driver.h>
  28 #include <rte_bus_pci.h>
  29 #include <rte_mbuf.h>
  30 #include <rte_common.h>
  31 #include <rte_interrupts.h>
  32 #include <rte_malloc.h>
  33 #include <rte_string_fns.h>
  34 #include <rte_rwlock.h>
  35 #include <rte_cycles.h>
  36
  37 #include <mlx5_glue.h>
  38 #include <mlx5_devx_cmds.h>
  39 #include <mlx5_common.h>
  40 #include <mlx5_malloc.h>
  41
  42 #include "mlx5.h"
  43 #include "mlx5_rxtx.h"
  44 #include "mlx5_utils.h"
  45
  46 /* Supported speed values found in /usr/include/linux/ethtool.h */
  47 #ifndef HAVE_SUPPORTED_40000baseKR4_Full
  48 #define SUPPORTED_40000baseKR4_Full (1 << 23)
  49 #endif
  50 #ifndef HAVE_SUPPORTED_40000baseCR4_Full
  51 #define SUPPORTED_40000baseCR4_Full (1 << 24)
  52 #endif
  53 #ifndef HAVE_SUPPORTED_40000baseSR4_Full
  54 #define SUPPORTED_40000baseSR4_Full (1 << 25)
  55 #endif
  56 #ifndef HAVE_SUPPORTED_40000baseLR4_Full
  57 #define SUPPORTED_40000baseLR4_Full (1 << 26)
  58 #endif
  59 #ifndef HAVE_SUPPORTED_56000baseKR4_Full
  60 #define SUPPORTED_56000baseKR4_Full (1 << 27)
  61 #endif
  62 #ifndef HAVE_SUPPORTED_56000baseCR4_Full
  63 #define SUPPORTED_56000baseCR4_Full (1 << 28)
  64 #endif
  65 #ifndef HAVE_SUPPORTED_56000baseSR4_Full
  66 #define SUPPORTED_56000baseSR4_Full (1 << 29)
  67 #endif
  68 #ifndef HAVE_SUPPORTED_56000baseLR4_Full
  69 #define SUPPORTED_56000baseLR4_Full (1 << 30)
  70 #endif
  71
  72 /* Add defines in case the running kernel is not the same as user headers. */
  73 #ifndef ETHTOOL_GLINKSETTINGS
  74 struct ethtool_link_settings {
  75         uint32_t cmd;
  76         uint32_t speed;
  77         uint8_t duplex;
  78         uint8_t port;
  79         uint8_t phy_address;
  80         uint8_t autoneg;
  81         uint8_t mdio_support;
  82         uint8_t eth_to_mdix;
  83         uint8_t eth_tp_mdix_ctrl;
  84         int8_t link_mode_masks_nwords;
  85         uint32_t reserved[8];
  86         uint32_t link_mode_masks[];
  87 };
  88
  89 /* The kernel values can be found in /include/uapi/linux/ethtool.h */
  90 #define ETHTOOL_GLINKSETTINGS 0x0000004c
  91 #define ETHTOOL_LINK_MODE_1000baseT_Full_BIT 5
  92 #define ETHTOOL_LINK_MODE_Autoneg_BIT 6
  93 #define ETHTOOL_LINK_MODE_1000baseKX_Full_BIT 17
  94 #define ETHTOOL_LINK_MODE_10000baseKX4_Full_BIT 18
  95 #define ETHTOOL_LINK_MODE_10000baseKR_Full_BIT 19
  96 #define ETHTOOL_LINK_MODE_10000baseR_FEC_BIT 20
  97 #define ETHTOOL_LINK_MODE_20000baseMLD2_Full_BIT 21
  98 #define ETHTOOL_LINK_MODE_20000baseKR2_Full_BIT 22
  99 #define ETHTOOL_LINK_MODE_40000baseKR4_Full_BIT 23
 100 #define ETHTOOL_LINK_MODE_40000baseCR4_Full_BIT 24
 101 #define ETHTOOL_LINK_MODE_40000baseSR4_Full_BIT 25
 102 #define ETHTOOL_LINK_MODE_40000baseLR4_Full_BIT 26
 103 #define ETHTOOL_LINK_MODE_56000baseKR4_Full_BIT 27
 104 #define ETHTOOL_LINK_MODE_56000baseCR4_Full_BIT 28
 105 #define ETHTOOL_LINK_MODE_56000baseSR4_Full_BIT 29
 106 #define ETHTOOL_LINK_MODE_56000baseLR4_Full_BIT 30
 107 #endif
 108 #ifndef HAVE_ETHTOOL_LINK_MODE_25G
 109 #define ETHTOOL_LINK_MODE_25000baseCR_Full_BIT 31
 110 #define ETHTOOL_LINK_MODE_25000baseKR_Full_BIT 32
 111 #define ETHTOOL_LINK_MODE_25000baseSR_Full_BIT 33
 112 #endif
 113 #ifndef HAVE_ETHTOOL_LINK_MODE_50G
 114 #define ETHTOOL_LINK_MODE_50000baseCR2_Full_BIT 34
 115 #define ETHTOOL_LINK_MODE_50000baseKR2_Full_BIT 35
 116 #endif
 117 #ifndef HAVE_ETHTOOL_LINK_MODE_100G
 118 #define ETHTOOL_LINK_MODE_100000baseKR4_Full_BIT 36
 119 #define ETHTOOL_LINK_MODE_100000baseSR4_Full_BIT 37
 120 #define ETHTOOL_LINK_MODE_100000baseCR4_Full_BIT 38
 121 #define ETHTOOL_LINK_MODE_100000baseLR4_ER4_Full_BIT 39
 122 #endif
 123 #ifndef HAVE_ETHTOOL_LINK_MODE_200G
 124 #define ETHTOOL_LINK_MODE_200000baseKR4_Full_BIT 62
 125 #define ETHTOOL_LINK_MODE_200000baseSR4_Full_BIT 63
 126 #define ETHTOOL_LINK_MODE_200000baseLR4_ER4_FR4_Full_BIT 0 /* 64 - 64 */
 127 #define ETHTOOL_LINK_MODE_200000baseDR4_Full_BIT 1 /* 65 - 64 */
 128 #define ETHTOOL_LINK_MODE_200000baseCR4_Full_BIT 2 /* 66 - 64 */
 129 #endif
 130
 131
 132 /**
 133  * Get interface name from private structure.
 134  *
 135  * This is a port representor-aware version of mlx5_get_ifname_sysfs().
 136  *
 137  * @param[in] dev
 138  *   Pointer to Ethernet device.
 139  * @param[out] ifname
 140  *   Interface name output buffer.
 141  *
 142  * @return
 143  *   0 on success, a negative errno value otherwise and rte_errno is set.
 144  */
 145 int
 146 mlx5_get_ifname(const struct rte_eth_dev *dev, char (*ifname)[MLX5_NAMESIZE])
 147 {
 148         struct mlx5_priv *priv = dev->data->dev_private;
 149         unsigned int ifindex;
 150
 151         MLX5_ASSERT(priv);
 152         MLX5_ASSERT(priv->sh);
 153         if (priv->master && priv->sh->bond.ifindex > 0) {
 154                 memcpy(ifname, priv->sh->bond.ifname, MLX5_NAMESIZE);
 155                 return 0;
 156         }
 157         ifindex = mlx5_ifindex(dev);
 158         if (!ifindex) {
 159                 if (!priv->representor)
 160                         return mlx5_get_ifname_sysfs(priv->sh->ibdev_path,
 161                                                      *ifname);
 162                 rte_errno = ENXIO;
 163                 return -rte_errno;
 164         }
 165         if (if_indextoname(ifindex, &(*ifname)[0]))
 166                 return 0;
 167         rte_errno = errno;
 168         return -rte_errno;
 169 }
 170
 171 /**
 172  * Perform ifreq ioctl() on associated Ethernet device.
 173  *
 174  * @param[in] dev
 175  *   Pointer to Ethernet device.
 176  * @param req
 177  *   Request number to pass to ioctl().
 178  * @param[out] ifr
 179  *   Interface request structure output buffer.
 180  *
 181  * @return
 182  *   0 on success, a negative errno value otherwise and rte_errno is set.
 183  */
 184 static int
 185 mlx5_ifreq(const struct rte_eth_dev *dev, int req, struct ifreq *ifr)
 186 {
 187         int sock = socket(PF_INET, SOCK_DGRAM, IPPROTO_IP);
 188         int ret = 0;
 189
 190         if (sock == -1) {
 191                 rte_errno = errno;
 192                 return -rte_errno;
 193         }
 194         ret = mlx5_get_ifname(dev, &ifr->ifr_name);
 195         if (ret)
 196                 goto error;
 197         ret = ioctl(sock, req, ifr);
 198         if (ret == -1) {
 199                 rte_errno = errno;
 200                 goto error;
 201         }
 202         close(sock);
 203         return 0;
 204 error:
 205         close(sock);
 206         return -rte_errno;
 207 }
 208
 209 /**
 210  * Get device MTU.
 211  *
 212  * @param dev
 213  *   Pointer to Ethernet device.
 214  * @param[out] mtu
 215  *   MTU value output buffer.
 216  *
 217  * @return
 218  *   0 on success, a negative errno value otherwise and rte_errno is set.
 219  */
 220 int
 221 mlx5_get_mtu(struct rte_eth_dev *dev, uint16_t *mtu)
 222 {
 223         struct ifreq request;
 224         int ret = mlx5_ifreq(dev, SIOCGIFMTU, &request);
 225
 226         if (ret)
 227                 return ret;
 228         *mtu = request.ifr_mtu;
 229         return 0;
 230 }
 231
 232 /**
 233  * Set device MTU.
 234  *
 235  * @param dev
 236  *   Pointer to Ethernet device.
 237  * @param mtu
 238  *   MTU value to set.
 239  *
 240  * @return
 241  *   0 on success, a negative errno value otherwise and rte_errno is set.
 242  */
 243 int
 244 mlx5_set_mtu(struct rte_eth_dev *dev, uint16_t mtu)
 245 {
 246         struct ifreq request = { .ifr_mtu = mtu, };
 247
 248         return mlx5_ifreq(dev, SIOCSIFMTU, &request);
 249 }
 250
 251 /**
 252  * Set device flags.
 253  *
 254  * @param dev
 255  *   Pointer to Ethernet device.
 256  * @param keep
 257  *   Bitmask for flags that must remain untouched.
 258  * @param flags
 259  *   Bitmask for flags to modify.
 260  *
 261  * @return
 262  *   0 on success, a negative errno value otherwise and rte_errno is set.
 263  */
 264 static int
 265 mlx5_set_flags(struct rte_eth_dev *dev, unsigned int keep, unsigned int flags)
 266 {
 267         struct ifreq request;
 268         int ret = mlx5_ifreq(dev, SIOCGIFFLAGS, &request);
 269
 270         if (ret)
 271                 return ret;
 272         request.ifr_flags &= keep;
 273         request.ifr_flags |= flags & ~keep;
 274         return mlx5_ifreq(dev, SIOCSIFFLAGS, &request);
 275 }
 276
 277 /**
 278  * Get device current raw clock counter
 279  *
 280  * @param dev
 281  *   Pointer to Ethernet device structure.
 282  * @param[out] time
 283  *   Current raw clock counter of the device.
 284  *
 285  * @return
 286  *   0 if the clock has correctly been read
 287  *   The value of errno in case of error
 288  */
 289 int
 290 mlx5_read_clock(struct rte_eth_dev *dev, uint64_t *clock)
 291 {
 292         struct mlx5_priv *priv = dev->data->dev_private;
 293         struct ibv_context *ctx = priv->sh->ctx;
 294         struct ibv_values_ex values;
 295         int err = 0;
 296
 297         values.comp_mask = IBV_VALUES_MASK_RAW_CLOCK;
 298         err = mlx5_glue->query_rt_values_ex(ctx, &values);
 299         if (err != 0) {
 300                 DRV_LOG(WARNING, "Could not query the clock !");
 301                 return err;
 302         }
 303         *clock = values.raw_clock.tv_nsec;
 304         return 0;
 305 }
 306
 307 /**
 308  * Retrieve the master device for representor in the same switch domain.
 309  *
 310  * @param dev
 311  *   Pointer to representor Ethernet device structure.
 312  *
 313  * @return
 314  *   Master device structure  on success, NULL otherwise.
 315  */
 316 static struct rte_eth_dev *
 317 mlx5_find_master_dev(struct rte_eth_dev *dev)
 318 {
 319         struct mlx5_priv *priv;
 320         uint16_t port_id;
 321         uint16_t domain_id;
 322
 323         priv = dev->data->dev_private;
 324         domain_id = priv->domain_id;
 325         MLX5_ASSERT(priv->representor);
 326         MLX5_ETH_FOREACH_DEV(port_id, priv->pci_dev) {
 327                 struct mlx5_priv *opriv =
 328                         rte_eth_devices[port_id].data->dev_private;
 329                 if (opriv &&
 330                     opriv->master &&
 331                     opriv->domain_id == domain_id &&
 332                     opriv->sh == priv->sh)
 333                         return &rte_eth_devices[port_id];
 334         }
 335         return NULL;
 336 }
 337
 338 /**
 339  * DPDK callback to retrieve physical link information.
 340  *
 341  * @param dev
 342  *   Pointer to Ethernet device structure.
 343  * @param[out] link
 344  *   Storage for current link status.
 345  *
 346  * @return
 347  *   0 on success, a negative errno value otherwise and rte_errno is set.
 348  */
 349 static int
 350 mlx5_link_update_unlocked_gset(struct rte_eth_dev *dev,
 351                                struct rte_eth_link *link)
 352 {
 353         struct mlx5_priv *priv = dev->data->dev_private;
 354         struct ethtool_cmd edata = {
 355                 .cmd = ETHTOOL_GSET /* Deprecated since Linux v4.5. */
 356         };
 357         struct ifreq ifr;
 358         struct rte_eth_link dev_link;
 359         int link_speed = 0;
 360         int ret;
 361
 362         ret = mlx5_ifreq(dev, SIOCGIFFLAGS, &ifr);
 363         if (ret) {
 364                 DRV_LOG(WARNING, "port %u ioctl(SIOCGIFFLAGS) failed: %s",
 365                         dev->data->port_id, strerror(rte_errno));
 366                 return ret;
 367         }
 368         dev_link = (struct rte_eth_link) {
 369                 .link_status = ((ifr.ifr_flags & IFF_UP) &&
 370                                 (ifr.ifr_flags & IFF_RUNNING)),
 371         };
 372         ifr = (struct ifreq) {
 373                 .ifr_data = (void *)&edata,
 374         };
 375         ret = mlx5_ifreq(dev, SIOCETHTOOL, &ifr);
 376         if (ret) {
 377                 if (ret == -ENOTSUP && priv->representor) {
 378                         struct rte_eth_dev *master;
 379
 380                         /*
 381                          * For representors we can try to inherit link
 382                          * settings from the master device. Actually
 383                          * link settings do not make a lot of sense
 384                          * for representors due to missing physical
 385                          * link. The old kernel drivers supported
 386                          * emulated settings query for representors,
 387                          * the new ones do not, so we have to add
 388                          * this code for compatibility issues.
 389                          */
 390                         master = mlx5_find_master_dev(dev);
 391                         if (master) {
 392                                 ifr = (struct ifreq) {
 393                                         .ifr_data = (void *)&edata,
 394                                 };
 395                                 ret = mlx5_ifreq(master, SIOCETHTOOL, &ifr);
 396                         }
 397                 }
 398                 if (ret) {
 399                         DRV_LOG(WARNING,
 400                                 "port %u ioctl(SIOCETHTOOL,"
 401                                 " ETHTOOL_GSET) failed: %s",
 402                                 dev->data->port_id, strerror(rte_errno));
 403                         return ret;
 404                 }
 405         }
 406         link_speed = ethtool_cmd_speed(&edata);
 407         if (link_speed == -1)
 408                 dev_link.link_speed = ETH_SPEED_NUM_UNKNOWN;
 409         else
 410                 dev_link.link_speed = link_speed;
 411         priv->link_speed_capa = 0;
 412         if (edata.supported & SUPPORTED_Autoneg)
 413                 priv->link_speed_capa |= ETH_LINK_SPEED_AUTONEG;
 414         if (edata.supported & (SUPPORTED_1000baseT_Full |
 415                                SUPPORTED_1000baseKX_Full))
 416                 priv->link_speed_capa |= ETH_LINK_SPEED_1G;
 417         if (edata.supported & SUPPORTED_10000baseKR_Full)
 418                 priv->link_speed_capa |= ETH_LINK_SPEED_10G;
 419         if (edata.supported & (SUPPORTED_40000baseKR4_Full |
 420                                SUPPORTED_40000baseCR4_Full |
 421                                SUPPORTED_40000baseSR4_Full |
 422                                SUPPORTED_40000baseLR4_Full))
 423                 priv->link_speed_capa |= ETH_LINK_SPEED_40G;
 424         dev_link.link_duplex = ((edata.duplex == DUPLEX_HALF) ?
 425                                 ETH_LINK_HALF_DUPLEX : ETH_LINK_FULL_DUPLEX);
 426         dev_link.link_autoneg = !(dev->data->dev_conf.link_speeds &
 427                         ETH_LINK_SPEED_FIXED);
 428         *link = dev_link;
 429         return 0;
 430 }
 431
 432 /**
 433  * Retrieve physical link information (unlocked version using new ioctl).
 434  *
 435  * @param dev
 436  *   Pointer to Ethernet device structure.
 437  * @param[out] link
 438  *   Storage for current link status.
 439  *
 440  * @return
 441  *   0 on success, a negative errno value otherwise and rte_errno is set.
 442  */
 443 static int
 444 mlx5_link_update_unlocked_gs(struct rte_eth_dev *dev,
 445                              struct rte_eth_link *link)
 446
 447 {
 448         struct mlx5_priv *priv = dev->data->dev_private;
 449         struct ethtool_link_settings gcmd = { .cmd = ETHTOOL_GLINKSETTINGS };
 450         struct ifreq ifr;
 451         struct rte_eth_link dev_link;
 452         struct rte_eth_dev *master = NULL;
 453         uint64_t sc;
 454         int ret;
 455
 456         ret = mlx5_ifreq(dev, SIOCGIFFLAGS, &ifr);
 457         if (ret) {
 458                 DRV_LOG(WARNING, "port %u ioctl(SIOCGIFFLAGS) failed: %s",
 459                         dev->data->port_id, strerror(rte_errno));
 460                 return ret;
 461         }
 462         dev_link = (struct rte_eth_link) {
 463                 .link_status = ((ifr.ifr_flags & IFF_UP) &&
 464                                 (ifr.ifr_flags & IFF_RUNNING)),
 465         };
 466         ifr = (struct ifreq) {
 467                 .ifr_data = (void *)&gcmd,
 468         };
 469         ret = mlx5_ifreq(dev, SIOCETHTOOL, &ifr);
 470         if (ret) {
 471                 if (ret == -ENOTSUP && priv->representor) {
 472                         /*
 473                          * For representors we can try to inherit link
 474                          * settings from the master device. Actually
 475                          * link settings do not make a lot of sense
 476                          * for representors due to missing physical
 477                          * link. The old kernel drivers supported
 478                          * emulated settings query for representors,
 479                          * the new ones do not, so we have to add
 480                          * this code for compatibility issues.
 481                          */
 482                         master = mlx5_find_master_dev(dev);
 483                         if (master) {
 484                                 ifr = (struct ifreq) {
 485                                         .ifr_data = (void *)&gcmd,
 486                                 };
 487                                 ret = mlx5_ifreq(master, SIOCETHTOOL, &ifr);
 488                         }
 489                 }
 490                 if (ret) {
 491                         DRV_LOG(DEBUG,
 492                                 "port %u ioctl(SIOCETHTOOL,"
 493                                 " ETHTOOL_GLINKSETTINGS) failed: %s",
 494                                 dev->data->port_id, strerror(rte_errno));
 495                         return ret;
 496                 }
 497         }
 498         gcmd.link_mode_masks_nwords = -gcmd.link_mode_masks_nwords;
 499
 500         alignas(struct ethtool_link_settings)
 501         uint8_t data[offsetof(struct ethtool_link_settings, link_mode_masks) +
 502                      sizeof(uint32_t) * gcmd.link_mode_masks_nwords * 3];
 503         struct ethtool_link_settings *ecmd = (void *)data;
 504
 505         *ecmd = gcmd;
 506         ifr.ifr_data = (void *)ecmd;
 507         ret = mlx5_ifreq(master ? master : dev, SIOCETHTOOL, &ifr);
 508         if (ret) {
 509                 DRV_LOG(DEBUG,
 510                         "port %u ioctl(SIOCETHTOOL,"
 511                         "ETHTOOL_GLINKSETTINGS) failed: %s",
 512                         dev->data->port_id, strerror(rte_errno));
 513                 return ret;
 514         }
 515         dev_link.link_speed = (ecmd->speed == UINT32_MAX) ?
 516                                 ETH_SPEED_NUM_UNKNOWN : ecmd->speed;
 517         sc = ecmd->link_mode_masks[0] |
 518                 ((uint64_t)ecmd->link_mode_masks[1] << 32);
 519         priv->link_speed_capa = 0;
 520         if (sc & MLX5_BITSHIFT(ETHTOOL_LINK_MODE_Autoneg_BIT))
 521                 priv->link_speed_capa |= ETH_LINK_SPEED_AUTONEG;
 522         if (sc & (MLX5_BITSHIFT(ETHTOOL_LINK_MODE_1000baseT_Full_BIT) |
 523                   MLX5_BITSHIFT(ETHTOOL_LINK_MODE_1000baseKX_Full_BIT)))
 524                 priv->link_speed_capa |= ETH_LINK_SPEED_1G;
 525         if (sc & (MLX5_BITSHIFT(ETHTOOL_LINK_MODE_10000baseKX4_Full_BIT) |
 526                   MLX5_BITSHIFT(ETHTOOL_LINK_MODE_10000baseKR_Full_BIT) |
 527                   MLX5_BITSHIFT(ETHTOOL_LINK_MODE_10000baseR_FEC_BIT)))
 528                 priv->link_speed_capa |= ETH_LINK_SPEED_10G;
 529         if (sc & (MLX5_BITSHIFT(ETHTOOL_LINK_MODE_20000baseMLD2_Full_BIT) |
 530                   MLX5_BITSHIFT(ETHTOOL_LINK_MODE_20000baseKR2_Full_BIT)))
 531                 priv->link_speed_capa |= ETH_LINK_SPEED_20G;
 532         if (sc & (MLX5_BITSHIFT(ETHTOOL_LINK_MODE_40000baseKR4_Full_BIT) |
 533                   MLX5_BITSHIFT(ETHTOOL_LINK_MODE_40000baseCR4_Full_BIT) |
 534                   MLX5_BITSHIFT(ETHTOOL_LINK_MODE_40000baseSR4_Full_BIT) |
 535                   MLX5_BITSHIFT(ETHTOOL_LINK_MODE_40000baseLR4_Full_BIT)))
 536                 priv->link_speed_capa |= ETH_LINK_SPEED_40G;
 537         if (sc & (MLX5_BITSHIFT(ETHTOOL_LINK_MODE_56000baseKR4_Full_BIT) |
 538                   MLX5_BITSHIFT(ETHTOOL_LINK_MODE_56000baseCR4_Full_BIT) |
 539                   MLX5_BITSHIFT(ETHTOOL_LINK_MODE_56000baseSR4_Full_BIT) |
 540                   MLX5_BITSHIFT(ETHTOOL_LINK_MODE_56000baseLR4_Full_BIT)))
 541                 priv->link_speed_capa |= ETH_LINK_SPEED_56G;
 542         if (sc & (MLX5_BITSHIFT(ETHTOOL_LINK_MODE_25000baseCR_Full_BIT) |
 543                   MLX5_BITSHIFT(ETHTOOL_LINK_MODE_25000baseKR_Full_BIT) |
 544                   MLX5_BITSHIFT(ETHTOOL_LINK_MODE_25000baseSR_Full_BIT)))
 545                 priv->link_speed_capa |= ETH_LINK_SPEED_25G;
 546         if (sc & (MLX5_BITSHIFT(ETHTOOL_LINK_MODE_50000baseCR2_Full_BIT) |
 547                   MLX5_BITSHIFT(ETHTOOL_LINK_MODE_50000baseKR2_Full_BIT)))
 548                 priv->link_speed_capa |= ETH_LINK_SPEED_50G;
 549         if (sc & (MLX5_BITSHIFT(ETHTOOL_LINK_MODE_100000baseKR4_Full_BIT) |
 550                   MLX5_BITSHIFT(ETHTOOL_LINK_MODE_100000baseSR4_Full_BIT) |
 551                   MLX5_BITSHIFT(ETHTOOL_LINK_MODE_100000baseCR4_Full_BIT) |
 552                   MLX5_BITSHIFT(ETHTOOL_LINK_MODE_100000baseLR4_ER4_Full_BIT)))
 553                 priv->link_speed_capa |= ETH_LINK_SPEED_100G;
 554         if (sc & (MLX5_BITSHIFT(ETHTOOL_LINK_MODE_200000baseKR4_Full_BIT) |
 555                   MLX5_BITSHIFT(ETHTOOL_LINK_MODE_200000baseSR4_Full_BIT)))
 556                 priv->link_speed_capa |= ETH_LINK_SPEED_200G;
 557
 558         sc = ecmd->link_mode_masks[2] |
 559                 ((uint64_t)ecmd->link_mode_masks[3] << 32);
 560         if (sc & (MLX5_BITSHIFT(ETHTOOL_LINK_MODE_200000baseCR4_Full_BIT) |
 561                   MLX5_BITSHIFT
 562                        (ETHTOOL_LINK_MODE_200000baseLR4_ER4_FR4_Full_BIT) |
 563                   MLX5_BITSHIFT(ETHTOOL_LINK_MODE_200000baseDR4_Full_BIT)))
 564                 priv->link_speed_capa |= ETH_LINK_SPEED_200G;
 565         dev_link.link_duplex = ((ecmd->duplex == DUPLEX_HALF) ?
 566                                 ETH_LINK_HALF_DUPLEX : ETH_LINK_FULL_DUPLEX);
 567         dev_link.link_autoneg = !(dev->data->dev_conf.link_speeds &
 568                                   ETH_LINK_SPEED_FIXED);
 569         *link = dev_link;
 570         return 0;
 571 }
 572
 573 /**
 574  * DPDK callback to retrieve physical link information.
 575  *
 576  * @param dev
 577  *   Pointer to Ethernet device structure.
 578  * @param wait_to_complete
 579  *   Wait for request completion.
 580  *
 581  * @return
 582  *   0 if link status was not updated, positive if it was, a negative errno
 583  *   value otherwise and rte_errno is set.
 584  */
 585 int
 586 mlx5_link_update(struct rte_eth_dev *dev, int wait_to_complete)
 587 {
 588         int ret;
 589         struct rte_eth_link dev_link;
 590         time_t start_time = time(NULL);
 591         int retry = MLX5_GET_LINK_STATUS_RETRY_COUNT;
 592
 593         do {
 594                 ret = mlx5_link_update_unlocked_gs(dev, &dev_link);
 595                 if (ret == -ENOTSUP)
 596                         ret = mlx5_link_update_unlocked_gset(dev, &dev_link);
 597                 if (ret == 0)
 598                         break;
 599                 /* Handle wait to complete situation. */
 600                 if ((wait_to_complete || retry) && ret == -EAGAIN) {
 601                         if (abs((int)difftime(time(NULL), start_time)) <
 602                             MLX5_LINK_STATUS_TIMEOUT) {
 603                                 usleep(0);
 604                                 continue;
 605                         } else {
 606                                 rte_errno = EBUSY;
 607                                 return -rte_errno;
 608                         }
 609                 } else if (ret < 0) {
 610                         return ret;
 611                 }
 612         } while (wait_to_complete || retry-- > 0);
 613         ret = !!memcmp(&dev->data->dev_link, &dev_link,
 614                        sizeof(struct rte_eth_link));
 615         dev->data->dev_link = dev_link;
 616         return ret;
 617 }
 618
 619 /**
 620  * DPDK callback to get flow control status.
 621  *
 622  * @param dev
 623  *   Pointer to Ethernet device structure.
 624  * @param[out] fc_conf
 625  *   Flow control output buffer.
 626  *
 627  * @return
 628  *   0 on success, a negative errno value otherwise and rte_errno is set.
 629  */
 630 int
 631 mlx5_dev_get_flow_ctrl(struct rte_eth_dev *dev, struct rte_eth_fc_conf *fc_conf)
 632 {
 633         struct ifreq ifr;
 634         struct ethtool_pauseparam ethpause = {
 635                 .cmd = ETHTOOL_GPAUSEPARAM
 636         };
 637         int ret;
 638
 639         ifr.ifr_data = (void *)&ethpause;
 640         ret = mlx5_ifreq(dev, SIOCETHTOOL, &ifr);
 641         if (ret) {
 642                 DRV_LOG(WARNING,
 643                         "port %u ioctl(SIOCETHTOOL, ETHTOOL_GPAUSEPARAM) failed:"
 644                         " %s",
 645                         dev->data->port_id, strerror(rte_errno));
 646                 return ret;
 647         }
 648         fc_conf->autoneg = ethpause.autoneg;
 649         if (ethpause.rx_pause && ethpause.tx_pause)
 650                 fc_conf->mode = RTE_FC_FULL;
 651         else if (ethpause.rx_pause)
 652                 fc_conf->mode = RTE_FC_RX_PAUSE;
 653         else if (ethpause.tx_pause)
 654                 fc_conf->mode = RTE_FC_TX_PAUSE;
 655         else
 656                 fc_conf->mode = RTE_FC_NONE;
 657         return 0;
 658 }
 659
 660 /**
 661  * DPDK callback to modify flow control parameters.
 662  *
 663  * @param dev
 664  *   Pointer to Ethernet device structure.
 665  * @param[in] fc_conf
 666  *   Flow control parameters.
 667  *
 668  * @return
 669  *   0 on success, a negative errno value otherwise and rte_errno is set.
 670  */
 671 int
 672 mlx5_dev_set_flow_ctrl(struct rte_eth_dev *dev, struct rte_eth_fc_conf *fc_conf)
 673 {
 674         struct ifreq ifr;
 675         struct ethtool_pauseparam ethpause = {
 676                 .cmd = ETHTOOL_SPAUSEPARAM
 677         };
 678         int ret;
 679
 680         ifr.ifr_data = (void *)&ethpause;
 681         ethpause.autoneg = fc_conf->autoneg;
 682         if (((fc_conf->mode & RTE_FC_FULL) == RTE_FC_FULL) ||
 683             (fc_conf->mode & RTE_FC_RX_PAUSE))
 684                 ethpause.rx_pause = 1;
 685         else
 686                 ethpause.rx_pause = 0;
 687
 688         if (((fc_conf->mode & RTE_FC_FULL) == RTE_FC_FULL) ||
 689             (fc_conf->mode & RTE_FC_TX_PAUSE))
 690                 ethpause.tx_pause = 1;
 691         else
 692                 ethpause.tx_pause = 0;
 693         ret = mlx5_ifreq(dev, SIOCETHTOOL, &ifr);
 694         if (ret) {
 695                 DRV_LOG(WARNING,
 696                         "port %u ioctl(SIOCETHTOOL, ETHTOOL_SPAUSEPARAM)"
 697                         " failed: %s",
 698                         dev->data->port_id, strerror(rte_errno));
 699                 return ret;
 700         }
 701         return 0;
 702 }
 703
 704 /**
 705  * Handle asynchronous removal event for entire multiport device.
 706  *
 707  * @param sh
 708  *   Infiniband device shared context.
 709  */
 710 static void
 711 mlx5_dev_interrupt_device_fatal(struct mlx5_dev_ctx_shared *sh)
 712 {
 713         uint32_t i;
 714
 715         for (i = 0; i < sh->max_port; ++i) {
 716                 struct rte_eth_dev *dev;
 717
 718                 if (sh->port[i].ih_port_id >= RTE_MAX_ETHPORTS) {
 719                         /*
 720                          * Or not existing port either no
 721                          * handler installed for this port.
 722                          */
 723                         continue;
 724                 }
 725                 dev = &rte_eth_devices[sh->port[i].ih_port_id];
 726                 MLX5_ASSERT(dev);
 727                 if (dev->data->dev_conf.intr_conf.rmv)
 728                         rte_eth_dev_callback_process
 729                                 (dev, RTE_ETH_EVENT_INTR_RMV, NULL);
 730         }
 731 }
 732
 733 /**
 734  * Handle shared asynchronous events the NIC (removal event
 735  * and link status change). Supports multiport IB device.
 736  *
 737  * @param cb_arg
 738  *   Callback argument.
 739  */
 740 void
 741 mlx5_dev_interrupt_handler(void *cb_arg)
 742 {
 743         struct mlx5_dev_ctx_shared *sh = cb_arg;
 744         struct ibv_async_event event;
 745
 746         /* Read all message from the IB device and acknowledge them. */
 747         for (;;) {
 748                 struct rte_eth_dev *dev;
 749                 uint32_t tmp;
 750
 751                 if (mlx5_glue->get_async_event(sh->ctx, &event))
 752                         break;
 753                 /* Retrieve and check IB port index. */
 754                 tmp = (uint32_t)event.element.port_num;
 755                 if (!tmp && event.event_type == IBV_EVENT_DEVICE_FATAL) {
 756                         /*
 757                          * The DEVICE_FATAL event is called once for
 758                          * entire device without port specifying.
 759                          * We should notify all existing ports.
 760                          */
 761                         mlx5_glue->ack_async_event(&event);
 762                         mlx5_dev_interrupt_device_fatal(sh);
 763                         continue;
 764                 }
 765                 MLX5_ASSERT(tmp && (tmp <= sh->max_port));
 766                 if (!tmp) {
 767                         /* Unsupported device level event. */
 768                         mlx5_glue->ack_async_event(&event);
 769                         DRV_LOG(DEBUG,
 770                                 "unsupported common event (type %d)",
 771                                 event.event_type);
 772                         continue;
 773                 }
 774                 if (tmp > sh->max_port) {
 775                         /* Invalid IB port index. */
 776                         mlx5_glue->ack_async_event(&event);
 777                         DRV_LOG(DEBUG,
 778                                 "cannot handle an event (type %d)"
 779                                 "due to invalid IB port index (%u)",
 780                                 event.event_type, tmp);
 781                         continue;
 782                 }
 783                 if (sh->port[tmp - 1].ih_port_id >= RTE_MAX_ETHPORTS) {
 784                         /* No handler installed. */
 785                         mlx5_glue->ack_async_event(&event);
 786                         DRV_LOG(DEBUG,
 787                                 "cannot handle an event (type %d)"
 788                                 "due to no handler installed for port %u",
 789                                 event.event_type, tmp);
 790                         continue;
 791                 }
 792                 /* Retrieve ethernet device descriptor. */
 793                 tmp = sh->port[tmp - 1].ih_port_id;
 794                 dev = &rte_eth_devices[tmp];
 795                 MLX5_ASSERT(dev);
 796                 if ((event.event_type == IBV_EVENT_PORT_ACTIVE ||
 797                      event.event_type == IBV_EVENT_PORT_ERR) &&
 798                         dev->data->dev_conf.intr_conf.lsc) {
 799                         mlx5_glue->ack_async_event(&event);
 800                         if (mlx5_link_update(dev, 0) == -EAGAIN) {
 801                                 usleep(0);
 802                                 continue;
 803                         }
 804                         rte_eth_dev_callback_process
 805                                 (dev, RTE_ETH_EVENT_INTR_LSC, NULL);
 806                         continue;
 807                 }
 808                 DRV_LOG(DEBUG,
 809                         "port %u cannot handle an unknown event (type %d)",
 810                         dev->data->port_id, event.event_type);
 811                 mlx5_glue->ack_async_event(&event);
 812         }
 813 }
 814
 815 /*
 816  * Unregister callback handler safely. The handler may be active
 817  * while we are trying to unregister it, in this case code -EAGAIN
 818  * is returned by rte_intr_callback_unregister(). This routine checks
 819  * the return code and tries to unregister handler again.
 820  *
 821  * @param handle
 822  *   interrupt handle
 823  * @param cb_fn
 824  *   pointer to callback routine
 825  * @cb_arg
 826  *   opaque callback parameter
 827  */
 828 void
 829 mlx5_intr_callback_unregister(const struct rte_intr_handle *handle,
 830                               rte_intr_callback_fn cb_fn, void *cb_arg)
 831 {
 832         /*
 833          * Try to reduce timeout management overhead by not calling
 834          * the timer related routines on the first iteration. If the
 835          * unregistering succeeds on first call there will be no
 836          * timer calls at all.
 837          */
 838         uint64_t twait = 0;
 839         uint64_t start = 0;
 840
 841         do {
 842                 int ret;
 843
 844                 ret = rte_intr_callback_unregister(handle, cb_fn, cb_arg);
 845                 if (ret >= 0)
 846                         return;
 847                 if (ret != -EAGAIN) {
 848                         DRV_LOG(INFO, "failed to unregister interrupt"
 849                                       " handler (error: %d)", ret);
 850                         MLX5_ASSERT(false);
 851                         return;
 852                 }
 853                 if (twait) {
 854                         struct timespec onems;
 855
 856                         /* Wait one millisecond and try again. */
 857                         onems.tv_sec = 0;
 858                         onems.tv_nsec = NS_PER_S / MS_PER_S;
 859                         nanosleep(&onems, 0);
 860                         /* Check whether one second elapsed. */
 861                         if ((rte_get_timer_cycles() - start) <= twait)
 862                                 continue;
 863                 } else {
 864                         /*
 865                          * We get the amount of timer ticks for one second.
 866                          * If this amount elapsed it means we spent one
 867                          * second in waiting. This branch is executed once
 868                          * on first iteration.
 869                          */
 870                         twait = rte_get_timer_hz();
 871                         MLX5_ASSERT(twait);
 872                 }
 873                 /*
 874                  * Timeout elapsed, show message (once a second) and retry.
 875                  * We have no other acceptable option here, if we ignore
 876                  * the unregistering return code the handler will not
 877                  * be unregistered, fd will be closed and we may get the
 878                  * crush. Hanging and messaging in the loop seems not to be
 879                  * the worst choice.
 880                  */
 881                 DRV_LOG(INFO, "Retrying to unregister interrupt handler");
 882                 start = rte_get_timer_cycles();
 883         } while (true);
 884 }
 885
 886 /**
 887  * Handle DEVX interrupts from the NIC.
 888  * This function is probably called from the DPDK host thread.
 889  *
 890  * @param cb_arg
 891  *   Callback argument.
 892  */
 893 void
 894 mlx5_dev_interrupt_handler_devx(void *cb_arg)
 895 {
 896 #ifndef HAVE_IBV_DEVX_ASYNC
 897         (void)cb_arg;
 898         return;
 899 #else
 900         struct mlx5_dev_ctx_shared *sh = cb_arg;
 901         union {
 902                 struct mlx5dv_devx_async_cmd_hdr cmd_resp;
 903                 uint8_t buf[MLX5_ST_SZ_BYTES(query_flow_counter_out) +
 904                             MLX5_ST_SZ_BYTES(traffic_counter) +
 905                             sizeof(struct mlx5dv_devx_async_cmd_hdr)];
 906         } out;
 907         uint8_t *buf = out.buf + sizeof(out.cmd_resp);
 908
 909         while (!mlx5_glue->devx_get_async_cmd_comp(sh->devx_comp,
 910                                                    &out.cmd_resp,
 911                                                    sizeof(out.buf)))
 912                 mlx5_flow_async_pool_query_handle
 913                         (sh, (uint64_t)out.cmd_resp.wr_id,
 914                          mlx5_devx_get_out_command_status(buf));
 915 #endif /* HAVE_IBV_DEVX_ASYNC */
 916 }
 917
 918 /**
 919  * DPDK callback to bring the link DOWN.
 920  *
 921  * @param dev
 922  *   Pointer to Ethernet device structure.
 923  *
 924  * @return
 925  *   0 on success, a negative errno value otherwise and rte_errno is set.
 926  */
 927 int
 928 mlx5_set_link_down(struct rte_eth_dev *dev)
 929 {
 930         return mlx5_set_flags(dev, ~IFF_UP, ~IFF_UP);
 931 }
 932
 933 /**
 934  * DPDK callback to bring the link UP.
 935  *
 936  * @param dev
 937  *   Pointer to Ethernet device structure.
 938  *
 939  * @return
 940  *   0 on success, a negative errno value otherwise and rte_errno is set.
 941  */
 942 int
 943 mlx5_set_link_up(struct rte_eth_dev *dev)
 944 {
 945         return mlx5_set_flags(dev, ~IFF_UP, IFF_UP);
 946 }
 947
 948 /**
 949  * Check if mlx5 device was removed.
 950  *
 951  * @param dev
 952  *   Pointer to Ethernet device structure.
 953  *
 954  * @return
 955  *   1 when device is removed, otherwise 0.
 956  */
 957 int
 958 mlx5_is_removed(struct rte_eth_dev *dev)
 959 {
 960         struct ibv_device_attr device_attr;
 961         struct mlx5_priv *priv = dev->data->dev_private;
 962
 963         if (mlx5_glue->query_device(priv->sh->ctx, &device_attr) == EIO)
 964                 return 1;
 965         return 0;
 966 }
 967
 968 /**
 969  * Analyze gathered port parameters via sysfs to recognize master
 970  * and representor devices for E-Switch configuration.
 971  *
 972  * @param[in] device_dir
 973  *   flag of presence of "device" directory under port device key.
 974  * @param[inout] switch_info
 975  *   Port information, including port name as a number and port name
 976  *   type if recognized
 977  *
 978  * @return
 979  *   master and representor flags are set in switch_info according to
 980  *   recognized parameters (if any).
 981  */
 982 static void
 983 mlx5_sysfs_check_switch_info(bool device_dir,
 984                              struct mlx5_switch_info *switch_info)
 985 {
 986         switch (switch_info->name_type) {
 987         case MLX5_PHYS_PORT_NAME_TYPE_UNKNOWN:
 988                 /*
 989                  * Name is not recognized, assume the master,
 990                  * check the device directory presence.
 991                  */
 992                 switch_info->master = device_dir;
 993                 break;
 994         case MLX5_PHYS_PORT_NAME_TYPE_NOTSET:
 995                 /*
 996                  * Name is not set, this assumes the legacy naming
 997                  * schema for master, just check if there is
 998                  * a device directory.
 999                  */
1000                 switch_info->master = device_dir;
1001                 break;
1002         case MLX5_PHYS_PORT_NAME_TYPE_UPLINK:
1003                 /* New uplink naming schema recognized. */
1004                 switch_info->master = 1;
1005                 break;
1006         case MLX5_PHYS_PORT_NAME_TYPE_LEGACY:
1007                 /* Legacy representors naming schema. */
1008                 switch_info->representor = !device_dir;
1009                 break;
1010         case MLX5_PHYS_PORT_NAME_TYPE_PFHPF:
1011                 /* Fallthrough */
1012         case MLX5_PHYS_PORT_NAME_TYPE_PFVF:
1013                 /* Fallthrough */
1014         case MLX5_PHYS_PORT_NAME_TYPE_PFSF:
1015                 /* New representors naming schema. */
1016                 switch_info->representor = 1;
1017                 break;
1018         default:
1019                 switch_info->master = device_dir;
1020                 break;
1021         }
1022 }
1023
1024 /**
1025  * Get switch information associated with network interface.
1026  *
1027  * @param ifindex
1028  *   Network interface index.
1029  * @param[out] info
1030  *   Switch information object, populated in case of success.
1031  *
1032  * @return
1033  *   0 on success, a negative errno value otherwise and rte_errno is set.
1034  */
1035 int
1036 mlx5_sysfs_switch_info(unsigned int ifindex, struct mlx5_switch_info *info)
1037 {
1038         char ifname[IF_NAMESIZE];
1039         char port_name[IF_NAMESIZE];
1040         FILE *file;
1041         struct mlx5_switch_info data = {
1042                 .master = 0,
1043                 .representor = 0,
1044                 .name_type = MLX5_PHYS_PORT_NAME_TYPE_NOTSET,
1045                 .port_name = 0,
1046                 .switch_id = 0,
1047         };
1048         DIR *dir;
1049         bool port_switch_id_set = false;
1050         bool device_dir = false;
1051         char c;
1052         int ret;
1053
1054         if (!if_indextoname(ifindex, ifname)) {
1055                 rte_errno = errno;
1056                 return -rte_errno;
1057         }
1058
1059         MKSTR(phys_port_name, "/sys/class/net/%s/phys_port_name",
1060               ifname);
1061         MKSTR(phys_switch_id, "/sys/class/net/%s/phys_switch_id",
1062               ifname);
1063         MKSTR(pci_device, "/sys/class/net/%s/device",
1064               ifname);
1065
1066         file = fopen(phys_port_name, "rb");
1067         if (file != NULL) {
1068                 ret = fscanf(file, "%" RTE_STR(IF_NAMESIZE) "s", port_name);
1069                 fclose(file);
1070                 if (ret == 1)
1071                         mlx5_translate_port_name(port_name, &data);
1072         }
1073         file = fopen(phys_switch_id, "rb");
1074         if (file == NULL) {
1075                 rte_errno = errno;
1076                 return -rte_errno;
1077         }
1078         port_switch_id_set =
1079                 fscanf(file, "%" SCNx64 "%c", &data.switch_id, &c) == 2 &&
1080                 c == '\n';
1081         fclose(file);
1082         dir = opendir(pci_device);
1083         if (dir != NULL) {
1084                 closedir(dir);
1085                 device_dir = true;
1086         }
1087         if (port_switch_id_set) {
1088                 /* We have some E-Switch configuration. */
1089                 mlx5_sysfs_check_switch_info(device_dir, &data);
1090         }
1091         *info = data;
1092         MLX5_ASSERT(!(data.master && data.representor));
1093         if (data.master && data.representor) {
1094                 DRV_LOG(ERR, "ifindex %u device is recognized as master"
1095                              " and as representor", ifindex);
1096                 rte_errno = ENODEV;
1097                 return -rte_errno;
1098         }
1099         return 0;
1100 }
1101
1102 /**
1103  * Get bond information associated with network interface.
1104  *
1105  * @param pf_ifindex
1106  *   Network interface index of bond slave interface
1107  * @param[out] ifindex
1108  *   Pointer to bond ifindex.
1109  * @param[out] ifname
1110  *   Pointer to bond ifname.
1111  *
1112  * @return
1113  *   0 on success, a negative errno value otherwise and rte_errno is set.
1114  */
1115 int
1116 mlx5_sysfs_bond_info(unsigned int pf_ifindex, unsigned int *ifindex,
1117                      char *ifname)
1118 {
1119         char name[IF_NAMESIZE];
1120         FILE *file;
1121         unsigned int index;
1122         int ret;
1123
1124         if (!if_indextoname(pf_ifindex, name) || !strlen(name)) {
1125                 rte_errno = errno;
1126                 return -rte_errno;
1127         }
1128         MKSTR(bond_if, "/sys/class/net/%s/master/ifindex", name);
1129         /* read bond ifindex */
1130         file = fopen(bond_if, "rb");
1131         if (file == NULL) {
1132                 rte_errno = errno;
1133                 return -rte_errno;
1134         }
1135         ret = fscanf(file, "%u", &index);
1136         fclose(file);
1137         if (ret <= 0) {
1138                 rte_errno = errno;
1139                 return -rte_errno;
1140         }
1141         if (ifindex)
1142                 *ifindex = index;
1143
1144         /* read bond device name from symbol link */
1145         if (ifname) {
1146                 if (!if_indextoname(index, ifname)) {
1147                         rte_errno = errno;
1148                         return -rte_errno;
1149                 }
1150         }
1151         return 0;
1152 }
1153
1154 /**
1155  * DPDK callback to retrieve plug-in module EEPROM information (type and size).
1156  *
1157  * @param dev
1158  *   Pointer to Ethernet device structure.
1159  * @param[out] modinfo
1160  *   Storage for plug-in module EEPROM information.
1161  *
1162  * @return
1163  *   0 on success, a negative errno value otherwise and rte_errno is set.
1164  */
1165 int
1166 mlx5_get_module_info(struct rte_eth_dev *dev,
1167                      struct rte_eth_dev_module_info *modinfo)
1168 {
1169         struct ethtool_modinfo info = {
1170                 .cmd = ETHTOOL_GMODULEINFO,
1171         };
1172         struct ifreq ifr = (struct ifreq) {
1173                 .ifr_data = (void *)&info,
1174         };
1175         int ret = 0;
1176
1177         if (!dev || !modinfo) {
1178                 DRV_LOG(WARNING, "missing argument, cannot get module info");
1179                 rte_errno = EINVAL;
1180                 return -rte_errno;
1181         }
1182         ret = mlx5_ifreq(dev, SIOCETHTOOL, &ifr);
1183         if (ret) {
1184                 DRV_LOG(WARNING, "port %u ioctl(SIOCETHTOOL) failed: %s",
1185                         dev->data->port_id, strerror(rte_errno));
1186                 return ret;
1187         }
1188         modinfo->type = info.type;
1189         modinfo->eeprom_len = info.eeprom_len;
1190         return ret;
1191 }
1192
1193 /**
1194  * DPDK callback to retrieve plug-in module EEPROM data.
1195  *
1196  * @param dev
1197  *   Pointer to Ethernet device structure.
1198  * @param[out] info
1199  *   Storage for plug-in module EEPROM data.
1200  *
1201  * @return
1202  *   0 on success, a negative errno value otherwise and rte_errno is set.
1203  */
1204 int mlx5_get_module_eeprom(struct rte_eth_dev *dev,
1205                            struct rte_dev_eeprom_info *info)
1206 {
1207         struct ethtool_eeprom *eeprom;
1208         struct ifreq ifr;
1209         int ret = 0;
1210
1211         if (!dev || !info) {
1212                 DRV_LOG(WARNING, "missing argument, cannot get module eeprom");
1213                 rte_errno = EINVAL;
1214                 return -rte_errno;
1215         }
1216         eeprom = mlx5_malloc(MLX5_MEM_ZERO,
1217                              (sizeof(struct ethtool_eeprom) + info->length), 0,
1218                              SOCKET_ID_ANY);
1219         if (!eeprom) {
1220                 DRV_LOG(WARNING, "port %u cannot allocate memory for "
1221                         "eeprom data", dev->data->port_id);
1222                 rte_errno = ENOMEM;
1223                 return -rte_errno;
1224         }
1225         eeprom->cmd = ETHTOOL_GMODULEEEPROM;
1226         eeprom->offset = info->offset;
1227         eeprom->len = info->length;
1228         ifr = (struct ifreq) {
1229                 .ifr_data = (void *)eeprom,
1230         };
1231         ret = mlx5_ifreq(dev, SIOCETHTOOL, &ifr);
1232         if (ret)
1233                 DRV_LOG(WARNING, "port %u ioctl(SIOCETHTOOL) failed: %s",
1234                         dev->data->port_id, strerror(rte_errno));
1235         else
1236                 rte_memcpy(info->data, eeprom->data, info->length);
1237         mlx5_free(eeprom);
1238         return ret;
1239 }
1240
1241 /**
1242  * Read device counters table.
1243  *
1244  * @param dev
1245  *   Pointer to Ethernet device.
1246  * @param[out] stats
1247  *   Counters table output buffer.
1248  *
1249  * @return
1250  *   0 on success and stats is filled, negative errno value otherwise and
1251  *   rte_errno is set.
1252  */
1253 int
1254 mlx5_os_read_dev_counters(struct rte_eth_dev *dev, uint64_t *stats)
1255 {
1256         struct mlx5_priv *priv = dev->data->dev_private;
1257         struct mlx5_xstats_ctrl *xstats_ctrl = &priv->xstats_ctrl;
1258         unsigned int i;
1259         struct ifreq ifr;
1260         unsigned int stats_sz = xstats_ctrl->stats_n * sizeof(uint64_t);
1261         unsigned char et_stat_buf[sizeof(struct ethtool_stats) + stats_sz];
1262         struct ethtool_stats *et_stats = (struct ethtool_stats *)et_stat_buf;
1263         int ret;
1264
1265         et_stats->cmd = ETHTOOL_GSTATS;
1266         et_stats->n_stats = xstats_ctrl->stats_n;
1267         ifr.ifr_data = (caddr_t)et_stats;
1268         ret = mlx5_ifreq(dev, SIOCETHTOOL, &ifr);
1269         if (ret) {
1270                 DRV_LOG(WARNING,
1271                         "port %u unable to read statistic values from device",
1272                         dev->data->port_id);
1273                 return ret;
1274         }
1275         for (i = 0; i != xstats_ctrl->mlx5_stats_n; ++i) {
1276                 if (xstats_ctrl->info[i].dev) {
1277                         ret = mlx5_os_read_dev_stat(priv,
1278                                             xstats_ctrl->info[i].ctr_name,
1279                                             &stats[i]);
1280                         /* return last xstats counter if fail to read. */
1281                         if (ret == 0)
1282                                 xstats_ctrl->xstats[i] = stats[i];
1283                         else
1284                                 stats[i] = xstats_ctrl->xstats[i];
1285                 } else {
1286                         stats[i] = (uint64_t)
1287                                 et_stats->data[xstats_ctrl->dev_table_idx[i]];
1288                 }
1289         }
1290         return 0;
1291 }
1292
1293 /**
1294  * Query the number of statistics provided by ETHTOOL.
1295  *
1296  * @param dev
1297  *   Pointer to Ethernet device.
1298  *
1299  * @return
1300  *   Number of statistics on success, negative errno value otherwise and
1301  *   rte_errno is set.
1302  */
1303 int
1304 mlx5_os_get_stats_n(struct rte_eth_dev *dev)
1305 {
1306         struct ethtool_drvinfo drvinfo;
1307         struct ifreq ifr;
1308         int ret;
1309
1310         drvinfo.cmd = ETHTOOL_GDRVINFO;
1311         ifr.ifr_data = (caddr_t)&drvinfo;
1312         ret = mlx5_ifreq(dev, SIOCETHTOOL, &ifr);
1313         if (ret) {
1314                 DRV_LOG(WARNING, "port %u unable to query number of statistics",
1315                         dev->data->port_id);
1316                 return ret;
1317         }
1318         return drvinfo.n_stats;
1319 }
1320
1321 static const struct mlx5_counter_ctrl mlx5_counters_init[] = {
1322         {
1323                 .dpdk_name = "rx_unicast_bytes",
1324                 .ctr_name = "rx_vport_unicast_bytes",
1325         },
1326         {
1327                 .dpdk_name = "rx_multicast_bytes",
1328                 .ctr_name = "rx_vport_multicast_bytes",
1329         },
1330         {
1331                 .dpdk_name = "rx_broadcast_bytes",
1332                 .ctr_name = "rx_vport_broadcast_bytes",
1333         },
1334         {
1335                 .dpdk_name = "rx_unicast_packets",
1336                 .ctr_name = "rx_vport_unicast_packets",
1337         },
1338         {
1339                 .dpdk_name = "rx_multicast_packets",
1340                 .ctr_name = "rx_vport_multicast_packets",
1341         },
1342         {
1343                 .dpdk_name = "rx_broadcast_packets",
1344                 .ctr_name = "rx_vport_broadcast_packets",
1345         },
1346         {
1347                 .dpdk_name = "tx_unicast_bytes",
1348                 .ctr_name = "tx_vport_unicast_bytes",
1349         },
1350         {
1351                 .dpdk_name = "tx_multicast_bytes",
1352                 .ctr_name = "tx_vport_multicast_bytes",
1353         },
1354         {
1355                 .dpdk_name = "tx_broadcast_bytes",
1356                 .ctr_name = "tx_vport_broadcast_bytes",
1357         },
1358         {
1359                 .dpdk_name = "tx_unicast_packets",
1360                 .ctr_name = "tx_vport_unicast_packets",
1361         },
1362         {
1363                 .dpdk_name = "tx_multicast_packets",
1364                 .ctr_name = "tx_vport_multicast_packets",
1365         },
1366         {
1367                 .dpdk_name = "tx_broadcast_packets",
1368                 .ctr_name = "tx_vport_broadcast_packets",
1369         },
1370         {
1371                 .dpdk_name = "rx_wqe_errors",
1372                 .ctr_name = "rx_wqe_err",
1373         },
1374         {
1375                 .dpdk_name = "rx_phy_crc_errors",
1376                 .ctr_name = "rx_crc_errors_phy",
1377         },
1378         {
1379                 .dpdk_name = "rx_phy_in_range_len_errors",
1380                 .ctr_name = "rx_in_range_len_errors_phy",
1381         },
1382         {
1383                 .dpdk_name = "rx_phy_symbol_errors",
1384                 .ctr_name = "rx_symbol_err_phy",
1385         },
1386         {
1387                 .dpdk_name = "tx_phy_errors",
1388                 .ctr_name = "tx_errors_phy",
1389         },
1390         {
1391                 .dpdk_name = "rx_out_of_buffer",
1392                 .ctr_name = "out_of_buffer",
1393                 .dev = 1,
1394         },
1395         {
1396                 .dpdk_name = "tx_phy_packets",
1397                 .ctr_name = "tx_packets_phy",
1398         },
1399         {
1400                 .dpdk_name = "rx_phy_packets",
1401                 .ctr_name = "rx_packets_phy",
1402         },
1403         {
1404                 .dpdk_name = "tx_phy_discard_packets",
1405                 .ctr_name = "tx_discards_phy",
1406         },
1407         {
1408                 .dpdk_name = "rx_phy_discard_packets",
1409                 .ctr_name = "rx_discards_phy",
1410         },
1411         {
1412                 .dpdk_name = "tx_phy_bytes",
1413                 .ctr_name = "tx_bytes_phy",
1414         },
1415         {
1416                 .dpdk_name = "rx_phy_bytes",
1417                 .ctr_name = "rx_bytes_phy",
1418         },
1419         /* Representor only */
1420         {
1421                 .dpdk_name = "rx_vport_packets",
1422                 .ctr_name = "vport_rx_packets",
1423         },
1424         {
1425                 .dpdk_name = "rx_vport_bytes",
1426                 .ctr_name = "vport_rx_bytes",
1427         },
1428         {
1429                 .dpdk_name = "tx_vport_packets",
1430                 .ctr_name = "vport_tx_packets",
1431         },
1432         {
1433                 .dpdk_name = "tx_vport_bytes",
1434                 .ctr_name = "vport_tx_bytes",
1435         },
1436 };
1437
1438 static const unsigned int xstats_n = RTE_DIM(mlx5_counters_init);
1439
1440 /**
1441  * Init the structures to read device counters.
1442  *
1443  * @param dev
1444  *   Pointer to Ethernet device.
1445  */
1446 void
1447 mlx5_os_stats_init(struct rte_eth_dev *dev)
1448 {
1449         struct mlx5_priv *priv = dev->data->dev_private;
1450         struct mlx5_xstats_ctrl *xstats_ctrl = &priv->xstats_ctrl;
1451         struct mlx5_stats_ctrl *stats_ctrl = &priv->stats_ctrl;
1452         unsigned int i;
1453         unsigned int j;
1454         struct ifreq ifr;
1455         struct ethtool_gstrings *strings = NULL;
1456         unsigned int dev_stats_n;
1457         unsigned int str_sz;
1458         int ret;
1459
1460         /* So that it won't aggregate for each init. */
1461         xstats_ctrl->mlx5_stats_n = 0;
1462         ret = mlx5_os_get_stats_n(dev);
1463         if (ret < 0) {
1464                 DRV_LOG(WARNING, "port %u no extended statistics available",
1465                         dev->data->port_id);
1466                 return;
1467         }
1468         dev_stats_n = ret;
1469         /* Allocate memory to grab stat names and values. */
1470         str_sz = dev_stats_n * ETH_GSTRING_LEN;
1471         strings = (struct ethtool_gstrings *)
1472                   mlx5_malloc(0, str_sz + sizeof(struct ethtool_gstrings), 0,
1473                               SOCKET_ID_ANY);
1474         if (!strings) {
1475                 DRV_LOG(WARNING, "port %u unable to allocate memory for xstats",
1476                      dev->data->port_id);
1477                 return;
1478         }
1479         strings->cmd = ETHTOOL_GSTRINGS;
1480         strings->string_set = ETH_SS_STATS;
1481         strings->len = dev_stats_n;
1482         ifr.ifr_data = (caddr_t)strings;
1483         ret = mlx5_ifreq(dev, SIOCETHTOOL, &ifr);
1484         if (ret) {
1485                 DRV_LOG(WARNING, "port %u unable to get statistic names",
1486                         dev->data->port_id);
1487                 goto free;
1488         }
1489         for (i = 0; i != dev_stats_n; ++i) {
1490                 const char *curr_string = (const char *)
1491                         &strings->data[i * ETH_GSTRING_LEN];
1492
1493                 for (j = 0; j != xstats_n; ++j) {
1494                         if (!strcmp(mlx5_counters_init[j].ctr_name,
1495                                     curr_string)) {
1496                                 unsigned int idx = xstats_ctrl->mlx5_stats_n++;
1497
1498                                 xstats_ctrl->dev_table_idx[idx] = i;
1499                                 xstats_ctrl->info[idx] = mlx5_counters_init[j];
1500                                 break;
1501                         }
1502                 }
1503         }
1504         /* Add dev counters. */
1505         for (i = 0; i != xstats_n; ++i) {
1506                 if (mlx5_counters_init[i].dev) {
1507                         unsigned int idx = xstats_ctrl->mlx5_stats_n++;
1508
1509                         xstats_ctrl->info[idx] = mlx5_counters_init[i];
1510                         xstats_ctrl->hw_stats[idx] = 0;
1511                 }
1512         }
1513         MLX5_ASSERT(xstats_ctrl->mlx5_stats_n <= MLX5_MAX_XSTATS);
1514         xstats_ctrl->stats_n = dev_stats_n;
1515         /* Copy to base at first time. */
1516         ret = mlx5_os_read_dev_counters(dev, xstats_ctrl->base);
1517         if (ret)
1518                 DRV_LOG(ERR, "port %u cannot read device counters: %s",
1519                         dev->data->port_id, strerror(rte_errno));
1520         mlx5_os_read_dev_stat(priv, "out_of_buffer", &stats_ctrl->imissed_base);
1521         stats_ctrl->imissed = 0;
1522 free:
1523         mlx5_free(strings);
1524 }
1525
1526 /**
1527  * Get MAC address by querying netdevice.
1528  *
1529  * @param[in] dev
1530  *   Pointer to Ethernet device.
1531  * @param[out] mac
1532  *   MAC address output buffer.
1533  *
1534  * @return
1535  *   0 on success, a negative errno value otherwise and rte_errno is set.
1536  */
1537 int
1538 mlx5_get_mac(struct rte_eth_dev *dev, uint8_t (*mac)[RTE_ETHER_ADDR_LEN])
1539 {
1540         struct ifreq request;
1541         int ret;
1542
1543         ret = mlx5_ifreq(dev, SIOCGIFHWADDR, &request);
1544         if (ret)
1545                 return ret;
1546         memcpy(mac, request.ifr_hwaddr.sa_data, RTE_ETHER_ADDR_LEN);
1547         return 0;
1548 }
1549