drivers/net/mlx5/linux/mlx5_ethdev_os.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright 2015 6WIND S.A.
   3  * Copyright 2015 Mellanox Technologies, Ltd
   4  */
   5
   6 #include <stddef.h>
   7 #include <inttypes.h>
   8 #include <unistd.h>
   9 #include <stdbool.h>
  10 #include <stdint.h>
  11 #include <stdio.h>
  12 #include <string.h>
  13 #include <stdlib.h>
  14 #include <errno.h>
  15 #include <dirent.h>
  16 #include <net/if.h>
  17 #include <sys/ioctl.h>
  18 #include <sys/socket.h>
  19 #include <netinet/in.h>
  20 #include <linux/ethtool.h>
  21 #include <linux/sockios.h>
  22 #include <fcntl.h>
  23 #include <stdalign.h>
  24 #include <sys/un.h>
  25 #include <time.h>
  26
  27 #include <ethdev_driver.h>
  28 #include <rte_bus_pci.h>
  29 #include <rte_mbuf.h>
  30 #include <rte_common.h>
  31 #include <rte_interrupts.h>
  32 #include <rte_malloc.h>
  33 #include <rte_string_fns.h>
  34 #include <rte_rwlock.h>
  35 #include <rte_cycles.h>
  36
  37 #include <mlx5_glue.h>
  38 #include <mlx5_devx_cmds.h>
  39 #include <mlx5_common.h>
  40 #include <mlx5_malloc.h>
  41
  42 #include "mlx5.h"
  43 #include "mlx5_rxtx.h"
  44 #include "mlx5_utils.h"
  45
  46 /* Supported speed values found in /usr/include/linux/ethtool.h */
  47 #ifndef HAVE_SUPPORTED_40000baseKR4_Full
  48 #define SUPPORTED_40000baseKR4_Full (1 << 23)
  49 #endif
  50 #ifndef HAVE_SUPPORTED_40000baseCR4_Full
  51 #define SUPPORTED_40000baseCR4_Full (1 << 24)
  52 #endif
  53 #ifndef HAVE_SUPPORTED_40000baseSR4_Full
  54 #define SUPPORTED_40000baseSR4_Full (1 << 25)
  55 #endif
  56 #ifndef HAVE_SUPPORTED_40000baseLR4_Full
  57 #define SUPPORTED_40000baseLR4_Full (1 << 26)
  58 #endif
  59 #ifndef HAVE_SUPPORTED_56000baseKR4_Full
  60 #define SUPPORTED_56000baseKR4_Full (1 << 27)
  61 #endif
  62 #ifndef HAVE_SUPPORTED_56000baseCR4_Full
  63 #define SUPPORTED_56000baseCR4_Full (1 << 28)
  64 #endif
  65 #ifndef HAVE_SUPPORTED_56000baseSR4_Full
  66 #define SUPPORTED_56000baseSR4_Full (1 << 29)
  67 #endif
  68 #ifndef HAVE_SUPPORTED_56000baseLR4_Full
  69 #define SUPPORTED_56000baseLR4_Full (1 << 30)
  70 #endif
  71
  72 /* Add defines in case the running kernel is not the same as user headers. */
  73 #ifndef ETHTOOL_GLINKSETTINGS
  74 struct ethtool_link_settings {
  75         uint32_t cmd;
  76         uint32_t speed;
  77         uint8_t duplex;
  78         uint8_t port;
  79         uint8_t phy_address;
  80         uint8_t autoneg;
  81         uint8_t mdio_support;
  82         uint8_t eth_to_mdix;
  83         uint8_t eth_tp_mdix_ctrl;
  84         int8_t link_mode_masks_nwords;
  85         uint32_t reserved[8];
  86         uint32_t link_mode_masks[];
  87 };
  88
  89 /* The kernel values can be found in /include/uapi/linux/ethtool.h */
  90 #define ETHTOOL_GLINKSETTINGS 0x0000004c
  91 #define ETHTOOL_LINK_MODE_1000baseT_Full_BIT 5
  92 #define ETHTOOL_LINK_MODE_Autoneg_BIT 6
  93 #define ETHTOOL_LINK_MODE_1000baseKX_Full_BIT 17
  94 #define ETHTOOL_LINK_MODE_10000baseKX4_Full_BIT 18
  95 #define ETHTOOL_LINK_MODE_10000baseKR_Full_BIT 19
  96 #define ETHTOOL_LINK_MODE_10000baseR_FEC_BIT 20
  97 #define ETHTOOL_LINK_MODE_20000baseMLD2_Full_BIT 21
  98 #define ETHTOOL_LINK_MODE_20000baseKR2_Full_BIT 22
  99 #define ETHTOOL_LINK_MODE_40000baseKR4_Full_BIT 23
 100 #define ETHTOOL_LINK_MODE_40000baseCR4_Full_BIT 24
 101 #define ETHTOOL_LINK_MODE_40000baseSR4_Full_BIT 25
 102 #define ETHTOOL_LINK_MODE_40000baseLR4_Full_BIT 26
 103 #define ETHTOOL_LINK_MODE_56000baseKR4_Full_BIT 27
 104 #define ETHTOOL_LINK_MODE_56000baseCR4_Full_BIT 28
 105 #define ETHTOOL_LINK_MODE_56000baseSR4_Full_BIT 29
 106 #define ETHTOOL_LINK_MODE_56000baseLR4_Full_BIT 30
 107 #endif
 108 #ifndef HAVE_ETHTOOL_LINK_MODE_25G
 109 #define ETHTOOL_LINK_MODE_25000baseCR_Full_BIT 31
 110 #define ETHTOOL_LINK_MODE_25000baseKR_Full_BIT 32
 111 #define ETHTOOL_LINK_MODE_25000baseSR_Full_BIT 33
 112 #endif
 113 #ifndef HAVE_ETHTOOL_LINK_MODE_50G
 114 #define ETHTOOL_LINK_MODE_50000baseCR2_Full_BIT 34
 115 #define ETHTOOL_LINK_MODE_50000baseKR2_Full_BIT 35
 116 #endif
 117 #ifndef HAVE_ETHTOOL_LINK_MODE_100G
 118 #define ETHTOOL_LINK_MODE_100000baseKR4_Full_BIT 36
 119 #define ETHTOOL_LINK_MODE_100000baseSR4_Full_BIT 37
 120 #define ETHTOOL_LINK_MODE_100000baseCR4_Full_BIT 38
 121 #define ETHTOOL_LINK_MODE_100000baseLR4_ER4_Full_BIT 39
 122 #endif
 123 #ifndef HAVE_ETHTOOL_LINK_MODE_200G
 124 #define ETHTOOL_LINK_MODE_200000baseKR4_Full_BIT 62
 125 #define ETHTOOL_LINK_MODE_200000baseSR4_Full_BIT 63
 126 #define ETHTOOL_LINK_MODE_200000baseLR4_ER4_FR4_Full_BIT 0 /* 64 - 64 */
 127 #define ETHTOOL_LINK_MODE_200000baseDR4_Full_BIT 1 /* 65 - 64 */
 128 #define ETHTOOL_LINK_MODE_200000baseCR4_Full_BIT 2 /* 66 - 64 */
 129 #endif
 130
 131
 132 /**
 133  * Get interface name from private structure.
 134  *
 135  * This is a port representor-aware version of mlx5_get_ifname_sysfs().
 136  *
 137  * @param[in] dev
 138  *   Pointer to Ethernet device.
 139  * @param[out] ifname
 140  *   Interface name output buffer.
 141  *
 142  * @return
 143  *   0 on success, a negative errno value otherwise and rte_errno is set.
 144  */
 145 int
 146 mlx5_get_ifname(const struct rte_eth_dev *dev, char (*ifname)[MLX5_NAMESIZE])
 147 {
 148         struct mlx5_priv *priv = dev->data->dev_private;
 149         unsigned int ifindex;
 150
 151         MLX5_ASSERT(priv);
 152         MLX5_ASSERT(priv->sh);
 153         if (priv->master && priv->sh->bond.ifindex > 0) {
 154                 memcpy(ifname, priv->sh->bond.ifname, MLX5_NAMESIZE);
 155                 return 0;
 156         }
 157         ifindex = mlx5_ifindex(dev);
 158         if (!ifindex) {
 159                 if (!priv->representor)
 160                         return mlx5_get_ifname_sysfs(priv->sh->ibdev_path,
 161                                                      *ifname);
 162                 rte_errno = ENXIO;
 163                 return -rte_errno;
 164         }
 165         if (if_indextoname(ifindex, &(*ifname)[0]))
 166                 return 0;
 167         rte_errno = errno;
 168         return -rte_errno;
 169 }
 170
 171 /**
 172  * Perform ifreq ioctl() on associated netdev ifname.
 173  *
 174  * @param[in] ifname
 175  *   Pointer to netdev name.
 176  * @param req
 177  *   Request number to pass to ioctl().
 178  * @param[out] ifr
 179  *   Interface request structure output buffer.
 180  *
 181  * @return
 182  *   0 on success, a negative errno value otherwise and rte_errno is set.
 183  */
 184 static int
 185 mlx5_ifreq_by_ifname(const char *ifname, int req, struct ifreq *ifr)
 186 {
 187         int sock = socket(PF_INET, SOCK_DGRAM, IPPROTO_IP);
 188         int ret = 0;
 189
 190         if (sock == -1) {
 191                 rte_errno = errno;
 192                 return -rte_errno;
 193         }
 194         rte_strscpy(ifr->ifr_name, ifname, sizeof(ifr->ifr_name));
 195         ret = ioctl(sock, req, ifr);
 196         if (ret == -1) {
 197                 rte_errno = errno;
 198                 goto error;
 199         }
 200         close(sock);
 201         return 0;
 202 error:
 203         close(sock);
 204         return -rte_errno;
 205 }
 206
 207 /**
 208  * Perform ifreq ioctl() on associated Ethernet device.
 209  *
 210  * @param[in] dev
 211  *   Pointer to Ethernet device.
 212  * @param req
 213  *   Request number to pass to ioctl().
 214  * @param[out] ifr
 215  *   Interface request structure output buffer.
 216  *
 217  * @return
 218  *   0 on success, a negative errno value otherwise and rte_errno is set.
 219  */
 220 static int
 221 mlx5_ifreq(const struct rte_eth_dev *dev, int req, struct ifreq *ifr)
 222 {
 223         char ifname[sizeof(ifr->ifr_name)];
 224         int ret;
 225
 226         ret = mlx5_get_ifname(dev, &ifname);
 227         if (ret)
 228                 return -rte_errno;
 229         return mlx5_ifreq_by_ifname(ifname, req, ifr);
 230 }
 231
 232 /**
 233  * Get device MTU.
 234  *
 235  * @param dev
 236  *   Pointer to Ethernet device.
 237  * @param[out] mtu
 238  *   MTU value output buffer.
 239  *
 240  * @return
 241  *   0 on success, a negative errno value otherwise and rte_errno is set.
 242  */
 243 int
 244 mlx5_get_mtu(struct rte_eth_dev *dev, uint16_t *mtu)
 245 {
 246         struct ifreq request;
 247         int ret = mlx5_ifreq(dev, SIOCGIFMTU, &request);
 248
 249         if (ret)
 250                 return ret;
 251         *mtu = request.ifr_mtu;
 252         return 0;
 253 }
 254
 255 /**
 256  * Set device MTU.
 257  *
 258  * @param dev
 259  *   Pointer to Ethernet device.
 260  * @param mtu
 261  *   MTU value to set.
 262  *
 263  * @return
 264  *   0 on success, a negative errno value otherwise and rte_errno is set.
 265  */
 266 int
 267 mlx5_set_mtu(struct rte_eth_dev *dev, uint16_t mtu)
 268 {
 269         struct ifreq request = { .ifr_mtu = mtu, };
 270
 271         return mlx5_ifreq(dev, SIOCSIFMTU, &request);
 272 }
 273
 274 /**
 275  * Set device flags.
 276  *
 277  * @param dev
 278  *   Pointer to Ethernet device.
 279  * @param keep
 280  *   Bitmask for flags that must remain untouched.
 281  * @param flags
 282  *   Bitmask for flags to modify.
 283  *
 284  * @return
 285  *   0 on success, a negative errno value otherwise and rte_errno is set.
 286  */
 287 static int
 288 mlx5_set_flags(struct rte_eth_dev *dev, unsigned int keep, unsigned int flags)
 289 {
 290         struct ifreq request;
 291         int ret = mlx5_ifreq(dev, SIOCGIFFLAGS, &request);
 292
 293         if (ret)
 294                 return ret;
 295         request.ifr_flags &= keep;
 296         request.ifr_flags |= flags & ~keep;
 297         return mlx5_ifreq(dev, SIOCSIFFLAGS, &request);
 298 }
 299
 300 /**
 301  * Get device current raw clock counter
 302  *
 303  * @param dev
 304  *   Pointer to Ethernet device structure.
 305  * @param[out] time
 306  *   Current raw clock counter of the device.
 307  *
 308  * @return
 309  *   0 if the clock has correctly been read
 310  *   The value of errno in case of error
 311  */
 312 int
 313 mlx5_read_clock(struct rte_eth_dev *dev, uint64_t *clock)
 314 {
 315         struct mlx5_priv *priv = dev->data->dev_private;
 316         struct ibv_context *ctx = priv->sh->ctx;
 317         struct ibv_values_ex values;
 318         int err = 0;
 319
 320         values.comp_mask = IBV_VALUES_MASK_RAW_CLOCK;
 321         err = mlx5_glue->query_rt_values_ex(ctx, &values);
 322         if (err != 0) {
 323                 DRV_LOG(WARNING, "Could not query the clock !");
 324                 return err;
 325         }
 326         *clock = values.raw_clock.tv_nsec;
 327         return 0;
 328 }
 329
 330 /**
 331  * Retrieve the master device for representor in the same switch domain.
 332  *
 333  * @param dev
 334  *   Pointer to representor Ethernet device structure.
 335  *
 336  * @return
 337  *   Master device structure  on success, NULL otherwise.
 338  */
 339 static struct rte_eth_dev *
 340 mlx5_find_master_dev(struct rte_eth_dev *dev)
 341 {
 342         struct mlx5_priv *priv;
 343         uint16_t port_id;
 344         uint16_t domain_id;
 345
 346         priv = dev->data->dev_private;
 347         domain_id = priv->domain_id;
 348         MLX5_ASSERT(priv->representor);
 349         MLX5_ETH_FOREACH_DEV(port_id, priv->pci_dev) {
 350                 struct mlx5_priv *opriv =
 351                         rte_eth_devices[port_id].data->dev_private;
 352                 if (opriv &&
 353                     opriv->master &&
 354                     opriv->domain_id == domain_id &&
 355                     opriv->sh == priv->sh)
 356                         return &rte_eth_devices[port_id];
 357         }
 358         return NULL;
 359 }
 360
 361 /**
 362  * DPDK callback to retrieve physical link information.
 363  *
 364  * @param dev
 365  *   Pointer to Ethernet device structure.
 366  * @param[out] link
 367  *   Storage for current link status.
 368  *
 369  * @return
 370  *   0 on success, a negative errno value otherwise and rte_errno is set.
 371  */
 372 static int
 373 mlx5_link_update_unlocked_gset(struct rte_eth_dev *dev,
 374                                struct rte_eth_link *link)
 375 {
 376         struct mlx5_priv *priv = dev->data->dev_private;
 377         struct ethtool_cmd edata = {
 378                 .cmd = ETHTOOL_GSET /* Deprecated since Linux v4.5. */
 379         };
 380         struct ifreq ifr;
 381         struct rte_eth_link dev_link;
 382         int link_speed = 0;
 383         int ret;
 384
 385         ret = mlx5_ifreq(dev, SIOCGIFFLAGS, &ifr);
 386         if (ret) {
 387                 DRV_LOG(WARNING, "port %u ioctl(SIOCGIFFLAGS) failed: %s",
 388                         dev->data->port_id, strerror(rte_errno));
 389                 return ret;
 390         }
 391         dev_link = (struct rte_eth_link) {
 392                 .link_status = ((ifr.ifr_flags & IFF_UP) &&
 393                                 (ifr.ifr_flags & IFF_RUNNING)),
 394         };
 395         ifr = (struct ifreq) {
 396                 .ifr_data = (void *)&edata,
 397         };
 398         ret = mlx5_ifreq(dev, SIOCETHTOOL, &ifr);
 399         if (ret) {
 400                 if (ret == -ENOTSUP && priv->representor) {
 401                         struct rte_eth_dev *master;
 402
 403                         /*
 404                          * For representors we can try to inherit link
 405                          * settings from the master device. Actually
 406                          * link settings do not make a lot of sense
 407                          * for representors due to missing physical
 408                          * link. The old kernel drivers supported
 409                          * emulated settings query for representors,
 410                          * the new ones do not, so we have to add
 411                          * this code for compatibility issues.
 412                          */
 413                         master = mlx5_find_master_dev(dev);
 414                         if (master) {
 415                                 ifr = (struct ifreq) {
 416                                         .ifr_data = (void *)&edata,
 417                                 };
 418                                 ret = mlx5_ifreq(master, SIOCETHTOOL, &ifr);
 419                         }
 420                 }
 421                 if (ret) {
 422                         DRV_LOG(WARNING,
 423                                 "port %u ioctl(SIOCETHTOOL,"
 424                                 " ETHTOOL_GSET) failed: %s",
 425                                 dev->data->port_id, strerror(rte_errno));
 426                         return ret;
 427                 }
 428         }
 429         link_speed = ethtool_cmd_speed(&edata);
 430         if (link_speed == -1)
 431                 dev_link.link_speed = ETH_SPEED_NUM_UNKNOWN;
 432         else
 433                 dev_link.link_speed = link_speed;
 434         priv->link_speed_capa = 0;
 435         if (edata.supported & SUPPORTED_Autoneg)
 436                 priv->link_speed_capa |= ETH_LINK_SPEED_AUTONEG;
 437         if (edata.supported & (SUPPORTED_1000baseT_Full |
 438                                SUPPORTED_1000baseKX_Full))
 439                 priv->link_speed_capa |= ETH_LINK_SPEED_1G;
 440         if (edata.supported & SUPPORTED_10000baseKR_Full)
 441                 priv->link_speed_capa |= ETH_LINK_SPEED_10G;
 442         if (edata.supported & (SUPPORTED_40000baseKR4_Full |
 443                                SUPPORTED_40000baseCR4_Full |
 444                                SUPPORTED_40000baseSR4_Full |
 445                                SUPPORTED_40000baseLR4_Full))
 446                 priv->link_speed_capa |= ETH_LINK_SPEED_40G;
 447         dev_link.link_duplex = ((edata.duplex == DUPLEX_HALF) ?
 448                                 ETH_LINK_HALF_DUPLEX : ETH_LINK_FULL_DUPLEX);
 449         dev_link.link_autoneg = !(dev->data->dev_conf.link_speeds &
 450                         ETH_LINK_SPEED_FIXED);
 451         *link = dev_link;
 452         return 0;
 453 }
 454
 455 /**
 456  * Retrieve physical link information (unlocked version using new ioctl).
 457  *
 458  * @param dev
 459  *   Pointer to Ethernet device structure.
 460  * @param[out] link
 461  *   Storage for current link status.
 462  *
 463  * @return
 464  *   0 on success, a negative errno value otherwise and rte_errno is set.
 465  */
 466 static int
 467 mlx5_link_update_unlocked_gs(struct rte_eth_dev *dev,
 468                              struct rte_eth_link *link)
 469
 470 {
 471         struct mlx5_priv *priv = dev->data->dev_private;
 472         struct ethtool_link_settings gcmd = { .cmd = ETHTOOL_GLINKSETTINGS };
 473         struct ifreq ifr;
 474         struct rte_eth_link dev_link;
 475         struct rte_eth_dev *master = NULL;
 476         uint64_t sc;
 477         int ret;
 478
 479         ret = mlx5_ifreq(dev, SIOCGIFFLAGS, &ifr);
 480         if (ret) {
 481                 DRV_LOG(WARNING, "port %u ioctl(SIOCGIFFLAGS) failed: %s",
 482                         dev->data->port_id, strerror(rte_errno));
 483                 return ret;
 484         }
 485         dev_link = (struct rte_eth_link) {
 486                 .link_status = ((ifr.ifr_flags & IFF_UP) &&
 487                                 (ifr.ifr_flags & IFF_RUNNING)),
 488         };
 489         ifr = (struct ifreq) {
 490                 .ifr_data = (void *)&gcmd,
 491         };
 492         ret = mlx5_ifreq(dev, SIOCETHTOOL, &ifr);
 493         if (ret) {
 494                 if (ret == -ENOTSUP && priv->representor) {
 495                         /*
 496                          * For representors we can try to inherit link
 497                          * settings from the master device. Actually
 498                          * link settings do not make a lot of sense
 499                          * for representors due to missing physical
 500                          * link. The old kernel drivers supported
 501                          * emulated settings query for representors,
 502                          * the new ones do not, so we have to add
 503                          * this code for compatibility issues.
 504                          */
 505                         master = mlx5_find_master_dev(dev);
 506                         if (master) {
 507                                 ifr = (struct ifreq) {
 508                                         .ifr_data = (void *)&gcmd,
 509                                 };
 510                                 ret = mlx5_ifreq(master, SIOCETHTOOL, &ifr);
 511                         }
 512                 }
 513                 if (ret) {
 514                         DRV_LOG(DEBUG,
 515                                 "port %u ioctl(SIOCETHTOOL,"
 516                                 " ETHTOOL_GLINKSETTINGS) failed: %s",
 517                                 dev->data->port_id, strerror(rte_errno));
 518                         return ret;
 519                 }
 520         }
 521         gcmd.link_mode_masks_nwords = -gcmd.link_mode_masks_nwords;
 522
 523         alignas(struct ethtool_link_settings)
 524         uint8_t data[offsetof(struct ethtool_link_settings, link_mode_masks) +
 525                      sizeof(uint32_t) * gcmd.link_mode_masks_nwords * 3];
 526         struct ethtool_link_settings *ecmd = (void *)data;
 527
 528         *ecmd = gcmd;
 529         ifr.ifr_data = (void *)ecmd;
 530         ret = mlx5_ifreq(master ? master : dev, SIOCETHTOOL, &ifr);
 531         if (ret) {
 532                 DRV_LOG(DEBUG,
 533                         "port %u ioctl(SIOCETHTOOL,"
 534                         "ETHTOOL_GLINKSETTINGS) failed: %s",
 535                         dev->data->port_id, strerror(rte_errno));
 536                 return ret;
 537         }
 538         dev_link.link_speed = (ecmd->speed == UINT32_MAX) ?
 539                                 ETH_SPEED_NUM_UNKNOWN : ecmd->speed;
 540         sc = ecmd->link_mode_masks[0] |
 541                 ((uint64_t)ecmd->link_mode_masks[1] << 32);
 542         priv->link_speed_capa = 0;
 543         if (sc & MLX5_BITSHIFT(ETHTOOL_LINK_MODE_Autoneg_BIT))
 544                 priv->link_speed_capa |= ETH_LINK_SPEED_AUTONEG;
 545         if (sc & (MLX5_BITSHIFT(ETHTOOL_LINK_MODE_1000baseT_Full_BIT) |
 546                   MLX5_BITSHIFT(ETHTOOL_LINK_MODE_1000baseKX_Full_BIT)))
 547                 priv->link_speed_capa |= ETH_LINK_SPEED_1G;
 548         if (sc & (MLX5_BITSHIFT(ETHTOOL_LINK_MODE_10000baseKX4_Full_BIT) |
 549                   MLX5_BITSHIFT(ETHTOOL_LINK_MODE_10000baseKR_Full_BIT) |
 550                   MLX5_BITSHIFT(ETHTOOL_LINK_MODE_10000baseR_FEC_BIT)))
 551                 priv->link_speed_capa |= ETH_LINK_SPEED_10G;
 552         if (sc & (MLX5_BITSHIFT(ETHTOOL_LINK_MODE_20000baseMLD2_Full_BIT) |
 553                   MLX5_BITSHIFT(ETHTOOL_LINK_MODE_20000baseKR2_Full_BIT)))
 554                 priv->link_speed_capa |= ETH_LINK_SPEED_20G;
 555         if (sc & (MLX5_BITSHIFT(ETHTOOL_LINK_MODE_40000baseKR4_Full_BIT) |
 556                   MLX5_BITSHIFT(ETHTOOL_LINK_MODE_40000baseCR4_Full_BIT) |
 557                   MLX5_BITSHIFT(ETHTOOL_LINK_MODE_40000baseSR4_Full_BIT) |
 558                   MLX5_BITSHIFT(ETHTOOL_LINK_MODE_40000baseLR4_Full_BIT)))
 559                 priv->link_speed_capa |= ETH_LINK_SPEED_40G;
 560         if (sc & (MLX5_BITSHIFT(ETHTOOL_LINK_MODE_56000baseKR4_Full_BIT) |
 561                   MLX5_BITSHIFT(ETHTOOL_LINK_MODE_56000baseCR4_Full_BIT) |
 562                   MLX5_BITSHIFT(ETHTOOL_LINK_MODE_56000baseSR4_Full_BIT) |
 563                   MLX5_BITSHIFT(ETHTOOL_LINK_MODE_56000baseLR4_Full_BIT)))
 564                 priv->link_speed_capa |= ETH_LINK_SPEED_56G;
 565         if (sc & (MLX5_BITSHIFT(ETHTOOL_LINK_MODE_25000baseCR_Full_BIT) |
 566                   MLX5_BITSHIFT(ETHTOOL_LINK_MODE_25000baseKR_Full_BIT) |
 567                   MLX5_BITSHIFT(ETHTOOL_LINK_MODE_25000baseSR_Full_BIT)))
 568                 priv->link_speed_capa |= ETH_LINK_SPEED_25G;
 569         if (sc & (MLX5_BITSHIFT(ETHTOOL_LINK_MODE_50000baseCR2_Full_BIT) |
 570                   MLX5_BITSHIFT(ETHTOOL_LINK_MODE_50000baseKR2_Full_BIT)))
 571                 priv->link_speed_capa |= ETH_LINK_SPEED_50G;
 572         if (sc & (MLX5_BITSHIFT(ETHTOOL_LINK_MODE_100000baseKR4_Full_BIT) |
 573                   MLX5_BITSHIFT(ETHTOOL_LINK_MODE_100000baseSR4_Full_BIT) |
 574                   MLX5_BITSHIFT(ETHTOOL_LINK_MODE_100000baseCR4_Full_BIT) |
 575                   MLX5_BITSHIFT(ETHTOOL_LINK_MODE_100000baseLR4_ER4_Full_BIT)))
 576                 priv->link_speed_capa |= ETH_LINK_SPEED_100G;
 577         if (sc & (MLX5_BITSHIFT(ETHTOOL_LINK_MODE_200000baseKR4_Full_BIT) |
 578                   MLX5_BITSHIFT(ETHTOOL_LINK_MODE_200000baseSR4_Full_BIT)))
 579                 priv->link_speed_capa |= ETH_LINK_SPEED_200G;
 580
 581         sc = ecmd->link_mode_masks[2] |
 582                 ((uint64_t)ecmd->link_mode_masks[3] << 32);
 583         if (sc & (MLX5_BITSHIFT(ETHTOOL_LINK_MODE_200000baseCR4_Full_BIT) |
 584                   MLX5_BITSHIFT
 585                        (ETHTOOL_LINK_MODE_200000baseLR4_ER4_FR4_Full_BIT) |
 586                   MLX5_BITSHIFT(ETHTOOL_LINK_MODE_200000baseDR4_Full_BIT)))
 587                 priv->link_speed_capa |= ETH_LINK_SPEED_200G;
 588         dev_link.link_duplex = ((ecmd->duplex == DUPLEX_HALF) ?
 589                                 ETH_LINK_HALF_DUPLEX : ETH_LINK_FULL_DUPLEX);
 590         dev_link.link_autoneg = !(dev->data->dev_conf.link_speeds &
 591                                   ETH_LINK_SPEED_FIXED);
 592         *link = dev_link;
 593         return 0;
 594 }
 595
 596 /**
 597  * DPDK callback to retrieve physical link information.
 598  *
 599  * @param dev
 600  *   Pointer to Ethernet device structure.
 601  * @param wait_to_complete
 602  *   Wait for request completion.
 603  *
 604  * @return
 605  *   0 if link status was not updated, positive if it was, a negative errno
 606  *   value otherwise and rte_errno is set.
 607  */
 608 int
 609 mlx5_link_update(struct rte_eth_dev *dev, int wait_to_complete)
 610 {
 611         int ret;
 612         struct rte_eth_link dev_link;
 613         time_t start_time = time(NULL);
 614         int retry = MLX5_GET_LINK_STATUS_RETRY_COUNT;
 615
 616         do {
 617                 ret = mlx5_link_update_unlocked_gs(dev, &dev_link);
 618                 if (ret == -ENOTSUP)
 619                         ret = mlx5_link_update_unlocked_gset(dev, &dev_link);
 620                 if (ret == 0)
 621                         break;
 622                 /* Handle wait to complete situation. */
 623                 if ((wait_to_complete || retry) && ret == -EAGAIN) {
 624                         if (abs((int)difftime(time(NULL), start_time)) <
 625                             MLX5_LINK_STATUS_TIMEOUT) {
 626                                 usleep(0);
 627                                 continue;
 628                         } else {
 629                                 rte_errno = EBUSY;
 630                                 return -rte_errno;
 631                         }
 632                 } else if (ret < 0) {
 633                         return ret;
 634                 }
 635         } while (wait_to_complete || retry-- > 0);
 636         ret = !!memcmp(&dev->data->dev_link, &dev_link,
 637                        sizeof(struct rte_eth_link));
 638         dev->data->dev_link = dev_link;
 639         return ret;
 640 }
 641
 642 /**
 643  * DPDK callback to get flow control status.
 644  *
 645  * @param dev
 646  *   Pointer to Ethernet device structure.
 647  * @param[out] fc_conf
 648  *   Flow control output buffer.
 649  *
 650  * @return
 651  *   0 on success, a negative errno value otherwise and rte_errno is set.
 652  */
 653 int
 654 mlx5_dev_get_flow_ctrl(struct rte_eth_dev *dev, struct rte_eth_fc_conf *fc_conf)
 655 {
 656         struct ifreq ifr;
 657         struct ethtool_pauseparam ethpause = {
 658                 .cmd = ETHTOOL_GPAUSEPARAM
 659         };
 660         int ret;
 661
 662         ifr.ifr_data = (void *)&ethpause;
 663         ret = mlx5_ifreq(dev, SIOCETHTOOL, &ifr);
 664         if (ret) {
 665                 DRV_LOG(WARNING,
 666                         "port %u ioctl(SIOCETHTOOL, ETHTOOL_GPAUSEPARAM) failed:"
 667                         " %s",
 668                         dev->data->port_id, strerror(rte_errno));
 669                 return ret;
 670         }
 671         fc_conf->autoneg = ethpause.autoneg;
 672         if (ethpause.rx_pause && ethpause.tx_pause)
 673                 fc_conf->mode = RTE_FC_FULL;
 674         else if (ethpause.rx_pause)
 675                 fc_conf->mode = RTE_FC_RX_PAUSE;
 676         else if (ethpause.tx_pause)
 677                 fc_conf->mode = RTE_FC_TX_PAUSE;
 678         else
 679                 fc_conf->mode = RTE_FC_NONE;
 680         return 0;
 681 }
 682
 683 /**
 684  * DPDK callback to modify flow control parameters.
 685  *
 686  * @param dev
 687  *   Pointer to Ethernet device structure.
 688  * @param[in] fc_conf
 689  *   Flow control parameters.
 690  *
 691  * @return
 692  *   0 on success, a negative errno value otherwise and rte_errno is set.
 693  */
 694 int
 695 mlx5_dev_set_flow_ctrl(struct rte_eth_dev *dev, struct rte_eth_fc_conf *fc_conf)
 696 {
 697         struct ifreq ifr;
 698         struct ethtool_pauseparam ethpause = {
 699                 .cmd = ETHTOOL_SPAUSEPARAM
 700         };
 701         int ret;
 702
 703         ifr.ifr_data = (void *)&ethpause;
 704         ethpause.autoneg = fc_conf->autoneg;
 705         if (((fc_conf->mode & RTE_FC_FULL) == RTE_FC_FULL) ||
 706             (fc_conf->mode & RTE_FC_RX_PAUSE))
 707                 ethpause.rx_pause = 1;
 708         else
 709                 ethpause.rx_pause = 0;
 710
 711         if (((fc_conf->mode & RTE_FC_FULL) == RTE_FC_FULL) ||
 712             (fc_conf->mode & RTE_FC_TX_PAUSE))
 713                 ethpause.tx_pause = 1;
 714         else
 715                 ethpause.tx_pause = 0;
 716         ret = mlx5_ifreq(dev, SIOCETHTOOL, &ifr);
 717         if (ret) {
 718                 DRV_LOG(WARNING,
 719                         "port %u ioctl(SIOCETHTOOL, ETHTOOL_SPAUSEPARAM)"
 720                         " failed: %s",
 721                         dev->data->port_id, strerror(rte_errno));
 722                 return ret;
 723         }
 724         return 0;
 725 }
 726
 727 /**
 728  * Handle asynchronous removal event for entire multiport device.
 729  *
 730  * @param sh
 731  *   Infiniband device shared context.
 732  */
 733 static void
 734 mlx5_dev_interrupt_device_fatal(struct mlx5_dev_ctx_shared *sh)
 735 {
 736         uint32_t i;
 737
 738         for (i = 0; i < sh->max_port; ++i) {
 739                 struct rte_eth_dev *dev;
 740
 741                 if (sh->port[i].ih_port_id >= RTE_MAX_ETHPORTS) {
 742                         /*
 743                          * Or not existing port either no
 744                          * handler installed for this port.
 745                          */
 746                         continue;
 747                 }
 748                 dev = &rte_eth_devices[sh->port[i].ih_port_id];
 749                 MLX5_ASSERT(dev);
 750                 if (dev->data->dev_conf.intr_conf.rmv)
 751                         rte_eth_dev_callback_process
 752                                 (dev, RTE_ETH_EVENT_INTR_RMV, NULL);
 753         }
 754 }
 755
 756 /**
 757  * Handle shared asynchronous events the NIC (removal event
 758  * and link status change). Supports multiport IB device.
 759  *
 760  * @param cb_arg
 761  *   Callback argument.
 762  */
 763 void
 764 mlx5_dev_interrupt_handler(void *cb_arg)
 765 {
 766         struct mlx5_dev_ctx_shared *sh = cb_arg;
 767         struct ibv_async_event event;
 768
 769         /* Read all message from the IB device and acknowledge them. */
 770         for (;;) {
 771                 struct rte_eth_dev *dev;
 772                 uint32_t tmp;
 773
 774                 if (mlx5_glue->get_async_event(sh->ctx, &event))
 775                         break;
 776                 /* Retrieve and check IB port index. */
 777                 tmp = (uint32_t)event.element.port_num;
 778                 if (!tmp && event.event_type == IBV_EVENT_DEVICE_FATAL) {
 779                         /*
 780                          * The DEVICE_FATAL event is called once for
 781                          * entire device without port specifying.
 782                          * We should notify all existing ports.
 783                          */
 784                         mlx5_glue->ack_async_event(&event);
 785                         mlx5_dev_interrupt_device_fatal(sh);
 786                         continue;
 787                 }
 788                 MLX5_ASSERT(tmp && (tmp <= sh->max_port));
 789                 if (!tmp) {
 790                         /* Unsupported device level event. */
 791                         mlx5_glue->ack_async_event(&event);
 792                         DRV_LOG(DEBUG,
 793                                 "unsupported common event (type %d)",
 794                                 event.event_type);
 795                         continue;
 796                 }
 797                 if (tmp > sh->max_port) {
 798                         /* Invalid IB port index. */
 799                         mlx5_glue->ack_async_event(&event);
 800                         DRV_LOG(DEBUG,
 801                                 "cannot handle an event (type %d)"
 802                                 "due to invalid IB port index (%u)",
 803                                 event.event_type, tmp);
 804                         continue;
 805                 }
 806                 if (sh->port[tmp - 1].ih_port_id >= RTE_MAX_ETHPORTS) {
 807                         /* No handler installed. */
 808                         mlx5_glue->ack_async_event(&event);
 809                         DRV_LOG(DEBUG,
 810                                 "cannot handle an event (type %d)"
 811                                 "due to no handler installed for port %u",
 812                                 event.event_type, tmp);
 813                         continue;
 814                 }
 815                 /* Retrieve ethernet device descriptor. */
 816                 tmp = sh->port[tmp - 1].ih_port_id;
 817                 dev = &rte_eth_devices[tmp];
 818                 MLX5_ASSERT(dev);
 819                 if ((event.event_type == IBV_EVENT_PORT_ACTIVE ||
 820                      event.event_type == IBV_EVENT_PORT_ERR) &&
 821                         dev->data->dev_conf.intr_conf.lsc) {
 822                         mlx5_glue->ack_async_event(&event);
 823                         if (mlx5_link_update(dev, 0) == -EAGAIN) {
 824                                 usleep(0);
 825                                 continue;
 826                         }
 827                         rte_eth_dev_callback_process
 828                                 (dev, RTE_ETH_EVENT_INTR_LSC, NULL);
 829                         continue;
 830                 }
 831                 DRV_LOG(DEBUG,
 832                         "port %u cannot handle an unknown event (type %d)",
 833                         dev->data->port_id, event.event_type);
 834                 mlx5_glue->ack_async_event(&event);
 835         }
 836 }
 837
 838 /*
 839  * Unregister callback handler safely. The handler may be active
 840  * while we are trying to unregister it, in this case code -EAGAIN
 841  * is returned by rte_intr_callback_unregister(). This routine checks
 842  * the return code and tries to unregister handler again.
 843  *
 844  * @param handle
 845  *   interrupt handle
 846  * @param cb_fn
 847  *   pointer to callback routine
 848  * @cb_arg
 849  *   opaque callback parameter
 850  */
 851 void
 852 mlx5_intr_callback_unregister(const struct rte_intr_handle *handle,
 853                               rte_intr_callback_fn cb_fn, void *cb_arg)
 854 {
 855         /*
 856          * Try to reduce timeout management overhead by not calling
 857          * the timer related routines on the first iteration. If the
 858          * unregistering succeeds on first call there will be no
 859          * timer calls at all.
 860          */
 861         uint64_t twait = 0;
 862         uint64_t start = 0;
 863
 864         do {
 865                 int ret;
 866
 867                 ret = rte_intr_callback_unregister(handle, cb_fn, cb_arg);
 868                 if (ret >= 0)
 869                         return;
 870                 if (ret != -EAGAIN) {
 871                         DRV_LOG(INFO, "failed to unregister interrupt"
 872                                       " handler (error: %d)", ret);
 873                         MLX5_ASSERT(false);
 874                         return;
 875                 }
 876                 if (twait) {
 877                         struct timespec onems;
 878
 879                         /* Wait one millisecond and try again. */
 880                         onems.tv_sec = 0;
 881                         onems.tv_nsec = NS_PER_S / MS_PER_S;
 882                         nanosleep(&onems, 0);
 883                         /* Check whether one second elapsed. */
 884                         if ((rte_get_timer_cycles() - start) <= twait)
 885                                 continue;
 886                 } else {
 887                         /*
 888                          * We get the amount of timer ticks for one second.
 889                          * If this amount elapsed it means we spent one
 890                          * second in waiting. This branch is executed once
 891                          * on first iteration.
 892                          */
 893                         twait = rte_get_timer_hz();
 894                         MLX5_ASSERT(twait);
 895                 }
 896                 /*
 897                  * Timeout elapsed, show message (once a second) and retry.
 898                  * We have no other acceptable option here, if we ignore
 899                  * the unregistering return code the handler will not
 900                  * be unregistered, fd will be closed and we may get the
 901                  * crush. Hanging and messaging in the loop seems not to be
 902                  * the worst choice.
 903                  */
 904                 DRV_LOG(INFO, "Retrying to unregister interrupt handler");
 905                 start = rte_get_timer_cycles();
 906         } while (true);
 907 }
 908
 909 /**
 910  * Handle DEVX interrupts from the NIC.
 911  * This function is probably called from the DPDK host thread.
 912  *
 913  * @param cb_arg
 914  *   Callback argument.
 915  */
 916 void
 917 mlx5_dev_interrupt_handler_devx(void *cb_arg)
 918 {
 919 #ifndef HAVE_IBV_DEVX_ASYNC
 920         (void)cb_arg;
 921         return;
 922 #else
 923         struct mlx5_dev_ctx_shared *sh = cb_arg;
 924         union {
 925                 struct mlx5dv_devx_async_cmd_hdr cmd_resp;
 926                 uint8_t buf[MLX5_ST_SZ_BYTES(query_flow_counter_out) +
 927                             MLX5_ST_SZ_BYTES(traffic_counter) +
 928                             sizeof(struct mlx5dv_devx_async_cmd_hdr)];
 929         } out;
 930         uint8_t *buf = out.buf + sizeof(out.cmd_resp);
 931
 932         while (!mlx5_glue->devx_get_async_cmd_comp(sh->devx_comp,
 933                                                    &out.cmd_resp,
 934                                                    sizeof(out.buf)))
 935                 mlx5_flow_async_pool_query_handle
 936                         (sh, (uint64_t)out.cmd_resp.wr_id,
 937                          mlx5_devx_get_out_command_status(buf));
 938 #endif /* HAVE_IBV_DEVX_ASYNC */
 939 }
 940
 941 /**
 942  * DPDK callback to bring the link DOWN.
 943  *
 944  * @param dev
 945  *   Pointer to Ethernet device structure.
 946  *
 947  * @return
 948  *   0 on success, a negative errno value otherwise and rte_errno is set.
 949  */
 950 int
 951 mlx5_set_link_down(struct rte_eth_dev *dev)
 952 {
 953         return mlx5_set_flags(dev, ~IFF_UP, ~IFF_UP);
 954 }
 955
 956 /**
 957  * DPDK callback to bring the link UP.
 958  *
 959  * @param dev
 960  *   Pointer to Ethernet device structure.
 961  *
 962  * @return
 963  *   0 on success, a negative errno value otherwise and rte_errno is set.
 964  */
 965 int
 966 mlx5_set_link_up(struct rte_eth_dev *dev)
 967 {
 968         return mlx5_set_flags(dev, ~IFF_UP, IFF_UP);
 969 }
 970
 971 /**
 972  * Check if mlx5 device was removed.
 973  *
 974  * @param dev
 975  *   Pointer to Ethernet device structure.
 976  *
 977  * @return
 978  *   1 when device is removed, otherwise 0.
 979  */
 980 int
 981 mlx5_is_removed(struct rte_eth_dev *dev)
 982 {
 983         struct ibv_device_attr device_attr;
 984         struct mlx5_priv *priv = dev->data->dev_private;
 985
 986         if (mlx5_glue->query_device(priv->sh->ctx, &device_attr) == EIO)
 987                 return 1;
 988         return 0;
 989 }
 990
 991 /**
 992  * Analyze gathered port parameters via sysfs to recognize master
 993  * and representor devices for E-Switch configuration.
 994  *
 995  * @param[in] device_dir
 996  *   flag of presence of "device" directory under port device key.
 997  * @param[inout] switch_info
 998  *   Port information, including port name as a number and port name
 999  *   type if recognized
1000  *
1001  * @return
1002  *   master and representor flags are set in switch_info according to
1003  *   recognized parameters (if any).
1004  */
1005 static void
1006 mlx5_sysfs_check_switch_info(bool device_dir,
1007                              struct mlx5_switch_info *switch_info)
1008 {
1009         switch (switch_info->name_type) {
1010         case MLX5_PHYS_PORT_NAME_TYPE_UNKNOWN:
1011                 /*
1012                  * Name is not recognized, assume the master,
1013                  * check the device directory presence.
1014                  */
1015                 switch_info->master = device_dir;
1016                 break;
1017         case MLX5_PHYS_PORT_NAME_TYPE_NOTSET:
1018                 /*
1019                  * Name is not set, this assumes the legacy naming
1020                  * schema for master, just check if there is
1021                  * a device directory.
1022                  */
1023                 switch_info->master = device_dir;
1024                 break;
1025         case MLX5_PHYS_PORT_NAME_TYPE_UPLINK:
1026                 /* New uplink naming schema recognized. */
1027                 switch_info->master = 1;
1028                 break;
1029         case MLX5_PHYS_PORT_NAME_TYPE_LEGACY:
1030                 /* Legacy representors naming schema. */
1031                 switch_info->representor = !device_dir;
1032                 break;
1033         case MLX5_PHYS_PORT_NAME_TYPE_PFHPF:
1034                 /* Fallthrough */
1035         case MLX5_PHYS_PORT_NAME_TYPE_PFVF:
1036                 /* Fallthrough */
1037         case MLX5_PHYS_PORT_NAME_TYPE_PFSF:
1038                 /* New representors naming schema. */
1039                 switch_info->representor = 1;
1040                 break;
1041         default:
1042                 switch_info->master = device_dir;
1043                 break;
1044         }
1045 }
1046
1047 /**
1048  * Get switch information associated with network interface.
1049  *
1050  * @param ifindex
1051  *   Network interface index.
1052  * @param[out] info
1053  *   Switch information object, populated in case of success.
1054  *
1055  * @return
1056  *   0 on success, a negative errno value otherwise and rte_errno is set.
1057  */
1058 int
1059 mlx5_sysfs_switch_info(unsigned int ifindex, struct mlx5_switch_info *info)
1060 {
1061         char ifname[IF_NAMESIZE];
1062         char port_name[IF_NAMESIZE];
1063         FILE *file;
1064         struct mlx5_switch_info data = {
1065                 .master = 0,
1066                 .representor = 0,
1067                 .name_type = MLX5_PHYS_PORT_NAME_TYPE_NOTSET,
1068                 .port_name = 0,
1069                 .switch_id = 0,
1070         };
1071         DIR *dir;
1072         bool port_switch_id_set = false;
1073         bool device_dir = false;
1074         char c;
1075         int ret;
1076
1077         if (!if_indextoname(ifindex, ifname)) {
1078                 rte_errno = errno;
1079                 return -rte_errno;
1080         }
1081
1082         MKSTR(phys_port_name, "/sys/class/net/%s/phys_port_name",
1083               ifname);
1084         MKSTR(phys_switch_id, "/sys/class/net/%s/phys_switch_id",
1085               ifname);
1086         MKSTR(pci_device, "/sys/class/net/%s/device",
1087               ifname);
1088
1089         file = fopen(phys_port_name, "rb");
1090         if (file != NULL) {
1091                 ret = fscanf(file, "%" RTE_STR(IF_NAMESIZE) "s", port_name);
1092                 fclose(file);
1093                 if (ret == 1)
1094                         mlx5_translate_port_name(port_name, &data);
1095         }
1096         file = fopen(phys_switch_id, "rb");
1097         if (file == NULL) {
1098                 rte_errno = errno;
1099                 return -rte_errno;
1100         }
1101         port_switch_id_set =
1102                 fscanf(file, "%" SCNx64 "%c", &data.switch_id, &c) == 2 &&
1103                 c == '\n';
1104         fclose(file);
1105         dir = opendir(pci_device);
1106         if (dir != NULL) {
1107                 closedir(dir);
1108                 device_dir = true;
1109         }
1110         if (port_switch_id_set) {
1111                 /* We have some E-Switch configuration. */
1112                 mlx5_sysfs_check_switch_info(device_dir, &data);
1113         }
1114         *info = data;
1115         MLX5_ASSERT(!(data.master && data.representor));
1116         if (data.master && data.representor) {
1117                 DRV_LOG(ERR, "ifindex %u device is recognized as master"
1118                              " and as representor", ifindex);
1119                 rte_errno = ENODEV;
1120                 return -rte_errno;
1121         }
1122         return 0;
1123 }
1124
1125 /**
1126  * Get bond information associated with network interface.
1127  *
1128  * @param pf_ifindex
1129  *   Network interface index of bond slave interface
1130  * @param[out] ifindex
1131  *   Pointer to bond ifindex.
1132  * @param[out] ifname
1133  *   Pointer to bond ifname.
1134  *
1135  * @return
1136  *   0 on success, a negative errno value otherwise and rte_errno is set.
1137  */
1138 int
1139 mlx5_sysfs_bond_info(unsigned int pf_ifindex, unsigned int *ifindex,
1140                      char *ifname)
1141 {
1142         char name[IF_NAMESIZE];
1143         FILE *file;
1144         unsigned int index;
1145         int ret;
1146
1147         if (!if_indextoname(pf_ifindex, name) || !strlen(name)) {
1148                 rte_errno = errno;
1149                 return -rte_errno;
1150         }
1151         MKSTR(bond_if, "/sys/class/net/%s/master/ifindex", name);
1152         /* read bond ifindex */
1153         file = fopen(bond_if, "rb");
1154         if (file == NULL) {
1155                 rte_errno = errno;
1156                 return -rte_errno;
1157         }
1158         ret = fscanf(file, "%u", &index);
1159         fclose(file);
1160         if (ret <= 0) {
1161                 rte_errno = errno;
1162                 return -rte_errno;
1163         }
1164         if (ifindex)
1165                 *ifindex = index;
1166
1167         /* read bond device name from symbol link */
1168         if (ifname) {
1169                 if (!if_indextoname(index, ifname)) {
1170                         rte_errno = errno;
1171                         return -rte_errno;
1172                 }
1173         }
1174         return 0;
1175 }
1176
1177 /**
1178  * DPDK callback to retrieve plug-in module EEPROM information (type and size).
1179  *
1180  * @param dev
1181  *   Pointer to Ethernet device structure.
1182  * @param[out] modinfo
1183  *   Storage for plug-in module EEPROM information.
1184  *
1185  * @return
1186  *   0 on success, a negative errno value otherwise and rte_errno is set.
1187  */
1188 int
1189 mlx5_get_module_info(struct rte_eth_dev *dev,
1190                      struct rte_eth_dev_module_info *modinfo)
1191 {
1192         struct ethtool_modinfo info = {
1193                 .cmd = ETHTOOL_GMODULEINFO,
1194         };
1195         struct ifreq ifr = (struct ifreq) {
1196                 .ifr_data = (void *)&info,
1197         };
1198         int ret = 0;
1199
1200         if (!dev || !modinfo) {
1201                 DRV_LOG(WARNING, "missing argument, cannot get module info");
1202                 rte_errno = EINVAL;
1203                 return -rte_errno;
1204         }
1205         ret = mlx5_ifreq(dev, SIOCETHTOOL, &ifr);
1206         if (ret) {
1207                 DRV_LOG(WARNING, "port %u ioctl(SIOCETHTOOL) failed: %s",
1208                         dev->data->port_id, strerror(rte_errno));
1209                 return ret;
1210         }
1211         modinfo->type = info.type;
1212         modinfo->eeprom_len = info.eeprom_len;
1213         return ret;
1214 }
1215
1216 /**
1217  * DPDK callback to retrieve plug-in module EEPROM data.
1218  *
1219  * @param dev
1220  *   Pointer to Ethernet device structure.
1221  * @param[out] info
1222  *   Storage for plug-in module EEPROM data.
1223  *
1224  * @return
1225  *   0 on success, a negative errno value otherwise and rte_errno is set.
1226  */
1227 int mlx5_get_module_eeprom(struct rte_eth_dev *dev,
1228                            struct rte_dev_eeprom_info *info)
1229 {
1230         struct ethtool_eeprom *eeprom;
1231         struct ifreq ifr;
1232         int ret = 0;
1233
1234         if (!dev || !info) {
1235                 DRV_LOG(WARNING, "missing argument, cannot get module eeprom");
1236                 rte_errno = EINVAL;
1237                 return -rte_errno;
1238         }
1239         eeprom = mlx5_malloc(MLX5_MEM_ZERO,
1240                              (sizeof(struct ethtool_eeprom) + info->length), 0,
1241                              SOCKET_ID_ANY);
1242         if (!eeprom) {
1243                 DRV_LOG(WARNING, "port %u cannot allocate memory for "
1244                         "eeprom data", dev->data->port_id);
1245                 rte_errno = ENOMEM;
1246                 return -rte_errno;
1247         }
1248         eeprom->cmd = ETHTOOL_GMODULEEEPROM;
1249         eeprom->offset = info->offset;
1250         eeprom->len = info->length;
1251         ifr = (struct ifreq) {
1252                 .ifr_data = (void *)eeprom,
1253         };
1254         ret = mlx5_ifreq(dev, SIOCETHTOOL, &ifr);
1255         if (ret)
1256                 DRV_LOG(WARNING, "port %u ioctl(SIOCETHTOOL) failed: %s",
1257                         dev->data->port_id, strerror(rte_errno));
1258         else
1259                 rte_memcpy(info->data, eeprom->data, info->length);
1260         mlx5_free(eeprom);
1261         return ret;
1262 }
1263
1264 /**
1265  * Read device counters table.
1266  *
1267  * @param dev
1268  *   Pointer to Ethernet device.
1269  * @param[in] pf
1270  *   PF index in case of bonding device, -1 otherwise
1271  * @param[out] stats
1272  *   Counters table output buffer.
1273  *
1274  * @return
1275  *   0 on success and stats is filled, negative errno value otherwise and
1276  *   rte_errno is set.
1277  */
1278 static int
1279 _mlx5_os_read_dev_counters(struct rte_eth_dev *dev, int pf, uint64_t *stats)
1280 {
1281         struct mlx5_priv *priv = dev->data->dev_private;
1282         struct mlx5_xstats_ctrl *xstats_ctrl = &priv->xstats_ctrl;
1283         unsigned int i;
1284         struct ifreq ifr;
1285         unsigned int stats_sz = xstats_ctrl->stats_n * sizeof(uint64_t);
1286         unsigned char et_stat_buf[sizeof(struct ethtool_stats) + stats_sz];
1287         struct ethtool_stats *et_stats = (struct ethtool_stats *)et_stat_buf;
1288         int ret;
1289
1290         et_stats->cmd = ETHTOOL_GSTATS;
1291         et_stats->n_stats = xstats_ctrl->stats_n;
1292         ifr.ifr_data = (caddr_t)et_stats;
1293         if (pf >= 0)
1294                 ret = mlx5_ifreq_by_ifname(priv->sh->bond.ports[pf].ifname,
1295                                            SIOCETHTOOL, &ifr);
1296         else
1297                 ret = mlx5_ifreq(dev, SIOCETHTOOL, &ifr);
1298         if (ret) {
1299                 DRV_LOG(WARNING,
1300                         "port %u unable to read statistic values from device",
1301                         dev->data->port_id);
1302                 return ret;
1303         }
1304         for (i = 0; i != xstats_ctrl->mlx5_stats_n; ++i) {
1305                 if (xstats_ctrl->info[i].dev)
1306                         continue;
1307                 stats[i] += (uint64_t)
1308                             et_stats->data[xstats_ctrl->dev_table_idx[i]];
1309         }
1310         return 0;
1311 }
1312
1313 /**
1314  * Read device counters.
1315  *
1316  * @param dev
1317  *   Pointer to Ethernet device.
1318  * @param[out] stats
1319  *   Counters table output buffer.
1320  *
1321  * @return
1322  *   0 on success and stats is filled, negative errno value otherwise and
1323  *   rte_errno is set.
1324  */
1325 int
1326 mlx5_os_read_dev_counters(struct rte_eth_dev *dev, uint64_t *stats)
1327 {
1328         struct mlx5_priv *priv = dev->data->dev_private;
1329         struct mlx5_xstats_ctrl *xstats_ctrl = &priv->xstats_ctrl;
1330         int ret = 0, i;
1331
1332         memset(stats, 0, sizeof(*stats) * xstats_ctrl->mlx5_stats_n);
1333         /* Read ifreq counters. */
1334         if (priv->master && priv->pf_bond >= 0) {
1335                 /* Sum xstats from bonding device member ports. */
1336                 for (i = 0; i < priv->sh->bond.n_port; i++) {
1337                         ret = _mlx5_os_read_dev_counters(dev, i, stats);
1338                         if (ret)
1339                                 return ret;
1340                 }
1341         } else {
1342                 ret = _mlx5_os_read_dev_counters(dev, -1, stats);
1343         }
1344         /* Read IB counters. */
1345         for (i = 0; i != xstats_ctrl->mlx5_stats_n; ++i) {
1346                 if (!xstats_ctrl->info[i].dev)
1347                         continue;
1348                 ret = mlx5_os_read_dev_stat(priv, xstats_ctrl->info[i].ctr_name,
1349                                             &stats[i]);
1350                 /* return last xstats counter if fail to read. */
1351                 if (ret != 0)
1352                         xstats_ctrl->xstats[i] = stats[i];
1353                 else
1354                         stats[i] = xstats_ctrl->xstats[i];
1355         }
1356         return ret;
1357 }
1358
1359 /**
1360  * Query the number of statistics provided by ETHTOOL.
1361  *
1362  * @param dev
1363  *   Pointer to Ethernet device.
1364  *
1365  * @return
1366  *   Number of statistics on success, negative errno value otherwise and
1367  *   rte_errno is set.
1368  */
1369 int
1370 mlx5_os_get_stats_n(struct rte_eth_dev *dev)
1371 {
1372         struct mlx5_priv *priv = dev->data->dev_private;
1373         struct ethtool_drvinfo drvinfo;
1374         struct ifreq ifr;
1375         int ret;
1376
1377         drvinfo.cmd = ETHTOOL_GDRVINFO;
1378         ifr.ifr_data = (caddr_t)&drvinfo;
1379         if (priv->master && priv->pf_bond >= 0)
1380                 /* Bonding PF. */
1381                 ret = mlx5_ifreq_by_ifname(priv->sh->bond.ports[0].ifname,
1382                                            SIOCETHTOOL, &ifr);
1383         else
1384                 ret = mlx5_ifreq(dev, SIOCETHTOOL, &ifr);
1385         if (ret) {
1386                 DRV_LOG(WARNING, "port %u unable to query number of statistics",
1387                         dev->data->port_id);
1388                 return ret;
1389         }
1390         return drvinfo.n_stats;
1391 }
1392
1393 static const struct mlx5_counter_ctrl mlx5_counters_init[] = {
1394         {
1395                 .dpdk_name = "rx_unicast_bytes",
1396                 .ctr_name = "rx_vport_unicast_bytes",
1397         },
1398         {
1399                 .dpdk_name = "rx_multicast_bytes",
1400                 .ctr_name = "rx_vport_multicast_bytes",
1401         },
1402         {
1403                 .dpdk_name = "rx_broadcast_bytes",
1404                 .ctr_name = "rx_vport_broadcast_bytes",
1405         },
1406         {
1407                 .dpdk_name = "rx_unicast_packets",
1408                 .ctr_name = "rx_vport_unicast_packets",
1409         },
1410         {
1411                 .dpdk_name = "rx_multicast_packets",
1412                 .ctr_name = "rx_vport_multicast_packets",
1413         },
1414         {
1415                 .dpdk_name = "rx_broadcast_packets",
1416                 .ctr_name = "rx_vport_broadcast_packets",
1417         },
1418         {
1419                 .dpdk_name = "tx_unicast_bytes",
1420                 .ctr_name = "tx_vport_unicast_bytes",
1421         },
1422         {
1423                 .dpdk_name = "tx_multicast_bytes",
1424                 .ctr_name = "tx_vport_multicast_bytes",
1425         },
1426         {
1427                 .dpdk_name = "tx_broadcast_bytes",
1428                 .ctr_name = "tx_vport_broadcast_bytes",
1429         },
1430         {
1431                 .dpdk_name = "tx_unicast_packets",
1432                 .ctr_name = "tx_vport_unicast_packets",
1433         },
1434         {
1435                 .dpdk_name = "tx_multicast_packets",
1436                 .ctr_name = "tx_vport_multicast_packets",
1437         },
1438         {
1439                 .dpdk_name = "tx_broadcast_packets",
1440                 .ctr_name = "tx_vport_broadcast_packets",
1441         },
1442         {
1443                 .dpdk_name = "rx_wqe_errors",
1444                 .ctr_name = "rx_wqe_err",
1445         },
1446         {
1447                 .dpdk_name = "rx_phy_crc_errors",
1448                 .ctr_name = "rx_crc_errors_phy",
1449         },
1450         {
1451                 .dpdk_name = "rx_phy_in_range_len_errors",
1452                 .ctr_name = "rx_in_range_len_errors_phy",
1453         },
1454         {
1455                 .dpdk_name = "rx_phy_symbol_errors",
1456                 .ctr_name = "rx_symbol_err_phy",
1457         },
1458         {
1459                 .dpdk_name = "tx_phy_errors",
1460                 .ctr_name = "tx_errors_phy",
1461         },
1462         {
1463                 .dpdk_name = "rx_out_of_buffer",
1464                 .ctr_name = "out_of_buffer",
1465                 .dev = 1,
1466         },
1467         {
1468                 .dpdk_name = "tx_phy_packets",
1469                 .ctr_name = "tx_packets_phy",
1470         },
1471         {
1472                 .dpdk_name = "rx_phy_packets",
1473                 .ctr_name = "rx_packets_phy",
1474         },
1475         {
1476                 .dpdk_name = "tx_phy_discard_packets",
1477                 .ctr_name = "tx_discards_phy",
1478         },
1479         {
1480                 .dpdk_name = "rx_phy_discard_packets",
1481                 .ctr_name = "rx_discards_phy",
1482         },
1483         {
1484                 .dpdk_name = "tx_phy_bytes",
1485                 .ctr_name = "tx_bytes_phy",
1486         },
1487         {
1488                 .dpdk_name = "rx_phy_bytes",
1489                 .ctr_name = "rx_bytes_phy",
1490         },
1491         /* Representor only */
1492         {
1493                 .dpdk_name = "rx_vport_packets",
1494                 .ctr_name = "vport_rx_packets",
1495         },
1496         {
1497                 .dpdk_name = "rx_vport_bytes",
1498                 .ctr_name = "vport_rx_bytes",
1499         },
1500         {
1501                 .dpdk_name = "tx_vport_packets",
1502                 .ctr_name = "vport_tx_packets",
1503         },
1504         {
1505                 .dpdk_name = "tx_vport_bytes",
1506                 .ctr_name = "vport_tx_bytes",
1507         },
1508 };
1509
1510 static const unsigned int xstats_n = RTE_DIM(mlx5_counters_init);
1511
1512 /**
1513  * Init the structures to read device counters.
1514  *
1515  * @param dev
1516  *   Pointer to Ethernet device.
1517  */
1518 void
1519 mlx5_os_stats_init(struct rte_eth_dev *dev)
1520 {
1521         struct mlx5_priv *priv = dev->data->dev_private;
1522         struct mlx5_xstats_ctrl *xstats_ctrl = &priv->xstats_ctrl;
1523         struct mlx5_stats_ctrl *stats_ctrl = &priv->stats_ctrl;
1524         unsigned int i;
1525         unsigned int j;
1526         struct ifreq ifr;
1527         struct ethtool_gstrings *strings = NULL;
1528         unsigned int dev_stats_n;
1529         unsigned int str_sz;
1530         int ret;
1531
1532         /* So that it won't aggregate for each init. */
1533         xstats_ctrl->mlx5_stats_n = 0;
1534         ret = mlx5_os_get_stats_n(dev);
1535         if (ret < 0) {
1536                 DRV_LOG(WARNING, "port %u no extended statistics available",
1537                         dev->data->port_id);
1538                 return;
1539         }
1540         dev_stats_n = ret;
1541         /* Allocate memory to grab stat names and values. */
1542         str_sz = dev_stats_n * ETH_GSTRING_LEN;
1543         strings = (struct ethtool_gstrings *)
1544                   mlx5_malloc(0, str_sz + sizeof(struct ethtool_gstrings), 0,
1545                               SOCKET_ID_ANY);
1546         if (!strings) {
1547                 DRV_LOG(WARNING, "port %u unable to allocate memory for xstats",
1548                      dev->data->port_id);
1549                 return;
1550         }
1551         strings->cmd = ETHTOOL_GSTRINGS;
1552         strings->string_set = ETH_SS_STATS;
1553         strings->len = dev_stats_n;
1554         ifr.ifr_data = (caddr_t)strings;
1555         if (priv->master && priv->pf_bond >= 0)
1556                 /* Bonding master. */
1557                 ret = mlx5_ifreq_by_ifname(priv->sh->bond.ports[0].ifname,
1558                                            SIOCETHTOOL, &ifr);
1559         else
1560                 ret = mlx5_ifreq(dev, SIOCETHTOOL, &ifr);
1561         if (ret) {
1562                 DRV_LOG(WARNING, "port %u unable to get statistic names",
1563                         dev->data->port_id);
1564                 goto free;
1565         }
1566         for (i = 0; i != dev_stats_n; ++i) {
1567                 const char *curr_string = (const char *)
1568                         &strings->data[i * ETH_GSTRING_LEN];
1569
1570                 for (j = 0; j != xstats_n; ++j) {
1571                         if (!strcmp(mlx5_counters_init[j].ctr_name,
1572                                     curr_string)) {
1573                                 unsigned int idx = xstats_ctrl->mlx5_stats_n++;
1574
1575                                 xstats_ctrl->dev_table_idx[idx] = i;
1576                                 xstats_ctrl->info[idx] = mlx5_counters_init[j];
1577                                 break;
1578                         }
1579                 }
1580         }
1581         /* Add dev counters. */
1582         for (i = 0; i != xstats_n; ++i) {
1583                 if (mlx5_counters_init[i].dev) {
1584                         unsigned int idx = xstats_ctrl->mlx5_stats_n++;
1585
1586                         xstats_ctrl->info[idx] = mlx5_counters_init[i];
1587                         xstats_ctrl->hw_stats[idx] = 0;
1588                 }
1589         }
1590         MLX5_ASSERT(xstats_ctrl->mlx5_stats_n <= MLX5_MAX_XSTATS);
1591         xstats_ctrl->stats_n = dev_stats_n;
1592         /* Copy to base at first time. */
1593         ret = mlx5_os_read_dev_counters(dev, xstats_ctrl->base);
1594         if (ret)
1595                 DRV_LOG(ERR, "port %u cannot read device counters: %s",
1596                         dev->data->port_id, strerror(rte_errno));
1597         mlx5_os_read_dev_stat(priv, "out_of_buffer", &stats_ctrl->imissed_base);
1598         stats_ctrl->imissed = 0;
1599 free:
1600         mlx5_free(strings);
1601 }
1602
1603 /**
1604  * Get MAC address by querying netdevice.
1605  *
1606  * @param[in] dev
1607  *   Pointer to Ethernet device.
1608  * @param[out] mac
1609  *   MAC address output buffer.
1610  *
1611  * @return
1612  *   0 on success, a negative errno value otherwise and rte_errno is set.
1613  */
1614 int
1615 mlx5_get_mac(struct rte_eth_dev *dev, uint8_t (*mac)[RTE_ETHER_ADDR_LEN])
1616 {
1617         struct ifreq request;
1618         int ret;
1619
1620         ret = mlx5_ifreq(dev, SIOCGIFHWADDR, &request);
1621         if (ret)
1622                 return ret;
1623         memcpy(mac, request.ifr_hwaddr.sa_data, RTE_ETHER_ADDR_LEN);
1624         return 0;
1625 }
1626