2e8a2969fe7328b03b7f5ece85618497ed780021
[dpdk.git] / drivers / net / mlx5 / mlx5_ethdev.c
1 /* SPDX-License-Identifier: BSD-3-Clause
2  * Copyright 2015 6WIND S.A.
3  * Copyright 2015 Mellanox Technologies, Ltd
4  */
5
6 #include <stddef.h>
7 #include <assert.h>
8 #include <inttypes.h>
9 #include <unistd.h>
10 #include <stdint.h>
11 #include <stdio.h>
12 #include <string.h>
13 #include <stdlib.h>
14 #include <errno.h>
15 #include <dirent.h>
16 #include <net/if.h>
17 #include <sys/ioctl.h>
18 #include <sys/socket.h>
19 #include <netinet/in.h>
20 #include <linux/ethtool.h>
21 #include <linux/sockios.h>
22 #include <fcntl.h>
23 #include <stdalign.h>
24 #include <sys/un.h>
25 #include <time.h>
26
27 #include <rte_atomic.h>
28 #include <rte_ethdev_driver.h>
29 #include <rte_bus_pci.h>
30 #include <rte_mbuf.h>
31 #include <rte_common.h>
32 #include <rte_interrupts.h>
33 #include <rte_malloc.h>
34 #include <rte_string_fns.h>
35 #include <rte_rwlock.h>
36
37 #include "mlx5.h"
38 #include "mlx5_glue.h"
39 #include "mlx5_rxtx.h"
40 #include "mlx5_utils.h"
41
42 /* Supported speed values found in /usr/include/linux/ethtool.h */
43 #ifndef HAVE_SUPPORTED_40000baseKR4_Full
44 #define SUPPORTED_40000baseKR4_Full (1 << 23)
45 #endif
46 #ifndef HAVE_SUPPORTED_40000baseCR4_Full
47 #define SUPPORTED_40000baseCR4_Full (1 << 24)
48 #endif
49 #ifndef HAVE_SUPPORTED_40000baseSR4_Full
50 #define SUPPORTED_40000baseSR4_Full (1 << 25)
51 #endif
52 #ifndef HAVE_SUPPORTED_40000baseLR4_Full
53 #define SUPPORTED_40000baseLR4_Full (1 << 26)
54 #endif
55 #ifndef HAVE_SUPPORTED_56000baseKR4_Full
56 #define SUPPORTED_56000baseKR4_Full (1 << 27)
57 #endif
58 #ifndef HAVE_SUPPORTED_56000baseCR4_Full
59 #define SUPPORTED_56000baseCR4_Full (1 << 28)
60 #endif
61 #ifndef HAVE_SUPPORTED_56000baseSR4_Full
62 #define SUPPORTED_56000baseSR4_Full (1 << 29)
63 #endif
64 #ifndef HAVE_SUPPORTED_56000baseLR4_Full
65 #define SUPPORTED_56000baseLR4_Full (1 << 30)
66 #endif
67
68 /* Add defines in case the running kernel is not the same as user headers. */
69 #ifndef ETHTOOL_GLINKSETTINGS
70 struct ethtool_link_settings {
71         uint32_t cmd;
72         uint32_t speed;
73         uint8_t duplex;
74         uint8_t port;
75         uint8_t phy_address;
76         uint8_t autoneg;
77         uint8_t mdio_support;
78         uint8_t eth_to_mdix;
79         uint8_t eth_tp_mdix_ctrl;
80         int8_t link_mode_masks_nwords;
81         uint32_t reserved[8];
82         uint32_t link_mode_masks[];
83 };
84
85 #define ETHTOOL_GLINKSETTINGS 0x0000004c
86 #define ETHTOOL_LINK_MODE_1000baseT_Full_BIT 5
87 #define ETHTOOL_LINK_MODE_Autoneg_BIT 6
88 #define ETHTOOL_LINK_MODE_1000baseKX_Full_BIT 17
89 #define ETHTOOL_LINK_MODE_10000baseKX4_Full_BIT 18
90 #define ETHTOOL_LINK_MODE_10000baseKR_Full_BIT 19
91 #define ETHTOOL_LINK_MODE_10000baseR_FEC_BIT 20
92 #define ETHTOOL_LINK_MODE_20000baseMLD2_Full_BIT 21
93 #define ETHTOOL_LINK_MODE_20000baseKR2_Full_BIT 22
94 #define ETHTOOL_LINK_MODE_40000baseKR4_Full_BIT 23
95 #define ETHTOOL_LINK_MODE_40000baseCR4_Full_BIT 24
96 #define ETHTOOL_LINK_MODE_40000baseSR4_Full_BIT 25
97 #define ETHTOOL_LINK_MODE_40000baseLR4_Full_BIT 26
98 #define ETHTOOL_LINK_MODE_56000baseKR4_Full_BIT 27
99 #define ETHTOOL_LINK_MODE_56000baseCR4_Full_BIT 28
100 #define ETHTOOL_LINK_MODE_56000baseSR4_Full_BIT 29
101 #define ETHTOOL_LINK_MODE_56000baseLR4_Full_BIT 30
102 #endif
103 #ifndef HAVE_ETHTOOL_LINK_MODE_25G
104 #define ETHTOOL_LINK_MODE_25000baseCR_Full_BIT 31
105 #define ETHTOOL_LINK_MODE_25000baseKR_Full_BIT 32
106 #define ETHTOOL_LINK_MODE_25000baseSR_Full_BIT 33
107 #endif
108 #ifndef HAVE_ETHTOOL_LINK_MODE_50G
109 #define ETHTOOL_LINK_MODE_50000baseCR2_Full_BIT 34
110 #define ETHTOOL_LINK_MODE_50000baseKR2_Full_BIT 35
111 #endif
112 #ifndef HAVE_ETHTOOL_LINK_MODE_100G
113 #define ETHTOOL_LINK_MODE_100000baseKR4_Full_BIT 36
114 #define ETHTOOL_LINK_MODE_100000baseSR4_Full_BIT 37
115 #define ETHTOOL_LINK_MODE_100000baseCR4_Full_BIT 38
116 #define ETHTOOL_LINK_MODE_100000baseLR4_ER4_Full_BIT 39
117 #endif
118
119 /**
120  * Get master interface name from private structure.
121  *
122  * @param[in] dev
123  *   Pointer to Ethernet device.
124  * @param[out] ifname
125  *   Interface name output buffer.
126  *
127  * @return
128  *   0 on success, a negative errno value otherwise and rte_errno is set.
129  */
130 static int
131 mlx5_get_master_ifname(const struct rte_eth_dev *dev,
132                        char (*ifname)[IF_NAMESIZE])
133 {
134         struct mlx5_priv *priv = dev->data->dev_private;
135         DIR *dir;
136         struct dirent *dent;
137         unsigned int dev_type = 0;
138         unsigned int dev_port_prev = ~0u;
139         char match[IF_NAMESIZE] = "";
140
141         {
142                 MKSTR(path, "%s/device/net", priv->ibdev_path);
143
144                 dir = opendir(path);
145                 if (dir == NULL) {
146                         rte_errno = errno;
147                         return -rte_errno;
148                 }
149         }
150         while ((dent = readdir(dir)) != NULL) {
151                 char *name = dent->d_name;
152                 FILE *file;
153                 unsigned int dev_port;
154                 int r;
155
156                 if ((name[0] == '.') &&
157                     ((name[1] == '\0') ||
158                      ((name[1] == '.') && (name[2] == '\0'))))
159                         continue;
160
161                 MKSTR(path, "%s/device/net/%s/%s",
162                       priv->ibdev_path, name,
163                       (dev_type ? "dev_id" : "dev_port"));
164
165                 file = fopen(path, "rb");
166                 if (file == NULL) {
167                         if (errno != ENOENT)
168                                 continue;
169                         /*
170                          * Switch to dev_id when dev_port does not exist as
171                          * is the case with Linux kernel versions < 3.15.
172                          */
173 try_dev_id:
174                         match[0] = '\0';
175                         if (dev_type)
176                                 break;
177                         dev_type = 1;
178                         dev_port_prev = ~0u;
179                         rewinddir(dir);
180                         continue;
181                 }
182                 r = fscanf(file, (dev_type ? "%x" : "%u"), &dev_port);
183                 fclose(file);
184                 if (r != 1)
185                         continue;
186                 /*
187                  * Switch to dev_id when dev_port returns the same value for
188                  * all ports. May happen when using a MOFED release older than
189                  * 3.0 with a Linux kernel >= 3.15.
190                  */
191                 if (dev_port == dev_port_prev)
192                         goto try_dev_id;
193                 dev_port_prev = dev_port;
194                 if (dev_port == 0)
195                         strlcpy(match, name, sizeof(match));
196         }
197         closedir(dir);
198         if (match[0] == '\0') {
199                 rte_errno = ENOENT;
200                 return -rte_errno;
201         }
202         strncpy(*ifname, match, sizeof(*ifname));
203         return 0;
204 }
205
206 /**
207  * Get interface name from private structure.
208  *
209  * This is a port representor-aware version of mlx5_get_master_ifname().
210  *
211  * @param[in] dev
212  *   Pointer to Ethernet device.
213  * @param[out] ifname
214  *   Interface name output buffer.
215  *
216  * @return
217  *   0 on success, a negative errno value otherwise and rte_errno is set.
218  */
219 int
220 mlx5_get_ifname(const struct rte_eth_dev *dev, char (*ifname)[IF_NAMESIZE])
221 {
222         struct mlx5_priv *priv = dev->data->dev_private;
223         unsigned int ifindex =
224                 priv->nl_socket_rdma >= 0 ?
225                 mlx5_nl_ifindex(priv->nl_socket_rdma, priv->ibdev_name) : 0;
226
227         if (!ifindex) {
228                 if (!priv->representor)
229                         return mlx5_get_master_ifname(dev, ifname);
230                 rte_errno = ENXIO;
231                 return -rte_errno;
232         }
233         if (if_indextoname(ifindex, &(*ifname)[0]))
234                 return 0;
235         rte_errno = errno;
236         return -rte_errno;
237 }
238
239 /**
240  * Get the interface index from device name.
241  *
242  * @param[in] dev
243  *   Pointer to Ethernet device.
244  *
245  * @return
246  *   Nonzero interface index on success, zero otherwise and rte_errno is set.
247  */
248 unsigned int
249 mlx5_ifindex(const struct rte_eth_dev *dev)
250 {
251         char ifname[IF_NAMESIZE];
252         unsigned int ifindex;
253
254         if (mlx5_get_ifname(dev, &ifname))
255                 return 0;
256         ifindex = if_nametoindex(ifname);
257         if (!ifindex)
258                 rte_errno = errno;
259         return ifindex;
260 }
261
262 /**
263  * Perform ifreq ioctl() on associated Ethernet device.
264  *
265  * @param[in] dev
266  *   Pointer to Ethernet device.
267  * @param req
268  *   Request number to pass to ioctl().
269  * @param[out] ifr
270  *   Interface request structure output buffer.
271  *
272  * @return
273  *   0 on success, a negative errno value otherwise and rte_errno is set.
274  */
275 int
276 mlx5_ifreq(const struct rte_eth_dev *dev, int req, struct ifreq *ifr)
277 {
278         int sock = socket(PF_INET, SOCK_DGRAM, IPPROTO_IP);
279         int ret = 0;
280
281         if (sock == -1) {
282                 rte_errno = errno;
283                 return -rte_errno;
284         }
285         ret = mlx5_get_ifname(dev, &ifr->ifr_name);
286         if (ret)
287                 goto error;
288         ret = ioctl(sock, req, ifr);
289         if (ret == -1) {
290                 rte_errno = errno;
291                 goto error;
292         }
293         close(sock);
294         return 0;
295 error:
296         close(sock);
297         return -rte_errno;
298 }
299
300 /**
301  * Get device MTU.
302  *
303  * @param dev
304  *   Pointer to Ethernet device.
305  * @param[out] mtu
306  *   MTU value output buffer.
307  *
308  * @return
309  *   0 on success, a negative errno value otherwise and rte_errno is set.
310  */
311 int
312 mlx5_get_mtu(struct rte_eth_dev *dev, uint16_t *mtu)
313 {
314         struct ifreq request;
315         int ret = mlx5_ifreq(dev, SIOCGIFMTU, &request);
316
317         if (ret)
318                 return ret;
319         *mtu = request.ifr_mtu;
320         return 0;
321 }
322
323 /**
324  * Set device MTU.
325  *
326  * @param dev
327  *   Pointer to Ethernet device.
328  * @param mtu
329  *   MTU value to set.
330  *
331  * @return
332  *   0 on success, a negative errno value otherwise and rte_errno is set.
333  */
334 static int
335 mlx5_set_mtu(struct rte_eth_dev *dev, uint16_t mtu)
336 {
337         struct ifreq request = { .ifr_mtu = mtu, };
338
339         return mlx5_ifreq(dev, SIOCSIFMTU, &request);
340 }
341
342 /**
343  * Set device flags.
344  *
345  * @param dev
346  *   Pointer to Ethernet device.
347  * @param keep
348  *   Bitmask for flags that must remain untouched.
349  * @param flags
350  *   Bitmask for flags to modify.
351  *
352  * @return
353  *   0 on success, a negative errno value otherwise and rte_errno is set.
354  */
355 int
356 mlx5_set_flags(struct rte_eth_dev *dev, unsigned int keep, unsigned int flags)
357 {
358         struct ifreq request;
359         int ret = mlx5_ifreq(dev, SIOCGIFFLAGS, &request);
360
361         if (ret)
362                 return ret;
363         request.ifr_flags &= keep;
364         request.ifr_flags |= flags & ~keep;
365         return mlx5_ifreq(dev, SIOCSIFFLAGS, &request);
366 }
367
368 /**
369  * DPDK callback for Ethernet device configuration.
370  *
371  * @param dev
372  *   Pointer to Ethernet device structure.
373  *
374  * @return
375  *   0 on success, a negative errno value otherwise and rte_errno is set.
376  */
377 int
378 mlx5_dev_configure(struct rte_eth_dev *dev)
379 {
380         struct mlx5_priv *priv = dev->data->dev_private;
381         unsigned int rxqs_n = dev->data->nb_rx_queues;
382         unsigned int txqs_n = dev->data->nb_tx_queues;
383         unsigned int i;
384         unsigned int j;
385         unsigned int reta_idx_n;
386         const uint8_t use_app_rss_key =
387                 !!dev->data->dev_conf.rx_adv_conf.rss_conf.rss_key;
388         int ret = 0;
389
390         if (use_app_rss_key &&
391             (dev->data->dev_conf.rx_adv_conf.rss_conf.rss_key_len !=
392              MLX5_RSS_HASH_KEY_LEN)) {
393                 DRV_LOG(ERR, "port %u RSS key len must be %s Bytes long",
394                         dev->data->port_id, RTE_STR(MLX5_RSS_HASH_KEY_LEN));
395                 rte_errno = EINVAL;
396                 return -rte_errno;
397         }
398         priv->rss_conf.rss_key =
399                 rte_realloc(priv->rss_conf.rss_key,
400                             MLX5_RSS_HASH_KEY_LEN, 0);
401         if (!priv->rss_conf.rss_key) {
402                 DRV_LOG(ERR, "port %u cannot allocate RSS hash key memory (%u)",
403                         dev->data->port_id, rxqs_n);
404                 rte_errno = ENOMEM;
405                 return -rte_errno;
406         }
407         memcpy(priv->rss_conf.rss_key,
408                use_app_rss_key ?
409                dev->data->dev_conf.rx_adv_conf.rss_conf.rss_key :
410                rss_hash_default_key,
411                MLX5_RSS_HASH_KEY_LEN);
412         priv->rss_conf.rss_key_len = MLX5_RSS_HASH_KEY_LEN;
413         priv->rss_conf.rss_hf = dev->data->dev_conf.rx_adv_conf.rss_conf.rss_hf;
414         priv->rxqs = (void *)dev->data->rx_queues;
415         priv->txqs = (void *)dev->data->tx_queues;
416         if (txqs_n != priv->txqs_n) {
417                 DRV_LOG(INFO, "port %u Tx queues number update: %u -> %u",
418                         dev->data->port_id, priv->txqs_n, txqs_n);
419                 priv->txqs_n = txqs_n;
420         }
421         if (rxqs_n > priv->config.ind_table_max_size) {
422                 DRV_LOG(ERR, "port %u cannot handle this many Rx queues (%u)",
423                         dev->data->port_id, rxqs_n);
424                 rte_errno = EINVAL;
425                 return -rte_errno;
426         }
427         if (rxqs_n == priv->rxqs_n)
428                 return 0;
429         DRV_LOG(INFO, "port %u Rx queues number update: %u -> %u",
430                 dev->data->port_id, priv->rxqs_n, rxqs_n);
431         priv->rxqs_n = rxqs_n;
432         /* If the requested number of RX queues is not a power of two, use the
433          * maximum indirection table size for better balancing.
434          * The result is always rounded to the next power of two. */
435         reta_idx_n = (1 << log2above((rxqs_n & (rxqs_n - 1)) ?
436                                      priv->config.ind_table_max_size :
437                                      rxqs_n));
438         ret = mlx5_rss_reta_index_resize(dev, reta_idx_n);
439         if (ret)
440                 return ret;
441         /* When the number of RX queues is not a power of two, the remaining
442          * table entries are padded with reused WQs and hashes are not spread
443          * uniformly. */
444         for (i = 0, j = 0; (i != reta_idx_n); ++i) {
445                 (*priv->reta_idx)[i] = j;
446                 if (++j == rxqs_n)
447                         j = 0;
448         }
449         return 0;
450 }
451
452 /**
453  * Sets default tuning parameters.
454  *
455  * @param dev
456  *   Pointer to Ethernet device.
457  * @param[out] info
458  *   Info structure output buffer.
459  */
460 static void
461 mlx5_set_default_params(struct rte_eth_dev *dev, struct rte_eth_dev_info *info)
462 {
463         struct mlx5_priv *priv = dev->data->dev_private;
464
465         /* Minimum CPU utilization. */
466         info->default_rxportconf.ring_size = 256;
467         info->default_txportconf.ring_size = 256;
468         info->default_rxportconf.burst_size = 64;
469         info->default_txportconf.burst_size = 64;
470         if (priv->link_speed_capa & ETH_LINK_SPEED_100G) {
471                 info->default_rxportconf.nb_queues = 16;
472                 info->default_txportconf.nb_queues = 16;
473                 if (dev->data->nb_rx_queues > 2 ||
474                     dev->data->nb_tx_queues > 2) {
475                         /* Max Throughput. */
476                         info->default_rxportconf.ring_size = 2048;
477                         info->default_txportconf.ring_size = 2048;
478                 }
479         } else {
480                 info->default_rxportconf.nb_queues = 8;
481                 info->default_txportconf.nb_queues = 8;
482                 if (dev->data->nb_rx_queues > 2 ||
483                     dev->data->nb_tx_queues > 2) {
484                         /* Max Throughput. */
485                         info->default_rxportconf.ring_size = 4096;
486                         info->default_txportconf.ring_size = 4096;
487                 }
488         }
489 }
490
491 /**
492  * DPDK callback to get information about the device.
493  *
494  * @param dev
495  *   Pointer to Ethernet device structure.
496  * @param[out] info
497  *   Info structure output buffer.
498  */
499 void
500 mlx5_dev_infos_get(struct rte_eth_dev *dev, struct rte_eth_dev_info *info)
501 {
502         struct mlx5_priv *priv = dev->data->dev_private;
503         struct mlx5_dev_config *config = &priv->config;
504         unsigned int max;
505         char ifname[IF_NAMESIZE];
506
507         /* FIXME: we should ask the device for these values. */
508         info->min_rx_bufsize = 32;
509         info->max_rx_pktlen = 65536;
510         /*
511          * Since we need one CQ per QP, the limit is the minimum number
512          * between the two values.
513          */
514         max = RTE_MIN(priv->device_attr.orig_attr.max_cq,
515                       priv->device_attr.orig_attr.max_qp);
516         /* If max >= 65535 then max = 0, max_rx_queues is uint16_t. */
517         if (max >= 65535)
518                 max = 65535;
519         info->max_rx_queues = max;
520         info->max_tx_queues = max;
521         info->max_mac_addrs = MLX5_MAX_UC_MAC_ADDRESSES;
522         info->rx_queue_offload_capa = mlx5_get_rx_queue_offloads(dev);
523         info->rx_offload_capa = (mlx5_get_rx_port_offloads() |
524                                  info->rx_queue_offload_capa);
525         info->tx_offload_capa = mlx5_get_tx_port_offloads(dev);
526         if (mlx5_get_ifname(dev, &ifname) == 0)
527                 info->if_index = if_nametoindex(ifname);
528         info->reta_size = priv->reta_idx_n ?
529                 priv->reta_idx_n : config->ind_table_max_size;
530         info->hash_key_size = MLX5_RSS_HASH_KEY_LEN;
531         info->speed_capa = priv->link_speed_capa;
532         info->flow_type_rss_offloads = ~MLX5_RSS_HF_MASK;
533         mlx5_set_default_params(dev, info);
534         info->switch_info.name = dev->data->name;
535         info->switch_info.domain_id = priv->domain_id;
536         info->switch_info.port_id = priv->representor_id;
537         if (priv->representor) {
538                 unsigned int i = mlx5_dev_to_port_id(dev->device, NULL, 0);
539                 uint16_t port_id[i];
540
541                 i = RTE_MIN(mlx5_dev_to_port_id(dev->device, port_id, i), i);
542                 while (i--) {
543                         struct mlx5_priv *opriv =
544                                 rte_eth_devices[port_id[i]].data->dev_private;
545
546                         if (!opriv ||
547                             opriv->representor ||
548                             opriv->domain_id != priv->domain_id)
549                                 continue;
550                         /*
551                          * Override switch name with that of the master
552                          * device.
553                          */
554                         info->switch_info.name = opriv->dev_data->name;
555                         break;
556                 }
557         }
558 }
559
560 /**
561  * Get firmware version of a device.
562  *
563  * @param dev
564  *   Ethernet device port.
565  * @param fw_ver
566  *   String output allocated by caller.
567  * @param fw_size
568  *   Size of the output string, including terminating null byte.
569  *
570  * @return
571  *   0 on success, or the size of the non truncated string if too big.
572  */
573 int mlx5_fw_version_get(struct rte_eth_dev *dev, char *fw_ver, size_t fw_size)
574 {
575         struct mlx5_priv *priv = dev->data->dev_private;
576         struct ibv_device_attr *attr = &priv->device_attr.orig_attr;
577         size_t size = strnlen(attr->fw_ver, sizeof(attr->fw_ver)) + 1;
578
579         if (fw_size < size)
580                 return size;
581         if (fw_ver != NULL)
582                 strlcpy(fw_ver, attr->fw_ver, fw_size);
583         return 0;
584 }
585
586 /**
587  * Get supported packet types.
588  *
589  * @param dev
590  *   Pointer to Ethernet device structure.
591  *
592  * @return
593  *   A pointer to the supported Packet types array.
594  */
595 const uint32_t *
596 mlx5_dev_supported_ptypes_get(struct rte_eth_dev *dev)
597 {
598         static const uint32_t ptypes[] = {
599                 /* refers to rxq_cq_to_pkt_type() */
600                 RTE_PTYPE_L2_ETHER,
601                 RTE_PTYPE_L3_IPV4_EXT_UNKNOWN,
602                 RTE_PTYPE_L3_IPV6_EXT_UNKNOWN,
603                 RTE_PTYPE_L4_NONFRAG,
604                 RTE_PTYPE_L4_FRAG,
605                 RTE_PTYPE_L4_TCP,
606                 RTE_PTYPE_L4_UDP,
607                 RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN,
608                 RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN,
609                 RTE_PTYPE_INNER_L4_NONFRAG,
610                 RTE_PTYPE_INNER_L4_FRAG,
611                 RTE_PTYPE_INNER_L4_TCP,
612                 RTE_PTYPE_INNER_L4_UDP,
613                 RTE_PTYPE_UNKNOWN
614         };
615
616         if (dev->rx_pkt_burst == mlx5_rx_burst ||
617             dev->rx_pkt_burst == mlx5_rx_burst_mprq ||
618             dev->rx_pkt_burst == mlx5_rx_burst_vec)
619                 return ptypes;
620         return NULL;
621 }
622
623 /**
624  * DPDK callback to retrieve physical link information.
625  *
626  * @param dev
627  *   Pointer to Ethernet device structure.
628  * @param[out] link
629  *   Storage for current link status.
630  *
631  * @return
632  *   0 on success, a negative errno value otherwise and rte_errno is set.
633  */
634 static int
635 mlx5_link_update_unlocked_gset(struct rte_eth_dev *dev,
636                                struct rte_eth_link *link)
637 {
638         struct mlx5_priv *priv = dev->data->dev_private;
639         struct ethtool_cmd edata = {
640                 .cmd = ETHTOOL_GSET /* Deprecated since Linux v4.5. */
641         };
642         struct ifreq ifr;
643         struct rte_eth_link dev_link;
644         int link_speed = 0;
645         int ret;
646
647         ret = mlx5_ifreq(dev, SIOCGIFFLAGS, &ifr);
648         if (ret) {
649                 DRV_LOG(WARNING, "port %u ioctl(SIOCGIFFLAGS) failed: %s",
650                         dev->data->port_id, strerror(rte_errno));
651                 return ret;
652         }
653         dev_link = (struct rte_eth_link) {
654                 .link_status = ((ifr.ifr_flags & IFF_UP) &&
655                                 (ifr.ifr_flags & IFF_RUNNING)),
656         };
657         ifr = (struct ifreq) {
658                 .ifr_data = (void *)&edata,
659         };
660         ret = mlx5_ifreq(dev, SIOCETHTOOL, &ifr);
661         if (ret) {
662                 DRV_LOG(WARNING,
663                         "port %u ioctl(SIOCETHTOOL, ETHTOOL_GSET) failed: %s",
664                         dev->data->port_id, strerror(rte_errno));
665                 return ret;
666         }
667         link_speed = ethtool_cmd_speed(&edata);
668         if (link_speed == -1)
669                 dev_link.link_speed = ETH_SPEED_NUM_NONE;
670         else
671                 dev_link.link_speed = link_speed;
672         priv->link_speed_capa = 0;
673         if (edata.supported & SUPPORTED_Autoneg)
674                 priv->link_speed_capa |= ETH_LINK_SPEED_AUTONEG;
675         if (edata.supported & (SUPPORTED_1000baseT_Full |
676                                SUPPORTED_1000baseKX_Full))
677                 priv->link_speed_capa |= ETH_LINK_SPEED_1G;
678         if (edata.supported & SUPPORTED_10000baseKR_Full)
679                 priv->link_speed_capa |= ETH_LINK_SPEED_10G;
680         if (edata.supported & (SUPPORTED_40000baseKR4_Full |
681                                SUPPORTED_40000baseCR4_Full |
682                                SUPPORTED_40000baseSR4_Full |
683                                SUPPORTED_40000baseLR4_Full))
684                 priv->link_speed_capa |= ETH_LINK_SPEED_40G;
685         dev_link.link_duplex = ((edata.duplex == DUPLEX_HALF) ?
686                                 ETH_LINK_HALF_DUPLEX : ETH_LINK_FULL_DUPLEX);
687         dev_link.link_autoneg = !(dev->data->dev_conf.link_speeds &
688                         ETH_LINK_SPEED_FIXED);
689         if (((dev_link.link_speed && !dev_link.link_status) ||
690              (!dev_link.link_speed && dev_link.link_status))) {
691                 rte_errno = EAGAIN;
692                 return -rte_errno;
693         }
694         *link = dev_link;
695         return 0;
696 }
697
698 /**
699  * Retrieve physical link information (unlocked version using new ioctl).
700  *
701  * @param dev
702  *   Pointer to Ethernet device structure.
703  * @param[out] link
704  *   Storage for current link status.
705  *
706  * @return
707  *   0 on success, a negative errno value otherwise and rte_errno is set.
708  */
709 static int
710 mlx5_link_update_unlocked_gs(struct rte_eth_dev *dev,
711                              struct rte_eth_link *link)
712
713 {
714         struct mlx5_priv *priv = dev->data->dev_private;
715         struct ethtool_link_settings gcmd = { .cmd = ETHTOOL_GLINKSETTINGS };
716         struct ifreq ifr;
717         struct rte_eth_link dev_link;
718         uint64_t sc;
719         int ret;
720
721         ret = mlx5_ifreq(dev, SIOCGIFFLAGS, &ifr);
722         if (ret) {
723                 DRV_LOG(WARNING, "port %u ioctl(SIOCGIFFLAGS) failed: %s",
724                         dev->data->port_id, strerror(rte_errno));
725                 return ret;
726         }
727         dev_link = (struct rte_eth_link) {
728                 .link_status = ((ifr.ifr_flags & IFF_UP) &&
729                                 (ifr.ifr_flags & IFF_RUNNING)),
730         };
731         ifr = (struct ifreq) {
732                 .ifr_data = (void *)&gcmd,
733         };
734         ret = mlx5_ifreq(dev, SIOCETHTOOL, &ifr);
735         if (ret) {
736                 DRV_LOG(DEBUG,
737                         "port %u ioctl(SIOCETHTOOL, ETHTOOL_GLINKSETTINGS)"
738                         " failed: %s",
739                         dev->data->port_id, strerror(rte_errno));
740                 return ret;
741         }
742         gcmd.link_mode_masks_nwords = -gcmd.link_mode_masks_nwords;
743
744         alignas(struct ethtool_link_settings)
745         uint8_t data[offsetof(struct ethtool_link_settings, link_mode_masks) +
746                      sizeof(uint32_t) * gcmd.link_mode_masks_nwords * 3];
747         struct ethtool_link_settings *ecmd = (void *)data;
748
749         *ecmd = gcmd;
750         ifr.ifr_data = (void *)ecmd;
751         ret = mlx5_ifreq(dev, SIOCETHTOOL, &ifr);
752         if (ret) {
753                 DRV_LOG(DEBUG,
754                         "port %u ioctl(SIOCETHTOOL, ETHTOOL_GLINKSETTINGS)"
755                         " failed: %s",
756                         dev->data->port_id, strerror(rte_errno));
757                 return ret;
758         }
759         dev_link.link_speed = ecmd->speed;
760         sc = ecmd->link_mode_masks[0] |
761                 ((uint64_t)ecmd->link_mode_masks[1] << 32);
762         priv->link_speed_capa = 0;
763         if (sc & MLX5_BITSHIFT(ETHTOOL_LINK_MODE_Autoneg_BIT))
764                 priv->link_speed_capa |= ETH_LINK_SPEED_AUTONEG;
765         if (sc & (MLX5_BITSHIFT(ETHTOOL_LINK_MODE_1000baseT_Full_BIT) |
766                   MLX5_BITSHIFT(ETHTOOL_LINK_MODE_1000baseKX_Full_BIT)))
767                 priv->link_speed_capa |= ETH_LINK_SPEED_1G;
768         if (sc & (MLX5_BITSHIFT(ETHTOOL_LINK_MODE_10000baseKX4_Full_BIT) |
769                   MLX5_BITSHIFT(ETHTOOL_LINK_MODE_10000baseKR_Full_BIT) |
770                   MLX5_BITSHIFT(ETHTOOL_LINK_MODE_10000baseR_FEC_BIT)))
771                 priv->link_speed_capa |= ETH_LINK_SPEED_10G;
772         if (sc & (MLX5_BITSHIFT(ETHTOOL_LINK_MODE_20000baseMLD2_Full_BIT) |
773                   MLX5_BITSHIFT(ETHTOOL_LINK_MODE_20000baseKR2_Full_BIT)))
774                 priv->link_speed_capa |= ETH_LINK_SPEED_20G;
775         if (sc & (MLX5_BITSHIFT(ETHTOOL_LINK_MODE_40000baseKR4_Full_BIT) |
776                   MLX5_BITSHIFT(ETHTOOL_LINK_MODE_40000baseCR4_Full_BIT) |
777                   MLX5_BITSHIFT(ETHTOOL_LINK_MODE_40000baseSR4_Full_BIT) |
778                   MLX5_BITSHIFT(ETHTOOL_LINK_MODE_40000baseLR4_Full_BIT)))
779                 priv->link_speed_capa |= ETH_LINK_SPEED_40G;
780         if (sc & (MLX5_BITSHIFT(ETHTOOL_LINK_MODE_56000baseKR4_Full_BIT) |
781                   MLX5_BITSHIFT(ETHTOOL_LINK_MODE_56000baseCR4_Full_BIT) |
782                   MLX5_BITSHIFT(ETHTOOL_LINK_MODE_56000baseSR4_Full_BIT) |
783                   MLX5_BITSHIFT(ETHTOOL_LINK_MODE_56000baseLR4_Full_BIT)))
784                 priv->link_speed_capa |= ETH_LINK_SPEED_56G;
785         if (sc & (MLX5_BITSHIFT(ETHTOOL_LINK_MODE_25000baseCR_Full_BIT) |
786                   MLX5_BITSHIFT(ETHTOOL_LINK_MODE_25000baseKR_Full_BIT) |
787                   MLX5_BITSHIFT(ETHTOOL_LINK_MODE_25000baseSR_Full_BIT)))
788                 priv->link_speed_capa |= ETH_LINK_SPEED_25G;
789         if (sc & (MLX5_BITSHIFT(ETHTOOL_LINK_MODE_50000baseCR2_Full_BIT) |
790                   MLX5_BITSHIFT(ETHTOOL_LINK_MODE_50000baseKR2_Full_BIT)))
791                 priv->link_speed_capa |= ETH_LINK_SPEED_50G;
792         if (sc & (MLX5_BITSHIFT(ETHTOOL_LINK_MODE_100000baseKR4_Full_BIT) |
793                   MLX5_BITSHIFT(ETHTOOL_LINK_MODE_100000baseSR4_Full_BIT) |
794                   MLX5_BITSHIFT(ETHTOOL_LINK_MODE_100000baseCR4_Full_BIT) |
795                   MLX5_BITSHIFT(ETHTOOL_LINK_MODE_100000baseLR4_ER4_Full_BIT)))
796                 priv->link_speed_capa |= ETH_LINK_SPEED_100G;
797         dev_link.link_duplex = ((ecmd->duplex == DUPLEX_HALF) ?
798                                 ETH_LINK_HALF_DUPLEX : ETH_LINK_FULL_DUPLEX);
799         dev_link.link_autoneg = !(dev->data->dev_conf.link_speeds &
800                                   ETH_LINK_SPEED_FIXED);
801         if (((dev_link.link_speed && !dev_link.link_status) ||
802              (!dev_link.link_speed && dev_link.link_status))) {
803                 rte_errno = EAGAIN;
804                 return -rte_errno;
805         }
806         *link = dev_link;
807         return 0;
808 }
809
810 /**
811  * DPDK callback to retrieve physical link information.
812  *
813  * @param dev
814  *   Pointer to Ethernet device structure.
815  * @param wait_to_complete
816  *   Wait for request completion.
817  *
818  * @return
819  *   0 if link status was not updated, positive if it was, a negative errno
820  *   value otherwise and rte_errno is set.
821  */
822 int
823 mlx5_link_update(struct rte_eth_dev *dev, int wait_to_complete)
824 {
825         int ret;
826         struct rte_eth_link dev_link;
827         time_t start_time = time(NULL);
828
829         do {
830                 ret = mlx5_link_update_unlocked_gs(dev, &dev_link);
831                 if (ret)
832                         ret = mlx5_link_update_unlocked_gset(dev, &dev_link);
833                 if (ret == 0)
834                         break;
835                 /* Handle wait to complete situation. */
836                 if (wait_to_complete && ret == -EAGAIN) {
837                         if (abs((int)difftime(time(NULL), start_time)) <
838                             MLX5_LINK_STATUS_TIMEOUT) {
839                                 usleep(0);
840                                 continue;
841                         } else {
842                                 rte_errno = EBUSY;
843                                 return -rte_errno;
844                         }
845                 } else if (ret < 0) {
846                         return ret;
847                 }
848         } while (wait_to_complete);
849         ret = !!memcmp(&dev->data->dev_link, &dev_link,
850                        sizeof(struct rte_eth_link));
851         dev->data->dev_link = dev_link;
852         return ret;
853 }
854
855 /**
856  * DPDK callback to change the MTU.
857  *
858  * @param dev
859  *   Pointer to Ethernet device structure.
860  * @param in_mtu
861  *   New MTU.
862  *
863  * @return
864  *   0 on success, a negative errno value otherwise and rte_errno is set.
865  */
866 int
867 mlx5_dev_set_mtu(struct rte_eth_dev *dev, uint16_t mtu)
868 {
869         struct mlx5_priv *priv = dev->data->dev_private;
870         uint16_t kern_mtu = 0;
871         int ret;
872
873         ret = mlx5_get_mtu(dev, &kern_mtu);
874         if (ret)
875                 return ret;
876         /* Set kernel interface MTU first. */
877         ret = mlx5_set_mtu(dev, mtu);
878         if (ret)
879                 return ret;
880         ret = mlx5_get_mtu(dev, &kern_mtu);
881         if (ret)
882                 return ret;
883         if (kern_mtu == mtu) {
884                 priv->mtu = mtu;
885                 DRV_LOG(DEBUG, "port %u adapter MTU set to %u",
886                         dev->data->port_id, mtu);
887                 return 0;
888         }
889         rte_errno = EAGAIN;
890         return -rte_errno;
891 }
892
893 /**
894  * DPDK callback to get flow control status.
895  *
896  * @param dev
897  *   Pointer to Ethernet device structure.
898  * @param[out] fc_conf
899  *   Flow control output buffer.
900  *
901  * @return
902  *   0 on success, a negative errno value otherwise and rte_errno is set.
903  */
904 int
905 mlx5_dev_get_flow_ctrl(struct rte_eth_dev *dev, struct rte_eth_fc_conf *fc_conf)
906 {
907         struct ifreq ifr;
908         struct ethtool_pauseparam ethpause = {
909                 .cmd = ETHTOOL_GPAUSEPARAM
910         };
911         int ret;
912
913         ifr.ifr_data = (void *)&ethpause;
914         ret = mlx5_ifreq(dev, SIOCETHTOOL, &ifr);
915         if (ret) {
916                 DRV_LOG(WARNING,
917                         "port %u ioctl(SIOCETHTOOL, ETHTOOL_GPAUSEPARAM) failed:"
918                         " %s",
919                         dev->data->port_id, strerror(rte_errno));
920                 return ret;
921         }
922         fc_conf->autoneg = ethpause.autoneg;
923         if (ethpause.rx_pause && ethpause.tx_pause)
924                 fc_conf->mode = RTE_FC_FULL;
925         else if (ethpause.rx_pause)
926                 fc_conf->mode = RTE_FC_RX_PAUSE;
927         else if (ethpause.tx_pause)
928                 fc_conf->mode = RTE_FC_TX_PAUSE;
929         else
930                 fc_conf->mode = RTE_FC_NONE;
931         return 0;
932 }
933
934 /**
935  * DPDK callback to modify flow control parameters.
936  *
937  * @param dev
938  *   Pointer to Ethernet device structure.
939  * @param[in] fc_conf
940  *   Flow control parameters.
941  *
942  * @return
943  *   0 on success, a negative errno value otherwise and rte_errno is set.
944  */
945 int
946 mlx5_dev_set_flow_ctrl(struct rte_eth_dev *dev, struct rte_eth_fc_conf *fc_conf)
947 {
948         struct ifreq ifr;
949         struct ethtool_pauseparam ethpause = {
950                 .cmd = ETHTOOL_SPAUSEPARAM
951         };
952         int ret;
953
954         ifr.ifr_data = (void *)&ethpause;
955         ethpause.autoneg = fc_conf->autoneg;
956         if (((fc_conf->mode & RTE_FC_FULL) == RTE_FC_FULL) ||
957             (fc_conf->mode & RTE_FC_RX_PAUSE))
958                 ethpause.rx_pause = 1;
959         else
960                 ethpause.rx_pause = 0;
961
962         if (((fc_conf->mode & RTE_FC_FULL) == RTE_FC_FULL) ||
963             (fc_conf->mode & RTE_FC_TX_PAUSE))
964                 ethpause.tx_pause = 1;
965         else
966                 ethpause.tx_pause = 0;
967         ret = mlx5_ifreq(dev, SIOCETHTOOL, &ifr);
968         if (ret) {
969                 DRV_LOG(WARNING,
970                         "port %u ioctl(SIOCETHTOOL, ETHTOOL_SPAUSEPARAM)"
971                         " failed: %s",
972                         dev->data->port_id, strerror(rte_errno));
973                 return ret;
974         }
975         return 0;
976 }
977
978 /**
979  * Get PCI information from struct ibv_device.
980  *
981  * @param device
982  *   Pointer to Ethernet device structure.
983  * @param[out] pci_addr
984  *   PCI bus address output buffer.
985  *
986  * @return
987  *   0 on success, a negative errno value otherwise and rte_errno is set.
988  */
989 int
990 mlx5_ibv_device_to_pci_addr(const struct ibv_device *device,
991                             struct rte_pci_addr *pci_addr)
992 {
993         FILE *file;
994         char line[32];
995         MKSTR(path, "%s/device/uevent", device->ibdev_path);
996
997         file = fopen(path, "rb");
998         if (file == NULL) {
999                 rte_errno = errno;
1000                 return -rte_errno;
1001         }
1002         while (fgets(line, sizeof(line), file) == line) {
1003                 size_t len = strlen(line);
1004                 int ret;
1005
1006                 /* Truncate long lines. */
1007                 if (len == (sizeof(line) - 1))
1008                         while (line[(len - 1)] != '\n') {
1009                                 ret = fgetc(file);
1010                                 if (ret == EOF)
1011                                         break;
1012                                 line[(len - 1)] = ret;
1013                         }
1014                 /* Extract information. */
1015                 if (sscanf(line,
1016                            "PCI_SLOT_NAME="
1017                            "%" SCNx32 ":%" SCNx8 ":%" SCNx8 ".%" SCNx8 "\n",
1018                            &pci_addr->domain,
1019                            &pci_addr->bus,
1020                            &pci_addr->devid,
1021                            &pci_addr->function) == 4) {
1022                         ret = 0;
1023                         break;
1024                 }
1025         }
1026         fclose(file);
1027         return 0;
1028 }
1029
1030 /**
1031  * Device status handler.
1032  *
1033  * @param dev
1034  *   Pointer to Ethernet device.
1035  * @param events
1036  *   Pointer to event flags holder.
1037  *
1038  * @return
1039  *   Events bitmap of callback process which can be called immediately.
1040  */
1041 static uint32_t
1042 mlx5_dev_status_handler(struct rte_eth_dev *dev)
1043 {
1044         struct mlx5_priv *priv = dev->data->dev_private;
1045         struct ibv_async_event event;
1046         uint32_t ret = 0;
1047
1048         if (mlx5_link_update(dev, 0) == -EAGAIN) {
1049                 usleep(0);
1050                 return 0;
1051         }
1052         /* Read all message and acknowledge them. */
1053         for (;;) {
1054                 if (mlx5_glue->get_async_event(priv->ctx, &event))
1055                         break;
1056                 if ((event.event_type == IBV_EVENT_PORT_ACTIVE ||
1057                         event.event_type == IBV_EVENT_PORT_ERR) &&
1058                         (dev->data->dev_conf.intr_conf.lsc == 1))
1059                         ret |= (1 << RTE_ETH_EVENT_INTR_LSC);
1060                 else if (event.event_type == IBV_EVENT_DEVICE_FATAL &&
1061                         dev->data->dev_conf.intr_conf.rmv == 1)
1062                         ret |= (1 << RTE_ETH_EVENT_INTR_RMV);
1063                 else
1064                         DRV_LOG(DEBUG,
1065                                 "port %u event type %d on not handled",
1066                                 dev->data->port_id, event.event_type);
1067                 mlx5_glue->ack_async_event(&event);
1068         }
1069         return ret;
1070 }
1071
1072 /**
1073  * Handle interrupts from the NIC.
1074  *
1075  * @param[in] intr_handle
1076  *   Interrupt handler.
1077  * @param cb_arg
1078  *   Callback argument.
1079  */
1080 void
1081 mlx5_dev_interrupt_handler(void *cb_arg)
1082 {
1083         struct rte_eth_dev *dev = cb_arg;
1084         uint32_t events;
1085
1086         events = mlx5_dev_status_handler(dev);
1087         if (events & (1 << RTE_ETH_EVENT_INTR_LSC))
1088                 _rte_eth_dev_callback_process(dev, RTE_ETH_EVENT_INTR_LSC, NULL);
1089         if (events & (1 << RTE_ETH_EVENT_INTR_RMV))
1090                 _rte_eth_dev_callback_process(dev, RTE_ETH_EVENT_INTR_RMV, NULL);
1091 }
1092
1093 /**
1094  * Handle interrupts from the socket.
1095  *
1096  * @param cb_arg
1097  *   Callback argument.
1098  */
1099 static void
1100 mlx5_dev_handler_socket(void *cb_arg)
1101 {
1102         struct rte_eth_dev *dev = cb_arg;
1103
1104         mlx5_socket_handle(dev);
1105 }
1106
1107 /**
1108  * Uninstall interrupt handler.
1109  *
1110  * @param dev
1111  *   Pointer to Ethernet device.
1112  */
1113 void
1114 mlx5_dev_interrupt_handler_uninstall(struct rte_eth_dev *dev)
1115 {
1116         struct mlx5_priv *priv = dev->data->dev_private;
1117
1118         if (dev->data->dev_conf.intr_conf.lsc ||
1119             dev->data->dev_conf.intr_conf.rmv)
1120                 rte_intr_callback_unregister(&priv->intr_handle,
1121                                              mlx5_dev_interrupt_handler, dev);
1122         if (priv->primary_socket)
1123                 rte_intr_callback_unregister(&priv->intr_handle_socket,
1124                                              mlx5_dev_handler_socket, dev);
1125         priv->intr_handle.fd = 0;
1126         priv->intr_handle.type = RTE_INTR_HANDLE_UNKNOWN;
1127         priv->intr_handle_socket.fd = 0;
1128         priv->intr_handle_socket.type = RTE_INTR_HANDLE_UNKNOWN;
1129 }
1130
1131 /**
1132  * Install interrupt handler.
1133  *
1134  * @param dev
1135  *   Pointer to Ethernet device.
1136  */
1137 void
1138 mlx5_dev_interrupt_handler_install(struct rte_eth_dev *dev)
1139 {
1140         struct mlx5_priv *priv = dev->data->dev_private;
1141         int ret;
1142         int flags;
1143
1144         assert(priv->ctx->async_fd > 0);
1145         flags = fcntl(priv->ctx->async_fd, F_GETFL);
1146         ret = fcntl(priv->ctx->async_fd, F_SETFL, flags | O_NONBLOCK);
1147         if (ret) {
1148                 DRV_LOG(INFO,
1149                         "port %u failed to change file descriptor async event"
1150                         " queue",
1151                         dev->data->port_id);
1152                 dev->data->dev_conf.intr_conf.lsc = 0;
1153                 dev->data->dev_conf.intr_conf.rmv = 0;
1154         }
1155         if (dev->data->dev_conf.intr_conf.lsc ||
1156             dev->data->dev_conf.intr_conf.rmv) {
1157                 priv->intr_handle.fd = priv->ctx->async_fd;
1158                 priv->intr_handle.type = RTE_INTR_HANDLE_EXT;
1159                 rte_intr_callback_register(&priv->intr_handle,
1160                                            mlx5_dev_interrupt_handler, dev);
1161         }
1162         ret = mlx5_socket_init(dev);
1163         if (ret)
1164                 DRV_LOG(ERR, "port %u cannot initialise socket: %s",
1165                         dev->data->port_id, strerror(rte_errno));
1166         else if (priv->primary_socket) {
1167                 priv->intr_handle_socket.fd = priv->primary_socket;
1168                 priv->intr_handle_socket.type = RTE_INTR_HANDLE_EXT;
1169                 rte_intr_callback_register(&priv->intr_handle_socket,
1170                                            mlx5_dev_handler_socket, dev);
1171         }
1172 }
1173
1174 /**
1175  * DPDK callback to bring the link DOWN.
1176  *
1177  * @param dev
1178  *   Pointer to Ethernet device structure.
1179  *
1180  * @return
1181  *   0 on success, a negative errno value otherwise and rte_errno is set.
1182  */
1183 int
1184 mlx5_set_link_down(struct rte_eth_dev *dev)
1185 {
1186         return mlx5_set_flags(dev, ~IFF_UP, ~IFF_UP);
1187 }
1188
1189 /**
1190  * DPDK callback to bring the link UP.
1191  *
1192  * @param dev
1193  *   Pointer to Ethernet device structure.
1194  *
1195  * @return
1196  *   0 on success, a negative errno value otherwise and rte_errno is set.
1197  */
1198 int
1199 mlx5_set_link_up(struct rte_eth_dev *dev)
1200 {
1201         return mlx5_set_flags(dev, ~IFF_UP, IFF_UP);
1202 }
1203
1204 /**
1205  * Configure the TX function to use.
1206  *
1207  * @param dev
1208  *   Pointer to private data structure.
1209  *
1210  * @return
1211  *   Pointer to selected Tx burst function.
1212  */
1213 eth_tx_burst_t
1214 mlx5_select_tx_function(struct rte_eth_dev *dev)
1215 {
1216         struct mlx5_priv *priv = dev->data->dev_private;
1217         eth_tx_burst_t tx_pkt_burst = mlx5_tx_burst;
1218         struct mlx5_dev_config *config = &priv->config;
1219         uint64_t tx_offloads = dev->data->dev_conf.txmode.offloads;
1220         int tso = !!(tx_offloads & (DEV_TX_OFFLOAD_TCP_TSO |
1221                                     DEV_TX_OFFLOAD_VXLAN_TNL_TSO |
1222                                     DEV_TX_OFFLOAD_GRE_TNL_TSO |
1223                                     DEV_TX_OFFLOAD_IP_TNL_TSO |
1224                                     DEV_TX_OFFLOAD_UDP_TNL_TSO));
1225         int swp = !!(tx_offloads & (DEV_TX_OFFLOAD_IP_TNL_TSO |
1226                                     DEV_TX_OFFLOAD_UDP_TNL_TSO |
1227                                     DEV_TX_OFFLOAD_OUTER_IPV4_CKSUM));
1228         int vlan_insert = !!(tx_offloads & DEV_TX_OFFLOAD_VLAN_INSERT);
1229
1230         assert(priv != NULL);
1231         /* Select appropriate TX function. */
1232         if (vlan_insert || tso || swp)
1233                 return tx_pkt_burst;
1234         if (config->mps == MLX5_MPW_ENHANCED) {
1235                 if (mlx5_check_vec_tx_support(dev) > 0) {
1236                         if (mlx5_check_raw_vec_tx_support(dev) > 0)
1237                                 tx_pkt_burst = mlx5_tx_burst_raw_vec;
1238                         else
1239                                 tx_pkt_burst = mlx5_tx_burst_vec;
1240                         DRV_LOG(DEBUG,
1241                                 "port %u selected enhanced MPW Tx vectorized"
1242                                 " function",
1243                                 dev->data->port_id);
1244                 } else {
1245                         tx_pkt_burst = mlx5_tx_burst_empw;
1246                         DRV_LOG(DEBUG,
1247                                 "port %u selected enhanced MPW Tx function",
1248                                 dev->data->port_id);
1249                 }
1250         } else if (config->mps && (config->txq_inline > 0)) {
1251                 tx_pkt_burst = mlx5_tx_burst_mpw_inline;
1252                 DRV_LOG(DEBUG, "port %u selected MPW inline Tx function",
1253                         dev->data->port_id);
1254         } else if (config->mps) {
1255                 tx_pkt_burst = mlx5_tx_burst_mpw;
1256                 DRV_LOG(DEBUG, "port %u selected MPW Tx function",
1257                         dev->data->port_id);
1258         }
1259         return tx_pkt_burst;
1260 }
1261
1262 /**
1263  * Configure the RX function to use.
1264  *
1265  * @param dev
1266  *   Pointer to private data structure.
1267  *
1268  * @return
1269  *   Pointer to selected Rx burst function.
1270  */
1271 eth_rx_burst_t
1272 mlx5_select_rx_function(struct rte_eth_dev *dev)
1273 {
1274         eth_rx_burst_t rx_pkt_burst = mlx5_rx_burst;
1275
1276         assert(dev != NULL);
1277         if (mlx5_check_vec_rx_support(dev) > 0) {
1278                 rx_pkt_burst = mlx5_rx_burst_vec;
1279                 DRV_LOG(DEBUG, "port %u selected Rx vectorized function",
1280                         dev->data->port_id);
1281         } else if (mlx5_mprq_enabled(dev)) {
1282                 rx_pkt_burst = mlx5_rx_burst_mprq;
1283         }
1284         return rx_pkt_burst;
1285 }
1286
1287 /**
1288  * Check if mlx5 device was removed.
1289  *
1290  * @param dev
1291  *   Pointer to Ethernet device structure.
1292  *
1293  * @return
1294  *   1 when device is removed, otherwise 0.
1295  */
1296 int
1297 mlx5_is_removed(struct rte_eth_dev *dev)
1298 {
1299         struct ibv_device_attr device_attr;
1300         struct mlx5_priv *priv = dev->data->dev_private;
1301
1302         if (mlx5_glue->query_device(priv->ctx, &device_attr) == EIO)
1303                 return 1;
1304         return 0;
1305 }
1306
1307 /**
1308  * Get port ID list of mlx5 instances sharing a common device.
1309  *
1310  * @param[in] dev
1311  *   Device to look for.
1312  * @param[out] port_list
1313  *   Result buffer for collected port IDs.
1314  * @param port_list_n
1315  *   Maximum number of entries in result buffer. If 0, @p port_list can be
1316  *   NULL.
1317  *
1318  * @return
1319  *   Number of matching instances regardless of the @p port_list_n
1320  *   parameter, 0 if none were found.
1321  */
1322 unsigned int
1323 mlx5_dev_to_port_id(const struct rte_device *dev, uint16_t *port_list,
1324                     unsigned int port_list_n)
1325 {
1326         uint16_t id;
1327         unsigned int n = 0;
1328
1329         RTE_ETH_FOREACH_DEV(id) {
1330                 struct rte_eth_dev *ldev = &rte_eth_devices[id];
1331
1332                 if (ldev->device != dev)
1333                         continue;
1334                 if (n < port_list_n)
1335                         port_list[n] = id;
1336                 n++;
1337         }
1338         return n;
1339 }
1340
1341 /**
1342  * Get switch information associated with network interface.
1343  *
1344  * @param ifindex
1345  *   Network interface index.
1346  * @param[out] info
1347  *   Switch information object, populated in case of success.
1348  *
1349  * @return
1350  *   0 on success, a negative errno value otherwise and rte_errno is set.
1351  */
1352 int
1353 mlx5_sysfs_switch_info(unsigned int ifindex, struct mlx5_switch_info *info)
1354 {
1355         char ifname[IF_NAMESIZE];
1356         char port_name[IF_NAMESIZE];
1357         FILE *file;
1358         struct mlx5_switch_info data = {
1359                 .master = 0,
1360                 .representor = 0,
1361                 .port_name_new = 0,
1362                 .port_name = 0,
1363                 .switch_id = 0,
1364         };
1365         bool port_name_set = false;
1366         bool port_switch_id_set = false;
1367         char c;
1368         int ret;
1369
1370         if (!if_indextoname(ifindex, ifname)) {
1371                 rte_errno = errno;
1372                 return -rte_errno;
1373         }
1374
1375         MKSTR(phys_port_name, "/sys/class/net/%s/phys_port_name",
1376               ifname);
1377         MKSTR(phys_switch_id, "/sys/class/net/%s/phys_switch_id",
1378               ifname);
1379
1380         file = fopen(phys_port_name, "rb");
1381         if (file != NULL) {
1382                 ret = fscanf(file, "%s", port_name);
1383                 fclose(file);
1384                 if (ret == 1)
1385                         port_name_set = mlx5_translate_port_name(port_name,
1386                                                                  &data);
1387         }
1388         file = fopen(phys_switch_id, "rb");
1389         if (file == NULL) {
1390                 rte_errno = errno;
1391                 return -rte_errno;
1392         }
1393         port_switch_id_set =
1394                 fscanf(file, "%" SCNx64 "%c", &data.switch_id, &c) == 2 &&
1395                 c == '\n';
1396         fclose(file);
1397         data.master = port_switch_id_set && !port_name_set;
1398         data.representor = port_switch_id_set && port_name_set;
1399         *info = data;
1400         return 0;
1401 }
1402
1403 /**
1404  * Extract port name, as a number, from sysfs or netlink information.
1405  *
1406  * @param[in] port_name_in
1407  *   String representing the port name.
1408  * @param[out] port_info_out
1409  *   Port information, including port name as a number.
1410  *
1411  * @return
1412  *   true on success, false otherwise.
1413  */
1414 bool
1415 mlx5_translate_port_name(const char *port_name_in,
1416                          struct mlx5_switch_info *port_info_out)
1417 {
1418         char pf_c1, pf_c2, vf_c1, vf_c2;
1419         char *end;
1420         int32_t pf_num;
1421         bool port_name_set = false;
1422
1423         /*
1424          * Check for port-name as a string of the form pf0vf0
1425          * (support kernel ver >= 5.0)
1426          */
1427         port_name_set = (sscanf(port_name_in, "%c%c%d%c%c%d", &pf_c1, &pf_c2,
1428                                 &pf_num, &vf_c1, &vf_c2,
1429                                 &port_info_out->port_name) == 6);
1430         if (port_name_set) {
1431                 port_info_out->port_name_new = 1;
1432         } else {
1433                 /* Check for port-name as a number (support kernel ver < 5.0 */
1434                 errno = 0;
1435                 port_info_out->port_name = strtol(port_name_in, &end, 0);
1436                 if (!errno &&
1437                     (size_t)(end - port_name_in) == strlen(port_name_in))
1438                         port_name_set = true;
1439         }
1440         return port_name_set;
1441 }