4 * Copyright 2015 6WIND S.A.
5 * Copyright 2015 Mellanox.
7 * Redistribution and use in source and binary forms, with or without
8 * modification, are permitted provided that the following conditions
11 * * Redistributions of source code must retain the above copyright
12 * notice, this list of conditions and the following disclaimer.
13 * * Redistributions in binary form must reproduce the above copyright
14 * notice, this list of conditions and the following disclaimer in
15 * the documentation and/or other materials provided with the
17 * * Neither the name of 6WIND S.A. nor the names of its
18 * contributors may be used to endorse or promote products derived
19 * from this software without specific prior written permission.
21 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
22 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
23 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
24 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
25 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
26 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
27 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
28 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
29 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
30 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
31 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
44 #include <sys/ioctl.h>
45 #include <sys/socket.h>
46 #include <netinet/in.h>
48 #include <linux/ethtool.h>
49 #include <linux/sockios.h>
52 /* DPDK headers don't like -pedantic. */
54 #pragma GCC diagnostic ignored "-pedantic"
56 #include <rte_atomic.h>
57 #include <rte_ethdev.h>
59 #include <rte_common.h>
60 #include <rte_interrupts.h>
61 #include <rte_alarm.h>
62 #include <rte_malloc.h>
64 #pragma GCC diagnostic error "-pedantic"
68 #include "mlx5_rxtx.h"
69 #include "mlx5_utils.h"
72 * Return private structure associated with an Ethernet device.
75 * Pointer to Ethernet device structure.
78 * Pointer to private structure.
81 mlx5_get_priv(struct rte_eth_dev *dev)
83 struct mlx5_secondary_data *sd;
85 if (!mlx5_is_secondary())
86 return dev->data->dev_private;
87 sd = &mlx5_secondary_data[dev->data->port_id];
88 return sd->data.dev_private;
92 * Check if running as a secondary process.
95 * Nonzero if running as a secondary process.
98 mlx5_is_secondary(void)
100 return rte_eal_process_type() != RTE_PROC_PRIMARY;
104 * Get interface name from private structure.
107 * Pointer to private structure.
109 * Interface name output buffer.
112 * 0 on success, -1 on failure and errno is set.
115 priv_get_ifname(const struct priv *priv, char (*ifname)[IF_NAMESIZE])
119 unsigned int dev_type = 0;
120 unsigned int dev_port_prev = ~0u;
121 char match[IF_NAMESIZE] = "";
124 MKSTR(path, "%s/device/net", priv->ctx->device->ibdev_path);
130 while ((dent = readdir(dir)) != NULL) {
131 char *name = dent->d_name;
133 unsigned int dev_port;
136 if ((name[0] == '.') &&
137 ((name[1] == '\0') ||
138 ((name[1] == '.') && (name[2] == '\0'))))
141 MKSTR(path, "%s/device/net/%s/%s",
142 priv->ctx->device->ibdev_path, name,
143 (dev_type ? "dev_id" : "dev_port"));
145 file = fopen(path, "rb");
150 * Switch to dev_id when dev_port does not exist as
151 * is the case with Linux kernel versions < 3.15.
162 r = fscanf(file, (dev_type ? "%x" : "%u"), &dev_port);
167 * Switch to dev_id when dev_port returns the same value for
168 * all ports. May happen when using a MOFED release older than
169 * 3.0 with a Linux kernel >= 3.15.
171 if (dev_port == dev_port_prev)
173 dev_port_prev = dev_port;
174 if (dev_port == (priv->port - 1u))
175 snprintf(match, sizeof(match), "%s", name);
178 if (match[0] == '\0')
180 strncpy(*ifname, match, sizeof(*ifname));
185 * Read from sysfs entry.
188 * Pointer to private structure.
190 * Entry name relative to sysfs path.
192 * Data output buffer.
197 * 0 on success, -1 on failure and errno is set.
200 priv_sysfs_read(const struct priv *priv, const char *entry,
201 char *buf, size_t size)
203 char ifname[IF_NAMESIZE];
208 if (priv_get_ifname(priv, &ifname))
211 MKSTR(path, "%s/device/net/%s/%s", priv->ctx->device->ibdev_path,
214 file = fopen(path, "rb");
217 ret = fread(buf, 1, size, file);
219 if (((size_t)ret < size) && (ferror(file)))
229 * Write to sysfs entry.
232 * Pointer to private structure.
234 * Entry name relative to sysfs path.
241 * 0 on success, -1 on failure and errno is set.
244 priv_sysfs_write(const struct priv *priv, const char *entry,
245 char *buf, size_t size)
247 char ifname[IF_NAMESIZE];
252 if (priv_get_ifname(priv, &ifname))
255 MKSTR(path, "%s/device/net/%s/%s", priv->ctx->device->ibdev_path,
258 file = fopen(path, "wb");
261 ret = fwrite(buf, 1, size, file);
263 if (((size_t)ret < size) || (ferror(file)))
273 * Get unsigned long sysfs property.
276 * Pointer to private structure.
278 * Entry name relative to sysfs path.
280 * Value output buffer.
283 * 0 on success, -1 on failure and errno is set.
286 priv_get_sysfs_ulong(struct priv *priv, const char *name, unsigned long *value)
289 unsigned long value_ret;
292 ret = priv_sysfs_read(priv, name, value_str, (sizeof(value_str) - 1));
294 DEBUG("cannot read %s value from sysfs: %s",
295 name, strerror(errno));
298 value_str[ret] = '\0';
300 value_ret = strtoul(value_str, NULL, 0);
302 DEBUG("invalid %s value `%s': %s", name, value_str,
311 * Set unsigned long sysfs property.
314 * Pointer to private structure.
316 * Entry name relative to sysfs path.
321 * 0 on success, -1 on failure and errno is set.
324 priv_set_sysfs_ulong(struct priv *priv, const char *name, unsigned long value)
327 MKSTR(value_str, "%lu", value);
329 ret = priv_sysfs_write(priv, name, value_str, (sizeof(value_str) - 1));
331 DEBUG("cannot write %s `%s' (%lu) to sysfs: %s",
332 name, value_str, value, strerror(errno));
339 * Perform ifreq ioctl() on associated Ethernet device.
342 * Pointer to private structure.
344 * Request number to pass to ioctl().
346 * Interface request structure output buffer.
349 * 0 on success, -1 on failure and errno is set.
352 priv_ifreq(const struct priv *priv, int req, struct ifreq *ifr)
354 int sock = socket(PF_INET, SOCK_DGRAM, IPPROTO_IP);
359 if (priv_get_ifname(priv, &ifr->ifr_name) == 0)
360 ret = ioctl(sock, req, ifr);
369 * Pointer to private structure.
371 * MTU value output buffer.
374 * 0 on success, -1 on failure and errno is set.
377 priv_get_mtu(struct priv *priv, uint16_t *mtu)
379 unsigned long ulong_mtu;
381 if (priv_get_sysfs_ulong(priv, "mtu", &ulong_mtu) == -1)
391 * Pointer to private structure.
396 * 0 on success, -1 on failure and errno is set.
399 priv_set_mtu(struct priv *priv, uint16_t mtu)
401 return priv_set_sysfs_ulong(priv, "mtu", mtu);
408 * Pointer to private structure.
410 * Bitmask for flags that must remain untouched.
412 * Bitmask for flags to modify.
415 * 0 on success, -1 on failure and errno is set.
418 priv_set_flags(struct priv *priv, unsigned int keep, unsigned int flags)
422 if (priv_get_sysfs_ulong(priv, "flags", &tmp) == -1)
426 return priv_set_sysfs_ulong(priv, "flags", tmp);
430 * Ethernet device configuration.
432 * Prepare the driver for a given number of TX and RX queues.
435 * Pointer to Ethernet device structure.
438 * 0 on success, errno value on failure.
441 dev_configure(struct rte_eth_dev *dev)
443 struct priv *priv = dev->data->dev_private;
444 unsigned int rxqs_n = dev->data->nb_rx_queues;
445 unsigned int txqs_n = dev->data->nb_tx_queues;
448 unsigned int reta_idx_n;
450 priv->rxqs = (void *)dev->data->rx_queues;
451 priv->txqs = (void *)dev->data->tx_queues;
452 if (txqs_n != priv->txqs_n) {
453 INFO("%p: TX queues number update: %u -> %u",
454 (void *)dev, priv->txqs_n, txqs_n);
455 priv->txqs_n = txqs_n;
457 if (rxqs_n > priv->ind_table_max_size) {
458 ERROR("cannot handle this many RX queues (%u)", rxqs_n);
461 if (rxqs_n == priv->rxqs_n)
463 INFO("%p: RX queues number update: %u -> %u",
464 (void *)dev, priv->rxqs_n, rxqs_n);
465 priv->rxqs_n = rxqs_n;
466 /* If the requested number of RX queues is not a power of two, use the
467 * maximum indirection table size for better balancing.
468 * The result is always rounded to the next power of two. */
469 reta_idx_n = (1 << log2above((rxqs_n & (rxqs_n - 1)) ?
470 priv->ind_table_max_size :
472 if (priv_rss_reta_index_resize(priv, reta_idx_n))
474 /* When the number of RX queues is not a power of two, the remaining
475 * table entries are padded with reused WQs and hashes are not spread
477 for (i = 0, j = 0; (i != reta_idx_n); ++i) {
478 (*priv->reta_idx)[i] = j;
486 * DPDK callback for Ethernet device configuration.
489 * Pointer to Ethernet device structure.
492 * 0 on success, negative errno value on failure.
495 mlx5_dev_configure(struct rte_eth_dev *dev)
497 struct priv *priv = dev->data->dev_private;
500 if (mlx5_is_secondary())
501 return -E_RTE_SECONDARY;
504 ret = dev_configure(dev);
511 * DPDK callback to get information about the device.
514 * Pointer to Ethernet device structure.
516 * Info structure output buffer.
519 mlx5_dev_infos_get(struct rte_eth_dev *dev, struct rte_eth_dev_info *info)
521 struct priv *priv = mlx5_get_priv(dev);
523 char ifname[IF_NAMESIZE];
526 /* FIXME: we should ask the device for these values. */
527 info->min_rx_bufsize = 32;
528 info->max_rx_pktlen = 65536;
530 * Since we need one CQ per QP, the limit is the minimum number
531 * between the two values.
533 max = ((priv->device_attr.max_cq > priv->device_attr.max_qp) ?
534 priv->device_attr.max_qp : priv->device_attr.max_cq);
535 /* If max >= 65535 then max = 0, max_rx_queues is uint16_t. */
538 info->max_rx_queues = max;
539 info->max_tx_queues = max;
540 info->max_mac_addrs = RTE_DIM(priv->mac);
541 info->rx_offload_capa =
543 (DEV_RX_OFFLOAD_IPV4_CKSUM |
544 DEV_RX_OFFLOAD_UDP_CKSUM |
545 DEV_RX_OFFLOAD_TCP_CKSUM) :
547 info->tx_offload_capa =
549 (DEV_TX_OFFLOAD_IPV4_CKSUM |
550 DEV_TX_OFFLOAD_UDP_CKSUM |
551 DEV_TX_OFFLOAD_TCP_CKSUM) :
553 if (priv_get_ifname(priv, &ifname) == 0)
554 info->if_index = if_nametoindex(ifname);
555 /* FIXME: RETA update/query API expects the callee to know the size of
556 * the indirection table, for this PMD the size varies depending on
557 * the number of RX queues, it becomes impossible to find the correct
558 * size if it is not fixed.
559 * The API should be updated to solve this problem. */
560 info->reta_size = priv->ind_table_max_size;
565 mlx5_dev_supported_ptypes_get(struct rte_eth_dev *dev)
567 static const uint32_t ptypes[] = {
568 /* refers to rxq_cq_to_pkt_type() */
571 RTE_PTYPE_INNER_L3_IPV4,
572 RTE_PTYPE_INNER_L3_IPV6,
577 if (dev->rx_pkt_burst == mlx5_rx_burst ||
578 dev->rx_pkt_burst == mlx5_rx_burst_sp)
584 * DPDK callback to retrieve physical link information (unlocked version).
587 * Pointer to Ethernet device structure.
588 * @param wait_to_complete
589 * Wait for request completion (ignored).
592 mlx5_link_update_unlocked(struct rte_eth_dev *dev, int wait_to_complete)
594 struct priv *priv = mlx5_get_priv(dev);
595 struct ethtool_cmd edata = {
599 struct rte_eth_link dev_link;
602 (void)wait_to_complete;
603 if (priv_ifreq(priv, SIOCGIFFLAGS, &ifr)) {
604 WARN("ioctl(SIOCGIFFLAGS) failed: %s", strerror(errno));
607 memset(&dev_link, 0, sizeof(dev_link));
608 dev_link.link_status = ((ifr.ifr_flags & IFF_UP) &&
609 (ifr.ifr_flags & IFF_RUNNING));
610 ifr.ifr_data = &edata;
611 if (priv_ifreq(priv, SIOCETHTOOL, &ifr)) {
612 WARN("ioctl(SIOCETHTOOL, ETHTOOL_GSET) failed: %s",
616 link_speed = ethtool_cmd_speed(&edata);
617 if (link_speed == -1)
618 dev_link.link_speed = 0;
620 dev_link.link_speed = link_speed;
621 dev_link.link_duplex = ((edata.duplex == DUPLEX_HALF) ?
622 ETH_LINK_HALF_DUPLEX : ETH_LINK_FULL_DUPLEX);
623 if (memcmp(&dev_link, &dev->data->dev_link, sizeof(dev_link))) {
624 /* Link status changed. */
625 dev->data->dev_link = dev_link;
628 /* Link status is still the same. */
633 * DPDK callback to retrieve physical link information.
636 * Pointer to Ethernet device structure.
637 * @param wait_to_complete
638 * Wait for request completion (ignored).
641 mlx5_link_update(struct rte_eth_dev *dev, int wait_to_complete)
643 struct priv *priv = mlx5_get_priv(dev);
647 ret = mlx5_link_update_unlocked(dev, wait_to_complete);
653 * DPDK callback to change the MTU.
655 * Setting the MTU affects hardware MRU (packets larger than the MTU cannot be
656 * received). Use this as a hint to enable/disable scattered packets support
657 * and improve performance when not needed.
658 * Since failure is not an option, reconfiguring queues on the fly is not
662 * Pointer to Ethernet device structure.
667 * 0 on success, negative errno value on failure.
670 mlx5_dev_set_mtu(struct rte_eth_dev *dev, uint16_t mtu)
672 struct priv *priv = dev->data->dev_private;
675 uint16_t (*rx_func)(void *, struct rte_mbuf **, uint16_t) =
678 if (mlx5_is_secondary())
679 return -E_RTE_SECONDARY;
682 /* Set kernel interface MTU first. */
683 if (priv_set_mtu(priv, mtu)) {
685 WARN("cannot set port %u MTU to %u: %s", priv->port, mtu,
689 DEBUG("adapter port %u MTU set to %u", priv->port, mtu);
691 /* Temporarily replace RX handler with a fake one, assuming it has not
692 * been copied elsewhere. */
693 dev->rx_pkt_burst = removed_rx_burst;
694 /* Make sure everyone has left mlx5_rx_burst() and uses
695 * removed_rx_burst() instead. */
698 /* Reconfigure each RX queue. */
699 for (i = 0; (i != priv->rxqs_n); ++i) {
700 struct rxq *rxq = (*priv->rxqs)[i];
701 unsigned int max_frame_len;
706 /* Calculate new maximum frame length according to MTU and
707 * toggle scattered support (sp) if necessary. */
708 max_frame_len = (priv->mtu + ETHER_HDR_LEN +
709 (ETHER_MAX_VLAN_FRAME_LEN - ETHER_MAX_LEN));
710 sp = (max_frame_len > (rxq->mb_len - RTE_PKTMBUF_HEADROOM));
711 /* Provide new values to rxq_setup(). */
712 dev->data->dev_conf.rxmode.jumbo_frame = sp;
713 dev->data->dev_conf.rxmode.max_rx_pkt_len = max_frame_len;
714 ret = rxq_rehash(dev, rxq);
716 /* Force SP RX if that queue requires it and abort. */
718 rx_func = mlx5_rx_burst_sp;
721 /* Scattered burst function takes priority. */
723 rx_func = mlx5_rx_burst_sp;
725 /* Burst functions can now be called again. */
727 dev->rx_pkt_burst = rx_func;
735 * DPDK callback to get flow control status.
738 * Pointer to Ethernet device structure.
739 * @param[out] fc_conf
740 * Flow control output buffer.
743 * 0 on success, negative errno value on failure.
746 mlx5_dev_get_flow_ctrl(struct rte_eth_dev *dev, struct rte_eth_fc_conf *fc_conf)
748 struct priv *priv = dev->data->dev_private;
750 struct ethtool_pauseparam ethpause = {
751 .cmd = ETHTOOL_GPAUSEPARAM
755 if (mlx5_is_secondary())
756 return -E_RTE_SECONDARY;
758 ifr.ifr_data = ðpause;
760 if (priv_ifreq(priv, SIOCETHTOOL, &ifr)) {
762 WARN("ioctl(SIOCETHTOOL, ETHTOOL_GPAUSEPARAM)"
768 fc_conf->autoneg = ethpause.autoneg;
769 if (ethpause.rx_pause && ethpause.tx_pause)
770 fc_conf->mode = RTE_FC_FULL;
771 else if (ethpause.rx_pause)
772 fc_conf->mode = RTE_FC_RX_PAUSE;
773 else if (ethpause.tx_pause)
774 fc_conf->mode = RTE_FC_TX_PAUSE;
776 fc_conf->mode = RTE_FC_NONE;
786 * DPDK callback to modify flow control parameters.
789 * Pointer to Ethernet device structure.
791 * Flow control parameters.
794 * 0 on success, negative errno value on failure.
797 mlx5_dev_set_flow_ctrl(struct rte_eth_dev *dev, struct rte_eth_fc_conf *fc_conf)
799 struct priv *priv = dev->data->dev_private;
801 struct ethtool_pauseparam ethpause = {
802 .cmd = ETHTOOL_SPAUSEPARAM
806 if (mlx5_is_secondary())
807 return -E_RTE_SECONDARY;
809 ifr.ifr_data = ðpause;
810 ethpause.autoneg = fc_conf->autoneg;
811 if (((fc_conf->mode & RTE_FC_FULL) == RTE_FC_FULL) ||
812 (fc_conf->mode & RTE_FC_RX_PAUSE))
813 ethpause.rx_pause = 1;
815 ethpause.rx_pause = 0;
817 if (((fc_conf->mode & RTE_FC_FULL) == RTE_FC_FULL) ||
818 (fc_conf->mode & RTE_FC_TX_PAUSE))
819 ethpause.tx_pause = 1;
821 ethpause.tx_pause = 0;
824 if (priv_ifreq(priv, SIOCETHTOOL, &ifr)) {
826 WARN("ioctl(SIOCETHTOOL, ETHTOOL_SPAUSEPARAM)"
840 * Get PCI information from struct ibv_device.
843 * Pointer to Ethernet device structure.
844 * @param[out] pci_addr
845 * PCI bus address output buffer.
848 * 0 on success, -1 on failure and errno is set.
851 mlx5_ibv_device_to_pci_addr(const struct ibv_device *device,
852 struct rte_pci_addr *pci_addr)
856 MKSTR(path, "%s/device/uevent", device->ibdev_path);
858 file = fopen(path, "rb");
861 while (fgets(line, sizeof(line), file) == line) {
862 size_t len = strlen(line);
865 /* Truncate long lines. */
866 if (len == (sizeof(line) - 1))
867 while (line[(len - 1)] != '\n') {
871 line[(len - 1)] = ret;
873 /* Extract information. */
876 "%" SCNx16 ":%" SCNx8 ":%" SCNx8 ".%" SCNx8 "\n",
880 &pci_addr->function) == 4) {
890 * Link status handler.
893 * Pointer to private structure.
895 * Pointer to the rte_eth_dev structure.
898 * Nonzero if the callback process can be called immediately.
901 priv_dev_link_status_handler(struct priv *priv, struct rte_eth_dev *dev)
903 struct ibv_async_event event;
907 /* Read all message and acknowledge them. */
909 if (ibv_get_async_event(priv->ctx, &event))
912 if (event.event_type == IBV_EVENT_PORT_ACTIVE ||
913 event.event_type == IBV_EVENT_PORT_ERR)
916 DEBUG("event type %d on port %d not handled",
917 event.event_type, event.element.port_num);
918 ibv_ack_async_event(&event);
921 if (port_change ^ priv->pending_alarm) {
922 struct rte_eth_link *link = &dev->data->dev_link;
924 priv->pending_alarm = 0;
925 mlx5_link_update_unlocked(dev, 0);
926 if (((link->link_speed == 0) && link->link_status) ||
927 ((link->link_speed != 0) && !link->link_status)) {
928 /* Inconsistent status, check again later. */
929 priv->pending_alarm = 1;
930 rte_eal_alarm_set(MLX5_ALARM_TIMEOUT_US,
931 mlx5_dev_link_status_handler,
940 * Handle delayed link status event.
943 * Registered argument.
946 mlx5_dev_link_status_handler(void *arg)
948 struct rte_eth_dev *dev = arg;
949 struct priv *priv = dev->data->dev_private;
953 assert(priv->pending_alarm == 1);
954 ret = priv_dev_link_status_handler(priv, dev);
957 _rte_eth_dev_callback_process(dev, RTE_ETH_EVENT_INTR_LSC);
961 * Handle interrupts from the NIC.
963 * @param[in] intr_handle
969 mlx5_dev_interrupt_handler(struct rte_intr_handle *intr_handle, void *cb_arg)
971 struct rte_eth_dev *dev = cb_arg;
972 struct priv *priv = dev->data->dev_private;
977 ret = priv_dev_link_status_handler(priv, dev);
980 _rte_eth_dev_callback_process(dev, RTE_ETH_EVENT_INTR_LSC);
984 * Uninstall interrupt handler.
987 * Pointer to private structure.
989 * Pointer to the rte_eth_dev structure.
992 priv_dev_interrupt_handler_uninstall(struct priv *priv, struct rte_eth_dev *dev)
994 if (!dev->data->dev_conf.intr_conf.lsc)
996 rte_intr_callback_unregister(&priv->intr_handle,
997 mlx5_dev_interrupt_handler,
999 if (priv->pending_alarm)
1000 rte_eal_alarm_cancel(mlx5_dev_link_status_handler, dev);
1001 priv->pending_alarm = 0;
1002 priv->intr_handle.fd = 0;
1003 priv->intr_handle.type = 0;
1007 * Install interrupt handler.
1010 * Pointer to private structure.
1012 * Pointer to the rte_eth_dev structure.
1015 priv_dev_interrupt_handler_install(struct priv *priv, struct rte_eth_dev *dev)
1019 if (!dev->data->dev_conf.intr_conf.lsc)
1021 assert(priv->ctx->async_fd > 0);
1022 flags = fcntl(priv->ctx->async_fd, F_GETFL);
1023 rc = fcntl(priv->ctx->async_fd, F_SETFL, flags | O_NONBLOCK);
1025 INFO("failed to change file descriptor async event queue");
1026 dev->data->dev_conf.intr_conf.lsc = 0;
1028 priv->intr_handle.fd = priv->ctx->async_fd;
1029 priv->intr_handle.type = RTE_INTR_HANDLE_EXT;
1030 rte_intr_callback_register(&priv->intr_handle,
1031 mlx5_dev_interrupt_handler,
1037 * Change the link state (UP / DOWN).
1040 * Pointer to Ethernet device structure.
1042 * Nonzero for link up, otherwise link down.
1045 * 0 on success, errno value on failure.
1048 priv_set_link(struct priv *priv, int up)
1050 struct rte_eth_dev *dev = priv->dev;
1055 err = priv_set_flags(priv, ~IFF_UP, IFF_UP);
1058 for (i = 0; i < priv->rxqs_n; i++)
1059 if ((*priv->rxqs)[i]->sp)
1061 /* Check if an sp queue exists.
1062 * Note: Some old frames might be received.
1064 if (i == priv->rxqs_n)
1065 dev->rx_pkt_burst = mlx5_rx_burst;
1067 dev->rx_pkt_burst = mlx5_rx_burst_sp;
1068 dev->tx_pkt_burst = mlx5_tx_burst;
1070 err = priv_set_flags(priv, ~IFF_UP, ~IFF_UP);
1073 dev->rx_pkt_burst = removed_rx_burst;
1074 dev->tx_pkt_burst = removed_tx_burst;
1080 * DPDK callback to bring the link DOWN.
1083 * Pointer to Ethernet device structure.
1086 * 0 on success, errno value on failure.
1089 mlx5_set_link_down(struct rte_eth_dev *dev)
1091 struct priv *priv = dev->data->dev_private;
1095 err = priv_set_link(priv, 0);
1101 * DPDK callback to bring the link UP.
1104 * Pointer to Ethernet device structure.
1107 * 0 on success, errno value on failure.
1110 mlx5_set_link_up(struct rte_eth_dev *dev)
1112 struct priv *priv = dev->data->dev_private;
1116 err = priv_set_link(priv, 1);
1122 * Configure secondary process queues from a private data pointer (primary
1123 * or secondary) and update burst callbacks. Can take place only once.
1125 * All queues must have been previously created by the primary process to
1126 * avoid undefined behavior.
1129 * Private data pointer from either primary or secondary process.
1132 * Private data pointer from secondary process, NULL in case of error.
1135 mlx5_secondary_data_setup(struct priv *priv)
1137 unsigned int port_id = 0;
1138 struct mlx5_secondary_data *sd;
1141 unsigned int nb_tx_queues;
1142 unsigned int nb_rx_queues;
1145 /* priv must be valid at this point. */
1146 assert(priv != NULL);
1147 /* priv->dev must also be valid but may point to local memory from
1148 * another process, possibly with the same address and must not
1149 * be dereferenced yet. */
1150 assert(priv->dev != NULL);
1151 /* Determine port ID by finding out where priv comes from. */
1153 sd = &mlx5_secondary_data[port_id];
1154 rte_spinlock_lock(&sd->lock);
1155 /* Primary process? */
1156 if (sd->primary_priv == priv)
1158 /* Secondary process? */
1159 if (sd->data.dev_private == priv)
1161 rte_spinlock_unlock(&sd->lock);
1162 if (++port_id == RTE_DIM(mlx5_secondary_data))
1165 /* Switch to secondary private structure. If private data has already
1166 * been updated by another thread, there is nothing else to do. */
1167 priv = sd->data.dev_private;
1168 if (priv->dev->data == &sd->data)
1170 /* Sanity checks. Secondary private structure is supposed to point
1171 * to local eth_dev, itself still pointing to the shared device data
1172 * structure allocated by the primary process. */
1173 assert(sd->shared_dev_data != &sd->data);
1174 assert(sd->data.nb_tx_queues == 0);
1175 assert(sd->data.tx_queues == NULL);
1176 assert(sd->data.nb_rx_queues == 0);
1177 assert(sd->data.rx_queues == NULL);
1178 assert(priv != sd->primary_priv);
1179 assert(priv->dev->data == sd->shared_dev_data);
1180 assert(priv->txqs_n == 0);
1181 assert(priv->txqs == NULL);
1182 assert(priv->rxqs_n == 0);
1183 assert(priv->rxqs == NULL);
1184 nb_tx_queues = sd->shared_dev_data->nb_tx_queues;
1185 nb_rx_queues = sd->shared_dev_data->nb_rx_queues;
1186 /* Allocate local storage for queues. */
1187 tx_queues = rte_zmalloc("secondary ethdev->tx_queues",
1188 sizeof(sd->data.tx_queues[0]) * nb_tx_queues,
1189 RTE_CACHE_LINE_SIZE);
1190 rx_queues = rte_zmalloc("secondary ethdev->rx_queues",
1191 sizeof(sd->data.rx_queues[0]) * nb_rx_queues,
1192 RTE_CACHE_LINE_SIZE);
1193 if (tx_queues == NULL || rx_queues == NULL)
1195 /* Lock to prevent control operations during setup. */
1198 for (i = 0; i != nb_tx_queues; ++i) {
1199 struct txq *primary_txq = (*sd->primary_priv->txqs)[i];
1202 if (primary_txq == NULL)
1204 txq = rte_calloc_socket("TXQ", 1, sizeof(*txq), 0,
1205 primary_txq->socket);
1207 if (txq_setup(priv->dev,
1209 primary_txq->elts_n * MLX5_PMD_SGE_WR_N,
1210 primary_txq->socket,
1212 txq->stats.idx = primary_txq->stats.idx;
1219 txq = tx_queues[--i];
1226 for (i = 0; i != nb_rx_queues; ++i) {
1227 struct rxq *primary_rxq = (*sd->primary_priv->rxqs)[i];
1229 if (primary_rxq == NULL)
1231 /* Not supported yet. */
1232 rx_queues[i] = NULL;
1234 /* Update everything. */
1235 priv->txqs = (void *)tx_queues;
1236 priv->txqs_n = nb_tx_queues;
1237 priv->rxqs = (void *)rx_queues;
1238 priv->rxqs_n = nb_rx_queues;
1239 sd->data.rx_queues = rx_queues;
1240 sd->data.tx_queues = tx_queues;
1241 sd->data.nb_rx_queues = nb_rx_queues;
1242 sd->data.nb_tx_queues = nb_tx_queues;
1243 sd->data.dev_link = sd->shared_dev_data->dev_link;
1244 sd->data.mtu = sd->shared_dev_data->mtu;
1245 memcpy(sd->data.rx_queue_state, sd->shared_dev_data->rx_queue_state,
1246 sizeof(sd->data.rx_queue_state));
1247 memcpy(sd->data.tx_queue_state, sd->shared_dev_data->tx_queue_state,
1248 sizeof(sd->data.tx_queue_state));
1249 sd->data.dev_flags = sd->shared_dev_data->dev_flags;
1250 /* Use local data from now on. */
1252 priv->dev->data = &sd->data;
1254 priv->dev->tx_pkt_burst = mlx5_tx_burst;
1255 priv->dev->rx_pkt_burst = removed_rx_burst;
1258 /* More sanity checks. */
1259 assert(priv->dev->tx_pkt_burst == mlx5_tx_burst);
1260 assert(priv->dev->rx_pkt_burst == removed_rx_burst);
1261 assert(priv->dev->data == &sd->data);
1262 rte_spinlock_unlock(&sd->lock);
1266 rte_free(tx_queues);
1267 rte_free(rx_queues);
1268 rte_spinlock_unlock(&sd->lock);