+}
+
+/*
+ * Unregister callback handler safely. The handler may be active
+ * while we are trying to unregister it, in this case code -EAGAIN
+ * is returned by rte_intr_callback_unregister(). This routine checks
+ * the return code and tries to unregister handler again.
+ *
+ * @param handle
+ * interrupt handle
+ * @param cb_fn
+ * pointer to callback routine
+ * @cb_arg
+ * opaque callback parameter
+ */
+void
+mlx5_intr_callback_unregister(const struct rte_intr_handle *handle,
+ rte_intr_callback_fn cb_fn, void *cb_arg)
+{
+ /*
+ * Try to reduce timeout management overhead by not calling
+ * the timer related routines on the first iteration. If the
+ * unregistering succeeds on first call there will be no
+ * timer calls at all.
+ */
+ uint64_t twait = 0;
+ uint64_t start = 0;
+
+ do {
+ int ret;
+
+ ret = rte_intr_callback_unregister(handle, cb_fn, cb_arg);
+ if (ret >= 0)
+ return;
+ if (ret != -EAGAIN) {
+ DRV_LOG(INFO, "failed to unregister interrupt"
+ " handler (error: %d)", ret);
+ MLX5_ASSERT(false);
+ return;
+ }
+ if (twait) {
+ struct timespec onems;
+
+ /* Wait one millisecond and try again. */
+ onems.tv_sec = 0;
+ onems.tv_nsec = NS_PER_S / MS_PER_S;
+ nanosleep(&onems, 0);
+ /* Check whether one second elapsed. */
+ if ((rte_get_timer_cycles() - start) <= twait)
+ continue;
+ } else {
+ /*
+ * We get the amount of timer ticks for one second.
+ * If this amount elapsed it means we spent one
+ * second in waiting. This branch is executed once
+ * on first iteration.
+ */
+ twait = rte_get_timer_hz();
+ MLX5_ASSERT(twait);
+ }
+ /*
+ * Timeout elapsed, show message (once a second) and retry.
+ * We have no other acceptable option here, if we ignore
+ * the unregistering return code the handler will not
+ * be unregistered, fd will be closed and we may get the
+ * crush. Hanging and messaging in the loop seems not to be
+ * the worst choice.
+ */
+ DRV_LOG(INFO, "Retrying to unregister interrupt handler");
+ start = rte_get_timer_cycles();
+ } while (true);
+}
+
+/**
+ * Handle DEVX interrupts from the NIC.
+ * This function is probably called from the DPDK host thread.
+ *
+ * @param cb_arg
+ * Callback argument.
+ */
+void
+mlx5_dev_interrupt_handler_devx(void *cb_arg)
+{
+#ifndef HAVE_IBV_DEVX_ASYNC
+ (void)cb_arg;
+ return;
+#else
+ struct mlx5_ibv_shared *sh = cb_arg;
+ union {
+ struct mlx5dv_devx_async_cmd_hdr cmd_resp;
+ uint8_t buf[MLX5_ST_SZ_BYTES(query_flow_counter_out) +
+ MLX5_ST_SZ_BYTES(traffic_counter) +
+ sizeof(struct mlx5dv_devx_async_cmd_hdr)];
+ } out;
+ uint8_t *buf = out.buf + sizeof(out.cmd_resp);
+
+ while (!mlx5_glue->devx_get_async_cmd_comp(sh->devx_comp,
+ &out.cmd_resp,
+ sizeof(out.buf)))
+ mlx5_flow_async_pool_query_handle
+ (sh, (uint64_t)out.cmd_resp.wr_id,
+ mlx5_devx_get_out_command_status(buf));
+#endif /* HAVE_IBV_DEVX_ASYNC */
+}
+
+/**
+ * Uninstall shared asynchronous device events handler.
+ * This function is implemented to support event sharing
+ * between multiple ports of single IB device.
+ *
+ * @param dev
+ * Pointer to Ethernet device.
+ */
+static void
+mlx5_dev_shared_handler_uninstall(struct rte_eth_dev *dev)
+{
+ struct mlx5_priv *priv = dev->data->dev_private;
+ struct mlx5_ibv_shared *sh = priv->sh;
+
+ if (rte_eal_process_type() != RTE_PROC_PRIMARY)
+ return;
+ pthread_mutex_lock(&sh->intr_mutex);
+ MLX5_ASSERT(priv->ibv_port);
+ MLX5_ASSERT(priv->ibv_port <= sh->max_port);
+ MLX5_ASSERT(dev->data->port_id < RTE_MAX_ETHPORTS);
+ if (sh->port[priv->ibv_port - 1].ih_port_id >= RTE_MAX_ETHPORTS)
+ goto exit;
+ MLX5_ASSERT(sh->port[priv->ibv_port - 1].ih_port_id ==
+ (uint32_t)dev->data->port_id);
+ MLX5_ASSERT(sh->intr_cnt);
+ sh->port[priv->ibv_port - 1].ih_port_id = RTE_MAX_ETHPORTS;
+ if (!sh->intr_cnt || --sh->intr_cnt)
+ goto exit;
+ mlx5_intr_callback_unregister(&sh->intr_handle,
+ mlx5_dev_interrupt_handler, sh);
+ sh->intr_handle.fd = 0;
+ sh->intr_handle.type = RTE_INTR_HANDLE_UNKNOWN;
+exit:
+ pthread_mutex_unlock(&sh->intr_mutex);
+}
+
+/**
+ * Uninstall devx shared asynchronous device events handler.
+ * This function is implemeted to support event sharing
+ * between multiple ports of single IB device.
+ *
+ * @param dev
+ * Pointer to Ethernet device.
+ */
+static void
+mlx5_dev_shared_handler_devx_uninstall(struct rte_eth_dev *dev)
+{
+ struct mlx5_priv *priv = dev->data->dev_private;
+ struct mlx5_ibv_shared *sh = priv->sh;
+
+ if (rte_eal_process_type() != RTE_PROC_PRIMARY)
+ return;
+ pthread_mutex_lock(&sh->intr_mutex);
+ MLX5_ASSERT(priv->ibv_port);
+ MLX5_ASSERT(priv->ibv_port <= sh->max_port);
+ MLX5_ASSERT(dev->data->port_id < RTE_MAX_ETHPORTS);
+ if (sh->port[priv->ibv_port - 1].devx_ih_port_id >= RTE_MAX_ETHPORTS)
+ goto exit;
+ MLX5_ASSERT(sh->port[priv->ibv_port - 1].devx_ih_port_id ==
+ (uint32_t)dev->data->port_id);
+ sh->port[priv->ibv_port - 1].devx_ih_port_id = RTE_MAX_ETHPORTS;
+ if (!sh->devx_intr_cnt || --sh->devx_intr_cnt)
+ goto exit;
+ if (sh->intr_handle_devx.fd) {
+ rte_intr_callback_unregister(&sh->intr_handle_devx,
+ mlx5_dev_interrupt_handler_devx,
+ sh);
+ sh->intr_handle_devx.fd = 0;
+ sh->intr_handle_devx.type = RTE_INTR_HANDLE_UNKNOWN;
+ }
+ if (sh->devx_comp) {
+ mlx5_glue->devx_destroy_cmd_comp(sh->devx_comp);
+ sh->devx_comp = NULL;
+ }
+exit:
+ pthread_mutex_unlock(&sh->intr_mutex);
+}
+
+/**
+ * Install shared asynchronous device events handler.
+ * This function is implemented to support event sharing
+ * between multiple ports of single IB device.
+ *
+ * @param dev
+ * Pointer to Ethernet device.
+ */
+static void
+mlx5_dev_shared_handler_install(struct rte_eth_dev *dev)
+{
+ struct mlx5_priv *priv = dev->data->dev_private;
+ struct mlx5_ibv_shared *sh = priv->sh;
+ int ret;
+ int flags;
+
+ if (rte_eal_process_type() != RTE_PROC_PRIMARY)
+ return;
+ pthread_mutex_lock(&sh->intr_mutex);
+ MLX5_ASSERT(priv->ibv_port);
+ MLX5_ASSERT(priv->ibv_port <= sh->max_port);
+ MLX5_ASSERT(dev->data->port_id < RTE_MAX_ETHPORTS);
+ if (sh->port[priv->ibv_port - 1].ih_port_id < RTE_MAX_ETHPORTS) {
+ /* The handler is already installed for this port. */
+ MLX5_ASSERT(sh->intr_cnt);
+ goto exit;
+ }
+ if (sh->intr_cnt) {
+ sh->port[priv->ibv_port - 1].ih_port_id =
+ (uint32_t)dev->data->port_id;
+ sh->intr_cnt++;
+ goto exit;
+ }
+ /* No shared handler installed. */
+ MLX5_ASSERT(sh->ctx->async_fd > 0);
+ flags = fcntl(sh->ctx->async_fd, F_GETFL);
+ ret = fcntl(sh->ctx->async_fd, F_SETFL, flags | O_NONBLOCK);
+ if (ret) {
+ DRV_LOG(INFO, "failed to change file descriptor async event"
+ " queue");
+ /* Indicate there will be no interrupts. */
+ dev->data->dev_conf.intr_conf.lsc = 0;
+ dev->data->dev_conf.intr_conf.rmv = 0;
+ } else {
+ sh->intr_handle.fd = sh->ctx->async_fd;
+ sh->intr_handle.type = RTE_INTR_HANDLE_EXT;
+ rte_intr_callback_register(&sh->intr_handle,
+ mlx5_dev_interrupt_handler, sh);
+ sh->intr_cnt++;
+ sh->port[priv->ibv_port - 1].ih_port_id =
+ (uint32_t)dev->data->port_id;
+ }
+exit:
+ pthread_mutex_unlock(&sh->intr_mutex);
+}
+
+/**
+ * Install devx shared asyncronous device events handler.
+ * This function is implemeted to support event sharing
+ * between multiple ports of single IB device.
+ *
+ * @param dev
+ * Pointer to Ethernet device.
+ */
+static void
+mlx5_dev_shared_handler_devx_install(struct rte_eth_dev *dev)
+{
+ struct mlx5_priv *priv = dev->data->dev_private;
+ struct mlx5_ibv_shared *sh = priv->sh;
+
+ if (rte_eal_process_type() != RTE_PROC_PRIMARY)
+ return;
+ pthread_mutex_lock(&sh->intr_mutex);
+ MLX5_ASSERT(priv->ibv_port);
+ MLX5_ASSERT(priv->ibv_port <= sh->max_port);
+ MLX5_ASSERT(dev->data->port_id < RTE_MAX_ETHPORTS);
+ if (sh->port[priv->ibv_port - 1].devx_ih_port_id < RTE_MAX_ETHPORTS) {
+ /* The handler is already installed for this port. */
+ MLX5_ASSERT(sh->devx_intr_cnt);
+ goto exit;
+ }
+ if (sh->devx_intr_cnt) {
+ sh->devx_intr_cnt++;
+ sh->port[priv->ibv_port - 1].devx_ih_port_id =
+ (uint32_t)dev->data->port_id;
+ goto exit;
+ }
+ if (priv->config.devx) {
+#ifndef HAVE_IBV_DEVX_ASYNC
+ goto exit;
+#else
+ sh->devx_comp = mlx5_glue->devx_create_cmd_comp(sh->ctx);
+ if (sh->devx_comp) {
+ int flags = fcntl(sh->devx_comp->fd, F_GETFL);
+ int ret = fcntl(sh->devx_comp->fd, F_SETFL,
+ flags | O_NONBLOCK);
+
+ if (ret) {
+ DRV_LOG(INFO, "failed to change file descriptor"
+ " devx async event queue");
+ } else {
+ sh->intr_handle_devx.fd = sh->devx_comp->fd;
+ sh->intr_handle_devx.type = RTE_INTR_HANDLE_EXT;
+ rte_intr_callback_register
+ (&sh->intr_handle_devx,
+ mlx5_dev_interrupt_handler_devx, sh);
+ sh->devx_intr_cnt++;
+ sh->port[priv->ibv_port - 1].devx_ih_port_id =
+ (uint32_t)dev->data->port_id;
+ }
+ }
+#endif /* HAVE_IBV_DEVX_ASYNC */
+ }
+exit:
+ pthread_mutex_unlock(&sh->intr_mutex);
+}
+
+/**
+ * Uninstall interrupt handler.
+ *
+ * @param dev
+ * Pointer to Ethernet device.
+ */
+void
+mlx5_dev_interrupt_handler_uninstall(struct rte_eth_dev *dev)
+{
+ mlx5_dev_shared_handler_uninstall(dev);
+}
+
+/**
+ * Install interrupt handler.
+ *
+ * @param dev
+ * Pointer to Ethernet device.
+ */
+void
+mlx5_dev_interrupt_handler_install(struct rte_eth_dev *dev)
+{
+ mlx5_dev_shared_handler_install(dev);
+}
+
+/**
+ * Devx uninstall interrupt handler.
+ *
+ * @param dev
+ * Pointer to Ethernet device.
+ */
+void
+mlx5_dev_interrupt_handler_devx_uninstall(struct rte_eth_dev *dev)
+{
+ mlx5_dev_shared_handler_devx_uninstall(dev);
+}
+
+/**
+ * Devx install interrupt handler.
+ *
+ * @param dev
+ * Pointer to Ethernet device.
+ */
+void
+mlx5_dev_interrupt_handler_devx_install(struct rte_eth_dev *dev)
+{
+ mlx5_dev_shared_handler_devx_install(dev);
+}
+
+/**
+ * DPDK callback to bring the link DOWN.
+ *
+ * @param dev
+ * Pointer to Ethernet device structure.
+ *
+ * @return
+ * 0 on success, a negative errno value otherwise and rte_errno is set.
+ */
+int
+mlx5_set_link_down(struct rte_eth_dev *dev)
+{
+ return mlx5_set_flags(dev, ~IFF_UP, ~IFF_UP);
+}
+
+/**
+ * DPDK callback to bring the link UP.
+ *
+ * @param dev
+ * Pointer to Ethernet device structure.
+ *
+ * @return
+ * 0 on success, a negative errno value otherwise and rte_errno is set.
+ */
+int
+mlx5_set_link_up(struct rte_eth_dev *dev)
+{
+ return mlx5_set_flags(dev, ~IFF_UP, IFF_UP);
+}
+
+/**
+ * Configure the RX function to use.
+ *
+ * @param dev
+ * Pointer to private data structure.
+ *
+ * @return
+ * Pointer to selected Rx burst function.
+ */
+eth_rx_burst_t
+mlx5_select_rx_function(struct rte_eth_dev *dev)
+{
+ eth_rx_burst_t rx_pkt_burst = mlx5_rx_burst;
+
+ MLX5_ASSERT(dev != NULL);
+ if (mlx5_check_vec_rx_support(dev) > 0) {
+ rx_pkt_burst = mlx5_rx_burst_vec;
+ DRV_LOG(DEBUG, "port %u selected Rx vectorized function",
+ dev->data->port_id);
+ } else if (mlx5_mprq_enabled(dev)) {
+ rx_pkt_burst = mlx5_rx_burst_mprq;
+ }
+ return rx_pkt_burst;
+}
+
+/**
+ * Check if mlx5 device was removed.
+ *
+ * @param dev
+ * Pointer to Ethernet device structure.
+ *
+ * @return
+ * 1 when device is removed, otherwise 0.
+ */
+int
+mlx5_is_removed(struct rte_eth_dev *dev)
+{
+ struct ibv_device_attr device_attr;
+ struct mlx5_priv *priv = dev->data->dev_private;
+
+ if (mlx5_glue->query_device(priv->sh->ctx, &device_attr) == EIO)
+ return 1;
+ return 0;
+}
+
+/**
+ * Get the E-Switch parameters by port id.
+ *
+ * @param[in] port
+ * Device port id.
+ * @param[in] valid
+ * Device port id is valid, skip check. This flag is useful
+ * when trials are performed from probing and device is not
+ * flagged as valid yet (in attaching process).
+ * @param[out] es_domain_id
+ * E-Switch domain id.
+ * @param[out] es_port_id
+ * The port id of the port in the E-Switch.
+ *
+ * @return
+ * pointer to device private data structure containing data needed
+ * on success, NULL otherwise and rte_errno is set.
+ */
+struct mlx5_priv *
+mlx5_port_to_eswitch_info(uint16_t port, bool valid)
+{
+ struct rte_eth_dev *dev;
+ struct mlx5_priv *priv;
+
+ if (port >= RTE_MAX_ETHPORTS) {
+ rte_errno = EINVAL;
+ return NULL;
+ }
+ if (!valid && !rte_eth_dev_is_valid_port(port)) {
+ rte_errno = ENODEV;
+ return NULL;
+ }
+ dev = &rte_eth_devices[port];
+ priv = dev->data->dev_private;
+ if (!(priv->representor || priv->master)) {
+ rte_errno = EINVAL;
+ return NULL;
+ }
+ return priv;
+}
+
+/**
+ * Get the E-Switch parameters by device instance.
+ *
+ * @param[in] port
+ * Device port id.
+ * @param[out] es_domain_id
+ * E-Switch domain id.
+ * @param[out] es_port_id
+ * The port id of the port in the E-Switch.
+ *
+ * @return
+ * pointer to device private data structure containing data needed
+ * on success, NULL otherwise and rte_errno is set.
+ */
+struct mlx5_priv *
+mlx5_dev_to_eswitch_info(struct rte_eth_dev *dev)
+{
+ struct mlx5_priv *priv;
+
+ priv = dev->data->dev_private;
+ if (!(priv->representor || priv->master)) {
+ rte_errno = EINVAL;
+ return NULL;
+ }
+ return priv;
+}
+
+/**
+ * Get switch information associated with network interface.
+ *
+ * @param ifindex
+ * Network interface index.
+ * @param[out] info
+ * Switch information object, populated in case of success.
+ *
+ * @return
+ * 0 on success, a negative errno value otherwise and rte_errno is set.
+ */
+int
+mlx5_sysfs_switch_info(unsigned int ifindex, struct mlx5_switch_info *info)
+{
+ char ifname[IF_NAMESIZE];
+ char port_name[IF_NAMESIZE];
+ FILE *file;
+ struct mlx5_switch_info data = {
+ .master = 0,
+ .representor = 0,
+ .name_type = MLX5_PHYS_PORT_NAME_TYPE_NOTSET,
+ .port_name = 0,
+ .switch_id = 0,
+ };
+ DIR *dir;
+ bool port_switch_id_set = false;
+ bool device_dir = false;
+ char c;
+ int ret;
+
+ if (!if_indextoname(ifindex, ifname)) {
+ rte_errno = errno;
+ return -rte_errno;
+ }
+
+ MKSTR(phys_port_name, "/sys/class/net/%s/phys_port_name",
+ ifname);
+ MKSTR(phys_switch_id, "/sys/class/net/%s/phys_switch_id",
+ ifname);
+ MKSTR(pci_device, "/sys/class/net/%s/device",
+ ifname);
+
+ file = fopen(phys_port_name, "rb");
+ if (file != NULL) {
+ ret = fscanf(file, "%s", port_name);
+ fclose(file);
+ if (ret == 1)
+ mlx5_translate_port_name(port_name, &data);
+ }
+ file = fopen(phys_switch_id, "rb");
+ if (file == NULL) {
+ rte_errno = errno;
+ return -rte_errno;
+ }
+ port_switch_id_set =
+ fscanf(file, "%" SCNx64 "%c", &data.switch_id, &c) == 2 &&
+ c == '\n';