- goto err;
- }
- ret = mlx4_priv_flow_start(priv);
- if (ret) {
- ERROR("%p: flow start failed: %s",
- (void *)dev, strerror(ret));
- goto err;
- }
- priv_unlock(priv);
- return 0;
-err:
- /* Rollback. */
- priv_mac_addr_del(priv);
- priv->started = 0;
- priv_unlock(priv);
- return ret;
-}
-
-/**
- * DPDK callback to stop the device.
- *
- * Simulate device stop by detaching all configured flows.
- *
- * @param dev
- * Pointer to Ethernet device structure.
- */
-static void
-mlx4_dev_stop(struct rte_eth_dev *dev)
-{
- struct priv *priv = dev->data->dev_private;
-
- priv_lock(priv);
- if (!priv->started) {
- priv_unlock(priv);
- return;
- }
- DEBUG("%p: detaching flows from all RX queues", (void *)dev);
- priv->started = 0;
- mlx4_priv_flow_stop(priv);
- priv_mac_addr_del(priv);
- priv_unlock(priv);
-}
-
-/**
- * Dummy DPDK callback for TX.
- *
- * This function is used to temporarily replace the real callback during
- * unsafe control operations on the queue, or in case of error.
- *
- * @param dpdk_txq
- * Generic pointer to TX queue structure.
- * @param[in] pkts
- * Packets to transmit.
- * @param pkts_n
- * Number of packets in array.
- *
- * @return
- * Number of packets successfully transmitted (<= pkts_n).
- */
-static uint16_t
-removed_tx_burst(void *dpdk_txq, struct rte_mbuf **pkts, uint16_t pkts_n)
-{
- (void)dpdk_txq;
- (void)pkts;
- (void)pkts_n;
- return 0;
-}
-
-/**
- * Dummy DPDK callback for RX.
- *
- * This function is used to temporarily replace the real callback during
- * unsafe control operations on the queue, or in case of error.
- *
- * @param dpdk_rxq
- * Generic pointer to RX queue structure.
- * @param[out] pkts
- * Array to store received packets.
- * @param pkts_n
- * Maximum number of packets in array.
- *
- * @return
- * Number of packets successfully received (<= pkts_n).
- */
-static uint16_t
-removed_rx_burst(void *dpdk_rxq, struct rte_mbuf **pkts, uint16_t pkts_n)
-{
- (void)dpdk_rxq;
- (void)pkts;
- (void)pkts_n;
- return 0;
-}
-
-static int
-priv_dev_interrupt_handler_uninstall(struct priv *, struct rte_eth_dev *);
-
-static int
-priv_dev_removal_interrupt_handler_uninstall(struct priv *,
- struct rte_eth_dev *);
-
-static int
-priv_dev_link_interrupt_handler_uninstall(struct priv *, struct rte_eth_dev *);
-
-/**
- * DPDK callback to close the device.
- *
- * Destroy all queues and objects, free memory.
- *
- * @param dev
- * Pointer to Ethernet device structure.
- */
-static void
-mlx4_dev_close(struct rte_eth_dev *dev)
-{
- struct priv *priv = dev->data->dev_private;
- void *tmp;
- unsigned int i;
-
- if (priv == NULL)
- return;
- priv_lock(priv);
- DEBUG("%p: closing device \"%s\"",
- (void *)dev,
- ((priv->ctx != NULL) ? priv->ctx->device->name : ""));
- priv_mac_addr_del(priv);
- /*
- * Prevent crashes when queues are still in use. This is unfortunately
- * still required for DPDK 1.3 because some programs (such as testpmd)
- * never release them before closing the device.
- */
- dev->rx_pkt_burst = removed_rx_burst;
- dev->tx_pkt_burst = removed_tx_burst;
- if (priv->rxqs != NULL) {
- /* XXX race condition if mlx4_rx_burst() is still running. */
- usleep(1000);
- for (i = 0; (i != priv->rxqs_n); ++i) {
- tmp = (*priv->rxqs)[i];
- if (tmp == NULL)
- continue;
- (*priv->rxqs)[i] = NULL;
- rxq_cleanup(tmp);
- rte_free(tmp);
- }
- priv->rxqs_n = 0;
- priv->rxqs = NULL;
- }
- if (priv->txqs != NULL) {
- /* XXX race condition if mlx4_tx_burst() is still running. */
- usleep(1000);
- for (i = 0; (i != priv->txqs_n); ++i) {
- tmp = (*priv->txqs)[i];
- if (tmp == NULL)
- continue;
- (*priv->txqs)[i] = NULL;
- txq_cleanup(tmp);
- rte_free(tmp);
- }
- priv->txqs_n = 0;
- priv->txqs = NULL;
- }
- if (priv->pd != NULL) {
- assert(priv->ctx != NULL);
- claim_zero(ibv_dealloc_pd(priv->pd));
- claim_zero(ibv_close_device(priv->ctx));
- } else
- assert(priv->ctx == NULL);
- priv_dev_removal_interrupt_handler_uninstall(priv, dev);
- priv_dev_link_interrupt_handler_uninstall(priv, dev);
- priv_rx_intr_vec_disable(priv);
- priv_unlock(priv);
- memset(priv, 0, sizeof(*priv));
-}
-
-/**
- * Change the link state (UP / DOWN).
- *
- * @param priv
- * Pointer to Ethernet device private data.
- * @param up
- * Nonzero for link up, otherwise link down.
- *
- * @return
- * 0 on success, negative errno value otherwise and rte_errno is set.
- */
-static int
-priv_set_link(struct priv *priv, int up)
-{
- struct rte_eth_dev *dev = priv->dev;
- int err;
-
- if (up) {
- err = priv_set_flags(priv, ~IFF_UP, IFF_UP);
- if (err)
- return err;
- dev->rx_pkt_burst = mlx4_rx_burst;
- } else {
- err = priv_set_flags(priv, ~IFF_UP, ~IFF_UP);
- if (err)
- return err;
- dev->rx_pkt_burst = removed_rx_burst;
- dev->tx_pkt_burst = removed_tx_burst;
- }
- return 0;
-}
-
-/**
- * DPDK callback to bring the link DOWN.
- *
- * @param dev
- * Pointer to Ethernet device structure.
- *
- * @return
- * 0 on success, negative errno value otherwise and rte_errno is set.
- */
-static int
-mlx4_set_link_down(struct rte_eth_dev *dev)
-{
- struct priv *priv = dev->data->dev_private;
- int err;
-
- priv_lock(priv);
- err = priv_set_link(priv, 0);
- priv_unlock(priv);
- return err;
-}
-
-/**
- * DPDK callback to bring the link UP.
- *
- * @param dev
- * Pointer to Ethernet device structure.
- *
- * @return
- * 0 on success, negative errno value otherwise and rte_errno is set.
- */
-static int
-mlx4_set_link_up(struct rte_eth_dev *dev)
-{
- struct priv *priv = dev->data->dev_private;
- int err;
-
- priv_lock(priv);
- err = priv_set_link(priv, 1);
- priv_unlock(priv);
- return err;
-}
-
-/**
- * DPDK callback to get information about the device.
- *
- * @param dev
- * Pointer to Ethernet device structure.
- * @param[out] info
- * Info structure output buffer.
- */
-static void
-mlx4_dev_infos_get(struct rte_eth_dev *dev, struct rte_eth_dev_info *info)
-{
- struct priv *priv = dev->data->dev_private;
- unsigned int max;
- char ifname[IF_NAMESIZE];
-
- info->pci_dev = RTE_ETH_DEV_TO_PCI(dev);
- if (priv == NULL)
- return;
- priv_lock(priv);
- /* FIXME: we should ask the device for these values. */
- info->min_rx_bufsize = 32;
- info->max_rx_pktlen = 65536;
- /*
- * Since we need one CQ per QP, the limit is the minimum number
- * between the two values.
- */
- max = ((priv->device_attr.max_cq > priv->device_attr.max_qp) ?
- priv->device_attr.max_qp : priv->device_attr.max_cq);
- /* If max >= 65535 then max = 0, max_rx_queues is uint16_t. */
- if (max >= 65535)
- max = 65535;
- info->max_rx_queues = max;
- info->max_tx_queues = max;
- /* Last array entry is reserved for broadcast. */
- info->max_mac_addrs = 1;
- info->rx_offload_capa = 0;
- info->tx_offload_capa = 0;
- if (priv_get_ifname(priv, &ifname) == 0)
- info->if_index = if_nametoindex(ifname);
- info->speed_capa =
- ETH_LINK_SPEED_1G |
- ETH_LINK_SPEED_10G |
- ETH_LINK_SPEED_20G |
- ETH_LINK_SPEED_40G |
- ETH_LINK_SPEED_56G;
- priv_unlock(priv);
-}
-
-/**
- * DPDK callback to get device statistics.
- *
- * @param dev
- * Pointer to Ethernet device structure.
- * @param[out] stats
- * Stats structure output buffer.
- */
-static void
-mlx4_stats_get(struct rte_eth_dev *dev, struct rte_eth_stats *stats)
-{
- struct priv *priv = dev->data->dev_private;
- struct rte_eth_stats tmp = {0};
- unsigned int i;
- unsigned int idx;
-
- if (priv == NULL)
- return;
- priv_lock(priv);
- /* Add software counters. */
- for (i = 0; (i != priv->rxqs_n); ++i) {
- struct rxq *rxq = (*priv->rxqs)[i];
-
- if (rxq == NULL)
- continue;
- idx = rxq->stats.idx;
- if (idx < RTE_ETHDEV_QUEUE_STAT_CNTRS) {
- tmp.q_ipackets[idx] += rxq->stats.ipackets;
- tmp.q_ibytes[idx] += rxq->stats.ibytes;
- tmp.q_errors[idx] += (rxq->stats.idropped +
- rxq->stats.rx_nombuf);
- }
- tmp.ipackets += rxq->stats.ipackets;
- tmp.ibytes += rxq->stats.ibytes;
- tmp.ierrors += rxq->stats.idropped;
- tmp.rx_nombuf += rxq->stats.rx_nombuf;
- }
- for (i = 0; (i != priv->txqs_n); ++i) {
- struct txq *txq = (*priv->txqs)[i];
-
- if (txq == NULL)
- continue;
- idx = txq->stats.idx;
- if (idx < RTE_ETHDEV_QUEUE_STAT_CNTRS) {
- tmp.q_opackets[idx] += txq->stats.opackets;
- tmp.q_obytes[idx] += txq->stats.obytes;
- tmp.q_errors[idx] += txq->stats.odropped;
- }
- tmp.opackets += txq->stats.opackets;
- tmp.obytes += txq->stats.obytes;
- tmp.oerrors += txq->stats.odropped;
- }
- *stats = tmp;
- priv_unlock(priv);
-}
-
-/**
- * DPDK callback to clear device statistics.
- *
- * @param dev
- * Pointer to Ethernet device structure.
- */
-static void
-mlx4_stats_reset(struct rte_eth_dev *dev)
-{
- struct priv *priv = dev->data->dev_private;
- unsigned int i;
- unsigned int idx;
-
- if (priv == NULL)
- return;
- priv_lock(priv);
- for (i = 0; (i != priv->rxqs_n); ++i) {
- if ((*priv->rxqs)[i] == NULL)
- continue;
- idx = (*priv->rxqs)[i]->stats.idx;
- (*priv->rxqs)[i]->stats =
- (struct mlx4_rxq_stats){ .idx = idx };
- }
- for (i = 0; (i != priv->txqs_n); ++i) {
- if ((*priv->txqs)[i] == NULL)
- continue;
- idx = (*priv->txqs)[i]->stats.idx;
- (*priv->txqs)[i]->stats =
- (struct mlx4_txq_stats){ .idx = idx };
- }
- priv_unlock(priv);
-}
-
-/**
- * DPDK callback to retrieve physical link information.
- *
- * @param dev
- * Pointer to Ethernet device structure.
- * @param wait_to_complete
- * Wait for request completion (ignored).
- *
- * @return
- * 0 on success, negative errno value otherwise and rte_errno is set.
- */
-static int
-mlx4_link_update(struct rte_eth_dev *dev, int wait_to_complete)
-{
- const struct priv *priv = dev->data->dev_private;
- struct ethtool_cmd edata = {
- .cmd = ETHTOOL_GSET
- };
- struct ifreq ifr;
- struct rte_eth_link dev_link;
- int link_speed = 0;
-
- /* priv_lock() is not taken to allow concurrent calls. */
- if (priv == NULL) {
- rte_errno = EINVAL;
- return -rte_errno;
- }
- (void)wait_to_complete;
- if (priv_ifreq(priv, SIOCGIFFLAGS, &ifr)) {
- WARN("ioctl(SIOCGIFFLAGS) failed: %s", strerror(rte_errno));
- return -rte_errno;
- }
- memset(&dev_link, 0, sizeof(dev_link));
- dev_link.link_status = ((ifr.ifr_flags & IFF_UP) &&
- (ifr.ifr_flags & IFF_RUNNING));
- ifr.ifr_data = (void *)&edata;
- if (priv_ifreq(priv, SIOCETHTOOL, &ifr)) {
- WARN("ioctl(SIOCETHTOOL, ETHTOOL_GSET) failed: %s",
- strerror(rte_errno));
- return -rte_errno;
- }
- link_speed = ethtool_cmd_speed(&edata);
- if (link_speed == -1)
- dev_link.link_speed = 0;
- else
- dev_link.link_speed = link_speed;
- dev_link.link_duplex = ((edata.duplex == DUPLEX_HALF) ?
- ETH_LINK_HALF_DUPLEX : ETH_LINK_FULL_DUPLEX);
- dev_link.link_autoneg = !(dev->data->dev_conf.link_speeds &
- ETH_LINK_SPEED_FIXED);
- dev->data->dev_link = dev_link;
- return 0;
-}
-
-/**
- * DPDK callback to change the MTU.
- *
- * @param dev
- * Pointer to Ethernet device structure.
- * @param in_mtu
- * New MTU.
- *
- * @return
- * 0 on success, negative errno value otherwise and rte_errno is set.
- */
-static int
-mlx4_dev_set_mtu(struct rte_eth_dev *dev, uint16_t mtu)
-{
- struct priv *priv = dev->data->dev_private;
- int ret = 0;
-
- priv_lock(priv);
- /* Set kernel interface MTU first. */
- if (priv_set_mtu(priv, mtu)) {
- ret = rte_errno;
- WARN("cannot set port %u MTU to %u: %s", priv->port, mtu,
- strerror(rte_errno));
- goto out;
- } else
- DEBUG("adapter port %u MTU set to %u", priv->port, mtu);
- priv->mtu = mtu;
-out:
- priv_unlock(priv);
- assert(ret >= 0);
- return -ret;
-}
-
-/**
- * DPDK callback to get flow control status.
- *
- * @param dev
- * Pointer to Ethernet device structure.
- * @param[out] fc_conf
- * Flow control output buffer.
- *
- * @return
- * 0 on success, negative errno value otherwise and rte_errno is set.
- */
-static int
-mlx4_dev_get_flow_ctrl(struct rte_eth_dev *dev, struct rte_eth_fc_conf *fc_conf)
-{
- struct priv *priv = dev->data->dev_private;
- struct ifreq ifr;
- struct ethtool_pauseparam ethpause = {
- .cmd = ETHTOOL_GPAUSEPARAM
- };
- int ret;
-
- ifr.ifr_data = (void *)ðpause;
- priv_lock(priv);
- if (priv_ifreq(priv, SIOCETHTOOL, &ifr)) {
- ret = rte_errno;
- WARN("ioctl(SIOCETHTOOL, ETHTOOL_GPAUSEPARAM)"
- " failed: %s",
- strerror(rte_errno));
- goto out;
- }
- fc_conf->autoneg = ethpause.autoneg;
- if (ethpause.rx_pause && ethpause.tx_pause)
- fc_conf->mode = RTE_FC_FULL;
- else if (ethpause.rx_pause)
- fc_conf->mode = RTE_FC_RX_PAUSE;
- else if (ethpause.tx_pause)
- fc_conf->mode = RTE_FC_TX_PAUSE;
- else
- fc_conf->mode = RTE_FC_NONE;
- ret = 0;
-out:
- priv_unlock(priv);
- assert(ret >= 0);
- return -ret;
-}
-
-/**
- * DPDK callback to modify flow control parameters.
- *
- * @param dev
- * Pointer to Ethernet device structure.
- * @param[in] fc_conf
- * Flow control parameters.
- *
- * @return
- * 0 on success, negative errno value otherwise and rte_errno is set.
- */
-static int
-mlx4_dev_set_flow_ctrl(struct rte_eth_dev *dev, struct rte_eth_fc_conf *fc_conf)
-{
- struct priv *priv = dev->data->dev_private;
- struct ifreq ifr;
- struct ethtool_pauseparam ethpause = {
- .cmd = ETHTOOL_SPAUSEPARAM
- };
- int ret;
-
- ifr.ifr_data = (void *)ðpause;
- ethpause.autoneg = fc_conf->autoneg;
- if (((fc_conf->mode & RTE_FC_FULL) == RTE_FC_FULL) ||
- (fc_conf->mode & RTE_FC_RX_PAUSE))
- ethpause.rx_pause = 1;
- else
- ethpause.rx_pause = 0;
- if (((fc_conf->mode & RTE_FC_FULL) == RTE_FC_FULL) ||
- (fc_conf->mode & RTE_FC_TX_PAUSE))
- ethpause.tx_pause = 1;
- else
- ethpause.tx_pause = 0;
- priv_lock(priv);
- if (priv_ifreq(priv, SIOCETHTOOL, &ifr)) {
- ret = rte_errno;
- WARN("ioctl(SIOCETHTOOL, ETHTOOL_SPAUSEPARAM)"
- " failed: %s",
- strerror(rte_errno));
- goto out;
- }
- ret = 0;
-out:
- priv_unlock(priv);
- assert(ret >= 0);
- return -ret;
-}
-
-const struct rte_flow_ops mlx4_flow_ops = {
- .validate = mlx4_flow_validate,
- .create = mlx4_flow_create,
- .destroy = mlx4_flow_destroy,
- .flush = mlx4_flow_flush,
- .query = NULL,
- .isolate = mlx4_flow_isolate,
-};
-
-/**
- * Manage filter operations.
- *
- * @param dev
- * Pointer to Ethernet device structure.
- * @param filter_type
- * Filter type.
- * @param filter_op
- * Operation to perform.
- * @param arg
- * Pointer to operation-specific structure.
- *
- * @return
- * 0 on success, negative errno value otherwise and rte_errno is set.
- */
-static int
-mlx4_dev_filter_ctrl(struct rte_eth_dev *dev,
- enum rte_filter_type filter_type,
- enum rte_filter_op filter_op,
- void *arg)
-{
- switch (filter_type) {
- case RTE_ETH_FILTER_GENERIC:
- if (filter_op != RTE_ETH_FILTER_GET)
- break;
- *(const void **)arg = &mlx4_flow_ops;
- return 0;
- default:
- ERROR("%p: filter type (%d) not supported",
- (void *)dev, filter_type);
- break;
- }
- rte_errno = ENOTSUP;
- return -rte_errno;
-}
-
-static const struct eth_dev_ops mlx4_dev_ops = {
- .dev_configure = mlx4_dev_configure,
- .dev_start = mlx4_dev_start,
- .dev_stop = mlx4_dev_stop,
- .dev_set_link_down = mlx4_set_link_down,
- .dev_set_link_up = mlx4_set_link_up,
- .dev_close = mlx4_dev_close,
- .link_update = mlx4_link_update,
- .stats_get = mlx4_stats_get,
- .stats_reset = mlx4_stats_reset,
- .dev_infos_get = mlx4_dev_infos_get,
- .rx_queue_setup = mlx4_rx_queue_setup,
- .tx_queue_setup = mlx4_tx_queue_setup,
- .rx_queue_release = mlx4_rx_queue_release,
- .tx_queue_release = mlx4_tx_queue_release,
- .flow_ctrl_get = mlx4_dev_get_flow_ctrl,
- .flow_ctrl_set = mlx4_dev_set_flow_ctrl,
- .mtu_set = mlx4_dev_set_mtu,
- .filter_ctrl = mlx4_dev_filter_ctrl,
- .rx_queue_intr_enable = mlx4_rx_intr_enable,
- .rx_queue_intr_disable = mlx4_rx_intr_disable,
-};
-
-/**
- * Get PCI information from struct ibv_device.
- *
- * @param device
- * Pointer to Ethernet device structure.
- * @param[out] pci_addr
- * PCI bus address output buffer.
- *
- * @return
- * 0 on success, negative errno value otherwise and rte_errno is set.
- */
-static int
-mlx4_ibv_device_to_pci_addr(const struct ibv_device *device,
- struct rte_pci_addr *pci_addr)
-{
- FILE *file;
- char line[32];
- MKSTR(path, "%s/device/uevent", device->ibdev_path);
-
- file = fopen(path, "rb");
- if (file == NULL) {
- rte_errno = errno;
- return -rte_errno;
- }
- while (fgets(line, sizeof(line), file) == line) {
- size_t len = strlen(line);
- int ret;
-
- /* Truncate long lines. */
- if (len == (sizeof(line) - 1))
- while (line[(len - 1)] != '\n') {
- ret = fgetc(file);
- if (ret == EOF)
- break;
- line[(len - 1)] = ret;
- }
- /* Extract information. */
- if (sscanf(line,
- "PCI_SLOT_NAME="
- "%" SCNx32 ":%" SCNx8 ":%" SCNx8 ".%" SCNx8 "\n",
- &pci_addr->domain,
- &pci_addr->bus,
- &pci_addr->devid,
- &pci_addr->function) == 4) {
- ret = 0;
- break;
- }
- }
- fclose(file);
- return 0;
-}
-
-/**
- * Get MAC address by querying netdevice.
- *
- * @param[in] priv
- * struct priv for the requested device.
- * @param[out] mac
- * MAC address output buffer.
- *
- * @return
- * 0 on success, negative errno value otherwise and rte_errno is set.
- */
-static int
-priv_get_mac(struct priv *priv, uint8_t (*mac)[ETHER_ADDR_LEN])
-{
- struct ifreq request;
- int ret = priv_ifreq(priv, SIOCGIFHWADDR, &request);
-
- if (ret)
- return ret;
- memcpy(mac, request.ifr_hwaddr.sa_data, ETHER_ADDR_LEN);
- return 0;
-}
-
-static void
-mlx4_dev_link_status_handler(void *);
-static void
-mlx4_dev_interrupt_handler(void *);
-
-/**
- * Link/device status handler.
- *
- * @param priv
- * Pointer to private structure.
- * @param dev
- * Pointer to the rte_eth_dev structure.
- * @param events
- * Pointer to event flags holder.
- *
- * @return
- * Number of events
- */
-static int
-priv_dev_status_handler(struct priv *priv, struct rte_eth_dev *dev,
- uint32_t *events)
-{
- struct ibv_async_event event;
- int port_change = 0;
- struct rte_eth_link *link = &dev->data->dev_link;
- int ret = 0;
-
- *events = 0;
- /* Read all message and acknowledge them. */
- for (;;) {
- if (ibv_get_async_event(priv->ctx, &event))
- break;
- if ((event.event_type == IBV_EVENT_PORT_ACTIVE ||
- event.event_type == IBV_EVENT_PORT_ERR) &&
- (priv->intr_conf.lsc == 1)) {
- port_change = 1;
- ret++;
- } else if (event.event_type == IBV_EVENT_DEVICE_FATAL &&
- priv->intr_conf.rmv == 1) {
- *events |= (1 << RTE_ETH_EVENT_INTR_RMV);
- ret++;
- } else
- DEBUG("event type %d on port %d not handled",
- event.event_type, event.element.port_num);
- ibv_ack_async_event(&event);
- }
- if (!port_change)
- return ret;
- mlx4_link_update(dev, 0);
- if (((link->link_speed == 0) && link->link_status) ||
- ((link->link_speed != 0) && !link->link_status)) {
- if (!priv->pending_alarm) {
- /* Inconsistent status, check again later. */
- priv->pending_alarm = 1;
- rte_eal_alarm_set(MLX4_ALARM_TIMEOUT_US,
- mlx4_dev_link_status_handler,
- dev);
- }
- } else {
- *events |= (1 << RTE_ETH_EVENT_INTR_LSC);
- }
- return ret;
-}
-
-/**
- * Handle delayed link status event.
- *
- * @param arg
- * Registered argument.
- */
-static void
-mlx4_dev_link_status_handler(void *arg)
-{
- struct rte_eth_dev *dev = arg;
- struct priv *priv = dev->data->dev_private;
- uint32_t events;
- int ret;
-
- priv_lock(priv);
- assert(priv->pending_alarm == 1);
- priv->pending_alarm = 0;
- ret = priv_dev_status_handler(priv, dev, &events);
- priv_unlock(priv);
- if (ret > 0 && events & (1 << RTE_ETH_EVENT_INTR_LSC))
- _rte_eth_dev_callback_process(dev, RTE_ETH_EVENT_INTR_LSC,
- NULL, NULL);
-}
-
-/**
- * Handle interrupts from the NIC.
- *
- * @param[in] intr_handle
- * Interrupt handler.
- * @param cb_arg
- * Callback argument.
- */
-static void
-mlx4_dev_interrupt_handler(void *cb_arg)
-{
- struct rte_eth_dev *dev = cb_arg;
- struct priv *priv = dev->data->dev_private;
- int ret;
- uint32_t ev;
- int i;
-
- priv_lock(priv);
- ret = priv_dev_status_handler(priv, dev, &ev);
- priv_unlock(priv);
- if (ret > 0) {
- for (i = RTE_ETH_EVENT_UNKNOWN;
- i < RTE_ETH_EVENT_MAX;
- i++) {
- if (ev & (1 << i)) {
- ev &= ~(1 << i);
- _rte_eth_dev_callback_process(dev, i, NULL,
- NULL);
- ret--;
- }
- }
- if (ret)
- WARN("%d event%s not processed", ret,
- (ret > 1 ? "s were" : " was"));
- }
-}
-
-/**
- * Uninstall interrupt handler.
- *
- * @param priv
- * Pointer to private structure.
- * @param dev
- * Pointer to the rte_eth_dev structure.
- *
- * @return
- * 0 on success, negative errno value otherwise and rte_errno is set.
- */
-static int
-priv_dev_interrupt_handler_uninstall(struct priv *priv, struct rte_eth_dev *dev)
-{
- int ret;
-
- if (priv->intr_conf.lsc ||
- priv->intr_conf.rmv)
- return 0;
- ret = rte_intr_callback_unregister(&priv->intr_handle,
- mlx4_dev_interrupt_handler,
- dev);
- if (ret < 0) {
- rte_errno = ret;
- ERROR("rte_intr_callback_unregister failed with %d %s",
- ret, strerror(rte_errno));
- }
- priv->intr_handle.fd = 0;
- priv->intr_handle.type = RTE_INTR_HANDLE_UNKNOWN;
- return ret;
-}
-
-/**
- * Install interrupt handler.
- *
- * @param priv
- * Pointer to private structure.
- * @param dev
- * Pointer to the rte_eth_dev structure.
- *
- * @return
- * 0 on success, negative errno value otherwise and rte_errno is set.
- */
-static int
-priv_dev_interrupt_handler_install(struct priv *priv,
- struct rte_eth_dev *dev)
-{
- int flags;
- int rc;
-
- /*
- * Check whether the interrupt handler has already been installed
- * for either type of interrupt.
- */
- if (priv->intr_conf.lsc &&
- priv->intr_conf.rmv &&
- priv->intr_handle.fd)
- return 0;
- assert(priv->ctx->async_fd > 0);
- flags = fcntl(priv->ctx->async_fd, F_GETFL);
- rc = fcntl(priv->ctx->async_fd, F_SETFL, flags | O_NONBLOCK);
- if (rc < 0) {
- rte_errno = errno ? errno : EINVAL;
- INFO("failed to change file descriptor async event queue");
- dev->data->dev_conf.intr_conf.lsc = 0;
- dev->data->dev_conf.intr_conf.rmv = 0;
- return -rte_errno;
- } else {
- priv->intr_handle.fd = priv->ctx->async_fd;
- priv->intr_handle.type = RTE_INTR_HANDLE_EXT;
- rc = rte_intr_callback_register(&priv->intr_handle,
- mlx4_dev_interrupt_handler,
- dev);
- if (rc) {
- rte_errno = -rc;
- ERROR("rte_intr_callback_register failed "
- " (rte_errno: %s)", strerror(rte_errno));
- return -rte_errno;
- }
- }
- return 0;
-}
-
-/**
- * Uninstall interrupt handler.
- *
- * @param priv
- * Pointer to private structure.
- * @param dev
- * Pointer to the rte_eth_dev structure.
- *
- * @return
- * 0 on success, negative errno value otherwise and rte_errno is set.
- */
-static int
-priv_dev_removal_interrupt_handler_uninstall(struct priv *priv,
- struct rte_eth_dev *dev)
-{
- if (dev->data->dev_conf.intr_conf.rmv) {
- priv->intr_conf.rmv = 0;
- return priv_dev_interrupt_handler_uninstall(priv, dev);
- }
- return 0;
-}
-
-/**
- * Uninstall interrupt handler.
- *
- * @param priv
- * Pointer to private structure.
- * @param dev
- * Pointer to the rte_eth_dev structure.
- *
- * @return
- * 0 on success, negative errno value otherwise and rte_errno is set.
- */
-static int
-priv_dev_link_interrupt_handler_uninstall(struct priv *priv,
- struct rte_eth_dev *dev)
-{
- int ret = 0;
-
- if (dev->data->dev_conf.intr_conf.lsc) {
- priv->intr_conf.lsc = 0;
- ret = priv_dev_interrupt_handler_uninstall(priv, dev);
- if (ret)
- return ret;
- }
- if (priv->pending_alarm)
- if (rte_eal_alarm_cancel(mlx4_dev_link_status_handler,
- dev)) {
- ERROR("rte_eal_alarm_cancel failed "
- " (rte_errno: %s)", strerror(rte_errno));
- return -rte_errno;
- }
- priv->pending_alarm = 0;
- return 0;
-}
-
-/**
- * Install link interrupt handler.
- *
- * @param priv
- * Pointer to private structure.
- * @param dev
- * Pointer to the rte_eth_dev structure.
- *
- * @return
- * 0 on success, negative errno value otherwise and rte_errno is set.
- */
-static int
-priv_dev_link_interrupt_handler_install(struct priv *priv,
- struct rte_eth_dev *dev)
-{
- int ret;
-
- if (dev->data->dev_conf.intr_conf.lsc) {
- ret = priv_dev_interrupt_handler_install(priv, dev);
- if (ret)
- return ret;
- priv->intr_conf.lsc = 1;
- }
- return 0;
-}
-
-/**
- * Install removal interrupt handler.
- *
- * @param priv
- * Pointer to private structure.
- * @param dev
- * Pointer to the rte_eth_dev structure.
- *
- * @return
- * 0 on success, negative errno value otherwise and rte_errno is set.
- */
-static int
-priv_dev_removal_interrupt_handler_install(struct priv *priv,
- struct rte_eth_dev *dev)
-{
- int ret;
-
- if (dev->data->dev_conf.intr_conf.rmv) {
- ret = priv_dev_interrupt_handler_install(priv, dev);
- if (ret)
- return ret;
- priv->intr_conf.rmv = 1;