#include <netinet/in.h>
#include <linux/ethtool.h>
#include <linux/sockios.h>
-#include <fcntl.h>
#include <rte_ether.h>
#include <rte_ethdev.h>
#include <rte_mempool.h>
#include <rte_prefetch.h>
#include <rte_malloc.h>
-#include <rte_alarm.h>
#include <rte_memory.h>
#include <rte_flow.h>
#include <rte_kvargs.h>
/* PMD headers. */
#include "mlx4.h"
#include "mlx4_flow.h"
+#include "mlx4_rxtx.h"
#include "mlx4_utils.h"
/** Configuration structure for device arguments. */
NULL,
};
-static int
-mlx4_rx_intr_enable(struct rte_eth_dev *dev, uint16_t idx);
-
-static int
-mlx4_rx_intr_disable(struct rte_eth_dev *dev, uint16_t idx);
-
-static int
-priv_rx_intr_vec_enable(struct priv *priv);
-
-static void
-priv_rx_intr_vec_disable(struct priv *priv);
-
/* Allocate a buffer on the stack and fill it with a printf format string. */
#define MKSTR(name, ...) \
char name[snprintf(NULL, 0, __VA_ARGS__) + 1]; \
(void *)dev, strerror(rte_errno));
goto error;
}
+ if (mlx4_fd_set_non_blocking(tmpl.channel->fd) < 0) {
+ ERROR("%p: unable to make Rx interrupt completion"
+ " channel non-blocking: %s",
+ (void *)dev, strerror(rte_errno));
+ goto error;
+ }
}
tmpl.cq = ibv_create_cq(priv->ctx, desc, NULL, tmpl.channel, 0);
if (tmpl.cq == NULL) {
rte_free(rxq);
}
-static int
-priv_dev_interrupt_handler_install(struct priv *, struct rte_eth_dev *);
-
-static int
-priv_dev_removal_interrupt_handler_install(struct priv *, struct rte_eth_dev *);
-
-static int
-priv_dev_link_interrupt_handler_install(struct priv *, struct rte_eth_dev *);
-
/**
* DPDK callback to start the device.
*
ret = priv_mac_addr_add(priv);
if (ret)
goto err;
- ret = priv_dev_link_interrupt_handler_install(priv, dev);
+ ret = mlx4_intr_install(priv);
if (ret) {
- ERROR("%p: LSC handler install failed",
+ ERROR("%p: interrupt handler installation failed",
(void *)dev);
goto err;
}
- ret = priv_dev_removal_interrupt_handler_install(priv, dev);
- if (ret) {
- ERROR("%p: RMV handler install failed",
- (void *)dev);
- goto err;
- }
- ret = priv_rx_intr_vec_enable(priv);
- if (ret) {
- ERROR("%p: Rx interrupt vector creation failed",
- (void *)dev);
- goto err;
- }
ret = mlx4_priv_flow_start(priv);
if (ret) {
ERROR("%p: flow start failed: %s",
DEBUG("%p: detaching flows from all RX queues", (void *)dev);
priv->started = 0;
mlx4_priv_flow_stop(priv);
+ mlx4_intr_uninstall(priv);
priv_mac_addr_del(priv);
}
return 0;
}
-static int
-priv_dev_interrupt_handler_uninstall(struct priv *, struct rte_eth_dev *);
-
-static int
-priv_dev_removal_interrupt_handler_uninstall(struct priv *,
- struct rte_eth_dev *);
-
-static int
-priv_dev_link_interrupt_handler_uninstall(struct priv *, struct rte_eth_dev *);
-
/**
* DPDK callback to close the device.
*
claim_zero(ibv_close_device(priv->ctx));
} else
assert(priv->ctx == NULL);
- priv_dev_removal_interrupt_handler_uninstall(priv, dev);
- priv_dev_link_interrupt_handler_uninstall(priv, dev);
- priv_rx_intr_vec_disable(priv);
+ mlx4_intr_uninstall(priv);
memset(priv, 0, sizeof(*priv));
}
* @return
* 0 on success, negative errno value otherwise and rte_errno is set.
*/
-static int
+int
mlx4_link_update(struct rte_eth_dev *dev, int wait_to_complete)
{
const struct priv *priv = dev->data->dev_private;
return 0;
}
-static void
-mlx4_dev_link_status_handler(void *);
-static void
-mlx4_dev_interrupt_handler(void *);
-
-/**
- * Link/device status handler.
- *
- * @param priv
- * Pointer to private structure.
- * @param dev
- * Pointer to the rte_eth_dev structure.
- * @param events
- * Pointer to event flags holder.
- *
- * @return
- * Number of events
- */
-static int
-priv_dev_status_handler(struct priv *priv, struct rte_eth_dev *dev,
- uint32_t *events)
-{
- struct ibv_async_event event;
- int port_change = 0;
- struct rte_eth_link *link = &dev->data->dev_link;
- int ret = 0;
-
- *events = 0;
- /* Read all message and acknowledge them. */
- for (;;) {
- if (ibv_get_async_event(priv->ctx, &event))
- break;
- if ((event.event_type == IBV_EVENT_PORT_ACTIVE ||
- event.event_type == IBV_EVENT_PORT_ERR) &&
- (priv->intr_conf.lsc == 1)) {
- port_change = 1;
- ret++;
- } else if (event.event_type == IBV_EVENT_DEVICE_FATAL &&
- priv->intr_conf.rmv == 1) {
- *events |= (1 << RTE_ETH_EVENT_INTR_RMV);
- ret++;
- } else
- DEBUG("event type %d on port %d not handled",
- event.event_type, event.element.port_num);
- ibv_ack_async_event(&event);
- }
- if (!port_change)
- return ret;
- mlx4_link_update(dev, 0);
- if (((link->link_speed == 0) && link->link_status) ||
- ((link->link_speed != 0) && !link->link_status)) {
- if (!priv->pending_alarm) {
- /* Inconsistent status, check again later. */
- priv->pending_alarm = 1;
- rte_eal_alarm_set(MLX4_ALARM_TIMEOUT_US,
- mlx4_dev_link_status_handler,
- dev);
- }
- } else {
- *events |= (1 << RTE_ETH_EVENT_INTR_LSC);
- }
- return ret;
-}
-
-/**
- * Handle delayed link status event.
- *
- * @param arg
- * Registered argument.
- */
-static void
-mlx4_dev_link_status_handler(void *arg)
-{
- struct rte_eth_dev *dev = arg;
- struct priv *priv = dev->data->dev_private;
- uint32_t events;
- int ret;
-
- assert(priv->pending_alarm == 1);
- priv->pending_alarm = 0;
- ret = priv_dev_status_handler(priv, dev, &events);
- if (ret > 0 && events & (1 << RTE_ETH_EVENT_INTR_LSC))
- _rte_eth_dev_callback_process(dev, RTE_ETH_EVENT_INTR_LSC,
- NULL, NULL);
-}
-
-/**
- * Handle interrupts from the NIC.
- *
- * @param[in] intr_handle
- * Interrupt handler.
- * @param cb_arg
- * Callback argument.
- */
-static void
-mlx4_dev_interrupt_handler(void *cb_arg)
-{
- struct rte_eth_dev *dev = cb_arg;
- struct priv *priv = dev->data->dev_private;
- int ret;
- uint32_t ev;
- int i;
-
- ret = priv_dev_status_handler(priv, dev, &ev);
- if (ret > 0) {
- for (i = RTE_ETH_EVENT_UNKNOWN;
- i < RTE_ETH_EVENT_MAX;
- i++) {
- if (ev & (1 << i)) {
- ev &= ~(1 << i);
- _rte_eth_dev_callback_process(dev, i, NULL,
- NULL);
- ret--;
- }
- }
- if (ret)
- WARN("%d event%s not processed", ret,
- (ret > 1 ? "s were" : " was"));
- }
-}
-
-/**
- * Uninstall interrupt handler.
- *
- * @param priv
- * Pointer to private structure.
- * @param dev
- * Pointer to the rte_eth_dev structure.
- *
- * @return
- * 0 on success, negative errno value otherwise and rte_errno is set.
- */
-static int
-priv_dev_interrupt_handler_uninstall(struct priv *priv, struct rte_eth_dev *dev)
-{
- int ret;
-
- if (priv->intr_conf.lsc ||
- priv->intr_conf.rmv)
- return 0;
- ret = rte_intr_callback_unregister(&priv->intr_handle,
- mlx4_dev_interrupt_handler,
- dev);
- if (ret < 0) {
- rte_errno = ret;
- ERROR("rte_intr_callback_unregister failed with %d %s",
- ret, strerror(rte_errno));
- }
- priv->intr_handle.fd = -1;
- return ret;
-}
-
-/**
- * Install interrupt handler.
- *
- * @param priv
- * Pointer to private structure.
- * @param dev
- * Pointer to the rte_eth_dev structure.
- *
- * @return
- * 0 on success, negative errno value otherwise and rte_errno is set.
- */
-static int
-priv_dev_interrupt_handler_install(struct priv *priv,
- struct rte_eth_dev *dev)
-{
- int flags;
- int rc;
-
- /*
- * Check whether the interrupt handler has already been installed
- * for either type of interrupt.
- */
- if (priv->intr_conf.lsc &&
- priv->intr_conf.rmv &&
- priv->intr_handle.fd)
- return 0;
- assert(priv->ctx->async_fd > 0);
- flags = fcntl(priv->ctx->async_fd, F_GETFL);
- rc = fcntl(priv->ctx->async_fd, F_SETFL, flags | O_NONBLOCK);
- if (rc < 0) {
- rte_errno = errno ? errno : EINVAL;
- INFO("failed to change file descriptor async event queue");
- dev->data->dev_conf.intr_conf.lsc = 0;
- dev->data->dev_conf.intr_conf.rmv = 0;
- return -rte_errno;
- } else {
- priv->intr_handle.fd = priv->ctx->async_fd;
- rc = rte_intr_callback_register(&priv->intr_handle,
- mlx4_dev_interrupt_handler,
- dev);
- if (rc) {
- rte_errno = -rc;
- ERROR("rte_intr_callback_register failed "
- " (rte_errno: %s)", strerror(rte_errno));
- priv->intr_handle.fd = -1;
- return -rte_errno;
- }
- }
- return 0;
-}
-
-/**
- * Uninstall interrupt handler.
- *
- * @param priv
- * Pointer to private structure.
- * @param dev
- * Pointer to the rte_eth_dev structure.
- *
- * @return
- * 0 on success, negative errno value otherwise and rte_errno is set.
- */
-static int
-priv_dev_removal_interrupt_handler_uninstall(struct priv *priv,
- struct rte_eth_dev *dev)
-{
- if (dev->data->dev_conf.intr_conf.rmv) {
- priv->intr_conf.rmv = 0;
- return priv_dev_interrupt_handler_uninstall(priv, dev);
- }
- return 0;
-}
-
-/**
- * Uninstall interrupt handler.
- *
- * @param priv
- * Pointer to private structure.
- * @param dev
- * Pointer to the rte_eth_dev structure.
- *
- * @return
- * 0 on success, negative errno value otherwise and rte_errno is set.
- */
-static int
-priv_dev_link_interrupt_handler_uninstall(struct priv *priv,
- struct rte_eth_dev *dev)
-{
- int ret = 0;
-
- if (dev->data->dev_conf.intr_conf.lsc) {
- priv->intr_conf.lsc = 0;
- ret = priv_dev_interrupt_handler_uninstall(priv, dev);
- if (ret)
- return ret;
- }
- if (priv->pending_alarm)
- if (rte_eal_alarm_cancel(mlx4_dev_link_status_handler,
- dev)) {
- ERROR("rte_eal_alarm_cancel failed "
- " (rte_errno: %s)", strerror(rte_errno));
- return -rte_errno;
- }
- priv->pending_alarm = 0;
- return 0;
-}
-
-/**
- * Install link interrupt handler.
- *
- * @param priv
- * Pointer to private structure.
- * @param dev
- * Pointer to the rte_eth_dev structure.
- *
- * @return
- * 0 on success, negative errno value otherwise and rte_errno is set.
- */
-static int
-priv_dev_link_interrupt_handler_install(struct priv *priv,
- struct rte_eth_dev *dev)
-{
- int ret;
-
- if (dev->data->dev_conf.intr_conf.lsc) {
- ret = priv_dev_interrupt_handler_install(priv, dev);
- if (ret)
- return ret;
- priv->intr_conf.lsc = 1;
- }
- return 0;
-}
-
-/**
- * Install removal interrupt handler.
- *
- * @param priv
- * Pointer to private structure.
- * @param dev
- * Pointer to the rte_eth_dev structure.
- *
- * @return
- * 0 on success, negative errno value otherwise and rte_errno is set.
- */
-static int
-priv_dev_removal_interrupt_handler_install(struct priv *priv,
- struct rte_eth_dev *dev)
-{
- int ret;
-
- if (dev->data->dev_conf.intr_conf.rmv) {
- ret = priv_dev_interrupt_handler_install(priv, dev);
- if (ret)
- return ret;
- priv->intr_conf.rmv = 1;
- }
- return 0;
-}
-
-/**
- * Allocate queue vector and fill epoll fd list for Rx interrupts.
- *
- * @param priv
- * Pointer to private structure.
- *
- * @return
- * 0 on success, negative errno value otherwise and rte_errno is set.
- */
-static int
-priv_rx_intr_vec_enable(struct priv *priv)
-{
- unsigned int i;
- unsigned int rxqs_n = priv->rxqs_n;
- unsigned int n = RTE_MIN(rxqs_n, (uint32_t)RTE_MAX_RXTX_INTR_VEC_ID);
- unsigned int count = 0;
- struct rte_intr_handle *intr_handle = &priv->intr_handle;
-
- if (!priv->dev->data->dev_conf.intr_conf.rxq)
- return 0;
- priv_rx_intr_vec_disable(priv);
- intr_handle->intr_vec = malloc(sizeof(intr_handle->intr_vec[rxqs_n]));
- if (intr_handle->intr_vec == NULL) {
- rte_errno = ENOMEM;
- ERROR("failed to allocate memory for interrupt vector,"
- " Rx interrupts will not be supported");
- return -rte_errno;
- }
- for (i = 0; i != n; ++i) {
- struct rxq *rxq = (*priv->rxqs)[i];
- int fd;
- int flags;
- int rc;
-
- /* Skip queues that cannot request interrupts. */
- if (!rxq || !rxq->channel) {
- /* Use invalid intr_vec[] index to disable entry. */
- intr_handle->intr_vec[i] =
- RTE_INTR_VEC_RXTX_OFFSET +
- RTE_MAX_RXTX_INTR_VEC_ID;
- continue;
- }
- if (count >= RTE_MAX_RXTX_INTR_VEC_ID) {
- rte_errno = E2BIG;
- ERROR("too many Rx queues for interrupt vector size"
- " (%d), Rx interrupts cannot be enabled",
- RTE_MAX_RXTX_INTR_VEC_ID);
- priv_rx_intr_vec_disable(priv);
- return -rte_errno;
- }
- fd = rxq->channel->fd;
- flags = fcntl(fd, F_GETFL);
- rc = fcntl(fd, F_SETFL, flags | O_NONBLOCK);
- if (rc < 0) {
- rte_errno = errno;
- ERROR("failed to make Rx interrupt file descriptor"
- " %d non-blocking for queue index %d", fd, i);
- priv_rx_intr_vec_disable(priv);
- return -rte_errno;
- }
- intr_handle->intr_vec[i] = RTE_INTR_VEC_RXTX_OFFSET + count;
- intr_handle->efds[count] = fd;
- count++;
- }
- if (!count)
- priv_rx_intr_vec_disable(priv);
- else
- intr_handle->nb_efd = count;
- return 0;
-}
-
-/**
- * Clean up Rx interrupts handler.
- *
- * @param priv
- * Pointer to private structure.
- */
-static void
-priv_rx_intr_vec_disable(struct priv *priv)
-{
- struct rte_intr_handle *intr_handle = &priv->intr_handle;
-
- rte_intr_free_epoll_fd(intr_handle);
- free(intr_handle->intr_vec);
- intr_handle->nb_efd = 0;
- intr_handle->intr_vec = NULL;
-}
-
-/**
- * DPDK callback for Rx queue interrupt enable.
- *
- * @param dev
- * Pointer to Ethernet device structure.
- * @param idx
- * Rx queue index.
- *
- * @return
- * 0 on success, negative errno value otherwise and rte_errno is set.
- */
-static int
-mlx4_rx_intr_enable(struct rte_eth_dev *dev, uint16_t idx)
-{
- struct priv *priv = dev->data->dev_private;
- struct rxq *rxq = (*priv->rxqs)[idx];
- int ret;
-
- if (!rxq || !rxq->channel)
- ret = EINVAL;
- else
- ret = ibv_req_notify_cq(rxq->cq, 0);
- if (ret) {
- rte_errno = ret;
- WARN("unable to arm interrupt on rx queue %d", idx);
- }
- return -ret;
-}
-
-/**
- * DPDK callback for Rx queue interrupt disable.
- *
- * @param dev
- * Pointer to Ethernet device structure.
- * @param idx
- * Rx queue index.
- *
- * @return
- * 0 on success, negative errno value otherwise and rte_errno is set.
- */
-static int
-mlx4_rx_intr_disable(struct rte_eth_dev *dev, uint16_t idx)
-{
- struct priv *priv = dev->data->dev_private;
- struct rxq *rxq = (*priv->rxqs)[idx];
- struct ibv_cq *ev_cq;
- void *ev_ctx;
- int ret;
-
- if (!rxq || !rxq->channel) {
- ret = EINVAL;
- } else {
- ret = ibv_get_cq_event(rxq->cq->channel, &ev_cq, &ev_ctx);
- if (ret || ev_cq != rxq->cq)
- ret = EINVAL;
- }
- if (ret) {
- rte_errno = ret;
- WARN("unable to disable interrupt on rx queue %d",
- idx);
- } else {
- ibv_ack_cq_events(rxq->cq, 1);
- }
- return -ret;
-}
-
/**
* Verify and store value for device argument.
*
DEBUG("port %d is not active: \"%s\" (%d)",
port, ibv_port_state_str(port_attr.state),
port_attr.state);
+ /* Make asynchronous FD non-blocking to handle interrupts. */
+ if (mlx4_fd_set_non_blocking(ctx->async_fd) < 0) {
+ ERROR("cannot make asynchronous FD non-blocking: %s",
+ strerror(rte_errno));
+ goto port_error;
+ }
/* Allocate protection domain. */
pd = ibv_alloc_pd(ctx);
if (pd == NULL) {
git.droids-corp.org / dpdk.git / blobdiff