X-Git-Url: http://git.droids-corp.org/?a=blobdiff_plain;f=drivers%2Fnet%2Fmlx4%2Fmlx4_intr.c;h=01057482ec750e57417bae65340ee23409121292;hb=8b433d04adc9c62040d229d3eebfa2bdc2ced620;hp=d7f10989d27601d307f1a7e55fbb4c31e9c0b7b3;hpb=ebada48456bdaeeec556567ffab9bec5642b1304;p=dpdk.git

diff --git a/drivers/net/mlx4/mlx4_intr.c b/drivers/net/mlx4/mlx4_intr.c
index d7f10989d2..01057482ec 100644
--- a/drivers/net/mlx4/mlx4_intr.c
+++ b/drivers/net/mlx4/mlx4_intr.c
@@ -1,34 +1,6 @@
-/*-
- *   BSD LICENSE
- *
- *   Copyright 2017 6WIND S.A.
- *   Copyright 2017 Mellanox
- *
- *   Redistribution and use in source and binary forms, with or without
- *   modification, are permitted provided that the following conditions
- *   are met:
- *
- *     * Redistributions of source code must retain the above copyright
- *       notice, this list of conditions and the following disclaimer.
- *     * Redistributions in binary form must reproduce the above copyright
- *       notice, this list of conditions and the following disclaimer in
- *       the documentation and/or other materials provided with the
- *       distribution.
- *     * Neither the name of 6WIND S.A. nor the names of its
- *       contributors may be used to endorse or promote products derived
- *       from this software without specific prior written permission.
- *
- *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
- *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
- *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
- *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
- *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
- *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
- *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
- *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
- *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
- *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
- *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright 2017 6WIND S.A.
+ * Copyright 2017 Mellanox Technologies, Ltd
  */
 
 /**
@@ -36,7 +8,6 @@
  * Interrupts handling for mlx4 driver.
  */
 
-#include <assert.h>
 #include <errno.h>
 #include <stdint.h>
 #include <stdlib.h>
@@ -52,14 +23,16 @@
 
 #include <rte_alarm.h>
 #include <rte_errno.h>
-#include <rte_ethdev.h>
+#include <ethdev_driver.h>
+#include <rte_io.h>
 #include <rte_interrupts.h>
 
 #include "mlx4.h"
+#include "mlx4_glue.h"
 #include "mlx4_rxtx.h"
 #include "mlx4_utils.h"
 
-static void mlx4_link_status_alarm(struct priv *priv);
+static int mlx4_link_status_check(struct mlx4_priv *priv);
 
 /**
  * Clean up Rx interrupts handler.
@@ -68,14 +41,14 @@ static void mlx4_link_status_alarm(struct priv *priv);
  *   Pointer to private structure.
  */
 static void
-mlx4_rx_intr_vec_disable(struct priv *priv)
+mlx4_rx_intr_vec_disable(struct mlx4_priv *priv)
 {
-	struct rte_intr_handle *intr_handle = &priv->intr_handle;
+	struct rte_intr_handle *intr_handle = priv->intr_handle;
 
 	rte_intr_free_epoll_fd(intr_handle);
-	free(intr_handle->intr_vec);
-	intr_handle->nb_efd = 0;
-	intr_handle->intr_vec = NULL;
+	rte_intr_vec_list_free(intr_handle);
+
+	rte_intr_nb_efd_set(intr_handle, 0);
 }
 
 /**
@@ -88,31 +61,30 @@ mlx4_rx_intr_vec_disable(struct priv *priv)
  *   0 on success, negative errno value otherwise and rte_errno is set.
  */
 static int
-mlx4_rx_intr_vec_enable(struct priv *priv)
+mlx4_rx_intr_vec_enable(struct mlx4_priv *priv)
 {
 	unsigned int i;
-	unsigned int rxqs_n = priv->dev->data->nb_rx_queues;
+	unsigned int rxqs_n = ETH_DEV(priv)->data->nb_rx_queues;
 	unsigned int n = RTE_MIN(rxqs_n, (uint32_t)RTE_MAX_RXTX_INTR_VEC_ID);
 	unsigned int count = 0;
-	struct rte_intr_handle *intr_handle = &priv->intr_handle;
+	struct rte_intr_handle *intr_handle = priv->intr_handle;
 
 	mlx4_rx_intr_vec_disable(priv);
-	intr_handle->intr_vec = malloc(sizeof(intr_handle->intr_vec[rxqs_n]));
-	if (intr_handle->intr_vec == NULL) {
+	if (rte_intr_vec_list_alloc(intr_handle, NULL, n)) {
 		rte_errno = ENOMEM;
 		ERROR("failed to allocate memory for interrupt vector,"
 		      " Rx interrupts will not be supported");
 		return -rte_errno;
 	}
 	for (i = 0; i != n; ++i) {
-		struct rxq *rxq = priv->dev->data->rx_queues[i];
+		struct rxq *rxq = ETH_DEV(priv)->data->rx_queues[i];
 
 		/* Skip queues that cannot request interrupts. */
 		if (!rxq || !rxq->channel) {
 			/* Use invalid intr_vec[] index to disable entry. */
-			intr_handle->intr_vec[i] =
-				RTE_INTR_VEC_RXTX_OFFSET +
-				RTE_MAX_RXTX_INTR_VEC_ID;
+			if (rte_intr_vec_list_index_set(intr_handle, i,
+			RTE_INTR_VEC_RXTX_OFFSET + RTE_MAX_RXTX_INTR_VEC_ID))
+				return -rte_errno;
 			continue;
 		}
 		if (count >= RTE_MAX_RXTX_INTR_VEC_ID) {
@@ -123,132 +95,155 @@ mlx4_rx_intr_vec_enable(struct priv *priv)
 			mlx4_rx_intr_vec_disable(priv);
 			return -rte_errno;
 		}
-		intr_handle->intr_vec[i] = RTE_INTR_VEC_RXTX_OFFSET + count;
-		intr_handle->efds[count] = rxq->channel->fd;
+
+		if (rte_intr_vec_list_index_set(intr_handle, i,
+					RTE_INTR_VEC_RXTX_OFFSET + count))
+			return -rte_errno;
+
+		if (rte_intr_efds_index_set(intr_handle, i,
+						   rxq->channel->fd))
+			return -rte_errno;
+
 		count++;
 	}
 	if (!count)
 		mlx4_rx_intr_vec_disable(priv);
-	else
-		intr_handle->nb_efd = count;
+	else if (rte_intr_nb_efd_set(intr_handle, count))
+		return -rte_errno;
 	return 0;
 }
 
 /**
- * Collect interrupt events.
+ * Process scheduled link status check.
+ *
+ * If LSC interrupts are requested, process related callback.
+ *
+ * @param priv
+ *   Pointer to private structure.
+ */
+static void
+mlx4_link_status_alarm(struct mlx4_priv *priv)
+{
+	const struct rte_eth_intr_conf *const intr_conf =
+		&ETH_DEV(priv)->data->dev_conf.intr_conf;
+
+	MLX4_ASSERT(priv->intr_alarm == 1);
+	priv->intr_alarm = 0;
+	if (intr_conf->lsc && !mlx4_link_status_check(priv))
+		rte_eth_dev_callback_process(ETH_DEV(priv),
+					     RTE_ETH_EVENT_INTR_LSC,
+					     NULL);
+}
+
+/**
+ * Check link status.
+ *
+ * In case of inconsistency, another check is scheduled.
  *
  * @param priv
  *   Pointer to private structure.
- * @param events
- *   Pointer to event flags holder.
  *
  * @return
- *   Number of events.
+ *   0 on success (link status is consistent), negative errno value
+ *   otherwise and rte_errno is set.
  */
 static int
-mlx4_collect_interrupt_events(struct priv *priv, uint32_t *events)
+mlx4_link_status_check(struct mlx4_priv *priv)
 {
-	struct ibv_async_event event;
-	int port_change = 0;
-	struct rte_eth_link *link = &priv->dev->data->dev_link;
-	const struct rte_intr_conf *const intr_conf =
-		&priv->dev->data->dev_conf.intr_conf;
-	int ret = 0;
+	struct rte_eth_link *link = &ETH_DEV(priv)->data->dev_link;
+	int ret = mlx4_link_update(ETH_DEV(priv), 0);
 
-	*events = 0;
-	/* Read all message and acknowledge them. */
-	for (;;) {
-		if (ibv_get_async_event(priv->ctx, &event))
-			break;
-		switch (event.event_type) {
-		case IBV_EVENT_PORT_ACTIVE:
-		case IBV_EVENT_PORT_ERR:
-			if (!intr_conf->lsc)
-				break;
-			port_change = 1;
-			ret++;
-			break;
-		case IBV_EVENT_DEVICE_FATAL:
-			if (!intr_conf->rmv)
-				break;
-			*events |= (1 << RTE_ETH_EVENT_INTR_RMV);
-			ret++;
-			break;
-		default:
-			DEBUG("event type %d on port %d not handled",
-			      event.event_type, event.element.port_num);
-		}
-		ibv_ack_async_event(&event);
-	}
-	if (!port_change)
+	if (ret)
 		return ret;
-	mlx4_link_update(priv->dev, 0);
-	if (((link->link_speed == 0) && link->link_status) ||
-	    ((link->link_speed != 0) && !link->link_status)) {
+	if ((!link->link_speed && link->link_status) ||
+	    (link->link_speed && !link->link_status)) {
 		if (!priv->intr_alarm) {
 			/* Inconsistent status, check again later. */
+			ret = rte_eal_alarm_set(MLX4_INTR_ALARM_TIMEOUT,
+						(void (*)(void *))
+						mlx4_link_status_alarm,
+						priv);
+			if (ret)
+				return ret;
 			priv->intr_alarm = 1;
-			rte_eal_alarm_set(MLX4_INTR_ALARM_TIMEOUT,
-					  (void (*)(void *))
-					  mlx4_link_status_alarm,
-					  priv);
 		}
-	} else {
-		*events |= (1 << RTE_ETH_EVENT_INTR_LSC);
+		rte_errno = EINPROGRESS;
+		return -rte_errno;
 	}
-	return ret;
+	return 0;
 }
 
 /**
- * Process scheduled link status check.
+ * Handle interrupts from the NIC.
  *
  * @param priv
  *   Pointer to private structure.
  */
 static void
-mlx4_link_status_alarm(struct priv *priv)
+mlx4_interrupt_handler(struct mlx4_priv *priv)
 {
-	uint32_t events;
-	int ret;
+	enum { LSC, RMV, };
+	static const enum rte_eth_event_type type[] = {
+		[LSC] = RTE_ETH_EVENT_INTR_LSC,
+		[RMV] = RTE_ETH_EVENT_INTR_RMV,
+	};
+	uint32_t caught[RTE_DIM(type)] = { 0 };
+	struct ibv_async_event event;
+	const struct rte_eth_intr_conf *const intr_conf =
+		&ETH_DEV(priv)->data->dev_conf.intr_conf;
+	unsigned int i;
 
-	assert(priv->intr_alarm == 1);
-	priv->intr_alarm = 0;
-	ret = mlx4_collect_interrupt_events(priv, &events);
-	if (ret > 0 && events & (1 << RTE_ETH_EVENT_INTR_LSC))
-		_rte_eth_dev_callback_process(priv->dev,
-					      RTE_ETH_EVENT_INTR_LSC,
-					      NULL, NULL);
+	/* Read all message and acknowledge them. */
+	while (!mlx4_glue->get_async_event(priv->ctx, &event)) {
+		switch (event.event_type) {
+		case IBV_EVENT_PORT_ACTIVE:
+		case IBV_EVENT_PORT_ERR:
+			if (intr_conf->lsc && !mlx4_link_status_check(priv))
+				++caught[LSC];
+			break;
+		case IBV_EVENT_DEVICE_FATAL:
+			if (intr_conf->rmv)
+				++caught[RMV];
+			break;
+		default:
+			DEBUG("event type %d on physical port %d not handled",
+			      event.event_type, event.element.port_num);
+		}
+		mlx4_glue->ack_async_event(&event);
+	}
+	for (i = 0; i != RTE_DIM(caught); ++i)
+		if (caught[i])
+			rte_eth_dev_callback_process(ETH_DEV(priv), type[i],
+						     NULL);
 }
 
 /**
- * Handle interrupts from the NIC.
+ * MLX4 CQ notification .
  *
- * @param priv
- *   Pointer to private structure.
+ * @param rxq
+ *   Pointer to receive queue structure.
+ * @param solicited
+ *   Is request solicited or not.
  */
 static void
-mlx4_interrupt_handler(struct priv *priv)
+mlx4_arm_cq(struct rxq *rxq, int solicited)
 {
-	int ret;
-	uint32_t ev;
-	int i;
-
-	ret = mlx4_collect_interrupt_events(priv, &ev);
-	if (ret > 0) {
-		for (i = RTE_ETH_EVENT_UNKNOWN;
-		     i < RTE_ETH_EVENT_MAX;
-		     i++) {
-			if (ev & (1 << i)) {
-				ev &= ~(1 << i);
-				_rte_eth_dev_callback_process(priv->dev, i,
-							      NULL, NULL);
-				ret--;
-			}
-		}
-		if (ret)
-			WARN("%d event%s not processed", ret,
-			     (ret > 1 ? "s were" : " was"));
-	}
+	struct mlx4_cq *cq = &rxq->mcq;
+	uint64_t doorbell;
+	uint32_t sn = cq->arm_sn & MLX4_CQ_DB_GEQ_N_MASK;
+	uint32_t ci = cq->cons_index & MLX4_CQ_DB_CI_MASK;
+	uint32_t cmd = solicited ? MLX4_CQ_DB_REQ_NOT_SOL : MLX4_CQ_DB_REQ_NOT;
+
+	*cq->arm_db = rte_cpu_to_be_32(sn << 28 | cmd | ci);
+	/*
+	 * Make sure that the doorbell record in host memory is
+	 * written before ringing the doorbell via PCI MMIO.
+	 */
+	rte_wmb();
+	doorbell = sn << 28 | cmd | cq->cqn;
+	doorbell <<= 32;
+	doorbell |= ci;
+	rte_write64(rte_cpu_to_be_64(doorbell), cq->cq_db_reg);
 }
 
 /**
@@ -261,20 +256,21 @@ mlx4_interrupt_handler(struct priv *priv)
  *   0 on success, negative errno value otherwise and rte_errno is set.
  */
 int
-mlx4_intr_uninstall(struct priv *priv)
+mlx4_intr_uninstall(struct mlx4_priv *priv)
 {
 	int err = rte_errno; /* Make sure rte_errno remains unchanged. */
 
-	if (priv->intr_handle.fd != -1) {
-		rte_intr_callback_unregister(&priv->intr_handle,
+	if (rte_intr_fd_get(priv->intr_handle) != -1) {
+		rte_intr_callback_unregister(priv->intr_handle,
 					     (void (*)(void *))
 					     mlx4_interrupt_handler,
 					     priv);
-		priv->intr_handle.fd = -1;
+		if (rte_intr_fd_set(priv->intr_handle, -1))
+			return -rte_errno;
 	}
 	rte_eal_alarm_cancel((void (*)(void *))mlx4_link_status_alarm, priv);
 	priv->intr_alarm = 0;
-	mlx4_rx_intr_vec_disable(priv);
+	mlx4_rxq_intr_disable(priv);
 	rte_errno = err;
 	return 0;
 }
@@ -289,18 +285,18 @@ mlx4_intr_uninstall(struct priv *priv)
  *   0 on success, negative errno value otherwise and rte_errno is set.
  */
 int
-mlx4_intr_install(struct priv *priv)
+mlx4_intr_install(struct mlx4_priv *priv)
 {
-	const struct rte_intr_conf *const intr_conf =
-		&priv->dev->data->dev_conf.intr_conf;
+	const struct rte_eth_intr_conf *const intr_conf =
+		&ETH_DEV(priv)->data->dev_conf.intr_conf;
 	int rc;
 
 	mlx4_intr_uninstall(priv);
-	if (intr_conf->rxq && mlx4_rx_intr_vec_enable(priv) < 0)
-		goto error;
 	if (intr_conf->lsc | intr_conf->rmv) {
-		priv->intr_handle.fd = priv->ctx->async_fd;
-		rc = rte_intr_callback_register(&priv->intr_handle,
+		if (rte_intr_fd_set(priv->intr_handle, priv->ctx->async_fd))
+			return -rte_errno;
+
+		rc = rte_intr_callback_register(priv->intr_handle,
 						(void (*)(void *))
 						mlx4_interrupt_handler,
 						priv);
@@ -337,16 +333,25 @@ mlx4_rx_intr_disable(struct rte_eth_dev *dev, uint16_t idx)
 	if (!rxq || !rxq->channel) {
 		ret = EINVAL;
 	} else {
-		ret = ibv_get_cq_event(rxq->cq->channel, &ev_cq, &ev_ctx);
-		if (ret || ev_cq != rxq->cq)
+		ret = mlx4_glue->get_cq_event(rxq->cq->channel, &ev_cq,
+					      &ev_ctx);
+		/** For non-zero ret save the errno (may be EAGAIN
+		 * which means the get_cq_event function was called before
+		 * receiving one).
+		 */
+		if (ret)
+			ret = errno;
+		else if (ev_cq != rxq->cq)
 			ret = EINVAL;
 	}
 	if (ret) {
 		rte_errno = ret;
-		WARN("unable to disable interrupt on rx queue %d",
-		     idx);
+		if (ret != EAGAIN)
+			WARN("unable to disable interrupt on rx queue %d",
+			     idx);
 	} else {
-		ibv_ack_cq_events(rxq->cq, 1);
+		rxq->mcq.arm_sn++;
+		mlx4_glue->ack_cq_events(rxq->cq, 1);
 	}
 	return -ret;
 }
@@ -366,15 +371,51 @@ int
 mlx4_rx_intr_enable(struct rte_eth_dev *dev, uint16_t idx)
 {
 	struct rxq *rxq = dev->data->rx_queues[idx];
-	int ret;
+	int ret = 0;
 
-	if (!rxq || !rxq->channel)
+	if (!rxq || !rxq->channel) {
 		ret = EINVAL;
-	else
-		ret = ibv_req_notify_cq(rxq->cq, 0);
-	if (ret) {
 		rte_errno = ret;
 		WARN("unable to arm interrupt on rx queue %d", idx);
+	} else {
+		mlx4_arm_cq(rxq, 0);
 	}
 	return -ret;
 }
+
+/**
+ * Enable datapath interrupts.
+ *
+ * @param priv
+ *   Pointer to private structure.
+ *
+ * @return
+ *   0 on success, negative errno value otherwise and rte_errno is set.
+ */
+int
+mlx4_rxq_intr_enable(struct mlx4_priv *priv)
+{
+	const struct rte_eth_intr_conf *const intr_conf =
+		&ETH_DEV(priv)->data->dev_conf.intr_conf;
+
+	if (intr_conf->rxq && mlx4_rx_intr_vec_enable(priv) < 0)
+		goto error;
+	return 0;
+error:
+	return -rte_errno;
+}
+
+/**
+ * Disable datapath interrupts, keeping other interrupts intact.
+ *
+ * @param priv
+ *   Pointer to private structure.
+ */
+void
+mlx4_rxq_intr_disable(struct mlx4_priv *priv)
+{
+	int err = rte_errno; /* Make sure rte_errno remains unchanged. */
+
+	mlx4_rx_intr_vec_disable(priv);
+	rte_errno = err;
+}