net/mlx4: fix premature disabling of interrupt
[dpdk.git] / drivers / net / mlx4 / mlx4_intr.c
index 76d2e01..19af935 100644 (file)
@@ -1,34 +1,6 @@
-/*-
- *   BSD LICENSE
- *
- *   Copyright 2017 6WIND S.A.
- *   Copyright 2017 Mellanox
- *
- *   Redistribution and use in source and binary forms, with or without
- *   modification, are permitted provided that the following conditions
- *   are met:
- *
- *     * Redistributions of source code must retain the above copyright
- *       notice, this list of conditions and the following disclaimer.
- *     * Redistributions in binary form must reproduce the above copyright
- *       notice, this list of conditions and the following disclaimer in
- *       the documentation and/or other materials provided with the
- *       distribution.
- *     * Neither the name of 6WIND S.A. nor the names of its
- *       contributors may be used to endorse or promote products derived
- *       from this software without specific prior written permission.
- *
- *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
- *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
- *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
- *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
- *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
- *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
- *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
- *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
- *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
- *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
- *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright 2017 6WIND S.A.
+ * Copyright 2017 Mellanox Technologies, Ltd
  */
 
 /**
@@ -36,7 +8,6 @@
  * Interrupts handling for mlx4 driver.
  */
 
-#include <assert.h>
 #include <errno.h>
 #include <stdint.h>
 #include <stdlib.h>
 
 #include <rte_alarm.h>
 #include <rte_errno.h>
-#include <rte_ethdev.h>
+#include <rte_ethdev_driver.h>
+#include <rte_io.h>
 #include <rte_interrupts.h>
 
 #include "mlx4.h"
+#include "mlx4_glue.h"
 #include "mlx4_rxtx.h"
 #include "mlx4_utils.h"
 
-static void mlx4_link_status_alarm(struct priv *priv);
+static int mlx4_link_status_check(struct mlx4_priv *priv);
 
 /**
  * Clean up Rx interrupts handler.
@@ -68,7 +41,7 @@ static void mlx4_link_status_alarm(struct priv *priv);
  *   Pointer to private structure.
  */
 static void
-mlx4_rx_intr_vec_disable(struct priv *priv)
+mlx4_rx_intr_vec_disable(struct mlx4_priv *priv)
 {
        struct rte_intr_handle *intr_handle = &priv->intr_handle;
 
@@ -88,16 +61,16 @@ mlx4_rx_intr_vec_disable(struct priv *priv)
  *   0 on success, negative errno value otherwise and rte_errno is set.
  */
 static int
-mlx4_rx_intr_vec_enable(struct priv *priv)
+mlx4_rx_intr_vec_enable(struct mlx4_priv *priv)
 {
        unsigned int i;
-       unsigned int rxqs_n = priv->rxqs_n;
+       unsigned int rxqs_n = ETH_DEV(priv)->data->nb_rx_queues;
        unsigned int n = RTE_MIN(rxqs_n, (uint32_t)RTE_MAX_RXTX_INTR_VEC_ID);
        unsigned int count = 0;
        struct rte_intr_handle *intr_handle = &priv->intr_handle;
 
        mlx4_rx_intr_vec_disable(priv);
-       intr_handle->intr_vec = malloc(sizeof(intr_handle->intr_vec[rxqs_n]));
+       intr_handle->intr_vec = malloc(n * sizeof(intr_handle->intr_vec[0]));
        if (intr_handle->intr_vec == NULL) {
                rte_errno = ENOMEM;
                ERROR("failed to allocate memory for interrupt vector,"
@@ -105,7 +78,7 @@ mlx4_rx_intr_vec_enable(struct priv *priv)
                return -rte_errno;
        }
        for (i = 0; i != n; ++i) {
-               struct rxq *rxq = (*priv->rxqs)[i];
+               struct rxq *rxq = ETH_DEV(priv)->data->rx_queues[i];
 
                /* Skip queues that cannot request interrupts. */
                if (!rxq || !rxq->channel) {
@@ -135,115 +108,136 @@ mlx4_rx_intr_vec_enable(struct priv *priv)
 }
 
 /**
- * Collect interrupt events.
+ * Process scheduled link status check.
+ *
+ * If LSC interrupts are requested, process related callback.
+ *
+ * @param priv
+ *   Pointer to private structure.
+ */
+static void
+mlx4_link_status_alarm(struct mlx4_priv *priv)
+{
+       const struct rte_intr_conf *const intr_conf =
+               &ETH_DEV(priv)->data->dev_conf.intr_conf;
+
+       MLX4_ASSERT(priv->intr_alarm == 1);
+       priv->intr_alarm = 0;
+       if (intr_conf->lsc && !mlx4_link_status_check(priv))
+               _rte_eth_dev_callback_process(ETH_DEV(priv),
+                                             RTE_ETH_EVENT_INTR_LSC,
+                                             NULL);
+}
+
+/**
+ * Check link status.
+ *
+ * In case of inconsistency, another check is scheduled.
  *
  * @param priv
  *   Pointer to private structure.
- * @param events
- *   Pointer to event flags holder.
  *
  * @return
- *   Number of events.
+ *   0 on success (link status is consistent), negative errno value
+ *   otherwise and rte_errno is set.
  */
 static int
-mlx4_collect_interrupt_events(struct priv *priv, uint32_t *events)
+mlx4_link_status_check(struct mlx4_priv *priv)
 {
-       struct ibv_async_event event;
-       int port_change = 0;
-       struct rte_eth_link *link = &priv->dev->data->dev_link;
-       const struct rte_intr_conf *const intr_conf =
-               &priv->dev->data->dev_conf.intr_conf;
-       int ret = 0;
+       struct rte_eth_link *link = &ETH_DEV(priv)->data->dev_link;
+       int ret = mlx4_link_update(ETH_DEV(priv), 0);
 
-       *events = 0;
-       /* Read all message and acknowledge them. */
-       for (;;) {
-               if (ibv_get_async_event(priv->ctx, &event))
-                       break;
-               if ((event.event_type == IBV_EVENT_PORT_ACTIVE ||
-                    event.event_type == IBV_EVENT_PORT_ERR) &&
-                   intr_conf->lsc) {
-                       port_change = 1;
-                       ret++;
-               } else if (event.event_type == IBV_EVENT_DEVICE_FATAL &&
-                          intr_conf->rmv) {
-                       *events |= (1 << RTE_ETH_EVENT_INTR_RMV);
-                       ret++;
-               } else {
-                       DEBUG("event type %d on port %d not handled",
-                             event.event_type, event.element.port_num);
-               }
-               ibv_ack_async_event(&event);
-       }
-       if (!port_change)
+       if (ret)
                return ret;
-       mlx4_link_update(priv->dev, 0);
-       if (((link->link_speed == 0) && link->link_status) ||
-           ((link->link_speed != 0) && !link->link_status)) {
+       if ((!link->link_speed && link->link_status) ||
+           (link->link_speed && !link->link_status)) {
                if (!priv->intr_alarm) {
                        /* Inconsistent status, check again later. */
+                       ret = rte_eal_alarm_set(MLX4_INTR_ALARM_TIMEOUT,
+                                               (void (*)(void *))
+                                               mlx4_link_status_alarm,
+                                               priv);
+                       if (ret)
+                               return ret;
                        priv->intr_alarm = 1;
-                       rte_eal_alarm_set(MLX4_INTR_ALARM_TIMEOUT,
-                                         (void (*)(void *))
-                                         mlx4_link_status_alarm,
-                                         priv);
                }
-       } else {
-               *events |= (1 << RTE_ETH_EVENT_INTR_LSC);
+               rte_errno = EINPROGRESS;
+               return -rte_errno;
        }
-       return ret;
+       return 0;
 }
 
 /**
- * Process scheduled link status check.
+ * Handle interrupts from the NIC.
  *
  * @param priv
  *   Pointer to private structure.
  */
 static void
-mlx4_link_status_alarm(struct priv *priv)
+mlx4_interrupt_handler(struct mlx4_priv *priv)
 {
-       uint32_t events;
-       int ret;
+       enum { LSC, RMV, };
+       static const enum rte_eth_event_type type[] = {
+               [LSC] = RTE_ETH_EVENT_INTR_LSC,
+               [RMV] = RTE_ETH_EVENT_INTR_RMV,
+       };
+       uint32_t caught[RTE_DIM(type)] = { 0 };
+       struct ibv_async_event event;
+       const struct rte_intr_conf *const intr_conf =
+               &ETH_DEV(priv)->data->dev_conf.intr_conf;
+       unsigned int i;
 
-       assert(priv->intr_alarm == 1);
-       priv->intr_alarm = 0;
-       ret = mlx4_collect_interrupt_events(priv, &events);
-       if (ret > 0 && events & (1 << RTE_ETH_EVENT_INTR_LSC))
-               _rte_eth_dev_callback_process(priv->dev,
-                                             RTE_ETH_EVENT_INTR_LSC,
-                                             NULL, NULL);
+       /* Read all message and acknowledge them. */
+       while (!mlx4_glue->get_async_event(priv->ctx, &event)) {
+               switch (event.event_type) {
+               case IBV_EVENT_PORT_ACTIVE:
+               case IBV_EVENT_PORT_ERR:
+                       if (intr_conf->lsc && !mlx4_link_status_check(priv))
+                               ++caught[LSC];
+                       break;
+               case IBV_EVENT_DEVICE_FATAL:
+                       if (intr_conf->rmv)
+                               ++caught[RMV];
+                       break;
+               default:
+                       DEBUG("event type %d on physical port %d not handled",
+                             event.event_type, event.element.port_num);
+               }
+               mlx4_glue->ack_async_event(&event);
+       }
+       for (i = 0; i != RTE_DIM(caught); ++i)
+               if (caught[i])
+                       _rte_eth_dev_callback_process(ETH_DEV(priv), type[i],
+                                                     NULL);
 }
 
 /**
- * Handle interrupts from the NIC.
+ * MLX4 CQ notification .
  *
- * @param priv
- *   Pointer to private structure.
+ * @param rxq
+ *   Pointer to receive queue structure.
+ * @param solicited
+ *   Is request solicited or not.
  */
 static void
-mlx4_interrupt_handler(struct priv *priv)
+mlx4_arm_cq(struct rxq *rxq, int solicited)
 {
-       int ret;
-       uint32_t ev;
-       int i;
+       struct mlx4_cq *cq = &rxq->mcq;
+       uint64_t doorbell;
+       uint32_t sn = cq->arm_sn & MLX4_CQ_DB_GEQ_N_MASK;
+       uint32_t ci = cq->cons_index & MLX4_CQ_DB_CI_MASK;
+       uint32_t cmd = solicited ? MLX4_CQ_DB_REQ_NOT_SOL : MLX4_CQ_DB_REQ_NOT;
 
-       ret = mlx4_collect_interrupt_events(priv, &ev);
-       if (ret > 0) {
-               for (i = RTE_ETH_EVENT_UNKNOWN;
-                    i < RTE_ETH_EVENT_MAX;
-                    i++) {
-                       if (ev & (1 << i)) {
-                               ev &= ~(1 << i);
-                               _rte_eth_dev_callback_process(priv->dev, i,
-                                                             NULL, NULL);
-                               ret--;
-                       }
-               }
-               if (ret)
-                       WARN("%d event%s not processed", ret,
-                            (ret > 1 ? "s were" : " was"));
-       }
+       *cq->arm_db = rte_cpu_to_be_32(sn << 28 | cmd | ci);
+       /*
+        * Make sure that the doorbell record in host memory is
+        * written before ringing the doorbell via PCI MMIO.
+        */
+       rte_wmb();
+       doorbell = sn << 28 | cmd | cq->cqn;
+       doorbell <<= 32;
+       doorbell |= ci;
+       rte_write64(rte_cpu_to_be_64(doorbell), cq->cq_db_reg);
 }
 
 /**
@@ -256,7 +250,7 @@ mlx4_interrupt_handler(struct priv *priv)
  *   0 on success, negative errno value otherwise and rte_errno is set.
  */
 int
-mlx4_intr_uninstall(struct priv *priv)
+mlx4_intr_uninstall(struct mlx4_priv *priv)
 {
        int err = rte_errno; /* Make sure rte_errno remains unchanged. */
 
@@ -269,7 +263,7 @@ mlx4_intr_uninstall(struct priv *priv)
        }
        rte_eal_alarm_cancel((void (*)(void *))mlx4_link_status_alarm, priv);
        priv->intr_alarm = 0;
-       mlx4_rx_intr_vec_disable(priv);
+       mlx4_rxq_intr_disable(priv);
        rte_errno = err;
        return 0;
 }
@@ -284,15 +278,13 @@ mlx4_intr_uninstall(struct priv *priv)
  *   0 on success, negative errno value otherwise and rte_errno is set.
  */
 int
-mlx4_intr_install(struct priv *priv)
+mlx4_intr_install(struct mlx4_priv *priv)
 {
        const struct rte_intr_conf *const intr_conf =
-               &priv->dev->data->dev_conf.intr_conf;
+               &ETH_DEV(priv)->data->dev_conf.intr_conf;
        int rc;
 
        mlx4_intr_uninstall(priv);
-       if (intr_conf->rxq && mlx4_rx_intr_vec_enable(priv) < 0)
-               goto error;
        if (intr_conf->lsc | intr_conf->rmv) {
                priv->intr_handle.fd = priv->ctx->async_fd;
                rc = rte_intr_callback_register(&priv->intr_handle,
@@ -324,8 +316,7 @@ error:
 int
 mlx4_rx_intr_disable(struct rte_eth_dev *dev, uint16_t idx)
 {
-       struct priv *priv = dev->data->dev_private;
-       struct rxq *rxq = (*priv->rxqs)[idx];
+       struct rxq *rxq = dev->data->rx_queues[idx];
        struct ibv_cq *ev_cq;
        void *ev_ctx;
        int ret;
@@ -333,16 +324,25 @@ mlx4_rx_intr_disable(struct rte_eth_dev *dev, uint16_t idx)
        if (!rxq || !rxq->channel) {
                ret = EINVAL;
        } else {
-               ret = ibv_get_cq_event(rxq->cq->channel, &ev_cq, &ev_ctx);
-               if (ret || ev_cq != rxq->cq)
+               ret = mlx4_glue->get_cq_event(rxq->cq->channel, &ev_cq,
+                                             &ev_ctx);
+               /** For non-zero ret save the errno (may be EAGAIN
+                * which means the get_cq_event function was called before
+                * receiving one).
+                */
+               if (ret)
+                       ret = errno;
+               else if (ev_cq != rxq->cq)
                        ret = EINVAL;
        }
        if (ret) {
                rte_errno = ret;
-               WARN("unable to disable interrupt on rx queue %d",
-                    idx);
+               if (ret != EAGAIN)
+                       WARN("unable to disable interrupt on rx queue %d",
+                            idx);
        } else {
-               ibv_ack_cq_events(rxq->cq, 1);
+               rxq->mcq.arm_sn++;
+               mlx4_glue->ack_cq_events(rxq->cq, 1);
        }
        return -ret;
 }
@@ -361,17 +361,52 @@ mlx4_rx_intr_disable(struct rte_eth_dev *dev, uint16_t idx)
 int
 mlx4_rx_intr_enable(struct rte_eth_dev *dev, uint16_t idx)
 {
-       struct priv *priv = dev->data->dev_private;
-       struct rxq *rxq = (*priv->rxqs)[idx];
-       int ret;
+       struct rxq *rxq = dev->data->rx_queues[idx];
+       int ret = 0;
 
-       if (!rxq || !rxq->channel)
+       if (!rxq || !rxq->channel) {
                ret = EINVAL;
-       else
-               ret = ibv_req_notify_cq(rxq->cq, 0);
-       if (ret) {
                rte_errno = ret;
                WARN("unable to arm interrupt on rx queue %d", idx);
+       } else {
+               mlx4_arm_cq(rxq, 0);
        }
        return -ret;
 }
+
+/**
+ * Enable datapath interrupts.
+ *
+ * @param priv
+ *   Pointer to private structure.
+ *
+ * @return
+ *   0 on success, negative errno value otherwise and rte_errno is set.
+ */
+int
+mlx4_rxq_intr_enable(struct mlx4_priv *priv)
+{
+       const struct rte_intr_conf *const intr_conf =
+               &ETH_DEV(priv)->data->dev_conf.intr_conf;
+
+       if (intr_conf->rxq && mlx4_rx_intr_vec_enable(priv) < 0)
+               goto error;
+       return 0;
+error:
+       return -rte_errno;
+}
+
+/**
+ * Disable datapath interrupts, keeping other interrupts intact.
+ *
+ * @param priv
+ *   Pointer to private structure.
+ */
+void
+mlx4_rxq_intr_disable(struct mlx4_priv *priv)
+{
+       int err = rte_errno; /* Make sure rte_errno remains unchanged. */
+
+       mlx4_rx_intr_vec_disable(priv);
+       rte_errno = err;
+}