net/mlx4: fix no Rx interrupts

This commit addresses the issue of Rx interrupts support with
the new Rx datapath introduced in DPDK version 17.11.
In order to generate an Rx interrupt an event queue is armed with the
consumer index of the Rx completion queue. Since version 17.11 this
index is handled by the PMD so it is now the responsibility of the
PMD to write this value when enabling Rx interrupts.

Fixes: 6681b845034c ("net/mlx4: add Rx bypassing Verbs")
Signed-off-by: Moti Haimovsky <motih@mellanox.com>
Acked-by: Nelio Laranjeiro <nelio.laranjeiro@6wind.com>

diff --git a/drivers/net/mlx4/mlx4_intr.c b/drivers/net/mlx4/mlx4_intr.c
index 3806322e5f9a61df20206c23d26a557b3d1ac955..b17d109a369fc4e791611a3d93b91817dadc4052 100644 (file)
--- a/drivers/net/mlx4/mlx4_intr.c
+++ b/drivers/net/mlx4/mlx4_intr.c
@@ -53,6 +53,7 @@
 #include <rte_alarm.h>
 #include <rte_errno.h>
 #include <rte_ethdev.h>
+#include <rte_io.h>
 #include <rte_interrupts.h>
 
 #include "mlx4.h"
@@ -238,6 +239,35 @@ mlx4_interrupt_handler(struct priv *priv)
                                                      NULL, NULL);
 }
 
+/**
+ * MLX4 CQ notification .
+ *
+ * @param rxq
+ *   Pointer to receive queue structure.
+ * @param solicited
+ *   Is request solicited or not.
+ */
+static void
+mlx4_arm_cq(struct rxq *rxq, int solicited)
+{
+       struct mlx4_cq *cq = &rxq->mcq;
+       uint64_t doorbell;
+       uint32_t sn = cq->arm_sn & MLX4_CQ_DB_GEQ_N_MASK;
+       uint32_t ci = cq->cons_index & MLX4_CQ_DB_CI_MASK;
+       uint32_t cmd = solicited ? MLX4_CQ_DB_REQ_NOT_SOL : MLX4_CQ_DB_REQ_NOT;
+
+       *cq->arm_db = rte_cpu_to_be_32(sn << 28 | cmd | ci);
+       /*
+        * Make sure that the doorbell record in host memory is
+        * written before ringing the doorbell via PCI MMIO.
+        */
+       rte_wmb();
+       doorbell = sn << 28 | cmd | cq->cqn;
+       doorbell <<= 32;
+       doorbell |= ci;
+       rte_write64(rte_cpu_to_be_64(doorbell), cq->cq_db_reg);
+}
+
 /**
  * Uninstall interrupt handler.
  *
@@ -333,6 +363,7 @@ mlx4_rx_intr_disable(struct rte_eth_dev *dev, uint16_t idx)
                WARN("unable to disable interrupt on rx queue %d",
                     idx);
        } else {
+               rxq->mcq.arm_sn++;
                ibv_ack_cq_events(rxq->cq, 1);
        }
        return -ret;
@@ -353,15 +384,14 @@ int
 mlx4_rx_intr_enable(struct rte_eth_dev *dev, uint16_t idx)
 {
        struct rxq *rxq = dev->data->rx_queues[idx];
-       int ret;
+       int ret = 0;
 
-       if (!rxq || !rxq->channel)
+       if (!rxq || !rxq->channel) {
                ret = EINVAL;
-       else
-               ret = ibv_req_notify_cq(rxq->cq, 0);
-       if (ret) {
                rte_errno = ret;
                WARN("unable to arm interrupt on rx queue %d", idx);
+       } else {
+               mlx4_arm_cq(rxq, 0);
        }
        return -ret;
 }

diff --git a/drivers/net/mlx4/mlx4_prm.h b/drivers/net/mlx4/mlx4_prm.h
index bd9a57456be77fd5c474ff58a783eeb3f2d6be5b..b0fd9820f77e85b0f62643ccc0de81a5f42e4a14 100644 (file)
--- a/drivers/net/mlx4/mlx4_prm.h
+++ b/drivers/net/mlx4/mlx4_prm.h
@@ -93,16 +93,23 @@ struct mlx4_sq {
 
 #define mlx4_get_send_wqe(sq, n) ((sq)->buf + ((n) * (MLX4_TXBB_SIZE)))
 
-/* Completion queue consumer index mask. */
+/* Completion queue events, numbers and masks. */
+#define MLX4_CQ_DB_GEQ_N_MASK 0x3
+#define MLX4_CQ_DOORBELL 0x20
 #define MLX4_CQ_DB_CI_MASK 0xffffff
 
 /* Completion queue information. */
 struct mlx4_cq {
+       void *cq_uar; /**< CQ user access region. */
+       void *cq_db_reg; /**< CQ doorbell register. */
+       uint32_t *set_ci_db; /**< Pointer to the completion queue doorbell. */
+       uint32_t *arm_db; /**< Pointer to doorbell for arming Rx events. */
        uint8_t *buf; /**< Pointer to the completion queue buffer. */
        uint32_t cqe_cnt; /**< Number of entries in the queue. */
        uint32_t cqe_64:1; /**< CQ entry size is 64 bytes. */
        uint32_t cons_index; /**< Last queue entry that was handled. */
-       uint32_t *set_ci_db; /**< Pointer to the completion queue doorbell. */
+       uint32_t cqn; /**< CQ number. */
+       int arm_sn; /**< Rx event counter. */
 };
 
 /**

diff --git a/drivers/net/mlx4/mlx4_rxq.c b/drivers/net/mlx4/mlx4_rxq.c
index ad55934b446f52c47799442363d9fa148453f75a..7fe21b6fb71c4081e34c1cec112281f34060bc8c 100644 (file)
--- a/drivers/net/mlx4/mlx4_rxq.c
+++ b/drivers/net/mlx4/mlx4_rxq.c
@@ -510,7 +510,7 @@ mlx4_rxq_attach(struct rxq *rxq)
        struct rte_mbuf *(*elts)[elts_n] = rxq->elts;
        struct mlx4dv_obj mlxdv;
        struct mlx4dv_rwq dv_rwq;
-       struct mlx4dv_cq dv_cq;
+       struct mlx4dv_cq dv_cq = { .comp_mask = MLX4DV_CQ_MASK_UAR, };
        const char *msg;
        struct ibv_cq *cq = NULL;
        struct ibv_wq *wq = NULL;
@@ -604,6 +604,11 @@ mlx4_rxq_attach(struct rxq *rxq)
        rxq->mcq.cqe_cnt = dv_cq.cqe_cnt;
        rxq->mcq.set_ci_db = dv_cq.set_ci_db;
        rxq->mcq.cqe_64 = (dv_cq.cqe_size & 64) ? 1 : 0;
+       rxq->mcq.arm_db = dv_cq.arm_db;
+       rxq->mcq.arm_sn = dv_cq.arm_sn;
+       rxq->mcq.cqn = dv_cq.cqn;
+       rxq->mcq.cq_uar = dv_cq.cq_uar;
+       rxq->mcq.cq_db_reg = (uint8_t *)dv_cq.cq_uar + MLX4_CQ_DOORBELL;
        /* Update doorbell counter. */
        rxq->rq_ci = elts_n / sges_n;
        rte_wmb();