net/mlx5: select Rx/Tx callbacks when starting device

[dpdk.git] / drivers / net / mlx5 / mlx5_txq.c

diff --git a/drivers/net/mlx5/mlx5_txq.c b/drivers/net/mlx5/mlx5_txq.c
index f80740a..03e23f5 100644 (file)
--- a/drivers/net/mlx5/mlx5_txq.c
+++ b/drivers/net/mlx5/mlx5_txq.c
@@ -103,9 +103,10 @@ txq_alloc_elts(struct txq_ctrl *txq_ctrl, unsigned int elts_n)
 static void
 txq_free_elts(struct txq_ctrl *txq_ctrl)
 {
-       unsigned int elts_n = 1 << txq_ctrl->txq.elts_n;
-       unsigned int elts_head = txq_ctrl->txq.elts_head;
-       unsigned int elts_tail = txq_ctrl->txq.elts_tail;
+       const uint16_t elts_n = 1 << txq_ctrl->txq.elts_n;
+       const uint16_t elts_m = elts_n - 1;
+       uint16_t elts_head = txq_ctrl->txq.elts_head;
+       uint16_t elts_tail = txq_ctrl->txq.elts_tail;
        struct rte_mbuf *(*elts)[elts_n] = txq_ctrl->txq.elts;
 
        DEBUG("%p: freeing WRs", (void *)txq_ctrl);
@@ -114,18 +115,17 @@ txq_free_elts(struct txq_ctrl *txq_ctrl)
        txq_ctrl->txq.elts_comp = 0;
 
        while (elts_tail != elts_head) {
-               struct rte_mbuf *elt = (*elts)[elts_tail];
+               struct rte_mbuf *elt = (*elts)[elts_tail & elts_m];
 
                assert(elt != NULL);
-               rte_pktmbuf_free(elt);
+               rte_pktmbuf_free_seg(elt);
 #ifndef NDEBUG
                /* Poisoning. */
-               memset(&(*elts)[elts_tail],
+               memset(&(*elts)[elts_tail & elts_m],
                       0x77,
-                      sizeof((*elts)[elts_tail]));
+                      sizeof((*elts)[elts_tail & elts_m]));
 #endif
-               if (++elts_tail == elts_n)
-                       elts_tail = 0;
+               ++elts_tail;
        }
 }
 
@@ -149,9 +149,8 @@ txq_cleanup(struct txq_ctrl *txq_ctrl)
        if (txq_ctrl->cq != NULL)
                claim_zero(ibv_destroy_cq(txq_ctrl->cq));
        for (i = 0; (i != RTE_DIM(txq_ctrl->txq.mp2mr)); ++i) {
-               if (txq_ctrl->txq.mp2mr[i].mp == NULL)
+               if (txq_ctrl->txq.mp2mr[i].mr == NULL)
                        break;
-               assert(txq_ctrl->txq.mp2mr[i].mr != NULL);
                claim_zero(ibv_dereg_mr(txq_ctrl->txq.mp2mr[i].mr));
        }
        memset(txq_ctrl, 0, sizeof(*txq_ctrl));
@@ -173,23 +172,27 @@ txq_setup(struct txq_ctrl *tmpl, struct txq_ctrl *txq_ctrl)
 {
        struct mlx5_qp *qp = to_mqp(tmpl->qp);
        struct ibv_cq *ibcq = tmpl->cq;
-       struct mlx5_cq *cq = to_mxxx(cq, cq);
+       struct ibv_mlx5_cq_info cq_info;
 
-       if (cq->cqe_sz != RTE_CACHE_LINE_SIZE) {
+       if (ibv_mlx5_exp_get_cq_info(ibcq, &cq_info)) {
+               ERROR("Unable to query CQ info. check your OFED.");
+               return ENOTSUP;
+       }
+       if (cq_info.cqe_size != RTE_CACHE_LINE_SIZE) {
                ERROR("Wrong MLX5_CQE_SIZE environment variable value: "
                      "it should be set to %u", RTE_CACHE_LINE_SIZE);
                return EINVAL;
        }
-       tmpl->txq.cqe_n = log2above(ibcq->cqe);
+       tmpl->txq.cqe_n = log2above(cq_info.cqe_cnt);
        tmpl->txq.qp_num_8s = qp->ctrl_seg.qp_num << 8;
        tmpl->txq.wqes = qp->gen_data.sqstart;
        tmpl->txq.wqe_n = log2above(qp->sq.wqe_cnt);
        tmpl->txq.qp_db = &qp->gen_data.db[MLX5_SND_DBR];
        tmpl->txq.bf_reg = qp->gen_data.bf->reg;
-       tmpl->txq.cq_db = cq->dbrec;
+       tmpl->txq.cq_db = cq_info.dbrec;
        tmpl->txq.cqes =
                (volatile struct mlx5_cqe (*)[])
-               (uintptr_t)cq->active_buf->buf;
+               (uintptr_t)cq_info.buf;
        tmpl->txq.elts =
                (struct rte_mbuf *(*)[1 << tmpl->txq.elts_n])
                ((uintptr_t)txq_ctrl + sizeof(*txq_ctrl));
@@ -230,6 +233,9 @@ txq_ctrl_setup(struct rte_eth_dev *dev, struct txq_ctrl *txq_ctrl,
                struct ibv_exp_cq_attr cq_attr;
        } attr;
        unsigned int cqe_n;
+       const unsigned int max_tso_inline = ((MLX5_MAX_TSO_HEADER +
+                                            (RTE_CACHE_LINE_SIZE - 1)) /
+                                             RTE_CACHE_LINE_SIZE);
        int ret = 0;
 
        if (mlx5_getenv_int("MLX5_ENABLE_CQE_COMPRESSION")) {
@@ -307,16 +313,23 @@ txq_ctrl_setup(struct rte_eth_dev *dev, struct txq_ctrl *txq_ctrl,
                                          priv->inline_max_packet_sz) +
                                  (RTE_CACHE_LINE_SIZE - 1)) /
                                 RTE_CACHE_LINE_SIZE) * RTE_CACHE_LINE_SIZE;
+               } else if (priv->tso) {
+                       int inline_diff = tmpl.txq.max_inline - max_tso_inline;
+
+                       /*
+                        * Adjust inline value as Verbs aggregates
+                        * tso_inline and txq_inline fields.
+                        */
+                       attr.init.cap.max_inline_data = inline_diff > 0 ?
+                                                       inline_diff *
+                                                       RTE_CACHE_LINE_SIZE :
+                                                       0;
                } else {
                        attr.init.cap.max_inline_data =
                                tmpl.txq.max_inline * RTE_CACHE_LINE_SIZE;
                }
        }
        if (priv->tso) {
-               uint16_t max_tso_inline = ((MLX5_MAX_TSO_HEADER +
-                                          (RTE_CACHE_LINE_SIZE - 1)) /
-                                           RTE_CACHE_LINE_SIZE);
-
                attr.init.max_tso_header =
                        max_tso_inline * RTE_CACHE_LINE_SIZE;
                attr.init.comp_mask |= IBV_EXP_QP_INIT_ATTR_MAX_TSO_HEADER;
@@ -483,8 +496,6 @@ mlx5_tx_queue_setup(struct rte_eth_dev *dev, uint16_t idx, uint16_t desc,
                DEBUG("%p: adding TX queue %p to list",
                      (void *)dev, (void *)txq_ctrl);
                (*priv->txqs)[idx] = &txq_ctrl->txq;
-               /* Update send callback. */
-               priv_select_tx_function(priv);
        }
        priv_unlock(priv);
        return -ret;