From 72ba7fadbf6e4a1c65e93cbfbdf407507ef3374d Mon Sep 17 00:00:00 2001 From: Adrien Mazarguil Date: Fri, 1 Sep 2017 10:06:37 +0200 Subject: [PATCH] net/mlx4: revert resource domain support This reverts commit 3e49c148b715c3c0a12c1200295bb9b312f7028e. Resource domains are not part of the standard Verbs interface. The performance improvement they bring will be restored later through a different data path implementation. This commit makes the PMD not rely on the non-standard QP allocation interface. Signed-off-by: Moti Haimovsky Signed-off-by: Adrien Mazarguil --- drivers/net/mlx4/mlx4.c | 88 ++++-------------------------------- drivers/net/mlx4/mlx4.h | 2 - drivers/net/mlx4/mlx4_flow.c | 30 +++++------- 3 files changed, 20 insertions(+), 100 deletions(-) diff --git a/drivers/net/mlx4/mlx4.c b/drivers/net/mlx4/mlx4.c index 682307fb8c..7dbed93047 100644 --- a/drivers/net/mlx4/mlx4.c +++ b/drivers/net/mlx4/mlx4.c @@ -697,17 +697,6 @@ txq_cleanup(struct txq *txq) claim_zero(ibv_destroy_qp(txq->qp)); if (txq->cq != NULL) claim_zero(ibv_destroy_cq(txq->cq)); - if (txq->rd != NULL) { - struct ibv_exp_destroy_res_domain_attr attr = { - .comp_mask = 0, - }; - - assert(txq->priv != NULL); - assert(txq->priv->ctx != NULL); - claim_zero(ibv_exp_destroy_res_domain(txq->priv->ctx, - txq->rd, - &attr)); - } for (i = 0; (i != elemof(txq->mp2mr)); ++i) { if (txq->mp2mr[i].mp == NULL) break; @@ -1175,9 +1164,7 @@ txq_setup(struct rte_eth_dev *dev, struct txq *txq, uint16_t desc, }; union { struct ibv_exp_query_intf_params params; - struct ibv_exp_qp_init_attr init; - struct ibv_exp_res_domain_init_attr rd; - struct ibv_exp_cq_init_attr cq; + struct ibv_qp_init_attr init; struct ibv_qp_attr mod; } attr; enum ibv_exp_query_intf_status status; @@ -1191,24 +1178,7 @@ txq_setup(struct rte_eth_dev *dev, struct txq *txq, uint16_t desc, return EINVAL; } /* MRs will be registered in mp2mr[] later. */ - attr.rd = (struct ibv_exp_res_domain_init_attr){ - .comp_mask = (IBV_EXP_RES_DOMAIN_THREAD_MODEL | - IBV_EXP_RES_DOMAIN_MSG_MODEL), - .thread_model = IBV_EXP_THREAD_SINGLE, - .msg_model = IBV_EXP_MSG_HIGH_BW, - }; - tmpl.rd = ibv_exp_create_res_domain(priv->ctx, &attr.rd); - if (tmpl.rd == NULL) { - ret = ENOMEM; - ERROR("%p: RD creation failure: %s", - (void *)dev, strerror(ret)); - goto error; - } - attr.cq = (struct ibv_exp_cq_init_attr){ - .comp_mask = IBV_EXP_CQ_INIT_ATTR_RES_DOMAIN, - .res_domain = tmpl.rd, - }; - tmpl.cq = ibv_exp_create_cq(priv->ctx, desc, NULL, NULL, 0, &attr.cq); + tmpl.cq = ibv_create_cq(priv->ctx, desc, NULL, NULL, 0); if (tmpl.cq == NULL) { ret = ENOMEM; ERROR("%p: CQ creation failure: %s", @@ -1219,7 +1189,7 @@ txq_setup(struct rte_eth_dev *dev, struct txq *txq, uint16_t desc, priv->device_attr.max_qp_wr); DEBUG("priv->device_attr.max_sge is %d", priv->device_attr.max_sge); - attr.init = (struct ibv_exp_qp_init_attr){ + attr.init = (struct ibv_qp_init_attr){ /* CQ to be associated with the send queue. */ .send_cq = tmpl.cq, /* CQ to be associated with the receive queue. */ @@ -1237,12 +1207,8 @@ txq_setup(struct rte_eth_dev *dev, struct txq *txq, uint16_t desc, /* Do *NOT* enable this, completions events are managed per * TX burst. */ .sq_sig_all = 0, - .pd = priv->pd, - .res_domain = tmpl.rd, - .comp_mask = (IBV_EXP_QP_INIT_ATTR_PD | - IBV_EXP_QP_INIT_ATTR_RES_DOMAIN), }; - tmpl.qp = ibv_exp_create_qp(priv->ctx, &attr.init); + tmpl.qp = ibv_create_qp(priv->pd, &attr.init); if (tmpl.qp == NULL) { ret = (errno ? errno : EINVAL); ERROR("%p: QP creation failure: %s", @@ -1710,17 +1676,6 @@ rxq_cleanup(struct rxq *rxq) claim_zero(ibv_destroy_cq(rxq->cq)); if (rxq->channel != NULL) claim_zero(ibv_destroy_comp_channel(rxq->channel)); - if (rxq->rd != NULL) { - struct ibv_exp_destroy_res_domain_attr attr = { - .comp_mask = 0, - }; - - assert(rxq->priv != NULL); - assert(rxq->priv->ctx != NULL); - claim_zero(ibv_exp_destroy_res_domain(rxq->priv->ctx, - rxq->rd, - &attr)); - } if (rxq->mr != NULL) claim_zero(ibv_dereg_mr(rxq->mr)); memset(rxq, 0, sizeof(*rxq)); @@ -1890,10 +1845,9 @@ repost: * QP pointer or NULL in case of error. */ static struct ibv_qp * -rxq_setup_qp(struct priv *priv, struct ibv_cq *cq, uint16_t desc, - struct ibv_exp_res_domain *rd) +rxq_setup_qp(struct priv *priv, struct ibv_cq *cq, uint16_t desc) { - struct ibv_exp_qp_init_attr attr = { + struct ibv_qp_init_attr attr = { /* CQ to be associated with the send queue. */ .send_cq = cq, /* CQ to be associated with the receive queue. */ @@ -1907,13 +1861,9 @@ rxq_setup_qp(struct priv *priv, struct ibv_cq *cq, uint16_t desc, .max_recv_sge = 1, }, .qp_type = IBV_QPT_RAW_PACKET, - .comp_mask = (IBV_EXP_QP_INIT_ATTR_PD | - IBV_EXP_QP_INIT_ATTR_RES_DOMAIN), - .pd = priv->pd, - .res_domain = rd, }; - return ibv_exp_create_qp(priv->ctx, &attr); + return ibv_create_qp(priv->pd, &attr); } /** @@ -1949,8 +1899,6 @@ rxq_setup(struct rte_eth_dev *dev, struct rxq *rxq, uint16_t desc, struct ibv_qp_attr mod; union { struct ibv_exp_query_intf_params params; - struct ibv_exp_cq_init_attr cq; - struct ibv_exp_res_domain_init_attr rd; } attr; enum ibv_exp_query_intf_status status; struct ibv_recv_wr *bad_wr; @@ -1988,19 +1936,6 @@ rxq_setup(struct rte_eth_dev *dev, struct rxq *rxq, uint16_t desc, (void *)dev, strerror(ret)); goto error; } - attr.rd = (struct ibv_exp_res_domain_init_attr){ - .comp_mask = (IBV_EXP_RES_DOMAIN_THREAD_MODEL | - IBV_EXP_RES_DOMAIN_MSG_MODEL), - .thread_model = IBV_EXP_THREAD_SINGLE, - .msg_model = IBV_EXP_MSG_HIGH_BW, - }; - tmpl.rd = ibv_exp_create_res_domain(priv->ctx, &attr.rd); - if (tmpl.rd == NULL) { - ret = ENOMEM; - ERROR("%p: RD creation failure: %s", - (void *)dev, strerror(ret)); - goto error; - } if (dev->data->dev_conf.intr_conf.rxq) { tmpl.channel = ibv_create_comp_channel(priv->ctx); if (tmpl.channel == NULL) { @@ -2011,12 +1946,7 @@ rxq_setup(struct rte_eth_dev *dev, struct rxq *rxq, uint16_t desc, goto error; } } - attr.cq = (struct ibv_exp_cq_init_attr){ - .comp_mask = IBV_EXP_CQ_INIT_ATTR_RES_DOMAIN, - .res_domain = tmpl.rd, - }; - tmpl.cq = ibv_exp_create_cq(priv->ctx, desc, NULL, tmpl.channel, 0, - &attr.cq); + tmpl.cq = ibv_create_cq(priv->ctx, desc, NULL, tmpl.channel, 0); if (tmpl.cq == NULL) { ret = ENOMEM; ERROR("%p: CQ creation failure: %s", @@ -2027,7 +1957,7 @@ rxq_setup(struct rte_eth_dev *dev, struct rxq *rxq, uint16_t desc, priv->device_attr.max_qp_wr); DEBUG("priv->device_attr.max_sge is %d", priv->device_attr.max_sge); - tmpl.qp = rxq_setup_qp(priv, tmpl.cq, desc, tmpl.rd); + tmpl.qp = rxq_setup_qp(priv, tmpl.cq, desc); if (tmpl.qp == NULL) { ret = (errno ? errno : EINVAL); ERROR("%p: QP creation failure: %s", diff --git a/drivers/net/mlx4/mlx4.h b/drivers/net/mlx4/mlx4.h index 9cbde1d1d9..edec40c607 100644 --- a/drivers/net/mlx4/mlx4.h +++ b/drivers/net/mlx4/mlx4.h @@ -167,7 +167,6 @@ struct rxq { struct rxq_elt (*elts)[]; /* Rx elements. */ struct mlx4_rxq_stats stats; /* RX queue counters. */ unsigned int socket; /* CPU socket ID for allocations. */ - struct ibv_exp_res_domain *rd; /* Resource Domain. */ }; /* TX element. */ @@ -204,7 +203,6 @@ struct txq { unsigned int elts_comp_cd_init; /* Initial value for countdown. */ struct mlx4_txq_stats stats; /* TX queue counters. */ unsigned int socket; /* CPU socket ID for allocations. */ - struct ibv_exp_res_domain *rd; /* Resource Domain. */ }; struct rte_flow; diff --git a/drivers/net/mlx4/mlx4_flow.c b/drivers/net/mlx4/mlx4_flow.c index 2c5dc3c188..58d469856e 100644 --- a/drivers/net/mlx4/mlx4_flow.c +++ b/drivers/net/mlx4/mlx4_flow.c @@ -752,29 +752,21 @@ mlx4_flow_create_drop_queue(struct priv *priv) ERROR("Cannot allocate memory for drop struct"); goto err; } - cq = ibv_exp_create_cq(priv->ctx, 1, NULL, NULL, 0, - &(struct ibv_exp_cq_init_attr){ - .comp_mask = 0, - }); + cq = ibv_create_cq(priv->ctx, 1, NULL, NULL, 0); if (!cq) { ERROR("Cannot create drop CQ"); goto err_create_cq; } - qp = ibv_exp_create_qp(priv->ctx, - &(struct ibv_exp_qp_init_attr){ - .send_cq = cq, - .recv_cq = cq, - .cap = { - .max_recv_wr = 1, - .max_recv_sge = 1, - }, - .qp_type = IBV_QPT_RAW_PACKET, - .comp_mask = - IBV_EXP_QP_INIT_ATTR_PD | - IBV_EXP_QP_INIT_ATTR_PORT, - .pd = priv->pd, - .port_num = priv->port, - }); + qp = ibv_create_qp(priv->pd, + &(struct ibv_qp_init_attr){ + .send_cq = cq, + .recv_cq = cq, + .cap = { + .max_recv_wr = 1, + .max_recv_sge = 1, + }, + .qp_type = IBV_QPT_RAW_PACKET, + }); if (!qp) { ERROR("Cannot create drop QP"); goto err_create_qp; -- 2.20.1