X-Git-Url: http://git.droids-corp.org/?a=blobdiff_plain;f=drivers%2Fnet%2Fmlx4%2Fmlx4_txq.c;h=fe6a8e07e35a0b6882a85837cf5717f9017e69a6;hb=cd8c7c7ce241;hp=f102c685784cca14d1d49f5307a57644b315768e;hpb=79770826499bc2e9c74d4d4e49970a16ed7150cc;p=dpdk.git diff --git a/drivers/net/mlx4/mlx4_txq.c b/drivers/net/mlx4/mlx4_txq.c index f102c68578..fe6a8e07e3 100644 --- a/drivers/net/mlx4/mlx4_txq.c +++ b/drivers/net/mlx4/mlx4_txq.c @@ -1,34 +1,6 @@ -/*- - * BSD LICENSE - * - * Copyright 2017 6WIND S.A. - * Copyright 2017 Mellanox - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * - * * Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in - * the documentation and/or other materials provided with the - * distribution. - * * Neither the name of 6WIND S.A. nor the names of its - * contributors may be used to endorse or promote products derived - * from this software without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright 2017 6WIND S.A. + * Copyright 2017 Mellanox Technologies, Ltd */ /** @@ -41,6 +13,7 @@ #include #include #include +#include /* Verbs headers do not support -pedantic. */ #ifdef PEDANTIC @@ -53,69 +26,17 @@ #include #include -#include +#include #include #include #include #include "mlx4.h" -#include "mlx4_autoconf.h" +#include "mlx4_glue.h" +#include "mlx4_prm.h" #include "mlx4_rxtx.h" #include "mlx4_utils.h" -/** - * Allocate Tx queue elements. - * - * @param txq - * Pointer to Tx queue structure. - * @param elts_n - * Number of elements to allocate. - * - * @return - * 0 on success, negative errno value otherwise and rte_errno is set. - */ -static int -mlx4_txq_alloc_elts(struct txq *txq, unsigned int elts_n) -{ - unsigned int i; - struct txq_elt (*elts)[elts_n] = - rte_calloc_socket("TXQ", 1, sizeof(*elts), 0, txq->socket); - int ret = 0; - - if (elts == NULL) { - ERROR("%p: can't allocate packets array", (void *)txq); - ret = ENOMEM; - goto error; - } - for (i = 0; (i != elts_n); ++i) { - struct txq_elt *elt = &(*elts)[i]; - - elt->buf = NULL; - } - DEBUG("%p: allocated and configured %u WRs", (void *)txq, elts_n); - txq->elts_n = elts_n; - txq->elts = elts; - txq->elts_head = 0; - txq->elts_tail = 0; - txq->elts_comp = 0; - /* - * Request send completion every MLX4_PMD_TX_PER_COMP_REQ packets or - * at least 4 times per ring. - */ - txq->elts_comp_cd_init = - ((MLX4_PMD_TX_PER_COMP_REQ < (elts_n / 4)) ? - MLX4_PMD_TX_PER_COMP_REQ : (elts_n / 4)); - txq->elts_comp_cd = txq->elts_comp_cd_init; - assert(ret == 0); - return 0; -error: - rte_free(elts); - DEBUG("%p: failed, freed everything", (void *)txq); - assert(ret > 0); - rte_errno = ret; - return -rte_errno; -} - /** * Free Tx queue elements. * @@ -125,34 +46,21 @@ error: static void mlx4_txq_free_elts(struct txq *txq) { - unsigned int elts_n = txq->elts_n; unsigned int elts_head = txq->elts_head; unsigned int elts_tail = txq->elts_tail; - struct txq_elt (*elts)[elts_n] = txq->elts; + struct txq_elt (*elts)[txq->elts_n] = txq->elts; + unsigned int elts_m = txq->elts_n - 1; DEBUG("%p: freeing WRs", (void *)txq); - txq->elts_n = 0; - txq->elts_head = 0; - txq->elts_tail = 0; - txq->elts_comp = 0; - txq->elts_comp_cd = 0; - txq->elts_comp_cd_init = 0; - txq->elts = NULL; - if (elts == NULL) - return; while (elts_tail != elts_head) { - struct txq_elt *elt = &(*elts)[elts_tail]; + struct txq_elt *elt = &(*elts)[elts_tail++ & elts_m]; assert(elt->buf != NULL); rte_pktmbuf_free(elt->buf); -#ifndef NDEBUG - /* Poisoning. */ - memset(elt, 0x77, sizeof(*elt)); -#endif - if (++elts_tail == elts_n) - elts_tail = 0; + elt->buf = NULL; + elt->wqe = NULL; } - rte_free(elts); + txq->elts_tail = txq->elts_head; } struct txq_mp2mr_mbuf_check_data { @@ -213,6 +121,84 @@ mlx4_txq_mp2mr_iter(struct rte_mempool *mp, void *arg) mlx4_txq_mp2mr(txq, mp); } +/** + * Retrieves information needed in order to directly access the Tx queue. + * + * @param txq + * Pointer to Tx queue structure. + * @param mlxdv + * Pointer to device information for this Tx queue. + */ +static void +mlx4_txq_fill_dv_obj_info(struct txq *txq, struct mlx4dv_obj *mlxdv) +{ + struct mlx4_sq *sq = &txq->msq; + struct mlx4_cq *cq = &txq->mcq; + struct mlx4dv_qp *dqp = mlxdv->qp.out; + struct mlx4dv_cq *dcq = mlxdv->cq.out; + + /* Total length, including headroom and spare WQEs. */ + sq->size = (uint32_t)dqp->rq.offset - (uint32_t)dqp->sq.offset; + sq->buf = (uint8_t *)dqp->buf.buf + dqp->sq.offset; + sq->eob = sq->buf + sq->size; + uint32_t headroom_size = 2048 + (1 << dqp->sq.wqe_shift); + /* Continuous headroom size bytes must always stay freed. */ + sq->remain_size = sq->size - headroom_size; + sq->owner_opcode = MLX4_OPCODE_SEND | (0 << MLX4_SQ_OWNER_BIT); + sq->stamp = rte_cpu_to_be_32(MLX4_SQ_STAMP_VAL | + (0 << MLX4_SQ_OWNER_BIT)); + sq->db = dqp->sdb; + sq->doorbell_qpn = dqp->doorbell_qpn; + cq->buf = dcq->buf.buf; + cq->cqe_cnt = dcq->cqe_cnt; + cq->set_ci_db = dcq->set_ci_db; + cq->cqe_64 = (dcq->cqe_size & 64) ? 1 : 0; +} + +/** + * Returns the per-port supported offloads. + * + * @param priv + * Pointer to private structure. + * + * @return + * Supported Tx offloads. + */ +uint64_t +mlx4_get_tx_port_offloads(struct priv *priv) +{ + uint64_t offloads = DEV_TX_OFFLOAD_MULTI_SEGS; + + if (priv->hw_csum) { + offloads |= (DEV_TX_OFFLOAD_IPV4_CKSUM | + DEV_TX_OFFLOAD_UDP_CKSUM | + DEV_TX_OFFLOAD_TCP_CKSUM); + } + if (priv->hw_csum_l2tun) + offloads |= DEV_TX_OFFLOAD_OUTER_IPV4_CKSUM; + return offloads; +} + +/** + * Checks if the per-queue offload configuration is valid. + * + * @param priv + * Pointer to private structure. + * @param requested + * Per-queue offloads configuration. + * + * @return + * Nonzero when configuration is valid. + */ +static int +mlx4_check_tx_queue_offloads(struct priv *priv, uint64_t requested) +{ + uint64_t mandatory = priv->dev->data->dev_conf.txmode.offloads; + uint64_t supported = mlx4_get_tx_port_offloads(priv); + + return !((mandatory ^ requested) & supported); +} + /** * DPDK callback to configure a Tx queue. * @@ -235,13 +221,48 @@ mlx4_tx_queue_setup(struct rte_eth_dev *dev, uint16_t idx, uint16_t desc, unsigned int socket, const struct rte_eth_txconf *conf) { struct priv *priv = dev->data->dev_private; + struct mlx4dv_obj mlxdv; + struct mlx4dv_qp dv_qp; + struct mlx4dv_cq dv_cq; + struct txq_elt (*elts)[rte_align32pow2(desc)]; struct ibv_qp_init_attr qp_init_attr; struct txq *txq; + uint8_t *bounce_buf; + struct mlx4_malloc_vec vec[] = { + { + .align = RTE_CACHE_LINE_SIZE, + .size = sizeof(*txq), + .addr = (void **)&txq, + }, + { + .align = RTE_CACHE_LINE_SIZE, + .size = sizeof(*elts), + .addr = (void **)&elts, + }, + { + .align = RTE_CACHE_LINE_SIZE, + .size = MLX4_MAX_WQE_SIZE, + .addr = (void **)&bounce_buf, + }, + }; int ret; - (void)conf; /* Thresholds configuration (ignored). */ DEBUG("%p: configuring queue %u for %u descriptors", (void *)dev, idx, desc); + /* + * Don't verify port offloads for application which + * use the old API. + */ + if ((conf->txq_flags & ETH_TXQ_FLAGS_IGNORE) && + !mlx4_check_tx_queue_offloads(priv, conf->offloads)) { + rte_errno = ENOTSUP; + ERROR("%p: Tx queue offloads 0x%" PRIx64 " don't match port " + "offloads 0x%" PRIx64 " or supported offloads 0x%" PRIx64, + (void *)dev, conf->offloads, + dev->data->dev_conf.txmode.offloads, + mlx4_get_tx_port_offloads(priv)); + return -rte_errno; + } if (idx >= dev->data->nb_tx_queues) { rte_errno = EOVERFLOW; ERROR("%p: queue index out of range (%u >= %u)", @@ -260,20 +281,49 @@ mlx4_tx_queue_setup(struct rte_eth_dev *dev, uint16_t idx, uint16_t desc, ERROR("%p: invalid number of Tx descriptors", (void *)dev); return -rte_errno; } + if (desc != RTE_DIM(*elts)) { + desc = RTE_DIM(*elts); + WARN("%p: increased number of descriptors in Tx queue %u" + " to the next power of two (%u)", + (void *)dev, idx, desc); + } /* Allocate and initialize Tx queue. */ - txq = rte_calloc_socket("TXQ", 1, sizeof(*txq), 0, socket); + mlx4_zmallocv_socket("TXQ", vec, RTE_DIM(vec), socket); if (!txq) { - rte_errno = ENOMEM; ERROR("%p: unable to allocate queue index %u", (void *)dev, idx); return -rte_errno; } *txq = (struct txq){ .priv = priv, - .stats.idx = idx, + .stats = { + .idx = idx, + }, .socket = socket, + .elts_n = desc, + .elts = elts, + .elts_head = 0, + .elts_tail = 0, + /* + * Request send completion every MLX4_PMD_TX_PER_COMP_REQ + * packets or at least 4 times per ring. + */ + .elts_comp_cd = + RTE_MIN(MLX4_PMD_TX_PER_COMP_REQ, desc / 4), + .elts_comp_cd_init = + RTE_MIN(MLX4_PMD_TX_PER_COMP_REQ, desc / 4), + .csum = priv->hw_csum && + (conf->offloads & (DEV_TX_OFFLOAD_IPV4_CKSUM | + DEV_TX_OFFLOAD_UDP_CKSUM | + DEV_TX_OFFLOAD_TCP_CKSUM)), + .csum_l2tun = priv->hw_csum_l2tun && + (conf->offloads & + DEV_TX_OFFLOAD_OUTER_IPV4_CKSUM), + /* Enable Tx loopback for VF devices. */ + .lb = !!priv->vf, + .bounce_buf = bounce_buf, }; - txq->cq = ibv_create_cq(priv->ctx, desc, NULL, NULL, 0); + txq->cq = mlx4_glue->create_cq(priv->ctx, desc, NULL, NULL, 0); if (!txq->cq) { rte_errno = ENOMEM; ERROR("%p: CQ creation failure: %s", @@ -293,7 +343,7 @@ mlx4_tx_queue_setup(struct rte_eth_dev *dev, uint16_t idx, uint16_t desc, /* No completion events must occur by default. */ .sq_sig_all = 0, }; - txq->qp = ibv_create_qp(priv->pd, &qp_init_attr); + txq->qp = mlx4_glue->create_qp(priv->pd, &qp_init_attr); if (!txq->qp) { rte_errno = errno ? errno : EINVAL; ERROR("%p: QP creation failure: %s", @@ -301,7 +351,7 @@ mlx4_tx_queue_setup(struct rte_eth_dev *dev, uint16_t idx, uint16_t desc, goto error; } txq->max_inline = qp_init_attr.cap.max_inline_data; - ret = ibv_modify_qp + ret = mlx4_glue->modify_qp (txq->qp, &(struct ibv_qp_attr){ .qp_state = IBV_QPS_INIT, @@ -314,13 +364,7 @@ mlx4_tx_queue_setup(struct rte_eth_dev *dev, uint16_t idx, uint16_t desc, (void *)dev, strerror(rte_errno)); goto error; } - ret = mlx4_txq_alloc_elts(txq, desc); - if (ret) { - ERROR("%p: TXQ allocation failed: %s", - (void *)dev, strerror(rte_errno)); - goto error; - } - ret = ibv_modify_qp + ret = mlx4_glue->modify_qp (txq->qp, &(struct ibv_qp_attr){ .qp_state = IBV_QPS_RTR, @@ -332,7 +376,7 @@ mlx4_tx_queue_setup(struct rte_eth_dev *dev, uint16_t idx, uint16_t desc, (void *)dev, strerror(rte_errno)); goto error; } - ret = ibv_modify_qp + ret = mlx4_glue->modify_qp (txq->qp, &(struct ibv_qp_attr){ .qp_state = IBV_QPS_RTS, @@ -344,6 +388,22 @@ mlx4_tx_queue_setup(struct rte_eth_dev *dev, uint16_t idx, uint16_t desc, (void *)dev, strerror(rte_errno)); goto error; } + /* Retrieve device queue information. */ + mlxdv.cq.in = txq->cq; + mlxdv.cq.out = &dv_cq; + mlxdv.qp.in = txq->qp; + mlxdv.qp.out = &dv_qp; + ret = mlx4_glue->dv_init_obj(&mlxdv, MLX4DV_OBJ_QP | MLX4DV_OBJ_CQ); + if (ret) { + rte_errno = EINVAL; + ERROR("%p: failed to obtain information needed for" + " accessing the device queues", (void *)dev); + goto error; + } + mlx4_txq_fill_dv_obj_info(txq, &mlxdv); + /* Save first wqe pointer in the first element. */ + (&(*txq->elts)[0])->wqe = + (volatile struct mlx4_wqe_ctrl_seg *)txq->msq.buf; /* Pre-register known mempools. */ rte_mempool_walk(mlx4_txq_mp2mr_iter, txq); DEBUG("%p: adding Tx queue %p to list", (void *)dev, (void *)txq); @@ -383,14 +443,14 @@ mlx4_tx_queue_release(void *dpdk_txq) } mlx4_txq_free_elts(txq); if (txq->qp) - claim_zero(ibv_destroy_qp(txq->qp)); + claim_zero(mlx4_glue->destroy_qp(txq->qp)); if (txq->cq) - claim_zero(ibv_destroy_cq(txq->cq)); + claim_zero(mlx4_glue->destroy_cq(txq->cq)); for (i = 0; i != RTE_DIM(txq->mp2mr); ++i) { if (!txq->mp2mr[i].mp) break; assert(txq->mp2mr[i].mr); - claim_zero(ibv_dereg_mr(txq->mp2mr[i].mr)); + mlx4_mr_put(txq->mp2mr[i].mr); } rte_free(txq); }