X-Git-Url: http://git.droids-corp.org/?a=blobdiff_plain;f=drivers%2Fnet%2Fmlx5%2Fmlx5_txq.c;h=fe3b4ec76d74cd28d2bcbeec06f715bbba843251;hb=5a93e173b874e549f0adac9796f7284837a98566;hp=51210f2d3b3a01fb961ddadc98cc4453d6e9e26f;hpb=36271e74466cd307ebe13ef30fdb275c395d7012;p=dpdk.git diff --git a/drivers/net/mlx5/mlx5_txq.c b/drivers/net/mlx5/mlx5_txq.c index 51210f2d3b..fe3b4ec76d 100644 --- a/drivers/net/mlx5/mlx5_txq.c +++ b/drivers/net/mlx5/mlx5_txq.c @@ -1,34 +1,6 @@ -/*- - * BSD LICENSE - * - * Copyright 2015 6WIND S.A. - * Copyright 2015 Mellanox. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * - * * Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in - * the documentation and/or other materials provided with the - * distribution. - * * Neither the name of 6WIND S.A. nor the names of its - * contributors may be used to endorse or promote products derived - * from this software without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright 2015 6WIND S.A. + * Copyright 2015 Mellanox Technologies, Ltd */ #include @@ -36,84 +8,51 @@ #include #include #include +#include +#include +#include /* Verbs header. */ /* ISO C doesn't support unnamed structs/unions, disabling -pedantic. */ #ifdef PEDANTIC -#pragma GCC diagnostic ignored "-pedantic" +#pragma GCC diagnostic ignored "-Wpedantic" #endif #include #ifdef PEDANTIC -#pragma GCC diagnostic error "-pedantic" +#pragma GCC diagnostic error "-Wpedantic" #endif -/* DPDK headers don't like -pedantic. */ -#ifdef PEDANTIC -#pragma GCC diagnostic ignored "-pedantic" -#endif #include #include -#include +#include #include -#ifdef PEDANTIC -#pragma GCC diagnostic error "-pedantic" -#endif #include "mlx5_utils.h" +#include "mlx5_defs.h" #include "mlx5.h" #include "mlx5_rxtx.h" #include "mlx5_autoconf.h" -#include "mlx5_defs.h" +#include "mlx5_glue.h" /** * Allocate TX queue elements. * * @param txq_ctrl * Pointer to TX queue structure. - * @param elts_n - * Number of elements to allocate. - * - * @return - * 0 on success, errno value on failure. */ -static int -txq_alloc_elts(struct txq_ctrl *txq_ctrl, unsigned int elts_n) +void +txq_alloc_elts(struct mlx5_txq_ctrl *txq_ctrl) { + const unsigned int elts_n = 1 << txq_ctrl->txq.elts_n; unsigned int i; - struct txq_elt (*elts)[elts_n] = - rte_calloc_socket("TXQ", 1, sizeof(*elts), 0, txq_ctrl->socket); - int ret = 0; - - if (elts == NULL) { - ERROR("%p: can't allocate packets array", (void *)txq_ctrl); - ret = ENOMEM; - goto error; - } - for (i = 0; (i != elts_n); ++i) { - struct txq_elt *elt = &(*elts)[i]; - elt->buf = NULL; - } - DEBUG("%p: allocated and configured %u WRs", (void *)txq_ctrl, elts_n); - txq_ctrl->txq.elts_n = elts_n; - txq_ctrl->txq.elts = elts; + for (i = 0; (i != elts_n); ++i) + txq_ctrl->txq.elts[i] = NULL; + DRV_LOG(DEBUG, "port %u Tx queue %u allocated and configured %u WRs", + PORT_ID(txq_ctrl->priv), txq_ctrl->txq.idx, elts_n); txq_ctrl->txq.elts_head = 0; txq_ctrl->txq.elts_tail = 0; txq_ctrl->txq.elts_comp = 0; - /* Request send completion every MLX5_PMD_TX_PER_COMP_REQ packets or - * at least 4 times per ring. */ - txq_ctrl->txq.elts_comp_cd_init = - ((MLX5_PMD_TX_PER_COMP_REQ < (elts_n / 4)) ? - MLX5_PMD_TX_PER_COMP_REQ : (elts_n / 4)); - txq_ctrl->txq.elts_comp_cd = txq_ctrl->txq.elts_comp_cd_init; - assert(ret == 0); - return 0; -error: - rte_free(elts); - - DEBUG("%p: failed, freed everything", (void *)txq_ctrl); - assert(ret > 0); - return ret; } /** @@ -123,110 +62,86 @@ error: * Pointer to TX queue structure. */ static void -txq_free_elts(struct txq_ctrl *txq_ctrl) +txq_free_elts(struct mlx5_txq_ctrl *txq_ctrl) { - unsigned int elts_n = txq_ctrl->txq.elts_n; - unsigned int elts_head = txq_ctrl->txq.elts_head; - unsigned int elts_tail = txq_ctrl->txq.elts_tail; - struct txq_elt (*elts)[elts_n] = txq_ctrl->txq.elts; + const uint16_t elts_n = 1 << txq_ctrl->txq.elts_n; + const uint16_t elts_m = elts_n - 1; + uint16_t elts_head = txq_ctrl->txq.elts_head; + uint16_t elts_tail = txq_ctrl->txq.elts_tail; + struct rte_mbuf *(*elts)[elts_n] = &txq_ctrl->txq.elts; - DEBUG("%p: freeing WRs", (void *)txq_ctrl); - txq_ctrl->txq.elts_n = 0; + DRV_LOG(DEBUG, "port %u Tx queue %u freeing WRs", + PORT_ID(txq_ctrl->priv), txq_ctrl->txq.idx); txq_ctrl->txq.elts_head = 0; txq_ctrl->txq.elts_tail = 0; txq_ctrl->txq.elts_comp = 0; - txq_ctrl->txq.elts_comp_cd = 0; - txq_ctrl->txq.elts_comp_cd_init = 0; - txq_ctrl->txq.elts = NULL; - if (elts == NULL) - return; while (elts_tail != elts_head) { - struct txq_elt *elt = &(*elts)[elts_tail]; + struct rte_mbuf *elt = (*elts)[elts_tail & elts_m]; - assert(elt->buf != NULL); - rte_pktmbuf_free(elt->buf); + assert(elt != NULL); + rte_pktmbuf_free_seg(elt); #ifndef NDEBUG /* Poisoning. */ - memset(elt, 0x77, sizeof(*elt)); + memset(&(*elts)[elts_tail & elts_m], + 0x77, + sizeof((*elts)[elts_tail & elts_m])); #endif - if (++elts_tail == elts_n) - elts_tail = 0; + ++elts_tail; } - rte_free(elts); } /** - * Clean up a TX queue. + * Returns the per-port supported offloads. * - * Destroy objects, free allocated memory and reset the structure for reuse. + * @param dev + * Pointer to Ethernet device. * - * @param txq_ctrl - * Pointer to TX queue structure. + * @return + * Supported Tx offloads. */ -void -txq_cleanup(struct txq_ctrl *txq_ctrl) +uint64_t +mlx5_get_tx_port_offloads(struct rte_eth_dev *dev) { - struct ibv_exp_release_intf_params params; - size_t i; - - DEBUG("cleaning up %p", (void *)txq_ctrl); - txq_free_elts(txq_ctrl); - txq_ctrl->txq.poll_cnt = NULL; - txq_ctrl->txq.send_flush = NULL; - if (txq_ctrl->if_qp != NULL) { - assert(txq_ctrl->txq.priv != NULL); - assert(txq_ctrl->txq.priv->ctx != NULL); - assert(txq_ctrl->txq.qp != NULL); - params = (struct ibv_exp_release_intf_params){ - .comp_mask = 0, - }; - claim_zero(ibv_exp_release_intf(txq_ctrl->txq.priv->ctx, - txq_ctrl->if_qp, - ¶ms)); - } - if (txq_ctrl->if_cq != NULL) { - assert(txq_ctrl->txq.priv != NULL); - assert(txq_ctrl->txq.priv->ctx != NULL); - assert(txq_ctrl->txq.cq != NULL); - params = (struct ibv_exp_release_intf_params){ - .comp_mask = 0, - }; - claim_zero(ibv_exp_release_intf(txq_ctrl->txq.priv->ctx, - txq_ctrl->if_cq, - ¶ms)); - } - if (txq_ctrl->txq.qp != NULL) - claim_zero(ibv_destroy_qp(txq_ctrl->txq.qp)); - if (txq_ctrl->txq.cq != NULL) - claim_zero(ibv_destroy_cq(txq_ctrl->txq.cq)); - if (txq_ctrl->rd != NULL) { - struct ibv_exp_destroy_res_domain_attr attr = { - .comp_mask = 0, - }; - - assert(txq_ctrl->txq.priv != NULL); - assert(txq_ctrl->txq.priv->ctx != NULL); - claim_zero(ibv_exp_destroy_res_domain(txq_ctrl->txq.priv->ctx, - txq_ctrl->rd, - &attr)); - } - for (i = 0; (i != RTE_DIM(txq_ctrl->txq.mp2mr)); ++i) { - if (txq_ctrl->txq.mp2mr[i].mp == NULL) - break; - assert(txq_ctrl->txq.mp2mr[i].mr != NULL); - claim_zero(ibv_dereg_mr(txq_ctrl->txq.mp2mr[i].mr)); + struct mlx5_priv *priv = dev->data->dev_private; + uint64_t offloads = (DEV_TX_OFFLOAD_MULTI_SEGS | + DEV_TX_OFFLOAD_VLAN_INSERT); + struct mlx5_dev_config *config = &priv->config; + + if (config->hw_csum) + offloads |= (DEV_TX_OFFLOAD_IPV4_CKSUM | + DEV_TX_OFFLOAD_UDP_CKSUM | + DEV_TX_OFFLOAD_TCP_CKSUM); + if (config->tso) + offloads |= DEV_TX_OFFLOAD_TCP_TSO; + if (config->swp) { + if (config->hw_csum) + offloads |= DEV_TX_OFFLOAD_OUTER_IPV4_CKSUM; + if (config->tso) + offloads |= (DEV_TX_OFFLOAD_IP_TNL_TSO | + DEV_TX_OFFLOAD_UDP_TNL_TSO); } - memset(txq_ctrl, 0, sizeof(*txq_ctrl)); + if (config->tunnel_en) { + if (config->hw_csum) + offloads |= DEV_TX_OFFLOAD_OUTER_IPV4_CKSUM; + if (config->tso) + offloads |= (DEV_TX_OFFLOAD_VXLAN_TNL_TSO | + DEV_TX_OFFLOAD_GRE_TNL_TSO); + } +#ifdef HAVE_IBV_FLOW_DV_SUPPORT + if (config->dv_flow_en) + offloads |= DEV_TX_OFFLOAD_MATCH_METADATA; +#endif + return offloads; } /** - * Configure a TX queue. + * DPDK callback to configure a TX queue. * * @param dev * Pointer to Ethernet device structure. - * @param txq_ctrl - * Pointer to TX queue structure. + * @param idx + * TX queue index. * @param desc * Number of descriptors to configure in queue. * @param socket @@ -235,186 +150,756 @@ txq_cleanup(struct txq_ctrl *txq_ctrl) * Thresholds parameters. * * @return - * 0 on success, errno value on failure. + * 0 on success, a negative errno value otherwise and rte_errno is set. */ int -txq_setup(struct rte_eth_dev *dev, struct txq_ctrl *txq_ctrl, uint16_t desc, - unsigned int socket, const struct rte_eth_txconf *conf) +mlx5_tx_queue_setup(struct rte_eth_dev *dev, uint16_t idx, uint16_t desc, + unsigned int socket, const struct rte_eth_txconf *conf) { - struct priv *priv = mlx5_get_priv(dev); - struct txq_ctrl tmpl = { - .socket = socket, - .txq = { - .priv = priv, - }, - }; + struct mlx5_priv *priv = dev->data->dev_private; + struct mlx5_txq_data *txq = (*priv->txqs)[idx]; + struct mlx5_txq_ctrl *txq_ctrl = + container_of(txq, struct mlx5_txq_ctrl, txq); + + if (desc <= MLX5_TX_COMP_THRESH) { + DRV_LOG(WARNING, + "port %u number of descriptors requested for Tx queue" + " %u must be higher than MLX5_TX_COMP_THRESH, using %u" + " instead of %u", + dev->data->port_id, idx, MLX5_TX_COMP_THRESH + 1, desc); + desc = MLX5_TX_COMP_THRESH + 1; + } + if (!rte_is_power_of_2(desc)) { + desc = 1 << log2above(desc); + DRV_LOG(WARNING, + "port %u increased number of descriptors in Tx queue" + " %u to the next power of two (%d)", + dev->data->port_id, idx, desc); + } + DRV_LOG(DEBUG, "port %u configuring queue %u for %u descriptors", + dev->data->port_id, idx, desc); + if (idx >= priv->txqs_n) { + DRV_LOG(ERR, "port %u Tx queue index out of range (%u >= %u)", + dev->data->port_id, idx, priv->txqs_n); + rte_errno = EOVERFLOW; + return -rte_errno; + } + if (!mlx5_txq_releasable(dev, idx)) { + rte_errno = EBUSY; + DRV_LOG(ERR, "port %u unable to release queue index %u", + dev->data->port_id, idx); + return -rte_errno; + } + mlx5_txq_release(dev, idx); + txq_ctrl = mlx5_txq_new(dev, idx, desc, socket, conf); + if (!txq_ctrl) { + DRV_LOG(ERR, "port %u unable to allocate queue index %u", + dev->data->port_id, idx); + return -rte_errno; + } + DRV_LOG(DEBUG, "port %u adding Tx queue %u to list", + dev->data->port_id, idx); + (*priv->txqs)[idx] = &txq_ctrl->txq; + return 0; +} + +/** + * DPDK callback to release a TX queue. + * + * @param dpdk_txq + * Generic TX queue pointer. + */ +void +mlx5_tx_queue_release(void *dpdk_txq) +{ + struct mlx5_txq_data *txq = (struct mlx5_txq_data *)dpdk_txq; + struct mlx5_txq_ctrl *txq_ctrl; + struct mlx5_priv *priv; + unsigned int i; + + if (txq == NULL) + return; + txq_ctrl = container_of(txq, struct mlx5_txq_ctrl, txq); + priv = txq_ctrl->priv; + for (i = 0; (i != priv->txqs_n); ++i) + if ((*priv->txqs)[i] == txq) { + mlx5_txq_release(ETH_DEV(priv), i); + DRV_LOG(DEBUG, "port %u removing Tx queue %u from list", + PORT_ID(priv), txq->idx); + break; + } +} + +/** + * Initialize Tx UAR registers for primary process. + * + * @param txq_ctrl + * Pointer to Tx queue control structure. + */ +static void +txq_uar_init(struct mlx5_txq_ctrl *txq_ctrl) +{ + struct mlx5_priv *priv = txq_ctrl->priv; + struct mlx5_proc_priv *ppriv = MLX5_PROC_PRIV(PORT_ID(priv)); +#ifndef RTE_ARCH_64 + unsigned int lock_idx; + const size_t page_size = sysconf(_SC_PAGESIZE); +#endif + + assert(rte_eal_process_type() == RTE_PROC_PRIMARY); + assert(ppriv); + ppriv->uar_table[txq_ctrl->txq.idx] = txq_ctrl->bf_reg; +#ifndef RTE_ARCH_64 + /* Assign an UAR lock according to UAR page number */ + lock_idx = (txq_ctrl->uar_mmap_offset / page_size) & + MLX5_UAR_PAGE_NUM_MASK; + txq_ctrl->txq.uar_lock = &priv->uar_lock[lock_idx]; +#endif +} + +/** + * Remap UAR register of a Tx queue for secondary process. + * + * Remapped address is stored at the table in the process private structure of + * the device, indexed by queue index. + * + * @param txq_ctrl + * Pointer to Tx queue control structure. + * @param fd + * Verbs file descriptor to map UAR pages. + * + * @return + * 0 on success, a negative errno value otherwise and rte_errno is set. + */ +static int +txq_uar_init_secondary(struct mlx5_txq_ctrl *txq_ctrl, int fd) +{ + struct mlx5_priv *priv = txq_ctrl->priv; + struct mlx5_proc_priv *ppriv = MLX5_PROC_PRIV(PORT_ID(priv)); + struct mlx5_txq_data *txq = &txq_ctrl->txq; + void *addr; + uintptr_t uar_va; + uintptr_t offset; + const size_t page_size = sysconf(_SC_PAGESIZE); + + assert(ppriv); + /* + * As rdma-core, UARs are mapped in size of OS page + * size. Ref to libmlx5 function: mlx5_init_context() + */ + uar_va = (uintptr_t)txq_ctrl->bf_reg; + offset = uar_va & (page_size - 1); /* Offset in page. */ + addr = mmap(NULL, page_size, PROT_WRITE, MAP_SHARED, fd, + txq_ctrl->uar_mmap_offset); + if (addr == MAP_FAILED) { + DRV_LOG(ERR, + "port %u mmap failed for BF reg of txq %u", + txq->port_id, txq->idx); + rte_errno = ENXIO; + return -rte_errno; + } + addr = RTE_PTR_ADD(addr, offset); + ppriv->uar_table[txq->idx] = addr; + return 0; +} + +/** + * Unmap UAR register of a Tx queue for secondary process. + * + * @param txq_ctrl + * Pointer to Tx queue control structure. + */ +static void +txq_uar_uninit_secondary(struct mlx5_txq_ctrl *txq_ctrl) +{ + struct mlx5_proc_priv *ppriv = MLX5_PROC_PRIV(PORT_ID(txq_ctrl->priv)); + const size_t page_size = sysconf(_SC_PAGESIZE); + void *addr; + + addr = ppriv->uar_table[txq_ctrl->txq.idx]; + munmap(RTE_PTR_ALIGN_FLOOR(addr, page_size), page_size); +} + +/** + * Initialize Tx UAR registers for secondary process. + * + * @param dev + * Pointer to Ethernet device. + * @param fd + * Verbs file descriptor to map UAR pages. + * + * @return + * 0 on success, a negative errno value otherwise and rte_errno is set. + */ +int +mlx5_tx_uar_init_secondary(struct rte_eth_dev *dev, int fd) +{ + struct mlx5_priv *priv = dev->data->dev_private; + struct mlx5_txq_data *txq; + struct mlx5_txq_ctrl *txq_ctrl; + unsigned int i; + int ret; + + assert(rte_eal_process_type() == RTE_PROC_SECONDARY); + for (i = 0; i != priv->txqs_n; ++i) { + if (!(*priv->txqs)[i]) + continue; + txq = (*priv->txqs)[i]; + txq_ctrl = container_of(txq, struct mlx5_txq_ctrl, txq); + assert(txq->idx == (uint16_t)i); + ret = txq_uar_init_secondary(txq_ctrl, fd); + if (ret) + goto error; + } + return 0; +error: + /* Rollback. */ + do { + if (!(*priv->txqs)[i]) + continue; + txq = (*priv->txqs)[i]; + txq_ctrl = container_of(txq, struct mlx5_txq_ctrl, txq); + txq_uar_uninit_secondary(txq_ctrl); + } while (i--); + return -rte_errno; +} + +/** + * Create the Tx queue Verbs object. + * + * @param dev + * Pointer to Ethernet device. + * @param idx + * Queue index in DPDK Tx queue array. + * + * @return + * The Verbs object initialised, NULL otherwise and rte_errno is set. + */ +struct mlx5_txq_ibv * +mlx5_txq_ibv_new(struct rte_eth_dev *dev, uint16_t idx) +{ + struct mlx5_priv *priv = dev->data->dev_private; + struct mlx5_txq_data *txq_data = (*priv->txqs)[idx]; + struct mlx5_txq_ctrl *txq_ctrl = + container_of(txq_data, struct mlx5_txq_ctrl, txq); + struct mlx5_txq_ibv tmpl; + struct mlx5_txq_ibv *txq_ibv = NULL; union { - struct ibv_exp_query_intf_params params; - struct ibv_exp_qp_init_attr init; - struct ibv_exp_res_domain_init_attr rd; - struct ibv_exp_cq_init_attr cq; - struct ibv_exp_qp_attr mod; + struct ibv_qp_init_attr_ex init; + struct ibv_cq_init_attr_ex cq; + struct ibv_qp_attr mod; } attr; - enum ibv_exp_query_intf_status status; + unsigned int cqe_n; + struct mlx5dv_qp qp = { .comp_mask = MLX5DV_QP_MASK_UAR_MMAP_OFFSET }; + struct mlx5dv_cq cq_info; + struct mlx5dv_obj obj; + const int desc = 1 << txq_data->elts_n; int ret = 0; - (void)conf; /* Thresholds configuration (ignored). */ - if (desc == 0) { - ERROR("%p: invalid number of TX descriptors", (void *)dev); - return EINVAL; - } - /* MRs will be registered in mp2mr[] later. */ - attr.rd = (struct ibv_exp_res_domain_init_attr){ - .comp_mask = (IBV_EXP_RES_DOMAIN_THREAD_MODEL | - IBV_EXP_RES_DOMAIN_MSG_MODEL), - .thread_model = IBV_EXP_THREAD_SINGLE, - .msg_model = IBV_EXP_MSG_HIGH_BW, - }; - tmpl.rd = ibv_exp_create_res_domain(priv->ctx, &attr.rd); - if (tmpl.rd == NULL) { - ret = ENOMEM; - ERROR("%p: RD creation failure: %s", - (void *)dev, strerror(ret)); - goto error; +#ifdef HAVE_IBV_FLOW_DV_SUPPORT + /* If using DevX, need additional mask to read tisn value. */ + if (priv->config.devx && !priv->sh->tdn) + qp.comp_mask |= MLX5DV_QP_MASK_RAW_QP_HANDLES; +#endif + assert(txq_data); + priv->verbs_alloc_ctx.type = MLX5_VERBS_ALLOC_TYPE_TX_QUEUE; + priv->verbs_alloc_ctx.obj = txq_ctrl; + if (mlx5_getenv_int("MLX5_ENABLE_CQE_COMPRESSION")) { + DRV_LOG(ERR, + "port %u MLX5_ENABLE_CQE_COMPRESSION must never be set", + dev->data->port_id); + rte_errno = EINVAL; + return NULL; } - attr.cq = (struct ibv_exp_cq_init_attr){ - .comp_mask = IBV_EXP_CQ_INIT_ATTR_RES_DOMAIN, - .res_domain = tmpl.rd, + memset(&tmpl, 0, sizeof(struct mlx5_txq_ibv)); + attr.cq = (struct ibv_cq_init_attr_ex){ + .comp_mask = 0, }; - tmpl.txq.cq = ibv_exp_create_cq(priv->ctx, desc, NULL, NULL, 0, - &attr.cq); - if (tmpl.txq.cq == NULL) { - ret = ENOMEM; - ERROR("%p: CQ creation failure: %s", - (void *)dev, strerror(ret)); + cqe_n = desc / MLX5_TX_COMP_THRESH + + 1 + MLX5_TX_COMP_THRESH_INLINE_DIV; + tmpl.cq = mlx5_glue->create_cq(priv->sh->ctx, cqe_n, NULL, NULL, 0); + if (tmpl.cq == NULL) { + DRV_LOG(ERR, "port %u Tx queue %u CQ creation failure", + dev->data->port_id, idx); + rte_errno = errno; goto error; } - DEBUG("priv->device_attr.max_qp_wr is %d", - priv->device_attr.max_qp_wr); - DEBUG("priv->device_attr.max_sge is %d", - priv->device_attr.max_sge); - attr.init = (struct ibv_exp_qp_init_attr){ + attr.init = (struct ibv_qp_init_attr_ex){ /* CQ to be associated with the send queue. */ - .send_cq = tmpl.txq.cq, + .send_cq = tmpl.cq, /* CQ to be associated with the receive queue. */ - .recv_cq = tmpl.txq.cq, + .recv_cq = tmpl.cq, .cap = { /* Max number of outstanding WRs. */ - .max_send_wr = ((priv->device_attr.max_qp_wr < desc) ? - priv->device_attr.max_qp_wr : - desc), - /* Max number of scatter/gather elements in a WR. */ + .max_send_wr = + ((priv->sh->device_attr.orig_attr.max_qp_wr < + desc) ? + priv->sh->device_attr.orig_attr.max_qp_wr : + desc), + /* + * Max number of scatter/gather elements in a WR, + * must be 1 to prevent libmlx5 from trying to affect + * too much memory. TX gather is not impacted by the + * device_attr.max_sge limit and will still work + * properly. + */ .max_send_sge = 1, }, .qp_type = IBV_QPT_RAW_PACKET, - /* Do *NOT* enable this, completions events are managed per - * TX burst. */ + /* + * Do *NOT* enable this, completions events are managed per + * Tx burst. + */ .sq_sig_all = 0, - .pd = priv->pd, - .res_domain = tmpl.rd, - .comp_mask = (IBV_EXP_QP_INIT_ATTR_PD | - IBV_EXP_QP_INIT_ATTR_RES_DOMAIN), + .pd = priv->sh->pd, + .comp_mask = IBV_QP_INIT_ATTR_PD, }; - tmpl.txq.qp = ibv_exp_create_qp(priv->ctx, &attr.init); - if (tmpl.txq.qp == NULL) { - ret = (errno ? errno : EINVAL); - ERROR("%p: QP creation failure: %s", - (void *)dev, strerror(ret)); + if (txq_data->inlen_send) + attr.init.cap.max_inline_data = txq_ctrl->max_inline_data; + if (txq_data->tso_en) { + attr.init.max_tso_header = txq_ctrl->max_tso_header; + attr.init.comp_mask |= IBV_QP_INIT_ATTR_MAX_TSO_HEADER; + } + tmpl.qp = mlx5_glue->create_qp_ex(priv->sh->ctx, &attr.init); + if (tmpl.qp == NULL) { + DRV_LOG(ERR, "port %u Tx queue %u QP creation failure", + dev->data->port_id, idx); + rte_errno = errno; goto error; } - attr.mod = (struct ibv_exp_qp_attr){ + attr.mod = (struct ibv_qp_attr){ /* Move the QP to this state. */ .qp_state = IBV_QPS_INIT, - /* Primary port number. */ - .port_num = priv->port + /* IB device port number. */ + .port_num = (uint8_t)priv->ibv_port, }; - ret = ibv_exp_modify_qp(tmpl.txq.qp, &attr.mod, - (IBV_EXP_QP_STATE | IBV_EXP_QP_PORT)); + ret = mlx5_glue->modify_qp(tmpl.qp, &attr.mod, + (IBV_QP_STATE | IBV_QP_PORT)); if (ret) { - ERROR("%p: QP state to IBV_QPS_INIT failed: %s", - (void *)dev, strerror(ret)); + DRV_LOG(ERR, + "port %u Tx queue %u QP state to IBV_QPS_INIT failed", + dev->data->port_id, idx); + rte_errno = errno; goto error; } - ret = txq_alloc_elts(&tmpl, desc); - if (ret) { - ERROR("%p: TXQ allocation failed: %s", - (void *)dev, strerror(ret)); - goto error; - } - attr.mod = (struct ibv_exp_qp_attr){ + attr.mod = (struct ibv_qp_attr){ .qp_state = IBV_QPS_RTR }; - ret = ibv_exp_modify_qp(tmpl.txq.qp, &attr.mod, IBV_EXP_QP_STATE); + ret = mlx5_glue->modify_qp(tmpl.qp, &attr.mod, IBV_QP_STATE); if (ret) { - ERROR("%p: QP state to IBV_QPS_RTR failed: %s", - (void *)dev, strerror(ret)); + DRV_LOG(ERR, + "port %u Tx queue %u QP state to IBV_QPS_RTR failed", + dev->data->port_id, idx); + rte_errno = errno; goto error; } attr.mod.qp_state = IBV_QPS_RTS; - ret = ibv_exp_modify_qp(tmpl.txq.qp, &attr.mod, IBV_EXP_QP_STATE); + ret = mlx5_glue->modify_qp(tmpl.qp, &attr.mod, IBV_QP_STATE); if (ret) { - ERROR("%p: QP state to IBV_QPS_RTS failed: %s", - (void *)dev, strerror(ret)); + DRV_LOG(ERR, + "port %u Tx queue %u QP state to IBV_QPS_RTS failed", + dev->data->port_id, idx); + rte_errno = errno; goto error; } - attr.params = (struct ibv_exp_query_intf_params){ - .intf_scope = IBV_EXP_INTF_GLOBAL, - .intf = IBV_EXP_INTF_CQ, - .obj = tmpl.txq.cq, - }; - tmpl.if_cq = ibv_exp_query_intf(priv->ctx, &attr.params, &status); - if (tmpl.if_cq == NULL) { - ret = EINVAL; - ERROR("%p: CQ interface family query failed with status %d", - (void *)dev, status); + txq_ibv = rte_calloc_socket(__func__, 1, sizeof(struct mlx5_txq_ibv), 0, + txq_ctrl->socket); + if (!txq_ibv) { + DRV_LOG(ERR, "port %u Tx queue %u cannot allocate memory", + dev->data->port_id, idx); + rte_errno = ENOMEM; goto error; } - attr.params = (struct ibv_exp_query_intf_params){ - .intf_scope = IBV_EXP_INTF_GLOBAL, - .intf = IBV_EXP_INTF_QP_BURST, - .obj = tmpl.txq.qp, -#ifdef HAVE_VERBS_VLAN_INSERTION - .intf_version = 1, -#endif - /* Enable multi-packet send if supported. */ - .family_flags = - (priv->mps ? - IBV_EXP_QP_BURST_CREATE_ENABLE_MULTI_PACKET_SEND_WR : - 0), - }; - tmpl.if_qp = ibv_exp_query_intf(priv->ctx, &attr.params, &status); - if (tmpl.if_qp == NULL) { - ret = EINVAL; - ERROR("%p: QP interface family query failed with status %d", - (void *)dev, status); + obj.cq.in = tmpl.cq; + obj.cq.out = &cq_info; + obj.qp.in = tmpl.qp; + obj.qp.out = &qp; + ret = mlx5_glue->dv_init_obj(&obj, MLX5DV_OBJ_CQ | MLX5DV_OBJ_QP); + if (ret != 0) { + rte_errno = errno; goto error; } - /* Clean up txq in case we're reinitializing it. */ - DEBUG("%p: cleaning-up old txq just in case", (void *)txq_ctrl); - txq_cleanup(txq_ctrl); - *txq_ctrl = tmpl; - txq_ctrl->txq.poll_cnt = txq_ctrl->if_cq->poll_cnt; - txq_ctrl->txq.send_pending = txq_ctrl->if_qp->send_pending; -#ifdef HAVE_VERBS_VLAN_INSERTION - txq_ctrl->txq.send_pending_vlan = txq_ctrl->if_qp->send_pending_vlan; + if (cq_info.cqe_size != RTE_CACHE_LINE_SIZE) { + DRV_LOG(ERR, + "port %u wrong MLX5_CQE_SIZE environment variable" + " value: it should be set to %u", + dev->data->port_id, RTE_CACHE_LINE_SIZE); + rte_errno = EINVAL; + goto error; + } + txq_data->cqe_n = log2above(cq_info.cqe_cnt); + txq_data->cqe_s = 1 << txq_data->cqe_n; + txq_data->cqe_m = txq_data->cqe_s - 1; + txq_data->qp_num_8s = tmpl.qp->qp_num << 8; + txq_data->wqes = qp.sq.buf; + txq_data->wqe_n = log2above(qp.sq.wqe_cnt); + txq_data->wqe_s = 1 << txq_data->wqe_n; + txq_data->wqe_m = txq_data->wqe_s - 1; + txq_data->wqes_end = txq_data->wqes + txq_data->wqe_s; + txq_data->qp_db = &qp.dbrec[MLX5_SND_DBR]; + txq_data->cq_db = cq_info.dbrec; + txq_data->cqes = (volatile struct mlx5_cqe *)cq_info.buf; + txq_data->cq_ci = 0; +#ifndef NDEBUG + txq_data->cq_pi = 0; #endif - txq_ctrl->txq.send_flush = txq_ctrl->if_qp->send_flush; - DEBUG("%p: txq updated with %p", (void *)txq_ctrl, (void *)&tmpl); - /* Pre-register known mempools. */ - rte_mempool_walk(txq_mp2mr_iter, txq_ctrl); - assert(ret == 0); - return 0; + txq_data->wqe_ci = 0; + txq_data->wqe_pi = 0; + txq_data->wqe_comp = 0; + txq_data->wqe_thres = txq_data->wqe_s / MLX5_TX_COMP_THRESH_INLINE_DIV; +#ifdef HAVE_IBV_FLOW_DV_SUPPORT + /* + * If using DevX need to query and store TIS transport domain value. + * This is done once per port. + * Will use this value on Rx, when creating matching TIR. + */ + if (priv->config.devx && !priv->sh->tdn) { + ret = mlx5_devx_cmd_qp_query_tis_td(tmpl.qp, qp.tisn, + &priv->sh->tdn); + if (ret) { + DRV_LOG(ERR, "Fail to query port %u Tx queue %u QP TIS " + "transport domain", dev->data->port_id, idx); + rte_errno = EINVAL; + goto error; + } else { + DRV_LOG(DEBUG, "port %u Tx queue %u TIS number %d " + "transport domain %d", dev->data->port_id, + idx, qp.tisn, priv->sh->tdn); + } + } +#endif + txq_ibv->qp = tmpl.qp; + txq_ibv->cq = tmpl.cq; + rte_atomic32_inc(&txq_ibv->refcnt); + txq_ctrl->bf_reg = qp.bf.reg; + txq_uar_init(txq_ctrl); + if (qp.comp_mask & MLX5DV_QP_MASK_UAR_MMAP_OFFSET) { + txq_ctrl->uar_mmap_offset = qp.uar_mmap_offset; + DRV_LOG(DEBUG, "port %u: uar_mmap_offset 0x%"PRIx64, + dev->data->port_id, txq_ctrl->uar_mmap_offset); + } else { + DRV_LOG(ERR, + "port %u failed to retrieve UAR info, invalid" + " libmlx5.so", + dev->data->port_id); + rte_errno = EINVAL; + goto error; + } + LIST_INSERT_HEAD(&priv->txqsibv, txq_ibv, next); + txq_ibv->txq_ctrl = txq_ctrl; + priv->verbs_alloc_ctx.type = MLX5_VERBS_ALLOC_TYPE_NONE; + return txq_ibv; error: - txq_cleanup(&tmpl); - assert(ret > 0); + ret = rte_errno; /* Save rte_errno before cleanup. */ + if (tmpl.cq) + claim_zero(mlx5_glue->destroy_cq(tmpl.cq)); + if (tmpl.qp) + claim_zero(mlx5_glue->destroy_qp(tmpl.qp)); + if (txq_ibv) + rte_free(txq_ibv); + priv->verbs_alloc_ctx.type = MLX5_VERBS_ALLOC_TYPE_NONE; + rte_errno = ret; /* Restore rte_errno. */ + return NULL; +} + +/** + * Get an Tx queue Verbs object. + * + * @param dev + * Pointer to Ethernet device. + * @param idx + * Queue index in DPDK Tx queue array. + * + * @return + * The Verbs object if it exists. + */ +struct mlx5_txq_ibv * +mlx5_txq_ibv_get(struct rte_eth_dev *dev, uint16_t idx) +{ + struct mlx5_priv *priv = dev->data->dev_private; + struct mlx5_txq_ctrl *txq_ctrl; + + if (idx >= priv->txqs_n) + return NULL; + if (!(*priv->txqs)[idx]) + return NULL; + txq_ctrl = container_of((*priv->txqs)[idx], struct mlx5_txq_ctrl, txq); + if (txq_ctrl->ibv) + rte_atomic32_inc(&txq_ctrl->ibv->refcnt); + return txq_ctrl->ibv; +} + +/** + * Release an Tx verbs queue object. + * + * @param txq_ibv + * Verbs Tx queue object. + * + * @return + * 1 while a reference on it exists, 0 when freed. + */ +int +mlx5_txq_ibv_release(struct mlx5_txq_ibv *txq_ibv) +{ + assert(txq_ibv); + if (rte_atomic32_dec_and_test(&txq_ibv->refcnt)) { + claim_zero(mlx5_glue->destroy_qp(txq_ibv->qp)); + claim_zero(mlx5_glue->destroy_cq(txq_ibv->cq)); + LIST_REMOVE(txq_ibv, next); + rte_free(txq_ibv); + return 0; + } + return 1; +} + +/** + * Verify the Verbs Tx queue list is empty + * + * @param dev + * Pointer to Ethernet device. + * + * @return + * The number of object not released. + */ +int +mlx5_txq_ibv_verify(struct rte_eth_dev *dev) +{ + struct mlx5_priv *priv = dev->data->dev_private; + int ret = 0; + struct mlx5_txq_ibv *txq_ibv; + + LIST_FOREACH(txq_ibv, &priv->txqsibv, next) { + DRV_LOG(DEBUG, "port %u Verbs Tx queue %u still referenced", + dev->data->port_id, txq_ibv->txq_ctrl->txq.idx); + ++ret; + } return ret; } /** - * DPDK callback to configure a TX queue. + * Calculate the total number of WQEBB for Tx queue. + * + * Simplified version of calc_sq_size() in rdma-core. + * + * @param txq_ctrl + * Pointer to Tx queue control structure. + * + * @return + * The number of WQEBB. + */ +static int +txq_calc_wqebb_cnt(struct mlx5_txq_ctrl *txq_ctrl) +{ + unsigned int wqe_size; + const unsigned int desc = 1 << txq_ctrl->txq.elts_n; + + wqe_size = MLX5_WQE_CSEG_SIZE + + MLX5_WQE_ESEG_SIZE + + MLX5_WSEG_SIZE - + MLX5_ESEG_MIN_INLINE_SIZE + + txq_ctrl->max_inline_data; + return rte_align32pow2(wqe_size * desc) / MLX5_WQE_SIZE; +} + +/** + * Set Tx queue parameters from device configuration. + * + * @param txq_ctrl + * Pointer to Tx queue control structure. + */ +static void +txq_set_params(struct mlx5_txq_ctrl *txq_ctrl) +{ + struct mlx5_priv *priv = txq_ctrl->priv; + struct mlx5_dev_config *config = &priv->config; + unsigned int inlen_send; /* Inline data for ordinary SEND.*/ + unsigned int inlen_empw; /* Inline data for enhanced MPW. */ + unsigned int inlen_mode; /* Minimal required Inline data. */ + unsigned int txqs_inline; /* Min Tx queues to enable inline. */ + uint64_t dev_txoff = priv->dev_data->dev_conf.txmode.offloads; + bool tso = txq_ctrl->txq.offloads & (DEV_TX_OFFLOAD_TCP_TSO | + DEV_TX_OFFLOAD_VXLAN_TNL_TSO | + DEV_TX_OFFLOAD_GRE_TNL_TSO | + DEV_TX_OFFLOAD_IP_TNL_TSO | + DEV_TX_OFFLOAD_UDP_TNL_TSO); + bool vlan_inline; + unsigned int temp; + + if (config->txqs_inline == MLX5_ARG_UNSET) + txqs_inline = +#if defined(RTE_ARCH_ARM64) + (priv->sh->pci_dev->id.device_id == + PCI_DEVICE_ID_MELLANOX_CONNECTX5BF) ? + MLX5_INLINE_MAX_TXQS_BLUEFIELD : +#endif + MLX5_INLINE_MAX_TXQS; + else + txqs_inline = (unsigned int)config->txqs_inline; + inlen_send = (config->txq_inline_max == MLX5_ARG_UNSET) ? + MLX5_SEND_DEF_INLINE_LEN : + (unsigned int)config->txq_inline_max; + inlen_empw = (config->txq_inline_mpw == MLX5_ARG_UNSET) ? + MLX5_EMPW_DEF_INLINE_LEN : + (unsigned int)config->txq_inline_mpw; + inlen_mode = (config->txq_inline_min == MLX5_ARG_UNSET) ? + 0 : (unsigned int)config->txq_inline_min; + if (config->mps != MLX5_MPW_ENHANCED) + inlen_empw = 0; + /* + * If there is requested minimal amount of data to inline + * we MUST enable inlining. This is a case for ConnectX-4 + * which usually requires L2 inlined for correct operating + * and ConnectX-4LX which requires L2-L4 inlined to + * support E-Switch Flows. + */ + if (inlen_mode) { + if (inlen_mode <= MLX5_ESEG_MIN_INLINE_SIZE) { + /* + * Optimize minimal inlining for single + * segment packets to fill one WQEBB + * without gaps. + */ + temp = MLX5_ESEG_MIN_INLINE_SIZE; + } else { + temp = inlen_mode - MLX5_ESEG_MIN_INLINE_SIZE; + temp = RTE_ALIGN(temp, MLX5_WSEG_SIZE) + + MLX5_ESEG_MIN_INLINE_SIZE; + temp = RTE_MIN(temp, MLX5_SEND_MAX_INLINE_LEN); + } + if (temp != inlen_mode) { + DRV_LOG(INFO, + "port %u minimal required inline setting" + " aligned from %u to %u", + PORT_ID(priv), inlen_mode, temp); + inlen_mode = temp; + } + } + /* + * If port is configured to support VLAN insertion and device + * does not support this feature by HW (for NICs before ConnectX-5 + * or in case of wqe_vlan_insert flag is not set) we must enable + * data inline on all queues because it is supported by single + * tx_burst routine. + */ + txq_ctrl->txq.vlan_en = config->hw_vlan_insert; + vlan_inline = (dev_txoff & DEV_TX_OFFLOAD_VLAN_INSERT) && + !config->hw_vlan_insert; + if (vlan_inline) + inlen_send = RTE_MAX(inlen_send, MLX5_ESEG_MIN_INLINE_SIZE); + /* + * If there are few Tx queues it is prioritized + * to save CPU cycles and disable data inlining at all. + */ + if ((inlen_send && priv->txqs_n >= txqs_inline) || vlan_inline) { + /* + * The data sent with ordinal MLX5_OPCODE_SEND + * may be inlined in Ethernet Segment, align the + * length accordingly to fit entire WQEBBs. + */ + temp = (inlen_send / MLX5_WQE_SIZE) * MLX5_WQE_SIZE + + MLX5_ESEG_MIN_INLINE_SIZE + MLX5_WQE_DSEG_SIZE; + temp = RTE_MIN(temp, MLX5_WQE_SIZE_MAX + + MLX5_ESEG_MIN_INLINE_SIZE - + MLX5_WQE_CSEG_SIZE - + MLX5_WQE_ESEG_SIZE - + MLX5_WQE_DSEG_SIZE * 2); + temp = RTE_MIN(temp, MLX5_SEND_MAX_INLINE_LEN); + temp = RTE_MAX(temp, inlen_mode); + if (temp != inlen_send) { + DRV_LOG(INFO, + "port %u ordinary send inline setting" + " aligned from %u to %u", + PORT_ID(priv), inlen_send, temp); + inlen_send = temp; + } + /* + * Not aligned to cache lines, but to WQEs. + * First bytes of data (initial alignment) + * is going to be copied explicitly at the + * beginning of inlining buffer in Ethernet + * Segment. + */ + assert(inlen_send >= MLX5_ESEG_MIN_INLINE_SIZE); + assert(inlen_send <= MLX5_WQE_SIZE_MAX + + MLX5_ESEG_MIN_INLINE_SIZE - + MLX5_WQE_CSEG_SIZE - + MLX5_WQE_ESEG_SIZE - + MLX5_WQE_DSEG_SIZE * 2); + txq_ctrl->txq.inlen_send = inlen_send; + txq_ctrl->txq.inlen_mode = inlen_mode; + txq_ctrl->txq.inlen_empw = 0; + } else { + /* + * If minimal inlining is requested we must + * enable inlining in general, despite the + * number of configured queues. + */ + inlen_send = inlen_mode; + if (inlen_mode) { + /* + * Extend space for inline data to allow + * optional alignment of data buffer + * start address, it may improve PCIe + * performance. + */ + inlen_send = RTE_MIN(inlen_send + MLX5_WQE_SIZE, + MLX5_SEND_MAX_INLINE_LEN); + } + txq_ctrl->txq.inlen_send = inlen_send; + txq_ctrl->txq.inlen_mode = inlen_mode; + txq_ctrl->txq.inlen_empw = 0; + inlen_send = 0; + inlen_empw = 0; + } + if (inlen_send && inlen_empw && priv->txqs_n >= txqs_inline) { + /* + * The data sent with MLX5_OPCODE_ENHANCED_MPSW + * may be inlined in Data Segment, align the + * length accordingly to fit entire WQEBBs. + */ + temp = (inlen_empw + MLX5_WQE_SIZE - 1) / MLX5_WQE_SIZE; + temp = temp * MLX5_WQE_SIZE + + MLX5_DSEG_MIN_INLINE_SIZE - MLX5_WQE_DSEG_SIZE; + temp = RTE_MIN(temp, MLX5_WQE_SIZE_MAX + + MLX5_DSEG_MIN_INLINE_SIZE - + MLX5_WQE_CSEG_SIZE - + MLX5_WQE_ESEG_SIZE - + MLX5_WQE_DSEG_SIZE); + temp = RTE_MIN(temp, MLX5_EMPW_MAX_INLINE_LEN); + if (temp != inlen_empw) { + DRV_LOG(INFO, + "port %u enhanced empw inline setting" + " aligned from %u to %u", + PORT_ID(priv), inlen_empw, temp); + inlen_empw = temp; + } + assert(inlen_empw >= MLX5_ESEG_MIN_INLINE_SIZE); + assert(inlen_empw <= MLX5_WQE_SIZE_MAX + + MLX5_DSEG_MIN_INLINE_SIZE - + MLX5_WQE_CSEG_SIZE - + MLX5_WQE_ESEG_SIZE - + MLX5_WQE_DSEG_SIZE); + txq_ctrl->txq.inlen_empw = inlen_empw; + } + txq_ctrl->max_inline_data = RTE_MAX(inlen_send, inlen_empw); + if (tso) { + txq_ctrl->max_tso_header = MLX5_MAX_TSO_HEADER; + txq_ctrl->max_inline_data = RTE_MAX(txq_ctrl->max_inline_data, + MLX5_MAX_TSO_HEADER); + txq_ctrl->txq.tso_en = 1; + } + txq_ctrl->txq.tunnel_en = config->tunnel_en | config->swp; + txq_ctrl->txq.swp_en = ((DEV_TX_OFFLOAD_IP_TNL_TSO | + DEV_TX_OFFLOAD_UDP_TNL_TSO | + DEV_TX_OFFLOAD_OUTER_IPV4_CKSUM) & + txq_ctrl->txq.offloads) && config->swp; +} + +/** + * Create a DPDK Tx queue. * * @param dev - * Pointer to Ethernet device structure. + * Pointer to Ethernet device. * @param idx * TX queue index. * @param desc @@ -422,138 +907,164 @@ error: * @param socket * NUMA socket on which memory must be allocated. * @param[in] conf - * Thresholds parameters. + * Thresholds parameters. * * @return - * 0 on success, negative errno value on failure. + * A DPDK queue object on success, NULL otherwise and rte_errno is set. */ -int -mlx5_tx_queue_setup(struct rte_eth_dev *dev, uint16_t idx, uint16_t desc, - unsigned int socket, const struct rte_eth_txconf *conf) +struct mlx5_txq_ctrl * +mlx5_txq_new(struct rte_eth_dev *dev, uint16_t idx, uint16_t desc, + unsigned int socket, const struct rte_eth_txconf *conf) { - struct priv *priv = dev->data->dev_private; - struct txq *txq = (*priv->txqs)[idx]; - struct txq_ctrl *txq_ctrl; - int ret; - - if (mlx5_is_secondary()) - return -E_RTE_SECONDARY; + struct mlx5_priv *priv = dev->data->dev_private; + struct mlx5_txq_ctrl *tmpl; - priv_lock(priv); - if (txq) - txq_ctrl = container_of(txq, struct txq_ctrl, txq); - DEBUG("%p: configuring queue %u for %u descriptors", - (void *)dev, idx, desc); - if (idx >= priv->txqs_n) { - ERROR("%p: queue index out of range (%u >= %u)", - (void *)dev, idx, priv->txqs_n); - priv_unlock(priv); - return -EOVERFLOW; - } - if (txq != NULL) { - DEBUG("%p: reusing already allocated queue index %u (%p)", - (void *)dev, idx, (void *)txq); - if (priv->started) { - priv_unlock(priv); - return -EEXIST; - } - (*priv->txqs)[idx] = NULL; - txq_cleanup(txq_ctrl); - } else { - txq_ctrl = rte_calloc_socket("TXQ", 1, sizeof(*txq_ctrl), - 0, socket); - if (txq_ctrl == NULL) { - ERROR("%p: unable to allocate queue index %u", - (void *)dev, idx); - priv_unlock(priv); - return -ENOMEM; - } + tmpl = rte_calloc_socket("TXQ", 1, + sizeof(*tmpl) + + desc * sizeof(struct rte_mbuf *), + 0, socket); + if (!tmpl) { + rte_errno = ENOMEM; + return NULL; + } + if (mlx5_mr_btree_init(&tmpl->txq.mr_ctrl.cache_bh, + MLX5_MR_BTREE_CACHE_N, socket)) { + /* rte_errno is already set. */ + goto error; + } + /* Save pointer of global generation number to check memory event. */ + tmpl->txq.mr_ctrl.dev_gen_ptr = &priv->sh->mr.dev_gen; + assert(desc > MLX5_TX_COMP_THRESH); + tmpl->txq.offloads = conf->offloads | + dev->data->dev_conf.txmode.offloads; + tmpl->priv = priv; + tmpl->socket = socket; + tmpl->txq.elts_n = log2above(desc); + tmpl->txq.elts_s = desc; + tmpl->txq.elts_m = desc - 1; + tmpl->txq.port_id = dev->data->port_id; + tmpl->txq.idx = idx; + txq_set_params(tmpl); + if (txq_calc_wqebb_cnt(tmpl) > + priv->sh->device_attr.orig_attr.max_qp_wr) { + DRV_LOG(ERR, + "port %u Tx WQEBB count (%d) exceeds the limit (%d)," + " try smaller queue size", + dev->data->port_id, txq_calc_wqebb_cnt(tmpl), + priv->sh->device_attr.orig_attr.max_qp_wr); + rte_errno = ENOMEM; + goto error; } - ret = txq_setup(dev, txq_ctrl, desc, socket, conf); - if (ret) - rte_free(txq_ctrl); - else { - txq_ctrl->txq.stats.idx = idx; - DEBUG("%p: adding TX queue %p to list", - (void *)dev, (void *)txq_ctrl); - (*priv->txqs)[idx] = &txq_ctrl->txq; - /* Update send callback. */ - dev->tx_pkt_burst = mlx5_tx_burst; - } - priv_unlock(priv); - return -ret; + rte_atomic32_inc(&tmpl->refcnt); + LIST_INSERT_HEAD(&priv->txqsctrl, tmpl, next); + return tmpl; +error: + rte_free(tmpl); + return NULL; } /** - * DPDK callback to release a TX queue. + * Get a Tx queue. * - * @param dpdk_txq - * Generic TX queue pointer. + * @param dev + * Pointer to Ethernet device. + * @param idx + * TX queue index. + * + * @return + * A pointer to the queue if it exists. */ -void -mlx5_tx_queue_release(void *dpdk_txq) +struct mlx5_txq_ctrl * +mlx5_txq_get(struct rte_eth_dev *dev, uint16_t idx) { - struct txq *txq = (struct txq *)dpdk_txq; - struct txq_ctrl *txq_ctrl; - struct priv *priv; - unsigned int i; - - if (mlx5_is_secondary()) - return; + struct mlx5_priv *priv = dev->data->dev_private; + struct mlx5_txq_ctrl *ctrl = NULL; - if (txq == NULL) - return; - txq_ctrl = container_of(txq, struct txq_ctrl, txq); - priv = txq->priv; - priv_lock(priv); - for (i = 0; (i != priv->txqs_n); ++i) - if ((*priv->txqs)[i] == txq) { - DEBUG("%p: removing TX queue %p from list", - (void *)priv->dev, (void *)txq_ctrl); - (*priv->txqs)[i] = NULL; - break; - } - txq_cleanup(txq_ctrl); - rte_free(txq_ctrl); - priv_unlock(priv); + if ((*priv->txqs)[idx]) { + ctrl = container_of((*priv->txqs)[idx], struct mlx5_txq_ctrl, + txq); + mlx5_txq_ibv_get(dev, idx); + rte_atomic32_inc(&ctrl->refcnt); + } + return ctrl; } /** - * DPDK callback for TX in secondary processes. - * - * This function configures all queues from primary process information - * if necessary before reverting to the normal TX burst callback. + * Release a Tx queue. * - * @param dpdk_txq - * Generic pointer to TX queue structure. - * @param[in] pkts - * Packets to transmit. - * @param pkts_n - * Number of packets in array. + * @param dev + * Pointer to Ethernet device. + * @param idx + * TX queue index. * * @return - * Number of packets successfully transmitted (<= pkts_n). + * 1 while a reference on it exists, 0 when freed. */ -uint16_t -mlx5_tx_burst_secondary_setup(void *dpdk_txq, struct rte_mbuf **pkts, - uint16_t pkts_n) +int +mlx5_txq_release(struct rte_eth_dev *dev, uint16_t idx) { - struct txq *txq = dpdk_txq; - struct priv *priv = mlx5_secondary_data_setup(txq->priv); - struct priv *primary_priv; - unsigned int index; + struct mlx5_priv *priv = dev->data->dev_private; + struct mlx5_txq_ctrl *txq; - if (priv == NULL) + if (!(*priv->txqs)[idx]) return 0; - primary_priv = - mlx5_secondary_data[priv->dev->data->port_id].primary_priv; - /* Look for queue index in both private structures. */ - for (index = 0; index != priv->txqs_n; ++index) - if (((*primary_priv->txqs)[index] == txq) || - ((*priv->txqs)[index] == txq)) - break; - if (index == priv->txqs_n) + txq = container_of((*priv->txqs)[idx], struct mlx5_txq_ctrl, txq); + if (txq->ibv && !mlx5_txq_ibv_release(txq->ibv)) + txq->ibv = NULL; + if (rte_atomic32_dec_and_test(&txq->refcnt)) { + txq_free_elts(txq); + mlx5_mr_btree_free(&txq->txq.mr_ctrl.cache_bh); + LIST_REMOVE(txq, next); + rte_free(txq); + (*priv->txqs)[idx] = NULL; return 0; - txq = (*priv->txqs)[index]; - return priv->dev->tx_pkt_burst(txq, pkts, pkts_n); + } + return 1; +} + +/** + * Verify if the queue can be released. + * + * @param dev + * Pointer to Ethernet device. + * @param idx + * TX queue index. + * + * @return + * 1 if the queue can be released. + */ +int +mlx5_txq_releasable(struct rte_eth_dev *dev, uint16_t idx) +{ + struct mlx5_priv *priv = dev->data->dev_private; + struct mlx5_txq_ctrl *txq; + + if (!(*priv->txqs)[idx]) + return -1; + txq = container_of((*priv->txqs)[idx], struct mlx5_txq_ctrl, txq); + return (rte_atomic32_read(&txq->refcnt) == 1); +} + +/** + * Verify the Tx Queue list is empty + * + * @param dev + * Pointer to Ethernet device. + * + * @return + * The number of object not released. + */ +int +mlx5_txq_verify(struct rte_eth_dev *dev) +{ + struct mlx5_priv *priv = dev->data->dev_private; + struct mlx5_txq_ctrl *txq_ctrl; + int ret = 0; + + LIST_FOREACH(txq_ctrl, &priv->txqsctrl, next) { + DRV_LOG(DEBUG, "port %u Tx queue %u still referenced", + dev->data->port_id, txq_ctrl->txq.idx); + ++ret; + } + return ret; }