X-Git-Url: http://git.droids-corp.org/?a=blobdiff_plain;f=drivers%2Fnet%2Fmlx4%2Fmlx4_rxq.c;h=87688c1c7ed5b7a65d117832c3965c99a1622bbc;hb=399421100e08b79e1bf68f3406b3bc414925c967;hp=ad55934b446f52c47799442363d9fa148453f75a;hpb=5697a4142107ae1bc69ec13699d146b157d06652;p=dpdk.git diff --git a/drivers/net/mlx4/mlx4_rxq.c b/drivers/net/mlx4/mlx4_rxq.c index ad55934b44..87688c1c7e 100644 --- a/drivers/net/mlx4/mlx4_rxq.c +++ b/drivers/net/mlx4/mlx4_rxq.c @@ -1,34 +1,6 @@ -/*- - * BSD LICENSE - * - * Copyright 2017 6WIND S.A. - * Copyright 2017 Mellanox - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * - * * Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in - * the documentation and/or other materials provided with the - * distribution. - * * Neither the name of 6WIND S.A. nor the names of its - * contributors may be used to endorse or promote products derived - * from this software without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright 2017 6WIND S.A. + * Copyright 2017 Mellanox Technologies, Ltd */ /** @@ -55,13 +27,14 @@ #include #include #include -#include +#include #include #include #include #include #include "mlx4.h" +#include "mlx4_glue.h" #include "mlx4_flow.h" #include "mlx4_rxtx.h" #include "mlx4_utils.h" @@ -115,7 +88,7 @@ mlx4_rss_hash_key_default[MLX4_RSS_HASH_KEY_SIZE] = { */ struct mlx4_rss * mlx4_rss_get(struct priv *priv, uint64_t fields, - uint8_t key[MLX4_RSS_HASH_KEY_SIZE], + const uint8_t key[MLX4_RSS_HASH_KEY_SIZE], uint16_t queues, const uint16_t queue_id[]) { struct mlx4_rss *rss; @@ -164,7 +137,8 @@ error: * @param rss * RSS context to release. */ -void mlx4_rss_put(struct mlx4_rss *rss) +void +mlx4_rss_put(struct mlx4_rss *rss) { assert(rss->refcnt); if (--rss->refcnt) @@ -190,7 +164,8 @@ void mlx4_rss_put(struct mlx4_rss *rss) * @return * 0 on success, a negative errno value otherwise and rte_errno is set. */ -int mlx4_rss_attach(struct mlx4_rss *rss) +int +mlx4_rss_attach(struct mlx4_rss *rss) { assert(rss->refcnt); if (rss->usecnt++) { @@ -229,7 +204,7 @@ int mlx4_rss_attach(struct mlx4_rss *rss) } ind_tbl[i] = rxq->wq; } - rss->ind = ibv_create_rwq_ind_table + rss->ind = mlx4_glue->create_rwq_ind_table (priv->ctx, &(struct ibv_rwq_ind_table_init_attr){ .log_ind_tbl_size = rte_log2_u32(RTE_DIM(ind_tbl)), @@ -241,7 +216,7 @@ int mlx4_rss_attach(struct mlx4_rss *rss) msg = "RSS indirection table creation failure"; goto error; } - rss->qp = ibv_create_qp_ex + rss->qp = mlx4_glue->create_qp_ex (priv->ctx, &(struct ibv_qp_init_attr_ex){ .comp_mask = (IBV_QP_INIT_ATTR_PD | @@ -262,7 +237,7 @@ int mlx4_rss_attach(struct mlx4_rss *rss) msg = "RSS hash QP creation failure"; goto error; } - ret = ibv_modify_qp + ret = mlx4_glue->modify_qp (rss->qp, &(struct ibv_qp_attr){ .qp_state = IBV_QPS_INIT, @@ -273,7 +248,7 @@ int mlx4_rss_attach(struct mlx4_rss *rss) msg = "failed to switch RSS hash QP to INIT state"; goto error; } - ret = ibv_modify_qp + ret = mlx4_glue->modify_qp (rss->qp, &(struct ibv_qp_attr){ .qp_state = IBV_QPS_RTR, @@ -286,11 +261,11 @@ int mlx4_rss_attach(struct mlx4_rss *rss) return 0; error: if (rss->qp) { - claim_zero(ibv_destroy_qp(rss->qp)); + claim_zero(mlx4_glue->destroy_qp(rss->qp)); rss->qp = NULL; } if (rss->ind) { - claim_zero(ibv_destroy_rwq_ind_table(rss->ind)); + claim_zero(mlx4_glue->destroy_rwq_ind_table(rss->ind)); rss->ind = NULL; } while (i--) @@ -312,7 +287,8 @@ error: * @param rss * RSS context to detach from. */ -void mlx4_rss_detach(struct mlx4_rss *rss) +void +mlx4_rss_detach(struct mlx4_rss *rss) { struct priv *priv = rss->priv; unsigned int i; @@ -322,9 +298,9 @@ void mlx4_rss_detach(struct mlx4_rss *rss) assert(rss->ind); if (--rss->usecnt) return; - claim_zero(ibv_destroy_qp(rss->qp)); + claim_zero(mlx4_glue->destroy_qp(rss->qp)); rss->qp = NULL; - claim_zero(ibv_destroy_rwq_ind_table(rss->ind)); + claim_zero(mlx4_glue->destroy_rwq_ind_table(rss->ind)); rss->ind = NULL; for (i = 0; i != rss->queues; ++i) mlx4_rxq_detach(priv->dev->data->rx_queues[rss->queue_id[i]]); @@ -360,10 +336,13 @@ mlx4_rss_init(struct priv *priv) unsigned int i; int ret; + if (priv->rss_init) + return 0; /* Prepare range for RSS contexts before creating the first WQ. */ - ret = mlx4dv_set_context_attr(priv->ctx, - MLX4DV_SET_CTX_ATTR_LOG_WQS_RANGE_SZ, - &log2_range); + ret = mlx4_glue->dv_set_context_attr + (priv->ctx, + MLX4DV_SET_CTX_ATTR_LOG_WQS_RANGE_SZ, + &log2_range); if (ret) { ERROR("cannot set up range size for RSS context to %u" " (for %u Rx queues), error: %s", @@ -399,13 +378,13 @@ mlx4_rss_init(struct priv *priv) * sequentially and are guaranteed to never be reused in the * same context by the underlying implementation. */ - cq = ibv_create_cq(priv->ctx, 1, NULL, NULL, 0); + cq = mlx4_glue->create_cq(priv->ctx, 1, NULL, NULL, 0); if (!cq) { ret = ENOMEM; msg = "placeholder CQ creation failure"; goto error; } - wq = ibv_create_wq + wq = mlx4_glue->create_wq (priv->ctx, &(struct ibv_wq_init_attr){ .wq_type = IBV_WQT_RQ, @@ -416,11 +395,11 @@ mlx4_rss_init(struct priv *priv) }); if (wq) { wq_num = wq->wq_num; - claim_zero(ibv_destroy_wq(wq)); + claim_zero(mlx4_glue->destroy_wq(wq)); } else { wq_num = 0; /* Shut up GCC 4.8 warnings. */ } - claim_zero(ibv_destroy_cq(cq)); + claim_zero(mlx4_glue->destroy_cq(cq)); if (!wq) { ret = ENOMEM; msg = "placeholder WQ creation failure"; @@ -441,6 +420,7 @@ wq_num_check: } wq_num_prev = wq_num; } + priv->rss_init = 1; return 0; error: ERROR("cannot initialize common RSS resources (queue %u): %s: %s", @@ -469,6 +449,8 @@ mlx4_rss_deinit(struct priv *priv) { unsigned int i; + if (!priv->rss_init) + return; for (i = 0; i != priv->dev->data->nb_rx_queues; ++i) { struct rxq *rxq = priv->dev->data->rx_queues[i]; @@ -477,6 +459,7 @@ mlx4_rss_deinit(struct priv *priv) mlx4_rxq_detach(rxq); } } + priv->rss_init = 0; } /** @@ -505,27 +488,36 @@ mlx4_rxq_attach(struct rxq *rxq) } struct priv *priv = rxq->priv; + struct rte_eth_dev *dev = priv->dev; const uint32_t elts_n = 1 << rxq->elts_n; const uint32_t sges_n = 1 << rxq->sges_n; struct rte_mbuf *(*elts)[elts_n] = rxq->elts; struct mlx4dv_obj mlxdv; struct mlx4dv_rwq dv_rwq; - struct mlx4dv_cq dv_cq; + struct mlx4dv_cq dv_cq = { .comp_mask = MLX4DV_CQ_MASK_UAR, }; const char *msg; struct ibv_cq *cq = NULL; struct ibv_wq *wq = NULL; + uint32_t create_flags = 0; + uint32_t comp_mask = 0; volatile struct mlx4_wqe_data_seg (*wqes)[]; unsigned int i; int ret; assert(rte_is_power_of_2(elts_n)); - cq = ibv_create_cq(priv->ctx, elts_n / sges_n, NULL, rxq->channel, 0); + cq = mlx4_glue->create_cq(priv->ctx, elts_n / sges_n, NULL, + rxq->channel, 0); if (!cq) { ret = ENOMEM; msg = "CQ creation failure"; goto error; } - wq = ibv_create_wq + /* By default, FCS (CRC) is stripped by hardware. */ + if (rxq->crc_present) { + create_flags |= IBV_WQ_FLAGS_SCATTER_FCS; + comp_mask |= IBV_WQ_INIT_ATTR_FLAGS; + } + wq = mlx4_glue->create_wq (priv->ctx, &(struct ibv_wq_init_attr){ .wq_type = IBV_WQT_RQ, @@ -533,13 +525,15 @@ mlx4_rxq_attach(struct rxq *rxq) .max_sge = sges_n, .pd = priv->pd, .cq = cq, + .comp_mask = comp_mask, + .create_flags = create_flags, }); if (!wq) { ret = errno ? errno : EINVAL; msg = "WQ creation failure"; goto error; } - ret = ibv_modify_wq + ret = mlx4_glue->modify_wq (wq, &(struct ibv_wq_attr){ .attr_mask = IBV_WQ_ATTR_STATE, @@ -554,11 +548,16 @@ mlx4_rxq_attach(struct rxq *rxq) mlxdv.cq.out = &dv_cq; mlxdv.rwq.in = wq; mlxdv.rwq.out = &dv_rwq; - ret = mlx4dv_init_obj(&mlxdv, MLX4DV_OBJ_RWQ | MLX4DV_OBJ_CQ); + ret = mlx4_glue->dv_init_obj(&mlxdv, MLX4DV_OBJ_RWQ | MLX4DV_OBJ_CQ); if (ret) { msg = "failed to obtain device information from WQ/CQ objects"; goto error; } + /* Pre-register Rx mempool. */ + DEBUG("port %u Rx queue %u registering mp %s having %u chunks", + priv->dev->data->port_id, rxq->stats.idx, + rxq->mp->name, rxq->mp->nb_mem_chunks); + mlx4_mr_update_mp(dev, &rxq->mr_ctrl, rxq->mp); wqes = (volatile struct mlx4_wqe_data_seg (*)[]) ((uintptr_t)dv_rwq.buf.buf + dv_rwq.rq.offset); for (i = 0; i != RTE_DIM(*elts); ++i) { @@ -590,7 +589,7 @@ mlx4_rxq_attach(struct rxq *rxq) .addr = rte_cpu_to_be_64(rte_pktmbuf_mtod(buf, uintptr_t)), .byte_count = rte_cpu_to_be_32(buf->data_len), - .lkey = rte_cpu_to_be_32(rxq->mr->lkey), + .lkey = mlx4_rx_mb2mr(rxq, buf), }; (*elts)[i] = buf; } @@ -604,6 +603,11 @@ mlx4_rxq_attach(struct rxq *rxq) rxq->mcq.cqe_cnt = dv_cq.cqe_cnt; rxq->mcq.set_ci_db = dv_cq.set_ci_db; rxq->mcq.cqe_64 = (dv_cq.cqe_size & 64) ? 1 : 0; + rxq->mcq.arm_db = dv_cq.arm_db; + rxq->mcq.arm_sn = dv_cq.arm_sn; + rxq->mcq.cqn = dv_cq.cqn; + rxq->mcq.cq_uar = dv_cq.cq_uar; + rxq->mcq.cq_db_reg = (uint8_t *)dv_cq.cq_uar + MLX4_CQ_DOORBELL; /* Update doorbell counter. */ rxq->rq_ci = elts_n / sges_n; rte_wmb(); @@ -611,9 +615,10 @@ mlx4_rxq_attach(struct rxq *rxq) return 0; error: if (wq) - claim_zero(ibv_destroy_wq(wq)); + claim_zero(mlx4_glue->destroy_wq(wq)); if (cq) - claim_zero(ibv_destroy_cq(cq)); + claim_zero(mlx4_glue->destroy_cq(cq)); + --rxq->usecnt; rte_errno = ret; ERROR("error while attaching Rx queue %p: %s: %s", (void *)rxq, msg, strerror(ret)); @@ -641,9 +646,9 @@ mlx4_rxq_detach(struct rxq *rxq) memset(&rxq->mcq, 0, sizeof(rxq->mcq)); rxq->rq_db = NULL; rxq->wqes = NULL; - claim_zero(ibv_destroy_wq(rxq->wq)); + claim_zero(mlx4_glue->destroy_wq(rxq->wq)); rxq->wq = NULL; - claim_zero(ibv_destroy_cq(rxq->cq)); + claim_zero(mlx4_glue->destroy_cq(rxq->cq)); rxq->cq = NULL; DEBUG("%p: freeing Rx queue elements", (void *)rxq); for (i = 0; (i != RTE_DIM(*elts)); ++i) { @@ -654,6 +659,44 @@ mlx4_rxq_detach(struct rxq *rxq) } } +/** + * Returns the per-queue supported offloads. + * + * @param priv + * Pointer to private structure. + * + * @return + * Supported Tx offloads. + */ +uint64_t +mlx4_get_rx_queue_offloads(struct priv *priv) +{ + uint64_t offloads = DEV_RX_OFFLOAD_SCATTER | + DEV_RX_OFFLOAD_CRC_STRIP; + + if (priv->hw_csum) + offloads |= DEV_RX_OFFLOAD_CHECKSUM; + return offloads; +} + +/** + * Returns the per-port supported offloads. + * + * @param priv + * Pointer to private structure. + * + * @return + * Supported Rx offloads. + */ +uint64_t +mlx4_get_rx_port_offloads(struct priv *priv) +{ + uint64_t offloads = DEV_RX_OFFLOAD_VLAN_FILTER; + + (void)priv; + return offloads; +} + /** * DPDK callback to configure a Rx queue. * @@ -695,10 +738,14 @@ mlx4_rx_queue_setup(struct rte_eth_dev *dev, uint16_t idx, uint16_t desc, }, }; int ret; + uint32_t crc_present; + uint64_t offloads; + + offloads = conf->offloads | dev->data->dev_conf.rxmode.offloads; - (void)conf; /* Thresholds configuration (ignored). */ DEBUG("%p: configuring queue %u for %u descriptors", (void *)dev, idx, desc); + if (idx >= dev->data->nb_rx_queues) { rte_errno = EOVERFLOW; ERROR("%p: queue index out of range (%u >= %u)", @@ -723,6 +770,23 @@ mlx4_rx_queue_setup(struct rte_eth_dev *dev, uint16_t idx, uint16_t desc, " to the next power of two (%u)", (void *)dev, idx, desc); } + /* By default, FCS (CRC) is stripped by hardware. */ + if (offloads & DEV_RX_OFFLOAD_CRC_STRIP) { + crc_present = 0; + } else if (priv->hw_fcs_strip) { + crc_present = 1; + } else { + WARN("%p: CRC stripping has been disabled but will still" + " be performed by hardware, make sure MLNX_OFED and" + " firmware are up to date", + (void *)dev); + crc_present = 0; + } + DEBUG("%p: CRC stripping is %s, %u bytes will be subtracted from" + " incoming frames to hide it", + (void *)dev, + crc_present ? "disabled" : "enabled", + crc_present << 2); /* Allocate and initialize Rx queue. */ mlx4_zmallocv_socket("RXQ", vec, RTE_DIM(vec), socket); if (!rxq) { @@ -738,10 +802,12 @@ mlx4_rx_queue_setup(struct rte_eth_dev *dev, uint16_t idx, uint16_t desc, .elts_n = rte_log2_u32(desc), .elts = elts, /* Toggle Rx checksum offload if hardware supports it. */ - .csum = (priv->hw_csum && - dev->data->dev_conf.rxmode.hw_ip_checksum), - .csum_l2tun = (priv->hw_csum_l2tun && - dev->data->dev_conf.rxmode.hw_ip_checksum), + .csum = priv->hw_csum && + (offloads & DEV_RX_OFFLOAD_CHECKSUM), + .csum_l2tun = priv->hw_csum_l2tun && + (offloads & DEV_RX_OFFLOAD_CHECKSUM), + .crc_present = crc_present, + .l2tun_offload = priv->hw_csum_l2tun, .stats = { .idx = idx, }, @@ -752,7 +818,7 @@ mlx4_rx_queue_setup(struct rte_eth_dev *dev, uint16_t idx, uint16_t desc, if (dev->data->dev_conf.rxmode.max_rx_pkt_len <= (mb_len - RTE_PKTMBUF_HEADROOM)) { ; - } else if (dev->data->dev_conf.rxmode.enable_scatter) { + } else if (offloads & DEV_RX_OFFLOAD_SCATTER) { uint32_t size = RTE_PKTMBUF_HEADROOM + dev->data->dev_conf.rxmode.max_rx_pkt_len; @@ -795,16 +861,13 @@ mlx4_rx_queue_setup(struct rte_eth_dev *dev, uint16_t idx, uint16_t desc, 1 << rxq->sges_n); goto error; } - /* Use the entire Rx mempool as the memory region. */ - rxq->mr = mlx4_mp2mr(priv->pd, mp); - if (!rxq->mr) { - rte_errno = EINVAL; - ERROR("%p: MR creation failure: %s", - (void *)dev, strerror(rte_errno)); + if (mlx4_mr_btree_init(&rxq->mr_ctrl.cache_bh, + MLX4_MR_BTREE_CACHE_N, socket)) { + /* rte_errno is already set. */ goto error; } if (dev->data->dev_conf.intr_conf.rxq) { - rxq->channel = ibv_create_comp_channel(priv->ctx); + rxq->channel = mlx4_glue->create_comp_channel(priv->ctx); if (rxq->channel == NULL) { rte_errno = ENOMEM; ERROR("%p: Rx interrupt completion channel creation" @@ -859,8 +922,7 @@ mlx4_rx_queue_release(void *dpdk_rxq) assert(!rxq->wqes); assert(!rxq->rq_db); if (rxq->channel) - claim_zero(ibv_destroy_comp_channel(rxq->channel)); - if (rxq->mr) - claim_zero(ibv_dereg_mr(rxq->mr)); + claim_zero(mlx4_glue->destroy_comp_channel(rxq->channel)); + mlx4_mr_btree_free(&rxq->mr_ctrl.cache_bh); rte_free(rxq); }