X-Git-Url: http://git.droids-corp.org/?a=blobdiff_plain;f=drivers%2Fnet%2Fmlx4%2Fmlx4_rxq.c;h=87688c1c7ed5b7a65d117832c3965c99a1622bbc;hb=399421100e08b79e1bf68f3406b3bc414925c967;hp=98ab1d266f18b51a117b70e2ebc2405d478023fd;hpb=597d2ce5b4698dafcba52b1087575221d798c06f;p=dpdk.git diff --git a/drivers/net/mlx4/mlx4_rxq.c b/drivers/net/mlx4/mlx4_rxq.c index 98ab1d266f..87688c1c7e 100644 --- a/drivers/net/mlx4/mlx4_rxq.c +++ b/drivers/net/mlx4/mlx4_rxq.c @@ -1,34 +1,6 @@ -/*- - * BSD LICENSE - * - * Copyright 2017 6WIND S.A. - * Copyright 2017 Mellanox - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * - * * Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in - * the documentation and/or other materials provided with the - * distribution. - * * Neither the name of 6WIND S.A. nor the names of its - * contributors may be used to endorse or promote products derived - * from this software without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright 2017 6WIND S.A. + * Copyright 2017 Mellanox Technologies, Ltd */ /** @@ -55,13 +27,14 @@ #include #include #include -#include +#include #include #include #include #include #include "mlx4.h" +#include "mlx4_glue.h" #include "mlx4_flow.h" #include "mlx4_rxtx.h" #include "mlx4_utils.h" @@ -115,7 +88,7 @@ mlx4_rss_hash_key_default[MLX4_RSS_HASH_KEY_SIZE] = { */ struct mlx4_rss * mlx4_rss_get(struct priv *priv, uint64_t fields, - uint8_t key[MLX4_RSS_HASH_KEY_SIZE], + const uint8_t key[MLX4_RSS_HASH_KEY_SIZE], uint16_t queues, const uint16_t queue_id[]) { struct mlx4_rss *rss; @@ -231,7 +204,7 @@ mlx4_rss_attach(struct mlx4_rss *rss) } ind_tbl[i] = rxq->wq; } - rss->ind = ibv_create_rwq_ind_table + rss->ind = mlx4_glue->create_rwq_ind_table (priv->ctx, &(struct ibv_rwq_ind_table_init_attr){ .log_ind_tbl_size = rte_log2_u32(RTE_DIM(ind_tbl)), @@ -243,7 +216,7 @@ mlx4_rss_attach(struct mlx4_rss *rss) msg = "RSS indirection table creation failure"; goto error; } - rss->qp = ibv_create_qp_ex + rss->qp = mlx4_glue->create_qp_ex (priv->ctx, &(struct ibv_qp_init_attr_ex){ .comp_mask = (IBV_QP_INIT_ATTR_PD | @@ -264,7 +237,7 @@ mlx4_rss_attach(struct mlx4_rss *rss) msg = "RSS hash QP creation failure"; goto error; } - ret = ibv_modify_qp + ret = mlx4_glue->modify_qp (rss->qp, &(struct ibv_qp_attr){ .qp_state = IBV_QPS_INIT, @@ -275,7 +248,7 @@ mlx4_rss_attach(struct mlx4_rss *rss) msg = "failed to switch RSS hash QP to INIT state"; goto error; } - ret = ibv_modify_qp + ret = mlx4_glue->modify_qp (rss->qp, &(struct ibv_qp_attr){ .qp_state = IBV_QPS_RTR, @@ -288,11 +261,11 @@ mlx4_rss_attach(struct mlx4_rss *rss) return 0; error: if (rss->qp) { - claim_zero(ibv_destroy_qp(rss->qp)); + claim_zero(mlx4_glue->destroy_qp(rss->qp)); rss->qp = NULL; } if (rss->ind) { - claim_zero(ibv_destroy_rwq_ind_table(rss->ind)); + claim_zero(mlx4_glue->destroy_rwq_ind_table(rss->ind)); rss->ind = NULL; } while (i--) @@ -325,9 +298,9 @@ mlx4_rss_detach(struct mlx4_rss *rss) assert(rss->ind); if (--rss->usecnt) return; - claim_zero(ibv_destroy_qp(rss->qp)); + claim_zero(mlx4_glue->destroy_qp(rss->qp)); rss->qp = NULL; - claim_zero(ibv_destroy_rwq_ind_table(rss->ind)); + claim_zero(mlx4_glue->destroy_rwq_ind_table(rss->ind)); rss->ind = NULL; for (i = 0; i != rss->queues; ++i) mlx4_rxq_detach(priv->dev->data->rx_queues[rss->queue_id[i]]); @@ -363,10 +336,13 @@ mlx4_rss_init(struct priv *priv) unsigned int i; int ret; + if (priv->rss_init) + return 0; /* Prepare range for RSS contexts before creating the first WQ. */ - ret = mlx4dv_set_context_attr(priv->ctx, - MLX4DV_SET_CTX_ATTR_LOG_WQS_RANGE_SZ, - &log2_range); + ret = mlx4_glue->dv_set_context_attr + (priv->ctx, + MLX4DV_SET_CTX_ATTR_LOG_WQS_RANGE_SZ, + &log2_range); if (ret) { ERROR("cannot set up range size for RSS context to %u" " (for %u Rx queues), error: %s", @@ -402,13 +378,13 @@ mlx4_rss_init(struct priv *priv) * sequentially and are guaranteed to never be reused in the * same context by the underlying implementation. */ - cq = ibv_create_cq(priv->ctx, 1, NULL, NULL, 0); + cq = mlx4_glue->create_cq(priv->ctx, 1, NULL, NULL, 0); if (!cq) { ret = ENOMEM; msg = "placeholder CQ creation failure"; goto error; } - wq = ibv_create_wq + wq = mlx4_glue->create_wq (priv->ctx, &(struct ibv_wq_init_attr){ .wq_type = IBV_WQT_RQ, @@ -419,11 +395,11 @@ mlx4_rss_init(struct priv *priv) }); if (wq) { wq_num = wq->wq_num; - claim_zero(ibv_destroy_wq(wq)); + claim_zero(mlx4_glue->destroy_wq(wq)); } else { wq_num = 0; /* Shut up GCC 4.8 warnings. */ } - claim_zero(ibv_destroy_cq(cq)); + claim_zero(mlx4_glue->destroy_cq(cq)); if (!wq) { ret = ENOMEM; msg = "placeholder WQ creation failure"; @@ -444,6 +420,7 @@ wq_num_check: } wq_num_prev = wq_num; } + priv->rss_init = 1; return 0; error: ERROR("cannot initialize common RSS resources (queue %u): %s: %s", @@ -472,6 +449,8 @@ mlx4_rss_deinit(struct priv *priv) { unsigned int i; + if (!priv->rss_init) + return; for (i = 0; i != priv->dev->data->nb_rx_queues; ++i) { struct rxq *rxq = priv->dev->data->rx_queues[i]; @@ -480,6 +459,7 @@ mlx4_rss_deinit(struct priv *priv) mlx4_rxq_detach(rxq); } } + priv->rss_init = 0; } /** @@ -508,6 +488,7 @@ mlx4_rxq_attach(struct rxq *rxq) } struct priv *priv = rxq->priv; + struct rte_eth_dev *dev = priv->dev; const uint32_t elts_n = 1 << rxq->elts_n; const uint32_t sges_n = 1 << rxq->sges_n; struct rte_mbuf *(*elts)[elts_n] = rxq->elts; @@ -517,18 +498,26 @@ mlx4_rxq_attach(struct rxq *rxq) const char *msg; struct ibv_cq *cq = NULL; struct ibv_wq *wq = NULL; + uint32_t create_flags = 0; + uint32_t comp_mask = 0; volatile struct mlx4_wqe_data_seg (*wqes)[]; unsigned int i; int ret; assert(rte_is_power_of_2(elts_n)); - cq = ibv_create_cq(priv->ctx, elts_n / sges_n, NULL, rxq->channel, 0); + cq = mlx4_glue->create_cq(priv->ctx, elts_n / sges_n, NULL, + rxq->channel, 0); if (!cq) { ret = ENOMEM; msg = "CQ creation failure"; goto error; } - wq = ibv_create_wq + /* By default, FCS (CRC) is stripped by hardware. */ + if (rxq->crc_present) { + create_flags |= IBV_WQ_FLAGS_SCATTER_FCS; + comp_mask |= IBV_WQ_INIT_ATTR_FLAGS; + } + wq = mlx4_glue->create_wq (priv->ctx, &(struct ibv_wq_init_attr){ .wq_type = IBV_WQT_RQ, @@ -536,13 +525,15 @@ mlx4_rxq_attach(struct rxq *rxq) .max_sge = sges_n, .pd = priv->pd, .cq = cq, + .comp_mask = comp_mask, + .create_flags = create_flags, }); if (!wq) { ret = errno ? errno : EINVAL; msg = "WQ creation failure"; goto error; } - ret = ibv_modify_wq + ret = mlx4_glue->modify_wq (wq, &(struct ibv_wq_attr){ .attr_mask = IBV_WQ_ATTR_STATE, @@ -557,11 +548,16 @@ mlx4_rxq_attach(struct rxq *rxq) mlxdv.cq.out = &dv_cq; mlxdv.rwq.in = wq; mlxdv.rwq.out = &dv_rwq; - ret = mlx4dv_init_obj(&mlxdv, MLX4DV_OBJ_RWQ | MLX4DV_OBJ_CQ); + ret = mlx4_glue->dv_init_obj(&mlxdv, MLX4DV_OBJ_RWQ | MLX4DV_OBJ_CQ); if (ret) { msg = "failed to obtain device information from WQ/CQ objects"; goto error; } + /* Pre-register Rx mempool. */ + DEBUG("port %u Rx queue %u registering mp %s having %u chunks", + priv->dev->data->port_id, rxq->stats.idx, + rxq->mp->name, rxq->mp->nb_mem_chunks); + mlx4_mr_update_mp(dev, &rxq->mr_ctrl, rxq->mp); wqes = (volatile struct mlx4_wqe_data_seg (*)[]) ((uintptr_t)dv_rwq.buf.buf + dv_rwq.rq.offset); for (i = 0; i != RTE_DIM(*elts); ++i) { @@ -593,7 +589,7 @@ mlx4_rxq_attach(struct rxq *rxq) .addr = rte_cpu_to_be_64(rte_pktmbuf_mtod(buf, uintptr_t)), .byte_count = rte_cpu_to_be_32(buf->data_len), - .lkey = rte_cpu_to_be_32(rxq->mr->lkey), + .lkey = mlx4_rx_mb2mr(rxq, buf), }; (*elts)[i] = buf; } @@ -619,9 +615,10 @@ mlx4_rxq_attach(struct rxq *rxq) return 0; error: if (wq) - claim_zero(ibv_destroy_wq(wq)); + claim_zero(mlx4_glue->destroy_wq(wq)); if (cq) - claim_zero(ibv_destroy_cq(cq)); + claim_zero(mlx4_glue->destroy_cq(cq)); + --rxq->usecnt; rte_errno = ret; ERROR("error while attaching Rx queue %p: %s: %s", (void *)rxq, msg, strerror(ret)); @@ -649,9 +646,9 @@ mlx4_rxq_detach(struct rxq *rxq) memset(&rxq->mcq, 0, sizeof(rxq->mcq)); rxq->rq_db = NULL; rxq->wqes = NULL; - claim_zero(ibv_destroy_wq(rxq->wq)); + claim_zero(mlx4_glue->destroy_wq(rxq->wq)); rxq->wq = NULL; - claim_zero(ibv_destroy_cq(rxq->cq)); + claim_zero(mlx4_glue->destroy_cq(rxq->cq)); rxq->cq = NULL; DEBUG("%p: freeing Rx queue elements", (void *)rxq); for (i = 0; (i != RTE_DIM(*elts)); ++i) { @@ -674,7 +671,8 @@ mlx4_rxq_detach(struct rxq *rxq) uint64_t mlx4_get_rx_queue_offloads(struct priv *priv) { - uint64_t offloads = DEV_RX_OFFLOAD_SCATTER; + uint64_t offloads = DEV_RX_OFFLOAD_SCATTER | + DEV_RX_OFFLOAD_CRC_STRIP; if (priv->hw_csum) offloads |= DEV_RX_OFFLOAD_CHECKSUM; @@ -699,26 +697,6 @@ mlx4_get_rx_port_offloads(struct priv *priv) return offloads; } -/** - * Checks if the per-queue offload configuration is valid. - * - * @param priv - * Pointer to private structure. - * @param requested - * Per-queue offloads configuration. - * - * @return - * Nonzero when configuration is valid. - */ -static int -mlx4_check_rx_queue_offloads(struct priv *priv, uint64_t requested) -{ - uint64_t mandatory = priv->dev->data->dev_conf.rxmode.offloads; - uint64_t supported = mlx4_get_rx_port_offloads(priv); - - return !((mandatory ^ requested) & supported); -} - /** * DPDK callback to configure a Rx queue. * @@ -760,20 +738,14 @@ mlx4_rx_queue_setup(struct rte_eth_dev *dev, uint16_t idx, uint16_t desc, }, }; int ret; + uint32_t crc_present; + uint64_t offloads; + + offloads = conf->offloads | dev->data->dev_conf.rxmode.offloads; - (void)conf; /* Thresholds configuration (ignored). */ DEBUG("%p: configuring queue %u for %u descriptors", (void *)dev, idx, desc); - if (!mlx4_check_rx_queue_offloads(priv, conf->offloads)) { - rte_errno = ENOTSUP; - ERROR("%p: Rx queue offloads 0x%" PRIx64 " don't match port " - "offloads 0x%" PRIx64 " or supported offloads 0x%" PRIx64, - (void *)dev, conf->offloads, - dev->data->dev_conf.rxmode.offloads, - (mlx4_get_rx_port_offloads(priv) | - mlx4_get_rx_queue_offloads(priv))); - return -rte_errno; - } + if (idx >= dev->data->nb_rx_queues) { rte_errno = EOVERFLOW; ERROR("%p: queue index out of range (%u >= %u)", @@ -798,6 +770,23 @@ mlx4_rx_queue_setup(struct rte_eth_dev *dev, uint16_t idx, uint16_t desc, " to the next power of two (%u)", (void *)dev, idx, desc); } + /* By default, FCS (CRC) is stripped by hardware. */ + if (offloads & DEV_RX_OFFLOAD_CRC_STRIP) { + crc_present = 0; + } else if (priv->hw_fcs_strip) { + crc_present = 1; + } else { + WARN("%p: CRC stripping has been disabled but will still" + " be performed by hardware, make sure MLNX_OFED and" + " firmware are up to date", + (void *)dev); + crc_present = 0; + } + DEBUG("%p: CRC stripping is %s, %u bytes will be subtracted from" + " incoming frames to hide it", + (void *)dev, + crc_present ? "disabled" : "enabled", + crc_present << 2); /* Allocate and initialize Rx queue. */ mlx4_zmallocv_socket("RXQ", vec, RTE_DIM(vec), socket); if (!rxq) { @@ -814,9 +803,10 @@ mlx4_rx_queue_setup(struct rte_eth_dev *dev, uint16_t idx, uint16_t desc, .elts = elts, /* Toggle Rx checksum offload if hardware supports it. */ .csum = priv->hw_csum && - (conf->offloads & DEV_RX_OFFLOAD_CHECKSUM), + (offloads & DEV_RX_OFFLOAD_CHECKSUM), .csum_l2tun = priv->hw_csum_l2tun && - (conf->offloads & DEV_RX_OFFLOAD_CHECKSUM), + (offloads & DEV_RX_OFFLOAD_CHECKSUM), + .crc_present = crc_present, .l2tun_offload = priv->hw_csum_l2tun, .stats = { .idx = idx, @@ -828,7 +818,7 @@ mlx4_rx_queue_setup(struct rte_eth_dev *dev, uint16_t idx, uint16_t desc, if (dev->data->dev_conf.rxmode.max_rx_pkt_len <= (mb_len - RTE_PKTMBUF_HEADROOM)) { ; - } else if (conf->offloads & DEV_RX_OFFLOAD_SCATTER) { + } else if (offloads & DEV_RX_OFFLOAD_SCATTER) { uint32_t size = RTE_PKTMBUF_HEADROOM + dev->data->dev_conf.rxmode.max_rx_pkt_len; @@ -871,15 +861,13 @@ mlx4_rx_queue_setup(struct rte_eth_dev *dev, uint16_t idx, uint16_t desc, 1 << rxq->sges_n); goto error; } - /* Use the entire Rx mempool as the memory region. */ - rxq->mr = mlx4_mr_get(priv, mp); - if (!rxq->mr) { - ERROR("%p: MR creation failure: %s", - (void *)dev, strerror(rte_errno)); + if (mlx4_mr_btree_init(&rxq->mr_ctrl.cache_bh, + MLX4_MR_BTREE_CACHE_N, socket)) { + /* rte_errno is already set. */ goto error; } if (dev->data->dev_conf.intr_conf.rxq) { - rxq->channel = ibv_create_comp_channel(priv->ctx); + rxq->channel = mlx4_glue->create_comp_channel(priv->ctx); if (rxq->channel == NULL) { rte_errno = ENOMEM; ERROR("%p: Rx interrupt completion channel creation" @@ -934,8 +922,7 @@ mlx4_rx_queue_release(void *dpdk_rxq) assert(!rxq->wqes); assert(!rxq->rq_db); if (rxq->channel) - claim_zero(ibv_destroy_comp_channel(rxq->channel)); - if (rxq->mr) - mlx4_mr_put(rxq->mr); + claim_zero(mlx4_glue->destroy_comp_channel(rxq->channel)); + mlx4_mr_btree_free(&rxq->mr_ctrl.cache_bh); rte_free(rxq); }