X-Git-Url: http://git.droids-corp.org/?a=blobdiff_plain;f=drivers%2Fnet%2Fmlx5%2Fmlx5_rxq.c;h=580d7ae868139b88f83df1ccf97c4b2692e77538;hb=8e83ba285abe4341b7666927d3fc265b35446c06;hp=47dc24793b4e291dae00148d928ec392bcb56f04;hpb=8fbce96fbed60ac6fada000b30d5671e2136eeb5;p=dpdk.git diff --git a/drivers/net/mlx5/mlx5_rxq.c b/drivers/net/mlx5/mlx5_rxq.c index 47dc24793b..580d7ae868 100644 --- a/drivers/net/mlx5/mlx5_rxq.c +++ b/drivers/net/mlx5/mlx5_rxq.c @@ -67,7 +67,7 @@ mlx5_rxq_cqe_num(struct mlx5_rxq_data *rxq_data) unsigned int wqe_n = 1 << rxq_data->elts_n; if (mlx5_rxq_mprq_enabled(rxq_data)) - cqe_n = wqe_n * (1 << rxq_data->strd_num_n) - 1; + cqe_n = wqe_n * RTE_BIT32(rxq_data->log_strd_num) - 1; else cqe_n = wqe_n - 1; return cqe_n; @@ -137,8 +137,10 @@ rxq_alloc_elts_sprq(struct mlx5_rxq_ctrl *rxq_ctrl) { const unsigned int sges_n = 1 << rxq_ctrl->rxq.sges_n; unsigned int elts_n = mlx5_rxq_mprq_enabled(&rxq_ctrl->rxq) ? - (1 << rxq_ctrl->rxq.elts_n) * (1 << rxq_ctrl->rxq.strd_num_n) : - (1 << rxq_ctrl->rxq.elts_n); + RTE_BIT32(rxq_ctrl->rxq.elts_n) * + RTE_BIT32(rxq_ctrl->rxq.log_strd_num) : + RTE_BIT32(rxq_ctrl->rxq.elts_n); + bool has_vec_support = mlx5_rxq_check_vec_support(&rxq_ctrl->rxq) > 0; unsigned int i; int err; @@ -160,8 +162,9 @@ rxq_alloc_elts_sprq(struct mlx5_rxq_ctrl *rxq_ctrl) rte_errno = ENOMEM; goto error; } - /* Headroom is reserved by rte_pktmbuf_alloc(). */ - MLX5_ASSERT(DATA_OFF(buf) == RTE_PKTMBUF_HEADROOM); + /* Only vectored Rx routines rely on headroom size. */ + MLX5_ASSERT(!has_vec_support || + DATA_OFF(buf) >= RTE_PKTMBUF_HEADROOM); /* Buffer is supposed to be empty. */ MLX5_ASSERT(rte_pktmbuf_data_len(buf) == 0); MLX5_ASSERT(rte_pktmbuf_pkt_len(buf) == 0); @@ -174,7 +177,7 @@ rxq_alloc_elts_sprq(struct mlx5_rxq_ctrl *rxq_ctrl) (*rxq_ctrl->rxq.elts)[i] = buf; } /* If Rx vector is activated. */ - if (mlx5_rxq_check_vec_support(&rxq_ctrl->rxq) > 0) { + if (has_vec_support) { struct mlx5_rxq_data *rxq = &rxq_ctrl->rxq; struct rte_mbuf *mbuf_init = &rxq->fake_mbuf; struct rte_pktmbuf_pool_private *priv = @@ -293,8 +296,8 @@ rxq_free_elts_sprq(struct mlx5_rxq_ctrl *rxq_ctrl) { struct mlx5_rxq_data *rxq = &rxq_ctrl->rxq; const uint16_t q_n = mlx5_rxq_mprq_enabled(&rxq_ctrl->rxq) ? - (1 << rxq->elts_n) * (1 << rxq->strd_num_n) : - (1 << rxq->elts_n); + RTE_BIT32(rxq->elts_n) * RTE_BIT32(rxq->log_strd_num) : + RTE_BIT32(rxq->elts_n); const uint16_t q_mask = q_n - 1; uint16_t elts_ci = mlx5_rxq_mprq_enabled(&rxq_ctrl->rxq) ? rxq->elts_ci : rxq->rq_ci; @@ -775,11 +778,14 @@ mlx5_shared_rxq_match(struct mlx5_rxq_ctrl *rxq_ctrl, struct rte_eth_dev *dev, dev->data->port_id, idx); return false; } else if (mp == NULL) { + if (conf->rx_nseg != rxq_ctrl->rxseg_n) { + DRV_LOG(ERR, "port %u queue index %u failed to join shared group: segment number mismatch", + dev->data->port_id, idx); + return false; + } for (i = 0; i < conf->rx_nseg; i++) { - if (conf->rx_seg[i].split.mp != - rxq_ctrl->rxq.rxseg[i].mp || - conf->rx_seg[i].split.length != - rxq_ctrl->rxq.rxseg[i].length) { + if (memcmp(&conf->rx_seg[i].split, &rxq_ctrl->rxseg[i], + sizeof(struct rte_eth_rxseg_split))) { DRV_LOG(ERR, "port %u queue index %u failed to join shared group: segment %u configuration mismatch", dev->data->port_id, idx, i); return false; @@ -1271,6 +1277,8 @@ mlx5_rxq_obj_verify(struct rte_eth_dev *dev) struct mlx5_rxq_obj *rxq_obj; LIST_FOREACH(rxq_obj, &priv->rxqsobj, next) { + if (rxq_obj->rxq_ctrl == NULL) + continue; if (rxq_obj->rxq_ctrl->rxq.shared && !LIST_EMPTY(&rxq_obj->rxq_ctrl->owners)) continue; @@ -1373,8 +1381,8 @@ mlx5_mprq_alloc_mp(struct rte_eth_dev *dev) unsigned int buf_len; unsigned int obj_num; unsigned int obj_size; - unsigned int strd_num_n = 0; - unsigned int strd_sz_n = 0; + unsigned int log_strd_num = 0; + unsigned int log_strd_sz = 0; unsigned int i; unsigned int n_ibv = 0; int ret; @@ -1393,16 +1401,18 @@ mlx5_mprq_alloc_mp(struct rte_eth_dev *dev) n_ibv++; desc += 1 << rxq->elts_n; /* Get the max number of strides. */ - if (strd_num_n < rxq->strd_num_n) - strd_num_n = rxq->strd_num_n; + if (log_strd_num < rxq->log_strd_num) + log_strd_num = rxq->log_strd_num; /* Get the max size of a stride. */ - if (strd_sz_n < rxq->strd_sz_n) - strd_sz_n = rxq->strd_sz_n; - } - MLX5_ASSERT(strd_num_n && strd_sz_n); - buf_len = (1 << strd_num_n) * (1 << strd_sz_n); - obj_size = sizeof(struct mlx5_mprq_buf) + buf_len + (1 << strd_num_n) * - sizeof(struct rte_mbuf_ext_shared_info) + RTE_PKTMBUF_HEADROOM; + if (log_strd_sz < rxq->log_strd_sz) + log_strd_sz = rxq->log_strd_sz; + } + MLX5_ASSERT(log_strd_num && log_strd_sz); + buf_len = RTE_BIT32(log_strd_num) * RTE_BIT32(log_strd_sz); + obj_size = sizeof(struct mlx5_mprq_buf) + buf_len + + RTE_BIT32(log_strd_num) * + sizeof(struct rte_mbuf_ext_shared_info) + + RTE_PKTMBUF_HEADROOM; /* * Received packets can be either memcpy'd or externally referenced. In * case that the packet is attached to an mbuf as an external buffer, as @@ -1448,7 +1458,7 @@ mlx5_mprq_alloc_mp(struct rte_eth_dev *dev) snprintf(name, sizeof(name), "port-%u-mprq", dev->data->port_id); mp = rte_mempool_create(name, obj_num, obj_size, MLX5_MPRQ_MP_CACHE_SZ, 0, NULL, NULL, mlx5_mprq_buf_init, - (void *)((uintptr_t)1 << strd_num_n), + (void *)((uintptr_t)1 << log_strd_num), dev->device->numa_node, 0); if (mp == NULL) { DRV_LOG(ERR, @@ -1525,6 +1535,126 @@ mlx5_max_lro_msg_size_adjust(struct rte_eth_dev *dev, uint16_t idx, priv->max_lro_msg_size * MLX5_LRO_SEG_CHUNK_SIZE); } +/** + * Prepare both size and number of stride for Multi-Packet RQ. + * + * @param dev + * Pointer to Ethernet device. + * @param idx + * RX queue index. + * @param desc + * Number of descriptors to configure in queue. + * @param rx_seg_en + * Indicator if Rx segment enables, if so Multi-Packet RQ doesn't enable. + * @param min_mbuf_size + * Non scatter min mbuf size, max_rx_pktlen plus overhead. + * @param actual_log_stride_num + * Log number of strides to configure for this queue. + * @param actual_log_stride_size + * Log stride size to configure for this queue. + * + * @return + * 0 if Multi-Packet RQ is supported, otherwise -1. + */ +static int +mlx5_mprq_prepare(struct rte_eth_dev *dev, uint16_t idx, uint16_t desc, + bool rx_seg_en, uint32_t min_mbuf_size, + uint32_t *actual_log_stride_num, + uint32_t *actual_log_stride_size) +{ + struct mlx5_priv *priv = dev->data->dev_private; + struct mlx5_dev_config *config = &priv->config; + uint32_t log_min_stride_num = config->mprq.log_min_stride_num; + uint32_t log_max_stride_num = config->mprq.log_max_stride_num; + uint32_t log_def_stride_num = + RTE_MIN(RTE_MAX(MLX5_MPRQ_DEFAULT_LOG_STRIDE_NUM, + log_min_stride_num), + log_max_stride_num); + uint32_t log_min_stride_size = config->mprq.log_min_stride_size; + uint32_t log_max_stride_size = config->mprq.log_max_stride_size; + uint32_t log_def_stride_size = + RTE_MIN(RTE_MAX(MLX5_MPRQ_DEFAULT_LOG_STRIDE_SIZE, + log_min_stride_size), + log_max_stride_size); + uint32_t log_stride_wqe_size; + + if (mlx5_check_mprq_support(dev) != 1 || rx_seg_en) + goto unsupport; + /* Checks if chosen number of strides is in supported range. */ + if (config->mprq.log_stride_num > log_max_stride_num || + config->mprq.log_stride_num < log_min_stride_num) { + *actual_log_stride_num = log_def_stride_num; + DRV_LOG(WARNING, + "Port %u Rx queue %u number of strides for Multi-Packet RQ is out of range, setting default value (%u)", + dev->data->port_id, idx, RTE_BIT32(log_def_stride_num)); + } else { + *actual_log_stride_num = config->mprq.log_stride_num; + } + if (config->mprq.log_stride_size) { + /* Checks if chosen size of stride is in supported range. */ + if (config->mprq.log_stride_size > log_max_stride_size || + config->mprq.log_stride_size < log_min_stride_size) { + *actual_log_stride_size = log_def_stride_size; + DRV_LOG(WARNING, + "Port %u Rx queue %u size of a stride for Multi-Packet RQ is out of range, setting default value (%u)", + dev->data->port_id, idx, + RTE_BIT32(log_def_stride_size)); + } else { + *actual_log_stride_size = config->mprq.log_stride_size; + } + } else { + if (min_mbuf_size <= RTE_BIT32(log_max_stride_size)) + *actual_log_stride_size = log2above(min_mbuf_size); + else + goto unsupport; + } + log_stride_wqe_size = *actual_log_stride_num + *actual_log_stride_size; + /* Check if WQE buffer size is supported by hardware. */ + if (log_stride_wqe_size < config->mprq.log_min_stride_wqe_size) { + *actual_log_stride_num = log_def_stride_num; + *actual_log_stride_size = log_def_stride_size; + DRV_LOG(WARNING, + "Port %u Rx queue %u size of WQE buffer for Multi-Packet RQ is too small, setting default values (stride_num_n=%u, stride_size_n=%u)", + dev->data->port_id, idx, RTE_BIT32(log_def_stride_num), + RTE_BIT32(log_def_stride_size)); + log_stride_wqe_size = log_def_stride_num + log_def_stride_size; + } + MLX5_ASSERT(log_stride_wqe_size >= config->mprq.log_min_stride_wqe_size); + if (desc <= RTE_BIT32(*actual_log_stride_num)) + goto unsupport; + if (min_mbuf_size > RTE_BIT32(log_stride_wqe_size)) { + DRV_LOG(WARNING, "Port %u Rx queue %u " + "Multi-Packet RQ is unsupported, WQE buffer size (%u) " + "is smaller than min mbuf size (%u)", + dev->data->port_id, idx, RTE_BIT32(log_stride_wqe_size), + min_mbuf_size); + goto unsupport; + } + DRV_LOG(DEBUG, "Port %u Rx queue %u " + "Multi-Packet RQ is enabled strd_num_n = %u, strd_sz_n = %u", + dev->data->port_id, idx, RTE_BIT32(*actual_log_stride_num), + RTE_BIT32(*actual_log_stride_size)); + return 0; +unsupport: + if (config->mprq.enabled) + DRV_LOG(WARNING, + "Port %u MPRQ is requested but cannot be enabled\n" + " (requested: pkt_sz = %u, desc_num = %u," + " rxq_num = %u, stride_sz = %u, stride_num = %u\n" + " supported: min_rxqs_num = %u, min_buf_wqe_sz = %u" + " min_stride_sz = %u, max_stride_sz = %u).\n" + "Rx segment is %senable.", + dev->data->port_id, min_mbuf_size, desc, priv->rxqs_n, + RTE_BIT32(config->mprq.log_stride_size), + RTE_BIT32(config->mprq.log_stride_num), + config->mprq.min_rxqs_num, + RTE_BIT32(config->mprq.log_min_stride_wqe_size), + RTE_BIT32(config->mprq.log_min_stride_size), + RTE_BIT32(config->mprq.log_max_stride_size), + rx_seg_en ? "" : "not "); + return -1; +} + /** * Create a DPDK Rx queue. * @@ -1562,30 +1692,28 @@ mlx5_rxq_new(struct rte_eth_dev *dev, struct mlx5_rxq_priv *rxq, RTE_PKTMBUF_HEADROOM; unsigned int max_lro_size = 0; unsigned int first_mb_free_size = mb_len - RTE_PKTMBUF_HEADROOM; - const int mprq_en = mlx5_check_mprq_support(dev) > 0 && n_seg == 1 && - !rx_seg[0].offset && !rx_seg[0].length; - unsigned int mprq_stride_nums = config->mprq.stride_num_n ? - config->mprq.stride_num_n : MLX5_MPRQ_STRIDE_NUM_N; - unsigned int mprq_stride_size = non_scatter_min_mbuf_size <= - (1U << config->mprq.max_stride_size_n) ? - log2above(non_scatter_min_mbuf_size) : MLX5_MPRQ_STRIDE_SIZE_N; - unsigned int mprq_stride_cap = (config->mprq.stride_num_n ? - (1U << config->mprq.stride_num_n) : (1U << mprq_stride_nums)) * - (config->mprq.stride_size_n ? - (1U << config->mprq.stride_size_n) : (1U << mprq_stride_size)); + uint32_t mprq_log_actual_stride_num = 0; + uint32_t mprq_log_actual_stride_size = 0; + bool rx_seg_en = n_seg != 1 || rx_seg[0].offset || rx_seg[0].length; + const int mprq_en = !mlx5_mprq_prepare(dev, idx, desc, rx_seg_en, + non_scatter_min_mbuf_size, + &mprq_log_actual_stride_num, + &mprq_log_actual_stride_size); /* * Always allocate extra slots, even if eventually * the vector Rx will not be used. */ uint16_t desc_n = desc + config->rx_vec_en * MLX5_VPMD_DESCS_PER_LOOP; + size_t alloc_size = sizeof(*tmpl) + desc_n * sizeof(struct rte_mbuf *); const struct rte_eth_rxseg_split *qs_seg = rx_seg; unsigned int tail_len; - tmpl = mlx5_malloc(MLX5_MEM_RTE | MLX5_MEM_ZERO, - sizeof(*tmpl) + desc_n * sizeof(struct rte_mbuf *) + - (!!mprq_en) * - (desc >> mprq_stride_nums) * sizeof(struct mlx5_mprq_buf *), - 0, socket); + if (mprq_en) { + /* Trim the number of descs needed. */ + desc >>= mprq_log_actual_stride_num; + alloc_size += desc * sizeof(struct mlx5_mprq_buf *); + } + tmpl = mlx5_malloc(MLX5_MEM_RTE | MLX5_MEM_ZERO, alloc_size, 0, socket); if (!tmpl) { rte_errno = ENOMEM; return NULL; @@ -1600,6 +1728,13 @@ mlx5_rxq_new(struct rte_eth_dev *dev, struct mlx5_rxq_priv *rxq, rxq->ctrl = tmpl; LIST_INSERT_HEAD(&tmpl->owners, rxq, owner_entry); MLX5_ASSERT(n_seg && n_seg <= MLX5_MAX_RXQ_NSEG); + /* + * Save the original segment configuration in the shared queue + * descriptor for the later check on the sibling queue creation. + */ + tmpl->rxseg_n = n_seg; + rte_memcpy(tmpl->rxseg, qs_seg, + sizeof(struct rte_eth_rxseg_split) * n_seg); /* * Build the array of actual buffer offsets and lengths. * Pad with the buffers from the last memory pool if @@ -1683,43 +1818,19 @@ mlx5_rxq_new(struct rte_eth_dev *dev, struct mlx5_rxq_priv *rxq, tmpl->socket = socket; if (dev->data->dev_conf.intr_conf.rxq) tmpl->irq = 1; - /* - * This Rx queue can be configured as a Multi-Packet RQ if all of the - * following conditions are met: - * - MPRQ is enabled. - * - The number of descs is more than the number of strides. - * - max_rx_pktlen plus overhead is less than the max size - * of a stride or mprq_stride_size is specified by a user. - * Need to make sure that there are enough strides to encap - * the maximum packet size in case mprq_stride_size is set. - * Otherwise, enable Rx scatter if necessary. - */ - if (mprq_en && desc > (1U << mprq_stride_nums) && - (non_scatter_min_mbuf_size <= - (1U << config->mprq.max_stride_size_n) || - (config->mprq.stride_size_n && - non_scatter_min_mbuf_size <= mprq_stride_cap))) { + if (mprq_en) { /* TODO: Rx scatter isn't supported yet. */ tmpl->rxq.sges_n = 0; - /* Trim the number of descs needed. */ - desc >>= mprq_stride_nums; - tmpl->rxq.strd_num_n = config->mprq.stride_num_n ? - config->mprq.stride_num_n : mprq_stride_nums; - tmpl->rxq.strd_sz_n = config->mprq.stride_size_n ? - config->mprq.stride_size_n : mprq_stride_size; + tmpl->rxq.log_strd_num = mprq_log_actual_stride_num; + tmpl->rxq.log_strd_sz = mprq_log_actual_stride_size; tmpl->rxq.strd_shift_en = MLX5_MPRQ_TWO_BYTE_SHIFT; tmpl->rxq.strd_scatter_en = !!(offloads & RTE_ETH_RX_OFFLOAD_SCATTER); tmpl->rxq.mprq_max_memcpy_len = RTE_MIN(first_mb_free_size, config->mprq.max_memcpy_len); max_lro_size = RTE_MIN(max_rx_pktlen, - (1u << tmpl->rxq.strd_num_n) * - (1u << tmpl->rxq.strd_sz_n)); - DRV_LOG(DEBUG, - "port %u Rx queue %u: Multi-Packet RQ is enabled" - " strd_num_n = %u, strd_sz_n = %u", - dev->data->port_id, idx, - tmpl->rxq.strd_num_n, tmpl->rxq.strd_sz_n); + RTE_BIT32(tmpl->rxq.log_strd_num) * + RTE_BIT32(tmpl->rxq.log_strd_sz)); } else if (tmpl->rxq.rxseg_n == 1) { MLX5_ASSERT(max_rx_pktlen <= first_mb_free_size); tmpl->rxq.sges_n = 0; @@ -1753,24 +1864,6 @@ mlx5_rxq_new(struct rte_eth_dev *dev, struct mlx5_rxq_priv *rxq, tmpl->rxq.sges_n = sges_n; max_lro_size = max_rx_pktlen; } - if (config->mprq.enabled && !mlx5_rxq_mprq_enabled(&tmpl->rxq)) - DRV_LOG(WARNING, - "port %u MPRQ is requested but cannot be enabled\n" - " (requested: pkt_sz = %u, desc_num = %u," - " rxq_num = %u, stride_sz = %u, stride_num = %u\n" - " supported: min_rxqs_num = %u," - " min_stride_sz = %u, max_stride_sz = %u).", - dev->data->port_id, non_scatter_min_mbuf_size, - desc, priv->rxqs_n, - config->mprq.stride_size_n ? - (1U << config->mprq.stride_size_n) : - (1U << mprq_stride_size), - config->mprq.stride_num_n ? - (1U << config->mprq.stride_num_n) : - (1U << mprq_stride_nums), - config->mprq.min_rxqs_num, - (1U << config->mprq.min_stride_size_n), - (1U << config->mprq.max_stride_size_n)); DRV_LOG(DEBUG, "port %u maximum number of segments per packet: %u", dev->data->port_id, 1 << tmpl->rxq.sges_n); if (desc % (1 << tmpl->rxq.sges_n)) { @@ -1828,17 +1921,15 @@ mlx5_rxq_new(struct rte_eth_dev *dev, struct mlx5_rxq_priv *rxq, dev->data->port_id, tmpl->rxq.crc_present ? "disabled" : "enabled", tmpl->rxq.crc_present << 2); - /* Save port ID. */ tmpl->rxq.rss_hash = !!priv->rss_conf.rss_hf && (!!(dev->data->dev_conf.rxmode.mq_mode & RTE_ETH_MQ_RX_RSS)); + /* Save port ID. */ tmpl->rxq.port_id = dev->data->port_id; tmpl->sh = priv->sh; tmpl->rxq.mp = rx_seg[0].mp; tmpl->rxq.elts_n = log2above(desc); - tmpl->rxq.rq_repl_thresh = - MLX5_VPMD_RXQ_RPLNSH_THRESH(desc_n); - tmpl->rxq.elts = - (struct rte_mbuf *(*)[desc_n])(tmpl + 1); + tmpl->rxq.rq_repl_thresh = MLX5_VPMD_RXQ_RPLNSH_THRESH(desc_n); + tmpl->rxq.elts = (struct rte_mbuf *(*)[desc_n])(tmpl + 1); tmpl->rxq.mprq_bufs = (struct mlx5_mprq_buf *(*)[desc])(*tmpl->rxq.elts + desc_n); tmpl->rxq.idx = idx; @@ -2140,7 +2231,7 @@ mlx5_rxq_get_hairpin_conf(struct rte_eth_dev *dev, uint16_t idx) * Number of queues in the array. * * @return - * 1 if all queues in indirection table match 0 othrwise. + * 1 if all queues in indirection table match 0 otherwise. */ static int mlx5_ind_table_obj_match_queues(const struct mlx5_ind_table_obj *ind_tbl, @@ -2261,39 +2352,45 @@ mlx5_ind_table_obj_verify(struct rte_eth_dev *dev) * Pointer to Ethernet device. * @param ind_table * Indirection table to modify. + * @param ref_qs + * Whether to increment RxQ reference counters. * * @return * 0 on success, a negative errno value otherwise and rte_errno is set. */ int mlx5_ind_table_obj_setup(struct rte_eth_dev *dev, - struct mlx5_ind_table_obj *ind_tbl) + struct mlx5_ind_table_obj *ind_tbl, + bool ref_qs) { struct mlx5_priv *priv = dev->data->dev_private; uint32_t queues_n = ind_tbl->queues_n; uint16_t *queues = ind_tbl->queues; - unsigned int i, j; + unsigned int i = 0, j; int ret = 0, err; const unsigned int n = rte_is_power_of_2(queues_n) ? log2above(queues_n) : log2above(priv->config.ind_table_max_size); - for (i = 0; i != queues_n; ++i) { - if (mlx5_rxq_ref(dev, queues[i]) == NULL) { - ret = -rte_errno; - goto error; + if (ref_qs) + for (i = 0; i != queues_n; ++i) { + if (mlx5_rxq_ref(dev, queues[i]) == NULL) { + ret = -rte_errno; + goto error; + } } - } ret = priv->obj_ops.ind_table_new(dev, n, ind_tbl); if (ret) goto error; __atomic_fetch_add(&ind_tbl->refcnt, 1, __ATOMIC_RELAXED); return 0; error: - err = rte_errno; - for (j = 0; j < i; j++) - mlx5_rxq_deref(dev, ind_tbl->queues[j]); - rte_errno = err; + if (ref_qs) { + err = rte_errno; + for (j = 0; j < i; j++) + mlx5_rxq_deref(dev, queues[j]); + rte_errno = err; + } DRV_LOG(DEBUG, "Port %u cannot setup indirection table.", dev->data->port_id); return ret; @@ -2310,13 +2407,15 @@ error: * Number of queues in the array. * @param standalone * Indirection table for Standalone queue. + * @param ref_qs + * Whether to increment RxQ reference counters. * * @return * The Verbs/DevX object initialized, NULL otherwise and rte_errno is set. */ static struct mlx5_ind_table_obj * mlx5_ind_table_obj_new(struct rte_eth_dev *dev, const uint16_t *queues, - uint32_t queues_n, bool standalone) + uint32_t queues_n, bool standalone, bool ref_qs) { struct mlx5_priv *priv = dev->data->dev_private; struct mlx5_ind_table_obj *ind_tbl; @@ -2331,7 +2430,7 @@ mlx5_ind_table_obj_new(struct rte_eth_dev *dev, const uint16_t *queues, ind_tbl->queues_n = queues_n; ind_tbl->queues = (uint16_t *)(ind_tbl + 1); memcpy(ind_tbl->queues, queues, queues_n * sizeof(*queues)); - ret = mlx5_ind_table_obj_setup(dev, ind_tbl); + ret = mlx5_ind_table_obj_setup(dev, ind_tbl, ref_qs); if (ret < 0) { mlx5_free(ind_tbl); return NULL; @@ -2377,6 +2476,10 @@ mlx5_ind_table_obj_check_standalone(struct rte_eth_dev *dev __rte_unused, * Number of queues in the array. * @param standalone * Indirection table for Standalone queue. + * @param ref_new_qs + * Whether to increment new RxQ set reference counters. + * @param deref_old_qs + * Whether to decrement old RxQ set reference counters. * * @return * 0 on success, a negative errno value otherwise and rte_errno is set. @@ -2385,10 +2488,10 @@ int mlx5_ind_table_obj_modify(struct rte_eth_dev *dev, struct mlx5_ind_table_obj *ind_tbl, uint16_t *queues, const uint32_t queues_n, - bool standalone) + bool standalone, bool ref_new_qs, bool deref_old_qs) { struct mlx5_priv *priv = dev->data->dev_private; - unsigned int i; + unsigned int i = 0, j; int ret = 0, err; const unsigned int n = rte_is_power_of_2(queues_n) ? log2above(queues_n) : @@ -2398,22 +2501,30 @@ mlx5_ind_table_obj_modify(struct rte_eth_dev *dev, RTE_SET_USED(standalone); if (mlx5_ind_table_obj_check_standalone(dev, ind_tbl) < 0) return -rte_errno; - for (i = 0; i != queues_n; ++i) { - if (!mlx5_rxq_get(dev, queues[i])) { - ret = -rte_errno; - goto error; + if (ref_new_qs) + for (i = 0; i != queues_n; ++i) { + if (!mlx5_rxq_ref(dev, queues[i])) { + ret = -rte_errno; + goto error; + } } - } MLX5_ASSERT(priv->obj_ops.ind_table_modify); ret = priv->obj_ops.ind_table_modify(dev, n, queues, queues_n, ind_tbl); if (ret) goto error; + if (deref_old_qs) + for (i = 0; i < ind_tbl->queues_n; i++) + claim_nonzero(mlx5_rxq_deref(dev, ind_tbl->queues[i])); ind_tbl->queues_n = queues_n; ind_tbl->queues = queues; return 0; error: - err = rte_errno; - rte_errno = err; + if (ref_new_qs) { + err = rte_errno; + for (j = 0; j < i; j++) + mlx5_rxq_deref(dev, queues[j]); + rte_errno = err; + } DRV_LOG(DEBUG, "Port %u cannot setup indirection table.", dev->data->port_id); return ret; @@ -2434,19 +2545,17 @@ int mlx5_ind_table_obj_attach(struct rte_eth_dev *dev, struct mlx5_ind_table_obj *ind_tbl) { - unsigned int i; int ret; ret = mlx5_ind_table_obj_modify(dev, ind_tbl, ind_tbl->queues, - ind_tbl->queues_n, true); - if (ret != 0) { + ind_tbl->queues_n, + true /* standalone */, + true /* ref_new_qs */, + false /* deref_old_qs */); + if (ret != 0) DRV_LOG(ERR, "Port %u could not modify indirect table obj %p", dev->data->port_id, (void *)ind_tbl); - return ret; - } - for (i = 0; i < ind_tbl->queues_n; i++) - mlx5_rxq_ref(dev, ind_tbl->queues[i]); - return 0; + return ret; } /** @@ -2535,6 +2644,7 @@ mlx5_hrxq_modify(struct rte_eth_dev *dev, uint32_t hrxq_idx, struct mlx5_priv *priv = dev->data->dev_private; struct mlx5_hrxq *hrxq = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_HRXQ], hrxq_idx); + bool dev_started = !!dev->data->dev_started; int ret; if (!hrxq) { @@ -2555,7 +2665,7 @@ mlx5_hrxq_modify(struct rte_eth_dev *dev, uint32_t hrxq_idx, if (hrxq->standalone) { /* * Replacement of indirection table unsupported for - * stanalone hrxq objects (used by shared RSS). + * standalone hrxq objects (used by shared RSS). */ rte_errno = ENOTSUP; return -rte_errno; @@ -2563,7 +2673,8 @@ mlx5_hrxq_modify(struct rte_eth_dev *dev, uint32_t hrxq_idx, ind_tbl = mlx5_ind_table_obj_get(dev, queues, queues_n); if (!ind_tbl) ind_tbl = mlx5_ind_table_obj_new(dev, queues, queues_n, - hrxq->standalone); + hrxq->standalone, + dev_started); } if (!ind_tbl) { rte_errno = ENOMEM; @@ -2655,7 +2766,8 @@ __mlx5_hrxq_create(struct rte_eth_dev *dev, ind_tbl = mlx5_ind_table_obj_get(dev, queues, queues_n); if (!ind_tbl) ind_tbl = mlx5_ind_table_obj_new(dev, queues, queues_n, - standalone); + standalone, + !!dev->data->dev_started); if (!ind_tbl) return NULL; hrxq = mlx5_ipool_zmalloc(priv->sh->ipool[MLX5_IPOOL_HRXQ], &hrxq_idx);