X-Git-Url: http://git.droids-corp.org/?a=blobdiff_plain;f=drivers%2Fnet%2Fmlx5%2Fmlx5_rx.h;h=38335fd744828e45341b9141570f98c018cef122;hb=42431df9240da9c8e30ad61dc9fb3e00b6eb0284;hp=c04c0c733498e79fd9b03f1afdc663d9fbc7bdd9;hpb=5db77fef78b4bf3559d1f5e83f905f9be729c78b;p=dpdk.git diff --git a/drivers/net/mlx5/mlx5_rx.h b/drivers/net/mlx5/mlx5_rx.h index c04c0c7334..38335fd744 100644 --- a/drivers/net/mlx5/mlx5_rx.h +++ b/drivers/net/mlx5/mlx5_rx.h @@ -88,14 +88,16 @@ struct mlx5_rxq_data { unsigned int elts_n:4; /* Log 2 of Mbufs. */ unsigned int rss_hash:1; /* RSS hash result is enabled. */ unsigned int mark:1; /* Marked flow available on the queue. */ - unsigned int strd_num_n:5; /* Log 2 of the number of stride. */ - unsigned int strd_sz_n:4; /* Log 2 of stride size. */ + unsigned int log_strd_num:5; /* Log 2 of the number of stride. */ + unsigned int log_strd_sz:4; /* Log 2 of stride size. */ unsigned int strd_shift_en:1; /* Enable 2bytes shift on a stride. */ unsigned int err_state:2; /* enum mlx5_rxq_err_state. */ unsigned int strd_scatter_en:1; /* Scattered packets from a stride. */ unsigned int lro:1; /* Enable LRO. */ unsigned int dynf_meta:1; /* Dynamic metadata is configured. */ unsigned int mcqe_format:3; /* CQE compression format. */ + unsigned int shared:1; /* Shared RXQ. */ + unsigned int delay_drop:1; /* Enable delay drop. */ volatile uint32_t *rq_db; volatile uint32_t *cq_db; uint16_t port_id; @@ -125,13 +127,9 @@ struct mlx5_rxq_data { struct mlx5_rxq_stats stats; rte_xmm_t mbuf_initializer; /* Default rearm/flags for vectorized Rx. */ struct rte_mbuf fake_mbuf; /* elts padding for vectorized Rx. */ - void *cq_uar; /* Verbs CQ user access region. */ + struct mlx5_uar_data uar_data; /* CQ doorbell. */ uint32_t cqn; /* CQ number. */ uint8_t cq_arm_sn; /* CQ arm seq number. */ -#ifndef RTE_ARCH_64 - rte_spinlock_t *uar_lock_cq; - /* CQ (UAR) access lock required for 32bit implementations */ -#endif uint32_t tunnel; /* Tunnel information. */ int timestamp_offset; /* Dynamic mbuf field for timestamp. */ uint64_t timestamp_rx_flag; /* Dynamic mbuf flag for timestamp. */ @@ -158,12 +156,16 @@ struct mlx5_rxq_ctrl { struct mlx5_dev_ctx_shared *sh; /* Shared context. */ enum mlx5_rxq_type type; /* Rxq type. */ unsigned int socket; /* CPU socket ID for allocations. */ + LIST_ENTRY(mlx5_rxq_ctrl) share_entry; /* Entry in shared RXQ list. */ uint32_t share_group; /* Group ID of shared RXQ. */ uint16_t share_qid; /* Shared RxQ ID in group. */ + unsigned int started:1; /* Whether (shared) RXQ has been started. */ unsigned int irq:1; /* Whether IRQ is enabled. */ - uint32_t flow_mark_n; /* Number of Mark/Flag flows using this Queue. */ uint32_t flow_tunnels_n[MLX5_FLOW_TUNNEL]; /* Tunnels counters. */ uint32_t wqn; /* WQ number. */ + uint32_t rxseg_n; /* Number of split segment descriptions. */ + struct rte_eth_rxseg_split rxseg[MLX5_MAX_RXQ_NSEG]; + /* Saved original buffer split segment configuration. */ uint16_t dump_file_n; /* Number of dump files. */ }; @@ -174,6 +176,7 @@ struct mlx5_rxq_priv { struct mlx5_rxq_ctrl *ctrl; /* Shared Rx Queue. */ LIST_ENTRY(mlx5_rxq_priv) owner_entry; /* Entry in shared rxq_ctrl. */ struct mlx5_priv *priv; /* Back pointer to private data. */ + struct mlx5_devx_rq devx_rq; struct rte_eth_hairpin_conf hairpin_conf; /* Hairpin configuration. */ uint32_t hairpin_status; /* Hairpin binding status. */ }; @@ -201,8 +204,7 @@ void mlx5_rx_intr_vec_disable(struct rte_eth_dev *dev); int mlx5_rx_intr_enable(struct rte_eth_dev *dev, uint16_t rx_queue_id); int mlx5_rx_intr_disable(struct rte_eth_dev *dev, uint16_t rx_queue_id); int mlx5_rxq_obj_verify(struct rte_eth_dev *dev); -struct mlx5_rxq_ctrl *mlx5_rxq_new(struct rte_eth_dev *dev, - struct mlx5_rxq_priv *rxq, +struct mlx5_rxq_ctrl *mlx5_rxq_new(struct rte_eth_dev *dev, uint16_t idx, uint16_t desc, unsigned int socket, const struct rte_eth_rxconf *conf, const struct rte_eth_rxseg_split *rx_seg, @@ -224,13 +226,16 @@ struct mlx5_ind_table_obj *mlx5_ind_table_obj_get(struct rte_eth_dev *dev, uint32_t queues_n); int mlx5_ind_table_obj_release(struct rte_eth_dev *dev, struct mlx5_ind_table_obj *ind_tbl, - bool standalone); + bool standalone, + bool deref_rxqs); int mlx5_ind_table_obj_setup(struct rte_eth_dev *dev, - struct mlx5_ind_table_obj *ind_tbl); + struct mlx5_ind_table_obj *ind_tbl, + bool ref_qs); int mlx5_ind_table_obj_modify(struct rte_eth_dev *dev, struct mlx5_ind_table_obj *ind_tbl, uint16_t *queues, const uint32_t queues_n, - bool standalone); + bool standalone, + bool ref_new_qs, bool deref_old_qs); int mlx5_ind_table_obj_attach(struct rte_eth_dev *dev, struct mlx5_ind_table_obj *ind_tbl); int mlx5_ind_table_obj_detach(struct rte_eth_dev *dev, @@ -269,8 +274,6 @@ __rte_noinline int mlx5_rx_err_handle(struct mlx5_rxq_data *rxq, uint8_t vec); void mlx5_mprq_buf_free(struct mlx5_mprq_buf *buf); uint16_t mlx5_rx_burst_mprq(void *dpdk_rxq, struct rte_mbuf **pkts, uint16_t pkts_n); -uint16_t removed_rx_burst(void *dpdk_rxq, struct rte_mbuf **pkts, - uint16_t pkts_n); int mlx5_rx_descriptor_status(void *rx_queue, uint16_t offset); uint32_t mlx5_rx_queue_count(void *rx_queue); void mlx5_rxq_info_get(struct rte_eth_dev *dev, uint16_t queue_id, @@ -290,7 +293,7 @@ uint16_t mlx5_rx_burst_mprq_vec(void *dpdk_rxq, struct rte_mbuf **pkts, static int mlx5_rxq_mprq_enabled(struct mlx5_rxq_data *rxq); /** - * Query LKey from a packet buffer for Rx. No need to flush local caches + * Query LKey for an address on Rx. No need to flush local caches * as the Rx mempool database entries are valid for the lifetime of the queue. * * @param rxq @@ -306,7 +309,6 @@ static __rte_always_inline uint32_t mlx5_rx_addr2mr(struct mlx5_rxq_data *rxq, uintptr_t addr) { struct mlx5_mr_ctrl *mr_ctrl = &rxq->mr_ctrl; - struct mlx5_rxq_ctrl *rxq_ctrl; struct rte_mempool *mp; uint32_t lkey; @@ -315,17 +317,38 @@ mlx5_rx_addr2mr(struct mlx5_rxq_data *rxq, uintptr_t addr) MLX5_MR_CACHE_N, addr); if (likely(lkey != UINT32_MAX)) return lkey; - /* - * Slower search in the mempool database on miss. - * During queue creation rxq->sh is not yet set, so we use rxq_ctrl. - */ - rxq_ctrl = container_of(rxq, struct mlx5_rxq_ctrl, rxq); mp = mlx5_rxq_mprq_enabled(rxq) ? rxq->mprq_mp : rxq->mp; - return mlx5_mr_mempool2mr_bh(&rxq_ctrl->sh->cdev->mr_scache, - mr_ctrl, mp, addr); + return mlx5_mr_mempool2mr_bh(mr_ctrl, mp, addr); } -#define mlx5_rx_mb2mr(rxq, mb) mlx5_rx_addr2mr(rxq, (uintptr_t)((mb)->buf_addr)) +/** + * Query LKey from a packet buffer for Rx. No need to flush local caches + * as the Rx mempool database entries are valid for the lifetime of the queue. + * + * @param rxq + * Pointer to Rx queue structure. + * @param mb + * Buffer to search the address of. + * + * @return + * Searched LKey on success, UINT32_MAX on no match. + * This function always succeeds on valid input. + */ +static __rte_always_inline uint32_t +mlx5_rx_mb2mr(struct mlx5_rxq_data *rxq, struct rte_mbuf *mb) +{ + struct mlx5_mr_ctrl *mr_ctrl = &rxq->mr_ctrl; + uintptr_t addr = (uintptr_t)mb->buf_addr; + uint32_t lkey; + + /* Linear search on MR cache array. */ + lkey = mlx5_mr_lookup_lkey(mr_ctrl->cache, &mr_ctrl->mru, + MLX5_MR_CACHE_N, addr); + if (likely(lkey != UINT32_MAX)) + return lkey; + /* Slower search in the mempool database on miss. */ + return mlx5_mr_mempool2mr_bh(mr_ctrl, mb->pool, addr); +} /** * Convert timestamp from HW format to linear counter @@ -374,7 +397,7 @@ mlx5_timestamp_set(struct rte_mbuf *mbuf, int offset, static __rte_always_inline void mprq_buf_replace(struct mlx5_rxq_data *rxq, uint16_t rq_idx) { - const uint32_t strd_n = 1 << rxq->strd_num_n; + const uint32_t strd_n = RTE_BIT32(rxq->log_strd_num); struct mlx5_mprq_buf *rep = rxq->mprq_repl; volatile struct mlx5_wqe_data_seg *wqe = &((volatile struct mlx5_wqe_mprq *)rxq->wqes)[rq_idx].dseg; @@ -432,8 +455,8 @@ static __rte_always_inline enum mlx5_rqx_code mprq_buf_to_pkt(struct mlx5_rxq_data *rxq, struct rte_mbuf *pkt, uint32_t len, struct mlx5_mprq_buf *buf, uint16_t strd_idx, uint16_t strd_cnt) { - const uint32_t strd_n = 1 << rxq->strd_num_n; - const uint16_t strd_sz = 1 << rxq->strd_sz_n; + const uint32_t strd_n = RTE_BIT32(rxq->log_strd_num); + const uint16_t strd_sz = RTE_BIT32(rxq->log_strd_sz); const uint16_t strd_shift = MLX5_MPRQ_STRIDE_SHIFT_BYTE * rxq->strd_shift_en; const int32_t hdrm_overlap = @@ -516,7 +539,7 @@ mprq_buf_to_pkt(struct mlx5_rxq_data *rxq, struct rte_mbuf *pkt, uint32_t len, buf_len, shinfo); /* Set mbuf head-room. */ SET_DATA_OFF(pkt, RTE_PKTMBUF_HEADROOM); - MLX5_ASSERT(pkt->ol_flags == RTE_MBUF_F_EXTERNAL); + MLX5_ASSERT(pkt->ol_flags & RTE_MBUF_F_EXTERNAL); MLX5_ASSERT(rte_pktmbuf_tailroom(pkt) >= len - (hdrm_overlap > 0 ? hdrm_overlap : 0)); DATA_LEN(pkt) = len; @@ -578,7 +601,7 @@ mlx5_check_mprq_support(struct rte_eth_dev *dev) static __rte_always_inline int mlx5_rxq_mprq_enabled(struct mlx5_rxq_data *rxq) { - return rxq->strd_num_n > 0; + return rxq->log_strd_num > 0; } /** @@ -602,14 +625,13 @@ mlx5_mprq_enabled(struct rte_eth_dev *dev) return 0; /* All the configured queues should be enabled. */ for (i = 0; i < priv->rxqs_n; ++i) { - struct mlx5_rxq_data *rxq = (*priv->rxqs)[i]; - struct mlx5_rxq_ctrl *rxq_ctrl = container_of - (rxq, struct mlx5_rxq_ctrl, rxq); + struct mlx5_rxq_ctrl *rxq_ctrl = mlx5_rxq_ctrl_get(dev, i); - if (rxq == NULL || rxq_ctrl->type != MLX5_RXQ_TYPE_STANDARD) + if (rxq_ctrl == NULL || + rxq_ctrl->type != MLX5_RXQ_TYPE_STANDARD) continue; n_ibv++; - if (mlx5_rxq_mprq_enabled(rxq)) + if (mlx5_rxq_mprq_enabled(&rxq_ctrl->rxq)) ++n; } /* Multi-Packet RQ can't be partially configured. */