X-Git-Url: http://git.droids-corp.org/?a=blobdiff_plain;f=drivers%2Fnet%2Fmlx5%2Fmlx5_rx.h;h=eda6eca8dea2ccc81542a1449c3c6fa8cd60f257;hb=09c2555303be8fa21be6b2261ca8ec262a21e201;hp=d5a2de84d10aaef9aff7c4b0e571ece02be1d754;hpb=a96102c8698aa61dcb72d392a6c844323b8286a5;p=dpdk.git diff --git a/drivers/net/mlx5/mlx5_rx.h b/drivers/net/mlx5/mlx5_rx.h index d5a2de84d1..eda6eca8de 100644 --- a/drivers/net/mlx5/mlx5_rx.h +++ b/drivers/net/mlx5/mlx5_rx.h @@ -18,11 +18,17 @@ #include "mlx5.h" #include "mlx5_autoconf.h" -#include "mlx5_mr.h" /* Support tunnel matching. */ #define MLX5_FLOW_TUNNEL 10 +#define RXQ_PORT(rxq_ctrl) LIST_FIRST(&(rxq_ctrl)->owners)->priv +#define RXQ_DEV(rxq_ctrl) ETH_DEV(RXQ_PORT(rxq_ctrl)) +#define RXQ_PORT_ID(rxq_ctrl) PORT_ID(RXQ_PORT(rxq_ctrl)) + +/* First entry must be NULL for comparison. */ +#define mlx5_mr_btree_len(bt) ((bt)->len - 1) + struct mlx5_rxq_stats { #ifdef MLX5_PMD_SOFT_COUNTERS uint64_t ipackets; /**< Total of successfully received packets. */ @@ -41,19 +47,6 @@ struct rxq_zip { uint32_t cqe_cnt; /* Number of CQEs. */ }; -/* Multi-Packet RQ buffer header. */ -struct mlx5_mprq_buf { - struct rte_mempool *mp; - uint16_t refcnt; /* Atomically accessed refcnt. */ - uint8_t pad[RTE_PKTMBUF_HEADROOM]; /* Headroom for the first packet. */ - struct rte_mbuf_ext_shared_info shinfos[]; - /* - * Shared information per stride. - * More memory will be allocated for the first stride head-room and for - * the strides data. - */ -} __rte_cache_aligned; - /* Get pointer to the first stride. */ #define mlx5_mprq_buf_addr(ptr, strd_n) (RTE_PTR_ADD((ptr), \ sizeof(struct mlx5_mprq_buf) + \ @@ -103,6 +96,7 @@ struct mlx5_rxq_data { unsigned int lro:1; /* Enable LRO. */ unsigned int dynf_meta:1; /* Dynamic metadata is configured. */ unsigned int mcqe_format:3; /* CQE compression format. */ + unsigned int shared:1; /* Shared RXQ. */ volatile uint32_t *rq_db; volatile uint32_t *cq_db; uint16_t port_id; @@ -160,16 +154,30 @@ enum mlx5_rxq_type { struct mlx5_rxq_ctrl { struct mlx5_rxq_data rxq; /* Data path structure. */ LIST_ENTRY(mlx5_rxq_ctrl) next; /* Pointer to the next element. */ - uint32_t refcnt; /* Reference counter. */ + LIST_HEAD(priv, mlx5_rxq_priv) owners; /* Owner rxq list. */ struct mlx5_rxq_obj *obj; /* Verbs/DevX elements. */ - struct mlx5_priv *priv; /* Back pointer to private data. */ + struct mlx5_dev_ctx_shared *sh; /* Shared context. */ enum mlx5_rxq_type type; /* Rxq type. */ unsigned int socket; /* CPU socket ID for allocations. */ + LIST_ENTRY(mlx5_rxq_ctrl) share_entry; /* Entry in shared RXQ list. */ + uint32_t share_group; /* Group ID of shared RXQ. */ + uint16_t share_qid; /* Shared RxQ ID in group. */ + unsigned int started:1; /* Whether (shared) RXQ has been started. */ unsigned int irq:1; /* Whether IRQ is enabled. */ uint32_t flow_mark_n; /* Number of Mark/Flag flows using this Queue. */ uint32_t flow_tunnels_n[MLX5_FLOW_TUNNEL]; /* Tunnels counters. */ uint32_t wqn; /* WQ number. */ uint16_t dump_file_n; /* Number of dump files. */ +}; + +/* RX queue private data. */ +struct mlx5_rxq_priv { + uint16_t idx; /* Queue index. */ + uint32_t refcnt; /* Reference counter. */ + struct mlx5_rxq_ctrl *ctrl; /* Shared Rx Queue. */ + LIST_ENTRY(mlx5_rxq_priv) owner_entry; /* Entry in shared rxq_ctrl. */ + struct mlx5_priv *priv; /* Back pointer to private data. */ + struct mlx5_devx_rq devx_rq; struct rte_eth_hairpin_conf hairpin_conf; /* Hairpin configuration. */ uint32_t hairpin_status; /* Hairpin binding status. */ }; @@ -191,21 +199,26 @@ int mlx5_rx_queue_setup(struct rte_eth_dev *dev, uint16_t idx, uint16_t desc, int mlx5_rx_hairpin_queue_setup (struct rte_eth_dev *dev, uint16_t idx, uint16_t desc, const struct rte_eth_hairpin_conf *hairpin_conf); -void mlx5_rx_queue_release(void *dpdk_rxq); +void mlx5_rx_queue_release(struct rte_eth_dev *dev, uint16_t qid); int mlx5_rx_intr_vec_enable(struct rte_eth_dev *dev); void mlx5_rx_intr_vec_disable(struct rte_eth_dev *dev); int mlx5_rx_intr_enable(struct rte_eth_dev *dev, uint16_t rx_queue_id); int mlx5_rx_intr_disable(struct rte_eth_dev *dev, uint16_t rx_queue_id); int mlx5_rxq_obj_verify(struct rte_eth_dev *dev); -struct mlx5_rxq_ctrl *mlx5_rxq_new(struct rte_eth_dev *dev, uint16_t idx, +struct mlx5_rxq_ctrl *mlx5_rxq_new(struct rte_eth_dev *dev, + struct mlx5_rxq_priv *rxq, uint16_t desc, unsigned int socket, const struct rte_eth_rxconf *conf, const struct rte_eth_rxseg_split *rx_seg, uint16_t n_seg); struct mlx5_rxq_ctrl *mlx5_rxq_hairpin_new - (struct rte_eth_dev *dev, uint16_t idx, uint16_t desc, + (struct rte_eth_dev *dev, struct mlx5_rxq_priv *rxq, uint16_t desc, const struct rte_eth_hairpin_conf *hairpin_conf); -struct mlx5_rxq_ctrl *mlx5_rxq_get(struct rte_eth_dev *dev, uint16_t idx); +struct mlx5_rxq_priv *mlx5_rxq_ref(struct rte_eth_dev *dev, uint16_t idx); +uint32_t mlx5_rxq_deref(struct rte_eth_dev *dev, uint16_t idx); +struct mlx5_rxq_priv *mlx5_rxq_get(struct rte_eth_dev *dev, uint16_t idx); +struct mlx5_rxq_ctrl *mlx5_rxq_ctrl_get(struct rte_eth_dev *dev, uint16_t idx); +struct mlx5_rxq_data *mlx5_rxq_data_get(struct rte_eth_dev *dev, uint16_t idx); int mlx5_rxq_release(struct rte_eth_dev *dev, uint16_t idx); int mlx5_rxq_verify(struct rte_eth_dev *dev); int rxq_alloc_elts(struct mlx5_rxq_ctrl *rxq_ctrl); @@ -222,13 +235,19 @@ int mlx5_ind_table_obj_modify(struct rte_eth_dev *dev, struct mlx5_ind_table_obj *ind_tbl, uint16_t *queues, const uint32_t queues_n, bool standalone); -struct mlx5_cache_entry *mlx5_hrxq_create_cb(struct mlx5_cache_list *list, - struct mlx5_cache_entry *entry __rte_unused, void *cb_ctx); -int mlx5_hrxq_match_cb(struct mlx5_cache_list *list, - struct mlx5_cache_entry *entry, +int mlx5_ind_table_obj_attach(struct rte_eth_dev *dev, + struct mlx5_ind_table_obj *ind_tbl); +int mlx5_ind_table_obj_detach(struct rte_eth_dev *dev, + struct mlx5_ind_table_obj *ind_tbl); +struct mlx5_list_entry *mlx5_hrxq_create_cb(void *tool_ctx, void *cb_ctx); +int mlx5_hrxq_match_cb(void *tool_ctx, struct mlx5_list_entry *entry, void *cb_ctx); -void mlx5_hrxq_remove_cb(struct mlx5_cache_list *list, - struct mlx5_cache_entry *entry); +void mlx5_hrxq_remove_cb(void *tool_ctx, struct mlx5_list_entry *entry); +struct mlx5_list_entry *mlx5_hrxq_clone_cb(void *tool_ctx, + struct mlx5_list_entry *entry, + void *cb_ctx __rte_unused); +void mlx5_hrxq_clone_free_cb(void *tool_ctx __rte_unused, + struct mlx5_list_entry *entry); uint32_t mlx5_hrxq_get(struct rte_eth_dev *dev, struct mlx5_flow_rss_desc *rss_desc); int mlx5_hrxq_release(struct rte_eth_dev *dev, uint32_t hxrq_idx); @@ -251,18 +270,18 @@ int mlx5_hrxq_modify(struct rte_eth_dev *dev, uint32_t hxrq_idx, uint16_t mlx5_rx_burst(void *dpdk_rxq, struct rte_mbuf **pkts, uint16_t pkts_n); void mlx5_rxq_initialize(struct mlx5_rxq_data *rxq); __rte_noinline int mlx5_rx_err_handle(struct mlx5_rxq_data *rxq, uint8_t vec); -void mlx5_mprq_buf_free_cb(void *addr, void *opaque); void mlx5_mprq_buf_free(struct mlx5_mprq_buf *buf); uint16_t mlx5_rx_burst_mprq(void *dpdk_rxq, struct rte_mbuf **pkts, uint16_t pkts_n); uint16_t removed_rx_burst(void *dpdk_rxq, struct rte_mbuf **pkts, uint16_t pkts_n); int mlx5_rx_descriptor_status(void *rx_queue, uint16_t offset); -uint32_t mlx5_rx_queue_count(struct rte_eth_dev *dev, uint16_t rx_queue_id); +uint32_t mlx5_rx_queue_count(void *rx_queue); void mlx5_rxq_info_get(struct rte_eth_dev *dev, uint16_t queue_id, struct rte_eth_rxq_info *qinfo); int mlx5_rx_burst_mode_get(struct rte_eth_dev *dev, uint16_t rx_queue_id, struct rte_eth_burst_mode *mode); +int mlx5_get_monitor_addr(void *rx_queue, struct rte_power_monitor_cond *pmc); /* Vectorized version of mlx5_rx.c */ int mlx5_rxq_check_vec_support(struct mlx5_rxq_data *rxq_data); @@ -272,13 +291,11 @@ uint16_t mlx5_rx_burst_vec(void *dpdk_rxq, struct rte_mbuf **pkts, uint16_t mlx5_rx_burst_mprq_vec(void *dpdk_rxq, struct rte_mbuf **pkts, uint16_t pkts_n); -/* mlx5_mr.c */ - -uint32_t mlx5_rx_addr2mr_bh(struct mlx5_rxq_data *rxq, uintptr_t addr); +static int mlx5_rxq_mprq_enabled(struct mlx5_rxq_data *rxq); /** - * Query LKey from a packet buffer for Rx. No need to flush local caches for Rx - * as mempool is pre-configured and static. + * Query LKey from a packet buffer for Rx. No need to flush local caches + * as the Rx mempool database entries are valid for the lifetime of the queue. * * @param rxq * Pointer to Rx queue structure. @@ -287,11 +304,14 @@ uint32_t mlx5_rx_addr2mr_bh(struct mlx5_rxq_data *rxq, uintptr_t addr); * * @return * Searched LKey on success, UINT32_MAX on no match. + * This function always succeeds on valid input. */ static __rte_always_inline uint32_t mlx5_rx_addr2mr(struct mlx5_rxq_data *rxq, uintptr_t addr) { struct mlx5_mr_ctrl *mr_ctrl = &rxq->mr_ctrl; + struct mlx5_rxq_ctrl *rxq_ctrl; + struct rte_mempool *mp; uint32_t lkey; /* Linear search on MR cache array. */ @@ -299,8 +319,14 @@ mlx5_rx_addr2mr(struct mlx5_rxq_data *rxq, uintptr_t addr) MLX5_MR_CACHE_N, addr); if (likely(lkey != UINT32_MAX)) return lkey; - /* Take slower bottom-half (Binary Search) on miss. */ - return mlx5_rx_addr2mr_bh(rxq, addr); + /* + * Slower search in the mempool database on miss. + * During queue creation rxq->sh is not yet set, so we use rxq_ctrl. + */ + rxq_ctrl = container_of(rxq, struct mlx5_rxq_ctrl, rxq); + mp = mlx5_rxq_mprq_enabled(rxq) ? rxq->mprq_mp : rxq->mp; + return mlx5_mr_mempool2mr_bh(&rxq_ctrl->sh->cdev->mr_scache, + mr_ctrl, mp, addr); } #define mlx5_rx_mb2mr(rxq, mb) mlx5_rx_addr2mr(rxq, (uintptr_t)((mb)->buf_addr)) @@ -485,7 +511,7 @@ mprq_buf_to_pkt(struct mlx5_rxq_data *rxq, struct rte_mbuf *pkt, uint32_t len, shinfo = &buf->shinfos[strd_idx]; rte_mbuf_ext_refcnt_set(shinfo, 1); /* - * EXT_ATTACHED_MBUF will be set to pkt->ol_flags when + * RTE_MBUF_F_EXTERNAL will be set to pkt->ol_flags when * attaching the stride to mbuf and more offload flags * will be added below by calling rxq_cq_to_mbuf(). * Other fields will be overwritten. @@ -494,7 +520,7 @@ mprq_buf_to_pkt(struct mlx5_rxq_data *rxq, struct rte_mbuf *pkt, uint32_t len, buf_len, shinfo); /* Set mbuf head-room. */ SET_DATA_OFF(pkt, RTE_PKTMBUF_HEADROOM); - MLX5_ASSERT(pkt->ol_flags == EXT_ATTACHED_MBUF); + MLX5_ASSERT(pkt->ol_flags == RTE_MBUF_F_EXTERNAL); MLX5_ASSERT(rte_pktmbuf_tailroom(pkt) >= len - (hdrm_overlap > 0 ? hdrm_overlap : 0)); DATA_LEN(pkt) = len; @@ -580,14 +606,13 @@ mlx5_mprq_enabled(struct rte_eth_dev *dev) return 0; /* All the configured queues should be enabled. */ for (i = 0; i < priv->rxqs_n; ++i) { - struct mlx5_rxq_data *rxq = (*priv->rxqs)[i]; - struct mlx5_rxq_ctrl *rxq_ctrl = container_of - (rxq, struct mlx5_rxq_ctrl, rxq); + struct mlx5_rxq_ctrl *rxq_ctrl = mlx5_rxq_ctrl_get(dev, i); - if (rxq == NULL || rxq_ctrl->type != MLX5_RXQ_TYPE_STANDARD) + if (rxq_ctrl == NULL || + rxq_ctrl->type != MLX5_RXQ_TYPE_STANDARD) continue; n_ibv++; - if (mlx5_rxq_mprq_enabled(rxq)) + if (mlx5_rxq_mprq_enabled(&rxq_ctrl->rxq)) ++n; } /* Multi-Packet RQ can't be partially configured. */