#include <rte_spinlock.h>
#include <rte_io.h>
#include <rte_bus_pci.h>
+#include <rte_malloc.h>
#include "mlx5_utils.h"
#include "mlx5.h"
#include "mlx5_glue.h"
/* Support tunnel matching. */
-#define MLX5_FLOW_TUNNEL 5
+#define MLX5_FLOW_TUNNEL 6
struct mlx5_rxq_stats {
#ifdef MLX5_PMD_SOFT_COUNTERS
struct rte_mempool *mp;
rte_atomic16_t refcnt; /* Atomically accessed refcnt. */
uint8_t pad[RTE_PKTMBUF_HEADROOM]; /* Headroom for the first packet. */
+ struct rte_mbuf_ext_shared_info shinfos[];
+ /*
+ * Shared information per stride.
+ * More memory will be allocated for the first stride head-room and for
+ * the strides data.
+ */
} __rte_cache_aligned;
/* Get pointer to the first stride. */
-#define mlx5_mprq_buf_addr(ptr) ((ptr) + 1)
+#define mlx5_mprq_buf_addr(ptr, strd_n) (RTE_PTR_ADD((ptr), \
+ sizeof(struct mlx5_mprq_buf) + \
+ (strd_n) * \
+ sizeof(struct rte_mbuf_ext_shared_info) + \
+ RTE_PKTMBUF_HEADROOM))
+
+#define MLX5_MIN_SINGLE_STRIDE_LOG_NUM_BYTES 6
+#define MLX5_MIN_SINGLE_WQE_LOG_NUM_STRIDES 9
enum mlx5_rxq_err_state {
MLX5_RXQ_ERR_STATE_NO_ERROR = 0,
unsigned int hw_timestamp:1; /* Enable HW timestamp. */
unsigned int vlan_strip:1; /* Enable VLAN stripping. */
unsigned int crc_present:1; /* CRC must be subtracted. */
- unsigned int sges_n:2; /* Log 2 of SGEs (max buffers per packet). */
+ unsigned int sges_n:3; /* Log 2 of SGEs (max buffers per packet). */
unsigned int cqe_n:4; /* Log 2 of CQ elements. */
unsigned int elts_n:4; /* Log 2 of Mbufs. */
unsigned int rss_hash:1; /* RSS hash result is enabled. */
unsigned int strd_sz_n:4; /* Log 2 of stride size. */
unsigned int strd_shift_en:1; /* Enable 2bytes shift on a stride. */
unsigned int err_state:2; /* enum mlx5_rxq_err_state. */
- unsigned int :4; /* Remaining bits. */
+ unsigned int strd_headroom_en:1; /* Enable mbuf headroom in MPRQ. */
+ unsigned int :2; /* Remaining bits. */
volatile uint32_t *rq_db;
volatile uint32_t *cq_db;
uint16_t port_id;
uint32_t tunnel; /* Tunnel information. */
} __rte_cache_aligned;
-/* Verbs Rx queue elements. */
-struct mlx5_rxq_ibv {
- LIST_ENTRY(mlx5_rxq_ibv) next; /* Pointer to the next element. */
+enum mlx5_rxq_obj_type {
+ MLX5_RXQ_OBJ_TYPE_IBV, /* mlx5_rxq_obj with ibv_wq. */
+ MLX5_RXQ_OBJ_TYPE_DEVX_RQ, /* mlx5_rxq_obj with mlx5_devx_rq. */
+};
+
+/* Verbs/DevX Rx queue elements. */
+struct mlx5_rxq_obj {
+ LIST_ENTRY(mlx5_rxq_obj) next; /* Pointer to the next element. */
rte_atomic32_t refcnt; /* Reference counter. */
struct mlx5_rxq_ctrl *rxq_ctrl; /* Back pointer to parent. */
struct ibv_cq *cq; /* Completion Queue. */
- struct ibv_wq *wq; /* Work Queue. */
+ enum mlx5_rxq_obj_type type;
+ RTE_STD_C11
+ union {
+ struct ibv_wq *wq; /* Work Queue. */
+ struct mlx5_devx_obj *rq; /* DevX object for Rx Queue. */
+ };
struct ibv_comp_channel *channel;
};
struct mlx5_rxq_data rxq; /* Data path structure. */
LIST_ENTRY(mlx5_rxq_ctrl) next; /* Pointer to the next element. */
rte_atomic32_t refcnt; /* Reference counter. */
- struct mlx5_rxq_ibv *ibv; /* Verbs elements. */
+ struct mlx5_rxq_obj *obj; /* Verbs/DevX elements. */
struct mlx5_priv *priv; /* Back pointer to private data. */
unsigned int socket; /* CPU socket ID for allocations. */
unsigned int irq:1; /* Whether IRQ is enabled. */
+ unsigned int dbr_umem_id_valid:1; /* dbr_umem_id holds a valid value. */
uint32_t flow_mark_n; /* Number of Mark/Flag flows using this Queue. */
uint32_t flow_tunnels_n[MLX5_FLOW_TUNNEL]; /* Tunnels counters. */
uint32_t wqn; /* WQ number. */
uint16_t dump_file_n; /* Number of dump files. */
+ uint32_t dbr_umem_id; /* Storing door-bell information, */
+ uint64_t dbr_offset; /* needed when freeing door-bell. */
+ struct mlx5dv_devx_umem *wq_umem; /* WQ buffer registration info. */
+};
+
+enum mlx5_ind_tbl_type {
+ MLX5_IND_TBL_TYPE_IBV,
+ MLX5_IND_TBL_TYPE_DEVX,
};
/* Indirection table. */
-struct mlx5_ind_table_ibv {
- LIST_ENTRY(mlx5_ind_table_ibv) next; /* Pointer to the next element. */
+struct mlx5_ind_table_obj {
+ LIST_ENTRY(mlx5_ind_table_obj) next; /* Pointer to the next element. */
rte_atomic32_t refcnt; /* Reference counter. */
- struct ibv_rwq_ind_table *ind_table; /**< Indirection table. */
+ enum mlx5_ind_tbl_type type;
+ RTE_STD_C11
+ union {
+ struct ibv_rwq_ind_table *ind_table; /**< Indirection table. */
+ struct mlx5_devx_obj *rqt; /* DevX RQT object. */
+ };
uint32_t queues_n; /**< Number of queues in the list. */
uint16_t queues[]; /**< Queue list. */
};
struct mlx5_hrxq {
LIST_ENTRY(mlx5_hrxq) next; /* Pointer to the next element. */
rte_atomic32_t refcnt; /* Reference counter. */
- struct mlx5_ind_table_ibv *ind_table; /* Indirection table. */
- struct ibv_qp *qp; /* Verbs queue pair. */
+ struct mlx5_ind_table_obj *ind_table; /* Indirection table. */
+ RTE_STD_C11
+ union {
+ struct ibv_qp *qp; /* Verbs queue pair. */
+ struct mlx5_devx_obj *tir; /* DevX TIR object. */
+ };
#ifdef HAVE_IBV_FLOW_DV_SUPPORT
void *action; /* DV QP action pointer. */
#endif
void mlx5_rx_intr_vec_disable(struct rte_eth_dev *dev);
int mlx5_rx_intr_enable(struct rte_eth_dev *dev, uint16_t rx_queue_id);
int mlx5_rx_intr_disable(struct rte_eth_dev *dev, uint16_t rx_queue_id);
-struct mlx5_rxq_ibv *mlx5_rxq_ibv_new(struct rte_eth_dev *dev, uint16_t idx);
-int mlx5_rxq_ibv_verify(struct rte_eth_dev *dev);
+struct mlx5_rxq_obj *mlx5_rxq_obj_new(struct rte_eth_dev *dev, uint16_t idx,
+ enum mlx5_rxq_obj_type type);
+int mlx5_rxq_obj_verify(struct rte_eth_dev *dev);
struct mlx5_rxq_ctrl *mlx5_rxq_new(struct rte_eth_dev *dev, uint16_t idx,
uint16_t desc, unsigned int socket,
const struct rte_eth_rxconf *conf,
int mlx5_rxq_release(struct rte_eth_dev *dev, uint16_t idx);
int mlx5_rxq_verify(struct rte_eth_dev *dev);
int rxq_alloc_elts(struct mlx5_rxq_ctrl *rxq_ctrl);
-int mlx5_ind_table_ibv_verify(struct rte_eth_dev *dev);
+int mlx5_ind_table_obj_verify(struct rte_eth_dev *dev);
struct mlx5_hrxq *mlx5_hrxq_new(struct rte_eth_dev *dev,
const uint8_t *rss_key, uint32_t rss_key_len,
uint64_t hash_fields,
uint64_t hash_fields,
const uint16_t *queues, uint32_t queues_n);
int mlx5_hrxq_release(struct rte_eth_dev *dev, struct mlx5_hrxq *hxrq);
-int mlx5_hrxq_ibv_verify(struct rte_eth_dev *dev);
+int mlx5_hrxq_verify(struct rte_eth_dev *dev);
struct mlx5_hrxq *mlx5_hrxq_drop_new(struct rte_eth_dev *dev);
void mlx5_hrxq_drop_release(struct rte_eth_dev *dev);
-uint64_t mlx5_get_rx_port_offloads(void);
+uint64_t mlx5_get_rx_port_offloads(struct rte_eth_dev *dev);
uint64_t mlx5_get_rx_queue_offloads(struct rte_eth_dev *dev);
+int mlx5_lro_on(struct rte_eth_dev *dev);
/* mlx5_txq.c */
void mlx5_set_ptype_table(void);
void mlx5_set_cksum_table(void);
void mlx5_set_swp_types_table(void);
-__rte_noinline uint16_t mlx5_tx_error_cqe_handle(struct mlx5_txq_data *txq,
- volatile struct mlx5_err_cqe *err_cqe);
+__rte_noinline uint16_t mlx5_tx_error_cqe_handle
+ (struct mlx5_txq_data *restrict txq,
+ volatile struct mlx5_err_cqe *err_cqe);
uint16_t mlx5_rx_burst(void *dpdk_rxq, struct rte_mbuf **pkts, uint16_t pkts_n);
void mlx5_rxq_initialize(struct mlx5_rxq_data *rxq);
__rte_noinline int mlx5_rx_err_handle(struct mlx5_rxq_data *rxq,
git.droids-corp.org / dpdk.git / blobdiff