+/* Default PMD specific parameter value. */
+#define MLX5_ARG_UNSET (-1)
+
+#define MLX5_LRO_SUPPORTED(dev) \
+ (((struct mlx5_priv *)((dev)->data->dev_private))->config.lro.supported)
+
+/* LRO configurations structure. */
+struct mlx5_lro_config {
+ uint32_t supported:1; /* Whether LRO is supported. */
+ uint32_t timeout; /* User configuration. */
+};
+
+/*
+ * Device configuration structure.
+ *
+ * Merged configuration from:
+ *
+ * - Device capabilities,
+ * - User device parameters disabled features.
+ */
+struct mlx5_dev_config {
+ unsigned int hw_csum:1; /* Checksum offload is supported. */
+ unsigned int hw_vlan_strip:1; /* VLAN stripping is supported. */
+ unsigned int hw_vlan_insert:1; /* VLAN insertion in WQE is supported. */
+ unsigned int hw_fcs_strip:1; /* FCS stripping is supported. */
+ unsigned int hw_padding:1; /* End alignment padding is supported. */
+ unsigned int vf:1; /* This is a VF. */
+ unsigned int tunnel_en:1;
+ /* Whether tunnel stateless offloads are supported. */
+ unsigned int mpls_en:1; /* MPLS over GRE/UDP is enabled. */
+ unsigned int cqe_comp:1; /* CQE compression is enabled. */
+ unsigned int cqe_pad:1; /* CQE padding is enabled. */
+ unsigned int tso:1; /* Whether TSO is supported. */
+ unsigned int rx_vec_en:1; /* Rx vector is enabled. */
+ unsigned int mr_ext_memseg_en:1;
+ /* Whether memseg should be extended for MR creation. */
+ unsigned int l3_vxlan_en:1; /* Enable L3 VXLAN flow creation. */
+ unsigned int vf_nl_en:1; /* Enable Netlink requests in VF mode. */
+ unsigned int dv_esw_en:1; /* Enable E-Switch DV flow. */
+ unsigned int dv_flow_en:1; /* Enable DV flow. */
+ unsigned int dv_xmeta_en:2; /* Enable extensive flow metadata. */
+ unsigned int swp:1; /* Tx generic tunnel checksum and TSO offload. */
+ unsigned int devx:1; /* Whether devx interface is available or not. */
+ unsigned int dest_tir:1; /* Whether advanced DR API is available. */
+ struct {
+ unsigned int enabled:1; /* Whether MPRQ is enabled. */
+ unsigned int stride_num_n; /* Number of strides. */
+ unsigned int min_stride_size_n; /* Min size of a stride. */
+ unsigned int max_stride_size_n; /* Max size of a stride. */
+ unsigned int max_memcpy_len;
+ /* Maximum packet size to memcpy Rx packets. */
+ unsigned int min_rxqs_num;
+ /* Rx queue count threshold to enable MPRQ. */
+ } mprq; /* Configurations for Multi-Packet RQ. */
+ int mps; /* Multi-packet send supported mode. */
+ int dbnc; /* Skip doorbell register write barrier. */
+ unsigned int flow_prio; /* Number of flow priorities. */
+ enum modify_reg flow_mreg_c[MLX5_MREG_C_NUM];
+ /* Availibility of mreg_c's. */
+ unsigned int tso_max_payload_sz; /* Maximum TCP payload for TSO. */
+ unsigned int ind_table_max_size; /* Maximum indirection table size. */
+ unsigned int max_dump_files_num; /* Maximum dump files per queue. */
+ int txqs_inline; /* Queue number threshold for inlining. */
+ int txq_inline_min; /* Minimal amount of data bytes to inline. */
+ int txq_inline_max; /* Max packet size for inlining with SEND. */
+ int txq_inline_mpw; /* Max packet size for inlining with eMPW. */
+ struct mlx5_hca_attr hca_attr; /* HCA attributes. */
+ struct mlx5_lro_config lro; /* LRO configuration. */
+};
+
+struct mlx5_devx_wq_attr {
+ uint32_t wq_type:4;
+ uint32_t wq_signature:1;
+ uint32_t end_padding_mode:2;
+ uint32_t cd_slave:1;
+ uint32_t hds_skip_first_sge:1;
+ uint32_t log2_hds_buf_size:3;
+ uint32_t page_offset:5;
+ uint32_t lwm:16;
+ uint32_t pd:24;
+ uint32_t uar_page:24;
+ uint64_t dbr_addr;
+ uint32_t hw_counter;
+ uint32_t sw_counter;
+ uint32_t log_wq_stride:4;
+ uint32_t log_wq_pg_sz:5;
+ uint32_t log_wq_sz:5;
+ uint32_t dbr_umem_valid:1;
+ uint32_t wq_umem_valid:1;
+ uint32_t log_hairpin_num_packets:5;
+ uint32_t log_hairpin_data_sz:5;
+ uint32_t single_wqe_log_num_of_strides:4;
+ uint32_t two_byte_shift_en:1;
+ uint32_t single_stride_log_num_of_bytes:3;
+ uint32_t dbr_umem_id;
+ uint32_t wq_umem_id;
+ uint64_t wq_umem_offset;
+};
+
+/* Create RQ attributes structure, used by create RQ operation. */
+struct mlx5_devx_create_rq_attr {
+ uint32_t rlky:1;
+ uint32_t delay_drop_en:1;
+ uint32_t scatter_fcs:1;
+ uint32_t vsd:1;
+ uint32_t mem_rq_type:4;
+ uint32_t state:4;
+ uint32_t flush_in_error_en:1;
+ uint32_t hairpin:1;
+ uint32_t user_index:24;
+ uint32_t cqn:24;
+ uint32_t counter_set_id:8;
+ uint32_t rmpn:24;
+ struct mlx5_devx_wq_attr wq_attr;
+};
+
+/* Modify RQ attributes structure, used by modify RQ operation. */
+struct mlx5_devx_modify_rq_attr {
+ uint32_t rqn:24;
+ uint32_t rq_state:4; /* Current RQ state. */
+ uint32_t state:4; /* Required RQ state. */
+ uint32_t scatter_fcs:1;
+ uint32_t vsd:1;
+ uint32_t counter_set_id:8;
+ uint32_t hairpin_peer_sq:24;
+ uint32_t hairpin_peer_vhca:16;
+ uint64_t modify_bitmask;
+ uint32_t lwm:16; /* Contained WQ lwm. */
+};
+
+struct mlx5_rx_hash_field_select {
+ uint32_t l3_prot_type:1;
+ uint32_t l4_prot_type:1;
+ uint32_t selected_fields:30;
+};
+
+/* TIR attributes structure, used by TIR operations. */
+struct mlx5_devx_tir_attr {
+ uint32_t disp_type:4;
+ uint32_t lro_timeout_period_usecs:16;
+ uint32_t lro_enable_mask:4;
+ uint32_t lro_max_msg_sz:8;
+ uint32_t inline_rqn:24;
+ uint32_t rx_hash_symmetric:1;
+ uint32_t tunneled_offload_en:1;
+ uint32_t indirect_table:24;
+ uint32_t rx_hash_fn:4;
+ uint32_t self_lb_block:2;
+ uint32_t transport_domain:24;
+ uint32_t rx_hash_toeplitz_key[10];
+ struct mlx5_rx_hash_field_select rx_hash_field_selector_outer;
+ struct mlx5_rx_hash_field_select rx_hash_field_selector_inner;
+};
+
+/* RQT attributes structure, used by RQT operations. */
+struct mlx5_devx_rqt_attr {
+ uint32_t rqt_max_size:16;
+ uint32_t rqt_actual_size:16;
+ uint32_t rq_list[];
+};
+
+/* TIS attributes structure. */
+struct mlx5_devx_tis_attr {
+ uint32_t strict_lag_tx_port_affinity:1;
+ uint32_t tls_en:1;
+ uint32_t lag_tx_port_affinity:4;
+ uint32_t prio:4;
+ uint32_t transport_domain:24;
+};
+
+/* SQ attributes structure, used by SQ create operation. */
+struct mlx5_devx_create_sq_attr {
+ uint32_t rlky:1;
+ uint32_t cd_master:1;
+ uint32_t fre:1;
+ uint32_t flush_in_error_en:1;
+ uint32_t allow_multi_pkt_send_wqe:1;
+ uint32_t min_wqe_inline_mode:3;
+ uint32_t state:4;
+ uint32_t reg_umr:1;
+ uint32_t allow_swp:1;
+ uint32_t hairpin:1;
+ uint32_t user_index:24;
+ uint32_t cqn:24;
+ uint32_t packet_pacing_rate_limit_index:16;
+ uint32_t tis_lst_sz:16;
+ uint32_t tis_num:24;
+ struct mlx5_devx_wq_attr wq_attr;
+};
+
+/* SQ attributes structure, used by SQ modify operation. */
+struct mlx5_devx_modify_sq_attr {
+ uint32_t sq_state:4;
+ uint32_t state:4;
+ uint32_t hairpin_peer_rq:24;
+ uint32_t hairpin_peer_vhca:16;
+};
+
+/**
+ * Type of object being allocated.
+ */
+enum mlx5_verbs_alloc_type {
+ MLX5_VERBS_ALLOC_TYPE_NONE,
+ MLX5_VERBS_ALLOC_TYPE_TX_QUEUE,
+ MLX5_VERBS_ALLOC_TYPE_RX_QUEUE,
+};
+
+/* VLAN netdev for VLAN workaround. */
+struct mlx5_vlan_dev {
+ uint32_t refcnt;
+ uint32_t ifindex; /**< Own interface index. */
+};
+
+/* Structure for VF VLAN workaround. */
+struct mlx5_vf_vlan {
+ uint32_t tag:12;
+ uint32_t created:1;
+};
+
+/*
+ * Array of VLAN devices created on the base of VF
+ * used for workaround in virtual environments.
+ */
+struct mlx5_vlan_vmwa_context {
+ int nl_socket;
+ uint32_t nl_sn;
+ uint32_t vf_ifindex;
+ struct rte_eth_dev *dev;
+ struct mlx5_vlan_dev vlan_dev[4096];
+};
+
+/**
+ * Verbs allocator needs a context to know in the callback which kind of
+ * resources it is allocating.
+ */
+struct mlx5_verbs_alloc_ctx {
+ enum mlx5_verbs_alloc_type type; /* Kind of object being allocated. */
+ const void *obj; /* Pointer to the DPDK object. */
+};
+
+LIST_HEAD(mlx5_mr_list, mlx5_mr);
+
+/* Flow drop context necessary due to Verbs API. */
+struct mlx5_drop {
+ struct mlx5_hrxq *hrxq; /* Hash Rx queue queue. */
+ struct mlx5_rxq_obj *rxq; /* Rx queue object. */
+};
+
+#define MLX5_COUNTERS_PER_POOL 512
+#define MLX5_MAX_PENDING_QUERIES 4
+
+struct mlx5_flow_counter_pool;
+
+struct flow_counter_stats {
+ uint64_t hits;
+ uint64_t bytes;
+};
+
+/* Counters information. */
+struct mlx5_flow_counter {
+ TAILQ_ENTRY(mlx5_flow_counter) next;
+ /**< Pointer to the next flow counter structure. */
+ uint32_t shared:1; /**< Share counter ID with other flow rules. */
+ uint32_t batch: 1;
+ /**< Whether the counter was allocated by batch command. */
+ uint32_t ref_cnt:30; /**< Reference counter. */
+ uint32_t id; /**< Counter ID. */
+ union { /**< Holds the counters for the rule. */
+#if defined(HAVE_IBV_DEVICE_COUNTERS_SET_V42)
+ struct ibv_counter_set *cs;
+#elif defined(HAVE_IBV_DEVICE_COUNTERS_SET_V45)
+ struct ibv_counters *cs;
+#endif
+ struct mlx5_devx_obj *dcs; /**< Counter Devx object. */
+ struct mlx5_flow_counter_pool *pool; /**< The counter pool. */
+ };
+ union {
+ uint64_t hits; /**< Reset value of hits packets. */
+ int64_t query_gen; /**< Generation of the last release. */
+ };
+ uint64_t bytes; /**< Reset value of bytes. */
+ void *action; /**< Pointer to the dv action. */
+};
+
+TAILQ_HEAD(mlx5_counters, mlx5_flow_counter);
+
+/* Counter pool structure - query is in pool resolution. */
+struct mlx5_flow_counter_pool {
+ TAILQ_ENTRY(mlx5_flow_counter_pool) next;
+ struct mlx5_counters counters; /* Free counter list. */
+ union {
+ struct mlx5_devx_obj *min_dcs;
+ rte_atomic64_t a64_dcs;
+ };
+ /* The devx object of the minimum counter ID. */
+ rte_atomic64_t query_gen;
+ uint32_t n_counters: 16; /* Number of devx allocated counters. */
+ rte_spinlock_t sl; /* The pool lock. */
+ struct mlx5_counter_stats_raw *raw;
+ struct mlx5_counter_stats_raw *raw_hw; /* The raw on HW working. */
+ struct mlx5_flow_counter counters_raw[]; /* The pool counters memory. */
+};
+
+struct mlx5_counter_stats_raw;
+
+/* Memory management structure for group of counter statistics raws. */
+struct mlx5_counter_stats_mem_mng {
+ LIST_ENTRY(mlx5_counter_stats_mem_mng) next;
+ struct mlx5_counter_stats_raw *raws;
+ struct mlx5_devx_obj *dm;
+ struct mlx5dv_devx_umem *umem;
+};
+
+/* Raw memory structure for the counter statistics values of a pool. */
+struct mlx5_counter_stats_raw {
+ LIST_ENTRY(mlx5_counter_stats_raw) next;
+ int min_dcs_id;
+ struct mlx5_counter_stats_mem_mng *mem_mng;
+ volatile struct flow_counter_stats *data;
+};
+
+TAILQ_HEAD(mlx5_counter_pools, mlx5_flow_counter_pool);
+
+/* Container structure for counter pools. */
+struct mlx5_pools_container {
+ rte_atomic16_t n_valid; /* Number of valid pools. */
+ uint16_t n; /* Number of pools. */
+ struct mlx5_counter_pools pool_list; /* Counter pool list. */
+ struct mlx5_flow_counter_pool **pools; /* Counter pool array. */
+ struct mlx5_counter_stats_mem_mng *init_mem_mng;
+ /* Hold the memory management for the next allocated pools raws. */
+};
+
+/* Counter global management structure. */
+struct mlx5_flow_counter_mng {
+ uint8_t mhi[2]; /* master \ host container index. */
+ struct mlx5_pools_container ccont[2 * 2];
+ /* 2 containers for single and for batch for double-buffer. */
+ struct mlx5_counters flow_counters; /* Legacy flow counter list. */
+ uint8_t pending_queries;
+ uint8_t batch;
+ uint16_t pool_index;
+ uint8_t query_thread_on;
+ LIST_HEAD(mem_mngs, mlx5_counter_stats_mem_mng) mem_mngs;
+ LIST_HEAD(stat_raws, mlx5_counter_stats_raw) free_stat_raws;
+};
+
+/* Per port data of shared IB device. */
+struct mlx5_ibv_shared_port {
+ uint32_t ih_port_id;
+ uint32_t devx_ih_port_id;
+ /*
+ * Interrupt handler port_id. Used by shared interrupt
+ * handler to find the corresponding rte_eth device
+ * by IB port index. If value is equal or greater
+ * RTE_MAX_ETHPORTS it means there is no subhandler
+ * installed for specified IB port index.
+ */
+};
+
+/* Table key of the hash organization. */
+union mlx5_flow_tbl_key {
+ struct {
+ /* Table ID should be at the lowest address. */
+ uint32_t table_id; /**< ID of the table. */
+ uint16_t reserved; /**< must be zero for comparison. */
+ uint8_t domain; /**< 1 - FDB, 0 - NIC TX/RX. */
+ uint8_t direction; /**< 1 - egress, 0 - ingress. */
+ };
+ uint64_t v64; /**< full 64bits value of key */
+};
+
+/* Table structure. */
+struct mlx5_flow_tbl_resource {
+ void *obj; /**< Pointer to DR table object. */
+ rte_atomic32_t refcnt; /**< Reference counter. */
+};
+
+#define MLX5_MAX_TABLES UINT16_MAX
+#define MLX5_FLOW_TABLE_LEVEL_METER (UINT16_MAX - 3)
+#define MLX5_FLOW_TABLE_LEVEL_SUFFIX (UINT16_MAX - 2)
+#define MLX5_HAIRPIN_TX_TABLE (UINT16_MAX - 1)
+/* Reserve the last two tables for metadata register copy. */
+#define MLX5_FLOW_MREG_ACT_TABLE_GROUP (MLX5_MAX_TABLES - 1)
+#define MLX5_FLOW_MREG_CP_TABLE_GROUP (MLX5_MAX_TABLES - 2)
+/* Tables for metering splits should be added here. */
+#define MLX5_MAX_TABLES_EXTERNAL (MLX5_MAX_TABLES - 3)
+#define MLX5_MAX_TABLES_FDB UINT16_MAX
+
+#define MLX5_DBR_PAGE_SIZE 4096 /* Must be >= 512. */
+#define MLX5_DBR_SIZE 8
+#define MLX5_DBR_PER_PAGE (MLX5_DBR_PAGE_SIZE / MLX5_DBR_SIZE)
+#define MLX5_DBR_BITMAP_SIZE (MLX5_DBR_PER_PAGE / 64)
+
+struct mlx5_devx_dbr_page {
+ /* Door-bell records, must be first member in structure. */
+ uint8_t dbrs[MLX5_DBR_PAGE_SIZE];
+ LIST_ENTRY(mlx5_devx_dbr_page) next; /* Pointer to the next element. */
+ struct mlx5dv_devx_umem *umem;
+ uint32_t dbr_count; /* Number of door-bell records in use. */
+ /* 1 bit marks matching door-bell is in use. */
+ uint64_t dbr_bitmap[MLX5_DBR_BITMAP_SIZE];
+};
+
+/* ID generation structure. */
+struct mlx5_flow_id_pool {
+ uint32_t *free_arr; /**< Pointer to the a array of free values. */
+ uint32_t base_index;
+ /**< The next index that can be used without any free elements. */
+ uint32_t *curr; /**< Pointer to the index to pop. */
+ uint32_t *last; /**< Pointer to the last element in the empty arrray. */
+};
+
+/*
+ * Shared Infiniband device context for Master/Representors
+ * which belong to same IB device with multiple IB ports.
+ **/
+struct mlx5_ibv_shared {
+ LIST_ENTRY(mlx5_ibv_shared) next;
+ uint32_t refcnt;
+ uint32_t devx:1; /* Opened with DV. */
+ uint32_t max_port; /* Maximal IB device port index. */
+ struct ibv_context *ctx; /* Verbs/DV context. */