+TAILQ_HEAD(mlx5_counters, mlx5_flow_counter);
+
+/* Generic counter pool structure - query is in pool resolution. */
+struct mlx5_flow_counter_pool {
+ TAILQ_ENTRY(mlx5_flow_counter_pool) next;
+ struct mlx5_counters counters[2]; /* Free counter list. */
+ union {
+ struct mlx5_devx_obj *min_dcs;
+ rte_atomic64_t a64_dcs;
+ };
+ /* The devx object of the minimum counter ID. */
+ uint32_t index:29; /* Pool index in container. */
+ uint32_t type:2; /* Memory type behind the counter array. */
+ volatile uint32_t query_gen:1; /* Query round. */
+ rte_spinlock_t sl; /* The pool lock. */
+ struct mlx5_counter_stats_raw *raw;
+ struct mlx5_counter_stats_raw *raw_hw; /* The raw on HW working. */
+};
+
+struct mlx5_counter_stats_raw;
+
+/* Memory management structure for group of counter statistics raws. */
+struct mlx5_counter_stats_mem_mng {
+ LIST_ENTRY(mlx5_counter_stats_mem_mng) next;
+ struct mlx5_counter_stats_raw *raws;
+ struct mlx5_devx_obj *dm;
+ void *umem;
+};
+
+/* Raw memory structure for the counter statistics values of a pool. */
+struct mlx5_counter_stats_raw {
+ LIST_ENTRY(mlx5_counter_stats_raw) next;
+ int min_dcs_id;
+ struct mlx5_counter_stats_mem_mng *mem_mng;
+ volatile struct flow_counter_stats *data;
+};
+
+TAILQ_HEAD(mlx5_counter_pools, mlx5_flow_counter_pool);
+
+/* Container structure for counter pools. */
+struct mlx5_pools_container {
+ rte_atomic16_t n_valid; /* Number of valid pools. */
+ uint16_t n; /* Number of pools. */
+ uint16_t last_pool_idx; /* Last used pool index */
+ int min_id; /* The minimum counter ID in the pools. */
+ int max_id; /* The maximum counter ID in the pools. */
+ rte_spinlock_t resize_sl; /* The resize lock. */
+ rte_spinlock_t csl; /* The counter free list lock. */
+ struct mlx5_counters counters; /* Free counter list. */
+ struct mlx5_counter_pools pool_list; /* Counter pool list. */
+ struct mlx5_flow_counter_pool **pools; /* Counter pool array. */
+ struct mlx5_counter_stats_mem_mng *mem_mng;
+ /* Hold the memory management for the next allocated pools raws. */
+};
+
+/* Counter global management structure. */
+struct mlx5_flow_counter_mng {
+ struct mlx5_pools_container ccont[MLX5_CCONT_TYPE_MAX];
+ struct mlx5_counters flow_counters; /* Legacy flow counter list. */
+ uint8_t pending_queries;
+ uint8_t batch;
+ uint16_t pool_index;
+ uint8_t age;
+ uint8_t query_thread_on;
+ LIST_HEAD(mem_mngs, mlx5_counter_stats_mem_mng) mem_mngs;
+ LIST_HEAD(stat_raws, mlx5_counter_stats_raw) free_stat_raws;
+};
+
+/* Default miss action resource structure. */
+struct mlx5_flow_default_miss_resource {
+ void *action; /* Pointer to the rdma-core action. */
+ rte_atomic32_t refcnt; /* Default miss action reference counter. */
+};
+
+#define MLX5_AGE_EVENT_NEW 1
+#define MLX5_AGE_TRIGGER 2
+#define MLX5_AGE_SET(age_info, BIT) \
+ ((age_info)->flags |= (1 << (BIT)))
+#define MLX5_AGE_GET(age_info, BIT) \
+ ((age_info)->flags & (1 << (BIT)))
+#define GET_PORT_AGE_INFO(priv) \
+ (&((priv)->sh->port[(priv)->dev_port - 1].age_info))
+
+/* Aging information for per port. */
+struct mlx5_age_info {
+ uint8_t flags; /*Indicate if is new event or need be trigered*/
+ struct mlx5_counters aged_counters; /* Aged flow counter list. */
+ rte_spinlock_t aged_sl; /* Aged flow counter list lock. */
+};
+
+/* Per port data of shared IB device. */
+struct mlx5_dev_shared_port {
+ uint32_t ih_port_id;
+ uint32_t devx_ih_port_id;
+ /*
+ * Interrupt handler port_id. Used by shared interrupt
+ * handler to find the corresponding rte_eth device
+ * by IB port index. If value is equal or greater
+ * RTE_MAX_ETHPORTS it means there is no subhandler
+ * installed for specified IB port index.
+ */
+ struct mlx5_age_info age_info;
+ /* Aging information for per port. */
+};
+
+/* Table key of the hash organization. */
+union mlx5_flow_tbl_key {
+ struct {
+ /* Table ID should be at the lowest address. */
+ uint32_t table_id; /**< ID of the table. */
+ uint16_t reserved; /**< must be zero for comparison. */
+ uint8_t domain; /**< 1 - FDB, 0 - NIC TX/RX. */
+ uint8_t direction; /**< 1 - egress, 0 - ingress. */
+ };
+ uint64_t v64; /**< full 64bits value of key */
+};
+
+/* Table structure. */
+struct mlx5_flow_tbl_resource {
+ void *obj; /**< Pointer to DR table object. */
+ rte_atomic32_t refcnt; /**< Reference counter. */
+};
+
+#define MLX5_MAX_TABLES UINT16_MAX
+#define MLX5_FLOW_TABLE_LEVEL_METER (UINT16_MAX - 3)
+#define MLX5_FLOW_TABLE_LEVEL_SUFFIX (UINT16_MAX - 2)
+#define MLX5_HAIRPIN_TX_TABLE (UINT16_MAX - 1)
+/* Reserve the last two tables for metadata register copy. */
+#define MLX5_FLOW_MREG_ACT_TABLE_GROUP (MLX5_MAX_TABLES - 1)
+#define MLX5_FLOW_MREG_CP_TABLE_GROUP (MLX5_MAX_TABLES - 2)
+/* Tables for metering splits should be added here. */
+#define MLX5_MAX_TABLES_EXTERNAL (MLX5_MAX_TABLES - 3)
+#define MLX5_MAX_TABLES_FDB UINT16_MAX
+
+/* ID generation structure. */
+struct mlx5_flow_id_pool {
+ uint32_t *free_arr; /**< Pointer to the a array of free values. */
+ uint32_t base_index;
+ /**< The next index that can be used without any free elements. */
+ uint32_t *curr; /**< Pointer to the index to pop. */
+ uint32_t *last; /**< Pointer to the last element in the empty arrray. */
+ uint32_t max_id; /**< Maximum id can be allocated from the pool. */
+};
+
+/*
+ * Shared Infiniband device context for Master/Representors
+ * which belong to same IB device with multiple IB ports.
+ **/
+struct mlx5_dev_ctx_shared {
+ LIST_ENTRY(mlx5_dev_ctx_shared) next;
+ uint32_t refcnt;
+ uint32_t devx:1; /* Opened with DV. */
+ uint32_t max_port; /* Maximal IB device port index. */
+ void *ctx; /* Verbs/DV/DevX context. */
+ void *pd; /* Protection Domain. */
+ uint32_t pdn; /* Protection Domain number. */
+ uint32_t tdn; /* Transport Domain number. */
+ char ibdev_name[DEV_SYSFS_NAME_MAX]; /* SYSFS dev name. */
+ char ibdev_path[DEV_SYSFS_PATH_MAX]; /* SYSFS dev path for secondary */
+ struct mlx5_dev_attr device_attr; /* Device properties. */
+ LIST_ENTRY(mlx5_dev_ctx_shared) mem_event_cb;
+ /**< Called by memory event callback. */
+ struct mlx5_mr_share_cache share_cache;
+ /* Shared DV/DR flow data section. */
+ pthread_mutex_t dv_mutex; /* DV context mutex. */
+ uint32_t dv_meta_mask; /* flow META metadata supported mask. */
+ uint32_t dv_mark_mask; /* flow MARK metadata supported mask. */
+ uint32_t dv_regc0_mask; /* available bits of metatada reg_c[0]. */
+ uint32_t dv_refcnt; /* DV/DR data reference counter. */
+ void *fdb_domain; /* FDB Direct Rules name space handle. */
+ void *rx_domain; /* RX Direct Rules name space handle. */
+ void *tx_domain; /* TX Direct Rules name space handle. */
+ struct mlx5_hlist *flow_tbls;
+ /* Direct Rules tables for FDB, NIC TX+RX */
+ void *esw_drop_action; /* Pointer to DR E-Switch drop action. */
+ void *pop_vlan_action; /* Pointer to DR pop VLAN action. */
+ uint32_t encaps_decaps; /* Encap/decap action indexed memory list. */
+ LIST_HEAD(modify_cmd, mlx5_flow_dv_modify_hdr_resource) modify_cmds;
+ struct mlx5_hlist *tag_table;
+ uint32_t port_id_action_list; /* List of port ID actions. */
+ uint32_t push_vlan_action_list; /* List of push VLAN actions. */
+ struct mlx5_flow_counter_mng cmng; /* Counters management structure. */
+ struct mlx5_flow_default_miss_resource default_miss;
+ /* Default miss action resource structure. */
+ struct mlx5_indexed_pool *ipool[MLX5_IPOOL_MAX];
+ /* Memory Pool for mlx5 flow resources. */
+ struct mlx5_l3t_tbl *cnt_id_tbl; /* Shared counter lookup table. */
+ /* Shared interrupt handler section. */
+ struct rte_intr_handle intr_handle; /* Interrupt handler for device. */
+ struct rte_intr_handle intr_handle_devx; /* DEVX interrupt handler. */
+ void *devx_comp; /* DEVX async comp obj. */
+ struct mlx5_devx_obj *tis; /* TIS object. */
+ struct mlx5_devx_obj *td; /* Transport domain. */
+ struct mlx5_flow_id_pool *flow_id_pool; /* Flow ID pool. */
+ struct mlx5_dev_shared_port port[]; /* per device port data array. */
+};
+
+/* Per-process private structure. */
+struct mlx5_proc_priv {
+ size_t uar_table_sz;
+ /* Size of UAR register table. */
+ void *uar_table[];
+ /* Table of UAR registers for each process. */
+};
+
+/* MTR profile list. */
+TAILQ_HEAD(mlx5_mtr_profiles, mlx5_flow_meter_profile);
+/* MTR list. */
+TAILQ_HEAD(mlx5_flow_meters, mlx5_flow_meter);
+
+#define MLX5_PROC_PRIV(port_id) \
+ ((struct mlx5_proc_priv *)rte_eth_devices[port_id].process_private)
+
+struct mlx5_priv {