+ uint64_t xstats[MLX5_MAX_XSTATS];
+ uint64_t hw_stats[MLX5_MAX_XSTATS];
+ struct mlx5_counter_ctrl info[MLX5_MAX_XSTATS];
+};
+
+struct mlx5_stats_ctrl {
+ /* Base for imissed counter. */
+ uint64_t imissed_base;
+ uint64_t imissed;
+};
+
+/* Default PMD specific parameter value. */
+#define MLX5_ARG_UNSET (-1)
+
+#define MLX5_LRO_SUPPORTED(dev) \
+ (((struct mlx5_priv *)((dev)->data->dev_private))->config.lro.supported)
+
+/* Maximal size of coalesced segment for LRO is set in chunks of 256 Bytes. */
+#define MLX5_LRO_SEG_CHUNK_SIZE 256u
+
+/* Maximal size of aggregated LRO packet. */
+#define MLX5_MAX_LRO_SIZE (UINT8_MAX * MLX5_LRO_SEG_CHUNK_SIZE)
+
+/* LRO configurations structure. */
+struct mlx5_lro_config {
+ uint32_t supported:1; /* Whether LRO is supported. */
+ uint32_t timeout; /* User configuration. */
+};
+
+/*
+ * Device configuration structure.
+ *
+ * Merged configuration from:
+ *
+ * - Device capabilities,
+ * - User device parameters disabled features.
+ */
+struct mlx5_dev_config {
+ unsigned int hw_csum:1; /* Checksum offload is supported. */
+ unsigned int hw_vlan_strip:1; /* VLAN stripping is supported. */
+ unsigned int hw_vlan_insert:1; /* VLAN insertion in WQE is supported. */
+ unsigned int hw_fcs_strip:1; /* FCS stripping is supported. */
+ unsigned int hw_padding:1; /* End alignment padding is supported. */
+ unsigned int vf:1; /* This is a VF. */
+ unsigned int tunnel_en:1;
+ /* Whether tunnel stateless offloads are supported. */
+ unsigned int mpls_en:1; /* MPLS over GRE/UDP is enabled. */
+ unsigned int cqe_comp:1; /* CQE compression is enabled. */
+ unsigned int cqe_pad:1; /* CQE padding is enabled. */
+ unsigned int tso:1; /* Whether TSO is supported. */
+ unsigned int rx_vec_en:1; /* Rx vector is enabled. */
+ unsigned int mr_ext_memseg_en:1;
+ /* Whether memseg should be extended for MR creation. */
+ unsigned int l3_vxlan_en:1; /* Enable L3 VXLAN flow creation. */
+ unsigned int vf_nl_en:1; /* Enable Netlink requests in VF mode. */
+ unsigned int dv_esw_en:1; /* Enable E-Switch DV flow. */
+ unsigned int dv_flow_en:1; /* Enable DV flow. */
+ unsigned int dv_xmeta_en:2; /* Enable extensive flow metadata. */
+ unsigned int lacp_by_user:1;
+ /* Enable user to manage LACP traffic. */
+ unsigned int swp:1; /* Tx generic tunnel checksum and TSO offload. */
+ unsigned int devx:1; /* Whether devx interface is available or not. */
+ unsigned int dest_tir:1; /* Whether advanced DR API is available. */
+ unsigned int reclaim_mode:2; /* Memory reclaim mode. */
+ struct {
+ unsigned int enabled:1; /* Whether MPRQ is enabled. */
+ unsigned int stride_num_n; /* Number of strides. */
+ unsigned int stride_size_n; /* Size of a stride. */
+ unsigned int min_stride_size_n; /* Min size of a stride. */
+ unsigned int max_stride_size_n; /* Max size of a stride. */
+ unsigned int max_memcpy_len;
+ /* Maximum packet size to memcpy Rx packets. */
+ unsigned int min_rxqs_num;
+ /* Rx queue count threshold to enable MPRQ. */
+ } mprq; /* Configurations for Multi-Packet RQ. */
+ int mps; /* Multi-packet send supported mode. */
+ int dbnc; /* Skip doorbell register write barrier. */
+ unsigned int flow_prio; /* Number of flow priorities. */
+ enum modify_reg flow_mreg_c[MLX5_MREG_C_NUM];
+ /* Availibility of mreg_c's. */
+ unsigned int tso_max_payload_sz; /* Maximum TCP payload for TSO. */
+ unsigned int ind_table_max_size; /* Maximum indirection table size. */
+ unsigned int max_dump_files_num; /* Maximum dump files per queue. */
+ unsigned int log_hp_size; /* Single hairpin queue data size in total. */
+ int txqs_inline; /* Queue number threshold for inlining. */
+ int txq_inline_min; /* Minimal amount of data bytes to inline. */
+ int txq_inline_max; /* Max packet size for inlining with SEND. */
+ int txq_inline_mpw; /* Max packet size for inlining with eMPW. */
+ struct mlx5_hca_attr hca_attr; /* HCA attributes. */
+ struct mlx5_lro_config lro; /* LRO configuration. */
+};
+
+
+/**
+ * Type of object being allocated.
+ */
+enum mlx5_verbs_alloc_type {
+ MLX5_VERBS_ALLOC_TYPE_NONE,
+ MLX5_VERBS_ALLOC_TYPE_TX_QUEUE,
+ MLX5_VERBS_ALLOC_TYPE_RX_QUEUE,
+};
+
+/* Structure for VF VLAN workaround. */
+struct mlx5_vf_vlan {
+ uint32_t tag:12;
+ uint32_t created:1;
+};
+
+/**
+ * Verbs allocator needs a context to know in the callback which kind of
+ * resources it is allocating.
+ */
+struct mlx5_verbs_alloc_ctx {
+ enum mlx5_verbs_alloc_type type; /* Kind of object being allocated. */
+ const void *obj; /* Pointer to the DPDK object. */
+};
+
+/* Flow drop context necessary due to Verbs API. */
+struct mlx5_drop {
+ struct mlx5_hrxq *hrxq; /* Hash Rx queue queue. */
+ struct mlx5_rxq_obj *rxq; /* Rx queue object. */
+};
+
+#define MLX5_COUNTERS_PER_POOL 512
+#define MLX5_MAX_PENDING_QUERIES 4
+#define MLX5_CNT_CONTAINER_RESIZE 64
+#define MLX5_CNT_AGE_OFFSET 0x80000000
+#define CNT_SIZE (sizeof(struct mlx5_flow_counter))
+#define CNTEXT_SIZE (sizeof(struct mlx5_flow_counter_ext))
+#define AGE_SIZE (sizeof(struct mlx5_age_param))
+#define MLX5_AGING_TIME_DELAY 7
+#define CNT_POOL_TYPE_EXT (1 << 0)
+#define CNT_POOL_TYPE_AGE (1 << 1)
+#define IS_EXT_POOL(pool) (((pool)->type) & CNT_POOL_TYPE_EXT)
+#define IS_AGE_POOL(pool) (((pool)->type) & CNT_POOL_TYPE_AGE)
+#define MLX_CNT_IS_AGE(counter) ((counter) & MLX5_CNT_AGE_OFFSET ? 1 : 0)
+#define MLX5_CNT_LEN(pool) \
+ (CNT_SIZE + \
+ (IS_AGE_POOL(pool) ? AGE_SIZE : 0) + \
+ (IS_EXT_POOL(pool) ? CNTEXT_SIZE : 0))
+#define MLX5_POOL_GET_CNT(pool, index) \
+ ((struct mlx5_flow_counter *) \
+ ((uint8_t *)((pool) + 1) + (index) * (MLX5_CNT_LEN(pool))))
+#define MLX5_CNT_ARRAY_IDX(pool, cnt) \
+ ((int)(((uint8_t *)(cnt) - (uint8_t *)((pool) + 1)) / \
+ MLX5_CNT_LEN(pool)))
+/*
+ * The pool index and offset of counter in the pool array makes up the
+ * counter index. In case the counter is from pool 0 and offset 0, it
+ * should plus 1 to avoid index 0, since 0 means invalid counter index
+ * currently.
+ */
+#define MLX5_MAKE_CNT_IDX(pi, offset) \
+ ((pi) * MLX5_COUNTERS_PER_POOL + (offset) + 1)
+#define MLX5_CNT_TO_CNT_EXT(pool, cnt) \
+ ((struct mlx5_flow_counter_ext *)\
+ ((uint8_t *)((cnt) + 1) + \
+ (IS_AGE_POOL(pool) ? AGE_SIZE : 0)))
+#define MLX5_GET_POOL_CNT_EXT(pool, offset) \
+ MLX5_CNT_TO_CNT_EXT(pool, MLX5_POOL_GET_CNT((pool), (offset)))
+#define MLX5_CNT_TO_AGE(cnt) \
+ ((struct mlx5_age_param *)((cnt) + 1))
+/*
+ * The maximum single counter is 0x800000 as MLX5_CNT_BATCH_OFFSET
+ * defines. The pool size is 512, pool index should never reach
+ * INT16_MAX.
+ */
+#define POOL_IDX_INVALID UINT16_MAX
+
+struct mlx5_flow_counter_pool;
+
+/*age status*/
+enum {
+ AGE_FREE, /* Initialized state. */
+ AGE_CANDIDATE, /* Counter assigned to flows. */
+ AGE_TMOUT, /* Timeout, wait for rte_flow_get_aged_flows and destroy. */
+};
+
+#define MLX5_CNT_CONTAINER(sh, batch, age) (&(sh)->cmng.ccont \
+ [(batch) * 2 + (age)])
+
+enum {
+ MLX5_CCONT_TYPE_SINGLE,
+ MLX5_CCONT_TYPE_SINGLE_FOR_AGE,
+ MLX5_CCONT_TYPE_BATCH,
+ MLX5_CCONT_TYPE_BATCH_FOR_AGE,
+ MLX5_CCONT_TYPE_MAX,
+};
+
+/* Counter age parameter. */
+struct mlx5_age_param {
+ rte_atomic16_t state; /**< Age state. */
+ uint16_t port_id; /**< Port id of the counter. */
+ uint32_t timeout:15; /**< Age timeout in unit of 0.1sec. */
+ uint32_t expire:16; /**< Expire time(0.1sec) in the future. */
+ void *context; /**< Flow counter age context. */
+};
+
+struct flow_counter_stats {
+ uint64_t hits;
+ uint64_t bytes;
+};
+
+struct mlx5_flow_counter_pool;
+/* Generic counters information. */
+struct mlx5_flow_counter {
+ TAILQ_ENTRY(mlx5_flow_counter) next;
+ /**< Pointer to the next flow counter structure. */
+ union {
+ uint64_t hits; /**< Reset value of hits packets. */
+ struct mlx5_flow_counter_pool *pool; /**< Counter pool. */
+ };
+ uint64_t bytes; /**< Reset value of bytes. */
+ void *action; /**< Pointer to the dv action. */
+};
+
+/* Extend counters information for none batch counters. */
+struct mlx5_flow_counter_ext {
+ uint32_t shared:1; /**< Share counter ID with other flow rules. */
+ uint32_t batch: 1;
+ /**< Whether the counter was allocated by batch command. */
+ uint32_t ref_cnt:30; /**< Reference counter. */
+ uint32_t id; /**< User counter ID. */
+ union { /**< Holds the counters for the rule. */
+#if defined(HAVE_IBV_DEVICE_COUNTERS_SET_V42)
+ struct ibv_counter_set *cs;
+#elif defined(HAVE_IBV_DEVICE_COUNTERS_SET_V45)
+ struct ibv_counters *cs;
+#endif
+ struct mlx5_devx_obj *dcs; /**< Counter Devx object. */
+ };
+};
+
+TAILQ_HEAD(mlx5_counters, mlx5_flow_counter);
+
+/* Generic counter pool structure - query is in pool resolution. */
+struct mlx5_flow_counter_pool {
+ TAILQ_ENTRY(mlx5_flow_counter_pool) next;
+ struct mlx5_counters counters[2]; /* Free counter list. */
+ union {
+ struct mlx5_devx_obj *min_dcs;
+ rte_atomic64_t a64_dcs;
+ };
+ /* The devx object of the minimum counter ID. */
+ uint32_t index:29; /* Pool index in container. */
+ uint32_t type:2; /* Memory type behind the counter array. */
+ volatile uint32_t query_gen:1; /* Query round. */
+ rte_spinlock_t sl; /* The pool lock. */
+ struct mlx5_counter_stats_raw *raw;
+ struct mlx5_counter_stats_raw *raw_hw; /* The raw on HW working. */
+};
+
+struct mlx5_counter_stats_raw;
+
+/* Memory management structure for group of counter statistics raws. */
+struct mlx5_counter_stats_mem_mng {
+ LIST_ENTRY(mlx5_counter_stats_mem_mng) next;
+ struct mlx5_counter_stats_raw *raws;
+ struct mlx5_devx_obj *dm;
+ void *umem;
+};
+
+/* Raw memory structure for the counter statistics values of a pool. */
+struct mlx5_counter_stats_raw {
+ LIST_ENTRY(mlx5_counter_stats_raw) next;
+ int min_dcs_id;
+ struct mlx5_counter_stats_mem_mng *mem_mng;
+ volatile struct flow_counter_stats *data;
+};
+
+TAILQ_HEAD(mlx5_counter_pools, mlx5_flow_counter_pool);
+
+/* Container structure for counter pools. */
+struct mlx5_pools_container {
+ rte_atomic16_t n_valid; /* Number of valid pools. */
+ uint16_t n; /* Number of pools. */
+ uint16_t last_pool_idx; /* Last used pool index */
+ int min_id; /* The minimum counter ID in the pools. */
+ int max_id; /* The maximum counter ID in the pools. */
+ rte_spinlock_t resize_sl; /* The resize lock. */
+ rte_spinlock_t csl; /* The counter free list lock. */
+ struct mlx5_counters counters; /* Free counter list. */
+ struct mlx5_counter_pools pool_list; /* Counter pool list. */
+ struct mlx5_flow_counter_pool **pools; /* Counter pool array. */
+ struct mlx5_counter_stats_mem_mng *mem_mng;
+ /* Hold the memory management for the next allocated pools raws. */
+};
+
+/* Counter global management structure. */
+struct mlx5_flow_counter_mng {
+ struct mlx5_pools_container ccont[MLX5_CCONT_TYPE_MAX];
+ struct mlx5_counters flow_counters; /* Legacy flow counter list. */
+ uint8_t pending_queries;
+ uint8_t batch;
+ uint16_t pool_index;
+ uint8_t age;
+ uint8_t query_thread_on;
+ LIST_HEAD(mem_mngs, mlx5_counter_stats_mem_mng) mem_mngs;
+ LIST_HEAD(stat_raws, mlx5_counter_stats_raw) free_stat_raws;
+};
+
+/* Default miss action resource structure. */
+struct mlx5_flow_default_miss_resource {
+ void *action; /* Pointer to the rdma-core action. */
+ rte_atomic32_t refcnt; /* Default miss action reference counter. */
+};
+
+#define MLX5_AGE_EVENT_NEW 1
+#define MLX5_AGE_TRIGGER 2
+#define MLX5_AGE_SET(age_info, BIT) \
+ ((age_info)->flags |= (1 << (BIT)))
+#define MLX5_AGE_GET(age_info, BIT) \
+ ((age_info)->flags & (1 << (BIT)))
+#define GET_PORT_AGE_INFO(priv) \
+ (&((priv)->sh->port[(priv)->dev_port - 1].age_info))
+
+/* Aging information for per port. */
+struct mlx5_age_info {
+ uint8_t flags; /*Indicate if is new event or need be trigered*/
+ struct mlx5_counters aged_counters; /* Aged flow counter list. */
+ rte_spinlock_t aged_sl; /* Aged flow counter list lock. */