+#define MLX5_FLOW_MIN_ID_POOL_SIZE 512
+#define MLX5_ID_GENERATION_ARRAY_FACTOR 16
+
+#define MLX5_FLOW_TABLE_HLIST_ARRAY_SIZE 4096
+#define MLX5_TAGS_HLIST_ARRAY_SIZE 8192
+
+/**
+ * Allocate ID pool structure.
+ *
+ * @return
+ * Pointer to pool object, NULL value otherwise.
+ */
+struct mlx5_flow_id_pool *
+mlx5_flow_id_pool_alloc(void)
+{
+ struct mlx5_flow_id_pool *pool;
+ void *mem;
+
+ pool = rte_zmalloc("id pool allocation", sizeof(*pool),
+ RTE_CACHE_LINE_SIZE);
+ if (!pool) {
+ DRV_LOG(ERR, "can't allocate id pool");
+ rte_errno = ENOMEM;
+ return NULL;
+ }
+ mem = rte_zmalloc("", MLX5_FLOW_MIN_ID_POOL_SIZE * sizeof(uint32_t),
+ RTE_CACHE_LINE_SIZE);
+ if (!mem) {
+ DRV_LOG(ERR, "can't allocate mem for id pool");
+ rte_errno = ENOMEM;
+ goto error;
+ }
+ pool->free_arr = mem;
+ pool->curr = pool->free_arr;
+ pool->last = pool->free_arr + MLX5_FLOW_MIN_ID_POOL_SIZE;
+ pool->base_index = 0;
+ return pool;
+error:
+ rte_free(pool);
+ return NULL;
+}
+
+/**
+ * Release ID pool structure.
+ *
+ * @param[in] pool
+ * Pointer to flow id pool object to free.
+ */
+void
+mlx5_flow_id_pool_release(struct mlx5_flow_id_pool *pool)
+{
+ rte_free(pool->free_arr);
+ rte_free(pool);
+}
+
+/**
+ * Generate ID.
+ *
+ * @param[in] pool
+ * Pointer to flow id pool.
+ * @param[out] id
+ * The generated ID.
+ *
+ * @return
+ * 0 on success, error value otherwise.
+ */
+uint32_t
+mlx5_flow_id_get(struct mlx5_flow_id_pool *pool, uint32_t *id)
+{
+ if (pool->curr == pool->free_arr) {
+ if (pool->base_index == UINT32_MAX) {
+ rte_errno = ENOMEM;
+ DRV_LOG(ERR, "no free id");
+ return -rte_errno;
+ }
+ *id = ++pool->base_index;
+ return 0;
+ }
+ *id = *(--pool->curr);
+ return 0;
+}
+
+/**
+ * Release ID.
+ *
+ * @param[in] pool
+ * Pointer to flow id pool.
+ * @param[out] id
+ * The generated ID.
+ *
+ * @return
+ * 0 on success, error value otherwise.
+ */
+uint32_t
+mlx5_flow_id_release(struct mlx5_flow_id_pool *pool, uint32_t id)
+{
+ uint32_t size;
+ uint32_t size2;
+ void *mem;
+
+ if (pool->curr == pool->last) {
+ size = pool->curr - pool->free_arr;
+ size2 = size * MLX5_ID_GENERATION_ARRAY_FACTOR;
+ assert(size2 > size);
+ mem = rte_malloc("", size2 * sizeof(uint32_t), 0);
+ if (!mem) {
+ DRV_LOG(ERR, "can't allocate mem for id pool");
+ rte_errno = ENOMEM;
+ return -rte_errno;
+ }
+ memcpy(mem, pool->free_arr, size * sizeof(uint32_t));
+ rte_free(pool->free_arr);
+ pool->free_arr = mem;
+ pool->curr = pool->free_arr + size;
+ pool->last = pool->free_arr + size2;
+ }
+ *pool->curr = id;
+ pool->curr++;
+ return 0;
+}
+
+/**
+ * Initialize the counters management structure.
+ *
+ * @param[in] sh
+ * Pointer to mlx5_ibv_shared object to free
+ */
+static void
+mlx5_flow_counters_mng_init(struct mlx5_ibv_shared *sh)
+{
+ uint8_t i;
+
+ TAILQ_INIT(&sh->cmng.flow_counters);
+ for (i = 0; i < RTE_DIM(sh->cmng.ccont); ++i)
+ TAILQ_INIT(&sh->cmng.ccont[i].pool_list);
+}
+
+/**
+ * Destroy all the resources allocated for a counter memory management.
+ *
+ * @param[in] mng
+ * Pointer to the memory management structure.
+ */
+static void
+mlx5_flow_destroy_counter_stat_mem_mng(struct mlx5_counter_stats_mem_mng *mng)
+{
+ uint8_t *mem = (uint8_t *)(uintptr_t)mng->raws[0].data;
+
+ LIST_REMOVE(mng, next);
+ claim_zero(mlx5_devx_cmd_destroy(mng->dm));
+ claim_zero(mlx5_glue->devx_umem_dereg(mng->umem));
+ rte_free(mem);
+}
+
+/**
+ * Close and release all the resources of the counters management.
+ *
+ * @param[in] sh
+ * Pointer to mlx5_ibv_shared object to free.
+ */
+static void
+mlx5_flow_counters_mng_close(struct mlx5_ibv_shared *sh)
+{
+ struct mlx5_counter_stats_mem_mng *mng;
+ uint8_t i;
+ int j;
+ int retries = 1024;
+
+ rte_errno = 0;
+ while (--retries) {
+ rte_eal_alarm_cancel(mlx5_flow_query_alarm, sh);
+ if (rte_errno != EINPROGRESS)
+ break;
+ rte_pause();
+ }
+ for (i = 0; i < RTE_DIM(sh->cmng.ccont); ++i) {
+ struct mlx5_flow_counter_pool *pool;
+ uint32_t batch = !!(i % 2);
+
+ if (!sh->cmng.ccont[i].pools)
+ continue;
+ pool = TAILQ_FIRST(&sh->cmng.ccont[i].pool_list);
+ while (pool) {
+ if (batch) {
+ if (pool->min_dcs)
+ claim_zero
+ (mlx5_devx_cmd_destroy(pool->min_dcs));
+ }
+ for (j = 0; j < MLX5_COUNTERS_PER_POOL; ++j) {
+ if (pool->counters_raw[j].action)
+ claim_zero
+ (mlx5_glue->destroy_flow_action
+ (pool->counters_raw[j].action));
+ if (!batch && pool->counters_raw[j].dcs)
+ claim_zero(mlx5_devx_cmd_destroy
+ (pool->counters_raw[j].dcs));
+ }
+ TAILQ_REMOVE(&sh->cmng.ccont[i].pool_list, pool,
+ next);
+ rte_free(pool);
+ pool = TAILQ_FIRST(&sh->cmng.ccont[i].pool_list);
+ }
+ rte_free(sh->cmng.ccont[i].pools);
+ }
+ mng = LIST_FIRST(&sh->cmng.mem_mngs);
+ while (mng) {
+ mlx5_flow_destroy_counter_stat_mem_mng(mng);
+ mng = LIST_FIRST(&sh->cmng.mem_mngs);
+ }
+ memset(&sh->cmng, 0, sizeof(sh->cmng));
+}
+
+/**
+ * Extract pdn of PD object using DV API.
+ *
+ * @param[in] pd
+ * Pointer to the verbs PD object.
+ * @param[out] pdn
+ * Pointer to the PD object number variable.
+ *
+ * @return
+ * 0 on success, error value otherwise.
+ */
+#ifdef HAVE_IBV_FLOW_DV_SUPPORT
+static int
+mlx5_get_pdn(struct ibv_pd *pd __rte_unused, uint32_t *pdn __rte_unused)
+{
+ struct mlx5dv_obj obj;
+ struct mlx5dv_pd pd_info;
+ int ret = 0;
+
+ obj.pd.in = pd;
+ obj.pd.out = &pd_info;
+ ret = mlx5_glue->dv_init_obj(&obj, MLX5DV_OBJ_PD);
+ if (ret) {
+ DRV_LOG(DEBUG, "Fail to get PD object info");
+ return ret;
+ }
+ *pdn = pd_info.pdn;
+ return 0;
+}
+#endif /* HAVE_IBV_FLOW_DV_SUPPORT */
+
+static int
+mlx5_config_doorbell_mapping_env(const struct mlx5_dev_config *config)
+{
+ char *env;
+ int value;
+
+ assert(rte_eal_process_type() == RTE_PROC_PRIMARY);
+ /* Get environment variable to store. */
+ env = getenv(MLX5_SHUT_UP_BF);
+ value = env ? !!strcmp(env, "0") : MLX5_ARG_UNSET;
+ if (config->dbnc == MLX5_ARG_UNSET)
+ setenv(MLX5_SHUT_UP_BF, MLX5_SHUT_UP_BF_DEFAULT, 1);
+ else
+ setenv(MLX5_SHUT_UP_BF,
+ config->dbnc == MLX5_TXDB_NCACHED ? "1" : "0", 1);
+ return value;
+}
+
+static void
+mlx5_restore_doorbell_mapping_env(int value)
+{
+ assert(rte_eal_process_type() == RTE_PROC_PRIMARY);
+ /* Restore the original environment variable state. */
+ if (value == MLX5_ARG_UNSET)
+ unsetenv(MLX5_SHUT_UP_BF);
+ else
+ setenv(MLX5_SHUT_UP_BF, value ? "1" : "0", 1);
+}
+