net/hns3: add RSS hash offload to capabilities
[dpdk.git] / drivers / net / mlx5 / mlx5_flow_dv.c
index 6a49bc9..b8d03d4 100644 (file)
@@ -3820,132 +3820,41 @@ flow_dv_modify_hdr_resource_register
        return 0;
 }
 
-#define MLX5_CNT_CONTAINER_RESIZE 64
-
 /**
- * Get or create a flow counter.
+ * Get DV flow counter by index.
  *
  * @param[in] dev
  *   Pointer to the Ethernet device structure.
- * @param[in] shared
- *   Indicate if this counter is shared with other flows.
- * @param[in] id
- *   Counter identifier.
+ * @param[in] idx
+ *   mlx5 flow counter index in the container.
+ * @param[out] ppool
+ *   mlx5 flow counter pool in the container,
  *
  * @return
- *   pointer to flow counter on success, NULL otherwise and rte_errno is set.
+ *   Pointer to the counter, NULL otherwise.
  */
 static struct mlx5_flow_counter *
-flow_dv_counter_alloc_fallback(struct rte_eth_dev *dev, uint32_t shared,
-                              uint32_t id)
-{
-       struct mlx5_priv *priv = dev->data->dev_private;
-       struct mlx5_flow_counter *cnt = NULL;
-       struct mlx5_devx_obj *dcs = NULL;
-
-       if (!priv->config.devx) {
-               rte_errno = ENOTSUP;
-               return NULL;
-       }
-       if (shared) {
-               TAILQ_FOREACH(cnt, &priv->sh->cmng.flow_counters, next) {
-                       if (cnt->shared && cnt->id == id) {
-                               cnt->ref_cnt++;
-                               return cnt;
-                       }
-               }
-       }
-       dcs = mlx5_devx_cmd_flow_counter_alloc(priv->sh->ctx, 0);
-       if (!dcs)
-               return NULL;
-       cnt = rte_calloc(__func__, 1, sizeof(*cnt), 0);
-       if (!cnt) {
-               claim_zero(mlx5_devx_cmd_destroy(cnt->dcs));
-               rte_errno = ENOMEM;
-               return NULL;
-       }
-       struct mlx5_flow_counter tmpl = {
-               .shared = shared,
-               .ref_cnt = 1,
-               .id = id,
-               .dcs = dcs,
-       };
-       tmpl.action = mlx5_glue->dv_create_flow_action_counter(dcs->obj, 0);
-       if (!tmpl.action) {
-               claim_zero(mlx5_devx_cmd_destroy(cnt->dcs));
-               rte_errno = errno;
-               rte_free(cnt);
-               return NULL;
-       }
-       *cnt = tmpl;
-       TAILQ_INSERT_HEAD(&priv->sh->cmng.flow_counters, cnt, next);
-       return cnt;
-}
-
-/**
- * Release a flow counter.
- *
- * @param[in] dev
- *   Pointer to the Ethernet device structure.
- * @param[in] counter
- *   Pointer to the counter handler.
- */
-static void
-flow_dv_counter_release_fallback(struct rte_eth_dev *dev,
-                                struct mlx5_flow_counter *counter)
+flow_dv_counter_get_by_idx(struct rte_eth_dev *dev,
+                          uint32_t idx,
+                          struct mlx5_flow_counter_pool **ppool)
 {
        struct mlx5_priv *priv = dev->data->dev_private;
+       struct mlx5_pools_container *cont;
+       struct mlx5_flow_counter_pool *pool;
+       uint32_t batch = 0;
 
-       if (!counter)
-               return;
-       if (--counter->ref_cnt == 0) {
-               TAILQ_REMOVE(&priv->sh->cmng.flow_counters, counter, next);
-               claim_zero(mlx5_devx_cmd_destroy(counter->dcs));
-               rte_free(counter);
+       idx--;
+       if (idx >= MLX5_CNT_BATCH_OFFSET) {
+               idx -= MLX5_CNT_BATCH_OFFSET;
+               batch = 1;
        }
-}
-
-/**
- * Query a devx flow counter.
- *
- * @param[in] dev
- *   Pointer to the Ethernet device structure.
- * @param[in] cnt
- *   Pointer to the flow counter.
- * @param[out] pkts
- *   The statistics value of packets.
- * @param[out] bytes
- *   The statistics value of bytes.
- *
- * @return
- *   0 on success, otherwise a negative errno value and rte_errno is set.
- */
-static inline int
-_flow_dv_query_count_fallback(struct rte_eth_dev *dev __rte_unused,
-                    struct mlx5_flow_counter *cnt, uint64_t *pkts,
-                    uint64_t *bytes)
-{
-       return mlx5_devx_cmd_flow_counter_query(cnt->dcs, 0, 0, pkts, bytes,
-                                               0, NULL, NULL, 0);
-}
-
-/**
- * Get a pool by a counter.
- *
- * @param[in] cnt
- *   Pointer to the counter.
- *
- * @return
- *   The counter pool.
- */
-static struct mlx5_flow_counter_pool *
-flow_dv_counter_pool_get(struct mlx5_flow_counter *cnt)
-{
-       if (!cnt->batch) {
-               cnt -= cnt->dcs->id % MLX5_COUNTERS_PER_POOL;
-               return (struct mlx5_flow_counter_pool *)cnt - 1;
-       }
-       return cnt->pool;
+       cont = MLX5_CNT_CONTAINER(priv->sh, batch, 0);
+       MLX5_ASSERT(idx / MLX5_COUNTERS_PER_POOL < cont->n);
+       pool = cont->pools[idx / MLX5_COUNTERS_PER_POOL];
+       MLX5_ASSERT(pool);
+       if (ppool)
+               *ppool = pool;
+       return &pool->counters_raw[idx % MLX5_COUNTERS_PER_POOL];
 }
 
 /**
@@ -3962,15 +3871,26 @@ flow_dv_counter_pool_get(struct mlx5_flow_counter *cnt)
 static struct mlx5_flow_counter_pool *
 flow_dv_find_pool_by_id(struct mlx5_pools_container *cont, int id)
 {
-       struct mlx5_flow_counter_pool *pool;
+       uint32_t i;
+       uint32_t n_valid = rte_atomic16_read(&cont->n_valid);
 
-       TAILQ_FOREACH(pool, &cont->pool_list, next) {
+       for (i = 0; i < n_valid; i++) {
+               struct mlx5_flow_counter_pool *pool = cont->pools[i];
                int base = (pool->min_dcs->id / MLX5_COUNTERS_PER_POOL) *
-                               MLX5_COUNTERS_PER_POOL;
+                          MLX5_COUNTERS_PER_POOL;
 
-               if (id >= base && id < base + MLX5_COUNTERS_PER_POOL)
+               if (id >= base && id < base + MLX5_COUNTERS_PER_POOL) {
+                       /*
+                        * Move the pool to the head, as counter allocate
+                        * always gets the first pool in the container.
+                        */
+                       if (pool != TAILQ_FIRST(&cont->pool_list)) {
+                               TAILQ_REMOVE(&cont->pool_list, pool, next);
+                               TAILQ_INSERT_HEAD(&cont->pool_list, pool, next);
+                       }
                        return pool;
-       };
+               }
+       }
        return NULL;
 }
 
@@ -4060,12 +3980,14 @@ flow_dv_container_resize(struct rte_eth_dev *dev, uint32_t batch)
                        MLX5_CNT_CONTAINER(priv->sh, batch, 0);
        struct mlx5_pools_container *new_cont =
                        MLX5_CNT_CONTAINER_UNUSED(priv->sh, batch, 0);
-       struct mlx5_counter_stats_mem_mng *mem_mng;
+       struct mlx5_counter_stats_mem_mng *mem_mng = NULL;
        uint32_t resize = cont->n + MLX5_CNT_CONTAINER_RESIZE;
        uint32_t mem_size = sizeof(struct mlx5_flow_counter_pool *) * resize;
        int i;
 
-       if (cont != MLX5_CNT_CONTAINER(priv->sh, batch, 1)) {
+       /* Fallback mode has no background thread. Skip the check. */
+       if (!priv->counter_fallback &&
+           cont != MLX5_CNT_CONTAINER(priv->sh, batch, 1)) {
                /* The last resize still hasn't detected by the host thread. */
                rte_errno = EAGAIN;
                return NULL;
@@ -4078,16 +4000,29 @@ flow_dv_container_resize(struct rte_eth_dev *dev, uint32_t batch)
        if (cont->n)
                memcpy(new_cont->pools, cont->pools, cont->n *
                       sizeof(struct mlx5_flow_counter_pool *));
-       mem_mng = flow_dv_create_counter_stat_mem_mng(dev,
-               MLX5_CNT_CONTAINER_RESIZE + MLX5_MAX_PENDING_QUERIES);
-       if (!mem_mng) {
-               rte_free(new_cont->pools);
-               return NULL;
+       /*
+        * Fallback mode query the counter directly, no background query
+        * resources are needed.
+        */
+       if (!priv->counter_fallback) {
+               mem_mng = flow_dv_create_counter_stat_mem_mng(dev,
+                       MLX5_CNT_CONTAINER_RESIZE + MLX5_MAX_PENDING_QUERIES);
+               if (!mem_mng) {
+                       rte_free(new_cont->pools);
+                       return NULL;
+               }
+               for (i = 0; i < MLX5_MAX_PENDING_QUERIES; ++i)
+                       LIST_INSERT_HEAD(&priv->sh->cmng.free_stat_raws,
+                                        mem_mng->raws +
+                                        MLX5_CNT_CONTAINER_RESIZE +
+                                        i, next);
+       } else {
+               /*
+                * Release the old container pools directly as no background
+                * thread helps that.
+                */
+               rte_free(cont->pools);
        }
-       for (i = 0; i < MLX5_MAX_PENDING_QUERIES; ++i)
-               LIST_INSERT_HEAD(&priv->sh->cmng.free_stat_raws,
-                                mem_mng->raws + MLX5_CNT_CONTAINER_RESIZE +
-                                i, next);
        new_cont->n = resize;
        rte_atomic16_set(&new_cont->n_valid, rte_atomic16_read(&cont->n_valid));
        TAILQ_INIT(&new_cont->pool_list);
@@ -4105,7 +4040,7 @@ flow_dv_container_resize(struct rte_eth_dev *dev, uint32_t batch)
  * @param[in] dev
  *   Pointer to the Ethernet device structure.
  * @param[in] cnt
- *   Pointer to the flow counter.
+ *   Index to the flow counter.
  * @param[out] pkts
  *   The statistics value of packets.
  * @param[out] bytes
@@ -4115,17 +4050,23 @@ flow_dv_container_resize(struct rte_eth_dev *dev, uint32_t batch)
  *   0 on success, otherwise a negative errno value and rte_errno is set.
  */
 static inline int
-_flow_dv_query_count(struct rte_eth_dev *dev,
-                    struct mlx5_flow_counter *cnt, uint64_t *pkts,
+_flow_dv_query_count(struct rte_eth_dev *dev, uint32_t counter, uint64_t *pkts,
                     uint64_t *bytes)
 {
        struct mlx5_priv *priv = dev->data->dev_private;
-       struct mlx5_flow_counter_pool *pool =
-                       flow_dv_counter_pool_get(cnt);
-       int offset = cnt - &pool->counters_raw[0];
+       struct mlx5_flow_counter_pool *pool = NULL;
+       struct mlx5_flow_counter *cnt;
+       struct mlx5_flow_counter_ext *cnt_ext = NULL;
+       int offset;
 
-       if (priv->counter_fallback)
-               return _flow_dv_query_count_fallback(dev, cnt, pkts, bytes);
+       cnt = flow_dv_counter_get_by_idx(dev, counter, &pool);
+       MLX5_ASSERT(pool);
+       if (counter < MLX5_CNT_BATCH_OFFSET) {
+               cnt_ext = MLX5_CNT_TO_CNT_EXT(pool, cnt);
+               if (priv->counter_fallback)
+                       return mlx5_devx_cmd_flow_counter_query(cnt_ext->dcs, 0,
+                                       0, pkts, bytes, 0, NULL, NULL, 0);
+       }
 
        rte_spinlock_lock(&pool->sl);
        /*
@@ -4133,10 +4074,11 @@ _flow_dv_query_count(struct rte_eth_dev *dev,
         * current allocated in parallel to the host reading.
         * In this case the new counter values must be reported as 0.
         */
-       if (unlikely(!cnt->batch && cnt->dcs->id < pool->raw->min_dcs_id)) {
+       if (unlikely(cnt_ext && cnt_ext->dcs->id < pool->raw->min_dcs_id)) {
                *pkts = 0;
                *bytes = 0;
        } else {
+               offset = cnt - &pool->counters_raw[0];
                *pkts = rte_be_to_cpu_64(pool->raw->data[offset].hits);
                *bytes = rte_be_to_cpu_64(pool->raw->data[offset].bytes);
        }
@@ -4175,25 +4117,39 @@ flow_dv_pool_create(struct rte_eth_dev *dev, struct mlx5_devx_obj *dcs,
                if (!cont)
                        return NULL;
        }
-       size = sizeof(*pool) + MLX5_COUNTERS_PER_POOL *
-                       sizeof(struct mlx5_flow_counter);
+       size = sizeof(*pool);
+       if (!batch)
+               size += MLX5_COUNTERS_PER_POOL *
+                       sizeof(struct mlx5_flow_counter_ext);
        pool = rte_calloc(__func__, 1, size, 0);
        if (!pool) {
                rte_errno = ENOMEM;
                return NULL;
        }
        pool->min_dcs = dcs;
-       pool->raw = cont->init_mem_mng->raws + n_valid %
+       if (!priv->counter_fallback)
+               pool->raw = cont->init_mem_mng->raws + n_valid %
                                                     MLX5_CNT_CONTAINER_RESIZE;
        pool->raw_hw = NULL;
        rte_spinlock_init(&pool->sl);
        /*
         * The generation of the new allocated counters in this pool is 0, 2 in
         * the pool generation makes all the counters valid for allocation.
+        * The start and end query generation protect the counters be released
+        * between the query and update gap period will not be reallocated
+        * without the last query finished and stats updated to the memory.
+        */
+       rte_atomic64_set(&pool->start_query_gen, 0x2);
+       /*
+        * There's no background query thread for fallback mode, set the
+        * end_query_gen to the maximum value since no need to wait for
+        * statistics update.
         */
-       rte_atomic64_set(&pool->query_gen, 0x2);
+       rte_atomic64_set(&pool->end_query_gen, priv->counter_fallback ?
+                        INT64_MAX : 0x2);
        TAILQ_INIT(&pool->counters);
        TAILQ_INSERT_HEAD(&cont->pool_list, pool, next);
+       pool->index = n_valid;
        cont->pools[n_valid] = pool;
        /* Pool initialization must be updated before host thread access. */
        rte_cio_wmb();
@@ -4245,9 +4201,10 @@ flow_dv_counter_pool_prepare(struct rte_eth_dev *dev,
                        rte_atomic64_set(&pool->a64_dcs,
                                         (int64_t)(uintptr_t)dcs);
                }
-               cnt = &pool->counters_raw[dcs->id % MLX5_COUNTERS_PER_POOL];
+               i = dcs->id % MLX5_COUNTERS_PER_POOL;
+               cnt = &pool->counters_raw[i];
                TAILQ_INSERT_HEAD(&pool->counters, cnt, next);
-               cnt->dcs = dcs;
+               MLX5_GET_POOL_CNT_EXT(pool, i)->dcs = dcs;
                *cnt_free = cnt;
                return cont;
        }
@@ -4266,7 +4223,6 @@ flow_dv_counter_pool_prepare(struct rte_eth_dev *dev,
        pool = TAILQ_FIRST(&cont->pool_list);
        for (i = 0; i < MLX5_COUNTERS_PER_POOL; ++i) {
                cnt = &pool->counters_raw[i];
-               cnt->pool = pool;
                TAILQ_INSERT_HEAD(&pool->counters, cnt, next);
        }
        *cnt_free = &pool->counters_raw[0];
@@ -4280,23 +4236,30 @@ flow_dv_counter_pool_prepare(struct rte_eth_dev *dev,
  *   Pointer to the relevant counter pool container.
  * @param[in] id
  *   The shared counter ID to search.
+ * @param[out] ppool
+ *   mlx5 flow counter pool in the container,
  *
  * @return
- *   NULL if not existed, otherwise pointer to the shared counter.
+ *   NULL if not existed, otherwise pointer to the shared extend counter.
  */
-static struct mlx5_flow_counter *
-flow_dv_counter_shared_search(struct mlx5_pools_container *cont,
-                             uint32_t id)
+static struct mlx5_flow_counter_ext *
+flow_dv_counter_shared_search(struct mlx5_pools_container *cont, uint32_t id,
+                             struct mlx5_flow_counter_pool **ppool)
 {
-       static struct mlx5_flow_counter *cnt;
+       static struct mlx5_flow_counter_ext *cnt;
        struct mlx5_flow_counter_pool *pool;
-       int i;
+       uint32_t i;
+       uint32_t n_valid = rte_atomic16_read(&cont->n_valid);
 
-       TAILQ_FOREACH(pool, &cont->pool_list, next) {
+       for (i = 0; i < n_valid; i++) {
+               pool = cont->pools[i];
                for (i = 0; i < MLX5_COUNTERS_PER_POOL; ++i) {
-                       cnt = &pool->counters_raw[i];
-                       if (cnt->ref_cnt && cnt->shared && cnt->id == id)
+                       cnt = MLX5_GET_POOL_CNT_EXT(pool, i);
+                       if (cnt->ref_cnt && cnt->shared && cnt->id == id) {
+                               if (ppool)
+                                       *ppool = cont->pools[i];
                                return cnt;
+                       }
                }
        }
        return NULL;
@@ -4315,15 +4278,16 @@ flow_dv_counter_shared_search(struct mlx5_pools_container *cont,
  *   Counter flow group.
  *
  * @return
- *   pointer to flow counter on success, NULL otherwise and rte_errno is set.
+ *   Index to flow counter on success, 0 otherwise and rte_errno is set.
  */
-static struct mlx5_flow_counter *
+static uint32_t
 flow_dv_counter_alloc(struct rte_eth_dev *dev, uint32_t shared, uint32_t id,
                      uint16_t group)
 {
        struct mlx5_priv *priv = dev->data->dev_private;
        struct mlx5_flow_counter_pool *pool = NULL;
        struct mlx5_flow_counter *cnt_free = NULL;
+       struct mlx5_flow_counter_ext *cnt_ext = NULL;
        /*
         * Currently group 0 flow counter cannot be assigned to a flow if it is
         * not the first one in the batch counter allocation, so it is better
@@ -4332,25 +4296,27 @@ flow_dv_counter_alloc(struct rte_eth_dev *dev, uint32_t shared, uint32_t id,
         * A counter can be shared between different groups so need to take
         * shared counters from the single container.
         */
-       uint32_t batch = (group && !shared) ? 1 : 0;
+       uint32_t batch = (group && !shared && !priv->counter_fallback) ? 1 : 0;
        struct mlx5_pools_container *cont = MLX5_CNT_CONTAINER(priv->sh, batch,
                                                               0);
+       uint32_t cnt_idx;
 
-       if (priv->counter_fallback)
-               return flow_dv_counter_alloc_fallback(dev, shared, id);
        if (!priv->config.devx) {
                rte_errno = ENOTSUP;
-               return NULL;
+               return 0;
        }
        if (shared) {
-               cnt_free = flow_dv_counter_shared_search(cont, id);
-               if (cnt_free) {
-                       if (cnt_free->ref_cnt + 1 == 0) {
+               cnt_ext = flow_dv_counter_shared_search(cont, id, &pool);
+               if (cnt_ext) {
+                       if (cnt_ext->ref_cnt + 1 == 0) {
                                rte_errno = E2BIG;
-                               return NULL;
+                               return 0;
                        }
-                       cnt_free->ref_cnt++;
-                       return cnt_free;
+                       cnt_ext->ref_cnt++;
+                       cnt_idx = pool->index * MLX5_COUNTERS_PER_POOL +
+                                 (cnt_ext->dcs->id % MLX5_COUNTERS_PER_POOL)
+                                 + 1;
+                       return cnt_idx;
                }
        }
        /* Pools which has a free counters are in the start. */
@@ -4365,18 +4331,19 @@ flow_dv_counter_alloc(struct rte_eth_dev *dev, uint32_t shared, uint32_t id,
                 * updated too.
                 */
                cnt_free = TAILQ_FIRST(&pool->counters);
-               if (cnt_free && cnt_free->query_gen + 1 <
-                   rte_atomic64_read(&pool->query_gen))
+               if (cnt_free && cnt_free->query_gen <
+                   rte_atomic64_read(&pool->end_query_gen))
                        break;
                cnt_free = NULL;
        }
        if (!cnt_free) {
                cont = flow_dv_counter_pool_prepare(dev, &cnt_free, batch);
                if (!cont)
-                       return NULL;
+                       return 0;
                pool = TAILQ_FIRST(&cont->pool_list);
        }
-       cnt_free->batch = batch;
+       if (!batch)
+               cnt_ext = MLX5_CNT_TO_CNT_EXT(pool, cnt_free);
        /* Create a DV counter action only in the first time usage. */
        if (!cnt_free->action) {
                uint16_t offset;
@@ -4387,23 +4354,28 @@ flow_dv_counter_alloc(struct rte_eth_dev *dev, uint32_t shared, uint32_t id,
                        dcs = pool->min_dcs;
                } else {
                        offset = 0;
-                       dcs = cnt_free->dcs;
+                       dcs = cnt_ext->dcs;
                }
                cnt_free->action = mlx5_glue->dv_create_flow_action_counter
                                        (dcs->obj, offset);
                if (!cnt_free->action) {
                        rte_errno = errno;
-                       return NULL;
+                       return 0;
                }
        }
+       cnt_idx = MLX5_MAKE_CNT_IDX(pool->index,
+                                   (cnt_free - pool->counters_raw));
+       cnt_idx += batch * MLX5_CNT_BATCH_OFFSET;
        /* Update the counter reset values. */
-       if (_flow_dv_query_count(dev, cnt_free, &cnt_free->hits,
+       if (_flow_dv_query_count(dev, cnt_idx, &cnt_free->hits,
                                 &cnt_free->bytes))
-               return NULL;
-       cnt_free->shared = shared;
-       cnt_free->ref_cnt = 1;
-       cnt_free->id = id;
-       if (!priv->sh->cmng.query_thread_on)
+               return 0;
+       if (cnt_ext) {
+               cnt_ext->shared = shared;
+               cnt_ext->ref_cnt = 1;
+               cnt_ext->id = id;
+       }
+       if (!priv->counter_fallback && !priv->sh->cmng.query_thread_on)
                /* Start the asynchronous batch query by the host thread. */
                mlx5_set_query_alarm(priv->sh);
        TAILQ_REMOVE(&pool->counters, cnt_free, next);
@@ -4412,7 +4384,7 @@ flow_dv_counter_alloc(struct rte_eth_dev *dev, uint32_t shared, uint32_t id,
                TAILQ_REMOVE(&cont->pool_list, pool, next);
                TAILQ_INSERT_TAIL(&cont->pool_list, pool, next);
        }
-       return cnt_free;
+       return cnt_idx;
 }
 
 /**
@@ -4421,28 +4393,33 @@ flow_dv_counter_alloc(struct rte_eth_dev *dev, uint32_t shared, uint32_t id,
  * @param[in] dev
  *   Pointer to the Ethernet device structure.
  * @param[in] counter
- *   Pointer to the counter handler.
+ *   Index to the counter handler.
  */
 static void
-flow_dv_counter_release(struct rte_eth_dev *dev,
-                       struct mlx5_flow_counter *counter)
+flow_dv_counter_release(struct rte_eth_dev *dev, uint32_t counter)
 {
-       struct mlx5_priv *priv = dev->data->dev_private;
+       struct mlx5_flow_counter_pool *pool = NULL;
+       struct mlx5_flow_counter *cnt;
+       struct mlx5_flow_counter_ext *cnt_ext = NULL;
 
        if (!counter)
                return;
-       if (priv->counter_fallback) {
-               flow_dv_counter_release_fallback(dev, counter);
-               return;
-       }
-       if (--counter->ref_cnt == 0) {
-               struct mlx5_flow_counter_pool *pool =
-                               flow_dv_counter_pool_get(counter);
-
-               /* Put the counter in the end - the last updated one. */
-               TAILQ_INSERT_TAIL(&pool->counters, counter, next);
-               counter->query_gen = rte_atomic64_read(&pool->query_gen);
+       cnt = flow_dv_counter_get_by_idx(dev, counter, &pool);
+       MLX5_ASSERT(pool);
+       if (counter < MLX5_CNT_BATCH_OFFSET) {
+               cnt_ext = MLX5_CNT_TO_CNT_EXT(pool, cnt);
+               if (cnt_ext && --cnt_ext->ref_cnt)
+                       return;
        }
+       /* Put the counter in the end - the last updated one. */
+       TAILQ_INSERT_TAIL(&pool->counters, cnt, next);
+       /*
+        * Counters released between query trigger and handler need
+        * to wait the next round of query. Since the packets arrive
+        * in the gap period will not be taken into account to the
+        * old counter.
+        */
+       cnt->query_gen = rte_atomic64_read(&pool->start_query_gen);
 }
 
 /**
@@ -7506,10 +7483,11 @@ __flow_dv_translate(struct rte_eth_dev *dev,
                                                        count->shared,
                                                        count->id,
                                                        dev_flow->dv.group);
-                       if (flow->counter == NULL)
+                       if (!flow->counter)
                                goto cnt_err;
                        dev_flow->dv.actions[actions_n++] =
-                               flow->counter->action;
+                                 (flow_dv_counter_get_by_idx(dev,
+                                 flow->counter, NULL))->action;
                        action_flags |= MLX5_FLOW_ACTION_COUNT;
                        break;
 cnt_err:
@@ -8382,7 +8360,7 @@ __flow_dv_destroy(struct rte_eth_dev *dev, struct rte_flow *flow)
        __flow_dv_remove(dev, flow);
        if (flow->counter) {
                flow_dv_counter_release(dev, flow->counter);
-               flow->counter = NULL;
+               flow->counter = 0;
        }
        if (flow->meter) {
                mlx5_flow_meter_detach(flow->meter);
@@ -8439,6 +8417,10 @@ flow_dv_query_count(struct rte_eth_dev *dev, struct rte_flow *flow,
                                          "counters are not supported");
        if (flow->counter) {
                uint64_t pkts, bytes;
+               struct mlx5_flow_counter *cnt;
+
+               cnt = flow_dv_counter_get_by_idx(dev, flow->counter,
+                                                NULL);
                int err = _flow_dv_query_count(dev, flow->counter, &pkts,
                                               &bytes);
 
@@ -8448,11 +8430,11 @@ flow_dv_query_count(struct rte_eth_dev *dev, struct rte_flow *flow,
                                        NULL, "cannot read counters");
                qc->hits_set = 1;
                qc->bytes_set = 1;
-               qc->hits = pkts - flow->counter->hits;
-               qc->bytes = bytes - flow->counter->bytes;
+               qc->hits = pkts - cnt->hits;
+               qc->bytes = bytes - cnt->bytes;
                if (qc->reset) {
-                       flow->counter->hits = pkts;
-                       flow->counter->bytes = bytes;
+                       cnt->hits = pkts;
+                       cnt->bytes = bytes;
                }
                return 0;
        }
@@ -8702,9 +8684,12 @@ flow_dv_create_mtr_tbl(struct rte_eth_dev *dev,
        }
        /* Create meter count actions */
        for (i = 0; i <= RTE_MTR_DROPPED; i++) {
+               struct mlx5_flow_counter *cnt;
                if (!fm->policer_stats.cnt[i])
                        continue;
-               mtb->count_actns[i] = fm->policer_stats.cnt[i]->action;
+               cnt = flow_dv_counter_get_by_idx(dev,
+                     fm->policer_stats.cnt[i], NULL);
+               mtb->count_actns[i] = cnt->action;
        }
        /* Create drop action. */
        mtb->drop_actn = mlx5_glue->dr_create_flow_action_drop();
@@ -8923,7 +8908,7 @@ error:
  * @param[in] dev
  *   Pointer to the Ethernet device structure.
  * @param[in] cnt
- *   Pointer to the flow counter.
+ *   Index to the flow counter.
  * @param[in] clear
  *   Set to clear the counter statistics.
  * @param[out] pkts
@@ -8935,19 +8920,21 @@ error:
  *   0 on success, otherwise return -1.
  */
 static int
-flow_dv_counter_query(struct rte_eth_dev *dev,
-                     struct mlx5_flow_counter *cnt, bool clear,
+flow_dv_counter_query(struct rte_eth_dev *dev, uint32_t counter, bool clear,
                      uint64_t *pkts, uint64_t *bytes)
 {
        struct mlx5_priv *priv = dev->data->dev_private;
+       struct mlx5_flow_counter *cnt;
        uint64_t inn_pkts, inn_bytes;
        int ret;
 
        if (!priv->config.devx)
                return -1;
-       ret = _flow_dv_query_count(dev, cnt, &inn_pkts, &inn_bytes);
+
+       ret = _flow_dv_query_count(dev, counter, &inn_pkts, &inn_bytes);
        if (ret)
                return -1;
+       cnt = flow_dv_counter_get_by_idx(dev, counter, NULL);
        *pkts = inn_pkts - cnt->hits;
        *bytes = inn_bytes - cnt->bytes;
        if (clear) {
@@ -9017,10 +9004,10 @@ flow_dv_destroy(struct rte_eth_dev *dev, struct rte_flow *flow)
 /*
  * Mutex-protected thunk to lock-free flow_dv_counter_alloc().
  */
-static struct mlx5_flow_counter *
+static uint32_t
 flow_dv_counter_allocate(struct rte_eth_dev *dev)
 {
-       struct mlx5_flow_counter *cnt;
+       uint32_t cnt;
 
        flow_dv_shared_lock(dev);
        cnt = flow_dv_counter_alloc(dev, 0, 0, 1);
@@ -9032,7 +9019,7 @@ flow_dv_counter_allocate(struct rte_eth_dev *dev)
  * Mutex-protected thunk to lock-free flow_dv_counter_release().
  */
 static void
-flow_dv_counter_free(struct rte_eth_dev *dev, struct mlx5_flow_counter *cnt)
+flow_dv_counter_free(struct rte_eth_dev *dev, uint32_t cnt)
 {
        flow_dv_shared_lock(dev);
        flow_dv_counter_release(dev, cnt);