net/mlx5: use indexed pool as id generator
[dpdk.git] / drivers / net / mlx5 / mlx5_flow_dv.c
index dafe07f..bfcba45 100644 (file)
@@ -33,6 +33,7 @@
 #include "mlx5_flow.h"
 #include "mlx5_flow_os.h"
 #include "mlx5_rxtx.h"
+#include "rte_pmd_mlx5.h"
 
 #ifdef HAVE_IBV_FLOW_DV_SUPPORT
 
@@ -2873,8 +2874,10 @@ flow_dv_encap_decap_resource_register
                        struct mlx5_flow_dv_encap_decap_resource, entry);
                DRV_LOG(DEBUG, "encap/decap resource %p: refcnt %d++",
                        (void *)cache_resource,
-                       rte_atomic32_read(&cache_resource->refcnt));
-               rte_atomic32_inc(&cache_resource->refcnt);
+                       __atomic_load_n(&cache_resource->refcnt,
+                                       __ATOMIC_RELAXED));
+               __atomic_fetch_add(&cache_resource->refcnt, 1,
+                                  __ATOMIC_RELAXED);
                dev_flow->handle->dvh.rix_encap_decap = cache_resource->idx;
                dev_flow->dv.encap_decap = cache_resource;
                return 0;
@@ -2897,8 +2900,7 @@ flow_dv_encap_decap_resource_register
                                          RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
                                          NULL, "cannot create action");
        }
-       rte_atomic32_init(&cache_resource->refcnt);
-       rte_atomic32_inc(&cache_resource->refcnt);
+       __atomic_store_n(&cache_resource->refcnt, 1, __ATOMIC_RELAXED);
        if (mlx5_hlist_insert_ex(sh->encaps_decaps, &cache_resource->entry,
                                 flow_dv_encap_decap_resource_match,
                                 (void *)cache_resource)) {
@@ -2913,7 +2915,7 @@ flow_dv_encap_decap_resource_register
        dev_flow->dv.encap_decap = cache_resource;
        DRV_LOG(DEBUG, "new encap/decap resource %p: refcnt %d++",
                (void *)cache_resource,
-               rte_atomic32_read(&cache_resource->refcnt));
+               __atomic_load_n(&cache_resource->refcnt, __ATOMIC_RELAXED));
        return 0;
 }
 
@@ -2944,7 +2946,7 @@ flow_dv_jump_tbl_resource_register
        int cnt, ret;
 
        MLX5_ASSERT(tbl);
-       cnt = rte_atomic32_read(&tbl_data->jump.refcnt);
+       cnt = __atomic_load_n(&tbl_data->jump.refcnt, __ATOMIC_ACQUIRE);
        if (!cnt) {
                ret = mlx5_flow_os_create_flow_action_dest_flow_tbl
                                (tbl->obj, &tbl_data->jump.action);
@@ -2961,7 +2963,7 @@ flow_dv_jump_tbl_resource_register
                DRV_LOG(DEBUG, "existed jump table resource %p: refcnt %d++",
                        (void *)&tbl_data->jump, cnt);
        }
-       rte_atomic32_inc(&tbl_data->jump.refcnt);
+       __atomic_fetch_add(&tbl_data->jump.refcnt, 1, __ATOMIC_RELEASE);
        dev_flow->handle->rix_jump = tbl_data->idx;
        dev_flow->dv.jump = &tbl_data->jump;
        return 0;
@@ -2986,7 +2988,7 @@ flow_dv_default_miss_resource_register(struct rte_eth_dev *dev,
        struct mlx5_dev_ctx_shared *sh = priv->sh;
        struct mlx5_flow_default_miss_resource *cache_resource =
                        &sh->default_miss;
-       int cnt = rte_atomic32_read(&cache_resource->refcnt);
+       int cnt = __atomic_load_n(&cache_resource->refcnt, __ATOMIC_ACQUIRE);
 
        if (!cnt) {
                MLX5_ASSERT(cache_resource->action);
@@ -2999,7 +3001,7 @@ flow_dv_default_miss_resource_register(struct rte_eth_dev *dev,
                DRV_LOG(DEBUG, "new default miss resource %p: refcnt %d++",
                                (void *)cache_resource->action, cnt);
        }
-       rte_atomic32_inc(&cache_resource->refcnt);
+       __atomic_fetch_add(&cache_resource->refcnt, 1, __ATOMIC_RELEASE);
        return 0;
 }
 
@@ -3038,8 +3040,10 @@ flow_dv_port_id_action_resource_register
                        DRV_LOG(DEBUG, "port id action resource resource %p: "
                                "refcnt %d++",
                                (void *)cache_resource,
-                               rte_atomic32_read(&cache_resource->refcnt));
-                       rte_atomic32_inc(&cache_resource->refcnt);
+                               __atomic_load_n(&cache_resource->refcnt,
+                                               __ATOMIC_RELAXED));
+                       __atomic_fetch_add(&cache_resource->refcnt, 1,
+                                          __ATOMIC_RELAXED);
                        dev_flow->handle->rix_port_id_action = idx;
                        dev_flow->dv.port_id_action = cache_resource;
                        return 0;
@@ -3062,15 +3066,14 @@ flow_dv_port_id_action_resource_register
                                          RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
                                          NULL, "cannot create action");
        }
-       rte_atomic32_init(&cache_resource->refcnt);
-       rte_atomic32_inc(&cache_resource->refcnt);
+       __atomic_store_n(&cache_resource->refcnt, 1, __ATOMIC_RELAXED);
        ILIST_INSERT(sh->ipool[MLX5_IPOOL_PORT_ID], &sh->port_id_action_list,
                     dev_flow->handle->rix_port_id_action, cache_resource,
                     next);
        dev_flow->dv.port_id_action = cache_resource;
        DRV_LOG(DEBUG, "new port id action resource %p: refcnt %d++",
                (void *)cache_resource,
-               rte_atomic32_read(&cache_resource->refcnt));
+               __atomic_load_n(&cache_resource->refcnt, __ATOMIC_RELAXED));
        return 0;
 }
 
@@ -3111,8 +3114,10 @@ flow_dv_push_vlan_action_resource_register
                        DRV_LOG(DEBUG, "push-VLAN action resource resource %p: "
                                "refcnt %d++",
                                (void *)cache_resource,
-                               rte_atomic32_read(&cache_resource->refcnt));
-                       rte_atomic32_inc(&cache_resource->refcnt);
+                               __atomic_load_n(&cache_resource->refcnt,
+                                               __ATOMIC_RELAXED));
+                       __atomic_fetch_add(&cache_resource->refcnt, 1,
+                                          __ATOMIC_RELAXED);
                        dev_flow->handle->dvh.rix_push_vlan = idx;
                        dev_flow->dv.push_vlan_res = cache_resource;
                        return 0;
@@ -3141,8 +3146,7 @@ flow_dv_push_vlan_action_resource_register
                                          RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
                                          NULL, "cannot create action");
        }
-       rte_atomic32_init(&cache_resource->refcnt);
-       rte_atomic32_inc(&cache_resource->refcnt);
+       __atomic_store_n(&cache_resource->refcnt, 1, __ATOMIC_RELAXED);
        ILIST_INSERT(sh->ipool[MLX5_IPOOL_PUSH_VLAN],
                     &sh->push_vlan_action_list,
                     dev_flow->handle->dvh.rix_push_vlan,
@@ -3150,7 +3154,7 @@ flow_dv_push_vlan_action_resource_register
        dev_flow->dv.push_vlan_res = cache_resource;
        DRV_LOG(DEBUG, "new push vlan action resource %p: refcnt %d++",
                (void *)cache_resource,
-               rte_atomic32_read(&cache_resource->refcnt));
+               __atomic_load_n(&cache_resource->refcnt, __ATOMIC_RELAXED));
        return 0;
 }
 /**
@@ -4550,8 +4554,10 @@ flow_dv_modify_hdr_resource_register
                                        entry);
                DRV_LOG(DEBUG, "modify-header resource %p: refcnt %d++",
                        (void *)cache_resource,
-                       rte_atomic32_read(&cache_resource->refcnt));
-               rte_atomic32_inc(&cache_resource->refcnt);
+                       __atomic_load_n(&cache_resource->refcnt,
+                                       __ATOMIC_RELAXED));
+               __atomic_fetch_add(&cache_resource->refcnt, 1,
+                                  __ATOMIC_RELAXED);
                dev_flow->handle->dvh.modify_hdr = cache_resource;
                return 0;
 
@@ -4575,8 +4581,7 @@ flow_dv_modify_hdr_resource_register
                                          RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
                                          NULL, "cannot create action");
        }
-       rte_atomic32_init(&cache_resource->refcnt);
-       rte_atomic32_inc(&cache_resource->refcnt);
+       __atomic_store_n(&cache_resource->refcnt, 1, __ATOMIC_RELAXED);
        if (mlx5_hlist_insert_ex(sh->modify_cmds, &cache_resource->entry,
                                 flow_dv_modify_hdr_resource_match,
                                 (void *)cache_resource)) {
@@ -4590,7 +4595,7 @@ flow_dv_modify_hdr_resource_register
        dev_flow->handle->dvh.modify_hdr = cache_resource;
        DRV_LOG(DEBUG, "new modify-header resource %p: refcnt %d++",
                (void *)cache_resource,
-               rte_atomic32_read(&cache_resource->refcnt));
+               __atomic_load_n(&cache_resource->refcnt, __ATOMIC_RELAXED));
        return 0;
 }
 
@@ -6248,9 +6253,11 @@ flow_dv_prepare(struct rte_eth_dev *dev,
        struct mlx5_flow *dev_flow;
        struct mlx5_flow_handle *dev_handle;
        struct mlx5_priv *priv = dev->data->dev_private;
+       struct mlx5_flow_workspace *wks = mlx5_flow_get_thread_workspace();
 
+       MLX5_ASSERT(wks);
        /* In case of corrupting the memory. */
-       if (priv->flow_idx >= MLX5_NUM_MAX_DEV_FLOWS) {
+       if (wks->flow_idx >= MLX5_NUM_MAX_DEV_FLOWS) {
                rte_flow_error_set(error, ENOSPC,
                                   RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
                                   "not free temporary device flow");
@@ -6264,8 +6271,8 @@ flow_dv_prepare(struct rte_eth_dev *dev,
                                   "not enough memory to create flow handle");
                return NULL;
        }
-       /* No multi-thread supporting. */
-       dev_flow = &((struct mlx5_flow *)priv->inter_flows)[priv->flow_idx++];
+       MLX5_ASSERT(wks->flow_idx + 1 < RTE_DIM(wks->flows));
+       dev_flow = &wks->flows[wks->flow_idx++];
        dev_flow->handle = dev_handle;
        dev_flow->handle_idx = handle_idx;
        /*
@@ -8021,7 +8028,7 @@ flow_dv_tbl_resource_get(struct rte_eth_dev *dev,
                tbl_data = container_of(pos, struct mlx5_flow_tbl_data_entry,
                                        entry);
                tbl = &tbl_data->tbl;
-               rte_atomic32_inc(&tbl->refcnt);
+               __atomic_fetch_add(&tbl->refcnt, 1, __ATOMIC_RELAXED);
                return tbl;
        }
        tbl_data = mlx5_ipool_zmalloc(sh->ipool[MLX5_IPOOL_JUMP], &idx);
@@ -8056,9 +8063,9 @@ flow_dv_tbl_resource_get(struct rte_eth_dev *dev,
         * No multi-threads now, but still better to initialize the reference
         * count before insert it into the hash list.
         */
-       rte_atomic32_init(&tbl->refcnt);
+       __atomic_store_n(&tbl->refcnt, 0, __ATOMIC_RELAXED);
        /* Jump action reference count is initialized here. */
-       rte_atomic32_init(&tbl_data->jump.refcnt);
+       __atomic_store_n(&tbl_data->jump.refcnt, 0, __ATOMIC_RELAXED);
        pos->key = table_key.v64;
        ret = mlx5_hlist_insert(sh->flow_tbls, pos);
        if (ret < 0) {
@@ -8068,7 +8075,7 @@ flow_dv_tbl_resource_get(struct rte_eth_dev *dev,
                mlx5_flow_os_destroy_flow_tbl(tbl->obj);
                mlx5_ipool_free(sh->ipool[MLX5_IPOOL_JUMP], idx);
        }
-       rte_atomic32_inc(&tbl->refcnt);
+       __atomic_fetch_add(&tbl->refcnt, 1, __ATOMIC_RELAXED);
        return tbl;
 }
 
@@ -8094,7 +8101,7 @@ flow_dv_tbl_resource_release(struct rte_eth_dev *dev,
 
        if (!tbl)
                return 0;
-       if (rte_atomic32_dec_and_test(&tbl->refcnt)) {
+       if (__atomic_sub_fetch(&tbl->refcnt, 1, __ATOMIC_RELAXED) == 0) {
                struct mlx5_hlist_entry *pos = &tbl_data->entry;
 
                mlx5_flow_os_destroy_flow_tbl(tbl->obj);
@@ -8125,8 +8132,8 @@ flow_dv_tbl_resource_release(struct rte_eth_dev *dev,
                                mlx5_hlist_remove(tunnel_grp_hash, he);
                                mlx5_free(tte);
                        }
-                       mlx5_flow_id_release(mlx5_tunnel_hub(dev)->table_ids,
-                                            tunnel_flow_tbl_to_id(table_id));
+                       mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_TNL_TBL_ID],
+                                       tunnel_flow_tbl_to_id(table_id));
                        DRV_LOG(DEBUG,
                                "port %u release table_id %#x tunnel %u group %u",
                                dev->data->port_id, table_id,
@@ -8197,8 +8204,10 @@ flow_dv_matcher_register(struct rte_eth_dev *dev,
                                cache_matcher->priority,
                                key->direction ? "tx" : "rx",
                                (void *)cache_matcher,
-                               rte_atomic32_read(&cache_matcher->refcnt));
-                       rte_atomic32_inc(&cache_matcher->refcnt);
+                               __atomic_load_n(&cache_matcher->refcnt,
+                                               __ATOMIC_RELAXED));
+                       __atomic_fetch_add(&cache_matcher->refcnt, 1,
+                                          __ATOMIC_RELAXED);
                        dev_flow->handle->dvh.matcher = cache_matcher;
                        /* old matcher should not make the table ref++. */
                        flow_dv_tbl_resource_release(dev, tbl);
@@ -8233,16 +8242,15 @@ flow_dv_matcher_register(struct rte_eth_dev *dev,
        }
        /* Save the table information */
        cache_matcher->tbl = tbl;
-       rte_atomic32_init(&cache_matcher->refcnt);
        /* only matcher ref++, table ref++ already done above in get API. */
-       rte_atomic32_inc(&cache_matcher->refcnt);
+       __atomic_store_n(&cache_matcher->refcnt, 1, __ATOMIC_RELAXED);
        LIST_INSERT_HEAD(&tbl_data->matchers, cache_matcher, next);
        dev_flow->handle->dvh.matcher = cache_matcher;
        DRV_LOG(DEBUG, "%s group %u priority %hd new %s matcher %p: refcnt %d",
                key->domain ? "FDB" : "NIC", key->table_id,
                cache_matcher->priority,
                key->direction ? "tx" : "rx", (void *)cache_matcher,
-               rte_atomic32_read(&cache_matcher->refcnt));
+               __atomic_load_n(&cache_matcher->refcnt, __ATOMIC_RELAXED));
        return 0;
 }
 
@@ -8279,12 +8287,14 @@ flow_dv_tag_resource_register
        if (entry) {
                cache_resource = container_of
                        (entry, struct mlx5_flow_dv_tag_resource, entry);
-               rte_atomic32_inc(&cache_resource->refcnt);
+               __atomic_fetch_add(&cache_resource->refcnt, 1,
+                                  __ATOMIC_RELAXED);
                dev_flow->handle->dvh.rix_tag = cache_resource->idx;
                dev_flow->dv.tag_resource = cache_resource;
                DRV_LOG(DEBUG, "cached tag resource %p: refcnt now %d++",
                        (void *)cache_resource,
-                       rte_atomic32_read(&cache_resource->refcnt));
+                       __atomic_load_n(&cache_resource->refcnt,
+                                       __ATOMIC_RELAXED));
                return 0;
        }
        /* Register new resource. */
@@ -8303,8 +8313,7 @@ flow_dv_tag_resource_register
                                          RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
                                          NULL, "cannot create action");
        }
-       rte_atomic32_init(&cache_resource->refcnt);
-       rte_atomic32_inc(&cache_resource->refcnt);
+       __atomic_store_n(&cache_resource->refcnt, 1, __ATOMIC_RELAXED);
        if (mlx5_hlist_insert(sh->tag_table, &cache_resource->entry)) {
                mlx5_flow_os_destroy_flow_action(cache_resource->action);
                mlx5_free(cache_resource);
@@ -8315,7 +8324,7 @@ flow_dv_tag_resource_register
        dev_flow->dv.tag_resource = cache_resource;
        DRV_LOG(DEBUG, "new tag resource %p: refcnt now %d++",
                (void *)cache_resource,
-               rte_atomic32_read(&cache_resource->refcnt));
+               __atomic_load_n(&cache_resource->refcnt, __ATOMIC_RELAXED));
        return 0;
 }
 
@@ -8343,8 +8352,8 @@ flow_dv_tag_release(struct rte_eth_dev *dev,
                return 0;
        DRV_LOG(DEBUG, "port %u tag %p: refcnt %d--",
                dev->data->port_id, (void *)tag,
-               rte_atomic32_read(&tag->refcnt));
-       if (rte_atomic32_dec_and_test(&tag->refcnt)) {
+               __atomic_load_n(&tag->refcnt, __ATOMIC_RELAXED));
+       if (__atomic_sub_fetch(&tag->refcnt, 1, __ATOMIC_RELAXED) == 0) {
                claim_zero(mlx5_flow_os_destroy_flow_action(tag->action));
                mlx5_hlist_remove(sh->tag_table, &tag->entry);
                DRV_LOG(DEBUG, "port %u tag %p: removed",
@@ -8935,11 +8944,12 @@ flow_dv_translate_action_sample(struct rte_eth_dev *dev,
        const struct rte_flow_action_queue *queue;
        struct mlx5_flow_sub_actions_list *sample_act;
        struct mlx5_flow_sub_actions_idx *sample_idx;
-       struct mlx5_flow_rss_desc *rss_desc = &((struct mlx5_flow_rss_desc *)
-                                             priv->rss_desc)
-                                             [!!priv->flow_nested_idx];
+       struct mlx5_flow_workspace *wks = mlx5_flow_get_thread_workspace();
+       struct mlx5_flow_rss_desc *rss_desc;
        uint64_t action_flags = 0;
 
+       MLX5_ASSERT(wks);
+       rss_desc = &wks->rss_desc[!!wks->flow_nested_idx];
        sample_act = &res->sample_act;
        sample_idx = &res->sample_idx;
        sample_action = (const struct rte_flow_action_sample *)action->conf;
@@ -9141,18 +9151,18 @@ flow_dv_create_action_sample(struct rte_eth_dev *dev,
                             uint64_t action_flags,
                             struct rte_flow_error *error)
 {
-       struct mlx5_priv *priv = dev->data->dev_private;
        /* update normal path action resource into last index of array */
        uint32_t dest_index = MLX5_MAX_DEST_NUM - 1;
        struct mlx5_flow_sub_actions_list *sample_act =
                                        &mdest_res->sample_act[dest_index];
-       struct mlx5_flow_rss_desc *rss_desc = &((struct mlx5_flow_rss_desc *)
-                                             priv->rss_desc)
-                                             [!!priv->flow_nested_idx];
+       struct mlx5_flow_workspace *wks = mlx5_flow_get_thread_workspace();
+       struct mlx5_flow_rss_desc *rss_desc;
        uint32_t normal_idx = 0;
        struct mlx5_hrxq *hrxq;
        uint32_t hrxq_idx;
 
+       MLX5_ASSERT(wks);
+       rss_desc = &wks->rss_desc[!!wks->flow_nested_idx];
        if (num_of_dest > 1) {
                if (sample_act->action_flags & MLX5_FLOW_ACTION_QUEUE) {
                        /* Handle QP action for mirroring */
@@ -9242,9 +9252,8 @@ __flow_dv_translate(struct rte_eth_dev *dev,
        struct mlx5_dev_config *dev_conf = &priv->config;
        struct rte_flow *flow = dev_flow->flow;
        struct mlx5_flow_handle *handle = dev_flow->handle;
-       struct mlx5_flow_rss_desc *rss_desc = &((struct mlx5_flow_rss_desc *)
-                                             priv->rss_desc)
-                                             [!!priv->flow_nested_idx];
+       struct mlx5_flow_workspace *wks = mlx5_flow_get_thread_workspace();
+       struct mlx5_flow_rss_desc *rss_desc;
        uint64_t item_flags = 0;
        uint64_t last_item = 0;
        uint64_t action_flags = 0;
@@ -9290,6 +9299,8 @@ __flow_dv_translate(struct rte_eth_dev *dev,
                .fdb_def_rule = !!priv->fdb_def_rule,
        };
 
+       MLX5_ASSERT(wks);
+       rss_desc = &wks->rss_desc[!!wks->flow_nested_idx];
        memset(&mdest_res, 0, sizeof(struct mlx5_flow_dv_dest_array_resource));
        memset(&sample_res, 0, sizeof(struct mlx5_flow_dv_sample_resource));
        mhdr_res->ft_type = attr->egress ? MLX5DV_FLOW_TABLE_TYPE_NIC_TX :
@@ -10273,6 +10284,7 @@ __flow_dv_rss_get_hrxq(struct rte_eth_dev *dev, struct rte_flow *flow,
                           struct mlx5_hrxq **hrxq)
 {
        struct mlx5_priv *priv = dev->data->dev_private;
+       struct mlx5_flow_workspace *wks = mlx5_flow_get_thread_workspace();
        uint32_t hrxq_idx;
 
        if (flow->shared_rss) {
@@ -10283,12 +10295,12 @@ __flow_dv_rss_get_hrxq(struct rte_eth_dev *dev, struct rte_flow *flow,
                if (hrxq_idx) {
                        *hrxq = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_HRXQ],
                                               hrxq_idx);
-                       rte_atomic32_inc(&(*hrxq)->refcnt);
+                       __atomic_fetch_add(&(*hrxq)->refcnt, 1,
+                                          __ATOMIC_RELAXED);
                }
        } else {
                struct mlx5_flow_rss_desc *rss_desc =
-                               &((struct mlx5_flow_rss_desc *)priv->rss_desc)
-                               [!!priv->flow_nested_idx];
+                               &wks->rss_desc[!!wks->flow_nested_idx];
 
                MLX5_ASSERT(rss_desc->queue_num);
                hrxq_idx = mlx5_hrxq_get(dev, rss_desc->key,
@@ -10339,9 +10351,11 @@ __flow_dv_apply(struct rte_eth_dev *dev, struct rte_flow *flow,
        int n;
        int err;
        int idx;
+       struct mlx5_flow_workspace *wks = mlx5_flow_get_thread_workspace();
 
-       for (idx = priv->flow_idx - 1; idx >= priv->flow_nested_idx; idx--) {
-               dev_flow = &((struct mlx5_flow *)priv->inter_flows)[idx];
+       MLX5_ASSERT(wks);
+       for (idx = wks->flow_idx - 1; idx >= wks->flow_nested_idx; idx--) {
+               dev_flow = &wks->flows[idx];
                dv = &dev_flow->dv;
                dh = dev_flow->handle;
                dv_h = &dh->dvh;
@@ -10462,8 +10476,8 @@ flow_dv_matcher_release(struct rte_eth_dev *dev,
        MLX5_ASSERT(matcher->matcher_object);
        DRV_LOG(DEBUG, "port %u matcher %p: refcnt %d--",
                dev->data->port_id, (void *)matcher,
-               rte_atomic32_read(&matcher->refcnt));
-       if (rte_atomic32_dec_and_test(&matcher->refcnt)) {
+               __atomic_load_n(&matcher->refcnt, __ATOMIC_RELAXED));
+       if (__atomic_sub_fetch(&matcher->refcnt, 1, __ATOMIC_RELAXED) == 0) {
                claim_zero(mlx5_flow_os_destroy_flow_matcher
                           (matcher->matcher_object));
                LIST_REMOVE(matcher, next);
@@ -10503,8 +10517,9 @@ flow_dv_encap_decap_resource_release(struct rte_eth_dev *dev,
        MLX5_ASSERT(cache_resource->action);
        DRV_LOG(DEBUG, "encap/decap resource %p: refcnt %d--",
                (void *)cache_resource,
-               rte_atomic32_read(&cache_resource->refcnt));
-       if (rte_atomic32_dec_and_test(&cache_resource->refcnt)) {
+               __atomic_load_n(&cache_resource->refcnt, __ATOMIC_RELAXED));
+       if (__atomic_sub_fetch(&cache_resource->refcnt, 1,
+                              __ATOMIC_RELAXED) == 0) {
                claim_zero(mlx5_flow_os_destroy_flow_action
                                                (cache_resource->action));
                mlx5_hlist_remove(priv->sh->encaps_decaps,
@@ -10544,8 +10559,9 @@ flow_dv_jump_tbl_resource_release(struct rte_eth_dev *dev,
        MLX5_ASSERT(cache_resource->action);
        DRV_LOG(DEBUG, "jump table resource %p: refcnt %d--",
                (void *)cache_resource,
-               rte_atomic32_read(&cache_resource->refcnt));
-       if (rte_atomic32_dec_and_test(&cache_resource->refcnt)) {
+               __atomic_load_n(&cache_resource->refcnt, __ATOMIC_RELAXED));
+       if (__atomic_sub_fetch(&cache_resource->refcnt, 1,
+                              __ATOMIC_RELAXED) == 0) {
                claim_zero(mlx5_flow_os_destroy_flow_action
                                                (cache_resource->action));
                /* jump action memory free is inside the table release. */
@@ -10576,8 +10592,10 @@ flow_dv_default_miss_resource_release(struct rte_eth_dev *dev)
        MLX5_ASSERT(cache_resource->action);
        DRV_LOG(DEBUG, "default miss resource %p: refcnt %d--",
                        (void *)cache_resource->action,
-                       rte_atomic32_read(&cache_resource->refcnt));
-       if (rte_atomic32_dec_and_test(&cache_resource->refcnt)) {
+                       __atomic_load_n(&cache_resource->refcnt,
+                                       __ATOMIC_RELAXED));
+       if (__atomic_sub_fetch(&cache_resource->refcnt, 1,
+                              __ATOMIC_RELAXED) == 0) {
                claim_zero(mlx5_glue->destroy_flow_action
                                (cache_resource->action));
                DRV_LOG(DEBUG, "default miss resource %p: removed",
@@ -10609,8 +10627,9 @@ flow_dv_modify_hdr_resource_release(struct rte_eth_dev *dev,
        MLX5_ASSERT(cache_resource->action);
        DRV_LOG(DEBUG, "modify-header resource %p: refcnt %d--",
                (void *)cache_resource,
-               rte_atomic32_read(&cache_resource->refcnt));
-       if (rte_atomic32_dec_and_test(&cache_resource->refcnt)) {
+               __atomic_load_n(&cache_resource->refcnt, __ATOMIC_RELAXED));
+       if (__atomic_sub_fetch(&cache_resource->refcnt, 1,
+                               __ATOMIC_RELAXED) == 0) {
                claim_zero(mlx5_flow_os_destroy_flow_action
                                                (cache_resource->action));
                mlx5_hlist_remove(priv->sh->modify_cmds,
@@ -10649,8 +10668,9 @@ flow_dv_port_id_action_resource_release(struct rte_eth_dev *dev,
        MLX5_ASSERT(cache_resource->action);
        DRV_LOG(DEBUG, "port ID action resource %p: refcnt %d--",
                (void *)cache_resource,
-               rte_atomic32_read(&cache_resource->refcnt));
-       if (rte_atomic32_dec_and_test(&cache_resource->refcnt)) {
+               __atomic_load_n(&cache_resource->refcnt, __ATOMIC_RELAXED));
+       if (__atomic_sub_fetch(&cache_resource->refcnt, 1,
+                              __ATOMIC_RELAXED) == 0) {
                claim_zero(mlx5_flow_os_destroy_flow_action
                                                (cache_resource->action));
                ILIST_REMOVE(priv->sh->ipool[MLX5_IPOOL_PORT_ID],
@@ -10690,8 +10710,9 @@ flow_dv_push_vlan_action_resource_release(struct rte_eth_dev *dev,
        MLX5_ASSERT(cache_resource->action);
        DRV_LOG(DEBUG, "push VLAN action resource %p: refcnt %d--",
                (void *)cache_resource,
-               rte_atomic32_read(&cache_resource->refcnt));
-       if (rte_atomic32_dec_and_test(&cache_resource->refcnt)) {
+               __atomic_load_n(&cache_resource->refcnt, __ATOMIC_RELAXED));
+       if (__atomic_sub_fetch(&cache_resource->refcnt, 1,
+                              __ATOMIC_RELAXED) == 0) {
                claim_zero(mlx5_flow_os_destroy_flow_action
                                                (cache_resource->action));
                ILIST_REMOVE(priv->sh->ipool[MLX5_IPOOL_PUSH_VLAN],
@@ -12310,6 +12331,31 @@ flow_dv_action_update(struct rte_eth_dev *dev,
        return ret;
 }
 
+static int
+flow_dv_sync_domain(struct rte_eth_dev *dev, uint32_t domains, uint32_t flags)
+{
+       struct mlx5_priv *priv = dev->data->dev_private;
+       int ret = 0;
+
+       if ((domains & MLX5_DOMAIN_BIT_NIC_RX) && priv->sh->rx_domain != NULL) {
+               ret = mlx5_glue->dr_sync_domain(priv->sh->rx_domain,
+                                               flags);
+               if (ret != 0)
+                       return ret;
+       }
+       if ((domains & MLX5_DOMAIN_BIT_NIC_TX) && priv->sh->tx_domain != NULL) {
+               ret = mlx5_glue->dr_sync_domain(priv->sh->tx_domain, flags);
+               if (ret != 0)
+                       return ret;
+       }
+       if ((domains & MLX5_DOMAIN_BIT_FDB) && priv->sh->fdb_domain != NULL) {
+               ret = mlx5_glue->dr_sync_domain(priv->sh->fdb_domain, flags);
+               if (ret != 0)
+                       return ret;
+       }
+       return 0;
+}
+
 const struct mlx5_flow_driver_ops mlx5_flow_dv_drv_ops = {
        .validate = flow_dv_validate,
        .prepare = flow_dv_prepare,
@@ -12330,6 +12376,7 @@ const struct mlx5_flow_driver_ops mlx5_flow_dv_drv_ops = {
        .action_create = flow_dv_action_create,
        .action_destroy = flow_dv_action_destroy,
        .action_update = flow_dv_action_update,
+       .sync_domain = flow_dv_sync_domain,
 };
 
 #endif /* HAVE_IBV_FLOW_DV_SUPPORT */