net/mlx5: manage modify actions with hashed list

To manage header modify actions mlx5 PMD used the single linked list and
lookup and insertion operations took too long times if there were
millions of objects and this impacted the flow insertion/deletion rate.

In order to optimize the performance the hashed list is engaged. The
list implementation is updated to support non-unique keys with few
collisions.

Signed-off-by: Suanming Mou <suanmingm@mellanox.com>
Acked-by: Viacheslav Ovsiienko <viacheslavo@mellanox.com>

diff --git a/drivers/net/mlx5/linux/mlx5_os.c b/drivers/net/mlx5/linux/mlx5_os.c
index 69123e12c368cc4fc675eec81350afea3609f8ea..db955ae6bf6369f286907cf1bab3f0fd63617126 100644 (file)
--- a/drivers/net/mlx5/linux/mlx5_os.c
+++ b/drivers/net/mlx5/linux/mlx5_os.c
@@ -241,6 +241,13 @@ mlx5_alloc_shared_dr(struct mlx5_priv *priv)
                err = ENOMEM;
                goto error;
        }
+       snprintf(s, sizeof(s), "%s_hdr_modify", sh->ibdev_name);
+       sh->modify_cmds = mlx5_hlist_create(s, MLX5_FLOW_HDR_MODIFY_HTABLE_SZ);
+       if (!sh->modify_cmds) {
+               DRV_LOG(ERR, "hdr modify hash creation failed");
+               err = ENOMEM;
+               goto error;
+       }
 #ifdef HAVE_MLX5DV_DR
        void *domain;
 
@@ -314,6 +321,10 @@ error:
                mlx5_glue->destroy_flow_action(sh->pop_vlan_action);
                sh->pop_vlan_action = NULL;
        }
+       if (sh->modify_cmds) {
+               mlx5_hlist_destroy(sh->modify_cmds, NULL, NULL);
+               sh->modify_cmds = NULL;
+       }
        if (sh->tag_table) {
                /* tags should be destroyed with flow before. */
                mlx5_hlist_destroy(sh->tag_table, NULL, NULL);
@@ -367,6 +378,10 @@ mlx5_os_free_shared_dr(struct mlx5_priv *priv)
        }
        pthread_mutex_destroy(&sh->dv_mutex);
 #endif /* HAVE_MLX5DV_DR */
+       if (sh->modify_cmds) {
+               mlx5_hlist_destroy(sh->modify_cmds, NULL, NULL);
+               sh->modify_cmds = NULL;
+       }
        if (sh->tag_table) {
                /* tags should be destroyed with flow before. */
                mlx5_hlist_destroy(sh->tag_table, NULL, NULL);

diff --git a/drivers/net/mlx5/mlx5.h b/drivers/net/mlx5/mlx5.h
index 78d6eb72816f5fa9b093c2032b185732fbffa601..1880a82ef5de4afb49b5810a6100dfef315c37cf 100644 (file)
--- a/drivers/net/mlx5/mlx5.h
+++ b/drivers/net/mlx5/mlx5.h
@@ -638,7 +638,7 @@ struct mlx5_dev_ctx_shared {
        void *esw_drop_action; /* Pointer to DR E-Switch drop action. */
        void *pop_vlan_action; /* Pointer to DR pop VLAN action. */
        uint32_t encaps_decaps; /* Encap/decap action indexed memory list. */
-       LIST_HEAD(modify_cmd, mlx5_flow_dv_modify_hdr_resource) modify_cmds;
+       struct mlx5_hlist *modify_cmds;
        struct mlx5_hlist *tag_table;
        uint32_t port_id_action_list; /* List of port ID actions. */
        uint32_t push_vlan_action_list; /* List of push VLAN actions. */

diff --git a/drivers/net/mlx5/mlx5_defs.h b/drivers/net/mlx5/mlx5_defs.h
index c26d5a292754fc3b79402df39c9a1d7bf14d893c..90f1839054f468d9f8506fdcc19fb9680461890e 100644 (file)
--- a/drivers/net/mlx5/mlx5_defs.h
+++ b/drivers/net/mlx5/mlx5_defs.h
@@ -187,6 +187,9 @@
 #define MLX5_FLOW_MREG_HNAME "MARK_COPY_TABLE"
 #define MLX5_DEFAULT_COPY_ID UINT32_MAX
 
+/* Size of the simple hash table for header modify table. */
+#define MLX5_FLOW_HDR_MODIFY_HTABLE_SZ (1 << 16)
+
 /* Hairpin TX/RX queue configuration parameters. */
 #define MLX5_HAIRPIN_QUEUE_STRIDE 6
 #define MLX5_HAIRPIN_JUMBO_LOG_SIZE (14 + 2)

diff --git a/drivers/net/mlx5/mlx5_flow.h b/drivers/net/mlx5/mlx5_flow.h
index 66caefce4683dd039a5d8f02ca08cc1c29e3389b..92301e494cc07d066dad4d8d21f116145640cabb 100644 (file)
--- a/drivers/net/mlx5/mlx5_flow.h
+++ b/drivers/net/mlx5/mlx5_flow.h
@@ -427,7 +427,7 @@ struct mlx5_flow_dv_tag_resource {
 
 /* Modify resource structure */
 struct mlx5_flow_dv_modify_hdr_resource {
-       LIST_ENTRY(mlx5_flow_dv_modify_hdr_resource) next;
+       struct mlx5_hlist_entry entry;
        /* Pointer to next element. */
        rte_atomic32_t refcnt; /**< Reference counter. */
        void *action;
@@ -439,6 +439,17 @@ struct mlx5_flow_dv_modify_hdr_resource {
        /**< Modification actions. */
 };
 
+/* Modify resource key of the hash organization. */
+union mlx5_flow_modify_hdr_key {
+       struct {
+               uint32_t ft_type:8;     /**< Flow table type, Rx or Tx. */
+               uint32_t actions_num:5; /**< Number of modification actions. */
+               uint32_t group:19;      /**< Flow group id. */
+               uint32_t cksum;         /**< Actions check sum. */
+       };
+       uint64_t v64;                   /**< full 64bits value of key */
+};
+
 /* Jump action resource structure. */
 struct mlx5_flow_dv_jump_tbl_resource {
        rte_atomic32_t refcnt; /**< Reference counter. */

diff --git a/drivers/net/mlx5/mlx5_flow_dv.c b/drivers/net/mlx5/mlx5_flow_dv.c
index 53399800ff6f7da55dba62362789fea3bfd3d498..dd359597e4a3fec3d634bab9e91be8c65f2b76d4 100644 (file)
--- a/drivers/net/mlx5/mlx5_flow_dv.c
+++ b/drivers/net/mlx5/mlx5_flow_dv.c
@@ -3957,6 +3957,40 @@ flow_dv_validate_action_modify_ipv6_dscp(const uint64_t action_flags,
        return ret;
 }
 
+/**
+ * Match modify-header resource.
+ *
+ * @param entry
+ *   Pointer to exist resource entry object.
+ * @param ctx
+ *   Pointer to new modify-header resource.
+ *
+ * @return
+ *   0 on matching, -1 otherwise.
+ */
+static int
+flow_dv_modify_hdr_resource_match(struct mlx5_hlist_entry *entry, void *ctx)
+{
+       struct mlx5_flow_dv_modify_hdr_resource *resource;
+       struct mlx5_flow_dv_modify_hdr_resource *cache_resource;
+       uint32_t actions_len;
+
+       resource = (struct mlx5_flow_dv_modify_hdr_resource *)ctx;
+       cache_resource = container_of(entry,
+                                     struct mlx5_flow_dv_modify_hdr_resource,
+                                     entry);
+       actions_len = resource->actions_num * sizeof(resource->actions[0]);
+       if (resource->entry.key == cache_resource->entry.key &&
+           resource->ft_type == cache_resource->ft_type &&
+           resource->actions_num == cache_resource->actions_num &&
+           resource->flags == cache_resource->flags &&
+           !memcmp((const void *)resource->actions,
+                   (const void *)cache_resource->actions,
+                   actions_len))
+               return 0;
+       return -1;
+}
+
 /**
  * Find existing modify-header resource or create and register a new one.
  *
@@ -3984,6 +4018,15 @@ flow_dv_modify_hdr_resource_register
        struct mlx5_flow_dv_modify_hdr_resource *cache_resource;
        struct mlx5dv_dr_domain *ns;
        uint32_t actions_len;
+       struct mlx5_hlist_entry *entry;
+       union mlx5_flow_modify_hdr_key hdr_mod_key = {
+               {
+                       .ft_type = resource->ft_type,
+                       .actions_num = resource->actions_num,
+                       .group = dev_flow->dv.group,
+                       .cksum = 0,
+               }
+       };
        int ret;
 
        resource->flags = dev_flow->dv.group ? 0 :
@@ -4001,20 +4044,22 @@ flow_dv_modify_hdr_resource_register
                ns = sh->rx_domain;
        /* Lookup a matching resource from cache. */
        actions_len = resource->actions_num * sizeof(resource->actions[0]);
-       LIST_FOREACH(cache_resource, &sh->modify_cmds, next) {
-               if (resource->ft_type == cache_resource->ft_type &&
-                   resource->actions_num == cache_resource->actions_num &&
-                   resource->flags == cache_resource->flags &&
-                   !memcmp((const void *)resource->actions,
-                           (const void *)cache_resource->actions,
-                           actions_len)) {
-                       DRV_LOG(DEBUG, "modify-header resource %p: refcnt %d++",
-                               (void *)cache_resource,
-                               rte_atomic32_read(&cache_resource->refcnt));
-                       rte_atomic32_inc(&cache_resource->refcnt);
-                       dev_flow->handle->dvh.modify_hdr = cache_resource;
-                       return 0;
-               }
+       hdr_mod_key.cksum = __rte_raw_cksum(resource->actions, actions_len, 0);
+       resource->entry.key = hdr_mod_key.v64;
+       entry = mlx5_hlist_lookup_ex(sh->modify_cmds, resource->entry.key,
+                                    flow_dv_modify_hdr_resource_match,
+                                    (void *)resource);
+       if (entry) {
+               cache_resource = container_of(entry,
+                                       struct mlx5_flow_dv_modify_hdr_resource,
+                                       entry);
+               DRV_LOG(DEBUG, "modify-header resource %p: refcnt %d++",
+                       (void *)cache_resource,
+                       rte_atomic32_read(&cache_resource->refcnt));
+               rte_atomic32_inc(&cache_resource->refcnt);
+               dev_flow->handle->dvh.modify_hdr = cache_resource;
+               return 0;
+
        }
        /* Register new modify-header resource. */
        cache_resource = mlx5_malloc(MLX5_MEM_ZERO,
@@ -4037,7 +4082,16 @@ flow_dv_modify_hdr_resource_register
        }
        rte_atomic32_init(&cache_resource->refcnt);
        rte_atomic32_inc(&cache_resource->refcnt);
-       LIST_INSERT_HEAD(&sh->modify_cmds, cache_resource, next);
+       if (mlx5_hlist_insert_ex(sh->modify_cmds, &cache_resource->entry,
+                                flow_dv_modify_hdr_resource_match,
+                                (void *)cache_resource)) {
+               claim_zero(mlx5_flow_os_destroy_flow_action
+                                               (cache_resource->action));
+               mlx5_free(cache_resource);
+               return rte_flow_error_set(error, EEXIST,
+                                         RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
+                                         NULL, "action exist");
+       }
        dev_flow->handle->dvh.modify_hdr = cache_resource;
        DRV_LOG(DEBUG, "new modify-header resource %p: refcnt %d++",
                (void *)cache_resource,
@@ -9122,6 +9176,8 @@ flow_dv_default_miss_resource_release(struct rte_eth_dev *dev)
 /**
  * Release a modify-header resource.
  *
+ * @param dev
+ *   Pointer to Ethernet device.
  * @param handle
  *   Pointer to mlx5_flow_handle.
  *
@@ -9129,8 +9185,10 @@ flow_dv_default_miss_resource_release(struct rte_eth_dev *dev)
  *   1 while a reference on it exists, 0 when freed.
  */
 static int
-flow_dv_modify_hdr_resource_release(struct mlx5_flow_handle *handle)
+flow_dv_modify_hdr_resource_release(struct rte_eth_dev *dev,
+                                   struct mlx5_flow_handle *handle)
 {
+       struct mlx5_priv *priv = dev->data->dev_private;
        struct mlx5_flow_dv_modify_hdr_resource *cache_resource =
                                                        handle->dvh.modify_hdr;
 
@@ -9141,7 +9199,8 @@ flow_dv_modify_hdr_resource_release(struct mlx5_flow_handle *handle)
        if (rte_atomic32_dec_and_test(&cache_resource->refcnt)) {
                claim_zero(mlx5_flow_os_destroy_flow_action
                                                (cache_resource->action));
-               LIST_REMOVE(cache_resource, next);
+               mlx5_hlist_remove(priv->sh->modify_cmds,
+                                 &cache_resource->entry);
                mlx5_free(cache_resource);
                DRV_LOG(DEBUG, "modify-header resource %p: removed",
                        (void *)cache_resource);
@@ -9351,7 +9410,7 @@ __flow_dv_destroy(struct rte_eth_dev *dev, struct rte_flow *flow)
                if (dev_handle->dvh.rix_encap_decap)
                        flow_dv_encap_decap_resource_release(dev, dev_handle);
                if (dev_handle->dvh.modify_hdr)
-                       flow_dv_modify_hdr_resource_release(dev_handle);
+                       flow_dv_modify_hdr_resource_release(dev, dev_handle);
                if (dev_handle->dvh.rix_push_vlan)
                        flow_dv_push_vlan_action_resource_release(dev,
                                                                  dev_handle);