net/mlx5: manage header reformat actions with hashed list

To manage encap decap header format actions mlx5 PMD used the single
linked list and lookup and insertion operations took too long times if
there were millions of objects and this impacted the flow
insertion/deletion rate.

In order to optimize the performance the hashed list is engaged. The
list implementation is updated to support non-unique keys with few
collisions.

Signed-off-by: Suanming Mou <suanmingm@nvidia.com>
Acked-by: Viacheslav Ovsiienko <viacheslavo@nvidia.com>

diff --git a/drivers/net/mlx5/linux/mlx5_os.c b/drivers/net/mlx5/linux/mlx5_os.c
index 26128b69d313253a9f41cfa18ee02f2377010a5d..81a2e99e71f5bfdc3a8e833fbbae01933c258ff2 100644 (file)
--- a/drivers/net/mlx5/linux/mlx5_os.c
+++ b/drivers/net/mlx5/linux/mlx5_os.c
@@ -250,6 +250,14 @@ mlx5_alloc_shared_dr(struct mlx5_priv *priv)
                err = ENOMEM;
                goto error;
        }
+       snprintf(s, sizeof(s), "%s_encaps_decaps", sh->ibdev_name);
+       sh->encaps_decaps = mlx5_hlist_create(s,
+                                             MLX5_FLOW_ENCAP_DECAP_HTABLE_SZ);
+       if (!sh->encaps_decaps) {
+               DRV_LOG(ERR, "encap decap hash creation failed");
+               err = ENOMEM;
+               goto error;
+       }
 #ifdef HAVE_MLX5DV_DR
        void *domain;
 
@@ -323,6 +331,10 @@ error:
                mlx5_glue->destroy_flow_action(sh->pop_vlan_action);
                sh->pop_vlan_action = NULL;
        }
+       if (sh->encaps_decaps) {
+               mlx5_hlist_destroy(sh->encaps_decaps, NULL, NULL);
+               sh->encaps_decaps = NULL;
+       }
        if (sh->modify_cmds) {
                mlx5_hlist_destroy(sh->modify_cmds, NULL, NULL);
                sh->modify_cmds = NULL;
@@ -380,6 +392,10 @@ mlx5_os_free_shared_dr(struct mlx5_priv *priv)
        }
        pthread_mutex_destroy(&sh->dv_mutex);
 #endif /* HAVE_MLX5DV_DR */
+       if (sh->encaps_decaps) {
+               mlx5_hlist_destroy(sh->encaps_decaps, NULL, NULL);
+               sh->encaps_decaps = NULL;
+       }
        if (sh->modify_cmds) {
                mlx5_hlist_destroy(sh->modify_cmds, NULL, NULL);
                sh->modify_cmds = NULL;

diff --git a/drivers/net/mlx5/mlx5.h b/drivers/net/mlx5/mlx5.h
index 1cdec47431837c585ec3705f4ca794a49e7bb3cc..0907506755a8d3936cb423f4c32f7bbcac6f1c25 100644 (file)
--- a/drivers/net/mlx5/mlx5.h
+++ b/drivers/net/mlx5/mlx5.h
@@ -635,7 +635,7 @@ struct mlx5_dev_ctx_shared {
        /* Direct Rules tables for FDB, NIC TX+RX */
        void *esw_drop_action; /* Pointer to DR E-Switch drop action. */
        void *pop_vlan_action; /* Pointer to DR pop VLAN action. */
-       uint32_t encaps_decaps; /* Encap/decap action indexed memory list. */
+       struct mlx5_hlist *encaps_decaps; /* Encap/decap action hash list. */
        struct mlx5_hlist *modify_cmds;
        struct mlx5_hlist *tag_table;
        uint32_t port_id_action_list; /* List of port ID actions. */

diff --git a/drivers/net/mlx5/mlx5_defs.h b/drivers/net/mlx5/mlx5_defs.h
index 90f1839054f468d9f8506fdcc19fb9680461890e..0df47391eeb9523915b8a2ad77c318df45da084e 100644 (file)
--- a/drivers/net/mlx5/mlx5_defs.h
+++ b/drivers/net/mlx5/mlx5_defs.h
@@ -190,6 +190,9 @@
 /* Size of the simple hash table for header modify table. */
 #define MLX5_FLOW_HDR_MODIFY_HTABLE_SZ (1 << 16)
 
+/* Size of the simple hash table for encap decap table. */
+#define MLX5_FLOW_ENCAP_DECAP_HTABLE_SZ (1 << 16)
+
 /* Hairpin TX/RX queue configuration parameters. */
 #define MLX5_HAIRPIN_QUEUE_STRIDE 6
 #define MLX5_HAIRPIN_JUMBO_LOG_SIZE (14 + 2)

diff --git a/drivers/net/mlx5/mlx5_flow.h b/drivers/net/mlx5/mlx5_flow.h
index 92301e494cc07d066dad4d8d21f116145640cabb..279daf21f5dc379623f3fcbd88645de3b81caa6e 100644 (file)
--- a/drivers/net/mlx5/mlx5_flow.h
+++ b/drivers/net/mlx5/mlx5_flow.h
@@ -388,9 +388,21 @@ struct mlx5_flow_dv_matcher {
 
 #define MLX5_ENCAP_MAX_LEN 132
 
+/* Encap/decap resource key of the hash organization. */
+union mlx5_flow_encap_decap_key {
+       struct {
+               uint32_t ft_type:8;     /**< Flow table type, Rx or Tx. */
+               uint32_t refmt_type:8;  /**< Header reformat type. */
+               uint32_t buf_size:8;    /**< Encap buf size. */
+               uint32_t table_level:8; /**< Root table or not. */
+               uint32_t cksum;         /**< Encap buf check sum. */
+       };
+       uint64_t v64;                   /**< full 64bits value of key */
+};
+
 /* Encap/decap resource structure. */
 struct mlx5_flow_dv_encap_decap_resource {
-       ILIST_ENTRY(uint32_t)next;
+       struct mlx5_hlist_entry entry;
        /* Pointer to next element. */
        rte_atomic32_t refcnt; /**< Reference counter. */
        void *action;
@@ -400,6 +412,7 @@ struct mlx5_flow_dv_encap_decap_resource {
        uint8_t reformat_type;
        uint8_t ft_type;
        uint64_t flags; /**< Flags for RDMA API. */
+       uint32_t idx; /**< Index for the index memory pool. */
 };
 
 /* Tag resource structure. */

diff --git a/drivers/net/mlx5/mlx5_flow_dv.c b/drivers/net/mlx5/mlx5_flow_dv.c
index ca1f39f7c0b2d82c3fcf95832d134314ff6703a1..79fdf34c0eee4ae3e34665a0b877c0de1fb13add 100644 (file)
--- a/drivers/net/mlx5/mlx5_flow_dv.c
+++ b/drivers/net/mlx5/mlx5_flow_dv.c
@@ -2545,6 +2545,39 @@ flow_dv_validate_action_raw_encap_decap
        return 0;
 }
 
+/**
+ * Match encap_decap resource.
+ *
+ * @param entry
+ *   Pointer to exist resource entry object.
+ * @param ctx
+ *   Pointer to new encap_decap resource.
+ *
+ * @return
+ *   0 on matching, -1 otherwise.
+ */
+static int
+flow_dv_encap_decap_resource_match(struct mlx5_hlist_entry *entry, void *ctx)
+{
+       struct mlx5_flow_dv_encap_decap_resource *resource;
+       struct mlx5_flow_dv_encap_decap_resource *cache_resource;
+
+       resource = (struct mlx5_flow_dv_encap_decap_resource *)ctx;
+       cache_resource = container_of(entry,
+                                     struct mlx5_flow_dv_encap_decap_resource,
+                                     entry);
+       if (resource->entry.key == cache_resource->entry.key &&
+           resource->reformat_type == cache_resource->reformat_type &&
+           resource->ft_type == cache_resource->ft_type &&
+           resource->flags == cache_resource->flags &&
+           resource->size == cache_resource->size &&
+           !memcmp((const void *)resource->buf,
+                   (const void *)cache_resource->buf,
+                   resource->size))
+               return 0;
+       return -1;
+}
+
 /**
  * Find existing encap/decap resource or create and register a new one.
  *
@@ -2571,7 +2604,16 @@ flow_dv_encap_decap_resource_register
        struct mlx5_dev_ctx_shared *sh = priv->sh;
        struct mlx5_flow_dv_encap_decap_resource *cache_resource;
        struct mlx5dv_dr_domain *domain;
-       uint32_t idx = 0;
+       struct mlx5_hlist_entry *entry;
+       union mlx5_flow_encap_decap_key encap_decap_key = {
+               {
+                       .ft_type = resource->ft_type,
+                       .refmt_type = resource->reformat_type,
+                       .buf_size = resource->size,
+                       .table_level = !!dev_flow->dv.group,
+                       .cksum = 0,
+               }
+       };
        int ret;
 
        resource->flags = dev_flow->dv.group ? 0 : 1;
@@ -2581,24 +2623,23 @@ flow_dv_encap_decap_resource_register
                domain = sh->rx_domain;
        else
                domain = sh->tx_domain;
+       encap_decap_key.cksum = __rte_raw_cksum(resource->buf,
+                                               resource->size, 0);
+       resource->entry.key = encap_decap_key.v64;
        /* Lookup a matching resource from cache. */
-       ILIST_FOREACH(sh->ipool[MLX5_IPOOL_DECAP_ENCAP], sh->encaps_decaps, idx,
-                     cache_resource, next) {
-               if (resource->reformat_type == cache_resource->reformat_type &&
-                   resource->ft_type == cache_resource->ft_type &&
-                   resource->flags == cache_resource->flags &&
-                   resource->size == cache_resource->size &&
-                   !memcmp((const void *)resource->buf,
-                           (const void *)cache_resource->buf,
-                           resource->size)) {
-                       DRV_LOG(DEBUG, "encap/decap resource %p: refcnt %d++",
-                               (void *)cache_resource,
-                               rte_atomic32_read(&cache_resource->refcnt));
-                       rte_atomic32_inc(&cache_resource->refcnt);
-                       dev_flow->handle->dvh.rix_encap_decap = idx;
-                       dev_flow->dv.encap_decap = cache_resource;
-                       return 0;
-               }
+       entry = mlx5_hlist_lookup_ex(sh->encaps_decaps, resource->entry.key,
+                                    flow_dv_encap_decap_resource_match,
+                                    (void *)resource);
+       if (entry) {
+               cache_resource = container_of(entry,
+                       struct mlx5_flow_dv_encap_decap_resource, entry);
+               DRV_LOG(DEBUG, "encap/decap resource %p: refcnt %d++",
+                       (void *)cache_resource,
+                       rte_atomic32_read(&cache_resource->refcnt));
+               rte_atomic32_inc(&cache_resource->refcnt);
+               dev_flow->handle->dvh.rix_encap_decap = cache_resource->idx;
+               dev_flow->dv.encap_decap = cache_resource;
+               return 0;
        }
        /* Register new encap/decap resource. */
        cache_resource = mlx5_ipool_zmalloc(sh->ipool[MLX5_IPOOL_DECAP_ENCAP],
@@ -2608,6 +2649,7 @@ flow_dv_encap_decap_resource_register
                                          RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
                                          "cannot allocate resource memory");
        *cache_resource = *resource;
+       cache_resource->idx = dev_flow->handle->dvh.rix_encap_decap;
        ret = mlx5_flow_os_create_flow_action_packet_reformat
                                        (sh->ctx, domain, cache_resource,
                                         &cache_resource->action);
@@ -2619,9 +2661,17 @@ flow_dv_encap_decap_resource_register
        }
        rte_atomic32_init(&cache_resource->refcnt);
        rte_atomic32_inc(&cache_resource->refcnt);
-       ILIST_INSERT(sh->ipool[MLX5_IPOOL_DECAP_ENCAP], &sh->encaps_decaps,
-                    dev_flow->handle->dvh.rix_encap_decap, cache_resource,
-                    next);
+       if (mlx5_hlist_insert_ex(sh->encaps_decaps, &cache_resource->entry,
+                                flow_dv_encap_decap_resource_match,
+                                (void *)cache_resource)) {
+               claim_zero(mlx5_flow_os_destroy_flow_action
+                                               (cache_resource->action));
+               mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_DECAP_ENCAP],
+                               cache_resource->idx);
+               return rte_flow_error_set(error, EEXIST,
+                                         RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
+                                         NULL, "action exist");
+       }
        dev_flow->dv.encap_decap = cache_resource;
        DRV_LOG(DEBUG, "new encap/decap resource %p: refcnt %d++",
                (void *)cache_resource,
@@ -9092,9 +9142,8 @@ flow_dv_encap_decap_resource_release(struct rte_eth_dev *dev,
        if (rte_atomic32_dec_and_test(&cache_resource->refcnt)) {
                claim_zero(mlx5_flow_os_destroy_flow_action
                                                (cache_resource->action));
-               ILIST_REMOVE(priv->sh->ipool[MLX5_IPOOL_DECAP_ENCAP],
-                            &priv->sh->encaps_decaps, idx,
-                            cache_resource, next);
+               mlx5_hlist_remove(priv->sh->encaps_decaps,
+                                 &cache_resource->entry);
                mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_DECAP_ENCAP], idx);
                DRV_LOG(DEBUG, "encap/decap resource %p: removed",
                        (void *)cache_resource);