net/mlx5: convert encap/decap resource to indexed

This commit converts the flow encap/decap resource to indexed.

Using the uint32_t index instead of pointer saves 4 bytes memory for the
flow handle. For millions flows, it will save several MBytes of memory.

Signed-off-by: Suanming Mou <suanmingm@mellanox.com>
Acked-by: Viacheslav Ovsiienko <viacheslavo@mellanox.com>

diff --git a/drivers/net/mlx5/mlx5.c b/drivers/net/mlx5/mlx5.c
index f8b134c..3fb0bb5 100644 (file)
--- a/drivers/net/mlx5/mlx5.c
+++ b/drivers/net/mlx5/mlx5.c
@@ -198,6 +198,21 @@ struct mlx5_dev_spawn_data {
 static LIST_HEAD(, mlx5_ibv_shared) mlx5_ibv_list = LIST_HEAD_INITIALIZER();
 static pthread_mutex_t mlx5_ibv_list_mutex = PTHREAD_MUTEX_INITIALIZER;
 
+static struct mlx5_indexed_pool_config mlx5_ipool_cfg[] = {
+       {
+               .size = sizeof(struct mlx5_flow_dv_encap_decap_resource),
+               .trunk_size = 64,
+               .grow_trunk = 3,
+               .grow_shift = 2,
+               .need_lock = 0,
+               .release_mem_en = 1,
+               .malloc = rte_malloc_socket,
+               .free = rte_free,
+               .type = "mlx5_encap_decap_ipool",
+       },
+};
+
+
 #define MLX5_FLOW_MIN_ID_POOL_SIZE 512
 #define MLX5_ID_GENERATION_ARRAY_FACTOR 16
 
@@ -416,6 +431,36 @@ mlx5_flow_counters_mng_close(struct mlx5_ibv_shared *sh)
        memset(&sh->cmng, 0, sizeof(sh->cmng));
 }
 
+/**
+ * Initialize the flow resources' indexed mempool.
+ *
+ * @param[in] sh
+ *   Pointer to mlx5_ibv_shared object.
+ */
+static void
+mlx5_flow_ipool_create(struct mlx5_ibv_shared *sh)
+{
+       uint8_t i;
+
+       for (i = 0; i < MLX5_IPOOL_MAX; ++i)
+               sh->ipool[i] = mlx5_ipool_create(&mlx5_ipool_cfg[i]);
+}
+
+/**
+ * Release the flow resources' indexed mempool.
+ *
+ * @param[in] sh
+ *   Pointer to mlx5_ibv_shared object.
+ */
+static void
+mlx5_flow_ipool_destroy(struct mlx5_ibv_shared *sh)
+{
+       uint8_t i;
+
+       for (i = 0; i < MLX5_IPOOL_MAX; ++i)
+               mlx5_ipool_destroy(sh->ipool[i]);
+}
+
 /**
  * Extract pdn of PD object using DV API.
  *
@@ -631,6 +676,7 @@ mlx5_alloc_shared_ibctx(const struct mlx5_dev_spawn_data *spawn,
                goto error;
        }
        mlx5_flow_counters_mng_init(sh);
+       mlx5_flow_ipool_create(sh);
        /* Add device to memory callback list. */
        rte_rwlock_write_lock(&mlx5_shared_data->mem_event_rwlock);
        LIST_INSERT_HEAD(&mlx5_shared_data->mem_event_cb_list,
@@ -703,6 +749,7 @@ mlx5_free_shared_ibctx(struct mlx5_ibv_shared *sh)
         *  Only primary process handles async device events.
         **/
        mlx5_flow_counters_mng_close(sh);
+       mlx5_flow_ipool_destroy(sh);
        MLX5_ASSERT(!sh->intr_cnt);
        if (sh->intr_cnt)
                mlx5_intr_callback_unregister

diff --git a/drivers/net/mlx5/mlx5.h b/drivers/net/mlx5/mlx5.h
index 01112f7..4286969 100644 (file)
--- a/drivers/net/mlx5/mlx5.h
+++ b/drivers/net/mlx5/mlx5.h
@@ -43,6 +43,12 @@
 #include "mlx5_utils.h"
 #include "mlx5_autoconf.h"
 
+
+enum mlx5_ipool_index {
+       MLX5_IPOOL_DECAP_ENCAP = 0, /* Pool for encap/decap resource. */
+       MLX5_IPOOL_MAX,
+};
+
 /** Key string for IPC. */
 #define MLX5_MP_NAME "net_mlx5_mp"
 
@@ -423,7 +429,7 @@ struct mlx5_ibv_shared {
        /* Direct Rules tables for FDB, NIC TX+RX */
        void *esw_drop_action; /* Pointer to DR E-Switch drop action. */
        void *pop_vlan_action; /* Pointer to DR pop VLAN action. */
-       LIST_HEAD(encap_decap, mlx5_flow_dv_encap_decap_resource) encaps_decaps;
+       uint32_t encaps_decaps; /* Encap/decap action indexed memory list. */
        LIST_HEAD(modify_cmd, mlx5_flow_dv_modify_hdr_resource) modify_cmds;
        struct mlx5_hlist *tag_table;
        LIST_HEAD(port_id_action_list, mlx5_flow_dv_port_id_action_resource)
@@ -431,6 +437,8 @@ struct mlx5_ibv_shared {
        LIST_HEAD(push_vlan_action_list, mlx5_flow_dv_push_vlan_action_resource)
                push_vlan_action_list; /* List of push VLAN actions. */
        struct mlx5_flow_counter_mng cmng; /* Counters management structure. */
+       struct mlx5_indexed_pool *ipool[MLX5_IPOOL_MAX];
+       /* Memory Pool for mlx5 flow resources. */
        /* Shared interrupt handler section. */
        pthread_mutex_t intr_mutex; /* Interrupt config mutex. */
        uint32_t intr_cnt; /* Interrupt handler reference counter. */

diff --git a/drivers/net/mlx5/mlx5_flow.h b/drivers/net/mlx5/mlx5_flow.h
index df09d0b..6069403 100644 (file)
--- a/drivers/net/mlx5/mlx5_flow.h
+++ b/drivers/net/mlx5/mlx5_flow.h
@@ -364,7 +364,7 @@ struct mlx5_flow_dv_matcher {
 
 /* Encap/decap resource structure. */
 struct mlx5_flow_dv_encap_decap_resource {
-       LIST_ENTRY(mlx5_flow_dv_encap_decap_resource) next;
+       ILIST_ENTRY(uint32_t)next;
        /* Pointer to next element. */
        rte_atomic32_t refcnt; /**< Reference counter. */
        void *verbs_action;
@@ -482,8 +482,8 @@ struct mlx5_flow_rss {
 struct mlx5_flow_handle_dv {
        /* Flow DV api: */
        struct mlx5_flow_dv_matcher *matcher; /**< Cache to matcher. */
-       struct mlx5_flow_dv_encap_decap_resource *encap_decap;
-       /**< Pointer to encap/decap resource in cache. */
+       uint32_t encap_decap;
+       /**< Index to encap/decap resource in cache. */
        struct mlx5_flow_dv_modify_hdr_resource *modify_hdr;
        /**< Pointer to modify header resource in cache. */
        struct mlx5_flow_dv_jump_tbl_resource *jump;
@@ -543,6 +543,8 @@ struct mlx5_flow_dv_workspace {
        uint8_t transfer; /**< 1 if the flow is E-Switch flow. */
        int actions_n; /**< number of actions. */
        void *actions[MLX5_DV_MAX_NUMBER_OF_ACTIONS]; /**< Action list. */
+       struct mlx5_flow_dv_encap_decap_resource *encap_decap;
+       /**< Pointer to encap/decap resource in cache. */
        struct mlx5_flow_dv_match_params value;
        /**< Holds the value that the packet is compared to. */
 };

diff --git a/drivers/net/mlx5/mlx5_flow_dv.c b/drivers/net/mlx5/mlx5_flow_dv.c
index ae00ca4..16164d8 100644 (file)
--- a/drivers/net/mlx5/mlx5_flow_dv.c
+++ b/drivers/net/mlx5/mlx5_flow_dv.c
@@ -2467,6 +2467,7 @@ flow_dv_encap_decap_resource_register
        struct mlx5_ibv_shared *sh = priv->sh;
        struct mlx5_flow_dv_encap_decap_resource *cache_resource;
        struct mlx5dv_dr_domain *domain;
+       uint32_t idx = 0;
 
        resource->flags = dev_flow->dv.group ? 0 : 1;
        if (resource->ft_type == MLX5DV_FLOW_TABLE_TYPE_FDB)
@@ -2476,7 +2477,8 @@ flow_dv_encap_decap_resource_register
        else
                domain = sh->tx_domain;
        /* Lookup a matching resource from cache. */
-       LIST_FOREACH(cache_resource, &sh->encaps_decaps, next) {
+       ILIST_FOREACH(sh->ipool[MLX5_IPOOL_DECAP_ENCAP], sh->encaps_decaps, idx,
+                     cache_resource, next) {
                if (resource->reformat_type == cache_resource->reformat_type &&
                    resource->ft_type == cache_resource->ft_type &&
                    resource->flags == cache_resource->flags &&
@@ -2488,12 +2490,14 @@ flow_dv_encap_decap_resource_register
                                (void *)cache_resource,
                                rte_atomic32_read(&cache_resource->refcnt));
                        rte_atomic32_inc(&cache_resource->refcnt);
-                       dev_flow->handle->dvh.encap_decap = cache_resource;
+                       dev_flow->handle->dvh.encap_decap = idx;
+                       dev_flow->dv.encap_decap = cache_resource;
                        return 0;
                }
        }
        /* Register new encap/decap resource. */
-       cache_resource = rte_calloc(__func__, 1, sizeof(*cache_resource), 0);
+       cache_resource = mlx5_ipool_zmalloc(sh->ipool[MLX5_IPOOL_DECAP_ENCAP],
+                                      &dev_flow->handle->dvh.encap_decap);
        if (!cache_resource)
                return rte_flow_error_set(error, ENOMEM,
                                          RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
@@ -2513,8 +2517,9 @@ flow_dv_encap_decap_resource_register
        }
        rte_atomic32_init(&cache_resource->refcnt);
        rte_atomic32_inc(&cache_resource->refcnt);
-       LIST_INSERT_HEAD(&sh->encaps_decaps, cache_resource, next);
-       dev_flow->handle->dvh.encap_decap = cache_resource;
+       ILIST_INSERT(sh->ipool[MLX5_IPOOL_DECAP_ENCAP], &sh->encaps_decaps,
+                    dev_flow->handle->dvh.encap_decap, cache_resource, next);
+       dev_flow->dv.encap_decap = cache_resource;
        DRV_LOG(DEBUG, "new encap/decap resource %p: refcnt %d++",
                (void *)cache_resource,
                rte_atomic32_read(&cache_resource->refcnt));
@@ -3112,6 +3117,7 @@ flow_dv_create_action_raw_encap(struct rte_eth_dev *dev,
        const struct rte_flow_action_raw_encap *encap_data;
        struct mlx5_flow_dv_encap_decap_resource res;
 
+       memset(&res, 0, sizeof(res));
        encap_data = (const struct rte_flow_action_raw_encap *)action->conf;
        res.size = encap_data->size;
        memcpy(res.buf, encap_data->data, res.size);
@@ -7594,7 +7600,7 @@ cnt_err:
                                                           error))
                                return -rte_errno;
                        dev_flow->dv.actions[actions_n++] =
-                                       handle->dvh.encap_decap->verbs_action;
+                                       dev_flow->dv.encap_decap->verbs_action;
                        action_flags |= MLX5_FLOW_ACTION_ENCAP;
                        break;
                case RTE_FLOW_ACTION_TYPE_VXLAN_DECAP:
@@ -7604,7 +7610,7 @@ cnt_err:
                                                           error))
                                return -rte_errno;
                        dev_flow->dv.actions[actions_n++] =
-                                       handle->dvh.encap_decap->verbs_action;
+                                       dev_flow->dv.encap_decap->verbs_action;
                        action_flags |= MLX5_FLOW_ACTION_DECAP;
                        break;
                case RTE_FLOW_ACTION_TYPE_RAW_ENCAP:
@@ -7614,7 +7620,7 @@ cnt_err:
                                        (dev, actions, dev_flow, attr, error))
                                        return -rte_errno;
                                dev_flow->dv.actions[actions_n++] =
-                                       handle->dvh.encap_decap->verbs_action;
+                                       dev_flow->dv.encap_decap->verbs_action;
                        } else {
                                /* Handle encap without preceding decap. */
                                if (flow_dv_create_action_l2_encap
@@ -7622,7 +7628,7 @@ cnt_err:
                                     error))
                                        return -rte_errno;
                                dev_flow->dv.actions[actions_n++] =
-                                       handle->dvh.encap_decap->verbs_action;
+                                       dev_flow->dv.encap_decap->verbs_action;
                        }
                        action_flags |= MLX5_FLOW_ACTION_ENCAP;
                        break;
@@ -7634,7 +7640,7 @@ cnt_err:
                                    (dev, dev_flow, attr->transfer, error))
                                        return -rte_errno;
                                dev_flow->dv.actions[actions_n++] =
-                                       handle->dvh.encap_decap->verbs_action;
+                                       dev_flow->dv.encap_decap->verbs_action;
                        }
                        /* If decap is followed by encap, handle it at encap. */
                        action_flags |= MLX5_FLOW_ACTION_DECAP;
@@ -8188,6 +8194,8 @@ flow_dv_matcher_release(struct rte_eth_dev *dev,
 /**
  * Release an encap/decap resource.
  *
+ * @param dev
+ *   Pointer to Ethernet device.
  * @param handle
  *   Pointer to mlx5_flow_handle.
  *
@@ -8195,11 +8203,17 @@ flow_dv_matcher_release(struct rte_eth_dev *dev,
  *   1 while a reference on it exists, 0 when freed.
  */
 static int
-flow_dv_encap_decap_resource_release(struct mlx5_flow_handle *handle)
+flow_dv_encap_decap_resource_release(struct rte_eth_dev *dev,
+                                    struct mlx5_flow_handle *handle)
 {
-       struct mlx5_flow_dv_encap_decap_resource *cache_resource =
-                                               handle->dvh.encap_decap;
+       struct mlx5_priv *priv = dev->data->dev_private;
+       uint32_t idx = handle->dvh.encap_decap;
+       struct mlx5_flow_dv_encap_decap_resource *cache_resource;
 
+       cache_resource = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_DECAP_ENCAP],
+                        idx);
+       if (!cache_resource)
+               return 0;
        MLX5_ASSERT(cache_resource->verbs_action);
        DRV_LOG(DEBUG, "encap/decap resource %p: refcnt %d--",
                (void *)cache_resource,
@@ -8207,8 +8221,10 @@ flow_dv_encap_decap_resource_release(struct mlx5_flow_handle *handle)
        if (rte_atomic32_dec_and_test(&cache_resource->refcnt)) {
                claim_zero(mlx5_glue->destroy_flow_action
                                (cache_resource->verbs_action));
-               LIST_REMOVE(cache_resource, next);
-               rte_free(cache_resource);
+               ILIST_REMOVE(priv->sh->ipool[MLX5_IPOOL_DECAP_ENCAP],
+                            &priv->sh->encaps_decaps, idx,
+                            cache_resource, next);
+               mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_DECAP_ENCAP], idx);
                DRV_LOG(DEBUG, "encap/decap resource %p: removed",
                        (void *)cache_resource);
                return 0;
@@ -8410,7 +8426,7 @@ __flow_dv_destroy(struct rte_eth_dev *dev, struct rte_flow *flow)
                if (dev_handle->dvh.matcher)
                        flow_dv_matcher_release(dev, dev_handle);
                if (dev_handle->dvh.encap_decap)
-                       flow_dv_encap_decap_resource_release(dev_handle);
+                       flow_dv_encap_decap_resource_release(dev, dev_handle);
                if (dev_handle->dvh.modify_hdr)
                        flow_dv_modify_hdr_resource_release(dev_handle);
                if (dev_handle->dvh.jump)