X-Git-Url: http://git.droids-corp.org/?a=blobdiff_plain;f=drivers%2Fnet%2Fmlx5%2Fmlx5_utils.c;h=9a54fda9b0a34a00df6be1862f4455234325a7bc;hb=e4b7b8d082db943355100fd7270113bd4be0fff6;hp=bf67192980f8a5c8b309a47ca34758c6c687b1c4;hpb=bd81eaebd97ce33df3e23290f190e8dba1814f28;p=dpdk.git

diff --git a/drivers/net/mlx5/mlx5_utils.c b/drivers/net/mlx5/mlx5_utils.c
index bf67192980..9a54fda9b0 100644
--- a/drivers/net/mlx5/mlx5_utils.c
+++ b/drivers/net/mlx5/mlx5_utils.c
@@ -5,6 +5,8 @@
 #include <rte_malloc.h>
 #include <rte_hash_crc.h>
 
+#include <mlx5_malloc.h>
+
 #include "mlx5_utils.h"
 
 struct mlx5_hlist *
@@ -27,7 +29,8 @@ mlx5_hlist_create(const char *name, uint32_t size)
 	alloc_size = sizeof(struct mlx5_hlist) +
 		     sizeof(struct mlx5_hlist_head) * act_size;
 	/* Using zmalloc, then no need to initialize the heads. */
-	h = rte_zmalloc(name, alloc_size, RTE_CACHE_LINE_SIZE);
+	h = mlx5_malloc(MLX5_MEM_ZERO, alloc_size, RTE_CACHE_LINE_SIZE,
+			SOCKET_ID_ANY);
 	if (!h) {
 		DRV_LOG(ERR, "No memory for hash list %s creation",
 			name ? name : "None");
@@ -78,6 +81,44 @@ mlx5_hlist_insert(struct mlx5_hlist *h, struct mlx5_hlist_entry *entry)
 	return 0;
 }
 
+struct mlx5_hlist_entry *
+mlx5_hlist_lookup_ex(struct mlx5_hlist *h, uint64_t key,
+		     mlx5_hlist_match_callback_fn cb, void *ctx)
+{
+	uint32_t idx;
+	struct mlx5_hlist_head *first;
+	struct mlx5_hlist_entry *node;
+
+	MLX5_ASSERT(h && cb && ctx);
+	idx = rte_hash_crc_8byte(key, 0) & h->mask;
+	first = &h->heads[idx];
+	LIST_FOREACH(node, first, next) {
+		if (!cb(node, ctx))
+			return node;
+	}
+	return NULL;
+}
+
+int
+mlx5_hlist_insert_ex(struct mlx5_hlist *h, struct mlx5_hlist_entry *entry,
+		     mlx5_hlist_match_callback_fn cb, void *ctx)
+{
+	uint32_t idx;
+	struct mlx5_hlist_head *first;
+	struct mlx5_hlist_entry *node;
+
+	MLX5_ASSERT(h && entry && cb && ctx);
+	idx = rte_hash_crc_8byte(entry->key, 0) & h->mask;
+	first = &h->heads[idx];
+	/* No need to reuse the lookup function. */
+	LIST_FOREACH(node, first, next) {
+		if (!cb(node, ctx))
+			return -EEXIST;
+	}
+	LIST_INSERT_HEAD(first, entry, next);
+	return 0;
+}
+
 void
 mlx5_hlist_remove(struct mlx5_hlist *h __rte_unused,
 		  struct mlx5_hlist_entry *entry)
@@ -112,10 +153,10 @@ mlx5_hlist_destroy(struct mlx5_hlist *h,
 			if (cb)
 				cb(entry, ctx);
 			else
-				rte_free(entry);
+				mlx5_free(entry);
 		}
 	}
-	rte_free(h);
+	mlx5_free(h);
 }
 
 static inline void
@@ -193,16 +234,17 @@ mlx5_ipool_create(struct mlx5_indexed_pool_config *cfg)
 	    (cfg->trunk_size && ((cfg->trunk_size & (cfg->trunk_size - 1)) ||
 	    ((__builtin_ffs(cfg->trunk_size) + TRUNK_IDX_BITS) > 32))))
 		return NULL;
-	pool = rte_zmalloc("mlx5_ipool", sizeof(*pool) + cfg->grow_trunk *
-				sizeof(pool->grow_tbl[0]), RTE_CACHE_LINE_SIZE);
+	pool = mlx5_malloc(MLX5_MEM_ZERO, sizeof(*pool) + cfg->grow_trunk *
+			   sizeof(pool->grow_tbl[0]), RTE_CACHE_LINE_SIZE,
+			   SOCKET_ID_ANY);
 	if (!pool)
 		return NULL;
 	pool->cfg = *cfg;
 	if (!pool->cfg.trunk_size)
 		pool->cfg.trunk_size = MLX5_IPOOL_DEFAULT_TRUNK_SIZE;
 	if (!cfg->malloc && !cfg->free) {
-		pool->cfg.malloc = rte_malloc_socket;
-		pool->cfg.free = rte_free;
+		pool->cfg.malloc = mlx5_malloc;
+		pool->cfg.free = mlx5_free;
 	}
 	pool->free_list = TRUNK_INVALID;
 	if (pool->cfg.need_lock)
@@ -237,10 +279,9 @@ mlx5_ipool_grow(struct mlx5_indexed_pool *pool)
 		int n_grow = pool->n_trunk_valid ? pool->n_trunk :
 			     RTE_CACHE_LINE_SIZE / sizeof(void *);
 
-		p = pool->cfg.malloc(pool->cfg.type,
-				 (pool->n_trunk_valid + n_grow) *
-				 sizeof(struct mlx5_indexed_trunk *),
-				 RTE_CACHE_LINE_SIZE, rte_socket_id());
+		p = pool->cfg.malloc(0, (pool->n_trunk_valid + n_grow) *
+				     sizeof(struct mlx5_indexed_trunk *),
+				     RTE_CACHE_LINE_SIZE, rte_socket_id());
 		if (!p)
 			return -ENOMEM;
 		if (pool->trunks)
@@ -268,7 +309,7 @@ mlx5_ipool_grow(struct mlx5_indexed_pool *pool)
 	/* rte_bitmap requires memory cacheline aligned. */
 	trunk_size += RTE_CACHE_LINE_ROUNDUP(data_size * pool->cfg.size);
 	trunk_size += bmp_size;
-	trunk = pool->cfg.malloc(pool->cfg.type, trunk_size,
+	trunk = pool->cfg.malloc(0, trunk_size,
 				 RTE_CACHE_LINE_SIZE, rte_socket_id());
 	if (!trunk)
 		return -ENOMEM;
@@ -464,7 +505,7 @@ mlx5_ipool_destroy(struct mlx5_indexed_pool *pool)
 	if (!pool->trunks)
 		pool->cfg.free(pool->trunks);
 	mlx5_ipool_unlock(pool);
-	rte_free(pool);
+	mlx5_free(pool);
 	return 0;
 }
 
@@ -493,15 +534,16 @@ mlx5_l3t_create(enum mlx5_l3t_type type)
 		.grow_shift = 1,
 		.need_lock = 0,
 		.release_mem_en = 1,
-		.malloc = rte_malloc_socket,
-		.free = rte_free,
+		.malloc = mlx5_malloc,
+		.free = mlx5_free,
 	};
 
 	if (type >= MLX5_L3T_TYPE_MAX) {
 		rte_errno = EINVAL;
 		return NULL;
 	}
-	tbl = rte_zmalloc(NULL, sizeof(struct mlx5_l3t_tbl), 1);
+	tbl = mlx5_malloc(MLX5_MEM_ZERO, sizeof(struct mlx5_l3t_tbl), 1,
+			  SOCKET_ID_ANY);
 	if (!tbl) {
 		rte_errno = ENOMEM;
 		return NULL;
@@ -509,30 +551,27 @@ mlx5_l3t_create(enum mlx5_l3t_type type)
 	tbl->type = type;
 	switch (type) {
 	case MLX5_L3T_TYPE_WORD:
-		l3t_ip_cfg.size = sizeof(struct mlx5_l3t_entry_word) +
-				  sizeof(uint16_t) * MLX5_L3T_ET_SIZE;
+		l3t_ip_cfg.size = sizeof(struct mlx5_l3t_entry_word);
 		l3t_ip_cfg.type = "mlx5_l3t_e_tbl_w";
 		break;
 	case MLX5_L3T_TYPE_DWORD:
-		l3t_ip_cfg.size = sizeof(struct mlx5_l3t_entry_dword) +
-				  sizeof(uint32_t) * MLX5_L3T_ET_SIZE;
+		l3t_ip_cfg.size = sizeof(struct mlx5_l3t_entry_dword);
 		l3t_ip_cfg.type = "mlx5_l3t_e_tbl_dw";
 		break;
 	case MLX5_L3T_TYPE_QWORD:
-		l3t_ip_cfg.size = sizeof(struct mlx5_l3t_entry_qword) +
-				  sizeof(uint64_t) * MLX5_L3T_ET_SIZE;
+		l3t_ip_cfg.size = sizeof(struct mlx5_l3t_entry_qword);
 		l3t_ip_cfg.type = "mlx5_l3t_e_tbl_qw";
 		break;
 	default:
-		l3t_ip_cfg.size = sizeof(struct mlx5_l3t_entry_ptr) +
-				  sizeof(void *) * MLX5_L3T_ET_SIZE;
+		l3t_ip_cfg.size = sizeof(struct mlx5_l3t_entry_ptr);
 		l3t_ip_cfg.type = "mlx5_l3t_e_tbl_tpr";
 		break;
 	}
+	rte_spinlock_init(&tbl->sl);
 	tbl->eip = mlx5_ipool_create(&l3t_ip_cfg);
 	if (!tbl->eip) {
 		rte_errno = ENOMEM;
-		rte_free(tbl);
+		mlx5_free(tbl);
 		tbl = NULL;
 	}
 	return tbl;
@@ -565,24 +604,28 @@ mlx5_l3t_destroy(struct mlx5_l3t_tbl *tbl)
 					break;
 			}
 			MLX5_ASSERT(!m_tbl->ref_cnt);
-			rte_free(g_tbl->tbl[i]);
+			mlx5_free(g_tbl->tbl[i]);
 			g_tbl->tbl[i] = 0;
 			if (!(--g_tbl->ref_cnt))
 				break;
 		}
 		MLX5_ASSERT(!g_tbl->ref_cnt);
-		rte_free(tbl->tbl);
+		mlx5_free(tbl->tbl);
 		tbl->tbl = 0;
 	}
 	mlx5_ipool_destroy(tbl->eip);
-	rte_free(tbl);
+	mlx5_free(tbl);
 }
 
-uint32_t
-mlx5_l3t_get_entry(struct mlx5_l3t_tbl *tbl, uint32_t idx,
-		   union mlx5_l3t_data *data)
+static int32_t
+__l3t_get_entry(struct mlx5_l3t_tbl *tbl, uint32_t idx,
+		union mlx5_l3t_data *data)
 {
 	struct mlx5_l3t_level_tbl *g_tbl, *m_tbl;
+	struct mlx5_l3t_entry_word *w_e_tbl;
+	struct mlx5_l3t_entry_dword *dw_e_tbl;
+	struct mlx5_l3t_entry_qword *qw_e_tbl;
+	struct mlx5_l3t_entry_ptr *ptr_e_tbl;
 	void *e_tbl;
 	uint32_t entry_idx;
 
@@ -598,26 +641,46 @@ mlx5_l3t_get_entry(struct mlx5_l3t_tbl *tbl, uint32_t idx,
 	entry_idx = idx & MLX5_L3T_ET_MASK;
 	switch (tbl->type) {
 	case MLX5_L3T_TYPE_WORD:
-		data->word = ((struct mlx5_l3t_entry_word *)e_tbl)->entry
-			     [entry_idx];
+		w_e_tbl = (struct mlx5_l3t_entry_word *)e_tbl;
+		data->word = w_e_tbl->entry[entry_idx].data;
+		if (w_e_tbl->entry[entry_idx].data)
+			w_e_tbl->entry[entry_idx].ref_cnt++;
 		break;
 	case MLX5_L3T_TYPE_DWORD:
-		data->dword = ((struct mlx5_l3t_entry_dword *)e_tbl)->entry
-			     [entry_idx];
+		dw_e_tbl = (struct mlx5_l3t_entry_dword *)e_tbl;
+		data->dword = dw_e_tbl->entry[entry_idx].data;
+		if (dw_e_tbl->entry[entry_idx].data)
+			dw_e_tbl->entry[entry_idx].ref_cnt++;
 		break;
 	case MLX5_L3T_TYPE_QWORD:
-		data->qword = ((struct mlx5_l3t_entry_qword *)e_tbl)->entry
-			      [entry_idx];
+		qw_e_tbl = (struct mlx5_l3t_entry_qword *)e_tbl;
+		data->qword = qw_e_tbl->entry[entry_idx].data;
+		if (qw_e_tbl->entry[entry_idx].data)
+			qw_e_tbl->entry[entry_idx].ref_cnt++;
 		break;
 	default:
-		data->ptr = ((struct mlx5_l3t_entry_ptr *)e_tbl)->entry
-			    [entry_idx];
+		ptr_e_tbl = (struct mlx5_l3t_entry_ptr *)e_tbl;
+		data->ptr = ptr_e_tbl->entry[entry_idx].data;
+		if (ptr_e_tbl->entry[entry_idx].data)
+			ptr_e_tbl->entry[entry_idx].ref_cnt++;
 		break;
 	}
 	return 0;
 }
 
-void
+int32_t
+mlx5_l3t_get_entry(struct mlx5_l3t_tbl *tbl, uint32_t idx,
+		   union mlx5_l3t_data *data)
+{
+	int ret;
+
+	rte_spinlock_lock(&tbl->sl);
+	ret = __l3t_get_entry(tbl, idx, data);
+	rte_spinlock_unlock(&tbl->sl);
+	return ret;
+}
+
+int32_t
 mlx5_l3t_clear_entry(struct mlx5_l3t_tbl *tbl, uint32_t idx)
 {
 	struct mlx5_l3t_level_tbl *g_tbl, *m_tbl;
@@ -628,36 +691,54 @@ mlx5_l3t_clear_entry(struct mlx5_l3t_tbl *tbl, uint32_t idx)
 	void *e_tbl;
 	uint32_t entry_idx;
 	uint64_t ref_cnt;
+	int32_t ret = -1;
 
+	rte_spinlock_lock(&tbl->sl);
 	g_tbl = tbl->tbl;
 	if (!g_tbl)
-		return;
+		goto out;
 	m_tbl = g_tbl->tbl[(idx >> MLX5_L3T_GT_OFFSET) & MLX5_L3T_GT_MASK];
 	if (!m_tbl)
-		return;
+		goto out;
 	e_tbl = m_tbl->tbl[(idx >> MLX5_L3T_MT_OFFSET) & MLX5_L3T_MT_MASK];
 	if (!e_tbl)
-		return;
+		goto out;
 	entry_idx = idx & MLX5_L3T_ET_MASK;
 	switch (tbl->type) {
 	case MLX5_L3T_TYPE_WORD:
 		w_e_tbl = (struct mlx5_l3t_entry_word *)e_tbl;
-		w_e_tbl->entry[entry_idx] = 0;
+		MLX5_ASSERT(w_e_tbl->entry[entry_idx].ref_cnt);
+		ret = --w_e_tbl->entry[entry_idx].ref_cnt;
+		if (ret)
+			goto out;
+		w_e_tbl->entry[entry_idx].data = 0;
 		ref_cnt = --w_e_tbl->ref_cnt;
 		break;
 	case MLX5_L3T_TYPE_DWORD:
 		dw_e_tbl = (struct mlx5_l3t_entry_dword *)e_tbl;
-		dw_e_tbl->entry[entry_idx] = 0;
+		MLX5_ASSERT(dw_e_tbl->entry[entry_idx].ref_cnt);
+		ret = --dw_e_tbl->entry[entry_idx].ref_cnt;
+		if (ret)
+			goto out;
+		dw_e_tbl->entry[entry_idx].data = 0;
 		ref_cnt = --dw_e_tbl->ref_cnt;
 		break;
 	case MLX5_L3T_TYPE_QWORD:
 		qw_e_tbl = (struct mlx5_l3t_entry_qword *)e_tbl;
-		qw_e_tbl->entry[entry_idx] = 0;
+		MLX5_ASSERT(qw_e_tbl->entry[entry_idx].ref_cnt);
+		ret = --qw_e_tbl->entry[entry_idx].ref_cnt;
+		if (ret)
+			goto out;
+		qw_e_tbl->entry[entry_idx].data = 0;
 		ref_cnt = --qw_e_tbl->ref_cnt;
 		break;
 	default:
 		ptr_e_tbl = (struct mlx5_l3t_entry_ptr *)e_tbl;
-		ptr_e_tbl->entry[entry_idx] = NULL;
+		MLX5_ASSERT(ptr_e_tbl->entry[entry_idx].ref_cnt);
+		ret = --ptr_e_tbl->entry[entry_idx].ref_cnt;
+		if (ret)
+			goto out;
+		ptr_e_tbl->entry[entry_idx].data = NULL;
 		ref_cnt = --ptr_e_tbl->ref_cnt;
 		break;
 	}
@@ -667,20 +748,23 @@ mlx5_l3t_clear_entry(struct mlx5_l3t_tbl *tbl, uint32_t idx)
 		m_tbl->tbl[(idx >> MLX5_L3T_MT_OFFSET) & MLX5_L3T_MT_MASK] =
 									NULL;
 		if (!(--m_tbl->ref_cnt)) {
-			rte_free(m_tbl);
+			mlx5_free(m_tbl);
 			g_tbl->tbl
 			[(idx >> MLX5_L3T_GT_OFFSET) & MLX5_L3T_GT_MASK] = NULL;
 			if (!(--g_tbl->ref_cnt)) {
-				rte_free(g_tbl);
+				mlx5_free(g_tbl);
 				tbl->tbl = 0;
 			}
 		}
 	}
+out:
+	rte_spinlock_unlock(&tbl->sl);
+	return ret;
 }
 
-uint32_t
-mlx5_l3t_set_entry(struct mlx5_l3t_tbl *tbl, uint32_t idx,
-		   union mlx5_l3t_data *data)
+static int32_t
+__l3t_set_entry(struct mlx5_l3t_tbl *tbl, uint32_t idx,
+		union mlx5_l3t_data *data)
 {
 	struct mlx5_l3t_level_tbl *g_tbl, *m_tbl;
 	struct mlx5_l3t_entry_word *w_e_tbl;
@@ -693,8 +777,10 @@ mlx5_l3t_set_entry(struct mlx5_l3t_tbl *tbl, uint32_t idx,
 	/* Check the global table, create it if empty. */
 	g_tbl = tbl->tbl;
 	if (!g_tbl) {
-		g_tbl = rte_zmalloc(NULL, sizeof(struct mlx5_l3t_level_tbl) +
-				    sizeof(void *) * MLX5_L3T_GT_SIZE, 1);
+		g_tbl = mlx5_malloc(MLX5_MEM_ZERO,
+				    sizeof(struct mlx5_l3t_level_tbl) +
+				    sizeof(void *) * MLX5_L3T_GT_SIZE, 1,
+				    SOCKET_ID_ANY);
 		if (!g_tbl) {
 			rte_errno = ENOMEM;
 			return -1;
@@ -707,8 +793,10 @@ mlx5_l3t_set_entry(struct mlx5_l3t_tbl *tbl, uint32_t idx,
 	 */
 	m_tbl = g_tbl->tbl[(idx >> MLX5_L3T_GT_OFFSET) & MLX5_L3T_GT_MASK];
 	if (!m_tbl) {
-		m_tbl = rte_zmalloc(NULL, sizeof(struct mlx5_l3t_level_tbl) +
-				    sizeof(void *) * MLX5_L3T_MT_SIZE, 1);
+		m_tbl = mlx5_malloc(MLX5_MEM_ZERO,
+				    sizeof(struct mlx5_l3t_level_tbl) +
+				    sizeof(void *) * MLX5_L3T_MT_SIZE, 1,
+				    SOCKET_ID_ANY);
 		if (!m_tbl) {
 			rte_errno = ENOMEM;
 			return -1;
@@ -737,24 +825,105 @@ mlx5_l3t_set_entry(struct mlx5_l3t_tbl *tbl, uint32_t idx,
 	switch (tbl->type) {
 	case MLX5_L3T_TYPE_WORD:
 		w_e_tbl = (struct mlx5_l3t_entry_word *)e_tbl;
-		w_e_tbl->entry[entry_idx] = data->word;
+		if (w_e_tbl->entry[entry_idx].data) {
+			data->word = w_e_tbl->entry[entry_idx].data;
+			w_e_tbl->entry[entry_idx].ref_cnt++;
+			rte_errno = EEXIST;
+			return -1;
+		}
+		w_e_tbl->entry[entry_idx].data = data->word;
+		w_e_tbl->entry[entry_idx].ref_cnt = 1;
 		w_e_tbl->ref_cnt++;
 		break;
 	case MLX5_L3T_TYPE_DWORD:
 		dw_e_tbl = (struct mlx5_l3t_entry_dword *)e_tbl;
-		dw_e_tbl->entry[entry_idx] = data->dword;
+		if (dw_e_tbl->entry[entry_idx].data) {
+			data->dword = dw_e_tbl->entry[entry_idx].data;
+			dw_e_tbl->entry[entry_idx].ref_cnt++;
+			rte_errno = EEXIST;
+			return -1;
+		}
+		dw_e_tbl->entry[entry_idx].data = data->dword;
+		dw_e_tbl->entry[entry_idx].ref_cnt = 1;
 		dw_e_tbl->ref_cnt++;
 		break;
 	case MLX5_L3T_TYPE_QWORD:
 		qw_e_tbl = (struct mlx5_l3t_entry_qword *)e_tbl;
-		qw_e_tbl->entry[entry_idx] = data->qword;
+		if (qw_e_tbl->entry[entry_idx].data) {
+			data->qword = qw_e_tbl->entry[entry_idx].data;
+			qw_e_tbl->entry[entry_idx].ref_cnt++;
+			rte_errno = EEXIST;
+			return -1;
+		}
+		qw_e_tbl->entry[entry_idx].data = data->qword;
+		qw_e_tbl->entry[entry_idx].ref_cnt = 1;
 		qw_e_tbl->ref_cnt++;
 		break;
 	default:
 		ptr_e_tbl = (struct mlx5_l3t_entry_ptr *)e_tbl;
-		ptr_e_tbl->entry[entry_idx] = data->ptr;
+		if (ptr_e_tbl->entry[entry_idx].data) {
+			data->ptr = ptr_e_tbl->entry[entry_idx].data;
+			ptr_e_tbl->entry[entry_idx].ref_cnt++;
+			rte_errno = EEXIST;
+			return -1;
+		}
+		ptr_e_tbl->entry[entry_idx].data = data->ptr;
+		ptr_e_tbl->entry[entry_idx].ref_cnt = 1;
 		ptr_e_tbl->ref_cnt++;
 		break;
 	}
 	return 0;
 }
+
+int32_t
+mlx5_l3t_set_entry(struct mlx5_l3t_tbl *tbl, uint32_t idx,
+		   union mlx5_l3t_data *data)
+{
+	int ret;
+
+	rte_spinlock_lock(&tbl->sl);
+	ret = __l3t_set_entry(tbl, idx, data);
+	rte_spinlock_unlock(&tbl->sl);
+	return ret;
+}
+
+int32_t
+mlx5_l3t_prepare_entry(struct mlx5_l3t_tbl *tbl, uint32_t idx,
+		       union mlx5_l3t_data *data,
+		       mlx5_l3t_alloc_callback_fn cb, void *ctx)
+{
+	int32_t ret;
+
+	rte_spinlock_lock(&tbl->sl);
+	/* Check if entry data is ready. */
+	ret = __l3t_get_entry(tbl, idx, data);
+	if (!ret) {
+		switch (tbl->type) {
+		case MLX5_L3T_TYPE_WORD:
+			if (data->word)
+				goto out;
+			break;
+		case MLX5_L3T_TYPE_DWORD:
+			if (data->dword)
+				goto out;
+			break;
+		case MLX5_L3T_TYPE_QWORD:
+			if (data->qword)
+				goto out;
+			break;
+		default:
+			if (data->ptr)
+				goto out;
+			break;
+		}
+	}
+	/* Entry data is not ready, use user callback to create it. */
+	ret = cb(ctx, data);
+	if (ret)
+		goto out;
+	/* Save the new allocated data to entry. */
+	ret = __l3t_set_entry(tbl, idx, data);
+out:
+	rte_spinlock_unlock(&tbl->sl);
+	return ret;
+}