From db2a347a501d6bd437308e2c8aec9cfe71264882 Mon Sep 17 00:00:00 2001 From: Ashwin Sekhar T K Date: Thu, 8 Apr 2021 15:20:46 +0530 Subject: [PATCH] mempool/cnxk: add batch operation init Marvell CN10k mempool supports batch enqueue/dequeue which can dequeue up to 512 pointers and enqueue up to 15 pointers using a single instruction. These batch operations require a DMA memory to enqueue/dequeue pointers. This patch adds the initialization of this DMA memory. Signed-off-by: Ashwin Sekhar T K --- doc/guides/mempool/cnxk.rst | 5 + drivers/mempool/cnxk/cn10k_mempool_ops.c | 147 ++++++++++++++++++++++- drivers/mempool/cnxk/cnxk_mempool.h | 2 + drivers/mempool/cnxk/cnxk_mempool_ops.c | 11 +- 4 files changed, 160 insertions(+), 5 deletions(-) diff --git a/doc/guides/mempool/cnxk.rst b/doc/guides/mempool/cnxk.rst index 76a024426f..d883b83f7b 100644 --- a/doc/guides/mempool/cnxk.rst +++ b/doc/guides/mempool/cnxk.rst @@ -25,6 +25,11 @@ CN9k NPA supports: - Burst alloc of up to 32 pointers. +CN10k NPA supports: + +- Batch dequeue of up to 512 pointers with single instruction. +- Batch enqueue of up to 15 pointers with single instruction. + Prerequisites and Compilation procedure --------------------------------------- diff --git a/drivers/mempool/cnxk/cn10k_mempool_ops.c b/drivers/mempool/cnxk/cn10k_mempool_ops.c index 9b63789006..a3aef0ddb2 100644 --- a/drivers/mempool/cnxk/cn10k_mempool_ops.c +++ b/drivers/mempool/cnxk/cn10k_mempool_ops.c @@ -7,11 +7,136 @@ #include "roc_api.h" #include "cnxk_mempool.h" +#define BATCH_ALLOC_SZ ROC_CN10K_NPA_BATCH_ALLOC_MAX_PTRS +#define BATCH_OP_DATA_TABLE_MZ_NAME "batch_op_data_table_mz" + +enum batch_op_status { + BATCH_ALLOC_OP_NOT_ISSUED = 0, + BATCH_ALLOC_OP_ISSUED = 1, + BATCH_ALLOC_OP_DONE +}; + +struct batch_op_mem { + unsigned int sz; + enum batch_op_status status; + uint64_t objs[BATCH_ALLOC_SZ] __rte_aligned(ROC_ALIGN); +}; + +struct batch_op_data { + uint64_t lmt_addr; + struct batch_op_mem mem[RTE_MAX_LCORE] __rte_aligned(ROC_ALIGN); +}; + +static struct batch_op_data **batch_op_data_tbl; + +static int +batch_op_data_table_create(void) +{ + const struct rte_memzone *mz; + + /* If table is already set, nothing to do */ + if (batch_op_data_tbl) + return 0; + + mz = rte_memzone_lookup(BATCH_OP_DATA_TABLE_MZ_NAME); + if (mz == NULL) { + if (rte_eal_process_type() == RTE_PROC_PRIMARY) { + unsigned int maxpools, sz; + + maxpools = roc_idev_npa_maxpools_get(); + sz = maxpools * sizeof(struct batch_op_data *); + + mz = rte_memzone_reserve_aligned( + BATCH_OP_DATA_TABLE_MZ_NAME, sz, SOCKET_ID_ANY, + 0, ROC_ALIGN); + } + if (mz == NULL) { + plt_err("Failed to reserve batch op data table"); + return -ENOMEM; + } + } + batch_op_data_tbl = mz->addr; + rte_wmb(); + return 0; +} + +static inline struct batch_op_data * +batch_op_data_get(uint64_t pool_id) +{ + uint64_t aura = roc_npa_aura_handle_to_aura(pool_id); + + return batch_op_data_tbl[aura]; +} + +static inline void +batch_op_data_set(uint64_t pool_id, struct batch_op_data *op_data) +{ + uint64_t aura = roc_npa_aura_handle_to_aura(pool_id); + + batch_op_data_tbl[aura] = op_data; +} + +static int +batch_op_init(struct rte_mempool *mp) +{ + struct batch_op_data *op_data; + int i; + + op_data = batch_op_data_get(mp->pool_id); + /* The data should not have been allocated previously */ + RTE_ASSERT(op_data == NULL); + + op_data = rte_zmalloc(NULL, sizeof(struct batch_op_data), ROC_ALIGN); + if (op_data == NULL) + return -ENOMEM; + + for (i = 0; i < RTE_MAX_LCORE; i++) { + op_data->mem[i].sz = 0; + op_data->mem[i].status = BATCH_ALLOC_OP_NOT_ISSUED; + } + + op_data->lmt_addr = roc_idev_lmt_base_addr_get(); + batch_op_data_set(mp->pool_id, op_data); + rte_wmb(); + + return 0; +} + +static void +batch_op_fini(struct rte_mempool *mp) +{ + struct batch_op_data *op_data; + int i; + + op_data = batch_op_data_get(mp->pool_id); + + rte_wmb(); + for (i = 0; i < RTE_MAX_LCORE; i++) { + struct batch_op_mem *mem = &op_data->mem[i]; + + if (mem->status == BATCH_ALLOC_OP_ISSUED) { + mem->sz = roc_npa_aura_batch_alloc_extract( + mem->objs, mem->objs, BATCH_ALLOC_SZ); + mem->status = BATCH_ALLOC_OP_DONE; + } + if (mem->status == BATCH_ALLOC_OP_DONE) { + roc_npa_aura_op_bulk_free(mp->pool_id, mem->objs, + mem->sz, 1); + mem->status = BATCH_ALLOC_OP_NOT_ISSUED; + } + } + + rte_free(op_data); + batch_op_data_set(mp->pool_id, NULL); + rte_wmb(); +} + static int cn10k_mempool_alloc(struct rte_mempool *mp) { uint32_t block_size; size_t padding; + int rc; block_size = mp->elt_size + mp->header_size + mp->trailer_size; /* Align header size to ROC_ALIGN */ @@ -29,15 +154,35 @@ cn10k_mempool_alloc(struct rte_mempool *mp) block_size += padding; } - return cnxk_mempool_alloc(mp); + rc = cnxk_mempool_alloc(mp); + if (rc) + return rc; + + rc = batch_op_init(mp); + if (rc) { + plt_err("Failed to init batch alloc mem rc=%d", rc); + goto error; + } + + return 0; +error: + cnxk_mempool_free(mp); + return rc; } static void cn10k_mempool_free(struct rte_mempool *mp) { + batch_op_fini(mp); cnxk_mempool_free(mp); } +int +cn10k_mempool_plt_init(void) +{ + return batch_op_data_table_create(); +} + static struct rte_mempool_ops cn10k_mempool_ops = { .name = "cn10k_mempool_ops", .alloc = cn10k_mempool_alloc, diff --git a/drivers/mempool/cnxk/cnxk_mempool.h b/drivers/mempool/cnxk/cnxk_mempool.h index 099b7f6998..3405aa7663 100644 --- a/drivers/mempool/cnxk/cnxk_mempool.h +++ b/drivers/mempool/cnxk/cnxk_mempool.h @@ -23,4 +23,6 @@ int __rte_hot cnxk_mempool_enq(struct rte_mempool *mp, void *const *obj_table, int __rte_hot cnxk_mempool_deq(struct rte_mempool *mp, void **obj_table, unsigned int n); +int cn10k_mempool_plt_init(void); + #endif diff --git a/drivers/mempool/cnxk/cnxk_mempool_ops.c b/drivers/mempool/cnxk/cnxk_mempool_ops.c index 42c02bf14e..c7b75f026d 100644 --- a/drivers/mempool/cnxk/cnxk_mempool_ops.c +++ b/drivers/mempool/cnxk/cnxk_mempool_ops.c @@ -174,12 +174,15 @@ cnxk_mempool_populate(struct rte_mempool *mp, unsigned int max_objs, static int cnxk_mempool_plt_init(void) { - if (roc_model_is_cn9k()) + int rc = 0; + + if (roc_model_is_cn9k()) { rte_mbuf_set_platform_mempool_ops("cn9k_mempool_ops"); - else if (roc_model_is_cn10k()) + } else if (roc_model_is_cn10k()) { rte_mbuf_set_platform_mempool_ops("cn10k_mempool_ops"); - - return 0; + rc = cn10k_mempool_plt_init(); + } + return rc; } RTE_INIT(cnxk_mempool_ops_init) -- 2.20.1