1 /* SPDX-License-Identifier: BSD-3-Clause
2 * Copyright(C) 2021 Marvell.
5 #include <rte_mempool.h>
8 #include "cnxk_mempool.h"
10 #define BATCH_ALLOC_SZ ROC_CN10K_NPA_BATCH_ALLOC_MAX_PTRS
11 #define BATCH_OP_DATA_TABLE_MZ_NAME "batch_op_data_table_mz"
13 enum batch_op_status {
14 BATCH_ALLOC_OP_NOT_ISSUED = 0,
15 BATCH_ALLOC_OP_ISSUED = 1,
21 enum batch_op_status status;
22 uint64_t objs[BATCH_ALLOC_SZ] __rte_aligned(ROC_ALIGN);
25 struct batch_op_data {
27 struct batch_op_mem mem[RTE_MAX_LCORE] __rte_aligned(ROC_ALIGN);
30 static struct batch_op_data **batch_op_data_tbl;
33 batch_op_data_table_create(void)
35 const struct rte_memzone *mz;
37 /* If table is already set, nothing to do */
38 if (batch_op_data_tbl)
41 mz = rte_memzone_lookup(BATCH_OP_DATA_TABLE_MZ_NAME);
43 if (rte_eal_process_type() == RTE_PROC_PRIMARY) {
44 unsigned int maxpools, sz;
46 maxpools = roc_idev_npa_maxpools_get();
47 sz = maxpools * sizeof(struct batch_op_data *);
49 mz = rte_memzone_reserve_aligned(
50 BATCH_OP_DATA_TABLE_MZ_NAME, sz, SOCKET_ID_ANY,
54 plt_err("Failed to reserve batch op data table");
58 batch_op_data_tbl = mz->addr;
63 static inline struct batch_op_data *
64 batch_op_data_get(uint64_t pool_id)
66 uint64_t aura = roc_npa_aura_handle_to_aura(pool_id);
68 return batch_op_data_tbl[aura];
72 batch_op_data_set(uint64_t pool_id, struct batch_op_data *op_data)
74 uint64_t aura = roc_npa_aura_handle_to_aura(pool_id);
76 batch_op_data_tbl[aura] = op_data;
80 batch_op_init(struct rte_mempool *mp)
82 struct batch_op_data *op_data;
85 op_data = batch_op_data_get(mp->pool_id);
86 /* The data should not have been allocated previously */
87 RTE_ASSERT(op_data == NULL);
89 op_data = rte_zmalloc(NULL, sizeof(struct batch_op_data), ROC_ALIGN);
93 for (i = 0; i < RTE_MAX_LCORE; i++) {
94 op_data->mem[i].sz = 0;
95 op_data->mem[i].status = BATCH_ALLOC_OP_NOT_ISSUED;
98 op_data->lmt_addr = roc_idev_lmt_base_addr_get();
99 batch_op_data_set(mp->pool_id, op_data);
106 batch_op_fini(struct rte_mempool *mp)
108 struct batch_op_data *op_data;
111 op_data = batch_op_data_get(mp->pool_id);
114 for (i = 0; i < RTE_MAX_LCORE; i++) {
115 struct batch_op_mem *mem = &op_data->mem[i];
117 if (mem->status == BATCH_ALLOC_OP_ISSUED) {
118 mem->sz = roc_npa_aura_batch_alloc_extract(
119 mem->objs, mem->objs, BATCH_ALLOC_SZ);
120 mem->status = BATCH_ALLOC_OP_DONE;
122 if (mem->status == BATCH_ALLOC_OP_DONE) {
123 roc_npa_aura_op_bulk_free(mp->pool_id, mem->objs,
125 mem->status = BATCH_ALLOC_OP_NOT_ISSUED;
130 batch_op_data_set(mp->pool_id, NULL);
135 cn10k_mempool_enq(struct rte_mempool *mp, void *const *obj_table,
138 const uint64_t *ptr = (const uint64_t *)obj_table;
139 uint64_t lmt_addr = 0, lmt_id = 0;
140 struct batch_op_data *op_data;
142 /* Ensure mbuf init changes are written before the free pointers are
143 * enqueued to the stack.
148 roc_npa_aura_op_free(mp->pool_id, 1, ptr[0]);
152 op_data = batch_op_data_get(mp->pool_id);
153 lmt_addr = op_data->lmt_addr;
154 ROC_LMT_BASE_ID_GET(lmt_addr, lmt_id);
155 roc_npa_aura_op_batch_free(mp->pool_id, ptr, n, 1, lmt_addr, lmt_id);
161 cn10k_mempool_alloc(struct rte_mempool *mp)
167 block_size = mp->elt_size + mp->header_size + mp->trailer_size;
168 /* Align header size to ROC_ALIGN */
169 if (mp->header_size % ROC_ALIGN != 0) {
170 padding = RTE_ALIGN_CEIL(mp->header_size, ROC_ALIGN) -
172 mp->header_size += padding;
173 block_size += padding;
176 /* Align block size to ROC_ALIGN */
177 if (block_size % ROC_ALIGN != 0) {
178 padding = RTE_ALIGN_CEIL(block_size, ROC_ALIGN) - block_size;
179 mp->trailer_size += padding;
180 block_size += padding;
183 rc = cnxk_mempool_alloc(mp);
187 rc = batch_op_init(mp);
189 plt_err("Failed to init batch alloc mem rc=%d", rc);
195 cnxk_mempool_free(mp);
200 cn10k_mempool_free(struct rte_mempool *mp)
203 cnxk_mempool_free(mp);
207 cn10k_mempool_plt_init(void)
209 return batch_op_data_table_create();
212 static struct rte_mempool_ops cn10k_mempool_ops = {
213 .name = "cn10k_mempool_ops",
214 .alloc = cn10k_mempool_alloc,
215 .free = cn10k_mempool_free,
216 .enqueue = cn10k_mempool_enq,
217 .dequeue = cnxk_mempool_deq,
218 .get_count = cnxk_mempool_get_count,
219 .calc_mem_size = cnxk_mempool_calc_mem_size,
220 .populate = cnxk_mempool_populate,
223 MEMPOOL_REGISTER_OPS(cn10k_mempool_ops);