1 /* SPDX-License-Identifier: BSD-3-Clause
2 * Copyright(C) 2021 Marvell.
5 #include <rte_mempool.h>
8 #include "cnxk_mempool.h"
10 #define BATCH_ALLOC_SZ ROC_CN10K_NPA_BATCH_ALLOC_MAX_PTRS
11 #define BATCH_OP_DATA_TABLE_MZ_NAME "batch_op_data_table_mz"
13 enum batch_op_status {
14 BATCH_ALLOC_OP_NOT_ISSUED = 0,
15 BATCH_ALLOC_OP_ISSUED = 1,
21 enum batch_op_status status;
22 uint64_t objs[BATCH_ALLOC_SZ] __rte_aligned(ROC_ALIGN);
25 struct batch_op_data {
27 struct batch_op_mem mem[RTE_MAX_LCORE] __rte_aligned(ROC_ALIGN);
30 static struct batch_op_data **batch_op_data_tbl;
33 batch_op_data_table_create(void)
35 const struct rte_memzone *mz;
37 /* If table is already set, nothing to do */
38 if (batch_op_data_tbl)
41 mz = rte_memzone_lookup(BATCH_OP_DATA_TABLE_MZ_NAME);
43 if (rte_eal_process_type() == RTE_PROC_PRIMARY) {
44 unsigned int maxpools, sz;
46 maxpools = roc_idev_npa_maxpools_get();
47 sz = maxpools * sizeof(struct batch_op_data *);
49 mz = rte_memzone_reserve_aligned(
50 BATCH_OP_DATA_TABLE_MZ_NAME, sz, SOCKET_ID_ANY,
54 plt_err("Failed to reserve batch op data table");
58 batch_op_data_tbl = mz->addr;
63 static inline struct batch_op_data *
64 batch_op_data_get(uint64_t pool_id)
66 uint64_t aura = roc_npa_aura_handle_to_aura(pool_id);
68 return batch_op_data_tbl[aura];
72 batch_op_data_set(uint64_t pool_id, struct batch_op_data *op_data)
74 uint64_t aura = roc_npa_aura_handle_to_aura(pool_id);
76 batch_op_data_tbl[aura] = op_data;
80 batch_op_init(struct rte_mempool *mp)
82 struct batch_op_data *op_data;
85 op_data = batch_op_data_get(mp->pool_id);
86 /* The data should not have been allocated previously */
87 RTE_ASSERT(op_data == NULL);
89 op_data = rte_zmalloc(NULL, sizeof(struct batch_op_data), ROC_ALIGN);
93 for (i = 0; i < RTE_MAX_LCORE; i++) {
94 op_data->mem[i].sz = 0;
95 op_data->mem[i].status = BATCH_ALLOC_OP_NOT_ISSUED;
98 op_data->lmt_addr = roc_idev_lmt_base_addr_get();
99 batch_op_data_set(mp->pool_id, op_data);
106 batch_op_fini(struct rte_mempool *mp)
108 struct batch_op_data *op_data;
111 op_data = batch_op_data_get(mp->pool_id);
114 for (i = 0; i < RTE_MAX_LCORE; i++) {
115 struct batch_op_mem *mem = &op_data->mem[i];
117 if (mem->status == BATCH_ALLOC_OP_ISSUED) {
118 mem->sz = roc_npa_aura_batch_alloc_extract(
119 mem->objs, mem->objs, BATCH_ALLOC_SZ);
120 mem->status = BATCH_ALLOC_OP_DONE;
122 if (mem->status == BATCH_ALLOC_OP_DONE) {
123 roc_npa_aura_op_bulk_free(mp->pool_id, mem->objs,
125 mem->status = BATCH_ALLOC_OP_NOT_ISSUED;
130 batch_op_data_set(mp->pool_id, NULL);
135 cn10k_mempool_enq(struct rte_mempool *mp, void *const *obj_table,
138 const uint64_t *ptr = (const uint64_t *)obj_table;
139 uint64_t lmt_addr = 0, lmt_id = 0;
140 struct batch_op_data *op_data;
142 /* Ensure mbuf init changes are written before the free pointers are
143 * enqueued to the stack.
148 roc_npa_aura_op_free(mp->pool_id, 1, ptr[0]);
152 op_data = batch_op_data_get(mp->pool_id);
153 lmt_addr = op_data->lmt_addr;
154 ROC_LMT_BASE_ID_GET(lmt_addr, lmt_id);
155 roc_npa_aura_op_batch_free(mp->pool_id, ptr, n, 1, lmt_addr, lmt_id);
161 cn10k_mempool_get_count(const struct rte_mempool *mp)
163 struct batch_op_data *op_data;
164 unsigned int count = 0;
167 op_data = batch_op_data_get(mp->pool_id);
170 for (i = 0; i < RTE_MAX_LCORE; i++) {
171 struct batch_op_mem *mem = &op_data->mem[i];
173 if (mem->status == BATCH_ALLOC_OP_ISSUED)
174 count += roc_npa_aura_batch_alloc_count(mem->objs,
177 if (mem->status == BATCH_ALLOC_OP_DONE)
181 count += cnxk_mempool_get_count(mp);
187 cn10k_mempool_deq(struct rte_mempool *mp, void **obj_table, unsigned int n)
189 struct batch_op_data *op_data;
190 struct batch_op_mem *mem;
191 unsigned int count = 0;
195 op_data = batch_op_data_get(mp->pool_id);
196 tid = rte_lcore_id();
197 mem = &op_data->mem[tid];
199 /* Issue batch alloc */
200 if (mem->status == BATCH_ALLOC_OP_NOT_ISSUED) {
201 rc = roc_npa_aura_batch_alloc_issue(mp->pool_id, mem->objs,
202 BATCH_ALLOC_SZ, 0, 1);
203 /* If issue fails, try falling back to default alloc */
205 return cn10k_mempool_enq(mp, obj_table, n);
206 mem->status = BATCH_ALLOC_OP_ISSUED;
213 if (mem->status == BATCH_ALLOC_OP_ISSUED) {
214 mem->sz = roc_npa_aura_batch_alloc_extract(
215 mem->objs, mem->objs, BATCH_ALLOC_SZ);
217 /* If partial alloc reduce the retry count */
218 retry -= (mem->sz != BATCH_ALLOC_SZ);
219 /* Break the loop if retry count exhausted */
221 mem->status = BATCH_ALLOC_OP_DONE;
225 if (cur_sz > mem->sz)
228 /* Dequeue the pointers */
229 memcpy(&obj_table[count], &mem->objs[mem->sz - cur_sz],
230 cur_sz * sizeof(uintptr_t));
234 /* Break loop if the required pointers has been dequeued */
235 loop &= (count != n);
237 /* Issue next batch alloc if pointers are exhausted */
239 rc = roc_npa_aura_batch_alloc_issue(
240 mp->pool_id, mem->objs, BATCH_ALLOC_SZ, 0, 1);
241 /* Break loop if issue failed and set status */
247 if (unlikely(count != n)) {
248 /* No partial alloc allowed. Free up allocated pointers */
249 cn10k_mempool_enq(mp, obj_table, count);
257 cn10k_mempool_alloc(struct rte_mempool *mp)
263 block_size = mp->elt_size + mp->header_size + mp->trailer_size;
264 /* Align header size to ROC_ALIGN */
265 if (mp->header_size % ROC_ALIGN != 0) {
266 padding = RTE_ALIGN_CEIL(mp->header_size, ROC_ALIGN) -
268 mp->header_size += padding;
269 block_size += padding;
272 /* Align block size to ROC_ALIGN */
273 if (block_size % ROC_ALIGN != 0) {
274 padding = RTE_ALIGN_CEIL(block_size, ROC_ALIGN) - block_size;
275 mp->trailer_size += padding;
276 block_size += padding;
279 rc = cnxk_mempool_alloc(mp);
283 rc = batch_op_init(mp);
285 plt_err("Failed to init batch alloc mem rc=%d", rc);
291 cnxk_mempool_free(mp);
296 cn10k_mempool_free(struct rte_mempool *mp)
299 cnxk_mempool_free(mp);
303 cn10k_mempool_plt_init(void)
305 return batch_op_data_table_create();
308 static struct rte_mempool_ops cn10k_mempool_ops = {
309 .name = "cn10k_mempool_ops",
310 .alloc = cn10k_mempool_alloc,
311 .free = cn10k_mempool_free,
312 .enqueue = cn10k_mempool_enq,
313 .dequeue = cn10k_mempool_deq,
314 .get_count = cn10k_mempool_get_count,
315 .calc_mem_size = cnxk_mempool_calc_mem_size,
316 .populate = cnxk_mempool_populate,
319 RTE_MEMPOOL_REGISTER_OPS(cn10k_mempool_ops);