1 /* SPDX-License-Identifier: BSD-3-Clause
2 * Copyright(C) 2019 Marvell International Ltd.
5 #include <rte_mempool.h>
8 #include "otx2_mempool.h"
11 otx2_npa_enq(struct rte_mempool *mp, void * const *obj_table, unsigned int n)
13 unsigned int index; const uint64_t aura_handle = mp->pool_id;
14 const uint64_t reg = npa_lf_aura_handle_to_aura(aura_handle);
15 const uint64_t addr = npa_lf_aura_handle_to_base(aura_handle) +
18 for (index = 0; index < n; index++)
19 otx2_store_pair((uint64_t)obj_table[index], reg, addr);
24 static __rte_noinline int
25 npa_lf_aura_op_alloc_one(const int64_t wdata, int64_t * const addr,
26 void **obj_table, uint8_t i)
31 obj_table[i] = (void *)otx2_atomic64_add_nosync(wdata, addr);
32 if (obj_table[i] != NULL)
40 #if defined(RTE_ARCH_ARM64)
41 static __rte_noinline int
42 npa_lf_aura_op_search_alloc(const int64_t wdata, int64_t * const addr,
43 void **obj_table, unsigned int n)
47 for (i = 0; i < n; i++) {
48 if (obj_table[i] != NULL)
50 if (npa_lf_aura_op_alloc_one(wdata, addr, obj_table, i))
58 * Some versions of the compiler don't have support for __int128_t for
59 * CASP inline-asm. i.e. if the optimization level is reduced to -O0 the
60 * CASP restrictions aren't followed and the compiler might end up violation the
61 * CASP rules. Fix it by explicitly providing ((optimize("-O3"))).
64 * ccSPMGzq.s:1648: Error: reg pair must start from even reg at
65 * operand 1 - `casp x21,x22,x0,x1,[x19]'
67 static __attribute__((optimize("-O3"))) __rte_noinline int __hot
68 npa_lf_aura_op_alloc_bulk(const int64_t wdata, int64_t * const addr,
69 unsigned int n, void **obj_table)
71 const __uint128_t wdata128 = ((__uint128_t)wdata << 64) | wdata;
72 uint64x2_t failed = vdupq_n_u64(~0);
77 __uint128_t t0, t1, t2, t3, t4, t5, t6, t7, t8, t9;
82 "casp %[t0], %H[t0], %[wdata], %H[wdata], [%[loc]]\n"
83 "casp %[t1], %H[t1], %[wdata], %H[wdata], [%[loc]]\n"
84 "casp %[t2], %H[t2], %[wdata], %H[wdata], [%[loc]]\n"
85 "casp %[t3], %H[t3], %[wdata], %H[wdata], [%[loc]]\n"
86 "casp %[t4], %H[t4], %[wdata], %H[wdata], [%[loc]]\n"
87 "casp %[t5], %H[t5], %[wdata], %H[wdata], [%[loc]]\n"
88 "casp %[t6], %H[t6], %[wdata], %H[wdata], [%[loc]]\n"
89 "casp %[t7], %H[t7], %[wdata], %H[wdata], [%[loc]]\n"
90 "casp %[t8], %H[t8], %[wdata], %H[wdata], [%[loc]]\n"
91 "casp %[t9], %H[t9], %[wdata], %H[wdata], [%[loc]]\n"
92 "casp %[t10], %H[t10], %[wdata], %H[wdata], [%[loc]]\n"
93 "casp %[t11], %H[t11], %[wdata], %H[wdata], [%[loc]]\n"
95 "fmov v16.D[1], %H[t0]\n"
96 "casp %[t0], %H[t0], %[wdata], %H[wdata], [%[loc]]\n"
98 "fmov v17.D[1], %H[t1]\n"
99 "casp %[t1], %H[t1], %[wdata], %H[wdata], [%[loc]]\n"
101 "fmov v18.D[1], %H[t2]\n"
102 "casp %[t2], %H[t2], %[wdata], %H[wdata], [%[loc]]\n"
104 "fmov v19.D[1], %H[t3]\n"
105 "casp %[t3], %H[t3], %[wdata], %H[wdata], [%[loc]]\n"
106 "and %[failed].16B, %[failed].16B, v16.16B\n"
107 "and %[failed].16B, %[failed].16B, v17.16B\n"
108 "and %[failed].16B, %[failed].16B, v18.16B\n"
109 "and %[failed].16B, %[failed].16B, v19.16B\n"
111 "fmov v20.D[1], %H[t4]\n"
113 "fmov v21.D[1], %H[t5]\n"
115 "fmov v22.D[1], %H[t6]\n"
117 "fmov v23.D[1], %H[t7]\n"
118 "and %[failed].16B, %[failed].16B, v20.16B\n"
119 "and %[failed].16B, %[failed].16B, v21.16B\n"
120 "and %[failed].16B, %[failed].16B, v22.16B\n"
121 "and %[failed].16B, %[failed].16B, v23.16B\n"
122 "st1 { v16.2d, v17.2d, v18.2d, v19.2d}, [%[dst]], 64\n"
123 "st1 { v20.2d, v21.2d, v22.2d, v23.2d}, [%[dst]], 64\n"
125 "fmov v16.D[1], %H[t8]\n"
127 "fmov v17.D[1], %H[t9]\n"
129 "fmov v18.D[1], %H[t10]\n"
131 "fmov v19.D[1], %H[t11]\n"
132 "and %[failed].16B, %[failed].16B, v16.16B\n"
133 "and %[failed].16B, %[failed].16B, v17.16B\n"
134 "and %[failed].16B, %[failed].16B, v18.16B\n"
135 "and %[failed].16B, %[failed].16B, v19.16B\n"
137 "fmov v20.D[1], %H[t0]\n"
139 "fmov v21.D[1], %H[t1]\n"
141 "fmov v22.D[1], %H[t2]\n"
143 "fmov v23.D[1], %H[t3]\n"
144 "and %[failed].16B, %[failed].16B, v20.16B\n"
145 "and %[failed].16B, %[failed].16B, v21.16B\n"
146 "and %[failed].16B, %[failed].16B, v22.16B\n"
147 "and %[failed].16B, %[failed].16B, v23.16B\n"
148 "st1 { v16.2d, v17.2d, v18.2d, v19.2d}, [%[dst]], 64\n"
149 "st1 { v20.2d, v21.2d, v22.2d, v23.2d}, [%[dst]], 64\n"
150 : "+Q" (*addr), [failed] "=&w" (failed),
151 [t0] "=&r" (t0), [t1] "=&r" (t1), [t2] "=&r" (t2),
152 [t3] "=&r" (t3), [t4] "=&r" (t4), [t5] "=&r" (t5),
153 [t6] "=&r" (t6), [t7] "=&r" (t7), [t8] "=&r" (t8),
154 [t9] "=&r" (t9), [t10] "=&r" (t10), [t11] "=&r" (t11)
155 : [wdata] "r" (wdata128), [dst] "r" (obj_table),
157 : "memory", "v16", "v17", "v18",
158 "v19", "v20", "v21", "v22", "v23"
164 __uint128_t t0, t1, t2, t3, t4, t5, t6, t7;
168 "casp %[t0], %H[t0], %[wdata], %H[wdata], [%[loc]]\n"
169 "casp %[t1], %H[t1], %[wdata], %H[wdata], [%[loc]]\n"
170 "casp %[t2], %H[t2], %[wdata], %H[wdata], [%[loc]]\n"
171 "casp %[t3], %H[t3], %[wdata], %H[wdata], [%[loc]]\n"
172 "casp %[t4], %H[t4], %[wdata], %H[wdata], [%[loc]]\n"
173 "casp %[t5], %H[t5], %[wdata], %H[wdata], [%[loc]]\n"
174 "casp %[t6], %H[t6], %[wdata], %H[wdata], [%[loc]]\n"
175 "casp %[t7], %H[t7], %[wdata], %H[wdata], [%[loc]]\n"
177 "fmov v16.D[1], %H[t0]\n"
179 "fmov v17.D[1], %H[t1]\n"
181 "fmov v18.D[1], %H[t2]\n"
183 "fmov v19.D[1], %H[t3]\n"
184 "and %[failed].16B, %[failed].16B, v16.16B\n"
185 "and %[failed].16B, %[failed].16B, v17.16B\n"
186 "and %[failed].16B, %[failed].16B, v18.16B\n"
187 "and %[failed].16B, %[failed].16B, v19.16B\n"
189 "fmov v20.D[1], %H[t4]\n"
191 "fmov v21.D[1], %H[t5]\n"
193 "fmov v22.D[1], %H[t6]\n"
195 "fmov v23.D[1], %H[t7]\n"
196 "and %[failed].16B, %[failed].16B, v20.16B\n"
197 "and %[failed].16B, %[failed].16B, v21.16B\n"
198 "and %[failed].16B, %[failed].16B, v22.16B\n"
199 "and %[failed].16B, %[failed].16B, v23.16B\n"
200 "st1 { v16.2d, v17.2d, v18.2d, v19.2d}, [%[dst]], 64\n"
201 "st1 { v20.2d, v21.2d, v22.2d, v23.2d}, [%[dst]], 64\n"
202 : "+Q" (*addr), [failed] "=&w" (failed),
203 [t0] "=&r" (t0), [t1] "=&r" (t1), [t2] "=&r" (t2),
204 [t3] "=&r" (t3), [t4] "=&r" (t4), [t5] "=&r" (t5),
205 [t6] "=&r" (t6), [t7] "=&r" (t7)
206 : [wdata] "r" (wdata128), [dst] "r" (obj_table),
208 : "memory", "v16", "v17", "v18", "v19",
209 "v20", "v21", "v22", "v23"
215 __uint128_t t0, t1, t2, t3;
219 "casp %[t0], %H[t0], %[wdata], %H[wdata], [%[loc]]\n"
220 "casp %[t1], %H[t1], %[wdata], %H[wdata], [%[loc]]\n"
221 "casp %[t2], %H[t2], %[wdata], %H[wdata], [%[loc]]\n"
222 "casp %[t3], %H[t3], %[wdata], %H[wdata], [%[loc]]\n"
224 "fmov v16.D[1], %H[t0]\n"
226 "fmov v17.D[1], %H[t1]\n"
228 "fmov v18.D[1], %H[t2]\n"
230 "fmov v19.D[1], %H[t3]\n"
231 "and %[failed].16B, %[failed].16B, v16.16B\n"
232 "and %[failed].16B, %[failed].16B, v17.16B\n"
233 "and %[failed].16B, %[failed].16B, v18.16B\n"
234 "and %[failed].16B, %[failed].16B, v19.16B\n"
235 "st1 { v16.2d, v17.2d, v18.2d, v19.2d}, [%[dst]], 64\n"
236 : "+Q" (*addr), [failed] "=&w" (failed),
237 [t0] "=&r" (t0), [t1] "=&r" (t1), [t2] "=&r" (t2),
239 : [wdata] "r" (wdata128), [dst] "r" (obj_table),
241 : "memory", "v16", "v17", "v18", "v19"
251 "casp %[t0], %H[t0], %[wdata], %H[wdata], [%[loc]]\n"
252 "casp %[t1], %H[t1], %[wdata], %H[wdata], [%[loc]]\n"
254 "fmov v16.D[1], %H[t0]\n"
256 "fmov v17.D[1], %H[t1]\n"
257 "and %[failed].16B, %[failed].16B, v16.16B\n"
258 "and %[failed].16B, %[failed].16B, v17.16B\n"
259 "st1 { v16.2d, v17.2d}, [%[dst]], 32\n"
260 : "+Q" (*addr), [failed] "=&w" (failed),
261 [t0] "=&r" (t0), [t1] "=&r" (t1)
262 : [wdata] "r" (wdata128), [dst] "r" (obj_table),
264 : "memory", "v16", "v17"
274 "casp %[t0], %H[t0], %[wdata], %H[wdata], [%[loc]]\n"
276 "fmov v16.D[1], %H[t0]\n"
277 "and %[failed].16B, %[failed].16B, v16.16B\n"
278 "st1 { v16.2d}, [%[dst]], 16\n"
279 : "+Q" (*addr), [failed] "=&w" (failed),
281 : [wdata] "r" (wdata128), [dst] "r" (obj_table),
288 return npa_lf_aura_op_alloc_one(wdata, addr, obj_table, 0);
291 if (unlikely(!(vgetq_lane_u64(failed, 0) & vgetq_lane_u64(failed, 1))))
292 return npa_lf_aura_op_search_alloc(wdata, addr, (void **)
293 ((char *)obj_table - (sizeof(uint64_t) * n)), n);
298 static __rte_noinline void
299 otx2_npa_clear_alloc(struct rte_mempool *mp, void **obj_table, unsigned int n)
303 for (i = 0; i < n; i++) {
304 if (obj_table[i] != NULL) {
305 otx2_npa_enq(mp, &obj_table[i], 1);
311 static inline int __hot
312 otx2_npa_deq_arm64(struct rte_mempool *mp, void **obj_table, unsigned int n)
314 const int64_t wdata = npa_lf_aura_handle_to_aura(mp->pool_id);
315 void **obj_table_bak = obj_table;
316 const unsigned int nfree = n;
319 int64_t * const addr = (int64_t * const)
320 (npa_lf_aura_handle_to_base(mp->pool_id) +
321 NPA_LF_AURA_OP_ALLOCX(0));
323 parts = n > 31 ? 32 : rte_align32prevpow2(n);
325 if (unlikely(npa_lf_aura_op_alloc_bulk(wdata, addr,
326 parts, obj_table))) {
327 otx2_npa_clear_alloc(mp, obj_table_bak, nfree - n);
337 static inline int __hot
338 otx2_npa_deq(struct rte_mempool *mp, void **obj_table, unsigned int n)
340 const int64_t wdata = npa_lf_aura_handle_to_aura(mp->pool_id);
344 int64_t * const addr = (int64_t * const)
345 (npa_lf_aura_handle_to_base(mp->pool_id) +
346 NPA_LF_AURA_OP_ALLOCX(0));
347 for (index = 0; index < n; index++, obj_table++) {
348 obj = npa_lf_aura_op_alloc_one(wdata, addr, obj_table, 0);
350 for (; index > 0; index--) {
352 otx2_npa_enq(mp, obj_table, 1);
356 *obj_table = (void *)obj;
363 otx2_npa_get_count(const struct rte_mempool *mp)
365 return (unsigned int)npa_lf_aura_op_available(mp->pool_id);
369 npa_lf_aura_pool_init(struct otx2_mbox *mbox, uint32_t aura_id,
370 struct npa_aura_s *aura, struct npa_pool_s *pool)
372 struct npa_aq_enq_req *aura_init_req, *pool_init_req;
373 struct npa_aq_enq_rsp *aura_init_rsp, *pool_init_rsp;
374 struct otx2_mbox_dev *mdev = &mbox->dev[0];
377 aura_init_req = otx2_mbox_alloc_msg_npa_aq_enq(mbox);
379 aura_init_req->aura_id = aura_id;
380 aura_init_req->ctype = NPA_AQ_CTYPE_AURA;
381 aura_init_req->op = NPA_AQ_INSTOP_INIT;
382 memcpy(&aura_init_req->aura, aura, sizeof(*aura));
384 pool_init_req = otx2_mbox_alloc_msg_npa_aq_enq(mbox);
386 pool_init_req->aura_id = aura_id;
387 pool_init_req->ctype = NPA_AQ_CTYPE_POOL;
388 pool_init_req->op = NPA_AQ_INSTOP_INIT;
389 memcpy(&pool_init_req->pool, pool, sizeof(*pool));
391 otx2_mbox_msg_send(mbox, 0);
392 rc = otx2_mbox_wait_for_rsp(mbox, 0);
396 off = mbox->rx_start +
397 RTE_ALIGN(sizeof(struct mbox_hdr), MBOX_MSG_ALIGN);
398 aura_init_rsp = (struct npa_aq_enq_rsp *)((uintptr_t)mdev->mbase + off);
399 off = mbox->rx_start + aura_init_rsp->hdr.next_msgoff;
400 pool_init_rsp = (struct npa_aq_enq_rsp *)((uintptr_t)mdev->mbase + off);
402 if (rc == 2 && aura_init_rsp->hdr.rc == 0 && pool_init_rsp->hdr.rc == 0)
405 return NPA_LF_ERR_AURA_POOL_INIT;
409 npa_lf_aura_pool_fini(struct otx2_mbox *mbox,
411 uint64_t aura_handle)
413 struct npa_aq_enq_req *aura_req, *pool_req;
414 struct npa_aq_enq_rsp *aura_rsp, *pool_rsp;
415 struct otx2_mbox_dev *mdev = &mbox->dev[0];
416 struct ndc_sync_op *ndc_req;
419 /* Procedure for disabling an aura/pool */
421 npa_lf_aura_op_alloc(aura_handle, 0);
423 pool_req = otx2_mbox_alloc_msg_npa_aq_enq(mbox);
424 pool_req->aura_id = aura_id;
425 pool_req->ctype = NPA_AQ_CTYPE_POOL;
426 pool_req->op = NPA_AQ_INSTOP_WRITE;
427 pool_req->pool.ena = 0;
428 pool_req->pool_mask.ena = ~pool_req->pool_mask.ena;
430 aura_req = otx2_mbox_alloc_msg_npa_aq_enq(mbox);
431 aura_req->aura_id = aura_id;
432 aura_req->ctype = NPA_AQ_CTYPE_AURA;
433 aura_req->op = NPA_AQ_INSTOP_WRITE;
434 aura_req->aura.ena = 0;
435 aura_req->aura_mask.ena = ~aura_req->aura_mask.ena;
437 otx2_mbox_msg_send(mbox, 0);
438 rc = otx2_mbox_wait_for_rsp(mbox, 0);
442 off = mbox->rx_start +
443 RTE_ALIGN(sizeof(struct mbox_hdr), MBOX_MSG_ALIGN);
444 pool_rsp = (struct npa_aq_enq_rsp *)((uintptr_t)mdev->mbase + off);
446 off = mbox->rx_start + pool_rsp->hdr.next_msgoff;
447 aura_rsp = (struct npa_aq_enq_rsp *)((uintptr_t)mdev->mbase + off);
449 if (rc != 2 || aura_rsp->hdr.rc != 0 || pool_rsp->hdr.rc != 0)
450 return NPA_LF_ERR_AURA_POOL_FINI;
452 /* Sync NDC-NPA for LF */
453 ndc_req = otx2_mbox_alloc_msg_ndc_sync_op(mbox);
454 ndc_req->npa_lf_sync = 1;
456 rc = otx2_mbox_process(mbox);
458 otx2_err("Error on NDC-NPA LF sync, rc %d", rc);
459 return NPA_LF_ERR_AURA_POOL_FINI;
465 npa_lf_stack_memzone_name(struct otx2_npa_lf *lf, int pool_id, char *name)
467 snprintf(name, RTE_MEMZONE_NAMESIZE, "otx2_npa_stack_%x_%d",
468 lf->pf_func, pool_id);
473 static inline const struct rte_memzone *
474 npa_lf_stack_dma_alloc(struct otx2_npa_lf *lf, char *name,
475 int pool_id, size_t size)
477 return rte_memzone_reserve_aligned(
478 npa_lf_stack_memzone_name(lf, pool_id, name), size, 0,
479 RTE_MEMZONE_IOVA_CONTIG, OTX2_ALIGN);
483 npa_lf_stack_dma_free(struct otx2_npa_lf *lf, char *name, int pool_id)
485 const struct rte_memzone *mz;
487 mz = rte_memzone_lookup(npa_lf_stack_memzone_name(lf, pool_id, name));
491 return rte_memzone_free(mz);
495 bitmap_ctzll(uint64_t slab)
500 return __builtin_ctzll(slab);
504 npa_lf_aura_pool_pair_alloc(struct otx2_npa_lf *lf, const uint32_t block_size,
505 const uint32_t block_count, struct npa_aura_s *aura,
506 struct npa_pool_s *pool, uint64_t *aura_handle)
508 int rc, aura_id, pool_id, stack_size, alloc_size;
509 char name[RTE_MEMZONE_NAMESIZE];
510 const struct rte_memzone *mz;
515 if (!lf || !block_size || !block_count ||
516 !pool || !aura || !aura_handle)
517 return NPA_LF_ERR_PARAM;
519 /* Block size should be cache line aligned and in range of 128B-128KB */
520 if (block_size % OTX2_ALIGN || block_size < 128 ||
521 block_size > 128 * 1024)
522 return NPA_LF_ERR_INVALID_BLOCK_SZ;
525 /* Scan from the beginning */
526 __rte_bitmap_scan_init(lf->npa_bmp);
527 /* Scan bitmap to get the free pool */
528 rc = rte_bitmap_scan(lf->npa_bmp, &pos, &slab);
531 otx2_err("Mempools exhausted, 'max_pools' devargs to increase");
535 /* Get aura_id from resource bitmap */
536 aura_id = pos + bitmap_ctzll(slab);
537 /* Mark pool as reserved */
538 rte_bitmap_clear(lf->npa_bmp, aura_id);
540 /* Configuration based on each aura has separate pool(aura-pool pair) */
542 rc = (aura_id < 0 || pool_id >= (int)lf->nr_pools || aura_id >=
543 (int)BIT_ULL(6 + lf->aura_sz)) ? NPA_LF_ERR_AURA_ID_ALLOC : 0;
547 /* Allocate stack memory */
548 stack_size = (block_count + lf->stack_pg_ptrs - 1) / lf->stack_pg_ptrs;
549 alloc_size = stack_size * lf->stack_pg_bytes;
551 mz = npa_lf_stack_dma_alloc(lf, name, pool_id, alloc_size);
557 /* Update aura fields */
558 aura->pool_addr = pool_id;/* AF will translate to associated poolctx */
560 aura->shift = __builtin_clz(block_count) - 8;
561 aura->limit = block_count;
562 aura->pool_caching = 1;
563 aura->err_int_ena = BIT(NPA_AURA_ERR_INT_AURA_ADD_OVER);
564 aura->err_int_ena |= BIT(NPA_AURA_ERR_INT_AURA_ADD_UNDER);
565 aura->err_int_ena |= BIT(NPA_AURA_ERR_INT_AURA_FREE_UNDER);
566 aura->err_int_ena |= BIT(NPA_AURA_ERR_INT_POOL_DIS);
567 /* Many to one reduction */
568 aura->err_qint_idx = aura_id % lf->qints;
570 /* Update pool fields */
571 pool->stack_base = mz->iova;
573 pool->buf_size = block_size / OTX2_ALIGN;
574 pool->stack_max_pages = stack_size;
575 pool->shift = __builtin_clz(block_count) - 8;
578 pool->stack_caching = 1;
579 pool->err_int_ena = BIT(NPA_POOL_ERR_INT_OVFLS);
580 pool->err_int_ena |= BIT(NPA_POOL_ERR_INT_RANGE);
581 pool->err_int_ena |= BIT(NPA_POOL_ERR_INT_PERR);
583 /* Many to one reduction */
584 pool->err_qint_idx = pool_id % lf->qints;
586 /* Issue AURA_INIT and POOL_INIT op */
587 rc = npa_lf_aura_pool_init(lf->mbox, aura_id, aura, pool);
591 *aura_handle = npa_lf_aura_handle_gen(aura_id, lf->base);
593 /* Update aura count */
594 npa_lf_aura_op_cnt_set(*aura_handle, 0, block_count);
595 /* Read it back to make sure aura count is updated */
596 npa_lf_aura_op_cnt_get(*aura_handle);
601 rte_memzone_free(mz);
603 rte_bitmap_set(lf->npa_bmp, aura_id);
609 npa_lf_aura_pool_pair_free(struct otx2_npa_lf *lf, uint64_t aura_handle)
611 char name[RTE_MEMZONE_NAMESIZE];
612 int aura_id, pool_id, rc;
614 if (!lf || !aura_handle)
615 return NPA_LF_ERR_PARAM;
617 aura_id = pool_id = npa_lf_aura_handle_to_aura(aura_handle);
618 rc = npa_lf_aura_pool_fini(lf->mbox, aura_id, aura_handle);
619 rc |= npa_lf_stack_dma_free(lf, name, pool_id);
621 rte_bitmap_set(lf->npa_bmp, aura_id);
627 otx2_npa_alloc(struct rte_mempool *mp)
629 uint32_t block_size, block_count;
630 struct otx2_npa_lf *lf;
631 struct npa_aura_s aura;
632 struct npa_pool_s pool;
633 uint64_t aura_handle;
636 lf = otx2_npa_lf_obj_get();
642 block_size = mp->elt_size + mp->header_size + mp->trailer_size;
643 block_count = mp->size;
645 if (block_size % OTX2_ALIGN != 0) {
646 otx2_err("Block size should be multiple of 128B");
651 memset(&aura, 0, sizeof(struct npa_aura_s));
652 memset(&pool, 0, sizeof(struct npa_pool_s));
656 if ((uint32_t)pool.buf_offset * OTX2_ALIGN != mp->header_size) {
657 otx2_err("Unsupported mp->header_size=%d", mp->header_size);
662 /* Use driver specific mp->pool_config to override aura config */
663 if (mp->pool_config != NULL)
664 memcpy(&aura, mp->pool_config, sizeof(struct npa_aura_s));
666 rc = npa_lf_aura_pool_pair_alloc(lf, block_size, block_count,
667 &aura, &pool, &aura_handle);
669 otx2_err("Failed to alloc pool or aura rc=%d", rc);
673 /* Store aura_handle for future queue operations */
674 mp->pool_id = aura_handle;
675 otx2_npa_dbg("lf=%p block_sz=%d block_count=%d aura_handle=0x%"PRIx64,
676 lf, block_size, block_count, aura_handle);
678 /* Just hold the reference of the object */
679 otx2_npa_lf_obj_ref();
686 otx2_npa_free(struct rte_mempool *mp)
688 struct otx2_npa_lf *lf = otx2_npa_lf_obj_get();
691 otx2_npa_dbg("lf=%p aura_handle=0x%"PRIx64, lf, mp->pool_id);
693 rc = npa_lf_aura_pool_pair_free(lf, mp->pool_id);
696 otx2_err("Failed to free pool or aura rc=%d", rc);
698 /* Release the reference of npalf */
703 otx2_npa_calc_mem_size(const struct rte_mempool *mp, uint32_t obj_num,
704 uint32_t pg_shift, size_t *min_chunk_size, size_t *align)
709 * Simply need space for one more object to be able to
710 * fulfill alignment requirements.
712 mem_size = rte_mempool_op_calc_mem_size_default(mp, obj_num + 1,
714 min_chunk_size, align);
717 * Memory area which contains objects must be physically
720 *min_chunk_size = mem_size;
727 otx2_npa_populate(struct rte_mempool *mp, unsigned int max_objs, void *vaddr,
728 rte_iova_t iova, size_t len,
729 rte_mempool_populate_obj_cb_t *obj_cb, void *obj_cb_arg)
734 if (iova == RTE_BAD_IOVA)
737 total_elt_sz = mp->header_size + mp->elt_size + mp->trailer_size;
739 /* Align object start address to a multiple of total_elt_sz */
740 off = total_elt_sz - ((uintptr_t)vaddr % total_elt_sz);
745 vaddr = (char *)vaddr + off;
749 npa_lf_aura_op_range_set(mp->pool_id, iova, iova + len);
751 return rte_mempool_op_populate_default(mp, max_objs, vaddr, iova, len,
755 static struct rte_mempool_ops otx2_npa_ops = {
756 .name = "octeontx2_npa",
757 .alloc = otx2_npa_alloc,
758 .free = otx2_npa_free,
759 .enqueue = otx2_npa_enq,
760 .get_count = otx2_npa_get_count,
761 .calc_mem_size = otx2_npa_calc_mem_size,
762 .populate = otx2_npa_populate,
763 #if defined(RTE_ARCH_ARM64)
764 .dequeue = otx2_npa_deq_arm64,
766 .dequeue = otx2_npa_deq,
770 MEMPOOL_REGISTER_OPS(otx2_npa_ops);