1 /* SPDX-License-Identifier: BSD-3-Clause
2 * Copyright(C) 2019 Marvell International Ltd.
5 #include <rte_mempool.h>
8 #include "otx2_mempool.h"
11 npa_lf_aura_pool_init(struct otx2_mbox *mbox, uint32_t aura_id,
12 struct npa_aura_s *aura, struct npa_pool_s *pool)
14 struct npa_aq_enq_req *aura_init_req, *pool_init_req;
15 struct npa_aq_enq_rsp *aura_init_rsp, *pool_init_rsp;
16 struct otx2_mbox_dev *mdev = &mbox->dev[0];
19 aura_init_req = otx2_mbox_alloc_msg_npa_aq_enq(mbox);
21 aura_init_req->aura_id = aura_id;
22 aura_init_req->ctype = NPA_AQ_CTYPE_AURA;
23 aura_init_req->op = NPA_AQ_INSTOP_INIT;
24 memcpy(&aura_init_req->aura, aura, sizeof(*aura));
26 pool_init_req = otx2_mbox_alloc_msg_npa_aq_enq(mbox);
28 pool_init_req->aura_id = aura_id;
29 pool_init_req->ctype = NPA_AQ_CTYPE_POOL;
30 pool_init_req->op = NPA_AQ_INSTOP_INIT;
31 memcpy(&pool_init_req->pool, pool, sizeof(*pool));
33 otx2_mbox_msg_send(mbox, 0);
34 rc = otx2_mbox_wait_for_rsp(mbox, 0);
38 off = mbox->rx_start +
39 RTE_ALIGN(sizeof(struct mbox_hdr), MBOX_MSG_ALIGN);
40 aura_init_rsp = (struct npa_aq_enq_rsp *)((uintptr_t)mdev->mbase + off);
41 off = mbox->rx_start + aura_init_rsp->hdr.next_msgoff;
42 pool_init_rsp = (struct npa_aq_enq_rsp *)((uintptr_t)mdev->mbase + off);
44 if (rc == 2 && aura_init_rsp->hdr.rc == 0 && pool_init_rsp->hdr.rc == 0)
47 return NPA_LF_ERR_AURA_POOL_INIT;
51 npa_lf_aura_pool_fini(struct otx2_mbox *mbox,
55 struct npa_aq_enq_req *aura_req, *pool_req;
56 struct npa_aq_enq_rsp *aura_rsp, *pool_rsp;
57 struct otx2_mbox_dev *mdev = &mbox->dev[0];
58 struct ndc_sync_op *ndc_req;
61 /* Procedure for disabling an aura/pool */
63 npa_lf_aura_op_alloc(aura_handle, 0);
65 pool_req = otx2_mbox_alloc_msg_npa_aq_enq(mbox);
66 pool_req->aura_id = aura_id;
67 pool_req->ctype = NPA_AQ_CTYPE_POOL;
68 pool_req->op = NPA_AQ_INSTOP_WRITE;
69 pool_req->pool.ena = 0;
70 pool_req->pool_mask.ena = ~pool_req->pool_mask.ena;
72 aura_req = otx2_mbox_alloc_msg_npa_aq_enq(mbox);
73 aura_req->aura_id = aura_id;
74 aura_req->ctype = NPA_AQ_CTYPE_AURA;
75 aura_req->op = NPA_AQ_INSTOP_WRITE;
76 aura_req->aura.ena = 0;
77 aura_req->aura_mask.ena = ~aura_req->aura_mask.ena;
79 otx2_mbox_msg_send(mbox, 0);
80 rc = otx2_mbox_wait_for_rsp(mbox, 0);
84 off = mbox->rx_start +
85 RTE_ALIGN(sizeof(struct mbox_hdr), MBOX_MSG_ALIGN);
86 pool_rsp = (struct npa_aq_enq_rsp *)((uintptr_t)mdev->mbase + off);
88 off = mbox->rx_start + pool_rsp->hdr.next_msgoff;
89 aura_rsp = (struct npa_aq_enq_rsp *)((uintptr_t)mdev->mbase + off);
91 if (rc != 2 || aura_rsp->hdr.rc != 0 || pool_rsp->hdr.rc != 0)
92 return NPA_LF_ERR_AURA_POOL_FINI;
94 /* Sync NDC-NPA for LF */
95 ndc_req = otx2_mbox_alloc_msg_ndc_sync_op(mbox);
96 ndc_req->npa_lf_sync = 1;
98 rc = otx2_mbox_process(mbox);
100 otx2_err("Error on NDC-NPA LF sync, rc %d", rc);
101 return NPA_LF_ERR_AURA_POOL_FINI;
107 npa_lf_stack_memzone_name(struct otx2_npa_lf *lf, int pool_id, char *name)
109 snprintf(name, RTE_MEMZONE_NAMESIZE, "otx2_npa_stack_%x_%d",
110 lf->pf_func, pool_id);
115 static inline const struct rte_memzone *
116 npa_lf_stack_dma_alloc(struct otx2_npa_lf *lf, char *name,
117 int pool_id, size_t size)
119 return rte_memzone_reserve_aligned(
120 npa_lf_stack_memzone_name(lf, pool_id, name), size, 0,
121 RTE_MEMZONE_IOVA_CONTIG, OTX2_ALIGN);
125 npa_lf_stack_dma_free(struct otx2_npa_lf *lf, char *name, int pool_id)
127 const struct rte_memzone *mz;
129 mz = rte_memzone_lookup(npa_lf_stack_memzone_name(lf, pool_id, name));
133 return rte_memzone_free(mz);
137 bitmap_ctzll(uint64_t slab)
142 return __builtin_ctzll(slab);
146 npa_lf_aura_pool_pair_alloc(struct otx2_npa_lf *lf, const uint32_t block_size,
147 const uint32_t block_count, struct npa_aura_s *aura,
148 struct npa_pool_s *pool, uint64_t *aura_handle)
150 int rc, aura_id, pool_id, stack_size, alloc_size;
151 char name[RTE_MEMZONE_NAMESIZE];
152 const struct rte_memzone *mz;
157 if (!lf || !block_size || !block_count ||
158 !pool || !aura || !aura_handle)
159 return NPA_LF_ERR_PARAM;
161 /* Block size should be cache line aligned and in range of 128B-128KB */
162 if (block_size % OTX2_ALIGN || block_size < 128 ||
163 block_size > 128 * 1024)
164 return NPA_LF_ERR_INVALID_BLOCK_SZ;
167 /* Scan from the beginning */
168 __rte_bitmap_scan_init(lf->npa_bmp);
169 /* Scan bitmap to get the free pool */
170 rc = rte_bitmap_scan(lf->npa_bmp, &pos, &slab);
173 otx2_err("Mempools exhausted, 'max_pools' devargs to increase");
177 /* Get aura_id from resource bitmap */
178 aura_id = pos + bitmap_ctzll(slab);
179 /* Mark pool as reserved */
180 rte_bitmap_clear(lf->npa_bmp, aura_id);
182 /* Configuration based on each aura has separate pool(aura-pool pair) */
184 rc = (aura_id < 0 || pool_id >= (int)lf->nr_pools || aura_id >=
185 (int)BIT_ULL(6 + lf->aura_sz)) ? NPA_LF_ERR_AURA_ID_ALLOC : 0;
189 /* Allocate stack memory */
190 stack_size = (block_count + lf->stack_pg_ptrs - 1) / lf->stack_pg_ptrs;
191 alloc_size = stack_size * lf->stack_pg_bytes;
193 mz = npa_lf_stack_dma_alloc(lf, name, pool_id, alloc_size);
199 /* Update aura fields */
200 aura->pool_addr = pool_id;/* AF will translate to associated poolctx */
202 aura->shift = __builtin_clz(block_count) - 8;
203 aura->limit = block_count;
204 aura->pool_caching = 1;
205 aura->err_int_ena = BIT(NPA_AURA_ERR_INT_AURA_ADD_OVER);
206 aura->err_int_ena |= BIT(NPA_AURA_ERR_INT_AURA_ADD_UNDER);
207 aura->err_int_ena |= BIT(NPA_AURA_ERR_INT_AURA_FREE_UNDER);
208 aura->err_int_ena |= BIT(NPA_AURA_ERR_INT_POOL_DIS);
209 /* Many to one reduction */
210 aura->err_qint_idx = aura_id % lf->qints;
212 /* Update pool fields */
213 pool->stack_base = mz->iova;
215 pool->buf_size = block_size / OTX2_ALIGN;
216 pool->stack_max_pages = stack_size;
217 pool->shift = __builtin_clz(block_count) - 8;
220 pool->stack_caching = 1;
221 pool->err_int_ena = BIT(NPA_POOL_ERR_INT_OVFLS);
222 pool->err_int_ena |= BIT(NPA_POOL_ERR_INT_RANGE);
223 pool->err_int_ena |= BIT(NPA_POOL_ERR_INT_PERR);
225 /* Many to one reduction */
226 pool->err_qint_idx = pool_id % lf->qints;
228 /* Issue AURA_INIT and POOL_INIT op */
229 rc = npa_lf_aura_pool_init(lf->mbox, aura_id, aura, pool);
233 *aura_handle = npa_lf_aura_handle_gen(aura_id, lf->base);
235 /* Update aura count */
236 npa_lf_aura_op_cnt_set(*aura_handle, 0, block_count);
237 /* Read it back to make sure aura count is updated */
238 npa_lf_aura_op_cnt_get(*aura_handle);
243 rte_memzone_free(mz);
245 rte_bitmap_set(lf->npa_bmp, aura_id);
251 npa_lf_aura_pool_pair_free(struct otx2_npa_lf *lf, uint64_t aura_handle)
253 char name[RTE_MEMZONE_NAMESIZE];
254 int aura_id, pool_id, rc;
256 if (!lf || !aura_handle)
257 return NPA_LF_ERR_PARAM;
259 aura_id = pool_id = npa_lf_aura_handle_to_aura(aura_handle);
260 rc = npa_lf_aura_pool_fini(lf->mbox, aura_id, aura_handle);
261 rc |= npa_lf_stack_dma_free(lf, name, pool_id);
263 rte_bitmap_set(lf->npa_bmp, aura_id);
269 otx2_npa_alloc(struct rte_mempool *mp)
271 uint32_t block_size, block_count;
272 struct otx2_npa_lf *lf;
273 struct npa_aura_s aura;
274 struct npa_pool_s pool;
275 uint64_t aura_handle;
278 lf = otx2_npa_lf_obj_get();
284 block_size = mp->elt_size + mp->header_size + mp->trailer_size;
285 block_count = mp->size;
287 if (block_size % OTX2_ALIGN != 0) {
288 otx2_err("Block size should be multiple of 128B");
293 memset(&aura, 0, sizeof(struct npa_aura_s));
294 memset(&pool, 0, sizeof(struct npa_pool_s));
298 if ((uint32_t)pool.buf_offset * OTX2_ALIGN != mp->header_size) {
299 otx2_err("Unsupported mp->header_size=%d", mp->header_size);
304 /* Use driver specific mp->pool_config to override aura config */
305 if (mp->pool_config != NULL)
306 memcpy(&aura, mp->pool_config, sizeof(struct npa_aura_s));
308 rc = npa_lf_aura_pool_pair_alloc(lf, block_size, block_count,
309 &aura, &pool, &aura_handle);
311 otx2_err("Failed to alloc pool or aura rc=%d", rc);
315 /* Store aura_handle for future queue operations */
316 mp->pool_id = aura_handle;
317 otx2_npa_dbg("lf=%p block_sz=%d block_count=%d aura_handle=0x%"PRIx64,
318 lf, block_size, block_count, aura_handle);
320 /* Just hold the reference of the object */
321 otx2_npa_lf_obj_ref();
328 otx2_npa_free(struct rte_mempool *mp)
330 struct otx2_npa_lf *lf = otx2_npa_lf_obj_get();
333 otx2_npa_dbg("lf=%p aura_handle=0x%"PRIx64, lf, mp->pool_id);
335 rc = npa_lf_aura_pool_pair_free(lf, mp->pool_id);
338 otx2_err("Failed to free pool or aura rc=%d", rc);
340 /* Release the reference of npalf */
344 static struct rte_mempool_ops otx2_npa_ops = {
345 .name = "octeontx2_npa",
346 .alloc = otx2_npa_alloc,
347 .free = otx2_npa_free,
350 MEMPOOL_REGISTER_OPS(otx2_npa_ops);