4 * Copyright (c) 2016 Freescale Semiconductor, Inc. All rights reserved.
7 * Redistribution and use in source and binary forms, with or without
8 * modification, are permitted provided that the following conditions
11 * * Redistributions of source code must retain the above copyright
12 * notice, this list of conditions and the following disclaimer.
13 * * Redistributions in binary form must reproduce the above copyright
14 * notice, this list of conditions and the following disclaimer in
15 * the documentation and/or other materials provided with the
17 * * Neither the name of Freescale Semiconductor, Inc nor the names of its
18 * contributors may be used to endorse or promote products derived
19 * from this software without specific prior written permission.
21 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
22 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
23 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
24 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
25 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
26 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
27 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
28 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
29 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
30 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
31 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
36 #include <sys/types.h>
43 #include <rte_ethdev.h>
44 #include <rte_malloc.h>
45 #include <rte_memcpy.h>
46 #include <rte_string_fns.h>
47 #include <rte_cycles.h>
48 #include <rte_kvargs.h>
51 #include <fslmc_logs.h>
52 #include <mc/fsl_dpbp.h>
53 #include <portal/dpaa2_hw_pvt.h>
54 #include <portal/dpaa2_hw_dpio.h>
55 #include "dpaa2_hw_mempool.h"
57 struct dpaa2_bp_info rte_dpaa2_bpid_info[MAX_BPID];
58 static struct dpaa2_bp_list *h_bp_list;
61 rte_hw_mbuf_create_pool(struct rte_mempool *mp)
63 struct dpaa2_bp_list *bp_list;
64 struct dpaa2_dpbp_dev *avail_dpbp;
65 struct dpaa2_bp_info *bp_info;
66 struct dpbp_attr dpbp_attr;
70 avail_dpbp = dpaa2_alloc_dpbp_dev();
73 PMD_DRV_LOG(ERR, "DPAA2 resources not available");
77 if (unlikely(!DPAA2_PER_LCORE_DPIO)) {
78 ret = dpaa2_affine_qbman_swp();
80 RTE_LOG(ERR, PMD, "Failure in affining portal\n");
85 ret = dpbp_enable(&avail_dpbp->dpbp, CMD_PRI_LOW, avail_dpbp->token);
87 PMD_INIT_LOG(ERR, "Resource enable failure with"
88 " err code: %d\n", ret);
92 ret = dpbp_get_attributes(&avail_dpbp->dpbp, CMD_PRI_LOW,
93 avail_dpbp->token, &dpbp_attr);
95 PMD_INIT_LOG(ERR, "Resource read failure with"
96 " err code: %d\n", ret);
100 bp_info = rte_malloc(NULL,
101 sizeof(struct dpaa2_bp_info),
102 RTE_CACHE_LINE_SIZE);
104 PMD_INIT_LOG(ERR, "No heap memory available for bp_info");
109 /* Allocate the bp_list which will be added into global_bp_list */
110 bp_list = rte_malloc(NULL, sizeof(struct dpaa2_bp_list),
111 RTE_CACHE_LINE_SIZE);
113 PMD_INIT_LOG(ERR, "No heap memory available");
118 /* Set parameters of buffer pool list */
119 bp_list->buf_pool.num_bufs = mp->size;
120 bp_list->buf_pool.size = mp->elt_size
121 - sizeof(struct rte_mbuf) - rte_pktmbuf_priv_size(mp);
122 bp_list->buf_pool.bpid = dpbp_attr.bpid;
123 bp_list->buf_pool.h_bpool_mem = NULL;
124 bp_list->buf_pool.dpbp_node = avail_dpbp;
125 /* Identification for our offloaded pool_data structure */
126 bp_list->dpaa2_ops_index = mp->ops_index;
127 bp_list->next = h_bp_list;
130 bpid = dpbp_attr.bpid;
132 rte_dpaa2_bpid_info[bpid].meta_data_size = sizeof(struct rte_mbuf)
133 + rte_pktmbuf_priv_size(mp);
134 rte_dpaa2_bpid_info[bpid].bp_list = bp_list;
135 rte_dpaa2_bpid_info[bpid].bpid = bpid;
137 rte_memcpy(bp_info, (void *)&rte_dpaa2_bpid_info[bpid],
138 sizeof(struct dpaa2_bp_info));
139 mp->pool_data = (void *)bp_info;
141 PMD_INIT_LOG(DEBUG, "BP List created for bpid =%d", dpbp_attr.bpid);
148 dpbp_disable(&avail_dpbp->dpbp, CMD_PRI_LOW, avail_dpbp->token);
150 dpaa2_free_dpbp_dev(avail_dpbp);
156 rte_hw_mbuf_free_pool(struct rte_mempool *mp)
158 struct dpaa2_bp_info *bpinfo;
159 struct dpaa2_bp_list *bp;
160 struct dpaa2_dpbp_dev *dpbp_node;
162 if (!mp->pool_data) {
163 PMD_DRV_LOG(ERR, "Not a valid dpaa22 pool");
167 bpinfo = (struct dpaa2_bp_info *)mp->pool_data;
168 bp = bpinfo->bp_list;
169 dpbp_node = bp->buf_pool.dpbp_node;
171 dpbp_disable(&(dpbp_node->dpbp), CMD_PRI_LOW, dpbp_node->token);
173 if (h_bp_list == bp) {
174 h_bp_list = h_bp_list->next;
175 } else { /* if it is not the first node */
176 struct dpaa2_bp_list *prev = h_bp_list, *temp;
177 temp = h_bp_list->next;
180 prev->next = temp->next;
189 rte_free(mp->pool_data);
190 dpaa2_free_dpbp_dev(dpbp_node);
194 rte_dpaa2_mbuf_release(struct rte_mempool *pool __rte_unused,
195 void * const *obj_table,
197 uint32_t meta_data_size,
200 struct qbman_release_desc releasedesc;
201 struct qbman_swp *swp;
204 uint64_t bufs[DPAA2_MBUF_MAX_ACQ_REL];
206 if (unlikely(!DPAA2_PER_LCORE_DPIO)) {
207 ret = dpaa2_affine_qbman_swp();
209 RTE_LOG(ERR, PMD, "Failed to allocate IO portal\n");
213 swp = DPAA2_PER_LCORE_PORTAL;
215 /* Create a release descriptor required for releasing
218 qbman_release_desc_clear(&releasedesc);
219 qbman_release_desc_set_bpid(&releasedesc, bpid);
221 n = count % DPAA2_MBUF_MAX_ACQ_REL;
225 /* convert mbuf to buffers for the remainder */
226 for (i = 0; i < n ; i++) {
227 #ifdef RTE_LIBRTE_DPAA2_USE_PHYS_IOVA
228 bufs[i] = (uint64_t)rte_mempool_virt2phy(pool, obj_table[i])
231 bufs[i] = (uint64_t)obj_table[i] + meta_data_size;
235 /* feed them to bman */
237 ret = qbman_swp_release(swp, &releasedesc, bufs, n);
238 } while (ret == -EBUSY);
241 /* if there are more buffers to free */
243 /* convert mbuf to buffers */
244 for (i = 0; i < DPAA2_MBUF_MAX_ACQ_REL; i++) {
245 #ifdef RTE_LIBRTE_DPAA2_USE_PHYS_IOVA
247 rte_mempool_virt2phy(pool, obj_table[n + i])
250 bufs[i] = (uint64_t)obj_table[n + i] + meta_data_size;
255 ret = qbman_swp_release(swp, &releasedesc, bufs,
256 DPAA2_MBUF_MAX_ACQ_REL);
257 } while (ret == -EBUSY);
258 n += DPAA2_MBUF_MAX_ACQ_REL;
263 rte_dpaa2_mbuf_alloc_bulk(struct rte_mempool *pool,
264 void **obj_table, unsigned int count)
266 #ifdef RTE_LIBRTE_DPAA2_DEBUG_DRIVER
269 struct qbman_swp *swp;
271 uint64_t bufs[DPAA2_MBUF_MAX_ACQ_REL];
274 struct dpaa2_bp_info *bp_info;
276 bp_info = mempool_to_bpinfo(pool);
278 if (!(bp_info->bp_list)) {
279 RTE_LOG(ERR, PMD, "DPAA2 buffer pool not configured\n");
283 bpid = bp_info->bpid;
285 if (unlikely(!DPAA2_PER_LCORE_DPIO)) {
286 ret = dpaa2_affine_qbman_swp();
288 RTE_LOG(ERR, PMD, "Failed to allocate IO portal\n");
292 swp = DPAA2_PER_LCORE_PORTAL;
295 /* Acquire is all-or-nothing, so we drain in 7s,
296 * then the remainder.
298 if ((count - n) > DPAA2_MBUF_MAX_ACQ_REL) {
299 ret = qbman_swp_acquire(swp, bpid, bufs,
300 DPAA2_MBUF_MAX_ACQ_REL);
302 ret = qbman_swp_acquire(swp, bpid, bufs,
305 /* In case of less than requested number of buffers available
306 * in pool, qbman_swp_acquire returns 0
309 PMD_TX_LOG(ERR, "Buffer acquire failed with"
310 " err code: %d", ret);
311 /* The API expect the exact number of requested bufs */
312 /* Releasing all buffers allocated */
313 rte_dpaa2_mbuf_release(pool, obj_table, bpid,
314 bp_info->meta_data_size, n);
317 /* assigning mbuf from the acquired objects */
318 for (i = 0; (i < ret) && bufs[i]; i++) {
319 DPAA2_MODIFY_IOVA_TO_VADDR(bufs[i], uint64_t);
320 obj_table[n] = (struct rte_mbuf *)
321 (bufs[i] - bp_info->meta_data_size);
322 PMD_TX_LOG(DEBUG, "Acquired %p address %p from BMAN",
323 (void *)bufs[i], (void *)obj_table[n]);
328 #ifdef RTE_LIBRTE_DPAA2_DEBUG_DRIVER
330 PMD_TX_LOG(DEBUG, "Total = %d , req = %d done = %d",
337 rte_hw_mbuf_free_bulk(struct rte_mempool *pool,
338 void * const *obj_table, unsigned int n)
340 struct dpaa2_bp_info *bp_info;
342 bp_info = mempool_to_bpinfo(pool);
343 if (!(bp_info->bp_list)) {
344 RTE_LOG(ERR, PMD, "DPAA2 buffer pool not configured\n");
347 rte_dpaa2_mbuf_release(pool, obj_table, bp_info->bpid,
348 bp_info->meta_data_size, n);
354 rte_hw_mbuf_get_count(const struct rte_mempool *mp)
357 unsigned int num_of_bufs = 0;
358 struct dpaa2_bp_info *bp_info;
359 struct dpaa2_dpbp_dev *dpbp_node;
361 if (!mp || !mp->pool_data) {
362 RTE_LOG(ERR, PMD, "Invalid mempool provided\n");
366 bp_info = (struct dpaa2_bp_info *)mp->pool_data;
367 dpbp_node = bp_info->bp_list->buf_pool.dpbp_node;
369 ret = dpbp_get_num_free_bufs(&dpbp_node->dpbp, CMD_PRI_LOW,
370 dpbp_node->token, &num_of_bufs);
372 RTE_LOG(ERR, PMD, "Unable to obtain free buf count (err=%d)\n",
377 RTE_LOG(DEBUG, PMD, "Free bufs = %u\n", num_of_bufs);
382 struct rte_mempool_ops dpaa2_mpool_ops = {
384 .alloc = rte_hw_mbuf_create_pool,
385 .free = rte_hw_mbuf_free_pool,
386 .enqueue = rte_hw_mbuf_free_bulk,
387 .dequeue = rte_dpaa2_mbuf_alloc_bulk,
388 .get_count = rte_hw_mbuf_get_count,
391 MEMPOOL_REGISTER_OPS(dpaa2_mpool_ops);