bus/pci: remove useless link dependency on ethdev
[dpdk.git] / drivers / mempool / dpaa2 / dpaa2_hw_mempool.c
1 /* SPDX-License-Identifier: BSD-3-Clause
2  *
3  *   Copyright (c) 2016 Freescale Semiconductor, Inc. All rights reserved.
4  *   Copyright 2016 NXP
5  *
6  */
7
8 #include <unistd.h>
9 #include <stdio.h>
10 #include <sys/types.h>
11 #include <string.h>
12 #include <stdlib.h>
13 #include <fcntl.h>
14 #include <errno.h>
15
16 #include <rte_mbuf.h>
17 #include <rte_ethdev_driver.h>
18 #include <rte_malloc.h>
19 #include <rte_memcpy.h>
20 #include <rte_string_fns.h>
21 #include <rte_cycles.h>
22 #include <rte_kvargs.h>
23 #include <rte_dev.h>
24 #include "rte_dpaa2_mempool.h"
25
26 #include "fslmc_vfio.h"
27 #include <fslmc_logs.h>
28 #include <mc/fsl_dpbp.h>
29 #include <portal/dpaa2_hw_pvt.h>
30 #include <portal/dpaa2_hw_dpio.h>
31 #include "dpaa2_hw_mempool.h"
32 #include "dpaa2_hw_mempool_logs.h"
33
34 #include <dpaax_iova_table.h>
35
36 struct dpaa2_bp_info *rte_dpaa2_bpid_info;
37 static struct dpaa2_bp_list *h_bp_list;
38
39 /* Dynamic logging identified for mempool */
40 int dpaa2_logtype_mempool;
41
42 static int
43 rte_hw_mbuf_create_pool(struct rte_mempool *mp)
44 {
45         struct dpaa2_bp_list *bp_list;
46         struct dpaa2_dpbp_dev *avail_dpbp;
47         struct dpaa2_bp_info *bp_info;
48         struct dpbp_attr dpbp_attr;
49         uint32_t bpid;
50         int ret;
51
52         avail_dpbp = dpaa2_alloc_dpbp_dev();
53
54         if (rte_dpaa2_bpid_info == NULL) {
55                 rte_dpaa2_bpid_info = (struct dpaa2_bp_info *)rte_malloc(NULL,
56                                       sizeof(struct dpaa2_bp_info) * MAX_BPID,
57                                       RTE_CACHE_LINE_SIZE);
58                 if (rte_dpaa2_bpid_info == NULL)
59                         return -ENOMEM;
60                 memset(rte_dpaa2_bpid_info, 0,
61                        sizeof(struct dpaa2_bp_info) * MAX_BPID);
62         }
63
64         if (!avail_dpbp) {
65                 DPAA2_MEMPOOL_ERR("DPAA2 pool not available!");
66                 return -ENOENT;
67         }
68
69         if (unlikely(!DPAA2_PER_LCORE_DPIO)) {
70                 ret = dpaa2_affine_qbman_swp();
71                 if (ret) {
72                         DPAA2_MEMPOOL_ERR("Failure in affining portal");
73                         goto err1;
74                 }
75         }
76
77         ret = dpbp_enable(&avail_dpbp->dpbp, CMD_PRI_LOW, avail_dpbp->token);
78         if (ret != 0) {
79                 DPAA2_MEMPOOL_ERR("Resource enable failure with err code: %d",
80                                   ret);
81                 goto err1;
82         }
83
84         ret = dpbp_get_attributes(&avail_dpbp->dpbp, CMD_PRI_LOW,
85                                   avail_dpbp->token, &dpbp_attr);
86         if (ret != 0) {
87                 DPAA2_MEMPOOL_ERR("Resource read failure with err code: %d",
88                                   ret);
89                 goto err2;
90         }
91
92         bp_info = rte_malloc(NULL,
93                              sizeof(struct dpaa2_bp_info),
94                              RTE_CACHE_LINE_SIZE);
95         if (!bp_info) {
96                 DPAA2_MEMPOOL_ERR("Unable to allocate buffer pool memory");
97                 ret = -ENOMEM;
98                 goto err2;
99         }
100
101         /* Allocate the bp_list which will be added into global_bp_list */
102         bp_list = rte_malloc(NULL, sizeof(struct dpaa2_bp_list),
103                              RTE_CACHE_LINE_SIZE);
104         if (!bp_list) {
105                 DPAA2_MEMPOOL_ERR("Unable to allocate buffer pool memory");
106                 ret = -ENOMEM;
107                 goto err3;
108         }
109
110         /* Set parameters of buffer pool list */
111         bp_list->buf_pool.num_bufs = mp->size;
112         bp_list->buf_pool.size = mp->elt_size
113                         - sizeof(struct rte_mbuf) - rte_pktmbuf_priv_size(mp);
114         bp_list->buf_pool.bpid = dpbp_attr.bpid;
115         bp_list->buf_pool.h_bpool_mem = NULL;
116         bp_list->buf_pool.dpbp_node = avail_dpbp;
117         /* Identification for our offloaded pool_data structure */
118         bp_list->dpaa2_ops_index = mp->ops_index;
119         bp_list->next = h_bp_list;
120         bp_list->mp = mp;
121
122         bpid = dpbp_attr.bpid;
123
124         rte_dpaa2_bpid_info[bpid].meta_data_size = sizeof(struct rte_mbuf)
125                                 + rte_pktmbuf_priv_size(mp);
126         rte_dpaa2_bpid_info[bpid].bp_list = bp_list;
127         rte_dpaa2_bpid_info[bpid].bpid = bpid;
128
129         rte_memcpy(bp_info, (void *)&rte_dpaa2_bpid_info[bpid],
130                    sizeof(struct dpaa2_bp_info));
131         mp->pool_data = (void *)bp_info;
132
133         DPAA2_MEMPOOL_DEBUG("BP List created for bpid =%d", dpbp_attr.bpid);
134
135         h_bp_list = bp_list;
136         return 0;
137 err3:
138         rte_free(bp_info);
139 err2:
140         dpbp_disable(&avail_dpbp->dpbp, CMD_PRI_LOW, avail_dpbp->token);
141 err1:
142         dpaa2_free_dpbp_dev(avail_dpbp);
143
144         return ret;
145 }
146
147 static void
148 rte_hw_mbuf_free_pool(struct rte_mempool *mp)
149 {
150         struct dpaa2_bp_info *bpinfo;
151         struct dpaa2_bp_list *bp;
152         struct dpaa2_dpbp_dev *dpbp_node;
153
154         if (!mp->pool_data) {
155                 DPAA2_MEMPOOL_ERR("Not a valid dpaa2 buffer pool");
156                 return;
157         }
158
159         bpinfo = (struct dpaa2_bp_info *)mp->pool_data;
160         bp = bpinfo->bp_list;
161         dpbp_node = bp->buf_pool.dpbp_node;
162
163         dpbp_disable(&(dpbp_node->dpbp), CMD_PRI_LOW, dpbp_node->token);
164
165         if (h_bp_list == bp) {
166                 h_bp_list = h_bp_list->next;
167         } else { /* if it is not the first node */
168                 struct dpaa2_bp_list *prev = h_bp_list, *temp;
169                 temp = h_bp_list->next;
170                 while (temp) {
171                         if (temp == bp) {
172                                 prev->next = temp->next;
173                                 rte_free(bp);
174                                 break;
175                         }
176                         prev = temp;
177                         temp = temp->next;
178                 }
179         }
180
181         rte_free(mp->pool_data);
182         dpaa2_free_dpbp_dev(dpbp_node);
183 }
184
185 static void
186 rte_dpaa2_mbuf_release(struct rte_mempool *pool __rte_unused,
187                         void * const *obj_table,
188                         uint32_t bpid,
189                         uint32_t meta_data_size,
190                         int count)
191 {
192         struct qbman_release_desc releasedesc;
193         struct qbman_swp *swp;
194         int ret;
195         int i, n;
196         uint64_t bufs[DPAA2_MBUF_MAX_ACQ_REL];
197
198         if (unlikely(!DPAA2_PER_LCORE_DPIO)) {
199                 ret = dpaa2_affine_qbman_swp();
200                 if (ret != 0) {
201                         DPAA2_MEMPOOL_ERR("Failed to allocate IO portal");
202                         return;
203                 }
204         }
205         swp = DPAA2_PER_LCORE_PORTAL;
206
207         /* Create a release descriptor required for releasing
208          * buffers into QBMAN
209          */
210         qbman_release_desc_clear(&releasedesc);
211         qbman_release_desc_set_bpid(&releasedesc, bpid);
212
213         n = count % DPAA2_MBUF_MAX_ACQ_REL;
214         if (unlikely(!n))
215                 goto aligned;
216
217         /* convert mbuf to buffers for the remainder */
218         for (i = 0; i < n ; i++) {
219 #ifdef RTE_LIBRTE_DPAA2_USE_PHYS_IOVA
220                 bufs[i] = (uint64_t)rte_mempool_virt2iova(obj_table[i])
221                                 + meta_data_size;
222 #else
223                 bufs[i] = (uint64_t)obj_table[i] + meta_data_size;
224 #endif
225         }
226
227         /* feed them to bman */
228         do {
229                 ret = qbman_swp_release(swp, &releasedesc, bufs, n);
230         } while (ret == -EBUSY);
231
232 aligned:
233         /* if there are more buffers to free */
234         while (n < count) {
235                 /* convert mbuf to buffers */
236                 for (i = 0; i < DPAA2_MBUF_MAX_ACQ_REL; i++) {
237 #ifdef RTE_LIBRTE_DPAA2_USE_PHYS_IOVA
238                         bufs[i] = (uint64_t)
239                                   rte_mempool_virt2iova(obj_table[n + i])
240                                   + meta_data_size;
241 #else
242                         bufs[i] = (uint64_t)obj_table[n + i] + meta_data_size;
243 #endif
244                 }
245
246                 do {
247                         ret = qbman_swp_release(swp, &releasedesc, bufs,
248                                                 DPAA2_MBUF_MAX_ACQ_REL);
249                 } while (ret == -EBUSY);
250                 n += DPAA2_MBUF_MAX_ACQ_REL;
251         }
252 }
253
254 uint16_t
255 rte_dpaa2_mbuf_pool_bpid(struct rte_mempool *mp)
256 {
257         struct dpaa2_bp_info *bp_info;
258
259         bp_info = mempool_to_bpinfo(mp);
260         if (!(bp_info->bp_list)) {
261                 RTE_LOG(ERR, PMD, "DPAA2 buffer pool not configured\n");
262                 return -ENOMEM;
263         }
264
265         return bp_info->bpid;
266 }
267
268 struct rte_mbuf *
269 rte_dpaa2_mbuf_from_buf_addr(struct rte_mempool *mp, void *buf_addr)
270 {
271         struct dpaa2_bp_info *bp_info;
272
273         bp_info = mempool_to_bpinfo(mp);
274         if (!(bp_info->bp_list)) {
275                 RTE_LOG(ERR, PMD, "DPAA2 buffer pool not configured\n");
276                 return NULL;
277         }
278
279         return (struct rte_mbuf *)((uint8_t *)buf_addr -
280                         bp_info->meta_data_size);
281 }
282
283 int
284 rte_dpaa2_mbuf_alloc_bulk(struct rte_mempool *pool,
285                           void **obj_table, unsigned int count)
286 {
287 #ifdef RTE_LIBRTE_DPAA2_DEBUG_DRIVER
288         static int alloc;
289 #endif
290         struct qbman_swp *swp;
291         uint16_t bpid;
292         size_t bufs[DPAA2_MBUF_MAX_ACQ_REL];
293         int i, ret;
294         unsigned int n = 0;
295         struct dpaa2_bp_info *bp_info;
296
297         bp_info = mempool_to_bpinfo(pool);
298
299         if (!(bp_info->bp_list)) {
300                 DPAA2_MEMPOOL_ERR("DPAA2 buffer pool not configured");
301                 return -ENOENT;
302         }
303
304         bpid = bp_info->bpid;
305
306         if (unlikely(!DPAA2_PER_LCORE_DPIO)) {
307                 ret = dpaa2_affine_qbman_swp();
308                 if (ret != 0) {
309                         DPAA2_MEMPOOL_ERR("Failed to allocate IO portal");
310                         return ret;
311                 }
312         }
313         swp = DPAA2_PER_LCORE_PORTAL;
314
315         while (n < count) {
316                 /* Acquire is all-or-nothing, so we drain in 7s,
317                  * then the remainder.
318                  */
319                 if ((count - n) > DPAA2_MBUF_MAX_ACQ_REL) {
320                         ret = qbman_swp_acquire(swp, bpid, (void *)bufs,
321                                                 DPAA2_MBUF_MAX_ACQ_REL);
322                 } else {
323                         ret = qbman_swp_acquire(swp, bpid, (void *)bufs,
324                                                 count - n);
325                 }
326                 /* In case of less than requested number of buffers available
327                  * in pool, qbman_swp_acquire returns 0
328                  */
329                 if (ret <= 0) {
330                         DPAA2_MEMPOOL_DP_DEBUG(
331                                 "Buffer acquire failed with err code: %d", ret);
332                         /* The API expect the exact number of requested bufs */
333                         /* Releasing all buffers allocated */
334                         rte_dpaa2_mbuf_release(pool, obj_table, bpid,
335                                            bp_info->meta_data_size, n);
336                         return -ENOBUFS;
337                 }
338                 /* assigning mbuf from the acquired objects */
339                 for (i = 0; (i < ret) && bufs[i]; i++) {
340                         DPAA2_MODIFY_IOVA_TO_VADDR(bufs[i], size_t);
341                         obj_table[n] = (struct rte_mbuf *)
342                                        (bufs[i] - bp_info->meta_data_size);
343                         DPAA2_MEMPOOL_DP_DEBUG(
344                                    "Acquired %p address %p from BMAN\n",
345                                    (void *)bufs[i], (void *)obj_table[n]);
346                         n++;
347                 }
348         }
349
350 #ifdef RTE_LIBRTE_DPAA2_DEBUG_DRIVER
351         alloc += n;
352         DPAA2_MEMPOOL_DP_DEBUG("Total = %d , req = %d done = %d\n",
353                                alloc, count, n);
354 #endif
355         return 0;
356 }
357
358 static int
359 rte_hw_mbuf_free_bulk(struct rte_mempool *pool,
360                   void * const *obj_table, unsigned int n)
361 {
362         struct dpaa2_bp_info *bp_info;
363
364         bp_info = mempool_to_bpinfo(pool);
365         if (!(bp_info->bp_list)) {
366                 DPAA2_MEMPOOL_ERR("DPAA2 buffer pool not configured");
367                 return -ENOENT;
368         }
369         rte_dpaa2_mbuf_release(pool, obj_table, bp_info->bpid,
370                            bp_info->meta_data_size, n);
371
372         return 0;
373 }
374
375 static unsigned int
376 rte_hw_mbuf_get_count(const struct rte_mempool *mp)
377 {
378         int ret;
379         unsigned int num_of_bufs = 0;
380         struct dpaa2_bp_info *bp_info;
381         struct dpaa2_dpbp_dev *dpbp_node;
382
383         if (!mp || !mp->pool_data) {
384                 DPAA2_MEMPOOL_ERR("Invalid mempool provided");
385                 return 0;
386         }
387
388         bp_info = (struct dpaa2_bp_info *)mp->pool_data;
389         dpbp_node = bp_info->bp_list->buf_pool.dpbp_node;
390
391         ret = dpbp_get_num_free_bufs(&dpbp_node->dpbp, CMD_PRI_LOW,
392                                      dpbp_node->token, &num_of_bufs);
393         if (ret) {
394                 DPAA2_MEMPOOL_ERR("Unable to obtain free buf count (err=%d)",
395                                   ret);
396                 return 0;
397         }
398
399         DPAA2_MEMPOOL_DP_DEBUG("Free bufs = %u\n", num_of_bufs);
400
401         return num_of_bufs;
402 }
403
404 static int
405 dpaa2_populate(struct rte_mempool *mp, unsigned int max_objs,
406               void *vaddr, rte_iova_t paddr, size_t len,
407               rte_mempool_populate_obj_cb_t *obj_cb, void *obj_cb_arg)
408 {
409         struct rte_memseg_list *msl;
410         /* The memsegment list exists incase the memory is not external.
411          * So, DMA-Map is required only when memory is provided by user,
412          * i.e. External.
413          */
414         msl = rte_mem_virt2memseg_list(vaddr);
415
416         if (!msl) {
417                 DPAA2_MEMPOOL_DEBUG("Memsegment is External.\n");
418                 rte_fslmc_vfio_mem_dmamap((size_t)vaddr,
419                                 (size_t)paddr, (size_t)len);
420         }
421         /* Insert entry into the PA->VA Table */
422         dpaax_iova_table_update(paddr, vaddr, len);
423
424         return rte_mempool_op_populate_default(mp, max_objs, vaddr, paddr, len,
425                                                obj_cb, obj_cb_arg);
426 }
427
428 static const struct rte_mempool_ops dpaa2_mpool_ops = {
429         .name = DPAA2_MEMPOOL_OPS_NAME,
430         .alloc = rte_hw_mbuf_create_pool,
431         .free = rte_hw_mbuf_free_pool,
432         .enqueue = rte_hw_mbuf_free_bulk,
433         .dequeue = rte_dpaa2_mbuf_alloc_bulk,
434         .get_count = rte_hw_mbuf_get_count,
435         .populate = dpaa2_populate,
436 };
437
438 MEMPOOL_REGISTER_OPS(dpaa2_mpool_ops);
439
440 RTE_INIT(dpaa2_mempool_init_log)
441 {
442         dpaa2_logtype_mempool = rte_log_register("mempool.dpaa2");
443         if (dpaa2_logtype_mempool >= 0)
444                 rte_log_set_level(dpaa2_logtype_mempool, RTE_LOG_NOTICE);
445 }