1 /* SPDX-License-Identifier: BSD-3-Clause
2 * Copyright 2018-2020 NXP
9 #include <rte_atomic.h>
10 #include <rte_lcore.h>
11 #include <rte_rawdev.h>
12 #include <rte_rawdev_pmd.h>
13 #include <rte_malloc.h>
15 #include <rte_mempool.h>
16 #include <rte_prefetch.h>
17 #include <rte_kvargs.h>
19 #include <mc/fsl_dpdmai.h>
20 #include <portal/dpaa2_hw_pvt.h>
21 #include <portal/dpaa2_hw_dpio.h>
23 #include "rte_pmd_dpaa2_qdma.h"
24 #include "dpaa2_qdma.h"
25 #include "dpaa2_qdma_logs.h"
27 #define DPAA2_QDMA_NO_PREFETCH "no_prefetch"
29 /* Dynamic log type identifier */
30 int dpaa2_qdma_logtype;
32 uint32_t dpaa2_coherent_no_alloc_cache;
33 uint32_t dpaa2_coherent_alloc_cache;
36 static struct qdma_device q_dev;
38 /* QDMA H/W queues list */
39 TAILQ_HEAD(qdma_hw_queue_list, qdma_hw_queue);
40 static struct qdma_hw_queue_list qdma_queue_list
41 = TAILQ_HEAD_INITIALIZER(qdma_queue_list);
43 /* QDMA per core data */
44 static struct qdma_per_core_info qdma_core_info[RTE_MAX_LCORE];
47 qdma_populate_fd_pci(phys_addr_t src, phys_addr_t dest,
48 uint32_t len, struct qbman_fd *fd,
49 struct rte_qdma_rbp *rbp)
51 fd->simple_pci.saddr_lo = lower_32_bits((uint64_t) (src));
52 fd->simple_pci.saddr_hi = upper_32_bits((uint64_t) (src));
54 fd->simple_pci.len_sl = len;
56 fd->simple_pci.bmt = 1;
57 fd->simple_pci.fmt = 3;
58 fd->simple_pci.sl = 1;
59 fd->simple_pci.ser = 1;
61 fd->simple_pci.sportid = rbp->sportid; /*pcie 3 */
62 fd->simple_pci.srbp = rbp->srbp;
64 fd->simple_pci.rdttype = 0;
66 fd->simple_pci.rdttype = dpaa2_coherent_alloc_cache;
68 /*dest is pcie memory */
69 fd->simple_pci.dportid = rbp->dportid; /*pcie 3 */
70 fd->simple_pci.drbp = rbp->drbp;
72 fd->simple_pci.wrttype = 0;
74 fd->simple_pci.wrttype = dpaa2_coherent_no_alloc_cache;
76 fd->simple_pci.daddr_lo = lower_32_bits((uint64_t) (dest));
77 fd->simple_pci.daddr_hi = upper_32_bits((uint64_t) (dest));
83 qdma_populate_fd_ddr(phys_addr_t src, phys_addr_t dest,
84 uint32_t len, struct qbman_fd *fd)
86 fd->simple_ddr.saddr_lo = lower_32_bits((uint64_t) (src));
87 fd->simple_ddr.saddr_hi = upper_32_bits((uint64_t) (src));
89 fd->simple_ddr.len = len;
91 fd->simple_ddr.bmt = 1;
92 fd->simple_ddr.fmt = 3;
93 fd->simple_ddr.sl = 1;
94 fd->simple_ddr.ser = 1;
96 * src If RBP=0 {NS,RDTTYPE[3:0]}: 0_1011
97 * Coherent copy of cacheable memory,
98 * lookup in downstream cache, no allocate
101 fd->simple_ddr.rns = 0;
102 fd->simple_ddr.rdttype = dpaa2_coherent_alloc_cache;
104 * dest If RBP=0 {NS,WRTTYPE[3:0]}: 0_0111
105 * Coherent write of cacheable memory,
106 * lookup in downstream cache, no allocate on miss
108 fd->simple_ddr.wns = 0;
109 fd->simple_ddr.wrttype = dpaa2_coherent_no_alloc_cache;
111 fd->simple_ddr.daddr_lo = lower_32_bits((uint64_t) (dest));
112 fd->simple_ddr.daddr_hi = upper_32_bits((uint64_t) (dest));
118 dpaa2_qdma_populate_fle(struct qbman_fle *fle,
120 struct rte_qdma_rbp *rbp,
121 uint64_t src, uint64_t dest,
122 size_t len, uint32_t flags, uint32_t fmt)
124 struct qdma_sdd *sdd;
127 sdd = (struct qdma_sdd *)
128 ((uintptr_t)(uint64_t)fle - QDMA_FLE_FLE_OFFSET +
129 QDMA_FLE_SDD_OFFSET);
130 sdd_iova = fle_iova - QDMA_FLE_FLE_OFFSET + QDMA_FLE_SDD_OFFSET;
132 /* first frame list to source descriptor */
133 DPAA2_SET_FLE_ADDR(fle, sdd_iova);
134 DPAA2_SET_FLE_LEN(fle, (2 * (sizeof(struct qdma_sdd))));
136 /* source and destination descriptor */
137 if (rbp && rbp->enable) {
139 sdd->read_cmd.portid = rbp->sportid;
140 sdd->rbpcmd_simple.pfid = rbp->spfid;
141 sdd->rbpcmd_simple.vfid = rbp->svfid;
144 sdd->read_cmd.rbp = rbp->srbp;
145 sdd->read_cmd.rdtype = DPAA2_RBP_MEM_RW;
147 sdd->read_cmd.rdtype = dpaa2_coherent_no_alloc_cache;
151 sdd->write_cmd.portid = rbp->dportid;
152 sdd->rbpcmd_simple.pfid = rbp->dpfid;
153 sdd->rbpcmd_simple.vfid = rbp->dvfid;
156 sdd->write_cmd.rbp = rbp->drbp;
157 sdd->write_cmd.wrttype = DPAA2_RBP_MEM_RW;
159 sdd->write_cmd.wrttype = dpaa2_coherent_alloc_cache;
163 sdd->read_cmd.rdtype = dpaa2_coherent_no_alloc_cache;
165 sdd->write_cmd.wrttype = dpaa2_coherent_alloc_cache;
168 /* source frame list to source buffer */
169 if (flags & RTE_QDMA_JOB_SRC_PHY) {
170 DPAA2_SET_FLE_ADDR(fle, src);
171 #ifdef RTE_LIBRTE_DPAA2_USE_PHYS_IOVA
172 DPAA2_SET_FLE_BMT(fle);
175 DPAA2_SET_FLE_ADDR(fle, DPAA2_VADDR_TO_IOVA(src));
177 fle->word4.fmt = fmt;
178 DPAA2_SET_FLE_LEN(fle, len);
181 /* destination frame list to destination buffer */
182 if (flags & RTE_QDMA_JOB_DEST_PHY) {
183 #ifdef RTE_LIBRTE_DPAA2_USE_PHYS_IOVA
184 DPAA2_SET_FLE_BMT(fle);
186 DPAA2_SET_FLE_ADDR(fle, dest);
188 DPAA2_SET_FLE_ADDR(fle, DPAA2_VADDR_TO_IOVA(dest));
190 fle->word4.fmt = fmt;
191 DPAA2_SET_FLE_LEN(fle, len);
193 /* Final bit: 1, for last frame list */
194 DPAA2_SET_FLE_FIN(fle);
197 static inline int dpdmai_dev_set_fd_us(
198 struct qdma_virt_queue *qdma_vq,
200 struct rte_qdma_job **job,
203 struct rte_qdma_rbp *rbp = &qdma_vq->rbp;
204 struct rte_qdma_job **ppjob;
208 for (loop = 0; loop < nb_jobs; loop++) {
209 if (job[loop]->src & QDMA_RBP_UPPER_ADDRESS_MASK)
210 iova = (size_t)job[loop]->dest;
212 iova = (size_t)job[loop]->src;
214 /* Set the metadata */
215 job[loop]->vq_id = qdma_vq->vq_id;
216 ppjob = (struct rte_qdma_job **)DPAA2_IOVA_TO_VADDR(iova) - 1;
219 if ((rbp->drbp == 1) || (rbp->srbp == 1))
220 ret = qdma_populate_fd_pci((phys_addr_t)job[loop]->src,
221 (phys_addr_t)job[loop]->dest,
222 job[loop]->len, &fd[loop], rbp);
224 ret = qdma_populate_fd_ddr((phys_addr_t)job[loop]->src,
225 (phys_addr_t)job[loop]->dest,
226 job[loop]->len, &fd[loop]);
232 static uint32_t qdma_populate_sg_entry(
233 struct rte_qdma_job **jobs,
234 struct qdma_sg_entry *src_sge,
235 struct qdma_sg_entry *dst_sge,
239 uint32_t total_len = 0;
242 for (i = 0; i < nb_jobs; i++) {
244 if (likely(jobs[i]->flags & RTE_QDMA_JOB_SRC_PHY)) {
245 src_sge->addr_lo = (uint32_t)jobs[i]->src;
246 src_sge->addr_hi = (jobs[i]->src >> 32);
248 iova = DPAA2_VADDR_TO_IOVA(jobs[i]->src);
249 src_sge->addr_lo = (uint32_t)iova;
250 src_sge->addr_hi = iova >> 32;
252 src_sge->data_len.data_len_sl0 = jobs[i]->len;
253 src_sge->ctrl.sl = QDMA_SG_SL_LONG;
254 src_sge->ctrl.fmt = QDMA_SG_FMT_SDB;
255 #ifdef RTE_LIBRTE_DPAA2_USE_PHYS_IOVA
256 src_sge->ctrl.bmt = QDMA_SG_BMT_ENABLE;
258 src_sge->ctrl.bmt = QDMA_SG_BMT_DISABLE;
261 if (likely(jobs[i]->flags & RTE_QDMA_JOB_DEST_PHY)) {
262 dst_sge->addr_lo = (uint32_t)jobs[i]->dest;
263 dst_sge->addr_hi = (jobs[i]->dest >> 32);
265 iova = DPAA2_VADDR_TO_IOVA(jobs[i]->dest);
266 dst_sge->addr_lo = (uint32_t)iova;
267 dst_sge->addr_hi = iova >> 32;
269 dst_sge->data_len.data_len_sl0 = jobs[i]->len;
270 dst_sge->ctrl.sl = QDMA_SG_SL_LONG;
271 dst_sge->ctrl.fmt = QDMA_SG_FMT_SDB;
272 #ifdef RTE_LIBRTE_DPAA2_USE_PHYS_IOVA
273 dst_sge->ctrl.bmt = QDMA_SG_BMT_ENABLE;
275 dst_sge->ctrl.bmt = QDMA_SG_BMT_DISABLE;
277 total_len += jobs[i]->len;
279 if (i == (nb_jobs - 1)) {
280 src_sge->ctrl.f = QDMA_SG_F;
281 dst_sge->ctrl.f = QDMA_SG_F;
293 static inline int dpdmai_dev_set_multi_fd_lf(
294 struct qdma_virt_queue *qdma_vq,
296 struct rte_qdma_job **job,
299 struct rte_qdma_rbp *rbp = &qdma_vq->rbp;
300 struct rte_qdma_job **ppjob;
303 void *elem[RTE_QDMA_BURST_NB_MAX];
304 struct qbman_fle *fle;
305 uint64_t elem_iova, fle_iova;
307 ret = rte_mempool_get_bulk(qdma_vq->fle_pool, elem, nb_jobs);
309 DPAA2_QDMA_DP_DEBUG("Memory alloc failed for FLE");
313 for (i = 0; i < nb_jobs; i++) {
314 #ifdef RTE_LIBRTE_DPAA2_USE_PHYS_IOVA
315 elem_iova = rte_mempool_virt2iova(elem[i]);
317 elem_iova = DPAA2_VADDR_TO_IOVA(elem[i]);
320 ppjob = (struct rte_qdma_job **)
321 ((uintptr_t)(uint64_t)elem[i] +
322 QDMA_FLE_SINGLE_JOB_OFFSET);
325 job[i]->vq_id = qdma_vq->vq_id;
327 fle = (struct qbman_fle *)
328 ((uintptr_t)(uint64_t)elem[i] + QDMA_FLE_FLE_OFFSET);
329 fle_iova = elem_iova + QDMA_FLE_FLE_OFFSET;
331 DPAA2_SET_FD_ADDR(&fd[i], fle_iova);
332 DPAA2_SET_FD_COMPOUND_FMT(&fd[i]);
333 DPAA2_SET_FD_FRC(&fd[i], QDMA_SER_CTX);
335 memset(fle, 0, DPAA2_QDMA_MAX_FLE * sizeof(struct qbman_fle) +
336 DPAA2_QDMA_MAX_SDD * sizeof(struct qdma_sdd));
338 dpaa2_qdma_populate_fle(fle, fle_iova, rbp,
339 job[i]->src, job[i]->dest, job[i]->len,
340 job[i]->flags, QBMAN_FLE_WORD4_FMT_SBF);
346 static inline int dpdmai_dev_set_sg_fd_lf(
347 struct qdma_virt_queue *qdma_vq,
349 struct rte_qdma_job **job,
352 struct rte_qdma_rbp *rbp = &qdma_vq->rbp;
353 struct rte_qdma_job **ppjob;
355 struct qbman_fle *fle;
356 uint64_t elem_iova, fle_iova, src, dst;
358 struct qdma_sg_entry *src_sge, *dst_sge;
359 uint32_t len, fmt, flags;
362 * Get an FLE/SDD from FLE pool.
363 * Note: IO metadata is before the FLE and SDD memory.
365 ret = rte_mempool_get(qdma_vq->fle_pool, (void **)(&elem));
367 DPAA2_QDMA_DP_DEBUG("Memory alloc failed for FLE");
371 #ifdef RTE_LIBRTE_DPAA2_USE_PHYS_IOVA
372 elem_iova = rte_mempool_virt2iova(elem);
374 elem_iova = DPAA2_VADDR_TO_IOVA(elem);
377 /* Set the metadata */
378 /* Save job context. */
380 ((uintptr_t)(uint64_t)elem + QDMA_FLE_JOB_NB_OFFSET)) = nb_jobs;
381 ppjob = (struct rte_qdma_job **)
382 ((uintptr_t)(uint64_t)elem + QDMA_FLE_SG_JOBS_OFFSET);
383 for (i = 0; i < nb_jobs; i++)
386 ppjob[0]->vq_id = qdma_vq->vq_id;
388 fle = (struct qbman_fle *)
389 ((uintptr_t)(uint64_t)elem + QDMA_FLE_FLE_OFFSET);
390 fle_iova = elem_iova + QDMA_FLE_FLE_OFFSET;
392 DPAA2_SET_FD_ADDR(fd, fle_iova);
393 DPAA2_SET_FD_COMPOUND_FMT(fd);
394 DPAA2_SET_FD_FRC(fd, QDMA_SER_CTX);
397 if (likely(nb_jobs > 1)) {
398 src_sge = (struct qdma_sg_entry *)
399 ((uintptr_t)(uint64_t)elem + QDMA_FLE_SG_ENTRY_OFFSET);
400 dst_sge = src_sge + DPAA2_QDMA_MAX_SG_NB;
401 src = elem_iova + QDMA_FLE_SG_ENTRY_OFFSET;
403 DPAA2_QDMA_MAX_SG_NB * sizeof(struct qdma_sg_entry);
404 len = qdma_populate_sg_entry(job, src_sge, dst_sge, nb_jobs);
405 fmt = QBMAN_FLE_WORD4_FMT_SGE;
406 flags = RTE_QDMA_JOB_SRC_PHY | RTE_QDMA_JOB_DEST_PHY;
411 fmt = QBMAN_FLE_WORD4_FMT_SBF;
412 flags = job[0]->flags;
415 memset(fle, 0, DPAA2_QDMA_MAX_FLE * sizeof(struct qbman_fle) +
416 DPAA2_QDMA_MAX_SDD * sizeof(struct qdma_sdd));
418 dpaa2_qdma_populate_fle(fle, fle_iova, rbp,
419 src, dst, len, flags, fmt);
424 static inline uint16_t dpdmai_dev_get_job_us(
425 struct qdma_virt_queue *qdma_vq __rte_unused,
426 const struct qbman_fd *fd,
427 struct rte_qdma_job **job, uint16_t *nb_jobs)
431 struct rte_qdma_job **ppjob;
433 if (fd->simple_pci.saddr_hi & (QDMA_RBP_UPPER_ADDRESS_MASK >> 32))
434 iova = (size_t)(((uint64_t)fd->simple_pci.daddr_hi) << 32
435 | (uint64_t)fd->simple_pci.daddr_lo);
437 iova = (size_t)(((uint64_t)fd->simple_pci.saddr_hi) << 32
438 | (uint64_t)fd->simple_pci.saddr_lo);
440 ppjob = (struct rte_qdma_job **)DPAA2_IOVA_TO_VADDR(iova) - 1;
441 *job = (struct rte_qdma_job *)*ppjob;
442 (*job)->status = (fd->simple_pci.acc_err << 8) |
443 (fd->simple_pci.error);
444 vqid = (*job)->vq_id;
450 static inline uint16_t dpdmai_dev_get_single_job_lf(
451 struct qdma_virt_queue *qdma_vq,
452 const struct qbman_fd *fd,
453 struct rte_qdma_job **job,
456 struct qbman_fle *fle;
457 struct rte_qdma_job **ppjob = NULL;
461 * Fetch metadata from FLE. job and vq_id were set
462 * in metadata in the enqueue operation.
464 fle = (struct qbman_fle *)
465 DPAA2_IOVA_TO_VADDR(DPAA2_GET_FD_ADDR(fd));
468 ppjob = (struct rte_qdma_job **)((uintptr_t)(uint64_t)fle -
469 QDMA_FLE_FLE_OFFSET + QDMA_FLE_SINGLE_JOB_OFFSET);
471 status = (DPAA2_GET_FD_ERR(fd) << 8) | (DPAA2_GET_FD_FRC(fd) & 0xFF);
474 (*job)->status = status;
476 /* Free FLE to the pool */
477 rte_mempool_put(qdma_vq->fle_pool,
479 ((uintptr_t)(uint64_t)fle - QDMA_FLE_FLE_OFFSET));
481 return (*job)->vq_id;
484 static inline uint16_t dpdmai_dev_get_sg_job_lf(
485 struct qdma_virt_queue *qdma_vq,
486 const struct qbman_fd *fd,
487 struct rte_qdma_job **job,
490 struct qbman_fle *fle;
491 struct rte_qdma_job **ppjob = NULL;
495 * Fetch metadata from FLE. job and vq_id were set
496 * in metadata in the enqueue operation.
498 fle = (struct qbman_fle *)
499 DPAA2_IOVA_TO_VADDR(DPAA2_GET_FD_ADDR(fd));
500 *nb_jobs = *((uint16_t *)((uintptr_t)(uint64_t)fle -
501 QDMA_FLE_FLE_OFFSET + QDMA_FLE_JOB_NB_OFFSET));
502 ppjob = (struct rte_qdma_job **)((uintptr_t)(uint64_t)fle -
503 QDMA_FLE_FLE_OFFSET + QDMA_FLE_SG_JOBS_OFFSET);
504 status = (DPAA2_GET_FD_ERR(fd) << 8) | (DPAA2_GET_FD_FRC(fd) & 0xFF);
506 for (i = 0; i < (*nb_jobs); i++) {
508 job[i]->status = status;
511 /* Free FLE to the pool */
512 rte_mempool_put(qdma_vq->fle_pool,
514 ((uintptr_t)(uint64_t)fle - QDMA_FLE_FLE_OFFSET));
516 return job[0]->vq_id;
519 /* Function to receive a QDMA job for a given device and queue*/
521 dpdmai_dev_dequeue_multijob_prefetch(
522 struct qdma_virt_queue *qdma_vq,
524 struct rte_qdma_job **job,
527 struct qdma_hw_queue *qdma_pq = qdma_vq->hw_queue;
528 struct dpaa2_dpdmai_dev *dpdmai_dev = qdma_pq->dpdmai_dev;
529 uint16_t rxq_id = qdma_pq->queue_id;
531 struct dpaa2_queue *rxq;
532 struct qbman_result *dq_storage, *dq_storage1 = NULL;
533 struct qbman_pull_desc pulldesc;
534 struct qbman_swp *swp;
535 struct queue_storage_info_t *q_storage;
537 uint8_t status, pending;
539 const struct qbman_fd *fd;
540 uint16_t vqid, num_rx_ret;
543 if (qdma_vq->flags & RTE_QDMA_VQ_FD_SG_FORMAT) {
544 /** Make sure there are enough space to get jobs.*/
545 if (unlikely(nb_jobs < DPAA2_QDMA_MAX_SG_NB))
550 if (unlikely(!DPAA2_PER_LCORE_DPIO)) {
551 ret = dpaa2_affine_qbman_swp();
554 "Failed to allocate IO portal, tid: %d\n",
559 swp = DPAA2_PER_LCORE_PORTAL;
561 pull_size = (nb_jobs > dpaa2_dqrr_size) ? dpaa2_dqrr_size : nb_jobs;
562 rxq = &(dpdmai_dev->rx_queue[rxq_id]);
564 q_storage = rxq->q_storage;
566 if (unlikely(!q_storage->active_dqs)) {
567 q_storage->toggle = 0;
568 dq_storage = q_storage->dq_storage[q_storage->toggle];
569 q_storage->last_num_pkts = pull_size;
570 qbman_pull_desc_clear(&pulldesc);
571 qbman_pull_desc_set_numframes(&pulldesc,
572 q_storage->last_num_pkts);
573 qbman_pull_desc_set_fq(&pulldesc, fqid);
574 qbman_pull_desc_set_storage(&pulldesc, dq_storage,
575 (size_t)(DPAA2_VADDR_TO_IOVA(dq_storage)), 1);
576 if (check_swp_active_dqs(DPAA2_PER_LCORE_DPIO->index)) {
577 while (!qbman_check_command_complete(
579 DPAA2_PER_LCORE_DPIO->index)))
581 clear_swp_active_dqs(DPAA2_PER_LCORE_DPIO->index);
584 if (qbman_swp_pull(swp, &pulldesc)) {
586 "VDQ command not issued.QBMAN busy\n");
587 /* Portal was busy, try again */
592 q_storage->active_dqs = dq_storage;
593 q_storage->active_dpio_id = DPAA2_PER_LCORE_DPIO->index;
594 set_swp_active_dqs(DPAA2_PER_LCORE_DPIO->index,
598 dq_storage = q_storage->active_dqs;
599 rte_prefetch0((void *)(size_t)(dq_storage));
600 rte_prefetch0((void *)(size_t)(dq_storage + 1));
602 /* Prepare next pull descriptor. This will give space for the
603 * prefething done on DQRR entries
605 q_storage->toggle ^= 1;
606 dq_storage1 = q_storage->dq_storage[q_storage->toggle];
607 qbman_pull_desc_clear(&pulldesc);
608 qbman_pull_desc_set_numframes(&pulldesc, pull_size);
609 qbman_pull_desc_set_fq(&pulldesc, fqid);
610 qbman_pull_desc_set_storage(&pulldesc, dq_storage1,
611 (size_t)(DPAA2_VADDR_TO_IOVA(dq_storage1)), 1);
613 /* Check if the previous issued command is completed.
614 * Also seems like the SWP is shared between the Ethernet Driver
615 * and the SEC driver.
617 while (!qbman_check_command_complete(dq_storage))
619 if (dq_storage == get_swp_active_dqs(q_storage->active_dpio_id))
620 clear_swp_active_dqs(q_storage->active_dpio_id);
625 /* Loop until the dq_storage is updated with
628 while (!qbman_check_new_result(dq_storage))
630 rte_prefetch0((void *)((size_t)(dq_storage + 2)));
631 /* Check whether Last Pull command is Expired and
632 * setting Condition for Loop termination
634 if (qbman_result_DQ_is_pull_complete(dq_storage)) {
636 /* Check for valid frame. */
637 status = qbman_result_DQ_flags(dq_storage);
638 if (unlikely((status & QBMAN_DQ_STAT_VALIDFRAME) == 0))
641 fd = qbman_result_DQ_fd(dq_storage);
643 vqid = qdma_vq->get_job(qdma_vq, fd, &job[num_rx],
646 vq_id[num_rx] = vqid;
649 num_rx += num_rx_ret;
652 if (check_swp_active_dqs(DPAA2_PER_LCORE_DPIO->index)) {
653 while (!qbman_check_command_complete(
654 get_swp_active_dqs(DPAA2_PER_LCORE_DPIO->index)))
656 clear_swp_active_dqs(DPAA2_PER_LCORE_DPIO->index);
658 /* issue a volatile dequeue command for next pull */
660 if (qbman_swp_pull(swp, &pulldesc)) {
662 "VDQ command is not issued. QBMAN is busy (2)\n");
668 q_storage->active_dqs = dq_storage1;
669 q_storage->active_dpio_id = DPAA2_PER_LCORE_DPIO->index;
670 set_swp_active_dqs(DPAA2_PER_LCORE_DPIO->index, dq_storage1);
676 dpdmai_dev_dequeue_multijob_no_prefetch(
677 struct qdma_virt_queue *qdma_vq,
679 struct rte_qdma_job **job,
682 struct qdma_hw_queue *qdma_pq = qdma_vq->hw_queue;
683 struct dpaa2_dpdmai_dev *dpdmai_dev = qdma_pq->dpdmai_dev;
684 uint16_t rxq_id = qdma_pq->queue_id;
686 struct dpaa2_queue *rxq;
687 struct qbman_result *dq_storage;
688 struct qbman_pull_desc pulldesc;
689 struct qbman_swp *swp;
691 uint8_t status, pending;
693 const struct qbman_fd *fd;
694 uint16_t vqid, num_rx_ret;
695 int ret, next_pull, num_pulled = 0;
697 if (qdma_vq->flags & RTE_QDMA_VQ_FD_SG_FORMAT) {
698 /** Make sure there are enough space to get jobs.*/
699 if (unlikely(nb_jobs < DPAA2_QDMA_MAX_SG_NB))
706 if (unlikely(!DPAA2_PER_LCORE_DPIO)) {
707 ret = dpaa2_affine_qbman_swp();
710 "Failed to allocate IO portal, tid: %d\n",
715 swp = DPAA2_PER_LCORE_PORTAL;
717 rxq = &(dpdmai_dev->rx_queue[rxq_id]);
721 dq_storage = rxq->q_storage->dq_storage[0];
722 /* Prepare dequeue descriptor */
723 qbman_pull_desc_clear(&pulldesc);
724 qbman_pull_desc_set_fq(&pulldesc, fqid);
725 qbman_pull_desc_set_storage(&pulldesc, dq_storage,
726 (uint64_t)(DPAA2_VADDR_TO_IOVA(dq_storage)), 1);
728 if (next_pull > dpaa2_dqrr_size) {
729 qbman_pull_desc_set_numframes(&pulldesc,
731 next_pull -= dpaa2_dqrr_size;
733 qbman_pull_desc_set_numframes(&pulldesc, next_pull);
738 if (qbman_swp_pull(swp, &pulldesc)) {
740 "VDQ command not issued. QBMAN busy");
741 /* Portal was busy, try again */
747 rte_prefetch0((void *)((size_t)(dq_storage + 1)));
748 /* Check if the previous issued command is completed. */
749 while (!qbman_check_command_complete(dq_storage))
756 /* Loop until dq_storage is updated
757 * with new token by QBMAN
759 while (!qbman_check_new_result(dq_storage))
761 rte_prefetch0((void *)((size_t)(dq_storage + 2)));
763 if (qbman_result_DQ_is_pull_complete(dq_storage)) {
765 /* Check for valid frame. */
766 status = qbman_result_DQ_flags(dq_storage);
767 if (unlikely((status &
768 QBMAN_DQ_STAT_VALIDFRAME) == 0))
771 fd = qbman_result_DQ_fd(dq_storage);
773 vqid = qdma_vq->get_job(qdma_vq, fd,
774 &job[num_rx], &num_rx_ret);
776 vq_id[num_rx] = vqid;
779 num_rx += num_rx_ret;
783 /* Last VDQ provided all packets and more packets are requested */
784 } while (next_pull && num_pulled == dpaa2_dqrr_size);
790 dpdmai_dev_enqueue_multi(
791 struct qdma_virt_queue *qdma_vq,
792 struct rte_qdma_job **job,
795 struct qdma_hw_queue *qdma_pq = qdma_vq->hw_queue;
796 struct dpaa2_dpdmai_dev *dpdmai_dev = qdma_pq->dpdmai_dev;
797 uint16_t txq_id = qdma_pq->queue_id;
799 struct qbman_fd fd[RTE_QDMA_BURST_NB_MAX];
800 struct dpaa2_queue *txq;
801 struct qbman_eq_desc eqdesc;
802 struct qbman_swp *swp;
804 uint32_t num_to_send = 0;
806 uint32_t enqueue_loop, retry_count, loop;
808 if (unlikely(!DPAA2_PER_LCORE_DPIO)) {
809 ret = dpaa2_affine_qbman_swp();
812 "Failed to allocate IO portal, tid: %d\n",
817 swp = DPAA2_PER_LCORE_PORTAL;
819 txq = &(dpdmai_dev->tx_queue[txq_id]);
821 /* Prepare enqueue descriptor */
822 qbman_eq_desc_clear(&eqdesc);
823 qbman_eq_desc_set_fq(&eqdesc, txq->fqid);
824 qbman_eq_desc_set_no_orp(&eqdesc, 0);
825 qbman_eq_desc_set_response(&eqdesc, 0, 0);
827 if (qdma_vq->flags & RTE_QDMA_VQ_FD_SG_FORMAT) {
829 uint16_t sg_entry_nb = nb_jobs > DPAA2_QDMA_MAX_SG_NB ?
830 DPAA2_QDMA_MAX_SG_NB : nb_jobs;
831 uint16_t job_idx = 0;
832 uint16_t fd_sg_nb[8];
833 uint16_t nb_jobs_ret = 0;
835 if (nb_jobs % DPAA2_QDMA_MAX_SG_NB)
836 fd_nb = nb_jobs / DPAA2_QDMA_MAX_SG_NB + 1;
838 fd_nb = nb_jobs / DPAA2_QDMA_MAX_SG_NB;
840 memset(&fd[0], 0, sizeof(struct qbman_fd) * fd_nb);
842 for (loop = 0; loop < fd_nb; loop++) {
843 ret = qdma_vq->set_fd(qdma_vq, &fd[loop], &job[job_idx],
845 if (unlikely(ret < 0))
847 fd_sg_nb[loop] = sg_entry_nb;
848 nb_jobs -= sg_entry_nb;
849 job_idx += sg_entry_nb;
850 sg_entry_nb = nb_jobs > DPAA2_QDMA_MAX_SG_NB ?
851 DPAA2_QDMA_MAX_SG_NB : nb_jobs;
854 /* Enqueue the packet to the QBMAN */
855 enqueue_loop = 0; retry_count = 0;
857 while (enqueue_loop < fd_nb) {
858 ret = qbman_swp_enqueue_multiple(swp,
859 &eqdesc, &fd[enqueue_loop],
860 NULL, fd_nb - enqueue_loop);
861 if (unlikely(ret < 0)) {
863 if (retry_count > DPAA2_MAX_TX_RETRY_COUNT)
866 for (loop = 0; loop < (uint32_t)ret; loop++)
868 fd_sg_nb[enqueue_loop + loop];
877 memset(fd, 0, nb_jobs * sizeof(struct qbman_fd));
879 while (nb_jobs > 0) {
880 num_to_send = (nb_jobs > dpaa2_eqcr_size) ?
881 dpaa2_eqcr_size : nb_jobs;
883 ret = qdma_vq->set_fd(qdma_vq, &fd[num_tx],
884 &job[num_tx], num_to_send);
885 if (unlikely(ret < 0))
888 /* Enqueue the packet to the QBMAN */
889 enqueue_loop = 0; retry_count = 0;
892 while (enqueue_loop < loop) {
893 ret = qbman_swp_enqueue_multiple(swp,
895 &fd[num_tx + enqueue_loop],
897 loop - enqueue_loop);
898 if (unlikely(ret < 0)) {
900 if (retry_count > DPAA2_MAX_TX_RETRY_COUNT)
907 num_tx += num_to_send;
913 static struct qdma_hw_queue *
914 alloc_hw_queue(uint32_t lcore_id)
916 struct qdma_hw_queue *queue = NULL;
918 DPAA2_QDMA_FUNC_TRACE();
920 /* Get a free queue from the list */
921 TAILQ_FOREACH(queue, &qdma_queue_list, next) {
922 if (queue->num_users == 0) {
923 queue->lcore_id = lcore_id;
933 free_hw_queue(struct qdma_hw_queue *queue)
935 DPAA2_QDMA_FUNC_TRACE();
941 static struct qdma_hw_queue *
942 get_hw_queue(struct qdma_device *qdma_dev, uint32_t lcore_id)
944 struct qdma_per_core_info *core_info;
945 struct qdma_hw_queue *queue, *temp;
946 uint32_t least_num_users;
947 int num_hw_queues, i;
949 DPAA2_QDMA_FUNC_TRACE();
951 core_info = &qdma_core_info[lcore_id];
952 num_hw_queues = core_info->num_hw_queues;
955 * Allocate a HW queue if there are less queues
956 * than maximum per core queues configured
958 if (num_hw_queues < qdma_dev->max_hw_queues_per_core) {
959 queue = alloc_hw_queue(lcore_id);
961 core_info->hw_queues[num_hw_queues] = queue;
962 core_info->num_hw_queues++;
967 queue = core_info->hw_queues[0];
968 /* In case there is no queue associated with the core return NULL */
972 /* Fetch the least loaded H/W queue */
973 least_num_users = core_info->hw_queues[0]->num_users;
974 for (i = 0; i < num_hw_queues; i++) {
975 temp = core_info->hw_queues[i];
976 if (temp->num_users < least_num_users)
987 put_hw_queue(struct qdma_hw_queue *queue)
989 struct qdma_per_core_info *core_info;
990 int lcore_id, num_hw_queues, i;
992 DPAA2_QDMA_FUNC_TRACE();
995 * If this is the last user of the queue free it.
996 * Also remove it from QDMA core info.
998 if (queue->num_users == 1) {
999 free_hw_queue(queue);
1001 /* Remove the physical queue from core info */
1002 lcore_id = queue->lcore_id;
1003 core_info = &qdma_core_info[lcore_id];
1004 num_hw_queues = core_info->num_hw_queues;
1005 for (i = 0; i < num_hw_queues; i++) {
1006 if (queue == core_info->hw_queues[i])
1009 for (; i < num_hw_queues - 1; i++)
1010 core_info->hw_queues[i] = core_info->hw_queues[i + 1];
1011 core_info->hw_queues[i] = NULL;
1018 dpaa2_qdma_attr_get(struct rte_rawdev *rawdev,
1019 __rte_unused const char *attr_name,
1020 uint64_t *attr_value)
1022 struct dpaa2_dpdmai_dev *dpdmai_dev = rawdev->dev_private;
1023 struct qdma_device *qdma_dev = dpdmai_dev->qdma_dev;
1024 struct rte_qdma_attr *qdma_attr = (struct rte_qdma_attr *)attr_value;
1026 DPAA2_QDMA_FUNC_TRACE();
1028 qdma_attr->num_hw_queues = qdma_dev->num_hw_queues;
1034 dpaa2_qdma_reset(struct rte_rawdev *rawdev)
1036 struct qdma_hw_queue *queue;
1037 struct dpaa2_dpdmai_dev *dpdmai_dev = rawdev->dev_private;
1038 struct qdma_device *qdma_dev = dpdmai_dev->qdma_dev;
1041 DPAA2_QDMA_FUNC_TRACE();
1043 /* In case QDMA device is not in stopped state, return -EBUSY */
1044 if (qdma_dev->state == 1) {
1046 "Device is in running state. Stop before reset.");
1050 /* In case there are pending jobs on any VQ, return -EBUSY */
1051 for (i = 0; i < qdma_dev->max_vqs; i++) {
1052 if (qdma_dev->vqs[i].in_use && (qdma_dev->vqs[i].num_enqueues !=
1053 qdma_dev->vqs[i].num_dequeues)) {
1054 DPAA2_QDMA_ERR("Jobs are still pending on VQ: %d", i);
1059 /* Reset HW queues */
1060 TAILQ_FOREACH(queue, &qdma_queue_list, next)
1061 queue->num_users = 0;
1063 /* Reset and free virtual queues */
1064 for (i = 0; i < qdma_dev->max_vqs; i++) {
1065 if (qdma_dev->vqs[i].status_ring)
1066 rte_ring_free(qdma_dev->vqs[i].status_ring);
1069 rte_free(qdma_dev->vqs);
1070 qdma_dev->vqs = NULL;
1072 /* Reset per core info */
1073 memset(&qdma_core_info, 0,
1074 sizeof(struct qdma_per_core_info) * RTE_MAX_LCORE);
1076 /* Reset QDMA device structure */
1077 qdma_dev->max_hw_queues_per_core = 0;
1078 qdma_dev->fle_queue_pool_cnt = 0;
1079 qdma_dev->max_vqs = 0;
1085 dpaa2_qdma_configure(const struct rte_rawdev *rawdev,
1086 rte_rawdev_obj_t config,
1089 char name[32]; /* RTE_MEMZONE_NAMESIZE = 32 */
1090 struct rte_qdma_config *qdma_config = (struct rte_qdma_config *)config;
1091 struct dpaa2_dpdmai_dev *dpdmai_dev = rawdev->dev_private;
1092 struct qdma_device *qdma_dev = dpdmai_dev->qdma_dev;
1094 DPAA2_QDMA_FUNC_TRACE();
1096 if (config_size != sizeof(*qdma_config))
1099 /* In case QDMA device is not in stopped state, return -EBUSY */
1100 if (qdma_dev->state == 1) {
1102 "Device is in running state. Stop before config.");
1106 /* Set max HW queue per core */
1107 if (qdma_config->max_hw_queues_per_core > MAX_HW_QUEUE_PER_CORE) {
1108 DPAA2_QDMA_ERR("H/W queues per core is more than: %d",
1109 MAX_HW_QUEUE_PER_CORE);
1112 qdma_dev->max_hw_queues_per_core =
1113 qdma_config->max_hw_queues_per_core;
1115 /* Allocate Virtual Queues */
1116 sprintf(name, "qdma_%d_vq", rawdev->dev_id);
1117 qdma_dev->vqs = rte_malloc(name,
1118 (sizeof(struct qdma_virt_queue) * qdma_config->max_vqs),
1119 RTE_CACHE_LINE_SIZE);
1120 if (!qdma_dev->vqs) {
1121 DPAA2_QDMA_ERR("qdma_virtual_queues allocation failed");
1124 qdma_dev->max_vqs = qdma_config->max_vqs;
1125 qdma_dev->fle_queue_pool_cnt = qdma_config->fle_queue_pool_cnt;
1131 dpaa2_qdma_start(struct rte_rawdev *rawdev)
1133 struct dpaa2_dpdmai_dev *dpdmai_dev = rawdev->dev_private;
1134 struct qdma_device *qdma_dev = dpdmai_dev->qdma_dev;
1136 DPAA2_QDMA_FUNC_TRACE();
1138 qdma_dev->state = 1;
1144 check_devargs_handler(__rte_unused const char *key, const char *value,
1145 __rte_unused void *opaque)
1147 if (strcmp(value, "1"))
1154 dpaa2_get_devargs(struct rte_devargs *devargs, const char *key)
1156 struct rte_kvargs *kvlist;
1161 kvlist = rte_kvargs_parse(devargs->args, NULL);
1165 if (!rte_kvargs_count(kvlist, key)) {
1166 rte_kvargs_free(kvlist);
1170 if (rte_kvargs_process(kvlist, key,
1171 check_devargs_handler, NULL) < 0) {
1172 rte_kvargs_free(kvlist);
1175 rte_kvargs_free(kvlist);
1181 dpaa2_qdma_queue_setup(struct rte_rawdev *rawdev,
1182 __rte_unused uint16_t queue_id,
1183 rte_rawdev_obj_t queue_conf,
1189 struct dpaa2_dpdmai_dev *dpdmai_dev = rawdev->dev_private;
1190 struct qdma_device *qdma_dev = dpdmai_dev->qdma_dev;
1191 struct rte_qdma_queue_config *q_config =
1192 (struct rte_qdma_queue_config *)queue_conf;
1195 DPAA2_QDMA_FUNC_TRACE();
1197 if (conf_size != sizeof(*q_config))
1200 rte_spinlock_lock(&qdma_dev->lock);
1202 /* Get a free Virtual Queue */
1203 for (i = 0; i < qdma_dev->max_vqs; i++) {
1204 if (qdma_dev->vqs[i].in_use == 0)
1208 /* Return in case no VQ is free */
1209 if (i == qdma_dev->max_vqs) {
1210 rte_spinlock_unlock(&qdma_dev->lock);
1211 DPAA2_QDMA_ERR("Unable to get lock on QDMA device");
1215 if (q_config->flags & RTE_QDMA_VQ_FD_SG_FORMAT) {
1216 if (!(q_config->flags & RTE_QDMA_VQ_EXCLUSIVE_PQ)) {
1218 "qDMA SG format only supports physical queue!");
1219 rte_spinlock_unlock(&qdma_dev->lock);
1222 if (!(q_config->flags & RTE_QDMA_VQ_FD_LONG_FORMAT)) {
1224 "qDMA SG format only supports long FD format!");
1225 rte_spinlock_unlock(&qdma_dev->lock);
1228 pool_size = QDMA_FLE_SG_POOL_SIZE;
1230 pool_size = QDMA_FLE_SINGLE_POOL_SIZE;
1233 if (q_config->flags & RTE_QDMA_VQ_EXCLUSIVE_PQ) {
1234 /* Allocate HW queue for a VQ */
1235 qdma_dev->vqs[i].hw_queue = alloc_hw_queue(q_config->lcore_id);
1236 qdma_dev->vqs[i].exclusive_hw_queue = 1;
1238 /* Allocate a Ring for Virtual Queue in VQ mode */
1239 snprintf(ring_name, sizeof(ring_name), "status ring %d", i);
1240 qdma_dev->vqs[i].status_ring = rte_ring_create(ring_name,
1241 qdma_dev->fle_queue_pool_cnt, rte_socket_id(), 0);
1242 if (!qdma_dev->vqs[i].status_ring) {
1243 DPAA2_QDMA_ERR("Status ring creation failed for vq");
1244 rte_spinlock_unlock(&qdma_dev->lock);
1248 /* Get a HW queue (shared) for a VQ */
1249 qdma_dev->vqs[i].hw_queue = get_hw_queue(qdma_dev,
1250 q_config->lcore_id);
1251 qdma_dev->vqs[i].exclusive_hw_queue = 0;
1254 if (qdma_dev->vqs[i].hw_queue == NULL) {
1255 DPAA2_QDMA_ERR("No H/W queue available for VQ");
1256 if (qdma_dev->vqs[i].status_ring)
1257 rte_ring_free(qdma_dev->vqs[i].status_ring);
1258 qdma_dev->vqs[i].status_ring = NULL;
1259 rte_spinlock_unlock(&qdma_dev->lock);
1263 snprintf(pool_name, sizeof(pool_name),
1264 "qdma_fle_pool%u_queue%d", getpid(), i);
1265 qdma_dev->vqs[i].fle_pool = rte_mempool_create(pool_name,
1266 qdma_dev->fle_queue_pool_cnt, pool_size,
1267 QDMA_FLE_CACHE_SIZE(qdma_dev->fle_queue_pool_cnt), 0,
1268 NULL, NULL, NULL, NULL, SOCKET_ID_ANY, 0);
1269 if (!qdma_dev->vqs[i].fle_pool) {
1270 DPAA2_QDMA_ERR("qdma_fle_pool create failed");
1271 rte_spinlock_unlock(&qdma_dev->lock);
1275 qdma_dev->vqs[i].flags = q_config->flags;
1276 qdma_dev->vqs[i].in_use = 1;
1277 qdma_dev->vqs[i].lcore_id = q_config->lcore_id;
1278 memset(&qdma_dev->vqs[i].rbp, 0, sizeof(struct rte_qdma_rbp));
1280 if (q_config->flags & RTE_QDMA_VQ_FD_LONG_FORMAT) {
1281 if (q_config->flags & RTE_QDMA_VQ_FD_SG_FORMAT) {
1282 qdma_dev->vqs[i].set_fd = dpdmai_dev_set_sg_fd_lf;
1283 qdma_dev->vqs[i].get_job = dpdmai_dev_get_sg_job_lf;
1285 qdma_dev->vqs[i].set_fd = dpdmai_dev_set_multi_fd_lf;
1286 qdma_dev->vqs[i].get_job = dpdmai_dev_get_single_job_lf;
1289 qdma_dev->vqs[i].set_fd = dpdmai_dev_set_fd_us;
1290 qdma_dev->vqs[i].get_job = dpdmai_dev_get_job_us;
1292 if (dpaa2_get_devargs(rawdev->device->devargs,
1293 DPAA2_QDMA_NO_PREFETCH) ||
1294 (getenv("DPAA2_NO_QDMA_PREFETCH_RX"))) {
1295 /* If no prefetch is configured. */
1296 qdma_dev->vqs[i].dequeue_job =
1297 dpdmai_dev_dequeue_multijob_no_prefetch;
1298 DPAA2_QDMA_INFO("No Prefetch RX Mode enabled");
1300 qdma_dev->vqs[i].dequeue_job =
1301 dpdmai_dev_dequeue_multijob_prefetch;
1304 qdma_dev->vqs[i].enqueue_job = dpdmai_dev_enqueue_multi;
1306 if (q_config->rbp != NULL)
1307 memcpy(&qdma_dev->vqs[i].rbp, q_config->rbp,
1308 sizeof(struct rte_qdma_rbp));
1310 rte_spinlock_unlock(&qdma_dev->lock);
1316 dpaa2_qdma_enqueue(struct rte_rawdev *rawdev,
1317 __rte_unused struct rte_rawdev_buf **buffers,
1318 unsigned int nb_jobs,
1319 rte_rawdev_obj_t context)
1321 struct dpaa2_dpdmai_dev *dpdmai_dev = rawdev->dev_private;
1322 struct rte_qdma_enqdeq *e_context =
1323 (struct rte_qdma_enqdeq *)context;
1324 struct qdma_virt_queue *qdma_vq =
1325 &dpdmai_dev->qdma_dev->vqs[e_context->vq_id];
1328 /* Return error in case of wrong lcore_id */
1329 if (rte_lcore_id() != qdma_vq->lcore_id) {
1330 DPAA2_QDMA_ERR("QDMA enqueue for vqid %d on wrong core",
1335 ret = qdma_vq->enqueue_job(qdma_vq, e_context->job, nb_jobs);
1337 DPAA2_QDMA_ERR("DPDMAI device enqueue failed: %d", ret);
1341 qdma_vq->num_enqueues += ret;
1347 dpaa2_qdma_dequeue(struct rte_rawdev *rawdev,
1348 __rte_unused struct rte_rawdev_buf **buffers,
1349 unsigned int nb_jobs,
1350 rte_rawdev_obj_t cntxt)
1352 struct dpaa2_dpdmai_dev *dpdmai_dev = rawdev->dev_private;
1353 struct qdma_device *qdma_dev = dpdmai_dev->qdma_dev;
1354 struct rte_qdma_enqdeq *context =
1355 (struct rte_qdma_enqdeq *)cntxt;
1356 struct qdma_virt_queue *qdma_vq = &qdma_dev->vqs[context->vq_id];
1357 struct qdma_virt_queue *temp_qdma_vq;
1359 unsigned int ring_count;
1361 if (qdma_vq->flags & RTE_QDMA_VQ_FD_SG_FORMAT) {
1362 /** Make sure there are enough space to get jobs.*/
1363 if (unlikely(nb_jobs < DPAA2_QDMA_MAX_SG_NB))
1367 /* Return error in case of wrong lcore_id */
1368 if (rte_lcore_id() != (unsigned int)(qdma_vq->lcore_id)) {
1369 DPAA2_QDMA_WARN("QDMA dequeue for vqid %d on wrong core",
1374 /* Only dequeue when there are pending jobs on VQ */
1375 if (qdma_vq->num_enqueues == qdma_vq->num_dequeues)
1378 if (!(qdma_vq->flags & RTE_QDMA_VQ_FD_SG_FORMAT) &&
1379 qdma_vq->num_enqueues < (qdma_vq->num_dequeues + nb_jobs))
1380 nb_jobs = (qdma_vq->num_enqueues - qdma_vq->num_dequeues);
1382 if (qdma_vq->exclusive_hw_queue) {
1383 /* In case of exclusive queue directly fetch from HW queue */
1384 ret = qdma_vq->dequeue_job(qdma_vq, NULL,
1385 context->job, nb_jobs);
1388 "Dequeue from DPDMAI device failed: %d", ret);
1391 qdma_vq->num_dequeues += ret;
1393 uint16_t temp_vq_id[RTE_QDMA_BURST_NB_MAX];
1395 * Get the QDMA completed jobs from the software ring.
1396 * In case they are not available on the ring poke the HW
1397 * to fetch completed jobs from corresponding HW queues
1399 ring_count = rte_ring_count(qdma_vq->status_ring);
1400 if (ring_count < nb_jobs) {
1401 /* TODO - How to have right budget */
1402 ret = qdma_vq->dequeue_job(qdma_vq,
1403 temp_vq_id, context->job, nb_jobs);
1404 for (i = 0; i < ret; i++) {
1405 temp_qdma_vq = &qdma_dev->vqs[temp_vq_id[i]];
1406 rte_ring_enqueue(temp_qdma_vq->status_ring,
1407 (void *)(context->job[i]));
1409 ring_count = rte_ring_count(
1410 qdma_vq->status_ring);
1414 /* Dequeue job from the software ring
1415 * to provide to the user
1417 ret = rte_ring_dequeue_bulk(qdma_vq->status_ring,
1418 (void **)context->job,
1421 qdma_vq->num_dequeues += ret;
1429 rte_qdma_vq_stats(struct rte_rawdev *rawdev,
1431 struct rte_qdma_vq_stats *vq_status)
1433 struct dpaa2_dpdmai_dev *dpdmai_dev = rawdev->dev_private;
1434 struct qdma_device *qdma_dev = dpdmai_dev->qdma_dev;
1435 struct qdma_virt_queue *qdma_vq = &qdma_dev->vqs[vq_id];
1437 if (qdma_vq->in_use) {
1438 vq_status->exclusive_hw_queue = qdma_vq->exclusive_hw_queue;
1439 vq_status->lcore_id = qdma_vq->lcore_id;
1440 vq_status->num_enqueues = qdma_vq->num_enqueues;
1441 vq_status->num_dequeues = qdma_vq->num_dequeues;
1442 vq_status->num_pending_jobs = vq_status->num_enqueues -
1443 vq_status->num_dequeues;
1448 dpaa2_qdma_queue_release(struct rte_rawdev *rawdev,
1451 struct dpaa2_dpdmai_dev *dpdmai_dev = rawdev->dev_private;
1452 struct qdma_device *qdma_dev = dpdmai_dev->qdma_dev;
1454 struct qdma_virt_queue *qdma_vq = &qdma_dev->vqs[vq_id];
1456 DPAA2_QDMA_FUNC_TRACE();
1458 /* In case there are pending jobs on any VQ, return -EBUSY */
1459 if (qdma_vq->num_enqueues != qdma_vq->num_dequeues)
1462 rte_spinlock_lock(&qdma_dev->lock);
1464 if (qdma_vq->exclusive_hw_queue)
1465 free_hw_queue(qdma_vq->hw_queue);
1467 if (qdma_vq->status_ring)
1468 rte_ring_free(qdma_vq->status_ring);
1470 put_hw_queue(qdma_vq->hw_queue);
1473 if (qdma_vq->fle_pool)
1474 rte_mempool_free(qdma_vq->fle_pool);
1476 memset(qdma_vq, 0, sizeof(struct qdma_virt_queue));
1478 rte_spinlock_unlock(&qdma_dev->lock);
1484 dpaa2_qdma_stop(struct rte_rawdev *rawdev)
1486 struct dpaa2_dpdmai_dev *dpdmai_dev = rawdev->dev_private;
1487 struct qdma_device *qdma_dev = dpdmai_dev->qdma_dev;
1489 DPAA2_QDMA_FUNC_TRACE();
1491 qdma_dev->state = 0;
1495 dpaa2_qdma_close(struct rte_rawdev *rawdev)
1497 DPAA2_QDMA_FUNC_TRACE();
1499 dpaa2_qdma_reset(rawdev);
1504 static struct rte_rawdev_ops dpaa2_qdma_ops = {
1505 .dev_configure = dpaa2_qdma_configure,
1506 .dev_start = dpaa2_qdma_start,
1507 .dev_stop = dpaa2_qdma_stop,
1508 .dev_reset = dpaa2_qdma_reset,
1509 .dev_close = dpaa2_qdma_close,
1510 .queue_setup = dpaa2_qdma_queue_setup,
1511 .queue_release = dpaa2_qdma_queue_release,
1512 .attr_get = dpaa2_qdma_attr_get,
1513 .enqueue_bufs = dpaa2_qdma_enqueue,
1514 .dequeue_bufs = dpaa2_qdma_dequeue,
1518 add_hw_queues_to_list(struct dpaa2_dpdmai_dev *dpdmai_dev)
1520 struct qdma_hw_queue *queue;
1523 DPAA2_QDMA_FUNC_TRACE();
1525 for (i = 0; i < dpdmai_dev->num_queues; i++) {
1526 queue = rte_zmalloc(NULL, sizeof(struct qdma_hw_queue), 0);
1529 "Memory allocation failed for QDMA queue");
1533 queue->dpdmai_dev = dpdmai_dev;
1534 queue->queue_id = i;
1536 TAILQ_INSERT_TAIL(&qdma_queue_list, queue, next);
1537 dpdmai_dev->qdma_dev->num_hw_queues++;
1544 remove_hw_queues_from_list(struct dpaa2_dpdmai_dev *dpdmai_dev)
1546 struct qdma_hw_queue *queue = NULL;
1547 struct qdma_hw_queue *tqueue = NULL;
1549 DPAA2_QDMA_FUNC_TRACE();
1551 TAILQ_FOREACH_SAFE(queue, &qdma_queue_list, next, tqueue) {
1552 if (queue->dpdmai_dev == dpdmai_dev) {
1553 TAILQ_REMOVE(&qdma_queue_list, queue, next);
1561 dpaa2_dpdmai_dev_uninit(struct rte_rawdev *rawdev)
1563 struct dpaa2_dpdmai_dev *dpdmai_dev = rawdev->dev_private;
1566 DPAA2_QDMA_FUNC_TRACE();
1568 /* Remove HW queues from global list */
1569 remove_hw_queues_from_list(dpdmai_dev);
1571 ret = dpdmai_disable(&dpdmai_dev->dpdmai, CMD_PRI_LOW,
1574 DPAA2_QDMA_ERR("dmdmai disable failed");
1576 /* Set up the DQRR storage for Rx */
1577 for (i = 0; i < dpdmai_dev->num_queues; i++) {
1578 struct dpaa2_queue *rxq = &(dpdmai_dev->rx_queue[i]);
1580 if (rxq->q_storage) {
1581 dpaa2_free_dq_storage(rxq->q_storage);
1582 rte_free(rxq->q_storage);
1586 /* Close the device at underlying layer*/
1587 ret = dpdmai_close(&dpdmai_dev->dpdmai, CMD_PRI_LOW, dpdmai_dev->token);
1589 DPAA2_QDMA_ERR("Failure closing dpdmai device");
1595 dpaa2_dpdmai_dev_init(struct rte_rawdev *rawdev, int dpdmai_id)
1597 struct dpaa2_dpdmai_dev *dpdmai_dev = rawdev->dev_private;
1598 struct dpdmai_rx_queue_cfg rx_queue_cfg;
1599 struct dpdmai_attr attr;
1600 struct dpdmai_rx_queue_attr rx_attr;
1601 struct dpdmai_tx_queue_attr tx_attr;
1604 DPAA2_QDMA_FUNC_TRACE();
1606 /* Open DPDMAI device */
1607 dpdmai_dev->dpdmai_id = dpdmai_id;
1608 dpdmai_dev->dpdmai.regs = dpaa2_get_mcp_ptr(MC_PORTAL_INDEX);
1609 dpdmai_dev->qdma_dev = &q_dev;
1610 ret = dpdmai_open(&dpdmai_dev->dpdmai, CMD_PRI_LOW,
1611 dpdmai_dev->dpdmai_id, &dpdmai_dev->token);
1613 DPAA2_QDMA_ERR("dpdmai_open() failed with err: %d", ret);
1617 /* Get DPDMAI attributes */
1618 ret = dpdmai_get_attributes(&dpdmai_dev->dpdmai, CMD_PRI_LOW,
1619 dpdmai_dev->token, &attr);
1621 DPAA2_QDMA_ERR("dpdmai get attributes failed with err: %d",
1625 dpdmai_dev->num_queues = attr.num_of_queues;
1627 /* Set up Rx Queues */
1628 for (i = 0; i < dpdmai_dev->num_queues; i++) {
1629 struct dpaa2_queue *rxq;
1631 memset(&rx_queue_cfg, 0, sizeof(struct dpdmai_rx_queue_cfg));
1632 ret = dpdmai_set_rx_queue(&dpdmai_dev->dpdmai,
1635 i, 0, &rx_queue_cfg);
1637 DPAA2_QDMA_ERR("Setting Rx queue failed with err: %d",
1642 /* Allocate DQ storage for the DPDMAI Rx queues */
1643 rxq = &(dpdmai_dev->rx_queue[i]);
1644 rxq->q_storage = rte_malloc("dq_storage",
1645 sizeof(struct queue_storage_info_t),
1646 RTE_CACHE_LINE_SIZE);
1647 if (!rxq->q_storage) {
1648 DPAA2_QDMA_ERR("q_storage allocation failed");
1653 memset(rxq->q_storage, 0, sizeof(struct queue_storage_info_t));
1654 ret = dpaa2_alloc_dq_storage(rxq->q_storage);
1656 DPAA2_QDMA_ERR("dpaa2_alloc_dq_storage failed");
1661 /* Get Rx and Tx queues FQID's */
1662 for (i = 0; i < dpdmai_dev->num_queues; i++) {
1663 ret = dpdmai_get_rx_queue(&dpdmai_dev->dpdmai, CMD_PRI_LOW,
1664 dpdmai_dev->token, i, 0, &rx_attr);
1666 DPAA2_QDMA_ERR("Reading device failed with err: %d",
1670 dpdmai_dev->rx_queue[i].fqid = rx_attr.fqid;
1672 ret = dpdmai_get_tx_queue(&dpdmai_dev->dpdmai, CMD_PRI_LOW,
1673 dpdmai_dev->token, i, 0, &tx_attr);
1675 DPAA2_QDMA_ERR("Reading device failed with err: %d",
1679 dpdmai_dev->tx_queue[i].fqid = tx_attr.fqid;
1682 /* Enable the device */
1683 ret = dpdmai_enable(&dpdmai_dev->dpdmai, CMD_PRI_LOW,
1686 DPAA2_QDMA_ERR("Enabling device failed with err: %d", ret);
1690 /* Add the HW queue to the global list */
1691 ret = add_hw_queues_to_list(dpdmai_dev);
1693 DPAA2_QDMA_ERR("Adding H/W queue to list failed");
1697 if (!dpaa2_coherent_no_alloc_cache) {
1698 if (dpaa2_svr_family == SVR_LX2160A) {
1699 dpaa2_coherent_no_alloc_cache =
1700 DPAA2_LX2_COHERENT_NO_ALLOCATE_CACHE;
1701 dpaa2_coherent_alloc_cache =
1702 DPAA2_LX2_COHERENT_ALLOCATE_CACHE;
1704 dpaa2_coherent_no_alloc_cache =
1705 DPAA2_COHERENT_NO_ALLOCATE_CACHE;
1706 dpaa2_coherent_alloc_cache =
1707 DPAA2_COHERENT_ALLOCATE_CACHE;
1711 DPAA2_QDMA_DEBUG("Initialized dpdmai object successfully");
1713 rte_spinlock_init(&dpdmai_dev->qdma_dev->lock);
1717 dpaa2_dpdmai_dev_uninit(rawdev);
1722 rte_dpaa2_qdma_probe(struct rte_dpaa2_driver *dpaa2_drv,
1723 struct rte_dpaa2_device *dpaa2_dev)
1725 struct rte_rawdev *rawdev;
1728 DPAA2_QDMA_FUNC_TRACE();
1730 rawdev = rte_rawdev_pmd_allocate(dpaa2_dev->device.name,
1731 sizeof(struct dpaa2_dpdmai_dev),
1734 DPAA2_QDMA_ERR("Unable to allocate rawdevice");
1738 dpaa2_dev->rawdev = rawdev;
1739 rawdev->dev_ops = &dpaa2_qdma_ops;
1740 rawdev->device = &dpaa2_dev->device;
1741 rawdev->driver_name = dpaa2_drv->driver.name;
1743 /* Invoke PMD device initialization function */
1744 ret = dpaa2_dpdmai_dev_init(rawdev, dpaa2_dev->object_id);
1746 rte_rawdev_pmd_release(rawdev);
1750 /* Reset the QDMA device */
1751 ret = dpaa2_qdma_reset(rawdev);
1753 DPAA2_QDMA_ERR("Resetting QDMA failed");
1761 rte_dpaa2_qdma_remove(struct rte_dpaa2_device *dpaa2_dev)
1763 struct rte_rawdev *rawdev = dpaa2_dev->rawdev;
1766 DPAA2_QDMA_FUNC_TRACE();
1768 dpaa2_dpdmai_dev_uninit(rawdev);
1770 ret = rte_rawdev_pmd_release(rawdev);
1772 DPAA2_QDMA_ERR("Device cleanup failed");
1777 static struct rte_dpaa2_driver rte_dpaa2_qdma_pmd = {
1778 .drv_flags = RTE_DPAA2_DRV_IOVA_AS_VA,
1779 .drv_type = DPAA2_QDMA,
1780 .probe = rte_dpaa2_qdma_probe,
1781 .remove = rte_dpaa2_qdma_remove,
1784 RTE_PMD_REGISTER_DPAA2(dpaa2_qdma, rte_dpaa2_qdma_pmd);
1785 RTE_PMD_REGISTER_PARAM_STRING(dpaa2_qdma,
1786 "no_prefetch=<int> ");
1787 RTE_LOG_REGISTER(dpaa2_qdma_logtype, pmd.raw.dpaa2.qdma, INFO);