2 * This file is provided under a dual BSD/GPLv2 license. When using or
3 * redistributing this file, you may do so under either license.
7 * Copyright 2008-2016 Freescale Semiconductor Inc.
10 * Redistribution and use in source and binary forms, with or without
11 * modification, are permitted provided that the following conditions are met:
12 * * Redistributions of source code must retain the above copyright
13 * notice, this list of conditions and the following disclaimer.
14 * * Redistributions in binary form must reproduce the above copyright
15 * notice, this list of conditions and the following disclaimer in the
16 * documentation and/or other materials provided with the distribution.
17 * * Neither the name of the above-listed copyright holders nor the
18 * names of any contributors may be used to endorse or promote products
19 * derived from this software without specific prior written permission.
23 * ALTERNATIVELY, this software may be distributed under the terms of the
24 * GNU General Public License ("GPL") as published by the Free Software
25 * Foundation, either version 2 of that License or (at your option) any
28 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
29 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
30 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
31 * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDERS OR CONTRIBUTORS BE
32 * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
33 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
34 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
35 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
36 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
37 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
38 * POSSIBILITY OF SUCH DAMAGE.
41 #include "qman_priv.h"
43 /***************************/
44 /* Portal register assists */
45 /***************************/
46 #define QM_REG_EQCR_PI_CINH 0x3000
47 #define QM_REG_EQCR_CI_CINH 0x3040
48 #define QM_REG_EQCR_ITR 0x3080
49 #define QM_REG_DQRR_PI_CINH 0x3100
50 #define QM_REG_DQRR_CI_CINH 0x3140
51 #define QM_REG_DQRR_ITR 0x3180
52 #define QM_REG_DQRR_DCAP 0x31C0
53 #define QM_REG_DQRR_SDQCR 0x3200
54 #define QM_REG_DQRR_VDQCR 0x3240
55 #define QM_REG_DQRR_PDQCR 0x3280
56 #define QM_REG_MR_PI_CINH 0x3300
57 #define QM_REG_MR_CI_CINH 0x3340
58 #define QM_REG_MR_ITR 0x3380
59 #define QM_REG_CFG 0x3500
60 #define QM_REG_ISR 0x3600
61 #define QM_REG_IIR 0x36C0
62 #define QM_REG_ITPR 0x3740
64 /* Cache-enabled register offsets */
65 #define QM_CL_EQCR 0x0000
66 #define QM_CL_DQRR 0x1000
67 #define QM_CL_MR 0x2000
68 #define QM_CL_EQCR_PI_CENA 0x3000
69 #define QM_CL_EQCR_CI_CENA 0x3040
70 #define QM_CL_DQRR_PI_CENA 0x3100
71 #define QM_CL_DQRR_CI_CENA 0x3140
72 #define QM_CL_MR_PI_CENA 0x3300
73 #define QM_CL_MR_CI_CENA 0x3340
74 #define QM_CL_CR 0x3800
75 #define QM_CL_RR0 0x3900
76 #define QM_CL_RR1 0x3940
78 /* BTW, the drivers (and h/w programming model) already obtain the required
79 * synchronisation for portal accesses via lwsync(), hwsync(), and
80 * data-dependencies. Use of barrier()s or other order-preserving primitives
81 * simply degrade performance. Hence the use of the __raw_*() interfaces, which
82 * simply ensure that the compiler treats the portal registers as volatile (ie.
86 /* Cache-inhibited register access. */
87 #define __qm_in(qm, o) be32_to_cpu(__raw_readl((qm)->ci + (o)))
88 #define __qm_out(qm, o, val) __raw_writel((cpu_to_be32(val)), \
90 #define qm_in(reg) __qm_in(&portal->addr, QM_REG_##reg)
91 #define qm_out(reg, val) __qm_out(&portal->addr, QM_REG_##reg, val)
93 /* Cache-enabled (index) register access */
94 #define __qm_cl_touch_ro(qm, o) dcbt_ro((qm)->ce + (o))
95 #define __qm_cl_touch_rw(qm, o) dcbt_rw((qm)->ce + (o))
96 #define __qm_cl_in(qm, o) be32_to_cpu(__raw_readl((qm)->ce + (o)))
97 #define __qm_cl_out(qm, o, val) \
99 u32 *__tmpclout = (qm)->ce + (o); \
100 __raw_writel(cpu_to_be32(val), __tmpclout); \
103 #define __qm_cl_invalidate(qm, o) dccivac((qm)->ce + (o))
104 #define qm_cl_touch_ro(reg) __qm_cl_touch_ro(&portal->addr, QM_CL_##reg##_CENA)
105 #define qm_cl_touch_rw(reg) __qm_cl_touch_rw(&portal->addr, QM_CL_##reg##_CENA)
106 #define qm_cl_in(reg) __qm_cl_in(&portal->addr, QM_CL_##reg##_CENA)
107 #define qm_cl_out(reg, val) __qm_cl_out(&portal->addr, QM_CL_##reg##_CENA, val)
108 #define qm_cl_invalidate(reg)\
109 __qm_cl_invalidate(&portal->addr, QM_CL_##reg##_CENA)
111 /* Cache-enabled ring access */
112 #define qm_cl(base, idx) ((void *)base + ((idx) << 6))
114 /* Cyclic helper for rings. FIXME: once we are able to do fine-grain perf
115 * analysis, look at using the "extra" bit in the ring index registers to avoid
118 static inline u8 qm_cyc_diff(u8 ringsize, u8 first, u8 last)
120 /* 'first' is included, 'last' is excluded */
123 return ringsize + last - first;
128 * pmode == production mode
129 * cmode == consumption mode,
130 * dmode == h/w dequeue mode.
131 * Enum values use 3 letter codes. First letter matches the portal mode,
132 * remaining two letters indicate;
133 * ci == cache-inhibited portal register
134 * ce == cache-enabled portal register
135 * vb == in-band valid-bit (cache-enabled)
136 * dc == DCA (Discrete Consumption Acknowledgment), DQRR-only
137 * As for "enum qm_dqrr_dmode", it should be self-explanatory.
139 enum qm_eqcr_pmode { /* matches QCSP_CFG::EPM */
140 qm_eqcr_pci = 0, /* PI index, cache-inhibited */
141 qm_eqcr_pce = 1, /* PI index, cache-enabled */
142 qm_eqcr_pvb = 2 /* valid-bit */
145 enum qm_dqrr_dmode { /* matches QCSP_CFG::DP */
146 qm_dqrr_dpush = 0, /* SDQCR + VDQCR */
147 qm_dqrr_dpull = 1 /* PDQCR */
150 enum qm_dqrr_pmode { /* s/w-only */
151 qm_dqrr_pci, /* reads DQRR_PI_CINH */
152 qm_dqrr_pce, /* reads DQRR_PI_CENA */
153 qm_dqrr_pvb /* reads valid-bit */
156 enum qm_dqrr_cmode { /* matches QCSP_CFG::DCM */
157 qm_dqrr_cci = 0, /* CI index, cache-inhibited */
158 qm_dqrr_cce = 1, /* CI index, cache-enabled */
159 qm_dqrr_cdc = 2 /* Discrete Consumption Acknowledgment */
162 enum qm_mr_pmode { /* s/w-only */
163 qm_mr_pci, /* reads MR_PI_CINH */
164 qm_mr_pce, /* reads MR_PI_CENA */
165 qm_mr_pvb /* reads valid-bit */
168 enum qm_mr_cmode { /* matches QCSP_CFG::MM */
169 qm_mr_cci = 0, /* CI index, cache-inhibited */
170 qm_mr_cce = 1 /* CI index, cache-enabled */
173 /* ------------------------- */
174 /* --- Portal structures --- */
176 #define QM_EQCR_SIZE 8
177 #define QM_DQRR_SIZE 16
181 struct qm_eqcr_entry *ring, *cursor;
182 u8 ci, available, ithresh, vbit;
183 #ifdef RTE_LIBRTE_DPAA_HWDEBUG
185 enum qm_eqcr_pmode pmode;
190 const struct qm_dqrr_entry *ring, *cursor;
191 u8 pi, ci, fill, ithresh, vbit;
192 #ifdef RTE_LIBRTE_DPAA_HWDEBUG
193 enum qm_dqrr_dmode dmode;
194 enum qm_dqrr_pmode pmode;
195 enum qm_dqrr_cmode cmode;
200 const struct qm_mr_entry *ring, *cursor;
201 u8 pi, ci, fill, ithresh, vbit;
202 #ifdef RTE_LIBRTE_DPAA_HWDEBUG
203 enum qm_mr_pmode pmode;
204 enum qm_mr_cmode cmode;
209 struct qm_mc_command *cr;
210 struct qm_mc_result *rr;
212 #ifdef RTE_LIBRTE_DPAA_HWDEBUG
214 /* Can be _mc_start()ed */
216 /* Can be _mc_commit()ed or _mc_abort()ed */
218 /* Can only be _mc_retry()ed */
224 #define QM_PORTAL_ALIGNMENT ____cacheline_aligned
227 void __iomem *ce; /* cache-enabled */
228 void __iomem *ci; /* cache-inhibited */
237 } QM_PORTAL_ALIGNMENT;
239 /* Bit-wise logic to wrap a ring pointer by clearing the "carry bit" */
240 #define EQCR_CARRYCLEAR(p) \
241 (void *)((unsigned long)(p) & (~(unsigned long)(QM_EQCR_SIZE << 6)))
243 extern dma_addr_t rte_mem_virt2iova(const void *addr);
245 /* Bit-wise logic to convert a ring pointer to a ring index */
246 static inline u8 EQCR_PTR2IDX(struct qm_eqcr_entry *e)
248 return ((uintptr_t)e >> 6) & (QM_EQCR_SIZE - 1);
251 /* Increment the 'cursor' ring pointer, taking 'vbit' into account */
252 static inline void EQCR_INC(struct qm_eqcr *eqcr)
254 /* NB: this is odd-looking, but experiments show that it generates fast
255 * code with essentially no branching overheads. We increment to the
256 * next EQCR pointer and handle overflow and 'vbit'.
258 struct qm_eqcr_entry *partial = eqcr->cursor + 1;
260 eqcr->cursor = EQCR_CARRYCLEAR(partial);
261 if (partial != eqcr->cursor)
262 eqcr->vbit ^= QM_EQCR_VERB_VBIT;
265 static inline struct qm_eqcr_entry *qm_eqcr_start_no_stash(struct qm_portal
268 register struct qm_eqcr *eqcr = &portal->eqcr;
270 #ifdef RTE_LIBRTE_DPAA_HWDEBUG
271 DPAA_ASSERT(!eqcr->busy);
273 if (!eqcr->available)
276 #ifdef RTE_LIBRTE_DPAA_HWDEBUG
283 static inline struct qm_eqcr_entry *qm_eqcr_start_stash(struct qm_portal
286 register struct qm_eqcr *eqcr = &portal->eqcr;
289 #ifdef RTE_LIBRTE_DPAA_HWDEBUG
290 DPAA_ASSERT(!eqcr->busy);
292 if (!eqcr->available) {
294 eqcr->ci = qm_cl_in(EQCR_CI) & (QM_EQCR_SIZE - 1);
295 diff = qm_cyc_diff(QM_EQCR_SIZE, old_ci, eqcr->ci);
296 eqcr->available += diff;
300 #ifdef RTE_LIBRTE_DPAA_HWDEBUG
306 static inline void qm_eqcr_abort(struct qm_portal *portal)
308 __maybe_unused register struct qm_eqcr *eqcr = &portal->eqcr;
310 #ifdef RTE_LIBRTE_DPAA_HWDEBUG
311 DPAA_ASSERT(eqcr->busy);
316 static inline struct qm_eqcr_entry *qm_eqcr_pend_and_next(
317 struct qm_portal *portal, u8 myverb)
319 register struct qm_eqcr *eqcr = &portal->eqcr;
321 #ifdef RTE_LIBRTE_DPAA_HWDEBUG
322 DPAA_ASSERT(eqcr->busy);
323 DPAA_ASSERT(eqcr->pmode != qm_eqcr_pvb);
325 if (eqcr->available == 1)
327 eqcr->cursor->__dont_write_directly__verb = myverb | eqcr->vbit;
334 #define EQCR_COMMIT_CHECKS(eqcr) \
336 DPAA_ASSERT(eqcr->busy); \
337 DPAA_ASSERT(eqcr->cursor->orp == (eqcr->cursor->orp & 0x00ffffff)); \
338 DPAA_ASSERT(eqcr->cursor->fqid == (eqcr->cursor->fqid & 0x00ffffff)); \
341 static inline void qm_eqcr_pci_commit(struct qm_portal *portal, u8 myverb)
343 register struct qm_eqcr *eqcr = &portal->eqcr;
345 #ifdef RTE_LIBRTE_DPAA_HWDEBUG
346 EQCR_COMMIT_CHECKS(eqcr);
347 DPAA_ASSERT(eqcr->pmode == qm_eqcr_pci);
349 eqcr->cursor->__dont_write_directly__verb = myverb | eqcr->vbit;
354 qm_out(EQCR_PI_CINH, EQCR_PTR2IDX(eqcr->cursor));
355 #ifdef RTE_LIBRTE_DPAA_HWDEBUG
360 static inline void qm_eqcr_pce_prefetch(struct qm_portal *portal)
362 __maybe_unused register struct qm_eqcr *eqcr = &portal->eqcr;
364 #ifdef RTE_LIBRTE_DPAA_HWDEBUG
365 DPAA_ASSERT(eqcr->pmode == qm_eqcr_pce);
367 qm_cl_invalidate(EQCR_PI);
368 qm_cl_touch_rw(EQCR_PI);
371 static inline void qm_eqcr_pce_commit(struct qm_portal *portal, u8 myverb)
373 register struct qm_eqcr *eqcr = &portal->eqcr;
375 #ifdef RTE_LIBRTE_DPAA_HWDEBUG
376 EQCR_COMMIT_CHECKS(eqcr);
377 DPAA_ASSERT(eqcr->pmode == qm_eqcr_pce);
379 eqcr->cursor->__dont_write_directly__verb = myverb | eqcr->vbit;
384 qm_cl_out(EQCR_PI, EQCR_PTR2IDX(eqcr->cursor));
385 #ifdef RTE_LIBRTE_DPAA_HWDEBUG
390 static inline void qm_eqcr_pvb_commit(struct qm_portal *portal, u8 myverb)
392 register struct qm_eqcr *eqcr = &portal->eqcr;
393 struct qm_eqcr_entry *eqcursor;
395 #ifdef RTE_LIBRTE_DPAA_HWDEBUG
396 EQCR_COMMIT_CHECKS(eqcr);
397 DPAA_ASSERT(eqcr->pmode == qm_eqcr_pvb);
400 eqcursor = eqcr->cursor;
401 eqcursor->__dont_write_directly__verb = myverb | eqcr->vbit;
405 #ifdef RTE_LIBRTE_DPAA_HWDEBUG
410 static inline u8 qm_eqcr_cci_update(struct qm_portal *portal)
412 register struct qm_eqcr *eqcr = &portal->eqcr;
413 u8 diff, old_ci = eqcr->ci;
415 eqcr->ci = qm_in(EQCR_CI_CINH) & (QM_EQCR_SIZE - 1);
416 diff = qm_cyc_diff(QM_EQCR_SIZE, old_ci, eqcr->ci);
417 eqcr->available += diff;
421 static inline void qm_eqcr_cce_prefetch(struct qm_portal *portal)
423 __maybe_unused register struct qm_eqcr *eqcr = &portal->eqcr;
425 qm_cl_touch_ro(EQCR_CI);
428 static inline u8 qm_eqcr_cce_update(struct qm_portal *portal)
430 register struct qm_eqcr *eqcr = &portal->eqcr;
431 u8 diff, old_ci = eqcr->ci;
433 eqcr->ci = qm_cl_in(EQCR_CI) & (QM_EQCR_SIZE - 1);
434 qm_cl_invalidate(EQCR_CI);
435 diff = qm_cyc_diff(QM_EQCR_SIZE, old_ci, eqcr->ci);
436 eqcr->available += diff;
440 static inline u8 qm_eqcr_get_ithresh(struct qm_portal *portal)
442 register struct qm_eqcr *eqcr = &portal->eqcr;
444 return eqcr->ithresh;
447 static inline void qm_eqcr_set_ithresh(struct qm_portal *portal, u8 ithresh)
449 register struct qm_eqcr *eqcr = &portal->eqcr;
451 eqcr->ithresh = ithresh;
452 qm_out(EQCR_ITR, ithresh);
455 static inline u8 qm_eqcr_get_avail(struct qm_portal *portal)
457 register struct qm_eqcr *eqcr = &portal->eqcr;
459 return eqcr->available;
462 static inline u8 qm_eqcr_get_fill(struct qm_portal *portal)
464 register struct qm_eqcr *eqcr = &portal->eqcr;
466 return QM_EQCR_SIZE - 1 - eqcr->available;
469 #define DQRR_CARRYCLEAR(p) \
470 (void *)((unsigned long)(p) & (~(unsigned long)(QM_DQRR_SIZE << 6)))
472 static inline u8 DQRR_PTR2IDX(const struct qm_dqrr_entry *e)
474 return ((uintptr_t)e >> 6) & (QM_DQRR_SIZE - 1);
477 static inline const struct qm_dqrr_entry *DQRR_INC(
478 const struct qm_dqrr_entry *e)
480 return DQRR_CARRYCLEAR(e + 1);
483 static inline void qm_dqrr_set_maxfill(struct qm_portal *portal, u8 mf)
485 qm_out(CFG, (qm_in(CFG) & 0xff0fffff) |
486 ((mf & (QM_DQRR_SIZE - 1)) << 20));
489 static inline const struct qm_dqrr_entry *qm_dqrr_current(
490 struct qm_portal *portal)
492 register struct qm_dqrr *dqrr = &portal->dqrr;
499 static inline u8 qm_dqrr_cursor(struct qm_portal *portal)
501 register struct qm_dqrr *dqrr = &portal->dqrr;
503 return DQRR_PTR2IDX(dqrr->cursor);
506 static inline u8 qm_dqrr_next(struct qm_portal *portal)
508 register struct qm_dqrr *dqrr = &portal->dqrr;
510 DPAA_ASSERT(dqrr->fill);
511 dqrr->cursor = DQRR_INC(dqrr->cursor);
515 static inline u8 qm_dqrr_pci_update(struct qm_portal *portal)
517 register struct qm_dqrr *dqrr = &portal->dqrr;
518 u8 diff, old_pi = dqrr->pi;
520 #ifdef RTE_LIBRTE_DPAA_HWDEBUG
521 DPAA_ASSERT(dqrr->pmode == qm_dqrr_pci);
523 dqrr->pi = qm_in(DQRR_PI_CINH) & (QM_DQRR_SIZE - 1);
524 diff = qm_cyc_diff(QM_DQRR_SIZE, old_pi, dqrr->pi);
529 static inline void qm_dqrr_pce_prefetch(struct qm_portal *portal)
531 __maybe_unused register struct qm_dqrr *dqrr = &portal->dqrr;
533 #ifdef RTE_LIBRTE_DPAA_HWDEBUG
534 DPAA_ASSERT(dqrr->pmode == qm_dqrr_pce);
536 qm_cl_invalidate(DQRR_PI);
537 qm_cl_touch_ro(DQRR_PI);
540 static inline u8 qm_dqrr_pce_update(struct qm_portal *portal)
542 register struct qm_dqrr *dqrr = &portal->dqrr;
543 u8 diff, old_pi = dqrr->pi;
545 #ifdef RTE_LIBRTE_DPAA_HWDEBUG
546 DPAA_ASSERT(dqrr->pmode == qm_dqrr_pce);
548 dqrr->pi = qm_cl_in(DQRR_PI) & (QM_DQRR_SIZE - 1);
549 diff = qm_cyc_diff(QM_DQRR_SIZE, old_pi, dqrr->pi);
554 static inline void qm_dqrr_pvb_update(struct qm_portal *portal)
556 register struct qm_dqrr *dqrr = &portal->dqrr;
557 const struct qm_dqrr_entry *res = qm_cl(dqrr->ring, dqrr->pi);
559 #ifdef RTE_LIBRTE_DPAA_HWDEBUG
560 DPAA_ASSERT(dqrr->pmode == qm_dqrr_pvb);
562 /* when accessing 'verb', use __raw_readb() to ensure that compiler
563 * inlining doesn't try to optimise out "excess reads".
565 if ((__raw_readb(&res->verb) & QM_DQRR_VERB_VBIT) == dqrr->vbit) {
566 dqrr->pi = (dqrr->pi + 1) & (QM_DQRR_SIZE - 1);
568 dqrr->vbit ^= QM_DQRR_VERB_VBIT;
573 static inline void qm_dqrr_cci_consume(struct qm_portal *portal, u8 num)
575 register struct qm_dqrr *dqrr = &portal->dqrr;
577 #ifdef RTE_LIBRTE_DPAA_HWDEBUG
578 DPAA_ASSERT(dqrr->cmode == qm_dqrr_cci);
580 dqrr->ci = (dqrr->ci + num) & (QM_DQRR_SIZE - 1);
581 qm_out(DQRR_CI_CINH, dqrr->ci);
584 static inline void qm_dqrr_cci_consume_to_current(struct qm_portal *portal)
586 register struct qm_dqrr *dqrr = &portal->dqrr;
588 #ifdef RTE_LIBRTE_DPAA_HWDEBUG
589 DPAA_ASSERT(dqrr->cmode == qm_dqrr_cci);
591 dqrr->ci = DQRR_PTR2IDX(dqrr->cursor);
592 qm_out(DQRR_CI_CINH, dqrr->ci);
595 static inline void qm_dqrr_cce_prefetch(struct qm_portal *portal)
597 __maybe_unused register struct qm_dqrr *dqrr = &portal->dqrr;
599 #ifdef RTE_LIBRTE_DPAA_HWDEBUG
600 DPAA_ASSERT(dqrr->cmode == qm_dqrr_cce);
602 qm_cl_invalidate(DQRR_CI);
603 qm_cl_touch_rw(DQRR_CI);
606 static inline void qm_dqrr_cce_consume(struct qm_portal *portal, u8 num)
608 register struct qm_dqrr *dqrr = &portal->dqrr;
610 #ifdef RTE_LIBRTE_DPAA_HWDEBUG
611 DPAA_ASSERT(dqrr->cmode == qm_dqrr_cce);
613 dqrr->ci = (dqrr->ci + num) & (QM_DQRR_SIZE - 1);
614 qm_cl_out(DQRR_CI, dqrr->ci);
617 static inline void qm_dqrr_cce_consume_to_current(struct qm_portal *portal)
619 register struct qm_dqrr *dqrr = &portal->dqrr;
621 #ifdef RTE_LIBRTE_DPAA_HWDEBUG
622 DPAA_ASSERT(dqrr->cmode == qm_dqrr_cce);
624 dqrr->ci = DQRR_PTR2IDX(dqrr->cursor);
625 qm_cl_out(DQRR_CI, dqrr->ci);
628 static inline void qm_dqrr_cdc_consume_1(struct qm_portal *portal, u8 idx,
631 __maybe_unused register struct qm_dqrr *dqrr = &portal->dqrr;
633 #ifdef RTE_LIBRTE_DPAA_HWDEBUG
634 DPAA_ASSERT(dqrr->cmode == qm_dqrr_cdc);
636 DPAA_ASSERT(idx < QM_DQRR_SIZE);
637 qm_out(DQRR_DCAP, (0 << 8) | /* S */
638 ((park ? 1 : 0) << 6) | /* PK */
642 static inline void qm_dqrr_cdc_consume_1ptr(struct qm_portal *portal,
643 const struct qm_dqrr_entry *dq,
646 __maybe_unused register struct qm_dqrr *dqrr = &portal->dqrr;
647 u8 idx = DQRR_PTR2IDX(dq);
649 #ifdef RTE_LIBRTE_DPAA_HWDEBUG
650 DPAA_ASSERT(dqrr->cmode == qm_dqrr_cdc);
652 DPAA_ASSERT(idx < QM_DQRR_SIZE);
653 qm_out(DQRR_DCAP, (0 << 8) | /* DQRR_DCAP::S */
654 ((park ? 1 : 0) << 6) | /* DQRR_DCAP::PK */
655 idx); /* DQRR_DCAP::DCAP_CI */
658 static inline void qm_dqrr_cdc_consume_n(struct qm_portal *portal, u16 bitmask)
660 __maybe_unused register struct qm_dqrr *dqrr = &portal->dqrr;
662 #ifdef RTE_LIBRTE_DPAA_HWDEBUG
663 DPAA_ASSERT(dqrr->cmode == qm_dqrr_cdc);
665 qm_out(DQRR_DCAP, (1 << 8) | /* DQRR_DCAP::S */
666 ((u32)bitmask << 16)); /* DQRR_DCAP::DCAP_CI */
667 dqrr->ci = qm_in(DQRR_CI_CINH) & (QM_DQRR_SIZE - 1);
668 dqrr->fill = qm_cyc_diff(QM_DQRR_SIZE, dqrr->ci, dqrr->pi);
671 static inline u8 qm_dqrr_cdc_cci(struct qm_portal *portal)
673 __maybe_unused register struct qm_dqrr *dqrr = &portal->dqrr;
675 #ifdef RTE_LIBRTE_DPAA_HWDEBUG
676 DPAA_ASSERT(dqrr->cmode == qm_dqrr_cdc);
678 return qm_in(DQRR_CI_CINH) & (QM_DQRR_SIZE - 1);
681 static inline void qm_dqrr_cdc_cce_prefetch(struct qm_portal *portal)
683 __maybe_unused register struct qm_dqrr *dqrr = &portal->dqrr;
685 #ifdef RTE_LIBRTE_DPAA_HWDEBUG
686 DPAA_ASSERT(dqrr->cmode == qm_dqrr_cdc);
688 qm_cl_invalidate(DQRR_CI);
689 qm_cl_touch_ro(DQRR_CI);
692 static inline u8 qm_dqrr_cdc_cce(struct qm_portal *portal)
694 __maybe_unused register struct qm_dqrr *dqrr = &portal->dqrr;
696 #ifdef RTE_LIBRTE_DPAA_HWDEBUG
697 DPAA_ASSERT(dqrr->cmode == qm_dqrr_cdc);
699 return qm_cl_in(DQRR_CI) & (QM_DQRR_SIZE - 1);
702 static inline u8 qm_dqrr_get_ci(struct qm_portal *portal)
704 register struct qm_dqrr *dqrr = &portal->dqrr;
706 #ifdef RTE_LIBRTE_DPAA_HWDEBUG
707 DPAA_ASSERT(dqrr->cmode != qm_dqrr_cdc);
712 static inline void qm_dqrr_park(struct qm_portal *portal, u8 idx)
714 __maybe_unused register struct qm_dqrr *dqrr = &portal->dqrr;
716 #ifdef RTE_LIBRTE_DPAA_HWDEBUG
717 DPAA_ASSERT(dqrr->cmode != qm_dqrr_cdc);
719 qm_out(DQRR_DCAP, (0 << 8) | /* S */
721 (idx & (QM_DQRR_SIZE - 1))); /* DCAP_CI */
724 static inline void qm_dqrr_park_current(struct qm_portal *portal)
726 register struct qm_dqrr *dqrr = &portal->dqrr;
728 #ifdef RTE_LIBRTE_DPAA_HWDEBUG
729 DPAA_ASSERT(dqrr->cmode != qm_dqrr_cdc);
731 qm_out(DQRR_DCAP, (0 << 8) | /* S */
733 DQRR_PTR2IDX(dqrr->cursor)); /* DCAP_CI */
736 static inline void qm_dqrr_sdqcr_set(struct qm_portal *portal, u32 sdqcr)
738 qm_out(DQRR_SDQCR, sdqcr);
741 static inline u32 qm_dqrr_sdqcr_get(struct qm_portal *portal)
743 return qm_in(DQRR_SDQCR);
746 static inline void qm_dqrr_vdqcr_set(struct qm_portal *portal, u32 vdqcr)
748 qm_out(DQRR_VDQCR, vdqcr);
751 static inline u32 qm_dqrr_vdqcr_get(struct qm_portal *portal)
753 return qm_in(DQRR_VDQCR);
756 static inline u8 qm_dqrr_get_ithresh(struct qm_portal *portal)
758 register struct qm_dqrr *dqrr = &portal->dqrr;
760 return dqrr->ithresh;
763 static inline void qm_dqrr_set_ithresh(struct qm_portal *portal, u8 ithresh)
765 qm_out(DQRR_ITR, ithresh);
768 static inline u8 qm_dqrr_get_maxfill(struct qm_portal *portal)
770 return (qm_in(CFG) & 0x00f00000) >> 20;
776 #define MR_CARRYCLEAR(p) \
777 (void *)((unsigned long)(p) & (~(unsigned long)(QM_MR_SIZE << 6)))
779 static inline u8 MR_PTR2IDX(const struct qm_mr_entry *e)
781 return ((uintptr_t)e >> 6) & (QM_MR_SIZE - 1);
784 static inline const struct qm_mr_entry *MR_INC(const struct qm_mr_entry *e)
786 return MR_CARRYCLEAR(e + 1);
789 static inline void qm_mr_finish(struct qm_portal *portal)
791 register struct qm_mr *mr = &portal->mr;
793 if (mr->ci != MR_PTR2IDX(mr->cursor))
794 pr_crit("Ignoring completed MR entries\n");
797 static inline const struct qm_mr_entry *qm_mr_current(struct qm_portal *portal)
799 register struct qm_mr *mr = &portal->mr;
806 static inline u8 qm_mr_next(struct qm_portal *portal)
808 register struct qm_mr *mr = &portal->mr;
810 DPAA_ASSERT(mr->fill);
811 mr->cursor = MR_INC(mr->cursor);
815 static inline void qm_mr_cci_consume(struct qm_portal *portal, u8 num)
817 register struct qm_mr *mr = &portal->mr;
819 #ifdef RTE_LIBRTE_DPAA_HWDEBUG
820 DPAA_ASSERT(mr->cmode == qm_mr_cci);
822 mr->ci = (mr->ci + num) & (QM_MR_SIZE - 1);
823 qm_out(MR_CI_CINH, mr->ci);
826 static inline void qm_mr_cci_consume_to_current(struct qm_portal *portal)
828 register struct qm_mr *mr = &portal->mr;
830 #ifdef RTE_LIBRTE_DPAA_HWDEBUG
831 DPAA_ASSERT(mr->cmode == qm_mr_cci);
833 mr->ci = MR_PTR2IDX(mr->cursor);
834 qm_out(MR_CI_CINH, mr->ci);
837 static inline void qm_mr_set_ithresh(struct qm_portal *portal, u8 ithresh)
839 qm_out(MR_ITR, ithresh);
842 /* ------------------------------ */
843 /* --- Management command API --- */
844 static inline int qm_mc_init(struct qm_portal *portal)
846 register struct qm_mc *mc = &portal->mc;
848 mc->cr = portal->addr.ce + QM_CL_CR;
849 mc->rr = portal->addr.ce + QM_CL_RR0;
850 mc->rridx = (__raw_readb(&mc->cr->__dont_write_directly__verb) &
851 QM_MCC_VERB_VBIT) ? 0 : 1;
852 mc->vbit = mc->rridx ? QM_MCC_VERB_VBIT : 0;
853 #ifdef RTE_LIBRTE_DPAA_HWDEBUG
854 mc->state = qman_mc_idle;
859 static inline void qm_mc_finish(struct qm_portal *portal)
861 __maybe_unused register struct qm_mc *mc = &portal->mc;
863 #ifdef RTE_LIBRTE_DPAA_HWDEBUG
864 DPAA_ASSERT(mc->state == qman_mc_idle);
865 if (mc->state != qman_mc_idle)
866 pr_crit("Losing incomplete MC command\n");
870 static inline struct qm_mc_command *qm_mc_start(struct qm_portal *portal)
872 register struct qm_mc *mc = &portal->mc;
874 #ifdef RTE_LIBRTE_DPAA_HWDEBUG
875 DPAA_ASSERT(mc->state == qman_mc_idle);
876 mc->state = qman_mc_user;
882 static inline void qm_mc_commit(struct qm_portal *portal, u8 myverb)
884 register struct qm_mc *mc = &portal->mc;
885 struct qm_mc_result *rr = mc->rr + mc->rridx;
887 #ifdef RTE_LIBRTE_DPAA_HWDEBUG
888 DPAA_ASSERT(mc->state == qman_mc_user);
891 mc->cr->__dont_write_directly__verb = myverb | mc->vbit;
894 #ifdef RTE_LIBRTE_DPAA_HWDEBUG
895 mc->state = qman_mc_hw;
899 static inline struct qm_mc_result *qm_mc_result(struct qm_portal *portal)
901 register struct qm_mc *mc = &portal->mc;
902 struct qm_mc_result *rr = mc->rr + mc->rridx;
904 #ifdef RTE_LIBRTE_DPAA_HWDEBUG
905 DPAA_ASSERT(mc->state == qman_mc_hw);
907 /* The inactive response register's verb byte always returns zero until
908 * its command is submitted and completed. This includes the valid-bit,
909 * in case you were wondering.
911 if (!__raw_readb(&rr->verb)) {
916 mc->vbit ^= QM_MCC_VERB_VBIT;
917 #ifdef RTE_LIBRTE_DPAA_HWDEBUG
918 mc->state = qman_mc_idle;
923 /* Portal interrupt register API */
924 static inline void qm_isr_set_iperiod(struct qm_portal *portal, u16 iperiod)
926 qm_out(ITPR, iperiod);
929 static inline u32 __qm_isr_read(struct qm_portal *portal, enum qm_isr_reg n)
931 #if defined(RTE_ARCH_ARM64)
932 return __qm_in(&portal->addr, QM_REG_ISR + (n << 6));
934 return __qm_in(&portal->addr, QM_REG_ISR + (n << 2));
938 static inline void __qm_isr_write(struct qm_portal *portal, enum qm_isr_reg n,
941 #if defined(RTE_ARCH_ARM64)
942 __qm_out(&portal->addr, QM_REG_ISR + (n << 6), val);
944 __qm_out(&portal->addr, QM_REG_ISR + (n << 2), val);