1 /* SPDX-License-Identifier: (BSD-3-Clause OR GPL-2.0)
3 * Copyright 2008-2016 Freescale Semiconductor Inc.
10 /***************************/
11 /* Portal register assists */
12 /***************************/
13 #define QM_REG_EQCR_PI_CINH 0x3000
14 #define QM_REG_EQCR_CI_CINH 0x3040
15 #define QM_REG_EQCR_ITR 0x3080
16 #define QM_REG_DQRR_PI_CINH 0x3100
17 #define QM_REG_DQRR_CI_CINH 0x3140
18 #define QM_REG_DQRR_ITR 0x3180
19 #define QM_REG_DQRR_DCAP 0x31C0
20 #define QM_REG_DQRR_SDQCR 0x3200
21 #define QM_REG_DQRR_VDQCR 0x3240
22 #define QM_REG_DQRR_PDQCR 0x3280
23 #define QM_REG_MR_PI_CINH 0x3300
24 #define QM_REG_MR_CI_CINH 0x3340
25 #define QM_REG_MR_ITR 0x3380
26 #define QM_REG_CFG 0x3500
27 #define QM_REG_ISR 0x3600
28 #define QM_REG_IIR 0x36C0
29 #define QM_REG_ITPR 0x3740
31 /* Cache-enabled register offsets */
32 #define QM_CL_EQCR 0x0000
33 #define QM_CL_DQRR 0x1000
34 #define QM_CL_MR 0x2000
35 #define QM_CL_EQCR_PI_CENA 0x3000
36 #define QM_CL_EQCR_CI_CENA 0x3040
37 #define QM_CL_DQRR_PI_CENA 0x3100
38 #define QM_CL_DQRR_CI_CENA 0x3140
39 #define QM_CL_MR_PI_CENA 0x3300
40 #define QM_CL_MR_CI_CENA 0x3340
41 #define QM_CL_CR 0x3800
42 #define QM_CL_RR0 0x3900
43 #define QM_CL_RR1 0x3940
45 /* BTW, the drivers (and h/w programming model) already obtain the required
46 * synchronisation for portal accesses via lwsync(), hwsync(), and
47 * data-dependencies. Use of barrier()s or other order-preserving primitives
48 * simply degrade performance. Hence the use of the __raw_*() interfaces, which
49 * simply ensure that the compiler treats the portal registers as volatile (ie.
53 /* Cache-inhibited register access. */
54 #define __qm_in(qm, o) be32_to_cpu(__raw_readl((qm)->ci + (o)))
55 #define __qm_out(qm, o, val) __raw_writel((cpu_to_be32(val)), \
57 #define qm_in(reg) __qm_in(&portal->addr, QM_REG_##reg)
58 #define qm_out(reg, val) __qm_out(&portal->addr, QM_REG_##reg, val)
60 /* Cache-enabled (index) register access */
61 #define __qm_cl_touch_ro(qm, o) dcbt_ro((qm)->ce + (o))
62 #define __qm_cl_touch_rw(qm, o) dcbt_rw((qm)->ce + (o))
63 #define __qm_cl_in(qm, o) be32_to_cpu(__raw_readl((qm)->ce + (o)))
64 #define __qm_cl_out(qm, o, val) \
66 u32 *__tmpclout = (qm)->ce + (o); \
67 __raw_writel(cpu_to_be32(val), __tmpclout); \
70 #define __qm_cl_invalidate(qm, o) dccivac((qm)->ce + (o))
71 #define qm_cl_touch_ro(reg) __qm_cl_touch_ro(&portal->addr, QM_CL_##reg##_CENA)
72 #define qm_cl_touch_rw(reg) __qm_cl_touch_rw(&portal->addr, QM_CL_##reg##_CENA)
73 #define qm_cl_in(reg) __qm_cl_in(&portal->addr, QM_CL_##reg##_CENA)
74 #define qm_cl_out(reg, val) __qm_cl_out(&portal->addr, QM_CL_##reg##_CENA, val)
75 #define qm_cl_invalidate(reg)\
76 __qm_cl_invalidate(&portal->addr, QM_CL_##reg##_CENA)
78 /* Cache-enabled ring access */
79 #define qm_cl(base, idx) ((void *)base + ((idx) << 6))
81 /* Cyclic helper for rings. FIXME: once we are able to do fine-grain perf
82 * analysis, look at using the "extra" bit in the ring index registers to avoid
85 static inline u8 qm_cyc_diff(u8 ringsize, u8 first, u8 last)
87 /* 'first' is included, 'last' is excluded */
90 return ringsize + last - first;
95 * pmode == production mode
96 * cmode == consumption mode,
97 * dmode == h/w dequeue mode.
98 * Enum values use 3 letter codes. First letter matches the portal mode,
99 * remaining two letters indicate;
100 * ci == cache-inhibited portal register
101 * ce == cache-enabled portal register
102 * vb == in-band valid-bit (cache-enabled)
103 * dc == DCA (Discrete Consumption Acknowledgment), DQRR-only
104 * As for "enum qm_dqrr_dmode", it should be self-explanatory.
106 enum qm_eqcr_pmode { /* matches QCSP_CFG::EPM */
107 qm_eqcr_pci = 0, /* PI index, cache-inhibited */
108 qm_eqcr_pce = 1, /* PI index, cache-enabled */
109 qm_eqcr_pvb = 2 /* valid-bit */
112 enum qm_dqrr_dmode { /* matches QCSP_CFG::DP */
113 qm_dqrr_dpush = 0, /* SDQCR + VDQCR */
114 qm_dqrr_dpull = 1 /* PDQCR */
117 enum qm_dqrr_pmode { /* s/w-only */
118 qm_dqrr_pci, /* reads DQRR_PI_CINH */
119 qm_dqrr_pce, /* reads DQRR_PI_CENA */
120 qm_dqrr_pvb /* reads valid-bit */
123 enum qm_dqrr_cmode { /* matches QCSP_CFG::DCM */
124 qm_dqrr_cci = 0, /* CI index, cache-inhibited */
125 qm_dqrr_cce = 1, /* CI index, cache-enabled */
126 qm_dqrr_cdc = 2 /* Discrete Consumption Acknowledgment */
129 enum qm_mr_pmode { /* s/w-only */
130 qm_mr_pci, /* reads MR_PI_CINH */
131 qm_mr_pce, /* reads MR_PI_CENA */
132 qm_mr_pvb /* reads valid-bit */
135 enum qm_mr_cmode { /* matches QCSP_CFG::MM */
136 qm_mr_cci = 0, /* CI index, cache-inhibited */
137 qm_mr_cce = 1 /* CI index, cache-enabled */
140 /* ------------------------- */
141 /* --- Portal structures --- */
143 #define QM_EQCR_SIZE 8
144 #define QM_DQRR_SIZE 16
148 struct qm_eqcr_entry *ring, *cursor;
149 u8 ci, available, ithresh, vbit;
150 #ifdef RTE_LIBRTE_DPAA_HWDEBUG
152 enum qm_eqcr_pmode pmode;
157 const struct qm_dqrr_entry *ring, *cursor;
158 u8 pi, ci, fill, ithresh, vbit;
159 #ifdef RTE_LIBRTE_DPAA_HWDEBUG
160 enum qm_dqrr_dmode dmode;
161 enum qm_dqrr_pmode pmode;
162 enum qm_dqrr_cmode cmode;
167 const struct qm_mr_entry *ring, *cursor;
168 u8 pi, ci, fill, ithresh, vbit;
169 #ifdef RTE_LIBRTE_DPAA_HWDEBUG
170 enum qm_mr_pmode pmode;
171 enum qm_mr_cmode cmode;
176 struct qm_mc_command *cr;
177 struct qm_mc_result *rr;
179 #ifdef RTE_LIBRTE_DPAA_HWDEBUG
181 /* Can be _mc_start()ed */
183 /* Can be _mc_commit()ed or _mc_abort()ed */
185 /* Can only be _mc_retry()ed */
191 #define QM_PORTAL_ALIGNMENT ____cacheline_aligned
194 void __iomem *ce; /* cache-enabled */
195 void __iomem *ci; /* cache-inhibited */
204 } QM_PORTAL_ALIGNMENT;
206 /* Bit-wise logic to wrap a ring pointer by clearing the "carry bit" */
207 #define EQCR_CARRYCLEAR(p) \
208 (void *)((unsigned long)(p) & (~(unsigned long)(QM_EQCR_SIZE << 6)))
210 extern dma_addr_t rte_mem_virt2iova(const void *addr);
212 /* Bit-wise logic to convert a ring pointer to a ring index */
213 static inline u8 EQCR_PTR2IDX(struct qm_eqcr_entry *e)
215 return ((uintptr_t)e >> 6) & (QM_EQCR_SIZE - 1);
218 /* Increment the 'cursor' ring pointer, taking 'vbit' into account */
219 static inline void EQCR_INC(struct qm_eqcr *eqcr)
221 /* NB: this is odd-looking, but experiments show that it generates fast
222 * code with essentially no branching overheads. We increment to the
223 * next EQCR pointer and handle overflow and 'vbit'.
225 struct qm_eqcr_entry *partial = eqcr->cursor + 1;
227 eqcr->cursor = EQCR_CARRYCLEAR(partial);
228 if (partial != eqcr->cursor)
229 eqcr->vbit ^= QM_EQCR_VERB_VBIT;
232 static inline struct qm_eqcr_entry *qm_eqcr_start_no_stash(struct qm_portal
235 register struct qm_eqcr *eqcr = &portal->eqcr;
237 #ifdef RTE_LIBRTE_DPAA_HWDEBUG
238 DPAA_ASSERT(!eqcr->busy);
240 if (!eqcr->available)
243 #ifdef RTE_LIBRTE_DPAA_HWDEBUG
250 static inline struct qm_eqcr_entry *qm_eqcr_start_stash(struct qm_portal
253 register struct qm_eqcr *eqcr = &portal->eqcr;
256 #ifdef RTE_LIBRTE_DPAA_HWDEBUG
257 DPAA_ASSERT(!eqcr->busy);
259 if (!eqcr->available) {
261 eqcr->ci = qm_cl_in(EQCR_CI) & (QM_EQCR_SIZE - 1);
262 diff = qm_cyc_diff(QM_EQCR_SIZE, old_ci, eqcr->ci);
263 eqcr->available += diff;
267 #ifdef RTE_LIBRTE_DPAA_HWDEBUG
273 static inline void qm_eqcr_abort(struct qm_portal *portal)
275 __maybe_unused register struct qm_eqcr *eqcr = &portal->eqcr;
277 #ifdef RTE_LIBRTE_DPAA_HWDEBUG
278 DPAA_ASSERT(eqcr->busy);
283 static inline struct qm_eqcr_entry *qm_eqcr_pend_and_next(
284 struct qm_portal *portal, u8 myverb)
286 register struct qm_eqcr *eqcr = &portal->eqcr;
288 #ifdef RTE_LIBRTE_DPAA_HWDEBUG
289 DPAA_ASSERT(eqcr->busy);
290 DPAA_ASSERT(eqcr->pmode != qm_eqcr_pvb);
292 if (eqcr->available == 1)
294 eqcr->cursor->__dont_write_directly__verb = myverb | eqcr->vbit;
301 #define EQCR_COMMIT_CHECKS(eqcr) \
303 DPAA_ASSERT(eqcr->busy); \
304 DPAA_ASSERT(eqcr->cursor->orp == (eqcr->cursor->orp & 0x00ffffff)); \
305 DPAA_ASSERT(eqcr->cursor->fqid == (eqcr->cursor->fqid & 0x00ffffff)); \
308 static inline void qm_eqcr_pci_commit(struct qm_portal *portal, u8 myverb)
310 register struct qm_eqcr *eqcr = &portal->eqcr;
312 #ifdef RTE_LIBRTE_DPAA_HWDEBUG
313 EQCR_COMMIT_CHECKS(eqcr);
314 DPAA_ASSERT(eqcr->pmode == qm_eqcr_pci);
316 eqcr->cursor->__dont_write_directly__verb = myverb | eqcr->vbit;
321 qm_out(EQCR_PI_CINH, EQCR_PTR2IDX(eqcr->cursor));
322 #ifdef RTE_LIBRTE_DPAA_HWDEBUG
327 static inline void qm_eqcr_pce_prefetch(struct qm_portal *portal)
329 __maybe_unused register struct qm_eqcr *eqcr = &portal->eqcr;
331 #ifdef RTE_LIBRTE_DPAA_HWDEBUG
332 DPAA_ASSERT(eqcr->pmode == qm_eqcr_pce);
334 qm_cl_invalidate(EQCR_PI);
335 qm_cl_touch_rw(EQCR_PI);
338 static inline void qm_eqcr_pce_commit(struct qm_portal *portal, u8 myverb)
340 register struct qm_eqcr *eqcr = &portal->eqcr;
342 #ifdef RTE_LIBRTE_DPAA_HWDEBUG
343 EQCR_COMMIT_CHECKS(eqcr);
344 DPAA_ASSERT(eqcr->pmode == qm_eqcr_pce);
346 eqcr->cursor->__dont_write_directly__verb = myverb | eqcr->vbit;
351 qm_cl_out(EQCR_PI, EQCR_PTR2IDX(eqcr->cursor));
352 #ifdef RTE_LIBRTE_DPAA_HWDEBUG
357 static inline void qm_eqcr_pvb_commit(struct qm_portal *portal, u8 myverb)
359 register struct qm_eqcr *eqcr = &portal->eqcr;
360 struct qm_eqcr_entry *eqcursor;
362 #ifdef RTE_LIBRTE_DPAA_HWDEBUG
363 EQCR_COMMIT_CHECKS(eqcr);
364 DPAA_ASSERT(eqcr->pmode == qm_eqcr_pvb);
367 eqcursor = eqcr->cursor;
368 eqcursor->__dont_write_directly__verb = myverb | eqcr->vbit;
372 #ifdef RTE_LIBRTE_DPAA_HWDEBUG
377 static inline u8 qm_eqcr_cci_update(struct qm_portal *portal)
379 register struct qm_eqcr *eqcr = &portal->eqcr;
380 u8 diff, old_ci = eqcr->ci;
382 eqcr->ci = qm_in(EQCR_CI_CINH) & (QM_EQCR_SIZE - 1);
383 diff = qm_cyc_diff(QM_EQCR_SIZE, old_ci, eqcr->ci);
384 eqcr->available += diff;
388 static inline void qm_eqcr_cce_prefetch(struct qm_portal *portal)
390 __maybe_unused register struct qm_eqcr *eqcr = &portal->eqcr;
392 qm_cl_touch_ro(EQCR_CI);
395 static inline u8 qm_eqcr_cce_update(struct qm_portal *portal)
397 register struct qm_eqcr *eqcr = &portal->eqcr;
398 u8 diff, old_ci = eqcr->ci;
400 eqcr->ci = qm_cl_in(EQCR_CI) & (QM_EQCR_SIZE - 1);
401 qm_cl_invalidate(EQCR_CI);
402 diff = qm_cyc_diff(QM_EQCR_SIZE, old_ci, eqcr->ci);
403 eqcr->available += diff;
407 static inline u8 qm_eqcr_get_ithresh(struct qm_portal *portal)
409 register struct qm_eqcr *eqcr = &portal->eqcr;
411 return eqcr->ithresh;
414 static inline void qm_eqcr_set_ithresh(struct qm_portal *portal, u8 ithresh)
416 register struct qm_eqcr *eqcr = &portal->eqcr;
418 eqcr->ithresh = ithresh;
419 qm_out(EQCR_ITR, ithresh);
422 static inline u8 qm_eqcr_get_avail(struct qm_portal *portal)
424 register struct qm_eqcr *eqcr = &portal->eqcr;
426 return eqcr->available;
429 static inline u8 qm_eqcr_get_fill(struct qm_portal *portal)
431 register struct qm_eqcr *eqcr = &portal->eqcr;
433 return QM_EQCR_SIZE - 1 - eqcr->available;
436 #define DQRR_CARRYCLEAR(p) \
437 (void *)((unsigned long)(p) & (~(unsigned long)(QM_DQRR_SIZE << 6)))
439 static inline u8 DQRR_PTR2IDX(const struct qm_dqrr_entry *e)
441 return ((uintptr_t)e >> 6) & (QM_DQRR_SIZE - 1);
444 static inline const struct qm_dqrr_entry *DQRR_INC(
445 const struct qm_dqrr_entry *e)
447 return DQRR_CARRYCLEAR(e + 1);
450 static inline void qm_dqrr_set_maxfill(struct qm_portal *portal, u8 mf)
452 qm_out(CFG, (qm_in(CFG) & 0xff0fffff) |
453 ((mf & (QM_DQRR_SIZE - 1)) << 20));
456 static inline const struct qm_dqrr_entry *qm_dqrr_current(
457 struct qm_portal *portal)
459 register struct qm_dqrr *dqrr = &portal->dqrr;
466 static inline u8 qm_dqrr_cursor(struct qm_portal *portal)
468 register struct qm_dqrr *dqrr = &portal->dqrr;
470 return DQRR_PTR2IDX(dqrr->cursor);
473 static inline u8 qm_dqrr_next(struct qm_portal *portal)
475 register struct qm_dqrr *dqrr = &portal->dqrr;
477 DPAA_ASSERT(dqrr->fill);
478 dqrr->cursor = DQRR_INC(dqrr->cursor);
482 static inline u8 qm_dqrr_pci_update(struct qm_portal *portal)
484 register struct qm_dqrr *dqrr = &portal->dqrr;
485 u8 diff, old_pi = dqrr->pi;
487 #ifdef RTE_LIBRTE_DPAA_HWDEBUG
488 DPAA_ASSERT(dqrr->pmode == qm_dqrr_pci);
490 dqrr->pi = qm_in(DQRR_PI_CINH) & (QM_DQRR_SIZE - 1);
491 diff = qm_cyc_diff(QM_DQRR_SIZE, old_pi, dqrr->pi);
496 static inline void qm_dqrr_pce_prefetch(struct qm_portal *portal)
498 __maybe_unused register struct qm_dqrr *dqrr = &portal->dqrr;
500 #ifdef RTE_LIBRTE_DPAA_HWDEBUG
501 DPAA_ASSERT(dqrr->pmode == qm_dqrr_pce);
503 qm_cl_invalidate(DQRR_PI);
504 qm_cl_touch_ro(DQRR_PI);
507 static inline u8 qm_dqrr_pce_update(struct qm_portal *portal)
509 register struct qm_dqrr *dqrr = &portal->dqrr;
510 u8 diff, old_pi = dqrr->pi;
512 #ifdef RTE_LIBRTE_DPAA_HWDEBUG
513 DPAA_ASSERT(dqrr->pmode == qm_dqrr_pce);
515 dqrr->pi = qm_cl_in(DQRR_PI) & (QM_DQRR_SIZE - 1);
516 diff = qm_cyc_diff(QM_DQRR_SIZE, old_pi, dqrr->pi);
521 static inline void qm_dqrr_pvb_update(struct qm_portal *portal)
523 register struct qm_dqrr *dqrr = &portal->dqrr;
524 const struct qm_dqrr_entry *res = qm_cl(dqrr->ring, dqrr->pi);
526 #ifdef RTE_LIBRTE_DPAA_HWDEBUG
527 DPAA_ASSERT(dqrr->pmode == qm_dqrr_pvb);
529 /* when accessing 'verb', use __raw_readb() to ensure that compiler
530 * inlining doesn't try to optimise out "excess reads".
532 if ((__raw_readb(&res->verb) & QM_DQRR_VERB_VBIT) == dqrr->vbit) {
533 dqrr->pi = (dqrr->pi + 1) & (QM_DQRR_SIZE - 1);
535 dqrr->vbit ^= QM_DQRR_VERB_VBIT;
540 static inline void qm_dqrr_cci_consume(struct qm_portal *portal, u8 num)
542 register struct qm_dqrr *dqrr = &portal->dqrr;
544 #ifdef RTE_LIBRTE_DPAA_HWDEBUG
545 DPAA_ASSERT(dqrr->cmode == qm_dqrr_cci);
547 dqrr->ci = (dqrr->ci + num) & (QM_DQRR_SIZE - 1);
548 qm_out(DQRR_CI_CINH, dqrr->ci);
551 static inline void qm_dqrr_cci_consume_to_current(struct qm_portal *portal)
553 register struct qm_dqrr *dqrr = &portal->dqrr;
555 #ifdef RTE_LIBRTE_DPAA_HWDEBUG
556 DPAA_ASSERT(dqrr->cmode == qm_dqrr_cci);
558 dqrr->ci = DQRR_PTR2IDX(dqrr->cursor);
559 qm_out(DQRR_CI_CINH, dqrr->ci);
562 static inline void qm_dqrr_cce_prefetch(struct qm_portal *portal)
564 __maybe_unused register struct qm_dqrr *dqrr = &portal->dqrr;
566 #ifdef RTE_LIBRTE_DPAA_HWDEBUG
567 DPAA_ASSERT(dqrr->cmode == qm_dqrr_cce);
569 qm_cl_invalidate(DQRR_CI);
570 qm_cl_touch_rw(DQRR_CI);
573 static inline void qm_dqrr_cce_consume(struct qm_portal *portal, u8 num)
575 register struct qm_dqrr *dqrr = &portal->dqrr;
577 #ifdef RTE_LIBRTE_DPAA_HWDEBUG
578 DPAA_ASSERT(dqrr->cmode == qm_dqrr_cce);
580 dqrr->ci = (dqrr->ci + num) & (QM_DQRR_SIZE - 1);
581 qm_cl_out(DQRR_CI, dqrr->ci);
584 static inline void qm_dqrr_cce_consume_to_current(struct qm_portal *portal)
586 register struct qm_dqrr *dqrr = &portal->dqrr;
588 #ifdef RTE_LIBRTE_DPAA_HWDEBUG
589 DPAA_ASSERT(dqrr->cmode == qm_dqrr_cce);
591 dqrr->ci = DQRR_PTR2IDX(dqrr->cursor);
592 qm_cl_out(DQRR_CI, dqrr->ci);
595 static inline void qm_dqrr_cdc_consume_1(struct qm_portal *portal, u8 idx,
598 __maybe_unused register struct qm_dqrr *dqrr = &portal->dqrr;
600 #ifdef RTE_LIBRTE_DPAA_HWDEBUG
601 DPAA_ASSERT(dqrr->cmode == qm_dqrr_cdc);
603 DPAA_ASSERT(idx < QM_DQRR_SIZE);
604 qm_out(DQRR_DCAP, (0 << 8) | /* S */
605 ((park ? 1 : 0) << 6) | /* PK */
609 static inline void qm_dqrr_cdc_consume_1ptr(struct qm_portal *portal,
610 const struct qm_dqrr_entry *dq,
613 __maybe_unused register struct qm_dqrr *dqrr = &portal->dqrr;
614 u8 idx = DQRR_PTR2IDX(dq);
616 #ifdef RTE_LIBRTE_DPAA_HWDEBUG
617 DPAA_ASSERT(dqrr->cmode == qm_dqrr_cdc);
619 DPAA_ASSERT(idx < QM_DQRR_SIZE);
620 qm_out(DQRR_DCAP, (0 << 8) | /* DQRR_DCAP::S */
621 ((park ? 1 : 0) << 6) | /* DQRR_DCAP::PK */
622 idx); /* DQRR_DCAP::DCAP_CI */
625 static inline void qm_dqrr_cdc_consume_n(struct qm_portal *portal, u16 bitmask)
627 __maybe_unused register struct qm_dqrr *dqrr = &portal->dqrr;
629 #ifdef RTE_LIBRTE_DPAA_HWDEBUG
630 DPAA_ASSERT(dqrr->cmode == qm_dqrr_cdc);
632 qm_out(DQRR_DCAP, (1 << 8) | /* DQRR_DCAP::S */
633 ((u32)bitmask << 16)); /* DQRR_DCAP::DCAP_CI */
634 dqrr->ci = qm_in(DQRR_CI_CINH) & (QM_DQRR_SIZE - 1);
635 dqrr->fill = qm_cyc_diff(QM_DQRR_SIZE, dqrr->ci, dqrr->pi);
638 static inline u8 qm_dqrr_cdc_cci(struct qm_portal *portal)
640 __maybe_unused register struct qm_dqrr *dqrr = &portal->dqrr;
642 #ifdef RTE_LIBRTE_DPAA_HWDEBUG
643 DPAA_ASSERT(dqrr->cmode == qm_dqrr_cdc);
645 return qm_in(DQRR_CI_CINH) & (QM_DQRR_SIZE - 1);
648 static inline void qm_dqrr_cdc_cce_prefetch(struct qm_portal *portal)
650 __maybe_unused register struct qm_dqrr *dqrr = &portal->dqrr;
652 #ifdef RTE_LIBRTE_DPAA_HWDEBUG
653 DPAA_ASSERT(dqrr->cmode == qm_dqrr_cdc);
655 qm_cl_invalidate(DQRR_CI);
656 qm_cl_touch_ro(DQRR_CI);
659 static inline u8 qm_dqrr_cdc_cce(struct qm_portal *portal)
661 __maybe_unused register struct qm_dqrr *dqrr = &portal->dqrr;
663 #ifdef RTE_LIBRTE_DPAA_HWDEBUG
664 DPAA_ASSERT(dqrr->cmode == qm_dqrr_cdc);
666 return qm_cl_in(DQRR_CI) & (QM_DQRR_SIZE - 1);
669 static inline u8 qm_dqrr_get_ci(struct qm_portal *portal)
671 register struct qm_dqrr *dqrr = &portal->dqrr;
673 #ifdef RTE_LIBRTE_DPAA_HWDEBUG
674 DPAA_ASSERT(dqrr->cmode != qm_dqrr_cdc);
679 static inline void qm_dqrr_park(struct qm_portal *portal, u8 idx)
681 __maybe_unused register struct qm_dqrr *dqrr = &portal->dqrr;
683 #ifdef RTE_LIBRTE_DPAA_HWDEBUG
684 DPAA_ASSERT(dqrr->cmode != qm_dqrr_cdc);
686 qm_out(DQRR_DCAP, (0 << 8) | /* S */
688 (idx & (QM_DQRR_SIZE - 1))); /* DCAP_CI */
691 static inline void qm_dqrr_park_current(struct qm_portal *portal)
693 register struct qm_dqrr *dqrr = &portal->dqrr;
695 #ifdef RTE_LIBRTE_DPAA_HWDEBUG
696 DPAA_ASSERT(dqrr->cmode != qm_dqrr_cdc);
698 qm_out(DQRR_DCAP, (0 << 8) | /* S */
700 DQRR_PTR2IDX(dqrr->cursor)); /* DCAP_CI */
703 static inline void qm_dqrr_sdqcr_set(struct qm_portal *portal, u32 sdqcr)
705 qm_out(DQRR_SDQCR, sdqcr);
708 static inline u32 qm_dqrr_sdqcr_get(struct qm_portal *portal)
710 return qm_in(DQRR_SDQCR);
713 static inline void qm_dqrr_vdqcr_set(struct qm_portal *portal, u32 vdqcr)
715 qm_out(DQRR_VDQCR, vdqcr);
718 static inline u32 qm_dqrr_vdqcr_get(struct qm_portal *portal)
720 return qm_in(DQRR_VDQCR);
723 static inline u8 qm_dqrr_get_ithresh(struct qm_portal *portal)
725 register struct qm_dqrr *dqrr = &portal->dqrr;
727 return dqrr->ithresh;
730 static inline void qm_dqrr_set_ithresh(struct qm_portal *portal, u8 ithresh)
732 qm_out(DQRR_ITR, ithresh);
735 static inline u8 qm_dqrr_get_maxfill(struct qm_portal *portal)
737 return (qm_in(CFG) & 0x00f00000) >> 20;
743 #define MR_CARRYCLEAR(p) \
744 (void *)((unsigned long)(p) & (~(unsigned long)(QM_MR_SIZE << 6)))
746 static inline u8 MR_PTR2IDX(const struct qm_mr_entry *e)
748 return ((uintptr_t)e >> 6) & (QM_MR_SIZE - 1);
751 static inline const struct qm_mr_entry *MR_INC(const struct qm_mr_entry *e)
753 return MR_CARRYCLEAR(e + 1);
756 static inline void qm_mr_finish(struct qm_portal *portal)
758 register struct qm_mr *mr = &portal->mr;
760 if (mr->ci != MR_PTR2IDX(mr->cursor))
761 pr_crit("Ignoring completed MR entries\n");
764 static inline const struct qm_mr_entry *qm_mr_current(struct qm_portal *portal)
766 register struct qm_mr *mr = &portal->mr;
773 static inline u8 qm_mr_next(struct qm_portal *portal)
775 register struct qm_mr *mr = &portal->mr;
777 DPAA_ASSERT(mr->fill);
778 mr->cursor = MR_INC(mr->cursor);
782 static inline void qm_mr_cci_consume(struct qm_portal *portal, u8 num)
784 register struct qm_mr *mr = &portal->mr;
786 #ifdef RTE_LIBRTE_DPAA_HWDEBUG
787 DPAA_ASSERT(mr->cmode == qm_mr_cci);
789 mr->ci = (mr->ci + num) & (QM_MR_SIZE - 1);
790 qm_out(MR_CI_CINH, mr->ci);
793 static inline void qm_mr_cci_consume_to_current(struct qm_portal *portal)
795 register struct qm_mr *mr = &portal->mr;
797 #ifdef RTE_LIBRTE_DPAA_HWDEBUG
798 DPAA_ASSERT(mr->cmode == qm_mr_cci);
800 mr->ci = MR_PTR2IDX(mr->cursor);
801 qm_out(MR_CI_CINH, mr->ci);
804 static inline void qm_mr_set_ithresh(struct qm_portal *portal, u8 ithresh)
806 qm_out(MR_ITR, ithresh);
809 /* ------------------------------ */
810 /* --- Management command API --- */
811 static inline int qm_mc_init(struct qm_portal *portal)
813 register struct qm_mc *mc = &portal->mc;
815 mc->cr = portal->addr.ce + QM_CL_CR;
816 mc->rr = portal->addr.ce + QM_CL_RR0;
817 mc->rridx = (__raw_readb(&mc->cr->__dont_write_directly__verb) &
818 QM_MCC_VERB_VBIT) ? 0 : 1;
819 mc->vbit = mc->rridx ? QM_MCC_VERB_VBIT : 0;
820 #ifdef RTE_LIBRTE_DPAA_HWDEBUG
821 mc->state = qman_mc_idle;
826 static inline void qm_mc_finish(struct qm_portal *portal)
828 __maybe_unused register struct qm_mc *mc = &portal->mc;
830 #ifdef RTE_LIBRTE_DPAA_HWDEBUG
831 DPAA_ASSERT(mc->state == qman_mc_idle);
832 if (mc->state != qman_mc_idle)
833 pr_crit("Losing incomplete MC command\n");
837 static inline struct qm_mc_command *qm_mc_start(struct qm_portal *portal)
839 register struct qm_mc *mc = &portal->mc;
841 #ifdef RTE_LIBRTE_DPAA_HWDEBUG
842 DPAA_ASSERT(mc->state == qman_mc_idle);
843 mc->state = qman_mc_user;
849 static inline void qm_mc_commit(struct qm_portal *portal, u8 myverb)
851 register struct qm_mc *mc = &portal->mc;
852 struct qm_mc_result *rr = mc->rr + mc->rridx;
854 #ifdef RTE_LIBRTE_DPAA_HWDEBUG
855 DPAA_ASSERT(mc->state == qman_mc_user);
858 mc->cr->__dont_write_directly__verb = myverb | mc->vbit;
861 #ifdef RTE_LIBRTE_DPAA_HWDEBUG
862 mc->state = qman_mc_hw;
866 static inline struct qm_mc_result *qm_mc_result(struct qm_portal *portal)
868 register struct qm_mc *mc = &portal->mc;
869 struct qm_mc_result *rr = mc->rr + mc->rridx;
871 #ifdef RTE_LIBRTE_DPAA_HWDEBUG
872 DPAA_ASSERT(mc->state == qman_mc_hw);
874 /* The inactive response register's verb byte always returns zero until
875 * its command is submitted and completed. This includes the valid-bit,
876 * in case you were wondering.
878 if (!__raw_readb(&rr->verb)) {
883 mc->vbit ^= QM_MCC_VERB_VBIT;
884 #ifdef RTE_LIBRTE_DPAA_HWDEBUG
885 mc->state = qman_mc_idle;
890 /* Portal interrupt register API */
891 static inline void qm_isr_set_iperiod(struct qm_portal *portal, u16 iperiod)
893 qm_out(ITPR, iperiod);
896 static inline u32 __qm_isr_read(struct qm_portal *portal, enum qm_isr_reg n)
898 #if defined(RTE_ARCH_ARM64)
899 return __qm_in(&portal->addr, QM_REG_ISR + (n << 6));
901 return __qm_in(&portal->addr, QM_REG_ISR + (n << 2));
905 static inline void __qm_isr_write(struct qm_portal *portal, enum qm_isr_reg n,
908 #if defined(RTE_ARCH_ARM64)
909 __qm_out(&portal->addr, QM_REG_ISR + (n << 6), val);
911 __qm_out(&portal->addr, QM_REG_ISR + (n << 2), val);