2 * This file is provided under a dual BSD/GPLv2 license. When using or
3 * redistributing this file, you may do so under either license.
7 * Copyright 2008-2016 Freescale Semiconductor Inc.
10 * Redistribution and use in source and binary forms, with or without
11 * modification, are permitted provided that the following conditions are met:
12 * * Redistributions of source code must retain the above copyright
13 * notice, this list of conditions and the following disclaimer.
14 * * Redistributions in binary form must reproduce the above copyright
15 * notice, this list of conditions and the following disclaimer in the
16 * documentation and/or other materials provided with the distribution.
17 * * Neither the name of the above-listed copyright holders nor the
18 * names of any contributors may be used to endorse or promote products
19 * derived from this software without specific prior written permission.
23 * ALTERNATIVELY, this software may be distributed under the terms of the
24 * GNU General Public License ("GPL") as published by the Free Software
25 * Foundation, either version 2 of that License or (at your option) any
28 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
29 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
30 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
31 * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDERS OR CONTRIBUTORS BE
32 * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
33 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
34 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
35 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
36 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
37 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
38 * POSSIBILITY OF SUCH DAMAGE.
42 #include <rte_branch_prediction.h>
44 /* Compilation constants */
45 #define DQRR_MAXFILL 15
46 #define EQCR_ITHRESH 4 /* if EQCR congests, interrupt threshold */
47 #define IRQNAME "QMan portal %d"
48 #define MAX_IRQNAME 16 /* big enough for "QMan portal %d" */
49 /* maximum number of DQRR entries to process in qman_poll() */
50 #define FSL_QMAN_POLL_LIMIT 8
52 /* Lock/unlock frame queues, subject to the "LOCKED" flag. This is about
53 * inter-processor locking only. Note, FQLOCK() is always called either under a
54 * local_irq_save() or from interrupt context - hence there's no need for irq
55 * protection (and indeed, attempting to nest irq-protection doesn't work, as
56 * the "irq en/disable" machinery isn't recursive...).
60 struct qman_fq *__fq478 = (fq); \
61 if (fq_isset(__fq478, QMAN_FQ_FLAG_LOCKED)) \
62 spin_lock(&__fq478->fqlock); \
64 #define FQUNLOCK(fq) \
66 struct qman_fq *__fq478 = (fq); \
67 if (fq_isset(__fq478, QMAN_FQ_FLAG_LOCKED)) \
68 spin_unlock(&__fq478->fqlock); \
71 static inline void fq_set(struct qman_fq *fq, u32 mask)
73 dpaa_set_bits(mask, &fq->flags);
76 static inline void fq_clear(struct qman_fq *fq, u32 mask)
78 dpaa_clear_bits(mask, &fq->flags);
81 static inline int fq_isset(struct qman_fq *fq, u32 mask)
83 return fq->flags & mask;
86 static inline int fq_isclear(struct qman_fq *fq, u32 mask)
88 return !(fq->flags & mask);
93 /* PORTAL_BITS_*** - dynamic, strictly internal */
95 /* interrupt sources processed by portal_isr(), configurable */
96 unsigned long irq_sources;
97 u32 use_eqcr_ci_stashing;
98 u32 slowpoll; /* only used when interrupts are off */
99 /* only 1 volatile dequeue at a time */
100 struct qman_fq *vdqcr_owned;
102 int dqrr_disable_ref;
103 /* A portal-specific handler for DCP ERNs. If this is NULL, the global
104 * handler is called instead.
106 qman_cb_dc_ern cb_dc_ern;
107 /* When the cpu-affine portal is activated, this is non-NULL */
108 const struct qm_portal_config *config;
109 struct dpa_rbtree retire_table;
110 char irqname[MAX_IRQNAME];
111 /* 2-element array. cgrs[0] is mask, cgrs[1] is snapshot. */
112 struct qman_cgrs *cgrs;
113 /* linked-list of CSCN handlers. */
114 struct list_head cgr_cbs;
117 /* track if memory was allocated by the driver */
118 #if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__
119 /* Keep a shadow copy of the DQRR on LE systems as the SW needs to
120 * do byte swaps of DQRR read only memory. First entry must be aligned
121 * to 2 ** 10 to ensure DQRR index calculations based shadow copy
122 * address (6 bits for address shift + 4 bits for the DQRR size).
124 struct qm_dqrr_entry shadow_dqrr[QM_DQRR_SIZE]
125 __attribute__((aligned(1024)));
129 /* Global handler for DCP ERNs. Used when the portal receiving the message does
130 * not have a portal-specific handler.
132 static qman_cb_dc_ern cb_dc_ern;
134 static cpumask_t affine_mask;
135 static DEFINE_SPINLOCK(affine_mask_lock);
136 static u16 affine_channels[NR_CPUS];
137 static RTE_DEFINE_PER_LCORE(struct qman_portal, qman_affine_portal);
139 static inline struct qman_portal *get_affine_portal(void)
141 return &RTE_PER_LCORE(qman_affine_portal);
144 /* This gives a FQID->FQ lookup to cover the fact that we can't directly demux
145 * retirement notifications (the fact they are sometimes h/w-consumed means that
146 * contextB isn't always a s/w demux - and as we can't know which case it is
147 * when looking at the notification, we have to use the slow lookup for all of
148 * them). NB, it's possible to have multiple FQ objects refer to the same FQID
149 * (though at most one of them should be the consumer), so this table isn't for
150 * all FQs - FQs are added when retirement commands are issued, and removed when
151 * they complete, which also massively reduces the size of this table.
153 IMPLEMENT_DPAA_RBTREE(fqtree, struct qman_fq, node, fqid);
155 * This is what everything can wait on, even if it migrates to a different cpu
156 * to the one whose affine portal it is waiting on.
158 static DECLARE_WAIT_QUEUE_HEAD(affine_queue);
160 static inline int table_push_fq(struct qman_portal *p, struct qman_fq *fq)
162 int ret = fqtree_push(&p->retire_table, fq);
165 pr_err("ERROR: double FQ-retirement %d\n", fq->fqid);
169 static inline void table_del_fq(struct qman_portal *p, struct qman_fq *fq)
171 fqtree_del(&p->retire_table, fq);
174 static inline struct qman_fq *table_find_fq(struct qman_portal *p, u32 fqid)
176 return fqtree_find(&p->retire_table, fqid);
179 #ifdef CONFIG_FSL_QMAN_FQ_LOOKUP
180 static void **qman_fq_lookup_table;
181 static size_t qman_fq_lookup_table_size;
183 int qman_setup_fq_lookup_table(size_t num_entries)
186 /* Allocate 1 more entry since the first entry is not used */
187 qman_fq_lookup_table = vmalloc((num_entries * sizeof(void *)));
188 if (!qman_fq_lookup_table) {
189 pr_err("QMan: Could not allocate fq lookup table\n");
192 memset(qman_fq_lookup_table, 0, num_entries * sizeof(void *));
193 qman_fq_lookup_table_size = num_entries;
194 pr_debug("QMan: Allocated lookup table at %p, entry count %lu\n",
195 qman_fq_lookup_table,
196 (unsigned long)qman_fq_lookup_table_size);
200 /* global structure that maintains fq object mapping */
201 static DEFINE_SPINLOCK(fq_hash_table_lock);
203 static int find_empty_fq_table_entry(u32 *entry, struct qman_fq *fq)
207 spin_lock(&fq_hash_table_lock);
208 /* Can't use index zero because this has special meaning
209 * in context_b field.
211 for (i = 1; i < qman_fq_lookup_table_size; i++) {
212 if (qman_fq_lookup_table[i] == NULL) {
214 qman_fq_lookup_table[i] = fq;
215 spin_unlock(&fq_hash_table_lock);
219 spin_unlock(&fq_hash_table_lock);
223 static void clear_fq_table_entry(u32 entry)
225 spin_lock(&fq_hash_table_lock);
226 DPAA_BUG_ON(entry >= qman_fq_lookup_table_size);
227 qman_fq_lookup_table[entry] = NULL;
228 spin_unlock(&fq_hash_table_lock);
231 static inline struct qman_fq *get_fq_table_entry(u32 entry)
233 DPAA_BUG_ON(entry >= qman_fq_lookup_table_size);
234 return qman_fq_lookup_table[entry];
238 static inline void cpu_to_hw_fqd(struct qm_fqd *fqd)
240 /* Byteswap the FQD to HW format */
241 fqd->fq_ctrl = cpu_to_be16(fqd->fq_ctrl);
242 fqd->dest_wq = cpu_to_be16(fqd->dest_wq);
243 fqd->ics_cred = cpu_to_be16(fqd->ics_cred);
244 fqd->context_b = cpu_to_be32(fqd->context_b);
245 fqd->context_a.opaque = cpu_to_be64(fqd->context_a.opaque);
246 fqd->opaque_td = cpu_to_be16(fqd->opaque_td);
249 static inline void hw_fqd_to_cpu(struct qm_fqd *fqd)
251 /* Byteswap the FQD to CPU format */
252 fqd->fq_ctrl = be16_to_cpu(fqd->fq_ctrl);
253 fqd->dest_wq = be16_to_cpu(fqd->dest_wq);
254 fqd->ics_cred = be16_to_cpu(fqd->ics_cred);
255 fqd->context_b = be32_to_cpu(fqd->context_b);
256 fqd->context_a.opaque = be64_to_cpu(fqd->context_a.opaque);
259 static inline void cpu_to_hw_fd(struct qm_fd *fd)
261 fd->addr = cpu_to_be40(fd->addr);
262 fd->status = cpu_to_be32(fd->status);
263 fd->opaque = cpu_to_be32(fd->opaque);
266 static inline void hw_fd_to_cpu(struct qm_fd *fd)
268 fd->addr = be40_to_cpu(fd->addr);
269 fd->status = be32_to_cpu(fd->status);
270 fd->opaque = be32_to_cpu(fd->opaque);
273 /* In the case that slow- and fast-path handling are both done by qman_poll()
274 * (ie. because there is no interrupt handling), we ought to balance how often
275 * we do the fast-path poll versus the slow-path poll. We'll use two decrementer
276 * sources, so we call the fast poll 'n' times before calling the slow poll
277 * once. The idle decrementer constant is used when the last slow-poll detected
278 * no work to do, and the busy decrementer constant when the last slow-poll had
281 #define SLOW_POLL_IDLE 1000
282 #define SLOW_POLL_BUSY 10
283 static u32 __poll_portal_slow(struct qman_portal *p, u32 is);
284 static inline unsigned int __poll_portal_fast(struct qman_portal *p,
285 unsigned int poll_limit);
287 /* Portal interrupt handler */
288 static irqreturn_t portal_isr(__always_unused int irq, void *ptr)
290 struct qman_portal *p = ptr;
292 * The CSCI/CCSCI source is cleared inside __poll_portal_slow(), because
293 * it could race against a Query Congestion State command also given
294 * as part of the handling of this interrupt source. We mustn't
295 * clear it a second time in this top-level function.
297 u32 clear = QM_DQAVAIL_MASK | (p->irq_sources &
298 ~(QM_PIRQ_CSCI | QM_PIRQ_CCSCI));
299 u32 is = qm_isr_status_read(&p->p) & p->irq_sources;
300 /* DQRR-handling if it's interrupt-driven */
301 if (is & QM_PIRQ_DQRI)
302 __poll_portal_fast(p, FSL_QMAN_POLL_LIMIT);
303 /* Handling of anything else that's interrupt-driven */
304 clear |= __poll_portal_slow(p, is);
305 qm_isr_status_clear(&p->p, clear);
309 /* This inner version is used privately by qman_create_affine_portal(), as well
310 * as by the exported qman_stop_dequeues().
312 static inline void qman_stop_dequeues_ex(struct qman_portal *p)
314 if (!(p->dqrr_disable_ref++))
315 qm_dqrr_set_maxfill(&p->p, 0);
318 static int drain_mr_fqrni(struct qm_portal *p)
320 const struct qm_mr_entry *msg;
322 msg = qm_mr_current(p);
325 * if MR was full and h/w had other FQRNI entries to produce, we
326 * need to allow it time to produce those entries once the
327 * existing entries are consumed. A worst-case situation
328 * (fully-loaded system) means h/w sequencers may have to do 3-4
329 * other things before servicing the portal's MR pump, each of
330 * which (if slow) may take ~50 qman cycles (which is ~200
331 * processor cycles). So rounding up and then multiplying this
332 * worst-case estimate by a factor of 10, just to be
333 * ultra-paranoid, goes as high as 10,000 cycles. NB, we consume
334 * one entry at a time, so h/w has an opportunity to produce new
335 * entries well before the ring has been fully consumed, so
336 * we're being *really* paranoid here.
338 u64 now, then = mfatb();
342 } while ((then + 10000) > now);
343 msg = qm_mr_current(p);
347 if ((msg->verb & QM_MR_VERB_TYPE_MASK) != QM_MR_VERB_FQRNI) {
348 /* We aren't draining anything but FQRNIs */
349 pr_err("Found verb 0x%x in MR\n", msg->verb);
353 qm_mr_cci_consume(p, 1);
357 static inline int qm_eqcr_init(struct qm_portal *portal,
358 enum qm_eqcr_pmode pmode,
359 unsigned int eq_stash_thresh,
362 /* This use of 'register', as well as all other occurrences, is because
363 * it has been observed to generate much faster code with gcc than is
364 * otherwise the case.
366 register struct qm_eqcr *eqcr = &portal->eqcr;
370 eqcr->ring = portal->addr.ce + QM_CL_EQCR;
371 eqcr->ci = qm_in(EQCR_CI_CINH) & (QM_EQCR_SIZE - 1);
372 qm_cl_invalidate(EQCR_CI);
373 pi = qm_in(EQCR_PI_CINH) & (QM_EQCR_SIZE - 1);
374 eqcr->cursor = eqcr->ring + pi;
375 eqcr->vbit = (qm_in(EQCR_PI_CINH) & QM_EQCR_SIZE) ?
376 QM_EQCR_VERB_VBIT : 0;
377 eqcr->available = QM_EQCR_SIZE - 1 -
378 qm_cyc_diff(QM_EQCR_SIZE, eqcr->ci, pi);
379 eqcr->ithresh = qm_in(EQCR_ITR);
380 #ifdef RTE_LIBRTE_DPAA_HWDEBUG
384 cfg = (qm_in(CFG) & 0x00ffffff) |
385 (eq_stash_thresh << 28) | /* QCSP_CFG: EST */
386 (eq_stash_prio << 26) | /* QCSP_CFG: EP */
387 ((pmode & 0x3) << 24); /* QCSP_CFG::EPM */
392 static inline void qm_eqcr_finish(struct qm_portal *portal)
394 register struct qm_eqcr *eqcr = &portal->eqcr;
399 * Disable EQCI stashing because the QMan only
400 * presents the value it previously stashed to
401 * maintain coherency. Setting the stash threshold
402 * to 1 then 0 ensures that QMan has resyncronized
403 * its internal copy so that the portal is clean
404 * when it is reinitialized in the future
406 cfg = (qm_in(CFG) & 0x0fffffff) |
407 (1 << 28); /* QCSP_CFG: EST */
409 cfg &= 0x0fffffff; /* stash threshold = 0 */
412 pi = qm_in(EQCR_PI_CINH) & (QM_EQCR_SIZE - 1);
413 ci = qm_in(EQCR_CI_CINH) & (QM_EQCR_SIZE - 1);
415 /* Refresh EQCR CI cache value */
416 qm_cl_invalidate(EQCR_CI);
417 eqcr->ci = qm_cl_in(EQCR_CI) & (QM_EQCR_SIZE - 1);
419 #ifdef RTE_LIBRTE_DPAA_HWDEBUG
420 DPAA_ASSERT(!eqcr->busy);
422 if (pi != EQCR_PTR2IDX(eqcr->cursor))
423 pr_crit("losing uncommitted EQCR entries\n");
425 pr_crit("missing existing EQCR completions\n");
426 if (eqcr->ci != EQCR_PTR2IDX(eqcr->cursor))
427 pr_crit("EQCR destroyed unquiesced\n");
430 static inline int qm_dqrr_init(struct qm_portal *portal,
431 __maybe_unused const struct qm_portal_config *config,
432 enum qm_dqrr_dmode dmode,
433 __maybe_unused enum qm_dqrr_pmode pmode,
434 enum qm_dqrr_cmode cmode, u8 max_fill)
436 register struct qm_dqrr *dqrr = &portal->dqrr;
439 /* Make sure the DQRR will be idle when we enable */
440 qm_out(DQRR_SDQCR, 0);
441 qm_out(DQRR_VDQCR, 0);
442 qm_out(DQRR_PDQCR, 0);
443 dqrr->ring = portal->addr.ce + QM_CL_DQRR;
444 dqrr->pi = qm_in(DQRR_PI_CINH) & (QM_DQRR_SIZE - 1);
445 dqrr->ci = qm_in(DQRR_CI_CINH) & (QM_DQRR_SIZE - 1);
446 dqrr->cursor = dqrr->ring + dqrr->ci;
447 dqrr->fill = qm_cyc_diff(QM_DQRR_SIZE, dqrr->ci, dqrr->pi);
448 dqrr->vbit = (qm_in(DQRR_PI_CINH) & QM_DQRR_SIZE) ?
449 QM_DQRR_VERB_VBIT : 0;
450 dqrr->ithresh = qm_in(DQRR_ITR);
451 #ifdef RTE_LIBRTE_DPAA_HWDEBUG
456 /* Invalidate every ring entry before beginning */
457 for (cfg = 0; cfg < QM_DQRR_SIZE; cfg++)
458 dccivac(qm_cl(dqrr->ring, cfg));
459 cfg = (qm_in(CFG) & 0xff000f00) |
460 ((max_fill & (QM_DQRR_SIZE - 1)) << 20) | /* DQRR_MF */
461 ((dmode & 1) << 18) | /* DP */
462 ((cmode & 3) << 16) | /* DCM */
464 (0 ? 0x40 : 0) | /* Ignore RP */
465 (0 ? 0x10 : 0); /* Ignore SP */
467 qm_dqrr_set_maxfill(portal, max_fill);
471 static inline void qm_dqrr_finish(struct qm_portal *portal)
473 __maybe_unused register struct qm_dqrr *dqrr = &portal->dqrr;
474 #ifdef RTE_LIBRTE_DPAA_HWDEBUG
475 if ((dqrr->cmode != qm_dqrr_cdc) &&
476 (dqrr->ci != DQRR_PTR2IDX(dqrr->cursor)))
477 pr_crit("Ignoring completed DQRR entries\n");
481 static inline int qm_mr_init(struct qm_portal *portal,
482 __maybe_unused enum qm_mr_pmode pmode,
483 enum qm_mr_cmode cmode)
485 register struct qm_mr *mr = &portal->mr;
488 mr->ring = portal->addr.ce + QM_CL_MR;
489 mr->pi = qm_in(MR_PI_CINH) & (QM_MR_SIZE - 1);
490 mr->ci = qm_in(MR_CI_CINH) & (QM_MR_SIZE - 1);
491 mr->cursor = mr->ring + mr->ci;
492 mr->fill = qm_cyc_diff(QM_MR_SIZE, mr->ci, mr->pi);
493 mr->vbit = (qm_in(MR_PI_CINH) & QM_MR_SIZE) ? QM_MR_VERB_VBIT : 0;
494 mr->ithresh = qm_in(MR_ITR);
495 #ifdef RTE_LIBRTE_DPAA_HWDEBUG
499 cfg = (qm_in(CFG) & 0xfffff0ff) |
500 ((cmode & 1) << 8); /* QCSP_CFG:MM */
505 static inline void qm_mr_pvb_update(struct qm_portal *portal)
507 register struct qm_mr *mr = &portal->mr;
508 const struct qm_mr_entry *res = qm_cl(mr->ring, mr->pi);
510 #ifdef RTE_LIBRTE_DPAA_HWDEBUG
511 DPAA_ASSERT(mr->pmode == qm_mr_pvb);
513 /* when accessing 'verb', use __raw_readb() to ensure that compiler
514 * inlining doesn't try to optimise out "excess reads".
516 if ((__raw_readb(&res->verb) & QM_MR_VERB_VBIT) == mr->vbit) {
517 mr->pi = (mr->pi + 1) & (QM_MR_SIZE - 1);
519 mr->vbit ^= QM_MR_VERB_VBIT;
527 struct qman_portal *qman_create_portal(
528 struct qman_portal *portal,
529 const struct qm_portal_config *c,
530 const struct qman_cgrs *cgrs)
539 portal->use_eqcr_ci_stashing = ((qman_ip_rev >= QMAN_REV30) ? 1 : 0);
541 * prep the low-level portal struct with the mapped addresses from the
542 * config, everything that follows depends on it and "config" is more
545 p->addr.ce = c->addr_virt[DPAA_PORTAL_CE];
546 p->addr.ci = c->addr_virt[DPAA_PORTAL_CI];
548 * If CI-stashing is used, the current defaults use a threshold of 3,
549 * and stash with high-than-DQRR priority.
551 if (qm_eqcr_init(p, qm_eqcr_pvb,
552 portal->use_eqcr_ci_stashing ? 3 : 0, 1)) {
553 pr_err("Qman EQCR initialisation failed\n");
556 if (qm_dqrr_init(p, c, qm_dqrr_dpush, qm_dqrr_pvb,
557 qm_dqrr_cdc, DQRR_MAXFILL)) {
558 pr_err("Qman DQRR initialisation failed\n");
561 if (qm_mr_init(p, qm_mr_pvb, qm_mr_cci)) {
562 pr_err("Qman MR initialisation failed\n");
566 pr_err("Qman MC initialisation failed\n");
570 /* static interrupt-gating controls */
571 qm_dqrr_set_ithresh(p, 0);
572 qm_mr_set_ithresh(p, 0);
573 qm_isr_set_iperiod(p, 0);
574 portal->cgrs = kmalloc(2 * sizeof(*cgrs), GFP_KERNEL);
577 /* initial snapshot is no-depletion */
578 qman_cgrs_init(&portal->cgrs[1]);
580 portal->cgrs[0] = *cgrs;
582 /* if the given mask is NULL, assume all CGRs can be seen */
583 qman_cgrs_fill(&portal->cgrs[0]);
584 INIT_LIST_HEAD(&portal->cgr_cbs);
585 spin_lock_init(&portal->cgr_lock);
587 portal->slowpoll = 0;
588 portal->sdqcr = QM_SDQCR_SOURCE_CHANNELS | QM_SDQCR_COUNT_UPTO3 |
589 QM_SDQCR_DEDICATED_PRECEDENCE | QM_SDQCR_TYPE_PRIO_QOS |
590 QM_SDQCR_TOKEN_SET(0xab) | QM_SDQCR_CHANNELS_DEDICATED;
591 portal->dqrr_disable_ref = 0;
592 portal->cb_dc_ern = NULL;
593 sprintf(buf, "qportal-%d", c->channel);
594 dpa_rbtree_init(&portal->retire_table);
596 qm_isr_disable_write(p, isdr);
597 portal->irq_sources = 0;
598 qm_isr_enable_write(p, portal->irq_sources);
599 qm_isr_status_clear(p, 0xffffffff);
600 snprintf(portal->irqname, MAX_IRQNAME, IRQNAME, c->cpu);
601 if (request_irq(c->irq, portal_isr, 0, portal->irqname,
603 pr_err("request_irq() failed\n");
607 /* Need EQCR to be empty before continuing */
608 isdr &= ~QM_PIRQ_EQCI;
609 qm_isr_disable_write(p, isdr);
610 ret = qm_eqcr_get_fill(p);
612 pr_err("Qman EQCR unclean\n");
613 goto fail_eqcr_empty;
615 isdr &= ~(QM_PIRQ_DQRI | QM_PIRQ_MRI);
616 qm_isr_disable_write(p, isdr);
617 if (qm_dqrr_current(p)) {
618 pr_err("Qman DQRR unclean\n");
619 qm_dqrr_cdc_consume_n(p, 0xffff);
621 if (qm_mr_current(p) && drain_mr_fqrni(p)) {
622 /* special handling, drain just in case it's a few FQRNIs */
623 if (drain_mr_fqrni(p))
624 goto fail_dqrr_mr_empty;
628 qm_isr_disable_write(p, 0);
630 /* Write a sane SDQCR */
631 qm_dqrr_sdqcr_set(p, portal->sdqcr);
635 free_irq(c->irq, portal);
638 spin_lock_destroy(&portal->cgr_lock);
651 struct qman_portal *qman_create_affine_portal(const struct qm_portal_config *c,
652 const struct qman_cgrs *cgrs)
654 struct qman_portal *res;
655 struct qman_portal *portal = get_affine_portal();
656 /* A criteria for calling this function (from qman_driver.c) is that
657 * we're already affine to the cpu and won't schedule onto another cpu.
660 res = qman_create_portal(portal, c, cgrs);
662 spin_lock(&affine_mask_lock);
663 CPU_SET(c->cpu, &affine_mask);
664 affine_channels[c->cpu] =
666 spin_unlock(&affine_mask_lock);
672 void qman_destroy_portal(struct qman_portal *qm)
674 const struct qm_portal_config *pcfg;
676 /* Stop dequeues on the portal */
677 qm_dqrr_sdqcr_set(&qm->p, 0);
680 * NB we do this to "quiesce" EQCR. If we add enqueue-completions or
681 * something related to QM_PIRQ_EQCI, this may need fixing.
682 * Also, due to the prefetching model used for CI updates in the enqueue
683 * path, this update will only invalidate the CI cacheline *after*
684 * working on it, so we need to call this twice to ensure a full update
685 * irrespective of where the enqueue processing was at when the teardown
688 qm_eqcr_cce_update(&qm->p);
689 qm_eqcr_cce_update(&qm->p);
692 free_irq(pcfg->irq, qm);
695 qm_mc_finish(&qm->p);
696 qm_mr_finish(&qm->p);
697 qm_dqrr_finish(&qm->p);
698 qm_eqcr_finish(&qm->p);
702 spin_lock_destroy(&qm->cgr_lock);
705 const struct qm_portal_config *qman_destroy_affine_portal(void)
707 /* We don't want to redirect if we're a slave, use "raw" */
708 struct qman_portal *qm = get_affine_portal();
709 const struct qm_portal_config *pcfg;
715 qman_destroy_portal(qm);
717 spin_lock(&affine_mask_lock);
718 CPU_CLR(cpu, &affine_mask);
719 spin_unlock(&affine_mask_lock);
723 int qman_get_portal_index(void)
725 struct qman_portal *p = get_affine_portal();
726 return p->config->index;
729 /* Inline helper to reduce nesting in __poll_portal_slow() */
730 static inline void fq_state_change(struct qman_portal *p, struct qman_fq *fq,
731 const struct qm_mr_entry *msg, u8 verb)
735 case QM_MR_VERB_FQRL:
736 DPAA_ASSERT(fq_isset(fq, QMAN_FQ_STATE_ORL));
737 fq_clear(fq, QMAN_FQ_STATE_ORL);
740 case QM_MR_VERB_FQRN:
741 DPAA_ASSERT((fq->state == qman_fq_state_parked) ||
742 (fq->state == qman_fq_state_sched));
743 DPAA_ASSERT(fq_isset(fq, QMAN_FQ_STATE_CHANGING));
744 fq_clear(fq, QMAN_FQ_STATE_CHANGING);
745 if (msg->fq.fqs & QM_MR_FQS_NOTEMPTY)
746 fq_set(fq, QMAN_FQ_STATE_NE);
747 if (msg->fq.fqs & QM_MR_FQS_ORLPRESENT)
748 fq_set(fq, QMAN_FQ_STATE_ORL);
751 fq->state = qman_fq_state_retired;
753 case QM_MR_VERB_FQPN:
754 DPAA_ASSERT(fq->state == qman_fq_state_sched);
755 DPAA_ASSERT(fq_isclear(fq, QMAN_FQ_STATE_CHANGING));
756 fq->state = qman_fq_state_parked;
761 static u32 __poll_portal_slow(struct qman_portal *p, u32 is)
763 const struct qm_mr_entry *msg;
764 struct qm_mr_entry swapped_msg;
766 if (is & QM_PIRQ_CSCI) {
767 struct qman_cgrs rr, c;
768 struct qm_mc_result *mcr;
769 struct qman_cgr *cgr;
771 spin_lock(&p->cgr_lock);
773 * The CSCI bit must be cleared _before_ issuing the
774 * Query Congestion State command, to ensure that a long
775 * CGR State Change callback cannot miss an intervening
778 qm_isr_status_clear(&p->p, QM_PIRQ_CSCI);
780 qm_mc_commit(&p->p, QM_MCC_VERB_QUERYCONGESTION);
781 while (!(mcr = qm_mc_result(&p->p)))
783 /* mask out the ones I'm not interested in */
784 qman_cgrs_and(&rr, (const struct qman_cgrs *)
785 &mcr->querycongestion.state, &p->cgrs[0]);
786 /* check previous snapshot for delta, enter/exit congestion */
787 qman_cgrs_xor(&c, &rr, &p->cgrs[1]);
788 /* update snapshot */
789 qman_cgrs_cp(&p->cgrs[1], &rr);
790 /* Invoke callback */
791 list_for_each_entry(cgr, &p->cgr_cbs, node)
792 if (cgr->cb && qman_cgrs_get(&c, cgr->cgrid))
793 cgr->cb(p, cgr, qman_cgrs_get(&rr, cgr->cgrid));
794 spin_unlock(&p->cgr_lock);
797 if (is & QM_PIRQ_EQRI) {
798 qm_eqcr_cce_update(&p->p);
799 qm_eqcr_set_ithresh(&p->p, 0);
800 wake_up(&affine_queue);
803 if (is & QM_PIRQ_MRI) {
807 qm_mr_pvb_update(&p->p);
808 msg = qm_mr_current(&p->p);
812 hw_fd_to_cpu(&swapped_msg.ern.fd);
813 verb = msg->verb & QM_MR_VERB_TYPE_MASK;
814 /* The message is a software ERN iff the 0x20 bit is set */
817 case QM_MR_VERB_FQRNI:
818 /* nada, we drop FQRNIs on the floor */
820 case QM_MR_VERB_FQRN:
821 case QM_MR_VERB_FQRL:
822 /* Lookup in the retirement table */
823 fq = table_find_fq(p,
824 be32_to_cpu(msg->fq.fqid));
826 fq_state_change(p, fq, &swapped_msg, verb);
828 fq->cb.fqs(p, fq, &swapped_msg);
830 case QM_MR_VERB_FQPN:
832 #ifdef CONFIG_FSL_QMAN_FQ_LOOKUP
833 fq = get_fq_table_entry(
834 be32_to_cpu(msg->fq.contextB));
836 fq = (void *)(uintptr_t)
837 be32_to_cpu(msg->fq.contextB);
839 fq_state_change(p, fq, msg, verb);
841 fq->cb.fqs(p, fq, &swapped_msg);
843 case QM_MR_VERB_DC_ERN:
846 p->cb_dc_ern(p, msg);
850 static int warn_once;
853 pr_crit("Leaking DCP ERNs!\n");
859 pr_crit("Invalid MR verb 0x%02x\n", verb);
862 /* Its a software ERN */
863 #ifdef CONFIG_FSL_QMAN_FQ_LOOKUP
864 fq = get_fq_table_entry(be32_to_cpu(msg->ern.tag));
866 fq = (void *)(uintptr_t)be32_to_cpu(msg->ern.tag);
868 fq->cb.ern(p, fq, &swapped_msg);
874 qm_mr_cci_consume(&p->p, num);
877 * QM_PIRQ_CSCI/CCSCI has already been cleared, as part of its specific
878 * processing. If that interrupt source has meanwhile been re-asserted,
879 * we mustn't clear it here (or in the top-level interrupt handler).
881 return is & (QM_PIRQ_EQCI | QM_PIRQ_EQRI | QM_PIRQ_MRI);
885 * remove some slowish-path stuff from the "fast path" and make sure it isn't
888 static noinline void clear_vdqcr(struct qman_portal *p, struct qman_fq *fq)
890 p->vdqcr_owned = NULL;
892 fq_clear(fq, QMAN_FQ_STATE_VDQCR);
894 wake_up(&affine_queue);
898 * The only states that would conflict with other things if they ran at the
899 * same time on the same cpu are:
901 * (i) setting/clearing vdqcr_owned, and
902 * (ii) clearing the NE (Not Empty) flag.
904 * Both are safe. Because;
906 * (i) this clearing can only occur after qman_set_vdq() has set the
907 * vdqcr_owned field (which it does before setting VDQCR), and
908 * qman_volatile_dequeue() blocks interrupts and preemption while this is
909 * done so that we can't interfere.
910 * (ii) the NE flag is only cleared after qman_retire_fq() has set it, and as
911 * with (i) that API prevents us from interfering until it's safe.
913 * The good thing is that qman_set_vdq() and qman_retire_fq() run far
914 * less frequently (ie. per-FQ) than __poll_portal_fast() does, so the nett
915 * advantage comes from this function not having to "lock" anything at all.
917 * Note also that the callbacks are invoked at points which are safe against the
918 * above potential conflicts, but that this function itself is not re-entrant
919 * (this is because the function tracks one end of each FIFO in the portal and
920 * we do *not* want to lock that). So the consequence is that it is safe for
921 * user callbacks to call into any QMan API.
923 static inline unsigned int __poll_portal_fast(struct qman_portal *p,
924 unsigned int poll_limit)
926 const struct qm_dqrr_entry *dq;
928 enum qman_cb_dqrr_result res;
929 unsigned int limit = 0;
930 #if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__
931 struct qm_dqrr_entry *shadow;
934 qm_dqrr_pvb_update(&p->p);
935 dq = qm_dqrr_current(&p->p);
938 #if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__
939 /* If running on an LE system the fields of the
940 * dequeue entry must be swapper. Because the
941 * QMan HW will ignore writes the DQRR entry is
942 * copied and the index stored within the copy
944 shadow = &p->shadow_dqrr[DQRR_PTR2IDX(dq)];
947 shadow->fqid = be32_to_cpu(shadow->fqid);
948 shadow->contextB = be32_to_cpu(shadow->contextB);
949 shadow->seqnum = be16_to_cpu(shadow->seqnum);
950 hw_fd_to_cpu(&shadow->fd);
953 if (dq->stat & QM_DQRR_STAT_UNSCHEDULED) {
955 * VDQCR: don't trust context_b as the FQ may have
956 * been configured for h/w consumption and we're
957 * draining it post-retirement.
961 * We only set QMAN_FQ_STATE_NE when retiring, so we
962 * only need to check for clearing it when doing
963 * volatile dequeues. It's one less thing to check
964 * in the critical path (SDQCR).
966 if (dq->stat & QM_DQRR_STAT_FQ_EMPTY)
967 fq_clear(fq, QMAN_FQ_STATE_NE);
969 * This is duplicated from the SDQCR code, but we
970 * have stuff to do before *and* after this callback,
971 * and we don't want multiple if()s in the critical
974 res = fq->cb.dqrr(p, fq, dq);
975 if (res == qman_cb_dqrr_stop)
977 /* Check for VDQCR completion */
978 if (dq->stat & QM_DQRR_STAT_DQCR_EXPIRED)
981 /* SDQCR: context_b points to the FQ */
982 #ifdef CONFIG_FSL_QMAN_FQ_LOOKUP
983 fq = get_fq_table_entry(dq->contextB);
985 fq = (void *)(uintptr_t)dq->contextB;
987 /* Now let the callback do its stuff */
988 res = fq->cb.dqrr(p, fq, dq);
990 * The callback can request that we exit without
991 * consuming this entry nor advancing;
993 if (res == qman_cb_dqrr_stop)
996 /* Interpret 'dq' from a driver perspective. */
998 * Parking isn't possible unless HELDACTIVE was set. NB,
999 * FORCEELIGIBLE implies HELDACTIVE, so we only need to
1000 * check for HELDACTIVE to cover both.
1002 DPAA_ASSERT((dq->stat & QM_DQRR_STAT_FQ_HELDACTIVE) ||
1003 (res != qman_cb_dqrr_park));
1004 /* just means "skip it, I'll consume it myself later on" */
1005 if (res != qman_cb_dqrr_defer)
1006 qm_dqrr_cdc_consume_1ptr(&p->p, dq,
1007 res == qman_cb_dqrr_park);
1009 qm_dqrr_next(&p->p);
1011 * Entry processed and consumed, increment our counter. The
1012 * callback can request that we exit after consuming the
1013 * entry, and we also exit if we reach our processing limit,
1014 * so loop back only if neither of these conditions is met.
1016 } while (++limit < poll_limit && res != qman_cb_dqrr_consume_stop);
1021 u16 qman_affine_channel(int cpu)
1024 struct qman_portal *portal = get_affine_portal();
1026 cpu = portal->config->cpu;
1028 DPAA_BUG_ON(!CPU_ISSET(cpu, &affine_mask));
1029 return affine_channels[cpu];
1032 struct qm_dqrr_entry *qman_dequeue(struct qman_fq *fq)
1034 struct qman_portal *p = get_affine_portal();
1035 const struct qm_dqrr_entry *dq;
1036 #if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__
1037 struct qm_dqrr_entry *shadow;
1040 qm_dqrr_pvb_update(&p->p);
1041 dq = qm_dqrr_current(&p->p);
1045 if (!(dq->stat & QM_DQRR_STAT_FD_VALID)) {
1046 /* Invalid DQRR - put the portal and consume the DQRR.
1047 * Return NULL to user as no packet is seen.
1049 qman_dqrr_consume(fq, (struct qm_dqrr_entry *)dq);
1053 #if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__
1054 shadow = &p->shadow_dqrr[DQRR_PTR2IDX(dq)];
1057 shadow->fqid = be32_to_cpu(shadow->fqid);
1058 shadow->contextB = be32_to_cpu(shadow->contextB);
1059 shadow->seqnum = be16_to_cpu(shadow->seqnum);
1060 hw_fd_to_cpu(&shadow->fd);
1063 if (dq->stat & QM_DQRR_STAT_FQ_EMPTY)
1064 fq_clear(fq, QMAN_FQ_STATE_NE);
1066 return (struct qm_dqrr_entry *)dq;
1069 void qman_dqrr_consume(struct qman_fq *fq,
1070 struct qm_dqrr_entry *dq)
1072 struct qman_portal *p = get_affine_portal();
1074 if (dq->stat & QM_DQRR_STAT_DQCR_EXPIRED)
1077 qm_dqrr_cdc_consume_1ptr(&p->p, dq, 0);
1078 qm_dqrr_next(&p->p);
1081 int qman_poll_dqrr(unsigned int limit)
1083 struct qman_portal *p = get_affine_portal();
1086 ret = __poll_portal_fast(p, limit);
1090 void qman_poll(void)
1092 struct qman_portal *p = get_affine_portal();
1094 if ((~p->irq_sources) & QM_PIRQ_SLOW) {
1095 if (!(p->slowpoll--)) {
1096 u32 is = qm_isr_status_read(&p->p) & ~p->irq_sources;
1097 u32 active = __poll_portal_slow(p, is);
1100 qm_isr_status_clear(&p->p, active);
1101 p->slowpoll = SLOW_POLL_BUSY;
1103 p->slowpoll = SLOW_POLL_IDLE;
1106 if ((~p->irq_sources) & QM_PIRQ_DQRI)
1107 __poll_portal_fast(p, FSL_QMAN_POLL_LIMIT);
1110 void qman_stop_dequeues(void)
1112 struct qman_portal *p = get_affine_portal();
1114 qman_stop_dequeues_ex(p);
1117 void qman_start_dequeues(void)
1119 struct qman_portal *p = get_affine_portal();
1121 DPAA_ASSERT(p->dqrr_disable_ref > 0);
1122 if (!(--p->dqrr_disable_ref))
1123 qm_dqrr_set_maxfill(&p->p, DQRR_MAXFILL);
1126 void qman_static_dequeue_add(u32 pools)
1128 struct qman_portal *p = get_affine_portal();
1130 pools &= p->config->pools;
1132 qm_dqrr_sdqcr_set(&p->p, p->sdqcr);
1135 void qman_static_dequeue_del(u32 pools)
1137 struct qman_portal *p = get_affine_portal();
1139 pools &= p->config->pools;
1141 qm_dqrr_sdqcr_set(&p->p, p->sdqcr);
1144 u32 qman_static_dequeue_get(void)
1146 struct qman_portal *p = get_affine_portal();
1150 void qman_dca(struct qm_dqrr_entry *dq, int park_request)
1152 struct qman_portal *p = get_affine_portal();
1154 qm_dqrr_cdc_consume_1ptr(&p->p, dq, park_request);
1157 /* Frame queue API */
1158 static const char *mcr_result_str(u8 result)
1161 case QM_MCR_RESULT_NULL:
1162 return "QM_MCR_RESULT_NULL";
1163 case QM_MCR_RESULT_OK:
1164 return "QM_MCR_RESULT_OK";
1165 case QM_MCR_RESULT_ERR_FQID:
1166 return "QM_MCR_RESULT_ERR_FQID";
1167 case QM_MCR_RESULT_ERR_FQSTATE:
1168 return "QM_MCR_RESULT_ERR_FQSTATE";
1169 case QM_MCR_RESULT_ERR_NOTEMPTY:
1170 return "QM_MCR_RESULT_ERR_NOTEMPTY";
1171 case QM_MCR_RESULT_PENDING:
1172 return "QM_MCR_RESULT_PENDING";
1173 case QM_MCR_RESULT_ERR_BADCOMMAND:
1174 return "QM_MCR_RESULT_ERR_BADCOMMAND";
1176 return "<unknown MCR result>";
1179 int qman_create_fq(u32 fqid, u32 flags, struct qman_fq *fq)
1182 struct qm_mcr_queryfq_np np;
1183 struct qm_mc_command *mcc;
1184 struct qm_mc_result *mcr;
1185 struct qman_portal *p;
1187 if (flags & QMAN_FQ_FLAG_DYNAMIC_FQID) {
1188 int ret = qman_alloc_fqid(&fqid);
1193 spin_lock_init(&fq->fqlock);
1196 fq->state = qman_fq_state_oos;
1197 fq->cgr_groupid = 0;
1198 #ifdef CONFIG_FSL_QMAN_FQ_LOOKUP
1199 if (unlikely(find_empty_fq_table_entry(&fq->key, fq))) {
1200 pr_info("Find empty table entry failed\n");
1204 if (!(flags & QMAN_FQ_FLAG_AS_IS) || (flags & QMAN_FQ_FLAG_NO_MODIFY))
1206 /* Everything else is AS_IS support */
1207 p = get_affine_portal();
1208 mcc = qm_mc_start(&p->p);
1209 mcc->queryfq.fqid = cpu_to_be32(fqid);
1210 qm_mc_commit(&p->p, QM_MCC_VERB_QUERYFQ);
1211 while (!(mcr = qm_mc_result(&p->p)))
1213 DPAA_ASSERT((mcr->verb & QM_MCR_VERB_MASK) == QM_MCC_VERB_QUERYFQ);
1214 if (mcr->result != QM_MCR_RESULT_OK) {
1215 pr_err("QUERYFQ failed: %s\n", mcr_result_str(mcr->result));
1218 fqd = mcr->queryfq.fqd;
1219 hw_fqd_to_cpu(&fqd);
1220 mcc = qm_mc_start(&p->p);
1221 mcc->queryfq_np.fqid = cpu_to_be32(fqid);
1222 qm_mc_commit(&p->p, QM_MCC_VERB_QUERYFQ_NP);
1223 while (!(mcr = qm_mc_result(&p->p)))
1225 DPAA_ASSERT((mcr->verb & QM_MCR_VERB_MASK) == QM_MCC_VERB_QUERYFQ_NP);
1226 if (mcr->result != QM_MCR_RESULT_OK) {
1227 pr_err("QUERYFQ_NP failed: %s\n", mcr_result_str(mcr->result));
1230 np = mcr->queryfq_np;
1231 /* Phew, have queryfq and queryfq_np results, stitch together
1232 * the FQ object from those.
1234 fq->cgr_groupid = fqd.cgid;
1235 switch (np.state & QM_MCR_NP_STATE_MASK) {
1236 case QM_MCR_NP_STATE_OOS:
1238 case QM_MCR_NP_STATE_RETIRED:
1239 fq->state = qman_fq_state_retired;
1241 fq_set(fq, QMAN_FQ_STATE_NE);
1243 case QM_MCR_NP_STATE_TEN_SCHED:
1244 case QM_MCR_NP_STATE_TRU_SCHED:
1245 case QM_MCR_NP_STATE_ACTIVE:
1246 fq->state = qman_fq_state_sched;
1247 if (np.state & QM_MCR_NP_STATE_R)
1248 fq_set(fq, QMAN_FQ_STATE_CHANGING);
1250 case QM_MCR_NP_STATE_PARKED:
1251 fq->state = qman_fq_state_parked;
1254 DPAA_ASSERT(NULL == "invalid FQ state");
1256 if (fqd.fq_ctrl & QM_FQCTRL_CGE)
1257 fq->state |= QMAN_FQ_STATE_CGR_EN;
1260 if (flags & QMAN_FQ_FLAG_DYNAMIC_FQID)
1261 qman_release_fqid(fqid);
1265 void qman_destroy_fq(struct qman_fq *fq, u32 flags __maybe_unused)
1268 * We don't need to lock the FQ as it is a pre-condition that the FQ be
1269 * quiesced. Instead, run some checks.
1271 switch (fq->state) {
1272 case qman_fq_state_parked:
1273 DPAA_ASSERT(flags & QMAN_FQ_DESTROY_PARKED);
1275 case qman_fq_state_oos:
1276 if (fq_isset(fq, QMAN_FQ_FLAG_DYNAMIC_FQID))
1277 qman_release_fqid(fq->fqid);
1278 #ifdef CONFIG_FSL_QMAN_FQ_LOOKUP
1279 clear_fq_table_entry(fq->key);
1285 DPAA_ASSERT(NULL == "qman_free_fq() on unquiesced FQ!");
1288 u32 qman_fq_fqid(struct qman_fq *fq)
1293 void qman_fq_state(struct qman_fq *fq, enum qman_fq_state *state, u32 *flags)
1301 int qman_init_fq(struct qman_fq *fq, u32 flags, struct qm_mcc_initfq *opts)
1303 struct qm_mc_command *mcc;
1304 struct qm_mc_result *mcr;
1305 struct qman_portal *p;
1307 u8 res, myverb = (flags & QMAN_INITFQ_FLAG_SCHED) ?
1308 QM_MCC_VERB_INITFQ_SCHED : QM_MCC_VERB_INITFQ_PARKED;
1310 if ((fq->state != qman_fq_state_oos) &&
1311 (fq->state != qman_fq_state_parked))
1313 #ifdef RTE_LIBRTE_DPAA_HWDEBUG
1314 if (unlikely(fq_isset(fq, QMAN_FQ_FLAG_NO_MODIFY)))
1317 if (opts && (opts->we_mask & QM_INITFQ_WE_OAC)) {
1318 /* And can't be set at the same time as TDTHRESH */
1319 if (opts->we_mask & QM_INITFQ_WE_TDTHRESH)
1322 /* Issue an INITFQ_[PARKED|SCHED] management command */
1323 p = get_affine_portal();
1325 if (unlikely((fq_isset(fq, QMAN_FQ_STATE_CHANGING)) ||
1326 ((fq->state != qman_fq_state_oos) &&
1327 (fq->state != qman_fq_state_parked)))) {
1331 mcc = qm_mc_start(&p->p);
1333 mcc->initfq = *opts;
1334 mcc->initfq.fqid = cpu_to_be32(fq->fqid);
1335 mcc->initfq.count = 0;
1337 * If the FQ does *not* have the TO_DCPORTAL flag, context_b is set as a
1338 * demux pointer. Otherwise, the caller-provided value is allowed to
1339 * stand, don't overwrite it.
1341 if (fq_isclear(fq, QMAN_FQ_FLAG_TO_DCPORTAL)) {
1344 mcc->initfq.we_mask |= QM_INITFQ_WE_CONTEXTB;
1345 #ifdef CONFIG_FSL_QMAN_FQ_LOOKUP
1346 mcc->initfq.fqd.context_b = fq->key;
1348 mcc->initfq.fqd.context_b = (u32)(uintptr_t)fq;
1351 * and the physical address - NB, if the user wasn't trying to
1352 * set CONTEXTA, clear the stashing settings.
1354 if (!(mcc->initfq.we_mask & QM_INITFQ_WE_CONTEXTA)) {
1355 mcc->initfq.we_mask |= QM_INITFQ_WE_CONTEXTA;
1356 memset(&mcc->initfq.fqd.context_a, 0,
1357 sizeof(mcc->initfq.fqd.context_a));
1359 phys_fq = rte_mem_virt2iova(fq);
1360 qm_fqd_stashing_set64(&mcc->initfq.fqd, phys_fq);
1363 if (flags & QMAN_INITFQ_FLAG_LOCAL) {
1364 mcc->initfq.fqd.dest.channel = p->config->channel;
1365 if (!(mcc->initfq.we_mask & QM_INITFQ_WE_DESTWQ)) {
1366 mcc->initfq.we_mask |= QM_INITFQ_WE_DESTWQ;
1367 mcc->initfq.fqd.dest.wq = 4;
1370 mcc->initfq.we_mask = cpu_to_be16(mcc->initfq.we_mask);
1371 cpu_to_hw_fqd(&mcc->initfq.fqd);
1372 qm_mc_commit(&p->p, myverb);
1373 while (!(mcr = qm_mc_result(&p->p)))
1375 DPAA_ASSERT((mcr->verb & QM_MCR_VERB_MASK) == myverb);
1377 if (res != QM_MCR_RESULT_OK) {
1382 if (opts->we_mask & QM_INITFQ_WE_FQCTRL) {
1383 if (opts->fqd.fq_ctrl & QM_FQCTRL_CGE)
1384 fq_set(fq, QMAN_FQ_STATE_CGR_EN);
1386 fq_clear(fq, QMAN_FQ_STATE_CGR_EN);
1388 if (opts->we_mask & QM_INITFQ_WE_CGID)
1389 fq->cgr_groupid = opts->fqd.cgid;
1391 fq->state = (flags & QMAN_INITFQ_FLAG_SCHED) ?
1392 qman_fq_state_sched : qman_fq_state_parked;
1397 int qman_schedule_fq(struct qman_fq *fq)
1399 struct qm_mc_command *mcc;
1400 struct qm_mc_result *mcr;
1401 struct qman_portal *p;
1406 if (fq->state != qman_fq_state_parked)
1408 #ifdef RTE_LIBRTE_DPAA_HWDEBUG
1409 if (unlikely(fq_isset(fq, QMAN_FQ_FLAG_NO_MODIFY)))
1412 /* Issue a ALTERFQ_SCHED management command */
1413 p = get_affine_portal();
1416 if (unlikely((fq_isset(fq, QMAN_FQ_STATE_CHANGING)) ||
1417 (fq->state != qman_fq_state_parked))) {
1421 mcc = qm_mc_start(&p->p);
1422 mcc->alterfq.fqid = cpu_to_be32(fq->fqid);
1423 qm_mc_commit(&p->p, QM_MCC_VERB_ALTER_SCHED);
1424 while (!(mcr = qm_mc_result(&p->p)))
1426 DPAA_ASSERT((mcr->verb & QM_MCR_VERB_MASK) == QM_MCR_VERB_ALTER_SCHED);
1428 if (res != QM_MCR_RESULT_OK) {
1432 fq->state = qman_fq_state_sched;
1439 int qman_retire_fq(struct qman_fq *fq, u32 *flags)
1441 struct qm_mc_command *mcc;
1442 struct qm_mc_result *mcr;
1443 struct qman_portal *p;
1448 if ((fq->state != qman_fq_state_parked) &&
1449 (fq->state != qman_fq_state_sched))
1451 #ifdef RTE_LIBRTE_DPAA_HWDEBUG
1452 if (unlikely(fq_isset(fq, QMAN_FQ_FLAG_NO_MODIFY)))
1455 p = get_affine_portal();
1458 if (unlikely((fq_isset(fq, QMAN_FQ_STATE_CHANGING)) ||
1459 (fq->state == qman_fq_state_retired) ||
1460 (fq->state == qman_fq_state_oos))) {
1464 rval = table_push_fq(p, fq);
1467 mcc = qm_mc_start(&p->p);
1468 mcc->alterfq.fqid = cpu_to_be32(fq->fqid);
1469 qm_mc_commit(&p->p, QM_MCC_VERB_ALTER_RETIRE);
1470 while (!(mcr = qm_mc_result(&p->p)))
1472 DPAA_ASSERT((mcr->verb & QM_MCR_VERB_MASK) == QM_MCR_VERB_ALTER_RETIRE);
1475 * "Elegant" would be to treat OK/PENDING the same way; set CHANGING,
1476 * and defer the flags until FQRNI or FQRN (respectively) show up. But
1477 * "Friendly" is to process OK immediately, and not set CHANGING. We do
1478 * friendly, otherwise the caller doesn't necessarily have a fully
1479 * "retired" FQ on return even if the retirement was immediate. However
1480 * this does mean some code duplication between here and
1481 * fq_state_change().
1483 if (likely(res == QM_MCR_RESULT_OK)) {
1485 /* Process 'fq' right away, we'll ignore FQRNI */
1486 if (mcr->alterfq.fqs & QM_MCR_FQS_NOTEMPTY)
1487 fq_set(fq, QMAN_FQ_STATE_NE);
1488 if (mcr->alterfq.fqs & QM_MCR_FQS_ORLPRESENT)
1489 fq_set(fq, QMAN_FQ_STATE_ORL);
1491 table_del_fq(p, fq);
1494 fq->state = qman_fq_state_retired;
1497 * Another issue with supporting "immediate" retirement
1498 * is that we're forced to drop FQRNIs, because by the
1499 * time they're seen it may already be "too late" (the
1500 * fq may have been OOS'd and free()'d already). But if
1501 * the upper layer wants a callback whether it's
1502 * immediate or not, we have to fake a "MR" entry to
1503 * look like an FQRNI...
1505 struct qm_mr_entry msg;
1507 msg.verb = QM_MR_VERB_FQRNI;
1508 msg.fq.fqs = mcr->alterfq.fqs;
1509 msg.fq.fqid = fq->fqid;
1510 #ifdef CONFIG_FSL_QMAN_FQ_LOOKUP
1511 msg.fq.contextB = fq->key;
1513 msg.fq.contextB = (u32)(uintptr_t)fq;
1515 fq->cb.fqs(p, fq, &msg);
1517 } else if (res == QM_MCR_RESULT_PENDING) {
1519 fq_set(fq, QMAN_FQ_STATE_CHANGING);
1522 table_del_fq(p, fq);
1529 int qman_oos_fq(struct qman_fq *fq)
1531 struct qm_mc_command *mcc;
1532 struct qm_mc_result *mcr;
1533 struct qman_portal *p;
1538 if (fq->state != qman_fq_state_retired)
1540 #ifdef RTE_LIBRTE_DPAA_HWDEBUG
1541 if (unlikely(fq_isset(fq, QMAN_FQ_FLAG_NO_MODIFY)))
1544 p = get_affine_portal();
1546 if (unlikely((fq_isset(fq, QMAN_FQ_STATE_BLOCKOOS)) ||
1547 (fq->state != qman_fq_state_retired))) {
1551 mcc = qm_mc_start(&p->p);
1552 mcc->alterfq.fqid = cpu_to_be32(fq->fqid);
1553 qm_mc_commit(&p->p, QM_MCC_VERB_ALTER_OOS);
1554 while (!(mcr = qm_mc_result(&p->p)))
1556 DPAA_ASSERT((mcr->verb & QM_MCR_VERB_MASK) == QM_MCR_VERB_ALTER_OOS);
1558 if (res != QM_MCR_RESULT_OK) {
1562 fq->state = qman_fq_state_oos;
1568 int qman_fq_flow_control(struct qman_fq *fq, int xon)
1570 struct qm_mc_command *mcc;
1571 struct qm_mc_result *mcr;
1572 struct qman_portal *p;
1578 if ((fq->state == qman_fq_state_oos) ||
1579 (fq->state == qman_fq_state_retired) ||
1580 (fq->state == qman_fq_state_parked))
1583 #ifdef RTE_LIBRTE_DPAA_HWDEBUG
1584 if (unlikely(fq_isset(fq, QMAN_FQ_FLAG_NO_MODIFY)))
1587 /* Issue a ALTER_FQXON or ALTER_FQXOFF management command */
1588 p = get_affine_portal();
1590 if (unlikely((fq_isset(fq, QMAN_FQ_STATE_CHANGING)) ||
1591 (fq->state == qman_fq_state_parked) ||
1592 (fq->state == qman_fq_state_oos) ||
1593 (fq->state == qman_fq_state_retired))) {
1597 mcc = qm_mc_start(&p->p);
1598 mcc->alterfq.fqid = fq->fqid;
1599 mcc->alterfq.count = 0;
1600 myverb = xon ? QM_MCC_VERB_ALTER_FQXON : QM_MCC_VERB_ALTER_FQXOFF;
1602 qm_mc_commit(&p->p, myverb);
1603 while (!(mcr = qm_mc_result(&p->p)))
1605 DPAA_ASSERT((mcr->verb & QM_MCR_VERB_MASK) == myverb);
1608 if (res != QM_MCR_RESULT_OK) {
1617 int qman_query_fq(struct qman_fq *fq, struct qm_fqd *fqd)
1619 struct qm_mc_command *mcc;
1620 struct qm_mc_result *mcr;
1621 struct qman_portal *p = get_affine_portal();
1625 mcc = qm_mc_start(&p->p);
1626 mcc->queryfq.fqid = cpu_to_be32(fq->fqid);
1627 qm_mc_commit(&p->p, QM_MCC_VERB_QUERYFQ);
1628 while (!(mcr = qm_mc_result(&p->p)))
1630 DPAA_ASSERT((mcr->verb & QM_MCR_VERB_MASK) == QM_MCR_VERB_QUERYFQ);
1632 if (res == QM_MCR_RESULT_OK)
1633 *fqd = mcr->queryfq.fqd;
1635 if (res != QM_MCR_RESULT_OK)
1640 int qman_query_fq_has_pkts(struct qman_fq *fq)
1642 struct qm_mc_command *mcc;
1643 struct qm_mc_result *mcr;
1644 struct qman_portal *p = get_affine_portal();
1649 mcc = qm_mc_start(&p->p);
1650 mcc->queryfq.fqid = cpu_to_be32(fq->fqid);
1651 qm_mc_commit(&p->p, QM_MCC_VERB_QUERYFQ_NP);
1652 while (!(mcr = qm_mc_result(&p->p)))
1655 if (res == QM_MCR_RESULT_OK)
1656 ret = !!mcr->queryfq_np.frm_cnt;
1660 int qman_query_fq_np(struct qman_fq *fq, struct qm_mcr_queryfq_np *np)
1662 struct qm_mc_command *mcc;
1663 struct qm_mc_result *mcr;
1664 struct qman_portal *p = get_affine_portal();
1668 mcc = qm_mc_start(&p->p);
1669 mcc->queryfq.fqid = cpu_to_be32(fq->fqid);
1670 qm_mc_commit(&p->p, QM_MCC_VERB_QUERYFQ_NP);
1671 while (!(mcr = qm_mc_result(&p->p)))
1673 DPAA_ASSERT((mcr->verb & QM_MCR_VERB_MASK) == QM_MCR_VERB_QUERYFQ_NP);
1675 if (res == QM_MCR_RESULT_OK) {
1676 *np = mcr->queryfq_np;
1677 np->fqd_link = be24_to_cpu(np->fqd_link);
1678 np->odp_seq = be16_to_cpu(np->odp_seq);
1679 np->orp_nesn = be16_to_cpu(np->orp_nesn);
1680 np->orp_ea_hseq = be16_to_cpu(np->orp_ea_hseq);
1681 np->orp_ea_tseq = be16_to_cpu(np->orp_ea_tseq);
1682 np->orp_ea_hptr = be24_to_cpu(np->orp_ea_hptr);
1683 np->orp_ea_tptr = be24_to_cpu(np->orp_ea_tptr);
1684 np->pfdr_hptr = be24_to_cpu(np->pfdr_hptr);
1685 np->pfdr_tptr = be24_to_cpu(np->pfdr_tptr);
1686 np->ics_surp = be16_to_cpu(np->ics_surp);
1687 np->byte_cnt = be32_to_cpu(np->byte_cnt);
1688 np->frm_cnt = be24_to_cpu(np->frm_cnt);
1689 np->ra1_sfdr = be16_to_cpu(np->ra1_sfdr);
1690 np->ra2_sfdr = be16_to_cpu(np->ra2_sfdr);
1691 np->od1_sfdr = be16_to_cpu(np->od1_sfdr);
1692 np->od2_sfdr = be16_to_cpu(np->od2_sfdr);
1693 np->od3_sfdr = be16_to_cpu(np->od3_sfdr);
1695 if (res == QM_MCR_RESULT_ERR_FQID)
1697 else if (res != QM_MCR_RESULT_OK)
1702 int qman_query_wq(u8 query_dedicated, struct qm_mcr_querywq *wq)
1704 struct qm_mc_command *mcc;
1705 struct qm_mc_result *mcr;
1706 struct qman_portal *p = get_affine_portal();
1710 myverb = (query_dedicated) ? QM_MCR_VERB_QUERYWQ_DEDICATED :
1711 QM_MCR_VERB_QUERYWQ;
1712 mcc = qm_mc_start(&p->p);
1713 mcc->querywq.channel.id = cpu_to_be16(wq->channel.id);
1714 qm_mc_commit(&p->p, myverb);
1715 while (!(mcr = qm_mc_result(&p->p)))
1717 DPAA_ASSERT((mcr->verb & QM_MCR_VERB_MASK) == myverb);
1719 if (res == QM_MCR_RESULT_OK) {
1722 wq->channel.id = be16_to_cpu(mcr->querywq.channel.id);
1723 array_len = ARRAY_SIZE(mcr->querywq.wq_len);
1724 for (i = 0; i < array_len; i++)
1725 wq->wq_len[i] = be32_to_cpu(mcr->querywq.wq_len[i]);
1727 if (res != QM_MCR_RESULT_OK) {
1728 pr_err("QUERYWQ failed: %s\n", mcr_result_str(res));
1734 int qman_testwrite_cgr(struct qman_cgr *cgr, u64 i_bcnt,
1735 struct qm_mcr_cgrtestwrite *result)
1737 struct qm_mc_command *mcc;
1738 struct qm_mc_result *mcr;
1739 struct qman_portal *p = get_affine_portal();
1743 mcc = qm_mc_start(&p->p);
1744 mcc->cgrtestwrite.cgid = cgr->cgrid;
1745 mcc->cgrtestwrite.i_bcnt_hi = (u8)(i_bcnt >> 32);
1746 mcc->cgrtestwrite.i_bcnt_lo = (u32)i_bcnt;
1747 qm_mc_commit(&p->p, QM_MCC_VERB_CGRTESTWRITE);
1748 while (!(mcr = qm_mc_result(&p->p)))
1750 DPAA_ASSERT((mcr->verb & QM_MCR_VERB_MASK) == QM_MCC_VERB_CGRTESTWRITE);
1752 if (res == QM_MCR_RESULT_OK)
1753 *result = mcr->cgrtestwrite;
1754 if (res != QM_MCR_RESULT_OK) {
1755 pr_err("CGR TEST WRITE failed: %s\n", mcr_result_str(res));
1761 int qman_query_cgr(struct qman_cgr *cgr, struct qm_mcr_querycgr *cgrd)
1763 struct qm_mc_command *mcc;
1764 struct qm_mc_result *mcr;
1765 struct qman_portal *p = get_affine_portal();
1769 mcc = qm_mc_start(&p->p);
1770 mcc->querycgr.cgid = cgr->cgrid;
1771 qm_mc_commit(&p->p, QM_MCC_VERB_QUERYCGR);
1772 while (!(mcr = qm_mc_result(&p->p)))
1774 DPAA_ASSERT((mcr->verb & QM_MCR_VERB_MASK) == QM_MCC_VERB_QUERYCGR);
1776 if (res == QM_MCR_RESULT_OK)
1777 *cgrd = mcr->querycgr;
1778 if (res != QM_MCR_RESULT_OK) {
1779 pr_err("QUERY_CGR failed: %s\n", mcr_result_str(res));
1782 cgrd->cgr.wr_parm_g.word =
1783 be32_to_cpu(cgrd->cgr.wr_parm_g.word);
1784 cgrd->cgr.wr_parm_y.word =
1785 be32_to_cpu(cgrd->cgr.wr_parm_y.word);
1786 cgrd->cgr.wr_parm_r.word =
1787 be32_to_cpu(cgrd->cgr.wr_parm_r.word);
1788 cgrd->cgr.cscn_targ = be32_to_cpu(cgrd->cgr.cscn_targ);
1789 cgrd->cgr.__cs_thres = be16_to_cpu(cgrd->cgr.__cs_thres);
1790 for (i = 0; i < ARRAY_SIZE(cgrd->cscn_targ_swp); i++)
1791 cgrd->cscn_targ_swp[i] =
1792 be32_to_cpu(cgrd->cscn_targ_swp[i]);
1796 int qman_query_congestion(struct qm_mcr_querycongestion *congestion)
1798 struct qm_mc_result *mcr;
1799 struct qman_portal *p = get_affine_portal();
1804 qm_mc_commit(&p->p, QM_MCC_VERB_QUERYCONGESTION);
1805 while (!(mcr = qm_mc_result(&p->p)))
1807 DPAA_ASSERT((mcr->verb & QM_MCR_VERB_MASK) ==
1808 QM_MCC_VERB_QUERYCONGESTION);
1810 if (res == QM_MCR_RESULT_OK)
1811 *congestion = mcr->querycongestion;
1812 if (res != QM_MCR_RESULT_OK) {
1813 pr_err("QUERY_CONGESTION failed: %s\n", mcr_result_str(res));
1816 for (i = 0; i < ARRAY_SIZE(congestion->state.state); i++)
1817 congestion->state.state[i] =
1818 be32_to_cpu(congestion->state.state[i]);
1822 int qman_set_vdq(struct qman_fq *fq, u16 num)
1824 struct qman_portal *p = get_affine_portal();
1828 vdqcr = QM_VDQCR_EXACT;
1829 vdqcr |= QM_VDQCR_NUMFRAMES_SET(num);
1831 if ((fq->state != qman_fq_state_parked) &&
1832 (fq->state != qman_fq_state_retired)) {
1836 if (fq_isset(fq, QMAN_FQ_STATE_VDQCR)) {
1840 vdqcr = (vdqcr & ~QM_VDQCR_FQID_MASK) | fq->fqid;
1842 if (!p->vdqcr_owned) {
1844 if (fq_isset(fq, QMAN_FQ_STATE_VDQCR))
1846 fq_set(fq, QMAN_FQ_STATE_VDQCR);
1848 p->vdqcr_owned = fq;
1853 qm_dqrr_vdqcr_set(&p->p, vdqcr);
1859 int qman_volatile_dequeue(struct qman_fq *fq, u32 flags __maybe_unused,
1862 struct qman_portal *p;
1865 if ((fq->state != qman_fq_state_parked) &&
1866 (fq->state != qman_fq_state_retired))
1868 if (vdqcr & QM_VDQCR_FQID_MASK)
1870 if (fq_isset(fq, QMAN_FQ_STATE_VDQCR))
1872 vdqcr = (vdqcr & ~QM_VDQCR_FQID_MASK) | fq->fqid;
1874 p = get_affine_portal();
1876 if (!p->vdqcr_owned) {
1878 if (fq_isset(fq, QMAN_FQ_STATE_VDQCR))
1880 fq_set(fq, QMAN_FQ_STATE_VDQCR);
1882 p->vdqcr_owned = fq;
1890 qm_dqrr_vdqcr_set(&p->p, vdqcr);
1894 static noinline void update_eqcr_ci(struct qman_portal *p, u8 avail)
1897 qm_eqcr_cce_prefetch(&p->p);
1899 qm_eqcr_cce_update(&p->p);
1902 int qman_eqcr_is_empty(void)
1904 struct qman_portal *p = get_affine_portal();
1907 update_eqcr_ci(p, 0);
1908 avail = qm_eqcr_get_fill(&p->p);
1909 return (avail == 0);
1912 void qman_set_dc_ern(qman_cb_dc_ern handler, int affine)
1915 struct qman_portal *p = get_affine_portal();
1917 p->cb_dc_ern = handler;
1919 cb_dc_ern = handler;
1922 static inline struct qm_eqcr_entry *try_p_eq_start(struct qman_portal *p,
1924 const struct qm_fd *fd,
1927 struct qm_eqcr_entry *eq;
1930 if (p->use_eqcr_ci_stashing) {
1932 * The stashing case is easy, only update if we need to in
1933 * order to try and liberate ring entries.
1935 eq = qm_eqcr_start_stash(&p->p);
1938 * The non-stashing case is harder, need to prefetch ahead of
1941 avail = qm_eqcr_get_avail(&p->p);
1943 update_eqcr_ci(p, avail);
1944 eq = qm_eqcr_start_no_stash(&p->p);
1950 if (flags & QMAN_ENQUEUE_FLAG_DCA)
1951 eq->dca = QM_EQCR_DCA_ENABLE |
1952 ((flags & QMAN_ENQUEUE_FLAG_DCA_PARK) ?
1953 QM_EQCR_DCA_PARK : 0) |
1954 ((flags >> 8) & QM_EQCR_DCA_IDXMASK);
1955 eq->fqid = cpu_to_be32(fq->fqid);
1956 #ifdef CONFIG_FSL_QMAN_FQ_LOOKUP
1957 eq->tag = cpu_to_be32(fq->key);
1959 eq->tag = cpu_to_be32((u32)(uintptr_t)fq);
1962 cpu_to_hw_fd(&eq->fd);
1966 int qman_enqueue(struct qman_fq *fq, const struct qm_fd *fd, u32 flags)
1968 struct qman_portal *p = get_affine_portal();
1969 struct qm_eqcr_entry *eq;
1971 eq = try_p_eq_start(p, fq, fd, flags);
1974 /* Note: QM_EQCR_VERB_INTERRUPT == QMAN_ENQUEUE_FLAG_WAIT_SYNC */
1975 qm_eqcr_pvb_commit(&p->p, QM_EQCR_VERB_CMD_ENQUEUE |
1976 (flags & (QM_EQCR_VERB_COLOUR_MASK | QM_EQCR_VERB_INTERRUPT)));
1977 /* Factor the below out, it's used from qman_enqueue_orp() too */
1981 int qman_enqueue_multi(struct qman_fq *fq,
1982 const struct qm_fd *fd,
1985 struct qman_portal *p = get_affine_portal();
1986 struct qm_portal *portal = &p->p;
1988 register struct qm_eqcr *eqcr = &portal->eqcr;
1989 struct qm_eqcr_entry *eq = eqcr->cursor, *prev_eq;
1991 u8 i, diff, old_ci, sent = 0;
1993 /* Update the available entries if no entry is free */
1994 if (!eqcr->available) {
1996 eqcr->ci = qm_cl_in(EQCR_CI) & (QM_EQCR_SIZE - 1);
1997 diff = qm_cyc_diff(QM_EQCR_SIZE, old_ci, eqcr->ci);
1998 eqcr->available += diff;
2003 /* try to send as many frames as possible */
2004 while (eqcr->available && frames_to_send--) {
2005 eq->fqid = cpu_to_be32(fq->fqid);
2006 #ifdef CONFIG_FSL_QMAN_FQ_LOOKUP
2007 eq->tag = cpu_to_be32(fq->key);
2009 eq->tag = cpu_to_be32((u32)(uintptr_t)fq);
2011 eq->fd.opaque_addr = fd->opaque_addr;
2012 eq->fd.addr = cpu_to_be40(fd->addr);
2013 eq->fd.status = cpu_to_be32(fd->status);
2014 eq->fd.opaque = cpu_to_be32(fd->opaque);
2016 eq = (void *)((unsigned long)(eq + 1) &
2017 (~(unsigned long)(QM_EQCR_SIZE << 6)));
2024 /* In order for flushes to complete faster, all lines are recorded in
2028 for (i = 0; i < sent; i++) {
2029 eq->__dont_write_directly__verb =
2030 QM_EQCR_VERB_CMD_ENQUEUE | eqcr->vbit;
2032 eq = (void *)((unsigned long)(eq + 1) &
2033 (~(unsigned long)(QM_EQCR_SIZE << 6)));
2034 if (unlikely((prev_eq + 1) != eq))
2035 eqcr->vbit ^= QM_EQCR_VERB_VBIT;
2038 /* We need to flush all the lines but without load/store operations
2042 for (i = 0; i < sent; i++) {
2044 eq = (void *)((unsigned long)(eq + 1) &
2045 (~(unsigned long)(QM_EQCR_SIZE << 6)));
2047 /* Update cursor for the next call */
2052 int qman_enqueue_orp(struct qman_fq *fq, const struct qm_fd *fd, u32 flags,
2053 struct qman_fq *orp, u16 orp_seqnum)
2055 struct qman_portal *p = get_affine_portal();
2056 struct qm_eqcr_entry *eq;
2058 eq = try_p_eq_start(p, fq, fd, flags);
2061 /* Process ORP-specifics here */
2062 if (flags & QMAN_ENQUEUE_FLAG_NLIS)
2063 orp_seqnum |= QM_EQCR_SEQNUM_NLIS;
2065 orp_seqnum &= ~QM_EQCR_SEQNUM_NLIS;
2066 if (flags & QMAN_ENQUEUE_FLAG_NESN)
2067 orp_seqnum |= QM_EQCR_SEQNUM_NESN;
2069 /* No need to check 4 QMAN_ENQUEUE_FLAG_HOLE */
2070 orp_seqnum &= ~QM_EQCR_SEQNUM_NESN;
2072 eq->seqnum = cpu_to_be16(orp_seqnum);
2073 eq->orp = cpu_to_be32(orp->fqid);
2074 /* Note: QM_EQCR_VERB_INTERRUPT == QMAN_ENQUEUE_FLAG_WAIT_SYNC */
2075 qm_eqcr_pvb_commit(&p->p, QM_EQCR_VERB_ORP |
2076 ((flags & (QMAN_ENQUEUE_FLAG_HOLE | QMAN_ENQUEUE_FLAG_NESN)) ?
2077 0 : QM_EQCR_VERB_CMD_ENQUEUE) |
2078 (flags & (QM_EQCR_VERB_COLOUR_MASK | QM_EQCR_VERB_INTERRUPT)));
2083 int qman_modify_cgr(struct qman_cgr *cgr, u32 flags,
2084 struct qm_mcc_initcgr *opts)
2086 struct qm_mc_command *mcc;
2087 struct qm_mc_result *mcr;
2088 struct qman_portal *p = get_affine_portal();
2091 u8 verb = QM_MCC_VERB_MODIFYCGR;
2093 mcc = qm_mc_start(&p->p);
2095 mcc->initcgr = *opts;
2096 mcc->initcgr.we_mask = cpu_to_be16(mcc->initcgr.we_mask);
2097 mcc->initcgr.cgr.wr_parm_g.word =
2098 cpu_to_be32(mcc->initcgr.cgr.wr_parm_g.word);
2099 mcc->initcgr.cgr.wr_parm_y.word =
2100 cpu_to_be32(mcc->initcgr.cgr.wr_parm_y.word);
2101 mcc->initcgr.cgr.wr_parm_r.word =
2102 cpu_to_be32(mcc->initcgr.cgr.wr_parm_r.word);
2103 mcc->initcgr.cgr.cscn_targ = cpu_to_be32(mcc->initcgr.cgr.cscn_targ);
2104 mcc->initcgr.cgr.__cs_thres = cpu_to_be16(mcc->initcgr.cgr.__cs_thres);
2106 mcc->initcgr.cgid = cgr->cgrid;
2107 if (flags & QMAN_CGR_FLAG_USE_INIT)
2108 verb = QM_MCC_VERB_INITCGR;
2109 qm_mc_commit(&p->p, verb);
2110 while (!(mcr = qm_mc_result(&p->p)))
2113 DPAA_ASSERT((mcr->verb & QM_MCR_VERB_MASK) == verb);
2115 return (res == QM_MCR_RESULT_OK) ? 0 : -EIO;
2118 #define TARG_MASK(n) (0x80000000 >> (n->config->channel - \
2119 QM_CHANNEL_SWPORTAL0))
2120 #define TARG_DCP_MASK(n) (0x80000000 >> (10 + n))
2121 #define PORTAL_IDX(n) (n->config->channel - QM_CHANNEL_SWPORTAL0)
2123 int qman_create_cgr(struct qman_cgr *cgr, u32 flags,
2124 struct qm_mcc_initcgr *opts)
2126 struct qm_mcr_querycgr cgr_state;
2127 struct qm_mcc_initcgr local_opts;
2129 struct qman_portal *p;
2131 /* We have to check that the provided CGRID is within the limits of the
2132 * data-structures, for obvious reasons. However we'll let h/w take
2133 * care of determining whether it's within the limits of what exists on
2136 if (cgr->cgrid >= __CGR_NUM)
2139 p = get_affine_portal();
2141 memset(&local_opts, 0, sizeof(struct qm_mcc_initcgr));
2142 cgr->chan = p->config->channel;
2143 spin_lock(&p->cgr_lock);
2145 /* if no opts specified, just add it to the list */
2149 ret = qman_query_cgr(cgr, &cgr_state);
2154 if ((qman_ip_rev & 0xFF00) >= QMAN_REV30)
2155 local_opts.cgr.cscn_targ_upd_ctrl =
2156 QM_CGR_TARG_UDP_CTRL_WRITE_BIT | PORTAL_IDX(p);
2158 /* Overwrite TARG */
2159 local_opts.cgr.cscn_targ = cgr_state.cgr.cscn_targ |
2161 local_opts.we_mask |= QM_CGR_WE_CSCN_TARG;
2163 /* send init if flags indicate so */
2164 if (opts && (flags & QMAN_CGR_FLAG_USE_INIT))
2165 ret = qman_modify_cgr(cgr, QMAN_CGR_FLAG_USE_INIT, &local_opts);
2167 ret = qman_modify_cgr(cgr, 0, &local_opts);
2171 list_add(&cgr->node, &p->cgr_cbs);
2173 /* Determine if newly added object requires its callback to be called */
2174 ret = qman_query_cgr(cgr, &cgr_state);
2176 /* we can't go back, so proceed and return success, but screen
2177 * and wail to the log file.
2179 pr_crit("CGR HW state partially modified\n");
2183 if (cgr->cb && cgr_state.cgr.cscn_en && qman_cgrs_get(&p->cgrs[1],
2187 spin_unlock(&p->cgr_lock);
2191 int qman_create_cgr_to_dcp(struct qman_cgr *cgr, u32 flags, u16 dcp_portal,
2192 struct qm_mcc_initcgr *opts)
2194 struct qm_mcc_initcgr local_opts;
2195 struct qm_mcr_querycgr cgr_state;
2198 if ((qman_ip_rev & 0xFF00) < QMAN_REV30) {
2199 pr_warn("QMan version doesn't support CSCN => DCP portal\n");
2202 /* We have to check that the provided CGRID is within the limits of the
2203 * data-structures, for obvious reasons. However we'll let h/w take
2204 * care of determining whether it's within the limits of what exists on
2207 if (cgr->cgrid >= __CGR_NUM)
2210 ret = qman_query_cgr(cgr, &cgr_state);
2214 memset(&local_opts, 0, sizeof(struct qm_mcc_initcgr));
2218 if ((qman_ip_rev & 0xFF00) >= QMAN_REV30)
2219 local_opts.cgr.cscn_targ_upd_ctrl =
2220 QM_CGR_TARG_UDP_CTRL_WRITE_BIT |
2221 QM_CGR_TARG_UDP_CTRL_DCP | dcp_portal;
2223 local_opts.cgr.cscn_targ = cgr_state.cgr.cscn_targ |
2224 TARG_DCP_MASK(dcp_portal);
2225 local_opts.we_mask |= QM_CGR_WE_CSCN_TARG;
2227 /* send init if flags indicate so */
2228 if (opts && (flags & QMAN_CGR_FLAG_USE_INIT))
2229 ret = qman_modify_cgr(cgr, QMAN_CGR_FLAG_USE_INIT,
2232 ret = qman_modify_cgr(cgr, 0, &local_opts);
2237 int qman_delete_cgr(struct qman_cgr *cgr)
2239 struct qm_mcr_querycgr cgr_state;
2240 struct qm_mcc_initcgr local_opts;
2243 struct qman_portal *p = get_affine_portal();
2245 if (cgr->chan != p->config->channel) {
2246 pr_crit("Attempting to delete cgr from different portal than"
2247 " it was create: create 0x%x, delete 0x%x\n",
2248 cgr->chan, p->config->channel);
2252 memset(&local_opts, 0, sizeof(struct qm_mcc_initcgr));
2253 spin_lock(&p->cgr_lock);
2254 list_del(&cgr->node);
2256 * If there are no other CGR objects for this CGRID in the list,
2257 * update CSCN_TARG accordingly
2259 list_for_each_entry(i, &p->cgr_cbs, node)
2260 if ((i->cgrid == cgr->cgrid) && i->cb)
2262 ret = qman_query_cgr(cgr, &cgr_state);
2264 /* add back to the list */
2265 list_add(&cgr->node, &p->cgr_cbs);
2268 /* Overwrite TARG */
2269 local_opts.we_mask = QM_CGR_WE_CSCN_TARG;
2270 if ((qman_ip_rev & 0xFF00) >= QMAN_REV30)
2271 local_opts.cgr.cscn_targ_upd_ctrl = PORTAL_IDX(p);
2273 local_opts.cgr.cscn_targ = cgr_state.cgr.cscn_targ &
2275 ret = qman_modify_cgr(cgr, 0, &local_opts);
2277 /* add back to the list */
2278 list_add(&cgr->node, &p->cgr_cbs);
2280 spin_unlock(&p->cgr_lock);
2285 int qman_shutdown_fq(u32 fqid)
2287 struct qman_portal *p;
2288 struct qm_portal *low_p;
2289 struct qm_mc_command *mcc;
2290 struct qm_mc_result *mcr;
2292 int orl_empty, fq_empty, drain = 0;
2297 p = get_affine_portal();
2300 /* Determine the state of the FQID */
2301 mcc = qm_mc_start(low_p);
2302 mcc->queryfq_np.fqid = cpu_to_be32(fqid);
2303 qm_mc_commit(low_p, QM_MCC_VERB_QUERYFQ_NP);
2304 while (!(mcr = qm_mc_result(low_p)))
2306 DPAA_ASSERT((mcr->verb & QM_MCR_VERB_MASK) == QM_MCR_VERB_QUERYFQ_NP);
2307 state = mcr->queryfq_np.state & QM_MCR_NP_STATE_MASK;
2308 if (state == QM_MCR_NP_STATE_OOS)
2309 return 0; /* Already OOS, no need to do anymore checks */
2311 /* Query which channel the FQ is using */
2312 mcc = qm_mc_start(low_p);
2313 mcc->queryfq.fqid = cpu_to_be32(fqid);
2314 qm_mc_commit(low_p, QM_MCC_VERB_QUERYFQ);
2315 while (!(mcr = qm_mc_result(low_p)))
2317 DPAA_ASSERT((mcr->verb & QM_MCR_VERB_MASK) == QM_MCR_VERB_QUERYFQ);
2319 /* Need to store these since the MCR gets reused */
2320 dest_wq = be16_to_cpu(mcr->queryfq.fqd.dest_wq);
2321 channel = dest_wq & 0x7;
2325 case QM_MCR_NP_STATE_TEN_SCHED:
2326 case QM_MCR_NP_STATE_TRU_SCHED:
2327 case QM_MCR_NP_STATE_ACTIVE:
2328 case QM_MCR_NP_STATE_PARKED:
2330 mcc = qm_mc_start(low_p);
2331 mcc->alterfq.fqid = cpu_to_be32(fqid);
2332 qm_mc_commit(low_p, QM_MCC_VERB_ALTER_RETIRE);
2333 while (!(mcr = qm_mc_result(low_p)))
2335 DPAA_ASSERT((mcr->verb & QM_MCR_VERB_MASK) ==
2336 QM_MCR_VERB_ALTER_RETIRE);
2337 result = mcr->result; /* Make a copy as we reuse MCR below */
2339 if (result == QM_MCR_RESULT_PENDING) {
2340 /* Need to wait for the FQRN in the message ring, which
2341 * will only occur once the FQ has been drained. In
2342 * order for the FQ to drain the portal needs to be set
2343 * to dequeue from the channel the FQ is scheduled on
2345 const struct qm_mr_entry *msg;
2346 const struct qm_dqrr_entry *dqrr = NULL;
2348 __maybe_unused u16 dequeue_wq = 0;
2350 /* Flag that we need to drain FQ */
2353 if (channel >= qm_channel_pool1 &&
2354 channel < (u16)(qm_channel_pool1 + 15)) {
2355 /* Pool channel, enable the bit in the portal */
2356 dequeue_wq = (channel -
2357 qm_channel_pool1 + 1) << 4 | wq;
2358 } else if (channel < qm_channel_pool1) {
2359 /* Dedicated channel */
2362 pr_info("Cannot recover FQ 0x%x,"
2363 " it is scheduled on channel 0x%x",
2367 /* Set the sdqcr to drain this channel */
2368 if (channel < qm_channel_pool1)
2369 qm_dqrr_sdqcr_set(low_p,
2370 QM_SDQCR_TYPE_ACTIVE |
2371 QM_SDQCR_CHANNELS_DEDICATED);
2373 qm_dqrr_sdqcr_set(low_p,
2374 QM_SDQCR_TYPE_ACTIVE |
2375 QM_SDQCR_CHANNELS_POOL_CONV
2377 while (!found_fqrn) {
2378 /* Keep draining DQRR while checking the MR*/
2379 qm_dqrr_pvb_update(low_p);
2380 dqrr = qm_dqrr_current(low_p);
2382 qm_dqrr_cdc_consume_1ptr(
2384 qm_dqrr_pvb_update(low_p);
2385 qm_dqrr_next(low_p);
2386 dqrr = qm_dqrr_current(low_p);
2388 /* Process message ring too */
2389 qm_mr_pvb_update(low_p);
2390 msg = qm_mr_current(low_p);
2393 QM_MR_VERB_TYPE_MASK)
2397 qm_mr_cci_consume_to_current(low_p);
2398 qm_mr_pvb_update(low_p);
2399 msg = qm_mr_current(low_p);
2404 if (result != QM_MCR_RESULT_OK &&
2405 result != QM_MCR_RESULT_PENDING) {
2407 pr_err("qman_retire_fq failed on FQ 0x%x,"
2408 " result=0x%x\n", fqid, result);
2411 if (!(mcr->alterfq.fqs & QM_MCR_FQS_ORLPRESENT)) {
2412 /* ORL had no entries, no need to wait until the
2417 /* Retirement succeeded, check to see if FQ needs
2420 if (drain || mcr->alterfq.fqs & QM_MCR_FQS_NOTEMPTY) {
2421 /* FQ is Not Empty, drain using volatile DQ commands */
2424 const struct qm_dqrr_entry *dqrr = NULL;
2425 u32 vdqcr = fqid | QM_VDQCR_NUMFRAMES_SET(3);
2427 qm_dqrr_vdqcr_set(low_p, vdqcr);
2429 /* Wait for a dequeue to occur */
2430 while (dqrr == NULL) {
2431 qm_dqrr_pvb_update(low_p);
2432 dqrr = qm_dqrr_current(low_p);
2436 /* Process the dequeues, making sure to
2437 * empty the ring completely.
2440 if (dqrr->fqid == fqid &&
2441 dqrr->stat & QM_DQRR_STAT_FQ_EMPTY)
2443 qm_dqrr_cdc_consume_1ptr(low_p,
2445 qm_dqrr_pvb_update(low_p);
2446 qm_dqrr_next(low_p);
2447 dqrr = qm_dqrr_current(low_p);
2449 } while (fq_empty == 0);
2451 qm_dqrr_sdqcr_set(low_p, 0);
2453 /* Wait for the ORL to have been completely drained */
2454 while (orl_empty == 0) {
2455 const struct qm_mr_entry *msg;
2457 qm_mr_pvb_update(low_p);
2458 msg = qm_mr_current(low_p);
2460 if ((msg->verb & QM_MR_VERB_TYPE_MASK) ==
2464 qm_mr_cci_consume_to_current(low_p);
2465 qm_mr_pvb_update(low_p);
2466 msg = qm_mr_current(low_p);
2470 mcc = qm_mc_start(low_p);
2471 mcc->alterfq.fqid = cpu_to_be32(fqid);
2472 qm_mc_commit(low_p, QM_MCC_VERB_ALTER_OOS);
2473 while (!(mcr = qm_mc_result(low_p)))
2475 DPAA_ASSERT((mcr->verb & QM_MCR_VERB_MASK) ==
2476 QM_MCR_VERB_ALTER_OOS);
2477 if (mcr->result != QM_MCR_RESULT_OK) {
2479 "OOS after drain Failed on FQID 0x%x, result 0x%x\n",
2485 case QM_MCR_NP_STATE_RETIRED:
2486 /* Send OOS Command */
2487 mcc = qm_mc_start(low_p);
2488 mcc->alterfq.fqid = cpu_to_be32(fqid);
2489 qm_mc_commit(low_p, QM_MCC_VERB_ALTER_OOS);
2490 while (!(mcr = qm_mc_result(low_p)))
2492 DPAA_ASSERT((mcr->verb & QM_MCR_VERB_MASK) ==
2493 QM_MCR_VERB_ALTER_OOS);
2495 pr_err("OOS Failed on FQID 0x%x\n", fqid);