1 /* SPDX-License-Identifier: (BSD-3-Clause OR GPL-2.0)
3 * Copyright 2008-2016 Freescale Semiconductor Inc.
9 #include <rte_branch_prediction.h>
10 #include <rte_dpaa_bus.h>
12 /* Compilation constants */
13 #define DQRR_MAXFILL 15
14 #define EQCR_ITHRESH 4 /* if EQCR congests, interrupt threshold */
15 #define IRQNAME "QMan portal %d"
16 #define MAX_IRQNAME 16 /* big enough for "QMan portal %d" */
17 /* maximum number of DQRR entries to process in qman_poll() */
18 #define FSL_QMAN_POLL_LIMIT 8
20 /* Lock/unlock frame queues, subject to the "LOCKED" flag. This is about
21 * inter-processor locking only. Note, FQLOCK() is always called either under a
22 * local_irq_save() or from interrupt context - hence there's no need for irq
23 * protection (and indeed, attempting to nest irq-protection doesn't work, as
24 * the "irq en/disable" machinery isn't recursive...).
28 struct qman_fq *__fq478 = (fq); \
29 if (fq_isset(__fq478, QMAN_FQ_FLAG_LOCKED)) \
30 spin_lock(&__fq478->fqlock); \
32 #define FQUNLOCK(fq) \
34 struct qman_fq *__fq478 = (fq); \
35 if (fq_isset(__fq478, QMAN_FQ_FLAG_LOCKED)) \
36 spin_unlock(&__fq478->fqlock); \
39 static inline void fq_set(struct qman_fq *fq, u32 mask)
41 dpaa_set_bits(mask, &fq->flags);
44 static inline void fq_clear(struct qman_fq *fq, u32 mask)
46 dpaa_clear_bits(mask, &fq->flags);
49 static inline int fq_isset(struct qman_fq *fq, u32 mask)
51 return fq->flags & mask;
54 static inline int fq_isclear(struct qman_fq *fq, u32 mask)
56 return !(fq->flags & mask);
61 /* PORTAL_BITS_*** - dynamic, strictly internal */
63 /* interrupt sources processed by portal_isr(), configurable */
64 unsigned long irq_sources;
65 u32 use_eqcr_ci_stashing;
66 u32 slowpoll; /* only used when interrupts are off */
67 /* only 1 volatile dequeue at a time */
68 struct qman_fq *vdqcr_owned;
71 /* A portal-specific handler for DCP ERNs. If this is NULL, the global
72 * handler is called instead.
74 qman_cb_dc_ern cb_dc_ern;
75 /* When the cpu-affine portal is activated, this is non-NULL */
76 const struct qm_portal_config *config;
77 struct dpa_rbtree retire_table;
78 char irqname[MAX_IRQNAME];
79 /* 2-element array. cgrs[0] is mask, cgrs[1] is snapshot. */
80 struct qman_cgrs *cgrs;
81 /* linked-list of CSCN handlers. */
82 struct list_head cgr_cbs;
85 /* track if memory was allocated by the driver */
86 #if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__
87 /* Keep a shadow copy of the DQRR on LE systems as the SW needs to
88 * do byte swaps of DQRR read only memory. First entry must be aligned
89 * to 2 ** 10 to ensure DQRR index calculations based shadow copy
90 * address (6 bits for address shift + 4 bits for the DQRR size).
92 struct qm_dqrr_entry shadow_dqrr[QM_DQRR_SIZE]
93 __attribute__((aligned(1024)));
97 /* Global handler for DCP ERNs. Used when the portal receiving the message does
98 * not have a portal-specific handler.
100 static qman_cb_dc_ern cb_dc_ern;
102 static cpumask_t affine_mask;
103 static DEFINE_SPINLOCK(affine_mask_lock);
104 static u16 affine_channels[NR_CPUS];
105 static RTE_DEFINE_PER_LCORE(struct qman_portal, qman_affine_portal);
107 static inline struct qman_portal *get_affine_portal(void)
109 return &RTE_PER_LCORE(qman_affine_portal);
112 /* This gives a FQID->FQ lookup to cover the fact that we can't directly demux
113 * retirement notifications (the fact they are sometimes h/w-consumed means that
114 * contextB isn't always a s/w demux - and as we can't know which case it is
115 * when looking at the notification, we have to use the slow lookup for all of
116 * them). NB, it's possible to have multiple FQ objects refer to the same FQID
117 * (though at most one of them should be the consumer), so this table isn't for
118 * all FQs - FQs are added when retirement commands are issued, and removed when
119 * they complete, which also massively reduces the size of this table.
121 IMPLEMENT_DPAA_RBTREE(fqtree, struct qman_fq, node, fqid);
123 * This is what everything can wait on, even if it migrates to a different cpu
124 * to the one whose affine portal it is waiting on.
126 static DECLARE_WAIT_QUEUE_HEAD(affine_queue);
128 static inline int table_push_fq(struct qman_portal *p, struct qman_fq *fq)
130 int ret = fqtree_push(&p->retire_table, fq);
133 pr_err("ERROR: double FQ-retirement %d\n", fq->fqid);
137 static inline void table_del_fq(struct qman_portal *p, struct qman_fq *fq)
139 fqtree_del(&p->retire_table, fq);
142 static inline struct qman_fq *table_find_fq(struct qman_portal *p, u32 fqid)
144 return fqtree_find(&p->retire_table, fqid);
147 #ifdef CONFIG_FSL_QMAN_FQ_LOOKUP
148 static void **qman_fq_lookup_table;
149 static size_t qman_fq_lookup_table_size;
151 int qman_setup_fq_lookup_table(size_t num_entries)
154 /* Allocate 1 more entry since the first entry is not used */
155 qman_fq_lookup_table = vmalloc((num_entries * sizeof(void *)));
156 if (!qman_fq_lookup_table) {
157 pr_err("QMan: Could not allocate fq lookup table\n");
160 memset(qman_fq_lookup_table, 0, num_entries * sizeof(void *));
161 qman_fq_lookup_table_size = num_entries;
162 pr_debug("QMan: Allocated lookup table at %p, entry count %lu\n",
163 qman_fq_lookup_table,
164 (unsigned long)qman_fq_lookup_table_size);
168 /* global structure that maintains fq object mapping */
169 static DEFINE_SPINLOCK(fq_hash_table_lock);
171 static int find_empty_fq_table_entry(u32 *entry, struct qman_fq *fq)
175 spin_lock(&fq_hash_table_lock);
176 /* Can't use index zero because this has special meaning
177 * in context_b field.
179 for (i = 1; i < qman_fq_lookup_table_size; i++) {
180 if (qman_fq_lookup_table[i] == NULL) {
182 qman_fq_lookup_table[i] = fq;
183 spin_unlock(&fq_hash_table_lock);
187 spin_unlock(&fq_hash_table_lock);
191 static void clear_fq_table_entry(u32 entry)
193 spin_lock(&fq_hash_table_lock);
194 DPAA_BUG_ON(entry >= qman_fq_lookup_table_size);
195 qman_fq_lookup_table[entry] = NULL;
196 spin_unlock(&fq_hash_table_lock);
199 static inline struct qman_fq *get_fq_table_entry(u32 entry)
201 DPAA_BUG_ON(entry >= qman_fq_lookup_table_size);
202 return qman_fq_lookup_table[entry];
206 static inline void cpu_to_hw_fqd(struct qm_fqd *fqd)
208 /* Byteswap the FQD to HW format */
209 fqd->fq_ctrl = cpu_to_be16(fqd->fq_ctrl);
210 fqd->dest_wq = cpu_to_be16(fqd->dest_wq);
211 fqd->ics_cred = cpu_to_be16(fqd->ics_cred);
212 fqd->context_b = cpu_to_be32(fqd->context_b);
213 fqd->context_a.opaque = cpu_to_be64(fqd->context_a.opaque);
214 fqd->opaque_td = cpu_to_be16(fqd->opaque_td);
217 static inline void hw_fqd_to_cpu(struct qm_fqd *fqd)
219 /* Byteswap the FQD to CPU format */
220 fqd->fq_ctrl = be16_to_cpu(fqd->fq_ctrl);
221 fqd->dest_wq = be16_to_cpu(fqd->dest_wq);
222 fqd->ics_cred = be16_to_cpu(fqd->ics_cred);
223 fqd->context_b = be32_to_cpu(fqd->context_b);
224 fqd->context_a.opaque = be64_to_cpu(fqd->context_a.opaque);
227 static inline void cpu_to_hw_fd(struct qm_fd *fd)
229 fd->addr = cpu_to_be40(fd->addr);
230 fd->status = cpu_to_be32(fd->status);
231 fd->opaque = cpu_to_be32(fd->opaque);
234 static inline void hw_fd_to_cpu(struct qm_fd *fd)
236 fd->addr = be40_to_cpu(fd->addr);
237 fd->status = be32_to_cpu(fd->status);
238 fd->opaque = be32_to_cpu(fd->opaque);
241 /* In the case that slow- and fast-path handling are both done by qman_poll()
242 * (ie. because there is no interrupt handling), we ought to balance how often
243 * we do the fast-path poll versus the slow-path poll. We'll use two decrementer
244 * sources, so we call the fast poll 'n' times before calling the slow poll
245 * once. The idle decrementer constant is used when the last slow-poll detected
246 * no work to do, and the busy decrementer constant when the last slow-poll had
249 #define SLOW_POLL_IDLE 1000
250 #define SLOW_POLL_BUSY 10
251 static u32 __poll_portal_slow(struct qman_portal *p, u32 is);
252 static inline unsigned int __poll_portal_fast(struct qman_portal *p,
253 unsigned int poll_limit);
255 /* Portal interrupt handler */
256 static irqreturn_t portal_isr(__always_unused int irq, void *ptr)
258 struct qman_portal *p = ptr;
260 * The CSCI/CCSCI source is cleared inside __poll_portal_slow(), because
261 * it could race against a Query Congestion State command also given
262 * as part of the handling of this interrupt source. We mustn't
263 * clear it a second time in this top-level function.
265 u32 clear = QM_DQAVAIL_MASK | (p->irq_sources &
266 ~(QM_PIRQ_CSCI | QM_PIRQ_CCSCI));
267 u32 is = qm_isr_status_read(&p->p) & p->irq_sources;
268 /* DQRR-handling if it's interrupt-driven */
269 if (is & QM_PIRQ_DQRI)
270 __poll_portal_fast(p, FSL_QMAN_POLL_LIMIT);
271 /* Handling of anything else that's interrupt-driven */
272 clear |= __poll_portal_slow(p, is);
273 qm_isr_status_clear(&p->p, clear);
277 /* This inner version is used privately by qman_create_affine_portal(), as well
278 * as by the exported qman_stop_dequeues().
280 static inline void qman_stop_dequeues_ex(struct qman_portal *p)
282 if (!(p->dqrr_disable_ref++))
283 qm_dqrr_set_maxfill(&p->p, 0);
286 static int drain_mr_fqrni(struct qm_portal *p)
288 const struct qm_mr_entry *msg;
290 msg = qm_mr_current(p);
293 * if MR was full and h/w had other FQRNI entries to produce, we
294 * need to allow it time to produce those entries once the
295 * existing entries are consumed. A worst-case situation
296 * (fully-loaded system) means h/w sequencers may have to do 3-4
297 * other things before servicing the portal's MR pump, each of
298 * which (if slow) may take ~50 qman cycles (which is ~200
299 * processor cycles). So rounding up and then multiplying this
300 * worst-case estimate by a factor of 10, just to be
301 * ultra-paranoid, goes as high as 10,000 cycles. NB, we consume
302 * one entry at a time, so h/w has an opportunity to produce new
303 * entries well before the ring has been fully consumed, so
304 * we're being *really* paranoid here.
306 u64 now, then = mfatb();
310 } while ((then + 10000) > now);
311 msg = qm_mr_current(p);
315 if ((msg->verb & QM_MR_VERB_TYPE_MASK) != QM_MR_VERB_FQRNI) {
316 /* We aren't draining anything but FQRNIs */
317 pr_err("Found verb 0x%x in MR\n", msg->verb);
321 qm_mr_cci_consume(p, 1);
325 static inline int qm_eqcr_init(struct qm_portal *portal,
326 enum qm_eqcr_pmode pmode,
327 unsigned int eq_stash_thresh,
330 /* This use of 'register', as well as all other occurrences, is because
331 * it has been observed to generate much faster code with gcc than is
332 * otherwise the case.
334 register struct qm_eqcr *eqcr = &portal->eqcr;
338 eqcr->ring = portal->addr.ce + QM_CL_EQCR;
339 eqcr->ci = qm_in(EQCR_CI_CINH) & (QM_EQCR_SIZE - 1);
340 qm_cl_invalidate(EQCR_CI);
341 pi = qm_in(EQCR_PI_CINH) & (QM_EQCR_SIZE - 1);
342 eqcr->cursor = eqcr->ring + pi;
343 eqcr->vbit = (qm_in(EQCR_PI_CINH) & QM_EQCR_SIZE) ?
344 QM_EQCR_VERB_VBIT : 0;
345 eqcr->available = QM_EQCR_SIZE - 1 -
346 qm_cyc_diff(QM_EQCR_SIZE, eqcr->ci, pi);
347 eqcr->ithresh = qm_in(EQCR_ITR);
348 #ifdef RTE_LIBRTE_DPAA_HWDEBUG
352 cfg = (qm_in(CFG) & 0x00ffffff) |
353 (eq_stash_thresh << 28) | /* QCSP_CFG: EST */
354 (eq_stash_prio << 26) | /* QCSP_CFG: EP */
355 ((pmode & 0x3) << 24); /* QCSP_CFG::EPM */
360 static inline void qm_eqcr_finish(struct qm_portal *portal)
362 register struct qm_eqcr *eqcr = &portal->eqcr;
367 * Disable EQCI stashing because the QMan only
368 * presents the value it previously stashed to
369 * maintain coherency. Setting the stash threshold
370 * to 1 then 0 ensures that QMan has resyncronized
371 * its internal copy so that the portal is clean
372 * when it is reinitialized in the future
374 cfg = (qm_in(CFG) & 0x0fffffff) |
375 (1 << 28); /* QCSP_CFG: EST */
377 cfg &= 0x0fffffff; /* stash threshold = 0 */
380 pi = qm_in(EQCR_PI_CINH) & (QM_EQCR_SIZE - 1);
381 ci = qm_in(EQCR_CI_CINH) & (QM_EQCR_SIZE - 1);
383 /* Refresh EQCR CI cache value */
384 qm_cl_invalidate(EQCR_CI);
385 eqcr->ci = qm_cl_in(EQCR_CI) & (QM_EQCR_SIZE - 1);
387 #ifdef RTE_LIBRTE_DPAA_HWDEBUG
388 DPAA_ASSERT(!eqcr->busy);
390 if (pi != EQCR_PTR2IDX(eqcr->cursor))
391 pr_crit("losing uncommitted EQCR entries\n");
393 pr_crit("missing existing EQCR completions\n");
394 if (eqcr->ci != EQCR_PTR2IDX(eqcr->cursor))
395 pr_crit("EQCR destroyed unquiesced\n");
398 static inline int qm_dqrr_init(struct qm_portal *portal,
399 __maybe_unused const struct qm_portal_config *config,
400 enum qm_dqrr_dmode dmode,
401 __maybe_unused enum qm_dqrr_pmode pmode,
402 enum qm_dqrr_cmode cmode, u8 max_fill)
404 register struct qm_dqrr *dqrr = &portal->dqrr;
407 /* Make sure the DQRR will be idle when we enable */
408 qm_out(DQRR_SDQCR, 0);
409 qm_out(DQRR_VDQCR, 0);
410 qm_out(DQRR_PDQCR, 0);
411 dqrr->ring = portal->addr.ce + QM_CL_DQRR;
412 dqrr->pi = qm_in(DQRR_PI_CINH) & (QM_DQRR_SIZE - 1);
413 dqrr->ci = qm_in(DQRR_CI_CINH) & (QM_DQRR_SIZE - 1);
414 dqrr->cursor = dqrr->ring + dqrr->ci;
415 dqrr->fill = qm_cyc_diff(QM_DQRR_SIZE, dqrr->ci, dqrr->pi);
416 dqrr->vbit = (qm_in(DQRR_PI_CINH) & QM_DQRR_SIZE) ?
417 QM_DQRR_VERB_VBIT : 0;
418 dqrr->ithresh = qm_in(DQRR_ITR);
419 #ifdef RTE_LIBRTE_DPAA_HWDEBUG
424 /* Invalidate every ring entry before beginning */
425 for (cfg = 0; cfg < QM_DQRR_SIZE; cfg++)
426 dccivac(qm_cl(dqrr->ring, cfg));
427 cfg = (qm_in(CFG) & 0xff000f00) |
428 ((max_fill & (QM_DQRR_SIZE - 1)) << 20) | /* DQRR_MF */
429 ((dmode & 1) << 18) | /* DP */
430 ((cmode & 3) << 16) | /* DCM */
432 (0 ? 0x40 : 0) | /* Ignore RP */
433 (0 ? 0x10 : 0); /* Ignore SP */
435 qm_dqrr_set_maxfill(portal, max_fill);
439 static inline void qm_dqrr_finish(struct qm_portal *portal)
441 __maybe_unused register struct qm_dqrr *dqrr = &portal->dqrr;
442 #ifdef RTE_LIBRTE_DPAA_HWDEBUG
443 if ((dqrr->cmode != qm_dqrr_cdc) &&
444 (dqrr->ci != DQRR_PTR2IDX(dqrr->cursor)))
445 pr_crit("Ignoring completed DQRR entries\n");
449 static inline int qm_mr_init(struct qm_portal *portal,
450 __maybe_unused enum qm_mr_pmode pmode,
451 enum qm_mr_cmode cmode)
453 register struct qm_mr *mr = &portal->mr;
456 mr->ring = portal->addr.ce + QM_CL_MR;
457 mr->pi = qm_in(MR_PI_CINH) & (QM_MR_SIZE - 1);
458 mr->ci = qm_in(MR_CI_CINH) & (QM_MR_SIZE - 1);
459 mr->cursor = mr->ring + mr->ci;
460 mr->fill = qm_cyc_diff(QM_MR_SIZE, mr->ci, mr->pi);
461 mr->vbit = (qm_in(MR_PI_CINH) & QM_MR_SIZE) ? QM_MR_VERB_VBIT : 0;
462 mr->ithresh = qm_in(MR_ITR);
463 #ifdef RTE_LIBRTE_DPAA_HWDEBUG
467 cfg = (qm_in(CFG) & 0xfffff0ff) |
468 ((cmode & 1) << 8); /* QCSP_CFG:MM */
473 static inline void qm_mr_pvb_update(struct qm_portal *portal)
475 register struct qm_mr *mr = &portal->mr;
476 const struct qm_mr_entry *res = qm_cl(mr->ring, mr->pi);
478 #ifdef RTE_LIBRTE_DPAA_HWDEBUG
479 DPAA_ASSERT(mr->pmode == qm_mr_pvb);
481 /* when accessing 'verb', use __raw_readb() to ensure that compiler
482 * inlining doesn't try to optimise out "excess reads".
484 if ((__raw_readb(&res->verb) & QM_MR_VERB_VBIT) == mr->vbit) {
485 mr->pi = (mr->pi + 1) & (QM_MR_SIZE - 1);
487 mr->vbit ^= QM_MR_VERB_VBIT;
495 struct qman_portal *qman_create_portal(
496 struct qman_portal *portal,
497 const struct qm_portal_config *c,
498 const struct qman_cgrs *cgrs)
507 if (dpaa_svr_family == SVR_LS1043A_FAMILY)
508 portal->use_eqcr_ci_stashing = 3;
510 portal->use_eqcr_ci_stashing =
511 ((qman_ip_rev >= QMAN_REV30) ? 1 : 0);
514 * prep the low-level portal struct with the mapped addresses from the
515 * config, everything that follows depends on it and "config" is more
518 p->addr.ce = c->addr_virt[DPAA_PORTAL_CE];
519 p->addr.ci = c->addr_virt[DPAA_PORTAL_CI];
521 * If CI-stashing is used, the current defaults use a threshold of 3,
522 * and stash with high-than-DQRR priority.
524 if (qm_eqcr_init(p, qm_eqcr_pvb,
525 portal->use_eqcr_ci_stashing, 1)) {
526 pr_err("Qman EQCR initialisation failed\n");
529 if (qm_dqrr_init(p, c, qm_dqrr_dpush, qm_dqrr_pvb,
530 qm_dqrr_cdc, DQRR_MAXFILL)) {
531 pr_err("Qman DQRR initialisation failed\n");
534 if (qm_mr_init(p, qm_mr_pvb, qm_mr_cci)) {
535 pr_err("Qman MR initialisation failed\n");
539 pr_err("Qman MC initialisation failed\n");
543 /* static interrupt-gating controls */
544 qm_dqrr_set_ithresh(p, 0);
545 qm_mr_set_ithresh(p, 0);
546 qm_isr_set_iperiod(p, 0);
547 portal->cgrs = kmalloc(2 * sizeof(*cgrs), GFP_KERNEL);
550 /* initial snapshot is no-depletion */
551 qman_cgrs_init(&portal->cgrs[1]);
553 portal->cgrs[0] = *cgrs;
555 /* if the given mask is NULL, assume all CGRs can be seen */
556 qman_cgrs_fill(&portal->cgrs[0]);
557 INIT_LIST_HEAD(&portal->cgr_cbs);
558 spin_lock_init(&portal->cgr_lock);
560 portal->slowpoll = 0;
561 portal->sdqcr = QM_SDQCR_SOURCE_CHANNELS | QM_SDQCR_COUNT_UPTO3 |
562 QM_SDQCR_DEDICATED_PRECEDENCE | QM_SDQCR_TYPE_PRIO_QOS |
563 QM_SDQCR_TOKEN_SET(0xab) | QM_SDQCR_CHANNELS_DEDICATED;
564 portal->dqrr_disable_ref = 0;
565 portal->cb_dc_ern = NULL;
566 sprintf(buf, "qportal-%d", c->channel);
567 dpa_rbtree_init(&portal->retire_table);
569 qm_isr_disable_write(p, isdr);
570 portal->irq_sources = 0;
571 qm_isr_enable_write(p, portal->irq_sources);
572 qm_isr_status_clear(p, 0xffffffff);
573 snprintf(portal->irqname, MAX_IRQNAME, IRQNAME, c->cpu);
574 if (request_irq(c->irq, portal_isr, 0, portal->irqname,
576 pr_err("request_irq() failed\n");
580 /* Need EQCR to be empty before continuing */
581 isdr &= ~QM_PIRQ_EQCI;
582 qm_isr_disable_write(p, isdr);
583 ret = qm_eqcr_get_fill(p);
585 pr_err("Qman EQCR unclean\n");
586 goto fail_eqcr_empty;
588 isdr &= ~(QM_PIRQ_DQRI | QM_PIRQ_MRI);
589 qm_isr_disable_write(p, isdr);
590 if (qm_dqrr_current(p)) {
591 pr_err("Qman DQRR unclean\n");
592 qm_dqrr_cdc_consume_n(p, 0xffff);
594 if (qm_mr_current(p) && drain_mr_fqrni(p)) {
595 /* special handling, drain just in case it's a few FQRNIs */
596 if (drain_mr_fqrni(p))
597 goto fail_dqrr_mr_empty;
601 qm_isr_disable_write(p, 0);
603 /* Write a sane SDQCR */
604 qm_dqrr_sdqcr_set(p, portal->sdqcr);
608 free_irq(c->irq, portal);
611 spin_lock_destroy(&portal->cgr_lock);
624 #define MAX_GLOBAL_PORTALS 8
625 static struct qman_portal global_portals[MAX_GLOBAL_PORTALS];
626 static int global_portals_used[MAX_GLOBAL_PORTALS];
628 static struct qman_portal *
629 qman_alloc_global_portal(void)
633 for (i = 0; i < MAX_GLOBAL_PORTALS; i++) {
634 if (global_portals_used[i] == 0) {
635 global_portals_used[i] = 1;
636 return &global_portals[i];
639 pr_err("No portal available (%x)\n", MAX_GLOBAL_PORTALS);
645 qman_free_global_portal(struct qman_portal *portal)
649 for (i = 0; i < MAX_GLOBAL_PORTALS; i++) {
650 if (&global_portals[i] == portal) {
651 global_portals_used[i] = 0;
658 struct qman_portal *qman_create_affine_portal(const struct qm_portal_config *c,
659 const struct qman_cgrs *cgrs,
662 struct qman_portal *res;
663 struct qman_portal *portal;
666 portal = qman_alloc_global_portal();
668 portal = get_affine_portal();
670 /* A criteria for calling this function (from qman_driver.c) is that
671 * we're already affine to the cpu and won't schedule onto another cpu.
674 res = qman_create_portal(portal, c, cgrs);
676 spin_lock(&affine_mask_lock);
677 CPU_SET(c->cpu, &affine_mask);
678 affine_channels[c->cpu] =
680 spin_unlock(&affine_mask_lock);
686 void qman_destroy_portal(struct qman_portal *qm)
688 const struct qm_portal_config *pcfg;
690 /* Stop dequeues on the portal */
691 qm_dqrr_sdqcr_set(&qm->p, 0);
694 * NB we do this to "quiesce" EQCR. If we add enqueue-completions or
695 * something related to QM_PIRQ_EQCI, this may need fixing.
696 * Also, due to the prefetching model used for CI updates in the enqueue
697 * path, this update will only invalidate the CI cacheline *after*
698 * working on it, so we need to call this twice to ensure a full update
699 * irrespective of where the enqueue processing was at when the teardown
702 qm_eqcr_cce_update(&qm->p);
703 qm_eqcr_cce_update(&qm->p);
706 free_irq(pcfg->irq, qm);
709 qm_mc_finish(&qm->p);
710 qm_mr_finish(&qm->p);
711 qm_dqrr_finish(&qm->p);
712 qm_eqcr_finish(&qm->p);
716 spin_lock_destroy(&qm->cgr_lock);
719 const struct qm_portal_config *
720 qman_destroy_affine_portal(struct qman_portal *qp)
722 /* We don't want to redirect if we're a slave, use "raw" */
723 struct qman_portal *qm;
724 const struct qm_portal_config *pcfg;
728 qm = get_affine_portal();
734 qman_destroy_portal(qm);
736 spin_lock(&affine_mask_lock);
737 CPU_CLR(cpu, &affine_mask);
738 spin_unlock(&affine_mask_lock);
740 qman_free_global_portal(qm);
745 int qman_get_portal_index(void)
747 struct qman_portal *p = get_affine_portal();
748 return p->config->index;
751 /* Inline helper to reduce nesting in __poll_portal_slow() */
752 static inline void fq_state_change(struct qman_portal *p, struct qman_fq *fq,
753 const struct qm_mr_entry *msg, u8 verb)
757 case QM_MR_VERB_FQRL:
758 DPAA_ASSERT(fq_isset(fq, QMAN_FQ_STATE_ORL));
759 fq_clear(fq, QMAN_FQ_STATE_ORL);
762 case QM_MR_VERB_FQRN:
763 DPAA_ASSERT((fq->state == qman_fq_state_parked) ||
764 (fq->state == qman_fq_state_sched));
765 DPAA_ASSERT(fq_isset(fq, QMAN_FQ_STATE_CHANGING));
766 fq_clear(fq, QMAN_FQ_STATE_CHANGING);
767 if (msg->fq.fqs & QM_MR_FQS_NOTEMPTY)
768 fq_set(fq, QMAN_FQ_STATE_NE);
769 if (msg->fq.fqs & QM_MR_FQS_ORLPRESENT)
770 fq_set(fq, QMAN_FQ_STATE_ORL);
773 fq->state = qman_fq_state_retired;
775 case QM_MR_VERB_FQPN:
776 DPAA_ASSERT(fq->state == qman_fq_state_sched);
777 DPAA_ASSERT(fq_isclear(fq, QMAN_FQ_STATE_CHANGING));
778 fq->state = qman_fq_state_parked;
783 static u32 __poll_portal_slow(struct qman_portal *p, u32 is)
785 const struct qm_mr_entry *msg;
786 struct qm_mr_entry swapped_msg;
788 if (is & QM_PIRQ_CSCI) {
789 struct qman_cgrs rr, c;
790 struct qm_mc_result *mcr;
791 struct qman_cgr *cgr;
793 spin_lock(&p->cgr_lock);
795 * The CSCI bit must be cleared _before_ issuing the
796 * Query Congestion State command, to ensure that a long
797 * CGR State Change callback cannot miss an intervening
800 qm_isr_status_clear(&p->p, QM_PIRQ_CSCI);
802 qm_mc_commit(&p->p, QM_MCC_VERB_QUERYCONGESTION);
803 while (!(mcr = qm_mc_result(&p->p)))
805 /* mask out the ones I'm not interested in */
806 qman_cgrs_and(&rr, (const struct qman_cgrs *)
807 &mcr->querycongestion.state, &p->cgrs[0]);
808 /* check previous snapshot for delta, enter/exit congestion */
809 qman_cgrs_xor(&c, &rr, &p->cgrs[1]);
810 /* update snapshot */
811 qman_cgrs_cp(&p->cgrs[1], &rr);
812 /* Invoke callback */
813 list_for_each_entry(cgr, &p->cgr_cbs, node)
814 if (cgr->cb && qman_cgrs_get(&c, cgr->cgrid))
815 cgr->cb(p, cgr, qman_cgrs_get(&rr, cgr->cgrid));
816 spin_unlock(&p->cgr_lock);
819 if (is & QM_PIRQ_EQRI) {
820 qm_eqcr_cce_update(&p->p);
821 qm_eqcr_set_ithresh(&p->p, 0);
822 wake_up(&affine_queue);
825 if (is & QM_PIRQ_MRI) {
829 qm_mr_pvb_update(&p->p);
830 msg = qm_mr_current(&p->p);
834 hw_fd_to_cpu(&swapped_msg.ern.fd);
835 verb = msg->verb & QM_MR_VERB_TYPE_MASK;
836 /* The message is a software ERN iff the 0x20 bit is set */
839 case QM_MR_VERB_FQRNI:
840 /* nada, we drop FQRNIs on the floor */
842 case QM_MR_VERB_FQRN:
843 case QM_MR_VERB_FQRL:
844 /* Lookup in the retirement table */
845 fq = table_find_fq(p,
846 be32_to_cpu(msg->fq.fqid));
848 fq_state_change(p, fq, &swapped_msg, verb);
850 fq->cb.fqs(p, fq, &swapped_msg);
852 case QM_MR_VERB_FQPN:
854 #ifdef CONFIG_FSL_QMAN_FQ_LOOKUP
855 fq = get_fq_table_entry(
856 be32_to_cpu(msg->fq.contextB));
858 fq = (void *)(uintptr_t)
859 be32_to_cpu(msg->fq.contextB);
861 fq_state_change(p, fq, msg, verb);
863 fq->cb.fqs(p, fq, &swapped_msg);
865 case QM_MR_VERB_DC_ERN:
868 p->cb_dc_ern(p, msg);
872 static int warn_once;
875 pr_crit("Leaking DCP ERNs!\n");
881 pr_crit("Invalid MR verb 0x%02x\n", verb);
884 /* Its a software ERN */
885 #ifdef CONFIG_FSL_QMAN_FQ_LOOKUP
886 fq = get_fq_table_entry(be32_to_cpu(msg->ern.tag));
888 fq = (void *)(uintptr_t)be32_to_cpu(msg->ern.tag);
890 fq->cb.ern(p, fq, &swapped_msg);
896 qm_mr_cci_consume(&p->p, num);
899 * QM_PIRQ_CSCI/CCSCI has already been cleared, as part of its specific
900 * processing. If that interrupt source has meanwhile been re-asserted,
901 * we mustn't clear it here (or in the top-level interrupt handler).
903 return is & (QM_PIRQ_EQCI | QM_PIRQ_EQRI | QM_PIRQ_MRI);
907 * remove some slowish-path stuff from the "fast path" and make sure it isn't
910 static noinline void clear_vdqcr(struct qman_portal *p, struct qman_fq *fq)
912 p->vdqcr_owned = NULL;
914 fq_clear(fq, QMAN_FQ_STATE_VDQCR);
916 wake_up(&affine_queue);
920 * The only states that would conflict with other things if they ran at the
921 * same time on the same cpu are:
923 * (i) setting/clearing vdqcr_owned, and
924 * (ii) clearing the NE (Not Empty) flag.
926 * Both are safe. Because;
928 * (i) this clearing can only occur after qman_set_vdq() has set the
929 * vdqcr_owned field (which it does before setting VDQCR), and
930 * qman_volatile_dequeue() blocks interrupts and preemption while this is
931 * done so that we can't interfere.
932 * (ii) the NE flag is only cleared after qman_retire_fq() has set it, and as
933 * with (i) that API prevents us from interfering until it's safe.
935 * The good thing is that qman_set_vdq() and qman_retire_fq() run far
936 * less frequently (ie. per-FQ) than __poll_portal_fast() does, so the nett
937 * advantage comes from this function not having to "lock" anything at all.
939 * Note also that the callbacks are invoked at points which are safe against the
940 * above potential conflicts, but that this function itself is not re-entrant
941 * (this is because the function tracks one end of each FIFO in the portal and
942 * we do *not* want to lock that). So the consequence is that it is safe for
943 * user callbacks to call into any QMan API.
945 static inline unsigned int __poll_portal_fast(struct qman_portal *p,
946 unsigned int poll_limit)
948 const struct qm_dqrr_entry *dq;
950 enum qman_cb_dqrr_result res;
951 unsigned int limit = 0;
952 #if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__
953 struct qm_dqrr_entry *shadow;
956 qm_dqrr_pvb_update(&p->p);
957 dq = qm_dqrr_current(&p->p);
960 #if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__
961 /* If running on an LE system the fields of the
962 * dequeue entry must be swapper. Because the
963 * QMan HW will ignore writes the DQRR entry is
964 * copied and the index stored within the copy
966 shadow = &p->shadow_dqrr[DQRR_PTR2IDX(dq)];
969 shadow->fqid = be32_to_cpu(shadow->fqid);
970 shadow->contextB = be32_to_cpu(shadow->contextB);
971 shadow->seqnum = be16_to_cpu(shadow->seqnum);
972 hw_fd_to_cpu(&shadow->fd);
975 if (dq->stat & QM_DQRR_STAT_UNSCHEDULED) {
977 * VDQCR: don't trust context_b as the FQ may have
978 * been configured for h/w consumption and we're
979 * draining it post-retirement.
983 * We only set QMAN_FQ_STATE_NE when retiring, so we
984 * only need to check for clearing it when doing
985 * volatile dequeues. It's one less thing to check
986 * in the critical path (SDQCR).
988 if (dq->stat & QM_DQRR_STAT_FQ_EMPTY)
989 fq_clear(fq, QMAN_FQ_STATE_NE);
991 * This is duplicated from the SDQCR code, but we
992 * have stuff to do before *and* after this callback,
993 * and we don't want multiple if()s in the critical
996 res = fq->cb.dqrr(p, fq, dq);
997 if (res == qman_cb_dqrr_stop)
999 /* Check for VDQCR completion */
1000 if (dq->stat & QM_DQRR_STAT_DQCR_EXPIRED)
1003 /* SDQCR: context_b points to the FQ */
1004 #ifdef CONFIG_FSL_QMAN_FQ_LOOKUP
1005 fq = get_fq_table_entry(dq->contextB);
1007 fq = (void *)(uintptr_t)dq->contextB;
1009 /* Now let the callback do its stuff */
1010 res = fq->cb.dqrr(p, fq, dq);
1012 * The callback can request that we exit without
1013 * consuming this entry nor advancing;
1015 if (res == qman_cb_dqrr_stop)
1018 /* Interpret 'dq' from a driver perspective. */
1020 * Parking isn't possible unless HELDACTIVE was set. NB,
1021 * FORCEELIGIBLE implies HELDACTIVE, so we only need to
1022 * check for HELDACTIVE to cover both.
1024 DPAA_ASSERT((dq->stat & QM_DQRR_STAT_FQ_HELDACTIVE) ||
1025 (res != qman_cb_dqrr_park));
1026 /* just means "skip it, I'll consume it myself later on" */
1027 if (res != qman_cb_dqrr_defer)
1028 qm_dqrr_cdc_consume_1ptr(&p->p, dq,
1029 res == qman_cb_dqrr_park);
1031 qm_dqrr_next(&p->p);
1033 * Entry processed and consumed, increment our counter. The
1034 * callback can request that we exit after consuming the
1035 * entry, and we also exit if we reach our processing limit,
1036 * so loop back only if neither of these conditions is met.
1038 } while (++limit < poll_limit && res != qman_cb_dqrr_consume_stop);
1043 u16 qman_affine_channel(int cpu)
1046 struct qman_portal *portal = get_affine_portal();
1048 cpu = portal->config->cpu;
1050 DPAA_BUG_ON(!CPU_ISSET(cpu, &affine_mask));
1051 return affine_channels[cpu];
1054 unsigned int qman_portal_poll_rx(unsigned int poll_limit,
1056 struct qman_portal *p)
1058 const struct qm_dqrr_entry *dq;
1060 enum qman_cb_dqrr_result res;
1061 unsigned int limit = 0;
1062 #if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__
1063 struct qm_dqrr_entry *shadow;
1065 unsigned int rx_number = 0;
1068 qm_dqrr_pvb_update(&p->p);
1069 dq = qm_dqrr_current(&p->p);
1072 #if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__
1073 /* If running on an LE system the fields of the
1074 * dequeue entry must be swapper. Because the
1075 * QMan HW will ignore writes the DQRR entry is
1076 * copied and the index stored within the copy
1078 shadow = &p->shadow_dqrr[DQRR_PTR2IDX(dq)];
1081 shadow->fqid = be32_to_cpu(shadow->fqid);
1082 shadow->contextB = be32_to_cpu(shadow->contextB);
1083 shadow->seqnum = be16_to_cpu(shadow->seqnum);
1084 hw_fd_to_cpu(&shadow->fd);
1087 /* SDQCR: context_b points to the FQ */
1088 #ifdef CONFIG_FSL_QMAN_FQ_LOOKUP
1089 fq = get_fq_table_entry(dq->contextB);
1091 fq = (void *)(uintptr_t)dq->contextB;
1093 /* Now let the callback do its stuff */
1094 res = fq->cb.dqrr_dpdk_cb(NULL, p, fq, dq, &bufs[rx_number]);
1096 /* Interpret 'dq' from a driver perspective. */
1098 * Parking isn't possible unless HELDACTIVE was set. NB,
1099 * FORCEELIGIBLE implies HELDACTIVE, so we only need to
1100 * check for HELDACTIVE to cover both.
1102 DPAA_ASSERT((dq->stat & QM_DQRR_STAT_FQ_HELDACTIVE) ||
1103 (res != qman_cb_dqrr_park));
1104 qm_dqrr_cdc_consume_1ptr(&p->p, dq, res == qman_cb_dqrr_park);
1106 qm_dqrr_next(&p->p);
1108 * Entry processed and consumed, increment our counter. The
1109 * callback can request that we exit after consuming the
1110 * entry, and we also exit if we reach our processing limit,
1111 * so loop back only if neither of these conditions is met.
1113 } while (likely(++limit < poll_limit));
1118 struct qm_dqrr_entry *qman_dequeue(struct qman_fq *fq)
1120 struct qman_portal *p = get_affine_portal();
1121 const struct qm_dqrr_entry *dq;
1122 #if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__
1123 struct qm_dqrr_entry *shadow;
1126 qm_dqrr_pvb_update(&p->p);
1127 dq = qm_dqrr_current(&p->p);
1131 if (!(dq->stat & QM_DQRR_STAT_FD_VALID)) {
1132 /* Invalid DQRR - put the portal and consume the DQRR.
1133 * Return NULL to user as no packet is seen.
1135 qman_dqrr_consume(fq, (struct qm_dqrr_entry *)dq);
1139 #if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__
1140 shadow = &p->shadow_dqrr[DQRR_PTR2IDX(dq)];
1143 shadow->fqid = be32_to_cpu(shadow->fqid);
1144 shadow->contextB = be32_to_cpu(shadow->contextB);
1145 shadow->seqnum = be16_to_cpu(shadow->seqnum);
1146 hw_fd_to_cpu(&shadow->fd);
1149 if (dq->stat & QM_DQRR_STAT_FQ_EMPTY)
1150 fq_clear(fq, QMAN_FQ_STATE_NE);
1152 return (struct qm_dqrr_entry *)dq;
1155 void qman_dqrr_consume(struct qman_fq *fq,
1156 struct qm_dqrr_entry *dq)
1158 struct qman_portal *p = get_affine_portal();
1160 if (dq->stat & QM_DQRR_STAT_DQCR_EXPIRED)
1163 qm_dqrr_cdc_consume_1ptr(&p->p, dq, 0);
1164 qm_dqrr_next(&p->p);
1167 int qman_poll_dqrr(unsigned int limit)
1169 struct qman_portal *p = get_affine_portal();
1172 ret = __poll_portal_fast(p, limit);
1176 void qman_poll(void)
1178 struct qman_portal *p = get_affine_portal();
1180 if ((~p->irq_sources) & QM_PIRQ_SLOW) {
1181 if (!(p->slowpoll--)) {
1182 u32 is = qm_isr_status_read(&p->p) & ~p->irq_sources;
1183 u32 active = __poll_portal_slow(p, is);
1186 qm_isr_status_clear(&p->p, active);
1187 p->slowpoll = SLOW_POLL_BUSY;
1189 p->slowpoll = SLOW_POLL_IDLE;
1192 if ((~p->irq_sources) & QM_PIRQ_DQRI)
1193 __poll_portal_fast(p, FSL_QMAN_POLL_LIMIT);
1196 void qman_stop_dequeues(void)
1198 struct qman_portal *p = get_affine_portal();
1200 qman_stop_dequeues_ex(p);
1203 void qman_start_dequeues(void)
1205 struct qman_portal *p = get_affine_portal();
1207 DPAA_ASSERT(p->dqrr_disable_ref > 0);
1208 if (!(--p->dqrr_disable_ref))
1209 qm_dqrr_set_maxfill(&p->p, DQRR_MAXFILL);
1212 void qman_static_dequeue_add(u32 pools, struct qman_portal *qp)
1214 struct qman_portal *p = qp ? qp : get_affine_portal();
1216 pools &= p->config->pools;
1218 qm_dqrr_sdqcr_set(&p->p, p->sdqcr);
1221 void qman_static_dequeue_del(u32 pools, struct qman_portal *qp)
1223 struct qman_portal *p = qp ? qp : get_affine_portal();
1225 pools &= p->config->pools;
1227 qm_dqrr_sdqcr_set(&p->p, p->sdqcr);
1230 u32 qman_static_dequeue_get(struct qman_portal *qp)
1232 struct qman_portal *p = qp ? qp : get_affine_portal();
1236 void qman_dca(struct qm_dqrr_entry *dq, int park_request)
1238 struct qman_portal *p = get_affine_portal();
1240 qm_dqrr_cdc_consume_1ptr(&p->p, dq, park_request);
1243 /* Frame queue API */
1244 static const char *mcr_result_str(u8 result)
1247 case QM_MCR_RESULT_NULL:
1248 return "QM_MCR_RESULT_NULL";
1249 case QM_MCR_RESULT_OK:
1250 return "QM_MCR_RESULT_OK";
1251 case QM_MCR_RESULT_ERR_FQID:
1252 return "QM_MCR_RESULT_ERR_FQID";
1253 case QM_MCR_RESULT_ERR_FQSTATE:
1254 return "QM_MCR_RESULT_ERR_FQSTATE";
1255 case QM_MCR_RESULT_ERR_NOTEMPTY:
1256 return "QM_MCR_RESULT_ERR_NOTEMPTY";
1257 case QM_MCR_RESULT_PENDING:
1258 return "QM_MCR_RESULT_PENDING";
1259 case QM_MCR_RESULT_ERR_BADCOMMAND:
1260 return "QM_MCR_RESULT_ERR_BADCOMMAND";
1262 return "<unknown MCR result>";
1265 int qman_create_fq(u32 fqid, u32 flags, struct qman_fq *fq)
1268 struct qm_mcr_queryfq_np np;
1269 struct qm_mc_command *mcc;
1270 struct qm_mc_result *mcr;
1271 struct qman_portal *p;
1273 if (flags & QMAN_FQ_FLAG_DYNAMIC_FQID) {
1274 int ret = qman_alloc_fqid(&fqid);
1279 spin_lock_init(&fq->fqlock);
1281 fq->fqid_le = cpu_to_be32(fqid);
1283 fq->state = qman_fq_state_oos;
1284 fq->cgr_groupid = 0;
1285 #ifdef CONFIG_FSL_QMAN_FQ_LOOKUP
1286 if (unlikely(find_empty_fq_table_entry(&fq->key, fq))) {
1287 pr_info("Find empty table entry failed\n");
1291 if (!(flags & QMAN_FQ_FLAG_AS_IS) || (flags & QMAN_FQ_FLAG_NO_MODIFY))
1293 /* Everything else is AS_IS support */
1294 p = get_affine_portal();
1295 mcc = qm_mc_start(&p->p);
1296 mcc->queryfq.fqid = cpu_to_be32(fqid);
1297 qm_mc_commit(&p->p, QM_MCC_VERB_QUERYFQ);
1298 while (!(mcr = qm_mc_result(&p->p)))
1300 DPAA_ASSERT((mcr->verb & QM_MCR_VERB_MASK) == QM_MCC_VERB_QUERYFQ);
1301 if (mcr->result != QM_MCR_RESULT_OK) {
1302 pr_err("QUERYFQ failed: %s\n", mcr_result_str(mcr->result));
1305 fqd = mcr->queryfq.fqd;
1306 hw_fqd_to_cpu(&fqd);
1307 mcc = qm_mc_start(&p->p);
1308 mcc->queryfq_np.fqid = cpu_to_be32(fqid);
1309 qm_mc_commit(&p->p, QM_MCC_VERB_QUERYFQ_NP);
1310 while (!(mcr = qm_mc_result(&p->p)))
1312 DPAA_ASSERT((mcr->verb & QM_MCR_VERB_MASK) == QM_MCC_VERB_QUERYFQ_NP);
1313 if (mcr->result != QM_MCR_RESULT_OK) {
1314 pr_err("QUERYFQ_NP failed: %s\n", mcr_result_str(mcr->result));
1317 np = mcr->queryfq_np;
1318 /* Phew, have queryfq and queryfq_np results, stitch together
1319 * the FQ object from those.
1321 fq->cgr_groupid = fqd.cgid;
1322 switch (np.state & QM_MCR_NP_STATE_MASK) {
1323 case QM_MCR_NP_STATE_OOS:
1325 case QM_MCR_NP_STATE_RETIRED:
1326 fq->state = qman_fq_state_retired;
1328 fq_set(fq, QMAN_FQ_STATE_NE);
1330 case QM_MCR_NP_STATE_TEN_SCHED:
1331 case QM_MCR_NP_STATE_TRU_SCHED:
1332 case QM_MCR_NP_STATE_ACTIVE:
1333 fq->state = qman_fq_state_sched;
1334 if (np.state & QM_MCR_NP_STATE_R)
1335 fq_set(fq, QMAN_FQ_STATE_CHANGING);
1337 case QM_MCR_NP_STATE_PARKED:
1338 fq->state = qman_fq_state_parked;
1341 DPAA_ASSERT(NULL == "invalid FQ state");
1343 if (fqd.fq_ctrl & QM_FQCTRL_CGE)
1344 fq->state |= QMAN_FQ_STATE_CGR_EN;
1347 if (flags & QMAN_FQ_FLAG_DYNAMIC_FQID)
1348 qman_release_fqid(fqid);
1352 void qman_destroy_fq(struct qman_fq *fq, u32 flags __maybe_unused)
1355 * We don't need to lock the FQ as it is a pre-condition that the FQ be
1356 * quiesced. Instead, run some checks.
1358 switch (fq->state) {
1359 case qman_fq_state_parked:
1360 DPAA_ASSERT(flags & QMAN_FQ_DESTROY_PARKED);
1362 case qman_fq_state_oos:
1363 if (fq_isset(fq, QMAN_FQ_FLAG_DYNAMIC_FQID))
1364 qman_release_fqid(fq->fqid);
1365 #ifdef CONFIG_FSL_QMAN_FQ_LOOKUP
1366 clear_fq_table_entry(fq->key);
1372 DPAA_ASSERT(NULL == "qman_free_fq() on unquiesced FQ!");
1375 u32 qman_fq_fqid(struct qman_fq *fq)
1380 void qman_fq_state(struct qman_fq *fq, enum qman_fq_state *state, u32 *flags)
1388 int qman_init_fq(struct qman_fq *fq, u32 flags, struct qm_mcc_initfq *opts)
1390 struct qm_mc_command *mcc;
1391 struct qm_mc_result *mcr;
1392 struct qman_portal *p;
1394 u8 res, myverb = (flags & QMAN_INITFQ_FLAG_SCHED) ?
1395 QM_MCC_VERB_INITFQ_SCHED : QM_MCC_VERB_INITFQ_PARKED;
1397 if ((fq->state != qman_fq_state_oos) &&
1398 (fq->state != qman_fq_state_parked))
1400 #ifdef RTE_LIBRTE_DPAA_HWDEBUG
1401 if (unlikely(fq_isset(fq, QMAN_FQ_FLAG_NO_MODIFY)))
1404 if (opts && (opts->we_mask & QM_INITFQ_WE_OAC)) {
1405 /* And can't be set at the same time as TDTHRESH */
1406 if (opts->we_mask & QM_INITFQ_WE_TDTHRESH)
1409 /* Issue an INITFQ_[PARKED|SCHED] management command */
1410 p = get_affine_portal();
1412 if (unlikely((fq_isset(fq, QMAN_FQ_STATE_CHANGING)) ||
1413 ((fq->state != qman_fq_state_oos) &&
1414 (fq->state != qman_fq_state_parked)))) {
1418 mcc = qm_mc_start(&p->p);
1420 mcc->initfq = *opts;
1421 mcc->initfq.fqid = cpu_to_be32(fq->fqid);
1422 mcc->initfq.count = 0;
1424 * If the FQ does *not* have the TO_DCPORTAL flag, context_b is set as a
1425 * demux pointer. Otherwise, the caller-provided value is allowed to
1426 * stand, don't overwrite it.
1428 if (fq_isclear(fq, QMAN_FQ_FLAG_TO_DCPORTAL)) {
1431 mcc->initfq.we_mask |= QM_INITFQ_WE_CONTEXTB;
1432 #ifdef CONFIG_FSL_QMAN_FQ_LOOKUP
1433 mcc->initfq.fqd.context_b = fq->key;
1435 mcc->initfq.fqd.context_b = (u32)(uintptr_t)fq;
1438 * and the physical address - NB, if the user wasn't trying to
1439 * set CONTEXTA, clear the stashing settings.
1441 if (!(mcc->initfq.we_mask & QM_INITFQ_WE_CONTEXTA)) {
1442 mcc->initfq.we_mask |= QM_INITFQ_WE_CONTEXTA;
1443 memset(&mcc->initfq.fqd.context_a, 0,
1444 sizeof(mcc->initfq.fqd.context_a));
1446 phys_fq = rte_mem_virt2iova(fq);
1447 qm_fqd_stashing_set64(&mcc->initfq.fqd, phys_fq);
1450 if (flags & QMAN_INITFQ_FLAG_LOCAL) {
1451 mcc->initfq.fqd.dest.channel = p->config->channel;
1452 if (!(mcc->initfq.we_mask & QM_INITFQ_WE_DESTWQ)) {
1453 mcc->initfq.we_mask |= QM_INITFQ_WE_DESTWQ;
1454 mcc->initfq.fqd.dest.wq = 4;
1457 mcc->initfq.we_mask = cpu_to_be16(mcc->initfq.we_mask);
1458 cpu_to_hw_fqd(&mcc->initfq.fqd);
1459 qm_mc_commit(&p->p, myverb);
1460 while (!(mcr = qm_mc_result(&p->p)))
1462 DPAA_ASSERT((mcr->verb & QM_MCR_VERB_MASK) == myverb);
1464 if (res != QM_MCR_RESULT_OK) {
1469 if (opts->we_mask & QM_INITFQ_WE_FQCTRL) {
1470 if (opts->fqd.fq_ctrl & QM_FQCTRL_CGE)
1471 fq_set(fq, QMAN_FQ_STATE_CGR_EN);
1473 fq_clear(fq, QMAN_FQ_STATE_CGR_EN);
1475 if (opts->we_mask & QM_INITFQ_WE_CGID)
1476 fq->cgr_groupid = opts->fqd.cgid;
1478 fq->state = (flags & QMAN_INITFQ_FLAG_SCHED) ?
1479 qman_fq_state_sched : qman_fq_state_parked;
1484 int qman_schedule_fq(struct qman_fq *fq)
1486 struct qm_mc_command *mcc;
1487 struct qm_mc_result *mcr;
1488 struct qman_portal *p;
1493 if (fq->state != qman_fq_state_parked)
1495 #ifdef RTE_LIBRTE_DPAA_HWDEBUG
1496 if (unlikely(fq_isset(fq, QMAN_FQ_FLAG_NO_MODIFY)))
1499 /* Issue a ALTERFQ_SCHED management command */
1500 p = get_affine_portal();
1503 if (unlikely((fq_isset(fq, QMAN_FQ_STATE_CHANGING)) ||
1504 (fq->state != qman_fq_state_parked))) {
1508 mcc = qm_mc_start(&p->p);
1509 mcc->alterfq.fqid = cpu_to_be32(fq->fqid);
1510 qm_mc_commit(&p->p, QM_MCC_VERB_ALTER_SCHED);
1511 while (!(mcr = qm_mc_result(&p->p)))
1513 DPAA_ASSERT((mcr->verb & QM_MCR_VERB_MASK) == QM_MCR_VERB_ALTER_SCHED);
1515 if (res != QM_MCR_RESULT_OK) {
1519 fq->state = qman_fq_state_sched;
1526 int qman_retire_fq(struct qman_fq *fq, u32 *flags)
1528 struct qm_mc_command *mcc;
1529 struct qm_mc_result *mcr;
1530 struct qman_portal *p;
1535 if ((fq->state != qman_fq_state_parked) &&
1536 (fq->state != qman_fq_state_sched))
1538 #ifdef RTE_LIBRTE_DPAA_HWDEBUG
1539 if (unlikely(fq_isset(fq, QMAN_FQ_FLAG_NO_MODIFY)))
1542 p = get_affine_portal();
1545 if (unlikely((fq_isset(fq, QMAN_FQ_STATE_CHANGING)) ||
1546 (fq->state == qman_fq_state_retired) ||
1547 (fq->state == qman_fq_state_oos))) {
1551 rval = table_push_fq(p, fq);
1554 mcc = qm_mc_start(&p->p);
1555 mcc->alterfq.fqid = cpu_to_be32(fq->fqid);
1556 qm_mc_commit(&p->p, QM_MCC_VERB_ALTER_RETIRE);
1557 while (!(mcr = qm_mc_result(&p->p)))
1559 DPAA_ASSERT((mcr->verb & QM_MCR_VERB_MASK) == QM_MCR_VERB_ALTER_RETIRE);
1562 * "Elegant" would be to treat OK/PENDING the same way; set CHANGING,
1563 * and defer the flags until FQRNI or FQRN (respectively) show up. But
1564 * "Friendly" is to process OK immediately, and not set CHANGING. We do
1565 * friendly, otherwise the caller doesn't necessarily have a fully
1566 * "retired" FQ on return even if the retirement was immediate. However
1567 * this does mean some code duplication between here and
1568 * fq_state_change().
1570 if (likely(res == QM_MCR_RESULT_OK)) {
1572 /* Process 'fq' right away, we'll ignore FQRNI */
1573 if (mcr->alterfq.fqs & QM_MCR_FQS_NOTEMPTY)
1574 fq_set(fq, QMAN_FQ_STATE_NE);
1575 if (mcr->alterfq.fqs & QM_MCR_FQS_ORLPRESENT)
1576 fq_set(fq, QMAN_FQ_STATE_ORL);
1578 table_del_fq(p, fq);
1581 fq->state = qman_fq_state_retired;
1584 * Another issue with supporting "immediate" retirement
1585 * is that we're forced to drop FQRNIs, because by the
1586 * time they're seen it may already be "too late" (the
1587 * fq may have been OOS'd and free()'d already). But if
1588 * the upper layer wants a callback whether it's
1589 * immediate or not, we have to fake a "MR" entry to
1590 * look like an FQRNI...
1592 struct qm_mr_entry msg;
1594 msg.verb = QM_MR_VERB_FQRNI;
1595 msg.fq.fqs = mcr->alterfq.fqs;
1596 msg.fq.fqid = fq->fqid;
1597 #ifdef CONFIG_FSL_QMAN_FQ_LOOKUP
1598 msg.fq.contextB = fq->key;
1600 msg.fq.contextB = (u32)(uintptr_t)fq;
1602 fq->cb.fqs(p, fq, &msg);
1604 } else if (res == QM_MCR_RESULT_PENDING) {
1606 fq_set(fq, QMAN_FQ_STATE_CHANGING);
1609 table_del_fq(p, fq);
1616 int qman_oos_fq(struct qman_fq *fq)
1618 struct qm_mc_command *mcc;
1619 struct qm_mc_result *mcr;
1620 struct qman_portal *p;
1625 if (fq->state != qman_fq_state_retired)
1627 #ifdef RTE_LIBRTE_DPAA_HWDEBUG
1628 if (unlikely(fq_isset(fq, QMAN_FQ_FLAG_NO_MODIFY)))
1631 p = get_affine_portal();
1633 if (unlikely((fq_isset(fq, QMAN_FQ_STATE_BLOCKOOS)) ||
1634 (fq->state != qman_fq_state_retired))) {
1638 mcc = qm_mc_start(&p->p);
1639 mcc->alterfq.fqid = cpu_to_be32(fq->fqid);
1640 qm_mc_commit(&p->p, QM_MCC_VERB_ALTER_OOS);
1641 while (!(mcr = qm_mc_result(&p->p)))
1643 DPAA_ASSERT((mcr->verb & QM_MCR_VERB_MASK) == QM_MCR_VERB_ALTER_OOS);
1645 if (res != QM_MCR_RESULT_OK) {
1649 fq->state = qman_fq_state_oos;
1655 int qman_fq_flow_control(struct qman_fq *fq, int xon)
1657 struct qm_mc_command *mcc;
1658 struct qm_mc_result *mcr;
1659 struct qman_portal *p;
1665 if ((fq->state == qman_fq_state_oos) ||
1666 (fq->state == qman_fq_state_retired) ||
1667 (fq->state == qman_fq_state_parked))
1670 #ifdef RTE_LIBRTE_DPAA_HWDEBUG
1671 if (unlikely(fq_isset(fq, QMAN_FQ_FLAG_NO_MODIFY)))
1674 /* Issue a ALTER_FQXON or ALTER_FQXOFF management command */
1675 p = get_affine_portal();
1677 if (unlikely((fq_isset(fq, QMAN_FQ_STATE_CHANGING)) ||
1678 (fq->state == qman_fq_state_parked) ||
1679 (fq->state == qman_fq_state_oos) ||
1680 (fq->state == qman_fq_state_retired))) {
1684 mcc = qm_mc_start(&p->p);
1685 mcc->alterfq.fqid = fq->fqid;
1686 mcc->alterfq.count = 0;
1687 myverb = xon ? QM_MCC_VERB_ALTER_FQXON : QM_MCC_VERB_ALTER_FQXOFF;
1689 qm_mc_commit(&p->p, myverb);
1690 while (!(mcr = qm_mc_result(&p->p)))
1692 DPAA_ASSERT((mcr->verb & QM_MCR_VERB_MASK) == myverb);
1695 if (res != QM_MCR_RESULT_OK) {
1704 int qman_query_fq(struct qman_fq *fq, struct qm_fqd *fqd)
1706 struct qm_mc_command *mcc;
1707 struct qm_mc_result *mcr;
1708 struct qman_portal *p = get_affine_portal();
1712 mcc = qm_mc_start(&p->p);
1713 mcc->queryfq.fqid = cpu_to_be32(fq->fqid);
1714 qm_mc_commit(&p->p, QM_MCC_VERB_QUERYFQ);
1715 while (!(mcr = qm_mc_result(&p->p)))
1717 DPAA_ASSERT((mcr->verb & QM_MCR_VERB_MASK) == QM_MCR_VERB_QUERYFQ);
1719 if (res == QM_MCR_RESULT_OK)
1720 *fqd = mcr->queryfq.fqd;
1722 if (res != QM_MCR_RESULT_OK)
1727 int qman_query_fq_has_pkts(struct qman_fq *fq)
1729 struct qm_mc_command *mcc;
1730 struct qm_mc_result *mcr;
1731 struct qman_portal *p = get_affine_portal();
1736 mcc = qm_mc_start(&p->p);
1737 mcc->queryfq.fqid = cpu_to_be32(fq->fqid);
1738 qm_mc_commit(&p->p, QM_MCC_VERB_QUERYFQ_NP);
1739 while (!(mcr = qm_mc_result(&p->p)))
1742 if (res == QM_MCR_RESULT_OK)
1743 ret = !!mcr->queryfq_np.frm_cnt;
1747 int qman_query_fq_np(struct qman_fq *fq, struct qm_mcr_queryfq_np *np)
1749 struct qm_mc_command *mcc;
1750 struct qm_mc_result *mcr;
1751 struct qman_portal *p = get_affine_portal();
1755 mcc = qm_mc_start(&p->p);
1756 mcc->queryfq.fqid = cpu_to_be32(fq->fqid);
1757 qm_mc_commit(&p->p, QM_MCC_VERB_QUERYFQ_NP);
1758 while (!(mcr = qm_mc_result(&p->p)))
1760 DPAA_ASSERT((mcr->verb & QM_MCR_VERB_MASK) == QM_MCR_VERB_QUERYFQ_NP);
1762 if (res == QM_MCR_RESULT_OK) {
1763 *np = mcr->queryfq_np;
1764 np->fqd_link = be24_to_cpu(np->fqd_link);
1765 np->odp_seq = be16_to_cpu(np->odp_seq);
1766 np->orp_nesn = be16_to_cpu(np->orp_nesn);
1767 np->orp_ea_hseq = be16_to_cpu(np->orp_ea_hseq);
1768 np->orp_ea_tseq = be16_to_cpu(np->orp_ea_tseq);
1769 np->orp_ea_hptr = be24_to_cpu(np->orp_ea_hptr);
1770 np->orp_ea_tptr = be24_to_cpu(np->orp_ea_tptr);
1771 np->pfdr_hptr = be24_to_cpu(np->pfdr_hptr);
1772 np->pfdr_tptr = be24_to_cpu(np->pfdr_tptr);
1773 np->ics_surp = be16_to_cpu(np->ics_surp);
1774 np->byte_cnt = be32_to_cpu(np->byte_cnt);
1775 np->frm_cnt = be24_to_cpu(np->frm_cnt);
1776 np->ra1_sfdr = be16_to_cpu(np->ra1_sfdr);
1777 np->ra2_sfdr = be16_to_cpu(np->ra2_sfdr);
1778 np->od1_sfdr = be16_to_cpu(np->od1_sfdr);
1779 np->od2_sfdr = be16_to_cpu(np->od2_sfdr);
1780 np->od3_sfdr = be16_to_cpu(np->od3_sfdr);
1782 if (res == QM_MCR_RESULT_ERR_FQID)
1784 else if (res != QM_MCR_RESULT_OK)
1789 int qman_query_fq_frm_cnt(struct qman_fq *fq, u32 *frm_cnt)
1791 struct qm_mc_command *mcc;
1792 struct qm_mc_result *mcr;
1793 struct qman_portal *p = get_affine_portal();
1795 mcc = qm_mc_start(&p->p);
1796 mcc->queryfq.fqid = cpu_to_be32(fq->fqid);
1797 qm_mc_commit(&p->p, QM_MCC_VERB_QUERYFQ_NP);
1798 while (!(mcr = qm_mc_result(&p->p)))
1800 DPAA_ASSERT((mcr->verb & QM_MCR_VERB_MASK) == QM_MCR_VERB_QUERYFQ_NP);
1802 if (mcr->result == QM_MCR_RESULT_OK)
1803 *frm_cnt = be24_to_cpu(mcr->queryfq_np.frm_cnt);
1804 else if (mcr->result == QM_MCR_RESULT_ERR_FQID)
1806 else if (mcr->result != QM_MCR_RESULT_OK)
1811 int qman_query_wq(u8 query_dedicated, struct qm_mcr_querywq *wq)
1813 struct qm_mc_command *mcc;
1814 struct qm_mc_result *mcr;
1815 struct qman_portal *p = get_affine_portal();
1819 myverb = (query_dedicated) ? QM_MCR_VERB_QUERYWQ_DEDICATED :
1820 QM_MCR_VERB_QUERYWQ;
1821 mcc = qm_mc_start(&p->p);
1822 mcc->querywq.channel.id = cpu_to_be16(wq->channel.id);
1823 qm_mc_commit(&p->p, myverb);
1824 while (!(mcr = qm_mc_result(&p->p)))
1826 DPAA_ASSERT((mcr->verb & QM_MCR_VERB_MASK) == myverb);
1828 if (res == QM_MCR_RESULT_OK) {
1831 wq->channel.id = be16_to_cpu(mcr->querywq.channel.id);
1832 array_len = ARRAY_SIZE(mcr->querywq.wq_len);
1833 for (i = 0; i < array_len; i++)
1834 wq->wq_len[i] = be32_to_cpu(mcr->querywq.wq_len[i]);
1836 if (res != QM_MCR_RESULT_OK) {
1837 pr_err("QUERYWQ failed: %s\n", mcr_result_str(res));
1843 int qman_testwrite_cgr(struct qman_cgr *cgr, u64 i_bcnt,
1844 struct qm_mcr_cgrtestwrite *result)
1846 struct qm_mc_command *mcc;
1847 struct qm_mc_result *mcr;
1848 struct qman_portal *p = get_affine_portal();
1852 mcc = qm_mc_start(&p->p);
1853 mcc->cgrtestwrite.cgid = cgr->cgrid;
1854 mcc->cgrtestwrite.i_bcnt_hi = (u8)(i_bcnt >> 32);
1855 mcc->cgrtestwrite.i_bcnt_lo = (u32)i_bcnt;
1856 qm_mc_commit(&p->p, QM_MCC_VERB_CGRTESTWRITE);
1857 while (!(mcr = qm_mc_result(&p->p)))
1859 DPAA_ASSERT((mcr->verb & QM_MCR_VERB_MASK) == QM_MCC_VERB_CGRTESTWRITE);
1861 if (res == QM_MCR_RESULT_OK)
1862 *result = mcr->cgrtestwrite;
1863 if (res != QM_MCR_RESULT_OK) {
1864 pr_err("CGR TEST WRITE failed: %s\n", mcr_result_str(res));
1870 int qman_query_cgr(struct qman_cgr *cgr, struct qm_mcr_querycgr *cgrd)
1872 struct qm_mc_command *mcc;
1873 struct qm_mc_result *mcr;
1874 struct qman_portal *p = get_affine_portal();
1878 mcc = qm_mc_start(&p->p);
1879 mcc->querycgr.cgid = cgr->cgrid;
1880 qm_mc_commit(&p->p, QM_MCC_VERB_QUERYCGR);
1881 while (!(mcr = qm_mc_result(&p->p)))
1883 DPAA_ASSERT((mcr->verb & QM_MCR_VERB_MASK) == QM_MCC_VERB_QUERYCGR);
1885 if (res == QM_MCR_RESULT_OK)
1886 *cgrd = mcr->querycgr;
1887 if (res != QM_MCR_RESULT_OK) {
1888 pr_err("QUERY_CGR failed: %s\n", mcr_result_str(res));
1891 cgrd->cgr.wr_parm_g.word =
1892 be32_to_cpu(cgrd->cgr.wr_parm_g.word);
1893 cgrd->cgr.wr_parm_y.word =
1894 be32_to_cpu(cgrd->cgr.wr_parm_y.word);
1895 cgrd->cgr.wr_parm_r.word =
1896 be32_to_cpu(cgrd->cgr.wr_parm_r.word);
1897 cgrd->cgr.cscn_targ = be32_to_cpu(cgrd->cgr.cscn_targ);
1898 cgrd->cgr.__cs_thres = be16_to_cpu(cgrd->cgr.__cs_thres);
1899 for (i = 0; i < ARRAY_SIZE(cgrd->cscn_targ_swp); i++)
1900 cgrd->cscn_targ_swp[i] =
1901 be32_to_cpu(cgrd->cscn_targ_swp[i]);
1905 int qman_query_congestion(struct qm_mcr_querycongestion *congestion)
1907 struct qm_mc_result *mcr;
1908 struct qman_portal *p = get_affine_portal();
1913 qm_mc_commit(&p->p, QM_MCC_VERB_QUERYCONGESTION);
1914 while (!(mcr = qm_mc_result(&p->p)))
1916 DPAA_ASSERT((mcr->verb & QM_MCR_VERB_MASK) ==
1917 QM_MCC_VERB_QUERYCONGESTION);
1919 if (res == QM_MCR_RESULT_OK)
1920 *congestion = mcr->querycongestion;
1921 if (res != QM_MCR_RESULT_OK) {
1922 pr_err("QUERY_CONGESTION failed: %s\n", mcr_result_str(res));
1925 for (i = 0; i < ARRAY_SIZE(congestion->state.state); i++)
1926 congestion->state.state[i] =
1927 be32_to_cpu(congestion->state.state[i]);
1931 int qman_set_vdq(struct qman_fq *fq, u16 num)
1933 struct qman_portal *p = get_affine_portal();
1937 vdqcr = QM_VDQCR_EXACT;
1938 vdqcr |= QM_VDQCR_NUMFRAMES_SET(num);
1940 if ((fq->state != qman_fq_state_parked) &&
1941 (fq->state != qman_fq_state_retired)) {
1945 if (fq_isset(fq, QMAN_FQ_STATE_VDQCR)) {
1949 vdqcr = (vdqcr & ~QM_VDQCR_FQID_MASK) | fq->fqid;
1951 if (!p->vdqcr_owned) {
1953 if (fq_isset(fq, QMAN_FQ_STATE_VDQCR))
1955 fq_set(fq, QMAN_FQ_STATE_VDQCR);
1957 p->vdqcr_owned = fq;
1962 qm_dqrr_vdqcr_set(&p->p, vdqcr);
1968 int qman_volatile_dequeue(struct qman_fq *fq, u32 flags __maybe_unused,
1971 struct qman_portal *p;
1974 if ((fq->state != qman_fq_state_parked) &&
1975 (fq->state != qman_fq_state_retired))
1977 if (vdqcr & QM_VDQCR_FQID_MASK)
1979 if (fq_isset(fq, QMAN_FQ_STATE_VDQCR))
1981 vdqcr = (vdqcr & ~QM_VDQCR_FQID_MASK) | fq->fqid;
1983 p = get_affine_portal();
1985 if (!p->vdqcr_owned) {
1987 if (fq_isset(fq, QMAN_FQ_STATE_VDQCR))
1989 fq_set(fq, QMAN_FQ_STATE_VDQCR);
1991 p->vdqcr_owned = fq;
1999 qm_dqrr_vdqcr_set(&p->p, vdqcr);
2003 static noinline void update_eqcr_ci(struct qman_portal *p, u8 avail)
2006 qm_eqcr_cce_prefetch(&p->p);
2008 qm_eqcr_cce_update(&p->p);
2011 int qman_eqcr_is_empty(void)
2013 struct qman_portal *p = get_affine_portal();
2016 update_eqcr_ci(p, 0);
2017 avail = qm_eqcr_get_fill(&p->p);
2018 return (avail == 0);
2021 void qman_set_dc_ern(qman_cb_dc_ern handler, int affine)
2024 struct qman_portal *p = get_affine_portal();
2026 p->cb_dc_ern = handler;
2028 cb_dc_ern = handler;
2031 static inline struct qm_eqcr_entry *try_p_eq_start(struct qman_portal *p,
2033 const struct qm_fd *fd,
2036 struct qm_eqcr_entry *eq;
2039 if (p->use_eqcr_ci_stashing) {
2041 * The stashing case is easy, only update if we need to in
2042 * order to try and liberate ring entries.
2044 eq = qm_eqcr_start_stash(&p->p);
2047 * The non-stashing case is harder, need to prefetch ahead of
2050 avail = qm_eqcr_get_avail(&p->p);
2052 update_eqcr_ci(p, avail);
2053 eq = qm_eqcr_start_no_stash(&p->p);
2059 if (flags & QMAN_ENQUEUE_FLAG_DCA)
2060 eq->dca = QM_EQCR_DCA_ENABLE |
2061 ((flags & QMAN_ENQUEUE_FLAG_DCA_PARK) ?
2062 QM_EQCR_DCA_PARK : 0) |
2063 ((flags >> 8) & QM_EQCR_DCA_IDXMASK);
2064 eq->fqid = cpu_to_be32(fq->fqid);
2065 #ifdef CONFIG_FSL_QMAN_FQ_LOOKUP
2066 eq->tag = cpu_to_be32(fq->key);
2068 eq->tag = cpu_to_be32((u32)(uintptr_t)fq);
2071 cpu_to_hw_fd(&eq->fd);
2075 int qman_enqueue(struct qman_fq *fq, const struct qm_fd *fd, u32 flags)
2077 struct qman_portal *p = get_affine_portal();
2078 struct qm_eqcr_entry *eq;
2080 eq = try_p_eq_start(p, fq, fd, flags);
2083 /* Note: QM_EQCR_VERB_INTERRUPT == QMAN_ENQUEUE_FLAG_WAIT_SYNC */
2084 qm_eqcr_pvb_commit(&p->p, QM_EQCR_VERB_CMD_ENQUEUE |
2085 (flags & (QM_EQCR_VERB_COLOUR_MASK | QM_EQCR_VERB_INTERRUPT)));
2086 /* Factor the below out, it's used from qman_enqueue_orp() too */
2090 int qman_enqueue_multi(struct qman_fq *fq,
2091 const struct qm_fd *fd,
2094 struct qman_portal *p = get_affine_portal();
2095 struct qm_portal *portal = &p->p;
2097 register struct qm_eqcr *eqcr = &portal->eqcr;
2098 struct qm_eqcr_entry *eq = eqcr->cursor, *prev_eq;
2100 u8 i, diff, old_ci, sent = 0;
2102 /* Update the available entries if no entry is free */
2103 if (!eqcr->available) {
2105 eqcr->ci = qm_cl_in(EQCR_CI) & (QM_EQCR_SIZE - 1);
2106 diff = qm_cyc_diff(QM_EQCR_SIZE, old_ci, eqcr->ci);
2107 eqcr->available += diff;
2112 /* try to send as many frames as possible */
2113 while (eqcr->available && frames_to_send--) {
2114 eq->fqid = fq->fqid_le;
2115 #ifdef CONFIG_FSL_QMAN_FQ_LOOKUP
2116 eq->tag = cpu_to_be32(fq->key);
2118 eq->tag = cpu_to_be32((u32)(uintptr_t)fq);
2120 eq->fd.opaque_addr = fd->opaque_addr;
2121 eq->fd.addr = cpu_to_be40(fd->addr);
2122 eq->fd.status = cpu_to_be32(fd->status);
2123 eq->fd.opaque = cpu_to_be32(fd->opaque);
2125 eq = (void *)((unsigned long)(eq + 1) &
2126 (~(unsigned long)(QM_EQCR_SIZE << 6)));
2133 /* In order for flushes to complete faster, all lines are recorded in
2137 for (i = 0; i < sent; i++) {
2138 eq->__dont_write_directly__verb =
2139 QM_EQCR_VERB_CMD_ENQUEUE | eqcr->vbit;
2141 eq = (void *)((unsigned long)(eq + 1) &
2142 (~(unsigned long)(QM_EQCR_SIZE << 6)));
2143 if (unlikely((prev_eq + 1) != eq))
2144 eqcr->vbit ^= QM_EQCR_VERB_VBIT;
2147 /* We need to flush all the lines but without load/store operations
2151 for (i = 0; i < sent; i++) {
2153 eq = (void *)((unsigned long)(eq + 1) &
2154 (~(unsigned long)(QM_EQCR_SIZE << 6)));
2156 /* Update cursor for the next call */
2162 qman_enqueue_multi_fq(struct qman_fq *fq[], const struct qm_fd *fd,
2165 struct qman_portal *p = get_affine_portal();
2166 struct qm_portal *portal = &p->p;
2168 register struct qm_eqcr *eqcr = &portal->eqcr;
2169 struct qm_eqcr_entry *eq = eqcr->cursor, *prev_eq;
2171 u8 i, diff, old_ci, sent = 0;
2173 /* Update the available entries if no entry is free */
2174 if (!eqcr->available) {
2176 eqcr->ci = qm_cl_in(EQCR_CI) & (QM_EQCR_SIZE - 1);
2177 diff = qm_cyc_diff(QM_EQCR_SIZE, old_ci, eqcr->ci);
2178 eqcr->available += diff;
2183 /* try to send as many frames as possible */
2184 while (eqcr->available && frames_to_send--) {
2185 eq->fqid = fq[sent]->fqid_le;
2186 eq->fd.opaque_addr = fd->opaque_addr;
2187 eq->fd.addr = cpu_to_be40(fd->addr);
2188 eq->fd.status = cpu_to_be32(fd->status);
2189 eq->fd.opaque = cpu_to_be32(fd->opaque);
2191 eq = (void *)((unsigned long)(eq + 1) &
2192 (~(unsigned long)(QM_EQCR_SIZE << 6)));
2199 /* In order for flushes to complete faster, all lines are recorded in
2203 for (i = 0; i < sent; i++) {
2204 eq->__dont_write_directly__verb =
2205 QM_EQCR_VERB_CMD_ENQUEUE | eqcr->vbit;
2207 eq = (void *)((unsigned long)(eq + 1) &
2208 (~(unsigned long)(QM_EQCR_SIZE << 6)));
2209 if (unlikely((prev_eq + 1) != eq))
2210 eqcr->vbit ^= QM_EQCR_VERB_VBIT;
2213 /* We need to flush all the lines but without load/store operations
2217 for (i = 0; i < sent; i++) {
2219 eq = (void *)((unsigned long)(eq + 1) &
2220 (~(unsigned long)(QM_EQCR_SIZE << 6)));
2222 /* Update cursor for the next call */
2227 int qman_enqueue_orp(struct qman_fq *fq, const struct qm_fd *fd, u32 flags,
2228 struct qman_fq *orp, u16 orp_seqnum)
2230 struct qman_portal *p = get_affine_portal();
2231 struct qm_eqcr_entry *eq;
2233 eq = try_p_eq_start(p, fq, fd, flags);
2236 /* Process ORP-specifics here */
2237 if (flags & QMAN_ENQUEUE_FLAG_NLIS)
2238 orp_seqnum |= QM_EQCR_SEQNUM_NLIS;
2240 orp_seqnum &= ~QM_EQCR_SEQNUM_NLIS;
2241 if (flags & QMAN_ENQUEUE_FLAG_NESN)
2242 orp_seqnum |= QM_EQCR_SEQNUM_NESN;
2244 /* No need to check 4 QMAN_ENQUEUE_FLAG_HOLE */
2245 orp_seqnum &= ~QM_EQCR_SEQNUM_NESN;
2247 eq->seqnum = cpu_to_be16(orp_seqnum);
2248 eq->orp = cpu_to_be32(orp->fqid);
2249 /* Note: QM_EQCR_VERB_INTERRUPT == QMAN_ENQUEUE_FLAG_WAIT_SYNC */
2250 qm_eqcr_pvb_commit(&p->p, QM_EQCR_VERB_ORP |
2251 ((flags & (QMAN_ENQUEUE_FLAG_HOLE | QMAN_ENQUEUE_FLAG_NESN)) ?
2252 0 : QM_EQCR_VERB_CMD_ENQUEUE) |
2253 (flags & (QM_EQCR_VERB_COLOUR_MASK | QM_EQCR_VERB_INTERRUPT)));
2258 int qman_modify_cgr(struct qman_cgr *cgr, u32 flags,
2259 struct qm_mcc_initcgr *opts)
2261 struct qm_mc_command *mcc;
2262 struct qm_mc_result *mcr;
2263 struct qman_portal *p = get_affine_portal();
2266 u8 verb = QM_MCC_VERB_MODIFYCGR;
2268 mcc = qm_mc_start(&p->p);
2270 mcc->initcgr = *opts;
2271 mcc->initcgr.we_mask = cpu_to_be16(mcc->initcgr.we_mask);
2272 mcc->initcgr.cgr.wr_parm_g.word =
2273 cpu_to_be32(mcc->initcgr.cgr.wr_parm_g.word);
2274 mcc->initcgr.cgr.wr_parm_y.word =
2275 cpu_to_be32(mcc->initcgr.cgr.wr_parm_y.word);
2276 mcc->initcgr.cgr.wr_parm_r.word =
2277 cpu_to_be32(mcc->initcgr.cgr.wr_parm_r.word);
2278 mcc->initcgr.cgr.cscn_targ = cpu_to_be32(mcc->initcgr.cgr.cscn_targ);
2279 mcc->initcgr.cgr.__cs_thres = cpu_to_be16(mcc->initcgr.cgr.__cs_thres);
2281 mcc->initcgr.cgid = cgr->cgrid;
2282 if (flags & QMAN_CGR_FLAG_USE_INIT)
2283 verb = QM_MCC_VERB_INITCGR;
2284 qm_mc_commit(&p->p, verb);
2285 while (!(mcr = qm_mc_result(&p->p)))
2288 DPAA_ASSERT((mcr->verb & QM_MCR_VERB_MASK) == verb);
2290 return (res == QM_MCR_RESULT_OK) ? 0 : -EIO;
2293 #define TARG_MASK(n) (0x80000000 >> (n->config->channel - \
2294 QM_CHANNEL_SWPORTAL0))
2295 #define TARG_DCP_MASK(n) (0x80000000 >> (10 + n))
2296 #define PORTAL_IDX(n) (n->config->channel - QM_CHANNEL_SWPORTAL0)
2298 int qman_create_cgr(struct qman_cgr *cgr, u32 flags,
2299 struct qm_mcc_initcgr *opts)
2301 struct qm_mcr_querycgr cgr_state;
2302 struct qm_mcc_initcgr local_opts;
2304 struct qman_portal *p;
2306 /* We have to check that the provided CGRID is within the limits of the
2307 * data-structures, for obvious reasons. However we'll let h/w take
2308 * care of determining whether it's within the limits of what exists on
2311 if (cgr->cgrid >= __CGR_NUM)
2314 p = get_affine_portal();
2316 memset(&local_opts, 0, sizeof(struct qm_mcc_initcgr));
2317 cgr->chan = p->config->channel;
2318 spin_lock(&p->cgr_lock);
2320 /* if no opts specified, just add it to the list */
2324 ret = qman_query_cgr(cgr, &cgr_state);
2329 if ((qman_ip_rev & 0xFF00) >= QMAN_REV30)
2330 local_opts.cgr.cscn_targ_upd_ctrl =
2331 QM_CGR_TARG_UDP_CTRL_WRITE_BIT | PORTAL_IDX(p);
2333 /* Overwrite TARG */
2334 local_opts.cgr.cscn_targ = cgr_state.cgr.cscn_targ |
2336 local_opts.we_mask |= QM_CGR_WE_CSCN_TARG;
2338 /* send init if flags indicate so */
2339 if (opts && (flags & QMAN_CGR_FLAG_USE_INIT))
2340 ret = qman_modify_cgr(cgr, QMAN_CGR_FLAG_USE_INIT, &local_opts);
2342 ret = qman_modify_cgr(cgr, 0, &local_opts);
2346 list_add(&cgr->node, &p->cgr_cbs);
2348 /* Determine if newly added object requires its callback to be called */
2349 ret = qman_query_cgr(cgr, &cgr_state);
2351 /* we can't go back, so proceed and return success, but screen
2352 * and wail to the log file.
2354 pr_crit("CGR HW state partially modified\n");
2358 if (cgr->cb && cgr_state.cgr.cscn_en && qman_cgrs_get(&p->cgrs[1],
2362 spin_unlock(&p->cgr_lock);
2366 int qman_create_cgr_to_dcp(struct qman_cgr *cgr, u32 flags, u16 dcp_portal,
2367 struct qm_mcc_initcgr *opts)
2369 struct qm_mcc_initcgr local_opts;
2370 struct qm_mcr_querycgr cgr_state;
2373 if ((qman_ip_rev & 0xFF00) < QMAN_REV30) {
2374 pr_warn("QMan version doesn't support CSCN => DCP portal\n");
2377 /* We have to check that the provided CGRID is within the limits of the
2378 * data-structures, for obvious reasons. However we'll let h/w take
2379 * care of determining whether it's within the limits of what exists on
2382 if (cgr->cgrid >= __CGR_NUM)
2385 ret = qman_query_cgr(cgr, &cgr_state);
2389 memset(&local_opts, 0, sizeof(struct qm_mcc_initcgr));
2393 if ((qman_ip_rev & 0xFF00) >= QMAN_REV30)
2394 local_opts.cgr.cscn_targ_upd_ctrl =
2395 QM_CGR_TARG_UDP_CTRL_WRITE_BIT |
2396 QM_CGR_TARG_UDP_CTRL_DCP | dcp_portal;
2398 local_opts.cgr.cscn_targ = cgr_state.cgr.cscn_targ |
2399 TARG_DCP_MASK(dcp_portal);
2400 local_opts.we_mask |= QM_CGR_WE_CSCN_TARG;
2402 /* send init if flags indicate so */
2403 if (opts && (flags & QMAN_CGR_FLAG_USE_INIT))
2404 ret = qman_modify_cgr(cgr, QMAN_CGR_FLAG_USE_INIT,
2407 ret = qman_modify_cgr(cgr, 0, &local_opts);
2412 int qman_delete_cgr(struct qman_cgr *cgr)
2414 struct qm_mcr_querycgr cgr_state;
2415 struct qm_mcc_initcgr local_opts;
2418 struct qman_portal *p = get_affine_portal();
2420 if (cgr->chan != p->config->channel) {
2421 pr_crit("Attempting to delete cgr from different portal than"
2422 " it was create: create 0x%x, delete 0x%x\n",
2423 cgr->chan, p->config->channel);
2427 memset(&local_opts, 0, sizeof(struct qm_mcc_initcgr));
2428 spin_lock(&p->cgr_lock);
2429 list_del(&cgr->node);
2431 * If there are no other CGR objects for this CGRID in the list,
2432 * update CSCN_TARG accordingly
2434 list_for_each_entry(i, &p->cgr_cbs, node)
2435 if ((i->cgrid == cgr->cgrid) && i->cb)
2437 ret = qman_query_cgr(cgr, &cgr_state);
2439 /* add back to the list */
2440 list_add(&cgr->node, &p->cgr_cbs);
2443 /* Overwrite TARG */
2444 local_opts.we_mask = QM_CGR_WE_CSCN_TARG;
2445 if ((qman_ip_rev & 0xFF00) >= QMAN_REV30)
2446 local_opts.cgr.cscn_targ_upd_ctrl = PORTAL_IDX(p);
2448 local_opts.cgr.cscn_targ = cgr_state.cgr.cscn_targ &
2450 ret = qman_modify_cgr(cgr, 0, &local_opts);
2452 /* add back to the list */
2453 list_add(&cgr->node, &p->cgr_cbs);
2455 spin_unlock(&p->cgr_lock);
2460 int qman_shutdown_fq(u32 fqid)
2462 struct qman_portal *p;
2463 struct qm_portal *low_p;
2464 struct qm_mc_command *mcc;
2465 struct qm_mc_result *mcr;
2467 int orl_empty, fq_empty, drain = 0;
2472 p = get_affine_portal();
2475 /* Determine the state of the FQID */
2476 mcc = qm_mc_start(low_p);
2477 mcc->queryfq_np.fqid = cpu_to_be32(fqid);
2478 qm_mc_commit(low_p, QM_MCC_VERB_QUERYFQ_NP);
2479 while (!(mcr = qm_mc_result(low_p)))
2481 DPAA_ASSERT((mcr->verb & QM_MCR_VERB_MASK) == QM_MCR_VERB_QUERYFQ_NP);
2482 state = mcr->queryfq_np.state & QM_MCR_NP_STATE_MASK;
2483 if (state == QM_MCR_NP_STATE_OOS)
2484 return 0; /* Already OOS, no need to do anymore checks */
2486 /* Query which channel the FQ is using */
2487 mcc = qm_mc_start(low_p);
2488 mcc->queryfq.fqid = cpu_to_be32(fqid);
2489 qm_mc_commit(low_p, QM_MCC_VERB_QUERYFQ);
2490 while (!(mcr = qm_mc_result(low_p)))
2492 DPAA_ASSERT((mcr->verb & QM_MCR_VERB_MASK) == QM_MCR_VERB_QUERYFQ);
2494 /* Need to store these since the MCR gets reused */
2495 dest_wq = be16_to_cpu(mcr->queryfq.fqd.dest_wq);
2496 channel = dest_wq & 0x7;
2500 case QM_MCR_NP_STATE_TEN_SCHED:
2501 case QM_MCR_NP_STATE_TRU_SCHED:
2502 case QM_MCR_NP_STATE_ACTIVE:
2503 case QM_MCR_NP_STATE_PARKED:
2505 mcc = qm_mc_start(low_p);
2506 mcc->alterfq.fqid = cpu_to_be32(fqid);
2507 qm_mc_commit(low_p, QM_MCC_VERB_ALTER_RETIRE);
2508 while (!(mcr = qm_mc_result(low_p)))
2510 DPAA_ASSERT((mcr->verb & QM_MCR_VERB_MASK) ==
2511 QM_MCR_VERB_ALTER_RETIRE);
2512 result = mcr->result; /* Make a copy as we reuse MCR below */
2514 if (result == QM_MCR_RESULT_PENDING) {
2515 /* Need to wait for the FQRN in the message ring, which
2516 * will only occur once the FQ has been drained. In
2517 * order for the FQ to drain the portal needs to be set
2518 * to dequeue from the channel the FQ is scheduled on
2520 const struct qm_mr_entry *msg;
2521 const struct qm_dqrr_entry *dqrr = NULL;
2523 __maybe_unused u16 dequeue_wq = 0;
2525 /* Flag that we need to drain FQ */
2528 if (channel >= qm_channel_pool1 &&
2529 channel < (u16)(qm_channel_pool1 + 15)) {
2530 /* Pool channel, enable the bit in the portal */
2531 dequeue_wq = (channel -
2532 qm_channel_pool1 + 1) << 4 | wq;
2533 } else if (channel < qm_channel_pool1) {
2534 /* Dedicated channel */
2537 pr_info("Cannot recover FQ 0x%x,"
2538 " it is scheduled on channel 0x%x",
2542 /* Set the sdqcr to drain this channel */
2543 if (channel < qm_channel_pool1)
2544 qm_dqrr_sdqcr_set(low_p,
2545 QM_SDQCR_TYPE_ACTIVE |
2546 QM_SDQCR_CHANNELS_DEDICATED);
2548 qm_dqrr_sdqcr_set(low_p,
2549 QM_SDQCR_TYPE_ACTIVE |
2550 QM_SDQCR_CHANNELS_POOL_CONV
2552 while (!found_fqrn) {
2553 /* Keep draining DQRR while checking the MR*/
2554 qm_dqrr_pvb_update(low_p);
2555 dqrr = qm_dqrr_current(low_p);
2557 qm_dqrr_cdc_consume_1ptr(
2559 qm_dqrr_pvb_update(low_p);
2560 qm_dqrr_next(low_p);
2561 dqrr = qm_dqrr_current(low_p);
2563 /* Process message ring too */
2564 qm_mr_pvb_update(low_p);
2565 msg = qm_mr_current(low_p);
2568 QM_MR_VERB_TYPE_MASK)
2572 qm_mr_cci_consume_to_current(low_p);
2573 qm_mr_pvb_update(low_p);
2574 msg = qm_mr_current(low_p);
2579 if (result != QM_MCR_RESULT_OK &&
2580 result != QM_MCR_RESULT_PENDING) {
2582 pr_err("qman_retire_fq failed on FQ 0x%x,"
2583 " result=0x%x\n", fqid, result);
2586 if (!(mcr->alterfq.fqs & QM_MCR_FQS_ORLPRESENT)) {
2587 /* ORL had no entries, no need to wait until the
2592 /* Retirement succeeded, check to see if FQ needs
2595 if (drain || mcr->alterfq.fqs & QM_MCR_FQS_NOTEMPTY) {
2596 /* FQ is Not Empty, drain using volatile DQ commands */
2599 const struct qm_dqrr_entry *dqrr = NULL;
2600 u32 vdqcr = fqid | QM_VDQCR_NUMFRAMES_SET(3);
2602 qm_dqrr_vdqcr_set(low_p, vdqcr);
2604 /* Wait for a dequeue to occur */
2605 while (dqrr == NULL) {
2606 qm_dqrr_pvb_update(low_p);
2607 dqrr = qm_dqrr_current(low_p);
2611 /* Process the dequeues, making sure to
2612 * empty the ring completely.
2615 if (dqrr->fqid == fqid &&
2616 dqrr->stat & QM_DQRR_STAT_FQ_EMPTY)
2618 qm_dqrr_cdc_consume_1ptr(low_p,
2620 qm_dqrr_pvb_update(low_p);
2621 qm_dqrr_next(low_p);
2622 dqrr = qm_dqrr_current(low_p);
2624 } while (fq_empty == 0);
2626 qm_dqrr_sdqcr_set(low_p, 0);
2628 /* Wait for the ORL to have been completely drained */
2629 while (orl_empty == 0) {
2630 const struct qm_mr_entry *msg;
2632 qm_mr_pvb_update(low_p);
2633 msg = qm_mr_current(low_p);
2635 if ((msg->verb & QM_MR_VERB_TYPE_MASK) ==
2639 qm_mr_cci_consume_to_current(low_p);
2640 qm_mr_pvb_update(low_p);
2641 msg = qm_mr_current(low_p);
2645 mcc = qm_mc_start(low_p);
2646 mcc->alterfq.fqid = cpu_to_be32(fqid);
2647 qm_mc_commit(low_p, QM_MCC_VERB_ALTER_OOS);
2648 while (!(mcr = qm_mc_result(low_p)))
2650 DPAA_ASSERT((mcr->verb & QM_MCR_VERB_MASK) ==
2651 QM_MCR_VERB_ALTER_OOS);
2652 if (mcr->result != QM_MCR_RESULT_OK) {
2654 "OOS after drain Failed on FQID 0x%x, result 0x%x\n",
2660 case QM_MCR_NP_STATE_RETIRED:
2661 /* Send OOS Command */
2662 mcc = qm_mc_start(low_p);
2663 mcc->alterfq.fqid = cpu_to_be32(fqid);
2664 qm_mc_commit(low_p, QM_MCC_VERB_ALTER_OOS);
2665 while (!(mcr = qm_mc_result(low_p)))
2667 DPAA_ASSERT((mcr->verb & QM_MCR_VERB_MASK) ==
2668 QM_MCR_VERB_ALTER_OOS);
2670 pr_err("OOS Failed on FQID 0x%x\n", fqid);