drivers/bus/dpaa/base/qbman/qman.c

   1 /* SPDX-License-Identifier: (BSD-3-Clause OR GPL-2.0)
   2  *
   3  * Copyright 2008-2016 Freescale Semiconductor Inc.
   4  * Copyright 2017 NXP
   5  *
   6  */
   7
   8 #include "qman.h"
   9 #include <rte_branch_prediction.h>
  10 #include <rte_dpaa_bus.h>
  11 #include <rte_eventdev.h>
  12 #include <rte_byteorder.h>
  13
  14 /* Compilation constants */
  15 #define DQRR_MAXFILL    15
  16 #define EQCR_ITHRESH    4       /* if EQCR congests, interrupt threshold */
  17 #define IRQNAME         "QMan portal %d"
  18 #define MAX_IRQNAME     16      /* big enough for "QMan portal %d" */
  19 /* maximum number of DQRR entries to process in qman_poll() */
  20 #define FSL_QMAN_POLL_LIMIT 8
  21
  22 /* Lock/unlock frame queues, subject to the "LOCKED" flag. This is about
  23  * inter-processor locking only. Note, FQLOCK() is always called either under a
  24  * local_irq_save() or from interrupt context - hence there's no need for irq
  25  * protection (and indeed, attempting to nest irq-protection doesn't work, as
  26  * the "irq en/disable" machinery isn't recursive...).
  27  */
  28 #define FQLOCK(fq) \
  29         do { \
  30                 struct qman_fq *__fq478 = (fq); \
  31                 if (fq_isset(__fq478, QMAN_FQ_FLAG_LOCKED)) \
  32                         spin_lock(&__fq478->fqlock); \
  33         } while (0)
  34 #define FQUNLOCK(fq) \
  35         do { \
  36                 struct qman_fq *__fq478 = (fq); \
  37                 if (fq_isset(__fq478, QMAN_FQ_FLAG_LOCKED)) \
  38                         spin_unlock(&__fq478->fqlock); \
  39         } while (0)
  40
  41 static inline void fq_set(struct qman_fq *fq, u32 mask)
  42 {
  43         dpaa_set_bits(mask, &fq->flags);
  44 }
  45
  46 static inline void fq_clear(struct qman_fq *fq, u32 mask)
  47 {
  48         dpaa_clear_bits(mask, &fq->flags);
  49 }
  50
  51 static inline int fq_isset(struct qman_fq *fq, u32 mask)
  52 {
  53         return fq->flags & mask;
  54 }
  55
  56 static inline int fq_isclear(struct qman_fq *fq, u32 mask)
  57 {
  58         return !(fq->flags & mask);
  59 }
  60
  61 struct qman_portal {
  62         struct qm_portal p;
  63         /* PORTAL_BITS_*** - dynamic, strictly internal */
  64         unsigned long bits;
  65         /* interrupt sources processed by portal_isr(), configurable */
  66         unsigned long irq_sources;
  67         u32 use_eqcr_ci_stashing;
  68         u32 slowpoll;   /* only used when interrupts are off */
  69         /* only 1 volatile dequeue at a time */
  70         struct qman_fq *vdqcr_owned;
  71         u32 sdqcr;
  72         int dqrr_disable_ref;
  73         /* A portal-specific handler for DCP ERNs. If this is NULL, the global
  74          * handler is called instead.
  75          */
  76         qman_cb_dc_ern cb_dc_ern;
  77         /* When the cpu-affine portal is activated, this is non-NULL */
  78         const struct qm_portal_config *config;
  79         struct dpa_rbtree retire_table;
  80         char irqname[MAX_IRQNAME];
  81         /* 2-element array. cgrs[0] is mask, cgrs[1] is snapshot. */
  82         struct qman_cgrs *cgrs;
  83         /* linked-list of CSCN handlers. */
  84         struct list_head cgr_cbs;
  85         /* list lock */
  86         spinlock_t cgr_lock;
  87         /* track if memory was allocated by the driver */
  88 #if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__
  89         /* Keep a shadow copy of the DQRR on LE systems as the SW needs to
  90          * do byte swaps of DQRR read only memory.  First entry must be aligned
  91          * to 2 ** 10 to ensure DQRR index calculations based shadow copy
  92          * address (6 bits for address shift + 4 bits for the DQRR size).
  93          */
  94         struct qm_dqrr_entry shadow_dqrr[QM_DQRR_SIZE]
  95                     __attribute__((aligned(1024)));
  96 #endif
  97 };
  98
  99 /* Global handler for DCP ERNs. Used when the portal receiving the message does
 100  * not have a portal-specific handler.
 101  */
 102 static qman_cb_dc_ern cb_dc_ern;
 103
 104 static cpumask_t affine_mask;
 105 static DEFINE_SPINLOCK(affine_mask_lock);
 106 static u16 affine_channels[NR_CPUS];
 107 static RTE_DEFINE_PER_LCORE(struct qman_portal, qman_affine_portal);
 108
 109 static inline struct qman_portal *get_affine_portal(void)
 110 {
 111         return &RTE_PER_LCORE(qman_affine_portal);
 112 }
 113
 114 /* This gives a FQID->FQ lookup to cover the fact that we can't directly demux
 115  * retirement notifications (the fact they are sometimes h/w-consumed means that
 116  * contextB isn't always a s/w demux - and as we can't know which case it is
 117  * when looking at the notification, we have to use the slow lookup for all of
 118  * them). NB, it's possible to have multiple FQ objects refer to the same FQID
 119  * (though at most one of them should be the consumer), so this table isn't for
 120  * all FQs - FQs are added when retirement commands are issued, and removed when
 121  * they complete, which also massively reduces the size of this table.
 122  */
 123 IMPLEMENT_DPAA_RBTREE(fqtree, struct qman_fq, node, fqid);
 124 /*
 125  * This is what everything can wait on, even if it migrates to a different cpu
 126  * to the one whose affine portal it is waiting on.
 127  */
 128 static DECLARE_WAIT_QUEUE_HEAD(affine_queue);
 129
 130 static inline int table_push_fq(struct qman_portal *p, struct qman_fq *fq)
 131 {
 132         int ret = fqtree_push(&p->retire_table, fq);
 133
 134         if (ret)
 135                 pr_err("ERROR: double FQ-retirement %d\n", fq->fqid);
 136         return ret;
 137 }
 138
 139 static inline void table_del_fq(struct qman_portal *p, struct qman_fq *fq)
 140 {
 141         fqtree_del(&p->retire_table, fq);
 142 }
 143
 144 static inline struct qman_fq *table_find_fq(struct qman_portal *p, u32 fqid)
 145 {
 146         return fqtree_find(&p->retire_table, fqid);
 147 }
 148
 149 #ifdef CONFIG_FSL_QMAN_FQ_LOOKUP
 150 static void **qman_fq_lookup_table;
 151 static size_t qman_fq_lookup_table_size;
 152
 153 int qman_setup_fq_lookup_table(size_t num_entries)
 154 {
 155         num_entries++;
 156         /* Allocate 1 more entry since the first entry is not used */
 157         qman_fq_lookup_table = vmalloc((num_entries * sizeof(void *)));
 158         if (!qman_fq_lookup_table) {
 159                 pr_err("QMan: Could not allocate fq lookup table\n");
 160                 return -ENOMEM;
 161         }
 162         memset(qman_fq_lookup_table, 0, num_entries * sizeof(void *));
 163         qman_fq_lookup_table_size = num_entries;
 164         pr_debug("QMan: Allocated lookup table at %p, entry count %lu\n",
 165                 qman_fq_lookup_table,
 166                         (unsigned long)qman_fq_lookup_table_size);
 167         return 0;
 168 }
 169
 170 /* global structure that maintains fq object mapping */
 171 static DEFINE_SPINLOCK(fq_hash_table_lock);
 172
 173 static int find_empty_fq_table_entry(u32 *entry, struct qman_fq *fq)
 174 {
 175         u32 i;
 176
 177         spin_lock(&fq_hash_table_lock);
 178         /* Can't use index zero because this has special meaning
 179          * in context_b field.
 180          */
 181         for (i = 1; i < qman_fq_lookup_table_size; i++) {
 182                 if (qman_fq_lookup_table[i] == NULL) {
 183                         *entry = i;
 184                         qman_fq_lookup_table[i] = fq;
 185                         spin_unlock(&fq_hash_table_lock);
 186                         return 0;
 187                 }
 188         }
 189         spin_unlock(&fq_hash_table_lock);
 190         return -ENOMEM;
 191 }
 192
 193 static void clear_fq_table_entry(u32 entry)
 194 {
 195         spin_lock(&fq_hash_table_lock);
 196         DPAA_BUG_ON(entry >= qman_fq_lookup_table_size);
 197         qman_fq_lookup_table[entry] = NULL;
 198         spin_unlock(&fq_hash_table_lock);
 199 }
 200
 201 static inline struct qman_fq *get_fq_table_entry(u32 entry)
 202 {
 203         DPAA_BUG_ON(entry >= qman_fq_lookup_table_size);
 204         return qman_fq_lookup_table[entry];
 205 }
 206 #endif
 207
 208 static inline void cpu_to_hw_fqd(struct qm_fqd *fqd)
 209 {
 210         /* Byteswap the FQD to HW format */
 211         fqd->fq_ctrl = cpu_to_be16(fqd->fq_ctrl);
 212         fqd->dest_wq = cpu_to_be16(fqd->dest_wq);
 213         fqd->ics_cred = cpu_to_be16(fqd->ics_cred);
 214         fqd->context_b = cpu_to_be32(fqd->context_b);
 215         fqd->context_a.opaque = cpu_to_be64(fqd->context_a.opaque);
 216         fqd->opaque_td = cpu_to_be16(fqd->opaque_td);
 217 }
 218
 219 static inline void hw_fqd_to_cpu(struct qm_fqd *fqd)
 220 {
 221         /* Byteswap the FQD to CPU format */
 222         fqd->fq_ctrl = be16_to_cpu(fqd->fq_ctrl);
 223         fqd->dest_wq = be16_to_cpu(fqd->dest_wq);
 224         fqd->ics_cred = be16_to_cpu(fqd->ics_cred);
 225         fqd->context_b = be32_to_cpu(fqd->context_b);
 226         fqd->context_a.opaque = be64_to_cpu(fqd->context_a.opaque);
 227 }
 228
 229 static inline void cpu_to_hw_fd(struct qm_fd *fd)
 230 {
 231         fd->addr = cpu_to_be40(fd->addr);
 232         fd->status = cpu_to_be32(fd->status);
 233         fd->opaque = cpu_to_be32(fd->opaque);
 234 }
 235
 236 static inline void hw_fd_to_cpu(struct qm_fd *fd)
 237 {
 238         fd->addr = be40_to_cpu(fd->addr);
 239         fd->status = be32_to_cpu(fd->status);
 240         fd->opaque = be32_to_cpu(fd->opaque);
 241 }
 242
 243 /* In the case that slow- and fast-path handling are both done by qman_poll()
 244  * (ie. because there is no interrupt handling), we ought to balance how often
 245  * we do the fast-path poll versus the slow-path poll. We'll use two decrementer
 246  * sources, so we call the fast poll 'n' times before calling the slow poll
 247  * once. The idle decrementer constant is used when the last slow-poll detected
 248  * no work to do, and the busy decrementer constant when the last slow-poll had
 249  * work to do.
 250  */
 251 #define SLOW_POLL_IDLE   1000
 252 #define SLOW_POLL_BUSY   10
 253 static u32 __poll_portal_slow(struct qman_portal *p, u32 is);
 254 static inline unsigned int __poll_portal_fast(struct qman_portal *p,
 255                                               unsigned int poll_limit);
 256
 257 /* Portal interrupt handler */
 258 static irqreturn_t portal_isr(__always_unused int irq, void *ptr)
 259 {
 260         struct qman_portal *p = ptr;
 261         /*
 262          * The CSCI/CCSCI source is cleared inside __poll_portal_slow(), because
 263          * it could race against a Query Congestion State command also given
 264          * as part of the handling of this interrupt source. We mustn't
 265          * clear it a second time in this top-level function.
 266          */
 267         u32 clear = QM_DQAVAIL_MASK | (p->irq_sources &
 268                 ~(QM_PIRQ_CSCI | QM_PIRQ_CCSCI));
 269         u32 is = qm_isr_status_read(&p->p) & p->irq_sources;
 270         /* DQRR-handling if it's interrupt-driven */
 271         if (is & QM_PIRQ_DQRI)
 272                 __poll_portal_fast(p, FSL_QMAN_POLL_LIMIT);
 273         /* Handling of anything else that's interrupt-driven */
 274         clear |= __poll_portal_slow(p, is);
 275         qm_isr_status_clear(&p->p, clear);
 276         return IRQ_HANDLED;
 277 }
 278
 279 /* This inner version is used privately by qman_create_affine_portal(), as well
 280  * as by the exported qman_stop_dequeues().
 281  */
 282 static inline void qman_stop_dequeues_ex(struct qman_portal *p)
 283 {
 284         if (!(p->dqrr_disable_ref++))
 285                 qm_dqrr_set_maxfill(&p->p, 0);
 286 }
 287
 288 static int drain_mr_fqrni(struct qm_portal *p)
 289 {
 290         const struct qm_mr_entry *msg;
 291 loop:
 292         msg = qm_mr_current(p);
 293         if (!msg) {
 294                 /*
 295                  * if MR was full and h/w had other FQRNI entries to produce, we
 296                  * need to allow it time to produce those entries once the
 297                  * existing entries are consumed. A worst-case situation
 298                  * (fully-loaded system) means h/w sequencers may have to do 3-4
 299                  * other things before servicing the portal's MR pump, each of
 300                  * which (if slow) may take ~50 qman cycles (which is ~200
 301                  * processor cycles). So rounding up and then multiplying this
 302                  * worst-case estimate by a factor of 10, just to be
 303                  * ultra-paranoid, goes as high as 10,000 cycles. NB, we consume
 304                  * one entry at a time, so h/w has an opportunity to produce new
 305                  * entries well before the ring has been fully consumed, so
 306                  * we're being *really* paranoid here.
 307                  */
 308                 u64 now, then = mfatb();
 309
 310                 do {
 311                         now = mfatb();
 312                 } while ((then + 10000) > now);
 313                 msg = qm_mr_current(p);
 314                 if (!msg)
 315                         return 0;
 316         }
 317         if ((msg->ern.verb & QM_MR_VERB_TYPE_MASK) != QM_MR_VERB_FQRNI) {
 318                 /* We aren't draining anything but FQRNIs */
 319                 pr_err("Found verb 0x%x in MR\n", msg->ern.verb);
 320                 return -1;
 321         }
 322         qm_mr_next(p);
 323         qm_mr_cci_consume(p, 1);
 324         goto loop;
 325 }
 326
 327 static inline int qm_eqcr_init(struct qm_portal *portal,
 328                                enum qm_eqcr_pmode pmode,
 329                                unsigned int eq_stash_thresh,
 330                                int eq_stash_prio)
 331 {
 332         /* This use of 'register', as well as all other occurrences, is because
 333          * it has been observed to generate much faster code with gcc than is
 334          * otherwise the case.
 335          */
 336         register struct qm_eqcr *eqcr = &portal->eqcr;
 337         u32 cfg;
 338         u8 pi;
 339
 340         eqcr->ring = portal->addr.ce + QM_CL_EQCR;
 341         eqcr->ci = qm_in(EQCR_CI_CINH) & (QM_EQCR_SIZE - 1);
 342         qm_cl_invalidate(EQCR_CI);
 343         pi = qm_in(EQCR_PI_CINH) & (QM_EQCR_SIZE - 1);
 344         eqcr->cursor = eqcr->ring + pi;
 345         eqcr->vbit = (qm_in(EQCR_PI_CINH) & QM_EQCR_SIZE) ?
 346                         QM_EQCR_VERB_VBIT : 0;
 347         eqcr->available = QM_EQCR_SIZE - 1 -
 348                         qm_cyc_diff(QM_EQCR_SIZE, eqcr->ci, pi);
 349         eqcr->ithresh = qm_in(EQCR_ITR);
 350 #ifdef RTE_LIBRTE_DPAA_HWDEBUG
 351         eqcr->busy = 0;
 352         eqcr->pmode = pmode;
 353 #endif
 354         cfg = (qm_in(CFG) & 0x00ffffff) |
 355                 (eq_stash_thresh << 28) | /* QCSP_CFG: EST */
 356                 (eq_stash_prio << 26)   | /* QCSP_CFG: EP */
 357                 ((pmode & 0x3) << 24);  /* QCSP_CFG::EPM */
 358         qm_out(CFG, cfg);
 359         return 0;
 360 }
 361
 362 static inline void qm_eqcr_finish(struct qm_portal *portal)
 363 {
 364         register struct qm_eqcr *eqcr = &portal->eqcr;
 365         u8 pi, ci;
 366         u32 cfg;
 367
 368         /*
 369          * Disable EQCI stashing because the QMan only
 370          * presents the value it previously stashed to
 371          * maintain coherency.  Setting the stash threshold
 372          * to 1 then 0 ensures that QMan has resyncronized
 373          * its internal copy so that the portal is clean
 374          * when it is reinitialized in the future
 375          */
 376         cfg = (qm_in(CFG) & 0x0fffffff) |
 377                 (1 << 28); /* QCSP_CFG: EST */
 378         qm_out(CFG, cfg);
 379         cfg &= 0x0fffffff; /* stash threshold = 0 */
 380         qm_out(CFG, cfg);
 381
 382         pi = qm_in(EQCR_PI_CINH) & (QM_EQCR_SIZE - 1);
 383         ci = qm_in(EQCR_CI_CINH) & (QM_EQCR_SIZE - 1);
 384
 385         /* Refresh EQCR CI cache value */
 386         qm_cl_invalidate(EQCR_CI);
 387         eqcr->ci = qm_cl_in(EQCR_CI) & (QM_EQCR_SIZE - 1);
 388
 389 #ifdef RTE_LIBRTE_DPAA_HWDEBUG
 390         DPAA_ASSERT(!eqcr->busy);
 391 #endif
 392         if (pi != EQCR_PTR2IDX(eqcr->cursor))
 393                 pr_crit("losing uncommitted EQCR entries\n");
 394         if (ci != eqcr->ci)
 395                 pr_crit("missing existing EQCR completions\n");
 396         if (eqcr->ci != EQCR_PTR2IDX(eqcr->cursor))
 397                 pr_crit("EQCR destroyed unquiesced\n");
 398 }
 399
 400 static inline int qm_dqrr_init(struct qm_portal *portal,
 401                         __maybe_unused const struct qm_portal_config *config,
 402                         enum qm_dqrr_dmode dmode,
 403                         __maybe_unused enum qm_dqrr_pmode pmode,
 404                         enum qm_dqrr_cmode cmode, u8 max_fill)
 405 {
 406         register struct qm_dqrr *dqrr = &portal->dqrr;
 407         u32 cfg;
 408
 409         /* Make sure the DQRR will be idle when we enable */
 410         qm_out(DQRR_SDQCR, 0);
 411         qm_out(DQRR_VDQCR, 0);
 412         qm_out(DQRR_PDQCR, 0);
 413         dqrr->ring = portal->addr.ce + QM_CL_DQRR;
 414         dqrr->pi = qm_in(DQRR_PI_CINH) & (QM_DQRR_SIZE - 1);
 415         dqrr->ci = qm_in(DQRR_CI_CINH) & (QM_DQRR_SIZE - 1);
 416         dqrr->cursor = dqrr->ring + dqrr->ci;
 417         dqrr->fill = qm_cyc_diff(QM_DQRR_SIZE, dqrr->ci, dqrr->pi);
 418         dqrr->vbit = (qm_in(DQRR_PI_CINH) & QM_DQRR_SIZE) ?
 419                         QM_DQRR_VERB_VBIT : 0;
 420         dqrr->ithresh = qm_in(DQRR_ITR);
 421 #ifdef RTE_LIBRTE_DPAA_HWDEBUG
 422         dqrr->dmode = dmode;
 423         dqrr->pmode = pmode;
 424         dqrr->cmode = cmode;
 425 #endif
 426         /* Invalidate every ring entry before beginning */
 427         for (cfg = 0; cfg < QM_DQRR_SIZE; cfg++)
 428                 dccivac(qm_cl(dqrr->ring, cfg));
 429         cfg = (qm_in(CFG) & 0xff000f00) |
 430                 ((max_fill & (QM_DQRR_SIZE - 1)) << 20) | /* DQRR_MF */
 431                 ((dmode & 1) << 18) |                   /* DP */
 432                 ((cmode & 3) << 16) |                   /* DCM */
 433                 0xa0 |                                  /* RE+SE */
 434                 (0 ? 0x40 : 0) |                        /* Ignore RP */
 435                 (0 ? 0x10 : 0);                         /* Ignore SP */
 436         qm_out(CFG, cfg);
 437         qm_dqrr_set_maxfill(portal, max_fill);
 438         return 0;
 439 }
 440
 441 static inline void qm_dqrr_finish(struct qm_portal *portal)
 442 {
 443         __maybe_unused register struct qm_dqrr *dqrr = &portal->dqrr;
 444 #ifdef RTE_LIBRTE_DPAA_HWDEBUG
 445         if ((dqrr->cmode != qm_dqrr_cdc) &&
 446             (dqrr->ci != DQRR_PTR2IDX(dqrr->cursor)))
 447                 pr_crit("Ignoring completed DQRR entries\n");
 448 #endif
 449 }
 450
 451 static inline int qm_mr_init(struct qm_portal *portal,
 452                              __maybe_unused enum qm_mr_pmode pmode,
 453                              enum qm_mr_cmode cmode)
 454 {
 455         register struct qm_mr *mr = &portal->mr;
 456         u32 cfg;
 457
 458         mr->ring = portal->addr.ce + QM_CL_MR;
 459         mr->pi = qm_in(MR_PI_CINH) & (QM_MR_SIZE - 1);
 460         mr->ci = qm_in(MR_CI_CINH) & (QM_MR_SIZE - 1);
 461         mr->cursor = mr->ring + mr->ci;
 462         mr->fill = qm_cyc_diff(QM_MR_SIZE, mr->ci, mr->pi);
 463         mr->vbit = (qm_in(MR_PI_CINH) & QM_MR_SIZE) ? QM_MR_VERB_VBIT : 0;
 464         mr->ithresh = qm_in(MR_ITR);
 465 #ifdef RTE_LIBRTE_DPAA_HWDEBUG
 466         mr->pmode = pmode;
 467         mr->cmode = cmode;
 468 #endif
 469         cfg = (qm_in(CFG) & 0xfffff0ff) |
 470                 ((cmode & 1) << 8);             /* QCSP_CFG:MM */
 471         qm_out(CFG, cfg);
 472         return 0;
 473 }
 474
 475 static inline void qm_mr_pvb_update(struct qm_portal *portal)
 476 {
 477         register struct qm_mr *mr = &portal->mr;
 478         const struct qm_mr_entry *res = qm_cl(mr->ring, mr->pi);
 479
 480 #ifdef RTE_LIBRTE_DPAA_HWDEBUG
 481         DPAA_ASSERT(mr->pmode == qm_mr_pvb);
 482 #endif
 483         /* when accessing 'verb', use __raw_readb() to ensure that compiler
 484          * inlining doesn't try to optimise out "excess reads".
 485          */
 486         if ((__raw_readb(&res->ern.verb) & QM_MR_VERB_VBIT) == mr->vbit) {
 487                 mr->pi = (mr->pi + 1) & (QM_MR_SIZE - 1);
 488                 if (!mr->pi)
 489                         mr->vbit ^= QM_MR_VERB_VBIT;
 490                 mr->fill++;
 491                 res = MR_INC(res);
 492         }
 493         dcbit_ro(res);
 494 }
 495
 496 static inline
 497 struct qman_portal *qman_create_portal(
 498                         struct qman_portal *portal,
 499                               const struct qm_portal_config *c,
 500                               const struct qman_cgrs *cgrs)
 501 {
 502         struct qm_portal *p;
 503         char buf[16];
 504         int ret;
 505         u32 isdr;
 506
 507         p = &portal->p;
 508
 509         if (dpaa_svr_family == SVR_LS1043A_FAMILY)
 510                 portal->use_eqcr_ci_stashing = 3;
 511         else
 512                 portal->use_eqcr_ci_stashing =
 513                                         ((qman_ip_rev >= QMAN_REV30) ? 1 : 0);
 514
 515         /*
 516          * prep the low-level portal struct with the mapped addresses from the
 517          * config, everything that follows depends on it and "config" is more
 518          * for (de)reference
 519          */
 520         p->addr.ce = c->addr_virt[DPAA_PORTAL_CE];
 521         p->addr.ci = c->addr_virt[DPAA_PORTAL_CI];
 522         /*
 523          * If CI-stashing is used, the current defaults use a threshold of 3,
 524          * and stash with high-than-DQRR priority.
 525          */
 526         if (qm_eqcr_init(p, qm_eqcr_pvb,
 527                          portal->use_eqcr_ci_stashing, 1)) {
 528                 pr_err("Qman EQCR initialisation failed\n");
 529                 goto fail_eqcr;
 530         }
 531         if (qm_dqrr_init(p, c, qm_dqrr_dpush, qm_dqrr_pvb,
 532                          qm_dqrr_cdc, DQRR_MAXFILL)) {
 533                 pr_err("Qman DQRR initialisation failed\n");
 534                 goto fail_dqrr;
 535         }
 536         if (qm_mr_init(p, qm_mr_pvb, qm_mr_cci)) {
 537                 pr_err("Qman MR initialisation failed\n");
 538                 goto fail_mr;
 539         }
 540         if (qm_mc_init(p)) {
 541                 pr_err("Qman MC initialisation failed\n");
 542                 goto fail_mc;
 543         }
 544
 545         /* static interrupt-gating controls */
 546         qm_dqrr_set_ithresh(p, 0);
 547         qm_mr_set_ithresh(p, 0);
 548         qm_isr_set_iperiod(p, 0);
 549         portal->cgrs = kmalloc(2 * sizeof(*cgrs), GFP_KERNEL);
 550         if (!portal->cgrs)
 551                 goto fail_cgrs;
 552         /* initial snapshot is no-depletion */
 553         qman_cgrs_init(&portal->cgrs[1]);
 554         if (cgrs)
 555                 portal->cgrs[0] = *cgrs;
 556         else
 557                 /* if the given mask is NULL, assume all CGRs can be seen */
 558                 qman_cgrs_fill(&portal->cgrs[0]);
 559         INIT_LIST_HEAD(&portal->cgr_cbs);
 560         spin_lock_init(&portal->cgr_lock);
 561         portal->bits = 0;
 562         portal->slowpoll = 0;
 563         portal->sdqcr = QM_SDQCR_SOURCE_CHANNELS | QM_SDQCR_COUNT_UPTO3 |
 564                         QM_SDQCR_DEDICATED_PRECEDENCE | QM_SDQCR_TYPE_PRIO_QOS |
 565                         QM_SDQCR_TOKEN_SET(0xab) | QM_SDQCR_CHANNELS_DEDICATED;
 566         portal->dqrr_disable_ref = 0;
 567         portal->cb_dc_ern = NULL;
 568         sprintf(buf, "qportal-%d", c->channel);
 569         dpa_rbtree_init(&portal->retire_table);
 570         isdr = 0xffffffff;
 571         qm_isr_disable_write(p, isdr);
 572         portal->irq_sources = 0;
 573         qm_isr_enable_write(p, portal->irq_sources);
 574         qm_isr_status_clear(p, 0xffffffff);
 575         snprintf(portal->irqname, MAX_IRQNAME, IRQNAME, c->cpu);
 576         if (request_irq(c->irq, portal_isr, 0, portal->irqname,
 577                         portal)) {
 578                 pr_err("request_irq() failed\n");
 579                 goto fail_irq;
 580         }
 581
 582         /* Need EQCR to be empty before continuing */
 583         isdr &= ~QM_PIRQ_EQCI;
 584         qm_isr_disable_write(p, isdr);
 585         ret = qm_eqcr_get_fill(p);
 586         if (ret) {
 587                 pr_err("Qman EQCR unclean\n");
 588                 goto fail_eqcr_empty;
 589         }
 590         isdr &= ~(QM_PIRQ_DQRI | QM_PIRQ_MRI);
 591         qm_isr_disable_write(p, isdr);
 592         if (qm_dqrr_current(p)) {
 593                 pr_err("Qman DQRR unclean\n");
 594                 qm_dqrr_cdc_consume_n(p, 0xffff);
 595         }
 596         if (qm_mr_current(p) && drain_mr_fqrni(p)) {
 597                 /* special handling, drain just in case it's a few FQRNIs */
 598                 if (drain_mr_fqrni(p))
 599                         goto fail_dqrr_mr_empty;
 600         }
 601         /* Success */
 602         portal->config = c;
 603         qm_isr_disable_write(p, 0);
 604         qm_isr_uninhibit(p);
 605         /* Write a sane SDQCR */
 606         qm_dqrr_sdqcr_set(p, portal->sdqcr);
 607         return portal;
 608 fail_dqrr_mr_empty:
 609 fail_eqcr_empty:
 610         free_irq(c->irq, portal);
 611 fail_irq:
 612         kfree(portal->cgrs);
 613         spin_lock_destroy(&portal->cgr_lock);
 614 fail_cgrs:
 615         qm_mc_finish(p);
 616 fail_mc:
 617         qm_mr_finish(p);
 618 fail_mr:
 619         qm_dqrr_finish(p);
 620 fail_dqrr:
 621         qm_eqcr_finish(p);
 622 fail_eqcr:
 623         return NULL;
 624 }
 625
 626 #define MAX_GLOBAL_PORTALS 8
 627 static struct qman_portal global_portals[MAX_GLOBAL_PORTALS];
 628 static rte_atomic16_t global_portals_used[MAX_GLOBAL_PORTALS];
 629
 630 static struct qman_portal *
 631 qman_alloc_global_portal(void)
 632 {
 633         unsigned int i;
 634
 635         for (i = 0; i < MAX_GLOBAL_PORTALS; i++) {
 636                 if (rte_atomic16_test_and_set(&global_portals_used[i]))
 637                         return &global_portals[i];
 638         }
 639         pr_err("No portal available (%x)\n", MAX_GLOBAL_PORTALS);
 640
 641         return NULL;
 642 }
 643
 644 static int
 645 qman_free_global_portal(struct qman_portal *portal)
 646 {
 647         unsigned int i;
 648
 649         for (i = 0; i < MAX_GLOBAL_PORTALS; i++) {
 650                 if (&global_portals[i] == portal) {
 651                         rte_atomic16_clear(&global_portals_used[i]);
 652                         return 0;
 653                 }
 654         }
 655         return -1;
 656 }
 657
 658 struct qman_portal *qman_create_affine_portal(const struct qm_portal_config *c,
 659                                               const struct qman_cgrs *cgrs,
 660                                               int alloc)
 661 {
 662         struct qman_portal *res;
 663         struct qman_portal *portal;
 664
 665         if (alloc)
 666                 portal = qman_alloc_global_portal();
 667         else
 668                 portal = get_affine_portal();
 669
 670         /* A criteria for calling this function (from qman_driver.c) is that
 671          * we're already affine to the cpu and won't schedule onto another cpu.
 672          */
 673
 674         res = qman_create_portal(portal, c, cgrs);
 675         if (res) {
 676                 spin_lock(&affine_mask_lock);
 677                 CPU_SET(c->cpu, &affine_mask);
 678                 affine_channels[c->cpu] =
 679                         c->channel;
 680                 spin_unlock(&affine_mask_lock);
 681         }
 682         return res;
 683 }
 684
 685 static inline
 686 void qman_destroy_portal(struct qman_portal *qm)
 687 {
 688         const struct qm_portal_config *pcfg;
 689
 690         /* Stop dequeues on the portal */
 691         qm_dqrr_sdqcr_set(&qm->p, 0);
 692
 693         /*
 694          * NB we do this to "quiesce" EQCR. If we add enqueue-completions or
 695          * something related to QM_PIRQ_EQCI, this may need fixing.
 696          * Also, due to the prefetching model used for CI updates in the enqueue
 697          * path, this update will only invalidate the CI cacheline *after*
 698          * working on it, so we need to call this twice to ensure a full update
 699          * irrespective of where the enqueue processing was at when the teardown
 700          * began.
 701          */
 702         qm_eqcr_cce_update(&qm->p);
 703         qm_eqcr_cce_update(&qm->p);
 704         pcfg = qm->config;
 705
 706         free_irq(pcfg->irq, qm);
 707
 708         kfree(qm->cgrs);
 709         qm_mc_finish(&qm->p);
 710         qm_mr_finish(&qm->p);
 711         qm_dqrr_finish(&qm->p);
 712         qm_eqcr_finish(&qm->p);
 713
 714         qm->config = NULL;
 715
 716         spin_lock_destroy(&qm->cgr_lock);
 717 }
 718
 719 const struct qm_portal_config *
 720 qman_destroy_affine_portal(struct qman_portal *qp)
 721 {
 722         /* We don't want to redirect if we're a slave, use "raw" */
 723         struct qman_portal *qm;
 724         const struct qm_portal_config *pcfg;
 725         int cpu;
 726
 727         if (qp == NULL)
 728                 qm = get_affine_portal();
 729         else
 730                 qm = qp;
 731         pcfg = qm->config;
 732         cpu = pcfg->cpu;
 733
 734         qman_destroy_portal(qm);
 735
 736         spin_lock(&affine_mask_lock);
 737         CPU_CLR(cpu, &affine_mask);
 738         spin_unlock(&affine_mask_lock);
 739
 740         qman_free_global_portal(qm);
 741
 742         return pcfg;
 743 }
 744
 745 int qman_get_portal_index(void)
 746 {
 747         struct qman_portal *p = get_affine_portal();
 748         return p->config->index;
 749 }
 750
 751 /* Inline helper to reduce nesting in __poll_portal_slow() */
 752 static inline void fq_state_change(struct qman_portal *p, struct qman_fq *fq,
 753                                    const struct qm_mr_entry *msg, u8 verb)
 754 {
 755         FQLOCK(fq);
 756         switch (verb) {
 757         case QM_MR_VERB_FQRL:
 758                 DPAA_ASSERT(fq_isset(fq, QMAN_FQ_STATE_ORL));
 759                 fq_clear(fq, QMAN_FQ_STATE_ORL);
 760                 table_del_fq(p, fq);
 761                 break;
 762         case QM_MR_VERB_FQRN:
 763                 DPAA_ASSERT((fq->state == qman_fq_state_parked) ||
 764                             (fq->state == qman_fq_state_sched));
 765                 DPAA_ASSERT(fq_isset(fq, QMAN_FQ_STATE_CHANGING));
 766                 fq_clear(fq, QMAN_FQ_STATE_CHANGING);
 767                 if (msg->fq.fqs & QM_MR_FQS_NOTEMPTY)
 768                         fq_set(fq, QMAN_FQ_STATE_NE);
 769                 if (msg->fq.fqs & QM_MR_FQS_ORLPRESENT)
 770                         fq_set(fq, QMAN_FQ_STATE_ORL);
 771                 else
 772                         table_del_fq(p, fq);
 773                 fq->state = qman_fq_state_retired;
 774                 break;
 775         case QM_MR_VERB_FQPN:
 776                 DPAA_ASSERT(fq->state == qman_fq_state_sched);
 777                 DPAA_ASSERT(fq_isclear(fq, QMAN_FQ_STATE_CHANGING));
 778                 fq->state = qman_fq_state_parked;
 779         }
 780         FQUNLOCK(fq);
 781 }
 782
 783 static u32 __poll_portal_slow(struct qman_portal *p, u32 is)
 784 {
 785         const struct qm_mr_entry *msg;
 786         struct qm_mr_entry swapped_msg;
 787
 788         if (is & QM_PIRQ_CSCI) {
 789                 struct qman_cgrs rr, c;
 790                 struct qm_mc_result *mcr;
 791                 struct qman_cgr *cgr;
 792
 793                 spin_lock(&p->cgr_lock);
 794                 /*
 795                  * The CSCI bit must be cleared _before_ issuing the
 796                  * Query Congestion State command, to ensure that a long
 797                  * CGR State Change callback cannot miss an intervening
 798                  * state change.
 799                  */
 800                 qm_isr_status_clear(&p->p, QM_PIRQ_CSCI);
 801                 qm_mc_start(&p->p);
 802                 qm_mc_commit(&p->p, QM_MCC_VERB_QUERYCONGESTION);
 803                 while (!(mcr = qm_mc_result(&p->p)))
 804                         cpu_relax();
 805                 /* mask out the ones I'm not interested in */
 806                 qman_cgrs_and(&rr, (const struct qman_cgrs *)
 807                         &mcr->querycongestion.state, &p->cgrs[0]);
 808                 /* check previous snapshot for delta, enter/exit congestion */
 809                 qman_cgrs_xor(&c, &rr, &p->cgrs[1]);
 810                 /* update snapshot */
 811                 qman_cgrs_cp(&p->cgrs[1], &rr);
 812                 /* Invoke callback */
 813                 list_for_each_entry(cgr, &p->cgr_cbs, node)
 814                         if (cgr->cb && qman_cgrs_get(&c, cgr->cgrid))
 815                                 cgr->cb(p, cgr, qman_cgrs_get(&rr, cgr->cgrid));
 816                 spin_unlock(&p->cgr_lock);
 817         }
 818
 819         if (is & QM_PIRQ_EQRI) {
 820                 qm_eqcr_cce_update(&p->p);
 821                 qm_eqcr_set_ithresh(&p->p, 0);
 822                 wake_up(&affine_queue);
 823         }
 824
 825         if (is & QM_PIRQ_MRI) {
 826                 struct qman_fq *fq;
 827                 u8 verb, num = 0;
 828 mr_loop:
 829                 qm_mr_pvb_update(&p->p);
 830                 msg = qm_mr_current(&p->p);
 831                 if (!msg)
 832                         goto mr_done;
 833                 swapped_msg = *msg;
 834                 hw_fd_to_cpu(&swapped_msg.ern.fd);
 835                 verb = msg->ern.verb & QM_MR_VERB_TYPE_MASK;
 836                 /* The message is a software ERN iff the 0x20 bit is set */
 837                 if (verb & 0x20) {
 838                         switch (verb) {
 839                         case QM_MR_VERB_FQRNI:
 840                                 /* nada, we drop FQRNIs on the floor */
 841                                 break;
 842                         case QM_MR_VERB_FQRN:
 843                         case QM_MR_VERB_FQRL:
 844                                 /* Lookup in the retirement table */
 845                                 fq = table_find_fq(p,
 846                                                    be32_to_cpu(msg->fq.fqid));
 847                                 DPAA_BUG_ON(!fq);
 848                                 fq_state_change(p, fq, &swapped_msg, verb);
 849                                 if (fq->cb.fqs)
 850                                         fq->cb.fqs(p, fq, &swapped_msg);
 851                                 break;
 852                         case QM_MR_VERB_FQPN:
 853                                 /* Parked */
 854 #ifdef CONFIG_FSL_QMAN_FQ_LOOKUP
 855                                 fq = get_fq_table_entry(
 856                                         be32_to_cpu(msg->fq.contextB));
 857 #else
 858                                 fq = (void *)(uintptr_t)
 859                                         be32_to_cpu(msg->fq.contextB);
 860 #endif
 861                                 fq_state_change(p, fq, msg, verb);
 862                                 if (fq->cb.fqs)
 863                                         fq->cb.fqs(p, fq, &swapped_msg);
 864                                 break;
 865                         case QM_MR_VERB_DC_ERN:
 866                                 /* DCP ERN */
 867                                 if (p->cb_dc_ern)
 868                                         p->cb_dc_ern(p, msg);
 869                                 else if (cb_dc_ern)
 870                                         cb_dc_ern(p, msg);
 871                                 else {
 872                                         static int warn_once;
 873
 874                                         if (!warn_once) {
 875                                                 pr_crit("Leaking DCP ERNs!\n");
 876                                                 warn_once = 1;
 877                                         }
 878                                 }
 879                                 break;
 880                         default:
 881                                 pr_crit("Invalid MR verb 0x%02x\n", verb);
 882                         }
 883                 } else {
 884                         /* Its a software ERN */
 885 #ifdef CONFIG_FSL_QMAN_FQ_LOOKUP
 886                         fq = get_fq_table_entry(be32_to_cpu(msg->ern.tag));
 887 #else
 888                         fq = (void *)(uintptr_t)be32_to_cpu(msg->ern.tag);
 889 #endif
 890                         fq->cb.ern(p, fq, &swapped_msg);
 891                 }
 892                 num++;
 893                 qm_mr_next(&p->p);
 894                 goto mr_loop;
 895 mr_done:
 896                 qm_mr_cci_consume(&p->p, num);
 897         }
 898         /*
 899          * QM_PIRQ_CSCI/CCSCI has already been cleared, as part of its specific
 900          * processing. If that interrupt source has meanwhile been re-asserted,
 901          * we mustn't clear it here (or in the top-level interrupt handler).
 902          */
 903         return is & (QM_PIRQ_EQCI | QM_PIRQ_EQRI | QM_PIRQ_MRI);
 904 }
 905
 906 /*
 907  * remove some slowish-path stuff from the "fast path" and make sure it isn't
 908  * inlined.
 909  */
 910 static noinline void clear_vdqcr(struct qman_portal *p, struct qman_fq *fq)
 911 {
 912         p->vdqcr_owned = NULL;
 913         FQLOCK(fq);
 914         fq_clear(fq, QMAN_FQ_STATE_VDQCR);
 915         FQUNLOCK(fq);
 916         wake_up(&affine_queue);
 917 }
 918
 919 /*
 920  * The only states that would conflict with other things if they ran at the
 921  * same time on the same cpu are:
 922  *
 923  *   (i) setting/clearing vdqcr_owned, and
 924  *  (ii) clearing the NE (Not Empty) flag.
 925  *
 926  * Both are safe. Because;
 927  *
 928  *   (i) this clearing can only occur after qman_set_vdq() has set the
 929  *       vdqcr_owned field (which it does before setting VDQCR), and
 930  *       qman_volatile_dequeue() blocks interrupts and preemption while this is
 931  *       done so that we can't interfere.
 932  *  (ii) the NE flag is only cleared after qman_retire_fq() has set it, and as
 933  *       with (i) that API prevents us from interfering until it's safe.
 934  *
 935  * The good thing is that qman_set_vdq() and qman_retire_fq() run far
 936  * less frequently (ie. per-FQ) than __poll_portal_fast() does, so the nett
 937  * advantage comes from this function not having to "lock" anything at all.
 938  *
 939  * Note also that the callbacks are invoked at points which are safe against the
 940  * above potential conflicts, but that this function itself is not re-entrant
 941  * (this is because the function tracks one end of each FIFO in the portal and
 942  * we do *not* want to lock that). So the consequence is that it is safe for
 943  * user callbacks to call into any QMan API.
 944  */
 945 static inline unsigned int __poll_portal_fast(struct qman_portal *p,
 946                                               unsigned int poll_limit)
 947 {
 948         const struct qm_dqrr_entry *dq;
 949         struct qman_fq *fq;
 950         enum qman_cb_dqrr_result res;
 951         unsigned int limit = 0;
 952 #if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__
 953         struct qm_dqrr_entry *shadow;
 954 #endif
 955         do {
 956                 qm_dqrr_pvb_update(&p->p);
 957                 dq = qm_dqrr_current(&p->p);
 958                 if (unlikely(!dq))
 959                         break;
 960 #if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__
 961         /* If running on an LE system the fields of the
 962          * dequeue entry must be swapper.  Because the
 963          * QMan HW will ignore writes the DQRR entry is
 964          * copied and the index stored within the copy
 965          */
 966                 shadow = &p->shadow_dqrr[DQRR_PTR2IDX(dq)];
 967                 *shadow = *dq;
 968                 dq = shadow;
 969                 shadow->fqid = be32_to_cpu(shadow->fqid);
 970                 shadow->contextB = be32_to_cpu(shadow->contextB);
 971                 shadow->seqnum = be16_to_cpu(shadow->seqnum);
 972                 hw_fd_to_cpu(&shadow->fd);
 973 #endif
 974
 975                 if (dq->stat & QM_DQRR_STAT_UNSCHEDULED) {
 976                         /*
 977                          * VDQCR: don't trust context_b as the FQ may have
 978                          * been configured for h/w consumption and we're
 979                          * draining it post-retirement.
 980                          */
 981                         fq = p->vdqcr_owned;
 982                         /*
 983                          * We only set QMAN_FQ_STATE_NE when retiring, so we
 984                          * only need to check for clearing it when doing
 985                          * volatile dequeues.  It's one less thing to check
 986                          * in the critical path (SDQCR).
 987                          */
 988                         if (dq->stat & QM_DQRR_STAT_FQ_EMPTY)
 989                                 fq_clear(fq, QMAN_FQ_STATE_NE);
 990                         /*
 991                          * This is duplicated from the SDQCR code, but we
 992                          * have stuff to do before *and* after this callback,
 993                          * and we don't want multiple if()s in the critical
 994                          * path (SDQCR).
 995                          */
 996                         res = fq->cb.dqrr(p, fq, dq);
 997                         if (res == qman_cb_dqrr_stop)
 998                                 break;
 999                         /* Check for VDQCR completion */
1000                         if (dq->stat & QM_DQRR_STAT_DQCR_EXPIRED)
1001                                 clear_vdqcr(p, fq);
1002                 } else {
1003                         /* SDQCR: context_b points to the FQ */
1004 #ifdef CONFIG_FSL_QMAN_FQ_LOOKUP
1005                         fq = get_fq_table_entry(dq->contextB);
1006 #else
1007                         fq = (void *)(uintptr_t)dq->contextB;
1008 #endif
1009                         /* Now let the callback do its stuff */
1010                         res = fq->cb.dqrr(p, fq, dq);
1011                         /*
1012                          * The callback can request that we exit without
1013                          * consuming this entry nor advancing;
1014                          */
1015                         if (res == qman_cb_dqrr_stop)
1016                                 break;
1017                 }
1018                 /* Interpret 'dq' from a driver perspective. */
1019                 /*
1020                  * Parking isn't possible unless HELDACTIVE was set. NB,
1021                  * FORCEELIGIBLE implies HELDACTIVE, so we only need to
1022                  * check for HELDACTIVE to cover both.
1023                  */
1024                 DPAA_ASSERT((dq->stat & QM_DQRR_STAT_FQ_HELDACTIVE) ||
1025                             (res != qman_cb_dqrr_park));
1026                 /* just means "skip it, I'll consume it myself later on" */
1027                 if (res != qman_cb_dqrr_defer)
1028                         qm_dqrr_cdc_consume_1ptr(&p->p, dq,
1029                                                  res == qman_cb_dqrr_park);
1030                 /* Move forward */
1031                 qm_dqrr_next(&p->p);
1032                 /*
1033                  * Entry processed and consumed, increment our counter.  The
1034                  * callback can request that we exit after consuming the
1035                  * entry, and we also exit if we reach our processing limit,
1036                  * so loop back only if neither of these conditions is met.
1037                  */
1038         } while (++limit < poll_limit && res != qman_cb_dqrr_consume_stop);
1039
1040         return limit;
1041 }
1042
1043 int qman_irqsource_add(u32 bits)
1044 {
1045         struct qman_portal *p = get_affine_portal();
1046
1047         bits = bits & QM_PIRQ_VISIBLE;
1048
1049         /* Clear any previously remaining interrupt conditions in
1050          * QCSP_ISR. This prevents raising a false interrupt when
1051          * interrupt conditions are enabled in QCSP_IER.
1052          */
1053         qm_isr_status_clear(&p->p, bits);
1054         dpaa_set_bits(bits, &p->irq_sources);
1055         qm_isr_enable_write(&p->p, p->irq_sources);
1056
1057
1058         return 0;
1059 }
1060
1061 int qman_irqsource_remove(u32 bits)
1062 {
1063         struct qman_portal *p = get_affine_portal();
1064         u32 ier;
1065
1066         /* Our interrupt handler only processes+clears status register bits that
1067          * are in p->irq_sources. As we're trimming that mask, if one of them
1068          * were to assert in the status register just before we remove it from
1069          * the enable register, there would be an interrupt-storm when we
1070          * release the IRQ lock. So we wait for the enable register update to
1071          * take effect in h/w (by reading it back) and then clear all other bits
1072          * in the status register. Ie. we clear them from ISR once it's certain
1073          * IER won't allow them to reassert.
1074          */
1075
1076         bits &= QM_PIRQ_VISIBLE;
1077         dpaa_clear_bits(bits, &p->irq_sources);
1078         qm_isr_enable_write(&p->p, p->irq_sources);
1079         ier = qm_isr_enable_read(&p->p);
1080         /* Using "~ier" (rather than "bits" or "~p->irq_sources") creates a
1081          * data-dependency, ie. to protect against re-ordering.
1082          */
1083         qm_isr_status_clear(&p->p, ~ier);
1084         return 0;
1085 }
1086
1087 u16 qman_affine_channel(int cpu)
1088 {
1089         if (cpu < 0) {
1090                 struct qman_portal *portal = get_affine_portal();
1091
1092                 cpu = portal->config->cpu;
1093         }
1094         DPAA_BUG_ON(!CPU_ISSET(cpu, &affine_mask));
1095         return affine_channels[cpu];
1096 }
1097
1098 unsigned int qman_portal_poll_rx(unsigned int poll_limit,
1099                                  void **bufs,
1100                                  struct qman_portal *p)
1101 {
1102         struct qm_portal *portal = &p->p;
1103         register struct qm_dqrr *dqrr = &portal->dqrr;
1104         struct qm_dqrr_entry *dq[QM_DQRR_SIZE], *shadow[QM_DQRR_SIZE];
1105         struct qman_fq *fq;
1106         unsigned int limit = 0, rx_number = 0;
1107         uint32_t consume = 0;
1108
1109         do {
1110                 qm_dqrr_pvb_update(&p->p);
1111                 if (!dqrr->fill)
1112                         break;
1113
1114                 dq[rx_number] = dqrr->cursor;
1115                 dqrr->cursor = DQRR_CARRYCLEAR(dqrr->cursor + 1);
1116                 /* Prefetch the next DQRR entry */
1117                 rte_prefetch0(dqrr->cursor);
1118
1119 #if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__
1120                 /* If running on an LE system the fields of the
1121                  * dequeue entry must be swapper.  Because the
1122                  * QMan HW will ignore writes the DQRR entry is
1123                  * copied and the index stored within the copy
1124                  */
1125                 shadow[rx_number] =
1126                         &p->shadow_dqrr[DQRR_PTR2IDX(dq[rx_number])];
1127                 shadow[rx_number]->fd.opaque_addr =
1128                         dq[rx_number]->fd.opaque_addr;
1129                 shadow[rx_number]->fd.addr =
1130                         be40_to_cpu(dq[rx_number]->fd.addr);
1131                 shadow[rx_number]->fd.opaque =
1132                         be32_to_cpu(dq[rx_number]->fd.opaque);
1133 #else
1134                 shadow[rx_number] = dq[rx_number];
1135 #endif
1136
1137                 /* SDQCR: context_b points to the FQ */
1138 #ifdef CONFIG_FSL_QMAN_FQ_LOOKUP
1139                 fq = qman_fq_lookup_table[be32_to_cpu(dq[rx_number]->contextB)];
1140 #else
1141                 fq = (void *)be32_to_cpu(dq[rx_number]->contextB);
1142 #endif
1143                 if (fq->cb.dqrr_prepare)
1144                         fq->cb.dqrr_prepare(shadow[rx_number],
1145                                             &bufs[rx_number]);
1146
1147                 consume |= (1 << (31 - DQRR_PTR2IDX(shadow[rx_number])));
1148                 rx_number++;
1149                 --dqrr->fill;
1150         } while (++limit < poll_limit);
1151
1152         if (rx_number)
1153                 fq->cb.dqrr_dpdk_pull_cb(&fq, shadow, bufs, rx_number);
1154
1155         /* Consume all the DQRR enries together */
1156         qm_out(DQRR_DCAP, (1 << 8) | consume);
1157
1158         return rx_number;
1159 }
1160
1161 void qman_clear_irq(void)
1162 {
1163         struct qman_portal *p = get_affine_portal();
1164         u32 clear = QM_DQAVAIL_MASK | (p->irq_sources &
1165                 ~(QM_PIRQ_CSCI | QM_PIRQ_CCSCI));
1166         qm_isr_status_clear(&p->p, clear);
1167 }
1168
1169 u32 qman_portal_dequeue(struct rte_event ev[], unsigned int poll_limit,
1170                         void **bufs)
1171 {
1172         const struct qm_dqrr_entry *dq;
1173         struct qman_fq *fq;
1174         enum qman_cb_dqrr_result res;
1175         unsigned int limit = 0;
1176         struct qman_portal *p = get_affine_portal();
1177 #if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN
1178         struct qm_dqrr_entry *shadow;
1179 #endif
1180         unsigned int rx_number = 0;
1181
1182         do {
1183                 qm_dqrr_pvb_update(&p->p);
1184                 dq = qm_dqrr_current(&p->p);
1185                 if (!dq)
1186                         break;
1187 #if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN
1188                 /*
1189                  * If running on an LE system the fields of the
1190                  * dequeue entry must be swapper.  Because the
1191                  * QMan HW will ignore writes the DQRR entry is
1192                  * copied and the index stored within the copy
1193                  */
1194                 shadow = &p->shadow_dqrr[DQRR_PTR2IDX(dq)];
1195                 *shadow = *dq;
1196                 dq = shadow;
1197                 shadow->fqid = be32_to_cpu(shadow->fqid);
1198                 shadow->contextB = be32_to_cpu(shadow->contextB);
1199                 shadow->seqnum = be16_to_cpu(shadow->seqnum);
1200                 hw_fd_to_cpu(&shadow->fd);
1201 #endif
1202
1203                /* SDQCR: context_b points to the FQ */
1204 #ifdef CONFIG_FSL_QMAN_FQ_LOOKUP
1205                 fq = get_fq_table_entry(dq->contextB);
1206 #else
1207                 fq = (void *)(uintptr_t)dq->contextB;
1208 #endif
1209                 /* Now let the callback do its stuff */
1210                 res = fq->cb.dqrr_dpdk_cb(&ev[rx_number], p, fq,
1211                                          dq, &bufs[rx_number]);
1212                 rx_number++;
1213                 /* Interpret 'dq' from a driver perspective. */
1214                 /*
1215                  * Parking isn't possible unless HELDACTIVE was set. NB,
1216                  * FORCEELIGIBLE implies HELDACTIVE, so we only need to
1217                  * check for HELDACTIVE to cover both.
1218                  */
1219                 DPAA_ASSERT((dq->stat & QM_DQRR_STAT_FQ_HELDACTIVE) ||
1220                             (res != qman_cb_dqrr_park));
1221                 if (res != qman_cb_dqrr_defer)
1222                         qm_dqrr_cdc_consume_1ptr(&p->p, dq,
1223                                                  res == qman_cb_dqrr_park);
1224                 /* Move forward */
1225                 qm_dqrr_next(&p->p);
1226                 /*
1227                  * Entry processed and consumed, increment our counter.  The
1228                  * callback can request that we exit after consuming the
1229                  * entry, and we also exit if we reach our processing limit,
1230                  * so loop back only if neither of these conditions is met.
1231                  */
1232         } while (++limit < poll_limit);
1233
1234         return limit;
1235 }
1236
1237 struct qm_dqrr_entry *qman_dequeue(struct qman_fq *fq)
1238 {
1239         struct qman_portal *p = get_affine_portal();
1240         const struct qm_dqrr_entry *dq;
1241 #if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__
1242         struct qm_dqrr_entry *shadow;
1243 #endif
1244
1245         qm_dqrr_pvb_update(&p->p);
1246         dq = qm_dqrr_current(&p->p);
1247         if (!dq)
1248                 return NULL;
1249
1250         if (!(dq->stat & QM_DQRR_STAT_FD_VALID)) {
1251                 /* Invalid DQRR - put the portal and consume the DQRR.
1252                  * Return NULL to user as no packet is seen.
1253                  */
1254                 qman_dqrr_consume(fq, (struct qm_dqrr_entry *)dq);
1255                 return NULL;
1256         }
1257
1258 #if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__
1259         shadow = &p->shadow_dqrr[DQRR_PTR2IDX(dq)];
1260         *shadow = *dq;
1261         dq = shadow;
1262         shadow->fqid = be32_to_cpu(shadow->fqid);
1263         shadow->contextB = be32_to_cpu(shadow->contextB);
1264         shadow->seqnum = be16_to_cpu(shadow->seqnum);
1265         hw_fd_to_cpu(&shadow->fd);
1266 #endif
1267
1268         if (dq->stat & QM_DQRR_STAT_FQ_EMPTY)
1269                 fq_clear(fq, QMAN_FQ_STATE_NE);
1270
1271         return (struct qm_dqrr_entry *)dq;
1272 }
1273
1274 void qman_dqrr_consume(struct qman_fq *fq,
1275                        struct qm_dqrr_entry *dq)
1276 {
1277         struct qman_portal *p = get_affine_portal();
1278
1279         if (dq->stat & QM_DQRR_STAT_DQCR_EXPIRED)
1280                 clear_vdqcr(p, fq);
1281
1282         qm_dqrr_cdc_consume_1ptr(&p->p, dq, 0);
1283         qm_dqrr_next(&p->p);
1284 }
1285
1286 int qman_poll_dqrr(unsigned int limit)
1287 {
1288         struct qman_portal *p = get_affine_portal();
1289         int ret;
1290
1291         ret = __poll_portal_fast(p, limit);
1292         return ret;
1293 }
1294
1295 void qman_poll(void)
1296 {
1297         struct qman_portal *p = get_affine_portal();
1298
1299         if ((~p->irq_sources) & QM_PIRQ_SLOW) {
1300                 if (!(p->slowpoll--)) {
1301                         u32 is = qm_isr_status_read(&p->p) & ~p->irq_sources;
1302                         u32 active = __poll_portal_slow(p, is);
1303
1304                         if (active) {
1305                                 qm_isr_status_clear(&p->p, active);
1306                                 p->slowpoll = SLOW_POLL_BUSY;
1307                         } else
1308                                 p->slowpoll = SLOW_POLL_IDLE;
1309                 }
1310         }
1311         if ((~p->irq_sources) & QM_PIRQ_DQRI)
1312                 __poll_portal_fast(p, FSL_QMAN_POLL_LIMIT);
1313 }
1314
1315 void qman_stop_dequeues(void)
1316 {
1317         struct qman_portal *p = get_affine_portal();
1318
1319         qman_stop_dequeues_ex(p);
1320 }
1321
1322 void qman_start_dequeues(void)
1323 {
1324         struct qman_portal *p = get_affine_portal();
1325
1326         DPAA_ASSERT(p->dqrr_disable_ref > 0);
1327         if (!(--p->dqrr_disable_ref))
1328                 qm_dqrr_set_maxfill(&p->p, DQRR_MAXFILL);
1329 }
1330
1331 void qman_static_dequeue_add(u32 pools, struct qman_portal *qp)
1332 {
1333         struct qman_portal *p = qp ? qp : get_affine_portal();
1334
1335         pools &= p->config->pools;
1336         p->sdqcr |= pools;
1337         qm_dqrr_sdqcr_set(&p->p, p->sdqcr);
1338 }
1339
1340 void qman_static_dequeue_del(u32 pools, struct qman_portal *qp)
1341 {
1342         struct qman_portal *p = qp ? qp : get_affine_portal();
1343
1344         pools &= p->config->pools;
1345         p->sdqcr &= ~pools;
1346         qm_dqrr_sdqcr_set(&p->p, p->sdqcr);
1347 }
1348
1349 u32 qman_static_dequeue_get(struct qman_portal *qp)
1350 {
1351         struct qman_portal *p = qp ? qp : get_affine_portal();
1352         return p->sdqcr;
1353 }
1354
1355 void qman_dca(const struct qm_dqrr_entry *dq, int park_request)
1356 {
1357         struct qman_portal *p = get_affine_portal();
1358
1359         qm_dqrr_cdc_consume_1ptr(&p->p, dq, park_request);
1360 }
1361
1362 void qman_dca_index(u8 index, int park_request)
1363 {
1364         struct qman_portal *p = get_affine_portal();
1365
1366         qm_dqrr_cdc_consume_1(&p->p, index, park_request);
1367 }
1368
1369 /* Frame queue API */
1370 static const char *mcr_result_str(u8 result)
1371 {
1372         switch (result) {
1373         case QM_MCR_RESULT_NULL:
1374                 return "QM_MCR_RESULT_NULL";
1375         case QM_MCR_RESULT_OK:
1376                 return "QM_MCR_RESULT_OK";
1377         case QM_MCR_RESULT_ERR_FQID:
1378                 return "QM_MCR_RESULT_ERR_FQID";
1379         case QM_MCR_RESULT_ERR_FQSTATE:
1380                 return "QM_MCR_RESULT_ERR_FQSTATE";
1381         case QM_MCR_RESULT_ERR_NOTEMPTY:
1382                 return "QM_MCR_RESULT_ERR_NOTEMPTY";
1383         case QM_MCR_RESULT_PENDING:
1384                 return "QM_MCR_RESULT_PENDING";
1385         case QM_MCR_RESULT_ERR_BADCOMMAND:
1386                 return "QM_MCR_RESULT_ERR_BADCOMMAND";
1387         }
1388         return "<unknown MCR result>";
1389 }
1390
1391 int qman_create_fq(u32 fqid, u32 flags, struct qman_fq *fq)
1392 {
1393         struct qm_fqd fqd;
1394         struct qm_mcr_queryfq_np np;
1395         struct qm_mc_command *mcc;
1396         struct qm_mc_result *mcr;
1397         struct qman_portal *p;
1398
1399         if (flags & QMAN_FQ_FLAG_DYNAMIC_FQID) {
1400                 int ret = qman_alloc_fqid(&fqid);
1401
1402                 if (ret)
1403                         return ret;
1404         }
1405         spin_lock_init(&fq->fqlock);
1406         fq->fqid = fqid;
1407         fq->fqid_le = cpu_to_be32(fqid);
1408         fq->flags = flags;
1409         fq->state = qman_fq_state_oos;
1410         fq->cgr_groupid = 0;
1411 #ifdef CONFIG_FSL_QMAN_FQ_LOOKUP
1412         if (unlikely(find_empty_fq_table_entry(&fq->key, fq))) {
1413                 pr_info("Find empty table entry failed\n");
1414                 return -ENOMEM;
1415         }
1416 #endif
1417         if (!(flags & QMAN_FQ_FLAG_AS_IS) || (flags & QMAN_FQ_FLAG_NO_MODIFY))
1418                 return 0;
1419         /* Everything else is AS_IS support */
1420         p = get_affine_portal();
1421         mcc = qm_mc_start(&p->p);
1422         mcc->queryfq.fqid = cpu_to_be32(fqid);
1423         qm_mc_commit(&p->p, QM_MCC_VERB_QUERYFQ);
1424         while (!(mcr = qm_mc_result(&p->p)))
1425                 cpu_relax();
1426         DPAA_ASSERT((mcr->verb & QM_MCR_VERB_MASK) == QM_MCC_VERB_QUERYFQ);
1427         if (mcr->result != QM_MCR_RESULT_OK) {
1428                 pr_err("QUERYFQ failed: %s\n", mcr_result_str(mcr->result));
1429                 goto err;
1430         }
1431         fqd = mcr->queryfq.fqd;
1432         hw_fqd_to_cpu(&fqd);
1433         mcc = qm_mc_start(&p->p);
1434         mcc->queryfq_np.fqid = cpu_to_be32(fqid);
1435         qm_mc_commit(&p->p, QM_MCC_VERB_QUERYFQ_NP);
1436         while (!(mcr = qm_mc_result(&p->p)))
1437                 cpu_relax();
1438         DPAA_ASSERT((mcr->verb & QM_MCR_VERB_MASK) == QM_MCC_VERB_QUERYFQ_NP);
1439         if (mcr->result != QM_MCR_RESULT_OK) {
1440                 pr_err("QUERYFQ_NP failed: %s\n", mcr_result_str(mcr->result));
1441                 goto err;
1442         }
1443         np = mcr->queryfq_np;
1444         /* Phew, have queryfq and queryfq_np results, stitch together
1445          * the FQ object from those.
1446          */
1447         fq->cgr_groupid = fqd.cgid;
1448         switch (np.state & QM_MCR_NP_STATE_MASK) {
1449         case QM_MCR_NP_STATE_OOS:
1450                 break;
1451         case QM_MCR_NP_STATE_RETIRED:
1452                 fq->state = qman_fq_state_retired;
1453                 if (np.frm_cnt)
1454                         fq_set(fq, QMAN_FQ_STATE_NE);
1455                 break;
1456         case QM_MCR_NP_STATE_TEN_SCHED:
1457         case QM_MCR_NP_STATE_TRU_SCHED:
1458         case QM_MCR_NP_STATE_ACTIVE:
1459                 fq->state = qman_fq_state_sched;
1460                 if (np.state & QM_MCR_NP_STATE_R)
1461                         fq_set(fq, QMAN_FQ_STATE_CHANGING);
1462                 break;
1463         case QM_MCR_NP_STATE_PARKED:
1464                 fq->state = qman_fq_state_parked;
1465                 break;
1466         default:
1467                 DPAA_ASSERT(NULL == "invalid FQ state");
1468         }
1469         if (fqd.fq_ctrl & QM_FQCTRL_CGE)
1470                 fq->state |= QMAN_FQ_STATE_CGR_EN;
1471         return 0;
1472 err:
1473         if (flags & QMAN_FQ_FLAG_DYNAMIC_FQID)
1474                 qman_release_fqid(fqid);
1475         return -EIO;
1476 }
1477
1478 void qman_destroy_fq(struct qman_fq *fq, u32 flags __maybe_unused)
1479 {
1480         /*
1481          * We don't need to lock the FQ as it is a pre-condition that the FQ be
1482          * quiesced. Instead, run some checks.
1483          */
1484         switch (fq->state) {
1485         case qman_fq_state_parked:
1486                 DPAA_ASSERT(flags & QMAN_FQ_DESTROY_PARKED);
1487                 /* Fallthrough */
1488         case qman_fq_state_oos:
1489                 if (fq_isset(fq, QMAN_FQ_FLAG_DYNAMIC_FQID))
1490                         qman_release_fqid(fq->fqid);
1491 #ifdef CONFIG_FSL_QMAN_FQ_LOOKUP
1492                 clear_fq_table_entry(fq->key);
1493 #endif
1494                 return;
1495         default:
1496                 break;
1497         }
1498         DPAA_ASSERT(NULL == "qman_free_fq() on unquiesced FQ!");
1499 }
1500
1501 u32 qman_fq_fqid(struct qman_fq *fq)
1502 {
1503         return fq->fqid;
1504 }
1505
1506 void qman_fq_state(struct qman_fq *fq, enum qman_fq_state *state, u32 *flags)
1507 {
1508         if (state)
1509                 *state = fq->state;
1510         if (flags)
1511                 *flags = fq->flags;
1512 }
1513
1514 int qman_init_fq(struct qman_fq *fq, u32 flags, struct qm_mcc_initfq *opts)
1515 {
1516         struct qm_mc_command *mcc;
1517         struct qm_mc_result *mcr;
1518         struct qman_portal *p;
1519
1520         u8 res, myverb = (flags & QMAN_INITFQ_FLAG_SCHED) ?
1521                 QM_MCC_VERB_INITFQ_SCHED : QM_MCC_VERB_INITFQ_PARKED;
1522
1523         if ((fq->state != qman_fq_state_oos) &&
1524             (fq->state != qman_fq_state_parked))
1525                 return -EINVAL;
1526 #ifdef RTE_LIBRTE_DPAA_HWDEBUG
1527         if (unlikely(fq_isset(fq, QMAN_FQ_FLAG_NO_MODIFY)))
1528                 return -EINVAL;
1529 #endif
1530         if (opts && (opts->we_mask & QM_INITFQ_WE_OAC)) {
1531                 /* And can't be set at the same time as TDTHRESH */
1532                 if (opts->we_mask & QM_INITFQ_WE_TDTHRESH)
1533                         return -EINVAL;
1534         }
1535         /* Issue an INITFQ_[PARKED|SCHED] management command */
1536         p = get_affine_portal();
1537         FQLOCK(fq);
1538         if (unlikely((fq_isset(fq, QMAN_FQ_STATE_CHANGING)) ||
1539                      ((fq->state != qman_fq_state_oos) &&
1540                                 (fq->state != qman_fq_state_parked)))) {
1541                 FQUNLOCK(fq);
1542                 return -EBUSY;
1543         }
1544         mcc = qm_mc_start(&p->p);
1545         if (opts)
1546                 mcc->initfq = *opts;
1547         mcc->initfq.fqid = cpu_to_be32(fq->fqid);
1548         mcc->initfq.count = 0;
1549         /*
1550          * If the FQ does *not* have the TO_DCPORTAL flag, context_b is set as a
1551          * demux pointer. Otherwise, the caller-provided value is allowed to
1552          * stand, don't overwrite it.
1553          */
1554         if (fq_isclear(fq, QMAN_FQ_FLAG_TO_DCPORTAL)) {
1555                 dma_addr_t phys_fq;
1556
1557                 mcc->initfq.we_mask |= QM_INITFQ_WE_CONTEXTB;
1558 #ifdef CONFIG_FSL_QMAN_FQ_LOOKUP
1559                 mcc->initfq.fqd.context_b = fq->key;
1560 #else
1561                 mcc->initfq.fqd.context_b = (u32)(uintptr_t)fq;
1562 #endif
1563                 /*
1564                  *  and the physical address - NB, if the user wasn't trying to
1565                  * set CONTEXTA, clear the stashing settings.
1566                  */
1567                 if (!(mcc->initfq.we_mask & QM_INITFQ_WE_CONTEXTA)) {
1568                         mcc->initfq.we_mask |= QM_INITFQ_WE_CONTEXTA;
1569                         memset(&mcc->initfq.fqd.context_a, 0,
1570                                sizeof(mcc->initfq.fqd.context_a));
1571                 } else {
1572                         phys_fq = rte_mem_virt2iova(fq);
1573                         qm_fqd_stashing_set64(&mcc->initfq.fqd, phys_fq);
1574                 }
1575         }
1576         if (flags & QMAN_INITFQ_FLAG_LOCAL) {
1577                 mcc->initfq.fqd.dest.channel = p->config->channel;
1578                 if (!(mcc->initfq.we_mask & QM_INITFQ_WE_DESTWQ)) {
1579                         mcc->initfq.we_mask |= QM_INITFQ_WE_DESTWQ;
1580                         mcc->initfq.fqd.dest.wq = 4;
1581                 }
1582         }
1583         mcc->initfq.we_mask = cpu_to_be16(mcc->initfq.we_mask);
1584         cpu_to_hw_fqd(&mcc->initfq.fqd);
1585         qm_mc_commit(&p->p, myverb);
1586         while (!(mcr = qm_mc_result(&p->p)))
1587                 cpu_relax();
1588         DPAA_ASSERT((mcr->verb & QM_MCR_VERB_MASK) == myverb);
1589         res = mcr->result;
1590         if (res != QM_MCR_RESULT_OK) {
1591                 FQUNLOCK(fq);
1592                 return -EIO;
1593         }
1594         if (opts) {
1595                 if (opts->we_mask & QM_INITFQ_WE_FQCTRL) {
1596                         if (opts->fqd.fq_ctrl & QM_FQCTRL_CGE)
1597                                 fq_set(fq, QMAN_FQ_STATE_CGR_EN);
1598                         else
1599                                 fq_clear(fq, QMAN_FQ_STATE_CGR_EN);
1600                 }
1601                 if (opts->we_mask & QM_INITFQ_WE_CGID)
1602                         fq->cgr_groupid = opts->fqd.cgid;
1603         }
1604         fq->state = (flags & QMAN_INITFQ_FLAG_SCHED) ?
1605                 qman_fq_state_sched : qman_fq_state_parked;
1606         FQUNLOCK(fq);
1607         return 0;
1608 }
1609
1610 int qman_schedule_fq(struct qman_fq *fq)
1611 {
1612         struct qm_mc_command *mcc;
1613         struct qm_mc_result *mcr;
1614         struct qman_portal *p;
1615
1616         int ret = 0;
1617         u8 res;
1618
1619         if (fq->state != qman_fq_state_parked)
1620                 return -EINVAL;
1621 #ifdef RTE_LIBRTE_DPAA_HWDEBUG
1622         if (unlikely(fq_isset(fq, QMAN_FQ_FLAG_NO_MODIFY)))
1623                 return -EINVAL;
1624 #endif
1625         /* Issue a ALTERFQ_SCHED management command */
1626         p = get_affine_portal();
1627
1628         FQLOCK(fq);
1629         if (unlikely((fq_isset(fq, QMAN_FQ_STATE_CHANGING)) ||
1630                      (fq->state != qman_fq_state_parked))) {
1631                 ret = -EBUSY;
1632                 goto out;
1633         }
1634         mcc = qm_mc_start(&p->p);
1635         mcc->alterfq.fqid = cpu_to_be32(fq->fqid);
1636         qm_mc_commit(&p->p, QM_MCC_VERB_ALTER_SCHED);
1637         while (!(mcr = qm_mc_result(&p->p)))
1638                 cpu_relax();
1639         DPAA_ASSERT((mcr->verb & QM_MCR_VERB_MASK) == QM_MCR_VERB_ALTER_SCHED);
1640         res = mcr->result;
1641         if (res != QM_MCR_RESULT_OK) {
1642                 ret = -EIO;
1643                 goto out;
1644         }
1645         fq->state = qman_fq_state_sched;
1646 out:
1647         FQUNLOCK(fq);
1648
1649         return ret;
1650 }
1651
1652 int qman_retire_fq(struct qman_fq *fq, u32 *flags)
1653 {
1654         struct qm_mc_command *mcc;
1655         struct qm_mc_result *mcr;
1656         struct qman_portal *p;
1657
1658         int rval;
1659         u8 res;
1660
1661         if ((fq->state != qman_fq_state_parked) &&
1662             (fq->state != qman_fq_state_sched))
1663                 return -EINVAL;
1664 #ifdef RTE_LIBRTE_DPAA_HWDEBUG
1665         if (unlikely(fq_isset(fq, QMAN_FQ_FLAG_NO_MODIFY)))
1666                 return -EINVAL;
1667 #endif
1668         p = get_affine_portal();
1669
1670         FQLOCK(fq);
1671         if (unlikely((fq_isset(fq, QMAN_FQ_STATE_CHANGING)) ||
1672                      (fq->state == qman_fq_state_retired) ||
1673                                 (fq->state == qman_fq_state_oos))) {
1674                 rval = -EBUSY;
1675                 goto out;
1676         }
1677         rval = table_push_fq(p, fq);
1678         if (rval)
1679                 goto out;
1680         mcc = qm_mc_start(&p->p);
1681         mcc->alterfq.fqid = cpu_to_be32(fq->fqid);
1682         qm_mc_commit(&p->p, QM_MCC_VERB_ALTER_RETIRE);
1683         while (!(mcr = qm_mc_result(&p->p)))
1684                 cpu_relax();
1685         DPAA_ASSERT((mcr->verb & QM_MCR_VERB_MASK) == QM_MCR_VERB_ALTER_RETIRE);
1686         res = mcr->result;
1687         /*
1688          * "Elegant" would be to treat OK/PENDING the same way; set CHANGING,
1689          * and defer the flags until FQRNI or FQRN (respectively) show up. But
1690          * "Friendly" is to process OK immediately, and not set CHANGING. We do
1691          * friendly, otherwise the caller doesn't necessarily have a fully
1692          * "retired" FQ on return even if the retirement was immediate. However
1693          * this does mean some code duplication between here and
1694          * fq_state_change().
1695          */
1696         if (likely(res == QM_MCR_RESULT_OK)) {
1697                 rval = 0;
1698                 /* Process 'fq' right away, we'll ignore FQRNI */
1699                 if (mcr->alterfq.fqs & QM_MCR_FQS_NOTEMPTY)
1700                         fq_set(fq, QMAN_FQ_STATE_NE);
1701                 if (mcr->alterfq.fqs & QM_MCR_FQS_ORLPRESENT)
1702                         fq_set(fq, QMAN_FQ_STATE_ORL);
1703                 else
1704                         table_del_fq(p, fq);
1705                 if (flags)
1706                         *flags = fq->flags;
1707                 fq->state = qman_fq_state_retired;
1708                 if (fq->cb.fqs) {
1709                         /*
1710                          * Another issue with supporting "immediate" retirement
1711                          * is that we're forced to drop FQRNIs, because by the
1712                          * time they're seen it may already be "too late" (the
1713                          * fq may have been OOS'd and free()'d already). But if
1714                          * the upper layer wants a callback whether it's
1715                          * immediate or not, we have to fake a "MR" entry to
1716                          * look like an FQRNI...
1717                          */
1718                         struct qm_mr_entry msg;
1719
1720                         msg.ern.verb = QM_MR_VERB_FQRNI;
1721                         msg.fq.fqs = mcr->alterfq.fqs;
1722                         msg.fq.fqid = fq->fqid;
1723 #ifdef CONFIG_FSL_QMAN_FQ_LOOKUP
1724                         msg.fq.contextB = fq->key;
1725 #else
1726                         msg.fq.contextB = (u32)(uintptr_t)fq;
1727 #endif
1728                         fq->cb.fqs(p, fq, &msg);
1729                 }
1730         } else if (res == QM_MCR_RESULT_PENDING) {
1731                 rval = 1;
1732                 fq_set(fq, QMAN_FQ_STATE_CHANGING);
1733         } else {
1734                 rval = -EIO;
1735                 table_del_fq(p, fq);
1736         }
1737 out:
1738         FQUNLOCK(fq);
1739         return rval;
1740 }
1741
1742 int qman_oos_fq(struct qman_fq *fq)
1743 {
1744         struct qm_mc_command *mcc;
1745         struct qm_mc_result *mcr;
1746         struct qman_portal *p;
1747
1748         int ret = 0;
1749         u8 res;
1750
1751         if (fq->state != qman_fq_state_retired)
1752                 return -EINVAL;
1753 #ifdef RTE_LIBRTE_DPAA_HWDEBUG
1754         if (unlikely(fq_isset(fq, QMAN_FQ_FLAG_NO_MODIFY)))
1755                 return -EINVAL;
1756 #endif
1757         p = get_affine_portal();
1758         FQLOCK(fq);
1759         if (unlikely((fq_isset(fq, QMAN_FQ_STATE_BLOCKOOS)) ||
1760                      (fq->state != qman_fq_state_retired))) {
1761                 ret = -EBUSY;
1762                 goto out;
1763         }
1764         mcc = qm_mc_start(&p->p);
1765         mcc->alterfq.fqid = cpu_to_be32(fq->fqid);
1766         qm_mc_commit(&p->p, QM_MCC_VERB_ALTER_OOS);
1767         while (!(mcr = qm_mc_result(&p->p)))
1768                 cpu_relax();
1769         DPAA_ASSERT((mcr->verb & QM_MCR_VERB_MASK) == QM_MCR_VERB_ALTER_OOS);
1770         res = mcr->result;
1771         if (res != QM_MCR_RESULT_OK) {
1772                 ret = -EIO;
1773                 goto out;
1774         }
1775         fq->state = qman_fq_state_oos;
1776 out:
1777         FQUNLOCK(fq);
1778         return ret;
1779 }
1780
1781 int qman_fq_flow_control(struct qman_fq *fq, int xon)
1782 {
1783         struct qm_mc_command *mcc;
1784         struct qm_mc_result *mcr;
1785         struct qman_portal *p;
1786
1787         int ret = 0;
1788         u8 res;
1789         u8 myverb;
1790
1791         if ((fq->state == qman_fq_state_oos) ||
1792             (fq->state == qman_fq_state_retired) ||
1793                 (fq->state == qman_fq_state_parked))
1794                 return -EINVAL;
1795
1796 #ifdef RTE_LIBRTE_DPAA_HWDEBUG
1797         if (unlikely(fq_isset(fq, QMAN_FQ_FLAG_NO_MODIFY)))
1798                 return -EINVAL;
1799 #endif
1800         /* Issue a ALTER_FQXON or ALTER_FQXOFF management command */
1801         p = get_affine_portal();
1802         FQLOCK(fq);
1803         if (unlikely((fq_isset(fq, QMAN_FQ_STATE_CHANGING)) ||
1804                      (fq->state == qman_fq_state_parked) ||
1805                         (fq->state == qman_fq_state_oos) ||
1806                         (fq->state == qman_fq_state_retired))) {
1807                 ret = -EBUSY;
1808                 goto out;
1809         }
1810         mcc = qm_mc_start(&p->p);
1811         mcc->alterfq.fqid = fq->fqid;
1812         mcc->alterfq.count = 0;
1813         myverb = xon ? QM_MCC_VERB_ALTER_FQXON : QM_MCC_VERB_ALTER_FQXOFF;
1814
1815         qm_mc_commit(&p->p, myverb);
1816         while (!(mcr = qm_mc_result(&p->p)))
1817                 cpu_relax();
1818         DPAA_ASSERT((mcr->verb & QM_MCR_VERB_MASK) == myverb);
1819
1820         res = mcr->result;
1821         if (res != QM_MCR_RESULT_OK) {
1822                 ret = -EIO;
1823                 goto out;
1824         }
1825 out:
1826         FQUNLOCK(fq);
1827         return ret;
1828 }
1829
1830 int qman_query_fq(struct qman_fq *fq, struct qm_fqd *fqd)
1831 {
1832         struct qm_mc_command *mcc;
1833         struct qm_mc_result *mcr;
1834         struct qman_portal *p = get_affine_portal();
1835
1836         u8 res;
1837
1838         mcc = qm_mc_start(&p->p);
1839         mcc->queryfq.fqid = cpu_to_be32(fq->fqid);
1840         qm_mc_commit(&p->p, QM_MCC_VERB_QUERYFQ);
1841         while (!(mcr = qm_mc_result(&p->p)))
1842                 cpu_relax();
1843         DPAA_ASSERT((mcr->verb & QM_MCR_VERB_MASK) == QM_MCR_VERB_QUERYFQ);
1844         res = mcr->result;
1845         if (res == QM_MCR_RESULT_OK)
1846                 *fqd = mcr->queryfq.fqd;
1847         hw_fqd_to_cpu(fqd);
1848         if (res != QM_MCR_RESULT_OK)
1849                 return -EIO;
1850         return 0;
1851 }
1852
1853 int qman_query_fq_has_pkts(struct qman_fq *fq)
1854 {
1855         struct qm_mc_command *mcc;
1856         struct qm_mc_result *mcr;
1857         struct qman_portal *p = get_affine_portal();
1858
1859         int ret = 0;
1860         u8 res;
1861
1862         mcc = qm_mc_start(&p->p);
1863         mcc->queryfq.fqid = cpu_to_be32(fq->fqid);
1864         qm_mc_commit(&p->p, QM_MCC_VERB_QUERYFQ_NP);
1865         while (!(mcr = qm_mc_result(&p->p)))
1866                 cpu_relax();
1867         res = mcr->result;
1868         if (res == QM_MCR_RESULT_OK)
1869                 ret = !!mcr->queryfq_np.frm_cnt;
1870         return ret;
1871 }
1872
1873 int qman_query_fq_np(struct qman_fq *fq, struct qm_mcr_queryfq_np *np)
1874 {
1875         struct qm_mc_command *mcc;
1876         struct qm_mc_result *mcr;
1877         struct qman_portal *p = get_affine_portal();
1878
1879         u8 res;
1880
1881         mcc = qm_mc_start(&p->p);
1882         mcc->queryfq.fqid = cpu_to_be32(fq->fqid);
1883         qm_mc_commit(&p->p, QM_MCC_VERB_QUERYFQ_NP);
1884         while (!(mcr = qm_mc_result(&p->p)))
1885                 cpu_relax();
1886         DPAA_ASSERT((mcr->verb & QM_MCR_VERB_MASK) == QM_MCR_VERB_QUERYFQ_NP);
1887         res = mcr->result;
1888         if (res == QM_MCR_RESULT_OK) {
1889                 *np = mcr->queryfq_np;
1890                 np->fqd_link = be24_to_cpu(np->fqd_link);
1891                 np->odp_seq = be16_to_cpu(np->odp_seq);
1892                 np->orp_nesn = be16_to_cpu(np->orp_nesn);
1893                 np->orp_ea_hseq  = be16_to_cpu(np->orp_ea_hseq);
1894                 np->orp_ea_tseq  = be16_to_cpu(np->orp_ea_tseq);
1895                 np->orp_ea_hptr = be24_to_cpu(np->orp_ea_hptr);
1896                 np->orp_ea_tptr = be24_to_cpu(np->orp_ea_tptr);
1897                 np->pfdr_hptr = be24_to_cpu(np->pfdr_hptr);
1898                 np->pfdr_tptr = be24_to_cpu(np->pfdr_tptr);
1899                 np->ics_surp = be16_to_cpu(np->ics_surp);
1900                 np->byte_cnt = be32_to_cpu(np->byte_cnt);
1901                 np->frm_cnt = be24_to_cpu(np->frm_cnt);
1902                 np->ra1_sfdr = be16_to_cpu(np->ra1_sfdr);
1903                 np->ra2_sfdr = be16_to_cpu(np->ra2_sfdr);
1904                 np->od1_sfdr = be16_to_cpu(np->od1_sfdr);
1905                 np->od2_sfdr = be16_to_cpu(np->od2_sfdr);
1906                 np->od3_sfdr = be16_to_cpu(np->od3_sfdr);
1907         }
1908         if (res == QM_MCR_RESULT_ERR_FQID)
1909                 return -ERANGE;
1910         else if (res != QM_MCR_RESULT_OK)
1911                 return -EIO;
1912         return 0;
1913 }
1914
1915 int qman_query_fq_frm_cnt(struct qman_fq *fq, u32 *frm_cnt)
1916 {
1917         struct qm_mc_command *mcc;
1918         struct qm_mc_result *mcr;
1919         struct qman_portal *p = get_affine_portal();
1920
1921         mcc = qm_mc_start(&p->p);
1922         mcc->queryfq.fqid = cpu_to_be32(fq->fqid);
1923         qm_mc_commit(&p->p, QM_MCC_VERB_QUERYFQ_NP);
1924         while (!(mcr = qm_mc_result(&p->p)))
1925                 cpu_relax();
1926         DPAA_ASSERT((mcr->verb & QM_MCR_VERB_MASK) == QM_MCR_VERB_QUERYFQ_NP);
1927
1928         if (mcr->result == QM_MCR_RESULT_OK)
1929                 *frm_cnt = be24_to_cpu(mcr->queryfq_np.frm_cnt);
1930         else if (mcr->result == QM_MCR_RESULT_ERR_FQID)
1931                 return -ERANGE;
1932         else if (mcr->result != QM_MCR_RESULT_OK)
1933                 return -EIO;
1934         return 0;
1935 }
1936
1937 int qman_query_wq(u8 query_dedicated, struct qm_mcr_querywq *wq)
1938 {
1939         struct qm_mc_command *mcc;
1940         struct qm_mc_result *mcr;
1941         struct qman_portal *p = get_affine_portal();
1942
1943         u8 res, myverb;
1944
1945         myverb = (query_dedicated) ? QM_MCR_VERB_QUERYWQ_DEDICATED :
1946                                  QM_MCR_VERB_QUERYWQ;
1947         mcc = qm_mc_start(&p->p);
1948         mcc->querywq.channel.id = cpu_to_be16(wq->channel.id);
1949         qm_mc_commit(&p->p, myverb);
1950         while (!(mcr = qm_mc_result(&p->p)))
1951                 cpu_relax();
1952         DPAA_ASSERT((mcr->verb & QM_MCR_VERB_MASK) == myverb);
1953         res = mcr->result;
1954         if (res == QM_MCR_RESULT_OK) {
1955                 int i, array_len;
1956
1957                 wq->channel.id = be16_to_cpu(mcr->querywq.channel.id);
1958                 array_len = ARRAY_SIZE(mcr->querywq.wq_len);
1959                 for (i = 0; i < array_len; i++)
1960                         wq->wq_len[i] = be32_to_cpu(mcr->querywq.wq_len[i]);
1961         }
1962         if (res != QM_MCR_RESULT_OK) {
1963                 pr_err("QUERYWQ failed: %s\n", mcr_result_str(res));
1964                 return -EIO;
1965         }
1966         return 0;
1967 }
1968
1969 int qman_testwrite_cgr(struct qman_cgr *cgr, u64 i_bcnt,
1970                        struct qm_mcr_cgrtestwrite *result)
1971 {
1972         struct qm_mc_command *mcc;
1973         struct qm_mc_result *mcr;
1974         struct qman_portal *p = get_affine_portal();
1975
1976         u8 res;
1977
1978         mcc = qm_mc_start(&p->p);
1979         mcc->cgrtestwrite.cgid = cgr->cgrid;
1980         mcc->cgrtestwrite.i_bcnt_hi = (u8)(i_bcnt >> 32);
1981         mcc->cgrtestwrite.i_bcnt_lo = (u32)i_bcnt;
1982         qm_mc_commit(&p->p, QM_MCC_VERB_CGRTESTWRITE);
1983         while (!(mcr = qm_mc_result(&p->p)))
1984                 cpu_relax();
1985         DPAA_ASSERT((mcr->verb & QM_MCR_VERB_MASK) == QM_MCC_VERB_CGRTESTWRITE);
1986         res = mcr->result;
1987         if (res == QM_MCR_RESULT_OK)
1988                 *result = mcr->cgrtestwrite;
1989         if (res != QM_MCR_RESULT_OK) {
1990                 pr_err("CGR TEST WRITE failed: %s\n", mcr_result_str(res));
1991                 return -EIO;
1992         }
1993         return 0;
1994 }
1995
1996 int qman_query_cgr(struct qman_cgr *cgr, struct qm_mcr_querycgr *cgrd)
1997 {
1998         struct qm_mc_command *mcc;
1999         struct qm_mc_result *mcr;
2000         struct qman_portal *p = get_affine_portal();
2001         u8 res;
2002         unsigned int i;
2003
2004         mcc = qm_mc_start(&p->p);
2005         mcc->querycgr.cgid = cgr->cgrid;
2006         qm_mc_commit(&p->p, QM_MCC_VERB_QUERYCGR);
2007         while (!(mcr = qm_mc_result(&p->p)))
2008                 cpu_relax();
2009         DPAA_ASSERT((mcr->verb & QM_MCR_VERB_MASK) == QM_MCC_VERB_QUERYCGR);
2010         res = mcr->result;
2011         if (res == QM_MCR_RESULT_OK)
2012                 *cgrd = mcr->querycgr;
2013         if (res != QM_MCR_RESULT_OK) {
2014                 pr_err("QUERY_CGR failed: %s\n", mcr_result_str(res));
2015                 return -EIO;
2016         }
2017         cgrd->cgr.wr_parm_g.word =
2018                 be32_to_cpu(cgrd->cgr.wr_parm_g.word);
2019         cgrd->cgr.wr_parm_y.word =
2020                 be32_to_cpu(cgrd->cgr.wr_parm_y.word);
2021         cgrd->cgr.wr_parm_r.word =
2022                 be32_to_cpu(cgrd->cgr.wr_parm_r.word);
2023         cgrd->cgr.cscn_targ =  be32_to_cpu(cgrd->cgr.cscn_targ);
2024         cgrd->cgr.__cs_thres = be16_to_cpu(cgrd->cgr.__cs_thres);
2025         for (i = 0; i < ARRAY_SIZE(cgrd->cscn_targ_swp); i++)
2026                 cgrd->cscn_targ_swp[i] =
2027                         be32_to_cpu(cgrd->cscn_targ_swp[i]);
2028         return 0;
2029 }
2030
2031 int qman_query_congestion(struct qm_mcr_querycongestion *congestion)
2032 {
2033         struct qm_mc_result *mcr;
2034         struct qman_portal *p = get_affine_portal();
2035         u8 res;
2036         unsigned int i;
2037
2038         qm_mc_start(&p->p);
2039         qm_mc_commit(&p->p, QM_MCC_VERB_QUERYCONGESTION);
2040         while (!(mcr = qm_mc_result(&p->p)))
2041                 cpu_relax();
2042         DPAA_ASSERT((mcr->verb & QM_MCR_VERB_MASK) ==
2043                         QM_MCC_VERB_QUERYCONGESTION);
2044         res = mcr->result;
2045         if (res == QM_MCR_RESULT_OK)
2046                 *congestion = mcr->querycongestion;
2047         if (res != QM_MCR_RESULT_OK) {
2048                 pr_err("QUERY_CONGESTION failed: %s\n", mcr_result_str(res));
2049                 return -EIO;
2050         }
2051         for (i = 0; i < ARRAY_SIZE(congestion->state.state); i++)
2052                 congestion->state.state[i] =
2053                         be32_to_cpu(congestion->state.state[i]);
2054         return 0;
2055 }
2056
2057 int qman_set_vdq(struct qman_fq *fq, u16 num, uint32_t vdqcr_flags)
2058 {
2059         struct qman_portal *p = get_affine_portal();
2060         uint32_t vdqcr;
2061         int ret = -EBUSY;
2062
2063         vdqcr = vdqcr_flags;
2064         vdqcr |= QM_VDQCR_NUMFRAMES_SET(num);
2065
2066         if ((fq->state != qman_fq_state_parked) &&
2067             (fq->state != qman_fq_state_retired)) {
2068                 ret = -EINVAL;
2069                 goto out;
2070         }
2071         if (fq_isset(fq, QMAN_FQ_STATE_VDQCR)) {
2072                 ret = -EBUSY;
2073                 goto out;
2074         }
2075         vdqcr = (vdqcr & ~QM_VDQCR_FQID_MASK) | fq->fqid;
2076
2077         if (!p->vdqcr_owned) {
2078                 FQLOCK(fq);
2079                 if (fq_isset(fq, QMAN_FQ_STATE_VDQCR))
2080                         goto escape;
2081                 fq_set(fq, QMAN_FQ_STATE_VDQCR);
2082                 FQUNLOCK(fq);
2083                 p->vdqcr_owned = fq;
2084                 ret = 0;
2085         }
2086 escape:
2087         if (!ret)
2088                 qm_dqrr_vdqcr_set(&p->p, vdqcr);
2089
2090 out:
2091         return ret;
2092 }
2093
2094 int qman_volatile_dequeue(struct qman_fq *fq, u32 flags __maybe_unused,
2095                           u32 vdqcr)
2096 {
2097         struct qman_portal *p;
2098         int ret = -EBUSY;
2099
2100         if ((fq->state != qman_fq_state_parked) &&
2101             (fq->state != qman_fq_state_retired))
2102                 return -EINVAL;
2103         if (vdqcr & QM_VDQCR_FQID_MASK)
2104                 return -EINVAL;
2105         if (fq_isset(fq, QMAN_FQ_STATE_VDQCR))
2106                 return -EBUSY;
2107         vdqcr = (vdqcr & ~QM_VDQCR_FQID_MASK) | fq->fqid;
2108
2109         p = get_affine_portal();
2110
2111         if (!p->vdqcr_owned) {
2112                 FQLOCK(fq);
2113                 if (fq_isset(fq, QMAN_FQ_STATE_VDQCR))
2114                         goto escape;
2115                 fq_set(fq, QMAN_FQ_STATE_VDQCR);
2116                 FQUNLOCK(fq);
2117                 p->vdqcr_owned = fq;
2118                 ret = 0;
2119         }
2120 escape:
2121         if (ret)
2122                 return ret;
2123
2124         /* VDQCR is set */
2125         qm_dqrr_vdqcr_set(&p->p, vdqcr);
2126         return 0;
2127 }
2128
2129 static noinline void update_eqcr_ci(struct qman_portal *p, u8 avail)
2130 {
2131         if (avail)
2132                 qm_eqcr_cce_prefetch(&p->p);
2133         else
2134                 qm_eqcr_cce_update(&p->p);
2135 }
2136
2137 int qman_eqcr_is_empty(void)
2138 {
2139         struct qman_portal *p = get_affine_portal();
2140         u8 avail;
2141
2142         update_eqcr_ci(p, 0);
2143         avail = qm_eqcr_get_fill(&p->p);
2144         return (avail == 0);
2145 }
2146
2147 void qman_set_dc_ern(qman_cb_dc_ern handler, int affine)
2148 {
2149         if (affine) {
2150                 struct qman_portal *p = get_affine_portal();
2151
2152                 p->cb_dc_ern = handler;
2153         } else
2154                 cb_dc_ern = handler;
2155 }
2156
2157 static inline struct qm_eqcr_entry *try_p_eq_start(struct qman_portal *p,
2158                                         struct qman_fq *fq,
2159                                         const struct qm_fd *fd,
2160                                         u32 flags)
2161 {
2162         struct qm_eqcr_entry *eq;
2163         u8 avail;
2164
2165         if (p->use_eqcr_ci_stashing) {
2166                 /*
2167                  * The stashing case is easy, only update if we need to in
2168                  * order to try and liberate ring entries.
2169                  */
2170                 eq = qm_eqcr_start_stash(&p->p);
2171         } else {
2172                 /*
2173                  * The non-stashing case is harder, need to prefetch ahead of
2174                  * time.
2175                  */
2176                 avail = qm_eqcr_get_avail(&p->p);
2177                 if (avail < 2)
2178                         update_eqcr_ci(p, avail);
2179                 eq = qm_eqcr_start_no_stash(&p->p);
2180         }
2181
2182         if (unlikely(!eq))
2183                 return NULL;
2184
2185         if (flags & QMAN_ENQUEUE_FLAG_DCA)
2186                 eq->dca = QM_EQCR_DCA_ENABLE |
2187                         ((flags & QMAN_ENQUEUE_FLAG_DCA_PARK) ?
2188                                         QM_EQCR_DCA_PARK : 0) |
2189                         ((flags >> 8) & QM_EQCR_DCA_IDXMASK);
2190         eq->fqid = cpu_to_be32(fq->fqid);
2191 #ifdef CONFIG_FSL_QMAN_FQ_LOOKUP
2192         eq->tag = cpu_to_be32(fq->key);
2193 #else
2194         eq->tag = cpu_to_be32((u32)(uintptr_t)fq);
2195 #endif
2196         eq->fd = *fd;
2197         cpu_to_hw_fd(&eq->fd);
2198         return eq;
2199 }
2200
2201 int qman_enqueue(struct qman_fq *fq, const struct qm_fd *fd, u32 flags)
2202 {
2203         struct qman_portal *p = get_affine_portal();
2204         struct qm_eqcr_entry *eq;
2205
2206         eq = try_p_eq_start(p, fq, fd, flags);
2207         if (!eq)
2208                 return -EBUSY;
2209         /* Note: QM_EQCR_VERB_INTERRUPT == QMAN_ENQUEUE_FLAG_WAIT_SYNC */
2210         qm_eqcr_pvb_commit(&p->p, QM_EQCR_VERB_CMD_ENQUEUE |
2211                 (flags & (QM_EQCR_VERB_COLOUR_MASK | QM_EQCR_VERB_INTERRUPT)));
2212         /* Factor the below out, it's used from qman_enqueue_orp() too */
2213         return 0;
2214 }
2215
2216 int qman_enqueue_multi(struct qman_fq *fq,
2217                        const struct qm_fd *fd, u32 *flags,
2218                 int frames_to_send)
2219 {
2220         struct qman_portal *p = get_affine_portal();
2221         struct qm_portal *portal = &p->p;
2222
2223         register struct qm_eqcr *eqcr = &portal->eqcr;
2224         struct qm_eqcr_entry *eq = eqcr->cursor, *prev_eq;
2225
2226         u8 i = 0, diff, old_ci, sent = 0;
2227
2228         /* Update the available entries if no entry is free */
2229         if (!eqcr->available) {
2230                 old_ci = eqcr->ci;
2231                 eqcr->ci = qm_cl_in(EQCR_CI) & (QM_EQCR_SIZE - 1);
2232                 diff = qm_cyc_diff(QM_EQCR_SIZE, old_ci, eqcr->ci);
2233                 eqcr->available += diff;
2234                 if (!diff)
2235                         return 0;
2236         }
2237
2238         /* try to send as many frames as possible */
2239         while (eqcr->available && frames_to_send--) {
2240                 eq->fqid = fq->fqid_le;
2241 #ifdef CONFIG_FSL_QMAN_FQ_LOOKUP
2242                 eq->tag = cpu_to_be32(fq->key);
2243 #else
2244                 eq->tag = cpu_to_be32((u32)(uintptr_t)fq);
2245 #endif
2246                 eq->fd.opaque_addr = fd->opaque_addr;
2247                 eq->fd.addr = cpu_to_be40(fd->addr);
2248                 eq->fd.status = cpu_to_be32(fd->status);
2249                 eq->fd.opaque = cpu_to_be32(fd->opaque);
2250                 if (flags && (flags[i] & QMAN_ENQUEUE_FLAG_DCA)) {
2251                         eq->dca = QM_EQCR_DCA_ENABLE |
2252                                 ((flags[i] >> 8) & QM_EQCR_DCA_IDXMASK);
2253                 }
2254                 i++;
2255                 eq = (void *)((unsigned long)(eq + 1) &
2256                         (~(unsigned long)(QM_EQCR_SIZE << 6)));
2257                 eqcr->available--;
2258                 sent++;
2259                 fd++;
2260         }
2261         lwsync();
2262
2263         /* In order for flushes to complete faster, all lines are recorded in
2264          * 32 bit word.
2265          */
2266         eq = eqcr->cursor;
2267         for (i = 0; i < sent; i++) {
2268                 eq->__dont_write_directly__verb =
2269                         QM_EQCR_VERB_CMD_ENQUEUE | eqcr->vbit;
2270                 prev_eq = eq;
2271                 eq = (void *)((unsigned long)(eq + 1) &
2272                         (~(unsigned long)(QM_EQCR_SIZE << 6)));
2273                 if (unlikely((prev_eq + 1) != eq))
2274                         eqcr->vbit ^= QM_EQCR_VERB_VBIT;
2275         }
2276
2277         /* We need  to flush all the lines but without load/store operations
2278          * between them
2279          */
2280         eq = eqcr->cursor;
2281         for (i = 0; i < sent; i++) {
2282                 dcbf(eq);
2283                 eq = (void *)((unsigned long)(eq + 1) &
2284                         (~(unsigned long)(QM_EQCR_SIZE << 6)));
2285         }
2286         /* Update cursor for the next call */
2287         eqcr->cursor = eq;
2288         return sent;
2289 }
2290
2291 int
2292 qman_enqueue_multi_fq(struct qman_fq *fq[], const struct qm_fd *fd,
2293                       int frames_to_send)
2294 {
2295         struct qman_portal *p = get_affine_portal();
2296         struct qm_portal *portal = &p->p;
2297
2298         register struct qm_eqcr *eqcr = &portal->eqcr;
2299         struct qm_eqcr_entry *eq = eqcr->cursor, *prev_eq;
2300
2301         u8 i, diff, old_ci, sent = 0;
2302
2303         /* Update the available entries if no entry is free */
2304         if (!eqcr->available) {
2305                 old_ci = eqcr->ci;
2306                 eqcr->ci = qm_cl_in(EQCR_CI) & (QM_EQCR_SIZE - 1);
2307                 diff = qm_cyc_diff(QM_EQCR_SIZE, old_ci, eqcr->ci);
2308                 eqcr->available += diff;
2309                 if (!diff)
2310                         return 0;
2311         }
2312
2313         /* try to send as many frames as possible */
2314         while (eqcr->available && frames_to_send--) {
2315                 eq->fqid = fq[sent]->fqid_le;
2316                 eq->fd.opaque_addr = fd->opaque_addr;
2317                 eq->fd.addr = cpu_to_be40(fd->addr);
2318                 eq->fd.status = cpu_to_be32(fd->status);
2319                 eq->fd.opaque = cpu_to_be32(fd->opaque);
2320
2321                 eq = (void *)((unsigned long)(eq + 1) &
2322                         (~(unsigned long)(QM_EQCR_SIZE << 6)));
2323                 eqcr->available--;
2324                 sent++;
2325                 fd++;
2326         }
2327         lwsync();
2328
2329         /* In order for flushes to complete faster, all lines are recorded in
2330          * 32 bit word.
2331          */
2332         eq = eqcr->cursor;
2333         for (i = 0; i < sent; i++) {
2334                 eq->__dont_write_directly__verb =
2335                         QM_EQCR_VERB_CMD_ENQUEUE | eqcr->vbit;
2336                 prev_eq = eq;
2337                 eq = (void *)((unsigned long)(eq + 1) &
2338                         (~(unsigned long)(QM_EQCR_SIZE << 6)));
2339                 if (unlikely((prev_eq + 1) != eq))
2340                         eqcr->vbit ^= QM_EQCR_VERB_VBIT;
2341         }
2342
2343         /* We need  to flush all the lines but without load/store operations
2344          * between them
2345          */
2346         eq = eqcr->cursor;
2347         for (i = 0; i < sent; i++) {
2348                 dcbf(eq);
2349                 eq = (void *)((unsigned long)(eq + 1) &
2350                         (~(unsigned long)(QM_EQCR_SIZE << 6)));
2351         }
2352         /* Update cursor for the next call */
2353         eqcr->cursor = eq;
2354         return sent;
2355 }
2356
2357 int qman_enqueue_orp(struct qman_fq *fq, const struct qm_fd *fd, u32 flags,
2358                      struct qman_fq *orp, u16 orp_seqnum)
2359 {
2360         struct qman_portal *p  = get_affine_portal();
2361         struct qm_eqcr_entry *eq;
2362
2363         eq = try_p_eq_start(p, fq, fd, flags);
2364         if (!eq)
2365                 return -EBUSY;
2366         /* Process ORP-specifics here */
2367         if (flags & QMAN_ENQUEUE_FLAG_NLIS)
2368                 orp_seqnum |= QM_EQCR_SEQNUM_NLIS;
2369         else {
2370                 orp_seqnum &= ~QM_EQCR_SEQNUM_NLIS;
2371                 if (flags & QMAN_ENQUEUE_FLAG_NESN)
2372                         orp_seqnum |= QM_EQCR_SEQNUM_NESN;
2373                 else
2374                         /* No need to check 4 QMAN_ENQUEUE_FLAG_HOLE */
2375                         orp_seqnum &= ~QM_EQCR_SEQNUM_NESN;
2376         }
2377         eq->seqnum = cpu_to_be16(orp_seqnum);
2378         eq->orp = cpu_to_be32(orp->fqid);
2379         /* Note: QM_EQCR_VERB_INTERRUPT == QMAN_ENQUEUE_FLAG_WAIT_SYNC */
2380         qm_eqcr_pvb_commit(&p->p, QM_EQCR_VERB_ORP |
2381                 ((flags & (QMAN_ENQUEUE_FLAG_HOLE | QMAN_ENQUEUE_FLAG_NESN)) ?
2382                                 0 : QM_EQCR_VERB_CMD_ENQUEUE) |
2383                 (flags & (QM_EQCR_VERB_COLOUR_MASK | QM_EQCR_VERB_INTERRUPT)));
2384
2385         return 0;
2386 }
2387
2388 int qman_modify_cgr(struct qman_cgr *cgr, u32 flags,
2389                     struct qm_mcc_initcgr *opts)
2390 {
2391         struct qm_mc_command *mcc;
2392         struct qm_mc_result *mcr;
2393         struct qman_portal *p = get_affine_portal();
2394
2395         u8 res;
2396         u8 verb = QM_MCC_VERB_MODIFYCGR;
2397
2398         mcc = qm_mc_start(&p->p);
2399         if (opts)
2400                 mcc->initcgr = *opts;
2401         mcc->initcgr.we_mask = cpu_to_be16(mcc->initcgr.we_mask);
2402         mcc->initcgr.cgr.wr_parm_g.word =
2403                 cpu_to_be32(mcc->initcgr.cgr.wr_parm_g.word);
2404         mcc->initcgr.cgr.wr_parm_y.word =
2405                 cpu_to_be32(mcc->initcgr.cgr.wr_parm_y.word);
2406         mcc->initcgr.cgr.wr_parm_r.word =
2407                 cpu_to_be32(mcc->initcgr.cgr.wr_parm_r.word);
2408         mcc->initcgr.cgr.cscn_targ =  cpu_to_be32(mcc->initcgr.cgr.cscn_targ);
2409         mcc->initcgr.cgr.__cs_thres = cpu_to_be16(mcc->initcgr.cgr.__cs_thres);
2410
2411         mcc->initcgr.cgid = cgr->cgrid;
2412         if (flags & QMAN_CGR_FLAG_USE_INIT)
2413                 verb = QM_MCC_VERB_INITCGR;
2414         qm_mc_commit(&p->p, verb);
2415         while (!(mcr = qm_mc_result(&p->p)))
2416                 cpu_relax();
2417
2418         DPAA_ASSERT((mcr->verb & QM_MCR_VERB_MASK) == verb);
2419         res = mcr->result;
2420         return (res == QM_MCR_RESULT_OK) ? 0 : -EIO;
2421 }
2422
2423 #define TARG_MASK(n) (0x80000000 >> (n->config->channel - \
2424                                         QM_CHANNEL_SWPORTAL0))
2425 #define TARG_DCP_MASK(n) (0x80000000 >> (10 + n))
2426 #define PORTAL_IDX(n) (n->config->channel - QM_CHANNEL_SWPORTAL0)
2427
2428 int qman_create_cgr(struct qman_cgr *cgr, u32 flags,
2429                     struct qm_mcc_initcgr *opts)
2430 {
2431         struct qm_mcr_querycgr cgr_state;
2432         struct qm_mcc_initcgr local_opts;
2433         int ret;
2434         struct qman_portal *p;
2435
2436         /* We have to check that the provided CGRID is within the limits of the
2437          * data-structures, for obvious reasons. However we'll let h/w take
2438          * care of determining whether it's within the limits of what exists on
2439          * the SoC.
2440          */
2441         if (cgr->cgrid >= __CGR_NUM)
2442                 return -EINVAL;
2443
2444         p = get_affine_portal();
2445
2446         memset(&local_opts, 0, sizeof(struct qm_mcc_initcgr));
2447         cgr->chan = p->config->channel;
2448         spin_lock(&p->cgr_lock);
2449
2450         /* if no opts specified, just add it to the list */
2451         if (!opts)
2452                 goto add_list;
2453
2454         ret = qman_query_cgr(cgr, &cgr_state);
2455         if (ret)
2456                 goto release_lock;
2457         if (opts)
2458                 local_opts = *opts;
2459         if ((qman_ip_rev & 0xFF00) >= QMAN_REV30)
2460                 local_opts.cgr.cscn_targ_upd_ctrl =
2461                         QM_CGR_TARG_UDP_CTRL_WRITE_BIT | PORTAL_IDX(p);
2462         else
2463                 /* Overwrite TARG */
2464                 local_opts.cgr.cscn_targ = cgr_state.cgr.cscn_targ |
2465                                                         TARG_MASK(p);
2466         local_opts.we_mask |= QM_CGR_WE_CSCN_TARG;
2467
2468         /* send init if flags indicate so */
2469         if (opts && (flags & QMAN_CGR_FLAG_USE_INIT))
2470                 ret = qman_modify_cgr(cgr, QMAN_CGR_FLAG_USE_INIT, &local_opts);
2471         else
2472                 ret = qman_modify_cgr(cgr, 0, &local_opts);
2473         if (ret)
2474                 goto release_lock;
2475 add_list:
2476         list_add(&cgr->node, &p->cgr_cbs);
2477
2478         /* Determine if newly added object requires its callback to be called */
2479         ret = qman_query_cgr(cgr, &cgr_state);
2480         if (ret) {
2481                 /* we can't go back, so proceed and return success, but screen
2482                  * and wail to the log file.
2483                  */
2484                 pr_crit("CGR HW state partially modified\n");
2485                 ret = 0;
2486                 goto release_lock;
2487         }
2488         if (cgr->cb && cgr_state.cgr.cscn_en && qman_cgrs_get(&p->cgrs[1],
2489                                                               cgr->cgrid))
2490                 cgr->cb(p, cgr, 1);
2491 release_lock:
2492         spin_unlock(&p->cgr_lock);
2493         return ret;
2494 }
2495
2496 int qman_create_cgr_to_dcp(struct qman_cgr *cgr, u32 flags, u16 dcp_portal,
2497                            struct qm_mcc_initcgr *opts)
2498 {
2499         struct qm_mcc_initcgr local_opts;
2500         struct qm_mcr_querycgr cgr_state;
2501         int ret;
2502
2503         if ((qman_ip_rev & 0xFF00) < QMAN_REV30) {
2504                 pr_warn("QMan version doesn't support CSCN => DCP portal\n");
2505                 return -EINVAL;
2506         }
2507         /* We have to check that the provided CGRID is within the limits of the
2508          * data-structures, for obvious reasons. However we'll let h/w take
2509          * care of determining whether it's within the limits of what exists on
2510          * the SoC.
2511          */
2512         if (cgr->cgrid >= __CGR_NUM)
2513                 return -EINVAL;
2514
2515         ret = qman_query_cgr(cgr, &cgr_state);
2516         if (ret)
2517                 return ret;
2518
2519         memset(&local_opts, 0, sizeof(struct qm_mcc_initcgr));
2520         if (opts)
2521                 local_opts = *opts;
2522
2523         if ((qman_ip_rev & 0xFF00) >= QMAN_REV30)
2524                 local_opts.cgr.cscn_targ_upd_ctrl =
2525                                 QM_CGR_TARG_UDP_CTRL_WRITE_BIT |
2526                                 QM_CGR_TARG_UDP_CTRL_DCP | dcp_portal;
2527         else
2528                 local_opts.cgr.cscn_targ = cgr_state.cgr.cscn_targ |
2529                                         TARG_DCP_MASK(dcp_portal);
2530         local_opts.we_mask |= QM_CGR_WE_CSCN_TARG;
2531
2532         /* send init if flags indicate so */
2533         if (opts && (flags & QMAN_CGR_FLAG_USE_INIT))
2534                 ret = qman_modify_cgr(cgr, QMAN_CGR_FLAG_USE_INIT,
2535                                       &local_opts);
2536         else
2537                 ret = qman_modify_cgr(cgr, 0, &local_opts);
2538
2539         return ret;
2540 }
2541
2542 int qman_delete_cgr(struct qman_cgr *cgr)
2543 {
2544         struct qm_mcr_querycgr cgr_state;
2545         struct qm_mcc_initcgr local_opts;
2546         int ret = 0;
2547         struct qman_cgr *i;
2548         struct qman_portal *p = get_affine_portal();
2549
2550         if (cgr->chan != p->config->channel) {
2551                 pr_crit("Attempting to delete cgr from different portal than"
2552                         " it was create: create 0x%x, delete 0x%x\n",
2553                         cgr->chan, p->config->channel);
2554                 ret = -EINVAL;
2555                 goto put_portal;
2556         }
2557         memset(&local_opts, 0, sizeof(struct qm_mcc_initcgr));
2558         spin_lock(&p->cgr_lock);
2559         list_del(&cgr->node);
2560         /*
2561          * If there are no other CGR objects for this CGRID in the list,
2562          * update CSCN_TARG accordingly
2563          */
2564         list_for_each_entry(i, &p->cgr_cbs, node)
2565                 if ((i->cgrid == cgr->cgrid) && i->cb)
2566                         goto release_lock;
2567         ret = qman_query_cgr(cgr, &cgr_state);
2568         if (ret)  {
2569                 /* add back to the list */
2570                 list_add(&cgr->node, &p->cgr_cbs);
2571                 goto release_lock;
2572         }
2573         /* Overwrite TARG */
2574         local_opts.we_mask = QM_CGR_WE_CSCN_TARG;
2575         if ((qman_ip_rev & 0xFF00) >= QMAN_REV30)
2576                 local_opts.cgr.cscn_targ_upd_ctrl = PORTAL_IDX(p);
2577         else
2578                 local_opts.cgr.cscn_targ = cgr_state.cgr.cscn_targ &
2579                                                          ~(TARG_MASK(p));
2580         ret = qman_modify_cgr(cgr, 0, &local_opts);
2581         if (ret)
2582                 /* add back to the list */
2583                 list_add(&cgr->node, &p->cgr_cbs);
2584 release_lock:
2585         spin_unlock(&p->cgr_lock);
2586 put_portal:
2587         return ret;
2588 }
2589
2590 int qman_shutdown_fq(u32 fqid)
2591 {
2592         struct qman_portal *p;
2593         struct qm_portal *low_p;
2594         struct qm_mc_command *mcc;
2595         struct qm_mc_result *mcr;
2596         u8 state;
2597         int orl_empty, fq_empty, drain = 0;
2598         u32 result;
2599         u32 channel, wq;
2600         u16 dest_wq;
2601
2602         p = get_affine_portal();
2603         low_p = &p->p;
2604
2605         /* Determine the state of the FQID */
2606         mcc = qm_mc_start(low_p);
2607         mcc->queryfq_np.fqid = cpu_to_be32(fqid);
2608         qm_mc_commit(low_p, QM_MCC_VERB_QUERYFQ_NP);
2609         while (!(mcr = qm_mc_result(low_p)))
2610                 cpu_relax();
2611         DPAA_ASSERT((mcr->verb & QM_MCR_VERB_MASK) == QM_MCR_VERB_QUERYFQ_NP);
2612         state = mcr->queryfq_np.state & QM_MCR_NP_STATE_MASK;
2613         if (state == QM_MCR_NP_STATE_OOS)
2614                 return 0; /* Already OOS, no need to do anymore checks */
2615
2616         /* Query which channel the FQ is using */
2617         mcc = qm_mc_start(low_p);
2618         mcc->queryfq.fqid = cpu_to_be32(fqid);
2619         qm_mc_commit(low_p, QM_MCC_VERB_QUERYFQ);
2620         while (!(mcr = qm_mc_result(low_p)))
2621                 cpu_relax();
2622         DPAA_ASSERT((mcr->verb & QM_MCR_VERB_MASK) == QM_MCR_VERB_QUERYFQ);
2623
2624         /* Need to store these since the MCR gets reused */
2625         dest_wq = be16_to_cpu(mcr->queryfq.fqd.dest_wq);
2626         channel = dest_wq & 0x7;
2627         wq = dest_wq >> 3;
2628
2629         switch (state) {
2630         case QM_MCR_NP_STATE_TEN_SCHED:
2631         case QM_MCR_NP_STATE_TRU_SCHED:
2632         case QM_MCR_NP_STATE_ACTIVE:
2633         case QM_MCR_NP_STATE_PARKED:
2634                 orl_empty = 0;
2635                 mcc = qm_mc_start(low_p);
2636                 mcc->alterfq.fqid = cpu_to_be32(fqid);
2637                 qm_mc_commit(low_p, QM_MCC_VERB_ALTER_RETIRE);
2638                 while (!(mcr = qm_mc_result(low_p)))
2639                         cpu_relax();
2640                 DPAA_ASSERT((mcr->verb & QM_MCR_VERB_MASK) ==
2641                            QM_MCR_VERB_ALTER_RETIRE);
2642                 result = mcr->result; /* Make a copy as we reuse MCR below */
2643
2644                 if (result == QM_MCR_RESULT_PENDING) {
2645                         /* Need to wait for the FQRN in the message ring, which
2646                          * will only occur once the FQ has been drained.  In
2647                          * order for the FQ to drain the portal needs to be set
2648                          * to dequeue from the channel the FQ is scheduled on
2649                          */
2650                         const struct qm_mr_entry *msg;
2651                         const struct qm_dqrr_entry *dqrr = NULL;
2652                         int found_fqrn = 0;
2653                         __maybe_unused u16 dequeue_wq = 0;
2654
2655                         /* Flag that we need to drain FQ */
2656                         drain = 1;
2657
2658                         if (channel >= qm_channel_pool1 &&
2659                             channel < (u16)(qm_channel_pool1 + 15)) {
2660                                 /* Pool channel, enable the bit in the portal */
2661                                 dequeue_wq = (channel -
2662                                               qm_channel_pool1 + 1) << 4 | wq;
2663                         } else if (channel < qm_channel_pool1) {
2664                                 /* Dedicated channel */
2665                                 dequeue_wq = wq;
2666                         } else {
2667                                 pr_info("Cannot recover FQ 0x%x,"
2668                                         " it is scheduled on channel 0x%x",
2669                                         fqid, channel);
2670                                 return -EBUSY;
2671                         }
2672                         /* Set the sdqcr to drain this channel */
2673                         if (channel < qm_channel_pool1)
2674                                 qm_dqrr_sdqcr_set(low_p,
2675                                                   QM_SDQCR_TYPE_ACTIVE |
2676                                           QM_SDQCR_CHANNELS_DEDICATED);
2677                         else
2678                                 qm_dqrr_sdqcr_set(low_p,
2679                                                   QM_SDQCR_TYPE_ACTIVE |
2680                                                   QM_SDQCR_CHANNELS_POOL_CONV
2681                                                   (channel));
2682                         while (!found_fqrn) {
2683                                 /* Keep draining DQRR while checking the MR*/
2684                                 qm_dqrr_pvb_update(low_p);
2685                                 dqrr = qm_dqrr_current(low_p);
2686                                 while (dqrr) {
2687                                         qm_dqrr_cdc_consume_1ptr(
2688                                                 low_p, dqrr, 0);
2689                                         qm_dqrr_pvb_update(low_p);
2690                                         qm_dqrr_next(low_p);
2691                                         dqrr = qm_dqrr_current(low_p);
2692                                 }
2693                                 /* Process message ring too */
2694                                 qm_mr_pvb_update(low_p);
2695                                 msg = qm_mr_current(low_p);
2696                                 while (msg) {
2697                                         if ((msg->ern.verb &
2698                                              QM_MR_VERB_TYPE_MASK)
2699                                             == QM_MR_VERB_FQRN)
2700                                                 found_fqrn = 1;
2701                                         qm_mr_next(low_p);
2702                                         qm_mr_cci_consume_to_current(low_p);
2703                                         qm_mr_pvb_update(low_p);
2704                                         msg = qm_mr_current(low_p);
2705                                 }
2706                                 cpu_relax();
2707                         }
2708                 }
2709                 if (result != QM_MCR_RESULT_OK &&
2710                     result !=  QM_MCR_RESULT_PENDING) {
2711                         /* error */
2712                         pr_err("qman_retire_fq failed on FQ 0x%x,"
2713                                " result=0x%x\n", fqid, result);
2714                         return -1;
2715                 }
2716                 if (!(mcr->alterfq.fqs & QM_MCR_FQS_ORLPRESENT)) {
2717                         /* ORL had no entries, no need to wait until the
2718                          * ERNs come in.
2719                          */
2720                         orl_empty = 1;
2721                 }
2722                 /* Retirement succeeded, check to see if FQ needs
2723                  * to be drained.
2724                  */
2725                 if (drain || mcr->alterfq.fqs & QM_MCR_FQS_NOTEMPTY) {
2726                         /* FQ is Not Empty, drain using volatile DQ commands */
2727                         fq_empty = 0;
2728                         do {
2729                                 const struct qm_dqrr_entry *dqrr = NULL;
2730                                 u32 vdqcr = fqid | QM_VDQCR_NUMFRAMES_SET(3);
2731
2732                                 qm_dqrr_vdqcr_set(low_p, vdqcr);
2733
2734                                 /* Wait for a dequeue to occur */
2735                                 while (dqrr == NULL) {
2736                                         qm_dqrr_pvb_update(low_p);
2737                                         dqrr = qm_dqrr_current(low_p);
2738                                         if (!dqrr)
2739                                                 cpu_relax();
2740                                 }
2741                                 /* Process the dequeues, making sure to
2742                                  * empty the ring completely.
2743                                  */
2744                                 while (dqrr) {
2745                                         if (dqrr->fqid == fqid &&
2746                                             dqrr->stat & QM_DQRR_STAT_FQ_EMPTY)
2747                                                 fq_empty = 1;
2748                                         qm_dqrr_cdc_consume_1ptr(low_p,
2749                                                                  dqrr, 0);
2750                                         qm_dqrr_pvb_update(low_p);
2751                                         qm_dqrr_next(low_p);
2752                                         dqrr = qm_dqrr_current(low_p);
2753                                 }
2754                         } while (fq_empty == 0);
2755                 }
2756                 qm_dqrr_sdqcr_set(low_p, 0);
2757
2758                 /* Wait for the ORL to have been completely drained */
2759                 while (orl_empty == 0) {
2760                         const struct qm_mr_entry *msg;
2761
2762                         qm_mr_pvb_update(low_p);
2763                         msg = qm_mr_current(low_p);
2764                         while (msg) {
2765                                 if ((msg->ern.verb & QM_MR_VERB_TYPE_MASK) ==
2766                                     QM_MR_VERB_FQRL)
2767                                         orl_empty = 1;
2768                                 qm_mr_next(low_p);
2769                                 qm_mr_cci_consume_to_current(low_p);
2770                                 qm_mr_pvb_update(low_p);
2771                                 msg = qm_mr_current(low_p);
2772                         }
2773                         cpu_relax();
2774                 }
2775                 mcc = qm_mc_start(low_p);
2776                 mcc->alterfq.fqid = cpu_to_be32(fqid);
2777                 qm_mc_commit(low_p, QM_MCC_VERB_ALTER_OOS);
2778                 while (!(mcr = qm_mc_result(low_p)))
2779                         cpu_relax();
2780                 DPAA_ASSERT((mcr->verb & QM_MCR_VERB_MASK) ==
2781                            QM_MCR_VERB_ALTER_OOS);
2782                 if (mcr->result != QM_MCR_RESULT_OK) {
2783                         pr_err(
2784                         "OOS after drain Failed on FQID 0x%x, result 0x%x\n",
2785                                fqid, mcr->result);
2786                         return -1;
2787                 }
2788                 return 0;
2789
2790         case QM_MCR_NP_STATE_RETIRED:
2791                 /* Send OOS Command */
2792                 mcc = qm_mc_start(low_p);
2793                 mcc->alterfq.fqid = cpu_to_be32(fqid);
2794                 qm_mc_commit(low_p, QM_MCC_VERB_ALTER_OOS);
2795                 while (!(mcr = qm_mc_result(low_p)))
2796                         cpu_relax();
2797                 DPAA_ASSERT((mcr->verb & QM_MCR_VERB_MASK) ==
2798                            QM_MCR_VERB_ALTER_OOS);
2799                 if (mcr->result) {
2800                         pr_err("OOS Failed on FQID 0x%x\n", fqid);
2801                         return -1;
2802                 }
2803                 return 0;
2804
2805         }
2806         return -1;
2807 }