drivers/bus/dpaa/base/qbman/qman.c

   1 /* SPDX-License-Identifier: (BSD-3-Clause OR GPL-2.0)
   2  *
   3  * Copyright 2008-2016 Freescale Semiconductor Inc.
   4  * Copyright 2017,2019 NXP
   5  *
   6  */
   7
   8 #include "qman.h"
   9 #include <rte_branch_prediction.h>
  10 #include <rte_dpaa_bus.h>
  11 #include <rte_eventdev.h>
  12 #include <rte_byteorder.h>
  13
  14 /* Compilation constants */
  15 #define DQRR_MAXFILL    15
  16 #define EQCR_ITHRESH    4       /* if EQCR congests, interrupt threshold */
  17 #define IRQNAME         "QMan portal %d"
  18 #define MAX_IRQNAME     16      /* big enough for "QMan portal %d" */
  19 /* maximum number of DQRR entries to process in qman_poll() */
  20 #define FSL_QMAN_POLL_LIMIT 8
  21
  22 /* Lock/unlock frame queues, subject to the "LOCKED" flag. This is about
  23  * inter-processor locking only. Note, FQLOCK() is always called either under a
  24  * local_irq_save() or from interrupt context - hence there's no need for irq
  25  * protection (and indeed, attempting to nest irq-protection doesn't work, as
  26  * the "irq en/disable" machinery isn't recursive...).
  27  */
  28 #define FQLOCK(fq) \
  29         do { \
  30                 struct qman_fq *__fq478 = (fq); \
  31                 if (fq_isset(__fq478, QMAN_FQ_FLAG_LOCKED)) \
  32                         spin_lock(&__fq478->fqlock); \
  33         } while (0)
  34 #define FQUNLOCK(fq) \
  35         do { \
  36                 struct qman_fq *__fq478 = (fq); \
  37                 if (fq_isset(__fq478, QMAN_FQ_FLAG_LOCKED)) \
  38                         spin_unlock(&__fq478->fqlock); \
  39         } while (0)
  40
  41 static inline void fq_set(struct qman_fq *fq, u32 mask)
  42 {
  43         dpaa_set_bits(mask, &fq->flags);
  44 }
  45
  46 static inline void fq_clear(struct qman_fq *fq, u32 mask)
  47 {
  48         dpaa_clear_bits(mask, &fq->flags);
  49 }
  50
  51 static inline int fq_isset(struct qman_fq *fq, u32 mask)
  52 {
  53         return fq->flags & mask;
  54 }
  55
  56 static inline int fq_isclear(struct qman_fq *fq, u32 mask)
  57 {
  58         return !(fq->flags & mask);
  59 }
  60
  61 struct qman_portal {
  62         struct qm_portal p;
  63         /* PORTAL_BITS_*** - dynamic, strictly internal */
  64         unsigned long bits;
  65         /* interrupt sources processed by portal_isr(), configurable */
  66         unsigned long irq_sources;
  67         u32 use_eqcr_ci_stashing;
  68         u32 slowpoll;   /* only used when interrupts are off */
  69         /* only 1 volatile dequeue at a time */
  70         struct qman_fq *vdqcr_owned;
  71         u32 sdqcr;
  72         int dqrr_disable_ref;
  73         /* A portal-specific handler for DCP ERNs. If this is NULL, the global
  74          * handler is called instead.
  75          */
  76         qman_cb_dc_ern cb_dc_ern;
  77         /* When the cpu-affine portal is activated, this is non-NULL */
  78         const struct qm_portal_config *config;
  79         struct dpa_rbtree retire_table;
  80         char irqname[MAX_IRQNAME];
  81         /* 2-element array. cgrs[0] is mask, cgrs[1] is snapshot. */
  82         struct qman_cgrs *cgrs;
  83         /* linked-list of CSCN handlers. */
  84         struct list_head cgr_cbs;
  85         /* list lock */
  86         spinlock_t cgr_lock;
  87         /* track if memory was allocated by the driver */
  88 #if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__
  89         /* Keep a shadow copy of the DQRR on LE systems as the SW needs to
  90          * do byte swaps of DQRR read only memory.  First entry must be aligned
  91          * to 2 ** 10 to ensure DQRR index calculations based shadow copy
  92          * address (6 bits for address shift + 4 bits for the DQRR size).
  93          */
  94         struct qm_dqrr_entry shadow_dqrr[QM_DQRR_SIZE]
  95                     __attribute__((aligned(1024)));
  96 #endif
  97 };
  98
  99 /* Global handler for DCP ERNs. Used when the portal receiving the message does
 100  * not have a portal-specific handler.
 101  */
 102 static qman_cb_dc_ern cb_dc_ern;
 103
 104 static cpumask_t affine_mask;
 105 static DEFINE_SPINLOCK(affine_mask_lock);
 106 static u16 affine_channels[NR_CPUS];
 107 static RTE_DEFINE_PER_LCORE(struct qman_portal, qman_affine_portal);
 108
 109 static inline struct qman_portal *get_affine_portal(void)
 110 {
 111         return &RTE_PER_LCORE(qman_affine_portal);
 112 }
 113
 114 /* This gives a FQID->FQ lookup to cover the fact that we can't directly demux
 115  * retirement notifications (the fact they are sometimes h/w-consumed means that
 116  * contextB isn't always a s/w demux - and as we can't know which case it is
 117  * when looking at the notification, we have to use the slow lookup for all of
 118  * them). NB, it's possible to have multiple FQ objects refer to the same FQID
 119  * (though at most one of them should be the consumer), so this table isn't for
 120  * all FQs - FQs are added when retirement commands are issued, and removed when
 121  * they complete, which also massively reduces the size of this table.
 122  */
 123 IMPLEMENT_DPAA_RBTREE(fqtree, struct qman_fq, node, fqid);
 124 /*
 125  * This is what everything can wait on, even if it migrates to a different cpu
 126  * to the one whose affine portal it is waiting on.
 127  */
 128 static DECLARE_WAIT_QUEUE_HEAD(affine_queue);
 129
 130 static inline int table_push_fq(struct qman_portal *p, struct qman_fq *fq)
 131 {
 132         int ret = fqtree_push(&p->retire_table, fq);
 133
 134         if (ret)
 135                 pr_err("ERROR: double FQ-retirement %d\n", fq->fqid);
 136         return ret;
 137 }
 138
 139 static inline void table_del_fq(struct qman_portal *p, struct qman_fq *fq)
 140 {
 141         fqtree_del(&p->retire_table, fq);
 142 }
 143
 144 static inline struct qman_fq *table_find_fq(struct qman_portal *p, u32 fqid)
 145 {
 146         return fqtree_find(&p->retire_table, fqid);
 147 }
 148
 149 #ifdef CONFIG_FSL_QMAN_FQ_LOOKUP
 150 static void **qman_fq_lookup_table;
 151 static size_t qman_fq_lookup_table_size;
 152
 153 int qman_setup_fq_lookup_table(size_t num_entries)
 154 {
 155         num_entries++;
 156         /* Allocate 1 more entry since the first entry is not used */
 157         qman_fq_lookup_table = vmalloc((num_entries * sizeof(void *)));
 158         if (!qman_fq_lookup_table) {
 159                 pr_err("QMan: Could not allocate fq lookup table\n");
 160                 return -ENOMEM;
 161         }
 162         memset(qman_fq_lookup_table, 0, num_entries * sizeof(void *));
 163         qman_fq_lookup_table_size = num_entries;
 164         pr_debug("QMan: Allocated lookup table at %p, entry count %lu\n",
 165                 qman_fq_lookup_table,
 166                         (unsigned long)qman_fq_lookup_table_size);
 167         return 0;
 168 }
 169
 170 void qman_set_fq_lookup_table(void **fq_table)
 171 {
 172         qman_fq_lookup_table = fq_table;
 173 }
 174
 175 /* global structure that maintains fq object mapping */
 176 static DEFINE_SPINLOCK(fq_hash_table_lock);
 177
 178 static int find_empty_fq_table_entry(u32 *entry, struct qman_fq *fq)
 179 {
 180         u32 i;
 181
 182         spin_lock(&fq_hash_table_lock);
 183         /* Can't use index zero because this has special meaning
 184          * in context_b field.
 185          */
 186         for (i = 1; i < qman_fq_lookup_table_size; i++) {
 187                 if (qman_fq_lookup_table[i] == NULL) {
 188                         *entry = i;
 189                         qman_fq_lookup_table[i] = fq;
 190                         spin_unlock(&fq_hash_table_lock);
 191                         return 0;
 192                 }
 193         }
 194         spin_unlock(&fq_hash_table_lock);
 195         return -ENOMEM;
 196 }
 197
 198 static void clear_fq_table_entry(u32 entry)
 199 {
 200         spin_lock(&fq_hash_table_lock);
 201         DPAA_BUG_ON(entry >= qman_fq_lookup_table_size);
 202         qman_fq_lookup_table[entry] = NULL;
 203         spin_unlock(&fq_hash_table_lock);
 204 }
 205
 206 static inline struct qman_fq *get_fq_table_entry(u32 entry)
 207 {
 208         DPAA_BUG_ON(entry >= qman_fq_lookup_table_size);
 209         return qman_fq_lookup_table[entry];
 210 }
 211 #endif
 212
 213 static inline void cpu_to_hw_fqd(struct qm_fqd *fqd)
 214 {
 215         /* Byteswap the FQD to HW format */
 216         fqd->fq_ctrl = cpu_to_be16(fqd->fq_ctrl);
 217         fqd->dest_wq = cpu_to_be16(fqd->dest_wq);
 218         fqd->ics_cred = cpu_to_be16(fqd->ics_cred);
 219         fqd->context_b = cpu_to_be32(fqd->context_b);
 220         fqd->context_a.opaque = cpu_to_be64(fqd->context_a.opaque);
 221         fqd->opaque_td = cpu_to_be16(fqd->opaque_td);
 222 }
 223
 224 static inline void hw_fqd_to_cpu(struct qm_fqd *fqd)
 225 {
 226         /* Byteswap the FQD to CPU format */
 227         fqd->fq_ctrl = be16_to_cpu(fqd->fq_ctrl);
 228         fqd->dest_wq = be16_to_cpu(fqd->dest_wq);
 229         fqd->ics_cred = be16_to_cpu(fqd->ics_cred);
 230         fqd->context_b = be32_to_cpu(fqd->context_b);
 231         fqd->context_a.opaque = be64_to_cpu(fqd->context_a.opaque);
 232 }
 233
 234 static inline void cpu_to_hw_fd(struct qm_fd *fd)
 235 {
 236         fd->addr = cpu_to_be40(fd->addr);
 237         fd->status = cpu_to_be32(fd->status);
 238         fd->opaque = cpu_to_be32(fd->opaque);
 239 }
 240
 241 static inline void hw_fd_to_cpu(struct qm_fd *fd)
 242 {
 243         fd->addr = be40_to_cpu(fd->addr);
 244         fd->status = be32_to_cpu(fd->status);
 245         fd->opaque = be32_to_cpu(fd->opaque);
 246 }
 247
 248 /* In the case that slow- and fast-path handling are both done by qman_poll()
 249  * (ie. because there is no interrupt handling), we ought to balance how often
 250  * we do the fast-path poll versus the slow-path poll. We'll use two decrementer
 251  * sources, so we call the fast poll 'n' times before calling the slow poll
 252  * once. The idle decrementer constant is used when the last slow-poll detected
 253  * no work to do, and the busy decrementer constant when the last slow-poll had
 254  * work to do.
 255  */
 256 #define SLOW_POLL_IDLE   1000
 257 #define SLOW_POLL_BUSY   10
 258 static u32 __poll_portal_slow(struct qman_portal *p, u32 is);
 259 static inline unsigned int __poll_portal_fast(struct qman_portal *p,
 260                                               unsigned int poll_limit);
 261
 262 /* Portal interrupt handler */
 263 static irqreturn_t portal_isr(__always_unused int irq, void *ptr)
 264 {
 265         struct qman_portal *p = ptr;
 266         /*
 267          * The CSCI/CCSCI source is cleared inside __poll_portal_slow(), because
 268          * it could race against a Query Congestion State command also given
 269          * as part of the handling of this interrupt source. We mustn't
 270          * clear it a second time in this top-level function.
 271          */
 272         u32 clear = QM_DQAVAIL_MASK | (p->irq_sources &
 273                 ~(QM_PIRQ_CSCI | QM_PIRQ_CCSCI));
 274         u32 is = qm_isr_status_read(&p->p) & p->irq_sources;
 275         /* DQRR-handling if it's interrupt-driven */
 276         if (is & QM_PIRQ_DQRI)
 277                 __poll_portal_fast(p, FSL_QMAN_POLL_LIMIT);
 278         /* Handling of anything else that's interrupt-driven */
 279         clear |= __poll_portal_slow(p, is);
 280         qm_isr_status_clear(&p->p, clear);
 281         return IRQ_HANDLED;
 282 }
 283
 284 /* This inner version is used privately by qman_create_affine_portal(), as well
 285  * as by the exported qman_stop_dequeues().
 286  */
 287 static inline void qman_stop_dequeues_ex(struct qman_portal *p)
 288 {
 289         if (!(p->dqrr_disable_ref++))
 290                 qm_dqrr_set_maxfill(&p->p, 0);
 291 }
 292
 293 static int drain_mr_fqrni(struct qm_portal *p)
 294 {
 295         const struct qm_mr_entry *msg;
 296 loop:
 297         msg = qm_mr_current(p);
 298         if (!msg) {
 299                 /*
 300                  * if MR was full and h/w had other FQRNI entries to produce, we
 301                  * need to allow it time to produce those entries once the
 302                  * existing entries are consumed. A worst-case situation
 303                  * (fully-loaded system) means h/w sequencers may have to do 3-4
 304                  * other things before servicing the portal's MR pump, each of
 305                  * which (if slow) may take ~50 qman cycles (which is ~200
 306                  * processor cycles). So rounding up and then multiplying this
 307                  * worst-case estimate by a factor of 10, just to be
 308                  * ultra-paranoid, goes as high as 10,000 cycles. NB, we consume
 309                  * one entry at a time, so h/w has an opportunity to produce new
 310                  * entries well before the ring has been fully consumed, so
 311                  * we're being *really* paranoid here.
 312                  */
 313                 u64 now, then = mfatb();
 314
 315                 do {
 316                         now = mfatb();
 317                 } while ((then + 10000) > now);
 318                 msg = qm_mr_current(p);
 319                 if (!msg)
 320                         return 0;
 321         }
 322         if ((msg->ern.verb & QM_MR_VERB_TYPE_MASK) != QM_MR_VERB_FQRNI) {
 323                 /* We aren't draining anything but FQRNIs */
 324                 pr_err("Found verb 0x%x in MR\n", msg->ern.verb);
 325                 return -1;
 326         }
 327         qm_mr_next(p);
 328         qm_mr_cci_consume(p, 1);
 329         goto loop;
 330 }
 331
 332 static inline int qm_eqcr_init(struct qm_portal *portal,
 333                                enum qm_eqcr_pmode pmode,
 334                                unsigned int eq_stash_thresh,
 335                                int eq_stash_prio)
 336 {
 337         /* This use of 'register', as well as all other occurrences, is because
 338          * it has been observed to generate much faster code with gcc than is
 339          * otherwise the case.
 340          */
 341         register struct qm_eqcr *eqcr = &portal->eqcr;
 342         u32 cfg;
 343         u8 pi;
 344
 345         eqcr->ring = portal->addr.ce + QM_CL_EQCR;
 346         eqcr->ci = qm_in(EQCR_CI_CINH) & (QM_EQCR_SIZE - 1);
 347         qm_cl_invalidate(EQCR_CI);
 348         pi = qm_in(EQCR_PI_CINH) & (QM_EQCR_SIZE - 1);
 349         eqcr->cursor = eqcr->ring + pi;
 350         eqcr->vbit = (qm_in(EQCR_PI_CINH) & QM_EQCR_SIZE) ?
 351                         QM_EQCR_VERB_VBIT : 0;
 352         eqcr->available = QM_EQCR_SIZE - 1 -
 353                         qm_cyc_diff(QM_EQCR_SIZE, eqcr->ci, pi);
 354         eqcr->ithresh = qm_in(EQCR_ITR);
 355 #ifdef RTE_LIBRTE_DPAA_HWDEBUG
 356         eqcr->busy = 0;
 357         eqcr->pmode = pmode;
 358 #endif
 359         cfg = (qm_in(CFG) & 0x00ffffff) |
 360                 (eq_stash_thresh << 28) | /* QCSP_CFG: EST */
 361                 (eq_stash_prio << 26)   | /* QCSP_CFG: EP */
 362                 ((pmode & 0x3) << 24);  /* QCSP_CFG::EPM */
 363         qm_out(CFG, cfg);
 364         return 0;
 365 }
 366
 367 static inline void qm_eqcr_finish(struct qm_portal *portal)
 368 {
 369         register struct qm_eqcr *eqcr = &portal->eqcr;
 370         u8 pi, ci;
 371         u32 cfg;
 372
 373         /*
 374          * Disable EQCI stashing because the QMan only
 375          * presents the value it previously stashed to
 376          * maintain coherency.  Setting the stash threshold
 377          * to 1 then 0 ensures that QMan has resyncronized
 378          * its internal copy so that the portal is clean
 379          * when it is reinitialized in the future
 380          */
 381         cfg = (qm_in(CFG) & 0x0fffffff) |
 382                 (1 << 28); /* QCSP_CFG: EST */
 383         qm_out(CFG, cfg);
 384         cfg &= 0x0fffffff; /* stash threshold = 0 */
 385         qm_out(CFG, cfg);
 386
 387         pi = qm_in(EQCR_PI_CINH) & (QM_EQCR_SIZE - 1);
 388         ci = qm_in(EQCR_CI_CINH) & (QM_EQCR_SIZE - 1);
 389
 390         /* Refresh EQCR CI cache value */
 391         qm_cl_invalidate(EQCR_CI);
 392         eqcr->ci = qm_cl_in(EQCR_CI) & (QM_EQCR_SIZE - 1);
 393
 394 #ifdef RTE_LIBRTE_DPAA_HWDEBUG
 395         DPAA_ASSERT(!eqcr->busy);
 396 #endif
 397         if (pi != EQCR_PTR2IDX(eqcr->cursor))
 398                 pr_crit("losing uncommitted EQCR entries\n");
 399         if (ci != eqcr->ci)
 400                 pr_crit("missing existing EQCR completions\n");
 401         if (eqcr->ci != EQCR_PTR2IDX(eqcr->cursor))
 402                 pr_crit("EQCR destroyed unquiesced\n");
 403 }
 404
 405 static inline int qm_dqrr_init(struct qm_portal *portal,
 406                         __maybe_unused const struct qm_portal_config *config,
 407                         enum qm_dqrr_dmode dmode,
 408                         __maybe_unused enum qm_dqrr_pmode pmode,
 409                         enum qm_dqrr_cmode cmode, u8 max_fill)
 410 {
 411         register struct qm_dqrr *dqrr = &portal->dqrr;
 412         u32 cfg;
 413
 414         /* Make sure the DQRR will be idle when we enable */
 415         qm_out(DQRR_SDQCR, 0);
 416         qm_out(DQRR_VDQCR, 0);
 417         qm_out(DQRR_PDQCR, 0);
 418         dqrr->ring = portal->addr.ce + QM_CL_DQRR;
 419         dqrr->pi = qm_in(DQRR_PI_CINH) & (QM_DQRR_SIZE - 1);
 420         dqrr->ci = qm_in(DQRR_CI_CINH) & (QM_DQRR_SIZE - 1);
 421         dqrr->cursor = dqrr->ring + dqrr->ci;
 422         dqrr->fill = qm_cyc_diff(QM_DQRR_SIZE, dqrr->ci, dqrr->pi);
 423         dqrr->vbit = (qm_in(DQRR_PI_CINH) & QM_DQRR_SIZE) ?
 424                         QM_DQRR_VERB_VBIT : 0;
 425         dqrr->ithresh = qm_in(DQRR_ITR);
 426 #ifdef RTE_LIBRTE_DPAA_HWDEBUG
 427         dqrr->dmode = dmode;
 428         dqrr->pmode = pmode;
 429         dqrr->cmode = cmode;
 430 #endif
 431         /* Invalidate every ring entry before beginning */
 432         for (cfg = 0; cfg < QM_DQRR_SIZE; cfg++)
 433                 dccivac(qm_cl(dqrr->ring, cfg));
 434         cfg = (qm_in(CFG) & 0xff000f00) |
 435                 ((max_fill & (QM_DQRR_SIZE - 1)) << 20) | /* DQRR_MF */
 436                 ((dmode & 1) << 18) |                   /* DP */
 437                 ((cmode & 3) << 16) |                   /* DCM */
 438                 0xa0 |                                  /* RE+SE */
 439                 (0 ? 0x40 : 0) |                        /* Ignore RP */
 440                 (0 ? 0x10 : 0);                         /* Ignore SP */
 441         qm_out(CFG, cfg);
 442         qm_dqrr_set_maxfill(portal, max_fill);
 443         return 0;
 444 }
 445
 446 static inline void qm_dqrr_finish(struct qm_portal *portal)
 447 {
 448         __maybe_unused register struct qm_dqrr *dqrr = &portal->dqrr;
 449 #ifdef RTE_LIBRTE_DPAA_HWDEBUG
 450         if ((dqrr->cmode != qm_dqrr_cdc) &&
 451             (dqrr->ci != DQRR_PTR2IDX(dqrr->cursor)))
 452                 pr_crit("Ignoring completed DQRR entries\n");
 453 #endif
 454 }
 455
 456 static inline int qm_mr_init(struct qm_portal *portal,
 457                              __maybe_unused enum qm_mr_pmode pmode,
 458                              enum qm_mr_cmode cmode)
 459 {
 460         register struct qm_mr *mr = &portal->mr;
 461         u32 cfg;
 462
 463         mr->ring = portal->addr.ce + QM_CL_MR;
 464         mr->pi = qm_in(MR_PI_CINH) & (QM_MR_SIZE - 1);
 465         mr->ci = qm_in(MR_CI_CINH) & (QM_MR_SIZE - 1);
 466         mr->cursor = mr->ring + mr->ci;
 467         mr->fill = qm_cyc_diff(QM_MR_SIZE, mr->ci, mr->pi);
 468         mr->vbit = (qm_in(MR_PI_CINH) & QM_MR_SIZE) ? QM_MR_VERB_VBIT : 0;
 469         mr->ithresh = qm_in(MR_ITR);
 470 #ifdef RTE_LIBRTE_DPAA_HWDEBUG
 471         mr->pmode = pmode;
 472         mr->cmode = cmode;
 473 #endif
 474         cfg = (qm_in(CFG) & 0xfffff0ff) |
 475                 ((cmode & 1) << 8);             /* QCSP_CFG:MM */
 476         qm_out(CFG, cfg);
 477         return 0;
 478 }
 479
 480 static inline void qm_mr_pvb_update(struct qm_portal *portal)
 481 {
 482         register struct qm_mr *mr = &portal->mr;
 483         const struct qm_mr_entry *res = qm_cl(mr->ring, mr->pi);
 484
 485 #ifdef RTE_LIBRTE_DPAA_HWDEBUG
 486         DPAA_ASSERT(mr->pmode == qm_mr_pvb);
 487 #endif
 488         /* when accessing 'verb', use __raw_readb() to ensure that compiler
 489          * inlining doesn't try to optimise out "excess reads".
 490          */
 491         if ((__raw_readb(&res->ern.verb) & QM_MR_VERB_VBIT) == mr->vbit) {
 492                 mr->pi = (mr->pi + 1) & (QM_MR_SIZE - 1);
 493                 if (!mr->pi)
 494                         mr->vbit ^= QM_MR_VERB_VBIT;
 495                 mr->fill++;
 496                 res = MR_INC(res);
 497         }
 498         dcbit_ro(res);
 499 }
 500
 501 struct qman_portal *
 502 qman_init_portal(struct qman_portal *portal,
 503                    const struct qm_portal_config *c,
 504                    const struct qman_cgrs *cgrs)
 505 {
 506         struct qm_portal *p;
 507         char buf[16];
 508         int ret;
 509         u32 isdr;
 510
 511         p = &portal->p;
 512
 513         if (!c)
 514                 c = portal->config;
 515
 516         if (dpaa_svr_family == SVR_LS1043A_FAMILY)
 517                 portal->use_eqcr_ci_stashing = 3;
 518         else
 519                 portal->use_eqcr_ci_stashing =
 520                                         ((qman_ip_rev >= QMAN_REV30) ? 1 : 0);
 521
 522         /*
 523          * prep the low-level portal struct with the mapped addresses from the
 524          * config, everything that follows depends on it and "config" is more
 525          * for (de)reference
 526          */
 527         p->addr.ce = c->addr_virt[DPAA_PORTAL_CE];
 528         p->addr.ci = c->addr_virt[DPAA_PORTAL_CI];
 529         /*
 530          * If CI-stashing is used, the current defaults use a threshold of 3,
 531          * and stash with high-than-DQRR priority.
 532          */
 533         if (qm_eqcr_init(p, qm_eqcr_pvb,
 534                          portal->use_eqcr_ci_stashing, 1)) {
 535                 pr_err("Qman EQCR initialisation failed\n");
 536                 goto fail_eqcr;
 537         }
 538         if (qm_dqrr_init(p, c, qm_dqrr_dpush, qm_dqrr_pvb,
 539                          qm_dqrr_cdc, DQRR_MAXFILL)) {
 540                 pr_err("Qman DQRR initialisation failed\n");
 541                 goto fail_dqrr;
 542         }
 543         if (qm_mr_init(p, qm_mr_pvb, qm_mr_cci)) {
 544                 pr_err("Qman MR initialisation failed\n");
 545                 goto fail_mr;
 546         }
 547         if (qm_mc_init(p)) {
 548                 pr_err("Qman MC initialisation failed\n");
 549                 goto fail_mc;
 550         }
 551
 552         /* static interrupt-gating controls */
 553         qm_dqrr_set_ithresh(p, 0);
 554         qm_mr_set_ithresh(p, 0);
 555         qm_isr_set_iperiod(p, 0);
 556         portal->cgrs = kmalloc(2 * sizeof(*cgrs), GFP_KERNEL);
 557         if (!portal->cgrs)
 558                 goto fail_cgrs;
 559         /* initial snapshot is no-depletion */
 560         qman_cgrs_init(&portal->cgrs[1]);
 561         if (cgrs)
 562                 portal->cgrs[0] = *cgrs;
 563         else
 564                 /* if the given mask is NULL, assume all CGRs can be seen */
 565                 qman_cgrs_fill(&portal->cgrs[0]);
 566         INIT_LIST_HEAD(&portal->cgr_cbs);
 567         spin_lock_init(&portal->cgr_lock);
 568         portal->bits = 0;
 569         portal->slowpoll = 0;
 570         portal->sdqcr = QM_SDQCR_SOURCE_CHANNELS | QM_SDQCR_COUNT_UPTO3 |
 571                         QM_SDQCR_DEDICATED_PRECEDENCE | QM_SDQCR_TYPE_PRIO_QOS |
 572                         QM_SDQCR_TOKEN_SET(0xab) | QM_SDQCR_CHANNELS_DEDICATED;
 573         portal->dqrr_disable_ref = 0;
 574         portal->cb_dc_ern = NULL;
 575         sprintf(buf, "qportal-%d", c->channel);
 576         dpa_rbtree_init(&portal->retire_table);
 577         isdr = 0xffffffff;
 578         qm_isr_disable_write(p, isdr);
 579         portal->irq_sources = 0;
 580         qm_isr_enable_write(p, portal->irq_sources);
 581         qm_isr_status_clear(p, 0xffffffff);
 582         snprintf(portal->irqname, MAX_IRQNAME, IRQNAME, c->cpu);
 583         if (request_irq(c->irq, portal_isr, 0, portal->irqname,
 584                         portal)) {
 585                 pr_err("request_irq() failed\n");
 586                 goto fail_irq;
 587         }
 588
 589         /* Need EQCR to be empty before continuing */
 590         isdr &= ~QM_PIRQ_EQCI;
 591         qm_isr_disable_write(p, isdr);
 592         ret = qm_eqcr_get_fill(p);
 593         if (ret) {
 594                 pr_err("Qman EQCR unclean\n");
 595                 goto fail_eqcr_empty;
 596         }
 597         isdr &= ~(QM_PIRQ_DQRI | QM_PIRQ_MRI);
 598         qm_isr_disable_write(p, isdr);
 599         if (qm_dqrr_current(p)) {
 600                 pr_err("Qman DQRR unclean\n");
 601                 qm_dqrr_cdc_consume_n(p, 0xffff);
 602         }
 603         if (qm_mr_current(p) && drain_mr_fqrni(p)) {
 604                 /* special handling, drain just in case it's a few FQRNIs */
 605                 if (drain_mr_fqrni(p))
 606                         goto fail_dqrr_mr_empty;
 607         }
 608         /* Success */
 609         portal->config = c;
 610         qm_isr_disable_write(p, 0);
 611         qm_isr_uninhibit(p);
 612         /* Write a sane SDQCR */
 613         qm_dqrr_sdqcr_set(p, portal->sdqcr);
 614         return portal;
 615 fail_dqrr_mr_empty:
 616 fail_eqcr_empty:
 617         free_irq(c->irq, portal);
 618 fail_irq:
 619         kfree(portal->cgrs);
 620         spin_lock_destroy(&portal->cgr_lock);
 621 fail_cgrs:
 622         qm_mc_finish(p);
 623 fail_mc:
 624         qm_mr_finish(p);
 625 fail_mr:
 626         qm_dqrr_finish(p);
 627 fail_dqrr:
 628         qm_eqcr_finish(p);
 629 fail_eqcr:
 630         return NULL;
 631 }
 632
 633 #define MAX_GLOBAL_PORTALS 8
 634 static struct qman_portal global_portals[MAX_GLOBAL_PORTALS];
 635 static rte_atomic16_t global_portals_used[MAX_GLOBAL_PORTALS];
 636
 637 struct qman_portal *
 638 qman_alloc_global_portal(struct qm_portal_config *q_pcfg)
 639 {
 640         unsigned int i;
 641
 642         for (i = 0; i < MAX_GLOBAL_PORTALS; i++) {
 643                 if (rte_atomic16_test_and_set(&global_portals_used[i])) {
 644                         global_portals[i].config = q_pcfg;
 645                         return &global_portals[i];
 646                 }
 647         }
 648         pr_err("No portal available (%x)\n", MAX_GLOBAL_PORTALS);
 649
 650         return NULL;
 651 }
 652
 653 int
 654 qman_free_global_portal(struct qman_portal *portal)
 655 {
 656         unsigned int i;
 657
 658         for (i = 0; i < MAX_GLOBAL_PORTALS; i++) {
 659                 if (&global_portals[i] == portal) {
 660                         rte_atomic16_clear(&global_portals_used[i]);
 661                         return 0;
 662                 }
 663         }
 664         return -1;
 665 }
 666
 667 struct qman_portal *qman_create_affine_portal(const struct qm_portal_config *c,
 668                                               const struct qman_cgrs *cgrs)
 669 {
 670         struct qman_portal *res;
 671         struct qman_portal *portal = get_affine_portal();
 672
 673         /* A criteria for calling this function (from qman_driver.c) is that
 674          * we're already affine to the cpu and won't schedule onto another cpu.
 675          */
 676         res = qman_init_portal(portal, c, cgrs);
 677         if (res) {
 678                 spin_lock(&affine_mask_lock);
 679                 CPU_SET(c->cpu, &affine_mask);
 680                 affine_channels[c->cpu] =
 681                         c->channel;
 682                 spin_unlock(&affine_mask_lock);
 683         }
 684         return res;
 685 }
 686
 687 static inline
 688 void qman_destroy_portal(struct qman_portal *qm)
 689 {
 690         const struct qm_portal_config *pcfg;
 691
 692         /* Stop dequeues on the portal */
 693         qm_dqrr_sdqcr_set(&qm->p, 0);
 694
 695         /*
 696          * NB we do this to "quiesce" EQCR. If we add enqueue-completions or
 697          * something related to QM_PIRQ_EQCI, this may need fixing.
 698          * Also, due to the prefetching model used for CI updates in the enqueue
 699          * path, this update will only invalidate the CI cacheline *after*
 700          * working on it, so we need to call this twice to ensure a full update
 701          * irrespective of where the enqueue processing was at when the teardown
 702          * began.
 703          */
 704         qm_eqcr_cce_update(&qm->p);
 705         qm_eqcr_cce_update(&qm->p);
 706         pcfg = qm->config;
 707
 708         free_irq(pcfg->irq, qm);
 709
 710         kfree(qm->cgrs);
 711         qm_mc_finish(&qm->p);
 712         qm_mr_finish(&qm->p);
 713         qm_dqrr_finish(&qm->p);
 714         qm_eqcr_finish(&qm->p);
 715
 716         qm->config = NULL;
 717
 718         spin_lock_destroy(&qm->cgr_lock);
 719 }
 720
 721 const struct qm_portal_config *
 722 qman_destroy_affine_portal(struct qman_portal *qp)
 723 {
 724         /* We don't want to redirect if we're a slave, use "raw" */
 725         struct qman_portal *qm;
 726         const struct qm_portal_config *pcfg;
 727         int cpu;
 728
 729         if (qp == NULL)
 730                 qm = get_affine_portal();
 731         else
 732                 qm = qp;
 733         pcfg = qm->config;
 734         cpu = pcfg->cpu;
 735
 736         qman_destroy_portal(qm);
 737
 738         spin_lock(&affine_mask_lock);
 739         CPU_CLR(cpu, &affine_mask);
 740         spin_unlock(&affine_mask_lock);
 741
 742         qman_free_global_portal(qm);
 743
 744         return pcfg;
 745 }
 746
 747 int qman_get_portal_index(void)
 748 {
 749         struct qman_portal *p = get_affine_portal();
 750         return p->config->index;
 751 }
 752
 753 /* Inline helper to reduce nesting in __poll_portal_slow() */
 754 static inline void fq_state_change(struct qman_portal *p, struct qman_fq *fq,
 755                                    const struct qm_mr_entry *msg, u8 verb)
 756 {
 757         FQLOCK(fq);
 758         switch (verb) {
 759         case QM_MR_VERB_FQRL:
 760                 DPAA_ASSERT(fq_isset(fq, QMAN_FQ_STATE_ORL));
 761                 fq_clear(fq, QMAN_FQ_STATE_ORL);
 762                 table_del_fq(p, fq);
 763                 break;
 764         case QM_MR_VERB_FQRN:
 765                 DPAA_ASSERT((fq->state == qman_fq_state_parked) ||
 766                             (fq->state == qman_fq_state_sched));
 767                 DPAA_ASSERT(fq_isset(fq, QMAN_FQ_STATE_CHANGING));
 768                 fq_clear(fq, QMAN_FQ_STATE_CHANGING);
 769                 if (msg->fq.fqs & QM_MR_FQS_NOTEMPTY)
 770                         fq_set(fq, QMAN_FQ_STATE_NE);
 771                 if (msg->fq.fqs & QM_MR_FQS_ORLPRESENT)
 772                         fq_set(fq, QMAN_FQ_STATE_ORL);
 773                 else
 774                         table_del_fq(p, fq);
 775                 fq->state = qman_fq_state_retired;
 776                 break;
 777         case QM_MR_VERB_FQPN:
 778                 DPAA_ASSERT(fq->state == qman_fq_state_sched);
 779                 DPAA_ASSERT(fq_isclear(fq, QMAN_FQ_STATE_CHANGING));
 780                 fq->state = qman_fq_state_parked;
 781         }
 782         FQUNLOCK(fq);
 783 }
 784
 785 static u32 __poll_portal_slow(struct qman_portal *p, u32 is)
 786 {
 787         const struct qm_mr_entry *msg;
 788         struct qm_mr_entry swapped_msg;
 789
 790         if (is & QM_PIRQ_CSCI) {
 791                 struct qman_cgrs rr, c;
 792                 struct qm_mc_result *mcr;
 793                 struct qman_cgr *cgr;
 794
 795                 spin_lock(&p->cgr_lock);
 796                 /*
 797                  * The CSCI bit must be cleared _before_ issuing the
 798                  * Query Congestion State command, to ensure that a long
 799                  * CGR State Change callback cannot miss an intervening
 800                  * state change.
 801                  */
 802                 qm_isr_status_clear(&p->p, QM_PIRQ_CSCI);
 803                 qm_mc_start(&p->p);
 804                 qm_mc_commit(&p->p, QM_MCC_VERB_QUERYCONGESTION);
 805                 while (!(mcr = qm_mc_result(&p->p)))
 806                         cpu_relax();
 807                 /* mask out the ones I'm not interested in */
 808                 qman_cgrs_and(&rr, (const struct qman_cgrs *)
 809                         &mcr->querycongestion.state, &p->cgrs[0]);
 810                 /* check previous snapshot for delta, enter/exit congestion */
 811                 qman_cgrs_xor(&c, &rr, &p->cgrs[1]);
 812                 /* update snapshot */
 813                 qman_cgrs_cp(&p->cgrs[1], &rr);
 814                 /* Invoke callback */
 815                 list_for_each_entry(cgr, &p->cgr_cbs, node)
 816                         if (cgr->cb && qman_cgrs_get(&c, cgr->cgrid))
 817                                 cgr->cb(p, cgr, qman_cgrs_get(&rr, cgr->cgrid));
 818                 spin_unlock(&p->cgr_lock);
 819         }
 820
 821         if (is & QM_PIRQ_EQRI) {
 822                 qm_eqcr_cce_update(&p->p);
 823                 qm_eqcr_set_ithresh(&p->p, 0);
 824                 wake_up(&affine_queue);
 825         }
 826
 827         if (is & QM_PIRQ_MRI) {
 828                 struct qman_fq *fq;
 829                 u8 verb, num = 0;
 830 mr_loop:
 831                 qm_mr_pvb_update(&p->p);
 832                 msg = qm_mr_current(&p->p);
 833                 if (!msg)
 834                         goto mr_done;
 835                 swapped_msg = *msg;
 836                 hw_fd_to_cpu(&swapped_msg.ern.fd);
 837                 verb = msg->ern.verb & QM_MR_VERB_TYPE_MASK;
 838                 /* The message is a software ERN iff the 0x20 bit is set */
 839                 if (verb & 0x20) {
 840                         switch (verb) {
 841                         case QM_MR_VERB_FQRNI:
 842                                 /* nada, we drop FQRNIs on the floor */
 843                                 break;
 844                         case QM_MR_VERB_FQRN:
 845                         case QM_MR_VERB_FQRL:
 846                                 /* Lookup in the retirement table */
 847                                 fq = table_find_fq(p,
 848                                                    be32_to_cpu(msg->fq.fqid));
 849                                 DPAA_BUG_ON(!fq);
 850                                 fq_state_change(p, fq, &swapped_msg, verb);
 851                                 if (fq->cb.fqs)
 852                                         fq->cb.fqs(p, fq, &swapped_msg);
 853                                 break;
 854                         case QM_MR_VERB_FQPN:
 855                                 /* Parked */
 856 #ifdef CONFIG_FSL_QMAN_FQ_LOOKUP
 857                                 fq = get_fq_table_entry(msg->fq.contextB);
 858 #else
 859                                 fq = (void *)(uintptr_t)msg->fq.contextB;
 860 #endif
 861                                 fq_state_change(p, fq, msg, verb);
 862                                 if (fq->cb.fqs)
 863                                         fq->cb.fqs(p, fq, &swapped_msg);
 864                                 break;
 865                         case QM_MR_VERB_DC_ERN:
 866                                 /* DCP ERN */
 867                                 if (p->cb_dc_ern)
 868                                         p->cb_dc_ern(p, msg);
 869                                 else if (cb_dc_ern)
 870                                         cb_dc_ern(p, msg);
 871                                 else {
 872                                         static int warn_once;
 873
 874                                         if (!warn_once) {
 875                                                 pr_crit("Leaking DCP ERNs!\n");
 876                                                 warn_once = 1;
 877                                         }
 878                                 }
 879                                 break;
 880                         default:
 881                                 pr_crit("Invalid MR verb 0x%02x\n", verb);
 882                         }
 883                 } else {
 884                         /* Its a software ERN */
 885 #ifdef CONFIG_FSL_QMAN_FQ_LOOKUP
 886                         fq = get_fq_table_entry(be32_to_cpu(msg->ern.tag));
 887 #else
 888                         fq = (void *)(uintptr_t)be32_to_cpu(msg->ern.tag);
 889 #endif
 890                         fq->cb.ern(p, fq, &swapped_msg);
 891                 }
 892                 num++;
 893                 qm_mr_next(&p->p);
 894                 goto mr_loop;
 895 mr_done:
 896                 qm_mr_cci_consume(&p->p, num);
 897         }
 898         /*
 899          * QM_PIRQ_CSCI/CCSCI has already been cleared, as part of its specific
 900          * processing. If that interrupt source has meanwhile been re-asserted,
 901          * we mustn't clear it here (or in the top-level interrupt handler).
 902          */
 903         return is & (QM_PIRQ_EQCI | QM_PIRQ_EQRI | QM_PIRQ_MRI);
 904 }
 905
 906 /*
 907  * remove some slowish-path stuff from the "fast path" and make sure it isn't
 908  * inlined.
 909  */
 910 static noinline void clear_vdqcr(struct qman_portal *p, struct qman_fq *fq)
 911 {
 912         p->vdqcr_owned = NULL;
 913         FQLOCK(fq);
 914         fq_clear(fq, QMAN_FQ_STATE_VDQCR);
 915         FQUNLOCK(fq);
 916         wake_up(&affine_queue);
 917 }
 918
 919 /*
 920  * The only states that would conflict with other things if they ran at the
 921  * same time on the same cpu are:
 922  *
 923  *   (i) setting/clearing vdqcr_owned, and
 924  *  (ii) clearing the NE (Not Empty) flag.
 925  *
 926  * Both are safe. Because;
 927  *
 928  *   (i) this clearing can only occur after qman_set_vdq() has set the
 929  *       vdqcr_owned field (which it does before setting VDQCR), and
 930  *       qman_volatile_dequeue() blocks interrupts and preemption while this is
 931  *       done so that we can't interfere.
 932  *  (ii) the NE flag is only cleared after qman_retire_fq() has set it, and as
 933  *       with (i) that API prevents us from interfering until it's safe.
 934  *
 935  * The good thing is that qman_set_vdq() and qman_retire_fq() run far
 936  * less frequently (ie. per-FQ) than __poll_portal_fast() does, so the nett
 937  * advantage comes from this function not having to "lock" anything at all.
 938  *
 939  * Note also that the callbacks are invoked at points which are safe against the
 940  * above potential conflicts, but that this function itself is not re-entrant
 941  * (this is because the function tracks one end of each FIFO in the portal and
 942  * we do *not* want to lock that). So the consequence is that it is safe for
 943  * user callbacks to call into any QMan API.
 944  */
 945 static inline unsigned int __poll_portal_fast(struct qman_portal *p,
 946                                               unsigned int poll_limit)
 947 {
 948         const struct qm_dqrr_entry *dq;
 949         struct qman_fq *fq;
 950         enum qman_cb_dqrr_result res;
 951         unsigned int limit = 0;
 952 #if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__
 953         struct qm_dqrr_entry *shadow;
 954 #endif
 955         do {
 956                 qm_dqrr_pvb_update(&p->p);
 957                 dq = qm_dqrr_current(&p->p);
 958                 if (unlikely(!dq))
 959                         break;
 960 #if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__
 961         /* If running on an LE system the fields of the
 962          * dequeue entry must be swapper.  Because the
 963          * QMan HW will ignore writes the DQRR entry is
 964          * copied and the index stored within the copy
 965          */
 966                 shadow = &p->shadow_dqrr[DQRR_PTR2IDX(dq)];
 967                 *shadow = *dq;
 968                 dq = shadow;
 969                 shadow->fqid = be32_to_cpu(shadow->fqid);
 970                 shadow->seqnum = be16_to_cpu(shadow->seqnum);
 971                 hw_fd_to_cpu(&shadow->fd);
 972 #endif
 973
 974                 if (dq->stat & QM_DQRR_STAT_UNSCHEDULED) {
 975                         /*
 976                          * VDQCR: don't trust context_b as the FQ may have
 977                          * been configured for h/w consumption and we're
 978                          * draining it post-retirement.
 979                          */
 980                         fq = p->vdqcr_owned;
 981                         /*
 982                          * We only set QMAN_FQ_STATE_NE when retiring, so we
 983                          * only need to check for clearing it when doing
 984                          * volatile dequeues.  It's one less thing to check
 985                          * in the critical path (SDQCR).
 986                          */
 987                         if (dq->stat & QM_DQRR_STAT_FQ_EMPTY)
 988                                 fq_clear(fq, QMAN_FQ_STATE_NE);
 989                         /*
 990                          * This is duplicated from the SDQCR code, but we
 991                          * have stuff to do before *and* after this callback,
 992                          * and we don't want multiple if()s in the critical
 993                          * path (SDQCR).
 994                          */
 995                         res = fq->cb.dqrr(p, fq, dq);
 996                         if (res == qman_cb_dqrr_stop)
 997                                 break;
 998                         /* Check for VDQCR completion */
 999                         if (dq->stat & QM_DQRR_STAT_DQCR_EXPIRED)
1000                                 clear_vdqcr(p, fq);
1001                 } else {
1002                         /* SDQCR: context_b points to the FQ */
1003 #ifdef CONFIG_FSL_QMAN_FQ_LOOKUP
1004                         fq = get_fq_table_entry(dq->contextB);
1005 #else
1006                         fq = (void *)(uintptr_t)dq->contextB;
1007 #endif
1008                         /* Now let the callback do its stuff */
1009                         res = fq->cb.dqrr(p, fq, dq);
1010                         /*
1011                          * The callback can request that we exit without
1012                          * consuming this entry nor advancing;
1013                          */
1014                         if (res == qman_cb_dqrr_stop)
1015                                 break;
1016                 }
1017                 /* Interpret 'dq' from a driver perspective. */
1018                 /*
1019                  * Parking isn't possible unless HELDACTIVE was set. NB,
1020                  * FORCEELIGIBLE implies HELDACTIVE, so we only need to
1021                  * check for HELDACTIVE to cover both.
1022                  */
1023                 DPAA_ASSERT((dq->stat & QM_DQRR_STAT_FQ_HELDACTIVE) ||
1024                             (res != qman_cb_dqrr_park));
1025                 /* just means "skip it, I'll consume it myself later on" */
1026                 if (res != qman_cb_dqrr_defer)
1027                         qm_dqrr_cdc_consume_1ptr(&p->p, dq,
1028                                                  res == qman_cb_dqrr_park);
1029                 /* Move forward */
1030                 qm_dqrr_next(&p->p);
1031                 /*
1032                  * Entry processed and consumed, increment our counter.  The
1033                  * callback can request that we exit after consuming the
1034                  * entry, and we also exit if we reach our processing limit,
1035                  * so loop back only if neither of these conditions is met.
1036                  */
1037         } while (++limit < poll_limit && res != qman_cb_dqrr_consume_stop);
1038
1039         return limit;
1040 }
1041
1042 int qman_irqsource_add(u32 bits)
1043 {
1044         struct qman_portal *p = get_affine_portal();
1045
1046         bits = bits & QM_PIRQ_VISIBLE;
1047
1048         /* Clear any previously remaining interrupt conditions in
1049          * QCSP_ISR. This prevents raising a false interrupt when
1050          * interrupt conditions are enabled in QCSP_IER.
1051          */
1052         qm_isr_status_clear(&p->p, bits);
1053         dpaa_set_bits(bits, &p->irq_sources);
1054         qm_isr_enable_write(&p->p, p->irq_sources);
1055
1056
1057         return 0;
1058 }
1059
1060 int qman_irqsource_remove(u32 bits)
1061 {
1062         struct qman_portal *p = get_affine_portal();
1063         u32 ier;
1064
1065         /* Our interrupt handler only processes+clears status register bits that
1066          * are in p->irq_sources. As we're trimming that mask, if one of them
1067          * were to assert in the status register just before we remove it from
1068          * the enable register, there would be an interrupt-storm when we
1069          * release the IRQ lock. So we wait for the enable register update to
1070          * take effect in h/w (by reading it back) and then clear all other bits
1071          * in the status register. Ie. we clear them from ISR once it's certain
1072          * IER won't allow them to reassert.
1073          */
1074
1075         bits &= QM_PIRQ_VISIBLE;
1076         dpaa_clear_bits(bits, &p->irq_sources);
1077         qm_isr_enable_write(&p->p, p->irq_sources);
1078         ier = qm_isr_enable_read(&p->p);
1079         /* Using "~ier" (rather than "bits" or "~p->irq_sources") creates a
1080          * data-dependency, ie. to protect against re-ordering.
1081          */
1082         qm_isr_status_clear(&p->p, ~ier);
1083         return 0;
1084 }
1085
1086 u16 qman_affine_channel(int cpu)
1087 {
1088         if (cpu < 0) {
1089                 struct qman_portal *portal = get_affine_portal();
1090
1091                 cpu = portal->config->cpu;
1092         }
1093         DPAA_BUG_ON(!CPU_ISSET(cpu, &affine_mask));
1094         return affine_channels[cpu];
1095 }
1096
1097 unsigned int qman_portal_poll_rx(unsigned int poll_limit,
1098                                  void **bufs,
1099                                  struct qman_portal *p)
1100 {
1101         struct qm_portal *portal = &p->p;
1102         register struct qm_dqrr *dqrr = &portal->dqrr;
1103         struct qm_dqrr_entry *dq[QM_DQRR_SIZE], *shadow[QM_DQRR_SIZE];
1104         struct qman_fq *fq;
1105         unsigned int limit = 0, rx_number = 0;
1106         uint32_t consume = 0;
1107
1108         do {
1109                 qm_dqrr_pvb_update(&p->p);
1110                 if (!dqrr->fill)
1111                         break;
1112
1113                 dq[rx_number] = dqrr->cursor;
1114                 dqrr->cursor = DQRR_CARRYCLEAR(dqrr->cursor + 1);
1115                 /* Prefetch the next DQRR entry */
1116                 rte_prefetch0(dqrr->cursor);
1117
1118 #if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__
1119                 /* If running on an LE system the fields of the
1120                  * dequeue entry must be swapper.  Because the
1121                  * QMan HW will ignore writes the DQRR entry is
1122                  * copied and the index stored within the copy
1123                  */
1124                 shadow[rx_number] =
1125                         &p->shadow_dqrr[DQRR_PTR2IDX(dq[rx_number])];
1126                 shadow[rx_number]->fd.opaque_addr =
1127                         dq[rx_number]->fd.opaque_addr;
1128                 shadow[rx_number]->fd.addr =
1129                         be40_to_cpu(dq[rx_number]->fd.addr);
1130                 shadow[rx_number]->fd.opaque =
1131                         be32_to_cpu(dq[rx_number]->fd.opaque);
1132 #else
1133                 shadow[rx_number] = dq[rx_number];
1134 #endif
1135
1136                 /* SDQCR: context_b points to the FQ */
1137 #ifdef CONFIG_FSL_QMAN_FQ_LOOKUP
1138                 fq = qman_fq_lookup_table[dq[rx_number]->contextB];
1139 #else
1140                 fq = (void *)dq[rx_number]->contextB;
1141 #endif
1142                 if (fq->cb.dqrr_prepare)
1143                         fq->cb.dqrr_prepare(shadow[rx_number],
1144                                             &bufs[rx_number]);
1145
1146                 consume |= (1 << (31 - DQRR_PTR2IDX(shadow[rx_number])));
1147                 rx_number++;
1148                 --dqrr->fill;
1149         } while (++limit < poll_limit);
1150
1151         if (rx_number)
1152                 fq->cb.dqrr_dpdk_pull_cb(&fq, shadow, bufs, rx_number);
1153
1154         /* Consume all the DQRR enries together */
1155         qm_out(DQRR_DCAP, (1 << 8) | consume);
1156
1157         return rx_number;
1158 }
1159
1160 void qman_clear_irq(void)
1161 {
1162         struct qman_portal *p = get_affine_portal();
1163         u32 clear = QM_DQAVAIL_MASK | (p->irq_sources &
1164                 ~(QM_PIRQ_CSCI | QM_PIRQ_CCSCI));
1165         qm_isr_status_clear(&p->p, clear);
1166 }
1167
1168 u32 qman_portal_dequeue(struct rte_event ev[], unsigned int poll_limit,
1169                         void **bufs)
1170 {
1171         const struct qm_dqrr_entry *dq;
1172         struct qman_fq *fq;
1173         enum qman_cb_dqrr_result res;
1174         unsigned int limit = 0;
1175         struct qman_portal *p = get_affine_portal();
1176 #if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN
1177         struct qm_dqrr_entry *shadow;
1178 #endif
1179         unsigned int rx_number = 0;
1180
1181         do {
1182                 qm_dqrr_pvb_update(&p->p);
1183                 dq = qm_dqrr_current(&p->p);
1184                 if (!dq)
1185                         break;
1186 #if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN
1187                 /*
1188                  * If running on an LE system the fields of the
1189                  * dequeue entry must be swapper.  Because the
1190                  * QMan HW will ignore writes the DQRR entry is
1191                  * copied and the index stored within the copy
1192                  */
1193                 shadow = &p->shadow_dqrr[DQRR_PTR2IDX(dq)];
1194                 *shadow = *dq;
1195                 dq = shadow;
1196                 shadow->fqid = be32_to_cpu(shadow->fqid);
1197                 shadow->seqnum = be16_to_cpu(shadow->seqnum);
1198                 hw_fd_to_cpu(&shadow->fd);
1199 #endif
1200
1201                /* SDQCR: context_b points to the FQ */
1202 #ifdef CONFIG_FSL_QMAN_FQ_LOOKUP
1203                 fq = get_fq_table_entry(dq->contextB);
1204 #else
1205                 fq = (void *)(uintptr_t)dq->contextB;
1206 #endif
1207                 /* Now let the callback do its stuff */
1208                 res = fq->cb.dqrr_dpdk_cb(&ev[rx_number], p, fq,
1209                                          dq, &bufs[rx_number]);
1210                 rx_number++;
1211                 /* Interpret 'dq' from a driver perspective. */
1212                 /*
1213                  * Parking isn't possible unless HELDACTIVE was set. NB,
1214                  * FORCEELIGIBLE implies HELDACTIVE, so we only need to
1215                  * check for HELDACTIVE to cover both.
1216                  */
1217                 DPAA_ASSERT((dq->stat & QM_DQRR_STAT_FQ_HELDACTIVE) ||
1218                             (res != qman_cb_dqrr_park));
1219                 if (res != qman_cb_dqrr_defer)
1220                         qm_dqrr_cdc_consume_1ptr(&p->p, dq,
1221                                                  res == qman_cb_dqrr_park);
1222                 /* Move forward */
1223                 qm_dqrr_next(&p->p);
1224                 /*
1225                  * Entry processed and consumed, increment our counter.  The
1226                  * callback can request that we exit after consuming the
1227                  * entry, and we also exit if we reach our processing limit,
1228                  * so loop back only if neither of these conditions is met.
1229                  */
1230         } while (++limit < poll_limit);
1231
1232         return limit;
1233 }
1234
1235 struct qm_dqrr_entry *qman_dequeue(struct qman_fq *fq)
1236 {
1237         struct qman_portal *p = get_affine_portal();
1238         const struct qm_dqrr_entry *dq;
1239 #if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__
1240         struct qm_dqrr_entry *shadow;
1241 #endif
1242
1243         qm_dqrr_pvb_update(&p->p);
1244         dq = qm_dqrr_current(&p->p);
1245         if (!dq)
1246                 return NULL;
1247
1248         if (!(dq->stat & QM_DQRR_STAT_FD_VALID)) {
1249                 /* Invalid DQRR - put the portal and consume the DQRR.
1250                  * Return NULL to user as no packet is seen.
1251                  */
1252                 qman_dqrr_consume(fq, (struct qm_dqrr_entry *)dq);
1253                 return NULL;
1254         }
1255
1256 #if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__
1257         shadow = &p->shadow_dqrr[DQRR_PTR2IDX(dq)];
1258         *shadow = *dq;
1259         dq = shadow;
1260         shadow->fqid = be32_to_cpu(shadow->fqid);
1261         shadow->seqnum = be16_to_cpu(shadow->seqnum);
1262         hw_fd_to_cpu(&shadow->fd);
1263 #endif
1264
1265         if (dq->stat & QM_DQRR_STAT_FQ_EMPTY)
1266                 fq_clear(fq, QMAN_FQ_STATE_NE);
1267
1268         return (struct qm_dqrr_entry *)dq;
1269 }
1270
1271 void qman_dqrr_consume(struct qman_fq *fq,
1272                        struct qm_dqrr_entry *dq)
1273 {
1274         struct qman_portal *p = get_affine_portal();
1275
1276         if (dq->stat & QM_DQRR_STAT_DQCR_EXPIRED)
1277                 clear_vdqcr(p, fq);
1278
1279         qm_dqrr_cdc_consume_1ptr(&p->p, dq, 0);
1280         qm_dqrr_next(&p->p);
1281 }
1282
1283 int qman_poll_dqrr(unsigned int limit)
1284 {
1285         struct qman_portal *p = get_affine_portal();
1286         int ret;
1287
1288         ret = __poll_portal_fast(p, limit);
1289         return ret;
1290 }
1291
1292 void qman_poll(void)
1293 {
1294         struct qman_portal *p = get_affine_portal();
1295
1296         if ((~p->irq_sources) & QM_PIRQ_SLOW) {
1297                 if (!(p->slowpoll--)) {
1298                         u32 is = qm_isr_status_read(&p->p) & ~p->irq_sources;
1299                         u32 active = __poll_portal_slow(p, is);
1300
1301                         if (active) {
1302                                 qm_isr_status_clear(&p->p, active);
1303                                 p->slowpoll = SLOW_POLL_BUSY;
1304                         } else
1305                                 p->slowpoll = SLOW_POLL_IDLE;
1306                 }
1307         }
1308         if ((~p->irq_sources) & QM_PIRQ_DQRI)
1309                 __poll_portal_fast(p, FSL_QMAN_POLL_LIMIT);
1310 }
1311
1312 void qman_stop_dequeues(void)
1313 {
1314         struct qman_portal *p = get_affine_portal();
1315
1316         qman_stop_dequeues_ex(p);
1317 }
1318
1319 void qman_start_dequeues(void)
1320 {
1321         struct qman_portal *p = get_affine_portal();
1322
1323         DPAA_ASSERT(p->dqrr_disable_ref > 0);
1324         if (!(--p->dqrr_disable_ref))
1325                 qm_dqrr_set_maxfill(&p->p, DQRR_MAXFILL);
1326 }
1327
1328 void qman_static_dequeue_add(u32 pools, struct qman_portal *qp)
1329 {
1330         struct qman_portal *p = qp ? qp : get_affine_portal();
1331
1332         pools &= p->config->pools;
1333         p->sdqcr |= pools;
1334         qm_dqrr_sdqcr_set(&p->p, p->sdqcr);
1335 }
1336
1337 void qman_static_dequeue_del(u32 pools, struct qman_portal *qp)
1338 {
1339         struct qman_portal *p = qp ? qp : get_affine_portal();
1340
1341         pools &= p->config->pools;
1342         p->sdqcr &= ~pools;
1343         qm_dqrr_sdqcr_set(&p->p, p->sdqcr);
1344 }
1345
1346 u32 qman_static_dequeue_get(struct qman_portal *qp)
1347 {
1348         struct qman_portal *p = qp ? qp : get_affine_portal();
1349         return p->sdqcr;
1350 }
1351
1352 void qman_dca(const struct qm_dqrr_entry *dq, int park_request)
1353 {
1354         struct qman_portal *p = get_affine_portal();
1355
1356         qm_dqrr_cdc_consume_1ptr(&p->p, dq, park_request);
1357 }
1358
1359 void qman_dca_index(u8 index, int park_request)
1360 {
1361         struct qman_portal *p = get_affine_portal();
1362
1363         qm_dqrr_cdc_consume_1(&p->p, index, park_request);
1364 }
1365
1366 /* Frame queue API */
1367 static const char *mcr_result_str(u8 result)
1368 {
1369         switch (result) {
1370         case QM_MCR_RESULT_NULL:
1371                 return "QM_MCR_RESULT_NULL";
1372         case QM_MCR_RESULT_OK:
1373                 return "QM_MCR_RESULT_OK";
1374         case QM_MCR_RESULT_ERR_FQID:
1375                 return "QM_MCR_RESULT_ERR_FQID";
1376         case QM_MCR_RESULT_ERR_FQSTATE:
1377                 return "QM_MCR_RESULT_ERR_FQSTATE";
1378         case QM_MCR_RESULT_ERR_NOTEMPTY:
1379                 return "QM_MCR_RESULT_ERR_NOTEMPTY";
1380         case QM_MCR_RESULT_PENDING:
1381                 return "QM_MCR_RESULT_PENDING";
1382         case QM_MCR_RESULT_ERR_BADCOMMAND:
1383                 return "QM_MCR_RESULT_ERR_BADCOMMAND";
1384         }
1385         return "<unknown MCR result>";
1386 }
1387
1388 int qman_create_fq(u32 fqid, u32 flags, struct qman_fq *fq)
1389 {
1390         struct qm_fqd fqd;
1391         struct qm_mcr_queryfq_np np;
1392         struct qm_mc_command *mcc;
1393         struct qm_mc_result *mcr;
1394         struct qman_portal *p;
1395
1396         if (flags & QMAN_FQ_FLAG_DYNAMIC_FQID) {
1397                 int ret = qman_alloc_fqid(&fqid);
1398
1399                 if (ret)
1400                         return ret;
1401         }
1402         spin_lock_init(&fq->fqlock);
1403         fq->fqid = fqid;
1404         fq->fqid_le = cpu_to_be32(fqid);
1405         fq->flags = flags;
1406         fq->state = qman_fq_state_oos;
1407         fq->cgr_groupid = 0;
1408 #ifdef CONFIG_FSL_QMAN_FQ_LOOKUP
1409         if (unlikely(find_empty_fq_table_entry(&fq->key, fq))) {
1410                 pr_info("Find empty table entry failed\n");
1411                 return -ENOMEM;
1412         }
1413         fq->qman_fq_lookup_table = qman_fq_lookup_table;
1414 #endif
1415         if (!(flags & QMAN_FQ_FLAG_AS_IS) || (flags & QMAN_FQ_FLAG_NO_MODIFY))
1416                 return 0;
1417         /* Everything else is AS_IS support */
1418         p = get_affine_portal();
1419         mcc = qm_mc_start(&p->p);
1420         mcc->queryfq.fqid = cpu_to_be32(fqid);
1421         qm_mc_commit(&p->p, QM_MCC_VERB_QUERYFQ);
1422         while (!(mcr = qm_mc_result(&p->p)))
1423                 cpu_relax();
1424         DPAA_ASSERT((mcr->verb & QM_MCR_VERB_MASK) == QM_MCC_VERB_QUERYFQ);
1425         if (mcr->result != QM_MCR_RESULT_OK) {
1426                 pr_err("QUERYFQ failed: %s\n", mcr_result_str(mcr->result));
1427                 goto err;
1428         }
1429         fqd = mcr->queryfq.fqd;
1430         hw_fqd_to_cpu(&fqd);
1431         mcc = qm_mc_start(&p->p);
1432         mcc->queryfq_np.fqid = cpu_to_be32(fqid);
1433         qm_mc_commit(&p->p, QM_MCC_VERB_QUERYFQ_NP);
1434         while (!(mcr = qm_mc_result(&p->p)))
1435                 cpu_relax();
1436         DPAA_ASSERT((mcr->verb & QM_MCR_VERB_MASK) == QM_MCC_VERB_QUERYFQ_NP);
1437         if (mcr->result != QM_MCR_RESULT_OK) {
1438                 pr_err("QUERYFQ_NP failed: %s\n", mcr_result_str(mcr->result));
1439                 goto err;
1440         }
1441         np = mcr->queryfq_np;
1442         /* Phew, have queryfq and queryfq_np results, stitch together
1443          * the FQ object from those.
1444          */
1445         fq->cgr_groupid = fqd.cgid;
1446         switch (np.state & QM_MCR_NP_STATE_MASK) {
1447         case QM_MCR_NP_STATE_OOS:
1448                 break;
1449         case QM_MCR_NP_STATE_RETIRED:
1450                 fq->state = qman_fq_state_retired;
1451                 if (np.frm_cnt)
1452                         fq_set(fq, QMAN_FQ_STATE_NE);
1453                 break;
1454         case QM_MCR_NP_STATE_TEN_SCHED:
1455         case QM_MCR_NP_STATE_TRU_SCHED:
1456         case QM_MCR_NP_STATE_ACTIVE:
1457                 fq->state = qman_fq_state_sched;
1458                 if (np.state & QM_MCR_NP_STATE_R)
1459                         fq_set(fq, QMAN_FQ_STATE_CHANGING);
1460                 break;
1461         case QM_MCR_NP_STATE_PARKED:
1462                 fq->state = qman_fq_state_parked;
1463                 break;
1464         default:
1465                 DPAA_ASSERT(NULL == "invalid FQ state");
1466         }
1467         if (fqd.fq_ctrl & QM_FQCTRL_CGE)
1468                 fq->state |= QMAN_FQ_STATE_CGR_EN;
1469         return 0;
1470 err:
1471         if (flags & QMAN_FQ_FLAG_DYNAMIC_FQID)
1472                 qman_release_fqid(fqid);
1473         return -EIO;
1474 }
1475
1476 void qman_destroy_fq(struct qman_fq *fq, u32 flags __maybe_unused)
1477 {
1478         /*
1479          * We don't need to lock the FQ as it is a pre-condition that the FQ be
1480          * quiesced. Instead, run some checks.
1481          */
1482         switch (fq->state) {
1483         case qman_fq_state_parked:
1484                 DPAA_ASSERT(flags & QMAN_FQ_DESTROY_PARKED);
1485                 /* Fallthrough */
1486         case qman_fq_state_oos:
1487                 if (fq_isset(fq, QMAN_FQ_FLAG_DYNAMIC_FQID))
1488                         qman_release_fqid(fq->fqid);
1489 #ifdef CONFIG_FSL_QMAN_FQ_LOOKUP
1490                 clear_fq_table_entry(fq->key);
1491 #endif
1492                 return;
1493         default:
1494                 break;
1495         }
1496         DPAA_ASSERT(NULL == "qman_free_fq() on unquiesced FQ!");
1497 }
1498
1499 u32 qman_fq_fqid(struct qman_fq *fq)
1500 {
1501         return fq->fqid;
1502 }
1503
1504 void qman_fq_state(struct qman_fq *fq, enum qman_fq_state *state, u32 *flags)
1505 {
1506         if (state)
1507                 *state = fq->state;
1508         if (flags)
1509                 *flags = fq->flags;
1510 }
1511
1512 int qman_init_fq(struct qman_fq *fq, u32 flags, struct qm_mcc_initfq *opts)
1513 {
1514         struct qm_mc_command *mcc;
1515         struct qm_mc_result *mcr;
1516         struct qman_portal *p;
1517
1518         u8 res, myverb = (flags & QMAN_INITFQ_FLAG_SCHED) ?
1519                 QM_MCC_VERB_INITFQ_SCHED : QM_MCC_VERB_INITFQ_PARKED;
1520
1521         if ((fq->state != qman_fq_state_oos) &&
1522             (fq->state != qman_fq_state_parked))
1523                 return -EINVAL;
1524 #ifdef RTE_LIBRTE_DPAA_HWDEBUG
1525         if (unlikely(fq_isset(fq, QMAN_FQ_FLAG_NO_MODIFY)))
1526                 return -EINVAL;
1527 #endif
1528         if (opts && (opts->we_mask & QM_INITFQ_WE_OAC)) {
1529                 /* And can't be set at the same time as TDTHRESH */
1530                 if (opts->we_mask & QM_INITFQ_WE_TDTHRESH)
1531                         return -EINVAL;
1532         }
1533         /* Issue an INITFQ_[PARKED|SCHED] management command */
1534         p = get_affine_portal();
1535         FQLOCK(fq);
1536         if (unlikely((fq_isset(fq, QMAN_FQ_STATE_CHANGING)) ||
1537                      ((fq->state != qman_fq_state_oos) &&
1538                                 (fq->state != qman_fq_state_parked)))) {
1539                 FQUNLOCK(fq);
1540                 return -EBUSY;
1541         }
1542         mcc = qm_mc_start(&p->p);
1543         if (opts)
1544                 mcc->initfq = *opts;
1545         mcc->initfq.fqid = cpu_to_be32(fq->fqid);
1546         mcc->initfq.count = 0;
1547         /*
1548          * If the FQ does *not* have the TO_DCPORTAL flag, context_b is set as a
1549          * demux pointer. Otherwise, the caller-provided value is allowed to
1550          * stand, don't overwrite it.
1551          */
1552         if (fq_isclear(fq, QMAN_FQ_FLAG_TO_DCPORTAL)) {
1553                 dma_addr_t phys_fq;
1554
1555                 mcc->initfq.we_mask |= QM_INITFQ_WE_CONTEXTB;
1556 #ifdef CONFIG_FSL_QMAN_FQ_LOOKUP
1557                 mcc->initfq.fqd.context_b = cpu_to_be32(fq->key);
1558 #else
1559                 mcc->initfq.fqd.context_b = (u32)(uintptr_t)fq;
1560 #endif
1561                 /*
1562                  *  and the physical address - NB, if the user wasn't trying to
1563                  * set CONTEXTA, clear the stashing settings.
1564                  */
1565                 if (!(mcc->initfq.we_mask & QM_INITFQ_WE_CONTEXTA)) {
1566                         mcc->initfq.we_mask |= QM_INITFQ_WE_CONTEXTA;
1567                         memset(&mcc->initfq.fqd.context_a, 0,
1568                                sizeof(mcc->initfq.fqd.context_a));
1569                 } else {
1570                         phys_fq = rte_mem_virt2iova(fq);
1571                         qm_fqd_stashing_set64(&mcc->initfq.fqd, phys_fq);
1572                 }
1573         }
1574         if (flags & QMAN_INITFQ_FLAG_LOCAL) {
1575                 mcc->initfq.fqd.dest.channel = p->config->channel;
1576                 if (!(mcc->initfq.we_mask & QM_INITFQ_WE_DESTWQ)) {
1577                         mcc->initfq.we_mask |= QM_INITFQ_WE_DESTWQ;
1578                         mcc->initfq.fqd.dest.wq = 4;
1579                 }
1580         }
1581         mcc->initfq.we_mask = cpu_to_be16(mcc->initfq.we_mask);
1582         cpu_to_hw_fqd(&mcc->initfq.fqd);
1583         qm_mc_commit(&p->p, myverb);
1584         while (!(mcr = qm_mc_result(&p->p)))
1585                 cpu_relax();
1586         DPAA_ASSERT((mcr->verb & QM_MCR_VERB_MASK) == myverb);
1587         res = mcr->result;
1588         if (res != QM_MCR_RESULT_OK) {
1589                 FQUNLOCK(fq);
1590                 return -EIO;
1591         }
1592         if (opts) {
1593                 if (opts->we_mask & QM_INITFQ_WE_FQCTRL) {
1594                         if (opts->fqd.fq_ctrl & QM_FQCTRL_CGE)
1595                                 fq_set(fq, QMAN_FQ_STATE_CGR_EN);
1596                         else
1597                                 fq_clear(fq, QMAN_FQ_STATE_CGR_EN);
1598                 }
1599                 if (opts->we_mask & QM_INITFQ_WE_CGID)
1600                         fq->cgr_groupid = opts->fqd.cgid;
1601         }
1602         fq->state = (flags & QMAN_INITFQ_FLAG_SCHED) ?
1603                 qman_fq_state_sched : qman_fq_state_parked;
1604         FQUNLOCK(fq);
1605         return 0;
1606 }
1607
1608 int qman_schedule_fq(struct qman_fq *fq)
1609 {
1610         struct qm_mc_command *mcc;
1611         struct qm_mc_result *mcr;
1612         struct qman_portal *p;
1613
1614         int ret = 0;
1615         u8 res;
1616
1617         if (fq->state != qman_fq_state_parked)
1618                 return -EINVAL;
1619 #ifdef RTE_LIBRTE_DPAA_HWDEBUG
1620         if (unlikely(fq_isset(fq, QMAN_FQ_FLAG_NO_MODIFY)))
1621                 return -EINVAL;
1622 #endif
1623         /* Issue a ALTERFQ_SCHED management command */
1624         p = get_affine_portal();
1625
1626         FQLOCK(fq);
1627         if (unlikely((fq_isset(fq, QMAN_FQ_STATE_CHANGING)) ||
1628                      (fq->state != qman_fq_state_parked))) {
1629                 ret = -EBUSY;
1630                 goto out;
1631         }
1632         mcc = qm_mc_start(&p->p);
1633         mcc->alterfq.fqid = cpu_to_be32(fq->fqid);
1634         qm_mc_commit(&p->p, QM_MCC_VERB_ALTER_SCHED);
1635         while (!(mcr = qm_mc_result(&p->p)))
1636                 cpu_relax();
1637         DPAA_ASSERT((mcr->verb & QM_MCR_VERB_MASK) == QM_MCR_VERB_ALTER_SCHED);
1638         res = mcr->result;
1639         if (res != QM_MCR_RESULT_OK) {
1640                 ret = -EIO;
1641                 goto out;
1642         }
1643         fq->state = qman_fq_state_sched;
1644 out:
1645         FQUNLOCK(fq);
1646
1647         return ret;
1648 }
1649
1650 int qman_retire_fq(struct qman_fq *fq, u32 *flags)
1651 {
1652         struct qm_mc_command *mcc;
1653         struct qm_mc_result *mcr;
1654         struct qman_portal *p;
1655
1656         int rval;
1657         u8 res;
1658
1659         if ((fq->state != qman_fq_state_parked) &&
1660             (fq->state != qman_fq_state_sched))
1661                 return -EINVAL;
1662 #ifdef RTE_LIBRTE_DPAA_HWDEBUG
1663         if (unlikely(fq_isset(fq, QMAN_FQ_FLAG_NO_MODIFY)))
1664                 return -EINVAL;
1665 #endif
1666         p = get_affine_portal();
1667
1668         FQLOCK(fq);
1669         if (unlikely((fq_isset(fq, QMAN_FQ_STATE_CHANGING)) ||
1670                      (fq->state == qman_fq_state_retired) ||
1671                                 (fq->state == qman_fq_state_oos))) {
1672                 rval = -EBUSY;
1673                 goto out;
1674         }
1675         rval = table_push_fq(p, fq);
1676         if (rval)
1677                 goto out;
1678         mcc = qm_mc_start(&p->p);
1679         mcc->alterfq.fqid = cpu_to_be32(fq->fqid);
1680         qm_mc_commit(&p->p, QM_MCC_VERB_ALTER_RETIRE);
1681         while (!(mcr = qm_mc_result(&p->p)))
1682                 cpu_relax();
1683         DPAA_ASSERT((mcr->verb & QM_MCR_VERB_MASK) == QM_MCR_VERB_ALTER_RETIRE);
1684         res = mcr->result;
1685         /*
1686          * "Elegant" would be to treat OK/PENDING the same way; set CHANGING,
1687          * and defer the flags until FQRNI or FQRN (respectively) show up. But
1688          * "Friendly" is to process OK immediately, and not set CHANGING. We do
1689          * friendly, otherwise the caller doesn't necessarily have a fully
1690          * "retired" FQ on return even if the retirement was immediate. However
1691          * this does mean some code duplication between here and
1692          * fq_state_change().
1693          */
1694         if (likely(res == QM_MCR_RESULT_OK)) {
1695                 rval = 0;
1696                 /* Process 'fq' right away, we'll ignore FQRNI */
1697                 if (mcr->alterfq.fqs & QM_MCR_FQS_NOTEMPTY)
1698                         fq_set(fq, QMAN_FQ_STATE_NE);
1699                 if (mcr->alterfq.fqs & QM_MCR_FQS_ORLPRESENT)
1700                         fq_set(fq, QMAN_FQ_STATE_ORL);
1701                 else
1702                         table_del_fq(p, fq);
1703                 if (flags)
1704                         *flags = fq->flags;
1705                 fq->state = qman_fq_state_retired;
1706                 if (fq->cb.fqs) {
1707                         /*
1708                          * Another issue with supporting "immediate" retirement
1709                          * is that we're forced to drop FQRNIs, because by the
1710                          * time they're seen it may already be "too late" (the
1711                          * fq may have been OOS'd and free()'d already). But if
1712                          * the upper layer wants a callback whether it's
1713                          * immediate or not, we have to fake a "MR" entry to
1714                          * look like an FQRNI...
1715                          */
1716                         struct qm_mr_entry msg;
1717
1718                         msg.ern.verb = QM_MR_VERB_FQRNI;
1719                         msg.fq.fqs = mcr->alterfq.fqs;
1720                         msg.fq.fqid = fq->fqid;
1721 #ifdef CONFIG_FSL_QMAN_FQ_LOOKUP
1722                         msg.fq.contextB = fq->key;
1723 #else
1724                         msg.fq.contextB = (u32)(uintptr_t)fq;
1725 #endif
1726                         fq->cb.fqs(p, fq, &msg);
1727                 }
1728         } else if (res == QM_MCR_RESULT_PENDING) {
1729                 rval = 1;
1730                 fq_set(fq, QMAN_FQ_STATE_CHANGING);
1731         } else {
1732                 rval = -EIO;
1733                 table_del_fq(p, fq);
1734         }
1735 out:
1736         FQUNLOCK(fq);
1737         return rval;
1738 }
1739
1740 int qman_oos_fq(struct qman_fq *fq)
1741 {
1742         struct qm_mc_command *mcc;
1743         struct qm_mc_result *mcr;
1744         struct qman_portal *p;
1745
1746         int ret = 0;
1747         u8 res;
1748
1749         if (fq->state != qman_fq_state_retired)
1750                 return -EINVAL;
1751 #ifdef RTE_LIBRTE_DPAA_HWDEBUG
1752         if (unlikely(fq_isset(fq, QMAN_FQ_FLAG_NO_MODIFY)))
1753                 return -EINVAL;
1754 #endif
1755         p = get_affine_portal();
1756         FQLOCK(fq);
1757         if (unlikely((fq_isset(fq, QMAN_FQ_STATE_BLOCKOOS)) ||
1758                      (fq->state != qman_fq_state_retired))) {
1759                 ret = -EBUSY;
1760                 goto out;
1761         }
1762         mcc = qm_mc_start(&p->p);
1763         mcc->alterfq.fqid = cpu_to_be32(fq->fqid);
1764         qm_mc_commit(&p->p, QM_MCC_VERB_ALTER_OOS);
1765         while (!(mcr = qm_mc_result(&p->p)))
1766                 cpu_relax();
1767         DPAA_ASSERT((mcr->verb & QM_MCR_VERB_MASK) == QM_MCR_VERB_ALTER_OOS);
1768         res = mcr->result;
1769         if (res != QM_MCR_RESULT_OK) {
1770                 ret = -EIO;
1771                 goto out;
1772         }
1773         fq->state = qman_fq_state_oos;
1774 out:
1775         FQUNLOCK(fq);
1776         return ret;
1777 }
1778
1779 int qman_fq_flow_control(struct qman_fq *fq, int xon)
1780 {
1781         struct qm_mc_command *mcc;
1782         struct qm_mc_result *mcr;
1783         struct qman_portal *p;
1784
1785         int ret = 0;
1786         u8 res;
1787         u8 myverb;
1788
1789         if ((fq->state == qman_fq_state_oos) ||
1790             (fq->state == qman_fq_state_retired) ||
1791                 (fq->state == qman_fq_state_parked))
1792                 return -EINVAL;
1793
1794 #ifdef RTE_LIBRTE_DPAA_HWDEBUG
1795         if (unlikely(fq_isset(fq, QMAN_FQ_FLAG_NO_MODIFY)))
1796                 return -EINVAL;
1797 #endif
1798         /* Issue a ALTER_FQXON or ALTER_FQXOFF management command */
1799         p = get_affine_portal();
1800         FQLOCK(fq);
1801         if (unlikely((fq_isset(fq, QMAN_FQ_STATE_CHANGING)) ||
1802                      (fq->state == qman_fq_state_parked) ||
1803                         (fq->state == qman_fq_state_oos) ||
1804                         (fq->state == qman_fq_state_retired))) {
1805                 ret = -EBUSY;
1806                 goto out;
1807         }
1808         mcc = qm_mc_start(&p->p);
1809         mcc->alterfq.fqid = fq->fqid;
1810         mcc->alterfq.count = 0;
1811         myverb = xon ? QM_MCC_VERB_ALTER_FQXON : QM_MCC_VERB_ALTER_FQXOFF;
1812
1813         qm_mc_commit(&p->p, myverb);
1814         while (!(mcr = qm_mc_result(&p->p)))
1815                 cpu_relax();
1816         DPAA_ASSERT((mcr->verb & QM_MCR_VERB_MASK) == myverb);
1817
1818         res = mcr->result;
1819         if (res != QM_MCR_RESULT_OK) {
1820                 ret = -EIO;
1821                 goto out;
1822         }
1823 out:
1824         FQUNLOCK(fq);
1825         return ret;
1826 }
1827
1828 int qman_query_fq(struct qman_fq *fq, struct qm_fqd *fqd)
1829 {
1830         struct qm_mc_command *mcc;
1831         struct qm_mc_result *mcr;
1832         struct qman_portal *p = get_affine_portal();
1833
1834         u8 res;
1835
1836         mcc = qm_mc_start(&p->p);
1837         mcc->queryfq.fqid = cpu_to_be32(fq->fqid);
1838         qm_mc_commit(&p->p, QM_MCC_VERB_QUERYFQ);
1839         while (!(mcr = qm_mc_result(&p->p)))
1840                 cpu_relax();
1841         DPAA_ASSERT((mcr->verb & QM_MCR_VERB_MASK) == QM_MCR_VERB_QUERYFQ);
1842         res = mcr->result;
1843         if (res == QM_MCR_RESULT_OK)
1844                 *fqd = mcr->queryfq.fqd;
1845         hw_fqd_to_cpu(fqd);
1846         if (res != QM_MCR_RESULT_OK)
1847                 return -EIO;
1848         return 0;
1849 }
1850
1851 int qman_query_fq_has_pkts(struct qman_fq *fq)
1852 {
1853         struct qm_mc_command *mcc;
1854         struct qm_mc_result *mcr;
1855         struct qman_portal *p = get_affine_portal();
1856
1857         int ret = 0;
1858         u8 res;
1859
1860         mcc = qm_mc_start(&p->p);
1861         mcc->queryfq.fqid = cpu_to_be32(fq->fqid);
1862         qm_mc_commit(&p->p, QM_MCC_VERB_QUERYFQ_NP);
1863         while (!(mcr = qm_mc_result(&p->p)))
1864                 cpu_relax();
1865         res = mcr->result;
1866         if (res == QM_MCR_RESULT_OK)
1867                 ret = !!mcr->queryfq_np.frm_cnt;
1868         return ret;
1869 }
1870
1871 int qman_query_fq_np(struct qman_fq *fq, struct qm_mcr_queryfq_np *np)
1872 {
1873         struct qm_mc_command *mcc;
1874         struct qm_mc_result *mcr;
1875         struct qman_portal *p = get_affine_portal();
1876
1877         u8 res;
1878
1879         mcc = qm_mc_start(&p->p);
1880         mcc->queryfq.fqid = cpu_to_be32(fq->fqid);
1881         qm_mc_commit(&p->p, QM_MCC_VERB_QUERYFQ_NP);
1882         while (!(mcr = qm_mc_result(&p->p)))
1883                 cpu_relax();
1884         DPAA_ASSERT((mcr->verb & QM_MCR_VERB_MASK) == QM_MCR_VERB_QUERYFQ_NP);
1885         res = mcr->result;
1886         if (res == QM_MCR_RESULT_OK) {
1887                 *np = mcr->queryfq_np;
1888                 np->fqd_link = be24_to_cpu(np->fqd_link);
1889                 np->odp_seq = be16_to_cpu(np->odp_seq);
1890                 np->orp_nesn = be16_to_cpu(np->orp_nesn);
1891                 np->orp_ea_hseq  = be16_to_cpu(np->orp_ea_hseq);
1892                 np->orp_ea_tseq  = be16_to_cpu(np->orp_ea_tseq);
1893                 np->orp_ea_hptr = be24_to_cpu(np->orp_ea_hptr);
1894                 np->orp_ea_tptr = be24_to_cpu(np->orp_ea_tptr);
1895                 np->pfdr_hptr = be24_to_cpu(np->pfdr_hptr);
1896                 np->pfdr_tptr = be24_to_cpu(np->pfdr_tptr);
1897                 np->ics_surp = be16_to_cpu(np->ics_surp);
1898                 np->byte_cnt = be32_to_cpu(np->byte_cnt);
1899                 np->frm_cnt = be24_to_cpu(np->frm_cnt);
1900                 np->ra1_sfdr = be16_to_cpu(np->ra1_sfdr);
1901                 np->ra2_sfdr = be16_to_cpu(np->ra2_sfdr);
1902                 np->od1_sfdr = be16_to_cpu(np->od1_sfdr);
1903                 np->od2_sfdr = be16_to_cpu(np->od2_sfdr);
1904                 np->od3_sfdr = be16_to_cpu(np->od3_sfdr);
1905         }
1906         if (res == QM_MCR_RESULT_ERR_FQID)
1907                 return -ERANGE;
1908         else if (res != QM_MCR_RESULT_OK)
1909                 return -EIO;
1910         return 0;
1911 }
1912
1913 int qman_query_fq_frm_cnt(struct qman_fq *fq, u32 *frm_cnt)
1914 {
1915         struct qm_mc_command *mcc;
1916         struct qm_mc_result *mcr;
1917         struct qman_portal *p = get_affine_portal();
1918
1919         mcc = qm_mc_start(&p->p);
1920         mcc->queryfq.fqid = cpu_to_be32(fq->fqid);
1921         qm_mc_commit(&p->p, QM_MCC_VERB_QUERYFQ_NP);
1922         while (!(mcr = qm_mc_result(&p->p)))
1923                 cpu_relax();
1924         DPAA_ASSERT((mcr->verb & QM_MCR_VERB_MASK) == QM_MCR_VERB_QUERYFQ_NP);
1925
1926         if (mcr->result == QM_MCR_RESULT_OK)
1927                 *frm_cnt = be24_to_cpu(mcr->queryfq_np.frm_cnt);
1928         else if (mcr->result == QM_MCR_RESULT_ERR_FQID)
1929                 return -ERANGE;
1930         else if (mcr->result != QM_MCR_RESULT_OK)
1931                 return -EIO;
1932         return 0;
1933 }
1934
1935 int qman_query_wq(u8 query_dedicated, struct qm_mcr_querywq *wq)
1936 {
1937         struct qm_mc_command *mcc;
1938         struct qm_mc_result *mcr;
1939         struct qman_portal *p = get_affine_portal();
1940
1941         u8 res, myverb;
1942
1943         myverb = (query_dedicated) ? QM_MCR_VERB_QUERYWQ_DEDICATED :
1944                                  QM_MCR_VERB_QUERYWQ;
1945         mcc = qm_mc_start(&p->p);
1946         mcc->querywq.channel.id = cpu_to_be16(wq->channel.id);
1947         qm_mc_commit(&p->p, myverb);
1948         while (!(mcr = qm_mc_result(&p->p)))
1949                 cpu_relax();
1950         DPAA_ASSERT((mcr->verb & QM_MCR_VERB_MASK) == myverb);
1951         res = mcr->result;
1952         if (res == QM_MCR_RESULT_OK) {
1953                 int i, array_len;
1954
1955                 wq->channel.id = be16_to_cpu(mcr->querywq.channel.id);
1956                 array_len = ARRAY_SIZE(mcr->querywq.wq_len);
1957                 for (i = 0; i < array_len; i++)
1958                         wq->wq_len[i] = be32_to_cpu(mcr->querywq.wq_len[i]);
1959         }
1960         if (res != QM_MCR_RESULT_OK) {
1961                 pr_err("QUERYWQ failed: %s\n", mcr_result_str(res));
1962                 return -EIO;
1963         }
1964         return 0;
1965 }
1966
1967 int qman_testwrite_cgr(struct qman_cgr *cgr, u64 i_bcnt,
1968                        struct qm_mcr_cgrtestwrite *result)
1969 {
1970         struct qm_mc_command *mcc;
1971         struct qm_mc_result *mcr;
1972         struct qman_portal *p = get_affine_portal();
1973
1974         u8 res;
1975
1976         mcc = qm_mc_start(&p->p);
1977         mcc->cgrtestwrite.cgid = cgr->cgrid;
1978         mcc->cgrtestwrite.i_bcnt_hi = (u8)(i_bcnt >> 32);
1979         mcc->cgrtestwrite.i_bcnt_lo = (u32)i_bcnt;
1980         qm_mc_commit(&p->p, QM_MCC_VERB_CGRTESTWRITE);
1981         while (!(mcr = qm_mc_result(&p->p)))
1982                 cpu_relax();
1983         DPAA_ASSERT((mcr->verb & QM_MCR_VERB_MASK) == QM_MCC_VERB_CGRTESTWRITE);
1984         res = mcr->result;
1985         if (res == QM_MCR_RESULT_OK)
1986                 *result = mcr->cgrtestwrite;
1987         if (res != QM_MCR_RESULT_OK) {
1988                 pr_err("CGR TEST WRITE failed: %s\n", mcr_result_str(res));
1989                 return -EIO;
1990         }
1991         return 0;
1992 }
1993
1994 int qman_query_cgr(struct qman_cgr *cgr, struct qm_mcr_querycgr *cgrd)
1995 {
1996         struct qm_mc_command *mcc;
1997         struct qm_mc_result *mcr;
1998         struct qman_portal *p = get_affine_portal();
1999         u8 res;
2000         unsigned int i;
2001
2002         mcc = qm_mc_start(&p->p);
2003         mcc->querycgr.cgid = cgr->cgrid;
2004         qm_mc_commit(&p->p, QM_MCC_VERB_QUERYCGR);
2005         while (!(mcr = qm_mc_result(&p->p)))
2006                 cpu_relax();
2007         DPAA_ASSERT((mcr->verb & QM_MCR_VERB_MASK) == QM_MCC_VERB_QUERYCGR);
2008         res = mcr->result;
2009         if (res == QM_MCR_RESULT_OK)
2010                 *cgrd = mcr->querycgr;
2011         if (res != QM_MCR_RESULT_OK) {
2012                 pr_err("QUERY_CGR failed: %s\n", mcr_result_str(res));
2013                 return -EIO;
2014         }
2015         cgrd->cgr.wr_parm_g.word =
2016                 be32_to_cpu(cgrd->cgr.wr_parm_g.word);
2017         cgrd->cgr.wr_parm_y.word =
2018                 be32_to_cpu(cgrd->cgr.wr_parm_y.word);
2019         cgrd->cgr.wr_parm_r.word =
2020                 be32_to_cpu(cgrd->cgr.wr_parm_r.word);
2021         cgrd->cgr.cscn_targ =  be32_to_cpu(cgrd->cgr.cscn_targ);
2022         cgrd->cgr.__cs_thres = be16_to_cpu(cgrd->cgr.__cs_thres);
2023         for (i = 0; i < ARRAY_SIZE(cgrd->cscn_targ_swp); i++)
2024                 cgrd->cscn_targ_swp[i] =
2025                         be32_to_cpu(cgrd->cscn_targ_swp[i]);
2026         return 0;
2027 }
2028
2029 int qman_query_congestion(struct qm_mcr_querycongestion *congestion)
2030 {
2031         struct qm_mc_result *mcr;
2032         struct qman_portal *p = get_affine_portal();
2033         u8 res;
2034         unsigned int i;
2035
2036         qm_mc_start(&p->p);
2037         qm_mc_commit(&p->p, QM_MCC_VERB_QUERYCONGESTION);
2038         while (!(mcr = qm_mc_result(&p->p)))
2039                 cpu_relax();
2040         DPAA_ASSERT((mcr->verb & QM_MCR_VERB_MASK) ==
2041                         QM_MCC_VERB_QUERYCONGESTION);
2042         res = mcr->result;
2043         if (res == QM_MCR_RESULT_OK)
2044                 *congestion = mcr->querycongestion;
2045         if (res != QM_MCR_RESULT_OK) {
2046                 pr_err("QUERY_CONGESTION failed: %s\n", mcr_result_str(res));
2047                 return -EIO;
2048         }
2049         for (i = 0; i < ARRAY_SIZE(congestion->state.state); i++)
2050                 congestion->state.state[i] =
2051                         be32_to_cpu(congestion->state.state[i]);
2052         return 0;
2053 }
2054
2055 int qman_set_vdq(struct qman_fq *fq, u16 num, uint32_t vdqcr_flags)
2056 {
2057         struct qman_portal *p = get_affine_portal();
2058         uint32_t vdqcr;
2059         int ret = -EBUSY;
2060
2061         vdqcr = vdqcr_flags;
2062         vdqcr |= QM_VDQCR_NUMFRAMES_SET(num);
2063
2064         if ((fq->state != qman_fq_state_parked) &&
2065             (fq->state != qman_fq_state_retired)) {
2066                 ret = -EINVAL;
2067                 goto out;
2068         }
2069         if (fq_isset(fq, QMAN_FQ_STATE_VDQCR)) {
2070                 ret = -EBUSY;
2071                 goto out;
2072         }
2073         vdqcr = (vdqcr & ~QM_VDQCR_FQID_MASK) | fq->fqid;
2074
2075         if (!p->vdqcr_owned) {
2076                 FQLOCK(fq);
2077                 if (fq_isset(fq, QMAN_FQ_STATE_VDQCR))
2078                         goto escape;
2079                 fq_set(fq, QMAN_FQ_STATE_VDQCR);
2080                 FQUNLOCK(fq);
2081                 p->vdqcr_owned = fq;
2082                 ret = 0;
2083         }
2084 escape:
2085         if (!ret)
2086                 qm_dqrr_vdqcr_set(&p->p, vdqcr);
2087
2088 out:
2089         return ret;
2090 }
2091
2092 int qman_volatile_dequeue(struct qman_fq *fq, u32 flags __maybe_unused,
2093                           u32 vdqcr)
2094 {
2095         struct qman_portal *p;
2096         int ret = -EBUSY;
2097
2098         if ((fq->state != qman_fq_state_parked) &&
2099             (fq->state != qman_fq_state_retired))
2100                 return -EINVAL;
2101         if (vdqcr & QM_VDQCR_FQID_MASK)
2102                 return -EINVAL;
2103         if (fq_isset(fq, QMAN_FQ_STATE_VDQCR))
2104                 return -EBUSY;
2105         vdqcr = (vdqcr & ~QM_VDQCR_FQID_MASK) | fq->fqid;
2106
2107         p = get_affine_portal();
2108
2109         if (!p->vdqcr_owned) {
2110                 FQLOCK(fq);
2111                 if (fq_isset(fq, QMAN_FQ_STATE_VDQCR))
2112                         goto escape;
2113                 fq_set(fq, QMAN_FQ_STATE_VDQCR);
2114                 FQUNLOCK(fq);
2115                 p->vdqcr_owned = fq;
2116                 ret = 0;
2117         }
2118 escape:
2119         if (ret)
2120                 return ret;
2121
2122         /* VDQCR is set */
2123         qm_dqrr_vdqcr_set(&p->p, vdqcr);
2124         return 0;
2125 }
2126
2127 static noinline void update_eqcr_ci(struct qman_portal *p, u8 avail)
2128 {
2129         if (avail)
2130                 qm_eqcr_cce_prefetch(&p->p);
2131         else
2132                 qm_eqcr_cce_update(&p->p);
2133 }
2134
2135 int qman_eqcr_is_empty(void)
2136 {
2137         struct qman_portal *p = get_affine_portal();
2138         u8 avail;
2139
2140         update_eqcr_ci(p, 0);
2141         avail = qm_eqcr_get_fill(&p->p);
2142         return (avail == 0);
2143 }
2144
2145 void qman_set_dc_ern(qman_cb_dc_ern handler, int affine)
2146 {
2147         if (affine) {
2148                 struct qman_portal *p = get_affine_portal();
2149
2150                 p->cb_dc_ern = handler;
2151         } else
2152                 cb_dc_ern = handler;
2153 }
2154
2155 static inline struct qm_eqcr_entry *try_p_eq_start(struct qman_portal *p,
2156                                         struct qman_fq *fq,
2157                                         const struct qm_fd *fd,
2158                                         u32 flags)
2159 {
2160         struct qm_eqcr_entry *eq;
2161         u8 avail;
2162
2163         if (p->use_eqcr_ci_stashing) {
2164                 /*
2165                  * The stashing case is easy, only update if we need to in
2166                  * order to try and liberate ring entries.
2167                  */
2168                 eq = qm_eqcr_start_stash(&p->p);
2169         } else {
2170                 /*
2171                  * The non-stashing case is harder, need to prefetch ahead of
2172                  * time.
2173                  */
2174                 avail = qm_eqcr_get_avail(&p->p);
2175                 if (avail < 2)
2176                         update_eqcr_ci(p, avail);
2177                 eq = qm_eqcr_start_no_stash(&p->p);
2178         }
2179
2180         if (unlikely(!eq))
2181                 return NULL;
2182
2183         if (flags & QMAN_ENQUEUE_FLAG_DCA)
2184                 eq->dca = QM_EQCR_DCA_ENABLE |
2185                         ((flags & QMAN_ENQUEUE_FLAG_DCA_PARK) ?
2186                                         QM_EQCR_DCA_PARK : 0) |
2187                         ((flags >> 8) & QM_EQCR_DCA_IDXMASK);
2188         eq->fqid = cpu_to_be32(fq->fqid);
2189 #ifdef CONFIG_FSL_QMAN_FQ_LOOKUP
2190         eq->tag = cpu_to_be32(fq->key);
2191 #else
2192         eq->tag = cpu_to_be32((u32)(uintptr_t)fq);
2193 #endif
2194         eq->fd = *fd;
2195         cpu_to_hw_fd(&eq->fd);
2196         return eq;
2197 }
2198
2199 int qman_enqueue(struct qman_fq *fq, const struct qm_fd *fd, u32 flags)
2200 {
2201         struct qman_portal *p = get_affine_portal();
2202         struct qm_eqcr_entry *eq;
2203
2204         eq = try_p_eq_start(p, fq, fd, flags);
2205         if (!eq)
2206                 return -EBUSY;
2207         /* Note: QM_EQCR_VERB_INTERRUPT == QMAN_ENQUEUE_FLAG_WAIT_SYNC */
2208         qm_eqcr_pvb_commit(&p->p, QM_EQCR_VERB_CMD_ENQUEUE |
2209                 (flags & (QM_EQCR_VERB_COLOUR_MASK | QM_EQCR_VERB_INTERRUPT)));
2210         /* Factor the below out, it's used from qman_enqueue_orp() too */
2211         return 0;
2212 }
2213
2214 int qman_enqueue_multi(struct qman_fq *fq,
2215                        const struct qm_fd *fd, u32 *flags,
2216                 int frames_to_send)
2217 {
2218         struct qman_portal *p = get_affine_portal();
2219         struct qm_portal *portal = &p->p;
2220
2221         register struct qm_eqcr *eqcr = &portal->eqcr;
2222         struct qm_eqcr_entry *eq = eqcr->cursor, *prev_eq;
2223
2224         u8 i = 0, diff, old_ci, sent = 0;
2225
2226         /* Update the available entries if no entry is free */
2227         if (!eqcr->available) {
2228                 old_ci = eqcr->ci;
2229                 eqcr->ci = qm_cl_in(EQCR_CI) & (QM_EQCR_SIZE - 1);
2230                 diff = qm_cyc_diff(QM_EQCR_SIZE, old_ci, eqcr->ci);
2231                 eqcr->available += diff;
2232                 if (!diff)
2233                         return 0;
2234         }
2235
2236         /* try to send as many frames as possible */
2237         while (eqcr->available && frames_to_send--) {
2238                 eq->fqid = fq->fqid_le;
2239                 eq->fd.opaque_addr = fd->opaque_addr;
2240                 eq->fd.addr = cpu_to_be40(fd->addr);
2241                 eq->fd.status = cpu_to_be32(fd->status);
2242                 eq->fd.opaque = cpu_to_be32(fd->opaque);
2243                 if (flags && (flags[i] & QMAN_ENQUEUE_FLAG_DCA)) {
2244                         eq->dca = QM_EQCR_DCA_ENABLE |
2245                                 ((flags[i] >> 8) & QM_EQCR_DCA_IDXMASK);
2246                 }
2247                 i++;
2248                 eq = (void *)((unsigned long)(eq + 1) &
2249                         (~(unsigned long)(QM_EQCR_SIZE << 6)));
2250                 eqcr->available--;
2251                 sent++;
2252                 fd++;
2253         }
2254         lwsync();
2255
2256         /* In order for flushes to complete faster, all lines are recorded in
2257          * 32 bit word.
2258          */
2259         eq = eqcr->cursor;
2260         for (i = 0; i < sent; i++) {
2261                 eq->__dont_write_directly__verb =
2262                         QM_EQCR_VERB_CMD_ENQUEUE | eqcr->vbit;
2263                 prev_eq = eq;
2264                 eq = (void *)((unsigned long)(eq + 1) &
2265                         (~(unsigned long)(QM_EQCR_SIZE << 6)));
2266                 if (unlikely((prev_eq + 1) != eq))
2267                         eqcr->vbit ^= QM_EQCR_VERB_VBIT;
2268         }
2269
2270         /* We need  to flush all the lines but without load/store operations
2271          * between them
2272          */
2273         eq = eqcr->cursor;
2274         for (i = 0; i < sent; i++) {
2275                 dcbf(eq);
2276                 eq = (void *)((unsigned long)(eq + 1) &
2277                         (~(unsigned long)(QM_EQCR_SIZE << 6)));
2278         }
2279         /* Update cursor for the next call */
2280         eqcr->cursor = eq;
2281         return sent;
2282 }
2283
2284 int
2285 qman_enqueue_multi_fq(struct qman_fq *fq[], const struct qm_fd *fd,
2286                       u32 *flags, int frames_to_send)
2287 {
2288         struct qman_portal *p = get_affine_portal();
2289         struct qm_portal *portal = &p->p;
2290
2291         register struct qm_eqcr *eqcr = &portal->eqcr;
2292         struct qm_eqcr_entry *eq = eqcr->cursor, *prev_eq;
2293
2294         u8 i = 0, diff, old_ci, sent = 0;
2295
2296         /* Update the available entries if no entry is free */
2297         if (!eqcr->available) {
2298                 old_ci = eqcr->ci;
2299                 eqcr->ci = qm_cl_in(EQCR_CI) & (QM_EQCR_SIZE - 1);
2300                 diff = qm_cyc_diff(QM_EQCR_SIZE, old_ci, eqcr->ci);
2301                 eqcr->available += diff;
2302                 if (!diff)
2303                         return 0;
2304         }
2305
2306         /* try to send as many frames as possible */
2307         while (eqcr->available && frames_to_send--) {
2308                 eq->fqid = fq[sent]->fqid_le;
2309                 eq->fd.opaque_addr = fd->opaque_addr;
2310                 eq->fd.addr = cpu_to_be40(fd->addr);
2311                 eq->fd.status = cpu_to_be32(fd->status);
2312                 eq->fd.opaque = cpu_to_be32(fd->opaque);
2313                 if (flags && (flags[i] & QMAN_ENQUEUE_FLAG_DCA)) {
2314                         eq->dca = QM_EQCR_DCA_ENABLE |
2315                                 ((flags[i] >> 8) & QM_EQCR_DCA_IDXMASK);
2316                 }
2317                 i++;
2318
2319                 eq = (void *)((unsigned long)(eq + 1) &
2320                         (~(unsigned long)(QM_EQCR_SIZE << 6)));
2321                 eqcr->available--;
2322                 sent++;
2323                 fd++;
2324         }
2325         lwsync();
2326
2327         /* In order for flushes to complete faster, all lines are recorded in
2328          * 32 bit word.
2329          */
2330         eq = eqcr->cursor;
2331         for (i = 0; i < sent; i++) {
2332                 eq->__dont_write_directly__verb =
2333                         QM_EQCR_VERB_CMD_ENQUEUE | eqcr->vbit;
2334                 prev_eq = eq;
2335                 eq = (void *)((unsigned long)(eq + 1) &
2336                         (~(unsigned long)(QM_EQCR_SIZE << 6)));
2337                 if (unlikely((prev_eq + 1) != eq))
2338                         eqcr->vbit ^= QM_EQCR_VERB_VBIT;
2339         }
2340
2341         /* We need  to flush all the lines but without load/store operations
2342          * between them
2343          */
2344         eq = eqcr->cursor;
2345         for (i = 0; i < sent; i++) {
2346                 dcbf(eq);
2347                 eq = (void *)((unsigned long)(eq + 1) &
2348                         (~(unsigned long)(QM_EQCR_SIZE << 6)));
2349         }
2350         /* Update cursor for the next call */
2351         eqcr->cursor = eq;
2352         return sent;
2353 }
2354
2355 int qman_enqueue_orp(struct qman_fq *fq, const struct qm_fd *fd, u32 flags,
2356                      struct qman_fq *orp, u16 orp_seqnum)
2357 {
2358         struct qman_portal *p  = get_affine_portal();
2359         struct qm_eqcr_entry *eq;
2360
2361         eq = try_p_eq_start(p, fq, fd, flags);
2362         if (!eq)
2363                 return -EBUSY;
2364         /* Process ORP-specifics here */
2365         if (flags & QMAN_ENQUEUE_FLAG_NLIS)
2366                 orp_seqnum |= QM_EQCR_SEQNUM_NLIS;
2367         else {
2368                 orp_seqnum &= ~QM_EQCR_SEQNUM_NLIS;
2369                 if (flags & QMAN_ENQUEUE_FLAG_NESN)
2370                         orp_seqnum |= QM_EQCR_SEQNUM_NESN;
2371                 else
2372                         /* No need to check 4 QMAN_ENQUEUE_FLAG_HOLE */
2373                         orp_seqnum &= ~QM_EQCR_SEQNUM_NESN;
2374         }
2375         eq->seqnum = cpu_to_be16(orp_seqnum);
2376         eq->orp = cpu_to_be32(orp->fqid);
2377         /* Note: QM_EQCR_VERB_INTERRUPT == QMAN_ENQUEUE_FLAG_WAIT_SYNC */
2378         qm_eqcr_pvb_commit(&p->p, QM_EQCR_VERB_ORP |
2379                 ((flags & (QMAN_ENQUEUE_FLAG_HOLE | QMAN_ENQUEUE_FLAG_NESN)) ?
2380                                 0 : QM_EQCR_VERB_CMD_ENQUEUE) |
2381                 (flags & (QM_EQCR_VERB_COLOUR_MASK | QM_EQCR_VERB_INTERRUPT)));
2382
2383         return 0;
2384 }
2385
2386 int qman_modify_cgr(struct qman_cgr *cgr, u32 flags,
2387                     struct qm_mcc_initcgr *opts)
2388 {
2389         struct qm_mc_command *mcc;
2390         struct qm_mc_result *mcr;
2391         struct qman_portal *p = get_affine_portal();
2392
2393         u8 res;
2394         u8 verb = QM_MCC_VERB_MODIFYCGR;
2395
2396         mcc = qm_mc_start(&p->p);
2397         if (opts)
2398                 mcc->initcgr = *opts;
2399         mcc->initcgr.we_mask = cpu_to_be16(mcc->initcgr.we_mask);
2400         mcc->initcgr.cgr.wr_parm_g.word =
2401                 cpu_to_be32(mcc->initcgr.cgr.wr_parm_g.word);
2402         mcc->initcgr.cgr.wr_parm_y.word =
2403                 cpu_to_be32(mcc->initcgr.cgr.wr_parm_y.word);
2404         mcc->initcgr.cgr.wr_parm_r.word =
2405                 cpu_to_be32(mcc->initcgr.cgr.wr_parm_r.word);
2406         mcc->initcgr.cgr.cscn_targ =  cpu_to_be32(mcc->initcgr.cgr.cscn_targ);
2407         mcc->initcgr.cgr.__cs_thres = cpu_to_be16(mcc->initcgr.cgr.__cs_thres);
2408
2409         mcc->initcgr.cgid = cgr->cgrid;
2410         if (flags & QMAN_CGR_FLAG_USE_INIT)
2411                 verb = QM_MCC_VERB_INITCGR;
2412         qm_mc_commit(&p->p, verb);
2413         while (!(mcr = qm_mc_result(&p->p)))
2414                 cpu_relax();
2415
2416         DPAA_ASSERT((mcr->verb & QM_MCR_VERB_MASK) == verb);
2417         res = mcr->result;
2418         return (res == QM_MCR_RESULT_OK) ? 0 : -EIO;
2419 }
2420
2421 #define TARG_MASK(n) (0x80000000 >> (n->config->channel - \
2422                                         QM_CHANNEL_SWPORTAL0))
2423 #define TARG_DCP_MASK(n) (0x80000000 >> (10 + n))
2424 #define PORTAL_IDX(n) (n->config->channel - QM_CHANNEL_SWPORTAL0)
2425
2426 int qman_create_cgr(struct qman_cgr *cgr, u32 flags,
2427                     struct qm_mcc_initcgr *opts)
2428 {
2429         struct qm_mcr_querycgr cgr_state;
2430         struct qm_mcc_initcgr local_opts;
2431         int ret;
2432         struct qman_portal *p;
2433
2434         /* We have to check that the provided CGRID is within the limits of the
2435          * data-structures, for obvious reasons. However we'll let h/w take
2436          * care of determining whether it's within the limits of what exists on
2437          * the SoC.
2438          */
2439         if (cgr->cgrid >= __CGR_NUM)
2440                 return -EINVAL;
2441
2442         p = get_affine_portal();
2443
2444         memset(&local_opts, 0, sizeof(struct qm_mcc_initcgr));
2445         cgr->chan = p->config->channel;
2446         spin_lock(&p->cgr_lock);
2447
2448         /* if no opts specified, just add it to the list */
2449         if (!opts)
2450                 goto add_list;
2451
2452         ret = qman_query_cgr(cgr, &cgr_state);
2453         if (ret)
2454                 goto release_lock;
2455         if (opts)
2456                 local_opts = *opts;
2457         if ((qman_ip_rev & 0xFF00) >= QMAN_REV30)
2458                 local_opts.cgr.cscn_targ_upd_ctrl =
2459                         QM_CGR_TARG_UDP_CTRL_WRITE_BIT | PORTAL_IDX(p);
2460         else
2461                 /* Overwrite TARG */
2462                 local_opts.cgr.cscn_targ = cgr_state.cgr.cscn_targ |
2463                                                         TARG_MASK(p);
2464         local_opts.we_mask |= QM_CGR_WE_CSCN_TARG;
2465
2466         /* send init if flags indicate so */
2467         if (opts && (flags & QMAN_CGR_FLAG_USE_INIT))
2468                 ret = qman_modify_cgr(cgr, QMAN_CGR_FLAG_USE_INIT, &local_opts);
2469         else
2470                 ret = qman_modify_cgr(cgr, 0, &local_opts);
2471         if (ret)
2472                 goto release_lock;
2473 add_list:
2474         list_add(&cgr->node, &p->cgr_cbs);
2475
2476         /* Determine if newly added object requires its callback to be called */
2477         ret = qman_query_cgr(cgr, &cgr_state);
2478         if (ret) {
2479                 /* we can't go back, so proceed and return success, but screen
2480                  * and wail to the log file.
2481                  */
2482                 pr_crit("CGR HW state partially modified\n");
2483                 ret = 0;
2484                 goto release_lock;
2485         }
2486         if (cgr->cb && cgr_state.cgr.cscn_en && qman_cgrs_get(&p->cgrs[1],
2487                                                               cgr->cgrid))
2488                 cgr->cb(p, cgr, 1);
2489 release_lock:
2490         spin_unlock(&p->cgr_lock);
2491         return ret;
2492 }
2493
2494 int qman_create_cgr_to_dcp(struct qman_cgr *cgr, u32 flags, u16 dcp_portal,
2495                            struct qm_mcc_initcgr *opts)
2496 {
2497         struct qm_mcc_initcgr local_opts;
2498         struct qm_mcr_querycgr cgr_state;
2499         int ret;
2500
2501         if ((qman_ip_rev & 0xFF00) < QMAN_REV30) {
2502                 pr_warn("QMan version doesn't support CSCN => DCP portal\n");
2503                 return -EINVAL;
2504         }
2505         /* We have to check that the provided CGRID is within the limits of the
2506          * data-structures, for obvious reasons. However we'll let h/w take
2507          * care of determining whether it's within the limits of what exists on
2508          * the SoC.
2509          */
2510         if (cgr->cgrid >= __CGR_NUM)
2511                 return -EINVAL;
2512
2513         ret = qman_query_cgr(cgr, &cgr_state);
2514         if (ret)
2515                 return ret;
2516
2517         memset(&local_opts, 0, sizeof(struct qm_mcc_initcgr));
2518         if (opts)
2519                 local_opts = *opts;
2520
2521         if ((qman_ip_rev & 0xFF00) >= QMAN_REV30)
2522                 local_opts.cgr.cscn_targ_upd_ctrl =
2523                                 QM_CGR_TARG_UDP_CTRL_WRITE_BIT |
2524                                 QM_CGR_TARG_UDP_CTRL_DCP | dcp_portal;
2525         else
2526                 local_opts.cgr.cscn_targ = cgr_state.cgr.cscn_targ |
2527                                         TARG_DCP_MASK(dcp_portal);
2528         local_opts.we_mask |= QM_CGR_WE_CSCN_TARG;
2529
2530         /* send init if flags indicate so */
2531         if (opts && (flags & QMAN_CGR_FLAG_USE_INIT))
2532                 ret = qman_modify_cgr(cgr, QMAN_CGR_FLAG_USE_INIT,
2533                                       &local_opts);
2534         else
2535                 ret = qman_modify_cgr(cgr, 0, &local_opts);
2536
2537         return ret;
2538 }
2539
2540 int qman_delete_cgr(struct qman_cgr *cgr)
2541 {
2542         struct qm_mcr_querycgr cgr_state;
2543         struct qm_mcc_initcgr local_opts;
2544         int ret = 0;
2545         struct qman_cgr *i;
2546         struct qman_portal *p = get_affine_portal();
2547
2548         if (cgr->chan != p->config->channel) {
2549                 pr_crit("Attempting to delete cgr from different portal than"
2550                         " it was create: create 0x%x, delete 0x%x\n",
2551                         cgr->chan, p->config->channel);
2552                 ret = -EINVAL;
2553                 goto put_portal;
2554         }
2555         memset(&local_opts, 0, sizeof(struct qm_mcc_initcgr));
2556         spin_lock(&p->cgr_lock);
2557         list_del(&cgr->node);
2558         /*
2559          * If there are no other CGR objects for this CGRID in the list,
2560          * update CSCN_TARG accordingly
2561          */
2562         list_for_each_entry(i, &p->cgr_cbs, node)
2563                 if ((i->cgrid == cgr->cgrid) && i->cb)
2564                         goto release_lock;
2565         ret = qman_query_cgr(cgr, &cgr_state);
2566         if (ret)  {
2567                 /* add back to the list */
2568                 list_add(&cgr->node, &p->cgr_cbs);
2569                 goto release_lock;
2570         }
2571         /* Overwrite TARG */
2572         local_opts.we_mask = QM_CGR_WE_CSCN_TARG;
2573         if ((qman_ip_rev & 0xFF00) >= QMAN_REV30)
2574                 local_opts.cgr.cscn_targ_upd_ctrl = PORTAL_IDX(p);
2575         else
2576                 local_opts.cgr.cscn_targ = cgr_state.cgr.cscn_targ &
2577                                                          ~(TARG_MASK(p));
2578         ret = qman_modify_cgr(cgr, 0, &local_opts);
2579         if (ret)
2580                 /* add back to the list */
2581                 list_add(&cgr->node, &p->cgr_cbs);
2582 release_lock:
2583         spin_unlock(&p->cgr_lock);
2584 put_portal:
2585         return ret;
2586 }
2587
2588 int qman_shutdown_fq(u32 fqid)
2589 {
2590         struct qman_portal *p;
2591         struct qm_portal *low_p;
2592         struct qm_mc_command *mcc;
2593         struct qm_mc_result *mcr;
2594         u8 state;
2595         int orl_empty, fq_empty, drain = 0;
2596         u32 result;
2597         u32 channel, wq;
2598         u16 dest_wq;
2599
2600         p = get_affine_portal();
2601         low_p = &p->p;
2602
2603         /* Determine the state of the FQID */
2604         mcc = qm_mc_start(low_p);
2605         mcc->queryfq_np.fqid = cpu_to_be32(fqid);
2606         qm_mc_commit(low_p, QM_MCC_VERB_QUERYFQ_NP);
2607         while (!(mcr = qm_mc_result(low_p)))
2608                 cpu_relax();
2609         DPAA_ASSERT((mcr->verb & QM_MCR_VERB_MASK) == QM_MCR_VERB_QUERYFQ_NP);
2610         state = mcr->queryfq_np.state & QM_MCR_NP_STATE_MASK;
2611         if (state == QM_MCR_NP_STATE_OOS)
2612                 return 0; /* Already OOS, no need to do anymore checks */
2613
2614         /* Query which channel the FQ is using */
2615         mcc = qm_mc_start(low_p);
2616         mcc->queryfq.fqid = cpu_to_be32(fqid);
2617         qm_mc_commit(low_p, QM_MCC_VERB_QUERYFQ);
2618         while (!(mcr = qm_mc_result(low_p)))
2619                 cpu_relax();
2620         DPAA_ASSERT((mcr->verb & QM_MCR_VERB_MASK) == QM_MCR_VERB_QUERYFQ);
2621
2622         /* Need to store these since the MCR gets reused */
2623         dest_wq = be16_to_cpu(mcr->queryfq.fqd.dest_wq);
2624         channel = dest_wq & 0x7;
2625         wq = dest_wq >> 3;
2626
2627         switch (state) {
2628         case QM_MCR_NP_STATE_TEN_SCHED:
2629         case QM_MCR_NP_STATE_TRU_SCHED:
2630         case QM_MCR_NP_STATE_ACTIVE:
2631         case QM_MCR_NP_STATE_PARKED:
2632                 orl_empty = 0;
2633                 mcc = qm_mc_start(low_p);
2634                 mcc->alterfq.fqid = cpu_to_be32(fqid);
2635                 qm_mc_commit(low_p, QM_MCC_VERB_ALTER_RETIRE);
2636                 while (!(mcr = qm_mc_result(low_p)))
2637                         cpu_relax();
2638                 DPAA_ASSERT((mcr->verb & QM_MCR_VERB_MASK) ==
2639                            QM_MCR_VERB_ALTER_RETIRE);
2640                 result = mcr->result; /* Make a copy as we reuse MCR below */
2641
2642                 if (result == QM_MCR_RESULT_PENDING) {
2643                         /* Need to wait for the FQRN in the message ring, which
2644                          * will only occur once the FQ has been drained.  In
2645                          * order for the FQ to drain the portal needs to be set
2646                          * to dequeue from the channel the FQ is scheduled on
2647                          */
2648                         const struct qm_mr_entry *msg;
2649                         const struct qm_dqrr_entry *dqrr = NULL;
2650                         int found_fqrn = 0;
2651                         __maybe_unused u16 dequeue_wq = 0;
2652
2653                         /* Flag that we need to drain FQ */
2654                         drain = 1;
2655
2656                         if (channel >= qm_channel_pool1 &&
2657                             channel < (u16)(qm_channel_pool1 + 15)) {
2658                                 /* Pool channel, enable the bit in the portal */
2659                                 dequeue_wq = (channel -
2660                                               qm_channel_pool1 + 1) << 4 | wq;
2661                         } else if (channel < qm_channel_pool1) {
2662                                 /* Dedicated channel */
2663                                 dequeue_wq = wq;
2664                         } else {
2665                                 pr_info("Cannot recover FQ 0x%x,"
2666                                         " it is scheduled on channel 0x%x",
2667                                         fqid, channel);
2668                                 return -EBUSY;
2669                         }
2670                         /* Set the sdqcr to drain this channel */
2671                         if (channel < qm_channel_pool1)
2672                                 qm_dqrr_sdqcr_set(low_p,
2673                                                   QM_SDQCR_TYPE_ACTIVE |
2674                                           QM_SDQCR_CHANNELS_DEDICATED);
2675                         else
2676                                 qm_dqrr_sdqcr_set(low_p,
2677                                                   QM_SDQCR_TYPE_ACTIVE |
2678                                                   QM_SDQCR_CHANNELS_POOL_CONV
2679                                                   (channel));
2680                         while (!found_fqrn) {
2681                                 /* Keep draining DQRR while checking the MR*/
2682                                 qm_dqrr_pvb_update(low_p);
2683                                 dqrr = qm_dqrr_current(low_p);
2684                                 while (dqrr) {
2685                                         qm_dqrr_cdc_consume_1ptr(
2686                                                 low_p, dqrr, 0);
2687                                         qm_dqrr_pvb_update(low_p);
2688                                         qm_dqrr_next(low_p);
2689                                         dqrr = qm_dqrr_current(low_p);
2690                                 }
2691                                 /* Process message ring too */
2692                                 qm_mr_pvb_update(low_p);
2693                                 msg = qm_mr_current(low_p);
2694                                 while (msg) {
2695                                         if ((msg->ern.verb &
2696                                              QM_MR_VERB_TYPE_MASK)
2697                                             == QM_MR_VERB_FQRN)
2698                                                 found_fqrn = 1;
2699                                         qm_mr_next(low_p);
2700                                         qm_mr_cci_consume_to_current(low_p);
2701                                         qm_mr_pvb_update(low_p);
2702                                         msg = qm_mr_current(low_p);
2703                                 }
2704                                 cpu_relax();
2705                         }
2706                 }
2707                 if (result != QM_MCR_RESULT_OK &&
2708                     result !=  QM_MCR_RESULT_PENDING) {
2709                         /* error */
2710                         pr_err("qman_retire_fq failed on FQ 0x%x,"
2711                                " result=0x%x\n", fqid, result);
2712                         return -1;
2713                 }
2714                 if (!(mcr->alterfq.fqs & QM_MCR_FQS_ORLPRESENT)) {
2715                         /* ORL had no entries, no need to wait until the
2716                          * ERNs come in.
2717                          */
2718                         orl_empty = 1;
2719                 }
2720                 /* Retirement succeeded, check to see if FQ needs
2721                  * to be drained.
2722                  */
2723                 if (drain || mcr->alterfq.fqs & QM_MCR_FQS_NOTEMPTY) {
2724                         /* FQ is Not Empty, drain using volatile DQ commands */
2725                         fq_empty = 0;
2726                         do {
2727                                 const struct qm_dqrr_entry *dqrr = NULL;
2728                                 u32 vdqcr = fqid | QM_VDQCR_NUMFRAMES_SET(3);
2729
2730                                 qm_dqrr_vdqcr_set(low_p, vdqcr);
2731
2732                                 /* Wait for a dequeue to occur */
2733                                 while (dqrr == NULL) {
2734                                         qm_dqrr_pvb_update(low_p);
2735                                         dqrr = qm_dqrr_current(low_p);
2736                                         if (!dqrr)
2737                                                 cpu_relax();
2738                                 }
2739                                 /* Process the dequeues, making sure to
2740                                  * empty the ring completely.
2741                                  */
2742                                 while (dqrr) {
2743                                         if (dqrr->fqid == fqid &&
2744                                             dqrr->stat & QM_DQRR_STAT_FQ_EMPTY)
2745                                                 fq_empty = 1;
2746                                         qm_dqrr_cdc_consume_1ptr(low_p,
2747                                                                  dqrr, 0);
2748                                         qm_dqrr_pvb_update(low_p);
2749                                         qm_dqrr_next(low_p);
2750                                         dqrr = qm_dqrr_current(low_p);
2751                                 }
2752                         } while (fq_empty == 0);
2753                 }
2754                 qm_dqrr_sdqcr_set(low_p, 0);
2755
2756                 /* Wait for the ORL to have been completely drained */
2757                 while (orl_empty == 0) {
2758                         const struct qm_mr_entry *msg;
2759
2760                         qm_mr_pvb_update(low_p);
2761                         msg = qm_mr_current(low_p);
2762                         while (msg) {
2763                                 if ((msg->ern.verb & QM_MR_VERB_TYPE_MASK) ==
2764                                     QM_MR_VERB_FQRL)
2765                                         orl_empty = 1;
2766                                 qm_mr_next(low_p);
2767                                 qm_mr_cci_consume_to_current(low_p);
2768                                 qm_mr_pvb_update(low_p);
2769                                 msg = qm_mr_current(low_p);
2770                         }
2771                         cpu_relax();
2772                 }
2773                 mcc = qm_mc_start(low_p);
2774                 mcc->alterfq.fqid = cpu_to_be32(fqid);
2775                 qm_mc_commit(low_p, QM_MCC_VERB_ALTER_OOS);
2776                 while (!(mcr = qm_mc_result(low_p)))
2777                         cpu_relax();
2778                 DPAA_ASSERT((mcr->verb & QM_MCR_VERB_MASK) ==
2779                            QM_MCR_VERB_ALTER_OOS);
2780                 if (mcr->result != QM_MCR_RESULT_OK) {
2781                         pr_err(
2782                         "OOS after drain Failed on FQID 0x%x, result 0x%x\n",
2783                                fqid, mcr->result);
2784                         return -1;
2785                 }
2786                 return 0;
2787
2788         case QM_MCR_NP_STATE_RETIRED:
2789                 /* Send OOS Command */
2790                 mcc = qm_mc_start(low_p);
2791                 mcc->alterfq.fqid = cpu_to_be32(fqid);
2792                 qm_mc_commit(low_p, QM_MCC_VERB_ALTER_OOS);
2793                 while (!(mcr = qm_mc_result(low_p)))
2794                         cpu_relax();
2795                 DPAA_ASSERT((mcr->verb & QM_MCR_VERB_MASK) ==
2796                            QM_MCR_VERB_ALTER_OOS);
2797                 if (mcr->result) {
2798                         pr_err("OOS Failed on FQID 0x%x\n", fqid);
2799                         return -1;
2800                 }
2801                 return 0;
2802
2803         }
2804         return -1;
2805 }