drivers/bus/dpaa/base/qbman/qman.c

   1 /* SPDX-License-Identifier: (BSD-3-Clause OR GPL-2.0)
   2  *
   3  * Copyright 2008-2016 Freescale Semiconductor Inc.
   4  * Copyright 2017 NXP
   5  *
   6  */
   7
   8 #include "qman.h"
   9 #include <rte_branch_prediction.h>
  10 #include <rte_dpaa_bus.h>
  11
  12 /* Compilation constants */
  13 #define DQRR_MAXFILL    15
  14 #define EQCR_ITHRESH    4       /* if EQCR congests, interrupt threshold */
  15 #define IRQNAME         "QMan portal %d"
  16 #define MAX_IRQNAME     16      /* big enough for "QMan portal %d" */
  17 /* maximum number of DQRR entries to process in qman_poll() */
  18 #define FSL_QMAN_POLL_LIMIT 8
  19
  20 /* Lock/unlock frame queues, subject to the "LOCKED" flag. This is about
  21  * inter-processor locking only. Note, FQLOCK() is always called either under a
  22  * local_irq_save() or from interrupt context - hence there's no need for irq
  23  * protection (and indeed, attempting to nest irq-protection doesn't work, as
  24  * the "irq en/disable" machinery isn't recursive...).
  25  */
  26 #define FQLOCK(fq) \
  27         do { \
  28                 struct qman_fq *__fq478 = (fq); \
  29                 if (fq_isset(__fq478, QMAN_FQ_FLAG_LOCKED)) \
  30                         spin_lock(&__fq478->fqlock); \
  31         } while (0)
  32 #define FQUNLOCK(fq) \
  33         do { \
  34                 struct qman_fq *__fq478 = (fq); \
  35                 if (fq_isset(__fq478, QMAN_FQ_FLAG_LOCKED)) \
  36                         spin_unlock(&__fq478->fqlock); \
  37         } while (0)
  38
  39 static inline void fq_set(struct qman_fq *fq, u32 mask)
  40 {
  41         dpaa_set_bits(mask, &fq->flags);
  42 }
  43
  44 static inline void fq_clear(struct qman_fq *fq, u32 mask)
  45 {
  46         dpaa_clear_bits(mask, &fq->flags);
  47 }
  48
  49 static inline int fq_isset(struct qman_fq *fq, u32 mask)
  50 {
  51         return fq->flags & mask;
  52 }
  53
  54 static inline int fq_isclear(struct qman_fq *fq, u32 mask)
  55 {
  56         return !(fq->flags & mask);
  57 }
  58
  59 struct qman_portal {
  60         struct qm_portal p;
  61         /* PORTAL_BITS_*** - dynamic, strictly internal */
  62         unsigned long bits;
  63         /* interrupt sources processed by portal_isr(), configurable */
  64         unsigned long irq_sources;
  65         u32 use_eqcr_ci_stashing;
  66         u32 slowpoll;   /* only used when interrupts are off */
  67         /* only 1 volatile dequeue at a time */
  68         struct qman_fq *vdqcr_owned;
  69         u32 sdqcr;
  70         int dqrr_disable_ref;
  71         /* A portal-specific handler for DCP ERNs. If this is NULL, the global
  72          * handler is called instead.
  73          */
  74         qman_cb_dc_ern cb_dc_ern;
  75         /* When the cpu-affine portal is activated, this is non-NULL */
  76         const struct qm_portal_config *config;
  77         struct dpa_rbtree retire_table;
  78         char irqname[MAX_IRQNAME];
  79         /* 2-element array. cgrs[0] is mask, cgrs[1] is snapshot. */
  80         struct qman_cgrs *cgrs;
  81         /* linked-list of CSCN handlers. */
  82         struct list_head cgr_cbs;
  83         /* list lock */
  84         spinlock_t cgr_lock;
  85         /* track if memory was allocated by the driver */
  86 #if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__
  87         /* Keep a shadow copy of the DQRR on LE systems as the SW needs to
  88          * do byte swaps of DQRR read only memory.  First entry must be aligned
  89          * to 2 ** 10 to ensure DQRR index calculations based shadow copy
  90          * address (6 bits for address shift + 4 bits for the DQRR size).
  91          */
  92         struct qm_dqrr_entry shadow_dqrr[QM_DQRR_SIZE]
  93                     __attribute__((aligned(1024)));
  94 #endif
  95 };
  96
  97 /* Global handler for DCP ERNs. Used when the portal receiving the message does
  98  * not have a portal-specific handler.
  99  */
 100 static qman_cb_dc_ern cb_dc_ern;
 101
 102 static cpumask_t affine_mask;
 103 static DEFINE_SPINLOCK(affine_mask_lock);
 104 static u16 affine_channels[NR_CPUS];
 105 static RTE_DEFINE_PER_LCORE(struct qman_portal, qman_affine_portal);
 106
 107 static inline struct qman_portal *get_affine_portal(void)
 108 {
 109         return &RTE_PER_LCORE(qman_affine_portal);
 110 }
 111
 112 /* This gives a FQID->FQ lookup to cover the fact that we can't directly demux
 113  * retirement notifications (the fact they are sometimes h/w-consumed means that
 114  * contextB isn't always a s/w demux - and as we can't know which case it is
 115  * when looking at the notification, we have to use the slow lookup for all of
 116  * them). NB, it's possible to have multiple FQ objects refer to the same FQID
 117  * (though at most one of them should be the consumer), so this table isn't for
 118  * all FQs - FQs are added when retirement commands are issued, and removed when
 119  * they complete, which also massively reduces the size of this table.
 120  */
 121 IMPLEMENT_DPAA_RBTREE(fqtree, struct qman_fq, node, fqid);
 122 /*
 123  * This is what everything can wait on, even if it migrates to a different cpu
 124  * to the one whose affine portal it is waiting on.
 125  */
 126 static DECLARE_WAIT_QUEUE_HEAD(affine_queue);
 127
 128 static inline int table_push_fq(struct qman_portal *p, struct qman_fq *fq)
 129 {
 130         int ret = fqtree_push(&p->retire_table, fq);
 131
 132         if (ret)
 133                 pr_err("ERROR: double FQ-retirement %d\n", fq->fqid);
 134         return ret;
 135 }
 136
 137 static inline void table_del_fq(struct qman_portal *p, struct qman_fq *fq)
 138 {
 139         fqtree_del(&p->retire_table, fq);
 140 }
 141
 142 static inline struct qman_fq *table_find_fq(struct qman_portal *p, u32 fqid)
 143 {
 144         return fqtree_find(&p->retire_table, fqid);
 145 }
 146
 147 #ifdef CONFIG_FSL_QMAN_FQ_LOOKUP
 148 static void **qman_fq_lookup_table;
 149 static size_t qman_fq_lookup_table_size;
 150
 151 int qman_setup_fq_lookup_table(size_t num_entries)
 152 {
 153         num_entries++;
 154         /* Allocate 1 more entry since the first entry is not used */
 155         qman_fq_lookup_table = vmalloc((num_entries * sizeof(void *)));
 156         if (!qman_fq_lookup_table) {
 157                 pr_err("QMan: Could not allocate fq lookup table\n");
 158                 return -ENOMEM;
 159         }
 160         memset(qman_fq_lookup_table, 0, num_entries * sizeof(void *));
 161         qman_fq_lookup_table_size = num_entries;
 162         pr_debug("QMan: Allocated lookup table at %p, entry count %lu\n",
 163                 qman_fq_lookup_table,
 164                         (unsigned long)qman_fq_lookup_table_size);
 165         return 0;
 166 }
 167
 168 /* global structure that maintains fq object mapping */
 169 static DEFINE_SPINLOCK(fq_hash_table_lock);
 170
 171 static int find_empty_fq_table_entry(u32 *entry, struct qman_fq *fq)
 172 {
 173         u32 i;
 174
 175         spin_lock(&fq_hash_table_lock);
 176         /* Can't use index zero because this has special meaning
 177          * in context_b field.
 178          */
 179         for (i = 1; i < qman_fq_lookup_table_size; i++) {
 180                 if (qman_fq_lookup_table[i] == NULL) {
 181                         *entry = i;
 182                         qman_fq_lookup_table[i] = fq;
 183                         spin_unlock(&fq_hash_table_lock);
 184                         return 0;
 185                 }
 186         }
 187         spin_unlock(&fq_hash_table_lock);
 188         return -ENOMEM;
 189 }
 190
 191 static void clear_fq_table_entry(u32 entry)
 192 {
 193         spin_lock(&fq_hash_table_lock);
 194         DPAA_BUG_ON(entry >= qman_fq_lookup_table_size);
 195         qman_fq_lookup_table[entry] = NULL;
 196         spin_unlock(&fq_hash_table_lock);
 197 }
 198
 199 static inline struct qman_fq *get_fq_table_entry(u32 entry)
 200 {
 201         DPAA_BUG_ON(entry >= qman_fq_lookup_table_size);
 202         return qman_fq_lookup_table[entry];
 203 }
 204 #endif
 205
 206 static inline void cpu_to_hw_fqd(struct qm_fqd *fqd)
 207 {
 208         /* Byteswap the FQD to HW format */
 209         fqd->fq_ctrl = cpu_to_be16(fqd->fq_ctrl);
 210         fqd->dest_wq = cpu_to_be16(fqd->dest_wq);
 211         fqd->ics_cred = cpu_to_be16(fqd->ics_cred);
 212         fqd->context_b = cpu_to_be32(fqd->context_b);
 213         fqd->context_a.opaque = cpu_to_be64(fqd->context_a.opaque);
 214         fqd->opaque_td = cpu_to_be16(fqd->opaque_td);
 215 }
 216
 217 static inline void hw_fqd_to_cpu(struct qm_fqd *fqd)
 218 {
 219         /* Byteswap the FQD to CPU format */
 220         fqd->fq_ctrl = be16_to_cpu(fqd->fq_ctrl);
 221         fqd->dest_wq = be16_to_cpu(fqd->dest_wq);
 222         fqd->ics_cred = be16_to_cpu(fqd->ics_cred);
 223         fqd->context_b = be32_to_cpu(fqd->context_b);
 224         fqd->context_a.opaque = be64_to_cpu(fqd->context_a.opaque);
 225 }
 226
 227 static inline void cpu_to_hw_fd(struct qm_fd *fd)
 228 {
 229         fd->addr = cpu_to_be40(fd->addr);
 230         fd->status = cpu_to_be32(fd->status);
 231         fd->opaque = cpu_to_be32(fd->opaque);
 232 }
 233
 234 static inline void hw_fd_to_cpu(struct qm_fd *fd)
 235 {
 236         fd->addr = be40_to_cpu(fd->addr);
 237         fd->status = be32_to_cpu(fd->status);
 238         fd->opaque = be32_to_cpu(fd->opaque);
 239 }
 240
 241 /* In the case that slow- and fast-path handling are both done by qman_poll()
 242  * (ie. because there is no interrupt handling), we ought to balance how often
 243  * we do the fast-path poll versus the slow-path poll. We'll use two decrementer
 244  * sources, so we call the fast poll 'n' times before calling the slow poll
 245  * once. The idle decrementer constant is used when the last slow-poll detected
 246  * no work to do, and the busy decrementer constant when the last slow-poll had
 247  * work to do.
 248  */
 249 #define SLOW_POLL_IDLE   1000
 250 #define SLOW_POLL_BUSY   10
 251 static u32 __poll_portal_slow(struct qman_portal *p, u32 is);
 252 static inline unsigned int __poll_portal_fast(struct qman_portal *p,
 253                                               unsigned int poll_limit);
 254
 255 /* Portal interrupt handler */
 256 static irqreturn_t portal_isr(__always_unused int irq, void *ptr)
 257 {
 258         struct qman_portal *p = ptr;
 259         /*
 260          * The CSCI/CCSCI source is cleared inside __poll_portal_slow(), because
 261          * it could race against a Query Congestion State command also given
 262          * as part of the handling of this interrupt source. We mustn't
 263          * clear it a second time in this top-level function.
 264          */
 265         u32 clear = QM_DQAVAIL_MASK | (p->irq_sources &
 266                 ~(QM_PIRQ_CSCI | QM_PIRQ_CCSCI));
 267         u32 is = qm_isr_status_read(&p->p) & p->irq_sources;
 268         /* DQRR-handling if it's interrupt-driven */
 269         if (is & QM_PIRQ_DQRI)
 270                 __poll_portal_fast(p, FSL_QMAN_POLL_LIMIT);
 271         /* Handling of anything else that's interrupt-driven */
 272         clear |= __poll_portal_slow(p, is);
 273         qm_isr_status_clear(&p->p, clear);
 274         return IRQ_HANDLED;
 275 }
 276
 277 /* This inner version is used privately by qman_create_affine_portal(), as well
 278  * as by the exported qman_stop_dequeues().
 279  */
 280 static inline void qman_stop_dequeues_ex(struct qman_portal *p)
 281 {
 282         if (!(p->dqrr_disable_ref++))
 283                 qm_dqrr_set_maxfill(&p->p, 0);
 284 }
 285
 286 static int drain_mr_fqrni(struct qm_portal *p)
 287 {
 288         const struct qm_mr_entry *msg;
 289 loop:
 290         msg = qm_mr_current(p);
 291         if (!msg) {
 292                 /*
 293                  * if MR was full and h/w had other FQRNI entries to produce, we
 294                  * need to allow it time to produce those entries once the
 295                  * existing entries are consumed. A worst-case situation
 296                  * (fully-loaded system) means h/w sequencers may have to do 3-4
 297                  * other things before servicing the portal's MR pump, each of
 298                  * which (if slow) may take ~50 qman cycles (which is ~200
 299                  * processor cycles). So rounding up and then multiplying this
 300                  * worst-case estimate by a factor of 10, just to be
 301                  * ultra-paranoid, goes as high as 10,000 cycles. NB, we consume
 302                  * one entry at a time, so h/w has an opportunity to produce new
 303                  * entries well before the ring has been fully consumed, so
 304                  * we're being *really* paranoid here.
 305                  */
 306                 u64 now, then = mfatb();
 307
 308                 do {
 309                         now = mfatb();
 310                 } while ((then + 10000) > now);
 311                 msg = qm_mr_current(p);
 312                 if (!msg)
 313                         return 0;
 314         }
 315         if ((msg->verb & QM_MR_VERB_TYPE_MASK) != QM_MR_VERB_FQRNI) {
 316                 /* We aren't draining anything but FQRNIs */
 317                 pr_err("Found verb 0x%x in MR\n", msg->verb);
 318                 return -1;
 319         }
 320         qm_mr_next(p);
 321         qm_mr_cci_consume(p, 1);
 322         goto loop;
 323 }
 324
 325 static inline int qm_eqcr_init(struct qm_portal *portal,
 326                                enum qm_eqcr_pmode pmode,
 327                                unsigned int eq_stash_thresh,
 328                                int eq_stash_prio)
 329 {
 330         /* This use of 'register', as well as all other occurrences, is because
 331          * it has been observed to generate much faster code with gcc than is
 332          * otherwise the case.
 333          */
 334         register struct qm_eqcr *eqcr = &portal->eqcr;
 335         u32 cfg;
 336         u8 pi;
 337
 338         eqcr->ring = portal->addr.ce + QM_CL_EQCR;
 339         eqcr->ci = qm_in(EQCR_CI_CINH) & (QM_EQCR_SIZE - 1);
 340         qm_cl_invalidate(EQCR_CI);
 341         pi = qm_in(EQCR_PI_CINH) & (QM_EQCR_SIZE - 1);
 342         eqcr->cursor = eqcr->ring + pi;
 343         eqcr->vbit = (qm_in(EQCR_PI_CINH) & QM_EQCR_SIZE) ?
 344                         QM_EQCR_VERB_VBIT : 0;
 345         eqcr->available = QM_EQCR_SIZE - 1 -
 346                         qm_cyc_diff(QM_EQCR_SIZE, eqcr->ci, pi);
 347         eqcr->ithresh = qm_in(EQCR_ITR);
 348 #ifdef RTE_LIBRTE_DPAA_HWDEBUG
 349         eqcr->busy = 0;
 350         eqcr->pmode = pmode;
 351 #endif
 352         cfg = (qm_in(CFG) & 0x00ffffff) |
 353                 (eq_stash_thresh << 28) | /* QCSP_CFG: EST */
 354                 (eq_stash_prio << 26)   | /* QCSP_CFG: EP */
 355                 ((pmode & 0x3) << 24);  /* QCSP_CFG::EPM */
 356         qm_out(CFG, cfg);
 357         return 0;
 358 }
 359
 360 static inline void qm_eqcr_finish(struct qm_portal *portal)
 361 {
 362         register struct qm_eqcr *eqcr = &portal->eqcr;
 363         u8 pi, ci;
 364         u32 cfg;
 365
 366         /*
 367          * Disable EQCI stashing because the QMan only
 368          * presents the value it previously stashed to
 369          * maintain coherency.  Setting the stash threshold
 370          * to 1 then 0 ensures that QMan has resyncronized
 371          * its internal copy so that the portal is clean
 372          * when it is reinitialized in the future
 373          */
 374         cfg = (qm_in(CFG) & 0x0fffffff) |
 375                 (1 << 28); /* QCSP_CFG: EST */
 376         qm_out(CFG, cfg);
 377         cfg &= 0x0fffffff; /* stash threshold = 0 */
 378         qm_out(CFG, cfg);
 379
 380         pi = qm_in(EQCR_PI_CINH) & (QM_EQCR_SIZE - 1);
 381         ci = qm_in(EQCR_CI_CINH) & (QM_EQCR_SIZE - 1);
 382
 383         /* Refresh EQCR CI cache value */
 384         qm_cl_invalidate(EQCR_CI);
 385         eqcr->ci = qm_cl_in(EQCR_CI) & (QM_EQCR_SIZE - 1);
 386
 387 #ifdef RTE_LIBRTE_DPAA_HWDEBUG
 388         DPAA_ASSERT(!eqcr->busy);
 389 #endif
 390         if (pi != EQCR_PTR2IDX(eqcr->cursor))
 391                 pr_crit("losing uncommitted EQCR entries\n");
 392         if (ci != eqcr->ci)
 393                 pr_crit("missing existing EQCR completions\n");
 394         if (eqcr->ci != EQCR_PTR2IDX(eqcr->cursor))
 395                 pr_crit("EQCR destroyed unquiesced\n");
 396 }
 397
 398 static inline int qm_dqrr_init(struct qm_portal *portal,
 399                         __maybe_unused const struct qm_portal_config *config,
 400                         enum qm_dqrr_dmode dmode,
 401                         __maybe_unused enum qm_dqrr_pmode pmode,
 402                         enum qm_dqrr_cmode cmode, u8 max_fill)
 403 {
 404         register struct qm_dqrr *dqrr = &portal->dqrr;
 405         u32 cfg;
 406
 407         /* Make sure the DQRR will be idle when we enable */
 408         qm_out(DQRR_SDQCR, 0);
 409         qm_out(DQRR_VDQCR, 0);
 410         qm_out(DQRR_PDQCR, 0);
 411         dqrr->ring = portal->addr.ce + QM_CL_DQRR;
 412         dqrr->pi = qm_in(DQRR_PI_CINH) & (QM_DQRR_SIZE - 1);
 413         dqrr->ci = qm_in(DQRR_CI_CINH) & (QM_DQRR_SIZE - 1);
 414         dqrr->cursor = dqrr->ring + dqrr->ci;
 415         dqrr->fill = qm_cyc_diff(QM_DQRR_SIZE, dqrr->ci, dqrr->pi);
 416         dqrr->vbit = (qm_in(DQRR_PI_CINH) & QM_DQRR_SIZE) ?
 417                         QM_DQRR_VERB_VBIT : 0;
 418         dqrr->ithresh = qm_in(DQRR_ITR);
 419 #ifdef RTE_LIBRTE_DPAA_HWDEBUG
 420         dqrr->dmode = dmode;
 421         dqrr->pmode = pmode;
 422         dqrr->cmode = cmode;
 423 #endif
 424         /* Invalidate every ring entry before beginning */
 425         for (cfg = 0; cfg < QM_DQRR_SIZE; cfg++)
 426                 dccivac(qm_cl(dqrr->ring, cfg));
 427         cfg = (qm_in(CFG) & 0xff000f00) |
 428                 ((max_fill & (QM_DQRR_SIZE - 1)) << 20) | /* DQRR_MF */
 429                 ((dmode & 1) << 18) |                   /* DP */
 430                 ((cmode & 3) << 16) |                   /* DCM */
 431                 0xa0 |                                  /* RE+SE */
 432                 (0 ? 0x40 : 0) |                        /* Ignore RP */
 433                 (0 ? 0x10 : 0);                         /* Ignore SP */
 434         qm_out(CFG, cfg);
 435         qm_dqrr_set_maxfill(portal, max_fill);
 436         return 0;
 437 }
 438
 439 static inline void qm_dqrr_finish(struct qm_portal *portal)
 440 {
 441         __maybe_unused register struct qm_dqrr *dqrr = &portal->dqrr;
 442 #ifdef RTE_LIBRTE_DPAA_HWDEBUG
 443         if ((dqrr->cmode != qm_dqrr_cdc) &&
 444             (dqrr->ci != DQRR_PTR2IDX(dqrr->cursor)))
 445                 pr_crit("Ignoring completed DQRR entries\n");
 446 #endif
 447 }
 448
 449 static inline int qm_mr_init(struct qm_portal *portal,
 450                              __maybe_unused enum qm_mr_pmode pmode,
 451                              enum qm_mr_cmode cmode)
 452 {
 453         register struct qm_mr *mr = &portal->mr;
 454         u32 cfg;
 455
 456         mr->ring = portal->addr.ce + QM_CL_MR;
 457         mr->pi = qm_in(MR_PI_CINH) & (QM_MR_SIZE - 1);
 458         mr->ci = qm_in(MR_CI_CINH) & (QM_MR_SIZE - 1);
 459         mr->cursor = mr->ring + mr->ci;
 460         mr->fill = qm_cyc_diff(QM_MR_SIZE, mr->ci, mr->pi);
 461         mr->vbit = (qm_in(MR_PI_CINH) & QM_MR_SIZE) ? QM_MR_VERB_VBIT : 0;
 462         mr->ithresh = qm_in(MR_ITR);
 463 #ifdef RTE_LIBRTE_DPAA_HWDEBUG
 464         mr->pmode = pmode;
 465         mr->cmode = cmode;
 466 #endif
 467         cfg = (qm_in(CFG) & 0xfffff0ff) |
 468                 ((cmode & 1) << 8);             /* QCSP_CFG:MM */
 469         qm_out(CFG, cfg);
 470         return 0;
 471 }
 472
 473 static inline void qm_mr_pvb_update(struct qm_portal *portal)
 474 {
 475         register struct qm_mr *mr = &portal->mr;
 476         const struct qm_mr_entry *res = qm_cl(mr->ring, mr->pi);
 477
 478 #ifdef RTE_LIBRTE_DPAA_HWDEBUG
 479         DPAA_ASSERT(mr->pmode == qm_mr_pvb);
 480 #endif
 481         /* when accessing 'verb', use __raw_readb() to ensure that compiler
 482          * inlining doesn't try to optimise out "excess reads".
 483          */
 484         if ((__raw_readb(&res->verb) & QM_MR_VERB_VBIT) == mr->vbit) {
 485                 mr->pi = (mr->pi + 1) & (QM_MR_SIZE - 1);
 486                 if (!mr->pi)
 487                         mr->vbit ^= QM_MR_VERB_VBIT;
 488                 mr->fill++;
 489                 res = MR_INC(res);
 490         }
 491         dcbit_ro(res);
 492 }
 493
 494 static inline
 495 struct qman_portal *qman_create_portal(
 496                         struct qman_portal *portal,
 497                               const struct qm_portal_config *c,
 498                               const struct qman_cgrs *cgrs)
 499 {
 500         struct qm_portal *p;
 501         char buf[16];
 502         int ret;
 503         u32 isdr;
 504
 505         p = &portal->p;
 506
 507         if (dpaa_svr_family == SVR_LS1043A_FAMILY)
 508                 portal->use_eqcr_ci_stashing = 3;
 509         else
 510                 portal->use_eqcr_ci_stashing =
 511                                         ((qman_ip_rev >= QMAN_REV30) ? 1 : 0);
 512
 513         /*
 514          * prep the low-level portal struct with the mapped addresses from the
 515          * config, everything that follows depends on it and "config" is more
 516          * for (de)reference
 517          */
 518         p->addr.ce = c->addr_virt[DPAA_PORTAL_CE];
 519         p->addr.ci = c->addr_virt[DPAA_PORTAL_CI];
 520         /*
 521          * If CI-stashing is used, the current defaults use a threshold of 3,
 522          * and stash with high-than-DQRR priority.
 523          */
 524         if (qm_eqcr_init(p, qm_eqcr_pvb,
 525                          portal->use_eqcr_ci_stashing, 1)) {
 526                 pr_err("Qman EQCR initialisation failed\n");
 527                 goto fail_eqcr;
 528         }
 529         if (qm_dqrr_init(p, c, qm_dqrr_dpush, qm_dqrr_pvb,
 530                          qm_dqrr_cdc, DQRR_MAXFILL)) {
 531                 pr_err("Qman DQRR initialisation failed\n");
 532                 goto fail_dqrr;
 533         }
 534         if (qm_mr_init(p, qm_mr_pvb, qm_mr_cci)) {
 535                 pr_err("Qman MR initialisation failed\n");
 536                 goto fail_mr;
 537         }
 538         if (qm_mc_init(p)) {
 539                 pr_err("Qman MC initialisation failed\n");
 540                 goto fail_mc;
 541         }
 542
 543         /* static interrupt-gating controls */
 544         qm_dqrr_set_ithresh(p, 0);
 545         qm_mr_set_ithresh(p, 0);
 546         qm_isr_set_iperiod(p, 0);
 547         portal->cgrs = kmalloc(2 * sizeof(*cgrs), GFP_KERNEL);
 548         if (!portal->cgrs)
 549                 goto fail_cgrs;
 550         /* initial snapshot is no-depletion */
 551         qman_cgrs_init(&portal->cgrs[1]);
 552         if (cgrs)
 553                 portal->cgrs[0] = *cgrs;
 554         else
 555                 /* if the given mask is NULL, assume all CGRs can be seen */
 556                 qman_cgrs_fill(&portal->cgrs[0]);
 557         INIT_LIST_HEAD(&portal->cgr_cbs);
 558         spin_lock_init(&portal->cgr_lock);
 559         portal->bits = 0;
 560         portal->slowpoll = 0;
 561         portal->sdqcr = QM_SDQCR_SOURCE_CHANNELS | QM_SDQCR_COUNT_UPTO3 |
 562                         QM_SDQCR_DEDICATED_PRECEDENCE | QM_SDQCR_TYPE_PRIO_QOS |
 563                         QM_SDQCR_TOKEN_SET(0xab) | QM_SDQCR_CHANNELS_DEDICATED;
 564         portal->dqrr_disable_ref = 0;
 565         portal->cb_dc_ern = NULL;
 566         sprintf(buf, "qportal-%d", c->channel);
 567         dpa_rbtree_init(&portal->retire_table);
 568         isdr = 0xffffffff;
 569         qm_isr_disable_write(p, isdr);
 570         portal->irq_sources = 0;
 571         qm_isr_enable_write(p, portal->irq_sources);
 572         qm_isr_status_clear(p, 0xffffffff);
 573         snprintf(portal->irqname, MAX_IRQNAME, IRQNAME, c->cpu);
 574         if (request_irq(c->irq, portal_isr, 0, portal->irqname,
 575                         portal)) {
 576                 pr_err("request_irq() failed\n");
 577                 goto fail_irq;
 578         }
 579
 580         /* Need EQCR to be empty before continuing */
 581         isdr &= ~QM_PIRQ_EQCI;
 582         qm_isr_disable_write(p, isdr);
 583         ret = qm_eqcr_get_fill(p);
 584         if (ret) {
 585                 pr_err("Qman EQCR unclean\n");
 586                 goto fail_eqcr_empty;
 587         }
 588         isdr &= ~(QM_PIRQ_DQRI | QM_PIRQ_MRI);
 589         qm_isr_disable_write(p, isdr);
 590         if (qm_dqrr_current(p)) {
 591                 pr_err("Qman DQRR unclean\n");
 592                 qm_dqrr_cdc_consume_n(p, 0xffff);
 593         }
 594         if (qm_mr_current(p) && drain_mr_fqrni(p)) {
 595                 /* special handling, drain just in case it's a few FQRNIs */
 596                 if (drain_mr_fqrni(p))
 597                         goto fail_dqrr_mr_empty;
 598         }
 599         /* Success */
 600         portal->config = c;
 601         qm_isr_disable_write(p, 0);
 602         qm_isr_uninhibit(p);
 603         /* Write a sane SDQCR */
 604         qm_dqrr_sdqcr_set(p, portal->sdqcr);
 605         return portal;
 606 fail_dqrr_mr_empty:
 607 fail_eqcr_empty:
 608         free_irq(c->irq, portal);
 609 fail_irq:
 610         kfree(portal->cgrs);
 611         spin_lock_destroy(&portal->cgr_lock);
 612 fail_cgrs:
 613         qm_mc_finish(p);
 614 fail_mc:
 615         qm_mr_finish(p);
 616 fail_mr:
 617         qm_dqrr_finish(p);
 618 fail_dqrr:
 619         qm_eqcr_finish(p);
 620 fail_eqcr:
 621         return NULL;
 622 }
 623
 624 struct qman_portal *qman_create_affine_portal(const struct qm_portal_config *c,
 625                                               const struct qman_cgrs *cgrs)
 626 {
 627         struct qman_portal *res;
 628         struct qman_portal *portal = get_affine_portal();
 629         /* A criteria for calling this function (from qman_driver.c) is that
 630          * we're already affine to the cpu and won't schedule onto another cpu.
 631          */
 632
 633         res = qman_create_portal(portal, c, cgrs);
 634         if (res) {
 635                 spin_lock(&affine_mask_lock);
 636                 CPU_SET(c->cpu, &affine_mask);
 637                 affine_channels[c->cpu] =
 638                         c->channel;
 639                 spin_unlock(&affine_mask_lock);
 640         }
 641         return res;
 642 }
 643
 644 static inline
 645 void qman_destroy_portal(struct qman_portal *qm)
 646 {
 647         const struct qm_portal_config *pcfg;
 648
 649         /* Stop dequeues on the portal */
 650         qm_dqrr_sdqcr_set(&qm->p, 0);
 651
 652         /*
 653          * NB we do this to "quiesce" EQCR. If we add enqueue-completions or
 654          * something related to QM_PIRQ_EQCI, this may need fixing.
 655          * Also, due to the prefetching model used for CI updates in the enqueue
 656          * path, this update will only invalidate the CI cacheline *after*
 657          * working on it, so we need to call this twice to ensure a full update
 658          * irrespective of where the enqueue processing was at when the teardown
 659          * began.
 660          */
 661         qm_eqcr_cce_update(&qm->p);
 662         qm_eqcr_cce_update(&qm->p);
 663         pcfg = qm->config;
 664
 665         free_irq(pcfg->irq, qm);
 666
 667         kfree(qm->cgrs);
 668         qm_mc_finish(&qm->p);
 669         qm_mr_finish(&qm->p);
 670         qm_dqrr_finish(&qm->p);
 671         qm_eqcr_finish(&qm->p);
 672
 673         qm->config = NULL;
 674
 675         spin_lock_destroy(&qm->cgr_lock);
 676 }
 677
 678 const struct qm_portal_config *qman_destroy_affine_portal(void)
 679 {
 680         /* We don't want to redirect if we're a slave, use "raw" */
 681         struct qman_portal *qm = get_affine_portal();
 682         const struct qm_portal_config *pcfg;
 683         int cpu;
 684
 685         pcfg = qm->config;
 686         cpu = pcfg->cpu;
 687
 688         qman_destroy_portal(qm);
 689
 690         spin_lock(&affine_mask_lock);
 691         CPU_CLR(cpu, &affine_mask);
 692         spin_unlock(&affine_mask_lock);
 693         return pcfg;
 694 }
 695
 696 int qman_get_portal_index(void)
 697 {
 698         struct qman_portal *p = get_affine_portal();
 699         return p->config->index;
 700 }
 701
 702 /* Inline helper to reduce nesting in __poll_portal_slow() */
 703 static inline void fq_state_change(struct qman_portal *p, struct qman_fq *fq,
 704                                    const struct qm_mr_entry *msg, u8 verb)
 705 {
 706         FQLOCK(fq);
 707         switch (verb) {
 708         case QM_MR_VERB_FQRL:
 709                 DPAA_ASSERT(fq_isset(fq, QMAN_FQ_STATE_ORL));
 710                 fq_clear(fq, QMAN_FQ_STATE_ORL);
 711                 table_del_fq(p, fq);
 712                 break;
 713         case QM_MR_VERB_FQRN:
 714                 DPAA_ASSERT((fq->state == qman_fq_state_parked) ||
 715                             (fq->state == qman_fq_state_sched));
 716                 DPAA_ASSERT(fq_isset(fq, QMAN_FQ_STATE_CHANGING));
 717                 fq_clear(fq, QMAN_FQ_STATE_CHANGING);
 718                 if (msg->fq.fqs & QM_MR_FQS_NOTEMPTY)
 719                         fq_set(fq, QMAN_FQ_STATE_NE);
 720                 if (msg->fq.fqs & QM_MR_FQS_ORLPRESENT)
 721                         fq_set(fq, QMAN_FQ_STATE_ORL);
 722                 else
 723                         table_del_fq(p, fq);
 724                 fq->state = qman_fq_state_retired;
 725                 break;
 726         case QM_MR_VERB_FQPN:
 727                 DPAA_ASSERT(fq->state == qman_fq_state_sched);
 728                 DPAA_ASSERT(fq_isclear(fq, QMAN_FQ_STATE_CHANGING));
 729                 fq->state = qman_fq_state_parked;
 730         }
 731         FQUNLOCK(fq);
 732 }
 733
 734 static u32 __poll_portal_slow(struct qman_portal *p, u32 is)
 735 {
 736         const struct qm_mr_entry *msg;
 737         struct qm_mr_entry swapped_msg;
 738
 739         if (is & QM_PIRQ_CSCI) {
 740                 struct qman_cgrs rr, c;
 741                 struct qm_mc_result *mcr;
 742                 struct qman_cgr *cgr;
 743
 744                 spin_lock(&p->cgr_lock);
 745                 /*
 746                  * The CSCI bit must be cleared _before_ issuing the
 747                  * Query Congestion State command, to ensure that a long
 748                  * CGR State Change callback cannot miss an intervening
 749                  * state change.
 750                  */
 751                 qm_isr_status_clear(&p->p, QM_PIRQ_CSCI);
 752                 qm_mc_start(&p->p);
 753                 qm_mc_commit(&p->p, QM_MCC_VERB_QUERYCONGESTION);
 754                 while (!(mcr = qm_mc_result(&p->p)))
 755                         cpu_relax();
 756                 /* mask out the ones I'm not interested in */
 757                 qman_cgrs_and(&rr, (const struct qman_cgrs *)
 758                         &mcr->querycongestion.state, &p->cgrs[0]);
 759                 /* check previous snapshot for delta, enter/exit congestion */
 760                 qman_cgrs_xor(&c, &rr, &p->cgrs[1]);
 761                 /* update snapshot */
 762                 qman_cgrs_cp(&p->cgrs[1], &rr);
 763                 /* Invoke callback */
 764                 list_for_each_entry(cgr, &p->cgr_cbs, node)
 765                         if (cgr->cb && qman_cgrs_get(&c, cgr->cgrid))
 766                                 cgr->cb(p, cgr, qman_cgrs_get(&rr, cgr->cgrid));
 767                 spin_unlock(&p->cgr_lock);
 768         }
 769
 770         if (is & QM_PIRQ_EQRI) {
 771                 qm_eqcr_cce_update(&p->p);
 772                 qm_eqcr_set_ithresh(&p->p, 0);
 773                 wake_up(&affine_queue);
 774         }
 775
 776         if (is & QM_PIRQ_MRI) {
 777                 struct qman_fq *fq;
 778                 u8 verb, num = 0;
 779 mr_loop:
 780                 qm_mr_pvb_update(&p->p);
 781                 msg = qm_mr_current(&p->p);
 782                 if (!msg)
 783                         goto mr_done;
 784                 swapped_msg = *msg;
 785                 hw_fd_to_cpu(&swapped_msg.ern.fd);
 786                 verb = msg->verb & QM_MR_VERB_TYPE_MASK;
 787                 /* The message is a software ERN iff the 0x20 bit is set */
 788                 if (verb & 0x20) {
 789                         switch (verb) {
 790                         case QM_MR_VERB_FQRNI:
 791                                 /* nada, we drop FQRNIs on the floor */
 792                                 break;
 793                         case QM_MR_VERB_FQRN:
 794                         case QM_MR_VERB_FQRL:
 795                                 /* Lookup in the retirement table */
 796                                 fq = table_find_fq(p,
 797                                                    be32_to_cpu(msg->fq.fqid));
 798                                 DPAA_BUG_ON(!fq);
 799                                 fq_state_change(p, fq, &swapped_msg, verb);
 800                                 if (fq->cb.fqs)
 801                                         fq->cb.fqs(p, fq, &swapped_msg);
 802                                 break;
 803                         case QM_MR_VERB_FQPN:
 804                                 /* Parked */
 805 #ifdef CONFIG_FSL_QMAN_FQ_LOOKUP
 806                                 fq = get_fq_table_entry(
 807                                         be32_to_cpu(msg->fq.contextB));
 808 #else
 809                                 fq = (void *)(uintptr_t)
 810                                         be32_to_cpu(msg->fq.contextB);
 811 #endif
 812                                 fq_state_change(p, fq, msg, verb);
 813                                 if (fq->cb.fqs)
 814                                         fq->cb.fqs(p, fq, &swapped_msg);
 815                                 break;
 816                         case QM_MR_VERB_DC_ERN:
 817                                 /* DCP ERN */
 818                                 if (p->cb_dc_ern)
 819                                         p->cb_dc_ern(p, msg);
 820                                 else if (cb_dc_ern)
 821                                         cb_dc_ern(p, msg);
 822                                 else {
 823                                         static int warn_once;
 824
 825                                         if (!warn_once) {
 826                                                 pr_crit("Leaking DCP ERNs!\n");
 827                                                 warn_once = 1;
 828                                         }
 829                                 }
 830                                 break;
 831                         default:
 832                                 pr_crit("Invalid MR verb 0x%02x\n", verb);
 833                         }
 834                 } else {
 835                         /* Its a software ERN */
 836 #ifdef CONFIG_FSL_QMAN_FQ_LOOKUP
 837                         fq = get_fq_table_entry(be32_to_cpu(msg->ern.tag));
 838 #else
 839                         fq = (void *)(uintptr_t)be32_to_cpu(msg->ern.tag);
 840 #endif
 841                         fq->cb.ern(p, fq, &swapped_msg);
 842                 }
 843                 num++;
 844                 qm_mr_next(&p->p);
 845                 goto mr_loop;
 846 mr_done:
 847                 qm_mr_cci_consume(&p->p, num);
 848         }
 849         /*
 850          * QM_PIRQ_CSCI/CCSCI has already been cleared, as part of its specific
 851          * processing. If that interrupt source has meanwhile been re-asserted,
 852          * we mustn't clear it here (or in the top-level interrupt handler).
 853          */
 854         return is & (QM_PIRQ_EQCI | QM_PIRQ_EQRI | QM_PIRQ_MRI);
 855 }
 856
 857 /*
 858  * remove some slowish-path stuff from the "fast path" and make sure it isn't
 859  * inlined.
 860  */
 861 static noinline void clear_vdqcr(struct qman_portal *p, struct qman_fq *fq)
 862 {
 863         p->vdqcr_owned = NULL;
 864         FQLOCK(fq);
 865         fq_clear(fq, QMAN_FQ_STATE_VDQCR);
 866         FQUNLOCK(fq);
 867         wake_up(&affine_queue);
 868 }
 869
 870 /*
 871  * The only states that would conflict with other things if they ran at the
 872  * same time on the same cpu are:
 873  *
 874  *   (i) setting/clearing vdqcr_owned, and
 875  *  (ii) clearing the NE (Not Empty) flag.
 876  *
 877  * Both are safe. Because;
 878  *
 879  *   (i) this clearing can only occur after qman_set_vdq() has set the
 880  *       vdqcr_owned field (which it does before setting VDQCR), and
 881  *       qman_volatile_dequeue() blocks interrupts and preemption while this is
 882  *       done so that we can't interfere.
 883  *  (ii) the NE flag is only cleared after qman_retire_fq() has set it, and as
 884  *       with (i) that API prevents us from interfering until it's safe.
 885  *
 886  * The good thing is that qman_set_vdq() and qman_retire_fq() run far
 887  * less frequently (ie. per-FQ) than __poll_portal_fast() does, so the nett
 888  * advantage comes from this function not having to "lock" anything at all.
 889  *
 890  * Note also that the callbacks are invoked at points which are safe against the
 891  * above potential conflicts, but that this function itself is not re-entrant
 892  * (this is because the function tracks one end of each FIFO in the portal and
 893  * we do *not* want to lock that). So the consequence is that it is safe for
 894  * user callbacks to call into any QMan API.
 895  */
 896 static inline unsigned int __poll_portal_fast(struct qman_portal *p,
 897                                               unsigned int poll_limit)
 898 {
 899         const struct qm_dqrr_entry *dq;
 900         struct qman_fq *fq;
 901         enum qman_cb_dqrr_result res;
 902         unsigned int limit = 0;
 903 #if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__
 904         struct qm_dqrr_entry *shadow;
 905 #endif
 906         do {
 907                 qm_dqrr_pvb_update(&p->p);
 908                 dq = qm_dqrr_current(&p->p);
 909                 if (unlikely(!dq))
 910                         break;
 911 #if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__
 912         /* If running on an LE system the fields of the
 913          * dequeue entry must be swapper.  Because the
 914          * QMan HW will ignore writes the DQRR entry is
 915          * copied and the index stored within the copy
 916          */
 917                 shadow = &p->shadow_dqrr[DQRR_PTR2IDX(dq)];
 918                 *shadow = *dq;
 919                 dq = shadow;
 920                 shadow->fqid = be32_to_cpu(shadow->fqid);
 921                 shadow->contextB = be32_to_cpu(shadow->contextB);
 922                 shadow->seqnum = be16_to_cpu(shadow->seqnum);
 923                 hw_fd_to_cpu(&shadow->fd);
 924 #endif
 925
 926                 if (dq->stat & QM_DQRR_STAT_UNSCHEDULED) {
 927                         /*
 928                          * VDQCR: don't trust context_b as the FQ may have
 929                          * been configured for h/w consumption and we're
 930                          * draining it post-retirement.
 931                          */
 932                         fq = p->vdqcr_owned;
 933                         /*
 934                          * We only set QMAN_FQ_STATE_NE when retiring, so we
 935                          * only need to check for clearing it when doing
 936                          * volatile dequeues.  It's one less thing to check
 937                          * in the critical path (SDQCR).
 938                          */
 939                         if (dq->stat & QM_DQRR_STAT_FQ_EMPTY)
 940                                 fq_clear(fq, QMAN_FQ_STATE_NE);
 941                         /*
 942                          * This is duplicated from the SDQCR code, but we
 943                          * have stuff to do before *and* after this callback,
 944                          * and we don't want multiple if()s in the critical
 945                          * path (SDQCR).
 946                          */
 947                         res = fq->cb.dqrr(p, fq, dq);
 948                         if (res == qman_cb_dqrr_stop)
 949                                 break;
 950                         /* Check for VDQCR completion */
 951                         if (dq->stat & QM_DQRR_STAT_DQCR_EXPIRED)
 952                                 clear_vdqcr(p, fq);
 953                 } else {
 954                         /* SDQCR: context_b points to the FQ */
 955 #ifdef CONFIG_FSL_QMAN_FQ_LOOKUP
 956                         fq = get_fq_table_entry(dq->contextB);
 957 #else
 958                         fq = (void *)(uintptr_t)dq->contextB;
 959 #endif
 960                         /* Now let the callback do its stuff */
 961                         res = fq->cb.dqrr(p, fq, dq);
 962                         /*
 963                          * The callback can request that we exit without
 964                          * consuming this entry nor advancing;
 965                          */
 966                         if (res == qman_cb_dqrr_stop)
 967                                 break;
 968                 }
 969                 /* Interpret 'dq' from a driver perspective. */
 970                 /*
 971                  * Parking isn't possible unless HELDACTIVE was set. NB,
 972                  * FORCEELIGIBLE implies HELDACTIVE, so we only need to
 973                  * check for HELDACTIVE to cover both.
 974                  */
 975                 DPAA_ASSERT((dq->stat & QM_DQRR_STAT_FQ_HELDACTIVE) ||
 976                             (res != qman_cb_dqrr_park));
 977                 /* just means "skip it, I'll consume it myself later on" */
 978                 if (res != qman_cb_dqrr_defer)
 979                         qm_dqrr_cdc_consume_1ptr(&p->p, dq,
 980                                                  res == qman_cb_dqrr_park);
 981                 /* Move forward */
 982                 qm_dqrr_next(&p->p);
 983                 /*
 984                  * Entry processed and consumed, increment our counter.  The
 985                  * callback can request that we exit after consuming the
 986                  * entry, and we also exit if we reach our processing limit,
 987                  * so loop back only if neither of these conditions is met.
 988                  */
 989         } while (++limit < poll_limit && res != qman_cb_dqrr_consume_stop);
 990
 991         return limit;
 992 }
 993
 994 u16 qman_affine_channel(int cpu)
 995 {
 996         if (cpu < 0) {
 997                 struct qman_portal *portal = get_affine_portal();
 998
 999                 cpu = portal->config->cpu;
1000         }
1001         DPAA_BUG_ON(!CPU_ISSET(cpu, &affine_mask));
1002         return affine_channels[cpu];
1003 }
1004
1005 struct qm_dqrr_entry *qman_dequeue(struct qman_fq *fq)
1006 {
1007         struct qman_portal *p = get_affine_portal();
1008         const struct qm_dqrr_entry *dq;
1009 #if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__
1010         struct qm_dqrr_entry *shadow;
1011 #endif
1012
1013         qm_dqrr_pvb_update(&p->p);
1014         dq = qm_dqrr_current(&p->p);
1015         if (!dq)
1016                 return NULL;
1017
1018         if (!(dq->stat & QM_DQRR_STAT_FD_VALID)) {
1019                 /* Invalid DQRR - put the portal and consume the DQRR.
1020                  * Return NULL to user as no packet is seen.
1021                  */
1022                 qman_dqrr_consume(fq, (struct qm_dqrr_entry *)dq);
1023                 return NULL;
1024         }
1025
1026 #if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__
1027         shadow = &p->shadow_dqrr[DQRR_PTR2IDX(dq)];
1028         *shadow = *dq;
1029         dq = shadow;
1030         shadow->fqid = be32_to_cpu(shadow->fqid);
1031         shadow->contextB = be32_to_cpu(shadow->contextB);
1032         shadow->seqnum = be16_to_cpu(shadow->seqnum);
1033         hw_fd_to_cpu(&shadow->fd);
1034 #endif
1035
1036         if (dq->stat & QM_DQRR_STAT_FQ_EMPTY)
1037                 fq_clear(fq, QMAN_FQ_STATE_NE);
1038
1039         return (struct qm_dqrr_entry *)dq;
1040 }
1041
1042 void qman_dqrr_consume(struct qman_fq *fq,
1043                        struct qm_dqrr_entry *dq)
1044 {
1045         struct qman_portal *p = get_affine_portal();
1046
1047         if (dq->stat & QM_DQRR_STAT_DQCR_EXPIRED)
1048                 clear_vdqcr(p, fq);
1049
1050         qm_dqrr_cdc_consume_1ptr(&p->p, dq, 0);
1051         qm_dqrr_next(&p->p);
1052 }
1053
1054 int qman_poll_dqrr(unsigned int limit)
1055 {
1056         struct qman_portal *p = get_affine_portal();
1057         int ret;
1058
1059         ret = __poll_portal_fast(p, limit);
1060         return ret;
1061 }
1062
1063 void qman_poll(void)
1064 {
1065         struct qman_portal *p = get_affine_portal();
1066
1067         if ((~p->irq_sources) & QM_PIRQ_SLOW) {
1068                 if (!(p->slowpoll--)) {
1069                         u32 is = qm_isr_status_read(&p->p) & ~p->irq_sources;
1070                         u32 active = __poll_portal_slow(p, is);
1071
1072                         if (active) {
1073                                 qm_isr_status_clear(&p->p, active);
1074                                 p->slowpoll = SLOW_POLL_BUSY;
1075                         } else
1076                                 p->slowpoll = SLOW_POLL_IDLE;
1077                 }
1078         }
1079         if ((~p->irq_sources) & QM_PIRQ_DQRI)
1080                 __poll_portal_fast(p, FSL_QMAN_POLL_LIMIT);
1081 }
1082
1083 void qman_stop_dequeues(void)
1084 {
1085         struct qman_portal *p = get_affine_portal();
1086
1087         qman_stop_dequeues_ex(p);
1088 }
1089
1090 void qman_start_dequeues(void)
1091 {
1092         struct qman_portal *p = get_affine_portal();
1093
1094         DPAA_ASSERT(p->dqrr_disable_ref > 0);
1095         if (!(--p->dqrr_disable_ref))
1096                 qm_dqrr_set_maxfill(&p->p, DQRR_MAXFILL);
1097 }
1098
1099 void qman_static_dequeue_add(u32 pools)
1100 {
1101         struct qman_portal *p = get_affine_portal();
1102
1103         pools &= p->config->pools;
1104         p->sdqcr |= pools;
1105         qm_dqrr_sdqcr_set(&p->p, p->sdqcr);
1106 }
1107
1108 void qman_static_dequeue_del(u32 pools)
1109 {
1110         struct qman_portal *p = get_affine_portal();
1111
1112         pools &= p->config->pools;
1113         p->sdqcr &= ~pools;
1114         qm_dqrr_sdqcr_set(&p->p, p->sdqcr);
1115 }
1116
1117 u32 qman_static_dequeue_get(void)
1118 {
1119         struct qman_portal *p = get_affine_portal();
1120         return p->sdqcr;
1121 }
1122
1123 void qman_dca(struct qm_dqrr_entry *dq, int park_request)
1124 {
1125         struct qman_portal *p = get_affine_portal();
1126
1127         qm_dqrr_cdc_consume_1ptr(&p->p, dq, park_request);
1128 }
1129
1130 /* Frame queue API */
1131 static const char *mcr_result_str(u8 result)
1132 {
1133         switch (result) {
1134         case QM_MCR_RESULT_NULL:
1135                 return "QM_MCR_RESULT_NULL";
1136         case QM_MCR_RESULT_OK:
1137                 return "QM_MCR_RESULT_OK";
1138         case QM_MCR_RESULT_ERR_FQID:
1139                 return "QM_MCR_RESULT_ERR_FQID";
1140         case QM_MCR_RESULT_ERR_FQSTATE:
1141                 return "QM_MCR_RESULT_ERR_FQSTATE";
1142         case QM_MCR_RESULT_ERR_NOTEMPTY:
1143                 return "QM_MCR_RESULT_ERR_NOTEMPTY";
1144         case QM_MCR_RESULT_PENDING:
1145                 return "QM_MCR_RESULT_PENDING";
1146         case QM_MCR_RESULT_ERR_BADCOMMAND:
1147                 return "QM_MCR_RESULT_ERR_BADCOMMAND";
1148         }
1149         return "<unknown MCR result>";
1150 }
1151
1152 int qman_create_fq(u32 fqid, u32 flags, struct qman_fq *fq)
1153 {
1154         struct qm_fqd fqd;
1155         struct qm_mcr_queryfq_np np;
1156         struct qm_mc_command *mcc;
1157         struct qm_mc_result *mcr;
1158         struct qman_portal *p;
1159
1160         if (flags & QMAN_FQ_FLAG_DYNAMIC_FQID) {
1161                 int ret = qman_alloc_fqid(&fqid);
1162
1163                 if (ret)
1164                         return ret;
1165         }
1166         spin_lock_init(&fq->fqlock);
1167         fq->fqid = fqid;
1168         fq->fqid_le = cpu_to_be32(fqid);
1169         fq->flags = flags;
1170         fq->state = qman_fq_state_oos;
1171         fq->cgr_groupid = 0;
1172 #ifdef CONFIG_FSL_QMAN_FQ_LOOKUP
1173         if (unlikely(find_empty_fq_table_entry(&fq->key, fq))) {
1174                 pr_info("Find empty table entry failed\n");
1175                 return -ENOMEM;
1176         }
1177 #endif
1178         if (!(flags & QMAN_FQ_FLAG_AS_IS) || (flags & QMAN_FQ_FLAG_NO_MODIFY))
1179                 return 0;
1180         /* Everything else is AS_IS support */
1181         p = get_affine_portal();
1182         mcc = qm_mc_start(&p->p);
1183         mcc->queryfq.fqid = cpu_to_be32(fqid);
1184         qm_mc_commit(&p->p, QM_MCC_VERB_QUERYFQ);
1185         while (!(mcr = qm_mc_result(&p->p)))
1186                 cpu_relax();
1187         DPAA_ASSERT((mcr->verb & QM_MCR_VERB_MASK) == QM_MCC_VERB_QUERYFQ);
1188         if (mcr->result != QM_MCR_RESULT_OK) {
1189                 pr_err("QUERYFQ failed: %s\n", mcr_result_str(mcr->result));
1190                 goto err;
1191         }
1192         fqd = mcr->queryfq.fqd;
1193         hw_fqd_to_cpu(&fqd);
1194         mcc = qm_mc_start(&p->p);
1195         mcc->queryfq_np.fqid = cpu_to_be32(fqid);
1196         qm_mc_commit(&p->p, QM_MCC_VERB_QUERYFQ_NP);
1197         while (!(mcr = qm_mc_result(&p->p)))
1198                 cpu_relax();
1199         DPAA_ASSERT((mcr->verb & QM_MCR_VERB_MASK) == QM_MCC_VERB_QUERYFQ_NP);
1200         if (mcr->result != QM_MCR_RESULT_OK) {
1201                 pr_err("QUERYFQ_NP failed: %s\n", mcr_result_str(mcr->result));
1202                 goto err;
1203         }
1204         np = mcr->queryfq_np;
1205         /* Phew, have queryfq and queryfq_np results, stitch together
1206          * the FQ object from those.
1207          */
1208         fq->cgr_groupid = fqd.cgid;
1209         switch (np.state & QM_MCR_NP_STATE_MASK) {
1210         case QM_MCR_NP_STATE_OOS:
1211                 break;
1212         case QM_MCR_NP_STATE_RETIRED:
1213                 fq->state = qman_fq_state_retired;
1214                 if (np.frm_cnt)
1215                         fq_set(fq, QMAN_FQ_STATE_NE);
1216                 break;
1217         case QM_MCR_NP_STATE_TEN_SCHED:
1218         case QM_MCR_NP_STATE_TRU_SCHED:
1219         case QM_MCR_NP_STATE_ACTIVE:
1220                 fq->state = qman_fq_state_sched;
1221                 if (np.state & QM_MCR_NP_STATE_R)
1222                         fq_set(fq, QMAN_FQ_STATE_CHANGING);
1223                 break;
1224         case QM_MCR_NP_STATE_PARKED:
1225                 fq->state = qman_fq_state_parked;
1226                 break;
1227         default:
1228                 DPAA_ASSERT(NULL == "invalid FQ state");
1229         }
1230         if (fqd.fq_ctrl & QM_FQCTRL_CGE)
1231                 fq->state |= QMAN_FQ_STATE_CGR_EN;
1232         return 0;
1233 err:
1234         if (flags & QMAN_FQ_FLAG_DYNAMIC_FQID)
1235                 qman_release_fqid(fqid);
1236         return -EIO;
1237 }
1238
1239 void qman_destroy_fq(struct qman_fq *fq, u32 flags __maybe_unused)
1240 {
1241         /*
1242          * We don't need to lock the FQ as it is a pre-condition that the FQ be
1243          * quiesced. Instead, run some checks.
1244          */
1245         switch (fq->state) {
1246         case qman_fq_state_parked:
1247                 DPAA_ASSERT(flags & QMAN_FQ_DESTROY_PARKED);
1248                 /* Fallthrough */
1249         case qman_fq_state_oos:
1250                 if (fq_isset(fq, QMAN_FQ_FLAG_DYNAMIC_FQID))
1251                         qman_release_fqid(fq->fqid);
1252 #ifdef CONFIG_FSL_QMAN_FQ_LOOKUP
1253                 clear_fq_table_entry(fq->key);
1254 #endif
1255                 return;
1256         default:
1257                 break;
1258         }
1259         DPAA_ASSERT(NULL == "qman_free_fq() on unquiesced FQ!");
1260 }
1261
1262 u32 qman_fq_fqid(struct qman_fq *fq)
1263 {
1264         return fq->fqid;
1265 }
1266
1267 void qman_fq_state(struct qman_fq *fq, enum qman_fq_state *state, u32 *flags)
1268 {
1269         if (state)
1270                 *state = fq->state;
1271         if (flags)
1272                 *flags = fq->flags;
1273 }
1274
1275 int qman_init_fq(struct qman_fq *fq, u32 flags, struct qm_mcc_initfq *opts)
1276 {
1277         struct qm_mc_command *mcc;
1278         struct qm_mc_result *mcr;
1279         struct qman_portal *p;
1280
1281         u8 res, myverb = (flags & QMAN_INITFQ_FLAG_SCHED) ?
1282                 QM_MCC_VERB_INITFQ_SCHED : QM_MCC_VERB_INITFQ_PARKED;
1283
1284         if ((fq->state != qman_fq_state_oos) &&
1285             (fq->state != qman_fq_state_parked))
1286                 return -EINVAL;
1287 #ifdef RTE_LIBRTE_DPAA_HWDEBUG
1288         if (unlikely(fq_isset(fq, QMAN_FQ_FLAG_NO_MODIFY)))
1289                 return -EINVAL;
1290 #endif
1291         if (opts && (opts->we_mask & QM_INITFQ_WE_OAC)) {
1292                 /* And can't be set at the same time as TDTHRESH */
1293                 if (opts->we_mask & QM_INITFQ_WE_TDTHRESH)
1294                         return -EINVAL;
1295         }
1296         /* Issue an INITFQ_[PARKED|SCHED] management command */
1297         p = get_affine_portal();
1298         FQLOCK(fq);
1299         if (unlikely((fq_isset(fq, QMAN_FQ_STATE_CHANGING)) ||
1300                      ((fq->state != qman_fq_state_oos) &&
1301                                 (fq->state != qman_fq_state_parked)))) {
1302                 FQUNLOCK(fq);
1303                 return -EBUSY;
1304         }
1305         mcc = qm_mc_start(&p->p);
1306         if (opts)
1307                 mcc->initfq = *opts;
1308         mcc->initfq.fqid = cpu_to_be32(fq->fqid);
1309         mcc->initfq.count = 0;
1310         /*
1311          * If the FQ does *not* have the TO_DCPORTAL flag, context_b is set as a
1312          * demux pointer. Otherwise, the caller-provided value is allowed to
1313          * stand, don't overwrite it.
1314          */
1315         if (fq_isclear(fq, QMAN_FQ_FLAG_TO_DCPORTAL)) {
1316                 dma_addr_t phys_fq;
1317
1318                 mcc->initfq.we_mask |= QM_INITFQ_WE_CONTEXTB;
1319 #ifdef CONFIG_FSL_QMAN_FQ_LOOKUP
1320                 mcc->initfq.fqd.context_b = fq->key;
1321 #else
1322                 mcc->initfq.fqd.context_b = (u32)(uintptr_t)fq;
1323 #endif
1324                 /*
1325                  *  and the physical address - NB, if the user wasn't trying to
1326                  * set CONTEXTA, clear the stashing settings.
1327                  */
1328                 if (!(mcc->initfq.we_mask & QM_INITFQ_WE_CONTEXTA)) {
1329                         mcc->initfq.we_mask |= QM_INITFQ_WE_CONTEXTA;
1330                         memset(&mcc->initfq.fqd.context_a, 0,
1331                                sizeof(mcc->initfq.fqd.context_a));
1332                 } else {
1333                         phys_fq = rte_mem_virt2iova(fq);
1334                         qm_fqd_stashing_set64(&mcc->initfq.fqd, phys_fq);
1335                 }
1336         }
1337         if (flags & QMAN_INITFQ_FLAG_LOCAL) {
1338                 mcc->initfq.fqd.dest.channel = p->config->channel;
1339                 if (!(mcc->initfq.we_mask & QM_INITFQ_WE_DESTWQ)) {
1340                         mcc->initfq.we_mask |= QM_INITFQ_WE_DESTWQ;
1341                         mcc->initfq.fqd.dest.wq = 4;
1342                 }
1343         }
1344         mcc->initfq.we_mask = cpu_to_be16(mcc->initfq.we_mask);
1345         cpu_to_hw_fqd(&mcc->initfq.fqd);
1346         qm_mc_commit(&p->p, myverb);
1347         while (!(mcr = qm_mc_result(&p->p)))
1348                 cpu_relax();
1349         DPAA_ASSERT((mcr->verb & QM_MCR_VERB_MASK) == myverb);
1350         res = mcr->result;
1351         if (res != QM_MCR_RESULT_OK) {
1352                 FQUNLOCK(fq);
1353                 return -EIO;
1354         }
1355         if (opts) {
1356                 if (opts->we_mask & QM_INITFQ_WE_FQCTRL) {
1357                         if (opts->fqd.fq_ctrl & QM_FQCTRL_CGE)
1358                                 fq_set(fq, QMAN_FQ_STATE_CGR_EN);
1359                         else
1360                                 fq_clear(fq, QMAN_FQ_STATE_CGR_EN);
1361                 }
1362                 if (opts->we_mask & QM_INITFQ_WE_CGID)
1363                         fq->cgr_groupid = opts->fqd.cgid;
1364         }
1365         fq->state = (flags & QMAN_INITFQ_FLAG_SCHED) ?
1366                 qman_fq_state_sched : qman_fq_state_parked;
1367         FQUNLOCK(fq);
1368         return 0;
1369 }
1370
1371 int qman_schedule_fq(struct qman_fq *fq)
1372 {
1373         struct qm_mc_command *mcc;
1374         struct qm_mc_result *mcr;
1375         struct qman_portal *p;
1376
1377         int ret = 0;
1378         u8 res;
1379
1380         if (fq->state != qman_fq_state_parked)
1381                 return -EINVAL;
1382 #ifdef RTE_LIBRTE_DPAA_HWDEBUG
1383         if (unlikely(fq_isset(fq, QMAN_FQ_FLAG_NO_MODIFY)))
1384                 return -EINVAL;
1385 #endif
1386         /* Issue a ALTERFQ_SCHED management command */
1387         p = get_affine_portal();
1388
1389         FQLOCK(fq);
1390         if (unlikely((fq_isset(fq, QMAN_FQ_STATE_CHANGING)) ||
1391                      (fq->state != qman_fq_state_parked))) {
1392                 ret = -EBUSY;
1393                 goto out;
1394         }
1395         mcc = qm_mc_start(&p->p);
1396         mcc->alterfq.fqid = cpu_to_be32(fq->fqid);
1397         qm_mc_commit(&p->p, QM_MCC_VERB_ALTER_SCHED);
1398         while (!(mcr = qm_mc_result(&p->p)))
1399                 cpu_relax();
1400         DPAA_ASSERT((mcr->verb & QM_MCR_VERB_MASK) == QM_MCR_VERB_ALTER_SCHED);
1401         res = mcr->result;
1402         if (res != QM_MCR_RESULT_OK) {
1403                 ret = -EIO;
1404                 goto out;
1405         }
1406         fq->state = qman_fq_state_sched;
1407 out:
1408         FQUNLOCK(fq);
1409
1410         return ret;
1411 }
1412
1413 int qman_retire_fq(struct qman_fq *fq, u32 *flags)
1414 {
1415         struct qm_mc_command *mcc;
1416         struct qm_mc_result *mcr;
1417         struct qman_portal *p;
1418
1419         int rval;
1420         u8 res;
1421
1422         if ((fq->state != qman_fq_state_parked) &&
1423             (fq->state != qman_fq_state_sched))
1424                 return -EINVAL;
1425 #ifdef RTE_LIBRTE_DPAA_HWDEBUG
1426         if (unlikely(fq_isset(fq, QMAN_FQ_FLAG_NO_MODIFY)))
1427                 return -EINVAL;
1428 #endif
1429         p = get_affine_portal();
1430
1431         FQLOCK(fq);
1432         if (unlikely((fq_isset(fq, QMAN_FQ_STATE_CHANGING)) ||
1433                      (fq->state == qman_fq_state_retired) ||
1434                                 (fq->state == qman_fq_state_oos))) {
1435                 rval = -EBUSY;
1436                 goto out;
1437         }
1438         rval = table_push_fq(p, fq);
1439         if (rval)
1440                 goto out;
1441         mcc = qm_mc_start(&p->p);
1442         mcc->alterfq.fqid = cpu_to_be32(fq->fqid);
1443         qm_mc_commit(&p->p, QM_MCC_VERB_ALTER_RETIRE);
1444         while (!(mcr = qm_mc_result(&p->p)))
1445                 cpu_relax();
1446         DPAA_ASSERT((mcr->verb & QM_MCR_VERB_MASK) == QM_MCR_VERB_ALTER_RETIRE);
1447         res = mcr->result;
1448         /*
1449          * "Elegant" would be to treat OK/PENDING the same way; set CHANGING,
1450          * and defer the flags until FQRNI or FQRN (respectively) show up. But
1451          * "Friendly" is to process OK immediately, and not set CHANGING. We do
1452          * friendly, otherwise the caller doesn't necessarily have a fully
1453          * "retired" FQ on return even if the retirement was immediate. However
1454          * this does mean some code duplication between here and
1455          * fq_state_change().
1456          */
1457         if (likely(res == QM_MCR_RESULT_OK)) {
1458                 rval = 0;
1459                 /* Process 'fq' right away, we'll ignore FQRNI */
1460                 if (mcr->alterfq.fqs & QM_MCR_FQS_NOTEMPTY)
1461                         fq_set(fq, QMAN_FQ_STATE_NE);
1462                 if (mcr->alterfq.fqs & QM_MCR_FQS_ORLPRESENT)
1463                         fq_set(fq, QMAN_FQ_STATE_ORL);
1464                 else
1465                         table_del_fq(p, fq);
1466                 if (flags)
1467                         *flags = fq->flags;
1468                 fq->state = qman_fq_state_retired;
1469                 if (fq->cb.fqs) {
1470                         /*
1471                          * Another issue with supporting "immediate" retirement
1472                          * is that we're forced to drop FQRNIs, because by the
1473                          * time they're seen it may already be "too late" (the
1474                          * fq may have been OOS'd and free()'d already). But if
1475                          * the upper layer wants a callback whether it's
1476                          * immediate or not, we have to fake a "MR" entry to
1477                          * look like an FQRNI...
1478                          */
1479                         struct qm_mr_entry msg;
1480
1481                         msg.verb = QM_MR_VERB_FQRNI;
1482                         msg.fq.fqs = mcr->alterfq.fqs;
1483                         msg.fq.fqid = fq->fqid;
1484 #ifdef CONFIG_FSL_QMAN_FQ_LOOKUP
1485                         msg.fq.contextB = fq->key;
1486 #else
1487                         msg.fq.contextB = (u32)(uintptr_t)fq;
1488 #endif
1489                         fq->cb.fqs(p, fq, &msg);
1490                 }
1491         } else if (res == QM_MCR_RESULT_PENDING) {
1492                 rval = 1;
1493                 fq_set(fq, QMAN_FQ_STATE_CHANGING);
1494         } else {
1495                 rval = -EIO;
1496                 table_del_fq(p, fq);
1497         }
1498 out:
1499         FQUNLOCK(fq);
1500         return rval;
1501 }
1502
1503 int qman_oos_fq(struct qman_fq *fq)
1504 {
1505         struct qm_mc_command *mcc;
1506         struct qm_mc_result *mcr;
1507         struct qman_portal *p;
1508
1509         int ret = 0;
1510         u8 res;
1511
1512         if (fq->state != qman_fq_state_retired)
1513                 return -EINVAL;
1514 #ifdef RTE_LIBRTE_DPAA_HWDEBUG
1515         if (unlikely(fq_isset(fq, QMAN_FQ_FLAG_NO_MODIFY)))
1516                 return -EINVAL;
1517 #endif
1518         p = get_affine_portal();
1519         FQLOCK(fq);
1520         if (unlikely((fq_isset(fq, QMAN_FQ_STATE_BLOCKOOS)) ||
1521                      (fq->state != qman_fq_state_retired))) {
1522                 ret = -EBUSY;
1523                 goto out;
1524         }
1525         mcc = qm_mc_start(&p->p);
1526         mcc->alterfq.fqid = cpu_to_be32(fq->fqid);
1527         qm_mc_commit(&p->p, QM_MCC_VERB_ALTER_OOS);
1528         while (!(mcr = qm_mc_result(&p->p)))
1529                 cpu_relax();
1530         DPAA_ASSERT((mcr->verb & QM_MCR_VERB_MASK) == QM_MCR_VERB_ALTER_OOS);
1531         res = mcr->result;
1532         if (res != QM_MCR_RESULT_OK) {
1533                 ret = -EIO;
1534                 goto out;
1535         }
1536         fq->state = qman_fq_state_oos;
1537 out:
1538         FQUNLOCK(fq);
1539         return ret;
1540 }
1541
1542 int qman_fq_flow_control(struct qman_fq *fq, int xon)
1543 {
1544         struct qm_mc_command *mcc;
1545         struct qm_mc_result *mcr;
1546         struct qman_portal *p;
1547
1548         int ret = 0;
1549         u8 res;
1550         u8 myverb;
1551
1552         if ((fq->state == qman_fq_state_oos) ||
1553             (fq->state == qman_fq_state_retired) ||
1554                 (fq->state == qman_fq_state_parked))
1555                 return -EINVAL;
1556
1557 #ifdef RTE_LIBRTE_DPAA_HWDEBUG
1558         if (unlikely(fq_isset(fq, QMAN_FQ_FLAG_NO_MODIFY)))
1559                 return -EINVAL;
1560 #endif
1561         /* Issue a ALTER_FQXON or ALTER_FQXOFF management command */
1562         p = get_affine_portal();
1563         FQLOCK(fq);
1564         if (unlikely((fq_isset(fq, QMAN_FQ_STATE_CHANGING)) ||
1565                      (fq->state == qman_fq_state_parked) ||
1566                         (fq->state == qman_fq_state_oos) ||
1567                         (fq->state == qman_fq_state_retired))) {
1568                 ret = -EBUSY;
1569                 goto out;
1570         }
1571         mcc = qm_mc_start(&p->p);
1572         mcc->alterfq.fqid = fq->fqid;
1573         mcc->alterfq.count = 0;
1574         myverb = xon ? QM_MCC_VERB_ALTER_FQXON : QM_MCC_VERB_ALTER_FQXOFF;
1575
1576         qm_mc_commit(&p->p, myverb);
1577         while (!(mcr = qm_mc_result(&p->p)))
1578                 cpu_relax();
1579         DPAA_ASSERT((mcr->verb & QM_MCR_VERB_MASK) == myverb);
1580
1581         res = mcr->result;
1582         if (res != QM_MCR_RESULT_OK) {
1583                 ret = -EIO;
1584                 goto out;
1585         }
1586 out:
1587         FQUNLOCK(fq);
1588         return ret;
1589 }
1590
1591 int qman_query_fq(struct qman_fq *fq, struct qm_fqd *fqd)
1592 {
1593         struct qm_mc_command *mcc;
1594         struct qm_mc_result *mcr;
1595         struct qman_portal *p = get_affine_portal();
1596
1597         u8 res;
1598
1599         mcc = qm_mc_start(&p->p);
1600         mcc->queryfq.fqid = cpu_to_be32(fq->fqid);
1601         qm_mc_commit(&p->p, QM_MCC_VERB_QUERYFQ);
1602         while (!(mcr = qm_mc_result(&p->p)))
1603                 cpu_relax();
1604         DPAA_ASSERT((mcr->verb & QM_MCR_VERB_MASK) == QM_MCR_VERB_QUERYFQ);
1605         res = mcr->result;
1606         if (res == QM_MCR_RESULT_OK)
1607                 *fqd = mcr->queryfq.fqd;
1608         hw_fqd_to_cpu(fqd);
1609         if (res != QM_MCR_RESULT_OK)
1610                 return -EIO;
1611         return 0;
1612 }
1613
1614 int qman_query_fq_has_pkts(struct qman_fq *fq)
1615 {
1616         struct qm_mc_command *mcc;
1617         struct qm_mc_result *mcr;
1618         struct qman_portal *p = get_affine_portal();
1619
1620         int ret = 0;
1621         u8 res;
1622
1623         mcc = qm_mc_start(&p->p);
1624         mcc->queryfq.fqid = cpu_to_be32(fq->fqid);
1625         qm_mc_commit(&p->p, QM_MCC_VERB_QUERYFQ_NP);
1626         while (!(mcr = qm_mc_result(&p->p)))
1627                 cpu_relax();
1628         res = mcr->result;
1629         if (res == QM_MCR_RESULT_OK)
1630                 ret = !!mcr->queryfq_np.frm_cnt;
1631         return ret;
1632 }
1633
1634 int qman_query_fq_np(struct qman_fq *fq, struct qm_mcr_queryfq_np *np)
1635 {
1636         struct qm_mc_command *mcc;
1637         struct qm_mc_result *mcr;
1638         struct qman_portal *p = get_affine_portal();
1639
1640         u8 res;
1641
1642         mcc = qm_mc_start(&p->p);
1643         mcc->queryfq.fqid = cpu_to_be32(fq->fqid);
1644         qm_mc_commit(&p->p, QM_MCC_VERB_QUERYFQ_NP);
1645         while (!(mcr = qm_mc_result(&p->p)))
1646                 cpu_relax();
1647         DPAA_ASSERT((mcr->verb & QM_MCR_VERB_MASK) == QM_MCR_VERB_QUERYFQ_NP);
1648         res = mcr->result;
1649         if (res == QM_MCR_RESULT_OK) {
1650                 *np = mcr->queryfq_np;
1651                 np->fqd_link = be24_to_cpu(np->fqd_link);
1652                 np->odp_seq = be16_to_cpu(np->odp_seq);
1653                 np->orp_nesn = be16_to_cpu(np->orp_nesn);
1654                 np->orp_ea_hseq  = be16_to_cpu(np->orp_ea_hseq);
1655                 np->orp_ea_tseq  = be16_to_cpu(np->orp_ea_tseq);
1656                 np->orp_ea_hptr = be24_to_cpu(np->orp_ea_hptr);
1657                 np->orp_ea_tptr = be24_to_cpu(np->orp_ea_tptr);
1658                 np->pfdr_hptr = be24_to_cpu(np->pfdr_hptr);
1659                 np->pfdr_tptr = be24_to_cpu(np->pfdr_tptr);
1660                 np->ics_surp = be16_to_cpu(np->ics_surp);
1661                 np->byte_cnt = be32_to_cpu(np->byte_cnt);
1662                 np->frm_cnt = be24_to_cpu(np->frm_cnt);
1663                 np->ra1_sfdr = be16_to_cpu(np->ra1_sfdr);
1664                 np->ra2_sfdr = be16_to_cpu(np->ra2_sfdr);
1665                 np->od1_sfdr = be16_to_cpu(np->od1_sfdr);
1666                 np->od2_sfdr = be16_to_cpu(np->od2_sfdr);
1667                 np->od3_sfdr = be16_to_cpu(np->od3_sfdr);
1668         }
1669         if (res == QM_MCR_RESULT_ERR_FQID)
1670                 return -ERANGE;
1671         else if (res != QM_MCR_RESULT_OK)
1672                 return -EIO;
1673         return 0;
1674 }
1675
1676 int qman_query_wq(u8 query_dedicated, struct qm_mcr_querywq *wq)
1677 {
1678         struct qm_mc_command *mcc;
1679         struct qm_mc_result *mcr;
1680         struct qman_portal *p = get_affine_portal();
1681
1682         u8 res, myverb;
1683
1684         myverb = (query_dedicated) ? QM_MCR_VERB_QUERYWQ_DEDICATED :
1685                                  QM_MCR_VERB_QUERYWQ;
1686         mcc = qm_mc_start(&p->p);
1687         mcc->querywq.channel.id = cpu_to_be16(wq->channel.id);
1688         qm_mc_commit(&p->p, myverb);
1689         while (!(mcr = qm_mc_result(&p->p)))
1690                 cpu_relax();
1691         DPAA_ASSERT((mcr->verb & QM_MCR_VERB_MASK) == myverb);
1692         res = mcr->result;
1693         if (res == QM_MCR_RESULT_OK) {
1694                 int i, array_len;
1695
1696                 wq->channel.id = be16_to_cpu(mcr->querywq.channel.id);
1697                 array_len = ARRAY_SIZE(mcr->querywq.wq_len);
1698                 for (i = 0; i < array_len; i++)
1699                         wq->wq_len[i] = be32_to_cpu(mcr->querywq.wq_len[i]);
1700         }
1701         if (res != QM_MCR_RESULT_OK) {
1702                 pr_err("QUERYWQ failed: %s\n", mcr_result_str(res));
1703                 return -EIO;
1704         }
1705         return 0;
1706 }
1707
1708 int qman_testwrite_cgr(struct qman_cgr *cgr, u64 i_bcnt,
1709                        struct qm_mcr_cgrtestwrite *result)
1710 {
1711         struct qm_mc_command *mcc;
1712         struct qm_mc_result *mcr;
1713         struct qman_portal *p = get_affine_portal();
1714
1715         u8 res;
1716
1717         mcc = qm_mc_start(&p->p);
1718         mcc->cgrtestwrite.cgid = cgr->cgrid;
1719         mcc->cgrtestwrite.i_bcnt_hi = (u8)(i_bcnt >> 32);
1720         mcc->cgrtestwrite.i_bcnt_lo = (u32)i_bcnt;
1721         qm_mc_commit(&p->p, QM_MCC_VERB_CGRTESTWRITE);
1722         while (!(mcr = qm_mc_result(&p->p)))
1723                 cpu_relax();
1724         DPAA_ASSERT((mcr->verb & QM_MCR_VERB_MASK) == QM_MCC_VERB_CGRTESTWRITE);
1725         res = mcr->result;
1726         if (res == QM_MCR_RESULT_OK)
1727                 *result = mcr->cgrtestwrite;
1728         if (res != QM_MCR_RESULT_OK) {
1729                 pr_err("CGR TEST WRITE failed: %s\n", mcr_result_str(res));
1730                 return -EIO;
1731         }
1732         return 0;
1733 }
1734
1735 int qman_query_cgr(struct qman_cgr *cgr, struct qm_mcr_querycgr *cgrd)
1736 {
1737         struct qm_mc_command *mcc;
1738         struct qm_mc_result *mcr;
1739         struct qman_portal *p = get_affine_portal();
1740         u8 res;
1741         unsigned int i;
1742
1743         mcc = qm_mc_start(&p->p);
1744         mcc->querycgr.cgid = cgr->cgrid;
1745         qm_mc_commit(&p->p, QM_MCC_VERB_QUERYCGR);
1746         while (!(mcr = qm_mc_result(&p->p)))
1747                 cpu_relax();
1748         DPAA_ASSERT((mcr->verb & QM_MCR_VERB_MASK) == QM_MCC_VERB_QUERYCGR);
1749         res = mcr->result;
1750         if (res == QM_MCR_RESULT_OK)
1751                 *cgrd = mcr->querycgr;
1752         if (res != QM_MCR_RESULT_OK) {
1753                 pr_err("QUERY_CGR failed: %s\n", mcr_result_str(res));
1754                 return -EIO;
1755         }
1756         cgrd->cgr.wr_parm_g.word =
1757                 be32_to_cpu(cgrd->cgr.wr_parm_g.word);
1758         cgrd->cgr.wr_parm_y.word =
1759                 be32_to_cpu(cgrd->cgr.wr_parm_y.word);
1760         cgrd->cgr.wr_parm_r.word =
1761                 be32_to_cpu(cgrd->cgr.wr_parm_r.word);
1762         cgrd->cgr.cscn_targ =  be32_to_cpu(cgrd->cgr.cscn_targ);
1763         cgrd->cgr.__cs_thres = be16_to_cpu(cgrd->cgr.__cs_thres);
1764         for (i = 0; i < ARRAY_SIZE(cgrd->cscn_targ_swp); i++)
1765                 cgrd->cscn_targ_swp[i] =
1766                         be32_to_cpu(cgrd->cscn_targ_swp[i]);
1767         return 0;
1768 }
1769
1770 int qman_query_congestion(struct qm_mcr_querycongestion *congestion)
1771 {
1772         struct qm_mc_result *mcr;
1773         struct qman_portal *p = get_affine_portal();
1774         u8 res;
1775         unsigned int i;
1776
1777         qm_mc_start(&p->p);
1778         qm_mc_commit(&p->p, QM_MCC_VERB_QUERYCONGESTION);
1779         while (!(mcr = qm_mc_result(&p->p)))
1780                 cpu_relax();
1781         DPAA_ASSERT((mcr->verb & QM_MCR_VERB_MASK) ==
1782                         QM_MCC_VERB_QUERYCONGESTION);
1783         res = mcr->result;
1784         if (res == QM_MCR_RESULT_OK)
1785                 *congestion = mcr->querycongestion;
1786         if (res != QM_MCR_RESULT_OK) {
1787                 pr_err("QUERY_CONGESTION failed: %s\n", mcr_result_str(res));
1788                 return -EIO;
1789         }
1790         for (i = 0; i < ARRAY_SIZE(congestion->state.state); i++)
1791                 congestion->state.state[i] =
1792                         be32_to_cpu(congestion->state.state[i]);
1793         return 0;
1794 }
1795
1796 int qman_set_vdq(struct qman_fq *fq, u16 num)
1797 {
1798         struct qman_portal *p = get_affine_portal();
1799         uint32_t vdqcr;
1800         int ret = -EBUSY;
1801
1802         vdqcr = QM_VDQCR_EXACT;
1803         vdqcr |= QM_VDQCR_NUMFRAMES_SET(num);
1804
1805         if ((fq->state != qman_fq_state_parked) &&
1806             (fq->state != qman_fq_state_retired)) {
1807                 ret = -EINVAL;
1808                 goto out;
1809         }
1810         if (fq_isset(fq, QMAN_FQ_STATE_VDQCR)) {
1811                 ret = -EBUSY;
1812                 goto out;
1813         }
1814         vdqcr = (vdqcr & ~QM_VDQCR_FQID_MASK) | fq->fqid;
1815
1816         if (!p->vdqcr_owned) {
1817                 FQLOCK(fq);
1818                 if (fq_isset(fq, QMAN_FQ_STATE_VDQCR))
1819                         goto escape;
1820                 fq_set(fq, QMAN_FQ_STATE_VDQCR);
1821                 FQUNLOCK(fq);
1822                 p->vdqcr_owned = fq;
1823                 ret = 0;
1824         }
1825 escape:
1826         if (!ret)
1827                 qm_dqrr_vdqcr_set(&p->p, vdqcr);
1828
1829 out:
1830         return ret;
1831 }
1832
1833 int qman_volatile_dequeue(struct qman_fq *fq, u32 flags __maybe_unused,
1834                           u32 vdqcr)
1835 {
1836         struct qman_portal *p;
1837         int ret = -EBUSY;
1838
1839         if ((fq->state != qman_fq_state_parked) &&
1840             (fq->state != qman_fq_state_retired))
1841                 return -EINVAL;
1842         if (vdqcr & QM_VDQCR_FQID_MASK)
1843                 return -EINVAL;
1844         if (fq_isset(fq, QMAN_FQ_STATE_VDQCR))
1845                 return -EBUSY;
1846         vdqcr = (vdqcr & ~QM_VDQCR_FQID_MASK) | fq->fqid;
1847
1848         p = get_affine_portal();
1849
1850         if (!p->vdqcr_owned) {
1851                 FQLOCK(fq);
1852                 if (fq_isset(fq, QMAN_FQ_STATE_VDQCR))
1853                         goto escape;
1854                 fq_set(fq, QMAN_FQ_STATE_VDQCR);
1855                 FQUNLOCK(fq);
1856                 p->vdqcr_owned = fq;
1857                 ret = 0;
1858         }
1859 escape:
1860         if (ret)
1861                 return ret;
1862
1863         /* VDQCR is set */
1864         qm_dqrr_vdqcr_set(&p->p, vdqcr);
1865         return 0;
1866 }
1867
1868 static noinline void update_eqcr_ci(struct qman_portal *p, u8 avail)
1869 {
1870         if (avail)
1871                 qm_eqcr_cce_prefetch(&p->p);
1872         else
1873                 qm_eqcr_cce_update(&p->p);
1874 }
1875
1876 int qman_eqcr_is_empty(void)
1877 {
1878         struct qman_portal *p = get_affine_portal();
1879         u8 avail;
1880
1881         update_eqcr_ci(p, 0);
1882         avail = qm_eqcr_get_fill(&p->p);
1883         return (avail == 0);
1884 }
1885
1886 void qman_set_dc_ern(qman_cb_dc_ern handler, int affine)
1887 {
1888         if (affine) {
1889                 struct qman_portal *p = get_affine_portal();
1890
1891                 p->cb_dc_ern = handler;
1892         } else
1893                 cb_dc_ern = handler;
1894 }
1895
1896 static inline struct qm_eqcr_entry *try_p_eq_start(struct qman_portal *p,
1897                                         struct qman_fq *fq,
1898                                         const struct qm_fd *fd,
1899                                         u32 flags)
1900 {
1901         struct qm_eqcr_entry *eq;
1902         u8 avail;
1903
1904         if (p->use_eqcr_ci_stashing) {
1905                 /*
1906                  * The stashing case is easy, only update if we need to in
1907                  * order to try and liberate ring entries.
1908                  */
1909                 eq = qm_eqcr_start_stash(&p->p);
1910         } else {
1911                 /*
1912                  * The non-stashing case is harder, need to prefetch ahead of
1913                  * time.
1914                  */
1915                 avail = qm_eqcr_get_avail(&p->p);
1916                 if (avail < 2)
1917                         update_eqcr_ci(p, avail);
1918                 eq = qm_eqcr_start_no_stash(&p->p);
1919         }
1920
1921         if (unlikely(!eq))
1922                 return NULL;
1923
1924         if (flags & QMAN_ENQUEUE_FLAG_DCA)
1925                 eq->dca = QM_EQCR_DCA_ENABLE |
1926                         ((flags & QMAN_ENQUEUE_FLAG_DCA_PARK) ?
1927                                         QM_EQCR_DCA_PARK : 0) |
1928                         ((flags >> 8) & QM_EQCR_DCA_IDXMASK);
1929         eq->fqid = cpu_to_be32(fq->fqid);
1930 #ifdef CONFIG_FSL_QMAN_FQ_LOOKUP
1931         eq->tag = cpu_to_be32(fq->key);
1932 #else
1933         eq->tag = cpu_to_be32((u32)(uintptr_t)fq);
1934 #endif
1935         eq->fd = *fd;
1936         cpu_to_hw_fd(&eq->fd);
1937         return eq;
1938 }
1939
1940 int qman_enqueue(struct qman_fq *fq, const struct qm_fd *fd, u32 flags)
1941 {
1942         struct qman_portal *p = get_affine_portal();
1943         struct qm_eqcr_entry *eq;
1944
1945         eq = try_p_eq_start(p, fq, fd, flags);
1946         if (!eq)
1947                 return -EBUSY;
1948         /* Note: QM_EQCR_VERB_INTERRUPT == QMAN_ENQUEUE_FLAG_WAIT_SYNC */
1949         qm_eqcr_pvb_commit(&p->p, QM_EQCR_VERB_CMD_ENQUEUE |
1950                 (flags & (QM_EQCR_VERB_COLOUR_MASK | QM_EQCR_VERB_INTERRUPT)));
1951         /* Factor the below out, it's used from qman_enqueue_orp() too */
1952         return 0;
1953 }
1954
1955 int qman_enqueue_multi(struct qman_fq *fq,
1956                        const struct qm_fd *fd,
1957                        int frames_to_send)
1958 {
1959         struct qman_portal *p = get_affine_portal();
1960         struct qm_portal *portal = &p->p;
1961
1962         register struct qm_eqcr *eqcr = &portal->eqcr;
1963         struct qm_eqcr_entry *eq = eqcr->cursor, *prev_eq;
1964
1965         u8 i, diff, old_ci, sent = 0;
1966
1967         /* Update the available entries if no entry is free */
1968         if (!eqcr->available) {
1969                 old_ci = eqcr->ci;
1970                 eqcr->ci = qm_cl_in(EQCR_CI) & (QM_EQCR_SIZE - 1);
1971                 diff = qm_cyc_diff(QM_EQCR_SIZE, old_ci, eqcr->ci);
1972                 eqcr->available += diff;
1973                 if (!diff)
1974                         return 0;
1975         }
1976
1977         /* try to send as many frames as possible */
1978         while (eqcr->available && frames_to_send--) {
1979                 eq->fqid = fq->fqid_le;
1980 #ifdef CONFIG_FSL_QMAN_FQ_LOOKUP
1981                 eq->tag = cpu_to_be32(fq->key);
1982 #else
1983                 eq->tag = cpu_to_be32((u32)(uintptr_t)fq);
1984 #endif
1985                 eq->fd.opaque_addr = fd->opaque_addr;
1986                 eq->fd.addr = cpu_to_be40(fd->addr);
1987                 eq->fd.status = cpu_to_be32(fd->status);
1988                 eq->fd.opaque = cpu_to_be32(fd->opaque);
1989
1990                 eq = (void *)((unsigned long)(eq + 1) &
1991                         (~(unsigned long)(QM_EQCR_SIZE << 6)));
1992                 eqcr->available--;
1993                 sent++;
1994                 fd++;
1995         }
1996         lwsync();
1997
1998         /* In order for flushes to complete faster, all lines are recorded in
1999          * 32 bit word.
2000          */
2001         eq = eqcr->cursor;
2002         for (i = 0; i < sent; i++) {
2003                 eq->__dont_write_directly__verb =
2004                         QM_EQCR_VERB_CMD_ENQUEUE | eqcr->vbit;
2005                 prev_eq = eq;
2006                 eq = (void *)((unsigned long)(eq + 1) &
2007                         (~(unsigned long)(QM_EQCR_SIZE << 6)));
2008                 if (unlikely((prev_eq + 1) != eq))
2009                         eqcr->vbit ^= QM_EQCR_VERB_VBIT;
2010         }
2011
2012         /* We need  to flush all the lines but without load/store operations
2013          * between them
2014          */
2015         eq = eqcr->cursor;
2016         for (i = 0; i < sent; i++) {
2017                 dcbf(eq);
2018                 eq = (void *)((unsigned long)(eq + 1) &
2019                         (~(unsigned long)(QM_EQCR_SIZE << 6)));
2020         }
2021         /* Update cursor for the next call */
2022         eqcr->cursor = eq;
2023         return sent;
2024 }
2025
2026 int qman_enqueue_orp(struct qman_fq *fq, const struct qm_fd *fd, u32 flags,
2027                      struct qman_fq *orp, u16 orp_seqnum)
2028 {
2029         struct qman_portal *p  = get_affine_portal();
2030         struct qm_eqcr_entry *eq;
2031
2032         eq = try_p_eq_start(p, fq, fd, flags);
2033         if (!eq)
2034                 return -EBUSY;
2035         /* Process ORP-specifics here */
2036         if (flags & QMAN_ENQUEUE_FLAG_NLIS)
2037                 orp_seqnum |= QM_EQCR_SEQNUM_NLIS;
2038         else {
2039                 orp_seqnum &= ~QM_EQCR_SEQNUM_NLIS;
2040                 if (flags & QMAN_ENQUEUE_FLAG_NESN)
2041                         orp_seqnum |= QM_EQCR_SEQNUM_NESN;
2042                 else
2043                         /* No need to check 4 QMAN_ENQUEUE_FLAG_HOLE */
2044                         orp_seqnum &= ~QM_EQCR_SEQNUM_NESN;
2045         }
2046         eq->seqnum = cpu_to_be16(orp_seqnum);
2047         eq->orp = cpu_to_be32(orp->fqid);
2048         /* Note: QM_EQCR_VERB_INTERRUPT == QMAN_ENQUEUE_FLAG_WAIT_SYNC */
2049         qm_eqcr_pvb_commit(&p->p, QM_EQCR_VERB_ORP |
2050                 ((flags & (QMAN_ENQUEUE_FLAG_HOLE | QMAN_ENQUEUE_FLAG_NESN)) ?
2051                                 0 : QM_EQCR_VERB_CMD_ENQUEUE) |
2052                 (flags & (QM_EQCR_VERB_COLOUR_MASK | QM_EQCR_VERB_INTERRUPT)));
2053
2054         return 0;
2055 }
2056
2057 int qman_modify_cgr(struct qman_cgr *cgr, u32 flags,
2058                     struct qm_mcc_initcgr *opts)
2059 {
2060         struct qm_mc_command *mcc;
2061         struct qm_mc_result *mcr;
2062         struct qman_portal *p = get_affine_portal();
2063
2064         u8 res;
2065         u8 verb = QM_MCC_VERB_MODIFYCGR;
2066
2067         mcc = qm_mc_start(&p->p);
2068         if (opts)
2069                 mcc->initcgr = *opts;
2070         mcc->initcgr.we_mask = cpu_to_be16(mcc->initcgr.we_mask);
2071         mcc->initcgr.cgr.wr_parm_g.word =
2072                 cpu_to_be32(mcc->initcgr.cgr.wr_parm_g.word);
2073         mcc->initcgr.cgr.wr_parm_y.word =
2074                 cpu_to_be32(mcc->initcgr.cgr.wr_parm_y.word);
2075         mcc->initcgr.cgr.wr_parm_r.word =
2076                 cpu_to_be32(mcc->initcgr.cgr.wr_parm_r.word);
2077         mcc->initcgr.cgr.cscn_targ =  cpu_to_be32(mcc->initcgr.cgr.cscn_targ);
2078         mcc->initcgr.cgr.__cs_thres = cpu_to_be16(mcc->initcgr.cgr.__cs_thres);
2079
2080         mcc->initcgr.cgid = cgr->cgrid;
2081         if (flags & QMAN_CGR_FLAG_USE_INIT)
2082                 verb = QM_MCC_VERB_INITCGR;
2083         qm_mc_commit(&p->p, verb);
2084         while (!(mcr = qm_mc_result(&p->p)))
2085                 cpu_relax();
2086
2087         DPAA_ASSERT((mcr->verb & QM_MCR_VERB_MASK) == verb);
2088         res = mcr->result;
2089         return (res == QM_MCR_RESULT_OK) ? 0 : -EIO;
2090 }
2091
2092 #define TARG_MASK(n) (0x80000000 >> (n->config->channel - \
2093                                         QM_CHANNEL_SWPORTAL0))
2094 #define TARG_DCP_MASK(n) (0x80000000 >> (10 + n))
2095 #define PORTAL_IDX(n) (n->config->channel - QM_CHANNEL_SWPORTAL0)
2096
2097 int qman_create_cgr(struct qman_cgr *cgr, u32 flags,
2098                     struct qm_mcc_initcgr *opts)
2099 {
2100         struct qm_mcr_querycgr cgr_state;
2101         struct qm_mcc_initcgr local_opts;
2102         int ret;
2103         struct qman_portal *p;
2104
2105         /* We have to check that the provided CGRID is within the limits of the
2106          * data-structures, for obvious reasons. However we'll let h/w take
2107          * care of determining whether it's within the limits of what exists on
2108          * the SoC.
2109          */
2110         if (cgr->cgrid >= __CGR_NUM)
2111                 return -EINVAL;
2112
2113         p = get_affine_portal();
2114
2115         memset(&local_opts, 0, sizeof(struct qm_mcc_initcgr));
2116         cgr->chan = p->config->channel;
2117         spin_lock(&p->cgr_lock);
2118
2119         /* if no opts specified, just add it to the list */
2120         if (!opts)
2121                 goto add_list;
2122
2123         ret = qman_query_cgr(cgr, &cgr_state);
2124         if (ret)
2125                 goto release_lock;
2126         if (opts)
2127                 local_opts = *opts;
2128         if ((qman_ip_rev & 0xFF00) >= QMAN_REV30)
2129                 local_opts.cgr.cscn_targ_upd_ctrl =
2130                         QM_CGR_TARG_UDP_CTRL_WRITE_BIT | PORTAL_IDX(p);
2131         else
2132                 /* Overwrite TARG */
2133                 local_opts.cgr.cscn_targ = cgr_state.cgr.cscn_targ |
2134                                                         TARG_MASK(p);
2135         local_opts.we_mask |= QM_CGR_WE_CSCN_TARG;
2136
2137         /* send init if flags indicate so */
2138         if (opts && (flags & QMAN_CGR_FLAG_USE_INIT))
2139                 ret = qman_modify_cgr(cgr, QMAN_CGR_FLAG_USE_INIT, &local_opts);
2140         else
2141                 ret = qman_modify_cgr(cgr, 0, &local_opts);
2142         if (ret)
2143                 goto release_lock;
2144 add_list:
2145         list_add(&cgr->node, &p->cgr_cbs);
2146
2147         /* Determine if newly added object requires its callback to be called */
2148         ret = qman_query_cgr(cgr, &cgr_state);
2149         if (ret) {
2150                 /* we can't go back, so proceed and return success, but screen
2151                  * and wail to the log file.
2152                  */
2153                 pr_crit("CGR HW state partially modified\n");
2154                 ret = 0;
2155                 goto release_lock;
2156         }
2157         if (cgr->cb && cgr_state.cgr.cscn_en && qman_cgrs_get(&p->cgrs[1],
2158                                                               cgr->cgrid))
2159                 cgr->cb(p, cgr, 1);
2160 release_lock:
2161         spin_unlock(&p->cgr_lock);
2162         return ret;
2163 }
2164
2165 int qman_create_cgr_to_dcp(struct qman_cgr *cgr, u32 flags, u16 dcp_portal,
2166                            struct qm_mcc_initcgr *opts)
2167 {
2168         struct qm_mcc_initcgr local_opts;
2169         struct qm_mcr_querycgr cgr_state;
2170         int ret;
2171
2172         if ((qman_ip_rev & 0xFF00) < QMAN_REV30) {
2173                 pr_warn("QMan version doesn't support CSCN => DCP portal\n");
2174                 return -EINVAL;
2175         }
2176         /* We have to check that the provided CGRID is within the limits of the
2177          * data-structures, for obvious reasons. However we'll let h/w take
2178          * care of determining whether it's within the limits of what exists on
2179          * the SoC.
2180          */
2181         if (cgr->cgrid >= __CGR_NUM)
2182                 return -EINVAL;
2183
2184         ret = qman_query_cgr(cgr, &cgr_state);
2185         if (ret)
2186                 return ret;
2187
2188         memset(&local_opts, 0, sizeof(struct qm_mcc_initcgr));
2189         if (opts)
2190                 local_opts = *opts;
2191
2192         if ((qman_ip_rev & 0xFF00) >= QMAN_REV30)
2193                 local_opts.cgr.cscn_targ_upd_ctrl =
2194                                 QM_CGR_TARG_UDP_CTRL_WRITE_BIT |
2195                                 QM_CGR_TARG_UDP_CTRL_DCP | dcp_portal;
2196         else
2197                 local_opts.cgr.cscn_targ = cgr_state.cgr.cscn_targ |
2198                                         TARG_DCP_MASK(dcp_portal);
2199         local_opts.we_mask |= QM_CGR_WE_CSCN_TARG;
2200
2201         /* send init if flags indicate so */
2202         if (opts && (flags & QMAN_CGR_FLAG_USE_INIT))
2203                 ret = qman_modify_cgr(cgr, QMAN_CGR_FLAG_USE_INIT,
2204                                       &local_opts);
2205         else
2206                 ret = qman_modify_cgr(cgr, 0, &local_opts);
2207
2208         return ret;
2209 }
2210
2211 int qman_delete_cgr(struct qman_cgr *cgr)
2212 {
2213         struct qm_mcr_querycgr cgr_state;
2214         struct qm_mcc_initcgr local_opts;
2215         int ret = 0;
2216         struct qman_cgr *i;
2217         struct qman_portal *p = get_affine_portal();
2218
2219         if (cgr->chan != p->config->channel) {
2220                 pr_crit("Attempting to delete cgr from different portal than"
2221                         " it was create: create 0x%x, delete 0x%x\n",
2222                         cgr->chan, p->config->channel);
2223                 ret = -EINVAL;
2224                 goto put_portal;
2225         }
2226         memset(&local_opts, 0, sizeof(struct qm_mcc_initcgr));
2227         spin_lock(&p->cgr_lock);
2228         list_del(&cgr->node);
2229         /*
2230          * If there are no other CGR objects for this CGRID in the list,
2231          * update CSCN_TARG accordingly
2232          */
2233         list_for_each_entry(i, &p->cgr_cbs, node)
2234                 if ((i->cgrid == cgr->cgrid) && i->cb)
2235                         goto release_lock;
2236         ret = qman_query_cgr(cgr, &cgr_state);
2237         if (ret)  {
2238                 /* add back to the list */
2239                 list_add(&cgr->node, &p->cgr_cbs);
2240                 goto release_lock;
2241         }
2242         /* Overwrite TARG */
2243         local_opts.we_mask = QM_CGR_WE_CSCN_TARG;
2244         if ((qman_ip_rev & 0xFF00) >= QMAN_REV30)
2245                 local_opts.cgr.cscn_targ_upd_ctrl = PORTAL_IDX(p);
2246         else
2247                 local_opts.cgr.cscn_targ = cgr_state.cgr.cscn_targ &
2248                                                          ~(TARG_MASK(p));
2249         ret = qman_modify_cgr(cgr, 0, &local_opts);
2250         if (ret)
2251                 /* add back to the list */
2252                 list_add(&cgr->node, &p->cgr_cbs);
2253 release_lock:
2254         spin_unlock(&p->cgr_lock);
2255 put_portal:
2256         return ret;
2257 }
2258
2259 int qman_shutdown_fq(u32 fqid)
2260 {
2261         struct qman_portal *p;
2262         struct qm_portal *low_p;
2263         struct qm_mc_command *mcc;
2264         struct qm_mc_result *mcr;
2265         u8 state;
2266         int orl_empty, fq_empty, drain = 0;
2267         u32 result;
2268         u32 channel, wq;
2269         u16 dest_wq;
2270
2271         p = get_affine_portal();
2272         low_p = &p->p;
2273
2274         /* Determine the state of the FQID */
2275         mcc = qm_mc_start(low_p);
2276         mcc->queryfq_np.fqid = cpu_to_be32(fqid);
2277         qm_mc_commit(low_p, QM_MCC_VERB_QUERYFQ_NP);
2278         while (!(mcr = qm_mc_result(low_p)))
2279                 cpu_relax();
2280         DPAA_ASSERT((mcr->verb & QM_MCR_VERB_MASK) == QM_MCR_VERB_QUERYFQ_NP);
2281         state = mcr->queryfq_np.state & QM_MCR_NP_STATE_MASK;
2282         if (state == QM_MCR_NP_STATE_OOS)
2283                 return 0; /* Already OOS, no need to do anymore checks */
2284
2285         /* Query which channel the FQ is using */
2286         mcc = qm_mc_start(low_p);
2287         mcc->queryfq.fqid = cpu_to_be32(fqid);
2288         qm_mc_commit(low_p, QM_MCC_VERB_QUERYFQ);
2289         while (!(mcr = qm_mc_result(low_p)))
2290                 cpu_relax();
2291         DPAA_ASSERT((mcr->verb & QM_MCR_VERB_MASK) == QM_MCR_VERB_QUERYFQ);
2292
2293         /* Need to store these since the MCR gets reused */
2294         dest_wq = be16_to_cpu(mcr->queryfq.fqd.dest_wq);
2295         channel = dest_wq & 0x7;
2296         wq = dest_wq >> 3;
2297
2298         switch (state) {
2299         case QM_MCR_NP_STATE_TEN_SCHED:
2300         case QM_MCR_NP_STATE_TRU_SCHED:
2301         case QM_MCR_NP_STATE_ACTIVE:
2302         case QM_MCR_NP_STATE_PARKED:
2303                 orl_empty = 0;
2304                 mcc = qm_mc_start(low_p);
2305                 mcc->alterfq.fqid = cpu_to_be32(fqid);
2306                 qm_mc_commit(low_p, QM_MCC_VERB_ALTER_RETIRE);
2307                 while (!(mcr = qm_mc_result(low_p)))
2308                         cpu_relax();
2309                 DPAA_ASSERT((mcr->verb & QM_MCR_VERB_MASK) ==
2310                            QM_MCR_VERB_ALTER_RETIRE);
2311                 result = mcr->result; /* Make a copy as we reuse MCR below */
2312
2313                 if (result == QM_MCR_RESULT_PENDING) {
2314                         /* Need to wait for the FQRN in the message ring, which
2315                          * will only occur once the FQ has been drained.  In
2316                          * order for the FQ to drain the portal needs to be set
2317                          * to dequeue from the channel the FQ is scheduled on
2318                          */
2319                         const struct qm_mr_entry *msg;
2320                         const struct qm_dqrr_entry *dqrr = NULL;
2321                         int found_fqrn = 0;
2322                         __maybe_unused u16 dequeue_wq = 0;
2323
2324                         /* Flag that we need to drain FQ */
2325                         drain = 1;
2326
2327                         if (channel >= qm_channel_pool1 &&
2328                             channel < (u16)(qm_channel_pool1 + 15)) {
2329                                 /* Pool channel, enable the bit in the portal */
2330                                 dequeue_wq = (channel -
2331                                               qm_channel_pool1 + 1) << 4 | wq;
2332                         } else if (channel < qm_channel_pool1) {
2333                                 /* Dedicated channel */
2334                                 dequeue_wq = wq;
2335                         } else {
2336                                 pr_info("Cannot recover FQ 0x%x,"
2337                                         " it is scheduled on channel 0x%x",
2338                                         fqid, channel);
2339                                 return -EBUSY;
2340                         }
2341                         /* Set the sdqcr to drain this channel */
2342                         if (channel < qm_channel_pool1)
2343                                 qm_dqrr_sdqcr_set(low_p,
2344                                                   QM_SDQCR_TYPE_ACTIVE |
2345                                           QM_SDQCR_CHANNELS_DEDICATED);
2346                         else
2347                                 qm_dqrr_sdqcr_set(low_p,
2348                                                   QM_SDQCR_TYPE_ACTIVE |
2349                                                   QM_SDQCR_CHANNELS_POOL_CONV
2350                                                   (channel));
2351                         while (!found_fqrn) {
2352                                 /* Keep draining DQRR while checking the MR*/
2353                                 qm_dqrr_pvb_update(low_p);
2354                                 dqrr = qm_dqrr_current(low_p);
2355                                 while (dqrr) {
2356                                         qm_dqrr_cdc_consume_1ptr(
2357                                                 low_p, dqrr, 0);
2358                                         qm_dqrr_pvb_update(low_p);
2359                                         qm_dqrr_next(low_p);
2360                                         dqrr = qm_dqrr_current(low_p);
2361                                 }
2362                                 /* Process message ring too */
2363                                 qm_mr_pvb_update(low_p);
2364                                 msg = qm_mr_current(low_p);
2365                                 while (msg) {
2366                                         if ((msg->verb &
2367                                              QM_MR_VERB_TYPE_MASK)
2368                                             == QM_MR_VERB_FQRN)
2369                                                 found_fqrn = 1;
2370                                         qm_mr_next(low_p);
2371                                         qm_mr_cci_consume_to_current(low_p);
2372                                         qm_mr_pvb_update(low_p);
2373                                         msg = qm_mr_current(low_p);
2374                                 }
2375                                 cpu_relax();
2376                         }
2377                 }
2378                 if (result != QM_MCR_RESULT_OK &&
2379                     result !=  QM_MCR_RESULT_PENDING) {
2380                         /* error */
2381                         pr_err("qman_retire_fq failed on FQ 0x%x,"
2382                                " result=0x%x\n", fqid, result);
2383                         return -1;
2384                 }
2385                 if (!(mcr->alterfq.fqs & QM_MCR_FQS_ORLPRESENT)) {
2386                         /* ORL had no entries, no need to wait until the
2387                          * ERNs come in.
2388                          */
2389                         orl_empty = 1;
2390                 }
2391                 /* Retirement succeeded, check to see if FQ needs
2392                  * to be drained.
2393                  */
2394                 if (drain || mcr->alterfq.fqs & QM_MCR_FQS_NOTEMPTY) {
2395                         /* FQ is Not Empty, drain using volatile DQ commands */
2396                         fq_empty = 0;
2397                         do {
2398                                 const struct qm_dqrr_entry *dqrr = NULL;
2399                                 u32 vdqcr = fqid | QM_VDQCR_NUMFRAMES_SET(3);
2400
2401                                 qm_dqrr_vdqcr_set(low_p, vdqcr);
2402
2403                                 /* Wait for a dequeue to occur */
2404                                 while (dqrr == NULL) {
2405                                         qm_dqrr_pvb_update(low_p);
2406                                         dqrr = qm_dqrr_current(low_p);
2407                                         if (!dqrr)
2408                                                 cpu_relax();
2409                                 }
2410                                 /* Process the dequeues, making sure to
2411                                  * empty the ring completely.
2412                                  */
2413                                 while (dqrr) {
2414                                         if (dqrr->fqid == fqid &&
2415                                             dqrr->stat & QM_DQRR_STAT_FQ_EMPTY)
2416                                                 fq_empty = 1;
2417                                         qm_dqrr_cdc_consume_1ptr(low_p,
2418                                                                  dqrr, 0);
2419                                         qm_dqrr_pvb_update(low_p);
2420                                         qm_dqrr_next(low_p);
2421                                         dqrr = qm_dqrr_current(low_p);
2422                                 }
2423                         } while (fq_empty == 0);
2424                 }
2425                 qm_dqrr_sdqcr_set(low_p, 0);
2426
2427                 /* Wait for the ORL to have been completely drained */
2428                 while (orl_empty == 0) {
2429                         const struct qm_mr_entry *msg;
2430
2431                         qm_mr_pvb_update(low_p);
2432                         msg = qm_mr_current(low_p);
2433                         while (msg) {
2434                                 if ((msg->verb & QM_MR_VERB_TYPE_MASK) ==
2435                                     QM_MR_VERB_FQRL)
2436                                         orl_empty = 1;
2437                                 qm_mr_next(low_p);
2438                                 qm_mr_cci_consume_to_current(low_p);
2439                                 qm_mr_pvb_update(low_p);
2440                                 msg = qm_mr_current(low_p);
2441                         }
2442                         cpu_relax();
2443                 }
2444                 mcc = qm_mc_start(low_p);
2445                 mcc->alterfq.fqid = cpu_to_be32(fqid);
2446                 qm_mc_commit(low_p, QM_MCC_VERB_ALTER_OOS);
2447                 while (!(mcr = qm_mc_result(low_p)))
2448                         cpu_relax();
2449                 DPAA_ASSERT((mcr->verb & QM_MCR_VERB_MASK) ==
2450                            QM_MCR_VERB_ALTER_OOS);
2451                 if (mcr->result != QM_MCR_RESULT_OK) {
2452                         pr_err(
2453                         "OOS after drain Failed on FQID 0x%x, result 0x%x\n",
2454                                fqid, mcr->result);
2455                         return -1;
2456                 }
2457                 return 0;
2458
2459         case QM_MCR_NP_STATE_RETIRED:
2460                 /* Send OOS Command */
2461                 mcc = qm_mc_start(low_p);
2462                 mcc->alterfq.fqid = cpu_to_be32(fqid);
2463                 qm_mc_commit(low_p, QM_MCC_VERB_ALTER_OOS);
2464                 while (!(mcr = qm_mc_result(low_p)))
2465                         cpu_relax();
2466                 DPAA_ASSERT((mcr->verb & QM_MCR_VERB_MASK) ==
2467                            QM_MCR_VERB_ALTER_OOS);
2468                 if (mcr->result) {
2469                         pr_err("OOS Failed on FQID 0x%x\n", fqid);
2470                         return -1;
2471                 }
2472                 return 0;
2473
2474         }
2475         return -1;
2476 }