drivers/bus/fslmc/qbman/qbman_portal.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  *
   3  * Copyright (C) 2014-2016 Freescale Semiconductor, Inc.
   4  * Copyright 2018-2020 NXP
   5  *
   6  */
   7
   8 #include "qbman_sys.h"
   9 #include "qbman_portal.h"
  10
  11 /* QBMan portal management command codes */
  12 #define QBMAN_MC_ACQUIRE       0x30
  13 #define QBMAN_WQCHAN_CONFIGURE 0x46
  14
  15 /* Reverse mapping of QBMAN_CENA_SWP_DQRR() */
  16 #define QBMAN_IDX_FROM_DQRR(p) (((unsigned long)p & 0x1ff) >> 6)
  17
  18 /* QBMan FQ management command codes */
  19 #define QBMAN_FQ_SCHEDULE       0x48
  20 #define QBMAN_FQ_FORCE          0x49
  21 #define QBMAN_FQ_XON            0x4d
  22 #define QBMAN_FQ_XOFF           0x4e
  23
  24 /*******************************/
  25 /* Pre-defined attribute codes */
  26 /*******************************/
  27
  28 #define QBMAN_RESPONSE_VERB_MASK   0x7f
  29
  30 /*************************/
  31 /* SDQCR attribute codes */
  32 /*************************/
  33 #define QB_SDQCR_FC_SHIFT   29
  34 #define QB_SDQCR_FC_MASK    0x1
  35 #define QB_SDQCR_DCT_SHIFT  24
  36 #define QB_SDQCR_DCT_MASK   0x3
  37 #define QB_SDQCR_TOK_SHIFT  16
  38 #define QB_SDQCR_TOK_MASK   0xff
  39 #define QB_SDQCR_SRC_SHIFT  0
  40 #define QB_SDQCR_SRC_MASK   0xffff
  41
  42 /* opaque token for static dequeues */
  43 #define QMAN_SDQCR_TOKEN    0xbb
  44
  45 enum qbman_sdqcr_dct {
  46         qbman_sdqcr_dct_null = 0,
  47         qbman_sdqcr_dct_prio_ics,
  48         qbman_sdqcr_dct_active_ics,
  49         qbman_sdqcr_dct_active
  50 };
  51
  52 enum qbman_sdqcr_fc {
  53         qbman_sdqcr_fc_one = 0,
  54         qbman_sdqcr_fc_up_to_3 = 1
  55 };
  56
  57 /* We need to keep track of which SWP triggered a pull command
  58  * so keep an array of portal IDs and use the token field to
  59  * be able to find the proper portal
  60  */
  61 #define MAX_QBMAN_PORTALS  64
  62 static struct qbman_swp *portal_idx_map[MAX_QBMAN_PORTALS];
  63
  64 uint32_t qman_version;
  65
  66 /* Internal Function declaration */
  67 static int
  68 qbman_swp_enqueue_array_mode_direct(struct qbman_swp *s,
  69                 const struct qbman_eq_desc *d,
  70                 const struct qbman_fd *fd);
  71 static int
  72 qbman_swp_enqueue_array_mode_mem_back(struct qbman_swp *s,
  73                 const struct qbman_eq_desc *d,
  74                 const struct qbman_fd *fd);
  75
  76 static int
  77 qbman_swp_enqueue_ring_mode_direct(struct qbman_swp *s,
  78                 const struct qbman_eq_desc *d,
  79                 const struct qbman_fd *fd);
  80 static int
  81 qbman_swp_enqueue_ring_mode_cinh_read_direct(struct qbman_swp *s,
  82                 const struct qbman_eq_desc *d,
  83                 const struct qbman_fd *fd);
  84 static int
  85 qbman_swp_enqueue_ring_mode_cinh_direct(struct qbman_swp *s,
  86                 const struct qbman_eq_desc *d,
  87                 const struct qbman_fd *fd);
  88 static int
  89 qbman_swp_enqueue_ring_mode_mem_back(struct qbman_swp *s,
  90                 const struct qbman_eq_desc *d,
  91                 const struct qbman_fd *fd);
  92
  93 static int
  94 qbman_swp_enqueue_multiple_direct(struct qbman_swp *s,
  95                 const struct qbman_eq_desc *d,
  96                 const struct qbman_fd *fd,
  97                 uint32_t *flags,
  98                 int num_frames);
  99 static int
 100 qbman_swp_enqueue_multiple_cinh_read_direct(struct qbman_swp *s,
 101                 const struct qbman_eq_desc *d,
 102                 const struct qbman_fd *fd,
 103                 uint32_t *flags,
 104                 int num_frames);
 105 static int
 106 qbman_swp_enqueue_multiple_cinh_direct(struct qbman_swp *s,
 107                 const struct qbman_eq_desc *d,
 108                 const struct qbman_fd *fd,
 109                 uint32_t *flags,
 110                 int num_frames);
 111 static int
 112 qbman_swp_enqueue_multiple_mem_back(struct qbman_swp *s,
 113                 const struct qbman_eq_desc *d,
 114                 const struct qbman_fd *fd,
 115                 uint32_t *flags,
 116                 int num_frames);
 117
 118 static int
 119 qbman_swp_enqueue_multiple_fd_direct(struct qbman_swp *s,
 120                 const struct qbman_eq_desc *d,
 121                 struct qbman_fd **fd,
 122                 uint32_t *flags,
 123                 int num_frames);
 124 static int
 125 qbman_swp_enqueue_multiple_fd_cinh_read_direct(struct qbman_swp *s,
 126                 const struct qbman_eq_desc *d,
 127                 struct qbman_fd **fd,
 128                 uint32_t *flags,
 129                 int num_frames);
 130 static int
 131 qbman_swp_enqueue_multiple_fd_cinh_direct(struct qbman_swp *s,
 132                 const struct qbman_eq_desc *d,
 133                 struct qbman_fd **fd,
 134                 uint32_t *flags,
 135                 int num_frames);
 136 static int
 137 qbman_swp_enqueue_multiple_fd_mem_back(struct qbman_swp *s,
 138                 const struct qbman_eq_desc *d,
 139                 struct qbman_fd **fd,
 140                 uint32_t *flags,
 141                 int num_frames);
 142
 143 static int
 144 qbman_swp_enqueue_multiple_desc_direct(struct qbman_swp *s,
 145                 const struct qbman_eq_desc *d,
 146                 const struct qbman_fd *fd,
 147                 int num_frames);
 148 static int
 149 qbman_swp_enqueue_multiple_desc_cinh_read_direct(struct qbman_swp *s,
 150                 const struct qbman_eq_desc *d,
 151                 const struct qbman_fd *fd,
 152                 int num_frames);
 153 static int
 154 qbman_swp_enqueue_multiple_desc_cinh_direct(struct qbman_swp *s,
 155                 const struct qbman_eq_desc *d,
 156                 const struct qbman_fd *fd,
 157                 int num_frames);
 158 static int
 159 qbman_swp_enqueue_multiple_desc_mem_back(struct qbman_swp *s,
 160                 const struct qbman_eq_desc *d,
 161                 const struct qbman_fd *fd,
 162                 int num_frames);
 163
 164 static int
 165 qbman_swp_pull_direct(struct qbman_swp *s, struct qbman_pull_desc *d);
 166 static int
 167 qbman_swp_pull_cinh_direct(struct qbman_swp *s, struct qbman_pull_desc *d);
 168 static int
 169 qbman_swp_pull_mem_back(struct qbman_swp *s, struct qbman_pull_desc *d);
 170
 171 const struct qbman_result *qbman_swp_dqrr_next_direct(struct qbman_swp *s);
 172 const struct qbman_result *qbman_swp_dqrr_next_cinh_direct(struct qbman_swp *s);
 173 const struct qbman_result *qbman_swp_dqrr_next_mem_back(struct qbman_swp *s);
 174
 175 static int
 176 qbman_swp_release_direct(struct qbman_swp *s,
 177                 const struct qbman_release_desc *d,
 178                 const uint64_t *buffers, unsigned int num_buffers);
 179 static int
 180 qbman_swp_release_cinh_direct(struct qbman_swp *s,
 181                 const struct qbman_release_desc *d,
 182                 const uint64_t *buffers, unsigned int num_buffers);
 183 static int
 184 qbman_swp_release_mem_back(struct qbman_swp *s,
 185                 const struct qbman_release_desc *d,
 186                 const uint64_t *buffers, unsigned int num_buffers);
 187
 188 /* Function pointers */
 189 static int (*qbman_swp_enqueue_array_mode_ptr)(struct qbman_swp *s,
 190                 const struct qbman_eq_desc *d,
 191                 const struct qbman_fd *fd)
 192         = qbman_swp_enqueue_array_mode_direct;
 193
 194 static int (*qbman_swp_enqueue_ring_mode_ptr)(struct qbman_swp *s,
 195                 const struct qbman_eq_desc *d,
 196                 const struct qbman_fd *fd)
 197         = qbman_swp_enqueue_ring_mode_direct;
 198
 199 static int (*qbman_swp_enqueue_multiple_ptr)(struct qbman_swp *s,
 200                 const struct qbman_eq_desc *d,
 201                 const struct qbman_fd *fd,
 202                 uint32_t *flags,
 203                 int num_frames)
 204         = qbman_swp_enqueue_multiple_direct;
 205
 206 static int (*qbman_swp_enqueue_multiple_fd_ptr)(struct qbman_swp *s,
 207                 const struct qbman_eq_desc *d,
 208                 struct qbman_fd **fd,
 209                 uint32_t *flags,
 210                 int num_frames)
 211         = qbman_swp_enqueue_multiple_fd_direct;
 212
 213 static int (*qbman_swp_enqueue_multiple_desc_ptr)(struct qbman_swp *s,
 214                 const struct qbman_eq_desc *d,
 215                 const struct qbman_fd *fd,
 216                 int num_frames)
 217         = qbman_swp_enqueue_multiple_desc_direct;
 218
 219 static int (*qbman_swp_pull_ptr)(struct qbman_swp *s,
 220                 struct qbman_pull_desc *d)
 221         = qbman_swp_pull_direct;
 222
 223 const struct qbman_result *(*qbman_swp_dqrr_next_ptr)(struct qbman_swp *s)
 224                 = qbman_swp_dqrr_next_direct;
 225
 226 static int (*qbman_swp_release_ptr)(struct qbman_swp *s,
 227                         const struct qbman_release_desc *d,
 228                         const uint64_t *buffers, unsigned int num_buffers)
 229                         = qbman_swp_release_direct;
 230
 231 /*********************************/
 232 /* Portal constructor/destructor */
 233 /*********************************/
 234
 235 /* Software portals should always be in the power-on state when we initialise,
 236  * due to the CCSR-based portal reset functionality that MC has.
 237  *
 238  * Erk! Turns out that QMan versions prior to 4.1 do not correctly reset DQRR
 239  * valid-bits, so we need to support a workaround where we don't trust
 240  * valid-bits when detecting new entries until any stale ring entries have been
 241  * overwritten at least once. The idea is that we read PI for the first few
 242  * entries, then switch to valid-bit after that. The trick is to clear the
 243  * bug-work-around boolean once the PI wraps around the ring for the first time.
 244  *
 245  * Note: this still carries a slight additional cost once the decrementer hits
 246  * zero.
 247  */
 248 struct qbman_swp *qbman_swp_init(const struct qbman_swp_desc *d)
 249 {
 250         int ret;
 251         uint32_t eqcr_pi;
 252         uint32_t mask_size;
 253         struct qbman_swp *p = malloc(sizeof(*p));
 254
 255         if (!p)
 256                 return NULL;
 257
 258         memset(p, 0, sizeof(struct qbman_swp));
 259
 260         p->desc = *d;
 261 #ifdef QBMAN_CHECKING
 262         p->mc.check = swp_mc_can_start;
 263 #endif
 264         p->mc.valid_bit = QB_VALID_BIT;
 265         p->sdq |= qbman_sdqcr_dct_prio_ics << QB_SDQCR_DCT_SHIFT;
 266         p->sdq |= qbman_sdqcr_fc_up_to_3 << QB_SDQCR_FC_SHIFT;
 267         p->sdq |= QMAN_SDQCR_TOKEN << QB_SDQCR_TOK_SHIFT;
 268         if ((d->qman_version & QMAN_REV_MASK) >= QMAN_REV_5000
 269                         && (d->cena_access_mode == qman_cena_fastest_access))
 270                 p->mr.valid_bit = QB_VALID_BIT;
 271
 272         atomic_set(&p->vdq.busy, 1);
 273         p->vdq.valid_bit = QB_VALID_BIT;
 274         p->dqrr.valid_bit = QB_VALID_BIT;
 275         qman_version = p->desc.qman_version;
 276         if ((qman_version & QMAN_REV_MASK) < QMAN_REV_4100) {
 277                 p->dqrr.dqrr_size = 4;
 278                 p->dqrr.reset_bug = 1;
 279         } else {
 280                 p->dqrr.dqrr_size = 8;
 281                 p->dqrr.reset_bug = 0;
 282         }
 283
 284         ret = qbman_swp_sys_init(&p->sys, d, p->dqrr.dqrr_size);
 285         if (ret) {
 286                 free(p);
 287                 pr_err("qbman_swp_sys_init() failed %d\n", ret);
 288                 return NULL;
 289         }
 290
 291         /* Verify that the DQRRPI is 0 - if it is not the portal isn't
 292          * in default state which is an error
 293          */
 294         if (qbman_cinh_read(&p->sys, QBMAN_CINH_SWP_DQPI) & 0xF) {
 295                 pr_err("qbman DQRR PI is not zero, portal is not clean\n");
 296                 free(p);
 297                 return NULL;
 298         }
 299
 300         /* SDQCR needs to be initialized to 0 when no channels are
 301          * being dequeued from or else the QMan HW will indicate an
 302          * error.  The values that were calculated above will be
 303          * applied when dequeues from a specific channel are enabled.
 304          */
 305         qbman_cinh_write(&p->sys, QBMAN_CINH_SWP_SDQCR, 0);
 306
 307         p->eqcr.pi_ring_size = 8;
 308         if ((qman_version & QMAN_REV_MASK) >= QMAN_REV_5000
 309                         && (d->cena_access_mode == qman_cena_fastest_access)) {
 310                 p->eqcr.pi_ring_size = 32;
 311                 qbman_swp_enqueue_array_mode_ptr =
 312                         qbman_swp_enqueue_array_mode_mem_back;
 313                 qbman_swp_enqueue_ring_mode_ptr =
 314                         qbman_swp_enqueue_ring_mode_mem_back;
 315                 qbman_swp_enqueue_multiple_ptr =
 316                         qbman_swp_enqueue_multiple_mem_back;
 317                 qbman_swp_enqueue_multiple_fd_ptr =
 318                         qbman_swp_enqueue_multiple_fd_mem_back;
 319                 qbman_swp_enqueue_multiple_desc_ptr =
 320                         qbman_swp_enqueue_multiple_desc_mem_back;
 321                 qbman_swp_pull_ptr = qbman_swp_pull_mem_back;
 322                 qbman_swp_dqrr_next_ptr = qbman_swp_dqrr_next_mem_back;
 323                 qbman_swp_release_ptr = qbman_swp_release_mem_back;
 324         }
 325
 326         if (dpaa2_svr_family == SVR_LS1080A) {
 327                 qbman_swp_enqueue_ring_mode_ptr =
 328                         qbman_swp_enqueue_ring_mode_cinh_read_direct;
 329                 qbman_swp_enqueue_multiple_ptr =
 330                         qbman_swp_enqueue_multiple_cinh_read_direct;
 331                 qbman_swp_enqueue_multiple_fd_ptr =
 332                         qbman_swp_enqueue_multiple_fd_cinh_read_direct;
 333                 qbman_swp_enqueue_multiple_desc_ptr =
 334                         qbman_swp_enqueue_multiple_desc_cinh_read_direct;
 335         }
 336
 337         for (mask_size = p->eqcr.pi_ring_size; mask_size > 0; mask_size >>= 1)
 338                 p->eqcr.pi_ci_mask = (p->eqcr.pi_ci_mask<<1) + 1;
 339         eqcr_pi = qbman_cinh_read(&p->sys, QBMAN_CINH_SWP_EQCR_PI);
 340         p->eqcr.pi = eqcr_pi & p->eqcr.pi_ci_mask;
 341         p->eqcr.pi_vb = eqcr_pi & QB_VALID_BIT;
 342         p->eqcr.ci = qbman_cinh_read(&p->sys, QBMAN_CINH_SWP_EQCR_CI)
 343                         & p->eqcr.pi_ci_mask;
 344         p->eqcr.available = p->eqcr.pi_ring_size;
 345
 346         portal_idx_map[p->desc.idx] = p;
 347         return p;
 348 }
 349
 350 int qbman_swp_update(struct qbman_swp *p, int stash_off)
 351 {
 352         const struct qbman_swp_desc *d = &p->desc;
 353         struct qbman_swp_sys *s = &p->sys;
 354         int ret;
 355
 356         /* Nothing needs to be done for QBMAN rev > 5000 with fast access */
 357         if ((qman_version & QMAN_REV_MASK) >= QMAN_REV_5000
 358                         && (d->cena_access_mode == qman_cena_fastest_access))
 359                 return 0;
 360
 361         ret = qbman_swp_sys_update(s, d, p->dqrr.dqrr_size, stash_off);
 362         if (ret) {
 363                 pr_err("qbman_swp_sys_init() failed %d\n", ret);
 364                 return ret;
 365         }
 366
 367         p->stash_off = stash_off;
 368
 369         return 0;
 370 }
 371
 372 void qbman_swp_finish(struct qbman_swp *p)
 373 {
 374 #ifdef QBMAN_CHECKING
 375         QBMAN_BUG_ON(p->mc.check != swp_mc_can_start);
 376 #endif
 377         qbman_swp_sys_finish(&p->sys);
 378         portal_idx_map[p->desc.idx] = NULL;
 379         free(p);
 380 }
 381
 382 const struct qbman_swp_desc *qbman_swp_get_desc(struct qbman_swp *p)
 383 {
 384         return &p->desc;
 385 }
 386
 387 /**************/
 388 /* Interrupts */
 389 /**************/
 390
 391 uint32_t qbman_swp_interrupt_get_vanish(struct qbman_swp *p)
 392 {
 393         return qbman_cinh_read(&p->sys, QBMAN_CINH_SWP_ISDR);
 394 }
 395
 396 void qbman_swp_interrupt_set_vanish(struct qbman_swp *p, uint32_t mask)
 397 {
 398         qbman_cinh_write(&p->sys, QBMAN_CINH_SWP_ISDR, mask);
 399 }
 400
 401 uint32_t qbman_swp_interrupt_read_status(struct qbman_swp *p)
 402 {
 403         return qbman_cinh_read(&p->sys, QBMAN_CINH_SWP_ISR);
 404 }
 405
 406 void qbman_swp_interrupt_clear_status(struct qbman_swp *p, uint32_t mask)
 407 {
 408         qbman_cinh_write(&p->sys, QBMAN_CINH_SWP_ISR, mask);
 409 }
 410
 411 uint32_t qbman_swp_dqrr_thrshld_read_status(struct qbman_swp *p)
 412 {
 413         return qbman_cinh_read(&p->sys, QBMAN_CINH_SWP_DQRR_ITR);
 414 }
 415
 416 void qbman_swp_dqrr_thrshld_write(struct qbman_swp *p, uint32_t mask)
 417 {
 418         qbman_cinh_write(&p->sys, QBMAN_CINH_SWP_DQRR_ITR, mask);
 419 }
 420
 421 uint32_t qbman_swp_intr_timeout_read_status(struct qbman_swp *p)
 422 {
 423         return qbman_cinh_read(&p->sys, QBMAN_CINH_SWP_ITPR);
 424 }
 425
 426 void qbman_swp_intr_timeout_write(struct qbman_swp *p, uint32_t mask)
 427 {
 428         qbman_cinh_write(&p->sys, QBMAN_CINH_SWP_ITPR, mask);
 429 }
 430
 431 uint32_t qbman_swp_interrupt_get_trigger(struct qbman_swp *p)
 432 {
 433         return qbman_cinh_read(&p->sys, QBMAN_CINH_SWP_IER);
 434 }
 435
 436 void qbman_swp_interrupt_set_trigger(struct qbman_swp *p, uint32_t mask)
 437 {
 438         qbman_cinh_write(&p->sys, QBMAN_CINH_SWP_IER, mask);
 439 }
 440
 441 int qbman_swp_interrupt_get_inhibit(struct qbman_swp *p)
 442 {
 443         return qbman_cinh_read(&p->sys, QBMAN_CINH_SWP_IIR);
 444 }
 445
 446 void qbman_swp_interrupt_set_inhibit(struct qbman_swp *p, int inhibit)
 447 {
 448         qbman_cinh_write(&p->sys, QBMAN_CINH_SWP_IIR,
 449                          inhibit ? 0xffffffff : 0);
 450 }
 451
 452 /***********************/
 453 /* Management commands */
 454 /***********************/
 455
 456 /*
 457  * Internal code common to all types of management commands.
 458  */
 459
 460 void *qbman_swp_mc_start(struct qbman_swp *p)
 461 {
 462         void *ret;
 463 #ifdef QBMAN_CHECKING
 464         QBMAN_BUG_ON(p->mc.check != swp_mc_can_start);
 465 #endif
 466         if ((p->desc.qman_version & QMAN_REV_MASK) >= QMAN_REV_5000
 467                     && (p->desc.cena_access_mode == qman_cena_fastest_access))
 468                 ret = qbman_cena_write_start(&p->sys, QBMAN_CENA_SWP_CR_MEM);
 469         else
 470                 ret = qbman_cena_write_start(&p->sys, QBMAN_CENA_SWP_CR);
 471 #ifdef QBMAN_CHECKING
 472         if (!ret)
 473                 p->mc.check = swp_mc_can_submit;
 474 #endif
 475         return ret;
 476 }
 477
 478 void qbman_swp_mc_submit(struct qbman_swp *p, void *cmd, uint8_t cmd_verb)
 479 {
 480         uint8_t *v = cmd;
 481 #ifdef QBMAN_CHECKING
 482         QBMAN_BUG_ON(!(p->mc.check != swp_mc_can_submit));
 483 #endif
 484         /* TBD: "|=" is going to hurt performance. Need to move as many fields
 485          * out of word zero, and for those that remain, the "OR" needs to occur
 486          * at the caller side. This debug check helps to catch cases where the
 487          * caller wants to OR but has forgotten to do so.
 488          */
 489         QBMAN_BUG_ON((*v & cmd_verb) != *v);
 490         if ((p->desc.qman_version & QMAN_REV_MASK) >= QMAN_REV_5000
 491                     && (p->desc.cena_access_mode == qman_cena_fastest_access)) {
 492                 *v = cmd_verb | p->mr.valid_bit;
 493                 qbman_cena_write_complete(&p->sys, QBMAN_CENA_SWP_CR_MEM, cmd);
 494                 dma_wmb();
 495                 qbman_cinh_write(&p->sys, QBMAN_CINH_SWP_CR_RT, QMAN_RT_MODE);
 496         } else {
 497                 dma_wmb();
 498                 *v = cmd_verb | p->mc.valid_bit;
 499                 qbman_cena_write_complete(&p->sys, QBMAN_CENA_SWP_CR, cmd);
 500                 clean(cmd);
 501         }
 502 #ifdef QBMAN_CHECKING
 503         p->mc.check = swp_mc_can_poll;
 504 #endif
 505 }
 506
 507 void qbman_swp_mc_submit_cinh(struct qbman_swp *p, void *cmd, uint8_t cmd_verb)
 508 {
 509         uint8_t *v = cmd;
 510 #ifdef QBMAN_CHECKING
 511         QBMAN_BUG_ON(!(p->mc.check != swp_mc_can_submit));
 512 #endif
 513         /* TBD: "|=" is going to hurt performance. Need to move as many fields
 514          * out of word zero, and for those that remain, the "OR" needs to occur
 515          * at the caller side. This debug check helps to catch cases where the
 516          * caller wants to OR but has forgotten to do so.
 517          */
 518         QBMAN_BUG_ON((*v & cmd_verb) != *v);
 519         dma_wmb();
 520         *v = cmd_verb | p->mc.valid_bit;
 521         qbman_cinh_write_complete(&p->sys, QBMAN_CENA_SWP_CR, cmd);
 522         clean(cmd);
 523 #ifdef QBMAN_CHECKING
 524         p->mc.check = swp_mc_can_poll;
 525 #endif
 526 }
 527
 528 void *qbman_swp_mc_result(struct qbman_swp *p)
 529 {
 530         uint32_t *ret, verb;
 531 #ifdef QBMAN_CHECKING
 532         QBMAN_BUG_ON(p->mc.check != swp_mc_can_poll);
 533 #endif
 534         if ((p->desc.qman_version & QMAN_REV_MASK) >= QMAN_REV_5000
 535                 && (p->desc.cena_access_mode == qman_cena_fastest_access)) {
 536                 ret = qbman_cena_read(&p->sys, QBMAN_CENA_SWP_RR_MEM);
 537                 /* Command completed if the valid bit is toggled */
 538                 if (p->mr.valid_bit != (ret[0] & QB_VALID_BIT))
 539                         return NULL;
 540                 /* Remove the valid-bit -
 541                  * command completed iff the rest is non-zero
 542                  */
 543                 verb = ret[0] & ~QB_VALID_BIT;
 544                 if (!verb)
 545                         return NULL;
 546                 p->mr.valid_bit ^= QB_VALID_BIT;
 547         } else {
 548                 qbman_cena_invalidate_prefetch(&p->sys,
 549                         QBMAN_CENA_SWP_RR(p->mc.valid_bit));
 550                 ret = qbman_cena_read(&p->sys,
 551                                       QBMAN_CENA_SWP_RR(p->mc.valid_bit));
 552                 /* Remove the valid-bit -
 553                  * command completed iff the rest is non-zero
 554                  */
 555                 verb = ret[0] & ~QB_VALID_BIT;
 556                 if (!verb)
 557                         return NULL;
 558                 p->mc.valid_bit ^= QB_VALID_BIT;
 559         }
 560 #ifdef QBMAN_CHECKING
 561         p->mc.check = swp_mc_can_start;
 562 #endif
 563         return ret;
 564 }
 565
 566 void *qbman_swp_mc_result_cinh(struct qbman_swp *p)
 567 {
 568         uint32_t *ret, verb;
 569 #ifdef QBMAN_CHECKING
 570         QBMAN_BUG_ON(p->mc.check != swp_mc_can_poll);
 571 #endif
 572         ret = qbman_cinh_read_shadow(&p->sys,
 573                               QBMAN_CENA_SWP_RR(p->mc.valid_bit));
 574         /* Remove the valid-bit -
 575          * command completed iff the rest is non-zero
 576          */
 577         verb = ret[0] & ~QB_VALID_BIT;
 578         if (!verb)
 579                 return NULL;
 580         p->mc.valid_bit ^= QB_VALID_BIT;
 581 #ifdef QBMAN_CHECKING
 582         p->mc.check = swp_mc_can_start;
 583 #endif
 584         return ret;
 585 }
 586
 587 /***********/
 588 /* Enqueue */
 589 /***********/
 590
 591 #define QB_ENQUEUE_CMD_OPTIONS_SHIFT    0
 592 enum qb_enqueue_commands {
 593         enqueue_empty = 0,
 594         enqueue_response_always = 1,
 595         enqueue_rejects_to_fq = 2
 596 };
 597
 598 #define QB_ENQUEUE_CMD_EC_OPTION_MASK        0x3
 599 #define QB_ENQUEUE_CMD_ORP_ENABLE_SHIFT      2
 600 #define QB_ENQUEUE_CMD_IRQ_ON_DISPATCH_SHIFT 3
 601 #define QB_ENQUEUE_CMD_TARGET_TYPE_SHIFT     4
 602 #define QB_ENQUEUE_CMD_DCA_PK_SHIFT          6
 603 #define QB_ENQUEUE_CMD_DCA_EN_SHIFT          7
 604 #define QB_ENQUEUE_CMD_NLIS_SHIFT            14
 605 #define QB_ENQUEUE_CMD_IS_NESN_SHIFT         15
 606
 607 void qbman_eq_desc_clear(struct qbman_eq_desc *d)
 608 {
 609         memset(d, 0, sizeof(*d));
 610 }
 611
 612 void qbman_eq_desc_set_no_orp(struct qbman_eq_desc *d, int respond_success)
 613 {
 614         d->eq.verb &= ~(1 << QB_ENQUEUE_CMD_ORP_ENABLE_SHIFT);
 615         if (respond_success)
 616                 d->eq.verb |= enqueue_response_always;
 617         else
 618                 d->eq.verb |= enqueue_rejects_to_fq;
 619 }
 620
 621 void qbman_eq_desc_set_orp(struct qbman_eq_desc *d, int respond_success,
 622                            uint16_t opr_id, uint16_t seqnum, int incomplete)
 623 {
 624         d->eq.verb |= 1 << QB_ENQUEUE_CMD_ORP_ENABLE_SHIFT;
 625         if (respond_success)
 626                 d->eq.verb |= enqueue_response_always;
 627         else
 628                 d->eq.verb |= enqueue_rejects_to_fq;
 629
 630         d->eq.orpid = opr_id;
 631         d->eq.seqnum = seqnum;
 632         if (incomplete)
 633                 d->eq.seqnum |= 1 << QB_ENQUEUE_CMD_NLIS_SHIFT;
 634         else
 635                 d->eq.seqnum &= ~(1 << QB_ENQUEUE_CMD_NLIS_SHIFT);
 636 }
 637
 638 void qbman_eq_desc_set_orp_hole(struct qbman_eq_desc *d, uint16_t opr_id,
 639                                 uint16_t seqnum)
 640 {
 641         d->eq.verb |= 1 << QB_ENQUEUE_CMD_ORP_ENABLE_SHIFT;
 642         d->eq.verb &= ~QB_ENQUEUE_CMD_EC_OPTION_MASK;
 643         d->eq.orpid = opr_id;
 644         d->eq.seqnum = seqnum;
 645         d->eq.seqnum &= ~(1 << QB_ENQUEUE_CMD_NLIS_SHIFT);
 646         d->eq.seqnum &= ~(1 << QB_ENQUEUE_CMD_IS_NESN_SHIFT);
 647 }
 648
 649 void qbman_eq_desc_set_orp_nesn(struct qbman_eq_desc *d, uint16_t opr_id,
 650                                 uint16_t seqnum)
 651 {
 652         d->eq.verb |= 1 << QB_ENQUEUE_CMD_ORP_ENABLE_SHIFT;
 653         d->eq.verb &= ~QB_ENQUEUE_CMD_EC_OPTION_MASK;
 654         d->eq.orpid = opr_id;
 655         d->eq.seqnum = seqnum;
 656         d->eq.seqnum &= ~(1 << QB_ENQUEUE_CMD_NLIS_SHIFT);
 657         d->eq.seqnum |= 1 << QB_ENQUEUE_CMD_IS_NESN_SHIFT;
 658 }
 659
 660 void qbman_eq_desc_set_response(struct qbman_eq_desc *d,
 661                                 dma_addr_t storage_phys,
 662                                 int stash)
 663 {
 664         d->eq.rsp_addr = storage_phys;
 665         d->eq.wae = stash;
 666 }
 667
 668 void qbman_eq_desc_set_token(struct qbman_eq_desc *d, uint8_t token)
 669 {
 670         d->eq.rspid = token;
 671 }
 672
 673 void qbman_eq_desc_set_fq(struct qbman_eq_desc *d, uint32_t fqid)
 674 {
 675         d->eq.verb &= ~(1 << QB_ENQUEUE_CMD_TARGET_TYPE_SHIFT);
 676         d->eq.tgtid = fqid;
 677 }
 678
 679 void qbman_eq_desc_set_qd(struct qbman_eq_desc *d, uint32_t qdid,
 680                           uint16_t qd_bin, uint8_t qd_prio)
 681 {
 682         d->eq.verb |= 1 << QB_ENQUEUE_CMD_TARGET_TYPE_SHIFT;
 683         d->eq.tgtid = qdid;
 684         d->eq.qdbin = qd_bin;
 685         d->eq.qpri = qd_prio;
 686 }
 687
 688 void qbman_eq_desc_set_eqdi(struct qbman_eq_desc *d, int enable)
 689 {
 690         if (enable)
 691                 d->eq.verb |= 1 << QB_ENQUEUE_CMD_IRQ_ON_DISPATCH_SHIFT;
 692         else
 693                 d->eq.verb &= ~(1 << QB_ENQUEUE_CMD_IRQ_ON_DISPATCH_SHIFT);
 694 }
 695
 696 void qbman_eq_desc_set_dca(struct qbman_eq_desc *d, int enable,
 697                            uint8_t dqrr_idx, int park)
 698 {
 699         if (enable) {
 700                 d->eq.dca = dqrr_idx;
 701                 if (park)
 702                         d->eq.dca |= 1 << QB_ENQUEUE_CMD_DCA_PK_SHIFT;
 703                 else
 704                         d->eq.dca &= ~(1 << QB_ENQUEUE_CMD_DCA_PK_SHIFT);
 705                 d->eq.dca |= 1 << QB_ENQUEUE_CMD_DCA_EN_SHIFT;
 706         } else {
 707                 d->eq.dca &= ~(1 << QB_ENQUEUE_CMD_DCA_EN_SHIFT);
 708         }
 709 }
 710
 711 #define EQAR_IDX(eqar)     ((eqar) & 0x1f)
 712 #define EQAR_VB(eqar)      ((eqar) & 0x80)
 713 #define EQAR_SUCCESS(eqar) ((eqar) & 0x100)
 714
 715 static inline void qbman_write_eqcr_am_rt_register(struct qbman_swp *p,
 716                                                    uint8_t idx)
 717 {
 718         if (idx < 16)
 719                 qbman_cinh_write(&p->sys, QBMAN_CINH_SWP_EQCR_AM_RT + idx * 4,
 720                                      QMAN_RT_MODE);
 721         else
 722                 qbman_cinh_write(&p->sys, QBMAN_CINH_SWP_EQCR_AM_RT2 +
 723                                      (idx - 16) * 4,
 724                                      QMAN_RT_MODE);
 725 }
 726
 727 static void memcpy_byte_by_byte(void *to, const void *from, size_t n)
 728 {
 729         const uint8_t *src = from;
 730         volatile uint8_t *dest = to;
 731         size_t i;
 732
 733         for (i = 0; i < n; i++)
 734                 dest[i] = src[i];
 735 }
 736
 737
 738 static int qbman_swp_enqueue_array_mode_direct(struct qbman_swp *s,
 739                                                const struct qbman_eq_desc *d,
 740                                                const struct qbman_fd *fd)
 741 {
 742         uint32_t *p;
 743         const uint32_t *cl = qb_cl(d);
 744         uint32_t eqar = qbman_cinh_read(&s->sys, QBMAN_CINH_SWP_EQAR);
 745
 746         pr_debug("EQAR=%08x\n", eqar);
 747         if (!EQAR_SUCCESS(eqar))
 748                 return -EBUSY;
 749         p = qbman_cena_write_start_wo_shadow(&s->sys,
 750                         QBMAN_CENA_SWP_EQCR(EQAR_IDX(eqar)));
 751         memcpy(&p[1], &cl[1], 28);
 752         memcpy(&p[8], fd, sizeof(*fd));
 753
 754         /* Set the verb byte, have to substitute in the valid-bit */
 755         dma_wmb();
 756         p[0] = cl[0] | EQAR_VB(eqar);
 757         qbman_cena_write_complete_wo_shadow(&s->sys,
 758                                 QBMAN_CENA_SWP_EQCR(EQAR_IDX(eqar)));
 759         return 0;
 760 }
 761 static int qbman_swp_enqueue_array_mode_mem_back(struct qbman_swp *s,
 762                                                  const struct qbman_eq_desc *d,
 763                                                  const struct qbman_fd *fd)
 764 {
 765         uint32_t *p;
 766         const uint32_t *cl = qb_cl(d);
 767         uint32_t eqar = qbman_cinh_read(&s->sys, QBMAN_CINH_SWP_EQAR);
 768
 769         pr_debug("EQAR=%08x\n", eqar);
 770         if (!EQAR_SUCCESS(eqar))
 771                 return -EBUSY;
 772         p = qbman_cena_write_start_wo_shadow(&s->sys,
 773                         QBMAN_CENA_SWP_EQCR(EQAR_IDX(eqar)));
 774         memcpy(&p[1], &cl[1], 28);
 775         memcpy(&p[8], fd, sizeof(*fd));
 776
 777         /* Set the verb byte, have to substitute in the valid-bit */
 778         p[0] = cl[0] | EQAR_VB(eqar);
 779         dma_wmb();
 780         qbman_write_eqcr_am_rt_register(s, EQAR_IDX(eqar));
 781         return 0;
 782 }
 783
 784 static inline int qbman_swp_enqueue_array_mode(struct qbman_swp *s,
 785                                                const struct qbman_eq_desc *d,
 786                                                const struct qbman_fd *fd)
 787 {
 788         return qbman_swp_enqueue_array_mode_ptr(s, d, fd);
 789 }
 790
 791 static int qbman_swp_enqueue_ring_mode_direct(struct qbman_swp *s,
 792                                               const struct qbman_eq_desc *d,
 793                                               const struct qbman_fd *fd)
 794 {
 795         uint32_t *p;
 796         const uint32_t *cl = qb_cl(d);
 797         uint32_t eqcr_ci, full_mask, half_mask;
 798
 799         half_mask = (s->eqcr.pi_ci_mask>>1);
 800         full_mask = s->eqcr.pi_ci_mask;
 801         if (!s->eqcr.available) {
 802                 eqcr_ci = s->eqcr.ci;
 803                 s->eqcr.ci = qbman_cena_read_reg(&s->sys,
 804                                 QBMAN_CENA_SWP_EQCR_CI) & full_mask;
 805                 s->eqcr.available = qm_cyc_diff(s->eqcr.pi_ring_size,
 806                                 eqcr_ci, s->eqcr.ci);
 807                 if (!s->eqcr.available)
 808                         return -EBUSY;
 809         }
 810
 811         p = qbman_cena_write_start_wo_shadow(&s->sys,
 812                         QBMAN_CENA_SWP_EQCR(s->eqcr.pi & half_mask));
 813         memcpy(&p[1], &cl[1], 28);
 814         memcpy(&p[8], fd, sizeof(*fd));
 815         lwsync();
 816
 817         /* Set the verb byte, have to substitute in the valid-bit */
 818         p[0] = cl[0] | s->eqcr.pi_vb;
 819         qbman_cena_write_complete_wo_shadow(&s->sys,
 820                         QBMAN_CENA_SWP_EQCR(s->eqcr.pi & half_mask));
 821         s->eqcr.pi++;
 822         s->eqcr.pi &= full_mask;
 823         s->eqcr.available--;
 824         if (!(s->eqcr.pi & half_mask))
 825                 s->eqcr.pi_vb ^= QB_VALID_BIT;
 826
 827         return 0;
 828 }
 829
 830 static int qbman_swp_enqueue_ring_mode_cinh_read_direct(
 831                 struct qbman_swp *s,
 832                 const struct qbman_eq_desc *d,
 833                 const struct qbman_fd *fd)
 834 {
 835         uint32_t *p;
 836         const uint32_t *cl = qb_cl(d);
 837         uint32_t eqcr_ci, full_mask, half_mask;
 838
 839         half_mask = (s->eqcr.pi_ci_mask>>1);
 840         full_mask = s->eqcr.pi_ci_mask;
 841         if (!s->eqcr.available) {
 842                 eqcr_ci = s->eqcr.ci;
 843                 s->eqcr.ci = qbman_cinh_read(&s->sys,
 844                                 QBMAN_CINH_SWP_EQCR_CI) & full_mask;
 845                 s->eqcr.available = qm_cyc_diff(s->eqcr.pi_ring_size,
 846                                 eqcr_ci, s->eqcr.ci);
 847                 if (!s->eqcr.available)
 848                         return -EBUSY;
 849         }
 850
 851         p = qbman_cinh_write_start_wo_shadow(&s->sys,
 852                         QBMAN_CENA_SWP_EQCR(s->eqcr.pi & half_mask));
 853         memcpy(&p[1], &cl[1], 28);
 854         memcpy(&p[8], fd, sizeof(*fd));
 855         lwsync();
 856
 857         /* Set the verb byte, have to substitute in the valid-bit */
 858         p[0] = cl[0] | s->eqcr.pi_vb;
 859         s->eqcr.pi++;
 860         s->eqcr.pi &= full_mask;
 861         s->eqcr.available--;
 862         if (!(s->eqcr.pi & half_mask))
 863                 s->eqcr.pi_vb ^= QB_VALID_BIT;
 864
 865         return 0;
 866 }
 867
 868 static int qbman_swp_enqueue_ring_mode_cinh_direct(
 869                 struct qbman_swp *s,
 870                 const struct qbman_eq_desc *d,
 871                 const struct qbman_fd *fd)
 872 {
 873         uint32_t *p;
 874         const uint32_t *cl = qb_cl(d);
 875         uint32_t eqcr_ci, full_mask, half_mask;
 876
 877         half_mask = (s->eqcr.pi_ci_mask>>1);
 878         full_mask = s->eqcr.pi_ci_mask;
 879         if (!s->eqcr.available) {
 880                 eqcr_ci = s->eqcr.ci;
 881                 s->eqcr.ci = qbman_cinh_read(&s->sys,
 882                                 QBMAN_CINH_SWP_EQCR_CI) & full_mask;
 883                 s->eqcr.available = qm_cyc_diff(s->eqcr.pi_ring_size,
 884                                 eqcr_ci, s->eqcr.ci);
 885                 if (!s->eqcr.available)
 886                         return -EBUSY;
 887         }
 888
 889         p = qbman_cinh_write_start_wo_shadow(&s->sys,
 890                         QBMAN_CENA_SWP_EQCR(s->eqcr.pi & half_mask));
 891         memcpy_byte_by_byte(&p[1], &cl[1], 28);
 892         memcpy_byte_by_byte(&p[8], fd, sizeof(*fd));
 893         lwsync();
 894
 895         /* Set the verb byte, have to substitute in the valid-bit */
 896         p[0] = cl[0] | s->eqcr.pi_vb;
 897         s->eqcr.pi++;
 898         s->eqcr.pi &= full_mask;
 899         s->eqcr.available--;
 900         if (!(s->eqcr.pi & half_mask))
 901                 s->eqcr.pi_vb ^= QB_VALID_BIT;
 902
 903         return 0;
 904 }
 905
 906 static int qbman_swp_enqueue_ring_mode_mem_back(struct qbman_swp *s,
 907                                                 const struct qbman_eq_desc *d,
 908                                                 const struct qbman_fd *fd)
 909 {
 910         uint32_t *p;
 911         const uint32_t *cl = qb_cl(d);
 912         uint32_t eqcr_ci, full_mask, half_mask;
 913
 914         half_mask = (s->eqcr.pi_ci_mask>>1);
 915         full_mask = s->eqcr.pi_ci_mask;
 916         if (!s->eqcr.available) {
 917                 eqcr_ci = s->eqcr.ci;
 918                 s->eqcr.ci = qbman_cena_read_reg(&s->sys,
 919                                 QBMAN_CENA_SWP_EQCR_CI_MEMBACK) & full_mask;
 920                 s->eqcr.available = qm_cyc_diff(s->eqcr.pi_ring_size,
 921                                 eqcr_ci, s->eqcr.ci);
 922                 if (!s->eqcr.available)
 923                         return -EBUSY;
 924         }
 925
 926         p = qbman_cena_write_start_wo_shadow(&s->sys,
 927                         QBMAN_CENA_SWP_EQCR(s->eqcr.pi & half_mask));
 928         memcpy(&p[1], &cl[1], 28);
 929         memcpy(&p[8], fd, sizeof(*fd));
 930
 931         /* Set the verb byte, have to substitute in the valid-bit */
 932         p[0] = cl[0] | s->eqcr.pi_vb;
 933         s->eqcr.pi++;
 934         s->eqcr.pi &= full_mask;
 935         s->eqcr.available--;
 936         if (!(s->eqcr.pi & half_mask))
 937                 s->eqcr.pi_vb ^= QB_VALID_BIT;
 938         dma_wmb();
 939         qbman_cinh_write(&s->sys, QBMAN_CINH_SWP_EQCR_PI,
 940                                 (QB_RT_BIT)|(s->eqcr.pi)|s->eqcr.pi_vb);
 941         return 0;
 942 }
 943
 944 static int qbman_swp_enqueue_ring_mode(struct qbman_swp *s,
 945                                        const struct qbman_eq_desc *d,
 946                                        const struct qbman_fd *fd)
 947 {
 948         if (!s->stash_off)
 949                 return qbman_swp_enqueue_ring_mode_ptr(s, d, fd);
 950         else
 951                 return qbman_swp_enqueue_ring_mode_cinh_direct(s, d, fd);
 952 }
 953
 954 int qbman_swp_enqueue(struct qbman_swp *s, const struct qbman_eq_desc *d,
 955                       const struct qbman_fd *fd)
 956 {
 957         if (s->sys.eqcr_mode == qman_eqcr_vb_array)
 958                 return qbman_swp_enqueue_array_mode(s, d, fd);
 959         else    /* Use ring mode by default */
 960                 return qbman_swp_enqueue_ring_mode(s, d, fd);
 961 }
 962
 963 static int qbman_swp_enqueue_multiple_direct(struct qbman_swp *s,
 964                                              const struct qbman_eq_desc *d,
 965                                              const struct qbman_fd *fd,
 966                                              uint32_t *flags,
 967                                              int num_frames)
 968 {
 969         uint32_t *p = NULL;
 970         const uint32_t *cl = qb_cl(d);
 971         uint32_t eqcr_ci, eqcr_pi, half_mask, full_mask;
 972         int i, num_enqueued = 0;
 973         uint64_t addr_cena;
 974
 975         half_mask = (s->eqcr.pi_ci_mask>>1);
 976         full_mask = s->eqcr.pi_ci_mask;
 977         if (!s->eqcr.available) {
 978                 eqcr_ci = s->eqcr.ci;
 979                 s->eqcr.ci = qbman_cena_read_reg(&s->sys,
 980                                 QBMAN_CENA_SWP_EQCR_CI) & full_mask;
 981                 s->eqcr.available = qm_cyc_diff(s->eqcr.pi_ring_size,
 982                                 eqcr_ci, s->eqcr.ci);
 983                 if (!s->eqcr.available)
 984                         return 0;
 985         }
 986
 987         eqcr_pi = s->eqcr.pi;
 988         num_enqueued = (s->eqcr.available < num_frames) ?
 989                         s->eqcr.available : num_frames;
 990         s->eqcr.available -= num_enqueued;
 991         /* Fill in the EQCR ring */
 992         for (i = 0; i < num_enqueued; i++) {
 993                 p = qbman_cena_write_start_wo_shadow(&s->sys,
 994                                 QBMAN_CENA_SWP_EQCR(eqcr_pi & half_mask));
 995                 memcpy(&p[1], &cl[1], 28);
 996                 memcpy(&p[8], &fd[i], sizeof(*fd));
 997                 eqcr_pi++;
 998         }
 999
1000         lwsync();
1001
1002         /* Set the verb byte, have to substitute in the valid-bit */
1003         eqcr_pi = s->eqcr.pi;
1004         for (i = 0; i < num_enqueued; i++) {
1005                 p = qbman_cena_write_start_wo_shadow(&s->sys,
1006                                 QBMAN_CENA_SWP_EQCR(eqcr_pi & half_mask));
1007                 p[0] = cl[0] | s->eqcr.pi_vb;
1008                 if (flags && (flags[i] & QBMAN_ENQUEUE_FLAG_DCA)) {
1009                         struct qbman_eq_desc *d = (struct qbman_eq_desc *)p;
1010
1011                         d->eq.dca = (1 << QB_ENQUEUE_CMD_DCA_EN_SHIFT) |
1012                                 ((flags[i]) & QBMAN_EQCR_DCA_IDXMASK);
1013                 }
1014                 eqcr_pi++;
1015                 if (!(eqcr_pi & half_mask))
1016                         s->eqcr.pi_vb ^= QB_VALID_BIT;
1017         }
1018
1019         /* Flush all the cacheline without load/store in between */
1020         eqcr_pi = s->eqcr.pi;
1021         addr_cena = (size_t)s->sys.addr_cena;
1022         for (i = 0; i < num_enqueued; i++) {
1023                 dcbf((uintptr_t)(addr_cena +
1024                         QBMAN_CENA_SWP_EQCR(eqcr_pi & half_mask)));
1025                 eqcr_pi++;
1026         }
1027         s->eqcr.pi = eqcr_pi & full_mask;
1028
1029         return num_enqueued;
1030 }
1031
1032 static int qbman_swp_enqueue_multiple_cinh_read_direct(
1033                 struct qbman_swp *s,
1034                 const struct qbman_eq_desc *d,
1035                 const struct qbman_fd *fd,
1036                 uint32_t *flags,
1037                 int num_frames)
1038 {
1039         uint32_t *p = NULL;
1040         const uint32_t *cl = qb_cl(d);
1041         uint32_t eqcr_ci, eqcr_pi, half_mask, full_mask;
1042         int i, num_enqueued = 0;
1043         uint64_t addr_cena;
1044
1045         half_mask = (s->eqcr.pi_ci_mask>>1);
1046         full_mask = s->eqcr.pi_ci_mask;
1047         if (!s->eqcr.available) {
1048                 eqcr_ci = s->eqcr.ci;
1049                 s->eqcr.ci = qbman_cinh_read(&s->sys,
1050                                 QBMAN_CINH_SWP_EQCR_CI) & full_mask;
1051                 s->eqcr.available = qm_cyc_diff(s->eqcr.pi_ring_size,
1052                                 eqcr_ci, s->eqcr.ci);
1053                 if (!s->eqcr.available)
1054                         return 0;
1055         }
1056
1057         eqcr_pi = s->eqcr.pi;
1058         num_enqueued = (s->eqcr.available < num_frames) ?
1059                         s->eqcr.available : num_frames;
1060         s->eqcr.available -= num_enqueued;
1061         /* Fill in the EQCR ring */
1062         for (i = 0; i < num_enqueued; i++) {
1063                 p = qbman_cena_write_start_wo_shadow(&s->sys,
1064                                 QBMAN_CENA_SWP_EQCR(eqcr_pi & half_mask));
1065                 memcpy(&p[1], &cl[1], 28);
1066                 memcpy(&p[8], &fd[i], sizeof(*fd));
1067                 eqcr_pi++;
1068         }
1069
1070         lwsync();
1071
1072         /* Set the verb byte, have to substitute in the valid-bit */
1073         eqcr_pi = s->eqcr.pi;
1074         for (i = 0; i < num_enqueued; i++) {
1075                 p = qbman_cena_write_start_wo_shadow(&s->sys,
1076                                 QBMAN_CENA_SWP_EQCR(eqcr_pi & half_mask));
1077                 p[0] = cl[0] | s->eqcr.pi_vb;
1078                 if (flags && (flags[i] & QBMAN_ENQUEUE_FLAG_DCA)) {
1079                         struct qbman_eq_desc *d = (struct qbman_eq_desc *)p;
1080
1081                         d->eq.dca = (1 << QB_ENQUEUE_CMD_DCA_EN_SHIFT) |
1082                                 ((flags[i]) & QBMAN_EQCR_DCA_IDXMASK);
1083                 }
1084                 eqcr_pi++;
1085                 if (!(eqcr_pi & half_mask))
1086                         s->eqcr.pi_vb ^= QB_VALID_BIT;
1087         }
1088
1089         /* Flush all the cacheline without load/store in between */
1090         eqcr_pi = s->eqcr.pi;
1091         addr_cena = (size_t)s->sys.addr_cena;
1092         for (i = 0; i < num_enqueued; i++) {
1093                 dcbf(addr_cena +
1094                         QBMAN_CENA_SWP_EQCR(eqcr_pi & half_mask));
1095                 eqcr_pi++;
1096         }
1097         s->eqcr.pi = eqcr_pi & full_mask;
1098
1099         return num_enqueued;
1100 }
1101
1102 static int qbman_swp_enqueue_multiple_cinh_direct(
1103                 struct qbman_swp *s,
1104                 const struct qbman_eq_desc *d,
1105                 const struct qbman_fd *fd,
1106                 uint32_t *flags,
1107                 int num_frames)
1108 {
1109         uint32_t *p = NULL;
1110         const uint32_t *cl = qb_cl(d);
1111         uint32_t eqcr_ci, eqcr_pi, half_mask, full_mask;
1112         int i, num_enqueued = 0;
1113
1114         half_mask = (s->eqcr.pi_ci_mask>>1);
1115         full_mask = s->eqcr.pi_ci_mask;
1116         if (!s->eqcr.available) {
1117                 eqcr_ci = s->eqcr.ci;
1118                 s->eqcr.ci = qbman_cinh_read(&s->sys,
1119                                 QBMAN_CINH_SWP_EQCR_CI) & full_mask;
1120                 s->eqcr.available = qm_cyc_diff(s->eqcr.pi_ring_size,
1121                                 eqcr_ci, s->eqcr.ci);
1122                 if (!s->eqcr.available)
1123                         return 0;
1124         }
1125
1126         eqcr_pi = s->eqcr.pi;
1127         num_enqueued = (s->eqcr.available < num_frames) ?
1128                         s->eqcr.available : num_frames;
1129         s->eqcr.available -= num_enqueued;
1130         /* Fill in the EQCR ring */
1131         for (i = 0; i < num_enqueued; i++) {
1132                 p = qbman_cinh_write_start_wo_shadow(&s->sys,
1133                                 QBMAN_CENA_SWP_EQCR(eqcr_pi & half_mask));
1134                 memcpy_byte_by_byte(&p[1], &cl[1], 28);
1135                 memcpy_byte_by_byte(&p[8], &fd[i], sizeof(*fd));
1136                 eqcr_pi++;
1137         }
1138
1139         lwsync();
1140
1141         /* Set the verb byte, have to substitute in the valid-bit */
1142         eqcr_pi = s->eqcr.pi;
1143         for (i = 0; i < num_enqueued; i++) {
1144                 p = qbman_cinh_write_start_wo_shadow(&s->sys,
1145                                 QBMAN_CENA_SWP_EQCR(eqcr_pi & half_mask));
1146                 p[0] = cl[0] | s->eqcr.pi_vb;
1147                 if (flags && (flags[i] & QBMAN_ENQUEUE_FLAG_DCA)) {
1148                         struct qbman_eq_desc *d = (struct qbman_eq_desc *)p;
1149
1150                         d->eq.dca = (1 << QB_ENQUEUE_CMD_DCA_EN_SHIFT) |
1151                                 ((flags[i]) & QBMAN_EQCR_DCA_IDXMASK);
1152                 }
1153                 eqcr_pi++;
1154                 if (!(eqcr_pi & half_mask))
1155                         s->eqcr.pi_vb ^= QB_VALID_BIT;
1156         }
1157
1158         s->eqcr.pi = eqcr_pi & full_mask;
1159
1160         return num_enqueued;
1161 }
1162
1163 static int qbman_swp_enqueue_multiple_mem_back(struct qbman_swp *s,
1164                                                const struct qbman_eq_desc *d,
1165                                                const struct qbman_fd *fd,
1166                                                uint32_t *flags,
1167                                                int num_frames)
1168 {
1169         uint32_t *p = NULL;
1170         const uint32_t *cl = qb_cl(d);
1171         uint32_t eqcr_ci, eqcr_pi, half_mask, full_mask;
1172         int i, num_enqueued = 0;
1173
1174         half_mask = (s->eqcr.pi_ci_mask>>1);
1175         full_mask = s->eqcr.pi_ci_mask;
1176         if (!s->eqcr.available) {
1177                 eqcr_ci = s->eqcr.ci;
1178                 s->eqcr.ci = qbman_cena_read_reg(&s->sys,
1179                                 QBMAN_CENA_SWP_EQCR_CI_MEMBACK) & full_mask;
1180                 s->eqcr.available = qm_cyc_diff(s->eqcr.pi_ring_size,
1181                                         eqcr_ci, s->eqcr.ci);
1182                 if (!s->eqcr.available)
1183                         return 0;
1184         }
1185
1186         eqcr_pi = s->eqcr.pi;
1187         num_enqueued = (s->eqcr.available < num_frames) ?
1188                         s->eqcr.available : num_frames;
1189         s->eqcr.available -= num_enqueued;
1190         /* Fill in the EQCR ring */
1191         for (i = 0; i < num_enqueued; i++) {
1192                 p = qbman_cena_write_start_wo_shadow(&s->sys,
1193                                 QBMAN_CENA_SWP_EQCR(eqcr_pi & half_mask));
1194                 memcpy(&p[1], &cl[1], 28);
1195                 memcpy(&p[8], &fd[i], sizeof(*fd));
1196                 p[0] = cl[0] | s->eqcr.pi_vb;
1197
1198                 if (flags && (flags[i] & QBMAN_ENQUEUE_FLAG_DCA)) {
1199                         struct qbman_eq_desc *d = (struct qbman_eq_desc *)p;
1200
1201                         d->eq.dca = (1 << QB_ENQUEUE_CMD_DCA_EN_SHIFT) |
1202                                 ((flags[i]) & QBMAN_EQCR_DCA_IDXMASK);
1203                 }
1204                 eqcr_pi++;
1205
1206                 if (!(eqcr_pi & half_mask))
1207                         s->eqcr.pi_vb ^= QB_VALID_BIT;
1208         }
1209         s->eqcr.pi = eqcr_pi & full_mask;
1210
1211         dma_wmb();
1212         qbman_cinh_write(&s->sys, QBMAN_CINH_SWP_EQCR_PI,
1213                                 (QB_RT_BIT)|(s->eqcr.pi)|s->eqcr.pi_vb);
1214         return num_enqueued;
1215 }
1216
1217 int qbman_swp_enqueue_multiple(struct qbman_swp *s,
1218                                       const struct qbman_eq_desc *d,
1219                                       const struct qbman_fd *fd,
1220                                       uint32_t *flags,
1221                                       int num_frames)
1222 {
1223         if (!s->stash_off)
1224                 return qbman_swp_enqueue_multiple_ptr(s, d, fd, flags,
1225                                                 num_frames);
1226         else
1227                 return qbman_swp_enqueue_multiple_cinh_direct(s, d, fd, flags,
1228                                                 num_frames);
1229 }
1230
1231 static int qbman_swp_enqueue_multiple_fd_direct(struct qbman_swp *s,
1232                                                 const struct qbman_eq_desc *d,
1233                                                 struct qbman_fd **fd,
1234                                                 uint32_t *flags,
1235                                                 int num_frames)
1236 {
1237         uint32_t *p = NULL;
1238         const uint32_t *cl = qb_cl(d);
1239         uint32_t eqcr_ci, eqcr_pi, half_mask, full_mask;
1240         int i, num_enqueued = 0;
1241         uint64_t addr_cena;
1242
1243         half_mask = (s->eqcr.pi_ci_mask>>1);
1244         full_mask = s->eqcr.pi_ci_mask;
1245         if (!s->eqcr.available) {
1246                 eqcr_ci = s->eqcr.ci;
1247                 s->eqcr.ci = qbman_cena_read_reg(&s->sys,
1248                                 QBMAN_CENA_SWP_EQCR_CI) & full_mask;
1249                 s->eqcr.available = qm_cyc_diff(s->eqcr.pi_ring_size,
1250                                 eqcr_ci, s->eqcr.ci);
1251                 if (!s->eqcr.available)
1252                         return 0;
1253         }
1254
1255         eqcr_pi = s->eqcr.pi;
1256         num_enqueued = (s->eqcr.available < num_frames) ?
1257                         s->eqcr.available : num_frames;
1258         s->eqcr.available -= num_enqueued;
1259         /* Fill in the EQCR ring */
1260         for (i = 0; i < num_enqueued; i++) {
1261                 p = qbman_cena_write_start_wo_shadow(&s->sys,
1262                                 QBMAN_CENA_SWP_EQCR(eqcr_pi & half_mask));
1263                 memcpy(&p[1], &cl[1], 28);
1264                 memcpy(&p[8], fd[i], sizeof(struct qbman_fd));
1265                 eqcr_pi++;
1266         }
1267
1268         lwsync();
1269
1270         /* Set the verb byte, have to substitute in the valid-bit */
1271         eqcr_pi = s->eqcr.pi;
1272         for (i = 0; i < num_enqueued; i++) {
1273                 p = qbman_cena_write_start_wo_shadow(&s->sys,
1274                                 QBMAN_CENA_SWP_EQCR(eqcr_pi & half_mask));
1275                 p[0] = cl[0] | s->eqcr.pi_vb;
1276                 if (flags && (flags[i] & QBMAN_ENQUEUE_FLAG_DCA)) {
1277                         struct qbman_eq_desc *d = (struct qbman_eq_desc *)p;
1278
1279                         d->eq.dca = (1 << QB_ENQUEUE_CMD_DCA_EN_SHIFT) |
1280                                 ((flags[i]) & QBMAN_EQCR_DCA_IDXMASK);
1281                 }
1282                 eqcr_pi++;
1283                 if (!(eqcr_pi & half_mask))
1284                         s->eqcr.pi_vb ^= QB_VALID_BIT;
1285         }
1286
1287         /* Flush all the cacheline without load/store in between */
1288         eqcr_pi = s->eqcr.pi;
1289         addr_cena = (size_t)s->sys.addr_cena;
1290         for (i = 0; i < num_enqueued; i++) {
1291                 dcbf(addr_cena +
1292                         QBMAN_CENA_SWP_EQCR(eqcr_pi & half_mask));
1293                 eqcr_pi++;
1294         }
1295         s->eqcr.pi = eqcr_pi & full_mask;
1296
1297         return num_enqueued;
1298 }
1299
1300 static int qbman_swp_enqueue_multiple_fd_cinh_read_direct(
1301                 struct qbman_swp *s,
1302                 const struct qbman_eq_desc *d,
1303                 struct qbman_fd **fd,
1304                 uint32_t *flags,
1305                 int num_frames)
1306 {
1307         uint32_t *p = NULL;
1308         const uint32_t *cl = qb_cl(d);
1309         uint32_t eqcr_ci, eqcr_pi, half_mask, full_mask;
1310         int i, num_enqueued = 0;
1311         uint64_t addr_cena;
1312
1313         half_mask = (s->eqcr.pi_ci_mask>>1);
1314         full_mask = s->eqcr.pi_ci_mask;
1315         if (!s->eqcr.available) {
1316                 eqcr_ci = s->eqcr.ci;
1317                 s->eqcr.ci = qbman_cinh_read(&s->sys,
1318                                 QBMAN_CINH_SWP_EQCR_CI) & full_mask;
1319                 s->eqcr.available = qm_cyc_diff(s->eqcr.pi_ring_size,
1320                                 eqcr_ci, s->eqcr.ci);
1321                 if (!s->eqcr.available)
1322                         return 0;
1323         }
1324
1325         eqcr_pi = s->eqcr.pi;
1326         num_enqueued = (s->eqcr.available < num_frames) ?
1327                         s->eqcr.available : num_frames;
1328         s->eqcr.available -= num_enqueued;
1329         /* Fill in the EQCR ring */
1330         for (i = 0; i < num_enqueued; i++) {
1331                 p = qbman_cena_write_start_wo_shadow(&s->sys,
1332                                 QBMAN_CENA_SWP_EQCR(eqcr_pi & half_mask));
1333                 memcpy(&p[1], &cl[1], 28);
1334                 memcpy(&p[8], fd[i], sizeof(struct qbman_fd));
1335                 eqcr_pi++;
1336         }
1337
1338         lwsync();
1339
1340         /* Set the verb byte, have to substitute in the valid-bit */
1341         eqcr_pi = s->eqcr.pi;
1342         for (i = 0; i < num_enqueued; i++) {
1343                 p = qbman_cena_write_start_wo_shadow(&s->sys,
1344                                 QBMAN_CENA_SWP_EQCR(eqcr_pi & half_mask));
1345                 p[0] = cl[0] | s->eqcr.pi_vb;
1346                 if (flags && (flags[i] & QBMAN_ENQUEUE_FLAG_DCA)) {
1347                         struct qbman_eq_desc *d = (struct qbman_eq_desc *)p;
1348
1349                         d->eq.dca = (1 << QB_ENQUEUE_CMD_DCA_EN_SHIFT) |
1350                                 ((flags[i]) & QBMAN_EQCR_DCA_IDXMASK);
1351                 }
1352                 eqcr_pi++;
1353                 if (!(eqcr_pi & half_mask))
1354                         s->eqcr.pi_vb ^= QB_VALID_BIT;
1355         }
1356
1357         /* Flush all the cacheline without load/store in between */
1358         eqcr_pi = s->eqcr.pi;
1359         addr_cena = (size_t)s->sys.addr_cena;
1360         for (i = 0; i < num_enqueued; i++) {
1361                 dcbf(addr_cena +
1362                         QBMAN_CENA_SWP_EQCR(eqcr_pi & half_mask));
1363                 eqcr_pi++;
1364         }
1365         s->eqcr.pi = eqcr_pi & full_mask;
1366
1367         return num_enqueued;
1368 }
1369
1370 static int qbman_swp_enqueue_multiple_fd_cinh_direct(
1371                 struct qbman_swp *s,
1372                 const struct qbman_eq_desc *d,
1373                 struct qbman_fd **fd,
1374                 uint32_t *flags,
1375                 int num_frames)
1376 {
1377         uint32_t *p = NULL;
1378         const uint32_t *cl = qb_cl(d);
1379         uint32_t eqcr_ci, eqcr_pi, half_mask, full_mask;
1380         int i, num_enqueued = 0;
1381
1382         half_mask = (s->eqcr.pi_ci_mask>>1);
1383         full_mask = s->eqcr.pi_ci_mask;
1384         if (!s->eqcr.available) {
1385                 eqcr_ci = s->eqcr.ci;
1386                 s->eqcr.ci = qbman_cinh_read(&s->sys,
1387                                 QBMAN_CINH_SWP_EQCR_CI) & full_mask;
1388                 s->eqcr.available = qm_cyc_diff(s->eqcr.pi_ring_size,
1389                                 eqcr_ci, s->eqcr.ci);
1390                 if (!s->eqcr.available)
1391                         return 0;
1392         }
1393
1394         eqcr_pi = s->eqcr.pi;
1395         num_enqueued = (s->eqcr.available < num_frames) ?
1396                         s->eqcr.available : num_frames;
1397         s->eqcr.available -= num_enqueued;
1398         /* Fill in the EQCR ring */
1399         for (i = 0; i < num_enqueued; i++) {
1400                 p = qbman_cinh_write_start_wo_shadow(&s->sys,
1401                                 QBMAN_CENA_SWP_EQCR(eqcr_pi & half_mask));
1402                 memcpy_byte_by_byte(&p[1], &cl[1], 28);
1403                 memcpy_byte_by_byte(&p[8], fd[i], sizeof(struct qbman_fd));
1404                 eqcr_pi++;
1405         }
1406
1407         lwsync();
1408
1409         /* Set the verb byte, have to substitute in the valid-bit */
1410         eqcr_pi = s->eqcr.pi;
1411         for (i = 0; i < num_enqueued; i++) {
1412                 p = qbman_cinh_write_start_wo_shadow(&s->sys,
1413                                 QBMAN_CENA_SWP_EQCR(eqcr_pi & half_mask));
1414                 p[0] = cl[0] | s->eqcr.pi_vb;
1415                 if (flags && (flags[i] & QBMAN_ENQUEUE_FLAG_DCA)) {
1416                         struct qbman_eq_desc *d = (struct qbman_eq_desc *)p;
1417
1418                         d->eq.dca = (1 << QB_ENQUEUE_CMD_DCA_EN_SHIFT) |
1419                                 ((flags[i]) & QBMAN_EQCR_DCA_IDXMASK);
1420                 }
1421                 eqcr_pi++;
1422                 if (!(eqcr_pi & half_mask))
1423                         s->eqcr.pi_vb ^= QB_VALID_BIT;
1424         }
1425
1426         s->eqcr.pi = eqcr_pi & full_mask;
1427
1428         return num_enqueued;
1429 }
1430
1431 static int qbman_swp_enqueue_multiple_fd_mem_back(struct qbman_swp *s,
1432                                                   const struct qbman_eq_desc *d,
1433                                                   struct qbman_fd **fd,
1434                                                   uint32_t *flags,
1435                                                   int num_frames)
1436 {
1437         uint32_t *p = NULL;
1438         const uint32_t *cl = qb_cl(d);
1439         uint32_t eqcr_ci, eqcr_pi, half_mask, full_mask;
1440         int i, num_enqueued = 0;
1441
1442         half_mask = (s->eqcr.pi_ci_mask>>1);
1443         full_mask = s->eqcr.pi_ci_mask;
1444         if (!s->eqcr.available) {
1445                 eqcr_ci = s->eqcr.ci;
1446                 s->eqcr.ci = qbman_cena_read_reg(&s->sys,
1447                                 QBMAN_CENA_SWP_EQCR_CI_MEMBACK) & full_mask;
1448                 s->eqcr.available = qm_cyc_diff(s->eqcr.pi_ring_size,
1449                                         eqcr_ci, s->eqcr.ci);
1450                 if (!s->eqcr.available)
1451                         return 0;
1452         }
1453
1454         eqcr_pi = s->eqcr.pi;
1455         num_enqueued = (s->eqcr.available < num_frames) ?
1456                         s->eqcr.available : num_frames;
1457         s->eqcr.available -= num_enqueued;
1458         /* Fill in the EQCR ring */
1459         for (i = 0; i < num_enqueued; i++) {
1460                 p = qbman_cena_write_start_wo_shadow(&s->sys,
1461                                 QBMAN_CENA_SWP_EQCR(eqcr_pi & half_mask));
1462                 memcpy(&p[1], &cl[1], 28);
1463                 memcpy(&p[8], fd[i], sizeof(struct qbman_fd));
1464                 eqcr_pi++;
1465         }
1466
1467         /* Set the verb byte, have to substitute in the valid-bit */
1468         eqcr_pi = s->eqcr.pi;
1469         for (i = 0; i < num_enqueued; i++) {
1470                 p = qbman_cena_write_start_wo_shadow(&s->sys,
1471                                 QBMAN_CENA_SWP_EQCR(eqcr_pi & half_mask));
1472                 p[0] = cl[0] | s->eqcr.pi_vb;
1473                 if (flags && (flags[i] & QBMAN_ENQUEUE_FLAG_DCA)) {
1474                         struct qbman_eq_desc *d = (struct qbman_eq_desc *)p;
1475
1476                         d->eq.dca = (1 << QB_ENQUEUE_CMD_DCA_EN_SHIFT) |
1477                                 ((flags[i]) & QBMAN_EQCR_DCA_IDXMASK);
1478                 }
1479                 eqcr_pi++;
1480                 if (!(eqcr_pi & half_mask))
1481                         s->eqcr.pi_vb ^= QB_VALID_BIT;
1482         }
1483         s->eqcr.pi = eqcr_pi & full_mask;
1484
1485         dma_wmb();
1486         qbman_cinh_write(&s->sys, QBMAN_CINH_SWP_EQCR_PI,
1487                                 (QB_RT_BIT)|(s->eqcr.pi)|s->eqcr.pi_vb);
1488         return num_enqueued;
1489 }
1490
1491 int qbman_swp_enqueue_multiple_fd(struct qbman_swp *s,
1492                                          const struct qbman_eq_desc *d,
1493                                          struct qbman_fd **fd,
1494                                          uint32_t *flags,
1495                                          int num_frames)
1496 {
1497         if (!s->stash_off)
1498                 return qbman_swp_enqueue_multiple_fd_ptr(s, d, fd, flags,
1499                                         num_frames);
1500         else
1501                 return qbman_swp_enqueue_multiple_fd_cinh_direct(s, d, fd,
1502                                         flags, num_frames);
1503 }
1504
1505 static int qbman_swp_enqueue_multiple_desc_direct(struct qbman_swp *s,
1506                                         const struct qbman_eq_desc *d,
1507                                         const struct qbman_fd *fd,
1508                                         int num_frames)
1509 {
1510         uint32_t *p;
1511         const uint32_t *cl;
1512         uint32_t eqcr_ci, eqcr_pi, half_mask, full_mask;
1513         int i, num_enqueued = 0;
1514         uint64_t addr_cena;
1515
1516         half_mask = (s->eqcr.pi_ci_mask>>1);
1517         full_mask = s->eqcr.pi_ci_mask;
1518         if (!s->eqcr.available) {
1519                 eqcr_ci = s->eqcr.ci;
1520                 s->eqcr.ci = qbman_cena_read_reg(&s->sys,
1521                                 QBMAN_CENA_SWP_EQCR_CI) & full_mask;
1522                 s->eqcr.available = qm_cyc_diff(s->eqcr.pi_ring_size,
1523                                         eqcr_ci, s->eqcr.ci);
1524                 if (!s->eqcr.available)
1525                         return 0;
1526         }
1527
1528         eqcr_pi = s->eqcr.pi;
1529         num_enqueued = (s->eqcr.available < num_frames) ?
1530                         s->eqcr.available : num_frames;
1531         s->eqcr.available -= num_enqueued;
1532         /* Fill in the EQCR ring */
1533         for (i = 0; i < num_enqueued; i++) {
1534                 p = qbman_cena_write_start_wo_shadow(&s->sys,
1535                                 QBMAN_CENA_SWP_EQCR(eqcr_pi & half_mask));
1536                 cl = qb_cl(&d[i]);
1537                 memcpy(&p[1], &cl[1], 28);
1538                 memcpy(&p[8], &fd[i], sizeof(*fd));
1539                 eqcr_pi++;
1540         }
1541
1542         lwsync();
1543
1544         /* Set the verb byte, have to substitute in the valid-bit */
1545         eqcr_pi = s->eqcr.pi;
1546         for (i = 0; i < num_enqueued; i++) {
1547                 p = qbman_cena_write_start_wo_shadow(&s->sys,
1548                                 QBMAN_CENA_SWP_EQCR(eqcr_pi & half_mask));
1549                 cl = qb_cl(&d[i]);
1550                 p[0] = cl[0] | s->eqcr.pi_vb;
1551                 eqcr_pi++;
1552                 if (!(eqcr_pi & half_mask))
1553                         s->eqcr.pi_vb ^= QB_VALID_BIT;
1554         }
1555
1556         /* Flush all the cacheline without load/store in between */
1557         eqcr_pi = s->eqcr.pi;
1558         addr_cena = (size_t)s->sys.addr_cena;
1559         for (i = 0; i < num_enqueued; i++) {
1560                 dcbf((uintptr_t)(addr_cena +
1561                         QBMAN_CENA_SWP_EQCR(eqcr_pi & half_mask)));
1562                 eqcr_pi++;
1563         }
1564         s->eqcr.pi = eqcr_pi & full_mask;
1565
1566         return num_enqueued;
1567 }
1568
1569 static int qbman_swp_enqueue_multiple_desc_cinh_read_direct(
1570                 struct qbman_swp *s,
1571                 const struct qbman_eq_desc *d,
1572                 const struct qbman_fd *fd,
1573                 int num_frames)
1574 {
1575         uint32_t *p;
1576         const uint32_t *cl;
1577         uint32_t eqcr_ci, eqcr_pi, half_mask, full_mask;
1578         int i, num_enqueued = 0;
1579         uint64_t addr_cena;
1580
1581         half_mask = (s->eqcr.pi_ci_mask>>1);
1582         full_mask = s->eqcr.pi_ci_mask;
1583         if (!s->eqcr.available) {
1584                 eqcr_ci = s->eqcr.ci;
1585                 s->eqcr.ci = qbman_cinh_read(&s->sys,
1586                                 QBMAN_CINH_SWP_EQCR_CI) & full_mask;
1587                 s->eqcr.available = qm_cyc_diff(s->eqcr.pi_ring_size,
1588                                         eqcr_ci, s->eqcr.ci);
1589                 if (!s->eqcr.available)
1590                         return 0;
1591         }
1592
1593         eqcr_pi = s->eqcr.pi;
1594         num_enqueued = (s->eqcr.available < num_frames) ?
1595                         s->eqcr.available : num_frames;
1596         s->eqcr.available -= num_enqueued;
1597         /* Fill in the EQCR ring */
1598         for (i = 0; i < num_enqueued; i++) {
1599                 p = qbman_cena_write_start_wo_shadow(&s->sys,
1600                                 QBMAN_CENA_SWP_EQCR(eqcr_pi & half_mask));
1601                 cl = qb_cl(&d[i]);
1602                 memcpy(&p[1], &cl[1], 28);
1603                 memcpy(&p[8], &fd[i], sizeof(*fd));
1604                 eqcr_pi++;
1605         }
1606
1607         lwsync();
1608
1609         /* Set the verb byte, have to substitute in the valid-bit */
1610         eqcr_pi = s->eqcr.pi;
1611         for (i = 0; i < num_enqueued; i++) {
1612                 p = qbman_cena_write_start_wo_shadow(&s->sys,
1613                                 QBMAN_CENA_SWP_EQCR(eqcr_pi & half_mask));
1614                 cl = qb_cl(&d[i]);
1615                 p[0] = cl[0] | s->eqcr.pi_vb;
1616                 eqcr_pi++;
1617                 if (!(eqcr_pi & half_mask))
1618                         s->eqcr.pi_vb ^= QB_VALID_BIT;
1619         }
1620
1621         /* Flush all the cacheline without load/store in between */
1622         eqcr_pi = s->eqcr.pi;
1623         addr_cena = (size_t)s->sys.addr_cena;
1624         for (i = 0; i < num_enqueued; i++) {
1625                 dcbf(addr_cena +
1626                         QBMAN_CENA_SWP_EQCR(eqcr_pi & half_mask));
1627                 eqcr_pi++;
1628         }
1629         s->eqcr.pi = eqcr_pi & full_mask;
1630
1631         return num_enqueued;
1632 }
1633
1634 static int qbman_swp_enqueue_multiple_desc_cinh_direct(
1635                 struct qbman_swp *s,
1636                 const struct qbman_eq_desc *d,
1637                 const struct qbman_fd *fd,
1638                 int num_frames)
1639 {
1640         uint32_t *p;
1641         const uint32_t *cl;
1642         uint32_t eqcr_ci, eqcr_pi, half_mask, full_mask;
1643         int i, num_enqueued = 0;
1644
1645         half_mask = (s->eqcr.pi_ci_mask>>1);
1646         full_mask = s->eqcr.pi_ci_mask;
1647         if (!s->eqcr.available) {
1648                 eqcr_ci = s->eqcr.ci;
1649                 s->eqcr.ci = qbman_cinh_read(&s->sys,
1650                                 QBMAN_CINH_SWP_EQCR_CI) & full_mask;
1651                 s->eqcr.available = qm_cyc_diff(s->eqcr.pi_ring_size,
1652                                         eqcr_ci, s->eqcr.ci);
1653                 if (!s->eqcr.available)
1654                         return 0;
1655         }
1656
1657         eqcr_pi = s->eqcr.pi;
1658         num_enqueued = (s->eqcr.available < num_frames) ?
1659                         s->eqcr.available : num_frames;
1660         s->eqcr.available -= num_enqueued;
1661         /* Fill in the EQCR ring */
1662         for (i = 0; i < num_enqueued; i++) {
1663                 p = qbman_cinh_write_start_wo_shadow(&s->sys,
1664                                 QBMAN_CENA_SWP_EQCR(eqcr_pi & half_mask));
1665                 cl = qb_cl(&d[i]);
1666                 memcpy_byte_by_byte(&p[1], &cl[1], 28);
1667                 memcpy_byte_by_byte(&p[8], &fd[i], sizeof(*fd));
1668                 eqcr_pi++;
1669         }
1670
1671         lwsync();
1672
1673         /* Set the verb byte, have to substitute in the valid-bit */
1674         eqcr_pi = s->eqcr.pi;
1675         for (i = 0; i < num_enqueued; i++) {
1676                 p = qbman_cinh_write_start_wo_shadow(&s->sys,
1677                                 QBMAN_CENA_SWP_EQCR(eqcr_pi & half_mask));
1678                 cl = qb_cl(&d[i]);
1679                 p[0] = cl[0] | s->eqcr.pi_vb;
1680                 eqcr_pi++;
1681                 if (!(eqcr_pi & half_mask))
1682                         s->eqcr.pi_vb ^= QB_VALID_BIT;
1683         }
1684
1685         s->eqcr.pi = eqcr_pi & full_mask;
1686
1687         return num_enqueued;
1688 }
1689
1690 static int qbman_swp_enqueue_multiple_desc_mem_back(struct qbman_swp *s,
1691                                         const struct qbman_eq_desc *d,
1692                                         const struct qbman_fd *fd,
1693                                         int num_frames)
1694 {
1695         uint32_t *p;
1696         const uint32_t *cl;
1697         uint32_t eqcr_ci, eqcr_pi, half_mask, full_mask;
1698         int i, num_enqueued = 0;
1699
1700         half_mask = (s->eqcr.pi_ci_mask>>1);
1701         full_mask = s->eqcr.pi_ci_mask;
1702         if (!s->eqcr.available) {
1703                 eqcr_ci = s->eqcr.ci;
1704                 s->eqcr.ci = qbman_cena_read_reg(&s->sys,
1705                                 QBMAN_CENA_SWP_EQCR_CI_MEMBACK) & full_mask;
1706                 s->eqcr.available = qm_cyc_diff(s->eqcr.pi_ring_size,
1707                                         eqcr_ci, s->eqcr.ci);
1708                 if (!s->eqcr.available)
1709                         return 0;
1710         }
1711
1712         eqcr_pi = s->eqcr.pi;
1713         num_enqueued = (s->eqcr.available < num_frames) ?
1714                         s->eqcr.available : num_frames;
1715         s->eqcr.available -= num_enqueued;
1716         /* Fill in the EQCR ring */
1717         for (i = 0; i < num_enqueued; i++) {
1718                 p = qbman_cena_write_start_wo_shadow(&s->sys,
1719                                 QBMAN_CENA_SWP_EQCR(eqcr_pi & half_mask));
1720                 cl = qb_cl(&d[i]);
1721                 memcpy(&p[1], &cl[1], 28);
1722                 memcpy(&p[8], &fd[i], sizeof(*fd));
1723                 eqcr_pi++;
1724         }
1725
1726         /* Set the verb byte, have to substitute in the valid-bit */
1727         eqcr_pi = s->eqcr.pi;
1728         for (i = 0; i < num_enqueued; i++) {
1729                 p = qbman_cena_write_start_wo_shadow(&s->sys,
1730                                 QBMAN_CENA_SWP_EQCR(eqcr_pi & half_mask));
1731                 cl = qb_cl(&d[i]);
1732                 p[0] = cl[0] | s->eqcr.pi_vb;
1733                 eqcr_pi++;
1734                 if (!(eqcr_pi & half_mask))
1735                         s->eqcr.pi_vb ^= QB_VALID_BIT;
1736         }
1737
1738         s->eqcr.pi = eqcr_pi & full_mask;
1739
1740         dma_wmb();
1741         qbman_cinh_write(&s->sys, QBMAN_CINH_SWP_EQCR_PI,
1742                                 (QB_RT_BIT)|(s->eqcr.pi)|s->eqcr.pi_vb);
1743
1744         return num_enqueued;
1745 }
1746 int qbman_swp_enqueue_multiple_desc(struct qbman_swp *s,
1747                                            const struct qbman_eq_desc *d,
1748                                            const struct qbman_fd *fd,
1749                                            int num_frames)
1750 {
1751         if (!s->stash_off)
1752                 return qbman_swp_enqueue_multiple_desc_ptr(s, d, fd,
1753                                         num_frames);
1754         else
1755                 return qbman_swp_enqueue_multiple_desc_cinh_direct(s, d, fd,
1756                                         num_frames);
1757
1758 }
1759
1760 /*************************/
1761 /* Static (push) dequeue */
1762 /*************************/
1763
1764 void qbman_swp_push_get(struct qbman_swp *s, uint8_t channel_idx, int *enabled)
1765 {
1766         uint16_t src = (s->sdq >> QB_SDQCR_SRC_SHIFT) & QB_SDQCR_SRC_MASK;
1767
1768         QBMAN_BUG_ON(channel_idx > 15);
1769         *enabled = src | (1 << channel_idx);
1770 }
1771
1772 void qbman_swp_push_set(struct qbman_swp *s, uint8_t channel_idx, int enable)
1773 {
1774         uint16_t dqsrc;
1775
1776         QBMAN_BUG_ON(channel_idx > 15);
1777         if (enable)
1778                 s->sdq |= 1 << channel_idx;
1779         else
1780                 s->sdq &= ~(1 << channel_idx);
1781
1782         /* Read make the complete src map.  If no channels are enabled
1783          * the SDQCR must be 0 or else QMan will assert errors
1784          */
1785         dqsrc = (s->sdq >> QB_SDQCR_SRC_SHIFT) & QB_SDQCR_SRC_MASK;
1786         if (dqsrc != 0)
1787                 qbman_cinh_write(&s->sys, QBMAN_CINH_SWP_SDQCR, s->sdq);
1788         else
1789                 qbman_cinh_write(&s->sys, QBMAN_CINH_SWP_SDQCR, 0);
1790 }
1791
1792 /***************************/
1793 /* Volatile (pull) dequeue */
1794 /***************************/
1795
1796 /* These should be const, eventually */
1797 #define QB_VDQCR_VERB_DCT_SHIFT    0
1798 #define QB_VDQCR_VERB_DT_SHIFT     2
1799 #define QB_VDQCR_VERB_RLS_SHIFT    4
1800 #define QB_VDQCR_VERB_WAE_SHIFT    5
1801 #define QB_VDQCR_VERB_RAD_SHIFT    6
1802
1803 enum qb_pull_dt_e {
1804         qb_pull_dt_channel,
1805         qb_pull_dt_workqueue,
1806         qb_pull_dt_framequeue
1807 };
1808
1809 void qbman_pull_desc_clear(struct qbman_pull_desc *d)
1810 {
1811         memset(d, 0, sizeof(*d));
1812 }
1813
1814 void qbman_pull_desc_set_storage(struct qbman_pull_desc *d,
1815                                  struct qbman_result *storage,
1816                                  dma_addr_t storage_phys,
1817                                  int stash)
1818 {
1819         d->pull.rsp_addr_virt = (size_t)storage;
1820
1821         if (!storage) {
1822                 d->pull.verb &= ~(1 << QB_VDQCR_VERB_RLS_SHIFT);
1823                 return;
1824         }
1825         d->pull.verb |= 1 << QB_VDQCR_VERB_RLS_SHIFT;
1826         if (stash)
1827                 d->pull.verb |= 1 << QB_VDQCR_VERB_WAE_SHIFT;
1828         else
1829                 d->pull.verb &= ~(1 << QB_VDQCR_VERB_WAE_SHIFT);
1830
1831         d->pull.rsp_addr = storage_phys;
1832 }
1833
1834 void qbman_pull_desc_set_numframes(struct qbman_pull_desc *d,
1835                                    uint8_t numframes)
1836 {
1837         d->pull.numf = numframes - 1;
1838 }
1839
1840 void qbman_pull_desc_set_token(struct qbman_pull_desc *d, uint8_t token)
1841 {
1842         d->pull.tok = token;
1843 }
1844
1845 void qbman_pull_desc_set_fq(struct qbman_pull_desc *d, uint32_t fqid)
1846 {
1847         d->pull.verb |= 1 << QB_VDQCR_VERB_DCT_SHIFT;
1848         d->pull.verb |= qb_pull_dt_framequeue << QB_VDQCR_VERB_DT_SHIFT;
1849         d->pull.dq_src = fqid;
1850 }
1851
1852 void qbman_pull_desc_set_wq(struct qbman_pull_desc *d, uint32_t wqid,
1853                             enum qbman_pull_type_e dct)
1854 {
1855         d->pull.verb |= dct << QB_VDQCR_VERB_DCT_SHIFT;
1856         d->pull.verb |= qb_pull_dt_workqueue << QB_VDQCR_VERB_DT_SHIFT;
1857         d->pull.dq_src = wqid;
1858 }
1859
1860 void qbman_pull_desc_set_channel(struct qbman_pull_desc *d, uint32_t chid,
1861                                  enum qbman_pull_type_e dct)
1862 {
1863         d->pull.verb |= dct << QB_VDQCR_VERB_DCT_SHIFT;
1864         d->pull.verb |= qb_pull_dt_channel << QB_VDQCR_VERB_DT_SHIFT;
1865         d->pull.dq_src = chid;
1866 }
1867
1868 void qbman_pull_desc_set_rad(struct qbman_pull_desc *d, int rad)
1869 {
1870         if (d->pull.verb & (1 << QB_VDQCR_VERB_RLS_SHIFT)) {
1871                 if (rad)
1872                         d->pull.verb |= 1 << QB_VDQCR_VERB_RAD_SHIFT;
1873                 else
1874                         d->pull.verb &= ~(1 << QB_VDQCR_VERB_RAD_SHIFT);
1875         } else {
1876                 printf("The RAD feature is not valid when RLS = 0\n");
1877         }
1878 }
1879
1880 static int qbman_swp_pull_direct(struct qbman_swp *s,
1881                                  struct qbman_pull_desc *d)
1882 {
1883         uint32_t *p;
1884         uint32_t *cl = qb_cl(d);
1885
1886         if (!atomic_dec_and_test(&s->vdq.busy)) {
1887                 atomic_inc(&s->vdq.busy);
1888                 return -EBUSY;
1889         }
1890
1891         d->pull.tok = s->sys.idx + 1;
1892         s->vdq.storage = (void *)(size_t)d->pull.rsp_addr_virt;
1893         p = qbman_cena_write_start_wo_shadow(&s->sys, QBMAN_CENA_SWP_VDQCR);
1894         memcpy(&p[1], &cl[1], 12);
1895
1896         /* Set the verb byte, have to substitute in the valid-bit */
1897         lwsync();
1898         p[0] = cl[0] | s->vdq.valid_bit;
1899         s->vdq.valid_bit ^= QB_VALID_BIT;
1900         qbman_cena_write_complete_wo_shadow(&s->sys, QBMAN_CENA_SWP_VDQCR);
1901
1902         return 0;
1903 }
1904
1905 static int qbman_swp_pull_cinh_direct(struct qbman_swp *s,
1906                                  struct qbman_pull_desc *d)
1907 {
1908         uint32_t *p;
1909         uint32_t *cl = qb_cl(d);
1910
1911         if (!atomic_dec_and_test(&s->vdq.busy)) {
1912                 atomic_inc(&s->vdq.busy);
1913                 return -EBUSY;
1914         }
1915
1916         d->pull.tok = s->sys.idx + 1;
1917         s->vdq.storage = (void *)(size_t)d->pull.rsp_addr_virt;
1918         p = qbman_cinh_write_start_wo_shadow(&s->sys, QBMAN_CENA_SWP_VDQCR);
1919         memcpy_byte_by_byte(&p[1], &cl[1], 12);
1920
1921         /* Set the verb byte, have to substitute in the valid-bit */
1922         lwsync();
1923         p[0] = cl[0] | s->vdq.valid_bit;
1924         s->vdq.valid_bit ^= QB_VALID_BIT;
1925
1926         return 0;
1927 }
1928
1929 static int qbman_swp_pull_mem_back(struct qbman_swp *s,
1930                                    struct qbman_pull_desc *d)
1931 {
1932         uint32_t *p;
1933         uint32_t *cl = qb_cl(d);
1934
1935         if (!atomic_dec_and_test(&s->vdq.busy)) {
1936                 atomic_inc(&s->vdq.busy);
1937                 return -EBUSY;
1938         }
1939
1940         d->pull.tok = s->sys.idx + 1;
1941         s->vdq.storage = (void *)(size_t)d->pull.rsp_addr_virt;
1942         p = qbman_cena_write_start_wo_shadow(&s->sys, QBMAN_CENA_SWP_VDQCR_MEM);
1943         memcpy(&p[1], &cl[1], 12);
1944
1945         /* Set the verb byte, have to substitute in the valid-bit */
1946         p[0] = cl[0] | s->vdq.valid_bit;
1947         s->vdq.valid_bit ^= QB_VALID_BIT;
1948         dma_wmb();
1949         qbman_cinh_write(&s->sys, QBMAN_CINH_SWP_VDQCR_RT, QMAN_RT_MODE);
1950
1951         return 0;
1952 }
1953
1954 int qbman_swp_pull(struct qbman_swp *s, struct qbman_pull_desc *d)
1955 {
1956         if (!s->stash_off)
1957                 return qbman_swp_pull_ptr(s, d);
1958         else
1959                 return qbman_swp_pull_cinh_direct(s, d);
1960 }
1961
1962 /****************/
1963 /* Polling DQRR */
1964 /****************/
1965
1966 #define QMAN_DQRR_PI_MASK              0xf
1967
1968 #define QBMAN_RESULT_DQ        0x60
1969 #define QBMAN_RESULT_FQRN      0x21
1970 #define QBMAN_RESULT_FQRNI     0x22
1971 #define QBMAN_RESULT_FQPN      0x24
1972 #define QBMAN_RESULT_FQDAN     0x25
1973 #define QBMAN_RESULT_CDAN      0x26
1974 #define QBMAN_RESULT_CSCN_MEM  0x27
1975 #define QBMAN_RESULT_CGCU      0x28
1976 #define QBMAN_RESULT_BPSCN     0x29
1977 #define QBMAN_RESULT_CSCN_WQ   0x2a
1978
1979 #include <rte_prefetch.h>
1980
1981 void qbman_swp_prefetch_dqrr_next(struct qbman_swp *s)
1982 {
1983         const struct qbman_result *p;
1984
1985         p = qbman_cena_read_wo_shadow(&s->sys,
1986                 QBMAN_CENA_SWP_DQRR(s->dqrr.next_idx));
1987         rte_prefetch0(p);
1988 }
1989
1990 /* NULL return if there are no unconsumed DQRR entries. Returns a DQRR entry
1991  * only once, so repeated calls can return a sequence of DQRR entries, without
1992  * requiring they be consumed immediately or in any particular order.
1993  */
1994 const struct qbman_result *qbman_swp_dqrr_next(struct qbman_swp *s)
1995 {
1996         if (!s->stash_off)
1997                 return qbman_swp_dqrr_next_ptr(s);
1998         else
1999                 return qbman_swp_dqrr_next_cinh_direct(s);
2000 }
2001
2002 const struct qbman_result *qbman_swp_dqrr_next_direct(struct qbman_swp *s)
2003 {
2004         uint32_t verb;
2005         uint32_t response_verb;
2006         uint32_t flags;
2007         const struct qbman_result *p;
2008
2009         /* Before using valid-bit to detect if something is there, we have to
2010          * handle the case of the DQRR reset bug...
2011          */
2012         if (s->dqrr.reset_bug) {
2013                 /* We pick up new entries by cache-inhibited producer index,
2014                  * which means that a non-coherent mapping would require us to
2015                  * invalidate and read *only* once that PI has indicated that
2016                  * there's an entry here. The first trip around the DQRR ring
2017                  * will be much less efficient than all subsequent trips around
2018                  * it...
2019                  */
2020                 uint8_t pi = qbman_cinh_read(&s->sys, QBMAN_CINH_SWP_DQPI) &
2021                              QMAN_DQRR_PI_MASK;
2022
2023                 /* there are new entries if pi != next_idx */
2024                 if (pi == s->dqrr.next_idx)
2025                         return NULL;
2026
2027                 /* if next_idx is/was the last ring index, and 'pi' is
2028                  * different, we can disable the workaround as all the ring
2029                  * entries have now been DMA'd to so valid-bit checking is
2030                  * repaired. Note: this logic needs to be based on next_idx
2031                  * (which increments one at a time), rather than on pi (which
2032                  * can burst and wrap-around between our snapshots of it).
2033                  */
2034                 QBMAN_BUG_ON((s->dqrr.dqrr_size - 1) < 0);
2035                 if (s->dqrr.next_idx == (s->dqrr.dqrr_size - 1u)) {
2036                         pr_debug("DEBUG: next_idx=%d, pi=%d, clear reset bug\n",
2037                                  s->dqrr.next_idx, pi);
2038                         s->dqrr.reset_bug = 0;
2039                 }
2040                 qbman_cena_invalidate_prefetch(&s->sys,
2041                                         QBMAN_CENA_SWP_DQRR(s->dqrr.next_idx));
2042         }
2043         p = qbman_cena_read_wo_shadow(&s->sys,
2044                         QBMAN_CENA_SWP_DQRR(s->dqrr.next_idx));
2045
2046         verb = p->dq.verb;
2047
2048         /* If the valid-bit isn't of the expected polarity, nothing there. Note,
2049          * in the DQRR reset bug workaround, we shouldn't need to skip these
2050          * check, because we've already determined that a new entry is available
2051          * and we've invalidated the cacheline before reading it, so the
2052          * valid-bit behaviour is repaired and should tell us what we already
2053          * knew from reading PI.
2054          */
2055         if ((verb & QB_VALID_BIT) != s->dqrr.valid_bit)
2056                 return NULL;
2057
2058         /* There's something there. Move "next_idx" attention to the next ring
2059          * entry (and prefetch it) before returning what we found.
2060          */
2061         s->dqrr.next_idx++;
2062         if (s->dqrr.next_idx == s->dqrr.dqrr_size) {
2063                 s->dqrr.next_idx = 0;
2064                 s->dqrr.valid_bit ^= QB_VALID_BIT;
2065         }
2066         /* If this is the final response to a volatile dequeue command
2067          * indicate that the vdq is no longer busy
2068          */
2069         flags = p->dq.stat;
2070         response_verb = verb & QBMAN_RESPONSE_VERB_MASK;
2071         if ((response_verb == QBMAN_RESULT_DQ) &&
2072             (flags & QBMAN_DQ_STAT_VOLATILE) &&
2073             (flags & QBMAN_DQ_STAT_EXPIRED))
2074                 atomic_inc(&s->vdq.busy);
2075
2076         return p;
2077 }
2078
2079 const struct qbman_result *qbman_swp_dqrr_next_cinh_direct(struct qbman_swp *s)
2080 {
2081         uint32_t verb;
2082         uint32_t response_verb;
2083         uint32_t flags;
2084         const struct qbman_result *p;
2085
2086         /* Before using valid-bit to detect if something is there, we have to
2087          * handle the case of the DQRR reset bug...
2088          */
2089         if (s->dqrr.reset_bug) {
2090                 /* We pick up new entries by cache-inhibited producer index,
2091                  * which means that a non-coherent mapping would require us to
2092                  * invalidate and read *only* once that PI has indicated that
2093                  * there's an entry here. The first trip around the DQRR ring
2094                  * will be much less efficient than all subsequent trips around
2095                  * it...
2096                  */
2097                 uint8_t pi = qbman_cinh_read(&s->sys, QBMAN_CINH_SWP_DQPI) &
2098                              QMAN_DQRR_PI_MASK;
2099
2100                 /* there are new entries if pi != next_idx */
2101                 if (pi == s->dqrr.next_idx)
2102                         return NULL;
2103
2104                 /* if next_idx is/was the last ring index, and 'pi' is
2105                  * different, we can disable the workaround as all the ring
2106                  * entries have now been DMA'd to so valid-bit checking is
2107                  * repaired. Note: this logic needs to be based on next_idx
2108                  * (which increments one at a time), rather than on pi (which
2109                  * can burst and wrap-around between our snapshots of it).
2110                  */
2111                 QBMAN_BUG_ON((s->dqrr.dqrr_size - 1) < 0);
2112                 if (s->dqrr.next_idx == (s->dqrr.dqrr_size - 1u)) {
2113                         pr_debug("DEBUG: next_idx=%d, pi=%d, clear reset bug\n",
2114                                  s->dqrr.next_idx, pi);
2115                         s->dqrr.reset_bug = 0;
2116                 }
2117         }
2118         p = qbman_cinh_read_wo_shadow(&s->sys,
2119                         QBMAN_CENA_SWP_DQRR(s->dqrr.next_idx));
2120
2121         verb = p->dq.verb;
2122
2123         /* If the valid-bit isn't of the expected polarity, nothing there. Note,
2124          * in the DQRR reset bug workaround, we shouldn't need to skip these
2125          * check, because we've already determined that a new entry is available
2126          * and we've invalidated the cacheline before reading it, so the
2127          * valid-bit behaviour is repaired and should tell us what we already
2128          * knew from reading PI.
2129          */
2130         if ((verb & QB_VALID_BIT) != s->dqrr.valid_bit)
2131                 return NULL;
2132
2133         /* There's something there. Move "next_idx" attention to the next ring
2134          * entry (and prefetch it) before returning what we found.
2135          */
2136         s->dqrr.next_idx++;
2137         if (s->dqrr.next_idx == s->dqrr.dqrr_size) {
2138                 s->dqrr.next_idx = 0;
2139                 s->dqrr.valid_bit ^= QB_VALID_BIT;
2140         }
2141         /* If this is the final response to a volatile dequeue command
2142          * indicate that the vdq is no longer busy
2143          */
2144         flags = p->dq.stat;
2145         response_verb = verb & QBMAN_RESPONSE_VERB_MASK;
2146         if ((response_verb == QBMAN_RESULT_DQ) &&
2147             (flags & QBMAN_DQ_STAT_VOLATILE) &&
2148             (flags & QBMAN_DQ_STAT_EXPIRED))
2149                 atomic_inc(&s->vdq.busy);
2150
2151         return p;
2152 }
2153
2154 const struct qbman_result *qbman_swp_dqrr_next_mem_back(struct qbman_swp *s)
2155 {
2156         uint32_t verb;
2157         uint32_t response_verb;
2158         uint32_t flags;
2159         const struct qbman_result *p;
2160
2161         p = qbman_cena_read_wo_shadow(&s->sys,
2162                         QBMAN_CENA_SWP_DQRR_MEM(s->dqrr.next_idx));
2163
2164         verb = p->dq.verb;
2165
2166         /* If the valid-bit isn't of the expected polarity, nothing there. Note,
2167          * in the DQRR reset bug workaround, we shouldn't need to skip these
2168          * check, because we've already determined that a new entry is available
2169          * and we've invalidated the cacheline before reading it, so the
2170          * valid-bit behaviour is repaired and should tell us what we already
2171          * knew from reading PI.
2172          */
2173         if ((verb & QB_VALID_BIT) != s->dqrr.valid_bit)
2174                 return NULL;
2175
2176         /* There's something there. Move "next_idx" attention to the next ring
2177          * entry (and prefetch it) before returning what we found.
2178          */
2179         s->dqrr.next_idx++;
2180         if (s->dqrr.next_idx == s->dqrr.dqrr_size) {
2181                 s->dqrr.next_idx = 0;
2182                 s->dqrr.valid_bit ^= QB_VALID_BIT;
2183         }
2184         /* If this is the final response to a volatile dequeue command
2185          * indicate that the vdq is no longer busy
2186          */
2187         flags = p->dq.stat;
2188         response_verb = verb & QBMAN_RESPONSE_VERB_MASK;
2189         if ((response_verb == QBMAN_RESULT_DQ)
2190                         && (flags & QBMAN_DQ_STAT_VOLATILE)
2191                         && (flags & QBMAN_DQ_STAT_EXPIRED))
2192                 atomic_inc(&s->vdq.busy);
2193         return p;
2194 }
2195
2196 /* Consume DQRR entries previously returned from qbman_swp_dqrr_next(). */
2197 void qbman_swp_dqrr_consume(struct qbman_swp *s,
2198                             const struct qbman_result *dq)
2199 {
2200         qbman_cinh_write(&s->sys,
2201                         QBMAN_CINH_SWP_DCAP, QBMAN_IDX_FROM_DQRR(dq));
2202 }
2203
2204 /* Consume DQRR entries previously returned from qbman_swp_dqrr_next(). */
2205 void qbman_swp_dqrr_idx_consume(struct qbman_swp *s,
2206                             uint8_t dqrr_index)
2207 {
2208         qbman_cinh_write(&s->sys, QBMAN_CINH_SWP_DCAP, dqrr_index);
2209 }
2210
2211 /*********************************/
2212 /* Polling user-provided storage */
2213 /*********************************/
2214
2215 int qbman_result_has_new_result(struct qbman_swp *s,
2216                                 struct qbman_result *dq)
2217 {
2218         if (dq->dq.tok == 0)
2219                 return 0;
2220
2221         /*
2222          * Set token to be 0 so we will detect change back to 1
2223          * next time the looping is traversed. Const is cast away here
2224          * as we want users to treat the dequeue responses as read only.
2225          */
2226         ((struct qbman_result *)dq)->dq.tok = 0;
2227
2228         /*
2229          * VDQCR "no longer busy" hook - not quite the same as DQRR, because
2230          * the fact "VDQCR" shows busy doesn't mean that we hold the result
2231          * that makes it available. Eg. we may be looking at our 10th dequeue
2232          * result, having released VDQCR after the 1st result and it is now
2233          * busy due to some other command!
2234          */
2235         if (s->vdq.storage == dq) {
2236                 s->vdq.storage = NULL;
2237                 atomic_inc(&s->vdq.busy);
2238         }
2239
2240         return 1;
2241 }
2242
2243 int qbman_check_new_result(struct qbman_result *dq)
2244 {
2245         if (dq->dq.tok == 0)
2246                 return 0;
2247
2248         /*
2249          * Set token to be 0 so we will detect change back to 1
2250          * next time the looping is traversed. Const is cast away here
2251          * as we want users to treat the dequeue responses as read only.
2252          */
2253         ((struct qbman_result *)dq)->dq.tok = 0;
2254
2255         return 1;
2256 }
2257
2258 int qbman_check_command_complete(struct qbman_result *dq)
2259 {
2260         struct qbman_swp *s;
2261
2262         if (dq->dq.tok == 0)
2263                 return 0;
2264
2265         s = portal_idx_map[dq->dq.tok - 1];
2266         /*
2267          * VDQCR "no longer busy" hook - not quite the same as DQRR, because
2268          * the fact "VDQCR" shows busy doesn't mean that we hold the result
2269          * that makes it available. Eg. we may be looking at our 10th dequeue
2270          * result, having released VDQCR after the 1st result and it is now
2271          * busy due to some other command!
2272          */
2273         if (s->vdq.storage == dq) {
2274                 s->vdq.storage = NULL;
2275                 atomic_inc(&s->vdq.busy);
2276         }
2277
2278         return 1;
2279 }
2280
2281 /********************************/
2282 /* Categorising qbman results   */
2283 /********************************/
2284
2285 static inline int __qbman_result_is_x(const struct qbman_result *dq,
2286                                       uint8_t x)
2287 {
2288         uint8_t response_verb = dq->dq.verb & QBMAN_RESPONSE_VERB_MASK;
2289
2290         return (response_verb == x);
2291 }
2292
2293 int qbman_result_is_DQ(const struct qbman_result *dq)
2294 {
2295         return __qbman_result_is_x(dq, QBMAN_RESULT_DQ);
2296 }
2297
2298 int qbman_result_is_FQDAN(const struct qbman_result *dq)
2299 {
2300         return __qbman_result_is_x(dq, QBMAN_RESULT_FQDAN);
2301 }
2302
2303 int qbman_result_is_CDAN(const struct qbman_result *dq)
2304 {
2305         return __qbman_result_is_x(dq, QBMAN_RESULT_CDAN);
2306 }
2307
2308 int qbman_result_is_CSCN(const struct qbman_result *dq)
2309 {
2310         return __qbman_result_is_x(dq, QBMAN_RESULT_CSCN_MEM) ||
2311                 __qbman_result_is_x(dq, QBMAN_RESULT_CSCN_WQ);
2312 }
2313
2314 int qbman_result_is_BPSCN(const struct qbman_result *dq)
2315 {
2316         return __qbman_result_is_x(dq, QBMAN_RESULT_BPSCN);
2317 }
2318
2319 int qbman_result_is_CGCU(const struct qbman_result *dq)
2320 {
2321         return __qbman_result_is_x(dq, QBMAN_RESULT_CGCU);
2322 }
2323
2324 int qbman_result_is_FQRN(const struct qbman_result *dq)
2325 {
2326         return __qbman_result_is_x(dq, QBMAN_RESULT_FQRN);
2327 }
2328
2329 int qbman_result_is_FQRNI(const struct qbman_result *dq)
2330 {
2331         return __qbman_result_is_x(dq, QBMAN_RESULT_FQRNI);
2332 }
2333
2334 int qbman_result_is_FQPN(const struct qbman_result *dq)
2335 {
2336         return __qbman_result_is_x(dq, QBMAN_RESULT_FQPN);
2337 }
2338
2339 /*********************************/
2340 /* Parsing frame dequeue results */
2341 /*********************************/
2342
2343 /* These APIs assume qbman_result_is_DQ() is TRUE */
2344
2345 uint8_t qbman_result_DQ_flags(const struct qbman_result *dq)
2346 {
2347         return dq->dq.stat;
2348 }
2349
2350 uint16_t qbman_result_DQ_seqnum(const struct qbman_result *dq)
2351 {
2352         return dq->dq.seqnum;
2353 }
2354
2355 uint16_t qbman_result_DQ_odpid(const struct qbman_result *dq)
2356 {
2357         return dq->dq.oprid;
2358 }
2359
2360 uint32_t qbman_result_DQ_fqid(const struct qbman_result *dq)
2361 {
2362         return dq->dq.fqid;
2363 }
2364
2365 uint32_t qbman_result_DQ_byte_count(const struct qbman_result *dq)
2366 {
2367         return dq->dq.fq_byte_cnt;
2368 }
2369
2370 uint32_t qbman_result_DQ_frame_count(const struct qbman_result *dq)
2371 {
2372         return dq->dq.fq_frm_cnt;
2373 }
2374
2375 uint64_t qbman_result_DQ_fqd_ctx(const struct qbman_result *dq)
2376 {
2377         return dq->dq.fqd_ctx;
2378 }
2379
2380 const struct qbman_fd *qbman_result_DQ_fd(const struct qbman_result *dq)
2381 {
2382         return (const struct qbman_fd *)&dq->dq.fd[0];
2383 }
2384
2385 /**************************************/
2386 /* Parsing state-change notifications */
2387 /**************************************/
2388 uint8_t qbman_result_SCN_state(const struct qbman_result *scn)
2389 {
2390         return scn->scn.state;
2391 }
2392
2393 uint32_t qbman_result_SCN_rid(const struct qbman_result *scn)
2394 {
2395         return scn->scn.rid_tok;
2396 }
2397
2398 uint64_t qbman_result_SCN_ctx(const struct qbman_result *scn)
2399 {
2400         return scn->scn.ctx;
2401 }
2402
2403 /*****************/
2404 /* Parsing BPSCN */
2405 /*****************/
2406 uint16_t qbman_result_bpscn_bpid(const struct qbman_result *scn)
2407 {
2408         return (uint16_t)qbman_result_SCN_rid(scn) & 0x3FFF;
2409 }
2410
2411 int qbman_result_bpscn_has_free_bufs(const struct qbman_result *scn)
2412 {
2413         return !(int)(qbman_result_SCN_state(scn) & 0x1);
2414 }
2415
2416 int qbman_result_bpscn_is_depleted(const struct qbman_result *scn)
2417 {
2418         return (int)(qbman_result_SCN_state(scn) & 0x2);
2419 }
2420
2421 int qbman_result_bpscn_is_surplus(const struct qbman_result *scn)
2422 {
2423         return (int)(qbman_result_SCN_state(scn) & 0x4);
2424 }
2425
2426 uint64_t qbman_result_bpscn_ctx(const struct qbman_result *scn)
2427 {
2428         return qbman_result_SCN_ctx(scn);
2429 }
2430
2431 /*****************/
2432 /* Parsing CGCU  */
2433 /*****************/
2434 uint16_t qbman_result_cgcu_cgid(const struct qbman_result *scn)
2435 {
2436         return (uint16_t)qbman_result_SCN_rid(scn) & 0xFFFF;
2437 }
2438
2439 uint64_t qbman_result_cgcu_icnt(const struct qbman_result *scn)
2440 {
2441         return qbman_result_SCN_ctx(scn);
2442 }
2443
2444 /********************/
2445 /* Parsing EQ RESP  */
2446 /********************/
2447 struct qbman_fd *qbman_result_eqresp_fd(struct qbman_result *eqresp)
2448 {
2449         return (struct qbman_fd *)&eqresp->eq_resp.fd[0];
2450 }
2451
2452 void qbman_result_eqresp_set_rspid(struct qbman_result *eqresp, uint8_t val)
2453 {
2454         eqresp->eq_resp.rspid = val;
2455 }
2456
2457 uint8_t qbman_result_eqresp_rspid(struct qbman_result *eqresp)
2458 {
2459         return eqresp->eq_resp.rspid;
2460 }
2461
2462 uint8_t qbman_result_eqresp_rc(struct qbman_result *eqresp)
2463 {
2464         if (eqresp->eq_resp.rc == 0xE)
2465                 return 0;
2466         else
2467                 return -1;
2468 }
2469
2470 /******************/
2471 /* Buffer release */
2472 /******************/
2473 #define QB_BR_RC_VALID_SHIFT  5
2474 #define QB_BR_RCDI_SHIFT      6
2475
2476 void qbman_release_desc_clear(struct qbman_release_desc *d)
2477 {
2478         memset(d, 0, sizeof(*d));
2479         d->br.verb = 1 << QB_BR_RC_VALID_SHIFT;
2480 }
2481
2482 void qbman_release_desc_set_bpid(struct qbman_release_desc *d, uint16_t bpid)
2483 {
2484         d->br.bpid = bpid;
2485 }
2486
2487 void qbman_release_desc_set_rcdi(struct qbman_release_desc *d, int enable)
2488 {
2489         if (enable)
2490                 d->br.verb |= 1 << QB_BR_RCDI_SHIFT;
2491         else
2492                 d->br.verb &= ~(1 << QB_BR_RCDI_SHIFT);
2493 }
2494
2495 #define RAR_IDX(rar)     ((rar) & 0x7)
2496 #define RAR_VB(rar)      ((rar) & 0x80)
2497 #define RAR_SUCCESS(rar) ((rar) & 0x100)
2498
2499 static int qbman_swp_release_direct(struct qbman_swp *s,
2500                                     const struct qbman_release_desc *d,
2501                                     const uint64_t *buffers,
2502                                     unsigned int num_buffers)
2503 {
2504         uint32_t *p;
2505         const uint32_t *cl = qb_cl(d);
2506         uint32_t rar = qbman_cinh_read(&s->sys, QBMAN_CINH_SWP_RAR);
2507
2508         pr_debug("RAR=%08x\n", rar);
2509         if (!RAR_SUCCESS(rar))
2510                 return -EBUSY;
2511
2512         QBMAN_BUG_ON(!num_buffers || (num_buffers > 7));
2513
2514         /* Start the release command */
2515         p = qbman_cena_write_start_wo_shadow(&s->sys,
2516                                      QBMAN_CENA_SWP_RCR(RAR_IDX(rar)));
2517
2518         /* Copy the caller's buffer pointers to the command */
2519         u64_to_le32_copy(&p[2], buffers, num_buffers);
2520
2521         /* Set the verb byte, have to substitute in the valid-bit and the
2522          * number of buffers.
2523          */
2524         lwsync();
2525         p[0] = cl[0] | RAR_VB(rar) | num_buffers;
2526         qbman_cena_write_complete_wo_shadow(&s->sys,
2527                                     QBMAN_CENA_SWP_RCR(RAR_IDX(rar)));
2528
2529         return 0;
2530 }
2531
2532 static int qbman_swp_release_cinh_direct(struct qbman_swp *s,
2533                                     const struct qbman_release_desc *d,
2534                                     const uint64_t *buffers,
2535                                     unsigned int num_buffers)
2536 {
2537         uint32_t *p;
2538         const uint32_t *cl = qb_cl(d);
2539         uint32_t rar = qbman_cinh_read(&s->sys, QBMAN_CINH_SWP_RAR);
2540
2541         pr_debug("RAR=%08x\n", rar);
2542         if (!RAR_SUCCESS(rar))
2543                 return -EBUSY;
2544
2545         QBMAN_BUG_ON(!num_buffers || (num_buffers > 7));
2546
2547         /* Start the release command */
2548         p = qbman_cinh_write_start_wo_shadow(&s->sys,
2549                                      QBMAN_CENA_SWP_RCR(RAR_IDX(rar)));
2550
2551         /* Copy the caller's buffer pointers to the command */
2552         memcpy_byte_by_byte(&p[2], buffers, num_buffers * sizeof(uint64_t));
2553
2554         /* Set the verb byte, have to substitute in the valid-bit and the
2555          * number of buffers.
2556          */
2557         lwsync();
2558         p[0] = cl[0] | RAR_VB(rar) | num_buffers;
2559
2560         return 0;
2561 }
2562
2563 static int qbman_swp_release_mem_back(struct qbman_swp *s,
2564                                       const struct qbman_release_desc *d,
2565                                       const uint64_t *buffers,
2566                                       unsigned int num_buffers)
2567 {
2568         uint32_t *p;
2569         const uint32_t *cl = qb_cl(d);
2570         uint32_t rar = qbman_cinh_read(&s->sys, QBMAN_CINH_SWP_RAR);
2571
2572         pr_debug("RAR=%08x\n", rar);
2573         if (!RAR_SUCCESS(rar))
2574                 return -EBUSY;
2575
2576         QBMAN_BUG_ON(!num_buffers || (num_buffers > 7));
2577
2578         /* Start the release command */
2579         p = qbman_cena_write_start_wo_shadow(&s->sys,
2580                 QBMAN_CENA_SWP_RCR_MEM(RAR_IDX(rar)));
2581
2582         /* Copy the caller's buffer pointers to the command */
2583         u64_to_le32_copy(&p[2], buffers, num_buffers);
2584
2585         /* Set the verb byte, have to substitute in the valid-bit and the
2586          * number of buffers.
2587          */
2588         p[0] = cl[0] | RAR_VB(rar) | num_buffers;
2589         lwsync();
2590         qbman_cinh_write(&s->sys, QBMAN_CINH_SWP_RCR_AM_RT +
2591                 RAR_IDX(rar) * 4, QMAN_RT_MODE);
2592
2593         return 0;
2594 }
2595
2596 int qbman_swp_release(struct qbman_swp *s,
2597                              const struct qbman_release_desc *d,
2598                              const uint64_t *buffers,
2599                              unsigned int num_buffers)
2600 {
2601         if (!s->stash_off)
2602                 return qbman_swp_release_ptr(s, d, buffers, num_buffers);
2603         else
2604                 return qbman_swp_release_cinh_direct(s, d, buffers,
2605                                                 num_buffers);
2606 }
2607
2608 /*******************/
2609 /* Buffer acquires */
2610 /*******************/
2611 struct qbman_acquire_desc {
2612         uint8_t verb;
2613         uint8_t reserved;
2614         uint16_t bpid;
2615         uint8_t num;
2616         uint8_t reserved2[59];
2617 };
2618
2619 struct qbman_acquire_rslt {
2620         uint8_t verb;
2621         uint8_t rslt;
2622         uint16_t reserved;
2623         uint8_t num;
2624         uint8_t reserved2[3];
2625         uint64_t buf[7];
2626 };
2627
2628 static int qbman_swp_acquire_direct(struct qbman_swp *s, uint16_t bpid,
2629                                 uint64_t *buffers, unsigned int num_buffers)
2630 {
2631         struct qbman_acquire_desc *p;
2632         struct qbman_acquire_rslt *r;
2633
2634         if (!num_buffers || (num_buffers > 7))
2635                 return -EINVAL;
2636
2637         /* Start the management command */
2638         p = qbman_swp_mc_start(s);
2639
2640         if (!p)
2641                 return -EBUSY;
2642
2643         /* Encode the caller-provided attributes */
2644         p->bpid = bpid;
2645         p->num = num_buffers;
2646
2647         /* Complete the management command */
2648         r = qbman_swp_mc_complete(s, p, QBMAN_MC_ACQUIRE);
2649         if (!r) {
2650                 pr_err("qbman: acquire from BPID %d failed, no response\n",
2651                        bpid);
2652                 return -EIO;
2653         }
2654
2655         /* Decode the outcome */
2656         QBMAN_BUG_ON((r->verb & QBMAN_RESPONSE_VERB_MASK) != QBMAN_MC_ACQUIRE);
2657
2658         /* Determine success or failure */
2659         if (r->rslt != QBMAN_MC_RSLT_OK) {
2660                 pr_err("Acquire buffers from BPID 0x%x failed, code=0x%02x\n",
2661                        bpid, r->rslt);
2662                 return -EIO;
2663         }
2664
2665         QBMAN_BUG_ON(r->num > num_buffers);
2666
2667         /* Copy the acquired buffers to the caller's array */
2668         u64_from_le32_copy(buffers, &r->buf[0], r->num);
2669
2670         return (int)r->num;
2671 }
2672
2673 static int qbman_swp_acquire_cinh_direct(struct qbman_swp *s, uint16_t bpid,
2674                         uint64_t *buffers, unsigned int num_buffers)
2675 {
2676         struct qbman_acquire_desc *p;
2677         struct qbman_acquire_rslt *r;
2678
2679         if (!num_buffers || (num_buffers > 7))
2680                 return -EINVAL;
2681
2682         /* Start the management command */
2683         p = qbman_swp_mc_start(s);
2684
2685         if (!p)
2686                 return -EBUSY;
2687
2688         /* Encode the caller-provided attributes */
2689         p->bpid = bpid;
2690         p->num = num_buffers;
2691
2692         /* Complete the management command */
2693         r = qbman_swp_mc_complete_cinh(s, p, QBMAN_MC_ACQUIRE);
2694         if (!r) {
2695                 pr_err("qbman: acquire from BPID %d failed, no response\n",
2696                        bpid);
2697                 return -EIO;
2698         }
2699
2700         /* Decode the outcome */
2701         QBMAN_BUG_ON((r->verb & QBMAN_RESPONSE_VERB_MASK) != QBMAN_MC_ACQUIRE);
2702
2703         /* Determine success or failure */
2704         if (r->rslt != QBMAN_MC_RSLT_OK) {
2705                 pr_err("Acquire buffers from BPID 0x%x failed, code=0x%02x\n",
2706                        bpid, r->rslt);
2707                 return -EIO;
2708         }
2709
2710         QBMAN_BUG_ON(r->num > num_buffers);
2711
2712         /* Copy the acquired buffers to the caller's array */
2713         u64_from_le32_copy(buffers, &r->buf[0], r->num);
2714
2715         return (int)r->num;
2716 }
2717
2718 int qbman_swp_acquire(struct qbman_swp *s, uint16_t bpid, uint64_t *buffers,
2719                       unsigned int num_buffers)
2720 {
2721         if (!s->stash_off)
2722                 return qbman_swp_acquire_direct(s, bpid, buffers, num_buffers);
2723         else
2724                 return qbman_swp_acquire_cinh_direct(s, bpid, buffers,
2725                                         num_buffers);
2726 }
2727
2728 /*****************/
2729 /* FQ management */
2730 /*****************/
2731 struct qbman_alt_fq_state_desc {
2732         uint8_t verb;
2733         uint8_t reserved[3];
2734         uint32_t fqid;
2735         uint8_t reserved2[56];
2736 };
2737
2738 struct qbman_alt_fq_state_rslt {
2739         uint8_t verb;
2740         uint8_t rslt;
2741         uint8_t reserved[62];
2742 };
2743
2744 #define ALT_FQ_FQID_MASK 0x00FFFFFF
2745
2746 static int qbman_swp_alt_fq_state(struct qbman_swp *s, uint32_t fqid,
2747                                   uint8_t alt_fq_verb)
2748 {
2749         struct qbman_alt_fq_state_desc *p;
2750         struct qbman_alt_fq_state_rslt *r;
2751
2752         /* Start the management command */
2753         p = qbman_swp_mc_start(s);
2754         if (!p)
2755                 return -EBUSY;
2756
2757         p->fqid = fqid & ALT_FQ_FQID_MASK;
2758
2759         /* Complete the management command */
2760         r = qbman_swp_mc_complete(s, p, alt_fq_verb);
2761         if (!r) {
2762                 pr_err("qbman: mgmt cmd failed, no response (verb=0x%x)\n",
2763                        alt_fq_verb);
2764                 return -EIO;
2765         }
2766
2767         /* Decode the outcome */
2768         QBMAN_BUG_ON((r->verb & QBMAN_RESPONSE_VERB_MASK) != alt_fq_verb);
2769
2770         /* Determine success or failure */
2771         if (r->rslt != QBMAN_MC_RSLT_OK) {
2772                 pr_err("ALT FQID %d failed: verb = 0x%08x, code = 0x%02x\n",
2773                        fqid, alt_fq_verb, r->rslt);
2774                 return -EIO;
2775         }
2776
2777         return 0;
2778 }
2779
2780 int qbman_swp_fq_schedule(struct qbman_swp *s, uint32_t fqid)
2781 {
2782         return qbman_swp_alt_fq_state(s, fqid, QBMAN_FQ_SCHEDULE);
2783 }
2784
2785 int qbman_swp_fq_force(struct qbman_swp *s, uint32_t fqid)
2786 {
2787         return qbman_swp_alt_fq_state(s, fqid, QBMAN_FQ_FORCE);
2788 }
2789
2790 int qbman_swp_fq_xon(struct qbman_swp *s, uint32_t fqid)
2791 {
2792         return qbman_swp_alt_fq_state(s, fqid, QBMAN_FQ_XON);
2793 }
2794
2795 int qbman_swp_fq_xoff(struct qbman_swp *s, uint32_t fqid)
2796 {
2797         return qbman_swp_alt_fq_state(s, fqid, QBMAN_FQ_XOFF);
2798 }
2799
2800 /**********************/
2801 /* Channel management */
2802 /**********************/
2803
2804 struct qbman_cdan_ctrl_desc {
2805         uint8_t verb;
2806         uint8_t reserved;
2807         uint16_t ch;
2808         uint8_t we;
2809         uint8_t ctrl;
2810         uint16_t reserved2;
2811         uint64_t cdan_ctx;
2812         uint8_t reserved3[48];
2813
2814 };
2815
2816 struct qbman_cdan_ctrl_rslt {
2817         uint8_t verb;
2818         uint8_t rslt;
2819         uint16_t ch;
2820         uint8_t reserved[60];
2821 };
2822
2823 /* Hide "ICD" for now as we don't use it, don't set it, and don't test it, so it
2824  * would be irresponsible to expose it.
2825  */
2826 #define CODE_CDAN_WE_EN    0x1
2827 #define CODE_CDAN_WE_CTX   0x4
2828
2829 static int qbman_swp_CDAN_set(struct qbman_swp *s, uint16_t channelid,
2830                               uint8_t we_mask, uint8_t cdan_en,
2831                               uint64_t ctx)
2832 {
2833         struct qbman_cdan_ctrl_desc *p;
2834         struct qbman_cdan_ctrl_rslt *r;
2835
2836         /* Start the management command */
2837         p = qbman_swp_mc_start(s);
2838         if (!p)
2839                 return -EBUSY;
2840
2841         /* Encode the caller-provided attributes */
2842         p->ch = channelid;
2843         p->we = we_mask;
2844         if (cdan_en)
2845                 p->ctrl = 1;
2846         else
2847                 p->ctrl = 0;
2848         p->cdan_ctx = ctx;
2849
2850         /* Complete the management command */
2851         r = qbman_swp_mc_complete(s, p, QBMAN_WQCHAN_CONFIGURE);
2852         if (!r) {
2853                 pr_err("qbman: wqchan config failed, no response\n");
2854                 return -EIO;
2855         }
2856
2857         /* Decode the outcome */
2858         QBMAN_BUG_ON((r->verb & QBMAN_RESPONSE_VERB_MASK)
2859                      != QBMAN_WQCHAN_CONFIGURE);
2860
2861         /* Determine success or failure */
2862         if (r->rslt != QBMAN_MC_RSLT_OK) {
2863                 pr_err("CDAN cQID %d failed: code = 0x%02x\n",
2864                        channelid, r->rslt);
2865                 return -EIO;
2866         }
2867
2868         return 0;
2869 }
2870
2871 int qbman_swp_CDAN_set_context(struct qbman_swp *s, uint16_t channelid,
2872                                uint64_t ctx)
2873 {
2874         return qbman_swp_CDAN_set(s, channelid,
2875                                   CODE_CDAN_WE_CTX,
2876                                   0, ctx);
2877 }
2878
2879 int qbman_swp_CDAN_enable(struct qbman_swp *s, uint16_t channelid)
2880 {
2881         return qbman_swp_CDAN_set(s, channelid,
2882                                   CODE_CDAN_WE_EN,
2883                                   1, 0);
2884 }
2885
2886 int qbman_swp_CDAN_disable(struct qbman_swp *s, uint16_t channelid)
2887 {
2888         return qbman_swp_CDAN_set(s, channelid,
2889                                   CODE_CDAN_WE_EN,
2890                                   0, 0);
2891 }
2892
2893 int qbman_swp_CDAN_set_context_enable(struct qbman_swp *s, uint16_t channelid,
2894                                       uint64_t ctx)
2895 {
2896         return qbman_swp_CDAN_set(s, channelid,
2897                                   CODE_CDAN_WE_EN | CODE_CDAN_WE_CTX,
2898                                   1, ctx);
2899 }
2900
2901 uint8_t qbman_get_dqrr_idx(const struct qbman_result *dqrr)
2902 {
2903         return QBMAN_IDX_FROM_DQRR(dqrr);
2904 }
2905
2906 struct qbman_result *qbman_get_dqrr_from_idx(struct qbman_swp *s, uint8_t idx)
2907 {
2908         struct qbman_result *dq;
2909
2910         dq = qbman_cena_read(&s->sys, QBMAN_CENA_SWP_DQRR(idx));
2911         return dq;
2912 }