2 * This file is provided under a dual BSD/GPLv2 license. When using or
3 * redistributing this file, you may do so under either license.
7 * Copyright 2010-2016 Freescale Semiconductor Inc.
10 * Redistribution and use in source and binary forms, with or without
11 * modification, are permitted provided that the following conditions are met:
12 * * Redistributions of source code must retain the above copyright
13 * notice, this list of conditions and the following disclaimer.
14 * * Redistributions in binary form must reproduce the above copyright
15 * notice, this list of conditions and the following disclaimer in the
16 * documentation and/or other materials provided with the distribution.
17 * * Neither the name of the above-listed copyright holders nor the
18 * names of any contributors may be used to endorse or promote products
19 * derived from this software without specific prior written permission.
23 * ALTERNATIVELY, this software may be distributed under the terms of the
24 * GNU General Public License ("GPL") as published by the Free Software
25 * Foundation, either version 2 of that License or (at your option) any
28 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
29 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
30 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
31 * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDERS OR CONTRIBUTORS BE
32 * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
33 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
34 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
35 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
36 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
37 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
38 * POSSIBILITY OF SUCH DAMAGE.
44 #include "bman_priv.h"
46 /* Cache-inhibited register offsets */
47 #define BM_REG_RCR_PI_CINH 0x3000
48 #define BM_REG_RCR_CI_CINH 0x3100
49 #define BM_REG_RCR_ITR 0x3200
50 #define BM_REG_CFG 0x3300
51 #define BM_REG_SCN(n) (0x3400 + ((n) << 6))
52 #define BM_REG_ISR 0x3e00
53 #define BM_REG_IIR 0x3ec0
55 /* Cache-enabled register offsets */
56 #define BM_CL_CR 0x0000
57 #define BM_CL_RR0 0x0100
58 #define BM_CL_RR1 0x0140
59 #define BM_CL_RCR 0x1000
60 #define BM_CL_RCR_PI_CENA 0x3000
61 #define BM_CL_RCR_CI_CENA 0x3100
63 /* BTW, the drivers (and h/w programming model) already obtain the required
64 * synchronisation for portal accesses via lwsync(), hwsync(), and
65 * data-dependencies. Use of barrier()s or other order-preserving primitives
66 * simply degrade performance. Hence the use of the __raw_*() interfaces, which
67 * simply ensure that the compiler treats the portal registers as volatile (ie.
71 /* Cache-inhibited register access. */
72 #define __bm_in(bm, o) be32_to_cpu(__raw_readl((bm)->ci + (o)))
73 #define __bm_out(bm, o, val) __raw_writel(cpu_to_be32(val), \
75 #define bm_in(reg) __bm_in(&portal->addr, BM_REG_##reg)
76 #define bm_out(reg, val) __bm_out(&portal->addr, BM_REG_##reg, val)
78 /* Cache-enabled (index) register access */
79 #define __bm_cl_touch_ro(bm, o) dcbt_ro((bm)->ce + (o))
80 #define __bm_cl_touch_rw(bm, o) dcbt_rw((bm)->ce + (o))
81 #define __bm_cl_in(bm, o) be32_to_cpu(__raw_readl((bm)->ce + (o)))
82 #define __bm_cl_out(bm, o, val) \
84 u32 *__tmpclout = (bm)->ce + (o); \
85 __raw_writel(cpu_to_be32(val), __tmpclout); \
88 #define __bm_cl_invalidate(bm, o) dccivac((bm)->ce + (o))
89 #define bm_cl_touch_ro(reg) __bm_cl_touch_ro(&portal->addr, BM_CL_##reg##_CENA)
90 #define bm_cl_touch_rw(reg) __bm_cl_touch_rw(&portal->addr, BM_CL_##reg##_CENA)
91 #define bm_cl_in(reg) __bm_cl_in(&portal->addr, BM_CL_##reg##_CENA)
92 #define bm_cl_out(reg, val) __bm_cl_out(&portal->addr, BM_CL_##reg##_CENA, val)
93 #define bm_cl_invalidate(reg)\
94 __bm_cl_invalidate(&portal->addr, BM_CL_##reg##_CENA)
96 /* Cyclic helper for rings. FIXME: once we are able to do fine-grain perf
97 * analysis, look at using the "extra" bit in the ring index registers to avoid
100 static inline u8 bm_cyc_diff(u8 ringsize, u8 first, u8 last)
102 /* 'first' is included, 'last' is excluded */
105 return ringsize + last - first;
110 * pmode == production mode
111 * cmode == consumption mode,
112 * Enum values use 3 letter codes. First letter matches the portal mode,
113 * remaining two letters indicate;
114 * ci == cache-inhibited portal register
115 * ce == cache-enabled portal register
116 * vb == in-band valid-bit (cache-enabled)
118 enum bm_rcr_pmode { /* matches BCSP_CFG::RPM */
119 bm_rcr_pci = 0, /* PI index, cache-inhibited */
120 bm_rcr_pce = 1, /* PI index, cache-enabled */
121 bm_rcr_pvb = 2 /* valid-bit */
124 enum bm_rcr_cmode { /* s/w-only */
125 bm_rcr_cci, /* CI index, cache-inhibited */
126 bm_rcr_cce /* CI index, cache-enabled */
129 /* --- Portal structures --- */
131 #define BM_RCR_SIZE 8
134 struct bm_rcr_entry *ring, *cursor;
135 u8 ci, available, ithresh, vbit;
136 #ifdef RTE_LIBRTE_DPAA_HWDEBUG
138 enum bm_rcr_pmode pmode;
139 enum bm_rcr_cmode cmode;
144 struct bm_mc_command *cr;
145 struct bm_mc_result *rr;
147 #ifdef RTE_LIBRTE_DPAA_HWDEBUG
149 /* Can only be _mc_start()ed */
151 /* Can only be _mc_commit()ed or _mc_abort()ed */
153 /* Can only be _mc_retry()ed */
160 void __iomem *ce; /* cache-enabled */
161 void __iomem *ci; /* cache-inhibited */
168 struct bm_portal_config config;
169 } ____cacheline_aligned;
171 /* Bit-wise logic to wrap a ring pointer by clearing the "carry bit" */
172 #define RCR_CARRYCLEAR(p) \
173 (void *)((unsigned long)(p) & (~(unsigned long)(BM_RCR_SIZE << 6)))
175 /* Bit-wise logic to convert a ring pointer to a ring index */
176 static inline u8 RCR_PTR2IDX(struct bm_rcr_entry *e)
178 return ((uintptr_t)e >> 6) & (BM_RCR_SIZE - 1);
181 /* Increment the 'cursor' ring pointer, taking 'vbit' into account */
182 static inline void RCR_INC(struct bm_rcr *rcr)
184 /* NB: this is odd-looking, but experiments show that it generates
185 * fast code with essentially no branching overheads. We increment to
186 * the next RCR pointer and handle overflow and 'vbit'.
188 struct bm_rcr_entry *partial = rcr->cursor + 1;
190 rcr->cursor = RCR_CARRYCLEAR(partial);
191 if (partial != rcr->cursor)
192 rcr->vbit ^= BM_RCR_VERB_VBIT;
195 static inline int bm_rcr_init(struct bm_portal *portal, enum bm_rcr_pmode pmode,
196 __maybe_unused enum bm_rcr_cmode cmode)
198 /* This use of 'register', as well as all other occurrences, is because
199 * it has been observed to generate much faster code with gcc than is
200 * otherwise the case.
202 register struct bm_rcr *rcr = &portal->rcr;
206 rcr->ring = portal->addr.ce + BM_CL_RCR;
207 rcr->ci = bm_in(RCR_CI_CINH) & (BM_RCR_SIZE - 1);
209 pi = bm_in(RCR_PI_CINH) & (BM_RCR_SIZE - 1);
210 rcr->cursor = rcr->ring + pi;
211 rcr->vbit = (bm_in(RCR_PI_CINH) & BM_RCR_SIZE) ? BM_RCR_VERB_VBIT : 0;
212 rcr->available = BM_RCR_SIZE - 1
213 - bm_cyc_diff(BM_RCR_SIZE, rcr->ci, pi);
214 rcr->ithresh = bm_in(RCR_ITR);
215 #ifdef RTE_LIBRTE_DPAA_HWDEBUG
220 cfg = (bm_in(CFG) & 0xffffffe0) | (pmode & 0x3); /* BCSP_CFG::RPM */
225 static inline void bm_rcr_finish(struct bm_portal *portal)
227 register struct bm_rcr *rcr = &portal->rcr;
228 u8 pi = bm_in(RCR_PI_CINH) & (BM_RCR_SIZE - 1);
229 u8 ci = bm_in(RCR_CI_CINH) & (BM_RCR_SIZE - 1);
231 #ifdef RTE_LIBRTE_DPAA_HWDEBUG
232 DPAA_ASSERT(!rcr->busy);
234 if (pi != RCR_PTR2IDX(rcr->cursor))
235 pr_crit("losing uncommitted RCR entries\n");
237 pr_crit("missing existing RCR completions\n");
238 if (rcr->ci != RCR_PTR2IDX(rcr->cursor))
239 pr_crit("RCR destroyed unquiesced\n");
242 static inline struct bm_rcr_entry *bm_rcr_start(struct bm_portal *portal)
244 register struct bm_rcr *rcr = &portal->rcr;
246 #ifdef RTE_LIBRTE_DPAA_HWDEBUG
247 DPAA_ASSERT(!rcr->busy);
251 #ifdef RTE_LIBRTE_DPAA_HWDEBUG
254 dcbz_64(rcr->cursor);
258 static inline void bm_rcr_abort(struct bm_portal *portal)
260 __maybe_unused register struct bm_rcr *rcr = &portal->rcr;
262 #ifdef RTE_LIBRTE_DPAA_HWDEBUG
263 DPAA_ASSERT(rcr->busy);
268 static inline struct bm_rcr_entry *bm_rcr_pend_and_next(
269 struct bm_portal *portal, u8 myverb)
271 register struct bm_rcr *rcr = &portal->rcr;
273 #ifdef RTE_LIBRTE_DPAA_HWDEBUG
274 DPAA_ASSERT(rcr->busy);
275 DPAA_ASSERT(rcr->pmode != bm_rcr_pvb);
277 if (rcr->available == 1)
279 rcr->cursor->__dont_write_directly__verb = myverb | rcr->vbit;
280 dcbf_64(rcr->cursor);
283 dcbz_64(rcr->cursor);
287 static inline void bm_rcr_pci_commit(struct bm_portal *portal, u8 myverb)
289 register struct bm_rcr *rcr = &portal->rcr;
291 #ifdef RTE_LIBRTE_DPAA_HWDEBUG
292 DPAA_ASSERT(rcr->busy);
293 DPAA_ASSERT(rcr->pmode == bm_rcr_pci);
295 rcr->cursor->__dont_write_directly__verb = myverb | rcr->vbit;
299 bm_out(RCR_PI_CINH, RCR_PTR2IDX(rcr->cursor));
300 #ifdef RTE_LIBRTE_DPAA_HWDEBUG
305 static inline void bm_rcr_pce_prefetch(struct bm_portal *portal)
307 __maybe_unused register struct bm_rcr *rcr = &portal->rcr;
309 #ifdef RTE_LIBRTE_DPAA_HWDEBUG
310 DPAA_ASSERT(rcr->pmode == bm_rcr_pce);
312 bm_cl_invalidate(RCR_PI);
313 bm_cl_touch_rw(RCR_PI);
316 static inline void bm_rcr_pce_commit(struct bm_portal *portal, u8 myverb)
318 register struct bm_rcr *rcr = &portal->rcr;
320 #ifdef RTE_LIBRTE_DPAA_HWDEBUG
321 DPAA_ASSERT(rcr->busy);
322 DPAA_ASSERT(rcr->pmode == bm_rcr_pce);
324 rcr->cursor->__dont_write_directly__verb = myverb | rcr->vbit;
328 bm_cl_out(RCR_PI, RCR_PTR2IDX(rcr->cursor));
329 #ifdef RTE_LIBRTE_DPAA_HWDEBUG
334 static inline void bm_rcr_pvb_commit(struct bm_portal *portal, u8 myverb)
336 register struct bm_rcr *rcr = &portal->rcr;
337 struct bm_rcr_entry *rcursor;
339 #ifdef RTE_LIBRTE_DPAA_HWDEBUG
340 DPAA_ASSERT(rcr->busy);
341 DPAA_ASSERT(rcr->pmode == bm_rcr_pvb);
344 rcursor = rcr->cursor;
345 rcursor->__dont_write_directly__verb = myverb | rcr->vbit;
349 #ifdef RTE_LIBRTE_DPAA_HWDEBUG
354 static inline u8 bm_rcr_cci_update(struct bm_portal *portal)
356 register struct bm_rcr *rcr = &portal->rcr;
357 u8 diff, old_ci = rcr->ci;
359 #ifdef RTE_LIBRTE_DPAA_HWDEBUG
360 DPAA_ASSERT(rcr->cmode == bm_rcr_cci);
362 rcr->ci = bm_in(RCR_CI_CINH) & (BM_RCR_SIZE - 1);
363 diff = bm_cyc_diff(BM_RCR_SIZE, old_ci, rcr->ci);
364 rcr->available += diff;
368 static inline void bm_rcr_cce_prefetch(struct bm_portal *portal)
370 __maybe_unused register struct bm_rcr *rcr = &portal->rcr;
372 #ifdef RTE_LIBRTE_DPAA_HWDEBUG
373 DPAA_ASSERT(rcr->cmode == bm_rcr_cce);
375 bm_cl_touch_ro(RCR_CI);
378 static inline u8 bm_rcr_cce_update(struct bm_portal *portal)
380 register struct bm_rcr *rcr = &portal->rcr;
381 u8 diff, old_ci = rcr->ci;
383 #ifdef RTE_LIBRTE_DPAA_HWDEBUG
384 DPAA_ASSERT(rcr->cmode == bm_rcr_cce);
386 rcr->ci = bm_cl_in(RCR_CI) & (BM_RCR_SIZE - 1);
387 bm_cl_invalidate(RCR_CI);
388 diff = bm_cyc_diff(BM_RCR_SIZE, old_ci, rcr->ci);
389 rcr->available += diff;
393 static inline u8 bm_rcr_get_ithresh(struct bm_portal *portal)
395 register struct bm_rcr *rcr = &portal->rcr;
400 static inline void bm_rcr_set_ithresh(struct bm_portal *portal, u8 ithresh)
402 register struct bm_rcr *rcr = &portal->rcr;
404 rcr->ithresh = ithresh;
405 bm_out(RCR_ITR, ithresh);
408 static inline u8 bm_rcr_get_avail(struct bm_portal *portal)
410 register struct bm_rcr *rcr = &portal->rcr;
412 return rcr->available;
415 static inline u8 bm_rcr_get_fill(struct bm_portal *portal)
417 register struct bm_rcr *rcr = &portal->rcr;
419 return BM_RCR_SIZE - 1 - rcr->available;
422 /* --- Management command API --- */
424 static inline int bm_mc_init(struct bm_portal *portal)
426 register struct bm_mc *mc = &portal->mc;
428 mc->cr = portal->addr.ce + BM_CL_CR;
429 mc->rr = portal->addr.ce + BM_CL_RR0;
430 mc->rridx = (__raw_readb(&mc->cr->__dont_write_directly__verb) &
431 BM_MCC_VERB_VBIT) ? 0 : 1;
432 mc->vbit = mc->rridx ? BM_MCC_VERB_VBIT : 0;
433 #ifdef RTE_LIBRTE_DPAA_HWDEBUG
439 static inline void bm_mc_finish(struct bm_portal *portal)
441 __maybe_unused register struct bm_mc *mc = &portal->mc;
443 #ifdef RTE_LIBRTE_DPAA_HWDEBUG
444 DPAA_ASSERT(mc->state == mc_idle);
445 if (mc->state != mc_idle)
446 pr_crit("Losing incomplete MC command\n");
450 static inline struct bm_mc_command *bm_mc_start(struct bm_portal *portal)
452 register struct bm_mc *mc = &portal->mc;
454 #ifdef RTE_LIBRTE_DPAA_HWDEBUG
455 DPAA_ASSERT(mc->state == mc_idle);
462 static inline void bm_mc_abort(struct bm_portal *portal)
464 __maybe_unused register struct bm_mc *mc = &portal->mc;
466 #ifdef RTE_LIBRTE_DPAA_HWDEBUG
467 DPAA_ASSERT(mc->state == mc_user);
472 static inline void bm_mc_commit(struct bm_portal *portal, u8 myverb)
474 register struct bm_mc *mc = &portal->mc;
475 struct bm_mc_result *rr = mc->rr + mc->rridx;
477 #ifdef RTE_LIBRTE_DPAA_HWDEBUG
478 DPAA_ASSERT(mc->state == mc_user);
481 mc->cr->__dont_write_directly__verb = myverb | mc->vbit;
484 #ifdef RTE_LIBRTE_DPAA_HWDEBUG
489 static inline struct bm_mc_result *bm_mc_result(struct bm_portal *portal)
491 register struct bm_mc *mc = &portal->mc;
492 struct bm_mc_result *rr = mc->rr + mc->rridx;
494 #ifdef RTE_LIBRTE_DPAA_HWDEBUG
495 DPAA_ASSERT(mc->state == mc_hw);
497 /* The inactive response register's verb byte always returns zero until
498 * its command is submitted and completed. This includes the valid-bit,
499 * in case you were wondering.
501 if (!__raw_readb(&rr->verb)) {
506 mc->vbit ^= BM_MCC_VERB_VBIT;
507 #ifdef RTE_LIBRTE_DPAA_HWDEBUG
513 #define SCN_REG(bpid) BM_REG_SCN((bpid) / 32)
514 #define SCN_BIT(bpid) (0x80000000 >> (bpid & 31))
515 static inline void bm_isr_bscn_mask(struct bm_portal *portal, u8 bpid,
520 DPAA_ASSERT(bpid < bman_pool_max);
521 /* REG_SCN for bpid=0..31, REG_SCN+4 for bpid=32..63 */
522 val = __bm_in(&portal->addr, SCN_REG(bpid));
524 val |= SCN_BIT(bpid);
526 val &= ~SCN_BIT(bpid);
527 __bm_out(&portal->addr, SCN_REG(bpid), val);
530 static inline u32 __bm_isr_read(struct bm_portal *portal, enum bm_isr_reg n)
532 #if defined(RTE_ARCH_ARM64)
533 return __bm_in(&portal->addr, BM_REG_ISR + (n << 6));
535 return __bm_in(&portal->addr, BM_REG_ISR + (n << 2));
539 static inline void __bm_isr_write(struct bm_portal *portal, enum bm_isr_reg n,
542 #if defined(RTE_ARCH_ARM64)
543 __bm_out(&portal->addr, BM_REG_ISR + (n << 6), val);
545 __bm_out(&portal->addr, BM_REG_ISR + (n << 2), val);
549 /* Buffer Pool Cleanup */
550 static inline int bm_shutdown_pool(struct bm_portal *p, u32 bpid)
552 struct bm_mc_command *bm_cmd;
553 struct bm_mc_result *bm_res;
559 /* Acquire buffers until empty */
560 bm_cmd = bm_mc_start(p);
561 bm_cmd->acquire.bpid = bpid;
562 bm_mc_commit(p, BM_MCC_VERB_CMD_ACQUIRE | 1);
563 while (!(bm_res = bm_mc_result(p)))
565 if (!(bm_res->verb & BM_MCR_VERB_ACQUIRE_BUFCOUNT)) {
574 #endif /* __BMAN_H */