1/******************************************************************************
2
3 � 1995-2003, 2004, 2005-2011 Freescale Semiconductor, Inc.
4 All rights reserved.
5
6 This is proprietary source code of Freescale Semiconductor Inc.,
7 and its use is subject to the NetComm Device Drivers EULA.
8 The copyright notice above does not evidence any actual or intended
9 publication of such source code.
10
11 ALTERNATIVELY, redistribution and use in source and binary forms, with
12 or without modification, are permitted provided that the following
13 conditions are met:
14     * Redistributions of source code must retain the above copyright
15       notice, this list of conditions and the following disclaimer.
16     * Redistributions in binary form must reproduce the above copyright
17       notice, this list of conditions and the following disclaimer in the
18       documentation and/or other materials provided with the distribution.
19     * Neither the name of Freescale Semiconductor nor the
20       names of its contributors may be used to endorse or promote products
21       derived from this software without specific prior written permission.
22
23 THIS SOFTWARE IS PROVIDED BY Freescale Semiconductor ``AS IS'' AND ANY
24 EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
25 WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
26 DISCLAIMED. IN NO EVENT SHALL Freescale Semiconductor BE LIABLE FOR ANY
27 DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
28 (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
29 LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
30 ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
31 (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
32 SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
33 *
34
35 **************************************************************************/
36/******************************************************************************
37 @File          bman_low.c
38
39 @Description   BM low-level implementation
40*//***************************************************************************/
41#include <sys/cdefs.h>
42#include <sys/types.h>
43#include <machine/atomic.h>
44
45#include "std_ext.h"
46#include "core_ext.h"
47#include "xx_ext.h"
48#include "error_ext.h"
49
50#include "bman_private.h"
51
52
53/***************************/
54/* Portal register assists */
55/***************************/
56
57/* Cache-inhibited register offsets */
58#define REG_RCR_PI_CINH     0x0000
59#define REG_RCR_CI_CINH     0x0004
60#define REG_RCR_ITR         0x0008
61#define REG_CFG             0x0100
62#define REG_SCN(n)          (0x0200 + ((n) << 2))
63#define REG_ISR             0x0e00
64#define REG_IER             0x0e04
65#define REG_ISDR            0x0e08
66#define REG_IIR             0x0e0c
67
68/* Cache-enabled register offsets */
69#define CL_CR               0x0000
70#define CL_RR0              0x0100
71#define CL_RR1              0x0140
72#define CL_RCR              0x1000
73#define CL_RCR_PI_CENA      0x3000
74#define CL_RCR_CI_CENA      0x3100
75
76/* The h/w design requires mappings to be size-aligned so that "add"s can be
77 * reduced to "or"s. The primitives below do the same for s/w. */
78
79static __inline__ void *ptr_ADD(void *a, uintptr_t b)
80{
81    return (void *)((uintptr_t)a + b);
82}
83
84/* Bitwise-OR two pointers */
85static __inline__ void *ptr_OR(void *a, uintptr_t b)
86{
87    return (void *)((uintptr_t)a | b);
88}
89
90/* Cache-inhibited register access */
91static __inline__ uint32_t __bm_in(struct bm_addr *bm, uintptr_t offset)
92{
93    uint32_t    *tmp = (uint32_t *)ptr_ADD(bm->addr_ci, offset);
94    return GET_UINT32(*tmp);
95}
96static __inline__ void __bm_out(struct bm_addr *bm, uintptr_t offset, uint32_t val)
97{
98    uint32_t    *tmp = (uint32_t *)ptr_ADD(bm->addr_ci, offset);
99    WRITE_UINT32(*tmp, val);
100}
101#define bm_in(reg)        __bm_in(&portal->addr, REG_##reg)
102#define bm_out(reg, val)    __bm_out(&portal->addr, REG_##reg, val)
103
104/* Convert 'n' cachelines to a pointer value for bitwise OR */
105#define bm_cl(n)        (void *)((n) << 6)
106
107/* Cache-enabled (index) register access */
108static __inline__ void __bm_cl_touch_ro(struct bm_addr *bm, uintptr_t offset)
109{
110    dcbt_ro(ptr_ADD(bm->addr_ce, offset));
111}
112static __inline__ void __bm_cl_touch_rw(struct bm_addr *bm, uintptr_t offset)
113{
114    dcbt_rw(ptr_ADD(bm->addr_ce, offset));
115}
116static __inline__ uint32_t __bm_cl_in(struct bm_addr *bm, uintptr_t offset)
117{
118    uint32_t    *tmp = (uint32_t *)ptr_ADD(bm->addr_ce, offset);
119    return GET_UINT32(*tmp);
120}
121static __inline__ void __bm_cl_out(struct bm_addr *bm, uintptr_t offset, uint32_t val)
122{
123    uint32_t    *tmp = (uint32_t *)ptr_ADD(bm->addr_ce, offset);
124    WRITE_UINT32(*tmp, val);
125    dcbf(tmp);
126}
127static __inline__ void __bm_cl_invalidate(struct bm_addr *bm, uintptr_t offset)
128{
129    dcbi(ptr_ADD(bm->addr_ce, offset));
130}
131#define bm_cl_touch_ro(reg)    __bm_cl_touch_ro(&portal->addr, CL_##reg##_CENA)
132#define bm_cl_touch_rw(reg)    __bm_cl_touch_rw(&portal->addr, CL_##reg##_CENA)
133#define bm_cl_in(reg)        __bm_cl_in(&portal->addr, CL_##reg##_CENA)
134#define bm_cl_out(reg, val)    __bm_cl_out(&portal->addr, CL_##reg##_CENA, val)
135#define bm_cl_invalidate(reg) __bm_cl_invalidate(&portal->addr, CL_##reg##_CENA)
136
137/* Cyclic helper for rings. TODO: once we are able to do fine-grain perf
138 * analysis, look at using the "extra" bit in the ring index registers to avoid
139 * cyclic issues. */
140static __inline__ uint8_t cyc_diff(uint8_t ringsize, uint8_t first, uint8_t last)
141{
142    /* 'first' is included, 'last' is excluded */
143    if (first <= last)
144        return (uint8_t)(last - first);
145    return (uint8_t)(ringsize + last - first);
146}
147
148/* --------------- */
149/* --- RCR API --- */
150
151/* It's safer to code in terms of the 'rcr' object than the 'portal' object,
152 * because the latter runs the risk of copy-n-paste errors from other code where
153 * we could manipulate some other structure within 'portal'. */
154/* #define RCR_API_START()      register struct bm_rcr *rcr = &portal->rcr */
155
156/* Bit-wise logic to wrap a ring pointer by clearing the "carry bit" */
157#define RCR_CARRYCLEAR(p) \
158    (void *)((uintptr_t)(p) & (~(uintptr_t)(BM_RCR_SIZE << 6)))
159
160/* Bit-wise logic to convert a ring pointer to a ring index */
161static __inline__ uint8_t RCR_PTR2IDX(struct bm_rcr_entry *e)
162{
163    return (uint8_t)(((uintptr_t)e >> 6) & (BM_RCR_SIZE - 1));
164}
165
166/* Increment the 'cursor' ring pointer, taking 'vbit' into account */
167static __inline__ void RCR_INC(struct bm_rcr *rcr)
168{
169    /* NB: this is odd-looking, but experiments show that it generates
170     * fast code with essentially no branching overheads. We increment to
171     * the next RCR pointer and handle overflow and 'vbit'. */
172    struct bm_rcr_entry *partial = rcr->cursor + 1;
173    rcr->cursor = RCR_CARRYCLEAR(partial);
174    if (partial != rcr->cursor)
175        rcr->vbit ^= BM_RCR_VERB_VBIT;
176}
177
178t_Error bm_rcr_init(struct bm_portal *portal,
179                    e_BmPortalProduceMode pmode,
180                    e_BmPortalRcrConsumeMode cmode)
181{
182    register struct bm_rcr *rcr = &portal->rcr;
183    uint32_t cfg;
184    uint8_t pi;
185
186    rcr->ring = ptr_ADD(portal->addr.addr_ce, CL_RCR);
187    rcr->ci = (uint8_t)(bm_in(RCR_CI_CINH) & (BM_RCR_SIZE - 1));
188    pi = (uint8_t)(bm_in(RCR_PI_CINH) & (BM_RCR_SIZE - 1));
189    rcr->cursor = rcr->ring + pi;
190    rcr->vbit = (uint8_t)((bm_in(RCR_PI_CINH) & BM_RCR_SIZE) ?  BM_RCR_VERB_VBIT : 0);
191    rcr->available = (uint8_t)(BM_RCR_SIZE - 1 - cyc_diff(BM_RCR_SIZE, rcr->ci, pi));
192    rcr->ithresh = (uint8_t)bm_in(RCR_ITR);
193#ifdef BM_CHECKING
194    rcr->busy = 0;
195    rcr->pmode = pmode;
196    rcr->cmode = cmode;
197#else
198    UNUSED(cmode);
199#endif /* BM_CHECKING */
200    cfg = (bm_in(CFG) & 0xffffffe0) | (pmode & 0x3); /* BCSP_CFG::RPM */
201    bm_out(CFG, cfg);
202    return 0;
203}
204
205void bm_rcr_finish(struct bm_portal *portal)
206{
207    register struct bm_rcr *rcr = &portal->rcr;
208    uint8_t pi = (uint8_t)(bm_in(RCR_PI_CINH) & (BM_RCR_SIZE - 1));
209    uint8_t ci = (uint8_t)(bm_in(RCR_CI_CINH) & (BM_RCR_SIZE - 1));
210    ASSERT_COND(!rcr->busy);
211    if (pi != RCR_PTR2IDX(rcr->cursor))
212        REPORT_ERROR(WARNING, E_INVALID_STATE, ("losing uncommitted RCR entries"));
213    if (ci != rcr->ci)
214        REPORT_ERROR(WARNING, E_INVALID_STATE, ("missing existing RCR completions"));
215    if (rcr->ci != RCR_PTR2IDX(rcr->cursor))
216        REPORT_ERROR(WARNING, E_INVALID_STATE, ("RCR destroyed unquiesced"));
217}
218
219struct bm_rcr_entry *bm_rcr_start(struct bm_portal *portal)
220{
221    register struct bm_rcr *rcr = &portal->rcr;
222    ASSERT_COND(!rcr->busy);
223    if (!rcr->available)
224        return NULL;
225#ifdef BM_CHECKING
226    rcr->busy = 1;
227#endif /* BM_CHECKING */
228    dcbz_64(rcr->cursor);
229    return rcr->cursor;
230}
231
232void bm_rcr_abort(struct bm_portal *portal)
233{
234    register struct bm_rcr *rcr = &portal->rcr;
235    ASSERT_COND(rcr->busy);
236#ifdef BM_CHECKING
237    rcr->busy = 0;
238#else
239    UNUSED(rcr);
240#endif /* BM_CHECKING */
241}
242
243struct bm_rcr_entry *bm_rcr_pend_and_next(struct bm_portal *portal, uint8_t myverb)
244{
245    register struct bm_rcr *rcr = &portal->rcr;
246    ASSERT_COND(rcr->busy);
247    ASSERT_COND(rcr->pmode != e_BmPortalPVB);
248    if (rcr->available == 1)
249        return NULL;
250    rcr->cursor->__dont_write_directly__verb = (uint8_t)(myverb | rcr->vbit);
251    dcbf_64(rcr->cursor);
252    RCR_INC(rcr);
253    rcr->available--;
254    dcbz_64(rcr->cursor);
255    return rcr->cursor;
256}
257
258void bm_rcr_pci_commit(struct bm_portal *portal, uint8_t myverb)
259{
260    register struct bm_rcr *rcr = &portal->rcr;
261    ASSERT_COND(rcr->busy);
262    ASSERT_COND(rcr->pmode == e_BmPortalPCI);
263    rcr->cursor->__dont_write_directly__verb = (uint8_t)(myverb | rcr->vbit);
264    RCR_INC(rcr);
265    rcr->available--;
266    mb();
267    bm_out(RCR_PI_CINH, RCR_PTR2IDX(rcr->cursor));
268#ifdef BM_CHECKING
269    rcr->busy = 0;
270#endif /* BM_CHECKING */
271}
272
273void bm_rcr_pce_prefetch(struct bm_portal *portal)
274{
275    ASSERT_COND(((struct bm_rcr *)&portal->rcr)->pmode == e_BmPortalPCE);
276    bm_cl_invalidate(RCR_PI);
277    bm_cl_touch_rw(RCR_PI);
278}
279
280void bm_rcr_pce_commit(struct bm_portal *portal, uint8_t myverb)
281{
282    register struct bm_rcr *rcr = &portal->rcr;
283    ASSERT_COND(rcr->busy);
284    ASSERT_COND(rcr->pmode == e_BmPortalPCE);
285    rcr->cursor->__dont_write_directly__verb = (uint8_t)(myverb | rcr->vbit);
286    RCR_INC(rcr);
287    rcr->available--;
288    wmb();
289    bm_cl_out(RCR_PI, RCR_PTR2IDX(rcr->cursor));
290#ifdef BM_CHECKING
291    rcr->busy = 0;
292#endif /* BM_CHECKING */
293}
294
295void bm_rcr_pvb_commit(struct bm_portal *portal, uint8_t myverb)
296{
297    register struct bm_rcr *rcr = &portal->rcr;
298    struct bm_rcr_entry *rcursor;
299    ASSERT_COND(rcr->busy);
300    ASSERT_COND(rcr->pmode == e_BmPortalPVB);
301    rmb();
302    rcursor = rcr->cursor;
303    rcursor->__dont_write_directly__verb = (uint8_t)(myverb | rcr->vbit);
304    dcbf_64(rcursor);
305    RCR_INC(rcr);
306    rcr->available--;
307#ifdef BM_CHECKING
308    rcr->busy = 0;
309#endif /* BM_CHECKING */
310}
311
312
313uint8_t bm_rcr_cci_update(struct bm_portal *portal)
314{
315    register struct bm_rcr *rcr = &portal->rcr;
316    uint8_t diff, old_ci = rcr->ci;
317    ASSERT_COND(rcr->cmode == e_BmPortalRcrCCI);
318    rcr->ci = (uint8_t)(bm_in(RCR_CI_CINH) & (BM_RCR_SIZE - 1));
319    diff = cyc_diff(BM_RCR_SIZE, old_ci, rcr->ci);
320    rcr->available += diff;
321    return diff;
322}
323
324
325void bm_rcr_cce_prefetch(struct bm_portal *portal)
326{
327    ASSERT_COND(((struct bm_rcr *)&portal->rcr)->cmode == e_BmPortalRcrCCE);
328    bm_cl_touch_ro(RCR_CI);
329}
330
331
332uint8_t bm_rcr_cce_update(struct bm_portal *portal)
333{
334    register struct bm_rcr *rcr = &portal->rcr;
335    uint8_t diff, old_ci = rcr->ci;
336    ASSERT_COND(rcr->cmode == e_BmPortalRcrCCE);
337    rcr->ci = (uint8_t)(bm_cl_in(RCR_CI) & (BM_RCR_SIZE - 1));
338    bm_cl_invalidate(RCR_CI);
339    diff = cyc_diff(BM_RCR_SIZE, old_ci, rcr->ci);
340    rcr->available += diff;
341    return diff;
342}
343
344
345uint8_t bm_rcr_get_ithresh(struct bm_portal *portal)
346{
347    register struct bm_rcr *rcr = &portal->rcr;
348    return rcr->ithresh;
349}
350
351
352void bm_rcr_set_ithresh(struct bm_portal *portal, uint8_t ithresh)
353{
354    register struct bm_rcr *rcr = &portal->rcr;
355    rcr->ithresh = ithresh;
356    bm_out(RCR_ITR, ithresh);
357}
358
359
360uint8_t bm_rcr_get_avail(struct bm_portal *portal)
361{
362    register struct bm_rcr *rcr = &portal->rcr;
363    return rcr->available;
364}
365
366
367uint8_t bm_rcr_get_fill(struct bm_portal *portal)
368{
369    register struct bm_rcr *rcr = &portal->rcr;
370    return (uint8_t)(BM_RCR_SIZE - 1 - rcr->available);
371}
372
373
374/* ------------------------------ */
375/* --- Management command API --- */
376
377/* It's safer to code in terms of the 'mc' object than the 'portal' object,
378 * because the latter runs the risk of copy-n-paste errors from other code where
379 * we could manipulate some other structure within 'portal'. */
380/* #define MC_API_START()      register struct bm_mc *mc = &portal->mc */
381
382
383t_Error bm_mc_init(struct bm_portal *portal)
384{
385    register struct bm_mc *mc = &portal->mc;
386    mc->cr = ptr_ADD(portal->addr.addr_ce, CL_CR);
387    mc->rr = ptr_ADD(portal->addr.addr_ce, CL_RR0);
388    mc->rridx = (uint8_t)((mc->cr->__dont_write_directly__verb & BM_MCC_VERB_VBIT) ?
389            0 : 1);
390    mc->vbit = (uint8_t)(mc->rridx ? BM_MCC_VERB_VBIT : 0);
391#ifdef BM_CHECKING
392    mc->state = mc_idle;
393#endif /* BM_CHECKING */
394    return 0;
395}
396
397
398void bm_mc_finish(struct bm_portal *portal)
399{
400    register struct bm_mc *mc = &portal->mc;
401    ASSERT_COND(mc->state == mc_idle);
402#ifdef BM_CHECKING
403    if (mc->state != mc_idle)
404        REPORT_ERROR(WARNING, E_INVALID_STATE, ("Losing incomplete MC command"));
405#else
406    UNUSED(mc);
407#endif /* BM_CHECKING */
408}
409
410
411struct bm_mc_command *bm_mc_start(struct bm_portal *portal)
412{
413    register struct bm_mc *mc = &portal->mc;
414    ASSERT_COND(mc->state == mc_idle);
415#ifdef BM_CHECKING
416    mc->state = mc_user;
417#endif /* BM_CHECKING */
418    dcbz_64(mc->cr);
419    return mc->cr;
420}
421
422
423void bm_mc_abort(struct bm_portal *portal)
424{
425    register struct bm_mc *mc = &portal->mc;
426    ASSERT_COND(mc->state == mc_user);
427#ifdef BM_CHECKING
428    mc->state = mc_idle;
429#else
430    UNUSED(mc);
431#endif /* BM_CHECKING */
432}
433
434
435void bm_mc_commit(struct bm_portal *portal, uint8_t myverb)
436{
437    register struct bm_mc *mc = &portal->mc;
438    ASSERT_COND(mc->state == mc_user);
439    rmb();
440    mc->cr->__dont_write_directly__verb = (uint8_t)(myverb | mc->vbit);
441    dcbf_64(mc->cr);
442    dcbit_ro(mc->rr + mc->rridx);
443#ifdef BM_CHECKING
444    mc->state = mc_hw;
445#endif /* BM_CHECKING */
446}
447
448
449struct bm_mc_result *bm_mc_result(struct bm_portal *portal)
450{
451    register struct bm_mc *mc = &portal->mc;
452    struct bm_mc_result *rr = mc->rr + mc->rridx;
453    ASSERT_COND(mc->state == mc_hw);
454    /* The inactive response register's verb byte always returns zero until
455     * its command is submitted and completed. This includes the valid-bit,
456     * in case you were wondering... */
457    if (!rr->verb) {
458        dcbit_ro(rr);
459        return NULL;
460    }
461    mc->rridx ^= 1;
462    mc->vbit ^= BM_MCC_VERB_VBIT;
463#ifdef BM_CHECKING
464    mc->state = mc_idle;
465#endif /* BM_CHECKING */
466    return rr;
467}
468
469/* ------------------------------------- */
470/* --- Portal interrupt register API --- */
471
472#define SCN_REG(bpid) REG_SCN((bpid) / 32)
473#define SCN_BIT(bpid) (0x80000000 >> (bpid & 31))
474void bm_isr_bscn_mask(struct bm_portal *portal, uint8_t bpid, int enable)
475{
476    uint32_t val;
477    ASSERT_COND(bpid < BM_MAX_NUM_OF_POOLS);
478    /* REG_SCN for bpid=0..31, REG_SCN+4 for bpid=32..63 */
479    val = __bm_in(&portal->addr, SCN_REG(bpid));
480    if (enable)
481        val |= SCN_BIT(bpid);
482    else
483        val &= ~SCN_BIT(bpid);
484    __bm_out(&portal->addr, SCN_REG(bpid), val);
485}
486
487
488uint32_t __bm_isr_read(struct bm_portal *portal, enum bm_isr_reg n)
489{
490    return __bm_in(&portal->addr, REG_ISR + (n << 2));
491}
492
493
494void __bm_isr_write(struct bm_portal *portal, enum bm_isr_reg n, uint32_t val)
495{
496    __bm_out(&portal->addr, REG_ISR + (n << 2), val);
497}
498
499