1/*
2 * ARM lock definitions
3 */
4
5/*
6 * !!! DO NOT MODIFY THIS FILE !!!
7 * !!! IT WORKS PROPERLY, YOU MIGHT BREAK SOMETHING !!!
8 */
9
10#include <mach_assert.h>
11#include <assym.s>
12#include <arm/asm_help.h>
13#include <mach/arm/asm.h>
14
15/*
16 * OMAP3530 on BeagleBoard xM has issues with STREX/LDREX. Do ourselves a favor
17 * and not use them. It doesn't matter as this SoC is uniprocessor.
18 */
19#ifdef BOARD_CONFIG_OMAP3530
20#undef BOARD_CONFIG_OMAP3530
21#endif
22
23#ifdef NO_EXCLUSIVES
24#define ldrex           ldr
25#endif
26
27#ifndef _ARM_ARCH_7
28#undef EnterThumb
29#define EnterThumb EnterARM
30#endif
31
32/*
33 * Lock bit definitions are not defined here. Please don't touch. Thanks.
34 */
35
36/* Yes, this was blatantly taken from the real kernel, sorry about that. */
37
38/**
39 * hw_lock_init/arm_usimple_lock_init
40 *
41 * Initialize a lock and all of its bits to zero.
42 */
43EnterThumb(hw_lock_init)
44EnterThumb(arm_usimple_lock_init)
45    mov     r1, #0
46    str     r1, [r0]
47    bx      lr
48
49/**
50 * hw_lock_held
51 *
52 * Check a lock and see if the interlock bit is set or not.
53 */
54EnterThumb(hw_lock_held)
55    ldr     r3, [r0]
56    mov     r2, #1
57    and     r0, r2, r3
58    bx      lr
59
60.align 4
61
62/**
63 * arm_usimple_lock and friends
64 */
65EnterARM(arm_usimple_lock)
66EnterARM(lck_spin_lock)
67EnterARM(hw_lock_lock)
68    LoadLockHardwareRegister(r12)
69    IncrementPreemptLevel(r12, r2)
70    ldr     r3, [r0]
71    mov     r2, #1
72    orr     r1, r3, #1
73    ands    r2, r2, r3
74    streq   r1, [r0]
75    bxeq    lr
76hw_lock_lock_panic:
77    mov     r1, r0
78    ldr     r2, [r1]
79    adr     r0, _panicString
80    blx     _panic
81    b       .
82_panicString:
83    .asciz "hw_lock_lock(): PANIC: Lock 0x%08x = 0x%08x"
84
85
86/**
87 * hw_lock_unlock and friends
88 */
89EnterARM(lck_spin_unlock)
90EnterARM(hw_lock_unlock)
91    ldr     r3, [r0]
92    bic     r3, r3, #1
93    str     r3, [r0]
94    b       __enable_preemption
95
96/**
97 * arm_usimple_lock_try and friends
98 */
99EnterARM(arm_usimple_lock_try)
100EnterARM(lck_spin_try_lock)
101EnterARM(hw_lock_try)
102    mrs     r1, cpsr
103    cpsid   if
104    ldr     r3, [r0]
105    mov     r2, #1
106    orr     r12, r3, #1
107    ands    r2, r2, r3
108    streq   r12, [r0]
109    bne     hw_lock_try_fail
110    LoadLockHardwareRegister(r12)
111    IncrementPreemptLevel(r12, r2)
112    mov     r0, #1
113    b       hw_lock_try_return
114hw_lock_try_fail:
115    mov     r0, #0
116hw_lock_try_return:
117    msr     cpsr_cf, r1
118    bx      lr
119
120/**
121 * hw_lock_to
122 */
123EnterARM(hw_lock_to)
124    LoadLockHardwareRegister(r12)
125    IncrementPreemptLevel(r12, r2)
126    ldr     r3, [r0]
127    mov     r2, #1
128    orr     r1, r3, #1
129    ands    r2, r2, r3
130    streq   r1, [r0]
131    eors    r0, r2, #1
132    beq     hw_lock_to_enable_preempt
133    bx      lr
134hw_lock_to_enable_preempt:
135    stmfd   sp!, {r0,r1,r7,lr}
136    add     r7, sp, #8
137    blx     __enable_preemption
138    ldmfd   sp!, {r0,r1,r7,lr}
139    bx      lr
140
141/**
142 * lock_read and friends
143 */
144EnterARM(lock_read)
145EnterARM(lck_rw_lock_shared)
146#ifdef NO_EXCLUSIVES
147    mrs         r9, cpsr
148    orr         r3, r9, #0xc0
149    msr         cpsr_cf, r3
150#endif
151    mov     r3, #0xD
152rwlsloop:
153    ldrex   r1, [r0]
154    ands    r2, r1, r3
155    bne     rwlsopt
156rwlsloopres:
157    add     r1, r1, #0x10000
158#ifndef BOARD_CONFIG_OMAP3530
159    strex   r2, r1, [r0]
160    movs    r2, r2
161    bxeq    lr
162    b       rwlsloop
163    str     r1, [r0]
164    bx      lr
165#endif
166    str     r1, [r0]
167    bx      lr
168rwlsopt:
169#ifndef _ARM_ARCH_7
170    mov     r2, #0x0001
171    orr     r2, r2, #0x8000
172#else
173    mov     r2, #0x8001
174#endif
175    ands    r2, r1, r2
176    LOAD_ADDR(r12, lck_rw_lock_shared_gen)
177    bx      r12
178    movs    r2, r1, lsr#16
179    bne     rwlsloopres
180rwlsloolow:
181    LOAD_ADDR(r12, lck_rw_lock_shared_gen)
182    bx      r12
183
184/**
185 * lock_write and friends
186 */
187EnterARM(lock_write)
188EnterARM(lck_rw_lock_exclusive)
189    LoadConstantToReg(0xFFFF000D, r3)
190#ifdef NO_EXCLUSIVES
191    mrs         r12, cpsr
192    orr         r2, r12, #0xc0
193    msr         cpsr_cf, r2
194#endif
195rwleloop:
196    ldrex       r1, [r0]
197    ands        r2, r1, r3
198    bne         rwleslow
199    orr         r1, r1, #8
200#ifndef NO_EXCLUSIVES
201    strex       r2, r1, [r0]
202    movs        r2, r2
203    bxeq        lr
204    b           rwleloop
205#else
206    str         r1, [r0]
207    msr         cpsr_cf, r12
208    bx          lr
209#endif
210rwleslow:
211    LOAD_ADDR(r12, lck_rw_lock_exclusive_gen)
212    bx          r12
213
214/**
215 * lock_done and friends
216 */
217EnterARM(lock_done)
218EnterARM(lck_rw_done)
219#ifdef BOARD_CONFIG_OMAP3530
220    mrs         r9, cpsr
221    cpsid       if
222#endif
223_lock_done_enter:
224    ldrex       r1, [r0]
225    ands        r2, r1, #1
226    bne         rwldpanic
227    LoadConstantToReg(0xFFFF0000, r3)
228    ands        r2, r1, r3
229    beq         rwldexcl
230    sub         r1, r1, #0x10000
231    ands        r2, r1, r3
232    mov         r12, #0
233    mov         r3, #1
234    bne         rwldshared1
235    ands        r12, r1, #2
236    bics        r1, r1, #2
237rwldshared1:
238    b           rwldstore
239rwldexcl:
240    ands        r2, r1, #4
241    orrne       r2, r2, #2
242    bne         rwldexcl1
243    mov         r2, #0xA
244rwldexcl1:
245    mov         r3, #2
246    and         r12, r1, #2
247    bic         r1, r1, r2
248rwldstore:
249#ifndef BOARD_CONFIG_OMAP3530
250    strex       r2, r1, [r0]
251    movs        r2, r2
252    bne         _lock_done_enter
253#else
254    str         r1, [r0]
255#endif
256#ifdef BOARD_CONFIG_OMAP3530
257    msr         cpsr_cf, r9
258#endif
259    movs        r12, r12
260    moveq       r0, r3
261    bxeq        lr
262    stmfd       sp!,{r0,r3,r7,lr}
263    add         r0, r0, #8
264    blx         _thread_wakeup
265    ldmfd       sp!,{r0,r3,r7,lr}
266    mov         r0, r3
267    bx          lr
268rwldpanic:
269    mov         r2, r1
270    mov         r1, r0
271    adr         r0, lckPanicString
272    blx         _panic
273lckPanicString:
274    .asciz "lck_rw_done(): lock (0x%08x: 0x%08x)"
275
276/**
277 * lock_read_to_write and friends
278 */
279EnterARM(lock_read_to_write)
280EnterARM(lck_rw_lock_shared_to_exclusive)
281    ldrex       r1, [r0]
282    LoadConstantToReg(0xFFFF0000, r3)
283    ands        r2, r1, r3
284    beq         rwlsepanic
285    bic         r1, r1, r3
286    LOAD_ADDR(r12, lck_rw_lock_shared_gen)
287    subs        r2, r2, #0x10000
288    bxne        r12
289    ands        r3, r1, #5
290    bxne        r12
291    orr         r1, r1, #4
292#ifndef BOARD_CONFIG_OMAP3530
293    strex       r2, r1, [r0]
294    movs        r2, r2
295    bne         _lock_read_to_write
296#else
297    str         r1, [r0]
298#endif
299    mov         r0, #1
300    bx          lr
301rwlsepanic:
302    mov         r1, r2
303    mov         r1, r0
304    adr         r0, lckReadToWritePanicString
305    blx         _panic
306lckReadToWritePanicString:
307    .asciz "lck_rw_lock_shared_to_exclusive(): lock (0x%08x 0x%08x)"
308
309/**
310 * lock_write_to_read and friends
311 */
312EnterARM(lck_rw_lock_exclusive_to_shared)
313EnterARM(lock_write_to_read)
314#ifdef NO_EXCLUSIVES
315    mrs         r12, cpsr
316    orr         r2, r12, #0xc0
317    msr         cpsr_cf, r2
318#endif
319    ldrex       r1, [r0]
320    ands        r2, r1, #1
321    bne         rwlstexit
322    and         r2, r1, #2
323    ands        r3, r1, #4
324    mov         r3, #6
325    moveq       r3, #0xA
326    bic         r1, r1, r3
327    orr         r1, r1, #0x10000
328#ifndef BOARD_CONFIG_OMAP3530
329    strex       r3, r1, [r0]
330    movs        r3, r3
331    bne         _lock_write_to_read
332#else
333    str         r1, [r0]
334    msr         cpsr_cf, r12
335#endif
336    movs        r2, r2
337    bxeq        lr
338    add         r0, r0, #8
339    LOAD_ADDR(r12, thread_wakeup)
340    bx          r12
341rwlstexit:
342    mov         r2, r1
343    mov         r1, r0
344    adr         r0, rwlstePanicString
345    blx         _panic
346rwlstePanicString:
347    .asciz "lck_rw_lock_exclusive_to_shared(): lock (0x%08x 0x%08x)"
348
349
350/**
351 * lck_mtx_unlock
352 */
353EnterARM(lck_mtx_unlock)
354#ifdef NO_EXCLUSIVES
355    mrs     r9, cpsr
356    orr     r2, r9, #0xc0
357    msr     cpsr_cf, r2
358#endif
359    mov     r2, #0
360    LoadLockHardwareRegister(r12)
361mluloop:
362    ldrex       r1, [r0]
363    ands        r3, r1, #3
364    bne         lmuslow
365    bic         r3, r1, #3
366    cmp         r3, r12
367    bne         lmupanic
368#ifndef NO_EXCLUSIVES
369    strex       r1, r2, [r0]
370    movs        r1, r1
371    bxeq        lr
372#else
373    str         r2, [r0]
374    msr         cpsr_cf, r9
375    bx          lr
376#endif
377
378    b           mluloop
379lmuslow:
380#ifdef NO_EXCLUSIVES
381    msr         cpsr_cf, r9
382#endif
383    stmfd       sp!,{r0,r1,r7,lr}
384    add         r7, sp, #8
385    LoadLockHardwareRegister(r12)
386    IncrementPreemptLevel(r12, r2)
387    ldr         r1, [r0]
388    ands        r3, r1, #1
389    bne         lmupanic
390    orr         r3, r1, #1
391    str         r3, [r0]
392    bics        r1, r1, #3
393    ands        r2, r3, #2
394    beq         lmupanic
395    blx         _lck_mtx_unlock_wakeup
396    ldmfd       sp!,{r0,r1,r7,lr}
397    ldr         r1, [r0]
398    and         r3, r1, #2
399    str         r3, [r0]
400lmuret:
401    b           __enable_preemption
402lmupanic:
403    mov         r1, r0
404    ldr         r2, [r1]
405    adr         r0, lmuPanicString
406    blx         _panic
407lmuPanicString:
408    .asciz      "lck_mtx_unlock(): mutex (0x%08x, 0x%08x)"
409
410/**
411 * lck_mtx_lock
412 */
413EnterARM(lck_mtx_lock)
414    LoadLockHardwareRegister(r12)
415mlckretry:
416#ifdef NO_EXCLUSIVES
417    mrs     r2, cpsr
418    orr     r3, r2, #0xc0
419    msr     cpsr_cf, r3
420#endif
421    ldrex   r3, [r0]
422    movs    r3, r3
423    bne     mlckslow
424#ifndef NO_EXCLUSIVES
425    strex   r1, r12, [r0]
426    movs    r1, r1
427    bne     mlckretry
428#else
429    str     r12, [r0]
430    msr     cpsr_cf, r2
431#endif
432    bx      lr
433mlckslow:
434#ifdef NO_EXCLUSIVES
435    msr     cpsr_cf, r2
436#endif
437    stmfd   sp!,{r0,r1,r7,lr}
438    add     r7, sp, #8
439    LoadLockHardwareRegister(r12)
440    IncrementPreemptLevel(r12, r2)
441    ldr     r3, [r0]
442    ands    r2, r3, #1
443    bne     mlckpanic
444    bics    r1, r3, #2
445    bne     mlckwait
446    orr     r3, r3, #1
447    str     r3, [r0]
448    blx     _lck_mtx_lock_acquire
449    ands    r0, r0, r0
450    ldmfd   sp!,{r0,r1,r7,lr}
451    LoadLockHardwareRegister(r12)
452    mov     r3, r12
453    orrne   r3, r3, #2
454    str     r3, [r0]
455    b       __enable_preemption
456mlckwait:
457    orr     r3, r3, #0
458    str     r3, [r0]
459    blx     _lck_mtx_lock_wait
460    ldmfd   sp!,{r0,r1,r7,lr}
461    LoadLockHardwareRegister(r12)
462    b       mlckretry
463mlckpanic:
464    mov     r1, r0
465    ldr     r2, [r1]
466    adr     r0, mlckpanicString
467    blx     _panic
468mlckpanicString:
469    .asciz  "lck_mtx_lock(): mutex (0x%08x, 0x%08x)"
470
471/**
472 * lck_mtx_try_lock
473 */
474EnterARM(lck_mtx_try_lock)
475    LoadLockHardwareRegister(r12)
476#ifdef NO_EXCLUSIVES
477    mrs         r2, cpsr
478    orr         r3, r2, #0xc0
479    msr         cpsr_cf, r3
480#endif
481lmtstart:
482    ldrex       r3, [r0]
483    movs        r3, r3
484    bne         lmtslow
485#ifndef NO_EXCLUSIVES
486    strex       r1, r12, [r0]
487    movs        r1, r1
488    bne         lmtstart
489#else
490    str         r12, [r0]
491    msr         cpsr_cf, r2
492#endif
493    mov         r0, #1
494    bx          lr
495lmtslow:
496#ifdef NO_EXCLUSIVES
497    msr         cpsr_cf, r2
498#endif
499    mov         r1, #0
500    stmfd       sp!, {r0,r1,r7,lr}
501    add         r7, sp, #8
502    IncrementPreemptLevel(r12, r2)
503    ldr         r3, [r0]
504    ands        r2, r3, #1
505    bne         lmtpanic
506    bics        r2, r3, #2
507    bne         lmtret
508    orr         r3, r3, #1
509    str         r3, [r0]
510    blx         _lck_mtx_lock_acquire
511    ands        r0, r0, r0
512    ldmfd       sp!,{r0,r1,r7,lr}
513    LoadLockHardwareRegister(r12)
514    mov         r3, r12
515    orrne       r3, r3, #2
516    str         r3, [r0]
517    mov         r1, #1
518    stmfd       sp!,{r0,r1,r7,lr}
519    add         r7, sp, #8
520lmtret:
521    bl          __enable_preemption
522    ldmfd       sp!,{r0,r1,r7,lr}
523    mov         r0, r1
524    bx          lr
525lmtpanic:
526    mov         r1, r0
527    ldr         r2, [r1]
528    adr         r0, lmtPanicString
529    blx         _panic
530lmtPanicString:
531    .asciz "lck_mtx_try_lock(): mutex (0x%08x, 0x%08x)"
532
533/**
534 * lck_mtx_assert
535 */
536EnterARM(lck_mtx_assert)
537    ldr     r12, [r0]
538    bics    r3, r12, #3
539    cmp     r1, #1
540    bne     lck_mtx_assert_owned
541    LoadLockHardwareRegister(r12)
542    cmp     r3, r12
543    bxeq    lr
544    mov     r1, r0
545    adr     r0, panicString_lockNotOwned
546    blx     _panic
547lck_mtx_assert_owned:
548    cmp     r1, #2
549    bne     lck_mtx_assert_bad_arg
550    LoadLockHardwareRegister(r12)
551    cmp     r3, r12
552    bxne    lr
553    mov     r1, r0
554    adr     r0, panicString_lockOwned
555    blx     _panic
556lck_mtx_assert_bad_arg:
557    adr     r0, panicString_lockBadArgument
558    blx     _panic
559
560    .align 4
561panicString_lockNotOwned:
562    .asciz  "lck_mtx_assert(): mutex (0x%08X) not owned\n"
563    .align 4
564panicString_lockOwned:
565    .asciz  "lck_mtx_assert(): mutex (0x%08X) owned\n"
566    .align 4
567panicString_lockBadArgument:
568    .asciz  "lck_mtx_assert(): arg1 (0x%08X) invalid\n"
569
570/**
571 * lck_rw_try_lock_shared
572 */
573EnterARM(lck_rw_try_lock_shared)
574#ifdef NO_EXCLUSIVES
575    mrs     r9, cpsr
576    orr     r1, r9, #0xc0
577    msr     cpsr_cf, r1
578#endif
579    ldrex   r1, [r0]
580    ands    r2, r1, #1
581    bne     rwtlspanic
582    ands    r2, r1, #0xC
583    bne     rwtlsopt
584rwtlsloopres:
585    add     r1, r1, #0x10000
586#ifndef NO_EXCLUSIVES
587    strex   r2, r1, [r0]
588    movs    r2, r2
589    bne     _lck_rw_try_lock_shared
590#else
591    str     r1, [r0]
592    msr     cpsr_cf, r9
593#endif
594    mov     r0, #1
595    bx      lr
596rwtlsopt:
597    ands    r2, r1, #0x8000
598    bne     rwtlsfail
599    LoadConstantToReg(0xFFFF0000, r3)
600    ands    r2, r1, r3
601    bne     rwtlsloopres
602rwtlsfail:
603    mov     r0, #0
604    bx      lr
605rwtlspanic:
606    mov     r2, r1
607    mov     r1, r0
608    adr     r0, rwtlsPanicString
609    blx     _panic
610rwtlsPanicString:
611    .asciz  "lck_rw_try_lock_shared: lock (0x%08x, 0x%08x)"
612
613/**
614 * lck_rw_try_lock_exclusive
615 */
616EnterARM(lck_rw_try_lock_exclusive)
617#ifdef NO_EXCLUSIVES
618    mrs     r9, cpsr
619    orr     r1, r9, #0xc0
620    msr     cpsr_cf, r1
621#endif
622    ldrex   r1, [r0]
623    ands    r2, r1, #1
624    bne     rwtlepanic
625    LoadConstantToReg(0xFFFF0009, r3)
626    ands    r2, r1, r3
627    bne     rwtlefail
628    orr     r1, r1, #8
629#ifndef NO_EXCLUSIVES
630    strex   r2, r1, [r0]
631    movs    r2, r2
632    bne     _lck_rw_try_lock_exclusive
633#else
634    str     r1, [r0]
635    msr     cpsr_cf, r9
636#endif
637    mov     r0, #1
638    bx      lr
639rwtlefail:
640    mov     r0, #0
641    bx      lr
642rwtlepanic:
643    mov     r2, r1
644    mov     r1, r0
645    adr     r0, rwtlePanicString
646    blx     _panic
647rwtlePanicString:
648    .asciz  "lck_rw_try_lock_exclusive: lock (0x%08x, 0x%08x)"
649
650/**
651 * bitlock
652 */
653EnterARM(hw_lock_bit)
654    mov     r12, #1
655    mov     r12, r12, lsl r1
656    LoadLockHardwareRegister(r9)
657    IncrementPreemptLevel(r9, r2)
658    ldrex    r3, [r0]
659    orr     r1, r3, r12
660    ands    r2, r12, r3
661    streq   r1, [r0]
662    bxeq    lr
663hwlbitpanic:
664    mov     r0, r1
665    adr     r0, hwlbitPanicString
666    blx     _panic
667hwlbitPanicString:
668    .asciz "hw_lock_bit: lock 0x%08x"
669
670EnterARM(hw_unlock_bit)
671    mov     r12, #1
672    mov     r12, r12, lsl r1
673    ldr     r3, [r0]
674    bic     r3, r3, r12
675    str     r3, [r0]
676    b       __enable_preemption
677
678LOAD_ADDR_GEN_DEF(thread_wakeup)
679LOAD_ADDR_GEN_DEF(lck_rw_lock_shared_gen)
680LOAD_ADDR_GEN_DEF(lck_rw_lock_exclusive_gen)
681