1/***********************license start***************
2 * Copyright (c) 2003-2010  Cavium Inc. (support@cavium.com). All rights
3 * reserved.
4 *
5 *
6 * Redistribution and use in source and binary forms, with or without
7 * modification, are permitted provided that the following conditions are
8 * met:
9 *
10 *   * Redistributions of source code must retain the above copyright
11 *     notice, this list of conditions and the following disclaimer.
12 *
13 *   * Redistributions in binary form must reproduce the above
14 *     copyright notice, this list of conditions and the following
15 *     disclaimer in the documentation and/or other materials provided
16 *     with the distribution.
17
18 *   * Neither the name of Cavium Inc. nor the names of
19 *     its contributors may be used to endorse or promote products
20 *     derived from this software without specific prior written
21 *     permission.
22
23 * This Software, including technical data, may be subject to U.S. export  control
24 * laws, including the U.S. Export Administration Act and its  associated
25 * regulations, and may be subject to export or import  regulations in other
26 * countries.
27
28 * TO THE MAXIMUM EXTENT PERMITTED BY LAW, THE SOFTWARE IS PROVIDED "AS IS"
29 * AND WITH ALL FAULTS AND CAVIUM INC. MAKES NO PROMISES, REPRESENTATIONS OR
30 * WARRANTIES, EITHER EXPRESS, IMPLIED, STATUTORY, OR OTHERWISE, WITH RESPECT TO
31 * THE SOFTWARE, INCLUDING ITS CONDITION, ITS CONFORMITY TO ANY REPRESENTATION OR
32 * DESCRIPTION, OR THE EXISTENCE OF ANY LATENT OR PATENT DEFECTS, AND CAVIUM
33 * SPECIFICALLY DISCLAIMS ALL IMPLIED (IF ANY) WARRANTIES OF TITLE,
34 * MERCHANTABILITY, NONINFRINGEMENT, FITNESS FOR A PARTICULAR PURPOSE, LACK OF
35 * VIRUSES, ACCURACY OR COMPLETENESS, QUIET ENJOYMENT, QUIET POSSESSION OR
36 * CORRESPONDENCE TO DESCRIPTION. THE ENTIRE  RISK ARISING OUT OF USE OR
37 * PERFORMANCE OF THE SOFTWARE LIES WITH YOU.
38 ***********************license end**************************************/
39
40
41
42
43
44
45
46/**
47 * @file
48 *
49 * Implementation of spinlocks.
50 *
51 * <hr>$Revision: 70030 $<hr>
52 */
53
54
55#ifndef __CVMX_SPINLOCK_H__
56#define __CVMX_SPINLOCK_H__
57
58#include "cvmx-asm.h"
59
60#ifdef	__cplusplus
61extern "C" {
62#endif
63
64/* Spinlocks for Octeon */
65
66
67// define these to enable recursive spinlock debugging
68//#define CVMX_SPINLOCK_DEBUG
69
70
71/**
72 * Spinlocks for Octeon
73 */
74typedef struct {
75    volatile uint32_t value;
76} cvmx_spinlock_t;
77
78// note - macros not expanded in inline ASM, so values hardcoded
79#define  CVMX_SPINLOCK_UNLOCKED_VAL  0
80#define  CVMX_SPINLOCK_LOCKED_VAL    1
81
82
83#define CVMX_SPINLOCK_UNLOCKED_INITIALIZER  {CVMX_SPINLOCK_UNLOCKED_VAL}
84
85
86/**
87 * Initialize a spinlock
88 *
89 * @param lock   Lock to initialize
90 */
91static inline void cvmx_spinlock_init(cvmx_spinlock_t *lock)
92{
93    lock->value = CVMX_SPINLOCK_UNLOCKED_VAL;
94}
95
96
97/**
98 * Return non-zero if the spinlock is currently locked
99 *
100 * @param lock   Lock to check
101 * @return Non-zero if locked
102 */
103static inline int cvmx_spinlock_locked(cvmx_spinlock_t *lock)
104{
105    return (lock->value != CVMX_SPINLOCK_UNLOCKED_VAL);
106}
107
108
109/**
110 * Releases lock
111 *
112 * @param lock   pointer to lock structure
113 */
114static inline void cvmx_spinlock_unlock(cvmx_spinlock_t *lock)
115{
116    CVMX_SYNCWS;
117    lock->value = 0;
118    CVMX_SYNCWS;
119}
120
121
122/**
123 * Attempts to take the lock, but does not spin if lock is not available.
124 * May take some time to acquire the lock even if it is available
125 * due to the ll/sc not succeeding.
126 *
127 * @param lock   pointer to lock structure
128 *
129 * @return 0: lock successfully taken
130 *         1: lock not taken, held by someone else
131 * These return values match the Linux semantics.
132 */
133
134static inline unsigned int cvmx_spinlock_trylock(cvmx_spinlock_t *lock)
135{
136    unsigned int tmp;
137
138    __asm__ __volatile__(
139    ".set noreorder         \n"
140    "1: ll   %[tmp], %[val] \n"
141    "   bnez %[tmp], 2f     \n"  // if lock held, fail immediately
142    "   li   %[tmp], 1      \n"
143    "   sc   %[tmp], %[val] \n"
144    "   beqz %[tmp], 1b     \n"
145    "   li   %[tmp], 0      \n"
146    "2:                     \n"
147    ".set reorder           \n"
148    : [val] "+m" (lock->value), [tmp] "=&r" (tmp)
149    :
150    : "memory");
151
152    return (!!tmp);  /* normalize to 0 or 1 */
153}
154
155/**
156 * Gets lock, spins until lock is taken
157 *
158 * @param lock   pointer to lock structure
159 */
160static inline void cvmx_spinlock_lock(cvmx_spinlock_t *lock)
161{
162    unsigned int tmp;
163
164    __asm__ __volatile__(
165    ".set noreorder         \n"
166    "1: ll   %[tmp], %[val]  \n"
167    "   bnez %[tmp], 1b     \n"
168    "   li   %[tmp], 1      \n"
169    "   sc   %[tmp], %[val] \n"
170    "   beqz %[tmp], 1b     \n"
171    "   nop                \n"
172    ".set reorder           \n"
173    : [val] "+m" (lock->value), [tmp] "=&r" (tmp)
174    :
175    : "memory");
176
177}
178
179
180
181/** ********************************************************************
182 * Bit spinlocks
183 * These spinlocks use a single bit (bit 31) of a 32 bit word for locking.
184 * The rest of the bits in the word are left undisturbed.  This enables more
185 * compact data structures as only 1 bit is consumed for the lock.
186 *
187 */
188
189/**
190 * Gets lock, spins until lock is taken
191 * Preserves the low 31 bits of the 32 bit
192 * word used for the lock.
193 *
194 *
195 * @param word  word to lock bit 31 of
196 */
197static inline void cvmx_spinlock_bit_lock(uint32_t *word)
198{
199    unsigned int tmp;
200    unsigned int sav;
201
202    __asm__ __volatile__(
203    ".set noreorder         \n"
204    ".set noat              \n"
205    "1: ll    %[tmp], %[val]  \n"
206    "   bbit1 %[tmp], 31, 1b    \n"
207    "   li    $at, 1      \n"
208    "   ins   %[tmp], $at, 31, 1  \n"
209    "   sc    %[tmp], %[val] \n"
210    "   beqz  %[tmp], 1b     \n"
211    "   nop                \n"
212    ".set at              \n"
213    ".set reorder           \n"
214    : [val] "+m" (*word), [tmp] "=&r" (tmp), [sav] "=&r" (sav)
215    :
216    : "memory");
217
218}
219
220/**
221 * Attempts to get lock, returns immediately with success/failure
222 * Preserves the low 31 bits of the 32 bit
223 * word used for the lock.
224 *
225 *
226 * @param word  word to lock bit 31 of
227 * @return 0: lock successfully taken
228 *         1: lock not taken, held by someone else
229 * These return values match the Linux semantics.
230 */
231static inline unsigned int cvmx_spinlock_bit_trylock(uint32_t *word)
232{
233    unsigned int tmp;
234
235    __asm__ __volatile__(
236    ".set noreorder         \n"
237    ".set noat              \n"
238    "1: ll    %[tmp], %[val] \n"
239    "   bbit1 %[tmp], 31, 2f     \n"  // if lock held, fail immediately
240    "   li    $at, 1      \n"
241    "   ins   %[tmp], $at, 31, 1  \n"
242    "   sc    %[tmp], %[val] \n"
243    "   beqz  %[tmp], 1b     \n"
244    "   li    %[tmp], 0      \n"
245    "2:                     \n"
246    ".set at              \n"
247    ".set reorder           \n"
248    : [val] "+m" (*word), [tmp] "=&r" (tmp)
249    :
250    : "memory");
251
252    return (!!tmp);  /* normalize to 0 or 1 */
253}
254/**
255 * Releases bit lock
256 *
257 * Unconditionally clears bit 31 of the lock word.  Note that this is
258 * done non-atomically, as this implementation assumes that the rest
259 * of the bits in the word are protected by the lock.
260 *
261 * @param word  word to unlock bit 31 in
262 */
263static inline void cvmx_spinlock_bit_unlock(uint32_t *word)
264{
265    CVMX_SYNCWS;
266    *word &= ~(1UL << 31) ;
267    CVMX_SYNCWS;
268}
269
270
271
272/** ********************************************************************
273 * Recursive spinlocks
274 */
275typedef struct {
276	volatile unsigned int value;
277	volatile unsigned int core_num;
278} cvmx_spinlock_rec_t;
279
280
281/**
282 * Initialize a recursive spinlock
283 *
284 * @param lock   Lock to initialize
285 */
286static inline void cvmx_spinlock_rec_init(cvmx_spinlock_rec_t *lock)
287{
288    lock->value = CVMX_SPINLOCK_UNLOCKED_VAL;
289}
290
291
292/**
293 * Return non-zero if the recursive spinlock is currently locked
294 *
295 * @param lock   Lock to check
296 * @return Non-zero if locked
297 */
298static inline int cvmx_spinlock_rec_locked(cvmx_spinlock_rec_t *lock)
299{
300    return (lock->value != CVMX_SPINLOCK_UNLOCKED_VAL);
301}
302
303
304/**
305* Unlocks one level of recursive spinlock.  Lock is not unlocked
306* unless this is the final unlock call for that spinlock
307*
308* @param lock   ptr to recursive spinlock structure
309*/
310static inline void cvmx_spinlock_rec_unlock(cvmx_spinlock_rec_t *lock);
311
312#ifdef CVMX_SPINLOCK_DEBUG
313#define cvmx_spinlock_rec_unlock(x)  _int_cvmx_spinlock_rec_unlock((x), __FILE__, __LINE__)
314static inline void _int_cvmx_spinlock_rec_unlock(cvmx_spinlock_rec_t *lock, char *filename, int linenum)
315#else
316static inline void cvmx_spinlock_rec_unlock(cvmx_spinlock_rec_t *lock)
317#endif
318{
319
320	unsigned int temp, result;
321    int core_num;
322    core_num = cvmx_get_core_num();
323
324#ifdef CVMX_SPINLOCK_DEBUG
325    {
326        if (lock->core_num != core_num)
327        {
328            cvmx_dprintf("ERROR: Recursive spinlock release attemped by non-owner! file: %s, line: %d\n", filename, linenum);
329            return;
330        }
331    }
332#endif
333
334	__asm__ __volatile__(
335		".set  noreorder                 \n"
336		"     addi  %[tmp], %[pid], 0x80 \n"
337		"     sw    %[tmp], %[lid]       # set lid to invalid value\n"
338                CVMX_SYNCWS_STR
339		"1:   ll    %[tmp], %[val]       \n"
340		"     addu  %[res], %[tmp], -1   # decrement lock count\n"
341		"     sc    %[res], %[val]       \n"
342		"     beqz  %[res], 1b           \n"
343		"     nop                        \n"
344		"     beq   %[tmp], %[res], 2f   # res is 1 on successful sc       \n"
345		"     nop                        \n"
346		"     sw   %[pid], %[lid]        # set lid to pid, only if lock still held\n"
347		"2:                         \n"
348                CVMX_SYNCWS_STR
349		".set  reorder                   \n"
350		: [res] "=&r" (result), [tmp] "=&r" (temp), [val] "+m" (lock->value), [lid] "+m" (lock->core_num)
351		: [pid] "r" (core_num)
352		: "memory");
353
354
355#ifdef CVMX_SPINLOCK_DEBUG
356    {
357        if (lock->value == ~0UL)
358        {
359            cvmx_dprintf("ERROR: Recursive spinlock released too many times! file: %s, line: %d\n", filename, linenum);
360        }
361    }
362#endif
363
364
365}
366
367/**
368 * Takes recursive spinlock for a given core.  A core can take the lock multiple
369 * times, and the lock is released only when the corresponding number of
370 * unlocks have taken place.
371 *
372 * NOTE: This assumes only one thread per core, and that the core ID is used as
373 * the lock 'key'.  (This implementation cannot be generalized to allow
374 * multiple threads to use the same key (core id) .)
375 *
376 * @param lock   address of recursive spinlock structure.  Note that this is
377 *               distinct from the standard spinlock
378 */
379static inline void cvmx_spinlock_rec_lock(cvmx_spinlock_rec_t *lock);
380
381#ifdef CVMX_SPINLOCK_DEBUG
382#define cvmx_spinlock_rec_lock(x)  _int_cvmx_spinlock_rec_lock((x), __FILE__, __LINE__)
383static inline void _int_cvmx_spinlock_rec_lock(cvmx_spinlock_rec_t *lock, char *filename, int linenum)
384#else
385static inline void cvmx_spinlock_rec_lock(cvmx_spinlock_rec_t *lock)
386#endif
387{
388
389
390	volatile unsigned int tmp;
391	volatile int core_num;
392
393	core_num = cvmx_get_core_num();
394
395
396	__asm__ __volatile__(
397		".set  noreorder              \n"
398		"1: ll   %[tmp], %[val]       # load the count\n"
399		"   bnez %[tmp], 2f           # if count!=zero branch to 2\n"
400		"   addu %[tmp], %[tmp], 1    \n"
401		"   sc   %[tmp], %[val]       \n"
402		"   beqz %[tmp], 1b           # go back if not success\n"
403		"   nop                       \n"
404		"   j    3f                   # go to write core_num \n"
405		"2: lw   %[tmp], %[lid]       # load the core_num \n"
406		"   bne  %[tmp], %[pid], 1b   # core_num no match, restart\n"
407		"   nop                       \n"
408		"   lw   %[tmp], %[val]       \n"
409		"   addu %[tmp], %[tmp], 1    \n"
410		"   sw   %[tmp], %[val]       # update the count\n"
411		"3: sw   %[pid], %[lid]       # store the core_num\n"
412                CVMX_SYNCWS_STR
413		".set  reorder                \n"
414		: [tmp] "=&r" (tmp), [val] "+m" (lock->value), [lid] "+m" (lock->core_num)
415		: [pid] "r" (core_num)
416		: "memory");
417
418#ifdef CVMX_SPINLOCK_DEBUG
419    if (lock->core_num != core_num)
420    {
421        cvmx_dprintf("cvmx_spinlock_rec_lock: lock taken, but core_num is incorrect. file: %s, line: %d\n", filename, linenum);
422    }
423#endif
424
425
426}
427
428#ifdef	__cplusplus
429}
430#endif
431
432#endif /* __CVMX_SPINLOCK_H__ */
433