1/*
2 * Copyright (c) 2003-2012 Apple Inc. All rights reserved.
3 *
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
14 *
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
25 *
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27 */
28
29#ifndef _KERN_LOCKS_H_
30#define _KERN_LOCKS_H_
31
32#include	<sys/cdefs.h>
33#include	<sys/appleapiopts.h>
34#include	<mach/boolean.h>
35#include	<mach/mach_types.h>
36#include	<kern/kern_types.h>
37#include	<machine/locks.h>
38
39#ifdef	MACH_KERNEL_PRIVATE
40#include	<kern/queue.h>
41
42extern void				lck_mod_init(
43								void);
44
45typedef	unsigned int	lck_type_t;
46
47#define	LCK_TYPE_SPIN	1
48#define	LCK_TYPE_MTX	2
49#define	LCK_TYPE_RW	3
50
51#endif
52
53typedef	unsigned int		lck_sleep_action_t;
54
55#define	LCK_SLEEP_DEFAULT	0x00	/* Release the lock while waiting for the event, then reclaim */
56									/* RW locks are returned in the same mode */
57#define	LCK_SLEEP_UNLOCK	0x01	/* Release the lock and return unheld */
58#define	LCK_SLEEP_SHARED	0x02	/* Reclaim the lock in shared mode (RW only) */
59#define	LCK_SLEEP_EXCLUSIVE	0x04	/* Reclaim the lock in exclusive mode (RW only) */
60#define	LCK_SLEEP_SPIN		0x08	/* Reclaim the lock in spin mode (mutex only) */
61
62#define	LCK_SLEEP_MASK		0x0f	/* Valid actions */
63
64#ifdef	MACH_KERNEL_PRIVATE
65
66typedef struct {
67	uint64_t			lck_grp_spin_util_cnt;
68	uint64_t			lck_grp_spin_held_cnt;
69	uint64_t			lck_grp_spin_miss_cnt;
70	uint64_t			lck_grp_spin_held_max;
71	uint64_t			lck_grp_spin_held_cum;
72} lck_grp_spin_stat_t;
73
74typedef struct {
75	uint64_t			lck_grp_mtx_util_cnt;
76	/* On x86, this is used as the "direct wait" count */
77	uint64_t			lck_grp_mtx_held_cnt;
78	uint64_t			lck_grp_mtx_miss_cnt;
79	uint64_t			lck_grp_mtx_wait_cnt;
80	/* Rest currently unused */
81	uint64_t			lck_grp_mtx_held_max;
82	uint64_t			lck_grp_mtx_held_cum;
83	uint64_t			lck_grp_mtx_wait_max;
84	uint64_t			lck_grp_mtx_wait_cum;
85} lck_grp_mtx_stat_t;
86
87typedef struct {
88	uint64_t			lck_grp_rw_util_cnt;
89	uint64_t			lck_grp_rw_held_cnt;
90	uint64_t			lck_grp_rw_miss_cnt;
91	uint64_t			lck_grp_rw_wait_cnt;
92	uint64_t			lck_grp_rw_held_max;
93	uint64_t			lck_grp_rw_held_cum;
94	uint64_t			lck_grp_rw_wait_max;
95	uint64_t			lck_grp_rw_wait_cum;
96} lck_grp_rw_stat_t;
97
98typedef	struct _lck_grp_stat_ {
99	lck_grp_spin_stat_t	lck_grp_spin_stat;
100	lck_grp_mtx_stat_t	lck_grp_mtx_stat;
101	lck_grp_rw_stat_t	lck_grp_rw_stat;
102} lck_grp_stat_t;
103
104#define	LCK_GRP_MAX_NAME	64
105
106typedef	struct _lck_grp_ {
107	queue_chain_t		lck_grp_link;
108	uint32_t		lck_grp_refcnt;
109	uint32_t		lck_grp_spincnt;
110	uint32_t		lck_grp_mtxcnt;
111	uint32_t		lck_grp_rwcnt;
112	uint32_t		lck_grp_attr;
113	char			lck_grp_name[LCK_GRP_MAX_NAME];
114	lck_grp_stat_t		lck_grp_stat;
115} lck_grp_t;
116
117#define LCK_GRP_NULL	(lck_grp_t *)0
118
119#else
120typedef struct __lck_grp__ lck_grp_t;
121#endif
122
123#ifdef	MACH_KERNEL_PRIVATE
124typedef	struct _lck_grp_attr_ {
125	uint32_t	grp_attr_val;
126} lck_grp_attr_t;
127
128extern lck_grp_attr_t  LockDefaultGroupAttr;
129
130#define LCK_GRP_ATTR_STAT	0x1
131
132#else
133typedef struct __lck_grp_attr__ lck_grp_attr_t;
134#endif
135
136#define LCK_GRP_ATTR_NULL	(lck_grp_attr_t *)0
137
138__BEGIN_DECLS
139
140extern	lck_grp_attr_t	*lck_grp_attr_alloc_init(
141									void);
142
143extern	void			lck_grp_attr_setdefault(
144									lck_grp_attr_t	*attr);
145
146extern	void			lck_grp_attr_setstat(
147									lck_grp_attr_t  *attr);
148
149extern	void			lck_grp_attr_free(
150									lck_grp_attr_t	*attr);
151
152extern	lck_grp_t		*lck_grp_alloc_init(
153									const char*		grp_name,
154									lck_grp_attr_t	*attr);
155
156__END_DECLS
157
158#ifdef	MACH_KERNEL_PRIVATE
159extern	void			lck_grp_init(
160									lck_grp_t		*grp,
161									const char*		grp_name,
162									lck_grp_attr_t	*attr);
163
164extern	void			lck_grp_reference(
165									lck_grp_t		*grp);
166
167extern	void			lck_grp_deallocate(
168									lck_grp_t		 *grp);
169
170extern	void			lck_grp_lckcnt_incr(
171									lck_grp_t		*grp,
172									lck_type_t		lck_type);
173
174extern	void			lck_grp_lckcnt_decr(
175									lck_grp_t		*grp,
176									lck_type_t		lck_type);
177#endif
178
179__BEGIN_DECLS
180
181extern void				lck_grp_free(
182									lck_grp_t		*grp);
183
184__END_DECLS
185
186#ifdef	MACH_KERNEL_PRIVATE
187typedef	struct _lck_attr_ {
188	unsigned int	lck_attr_val;
189} lck_attr_t;
190
191extern lck_attr_t      LockDefaultLckAttr;
192
193#define LCK_ATTR_NONE		0
194
195#define	LCK_ATTR_DEBUG				0x00000001
196#define	LCK_ATTR_RW_SHARED_PRIORITY	0x00010000
197
198#else
199typedef struct __lck_attr__ lck_attr_t;
200#endif
201
202#define LCK_ATTR_NULL (lck_attr_t *)0
203
204__BEGIN_DECLS
205
206extern	lck_attr_t		*lck_attr_alloc_init(
207									void);
208
209extern	void			lck_attr_setdefault(
210									lck_attr_t		*attr);
211
212extern	void			lck_attr_setdebug(
213									lck_attr_t		*attr);
214
215extern	void			lck_attr_cleardebug(
216									lck_attr_t		*attr);
217
218#ifdef	XNU_KERNEL_PRIVATE
219extern	void			lck_attr_rw_shared_priority(
220									lck_attr_t		*attr);
221#endif
222
223extern	void			lck_attr_free(
224									lck_attr_t		*attr);
225
226#define decl_lck_spin_data(class,name)     class lck_spin_t name;
227
228extern lck_spin_t		*lck_spin_alloc_init(
229									lck_grp_t		*grp,
230									lck_attr_t		*attr);
231
232extern void				lck_spin_init(
233									lck_spin_t		*lck,
234									lck_grp_t		*grp,
235									lck_attr_t		*attr);
236
237extern void				lck_spin_lock(
238									lck_spin_t		*lck);
239
240extern void				lck_spin_unlock(
241									lck_spin_t		*lck);
242
243extern void				lck_spin_destroy(
244									lck_spin_t		*lck,
245									lck_grp_t		*grp);
246
247extern void				lck_spin_free(
248									lck_spin_t		*lck,
249									lck_grp_t		*grp);
250
251extern wait_result_t	lck_spin_sleep(
252									lck_spin_t			*lck,
253									lck_sleep_action_t	lck_sleep_action,
254									event_t				event,
255									wait_interrupt_t	interruptible);
256
257extern wait_result_t	lck_spin_sleep_deadline(
258									lck_spin_t			*lck,
259									lck_sleep_action_t	lck_sleep_action,
260									event_t				event,
261									wait_interrupt_t	interruptible,
262									uint64_t			deadline);
263
264#ifdef	KERNEL_PRIVATE
265
266extern boolean_t		lck_spin_try_lock(
267									lck_spin_t		*lck);
268
269struct _lck_mtx_ext_;
270extern void lck_mtx_init_ext(lck_mtx_t *lck, struct _lck_mtx_ext_ *lck_ext,
271    lck_grp_t *grp, lck_attr_t *attr);
272
273#endif
274
275
276#define decl_lck_mtx_data(class,name)     class lck_mtx_t name;
277
278extern lck_mtx_t		*lck_mtx_alloc_init(
279									lck_grp_t		*grp,
280									lck_attr_t		*attr);
281
282extern void				lck_mtx_init(
283									lck_mtx_t		*lck,
284									lck_grp_t		*grp,
285									lck_attr_t		*attr);
286extern void				lck_mtx_lock(
287									lck_mtx_t		*lck);
288
289extern void				lck_mtx_unlock(
290									lck_mtx_t		*lck);
291
292extern void				lck_mtx_destroy(
293									lck_mtx_t		*lck,
294									lck_grp_t		*grp);
295
296extern void				lck_mtx_free(
297									lck_mtx_t		*lck,
298									lck_grp_t		*grp);
299
300extern wait_result_t	lck_mtx_sleep(
301									lck_mtx_t			*lck,
302									lck_sleep_action_t	lck_sleep_action,
303									event_t				event,
304									wait_interrupt_t	interruptible);
305
306extern wait_result_t	lck_mtx_sleep_deadline(
307									lck_mtx_t			*lck,
308									lck_sleep_action_t	lck_sleep_action,
309									event_t				event,
310									wait_interrupt_t	interruptible,
311									uint64_t			deadline);
312
313#ifdef	KERNEL_PRIVATE
314
315extern boolean_t		lck_mtx_try_lock(
316									lck_mtx_t		*lck);
317
318extern void				mutex_pause(uint32_t);
319
320extern void 			lck_mtx_yield (
321									lck_mtx_t		*lck);
322
323#if defined(__i386__) || defined(__x86_64__)
324extern boolean_t		lck_mtx_try_lock_spin(
325									lck_mtx_t		*lck);
326
327extern boolean_t		lck_mtx_try_lock_spin_always(
328									lck_mtx_t		*lck);
329
330extern void			lck_mtx_lock_spin_always(
331									lck_mtx_t		*lck);
332
333extern void			lck_mtx_lock_spin(
334									lck_mtx_t		*lck);
335
336extern void			lck_mtx_convert_spin(
337									lck_mtx_t		*lck);
338
339#define lck_mtx_unlock_always(l)	lck_mtx_unlock(l)
340
341#else
342#define lck_mtx_try_lock_spin(l)	lck_mtx_try_lock(l)
343#define	lck_mtx_lock_spin(l)		lck_mtx_lock(l)
344#define lck_mtx_try_lock_spin_always(l)	lck_spin_try_lock(l)
345#define lck_mtx_lock_spin_always(l)	lck_spin_lock(l)
346#define lck_mtx_unlock_always(l)	lck_spin_unlock(l)
347#define	lck_mtx_convert_spin(l)		do {} while (0)
348#endif
349
350#endif	/* KERNEL_PRIVATE */
351
352extern void				lck_mtx_assert(
353									lck_mtx_t		*lck,
354									unsigned int	type);
355
356__END_DECLS
357
358#define	LCK_MTX_ASSERT_OWNED	0x01
359#define	LCK_MTX_ASSERT_NOTOWNED	0x02
360
361#ifdef	MACH_KERNEL_PRIVATE
362extern void				lck_mtx_lock_wait(
363									lck_mtx_t		*lck,
364									thread_t		holder);
365
366extern int				lck_mtx_lock_acquire(
367									lck_mtx_t		*lck);
368
369extern void				lck_mtx_unlock_wakeup(
370									lck_mtx_t		*lck,
371									thread_t		holder);
372extern void				lck_mtx_unlockspin_wakeup(
373							                lck_mtx_t		*lck);
374
375extern boolean_t		lck_mtx_ilk_unlock(
376									lck_mtx_t		*lck);
377
378#endif
379
380#define decl_lck_rw_data(class,name)     class lck_rw_t name;
381
382typedef unsigned int	 lck_rw_type_t;
383
384#define	LCK_RW_TYPE_SHARED			0x01
385#define	LCK_RW_TYPE_EXCLUSIVE		0x02
386
387#ifdef XNU_KERNEL_PRIVATE
388#define LCK_RW_ASSERT_SHARED	0x01
389#define LCK_RW_ASSERT_EXCLUSIVE	0x02
390#define LCK_RW_ASSERT_HELD		0x03
391#define LCK_RW_ASSERT_NOTHELD	0x04
392#endif
393
394__BEGIN_DECLS
395
396extern lck_rw_t			*lck_rw_alloc_init(
397									lck_grp_t		*grp,
398									lck_attr_t		*attr);
399
400extern void				lck_rw_init(
401									lck_rw_t		*lck,
402									lck_grp_t		*grp,
403									lck_attr_t		*attr);
404
405extern void				lck_rw_lock(
406									lck_rw_t		*lck,
407									lck_rw_type_t	lck_rw_type);
408
409extern void				lck_rw_unlock(
410									lck_rw_t		*lck,
411									lck_rw_type_t	lck_rw_type);
412
413extern void				lck_rw_lock_shared(
414									lck_rw_t		*lck);
415
416extern void				lck_rw_unlock_shared(
417									lck_rw_t		*lck);
418
419extern void				lck_rw_lock_exclusive(
420									lck_rw_t		*lck);
421
422extern void				lck_rw_unlock_exclusive(
423									lck_rw_t		*lck);
424
425#ifdef	XNU_KERNEL_PRIVATE
426/*
427 * CAUTION
428 * read-write locks do not have a concept of ownership, so lck_rw_assert()
429 * merely asserts that someone is holding the lock, not necessarily the caller.
430 */
431extern void				lck_rw_assert(
432									lck_rw_t		*lck,
433									unsigned int		type);
434
435extern void				lck_rw_clear_promotion(
436									thread_t		thread);
437#endif
438
439#ifdef	KERNEL_PRIVATE
440
441extern lck_rw_type_t	lck_rw_done(
442									lck_rw_t		*lck);
443#endif
444
445extern void				lck_rw_destroy(
446									lck_rw_t		*lck,
447									lck_grp_t		*grp);
448
449extern void				lck_rw_free(
450									lck_rw_t		*lck,
451									lck_grp_t		*grp);
452
453extern wait_result_t	lck_rw_sleep(
454									lck_rw_t			*lck,
455									lck_sleep_action_t	lck_sleep_action,
456									event_t				event,
457									wait_interrupt_t	interruptible);
458
459extern wait_result_t	lck_rw_sleep_deadline(
460									lck_rw_t			*lck,
461									lck_sleep_action_t	lck_sleep_action,
462									event_t				event,
463									wait_interrupt_t	interruptible,
464									uint64_t			deadline);
465
466extern boolean_t		lck_rw_lock_shared_to_exclusive(
467									lck_rw_t		*lck);
468
469extern void				lck_rw_lock_exclusive_to_shared(
470									lck_rw_t		*lck);
471
472extern boolean_t		lck_rw_try_lock(
473									lck_rw_t		*lck,
474									lck_rw_type_t	lck_rw_type);
475
476#ifdef	KERNEL_PRIVATE
477
478extern boolean_t		lck_rw_try_lock_shared(
479									lck_rw_t		*lck);
480
481extern boolean_t		lck_rw_try_lock_exclusive(
482									lck_rw_t		*lck);
483#endif
484
485__END_DECLS
486
487#endif /* _KERN_LOCKS_H_ */
488