osfmk/i386/i386_lock.s

149  *  Routines for general lock debugging.
153  * Checks for expected lock types and calls "panic" on
172  * (since a mutex lock may context switch, holding a simplelock
203 2:	String	"Attempt to recursively lock a non-recursive lock";	\
260  * a pointer into the lock code function body. At that
290 	movl	lck,4(%esp)	/* copy lock pointer to arg 1 */ ; \
353  *	Initialize a hardware lock.
356 	LOAD_HW_LOCK_REGISTER		/* fetch lock pointer */
357 	HW_LOCK_MOV_WORD $0, (HW_LOCK_REGISTER)		/* clear the lock */
364  *	Initialize a hardware byte lock.
367 	LOAD_HW_LOCK_REGISTER		/* fetch lock pointer */
368 	movb $0, (HW_LOCK_REGISTER)		/* clear the lock */
374  *	Acquire lock, spinning until it becomes available.
378 	LOAD_HW_LOCK_REGISTER		/* fetch lock pointer */
384 	test	HW_LOCK_EXAM_REGISTER,HW_LOCK_EXAM_REGISTER		/* lock locked? */
386 	lock; cmpxchg	HW_LOCK_THREAD_REGISTER,(HW_LOCK_REGISTER)	/* try to acquire the HW lock */
397  *	Acquire byte sized lock operand, spinning until it becomes available.
402 	LOAD_HW_LOCK_REGISTER		/* Load lock pointer */
404 	movl	$1, %ecx		/* Set lock value */
407 	testb	%al,%al			/* lock locked? */
409 	lock; cmpxchg	%cl,(HW_LOCK_REGISTER)	/* attempt atomic compare exchange */
419  *	Acquire lock, spinning until it becomes available or timeout.
424 	LOAD_HW_LOCK_REGISTER		/* fetch lock pointer */
428 	 * Attempt to grab the lock immediately
434 	test	HW_LOCK_EXAM_REGISTER,HW_LOCK_EXAM_REGISTER		/* lock locked? */
436 	lock; cmpxchg	HW_LOCK_THREAD_REGISTER,(HW_LOCK_REGISTER)	/* try to acquire the HW lock */
444 	 * Failed to get the lock so set the timeout
445 	 * and then spin re-checking the lock but pausing
460 	mov	%edi, %edx		/* load lock back into %edx */
472 	 * The inner-loop spin to look for the lock being freed.
481 	mov	(HW_LOCK_REGISTER),HW_LOCK_EXAM_REGISTER		/* spin checking lock value in cache */
520 	 * Here to try to grab the lock that now appears to be free
524 	lock; cmpxchg	HW_LOCK_THREAD_REGISTER,(HW_LOCK_REGISTER)	/* try to acquire the HW lock */
538  *	Unconditionally release lock.
542 	LOAD_HW_LOCK_REGISTER		/* fetch lock pointer */
543 	HW_LOCK_MOV_WORD $0, (HW_LOCK_REGISTER)		/* clear the lock */
550  *	Unconditionally release byte sized lock operand.
555 	LOAD_HW_LOCK_REGISTER		/* Load lock pointer */
556 	movb $0, (HW_LOCK_REGISTER)		/* Clear the lock byte */
565 	LOAD_HW_LOCK_REGISTER		/* fetch lock pointer */
572 	lock; cmpxchg	HW_LOCK_THREAD_REGISTER,(HW_LOCK_REGISTER)	/* try to acquire the HW lock */
589 	LOAD_HW_LOCK_REGISTER		/* fetch lock pointer */
590 	mov	(HW_LOCK_REGISTER),HW_LOCK_EXAM_REGISTER		/* check lock value */
598  * Reader-writer lock fastpaths. These currently exist for the
599  * shared lock acquire, the exclusive lock acquire, the shared to
650 	lock
663 	Fall thru when patched, counting on lock pointer in LCK_RW_REGISTER
688 	jne	3f			/* lock is busy */
692 	lock
705     /* Fall thru when patched, counting on lock pointer in LCK_RW_REGISTER  */
734 	lock
769 	lock
781     /* Fall thru when patched, counting on lock pointer in LCK_RW_REGISTER  */
797  *		Tries to get a write lock.
799  *		Returns FALSE if the lock is not held on return.
812 	lock
825     /* Fall thru when patched, counting on lock pointer in LCK_RW_REGISTER  */
849  * 	a read count on the lock that we have not yet released
870 	lock
886     /* Fall thru when patched, counting on lock pointer in LCK_RW_REGISTER  */
899 	lock
910 						/* lock is alread in %rdi */
924 	.asciz  "Releasing non-exclusive RW lock without a reader refcount!"
939 	testl	$(LCK_RW_SHARED_MASK), %ecx	/* if reader count == 0, must be exclusive lock */
952 	je	8f				/* lock is not 'owned', go panic */
971 	lock
982 				/* lock is in %rdi already */
1031 	lock
1066 	lock
1103  * recorded for a single lock acquire attempt via lck_mtx_lock
1115 #define LOCK_IF_ATOMIC_STAT_UPDATES lock
1124  * Eventually the lock owner is loaded into a register and examined.
1349  *	Takes the address of a lock, and an assertion type as parameters.
1351  *	parameter: either the lock is held by the current thread, and the
1358         LOAD_LMTX_REG(B_ARG0)	                   	/* Load lock address */
1417 	LOAD_LMTX_REG(B_ARG0)           /* fetch lock pointer */
1421 	LOAD_LMTX_REG(B_ARG0)		/* fetch lock pointer */
1433 	lock
1442 	incl	TH_MUTEX_COUNT(LMTX_A_REG)	/* lock statistic */
1452 	/* inherit lock pointer in LMTX_REG above */
1490 	LOAD_LMTX_REG(B_ARG0)		/* fetch lock pointer */
1502 	lock
1511 	incl	TH_MUTEX_COUNT(LMTX_A_REG)	/* lock statistic */
1531 	/* inherit lock pointer in LMTX_REG above */
1541 	/* inherit lock pointer in LMTX_REG above */
1602 	lock
1616 	incl	TH_MUTEX_COUNT(LMTX_A_REG)	/* lock statistic */
1647 	LOAD_LMTX_REG(B_ARG0)			/* fetch lock pointer */
1657 	lock
1666 	incl	TH_MUTEX_COUNT(LMTX_A_REG)	/* lock statistic */
1677 	/* inherit lock pointer in LMTX_REG above */
1710 	LOAD_LMTX_REG(B_ARG0)			/* fetch lock pointer */
1720 	lock
1729 	incl	TH_MUTEX_COUNT(LMTX_A_REG)	/* lock statistic */
1749 	/* inherit lock pointer in LMTX_REG from above */
1793 	LOAD_LMTX_REG(B_ARG0)			/* fetch lock pointer */
1821 	LOAD_LMTX_REG(B_ARG0)			/* fetch lock pointer */
1831 	LOAD_LMTX_REG(B_ARG0)			/* fetch lock pointer */
1850 	lock
1867 	/* perform lock statistics after drop to prevent delay */
1871 	decl	TH_MUTEX_COUNT(LMTX_A_REG)	/* lock statistic */
1890 	/* inherit lock pointer in LMTX_REG from above */
1900 	/* inherit lock pointer in LMTX_REG from above */
1924 	LOAD_LMTX_REG(L_ARG0)			/* fetch lock pointer - no indirection here */
1935 	LOAD_LMTX_REG(L_ARG0)			/* fetch lock pointer - no indirection here */
1946 	lock
1955 	incl	TH_MUTEX_COUNT(LMTX_A_REG)	/* lock statistic */
1985 	lock
2114 	lock
2121 	lock
2130 	lock
2139 	lock
2150 	lock
2161 	lock
2171 	lock
2182 	lock
2195 	lock
2205 	lock
2218 	lock
2225 	lock
2230 	lock
2237 	lock
2244 	lock
2254 	lock
2270 	lock 	xaddl %eax, (%rdi)		/* Atomic exchange and add */
2283 	lock	xaddl %eax, (%rdi)		/* Atomic exchange and add */
2298 	lock	cmpxchgl %edx, (%rdi)	/* Atomic CAS */
2314 	lock
2330 	lock	cmpxchgl %edx, (%rdi)	/* Atomic CAS */
2346 	lock	andl	%esi, (%rdi)		/* Atomic OR */