system/kernel/smp.cpp

117 	spinlock	*lock;
128 push_lock_caller(void* caller, spinlock* lock)
133 	sLastCaller[index].lock = lock;
138 find_lock_caller(spinlock* lock)
144 		if (sLastCaller[index].lock == lock)
164 	spinlock* lock = (spinlock*)(addr_t)address;
165 	kprintf("spinlock %p:\n", lock);
166 	bool locked = B_SPINLOCK_IS_LOCKED(lock);
168 		kprintf("  locked from %p\n", find_lock_caller(lock));
173 	kprintf("  failed try_acquire():		%d\n", lock->failed_try_acquire);
174 	kprintf("  total wait time:		%" B_PRIdBIGTIME "\n", lock->total_wait);
175 	kprintf("  total held time:		%" B_PRIdBIGTIME "\n", lock->total_held);
176 	kprintf("  last acquired at:		%" B_PRIdBIGTIME "\n", lock->last_acquired);
190 update_lock_contention(spinlock* lock, bigtime_t start)
193 	lock->last_acquired = now;
194 	lock->total_wait += (now - start);
199 update_lock_held(spinlock* lock)
201 	const bigtime_t held = (system_time() - lock->last_acquired);
202 	lock->total_held += held;
208 			lock, held, DEBUG_SPINLOCK_LATENCIES);
300 try_acquire_spinlock(spinlock* lock)
304 		panic("try_acquire_spinlock: attempt to acquire lock %p with "
305 			"interrupts enabled", lock);
309 	if (atomic_get_and_set(&lock->lock, 1) != 0) {
311 		atomic_add(&lock->failed_try_acquire, 1);
317 	update_lock_contention(lock, system_time());
321 	push_lock_caller(arch_debug_get_caller(), lock);
329 acquire_spinlock(spinlock* lock)
333 		panic("acquire_spinlock: attempt to acquire lock %p with interrupts "
334 			"enabled", lock);
345 			while (lock->lock != 0) {
350 						")", lock, find_lock_caller(lock), lock->lock);
353 						"for a long time (value: %" B_PRIx32 ")", lock,
354 						lock->lock);
360 				cpu_wait(&lock->lock, 0);
362 			if (atomic_get_and_set(&lock->lock, 1) == 0)
367 		update_lock_contention(lock, start);
371 		push_lock_caller(arch_debug_get_caller(), lock);
375 		lock->last_acquired = system_time();
378 		int32 oldValue = atomic_get_and_set(&lock->lock, 1);
380 			panic("acquire_spinlock: attempt to acquire lock %p twice on "
381 				"non-SMP system (last caller: %p, value %" B_PRIx32 ")", lock,
382 				find_lock_caller(lock), oldValue);
385 		push_lock_caller(arch_debug_get_caller(), lock);
392 acquire_spinlock_nocheck(spinlock *lock)
396 		panic("acquire_spinlock_nocheck: attempt to acquire lock %p with "
397 			"interrupts enabled", lock);
407 			while (lock->lock != 0) {
412 						B_PRIx32 ")", lock, find_lock_caller(lock), lock->lock);
416 						lock, lock->lock);
421 				cpu_wait(&lock->lock, 0);
424 			if (atomic_get_and_set(&lock->lock, 1) == 0)
429 		update_lock_contention(lock, start);
433 		push_lock_caller(arch_debug_get_caller(), lock);
437 		lock->last_acquired = system_time();
440 		int32 oldValue = atomic_get_and_set(&lock->lock, 1);
442 			panic("acquire_spinlock_nocheck: attempt to acquire lock %p twice "
444 				lock, find_lock_caller(lock), oldValue);
447 		push_lock_caller(arch_debug_get_caller(), lock);
455 acquire_spinlock_cpu(int32 currentCPU, spinlock *lock)
459 		panic("acquire_spinlock_cpu: attempt to acquire lock %p with "
460 			"interrupts enabled", lock);
470 			while (lock->lock != 0) {
475 						")", lock, find_lock_caller(lock), lock->lock);
478 						"%p for a long time (value: %" B_PRIx32 ")", lock,
479 						lock->lock);
485 				cpu_wait(&lock->lock, 0);
487 			if (atomic_get_and_set(&lock->lock, 1) == 0)
492 		update_lock_contention(lock, start);
496 		push_lock_caller(arch_debug_get_caller(), lock);
500 		lock->last_acquired = system_time();
503 		int32 oldValue = atomic_get_and_set(&lock->lock, 1);
505 			panic("acquire_spinlock_cpu(): attempt to acquire lock %p twice on "
506 				"non-SMP system (last caller: %p, value %" B_PRIx32 ")", lock,
507 				find_lock_caller(lock), oldValue);
510 		push_lock_caller(arch_debug_get_caller(), lock);
517 release_spinlock(spinlock *lock)
520 	update_lock_held(lock);
525 			panic("release_spinlock: attempt to release lock %p with "
526 				"interrupts enabled\n", lock);
530 		if (atomic_get_and_set(&lock->lock, 0) != 1)
531 			panic("release_spinlock: lock %p was already released\n", lock);
533 		atomic_set(&lock->lock, 0);
538 			panic("release_spinlock: attempt to release lock %p with "
539 				"interrupts enabled\n", lock);
541 		if (atomic_get_and_set(&lock->lock, 0) != 1)
542 			panic("release_spinlock: lock %p was already released\n", lock);
549 try_acquire_write_spinlock(rw_spinlock* lock)
553 		panic("try_acquire_write_spinlock: attempt to acquire lock %p with "
554 			"interrupts enabled", lock);
557 	if (sNumCPUs < 2 && lock->lock != 0) {
558 		panic("try_acquire_write_spinlock(): attempt to acquire lock %p twice "
559 			"on non-SMP system", lock);
563 	return atomic_test_and_set(&lock->lock, 1u << 31, 0) == 0;
568 acquire_write_spinlock(rw_spinlock* lock)
572 		panic("acquire_write_spinlock: attempt to acquire lock %p with "
573 			"interrupts enabled", lock);
580 		if (try_acquire_write_spinlock(lock))
583 		while (lock->lock != 0) {
586 					"for a long time!", lock);
591 			cpu_wait(&lock->lock, 0);
598 release_write_spinlock(rw_spinlock* lock)
601 	uint32 previous = atomic_get_and_set(&lock->lock, 0);
603 		panic("release_write_spinlock: lock %p was already released (value: "
604 			"%#" B_PRIx32 ")\n", lock, previous);
607 	atomic_set(&lock->lock, 0);
613 try_acquire_read_spinlock(rw_spinlock* lock)
617 		panic("try_acquire_read_spinlock: attempt to acquire lock %p with "
618 			"interrupts enabled", lock);
621 	if (sNumCPUs < 2 && lock->lock != 0) {
622 		panic("try_acquire_read_spinlock(): attempt to acquire lock %p twice "
623 			"on non-SMP system", lock);
627 	uint32 previous = atomic_add(&lock->lock, 1);
633 acquire_read_spinlock(rw_spinlock* lock)
637 		panic("acquire_read_spinlock: attempt to acquire lock %p with "
638 			"interrupts enabled", lock);
645 		if (try_acquire_read_spinlock(lock))
648 		while ((lock->lock & (1u << 31)) != 0) {
651 					"for a long time!", lock);
656 			cpu_wait(&lock->lock, 0);
663 release_read_spinlock(rw_spinlock* lock)
666 	uint32 previous = atomic_add(&lock->lock, -1);
668 		panic("release_read_spinlock: lock %p was already released (value:"
669 			" %#" B_PRIx32 ")\n", lock, previous);
672 	atomic_add(&lock->lock, -1);
679 try_acquire_write_seqlock(seqlock* lock)
681 	bool succeed = try_acquire_spinlock(&lock->lock);
683 		atomic_add((int32*)&lock->count, 1);
689 acquire_write_seqlock(seqlock* lock)
691 	acquire_spinlock(&lock->lock);
692 	atomic_add((int32*)&lock->count, 1);
697 release_write_seqlock(seqlock* lock)
699 	atomic_add((int32*)&lock->count, 1);
700 	release_spinlock(&lock->lock);
705 acquire_read_seqlock(seqlock* lock)
707 	return atomic_get((int32*)&lock->count);
712 release_read_seqlock(seqlock* lock, uint32 count)
716 	uint32 current = *(volatile int32*)&lock->count;
746 		// someone grabbed one while we were getting the lock,