kernel/locking/rwsem.c

370 rwsem_add_waiter(struct rw_semaphore *sem, struct rwsem_waiter *waiter)
373 	list_add_tail(&waiter->list, &sem->wait_list);
378  * Remove a waiter from the wait_list and clear flags.
386 rwsem_del_waiter(struct rw_semaphore *sem, struct rwsem_waiter *waiter)
389 	list_del(&waiter->list);
415 	struct rwsem_waiter *waiter, *tmp;
422 	 * Take a peek at the queue head waiter such that we can determine
425 	waiter = rwsem_first_waiter(sem);
427 	if (waiter->type == RWSEM_WAITING_FOR_WRITE) {
436 			wake_q_add(wake_q, waiter->task);
465 			if (time_after(jiffies, waiter->timeout)) {
470 				waiter->handoff_set = true;
482 		owner = waiter->task;
493 	 * reader phase (first waiter is a reader), all readers are eligible
500 	 * is because the to-be-woken waiter may not have slept yet. So it
501 	 * may see waiter->task got cleared, finish its critical section and
506 	 * 2) For each waiters in the new list, clear waiter->task and
510 	list_for_each_entry_safe(waiter, tmp, &sem->wait_list, list) {
511 		if (waiter->type == RWSEM_WAITING_FOR_WRITE)
515 		list_move_tail(&waiter->list, &wlist);
549 	list_for_each_entry_safe(waiter, tmp, &wlist, list) {
552 		tsk = waiter->task;
557 		 * waiter to nil such that rwsem_down_read_slowpath() cannot
561 		smp_store_release(&waiter->task, NULL);
564 		 * after setting the reader waiter to nil.
571  * Remove a waiter and try to wake up other waiters in the wait queue
577 rwsem_del_wake_waiter(struct rw_semaphore *sem, struct rwsem_waiter *waiter,
581 	bool first = rwsem_first_waiter(sem) == waiter;
586 	 * If the wait_list isn't empty and the waiter to be deleted is
587 	 * the first waiter, we wake up the remaining waiters as they may
590 	if (rwsem_del_waiter(sem, waiter) && first)
605 					struct rwsem_waiter *waiter)
619 			 * waiter is the one that set it. Otherwisee, we
622 			if (first->handoff_set && (waiter != first))
630 			 * A waiter (first or not) can set the handoff bit
634 			if (has_handoff || (!rt_task(waiter->task) &&
635 					    !time_after(jiffies, waiter->timeout)))
650 	 * the handoff bit. Only the first waiter can have its handoff_set
663 	list_del(&waiter->list);
968  * Prepare to wake up waiter(s) in the wait queue by putting them into the
971  * front waiter otherwise.
1000 	struct rwsem_waiter waiter;
1035 	waiter.task = current;
1036 	waiter.type = RWSEM_WAITING_FOR_READ;
1037 	waiter.timeout = jiffies + RWSEM_WAIT_TIMEOUT;
1038 	waiter.handoff_set = false;
1058 	rwsem_add_waiter(sem, &waiter);
1074 		if (!smp_load_acquire(&waiter.task)) {
1080 			if (waiter.task)
1096 	rwsem_del_wake_waiter(sem, &waiter, &wake_q);
1109 	struct rwsem_waiter waiter;
1122 	waiter.task = current;
1123 	waiter.type = RWSEM_WAITING_FOR_WRITE;
1124 	waiter.timeout = jiffies + RWSEM_WAIT_TIMEOUT;
1125 	waiter.handoff_set = false;
1128 	rwsem_add_waiter(sem, &waiter);
1131 	if (rwsem_first_waiter(sem) != &waiter) {
1152 		if (rwsem_try_write_lock(sem, &waiter)) {
1170 		if (waiter.handoff_set) {
1193 	rwsem_del_wake_waiter(sem, &waiter, &wake_q);
1200  * handle waking up a waiter on the semaphore