1/*	$NetBSD: linux_wait_bit.c,v 1.5 2021/12/19 12:36:09 riastradh Exp $	*/
2
3/*-
4 * Copyright (c) 2018 The NetBSD Foundation, Inc.
5 * All rights reserved.
6 *
7 * This code is derived from software contributed to The NetBSD Foundation
8 * by Taylor R. Campbell.
9 *
10 * Redistribution and use in source and binary forms, with or without
11 * modification, are permitted provided that the following conditions
12 * are met:
13 * 1. Redistributions of source code must retain the above copyright
14 *    notice, this list of conditions and the following disclaimer.
15 * 2. Redistributions in binary form must reproduce the above copyright
16 *    notice, this list of conditions and the following disclaimer in the
17 *    documentation and/or other materials provided with the distribution.
18 *
19 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
20 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
21 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
22 * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
23 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
24 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
25 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
26 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
27 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
28 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
29 * POSSIBILITY OF SUCH DAMAGE.
30 */
31
32#include <sys/cdefs.h>
33__KERNEL_RCSID(0, "$NetBSD: linux_wait_bit.c,v 1.5 2021/12/19 12:36:09 riastradh Exp $");
34
35#include <sys/param.h>
36#include <sys/types.h>
37#include <sys/bitops.h>
38#include <sys/condvar.h>
39#include <sys/mutex.h>
40#include <sys/systm.h>
41
42#include <linux/bitops.h>
43#include <linux/sched.h>
44#include <linux/wait_bit.h>
45
46static struct {
47	struct waitbitentry {
48		kmutex_t	lock;
49		kcondvar_t	cv;
50	}		ent;
51	char		pad[CACHE_LINE_SIZE - sizeof(struct waitbitentry)];
52} waitbittab[PAGE_SIZE/CACHE_LINE_SIZE] __cacheline_aligned;
53CTASSERT(sizeof(waitbittab) == PAGE_SIZE);
54CTASSERT(sizeof(waitbittab[0]) == CACHE_LINE_SIZE);
55
56int
57linux_wait_bit_init(void)
58{
59	size_t i;
60
61	for (i = 0; i < __arraycount(waitbittab); i++) {
62		mutex_init(&waitbittab[i].ent.lock, MUTEX_DEFAULT, IPL_VM);
63		cv_init(&waitbittab[i].ent.cv, "waitbit");
64	}
65
66	return 0;
67}
68
69void
70linux_wait_bit_fini(void)
71{
72	size_t i;
73
74	for (i = 0; i < __arraycount(waitbittab); i++) {
75		cv_destroy(&waitbittab[i].ent.cv);
76		mutex_destroy(&waitbittab[i].ent.lock);
77	}
78}
79
80static inline size_t
81wait_bit_hash(const volatile unsigned long *bitmap, unsigned bit)
82{
83	/* Try to avoid cache line collisions.  */
84	const volatile unsigned long *word = bitmap + bit/(NBBY*sizeof(*word));
85
86	return ((uintptr_t)word >> ilog2(CACHE_LINE_SIZE)) %
87	    __arraycount(waitbittab);
88}
89
90static struct waitbitentry *
91wait_bit_enter(const volatile unsigned long *bitmap, unsigned bit)
92{
93	struct waitbitentry *wbe = &waitbittab[wait_bit_hash(bitmap, bit)].ent;
94
95	mutex_enter(&wbe->lock);
96
97	return wbe;
98}
99
100static void
101wait_bit_exit(struct waitbitentry *wbe)
102{
103
104	mutex_exit(&wbe->lock);
105}
106
107/*
108 * clear_and_wake_up_bit(bit, bitmap)
109 *
110 *	Clear the specified bit in the bitmap and wake any waiters in
111 *	wait_on_bit or wait_on_bit_timeout that were waiting for it to
112 *	clear.
113 */
114void
115clear_and_wake_up_bit(int bit, volatile unsigned long *bitmap)
116{
117	struct waitbitentry *wbe;
118
119	wbe = wait_bit_enter(bitmap, bit);
120	clear_bit(bit, bitmap);
121	cv_broadcast(&wbe->cv);
122	wait_bit_exit(wbe);
123}
124
125/*
126 * wait_on_bit(bitmap, bit, flags)
127 *
128 *	Wait for the specified bit in bitmap to be cleared.  Returns 0
129 *	on success, -EINTR on signal, unless flags has
130 *	TASK_UNINTERRUPTIBLE set.
131 */
132int
133wait_on_bit(const volatile unsigned long *bitmap, unsigned bit, int flags)
134{
135	struct waitbitentry *wbe;
136	int error, ret;
137
138	if (test_bit(bit, bitmap) == 0)
139		return 0;
140
141	wbe = wait_bit_enter(bitmap, bit);
142
143	while (test_bit(bit, bitmap)) {
144		if (flags & TASK_UNINTERRUPTIBLE) {
145			cv_wait(&wbe->cv, &wbe->lock);
146		} else {
147			error = cv_wait_sig(&wbe->cv, &wbe->lock);
148			if (error) {
149				/* cv_wait_sig can only fail on signal.  */
150				KASSERTMSG(error == EINTR || error == ERESTART,
151				    "error=%d", error);
152				ret = -EINTR;
153				goto out;
154			}
155		}
156	}
157
158	/* Bit is clear.  Return zero on success.   */
159	KASSERT(test_bit(bit, bitmap) == 0);
160	ret = 0;
161
162out:	KASSERT(test_bit(bit, bitmap) == 0 || ret != 0);
163	wait_bit_exit(wbe);
164	return ret;
165}
166
167/*
168 * wait_on_bit_timeout(bitmap, bit, flags, timeout)
169 *
170 *	Wait for the specified bit in bitmap to be cleared.  Returns 0
171 *	on success, -EINTR on signal unless flags has
172 *	TASK_UNINTERRUPTIBLE set, or -EAGAIN on timeout.
173 */
174int
175wait_on_bit_timeout(const volatile unsigned long *bitmap, unsigned bit,
176    int flags, unsigned long timeout)
177{
178	struct waitbitentry *wbe;
179	int error, ret;
180
181	if (test_bit(bit, bitmap) == 0)
182		return 0;
183
184	wbe = wait_bit_enter(bitmap, bit);
185
186	while (test_bit(bit, bitmap)) {
187		unsigned starttime, endtime;
188
189		if (timeout == 0) {
190			ret = -EAGAIN;
191			goto out;
192		}
193
194		starttime = getticks();
195		if (flags & TASK_UNINTERRUPTIBLE) {
196			error = cv_timedwait(&wbe->cv, &wbe->lock,
197			    MIN(timeout, INT_MAX/2));
198		} else {
199			error = cv_timedwait_sig(&wbe->cv, &wbe->lock,
200			    MIN(timeout, INT_MAX/2));
201		}
202		endtime = getticks();
203
204		/*
205		 * If we were interrupted or timed out, massage the
206		 * error return and stop here.
207		 */
208		if (error) {
209			KASSERTMSG((error == EINTR || error == ERESTART ||
210				error == EWOULDBLOCK), "error=%d", error);
211			if (error == EINTR || error == ERESTART) {
212				ret = -EINTR;
213			} else if (error == EWOULDBLOCK) {
214				ret = -EAGAIN;
215			} else {
216				panic("invalid error=%d", error);
217			}
218			goto out;
219		}
220
221		/* Otherwise, debit the time spent.  */
222		timeout -= MIN(timeout, (endtime - starttime));
223	}
224
225	/* Bit is clear.  Return zero on success.  */
226	KASSERT(test_bit(bit, bitmap) == 0);
227	ret = timeout;
228
229out:	KASSERT(test_bit(bit, bitmap) == 0 || ret != 0);
230	wait_bit_exit(wbe);
231	return ret;
232}
233