1/* $NetBSD: linux_wait_bit.c,v 1.5 2021/12/19 12:36:09 riastradh Exp $ */ 2 3/*- 4 * Copyright (c) 2018 The NetBSD Foundation, Inc. 5 * All rights reserved. 6 * 7 * This code is derived from software contributed to The NetBSD Foundation 8 * by Taylor R. Campbell. 9 * 10 * Redistribution and use in source and binary forms, with or without 11 * modification, are permitted provided that the following conditions 12 * are met: 13 * 1. Redistributions of source code must retain the above copyright 14 * notice, this list of conditions and the following disclaimer. 15 * 2. Redistributions in binary form must reproduce the above copyright 16 * notice, this list of conditions and the following disclaimer in the 17 * documentation and/or other materials provided with the distribution. 18 * 19 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS 20 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 21 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 22 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS 23 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 24 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 25 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 26 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 27 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 28 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 29 * POSSIBILITY OF SUCH DAMAGE. 30 */ 31 32#include <sys/cdefs.h> 33__KERNEL_RCSID(0, "$NetBSD: linux_wait_bit.c,v 1.5 2021/12/19 12:36:09 riastradh Exp $"); 34 35#include <sys/param.h> 36#include <sys/types.h> 37#include <sys/bitops.h> 38#include <sys/condvar.h> 39#include <sys/mutex.h> 40#include <sys/systm.h> 41 42#include <linux/bitops.h> 43#include <linux/sched.h> 44#include <linux/wait_bit.h> 45 46static struct { 47 struct waitbitentry { 48 kmutex_t lock; 49 kcondvar_t cv; 50 } ent; 51 char pad[CACHE_LINE_SIZE - sizeof(struct waitbitentry)]; 52} waitbittab[PAGE_SIZE/CACHE_LINE_SIZE] __cacheline_aligned; 53CTASSERT(sizeof(waitbittab) == PAGE_SIZE); 54CTASSERT(sizeof(waitbittab[0]) == CACHE_LINE_SIZE); 55 56int 57linux_wait_bit_init(void) 58{ 59 size_t i; 60 61 for (i = 0; i < __arraycount(waitbittab); i++) { 62 mutex_init(&waitbittab[i].ent.lock, MUTEX_DEFAULT, IPL_VM); 63 cv_init(&waitbittab[i].ent.cv, "waitbit"); 64 } 65 66 return 0; 67} 68 69void 70linux_wait_bit_fini(void) 71{ 72 size_t i; 73 74 for (i = 0; i < __arraycount(waitbittab); i++) { 75 cv_destroy(&waitbittab[i].ent.cv); 76 mutex_destroy(&waitbittab[i].ent.lock); 77 } 78} 79 80static inline size_t 81wait_bit_hash(const volatile unsigned long *bitmap, unsigned bit) 82{ 83 /* Try to avoid cache line collisions. */ 84 const volatile unsigned long *word = bitmap + bit/(NBBY*sizeof(*word)); 85 86 return ((uintptr_t)word >> ilog2(CACHE_LINE_SIZE)) % 87 __arraycount(waitbittab); 88} 89 90static struct waitbitentry * 91wait_bit_enter(const volatile unsigned long *bitmap, unsigned bit) 92{ 93 struct waitbitentry *wbe = &waitbittab[wait_bit_hash(bitmap, bit)].ent; 94 95 mutex_enter(&wbe->lock); 96 97 return wbe; 98} 99 100static void 101wait_bit_exit(struct waitbitentry *wbe) 102{ 103 104 mutex_exit(&wbe->lock); 105} 106 107/* 108 * clear_and_wake_up_bit(bit, bitmap) 109 * 110 * Clear the specified bit in the bitmap and wake any waiters in 111 * wait_on_bit or wait_on_bit_timeout that were waiting for it to 112 * clear. 113 */ 114void 115clear_and_wake_up_bit(int bit, volatile unsigned long *bitmap) 116{ 117 struct waitbitentry *wbe; 118 119 wbe = wait_bit_enter(bitmap, bit); 120 clear_bit(bit, bitmap); 121 cv_broadcast(&wbe->cv); 122 wait_bit_exit(wbe); 123} 124 125/* 126 * wait_on_bit(bitmap, bit, flags) 127 * 128 * Wait for the specified bit in bitmap to be cleared. Returns 0 129 * on success, -EINTR on signal, unless flags has 130 * TASK_UNINTERRUPTIBLE set. 131 */ 132int 133wait_on_bit(const volatile unsigned long *bitmap, unsigned bit, int flags) 134{ 135 struct waitbitentry *wbe; 136 int error, ret; 137 138 if (test_bit(bit, bitmap) == 0) 139 return 0; 140 141 wbe = wait_bit_enter(bitmap, bit); 142 143 while (test_bit(bit, bitmap)) { 144 if (flags & TASK_UNINTERRUPTIBLE) { 145 cv_wait(&wbe->cv, &wbe->lock); 146 } else { 147 error = cv_wait_sig(&wbe->cv, &wbe->lock); 148 if (error) { 149 /* cv_wait_sig can only fail on signal. */ 150 KASSERTMSG(error == EINTR || error == ERESTART, 151 "error=%d", error); 152 ret = -EINTR; 153 goto out; 154 } 155 } 156 } 157 158 /* Bit is clear. Return zero on success. */ 159 KASSERT(test_bit(bit, bitmap) == 0); 160 ret = 0; 161 162out: KASSERT(test_bit(bit, bitmap) == 0 || ret != 0); 163 wait_bit_exit(wbe); 164 return ret; 165} 166 167/* 168 * wait_on_bit_timeout(bitmap, bit, flags, timeout) 169 * 170 * Wait for the specified bit in bitmap to be cleared. Returns 0 171 * on success, -EINTR on signal unless flags has 172 * TASK_UNINTERRUPTIBLE set, or -EAGAIN on timeout. 173 */ 174int 175wait_on_bit_timeout(const volatile unsigned long *bitmap, unsigned bit, 176 int flags, unsigned long timeout) 177{ 178 struct waitbitentry *wbe; 179 int error, ret; 180 181 if (test_bit(bit, bitmap) == 0) 182 return 0; 183 184 wbe = wait_bit_enter(bitmap, bit); 185 186 while (test_bit(bit, bitmap)) { 187 unsigned starttime, endtime; 188 189 if (timeout == 0) { 190 ret = -EAGAIN; 191 goto out; 192 } 193 194 starttime = getticks(); 195 if (flags & TASK_UNINTERRUPTIBLE) { 196 error = cv_timedwait(&wbe->cv, &wbe->lock, 197 MIN(timeout, INT_MAX/2)); 198 } else { 199 error = cv_timedwait_sig(&wbe->cv, &wbe->lock, 200 MIN(timeout, INT_MAX/2)); 201 } 202 endtime = getticks(); 203 204 /* 205 * If we were interrupted or timed out, massage the 206 * error return and stop here. 207 */ 208 if (error) { 209 KASSERTMSG((error == EINTR || error == ERESTART || 210 error == EWOULDBLOCK), "error=%d", error); 211 if (error == EINTR || error == ERESTART) { 212 ret = -EINTR; 213 } else if (error == EWOULDBLOCK) { 214 ret = -EAGAIN; 215 } else { 216 panic("invalid error=%d", error); 217 } 218 goto out; 219 } 220 221 /* Otherwise, debit the time spent. */ 222 timeout -= MIN(timeout, (endtime - starttime)); 223 } 224 225 /* Bit is clear. Return zero on success. */ 226 KASSERT(test_bit(bit, bitmap) == 0); 227 ret = timeout; 228 229out: KASSERT(test_bit(bit, bitmap) == 0 || ret != 0); 230 wait_bit_exit(wbe); 231 return ret; 232} 233