| kern_sx.c (161337) | kern_sx.c (164159) |
|---|---|
| 1/*- 2 * Copyright (C) 2001 Jason Evans <jasone@freebsd.org>. All rights reserved. 3 * 4 * Redistribution and use in source and binary forms, with or without 5 * modification, are permitted provided that the following conditions 6 * are met: 7 * 1. Redistributions of source code must retain the above copyright 8 * notice(s), this list of conditions and the following disclaimer as --- 20 unchanged lines hidden (view full) --- 29 * Shared/exclusive locks. This implementation assures deterministic lock 30 * granting behavior, so that slocks and xlocks are interleaved. 31 * 32 * Priority propagation will not generally raise the priority of lock holders, 33 * so should not be relied upon in combination with sx locks. 34 */ 35 36#include <sys/cdefs.h> | 1/*- 2 * Copyright (C) 2001 Jason Evans <jasone@freebsd.org>. All rights reserved. 3 * 4 * Redistribution and use in source and binary forms, with or without 5 * modification, are permitted provided that the following conditions 6 * are met: 7 * 1. Redistributions of source code must retain the above copyright 8 * notice(s), this list of conditions and the following disclaimer as --- 20 unchanged lines hidden (view full) --- 29 * Shared/exclusive locks. This implementation assures deterministic lock 30 * granting behavior, so that slocks and xlocks are interleaved. 31 * 32 * Priority propagation will not generally raise the priority of lock holders, 33 * so should not be relied upon in combination with sx locks. 34 */ 35 36#include <sys/cdefs.h> |
| 37__FBSDID("$FreeBSD: head/sys/kern/kern_sx.c 161337 2006-08-15 18:29:01Z jhb $"); | 37__FBSDID("$FreeBSD: head/sys/kern/kern_sx.c 164159 2006-11-11 03:18:07Z kmacy $"); |
| 38 39#include "opt_ddb.h" 40 41#include <sys/param.h> 42#include <sys/systm.h> 43#include <sys/ktr.h> 44#include <sys/linker_set.h> 45#include <sys/condvar.h> 46#include <sys/lock.h> 47#include <sys/mutex.h> 48#include <sys/proc.h> 49#include <sys/sx.h> | 38 39#include "opt_ddb.h" 40 41#include <sys/param.h> 42#include <sys/systm.h> 43#include <sys/ktr.h> 44#include <sys/linker_set.h> 45#include <sys/condvar.h> 46#include <sys/lock.h> 47#include <sys/mutex.h> 48#include <sys/proc.h> 49#include <sys/sx.h> |
| 50#include <sys/lock_profile.h> |
|
| 50 51#ifdef DDB 52#include <ddb/ddb.h> 53 54static void db_show_sx(struct lock_object *lock); 55#endif 56 57struct lock_class lock_class_sx = { --- 22 unchanged lines hidden (view full) --- 80 81 sx->sx_lock = mtx_pool_find(mtxpool_lockbuilder, sx); 82 sx->sx_cnt = 0; 83 cv_init(&sx->sx_shrd_cv, description); 84 sx->sx_shrd_wcnt = 0; 85 cv_init(&sx->sx_excl_cv, description); 86 sx->sx_excl_wcnt = 0; 87 sx->sx_xholder = NULL; | 51 52#ifdef DDB 53#include <ddb/ddb.h> 54 55static void db_show_sx(struct lock_object *lock); 56#endif 57 58struct lock_class lock_class_sx = { --- 22 unchanged lines hidden (view full) --- 81 82 sx->sx_lock = mtx_pool_find(mtxpool_lockbuilder, sx); 83 sx->sx_cnt = 0; 84 cv_init(&sx->sx_shrd_cv, description); 85 sx->sx_shrd_wcnt = 0; 86 cv_init(&sx->sx_excl_cv, description); 87 sx->sx_excl_wcnt = 0; 88 sx->sx_xholder = NULL; |
| 89 lock_profile_object_init(&sx->sx_object, description); |
|
| 88 lock_init(&sx->sx_object, &lock_class_sx, description, NULL, 89 LO_WITNESS | LO_RECURSABLE | LO_SLEEPABLE | LO_UPGRADABLE); 90} 91 92void 93sx_destroy(struct sx *sx) 94{ 95 96 KASSERT((sx->sx_cnt == 0 && sx->sx_shrd_wcnt == 0 && sx->sx_excl_wcnt == 97 0), ("%s (%s): holders or waiters\n", __func__, 98 sx->sx_object.lo_name)); 99 100 sx->sx_lock = NULL; 101 cv_destroy(&sx->sx_shrd_cv); 102 cv_destroy(&sx->sx_excl_cv); | 90 lock_init(&sx->sx_object, &lock_class_sx, description, NULL, 91 LO_WITNESS | LO_RECURSABLE | LO_SLEEPABLE | LO_UPGRADABLE); 92} 93 94void 95sx_destroy(struct sx *sx) 96{ 97 98 KASSERT((sx->sx_cnt == 0 && sx->sx_shrd_wcnt == 0 && sx->sx_excl_wcnt == 99 0), ("%s (%s): holders or waiters\n", __func__, 100 sx->sx_object.lo_name)); 101 102 sx->sx_lock = NULL; 103 cv_destroy(&sx->sx_shrd_cv); 104 cv_destroy(&sx->sx_excl_cv); |
| 103 | 105 106 lock_profile_object_destroy(&sx->sx_object); |
| 104 lock_destroy(&sx->sx_object); 105} 106 107void 108_sx_slock(struct sx *sx, const char *file, int line) 109{ | 107 lock_destroy(&sx->sx_object); 108} 109 110void 111_sx_slock(struct sx *sx, const char *file, int line) 112{ |
| 113 uint64_t waittime = 0; 114 int contested; |
|
| 110 111 mtx_lock(sx->sx_lock); 112 KASSERT(sx->sx_xholder != curthread, 113 ("%s (%s): slock while xlock is held @ %s:%d\n", __func__, 114 sx->sx_object.lo_name, file, line)); 115 WITNESS_CHECKORDER(&sx->sx_object, LOP_NEWORDER, file, line); 116 117 /* 118 * Loop in case we lose the race for lock acquisition. 119 */ | 115 116 mtx_lock(sx->sx_lock); 117 KASSERT(sx->sx_xholder != curthread, 118 ("%s (%s): slock while xlock is held @ %s:%d\n", __func__, 119 sx->sx_object.lo_name, file, line)); 120 WITNESS_CHECKORDER(&sx->sx_object, LOP_NEWORDER, file, line); 121 122 /* 123 * Loop in case we lose the race for lock acquisition. 124 */ |
| 125 if (sx->sx_cnt < 0) 126 lock_profile_waitstart(&waittime); |
|
| 120 while (sx->sx_cnt < 0) { 121 sx->sx_shrd_wcnt++; | 127 while (sx->sx_cnt < 0) { 128 sx->sx_shrd_wcnt++; |
| 129 lock_profile_obtain_lock_failed(&sx->sx_object, &contested); |
|
| 122 cv_wait(&sx->sx_shrd_cv, sx->sx_lock); 123 sx->sx_shrd_wcnt--; 124 } 125 126 /* Acquire a shared lock. */ 127 sx->sx_cnt++; 128 | 130 cv_wait(&sx->sx_shrd_cv, sx->sx_lock); 131 sx->sx_shrd_wcnt--; 132 } 133 134 /* Acquire a shared lock. */ 135 sx->sx_cnt++; 136 |
| 137 if (sx->sx_cnt == 1) 138 lock_profile_obtain_lock_success(&sx->sx_object, waittime, file, line); 139 |
|
| 129 LOCK_LOG_LOCK("SLOCK", &sx->sx_object, 0, 0, file, line); 130 WITNESS_LOCK(&sx->sx_object, 0, file, line); 131 curthread->td_locks++; 132 133 mtx_unlock(sx->sx_lock); 134} 135 136int --- 13 unchanged lines hidden (view full) --- 150 mtx_unlock(sx->sx_lock); 151 return (0); 152 } 153} 154 155void 156_sx_xlock(struct sx *sx, const char *file, int line) 157{ | 140 LOCK_LOG_LOCK("SLOCK", &sx->sx_object, 0, 0, file, line); 141 WITNESS_LOCK(&sx->sx_object, 0, file, line); 142 curthread->td_locks++; 143 144 mtx_unlock(sx->sx_lock); 145} 146 147int --- 13 unchanged lines hidden (view full) --- 161 mtx_unlock(sx->sx_lock); 162 return (0); 163 } 164} 165 166void 167_sx_xlock(struct sx *sx, const char *file, int line) 168{ |
| 169 int contested; 170 uint64_t waittime = 0; |
|
| 158 159 mtx_lock(sx->sx_lock); 160 161 /* 162 * With sx locks, we're absolutely not permitted to recurse on 163 * xlocks, as it is fatal (deadlock). Normally, recursion is handled 164 * by WITNESS, but as it is not semantically correct to hold the 165 * xlock while in here, we consider it API abuse and put it under 166 * INVARIANTS. 167 */ 168 KASSERT(sx->sx_xholder != curthread, 169 ("%s (%s): xlock already held @ %s:%d", __func__, 170 sx->sx_object.lo_name, file, line)); 171 WITNESS_CHECKORDER(&sx->sx_object, LOP_NEWORDER | LOP_EXCLUSIVE, file, 172 line); 173 | 171 172 mtx_lock(sx->sx_lock); 173 174 /* 175 * With sx locks, we're absolutely not permitted to recurse on 176 * xlocks, as it is fatal (deadlock). Normally, recursion is handled 177 * by WITNESS, but as it is not semantically correct to hold the 178 * xlock while in here, we consider it API abuse and put it under 179 * INVARIANTS. 180 */ 181 KASSERT(sx->sx_xholder != curthread, 182 ("%s (%s): xlock already held @ %s:%d", __func__, 183 sx->sx_object.lo_name, file, line)); 184 WITNESS_CHECKORDER(&sx->sx_object, LOP_NEWORDER | LOP_EXCLUSIVE, file, 185 line); 186 |
| 187 if (sx->sx_cnt) 188 lock_profile_waitstart(&waittime); |
|
| 174 /* Loop in case we lose the race for lock acquisition. */ 175 while (sx->sx_cnt != 0) { 176 sx->sx_excl_wcnt++; | 189 /* Loop in case we lose the race for lock acquisition. */ 190 while (sx->sx_cnt != 0) { 191 sx->sx_excl_wcnt++; |
| 192 lock_profile_obtain_lock_failed(&sx->sx_object, &contested); |
|
| 177 cv_wait(&sx->sx_excl_cv, sx->sx_lock); 178 sx->sx_excl_wcnt--; 179 } 180 181 MPASS(sx->sx_cnt == 0); 182 183 /* Acquire an exclusive lock. */ 184 sx->sx_cnt--; 185 sx->sx_xholder = curthread; 186 | 193 cv_wait(&sx->sx_excl_cv, sx->sx_lock); 194 sx->sx_excl_wcnt--; 195 } 196 197 MPASS(sx->sx_cnt == 0); 198 199 /* Acquire an exclusive lock. */ 200 sx->sx_cnt--; 201 sx->sx_xholder = curthread; 202 |
| 203 lock_profile_obtain_lock_success(&sx->sx_object, waittime, file, line); |
|
| 187 LOCK_LOG_LOCK("XLOCK", &sx->sx_object, 0, 0, file, line); 188 WITNESS_LOCK(&sx->sx_object, LOP_EXCLUSIVE, file, line); 189 curthread->td_locks++; 190 191 mtx_unlock(sx->sx_lock); 192} 193 194int --- 25 unchanged lines hidden (view full) --- 220 mtx_lock(sx->sx_lock); 221 222 curthread->td_locks--; 223 WITNESS_UNLOCK(&sx->sx_object, 0, file, line); 224 225 /* Release. */ 226 sx->sx_cnt--; 227 | 204 LOCK_LOG_LOCK("XLOCK", &sx->sx_object, 0, 0, file, line); 205 WITNESS_LOCK(&sx->sx_object, LOP_EXCLUSIVE, file, line); 206 curthread->td_locks++; 207 208 mtx_unlock(sx->sx_lock); 209} 210 211int --- 25 unchanged lines hidden (view full) --- 237 mtx_lock(sx->sx_lock); 238 239 curthread->td_locks--; 240 WITNESS_UNLOCK(&sx->sx_object, 0, file, line); 241 242 /* Release. */ 243 sx->sx_cnt--; 244 |
| 245 if (sx->sx_cnt == 0) 246 lock_profile_release_lock(&sx->sx_object); |
|
| 228 /* 229 * If we just released the last shared lock, wake any waiters up, giving 230 * exclusive lockers precedence. In order to make sure that exclusive 231 * lockers won't be blocked forever, don't wake shared lock waiters if 232 * there are exclusive lock waiters. 233 */ 234 if (sx->sx_excl_wcnt > 0) { 235 if (sx->sx_cnt == 0) --- 16 unchanged lines hidden (view full) --- 252 253 curthread->td_locks--; 254 WITNESS_UNLOCK(&sx->sx_object, LOP_EXCLUSIVE, file, line); 255 256 /* Release. */ 257 sx->sx_cnt++; 258 sx->sx_xholder = NULL; 259 | 247 /* 248 * If we just released the last shared lock, wake any waiters up, giving 249 * exclusive lockers precedence. In order to make sure that exclusive 250 * lockers won't be blocked forever, don't wake shared lock waiters if 251 * there are exclusive lock waiters. 252 */ 253 if (sx->sx_excl_wcnt > 0) { 254 if (sx->sx_cnt == 0) --- 16 unchanged lines hidden (view full) --- 271 272 curthread->td_locks--; 273 WITNESS_UNLOCK(&sx->sx_object, LOP_EXCLUSIVE, file, line); 274 275 /* Release. */ 276 sx->sx_cnt++; 277 sx->sx_xholder = NULL; 278 |
| 279 lock_profile_release_lock(&sx->sx_object); |
|
| 260 /* 261 * Wake up waiters if there are any. Give precedence to slock waiters. 262 */ 263 if (sx->sx_shrd_wcnt > 0) 264 cv_broadcast(&sx->sx_shrd_cv); 265 else if (sx->sx_excl_wcnt > 0) 266 cv_signal(&sx->sx_excl_cv); 267 --- 184 unchanged lines hidden --- | 280 /* 281 * Wake up waiters if there are any. Give precedence to slock waiters. 282 */ 283 if (sx->sx_shrd_wcnt > 0) 284 cv_broadcast(&sx->sx_shrd_cv); 285 else if (sx->sx_excl_wcnt > 0) 286 cv_signal(&sx->sx_excl_cv); 287 --- 184 unchanged lines hidden --- |