Deleted Added
sdiff udiff text old ( 168333 ) new ( 169394 )
full compact
kern_sx.c (168333) kern_sx.c (169394)
1/*-
2 * Copyright (c) 2007 Attilio Rao <attilio@freebsd.org>
3 * Copyright (c) 2001 Jason Evans <jasone@freebsd.org>
4 * All rights reserved.
5 *
6 * Redistribution and use in source and binary forms, with or without
7 * modification, are permitted provided that the following conditions
8 * are met:

--- 26 unchanged lines hidden (view full) ---

35 * Priority propagation will not generally raise the priority of lock holders,
36 * so should not be relied upon in combination with sx locks.
37 */
38
39#include "opt_adaptive_sx.h"
40#include "opt_ddb.h"
41
42#include <sys/cdefs.h>
1/*-
2 * Copyright (c) 2007 Attilio Rao <attilio@freebsd.org>
3 * Copyright (c) 2001 Jason Evans <jasone@freebsd.org>
4 * All rights reserved.
5 *
6 * Redistribution and use in source and binary forms, with or without
7 * modification, are permitted provided that the following conditions
8 * are met:

--- 26 unchanged lines hidden (view full) ---

35 * Priority propagation will not generally raise the priority of lock holders,
36 * so should not be relied upon in combination with sx locks.
37 */
38
39#include "opt_adaptive_sx.h"
40#include "opt_ddb.h"
41
42#include <sys/cdefs.h>
43__FBSDID("$FreeBSD: head/sys/kern/kern_sx.c 168333 2007-04-04 00:11:22Z kmacy $");
43__FBSDID("$FreeBSD: head/sys/kern/kern_sx.c 169394 2007-05-08 21:51:37Z jhb $");
44
45#include <sys/param.h>
46#include <sys/ktr.h>
47#include <sys/lock.h>
48#include <sys/lock_profile.h>
49#include <sys/mutex.h>
50#include <sys/proc.h>
51#include <sys/sleepqueue.h>

--- 136 unchanged lines hidden (view full) ---

188}
189
190void
191sx_destroy(struct sx *sx)
192{
193
194 KASSERT(sx->sx_lock == SX_LOCK_UNLOCKED, ("sx lock still held"));
195 KASSERT(sx->sx_recurse == 0, ("sx lock still recursed"));
44
45#include <sys/param.h>
46#include <sys/ktr.h>
47#include <sys/lock.h>
48#include <sys/lock_profile.h>
49#include <sys/mutex.h>
50#include <sys/proc.h>
51#include <sys/sleepqueue.h>

--- 136 unchanged lines hidden (view full) ---

188}
189
190void
191sx_destroy(struct sx *sx)
192{
193
194 KASSERT(sx->sx_lock == SX_LOCK_UNLOCKED, ("sx lock still held"));
195 KASSERT(sx->sx_recurse == 0, ("sx lock still recursed"));
196 sx->sx_lock = SX_LOCK_DESTROYED;
196 lock_profile_object_destroy(&sx->lock_object);
197 lock_destroy(&sx->lock_object);
198}
199
200void
201_sx_slock(struct sx *sx, const char *file, int line)
202{
203
204 MPASS(curthread != NULL);
197 lock_profile_object_destroy(&sx->lock_object);
198 lock_destroy(&sx->lock_object);
199}
200
201void
202_sx_slock(struct sx *sx, const char *file, int line)
203{
204
205 MPASS(curthread != NULL);
206 KASSERT(sx->sx_lock != SX_LOCK_DESTROYED,
207 ("sx_slock() of destroyed sx @ %s:%d", file, line));
205 WITNESS_CHECKORDER(&sx->lock_object, LOP_NEWORDER, file, line);
206 __sx_slock(sx, file, line);
207 LOCK_LOG_LOCK("SLOCK", &sx->lock_object, 0, 0, file, line);
208 WITNESS_LOCK(&sx->lock_object, 0, file, line);
209 curthread->td_locks++;
210}
211
212int
213_sx_try_slock(struct sx *sx, const char *file, int line)
214{
215 uintptr_t x;
216
217 x = sx->sx_lock;
208 WITNESS_CHECKORDER(&sx->lock_object, LOP_NEWORDER, file, line);
209 __sx_slock(sx, file, line);
210 LOCK_LOG_LOCK("SLOCK", &sx->lock_object, 0, 0, file, line);
211 WITNESS_LOCK(&sx->lock_object, 0, file, line);
212 curthread->td_locks++;
213}
214
215int
216_sx_try_slock(struct sx *sx, const char *file, int line)
217{
218 uintptr_t x;
219
220 x = sx->sx_lock;
221 KASSERT(x != SX_LOCK_DESTROYED,
222 ("sx_try_slock() of destroyed sx @ %s:%d", file, line));
218 if ((x & SX_LOCK_SHARED) && atomic_cmpset_acq_ptr(&sx->sx_lock, x,
219 x + SX_ONE_SHARER)) {
220 LOCK_LOG_TRY("SLOCK", &sx->lock_object, 0, 1, file, line);
221 WITNESS_LOCK(&sx->lock_object, LOP_TRYLOCK, file, line);
222 curthread->td_locks++;
223 return (1);
224 }
225
226 LOCK_LOG_TRY("SLOCK", &sx->lock_object, 0, 0, file, line);
227 return (0);
228}
229
230void
231_sx_xlock(struct sx *sx, const char *file, int line)
232{
233
234 MPASS(curthread != NULL);
223 if ((x & SX_LOCK_SHARED) && atomic_cmpset_acq_ptr(&sx->sx_lock, x,
224 x + SX_ONE_SHARER)) {
225 LOCK_LOG_TRY("SLOCK", &sx->lock_object, 0, 1, file, line);
226 WITNESS_LOCK(&sx->lock_object, LOP_TRYLOCK, file, line);
227 curthread->td_locks++;
228 return (1);
229 }
230
231 LOCK_LOG_TRY("SLOCK", &sx->lock_object, 0, 0, file, line);
232 return (0);
233}
234
235void
236_sx_xlock(struct sx *sx, const char *file, int line)
237{
238
239 MPASS(curthread != NULL);
240 KASSERT(sx->sx_lock != SX_LOCK_DESTROYED,
241 ("sx_xlock() of destroyed sx @ %s:%d", file, line));
235 WITNESS_CHECKORDER(&sx->lock_object, LOP_NEWORDER | LOP_EXCLUSIVE, file,
236 line);
237 __sx_xlock(sx, curthread, file, line);
238 LOCK_LOG_LOCK("XLOCK", &sx->lock_object, 0, sx->sx_recurse, file, line);
239 WITNESS_LOCK(&sx->lock_object, LOP_EXCLUSIVE, file, line);
240 curthread->td_locks++;
241}
242
243int
244_sx_try_xlock(struct sx *sx, const char *file, int line)
245{
246 int rval;
247
248 MPASS(curthread != NULL);
242 WITNESS_CHECKORDER(&sx->lock_object, LOP_NEWORDER | LOP_EXCLUSIVE, file,
243 line);
244 __sx_xlock(sx, curthread, file, line);
245 LOCK_LOG_LOCK("XLOCK", &sx->lock_object, 0, sx->sx_recurse, file, line);
246 WITNESS_LOCK(&sx->lock_object, LOP_EXCLUSIVE, file, line);
247 curthread->td_locks++;
248}
249
250int
251_sx_try_xlock(struct sx *sx, const char *file, int line)
252{
253 int rval;
254
255 MPASS(curthread != NULL);
256 KASSERT(sx->sx_lock != SX_LOCK_DESTROYED,
257 ("sx_try_xlock() of destroyed sx @ %s:%d", file, line));
249
250 if (sx_xlocked(sx)) {
251 sx->sx_recurse++;
252 atomic_set_ptr(&sx->sx_lock, SX_LOCK_RECURSED);
253 rval = 1;
254 } else
255 rval = atomic_cmpset_acq_ptr(&sx->sx_lock, SX_LOCK_UNLOCKED,
256 (uintptr_t)curthread);

--- 7 unchanged lines hidden (view full) ---

264 return (rval);
265}
266
267void
268_sx_sunlock(struct sx *sx, const char *file, int line)
269{
270
271 MPASS(curthread != NULL);
258
259 if (sx_xlocked(sx)) {
260 sx->sx_recurse++;
261 atomic_set_ptr(&sx->sx_lock, SX_LOCK_RECURSED);
262 rval = 1;
263 } else
264 rval = atomic_cmpset_acq_ptr(&sx->sx_lock, SX_LOCK_UNLOCKED,
265 (uintptr_t)curthread);

--- 7 unchanged lines hidden (view full) ---

273 return (rval);
274}
275
276void
277_sx_sunlock(struct sx *sx, const char *file, int line)
278{
279
280 MPASS(curthread != NULL);
281 KASSERT(sx->sx_lock != SX_LOCK_DESTROYED,
282 ("sx_sunlock() of destroyed sx @ %s:%d", file, line));
272 _sx_assert(sx, SX_SLOCKED, file, line);
273 curthread->td_locks--;
274 WITNESS_UNLOCK(&sx->lock_object, 0, file, line);
275 LOCK_LOG_LOCK("SUNLOCK", &sx->lock_object, 0, 0, file, line);
276 if (SX_SHARERS(sx->sx_lock) == 0)
277 lock_profile_release_lock(&sx->lock_object);
278 __sx_sunlock(sx, file, line);
279}
280
281void
282_sx_xunlock(struct sx *sx, const char *file, int line)
283{
284
285 MPASS(curthread != NULL);
283 _sx_assert(sx, SX_SLOCKED, file, line);
284 curthread->td_locks--;
285 WITNESS_UNLOCK(&sx->lock_object, 0, file, line);
286 LOCK_LOG_LOCK("SUNLOCK", &sx->lock_object, 0, 0, file, line);
287 if (SX_SHARERS(sx->sx_lock) == 0)
288 lock_profile_release_lock(&sx->lock_object);
289 __sx_sunlock(sx, file, line);
290}
291
292void
293_sx_xunlock(struct sx *sx, const char *file, int line)
294{
295
296 MPASS(curthread != NULL);
297 KASSERT(sx->sx_lock != SX_LOCK_DESTROYED,
298 ("sx_xunlock() of destroyed sx @ %s:%d", file, line));
286 _sx_assert(sx, SX_XLOCKED, file, line);
287 curthread->td_locks--;
288 WITNESS_UNLOCK(&sx->lock_object, LOP_EXCLUSIVE, file, line);
289 LOCK_LOG_LOCK("XUNLOCK", &sx->lock_object, 0, sx->sx_recurse, file,
290 line);
291 if (!sx_recursed(sx))
292 lock_profile_release_lock(&sx->lock_object);
293 __sx_xunlock(sx, curthread, file, line);

--- 5 unchanged lines hidden (view full) ---

299 * Return 1 if if the upgrade succeed, 0 otherwise.
300 */
301int
302_sx_try_upgrade(struct sx *sx, const char *file, int line)
303{
304 uintptr_t x;
305 int success;
306
299 _sx_assert(sx, SX_XLOCKED, file, line);
300 curthread->td_locks--;
301 WITNESS_UNLOCK(&sx->lock_object, LOP_EXCLUSIVE, file, line);
302 LOCK_LOG_LOCK("XUNLOCK", &sx->lock_object, 0, sx->sx_recurse, file,
303 line);
304 if (!sx_recursed(sx))
305 lock_profile_release_lock(&sx->lock_object);
306 __sx_xunlock(sx, curthread, file, line);

--- 5 unchanged lines hidden (view full) ---

312 * Return 1 if if the upgrade succeed, 0 otherwise.
313 */
314int
315_sx_try_upgrade(struct sx *sx, const char *file, int line)
316{
317 uintptr_t x;
318 int success;
319
320 KASSERT(sx->sx_lock != SX_LOCK_DESTROYED,
321 ("sx_try_upgrade() of destroyed sx @ %s:%d", file, line));
307 _sx_assert(sx, SX_SLOCKED, file, line);
308
309 /*
310 * Try to switch from one shared lock to an exclusive lock. We need
311 * to maintain the SX_LOCK_EXCLUSIVE_WAITERS flag if set so that
312 * we will wake up the exclusive waiters when we drop the lock.
313 */
314 x = sx->sx_lock & SX_LOCK_EXCLUSIVE_WAITERS;

--- 9 unchanged lines hidden (view full) ---

324/*
325 * Downgrade an unrecursed exclusive lock into a single shared lock.
326 */
327void
328_sx_downgrade(struct sx *sx, const char *file, int line)
329{
330 uintptr_t x;
331
322 _sx_assert(sx, SX_SLOCKED, file, line);
323
324 /*
325 * Try to switch from one shared lock to an exclusive lock. We need
326 * to maintain the SX_LOCK_EXCLUSIVE_WAITERS flag if set so that
327 * we will wake up the exclusive waiters when we drop the lock.
328 */
329 x = sx->sx_lock & SX_LOCK_EXCLUSIVE_WAITERS;

--- 9 unchanged lines hidden (view full) ---

339/*
340 * Downgrade an unrecursed exclusive lock into a single shared lock.
341 */
342void
343_sx_downgrade(struct sx *sx, const char *file, int line)
344{
345 uintptr_t x;
346
347 KASSERT(sx->sx_lock != SX_LOCK_DESTROYED,
348 ("sx_downgrade() of destroyed sx @ %s:%d", file, line));
332 _sx_assert(sx, SX_XLOCKED | SX_NOTRECURSED, file, line);
333#ifndef INVARIANTS
334 if (sx_recursed(sx))
335 panic("downgrade of a recursed lock");
336#endif
337
338 WITNESS_DOWNGRADE(&sx->lock_object, 0, file, line);
339

--- 571 unchanged lines hidden (view full) ---

911 struct thread *td;
912 struct sx *sx;
913
914 sx = (struct sx *)lock;
915
916 db_printf(" state: ");
917 if (sx->sx_lock == SX_LOCK_UNLOCKED)
918 db_printf("UNLOCKED\n");
349 _sx_assert(sx, SX_XLOCKED | SX_NOTRECURSED, file, line);
350#ifndef INVARIANTS
351 if (sx_recursed(sx))
352 panic("downgrade of a recursed lock");
353#endif
354
355 WITNESS_DOWNGRADE(&sx->lock_object, 0, file, line);
356

--- 571 unchanged lines hidden (view full) ---

928 struct thread *td;
929 struct sx *sx;
930
931 sx = (struct sx *)lock;
932
933 db_printf(" state: ");
934 if (sx->sx_lock == SX_LOCK_UNLOCKED)
935 db_printf("UNLOCKED\n");
919 else if (sx->sx_lock & SX_LOCK_SHARED)
936 else if (sx->sx_lock == SX_LOCK_DESTROYED) {
937 db_printf("DESTROYED\n");
938 return;
939 } else if (sx->sx_lock & SX_LOCK_SHARED)
920 db_printf("SLOCK: %ju\n", (uintmax_t)SX_SHARERS(sx->sx_lock));
921 else {
922 td = sx_xholder(sx);
923 db_printf("XLOCK: %p (tid %d, pid %d, \"%s\")\n", td,
924 td->td_tid, td->td_proc->p_pid, td->td_proc->p_comm);
925 if (sx_recursed(sx))
926 db_printf(" recursed: %d\n", sx->sx_recurse);
927 }

--- 49 unchanged lines hidden --- 		940 db_printf("SLOCK: %ju\n", (uintmax_t)SX_SHARERS(sx->sx_lock));
941 else {
942 td = sx_xholder(sx);
943 db_printf("XLOCK: %p (tid %d, pid %d, \"%s\")\n", td,
944 td->td_tid, td->td_proc->p_pid, td->td_proc->p_comm);
945 if (sx_recursed(sx))
946 db_printf(" recursed: %d\n", sx->sx_recurse);
947 }

--- 49 unchanged lines hidden ---