1173444Sups/*- 2173444Sups * Copyright (c) 2007 Stephan Uphoff <ups@FreeBSD.org> 3173444Sups * All rights reserved. 4173444Sups * 5173444Sups * Redistribution and use in source and binary forms, with or without 6173444Sups * modification, are permitted provided that the following conditions 7173444Sups * are met: 8173444Sups * 1. Redistributions of source code must retain the above copyright 9173444Sups * notice, this list of conditions and the following disclaimer. 10173444Sups * 2. Redistributions in binary form must reproduce the above copyright 11173444Sups * notice, this list of conditions and the following disclaimer in the 12173444Sups * documentation and/or other materials provided with the distribution. 13173444Sups * 3. Neither the name of the author nor the names of any co-contributors 14173444Sups * may be used to endorse or promote products derived from this software 15173444Sups * without specific prior written permission. 16173444Sups * 17173444Sups * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 18173444Sups * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 19173444Sups * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 20173444Sups * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 21173444Sups * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 22173444Sups * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 23173444Sups * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 24173444Sups * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 25173444Sups * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 26173444Sups * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 27173444Sups * SUCH DAMAGE. 28173444Sups */ 29173444Sups 30173444Sups/* 31173444Sups * Machine independent bits of reader/writer lock implementation. 32173444Sups */ 33173444Sups 34173444Sups#include <sys/cdefs.h> 35173444Sups__FBSDID("$FreeBSD$"); 36173444Sups 37173444Sups#include "opt_ddb.h" 38192853Ssson#include "opt_kdtrace.h" 39173444Sups 40173444Sups#include <sys/param.h> 41173444Sups#include <sys/systm.h> 42173444Sups 43173444Sups#include <sys/kernel.h> 44255862Sjhb#include <sys/kdb.h> 45173444Sups#include <sys/ktr.h> 46173444Sups#include <sys/lock.h> 47173444Sups#include <sys/mutex.h> 48173444Sups#include <sys/proc.h> 49173444Sups#include <sys/rmlock.h> 50173444Sups#include <sys/sched.h> 51173444Sups#include <sys/smp.h> 52173444Sups#include <sys/turnstile.h> 53173444Sups#include <sys/lock_profile.h> 54173444Sups#include <machine/cpu.h> 55173444Sups 56173444Sups#ifdef DDB 57173444Sups#include <ddb/ddb.h> 58173444Sups#endif 59191539Srwatson 60256001Sjhb/* 61256001Sjhb * A cookie to mark destroyed rmlocks. This is stored in the head of 62256001Sjhb * rm_activeReaders. 63256001Sjhb */ 64256001Sjhb#define RM_DESTROYED ((void *)0xdead) 65256001Sjhb 66256001Sjhb#define rm_destroyed(rm) \ 67256001Sjhb (LIST_FIRST(&(rm)->rm_activeReaders) == RM_DESTROYED) 68256001Sjhb 69173444Sups#define RMPF_ONQUEUE 1 70173444Sups#define RMPF_SIGNAL 2 71173444Sups 72256001Sjhb#ifndef INVARIANTS 73256001Sjhb#define _rm_assert(c, what, file, line) 74256001Sjhb#endif 75173444Sups 76191539Srwatsonstatic void assert_rm(struct lock_object *lock, int what); 77256001Sjhb#ifdef DDB 78256001Sjhbstatic void db_show_rm(struct lock_object *lock); 79256001Sjhb#endif 80191539Srwatsonstatic void lock_rm(struct lock_object *lock, int how); 81192853Ssson#ifdef KDTRACE_HOOKS 82192853Sssonstatic int owner_rm(struct lock_object *lock, struct thread **owner); 83192853Ssson#endif 84191539Srwatsonstatic int unlock_rm(struct lock_object *lock); 85173444Sups 86173444Supsstruct lock_class lock_class_rm = { 87173444Sups .lc_name = "rm", 88173444Sups .lc_flags = LC_SLEEPLOCK | LC_RECURSABLE, 89173733Sattilio .lc_assert = assert_rm, 90173444Sups#ifdef DDB 91256001Sjhb .lc_ddb_show = db_show_rm, 92173444Sups#endif 93256001Sjhb .lc_lock = lock_rm, 94256001Sjhb .lc_unlock = unlock_rm, 95256001Sjhb#ifdef KDTRACE_HOOKS 96256001Sjhb .lc_owner = owner_rm, 97173444Sups#endif 98256001Sjhb}; 99256001Sjhb 100256001Sjhbstruct lock_class lock_class_rm_sleepable = { 101256001Sjhb .lc_name = "sleepable rm", 102256001Sjhb .lc_flags = LC_SLEEPLOCK | LC_SLEEPABLE | LC_RECURSABLE, 103256001Sjhb .lc_assert = assert_rm, 104256001Sjhb#ifdef DDB 105256001Sjhb .lc_ddb_show = db_show_rm, 106256001Sjhb#endif 107173444Sups .lc_lock = lock_rm, 108173444Sups .lc_unlock = unlock_rm, 109192853Ssson#ifdef KDTRACE_HOOKS 110192853Ssson .lc_owner = owner_rm, 111192853Ssson#endif 112173444Sups}; 113173444Sups 114173444Supsstatic void 115173733Sattilioassert_rm(struct lock_object *lock, int what) 116173733Sattilio{ 117173733Sattilio 118256001Sjhb rm_assert((struct rmlock *)lock, what); 119173733Sattilio} 120173733Sattilio 121256001Sjhb/* 122256001Sjhb * These do not support read locks because it would be hard to make 123256001Sjhb * the tracker work correctly with the current lock_class API as you 124256001Sjhb * would need to have the tracker pointer available when calling 125256001Sjhb * rm_rlock() in lock_rm(). 126256001Sjhb */ 127173733Sattiliostatic void 128191539Srwatsonlock_rm(struct lock_object *lock, int how) 129191539Srwatson{ 130256001Sjhb struct rmlock *rm; 131191539Srwatson 132256001Sjhb rm = (struct rmlock *)lock; 133256001Sjhb if (how) 134256001Sjhb rm_wlock(rm); 135256001Sjhb#ifdef INVARIANTS 136256001Sjhb else 137256001Sjhb panic("lock_rm called in read mode"); 138256001Sjhb#endif 139173444Sups} 140173444Sups 141173444Supsstatic int 142191539Srwatsonunlock_rm(struct lock_object *lock) 143191539Srwatson{ 144256001Sjhb struct rmlock *rm; 145191539Srwatson 146256001Sjhb rm = (struct rmlock *)lock; 147256001Sjhb rm_wunlock(rm); 148256001Sjhb return (1); 149173444Sups} 150173444Sups 151192853Ssson#ifdef KDTRACE_HOOKS 152192853Sssonstatic int 153192853Sssonowner_rm(struct lock_object *lock, struct thread **owner) 154192853Ssson{ 155256001Sjhb struct rmlock *rm; 156256001Sjhb struct lock_class *lc; 157192853Ssson 158256001Sjhb rm = (struct rmlock *)lock; 159256001Sjhb lc = LOCK_CLASS(&rm->rm_wlock_object); 160256001Sjhb return (lc->lc_owner(&rm->rm_wlock_object, owner)); 161192853Ssson} 162192853Ssson#endif 163192853Ssson 164173444Supsstatic struct mtx rm_spinlock; 165173444Sups 166173444SupsMTX_SYSINIT(rm_spinlock, &rm_spinlock, "rm_spinlock", MTX_SPIN); 167173444Sups 168173444Sups/* 169200976Srwatson * Add or remove tracker from per-cpu list. 170191539Srwatson * 171200976Srwatson * The per-cpu list can be traversed at any time in forward direction from an 172191539Srwatson * interrupt on the *local* cpu. 173173444Sups */ 174191539Srwatsonstatic void inline 175191539Srwatsonrm_tracker_add(struct pcpu *pc, struct rm_priotracker *tracker) 176191539Srwatson{ 177191539Srwatson struct rm_queue *next; 178191539Srwatson 179173444Sups /* Initialize all tracker pointers */ 180173444Sups tracker->rmp_cpuQueue.rmq_prev = &pc->pc_rm_queue; 181173444Sups next = pc->pc_rm_queue.rmq_next; 182173444Sups tracker->rmp_cpuQueue.rmq_next = next; 183191539Srwatson 184191539Srwatson /* rmq_prev is not used during froward traversal. */ 185173444Sups next->rmq_prev = &tracker->rmp_cpuQueue; 186191539Srwatson 187191539Srwatson /* Update pointer to first element. */ 188201000Sbz pc->pc_rm_queue.rmq_next = &tracker->rmp_cpuQueue; 189173444Sups} 190173444Sups 191256001Sjhb/* 192256001Sjhb * Return a count of the number of trackers the thread 'td' already 193256001Sjhb * has on this CPU for the lock 'rm'. 194256001Sjhb */ 195256001Sjhbstatic int 196256001Sjhbrm_trackers_present(const struct pcpu *pc, const struct rmlock *rm, 197256001Sjhb const struct thread *td) 198256001Sjhb{ 199256001Sjhb struct rm_queue *queue; 200256001Sjhb struct rm_priotracker *tracker; 201256001Sjhb int count; 202256001Sjhb 203256001Sjhb count = 0; 204256001Sjhb for (queue = pc->pc_rm_queue.rmq_next; queue != &pc->pc_rm_queue; 205256001Sjhb queue = queue->rmq_next) { 206256001Sjhb tracker = (struct rm_priotracker *)queue; 207256001Sjhb if ((tracker->rmp_rmlock == rm) && (tracker->rmp_thread == td)) 208256001Sjhb count++; 209256001Sjhb } 210256001Sjhb return (count); 211256001Sjhb} 212256001Sjhb 213191539Srwatsonstatic void inline 214191539Srwatsonrm_tracker_remove(struct pcpu *pc, struct rm_priotracker *tracker) 215191539Srwatson{ 216191539Srwatson struct rm_queue *next, *prev; 217173444Sups 218191539Srwatson next = tracker->rmp_cpuQueue.rmq_next; 219191539Srwatson prev = tracker->rmp_cpuQueue.rmq_prev; 220191539Srwatson 221191539Srwatson /* Not used during forward traversal. */ 222173444Sups next->rmq_prev = prev; 223191539Srwatson 224191539Srwatson /* Remove from list. */ 225173444Sups prev->rmq_next = next; 226173444Sups} 227173444Sups 228191539Srwatsonstatic void 229191539Srwatsonrm_cleanIPI(void *arg) 230191539Srwatson{ 231173444Sups struct pcpu *pc; 232191539Srwatson struct rmlock *rm = arg; 233173444Sups struct rm_priotracker *tracker; 234191539Srwatson struct rm_queue *queue; 235173444Sups pc = pcpu_find(curcpu); 236173444Sups 237191539Srwatson for (queue = pc->pc_rm_queue.rmq_next; queue != &pc->pc_rm_queue; 238173444Sups queue = queue->rmq_next) { 239191539Srwatson tracker = (struct rm_priotracker *)queue; 240191539Srwatson if (tracker->rmp_rmlock == rm && tracker->rmp_flags == 0) { 241173444Sups tracker->rmp_flags = RMPF_ONQUEUE; 242173444Sups mtx_lock_spin(&rm_spinlock); 243191539Srwatson LIST_INSERT_HEAD(&rm->rm_activeReaders, tracker, 244173444Sups rmp_qentry); 245173444Sups mtx_unlock_spin(&rm_spinlock); 246173444Sups } 247173444Sups } 248173444Sups} 249173444Sups 250173444Supsvoid 251193030Srwatsonrm_init_flags(struct rmlock *rm, const char *name, int opts) 252173444Sups{ 253256001Sjhb struct lock_class *lc; 254193030Srwatson int liflags; 255191539Srwatson 256193030Srwatson liflags = 0; 257193030Srwatson if (!(opts & RM_NOWITNESS)) 258193030Srwatson liflags |= LO_WITNESS; 259193030Srwatson if (opts & RM_RECURSE) 260193030Srwatson liflags |= LO_RECURSABLE; 261212112Smlaier rm->rm_writecpus = all_cpus; 262173444Sups LIST_INIT(&rm->rm_activeReaders); 263212112Smlaier if (opts & RM_SLEEPABLE) { 264256001Sjhb liflags |= LO_SLEEPABLE; 265256001Sjhb lc = &lock_class_rm_sleepable; 266256001Sjhb sx_init_flags(&rm->rm_lock_sx, "rmlock_sx", SX_NOWITNESS); 267256001Sjhb } else { 268256001Sjhb lc = &lock_class_rm; 269212112Smlaier mtx_init(&rm->rm_lock_mtx, name, "rmlock_mtx", MTX_NOWITNESS); 270256001Sjhb } 271256001Sjhb lock_init(&rm->lock_object, lc, name, NULL, liflags); 272173444Sups} 273173444Sups 274173444Supsvoid 275193030Srwatsonrm_init(struct rmlock *rm, const char *name) 276193030Srwatson{ 277193030Srwatson 278193030Srwatson rm_init_flags(rm, name, 0); 279193030Srwatson} 280193030Srwatson 281193030Srwatsonvoid 282173444Supsrm_destroy(struct rmlock *rm) 283173444Sups{ 284191539Srwatson 285256001Sjhb rm_assert(rm, RA_UNLOCKED); 286256001Sjhb LIST_FIRST(&rm->rm_activeReaders) = RM_DESTROYED; 287256001Sjhb if (rm->lock_object.lo_flags & LO_SLEEPABLE) 288212112Smlaier sx_destroy(&rm->rm_lock_sx); 289212112Smlaier else 290212112Smlaier mtx_destroy(&rm->rm_lock_mtx); 291173444Sups lock_destroy(&rm->lock_object); 292173444Sups} 293173444Sups 294173520Srwatsonint 295173520Srwatsonrm_wowned(struct rmlock *rm) 296173520Srwatson{ 297173520Srwatson 298256001Sjhb if (rm->lock_object.lo_flags & LO_SLEEPABLE) 299212112Smlaier return (sx_xlocked(&rm->rm_lock_sx)); 300212112Smlaier else 301212112Smlaier return (mtx_owned(&rm->rm_lock_mtx)); 302173520Srwatson} 303173520Srwatson 304173444Supsvoid 305173444Supsrm_sysinit(void *arg) 306173444Sups{ 307193030Srwatson struct rm_args *args = arg; 308191539Srwatson 309193030Srwatson rm_init(args->ra_rm, args->ra_desc); 310173444Sups} 311173444Sups 312193030Srwatsonvoid 313193030Srwatsonrm_sysinit_flags(void *arg) 314193030Srwatson{ 315193030Srwatson struct rm_args_flags *args = arg; 316193030Srwatson 317193030Srwatson rm_init_flags(args->ra_rm, args->ra_desc, args->ra_opts); 318193030Srwatson} 319193030Srwatson 320212112Smlaierstatic int 321212112Smlaier_rm_rlock_hard(struct rmlock *rm, struct rm_priotracker *tracker, int trylock) 322173444Sups{ 323173444Sups struct pcpu *pc; 324173444Sups 325173444Sups critical_enter(); 326173444Sups pc = pcpu_find(curcpu); 327173444Sups 328191539Srwatson /* Check if we just need to do a proper critical_exit. */ 329223758Sattilio if (!CPU_ISSET(pc->pc_cpuid, &rm->rm_writecpus)) { 330173444Sups critical_exit(); 331212112Smlaier return (1); 332173444Sups } 333173444Sups 334200976Srwatson /* Remove our tracker from the per-cpu list. */ 335191539Srwatson rm_tracker_remove(pc, tracker); 336173444Sups 337191539Srwatson /* Check to see if the IPI granted us the lock after all. */ 338191539Srwatson if (tracker->rmp_flags) { 339191539Srwatson /* Just add back tracker - we hold the lock. */ 340191539Srwatson rm_tracker_add(pc, tracker); 341173444Sups critical_exit(); 342212112Smlaier return (1); 343173444Sups } 344173444Sups 345173444Sups /* 346191539Srwatson * We allow readers to aquire a lock even if a writer is blocked if 347191539Srwatson * the lock is recursive and the reader already holds the lock. 348173444Sups */ 349173444Sups if ((rm->lock_object.lo_flags & LO_RECURSABLE) != 0) { 350173444Sups /* 351200976Srwatson * Just grant the lock if this thread already has a tracker 352200976Srwatson * for this lock on the per-cpu queue. 353173444Sups */ 354256001Sjhb if (rm_trackers_present(pc, rm, curthread) != 0) { 355256001Sjhb mtx_lock_spin(&rm_spinlock); 356256001Sjhb LIST_INSERT_HEAD(&rm->rm_activeReaders, tracker, 357256001Sjhb rmp_qentry); 358256001Sjhb tracker->rmp_flags = RMPF_ONQUEUE; 359256001Sjhb mtx_unlock_spin(&rm_spinlock); 360256001Sjhb rm_tracker_add(pc, tracker); 361256001Sjhb critical_exit(); 362256001Sjhb return (1); 363173444Sups } 364173444Sups } 365173444Sups 366173444Sups sched_unpin(); 367173444Sups critical_exit(); 368173444Sups 369212112Smlaier if (trylock) { 370256001Sjhb if (rm->lock_object.lo_flags & LO_SLEEPABLE) { 371212112Smlaier if (!sx_try_xlock(&rm->rm_lock_sx)) 372212112Smlaier return (0); 373212112Smlaier } else { 374212112Smlaier if (!mtx_trylock(&rm->rm_lock_mtx)) 375212112Smlaier return (0); 376212112Smlaier } 377212112Smlaier } else { 378256001Sjhb if (rm->lock_object.lo_flags & LO_SLEEPABLE) 379212112Smlaier sx_xlock(&rm->rm_lock_sx); 380212112Smlaier else 381212112Smlaier mtx_lock(&rm->rm_lock_mtx); 382212112Smlaier } 383212112Smlaier 384173444Sups critical_enter(); 385173444Sups pc = pcpu_find(curcpu); 386223758Sattilio CPU_CLR(pc->pc_cpuid, &rm->rm_writecpus); 387191539Srwatson rm_tracker_add(pc, tracker); 388173444Sups sched_pin(); 389173444Sups critical_exit(); 390191539Srwatson 391256001Sjhb if (rm->lock_object.lo_flags & LO_SLEEPABLE) 392212112Smlaier sx_xunlock(&rm->rm_lock_sx); 393212112Smlaier else 394212112Smlaier mtx_unlock(&rm->rm_lock_mtx); 395212112Smlaier 396212112Smlaier return (1); 397173444Sups} 398173444Sups 399212112Smlaierint 400212112Smlaier_rm_rlock(struct rmlock *rm, struct rm_priotracker *tracker, int trylock) 401173444Sups{ 402173444Sups struct thread *td = curthread; 403173444Sups struct pcpu *pc; 404173444Sups 405235404Savg if (SCHEDULER_STOPPED()) 406235404Savg return (1); 407235404Savg 408173444Sups tracker->rmp_flags = 0; 409173444Sups tracker->rmp_thread = td; 410173444Sups tracker->rmp_rmlock = rm; 411173444Sups 412173444Sups td->td_critnest++; /* critical_enter(); */ 413173444Sups 414254169Smarius __compiler_membar(); 415173444Sups 416173444Sups pc = cpuid_to_pcpu[td->td_oncpu]; /* pcpu_find(td->td_oncpu); */ 417173444Sups 418191539Srwatson rm_tracker_add(pc, tracker); 419173444Sups 420193038Srwatson sched_pin(); 421173444Sups 422254169Smarius __compiler_membar(); 423173444Sups 424173444Sups td->td_critnest--; 425191539Srwatson 426191539Srwatson /* 427191539Srwatson * Fast path to combine two common conditions into a single 428191539Srwatson * conditional jump. 429173444Sups */ 430222813Sattilio if (0 == (td->td_owepreempt | 431223758Sattilio CPU_ISSET(pc->pc_cpuid, &rm->rm_writecpus))) 432212112Smlaier return (1); 433173444Sups 434191539Srwatson /* We do not have a read token and need to acquire one. */ 435212112Smlaier return _rm_rlock_hard(rm, tracker, trylock); 436173444Sups} 437173444Sups 438173444Supsstatic void 439191539Srwatson_rm_unlock_hard(struct thread *td,struct rm_priotracker *tracker) 440173444Sups{ 441191539Srwatson 442173444Sups if (td->td_owepreempt) { 443173444Sups td->td_critnest++; 444173444Sups critical_exit(); 445173444Sups } 446191539Srwatson 447191539Srwatson if (!tracker->rmp_flags) 448173444Sups return; 449173444Sups 450173444Sups mtx_lock_spin(&rm_spinlock); 451191539Srwatson LIST_REMOVE(tracker, rmp_qentry); 452173444Sups 453173444Sups if (tracker->rmp_flags & RMPF_SIGNAL) { 454173444Sups struct rmlock *rm; 455191539Srwatson struct turnstile *ts; 456173444Sups 457173444Sups rm = tracker->rmp_rmlock; 458191539Srwatson 459173444Sups turnstile_chain_lock(&rm->lock_object); 460173444Sups mtx_unlock_spin(&rm_spinlock); 461173444Sups 462173444Sups ts = turnstile_lookup(&rm->lock_object); 463173444Sups 464173444Sups turnstile_signal(ts, TS_EXCLUSIVE_QUEUE); 465173444Sups turnstile_unpend(ts, TS_EXCLUSIVE_LOCK); 466173444Sups turnstile_chain_unlock(&rm->lock_object); 467173444Sups } else 468173444Sups mtx_unlock_spin(&rm_spinlock); 469191539Srwatson} 470173444Sups 471173444Supsvoid 472191539Srwatson_rm_runlock(struct rmlock *rm, struct rm_priotracker *tracker) 473173444Sups{ 474173444Sups struct pcpu *pc; 475173444Sups struct thread *td = tracker->rmp_thread; 476173444Sups 477235404Savg if (SCHEDULER_STOPPED()) 478235404Savg return; 479235404Savg 480173444Sups td->td_critnest++; /* critical_enter(); */ 481173444Sups pc = cpuid_to_pcpu[td->td_oncpu]; /* pcpu_find(td->td_oncpu); */ 482191539Srwatson rm_tracker_remove(pc, tracker); 483173444Sups td->td_critnest--; 484193038Srwatson sched_unpin(); 485173444Sups 486191539Srwatson if (0 == (td->td_owepreempt | tracker->rmp_flags)) 487173444Sups return; 488173444Sups 489191539Srwatson _rm_unlock_hard(td, tracker); 490173444Sups} 491173444Sups 492173444Supsvoid 493173444Sups_rm_wlock(struct rmlock *rm) 494173444Sups{ 495173444Sups struct rm_priotracker *prio; 496173444Sups struct turnstile *ts; 497222813Sattilio cpuset_t readcpus; 498173444Sups 499235404Savg if (SCHEDULER_STOPPED()) 500235404Savg return; 501235404Savg 502256001Sjhb if (rm->lock_object.lo_flags & LO_SLEEPABLE) 503212112Smlaier sx_xlock(&rm->rm_lock_sx); 504212112Smlaier else 505212112Smlaier mtx_lock(&rm->rm_lock_mtx); 506173444Sups 507222813Sattilio if (CPU_CMP(&rm->rm_writecpus, &all_cpus)) { 508173444Sups /* Get all read tokens back */ 509222813Sattilio readcpus = all_cpus; 510222813Sattilio CPU_NAND(&readcpus, &rm->rm_writecpus); 511212112Smlaier rm->rm_writecpus = all_cpus; 512173444Sups 513191539Srwatson /* 514212112Smlaier * Assumes rm->rm_writecpus update is visible on other CPUs 515191539Srwatson * before rm_cleanIPI is called. 516173444Sups */ 517173444Sups#ifdef SMP 518212112Smlaier smp_rendezvous_cpus(readcpus, 519212112Smlaier smp_no_rendevous_barrier, 520173444Sups rm_cleanIPI, 521191539Srwatson smp_no_rendevous_barrier, 522191539Srwatson rm); 523173444Sups 524173444Sups#else 525173444Sups rm_cleanIPI(rm); 526173444Sups#endif 527173444Sups 528173444Sups mtx_lock_spin(&rm_spinlock); 529191539Srwatson while ((prio = LIST_FIRST(&rm->rm_activeReaders)) != NULL) { 530173444Sups ts = turnstile_trywait(&rm->lock_object); 531173444Sups prio->rmp_flags = RMPF_ONQUEUE | RMPF_SIGNAL; 532173444Sups mtx_unlock_spin(&rm_spinlock); 533191539Srwatson turnstile_wait(ts, prio->rmp_thread, 534191539Srwatson TS_EXCLUSIVE_QUEUE); 535173444Sups mtx_lock_spin(&rm_spinlock); 536173444Sups } 537173444Sups mtx_unlock_spin(&rm_spinlock); 538173444Sups } 539173444Sups} 540173444Sups 541173444Supsvoid 542173444Sups_rm_wunlock(struct rmlock *rm) 543173444Sups{ 544191539Srwatson 545256001Sjhb if (rm->lock_object.lo_flags & LO_SLEEPABLE) 546212112Smlaier sx_xunlock(&rm->rm_lock_sx); 547212112Smlaier else 548212112Smlaier mtx_unlock(&rm->rm_lock_mtx); 549173444Sups} 550173444Sups 551173444Sups#ifdef LOCK_DEBUG 552173444Sups 553256001Sjhbvoid 554256001Sjhb_rm_wlock_debug(struct rmlock *rm, const char *file, int line) 555173444Sups{ 556173444Sups 557235404Savg if (SCHEDULER_STOPPED()) 558235404Savg return; 559235404Savg 560255862Sjhb KASSERT(kdb_active != 0 || !TD_IS_IDLETHREAD(curthread), 561255862Sjhb ("rm_wlock() by idle thread %p on rmlock %s @ %s:%d", 562255862Sjhb curthread, rm->lock_object.lo_name, file, line)); 563256001Sjhb KASSERT(!rm_destroyed(rm), 564256001Sjhb ("rm_wlock() of destroyed rmlock @ %s:%d", file, line)); 565256001Sjhb _rm_assert(rm, RA_UNLOCKED, file, line); 566256001Sjhb 567191539Srwatson WITNESS_CHECKORDER(&rm->lock_object, LOP_NEWORDER | LOP_EXCLUSIVE, 568182914Sjhb file, line, NULL); 569173444Sups 570173444Sups _rm_wlock(rm); 571173444Sups 572173444Sups LOCK_LOG_LOCK("RMWLOCK", &rm->lock_object, 0, 0, file, line); 573173444Sups 574256001Sjhb WITNESS_LOCK(&rm->lock_object, LOP_EXCLUSIVE, file, line); 575173444Sups 576173444Sups curthread->td_locks++; 577173444Sups 578173444Sups} 579173444Sups 580191539Srwatsonvoid 581191539Srwatson_rm_wunlock_debug(struct rmlock *rm, const char *file, int line) 582173444Sups{ 583191539Srwatson 584235404Savg if (SCHEDULER_STOPPED()) 585235404Savg return; 586235404Savg 587256001Sjhb KASSERT(!rm_destroyed(rm), 588256001Sjhb ("rm_wunlock() of destroyed rmlock @ %s:%d", file, line)); 589256001Sjhb _rm_assert(rm, RA_WLOCKED, file, line); 590256001Sjhb WITNESS_UNLOCK(&rm->lock_object, LOP_EXCLUSIVE, file, line); 591173444Sups LOCK_LOG_LOCK("RMWUNLOCK", &rm->lock_object, 0, 0, file, line); 592173444Sups _rm_wunlock(rm); 593256001Sjhb curthread->td_locks--; 594191539Srwatson} 595173444Sups 596212112Smlaierint 597173444Sups_rm_rlock_debug(struct rmlock *rm, struct rm_priotracker *tracker, 598212112Smlaier int trylock, const char *file, int line) 599173444Sups{ 600235404Savg 601235404Savg if (SCHEDULER_STOPPED()) 602235404Savg return (1); 603235404Savg 604256001Sjhb#ifdef INVARIANTS 605256001Sjhb if (!(rm->lock_object.lo_flags & LO_RECURSABLE) && !trylock) { 606256001Sjhb critical_enter(); 607256001Sjhb KASSERT(rm_trackers_present(pcpu_find(curcpu), rm, 608256001Sjhb curthread) == 0, 609256001Sjhb ("rm_rlock: recursed on non-recursive rmlock %s @ %s:%d\n", 610256001Sjhb rm->lock_object.lo_name, file, line)); 611256001Sjhb critical_exit(); 612256001Sjhb } 613256001Sjhb#endif 614255862Sjhb KASSERT(kdb_active != 0 || !TD_IS_IDLETHREAD(curthread), 615255862Sjhb ("rm_rlock() by idle thread %p on rmlock %s @ %s:%d", 616255862Sjhb curthread, rm->lock_object.lo_name, file, line)); 617256001Sjhb KASSERT(!rm_destroyed(rm), 618256001Sjhb ("rm_rlock() of destroyed rmlock @ %s:%d", file, line)); 619256001Sjhb if (!trylock) { 620256001Sjhb KASSERT(!rm_wowned(rm), 621256001Sjhb ("rm_rlock: wlock already held for %s @ %s:%d", 622256001Sjhb rm->lock_object.lo_name, file, line)); 623256001Sjhb WITNESS_CHECKORDER(&rm->lock_object, LOP_NEWORDER, file, line, 624256001Sjhb NULL); 625256001Sjhb } 626173444Sups 627212112Smlaier if (_rm_rlock(rm, tracker, trylock)) { 628256001Sjhb if (trylock) 629256001Sjhb LOCK_LOG_TRY("RMRLOCK", &rm->lock_object, 0, 1, file, 630256001Sjhb line); 631256001Sjhb else 632256001Sjhb LOCK_LOG_LOCK("RMRLOCK", &rm->lock_object, 0, 0, file, 633256001Sjhb line); 634212112Smlaier WITNESS_LOCK(&rm->lock_object, 0, file, line); 635173444Sups 636212112Smlaier curthread->td_locks++; 637173444Sups 638212112Smlaier return (1); 639256001Sjhb } else if (trylock) 640256001Sjhb LOCK_LOG_TRY("RMRLOCK", &rm->lock_object, 0, 0, file, line); 641212112Smlaier 642212112Smlaier return (0); 643173444Sups} 644173444Sups 645191539Srwatsonvoid 646201000Sbz_rm_runlock_debug(struct rmlock *rm, struct rm_priotracker *tracker, 647191539Srwatson const char *file, int line) 648191539Srwatson{ 649191539Srwatson 650235404Savg if (SCHEDULER_STOPPED()) 651235404Savg return; 652235404Savg 653256001Sjhb KASSERT(!rm_destroyed(rm), 654256001Sjhb ("rm_runlock() of destroyed rmlock @ %s:%d", file, line)); 655256001Sjhb _rm_assert(rm, RA_RLOCKED, file, line); 656191539Srwatson WITNESS_UNLOCK(&rm->lock_object, 0, file, line); 657173444Sups LOCK_LOG_LOCK("RMRUNLOCK", &rm->lock_object, 0, 0, file, line); 658173444Sups _rm_runlock(rm, tracker); 659256001Sjhb curthread->td_locks--; 660173444Sups} 661173444Sups 662173444Sups#else 663173444Sups 664191539Srwatson/* 665191539Srwatson * Just strip out file and line arguments if no lock debugging is enabled in 666191539Srwatson * the kernel - we are called from a kernel module. 667191539Srwatson */ 668191539Srwatsonvoid 669191539Srwatson_rm_wlock_debug(struct rmlock *rm, const char *file, int line) 670191539Srwatson{ 671173444Sups 672173444Sups _rm_wlock(rm); 673173444Sups} 674173444Sups 675191539Srwatsonvoid 676191539Srwatson_rm_wunlock_debug(struct rmlock *rm, const char *file, int line) 677173444Sups{ 678191539Srwatson 679173444Sups _rm_wunlock(rm); 680191539Srwatson} 681191539Srwatson 682212112Smlaierint 683173444Sups_rm_rlock_debug(struct rmlock *rm, struct rm_priotracker *tracker, 684212112Smlaier int trylock, const char *file, int line) 685173444Sups{ 686191539Srwatson 687212112Smlaier return _rm_rlock(rm, tracker, trylock); 688173444Sups} 689173444Sups 690191539Srwatsonvoid 691201000Sbz_rm_runlock_debug(struct rmlock *rm, struct rm_priotracker *tracker, 692193039Srwatson const char *file, int line) 693193039Srwatson{ 694191539Srwatson 695173444Sups _rm_runlock(rm, tracker); 696173444Sups} 697173444Sups 698173444Sups#endif 699256001Sjhb 700256001Sjhb#ifdef INVARIANT_SUPPORT 701256001Sjhb#ifndef INVARIANTS 702256001Sjhb#undef _rm_assert 703256001Sjhb#endif 704256001Sjhb 705256001Sjhb/* 706256001Sjhb * Note that this does not need to use witness_assert() for read lock 707256001Sjhb * assertions since an exact count of read locks held by this thread 708256001Sjhb * is computable. 709256001Sjhb */ 710256001Sjhbvoid 711256001Sjhb_rm_assert(struct rmlock *rm, int what, const char *file, int line) 712256001Sjhb{ 713256001Sjhb int count; 714256001Sjhb 715256001Sjhb if (panicstr != NULL) 716256001Sjhb return; 717256001Sjhb switch (what) { 718256001Sjhb case RA_LOCKED: 719256001Sjhb case RA_LOCKED | RA_RECURSED: 720256001Sjhb case RA_LOCKED | RA_NOTRECURSED: 721256001Sjhb case RA_RLOCKED: 722256001Sjhb case RA_RLOCKED | RA_RECURSED: 723256001Sjhb case RA_RLOCKED | RA_NOTRECURSED: 724256001Sjhb /* 725256001Sjhb * Handle the write-locked case. Unlike other 726256001Sjhb * primitives, writers can never recurse. 727256001Sjhb */ 728256001Sjhb if (rm_wowned(rm)) { 729256001Sjhb if (what & RA_RLOCKED) 730256001Sjhb panic("Lock %s exclusively locked @ %s:%d\n", 731256001Sjhb rm->lock_object.lo_name, file, line); 732256001Sjhb if (what & RA_RECURSED) 733256001Sjhb panic("Lock %s not recursed @ %s:%d\n", 734256001Sjhb rm->lock_object.lo_name, file, line); 735256001Sjhb break; 736256001Sjhb } 737256001Sjhb 738256001Sjhb critical_enter(); 739256001Sjhb count = rm_trackers_present(pcpu_find(curcpu), rm, curthread); 740256001Sjhb critical_exit(); 741256001Sjhb 742256001Sjhb if (count == 0) 743256001Sjhb panic("Lock %s not %slocked @ %s:%d\n", 744256001Sjhb rm->lock_object.lo_name, (what & RA_RLOCKED) ? 745256001Sjhb "read " : "", file, line); 746256001Sjhb if (count > 1) { 747256001Sjhb if (what & RA_NOTRECURSED) 748256001Sjhb panic("Lock %s recursed @ %s:%d\n", 749256001Sjhb rm->lock_object.lo_name, file, line); 750256001Sjhb } else if (what & RA_RECURSED) 751256001Sjhb panic("Lock %s not recursed @ %s:%d\n", 752256001Sjhb rm->lock_object.lo_name, file, line); 753256001Sjhb break; 754256001Sjhb case RA_WLOCKED: 755256001Sjhb if (!rm_wowned(rm)) 756256001Sjhb panic("Lock %s not exclusively locked @ %s:%d\n", 757256001Sjhb rm->lock_object.lo_name, file, line); 758256001Sjhb break; 759256001Sjhb case RA_UNLOCKED: 760256001Sjhb if (rm_wowned(rm)) 761256001Sjhb panic("Lock %s exclusively locked @ %s:%d\n", 762256001Sjhb rm->lock_object.lo_name, file, line); 763256001Sjhb 764256001Sjhb critical_enter(); 765256001Sjhb count = rm_trackers_present(pcpu_find(curcpu), rm, curthread); 766256001Sjhb critical_exit(); 767256001Sjhb 768256001Sjhb if (count != 0) 769256001Sjhb panic("Lock %s read locked @ %s:%d\n", 770256001Sjhb rm->lock_object.lo_name, file, line); 771256001Sjhb break; 772256001Sjhb default: 773256001Sjhb panic("Unknown rm lock assertion: %d @ %s:%d", what, file, 774256001Sjhb line); 775256001Sjhb } 776256001Sjhb} 777256001Sjhb#endif /* INVARIANT_SUPPORT */ 778256001Sjhb 779256001Sjhb#ifdef DDB 780256001Sjhbstatic void 781256001Sjhbprint_tracker(struct rm_priotracker *tr) 782256001Sjhb{ 783256001Sjhb struct thread *td; 784256001Sjhb 785256001Sjhb td = tr->rmp_thread; 786256001Sjhb db_printf(" thread %p (tid %d, pid %d, \"%s\") {", td, td->td_tid, 787256001Sjhb td->td_proc->p_pid, td->td_name); 788256001Sjhb if (tr->rmp_flags & RMPF_ONQUEUE) { 789256001Sjhb db_printf("ONQUEUE"); 790256001Sjhb if (tr->rmp_flags & RMPF_SIGNAL) 791256001Sjhb db_printf(",SIGNAL"); 792256001Sjhb } else 793256001Sjhb db_printf("0"); 794256001Sjhb db_printf("}\n"); 795256001Sjhb} 796256001Sjhb 797256001Sjhbstatic void 798256001Sjhbdb_show_rm(struct lock_object *lock) 799256001Sjhb{ 800256001Sjhb struct rm_priotracker *tr; 801256001Sjhb struct rm_queue *queue; 802256001Sjhb struct rmlock *rm; 803256001Sjhb struct lock_class *lc; 804256001Sjhb struct pcpu *pc; 805256001Sjhb 806256001Sjhb rm = (struct rmlock *)lock; 807256001Sjhb db_printf(" writecpus: "); 808256001Sjhb ddb_display_cpuset(__DEQUALIFY(const cpuset_t *, &rm->rm_writecpus)); 809256001Sjhb db_printf("\n"); 810256001Sjhb db_printf(" per-CPU readers:\n"); 811256001Sjhb STAILQ_FOREACH(pc, &cpuhead, pc_allcpu) 812256001Sjhb for (queue = pc->pc_rm_queue.rmq_next; 813256001Sjhb queue != &pc->pc_rm_queue; queue = queue->rmq_next) { 814256001Sjhb tr = (struct rm_priotracker *)queue; 815256001Sjhb if (tr->rmp_rmlock == rm) 816256001Sjhb print_tracker(tr); 817256001Sjhb } 818256001Sjhb db_printf(" active readers:\n"); 819256001Sjhb LIST_FOREACH(tr, &rm->rm_activeReaders, rmp_qentry) 820256001Sjhb print_tracker(tr); 821256001Sjhb lc = LOCK_CLASS(&rm->rm_wlock_object); 822256001Sjhb db_printf("Backing write-lock (%s):\n", lc->lc_name); 823256001Sjhb lc->lc_ddb_show(&rm->rm_wlock_object); 824256001Sjhb} 825256001Sjhb#endif 826