1115396Skan/*- 2115396Skan * Copyright 1999, 2000 John D. Polstra. 3115396Skan * All rights reserved. 4115396Skan * 5115396Skan * Redistribution and use in source and binary forms, with or without 6115396Skan * modification, are permitted provided that the following conditions 7115396Skan * are met: 8115396Skan * 1. Redistributions of source code must retain the above copyright 9115396Skan * notice, this list of conditions and the following disclaimer. 10115396Skan * 2. Redistributions in binary form must reproduce the above copyright 11115396Skan * notice, this list of conditions and the following disclaimer in the 12115396Skan * documentation and/or other materials provided with the distribution. 13115396Skan * 14115396Skan * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 15115396Skan * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 16115396Skan * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 17115396Skan * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, 18115396Skan * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT 19115396Skan * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 20115396Skan * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 21115396Skan * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 22115396Skan * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF 23115396Skan * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 24115396Skan * 25115396Skan * from: FreeBSD: src/libexec/rtld-elf/sparc64/lockdflt.c,v 1.3 2002/10/09 26115396Skan * $FreeBSD: stable/10/libexec/rtld-elf/rtld_lock.c 312701 2017-01-24 17:30:13Z kib $ 27115396Skan */ 28115396Skan 29115396Skan/* 30115396Skan * Thread locking implementation for the dynamic linker. 31115396Skan * 32115396Skan * We use the "simple, non-scalable reader-preference lock" from: 33115396Skan * 34115396Skan * J. M. Mellor-Crummey and M. L. Scott. "Scalable Reader-Writer 35115396Skan * Synchronization for Shared-Memory Multiprocessors." 3rd ACM Symp. on 36115396Skan * Principles and Practice of Parallel Programming, April 1991. 37115396Skan * 38115396Skan * In this algorithm the lock is a single word. Its low-order bit is 39115396Skan * set when a writer holds the lock. The remaining high-order bits 40115396Skan * contain a count of readers desiring the lock. The algorithm requires 41312340Skib * atomic "compare_and_store" and "add" operations, which we take 42312340Skib * from machine/atomic.h. 43115396Skan */ 44115396Skan 45191303Srwatson#include <sys/param.h> 46115396Skan#include <signal.h> 47115396Skan#include <stdlib.h> 48115396Skan#include <time.h> 49115396Skan 50115396Skan#include "debug.h" 51115396Skan#include "rtld.h" 52115396Skan#include "rtld_machdep.h" 53115396Skan 54281453Skibvoid _rtld_thread_init(struct RtldLockInfo *) __exported; 55281453Skibvoid _rtld_atfork_pre(int *) __exported; 56281453Skibvoid _rtld_atfork_post(int *) __exported; 57281453Skib 58115396Skan#define WAFLAG 0x1 /* A writer holds the lock */ 59115396Skan#define RC_INCR 0x2 /* Adjusts count of readers desiring lock */ 60115396Skan 61115396Skantypedef struct Struct_Lock { 62168311Skan volatile u_int lock; 63115396Skan void *base; 64115396Skan} Lock; 65115396Skan 66115396Skanstatic sigset_t fullsigmask, oldsigmask; 67312701Skibstatic int thread_flag, wnested; 68115396Skan 69115396Skanstatic void * 70312340Skibdef_lock_create(void) 71115396Skan{ 72115396Skan void *base; 73115396Skan char *p; 74115396Skan uintptr_t r; 75115396Skan Lock *l; 76115396Skan 77115396Skan /* 78115396Skan * Arrange for the lock to occupy its own cache line. First, we 79115396Skan * optimistically allocate just a cache line, hoping that malloc 80115396Skan * will give us a well-aligned block of memory. If that doesn't 81115396Skan * work, we allocate a larger block and take a well-aligned cache 82115396Skan * line from it. 83115396Skan */ 84115396Skan base = xmalloc(CACHE_LINE_SIZE); 85115396Skan p = (char *)base; 86115396Skan if ((uintptr_t)p % CACHE_LINE_SIZE != 0) { 87115396Skan free(base); 88115396Skan base = xmalloc(2 * CACHE_LINE_SIZE); 89115396Skan p = (char *)base; 90115396Skan if ((r = (uintptr_t)p % CACHE_LINE_SIZE) != 0) 91115396Skan p += CACHE_LINE_SIZE - r; 92115396Skan } 93115396Skan l = (Lock *)p; 94115396Skan l->base = base; 95115396Skan l->lock = 0; 96115396Skan return l; 97115396Skan} 98115396Skan 99115396Skanstatic void 100115396Skandef_lock_destroy(void *lock) 101115396Skan{ 102115396Skan Lock *l = (Lock *)lock; 103115396Skan 104115396Skan free(l->base); 105115396Skan} 106115396Skan 107115396Skanstatic void 108115396Skandef_rlock_acquire(void *lock) 109115396Skan{ 110115396Skan Lock *l = (Lock *)lock; 111115396Skan 112115396Skan atomic_add_acq_int(&l->lock, RC_INCR); 113115396Skan while (l->lock & WAFLAG) 114115396Skan ; /* Spin */ 115115396Skan} 116115396Skan 117115396Skanstatic void 118115396Skandef_wlock_acquire(void *lock) 119115396Skan{ 120312701Skib Lock *l; 121312701Skib sigset_t tmp_oldsigmask; 122115396Skan 123312701Skib l = (Lock *)lock; 124312701Skib for (;;) { 125312701Skib sigprocmask(SIG_BLOCK, &fullsigmask, &tmp_oldsigmask); 126312701Skib if (atomic_cmpset_acq_int(&l->lock, 0, WAFLAG)) 127312701Skib break; 128312701Skib sigprocmask(SIG_SETMASK, &tmp_oldsigmask, NULL); 129312701Skib } 130312701Skib if (atomic_fetchadd_int(&wnested, 1) == 0) 131312701Skib oldsigmask = tmp_oldsigmask; 132115396Skan} 133115396Skan 134115396Skanstatic void 135115396Skandef_lock_release(void *lock) 136115396Skan{ 137312701Skib Lock *l; 138115396Skan 139312701Skib l = (Lock *)lock; 140312701Skib if ((l->lock & WAFLAG) == 0) 141312701Skib atomic_add_rel_int(&l->lock, -RC_INCR); 142312701Skib else { 143312701Skib assert(wnested > 0); 144312701Skib atomic_add_rel_int(&l->lock, -WAFLAG); 145312701Skib if (atomic_fetchadd_int(&wnested, -1) == 1) 146312701Skib sigprocmask(SIG_SETMASK, &oldsigmask, NULL); 147312701Skib } 148115396Skan} 149115396Skan 150115396Skanstatic int 151115396Skandef_thread_set_flag(int mask) 152115396Skan{ 153115396Skan int old_val = thread_flag; 154115396Skan thread_flag |= mask; 155115396Skan return (old_val); 156115396Skan} 157115396Skan 158115396Skanstatic int 159115396Skandef_thread_clr_flag(int mask) 160115396Skan{ 161115396Skan int old_val = thread_flag; 162115396Skan thread_flag &= ~mask; 163115396Skan return (old_val); 164115396Skan} 165115396Skan 166115396Skan/* 167115396Skan * Public interface exposed to the rest of the dynamic linker. 168115396Skan */ 169115396Skanstatic struct RtldLockInfo lockinfo; 170115396Skanstatic struct RtldLockInfo deflockinfo; 171115396Skan 172131575Sstefanfstatic __inline int 173115396Skanthread_mask_set(int mask) 174115396Skan{ 175115396Skan return lockinfo.thread_set_flag(mask); 176115396Skan} 177115396Skan 178131575Sstefanfstatic __inline void 179115396Skanthread_mask_clear(int mask) 180115396Skan{ 181115396Skan lockinfo.thread_clr_flag(mask); 182115396Skan} 183115396Skan 184178807Skib#define RTLD_LOCK_CNT 3 185115396Skanstruct rtld_lock { 186115396Skan void *handle; 187115396Skan int mask; 188115396Skan} rtld_locks[RTLD_LOCK_CNT]; 189115396Skan 190115396Skanrtld_lock_t rtld_bind_lock = &rtld_locks[0]; 191115396Skanrtld_lock_t rtld_libc_lock = &rtld_locks[1]; 192178807Skibrtld_lock_t rtld_phdr_lock = &rtld_locks[2]; 193115396Skan 194216695Skibvoid 195216695Skibrlock_acquire(rtld_lock_t lock, RtldLockState *lockstate) 196115396Skan{ 197216695Skib 198216695Skib if (lockstate == NULL) 199216695Skib return; 200216695Skib 201183061Sdavidxu if (thread_mask_set(lock->mask) & lock->mask) { 202216695Skib dbg("rlock_acquire: recursed"); 203216695Skib lockstate->lockstate = RTLD_LOCK_UNLOCKED; 204216695Skib return; 205115396Skan } 206115396Skan lockinfo.rlock_acquire(lock->handle); 207216695Skib lockstate->lockstate = RTLD_LOCK_RLOCKED; 208115396Skan} 209115396Skan 210216695Skibvoid 211216695Skibwlock_acquire(rtld_lock_t lock, RtldLockState *lockstate) 212115396Skan{ 213216695Skib 214216695Skib if (lockstate == NULL) 215216695Skib return; 216216695Skib 217183061Sdavidxu if (thread_mask_set(lock->mask) & lock->mask) { 218216695Skib dbg("wlock_acquire: recursed"); 219216695Skib lockstate->lockstate = RTLD_LOCK_UNLOCKED; 220216695Skib return; 221115396Skan } 222115396Skan lockinfo.wlock_acquire(lock->handle); 223216695Skib lockstate->lockstate = RTLD_LOCK_WLOCKED; 224115396Skan} 225115396Skan 226115396Skanvoid 227216695Skiblock_release(rtld_lock_t lock, RtldLockState *lockstate) 228115396Skan{ 229216695Skib 230216695Skib if (lockstate == NULL) 231216695Skib return; 232216695Skib 233216695Skib switch (lockstate->lockstate) { 234216695Skib case RTLD_LOCK_UNLOCKED: 235216695Skib break; 236216695Skib case RTLD_LOCK_RLOCKED: 237216695Skib case RTLD_LOCK_WLOCKED: 238216695Skib thread_mask_clear(lock->mask); 239216695Skib lockinfo.lock_release(lock->handle); 240216695Skib break; 241216695Skib default: 242216695Skib assert(0); 243216695Skib } 244115396Skan} 245115396Skan 246115396Skanvoid 247216695Skiblock_upgrade(rtld_lock_t lock, RtldLockState *lockstate) 248115396Skan{ 249216695Skib 250216695Skib if (lockstate == NULL) 251216695Skib return; 252216695Skib 253216695Skib lock_release(lock, lockstate); 254216695Skib wlock_acquire(lock, lockstate); 255115396Skan} 256115396Skan 257115396Skanvoid 258216695Skiblock_restart_for_upgrade(RtldLockState *lockstate) 259216695Skib{ 260216695Skib 261216695Skib if (lockstate == NULL) 262216695Skib return; 263216695Skib 264216695Skib switch (lockstate->lockstate) { 265216695Skib case RTLD_LOCK_UNLOCKED: 266216695Skib case RTLD_LOCK_WLOCKED: 267216695Skib break; 268216695Skib case RTLD_LOCK_RLOCKED: 269218476Skib siglongjmp(lockstate->env, 1); 270216695Skib break; 271216695Skib default: 272216695Skib assert(0); 273216695Skib } 274216695Skib} 275216695Skib 276216695Skibvoid 277312340Skiblockdflt_init(void) 278115396Skan{ 279115396Skan int i; 280115396Skan 281115396Skan deflockinfo.rtli_version = RTLI_VERSION; 282115396Skan deflockinfo.lock_create = def_lock_create; 283115396Skan deflockinfo.lock_destroy = def_lock_destroy; 284115396Skan deflockinfo.rlock_acquire = def_rlock_acquire; 285115396Skan deflockinfo.wlock_acquire = def_wlock_acquire; 286115396Skan deflockinfo.lock_release = def_lock_release; 287115396Skan deflockinfo.thread_set_flag = def_thread_set_flag; 288115396Skan deflockinfo.thread_clr_flag = def_thread_clr_flag; 289115396Skan deflockinfo.at_fork = NULL; 290115396Skan 291115396Skan for (i = 0; i < RTLD_LOCK_CNT; i++) { 292115396Skan rtld_locks[i].mask = (1 << i); 293115396Skan rtld_locks[i].handle = NULL; 294115396Skan } 295115396Skan 296115396Skan memcpy(&lockinfo, &deflockinfo, sizeof(lockinfo)); 297115396Skan _rtld_thread_init(NULL); 298115396Skan /* 299115396Skan * Construct a mask to block all signals except traps which might 300115396Skan * conceivably be generated within the dynamic linker itself. 301115396Skan */ 302115396Skan sigfillset(&fullsigmask); 303115396Skan sigdelset(&fullsigmask, SIGILL); 304115396Skan sigdelset(&fullsigmask, SIGTRAP); 305115396Skan sigdelset(&fullsigmask, SIGABRT); 306115396Skan sigdelset(&fullsigmask, SIGEMT); 307115396Skan sigdelset(&fullsigmask, SIGFPE); 308115396Skan sigdelset(&fullsigmask, SIGBUS); 309115396Skan sigdelset(&fullsigmask, SIGSEGV); 310115396Skan sigdelset(&fullsigmask, SIGSYS); 311115396Skan} 312115396Skan 313115396Skan/* 314115396Skan * Callback function to allow threads implementation to 315115396Skan * register their own locking primitives if the default 316115396Skan * one is not suitable. 317115396Skan * The current context should be the only context 318115396Skan * executing at the invocation time. 319115396Skan */ 320115396Skanvoid 321115396Skan_rtld_thread_init(struct RtldLockInfo *pli) 322115396Skan{ 323115396Skan int flags, i; 324115396Skan void *locks[RTLD_LOCK_CNT]; 325115396Skan 326115396Skan /* disable all locking while this function is running */ 327115396Skan flags = thread_mask_set(~0); 328115396Skan 329115396Skan if (pli == NULL) 330115396Skan pli = &deflockinfo; 331115396Skan 332115396Skan 333115396Skan for (i = 0; i < RTLD_LOCK_CNT; i++) 334115396Skan if ((locks[i] = pli->lock_create()) == NULL) 335115396Skan break; 336115396Skan 337115396Skan if (i < RTLD_LOCK_CNT) { 338115396Skan while (--i >= 0) 339115396Skan pli->lock_destroy(locks[i]); 340115396Skan abort(); 341115396Skan } 342115396Skan 343115396Skan for (i = 0; i < RTLD_LOCK_CNT; i++) { 344115396Skan if (rtld_locks[i].handle == NULL) 345115396Skan continue; 346115396Skan if (flags & rtld_locks[i].mask) 347115396Skan lockinfo.lock_release(rtld_locks[i].handle); 348115396Skan lockinfo.lock_destroy(rtld_locks[i].handle); 349115396Skan } 350115396Skan 351115396Skan for (i = 0; i < RTLD_LOCK_CNT; i++) { 352115396Skan rtld_locks[i].handle = locks[i]; 353115396Skan if (flags & rtld_locks[i].mask) 354115396Skan pli->wlock_acquire(rtld_locks[i].handle); 355115396Skan } 356115396Skan 357115396Skan lockinfo.lock_create = pli->lock_create; 358115396Skan lockinfo.lock_destroy = pli->lock_destroy; 359115396Skan lockinfo.rlock_acquire = pli->rlock_acquire; 360115396Skan lockinfo.wlock_acquire = pli->wlock_acquire; 361115396Skan lockinfo.lock_release = pli->lock_release; 362115396Skan lockinfo.thread_set_flag = pli->thread_set_flag; 363115396Skan lockinfo.thread_clr_flag = pli->thread_clr_flag; 364115396Skan lockinfo.at_fork = pli->at_fork; 365115396Skan 366115396Skan /* restore thread locking state, this time with new locks */ 367115396Skan thread_mask_clear(~0); 368115396Skan thread_mask_set(flags); 369115396Skan dbg("_rtld_thread_init: done"); 370115396Skan} 371185369Skib 372185369Skibvoid 373185369Skib_rtld_atfork_pre(int *locks) 374185369Skib{ 375216695Skib RtldLockState ls[2]; 376185369Skib 377267200Skib if (locks == NULL) 378267200Skib return; 379267200Skib 380267200Skib /* 381312701Skib * Warning: this did not worked well with the rtld compat 382312701Skib * locks above, when the thread signal mask was corrupted (set 383312701Skib * to all signals blocked) if two locks were taken 384312701Skib * simultaneously in the write mode. The caller of the 385312701Skib * _rtld_atfork_pre() must provide the working implementation 386312701Skib * of the locks anyway, and libthr locks are fine. 387267200Skib */ 388216695Skib wlock_acquire(rtld_phdr_lock, &ls[0]); 389267200Skib wlock_acquire(rtld_bind_lock, &ls[1]); 390216695Skib 391216695Skib /* XXXKIB: I am really sorry for this. */ 392216695Skib locks[0] = ls[1].lockstate; 393216695Skib locks[2] = ls[0].lockstate; 394185369Skib} 395185369Skib 396185369Skibvoid 397185369Skib_rtld_atfork_post(int *locks) 398185369Skib{ 399216695Skib RtldLockState ls[2]; 400185369Skib 401267200Skib if (locks == NULL) 402267200Skib return; 403267200Skib 404216695Skib bzero(ls, sizeof(ls)); 405216695Skib ls[0].lockstate = locks[2]; 406216695Skib ls[1].lockstate = locks[0]; 407216695Skib lock_release(rtld_bind_lock, &ls[1]); 408216695Skib lock_release(rtld_phdr_lock, &ls[0]); 409185369Skib} 410