1115396Skan/*- 2115396Skan * Copyright 1999, 2000 John D. Polstra. 3115396Skan * All rights reserved. 4115396Skan * 5115396Skan * Redistribution and use in source and binary forms, with or without 6115396Skan * modification, are permitted provided that the following conditions 7115396Skan * are met: 8115396Skan * 1. Redistributions of source code must retain the above copyright 9115396Skan * notice, this list of conditions and the following disclaimer. 10115396Skan * 2. Redistributions in binary form must reproduce the above copyright 11115396Skan * notice, this list of conditions and the following disclaimer in the 12115396Skan * documentation and/or other materials provided with the distribution. 13115396Skan * 14115396Skan * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 15115396Skan * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 16115396Skan * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 17115396Skan * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, 18115396Skan * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT 19115396Skan * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 20115396Skan * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 21115396Skan * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 22115396Skan * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF 23115396Skan * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 24115396Skan * 25115396Skan * from: FreeBSD: src/libexec/rtld-elf/sparc64/lockdflt.c,v 1.3 2002/10/09 26115396Skan * $FreeBSD$ 27115396Skan */ 28115396Skan 29115396Skan/* 30115396Skan * Thread locking implementation for the dynamic linker. 31115396Skan * 32115396Skan * We use the "simple, non-scalable reader-preference lock" from: 33115396Skan * 34115396Skan * J. M. Mellor-Crummey and M. L. Scott. "Scalable Reader-Writer 35115396Skan * Synchronization for Shared-Memory Multiprocessors." 3rd ACM Symp. on 36115396Skan * Principles and Practice of Parallel Programming, April 1991. 37115396Skan * 38115396Skan * In this algorithm the lock is a single word. Its low-order bit is 39115396Skan * set when a writer holds the lock. The remaining high-order bits 40115396Skan * contain a count of readers desiring the lock. The algorithm requires 41115396Skan * atomic "compare_and_store" and "add" operations, which we implement 42115396Skan * using assembly language sequences in "rtld_start.S". 43115396Skan */ 44115396Skan 45191303Srwatson#include <sys/param.h> 46115396Skan#include <signal.h> 47115396Skan#include <stdlib.h> 48115396Skan#include <time.h> 49115396Skan 50115396Skan#include "debug.h" 51115396Skan#include "rtld.h" 52115396Skan#include "rtld_machdep.h" 53115396Skan 54115396Skan#define WAFLAG 0x1 /* A writer holds the lock */ 55115396Skan#define RC_INCR 0x2 /* Adjusts count of readers desiring lock */ 56115396Skan 57115396Skantypedef struct Struct_Lock { 58168311Skan volatile u_int lock; 59115396Skan void *base; 60115396Skan} Lock; 61115396Skan 62115396Skanstatic sigset_t fullsigmask, oldsigmask; 63115396Skanstatic int thread_flag; 64115396Skan 65115396Skanstatic void * 66115396Skandef_lock_create() 67115396Skan{ 68115396Skan void *base; 69115396Skan char *p; 70115396Skan uintptr_t r; 71115396Skan Lock *l; 72115396Skan 73115396Skan /* 74115396Skan * Arrange for the lock to occupy its own cache line. First, we 75115396Skan * optimistically allocate just a cache line, hoping that malloc 76115396Skan * will give us a well-aligned block of memory. If that doesn't 77115396Skan * work, we allocate a larger block and take a well-aligned cache 78115396Skan * line from it. 79115396Skan */ 80115396Skan base = xmalloc(CACHE_LINE_SIZE); 81115396Skan p = (char *)base; 82115396Skan if ((uintptr_t)p % CACHE_LINE_SIZE != 0) { 83115396Skan free(base); 84115396Skan base = xmalloc(2 * CACHE_LINE_SIZE); 85115396Skan p = (char *)base; 86115396Skan if ((r = (uintptr_t)p % CACHE_LINE_SIZE) != 0) 87115396Skan p += CACHE_LINE_SIZE - r; 88115396Skan } 89115396Skan l = (Lock *)p; 90115396Skan l->base = base; 91115396Skan l->lock = 0; 92115396Skan return l; 93115396Skan} 94115396Skan 95115396Skanstatic void 96115396Skandef_lock_destroy(void *lock) 97115396Skan{ 98115396Skan Lock *l = (Lock *)lock; 99115396Skan 100115396Skan free(l->base); 101115396Skan} 102115396Skan 103115396Skanstatic void 104115396Skandef_rlock_acquire(void *lock) 105115396Skan{ 106115396Skan Lock *l = (Lock *)lock; 107115396Skan 108115396Skan atomic_add_acq_int(&l->lock, RC_INCR); 109115396Skan while (l->lock & WAFLAG) 110115396Skan ; /* Spin */ 111115396Skan} 112115396Skan 113115396Skanstatic void 114115396Skandef_wlock_acquire(void *lock) 115115396Skan{ 116115396Skan Lock *l = (Lock *)lock; 117115396Skan sigset_t tmp_oldsigmask; 118115396Skan 119115396Skan for ( ; ; ) { 120115396Skan sigprocmask(SIG_BLOCK, &fullsigmask, &tmp_oldsigmask); 121115396Skan if (atomic_cmpset_acq_int(&l->lock, 0, WAFLAG)) 122115396Skan break; 123115396Skan sigprocmask(SIG_SETMASK, &tmp_oldsigmask, NULL); 124115396Skan } 125115396Skan oldsigmask = tmp_oldsigmask; 126115396Skan} 127115396Skan 128115396Skanstatic void 129115396Skandef_lock_release(void *lock) 130115396Skan{ 131115396Skan Lock *l = (Lock *)lock; 132115396Skan 133115396Skan if ((l->lock & WAFLAG) == 0) 134115396Skan atomic_add_rel_int(&l->lock, -RC_INCR); 135115396Skan else { 136115396Skan atomic_add_rel_int(&l->lock, -WAFLAG); 137115396Skan sigprocmask(SIG_SETMASK, &oldsigmask, NULL); 138115396Skan } 139115396Skan} 140115396Skan 141115396Skanstatic int 142115396Skandef_thread_set_flag(int mask) 143115396Skan{ 144115396Skan int old_val = thread_flag; 145115396Skan thread_flag |= mask; 146115396Skan return (old_val); 147115396Skan} 148115396Skan 149115396Skanstatic int 150115396Skandef_thread_clr_flag(int mask) 151115396Skan{ 152115396Skan int old_val = thread_flag; 153115396Skan thread_flag &= ~mask; 154115396Skan return (old_val); 155115396Skan} 156115396Skan 157115396Skan/* 158115396Skan * Public interface exposed to the rest of the dynamic linker. 159115396Skan */ 160115396Skanstatic struct RtldLockInfo lockinfo; 161115396Skanstatic struct RtldLockInfo deflockinfo; 162115396Skan 163131575Sstefanfstatic __inline int 164115396Skanthread_mask_set(int mask) 165115396Skan{ 166115396Skan return lockinfo.thread_set_flag(mask); 167115396Skan} 168115396Skan 169131575Sstefanfstatic __inline void 170115396Skanthread_mask_clear(int mask) 171115396Skan{ 172115396Skan lockinfo.thread_clr_flag(mask); 173115396Skan} 174115396Skan 175178807Skib#define RTLD_LOCK_CNT 3 176115396Skanstruct rtld_lock { 177115396Skan void *handle; 178115396Skan int mask; 179115396Skan} rtld_locks[RTLD_LOCK_CNT]; 180115396Skan 181115396Skanrtld_lock_t rtld_bind_lock = &rtld_locks[0]; 182115396Skanrtld_lock_t rtld_libc_lock = &rtld_locks[1]; 183178807Skibrtld_lock_t rtld_phdr_lock = &rtld_locks[2]; 184115396Skan 185216695Skibvoid 186216695Skibrlock_acquire(rtld_lock_t lock, RtldLockState *lockstate) 187115396Skan{ 188216695Skib 189216695Skib if (lockstate == NULL) 190216695Skib return; 191216695Skib 192183061Sdavidxu if (thread_mask_set(lock->mask) & lock->mask) { 193216695Skib dbg("rlock_acquire: recursed"); 194216695Skib lockstate->lockstate = RTLD_LOCK_UNLOCKED; 195216695Skib return; 196115396Skan } 197115396Skan lockinfo.rlock_acquire(lock->handle); 198216695Skib lockstate->lockstate = RTLD_LOCK_RLOCKED; 199115396Skan} 200115396Skan 201216695Skibvoid 202216695Skibwlock_acquire(rtld_lock_t lock, RtldLockState *lockstate) 203115396Skan{ 204216695Skib 205216695Skib if (lockstate == NULL) 206216695Skib return; 207216695Skib 208183061Sdavidxu if (thread_mask_set(lock->mask) & lock->mask) { 209216695Skib dbg("wlock_acquire: recursed"); 210216695Skib lockstate->lockstate = RTLD_LOCK_UNLOCKED; 211216695Skib return; 212115396Skan } 213115396Skan lockinfo.wlock_acquire(lock->handle); 214216695Skib lockstate->lockstate = RTLD_LOCK_WLOCKED; 215115396Skan} 216115396Skan 217115396Skanvoid 218216695Skiblock_release(rtld_lock_t lock, RtldLockState *lockstate) 219115396Skan{ 220216695Skib 221216695Skib if (lockstate == NULL) 222216695Skib return; 223216695Skib 224216695Skib switch (lockstate->lockstate) { 225216695Skib case RTLD_LOCK_UNLOCKED: 226216695Skib break; 227216695Skib case RTLD_LOCK_RLOCKED: 228216695Skib case RTLD_LOCK_WLOCKED: 229216695Skib thread_mask_clear(lock->mask); 230216695Skib lockinfo.lock_release(lock->handle); 231216695Skib break; 232216695Skib default: 233216695Skib assert(0); 234216695Skib } 235115396Skan} 236115396Skan 237115396Skanvoid 238216695Skiblock_upgrade(rtld_lock_t lock, RtldLockState *lockstate) 239115396Skan{ 240216695Skib 241216695Skib if (lockstate == NULL) 242216695Skib return; 243216695Skib 244216695Skib lock_release(lock, lockstate); 245216695Skib wlock_acquire(lock, lockstate); 246115396Skan} 247115396Skan 248115396Skanvoid 249216695Skiblock_restart_for_upgrade(RtldLockState *lockstate) 250216695Skib{ 251216695Skib 252216695Skib if (lockstate == NULL) 253216695Skib return; 254216695Skib 255216695Skib switch (lockstate->lockstate) { 256216695Skib case RTLD_LOCK_UNLOCKED: 257216695Skib case RTLD_LOCK_WLOCKED: 258216695Skib break; 259216695Skib case RTLD_LOCK_RLOCKED: 260218476Skib siglongjmp(lockstate->env, 1); 261216695Skib break; 262216695Skib default: 263216695Skib assert(0); 264216695Skib } 265216695Skib} 266216695Skib 267216695Skibvoid 268115396Skanlockdflt_init() 269115396Skan{ 270115396Skan int i; 271115396Skan 272115396Skan deflockinfo.rtli_version = RTLI_VERSION; 273115396Skan deflockinfo.lock_create = def_lock_create; 274115396Skan deflockinfo.lock_destroy = def_lock_destroy; 275115396Skan deflockinfo.rlock_acquire = def_rlock_acquire; 276115396Skan deflockinfo.wlock_acquire = def_wlock_acquire; 277115396Skan deflockinfo.lock_release = def_lock_release; 278115396Skan deflockinfo.thread_set_flag = def_thread_set_flag; 279115396Skan deflockinfo.thread_clr_flag = def_thread_clr_flag; 280115396Skan deflockinfo.at_fork = NULL; 281115396Skan 282115396Skan for (i = 0; i < RTLD_LOCK_CNT; i++) { 283115396Skan rtld_locks[i].mask = (1 << i); 284115396Skan rtld_locks[i].handle = NULL; 285115396Skan } 286115396Skan 287115396Skan memcpy(&lockinfo, &deflockinfo, sizeof(lockinfo)); 288115396Skan _rtld_thread_init(NULL); 289115396Skan /* 290115396Skan * Construct a mask to block all signals except traps which might 291115396Skan * conceivably be generated within the dynamic linker itself. 292115396Skan */ 293115396Skan sigfillset(&fullsigmask); 294115396Skan sigdelset(&fullsigmask, SIGILL); 295115396Skan sigdelset(&fullsigmask, SIGTRAP); 296115396Skan sigdelset(&fullsigmask, SIGABRT); 297115396Skan sigdelset(&fullsigmask, SIGEMT); 298115396Skan sigdelset(&fullsigmask, SIGFPE); 299115396Skan sigdelset(&fullsigmask, SIGBUS); 300115396Skan sigdelset(&fullsigmask, SIGSEGV); 301115396Skan sigdelset(&fullsigmask, SIGSYS); 302115396Skan} 303115396Skan 304115396Skan/* 305115396Skan * Callback function to allow threads implementation to 306115396Skan * register their own locking primitives if the default 307115396Skan * one is not suitable. 308115396Skan * The current context should be the only context 309115396Skan * executing at the invocation time. 310115396Skan */ 311115396Skanvoid 312115396Skan_rtld_thread_init(struct RtldLockInfo *pli) 313115396Skan{ 314115396Skan int flags, i; 315115396Skan void *locks[RTLD_LOCK_CNT]; 316115396Skan 317115396Skan /* disable all locking while this function is running */ 318115396Skan flags = thread_mask_set(~0); 319115396Skan 320115396Skan if (pli == NULL) 321115396Skan pli = &deflockinfo; 322115396Skan 323115396Skan 324115396Skan for (i = 0; i < RTLD_LOCK_CNT; i++) 325115396Skan if ((locks[i] = pli->lock_create()) == NULL) 326115396Skan break; 327115396Skan 328115396Skan if (i < RTLD_LOCK_CNT) { 329115396Skan while (--i >= 0) 330115396Skan pli->lock_destroy(locks[i]); 331115396Skan abort(); 332115396Skan } 333115396Skan 334115396Skan for (i = 0; i < RTLD_LOCK_CNT; i++) { 335115396Skan if (rtld_locks[i].handle == NULL) 336115396Skan continue; 337115396Skan if (flags & rtld_locks[i].mask) 338115396Skan lockinfo.lock_release(rtld_locks[i].handle); 339115396Skan lockinfo.lock_destroy(rtld_locks[i].handle); 340115396Skan } 341115396Skan 342115396Skan for (i = 0; i < RTLD_LOCK_CNT; i++) { 343115396Skan rtld_locks[i].handle = locks[i]; 344115396Skan if (flags & rtld_locks[i].mask) 345115396Skan pli->wlock_acquire(rtld_locks[i].handle); 346115396Skan } 347115396Skan 348115396Skan lockinfo.lock_create = pli->lock_create; 349115396Skan lockinfo.lock_destroy = pli->lock_destroy; 350115396Skan lockinfo.rlock_acquire = pli->rlock_acquire; 351115396Skan lockinfo.wlock_acquire = pli->wlock_acquire; 352115396Skan lockinfo.lock_release = pli->lock_release; 353115396Skan lockinfo.thread_set_flag = pli->thread_set_flag; 354115396Skan lockinfo.thread_clr_flag = pli->thread_clr_flag; 355115396Skan lockinfo.at_fork = pli->at_fork; 356115396Skan 357115396Skan /* restore thread locking state, this time with new locks */ 358115396Skan thread_mask_clear(~0); 359115396Skan thread_mask_set(flags); 360115396Skan dbg("_rtld_thread_init: done"); 361115396Skan} 362185369Skib 363185369Skibvoid 364185369Skib_rtld_atfork_pre(int *locks) 365185369Skib{ 366216695Skib RtldLockState ls[2]; 367185369Skib 368216695Skib wlock_acquire(rtld_phdr_lock, &ls[0]); 369216695Skib rlock_acquire(rtld_bind_lock, &ls[1]); 370216695Skib 371216695Skib /* XXXKIB: I am really sorry for this. */ 372216695Skib locks[0] = ls[1].lockstate; 373216695Skib locks[2] = ls[0].lockstate; 374185369Skib} 375185369Skib 376185369Skibvoid 377185369Skib_rtld_atfork_post(int *locks) 378185369Skib{ 379216695Skib RtldLockState ls[2]; 380185369Skib 381216695Skib bzero(ls, sizeof(ls)); 382216695Skib ls[0].lockstate = locks[2]; 383216695Skib ls[1].lockstate = locks[0]; 384216695Skib lock_release(rtld_bind_lock, &ls[1]); 385216695Skib lock_release(rtld_phdr_lock, &ls[0]); 386185369Skib} 387