1331722Seadler/* 2157111Sdavidxu * Copyright (c) 2006, David Xu <davidxu@freebsd.org> 3157111Sdavidxu * All rights reserved. 4157111Sdavidxu * 5157111Sdavidxu * Redistribution and use in source and binary forms, with or without 6157111Sdavidxu * modification, are permitted provided that the following conditions 7157111Sdavidxu * are met: 8157111Sdavidxu * 1. Redistributions of source code must retain the above copyright 9157111Sdavidxu * notice unmodified, this list of conditions, and the following 10157111Sdavidxu * disclaimer. 11157111Sdavidxu * 2. Redistributions in binary form must reproduce the above copyright 12157111Sdavidxu * notice, this list of conditions and the following disclaimer in the 13157111Sdavidxu * documentation and/or other materials provided with the distribution. 14157111Sdavidxu * 15157111Sdavidxu * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 16157111Sdavidxu * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 17157111Sdavidxu * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 18157111Sdavidxu * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, 19157111Sdavidxu * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT 20157111Sdavidxu * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 21157111Sdavidxu * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 22157111Sdavidxu * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 23157111Sdavidxu * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF 24157111Sdavidxu * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 25157111Sdavidxu */ 26157111Sdavidxu 27297706Skib#include <sys/cdefs.h> 28297706Skib__FBSDID("$FreeBSD$"); 29297706Skib 30157111Sdavidxu /* 31157111Sdavidxu * A lockless rwlock for rtld. 32157111Sdavidxu */ 33157111Sdavidxu#include <sys/cdefs.h> 34217191Skib#include <sys/mman.h> 35266609Skib#include <sys/syscall.h> 36217191Skib#include <link.h> 37157111Sdavidxu#include <stdlib.h> 38212076Sdavidxu#include <string.h> 39157111Sdavidxu 40266609Skib#include "libc_private.h" 41157111Sdavidxu#include "rtld_lock.h" 42157111Sdavidxu#include "thr_private.h" 43157111Sdavidxu 44177445Sdavidxu#undef errno 45177445Sdavidxuextern int errno; 46177445Sdavidxu 47157111Sdavidxustatic int _thr_rtld_clr_flag(int); 48157111Sdavidxustatic void *_thr_rtld_lock_create(void); 49157111Sdavidxustatic void _thr_rtld_lock_destroy(void *); 50157111Sdavidxustatic void _thr_rtld_lock_release(void *); 51157111Sdavidxustatic void _thr_rtld_rlock_acquire(void *); 52157111Sdavidxustatic int _thr_rtld_set_flag(int); 53157111Sdavidxustatic void _thr_rtld_wlock_acquire(void *); 54157111Sdavidxu 55157111Sdavidxustruct rtld_lock { 56185558Skib struct urwlock lock; 57185558Skib char _pad[CACHE_LINE_SIZE - sizeof(struct urwlock)]; 58157111Sdavidxu}; 59157111Sdavidxu 60185558Skibstatic struct rtld_lock lock_place[MAX_RTLD_LOCKS] __aligned(CACHE_LINE_SIZE); 61185558Skibstatic int busy_places; 62185558Skib 63157111Sdavidxustatic void * 64157111Sdavidxu_thr_rtld_lock_create(void) 65157111Sdavidxu{ 66185558Skib int locki; 67185558Skib struct rtld_lock *l; 68185558Skib static const char fail[] = "_thr_rtld_lock_create failed\n"; 69157111Sdavidxu 70185558Skib for (locki = 0; locki < MAX_RTLD_LOCKS; locki++) { 71185558Skib if ((busy_places & (1 << locki)) == 0) 72185558Skib break; 73157111Sdavidxu } 74185558Skib if (locki == MAX_RTLD_LOCKS) { 75185558Skib write(2, fail, sizeof(fail) - 1); 76185558Skib return (NULL); 77185558Skib } 78185558Skib busy_places |= (1 << locki); 79185558Skib 80185558Skib l = &lock_place[locki]; 81178413Sdavidxu l->lock.rw_flags = URWLOCK_PREFER_READER; 82157111Sdavidxu return (l); 83157111Sdavidxu} 84157111Sdavidxu 85157111Sdavidxustatic void 86157111Sdavidxu_thr_rtld_lock_destroy(void *lock) 87157111Sdavidxu{ 88185558Skib int locki; 89211859Sdavidxu size_t i; 90185558Skib 91185558Skib locki = (struct rtld_lock *)lock - &lock_place[0]; 92211859Sdavidxu for (i = 0; i < sizeof(struct rtld_lock); ++i) 93211859Sdavidxu ((char *)lock)[i] = 0; 94185558Skib busy_places &= ~(1 << locki); 95157111Sdavidxu} 96157111Sdavidxu 97177445Sdavidxu#define SAVE_ERRNO() { \ 98177445Sdavidxu if (curthread != _thr_initial) \ 99177445Sdavidxu errsave = curthread->error; \ 100177445Sdavidxu else \ 101177445Sdavidxu errsave = errno; \ 102177445Sdavidxu} 103177445Sdavidxu 104177445Sdavidxu#define RESTORE_ERRNO() { \ 105177445Sdavidxu if (curthread != _thr_initial) \ 106177445Sdavidxu curthread->error = errsave; \ 107177445Sdavidxu else \ 108177445Sdavidxu errno = errsave; \ 109177445Sdavidxu} 110177445Sdavidxu 111157111Sdavidxustatic void 112157111Sdavidxu_thr_rtld_rlock_acquire(void *lock) 113157111Sdavidxu{ 114157111Sdavidxu struct pthread *curthread; 115157111Sdavidxu struct rtld_lock *l; 116177445Sdavidxu int errsave; 117157111Sdavidxu 118157111Sdavidxu curthread = _get_curthread(); 119177445Sdavidxu SAVE_ERRNO(); 120157111Sdavidxu l = (struct rtld_lock *)lock; 121157111Sdavidxu 122157111Sdavidxu THR_CRITICAL_ENTER(curthread); 123178413Sdavidxu while (_thr_rwlock_rdlock(&l->lock, 0, NULL) != 0) 124178413Sdavidxu ; 125195403Sattilio curthread->rdlock_count++; 126177445Sdavidxu RESTORE_ERRNO(); 127157111Sdavidxu} 128157111Sdavidxu 129157111Sdavidxustatic void 130157111Sdavidxu_thr_rtld_wlock_acquire(void *lock) 131157111Sdavidxu{ 132157111Sdavidxu struct pthread *curthread; 133157111Sdavidxu struct rtld_lock *l; 134177445Sdavidxu int errsave; 135157111Sdavidxu 136157111Sdavidxu curthread = _get_curthread(); 137177445Sdavidxu SAVE_ERRNO(); 138157111Sdavidxu l = (struct rtld_lock *)lock; 139157111Sdavidxu 140212076Sdavidxu THR_CRITICAL_ENTER(curthread); 141178413Sdavidxu while (_thr_rwlock_wrlock(&l->lock, NULL) != 0) 142178413Sdavidxu ; 143178413Sdavidxu RESTORE_ERRNO(); 144157111Sdavidxu} 145157111Sdavidxu 146157111Sdavidxustatic void 147157111Sdavidxu_thr_rtld_lock_release(void *lock) 148157111Sdavidxu{ 149157111Sdavidxu struct pthread *curthread; 150157111Sdavidxu struct rtld_lock *l; 151178413Sdavidxu int32_t state; 152177445Sdavidxu int errsave; 153157111Sdavidxu 154157111Sdavidxu curthread = _get_curthread(); 155177445Sdavidxu SAVE_ERRNO(); 156157111Sdavidxu l = (struct rtld_lock *)lock; 157157111Sdavidxu 158178413Sdavidxu state = l->lock.rw_state; 159178413Sdavidxu if (_thr_rwlock_unlock(&l->lock) == 0) { 160212076Sdavidxu if ((state & URWLOCK_WRITE_OWNER) == 0) 161211833Sdavidxu curthread->rdlock_count--; 162212076Sdavidxu THR_CRITICAL_LEAVE(curthread); 163157111Sdavidxu } 164177445Sdavidxu RESTORE_ERRNO(); 165157111Sdavidxu} 166157111Sdavidxu 167157111Sdavidxustatic int 168157457Sdavidxu_thr_rtld_set_flag(int mask __unused) 169157111Sdavidxu{ 170157111Sdavidxu /* 171157111Sdavidxu * The caller's code in rtld-elf is broken, it is not signal safe, 172157111Sdavidxu * just return zero to fool it. 173157111Sdavidxu */ 174157111Sdavidxu return (0); 175157111Sdavidxu} 176157111Sdavidxu 177157111Sdavidxustatic int 178157457Sdavidxu_thr_rtld_clr_flag(int mask __unused) 179157111Sdavidxu{ 180157111Sdavidxu return (0); 181157111Sdavidxu} 182157111Sdavidxu 183157111Sdavidxuvoid 184157111Sdavidxu_thr_rtld_init(void) 185157111Sdavidxu{ 186157111Sdavidxu struct RtldLockInfo li; 187157111Sdavidxu struct pthread *curthread; 188286582Skib ucontext_t *uc; 189185728Speter long dummy = -1; 190286582Skib int uc_len; 191157111Sdavidxu 192157111Sdavidxu curthread = _get_curthread(); 193157111Sdavidxu 194157111Sdavidxu /* force to resolve _umtx_op PLT */ 195177853Sdavidxu _umtx_op_err((struct umtx *)&dummy, UMTX_OP_WAKE, 1, 0, 0); 196177470Sdavidxu 197177470Sdavidxu /* force to resolve errno() PLT */ 198177470Sdavidxu __error(); 199157111Sdavidxu 200212076Sdavidxu /* force to resolve memcpy PLT */ 201212076Sdavidxu memcpy(&dummy, &dummy, sizeof(dummy)); 202212076Sdavidxu 203217191Skib mprotect(NULL, 0, 0); 204217191Skib _rtld_get_stack_prot(); 205217191Skib 206157111Sdavidxu li.lock_create = _thr_rtld_lock_create; 207157111Sdavidxu li.lock_destroy = _thr_rtld_lock_destroy; 208157111Sdavidxu li.rlock_acquire = _thr_rtld_rlock_acquire; 209157111Sdavidxu li.wlock_acquire = _thr_rtld_wlock_acquire; 210157111Sdavidxu li.lock_release = _thr_rtld_lock_release; 211157111Sdavidxu li.thread_set_flag = _thr_rtld_set_flag; 212157111Sdavidxu li.thread_clr_flag = _thr_rtld_clr_flag; 213157111Sdavidxu li.at_fork = NULL; 214266609Skib 215266609Skib /* 216266609Skib * Preresolve the symbols needed for the fork interposer. We 217266609Skib * call _rtld_atfork_pre() and _rtld_atfork_post() with NULL 218266609Skib * argument to indicate that no actual locking inside the 219266609Skib * functions should happen. Neither rtld compat locks nor 220266609Skib * libthr rtld locks cannot work there: 221266609Skib * - compat locks do not handle the case of two locks taken 222266609Skib * in write mode (the signal mask for the thread is corrupted); 223266609Skib * - libthr locks would work, but locked rtld_bind_lock prevents 224266609Skib * symbol resolution for _rtld_atfork_post. 225266609Skib */ 226266609Skib _rtld_atfork_pre(NULL); 227266609Skib _rtld_atfork_post(NULL); 228266609Skib _malloc_prefork(); 229266609Skib _malloc_postfork(); 230297139Skib getpid(); 231266609Skib syscall(SYS_getpid); 232266609Skib 233157111Sdavidxu /* mask signals, also force to resolve __sys_sigprocmask PLT */ 234157111Sdavidxu _thr_signal_block(curthread); 235157111Sdavidxu _rtld_thread_init(&li); 236157111Sdavidxu _thr_signal_unblock(curthread); 237286582Skib 238286582Skib uc_len = __getcontextx_size(); 239286582Skib uc = alloca(uc_len); 240286582Skib getcontext(uc); 241286582Skib __fillcontextx2((char *)uc); 242157111Sdavidxu} 243