1157111Sdavidxu/* 2157111Sdavidxu * Copyright (c) 2006, David Xu <davidxu@freebsd.org> 3157111Sdavidxu * All rights reserved. 4157111Sdavidxu * 5157111Sdavidxu * Redistribution and use in source and binary forms, with or without 6157111Sdavidxu * modification, are permitted provided that the following conditions 7157111Sdavidxu * are met: 8157111Sdavidxu * 1. Redistributions of source code must retain the above copyright 9157111Sdavidxu * notice unmodified, this list of conditions, and the following 10157111Sdavidxu * disclaimer. 11157111Sdavidxu * 2. Redistributions in binary form must reproduce the above copyright 12157111Sdavidxu * notice, this list of conditions and the following disclaimer in the 13157111Sdavidxu * documentation and/or other materials provided with the distribution. 14157111Sdavidxu * 15157111Sdavidxu * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 16157111Sdavidxu * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 17157111Sdavidxu * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 18157111Sdavidxu * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, 19157111Sdavidxu * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT 20157111Sdavidxu * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 21157111Sdavidxu * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 22157111Sdavidxu * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 23157111Sdavidxu * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF 24157111Sdavidxu * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 25157111Sdavidxu * 26157111Sdavidxu * $FreeBSD$ 27157111Sdavidxu * 28157111Sdavidxu */ 29157111Sdavidxu 30157111Sdavidxu /* 31157111Sdavidxu * A lockless rwlock for rtld. 32157111Sdavidxu */ 33157111Sdavidxu#include <sys/cdefs.h> 34217191Skib#include <sys/mman.h> 35217191Skib#include <link.h> 36157111Sdavidxu#include <stdlib.h> 37212076Sdavidxu#include <string.h> 38157111Sdavidxu 39157111Sdavidxu#include "rtld_lock.h" 40157111Sdavidxu#include "thr_private.h" 41157111Sdavidxu 42177445Sdavidxu#undef errno 43177445Sdavidxuextern int errno; 44177445Sdavidxu 45157111Sdavidxustatic int _thr_rtld_clr_flag(int); 46157111Sdavidxustatic void *_thr_rtld_lock_create(void); 47157111Sdavidxustatic void _thr_rtld_lock_destroy(void *); 48157111Sdavidxustatic void _thr_rtld_lock_release(void *); 49157111Sdavidxustatic void _thr_rtld_rlock_acquire(void *); 50157111Sdavidxustatic int _thr_rtld_set_flag(int); 51157111Sdavidxustatic void _thr_rtld_wlock_acquire(void *); 52157111Sdavidxu 53157111Sdavidxustruct rtld_lock { 54185558Skib struct urwlock lock; 55185558Skib char _pad[CACHE_LINE_SIZE - sizeof(struct urwlock)]; 56157111Sdavidxu}; 57157111Sdavidxu 58185558Skibstatic struct rtld_lock lock_place[MAX_RTLD_LOCKS] __aligned(CACHE_LINE_SIZE); 59185558Skibstatic int busy_places; 60185558Skib 61157111Sdavidxustatic void * 62157111Sdavidxu_thr_rtld_lock_create(void) 63157111Sdavidxu{ 64185558Skib int locki; 65185558Skib struct rtld_lock *l; 66185558Skib static const char fail[] = "_thr_rtld_lock_create failed\n"; 67157111Sdavidxu 68185558Skib for (locki = 0; locki < MAX_RTLD_LOCKS; locki++) { 69185558Skib if ((busy_places & (1 << locki)) == 0) 70185558Skib break; 71157111Sdavidxu } 72185558Skib if (locki == MAX_RTLD_LOCKS) { 73185558Skib write(2, fail, sizeof(fail) - 1); 74185558Skib return (NULL); 75185558Skib } 76185558Skib busy_places |= (1 << locki); 77185558Skib 78185558Skib l = &lock_place[locki]; 79178413Sdavidxu l->lock.rw_flags = URWLOCK_PREFER_READER; 80157111Sdavidxu return (l); 81157111Sdavidxu} 82157111Sdavidxu 83157111Sdavidxustatic void 84157111Sdavidxu_thr_rtld_lock_destroy(void *lock) 85157111Sdavidxu{ 86185558Skib int locki; 87211859Sdavidxu size_t i; 88185558Skib 89185558Skib locki = (struct rtld_lock *)lock - &lock_place[0]; 90211859Sdavidxu for (i = 0; i < sizeof(struct rtld_lock); ++i) 91211859Sdavidxu ((char *)lock)[i] = 0; 92185558Skib busy_places &= ~(1 << locki); 93157111Sdavidxu} 94157111Sdavidxu 95177445Sdavidxu#define SAVE_ERRNO() { \ 96177445Sdavidxu if (curthread != _thr_initial) \ 97177445Sdavidxu errsave = curthread->error; \ 98177445Sdavidxu else \ 99177445Sdavidxu errsave = errno; \ 100177445Sdavidxu} 101177445Sdavidxu 102177445Sdavidxu#define RESTORE_ERRNO() { \ 103177445Sdavidxu if (curthread != _thr_initial) \ 104177445Sdavidxu curthread->error = errsave; \ 105177445Sdavidxu else \ 106177445Sdavidxu errno = errsave; \ 107177445Sdavidxu} 108177445Sdavidxu 109157111Sdavidxustatic void 110157111Sdavidxu_thr_rtld_rlock_acquire(void *lock) 111157111Sdavidxu{ 112157111Sdavidxu struct pthread *curthread; 113157111Sdavidxu struct rtld_lock *l; 114177445Sdavidxu int errsave; 115157111Sdavidxu 116157111Sdavidxu curthread = _get_curthread(); 117177445Sdavidxu SAVE_ERRNO(); 118157111Sdavidxu l = (struct rtld_lock *)lock; 119157111Sdavidxu 120157111Sdavidxu THR_CRITICAL_ENTER(curthread); 121178413Sdavidxu while (_thr_rwlock_rdlock(&l->lock, 0, NULL) != 0) 122178413Sdavidxu ; 123195403Sattilio curthread->rdlock_count++; 124177445Sdavidxu RESTORE_ERRNO(); 125157111Sdavidxu} 126157111Sdavidxu 127157111Sdavidxustatic void 128157111Sdavidxu_thr_rtld_wlock_acquire(void *lock) 129157111Sdavidxu{ 130157111Sdavidxu struct pthread *curthread; 131157111Sdavidxu struct rtld_lock *l; 132177445Sdavidxu int errsave; 133157111Sdavidxu 134157111Sdavidxu curthread = _get_curthread(); 135177445Sdavidxu SAVE_ERRNO(); 136157111Sdavidxu l = (struct rtld_lock *)lock; 137157111Sdavidxu 138212076Sdavidxu THR_CRITICAL_ENTER(curthread); 139178413Sdavidxu while (_thr_rwlock_wrlock(&l->lock, NULL) != 0) 140178413Sdavidxu ; 141178413Sdavidxu RESTORE_ERRNO(); 142157111Sdavidxu} 143157111Sdavidxu 144157111Sdavidxustatic void 145157111Sdavidxu_thr_rtld_lock_release(void *lock) 146157111Sdavidxu{ 147157111Sdavidxu struct pthread *curthread; 148157111Sdavidxu struct rtld_lock *l; 149178413Sdavidxu int32_t state; 150177445Sdavidxu int errsave; 151157111Sdavidxu 152157111Sdavidxu curthread = _get_curthread(); 153177445Sdavidxu SAVE_ERRNO(); 154157111Sdavidxu l = (struct rtld_lock *)lock; 155157111Sdavidxu 156178413Sdavidxu state = l->lock.rw_state; 157178413Sdavidxu if (_thr_rwlock_unlock(&l->lock) == 0) { 158212076Sdavidxu if ((state & URWLOCK_WRITE_OWNER) == 0) 159211833Sdavidxu curthread->rdlock_count--; 160212076Sdavidxu THR_CRITICAL_LEAVE(curthread); 161157111Sdavidxu } 162177445Sdavidxu RESTORE_ERRNO(); 163157111Sdavidxu} 164157111Sdavidxu 165157111Sdavidxustatic int 166157457Sdavidxu_thr_rtld_set_flag(int mask __unused) 167157111Sdavidxu{ 168157111Sdavidxu /* 169157111Sdavidxu * The caller's code in rtld-elf is broken, it is not signal safe, 170157111Sdavidxu * just return zero to fool it. 171157111Sdavidxu */ 172157111Sdavidxu return (0); 173157111Sdavidxu} 174157111Sdavidxu 175157111Sdavidxustatic int 176157457Sdavidxu_thr_rtld_clr_flag(int mask __unused) 177157111Sdavidxu{ 178157111Sdavidxu return (0); 179157111Sdavidxu} 180157111Sdavidxu 181157111Sdavidxuvoid 182157111Sdavidxu_thr_rtld_init(void) 183157111Sdavidxu{ 184157111Sdavidxu struct RtldLockInfo li; 185157111Sdavidxu struct pthread *curthread; 186185728Speter long dummy = -1; 187157111Sdavidxu 188157111Sdavidxu curthread = _get_curthread(); 189157111Sdavidxu 190157111Sdavidxu /* force to resolve _umtx_op PLT */ 191177853Sdavidxu _umtx_op_err((struct umtx *)&dummy, UMTX_OP_WAKE, 1, 0, 0); 192177470Sdavidxu 193177470Sdavidxu /* force to resolve errno() PLT */ 194177470Sdavidxu __error(); 195157111Sdavidxu 196212076Sdavidxu /* force to resolve memcpy PLT */ 197212076Sdavidxu memcpy(&dummy, &dummy, sizeof(dummy)); 198212076Sdavidxu 199217191Skib mprotect(NULL, 0, 0); 200217191Skib _rtld_get_stack_prot(); 201217191Skib 202157111Sdavidxu li.lock_create = _thr_rtld_lock_create; 203157111Sdavidxu li.lock_destroy = _thr_rtld_lock_destroy; 204157111Sdavidxu li.rlock_acquire = _thr_rtld_rlock_acquire; 205157111Sdavidxu li.wlock_acquire = _thr_rtld_wlock_acquire; 206157111Sdavidxu li.lock_release = _thr_rtld_lock_release; 207157111Sdavidxu li.thread_set_flag = _thr_rtld_set_flag; 208157111Sdavidxu li.thread_clr_flag = _thr_rtld_clr_flag; 209157111Sdavidxu li.at_fork = NULL; 210157111Sdavidxu 211157111Sdavidxu /* mask signals, also force to resolve __sys_sigprocmask PLT */ 212157111Sdavidxu _thr_signal_block(curthread); 213157111Sdavidxu _rtld_thread_init(&li); 214157111Sdavidxu _thr_signal_unblock(curthread); 215157111Sdavidxu} 216157111Sdavidxu 217157111Sdavidxuvoid 218157111Sdavidxu_thr_rtld_fini(void) 219157111Sdavidxu{ 220157111Sdavidxu struct pthread *curthread; 221157111Sdavidxu 222157111Sdavidxu curthread = _get_curthread(); 223157111Sdavidxu _thr_signal_block(curthread); 224157111Sdavidxu _rtld_thread_init(NULL); 225157111Sdavidxu _thr_signal_unblock(curthread); 226157111Sdavidxu} 227