1/*- 2 * Copyright (c) 2015 The FreeBSD Foundation 3 * 4 * This software was developed by Konstantin Belousov 5 * under sponsorship from the FreeBSD Foundation. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 1. Redistributions of source code must retain the above copyright 11 * notice, this list of conditions and the following disclaimer. 12 * 2. Redistributions in binary form must reproduce the above copyright 13 * notice, this list of conditions and the following disclaimer in the 14 * documentation and/or other materials provided with the distribution. 15 * 16 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 17 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 18 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 19 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 20 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 21 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 22 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 23 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 24 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 25 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 26 * SUCH DAMAGE. 27 */ 28 29#include <sys/cdefs.h> 30__FBSDID("$FreeBSD$"); 31 32#include <sys/types.h> 33#include <sys/mman.h> 34#include <sys/queue.h> 35#include "namespace.h" 36#include <stdlib.h> 37#include "un-namespace.h" 38 39#include "thr_private.h" 40 41struct psh { 42 LIST_ENTRY(psh) link; 43 void *key; 44 void *val; 45}; 46 47LIST_HEAD(pshared_hash_head, psh); 48#define HASH_SIZE 128 49static struct pshared_hash_head pshared_hash[HASH_SIZE]; 50#define PSHARED_KEY_HASH(key) (((unsigned long)(key) >> 8) % HASH_SIZE) 51/* XXXKIB: lock could be split to per-hash chain, if appears contested */ 52static struct urwlock pshared_lock = DEFAULT_URWLOCK; 53 54void 55__thr_pshared_init(void) 56{ 57 int i; 58 59 _thr_urwlock_init(&pshared_lock); 60 for (i = 0; i < HASH_SIZE; i++) 61 LIST_INIT(&pshared_hash[i]); 62} 63 64static void 65pshared_rlock(struct pthread *curthread) 66{ 67 68 curthread->locklevel++; 69 _thr_rwl_rdlock(&pshared_lock); 70} 71 72static void 73pshared_wlock(struct pthread *curthread) 74{ 75 76 curthread->locklevel++; 77 _thr_rwl_wrlock(&pshared_lock); 78} 79 80static void 81pshared_unlock(struct pthread *curthread) 82{ 83 84 _thr_rwl_unlock(&pshared_lock); 85 curthread->locklevel--; 86 _thr_ast(curthread); 87} 88 89/* 90 * Among all processes sharing a lock only one executes 91 * pthread_lock_destroy(). Other processes still have the hash and 92 * mapped off-page. 93 * 94 * Mitigate the problem by checking the liveness of all hashed keys 95 * periodically. Right now this is executed on each 96 * pthread_lock_destroy(), but may be done less often if found to be 97 * too time-consuming. 98 */ 99static void 100pshared_gc(struct pthread *curthread) 101{ 102 struct pshared_hash_head *hd; 103 struct psh *h, *h1; 104 int error, i; 105 106 pshared_wlock(curthread); 107 for (i = 0; i < HASH_SIZE; i++) { 108 hd = &pshared_hash[i]; 109 LIST_FOREACH_SAFE(h, hd, link, h1) { 110 error = _umtx_op(NULL, UMTX_OP_SHM, UMTX_SHM_ALIVE, 111 h->val, NULL); 112 if (error == 0) 113 continue; 114 LIST_REMOVE(h, link); 115 munmap(h->val, PAGE_SIZE); 116 free(h); 117 } 118 } 119 pshared_unlock(curthread); 120} 121 122static void * 123pshared_lookup(void *key) 124{ 125 struct pshared_hash_head *hd; 126 struct psh *h; 127 128 hd = &pshared_hash[PSHARED_KEY_HASH(key)]; 129 LIST_FOREACH(h, hd, link) { 130 if (h->key == key) 131 return (h->val); 132 } 133 return (NULL); 134} 135 136static int 137pshared_insert(void *key, void **val) 138{ 139 struct pshared_hash_head *hd; 140 struct psh *h; 141 142 hd = &pshared_hash[PSHARED_KEY_HASH(key)]; 143 LIST_FOREACH(h, hd, link) { 144 /* 145 * When the key already exists in the hash, we should 146 * return either the new (just mapped) or old (hashed) 147 * val, and the other val should be unmapped to avoid 148 * address space leak. 149 * 150 * If two threads perform lock of the same object 151 * which is not yet stored in the pshared_hash, then 152 * the val already inserted by the first thread should 153 * be returned, and the second val freed (order is by 154 * the pshared_lock()). Otherwise, if we unmap the 155 * value obtained from the hash, the first thread 156 * might operate on an unmapped off-page object. 157 * 158 * There is still an issue: if hashed key was unmapped 159 * and then other page is mapped at the same key 160 * address, the hash would return the old val. I 161 * decided to handle the race of simultaneous hash 162 * insertion, leaving the unlikely remap problem 163 * unaddressed. 164 */ 165 if (h->key == key) { 166 if (h->val != *val) { 167 munmap(*val, PAGE_SIZE); 168 *val = h->val; 169 } 170 return (1); 171 } 172 } 173 174 h = malloc(sizeof(*h)); 175 if (h == NULL) 176 return (0); 177 h->key = key; 178 h->val = *val; 179 LIST_INSERT_HEAD(hd, h, link); 180 return (1); 181} 182 183static void * 184pshared_remove(void *key) 185{ 186 struct pshared_hash_head *hd; 187 struct psh *h; 188 void *val; 189 190 hd = &pshared_hash[PSHARED_KEY_HASH(key)]; 191 LIST_FOREACH(h, hd, link) { 192 if (h->key == key) { 193 LIST_REMOVE(h, link); 194 val = h->val; 195 free(h); 196 return (val); 197 } 198 } 199 return (NULL); 200} 201 202static void 203pshared_clean(void *key, void *val) 204{ 205 206 if (val != NULL) 207 munmap(val, PAGE_SIZE); 208 _umtx_op(NULL, UMTX_OP_SHM, UMTX_SHM_DESTROY, key, NULL); 209} 210 211void * 212__thr_pshared_offpage(void *key, int doalloc) 213{ 214 struct pthread *curthread; 215 void *res; 216 int fd, ins_done; 217 218 curthread = _get_curthread(); 219 pshared_rlock(curthread); 220 res = pshared_lookup(key); 221 pshared_unlock(curthread); 222 if (res != NULL) 223 return (res); 224 fd = _umtx_op(NULL, UMTX_OP_SHM, doalloc ? UMTX_SHM_CREAT : 225 UMTX_SHM_LOOKUP, key, NULL); 226 if (fd == -1) 227 return (NULL); 228 res = mmap(NULL, PAGE_SIZE, PROT_READ | PROT_WRITE, MAP_SHARED, fd, 0); 229 close(fd); 230 if (res == MAP_FAILED) 231 return (NULL); 232 pshared_wlock(curthread); 233 ins_done = pshared_insert(key, &res); 234 pshared_unlock(curthread); 235 if (!ins_done) { 236 pshared_clean(key, res); 237 res = NULL; 238 } 239 return (res); 240} 241 242void 243__thr_pshared_destroy(void *key) 244{ 245 struct pthread *curthread; 246 void *val; 247 248 curthread = _get_curthread(); 249 pshared_wlock(curthread); 250 val = pshared_remove(key); 251 pshared_unlock(curthread); 252 pshared_clean(key, val); 253 pshared_gc(curthread); 254} 255 256void 257__thr_pshared_atfork_pre(void) 258{ 259 260 _thr_rwl_rdlock(&pshared_lock); 261} 262 263void 264__thr_pshared_atfork_post(void) 265{ 266 267 _thr_rwl_unlock(&pshared_lock); 268} 269