subr_pcu.c revision 1.16
1/* $NetBSD: subr_pcu.c,v 1.16 2013/11/26 20:29:40 rmind Exp $ */ 2 3/*- 4 * Copyright (c) 2011 The NetBSD Foundation, Inc. 5 * All rights reserved. 6 * 7 * This code is derived from software contributed to The NetBSD Foundation 8 * by Mindaugas Rasiukevicius. 9 * 10 * Redistribution and use in source and binary forms, with or without 11 * modification, are permitted provided that the following conditions 12 * are met: 13 * 1. Redistributions of source code must retain the above copyright 14 * notice, this list of conditions and the following disclaimer. 15 * 2. Redistributions in binary form must reproduce the above copyright 16 * notice, this list of conditions and the following disclaimer in the 17 * documentation and/or other materials provided with the distribution. 18 * 19 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS 20 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 21 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 22 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS 23 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 24 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 25 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 26 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 27 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 28 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 29 * POSSIBILITY OF SUCH DAMAGE. 30 */ 31 32/* 33 * Per CPU Unit (PCU) - is an interface to manage synchronization of any 34 * per CPU context (unit) tied with LWP context. Typical use: FPU state. 35 * 36 * Concurrency notes: 37 * 38 * PCU state may be loaded only by the current LWP, that is, curlwp. 39 * Therefore, only LWP itself can set a CPU for lwp_t::l_pcu_cpu[id]. 40 * 41 * Request for a PCU release can be from owner LWP (whether PCU state 42 * is on current CPU or remote CPU) or any other LWP running on that 43 * CPU (in such case, owner LWP is on a remote CPU or sleeping). 44 * 45 * In any case, PCU state can only be changed from the running CPU. 46 * If said PCU state is on the remote CPU, a cross-call will be sent 47 * by the owner LWP. Therefore struct cpu_info::ci_pcu_curlwp[id] 48 * may only be changed by current CPU, and lwp_t::l_pcu_cpu[id] may 49 * only be unset by the CPU which has PCU state loaded. 50 * 51 * There is a race condition: LWP may have a PCU state on a remote CPU, 52 * which it requests to be released via cross-call. At the same time, 53 * other LWP on remote CPU might release existing PCU state and load 54 * its own one. Cross-call may arrive after this and release different 55 * PCU state than intended. In such case, such LWP would re-load its 56 * PCU state again. 57 */ 58 59#include <sys/cdefs.h> 60__KERNEL_RCSID(0, "$NetBSD: subr_pcu.c,v 1.16 2013/11/26 20:29:40 rmind Exp $"); 61 62#include <sys/param.h> 63#include <sys/cpu.h> 64#include <sys/lwp.h> 65#include <sys/pcu.h> 66#include <sys/xcall.h> 67 68#if PCU_UNIT_COUNT > 0 69 70static inline void pcu_do_op(const pcu_ops_t *, lwp_t * const, const int); 71static void pcu_cpu_op(const pcu_ops_t *, const int); 72static void pcu_lwp_op(const pcu_ops_t *, lwp_t *, const int); 73 74__CTASSERT(PCU_KERNEL == 1); 75 76#define PCU_SAVE (PCU_LOADED << 1) /* Save PCU state to the LWP. */ 77#define PCU_RELEASE (PCU_SAVE << 1) /* Release PCU state on the CPU. */ 78#define PCU_CLAIM (PCU_RELEASE << 1) /* CLAIM a PCU for a LWP. */ 79 80/* XXX */ 81extern const pcu_ops_t * const pcu_ops_md_defs[]; 82 83/* 84 * pcu_switchpoint: release PCU state if the LWP is being run on another CPU. 85 * 86 * On each context switches, called by mi_switch() with IPL_SCHED. 87 * 'l' is an LWP which is just we switched to. (the new curlwp) 88 */ 89 90void 91pcu_switchpoint(lwp_t *l) 92{ 93 const uint32_t pcu_kernel_inuse = l->l_pcu_used[PCU_KERNEL]; 94 uint32_t pcu_user_inuse = l->l_pcu_used[PCU_USER]; 95 /* int s; */ 96 97 KASSERTMSG(l == curlwp, "l %p != curlwp %p", l, curlwp); 98 99 if (__predict_false(pcu_kernel_inuse != 0)) { 100 for (u_int id = 0; id < PCU_UNIT_COUNT; id++) { 101 if ((pcu_kernel_inuse & (1 << id)) == 0) { 102 continue; 103 } 104 struct cpu_info * const pcu_ci = l->l_pcu_cpu[id]; 105 if (pcu_ci == NULL || pcu_ci == l->l_cpu) { 106 continue; 107 } 108 const pcu_ops_t * const pcu = pcu_ops_md_defs[id]; 109 /* 110 * Steal the PCU away from the current owner and 111 * take ownership of it. 112 */ 113 pcu_cpu_op(pcu, PCU_SAVE | PCU_RELEASE); 114 pcu_do_op(pcu, l, PCU_KERNEL | PCU_CLAIM | PCU_RELOAD); 115 pcu_user_inuse &= ~(1 << id); 116 } 117 } 118 119 if (__predict_true(pcu_user_inuse == 0)) { 120 /* PCUs are not in use. */ 121 return; 122 } 123 /* commented out as we know we are already at IPL_SCHED */ 124 /* s = splsoftserial(); */ 125 for (u_int id = 0; id < PCU_UNIT_COUNT; id++) { 126 if ((pcu_user_inuse & (1 << id)) == 0) { 127 continue; 128 } 129 struct cpu_info * const pcu_ci = l->l_pcu_cpu[id]; 130 if (pcu_ci == NULL || pcu_ci == l->l_cpu) { 131 continue; 132 } 133 const pcu_ops_t * const pcu = pcu_ops_md_defs[id]; 134 pcu->pcu_state_release(l, 0); 135 } 136 /* splx(s); */ 137} 138 139/* 140 * pcu_discard_all: discard PCU state of the given LWP. 141 * 142 * Used by exec and LWP exit. 143 */ 144 145void 146pcu_discard_all(lwp_t *l) 147{ 148 const uint32_t pcu_inuse = l->l_pcu_used[PCU_USER]; 149 150 KASSERT(l == curlwp || ((l->l_flag & LW_SYSTEM) && pcu_inuse == 0)); 151 KASSERT(l->l_pcu_used[PCU_KERNEL] == 0); 152 153 if (__predict_true(pcu_inuse == 0)) { 154 /* PCUs are not in use. */ 155 return; 156 } 157 const int s = splsoftserial(); 158 for (u_int id = 0; id < PCU_UNIT_COUNT; id++) { 159 if ((pcu_inuse & (1 << id)) == 0) { 160 continue; 161 } 162 if (__predict_true(l->l_pcu_cpu[id] == NULL)) { 163 continue; 164 } 165 const pcu_ops_t * const pcu = pcu_ops_md_defs[id]; 166 /* 167 * We aren't releasing since this LWP isn't giving up PCU, 168 * just saving it. 169 */ 170 pcu_lwp_op(pcu, l, PCU_RELEASE); 171 } 172 l->l_pcu_used[PCU_USER] = 0; 173 splx(s); 174} 175 176/* 177 * pcu_save_all: save PCU state of the given LWP so that eg. coredump can 178 * examine it. 179 */ 180 181void 182pcu_save_all(lwp_t *l) 183{ 184 const uint32_t pcu_inuse = l->l_pcu_used[PCU_USER]; 185 /* 186 * Unless LW_WCORE, we aren't releasing since this LWP isn't giving 187 * up PCU, just saving it. 188 */ 189 const int flags = PCU_SAVE | (l->l_flag & LW_WCORE ? PCU_RELEASE : 0); 190 191 /* 192 * Normally we save for the current LWP, but sometimes we get called 193 * with a different LWP (forking a system LWP or doing a coredump of 194 * a process with multiple threads) and we need to deal with that. 195 */ 196 KASSERT(l == curlwp 197 || (((l->l_flag & LW_SYSTEM) 198 || (curlwp->l_proc == l->l_proc && l->l_stat == LSSUSPENDED)) 199 && pcu_inuse == 0)); 200 KASSERT(l->l_pcu_used[PCU_KERNEL] == 0); 201 202 if (__predict_true(pcu_inuse == 0)) { 203 /* PCUs are not in use. */ 204 return; 205 } 206 const int s = splsoftserial(); 207 for (u_int id = 0; id < PCU_UNIT_COUNT; id++) { 208 if ((pcu_inuse & (1 << id)) == 0) { 209 continue; 210 } 211 if (__predict_true(l->l_pcu_cpu[id] == NULL)) { 212 continue; 213 } 214 const pcu_ops_t * const pcu = pcu_ops_md_defs[id]; 215 pcu_lwp_op(pcu, l, flags); 216 } 217 splx(s); 218} 219 220/* 221 * pcu_do_op: save/release PCU state on the current CPU. 222 * 223 * => Must be called at IPL_SOFTSERIAL or from the soft-interrupt. 224 */ 225static inline void 226pcu_do_op(const pcu_ops_t *pcu, lwp_t * const l, const int flags) 227{ 228 struct cpu_info * const ci = curcpu(); 229 const u_int id = pcu->pcu_id; 230 u_int state_flags = flags & (PCU_KERNEL|PCU_RELOAD|PCU_ENABLE); 231 uint32_t id_mask = 1 << id; 232 const bool kernel_p = (l->l_pcu_used[PCU_KERNEL] & id_mask) != 0; 233 234 KASSERT(l->l_pcu_cpu[id] == (flags & PCU_CLAIM ? NULL : ci)); 235 236 if (flags & PCU_SAVE) { 237 pcu->pcu_state_save(l, (kernel_p ? PCU_KERNEL : 0)); 238 } 239 if (flags & PCU_RELEASE) { 240 pcu->pcu_state_release(l, state_flags); 241 if (flags & PCU_KERNEL) { 242 l->l_pcu_used[PCU_KERNEL] &= ~id_mask; 243 } 244 ci->ci_pcu_curlwp[id] = NULL; 245 l->l_pcu_cpu[id] = NULL; 246 } 247 if (flags & PCU_CLAIM) { 248 if (l->l_pcu_used[(flags & PCU_KERNEL)] & id_mask) 249 state_flags |= PCU_LOADED; 250 pcu->pcu_state_load(l, state_flags); 251 l->l_pcu_cpu[id] = ci; 252 ci->ci_pcu_curlwp[id] = l; 253 l->l_pcu_used[flags & PCU_KERNEL] |= id_mask; 254 } 255 if (flags == PCU_KERNEL) { 256 KASSERT(ci->ci_pcu_curlwp[id] == l); 257 pcu->pcu_state_save(l, 0); 258 l->l_pcu_used[PCU_KERNEL] |= id_mask; 259 } 260} 261 262/* 263 * pcu_cpu_op: helper routine to call pcu_do_op() via xcall(9) or 264 * by pcu_load. 265 */ 266static void 267pcu_cpu_op(const pcu_ops_t *pcu, const int flags) 268{ 269 const u_int id = pcu->pcu_id; 270 lwp_t * const l = curcpu()->ci_pcu_curlwp[id]; 271 272 //KASSERT(cpu_softintr_p()); 273 274 /* If no state - nothing to do. */ 275 if (l == NULL) { 276 return; 277 } 278 pcu_do_op(pcu, l, flags); 279} 280 281/* 282 * pcu_lwp_op: perform PCU state save, release or both operations on LWP. 283 */ 284static void 285pcu_lwp_op(const pcu_ops_t *pcu, lwp_t *l, const int flags) 286{ 287 const u_int id = pcu->pcu_id; 288 struct cpu_info *ci; 289 uint64_t where; 290 int s; 291 292 /* 293 * Caller should have re-checked if there is any state to manage. 294 * Block the interrupts and inspect again, since cross-call sent 295 * by remote CPU could have changed the state. 296 */ 297 s = splsoftserial(); 298 ci = l->l_pcu_cpu[id]; 299 if (ci == curcpu()) { 300 /* 301 * State is on the current CPU - just perform the operations. 302 */ 303 KASSERT((flags & PCU_CLAIM) == 0); 304 KASSERTMSG(ci->ci_pcu_curlwp[id] == l, 305 "%s: cpu%u: pcu_curlwp[%u] (%p) != l (%p)", 306 __func__, cpu_index(ci), id, ci->ci_pcu_curlwp[id], l); 307 pcu_do_op(pcu, l, flags); 308 splx(s); 309 return; 310 } 311 312 if (__predict_false(ci == NULL)) { 313 if (flags & PCU_CLAIM) { 314 pcu_do_op(pcu, l, flags); 315 } 316 /* Cross-call has won the race - no state to manage. */ 317 splx(s); 318 return; 319 } 320 321 splx(s); 322 323 /* 324 * State is on the remote CPU - perform the operations there. 325 * Note: there is a race condition; see description in the top. 326 */ 327 where = xc_unicast(XC_HIGHPRI, (xcfunc_t)pcu_cpu_op, 328 __UNCONST(pcu), (void *)(uintptr_t)flags, ci); 329 xc_wait(where); 330 331 KASSERT((flags & PCU_RELEASE) == 0 || l->l_pcu_cpu[id] == NULL); 332} 333 334/* 335 * pcu_load: load/initialize the PCU state of current LWP on current CPU. 336 */ 337void 338pcu_load(const pcu_ops_t *pcu) 339{ 340 const u_int id = pcu->pcu_id; 341 struct cpu_info *ci, *curci; 342 lwp_t * const l = curlwp; 343 uint64_t where; 344 int s; 345 346 KASSERT(!cpu_intr_p() && !cpu_softintr_p()); 347 348 s = splsoftserial(); 349 curci = curcpu(); 350 ci = l->l_pcu_cpu[id]; 351 352 /* Does this CPU already have our PCU state loaded? */ 353 if (ci == curci) { 354 KASSERT(curci->ci_pcu_curlwp[id] == l); 355 pcu->pcu_state_load(l, PCU_ENABLE); /* Re-enable */ 356 splx(s); 357 return; 358 } 359 360 /* If PCU state of this LWP is on the remote CPU - save it there. */ 361 if (ci) { 362 splx(s); 363 /* Note: there is a race; see description in the top. */ 364 where = xc_unicast(XC_HIGHPRI, (xcfunc_t)pcu_cpu_op, 365 __UNCONST(pcu), (void *)(PCU_SAVE | PCU_RELEASE), ci); 366 xc_wait(where); 367 368 /* Enter IPL_SOFTSERIAL and re-fetch the current CPU. */ 369 s = splsoftserial(); 370 curci = curcpu(); 371 } 372 KASSERT(l->l_pcu_cpu[id] == NULL); 373 374 /* Save the PCU state on the current CPU, if there is any. */ 375 pcu_cpu_op(pcu, PCU_SAVE | PCU_RELEASE); 376 KASSERT(curci->ci_pcu_curlwp[id] == NULL); 377 378 /* 379 * Finally, load the state for this LWP on this CPU. Indicate to 380 * load function whether PCU was used before. Note the usage. 381 */ 382 pcu_do_op(pcu, l, PCU_CLAIM | PCU_ENABLE | PCU_RELOAD); 383 splx(s); 384} 385 386/* 387 * pcu_discard: discard the PCU state of current LWP. 388 * If the "usesw" flag is set, pcu_used_p() will return "true". 389 */ 390void 391pcu_discard(const pcu_ops_t *pcu, bool usesw) 392{ 393 const u_int id = pcu->pcu_id; 394 lwp_t * const l = curlwp; 395 396 KASSERT(!cpu_intr_p() && !cpu_softintr_p()); 397 398 if (usesw) 399 l->l_pcu_used[PCU_USER] |= (1 << id); 400 else 401 l->l_pcu_used[PCU_USER] &= ~(1 << id); 402 403 if (__predict_true(l->l_pcu_cpu[id] == NULL)) { 404 return; 405 } 406 pcu_lwp_op(pcu, l, PCU_RELEASE); 407} 408 409/* 410 * pcu_save_lwp: save PCU state to the given LWP. 411 */ 412void 413pcu_save(const pcu_ops_t *pcu) 414{ 415 const u_int id = pcu->pcu_id; 416 lwp_t * const l = curlwp; 417 418 KASSERT(!cpu_intr_p() && !cpu_softintr_p()); 419 420 if (__predict_true(l->l_pcu_cpu[id] == NULL)) { 421 return; 422 } 423 pcu_lwp_op(pcu, l, PCU_SAVE | PCU_RELEASE); 424} 425 426/* 427 * pcu_save_all_on_cpu: save all PCU state on current CPU 428 */ 429void 430pcu_save_all_on_cpu(void) 431{ 432 433 for (u_int id = 0; id < PCU_UNIT_COUNT; id++) { 434 pcu_cpu_op(pcu_ops_md_defs[id], PCU_SAVE | PCU_RELEASE); 435 } 436} 437 438/* 439 * pcu_used: return true if PCU was used (pcu_load() case) by the LWP. 440 */ 441bool 442pcu_used_p(const pcu_ops_t *pcu) 443{ 444 const u_int id = pcu->pcu_id; 445 lwp_t * const l = curlwp; 446 447 return l->l_pcu_used[PCU_USER] & (1 << id); 448} 449 450void 451pcu_kernel_acquire(const pcu_ops_t *pcu) 452{ 453 struct cpu_info * const ci = curcpu(); 454 lwp_t * const l = curlwp; 455 const u_int id = pcu->pcu_id; 456 457 /* 458 * If we own the PCU, save our user state. 459 */ 460 if (ci == l->l_pcu_cpu[id]) { 461 pcu_lwp_op(pcu, l, PCU_KERNEL); 462 return; 463 } 464 if (ci->ci_data.cpu_pcu_curlwp[id] != NULL) { 465 /* 466 * The PCU is owned by another LWP so save its state. 467 */ 468 pcu_cpu_op(pcu, PCU_SAVE | PCU_RELEASE); 469 } 470 /* 471 * Mark the PCU as hijacked and take ownership of it. 472 */ 473 pcu_lwp_op(pcu, l, PCU_KERNEL | PCU_CLAIM | PCU_ENABLE | PCU_RELOAD); 474} 475 476void 477pcu_kernel_release(const pcu_ops_t *pcu) 478{ 479 lwp_t * const l = curlwp; 480 481 KASSERT(l->l_pcu_used[PCU_KERNEL] & (1 << pcu->pcu_id)); 482 483 /* 484 * Release the PCU, if the curlwp wants to use it, it will have incur 485 * a trap to reenable it. 486 */ 487 pcu_lwp_op(pcu, l, PCU_KERNEL | PCU_RELEASE); 488} 489 490#endif /* PCU_UNIT_COUNT > 0 */ 491