subr_pcu.c revision 1.8
1/* $NetBSD: subr_pcu.c,v 1.8 2011/06/07 17:51:58 matt Exp $ */ 2 3/*- 4 * Copyright (c) 2011 The NetBSD Foundation, Inc. 5 * All rights reserved. 6 * 7 * This code is derived from software contributed to The NetBSD Foundation 8 * by Mindaugas Rasiukevicius. 9 * 10 * Redistribution and use in source and binary forms, with or without 11 * modification, are permitted provided that the following conditions 12 * are met: 13 * 1. Redistributions of source code must retain the above copyright 14 * notice, this list of conditions and the following disclaimer. 15 * 2. Redistributions in binary form must reproduce the above copyright 16 * notice, this list of conditions and the following disclaimer in the 17 * documentation and/or other materials provided with the distribution. 18 * 19 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS 20 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 21 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 22 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS 23 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 24 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 25 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 26 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 27 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 28 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 29 * POSSIBILITY OF SUCH DAMAGE. 30 */ 31 32/* 33 * Per CPU Unit (PCU) - is an interface to manage synchronization of any 34 * per CPU context (unit) tied with LWP context. Typical use: FPU state. 35 * 36 * Concurrency notes: 37 * 38 * PCU state may be loaded only by the current LWP, that is, curlwp. 39 * Therefore, only LWP itself can set a CPU for lwp_t::l_pcu_cpu[id]. 40 * 41 * Request for a PCU release can be from owner LWP (whether PCU state 42 * is on current CPU or remote CPU) or any other LWP running on that 43 * CPU (in such case, owner LWP is on a remote CPU or sleeping). 44 * 45 * In any case, PCU state can only be changed from the running CPU. 46 * If said PCU state is on the remote CPU, a cross-call will be sent 47 * by the owner LWP. Therefore struct cpu_info::ci_pcu_curlwp[id] 48 * may only be changed by current CPU, and lwp_t::l_pcu_cpu[id] may 49 * only be unset by the CPU which has PCU state loaded. 50 * 51 * There is a race condition: LWP may have a PCU state on a remote CPU, 52 * which it requests to be released via cross-call. At the same time, 53 * other LWP on remote CPU might release existing PCU state and load 54 * its own one. Cross-call may arrive after this and release different 55 * PCU state than intended. In such case, such LWP would re-load its 56 * PCU state again. 57 */ 58 59#include <sys/cdefs.h> 60__KERNEL_RCSID(0, "$NetBSD: subr_pcu.c,v 1.8 2011/06/07 17:51:58 matt Exp $"); 61 62#include <sys/param.h> 63#include <sys/cpu.h> 64#include <sys/lwp.h> 65#include <sys/pcu.h> 66#include <sys/xcall.h> 67 68#if PCU_UNIT_COUNT > 0 69 70static void pcu_lwp_op(const pcu_ops_t *, lwp_t *, int); 71 72#define PCU_SAVE 0x01 /* Save PCU state to the LWP. */ 73#define PCU_RELEASE 0x02 /* Release PCU state on the CPU. */ 74 75/* XXX */ 76extern const pcu_ops_t * const pcu_ops_md_defs[]; 77 78void 79pcu_switchpoint(lwp_t *l) 80{ 81 const uint32_t pcu_inuse = l->l_pcu_used; 82 u_int id; 83 /* int s; */ 84 85 KASSERT(l == curlwp); 86 87 if (__predict_true(pcu_inuse == 0)) { 88 /* PCUs are not in use. */ 89 return; 90 } 91 /* s = splsoftclock(); */ 92 for (id = 0; id < PCU_UNIT_COUNT; id++) { 93 if ((pcu_inuse & (1 << id)) == 0) { 94 continue; 95 } 96 struct cpu_info * const pcu_ci = l->l_pcu_cpu[id]; 97 if (pcu_ci == NULL || pcu_ci == l->l_cpu) { 98 continue; 99 } 100 const pcu_ops_t * const pcu = pcu_ops_md_defs[id]; 101 pcu->pcu_state_release(l); 102 } 103 /* splx(s); */ 104} 105 106void 107pcu_discard_all(lwp_t *l) 108{ 109 const uint32_t pcu_inuse = l->l_pcu_used; 110 111 KASSERT(l == curlwp || ((l->l_flag & LW_SYSTEM) && pcu_inuse == 0)); 112 113 if (__predict_true(pcu_inuse == 0)) { 114 /* PCUs are not in use. */ 115 return; 116 } 117 const int s = splsoftclock(); 118 for (u_int id = 0; id < PCU_UNIT_COUNT; id++) { 119 if ((pcu_inuse & (1 << id)) == 0) { 120 continue; 121 } 122 if (__predict_true(l->l_pcu_cpu[id] == NULL)) { 123 continue; 124 } 125 const pcu_ops_t * const pcu = pcu_ops_md_defs[id]; 126 /* 127 * We aren't releasing since this LWP isn't giving up PCU, 128 * just saving it. 129 */ 130 pcu_lwp_op(pcu, l, PCU_RELEASE); 131 } 132 l->l_pcu_used = 0; 133 splx(s); 134} 135 136void 137pcu_save_all(lwp_t *l) 138{ 139 const uint32_t pcu_inuse = l->l_pcu_used; 140 141 KASSERT(l == curlwp || ((l->l_flag & LW_SYSTEM) && pcu_inuse == 0)); 142 143 if (__predict_true(pcu_inuse == 0)) { 144 /* PCUs are not in use. */ 145 return; 146 } 147 const int s = splsoftclock(); 148 for (u_int id = 0; id < PCU_UNIT_COUNT; id++) { 149 if ((pcu_inuse & (1 << id)) == 0) { 150 continue; 151 } 152 if (__predict_true(l->l_pcu_cpu[id] == NULL)) { 153 continue; 154 } 155 const pcu_ops_t * const pcu = pcu_ops_md_defs[id]; 156 /* 157 * We aren't releasing since this LWP isn't giving up PCU, 158 * just saving it. 159 */ 160 pcu_lwp_op(pcu, l, PCU_SAVE); 161 } 162 splx(s); 163} 164 165/* 166 * pcu_do_op: save/release PCU state on the current CPU. 167 * 168 * => Must be called at IPL_SOFTCLOCK or from the soft-interrupt. 169 */ 170static inline void 171pcu_do_op(const pcu_ops_t *pcu, lwp_t * const l, const int flags) 172{ 173 struct cpu_info * const ci = curcpu(); 174 const u_int id = pcu->pcu_id; 175 176 KASSERT(l->l_pcu_cpu[id] == ci); 177 178 if (flags & PCU_SAVE) { 179 pcu->pcu_state_save(l); 180 } 181 if (flags & PCU_RELEASE) { 182 pcu->pcu_state_release(l); 183 ci->ci_pcu_curlwp[id] = NULL; 184 l->l_pcu_cpu[id] = NULL; 185 } 186} 187 188/* 189 * pcu_cpu_op: helper routine to call pcu_do_op() via xcall(9) or 190 * by pcu_load. 191 */ 192static void 193pcu_cpu_op(const pcu_ops_t *pcu, const int flags) 194{ 195 const u_int id = pcu->pcu_id; 196 lwp_t * const l = curcpu()->ci_pcu_curlwp[id]; 197 198 //KASSERT(cpu_softintr_p()); 199 200 /* If no state - nothing to do. */ 201 if (l == NULL) { 202 return; 203 } 204 pcu_do_op(pcu, l, flags); 205} 206 207/* 208 * pcu_lwp_op: perform PCU state save, release or both operations on LWP. 209 */ 210static void 211pcu_lwp_op(const pcu_ops_t *pcu, lwp_t *l, int flags) 212{ 213 const u_int id = pcu->pcu_id; 214 struct cpu_info *ci; 215 uint64_t where; 216 int s; 217 218 /* 219 * Caller should have re-checked if there is any state to manage. 220 * Block the interrupts and inspect again, since cross-call sent 221 * by remote CPU could have changed the state. 222 */ 223 s = splsoftclock(); 224 ci = l->l_pcu_cpu[id]; 225 if (ci == curcpu()) { 226 /* 227 * State is on the current CPU - just perform the operations. 228 */ 229 KASSERTMSG(ci->ci_pcu_curlwp[id] == l, 230 ("%s: cpu%u: pcu_curlwp[%u] (%p) != l (%p)", 231 __func__, cpu_index(ci), id, ci->ci_pcu_curlwp[id], l)); 232 pcu_do_op(pcu, l, flags); 233 splx(s); 234 return; 235 } 236 splx(s); 237 238 if (__predict_false(ci == NULL)) { 239 /* Cross-call has won the race - no state to manage. */ 240 return; 241 } 242 243 /* 244 * State is on the remote CPU - perform the operations there. 245 * Note: there is a race condition; see description in the top. 246 */ 247 where = xc_unicast(XC_HIGHPRI, (xcfunc_t)pcu_cpu_op, 248 __UNCONST(pcu), (void *)(uintptr_t)flags, ci); 249 xc_wait(where); 250 251 KASSERT((flags & PCU_RELEASE) == 0 || l->l_pcu_cpu[id] == NULL); 252} 253 254/* 255 * pcu_load: load/initialize the PCU state of current LWP on current CPU. 256 */ 257void 258pcu_load(const pcu_ops_t *pcu) 259{ 260 const u_int id = pcu->pcu_id; 261 struct cpu_info *ci, *curci; 262 lwp_t * const l = curlwp; 263 uint64_t where; 264 int s; 265 266 KASSERT(!cpu_intr_p() && !cpu_softintr_p()); 267 268 s = splsoftclock(); 269 curci = curcpu(); 270 ci = l->l_pcu_cpu[id]; 271 272 /* Does this CPU already have our PCU state loaded? */ 273 if (ci == curci) { 274 KASSERT(curci->ci_pcu_curlwp[id] == l); 275 splx(s); 276 return; 277 } 278 279 /* If PCU state of this LWP is on the remote CPU - save it there. */ 280 if (ci) { 281 splx(s); 282 /* Note: there is a race; see description in the top. */ 283 where = xc_unicast(XC_HIGHPRI, (xcfunc_t)pcu_cpu_op, 284 __UNCONST(pcu), (void *)(PCU_SAVE | PCU_RELEASE), ci); 285 xc_wait(where); 286 287 /* Enter IPL_SOFTCLOCK and re-fetch the current CPU. */ 288 s = splsoftclock(); 289 curci = curcpu(); 290 } 291 KASSERT(l->l_pcu_cpu[id] == NULL); 292 293 /* Save the PCU state on the current CPU, if there is any. */ 294 pcu_cpu_op(pcu, PCU_SAVE | PCU_RELEASE); 295 KASSERT(curci->ci_pcu_curlwp[id] == NULL); 296 297 /* 298 * Finally, load the state for this LWP on this CPU. Indicate to 299 * load function whether PCU was used before. Note the usage. 300 */ 301 pcu->pcu_state_load(l, ((1 << id) & l->l_pcu_used) != 0); 302 curci->ci_pcu_curlwp[id] = l; 303 l->l_pcu_cpu[id] = curci; 304 l->l_pcu_used |= (1 << id); 305 splx(s); 306} 307 308/* 309 * pcu_discard: discard the PCU state of current LWP. 310 */ 311void 312pcu_discard(const pcu_ops_t *pcu) 313{ 314 const u_int id = pcu->pcu_id; 315 lwp_t * const l = curlwp; 316 317 KASSERT(!cpu_intr_p() && !cpu_softintr_p()); 318 319 if (__predict_true(l->l_pcu_cpu[id] == NULL)) { 320 return; 321 } 322 pcu_lwp_op(pcu, l, PCU_RELEASE); 323 l->l_pcu_used &= ~(1 << id); 324} 325 326/* 327 * pcu_save_lwp: save PCU state to the given LWP. 328 */ 329void 330pcu_save(const pcu_ops_t *pcu) 331{ 332 const u_int id = pcu->pcu_id; 333 lwp_t * const l = curlwp; 334 335 KASSERT(!cpu_intr_p() && !cpu_softintr_p()); 336 337 if (__predict_true(l->l_pcu_cpu[id] == NULL)) { 338 return; 339 } 340 pcu_lwp_op(pcu, l, PCU_SAVE | PCU_RELEASE); 341} 342 343/* 344 * pcu_used: return true if PCU was used (pcu_load() case) by the LWP. 345 */ 346bool 347pcu_used_p(const pcu_ops_t *pcu) 348{ 349 const u_int id = pcu->pcu_id; 350 lwp_t * const l = curlwp; 351 352 return l->l_pcu_used & (1 << id); 353} 354 355#endif /* PCU_UNIT_COUNT > 0 */ 356