| subr_smp.c (222032) | subr_smp.c (222254) |
|---|---|
| 1/*- 2 * Copyright (c) 2001, John Baldwin <jhb@FreeBSD.org>. 3 * All rights reserved. 4 * 5 * Redistribution and use in source and binary forms, with or without 6 * modification, are permitted provided that the following conditions 7 * are met: 8 * 1. Redistributions of source code must retain the above copyright --- 19 unchanged lines hidden (view full) --- 28 */ 29 30/* 31 * This module holds the global variables and machine independent functions 32 * used for the kernel SMP support. 33 */ 34 35#include <sys/cdefs.h> | 1/*- 2 * Copyright (c) 2001, John Baldwin <jhb@FreeBSD.org>. 3 * All rights reserved. 4 * 5 * Redistribution and use in source and binary forms, with or without 6 * modification, are permitted provided that the following conditions 7 * are met: 8 * 1. Redistributions of source code must retain the above copyright --- 19 unchanged lines hidden (view full) --- 28 */ 29 30/* 31 * This module holds the global variables and machine independent functions 32 * used for the kernel SMP support. 33 */ 34 35#include <sys/cdefs.h> |
| 36__FBSDID("$FreeBSD: head/sys/kern/subr_smp.c 222032 2011-05-17 16:39:08Z jhb $"); | 36__FBSDID("$FreeBSD: head/sys/kern/subr_smp.c 222254 2011-05-24 13:36:41Z jhb $"); |
| 37 38#include <sys/param.h> 39#include <sys/systm.h> 40#include <sys/kernel.h> 41#include <sys/ktr.h> 42#include <sys/proc.h> 43#include <sys/bus.h> 44#include <sys/lock.h> --- 261 unchanged lines hidden (view full) --- 306 * resume. 307 * 308 * Note that the supplied external functions _must_ be reentrant and aware 309 * that they are running in parallel and in an unknown lock context. 310 */ 311void 312smp_rendezvous_action(void) 313{ | 37 38#include <sys/param.h> 39#include <sys/systm.h> 40#include <sys/kernel.h> 41#include <sys/ktr.h> 42#include <sys/proc.h> 43#include <sys/bus.h> 44#include <sys/lock.h> --- 261 unchanged lines hidden (view full) --- 306 * resume. 307 * 308 * Note that the supplied external functions _must_ be reentrant and aware 309 * that they are running in parallel and in an unknown lock context. 310 */ 311void 312smp_rendezvous_action(void) 313{ |
| 314 struct thread *td; |
|
| 314 void *local_func_arg; 315 void (*local_setup_func)(void*); 316 void (*local_action_func)(void*); 317 void (*local_teardown_func)(void*); 318 int generation; | 315 void *local_func_arg; 316 void (*local_setup_func)(void*); 317 void (*local_action_func)(void*); 318 void (*local_teardown_func)(void*); 319 int generation; |
| 320#ifdef INVARIANTS 321 int owepreempt; 322#endif |
|
| 319 320 /* Ensure we have up-to-date values. */ 321 atomic_add_acq_int(&smp_rv_waiters[0], 1); 322 while (smp_rv_waiters[0] < smp_rv_ncpus) 323 cpu_spinwait(); 324 325 /* Fetch rendezvous parameters after acquire barrier. */ 326 local_func_arg = smp_rv_func_arg; 327 local_setup_func = smp_rv_setup_func; 328 local_action_func = smp_rv_action_func; 329 local_teardown_func = smp_rv_teardown_func; 330 generation = smp_rv_generation; 331 332 /* | 323 324 /* Ensure we have up-to-date values. */ 325 atomic_add_acq_int(&smp_rv_waiters[0], 1); 326 while (smp_rv_waiters[0] < smp_rv_ncpus) 327 cpu_spinwait(); 328 329 /* Fetch rendezvous parameters after acquire barrier. */ 330 local_func_arg = smp_rv_func_arg; 331 local_setup_func = smp_rv_setup_func; 332 local_action_func = smp_rv_action_func; 333 local_teardown_func = smp_rv_teardown_func; 334 generation = smp_rv_generation; 335 336 /* |
| 337 * Use a nested critical section to prevent any preemptions 338 * from occurring during a rendezvous action routine. 339 * Specifically, if a rendezvous handler is invoked via an IPI 340 * and the interrupted thread was in the critical_exit() 341 * function after setting td_critnest to 0 but before 342 * performing a deferred preemption, this routine can be 343 * invoked with td_critnest set to 0 and td_owepreempt true. 344 * In that case, a critical_exit() during the rendezvous 345 * action would trigger a preemption which is not permitted in 346 * a rendezvous action. To fix this, wrap all of the 347 * rendezvous action handlers in a critical section. We 348 * cannot use a regular critical section however as having 349 * critical_exit() preempt from this routine would also be 350 * problematic (the preemption must not occur before the IPI 351 * has been acknowleged via an EOI). Instead, we 352 * intentionally ignore td_owepreempt when leaving the 353 * critical setion. This should be harmless because we do not 354 * permit rendezvous action routines to schedule threads, and 355 * thus td_owepreempt should never transition from 0 to 1 356 * during this routine. 357 */ 358 td = curthread; 359 td->td_critnest++; 360#ifdef INVARIANTS 361 owepreempt = td->td_owepreempt; 362#endif 363 364 /* |
|
| 333 * If requested, run a setup function before the main action 334 * function. Ensure all CPUs have completed the setup 335 * function before moving on to the action function. 336 */ 337 if (local_setup_func != smp_no_rendevous_barrier) { 338 if (smp_rv_setup_func != NULL) 339 smp_rv_setup_func(smp_rv_func_arg); 340 atomic_add_int(&smp_rv_waiters[1], 1); --- 16 unchanged lines hidden (view full) --- 357 * rendezvous to be initiated on the same CPU or a different 358 * CPU. In that case the exit sentinel may be cleared before 359 * all CPUs have noticed causing those CPUs to hang forever. 360 * Workaround this by using a generation count to notice when 361 * this race occurs and to exit the rendezvous in that case. 362 */ 363 MPASS(generation == smp_rv_generation); 364 atomic_add_int(&smp_rv_waiters[2], 1); | 365 * If requested, run a setup function before the main action 366 * function. Ensure all CPUs have completed the setup 367 * function before moving on to the action function. 368 */ 369 if (local_setup_func != smp_no_rendevous_barrier) { 370 if (smp_rv_setup_func != NULL) 371 smp_rv_setup_func(smp_rv_func_arg); 372 atomic_add_int(&smp_rv_waiters[1], 1); --- 16 unchanged lines hidden (view full) --- 389 * rendezvous to be initiated on the same CPU or a different 390 * CPU. In that case the exit sentinel may be cleared before 391 * all CPUs have noticed causing those CPUs to hang forever. 392 * Workaround this by using a generation count to notice when 393 * this race occurs and to exit the rendezvous in that case. 394 */ 395 MPASS(generation == smp_rv_generation); 396 atomic_add_int(&smp_rv_waiters[2], 1); |
| 365 if (local_teardown_func == smp_no_rendevous_barrier) 366 return; 367 while (smp_rv_waiters[2] < smp_rv_ncpus && 368 generation == smp_rv_generation) 369 cpu_spinwait(); | 397 if (local_teardown_func != smp_no_rendevous_barrier) { 398 while (smp_rv_waiters[2] < smp_rv_ncpus && 399 generation == smp_rv_generation) 400 cpu_spinwait(); |
| 370 | 401 |
| 371 if (local_teardown_func != NULL) 372 local_teardown_func(local_func_arg); | 402 if (local_teardown_func != NULL) 403 local_teardown_func(local_func_arg); 404 } 405 406 td->td_critnest--; 407 KASSERT(owepreempt == td->td_owepreempt, 408 ("rendezvous action changed td_owepreempt")); |
| 373} 374 375void 376smp_rendezvous_cpus(cpumask_t map, 377 void (* setup_func)(void *), 378 void (* action_func)(void *), 379 void (* teardown_func)(void *), 380 void *arg) --- 293 unchanged lines hidden --- | 409} 410 411void 412smp_rendezvous_cpus(cpumask_t map, 413 void (* setup_func)(void *), 414 void (* action_func)(void *), 415 void (* teardown_func)(void *), 416 void *arg) --- 293 unchanged lines hidden --- |