subr_smp.c revision 92723
125164Speter/* 276078Sjhb * Copyright (c) 2001 376078Sjhb * John Baldwin <jhb@FreeBSD.org>. All rights reserved. 425164Speter * 525164Speter * Redistribution and use in source and binary forms, with or without 625164Speter * modification, are permitted provided that the following conditions 725164Speter * are met: 825164Speter * 1. Redistributions of source code must retain the above copyright 925164Speter * notice, this list of conditions and the following disclaimer. 1076078Sjhb * 2. Redistributions in binary form must reproduce the above copyright 1176078Sjhb * notice, this list of conditions and the following disclaimer in the 1276078Sjhb * documentation and/or other materials provided with the distribution. 1376078Sjhb * 4. Neither the name of the author nor the names of any co-contributors 1476078Sjhb * may be used to endorse or promote products derived from this software 1576078Sjhb * without specific prior written permission. 1625164Speter * 1776078Sjhb * THIS SOFTWARE IS PROVIDED BY JOHN BALDWIN AND CONTRIBUTORS ``AS IS'' AND 1825164Speter * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 1925164Speter * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 2076078Sjhb * ARE DISCLAIMED. IN NO EVENT SHALL JOHN BALDWIN OR THE VOICES IN HIS HEAD 2176078Sjhb * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 2276078Sjhb * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 2376078Sjhb * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 2476078Sjhb * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 2576078Sjhb * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 2676078Sjhb * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF 2776078Sjhb * THE POSSIBILITY OF SUCH DAMAGE. 2825164Speter * 2950477Speter * $FreeBSD: head/sys/kern/subr_smp.c 92723 2002-03-19 21:25:46Z alfred $ 3025164Speter */ 3125164Speter 3276078Sjhb/* 3376078Sjhb * This module holds the global variables and machine independent functions 3476440Sjhb * used for the kernel SMP support. 3576078Sjhb */ 3625164Speter 3728743Sbde#include <sys/param.h> 3825164Speter#include <sys/systm.h> 3976440Sjhb#include <sys/kernel.h> 4076078Sjhb#include <sys/ktr.h> 4128808Speter#include <sys/proc.h> 4276078Sjhb#include <sys/lock.h> 4367365Sjhb#include <sys/mutex.h> 4476440Sjhb#include <sys/pcpu.h> 4576078Sjhb#include <sys/smp.h> 4676078Sjhb#include <sys/sysctl.h> 4725164Speter 4891778Sjake#include <machine/smp.h> 4991778Sjake 5076078Sjhbvolatile u_int stopped_cpus; 5176078Sjhbvolatile u_int started_cpus; 5225164Speter 5392723Salfredvoid (*cpustop_restartfunc)(void); 5476078Sjhbint mp_ncpus; 5528027Sfsmp 5685787Smarcelvolatile int smp_started; 5776078Sjhbu_int all_cpus; 5891673Sjeffu_int mp_maxid; 5925164Speter 6076078SjhbSYSCTL_NODE(_kern, OID_AUTO, smp, CTLFLAG_RD, NULL, "Kernel SMP"); 6125164Speter 6276078Sjhbint smp_active = 0; /* are the APs allowed to run? */ 6376078SjhbSYSCTL_INT(_kern_smp, OID_AUTO, active, CTLFLAG_RW, &smp_active, 0, ""); 6426155Sfsmp 6576078Sjhbint smp_cpus = 1; /* how many cpu's running */ 6676078SjhbSYSCTL_INT(_kern_smp, OID_AUTO, cpus, CTLFLAG_RD, &smp_cpus, 0, ""); 6727005Sfsmp 6871525Sjhb/* Enable forwarding of a signal to a process running on a different CPU */ 6971525Sjhbstatic int forward_signal_enabled = 1; 7076078SjhbSYSCTL_INT(_kern_smp, OID_AUTO, forward_signal_enabled, CTLFLAG_RW, 7171525Sjhb &forward_signal_enabled, 0, ""); 7271525Sjhb 7371525Sjhb/* Enable forwarding of roundrobin to all other cpus */ 7471525Sjhbstatic int forward_roundrobin_enabled = 1; 7576078SjhbSYSCTL_INT(_kern_smp, OID_AUTO, forward_roundrobin_enabled, CTLFLAG_RW, 7671525Sjhb &forward_roundrobin_enabled, 0, ""); 7771525Sjhb 7876078Sjhb/* Variables needed for SMP rendezvous. */ 7976078Sjhbstatic void (*smp_rv_setup_func)(void *arg); 8076078Sjhbstatic void (*smp_rv_action_func)(void *arg); 8176078Sjhbstatic void (*smp_rv_teardown_func)(void *arg); 8276078Sjhbstatic void *smp_rv_func_arg; 8376078Sjhbstatic volatile int smp_rv_waiters[2]; 8476078Sjhbstatic struct mtx smp_rv_mtx; 8591673Sjeffstatic int mp_probe_status; 8671525Sjhb 8725164Speter/* 8891673Sjeff * Initialize MI SMP variables. 8925164Speter */ 9071525Sjhbstatic void 9191673Sjeffmp_probe(void *dummy) 9291673Sjeff{ 9391673Sjeff mp_probe_status = cpu_mp_probe(); 9491673Sjeff} 9591673SjeffSYSINIT(cpu_mp_probe, SI_SUB_TUNABLES, SI_ORDER_FIRST, mp_probe, NULL); 9691673Sjeff 9791673Sjeff/* 9891673Sjeff * Call the MD SMP initialization code. 9991673Sjeff */ 10091673Sjeffstatic void 10176078Sjhbmp_start(void *dummy) 10271525Sjhb{ 10371525Sjhb 10476078Sjhb /* Probe for MP hardware. */ 10591673Sjeff if (mp_probe_status == 0) 10676078Sjhb return; 10776078Sjhb 10871576Sjasone mtx_init(&smp_rv_mtx, "smp rendezvous", MTX_SPIN); 10976078Sjhb cpu_mp_start(); 11076078Sjhb printf("FreeBSD/SMP: Multiprocessor System Detected: %d CPUs\n", 11176078Sjhb mp_ncpus); 11276078Sjhb cpu_mp_announce(); 11371525Sjhb} 11476078SjhbSYSINIT(cpu_mp, SI_SUB_CPU, SI_ORDER_SECOND, mp_start, NULL) 11571525Sjhb 11625164Spetervoid 11783366Sjulianforward_signal(struct thread *td) 11825164Speter{ 11976078Sjhb int id; 12025164Speter 12126108Sfsmp /* 12276078Sjhb * signotify() has already set PS_ASTPENDING on this process so all 12376078Sjhb * we need to do is poke it if it is currently executing so that it 12476078Sjhb * executes ast(). 12526108Sfsmp */ 12671525Sjhb mtx_assert(&sched_lock, MA_OWNED); 12783366Sjulian KASSERT(td->td_proc->p_stat == SRUN, ("forward_signal: process is not SRUN")); 12831639Sfsmp 12983366Sjulian CTR1(KTR_SMP, "forward_signal(%p)", td->td_proc); 13065557Sjasone 13176078Sjhb if (!smp_started || cold || panicstr) 13231639Sfsmp return; 13376078Sjhb if (!forward_signal_enabled) 13476078Sjhb return; 13531639Sfsmp 13676078Sjhb /* No need to IPI ourself. */ 13783366Sjulian if (td == curthread) 13831639Sfsmp return; 13931639Sfsmp 14083366Sjulian id = td->td_kse->ke_oncpu; 14176078Sjhb if (id == NOCPU) 14234020Stegge return; 14376078Sjhb ipi_selected(1 << id, IPI_AST); 14434020Stegge} 14534021Stegge 14636135Steggevoid 14736135Steggeforward_roundrobin(void) 14836135Stegge{ 14987702Sjhb struct pcpu *pc; 15083366Sjulian struct thread *td; 15176078Sjhb u_int id, map; 15234021Stegge 15376078Sjhb mtx_assert(&sched_lock, MA_OWNED); 15476078Sjhb 15571525Sjhb CTR0(KTR_SMP, "forward_roundrobin()"); 15671525Sjhb 15776078Sjhb if (!smp_started || cold || panicstr) 15836135Stegge return; 15936135Stegge if (!forward_roundrobin_enabled) 16036135Stegge return; 16176078Sjhb map = 0; 16287702Sjhb SLIST_FOREACH(pc, &cpuhead, pc_allcpu) { 16387702Sjhb td = pc->pc_curthread; 16488902Speter id = pc->pc_cpumask; 16588902Speter if (id != PCPU_GET(cpumask) && (id & stopped_cpus) == 0 && 16687702Sjhb td != pc->pc_idlethread) { 16783366Sjulian td->td_kse->ke_flags |= KEF_NEEDRESCHED; 16876078Sjhb map |= id; 16936135Stegge } 17036135Stegge } 17176078Sjhb ipi_selected(map, IPI_AST); 17236135Stegge} 17336135Stegge 17471525Sjhb/* 17571525Sjhb * When called the executing CPU will send an IPI to all other CPUs 17671525Sjhb * requesting that they halt execution. 17771525Sjhb * 17871525Sjhb * Usually (but not necessarily) called with 'other_cpus' as its arg. 17971525Sjhb * 18071525Sjhb * - Signals all CPUs in map to stop. 18171525Sjhb * - Waits for each to stop. 18271525Sjhb * 18371525Sjhb * Returns: 18471525Sjhb * -1: error 18571525Sjhb * 0: NA 18671525Sjhb * 1: ok 18771525Sjhb * 18871525Sjhb * XXX FIXME: this is not MP-safe, needs a lock to prevent multiple CPUs 18971525Sjhb * from executing at same time. 19071525Sjhb */ 19171525Sjhbint 19271525Sjhbstop_cpus(u_int map) 19371525Sjhb{ 19476078Sjhb int i; 19536135Stegge 19671525Sjhb if (!smp_started) 19771525Sjhb return 0; 19871525Sjhb 19976078Sjhb CTR1(KTR_SMP, "stop_cpus(%x)", map); 20076078Sjhb 20176078Sjhb /* send the stop IPI to all CPUs in map */ 20275421Sjhb ipi_selected(map, IPI_STOP); 20371525Sjhb 20476078Sjhb i = 0; 20576078Sjhb while ((atomic_load_acq_int(&stopped_cpus) & map) != map) { 20676078Sjhb /* spin */ 20776078Sjhb i++; 20871525Sjhb#ifdef DIAGNOSTIC 20976078Sjhb if (i == 100000) { 21076078Sjhb printf("timeout stopping cpus\n"); 21176078Sjhb break; 21276078Sjhb } 21371525Sjhb#endif 21476078Sjhb } 21571525Sjhb 21671525Sjhb return 1; 21771525Sjhb} 21871525Sjhb 21971525Sjhb 22071525Sjhb/* 22171525Sjhb * Called by a CPU to restart stopped CPUs. 22271525Sjhb * 22371525Sjhb * Usually (but not necessarily) called with 'stopped_cpus' as its arg. 22471525Sjhb * 22571525Sjhb * - Signals all CPUs in map to restart. 22671525Sjhb * - Waits for each to restart. 22771525Sjhb * 22871525Sjhb * Returns: 22971525Sjhb * -1: error 23071525Sjhb * 0: NA 23171525Sjhb * 1: ok 23271525Sjhb */ 23371525Sjhbint 23471525Sjhbrestart_cpus(u_int map) 23571525Sjhb{ 23671525Sjhb 23771525Sjhb if (!smp_started) 23871525Sjhb return 0; 23971525Sjhb 24076078Sjhb CTR1(KTR_SMP, "restart_cpus(%x)", map); 24171525Sjhb 24276078Sjhb /* signal other cpus to restart */ 24376078Sjhb atomic_store_rel_int(&started_cpus, map); 24476078Sjhb 24571525Sjhb /* wait for each to clear its bit */ 24676078Sjhb while ((atomic_load_acq_int(&stopped_cpus) & map) != 0) 24776078Sjhb ; /* nothing */ 24871525Sjhb 24971525Sjhb return 1; 25071525Sjhb} 25171525Sjhb 25234021Stegge/* 25348924Smsmith * All-CPU rendezvous. CPUs are signalled, all execute the setup function 25448924Smsmith * (if specified), rendezvous, execute the action function (if specified), 25548924Smsmith * rendezvous again, execute the teardown function (if specified), and then 25648924Smsmith * resume. 25748924Smsmith * 25848924Smsmith * Note that the supplied external functions _must_ be reentrant and aware 25948924Smsmith * that they are running in parallel and in an unknown lock context. 26048924Smsmith */ 26148924Smsmithvoid 26248924Smsmithsmp_rendezvous_action(void) 26348924Smsmith{ 26476078Sjhb 26548924Smsmith /* setup function */ 26648924Smsmith if (smp_rv_setup_func != NULL) 26748924Smsmith smp_rv_setup_func(smp_rv_func_arg); 26848924Smsmith /* spin on entry rendezvous */ 26948924Smsmith atomic_add_int(&smp_rv_waiters[0], 1); 27076078Sjhb while (atomic_load_acq_int(&smp_rv_waiters[0]) < mp_ncpus) 27176078Sjhb ; /* nothing */ 27248924Smsmith /* action function */ 27348924Smsmith if (smp_rv_action_func != NULL) 27448924Smsmith smp_rv_action_func(smp_rv_func_arg); 27548924Smsmith /* spin on exit rendezvous */ 27648924Smsmith atomic_add_int(&smp_rv_waiters[1], 1); 27776078Sjhb while (atomic_load_acq_int(&smp_rv_waiters[1]) < mp_ncpus) 27876078Sjhb ; /* nothing */ 27948924Smsmith /* teardown function */ 28048924Smsmith if (smp_rv_teardown_func != NULL) 28148924Smsmith smp_rv_teardown_func(smp_rv_func_arg); 28248924Smsmith} 28348924Smsmith 28448924Smsmithvoid 28548924Smsmithsmp_rendezvous(void (* setup_func)(void *), 28648924Smsmith void (* action_func)(void *), 28748924Smsmith void (* teardown_func)(void *), 28848924Smsmith void *arg) 28948924Smsmith{ 29071576Sjasone 29176078Sjhb if (!smp_started) { 29276078Sjhb if (setup_func != NULL) 29376078Sjhb setup_func(arg); 29476078Sjhb if (action_func != NULL) 29576078Sjhb action_func(arg); 29676078Sjhb if (teardown_func != NULL) 29776078Sjhb teardown_func(arg); 29876078Sjhb return; 29976078Sjhb } 30076078Sjhb 30148924Smsmith /* obtain rendezvous lock */ 30272200Sbmilekic mtx_lock_spin(&smp_rv_mtx); 30348924Smsmith 30448924Smsmith /* set static function pointers */ 30548924Smsmith smp_rv_setup_func = setup_func; 30648924Smsmith smp_rv_action_func = action_func; 30748924Smsmith smp_rv_teardown_func = teardown_func; 30848924Smsmith smp_rv_func_arg = arg; 30948924Smsmith smp_rv_waiters[0] = 0; 31048924Smsmith smp_rv_waiters[1] = 0; 31148924Smsmith 31276078Sjhb /* signal other processors, which will enter the IPI with interrupts off */ 31375421Sjhb ipi_all_but_self(IPI_RENDEZVOUS); 31448924Smsmith 31548924Smsmith /* call executor function */ 31648924Smsmith smp_rendezvous_action(); 31748924Smsmith 31848924Smsmith /* release lock */ 31972200Sbmilekic mtx_unlock_spin(&smp_rv_mtx); 32048924Smsmith} 321