subr_smp.c revision 112888
125164Speter/* 276078Sjhb * Copyright (c) 2001 376078Sjhb * John Baldwin <jhb@FreeBSD.org>. All rights reserved. 425164Speter * 525164Speter * Redistribution and use in source and binary forms, with or without 625164Speter * modification, are permitted provided that the following conditions 725164Speter * are met: 825164Speter * 1. Redistributions of source code must retain the above copyright 925164Speter * notice, this list of conditions and the following disclaimer. 1076078Sjhb * 2. Redistributions in binary form must reproduce the above copyright 1176078Sjhb * notice, this list of conditions and the following disclaimer in the 1276078Sjhb * documentation and/or other materials provided with the distribution. 1376078Sjhb * 4. Neither the name of the author nor the names of any co-contributors 1476078Sjhb * may be used to endorse or promote products derived from this software 1576078Sjhb * without specific prior written permission. 1625164Speter * 1776078Sjhb * THIS SOFTWARE IS PROVIDED BY JOHN BALDWIN AND CONTRIBUTORS ``AS IS'' AND 1825164Speter * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 1925164Speter * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 2076078Sjhb * ARE DISCLAIMED. IN NO EVENT SHALL JOHN BALDWIN OR THE VOICES IN HIS HEAD 2176078Sjhb * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 2276078Sjhb * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 2376078Sjhb * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 2476078Sjhb * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 2576078Sjhb * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 2676078Sjhb * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF 2776078Sjhb * THE POSSIBILITY OF SUCH DAMAGE. 2825164Speter * 2950477Speter * $FreeBSD: head/sys/kern/subr_smp.c 112888 2003-03-31 22:49:17Z jeff $ 3025164Speter */ 3125164Speter 3276078Sjhb/* 3376078Sjhb * This module holds the global variables and machine independent functions 3476440Sjhb * used for the kernel SMP support. 3576078Sjhb */ 3625164Speter 3728743Sbde#include <sys/param.h> 3825164Speter#include <sys/systm.h> 3976440Sjhb#include <sys/kernel.h> 4076078Sjhb#include <sys/ktr.h> 4128808Speter#include <sys/proc.h> 4276078Sjhb#include <sys/lock.h> 4367365Sjhb#include <sys/mutex.h> 4476440Sjhb#include <sys/pcpu.h> 4576078Sjhb#include <sys/smp.h> 4676078Sjhb#include <sys/sysctl.h> 4725164Speter 4891778Sjake#include <machine/smp.h> 4991778Sjake 5076078Sjhbvolatile u_int stopped_cpus; 5176078Sjhbvolatile u_int started_cpus; 5225164Speter 5392723Salfredvoid (*cpustop_restartfunc)(void); 5476078Sjhbint mp_ncpus; 5528027Sfsmp 5685787Smarcelvolatile int smp_started; 5776078Sjhbu_int all_cpus; 5891673Sjeffu_int mp_maxid; 5925164Speter 6076078SjhbSYSCTL_NODE(_kern, OID_AUTO, smp, CTLFLAG_RD, NULL, "Kernel SMP"); 6125164Speter 6276078Sjhbint smp_active = 0; /* are the APs allowed to run? */ 6376078SjhbSYSCTL_INT(_kern_smp, OID_AUTO, active, CTLFLAG_RW, &smp_active, 0, ""); 6426155Sfsmp 65108371Sjakeint smp_disabled = 0; /* has smp been disabled? */ 66108371SjakeSYSCTL_INT(_kern_smp, OID_AUTO, disabled, CTLFLAG_RD, &smp_disabled, 0, ""); 67108371SjakeTUNABLE_INT("kern.smp.disabled", &smp_disabled); 68108371Sjake 6976078Sjhbint smp_cpus = 1; /* how many cpu's running */ 7076078SjhbSYSCTL_INT(_kern_smp, OID_AUTO, cpus, CTLFLAG_RD, &smp_cpus, 0, ""); 7127005Sfsmp 7271525Sjhb/* Enable forwarding of a signal to a process running on a different CPU */ 7371525Sjhbstatic int forward_signal_enabled = 1; 7476078SjhbSYSCTL_INT(_kern_smp, OID_AUTO, forward_signal_enabled, CTLFLAG_RW, 7571525Sjhb &forward_signal_enabled, 0, ""); 7671525Sjhb 7771525Sjhb/* Enable forwarding of roundrobin to all other cpus */ 7871525Sjhbstatic int forward_roundrobin_enabled = 1; 7976078SjhbSYSCTL_INT(_kern_smp, OID_AUTO, forward_roundrobin_enabled, CTLFLAG_RW, 8071525Sjhb &forward_roundrobin_enabled, 0, ""); 8171525Sjhb 8276078Sjhb/* Variables needed for SMP rendezvous. */ 8376078Sjhbstatic void (*smp_rv_setup_func)(void *arg); 8476078Sjhbstatic void (*smp_rv_action_func)(void *arg); 8576078Sjhbstatic void (*smp_rv_teardown_func)(void *arg); 8676078Sjhbstatic void *smp_rv_func_arg; 8776078Sjhbstatic volatile int smp_rv_waiters[2]; 8876078Sjhbstatic struct mtx smp_rv_mtx; 8991673Sjeffstatic int mp_probe_status; 9071525Sjhb 9125164Speter/* 9291673Sjeff * Initialize MI SMP variables. 9325164Speter */ 9471525Sjhbstatic void 9591673Sjeffmp_probe(void *dummy) 9691673Sjeff{ 9791673Sjeff mp_probe_status = cpu_mp_probe(); 9891673Sjeff} 9992799SarrSYSINIT(cpu_mp_probe, SI_SUB_TUNABLES, SI_ORDER_FIRST, mp_probe, NULL) 10091673Sjeff 10191673Sjeff/* 10291673Sjeff * Call the MD SMP initialization code. 10391673Sjeff */ 10491673Sjeffstatic void 10576078Sjhbmp_start(void *dummy) 10671525Sjhb{ 10771525Sjhb 10876078Sjhb /* Probe for MP hardware. */ 109108371Sjake if (mp_probe_status == 0 || smp_disabled != 0) 11076078Sjhb return; 11176078Sjhb 11293818Sjhb mtx_init(&smp_rv_mtx, "smp rendezvous", NULL, MTX_SPIN); 11376078Sjhb cpu_mp_start(); 11476078Sjhb printf("FreeBSD/SMP: Multiprocessor System Detected: %d CPUs\n", 11576078Sjhb mp_ncpus); 11676078Sjhb cpu_mp_announce(); 11771525Sjhb} 11876078SjhbSYSINIT(cpu_mp, SI_SUB_CPU, SI_ORDER_SECOND, mp_start, NULL) 11971525Sjhb 12025164Spetervoid 12183366Sjulianforward_signal(struct thread *td) 12225164Speter{ 12376078Sjhb int id; 12425164Speter 12526108Sfsmp /* 126112888Sjeff * signotify() has already set TDF_ASTPENDING and TDF_NEEDSIGCHECK on 127112888Sjeff * this thread, so all we need to do is poke it if it is currently 12893873Sbde * executing so that it executes ast(). 12926108Sfsmp */ 13071525Sjhb mtx_assert(&sched_lock, MA_OWNED); 131103216Sjulian KASSERT(TD_IS_RUNNING(td), 13299072Sjulian ("forward_signal: thread is not TDS_RUNNING")); 13331639Sfsmp 13483366Sjulian CTR1(KTR_SMP, "forward_signal(%p)", td->td_proc); 13565557Sjasone 13676078Sjhb if (!smp_started || cold || panicstr) 13731639Sfsmp return; 13876078Sjhb if (!forward_signal_enabled) 13976078Sjhb return; 14031639Sfsmp 14176078Sjhb /* No need to IPI ourself. */ 14283366Sjulian if (td == curthread) 14331639Sfsmp return; 14431639Sfsmp 14583366Sjulian id = td->td_kse->ke_oncpu; 14676078Sjhb if (id == NOCPU) 14734020Stegge return; 14876078Sjhb ipi_selected(1 << id, IPI_AST); 14934020Stegge} 15034021Stegge 15136135Steggevoid 15236135Steggeforward_roundrobin(void) 15336135Stegge{ 15487702Sjhb struct pcpu *pc; 15583366Sjulian struct thread *td; 15676078Sjhb u_int id, map; 15734021Stegge 15876078Sjhb mtx_assert(&sched_lock, MA_OWNED); 15976078Sjhb 16071525Sjhb CTR0(KTR_SMP, "forward_roundrobin()"); 16171525Sjhb 16276078Sjhb if (!smp_started || cold || panicstr) 16336135Stegge return; 16436135Stegge if (!forward_roundrobin_enabled) 16536135Stegge return; 16676078Sjhb map = 0; 16787702Sjhb SLIST_FOREACH(pc, &cpuhead, pc_allcpu) { 16887702Sjhb td = pc->pc_curthread; 16988902Speter id = pc->pc_cpumask; 17088902Speter if (id != PCPU_GET(cpumask) && (id & stopped_cpus) == 0 && 17187702Sjhb td != pc->pc_idlethread) { 172111032Sjulian td->td_flags |= TDF_NEEDRESCHED; 17376078Sjhb map |= id; 17436135Stegge } 17536135Stegge } 17676078Sjhb ipi_selected(map, IPI_AST); 17736135Stegge} 17836135Stegge 17971525Sjhb/* 18071525Sjhb * When called the executing CPU will send an IPI to all other CPUs 18171525Sjhb * requesting that they halt execution. 18271525Sjhb * 18371525Sjhb * Usually (but not necessarily) called with 'other_cpus' as its arg. 18471525Sjhb * 18571525Sjhb * - Signals all CPUs in map to stop. 18671525Sjhb * - Waits for each to stop. 18771525Sjhb * 18871525Sjhb * Returns: 18971525Sjhb * -1: error 19071525Sjhb * 0: NA 19171525Sjhb * 1: ok 19271525Sjhb * 19371525Sjhb * XXX FIXME: this is not MP-safe, needs a lock to prevent multiple CPUs 19471525Sjhb * from executing at same time. 19571525Sjhb */ 19671525Sjhbint 19771525Sjhbstop_cpus(u_int map) 19871525Sjhb{ 19976078Sjhb int i; 20036135Stegge 20171525Sjhb if (!smp_started) 20271525Sjhb return 0; 20371525Sjhb 20476078Sjhb CTR1(KTR_SMP, "stop_cpus(%x)", map); 20576078Sjhb 20676078Sjhb /* send the stop IPI to all CPUs in map */ 20775421Sjhb ipi_selected(map, IPI_STOP); 20871525Sjhb 20976078Sjhb i = 0; 21076078Sjhb while ((atomic_load_acq_int(&stopped_cpus) & map) != map) { 21176078Sjhb /* spin */ 21276078Sjhb i++; 21371525Sjhb#ifdef DIAGNOSTIC 21476078Sjhb if (i == 100000) { 21576078Sjhb printf("timeout stopping cpus\n"); 21676078Sjhb break; 21776078Sjhb } 21871525Sjhb#endif 21976078Sjhb } 22071525Sjhb 22171525Sjhb return 1; 22271525Sjhb} 22371525Sjhb 22471525Sjhb 22571525Sjhb/* 22671525Sjhb * Called by a CPU to restart stopped CPUs. 22771525Sjhb * 22871525Sjhb * Usually (but not necessarily) called with 'stopped_cpus' as its arg. 22971525Sjhb * 23071525Sjhb * - Signals all CPUs in map to restart. 23171525Sjhb * - Waits for each to restart. 23271525Sjhb * 23371525Sjhb * Returns: 23471525Sjhb * -1: error 23571525Sjhb * 0: NA 23671525Sjhb * 1: ok 23771525Sjhb */ 23871525Sjhbint 23971525Sjhbrestart_cpus(u_int map) 24071525Sjhb{ 24171525Sjhb 24271525Sjhb if (!smp_started) 24371525Sjhb return 0; 24471525Sjhb 24576078Sjhb CTR1(KTR_SMP, "restart_cpus(%x)", map); 24671525Sjhb 24776078Sjhb /* signal other cpus to restart */ 24876078Sjhb atomic_store_rel_int(&started_cpus, map); 24976078Sjhb 25071525Sjhb /* wait for each to clear its bit */ 25176078Sjhb while ((atomic_load_acq_int(&stopped_cpus) & map) != 0) 25276078Sjhb ; /* nothing */ 25371525Sjhb 25471525Sjhb return 1; 25571525Sjhb} 25671525Sjhb 25734021Stegge/* 25848924Smsmith * All-CPU rendezvous. CPUs are signalled, all execute the setup function 25948924Smsmith * (if specified), rendezvous, execute the action function (if specified), 26048924Smsmith * rendezvous again, execute the teardown function (if specified), and then 26148924Smsmith * resume. 26248924Smsmith * 26348924Smsmith * Note that the supplied external functions _must_ be reentrant and aware 26448924Smsmith * that they are running in parallel and in an unknown lock context. 26548924Smsmith */ 26648924Smsmithvoid 26748924Smsmithsmp_rendezvous_action(void) 26848924Smsmith{ 26976078Sjhb 27048924Smsmith /* setup function */ 27148924Smsmith if (smp_rv_setup_func != NULL) 27248924Smsmith smp_rv_setup_func(smp_rv_func_arg); 27348924Smsmith /* spin on entry rendezvous */ 27448924Smsmith atomic_add_int(&smp_rv_waiters[0], 1); 27576078Sjhb while (atomic_load_acq_int(&smp_rv_waiters[0]) < mp_ncpus) 27676078Sjhb ; /* nothing */ 27748924Smsmith /* action function */ 27848924Smsmith if (smp_rv_action_func != NULL) 27948924Smsmith smp_rv_action_func(smp_rv_func_arg); 28048924Smsmith /* spin on exit rendezvous */ 28148924Smsmith atomic_add_int(&smp_rv_waiters[1], 1); 28276078Sjhb while (atomic_load_acq_int(&smp_rv_waiters[1]) < mp_ncpus) 28376078Sjhb ; /* nothing */ 28448924Smsmith /* teardown function */ 28548924Smsmith if (smp_rv_teardown_func != NULL) 28648924Smsmith smp_rv_teardown_func(smp_rv_func_arg); 28748924Smsmith} 28848924Smsmith 28948924Smsmithvoid 29048924Smsmithsmp_rendezvous(void (* setup_func)(void *), 29148924Smsmith void (* action_func)(void *), 29248924Smsmith void (* teardown_func)(void *), 29348924Smsmith void *arg) 29448924Smsmith{ 29571576Sjasone 29676078Sjhb if (!smp_started) { 29776078Sjhb if (setup_func != NULL) 29876078Sjhb setup_func(arg); 29976078Sjhb if (action_func != NULL) 30076078Sjhb action_func(arg); 30176078Sjhb if (teardown_func != NULL) 30276078Sjhb teardown_func(arg); 30376078Sjhb return; 30476078Sjhb } 30576078Sjhb 30648924Smsmith /* obtain rendezvous lock */ 30772200Sbmilekic mtx_lock_spin(&smp_rv_mtx); 30848924Smsmith 30948924Smsmith /* set static function pointers */ 31048924Smsmith smp_rv_setup_func = setup_func; 31148924Smsmith smp_rv_action_func = action_func; 31248924Smsmith smp_rv_teardown_func = teardown_func; 31348924Smsmith smp_rv_func_arg = arg; 31448924Smsmith smp_rv_waiters[0] = 0; 31548924Smsmith smp_rv_waiters[1] = 0; 31648924Smsmith 31776078Sjhb /* signal other processors, which will enter the IPI with interrupts off */ 31875421Sjhb ipi_all_but_self(IPI_RENDEZVOUS); 31948924Smsmith 32048924Smsmith /* call executor function */ 32148924Smsmith smp_rendezvous_action(); 32248924Smsmith 32348924Smsmith /* release lock */ 32472200Sbmilekic mtx_unlock_spin(&smp_rv_mtx); 32548924Smsmith} 326