subr_smp.c revision 134227
125164Speter/* 276078Sjhb * Copyright (c) 2001 376078Sjhb * John Baldwin <jhb@FreeBSD.org>. All rights reserved. 425164Speter * 525164Speter * Redistribution and use in source and binary forms, with or without 625164Speter * modification, are permitted provided that the following conditions 725164Speter * are met: 825164Speter * 1. Redistributions of source code must retain the above copyright 925164Speter * notice, this list of conditions and the following disclaimer. 1076078Sjhb * 2. Redistributions in binary form must reproduce the above copyright 1176078Sjhb * notice, this list of conditions and the following disclaimer in the 1276078Sjhb * documentation and/or other materials provided with the distribution. 1376078Sjhb * 4. Neither the name of the author nor the names of any co-contributors 1476078Sjhb * may be used to endorse or promote products derived from this software 1576078Sjhb * without specific prior written permission. 1625164Speter * 1776078Sjhb * THIS SOFTWARE IS PROVIDED BY JOHN BALDWIN AND CONTRIBUTORS ``AS IS'' AND 1825164Speter * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 1925164Speter * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 2076078Sjhb * ARE DISCLAIMED. IN NO EVENT SHALL JOHN BALDWIN OR THE VOICES IN HIS HEAD 2176078Sjhb * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 2276078Sjhb * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 2376078Sjhb * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 2476078Sjhb * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 2576078Sjhb * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 2676078Sjhb * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF 2776078Sjhb * THE POSSIBILITY OF SUCH DAMAGE. 2825164Speter */ 2925164Speter 3076078Sjhb/* 3176078Sjhb * This module holds the global variables and machine independent functions 3276440Sjhb * used for the kernel SMP support. 3376078Sjhb */ 3425164Speter 35116182Sobrien#include <sys/cdefs.h> 36116182Sobrien__FBSDID("$FreeBSD: head/sys/kern/subr_smp.c 134227 2004-08-23 21:39:29Z peter $"); 37116182Sobrien 3828743Sbde#include <sys/param.h> 3925164Speter#include <sys/systm.h> 4076440Sjhb#include <sys/kernel.h> 4176078Sjhb#include <sys/ktr.h> 4228808Speter#include <sys/proc.h> 43126763Snjl#include <sys/bus.h> 4476078Sjhb#include <sys/lock.h> 4567365Sjhb#include <sys/mutex.h> 4676440Sjhb#include <sys/pcpu.h> 4776078Sjhb#include <sys/smp.h> 4876078Sjhb#include <sys/sysctl.h> 4925164Speter 5091778Sjake#include <machine/smp.h> 5191778Sjake 52123125Sjhb#ifdef SMP 53127498Smarcelvolatile cpumask_t stopped_cpus; 54127498Smarcelvolatile cpumask_t started_cpus; 5525164Speter 5692723Salfredvoid (*cpustop_restartfunc)(void); 57123125Sjhb#endif 58123125Sjhb 5976078Sjhbint mp_ncpus; 60123766Salfred/* export this for libkvm consumers. */ 61123766Salfredint mp_maxcpus = MAXCPU; 6228027Sfsmp 63124925Sjeffstruct cpu_top *smp_topology; 6485787Smarcelvolatile int smp_started; 65127498Smarcelcpumask_t all_cpus; 6691673Sjeffu_int mp_maxid; 6725164Speter 6876078SjhbSYSCTL_NODE(_kern, OID_AUTO, smp, CTLFLAG_RD, NULL, "Kernel SMP"); 6925164Speter 70123766SalfredSYSCTL_INT(_kern_smp, OID_AUTO, maxcpus, CTLFLAG_RD, &mp_maxcpus, 0, 71123766Salfred "Max number of CPUs that the system was compiled for."); 72123766Salfred 7376078Sjhbint smp_active = 0; /* are the APs allowed to run? */ 74116270SdesSYSCTL_INT(_kern_smp, OID_AUTO, active, CTLFLAG_RW, &smp_active, 0, 75116270Sdes "Number of Auxillary Processors (APs) that were successfully started"); 7626155Sfsmp 77108371Sjakeint smp_disabled = 0; /* has smp been disabled? */ 78121307SsilbySYSCTL_INT(_kern_smp, OID_AUTO, disabled, CTLFLAG_RDTUN, &smp_disabled, 0, 79116270Sdes "SMP has been disabled from the loader"); 80108371SjakeTUNABLE_INT("kern.smp.disabled", &smp_disabled); 81108371Sjake 8276078Sjhbint smp_cpus = 1; /* how many cpu's running */ 83116270SdesSYSCTL_INT(_kern_smp, OID_AUTO, cpus, CTLFLAG_RD, &smp_cpus, 0, 84116270Sdes "Number of CPUs online"); 8527005Sfsmp 86123125Sjhb#ifdef SMP 8771525Sjhb/* Enable forwarding of a signal to a process running on a different CPU */ 8871525Sjhbstatic int forward_signal_enabled = 1; 8976078SjhbSYSCTL_INT(_kern_smp, OID_AUTO, forward_signal_enabled, CTLFLAG_RW, 90116270Sdes &forward_signal_enabled, 0, 91116270Sdes "Forwarding of a signal to a process on a different CPU"); 9271525Sjhb 9371525Sjhb/* Enable forwarding of roundrobin to all other cpus */ 9471525Sjhbstatic int forward_roundrobin_enabled = 1; 9576078SjhbSYSCTL_INT(_kern_smp, OID_AUTO, forward_roundrobin_enabled, CTLFLAG_RW, 96116270Sdes &forward_roundrobin_enabled, 0, 97116270Sdes "Forwarding of roundrobin to all other CPUs"); 9871525Sjhb 9976078Sjhb/* Variables needed for SMP rendezvous. */ 10076078Sjhbstatic void (*smp_rv_setup_func)(void *arg); 10176078Sjhbstatic void (*smp_rv_action_func)(void *arg); 10276078Sjhbstatic void (*smp_rv_teardown_func)(void *arg); 10376078Sjhbstatic void *smp_rv_func_arg; 10476078Sjhbstatic volatile int smp_rv_waiters[2]; 10571525Sjhb 106134227Speter/* 107134227Speter * Shared mutex to restrict busywaits between smp_rendezvous() and 108134227Speter * smp(_targeted)_tlb_shootdown(). A deadlock occurs if both of these 109134227Speter * functions trigger at once and cause multiple CPUs to busywait with 110134227Speter * interrupts disabled. 111134227Speter */ 112134227Speterstruct mtx smp_rv_mtx; 113134227Speter 11425164Speter/* 115122947Sjhb * Let the MD SMP code initialize mp_maxid very early if it can. 11625164Speter */ 11771525Sjhbstatic void 118122947Sjhbmp_setmaxid(void *dummy) 11991673Sjeff{ 120122947Sjhb cpu_mp_setmaxid(); 12191673Sjeff} 122122947SjhbSYSINIT(cpu_mp_setmaxid, SI_SUB_TUNABLES, SI_ORDER_FIRST, mp_setmaxid, NULL) 12391673Sjeff 12491673Sjeff/* 12591673Sjeff * Call the MD SMP initialization code. 12691673Sjeff */ 12791673Sjeffstatic void 12876078Sjhbmp_start(void *dummy) 12971525Sjhb{ 13071525Sjhb 13176078Sjhb /* Probe for MP hardware. */ 132122947Sjhb if (smp_disabled != 0 || cpu_mp_probe() == 0) { 133121756Sjhb mp_ncpus = 1; 134122947Sjhb all_cpus = PCPU_GET(cpumask); 13576078Sjhb return; 136121756Sjhb } 13776078Sjhb 13893818Sjhb mtx_init(&smp_rv_mtx, "smp rendezvous", NULL, MTX_SPIN); 13976078Sjhb cpu_mp_start(); 14076078Sjhb printf("FreeBSD/SMP: Multiprocessor System Detected: %d CPUs\n", 14176078Sjhb mp_ncpus); 14276078Sjhb cpu_mp_announce(); 14371525Sjhb} 14476078SjhbSYSINIT(cpu_mp, SI_SUB_CPU, SI_ORDER_SECOND, mp_start, NULL) 14571525Sjhb 14625164Spetervoid 14783366Sjulianforward_signal(struct thread *td) 14825164Speter{ 14976078Sjhb int id; 15025164Speter 15126108Sfsmp /* 152112888Sjeff * signotify() has already set TDF_ASTPENDING and TDF_NEEDSIGCHECK on 153112888Sjeff * this thread, so all we need to do is poke it if it is currently 15493873Sbde * executing so that it executes ast(). 15526108Sfsmp */ 15671525Sjhb mtx_assert(&sched_lock, MA_OWNED); 157103216Sjulian KASSERT(TD_IS_RUNNING(td), 15899072Sjulian ("forward_signal: thread is not TDS_RUNNING")); 15931639Sfsmp 16083366Sjulian CTR1(KTR_SMP, "forward_signal(%p)", td->td_proc); 16165557Sjasone 16276078Sjhb if (!smp_started || cold || panicstr) 16331639Sfsmp return; 16476078Sjhb if (!forward_signal_enabled) 16576078Sjhb return; 16631639Sfsmp 16776078Sjhb /* No need to IPI ourself. */ 16883366Sjulian if (td == curthread) 16931639Sfsmp return; 17031639Sfsmp 171113339Sjulian id = td->td_oncpu; 17276078Sjhb if (id == NOCPU) 17334020Stegge return; 17476078Sjhb ipi_selected(1 << id, IPI_AST); 17534020Stegge} 17634021Stegge 17736135Steggevoid 17836135Steggeforward_roundrobin(void) 17936135Stegge{ 18087702Sjhb struct pcpu *pc; 18183366Sjulian struct thread *td; 182133605Sjulian cpumask_t id, map, me; 18334021Stegge 18476078Sjhb mtx_assert(&sched_lock, MA_OWNED); 18576078Sjhb 18671525Sjhb CTR0(KTR_SMP, "forward_roundrobin()"); 18771525Sjhb 18876078Sjhb if (!smp_started || cold || panicstr) 18936135Stegge return; 19036135Stegge if (!forward_roundrobin_enabled) 19136135Stegge return; 19276078Sjhb map = 0; 193133605Sjulian me = PCPU_GET(cpumask); 19487702Sjhb SLIST_FOREACH(pc, &cpuhead, pc_allcpu) { 19587702Sjhb td = pc->pc_curthread; 19688902Speter id = pc->pc_cpumask; 197133605Sjulian if (id != me && (id & stopped_cpus) == 0 && 19887702Sjhb td != pc->pc_idlethread) { 199111032Sjulian td->td_flags |= TDF_NEEDRESCHED; 20076078Sjhb map |= id; 20136135Stegge } 20236135Stegge } 20376078Sjhb ipi_selected(map, IPI_AST); 20436135Stegge} 20536135Stegge 20671525Sjhb/* 20771525Sjhb * When called the executing CPU will send an IPI to all other CPUs 20871525Sjhb * requesting that they halt execution. 20971525Sjhb * 21071525Sjhb * Usually (but not necessarily) called with 'other_cpus' as its arg. 21171525Sjhb * 21271525Sjhb * - Signals all CPUs in map to stop. 21371525Sjhb * - Waits for each to stop. 21471525Sjhb * 21571525Sjhb * Returns: 21671525Sjhb * -1: error 21771525Sjhb * 0: NA 21871525Sjhb * 1: ok 21971525Sjhb * 22071525Sjhb * XXX FIXME: this is not MP-safe, needs a lock to prevent multiple CPUs 22171525Sjhb * from executing at same time. 22271525Sjhb */ 22371525Sjhbint 224127498Smarcelstop_cpus(cpumask_t map) 22571525Sjhb{ 22676078Sjhb int i; 22736135Stegge 22871525Sjhb if (!smp_started) 22971525Sjhb return 0; 23071525Sjhb 23176078Sjhb CTR1(KTR_SMP, "stop_cpus(%x)", map); 23276078Sjhb 23376078Sjhb /* send the stop IPI to all CPUs in map */ 23475421Sjhb ipi_selected(map, IPI_STOP); 235127498Smarcel 23676078Sjhb i = 0; 23776078Sjhb while ((atomic_load_acq_int(&stopped_cpus) & map) != map) { 23876078Sjhb /* spin */ 23976078Sjhb i++; 24071525Sjhb#ifdef DIAGNOSTIC 24176078Sjhb if (i == 100000) { 24276078Sjhb printf("timeout stopping cpus\n"); 24376078Sjhb break; 24476078Sjhb } 24571525Sjhb#endif 24676078Sjhb } 24771525Sjhb 24871525Sjhb return 1; 24971525Sjhb} 25071525Sjhb 25171525Sjhb 25271525Sjhb/* 25371525Sjhb * Called by a CPU to restart stopped CPUs. 25471525Sjhb * 25571525Sjhb * Usually (but not necessarily) called with 'stopped_cpus' as its arg. 25671525Sjhb * 25771525Sjhb * - Signals all CPUs in map to restart. 25871525Sjhb * - Waits for each to restart. 25971525Sjhb * 26071525Sjhb * Returns: 26171525Sjhb * -1: error 26271525Sjhb * 0: NA 26371525Sjhb * 1: ok 26471525Sjhb */ 26571525Sjhbint 266127498Smarcelrestart_cpus(cpumask_t map) 26771525Sjhb{ 26871525Sjhb 26971525Sjhb if (!smp_started) 27071525Sjhb return 0; 27171525Sjhb 27276078Sjhb CTR1(KTR_SMP, "restart_cpus(%x)", map); 27371525Sjhb 27476078Sjhb /* signal other cpus to restart */ 27576078Sjhb atomic_store_rel_int(&started_cpus, map); 27676078Sjhb 27771525Sjhb /* wait for each to clear its bit */ 27876078Sjhb while ((atomic_load_acq_int(&stopped_cpus) & map) != 0) 27976078Sjhb ; /* nothing */ 28071525Sjhb 28171525Sjhb return 1; 28271525Sjhb} 28371525Sjhb 28434021Stegge/* 28548924Smsmith * All-CPU rendezvous. CPUs are signalled, all execute the setup function 28648924Smsmith * (if specified), rendezvous, execute the action function (if specified), 28748924Smsmith * rendezvous again, execute the teardown function (if specified), and then 28848924Smsmith * resume. 28948924Smsmith * 29048924Smsmith * Note that the supplied external functions _must_ be reentrant and aware 29148924Smsmith * that they are running in parallel and in an unknown lock context. 29248924Smsmith */ 29348924Smsmithvoid 29448924Smsmithsmp_rendezvous_action(void) 29548924Smsmith{ 29676078Sjhb 29748924Smsmith /* setup function */ 29848924Smsmith if (smp_rv_setup_func != NULL) 29948924Smsmith smp_rv_setup_func(smp_rv_func_arg); 30048924Smsmith /* spin on entry rendezvous */ 30148924Smsmith atomic_add_int(&smp_rv_waiters[0], 1); 30276078Sjhb while (atomic_load_acq_int(&smp_rv_waiters[0]) < mp_ncpus) 30376078Sjhb ; /* nothing */ 30448924Smsmith /* action function */ 30548924Smsmith if (smp_rv_action_func != NULL) 30648924Smsmith smp_rv_action_func(smp_rv_func_arg); 30748924Smsmith /* spin on exit rendezvous */ 30848924Smsmith atomic_add_int(&smp_rv_waiters[1], 1); 30976078Sjhb while (atomic_load_acq_int(&smp_rv_waiters[1]) < mp_ncpus) 31076078Sjhb ; /* nothing */ 31148924Smsmith /* teardown function */ 31248924Smsmith if (smp_rv_teardown_func != NULL) 31348924Smsmith smp_rv_teardown_func(smp_rv_func_arg); 31448924Smsmith} 31548924Smsmith 31648924Smsmithvoid 31748924Smsmithsmp_rendezvous(void (* setup_func)(void *), 31848924Smsmith void (* action_func)(void *), 31948924Smsmith void (* teardown_func)(void *), 32048924Smsmith void *arg) 32148924Smsmith{ 32271576Sjasone 32376078Sjhb if (!smp_started) { 32476078Sjhb if (setup_func != NULL) 32576078Sjhb setup_func(arg); 32676078Sjhb if (action_func != NULL) 32776078Sjhb action_func(arg); 32876078Sjhb if (teardown_func != NULL) 32976078Sjhb teardown_func(arg); 33076078Sjhb return; 33176078Sjhb } 33276078Sjhb 33348924Smsmith /* obtain rendezvous lock */ 33472200Sbmilekic mtx_lock_spin(&smp_rv_mtx); 33548924Smsmith 33648924Smsmith /* set static function pointers */ 33748924Smsmith smp_rv_setup_func = setup_func; 33848924Smsmith smp_rv_action_func = action_func; 33948924Smsmith smp_rv_teardown_func = teardown_func; 34048924Smsmith smp_rv_func_arg = arg; 34148924Smsmith smp_rv_waiters[0] = 0; 34248924Smsmith smp_rv_waiters[1] = 0; 34348924Smsmith 34476078Sjhb /* signal other processors, which will enter the IPI with interrupts off */ 34575421Sjhb ipi_all_but_self(IPI_RENDEZVOUS); 34648924Smsmith 34748924Smsmith /* call executor function */ 34848924Smsmith smp_rendezvous_action(); 34948924Smsmith 35048924Smsmith /* release lock */ 35172200Sbmilekic mtx_unlock_spin(&smp_rv_mtx); 35248924Smsmith} 353123125Sjhb#else /* !SMP */ 354123125Sjhb 355123125Sjhb/* 356123125Sjhb * Provide dummy SMP support for UP kernels. Modules that need to use SMP 357123125Sjhb * APIs will still work using this dummy support. 358123125Sjhb */ 359123125Sjhbstatic void 360123125Sjhbmp_setvariables_for_up(void *dummy) 361123125Sjhb{ 362123125Sjhb mp_ncpus = 1; 363123125Sjhb mp_maxid = PCPU_GET(cpuid); 364123125Sjhb all_cpus = PCPU_GET(cpumask); 365123125Sjhb KASSERT(PCPU_GET(cpuid) == 0, ("UP must have a CPU ID of zero")); 366123125Sjhb} 367123125SjhbSYSINIT(cpu_mp_setvariables, SI_SUB_TUNABLES, SI_ORDER_FIRST, 368123125Sjhb mp_setvariables_for_up, NULL) 369123125Sjhb 370123125Sjhbvoid 371123125Sjhbsmp_rendezvous(void (* setup_func)(void *), 372123125Sjhb void (* action_func)(void *), 373123125Sjhb void (* teardown_func)(void *), 374123125Sjhb void *arg) 375123125Sjhb{ 376123125Sjhb 377123125Sjhb if (setup_func != NULL) 378123125Sjhb setup_func(arg); 379123125Sjhb if (action_func != NULL) 380123125Sjhb action_func(arg); 381123125Sjhb if (teardown_func != NULL) 382123125Sjhb teardown_func(arg); 383123125Sjhb} 384123125Sjhb#endif /* SMP */ 385