subr_smp.c revision 93873
125164Speter/* 276078Sjhb * Copyright (c) 2001 376078Sjhb * John Baldwin <jhb@FreeBSD.org>. All rights reserved. 425164Speter * 525164Speter * Redistribution and use in source and binary forms, with or without 625164Speter * modification, are permitted provided that the following conditions 725164Speter * are met: 825164Speter * 1. Redistributions of source code must retain the above copyright 925164Speter * notice, this list of conditions and the following disclaimer. 1076078Sjhb * 2. Redistributions in binary form must reproduce the above copyright 1176078Sjhb * notice, this list of conditions and the following disclaimer in the 1276078Sjhb * documentation and/or other materials provided with the distribution. 1376078Sjhb * 4. Neither the name of the author nor the names of any co-contributors 1476078Sjhb * may be used to endorse or promote products derived from this software 1576078Sjhb * without specific prior written permission. 1625164Speter * 1776078Sjhb * THIS SOFTWARE IS PROVIDED BY JOHN BALDWIN AND CONTRIBUTORS ``AS IS'' AND 1825164Speter * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 1925164Speter * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 2076078Sjhb * ARE DISCLAIMED. IN NO EVENT SHALL JOHN BALDWIN OR THE VOICES IN HIS HEAD 2176078Sjhb * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 2276078Sjhb * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 2376078Sjhb * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 2476078Sjhb * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 2576078Sjhb * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 2676078Sjhb * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF 2776078Sjhb * THE POSSIBILITY OF SUCH DAMAGE. 2825164Speter * 2950477Speter * $FreeBSD: head/sys/kern/subr_smp.c 93873 2002-04-05 10:00:37Z bde $ 3025164Speter */ 3125164Speter 3276078Sjhb/* 3376078Sjhb * This module holds the global variables and machine independent functions 3476440Sjhb * used for the kernel SMP support. 3576078Sjhb */ 3625164Speter 3728743Sbde#include <sys/param.h> 3825164Speter#include <sys/systm.h> 3976440Sjhb#include <sys/kernel.h> 4076078Sjhb#include <sys/ktr.h> 4128808Speter#include <sys/proc.h> 4276078Sjhb#include <sys/lock.h> 4367365Sjhb#include <sys/mutex.h> 4476440Sjhb#include <sys/pcpu.h> 4576078Sjhb#include <sys/smp.h> 4676078Sjhb#include <sys/sysctl.h> 4725164Speter 4891778Sjake#include <machine/smp.h> 4991778Sjake 5076078Sjhbvolatile u_int stopped_cpus; 5176078Sjhbvolatile u_int started_cpus; 5225164Speter 5392723Salfredvoid (*cpustop_restartfunc)(void); 5476078Sjhbint mp_ncpus; 5528027Sfsmp 5685787Smarcelvolatile int smp_started; 5776078Sjhbu_int all_cpus; 5891673Sjeffu_int mp_maxid; 5925164Speter 6076078SjhbSYSCTL_NODE(_kern, OID_AUTO, smp, CTLFLAG_RD, NULL, "Kernel SMP"); 6125164Speter 6276078Sjhbint smp_active = 0; /* are the APs allowed to run? */ 6376078SjhbSYSCTL_INT(_kern_smp, OID_AUTO, active, CTLFLAG_RW, &smp_active, 0, ""); 6426155Sfsmp 6576078Sjhbint smp_cpus = 1; /* how many cpu's running */ 6676078SjhbSYSCTL_INT(_kern_smp, OID_AUTO, cpus, CTLFLAG_RD, &smp_cpus, 0, ""); 6727005Sfsmp 6871525Sjhb/* Enable forwarding of a signal to a process running on a different CPU */ 6971525Sjhbstatic int forward_signal_enabled = 1; 7076078SjhbSYSCTL_INT(_kern_smp, OID_AUTO, forward_signal_enabled, CTLFLAG_RW, 7171525Sjhb &forward_signal_enabled, 0, ""); 7271525Sjhb 7371525Sjhb/* Enable forwarding of roundrobin to all other cpus */ 7471525Sjhbstatic int forward_roundrobin_enabled = 1; 7576078SjhbSYSCTL_INT(_kern_smp, OID_AUTO, forward_roundrobin_enabled, CTLFLAG_RW, 7671525Sjhb &forward_roundrobin_enabled, 0, ""); 7771525Sjhb 7876078Sjhb/* Variables needed for SMP rendezvous. */ 7976078Sjhbstatic void (*smp_rv_setup_func)(void *arg); 8076078Sjhbstatic void (*smp_rv_action_func)(void *arg); 8176078Sjhbstatic void (*smp_rv_teardown_func)(void *arg); 8276078Sjhbstatic void *smp_rv_func_arg; 8376078Sjhbstatic volatile int smp_rv_waiters[2]; 8476078Sjhbstatic struct mtx smp_rv_mtx; 8591673Sjeffstatic int mp_probe_status; 8671525Sjhb 8725164Speter/* 8891673Sjeff * Initialize MI SMP variables. 8925164Speter */ 9071525Sjhbstatic void 9191673Sjeffmp_probe(void *dummy) 9291673Sjeff{ 9391673Sjeff mp_probe_status = cpu_mp_probe(); 9491673Sjeff} 9592799SarrSYSINIT(cpu_mp_probe, SI_SUB_TUNABLES, SI_ORDER_FIRST, mp_probe, NULL) 9691673Sjeff 9791673Sjeff/* 9891673Sjeff * Call the MD SMP initialization code. 9991673Sjeff */ 10091673Sjeffstatic void 10176078Sjhbmp_start(void *dummy) 10271525Sjhb{ 10371525Sjhb 10476078Sjhb /* Probe for MP hardware. */ 10591673Sjeff if (mp_probe_status == 0) 10676078Sjhb return; 10776078Sjhb 10893818Sjhb mtx_init(&smp_rv_mtx, "smp rendezvous", NULL, MTX_SPIN); 10976078Sjhb cpu_mp_start(); 11076078Sjhb printf("FreeBSD/SMP: Multiprocessor System Detected: %d CPUs\n", 11176078Sjhb mp_ncpus); 11276078Sjhb cpu_mp_announce(); 11371525Sjhb} 11476078SjhbSYSINIT(cpu_mp, SI_SUB_CPU, SI_ORDER_SECOND, mp_start, NULL) 11571525Sjhb 11625164Spetervoid 11783366Sjulianforward_signal(struct thread *td) 11825164Speter{ 11976078Sjhb int id; 12025164Speter 12126108Sfsmp /* 12293873Sbde * signotify() has already set KEF_ASTPENDING and PS_NEEDSIGCHECK on 12393873Sbde * this process, so all we need to do is poke it if it is currently 12493873Sbde * executing so that it executes ast(). 12526108Sfsmp */ 12671525Sjhb mtx_assert(&sched_lock, MA_OWNED); 12793873Sbde KASSERT(td->td_proc->p_stat == SRUN, 12893873Sbde ("forward_signal: process is not SRUN")); 12931639Sfsmp 13083366Sjulian CTR1(KTR_SMP, "forward_signal(%p)", td->td_proc); 13165557Sjasone 13276078Sjhb if (!smp_started || cold || panicstr) 13331639Sfsmp return; 13476078Sjhb if (!forward_signal_enabled) 13576078Sjhb return; 13631639Sfsmp 13776078Sjhb /* No need to IPI ourself. */ 13883366Sjulian if (td == curthread) 13931639Sfsmp return; 14031639Sfsmp 14183366Sjulian id = td->td_kse->ke_oncpu; 14276078Sjhb if (id == NOCPU) 14334020Stegge return; 14476078Sjhb ipi_selected(1 << id, IPI_AST); 14534020Stegge} 14634021Stegge 14736135Steggevoid 14836135Steggeforward_roundrobin(void) 14936135Stegge{ 15087702Sjhb struct pcpu *pc; 15183366Sjulian struct thread *td; 15276078Sjhb u_int id, map; 15334021Stegge 15476078Sjhb mtx_assert(&sched_lock, MA_OWNED); 15576078Sjhb 15671525Sjhb CTR0(KTR_SMP, "forward_roundrobin()"); 15771525Sjhb 15876078Sjhb if (!smp_started || cold || panicstr) 15936135Stegge return; 16036135Stegge if (!forward_roundrobin_enabled) 16136135Stegge return; 16276078Sjhb map = 0; 16387702Sjhb SLIST_FOREACH(pc, &cpuhead, pc_allcpu) { 16487702Sjhb td = pc->pc_curthread; 16588902Speter id = pc->pc_cpumask; 16688902Speter if (id != PCPU_GET(cpumask) && (id & stopped_cpus) == 0 && 16787702Sjhb td != pc->pc_idlethread) { 16883366Sjulian td->td_kse->ke_flags |= KEF_NEEDRESCHED; 16976078Sjhb map |= id; 17036135Stegge } 17136135Stegge } 17276078Sjhb ipi_selected(map, IPI_AST); 17336135Stegge} 17436135Stegge 17571525Sjhb/* 17671525Sjhb * When called the executing CPU will send an IPI to all other CPUs 17771525Sjhb * requesting that they halt execution. 17871525Sjhb * 17971525Sjhb * Usually (but not necessarily) called with 'other_cpus' as its arg. 18071525Sjhb * 18171525Sjhb * - Signals all CPUs in map to stop. 18271525Sjhb * - Waits for each to stop. 18371525Sjhb * 18471525Sjhb * Returns: 18571525Sjhb * -1: error 18671525Sjhb * 0: NA 18771525Sjhb * 1: ok 18871525Sjhb * 18971525Sjhb * XXX FIXME: this is not MP-safe, needs a lock to prevent multiple CPUs 19071525Sjhb * from executing at same time. 19171525Sjhb */ 19271525Sjhbint 19371525Sjhbstop_cpus(u_int map) 19471525Sjhb{ 19576078Sjhb int i; 19636135Stegge 19771525Sjhb if (!smp_started) 19871525Sjhb return 0; 19971525Sjhb 20076078Sjhb CTR1(KTR_SMP, "stop_cpus(%x)", map); 20176078Sjhb 20276078Sjhb /* send the stop IPI to all CPUs in map */ 20375421Sjhb ipi_selected(map, IPI_STOP); 20471525Sjhb 20576078Sjhb i = 0; 20676078Sjhb while ((atomic_load_acq_int(&stopped_cpus) & map) != map) { 20776078Sjhb /* spin */ 20876078Sjhb i++; 20971525Sjhb#ifdef DIAGNOSTIC 21076078Sjhb if (i == 100000) { 21176078Sjhb printf("timeout stopping cpus\n"); 21276078Sjhb break; 21376078Sjhb } 21471525Sjhb#endif 21576078Sjhb } 21671525Sjhb 21771525Sjhb return 1; 21871525Sjhb} 21971525Sjhb 22071525Sjhb 22171525Sjhb/* 22271525Sjhb * Called by a CPU to restart stopped CPUs. 22371525Sjhb * 22471525Sjhb * Usually (but not necessarily) called with 'stopped_cpus' as its arg. 22571525Sjhb * 22671525Sjhb * - Signals all CPUs in map to restart. 22771525Sjhb * - Waits for each to restart. 22871525Sjhb * 22971525Sjhb * Returns: 23071525Sjhb * -1: error 23171525Sjhb * 0: NA 23271525Sjhb * 1: ok 23371525Sjhb */ 23471525Sjhbint 23571525Sjhbrestart_cpus(u_int map) 23671525Sjhb{ 23771525Sjhb 23871525Sjhb if (!smp_started) 23971525Sjhb return 0; 24071525Sjhb 24176078Sjhb CTR1(KTR_SMP, "restart_cpus(%x)", map); 24271525Sjhb 24376078Sjhb /* signal other cpus to restart */ 24476078Sjhb atomic_store_rel_int(&started_cpus, map); 24576078Sjhb 24671525Sjhb /* wait for each to clear its bit */ 24776078Sjhb while ((atomic_load_acq_int(&stopped_cpus) & map) != 0) 24876078Sjhb ; /* nothing */ 24971525Sjhb 25071525Sjhb return 1; 25171525Sjhb} 25271525Sjhb 25334021Stegge/* 25448924Smsmith * All-CPU rendezvous. CPUs are signalled, all execute the setup function 25548924Smsmith * (if specified), rendezvous, execute the action function (if specified), 25648924Smsmith * rendezvous again, execute the teardown function (if specified), and then 25748924Smsmith * resume. 25848924Smsmith * 25948924Smsmith * Note that the supplied external functions _must_ be reentrant and aware 26048924Smsmith * that they are running in parallel and in an unknown lock context. 26148924Smsmith */ 26248924Smsmithvoid 26348924Smsmithsmp_rendezvous_action(void) 26448924Smsmith{ 26576078Sjhb 26648924Smsmith /* setup function */ 26748924Smsmith if (smp_rv_setup_func != NULL) 26848924Smsmith smp_rv_setup_func(smp_rv_func_arg); 26948924Smsmith /* spin on entry rendezvous */ 27048924Smsmith atomic_add_int(&smp_rv_waiters[0], 1); 27176078Sjhb while (atomic_load_acq_int(&smp_rv_waiters[0]) < mp_ncpus) 27276078Sjhb ; /* nothing */ 27348924Smsmith /* action function */ 27448924Smsmith if (smp_rv_action_func != NULL) 27548924Smsmith smp_rv_action_func(smp_rv_func_arg); 27648924Smsmith /* spin on exit rendezvous */ 27748924Smsmith atomic_add_int(&smp_rv_waiters[1], 1); 27876078Sjhb while (atomic_load_acq_int(&smp_rv_waiters[1]) < mp_ncpus) 27976078Sjhb ; /* nothing */ 28048924Smsmith /* teardown function */ 28148924Smsmith if (smp_rv_teardown_func != NULL) 28248924Smsmith smp_rv_teardown_func(smp_rv_func_arg); 28348924Smsmith} 28448924Smsmith 28548924Smsmithvoid 28648924Smsmithsmp_rendezvous(void (* setup_func)(void *), 28748924Smsmith void (* action_func)(void *), 28848924Smsmith void (* teardown_func)(void *), 28948924Smsmith void *arg) 29048924Smsmith{ 29171576Sjasone 29276078Sjhb if (!smp_started) { 29376078Sjhb if (setup_func != NULL) 29476078Sjhb setup_func(arg); 29576078Sjhb if (action_func != NULL) 29676078Sjhb action_func(arg); 29776078Sjhb if (teardown_func != NULL) 29876078Sjhb teardown_func(arg); 29976078Sjhb return; 30076078Sjhb } 30176078Sjhb 30248924Smsmith /* obtain rendezvous lock */ 30372200Sbmilekic mtx_lock_spin(&smp_rv_mtx); 30448924Smsmith 30548924Smsmith /* set static function pointers */ 30648924Smsmith smp_rv_setup_func = setup_func; 30748924Smsmith smp_rv_action_func = action_func; 30848924Smsmith smp_rv_teardown_func = teardown_func; 30948924Smsmith smp_rv_func_arg = arg; 31048924Smsmith smp_rv_waiters[0] = 0; 31148924Smsmith smp_rv_waiters[1] = 0; 31248924Smsmith 31376078Sjhb /* signal other processors, which will enter the IPI with interrupts off */ 31475421Sjhb ipi_all_but_self(IPI_RENDEZVOUS); 31548924Smsmith 31648924Smsmith /* call executor function */ 31748924Smsmith smp_rendezvous_action(); 31848924Smsmith 31948924Smsmith /* release lock */ 32072200Sbmilekic mtx_unlock_spin(&smp_rv_mtx); 32148924Smsmith} 322