189051Sjake/*- 289051Sjake * Copyright (c) 1997 Berkeley Software Design, Inc. All rights reserved. 389051Sjake * 489051Sjake * Redistribution and use in source and binary forms, with or without 589051Sjake * modification, are permitted provided that the following conditions 689051Sjake * are met: 789051Sjake * 1. Redistributions of source code must retain the above copyright 889051Sjake * notice, this list of conditions and the following disclaimer. 989051Sjake * 2. Redistributions in binary form must reproduce the above copyright 1089051Sjake * notice, this list of conditions and the following disclaimer in the 1189051Sjake * documentation and/or other materials provided with the distribution. 1289051Sjake * 3. Berkeley Software Design Inc's name may not be used to endorse or 1389051Sjake * promote products derived from this software without specific prior 1489051Sjake * written permission. 1589051Sjake * 1689051Sjake * THIS SOFTWARE IS PROVIDED BY BERKELEY SOFTWARE DESIGN INC ``AS IS'' AND 1789051Sjake * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 1889051Sjake * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 1989051Sjake * ARE DISCLAIMED. IN NO EVENT SHALL BERKELEY SOFTWARE DESIGN INC BE LIABLE 2089051Sjake * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 2189051Sjake * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 2289051Sjake * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 2389051Sjake * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 2489051Sjake * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 2589051Sjake * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 2689051Sjake * SUCH DAMAGE. 2789051Sjake * 2889051Sjake * from BSDI: locore.s,v 1.36.2.15 1999/08/23 22:34:41 cp Exp 2989051Sjake */ 3089051Sjake/*- 3189051Sjake * Copyright (c) 2002 Jake Burkholder. 32211050Smarius * Copyright (c) 2007 - 2010 Marius Strobl <marius@FreeBSD.org> 3389051Sjake * All rights reserved. 3489051Sjake * 3589051Sjake * Redistribution and use in source and binary forms, with or without 3689051Sjake * modification, are permitted provided that the following conditions 3789051Sjake * are met: 3889051Sjake * 1. Redistributions of source code must retain the above copyright 3989051Sjake * notice, this list of conditions and the following disclaimer. 4089051Sjake * 2. Redistributions in binary form must reproduce the above copyright 4189051Sjake * notice, this list of conditions and the following disclaimer in the 4289051Sjake * documentation and/or other materials provided with the distribution. 4389051Sjake * 4489051Sjake * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 4589051Sjake * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 4689051Sjake * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 4789051Sjake * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 4889051Sjake * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 4989051Sjake * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 5089051Sjake * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 5189051Sjake * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 5289051Sjake * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 5389051Sjake * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 5489051Sjake * SUCH DAMAGE. 5589051Sjake */ 5689051Sjake 57145150Smarius#include <sys/cdefs.h> 58145150Smarius__FBSDID("$FreeBSD: releng/10.3/sys/sparc64/sparc64/mp_machdep.c 292771 2015-12-27 14:39:47Z marius $"); 59145150Smarius 6089051Sjake#include <sys/param.h> 6189051Sjake#include <sys/systm.h> 6289051Sjake#include <sys/lock.h> 63131950Smarcel#include <sys/kdb.h> 6489051Sjake#include <sys/kernel.h> 6589051Sjake#include <sys/ktr.h> 6689051Sjake#include <sys/mutex.h> 6789051Sjake#include <sys/pcpu.h> 6889051Sjake#include <sys/proc.h> 69170846Smarius#include <sys/sched.h> 7089051Sjake#include <sys/smp.h> 7189051Sjake 7289051Sjake#include <vm/vm.h> 7389051Sjake#include <vm/vm_param.h> 7489051Sjake#include <vm/pmap.h> 7589051Sjake#include <vm/vm_kern.h> 7689051Sjake#include <vm/vm_extern.h> 7789051Sjake#include <vm/vm_map.h> 7889051Sjake 7989051Sjake#include <dev/ofw/openfirm.h> 8089051Sjake 8189051Sjake#include <machine/asi.h> 8292205Sjake#include <machine/atomic.h> 83119696Smarcel#include <machine/bus.h> 84182730Smarius#include <machine/cpu.h> 85286055Smarius#include <machine/cpufunc.h> 8689051Sjake#include <machine/md_var.h> 8797511Sjake#include <machine/metadata.h> 8892205Sjake#include <machine/ofw_machdep.h> 89152022Sjhb#include <machine/pcb.h> 9089051Sjake#include <machine/smp.h> 9195132Sjake#include <machine/tick.h> 9291617Sjake#include <machine/tlb.h> 93216803Smarius#include <machine/tsb.h> 9489051Sjake#include <machine/tte.h> 95170846Smarius#include <machine/ver.h> 9689051Sjake 97183142Smarius#define SUNW_STARTCPU "SUNW,start-cpu" 98183142Smarius#define SUNW_STOPSELF "SUNW,stop-self" 99183142Smarius 10089051Sjakestatic ih_func_t cpu_ipi_ast; 101210601Smavstatic ih_func_t cpu_ipi_hardclock; 102178048Smariusstatic ih_func_t cpu_ipi_preempt; 10389051Sjakestatic ih_func_t cpu_ipi_stop; 10489051Sjake 10589051Sjake/* 10689051Sjake * Argument area used to pass data to non-boot processors as they start up. 107169796Smarius * This must be statically initialized with a known invalid CPU module ID, 108169796Smarius * since the other processors will use it before the boot CPU enters the 10989051Sjake * kernel. 11089051Sjake */ 111182730Smariusstruct cpu_start_args cpu_start_args = { 0, -1, -1, 0, 0, 0 }; 11297001Sjakestruct ipi_cache_args ipi_cache_args; 113211071Smariusstruct ipi_rd_args ipi_rd_args; 11491783Sjakestruct ipi_tlb_args ipi_tlb_args; 115152022Sjhbstruct pcb stoppcbs[MAXCPU]; 11689051Sjake 117286055Smariusstruct mtx ipi_mtx; 118286055Smarius 119170846Smariuscpu_ipi_selected_t *cpu_ipi_selected; 120211050Smariuscpu_ipi_single_t *cpu_ipi_single; 12191617Sjake 122170846Smariusstatic u_int cpuid_to_mid[MAXCPU]; 123292771Smariusstatic u_int cpuids = 1; 124222813Sattiliostatic volatile cpuset_t shutdown_cpus; 125292771Smariusstatic char ipi_pbuf[CPUSETBUFSIZ]; 126292771Smariusstatic vm_offset_t mp_tramp; 12792205Sjake 128204152Smariusstatic void ap_count(phandle_t node, u_int mid, u_int cpu_impl); 129204152Smariusstatic void ap_start(phandle_t node, u_int mid, u_int cpu_impl); 130169796Smariusstatic void cpu_mp_unleash(void *v); 131203838Smariusstatic void foreach_ap(phandle_t node, void (*func)(phandle_t node, 132204152Smarius u_int mid, u_int cpu_impl)); 133169796Smariusstatic void sun4u_startcpu(phandle_t cpu, void *func, u_long arg); 134169796Smarius 135170846Smariusstatic cpu_ipi_selected_t cheetah_ipi_selected; 136211050Smariusstatic cpu_ipi_single_t cheetah_ipi_single; 137211050Smariusstatic cpu_ipi_selected_t jalapeno_ipi_selected; 138211050Smariusstatic cpu_ipi_single_t jalapeno_ipi_single; 139170846Smariusstatic cpu_ipi_selected_t spitfire_ipi_selected; 140211050Smariusstatic cpu_ipi_single_t spitfire_ipi_single; 141170846Smarius 14291617SjakeSYSINIT(cpu_mp_unleash, SI_SUB_SMP, SI_ORDER_FIRST, cpu_mp_unleash, NULL); 14391617Sjake 144170846Smariusvoid 145286055Smariusmp_init(void) 14691617Sjake{ 14791617Sjake struct tte *tp; 14891617Sjake int i; 14991617Sjake 150170846Smarius mp_tramp = (vm_offset_t)OF_claim(NULL, PAGE_SIZE, PAGE_SIZE); 151170846Smarius if (mp_tramp == (vm_offset_t)-1) 152169796Smarius panic("%s", __func__); 153170846Smarius bcopy(mp_tramp_code, (void *)mp_tramp, mp_tramp_code_len); 154170846Smarius *(vm_offset_t *)(mp_tramp + mp_tramp_tlb_slots) = kernel_tlb_slots; 155170846Smarius *(vm_offset_t *)(mp_tramp + mp_tramp_func) = (vm_offset_t)mp_startup; 156170846Smarius tp = (struct tte *)(mp_tramp + mp_tramp_code_len); 15797511Sjake for (i = 0; i < kernel_tlb_slots; i++) { 158102042Sjake tp[i].tte_vpn = TV_VPN(kernel_tlbs[i].te_va, TS_4M); 15997511Sjake tp[i].tte_data = TD_V | TD_4M | TD_PA(kernel_tlbs[i].te_pa) | 16097511Sjake TD_L | TD_CP | TD_CV | TD_P | TD_W; 16197511Sjake } 162170846Smarius for (i = 0; i < PAGE_SIZE; i += sizeof(vm_offset_t)) 163170846Smarius flush(mp_tramp + i); 16491617Sjake} 16591617Sjake 166203838Smariusstatic void 167204152Smariusforeach_ap(phandle_t node, void (*func)(phandle_t node, u_int mid, 168204152Smarius u_int cpu_impl)) 169203838Smarius{ 170292771Smarius static char type[sizeof("cpu")]; 171203838Smarius phandle_t child; 172292771Smarius uint32_t cpu_impl, portid; 173211050Smarius 174203838Smarius /* There's no need to traverse the whole OFW tree twice. */ 175292771Smarius if (mp_maxid > 0 && cpuids > mp_maxid) 176203838Smarius return; 177203838Smarius 178203838Smarius for (; node != 0; node = OF_peer(node)) { 179203838Smarius child = OF_child(node); 180203838Smarius if (child > 0) 181203838Smarius foreach_ap(child, func); 182203838Smarius else { 183203838Smarius if (OF_getprop(node, "device_type", type, 184203838Smarius sizeof(type)) <= 0) 185203838Smarius continue; 186203838Smarius if (strcmp(type, "cpu") != 0) 187203838Smarius continue; 188204152Smarius if (OF_getprop(node, "implementation#", &cpu_impl, 189204152Smarius sizeof(cpu_impl)) <= 0) 190204152Smarius panic("%s: couldn't determine CPU " 191204152Smarius "implementation", __func__); 192292771Smarius if (OF_getprop(node, cpu_portid_prop(cpu_impl), 193292771Smarius &portid, sizeof(portid)) <= 0) 194292771Smarius panic("%s: couldn't determine CPU port ID", 195204152Smarius __func__); 196292771Smarius if (portid == PCPU_GET(mid)) 197203838Smarius continue; 198292771Smarius (*func)(node, portid, cpu_impl); 199203838Smarius } 200203838Smarius } 201203838Smarius} 202203838Smarius 20389051Sjake/* 204169796Smarius * Probe for other CPUs. 20589051Sjake */ 206122947Sjhbvoid 207286055Smariuscpu_mp_setmaxid(void) 20889051Sjake{ 20989051Sjake 210222813Sattilio CPU_SETOF(curcpu, &all_cpus); 21189051Sjake mp_ncpus = 1; 21289051Sjake 213203838Smarius foreach_ap(OF_child(OF_peer(0)), ap_count); 214292771Smarius mp_ncpus = MIN(mp_ncpus, MAXCPU); 215292771Smarius mp_maxid = mp_ncpus - 1; 21689051Sjake} 21789051Sjake 218203838Smariusstatic void 219204152Smariusap_count(phandle_t node __unused, u_int mid __unused, u_int cpu_impl __unused) 220203838Smarius{ 221203838Smarius 222292771Smarius mp_ncpus++; 223203838Smarius} 224203838Smarius 225122947Sjhbint 226122947Sjhbcpu_mp_probe(void) 227122947Sjhb{ 228122947Sjhb 229123126Sjhb return (mp_maxid > 0); 230122947Sjhb} 231122947Sjhb 232176734Sjeffstruct cpu_group * 233176734Sjeffcpu_topo(void) 234176734Sjeff{ 235176734Sjeff 236176994Smarius return (smp_topo_none()); 237176734Sjeff} 238176734Sjeff 23991617Sjakestatic void 24091617Sjakesun4u_startcpu(phandle_t cpu, void *func, u_long arg) 24191617Sjake{ 24291617Sjake static struct { 24391617Sjake cell_t name; 24491617Sjake cell_t nargs; 24591617Sjake cell_t nreturns; 24691617Sjake cell_t cpu; 24791617Sjake cell_t func; 24891617Sjake cell_t arg; 24991617Sjake } args = { 250183142Smarius (cell_t)SUNW_STARTCPU, 25191617Sjake 3, 25291617Sjake }; 25391617Sjake 25491617Sjake args.cpu = cpu; 25591617Sjake args.func = (cell_t)func; 25691617Sjake args.arg = (cell_t)arg; 257186347Snwhitehorn ofw_entry(&args); 25891617Sjake} 25991617Sjake 26089051Sjake/* 26189051Sjake * Fire up any non-boot processors. 26289051Sjake */ 26389051Sjakevoid 26489051Sjakecpu_mp_start(void) 26589051Sjake{ 266286055Smarius u_int cpu_impl, isjbus; 26789051Sjake 268286055Smarius mtx_init(&ipi_mtx, "ipi", NULL, MTX_SPIN); 269286055Smarius 270286055Smarius isjbus = 0; 271286055Smarius cpu_impl = PCPU_GET(impl); 272286055Smarius if (cpu_impl == CPU_IMPL_ULTRASPARCIIIi || 273286055Smarius cpu_impl == CPU_IMPL_ULTRASPARCIIIip) { 274286055Smarius isjbus = 1; 275286055Smarius cpu_ipi_selected = jalapeno_ipi_selected; 276286055Smarius cpu_ipi_single = jalapeno_ipi_single; 277286055Smarius } else if (cpu_impl == CPU_IMPL_SPARC64V || 278286055Smarius cpu_impl >= CPU_IMPL_ULTRASPARCIII) { 279286055Smarius cpu_ipi_selected = cheetah_ipi_selected; 280286055Smarius cpu_ipi_single = cheetah_ipi_single; 281286055Smarius } else { 282286055Smarius cpu_ipi_selected = spitfire_ipi_selected; 283286055Smarius cpu_ipi_single = spitfire_ipi_single; 284286055Smarius } 285286055Smarius 28689051Sjake intr_setup(PIL_AST, cpu_ipi_ast, -1, NULL, NULL); 28789051Sjake intr_setup(PIL_RENDEZVOUS, (ih_func_t *)smp_rendezvous_action, 28889051Sjake -1, NULL, NULL); 28989051Sjake intr_setup(PIL_STOP, cpu_ipi_stop, -1, NULL, NULL); 290178048Smarius intr_setup(PIL_PREEMPT, cpu_ipi_preempt, -1, NULL, NULL); 291210601Smav intr_setup(PIL_HARDCLOCK, cpu_ipi_hardclock, -1, NULL, NULL); 29289051Sjake 293169796Smarius cpuid_to_mid[curcpu] = PCPU_GET(mid); 294157240Smarius 295203838Smarius foreach_ap(OF_child(OF_peer(0)), ap_start); 296203838Smarius KASSERT(!isjbus || mp_ncpus <= IDR_JALAPENO_MAX_BN_PAIRS, 297203838Smarius ("%s: can only IPI a maximum of %d JBus-CPUs", 298203838Smarius __func__, IDR_JALAPENO_MAX_BN_PAIRS)); 299203838Smarius} 300203838Smarius 301203838Smariusstatic void 302204152Smariusap_start(phandle_t node, u_int mid, u_int cpu_impl) 303203838Smarius{ 304203838Smarius volatile struct cpu_start_args *csa; 305203838Smarius struct pcpu *pc; 306203838Smarius register_t s; 307203838Smarius vm_offset_t va; 308203838Smarius u_int cpuid; 309204152Smarius uint32_t clock; 310203838Smarius 311292771Smarius if (cpuids > mp_maxid) 312203838Smarius return; 313203838Smarius 314203838Smarius if (OF_getprop(node, "clock-frequency", &clock, sizeof(clock)) <= 0) 315204152Smarius panic("%s: couldn't determine CPU frequency", __func__); 316203838Smarius if (clock != PCPU_GET(clock)) 317214071Smarius tick_et_use_stick = 1; 318203838Smarius 31989051Sjake csa = &cpu_start_args; 320203838Smarius csa->csa_state = 0; 321203838Smarius sun4u_startcpu(node, (void *)mp_tramp, 0); 322203838Smarius s = intr_disable(); 323203838Smarius while (csa->csa_state != CPU_TICKSYNC) 324203838Smarius ; 325203838Smarius membar(StoreLoad); 326203838Smarius csa->csa_tick = rd(tick); 327207537Smarius if (cpu_impl == CPU_IMPL_SPARC64V || 328207537Smarius cpu_impl >= CPU_IMPL_ULTRASPARCIII) { 329203838Smarius while (csa->csa_state != CPU_STICKSYNC) 33091617Sjake ; 33191617Sjake membar(StoreLoad); 332203838Smarius csa->csa_stick = rdstick(); 333203838Smarius } 334203838Smarius while (csa->csa_state != CPU_INIT) 335203838Smarius ; 336203838Smarius csa->csa_tick = csa->csa_stick = 0; 337203838Smarius intr_restore(s); 33891617Sjake 339292771Smarius cpuid = cpuids++; 340203838Smarius cpuid_to_mid[cpuid] = mid; 341203838Smarius cpu_identify(csa->csa_ver, clock, cpuid); 34291617Sjake 343254025Sjeff va = kmem_malloc(kernel_arena, PCPU_PAGES * PAGE_SIZE, 344254025Sjeff M_WAITOK | M_ZERO); 345203838Smarius pc = (struct pcpu *)(va + (PCPU_PAGES * PAGE_SIZE)) - 1; 346203838Smarius pcpu_init(pc, cpuid, sizeof(*pc)); 347254025Sjeff dpcpu_init((void *)kmem_malloc(kernel_arena, DPCPU_SIZE, 348254025Sjeff M_WAITOK | M_ZERO), cpuid); 349203838Smarius pc->pc_addr = va; 350203838Smarius pc->pc_clock = clock; 351204152Smarius pc->pc_impl = cpu_impl; 352203838Smarius pc->pc_mid = mid; 353203838Smarius pc->pc_node = node; 35489051Sjake 355203838Smarius cache_init(pc); 356182689Smarius 357222813Sattilio CPU_SET(cpuid, &all_cpus); 358203838Smarius intr_add_cpu(cpuid); 35989051Sjake} 36089051Sjake 36189051Sjakevoid 36289051Sjakecpu_mp_announce(void) 36389051Sjake{ 364169796Smarius 36589051Sjake} 36689051Sjake 367169796Smariusstatic void 368286055Smariuscpu_mp_unleash(void *v __unused) 36989051Sjake{ 37091617Sjake volatile struct cpu_start_args *csa; 37191617Sjake struct pcpu *pc; 372169796Smarius register_t s; 37391617Sjake vm_offset_t va; 374113238Sjake vm_paddr_t pa; 375181701Smarius u_int ctx_inc; 37691617Sjake u_int ctx_min; 37791617Sjake int i; 37889051Sjake 37991783Sjake ctx_min = TLB_CTX_USER_MIN; 38091783Sjake ctx_inc = (TLB_CTX_USER_MAX - 1) / mp_ncpus; 38189051Sjake csa = &cpu_start_args; 38291783Sjake csa->csa_count = mp_ncpus; 383222531Snwhitehorn STAILQ_FOREACH(pc, &cpuhead, pc_allcpu) { 38491617Sjake pc->pc_tlb_ctx = ctx_min; 38591617Sjake pc->pc_tlb_ctx_min = ctx_min; 38691617Sjake pc->pc_tlb_ctx_max = ctx_min + ctx_inc; 38791617Sjake ctx_min += ctx_inc; 38889051Sjake 389169796Smarius if (pc->pc_cpuid == curcpu) 39091617Sjake continue; 39191617Sjake KASSERT(pc->pc_idlethread != NULL, 392169796Smarius ("%s: idlethread", __func__)); 393169796Smarius pc->pc_curthread = pc->pc_idlethread; 39491617Sjake pc->pc_curpcb = pc->pc_curthread->td_pcb; 39591617Sjake for (i = 0; i < PCPU_PAGES; i++) { 39691617Sjake va = pc->pc_addr + i * PAGE_SIZE; 39791617Sjake pa = pmap_kextract(va); 39891617Sjake if (pa == 0) 399169796Smarius panic("%s: pmap_kextract", __func__); 400102042Sjake csa->csa_ttes[i].tte_vpn = TV_VPN(va, TS_8K); 40191617Sjake csa->csa_ttes[i].tte_data = TD_V | TD_8K | TD_PA(pa) | 40291617Sjake TD_L | TD_CP | TD_CV | TD_P | TD_W; 40391617Sjake } 40491617Sjake csa->csa_state = 0; 40591783Sjake csa->csa_pcpu = pc->pc_addr; 40691617Sjake csa->csa_mid = pc->pc_mid; 40791617Sjake s = intr_disable(); 40891617Sjake while (csa->csa_state != CPU_BOOTSTRAP) 40991617Sjake ; 41091617Sjake intr_restore(s); 41191617Sjake } 41291783Sjake 41391783Sjake membar(StoreLoad); 414169796Smarius csa->csa_count = 0; 41591617Sjake} 41689051Sjake 41791617Sjakevoid 41891617Sjakecpu_mp_bootstrap(struct pcpu *pc) 41991617Sjake{ 42091617Sjake volatile struct cpu_start_args *csa; 42189051Sjake 42291617Sjake csa = &cpu_start_args; 423207248Smarius 424207248Smarius /* Do CPU-specific initialization. */ 425223719Smarius if (pc->pc_impl >= CPU_IMPL_ULTRASPARCIII) 426204152Smarius cheetah_init(pc->pc_impl); 427223719Smarius else if (pc->pc_impl == CPU_IMPL_SPARC64V) 428223719Smarius zeus_init(pc->pc_impl); 429223719Smarius 430207248Smarius /* 431207248Smarius * Enable the caches. Note that his may include applying workarounds. 432207248Smarius */ 433204152Smarius cache_enable(pc->pc_impl); 434207248Smarius 435213868Smarius /* 436213868Smarius * Clear (S)TICK timer(s) (including NPT) and ensure they are stopped. 437213868Smarius */ 438213868Smarius tick_clear(pc->pc_impl); 439213868Smarius tick_stop(pc->pc_impl); 440213868Smarius 441216803Smarius /* Set the kernel context. */ 442216803Smarius pmap_set_kctx(); 443207248Smarius 444216803Smarius /* Lock the kernel TSB in the TLB if necessary. */ 445216803Smarius if (tsb_kernel_ldd_phys == 0) 446216803Smarius pmap_map_tsb(); 447216803Smarius 448176994Smarius /* 449176994Smarius * Flush all non-locked TLB entries possibly left over by the 450176994Smarius * firmware. 451176994Smarius */ 452176994Smarius tlb_flush_nonlocked(); 453207248Smarius 454213868Smarius /* 455213868Smarius * Enable interrupts. 456213868Smarius * Note that the PIL we be lowered indirectly via sched_throw(NULL) 457213868Smarius * when fake spinlock held by the idle thread eventually is released. 458213868Smarius */ 459207248Smarius wrpr(pstate, 0, PSTATE_KERNEL); 460207248Smarius 46189051Sjake smp_cpus++; 462169796Smarius KASSERT(curthread != NULL, ("%s: curthread", __func__)); 463169796Smarius printf("SMP: AP CPU #%d Launched!\n", curcpu); 46489051Sjake 46591783Sjake csa->csa_count--; 46691783Sjake membar(StoreLoad); 46791617Sjake csa->csa_state = CPU_BOOTSTRAP; 46891783Sjake while (csa->csa_count != 0) 46991617Sjake ; 47089051Sjake 471286055Smarius if (smp_cpus == mp_ncpus) 472286055Smarius atomic_store_rel_int(&smp_started, 1); 473286055Smarius 474210601Smav /* Start per-CPU event timers. */ 475210601Smav cpu_initclocks_ap(); 476210601Smav 477182020Smarius /* Ok, now enter the scheduler. */ 478170303Sjeff sched_throw(NULL); 47989051Sjake} 48089051Sjake 48192205Sjakevoid 48292205Sjakecpu_mp_shutdown(void) 48392205Sjake{ 484222813Sattilio cpuset_t cpus; 48592205Sjake int i; 48692205Sjake 48792205Sjake critical_enter(); 488223346Smarius shutdown_cpus = all_cpus; 489223346Smarius CPU_CLR(PCPU_GET(cpuid), &shutdown_cpus); 490222813Sattilio cpus = shutdown_cpus; 491222813Sattilio 492222813Sattilio /* XXX: Stop all the CPUs which aren't already. */ 493222813Sattilio if (CPU_CMP(&stopped_cpus, &cpus)) { 494222813Sattilio 495223346Smarius /* cpus is just a flat "on" mask without curcpu. */ 496222813Sattilio CPU_NAND(&cpus, &stopped_cpus); 497222813Sattilio stop_cpus(cpus); 498222813Sattilio } 49992205Sjake i = 0; 500222813Sattilio while (!CPU_EMPTY(&shutdown_cpus)) { 50192205Sjake if (i++ > 100000) { 50292205Sjake printf("timeout shutting down CPUs.\n"); 50392205Sjake break; 50492205Sjake } 50592205Sjake } 50692205Sjake critical_exit(); 50792205Sjake} 50892205Sjake 50989051Sjakestatic void 510239864Smariuscpu_ipi_ast(struct trapframe *tf __unused) 51189051Sjake{ 512169796Smarius 51389051Sjake} 51489051Sjake 51589051Sjakestatic void 516239864Smariuscpu_ipi_stop(struct trapframe *tf __unused) 51789051Sjake{ 518223346Smarius u_int cpuid; 51992205Sjake 520169796Smarius CTR2(KTR_SMP, "%s: stopped %d", __func__, curcpu); 521222813Sattilio sched_pin(); 522169796Smarius savectx(&stoppcbs[curcpu]); 523223346Smarius cpuid = PCPU_GET(cpuid); 524223346Smarius CPU_SET_ATOMIC(cpuid, &stopped_cpus); 525223346Smarius while (!CPU_ISSET(cpuid, &started_cpus)) { 526223346Smarius if (CPU_ISSET(cpuid, &shutdown_cpus)) { 527223346Smarius CPU_CLR_ATOMIC(cpuid, &shutdown_cpus); 528183142Smarius (void)intr_disable(); 529183142Smarius for (;;) 530183142Smarius ; 53192205Sjake } 53292205Sjake } 533223346Smarius CPU_CLR_ATOMIC(cpuid, &started_cpus); 534223346Smarius CPU_CLR_ATOMIC(cpuid, &stopped_cpus); 535222813Sattilio sched_unpin(); 536169796Smarius CTR2(KTR_SMP, "%s: restarted %d", __func__, curcpu); 53789051Sjake} 53889051Sjake 539170846Smariusstatic void 540286055Smariuscpu_ipi_preempt(struct trapframe *tf __unused) 541178048Smarius{ 542178048Smarius 543178048Smarius sched_preempt(curthread); 544178048Smarius} 545178048Smarius 546178048Smariusstatic void 547210601Smavcpu_ipi_hardclock(struct trapframe *tf) 548210601Smav{ 549212541Smav struct trapframe *oldframe; 550212541Smav struct thread *td; 551210601Smav 552212541Smav critical_enter(); 553212541Smav td = curthread; 554212541Smav td->td_intr_nesting_level++; 555212541Smav oldframe = td->td_intr_frame; 556212541Smav td->td_intr_frame = tf; 557212541Smav hardclockintr(); 558212541Smav td->td_intr_frame = oldframe; 559212541Smav td->td_intr_nesting_level--; 560212541Smav critical_exit(); 561210601Smav} 562210601Smav 563210601Smavstatic void 564222813Sattiliospitfire_ipi_selected(cpuset_t cpus, u_long d0, u_long d1, u_long d2) 56589051Sjake{ 56689051Sjake u_int cpu; 56789051Sjake 568251703Sjeff while ((cpu = CPU_FFS(&cpus)) != 0) { 569222813Sattilio cpu--; 570222813Sattilio CPU_CLR(cpu, &cpus); 571211050Smarius spitfire_ipi_single(cpu, d0, d1, d2); 57289051Sjake } 57389051Sjake} 57489051Sjake 575169796Smariusstatic void 576211050Smariusspitfire_ipi_single(u_int cpu, u_long d0, u_long d1, u_long d2) 57789051Sjake{ 578169796Smarius register_t s; 579157240Smarius u_long ids; 580211050Smarius u_int mid; 58189051Sjake int i; 58289051Sjake 583286055Smarius mtx_assert(&ipi_mtx, MA_OWNED); 584211050Smarius KASSERT(cpu != curcpu, ("%s: CPU can't IPI itself", __func__)); 58589051Sjake KASSERT((ldxa(0, ASI_INTR_DISPATCH_STATUS) & IDR_BUSY) == 0, 586169796Smarius ("%s: outstanding dispatch", __func__)); 587286055Smarius 588211050Smarius mid = cpuid_to_mid[cpu]; 58989051Sjake for (i = 0; i < IPI_RETRIES; i++) { 59091783Sjake s = intr_disable(); 59189051Sjake stxa(AA_SDB_INTR_D0, ASI_SDB_INTR_W, d0); 59289051Sjake stxa(AA_SDB_INTR_D1, ASI_SDB_INTR_W, d1); 59389051Sjake stxa(AA_SDB_INTR_D2, ASI_SDB_INTR_W, d2); 594170846Smarius membar(Sync); 595169796Smarius stxa(AA_INTR_SEND | (mid << IDC_ITID_SHIFT), 596169796Smarius ASI_SDB_INTR_W, 0); 597157240Smarius /* 598157240Smarius * Workaround for SpitFire erratum #54; do a dummy read 599157240Smarius * from a SDB internal register before the MEMBAR #Sync 600157240Smarius * for the write to ASI_SDB_INTR_W (requiring another 601157240Smarius * MEMBAR #Sync in order to make sure the write has 602157240Smarius * occurred before the load). 603157240Smarius */ 60489051Sjake membar(Sync); 605157240Smarius (void)ldxa(AA_SDB_CNTL_HIGH, ASI_SDB_CONTROL_R); 606157240Smarius membar(Sync); 607169796Smarius while (((ids = ldxa(0, ASI_INTR_DISPATCH_STATUS)) & 608169796Smarius IDR_BUSY) != 0) 60989051Sjake ; 61091783Sjake intr_restore(s); 611170846Smarius if ((ids & (IDR_BUSY | IDR_NACK)) == 0) 61289051Sjake return; 61389051Sjake } 614190106Smarius if (kdb_active != 0 || panicstr != NULL) 615169796Smarius printf("%s: couldn't send IPI to module 0x%u\n", 616169796Smarius __func__, mid); 61792205Sjake else 618186395Smarius panic("%s: couldn't send IPI to module 0x%u", 619186395Smarius __func__, mid); 62089051Sjake} 62189051Sjake 622170846Smariusstatic void 623211050Smariuscheetah_ipi_single(u_int cpu, u_long d0, u_long d1, u_long d2) 624211050Smarius{ 625211050Smarius register_t s; 626211050Smarius u_long ids; 627211050Smarius u_int mid; 628211050Smarius int i; 629211050Smarius 630286055Smarius mtx_assert(&ipi_mtx, MA_OWNED); 631211050Smarius KASSERT(cpu != curcpu, ("%s: CPU can't IPI itself", __func__)); 632211050Smarius KASSERT((ldxa(0, ASI_INTR_DISPATCH_STATUS) & 633211050Smarius IDR_CHEETAH_ALL_BUSY) == 0, 634211050Smarius ("%s: outstanding dispatch", __func__)); 635286055Smarius 636211050Smarius mid = cpuid_to_mid[cpu]; 637211050Smarius for (i = 0; i < IPI_RETRIES; i++) { 638211050Smarius s = intr_disable(); 639211050Smarius stxa(AA_SDB_INTR_D0, ASI_SDB_INTR_W, d0); 640211050Smarius stxa(AA_SDB_INTR_D1, ASI_SDB_INTR_W, d1); 641211050Smarius stxa(AA_SDB_INTR_D2, ASI_SDB_INTR_W, d2); 642211050Smarius membar(Sync); 643211050Smarius stxa(AA_INTR_SEND | (mid << IDC_ITID_SHIFT), 644211050Smarius ASI_SDB_INTR_W, 0); 645211050Smarius membar(Sync); 646211050Smarius while (((ids = ldxa(0, ASI_INTR_DISPATCH_STATUS)) & 647211050Smarius IDR_BUSY) != 0) 648211050Smarius ; 649211050Smarius intr_restore(s); 650211050Smarius if ((ids & (IDR_BUSY | IDR_NACK)) == 0) 651211050Smarius return; 652211050Smarius } 653211050Smarius if (kdb_active != 0 || panicstr != NULL) 654211050Smarius printf("%s: couldn't send IPI to module 0x%u\n", 655211050Smarius __func__, mid); 656211050Smarius else 657211050Smarius panic("%s: couldn't send IPI to module 0x%u", 658211050Smarius __func__, mid); 659211050Smarius} 660211050Smarius 661211050Smariusstatic void 662222813Sattiliocheetah_ipi_selected(cpuset_t cpus, u_long d0, u_long d1, u_long d2) 663170846Smarius{ 664170846Smarius register_t s; 665170846Smarius u_long ids; 666170846Smarius u_int bnp; 667170846Smarius u_int cpu; 668170846Smarius int i; 669170846Smarius 670286055Smarius mtx_assert(&ipi_mtx, MA_OWNED); 671286055Smarius KASSERT(!CPU_EMPTY(&cpus), ("%s: no CPUs to IPI", __func__)); 672222813Sattilio KASSERT(!CPU_ISSET(curcpu, &cpus), ("%s: CPU can't IPI itself", 673222813Sattilio __func__)); 674170846Smarius KASSERT((ldxa(0, ASI_INTR_DISPATCH_STATUS) & 675170846Smarius IDR_CHEETAH_ALL_BUSY) == 0, 676170846Smarius ("%s: outstanding dispatch", __func__)); 677286055Smarius 678170846Smarius ids = 0; 679292771Smarius for (i = 0; i < IPI_RETRIES * smp_cpus; i++) { 680170846Smarius s = intr_disable(); 681170846Smarius stxa(AA_SDB_INTR_D0, ASI_SDB_INTR_W, d0); 682170846Smarius stxa(AA_SDB_INTR_D1, ASI_SDB_INTR_W, d1); 683170846Smarius stxa(AA_SDB_INTR_D2, ASI_SDB_INTR_W, d2); 684170846Smarius membar(Sync); 685170846Smarius bnp = 0; 686292771Smarius for (cpu = 0; cpu < smp_cpus; cpu++) { 687222813Sattilio if (CPU_ISSET(cpu, &cpus)) { 688211050Smarius stxa(AA_INTR_SEND | (cpuid_to_mid[cpu] << 689211050Smarius IDC_ITID_SHIFT) | bnp << IDC_BN_SHIFT, 690170846Smarius ASI_SDB_INTR_W, 0); 691170846Smarius membar(Sync); 692170846Smarius bnp++; 693223806Smarius if (bnp == IDR_CHEETAH_MAX_BN_PAIRS) 694223806Smarius break; 695170846Smarius } 696170846Smarius } 697170846Smarius while (((ids = ldxa(0, ASI_INTR_DISPATCH_STATUS)) & 698170846Smarius IDR_CHEETAH_ALL_BUSY) != 0) 699170846Smarius ; 700170846Smarius intr_restore(s); 701170846Smarius bnp = 0; 702292771Smarius for (cpu = 0; cpu < smp_cpus; cpu++) { 703222813Sattilio if (CPU_ISSET(cpu, &cpus)) { 704211050Smarius if ((ids & (IDR_NACK << (2 * bnp))) == 0) 705222813Sattilio CPU_CLR(cpu, &cpus); 706170846Smarius bnp++; 707170846Smarius } 708170846Smarius } 709222813Sattilio if (CPU_EMPTY(&cpus)) 710186395Smarius return; 711170846Smarius } 712190106Smarius if (kdb_active != 0 || panicstr != NULL) 713222813Sattilio printf("%s: couldn't send IPI (cpus=%s ids=0x%lu)\n", 714292771Smarius __func__, cpusetobj_strprint(ipi_pbuf, &cpus), ids); 715170846Smarius else 716222813Sattilio panic("%s: couldn't send IPI (cpus=%s ids=0x%lu)", 717292771Smarius __func__, cpusetobj_strprint(ipi_pbuf, &cpus), ids); 718170846Smarius} 719211050Smarius 720211050Smariusstatic void 721211050Smariusjalapeno_ipi_single(u_int cpu, u_long d0, u_long d1, u_long d2) 722211050Smarius{ 723211050Smarius register_t s; 724211050Smarius u_long ids; 725211050Smarius u_int busy, busynack, mid; 726211050Smarius int i; 727211050Smarius 728286055Smarius mtx_assert(&ipi_mtx, MA_OWNED); 729211050Smarius KASSERT(cpu != curcpu, ("%s: CPU can't IPI itself", __func__)); 730211050Smarius KASSERT((ldxa(0, ASI_INTR_DISPATCH_STATUS) & 731211050Smarius IDR_CHEETAH_ALL_BUSY) == 0, 732211050Smarius ("%s: outstanding dispatch", __func__)); 733286055Smarius 734211050Smarius mid = cpuid_to_mid[cpu]; 735211050Smarius busy = IDR_BUSY << (2 * mid); 736211050Smarius busynack = (IDR_BUSY | IDR_NACK) << (2 * mid); 737211050Smarius for (i = 0; i < IPI_RETRIES; i++) { 738211050Smarius s = intr_disable(); 739211050Smarius stxa(AA_SDB_INTR_D0, ASI_SDB_INTR_W, d0); 740211050Smarius stxa(AA_SDB_INTR_D1, ASI_SDB_INTR_W, d1); 741211050Smarius stxa(AA_SDB_INTR_D2, ASI_SDB_INTR_W, d2); 742211050Smarius membar(Sync); 743211050Smarius stxa(AA_INTR_SEND | (mid << IDC_ITID_SHIFT), 744211050Smarius ASI_SDB_INTR_W, 0); 745211050Smarius membar(Sync); 746211050Smarius while (((ids = ldxa(0, ASI_INTR_DISPATCH_STATUS)) & 747211050Smarius busy) != 0) 748211050Smarius ; 749211050Smarius intr_restore(s); 750211050Smarius if ((ids & busynack) == 0) 751211050Smarius return; 752211050Smarius } 753211050Smarius if (kdb_active != 0 || panicstr != NULL) 754211050Smarius printf("%s: couldn't send IPI to module 0x%u\n", 755211050Smarius __func__, mid); 756211050Smarius else 757211050Smarius panic("%s: couldn't send IPI to module 0x%u", 758211050Smarius __func__, mid); 759211050Smarius} 760211050Smarius 761211050Smariusstatic void 762222813Sattiliojalapeno_ipi_selected(cpuset_t cpus, u_long d0, u_long d1, u_long d2) 763211050Smarius{ 764211050Smarius register_t s; 765211050Smarius u_long ids; 766211050Smarius u_int cpu; 767211050Smarius int i; 768211050Smarius 769286055Smarius mtx_assert(&ipi_mtx, MA_OWNED); 770286055Smarius KASSERT(!CPU_EMPTY(&cpus), ("%s: no CPUs to IPI", __func__)); 771222813Sattilio KASSERT(!CPU_ISSET(curcpu, &cpus), ("%s: CPU can't IPI itself", 772222813Sattilio __func__)); 773211050Smarius KASSERT((ldxa(0, ASI_INTR_DISPATCH_STATUS) & 774211050Smarius IDR_CHEETAH_ALL_BUSY) == 0, 775211050Smarius ("%s: outstanding dispatch", __func__)); 776286055Smarius 777211050Smarius ids = 0; 778292771Smarius for (i = 0; i < IPI_RETRIES * smp_cpus; i++) { 779211050Smarius s = intr_disable(); 780211050Smarius stxa(AA_SDB_INTR_D0, ASI_SDB_INTR_W, d0); 781211050Smarius stxa(AA_SDB_INTR_D1, ASI_SDB_INTR_W, d1); 782211050Smarius stxa(AA_SDB_INTR_D2, ASI_SDB_INTR_W, d2); 783211050Smarius membar(Sync); 784292771Smarius for (cpu = 0; cpu < smp_cpus; cpu++) { 785222813Sattilio if (CPU_ISSET(cpu, &cpus)) { 786211050Smarius stxa(AA_INTR_SEND | (cpuid_to_mid[cpu] << 787211050Smarius IDC_ITID_SHIFT), ASI_SDB_INTR_W, 0); 788211050Smarius membar(Sync); 789211050Smarius } 790211050Smarius } 791211050Smarius while (((ids = ldxa(0, ASI_INTR_DISPATCH_STATUS)) & 792211050Smarius IDR_CHEETAH_ALL_BUSY) != 0) 793211050Smarius ; 794211050Smarius intr_restore(s); 795211050Smarius if ((ids & 796211050Smarius (IDR_CHEETAH_ALL_BUSY | IDR_CHEETAH_ALL_NACK)) == 0) 797211050Smarius return; 798292771Smarius for (cpu = 0; cpu < smp_cpus; cpu++) 799222813Sattilio if (CPU_ISSET(cpu, &cpus)) 800211050Smarius if ((ids & (IDR_NACK << 801211050Smarius (2 * cpuid_to_mid[cpu]))) == 0) 802222813Sattilio CPU_CLR(cpu, &cpus); 803211050Smarius } 804211050Smarius if (kdb_active != 0 || panicstr != NULL) 805222813Sattilio printf("%s: couldn't send IPI (cpus=%s ids=0x%lu)\n", 806292771Smarius __func__, cpusetobj_strprint(ipi_pbuf, &cpus), ids); 807211050Smarius else 808222813Sattilio panic("%s: couldn't send IPI (cpus=%s ids=0x%lu)", 809292771Smarius __func__, cpusetobj_strprint(ipi_pbuf, &cpus), ids); 810211050Smarius} 811