189051Sjake/*- 289051Sjake * Copyright (c) 1997 Berkeley Software Design, Inc. All rights reserved. 389051Sjake * 489051Sjake * Redistribution and use in source and binary forms, with or without 589051Sjake * modification, are permitted provided that the following conditions 689051Sjake * are met: 789051Sjake * 1. Redistributions of source code must retain the above copyright 889051Sjake * notice, this list of conditions and the following disclaimer. 989051Sjake * 2. Redistributions in binary form must reproduce the above copyright 1089051Sjake * notice, this list of conditions and the following disclaimer in the 1189051Sjake * documentation and/or other materials provided with the distribution. 1289051Sjake * 3. Berkeley Software Design Inc's name may not be used to endorse or 1389051Sjake * promote products derived from this software without specific prior 1489051Sjake * written permission. 1589051Sjake * 1689051Sjake * THIS SOFTWARE IS PROVIDED BY BERKELEY SOFTWARE DESIGN INC ``AS IS'' AND 1789051Sjake * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 1889051Sjake * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 1989051Sjake * ARE DISCLAIMED. IN NO EVENT SHALL BERKELEY SOFTWARE DESIGN INC BE LIABLE 2089051Sjake * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 2189051Sjake * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 2289051Sjake * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 2389051Sjake * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 2489051Sjake * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 2589051Sjake * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 2689051Sjake * SUCH DAMAGE. 2789051Sjake * 2889051Sjake * from BSDI: locore.s,v 1.36.2.15 1999/08/23 22:34:41 cp Exp 2989051Sjake */ 3089051Sjake/*- 3189051Sjake * Copyright (c) 2002 Jake Burkholder. 32211050Smarius * Copyright (c) 2007 - 2010 Marius Strobl <marius@FreeBSD.org> 3389051Sjake * All rights reserved. 3489051Sjake * 3589051Sjake * Redistribution and use in source and binary forms, with or without 3689051Sjake * modification, are permitted provided that the following conditions 3789051Sjake * are met: 3889051Sjake * 1. Redistributions of source code must retain the above copyright 3989051Sjake * notice, this list of conditions and the following disclaimer. 4089051Sjake * 2. Redistributions in binary form must reproduce the above copyright 4189051Sjake * notice, this list of conditions and the following disclaimer in the 4289051Sjake * documentation and/or other materials provided with the distribution. 4389051Sjake * 4489051Sjake * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 4589051Sjake * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 4689051Sjake * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 4789051Sjake * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 4889051Sjake * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 4989051Sjake * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 5089051Sjake * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 5189051Sjake * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 5289051Sjake * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 5389051Sjake * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 5489051Sjake * SUCH DAMAGE. 5589051Sjake */ 5689051Sjake 57145150Smarius#include <sys/cdefs.h> 58145150Smarius__FBSDID("$FreeBSD: releng/10.2/sys/sparc64/sparc64/mp_machdep.c 286060 2015-07-30 02:45:35Z marius $"); 59145150Smarius 6089051Sjake#include <sys/param.h> 6189051Sjake#include <sys/systm.h> 6289051Sjake#include <sys/lock.h> 63131950Smarcel#include <sys/kdb.h> 6489051Sjake#include <sys/kernel.h> 6589051Sjake#include <sys/ktr.h> 6689051Sjake#include <sys/mutex.h> 6789051Sjake#include <sys/pcpu.h> 6889051Sjake#include <sys/proc.h> 69170846Smarius#include <sys/sched.h> 7089051Sjake#include <sys/smp.h> 7189051Sjake 7289051Sjake#include <vm/vm.h> 7389051Sjake#include <vm/vm_param.h> 7489051Sjake#include <vm/pmap.h> 7589051Sjake#include <vm/vm_kern.h> 7689051Sjake#include <vm/vm_extern.h> 7789051Sjake#include <vm/vm_map.h> 7889051Sjake 7989051Sjake#include <dev/ofw/openfirm.h> 8089051Sjake 8189051Sjake#include <machine/asi.h> 8292205Sjake#include <machine/atomic.h> 83119696Smarcel#include <machine/bus.h> 84182730Smarius#include <machine/cpu.h> 85286060Smarius#include <machine/cpufunc.h> 8689051Sjake#include <machine/md_var.h> 8797511Sjake#include <machine/metadata.h> 8892205Sjake#include <machine/ofw_machdep.h> 89152022Sjhb#include <machine/pcb.h> 9089051Sjake#include <machine/smp.h> 9195132Sjake#include <machine/tick.h> 9291617Sjake#include <machine/tlb.h> 93216803Smarius#include <machine/tsb.h> 9489051Sjake#include <machine/tte.h> 95170846Smarius#include <machine/ver.h> 9689051Sjake 97183142Smarius#define SUNW_STARTCPU "SUNW,start-cpu" 98183142Smarius#define SUNW_STOPSELF "SUNW,stop-self" 99183142Smarius 10089051Sjakestatic ih_func_t cpu_ipi_ast; 101210601Smavstatic ih_func_t cpu_ipi_hardclock; 102178048Smariusstatic ih_func_t cpu_ipi_preempt; 10389051Sjakestatic ih_func_t cpu_ipi_stop; 10489051Sjake 10589051Sjake/* 10689051Sjake * Argument area used to pass data to non-boot processors as they start up. 107169796Smarius * This must be statically initialized with a known invalid CPU module ID, 108169796Smarius * since the other processors will use it before the boot CPU enters the 10989051Sjake * kernel. 11089051Sjake */ 111182730Smariusstruct cpu_start_args cpu_start_args = { 0, -1, -1, 0, 0, 0 }; 11297001Sjakestruct ipi_cache_args ipi_cache_args; 113211071Smariusstruct ipi_rd_args ipi_rd_args; 11491783Sjakestruct ipi_tlb_args ipi_tlb_args; 115152022Sjhbstruct pcb stoppcbs[MAXCPU]; 11689051Sjake 117286060Smariusstruct mtx ipi_mtx; 118286060Smarius 119170846Smariuscpu_ipi_selected_t *cpu_ipi_selected; 120211050Smariuscpu_ipi_single_t *cpu_ipi_single; 12191617Sjake 122170846Smariusstatic vm_offset_t mp_tramp; 123170846Smariusstatic u_int cpuid_to_mid[MAXCPU]; 124222813Sattiliostatic volatile cpuset_t shutdown_cpus; 12592205Sjake 126204152Smariusstatic void ap_count(phandle_t node, u_int mid, u_int cpu_impl); 127204152Smariusstatic void ap_start(phandle_t node, u_int mid, u_int cpu_impl); 128169796Smariusstatic void cpu_mp_unleash(void *v); 129203838Smariusstatic void foreach_ap(phandle_t node, void (*func)(phandle_t node, 130204152Smarius u_int mid, u_int cpu_impl)); 131169796Smariusstatic void sun4u_startcpu(phandle_t cpu, void *func, u_long arg); 132169796Smarius 133170846Smariusstatic cpu_ipi_selected_t cheetah_ipi_selected; 134211050Smariusstatic cpu_ipi_single_t cheetah_ipi_single; 135211050Smariusstatic cpu_ipi_selected_t jalapeno_ipi_selected; 136211050Smariusstatic cpu_ipi_single_t jalapeno_ipi_single; 137170846Smariusstatic cpu_ipi_selected_t spitfire_ipi_selected; 138211050Smariusstatic cpu_ipi_single_t spitfire_ipi_single; 139170846Smarius 14091617SjakeSYSINIT(cpu_mp_unleash, SI_SUB_SMP, SI_ORDER_FIRST, cpu_mp_unleash, NULL); 14191617Sjake 142170846Smariusvoid 143286060Smariusmp_init(void) 14491617Sjake{ 14591617Sjake struct tte *tp; 14691617Sjake int i; 14791617Sjake 148170846Smarius mp_tramp = (vm_offset_t)OF_claim(NULL, PAGE_SIZE, PAGE_SIZE); 149170846Smarius if (mp_tramp == (vm_offset_t)-1) 150169796Smarius panic("%s", __func__); 151170846Smarius bcopy(mp_tramp_code, (void *)mp_tramp, mp_tramp_code_len); 152170846Smarius *(vm_offset_t *)(mp_tramp + mp_tramp_tlb_slots) = kernel_tlb_slots; 153170846Smarius *(vm_offset_t *)(mp_tramp + mp_tramp_func) = (vm_offset_t)mp_startup; 154170846Smarius tp = (struct tte *)(mp_tramp + mp_tramp_code_len); 15597511Sjake for (i = 0; i < kernel_tlb_slots; i++) { 156102042Sjake tp[i].tte_vpn = TV_VPN(kernel_tlbs[i].te_va, TS_4M); 15797511Sjake tp[i].tte_data = TD_V | TD_4M | TD_PA(kernel_tlbs[i].te_pa) | 15897511Sjake TD_L | TD_CP | TD_CV | TD_P | TD_W; 15997511Sjake } 160170846Smarius for (i = 0; i < PAGE_SIZE; i += sizeof(vm_offset_t)) 161170846Smarius flush(mp_tramp + i); 16291617Sjake} 16391617Sjake 164203838Smariusstatic void 165204152Smariusforeach_ap(phandle_t node, void (*func)(phandle_t node, u_int mid, 166204152Smarius u_int cpu_impl)) 167203838Smarius{ 168203838Smarius char type[sizeof("cpu")]; 169203838Smarius phandle_t child; 170203838Smarius u_int cpuid; 171204152Smarius uint32_t cpu_impl; 172211050Smarius 173203838Smarius /* There's no need to traverse the whole OFW tree twice. */ 174203838Smarius if (mp_maxid > 0 && mp_ncpus >= mp_maxid + 1) 175203838Smarius return; 176203838Smarius 177203838Smarius for (; node != 0; node = OF_peer(node)) { 178203838Smarius child = OF_child(node); 179203838Smarius if (child > 0) 180203838Smarius foreach_ap(child, func); 181203838Smarius else { 182203838Smarius if (OF_getprop(node, "device_type", type, 183203838Smarius sizeof(type)) <= 0) 184203838Smarius continue; 185203838Smarius if (strcmp(type, "cpu") != 0) 186203838Smarius continue; 187204152Smarius if (OF_getprop(node, "implementation#", &cpu_impl, 188204152Smarius sizeof(cpu_impl)) <= 0) 189204152Smarius panic("%s: couldn't determine CPU " 190204152Smarius "implementation", __func__); 191204152Smarius if (OF_getprop(node, cpu_cpuid_prop(cpu_impl), &cpuid, 192211050Smarius sizeof(cpuid)) <= 0) 193204152Smarius panic("%s: couldn't determine CPU module ID", 194204152Smarius __func__); 195203838Smarius if (cpuid == PCPU_GET(mid)) 196203838Smarius continue; 197204152Smarius (*func)(node, cpuid, cpu_impl); 198203838Smarius } 199203838Smarius } 200203838Smarius} 201203838Smarius 20289051Sjake/* 203169796Smarius * Probe for other CPUs. 20489051Sjake */ 205122947Sjhbvoid 206286060Smariuscpu_mp_setmaxid(void) 20789051Sjake{ 20889051Sjake 209222813Sattilio CPU_SETOF(curcpu, &all_cpus); 21089051Sjake mp_ncpus = 1; 211203838Smarius mp_maxid = 0; 21289051Sjake 213203838Smarius foreach_ap(OF_child(OF_peer(0)), ap_count); 21489051Sjake} 21589051Sjake 216203838Smariusstatic void 217204152Smariusap_count(phandle_t node __unused, u_int mid __unused, u_int cpu_impl __unused) 218203838Smarius{ 219203838Smarius 220203838Smarius mp_maxid++; 221203838Smarius} 222203838Smarius 223122947Sjhbint 224122947Sjhbcpu_mp_probe(void) 225122947Sjhb{ 226122947Sjhb 227123126Sjhb return (mp_maxid > 0); 228122947Sjhb} 229122947Sjhb 230176734Sjeffstruct cpu_group * 231176734Sjeffcpu_topo(void) 232176734Sjeff{ 233176734Sjeff 234176994Smarius return (smp_topo_none()); 235176734Sjeff} 236176734Sjeff 23791617Sjakestatic void 23891617Sjakesun4u_startcpu(phandle_t cpu, void *func, u_long arg) 23991617Sjake{ 24091617Sjake static struct { 24191617Sjake cell_t name; 24291617Sjake cell_t nargs; 24391617Sjake cell_t nreturns; 24491617Sjake cell_t cpu; 24591617Sjake cell_t func; 24691617Sjake cell_t arg; 24791617Sjake } args = { 248183142Smarius (cell_t)SUNW_STARTCPU, 24991617Sjake 3, 25091617Sjake }; 25191617Sjake 25291617Sjake args.cpu = cpu; 25391617Sjake args.func = (cell_t)func; 25491617Sjake args.arg = (cell_t)arg; 255186347Snwhitehorn ofw_entry(&args); 25691617Sjake} 25791617Sjake 25889051Sjake/* 25989051Sjake * Fire up any non-boot processors. 26089051Sjake */ 26189051Sjakevoid 26289051Sjakecpu_mp_start(void) 26389051Sjake{ 264286060Smarius u_int cpu_impl, isjbus; 26589051Sjake 266286060Smarius mtx_init(&ipi_mtx, "ipi", NULL, MTX_SPIN); 267286060Smarius 268286060Smarius isjbus = 0; 269286060Smarius cpu_impl = PCPU_GET(impl); 270286060Smarius if (cpu_impl == CPU_IMPL_ULTRASPARCIIIi || 271286060Smarius cpu_impl == CPU_IMPL_ULTRASPARCIIIip) { 272286060Smarius isjbus = 1; 273286060Smarius cpu_ipi_selected = jalapeno_ipi_selected; 274286060Smarius cpu_ipi_single = jalapeno_ipi_single; 275286060Smarius } else if (cpu_impl == CPU_IMPL_SPARC64V || 276286060Smarius cpu_impl >= CPU_IMPL_ULTRASPARCIII) { 277286060Smarius cpu_ipi_selected = cheetah_ipi_selected; 278286060Smarius cpu_ipi_single = cheetah_ipi_single; 279286060Smarius } else { 280286060Smarius cpu_ipi_selected = spitfire_ipi_selected; 281286060Smarius cpu_ipi_single = spitfire_ipi_single; 282286060Smarius } 283286060Smarius 28489051Sjake intr_setup(PIL_AST, cpu_ipi_ast, -1, NULL, NULL); 28589051Sjake intr_setup(PIL_RENDEZVOUS, (ih_func_t *)smp_rendezvous_action, 28689051Sjake -1, NULL, NULL); 28789051Sjake intr_setup(PIL_STOP, cpu_ipi_stop, -1, NULL, NULL); 288178048Smarius intr_setup(PIL_PREEMPT, cpu_ipi_preempt, -1, NULL, NULL); 289210601Smav intr_setup(PIL_HARDCLOCK, cpu_ipi_hardclock, -1, NULL, NULL); 29089051Sjake 291169796Smarius cpuid_to_mid[curcpu] = PCPU_GET(mid); 292157240Smarius 293203838Smarius foreach_ap(OF_child(OF_peer(0)), ap_start); 294203838Smarius KASSERT(!isjbus || mp_ncpus <= IDR_JALAPENO_MAX_BN_PAIRS, 295203838Smarius ("%s: can only IPI a maximum of %d JBus-CPUs", 296203838Smarius __func__, IDR_JALAPENO_MAX_BN_PAIRS)); 297203838Smarius} 298203838Smarius 299203838Smariusstatic void 300204152Smariusap_start(phandle_t node, u_int mid, u_int cpu_impl) 301203838Smarius{ 302203838Smarius volatile struct cpu_start_args *csa; 303203838Smarius struct pcpu *pc; 304203838Smarius register_t s; 305203838Smarius vm_offset_t va; 306203838Smarius u_int cpuid; 307204152Smarius uint32_t clock; 308203838Smarius 309203838Smarius if (mp_ncpus > MAXCPU) 310203838Smarius return; 311203838Smarius 312203838Smarius if (OF_getprop(node, "clock-frequency", &clock, sizeof(clock)) <= 0) 313204152Smarius panic("%s: couldn't determine CPU frequency", __func__); 314203838Smarius if (clock != PCPU_GET(clock)) 315214071Smarius tick_et_use_stick = 1; 316203838Smarius 31789051Sjake csa = &cpu_start_args; 318203838Smarius csa->csa_state = 0; 319203838Smarius sun4u_startcpu(node, (void *)mp_tramp, 0); 320203838Smarius s = intr_disable(); 321203838Smarius while (csa->csa_state != CPU_TICKSYNC) 322203838Smarius ; 323203838Smarius membar(StoreLoad); 324203838Smarius csa->csa_tick = rd(tick); 325207537Smarius if (cpu_impl == CPU_IMPL_SPARC64V || 326207537Smarius cpu_impl >= CPU_IMPL_ULTRASPARCIII) { 327203838Smarius while (csa->csa_state != CPU_STICKSYNC) 32891617Sjake ; 32991617Sjake membar(StoreLoad); 330203838Smarius csa->csa_stick = rdstick(); 331203838Smarius } 332203838Smarius while (csa->csa_state != CPU_INIT) 333203838Smarius ; 334203838Smarius csa->csa_tick = csa->csa_stick = 0; 335203838Smarius intr_restore(s); 33691617Sjake 337203838Smarius cpuid = mp_ncpus++; 338203838Smarius cpuid_to_mid[cpuid] = mid; 339203838Smarius cpu_identify(csa->csa_ver, clock, cpuid); 34091617Sjake 341254025Sjeff va = kmem_malloc(kernel_arena, PCPU_PAGES * PAGE_SIZE, 342254025Sjeff M_WAITOK | M_ZERO); 343203838Smarius pc = (struct pcpu *)(va + (PCPU_PAGES * PAGE_SIZE)) - 1; 344203838Smarius pcpu_init(pc, cpuid, sizeof(*pc)); 345254025Sjeff dpcpu_init((void *)kmem_malloc(kernel_arena, DPCPU_SIZE, 346254025Sjeff M_WAITOK | M_ZERO), cpuid); 347203838Smarius pc->pc_addr = va; 348203838Smarius pc->pc_clock = clock; 349204152Smarius pc->pc_impl = cpu_impl; 350203838Smarius pc->pc_mid = mid; 351203838Smarius pc->pc_node = node; 35289051Sjake 353203838Smarius cache_init(pc); 354182689Smarius 355222813Sattilio CPU_SET(cpuid, &all_cpus); 356203838Smarius intr_add_cpu(cpuid); 35789051Sjake} 35889051Sjake 35989051Sjakevoid 36089051Sjakecpu_mp_announce(void) 36189051Sjake{ 362169796Smarius 36389051Sjake} 36489051Sjake 365169796Smariusstatic void 366286060Smariuscpu_mp_unleash(void *v __unused) 36789051Sjake{ 36891617Sjake volatile struct cpu_start_args *csa; 36991617Sjake struct pcpu *pc; 370169796Smarius register_t s; 37191617Sjake vm_offset_t va; 372113238Sjake vm_paddr_t pa; 373181701Smarius u_int ctx_inc; 37491617Sjake u_int ctx_min; 37591617Sjake int i; 37689051Sjake 37791783Sjake ctx_min = TLB_CTX_USER_MIN; 37891783Sjake ctx_inc = (TLB_CTX_USER_MAX - 1) / mp_ncpus; 37989051Sjake csa = &cpu_start_args; 38091783Sjake csa->csa_count = mp_ncpus; 381222531Snwhitehorn STAILQ_FOREACH(pc, &cpuhead, pc_allcpu) { 38291617Sjake pc->pc_tlb_ctx = ctx_min; 38391617Sjake pc->pc_tlb_ctx_min = ctx_min; 38491617Sjake pc->pc_tlb_ctx_max = ctx_min + ctx_inc; 38591617Sjake ctx_min += ctx_inc; 38689051Sjake 387169796Smarius if (pc->pc_cpuid == curcpu) 38891617Sjake continue; 38991617Sjake KASSERT(pc->pc_idlethread != NULL, 390169796Smarius ("%s: idlethread", __func__)); 391169796Smarius pc->pc_curthread = pc->pc_idlethread; 39291617Sjake pc->pc_curpcb = pc->pc_curthread->td_pcb; 39391617Sjake for (i = 0; i < PCPU_PAGES; i++) { 39491617Sjake va = pc->pc_addr + i * PAGE_SIZE; 39591617Sjake pa = pmap_kextract(va); 39691617Sjake if (pa == 0) 397169796Smarius panic("%s: pmap_kextract", __func__); 398102042Sjake csa->csa_ttes[i].tte_vpn = TV_VPN(va, TS_8K); 39991617Sjake csa->csa_ttes[i].tte_data = TD_V | TD_8K | TD_PA(pa) | 40091617Sjake TD_L | TD_CP | TD_CV | TD_P | TD_W; 40191617Sjake } 40291617Sjake csa->csa_state = 0; 40391783Sjake csa->csa_pcpu = pc->pc_addr; 40491617Sjake csa->csa_mid = pc->pc_mid; 40591617Sjake s = intr_disable(); 40691617Sjake while (csa->csa_state != CPU_BOOTSTRAP) 40791617Sjake ; 40891617Sjake intr_restore(s); 40991617Sjake } 41091783Sjake 41191783Sjake membar(StoreLoad); 412169796Smarius csa->csa_count = 0; 41391617Sjake} 41489051Sjake 41591617Sjakevoid 41691617Sjakecpu_mp_bootstrap(struct pcpu *pc) 41791617Sjake{ 41891617Sjake volatile struct cpu_start_args *csa; 41989051Sjake 42091617Sjake csa = &cpu_start_args; 421207248Smarius 422207248Smarius /* Do CPU-specific initialization. */ 423223719Smarius if (pc->pc_impl >= CPU_IMPL_ULTRASPARCIII) 424204152Smarius cheetah_init(pc->pc_impl); 425223719Smarius else if (pc->pc_impl == CPU_IMPL_SPARC64V) 426223719Smarius zeus_init(pc->pc_impl); 427223719Smarius 428207248Smarius /* 429207248Smarius * Enable the caches. Note that his may include applying workarounds. 430207248Smarius */ 431204152Smarius cache_enable(pc->pc_impl); 432207248Smarius 433213868Smarius /* 434213868Smarius * Clear (S)TICK timer(s) (including NPT) and ensure they are stopped. 435213868Smarius */ 436213868Smarius tick_clear(pc->pc_impl); 437213868Smarius tick_stop(pc->pc_impl); 438213868Smarius 439216803Smarius /* Set the kernel context. */ 440216803Smarius pmap_set_kctx(); 441207248Smarius 442216803Smarius /* Lock the kernel TSB in the TLB if necessary. */ 443216803Smarius if (tsb_kernel_ldd_phys == 0) 444216803Smarius pmap_map_tsb(); 445216803Smarius 446176994Smarius /* 447176994Smarius * Flush all non-locked TLB entries possibly left over by the 448176994Smarius * firmware. 449176994Smarius */ 450176994Smarius tlb_flush_nonlocked(); 451207248Smarius 452213868Smarius /* 453213868Smarius * Enable interrupts. 454213868Smarius * Note that the PIL we be lowered indirectly via sched_throw(NULL) 455213868Smarius * when fake spinlock held by the idle thread eventually is released. 456213868Smarius */ 457207248Smarius wrpr(pstate, 0, PSTATE_KERNEL); 458207248Smarius 45989051Sjake smp_cpus++; 460169796Smarius KASSERT(curthread != NULL, ("%s: curthread", __func__)); 461169796Smarius printf("SMP: AP CPU #%d Launched!\n", curcpu); 46289051Sjake 46391783Sjake csa->csa_count--; 46491783Sjake membar(StoreLoad); 46591617Sjake csa->csa_state = CPU_BOOTSTRAP; 46691783Sjake while (csa->csa_count != 0) 46791617Sjake ; 46889051Sjake 469286060Smarius if (smp_cpus == mp_ncpus) 470286060Smarius atomic_store_rel_int(&smp_started, 1); 471286060Smarius 472210601Smav /* Start per-CPU event timers. */ 473210601Smav cpu_initclocks_ap(); 474210601Smav 475182020Smarius /* Ok, now enter the scheduler. */ 476170303Sjeff sched_throw(NULL); 47789051Sjake} 47889051Sjake 47992205Sjakevoid 48092205Sjakecpu_mp_shutdown(void) 48192205Sjake{ 482222813Sattilio cpuset_t cpus; 48392205Sjake int i; 48492205Sjake 48592205Sjake critical_enter(); 486223346Smarius shutdown_cpus = all_cpus; 487223346Smarius CPU_CLR(PCPU_GET(cpuid), &shutdown_cpus); 488222813Sattilio cpus = shutdown_cpus; 489222813Sattilio 490222813Sattilio /* XXX: Stop all the CPUs which aren't already. */ 491222813Sattilio if (CPU_CMP(&stopped_cpus, &cpus)) { 492222813Sattilio 493223346Smarius /* cpus is just a flat "on" mask without curcpu. */ 494222813Sattilio CPU_NAND(&cpus, &stopped_cpus); 495222813Sattilio stop_cpus(cpus); 496222813Sattilio } 49792205Sjake i = 0; 498222813Sattilio while (!CPU_EMPTY(&shutdown_cpus)) { 49992205Sjake if (i++ > 100000) { 50092205Sjake printf("timeout shutting down CPUs.\n"); 50192205Sjake break; 50292205Sjake } 50392205Sjake } 50492205Sjake critical_exit(); 50592205Sjake} 50692205Sjake 50789051Sjakestatic void 508239864Smariuscpu_ipi_ast(struct trapframe *tf __unused) 50989051Sjake{ 510169796Smarius 51189051Sjake} 51289051Sjake 51389051Sjakestatic void 514239864Smariuscpu_ipi_stop(struct trapframe *tf __unused) 51589051Sjake{ 516223346Smarius u_int cpuid; 51792205Sjake 518169796Smarius CTR2(KTR_SMP, "%s: stopped %d", __func__, curcpu); 519222813Sattilio sched_pin(); 520169796Smarius savectx(&stoppcbs[curcpu]); 521223346Smarius cpuid = PCPU_GET(cpuid); 522223346Smarius CPU_SET_ATOMIC(cpuid, &stopped_cpus); 523223346Smarius while (!CPU_ISSET(cpuid, &started_cpus)) { 524223346Smarius if (CPU_ISSET(cpuid, &shutdown_cpus)) { 525223346Smarius CPU_CLR_ATOMIC(cpuid, &shutdown_cpus); 526183142Smarius (void)intr_disable(); 527183142Smarius for (;;) 528183142Smarius ; 52992205Sjake } 53092205Sjake } 531223346Smarius CPU_CLR_ATOMIC(cpuid, &started_cpus); 532223346Smarius CPU_CLR_ATOMIC(cpuid, &stopped_cpus); 533222813Sattilio sched_unpin(); 534169796Smarius CTR2(KTR_SMP, "%s: restarted %d", __func__, curcpu); 53589051Sjake} 53689051Sjake 537170846Smariusstatic void 538286060Smariuscpu_ipi_preempt(struct trapframe *tf __unused) 539178048Smarius{ 540178048Smarius 541178048Smarius sched_preempt(curthread); 542178048Smarius} 543178048Smarius 544178048Smariusstatic void 545210601Smavcpu_ipi_hardclock(struct trapframe *tf) 546210601Smav{ 547212541Smav struct trapframe *oldframe; 548212541Smav struct thread *td; 549210601Smav 550212541Smav critical_enter(); 551212541Smav td = curthread; 552212541Smav td->td_intr_nesting_level++; 553212541Smav oldframe = td->td_intr_frame; 554212541Smav td->td_intr_frame = tf; 555212541Smav hardclockintr(); 556212541Smav td->td_intr_frame = oldframe; 557212541Smav td->td_intr_nesting_level--; 558212541Smav critical_exit(); 559210601Smav} 560210601Smav 561210601Smavstatic void 562222813Sattiliospitfire_ipi_selected(cpuset_t cpus, u_long d0, u_long d1, u_long d2) 56389051Sjake{ 56489051Sjake u_int cpu; 56589051Sjake 566251703Sjeff while ((cpu = CPU_FFS(&cpus)) != 0) { 567222813Sattilio cpu--; 568222813Sattilio CPU_CLR(cpu, &cpus); 569211050Smarius spitfire_ipi_single(cpu, d0, d1, d2); 57089051Sjake } 57189051Sjake} 57289051Sjake 573169796Smariusstatic void 574211050Smariusspitfire_ipi_single(u_int cpu, u_long d0, u_long d1, u_long d2) 57589051Sjake{ 576169796Smarius register_t s; 577157240Smarius u_long ids; 578211050Smarius u_int mid; 57989051Sjake int i; 58089051Sjake 581286060Smarius mtx_assert(&ipi_mtx, MA_OWNED); 582211050Smarius KASSERT(cpu != curcpu, ("%s: CPU can't IPI itself", __func__)); 58389051Sjake KASSERT((ldxa(0, ASI_INTR_DISPATCH_STATUS) & IDR_BUSY) == 0, 584169796Smarius ("%s: outstanding dispatch", __func__)); 585286060Smarius 586211050Smarius mid = cpuid_to_mid[cpu]; 58789051Sjake for (i = 0; i < IPI_RETRIES; i++) { 58891783Sjake s = intr_disable(); 58989051Sjake stxa(AA_SDB_INTR_D0, ASI_SDB_INTR_W, d0); 59089051Sjake stxa(AA_SDB_INTR_D1, ASI_SDB_INTR_W, d1); 59189051Sjake stxa(AA_SDB_INTR_D2, ASI_SDB_INTR_W, d2); 592170846Smarius membar(Sync); 593169796Smarius stxa(AA_INTR_SEND | (mid << IDC_ITID_SHIFT), 594169796Smarius ASI_SDB_INTR_W, 0); 595157240Smarius /* 596157240Smarius * Workaround for SpitFire erratum #54; do a dummy read 597157240Smarius * from a SDB internal register before the MEMBAR #Sync 598157240Smarius * for the write to ASI_SDB_INTR_W (requiring another 599157240Smarius * MEMBAR #Sync in order to make sure the write has 600157240Smarius * occurred before the load). 601157240Smarius */ 60289051Sjake membar(Sync); 603157240Smarius (void)ldxa(AA_SDB_CNTL_HIGH, ASI_SDB_CONTROL_R); 604157240Smarius membar(Sync); 605169796Smarius while (((ids = ldxa(0, ASI_INTR_DISPATCH_STATUS)) & 606169796Smarius IDR_BUSY) != 0) 60789051Sjake ; 60891783Sjake intr_restore(s); 609170846Smarius if ((ids & (IDR_BUSY | IDR_NACK)) == 0) 61089051Sjake return; 61189051Sjake } 612190106Smarius if (kdb_active != 0 || panicstr != NULL) 613169796Smarius printf("%s: couldn't send IPI to module 0x%u\n", 614169796Smarius __func__, mid); 61592205Sjake else 616186395Smarius panic("%s: couldn't send IPI to module 0x%u", 617186395Smarius __func__, mid); 61889051Sjake} 61989051Sjake 620170846Smariusstatic void 621211050Smariuscheetah_ipi_single(u_int cpu, u_long d0, u_long d1, u_long d2) 622211050Smarius{ 623211050Smarius register_t s; 624211050Smarius u_long ids; 625211050Smarius u_int mid; 626211050Smarius int i; 627211050Smarius 628286060Smarius mtx_assert(&ipi_mtx, MA_OWNED); 629211050Smarius KASSERT(cpu != curcpu, ("%s: CPU can't IPI itself", __func__)); 630211050Smarius KASSERT((ldxa(0, ASI_INTR_DISPATCH_STATUS) & 631211050Smarius IDR_CHEETAH_ALL_BUSY) == 0, 632211050Smarius ("%s: outstanding dispatch", __func__)); 633286060Smarius 634211050Smarius mid = cpuid_to_mid[cpu]; 635211050Smarius for (i = 0; i < IPI_RETRIES; i++) { 636211050Smarius s = intr_disable(); 637211050Smarius stxa(AA_SDB_INTR_D0, ASI_SDB_INTR_W, d0); 638211050Smarius stxa(AA_SDB_INTR_D1, ASI_SDB_INTR_W, d1); 639211050Smarius stxa(AA_SDB_INTR_D2, ASI_SDB_INTR_W, d2); 640211050Smarius membar(Sync); 641211050Smarius stxa(AA_INTR_SEND | (mid << IDC_ITID_SHIFT), 642211050Smarius ASI_SDB_INTR_W, 0); 643211050Smarius membar(Sync); 644211050Smarius while (((ids = ldxa(0, ASI_INTR_DISPATCH_STATUS)) & 645211050Smarius IDR_BUSY) != 0) 646211050Smarius ; 647211050Smarius intr_restore(s); 648211050Smarius if ((ids & (IDR_BUSY | IDR_NACK)) == 0) 649211050Smarius return; 650211050Smarius } 651211050Smarius if (kdb_active != 0 || panicstr != NULL) 652211050Smarius printf("%s: couldn't send IPI to module 0x%u\n", 653211050Smarius __func__, mid); 654211050Smarius else 655211050Smarius panic("%s: couldn't send IPI to module 0x%u", 656211050Smarius __func__, mid); 657211050Smarius} 658211050Smarius 659211050Smariusstatic void 660222813Sattiliocheetah_ipi_selected(cpuset_t cpus, u_long d0, u_long d1, u_long d2) 661170846Smarius{ 662222813Sattilio char pbuf[CPUSETBUFSIZ]; 663170846Smarius register_t s; 664170846Smarius u_long ids; 665170846Smarius u_int bnp; 666170846Smarius u_int cpu; 667170846Smarius int i; 668170846Smarius 669286060Smarius mtx_assert(&ipi_mtx, MA_OWNED); 670286060Smarius KASSERT(!CPU_EMPTY(&cpus), ("%s: no CPUs to IPI", __func__)); 671222813Sattilio KASSERT(!CPU_ISSET(curcpu, &cpus), ("%s: CPU can't IPI itself", 672222813Sattilio __func__)); 673170846Smarius KASSERT((ldxa(0, ASI_INTR_DISPATCH_STATUS) & 674170846Smarius IDR_CHEETAH_ALL_BUSY) == 0, 675170846Smarius ("%s: outstanding dispatch", __func__)); 676286060Smarius 677170846Smarius ids = 0; 678170846Smarius for (i = 0; i < IPI_RETRIES * mp_ncpus; i++) { 679170846Smarius s = intr_disable(); 680170846Smarius stxa(AA_SDB_INTR_D0, ASI_SDB_INTR_W, d0); 681170846Smarius stxa(AA_SDB_INTR_D1, ASI_SDB_INTR_W, d1); 682170846Smarius stxa(AA_SDB_INTR_D2, ASI_SDB_INTR_W, d2); 683170846Smarius membar(Sync); 684170846Smarius bnp = 0; 685170846Smarius for (cpu = 0; cpu < mp_ncpus; cpu++) { 686222813Sattilio if (CPU_ISSET(cpu, &cpus)) { 687211050Smarius stxa(AA_INTR_SEND | (cpuid_to_mid[cpu] << 688211050Smarius IDC_ITID_SHIFT) | bnp << IDC_BN_SHIFT, 689170846Smarius ASI_SDB_INTR_W, 0); 690170846Smarius membar(Sync); 691170846Smarius bnp++; 692223806Smarius if (bnp == IDR_CHEETAH_MAX_BN_PAIRS) 693223806Smarius break; 694170846Smarius } 695170846Smarius } 696170846Smarius while (((ids = ldxa(0, ASI_INTR_DISPATCH_STATUS)) & 697170846Smarius IDR_CHEETAH_ALL_BUSY) != 0) 698170846Smarius ; 699170846Smarius intr_restore(s); 700170846Smarius bnp = 0; 701170846Smarius for (cpu = 0; cpu < mp_ncpus; cpu++) { 702222813Sattilio if (CPU_ISSET(cpu, &cpus)) { 703211050Smarius if ((ids & (IDR_NACK << (2 * bnp))) == 0) 704222813Sattilio CPU_CLR(cpu, &cpus); 705170846Smarius bnp++; 706170846Smarius } 707170846Smarius } 708222813Sattilio if (CPU_EMPTY(&cpus)) 709186395Smarius return; 710170846Smarius } 711190106Smarius if (kdb_active != 0 || panicstr != NULL) 712222813Sattilio printf("%s: couldn't send IPI (cpus=%s ids=0x%lu)\n", 713222813Sattilio __func__, cpusetobj_strprint(pbuf, &cpus), ids); 714170846Smarius else 715222813Sattilio panic("%s: couldn't send IPI (cpus=%s ids=0x%lu)", 716222813Sattilio __func__, cpusetobj_strprint(pbuf, &cpus), ids); 717170846Smarius} 718211050Smarius 719211050Smariusstatic void 720211050Smariusjalapeno_ipi_single(u_int cpu, u_long d0, u_long d1, u_long d2) 721211050Smarius{ 722211050Smarius register_t s; 723211050Smarius u_long ids; 724211050Smarius u_int busy, busynack, mid; 725211050Smarius int i; 726211050Smarius 727286060Smarius mtx_assert(&ipi_mtx, MA_OWNED); 728211050Smarius KASSERT(cpu != curcpu, ("%s: CPU can't IPI itself", __func__)); 729211050Smarius KASSERT((ldxa(0, ASI_INTR_DISPATCH_STATUS) & 730211050Smarius IDR_CHEETAH_ALL_BUSY) == 0, 731211050Smarius ("%s: outstanding dispatch", __func__)); 732286060Smarius 733211050Smarius mid = cpuid_to_mid[cpu]; 734211050Smarius busy = IDR_BUSY << (2 * mid); 735211050Smarius busynack = (IDR_BUSY | IDR_NACK) << (2 * mid); 736211050Smarius for (i = 0; i < IPI_RETRIES; i++) { 737211050Smarius s = intr_disable(); 738211050Smarius stxa(AA_SDB_INTR_D0, ASI_SDB_INTR_W, d0); 739211050Smarius stxa(AA_SDB_INTR_D1, ASI_SDB_INTR_W, d1); 740211050Smarius stxa(AA_SDB_INTR_D2, ASI_SDB_INTR_W, d2); 741211050Smarius membar(Sync); 742211050Smarius stxa(AA_INTR_SEND | (mid << IDC_ITID_SHIFT), 743211050Smarius ASI_SDB_INTR_W, 0); 744211050Smarius membar(Sync); 745211050Smarius while (((ids = ldxa(0, ASI_INTR_DISPATCH_STATUS)) & 746211050Smarius busy) != 0) 747211050Smarius ; 748211050Smarius intr_restore(s); 749211050Smarius if ((ids & busynack) == 0) 750211050Smarius return; 751211050Smarius } 752211050Smarius if (kdb_active != 0 || panicstr != NULL) 753211050Smarius printf("%s: couldn't send IPI to module 0x%u\n", 754211050Smarius __func__, mid); 755211050Smarius else 756211050Smarius panic("%s: couldn't send IPI to module 0x%u", 757211050Smarius __func__, mid); 758211050Smarius} 759211050Smarius 760211050Smariusstatic void 761222813Sattiliojalapeno_ipi_selected(cpuset_t cpus, u_long d0, u_long d1, u_long d2) 762211050Smarius{ 763222813Sattilio char pbuf[CPUSETBUFSIZ]; 764211050Smarius register_t s; 765211050Smarius u_long ids; 766211050Smarius u_int cpu; 767211050Smarius int i; 768211050Smarius 769286060Smarius mtx_assert(&ipi_mtx, MA_OWNED); 770286060Smarius KASSERT(!CPU_EMPTY(&cpus), ("%s: no CPUs to IPI", __func__)); 771222813Sattilio KASSERT(!CPU_ISSET(curcpu, &cpus), ("%s: CPU can't IPI itself", 772222813Sattilio __func__)); 773211050Smarius KASSERT((ldxa(0, ASI_INTR_DISPATCH_STATUS) & 774211050Smarius IDR_CHEETAH_ALL_BUSY) == 0, 775211050Smarius ("%s: outstanding dispatch", __func__)); 776286060Smarius 777211050Smarius ids = 0; 778211050Smarius for (i = 0; i < IPI_RETRIES * mp_ncpus; i++) { 779211050Smarius s = intr_disable(); 780211050Smarius stxa(AA_SDB_INTR_D0, ASI_SDB_INTR_W, d0); 781211050Smarius stxa(AA_SDB_INTR_D1, ASI_SDB_INTR_W, d1); 782211050Smarius stxa(AA_SDB_INTR_D2, ASI_SDB_INTR_W, d2); 783211050Smarius membar(Sync); 784211050Smarius for (cpu = 0; cpu < mp_ncpus; cpu++) { 785222813Sattilio if (CPU_ISSET(cpu, &cpus)) { 786211050Smarius stxa(AA_INTR_SEND | (cpuid_to_mid[cpu] << 787211050Smarius IDC_ITID_SHIFT), ASI_SDB_INTR_W, 0); 788211050Smarius membar(Sync); 789211050Smarius } 790211050Smarius } 791211050Smarius while (((ids = ldxa(0, ASI_INTR_DISPATCH_STATUS)) & 792211050Smarius IDR_CHEETAH_ALL_BUSY) != 0) 793211050Smarius ; 794211050Smarius intr_restore(s); 795211050Smarius if ((ids & 796211050Smarius (IDR_CHEETAH_ALL_BUSY | IDR_CHEETAH_ALL_NACK)) == 0) 797211050Smarius return; 798211050Smarius for (cpu = 0; cpu < mp_ncpus; cpu++) 799222813Sattilio if (CPU_ISSET(cpu, &cpus)) 800211050Smarius if ((ids & (IDR_NACK << 801211050Smarius (2 * cpuid_to_mid[cpu]))) == 0) 802222813Sattilio CPU_CLR(cpu, &cpus); 803211050Smarius } 804211050Smarius if (kdb_active != 0 || panicstr != NULL) 805222813Sattilio printf("%s: couldn't send IPI (cpus=%s ids=0x%lu)\n", 806222813Sattilio __func__, cpusetobj_strprint(pbuf, &cpus), ids); 807211050Smarius else 808222813Sattilio panic("%s: couldn't send IPI (cpus=%s ids=0x%lu)", 809222813Sattilio __func__, cpusetobj_strprint(pbuf, &cpus), ids); 810211050Smarius} 811