mp_machdep.c revision 122947
182547Smike/*- 282547Smike * Copyright (c) 1997 Berkeley Software Design, Inc. All rights reserved. 382547Smike * 482547Smike * Redistribution and use in source and binary forms, with or without 582547Smike * modification, are permitted provided that the following conditions 682547Smike * are met: 782547Smike * 1. Redistributions of source code must retain the above copyright 882547Smike * notice, this list of conditions and the following disclaimer. 982547Smike * 2. Redistributions in binary form must reproduce the above copyright 1082547Smike * notice, this list of conditions and the following disclaimer in the 1182547Smike * documentation and/or other materials provided with the distribution. 1282547Smike * 3. Berkeley Software Design Inc's name may not be used to endorse or 1382547Smike * promote products derived from this software without specific prior 1482547Smike * written permission. 1582547Smike * 1682547Smike * THIS SOFTWARE IS PROVIDED BY BERKELEY SOFTWARE DESIGN INC ``AS IS'' AND 1782547Smike * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 1882547Smike * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 1982547Smike * ARE DISCLAIMED. IN NO EVENT SHALL BERKELEY SOFTWARE DESIGN INC BE LIABLE 2082547Smike * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 2182547Smike * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 2282547Smike * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 2382547Smike * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 2482547Smike * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 2582547Smike * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 2682547Smike * SUCH DAMAGE. 2782547Smike * 2882547Smike * from BSDI: locore.s,v 1.36.2.15 1999/08/23 22:34:41 cp Exp 2982547Smike */ 3082547Smike/*- 31117280Scharnier * Copyright (c) 2002 Jake Burkholder. 32117280Scharnier * All rights reserved. 33117280Scharnier * 34149424Spjd * Redistribution and use in source and binary forms, with or without 35231912Strociny * modification, are permitted provided that the following conditions 36231909Strociny * are met: 3782547Smike * 1. Redistributions of source code must retain the above copyright 3882547Smike * notice, this list of conditions and the following disclaimer. 39129983Sphk * 2. Redistributions in binary form must reproduce the above copyright 40149424Spjd * notice, this list of conditions and the following disclaimer in the 41167700Strhodes * documentation and/or other materials provided with the distribution. 42231910Strociny * 43231910Strociny * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 4482547Smike * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 4582547Smike * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 4682547Smike * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 4782547Smike * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 48231910Strociny * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 49167700Strhodes * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 50231911Strociny * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 5182547Smike * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 5282547Smike * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 5382547Smike * SUCH DAMAGE. 5482547Smike * 5582547Smike * $FreeBSD: head/sys/sparc64/sparc64/mp_machdep.c 122947 2003-11-21 22:23:26Z jhb $ 56255521Sjmg */ 57231910Strociny 58255707Strociny#include "opt_ddb.h" 59255521Sjmg 60231909Strociny#include <sys/param.h> 6182547Smike#include <sys/systm.h> 6282547Smike#include <sys/lock.h> 63231911Strociny#include <sys/kernel.h> 64255521Sjmg#include <sys/ktr.h> 65255521Sjmg#include <sys/mutex.h> 6682547Smike#include <sys/pcpu.h> 6782547Smike#include <sys/proc.h> 6882547Smike#include <sys/smp.h> 6982547Smike 7082547Smike#include <vm/vm.h> 7182547Smike#include <vm/vm_param.h> 7282547Smike#include <vm/pmap.h> 73167700Strhodes#include <vm/vm_kern.h> 74167700Strhodes#include <vm/vm_extern.h> 75167700Strhodes#include <vm/vm_map.h> 76255521Sjmg 77255521Sjmg#include <dev/ofw/openfirm.h> 78255521Sjmg 79231911Strociny#include <ddb/ddb.h> 80231911Strociny 81231911Strociny#include <machine/asi.h> 82167356Strhodes#include <machine/atomic.h> 83167356Strhodes#include <machine/bus.h> 84167356Strhodes#include <machine/md_var.h> 8582547Smike#include <machine/metadata.h> 8682547Smike#include <machine/ofw_machdep.h> 8782547Smike#include <machine/smp.h> 8882547Smike#include <machine/tick.h> 8982547Smike#include <machine/tlb.h> 9082547Smike#include <machine/tte.h> 9182547Smike 9282547Smikestatic ih_func_t cpu_ipi_ast; 9382547Smikestatic ih_func_t cpu_ipi_stop; 94167356Strhodes 95255521Sjmg/* 96129983Sphk * Argument area used to pass data to non-boot processors as they start up. 97129983Sphk * This must be statically initialized with a known invalid upa module id, 98129983Sphk * since the other processors will use it before the boot cpu enters the 99129983Sphk * kernel. 100231909Strociny */ 101149424Spjdstruct cpu_start_args cpu_start_args = { 0, -1, -1, 0, 0 }; 102149424Spjdstruct ipi_cache_args ipi_cache_args; 103149424Spjdstruct ipi_tlb_args ipi_tlb_args; 104149424Spjd 105149424Spjdstruct mtx ipi_mtx; 106149424Spjd 107129983Sphkvm_offset_t mp_tramp; 108149424Spjd 109129983Sphku_int mp_boot_mid; 110255707Strociny 111255521Sjmgstatic volatile u_int shutdown_cpus; 112255521Sjmg 113255521Sjmgvoid cpu_mp_unleash(void *); 114255707StrocinySYSINIT(cpu_mp_unleash, SI_SUB_SMP, SI_ORDER_FIRST, cpu_mp_unleash, NULL); 115255707Strociny 116255707Strocinyvm_offset_t 117255521Sjmgmp_tramp_alloc(void) 118255521Sjmg{ 119255521Sjmg struct tte *tp; 120255521Sjmg char *v; 121255521Sjmg int i; 122255521Sjmg 123255521Sjmg v = OF_claim(NULL, PAGE_SIZE, PAGE_SIZE); 124129983Sphk if (v == NULL) 125255707Strociny panic("mp_tramp_alloc"); 126255707Strociny bcopy(mp_tramp_code, v, mp_tramp_code_len); 127255707Strociny *(u_long *)(v + mp_tramp_tlb_slots) = kernel_tlb_slots; 128255707Strociny *(u_long *)(v + mp_tramp_func) = (u_long)mp_startup; 129255707Strociny tp = (struct tte *)(v + mp_tramp_code_len); 130255707Strociny for (i = 0; i < kernel_tlb_slots; i++) { 131129983Sphk tp[i].tte_vpn = TV_VPN(kernel_tlbs[i].te_va, TS_4M); 132231910Strociny tp[i].tte_data = TD_V | TD_4M | TD_PA(kernel_tlbs[i].te_pa) | 133231911Strociny TD_L | TD_CP | TD_CV | TD_P | TD_W; 134231911Strociny } 135231911Strociny for (i = 0; i < PAGE_SIZE; i += sizeof(long)) 136231911Strociny flush(v + i); 137231911Strociny return (vm_offset_t)v; 138231911Strociny} 139231911Strociny 140231910Strociny/* 141231910Strociny * Probe for other cpus. 142231911Strociny */ 143231909Strocinyvoid 144231910Strocinycpu_mp_setmaxid(void) 145231910Strociny{ 146231910Strociny phandle_t child; 147255707Strociny phandle_t root; 148255707Strociny char buf[128]; 149255707Strociny int cpus; 150255707Strociny 151231910Strociny all_cpus = 1 << PCPU_GET(cpuid); 152231910Strociny mp_boot_mid = PCPU_GET(mid); 153231910Strociny mp_ncpus = 1; 154231910Strociny 155255707Strociny cpus = 0; 156255707Strociny root = OF_peer(0); 157255707Strociny for (child = OF_child(root); child != 0; child = OF_peer(child)) { 158255707Strociny if (OF_getprop(child, "device_type", buf, sizeof(buf)) > 0 && 159231910Strociny strcmp(buf, "cpu") == 0) 160231910Strociny cpus++; 161231910Strociny } 162231910Strociny mp_maxid = cpus; 163231910Strociny} 164231910Strociny 165255707Strocinyint 166255707Strocinycpu_mp_probe(void) 167255707Strociny{ 168255707Strociny 169231912Strociny return (mp_maxid > 1); 170231912Strociny} 171231912Strociny 172231912Strocinystatic void 173231912Strocinysun4u_startcpu(phandle_t cpu, void *func, u_long arg) 174231912Strociny{ 175231911Strociny static struct { 176231910Strociny cell_t name; 177231909Strociny cell_t nargs; 178231909Strociny cell_t nreturns; 179231909Strociny cell_t cpu; 180231909Strociny cell_t func; 181231909Strociny cell_t arg; 182255707Strociny } args = { 183255707Strociny (cell_t)"SUNW,start-cpu", 184231909Strociny 3, 185231909Strociny 0, 186231910Strociny 0, 187231910Strociny 0, 188231910Strociny 0 189231910Strociny }; 190231910Strociny 191231910Strociny args.cpu = cpu; 192231909Strociny args.func = (cell_t)func; 193149424Spjd args.arg = (cell_t)arg; 194129983Sphk openfirmware(&args); 195231909Strociny} 196231909Strociny 19782547Smike/* 198231909Strociny * Stop the calling CPU. 199129983Sphk */ 200231909Strocinystatic void 201231909Strocinysun4u_stopself(void) 202231909Strociny{ 203231909Strociny static struct { 204231909Strociny cell_t name; 205231909Strociny cell_t nargs; 206255521Sjmg cell_t nreturns; 207231909Strociny } args = { 208231911Strociny (cell_t)"SUNW,stop-self", 209231911Strociny 0, 210231911Strociny 0, 211231911Strociny }; 212255707Strociny 213231909Strociny openfirmware_exit(&args); 214255521Sjmg panic("sun4u_stopself: failed."); 215255707Strociny} 21682547Smike 21782547Smike/* 21882547Smike * Fire up any non-boot processors. 219231910Strociny */ 220231910Strocinyvoid 221231910Strocinycpu_mp_start(void) 222231910Strociny{ 223231910Strociny volatile struct cpu_start_args *csa; 224231910Strociny struct pcpu *pc; 225167700Strhodes phandle_t child; 226167356Strhodes phandle_t root; 227167356Strhodes vm_offset_t va; 228167356Strhodes char buf[128]; 229167700Strhodes u_int clock; 230167700Strhodes int cpuid; 231167700Strhodes u_int mid; 232167356Strhodes u_long s; 233167700Strhodes 234167700Strhodes mtx_init(&ipi_mtx, "ipi", NULL, MTX_SPIN); 235167356Strhodes 236167356Strhodes intr_setup(PIL_AST, cpu_ipi_ast, -1, NULL, NULL); 237231911Strociny intr_setup(PIL_RENDEZVOUS, (ih_func_t *)smp_rendezvous_action, 238231910Strociny -1, NULL, NULL); 239231909Strociny intr_setup(PIL_STOP, cpu_ipi_stop, -1, NULL, NULL); 240231911Strociny 241231909Strociny root = OF_peer(0); 242231911Strociny csa = &cpu_start_args; 243231910Strociny for (child = OF_child(root); child != 0; child = OF_peer(child)) { 244231910Strociny if (OF_getprop(child, "device_type", buf, sizeof(buf)) <= 0 || 245231910Strociny strcmp(buf, "cpu") != 0) 246231911Strociny continue; 247231909Strociny if (OF_getprop(child, "upa-portid", &mid, sizeof(mid)) <= 0 && 248231910Strociny OF_getprop(child, "portid", &mid, sizeof(mid)) <= 0) 249231910Strociny panic("cpu_mp_start: can't get module id"); 250236550Strociny if (mid == mp_boot_mid) 251236550Strociny continue; 252236550Strociny if (OF_getprop(child, "clock-frequency", &clock, 253236550Strociny sizeof(clock)) <= 0) 254231911Strociny panic("cpu_mp_start: can't get clock"); 255231910Strociny 256231911Strociny csa->csa_state = 0; 257231910Strociny sun4u_startcpu(child, (void *)mp_tramp, 0); 258231910Strociny s = intr_disable(); 259231911Strociny while (csa->csa_state != CPU_CLKSYNC) 260231910Strociny ; 261231910Strociny membar(StoreLoad); 262231910Strociny csa->csa_tick = rd(tick); 263231910Strociny while (csa->csa_state != CPU_INIT) 264231911Strociny ; 265231910Strociny csa->csa_tick = 0; 266231909Strociny intr_restore(s); 267231909Strociny 268231909Strociny cpuid = mp_ncpus++; 269231909Strociny cpu_identify(csa->csa_ver, clock, cpuid); 27082547Smike 27182547Smike va = kmem_alloc(kernel_map, PCPU_PAGES * PAGE_SIZE); 272129983Sphk pc = (struct pcpu *)(va + (PCPU_PAGES * PAGE_SIZE)) - 1; 273255521Sjmg pcpu_init(pc, cpuid, sizeof(*pc)); 274255521Sjmg pc->pc_addr = va; 27582547Smike pc->pc_mid = mid; 27682547Smike pc->pc_node = child; 277 278 all_cpus |= 1 << cpuid; 279 } 280 PCPU_SET(other_cpus, all_cpus & ~(1 << PCPU_GET(cpuid))); 281 smp_active = 1; 282} 283 284void 285cpu_mp_announce(void) 286{ 287} 288 289void 290cpu_mp_unleash(void *v) 291{ 292 volatile struct cpu_start_args *csa; 293 struct pcpu *pc; 294 vm_offset_t va; 295 vm_paddr_t pa; 296 u_int ctx_min; 297 u_int ctx_inc; 298 u_long s; 299 int i; 300 301 ctx_min = TLB_CTX_USER_MIN; 302 ctx_inc = (TLB_CTX_USER_MAX - 1) / mp_ncpus; 303 csa = &cpu_start_args; 304 csa->csa_count = mp_ncpus; 305 SLIST_FOREACH(pc, &cpuhead, pc_allcpu) { 306 pc->pc_tlb_ctx = ctx_min; 307 pc->pc_tlb_ctx_min = ctx_min; 308 pc->pc_tlb_ctx_max = ctx_min + ctx_inc; 309 ctx_min += ctx_inc; 310 311 if (pc->pc_cpuid == PCPU_GET(cpuid)) 312 continue; 313 KASSERT(pc->pc_idlethread != NULL, 314 ("cpu_mp_unleash: idlethread")); 315 KASSERT(pc->pc_curthread == pc->pc_idlethread, 316 ("cpu_mp_unleash: curthread")); 317 318 pc->pc_curpcb = pc->pc_curthread->td_pcb; 319 for (i = 0; i < PCPU_PAGES; i++) { 320 va = pc->pc_addr + i * PAGE_SIZE; 321 pa = pmap_kextract(va); 322 if (pa == 0) 323 panic("cpu_mp_unleash: pmap_kextract\n"); 324 csa->csa_ttes[i].tte_vpn = TV_VPN(va, TS_8K); 325 csa->csa_ttes[i].tte_data = TD_V | TD_8K | TD_PA(pa) | 326 TD_L | TD_CP | TD_CV | TD_P | TD_W; 327 } 328 csa->csa_state = 0; 329 csa->csa_pcpu = pc->pc_addr; 330 csa->csa_mid = pc->pc_mid; 331 s = intr_disable(); 332 while (csa->csa_state != CPU_BOOTSTRAP) 333 ; 334 intr_restore(s); 335 } 336 337 membar(StoreLoad); 338 csa->csa_count = 0; 339 smp_started = 1; 340} 341 342void 343cpu_mp_bootstrap(struct pcpu *pc) 344{ 345 volatile struct cpu_start_args *csa; 346 347 csa = &cpu_start_args; 348 pmap_map_tsb(); 349 cpu_setregs(pc); 350 tick_start_ap(); 351 352 smp_cpus++; 353 PCPU_SET(other_cpus, all_cpus & ~(1 << PCPU_GET(cpuid))); 354 printf("SMP: AP CPU #%d Launched!\n", PCPU_GET(cpuid)); 355 356 csa->csa_count--; 357 membar(StoreLoad); 358 csa->csa_state = CPU_BOOTSTRAP; 359 while (csa->csa_count != 0) 360 ; 361 362 binuptime(PCPU_PTR(switchtime)); 363 PCPU_SET(switchticks, ticks); 364 365 /* ok, now grab sched_lock and enter the scheduler */ 366 mtx_lock_spin(&sched_lock); 367 cpu_throw(NULL, choosethread()); /* doesn't return */ 368} 369 370void 371cpu_mp_shutdown(void) 372{ 373 int i; 374 375 critical_enter(); 376 shutdown_cpus = PCPU_GET(other_cpus); 377 if (stopped_cpus != PCPU_GET(other_cpus)) /* XXX */ 378 stop_cpus(stopped_cpus ^ PCPU_GET(other_cpus)); 379 i = 0; 380 while (shutdown_cpus != 0) { 381 if (i++ > 100000) { 382 printf("timeout shutting down CPUs.\n"); 383 break; 384 } 385 } 386 /* XXX: delay a bit to allow the CPUs to actually enter the PROM. */ 387 DELAY(100000); 388 critical_exit(); 389} 390 391static void 392cpu_ipi_ast(struct trapframe *tf) 393{ 394} 395 396static void 397cpu_ipi_stop(struct trapframe *tf) 398{ 399 400 CTR1(KTR_SMP, "cpu_ipi_stop: stopped %d", PCPU_GET(cpuid)); 401 atomic_set_acq_int(&stopped_cpus, PCPU_GET(cpumask)); 402 while ((started_cpus & PCPU_GET(cpumask)) == 0) { 403 if ((shutdown_cpus & PCPU_GET(cpumask)) != 0) { 404 atomic_clear_int(&shutdown_cpus, PCPU_GET(cpumask)); 405 sun4u_stopself(); 406 } 407 } 408 atomic_clear_rel_int(&started_cpus, PCPU_GET(cpumask)); 409 atomic_clear_rel_int(&stopped_cpus, PCPU_GET(cpumask)); 410 CTR1(KTR_SMP, "cpu_ipi_stop: restarted %d", PCPU_GET(cpuid)); 411} 412 413void 414cpu_ipi_selected(u_int cpus, u_long d0, u_long d1, u_long d2) 415{ 416 struct pcpu *pc; 417 u_int cpu; 418 419 while (cpus) { 420 cpu = ffs(cpus) - 1; 421 cpus &= ~(1 << cpu); 422 pc = pcpu_find(cpu); 423 cpu_ipi_send(pc->pc_mid, d0, d1, d2); 424 } 425} 426 427void 428cpu_ipi_send(u_int mid, u_long d0, u_long d1, u_long d2) 429{ 430 u_long s; 431 int i; 432 433 KASSERT((ldxa(0, ASI_INTR_DISPATCH_STATUS) & IDR_BUSY) == 0, 434 ("ipi_send: outstanding dispatch")); 435 for (i = 0; i < IPI_RETRIES; i++) { 436 s = intr_disable(); 437 stxa(AA_SDB_INTR_D0, ASI_SDB_INTR_W, d0); 438 stxa(AA_SDB_INTR_D1, ASI_SDB_INTR_W, d1); 439 stxa(AA_SDB_INTR_D2, ASI_SDB_INTR_W, d2); 440 stxa(AA_INTR_SEND | (mid << 14), ASI_SDB_INTR_W, 0); 441 membar(Sync); 442 while (ldxa(0, ASI_INTR_DISPATCH_STATUS) & IDR_BUSY) 443 ; 444 intr_restore(s); 445 if ((ldxa(0, ASI_INTR_DISPATCH_STATUS) & IDR_NACK) == 0) 446 return; 447 } 448 if ( 449#ifdef DDB 450 db_active || 451#endif 452 panicstr != NULL) 453 printf("ipi_send: couldn't send ipi to module %u\n", mid); 454 else 455 panic("ipi_send: couldn't send ipi"); 456} 457 458void 459ipi_selected(u_int cpus, u_int ipi) 460{ 461 cpu_ipi_selected(cpus, 0, (u_long)tl_ipi_level, ipi); 462} 463 464void 465ipi_all(u_int ipi) 466{ 467 panic("ipi_all"); 468} 469 470void 471ipi_all_but_self(u_int ipi) 472{ 473 cpu_ipi_selected(PCPU_GET(other_cpus), 0, (u_long)tl_ipi_level, ipi); 474} 475