1/*- 2 * Copyright (c) 2001-2005 Marcel Moolenaar 3 * Copyright (c) 2000 Doug Rabson 4 * All rights reserved. 5 * 6 * Redistribution and use in source and binary forms, with or without 7 * modification, are permitted provided that the following conditions 8 * are met: 9 * 1. Redistributions of source code must retain the above copyright 10 * notice, this list of conditions and the following disclaimer. 11 * 2. Redistributions in binary form must reproduce the above copyright 12 * notice, this list of conditions and the following disclaimer in the 13 * documentation and/or other materials provided with the distribution. 14 * 15 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 16 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 17 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 18 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 19 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 20 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 21 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 22 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 23 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 24 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 25 * SUCH DAMAGE. 26 */ 27 28#include <sys/cdefs.h> 29__FBSDID("$FreeBSD$"); 30 31#include "opt_kstack_pages.h" 32#include "opt_xtrace.h" 33 34#include <sys/param.h> 35#include <sys/systm.h> 36#include <sys/ktr.h> 37#include <sys/proc.h> 38#include <sys/bus.h> 39#include <sys/kthread.h> 40#include <sys/lock.h> 41#include <sys/malloc.h> 42#include <sys/mutex.h> 43#include <sys/kernel.h> 44#include <sys/pcpu.h> 45#include <sys/sched.h> 46#include <sys/smp.h> 47#include <sys/sysctl.h> 48#include <sys/uuid.h> 49 50#include <machine/atomic.h> 51#include <machine/bootinfo.h> 52#include <machine/cpu.h> 53#include <machine/fpu.h> 54#include <machine/intr.h> 55#include <machine/mca.h> 56#include <machine/md_var.h> 57#include <machine/pal.h> 58#include <machine/pcb.h> 59#include <machine/sal.h> 60#include <machine/smp.h> 61 62#include <vm/vm.h> 63#include <vm/pmap.h> 64#include <vm/vm_extern.h> 65#include <vm/vm_kern.h> 66 67extern uint64_t bdata[]; 68 69extern int smp_disabled; 70 71MALLOC_DEFINE(M_SMP, "SMP", "SMP related allocations"); 72 73void ia64_ap_startup(void); 74 75#define SAPIC_ID_GET_ID(x) ((u_int)((x) >> 8) & 0xff) 76#define SAPIC_ID_GET_EID(x) ((u_int)(x) & 0xff) 77#define SAPIC_ID_SET(id, eid) ((u_int)(((id) & 0xff) << 8) | ((eid) & 0xff)) 78 79/* State used to wake and bootstrap APs. */ 80struct ia64_ap_state ia64_ap_state; 81 82int ia64_ipi_ast; 83int ia64_ipi_hardclock; 84int ia64_ipi_highfp; 85int ia64_ipi_nmi; 86int ia64_ipi_preempt; 87int ia64_ipi_rndzvs; 88int ia64_ipi_stop; 89 90static u_int 91sz2shft(uint64_t sz) 92{ 93 uint64_t s; 94 u_int shft; 95 96 shft = 12; /* Start with 4K */ 97 s = 1 << shft; 98 while (s < sz) { 99 shft++; 100 s <<= 1; 101 } 102 return (shft); 103} 104 105static u_int 106ia64_ih_ast(struct thread *td, u_int xiv, struct trapframe *tf) 107{ 108 109 PCPU_INC(md.stats.pcs_nasts); 110 CTR1(KTR_SMP, "IPI_AST, cpuid=%d", PCPU_GET(cpuid)); 111 return (0); 112} 113 114static u_int 115ia64_ih_hardclock(struct thread *td, u_int xiv, struct trapframe *tf) 116{ 117 struct trapframe *stf; 118 119 PCPU_INC(md.stats.pcs_nhardclocks); 120 CTR1(KTR_SMP, "IPI_HARDCLOCK, cpuid=%d", PCPU_GET(cpuid)); 121 stf = td->td_intr_frame; 122 td->td_intr_frame = tf; 123 hardclockintr(); 124 td->td_intr_frame = stf; 125 return (0); 126} 127 128static u_int 129ia64_ih_highfp(struct thread *td, u_int xiv, struct trapframe *tf) 130{ 131 132 PCPU_INC(md.stats.pcs_nhighfps); 133 ia64_highfp_save_ipi(); 134 return (0); 135} 136 137static u_int 138ia64_ih_preempt(struct thread *td, u_int xiv, struct trapframe *tf) 139{ 140 141 PCPU_INC(md.stats.pcs_npreempts); 142 CTR1(KTR_SMP, "IPI_PREEMPT, cpuid=%d", PCPU_GET(cpuid)); 143 sched_preempt(curthread); 144 return (0); 145} 146 147static u_int 148ia64_ih_rndzvs(struct thread *td, u_int xiv, struct trapframe *tf) 149{ 150 151 PCPU_INC(md.stats.pcs_nrdvs); 152 CTR1(KTR_SMP, "IPI_RENDEZVOUS, cpuid=%d", PCPU_GET(cpuid)); 153 smp_rendezvous_action(); 154 return (0); 155} 156 157static u_int 158ia64_ih_stop(struct thread *td, u_int xiv, struct trapframe *tf) 159{ 160 u_int cpuid; 161 162 PCPU_INC(md.stats.pcs_nstops); 163 cpuid = PCPU_GET(cpuid); 164 165 savectx(PCPU_PTR(md.pcb)); 166 167 CPU_SET_ATOMIC(cpuid, &stopped_cpus); 168 while (!CPU_ISSET(cpuid, &started_cpus)) 169 cpu_spinwait(); 170 CPU_CLR_ATOMIC(cpuid, &started_cpus); 171 CPU_CLR_ATOMIC(cpuid, &stopped_cpus); 172 return (0); 173} 174 175struct cpu_group * 176cpu_topo(void) 177{ 178 179 return smp_topo_none(); 180} 181 182static void 183ia64_store_mca_state(void* arg) 184{ 185 struct pcpu *pc = arg; 186 struct thread *td = curthread; 187 188 /* 189 * ia64_mca_save_state() is CPU-sensitive, so bind ourself to our 190 * target CPU. 191 */ 192 thread_lock(td); 193 sched_bind(td, pc->pc_cpuid); 194 thread_unlock(td); 195 196 ia64_mca_init_ap(); 197 198 /* 199 * Get and save the CPU specific MCA records. Should we get the 200 * MCA state for each processor, or just the CMC state? 201 */ 202 ia64_mca_save_state(SAL_INFO_MCA); 203 ia64_mca_save_state(SAL_INFO_CMC); 204 205 kproc_exit(0); 206} 207 208void 209ia64_ap_startup(void) 210{ 211 uint64_t vhpt; 212 213 ia64_ap_state.as_trace = 0x100; 214 215 ia64_set_rr(IA64_RR_BASE(5), (5 << 8) | (PAGE_SHIFT << 2) | 1); 216 ia64_set_rr(IA64_RR_BASE(6), (6 << 8) | (LOG2_ID_PAGE_SIZE << 2)); 217 ia64_set_rr(IA64_RR_BASE(7), (7 << 8) | (LOG2_ID_PAGE_SIZE << 2)); 218 ia64_srlz_d(); 219 220 pcpup = ia64_ap_state.as_pcpu; 221 ia64_set_k4((intptr_t)pcpup); 222 223 ia64_ap_state.as_trace = 0x108; 224 225 vhpt = pcpup->pc_md.vhpt; 226 map_vhpt(vhpt); 227 ia64_set_pta(vhpt + (1 << 8) + (pmap_vhpt_log2size << 2) + 1); 228 ia64_srlz_i(); 229 230 ia64_ap_state.as_trace = 0x110; 231 232 ia64_ap_state.as_awake = 1; 233 ia64_ap_state.as_delay = 0; 234 235 map_pal_code(); 236 map_gateway_page(); 237 238 ia64_set_fpsr(IA64_FPSR_DEFAULT); 239 240#ifdef XTRACE 241 ia64_xtrace_init_ap(ia64_ap_state.as_xtrace_buffer); 242#endif 243 244 /* Wait until it's time for us to be unleashed */ 245 while (ia64_ap_state.as_spin) 246 cpu_spinwait(); 247 248 /* Initialize curthread. */ 249 KASSERT(pcpup->pc_idlethread != NULL, ("no idle thread")); 250 pcpup->pc_curthread = pcpup->pc_idlethread; 251 252 pmap_invalidate_all(); 253 254 atomic_add_int(&ia64_ap_state.as_awake, 1); 255 while (!smp_started) 256 cpu_spinwait(); 257 258 CTR1(KTR_SMP, "SMP: cpu%d launched", PCPU_GET(cpuid)); 259 260 cpu_initclocks(); 261 262 ia64_set_tpr(0); 263 ia64_srlz_d(); 264 265 sched_throw(NULL); 266 /* NOTREACHED */ 267} 268 269void 270cpu_mp_setmaxid(void) 271{ 272 273 /* 274 * Count the number of processors in the system by walking the ACPI 275 * tables. Note that we record the actual number of processors, even 276 * if this is larger than MAXCPU. We only activate MAXCPU processors. 277 */ 278 mp_ncpus = ia64_count_cpus(); 279 280 /* 281 * Set the largest cpuid we're going to use. This is necessary for 282 * VM initialization. 283 */ 284 mp_maxid = min(mp_ncpus, MAXCPU) - 1; 285} 286 287int 288cpu_mp_probe(void) 289{ 290 291 /* 292 * If there's only 1 processor, or we don't have a wake-up vector, 293 * we're not going to enable SMP. Note that no wake-up vector can 294 * also mean that the wake-up mechanism is not supported. In this 295 * case we can have multiple processors, but we simply can't wake 296 * them up... 297 */ 298 return (mp_ncpus > 1 && ia64_ipi_wakeup != 0); 299} 300 301void 302cpu_mp_add(u_int acpi_id, u_int id, u_int eid) 303{ 304 struct pcpu *pc; 305 void *dpcpu; 306 u_int cpuid, sapic_id; 307 308 if (smp_disabled) 309 return; 310 311 sapic_id = SAPIC_ID_SET(id, eid); 312 cpuid = (IA64_LID_GET_SAPIC_ID(ia64_get_lid()) == sapic_id) 313 ? 0 : smp_cpus++; 314 315 KASSERT(!CPU_ISSET(cpuid, &all_cpus), 316 ("%s: cpu%d already in CPU map", __func__, acpi_id)); 317 318 if (cpuid != 0) { 319 pc = (struct pcpu *)malloc(sizeof(*pc), M_SMP, M_WAITOK); 320 pcpu_init(pc, cpuid, sizeof(*pc)); 321 dpcpu = (void *)kmem_malloc(kernel_arena, DPCPU_SIZE, 322 M_WAITOK | M_ZERO); 323 dpcpu_init(dpcpu, cpuid); 324 } else 325 pc = pcpup; 326 327 cpu_pcpu_setup(pc, acpi_id, sapic_id); 328 329 CPU_SET(pc->pc_cpuid, &all_cpus); 330} 331 332void 333cpu_mp_announce() 334{ 335 struct pcpu *pc; 336 uint32_t sapic_id; 337 int i; 338 339 for (i = 0; i <= mp_maxid; i++) { 340 pc = pcpu_find(i); 341 if (pc != NULL) { 342 sapic_id = IA64_LID_GET_SAPIC_ID(pc->pc_md.lid); 343 printf("cpu%d: ACPI Id=%x, SAPIC Id=%x, SAPIC Eid=%x", 344 i, pc->pc_acpi_id, SAPIC_ID_GET_ID(sapic_id), 345 SAPIC_ID_GET_EID(sapic_id)); 346 if (i == 0) 347 printf(" (BSP)\n"); 348 else 349 printf("\n"); 350 } 351 } 352} 353 354void 355cpu_mp_start() 356{ 357 struct ia64_sal_result result; 358 struct ia64_fdesc *fd; 359 struct pcpu *pc; 360 uintptr_t state; 361 u_char *stp; 362 363 state = ia64_tpa((uintptr_t)&ia64_ap_state); 364 fd = (struct ia64_fdesc *) os_boot_rendez; 365 result = ia64_sal_entry(SAL_SET_VECTORS, SAL_OS_BOOT_RENDEZ, 366 ia64_tpa(fd->func), state, 0, 0, 0, 0); 367 368 ia64_ap_state.as_pgtbl_pte = PTE_PRESENT | PTE_MA_WB | 369 PTE_ACCESSED | PTE_DIRTY | PTE_PL_KERN | PTE_AR_RW | 370 (bootinfo->bi_pbvm_pgtbl & PTE_PPN_MASK); 371 ia64_ap_state.as_pgtbl_itir = sz2shft(bootinfo->bi_pbvm_pgtblsz) << 2; 372 ia64_ap_state.as_text_va = IA64_PBVM_BASE; 373 ia64_ap_state.as_text_pte = PTE_PRESENT | PTE_MA_WB | 374 PTE_ACCESSED | PTE_DIRTY | PTE_PL_KERN | PTE_AR_RX | 375 (ia64_tpa(IA64_PBVM_BASE) & PTE_PPN_MASK); 376 ia64_ap_state.as_text_itir = bootinfo->bi_text_mapped << 2; 377 ia64_ap_state.as_data_va = (uintptr_t)bdata; 378 ia64_ap_state.as_data_pte = PTE_PRESENT | PTE_MA_WB | 379 PTE_ACCESSED | PTE_DIRTY | PTE_PL_KERN | PTE_AR_RW | 380 (ia64_tpa((uintptr_t)bdata) & PTE_PPN_MASK); 381 ia64_ap_state.as_data_itir = bootinfo->bi_data_mapped << 2; 382 383 /* Keep 'em spinning until we unleash them... */ 384 ia64_ap_state.as_spin = 1; 385 386 STAILQ_FOREACH(pc, &cpuhead, pc_allcpu) { 387 pc->pc_md.current_pmap = kernel_pmap; 388 /* The BSP is obviously running already. */ 389 if (pc->pc_cpuid == 0) { 390 pc->pc_md.awake = 1; 391 continue; 392 } 393 394 ia64_ap_state.as_pcpu = pc; 395 pc->pc_md.vhpt = pmap_alloc_vhpt(); 396 if (pc->pc_md.vhpt == 0) { 397 printf("SMP: WARNING: unable to allocate VHPT" 398 " for cpu%d", pc->pc_cpuid); 399 continue; 400 } 401 402 stp = malloc(KSTACK_PAGES * PAGE_SIZE, M_SMP, M_WAITOK); 403 ia64_ap_state.as_kstack = stp; 404 ia64_ap_state.as_kstack_top = stp + KSTACK_PAGES * PAGE_SIZE; 405 406#ifdef XTRACE 407 ia64_ap_state.as_xtrace_buffer = ia64_xtrace_alloc(); 408#endif 409 410 ia64_ap_state.as_trace = 0; 411 ia64_ap_state.as_delay = 2000; 412 ia64_ap_state.as_awake = 0; 413 414 if (bootverbose) 415 printf("SMP: waking up cpu%d\n", pc->pc_cpuid); 416 417 /* Here she goes... */ 418 ipi_send(pc, ia64_ipi_wakeup); 419 do { 420 DELAY(1000); 421 } while (--ia64_ap_state.as_delay > 0); 422 423 pc->pc_md.awake = ia64_ap_state.as_awake; 424 425 if (!ia64_ap_state.as_awake) { 426 printf("SMP: WARNING: cpu%d did not wake up (code " 427 "%#lx)\n", pc->pc_cpuid, 428 ia64_ap_state.as_trace - state); 429 } 430 } 431} 432 433static void 434cpu_mp_unleash(void *dummy) 435{ 436 struct pcpu *pc; 437 int cpus; 438 439 if (mp_ncpus <= 1) 440 return; 441 442 /* Allocate XIVs for IPIs */ 443 ia64_ipi_ast = ia64_xiv_alloc(PI_DULL, IA64_XIV_IPI, ia64_ih_ast); 444 ia64_ipi_hardclock = ia64_xiv_alloc(PI_REALTIME, IA64_XIV_IPI, 445 ia64_ih_hardclock); 446 ia64_ipi_highfp = ia64_xiv_alloc(PI_AV, IA64_XIV_IPI, ia64_ih_highfp); 447 ia64_ipi_preempt = ia64_xiv_alloc(PI_SOFT, IA64_XIV_IPI, 448 ia64_ih_preempt); 449 ia64_ipi_rndzvs = ia64_xiv_alloc(PI_AV, IA64_XIV_IPI, ia64_ih_rndzvs); 450 ia64_ipi_stop = ia64_xiv_alloc(PI_REALTIME, IA64_XIV_IPI, ia64_ih_stop); 451 452 /* Reserve the NMI vector for IPI_STOP_HARD if possible */ 453 ia64_ipi_nmi = (ia64_xiv_reserve(2, IA64_XIV_IPI, ia64_ih_stop) != 0) 454 ? ia64_ipi_stop : 0x400; /* DM=NMI, Vector=n/a */ 455 456 cpus = 0; 457 smp_cpus = 0; 458 STAILQ_FOREACH(pc, &cpuhead, pc_allcpu) { 459 cpus++; 460 if (pc->pc_md.awake) { 461 kproc_create(ia64_store_mca_state, pc, NULL, 0, 0, 462 "mca %u", pc->pc_cpuid); 463 smp_cpus++; 464 } 465 } 466 467 ia64_ap_state.as_awake = 1; 468 ia64_ap_state.as_spin = 0; 469 470 while (ia64_ap_state.as_awake != smp_cpus) 471 cpu_spinwait(); 472 473 if (smp_cpus != cpus || cpus != mp_ncpus) { 474 printf("SMP: %d CPUs found; %d CPUs usable; %d CPUs woken\n", 475 mp_ncpus, cpus, smp_cpus); 476 } 477 478 /* XXX Atomic set operation? */ 479 smp_started = 1; 480 481 /* 482 * Now that all CPUs are up and running, bind interrupts to each of 483 * them. 484 */ 485 ia64_bind_intr(); 486} 487SYSINIT(start_aps, SI_SUB_KICK_SCHEDULER, SI_ORDER_ANY, cpu_mp_unleash, NULL); 488 489/* 490 * send an IPI to a set of cpus. 491 */ 492void 493ipi_selected(cpuset_t cpus, int ipi) 494{ 495 struct pcpu *pc; 496 497 STAILQ_FOREACH(pc, &cpuhead, pc_allcpu) { 498 if (CPU_ISSET(pc->pc_cpuid, &cpus)) 499 ipi_send(pc, ipi); 500 } 501} 502 503/* 504 * send an IPI to a specific CPU. 505 */ 506void 507ipi_cpu(int cpu, u_int ipi) 508{ 509 510 ipi_send(cpuid_to_pcpu[cpu], ipi); 511} 512 513/* 514 * send an IPI to all CPUs EXCEPT myself. 515 */ 516void 517ipi_all_but_self(int ipi) 518{ 519 struct pcpu *pc; 520 521 STAILQ_FOREACH(pc, &cpuhead, pc_allcpu) { 522 if (pc != pcpup) 523 ipi_send(pc, ipi); 524 } 525} 526 527/* 528 * Send an IPI to the specified processor. 529 */ 530void 531ipi_send(struct pcpu *cpu, int xiv) 532{ 533 u_int sapic_id; 534 535 KASSERT(xiv != 0, ("ipi_send")); 536 537 sapic_id = IA64_LID_GET_SAPIC_ID(cpu->pc_md.lid); 538 539 ia64_mf(); 540 ia64_st8(&(ia64_pib->ib_ipi[sapic_id][0]), xiv); 541 ia64_mf_a(); 542 CTR3(KTR_SMP, "ipi_send(%p, %d): cpuid=%d", cpu, xiv, PCPU_GET(cpuid)); 543} 544