mp_machdep.c revision 133129
1/*- 2 * Copyright (c) 1996, by Steve Passe 3 * All rights reserved. 4 * 5 * Redistribution and use in source and binary forms, with or without 6 * modification, are permitted provided that the following conditions 7 * are met: 8 * 1. Redistributions of source code must retain the above copyright 9 * notice, this list of conditions and the following disclaimer. 10 * 2. The name of the developer may NOT be used to endorse or promote products 11 * derived from this software without specific prior written permission. 12 * 13 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 14 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 15 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 16 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 17 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 18 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 19 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 20 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 21 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 22 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 23 * SUCH DAMAGE. 24 */ 25 26#include <sys/cdefs.h> 27__FBSDID("$FreeBSD: head/sys/i386/i386/mp_machdep.c 133129 2004-08-04 18:30:31Z markm $"); 28 29#include "opt_apic.h" 30#include "opt_cpu.h" 31#include "opt_kstack_pages.h" 32 33#if !defined(lint) 34#if !defined(SMP) 35#error How did you get here? 36#endif 37 38#if defined(I386_CPU) && !defined(COMPILING_LINT) 39#error SMP not supported with I386_CPU 40#endif 41#ifndef DEV_APIC 42#error The apic device is required for SMP, add "device apic" to your config file. 43#endif 44#if defined(CPU_DISABLE_CMPXCHG) && !defined(COMPILING_LINT) 45#error SMP not supported with CPU_DISABLE_CMPXCHG 46#endif 47#endif /* not lint */ 48 49#include <sys/param.h> 50#include <sys/systm.h> 51#include <sys/bus.h> 52#include <sys/cons.h> /* cngetc() */ 53#ifdef GPROF 54#include <sys/gmon.h> 55#endif 56#include <sys/kernel.h> 57#include <sys/ktr.h> 58#include <sys/lock.h> 59#include <sys/malloc.h> 60#include <sys/memrange.h> 61#include <sys/mutex.h> 62#include <sys/pcpu.h> 63#include <sys/proc.h> 64#include <sys/smp.h> 65#include <sys/sysctl.h> 66 67#include <vm/vm.h> 68#include <vm/vm_param.h> 69#include <vm/pmap.h> 70#include <vm/vm_kern.h> 71#include <vm/vm_extern.h> 72 73#include <machine/apicreg.h> 74#include <machine/clock.h> 75#include <machine/md_var.h> 76#include <machine/pcb.h> 77#include <machine/smp.h> 78#include <machine/smptests.h> /** COUNT_XINVLTLB_HITS */ 79#include <machine/specialreg.h> 80#include <machine/privatespace.h> 81 82#define WARMBOOT_TARGET 0 83#define WARMBOOT_OFF (KERNBASE + 0x0467) 84#define WARMBOOT_SEG (KERNBASE + 0x0469) 85 86#define CMOS_REG (0x70) 87#define CMOS_DATA (0x71) 88#define BIOS_RESET (0x0f) 89#define BIOS_WARM (0x0a) 90 91/* 92 * this code MUST be enabled here and in mpboot.s. 93 * it follows the very early stages of AP boot by placing values in CMOS ram. 94 * it NORMALLY will never be needed and thus the primitive method for enabling. 95 * 96#define CHECK_POINTS 97 */ 98 99#if defined(CHECK_POINTS) && !defined(PC98) 100#define CHECK_READ(A) (outb(CMOS_REG, (A)), inb(CMOS_DATA)) 101#define CHECK_WRITE(A,D) (outb(CMOS_REG, (A)), outb(CMOS_DATA, (D))) 102 103#define CHECK_INIT(D); \ 104 CHECK_WRITE(0x34, (D)); \ 105 CHECK_WRITE(0x35, (D)); \ 106 CHECK_WRITE(0x36, (D)); \ 107 CHECK_WRITE(0x37, (D)); \ 108 CHECK_WRITE(0x38, (D)); \ 109 CHECK_WRITE(0x39, (D)); 110 111#define CHECK_PRINT(S); \ 112 printf("%s: %d, %d, %d, %d, %d, %d\n", \ 113 (S), \ 114 CHECK_READ(0x34), \ 115 CHECK_READ(0x35), \ 116 CHECK_READ(0x36), \ 117 CHECK_READ(0x37), \ 118 CHECK_READ(0x38), \ 119 CHECK_READ(0x39)); 120 121#else /* CHECK_POINTS */ 122 123#define CHECK_INIT(D) 124#define CHECK_PRINT(S) 125#define CHECK_WRITE(A, D) 126 127#endif /* CHECK_POINTS */ 128 129/* 130 * Values to send to the POST hardware. 131 */ 132#define MP_BOOTADDRESS_POST 0x10 133#define MP_PROBE_POST 0x11 134#define MPTABLE_PASS1_POST 0x12 135 136#define MP_START_POST 0x13 137#define MP_ENABLE_POST 0x14 138#define MPTABLE_PASS2_POST 0x15 139 140#define START_ALL_APS_POST 0x16 141#define INSTALL_AP_TRAMP_POST 0x17 142#define START_AP_POST 0x18 143 144#define MP_ANNOUNCE_POST 0x19 145 146/* lock region used by kernel profiling */ 147int mcount_lock; 148 149/** XXX FIXME: where does this really belong, isa.h/isa.c perhaps? */ 150int current_postcode; 151 152int mp_naps; /* # of Applications processors */ 153int boot_cpu_id = -1; /* designated BSP */ 154extern int nkpt; 155 156/* 157 * CPU topology map datastructures for HTT. 158 */ 159static struct cpu_group mp_groups[MAXCPU]; 160static struct cpu_top mp_top; 161 162/* AP uses this during bootstrap. Do not staticize. */ 163char *bootSTK; 164static int bootAP; 165 166/* Hotwire a 0->4MB V==P mapping */ 167extern pt_entry_t *KPTphys; 168 169/* SMP page table page */ 170extern pt_entry_t *SMPpt; 171 172struct pcb stoppcbs[MAXCPU]; 173 174/* Variables needed for SMP tlb shootdown. */ 175vm_offset_t smp_tlb_addr1; 176vm_offset_t smp_tlb_addr2; 177volatile int smp_tlb_wait; 178struct mtx smp_tlb_mtx; 179 180/* 181 * Local data and functions. 182 */ 183 184static u_int logical_cpus; 185static u_int logical_cpus_mask; 186 187/* used to hold the AP's until we are ready to release them */ 188static struct mtx ap_boot_mtx; 189 190/* Set to 1 once we're ready to let the APs out of the pen. */ 191static volatile int aps_ready = 0; 192 193/* 194 * Store data from cpu_add() until later in the boot when we actually setup 195 * the APs. 196 */ 197struct cpu_info { 198 int cpu_present:1; 199 int cpu_bsp:1; 200} static cpu_info[MAXCPU]; 201static int cpu_apic_ids[MAXCPU]; 202 203static u_int boot_address; 204 205static void set_logical_apic_ids(void); 206static int start_all_aps(void); 207static void install_ap_tramp(void); 208static int start_ap(int apic_id); 209static void release_aps(void *dummy); 210 211static int hlt_cpus_mask; 212static int hlt_logical_cpus; 213static struct sysctl_ctx_list logical_cpu_clist; 214 215struct mem_range_softc mem_range_softc; 216 217static void 218mem_range_AP_init(void) 219{ 220 if (mem_range_softc.mr_op && mem_range_softc.mr_op->initAP) 221 mem_range_softc.mr_op->initAP(&mem_range_softc); 222} 223 224void 225mp_topology(void) 226{ 227 struct cpu_group *group; 228 int logical_cpus; 229 int apic_id; 230 int groups; 231 int cpu; 232 233 /* Build the smp_topology map. */ 234 /* Nothing to do if there is no HTT support. */ 235 if ((cpu_feature & CPUID_HTT) == 0) 236 return; 237 logical_cpus = (cpu_procinfo & CPUID_HTT_CORES) >> 16; 238 if (logical_cpus <= 1) 239 return; 240 group = &mp_groups[0]; 241 groups = 1; 242 for (cpu = 0, apic_id = 0; apic_id < MAXCPU; apic_id++) { 243 if (!cpu_info[apic_id].cpu_present) 244 continue; 245 /* 246 * If the current group has members and we're not a logical 247 * cpu, create a new group. 248 */ 249 if (group->cg_count != 0 && (apic_id % logical_cpus) == 0) { 250 group++; 251 groups++; 252 } 253 group->cg_count++; 254 group->cg_mask |= 1 << cpu; 255 cpu++; 256 } 257 258 mp_top.ct_count = groups; 259 mp_top.ct_group = mp_groups; 260 smp_topology = &mp_top; 261} 262 263 264/* 265 * Calculate usable address in base memory for AP trampoline code. 266 */ 267u_int 268mp_bootaddress(u_int basemem) 269{ 270 POSTCODE(MP_BOOTADDRESS_POST); 271 272 boot_address = trunc_page(basemem); /* round down to 4k boundary */ 273 if ((basemem - boot_address) < bootMP_size) 274 boot_address -= PAGE_SIZE; /* not enough, lower by 4k */ 275 276 return boot_address; 277} 278 279void 280cpu_add(u_int apic_id, char boot_cpu) 281{ 282 283 if (apic_id >= MAXCPU) { 284 printf("SMP: CPU %d exceeds maximum CPU %d, ignoring\n", 285 apic_id, MAXCPU - 1); 286 return; 287 } 288 KASSERT(cpu_info[apic_id].cpu_present == 0, ("CPU %d added twice", 289 apic_id)); 290 cpu_info[apic_id].cpu_present = 1; 291 if (boot_cpu) { 292 KASSERT(boot_cpu_id == -1, 293 ("CPU %d claims to be BSP, but CPU %d already is", apic_id, 294 boot_cpu_id)); 295 boot_cpu_id = apic_id; 296 cpu_info[apic_id].cpu_bsp = 1; 297 } 298 mp_ncpus++; 299 if (bootverbose) 300 printf("SMP: Added CPU %d (%s)\n", apic_id, boot_cpu ? "BSP" : 301 "AP"); 302 303} 304 305void 306cpu_mp_setmaxid(void) 307{ 308 309 mp_maxid = MAXCPU - 1; 310} 311 312int 313cpu_mp_probe(void) 314{ 315 316 /* 317 * Always record BSP in CPU map so that the mbuf init code works 318 * correctly. 319 */ 320 all_cpus = 1; 321 if (mp_ncpus == 0) { 322 /* 323 * No CPUs were found, so this must be a UP system. Setup 324 * the variables to represent a system with a single CPU 325 * with an id of 0. 326 */ 327 mp_ncpus = 1; 328 return (0); 329 } 330 331 /* At least one CPU was found. */ 332 if (mp_ncpus == 1) { 333 /* 334 * One CPU was found, so this must be a UP system with 335 * an I/O APIC. 336 */ 337 return (0); 338 } 339 340 /* At least two CPUs were found. */ 341 return (1); 342} 343 344/* 345 * Initialize the IPI handlers and start up the AP's. 346 */ 347void 348cpu_mp_start(void) 349{ 350 int i; 351 352 POSTCODE(MP_START_POST); 353 354 /* Initialize the logical ID to APIC ID table. */ 355 for (i = 0; i < MAXCPU; i++) 356 cpu_apic_ids[i] = -1; 357 358 /* Install an inter-CPU IPI for TLB invalidation */ 359 setidt(IPI_INVLTLB, IDTVEC(invltlb), 360 SDT_SYS386IGT, SEL_KPL, GSEL(GCODE_SEL, SEL_KPL)); 361 setidt(IPI_INVLPG, IDTVEC(invlpg), 362 SDT_SYS386IGT, SEL_KPL, GSEL(GCODE_SEL, SEL_KPL)); 363 setidt(IPI_INVLRNG, IDTVEC(invlrng), 364 SDT_SYS386IGT, SEL_KPL, GSEL(GCODE_SEL, SEL_KPL)); 365 366 /* Install an inter-CPU IPI for forwarding hardclock() */ 367 setidt(IPI_HARDCLOCK, IDTVEC(hardclock), 368 SDT_SYS386IGT, SEL_KPL, GSEL(GCODE_SEL, SEL_KPL)); 369 370 /* Install an inter-CPU IPI for forwarding statclock() */ 371 setidt(IPI_STATCLOCK, IDTVEC(statclock), 372 SDT_SYS386IGT, SEL_KPL, GSEL(GCODE_SEL, SEL_KPL)); 373 374 /* Install an inter-CPU IPI for lazy pmap release */ 375 setidt(IPI_LAZYPMAP, IDTVEC(lazypmap), 376 SDT_SYS386IGT, SEL_KPL, GSEL(GCODE_SEL, SEL_KPL)); 377 378 /* Install an inter-CPU IPI for all-CPU rendezvous */ 379 setidt(IPI_RENDEZVOUS, IDTVEC(rendezvous), 380 SDT_SYS386IGT, SEL_KPL, GSEL(GCODE_SEL, SEL_KPL)); 381 382 /* Install an inter-CPU IPI for forcing an additional software trap */ 383 setidt(IPI_AST, IDTVEC(cpuast), 384 SDT_SYS386IGT, SEL_KPL, GSEL(GCODE_SEL, SEL_KPL)); 385 386 /* Install an inter-CPU IPI for CPU stop/restart */ 387 setidt(IPI_STOP, IDTVEC(cpustop), 388 SDT_SYS386IGT, SEL_KPL, GSEL(GCODE_SEL, SEL_KPL)); 389 390 mtx_init(&smp_tlb_mtx, "tlb", NULL, MTX_SPIN); 391 392 /* Set boot_cpu_id if needed. */ 393 if (boot_cpu_id == -1) { 394 boot_cpu_id = PCPU_GET(apic_id); 395 cpu_info[boot_cpu_id].cpu_bsp = 1; 396 } else 397 KASSERT(boot_cpu_id == PCPU_GET(apic_id), 398 ("BSP's APIC ID doesn't match boot_cpu_id")); 399 cpu_apic_ids[0] = boot_cpu_id; 400 401 /* Start each Application Processor */ 402 start_all_aps(); 403 404 /* Setup the initial logical CPUs info. */ 405 logical_cpus = logical_cpus_mask = 0; 406 if (cpu_feature & CPUID_HTT) 407 logical_cpus = (cpu_procinfo & CPUID_HTT_CORES) >> 16; 408 409 set_logical_apic_ids(); 410} 411 412 413/* 414 * Print various information about the SMP system hardware and setup. 415 */ 416void 417cpu_mp_announce(void) 418{ 419 int i, x; 420 421 POSTCODE(MP_ANNOUNCE_POST); 422 423 /* List CPUs */ 424 printf(" cpu0 (BSP): APIC ID: %2d\n", boot_cpu_id); 425 for (i = 1, x = 0; x < MAXCPU; x++) { 426 if (cpu_info[x].cpu_present && !cpu_info[x].cpu_bsp) { 427 KASSERT(i < mp_ncpus, 428 ("mp_ncpus and actual cpus are out of whack")); 429 printf(" cpu%d (AP): APIC ID: %2d\n", i++, x); 430 } 431 } 432} 433 434/* 435 * AP CPU's call this to initialize themselves. 436 */ 437void 438init_secondary(void) 439{ 440 int gsel_tss; 441 int x, myid; 442 u_int cr0; 443 444 /* bootAP is set in start_ap() to our ID. */ 445 myid = bootAP; 446 gdt_segs[GPRIV_SEL].ssd_base = (int) &SMP_prvspace[myid]; 447 gdt_segs[GPROC0_SEL].ssd_base = 448 (int) &SMP_prvspace[myid].pcpu.pc_common_tss; 449 SMP_prvspace[myid].pcpu.pc_prvspace = 450 &SMP_prvspace[myid].pcpu; 451 452 for (x = 0; x < NGDT; x++) { 453 ssdtosd(&gdt_segs[x], &gdt[myid * NGDT + x].sd); 454 } 455 456 r_gdt.rd_limit = NGDT * sizeof(gdt[0]) - 1; 457 r_gdt.rd_base = (int) &gdt[myid * NGDT]; 458 lgdt(&r_gdt); /* does magic intra-segment return */ 459 460 lidt(&r_idt); 461 462 lldt(_default_ldt); 463 PCPU_SET(currentldt, _default_ldt); 464 465 gsel_tss = GSEL(GPROC0_SEL, SEL_KPL); 466 gdt[myid * NGDT + GPROC0_SEL].sd.sd_type = SDT_SYS386TSS; 467 PCPU_SET(common_tss.tss_esp0, 0); /* not used until after switch */ 468 PCPU_SET(common_tss.tss_ss0, GSEL(GDATA_SEL, SEL_KPL)); 469 PCPU_SET(common_tss.tss_ioopt, (sizeof (struct i386tss)) << 16); 470 PCPU_SET(tss_gdt, &gdt[myid * NGDT + GPROC0_SEL].sd); 471 PCPU_SET(common_tssd, *PCPU_GET(tss_gdt)); 472 ltr(gsel_tss); 473 474 /* 475 * Set to a known state: 476 * Set by mpboot.s: CR0_PG, CR0_PE 477 * Set by cpu_setregs: CR0_NE, CR0_MP, CR0_TS, CR0_WP, CR0_AM 478 */ 479 cr0 = rcr0(); 480 cr0 &= ~(CR0_CD | CR0_NW | CR0_EM); 481 load_cr0(cr0); 482 CHECK_WRITE(0x38, 5); 483 484 /* Disable local APIC just to be sure. */ 485 lapic_disable(); 486 487 /* signal our startup to the BSP. */ 488 mp_naps++; 489 CHECK_WRITE(0x39, 6); 490 491 /* Spin until the BSP releases the AP's. */ 492 while (!aps_ready) 493 ia32_pause(); 494 495 /* BSP may have changed PTD while we were waiting */ 496 invltlb(); 497 pmap_invalidate_range(kernel_pmap, 0, NKPT * NBPDR - 1); 498 499#if defined(I586_CPU) && !defined(NO_F00F_HACK) 500 lidt(&r_idt); 501#endif 502 503 /* set up CPU registers and state */ 504 cpu_setregs(); 505 506 /* set up FPU state on the AP */ 507 npxinit(__INITIAL_NPXCW__); 508 509 /* set up SSE registers */ 510 enable_sse(); 511 512 /* A quick check from sanity claus */ 513 if (PCPU_GET(apic_id) != lapic_id()) { 514 printf("SMP: cpuid = %d\n", PCPU_GET(cpuid)); 515 printf("SMP: actual apic_id = %d\n", lapic_id()); 516 printf("SMP: correct apic_id = %d\n", PCPU_GET(apic_id)); 517 printf("PTD[MPPTDI] = %#jx\n", (uintmax_t)PTD[MPPTDI]); 518 panic("cpuid mismatch! boom!!"); 519 } 520 521 mtx_lock_spin(&ap_boot_mtx); 522 523 /* Init local apic for irq's */ 524 lapic_setup(); 525 526 /* Set memory range attributes for this CPU to match the BSP */ 527 mem_range_AP_init(); 528 529 smp_cpus++; 530 531 CTR1(KTR_SMP, "SMP: AP CPU #%d Launched", PCPU_GET(cpuid)); 532 printf("SMP: AP CPU #%d Launched!\n", PCPU_GET(cpuid)); 533 534 /* Determine if we are a logical CPU. */ 535 if (logical_cpus > 1 && PCPU_GET(apic_id) % logical_cpus != 0) 536 logical_cpus_mask |= PCPU_GET(cpumask); 537 538 /* Build our map of 'other' CPUs. */ 539 PCPU_SET(other_cpus, all_cpus & ~PCPU_GET(cpumask)); 540 541 if (bootverbose) 542 lapic_dump("AP"); 543 544 if (smp_cpus == mp_ncpus) { 545 /* enable IPI's, tlb shootdown, freezes etc */ 546 atomic_store_rel_int(&smp_started, 1); 547 smp_active = 1; /* historic */ 548 } 549 550 mtx_unlock_spin(&ap_boot_mtx); 551 552 /* wait until all the AP's are up */ 553 while (smp_started == 0) 554 ia32_pause(); 555 556 /* ok, now grab sched_lock and enter the scheduler */ 557 mtx_lock_spin(&sched_lock); 558 559 binuptime(PCPU_PTR(switchtime)); 560 PCPU_SET(switchticks, ticks); 561 562 cpu_throw(NULL, choosethread()); /* doesn't return */ 563 564 panic("scheduler returned us to %s", __func__); 565 /* NOTREACHED */ 566} 567 568/******************************************************************* 569 * local functions and data 570 */ 571 572/* 573 * Set the APIC logical IDs. 574 * 575 * We want to cluster logical CPU's within the same APIC ID cluster. 576 * Since logical CPU's are aligned simply filling in the clusters in 577 * APIC ID order works fine. Note that this does not try to balance 578 * the number of CPU's in each cluster. (XXX?) 579 */ 580static void 581set_logical_apic_ids(void) 582{ 583 u_int apic_id, cluster, cluster_id; 584 585 /* Force us to allocate cluster 0 at the start. */ 586 cluster = -1; 587 cluster_id = APIC_MAX_INTRACLUSTER_ID; 588 for (apic_id = 0; apic_id < MAXCPU; apic_id++) { 589 if (!cpu_info[apic_id].cpu_present) 590 continue; 591 if (cluster_id == APIC_MAX_INTRACLUSTER_ID) { 592 cluster = ioapic_next_logical_cluster(); 593 cluster_id = 0; 594 } else 595 cluster_id++; 596 if (bootverbose) 597 printf("APIC ID: physical %u, logical %u:%u\n", 598 apic_id, cluster, cluster_id); 599 lapic_set_logical_id(apic_id, cluster, cluster_id); 600 } 601} 602 603/* 604 * start each AP in our list 605 */ 606static int 607start_all_aps(void) 608{ 609#ifndef PC98 610 u_char mpbiosreason; 611#endif 612 u_long mpbioswarmvec; 613 struct pcpu *pc; 614 char *stack; 615 uintptr_t kptbase; 616 int i, pg, apic_id, cpu; 617 618 POSTCODE(START_ALL_APS_POST); 619 620 mtx_init(&ap_boot_mtx, "ap boot", NULL, MTX_SPIN); 621 622 /* install the AP 1st level boot code */ 623 install_ap_tramp(); 624 625 /* save the current value of the warm-start vector */ 626 mpbioswarmvec = *((u_long *) WARMBOOT_OFF); 627#ifndef PC98 628 outb(CMOS_REG, BIOS_RESET); 629 mpbiosreason = inb(CMOS_DATA); 630#endif 631 632 /* set up temporary P==V mapping for AP boot */ 633 /* XXX this is a hack, we should boot the AP on its own stack/PTD */ 634 kptbase = (uintptr_t)(void *)KPTphys; 635 for (i = 0; i < NKPT; i++) 636 PTD[i] = (pd_entry_t)(PG_V | PG_RW | 637 ((kptbase + i * PAGE_SIZE) & PG_FRAME)); 638 invltlb(); 639 640 /* start each AP */ 641 for (cpu = 0, apic_id = 0; apic_id < MAXCPU; apic_id++) { 642 if (!cpu_info[apic_id].cpu_present || 643 cpu_info[apic_id].cpu_bsp) 644 continue; 645 cpu++; 646 647 /* save APIC ID for this logical ID */ 648 cpu_apic_ids[cpu] = apic_id; 649 650 /* first page of AP's private space */ 651 pg = cpu * i386_btop(sizeof(struct privatespace)); 652 653 /* allocate a new private data page */ 654 pc = (struct pcpu *)kmem_alloc(kernel_map, PAGE_SIZE); 655 656 /* wire it into the private page table page */ 657 SMPpt[pg] = (pt_entry_t)(PG_V | PG_RW | vtophys(pc)); 658 659 /* allocate and set up an idle stack data page */ 660 stack = (char *)kmem_alloc(kernel_map, KSTACK_PAGES * PAGE_SIZE); /* XXXKSE */ 661 for (i = 0; i < KSTACK_PAGES; i++) 662 SMPpt[pg + 1 + i] = (pt_entry_t) 663 (PG_V | PG_RW | vtophys(PAGE_SIZE * i + stack)); 664 665 /* prime data page for it to use */ 666 pcpu_init(pc, cpu, sizeof(struct pcpu)); 667 pc->pc_apic_id = apic_id; 668 669 /* setup a vector to our boot code */ 670 *((volatile u_short *) WARMBOOT_OFF) = WARMBOOT_TARGET; 671 *((volatile u_short *) WARMBOOT_SEG) = (boot_address >> 4); 672#ifndef PC98 673 outb(CMOS_REG, BIOS_RESET); 674 outb(CMOS_DATA, BIOS_WARM); /* 'warm-start' */ 675#endif 676 677 bootSTK = &SMP_prvspace[cpu].idlekstack[KSTACK_PAGES * 678 PAGE_SIZE]; 679 bootAP = cpu; 680 681 /* attempt to start the Application Processor */ 682 CHECK_INIT(99); /* setup checkpoints */ 683 if (!start_ap(apic_id)) { 684 printf("AP #%d (PHY# %d) failed!\n", cpu, apic_id); 685 CHECK_PRINT("trace"); /* show checkpoints */ 686 /* better panic as the AP may be running loose */ 687 printf("panic y/n? [y] "); 688 if (cngetc() != 'n') 689 panic("bye-bye"); 690 } 691 CHECK_PRINT("trace"); /* show checkpoints */ 692 693 all_cpus |= (1 << cpu); /* record AP in CPU map */ 694 } 695 696 /* build our map of 'other' CPUs */ 697 PCPU_SET(other_cpus, all_cpus & ~PCPU_GET(cpumask)); 698 699 /* restore the warmstart vector */ 700 *(u_long *) WARMBOOT_OFF = mpbioswarmvec; 701#ifndef PC98 702 outb(CMOS_REG, BIOS_RESET); 703 outb(CMOS_DATA, mpbiosreason); 704#endif 705 706 /* 707 * Set up the idle context for the BSP. Similar to above except 708 * that some was done by locore, some by pmap.c and some is implicit 709 * because the BSP is cpu#0 and the page is initially zero and also 710 * because we can refer to variables by name on the BSP.. 711 */ 712 713 /* Allocate and setup BSP idle stack */ 714 stack = (char *)kmem_alloc(kernel_map, KSTACK_PAGES * PAGE_SIZE); 715 for (i = 0; i < KSTACK_PAGES; i++) 716 SMPpt[1 + i] = (pt_entry_t) 717 (PG_V | PG_RW | vtophys(PAGE_SIZE * i + stack)); 718 719 for (i = 0; i < NKPT; i++) 720 PTD[i] = 0; 721 pmap_invalidate_range(kernel_pmap, 0, NKPT * NBPDR - 1); 722 723 /* number of APs actually started */ 724 return mp_naps; 725} 726 727/* 728 * load the 1st level AP boot code into base memory. 729 */ 730 731/* targets for relocation */ 732extern void bigJump(void); 733extern void bootCodeSeg(void); 734extern void bootDataSeg(void); 735extern void MPentry(void); 736extern u_int MP_GDT; 737extern u_int mp_gdtbase; 738 739static void 740install_ap_tramp(void) 741{ 742 int x; 743 int size = *(int *) ((u_long) & bootMP_size); 744 u_char *src = (u_char *) ((u_long) bootMP); 745 u_char *dst = (u_char *) boot_address + KERNBASE; 746 u_int boot_base = (u_int) bootMP; 747 u_int8_t *dst8; 748 u_int16_t *dst16; 749 u_int32_t *dst32; 750 751 POSTCODE(INSTALL_AP_TRAMP_POST); 752 753 pmap_kenter(boot_address + KERNBASE, boot_address); 754 for (x = 0; x < size; ++x) 755 *dst++ = *src++; 756 757 /* 758 * modify addresses in code we just moved to basemem. unfortunately we 759 * need fairly detailed info about mpboot.s for this to work. changes 760 * to mpboot.s might require changes here. 761 */ 762 763 /* boot code is located in KERNEL space */ 764 dst = (u_char *) boot_address + KERNBASE; 765 766 /* modify the lgdt arg */ 767 dst32 = (u_int32_t *) (dst + ((u_int) & mp_gdtbase - boot_base)); 768 *dst32 = boot_address + ((u_int) & MP_GDT - boot_base); 769 770 /* modify the ljmp target for MPentry() */ 771 dst32 = (u_int32_t *) (dst + ((u_int) bigJump - boot_base) + 1); 772 *dst32 = ((u_int) MPentry - KERNBASE); 773 774 /* modify the target for boot code segment */ 775 dst16 = (u_int16_t *) (dst + ((u_int) bootCodeSeg - boot_base)); 776 dst8 = (u_int8_t *) (dst16 + 1); 777 *dst16 = (u_int) boot_address & 0xffff; 778 *dst8 = ((u_int) boot_address >> 16) & 0xff; 779 780 /* modify the target for boot data segment */ 781 dst16 = (u_int16_t *) (dst + ((u_int) bootDataSeg - boot_base)); 782 dst8 = (u_int8_t *) (dst16 + 1); 783 *dst16 = (u_int) boot_address & 0xffff; 784 *dst8 = ((u_int) boot_address >> 16) & 0xff; 785} 786 787/* 788 * This function starts the AP (application processor) identified 789 * by the APIC ID 'physicalCpu'. It does quite a "song and dance" 790 * to accomplish this. This is necessary because of the nuances 791 * of the different hardware we might encounter. It isn't pretty, 792 * but it seems to work. 793 */ 794static int 795start_ap(int apic_id) 796{ 797 int vector, ms; 798 int cpus; 799 800 POSTCODE(START_AP_POST); 801 802 /* calculate the vector */ 803 vector = (boot_address >> 12) & 0xff; 804 805 /* used as a watchpoint to signal AP startup */ 806 cpus = mp_naps; 807 808 /* 809 * first we do an INIT/RESET IPI this INIT IPI might be run, reseting 810 * and running the target CPU. OR this INIT IPI might be latched (P5 811 * bug), CPU waiting for STARTUP IPI. OR this INIT IPI might be 812 * ignored. 813 */ 814 815 /* do an INIT IPI: assert RESET */ 816 lapic_ipi_raw(APIC_DEST_DESTFLD | APIC_TRIGMOD_EDGE | 817 APIC_LEVEL_ASSERT | APIC_DESTMODE_PHY | APIC_DELMODE_INIT, apic_id); 818 819 /* wait for pending status end */ 820 lapic_ipi_wait(-1); 821 822 /* do an INIT IPI: deassert RESET */ 823 lapic_ipi_raw(APIC_DEST_ALLESELF | APIC_TRIGMOD_LEVEL | 824 APIC_LEVEL_DEASSERT | APIC_DESTMODE_PHY | APIC_DELMODE_INIT, 0); 825 826 /* wait for pending status end */ 827 DELAY(10000); /* wait ~10mS */ 828 lapic_ipi_wait(-1); 829 830 /* 831 * next we do a STARTUP IPI: the previous INIT IPI might still be 832 * latched, (P5 bug) this 1st STARTUP would then terminate 833 * immediately, and the previously started INIT IPI would continue. OR 834 * the previous INIT IPI has already run. and this STARTUP IPI will 835 * run. OR the previous INIT IPI was ignored. and this STARTUP IPI 836 * will run. 837 */ 838 839 /* do a STARTUP IPI */ 840 lapic_ipi_raw(APIC_DEST_DESTFLD | APIC_TRIGMOD_EDGE | 841 APIC_LEVEL_DEASSERT | APIC_DESTMODE_PHY | APIC_DELMODE_STARTUP | 842 vector, apic_id); 843 lapic_ipi_wait(-1); 844 DELAY(200); /* wait ~200uS */ 845 846 /* 847 * finally we do a 2nd STARTUP IPI: this 2nd STARTUP IPI should run IF 848 * the previous STARTUP IPI was cancelled by a latched INIT IPI. OR 849 * this STARTUP IPI will be ignored, as only ONE STARTUP IPI is 850 * recognized after hardware RESET or INIT IPI. 851 */ 852 853 lapic_ipi_raw(APIC_DEST_DESTFLD | APIC_TRIGMOD_EDGE | 854 APIC_LEVEL_DEASSERT | APIC_DESTMODE_PHY | APIC_DELMODE_STARTUP | 855 vector, apic_id); 856 lapic_ipi_wait(-1); 857 DELAY(200); /* wait ~200uS */ 858 859 /* Wait up to 5 seconds for it to start. */ 860 for (ms = 0; ms < 5000; ms++) { 861 if (mp_naps > cpus) 862 return 1; /* return SUCCESS */ 863 DELAY(1000); 864 } 865 return 0; /* return FAILURE */ 866} 867 868#ifdef COUNT_XINVLTLB_HITS 869u_int xhits_gbl[MAXCPU]; 870u_int xhits_pg[MAXCPU]; 871u_int xhits_rng[MAXCPU]; 872SYSCTL_NODE(_debug, OID_AUTO, xhits, CTLFLAG_RW, 0, ""); 873SYSCTL_OPAQUE(_debug_xhits, OID_AUTO, global, CTLFLAG_RW, &xhits_gbl, 874 sizeof(xhits_gbl), "IU", ""); 875SYSCTL_OPAQUE(_debug_xhits, OID_AUTO, page, CTLFLAG_RW, &xhits_pg, 876 sizeof(xhits_pg), "IU", ""); 877SYSCTL_OPAQUE(_debug_xhits, OID_AUTO, range, CTLFLAG_RW, &xhits_rng, 878 sizeof(xhits_rng), "IU", ""); 879 880u_int ipi_global; 881u_int ipi_page; 882u_int ipi_range; 883u_int ipi_range_size; 884SYSCTL_INT(_debug_xhits, OID_AUTO, ipi_global, CTLFLAG_RW, &ipi_global, 0, ""); 885SYSCTL_INT(_debug_xhits, OID_AUTO, ipi_page, CTLFLAG_RW, &ipi_page, 0, ""); 886SYSCTL_INT(_debug_xhits, OID_AUTO, ipi_range, CTLFLAG_RW, &ipi_range, 0, ""); 887SYSCTL_INT(_debug_xhits, OID_AUTO, ipi_range_size, CTLFLAG_RW, &ipi_range_size, 888 0, ""); 889 890u_int ipi_masked_global; 891u_int ipi_masked_page; 892u_int ipi_masked_range; 893u_int ipi_masked_range_size; 894SYSCTL_INT(_debug_xhits, OID_AUTO, ipi_masked_global, CTLFLAG_RW, 895 &ipi_masked_global, 0, ""); 896SYSCTL_INT(_debug_xhits, OID_AUTO, ipi_masked_page, CTLFLAG_RW, 897 &ipi_masked_page, 0, ""); 898SYSCTL_INT(_debug_xhits, OID_AUTO, ipi_masked_range, CTLFLAG_RW, 899 &ipi_masked_range, 0, ""); 900SYSCTL_INT(_debug_xhits, OID_AUTO, ipi_masked_range_size, CTLFLAG_RW, 901 &ipi_masked_range_size, 0, ""); 902#endif /* COUNT_XINVLTLB_HITS */ 903 904/* 905 * Flush the TLB on all other CPU's 906 */ 907static void 908smp_tlb_shootdown(u_int vector, vm_offset_t addr1, vm_offset_t addr2) 909{ 910 u_int ncpu; 911 912 ncpu = mp_ncpus - 1; /* does not shootdown self */ 913 if (ncpu < 1) 914 return; /* no other cpus */ 915 mtx_assert(&smp_tlb_mtx, MA_OWNED); 916 smp_tlb_addr1 = addr1; 917 smp_tlb_addr2 = addr2; 918 atomic_store_rel_int(&smp_tlb_wait, 0); 919 ipi_all_but_self(vector); 920 while (smp_tlb_wait < ncpu) 921 ia32_pause(); 922} 923 924/* 925 * This is about as magic as it gets. fortune(1) has got similar code 926 * for reversing bits in a word. Who thinks up this stuff?? 927 * 928 * Yes, it does appear to be consistently faster than: 929 * while (i = ffs(m)) { 930 * m >>= i; 931 * bits++; 932 * } 933 * and 934 * while (lsb = (m & -m)) { // This is magic too 935 * m &= ~lsb; // or: m ^= lsb 936 * bits++; 937 * } 938 * Both of these latter forms do some very strange things on gcc-3.1 with 939 * -mcpu=pentiumpro and/or -march=pentiumpro and/or -O or -O2. 940 * There is probably an SSE or MMX popcnt instruction. 941 * 942 * I wonder if this should be in libkern? 943 * 944 * XXX Stop the presses! Another one: 945 * static __inline u_int32_t 946 * popcnt1(u_int32_t v) 947 * { 948 * v -= ((v >> 1) & 0x55555555); 949 * v = (v & 0x33333333) + ((v >> 2) & 0x33333333); 950 * v = (v + (v >> 4)) & 0x0F0F0F0F; 951 * return (v * 0x01010101) >> 24; 952 * } 953 * The downside is that it has a multiply. With a pentium3 with 954 * -mcpu=pentiumpro and -march=pentiumpro then gcc-3.1 will use 955 * an imull, and in that case it is faster. In most other cases 956 * it appears slightly slower. 957 * 958 * Another variant (also from fortune): 959 * #define BITCOUNT(x) (((BX_(x)+(BX_(x)>>4)) & 0x0F0F0F0F) % 255) 960 * #define BX_(x) ((x) - (((x)>>1)&0x77777777) \ 961 * - (((x)>>2)&0x33333333) \ 962 * - (((x)>>3)&0x11111111)) 963 */ 964static __inline u_int32_t 965popcnt(u_int32_t m) 966{ 967 968 m = (m & 0x55555555) + ((m & 0xaaaaaaaa) >> 1); 969 m = (m & 0x33333333) + ((m & 0xcccccccc) >> 2); 970 m = (m & 0x0f0f0f0f) + ((m & 0xf0f0f0f0) >> 4); 971 m = (m & 0x00ff00ff) + ((m & 0xff00ff00) >> 8); 972 m = (m & 0x0000ffff) + ((m & 0xffff0000) >> 16); 973 return m; 974} 975 976static void 977smp_targeted_tlb_shootdown(u_int mask, u_int vector, vm_offset_t addr1, vm_offset_t addr2) 978{ 979 int ncpu, othercpus; 980 981 othercpus = mp_ncpus - 1; 982 if (mask == (u_int)-1) { 983 ncpu = othercpus; 984 if (ncpu < 1) 985 return; 986 } else { 987 mask &= ~PCPU_GET(cpumask); 988 if (mask == 0) 989 return; 990 ncpu = popcnt(mask); 991 if (ncpu > othercpus) { 992 /* XXX this should be a panic offence */ 993 printf("SMP: tlb shootdown to %d other cpus (only have %d)\n", 994 ncpu, othercpus); 995 ncpu = othercpus; 996 } 997 /* XXX should be a panic, implied by mask == 0 above */ 998 if (ncpu < 1) 999 return; 1000 } 1001 mtx_assert(&smp_tlb_mtx, MA_OWNED); 1002 smp_tlb_addr1 = addr1; 1003 smp_tlb_addr2 = addr2; 1004 atomic_store_rel_int(&smp_tlb_wait, 0); 1005 if (mask == (u_int)-1) 1006 ipi_all_but_self(vector); 1007 else 1008 ipi_selected(mask, vector); 1009 while (smp_tlb_wait < ncpu) 1010 ia32_pause(); 1011} 1012 1013void 1014smp_invltlb(void) 1015{ 1016 if (smp_started) { 1017 smp_tlb_shootdown(IPI_INVLTLB, 0, 0); 1018#ifdef COUNT_XINVLTLB_HITS 1019 ipi_global++; 1020#endif 1021 } 1022} 1023 1024void 1025smp_invlpg(vm_offset_t addr) 1026{ 1027 if (smp_started) { 1028 smp_tlb_shootdown(IPI_INVLPG, addr, 0); 1029#ifdef COUNT_XINVLTLB_HITS 1030 ipi_page++; 1031#endif 1032 } 1033} 1034 1035void 1036smp_invlpg_range(vm_offset_t addr1, vm_offset_t addr2) 1037{ 1038 if (smp_started) { 1039 smp_tlb_shootdown(IPI_INVLRNG, addr1, addr2); 1040#ifdef COUNT_XINVLTLB_HITS 1041 ipi_range++; 1042 ipi_range_size += (addr2 - addr1) / PAGE_SIZE; 1043#endif 1044 } 1045} 1046 1047void 1048smp_masked_invltlb(u_int mask) 1049{ 1050 if (smp_started) { 1051 smp_targeted_tlb_shootdown(mask, IPI_INVLTLB, 0, 0); 1052#ifdef COUNT_XINVLTLB_HITS 1053 ipi_masked_global++; 1054#endif 1055 } 1056} 1057 1058void 1059smp_masked_invlpg(u_int mask, vm_offset_t addr) 1060{ 1061 if (smp_started) { 1062 smp_targeted_tlb_shootdown(mask, IPI_INVLPG, addr, 0); 1063#ifdef COUNT_XINVLTLB_HITS 1064 ipi_masked_page++; 1065#endif 1066 } 1067} 1068 1069void 1070smp_masked_invlpg_range(u_int mask, vm_offset_t addr1, vm_offset_t addr2) 1071{ 1072 if (smp_started) { 1073 smp_targeted_tlb_shootdown(mask, IPI_INVLRNG, addr1, addr2); 1074#ifdef COUNT_XINVLTLB_HITS 1075 ipi_masked_range++; 1076 ipi_masked_range_size += (addr2 - addr1) / PAGE_SIZE; 1077#endif 1078 } 1079} 1080 1081 1082/* 1083 * For statclock, we send an IPI to all CPU's to have them call this 1084 * function. 1085 */ 1086void 1087forwarded_statclock(struct clockframe frame) 1088{ 1089 struct thread *td; 1090 1091 CTR0(KTR_SMP, "forwarded_statclock"); 1092 td = curthread; 1093 td->td_intr_nesting_level++; 1094 if (profprocs != 0) 1095 profclock(&frame); 1096 if (pscnt == psdiv) 1097 statclock(&frame); 1098 td->td_intr_nesting_level--; 1099} 1100 1101void 1102forward_statclock(void) 1103{ 1104 int map; 1105 1106 CTR0(KTR_SMP, "forward_statclock"); 1107 1108 if (!smp_started || cold || panicstr) 1109 return; 1110 1111 map = PCPU_GET(other_cpus) & ~(stopped_cpus|hlt_cpus_mask); 1112 if (map != 0) 1113 ipi_selected(map, IPI_STATCLOCK); 1114} 1115 1116/* 1117 * For each hardclock(), we send an IPI to all other CPU's to have them 1118 * execute this function. It would be nice to reduce contention on 1119 * sched_lock if we could simply peek at the CPU to determine the user/kernel 1120 * state and call hardclock_process() on the CPU receiving the clock interrupt 1121 * and then just use a simple IPI to handle any ast's if needed. 1122 */ 1123void 1124forwarded_hardclock(struct clockframe frame) 1125{ 1126 struct thread *td; 1127 1128 CTR0(KTR_SMP, "forwarded_hardclock"); 1129 td = curthread; 1130 td->td_intr_nesting_level++; 1131 hardclock_process(&frame); 1132 td->td_intr_nesting_level--; 1133} 1134 1135void 1136forward_hardclock(void) 1137{ 1138 u_int map; 1139 1140 CTR0(KTR_SMP, "forward_hardclock"); 1141 1142 if (!smp_started || cold || panicstr) 1143 return; 1144 1145 map = PCPU_GET(other_cpus) & ~(stopped_cpus|hlt_cpus_mask); 1146 if (map != 0) 1147 ipi_selected(map, IPI_HARDCLOCK); 1148} 1149 1150/* 1151 * send an IPI to a set of cpus. 1152 */ 1153void 1154ipi_selected(u_int32_t cpus, u_int ipi) 1155{ 1156 int cpu; 1157 1158 CTR3(KTR_SMP, "%s: cpus: %x ipi: %x", __func__, cpus, ipi); 1159 while ((cpu = ffs(cpus)) != 0) { 1160 cpu--; 1161 KASSERT(cpu_apic_ids[cpu] != -1, 1162 ("IPI to non-existent CPU %d", cpu)); 1163 lapic_ipi_vectored(ipi, cpu_apic_ids[cpu]); 1164 cpus &= ~(1 << cpu); 1165 } 1166} 1167 1168/* 1169 * send an IPI INTerrupt containing 'vector' to all CPUs, including myself 1170 */ 1171void 1172ipi_all(u_int ipi) 1173{ 1174 1175 CTR2(KTR_SMP, "%s: ipi: %x", __func__, ipi); 1176 lapic_ipi_vectored(ipi, APIC_IPI_DEST_ALL); 1177} 1178 1179/* 1180 * send an IPI to all CPUs EXCEPT myself 1181 */ 1182void 1183ipi_all_but_self(u_int ipi) 1184{ 1185 1186 CTR2(KTR_SMP, "%s: ipi: %x", __func__, ipi); 1187 lapic_ipi_vectored(ipi, APIC_IPI_DEST_OTHERS); 1188} 1189 1190/* 1191 * send an IPI to myself 1192 */ 1193void 1194ipi_self(u_int ipi) 1195{ 1196 1197 CTR2(KTR_SMP, "%s: ipi: %x", __func__, ipi); 1198 lapic_ipi_vectored(ipi, APIC_IPI_DEST_SELF); 1199} 1200 1201/* 1202 * This is called once the rest of the system is up and running and we're 1203 * ready to let the AP's out of the pen. 1204 */ 1205static void 1206release_aps(void *dummy __unused) 1207{ 1208 1209 if (mp_ncpus == 1) 1210 return; 1211 mtx_lock_spin(&sched_lock); 1212 atomic_store_rel_int(&aps_ready, 1); 1213 while (smp_started == 0) 1214 ia32_pause(); 1215 mtx_unlock_spin(&sched_lock); 1216} 1217SYSINIT(start_aps, SI_SUB_SMP, SI_ORDER_FIRST, release_aps, NULL); 1218 1219static int 1220sysctl_hlt_cpus(SYSCTL_HANDLER_ARGS) 1221{ 1222 u_int mask; 1223 int error; 1224 1225 mask = hlt_cpus_mask; 1226 error = sysctl_handle_int(oidp, &mask, 0, req); 1227 if (error || !req->newptr) 1228 return (error); 1229 1230 if (logical_cpus_mask != 0 && 1231 (mask & logical_cpus_mask) == logical_cpus_mask) 1232 hlt_logical_cpus = 1; 1233 else 1234 hlt_logical_cpus = 0; 1235 1236 if ((mask & all_cpus) == all_cpus) 1237 mask &= ~(1<<0); 1238 hlt_cpus_mask = mask; 1239 return (error); 1240} 1241SYSCTL_PROC(_machdep, OID_AUTO, hlt_cpus, CTLTYPE_INT|CTLFLAG_RW, 1242 0, 0, sysctl_hlt_cpus, "IU", 1243 "Bitmap of CPUs to halt. 101 (binary) will halt CPUs 0 and 2."); 1244 1245static int 1246sysctl_hlt_logical_cpus(SYSCTL_HANDLER_ARGS) 1247{ 1248 int disable, error; 1249 1250 disable = hlt_logical_cpus; 1251 error = sysctl_handle_int(oidp, &disable, 0, req); 1252 if (error || !req->newptr) 1253 return (error); 1254 1255 if (disable) 1256 hlt_cpus_mask |= logical_cpus_mask; 1257 else 1258 hlt_cpus_mask &= ~logical_cpus_mask; 1259 1260 if ((hlt_cpus_mask & all_cpus) == all_cpus) 1261 hlt_cpus_mask &= ~(1<<0); 1262 1263 hlt_logical_cpus = disable; 1264 return (error); 1265} 1266 1267static void 1268cpu_hlt_setup(void *dummy __unused) 1269{ 1270 1271 if (logical_cpus_mask != 0) { 1272 TUNABLE_INT_FETCH("machdep.hlt_logical_cpus", 1273 &hlt_logical_cpus); 1274 sysctl_ctx_init(&logical_cpu_clist); 1275 SYSCTL_ADD_PROC(&logical_cpu_clist, 1276 SYSCTL_STATIC_CHILDREN(_machdep), OID_AUTO, 1277 "hlt_logical_cpus", CTLTYPE_INT|CTLFLAG_RW, 0, 0, 1278 sysctl_hlt_logical_cpus, "IU", ""); 1279 SYSCTL_ADD_UINT(&logical_cpu_clist, 1280 SYSCTL_STATIC_CHILDREN(_machdep), OID_AUTO, 1281 "logical_cpus_mask", CTLTYPE_INT|CTLFLAG_RD, 1282 &logical_cpus_mask, 0, ""); 1283 1284 if (hlt_logical_cpus) 1285 hlt_cpus_mask |= logical_cpus_mask; 1286 } 1287} 1288SYSINIT(cpu_hlt, SI_SUB_SMP, SI_ORDER_ANY, cpu_hlt_setup, NULL); 1289 1290int 1291mp_grab_cpu_hlt(void) 1292{ 1293 u_int mask = PCPU_GET(cpumask); 1294 int retval; 1295 1296 retval = mask & hlt_cpus_mask; 1297 while (mask & hlt_cpus_mask) 1298 __asm __volatile("sti; hlt" : : : "memory"); 1299 return (retval); 1300} 1301