mp_x86.c revision 111119
1/* 2 * Copyright (c) 1996, by Steve Passe 3 * All rights reserved. 4 * 5 * Redistribution and use in source and binary forms, with or without 6 * modification, are permitted provided that the following conditions 7 * are met: 8 * 1. Redistributions of source code must retain the above copyright 9 * notice, this list of conditions and the following disclaimer. 10 * 2. The name of the developer may NOT be used to endorse or promote products 11 * derived from this software without specific prior written permission. 12 * 13 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 14 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 15 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 16 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 17 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 18 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 19 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 20 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 21 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 22 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 23 * SUCH DAMAGE. 24 * 25 * $FreeBSD: head/sys/i386/i386/mp_machdep.c 111119 2003-02-19 05:47:46Z imp $ 26 */ 27 28#include "opt_cpu.h" 29#include "opt_kstack_pages.h" 30 31#ifdef SMP 32#include <machine/smptests.h> 33#else 34#error 35#endif 36 37#include <sys/param.h> 38#include <sys/systm.h> 39#include <sys/bus.h> 40#include <sys/cons.h> /* cngetc() */ 41#ifdef GPROF 42#include <sys/gmon.h> 43#endif 44#include <sys/kernel.h> 45#include <sys/ktr.h> 46#include <sys/lock.h> 47#include <sys/malloc.h> 48#include <sys/memrange.h> 49#include <sys/mutex.h> 50#include <sys/pcpu.h> 51#include <sys/proc.h> 52#include <sys/smp.h> 53#include <sys/sysctl.h> 54#include <sys/user.h> 55 56#include <vm/vm.h> 57#include <vm/vm_param.h> 58#include <vm/pmap.h> 59#include <vm/vm_kern.h> 60#include <vm/vm_extern.h> 61#include <vm/vm_map.h> 62 63#include <machine/apic.h> 64#include <machine/atomic.h> 65#include <machine/clock.h> 66#include <machine/cpu.h> 67#include <machine/cpufunc.h> 68#include <machine/mpapic.h> 69#include <machine/psl.h> 70#include <machine/segments.h> 71#include <machine/smp.h> 72#include <machine/smptests.h> /** TEST_DEFAULT_CONFIG, TEST_TEST1 */ 73#include <machine/tss.h> 74#include <machine/specialreg.h> 75#include <machine/privatespace.h> 76 77#if defined(APIC_IO) 78#include <machine/md_var.h> /* setidt() */ 79#include <i386/isa/icu.h> /* IPIs */ 80#include <i386/isa/intr_machdep.h> /* IPIs */ 81#endif /* APIC_IO */ 82 83#if defined(TEST_DEFAULT_CONFIG) 84#define MPFPS_MPFB1 TEST_DEFAULT_CONFIG 85#else 86#define MPFPS_MPFB1 mpfps->mpfb1 87#endif /* TEST_DEFAULT_CONFIG */ 88 89#define WARMBOOT_TARGET 0 90#define WARMBOOT_OFF (KERNBASE + 0x0467) 91#define WARMBOOT_SEG (KERNBASE + 0x0469) 92 93#ifdef PC98 94#define BIOS_BASE (0xe8000) 95#define BIOS_SIZE (0x18000) 96#else 97#define BIOS_BASE (0xf0000) 98#define BIOS_SIZE (0x10000) 99#endif 100#define BIOS_COUNT (BIOS_SIZE/4) 101 102#define CMOS_REG (0x70) 103#define CMOS_DATA (0x71) 104#define BIOS_RESET (0x0f) 105#define BIOS_WARM (0x0a) 106 107#define PROCENTRY_FLAG_EN 0x01 108#define PROCENTRY_FLAG_BP 0x02 109#define IOAPICENTRY_FLAG_EN 0x01 110 111 112/* MP Floating Pointer Structure */ 113typedef struct MPFPS { 114 char signature[4]; 115 void *pap; 116 u_char length; 117 u_char spec_rev; 118 u_char checksum; 119 u_char mpfb1; 120 u_char mpfb2; 121 u_char mpfb3; 122 u_char mpfb4; 123 u_char mpfb5; 124} *mpfps_t; 125 126/* MP Configuration Table Header */ 127typedef struct MPCTH { 128 char signature[4]; 129 u_short base_table_length; 130 u_char spec_rev; 131 u_char checksum; 132 u_char oem_id[8]; 133 u_char product_id[12]; 134 void *oem_table_pointer; 135 u_short oem_table_size; 136 u_short entry_count; 137 void *apic_address; 138 u_short extended_table_length; 139 u_char extended_table_checksum; 140 u_char reserved; 141} *mpcth_t; 142 143 144typedef struct PROCENTRY { 145 u_char type; 146 u_char apic_id; 147 u_char apic_version; 148 u_char cpu_flags; 149 u_long cpu_signature; 150 u_long feature_flags; 151 u_long reserved1; 152 u_long reserved2; 153} *proc_entry_ptr; 154 155typedef struct BUSENTRY { 156 u_char type; 157 u_char bus_id; 158 char bus_type[6]; 159} *bus_entry_ptr; 160 161typedef struct IOAPICENTRY { 162 u_char type; 163 u_char apic_id; 164 u_char apic_version; 165 u_char apic_flags; 166 void *apic_address; 167} *io_apic_entry_ptr; 168 169typedef struct INTENTRY { 170 u_char type; 171 u_char int_type; 172 u_short int_flags; 173 u_char src_bus_id; 174 u_char src_bus_irq; 175 u_char dst_apic_id; 176 u_char dst_apic_int; 177} *int_entry_ptr; 178 179/* descriptions of MP basetable entries */ 180typedef struct BASETABLE_ENTRY { 181 u_char type; 182 u_char length; 183 char name[16]; 184} basetable_entry; 185 186/* 187 * this code MUST be enabled here and in mpboot.s. 188 * it follows the very early stages of AP boot by placing values in CMOS ram. 189 * it NORMALLY will never be needed and thus the primitive method for enabling. 190 * 191#define CHECK_POINTS 192 */ 193 194#if defined(CHECK_POINTS) && !defined(PC98) 195#define CHECK_READ(A) (outb(CMOS_REG, (A)), inb(CMOS_DATA)) 196#define CHECK_WRITE(A,D) (outb(CMOS_REG, (A)), outb(CMOS_DATA, (D))) 197 198#define CHECK_INIT(D); \ 199 CHECK_WRITE(0x34, (D)); \ 200 CHECK_WRITE(0x35, (D)); \ 201 CHECK_WRITE(0x36, (D)); \ 202 CHECK_WRITE(0x37, (D)); \ 203 CHECK_WRITE(0x38, (D)); \ 204 CHECK_WRITE(0x39, (D)); 205 206#define CHECK_PRINT(S); \ 207 printf("%s: %d, %d, %d, %d, %d, %d\n", \ 208 (S), \ 209 CHECK_READ(0x34), \ 210 CHECK_READ(0x35), \ 211 CHECK_READ(0x36), \ 212 CHECK_READ(0x37), \ 213 CHECK_READ(0x38), \ 214 CHECK_READ(0x39)); 215 216#else /* CHECK_POINTS */ 217 218#define CHECK_INIT(D) 219#define CHECK_PRINT(S) 220 221#endif /* CHECK_POINTS */ 222 223/* 224 * Values to send to the POST hardware. 225 */ 226#define MP_BOOTADDRESS_POST 0x10 227#define MP_PROBE_POST 0x11 228#define MPTABLE_PASS1_POST 0x12 229 230#define MP_START_POST 0x13 231#define MP_ENABLE_POST 0x14 232#define MPTABLE_PASS2_POST 0x15 233 234#define START_ALL_APS_POST 0x16 235#define INSTALL_AP_TRAMP_POST 0x17 236#define START_AP_POST 0x18 237 238#define MP_ANNOUNCE_POST 0x19 239 240static int need_hyperthreading_fixup; 241static u_int logical_cpus; 242 243/* used to hold the AP's until we are ready to release them */ 244static struct mtx ap_boot_mtx; 245 246/** XXX FIXME: where does this really belong, isa.h/isa.c perhaps? */ 247int current_postcode; 248 249/** XXX FIXME: what system files declare these??? */ 250extern struct region_descriptor r_gdt, r_idt; 251 252int bsp_apic_ready = 0; /* flags useability of BSP apic */ 253int mp_naps; /* # of Applications processors */ 254int mp_nbusses; /* # of busses */ 255int mp_napics; /* # of IO APICs */ 256int boot_cpu_id; /* designated BSP */ 257vm_offset_t cpu_apic_address; 258vm_offset_t io_apic_address[NAPICID]; /* NAPICID is more than enough */ 259extern int nkpt; 260 261u_int32_t cpu_apic_versions[MAXCPU]; 262u_int32_t *io_apic_versions; 263 264#ifdef APIC_INTR_REORDER 265struct { 266 volatile int *location; 267 int bit; 268} apic_isrbit_location[32]; 269#endif 270 271struct apic_intmapinfo int_to_apicintpin[APIC_INTMAPSIZE]; 272 273/* 274 * APIC ID logical/physical mapping structures. 275 * We oversize these to simplify boot-time config. 276 */ 277int cpu_num_to_apic_id[NAPICID]; 278int io_num_to_apic_id[NAPICID]; 279int apic_id_to_logical[NAPICID]; 280 281 282/* AP uses this during bootstrap. Do not staticize. */ 283char *bootSTK; 284static int bootAP; 285 286/* Hotwire a 0->4MB V==P mapping */ 287extern pt_entry_t *KPTphys; 288 289/* SMP page table page */ 290extern pt_entry_t *SMPpt; 291 292struct pcb stoppcbs[MAXCPU]; 293 294#ifdef APIC_IO 295/* Variables needed for SMP tlb shootdown. */ 296vm_offset_t smp_tlb_addr1; 297vm_offset_t smp_tlb_addr2; 298volatile int smp_tlb_wait; 299static struct mtx smp_tlb_mtx; 300#endif 301 302/* 303 * Local data and functions. 304 */ 305 306/* Set to 1 once we're ready to let the APs out of the pen. */ 307static volatile int aps_ready = 0; 308 309static int mp_capable; 310static u_int boot_address; 311static u_int base_memory; 312 313static int picmode; /* 0: virtual wire mode, 1: PIC mode */ 314static mpfps_t mpfps; 315static int search_for_sig(u_int32_t target, int count); 316static void mp_enable(u_int boot_addr); 317 318static void mptable_hyperthread_fixup(u_int id_mask); 319static void mptable_pass1(void); 320static int mptable_pass2(void); 321static void default_mp_table(int type); 322static void fix_mp_table(void); 323static void setup_apic_irq_mapping(void); 324static void init_locks(void); 325static int start_all_aps(u_int boot_addr); 326static void install_ap_tramp(u_int boot_addr); 327static int start_ap(int logicalCpu, u_int boot_addr); 328void ap_init(void); 329static int apic_int_is_bus_type(int intr, int bus_type); 330static void release_aps(void *dummy); 331 332/* 333 * initialize all the SMP locks 334 */ 335 336/* lock region used by kernel profiling */ 337int mcount_lock; 338 339#ifdef USE_COMLOCK 340/* locks com (tty) data/hardware accesses: a FASTINTR() */ 341struct mtx com_mtx; 342#endif /* USE_COMLOCK */ 343 344static void 345init_locks(void) 346{ 347 348#ifdef USE_COMLOCK 349 mtx_init(&com_mtx, "com", NULL, MTX_SPIN); 350#endif /* USE_COMLOCK */ 351#ifdef APIC_IO 352 mtx_init(&smp_tlb_mtx, "tlb", NULL, MTX_SPIN); 353#endif 354} 355 356/* 357 * Calculate usable address in base memory for AP trampoline code. 358 */ 359u_int 360mp_bootaddress(u_int basemem) 361{ 362 POSTCODE(MP_BOOTADDRESS_POST); 363 364 base_memory = basemem * 1024; /* convert to bytes */ 365 366 boot_address = base_memory & ~0xfff; /* round down to 4k boundary */ 367 if ((base_memory - boot_address) < bootMP_size) 368 boot_address -= 4096; /* not enough, lower by 4k */ 369 370 return boot_address; 371} 372 373 374/* 375 * Look for an Intel MP spec table (ie, SMP capable hardware). 376 */ 377void 378i386_mp_probe(void) 379{ 380 int x; 381 u_long segment; 382 u_int32_t target; 383 384 POSTCODE(MP_PROBE_POST); 385 386 /* see if EBDA exists */ 387 if ((segment = (u_long) * (u_short *) (KERNBASE + 0x40e)) != 0) { 388 /* search first 1K of EBDA */ 389 target = (u_int32_t) (segment << 4); 390 if ((x = search_for_sig(target, 1024 / 4)) >= 0) 391 goto found; 392 } else { 393 /* last 1K of base memory, effective 'top of base' passed in */ 394 target = (u_int32_t) (base_memory - 0x400); 395 if ((x = search_for_sig(target, 1024 / 4)) >= 0) 396 goto found; 397 } 398 399 /* search the BIOS */ 400 target = (u_int32_t) BIOS_BASE; 401 if ((x = search_for_sig(target, BIOS_COUNT)) >= 0) 402 goto found; 403 404 /* nothing found */ 405 mpfps = (mpfps_t)0; 406 mp_capable = 0; 407 return; 408 409found: 410 /* calculate needed resources */ 411 mpfps = (mpfps_t)x; 412 mptable_pass1(); 413 414 /* flag fact that we are running multiple processors */ 415 mp_capable = 1; 416} 417 418int 419cpu_mp_probe(void) 420{ 421 /* 422 * Record BSP in CPU map 423 * This is done here so that MBUF init code works correctly. 424 */ 425 all_cpus = 1; 426 427 return (mp_capable); 428} 429 430/* 431 * Initialize the SMP hardware and the APIC and start up the AP's. 432 */ 433void 434cpu_mp_start(void) 435{ 436 POSTCODE(MP_START_POST); 437 438 /* look for MP capable motherboard */ 439 if (mp_capable) 440 mp_enable(boot_address); 441 else 442 panic("MP hardware not found!"); 443 444 cpu_setregs(); 445} 446 447 448/* 449 * Print various information about the SMP system hardware and setup. 450 */ 451void 452cpu_mp_announce(void) 453{ 454 int x; 455 456 POSTCODE(MP_ANNOUNCE_POST); 457 458 printf(" cpu0 (BSP): apic id: %2d", CPU_TO_ID(0)); 459 printf(", version: 0x%08x", cpu_apic_versions[0]); 460 printf(", at 0x%08x\n", cpu_apic_address); 461 for (x = 1; x <= mp_naps; ++x) { 462 printf(" cpu%d (AP): apic id: %2d", x, CPU_TO_ID(x)); 463 printf(", version: 0x%08x", cpu_apic_versions[x]); 464 printf(", at 0x%08x\n", cpu_apic_address); 465 } 466 467#if defined(APIC_IO) 468 for (x = 0; x < mp_napics; ++x) { 469 printf(" io%d (APIC): apic id: %2d", x, IO_TO_ID(x)); 470 printf(", version: 0x%08x", io_apic_versions[x]); 471 printf(", at 0x%08x\n", io_apic_address[x]); 472 } 473#else 474 printf(" Warning: APIC I/O disabled\n"); 475#endif /* APIC_IO */ 476} 477 478/* 479 * AP cpu's call this to sync up protected mode. 480 */ 481void 482init_secondary(void) 483{ 484 int gsel_tss; 485 int x, myid = bootAP; 486 u_int cr0; 487 488 gdt_segs[GPRIV_SEL].ssd_base = (int) &SMP_prvspace[myid]; 489 gdt_segs[GPROC0_SEL].ssd_base = 490 (int) &SMP_prvspace[myid].pcpu.pc_common_tss; 491 SMP_prvspace[myid].pcpu.pc_prvspace = 492 &SMP_prvspace[myid].pcpu; 493 494 for (x = 0; x < NGDT; x++) { 495 ssdtosd(&gdt_segs[x], &gdt[myid * NGDT + x].sd); 496 } 497 498 r_gdt.rd_limit = NGDT * sizeof(gdt[0]) - 1; 499 r_gdt.rd_base = (int) &gdt[myid * NGDT]; 500 lgdt(&r_gdt); /* does magic intra-segment return */ 501 502 lidt(&r_idt); 503 504 lldt(_default_ldt); 505 PCPU_SET(currentldt, _default_ldt); 506 507 gsel_tss = GSEL(GPROC0_SEL, SEL_KPL); 508 gdt[myid * NGDT + GPROC0_SEL].sd.sd_type = SDT_SYS386TSS; 509 PCPU_SET(common_tss.tss_esp0, 0); /* not used until after switch */ 510 PCPU_SET(common_tss.tss_ss0, GSEL(GDATA_SEL, SEL_KPL)); 511 PCPU_SET(common_tss.tss_ioopt, (sizeof (struct i386tss)) << 16); 512 PCPU_SET(tss_gdt, &gdt[myid * NGDT + GPROC0_SEL].sd); 513 PCPU_SET(common_tssd, *PCPU_GET(tss_gdt)); 514 ltr(gsel_tss); 515 516 /* 517 * Set to a known state: 518 * Set by mpboot.s: CR0_PG, CR0_PE 519 * Set by cpu_setregs: CR0_NE, CR0_MP, CR0_TS, CR0_WP, CR0_AM 520 */ 521 cr0 = rcr0(); 522 cr0 &= ~(CR0_CD | CR0_NW | CR0_EM); 523 load_cr0(cr0); 524 525 pmap_set_opt(); 526} 527 528 529#if defined(APIC_IO) 530/* 531 * Final configuration of the BSP's local APIC: 532 * - disable 'pic mode'. 533 * - disable 'virtual wire mode'. 534 * - enable NMI. 535 */ 536void 537bsp_apic_configure(void) 538{ 539 u_char byte; 540 u_int32_t temp; 541 542 /* leave 'pic mode' if necessary */ 543 if (picmode) { 544 outb(0x22, 0x70); /* select IMCR */ 545 byte = inb(0x23); /* current contents */ 546 byte |= 0x01; /* mask external INTR */ 547 outb(0x23, byte); /* disconnect 8259s/NMI */ 548 } 549 550 /* mask lint0 (the 8259 'virtual wire' connection) */ 551 temp = lapic.lvt_lint0; 552 temp |= APIC_LVT_M; /* set the mask */ 553 lapic.lvt_lint0 = temp; 554 555 /* setup lint1 to handle NMI */ 556 temp = lapic.lvt_lint1; 557 temp &= ~APIC_LVT_M; /* clear the mask */ 558 lapic.lvt_lint1 = temp; 559 560 if (bootverbose) 561 apic_dump("bsp_apic_configure()"); 562} 563#endif /* APIC_IO */ 564 565 566/******************************************************************* 567 * local functions and data 568 */ 569 570/* 571 * start the SMP system 572 */ 573static void 574mp_enable(u_int boot_addr) 575{ 576 int x; 577#if defined(APIC_IO) 578 int apic; 579 u_int ux; 580#endif /* APIC_IO */ 581 582 POSTCODE(MP_ENABLE_POST); 583 584 /* turn on 4MB of V == P addressing so we can get to MP table */ 585 *(int *)PTD = PG_V | PG_RW | ((uintptr_t)(void *)KPTphys & PG_FRAME); 586 invltlb(); 587 588 /* examine the MP table for needed info, uses physical addresses */ 589 x = mptable_pass2(); 590 591 *(int *)PTD = 0; 592 invltlb(); 593 594 /* can't process default configs till the CPU APIC is pmapped */ 595 if (x) 596 default_mp_table(x); 597 598 /* post scan cleanup */ 599 fix_mp_table(); 600 setup_apic_irq_mapping(); 601 602#if defined(APIC_IO) 603 604 /* fill the LOGICAL io_apic_versions table */ 605 for (apic = 0; apic < mp_napics; ++apic) { 606 ux = io_apic_read(apic, IOAPIC_VER); 607 io_apic_versions[apic] = ux; 608 io_apic_set_id(apic, IO_TO_ID(apic)); 609 } 610 611 /* program each IO APIC in the system */ 612 for (apic = 0; apic < mp_napics; ++apic) 613 if (io_apic_setup(apic) < 0) 614 panic("IO APIC setup failure"); 615 616 /* install a 'Spurious INTerrupt' vector */ 617 setidt(XSPURIOUSINT_OFFSET, Xspuriousint, 618 SDT_SYS386IGT, SEL_KPL, GSEL(GCODE_SEL, SEL_KPL)); 619 620 /* install an inter-CPU IPI for TLB invalidation */ 621 setidt(XINVLTLB_OFFSET, Xinvltlb, 622 SDT_SYS386IGT, SEL_KPL, GSEL(GCODE_SEL, SEL_KPL)); 623 setidt(XINVLPG_OFFSET, Xinvlpg, 624 SDT_SYS386IGT, SEL_KPL, GSEL(GCODE_SEL, SEL_KPL)); 625 setidt(XINVLRNG_OFFSET, Xinvlrng, 626 SDT_SYS386IGT, SEL_KPL, GSEL(GCODE_SEL, SEL_KPL)); 627 628 /* install an inter-CPU IPI for forwarding hardclock() */ 629 setidt(XHARDCLOCK_OFFSET, Xhardclock, 630 SDT_SYS386IGT, SEL_KPL, GSEL(GCODE_SEL, SEL_KPL)); 631 632 /* install an inter-CPU IPI for forwarding statclock() */ 633 setidt(XSTATCLOCK_OFFSET, Xstatclock, 634 SDT_SYS386IGT, SEL_KPL, GSEL(GCODE_SEL, SEL_KPL)); 635 636 /* install an inter-CPU IPI for all-CPU rendezvous */ 637 setidt(XRENDEZVOUS_OFFSET, Xrendezvous, 638 SDT_SYS386IGT, SEL_KPL, GSEL(GCODE_SEL, SEL_KPL)); 639 640 /* install an inter-CPU IPI for forcing an additional software trap */ 641 setidt(XCPUAST_OFFSET, Xcpuast, 642 SDT_SYS386IGT, SEL_KPL, GSEL(GCODE_SEL, SEL_KPL)); 643 644 /* install an inter-CPU IPI for CPU stop/restart */ 645 setidt(XCPUSTOP_OFFSET, Xcpustop, 646 SDT_SYS386IGT, SEL_KPL, GSEL(GCODE_SEL, SEL_KPL)); 647 648#if defined(TEST_TEST1) 649 /* install a "fake hardware INTerrupt" vector */ 650 setidt(XTEST1_OFFSET, Xtest1, 651 SDT_SYS386IGT, SEL_KPL, GSEL(GCODE_SEL, SEL_KPL)); 652#endif /** TEST_TEST1 */ 653 654#endif /* APIC_IO */ 655 656 /* initialize all SMP locks */ 657 init_locks(); 658 659 /* start each Application Processor */ 660 start_all_aps(boot_addr); 661} 662 663 664/* 665 * look for the MP spec signature 666 */ 667 668/* string defined by the Intel MP Spec as identifying the MP table */ 669#define MP_SIG 0x5f504d5f /* _MP_ */ 670#define NEXT(X) ((X) += 4) 671static int 672search_for_sig(u_int32_t target, int count) 673{ 674 int x; 675 u_int32_t *addr = (u_int32_t *) (KERNBASE + target); 676 677 for (x = 0; x < count; NEXT(x)) 678 if (addr[x] == MP_SIG) 679 /* make array index a byte index */ 680 return (target + (x * sizeof(u_int32_t))); 681 682 return -1; 683} 684 685 686static basetable_entry basetable_entry_types[] = 687{ 688 {0, 20, "Processor"}, 689 {1, 8, "Bus"}, 690 {2, 8, "I/O APIC"}, 691 {3, 8, "I/O INT"}, 692 {4, 8, "Local INT"} 693}; 694 695typedef struct BUSDATA { 696 u_char bus_id; 697 enum busTypes bus_type; 698} bus_datum; 699 700typedef struct INTDATA { 701 u_char int_type; 702 u_short int_flags; 703 u_char src_bus_id; 704 u_char src_bus_irq; 705 u_char dst_apic_id; 706 u_char dst_apic_int; 707 u_char int_vector; 708} io_int, local_int; 709 710typedef struct BUSTYPENAME { 711 u_char type; 712 char name[7]; 713} bus_type_name; 714 715static bus_type_name bus_type_table[] = 716{ 717 {CBUS, "CBUS"}, 718 {CBUSII, "CBUSII"}, 719 {EISA, "EISA"}, 720 {MCA, "MCA"}, 721 {UNKNOWN_BUSTYPE, "---"}, 722 {ISA, "ISA"}, 723 {MCA, "MCA"}, 724 {UNKNOWN_BUSTYPE, "---"}, 725 {UNKNOWN_BUSTYPE, "---"}, 726 {UNKNOWN_BUSTYPE, "---"}, 727 {UNKNOWN_BUSTYPE, "---"}, 728 {UNKNOWN_BUSTYPE, "---"}, 729 {PCI, "PCI"}, 730 {UNKNOWN_BUSTYPE, "---"}, 731 {UNKNOWN_BUSTYPE, "---"}, 732 {UNKNOWN_BUSTYPE, "---"}, 733 {UNKNOWN_BUSTYPE, "---"}, 734 {XPRESS, "XPRESS"}, 735 {UNKNOWN_BUSTYPE, "---"} 736}; 737/* from MP spec v1.4, table 5-1 */ 738static int default_data[7][5] = 739{ 740/* nbus, id0, type0, id1, type1 */ 741 {1, 0, ISA, 255, 255}, 742 {1, 0, EISA, 255, 255}, 743 {1, 0, EISA, 255, 255}, 744 {1, 0, MCA, 255, 255}, 745 {2, 0, ISA, 1, PCI}, 746 {2, 0, EISA, 1, PCI}, 747 {2, 0, MCA, 1, PCI} 748}; 749 750 751/* the bus data */ 752static bus_datum *bus_data; 753 754/* the IO INT data, one entry per possible APIC INTerrupt */ 755static io_int *io_apic_ints; 756 757static int nintrs; 758 759static int processor_entry(proc_entry_ptr entry, int cpu); 760static int bus_entry(bus_entry_ptr entry, int bus); 761static int io_apic_entry(io_apic_entry_ptr entry, int apic); 762static int int_entry(int_entry_ptr entry, int intr); 763static int lookup_bus_type(char *name); 764 765 766/* 767 * 1st pass on motherboard's Intel MP specification table. 768 * 769 * initializes: 770 * mp_ncpus = 1 771 * 772 * determines: 773 * cpu_apic_address (common to all CPUs) 774 * io_apic_address[N] 775 * mp_naps 776 * mp_nbusses 777 * mp_napics 778 * nintrs 779 */ 780static void 781mptable_pass1(void) 782{ 783 int x; 784 mpcth_t cth; 785 int totalSize; 786 void* position; 787 int count; 788 int type; 789 u_int id_mask; 790 791 POSTCODE(MPTABLE_PASS1_POST); 792 793 /* clear various tables */ 794 for (x = 0; x < NAPICID; ++x) { 795 io_apic_address[x] = ~0; /* IO APIC address table */ 796 } 797 798 /* init everything to empty */ 799 mp_naps = 0; 800 mp_nbusses = 0; 801 mp_napics = 0; 802 nintrs = 0; 803 id_mask = 0; 804 805 /* check for use of 'default' configuration */ 806 if (MPFPS_MPFB1 != 0) { 807 /* use default addresses */ 808 cpu_apic_address = DEFAULT_APIC_BASE; 809 io_apic_address[0] = DEFAULT_IO_APIC_BASE; 810 811 /* fill in with defaults */ 812 mp_naps = 2; /* includes BSP */ 813 mp_maxid = 1; 814 mp_nbusses = default_data[MPFPS_MPFB1 - 1][0]; 815#if defined(APIC_IO) 816 mp_napics = 1; 817 nintrs = 16; 818#endif /* APIC_IO */ 819 } 820 else { 821 if ((cth = mpfps->pap) == 0) 822 panic("MP Configuration Table Header MISSING!"); 823 824 cpu_apic_address = (vm_offset_t) cth->apic_address; 825 826 /* walk the table, recording info of interest */ 827 totalSize = cth->base_table_length - sizeof(struct MPCTH); 828 position = (u_char *) cth + sizeof(struct MPCTH); 829 count = cth->entry_count; 830 831 while (count--) { 832 switch (type = *(u_char *) position) { 833 case 0: /* processor_entry */ 834 if (((proc_entry_ptr)position)->cpu_flags 835 & PROCENTRY_FLAG_EN) { 836 ++mp_naps; 837 mp_maxid++; 838 id_mask |= 1 << 839 ((proc_entry_ptr)position)->apic_id; 840 } 841 break; 842 case 1: /* bus_entry */ 843 ++mp_nbusses; 844 break; 845 case 2: /* io_apic_entry */ 846 if (((io_apic_entry_ptr)position)->apic_flags 847 & IOAPICENTRY_FLAG_EN) 848 io_apic_address[mp_napics++] = 849 (vm_offset_t)((io_apic_entry_ptr) 850 position)->apic_address; 851 break; 852 case 3: /* int_entry */ 853 ++nintrs; 854 break; 855 case 4: /* int_entry */ 856 break; 857 default: 858 panic("mpfps Base Table HOSED!"); 859 /* NOTREACHED */ 860 } 861 862 totalSize -= basetable_entry_types[type].length; 863 (u_char*)position += basetable_entry_types[type].length; 864 } 865 } 866 867 /* qualify the numbers */ 868 if (mp_naps > MAXCPU) { 869 printf("Warning: only using %d of %d available CPUs!\n", 870 MAXCPU, mp_naps); 871 mp_naps = MAXCPU; 872 } 873 874 /* See if we need to fixup HT logical CPUs. */ 875 mptable_hyperthread_fixup(id_mask); 876 877 /* 878 * Count the BSP. 879 * This is also used as a counter while starting the APs. 880 */ 881 mp_ncpus = 1; 882 883 --mp_naps; /* subtract the BSP */ 884} 885 886 887/* 888 * 2nd pass on motherboard's Intel MP specification table. 889 * 890 * sets: 891 * boot_cpu_id 892 * ID_TO_IO(N), phy APIC ID to log CPU/IO table 893 * CPU_TO_ID(N), logical CPU to APIC ID table 894 * IO_TO_ID(N), logical IO to APIC ID table 895 * bus_data[N] 896 * io_apic_ints[N] 897 */ 898static int 899mptable_pass2(void) 900{ 901 struct PROCENTRY proc; 902 int x; 903 mpcth_t cth; 904 int totalSize; 905 void* position; 906 int count; 907 int type; 908 int apic, bus, cpu, intr; 909 int i, j; 910 int pgeflag; 911 912 POSTCODE(MPTABLE_PASS2_POST); 913 914 /* Initialize fake proc entry for use with HT fixup. */ 915 bzero(&proc, sizeof(proc)); 916 proc.type = 0; 917 proc.cpu_flags = PROCENTRY_FLAG_EN; 918 919 pgeflag = 0; /* XXX - Not used under SMP yet. */ 920 921 MALLOC(io_apic_versions, u_int32_t *, sizeof(u_int32_t) * mp_napics, 922 M_DEVBUF, M_WAITOK); 923 MALLOC(ioapic, volatile ioapic_t **, sizeof(ioapic_t *) * mp_napics, 924 M_DEVBUF, M_WAITOK); 925 MALLOC(io_apic_ints, io_int *, sizeof(io_int) * (nintrs + 1), 926 M_DEVBUF, M_WAITOK); 927 MALLOC(bus_data, bus_datum *, sizeof(bus_datum) * mp_nbusses, 928 M_DEVBUF, M_WAITOK); 929 930 bzero(ioapic, sizeof(ioapic_t *) * mp_napics); 931 932 for (i = 0; i < mp_napics; i++) { 933 for (j = 0; j < mp_napics; j++) { 934 /* same page frame as a previous IO apic? */ 935 if (((vm_offset_t)SMPpt[NPTEPG-2-j] & PG_FRAME) == 936 (io_apic_address[i] & PG_FRAME)) { 937 ioapic[i] = (ioapic_t *)((u_int)SMP_prvspace 938 + (NPTEPG-2-j) * PAGE_SIZE 939 + (io_apic_address[i] & PAGE_MASK)); 940 break; 941 } 942 /* use this slot if available */ 943 if (((vm_offset_t)SMPpt[NPTEPG-2-j] & PG_FRAME) == 0) { 944 SMPpt[NPTEPG-2-j] = (pt_entry_t)(PG_V | PG_RW | 945 pgeflag | (io_apic_address[i] & PG_FRAME)); 946 ioapic[i] = (ioapic_t *)((u_int)SMP_prvspace 947 + (NPTEPG-2-j) * PAGE_SIZE 948 + (io_apic_address[i] & PAGE_MASK)); 949 break; 950 } 951 } 952 } 953 954 /* clear various tables */ 955 for (x = 0; x < NAPICID; ++x) { 956 ID_TO_IO(x) = -1; /* phy APIC ID to log CPU/IO table */ 957 CPU_TO_ID(x) = -1; /* logical CPU to APIC ID table */ 958 IO_TO_ID(x) = -1; /* logical IO to APIC ID table */ 959 } 960 961 /* clear bus data table */ 962 for (x = 0; x < mp_nbusses; ++x) 963 bus_data[x].bus_id = 0xff; 964 965 /* clear IO APIC INT table */ 966 for (x = 0; x < (nintrs + 1); ++x) { 967 io_apic_ints[x].int_type = 0xff; 968 io_apic_ints[x].int_vector = 0xff; 969 } 970 971 /* setup the cpu/apic mapping arrays */ 972 boot_cpu_id = -1; 973 974 /* record whether PIC or virtual-wire mode */ 975 picmode = (mpfps->mpfb2 & 0x80) ? 1 : 0; 976 977 /* check for use of 'default' configuration */ 978 if (MPFPS_MPFB1 != 0) 979 return MPFPS_MPFB1; /* return default configuration type */ 980 981 if ((cth = mpfps->pap) == 0) 982 panic("MP Configuration Table Header MISSING!"); 983 984 /* walk the table, recording info of interest */ 985 totalSize = cth->base_table_length - sizeof(struct MPCTH); 986 position = (u_char *) cth + sizeof(struct MPCTH); 987 count = cth->entry_count; 988 apic = bus = intr = 0; 989 cpu = 1; /* pre-count the BSP */ 990 991 while (count--) { 992 switch (type = *(u_char *) position) { 993 case 0: 994 if (processor_entry(position, cpu)) 995 ++cpu; 996 997 if (need_hyperthreading_fixup) { 998 /* 999 * Create fake mptable processor entries 1000 * and feed them to processor_entry() to 1001 * enumerate the logical CPUs. 1002 */ 1003 proc.apic_id = ((proc_entry_ptr)position)->apic_id; 1004 for (i = 1; i < logical_cpus; i++) { 1005 proc.apic_id++; 1006 (void)processor_entry(&proc, cpu); 1007 cpu++; 1008 } 1009 } 1010 break; 1011 case 1: 1012 if (bus_entry(position, bus)) 1013 ++bus; 1014 break; 1015 case 2: 1016 if (io_apic_entry(position, apic)) 1017 ++apic; 1018 break; 1019 case 3: 1020 if (int_entry(position, intr)) 1021 ++intr; 1022 break; 1023 case 4: 1024 /* int_entry(position); */ 1025 break; 1026 default: 1027 panic("mpfps Base Table HOSED!"); 1028 /* NOTREACHED */ 1029 } 1030 1031 totalSize -= basetable_entry_types[type].length; 1032 (u_char *) position += basetable_entry_types[type].length; 1033 } 1034 1035 if (boot_cpu_id == -1) 1036 panic("NO BSP found!"); 1037 1038 /* report fact that its NOT a default configuration */ 1039 return 0; 1040} 1041 1042/* 1043 * Check if we should perform a hyperthreading "fix-up" to 1044 * enumerate any logical CPU's that aren't already listed 1045 * in the table. 1046 * 1047 * XXX: We assume that all of the physical CPUs in the 1048 * system have the same number of logical CPUs. 1049 * 1050 * XXX: We assume that APIC ID's are allocated such that 1051 * the APIC ID's for a physical processor are aligned 1052 * with the number of logical CPU's in the processor. 1053 */ 1054static void 1055mptable_hyperthread_fixup(u_int id_mask) 1056{ 1057 u_int i, id; 1058 1059 /* Nothing to do if there is no HTT support. */ 1060 if ((cpu_feature & CPUID_HTT) == 0) 1061 return; 1062 logical_cpus = (cpu_procinfo & CPUID_HTT_CORES) >> 16; 1063 if (logical_cpus <= 1) 1064 return; 1065 1066 /* 1067 * For each APIC ID of a CPU that is set in the mask, 1068 * scan the other candidate APIC ID's for this 1069 * physical processor. If any of those ID's are 1070 * already in the table, then kill the fixup. 1071 */ 1072 for (id = 0; id <= MAXCPU; id++) { 1073 if ((id_mask & 1 << id) == 0) 1074 continue; 1075 /* First, make sure we are on a logical_cpus boundary. */ 1076 if (id % logical_cpus != 0) 1077 return; 1078 for (i = id + 1; i < id + logical_cpus; i++) 1079 if ((id_mask & 1 << i) != 0) 1080 return; 1081 } 1082 1083 /* 1084 * Ok, the ID's checked out, so enable the fixup. We have to fixup 1085 * mp_naps and mp_maxid right now. 1086 */ 1087 need_hyperthreading_fixup = 1; 1088 mp_maxid *= logical_cpus; 1089 mp_naps *= logical_cpus; 1090} 1091 1092void 1093assign_apic_irq(int apic, int intpin, int irq) 1094{ 1095 int x; 1096 1097 if (int_to_apicintpin[irq].ioapic != -1) 1098 panic("assign_apic_irq: inconsistent table"); 1099 1100 int_to_apicintpin[irq].ioapic = apic; 1101 int_to_apicintpin[irq].int_pin = intpin; 1102 int_to_apicintpin[irq].apic_address = ioapic[apic]; 1103 int_to_apicintpin[irq].redirindex = IOAPIC_REDTBL + 2 * intpin; 1104 1105 for (x = 0; x < nintrs; x++) { 1106 if ((io_apic_ints[x].int_type == 0 || 1107 io_apic_ints[x].int_type == 3) && 1108 io_apic_ints[x].int_vector == 0xff && 1109 io_apic_ints[x].dst_apic_id == IO_TO_ID(apic) && 1110 io_apic_ints[x].dst_apic_int == intpin) 1111 io_apic_ints[x].int_vector = irq; 1112 } 1113} 1114 1115void 1116revoke_apic_irq(int irq) 1117{ 1118 int x; 1119 int oldapic; 1120 int oldintpin; 1121 1122 if (int_to_apicintpin[irq].ioapic == -1) 1123 panic("revoke_apic_irq: inconsistent table"); 1124 1125 oldapic = int_to_apicintpin[irq].ioapic; 1126 oldintpin = int_to_apicintpin[irq].int_pin; 1127 1128 int_to_apicintpin[irq].ioapic = -1; 1129 int_to_apicintpin[irq].int_pin = 0; 1130 int_to_apicintpin[irq].apic_address = NULL; 1131 int_to_apicintpin[irq].redirindex = 0; 1132 1133 for (x = 0; x < nintrs; x++) { 1134 if ((io_apic_ints[x].int_type == 0 || 1135 io_apic_ints[x].int_type == 3) && 1136 io_apic_ints[x].int_vector != 0xff && 1137 io_apic_ints[x].dst_apic_id == IO_TO_ID(oldapic) && 1138 io_apic_ints[x].dst_apic_int == oldintpin) 1139 io_apic_ints[x].int_vector = 0xff; 1140 } 1141} 1142 1143 1144static void 1145allocate_apic_irq(int intr) 1146{ 1147 int apic; 1148 int intpin; 1149 int irq; 1150 1151 if (io_apic_ints[intr].int_vector != 0xff) 1152 return; /* Interrupt handler already assigned */ 1153 1154 if (io_apic_ints[intr].int_type != 0 && 1155 (io_apic_ints[intr].int_type != 3 || 1156 (io_apic_ints[intr].dst_apic_id == IO_TO_ID(0) && 1157 io_apic_ints[intr].dst_apic_int == 0))) 1158 return; /* Not INT or ExtInt on != (0, 0) */ 1159 1160 irq = 0; 1161 while (irq < APIC_INTMAPSIZE && 1162 int_to_apicintpin[irq].ioapic != -1) 1163 irq++; 1164 1165 if (irq >= APIC_INTMAPSIZE) 1166 return; /* No free interrupt handlers */ 1167 1168 apic = ID_TO_IO(io_apic_ints[intr].dst_apic_id); 1169 intpin = io_apic_ints[intr].dst_apic_int; 1170 1171 assign_apic_irq(apic, intpin, irq); 1172 io_apic_setup_intpin(apic, intpin); 1173} 1174 1175 1176static void 1177swap_apic_id(int apic, int oldid, int newid) 1178{ 1179 int x; 1180 int oapic; 1181 1182 1183 if (oldid == newid) 1184 return; /* Nothing to do */ 1185 1186 printf("Changing APIC ID for IO APIC #%d from %d to %d in MP table\n", 1187 apic, oldid, newid); 1188 1189 /* Swap physical APIC IDs in interrupt entries */ 1190 for (x = 0; x < nintrs; x++) { 1191 if (io_apic_ints[x].dst_apic_id == oldid) 1192 io_apic_ints[x].dst_apic_id = newid; 1193 else if (io_apic_ints[x].dst_apic_id == newid) 1194 io_apic_ints[x].dst_apic_id = oldid; 1195 } 1196 1197 /* Swap physical APIC IDs in IO_TO_ID mappings */ 1198 for (oapic = 0; oapic < mp_napics; oapic++) 1199 if (IO_TO_ID(oapic) == newid) 1200 break; 1201 1202 if (oapic < mp_napics) { 1203 printf("Changing APIC ID for IO APIC #%d from " 1204 "%d to %d in MP table\n", 1205 oapic, newid, oldid); 1206 IO_TO_ID(oapic) = oldid; 1207 } 1208 IO_TO_ID(apic) = newid; 1209} 1210 1211 1212static void 1213fix_id_to_io_mapping(void) 1214{ 1215 int x; 1216 1217 for (x = 0; x < NAPICID; x++) 1218 ID_TO_IO(x) = -1; 1219 1220 for (x = 0; x <= mp_naps; x++) 1221 if (CPU_TO_ID(x) < NAPICID) 1222 ID_TO_IO(CPU_TO_ID(x)) = x; 1223 1224 for (x = 0; x < mp_napics; x++) 1225 if (IO_TO_ID(x) < NAPICID) 1226 ID_TO_IO(IO_TO_ID(x)) = x; 1227} 1228 1229 1230static int 1231first_free_apic_id(void) 1232{ 1233 int freeid, x; 1234 1235 for (freeid = 0; freeid < NAPICID; freeid++) { 1236 for (x = 0; x <= mp_naps; x++) 1237 if (CPU_TO_ID(x) == freeid) 1238 break; 1239 if (x <= mp_naps) 1240 continue; 1241 for (x = 0; x < mp_napics; x++) 1242 if (IO_TO_ID(x) == freeid) 1243 break; 1244 if (x < mp_napics) 1245 continue; 1246 return freeid; 1247 } 1248 return freeid; 1249} 1250 1251 1252static int 1253io_apic_id_acceptable(int apic, int id) 1254{ 1255 int cpu; /* Logical CPU number */ 1256 int oapic; /* Logical IO APIC number for other IO APIC */ 1257 1258 if (id >= NAPICID) 1259 return 0; /* Out of range */ 1260 1261 for (cpu = 0; cpu <= mp_naps; cpu++) 1262 if (CPU_TO_ID(cpu) == id) 1263 return 0; /* Conflict with CPU */ 1264 1265 for (oapic = 0; oapic < mp_napics && oapic < apic; oapic++) 1266 if (IO_TO_ID(oapic) == id) 1267 return 0; /* Conflict with other APIC */ 1268 1269 return 1; /* ID is acceptable for IO APIC */ 1270} 1271 1272 1273/* 1274 * parse an Intel MP specification table 1275 */ 1276static void 1277fix_mp_table(void) 1278{ 1279 int x; 1280 int id; 1281 int bus_0 = 0; /* Stop GCC warning */ 1282 int bus_pci = 0; /* Stop GCC warning */ 1283 int num_pci_bus; 1284 int apic; /* IO APIC unit number */ 1285 int freeid; /* Free physical APIC ID */ 1286 int physid; /* Current physical IO APIC ID */ 1287 1288 /* 1289 * Fix mis-numbering of the PCI bus and its INT entries if the BIOS 1290 * did it wrong. The MP spec says that when more than 1 PCI bus 1291 * exists the BIOS must begin with bus entries for the PCI bus and use 1292 * actual PCI bus numbering. This implies that when only 1 PCI bus 1293 * exists the BIOS can choose to ignore this ordering, and indeed many 1294 * MP motherboards do ignore it. This causes a problem when the PCI 1295 * sub-system makes requests of the MP sub-system based on PCI bus 1296 * numbers. So here we look for the situation and renumber the 1297 * busses and associated INTs in an effort to "make it right". 1298 */ 1299 1300 /* find bus 0, PCI bus, count the number of PCI busses */ 1301 for (num_pci_bus = 0, x = 0; x < mp_nbusses; ++x) { 1302 if (bus_data[x].bus_id == 0) { 1303 bus_0 = x; 1304 } 1305 if (bus_data[x].bus_type == PCI) { 1306 ++num_pci_bus; 1307 bus_pci = x; 1308 } 1309 } 1310 /* 1311 * bus_0 == slot of bus with ID of 0 1312 * bus_pci == slot of last PCI bus encountered 1313 */ 1314 1315 /* check the 1 PCI bus case for sanity */ 1316 /* if it is number 0 all is well */ 1317 if (num_pci_bus == 1 && 1318 bus_data[bus_pci].bus_id != 0) { 1319 1320 /* mis-numbered, swap with whichever bus uses slot 0 */ 1321 1322 /* swap the bus entry types */ 1323 bus_data[bus_pci].bus_type = bus_data[bus_0].bus_type; 1324 bus_data[bus_0].bus_type = PCI; 1325 1326 /* swap each relavant INTerrupt entry */ 1327 id = bus_data[bus_pci].bus_id; 1328 for (x = 0; x < nintrs; ++x) { 1329 if (io_apic_ints[x].src_bus_id == id) { 1330 io_apic_ints[x].src_bus_id = 0; 1331 } 1332 else if (io_apic_ints[x].src_bus_id == 0) { 1333 io_apic_ints[x].src_bus_id = id; 1334 } 1335 } 1336 } 1337 1338 /* Assign IO APIC IDs. 1339 * 1340 * First try the existing ID. If a conflict is detected, try 1341 * the ID in the MP table. If a conflict is still detected, find 1342 * a free id. 1343 * 1344 * We cannot use the ID_TO_IO table before all conflicts has been 1345 * resolved and the table has been corrected. 1346 */ 1347 for (apic = 0; apic < mp_napics; ++apic) { /* For all IO APICs */ 1348 1349 /* First try to use the value set by the BIOS */ 1350 physid = io_apic_get_id(apic); 1351 if (io_apic_id_acceptable(apic, physid)) { 1352 if (IO_TO_ID(apic) != physid) 1353 swap_apic_id(apic, IO_TO_ID(apic), physid); 1354 continue; 1355 } 1356 1357 /* Then check if the value in the MP table is acceptable */ 1358 if (io_apic_id_acceptable(apic, IO_TO_ID(apic))) 1359 continue; 1360 1361 /* Last resort, find a free APIC ID and use it */ 1362 freeid = first_free_apic_id(); 1363 if (freeid >= NAPICID) 1364 panic("No free physical APIC IDs found"); 1365 1366 if (io_apic_id_acceptable(apic, freeid)) { 1367 swap_apic_id(apic, IO_TO_ID(apic), freeid); 1368 continue; 1369 } 1370 panic("Free physical APIC ID not usable"); 1371 } 1372 fix_id_to_io_mapping(); 1373 1374 /* detect and fix broken Compaq MP table */ 1375 if (apic_int_type(0, 0) == -1) { 1376 printf("APIC_IO: MP table broken: 8259->APIC entry missing!\n"); 1377 io_apic_ints[nintrs].int_type = 3; /* ExtInt */ 1378 io_apic_ints[nintrs].int_vector = 0xff; /* Unassigned */ 1379 /* XXX fixme, set src bus id etc, but it doesn't seem to hurt */ 1380 io_apic_ints[nintrs].dst_apic_id = IO_TO_ID(0); 1381 io_apic_ints[nintrs].dst_apic_int = 0; /* Pin 0 */ 1382 nintrs++; 1383 } 1384} 1385 1386 1387/* Assign low level interrupt handlers */ 1388static void 1389setup_apic_irq_mapping(void) 1390{ 1391 int x; 1392 int int_vector; 1393 1394 /* Clear array */ 1395 for (x = 0; x < APIC_INTMAPSIZE; x++) { 1396 int_to_apicintpin[x].ioapic = -1; 1397 int_to_apicintpin[x].int_pin = 0; 1398 int_to_apicintpin[x].apic_address = NULL; 1399 int_to_apicintpin[x].redirindex = 0; 1400 } 1401 1402 /* First assign ISA/EISA interrupts */ 1403 for (x = 0; x < nintrs; x++) { 1404 int_vector = io_apic_ints[x].src_bus_irq; 1405 if (int_vector < APIC_INTMAPSIZE && 1406 io_apic_ints[x].int_vector == 0xff && 1407 int_to_apicintpin[int_vector].ioapic == -1 && 1408 (apic_int_is_bus_type(x, ISA) || 1409 apic_int_is_bus_type(x, EISA)) && 1410 io_apic_ints[x].int_type == 0) { 1411 assign_apic_irq(ID_TO_IO(io_apic_ints[x].dst_apic_id), 1412 io_apic_ints[x].dst_apic_int, 1413 int_vector); 1414 } 1415 } 1416 1417 /* Assign ExtInt entry if no ISA/EISA interrupt 0 entry */ 1418 for (x = 0; x < nintrs; x++) { 1419 if (io_apic_ints[x].dst_apic_int == 0 && 1420 io_apic_ints[x].dst_apic_id == IO_TO_ID(0) && 1421 io_apic_ints[x].int_vector == 0xff && 1422 int_to_apicintpin[0].ioapic == -1 && 1423 io_apic_ints[x].int_type == 3) { 1424 assign_apic_irq(0, 0, 0); 1425 break; 1426 } 1427 } 1428 /* PCI interrupt assignment is deferred */ 1429} 1430 1431 1432static int 1433processor_entry(proc_entry_ptr entry, int cpu) 1434{ 1435 /* check for usability */ 1436 if (!(entry->cpu_flags & PROCENTRY_FLAG_EN)) 1437 return 0; 1438 1439 if(entry->apic_id >= NAPICID) 1440 panic("CPU APIC ID out of range (0..%d)", NAPICID - 1); 1441 /* check for BSP flag */ 1442 if (entry->cpu_flags & PROCENTRY_FLAG_BP) { 1443 boot_cpu_id = entry->apic_id; 1444 CPU_TO_ID(0) = entry->apic_id; 1445 ID_TO_CPU(entry->apic_id) = 0; 1446 return 0; /* its already been counted */ 1447 } 1448 1449 /* add another AP to list, if less than max number of CPUs */ 1450 else if (cpu < MAXCPU) { 1451 CPU_TO_ID(cpu) = entry->apic_id; 1452 ID_TO_CPU(entry->apic_id) = cpu; 1453 return 1; 1454 } 1455 1456 return 0; 1457} 1458 1459 1460static int 1461bus_entry(bus_entry_ptr entry, int bus) 1462{ 1463 int x; 1464 char c, name[8]; 1465 1466 /* encode the name into an index */ 1467 for (x = 0; x < 6; ++x) { 1468 if ((c = entry->bus_type[x]) == ' ') 1469 break; 1470 name[x] = c; 1471 } 1472 name[x] = '\0'; 1473 1474 if ((x = lookup_bus_type(name)) == UNKNOWN_BUSTYPE) 1475 panic("unknown bus type: '%s'", name); 1476 1477 bus_data[bus].bus_id = entry->bus_id; 1478 bus_data[bus].bus_type = x; 1479 1480 return 1; 1481} 1482 1483 1484static int 1485io_apic_entry(io_apic_entry_ptr entry, int apic) 1486{ 1487 if (!(entry->apic_flags & IOAPICENTRY_FLAG_EN)) 1488 return 0; 1489 1490 IO_TO_ID(apic) = entry->apic_id; 1491 if (entry->apic_id < NAPICID) 1492 ID_TO_IO(entry->apic_id) = apic; 1493 1494 return 1; 1495} 1496 1497 1498static int 1499lookup_bus_type(char *name) 1500{ 1501 int x; 1502 1503 for (x = 0; x < MAX_BUSTYPE; ++x) 1504 if (strcmp(bus_type_table[x].name, name) == 0) 1505 return bus_type_table[x].type; 1506 1507 return UNKNOWN_BUSTYPE; 1508} 1509 1510 1511static int 1512int_entry(int_entry_ptr entry, int intr) 1513{ 1514 int apic; 1515 1516 io_apic_ints[intr].int_type = entry->int_type; 1517 io_apic_ints[intr].int_flags = entry->int_flags; 1518 io_apic_ints[intr].src_bus_id = entry->src_bus_id; 1519 io_apic_ints[intr].src_bus_irq = entry->src_bus_irq; 1520 if (entry->dst_apic_id == 255) { 1521 /* This signal goes to all IO APICS. Select an IO APIC 1522 with sufficient number of interrupt pins */ 1523 for (apic = 0; apic < mp_napics; apic++) 1524 if (((io_apic_read(apic, IOAPIC_VER) & 1525 IOART_VER_MAXREDIR) >> MAXREDIRSHIFT) >= 1526 entry->dst_apic_int) 1527 break; 1528 if (apic < mp_napics) 1529 io_apic_ints[intr].dst_apic_id = IO_TO_ID(apic); 1530 else 1531 io_apic_ints[intr].dst_apic_id = entry->dst_apic_id; 1532 } else 1533 io_apic_ints[intr].dst_apic_id = entry->dst_apic_id; 1534 io_apic_ints[intr].dst_apic_int = entry->dst_apic_int; 1535 1536 return 1; 1537} 1538 1539 1540static int 1541apic_int_is_bus_type(int intr, int bus_type) 1542{ 1543 int bus; 1544 1545 for (bus = 0; bus < mp_nbusses; ++bus) 1546 if ((bus_data[bus].bus_id == io_apic_ints[intr].src_bus_id) 1547 && ((int) bus_data[bus].bus_type == bus_type)) 1548 return 1; 1549 1550 return 0; 1551} 1552 1553 1554/* 1555 * Given a traditional ISA INT mask, return an APIC mask. 1556 */ 1557u_int 1558isa_apic_mask(u_int isa_mask) 1559{ 1560 int isa_irq; 1561 int apic_pin; 1562 1563#if defined(SKIP_IRQ15_REDIRECT) 1564 if (isa_mask == (1 << 15)) { 1565 printf("skipping ISA IRQ15 redirect\n"); 1566 return isa_mask; 1567 } 1568#endif /* SKIP_IRQ15_REDIRECT */ 1569 1570 isa_irq = ffs(isa_mask); /* find its bit position */ 1571 if (isa_irq == 0) /* doesn't exist */ 1572 return 0; 1573 --isa_irq; /* make it zero based */ 1574 1575 apic_pin = isa_apic_irq(isa_irq); /* look for APIC connection */ 1576 if (apic_pin == -1) 1577 return 0; 1578 1579 return (1 << apic_pin); /* convert pin# to a mask */ 1580} 1581 1582 1583/* 1584 * Determine which APIC pin an ISA/EISA INT is attached to. 1585 */ 1586#define INTTYPE(I) (io_apic_ints[(I)].int_type) 1587#define INTPIN(I) (io_apic_ints[(I)].dst_apic_int) 1588#define INTIRQ(I) (io_apic_ints[(I)].int_vector) 1589#define INTAPIC(I) (ID_TO_IO(io_apic_ints[(I)].dst_apic_id)) 1590 1591#define SRCBUSIRQ(I) (io_apic_ints[(I)].src_bus_irq) 1592int 1593isa_apic_irq(int isa_irq) 1594{ 1595 int intr; 1596 1597 for (intr = 0; intr < nintrs; ++intr) { /* check each record */ 1598 if (INTTYPE(intr) == 0) { /* standard INT */ 1599 if (SRCBUSIRQ(intr) == isa_irq) { 1600 if (apic_int_is_bus_type(intr, ISA) || 1601 apic_int_is_bus_type(intr, EISA)) { 1602 if (INTIRQ(intr) == 0xff) 1603 return -1; /* unassigned */ 1604 return INTIRQ(intr); /* found */ 1605 } 1606 } 1607 } 1608 } 1609 return -1; /* NOT found */ 1610} 1611 1612 1613/* 1614 * Determine which APIC pin a PCI INT is attached to. 1615 */ 1616#define SRCBUSID(I) (io_apic_ints[(I)].src_bus_id) 1617#define SRCBUSDEVICE(I) ((io_apic_ints[(I)].src_bus_irq >> 2) & 0x1f) 1618#define SRCBUSLINE(I) (io_apic_ints[(I)].src_bus_irq & 0x03) 1619int 1620pci_apic_irq(int pciBus, int pciDevice, int pciInt) 1621{ 1622 int intr; 1623 1624 --pciInt; /* zero based */ 1625 1626 for (intr = 0; intr < nintrs; ++intr) /* check each record */ 1627 if ((INTTYPE(intr) == 0) /* standard INT */ 1628 && (SRCBUSID(intr) == pciBus) 1629 && (SRCBUSDEVICE(intr) == pciDevice) 1630 && (SRCBUSLINE(intr) == pciInt)) /* a candidate IRQ */ 1631 if (apic_int_is_bus_type(intr, PCI)) { 1632 if (INTIRQ(intr) == 0xff) 1633 allocate_apic_irq(intr); 1634 if (INTIRQ(intr) == 0xff) 1635 return -1; /* unassigned */ 1636 return INTIRQ(intr); /* exact match */ 1637 } 1638 1639 return -1; /* NOT found */ 1640} 1641 1642int 1643next_apic_irq(int irq) 1644{ 1645 int intr, ointr; 1646 int bus, bustype; 1647 1648 bus = 0; 1649 bustype = 0; 1650 for (intr = 0; intr < nintrs; intr++) { 1651 if (INTIRQ(intr) != irq || INTTYPE(intr) != 0) 1652 continue; 1653 bus = SRCBUSID(intr); 1654 bustype = apic_bus_type(bus); 1655 if (bustype != ISA && 1656 bustype != EISA && 1657 bustype != PCI) 1658 continue; 1659 break; 1660 } 1661 if (intr >= nintrs) { 1662 return -1; 1663 } 1664 for (ointr = intr + 1; ointr < nintrs; ointr++) { 1665 if (INTTYPE(ointr) != 0) 1666 continue; 1667 if (bus != SRCBUSID(ointr)) 1668 continue; 1669 if (bustype == PCI) { 1670 if (SRCBUSDEVICE(intr) != SRCBUSDEVICE(ointr)) 1671 continue; 1672 if (SRCBUSLINE(intr) != SRCBUSLINE(ointr)) 1673 continue; 1674 } 1675 if (bustype == ISA || bustype == EISA) { 1676 if (SRCBUSIRQ(intr) != SRCBUSIRQ(ointr)) 1677 continue; 1678 } 1679 if (INTPIN(intr) == INTPIN(ointr)) 1680 continue; 1681 break; 1682 } 1683 if (ointr >= nintrs) { 1684 return -1; 1685 } 1686 return INTIRQ(ointr); 1687} 1688#undef SRCBUSLINE 1689#undef SRCBUSDEVICE 1690#undef SRCBUSID 1691#undef SRCBUSIRQ 1692 1693#undef INTPIN 1694#undef INTIRQ 1695#undef INTAPIC 1696#undef INTTYPE 1697 1698 1699/* 1700 * Reprogram the MB chipset to NOT redirect an ISA INTerrupt. 1701 * 1702 * XXX FIXME: 1703 * Exactly what this means is unclear at this point. It is a solution 1704 * for motherboards that redirect the MBIRQ0 pin. Generically a motherboard 1705 * could route any of the ISA INTs to upper (>15) IRQ values. But most would 1706 * NOT be redirected via MBIRQ0, thus "undirect()ing" them would NOT be an 1707 * option. 1708 */ 1709int 1710undirect_isa_irq(int rirq) 1711{ 1712#if defined(READY) 1713 if (bootverbose) 1714 printf("Freeing redirected ISA irq %d.\n", rirq); 1715 /** FIXME: tickle the MB redirector chip */ 1716 return -1; 1717#else 1718 if (bootverbose) 1719 printf("Freeing (NOT implemented) redirected ISA irq %d.\n", rirq); 1720 return 0; 1721#endif /* READY */ 1722} 1723 1724 1725/* 1726 * Reprogram the MB chipset to NOT redirect a PCI INTerrupt 1727 */ 1728int 1729undirect_pci_irq(int rirq) 1730{ 1731#if defined(READY) 1732 if (bootverbose) 1733 printf("Freeing redirected PCI irq %d.\n", rirq); 1734 1735 /** FIXME: tickle the MB redirector chip */ 1736 return -1; 1737#else 1738 if (bootverbose) 1739 printf("Freeing (NOT implemented) redirected PCI irq %d.\n", 1740 rirq); 1741 return 0; 1742#endif /* READY */ 1743} 1744 1745 1746/* 1747 * given a bus ID, return: 1748 * the bus type if found 1749 * -1 if NOT found 1750 */ 1751int 1752apic_bus_type(int id) 1753{ 1754 int x; 1755 1756 for (x = 0; x < mp_nbusses; ++x) 1757 if (bus_data[x].bus_id == id) 1758 return bus_data[x].bus_type; 1759 1760 return -1; 1761} 1762 1763 1764/* 1765 * given a LOGICAL APIC# and pin#, return: 1766 * the associated src bus ID if found 1767 * -1 if NOT found 1768 */ 1769int 1770apic_src_bus_id(int apic, int pin) 1771{ 1772 int x; 1773 1774 /* search each of the possible INTerrupt sources */ 1775 for (x = 0; x < nintrs; ++x) 1776 if ((apic == ID_TO_IO(io_apic_ints[x].dst_apic_id)) && 1777 (pin == io_apic_ints[x].dst_apic_int)) 1778 return (io_apic_ints[x].src_bus_id); 1779 1780 return -1; /* NOT found */ 1781} 1782 1783 1784/* 1785 * given a LOGICAL APIC# and pin#, return: 1786 * the associated src bus IRQ if found 1787 * -1 if NOT found 1788 */ 1789int 1790apic_src_bus_irq(int apic, int pin) 1791{ 1792 int x; 1793 1794 for (x = 0; x < nintrs; x++) 1795 if ((apic == ID_TO_IO(io_apic_ints[x].dst_apic_id)) && 1796 (pin == io_apic_ints[x].dst_apic_int)) 1797 return (io_apic_ints[x].src_bus_irq); 1798 1799 return -1; /* NOT found */ 1800} 1801 1802 1803/* 1804 * given a LOGICAL APIC# and pin#, return: 1805 * the associated INTerrupt type if found 1806 * -1 if NOT found 1807 */ 1808int 1809apic_int_type(int apic, int pin) 1810{ 1811 int x; 1812 1813 /* search each of the possible INTerrupt sources */ 1814 for (x = 0; x < nintrs; ++x) 1815 if ((apic == ID_TO_IO(io_apic_ints[x].dst_apic_id)) && 1816 (pin == io_apic_ints[x].dst_apic_int)) 1817 return (io_apic_ints[x].int_type); 1818 1819 return -1; /* NOT found */ 1820} 1821 1822int 1823apic_irq(int apic, int pin) 1824{ 1825 int x; 1826 int res; 1827 1828 for (x = 0; x < nintrs; ++x) 1829 if ((apic == ID_TO_IO(io_apic_ints[x].dst_apic_id)) && 1830 (pin == io_apic_ints[x].dst_apic_int)) { 1831 res = io_apic_ints[x].int_vector; 1832 if (res == 0xff) 1833 return -1; 1834 if (apic != int_to_apicintpin[res].ioapic) 1835 panic("apic_irq: inconsistent table"); 1836 if (pin != int_to_apicintpin[res].int_pin) 1837 panic("apic_irq inconsistent table (2)"); 1838 return res; 1839 } 1840 return -1; 1841} 1842 1843 1844/* 1845 * given a LOGICAL APIC# and pin#, return: 1846 * the associated trigger mode if found 1847 * -1 if NOT found 1848 */ 1849int 1850apic_trigger(int apic, int pin) 1851{ 1852 int x; 1853 1854 /* search each of the possible INTerrupt sources */ 1855 for (x = 0; x < nintrs; ++x) 1856 if ((apic == ID_TO_IO(io_apic_ints[x].dst_apic_id)) && 1857 (pin == io_apic_ints[x].dst_apic_int)) 1858 return ((io_apic_ints[x].int_flags >> 2) & 0x03); 1859 1860 return -1; /* NOT found */ 1861} 1862 1863 1864/* 1865 * given a LOGICAL APIC# and pin#, return: 1866 * the associated 'active' level if found 1867 * -1 if NOT found 1868 */ 1869int 1870apic_polarity(int apic, int pin) 1871{ 1872 int x; 1873 1874 /* search each of the possible INTerrupt sources */ 1875 for (x = 0; x < nintrs; ++x) 1876 if ((apic == ID_TO_IO(io_apic_ints[x].dst_apic_id)) && 1877 (pin == io_apic_ints[x].dst_apic_int)) 1878 return (io_apic_ints[x].int_flags & 0x03); 1879 1880 return -1; /* NOT found */ 1881} 1882 1883 1884/* 1885 * set data according to MP defaults 1886 * FIXME: probably not complete yet... 1887 */ 1888static void 1889default_mp_table(int type) 1890{ 1891 int ap_cpu_id; 1892#if defined(APIC_IO) 1893 int io_apic_id; 1894 int pin; 1895#endif /* APIC_IO */ 1896 1897#if 0 1898 printf(" MP default config type: %d\n", type); 1899 switch (type) { 1900 case 1: 1901 printf(" bus: ISA, APIC: 82489DX\n"); 1902 break; 1903 case 2: 1904 printf(" bus: EISA, APIC: 82489DX\n"); 1905 break; 1906 case 3: 1907 printf(" bus: EISA, APIC: 82489DX\n"); 1908 break; 1909 case 4: 1910 printf(" bus: MCA, APIC: 82489DX\n"); 1911 break; 1912 case 5: 1913 printf(" bus: ISA+PCI, APIC: Integrated\n"); 1914 break; 1915 case 6: 1916 printf(" bus: EISA+PCI, APIC: Integrated\n"); 1917 break; 1918 case 7: 1919 printf(" bus: MCA+PCI, APIC: Integrated\n"); 1920 break; 1921 default: 1922 printf(" future type\n"); 1923 break; 1924 /* NOTREACHED */ 1925 } 1926#endif /* 0 */ 1927 1928 boot_cpu_id = (lapic.id & APIC_ID_MASK) >> 24; 1929 ap_cpu_id = (boot_cpu_id == 0) ? 1 : 0; 1930 1931 /* BSP */ 1932 CPU_TO_ID(0) = boot_cpu_id; 1933 ID_TO_CPU(boot_cpu_id) = 0; 1934 1935 /* one and only AP */ 1936 CPU_TO_ID(1) = ap_cpu_id; 1937 ID_TO_CPU(ap_cpu_id) = 1; 1938 1939#if defined(APIC_IO) 1940 /* one and only IO APIC */ 1941 io_apic_id = (io_apic_read(0, IOAPIC_ID) & APIC_ID_MASK) >> 24; 1942 1943 /* 1944 * sanity check, refer to MP spec section 3.6.6, last paragraph 1945 * necessary as some hardware isn't properly setting up the IO APIC 1946 */ 1947#if defined(REALLY_ANAL_IOAPICID_VALUE) 1948 if (io_apic_id != 2) { 1949#else 1950 if ((io_apic_id == 0) || (io_apic_id == 1) || (io_apic_id == 15)) { 1951#endif /* REALLY_ANAL_IOAPICID_VALUE */ 1952 io_apic_set_id(0, 2); 1953 io_apic_id = 2; 1954 } 1955 IO_TO_ID(0) = io_apic_id; 1956 ID_TO_IO(io_apic_id) = 0; 1957#endif /* APIC_IO */ 1958 1959 /* fill out bus entries */ 1960 switch (type) { 1961 case 1: 1962 case 2: 1963 case 3: 1964 case 4: 1965 case 5: 1966 case 6: 1967 case 7: 1968 bus_data[0].bus_id = default_data[type - 1][1]; 1969 bus_data[0].bus_type = default_data[type - 1][2]; 1970 bus_data[1].bus_id = default_data[type - 1][3]; 1971 bus_data[1].bus_type = default_data[type - 1][4]; 1972 break; 1973 1974 /* case 4: case 7: MCA NOT supported */ 1975 default: /* illegal/reserved */ 1976 panic("BAD default MP config: %d", type); 1977 /* NOTREACHED */ 1978 } 1979 1980#if defined(APIC_IO) 1981 /* general cases from MP v1.4, table 5-2 */ 1982 for (pin = 0; pin < 16; ++pin) { 1983 io_apic_ints[pin].int_type = 0; 1984 io_apic_ints[pin].int_flags = 0x05; /* edge/active-hi */ 1985 io_apic_ints[pin].src_bus_id = 0; 1986 io_apic_ints[pin].src_bus_irq = pin; /* IRQ2 caught below */ 1987 io_apic_ints[pin].dst_apic_id = io_apic_id; 1988 io_apic_ints[pin].dst_apic_int = pin; /* 1-to-1 */ 1989 } 1990 1991 /* special cases from MP v1.4, table 5-2 */ 1992 if (type == 2) { 1993 io_apic_ints[2].int_type = 0xff; /* N/C */ 1994 io_apic_ints[13].int_type = 0xff; /* N/C */ 1995#if !defined(APIC_MIXED_MODE) 1996 /** FIXME: ??? */ 1997 panic("sorry, can't support type 2 default yet"); 1998#endif /* APIC_MIXED_MODE */ 1999 } 2000 else 2001 io_apic_ints[2].src_bus_irq = 0; /* ISA IRQ0 is on APIC INT 2 */ 2002 2003 if (type == 7) 2004 io_apic_ints[0].int_type = 0xff; /* N/C */ 2005 else 2006 io_apic_ints[0].int_type = 3; /* vectored 8259 */ 2007#endif /* APIC_IO */ 2008} 2009 2010 2011/* 2012 * start each AP in our list 2013 */ 2014static int 2015start_all_aps(u_int boot_addr) 2016{ 2017 int x, i, pg; 2018 u_char mpbiosreason; 2019 u_long mpbioswarmvec; 2020 struct pcpu *pc; 2021 char *stack; 2022 uintptr_t kptbase; 2023 2024 POSTCODE(START_ALL_APS_POST); 2025 2026 mtx_init(&ap_boot_mtx, "ap boot", NULL, MTX_SPIN); 2027 2028 /* initialize BSP's local APIC */ 2029 apic_initialize(); 2030 bsp_apic_ready = 1; 2031 2032 /* install the AP 1st level boot code */ 2033 install_ap_tramp(boot_addr); 2034 2035 2036 /* save the current value of the warm-start vector */ 2037 mpbioswarmvec = *((u_long *) WARMBOOT_OFF); 2038#ifndef PC98 2039 outb(CMOS_REG, BIOS_RESET); 2040 mpbiosreason = inb(CMOS_DATA); 2041#endif 2042 2043 /* set up temporary P==V mapping for AP boot */ 2044 /* XXX this is a hack, we should boot the AP on its own stack/PTD */ 2045 kptbase = (uintptr_t)(void *)KPTphys; 2046 for (x = 0; x < NKPT; x++) 2047 PTD[x] = (pd_entry_t)(PG_V | PG_RW | 2048 ((kptbase + x * PAGE_SIZE) & PG_FRAME)); 2049 invltlb(); 2050 2051 /* start each AP */ 2052 for (x = 1; x <= mp_naps; ++x) { 2053 2054 /* This is a bit verbose, it will go away soon. */ 2055 2056 /* first page of AP's private space */ 2057 pg = x * i386_btop(sizeof(struct privatespace)); 2058 2059 /* allocate a new private data page */ 2060 pc = (struct pcpu *)kmem_alloc(kernel_map, PAGE_SIZE); 2061 2062 /* wire it into the private page table page */ 2063 SMPpt[pg] = (pt_entry_t)(PG_V | PG_RW | vtophys(pc)); 2064 2065 /* allocate and set up an idle stack data page */ 2066 stack = (char *)kmem_alloc(kernel_map, KSTACK_PAGES * PAGE_SIZE); /* XXXKSE */ 2067 for (i = 0; i < KSTACK_PAGES; i++) 2068 SMPpt[pg + 1 + i] = (pt_entry_t) 2069 (PG_V | PG_RW | vtophys(PAGE_SIZE * i + stack)); 2070 2071 /* prime data page for it to use */ 2072 pcpu_init(pc, x, sizeof(struct pcpu)); 2073 2074 /* setup a vector to our boot code */ 2075 *((volatile u_short *) WARMBOOT_OFF) = WARMBOOT_TARGET; 2076 *((volatile u_short *) WARMBOOT_SEG) = (boot_addr >> 4); 2077#ifndef PC98 2078 outb(CMOS_REG, BIOS_RESET); 2079 outb(CMOS_DATA, BIOS_WARM); /* 'warm-start' */ 2080#endif 2081 2082 bootSTK = &SMP_prvspace[x].idlekstack[KSTACK_PAGES * PAGE_SIZE]; 2083 bootAP = x; 2084 2085 /* attempt to start the Application Processor */ 2086 CHECK_INIT(99); /* setup checkpoints */ 2087 if (!start_ap(x, boot_addr)) { 2088 printf("AP #%d (PHY# %d) failed!\n", x, CPU_TO_ID(x)); 2089 CHECK_PRINT("trace"); /* show checkpoints */ 2090 /* better panic as the AP may be running loose */ 2091 printf("panic y/n? [y] "); 2092 if (cngetc() != 'n') 2093 panic("bye-bye"); 2094 } 2095 CHECK_PRINT("trace"); /* show checkpoints */ 2096 2097 /* record its version info */ 2098 cpu_apic_versions[x] = cpu_apic_versions[0]; 2099 2100 all_cpus |= (1 << x); /* record AP in CPU map */ 2101 } 2102 2103 /* build our map of 'other' CPUs */ 2104 PCPU_SET(other_cpus, all_cpus & ~PCPU_GET(cpumask)); 2105 2106 /* fill in our (BSP) APIC version */ 2107 cpu_apic_versions[0] = lapic.version; 2108 2109 /* restore the warmstart vector */ 2110 *(u_long *) WARMBOOT_OFF = mpbioswarmvec; 2111#ifndef PC98 2112 outb(CMOS_REG, BIOS_RESET); 2113 outb(CMOS_DATA, mpbiosreason); 2114#endif 2115 2116 /* 2117 * Set up the idle context for the BSP. Similar to above except 2118 * that some was done by locore, some by pmap.c and some is implicit 2119 * because the BSP is cpu#0 and the page is initially zero, and also 2120 * because we can refer to variables by name on the BSP.. 2121 */ 2122 2123 /* Allocate and setup BSP idle stack */ 2124 stack = (char *)kmem_alloc(kernel_map, KSTACK_PAGES * PAGE_SIZE); 2125 for (i = 0; i < KSTACK_PAGES; i++) 2126 SMPpt[1 + i] = (pt_entry_t) 2127 (PG_V | PG_RW | vtophys(PAGE_SIZE * i + stack)); 2128 2129 for (x = 0; x < NKPT; x++) 2130 PTD[x] = 0; 2131 pmap_set_opt(); 2132 2133 /* number of APs actually started */ 2134 return mp_ncpus - 1; 2135} 2136 2137 2138/* 2139 * load the 1st level AP boot code into base memory. 2140 */ 2141 2142/* targets for relocation */ 2143extern void bigJump(void); 2144extern void bootCodeSeg(void); 2145extern void bootDataSeg(void); 2146extern void MPentry(void); 2147extern u_int MP_GDT; 2148extern u_int mp_gdtbase; 2149 2150static void 2151install_ap_tramp(u_int boot_addr) 2152{ 2153 int x; 2154 int size = *(int *) ((u_long) & bootMP_size); 2155 u_char *src = (u_char *) ((u_long) bootMP); 2156 u_char *dst = (u_char *) boot_addr + KERNBASE; 2157 u_int boot_base = (u_int) bootMP; 2158 u_int8_t *dst8; 2159 u_int16_t *dst16; 2160 u_int32_t *dst32; 2161 2162 POSTCODE(INSTALL_AP_TRAMP_POST); 2163 2164 for (x = 0; x < size; ++x) 2165 *dst++ = *src++; 2166 2167 /* 2168 * modify addresses in code we just moved to basemem. unfortunately we 2169 * need fairly detailed info about mpboot.s for this to work. changes 2170 * to mpboot.s might require changes here. 2171 */ 2172 2173 /* boot code is located in KERNEL space */ 2174 dst = (u_char *) boot_addr + KERNBASE; 2175 2176 /* modify the lgdt arg */ 2177 dst32 = (u_int32_t *) (dst + ((u_int) & mp_gdtbase - boot_base)); 2178 *dst32 = boot_addr + ((u_int) & MP_GDT - boot_base); 2179 2180 /* modify the ljmp target for MPentry() */ 2181 dst32 = (u_int32_t *) (dst + ((u_int) bigJump - boot_base) + 1); 2182 *dst32 = ((u_int) MPentry - KERNBASE); 2183 2184 /* modify the target for boot code segment */ 2185 dst16 = (u_int16_t *) (dst + ((u_int) bootCodeSeg - boot_base)); 2186 dst8 = (u_int8_t *) (dst16 + 1); 2187 *dst16 = (u_int) boot_addr & 0xffff; 2188 *dst8 = ((u_int) boot_addr >> 16) & 0xff; 2189 2190 /* modify the target for boot data segment */ 2191 dst16 = (u_int16_t *) (dst + ((u_int) bootDataSeg - boot_base)); 2192 dst8 = (u_int8_t *) (dst16 + 1); 2193 *dst16 = (u_int) boot_addr & 0xffff; 2194 *dst8 = ((u_int) boot_addr >> 16) & 0xff; 2195} 2196 2197 2198/* 2199 * this function starts the AP (application processor) identified 2200 * by the APIC ID 'physicalCpu'. It does quite a "song and dance" 2201 * to accomplish this. This is necessary because of the nuances 2202 * of the different hardware we might encounter. It ain't pretty, 2203 * but it seems to work. 2204 */ 2205static int 2206start_ap(int logical_cpu, u_int boot_addr) 2207{ 2208 int physical_cpu; 2209 int vector; 2210 int cpus; 2211 u_long icr_lo, icr_hi; 2212 2213 POSTCODE(START_AP_POST); 2214 2215 /* get the PHYSICAL APIC ID# */ 2216 physical_cpu = CPU_TO_ID(logical_cpu); 2217 2218 /* calculate the vector */ 2219 vector = (boot_addr >> 12) & 0xff; 2220 2221 /* used as a watchpoint to signal AP startup */ 2222 cpus = mp_ncpus; 2223 2224 /* 2225 * first we do an INIT/RESET IPI this INIT IPI might be run, reseting 2226 * and running the target CPU. OR this INIT IPI might be latched (P5 2227 * bug), CPU waiting for STARTUP IPI. OR this INIT IPI might be 2228 * ignored. 2229 */ 2230 2231 /* setup the address for the target AP */ 2232 icr_hi = lapic.icr_hi & ~APIC_ID_MASK; 2233 icr_hi |= (physical_cpu << 24); 2234 lapic.icr_hi = icr_hi; 2235 2236 /* do an INIT IPI: assert RESET */ 2237 icr_lo = lapic.icr_lo & 0xfff00000; 2238 lapic.icr_lo = icr_lo | 0x0000c500; 2239 2240 /* wait for pending status end */ 2241 while (lapic.icr_lo & APIC_DELSTAT_MASK) 2242 /* spin */ ; 2243 2244 /* do an INIT IPI: deassert RESET */ 2245 lapic.icr_lo = icr_lo | 0x00008500; 2246 2247 /* wait for pending status end */ 2248 u_sleep(10000); /* wait ~10mS */ 2249 while (lapic.icr_lo & APIC_DELSTAT_MASK) 2250 /* spin */ ; 2251 2252 /* 2253 * next we do a STARTUP IPI: the previous INIT IPI might still be 2254 * latched, (P5 bug) this 1st STARTUP would then terminate 2255 * immediately, and the previously started INIT IPI would continue. OR 2256 * the previous INIT IPI has already run. and this STARTUP IPI will 2257 * run. OR the previous INIT IPI was ignored. and this STARTUP IPI 2258 * will run. 2259 */ 2260 2261 /* do a STARTUP IPI */ 2262 lapic.icr_lo = icr_lo | 0x00000600 | vector; 2263 while (lapic.icr_lo & APIC_DELSTAT_MASK) 2264 /* spin */ ; 2265 u_sleep(200); /* wait ~200uS */ 2266 2267 /* 2268 * finally we do a 2nd STARTUP IPI: this 2nd STARTUP IPI should run IF 2269 * the previous STARTUP IPI was cancelled by a latched INIT IPI. OR 2270 * this STARTUP IPI will be ignored, as only ONE STARTUP IPI is 2271 * recognized after hardware RESET or INIT IPI. 2272 */ 2273 2274 lapic.icr_lo = icr_lo | 0x00000600 | vector; 2275 while (lapic.icr_lo & APIC_DELSTAT_MASK) 2276 /* spin */ ; 2277 u_sleep(200); /* wait ~200uS */ 2278 2279 /* wait for it to start */ 2280 set_apic_timer(5000000);/* == 5 seconds */ 2281 while (read_apic_timer()) 2282 if (mp_ncpus > cpus) 2283 return 1; /* return SUCCESS */ 2284 2285 return 0; /* return FAILURE */ 2286} 2287 2288#if defined(APIC_IO) 2289 2290#ifdef COUNT_XINVLTLB_HITS 2291u_int xhits_gbl[MAXCPU]; 2292u_int xhits_pg[MAXCPU]; 2293u_int xhits_rng[MAXCPU]; 2294SYSCTL_NODE(_debug, OID_AUTO, xhits, CTLFLAG_RW, 0, ""); 2295SYSCTL_OPAQUE(_debug_xhits, OID_AUTO, global, CTLFLAG_RW, &xhits_gbl, 2296 sizeof(xhits_gbl), "IU", ""); 2297SYSCTL_OPAQUE(_debug_xhits, OID_AUTO, page, CTLFLAG_RW, &xhits_pg, 2298 sizeof(xhits_pg), "IU", ""); 2299SYSCTL_OPAQUE(_debug_xhits, OID_AUTO, range, CTLFLAG_RW, &xhits_rng, 2300 sizeof(xhits_rng), "IU", ""); 2301 2302u_int ipi_global; 2303u_int ipi_page; 2304u_int ipi_range; 2305u_int ipi_range_size; 2306SYSCTL_INT(_debug_xhits, OID_AUTO, ipi_global, CTLFLAG_RW, &ipi_global, 0, ""); 2307SYSCTL_INT(_debug_xhits, OID_AUTO, ipi_page, CTLFLAG_RW, &ipi_page, 0, ""); 2308SYSCTL_INT(_debug_xhits, OID_AUTO, ipi_range, CTLFLAG_RW, &ipi_range, 0, ""); 2309SYSCTL_INT(_debug_xhits, OID_AUTO, ipi_range_size, CTLFLAG_RW, &ipi_range_size, 2310 0, ""); 2311 2312u_int ipi_masked_global; 2313u_int ipi_masked_page; 2314u_int ipi_masked_range; 2315u_int ipi_masked_range_size; 2316SYSCTL_INT(_debug_xhits, OID_AUTO, ipi_masked_global, CTLFLAG_RW, 2317 &ipi_masked_global, 0, ""); 2318SYSCTL_INT(_debug_xhits, OID_AUTO, ipi_masked_page, CTLFLAG_RW, 2319 &ipi_masked_page, 0, ""); 2320SYSCTL_INT(_debug_xhits, OID_AUTO, ipi_masked_range, CTLFLAG_RW, 2321 &ipi_masked_range, 0, ""); 2322SYSCTL_INT(_debug_xhits, OID_AUTO, ipi_masked_range_size, CTLFLAG_RW, 2323 &ipi_masked_range_size, 0, ""); 2324#endif 2325 2326/* 2327 * Flush the TLB on all other CPU's 2328 */ 2329static void 2330smp_tlb_shootdown(u_int vector, vm_offset_t addr1, vm_offset_t addr2) 2331{ 2332 u_int ncpu; 2333 register_t eflags; 2334 2335 ncpu = mp_ncpus - 1; /* does not shootdown self */ 2336 if (ncpu < 1) 2337 return; /* no other cpus */ 2338 eflags = read_eflags(); 2339 if ((eflags & PSL_I) == 0) 2340 panic("absolutely cannot call smp_ipi_shootdown with interrupts already disabled"); 2341 mtx_lock_spin(&smp_tlb_mtx); 2342 smp_tlb_addr1 = addr1; 2343 smp_tlb_addr2 = addr2; 2344 atomic_store_rel_int(&smp_tlb_wait, 0); 2345 ipi_all_but_self(vector); 2346 while (smp_tlb_wait < ncpu) 2347 ia32_pause(); 2348 mtx_unlock_spin(&smp_tlb_mtx); 2349} 2350 2351/* 2352 * This is about as magic as it gets. fortune(1) has got similar code 2353 * for reversing bits in a word. Who thinks up this stuff?? 2354 * 2355 * Yes, it does appear to be consistently faster than: 2356 * while (i = ffs(m)) { 2357 * m >>= i; 2358 * bits++; 2359 * } 2360 * and 2361 * while (lsb = (m & -m)) { // This is magic too 2362 * m &= ~lsb; // or: m ^= lsb 2363 * bits++; 2364 * } 2365 * Both of these latter forms do some very strange things on gcc-3.1 with 2366 * -mcpu=pentiumpro and/or -march=pentiumpro and/or -O or -O2. 2367 * There is probably an SSE or MMX popcnt instruction. 2368 * 2369 * I wonder if this should be in libkern? 2370 * 2371 * XXX Stop the presses! Another one: 2372 * static __inline u_int32_t 2373 * popcnt1(u_int32_t v) 2374 * { 2375 * v -= ((v >> 1) & 0x55555555); 2376 * v = (v & 0x33333333) + ((v >> 2) & 0x33333333); 2377 * v = (v + (v >> 4)) & 0x0F0F0F0F; 2378 * return (v * 0x01010101) >> 24; 2379 * } 2380 * The downside is that it has a multiply. With a pentium3 with 2381 * -mcpu=pentiumpro and -march=pentiumpro then gcc-3.1 will use 2382 * an imull, and in that case it is faster. In most other cases 2383 * it appears slightly slower. 2384 */ 2385static __inline u_int32_t 2386popcnt(u_int32_t m) 2387{ 2388 2389 m = (m & 0x55555555) + ((m & 0xaaaaaaaa) >> 1); 2390 m = (m & 0x33333333) + ((m & 0xcccccccc) >> 2); 2391 m = (m & 0x0f0f0f0f) + ((m & 0xf0f0f0f0) >> 4); 2392 m = (m & 0x00ff00ff) + ((m & 0xff00ff00) >> 8); 2393 m = (m & 0x0000ffff) + ((m & 0xffff0000) >> 16); 2394 return m; 2395} 2396 2397static void 2398smp_targeted_tlb_shootdown(u_int mask, u_int vector, vm_offset_t addr1, vm_offset_t addr2) 2399{ 2400 int ncpu, othercpus; 2401 register_t eflags; 2402 2403 othercpus = mp_ncpus - 1; 2404 if (mask == (u_int)-1) { 2405 ncpu = othercpus; 2406 if (ncpu < 1) 2407 return; 2408 } else { 2409 /* XXX there should be a pcpu self mask */ 2410 mask &= ~(1 << PCPU_GET(cpuid)); 2411 if (mask == 0) 2412 return; 2413 ncpu = popcnt(mask); 2414 if (ncpu > othercpus) { 2415 /* XXX this should be a panic offence */ 2416 printf("SMP: tlb shootdown to %d other cpus (only have %d)\n", 2417 ncpu, othercpus); 2418 ncpu = othercpus; 2419 } 2420 /* XXX should be a panic, implied by mask == 0 above */ 2421 if (ncpu < 1) 2422 return; 2423 } 2424 eflags = read_eflags(); 2425 if ((eflags & PSL_I) == 0) 2426 panic("absolutely cannot call smp_targeted_ipi_shootdown with interrupts already disabled"); 2427 mtx_lock_spin(&smp_tlb_mtx); 2428 smp_tlb_addr1 = addr1; 2429 smp_tlb_addr2 = addr2; 2430 atomic_store_rel_int(&smp_tlb_wait, 0); 2431 if (mask == (u_int)-1) 2432 ipi_all_but_self(vector); 2433 else 2434 ipi_selected(mask, vector); 2435 while (smp_tlb_wait < ncpu) 2436 ia32_pause(); 2437 mtx_unlock_spin(&smp_tlb_mtx); 2438} 2439#endif 2440 2441void 2442smp_invltlb(void) 2443{ 2444#if defined(APIC_IO) 2445 if (smp_started) { 2446 smp_tlb_shootdown(IPI_INVLTLB, 0, 0); 2447#ifdef COUNT_XINVLTLB_HITS 2448 ipi_global++; 2449#endif 2450 } 2451#endif /* APIC_IO */ 2452} 2453 2454void 2455smp_invlpg(vm_offset_t addr) 2456{ 2457#if defined(APIC_IO) 2458 if (smp_started) { 2459 smp_tlb_shootdown(IPI_INVLPG, addr, 0); 2460#ifdef COUNT_XINVLTLB_HITS 2461 ipi_page++; 2462#endif 2463 } 2464#endif /* APIC_IO */ 2465} 2466 2467void 2468smp_invlpg_range(vm_offset_t addr1, vm_offset_t addr2) 2469{ 2470#if defined(APIC_IO) 2471 if (smp_started) { 2472 smp_tlb_shootdown(IPI_INVLRNG, addr1, addr2); 2473#ifdef COUNT_XINVLTLB_HITS 2474 ipi_range++; 2475 ipi_range_size += (addr2 - addr1) / PAGE_SIZE; 2476#endif 2477 } 2478#endif /* APIC_IO */ 2479} 2480 2481void 2482smp_masked_invltlb(u_int mask) 2483{ 2484#if defined(APIC_IO) 2485 if (smp_started) { 2486 smp_targeted_tlb_shootdown(mask, IPI_INVLTLB, 0, 0); 2487#ifdef COUNT_XINVLTLB_HITS 2488 ipi_masked_global++; 2489#endif 2490 } 2491#endif /* APIC_IO */ 2492} 2493 2494void 2495smp_masked_invlpg(u_int mask, vm_offset_t addr) 2496{ 2497#if defined(APIC_IO) 2498 if (smp_started) { 2499 smp_targeted_tlb_shootdown(mask, IPI_INVLPG, addr, 0); 2500#ifdef COUNT_XINVLTLB_HITS 2501 ipi_masked_page++; 2502#endif 2503 } 2504#endif /* APIC_IO */ 2505} 2506 2507void 2508smp_masked_invlpg_range(u_int mask, vm_offset_t addr1, vm_offset_t addr2) 2509{ 2510#if defined(APIC_IO) 2511 if (smp_started) { 2512 smp_targeted_tlb_shootdown(mask, IPI_INVLRNG, addr1, addr2); 2513#ifdef COUNT_XINVLTLB_HITS 2514 ipi_masked_range++; 2515 ipi_masked_range_size += (addr2 - addr1) / PAGE_SIZE; 2516#endif 2517 } 2518#endif /* APIC_IO */ 2519} 2520 2521 2522/* 2523 * This is called once the rest of the system is up and running and we're 2524 * ready to let the AP's out of the pen. 2525 */ 2526void 2527ap_init(void) 2528{ 2529 u_int apic_id; 2530 2531 /* spin until all the AP's are ready */ 2532 while (!aps_ready) 2533 ia32_pause(); 2534 2535 /* BSP may have changed PTD while we were waiting */ 2536 invltlb(); 2537 2538#if defined(I586_CPU) && !defined(NO_F00F_HACK) 2539 lidt(&r_idt); 2540#endif 2541 2542 /* set up CPU registers and state */ 2543 cpu_setregs(); 2544 2545 /* set up FPU state on the AP */ 2546 npxinit(__INITIAL_NPXCW__); 2547 2548 /* set up SSE registers */ 2549 enable_sse(); 2550 2551 /* A quick check from sanity claus */ 2552 apic_id = (apic_id_to_logical[(lapic.id & 0x0f000000) >> 24]); 2553 if (PCPU_GET(cpuid) != apic_id) { 2554 printf("SMP: cpuid = %d\n", PCPU_GET(cpuid)); 2555 printf("SMP: apic_id = %d\n", apic_id); 2556 printf("PTD[MPPTDI] = %p\n", (void *)PTD[MPPTDI]); 2557 panic("cpuid mismatch! boom!!"); 2558 } 2559 2560 /* Init local apic for irq's */ 2561 apic_initialize(); 2562 2563 /* Set memory range attributes for this CPU to match the BSP */ 2564 mem_range_AP_init(); 2565 2566 mtx_lock_spin(&ap_boot_mtx); 2567 2568 smp_cpus++; 2569 2570 CTR1(KTR_SMP, "SMP: AP CPU #%d Launched", PCPU_GET(cpuid)); 2571 printf("SMP: AP CPU #%d Launched!\n", PCPU_GET(cpuid)); 2572 2573 /* Build our map of 'other' CPUs. */ 2574 PCPU_SET(other_cpus, all_cpus & ~PCPU_GET(cpumask)); 2575 2576 if (bootverbose) 2577 apic_dump("ap_init()"); 2578 2579 if (smp_cpus == mp_ncpus) { 2580 /* enable IPI's, tlb shootdown, freezes etc */ 2581 atomic_store_rel_int(&smp_started, 1); 2582 smp_active = 1; /* historic */ 2583 } 2584 2585 mtx_unlock_spin(&ap_boot_mtx); 2586 2587 /* wait until all the AP's are up */ 2588 while (smp_started == 0) 2589 ia32_pause(); 2590 2591 /* ok, now grab sched_lock and enter the scheduler */ 2592 mtx_lock_spin(&sched_lock); 2593 2594 binuptime(PCPU_PTR(switchtime)); 2595 PCPU_SET(switchticks, ticks); 2596 2597 cpu_throw(); /* doesn't return */ 2598 2599 panic("scheduler returned us to %s", __func__); 2600} 2601 2602/* 2603 * For statclock, we send an IPI to all CPU's to have them call this 2604 * function. 2605 * 2606 * WARNING! unpend() will call statclock() directly and skip this 2607 * routine. 2608 */ 2609void 2610forwarded_statclock(struct clockframe frame) 2611{ 2612 2613 if (profprocs != 0) 2614 profclock(&frame); 2615 if (pscnt == psdiv) 2616 statclock(&frame); 2617} 2618 2619void 2620forward_statclock(void) 2621{ 2622 int map; 2623 2624 CTR0(KTR_SMP, "forward_statclock"); 2625 2626 if (!smp_started || cold || panicstr) 2627 return; 2628 2629 map = PCPU_GET(other_cpus) & ~stopped_cpus ; 2630 if (map != 0) 2631 ipi_selected(map, IPI_STATCLOCK); 2632} 2633 2634/* 2635 * For each hardclock(), we send an IPI to all other CPU's to have them 2636 * execute this function. It would be nice to reduce contention on 2637 * sched_lock if we could simply peek at the CPU to determine the user/kernel 2638 * state and call hardclock_process() on the CPU receiving the clock interrupt 2639 * and then just use a simple IPI to handle any ast's if needed. 2640 * 2641 * WARNING! unpend() will call hardclock_process() directly and skip this 2642 * routine. 2643 */ 2644void 2645forwarded_hardclock(struct clockframe frame) 2646{ 2647 2648 hardclock_process(&frame); 2649} 2650 2651void 2652forward_hardclock(void) 2653{ 2654 u_int map; 2655 2656 CTR0(KTR_SMP, "forward_hardclock"); 2657 2658 if (!smp_started || cold || panicstr) 2659 return; 2660 2661 map = PCPU_GET(other_cpus) & ~stopped_cpus ; 2662 if (map != 0) 2663 ipi_selected(map, IPI_HARDCLOCK); 2664} 2665 2666#ifdef APIC_INTR_REORDER 2667/* 2668 * Maintain mapping from softintr vector to isr bit in local apic. 2669 */ 2670void 2671set_lapic_isrloc(int intr, int vector) 2672{ 2673 if (intr < 0 || intr > 32) 2674 panic("set_apic_isrloc: bad intr argument: %d",intr); 2675 if (vector < ICU_OFFSET || vector > 255) 2676 panic("set_apic_isrloc: bad vector argument: %d",vector); 2677 apic_isrbit_location[intr].location = &lapic.isr0 + ((vector>>5)<<2); 2678 apic_isrbit_location[intr].bit = (1<<(vector & 31)); 2679} 2680#endif 2681 2682/* 2683 * send an IPI to a set of cpus. 2684 */ 2685void 2686ipi_selected(u_int32_t cpus, u_int ipi) 2687{ 2688 2689 CTR3(KTR_SMP, "%s: cpus: %x ipi: %x", __func__, cpus, ipi); 2690 selected_apic_ipi(cpus, ipi, APIC_DELMODE_FIXED); 2691} 2692 2693/* 2694 * send an IPI INTerrupt containing 'vector' to all CPUs, including myself 2695 */ 2696void 2697ipi_all(u_int ipi) 2698{ 2699 2700 CTR2(KTR_SMP, "%s: ipi: %x", __func__, ipi); 2701 apic_ipi(APIC_DEST_ALLISELF, ipi, APIC_DELMODE_FIXED); 2702} 2703 2704/* 2705 * send an IPI to all CPUs EXCEPT myself 2706 */ 2707void 2708ipi_all_but_self(u_int ipi) 2709{ 2710 2711 CTR2(KTR_SMP, "%s: ipi: %x", __func__, ipi); 2712 apic_ipi(APIC_DEST_ALLESELF, ipi, APIC_DELMODE_FIXED); 2713} 2714 2715/* 2716 * send an IPI to myself 2717 */ 2718void 2719ipi_self(u_int ipi) 2720{ 2721 2722 CTR2(KTR_SMP, "%s: ipi: %x", __func__, ipi); 2723 apic_ipi(APIC_DEST_SELF, ipi, APIC_DELMODE_FIXED); 2724} 2725 2726static void 2727release_aps(void *dummy __unused) 2728{ 2729 2730 mtx_lock_spin(&sched_lock); 2731 atomic_store_rel_int(&aps_ready, 1); 2732 while (smp_started == 0) 2733 ia32_pause(); 2734 mtx_unlock_spin(&sched_lock); 2735} 2736 2737SYSINIT(start_aps, SI_SUB_SMP, SI_ORDER_FIRST, release_aps, NULL); 2738