subr_smp.c revision 36135
125164Speter/* 225164Speter * Copyright (c) 1996, by Steve Passe 325164Speter * All rights reserved. 425164Speter * 525164Speter * Redistribution and use in source and binary forms, with or without 625164Speter * modification, are permitted provided that the following conditions 725164Speter * are met: 825164Speter * 1. Redistributions of source code must retain the above copyright 925164Speter * notice, this list of conditions and the following disclaimer. 1025164Speter * 2. The name of the developer may NOT be used to endorse or promote products 1125164Speter * derived from this software without specific prior written permission. 1225164Speter * 1325164Speter * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 1425164Speter * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 1525164Speter * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 1625164Speter * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 1725164Speter * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 1825164Speter * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 1925164Speter * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 2025164Speter * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 2125164Speter * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 2225164Speter * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 2325164Speter * SUCH DAMAGE. 2425164Speter * 2536135Stegge * $Id: mp_machdep.c,v 1.75 1998/05/17 18:53:17 tegge Exp $ 2625164Speter */ 2725164Speter 2825164Speter#include "opt_smp.h" 2930265Speter#include "opt_vm86.h" 3034197Stegge#include "opt_cpu.h" 3125164Speter 3231639Sfsmp#ifdef SMP 3331639Sfsmp#include <machine/smptests.h> 3431639Sfsmp#else 3531639Sfsmp#error 3631639Sfsmp#endif 3731639Sfsmp 3828743Sbde#include <sys/param.h> 3925164Speter#include <sys/systm.h> 4028808Speter#include <sys/kernel.h> 4128808Speter#include <sys/proc.h> 4228808Speter#include <sys/sysctl.h> 4331639Sfsmp#ifdef BETTER_CLOCK 4431639Sfsmp#include <sys/dkstat.h> 4531639Sfsmp#endif 4625164Speter 4728808Speter#include <vm/vm.h> 4828808Speter#include <vm/vm_param.h> 4928808Speter#include <vm/pmap.h> 5026812Speter#include <vm/vm_kern.h> 5126812Speter#include <vm/vm_extern.h> 5231639Sfsmp#ifdef BETTER_CLOCK 5331639Sfsmp#include <sys/lock.h> 5431639Sfsmp#include <vm/vm_map.h> 5531639Sfsmp#include <sys/user.h> 5631689Stegge#ifdef GPROF 5731689Stegge#include <sys/gmon.h> 5831639Sfsmp#endif 5931689Stegge#endif 6025164Speter 6125164Speter#include <machine/smp.h> 6225164Speter#include <machine/apic.h> 6325164Speter#include <machine/mpapic.h> 6425164Speter#include <machine/segments.h> 6527697Sfsmp#include <machine/smptests.h> /** TEST_DEFAULT_CONFIG, TEST_TEST1 */ 6626812Speter#include <machine/tss.h> 6726896Stegge#include <machine/specialreg.h> 6830112Sdyson#include <machine/cputypes.h> 6935077Speter#include <machine/globaldata.h> 7025164Speter 7125164Speter#include <i386/i386/cons.h> /* cngetc() */ 7225164Speter 7325215Sfsmp#if defined(APIC_IO) 7427289Sfsmp#include <machine/md_var.h> /* setidt() */ 7527289Sfsmp#include <i386/isa/icu.h> /* IPIs */ 7627289Sfsmp#include <i386/isa/intr_machdep.h> /* IPIs */ 7725215Sfsmp#endif /* APIC_IO */ 7825164Speter 7928027Sfsmp#if defined(TEST_DEFAULT_CONFIG) 8028027Sfsmp#define MPFPS_MPFB1 TEST_DEFAULT_CONFIG 8128027Sfsmp#else 8228027Sfsmp#define MPFPS_MPFB1 mpfps->mpfb1 8328027Sfsmp#endif /* TEST_DEFAULT_CONFIG */ 8428027Sfsmp 8527005Sfsmp#define WARMBOOT_TARGET 0 8627005Sfsmp#define WARMBOOT_OFF (KERNBASE + 0x0467) 8727005Sfsmp#define WARMBOOT_SEG (KERNBASE + 0x0469) 8825164Speter 8927005Sfsmp#define BIOS_BASE (0xf0000) 9027005Sfsmp#define BIOS_SIZE (0x10000) 9127005Sfsmp#define BIOS_COUNT (BIOS_SIZE/4) 9225164Speter 9327005Sfsmp#define CMOS_REG (0x70) 9427005Sfsmp#define CMOS_DATA (0x71) 9527005Sfsmp#define BIOS_RESET (0x0f) 9627005Sfsmp#define BIOS_WARM (0x0a) 9725164Speter 9826155Sfsmp#define PROCENTRY_FLAG_EN 0x01 9926155Sfsmp#define PROCENTRY_FLAG_BP 0x02 10026155Sfsmp#define IOAPICENTRY_FLAG_EN 0x01 10126155Sfsmp 10227005Sfsmp 10326155Sfsmp/* MP Floating Pointer Structure */ 10426155Sfsmptypedef struct MPFPS { 10526155Sfsmp char signature[4]; 10626155Sfsmp void *pap; 10726155Sfsmp u_char length; 10826155Sfsmp u_char spec_rev; 10926155Sfsmp u_char checksum; 11026155Sfsmp u_char mpfb1; 11126155Sfsmp u_char mpfb2; 11226155Sfsmp u_char mpfb3; 11326155Sfsmp u_char mpfb4; 11426155Sfsmp u_char mpfb5; 11526155Sfsmp} *mpfps_t; 11626155Sfsmp 11726155Sfsmp/* MP Configuration Table Header */ 11826155Sfsmptypedef struct MPCTH { 11926155Sfsmp char signature[4]; 12026155Sfsmp u_short base_table_length; 12126155Sfsmp u_char spec_rev; 12226155Sfsmp u_char checksum; 12326155Sfsmp u_char oem_id[8]; 12426155Sfsmp u_char product_id[12]; 12526155Sfsmp void *oem_table_pointer; 12626155Sfsmp u_short oem_table_size; 12726155Sfsmp u_short entry_count; 12826155Sfsmp void *apic_address; 12926155Sfsmp u_short extended_table_length; 13026155Sfsmp u_char extended_table_checksum; 13126155Sfsmp u_char reserved; 13226155Sfsmp} *mpcth_t; 13326155Sfsmp 13426155Sfsmp 13526155Sfsmptypedef struct PROCENTRY { 13626155Sfsmp u_char type; 13726155Sfsmp u_char apic_id; 13826155Sfsmp u_char apic_version; 13926155Sfsmp u_char cpu_flags; 14026155Sfsmp u_long cpu_signature; 14126155Sfsmp u_long feature_flags; 14226155Sfsmp u_long reserved1; 14326155Sfsmp u_long reserved2; 14426155Sfsmp} *proc_entry_ptr; 14526155Sfsmp 14626155Sfsmptypedef struct BUSENTRY { 14726155Sfsmp u_char type; 14826155Sfsmp u_char bus_id; 14926155Sfsmp char bus_type[6]; 15026155Sfsmp} *bus_entry_ptr; 15126155Sfsmp 15226155Sfsmptypedef struct IOAPICENTRY { 15326155Sfsmp u_char type; 15426155Sfsmp u_char apic_id; 15526155Sfsmp u_char apic_version; 15626155Sfsmp u_char apic_flags; 15726155Sfsmp void *apic_address; 15826155Sfsmp} *io_apic_entry_ptr; 15926155Sfsmp 16026155Sfsmptypedef struct INTENTRY { 16126155Sfsmp u_char type; 16226155Sfsmp u_char int_type; 16326155Sfsmp u_short int_flags; 16426155Sfsmp u_char src_bus_id; 16526155Sfsmp u_char src_bus_irq; 16626155Sfsmp u_char dst_apic_id; 16726155Sfsmp u_char dst_apic_int; 16826155Sfsmp} *int_entry_ptr; 16926155Sfsmp 17026155Sfsmp/* descriptions of MP basetable entries */ 17126155Sfsmptypedef struct BASETABLE_ENTRY { 17226155Sfsmp u_char type; 17326155Sfsmp u_char length; 17426155Sfsmp char name[16]; 17526155Sfsmp} basetable_entry; 17626155Sfsmp 17725164Speter/* 17825164Speter * this code MUST be enabled here and in mpboot.s. 17925164Speter * it follows the very early stages of AP boot by placing values in CMOS ram. 18025164Speter * it NORMALLY will never be needed and thus the primitive method for enabling. 18125164Speter * 18225164Speter#define CHECK_POINTS 18325164Speter */ 18425164Speter 18525164Speter#if defined(CHECK_POINTS) 18625164Speter#define CHECK_READ(A) (outb(CMOS_REG, (A)), inb(CMOS_DATA)) 18725164Speter#define CHECK_WRITE(A,D) (outb(CMOS_REG, (A)), outb(CMOS_DATA, (D))) 18825164Speter 18925164Speter#define CHECK_INIT(D); \ 19025164Speter CHECK_WRITE(0x34, (D)); \ 19125164Speter CHECK_WRITE(0x35, (D)); \ 19225164Speter CHECK_WRITE(0x36, (D)); \ 19325164Speter CHECK_WRITE(0x37, (D)); \ 19425164Speter CHECK_WRITE(0x38, (D)); \ 19525164Speter CHECK_WRITE(0x39, (D)); 19625164Speter 19725164Speter#define CHECK_PRINT(S); \ 19825164Speter printf("%s: %d, %d, %d, %d, %d, %d\n", \ 19925164Speter (S), \ 20025164Speter CHECK_READ(0x34), \ 20125164Speter CHECK_READ(0x35), \ 20225164Speter CHECK_READ(0x36), \ 20325164Speter CHECK_READ(0x37), \ 20425164Speter CHECK_READ(0x38), \ 20525164Speter CHECK_READ(0x39)); 20625164Speter 20725164Speter#else /* CHECK_POINTS */ 20825164Speter 20925164Speter#define CHECK_INIT(D) 21025164Speter#define CHECK_PRINT(S) 21125164Speter 21225164Speter#endif /* CHECK_POINTS */ 21325164Speter 21427005Sfsmp/* 21527005Sfsmp * Values to send to the POST hardware. 21627005Sfsmp */ 21727005Sfsmp#define MP_BOOTADDRESS_POST 0x10 21827005Sfsmp#define MP_PROBE_POST 0x11 21929213Sfsmp#define MPTABLE_PASS1_POST 0x12 22029213Sfsmp 22129213Sfsmp#define MP_START_POST 0x13 22229213Sfsmp#define MP_ENABLE_POST 0x14 22327005Sfsmp#define MPTABLE_PASS2_POST 0x15 22427005Sfsmp 22529213Sfsmp#define START_ALL_APS_POST 0x16 22629213Sfsmp#define INSTALL_AP_TRAMP_POST 0x17 22729213Sfsmp#define START_AP_POST 0x18 22829213Sfsmp 22929213Sfsmp#define MP_ANNOUNCE_POST 0x19 23029213Sfsmp 23129213Sfsmp 23227289Sfsmp/** XXX FIXME: where does this really belong, isa.h/isa.c perhaps? */ 23327255Sfsmpint current_postcode; 23427255Sfsmp 23527289Sfsmp/** XXX FIXME: what system files declare these??? */ 23625164Speterextern struct region_descriptor r_gdt, r_idt; 23725164Speter 23828669Sfsmpint bsp_apic_ready = 0; /* flags useability of BSP apic */ 23926155Sfsmpint mp_ncpus; /* # of CPUs, including BSP */ 24026155Sfsmpint mp_naps; /* # of Applications processors */ 24126155Sfsmpint mp_nbusses; /* # of busses */ 24226155Sfsmpint mp_napics; /* # of IO APICs */ 24326155Sfsmpint boot_cpu_id; /* designated BSP */ 24425164Spetervm_offset_t cpu_apic_address; 24526108Sfsmpvm_offset_t io_apic_address[NAPICID]; /* NAPICID is more than enough */ 24629655Sdysonextern int nkpt; 24725164Speter 24825164Speteru_int32_t cpu_apic_versions[NCPU]; 24925164Speteru_int32_t io_apic_versions[NAPIC]; 25025164Speter 25134021Stegge#ifdef APIC_INTR_DIAGNOSTIC 25234021Steggeint apic_itrace_enter[32]; 25334021Steggeint apic_itrace_tryisrlock[32]; 25434021Steggeint apic_itrace_gotisrlock[32]; 25534021Steggeint apic_itrace_active[32]; 25634021Steggeint apic_itrace_masked[32]; 25734021Steggeint apic_itrace_noisrlock[32]; 25834021Steggeint apic_itrace_masked2[32]; 25934021Steggeint apic_itrace_unmask[32]; 26034021Steggeint apic_itrace_noforward[32]; 26134021Steggeint apic_itrace_leave[32]; 26234021Steggeint apic_itrace_enter2[32]; 26334021Steggeint apic_itrace_doreti[32]; 26434021Steggeint apic_itrace_splz[32]; 26534021Steggeint apic_itrace_eoi[32]; 26634021Stegge#ifdef APIC_INTR_DIAGNOSTIC_IRQ 26734021Steggeunsigned short apic_itrace_debugbuffer[32768]; 26834021Steggeint apic_itrace_debugbuffer_idx; 26934021Steggestruct simplelock apic_itrace_debuglock; 27034021Stegge#endif 27134021Stegge#endif 27234021Stegge 27334021Stegge#ifdef APIC_INTR_REORDER 27434021Steggestruct { 27534021Stegge volatile int *location; 27634021Stegge int bit; 27734021Stegge} apic_isrbit_location[32]; 27834021Stegge#endif 27934021Stegge 28025164Speter/* 28126108Sfsmp * APIC ID logical/physical mapping structures. 28226108Sfsmp * We oversize these to simplify boot-time config. 28325164Speter */ 28426108Sfsmpint cpu_num_to_apic_id[NAPICID]; 28526108Sfsmpint io_num_to_apic_id[NAPICID]; 28625164Speterint apic_id_to_logical[NAPICID]; 28725164Speter 28830112Sdyson 28927005Sfsmp/* Bitmap of all available CPUs */ 29027005Sfsmpu_int all_cpus; 29127005Sfsmp 29233181Seivind/* AP uses this PTD during bootstrap. Do not staticize. */ 29328808Speterpd_entry_t *bootPTD; 29426812Speter 29526812Speter/* Hotwire a 0->4MB V==P mapping */ 29628808Speterextern pt_entry_t *KPTphys; 29726812Speter 29827289Sfsmp/* Virtual address of per-cpu common_tss */ 29926812Speterextern struct i386tss common_tss; 30028984Sbde#ifdef VM86 30129663Speterextern struct segment_descriptor common_tssd; 30228984Sbdeextern u_int private_tss; /* flag indicating private tss */ 30329663Speterextern u_int my_tr; 30428984Sbde#endif /* VM86 */ 30526812Speter 30628808Speter/* IdlePTD per cpu */ 30728808Speterpd_entry_t *IdlePTDS[NCPU]; 30828808Speter 30928808Speter/* "my" private page table page, for BSP init */ 31028808Speterextern pt_entry_t SMP_prvpt[]; 31128808Speter 31228808Speter/* Private page pointer to curcpu's PTD, used during BSP init */ 31328808Speterextern pd_entry_t *my_idlePTD; 31428808Speter 31536135Steggestruct pcb stoppcbs[NCPU]; 31636135Stegge 31728809Speterstatic int smp_started; /* has the system started? */ 31828809Speter 31925164Speter/* 32027289Sfsmp * Local data and functions. 32125164Speter */ 32225164Speter 32326155Sfsmpstatic int mp_capable; 32426155Sfsmpstatic u_int boot_address; 32526155Sfsmpstatic u_int base_memory; 32625164Speter 32726155Sfsmpstatic int picmode; /* 0: virtual wire mode, 1: PIC mode */ 32826155Sfsmpstatic mpfps_t mpfps; 32926155Sfsmpstatic int search_for_sig(u_int32_t target, int count); 33026155Sfsmpstatic void mp_enable(u_int boot_addr); 33125164Speter 33226155Sfsmpstatic int mptable_pass1(void); 33326155Sfsmpstatic int mptable_pass2(void); 33426155Sfsmpstatic void default_mp_table(int type); 33528027Sfsmpstatic void fix_mp_table(void); 33627634Sfsmpstatic void init_locks(void); 33726155Sfsmpstatic int start_all_aps(u_int boot_addr); 33826155Sfsmpstatic void install_ap_tramp(u_int boot_addr); 33926155Sfsmpstatic int start_ap(int logicalCpu, u_int boot_addr); 34025164Speter 34125164Speter/* 34227289Sfsmp * Calculate usable address in base memory for AP trampoline code. 34325164Speter */ 34425164Speteru_int 34525164Spetermp_bootaddress(u_int basemem) 34625164Speter{ 34727005Sfsmp POSTCODE(MP_BOOTADDRESS_POST); 34827005Sfsmp 34925164Speter base_memory = basemem * 1024; /* convert to bytes */ 35025164Speter 35125164Speter boot_address = base_memory & ~0xfff; /* round down to 4k boundary */ 35225164Speter if ((base_memory - boot_address) < bootMP_size) 35325164Speter boot_address -= 4096; /* not enough, lower by 4k */ 35425164Speter 35525164Speter return boot_address; 35625164Speter} 35725164Speter 35825164Speter 35927289Sfsmp/* 36027289Sfsmp * Look for an Intel MP spec table (ie, SMP capable hardware). 36127289Sfsmp */ 36226155Sfsmpint 36326155Sfsmpmp_probe(void) 36426155Sfsmp{ 36526155Sfsmp int x; 36626155Sfsmp u_long segment; 36726155Sfsmp u_int32_t target; 36826155Sfsmp 36927005Sfsmp POSTCODE(MP_PROBE_POST); 37027005Sfsmp 37126155Sfsmp /* see if EBDA exists */ 37226155Sfsmp if (segment = (u_long) * (u_short *) (KERNBASE + 0x40e)) { 37326155Sfsmp /* search first 1K of EBDA */ 37426155Sfsmp target = (u_int32_t) (segment << 4); 37526155Sfsmp if ((x = search_for_sig(target, 1024 / 4)) >= 0) 37626155Sfsmp goto found; 37726155Sfsmp } else { 37826155Sfsmp /* last 1K of base memory, effective 'top of base' passed in */ 37926155Sfsmp target = (u_int32_t) (base_memory - 0x400); 38026155Sfsmp if ((x = search_for_sig(target, 1024 / 4)) >= 0) 38126155Sfsmp goto found; 38226155Sfsmp } 38326155Sfsmp 38426155Sfsmp /* search the BIOS */ 38526155Sfsmp target = (u_int32_t) BIOS_BASE; 38626155Sfsmp if ((x = search_for_sig(target, BIOS_COUNT)) >= 0) 38726155Sfsmp goto found; 38826155Sfsmp 38926155Sfsmp /* nothing found */ 39026155Sfsmp mpfps = (mpfps_t)0; 39126155Sfsmp mp_capable = 0; 39226155Sfsmp return 0; 39326155Sfsmp 39427289Sfsmpfound: 39526155Sfsmp /* calculate needed resources */ 39626155Sfsmp mpfps = (mpfps_t)x; 39726155Sfsmp if (mptable_pass1()) 39826155Sfsmp panic("you must reconfigure your kernel"); 39926155Sfsmp 40026155Sfsmp /* flag fact that we are running multiple processors */ 40126155Sfsmp mp_capable = 1; 40226155Sfsmp return 1; 40326155Sfsmp} 40426155Sfsmp 40526155Sfsmp 40625164Speter/* 40727289Sfsmp * Startup the SMP processors. 40825164Speter */ 40925164Spetervoid 41025164Spetermp_start(void) 41125164Speter{ 41227005Sfsmp POSTCODE(MP_START_POST); 41327005Sfsmp 41425164Speter /* look for MP capable motherboard */ 41526155Sfsmp if (mp_capable) 41625164Speter mp_enable(boot_address); 41726101Sfsmp else 41826155Sfsmp panic("MP hardware not found!"); 41925164Speter} 42025164Speter 42125164Speter 42225164Speter/* 42327289Sfsmp * Print various information about the SMP system hardware and setup. 42425164Speter */ 42525164Spetervoid 42625164Spetermp_announce(void) 42725164Speter{ 42825164Speter int x; 42925164Speter 43027005Sfsmp POSTCODE(MP_ANNOUNCE_POST); 43127005Sfsmp 43225164Speter printf("FreeBSD/SMP: Multiprocessor motherboard\n"); 43327489Sfsmp printf(" cpu0 (BSP): apic id: %2d", CPU_TO_ID(0)); 43426812Speter printf(", version: 0x%08x", cpu_apic_versions[0]); 43526812Speter printf(", at 0x%08x\n", cpu_apic_address); 43625164Speter for (x = 1; x <= mp_naps; ++x) { 43727561Sfsmp printf(" cpu%d (AP): apic id: %2d", x, CPU_TO_ID(x)); 43826812Speter printf(", version: 0x%08x", cpu_apic_versions[x]); 43926812Speter printf(", at 0x%08x\n", cpu_apic_address); 44025164Speter } 44125164Speter 44225164Speter#if defined(APIC_IO) 44325164Speter for (x = 0; x < mp_napics; ++x) { 44427489Sfsmp printf(" io%d (APIC): apic id: %2d", x, IO_TO_ID(x)); 44526812Speter printf(", version: 0x%08x", io_apic_versions[x]); 44626812Speter printf(", at 0x%08x\n", io_apic_address[x]); 44725164Speter } 44825164Speter#else 44925164Speter printf(" Warning: APIC I/O disabled\n"); 45025164Speter#endif /* APIC_IO */ 45125164Speter} 45225164Speter 45325164Speter/* 45425164Speter * AP cpu's call this to sync up protected mode. 45525164Speter */ 45625164Spetervoid 45725164Speterinit_secondary(void) 45825164Speter{ 45929663Speter int gsel_tss; 46029663Speter#ifndef VM86 46129663Speter u_int my_tr; 46229663Speter#endif 46325164Speter 46425164Speter r_gdt.rd_limit = sizeof(gdt[0]) * (NGDT + NCPU) - 1; 46525164Speter r_gdt.rd_base = (int) gdt; 46627289Sfsmp lgdt(&r_gdt); /* does magic intra-segment return */ 46725164Speter lidt(&r_idt); 46825164Speter lldt(_default_ldt); 46925164Speter 47029663Speter my_tr = NGDT + cpuid; 47129663Speter gsel_tss = GSEL(my_tr, SEL_KPL); 47229663Speter gdt[my_tr].sd.sd_type = SDT_SYS386TSS; 47326812Speter common_tss.tss_esp0 = 0; /* not used until after switch */ 47426812Speter common_tss.tss_ss0 = GSEL(GDATA_SEL, SEL_KPL); 47526812Speter common_tss.tss_ioopt = (sizeof common_tss) << 16; 47628984Sbde#ifdef VM86 47729663Speter common_tssd = gdt[my_tr].sd; 47828984Sbde private_tss = 0; 47928984Sbde#endif /* VM86 */ 48025164Speter ltr(gsel_tss); 48125164Speter 48227289Sfsmp load_cr0(0x8005003b); /* XXX! */ 48326812Speter 48426812Speter PTD[0] = 0; 48527484Sdyson pmap_set_opt((unsigned *)PTD); 48627484Sdyson 48730112Sdyson putmtrr(); 48835932Sdyson pmap_setvidram(); 48930112Sdyson 49026812Speter invltlb(); 49125164Speter} 49225164Speter 49325164Speter 49425164Speter#if defined(APIC_IO) 49527289Sfsmp/* 49627289Sfsmp * Final configuration of the BSP's local APIC: 49727289Sfsmp * - disable 'pic mode'. 49827289Sfsmp * - disable 'virtual wire mode'. 49927289Sfsmp * - enable NMI. 50027289Sfsmp */ 50125164Spetervoid 50227289Sfsmpbsp_apic_configure(void) 50325164Speter{ 50427289Sfsmp u_char byte; 50527289Sfsmp u_int32_t temp; 50625164Speter 50727289Sfsmp /* leave 'pic mode' if necessary */ 50825164Speter if (picmode) { 50925164Speter outb(0x22, 0x70); /* select IMCR */ 51025164Speter byte = inb(0x23); /* current contents */ 51127289Sfsmp byte |= 0x01; /* mask external INTR */ 51225164Speter outb(0x23, byte); /* disconnect 8259s/NMI */ 51325164Speter } 51427001Sfsmp 51527001Sfsmp /* mask lint0 (the 8259 'virtual wire' connection) */ 51626812Speter temp = lapic.lvt_lint0; 51727289Sfsmp temp |= APIC_LVT_M; /* set the mask */ 51826812Speter lapic.lvt_lint0 = temp; 51927001Sfsmp 52027001Sfsmp /* setup lint1 to handle NMI */ 52127001Sfsmp temp = lapic.lvt_lint1; 52227289Sfsmp temp &= ~APIC_LVT_M; /* clear the mask */ 52327289Sfsmp lapic.lvt_lint1 = temp; 52427001Sfsmp 52527353Sfsmp if (bootverbose) 52627561Sfsmp apic_dump("bsp_apic_configure()"); 52725164Speter} 52827005Sfsmp#endif /* APIC_IO */ 52925164Speter 53025164Speter 53125164Speter/******************************************************************* 53225164Speter * local functions and data 53325164Speter */ 53425164Speter 53525164Speter/* 53625164Speter * start the SMP system 53725164Speter */ 53825164Speterstatic void 53925164Spetermp_enable(u_int boot_addr) 54025164Speter{ 54125164Speter int x; 54225164Speter#if defined(APIC_IO) 54325164Speter int apic; 54425164Speter u_int ux; 54525164Speter#endif /* APIC_IO */ 54625164Speter 54730112Sdyson getmtrr(); 54835932Sdyson pmap_setvidram(); 54930112Sdyson 55027005Sfsmp POSTCODE(MP_ENABLE_POST); 55127005Sfsmp 55227289Sfsmp /* turn on 4MB of V == P addressing so we can get to MP table */ 55326812Speter *(int *)PTD = PG_V | PG_RW | ((u_long)KPTphys & PG_FRAME); 55426812Speter invltlb(); 55526812Speter 55626812Speter /* examine the MP table for needed info, uses physical addresses */ 55726155Sfsmp x = mptable_pass2(); 55825164Speter 55926812Speter *(int *)PTD = 0; 56026812Speter invltlb(); 56125164Speter 56225164Speter /* can't process default configs till the CPU APIC is pmapped */ 56325164Speter if (x) 56425164Speter default_mp_table(x); 56525164Speter 56628027Sfsmp /* post scan cleanup */ 56728027Sfsmp fix_mp_table(); 56828027Sfsmp 56925164Speter#if defined(APIC_IO) 57027353Sfsmp 57125164Speter /* fill the LOGICAL io_apic_versions table */ 57225164Speter for (apic = 0; apic < mp_napics; ++apic) { 57325164Speter ux = io_apic_read(apic, IOAPIC_VER); 57425164Speter io_apic_versions[apic] = ux; 57525164Speter } 57625164Speter 57725216Sfsmp /* program each IO APIC in the system */ 57825164Speter for (apic = 0; apic < mp_napics; ++apic) 57926266Speter if (io_apic_setup(apic) < 0) 58026266Speter panic("IO APIC setup failure"); 58125164Speter 58227353Sfsmp /* install a 'Spurious INTerrupt' vector */ 58327353Sfsmp setidt(XSPURIOUSINT_OFFSET, Xspuriousint, 58427353Sfsmp SDT_SYS386IGT, SEL_KPL, GSEL(GCODE_SEL, SEL_KPL)); 58527353Sfsmp 58625204Sfsmp /* install an inter-CPU IPI for TLB invalidation */ 58727005Sfsmp setidt(XINVLTLB_OFFSET, Xinvltlb, 58825216Sfsmp SDT_SYS386IGT, SEL_KPL, GSEL(GCODE_SEL, SEL_KPL)); 58927005Sfsmp 59031639Sfsmp#ifdef BETTER_CLOCK 59131639Sfsmp /* install an inter-CPU IPI for reading processor state */ 59231639Sfsmp setidt(XCPUCHECKSTATE_OFFSET, Xcpucheckstate, 59331639Sfsmp SDT_SYS386IGT, SEL_KPL, GSEL(GCODE_SEL, SEL_KPL)); 59434020Stegge#endif 59531639Sfsmp 59631639Sfsmp /* install an inter-CPU IPI for forcing an additional software trap */ 59731639Sfsmp setidt(XCPUAST_OFFSET, Xcpuast, 59831639Sfsmp SDT_SYS386IGT, SEL_KPL, GSEL(GCODE_SEL, SEL_KPL)); 59931639Sfsmp 60034021Stegge /* install an inter-CPU IPI for interrupt forwarding */ 60134021Stegge setidt(XFORWARD_IRQ_OFFSET, Xforward_irq, 60234021Stegge SDT_SYS386IGT, SEL_KPL, GSEL(GCODE_SEL, SEL_KPL)); 60334021Stegge 60427005Sfsmp /* install an inter-CPU IPI for CPU stop/restart */ 60527005Sfsmp setidt(XCPUSTOP_OFFSET, Xcpustop, 60627005Sfsmp SDT_SYS386IGT, SEL_KPL, GSEL(GCODE_SEL, SEL_KPL)); 60727255Sfsmp 60827353Sfsmp#if defined(TEST_TEST1) 60927517Sfsmp /* install a "fake hardware INTerrupt" vector */ 61027353Sfsmp setidt(XTEST1_OFFSET, Xtest1, 61127255Sfsmp SDT_SYS386IGT, SEL_KPL, GSEL(GCODE_SEL, SEL_KPL)); 61227353Sfsmp#endif /** TEST_TEST1 */ 61327353Sfsmp 61425164Speter#endif /* APIC_IO */ 61525164Speter 61627634Sfsmp /* initialize all SMP locks */ 61727634Sfsmp init_locks(); 61827634Sfsmp 61925164Speter /* start each Application Processor */ 62025164Speter start_all_aps(boot_addr); 62126896Stegge 62226896Stegge /* 62326896Stegge * The init process might be started on a different CPU now, 62426896Stegge * and the boot CPU might not call prepare_usermode to get 62526896Stegge * cr0 correctly configured. Thus we initialize cr0 here. 62626896Stegge */ 62726896Stegge load_cr0(rcr0() | CR0_WP | CR0_AM); 62825164Speter} 62925164Speter 63025164Speter 63125164Speter/* 63225164Speter * look for the MP spec signature 63325164Speter */ 63425164Speter 63525164Speter/* string defined by the Intel MP Spec as identifying the MP table */ 63625164Speter#define MP_SIG 0x5f504d5f /* _MP_ */ 63725164Speter#define NEXT(X) ((X) += 4) 63825164Speterstatic int 63925164Spetersearch_for_sig(u_int32_t target, int count) 64025164Speter{ 64125164Speter int x; 64225164Speter u_int32_t *addr = (u_int32_t *) (KERNBASE + target); 64325164Speter 64425164Speter for (x = 0; x < count; NEXT(x)) 64525164Speter if (addr[x] == MP_SIG) 64625164Speter /* make array index a byte index */ 64725164Speter return (target + (x * sizeof(u_int32_t))); 64825164Speter 64925164Speter return -1; 65025164Speter} 65125164Speter 65225164Speter 65325164Speterstatic basetable_entry basetable_entry_types[] = 65425164Speter{ 65525164Speter {0, 20, "Processor"}, 65625164Speter {1, 8, "Bus"}, 65725164Speter {2, 8, "I/O APIC"}, 65825164Speter {3, 8, "I/O INT"}, 65925164Speter {4, 8, "Local INT"} 66025164Speter}; 66125164Speter 66225164Spetertypedef struct BUSDATA { 66325164Speter u_char bus_id; 66425164Speter enum busTypes bus_type; 66525164Speter} bus_datum; 66625164Speter 66725164Spetertypedef struct INTDATA { 66825164Speter u_char int_type; 66925164Speter u_short int_flags; 67025164Speter u_char src_bus_id; 67125164Speter u_char src_bus_irq; 67225164Speter u_char dst_apic_id; 67325164Speter u_char dst_apic_int; 67425164Speter} io_int, local_int; 67525164Speter 67625164Spetertypedef struct BUSTYPENAME { 67725164Speter u_char type; 67825164Speter char name[7]; 67925164Speter} bus_type_name; 68025164Speter 68125164Speterstatic bus_type_name bus_type_table[] = 68225164Speter{ 68325164Speter {CBUS, "CBUS"}, 68425164Speter {CBUSII, "CBUSII"}, 68525164Speter {EISA, "EISA"}, 68625164Speter {UNKNOWN_BUSTYPE, "---"}, 68725164Speter {UNKNOWN_BUSTYPE, "---"}, 68825164Speter {ISA, "ISA"}, 68925164Speter {UNKNOWN_BUSTYPE, "---"}, 69025164Speter {UNKNOWN_BUSTYPE, "---"}, 69125164Speter {UNKNOWN_BUSTYPE, "---"}, 69225164Speter {UNKNOWN_BUSTYPE, "---"}, 69325164Speter {UNKNOWN_BUSTYPE, "---"}, 69425164Speter {UNKNOWN_BUSTYPE, "---"}, 69525164Speter {PCI, "PCI"}, 69625164Speter {UNKNOWN_BUSTYPE, "---"}, 69725164Speter {UNKNOWN_BUSTYPE, "---"}, 69825164Speter {UNKNOWN_BUSTYPE, "---"}, 69925164Speter {UNKNOWN_BUSTYPE, "---"}, 70025164Speter {XPRESS, "XPRESS"}, 70125164Speter {UNKNOWN_BUSTYPE, "---"} 70225164Speter}; 70325164Speter/* from MP spec v1.4, table 5-1 */ 70425164Speterstatic int default_data[7][5] = 70525164Speter{ 70625164Speter/* nbus, id0, type0, id1, type1 */ 70725164Speter {1, 0, ISA, 255, 255}, 70825164Speter {1, 0, EISA, 255, 255}, 70925164Speter {1, 0, EISA, 255, 255}, 71025164Speter {0, 255, 255, 255, 255},/* MCA not supported */ 71125164Speter {2, 0, ISA, 1, PCI}, 71225164Speter {2, 0, EISA, 1, PCI}, 71325164Speter {0, 255, 255, 255, 255} /* MCA not supported */ 71425164Speter}; 71525164Speter 71625164Speter 71725164Speter/* the bus data */ 71833181Seivindstatic bus_datum bus_data[NBUS]; 71925164Speter 72025164Speter/* the IO INT data, one entry per possible APIC INTerrupt */ 72133181Seivindstatic io_int io_apic_ints[NINTR]; 72225164Speter 72325164Speterstatic int nintrs; 72425164Speter 72526108Sfsmpstatic int processor_entry __P((proc_entry_ptr entry, int cpu)); 72626108Sfsmpstatic int bus_entry __P((bus_entry_ptr entry, int bus)); 72726108Sfsmpstatic int io_apic_entry __P((io_apic_entry_ptr entry, int apic)); 72826108Sfsmpstatic int int_entry __P((int_entry_ptr entry, int intr)); 72926108Sfsmpstatic int lookup_bus_type __P((char *name)); 73025164Speter 73125164Speter 73225164Speter/* 73326155Sfsmp * 1st pass on motherboard's Intel MP specification table. 73426155Sfsmp * 73526155Sfsmp * initializes: 73626155Sfsmp * mp_ncpus = 1 73726155Sfsmp * 73826155Sfsmp * determines: 73926155Sfsmp * cpu_apic_address (common to all CPUs) 74026155Sfsmp * io_apic_address[N] 74126155Sfsmp * mp_naps 74226155Sfsmp * mp_nbusses 74326155Sfsmp * mp_napics 74426155Sfsmp * nintrs 74525164Speter */ 74625164Speterstatic int 74726155Sfsmpmptable_pass1(void) 74825164Speter{ 74926108Sfsmp int x; 75026108Sfsmp mpcth_t cth; 75126108Sfsmp int totalSize; 75226108Sfsmp void* position; 75326108Sfsmp int count; 75426108Sfsmp int type; 75526108Sfsmp int mustpanic; 75625164Speter 75727005Sfsmp POSTCODE(MPTABLE_PASS1_POST); 75827005Sfsmp 75926108Sfsmp mustpanic = 0; 76026108Sfsmp 76126155Sfsmp /* clear various tables */ 76226155Sfsmp for (x = 0; x < NAPICID; ++x) { 76326155Sfsmp io_apic_address[x] = ~0; /* IO APIC address table */ 76426155Sfsmp } 76525164Speter 76626108Sfsmp /* init everything to empty */ 76726108Sfsmp mp_naps = 0; 76826108Sfsmp mp_nbusses = 0; 76926108Sfsmp mp_napics = 0; 77026108Sfsmp nintrs = 0; 77126108Sfsmp 77226108Sfsmp /* check for use of 'default' configuration */ 77328027Sfsmp if (MPFPS_MPFB1 != 0) { 77426108Sfsmp /* use default addresses */ 77526108Sfsmp cpu_apic_address = DEFAULT_APIC_BASE; 77626108Sfsmp io_apic_address[0] = DEFAULT_IO_APIC_BASE; 77726108Sfsmp 77826108Sfsmp /* fill in with defaults */ 77926882Sfsmp mp_naps = 2; /* includes BSP */ 78028027Sfsmp mp_nbusses = default_data[MPFPS_MPFB1 - 1][0]; 78126108Sfsmp#if defined(APIC_IO) 78226108Sfsmp mp_napics = 1; 78326108Sfsmp nintrs = 16; 78426108Sfsmp#endif /* APIC_IO */ 78526108Sfsmp } 78626108Sfsmp else { 78726155Sfsmp if ((cth = mpfps->pap) == 0) 78826108Sfsmp panic("MP Configuration Table Header MISSING!"); 78926108Sfsmp 79026108Sfsmp cpu_apic_address = (vm_offset_t) cth->apic_address; 79126108Sfsmp 79226108Sfsmp /* walk the table, recording info of interest */ 79326108Sfsmp totalSize = cth->base_table_length - sizeof(struct MPCTH); 79426108Sfsmp position = (u_char *) cth + sizeof(struct MPCTH); 79526108Sfsmp count = cth->entry_count; 79626108Sfsmp 79726108Sfsmp while (count--) { 79826108Sfsmp switch (type = *(u_char *) position) { 79926108Sfsmp case 0: /* processor_entry */ 80026108Sfsmp if (((proc_entry_ptr)position)->cpu_flags 80126108Sfsmp & PROCENTRY_FLAG_EN) 80226108Sfsmp ++mp_naps; 80326108Sfsmp break; 80426108Sfsmp case 1: /* bus_entry */ 80526108Sfsmp ++mp_nbusses; 80626108Sfsmp break; 80726108Sfsmp case 2: /* io_apic_entry */ 80826108Sfsmp if (((io_apic_entry_ptr)position)->apic_flags 80926108Sfsmp & IOAPICENTRY_FLAG_EN) 81026108Sfsmp io_apic_address[mp_napics++] = 81126108Sfsmp (vm_offset_t)((io_apic_entry_ptr) 81226108Sfsmp position)->apic_address; 81326108Sfsmp break; 81426108Sfsmp case 3: /* int_entry */ 81526108Sfsmp ++nintrs; 81626108Sfsmp break; 81726108Sfsmp case 4: /* int_entry */ 81826108Sfsmp break; 81926108Sfsmp default: 82026108Sfsmp panic("mpfps Base Table HOSED!"); 82126108Sfsmp /* NOTREACHED */ 82226108Sfsmp } 82326108Sfsmp 82426108Sfsmp totalSize -= basetable_entry_types[type].length; 82526108Sfsmp (u_char*)position += basetable_entry_types[type].length; 82626108Sfsmp } 82726108Sfsmp } 82826108Sfsmp 82926108Sfsmp /* qualify the numbers */ 83026108Sfsmp if (mp_naps > NCPU) 83128041Sfsmp#if 0 /* XXX FIXME: kern/4255 */ 83226108Sfsmp printf("Warning: only using %d of %d available CPUs!\n", 83326108Sfsmp NCPU, mp_naps); 83428041Sfsmp#else 83528041Sfsmp { 83628041Sfsmp printf("NCPU cannot be different than actual CPU count.\n"); 83728041Sfsmp printf(" add 'options NCPU=%d' to your kernel config file,\n", 83828041Sfsmp mp_naps); 83928041Sfsmp printf(" then rerun config & rebuild your SMP kernel\n"); 84026108Sfsmp mustpanic = 1; 84128041Sfsmp } 84228041Sfsmp#endif /* XXX FIXME: kern/4255 */ 84326108Sfsmp if (mp_nbusses > NBUS) { 84426108Sfsmp printf("found %d busses, increase NBUS\n", mp_nbusses); 84526108Sfsmp mustpanic = 1; 84626108Sfsmp } 84726108Sfsmp if (mp_napics > NAPIC) { 84826108Sfsmp printf("found %d apics, increase NAPIC\n", mp_napics); 84926108Sfsmp mustpanic = 1; 85026108Sfsmp } 85126108Sfsmp if (nintrs > NINTR) { 85226108Sfsmp printf("found %d intrs, increase NINTR\n", nintrs); 85326108Sfsmp mustpanic = 1; 85426108Sfsmp } 85526108Sfsmp 85626108Sfsmp /* 85726108Sfsmp * Count the BSP. 85826108Sfsmp * This is also used as a counter while starting the APs. 85926108Sfsmp */ 86026108Sfsmp mp_ncpus = 1; 86126108Sfsmp 86226108Sfsmp --mp_naps; /* subtract the BSP */ 86326108Sfsmp 86426108Sfsmp return mustpanic; 86526108Sfsmp} 86626108Sfsmp 86726108Sfsmp 86826108Sfsmp/* 86926155Sfsmp * 2nd pass on motherboard's Intel MP specification table. 87026155Sfsmp * 87126155Sfsmp * sets: 87226155Sfsmp * boot_cpu_id 87326155Sfsmp * ID_TO_IO(N), phy APIC ID to log CPU/IO table 87426155Sfsmp * CPU_TO_ID(N), logical CPU to APIC ID table 87526155Sfsmp * IO_TO_ID(N), logical IO to APIC ID table 87626155Sfsmp * bus_data[N] 87726155Sfsmp * io_apic_ints[N] 87826108Sfsmp */ 87926108Sfsmpstatic int 88026155Sfsmpmptable_pass2(void) 88126108Sfsmp{ 88226108Sfsmp int x; 88326108Sfsmp mpcth_t cth; 88426108Sfsmp int totalSize; 88526108Sfsmp void* position; 88626108Sfsmp int count; 88726108Sfsmp int type; 88826108Sfsmp int apic, bus, cpu, intr; 88926108Sfsmp 89027005Sfsmp POSTCODE(MPTABLE_PASS2_POST); 89127005Sfsmp 89226155Sfsmp /* clear various tables */ 89326155Sfsmp for (x = 0; x < NAPICID; ++x) { 89426155Sfsmp ID_TO_IO(x) = -1; /* phy APIC ID to log CPU/IO table */ 89526155Sfsmp CPU_TO_ID(x) = -1; /* logical CPU to APIC ID table */ 89626155Sfsmp IO_TO_ID(x) = -1; /* logical IO to APIC ID table */ 89726155Sfsmp } 89826155Sfsmp 89925164Speter /* clear bus data table */ 90025164Speter for (x = 0; x < NBUS; ++x) 90125164Speter bus_data[x].bus_id = 0xff; 90225164Speter 90325164Speter /* clear IO APIC INT table */ 90425164Speter for (x = 0; x < NINTR; ++x) 90525164Speter io_apic_ints[x].int_type = 0xff; 90625164Speter 90725164Speter /* setup the cpu/apic mapping arrays */ 90825164Speter boot_cpu_id = -1; 90925164Speter 91025164Speter /* record whether PIC or virtual-wire mode */ 91126155Sfsmp picmode = (mpfps->mpfb2 & 0x80) ? 1 : 0; 91225164Speter 91325164Speter /* check for use of 'default' configuration */ 91428027Sfsmp if (MPFPS_MPFB1 != 0) 91528027Sfsmp return MPFPS_MPFB1; /* return default configuration type */ 91625164Speter 91726155Sfsmp if ((cth = mpfps->pap) == 0) 91826108Sfsmp panic("MP Configuration Table Header MISSING!"); 91925164Speter 92026108Sfsmp /* walk the table, recording info of interest */ 92125164Speter totalSize = cth->base_table_length - sizeof(struct MPCTH); 92225164Speter position = (u_char *) cth + sizeof(struct MPCTH); 92325164Speter count = cth->entry_count; 92426108Sfsmp apic = bus = intr = 0; 92526108Sfsmp cpu = 1; /* pre-count the BSP */ 92625164Speter 92725164Speter while (count--) { 92825164Speter switch (type = *(u_char *) position) { 92925164Speter case 0: 93026108Sfsmp if (processor_entry(position, cpu)) 93126108Sfsmp ++cpu; 93225164Speter break; 93325164Speter case 1: 93426108Sfsmp if (bus_entry(position, bus)) 93526108Sfsmp ++bus; 93625164Speter break; 93725164Speter case 2: 93826108Sfsmp if (io_apic_entry(position, apic)) 93926108Sfsmp ++apic; 94025164Speter break; 94125164Speter case 3: 94226108Sfsmp if (int_entry(position, intr)) 94326108Sfsmp ++intr; 94425164Speter break; 94525164Speter case 4: 94625164Speter /* int_entry(position); */ 94725164Speter break; 94825164Speter default: 94926108Sfsmp panic("mpfps Base Table HOSED!"); 95025164Speter /* NOTREACHED */ 95125164Speter } 95225164Speter 95325164Speter totalSize -= basetable_entry_types[type].length; 95425164Speter (u_char *) position += basetable_entry_types[type].length; 95525164Speter } 95625164Speter 95726101Sfsmp if (boot_cpu_id == -1) 95826108Sfsmp panic("NO BSP found!"); 95925164Speter 96025164Speter /* report fact that its NOT a default configuration */ 96125164Speter return 0; 96225164Speter} 96325164Speter 96425164Speter 96525164Speter/* 96625164Speter * parse an Intel MP specification table 96725164Speter */ 96825164Speterstatic void 96925164Speterfix_mp_table(void) 97025164Speter{ 97125292Sfsmp int x; 97225292Sfsmp int id; 97325292Sfsmp int bus_0; 97425292Sfsmp int bus_pci; 97525292Sfsmp int num_pci_bus; 97625164Speter 97725164Speter /* 97825164Speter * Fix mis-numbering of the PCI bus and its INT entries if the BIOS 97925164Speter * did it wrong. The MP spec says that when more than 1 PCI bus 98025164Speter * exists the BIOS must begin with bus entries for the PCI bus and use 98125164Speter * actual PCI bus numbering. This implies that when only 1 PCI bus 98225164Speter * exists the BIOS can choose to ignore this ordering, and indeed many 98325164Speter * MP motherboards do ignore it. This causes a problem when the PCI 98425164Speter * sub-system makes requests of the MP sub-system based on PCI bus 98525164Speter * numbers. So here we look for the situation and renumber the 98625164Speter * busses and associated INTs in an effort to "make it right". 98725164Speter */ 98825164Speter 98925292Sfsmp /* find bus 0, PCI bus, count the number of PCI busses */ 99025164Speter for (num_pci_bus = 0, x = 0; x < mp_nbusses; ++x) { 99125292Sfsmp if (bus_data[x].bus_id == 0) { 99225292Sfsmp bus_0 = x; 99325292Sfsmp } 99425292Sfsmp if (bus_data[x].bus_type == PCI) { 99525164Speter ++num_pci_bus; 99625292Sfsmp bus_pci = x; 99725292Sfsmp } 99825164Speter } 99925292Sfsmp /* 100025292Sfsmp * bus_0 == slot of bus with ID of 0 100125292Sfsmp * bus_pci == slot of last PCI bus encountered 100225292Sfsmp */ 100325164Speter 100425164Speter /* check the 1 PCI bus case for sanity */ 100525164Speter if (num_pci_bus == 1) { 100625164Speter 100725292Sfsmp /* if it is number 0 all is well */ 100825292Sfsmp if (bus_data[bus_pci].bus_id == 0) 100925164Speter return; 101025164Speter 101125164Speter /* mis-numbered, swap with whichever bus uses slot 0 */ 101225164Speter 101325292Sfsmp /* swap the bus entry types */ 101425292Sfsmp bus_data[bus_pci].bus_type = bus_data[bus_0].bus_type; 101525292Sfsmp bus_data[bus_0].bus_type = PCI; 101625164Speter 101725164Speter /* swap each relavant INTerrupt entry */ 101825292Sfsmp id = bus_data[bus_pci].bus_id; 101925292Sfsmp for (x = 0; x < nintrs; ++x) { 102025292Sfsmp if (io_apic_ints[x].src_bus_id == id) { 102125292Sfsmp io_apic_ints[x].src_bus_id = 0; 102225292Sfsmp } 102325292Sfsmp else if (io_apic_ints[x].src_bus_id == 0) { 102425292Sfsmp io_apic_ints[x].src_bus_id = id; 102525292Sfsmp } 102625164Speter } 102725164Speter } 102825164Speter /* sanity check if more than 1 PCI bus */ 102925292Sfsmp else if (num_pci_bus > 1) { 103025292Sfsmp for (x = 0; x < mp_nbusses; ++x) { 103125292Sfsmp if (bus_data[x].bus_type != PCI) 103225292Sfsmp continue; 103326812Speter if (bus_data[x].bus_id >= num_pci_bus) 103426108Sfsmp panic("bad PCI bus numbering"); 103525164Speter } 103625292Sfsmp } 103725164Speter} 103825164Speter 103925164Speter 104026108Sfsmpstatic int 104126108Sfsmpprocessor_entry(proc_entry_ptr entry, int cpu) 104225164Speter{ 104325164Speter /* check for usability */ 104426108Sfsmp if ((cpu >= NCPU) || !(entry->cpu_flags & PROCENTRY_FLAG_EN)) 104526108Sfsmp return 0; 104625164Speter 104725164Speter /* check for BSP flag */ 104825164Speter if (entry->cpu_flags & PROCENTRY_FLAG_BP) { 104925164Speter boot_cpu_id = entry->apic_id; 105026108Sfsmp CPU_TO_ID(0) = entry->apic_id; 105126108Sfsmp ID_TO_CPU(entry->apic_id) = 0; 105226108Sfsmp return 0; /* its already been counted */ 105325164Speter } 105425164Speter 105526108Sfsmp /* add another AP to list, if less than max number of CPUs */ 105626108Sfsmp else { 105726108Sfsmp CPU_TO_ID(cpu) = entry->apic_id; 105826108Sfsmp ID_TO_CPU(entry->apic_id) = cpu; 105926108Sfsmp return 1; 106026108Sfsmp } 106125164Speter} 106225164Speter 106325164Speter 106426108Sfsmpstatic int 106526108Sfsmpbus_entry(bus_entry_ptr entry, int bus) 106625164Speter{ 106726108Sfsmp int x; 106826108Sfsmp char c, name[8]; 106925164Speter 107025164Speter /* encode the name into an index */ 107126108Sfsmp for (x = 0; x < 6; ++x) { 107226108Sfsmp if ((c = entry->bus_type[x]) == ' ') 107325164Speter break; 107426108Sfsmp name[x] = c; 107525164Speter } 107626108Sfsmp name[x] = '\0'; 107725164Speter 107826108Sfsmp if ((x = lookup_bus_type(name)) == UNKNOWN_BUSTYPE) 107926108Sfsmp panic("unknown bus type: '%s'", name); 108025164Speter 108126108Sfsmp bus_data[bus].bus_id = entry->bus_id; 108226108Sfsmp bus_data[bus].bus_type = x; 108326108Sfsmp 108426108Sfsmp return 1; 108525164Speter} 108625164Speter 108725164Speter 108826108Sfsmpstatic int 108926108Sfsmpio_apic_entry(io_apic_entry_ptr entry, int apic) 109025164Speter{ 109125164Speter if (!(entry->apic_flags & IOAPICENTRY_FLAG_EN)) 109226108Sfsmp return 0; 109325164Speter 109426108Sfsmp IO_TO_ID(apic) = entry->apic_id; 109526108Sfsmp ID_TO_IO(entry->apic_id) = apic; 109625164Speter 109726108Sfsmp return 1; 109825164Speter} 109925164Speter 110025164Speter 110125164Speterstatic int 110225164Speterlookup_bus_type(char *name) 110325164Speter{ 110425164Speter int x; 110525164Speter 110625164Speter for (x = 0; x < MAX_BUSTYPE; ++x) 110725164Speter if (strcmp(bus_type_table[x].name, name) == 0) 110825164Speter return bus_type_table[x].type; 110925164Speter 111025164Speter return UNKNOWN_BUSTYPE; 111125164Speter} 111225164Speter 111325164Speter 111426108Sfsmpstatic int 111526108Sfsmpint_entry(int_entry_ptr entry, int intr) 111625164Speter{ 111726108Sfsmp io_apic_ints[intr].int_type = entry->int_type; 111826108Sfsmp io_apic_ints[intr].int_flags = entry->int_flags; 111926108Sfsmp io_apic_ints[intr].src_bus_id = entry->src_bus_id; 112026108Sfsmp io_apic_ints[intr].src_bus_irq = entry->src_bus_irq; 112126108Sfsmp io_apic_ints[intr].dst_apic_id = entry->dst_apic_id; 112226108Sfsmp io_apic_ints[intr].dst_apic_int = entry->dst_apic_int; 112325164Speter 112426108Sfsmp return 1; 112525164Speter} 112625164Speter 112725164Speter 112825164Speterstatic int 112925164Speterapic_int_is_bus_type(int intr, int bus_type) 113025164Speter{ 113125164Speter int bus; 113225164Speter 113325164Speter for (bus = 0; bus < mp_nbusses; ++bus) 113425164Speter if ((bus_data[bus].bus_id == io_apic_ints[intr].src_bus_id) 113525164Speter && ((int) bus_data[bus].bus_type == bus_type)) 113625164Speter return 1; 113725164Speter 113825164Speter return 0; 113925164Speter} 114025164Speter 114125164Speter 114225164Speter/* 114326950Sfsmp * Given a traditional ISA INT mask, return an APIC mask. 114425164Speter */ 114525499Sfsmpu_int 114626950Sfsmpisa_apic_mask(u_int isa_mask) 114725419Sfsmp{ 114826950Sfsmp int isa_irq; 114926950Sfsmp int apic_pin; 115025419Sfsmp 115127255Sfsmp#if defined(SKIP_IRQ15_REDIRECT) 115227255Sfsmp if (isa_mask == (1 << 15)) { 115327255Sfsmp printf("skipping ISA IRQ15 redirect\n"); 115427255Sfsmp return isa_mask; 115527255Sfsmp } 115627255Sfsmp#endif /* SKIP_IRQ15_REDIRECT */ 115727255Sfsmp 115826950Sfsmp isa_irq = ffs(isa_mask); /* find its bit position */ 115926950Sfsmp if (isa_irq == 0) /* doesn't exist */ 116025419Sfsmp return 0; 116126950Sfsmp --isa_irq; /* make it zero based */ 116225419Sfsmp 116326950Sfsmp apic_pin = isa_apic_pin(isa_irq); /* look for APIC connection */ 116426950Sfsmp if (apic_pin == -1) 116526950Sfsmp return 0; 116625419Sfsmp 116726950Sfsmp return (1 << apic_pin); /* convert pin# to a mask */ 116825419Sfsmp} 116925419Sfsmp 117025419Sfsmp 117125419Sfsmp/* 117226950Sfsmp * Determine which APIC pin an ISA/EISA INT is attached to. 117325164Speter */ 117426950Sfsmp#define INTTYPE(I) (io_apic_ints[(I)].int_type) 117526950Sfsmp#define INTPIN(I) (io_apic_ints[(I)].dst_apic_int) 117626950Sfsmp 117725164Speter#define SRCBUSIRQ(I) (io_apic_ints[(I)].src_bus_irq) 117825164Speterint 117926950Sfsmpisa_apic_pin(int isa_irq) 118025164Speter{ 118125164Speter int intr; 118225164Speter 118326950Sfsmp for (intr = 0; intr < nintrs; ++intr) { /* check each record */ 118426950Sfsmp if (INTTYPE(intr) == 0) { /* standard INT */ 118526950Sfsmp if (SRCBUSIRQ(intr) == isa_irq) { 118626950Sfsmp if (apic_int_is_bus_type(intr, ISA) || 118726950Sfsmp apic_int_is_bus_type(intr, EISA)) 118826950Sfsmp return INTPIN(intr); /* found */ 118926950Sfsmp } 119026950Sfsmp } 119126950Sfsmp } 119226950Sfsmp return -1; /* NOT found */ 119325164Speter} 119425164Speter 119525164Speter 119625164Speter/* 119726950Sfsmp * Determine which APIC pin a PCI INT is attached to. 119825164Speter */ 119925164Speter#define SRCBUSID(I) (io_apic_ints[(I)].src_bus_id) 120025164Speter#define SRCBUSDEVICE(I) ((io_apic_ints[(I)].src_bus_irq >> 2) & 0x1f) 120125164Speter#define SRCBUSLINE(I) (io_apic_ints[(I)].src_bus_irq & 0x03) 120225164Speterint 120326950Sfsmppci_apic_pin(int pciBus, int pciDevice, int pciInt) 120425164Speter{ 120525164Speter int intr; 120625164Speter 120726950Sfsmp --pciInt; /* zero based */ 120825164Speter 120926950Sfsmp for (intr = 0; intr < nintrs; ++intr) /* check each record */ 121026950Sfsmp if ((INTTYPE(intr) == 0) /* standard INT */ 121125164Speter && (SRCBUSID(intr) == pciBus) 121225164Speter && (SRCBUSDEVICE(intr) == pciDevice) 121325164Speter && (SRCBUSLINE(intr) == pciInt)) /* a candidate IRQ */ 121426950Sfsmp if (apic_int_is_bus_type(intr, PCI)) 121525164Speter return INTPIN(intr); /* exact match */ 121625164Speter 121726950Sfsmp return -1; /* NOT found */ 121825164Speter} 121934990Stegge 122034990Steggeint 122134990Steggenext_apic_pin(int pin) 122234990Stegge{ 122334990Stegge int intr, ointr; 122434990Stegge int bus, bustype; 122534990Stegge 122634990Stegge bus = 0; 122734990Stegge bustype = 0; 122834990Stegge for (intr = 0; intr < nintrs; intr++) { 122934990Stegge if (INTPIN(intr) != pin || INTTYPE(intr) != 0) 123034990Stegge continue; 123134990Stegge bus = SRCBUSID(intr); 123234990Stegge bustype = apic_bus_type(bus); 123334990Stegge if (bustype != ISA && 123434990Stegge bustype != EISA && 123534990Stegge bustype != PCI) 123634990Stegge continue; 123734990Stegge break; 123834990Stegge } 123934990Stegge if (intr >= nintrs) { 124034990Stegge return -1; 124134990Stegge } 124234990Stegge for (ointr = intr + 1; ointr < nintrs; ointr++) { 124334990Stegge if (INTTYPE(ointr) != 0) 124434990Stegge continue; 124534990Stegge if (bus != SRCBUSID(ointr)) 124634990Stegge continue; 124734990Stegge if (bustype == PCI) { 124834990Stegge if (SRCBUSDEVICE(intr) != SRCBUSDEVICE(ointr)) 124934990Stegge continue; 125034990Stegge if (SRCBUSLINE(intr) != SRCBUSLINE(ointr)) 125134990Stegge continue; 125234990Stegge } 125334990Stegge if (bustype == ISA || bustype == EISA) { 125434990Stegge if (SRCBUSIRQ(intr) != SRCBUSIRQ(ointr)) 125534990Stegge continue; 125634990Stegge } 125734990Stegge if (INTPIN(intr) == INTPIN(ointr)) 125834990Stegge continue; 125934990Stegge break; 126034990Stegge } 126134990Stegge if (ointr >= nintrs) { 126234990Stegge return -1; 126334990Stegge } 126434990Stegge return INTPIN(ointr); 126534990Stegge} 126625164Speter#undef SRCBUSLINE 126725164Speter#undef SRCBUSDEVICE 126825164Speter#undef SRCBUSID 126934990Stegge#undef SRCBUSIRQ 127025164Speter 127125164Speter#undef INTPIN 127225164Speter#undef INTTYPE 127325164Speter 127425164Speter 127525499Sfsmp/* 127626950Sfsmp * Reprogram the MB chipset to NOT redirect an ISA INTerrupt. 127726950Sfsmp * 127826950Sfsmp * XXX FIXME: 127926950Sfsmp * Exactly what this means is unclear at this point. It is a solution 128026950Sfsmp * for motherboards that redirect the MBIRQ0 pin. Generically a motherboard 128126950Sfsmp * could route any of the ISA INTs to upper (>15) IRQ values. But most would 128226950Sfsmp * NOT be redirected via MBIRQ0, thus "undirect()ing" them would NOT be an 128326950Sfsmp * option. 128425499Sfsmp */ 128525164Speterint 128626950Sfsmpundirect_isa_irq(int rirq) 128725164Speter{ 128825164Speter#if defined(READY) 128926950Sfsmp printf("Freeing redirected ISA irq %d.\n", rirq); 129025164Speter /** FIXME: tickle the MB redirector chip */ 129125164Speter return ???; 129225164Speter#else 129326950Sfsmp printf("Freeing (NOT implemented) redirected ISA irq %d.\n", rirq); 129425164Speter return 0; 129525499Sfsmp#endif /* READY */ 129625164Speter} 129725164Speter 129825164Speter 129925164Speter/* 130026950Sfsmp * Reprogram the MB chipset to NOT redirect a PCI INTerrupt 130125499Sfsmp */ 130225499Sfsmpint 130326950Sfsmpundirect_pci_irq(int rirq) 130425499Sfsmp{ 130525499Sfsmp#if defined(READY) 130626950Sfsmp if (bootverbose) 130726950Sfsmp printf("Freeing redirected PCI irq %d.\n", rirq); 130826950Sfsmp 130925499Sfsmp /** FIXME: tickle the MB redirector chip */ 131025499Sfsmp return ???; 131125499Sfsmp#else 131226950Sfsmp if (bootverbose) 131326950Sfsmp printf("Freeing (NOT implemented) redirected PCI irq %d.\n", 131426950Sfsmp rirq); 131525499Sfsmp return 0; 131625499Sfsmp#endif /* READY */ 131725499Sfsmp} 131825499Sfsmp 131925499Sfsmp 132025499Sfsmp/* 132125164Speter * given a bus ID, return: 132225164Speter * the bus type if found 132325164Speter * -1 if NOT found 132425164Speter */ 132525164Speterint 132625164Speterapic_bus_type(int id) 132725164Speter{ 132825164Speter int x; 132925164Speter 133025164Speter for (x = 0; x < mp_nbusses; ++x) 133125164Speter if (bus_data[x].bus_id == id) 133225164Speter return bus_data[x].bus_type; 133325164Speter 133425164Speter return -1; 133525164Speter} 133625164Speter 133725164Speter 133825164Speter/* 133925164Speter * given a LOGICAL APIC# and pin#, return: 134025164Speter * the associated src bus ID if found 134125164Speter * -1 if NOT found 134225164Speter */ 134325164Speterint 134425164Speterapic_src_bus_id(int apic, int pin) 134525164Speter{ 134625164Speter int x; 134725164Speter 134825164Speter /* search each of the possible INTerrupt sources */ 134925164Speter for (x = 0; x < nintrs; ++x) 135025164Speter if ((apic == ID_TO_IO(io_apic_ints[x].dst_apic_id)) && 135125164Speter (pin == io_apic_ints[x].dst_apic_int)) 135225164Speter return (io_apic_ints[x].src_bus_id); 135325164Speter 135425164Speter return -1; /* NOT found */ 135525164Speter} 135625164Speter 135725164Speter 135825164Speter/* 135925164Speter * given a LOGICAL APIC# and pin#, return: 136025164Speter * the associated src bus IRQ if found 136125164Speter * -1 if NOT found 136225164Speter */ 136325164Speterint 136425164Speterapic_src_bus_irq(int apic, int pin) 136525164Speter{ 136625164Speter int x; 136725164Speter 136825164Speter for (x = 0; x < nintrs; x++) 136925164Speter if ((apic == ID_TO_IO(io_apic_ints[x].dst_apic_id)) && 137025164Speter (pin == io_apic_ints[x].dst_apic_int)) 137125164Speter return (io_apic_ints[x].src_bus_irq); 137225164Speter 137325164Speter return -1; /* NOT found */ 137425164Speter} 137525164Speter 137625164Speter 137725164Speter/* 137825164Speter * given a LOGICAL APIC# and pin#, return: 137925164Speter * the associated INTerrupt type if found 138025164Speter * -1 if NOT found 138125164Speter */ 138225164Speterint 138325164Speterapic_int_type(int apic, int pin) 138425164Speter{ 138525164Speter int x; 138625164Speter 138725164Speter /* search each of the possible INTerrupt sources */ 138825164Speter for (x = 0; x < nintrs; ++x) 138925164Speter if ((apic == ID_TO_IO(io_apic_ints[x].dst_apic_id)) && 139025164Speter (pin == io_apic_ints[x].dst_apic_int)) 139125164Speter return (io_apic_ints[x].int_type); 139225164Speter 139325164Speter return -1; /* NOT found */ 139425164Speter} 139525164Speter 139625164Speter 139725164Speter/* 139825164Speter * given a LOGICAL APIC# and pin#, return: 139925164Speter * the associated trigger mode if found 140025164Speter * -1 if NOT found 140125164Speter */ 140225164Speterint 140325164Speterapic_trigger(int apic, int pin) 140425164Speter{ 140525164Speter int x; 140625164Speter 140725164Speter /* search each of the possible INTerrupt sources */ 140825164Speter for (x = 0; x < nintrs; ++x) 140925164Speter if ((apic == ID_TO_IO(io_apic_ints[x].dst_apic_id)) && 141025164Speter (pin == io_apic_ints[x].dst_apic_int)) 141125164Speter return ((io_apic_ints[x].int_flags >> 2) & 0x03); 141225164Speter 141325164Speter return -1; /* NOT found */ 141425164Speter} 141525164Speter 141625164Speter 141725164Speter/* 141825164Speter * given a LOGICAL APIC# and pin#, return: 141925164Speter * the associated 'active' level if found 142025164Speter * -1 if NOT found 142125164Speter */ 142225164Speterint 142325164Speterapic_polarity(int apic, int pin) 142425164Speter{ 142525164Speter int x; 142625164Speter 142725164Speter /* search each of the possible INTerrupt sources */ 142825164Speter for (x = 0; x < nintrs; ++x) 142925164Speter if ((apic == ID_TO_IO(io_apic_ints[x].dst_apic_id)) && 143025164Speter (pin == io_apic_ints[x].dst_apic_int)) 143125164Speter return (io_apic_ints[x].int_flags & 0x03); 143225164Speter 143325164Speter return -1; /* NOT found */ 143425164Speter} 143525164Speter 143625164Speter 143725164Speter/* 143825164Speter * set data according to MP defaults 143925164Speter * FIXME: probably not complete yet... 144025164Speter */ 144125164Speterstatic void 144225164Speterdefault_mp_table(int type) 144325164Speter{ 144425164Speter int ap_cpu_id; 144525164Speter#if defined(APIC_IO) 144625164Speter u_int32_t ux; 144725164Speter int io_apic_id; 144825164Speter int pin; 144925164Speter#endif /* APIC_IO */ 145025164Speter 145125164Speter#if 0 145225164Speter printf(" MP default config type: %d\n", type); 145325164Speter switch (type) { 145425164Speter case 1: 145525164Speter printf(" bus: ISA, APIC: 82489DX\n"); 145625164Speter break; 145725164Speter case 2: 145825164Speter printf(" bus: EISA, APIC: 82489DX\n"); 145925164Speter break; 146025164Speter case 3: 146125164Speter printf(" bus: EISA, APIC: 82489DX\n"); 146225164Speter break; 146325164Speter case 4: 146425164Speter printf(" bus: MCA, APIC: 82489DX\n"); 146525164Speter break; 146625164Speter case 5: 146725164Speter printf(" bus: ISA+PCI, APIC: Integrated\n"); 146825164Speter break; 146925164Speter case 6: 147025164Speter printf(" bus: EISA+PCI, APIC: Integrated\n"); 147125164Speter break; 147225164Speter case 7: 147325164Speter printf(" bus: MCA+PCI, APIC: Integrated\n"); 147425164Speter break; 147525164Speter default: 147625164Speter printf(" future type\n"); 147725164Speter break; 147825164Speter /* NOTREACHED */ 147925164Speter } 148025164Speter#endif /* 0 */ 148125164Speter 148226812Speter boot_cpu_id = (lapic.id & APIC_ID_MASK) >> 24; 148325164Speter ap_cpu_id = (boot_cpu_id == 0) ? 1 : 0; 148425164Speter 148525164Speter /* BSP */ 148625164Speter CPU_TO_ID(0) = boot_cpu_id; 148725164Speter ID_TO_CPU(boot_cpu_id) = 0; 148825164Speter 148925164Speter /* one and only AP */ 149025164Speter CPU_TO_ID(1) = ap_cpu_id; 149125164Speter ID_TO_CPU(ap_cpu_id) = 1; 149225164Speter 149326108Sfsmp#if defined(APIC_IO) 149425164Speter /* one and only IO APIC */ 149525164Speter io_apic_id = (io_apic_read(0, IOAPIC_ID) & APIC_ID_MASK) >> 24; 149625164Speter 149725164Speter /* 149825164Speter * sanity check, refer to MP spec section 3.6.6, last paragraph 149925164Speter * necessary as some hardware isn't properly setting up the IO APIC 150025164Speter */ 150125164Speter#if defined(REALLY_ANAL_IOAPICID_VALUE) 150225164Speter if (io_apic_id != 2) { 150325164Speter#else 150425164Speter if ((io_apic_id == 0) || (io_apic_id == 1) || (io_apic_id == 15)) { 150525164Speter#endif /* REALLY_ANAL_IOAPICID_VALUE */ 150625164Speter ux = io_apic_read(0, IOAPIC_ID); /* get current contents */ 150725164Speter ux &= ~APIC_ID_MASK; /* clear the ID field */ 150825164Speter ux |= 0x02000000; /* set it to '2' */ 150925164Speter io_apic_write(0, IOAPIC_ID, ux); /* write new value */ 151025164Speter ux = io_apic_read(0, IOAPIC_ID); /* re-read && test */ 151126101Sfsmp if ((ux & APIC_ID_MASK) != 0x02000000) 151226108Sfsmp panic("can't control IO APIC ID, reg: 0x%08x", ux); 151325164Speter io_apic_id = 2; 151425164Speter } 151525164Speter IO_TO_ID(0) = io_apic_id; 151625164Speter ID_TO_IO(io_apic_id) = 0; 151725164Speter#endif /* APIC_IO */ 151825164Speter 151925164Speter /* fill out bus entries */ 152025164Speter switch (type) { 152125164Speter case 1: 152225164Speter case 2: 152325164Speter case 3: 152425164Speter case 5: 152525164Speter case 6: 152625164Speter bus_data[0].bus_id = default_data[type - 1][1]; 152725164Speter bus_data[0].bus_type = default_data[type - 1][2]; 152825164Speter bus_data[1].bus_id = default_data[type - 1][3]; 152925164Speter bus_data[1].bus_type = default_data[type - 1][4]; 153025164Speter break; 153125164Speter 153225164Speter /* case 4: case 7: MCA NOT supported */ 153325164Speter default: /* illegal/reserved */ 153426108Sfsmp panic("BAD default MP config: %d", type); 153526101Sfsmp /* NOTREACHED */ 153625164Speter } 153725164Speter 153825164Speter#if defined(APIC_IO) 153925164Speter /* general cases from MP v1.4, table 5-2 */ 154025164Speter for (pin = 0; pin < 16; ++pin) { 154125164Speter io_apic_ints[pin].int_type = 0; 154227728Sfsmp io_apic_ints[pin].int_flags = 0x05; /* edge/active-hi */ 154325164Speter io_apic_ints[pin].src_bus_id = 0; 154427728Sfsmp io_apic_ints[pin].src_bus_irq = pin; /* IRQ2 caught below */ 154525164Speter io_apic_ints[pin].dst_apic_id = io_apic_id; 154627728Sfsmp io_apic_ints[pin].dst_apic_int = pin; /* 1-to-1 */ 154725164Speter } 154825164Speter 154925164Speter /* special cases from MP v1.4, table 5-2 */ 155025164Speter if (type == 2) { 155125164Speter io_apic_ints[2].int_type = 0xff; /* N/C */ 155225164Speter io_apic_ints[13].int_type = 0xff; /* N/C */ 155325164Speter#if !defined(APIC_MIXED_MODE) 155425164Speter /** FIXME: ??? */ 155526108Sfsmp panic("sorry, can't support type 2 default yet"); 155625164Speter#endif /* APIC_MIXED_MODE */ 155726019Sfsmp } 155826019Sfsmp else 155925164Speter io_apic_ints[2].src_bus_irq = 0; /* ISA IRQ0 is on APIC INT 2 */ 156025164Speter 156125164Speter if (type == 7) 156225164Speter io_apic_ints[0].int_type = 0xff; /* N/C */ 156325164Speter else 156425164Speter io_apic_ints[0].int_type = 3; /* vectored 8259 */ 156525164Speter#endif /* APIC_IO */ 156625164Speter} 156725164Speter 156825164Speter 156925164Speter/* 157027634Sfsmp * initialize all the SMP locks 157127634Sfsmp */ 157228442Sfsmp 157328487Sfsmp/* critical region around IO APIC, apic_imen */ 157428487Sfsmpstruct simplelock imen_lock; 157528487Sfsmp 157629213Sfsmp/* critical region around splxx(), cpl, cml, cil, ipending */ 157728487Sfsmpstruct simplelock cpl_lock; 157828487Sfsmp 157928487Sfsmp/* Make FAST_INTR() routines sequential */ 158028487Sfsmpstruct simplelock fast_intr_lock; 158128487Sfsmp 158228487Sfsmp/* critical region around INTR() routines */ 158328487Sfsmpstruct simplelock intr_lock; 158428487Sfsmp 158528951Sfsmp/* lock regions protected in UP kernel via cli/sti */ 158628921Sfsmpstruct simplelock mpintr_lock; 158728921Sfsmp 158831723Stegge/* lock region used by kernel profiling */ 158931723Steggestruct simplelock mcount_lock; 159031723Stegge 159128951Sfsmp#ifdef USE_COMLOCK 159228951Sfsmp/* locks com (tty) data/hardware accesses: a FASTINTR() */ 159328442Sfsmpstruct simplelock com_lock; 159428951Sfsmp#endif /* USE_COMLOCK */ 159528442Sfsmp 159628999Sfsmp#ifdef USE_CLOCKLOCK 159728999Sfsmp/* lock regions around the clock hardware */ 159828999Sfsmpstruct simplelock clock_lock; 159928999Sfsmp#endif /* USE_CLOCKLOCK */ 160028999Sfsmp 160127634Sfsmpstatic void 160227634Sfsmpinit_locks(void) 160327634Sfsmp{ 160427634Sfsmp /* 160527634Sfsmp * Get the initial mp_lock with a count of 1 for the BSP. 160627634Sfsmp * This uses a LOGICAL cpu ID, ie BSP == 0. 160727634Sfsmp */ 160827634Sfsmp mp_lock = 0x00000001; 160927634Sfsmp 161028442Sfsmp /* ISR uses its own "giant lock" */ 161128921Sfsmp isr_lock = FREE_LOCK; 161228442Sfsmp 161334021Stegge#if defined(APIC_INTR_DIAGNOSTIC) && defined(APIC_INTR_DIAGNOSTIC_IRQ) 161434021Stegge s_lock_init((struct simplelock*)&apic_itrace_debuglock); 161534021Stegge#endif 161634021Stegge 161728921Sfsmp s_lock_init((struct simplelock*)&mpintr_lock); 161828999Sfsmp 161931723Stegge s_lock_init((struct simplelock*)&mcount_lock); 162031723Stegge 162128442Sfsmp s_lock_init((struct simplelock*)&fast_intr_lock); 162228442Sfsmp s_lock_init((struct simplelock*)&intr_lock); 162327780Sfsmp s_lock_init((struct simplelock*)&imen_lock); 162428442Sfsmp s_lock_init((struct simplelock*)&cpl_lock); 162528999Sfsmp 162628951Sfsmp#ifdef USE_COMLOCK 162728442Sfsmp s_lock_init((struct simplelock*)&com_lock); 162828951Sfsmp#endif /* USE_COMLOCK */ 162928999Sfsmp#ifdef USE_CLOCKLOCK 163028999Sfsmp s_lock_init((struct simplelock*)&clock_lock); 163128999Sfsmp#endif /* USE_CLOCKLOCK */ 163227634Sfsmp} 163327634Sfsmp 163427634Sfsmp 163527634Sfsmp/* 163625164Speter * start each AP in our list 163725164Speter */ 163825164Speterstatic int 163925164Speterstart_all_aps(u_int boot_addr) 164025164Speter{ 164126812Speter int x, i; 164225164Speter u_char mpbiosreason; 164325164Speter u_long mpbioswarmvec; 164428808Speter pd_entry_t *newptd; 164528808Speter pt_entry_t *newpt; 164635077Speter struct globaldata *gd; 164729655Sdyson char *stack; 164829655Sdyson pd_entry_t *myPTD; 164925164Speter 165027005Sfsmp POSTCODE(START_ALL_APS_POST); 165127005Sfsmp 165225164Speter /* initialize BSP's local APIC */ 165327289Sfsmp apic_initialize(); 165428669Sfsmp bsp_apic_ready = 1; 165525164Speter 165625164Speter /* install the AP 1st level boot code */ 165725164Speter install_ap_tramp(boot_addr); 165825164Speter 165926812Speter 166025164Speter /* save the current value of the warm-start vector */ 166125164Speter mpbioswarmvec = *((u_long *) WARMBOOT_OFF); 166225164Speter outb(CMOS_REG, BIOS_RESET); 166325164Speter mpbiosreason = inb(CMOS_DATA); 166425164Speter 166527005Sfsmp /* record BSP in CPU map */ 166627005Sfsmp all_cpus = 1; 166727005Sfsmp 166825164Speter /* start each AP */ 166925164Speter for (x = 1; x <= mp_naps; ++x) { 167025164Speter 167128808Speter /* This is a bit verbose, it will go away soon. */ 167226812Speter 167326812Speter /* alloc new page table directory */ 167428808Speter newptd = (pd_entry_t *)(kmem_alloc(kernel_map, PAGE_SIZE)); 167526812Speter 167628808Speter /* Store the virtual PTD address for this CPU */ 167728808Speter IdlePTDS[x] = newptd; 167828808Speter 167926812Speter /* clone currently active one (ie: IdlePTD) */ 168026812Speter bcopy(PTD, newptd, PAGE_SIZE); /* inc prv page pde */ 168126812Speter 168226812Speter /* set up 0 -> 4MB P==V mapping for AP boot */ 168328808Speter newptd[0] = (pd_entry_t) (PG_V | PG_RW | 168428808Speter ((u_long)KPTphys & PG_FRAME)); 168526812Speter 168628808Speter /* store PTD for this AP's boot sequence */ 168729655Sdyson myPTD = (pd_entry_t *)vtophys(newptd); 168826812Speter 168926812Speter /* alloc new page table page */ 169028808Speter newpt = (pt_entry_t *)(kmem_alloc(kernel_map, PAGE_SIZE)); 169126812Speter 169226812Speter /* set the new PTD's private page to point there */ 169328808Speter newptd[MPPTDI] = (pt_entry_t)(PG_V | PG_RW | vtophys(newpt)); 169426812Speter 169526812Speter /* install self referential entry */ 169628808Speter newptd[PTDPTDI] = (pd_entry_t)(PG_V | PG_RW | vtophys(newptd)); 169726812Speter 169828808Speter /* allocate a new private data page */ 169935077Speter gd = (struct globaldata *)kmem_alloc(kernel_map, PAGE_SIZE); 170026812Speter 170126812Speter /* wire it into the private page table page */ 170235077Speter newpt[0] = (pt_entry_t)(PG_V | PG_RW | vtophys(gd)); 170326812Speter 170426812Speter /* wire the ptp into itself for access */ 170528808Speter newpt[1] = (pt_entry_t)(PG_V | PG_RW | vtophys(newpt)); 170626812Speter 170728808Speter /* copy in the pointer to the local apic */ 170826812Speter newpt[2] = SMP_prvpt[2]; 170926812Speter 171026812Speter /* and the IO apic mapping[s] */ 171126812Speter for (i = 16; i < 32; i++) 171226812Speter newpt[i] = SMP_prvpt[i]; 171326812Speter 171428808Speter /* allocate and set up an idle stack data page */ 171529655Sdyson stack = (char *)kmem_alloc(kernel_map, UPAGES*PAGE_SIZE); 171629655Sdyson for (i = 0; i < UPAGES; i++) 171729655Sdyson newpt[i + 3] = (pt_entry_t)(PG_V | PG_RW | vtophys(PAGE_SIZE * i + stack)); 171826812Speter 171931030Stegge newpt[3 + UPAGES] = 0; /* *prv_CMAP1 */ 172031030Stegge newpt[4 + UPAGES] = 0; /* *prv_CMAP2 */ 172131030Stegge newpt[5 + UPAGES] = 0; /* *prv_CMAP3 */ 172236125Stegge newpt[6 + UPAGES] = 0; /* *prv_PMAP1 */ 172326812Speter 172428808Speter /* prime data page for it to use */ 172535077Speter gd->cpuid = x; 172635077Speter gd->cpu_lockid = x << 24; 172735077Speter gd->my_idlePTD = myPTD; 172835077Speter gd->prv_CMAP1 = &newpt[3 + UPAGES]; 172935077Speter gd->prv_CMAP2 = &newpt[4 + UPAGES]; 173035077Speter gd->prv_CMAP3 = &newpt[5 + UPAGES]; 173136125Stegge gd->prv_PMAP1 = &newpt[6 + UPAGES]; 173226812Speter 173325164Speter /* setup a vector to our boot code */ 173425164Speter *((volatile u_short *) WARMBOOT_OFF) = WARMBOOT_TARGET; 173525164Speter *((volatile u_short *) WARMBOOT_SEG) = (boot_addr >> 4); 173625164Speter outb(CMOS_REG, BIOS_RESET); 173725164Speter outb(CMOS_DATA, BIOS_WARM); /* 'warm-start' */ 173825164Speter 173929655Sdyson bootPTD = myPTD; 174025164Speter /* attempt to start the Application Processor */ 174125164Speter CHECK_INIT(99); /* setup checkpoints */ 174225164Speter if (!start_ap(x, boot_addr)) { 174325164Speter printf("AP #%d (PHY# %d) failed!\n", x, CPU_TO_ID(x)); 174425164Speter CHECK_PRINT("trace"); /* show checkpoints */ 174526155Sfsmp /* better panic as the AP may be running loose */ 174626155Sfsmp printf("panic y/n? [y] "); 174726101Sfsmp if (cngetc() != 'n') 174826108Sfsmp panic("bye-bye"); 174925164Speter } 175027005Sfsmp CHECK_PRINT("trace"); /* show checkpoints */ 175125164Speter 175225164Speter /* record its version info */ 175325164Speter cpu_apic_versions[x] = cpu_apic_versions[0]; 175427005Sfsmp 175527005Sfsmp all_cpus |= (1 << x); /* record AP in CPU map */ 175625164Speter } 175725164Speter 175827005Sfsmp /* build our map of 'other' CPUs */ 175927005Sfsmp other_cpus = all_cpus & ~(1 << cpuid); 176027005Sfsmp 176125164Speter /* fill in our (BSP) APIC version */ 176226812Speter cpu_apic_versions[0] = lapic.version; 176325164Speter 176425164Speter /* restore the warmstart vector */ 176525164Speter *(u_long *) WARMBOOT_OFF = mpbioswarmvec; 176625164Speter outb(CMOS_REG, BIOS_RESET); 176725164Speter outb(CMOS_DATA, mpbiosreason); 176825164Speter 176928808Speter /* 177028808Speter * Set up the idle context for the BSP. Similar to above except 177128808Speter * that some was done by locore, some by pmap.c and some is implicit 177228808Speter * because the BSP is cpu#0 and the page is initially zero, and also 177328808Speter * because we can refer to variables by name on the BSP.. 177428808Speter */ 177528808Speter newptd = (pd_entry_t *)(kmem_alloc(kernel_map, PAGE_SIZE)); 177628808Speter 177728808Speter bcopy(PTD, newptd, PAGE_SIZE); /* inc prv page pde */ 177828808Speter IdlePTDS[0] = newptd; 177928808Speter 178028808Speter /* Point PTD[] to this page instead of IdlePTD's physical page */ 178128808Speter newptd[PTDPTDI] = (pd_entry_t)(PG_V | PG_RW | vtophys(newptd)); 178228808Speter 178328808Speter my_idlePTD = (pd_entry_t *)vtophys(newptd); 178428808Speter 178528808Speter /* Allocate and setup BSP idle stack */ 178629655Sdyson stack = (char *)kmem_alloc(kernel_map, UPAGES * PAGE_SIZE); 178729655Sdyson for (i = 0; i < UPAGES; i++) 178829655Sdyson SMP_prvpt[i + 3] = (pt_entry_t)(PG_V | PG_RW | vtophys(PAGE_SIZE * i + stack)); 178928808Speter 179027484Sdyson pmap_set_opt_bsp(); 179127484Sdyson 179229655Sdyson for (i = 0; i < mp_ncpus; i++) { 179329655Sdyson bcopy( (int *) PTD + KPTDI, (int *) IdlePTDS[i] + KPTDI, NKPDE * sizeof (int)); 179429655Sdyson } 179529655Sdyson 179625164Speter /* number of APs actually started */ 179725164Speter return mp_ncpus - 1; 179825164Speter} 179925164Speter 180025164Speter 180125164Speter/* 180225164Speter * load the 1st level AP boot code into base memory. 180325164Speter */ 180425164Speter 180525164Speter/* targets for relocation */ 180625164Speterextern void bigJump(void); 180725164Speterextern void bootCodeSeg(void); 180825164Speterextern void bootDataSeg(void); 180925164Speterextern void MPentry(void); 181025164Speterextern u_int MP_GDT; 181125164Speterextern u_int mp_gdtbase; 181225164Speter 181325164Speterstatic void 181425164Speterinstall_ap_tramp(u_int boot_addr) 181525164Speter{ 181625164Speter int x; 181725164Speter int size = *(int *) ((u_long) & bootMP_size); 181825164Speter u_char *src = (u_char *) ((u_long) bootMP); 181925164Speter u_char *dst = (u_char *) boot_addr + KERNBASE; 182025164Speter u_int boot_base = (u_int) bootMP; 182125164Speter u_int8_t *dst8; 182225164Speter u_int16_t *dst16; 182325164Speter u_int32_t *dst32; 182425164Speter 182527005Sfsmp POSTCODE(INSTALL_AP_TRAMP_POST); 182627005Sfsmp 182725164Speter for (x = 0; x < size; ++x) 182825164Speter *dst++ = *src++; 182925164Speter 183025164Speter /* 183125164Speter * modify addresses in code we just moved to basemem. unfortunately we 183225164Speter * need fairly detailed info about mpboot.s for this to work. changes 183325164Speter * to mpboot.s might require changes here. 183425164Speter */ 183525164Speter 183625164Speter /* boot code is located in KERNEL space */ 183725164Speter dst = (u_char *) boot_addr + KERNBASE; 183825164Speter 183925164Speter /* modify the lgdt arg */ 184025164Speter dst32 = (u_int32_t *) (dst + ((u_int) & mp_gdtbase - boot_base)); 184125164Speter *dst32 = boot_addr + ((u_int) & MP_GDT - boot_base); 184225164Speter 184325164Speter /* modify the ljmp target for MPentry() */ 184425164Speter dst32 = (u_int32_t *) (dst + ((u_int) bigJump - boot_base) + 1); 184525164Speter *dst32 = ((u_int) MPentry - KERNBASE); 184625164Speter 184725164Speter /* modify the target for boot code segment */ 184825164Speter dst16 = (u_int16_t *) (dst + ((u_int) bootCodeSeg - boot_base)); 184925164Speter dst8 = (u_int8_t *) (dst16 + 1); 185025164Speter *dst16 = (u_int) boot_addr & 0xffff; 185125164Speter *dst8 = ((u_int) boot_addr >> 16) & 0xff; 185225164Speter 185325164Speter /* modify the target for boot data segment */ 185425164Speter dst16 = (u_int16_t *) (dst + ((u_int) bootDataSeg - boot_base)); 185525164Speter dst8 = (u_int8_t *) (dst16 + 1); 185625164Speter *dst16 = (u_int) boot_addr & 0xffff; 185725164Speter *dst8 = ((u_int) boot_addr >> 16) & 0xff; 185825164Speter} 185925164Speter 186025164Speter 186125164Speter/* 186225164Speter * this function starts the AP (application processor) identified 186325164Speter * by the APIC ID 'physicalCpu'. It does quite a "song and dance" 186425164Speter * to accomplish this. This is necessary because of the nuances 186525164Speter * of the different hardware we might encounter. It ain't pretty, 186625164Speter * but it seems to work. 186725164Speter */ 186825164Speterstatic int 186925164Speterstart_ap(int logical_cpu, u_int boot_addr) 187025164Speter{ 187125164Speter int physical_cpu; 187225164Speter int vector; 187325164Speter int cpus; 187425164Speter u_long icr_lo, icr_hi; 187525164Speter 187627005Sfsmp POSTCODE(START_AP_POST); 187727005Sfsmp 187825164Speter /* get the PHYSICAL APIC ID# */ 187925164Speter physical_cpu = CPU_TO_ID(logical_cpu); 188025164Speter 188125164Speter /* calculate the vector */ 188225164Speter vector = (boot_addr >> 12) & 0xff; 188325164Speter 188425164Speter /* used as a watchpoint to signal AP startup */ 188525164Speter cpus = mp_ncpus; 188625164Speter 188725164Speter /* 188825164Speter * first we do an INIT/RESET IPI this INIT IPI might be run, reseting 188925164Speter * and running the target CPU. OR this INIT IPI might be latched (P5 189025164Speter * bug), CPU waiting for STARTUP IPI. OR this INIT IPI might be 189125164Speter * ignored. 189225164Speter */ 189325164Speter 189425164Speter /* setup the address for the target AP */ 189526812Speter icr_hi = lapic.icr_hi & ~APIC_ID_MASK; 189625164Speter icr_hi |= (physical_cpu << 24); 189726812Speter lapic.icr_hi = icr_hi; 189825164Speter 189925164Speter /* do an INIT IPI: assert RESET */ 190026812Speter icr_lo = lapic.icr_lo & 0xfff00000; 190126812Speter lapic.icr_lo = icr_lo | 0x0000c500; 190225164Speter 190325164Speter /* wait for pending status end */ 190426812Speter while (lapic.icr_lo & APIC_DELSTAT_MASK) 190525164Speter /* spin */ ; 190625164Speter 190725164Speter /* do an INIT IPI: deassert RESET */ 190826812Speter lapic.icr_lo = icr_lo | 0x00008500; 190925164Speter 191025164Speter /* wait for pending status end */ 191125164Speter u_sleep(10000); /* wait ~10mS */ 191226812Speter while (lapic.icr_lo & APIC_DELSTAT_MASK) 191325164Speter /* spin */ ; 191425164Speter 191525164Speter /* 191625164Speter * next we do a STARTUP IPI: the previous INIT IPI might still be 191725164Speter * latched, (P5 bug) this 1st STARTUP would then terminate 191825164Speter * immediately, and the previously started INIT IPI would continue. OR 191925164Speter * the previous INIT IPI has already run. and this STARTUP IPI will 192025164Speter * run. OR the previous INIT IPI was ignored. and this STARTUP IPI 192125164Speter * will run. 192225164Speter */ 192325164Speter 192425164Speter /* do a STARTUP IPI */ 192526812Speter lapic.icr_lo = icr_lo | 0x00000600 | vector; 192626812Speter while (lapic.icr_lo & APIC_DELSTAT_MASK) 192725164Speter /* spin */ ; 192825164Speter u_sleep(200); /* wait ~200uS */ 192925164Speter 193025164Speter /* 193125164Speter * finally we do a 2nd STARTUP IPI: this 2nd STARTUP IPI should run IF 193225164Speter * the previous STARTUP IPI was cancelled by a latched INIT IPI. OR 193325164Speter * this STARTUP IPI will be ignored, as only ONE STARTUP IPI is 193425164Speter * recognized after hardware RESET or INIT IPI. 193525164Speter */ 193625164Speter 193726812Speter lapic.icr_lo = icr_lo | 0x00000600 | vector; 193826812Speter while (lapic.icr_lo & APIC_DELSTAT_MASK) 193925164Speter /* spin */ ; 194025164Speter u_sleep(200); /* wait ~200uS */ 194125164Speter 194225164Speter /* wait for it to start */ 194325164Speter set_apic_timer(5000000);/* == 5 seconds */ 194425164Speter while (read_apic_timer()) 194525164Speter if (mp_ncpus > cpus) 194625164Speter return 1; /* return SUCCESS */ 194725164Speter 194825164Speter return 0; /* return FAILURE */ 194925164Speter} 195025164Speter 195125164Speter 195225164Speter/* 195325164Speter * Flush the TLB on all other CPU's 195425164Speter * 195525164Speter * XXX: Needs to handshake and wait for completion before proceding. 195625164Speter */ 195725164Spetervoid 195825215Sfsmpsmp_invltlb(void) 195925164Speter{ 196025419Sfsmp#if defined(APIC_IO) 196128809Speter if (smp_started && invltlb_ok) 196227005Sfsmp all_but_self_ipi(XINVLTLB_OFFSET); 196325419Sfsmp#endif /* APIC_IO */ 196425164Speter} 196525164Speter 196625164Spetervoid 196725164Speterinvlpg(u_int addr) 196825164Speter{ 196925164Speter __asm __volatile("invlpg (%0)"::"r"(addr):"memory"); 197025215Sfsmp 197125215Sfsmp /* send a message to the other CPUs */ 197225164Speter smp_invltlb(); 197325164Speter} 197425164Speter 197525164Spetervoid 197625164Speterinvltlb(void) 197725164Speter{ 197825164Speter u_long temp; 197925215Sfsmp 198025164Speter /* 198125164Speter * This should be implemented as load_cr3(rcr3()) when load_cr3() is 198225164Speter * inlined. 198325164Speter */ 198425164Speter __asm __volatile("movl %%cr3, %0; movl %0, %%cr3":"=r"(temp) :: "memory"); 198525215Sfsmp 198625215Sfsmp /* send a message to the other CPUs */ 198725164Speter smp_invltlb(); 198825164Speter} 198927005Sfsmp 199027005Sfsmp 199127005Sfsmp/* 199227005Sfsmp * When called the executing CPU will send an IPI to all other CPUs 199327005Sfsmp * requesting that they halt execution. 199427005Sfsmp * 199527005Sfsmp * Usually (but not necessarily) called with 'other_cpus' as its arg. 199627005Sfsmp * 199727005Sfsmp * - Signals all CPUs in map to stop. 199827005Sfsmp * - Waits for each to stop. 199927005Sfsmp * 200027005Sfsmp * Returns: 200127005Sfsmp * -1: error 200227005Sfsmp * 0: NA 200327005Sfsmp * 1: ok 200427005Sfsmp * 200527005Sfsmp * XXX FIXME: this is not MP-safe, needs a lock to prevent multiple CPUs 200627005Sfsmp * from executing at same time. 200727005Sfsmp */ 200827005Sfsmpint 200928808Speterstop_cpus(u_int map) 201027005Sfsmp{ 201128809Speter if (!smp_started) 201227005Sfsmp return 0; 201327005Sfsmp 201427353Sfsmp /* send the Xcpustop IPI to all CPUs in map */ 201527005Sfsmp selected_apic_ipi(map, XCPUSTOP_OFFSET, APIC_DELMODE_FIXED); 201636135Stegge 201736135Stegge while ((stopped_cpus & map) != map) 201827005Sfsmp /* spin */ ; 201927005Sfsmp 202027005Sfsmp return 1; 202127005Sfsmp} 202227005Sfsmp 202327005Sfsmp 202427005Sfsmp/* 202527005Sfsmp * Called by a CPU to restart stopped CPUs. 202627005Sfsmp * 202727005Sfsmp * Usually (but not necessarily) called with 'stopped_cpus' as its arg. 202827005Sfsmp * 202927005Sfsmp * - Signals all CPUs in map to restart. 203027005Sfsmp * - Waits for each to restart. 203127005Sfsmp * 203227005Sfsmp * Returns: 203327005Sfsmp * -1: error 203427005Sfsmp * 0: NA 203527005Sfsmp * 1: ok 203627005Sfsmp */ 203727005Sfsmpint 203828808Speterrestart_cpus(u_int map) 203927005Sfsmp{ 204028809Speter if (!smp_started) 204127005Sfsmp return 0; 204227005Sfsmp 204327255Sfsmp started_cpus = map; /* signal other cpus to restart */ 204427255Sfsmp 204536135Stegge while ((stopped_cpus & map) != 0) /* wait for each to clear its bit */ 204627005Sfsmp /* spin */ ; 204727005Sfsmp 204827005Sfsmp return 1; 204927005Sfsmp} 205028808Speter 205128808Speterint smp_active = 0; /* are the APs allowed to run? */ 205228808SpeterSYSCTL_INT(_machdep, OID_AUTO, smp_active, CTLFLAG_RW, &smp_active, 0, ""); 205328808Speter 205428808Speter/* XXX maybe should be hw.ncpu */ 205533181Seivindstatic int smp_cpus = 1; /* how many cpu's running */ 205628808SpeterSYSCTL_INT(_machdep, OID_AUTO, smp_cpus, CTLFLAG_RD, &smp_cpus, 0, ""); 205728808Speter 205828808Speterint invltlb_ok = 0; /* throttle smp_invltlb() till safe */ 205928808SpeterSYSCTL_INT(_machdep, OID_AUTO, invltlb_ok, CTLFLAG_RW, &invltlb_ok, 0, ""); 206028808Speter 206133181Seivind/* Warning: Do not staticize. Used from swtch.s */ 206233936Sdysonint do_page_zero_idle = 1; /* bzero pages for fun and profit in idleloop */ 206328808SpeterSYSCTL_INT(_machdep, OID_AUTO, do_page_zero_idle, CTLFLAG_RW, 206428808Speter &do_page_zero_idle, 0, ""); 206528808Speter 206634021Stegge/* Is forwarding of a interrupt to the CPU holding the ISR lock enabled ? */ 206734021Steggeint forward_irq_enabled = 1; 206834021SteggeSYSCTL_INT(_machdep, OID_AUTO, forward_irq_enabled, CTLFLAG_RW, 206934021Stegge &forward_irq_enabled, 0, ""); 207034021Stegge 207134020Stegge/* Enable forwarding of a signal to a process running on a different CPU */ 207234020Steggeint forward_signal_enabled = 1; 207334020SteggeSYSCTL_INT(_machdep, OID_AUTO, forward_signal_enabled, CTLFLAG_RW, 207434020Stegge &forward_signal_enabled, 0, ""); 207528808Speter 207636135Stegge/* Enable forwarding of roundrobin to all other cpus */ 207736135Steggeint forward_roundrobin_enabled = 1; 207836135SteggeSYSCTL_INT(_machdep, OID_AUTO, forward_roundrobin_enabled, CTLFLAG_RW, 207936135Stegge &forward_roundrobin_enabled, 0, ""); 208036135Stegge 208128808Speter/* 208228808Speter * This is called once the rest of the system is up and running and we're 208328808Speter * ready to let the AP's out of the pen. 208428808Speter */ 208528808Spetervoid ap_init(void); 208628808Speter 208728808Spetervoid 208828808Speterap_init() 208928808Speter{ 209028808Speter u_int temp; 209128808Speter u_int apic_id; 209228808Speter 209328808Speter smp_cpus++; 209428808Speter 209534197Stegge#if defined(I586_CPU) && !defined(NO_F00F_HACK) 209634197Stegge lidt(&r_idt); 209734197Stegge#endif 209834197Stegge 209928808Speter /* Build our map of 'other' CPUs. */ 210028808Speter other_cpus = all_cpus & ~(1 << cpuid); 210128808Speter 210228808Speter printf("SMP: AP CPU #%d Launched!\n", cpuid); 210328808Speter 210428808Speter /* XXX FIXME: i386 specific, and redundant: Setup the FPU. */ 210528808Speter load_cr0((rcr0() & ~CR0_EM) | CR0_MP | CR0_NE | CR0_TS); 210628808Speter 210728808Speter /* A quick check from sanity claus */ 210828808Speter apic_id = (apic_id_to_logical[(lapic.id & 0x0f000000) >> 24]); 210928808Speter if (cpuid != apic_id) { 211028808Speter printf("SMP: cpuid = %d\n", cpuid); 211128808Speter printf("SMP: apic_id = %d\n", apic_id); 211228808Speter printf("PTD[MPPTDI] = %08x\n", PTD[MPPTDI]); 211328808Speter panic("cpuid mismatch! boom!!"); 211428808Speter } 211528808Speter 211635932Sdyson getmtrr(); 211735932Sdyson 211828808Speter /* Init local apic for irq's */ 211928808Speter apic_initialize(); 212028808Speter 212128808Speter /* 212228808Speter * Activate smp_invltlb, although strictly speaking, this isn't 212328808Speter * quite correct yet. We should have a bitfield for cpus willing 212428808Speter * to accept TLB flush IPI's or something and sync them. 212528808Speter */ 212628808Speter invltlb_ok = 1; 212728809Speter smp_started = 1; /* enable IPI's, tlb shootdown, freezes etc */ 212828808Speter smp_active = 1; /* historic */ 212928808Speter 213028808Speter curproc = NULL; /* make sure */ 213128808Speter} 213230112Sdyson 213331639Sfsmp#ifdef BETTER_CLOCK 213431639Sfsmp 213531639Sfsmp#define CHECKSTATE_USER 0 213631639Sfsmp#define CHECKSTATE_SYS 1 213731639Sfsmp#define CHECKSTATE_INTR 2 213831639Sfsmp 213933181Seivind/* Do not staticize. Used from apic_vector.s */ 214031639Sfsmpstruct proc* checkstate_curproc[NCPU]; 214131639Sfsmpint checkstate_cpustate[NCPU]; 214231639Sfsmpu_long checkstate_pc[NCPU]; 214331639Sfsmp 214431639Sfsmpextern long cp_time[CPUSTATES]; 214531639Sfsmp 214631639Sfsmp#define PC_TO_INDEX(pc, prof) \ 214731639Sfsmp ((int)(((u_quad_t)((pc) - (prof)->pr_off) * \ 214831639Sfsmp (u_quad_t)((prof)->pr_scale)) >> 16) & ~1) 214931639Sfsmp 215031639Sfsmpstatic void 215131639Sfsmpaddupc_intr_forwarded(struct proc *p, int id, int *astmap) 215231639Sfsmp{ 215331639Sfsmp int i; 215431639Sfsmp struct uprof *prof; 215531639Sfsmp u_long pc; 215631639Sfsmp 215731639Sfsmp pc = checkstate_pc[id]; 215831639Sfsmp prof = &p->p_stats->p_prof; 215931639Sfsmp if (pc >= prof->pr_off && 216031639Sfsmp (i = PC_TO_INDEX(pc, prof)) < prof->pr_size) { 216131639Sfsmp if ((p->p_flag & P_OWEUPC) == 0) { 216231639Sfsmp prof->pr_addr = pc; 216331639Sfsmp prof->pr_ticks = 1; 216431639Sfsmp p->p_flag |= P_OWEUPC; 216531639Sfsmp } 216631639Sfsmp *astmap |= (1 << id); 216731639Sfsmp } 216831639Sfsmp} 216931639Sfsmp 217031639Sfsmpstatic void 217131639Sfsmpforwarded_statclock(int id, int pscnt, int *astmap) 217231639Sfsmp{ 217331639Sfsmp struct pstats *pstats; 217431639Sfsmp long rss; 217531639Sfsmp struct rusage *ru; 217631639Sfsmp struct vmspace *vm; 217731639Sfsmp int cpustate; 217831639Sfsmp struct proc *p; 217931639Sfsmp#ifdef GPROF 218031639Sfsmp register struct gmonparam *g; 218131639Sfsmp int i; 218231639Sfsmp#endif 218331639Sfsmp 218431639Sfsmp p = checkstate_curproc[id]; 218531639Sfsmp cpustate = checkstate_cpustate[id]; 218631639Sfsmp 218731639Sfsmp switch (cpustate) { 218831639Sfsmp case CHECKSTATE_USER: 218931639Sfsmp if (p->p_flag & P_PROFIL) 219031639Sfsmp addupc_intr_forwarded(p, id, astmap); 219131639Sfsmp if (pscnt > 1) 219231639Sfsmp return; 219331639Sfsmp p->p_uticks++; 219431639Sfsmp if (p->p_nice > NZERO) 219531639Sfsmp cp_time[CP_NICE]++; 219631639Sfsmp else 219731639Sfsmp cp_time[CP_USER]++; 219831639Sfsmp break; 219931639Sfsmp case CHECKSTATE_SYS: 220031639Sfsmp#ifdef GPROF 220131639Sfsmp /* 220231639Sfsmp * Kernel statistics are just like addupc_intr, only easier. 220331639Sfsmp */ 220431639Sfsmp g = &_gmonparam; 220531639Sfsmp if (g->state == GMON_PROF_ON) { 220631639Sfsmp i = checkstate_pc[id] - g->lowpc; 220731639Sfsmp if (i < g->textsize) { 220831639Sfsmp i /= HISTFRACTION * sizeof(*g->kcount); 220931639Sfsmp g->kcount[i]++; 221031639Sfsmp } 221131639Sfsmp } 221231639Sfsmp#endif 221331639Sfsmp if (pscnt > 1) 221431639Sfsmp return; 221531639Sfsmp 221631639Sfsmp if (!p) 221731639Sfsmp cp_time[CP_IDLE]++; 221831639Sfsmp else { 221931639Sfsmp p->p_sticks++; 222031639Sfsmp cp_time[CP_SYS]++; 222131639Sfsmp } 222231639Sfsmp break; 222331639Sfsmp case CHECKSTATE_INTR: 222431639Sfsmp default: 222531639Sfsmp#ifdef GPROF 222631639Sfsmp /* 222731639Sfsmp * Kernel statistics are just like addupc_intr, only easier. 222831639Sfsmp */ 222931639Sfsmp g = &_gmonparam; 223031639Sfsmp if (g->state == GMON_PROF_ON) { 223131639Sfsmp i = checkstate_pc[id] - g->lowpc; 223231639Sfsmp if (i < g->textsize) { 223331639Sfsmp i /= HISTFRACTION * sizeof(*g->kcount); 223431639Sfsmp g->kcount[i]++; 223531639Sfsmp } 223631639Sfsmp } 223731639Sfsmp#endif 223831639Sfsmp if (pscnt > 1) 223931639Sfsmp return; 224031639Sfsmp if (p) 224131639Sfsmp p->p_iticks++; 224231639Sfsmp cp_time[CP_INTR]++; 224331639Sfsmp } 224431639Sfsmp if (p != NULL) { 224531639Sfsmp p->p_cpticks++; 224631639Sfsmp if (++p->p_estcpu == 0) 224731639Sfsmp p->p_estcpu--; 224831639Sfsmp if ((p->p_estcpu & 3) == 0) { 224931639Sfsmp resetpriority(p); 225031639Sfsmp if (p->p_priority >= PUSER) 225131639Sfsmp p->p_priority = p->p_usrpri; 225231639Sfsmp } 225331639Sfsmp 225431639Sfsmp /* Update resource usage integrals and maximums. */ 225531639Sfsmp if ((pstats = p->p_stats) != NULL && 225631639Sfsmp (ru = &pstats->p_ru) != NULL && 225731639Sfsmp (vm = p->p_vmspace) != NULL) { 225831639Sfsmp ru->ru_ixrss += vm->vm_tsize * PAGE_SIZE / 1024; 225931639Sfsmp ru->ru_idrss += vm->vm_dsize * PAGE_SIZE / 1024; 226031639Sfsmp ru->ru_isrss += vm->vm_ssize * PAGE_SIZE / 1024; 226131639Sfsmp rss = vm->vm_pmap.pm_stats.resident_count * 226231639Sfsmp PAGE_SIZE / 1024; 226331639Sfsmp if (ru->ru_maxrss < rss) 226431639Sfsmp ru->ru_maxrss = rss; 226531639Sfsmp } 226631639Sfsmp } 226731639Sfsmp} 226831639Sfsmp 226931639Sfsmpvoid 227031639Sfsmpforward_statclock(int pscnt) 227131639Sfsmp{ 227231639Sfsmp int map; 227331639Sfsmp int id; 227431639Sfsmp int i; 227531639Sfsmp 227631639Sfsmp /* Kludge. We don't yet have separate locks for the interrupts 227731639Sfsmp * and the kernel. This means that we cannot let the other processors 227831639Sfsmp * handle complex interrupts while inhibiting them from entering 227931639Sfsmp * the kernel in a non-interrupt context. 228031639Sfsmp * 228131639Sfsmp * What we can do, without changing the locking mechanisms yet, 228231639Sfsmp * is letting the other processors handle a very simple interrupt 228331639Sfsmp * (wich determines the processor states), and do the main 228431639Sfsmp * work ourself. 228531639Sfsmp */ 228631639Sfsmp 228731720Stegge if (!smp_started || !invltlb_ok || cold || panicstr) 228831639Sfsmp return; 228931639Sfsmp 229031639Sfsmp /* Step 1: Probe state (user, cpu, interrupt, spinlock, idle ) */ 229131639Sfsmp 229231720Stegge map = other_cpus & ~stopped_cpus ; 229331639Sfsmp checkstate_probed_cpus = 0; 229431720Stegge if (map != 0) 229531720Stegge selected_apic_ipi(map, 229631720Stegge XCPUCHECKSTATE_OFFSET, APIC_DELMODE_FIXED); 229731639Sfsmp 229831639Sfsmp i = 0; 229931639Sfsmp while (checkstate_probed_cpus != map) { 230031639Sfsmp /* spin */ 230131639Sfsmp i++; 230236135Stegge if (i == 100000) { 230336135Stegge#ifdef BETTER_CLOCK_DIAGNOSTIC 230431639Sfsmp printf("forward_statclock: checkstate %x\n", 230531639Sfsmp checkstate_probed_cpus); 230636135Stegge#endif 230731720Stegge break; 230831639Sfsmp } 230931639Sfsmp } 231031639Sfsmp 231131639Sfsmp /* 231231639Sfsmp * Step 2: walk through other processors processes, update ticks and 231331639Sfsmp * profiling info. 231431639Sfsmp */ 231531639Sfsmp 231631639Sfsmp map = 0; 231731639Sfsmp for (id = 0; id < mp_ncpus; id++) { 231831639Sfsmp if (id == cpuid) 231931639Sfsmp continue; 232031639Sfsmp if (((1 << id) & checkstate_probed_cpus) == 0) 232131720Stegge continue; 232231639Sfsmp forwarded_statclock(id, pscnt, &map); 232331639Sfsmp } 232431639Sfsmp if (map != 0) { 232531639Sfsmp checkstate_need_ast |= map; 232631639Sfsmp selected_apic_ipi(map, XCPUAST_OFFSET, APIC_DELMODE_FIXED); 232731639Sfsmp i = 0; 232834019Stegge while ((checkstate_need_ast & map) != 0) { 232931639Sfsmp /* spin */ 233031639Sfsmp i++; 233134019Stegge if (i > 100000) { 233234019Stegge#ifdef BETTER_CLOCK_DIAGNOSTIC 233331639Sfsmp printf("forward_statclock: dropped ast 0x%x\n", 233434019Stegge checkstate_need_ast & map); 233534019Stegge#endif 233631639Sfsmp break; 233731639Sfsmp } 233831639Sfsmp } 233931639Sfsmp } 234031639Sfsmp} 234131639Sfsmp 234231639Sfsmpvoid 234331639Sfsmpforward_hardclock(int pscnt) 234431639Sfsmp{ 234531639Sfsmp int map; 234631639Sfsmp int id; 234731639Sfsmp struct proc *p; 234831639Sfsmp struct pstats *pstats; 234931639Sfsmp int i; 235031639Sfsmp 235131639Sfsmp /* Kludge. We don't yet have separate locks for the interrupts 235231639Sfsmp * and the kernel. This means that we cannot let the other processors 235331639Sfsmp * handle complex interrupts while inhibiting them from entering 235431639Sfsmp * the kernel in a non-interrupt context. 235531639Sfsmp * 235631639Sfsmp * What we can do, without changing the locking mechanisms yet, 235731639Sfsmp * is letting the other processors handle a very simple interrupt 235831639Sfsmp * (wich determines the processor states), and do the main 235931639Sfsmp * work ourself. 236031639Sfsmp */ 236131639Sfsmp 236231720Stegge if (!smp_started || !invltlb_ok || cold || panicstr) 236331639Sfsmp return; 236431639Sfsmp 236531639Sfsmp /* Step 1: Probe state (user, cpu, interrupt, spinlock, idle) */ 236631639Sfsmp 236731720Stegge map = other_cpus & ~stopped_cpus ; 236831639Sfsmp checkstate_probed_cpus = 0; 236931720Stegge if (map != 0) 237031720Stegge selected_apic_ipi(map, 237131720Stegge XCPUCHECKSTATE_OFFSET, APIC_DELMODE_FIXED); 237231720Stegge 237331639Sfsmp i = 0; 237431639Sfsmp while (checkstate_probed_cpus != map) { 237531639Sfsmp /* spin */ 237631639Sfsmp i++; 237736135Stegge if (i == 100000) { 237836135Stegge#ifdef BETTER_CLOCK_DIAGNOSTIC 237931639Sfsmp printf("forward_hardclock: checkstate %x\n", 238031639Sfsmp checkstate_probed_cpus); 238136135Stegge#endif 238231720Stegge break; 238331639Sfsmp } 238431639Sfsmp } 238531639Sfsmp 238631639Sfsmp /* 238731639Sfsmp * Step 2: walk through other processors processes, update virtual 238831639Sfsmp * timer and profiling timer. If stathz == 0, also update ticks and 238931639Sfsmp * profiling info. 239031639Sfsmp */ 239131639Sfsmp 239231639Sfsmp map = 0; 239331639Sfsmp for (id = 0; id < mp_ncpus; id++) { 239431639Sfsmp if (id == cpuid) 239531639Sfsmp continue; 239631639Sfsmp if (((1 << id) & checkstate_probed_cpus) == 0) 239731720Stegge continue; 239831639Sfsmp p = checkstate_curproc[id]; 239931639Sfsmp if (p) { 240031639Sfsmp pstats = p->p_stats; 240131639Sfsmp if (checkstate_cpustate[id] == CHECKSTATE_USER && 240235058Sphk timevalisset(&pstats->p_timer[ITIMER_VIRTUAL].it_value) && 240331639Sfsmp itimerdecr(&pstats->p_timer[ITIMER_VIRTUAL], tick) == 0) { 240431639Sfsmp psignal(p, SIGVTALRM); 240531639Sfsmp map |= (1 << id); 240631639Sfsmp } 240735058Sphk if (timevalisset(&pstats->p_timer[ITIMER_PROF].it_value) && 240831639Sfsmp itimerdecr(&pstats->p_timer[ITIMER_PROF], tick) == 0) { 240931639Sfsmp psignal(p, SIGPROF); 241031639Sfsmp map |= (1 << id); 241131639Sfsmp } 241231639Sfsmp } 241331639Sfsmp if (stathz == 0) { 241431639Sfsmp forwarded_statclock( id, pscnt, &map); 241531639Sfsmp } 241631639Sfsmp } 241731639Sfsmp if (map != 0) { 241831639Sfsmp checkstate_need_ast |= map; 241931639Sfsmp selected_apic_ipi(map, XCPUAST_OFFSET, APIC_DELMODE_FIXED); 242031639Sfsmp i = 0; 242134019Stegge while ((checkstate_need_ast & map) != 0) { 242231639Sfsmp /* spin */ 242331639Sfsmp i++; 242434019Stegge if (i > 100000) { 242534019Stegge#ifdef BETTER_CLOCK_DIAGNOSTIC 242631639Sfsmp printf("forward_hardclock: dropped ast 0x%x\n", 242734019Stegge checkstate_need_ast & map); 242834019Stegge#endif 242931639Sfsmp break; 243031639Sfsmp } 243131639Sfsmp } 243231639Sfsmp } 243331639Sfsmp} 243431639Sfsmp 243531639Sfsmp#endif /* BETTER_CLOCK */ 243634020Stegge 243734020Steggevoid 243834020Steggeforward_signal(struct proc *p) 243934020Stegge{ 244034020Stegge int map; 244134020Stegge int id; 244234020Stegge int i; 244334020Stegge 244434020Stegge /* Kludge. We don't yet have separate locks for the interrupts 244534020Stegge * and the kernel. This means that we cannot let the other processors 244634020Stegge * handle complex interrupts while inhibiting them from entering 244734020Stegge * the kernel in a non-interrupt context. 244834020Stegge * 244934020Stegge * What we can do, without changing the locking mechanisms yet, 245034020Stegge * is letting the other processors handle a very simple interrupt 245134020Stegge * (wich determines the processor states), and do the main 245234020Stegge * work ourself. 245334020Stegge */ 245434020Stegge 245534020Stegge if (!smp_started || !invltlb_ok || cold || panicstr) 245634020Stegge return; 245734020Stegge if (!forward_signal_enabled) 245834020Stegge return; 245934020Stegge while (1) { 246034020Stegge if (p->p_stat != SRUN) 246134020Stegge return; 246234020Stegge id = (u_char) p->p_oncpu; 246334020Stegge if (id == 0xff) 246434020Stegge return; 246534020Stegge map = (1<<id); 246634020Stegge checkstate_need_ast |= map; 246734020Stegge selected_apic_ipi(map, XCPUAST_OFFSET, APIC_DELMODE_FIXED); 246834020Stegge i = 0; 246934020Stegge while ((checkstate_need_ast & map) != 0) { 247034020Stegge /* spin */ 247134020Stegge i++; 247234020Stegge if (i > 100000) { 247334020Stegge#if 0 247434020Stegge printf("forward_signal: dropped ast 0x%x\n", 247534020Stegge checkstate_need_ast & map); 247634020Stegge#endif 247734020Stegge break; 247834020Stegge } 247934020Stegge } 248034020Stegge if (id == (u_char) p->p_oncpu) 248134020Stegge return; 248234020Stegge } 248334020Stegge} 248434021Stegge 248536135Steggevoid 248636135Steggeforward_roundrobin(void) 248736135Stegge{ 248836135Stegge u_int map; 248936135Stegge int i; 249034021Stegge 249136135Stegge if (!smp_started || !invltlb_ok || cold || panicstr) 249236135Stegge return; 249336135Stegge if (!forward_roundrobin_enabled) 249436135Stegge return; 249536135Stegge resched_cpus |= other_cpus; 249636135Stegge map = other_cpus & ~stopped_cpus ; 249736135Stegge#if 1 249836135Stegge selected_apic_ipi(map, XCPUAST_OFFSET, APIC_DELMODE_FIXED); 249936135Stegge#else 250036135Stegge (void) all_but_self_ipi(XCPUAST_OFFSET); 250136135Stegge#endif 250236135Stegge i = 0; 250336135Stegge while ((checkstate_need_ast & map) != 0) { 250436135Stegge /* spin */ 250536135Stegge i++; 250636135Stegge if (i > 100000) { 250736135Stegge#if 0 250836135Stegge printf("forward_roundrobin: dropped ast 0x%x\n", 250936135Stegge checkstate_need_ast & map); 251036135Stegge#endif 251136135Stegge break; 251236135Stegge } 251336135Stegge } 251436135Stegge} 251536135Stegge 251636135Stegge 251734021Stegge#ifdef APIC_INTR_REORDER 251834021Stegge/* 251934021Stegge * Maintain mapping from softintr vector to isr bit in local apic. 252034021Stegge */ 252134021Steggevoid 252234021Steggeset_lapic_isrloc(int intr, int vector) 252334021Stegge{ 252434021Stegge if (intr < 0 || intr > 32) 252534021Stegge panic("set_apic_isrloc: bad intr argument: %d",intr); 252634021Stegge if (vector < ICU_OFFSET || vector > 255) 252734021Stegge panic("set_apic_isrloc: bad vector argument: %d",vector); 252834021Stegge apic_isrbit_location[intr].location = &lapic.isr0 + ((vector>>5)<<2); 252934021Stegge apic_isrbit_location[intr].bit = (1<<(vector & 31)); 253034021Stegge} 253134021Stegge#endif 2532