subr_smp.c revision 65557
125164Speter/* 225164Speter * Copyright (c) 1996, by Steve Passe 325164Speter * All rights reserved. 425164Speter * 525164Speter * Redistribution and use in source and binary forms, with or without 625164Speter * modification, are permitted provided that the following conditions 725164Speter * are met: 825164Speter * 1. Redistributions of source code must retain the above copyright 925164Speter * notice, this list of conditions and the following disclaimer. 1025164Speter * 2. The name of the developer may NOT be used to endorse or promote products 1125164Speter * derived from this software without specific prior written permission. 1225164Speter * 1325164Speter * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 1425164Speter * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 1525164Speter * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 1625164Speter * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 1725164Speter * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 1825164Speter * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 1925164Speter * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 2025164Speter * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 2125164Speter * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 2225164Speter * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 2325164Speter * SUCH DAMAGE. 2425164Speter * 2550477Speter * $FreeBSD: head/sys/kern/subr_smp.c 65557 2000-09-07 01:33:02Z jasone $ 2625164Speter */ 2725164Speter 2825164Speter#include "opt_smp.h" 2934197Stegge#include "opt_cpu.h" 3038422Smsmith#include "opt_user_ldt.h" 3125164Speter 3231639Sfsmp#ifdef SMP 3331639Sfsmp#include <machine/smptests.h> 3431639Sfsmp#else 3531639Sfsmp#error 3631639Sfsmp#endif 3731639Sfsmp 3828743Sbde#include <sys/param.h> 3965557Sjasone#include <sys/bus.h> 4025164Speter#include <sys/systm.h> 4128808Speter#include <sys/kernel.h> 4228808Speter#include <sys/proc.h> 4328808Speter#include <sys/sysctl.h> 4446703Speter#include <sys/malloc.h> 4546703Speter#include <sys/memrange.h> 4631639Sfsmp#ifdef BETTER_CLOCK 4731639Sfsmp#include <sys/dkstat.h> 4831639Sfsmp#endif 4949558Sphk#include <sys/cons.h> /* cngetc() */ 5025164Speter 5128808Speter#include <vm/vm.h> 5228808Speter#include <vm/vm_param.h> 5328808Speter#include <vm/pmap.h> 5426812Speter#include <vm/vm_kern.h> 5526812Speter#include <vm/vm_extern.h> 5631639Sfsmp#ifdef BETTER_CLOCK 5731639Sfsmp#include <sys/lock.h> 5831639Sfsmp#include <vm/vm_map.h> 5931639Sfsmp#include <sys/user.h> 6031689Stegge#ifdef GPROF 6131689Stegge#include <sys/gmon.h> 6231639Sfsmp#endif 6331689Stegge#endif 6425164Speter 6525164Speter#include <machine/smp.h> 6625164Speter#include <machine/apic.h> 6748924Smsmith#include <machine/atomic.h> 6848924Smsmith#include <machine/cpufunc.h> 6965557Sjasone#include <machine/mutex.h> 7025164Speter#include <machine/mpapic.h> 7148924Smsmith#include <machine/psl.h> 7225164Speter#include <machine/segments.h> 7327697Sfsmp#include <machine/smptests.h> /** TEST_DEFAULT_CONFIG, TEST_TEST1 */ 7426812Speter#include <machine/tss.h> 7526896Stegge#include <machine/specialreg.h> 7635077Speter#include <machine/globaldata.h> 7725164Speter 7825215Sfsmp#if defined(APIC_IO) 7927289Sfsmp#include <machine/md_var.h> /* setidt() */ 8027289Sfsmp#include <i386/isa/icu.h> /* IPIs */ 8127289Sfsmp#include <i386/isa/intr_machdep.h> /* IPIs */ 8225215Sfsmp#endif /* APIC_IO */ 8325164Speter 8428027Sfsmp#if defined(TEST_DEFAULT_CONFIG) 8528027Sfsmp#define MPFPS_MPFB1 TEST_DEFAULT_CONFIG 8628027Sfsmp#else 8728027Sfsmp#define MPFPS_MPFB1 mpfps->mpfb1 8828027Sfsmp#endif /* TEST_DEFAULT_CONFIG */ 8928027Sfsmp 9027005Sfsmp#define WARMBOOT_TARGET 0 9127005Sfsmp#define WARMBOOT_OFF (KERNBASE + 0x0467) 9227005Sfsmp#define WARMBOOT_SEG (KERNBASE + 0x0469) 9325164Speter 9440067Skato#ifdef PC98 9540067Skato#define BIOS_BASE (0xe8000) 9640067Skato#define BIOS_SIZE (0x18000) 9740067Skato#else 9827005Sfsmp#define BIOS_BASE (0xf0000) 9927005Sfsmp#define BIOS_SIZE (0x10000) 10040067Skato#endif 10127005Sfsmp#define BIOS_COUNT (BIOS_SIZE/4) 10225164Speter 10327005Sfsmp#define CMOS_REG (0x70) 10427005Sfsmp#define CMOS_DATA (0x71) 10527005Sfsmp#define BIOS_RESET (0x0f) 10627005Sfsmp#define BIOS_WARM (0x0a) 10725164Speter 10826155Sfsmp#define PROCENTRY_FLAG_EN 0x01 10926155Sfsmp#define PROCENTRY_FLAG_BP 0x02 11026155Sfsmp#define IOAPICENTRY_FLAG_EN 0x01 11126155Sfsmp 11227005Sfsmp 11326155Sfsmp/* MP Floating Pointer Structure */ 11426155Sfsmptypedef struct MPFPS { 11526155Sfsmp char signature[4]; 11626155Sfsmp void *pap; 11726155Sfsmp u_char length; 11826155Sfsmp u_char spec_rev; 11926155Sfsmp u_char checksum; 12026155Sfsmp u_char mpfb1; 12126155Sfsmp u_char mpfb2; 12226155Sfsmp u_char mpfb3; 12326155Sfsmp u_char mpfb4; 12426155Sfsmp u_char mpfb5; 12526155Sfsmp} *mpfps_t; 12626155Sfsmp 12726155Sfsmp/* MP Configuration Table Header */ 12826155Sfsmptypedef struct MPCTH { 12926155Sfsmp char signature[4]; 13026155Sfsmp u_short base_table_length; 13126155Sfsmp u_char spec_rev; 13226155Sfsmp u_char checksum; 13326155Sfsmp u_char oem_id[8]; 13426155Sfsmp u_char product_id[12]; 13526155Sfsmp void *oem_table_pointer; 13626155Sfsmp u_short oem_table_size; 13726155Sfsmp u_short entry_count; 13826155Sfsmp void *apic_address; 13926155Sfsmp u_short extended_table_length; 14026155Sfsmp u_char extended_table_checksum; 14126155Sfsmp u_char reserved; 14226155Sfsmp} *mpcth_t; 14326155Sfsmp 14426155Sfsmp 14526155Sfsmptypedef struct PROCENTRY { 14626155Sfsmp u_char type; 14726155Sfsmp u_char apic_id; 14826155Sfsmp u_char apic_version; 14926155Sfsmp u_char cpu_flags; 15026155Sfsmp u_long cpu_signature; 15126155Sfsmp u_long feature_flags; 15226155Sfsmp u_long reserved1; 15326155Sfsmp u_long reserved2; 15426155Sfsmp} *proc_entry_ptr; 15526155Sfsmp 15626155Sfsmptypedef struct BUSENTRY { 15726155Sfsmp u_char type; 15826155Sfsmp u_char bus_id; 15926155Sfsmp char bus_type[6]; 16026155Sfsmp} *bus_entry_ptr; 16126155Sfsmp 16226155Sfsmptypedef struct IOAPICENTRY { 16326155Sfsmp u_char type; 16426155Sfsmp u_char apic_id; 16526155Sfsmp u_char apic_version; 16626155Sfsmp u_char apic_flags; 16726155Sfsmp void *apic_address; 16826155Sfsmp} *io_apic_entry_ptr; 16926155Sfsmp 17026155Sfsmptypedef struct INTENTRY { 17126155Sfsmp u_char type; 17226155Sfsmp u_char int_type; 17326155Sfsmp u_short int_flags; 17426155Sfsmp u_char src_bus_id; 17526155Sfsmp u_char src_bus_irq; 17626155Sfsmp u_char dst_apic_id; 17726155Sfsmp u_char dst_apic_int; 17826155Sfsmp} *int_entry_ptr; 17926155Sfsmp 18026155Sfsmp/* descriptions of MP basetable entries */ 18126155Sfsmptypedef struct BASETABLE_ENTRY { 18226155Sfsmp u_char type; 18326155Sfsmp u_char length; 18426155Sfsmp char name[16]; 18526155Sfsmp} basetable_entry; 18626155Sfsmp 18725164Speter/* 18825164Speter * this code MUST be enabled here and in mpboot.s. 18925164Speter * it follows the very early stages of AP boot by placing values in CMOS ram. 19025164Speter * it NORMALLY will never be needed and thus the primitive method for enabling. 19125164Speter * 19225164Speter#define CHECK_POINTS 19325164Speter */ 19425164Speter 19540169Skato#if defined(CHECK_POINTS) && !defined(PC98) 19625164Speter#define CHECK_READ(A) (outb(CMOS_REG, (A)), inb(CMOS_DATA)) 19725164Speter#define CHECK_WRITE(A,D) (outb(CMOS_REG, (A)), outb(CMOS_DATA, (D))) 19825164Speter 19925164Speter#define CHECK_INIT(D); \ 20025164Speter CHECK_WRITE(0x34, (D)); \ 20125164Speter CHECK_WRITE(0x35, (D)); \ 20225164Speter CHECK_WRITE(0x36, (D)); \ 20325164Speter CHECK_WRITE(0x37, (D)); \ 20425164Speter CHECK_WRITE(0x38, (D)); \ 20525164Speter CHECK_WRITE(0x39, (D)); 20625164Speter 20725164Speter#define CHECK_PRINT(S); \ 20825164Speter printf("%s: %d, %d, %d, %d, %d, %d\n", \ 20925164Speter (S), \ 21025164Speter CHECK_READ(0x34), \ 21125164Speter CHECK_READ(0x35), \ 21225164Speter CHECK_READ(0x36), \ 21325164Speter CHECK_READ(0x37), \ 21425164Speter CHECK_READ(0x38), \ 21525164Speter CHECK_READ(0x39)); 21625164Speter 21725164Speter#else /* CHECK_POINTS */ 21825164Speter 21925164Speter#define CHECK_INIT(D) 22025164Speter#define CHECK_PRINT(S) 22125164Speter 22225164Speter#endif /* CHECK_POINTS */ 22325164Speter 22427005Sfsmp/* 22527005Sfsmp * Values to send to the POST hardware. 22627005Sfsmp */ 22727005Sfsmp#define MP_BOOTADDRESS_POST 0x10 22827005Sfsmp#define MP_PROBE_POST 0x11 22929213Sfsmp#define MPTABLE_PASS1_POST 0x12 23029213Sfsmp 23129213Sfsmp#define MP_START_POST 0x13 23229213Sfsmp#define MP_ENABLE_POST 0x14 23327005Sfsmp#define MPTABLE_PASS2_POST 0x15 23427005Sfsmp 23529213Sfsmp#define START_ALL_APS_POST 0x16 23629213Sfsmp#define INSTALL_AP_TRAMP_POST 0x17 23729213Sfsmp#define START_AP_POST 0x18 23829213Sfsmp 23929213Sfsmp#define MP_ANNOUNCE_POST 0x19 24029213Sfsmp 24165557Sjasone/* used to hold the AP's until we are ready to release them */ 24265557Sjasonestruct simplelock ap_boot_lock; 24329213Sfsmp 24427289Sfsmp/** XXX FIXME: where does this really belong, isa.h/isa.c perhaps? */ 24527255Sfsmpint current_postcode; 24627255Sfsmp 24727289Sfsmp/** XXX FIXME: what system files declare these??? */ 24825164Speterextern struct region_descriptor r_gdt, r_idt; 24925164Speter 25028669Sfsmpint bsp_apic_ready = 0; /* flags useability of BSP apic */ 25126155Sfsmpint mp_ncpus; /* # of CPUs, including BSP */ 25226155Sfsmpint mp_naps; /* # of Applications processors */ 25326155Sfsmpint mp_nbusses; /* # of busses */ 25426155Sfsmpint mp_napics; /* # of IO APICs */ 25526155Sfsmpint boot_cpu_id; /* designated BSP */ 25625164Spetervm_offset_t cpu_apic_address; 25726108Sfsmpvm_offset_t io_apic_address[NAPICID]; /* NAPICID is more than enough */ 25829655Sdysonextern int nkpt; 25925164Speter 26025164Speteru_int32_t cpu_apic_versions[NCPU]; 26125164Speteru_int32_t io_apic_versions[NAPIC]; 26225164Speter 26334021Stegge#ifdef APIC_INTR_DIAGNOSTIC 26434021Steggeint apic_itrace_enter[32]; 26534021Steggeint apic_itrace_tryisrlock[32]; 26634021Steggeint apic_itrace_gotisrlock[32]; 26734021Steggeint apic_itrace_active[32]; 26834021Steggeint apic_itrace_masked[32]; 26934021Steggeint apic_itrace_noisrlock[32]; 27034021Steggeint apic_itrace_masked2[32]; 27134021Steggeint apic_itrace_unmask[32]; 27234021Steggeint apic_itrace_noforward[32]; 27334021Steggeint apic_itrace_leave[32]; 27434021Steggeint apic_itrace_enter2[32]; 27534021Steggeint apic_itrace_doreti[32]; 27634021Steggeint apic_itrace_splz[32]; 27734021Steggeint apic_itrace_eoi[32]; 27834021Stegge#ifdef APIC_INTR_DIAGNOSTIC_IRQ 27934021Steggeunsigned short apic_itrace_debugbuffer[32768]; 28034021Steggeint apic_itrace_debugbuffer_idx; 28134021Steggestruct simplelock apic_itrace_debuglock; 28234021Stegge#endif 28334021Stegge#endif 28434021Stegge 28534021Stegge#ifdef APIC_INTR_REORDER 28634021Steggestruct { 28734021Stegge volatile int *location; 28834021Stegge int bit; 28934021Stegge} apic_isrbit_location[32]; 29034021Stegge#endif 29134021Stegge 29238888Steggestruct apic_intmapinfo int_to_apicintpin[APIC_INTMAPSIZE]; 29338888Stegge 29425164Speter/* 29526108Sfsmp * APIC ID logical/physical mapping structures. 29626108Sfsmp * We oversize these to simplify boot-time config. 29725164Speter */ 29826108Sfsmpint cpu_num_to_apic_id[NAPICID]; 29926108Sfsmpint io_num_to_apic_id[NAPICID]; 30025164Speterint apic_id_to_logical[NAPICID]; 30125164Speter 30230112Sdyson 30327005Sfsmp/* Bitmap of all available CPUs */ 30427005Sfsmpu_int all_cpus; 30527005Sfsmp 30646129Sluoqi/* AP uses this during bootstrap. Do not staticize. */ 30746129Sluoqichar *bootSTK; 30848144Sluoqistatic int bootAP; 30926812Speter 31026812Speter/* Hotwire a 0->4MB V==P mapping */ 31128808Speterextern pt_entry_t *KPTphys; 31226812Speter 31346129Sluoqi/* SMP page table page */ 31446129Sluoqiextern pt_entry_t *SMPpt; 31526812Speter 31636135Steggestruct pcb stoppcbs[NCPU]; 31736135Stegge 31838888Steggeint smp_started; /* has the system started? */ 31928809Speter 32025164Speter/* 32127289Sfsmp * Local data and functions. 32225164Speter */ 32325164Speter 32426155Sfsmpstatic int mp_capable; 32526155Sfsmpstatic u_int boot_address; 32626155Sfsmpstatic u_int base_memory; 32725164Speter 32826155Sfsmpstatic int picmode; /* 0: virtual wire mode, 1: PIC mode */ 32926155Sfsmpstatic mpfps_t mpfps; 33026155Sfsmpstatic int search_for_sig(u_int32_t target, int count); 33126155Sfsmpstatic void mp_enable(u_int boot_addr); 33225164Speter 33326155Sfsmpstatic int mptable_pass1(void); 33426155Sfsmpstatic int mptable_pass2(void); 33526155Sfsmpstatic void default_mp_table(int type); 33628027Sfsmpstatic void fix_mp_table(void); 33738888Steggestatic void setup_apic_irq_mapping(void); 33827634Sfsmpstatic void init_locks(void); 33926155Sfsmpstatic int start_all_aps(u_int boot_addr); 34026155Sfsmpstatic void install_ap_tramp(u_int boot_addr); 34126155Sfsmpstatic int start_ap(int logicalCpu, u_int boot_addr); 34255420Steggestatic int apic_int_is_bus_type(int intr, int bus_type); 34365557Sjasonestatic void release_aps(void *dummy); 34425164Speter 34525164Speter/* 34627289Sfsmp * Calculate usable address in base memory for AP trampoline code. 34725164Speter */ 34825164Speteru_int 34925164Spetermp_bootaddress(u_int basemem) 35025164Speter{ 35127005Sfsmp POSTCODE(MP_BOOTADDRESS_POST); 35227005Sfsmp 35325164Speter base_memory = basemem * 1024; /* convert to bytes */ 35425164Speter 35525164Speter boot_address = base_memory & ~0xfff; /* round down to 4k boundary */ 35625164Speter if ((base_memory - boot_address) < bootMP_size) 35725164Speter boot_address -= 4096; /* not enough, lower by 4k */ 35825164Speter 35925164Speter return boot_address; 36025164Speter} 36125164Speter 36225164Speter 36327289Sfsmp/* 36427289Sfsmp * Look for an Intel MP spec table (ie, SMP capable hardware). 36527289Sfsmp */ 36626155Sfsmpint 36726155Sfsmpmp_probe(void) 36826155Sfsmp{ 36926155Sfsmp int x; 37026155Sfsmp u_long segment; 37126155Sfsmp u_int32_t target; 37226155Sfsmp 37327005Sfsmp POSTCODE(MP_PROBE_POST); 37427005Sfsmp 37526155Sfsmp /* see if EBDA exists */ 37643314Sdillon if ((segment = (u_long) * (u_short *) (KERNBASE + 0x40e)) != 0) { 37726155Sfsmp /* search first 1K of EBDA */ 37826155Sfsmp target = (u_int32_t) (segment << 4); 37926155Sfsmp if ((x = search_for_sig(target, 1024 / 4)) >= 0) 38026155Sfsmp goto found; 38126155Sfsmp } else { 38226155Sfsmp /* last 1K of base memory, effective 'top of base' passed in */ 38326155Sfsmp target = (u_int32_t) (base_memory - 0x400); 38426155Sfsmp if ((x = search_for_sig(target, 1024 / 4)) >= 0) 38526155Sfsmp goto found; 38626155Sfsmp } 38726155Sfsmp 38826155Sfsmp /* search the BIOS */ 38926155Sfsmp target = (u_int32_t) BIOS_BASE; 39026155Sfsmp if ((x = search_for_sig(target, BIOS_COUNT)) >= 0) 39126155Sfsmp goto found; 39226155Sfsmp 39326155Sfsmp /* nothing found */ 39426155Sfsmp mpfps = (mpfps_t)0; 39526155Sfsmp mp_capable = 0; 39626155Sfsmp return 0; 39726155Sfsmp 39827289Sfsmpfound: 39926155Sfsmp /* calculate needed resources */ 40026155Sfsmp mpfps = (mpfps_t)x; 40126155Sfsmp if (mptable_pass1()) 40226155Sfsmp panic("you must reconfigure your kernel"); 40326155Sfsmp 40426155Sfsmp /* flag fact that we are running multiple processors */ 40526155Sfsmp mp_capable = 1; 40626155Sfsmp return 1; 40726155Sfsmp} 40826155Sfsmp 40926155Sfsmp 41025164Speter/* 41165557Sjasone * Initialize the SMP hardware and the APIC and start up the AP's. 41225164Speter */ 41325164Spetervoid 41425164Spetermp_start(void) 41525164Speter{ 41627005Sfsmp POSTCODE(MP_START_POST); 41727005Sfsmp 41825164Speter /* look for MP capable motherboard */ 41926155Sfsmp if (mp_capable) 42025164Speter mp_enable(boot_address); 42126101Sfsmp else 42226155Sfsmp panic("MP hardware not found!"); 42325164Speter} 42425164Speter 42525164Speter 42625164Speter/* 42727289Sfsmp * Print various information about the SMP system hardware and setup. 42825164Speter */ 42925164Spetervoid 43025164Spetermp_announce(void) 43125164Speter{ 43225164Speter int x; 43325164Speter 43427005Sfsmp POSTCODE(MP_ANNOUNCE_POST); 43527005Sfsmp 43625164Speter printf("FreeBSD/SMP: Multiprocessor motherboard\n"); 43727489Sfsmp printf(" cpu0 (BSP): apic id: %2d", CPU_TO_ID(0)); 43826812Speter printf(", version: 0x%08x", cpu_apic_versions[0]); 43926812Speter printf(", at 0x%08x\n", cpu_apic_address); 44025164Speter for (x = 1; x <= mp_naps; ++x) { 44127561Sfsmp printf(" cpu%d (AP): apic id: %2d", x, CPU_TO_ID(x)); 44226812Speter printf(", version: 0x%08x", cpu_apic_versions[x]); 44326812Speter printf(", at 0x%08x\n", cpu_apic_address); 44425164Speter } 44525164Speter 44625164Speter#if defined(APIC_IO) 44725164Speter for (x = 0; x < mp_napics; ++x) { 44827489Sfsmp printf(" io%d (APIC): apic id: %2d", x, IO_TO_ID(x)); 44926812Speter printf(", version: 0x%08x", io_apic_versions[x]); 45026812Speter printf(", at 0x%08x\n", io_apic_address[x]); 45125164Speter } 45225164Speter#else 45325164Speter printf(" Warning: APIC I/O disabled\n"); 45425164Speter#endif /* APIC_IO */ 45525164Speter} 45625164Speter 45725164Speter/* 45825164Speter * AP cpu's call this to sync up protected mode. 45925164Speter */ 46025164Spetervoid 46125164Speterinit_secondary(void) 46225164Speter{ 46329663Speter int gsel_tss; 46448144Sluoqi int x, myid = bootAP; 46525164Speter 46646129Sluoqi gdt_segs[GPRIV_SEL].ssd_base = (int) &SMP_prvspace[myid]; 46746129Sluoqi gdt_segs[GPROC0_SEL].ssd_base = 46846129Sluoqi (int) &SMP_prvspace[myid].globaldata.gd_common_tss; 46946129Sluoqi SMP_prvspace[myid].globaldata.gd_prvspace = &SMP_prvspace[myid]; 47046129Sluoqi 47146129Sluoqi for (x = 0; x < NGDT; x++) { 47246129Sluoqi ssdtosd(&gdt_segs[x], &gdt[myid * NGDT + x].sd); 47346129Sluoqi } 47446129Sluoqi 47546129Sluoqi r_gdt.rd_limit = NGDT * sizeof(gdt[0]) - 1; 47646129Sluoqi r_gdt.rd_base = (int) &gdt[myid * NGDT]; 47727289Sfsmp lgdt(&r_gdt); /* does magic intra-segment return */ 47846129Sluoqi 47925164Speter lidt(&r_idt); 48046129Sluoqi 48125164Speter lldt(_default_ldt); 48238422Smsmith#ifdef USER_LDT 48338422Smsmith currentldt = _default_ldt; 48438422Smsmith#endif 48525164Speter 48646129Sluoqi gsel_tss = GSEL(GPROC0_SEL, SEL_KPL); 48746129Sluoqi gdt[myid * NGDT + GPROC0_SEL].sd.sd_type = SDT_SYS386TSS; 48826812Speter common_tss.tss_esp0 = 0; /* not used until after switch */ 48926812Speter common_tss.tss_ss0 = GSEL(GDATA_SEL, SEL_KPL); 49026812Speter common_tss.tss_ioopt = (sizeof common_tss) << 16; 49147081Sluoqi tss_gdt = &gdt[myid * NGDT + GPROC0_SEL].sd; 49247081Sluoqi common_tssd = *tss_gdt; 49325164Speter ltr(gsel_tss); 49425164Speter 49548144Sluoqi pmap_set_opt(); 49625164Speter} 49725164Speter 49825164Speter 49925164Speter#if defined(APIC_IO) 50027289Sfsmp/* 50127289Sfsmp * Final configuration of the BSP's local APIC: 50227289Sfsmp * - disable 'pic mode'. 50327289Sfsmp * - disable 'virtual wire mode'. 50427289Sfsmp * - enable NMI. 50527289Sfsmp */ 50625164Spetervoid 50727289Sfsmpbsp_apic_configure(void) 50825164Speter{ 50927289Sfsmp u_char byte; 51027289Sfsmp u_int32_t temp; 51125164Speter 51227289Sfsmp /* leave 'pic mode' if necessary */ 51325164Speter if (picmode) { 51425164Speter outb(0x22, 0x70); /* select IMCR */ 51525164Speter byte = inb(0x23); /* current contents */ 51627289Sfsmp byte |= 0x01; /* mask external INTR */ 51725164Speter outb(0x23, byte); /* disconnect 8259s/NMI */ 51825164Speter } 51927001Sfsmp 52027001Sfsmp /* mask lint0 (the 8259 'virtual wire' connection) */ 52126812Speter temp = lapic.lvt_lint0; 52227289Sfsmp temp |= APIC_LVT_M; /* set the mask */ 52326812Speter lapic.lvt_lint0 = temp; 52427001Sfsmp 52527001Sfsmp /* setup lint1 to handle NMI */ 52627001Sfsmp temp = lapic.lvt_lint1; 52727289Sfsmp temp &= ~APIC_LVT_M; /* clear the mask */ 52827289Sfsmp lapic.lvt_lint1 = temp; 52927001Sfsmp 53027353Sfsmp if (bootverbose) 53127561Sfsmp apic_dump("bsp_apic_configure()"); 53225164Speter} 53327005Sfsmp#endif /* APIC_IO */ 53425164Speter 53525164Speter 53625164Speter/******************************************************************* 53725164Speter * local functions and data 53825164Speter */ 53925164Speter 54025164Speter/* 54125164Speter * start the SMP system 54225164Speter */ 54325164Speterstatic void 54425164Spetermp_enable(u_int boot_addr) 54525164Speter{ 54625164Speter int x; 54725164Speter#if defined(APIC_IO) 54825164Speter int apic; 54925164Speter u_int ux; 55025164Speter#endif /* APIC_IO */ 55125164Speter 55227005Sfsmp POSTCODE(MP_ENABLE_POST); 55327005Sfsmp 55427289Sfsmp /* turn on 4MB of V == P addressing so we can get to MP table */ 55538349Sbde *(int *)PTD = PG_V | PG_RW | ((uintptr_t)(void *)KPTphys & PG_FRAME); 55626812Speter invltlb(); 55726812Speter 55826812Speter /* examine the MP table for needed info, uses physical addresses */ 55926155Sfsmp x = mptable_pass2(); 56025164Speter 56126812Speter *(int *)PTD = 0; 56226812Speter invltlb(); 56325164Speter 56425164Speter /* can't process default configs till the CPU APIC is pmapped */ 56525164Speter if (x) 56625164Speter default_mp_table(x); 56725164Speter 56828027Sfsmp /* post scan cleanup */ 56928027Sfsmp fix_mp_table(); 57038888Stegge setup_apic_irq_mapping(); 57128027Sfsmp 57225164Speter#if defined(APIC_IO) 57327353Sfsmp 57425164Speter /* fill the LOGICAL io_apic_versions table */ 57525164Speter for (apic = 0; apic < mp_napics; ++apic) { 57625164Speter ux = io_apic_read(apic, IOAPIC_VER); 57725164Speter io_apic_versions[apic] = ux; 57861136Smsmith io_apic_set_id(apic, IO_TO_ID(apic)); 57925164Speter } 58025164Speter 58125216Sfsmp /* program each IO APIC in the system */ 58225164Speter for (apic = 0; apic < mp_napics; ++apic) 58326266Speter if (io_apic_setup(apic) < 0) 58426266Speter panic("IO APIC setup failure"); 58525164Speter 58627353Sfsmp /* install a 'Spurious INTerrupt' vector */ 58727353Sfsmp setidt(XSPURIOUSINT_OFFSET, Xspuriousint, 58827353Sfsmp SDT_SYS386IGT, SEL_KPL, GSEL(GCODE_SEL, SEL_KPL)); 58927353Sfsmp 59025204Sfsmp /* install an inter-CPU IPI for TLB invalidation */ 59127005Sfsmp setidt(XINVLTLB_OFFSET, Xinvltlb, 59225216Sfsmp SDT_SYS386IGT, SEL_KPL, GSEL(GCODE_SEL, SEL_KPL)); 59327005Sfsmp 59431639Sfsmp#ifdef BETTER_CLOCK 59531639Sfsmp /* install an inter-CPU IPI for reading processor state */ 59631639Sfsmp setidt(XCPUCHECKSTATE_OFFSET, Xcpucheckstate, 59731639Sfsmp SDT_SYS386IGT, SEL_KPL, GSEL(GCODE_SEL, SEL_KPL)); 59834020Stegge#endif 59931639Sfsmp 60048924Smsmith /* install an inter-CPU IPI for all-CPU rendezvous */ 60148924Smsmith setidt(XRENDEZVOUS_OFFSET, Xrendezvous, 60248924Smsmith SDT_SYS386IGT, SEL_KPL, GSEL(GCODE_SEL, SEL_KPL)); 60348924Smsmith 60431639Sfsmp /* install an inter-CPU IPI for forcing an additional software trap */ 60531639Sfsmp setidt(XCPUAST_OFFSET, Xcpuast, 60631639Sfsmp SDT_SYS386IGT, SEL_KPL, GSEL(GCODE_SEL, SEL_KPL)); 60731639Sfsmp 60834021Stegge /* install an inter-CPU IPI for interrupt forwarding */ 60934021Stegge setidt(XFORWARD_IRQ_OFFSET, Xforward_irq, 61034021Stegge SDT_SYS386IGT, SEL_KPL, GSEL(GCODE_SEL, SEL_KPL)); 61134021Stegge 61227005Sfsmp /* install an inter-CPU IPI for CPU stop/restart */ 61327005Sfsmp setidt(XCPUSTOP_OFFSET, Xcpustop, 61427005Sfsmp SDT_SYS386IGT, SEL_KPL, GSEL(GCODE_SEL, SEL_KPL)); 61527255Sfsmp 61627353Sfsmp#if defined(TEST_TEST1) 61727517Sfsmp /* install a "fake hardware INTerrupt" vector */ 61827353Sfsmp setidt(XTEST1_OFFSET, Xtest1, 61927255Sfsmp SDT_SYS386IGT, SEL_KPL, GSEL(GCODE_SEL, SEL_KPL)); 62027353Sfsmp#endif /** TEST_TEST1 */ 62127353Sfsmp 62225164Speter#endif /* APIC_IO */ 62325164Speter 62427634Sfsmp /* initialize all SMP locks */ 62527634Sfsmp init_locks(); 62627634Sfsmp 62765557Sjasone /* obtain the ap_boot_lock */ 62865557Sjasone s_lock(&ap_boot_lock); 62965557Sjasone 63025164Speter /* start each Application Processor */ 63125164Speter start_all_aps(boot_addr); 63225164Speter} 63325164Speter 63425164Speter 63525164Speter/* 63625164Speter * look for the MP spec signature 63725164Speter */ 63825164Speter 63925164Speter/* string defined by the Intel MP Spec as identifying the MP table */ 64025164Speter#define MP_SIG 0x5f504d5f /* _MP_ */ 64125164Speter#define NEXT(X) ((X) += 4) 64225164Speterstatic int 64325164Spetersearch_for_sig(u_int32_t target, int count) 64425164Speter{ 64525164Speter int x; 64625164Speter u_int32_t *addr = (u_int32_t *) (KERNBASE + target); 64725164Speter 64825164Speter for (x = 0; x < count; NEXT(x)) 64925164Speter if (addr[x] == MP_SIG) 65025164Speter /* make array index a byte index */ 65125164Speter return (target + (x * sizeof(u_int32_t))); 65225164Speter 65325164Speter return -1; 65425164Speter} 65525164Speter 65625164Speter 65725164Speterstatic basetable_entry basetable_entry_types[] = 65825164Speter{ 65925164Speter {0, 20, "Processor"}, 66025164Speter {1, 8, "Bus"}, 66125164Speter {2, 8, "I/O APIC"}, 66225164Speter {3, 8, "I/O INT"}, 66325164Speter {4, 8, "Local INT"} 66425164Speter}; 66525164Speter 66625164Spetertypedef struct BUSDATA { 66725164Speter u_char bus_id; 66825164Speter enum busTypes bus_type; 66925164Speter} bus_datum; 67025164Speter 67125164Spetertypedef struct INTDATA { 67225164Speter u_char int_type; 67325164Speter u_short int_flags; 67425164Speter u_char src_bus_id; 67525164Speter u_char src_bus_irq; 67625164Speter u_char dst_apic_id; 67725164Speter u_char dst_apic_int; 67838888Stegge u_char int_vector; 67925164Speter} io_int, local_int; 68025164Speter 68125164Spetertypedef struct BUSTYPENAME { 68225164Speter u_char type; 68325164Speter char name[7]; 68425164Speter} bus_type_name; 68525164Speter 68625164Speterstatic bus_type_name bus_type_table[] = 68725164Speter{ 68825164Speter {CBUS, "CBUS"}, 68925164Speter {CBUSII, "CBUSII"}, 69025164Speter {EISA, "EISA"}, 69155891Smdodd {MCA, "MCA"}, 69225164Speter {UNKNOWN_BUSTYPE, "---"}, 69325164Speter {ISA, "ISA"}, 69455891Smdodd {MCA, "MCA"}, 69525164Speter {UNKNOWN_BUSTYPE, "---"}, 69625164Speter {UNKNOWN_BUSTYPE, "---"}, 69725164Speter {UNKNOWN_BUSTYPE, "---"}, 69825164Speter {UNKNOWN_BUSTYPE, "---"}, 69925164Speter {UNKNOWN_BUSTYPE, "---"}, 70025164Speter {PCI, "PCI"}, 70125164Speter {UNKNOWN_BUSTYPE, "---"}, 70225164Speter {UNKNOWN_BUSTYPE, "---"}, 70325164Speter {UNKNOWN_BUSTYPE, "---"}, 70425164Speter {UNKNOWN_BUSTYPE, "---"}, 70525164Speter {XPRESS, "XPRESS"}, 70625164Speter {UNKNOWN_BUSTYPE, "---"} 70725164Speter}; 70825164Speter/* from MP spec v1.4, table 5-1 */ 70925164Speterstatic int default_data[7][5] = 71025164Speter{ 71125164Speter/* nbus, id0, type0, id1, type1 */ 71225164Speter {1, 0, ISA, 255, 255}, 71325164Speter {1, 0, EISA, 255, 255}, 71425164Speter {1, 0, EISA, 255, 255}, 71555891Smdodd {1, 0, MCA, 255, 255}, 71625164Speter {2, 0, ISA, 1, PCI}, 71725164Speter {2, 0, EISA, 1, PCI}, 71855891Smdodd {2, 0, MCA, 1, PCI} 71925164Speter}; 72025164Speter 72125164Speter 72225164Speter/* the bus data */ 72333181Seivindstatic bus_datum bus_data[NBUS]; 72425164Speter 72525164Speter/* the IO INT data, one entry per possible APIC INTerrupt */ 72633181Seivindstatic io_int io_apic_ints[NINTR]; 72725164Speter 72825164Speterstatic int nintrs; 72925164Speter 73026108Sfsmpstatic int processor_entry __P((proc_entry_ptr entry, int cpu)); 73126108Sfsmpstatic int bus_entry __P((bus_entry_ptr entry, int bus)); 73226108Sfsmpstatic int io_apic_entry __P((io_apic_entry_ptr entry, int apic)); 73326108Sfsmpstatic int int_entry __P((int_entry_ptr entry, int intr)); 73426108Sfsmpstatic int lookup_bus_type __P((char *name)); 73525164Speter 73625164Speter 73725164Speter/* 73826155Sfsmp * 1st pass on motherboard's Intel MP specification table. 73926155Sfsmp * 74026155Sfsmp * initializes: 74126155Sfsmp * mp_ncpus = 1 74226155Sfsmp * 74326155Sfsmp * determines: 74426155Sfsmp * cpu_apic_address (common to all CPUs) 74526155Sfsmp * io_apic_address[N] 74626155Sfsmp * mp_naps 74726155Sfsmp * mp_nbusses 74826155Sfsmp * mp_napics 74926155Sfsmp * nintrs 75025164Speter */ 75125164Speterstatic int 75226155Sfsmpmptable_pass1(void) 75325164Speter{ 75426108Sfsmp int x; 75526108Sfsmp mpcth_t cth; 75626108Sfsmp int totalSize; 75726108Sfsmp void* position; 75826108Sfsmp int count; 75926108Sfsmp int type; 76026108Sfsmp int mustpanic; 76125164Speter 76227005Sfsmp POSTCODE(MPTABLE_PASS1_POST); 76327005Sfsmp 76426108Sfsmp mustpanic = 0; 76526108Sfsmp 76626155Sfsmp /* clear various tables */ 76726155Sfsmp for (x = 0; x < NAPICID; ++x) { 76826155Sfsmp io_apic_address[x] = ~0; /* IO APIC address table */ 76926155Sfsmp } 77025164Speter 77126108Sfsmp /* init everything to empty */ 77226108Sfsmp mp_naps = 0; 77326108Sfsmp mp_nbusses = 0; 77426108Sfsmp mp_napics = 0; 77526108Sfsmp nintrs = 0; 77626108Sfsmp 77726108Sfsmp /* check for use of 'default' configuration */ 77828027Sfsmp if (MPFPS_MPFB1 != 0) { 77926108Sfsmp /* use default addresses */ 78026108Sfsmp cpu_apic_address = DEFAULT_APIC_BASE; 78126108Sfsmp io_apic_address[0] = DEFAULT_IO_APIC_BASE; 78226108Sfsmp 78326108Sfsmp /* fill in with defaults */ 78426882Sfsmp mp_naps = 2; /* includes BSP */ 78528027Sfsmp mp_nbusses = default_data[MPFPS_MPFB1 - 1][0]; 78626108Sfsmp#if defined(APIC_IO) 78726108Sfsmp mp_napics = 1; 78826108Sfsmp nintrs = 16; 78926108Sfsmp#endif /* APIC_IO */ 79026108Sfsmp } 79126108Sfsmp else { 79226155Sfsmp if ((cth = mpfps->pap) == 0) 79326108Sfsmp panic("MP Configuration Table Header MISSING!"); 79426108Sfsmp 79526108Sfsmp cpu_apic_address = (vm_offset_t) cth->apic_address; 79626108Sfsmp 79726108Sfsmp /* walk the table, recording info of interest */ 79826108Sfsmp totalSize = cth->base_table_length - sizeof(struct MPCTH); 79926108Sfsmp position = (u_char *) cth + sizeof(struct MPCTH); 80026108Sfsmp count = cth->entry_count; 80126108Sfsmp 80226108Sfsmp while (count--) { 80326108Sfsmp switch (type = *(u_char *) position) { 80426108Sfsmp case 0: /* processor_entry */ 80526108Sfsmp if (((proc_entry_ptr)position)->cpu_flags 80626108Sfsmp & PROCENTRY_FLAG_EN) 80726108Sfsmp ++mp_naps; 80826108Sfsmp break; 80926108Sfsmp case 1: /* bus_entry */ 81026108Sfsmp ++mp_nbusses; 81126108Sfsmp break; 81226108Sfsmp case 2: /* io_apic_entry */ 81326108Sfsmp if (((io_apic_entry_ptr)position)->apic_flags 81426108Sfsmp & IOAPICENTRY_FLAG_EN) 81526108Sfsmp io_apic_address[mp_napics++] = 81626108Sfsmp (vm_offset_t)((io_apic_entry_ptr) 81726108Sfsmp position)->apic_address; 81826108Sfsmp break; 81926108Sfsmp case 3: /* int_entry */ 82026108Sfsmp ++nintrs; 82126108Sfsmp break; 82226108Sfsmp case 4: /* int_entry */ 82326108Sfsmp break; 82426108Sfsmp default: 82526108Sfsmp panic("mpfps Base Table HOSED!"); 82626108Sfsmp /* NOTREACHED */ 82726108Sfsmp } 82826108Sfsmp 82926108Sfsmp totalSize -= basetable_entry_types[type].length; 83026108Sfsmp (u_char*)position += basetable_entry_types[type].length; 83126108Sfsmp } 83226108Sfsmp } 83326108Sfsmp 83426108Sfsmp /* qualify the numbers */ 83548145Smsmith if (mp_naps > NCPU) { 83626108Sfsmp printf("Warning: only using %d of %d available CPUs!\n", 83726108Sfsmp NCPU, mp_naps); 83848145Smsmith mp_naps = NCPU; 83928041Sfsmp } 84026108Sfsmp if (mp_nbusses > NBUS) { 84126108Sfsmp printf("found %d busses, increase NBUS\n", mp_nbusses); 84226108Sfsmp mustpanic = 1; 84326108Sfsmp } 84426108Sfsmp if (mp_napics > NAPIC) { 84526108Sfsmp printf("found %d apics, increase NAPIC\n", mp_napics); 84626108Sfsmp mustpanic = 1; 84726108Sfsmp } 84826108Sfsmp if (nintrs > NINTR) { 84926108Sfsmp printf("found %d intrs, increase NINTR\n", nintrs); 85026108Sfsmp mustpanic = 1; 85126108Sfsmp } 85226108Sfsmp 85326108Sfsmp /* 85426108Sfsmp * Count the BSP. 85526108Sfsmp * This is also used as a counter while starting the APs. 85626108Sfsmp */ 85726108Sfsmp mp_ncpus = 1; 85826108Sfsmp 85926108Sfsmp --mp_naps; /* subtract the BSP */ 86026108Sfsmp 86126108Sfsmp return mustpanic; 86226108Sfsmp} 86326108Sfsmp 86426108Sfsmp 86526108Sfsmp/* 86626155Sfsmp * 2nd pass on motherboard's Intel MP specification table. 86726155Sfsmp * 86826155Sfsmp * sets: 86926155Sfsmp * boot_cpu_id 87026155Sfsmp * ID_TO_IO(N), phy APIC ID to log CPU/IO table 87126155Sfsmp * CPU_TO_ID(N), logical CPU to APIC ID table 87226155Sfsmp * IO_TO_ID(N), logical IO to APIC ID table 87326155Sfsmp * bus_data[N] 87426155Sfsmp * io_apic_ints[N] 87526108Sfsmp */ 87626108Sfsmpstatic int 87726155Sfsmpmptable_pass2(void) 87826108Sfsmp{ 87926108Sfsmp int x; 88026108Sfsmp mpcth_t cth; 88126108Sfsmp int totalSize; 88226108Sfsmp void* position; 88326108Sfsmp int count; 88426108Sfsmp int type; 88526108Sfsmp int apic, bus, cpu, intr; 88626108Sfsmp 88727005Sfsmp POSTCODE(MPTABLE_PASS2_POST); 88827005Sfsmp 88926155Sfsmp /* clear various tables */ 89026155Sfsmp for (x = 0; x < NAPICID; ++x) { 89126155Sfsmp ID_TO_IO(x) = -1; /* phy APIC ID to log CPU/IO table */ 89226155Sfsmp CPU_TO_ID(x) = -1; /* logical CPU to APIC ID table */ 89326155Sfsmp IO_TO_ID(x) = -1; /* logical IO to APIC ID table */ 89426155Sfsmp } 89526155Sfsmp 89625164Speter /* clear bus data table */ 89725164Speter for (x = 0; x < NBUS; ++x) 89825164Speter bus_data[x].bus_id = 0xff; 89925164Speter 90025164Speter /* clear IO APIC INT table */ 90138888Stegge for (x = 0; x < NINTR; ++x) { 90225164Speter io_apic_ints[x].int_type = 0xff; 90338888Stegge io_apic_ints[x].int_vector = 0xff; 90438888Stegge } 90525164Speter 90625164Speter /* setup the cpu/apic mapping arrays */ 90725164Speter boot_cpu_id = -1; 90825164Speter 90925164Speter /* record whether PIC or virtual-wire mode */ 91026155Sfsmp picmode = (mpfps->mpfb2 & 0x80) ? 1 : 0; 91125164Speter 91225164Speter /* check for use of 'default' configuration */ 91328027Sfsmp if (MPFPS_MPFB1 != 0) 91428027Sfsmp return MPFPS_MPFB1; /* return default configuration type */ 91525164Speter 91626155Sfsmp if ((cth = mpfps->pap) == 0) 91726108Sfsmp panic("MP Configuration Table Header MISSING!"); 91825164Speter 91926108Sfsmp /* walk the table, recording info of interest */ 92025164Speter totalSize = cth->base_table_length - sizeof(struct MPCTH); 92125164Speter position = (u_char *) cth + sizeof(struct MPCTH); 92225164Speter count = cth->entry_count; 92326108Sfsmp apic = bus = intr = 0; 92426108Sfsmp cpu = 1; /* pre-count the BSP */ 92525164Speter 92625164Speter while (count--) { 92725164Speter switch (type = *(u_char *) position) { 92825164Speter case 0: 92926108Sfsmp if (processor_entry(position, cpu)) 93026108Sfsmp ++cpu; 93125164Speter break; 93225164Speter case 1: 93326108Sfsmp if (bus_entry(position, bus)) 93426108Sfsmp ++bus; 93525164Speter break; 93625164Speter case 2: 93726108Sfsmp if (io_apic_entry(position, apic)) 93826108Sfsmp ++apic; 93925164Speter break; 94025164Speter case 3: 94126108Sfsmp if (int_entry(position, intr)) 94226108Sfsmp ++intr; 94325164Speter break; 94425164Speter case 4: 94525164Speter /* int_entry(position); */ 94625164Speter break; 94725164Speter default: 94826108Sfsmp panic("mpfps Base Table HOSED!"); 94925164Speter /* NOTREACHED */ 95025164Speter } 95125164Speter 95225164Speter totalSize -= basetable_entry_types[type].length; 95325164Speter (u_char *) position += basetable_entry_types[type].length; 95425164Speter } 95525164Speter 95626101Sfsmp if (boot_cpu_id == -1) 95726108Sfsmp panic("NO BSP found!"); 95825164Speter 95925164Speter /* report fact that its NOT a default configuration */ 96025164Speter return 0; 96125164Speter} 96225164Speter 96325164Speter 96455420Steggevoid 96538888Steggeassign_apic_irq(int apic, int intpin, int irq) 96638888Stegge{ 96738888Stegge int x; 96838888Stegge 96938888Stegge if (int_to_apicintpin[irq].ioapic != -1) 97038888Stegge panic("assign_apic_irq: inconsistent table"); 97138888Stegge 97238888Stegge int_to_apicintpin[irq].ioapic = apic; 97338888Stegge int_to_apicintpin[irq].int_pin = intpin; 97438888Stegge int_to_apicintpin[irq].apic_address = ioapic[apic]; 97538888Stegge int_to_apicintpin[irq].redirindex = IOAPIC_REDTBL + 2 * intpin; 97638888Stegge 97738888Stegge for (x = 0; x < nintrs; x++) { 97838888Stegge if ((io_apic_ints[x].int_type == 0 || 97938888Stegge io_apic_ints[x].int_type == 3) && 98038888Stegge io_apic_ints[x].int_vector == 0xff && 98138888Stegge io_apic_ints[x].dst_apic_id == IO_TO_ID(apic) && 98238888Stegge io_apic_ints[x].dst_apic_int == intpin) 98338888Stegge io_apic_ints[x].int_vector = irq; 98438888Stegge } 98538888Stegge} 98638888Stegge 98755420Steggevoid 98855420Steggerevoke_apic_irq(int irq) 98955420Stegge{ 99055420Stegge int x; 99155420Stegge int oldapic; 99255420Stegge int oldintpin; 99355420Stegge 99455420Stegge if (int_to_apicintpin[irq].ioapic == -1) 99555420Stegge panic("assign_apic_irq: inconsistent table"); 99655420Stegge 99755420Stegge oldapic = int_to_apicintpin[irq].ioapic; 99855420Stegge oldintpin = int_to_apicintpin[irq].int_pin; 99955420Stegge 100055420Stegge int_to_apicintpin[irq].ioapic = -1; 100155420Stegge int_to_apicintpin[irq].int_pin = 0; 100255420Stegge int_to_apicintpin[irq].apic_address = NULL; 100355420Stegge int_to_apicintpin[irq].redirindex = 0; 100455420Stegge 100555420Stegge for (x = 0; x < nintrs; x++) { 100655420Stegge if ((io_apic_ints[x].int_type == 0 || 100755420Stegge io_apic_ints[x].int_type == 3) && 100855420Stegge io_apic_ints[x].int_vector == 0xff && 100955420Stegge io_apic_ints[x].dst_apic_id == IO_TO_ID(oldapic) && 101055420Stegge io_apic_ints[x].dst_apic_int == oldintpin) 101155420Stegge io_apic_ints[x].int_vector = 0xff; 101255420Stegge } 101355420Stegge} 101455420Stegge 101564290Stegge 101664290Stegge 101764290Steggestatic void 101864290Steggeswap_apic_id(int apic, int oldid, int newid) 101964290Stegge{ 102064290Stegge int x; 102164290Stegge int oapic; 102264290Stegge 102364290Stegge 102464290Stegge if (oldid == newid) 102564290Stegge return; /* Nothing to do */ 102664290Stegge 102764290Stegge printf("Changing APIC ID for IO APIC #%d from %d to %d in MP table\n", 102864290Stegge apic, oldid, newid); 102964290Stegge 103064290Stegge /* Swap physical APIC IDs in interrupt entries */ 103164290Stegge for (x = 0; x < nintrs; x++) { 103264290Stegge if (io_apic_ints[x].dst_apic_id == oldid) 103364290Stegge io_apic_ints[x].dst_apic_id = newid; 103464290Stegge else if (io_apic_ints[x].dst_apic_id == newid) 103564290Stegge io_apic_ints[x].dst_apic_id = oldid; 103664290Stegge } 103764290Stegge 103864290Stegge /* Swap physical APIC IDs in IO_TO_ID mappings */ 103964290Stegge for (oapic = 0; oapic < mp_napics; oapic++) 104064290Stegge if (IO_TO_ID(oapic) == newid) 104164290Stegge break; 104264290Stegge 104364290Stegge if (oapic < mp_napics) { 104464290Stegge printf("Changing APIC ID for IO APIC #%d from " 104564290Stegge "%d to %d in MP table\n", 104664290Stegge oapic, newid, oldid); 104764290Stegge IO_TO_ID(oapic) = oldid; 104864290Stegge } 104964290Stegge IO_TO_ID(apic) = newid; 105064290Stegge} 105164290Stegge 105264290Stegge 105364290Steggestatic void 105464290Steggefix_id_to_io_mapping(void) 105564290Stegge{ 105664290Stegge int x; 105764290Stegge 105864290Stegge for (x = 0; x < NAPICID; x++) 105964290Stegge ID_TO_IO(x) = -1; 106064290Stegge 106164290Stegge for (x = 0; x <= mp_naps; x++) 106264290Stegge if (CPU_TO_ID(x) < NAPICID) 106364290Stegge ID_TO_IO(CPU_TO_ID(x)) = x; 106464290Stegge 106564290Stegge for (x = 0; x < mp_napics; x++) 106664290Stegge if (IO_TO_ID(x) < NAPICID) 106764290Stegge ID_TO_IO(IO_TO_ID(x)) = x; 106864290Stegge} 106964290Stegge 107064290Stegge 107164290Steggestatic int 107264290Steggefirst_free_apic_id(void) 107364290Stegge{ 107464290Stegge int freeid, x; 107564290Stegge 107664290Stegge for (freeid = 0; freeid < NAPICID; freeid++) { 107764290Stegge for (x = 0; x <= mp_naps; x++) 107864290Stegge if (CPU_TO_ID(x) == freeid) 107964290Stegge break; 108064290Stegge if (x <= mp_naps) 108164290Stegge continue; 108264290Stegge for (x = 0; x < mp_napics; x++) 108364290Stegge if (IO_TO_ID(x) == freeid) 108464290Stegge break; 108564290Stegge if (x < mp_napics) 108664290Stegge continue; 108764290Stegge return freeid; 108864290Stegge } 108964290Stegge return freeid; 109064290Stegge} 109164290Stegge 109264290Stegge 109364290Steggestatic int 109464290Steggeio_apic_id_acceptable(int apic, int id) 109564290Stegge{ 109664290Stegge int cpu; /* Logical CPU number */ 109764290Stegge int oapic; /* Logical IO APIC number for other IO APIC */ 109864290Stegge 109964290Stegge if (id >= NAPICID) 110064290Stegge return 0; /* Out of range */ 110164290Stegge 110264290Stegge for (cpu = 0; cpu <= mp_naps; cpu++) 110364290Stegge if (CPU_TO_ID(cpu) == id) 110464290Stegge return 0; /* Conflict with CPU */ 110564290Stegge 110664290Stegge for (oapic = 0; oapic < mp_napics && oapic < apic; oapic++) 110764290Stegge if (IO_TO_ID(oapic) == id) 110864290Stegge return 0; /* Conflict with other APIC */ 110964290Stegge 111064290Stegge return 1; /* ID is acceptable for IO APIC */ 111164290Stegge} 111264290Stegge 111364290Stegge 111425164Speter/* 111525164Speter * parse an Intel MP specification table 111625164Speter */ 111725164Speterstatic void 111825164Speterfix_mp_table(void) 111925164Speter{ 112025292Sfsmp int x; 112125292Sfsmp int id; 112242543Seivind int bus_0 = 0; /* Stop GCC warning */ 112342543Seivind int bus_pci = 0; /* Stop GCC warning */ 112425292Sfsmp int num_pci_bus; 112564290Stegge int apic; /* IO APIC unit number */ 112664290Stegge int freeid; /* Free physical APIC ID */ 112764290Stegge int physid; /* Current physical IO APIC ID */ 112825164Speter 112925164Speter /* 113025164Speter * Fix mis-numbering of the PCI bus and its INT entries if the BIOS 113125164Speter * did it wrong. The MP spec says that when more than 1 PCI bus 113225164Speter * exists the BIOS must begin with bus entries for the PCI bus and use 113325164Speter * actual PCI bus numbering. This implies that when only 1 PCI bus 113425164Speter * exists the BIOS can choose to ignore this ordering, and indeed many 113525164Speter * MP motherboards do ignore it. This causes a problem when the PCI 113625164Speter * sub-system makes requests of the MP sub-system based on PCI bus 113725164Speter * numbers. So here we look for the situation and renumber the 113825164Speter * busses and associated INTs in an effort to "make it right". 113925164Speter */ 114025164Speter 114125292Sfsmp /* find bus 0, PCI bus, count the number of PCI busses */ 114225164Speter for (num_pci_bus = 0, x = 0; x < mp_nbusses; ++x) { 114325292Sfsmp if (bus_data[x].bus_id == 0) { 114425292Sfsmp bus_0 = x; 114525292Sfsmp } 114625292Sfsmp if (bus_data[x].bus_type == PCI) { 114725164Speter ++num_pci_bus; 114825292Sfsmp bus_pci = x; 114925292Sfsmp } 115025164Speter } 115125292Sfsmp /* 115225292Sfsmp * bus_0 == slot of bus with ID of 0 115325292Sfsmp * bus_pci == slot of last PCI bus encountered 115425292Sfsmp */ 115525164Speter 115625164Speter /* check the 1 PCI bus case for sanity */ 115764494Stegge /* if it is number 0 all is well */ 115864494Stegge if (num_pci_bus == 1 && 115964494Stegge bus_data[bus_pci].bus_id != 0) { 116064494Stegge 116125164Speter /* mis-numbered, swap with whichever bus uses slot 0 */ 116225164Speter 116325292Sfsmp /* swap the bus entry types */ 116425292Sfsmp bus_data[bus_pci].bus_type = bus_data[bus_0].bus_type; 116525292Sfsmp bus_data[bus_0].bus_type = PCI; 116625164Speter 116725164Speter /* swap each relavant INTerrupt entry */ 116825292Sfsmp id = bus_data[bus_pci].bus_id; 116925292Sfsmp for (x = 0; x < nintrs; ++x) { 117025292Sfsmp if (io_apic_ints[x].src_bus_id == id) { 117125292Sfsmp io_apic_ints[x].src_bus_id = 0; 117225292Sfsmp } 117325292Sfsmp else if (io_apic_ints[x].src_bus_id == 0) { 117425292Sfsmp io_apic_ints[x].src_bus_id = id; 117525292Sfsmp } 117625164Speter } 117725164Speter } 117864290Stegge 117964290Stegge /* Assign IO APIC IDs. 118064290Stegge * 118164290Stegge * First try the existing ID. If a conflict is detected, try 118264290Stegge * the ID in the MP table. If a conflict is still detected, find 118364290Stegge * a free id. 118464290Stegge * 118564290Stegge * We cannot use the ID_TO_IO table before all conflicts has been 118664290Stegge * resolved and the table has been corrected. 118764290Stegge */ 118864290Stegge for (apic = 0; apic < mp_napics; ++apic) { /* For all IO APICs */ 118964290Stegge 119064290Stegge /* First try to use the value set by the BIOS */ 119164290Stegge physid = io_apic_get_id(apic); 119264290Stegge if (io_apic_id_acceptable(apic, physid)) { 119364290Stegge if (IO_TO_ID(apic) != physid) 119464290Stegge swap_apic_id(apic, IO_TO_ID(apic), physid); 119564290Stegge continue; 119664290Stegge } 119764290Stegge 119864290Stegge /* Then check if the value in the MP table is acceptable */ 119964290Stegge if (io_apic_id_acceptable(apic, IO_TO_ID(apic))) 120064290Stegge continue; 120164290Stegge 120264290Stegge /* Last resort, find a free APIC ID and use it */ 120364290Stegge freeid = first_free_apic_id(); 120464290Stegge if (freeid >= NAPICID) 120564290Stegge panic("No free physical APIC IDs found"); 120664290Stegge 120764290Stegge if (io_apic_id_acceptable(apic, freeid)) { 120864290Stegge swap_apic_id(apic, IO_TO_ID(apic), freeid); 120964290Stegge continue; 121064290Stegge } 121164290Stegge panic("Free physical APIC ID not usable"); 121264290Stegge } 121364290Stegge fix_id_to_io_mapping(); 121425164Speter} 121525164Speter 121625164Speter 121755420Stegge/* Assign low level interrupt handlers */ 121838888Steggestatic void 121938888Steggesetup_apic_irq_mapping(void) 122038888Stegge{ 122138888Stegge int x; 122238888Stegge int int_vector; 122338888Stegge 122455420Stegge /* Clear array */ 122538888Stegge for (x = 0; x < APIC_INTMAPSIZE; x++) { 122638888Stegge int_to_apicintpin[x].ioapic = -1; 122738888Stegge int_to_apicintpin[x].int_pin = 0; 122838888Stegge int_to_apicintpin[x].apic_address = NULL; 122938888Stegge int_to_apicintpin[x].redirindex = 0; 123038888Stegge } 123155420Stegge 123255420Stegge /* First assign ISA/EISA interrupts */ 123338888Stegge for (x = 0; x < nintrs; x++) { 123455420Stegge int_vector = io_apic_ints[x].src_bus_irq; 123555420Stegge if (int_vector < APIC_INTMAPSIZE && 123655420Stegge io_apic_ints[x].int_vector == 0xff && 123755420Stegge int_to_apicintpin[int_vector].ioapic == -1 && 123855420Stegge (apic_int_is_bus_type(x, ISA) || 123955420Stegge apic_int_is_bus_type(x, EISA)) && 124055420Stegge io_apic_ints[x].int_type == 0) { 124155420Stegge assign_apic_irq(ID_TO_IO(io_apic_ints[x].dst_apic_id), 124255420Stegge io_apic_ints[x].dst_apic_int, 124355420Stegge int_vector); 124455420Stegge } 124555420Stegge } 124655420Stegge 124755420Stegge /* Assign interrupts on first 24 intpins on IOAPIC #0 */ 124855420Stegge for (x = 0; x < nintrs; x++) { 124955420Stegge int_vector = io_apic_ints[x].dst_apic_int; 125055420Stegge if (int_vector < APIC_INTMAPSIZE && 125138888Stegge io_apic_ints[x].dst_apic_id == IO_TO_ID(0) && 125238888Stegge io_apic_ints[x].int_vector == 0xff && 125355420Stegge int_to_apicintpin[int_vector].ioapic == -1 && 125438888Stegge (io_apic_ints[x].int_type == 0 || 125538888Stegge io_apic_ints[x].int_type == 3)) { 125655420Stegge assign_apic_irq(0, 125738888Stegge io_apic_ints[x].dst_apic_int, 125855420Stegge int_vector); 125938888Stegge } 126038888Stegge } 126155420Stegge /* 126255420Stegge * Assign interrupts for remaining intpins. 126355420Stegge * Skip IOAPIC #0 intpin 0 if the type is ExtInt, since this indicates 126455420Stegge * that an entry for ISA/EISA irq 0 exist, and a fallback to mixed mode 126555420Stegge * due to 8254 interrupts not being delivered can reuse that low level 126655420Stegge * interrupt handler. 126755420Stegge */ 126838888Stegge int_vector = 0; 126938888Stegge while (int_vector < APIC_INTMAPSIZE && 127038888Stegge int_to_apicintpin[int_vector].ioapic != -1) 127138888Stegge int_vector++; 127238888Stegge for (x = 0; x < nintrs && int_vector < APIC_INTMAPSIZE; x++) { 127338888Stegge if ((io_apic_ints[x].int_type == 0 || 127455420Stegge (io_apic_ints[x].int_type == 3 && 127555420Stegge (io_apic_ints[x].dst_apic_id != IO_TO_ID(0) || 127655420Stegge io_apic_ints[x].dst_apic_int != 0))) && 127738888Stegge io_apic_ints[x].int_vector == 0xff) { 127838888Stegge assign_apic_irq(ID_TO_IO(io_apic_ints[x].dst_apic_id), 127938888Stegge io_apic_ints[x].dst_apic_int, 128038888Stegge int_vector); 128138888Stegge int_vector++; 128238888Stegge while (int_vector < APIC_INTMAPSIZE && 128338888Stegge int_to_apicintpin[int_vector].ioapic != -1) 128438888Stegge int_vector++; 128538888Stegge } 128638888Stegge } 128738888Stegge} 128838888Stegge 128938888Stegge 129026108Sfsmpstatic int 129126108Sfsmpprocessor_entry(proc_entry_ptr entry, int cpu) 129225164Speter{ 129325164Speter /* check for usability */ 129455540Sluoqi if (!(entry->cpu_flags & PROCENTRY_FLAG_EN)) 129526108Sfsmp return 0; 129625164Speter 129764290Stegge if(entry->apic_id >= NAPICID) 129864290Stegge panic("CPU APIC ID out of range (0..%d)", NAPICID - 1); 129925164Speter /* check for BSP flag */ 130025164Speter if (entry->cpu_flags & PROCENTRY_FLAG_BP) { 130125164Speter boot_cpu_id = entry->apic_id; 130226108Sfsmp CPU_TO_ID(0) = entry->apic_id; 130326108Sfsmp ID_TO_CPU(entry->apic_id) = 0; 130426108Sfsmp return 0; /* its already been counted */ 130525164Speter } 130625164Speter 130726108Sfsmp /* add another AP to list, if less than max number of CPUs */ 130855540Sluoqi else if (cpu < NCPU) { 130926108Sfsmp CPU_TO_ID(cpu) = entry->apic_id; 131026108Sfsmp ID_TO_CPU(entry->apic_id) = cpu; 131126108Sfsmp return 1; 131226108Sfsmp } 131355540Sluoqi 131455540Sluoqi return 0; 131525164Speter} 131625164Speter 131725164Speter 131826108Sfsmpstatic int 131926108Sfsmpbus_entry(bus_entry_ptr entry, int bus) 132025164Speter{ 132126108Sfsmp int x; 132226108Sfsmp char c, name[8]; 132325164Speter 132425164Speter /* encode the name into an index */ 132526108Sfsmp for (x = 0; x < 6; ++x) { 132626108Sfsmp if ((c = entry->bus_type[x]) == ' ') 132725164Speter break; 132826108Sfsmp name[x] = c; 132925164Speter } 133026108Sfsmp name[x] = '\0'; 133125164Speter 133226108Sfsmp if ((x = lookup_bus_type(name)) == UNKNOWN_BUSTYPE) 133326108Sfsmp panic("unknown bus type: '%s'", name); 133425164Speter 133526108Sfsmp bus_data[bus].bus_id = entry->bus_id; 133626108Sfsmp bus_data[bus].bus_type = x; 133726108Sfsmp 133826108Sfsmp return 1; 133925164Speter} 134025164Speter 134125164Speter 134226108Sfsmpstatic int 134326108Sfsmpio_apic_entry(io_apic_entry_ptr entry, int apic) 134425164Speter{ 134525164Speter if (!(entry->apic_flags & IOAPICENTRY_FLAG_EN)) 134626108Sfsmp return 0; 134725164Speter 134826108Sfsmp IO_TO_ID(apic) = entry->apic_id; 134964290Stegge if (entry->apic_id < NAPICID) 135064290Stegge ID_TO_IO(entry->apic_id) = apic; 135125164Speter 135226108Sfsmp return 1; 135325164Speter} 135425164Speter 135525164Speter 135625164Speterstatic int 135725164Speterlookup_bus_type(char *name) 135825164Speter{ 135925164Speter int x; 136025164Speter 136125164Speter for (x = 0; x < MAX_BUSTYPE; ++x) 136225164Speter if (strcmp(bus_type_table[x].name, name) == 0) 136325164Speter return bus_type_table[x].type; 136425164Speter 136525164Speter return UNKNOWN_BUSTYPE; 136625164Speter} 136725164Speter 136825164Speter 136926108Sfsmpstatic int 137026108Sfsmpint_entry(int_entry_ptr entry, int intr) 137125164Speter{ 137241367Stegge int apic; 137341367Stegge 137426108Sfsmp io_apic_ints[intr].int_type = entry->int_type; 137526108Sfsmp io_apic_ints[intr].int_flags = entry->int_flags; 137626108Sfsmp io_apic_ints[intr].src_bus_id = entry->src_bus_id; 137726108Sfsmp io_apic_ints[intr].src_bus_irq = entry->src_bus_irq; 137841367Stegge if (entry->dst_apic_id == 255) { 137941367Stegge /* This signal goes to all IO APICS. Select an IO APIC 138041367Stegge with sufficient number of interrupt pins */ 138141367Stegge for (apic = 0; apic < mp_napics; apic++) 138241367Stegge if (((io_apic_read(apic, IOAPIC_VER) & 138341367Stegge IOART_VER_MAXREDIR) >> MAXREDIRSHIFT) >= 138441367Stegge entry->dst_apic_int) 138541367Stegge break; 138641367Stegge if (apic < mp_napics) 138741367Stegge io_apic_ints[intr].dst_apic_id = IO_TO_ID(apic); 138841367Stegge else 138941367Stegge io_apic_ints[intr].dst_apic_id = entry->dst_apic_id; 139041367Stegge } else 139141367Stegge io_apic_ints[intr].dst_apic_id = entry->dst_apic_id; 139226108Sfsmp io_apic_ints[intr].dst_apic_int = entry->dst_apic_int; 139325164Speter 139426108Sfsmp return 1; 139525164Speter} 139625164Speter 139725164Speter 139825164Speterstatic int 139925164Speterapic_int_is_bus_type(int intr, int bus_type) 140025164Speter{ 140125164Speter int bus; 140225164Speter 140325164Speter for (bus = 0; bus < mp_nbusses; ++bus) 140425164Speter if ((bus_data[bus].bus_id == io_apic_ints[intr].src_bus_id) 140525164Speter && ((int) bus_data[bus].bus_type == bus_type)) 140625164Speter return 1; 140725164Speter 140825164Speter return 0; 140925164Speter} 141025164Speter 141125164Speter 141225164Speter/* 141326950Sfsmp * Given a traditional ISA INT mask, return an APIC mask. 141425164Speter */ 141525499Sfsmpu_int 141626950Sfsmpisa_apic_mask(u_int isa_mask) 141725419Sfsmp{ 141826950Sfsmp int isa_irq; 141926950Sfsmp int apic_pin; 142025419Sfsmp 142127255Sfsmp#if defined(SKIP_IRQ15_REDIRECT) 142227255Sfsmp if (isa_mask == (1 << 15)) { 142327255Sfsmp printf("skipping ISA IRQ15 redirect\n"); 142427255Sfsmp return isa_mask; 142527255Sfsmp } 142627255Sfsmp#endif /* SKIP_IRQ15_REDIRECT */ 142727255Sfsmp 142826950Sfsmp isa_irq = ffs(isa_mask); /* find its bit position */ 142926950Sfsmp if (isa_irq == 0) /* doesn't exist */ 143025419Sfsmp return 0; 143126950Sfsmp --isa_irq; /* make it zero based */ 143225419Sfsmp 143338888Stegge apic_pin = isa_apic_irq(isa_irq); /* look for APIC connection */ 143426950Sfsmp if (apic_pin == -1) 143526950Sfsmp return 0; 143625419Sfsmp 143726950Sfsmp return (1 << apic_pin); /* convert pin# to a mask */ 143825419Sfsmp} 143925419Sfsmp 144025419Sfsmp 144125419Sfsmp/* 144226950Sfsmp * Determine which APIC pin an ISA/EISA INT is attached to. 144325164Speter */ 144426950Sfsmp#define INTTYPE(I) (io_apic_ints[(I)].int_type) 144526950Sfsmp#define INTPIN(I) (io_apic_ints[(I)].dst_apic_int) 144638888Stegge#define INTIRQ(I) (io_apic_ints[(I)].int_vector) 144738888Stegge#define INTAPIC(I) (ID_TO_IO(io_apic_ints[(I)].dst_apic_id)) 144826950Sfsmp 144925164Speter#define SRCBUSIRQ(I) (io_apic_ints[(I)].src_bus_irq) 145025164Speterint 145138888Steggeisa_apic_irq(int isa_irq) 145225164Speter{ 145325164Speter int intr; 145425164Speter 145526950Sfsmp for (intr = 0; intr < nintrs; ++intr) { /* check each record */ 145626950Sfsmp if (INTTYPE(intr) == 0) { /* standard INT */ 145726950Sfsmp if (SRCBUSIRQ(intr) == isa_irq) { 145826950Sfsmp if (apic_int_is_bus_type(intr, ISA) || 145926950Sfsmp apic_int_is_bus_type(intr, EISA)) 146038888Stegge return INTIRQ(intr); /* found */ 146126950Sfsmp } 146226950Sfsmp } 146326950Sfsmp } 146426950Sfsmp return -1; /* NOT found */ 146525164Speter} 146625164Speter 146725164Speter 146825164Speter/* 146926950Sfsmp * Determine which APIC pin a PCI INT is attached to. 147025164Speter */ 147125164Speter#define SRCBUSID(I) (io_apic_ints[(I)].src_bus_id) 147225164Speter#define SRCBUSDEVICE(I) ((io_apic_ints[(I)].src_bus_irq >> 2) & 0x1f) 147325164Speter#define SRCBUSLINE(I) (io_apic_ints[(I)].src_bus_irq & 0x03) 147425164Speterint 147538888Steggepci_apic_irq(int pciBus, int pciDevice, int pciInt) 147625164Speter{ 147725164Speter int intr; 147825164Speter 147926950Sfsmp --pciInt; /* zero based */ 148025164Speter 148126950Sfsmp for (intr = 0; intr < nintrs; ++intr) /* check each record */ 148226950Sfsmp if ((INTTYPE(intr) == 0) /* standard INT */ 148325164Speter && (SRCBUSID(intr) == pciBus) 148425164Speter && (SRCBUSDEVICE(intr) == pciDevice) 148525164Speter && (SRCBUSLINE(intr) == pciInt)) /* a candidate IRQ */ 148626950Sfsmp if (apic_int_is_bus_type(intr, PCI)) 148738888Stegge return INTIRQ(intr); /* exact match */ 148825164Speter 148926950Sfsmp return -1; /* NOT found */ 149025164Speter} 149134990Stegge 149234990Steggeint 149338888Steggenext_apic_irq(int irq) 149434990Stegge{ 149534990Stegge int intr, ointr; 149634990Stegge int bus, bustype; 149734990Stegge 149834990Stegge bus = 0; 149934990Stegge bustype = 0; 150034990Stegge for (intr = 0; intr < nintrs; intr++) { 150138888Stegge if (INTIRQ(intr) != irq || INTTYPE(intr) != 0) 150234990Stegge continue; 150334990Stegge bus = SRCBUSID(intr); 150434990Stegge bustype = apic_bus_type(bus); 150534990Stegge if (bustype != ISA && 150634990Stegge bustype != EISA && 150734990Stegge bustype != PCI) 150834990Stegge continue; 150934990Stegge break; 151034990Stegge } 151134990Stegge if (intr >= nintrs) { 151234990Stegge return -1; 151334990Stegge } 151434990Stegge for (ointr = intr + 1; ointr < nintrs; ointr++) { 151534990Stegge if (INTTYPE(ointr) != 0) 151634990Stegge continue; 151734990Stegge if (bus != SRCBUSID(ointr)) 151834990Stegge continue; 151934990Stegge if (bustype == PCI) { 152034990Stegge if (SRCBUSDEVICE(intr) != SRCBUSDEVICE(ointr)) 152134990Stegge continue; 152234990Stegge if (SRCBUSLINE(intr) != SRCBUSLINE(ointr)) 152334990Stegge continue; 152434990Stegge } 152534990Stegge if (bustype == ISA || bustype == EISA) { 152634990Stegge if (SRCBUSIRQ(intr) != SRCBUSIRQ(ointr)) 152734990Stegge continue; 152834990Stegge } 152934990Stegge if (INTPIN(intr) == INTPIN(ointr)) 153034990Stegge continue; 153134990Stegge break; 153234990Stegge } 153334990Stegge if (ointr >= nintrs) { 153434990Stegge return -1; 153534990Stegge } 153638888Stegge return INTIRQ(ointr); 153734990Stegge} 153825164Speter#undef SRCBUSLINE 153925164Speter#undef SRCBUSDEVICE 154025164Speter#undef SRCBUSID 154134990Stegge#undef SRCBUSIRQ 154225164Speter 154325164Speter#undef INTPIN 154438888Stegge#undef INTIRQ 154538888Stegge#undef INTAPIC 154625164Speter#undef INTTYPE 154725164Speter 154825164Speter 154925499Sfsmp/* 155026950Sfsmp * Reprogram the MB chipset to NOT redirect an ISA INTerrupt. 155126950Sfsmp * 155226950Sfsmp * XXX FIXME: 155326950Sfsmp * Exactly what this means is unclear at this point. It is a solution 155426950Sfsmp * for motherboards that redirect the MBIRQ0 pin. Generically a motherboard 155526950Sfsmp * could route any of the ISA INTs to upper (>15) IRQ values. But most would 155626950Sfsmp * NOT be redirected via MBIRQ0, thus "undirect()ing" them would NOT be an 155726950Sfsmp * option. 155825499Sfsmp */ 155925164Speterint 156026950Sfsmpundirect_isa_irq(int rirq) 156125164Speter{ 156225164Speter#if defined(READY) 156342880Sjkh if (bootverbose) 156442880Sjkh printf("Freeing redirected ISA irq %d.\n", rirq); 156525164Speter /** FIXME: tickle the MB redirector chip */ 156625164Speter return ???; 156725164Speter#else 156842880Sjkh if (bootverbose) 156942880Sjkh printf("Freeing (NOT implemented) redirected ISA irq %d.\n", rirq); 157025164Speter return 0; 157125499Sfsmp#endif /* READY */ 157225164Speter} 157325164Speter 157425164Speter 157525164Speter/* 157626950Sfsmp * Reprogram the MB chipset to NOT redirect a PCI INTerrupt 157725499Sfsmp */ 157825499Sfsmpint 157926950Sfsmpundirect_pci_irq(int rirq) 158025499Sfsmp{ 158125499Sfsmp#if defined(READY) 158226950Sfsmp if (bootverbose) 158326950Sfsmp printf("Freeing redirected PCI irq %d.\n", rirq); 158426950Sfsmp 158525499Sfsmp /** FIXME: tickle the MB redirector chip */ 158625499Sfsmp return ???; 158725499Sfsmp#else 158826950Sfsmp if (bootverbose) 158926950Sfsmp printf("Freeing (NOT implemented) redirected PCI irq %d.\n", 159026950Sfsmp rirq); 159125499Sfsmp return 0; 159225499Sfsmp#endif /* READY */ 159325499Sfsmp} 159425499Sfsmp 159525499Sfsmp 159625499Sfsmp/* 159725164Speter * given a bus ID, return: 159825164Speter * the bus type if found 159925164Speter * -1 if NOT found 160025164Speter */ 160125164Speterint 160225164Speterapic_bus_type(int id) 160325164Speter{ 160425164Speter int x; 160525164Speter 160625164Speter for (x = 0; x < mp_nbusses; ++x) 160725164Speter if (bus_data[x].bus_id == id) 160825164Speter return bus_data[x].bus_type; 160925164Speter 161025164Speter return -1; 161125164Speter} 161225164Speter 161325164Speter 161425164Speter/* 161525164Speter * given a LOGICAL APIC# and pin#, return: 161625164Speter * the associated src bus ID if found 161725164Speter * -1 if NOT found 161825164Speter */ 161925164Speterint 162025164Speterapic_src_bus_id(int apic, int pin) 162125164Speter{ 162225164Speter int x; 162325164Speter 162425164Speter /* search each of the possible INTerrupt sources */ 162525164Speter for (x = 0; x < nintrs; ++x) 162625164Speter if ((apic == ID_TO_IO(io_apic_ints[x].dst_apic_id)) && 162725164Speter (pin == io_apic_ints[x].dst_apic_int)) 162825164Speter return (io_apic_ints[x].src_bus_id); 162925164Speter 163025164Speter return -1; /* NOT found */ 163125164Speter} 163225164Speter 163325164Speter 163425164Speter/* 163525164Speter * given a LOGICAL APIC# and pin#, return: 163625164Speter * the associated src bus IRQ if found 163725164Speter * -1 if NOT found 163825164Speter */ 163925164Speterint 164025164Speterapic_src_bus_irq(int apic, int pin) 164125164Speter{ 164225164Speter int x; 164325164Speter 164425164Speter for (x = 0; x < nintrs; x++) 164525164Speter if ((apic == ID_TO_IO(io_apic_ints[x].dst_apic_id)) && 164625164Speter (pin == io_apic_ints[x].dst_apic_int)) 164725164Speter return (io_apic_ints[x].src_bus_irq); 164825164Speter 164925164Speter return -1; /* NOT found */ 165025164Speter} 165125164Speter 165225164Speter 165325164Speter/* 165425164Speter * given a LOGICAL APIC# and pin#, return: 165525164Speter * the associated INTerrupt type if found 165625164Speter * -1 if NOT found 165725164Speter */ 165825164Speterint 165925164Speterapic_int_type(int apic, int pin) 166025164Speter{ 166125164Speter int x; 166225164Speter 166325164Speter /* search each of the possible INTerrupt sources */ 166425164Speter for (x = 0; x < nintrs; ++x) 166525164Speter if ((apic == ID_TO_IO(io_apic_ints[x].dst_apic_id)) && 166625164Speter (pin == io_apic_ints[x].dst_apic_int)) 166725164Speter return (io_apic_ints[x].int_type); 166825164Speter 166925164Speter return -1; /* NOT found */ 167025164Speter} 167125164Speter 167238888Steggeint 167338888Steggeapic_irq(int apic, int pin) 167438888Stegge{ 167538888Stegge int x; 167638888Stegge int res; 167725164Speter 167838888Stegge for (x = 0; x < nintrs; ++x) 167938888Stegge if ((apic == ID_TO_IO(io_apic_ints[x].dst_apic_id)) && 168038888Stegge (pin == io_apic_ints[x].dst_apic_int)) { 168138888Stegge res = io_apic_ints[x].int_vector; 168238888Stegge if (res == 0xff) 168338888Stegge return -1; 168438888Stegge if (apic != int_to_apicintpin[res].ioapic) 168538888Stegge panic("apic_irq: inconsistent table"); 168638888Stegge if (pin != int_to_apicintpin[res].int_pin) 168738888Stegge panic("apic_irq inconsistent table (2)"); 168838888Stegge return res; 168938888Stegge } 169038888Stegge return -1; 169138888Stegge} 169238888Stegge 169338888Stegge 169425164Speter/* 169525164Speter * given a LOGICAL APIC# and pin#, return: 169625164Speter * the associated trigger mode if found 169725164Speter * -1 if NOT found 169825164Speter */ 169925164Speterint 170025164Speterapic_trigger(int apic, int pin) 170125164Speter{ 170225164Speter int x; 170325164Speter 170425164Speter /* search each of the possible INTerrupt sources */ 170525164Speter for (x = 0; x < nintrs; ++x) 170625164Speter if ((apic == ID_TO_IO(io_apic_ints[x].dst_apic_id)) && 170725164Speter (pin == io_apic_ints[x].dst_apic_int)) 170825164Speter return ((io_apic_ints[x].int_flags >> 2) & 0x03); 170925164Speter 171025164Speter return -1; /* NOT found */ 171125164Speter} 171225164Speter 171325164Speter 171425164Speter/* 171525164Speter * given a LOGICAL APIC# and pin#, return: 171625164Speter * the associated 'active' level if found 171725164Speter * -1 if NOT found 171825164Speter */ 171925164Speterint 172025164Speterapic_polarity(int apic, int pin) 172125164Speter{ 172225164Speter int x; 172325164Speter 172425164Speter /* search each of the possible INTerrupt sources */ 172525164Speter for (x = 0; x < nintrs; ++x) 172625164Speter if ((apic == ID_TO_IO(io_apic_ints[x].dst_apic_id)) && 172725164Speter (pin == io_apic_ints[x].dst_apic_int)) 172825164Speter return (io_apic_ints[x].int_flags & 0x03); 172925164Speter 173025164Speter return -1; /* NOT found */ 173125164Speter} 173225164Speter 173325164Speter 173425164Speter/* 173525164Speter * set data according to MP defaults 173625164Speter * FIXME: probably not complete yet... 173725164Speter */ 173825164Speterstatic void 173925164Speterdefault_mp_table(int type) 174025164Speter{ 174125164Speter int ap_cpu_id; 174225164Speter#if defined(APIC_IO) 174325164Speter int io_apic_id; 174425164Speter int pin; 174525164Speter#endif /* APIC_IO */ 174625164Speter 174725164Speter#if 0 174825164Speter printf(" MP default config type: %d\n", type); 174925164Speter switch (type) { 175025164Speter case 1: 175125164Speter printf(" bus: ISA, APIC: 82489DX\n"); 175225164Speter break; 175325164Speter case 2: 175425164Speter printf(" bus: EISA, APIC: 82489DX\n"); 175525164Speter break; 175625164Speter case 3: 175725164Speter printf(" bus: EISA, APIC: 82489DX\n"); 175825164Speter break; 175925164Speter case 4: 176025164Speter printf(" bus: MCA, APIC: 82489DX\n"); 176125164Speter break; 176225164Speter case 5: 176325164Speter printf(" bus: ISA+PCI, APIC: Integrated\n"); 176425164Speter break; 176525164Speter case 6: 176625164Speter printf(" bus: EISA+PCI, APIC: Integrated\n"); 176725164Speter break; 176825164Speter case 7: 176925164Speter printf(" bus: MCA+PCI, APIC: Integrated\n"); 177025164Speter break; 177125164Speter default: 177225164Speter printf(" future type\n"); 177325164Speter break; 177425164Speter /* NOTREACHED */ 177525164Speter } 177625164Speter#endif /* 0 */ 177725164Speter 177826812Speter boot_cpu_id = (lapic.id & APIC_ID_MASK) >> 24; 177925164Speter ap_cpu_id = (boot_cpu_id == 0) ? 1 : 0; 178025164Speter 178125164Speter /* BSP */ 178225164Speter CPU_TO_ID(0) = boot_cpu_id; 178325164Speter ID_TO_CPU(boot_cpu_id) = 0; 178425164Speter 178525164Speter /* one and only AP */ 178625164Speter CPU_TO_ID(1) = ap_cpu_id; 178725164Speter ID_TO_CPU(ap_cpu_id) = 1; 178825164Speter 178926108Sfsmp#if defined(APIC_IO) 179025164Speter /* one and only IO APIC */ 179125164Speter io_apic_id = (io_apic_read(0, IOAPIC_ID) & APIC_ID_MASK) >> 24; 179225164Speter 179325164Speter /* 179425164Speter * sanity check, refer to MP spec section 3.6.6, last paragraph 179525164Speter * necessary as some hardware isn't properly setting up the IO APIC 179625164Speter */ 179725164Speter#if defined(REALLY_ANAL_IOAPICID_VALUE) 179825164Speter if (io_apic_id != 2) { 179925164Speter#else 180025164Speter if ((io_apic_id == 0) || (io_apic_id == 1) || (io_apic_id == 15)) { 180125164Speter#endif /* REALLY_ANAL_IOAPICID_VALUE */ 180261136Smsmith io_apic_set_id(0, 2); 180325164Speter io_apic_id = 2; 180425164Speter } 180525164Speter IO_TO_ID(0) = io_apic_id; 180625164Speter ID_TO_IO(io_apic_id) = 0; 180725164Speter#endif /* APIC_IO */ 180825164Speter 180925164Speter /* fill out bus entries */ 181025164Speter switch (type) { 181125164Speter case 1: 181225164Speter case 2: 181325164Speter case 3: 181455891Smdodd case 4: 181525164Speter case 5: 181625164Speter case 6: 181755891Smdodd case 7: 181825164Speter bus_data[0].bus_id = default_data[type - 1][1]; 181925164Speter bus_data[0].bus_type = default_data[type - 1][2]; 182025164Speter bus_data[1].bus_id = default_data[type - 1][3]; 182125164Speter bus_data[1].bus_type = default_data[type - 1][4]; 182225164Speter break; 182325164Speter 182425164Speter /* case 4: case 7: MCA NOT supported */ 182525164Speter default: /* illegal/reserved */ 182626108Sfsmp panic("BAD default MP config: %d", type); 182726101Sfsmp /* NOTREACHED */ 182825164Speter } 182925164Speter 183025164Speter#if defined(APIC_IO) 183125164Speter /* general cases from MP v1.4, table 5-2 */ 183225164Speter for (pin = 0; pin < 16; ++pin) { 183325164Speter io_apic_ints[pin].int_type = 0; 183427728Sfsmp io_apic_ints[pin].int_flags = 0x05; /* edge/active-hi */ 183525164Speter io_apic_ints[pin].src_bus_id = 0; 183627728Sfsmp io_apic_ints[pin].src_bus_irq = pin; /* IRQ2 caught below */ 183725164Speter io_apic_ints[pin].dst_apic_id = io_apic_id; 183827728Sfsmp io_apic_ints[pin].dst_apic_int = pin; /* 1-to-1 */ 183925164Speter } 184025164Speter 184125164Speter /* special cases from MP v1.4, table 5-2 */ 184225164Speter if (type == 2) { 184325164Speter io_apic_ints[2].int_type = 0xff; /* N/C */ 184425164Speter io_apic_ints[13].int_type = 0xff; /* N/C */ 184525164Speter#if !defined(APIC_MIXED_MODE) 184625164Speter /** FIXME: ??? */ 184726108Sfsmp panic("sorry, can't support type 2 default yet"); 184825164Speter#endif /* APIC_MIXED_MODE */ 184926019Sfsmp } 185026019Sfsmp else 185125164Speter io_apic_ints[2].src_bus_irq = 0; /* ISA IRQ0 is on APIC INT 2 */ 185225164Speter 185325164Speter if (type == 7) 185425164Speter io_apic_ints[0].int_type = 0xff; /* N/C */ 185525164Speter else 185625164Speter io_apic_ints[0].int_type = 3; /* vectored 8259 */ 185725164Speter#endif /* APIC_IO */ 185825164Speter} 185925164Speter 186025164Speter 186125164Speter/* 186227634Sfsmp * initialize all the SMP locks 186327634Sfsmp */ 186428442Sfsmp 186528487Sfsmp/* critical region around IO APIC, apic_imen */ 186628487Sfsmpstruct simplelock imen_lock; 186728487Sfsmp 186829213Sfsmp/* critical region around splxx(), cpl, cml, cil, ipending */ 186928487Sfsmpstruct simplelock cpl_lock; 187028487Sfsmp 187128487Sfsmp/* Make FAST_INTR() routines sequential */ 187228487Sfsmpstruct simplelock fast_intr_lock; 187328487Sfsmp 187428487Sfsmp/* critical region around INTR() routines */ 187528487Sfsmpstruct simplelock intr_lock; 187628487Sfsmp 187731723Stegge/* lock region used by kernel profiling */ 187831723Steggestruct simplelock mcount_lock; 187931723Stegge 188028951Sfsmp#ifdef USE_COMLOCK 188128951Sfsmp/* locks com (tty) data/hardware accesses: a FASTINTR() */ 188228442Sfsmpstruct simplelock com_lock; 188328951Sfsmp#endif /* USE_COMLOCK */ 188428442Sfsmp 188528999Sfsmp#ifdef USE_CLOCKLOCK 188628999Sfsmp/* lock regions around the clock hardware */ 188728999Sfsmpstruct simplelock clock_lock; 188828999Sfsmp#endif /* USE_CLOCKLOCK */ 188928999Sfsmp 189048924Smsmith/* lock around the MP rendezvous */ 189148924Smsmithstatic struct simplelock smp_rv_lock; 189248924Smsmith 189365557Sjasone/* only 1 CPU can panic at a time :) */ 189465557Sjasonestruct simplelock panic_lock; 189565557Sjasone 189627634Sfsmpstatic void 189727634Sfsmpinit_locks(void) 189827634Sfsmp{ 189934021Stegge#if defined(APIC_INTR_DIAGNOSTIC) && defined(APIC_INTR_DIAGNOSTIC_IRQ) 190034021Stegge s_lock_init((struct simplelock*)&apic_itrace_debuglock); 190134021Stegge#endif 190234021Stegge 190331723Stegge s_lock_init((struct simplelock*)&mcount_lock); 190431723Stegge 190528442Sfsmp s_lock_init((struct simplelock*)&fast_intr_lock); 190628442Sfsmp s_lock_init((struct simplelock*)&intr_lock); 190727780Sfsmp s_lock_init((struct simplelock*)&imen_lock); 190828442Sfsmp s_lock_init((struct simplelock*)&cpl_lock); 190948924Smsmith s_lock_init(&smp_rv_lock); 191065557Sjasone s_lock_init(&panic_lock); 191128999Sfsmp 191228951Sfsmp#ifdef USE_COMLOCK 191328442Sfsmp s_lock_init((struct simplelock*)&com_lock); 191428951Sfsmp#endif /* USE_COMLOCK */ 191528999Sfsmp#ifdef USE_CLOCKLOCK 191628999Sfsmp s_lock_init((struct simplelock*)&clock_lock); 191728999Sfsmp#endif /* USE_CLOCKLOCK */ 191865557Sjasone 191965557Sjasone s_lock_init(&ap_boot_lock); 192027634Sfsmp} 192127634Sfsmp 192227634Sfsmp/* 192325164Speter * start each AP in our list 192425164Speter */ 192525164Speterstatic int 192625164Speterstart_all_aps(u_int boot_addr) 192725164Speter{ 192846129Sluoqi int x, i, pg; 192925164Speter u_char mpbiosreason; 193025164Speter u_long mpbioswarmvec; 193135077Speter struct globaldata *gd; 193229655Sdyson char *stack; 193325164Speter 193427005Sfsmp POSTCODE(START_ALL_APS_POST); 193527005Sfsmp 193625164Speter /* initialize BSP's local APIC */ 193727289Sfsmp apic_initialize(); 193828669Sfsmp bsp_apic_ready = 1; 193925164Speter 194025164Speter /* install the AP 1st level boot code */ 194125164Speter install_ap_tramp(boot_addr); 194225164Speter 194326812Speter 194425164Speter /* save the current value of the warm-start vector */ 194525164Speter mpbioswarmvec = *((u_long *) WARMBOOT_OFF); 194640179Skato#ifndef PC98 194725164Speter outb(CMOS_REG, BIOS_RESET); 194825164Speter mpbiosreason = inb(CMOS_DATA); 194940169Skato#endif 195025164Speter 195127005Sfsmp /* record BSP in CPU map */ 195227005Sfsmp all_cpus = 1; 195327005Sfsmp 195446129Sluoqi /* set up 0 -> 4MB P==V mapping for AP boot */ 195546129Sluoqi *(int *)PTD = PG_V | PG_RW | ((uintptr_t)(void *)KPTphys & PG_FRAME); 195646129Sluoqi invltlb(); 195746129Sluoqi 195825164Speter /* start each AP */ 195925164Speter for (x = 1; x <= mp_naps; ++x) { 196025164Speter 196128808Speter /* This is a bit verbose, it will go away soon. */ 196226812Speter 196346129Sluoqi /* first page of AP's private space */ 196446129Sluoqi pg = x * i386_btop(sizeof(struct privatespace)); 196526812Speter 196628808Speter /* allocate a new private data page */ 196735077Speter gd = (struct globaldata *)kmem_alloc(kernel_map, PAGE_SIZE); 196826812Speter 196926812Speter /* wire it into the private page table page */ 197046129Sluoqi SMPpt[pg] = (pt_entry_t)(PG_V | PG_RW | vtophys(gd)); 197126812Speter 197228808Speter /* allocate and set up an idle stack data page */ 197329655Sdyson stack = (char *)kmem_alloc(kernel_map, UPAGES*PAGE_SIZE); 197429655Sdyson for (i = 0; i < UPAGES; i++) 197546129Sluoqi SMPpt[pg + 5 + i] = (pt_entry_t) 197646129Sluoqi (PG_V | PG_RW | vtophys(PAGE_SIZE * i + stack)); 197726812Speter 197846129Sluoqi SMPpt[pg + 1] = 0; /* *prv_CMAP1 */ 197946129Sluoqi SMPpt[pg + 2] = 0; /* *prv_CMAP2 */ 198046129Sluoqi SMPpt[pg + 3] = 0; /* *prv_CMAP3 */ 198146129Sluoqi SMPpt[pg + 4] = 0; /* *prv_PMAP1 */ 198226812Speter 198328808Speter /* prime data page for it to use */ 198465557Sjasone SLIST_INSERT_HEAD(&cpuhead, gd, gd_allcpu); 198546129Sluoqi gd->gd_cpuid = x; 198646129Sluoqi gd->gd_cpu_lockid = x << 24; 198746129Sluoqi gd->gd_prv_CMAP1 = &SMPpt[pg + 1]; 198846129Sluoqi gd->gd_prv_CMAP2 = &SMPpt[pg + 2]; 198946129Sluoqi gd->gd_prv_CMAP3 = &SMPpt[pg + 3]; 199046129Sluoqi gd->gd_prv_PMAP1 = &SMPpt[pg + 4]; 199146129Sluoqi gd->gd_prv_CADDR1 = SMP_prvspace[x].CPAGE1; 199246129Sluoqi gd->gd_prv_CADDR2 = SMP_prvspace[x].CPAGE2; 199346129Sluoqi gd->gd_prv_CADDR3 = SMP_prvspace[x].CPAGE3; 199446129Sluoqi gd->gd_prv_PADDR1 = (unsigned *)SMP_prvspace[x].PPAGE1; 199526812Speter 199625164Speter /* setup a vector to our boot code */ 199725164Speter *((volatile u_short *) WARMBOOT_OFF) = WARMBOOT_TARGET; 199825164Speter *((volatile u_short *) WARMBOOT_SEG) = (boot_addr >> 4); 199940179Skato#ifndef PC98 200025164Speter outb(CMOS_REG, BIOS_RESET); 200125164Speter outb(CMOS_DATA, BIOS_WARM); /* 'warm-start' */ 200240169Skato#endif 200325164Speter 200446129Sluoqi bootSTK = &SMP_prvspace[x].idlestack[UPAGES*PAGE_SIZE]; 200548144Sluoqi bootAP = x; 200646129Sluoqi 200725164Speter /* attempt to start the Application Processor */ 200825164Speter CHECK_INIT(99); /* setup checkpoints */ 200925164Speter if (!start_ap(x, boot_addr)) { 201025164Speter printf("AP #%d (PHY# %d) failed!\n", x, CPU_TO_ID(x)); 201125164Speter CHECK_PRINT("trace"); /* show checkpoints */ 201226155Sfsmp /* better panic as the AP may be running loose */ 201326155Sfsmp printf("panic y/n? [y] "); 201426101Sfsmp if (cngetc() != 'n') 201526108Sfsmp panic("bye-bye"); 201625164Speter } 201727005Sfsmp CHECK_PRINT("trace"); /* show checkpoints */ 201825164Speter 201925164Speter /* record its version info */ 202025164Speter cpu_apic_versions[x] = cpu_apic_versions[0]; 202127005Sfsmp 202227005Sfsmp all_cpus |= (1 << x); /* record AP in CPU map */ 202325164Speter } 202425164Speter 202527005Sfsmp /* build our map of 'other' CPUs */ 202627005Sfsmp other_cpus = all_cpus & ~(1 << cpuid); 202727005Sfsmp 202825164Speter /* fill in our (BSP) APIC version */ 202926812Speter cpu_apic_versions[0] = lapic.version; 203025164Speter 203125164Speter /* restore the warmstart vector */ 203225164Speter *(u_long *) WARMBOOT_OFF = mpbioswarmvec; 203340179Skato#ifndef PC98 203425164Speter outb(CMOS_REG, BIOS_RESET); 203525164Speter outb(CMOS_DATA, mpbiosreason); 203640169Skato#endif 203725164Speter 203828808Speter /* 203928808Speter * Set up the idle context for the BSP. Similar to above except 204028808Speter * that some was done by locore, some by pmap.c and some is implicit 204128808Speter * because the BSP is cpu#0 and the page is initially zero, and also 204228808Speter * because we can refer to variables by name on the BSP.. 204328808Speter */ 204428808Speter 204528808Speter /* Allocate and setup BSP idle stack */ 204629655Sdyson stack = (char *)kmem_alloc(kernel_map, UPAGES * PAGE_SIZE); 204729655Sdyson for (i = 0; i < UPAGES; i++) 204846129Sluoqi SMPpt[5 + i] = (pt_entry_t) 204946129Sluoqi (PG_V | PG_RW | vtophys(PAGE_SIZE * i + stack)); 205028808Speter 205146129Sluoqi *(int *)PTD = 0; 205248144Sluoqi pmap_set_opt(); 205327484Sdyson 205425164Speter /* number of APs actually started */ 205525164Speter return mp_ncpus - 1; 205625164Speter} 205725164Speter 205825164Speter 205925164Speter/* 206025164Speter * load the 1st level AP boot code into base memory. 206125164Speter */ 206225164Speter 206325164Speter/* targets for relocation */ 206425164Speterextern void bigJump(void); 206525164Speterextern void bootCodeSeg(void); 206625164Speterextern void bootDataSeg(void); 206725164Speterextern void MPentry(void); 206825164Speterextern u_int MP_GDT; 206925164Speterextern u_int mp_gdtbase; 207025164Speter 207125164Speterstatic void 207225164Speterinstall_ap_tramp(u_int boot_addr) 207325164Speter{ 207425164Speter int x; 207525164Speter int size = *(int *) ((u_long) & bootMP_size); 207625164Speter u_char *src = (u_char *) ((u_long) bootMP); 207725164Speter u_char *dst = (u_char *) boot_addr + KERNBASE; 207825164Speter u_int boot_base = (u_int) bootMP; 207925164Speter u_int8_t *dst8; 208025164Speter u_int16_t *dst16; 208125164Speter u_int32_t *dst32; 208225164Speter 208327005Sfsmp POSTCODE(INSTALL_AP_TRAMP_POST); 208427005Sfsmp 208525164Speter for (x = 0; x < size; ++x) 208625164Speter *dst++ = *src++; 208725164Speter 208825164Speter /* 208925164Speter * modify addresses in code we just moved to basemem. unfortunately we 209025164Speter * need fairly detailed info about mpboot.s for this to work. changes 209125164Speter * to mpboot.s might require changes here. 209225164Speter */ 209325164Speter 209425164Speter /* boot code is located in KERNEL space */ 209525164Speter dst = (u_char *) boot_addr + KERNBASE; 209625164Speter 209725164Speter /* modify the lgdt arg */ 209825164Speter dst32 = (u_int32_t *) (dst + ((u_int) & mp_gdtbase - boot_base)); 209925164Speter *dst32 = boot_addr + ((u_int) & MP_GDT - boot_base); 210025164Speter 210125164Speter /* modify the ljmp target for MPentry() */ 210225164Speter dst32 = (u_int32_t *) (dst + ((u_int) bigJump - boot_base) + 1); 210325164Speter *dst32 = ((u_int) MPentry - KERNBASE); 210425164Speter 210525164Speter /* modify the target for boot code segment */ 210625164Speter dst16 = (u_int16_t *) (dst + ((u_int) bootCodeSeg - boot_base)); 210725164Speter dst8 = (u_int8_t *) (dst16 + 1); 210825164Speter *dst16 = (u_int) boot_addr & 0xffff; 210925164Speter *dst8 = ((u_int) boot_addr >> 16) & 0xff; 211025164Speter 211125164Speter /* modify the target for boot data segment */ 211225164Speter dst16 = (u_int16_t *) (dst + ((u_int) bootDataSeg - boot_base)); 211325164Speter dst8 = (u_int8_t *) (dst16 + 1); 211425164Speter *dst16 = (u_int) boot_addr & 0xffff; 211525164Speter *dst8 = ((u_int) boot_addr >> 16) & 0xff; 211625164Speter} 211725164Speter 211825164Speter 211925164Speter/* 212025164Speter * this function starts the AP (application processor) identified 212125164Speter * by the APIC ID 'physicalCpu'. It does quite a "song and dance" 212225164Speter * to accomplish this. This is necessary because of the nuances 212325164Speter * of the different hardware we might encounter. It ain't pretty, 212425164Speter * but it seems to work. 212525164Speter */ 212625164Speterstatic int 212725164Speterstart_ap(int logical_cpu, u_int boot_addr) 212825164Speter{ 212925164Speter int physical_cpu; 213025164Speter int vector; 213125164Speter int cpus; 213225164Speter u_long icr_lo, icr_hi; 213325164Speter 213427005Sfsmp POSTCODE(START_AP_POST); 213527005Sfsmp 213625164Speter /* get the PHYSICAL APIC ID# */ 213725164Speter physical_cpu = CPU_TO_ID(logical_cpu); 213825164Speter 213925164Speter /* calculate the vector */ 214025164Speter vector = (boot_addr >> 12) & 0xff; 214125164Speter 214225164Speter /* used as a watchpoint to signal AP startup */ 214325164Speter cpus = mp_ncpus; 214425164Speter 214525164Speter /* 214625164Speter * first we do an INIT/RESET IPI this INIT IPI might be run, reseting 214725164Speter * and running the target CPU. OR this INIT IPI might be latched (P5 214825164Speter * bug), CPU waiting for STARTUP IPI. OR this INIT IPI might be 214925164Speter * ignored. 215025164Speter */ 215125164Speter 215225164Speter /* setup the address for the target AP */ 215326812Speter icr_hi = lapic.icr_hi & ~APIC_ID_MASK; 215425164Speter icr_hi |= (physical_cpu << 24); 215526812Speter lapic.icr_hi = icr_hi; 215625164Speter 215725164Speter /* do an INIT IPI: assert RESET */ 215826812Speter icr_lo = lapic.icr_lo & 0xfff00000; 215926812Speter lapic.icr_lo = icr_lo | 0x0000c500; 216025164Speter 216125164Speter /* wait for pending status end */ 216226812Speter while (lapic.icr_lo & APIC_DELSTAT_MASK) 216325164Speter /* spin */ ; 216425164Speter 216525164Speter /* do an INIT IPI: deassert RESET */ 216626812Speter lapic.icr_lo = icr_lo | 0x00008500; 216725164Speter 216825164Speter /* wait for pending status end */ 216925164Speter u_sleep(10000); /* wait ~10mS */ 217026812Speter while (lapic.icr_lo & APIC_DELSTAT_MASK) 217125164Speter /* spin */ ; 217225164Speter 217325164Speter /* 217425164Speter * next we do a STARTUP IPI: the previous INIT IPI might still be 217525164Speter * latched, (P5 bug) this 1st STARTUP would then terminate 217625164Speter * immediately, and the previously started INIT IPI would continue. OR 217725164Speter * the previous INIT IPI has already run. and this STARTUP IPI will 217825164Speter * run. OR the previous INIT IPI was ignored. and this STARTUP IPI 217925164Speter * will run. 218025164Speter */ 218125164Speter 218225164Speter /* do a STARTUP IPI */ 218326812Speter lapic.icr_lo = icr_lo | 0x00000600 | vector; 218426812Speter while (lapic.icr_lo & APIC_DELSTAT_MASK) 218525164Speter /* spin */ ; 218625164Speter u_sleep(200); /* wait ~200uS */ 218725164Speter 218825164Speter /* 218925164Speter * finally we do a 2nd STARTUP IPI: this 2nd STARTUP IPI should run IF 219025164Speter * the previous STARTUP IPI was cancelled by a latched INIT IPI. OR 219125164Speter * this STARTUP IPI will be ignored, as only ONE STARTUP IPI is 219225164Speter * recognized after hardware RESET or INIT IPI. 219325164Speter */ 219425164Speter 219526812Speter lapic.icr_lo = icr_lo | 0x00000600 | vector; 219626812Speter while (lapic.icr_lo & APIC_DELSTAT_MASK) 219725164Speter /* spin */ ; 219825164Speter u_sleep(200); /* wait ~200uS */ 219925164Speter 220025164Speter /* wait for it to start */ 220125164Speter set_apic_timer(5000000);/* == 5 seconds */ 220225164Speter while (read_apic_timer()) 220325164Speter if (mp_ncpus > cpus) 220425164Speter return 1; /* return SUCCESS */ 220525164Speter 220625164Speter return 0; /* return FAILURE */ 220725164Speter} 220825164Speter 220925164Speter/* 221025164Speter * Flush the TLB on all other CPU's 221125164Speter * 221225164Speter * XXX: Needs to handshake and wait for completion before proceding. 221325164Speter */ 221425164Spetervoid 221525215Sfsmpsmp_invltlb(void) 221625164Speter{ 221725419Sfsmp#if defined(APIC_IO) 221828809Speter if (smp_started && invltlb_ok) 221927005Sfsmp all_but_self_ipi(XINVLTLB_OFFSET); 222025419Sfsmp#endif /* APIC_IO */ 222125164Speter} 222225164Speter 222325164Spetervoid 222425164Speterinvlpg(u_int addr) 222525164Speter{ 222625164Speter __asm __volatile("invlpg (%0)"::"r"(addr):"memory"); 222725215Sfsmp 222825215Sfsmp /* send a message to the other CPUs */ 222925164Speter smp_invltlb(); 223025164Speter} 223125164Speter 223225164Spetervoid 223325164Speterinvltlb(void) 223425164Speter{ 223525164Speter u_long temp; 223625215Sfsmp 223725164Speter /* 223825164Speter * This should be implemented as load_cr3(rcr3()) when load_cr3() is 223925164Speter * inlined. 224025164Speter */ 224125164Speter __asm __volatile("movl %%cr3, %0; movl %0, %%cr3":"=r"(temp) :: "memory"); 224225215Sfsmp 224325215Sfsmp /* send a message to the other CPUs */ 224425164Speter smp_invltlb(); 224525164Speter} 224627005Sfsmp 224727005Sfsmp 224827005Sfsmp/* 224927005Sfsmp * When called the executing CPU will send an IPI to all other CPUs 225027005Sfsmp * requesting that they halt execution. 225127005Sfsmp * 225227005Sfsmp * Usually (but not necessarily) called with 'other_cpus' as its arg. 225327005Sfsmp * 225427005Sfsmp * - Signals all CPUs in map to stop. 225527005Sfsmp * - Waits for each to stop. 225627005Sfsmp * 225727005Sfsmp * Returns: 225827005Sfsmp * -1: error 225927005Sfsmp * 0: NA 226027005Sfsmp * 1: ok 226127005Sfsmp * 226227005Sfsmp * XXX FIXME: this is not MP-safe, needs a lock to prevent multiple CPUs 226327005Sfsmp * from executing at same time. 226427005Sfsmp */ 226527005Sfsmpint 226628808Speterstop_cpus(u_int map) 226727005Sfsmp{ 226828809Speter if (!smp_started) 226927005Sfsmp return 0; 227027005Sfsmp 227127353Sfsmp /* send the Xcpustop IPI to all CPUs in map */ 227227005Sfsmp selected_apic_ipi(map, XCPUSTOP_OFFSET, APIC_DELMODE_FIXED); 227336135Stegge 227436135Stegge while ((stopped_cpus & map) != map) 227527005Sfsmp /* spin */ ; 227627005Sfsmp 227727005Sfsmp return 1; 227827005Sfsmp} 227927005Sfsmp 228027005Sfsmp 228127005Sfsmp/* 228227005Sfsmp * Called by a CPU to restart stopped CPUs. 228327005Sfsmp * 228427005Sfsmp * Usually (but not necessarily) called with 'stopped_cpus' as its arg. 228527005Sfsmp * 228627005Sfsmp * - Signals all CPUs in map to restart. 228727005Sfsmp * - Waits for each to restart. 228827005Sfsmp * 228927005Sfsmp * Returns: 229027005Sfsmp * -1: error 229127005Sfsmp * 0: NA 229227005Sfsmp * 1: ok 229327005Sfsmp */ 229427005Sfsmpint 229528808Speterrestart_cpus(u_int map) 229627005Sfsmp{ 229728809Speter if (!smp_started) 229827005Sfsmp return 0; 229927005Sfsmp 230027255Sfsmp started_cpus = map; /* signal other cpus to restart */ 230127255Sfsmp 230236135Stegge while ((stopped_cpus & map) != 0) /* wait for each to clear its bit */ 230327005Sfsmp /* spin */ ; 230427005Sfsmp 230527005Sfsmp return 1; 230627005Sfsmp} 230728808Speter 230828808Speterint smp_active = 0; /* are the APs allowed to run? */ 230928808SpeterSYSCTL_INT(_machdep, OID_AUTO, smp_active, CTLFLAG_RW, &smp_active, 0, ""); 231028808Speter 231128808Speter/* XXX maybe should be hw.ncpu */ 231233181Seivindstatic int smp_cpus = 1; /* how many cpu's running */ 231328808SpeterSYSCTL_INT(_machdep, OID_AUTO, smp_cpus, CTLFLAG_RD, &smp_cpus, 0, ""); 231428808Speter 231528808Speterint invltlb_ok = 0; /* throttle smp_invltlb() till safe */ 231628808SpeterSYSCTL_INT(_machdep, OID_AUTO, invltlb_ok, CTLFLAG_RW, &invltlb_ok, 0, ""); 231728808Speter 231833181Seivind/* Warning: Do not staticize. Used from swtch.s */ 231933936Sdysonint do_page_zero_idle = 1; /* bzero pages for fun and profit in idleloop */ 232028808SpeterSYSCTL_INT(_machdep, OID_AUTO, do_page_zero_idle, CTLFLAG_RW, 232128808Speter &do_page_zero_idle, 0, ""); 232228808Speter 232334021Stegge/* Is forwarding of a interrupt to the CPU holding the ISR lock enabled ? */ 232434021Steggeint forward_irq_enabled = 1; 232534021SteggeSYSCTL_INT(_machdep, OID_AUTO, forward_irq_enabled, CTLFLAG_RW, 232634021Stegge &forward_irq_enabled, 0, ""); 232734021Stegge 232834020Stegge/* Enable forwarding of a signal to a process running on a different CPU */ 232941362Seivindstatic int forward_signal_enabled = 1; 233034020SteggeSYSCTL_INT(_machdep, OID_AUTO, forward_signal_enabled, CTLFLAG_RW, 233134020Stegge &forward_signal_enabled, 0, ""); 233228808Speter 233336135Stegge/* Enable forwarding of roundrobin to all other cpus */ 233441362Seivindstatic int forward_roundrobin_enabled = 1; 233536135SteggeSYSCTL_INT(_machdep, OID_AUTO, forward_roundrobin_enabled, CTLFLAG_RW, 233636135Stegge &forward_roundrobin_enabled, 0, ""); 233736135Stegge 233828808Speter/* 233928808Speter * This is called once the rest of the system is up and running and we're 234028808Speter * ready to let the AP's out of the pen. 234128808Speter */ 234228808Spetervoid ap_init(void); 234328808Speter 234428808Spetervoid 234565557Sjasoneap_init(void) 234628808Speter{ 234728808Speter u_int apic_id; 234828808Speter 234965557Sjasone /* lock against other AP's that are waking up */ 235065557Sjasone s_lock(&ap_boot_lock); 235165557Sjasone 235248144Sluoqi /* BSP may have changed PTD while we're waiting for the lock */ 235348144Sluoqi cpu_invltlb(); 235448144Sluoqi 235528808Speter smp_cpus++; 235628808Speter 235734197Stegge#if defined(I586_CPU) && !defined(NO_F00F_HACK) 235834197Stegge lidt(&r_idt); 235934197Stegge#endif 236034197Stegge 236128808Speter /* Build our map of 'other' CPUs. */ 236228808Speter other_cpus = all_cpus & ~(1 << cpuid); 236328808Speter 236428808Speter printf("SMP: AP CPU #%d Launched!\n", cpuid); 236528808Speter 236664529Speter /* set up CPU registers and state */ 236764529Speter cpu_setregs(); 236828808Speter 236950972Speter /* set up FPU state on the AP */ 237050972Speter npxinit(__INITIAL_NPXCW__); 237150972Speter 237228808Speter /* A quick check from sanity claus */ 237328808Speter apic_id = (apic_id_to_logical[(lapic.id & 0x0f000000) >> 24]); 237428808Speter if (cpuid != apic_id) { 237528808Speter printf("SMP: cpuid = %d\n", cpuid); 237628808Speter printf("SMP: apic_id = %d\n", apic_id); 237738505Sbde printf("PTD[MPPTDI] = %p\n", (void *)PTD[MPPTDI]); 237828808Speter panic("cpuid mismatch! boom!!"); 237928808Speter } 238028808Speter 238128808Speter /* Init local apic for irq's */ 238228808Speter apic_initialize(); 238328808Speter 238446215Smsmith /* Set memory range attributes for this CPU to match the BSP */ 238546215Smsmith mem_range_AP_init(); 238646215Smsmith 238728808Speter /* 238828808Speter * Activate smp_invltlb, although strictly speaking, this isn't 238928808Speter * quite correct yet. We should have a bitfield for cpus willing 239028808Speter * to accept TLB flush IPI's or something and sync them. 239128808Speter */ 239238888Stegge if (smp_cpus == mp_ncpus) { 239338888Stegge invltlb_ok = 1; 239438888Stegge smp_started = 1; /* enable IPI's, tlb shootdown, freezes etc */ 239538888Stegge smp_active = 1; /* historic */ 239638888Stegge } 239765557Sjasone 239865557Sjasone /* let other AP's wake up now */ 239965557Sjasone s_unlock(&ap_boot_lock); 240065557Sjasone 240165557Sjasone /* wait until all the AP's are up */ 240265557Sjasone while (smp_started == 0) 240365557Sjasone ; /* nothing */ 240465557Sjasone 240565557Sjasone /* 240665557Sjasone * Set curproc to our per-cpu idleproc so that mutexes have 240765557Sjasone * something unique to lock with. 240865557Sjasone */ 240965557Sjasone PCPU_SET(curproc,idleproc); 241065557Sjasone PCPU_SET(prevproc,idleproc); 241165557Sjasone 241265557Sjasone microuptime(&switchtime); 241365557Sjasone switchticks = ticks; 241465557Sjasone 241565557Sjasone /* ok, now grab sched_lock and enter the scheduler */ 241665557Sjasone enable_intr(); 241765557Sjasone mtx_enter(&sched_lock, MTX_SPIN); 241865557Sjasone cpu_throw(); /* doesn't return */ 241965557Sjasone 242065557Sjasone panic("scheduler returned us to ap_init"); 242128808Speter} 242230112Sdyson 242331639Sfsmp#ifdef BETTER_CLOCK 242431639Sfsmp 242531639Sfsmp#define CHECKSTATE_USER 0 242631639Sfsmp#define CHECKSTATE_SYS 1 242731639Sfsmp#define CHECKSTATE_INTR 2 242831639Sfsmp 242933181Seivind/* Do not staticize. Used from apic_vector.s */ 243031639Sfsmpstruct proc* checkstate_curproc[NCPU]; 243131639Sfsmpint checkstate_cpustate[NCPU]; 243231639Sfsmpu_long checkstate_pc[NCPU]; 243331639Sfsmp 243431639Sfsmpextern long cp_time[CPUSTATES]; 243531639Sfsmp 243631639Sfsmp#define PC_TO_INDEX(pc, prof) \ 243731639Sfsmp ((int)(((u_quad_t)((pc) - (prof)->pr_off) * \ 243831639Sfsmp (u_quad_t)((prof)->pr_scale)) >> 16) & ~1) 243931639Sfsmp 244031639Sfsmpstatic void 244131639Sfsmpaddupc_intr_forwarded(struct proc *p, int id, int *astmap) 244231639Sfsmp{ 244331639Sfsmp int i; 244431639Sfsmp struct uprof *prof; 244531639Sfsmp u_long pc; 244631639Sfsmp 244731639Sfsmp pc = checkstate_pc[id]; 244831639Sfsmp prof = &p->p_stats->p_prof; 244931639Sfsmp if (pc >= prof->pr_off && 245031639Sfsmp (i = PC_TO_INDEX(pc, prof)) < prof->pr_size) { 245131639Sfsmp if ((p->p_flag & P_OWEUPC) == 0) { 245231639Sfsmp prof->pr_addr = pc; 245331639Sfsmp prof->pr_ticks = 1; 245431639Sfsmp p->p_flag |= P_OWEUPC; 245531639Sfsmp } 245631639Sfsmp *astmap |= (1 << id); 245731639Sfsmp } 245831639Sfsmp} 245931639Sfsmp 246031639Sfsmpstatic void 246131639Sfsmpforwarded_statclock(int id, int pscnt, int *astmap) 246231639Sfsmp{ 246331639Sfsmp struct pstats *pstats; 246431639Sfsmp long rss; 246531639Sfsmp struct rusage *ru; 246631639Sfsmp struct vmspace *vm; 246731639Sfsmp int cpustate; 246831639Sfsmp struct proc *p; 246931639Sfsmp#ifdef GPROF 247031639Sfsmp register struct gmonparam *g; 247131639Sfsmp int i; 247231639Sfsmp#endif 247331639Sfsmp 247431639Sfsmp p = checkstate_curproc[id]; 247531639Sfsmp cpustate = checkstate_cpustate[id]; 247631639Sfsmp 247765557Sjasone /* XXX */ 247865557Sjasone if (p->p_ithd) 247965557Sjasone cpustate = CHECKSTATE_INTR; 248065557Sjasone else if (p == idleproc) 248165557Sjasone cpustate = CHECKSTATE_SYS; 248265557Sjasone 248331639Sfsmp switch (cpustate) { 248431639Sfsmp case CHECKSTATE_USER: 248531639Sfsmp if (p->p_flag & P_PROFIL) 248631639Sfsmp addupc_intr_forwarded(p, id, astmap); 248731639Sfsmp if (pscnt > 1) 248831639Sfsmp return; 248931639Sfsmp p->p_uticks++; 249031639Sfsmp if (p->p_nice > NZERO) 249131639Sfsmp cp_time[CP_NICE]++; 249231639Sfsmp else 249331639Sfsmp cp_time[CP_USER]++; 249431639Sfsmp break; 249531639Sfsmp case CHECKSTATE_SYS: 249631639Sfsmp#ifdef GPROF 249731639Sfsmp /* 249831639Sfsmp * Kernel statistics are just like addupc_intr, only easier. 249931639Sfsmp */ 250031639Sfsmp g = &_gmonparam; 250131639Sfsmp if (g->state == GMON_PROF_ON) { 250231639Sfsmp i = checkstate_pc[id] - g->lowpc; 250331639Sfsmp if (i < g->textsize) { 250431639Sfsmp i /= HISTFRACTION * sizeof(*g->kcount); 250531639Sfsmp g->kcount[i]++; 250631639Sfsmp } 250731639Sfsmp } 250831639Sfsmp#endif 250931639Sfsmp if (pscnt > 1) 251031639Sfsmp return; 251131639Sfsmp 251265557Sjasone if (p == idleproc) { 251365557Sjasone p->p_sticks++; 251431639Sfsmp cp_time[CP_IDLE]++; 251565557Sjasone } else { 251631639Sfsmp p->p_sticks++; 251731639Sfsmp cp_time[CP_SYS]++; 251831639Sfsmp } 251931639Sfsmp break; 252031639Sfsmp case CHECKSTATE_INTR: 252131639Sfsmp default: 252231639Sfsmp#ifdef GPROF 252331639Sfsmp /* 252431639Sfsmp * Kernel statistics are just like addupc_intr, only easier. 252531639Sfsmp */ 252631639Sfsmp g = &_gmonparam; 252731639Sfsmp if (g->state == GMON_PROF_ON) { 252831639Sfsmp i = checkstate_pc[id] - g->lowpc; 252931639Sfsmp if (i < g->textsize) { 253031639Sfsmp i /= HISTFRACTION * sizeof(*g->kcount); 253131639Sfsmp g->kcount[i]++; 253231639Sfsmp } 253331639Sfsmp } 253431639Sfsmp#endif 253531639Sfsmp if (pscnt > 1) 253631639Sfsmp return; 253731639Sfsmp if (p) 253831639Sfsmp p->p_iticks++; 253931639Sfsmp cp_time[CP_INTR]++; 254031639Sfsmp } 254165557Sjasone if (p != idleproc) { 254253745Sbde schedclock(p); 254331639Sfsmp 254431639Sfsmp /* Update resource usage integrals and maximums. */ 254531639Sfsmp if ((pstats = p->p_stats) != NULL && 254631639Sfsmp (ru = &pstats->p_ru) != NULL && 254731639Sfsmp (vm = p->p_vmspace) != NULL) { 254844157Sluoqi ru->ru_ixrss += pgtok(vm->vm_tsize); 254944157Sluoqi ru->ru_idrss += pgtok(vm->vm_dsize); 255044157Sluoqi ru->ru_isrss += pgtok(vm->vm_ssize); 255144157Sluoqi rss = pgtok(vmspace_resident_count(vm)); 255231639Sfsmp if (ru->ru_maxrss < rss) 255331639Sfsmp ru->ru_maxrss = rss; 255431639Sfsmp } 255531639Sfsmp } 255631639Sfsmp} 255731639Sfsmp 255831639Sfsmpvoid 255931639Sfsmpforward_statclock(int pscnt) 256031639Sfsmp{ 256131639Sfsmp int map; 256231639Sfsmp int id; 256331639Sfsmp int i; 256431639Sfsmp 256531639Sfsmp /* Kludge. We don't yet have separate locks for the interrupts 256631639Sfsmp * and the kernel. This means that we cannot let the other processors 256731639Sfsmp * handle complex interrupts while inhibiting them from entering 256831639Sfsmp * the kernel in a non-interrupt context. 256931639Sfsmp * 257031639Sfsmp * What we can do, without changing the locking mechanisms yet, 257131639Sfsmp * is letting the other processors handle a very simple interrupt 257231639Sfsmp * (wich determines the processor states), and do the main 257331639Sfsmp * work ourself. 257431639Sfsmp */ 257531639Sfsmp 257631720Stegge if (!smp_started || !invltlb_ok || cold || panicstr) 257731639Sfsmp return; 257831639Sfsmp 257931639Sfsmp /* Step 1: Probe state (user, cpu, interrupt, spinlock, idle ) */ 258031639Sfsmp 258131720Stegge map = other_cpus & ~stopped_cpus ; 258231639Sfsmp checkstate_probed_cpus = 0; 258331720Stegge if (map != 0) 258431720Stegge selected_apic_ipi(map, 258531720Stegge XCPUCHECKSTATE_OFFSET, APIC_DELMODE_FIXED); 258631639Sfsmp 258731639Sfsmp i = 0; 258831639Sfsmp while (checkstate_probed_cpus != map) { 258931639Sfsmp /* spin */ 259031639Sfsmp i++; 259136135Stegge if (i == 100000) { 259236135Stegge#ifdef BETTER_CLOCK_DIAGNOSTIC 259331639Sfsmp printf("forward_statclock: checkstate %x\n", 259431639Sfsmp checkstate_probed_cpus); 259536135Stegge#endif 259631720Stegge break; 259731639Sfsmp } 259831639Sfsmp } 259931639Sfsmp 260031639Sfsmp /* 260131639Sfsmp * Step 2: walk through other processors processes, update ticks and 260231639Sfsmp * profiling info. 260331639Sfsmp */ 260431639Sfsmp 260531639Sfsmp map = 0; 260631639Sfsmp for (id = 0; id < mp_ncpus; id++) { 260731639Sfsmp if (id == cpuid) 260831639Sfsmp continue; 260931639Sfsmp if (((1 << id) & checkstate_probed_cpus) == 0) 261031720Stegge continue; 261131639Sfsmp forwarded_statclock(id, pscnt, &map); 261231639Sfsmp } 261331639Sfsmp if (map != 0) { 261431639Sfsmp checkstate_need_ast |= map; 261531639Sfsmp selected_apic_ipi(map, XCPUAST_OFFSET, APIC_DELMODE_FIXED); 261631639Sfsmp i = 0; 261734019Stegge while ((checkstate_need_ast & map) != 0) { 261831639Sfsmp /* spin */ 261931639Sfsmp i++; 262034019Stegge if (i > 100000) { 262134019Stegge#ifdef BETTER_CLOCK_DIAGNOSTIC 262231639Sfsmp printf("forward_statclock: dropped ast 0x%x\n", 262334019Stegge checkstate_need_ast & map); 262434019Stegge#endif 262531639Sfsmp break; 262631639Sfsmp } 262731639Sfsmp } 262831639Sfsmp } 262931639Sfsmp} 263031639Sfsmp 263131639Sfsmpvoid 263231639Sfsmpforward_hardclock(int pscnt) 263331639Sfsmp{ 263431639Sfsmp int map; 263531639Sfsmp int id; 263631639Sfsmp struct proc *p; 263731639Sfsmp struct pstats *pstats; 263831639Sfsmp int i; 263931639Sfsmp 264031639Sfsmp /* Kludge. We don't yet have separate locks for the interrupts 264131639Sfsmp * and the kernel. This means that we cannot let the other processors 264231639Sfsmp * handle complex interrupts while inhibiting them from entering 264331639Sfsmp * the kernel in a non-interrupt context. 264431639Sfsmp * 264531639Sfsmp * What we can do, without changing the locking mechanisms yet, 264631639Sfsmp * is letting the other processors handle a very simple interrupt 264731639Sfsmp * (wich determines the processor states), and do the main 264831639Sfsmp * work ourself. 264931639Sfsmp */ 265031639Sfsmp 265131720Stegge if (!smp_started || !invltlb_ok || cold || panicstr) 265231639Sfsmp return; 265331639Sfsmp 265431639Sfsmp /* Step 1: Probe state (user, cpu, interrupt, spinlock, idle) */ 265531639Sfsmp 265631720Stegge map = other_cpus & ~stopped_cpus ; 265731639Sfsmp checkstate_probed_cpus = 0; 265831720Stegge if (map != 0) 265931720Stegge selected_apic_ipi(map, 266031720Stegge XCPUCHECKSTATE_OFFSET, APIC_DELMODE_FIXED); 266131720Stegge 266231639Sfsmp i = 0; 266331639Sfsmp while (checkstate_probed_cpus != map) { 266431639Sfsmp /* spin */ 266531639Sfsmp i++; 266636135Stegge if (i == 100000) { 266736135Stegge#ifdef BETTER_CLOCK_DIAGNOSTIC 266831639Sfsmp printf("forward_hardclock: checkstate %x\n", 266931639Sfsmp checkstate_probed_cpus); 267036135Stegge#endif 267131720Stegge break; 267231639Sfsmp } 267331639Sfsmp } 267431639Sfsmp 267531639Sfsmp /* 267631639Sfsmp * Step 2: walk through other processors processes, update virtual 267731639Sfsmp * timer and profiling timer. If stathz == 0, also update ticks and 267831639Sfsmp * profiling info. 267931639Sfsmp */ 268031639Sfsmp 268131639Sfsmp map = 0; 268231639Sfsmp for (id = 0; id < mp_ncpus; id++) { 268331639Sfsmp if (id == cpuid) 268431639Sfsmp continue; 268531639Sfsmp if (((1 << id) & checkstate_probed_cpus) == 0) 268631720Stegge continue; 268731639Sfsmp p = checkstate_curproc[id]; 268831639Sfsmp if (p) { 268931639Sfsmp pstats = p->p_stats; 269031639Sfsmp if (checkstate_cpustate[id] == CHECKSTATE_USER && 269135058Sphk timevalisset(&pstats->p_timer[ITIMER_VIRTUAL].it_value) && 269231639Sfsmp itimerdecr(&pstats->p_timer[ITIMER_VIRTUAL], tick) == 0) { 269331639Sfsmp psignal(p, SIGVTALRM); 269431639Sfsmp map |= (1 << id); 269531639Sfsmp } 269635058Sphk if (timevalisset(&pstats->p_timer[ITIMER_PROF].it_value) && 269731639Sfsmp itimerdecr(&pstats->p_timer[ITIMER_PROF], tick) == 0) { 269831639Sfsmp psignal(p, SIGPROF); 269931639Sfsmp map |= (1 << id); 270031639Sfsmp } 270131639Sfsmp } 270231639Sfsmp if (stathz == 0) { 270331639Sfsmp forwarded_statclock( id, pscnt, &map); 270431639Sfsmp } 270531639Sfsmp } 270631639Sfsmp if (map != 0) { 270731639Sfsmp checkstate_need_ast |= map; 270831639Sfsmp selected_apic_ipi(map, XCPUAST_OFFSET, APIC_DELMODE_FIXED); 270931639Sfsmp i = 0; 271034019Stegge while ((checkstate_need_ast & map) != 0) { 271131639Sfsmp /* spin */ 271231639Sfsmp i++; 271334019Stegge if (i > 100000) { 271434019Stegge#ifdef BETTER_CLOCK_DIAGNOSTIC 271531639Sfsmp printf("forward_hardclock: dropped ast 0x%x\n", 271634019Stegge checkstate_need_ast & map); 271734019Stegge#endif 271831639Sfsmp break; 271931639Sfsmp } 272031639Sfsmp } 272131639Sfsmp } 272231639Sfsmp} 272331639Sfsmp 272431639Sfsmp#endif /* BETTER_CLOCK */ 272534020Stegge 272634020Steggevoid 272734020Steggeforward_signal(struct proc *p) 272834020Stegge{ 272934020Stegge int map; 273034020Stegge int id; 273134020Stegge int i; 273234020Stegge 273334020Stegge /* Kludge. We don't yet have separate locks for the interrupts 273434020Stegge * and the kernel. This means that we cannot let the other processors 273534020Stegge * handle complex interrupts while inhibiting them from entering 273634020Stegge * the kernel in a non-interrupt context. 273734020Stegge * 273834020Stegge * What we can do, without changing the locking mechanisms yet, 273934020Stegge * is letting the other processors handle a very simple interrupt 274034020Stegge * (wich determines the processor states), and do the main 274134020Stegge * work ourself. 274234020Stegge */ 274334020Stegge 274434020Stegge if (!smp_started || !invltlb_ok || cold || panicstr) 274534020Stegge return; 274634020Stegge if (!forward_signal_enabled) 274734020Stegge return; 274834020Stegge while (1) { 274934020Stegge if (p->p_stat != SRUN) 275034020Stegge return; 275144487Sbde id = p->p_oncpu; 275234020Stegge if (id == 0xff) 275334020Stegge return; 275434020Stegge map = (1<<id); 275534020Stegge checkstate_need_ast |= map; 275634020Stegge selected_apic_ipi(map, XCPUAST_OFFSET, APIC_DELMODE_FIXED); 275734020Stegge i = 0; 275834020Stegge while ((checkstate_need_ast & map) != 0) { 275934020Stegge /* spin */ 276034020Stegge i++; 276134020Stegge if (i > 100000) { 276234020Stegge#if 0 276334020Stegge printf("forward_signal: dropped ast 0x%x\n", 276434020Stegge checkstate_need_ast & map); 276534020Stegge#endif 276634020Stegge break; 276734020Stegge } 276834020Stegge } 276944487Sbde if (id == p->p_oncpu) 277034020Stegge return; 277134020Stegge } 277234020Stegge} 277334021Stegge 277436135Steggevoid 277536135Steggeforward_roundrobin(void) 277636135Stegge{ 277736135Stegge u_int map; 277836135Stegge int i; 277934021Stegge 278036135Stegge if (!smp_started || !invltlb_ok || cold || panicstr) 278136135Stegge return; 278236135Stegge if (!forward_roundrobin_enabled) 278336135Stegge return; 278436135Stegge resched_cpus |= other_cpus; 278536135Stegge map = other_cpus & ~stopped_cpus ; 278636135Stegge#if 1 278736135Stegge selected_apic_ipi(map, XCPUAST_OFFSET, APIC_DELMODE_FIXED); 278836135Stegge#else 278936135Stegge (void) all_but_self_ipi(XCPUAST_OFFSET); 279036135Stegge#endif 279136135Stegge i = 0; 279236135Stegge while ((checkstate_need_ast & map) != 0) { 279336135Stegge /* spin */ 279436135Stegge i++; 279536135Stegge if (i > 100000) { 279636135Stegge#if 0 279736135Stegge printf("forward_roundrobin: dropped ast 0x%x\n", 279836135Stegge checkstate_need_ast & map); 279936135Stegge#endif 280036135Stegge break; 280136135Stegge } 280236135Stegge } 280336135Stegge} 280436135Stegge 280536135Stegge 280634021Stegge#ifdef APIC_INTR_REORDER 280734021Stegge/* 280834021Stegge * Maintain mapping from softintr vector to isr bit in local apic. 280934021Stegge */ 281034021Steggevoid 281134021Steggeset_lapic_isrloc(int intr, int vector) 281234021Stegge{ 281334021Stegge if (intr < 0 || intr > 32) 281434021Stegge panic("set_apic_isrloc: bad intr argument: %d",intr); 281534021Stegge if (vector < ICU_OFFSET || vector > 255) 281634021Stegge panic("set_apic_isrloc: bad vector argument: %d",vector); 281734021Stegge apic_isrbit_location[intr].location = &lapic.isr0 + ((vector>>5)<<2); 281834021Stegge apic_isrbit_location[intr].bit = (1<<(vector & 31)); 281934021Stegge} 282034021Stegge#endif 282148924Smsmith 282248924Smsmith/* 282348924Smsmith * All-CPU rendezvous. CPUs are signalled, all execute the setup function 282448924Smsmith * (if specified), rendezvous, execute the action function (if specified), 282548924Smsmith * rendezvous again, execute the teardown function (if specified), and then 282648924Smsmith * resume. 282748924Smsmith * 282848924Smsmith * Note that the supplied external functions _must_ be reentrant and aware 282948924Smsmith * that they are running in parallel and in an unknown lock context. 283048924Smsmith */ 283148924Smsmithstatic void (*smp_rv_setup_func)(void *arg); 283248924Smsmithstatic void (*smp_rv_action_func)(void *arg); 283348924Smsmithstatic void (*smp_rv_teardown_func)(void *arg); 283448924Smsmithstatic void *smp_rv_func_arg; 283548924Smsmithstatic volatile int smp_rv_waiters[2]; 283648924Smsmith 283748924Smsmithvoid 283848924Smsmithsmp_rendezvous_action(void) 283948924Smsmith{ 284048924Smsmith /* setup function */ 284148924Smsmith if (smp_rv_setup_func != NULL) 284248924Smsmith smp_rv_setup_func(smp_rv_func_arg); 284348924Smsmith /* spin on entry rendezvous */ 284448924Smsmith atomic_add_int(&smp_rv_waiters[0], 1); 284548924Smsmith while (smp_rv_waiters[0] < mp_ncpus) 284648924Smsmith ; 284748924Smsmith /* action function */ 284848924Smsmith if (smp_rv_action_func != NULL) 284948924Smsmith smp_rv_action_func(smp_rv_func_arg); 285048924Smsmith /* spin on exit rendezvous */ 285148924Smsmith atomic_add_int(&smp_rv_waiters[1], 1); 285248924Smsmith while (smp_rv_waiters[1] < mp_ncpus) 285348924Smsmith ; 285448924Smsmith /* teardown function */ 285548924Smsmith if (smp_rv_teardown_func != NULL) 285648924Smsmith smp_rv_teardown_func(smp_rv_func_arg); 285748924Smsmith} 285848924Smsmith 285948924Smsmithvoid 286048924Smsmithsmp_rendezvous(void (* setup_func)(void *), 286148924Smsmith void (* action_func)(void *), 286248924Smsmith void (* teardown_func)(void *), 286348924Smsmith void *arg) 286448924Smsmith{ 286548924Smsmith u_int efl; 286648924Smsmith 286748924Smsmith /* obtain rendezvous lock */ 286848924Smsmith s_lock(&smp_rv_lock); /* XXX sleep here? NOWAIT flag? */ 286948924Smsmith 287048924Smsmith /* set static function pointers */ 287148924Smsmith smp_rv_setup_func = setup_func; 287248924Smsmith smp_rv_action_func = action_func; 287348924Smsmith smp_rv_teardown_func = teardown_func; 287448924Smsmith smp_rv_func_arg = arg; 287548924Smsmith smp_rv_waiters[0] = 0; 287648924Smsmith smp_rv_waiters[1] = 0; 287748924Smsmith 287848924Smsmith /* disable interrupts on this CPU, save interrupt status */ 287948924Smsmith efl = read_eflags(); 288048924Smsmith write_eflags(efl & ~PSL_I); 288148924Smsmith 288248924Smsmith /* signal other processors, which will enter the IPI with interrupts off */ 288348924Smsmith all_but_self_ipi(XRENDEZVOUS_OFFSET); 288448924Smsmith 288548924Smsmith /* call executor function */ 288648924Smsmith smp_rendezvous_action(); 288748924Smsmith 288848924Smsmith /* restore interrupt flag */ 288948924Smsmith write_eflags(efl); 289048924Smsmith 289148924Smsmith /* release lock */ 289248924Smsmith s_unlock(&smp_rv_lock); 289348924Smsmith} 289465557Sjasone 289565557Sjasonevoid 289665557Sjasonerelease_aps(void *dummy __unused) 289765557Sjasone{ 289865557Sjasone s_unlock(&ap_boot_lock); 289965557Sjasone} 290065557Sjasone 290165557SjasoneSYSINIT(start_aps, SI_SUB_SMP, SI_ORDER_FIRST, release_aps, NULL); 2902