subr_smp.c revision 31639
125164Speter/* 225164Speter * Copyright (c) 1996, by Steve Passe 325164Speter * All rights reserved. 425164Speter * 525164Speter * Redistribution and use in source and binary forms, with or without 625164Speter * modification, are permitted provided that the following conditions 725164Speter * are met: 825164Speter * 1. Redistributions of source code must retain the above copyright 925164Speter * notice, this list of conditions and the following disclaimer. 1025164Speter * 2. The name of the developer may NOT be used to endorse or promote products 1125164Speter * derived from this software without specific prior written permission. 1225164Speter * 1325164Speter * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 1425164Speter * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 1525164Speter * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 1625164Speter * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 1725164Speter * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 1825164Speter * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 1925164Speter * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 2025164Speter * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 2125164Speter * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 2225164Speter * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 2325164Speter * SUCH DAMAGE. 2425164Speter * 2531639Sfsmp * $Id: mp_machdep.c,v 1.40 1997/12/04 19:30:03 smp Exp smp $ 2625164Speter */ 2725164Speter 2825164Speter#include "opt_smp.h" 2930265Speter#include "opt_vm86.h" 3025164Speter 3131639Sfsmp#ifdef SMP 3231639Sfsmp#include <machine/smptests.h> 3331639Sfsmp#else 3431639Sfsmp#error 3531639Sfsmp#endif 3631639Sfsmp 3728743Sbde#include <sys/param.h> 3825164Speter#include <sys/systm.h> 3928808Speter#include <sys/kernel.h> 4028808Speter#include <sys/proc.h> 4128808Speter#include <sys/sysctl.h> 4231639Sfsmp#ifdef BETTER_CLOCK 4331639Sfsmp#include <sys/dkstat.h> 4431639Sfsmp#endif 4525164Speter 4628808Speter#include <vm/vm.h> 4728808Speter#include <vm/vm_param.h> 4828808Speter#include <vm/pmap.h> 4926812Speter#include <vm/vm_kern.h> 5026812Speter#include <vm/vm_extern.h> 5131639Sfsmp#ifdef BETTER_CLOCK 5231639Sfsmp#include <sys/lock.h> 5331639Sfsmp#include <vm/vm_map.h> 5431639Sfsmp#include <sys/user.h> 5531639Sfsmp#endif 5625164Speter 5725164Speter#include <machine/smp.h> 5825164Speter#include <machine/apic.h> 5925164Speter#include <machine/mpapic.h> 6025164Speter#include <machine/segments.h> 6127697Sfsmp#include <machine/smptests.h> /** TEST_DEFAULT_CONFIG, TEST_TEST1 */ 6226812Speter#include <machine/tss.h> 6326896Stegge#include <machine/specialreg.h> 6430112Sdyson#include <machine/cputypes.h> 6525164Speter 6625164Speter#include <i386/i386/cons.h> /* cngetc() */ 6725164Speter 6825215Sfsmp#if defined(APIC_IO) 6927289Sfsmp#include <machine/md_var.h> /* setidt() */ 7027289Sfsmp#include <i386/isa/icu.h> /* IPIs */ 7127289Sfsmp#include <i386/isa/intr_machdep.h> /* IPIs */ 7225215Sfsmp#endif /* APIC_IO */ 7325164Speter 7428027Sfsmp#if defined(TEST_DEFAULT_CONFIG) 7528027Sfsmp#define MPFPS_MPFB1 TEST_DEFAULT_CONFIG 7628027Sfsmp#else 7728027Sfsmp#define MPFPS_MPFB1 mpfps->mpfb1 7828027Sfsmp#endif /* TEST_DEFAULT_CONFIG */ 7928027Sfsmp 8027005Sfsmp#define WARMBOOT_TARGET 0 8127005Sfsmp#define WARMBOOT_OFF (KERNBASE + 0x0467) 8227005Sfsmp#define WARMBOOT_SEG (KERNBASE + 0x0469) 8325164Speter 8427005Sfsmp#define BIOS_BASE (0xf0000) 8527005Sfsmp#define BIOS_SIZE (0x10000) 8627005Sfsmp#define BIOS_COUNT (BIOS_SIZE/4) 8725164Speter 8827005Sfsmp#define CMOS_REG (0x70) 8927005Sfsmp#define CMOS_DATA (0x71) 9027005Sfsmp#define BIOS_RESET (0x0f) 9127005Sfsmp#define BIOS_WARM (0x0a) 9225164Speter 9326155Sfsmp#define PROCENTRY_FLAG_EN 0x01 9426155Sfsmp#define PROCENTRY_FLAG_BP 0x02 9526155Sfsmp#define IOAPICENTRY_FLAG_EN 0x01 9626155Sfsmp 9727005Sfsmp 9826155Sfsmp/* MP Floating Pointer Structure */ 9926155Sfsmptypedef struct MPFPS { 10026155Sfsmp char signature[4]; 10126155Sfsmp void *pap; 10226155Sfsmp u_char length; 10326155Sfsmp u_char spec_rev; 10426155Sfsmp u_char checksum; 10526155Sfsmp u_char mpfb1; 10626155Sfsmp u_char mpfb2; 10726155Sfsmp u_char mpfb3; 10826155Sfsmp u_char mpfb4; 10926155Sfsmp u_char mpfb5; 11026155Sfsmp} *mpfps_t; 11126155Sfsmp 11226155Sfsmp/* MP Configuration Table Header */ 11326155Sfsmptypedef struct MPCTH { 11426155Sfsmp char signature[4]; 11526155Sfsmp u_short base_table_length; 11626155Sfsmp u_char spec_rev; 11726155Sfsmp u_char checksum; 11826155Sfsmp u_char oem_id[8]; 11926155Sfsmp u_char product_id[12]; 12026155Sfsmp void *oem_table_pointer; 12126155Sfsmp u_short oem_table_size; 12226155Sfsmp u_short entry_count; 12326155Sfsmp void *apic_address; 12426155Sfsmp u_short extended_table_length; 12526155Sfsmp u_char extended_table_checksum; 12626155Sfsmp u_char reserved; 12726155Sfsmp} *mpcth_t; 12826155Sfsmp 12926155Sfsmp 13026155Sfsmptypedef struct PROCENTRY { 13126155Sfsmp u_char type; 13226155Sfsmp u_char apic_id; 13326155Sfsmp u_char apic_version; 13426155Sfsmp u_char cpu_flags; 13526155Sfsmp u_long cpu_signature; 13626155Sfsmp u_long feature_flags; 13726155Sfsmp u_long reserved1; 13826155Sfsmp u_long reserved2; 13926155Sfsmp} *proc_entry_ptr; 14026155Sfsmp 14126155Sfsmptypedef struct BUSENTRY { 14226155Sfsmp u_char type; 14326155Sfsmp u_char bus_id; 14426155Sfsmp char bus_type[6]; 14526155Sfsmp} *bus_entry_ptr; 14626155Sfsmp 14726155Sfsmptypedef struct IOAPICENTRY { 14826155Sfsmp u_char type; 14926155Sfsmp u_char apic_id; 15026155Sfsmp u_char apic_version; 15126155Sfsmp u_char apic_flags; 15226155Sfsmp void *apic_address; 15326155Sfsmp} *io_apic_entry_ptr; 15426155Sfsmp 15526155Sfsmptypedef struct INTENTRY { 15626155Sfsmp u_char type; 15726155Sfsmp u_char int_type; 15826155Sfsmp u_short int_flags; 15926155Sfsmp u_char src_bus_id; 16026155Sfsmp u_char src_bus_irq; 16126155Sfsmp u_char dst_apic_id; 16226155Sfsmp u_char dst_apic_int; 16326155Sfsmp} *int_entry_ptr; 16426155Sfsmp 16526155Sfsmp/* descriptions of MP basetable entries */ 16626155Sfsmptypedef struct BASETABLE_ENTRY { 16726155Sfsmp u_char type; 16826155Sfsmp u_char length; 16926155Sfsmp char name[16]; 17026155Sfsmp} basetable_entry; 17126155Sfsmp 17225164Speter/* 17325164Speter * this code MUST be enabled here and in mpboot.s. 17425164Speter * it follows the very early stages of AP boot by placing values in CMOS ram. 17525164Speter * it NORMALLY will never be needed and thus the primitive method for enabling. 17625164Speter * 17725164Speter#define CHECK_POINTS 17825164Speter */ 17925164Speter 18025164Speter#if defined(CHECK_POINTS) 18125164Speter#define CHECK_READ(A) (outb(CMOS_REG, (A)), inb(CMOS_DATA)) 18225164Speter#define CHECK_WRITE(A,D) (outb(CMOS_REG, (A)), outb(CMOS_DATA, (D))) 18325164Speter 18425164Speter#define CHECK_INIT(D); \ 18525164Speter CHECK_WRITE(0x34, (D)); \ 18625164Speter CHECK_WRITE(0x35, (D)); \ 18725164Speter CHECK_WRITE(0x36, (D)); \ 18825164Speter CHECK_WRITE(0x37, (D)); \ 18925164Speter CHECK_WRITE(0x38, (D)); \ 19025164Speter CHECK_WRITE(0x39, (D)); 19125164Speter 19225164Speter#define CHECK_PRINT(S); \ 19325164Speter printf("%s: %d, %d, %d, %d, %d, %d\n", \ 19425164Speter (S), \ 19525164Speter CHECK_READ(0x34), \ 19625164Speter CHECK_READ(0x35), \ 19725164Speter CHECK_READ(0x36), \ 19825164Speter CHECK_READ(0x37), \ 19925164Speter CHECK_READ(0x38), \ 20025164Speter CHECK_READ(0x39)); 20125164Speter 20225164Speter#else /* CHECK_POINTS */ 20325164Speter 20425164Speter#define CHECK_INIT(D) 20525164Speter#define CHECK_PRINT(S) 20625164Speter 20725164Speter#endif /* CHECK_POINTS */ 20825164Speter 20927005Sfsmp/* 21027005Sfsmp * Values to send to the POST hardware. 21127005Sfsmp */ 21227005Sfsmp#define MP_BOOTADDRESS_POST 0x10 21327005Sfsmp#define MP_PROBE_POST 0x11 21429213Sfsmp#define MPTABLE_PASS1_POST 0x12 21529213Sfsmp 21629213Sfsmp#define MP_START_POST 0x13 21729213Sfsmp#define MP_ENABLE_POST 0x14 21827005Sfsmp#define MPTABLE_PASS2_POST 0x15 21927005Sfsmp 22029213Sfsmp#define START_ALL_APS_POST 0x16 22129213Sfsmp#define INSTALL_AP_TRAMP_POST 0x17 22229213Sfsmp#define START_AP_POST 0x18 22329213Sfsmp 22429213Sfsmp#define MP_ANNOUNCE_POST 0x19 22529213Sfsmp 22629213Sfsmp 22727289Sfsmp/** XXX FIXME: where does this really belong, isa.h/isa.c perhaps? */ 22827255Sfsmpint current_postcode; 22927255Sfsmp 23027289Sfsmp/** XXX FIXME: what system files declare these??? */ 23125164Speterextern struct region_descriptor r_gdt, r_idt; 23225164Speter 23328669Sfsmpint bsp_apic_ready = 0; /* flags useability of BSP apic */ 23426155Sfsmpint mp_ncpus; /* # of CPUs, including BSP */ 23526155Sfsmpint mp_naps; /* # of Applications processors */ 23626155Sfsmpint mp_nbusses; /* # of busses */ 23726155Sfsmpint mp_napics; /* # of IO APICs */ 23826155Sfsmpint boot_cpu_id; /* designated BSP */ 23925164Spetervm_offset_t cpu_apic_address; 24026108Sfsmpvm_offset_t io_apic_address[NAPICID]; /* NAPICID is more than enough */ 24129655Sdysonextern int nkpt; 24225164Speter 24325164Speteru_int32_t cpu_apic_versions[NCPU]; 24425164Speteru_int32_t io_apic_versions[NAPIC]; 24525164Speter 24625164Speter/* 24726108Sfsmp * APIC ID logical/physical mapping structures. 24826108Sfsmp * We oversize these to simplify boot-time config. 24925164Speter */ 25026108Sfsmpint cpu_num_to_apic_id[NAPICID]; 25126108Sfsmpint io_num_to_apic_id[NAPICID]; 25225164Speterint apic_id_to_logical[NAPICID]; 25325164Speter 25430112Sdyson 25530343Speter#define NPPROVMTRR 8 25630343Speter#define PPRO_VMTRRphysBase0 0x200 25730343Speter#define PPRO_VMTRRphysMask0 0x201 25830112Sdysonstatic struct { 25930343Speter u_int64_t base, mask; 26030112Sdyson} PPro_vmtrr[NPPROVMTRR]; 26130112Sdyson 26227005Sfsmp/* Bitmap of all available CPUs */ 26327005Sfsmpu_int all_cpus; 26427005Sfsmp 26528808Speter/* AP uses this PTD during bootstrap */ 26628808Speterpd_entry_t *bootPTD; 26726812Speter 26826812Speter/* Hotwire a 0->4MB V==P mapping */ 26928808Speterextern pt_entry_t *KPTphys; 27026812Speter 27127289Sfsmp/* Virtual address of per-cpu common_tss */ 27226812Speterextern struct i386tss common_tss; 27328984Sbde#ifdef VM86 27429663Speterextern struct segment_descriptor common_tssd; 27528984Sbdeextern u_int private_tss; /* flag indicating private tss */ 27629663Speterextern u_int my_tr; 27728984Sbde#endif /* VM86 */ 27826812Speter 27928808Speter/* IdlePTD per cpu */ 28028808Speterpd_entry_t *IdlePTDS[NCPU]; 28128808Speter 28228808Speter/* "my" private page table page, for BSP init */ 28328808Speterextern pt_entry_t SMP_prvpt[]; 28428808Speter 28528808Speter/* Private page pointer to curcpu's PTD, used during BSP init */ 28628808Speterextern pd_entry_t *my_idlePTD; 28728808Speter 28828809Speterstatic int smp_started; /* has the system started? */ 28928809Speter 29025164Speter/* 29127289Sfsmp * Local data and functions. 29225164Speter */ 29325164Speter 29426155Sfsmpstatic int mp_capable; 29526155Sfsmpstatic u_int boot_address; 29626155Sfsmpstatic u_int base_memory; 29725164Speter 29826155Sfsmpstatic int picmode; /* 0: virtual wire mode, 1: PIC mode */ 29926155Sfsmpstatic mpfps_t mpfps; 30026155Sfsmpstatic int search_for_sig(u_int32_t target, int count); 30126155Sfsmpstatic void mp_enable(u_int boot_addr); 30225164Speter 30326155Sfsmpstatic int mptable_pass1(void); 30426155Sfsmpstatic int mptable_pass2(void); 30526155Sfsmpstatic void default_mp_table(int type); 30628027Sfsmpstatic void fix_mp_table(void); 30727634Sfsmpstatic void init_locks(void); 30826155Sfsmpstatic int start_all_aps(u_int boot_addr); 30926155Sfsmpstatic void install_ap_tramp(u_int boot_addr); 31026155Sfsmpstatic int start_ap(int logicalCpu, u_int boot_addr); 31130343Speterstatic void getmtrr(void); 31230343Speterstatic void putmtrr(void); 31330343Speterstatic void putfmtrr(void); 31425164Speter 31526155Sfsmp 31625164Speter/* 31727289Sfsmp * Calculate usable address in base memory for AP trampoline code. 31825164Speter */ 31925164Speteru_int 32025164Spetermp_bootaddress(u_int basemem) 32125164Speter{ 32227005Sfsmp POSTCODE(MP_BOOTADDRESS_POST); 32327005Sfsmp 32425164Speter base_memory = basemem * 1024; /* convert to bytes */ 32525164Speter 32625164Speter boot_address = base_memory & ~0xfff; /* round down to 4k boundary */ 32725164Speter if ((base_memory - boot_address) < bootMP_size) 32825164Speter boot_address -= 4096; /* not enough, lower by 4k */ 32925164Speter 33025164Speter return boot_address; 33125164Speter} 33225164Speter 33325164Speter 33427289Sfsmp/* 33527289Sfsmp * Look for an Intel MP spec table (ie, SMP capable hardware). 33627289Sfsmp */ 33726155Sfsmpint 33826155Sfsmpmp_probe(void) 33926155Sfsmp{ 34026155Sfsmp int x; 34126155Sfsmp u_long segment; 34226155Sfsmp u_int32_t target; 34326155Sfsmp 34427005Sfsmp POSTCODE(MP_PROBE_POST); 34527005Sfsmp 34626155Sfsmp /* see if EBDA exists */ 34726155Sfsmp if (segment = (u_long) * (u_short *) (KERNBASE + 0x40e)) { 34826155Sfsmp /* search first 1K of EBDA */ 34926155Sfsmp target = (u_int32_t) (segment << 4); 35026155Sfsmp if ((x = search_for_sig(target, 1024 / 4)) >= 0) 35126155Sfsmp goto found; 35226155Sfsmp } else { 35326155Sfsmp /* last 1K of base memory, effective 'top of base' passed in */ 35426155Sfsmp target = (u_int32_t) (base_memory - 0x400); 35526155Sfsmp if ((x = search_for_sig(target, 1024 / 4)) >= 0) 35626155Sfsmp goto found; 35726155Sfsmp } 35826155Sfsmp 35926155Sfsmp /* search the BIOS */ 36026155Sfsmp target = (u_int32_t) BIOS_BASE; 36126155Sfsmp if ((x = search_for_sig(target, BIOS_COUNT)) >= 0) 36226155Sfsmp goto found; 36326155Sfsmp 36426155Sfsmp /* nothing found */ 36526155Sfsmp mpfps = (mpfps_t)0; 36626155Sfsmp mp_capable = 0; 36726155Sfsmp return 0; 36826155Sfsmp 36927289Sfsmpfound: 37026155Sfsmp /* calculate needed resources */ 37126155Sfsmp mpfps = (mpfps_t)x; 37226155Sfsmp if (mptable_pass1()) 37326155Sfsmp panic("you must reconfigure your kernel"); 37426155Sfsmp 37526155Sfsmp /* flag fact that we are running multiple processors */ 37626155Sfsmp mp_capable = 1; 37726155Sfsmp return 1; 37826155Sfsmp} 37926155Sfsmp 38026155Sfsmp 38125164Speter/* 38227289Sfsmp * Startup the SMP processors. 38325164Speter */ 38425164Spetervoid 38525164Spetermp_start(void) 38625164Speter{ 38727005Sfsmp POSTCODE(MP_START_POST); 38827005Sfsmp 38925164Speter /* look for MP capable motherboard */ 39026155Sfsmp if (mp_capable) 39125164Speter mp_enable(boot_address); 39226101Sfsmp else 39326155Sfsmp panic("MP hardware not found!"); 39425164Speter} 39525164Speter 39625164Speter 39725164Speter/* 39827289Sfsmp * Print various information about the SMP system hardware and setup. 39925164Speter */ 40025164Spetervoid 40125164Spetermp_announce(void) 40225164Speter{ 40325164Speter int x; 40425164Speter 40527005Sfsmp POSTCODE(MP_ANNOUNCE_POST); 40627005Sfsmp 40725164Speter printf("FreeBSD/SMP: Multiprocessor motherboard\n"); 40827489Sfsmp printf(" cpu0 (BSP): apic id: %2d", CPU_TO_ID(0)); 40926812Speter printf(", version: 0x%08x", cpu_apic_versions[0]); 41026812Speter printf(", at 0x%08x\n", cpu_apic_address); 41125164Speter for (x = 1; x <= mp_naps; ++x) { 41227561Sfsmp printf(" cpu%d (AP): apic id: %2d", x, CPU_TO_ID(x)); 41326812Speter printf(", version: 0x%08x", cpu_apic_versions[x]); 41426812Speter printf(", at 0x%08x\n", cpu_apic_address); 41525164Speter } 41625164Speter 41725164Speter#if defined(APIC_IO) 41825164Speter for (x = 0; x < mp_napics; ++x) { 41927489Sfsmp printf(" io%d (APIC): apic id: %2d", x, IO_TO_ID(x)); 42026812Speter printf(", version: 0x%08x", io_apic_versions[x]); 42126812Speter printf(", at 0x%08x\n", io_apic_address[x]); 42225164Speter } 42325164Speter#else 42425164Speter printf(" Warning: APIC I/O disabled\n"); 42525164Speter#endif /* APIC_IO */ 42625164Speter} 42725164Speter 42825164Speter/* 42925164Speter * AP cpu's call this to sync up protected mode. 43025164Speter */ 43125164Spetervoid 43225164Speterinit_secondary(void) 43325164Speter{ 43429663Speter int gsel_tss; 43529663Speter#ifndef VM86 43629663Speter u_int my_tr; 43729663Speter#endif 43825164Speter 43925164Speter r_gdt.rd_limit = sizeof(gdt[0]) * (NGDT + NCPU) - 1; 44025164Speter r_gdt.rd_base = (int) gdt; 44127289Sfsmp lgdt(&r_gdt); /* does magic intra-segment return */ 44225164Speter lidt(&r_idt); 44325164Speter lldt(_default_ldt); 44425164Speter 44529663Speter my_tr = NGDT + cpuid; 44629663Speter gsel_tss = GSEL(my_tr, SEL_KPL); 44729663Speter gdt[my_tr].sd.sd_type = SDT_SYS386TSS; 44826812Speter common_tss.tss_esp0 = 0; /* not used until after switch */ 44926812Speter common_tss.tss_ss0 = GSEL(GDATA_SEL, SEL_KPL); 45026812Speter common_tss.tss_ioopt = (sizeof common_tss) << 16; 45128984Sbde#ifdef VM86 45229663Speter common_tssd = gdt[my_tr].sd; 45328984Sbde private_tss = 0; 45428984Sbde#endif /* VM86 */ 45525164Speter ltr(gsel_tss); 45625164Speter 45727289Sfsmp load_cr0(0x8005003b); /* XXX! */ 45826812Speter 45926812Speter PTD[0] = 0; 46027484Sdyson pmap_set_opt((unsigned *)PTD); 46127484Sdyson 46230112Sdyson putmtrr(); 46330112Sdyson putfmtrr(); 46430112Sdyson 46526812Speter invltlb(); 46625164Speter} 46725164Speter 46825164Speter 46925164Speter#if defined(APIC_IO) 47027289Sfsmp/* 47127289Sfsmp * Final configuration of the BSP's local APIC: 47227289Sfsmp * - disable 'pic mode'. 47327289Sfsmp * - disable 'virtual wire mode'. 47427289Sfsmp * - enable NMI. 47527289Sfsmp */ 47625164Spetervoid 47727289Sfsmpbsp_apic_configure(void) 47825164Speter{ 47927289Sfsmp u_char byte; 48027289Sfsmp u_int32_t temp; 48125164Speter 48227289Sfsmp /* leave 'pic mode' if necessary */ 48325164Speter if (picmode) { 48425164Speter outb(0x22, 0x70); /* select IMCR */ 48525164Speter byte = inb(0x23); /* current contents */ 48627289Sfsmp byte |= 0x01; /* mask external INTR */ 48725164Speter outb(0x23, byte); /* disconnect 8259s/NMI */ 48825164Speter } 48927001Sfsmp 49027001Sfsmp /* mask lint0 (the 8259 'virtual wire' connection) */ 49126812Speter temp = lapic.lvt_lint0; 49227289Sfsmp temp |= APIC_LVT_M; /* set the mask */ 49326812Speter lapic.lvt_lint0 = temp; 49427001Sfsmp 49527001Sfsmp /* setup lint1 to handle NMI */ 49627001Sfsmp temp = lapic.lvt_lint1; 49727289Sfsmp temp &= ~APIC_LVT_M; /* clear the mask */ 49827289Sfsmp lapic.lvt_lint1 = temp; 49927001Sfsmp 50027353Sfsmp if (bootverbose) 50127561Sfsmp apic_dump("bsp_apic_configure()"); 50225164Speter} 50327005Sfsmp#endif /* APIC_IO */ 50425164Speter 50525164Speter 50625164Speter/******************************************************************* 50725164Speter * local functions and data 50825164Speter */ 50925164Speter 51025164Speter/* 51125164Speter * start the SMP system 51225164Speter */ 51325164Speterstatic void 51425164Spetermp_enable(u_int boot_addr) 51525164Speter{ 51625164Speter int x; 51725164Speter#if defined(APIC_IO) 51825164Speter int apic; 51925164Speter u_int ux; 52025164Speter#endif /* APIC_IO */ 52125164Speter 52230112Sdyson getmtrr(); 52330112Sdyson putfmtrr(); 52430112Sdyson 52527005Sfsmp POSTCODE(MP_ENABLE_POST); 52627005Sfsmp 52727289Sfsmp /* turn on 4MB of V == P addressing so we can get to MP table */ 52826812Speter *(int *)PTD = PG_V | PG_RW | ((u_long)KPTphys & PG_FRAME); 52926812Speter invltlb(); 53026812Speter 53126812Speter /* examine the MP table for needed info, uses physical addresses */ 53226155Sfsmp x = mptable_pass2(); 53325164Speter 53426812Speter *(int *)PTD = 0; 53526812Speter invltlb(); 53625164Speter 53725164Speter /* can't process default configs till the CPU APIC is pmapped */ 53825164Speter if (x) 53925164Speter default_mp_table(x); 54025164Speter 54128027Sfsmp /* post scan cleanup */ 54228027Sfsmp fix_mp_table(); 54328027Sfsmp 54425164Speter#if defined(APIC_IO) 54527353Sfsmp 54625164Speter /* fill the LOGICAL io_apic_versions table */ 54725164Speter for (apic = 0; apic < mp_napics; ++apic) { 54825164Speter ux = io_apic_read(apic, IOAPIC_VER); 54925164Speter io_apic_versions[apic] = ux; 55025164Speter } 55125164Speter 55225216Sfsmp /* program each IO APIC in the system */ 55325164Speter for (apic = 0; apic < mp_napics; ++apic) 55426266Speter if (io_apic_setup(apic) < 0) 55526266Speter panic("IO APIC setup failure"); 55625164Speter 55727353Sfsmp /* install a 'Spurious INTerrupt' vector */ 55827353Sfsmp setidt(XSPURIOUSINT_OFFSET, Xspuriousint, 55927353Sfsmp SDT_SYS386IGT, SEL_KPL, GSEL(GCODE_SEL, SEL_KPL)); 56027353Sfsmp 56125204Sfsmp /* install an inter-CPU IPI for TLB invalidation */ 56227005Sfsmp setidt(XINVLTLB_OFFSET, Xinvltlb, 56325216Sfsmp SDT_SYS386IGT, SEL_KPL, GSEL(GCODE_SEL, SEL_KPL)); 56427005Sfsmp 56531639Sfsmp#ifdef BETTER_CLOCK 56631639Sfsmp /* install an inter-CPU IPI for reading processor state */ 56731639Sfsmp setidt(XCPUCHECKSTATE_OFFSET, Xcpucheckstate, 56831639Sfsmp SDT_SYS386IGT, SEL_KPL, GSEL(GCODE_SEL, SEL_KPL)); 56931639Sfsmp 57031639Sfsmp /* install an inter-CPU IPI for forcing an additional software trap */ 57131639Sfsmp setidt(XCPUAST_OFFSET, Xcpuast, 57231639Sfsmp SDT_SYS386IGT, SEL_KPL, GSEL(GCODE_SEL, SEL_KPL)); 57331639Sfsmp#endif 57431639Sfsmp 57527005Sfsmp /* install an inter-CPU IPI for CPU stop/restart */ 57627005Sfsmp setidt(XCPUSTOP_OFFSET, Xcpustop, 57727005Sfsmp SDT_SYS386IGT, SEL_KPL, GSEL(GCODE_SEL, SEL_KPL)); 57827255Sfsmp 57927353Sfsmp#if defined(TEST_TEST1) 58027517Sfsmp /* install a "fake hardware INTerrupt" vector */ 58127353Sfsmp setidt(XTEST1_OFFSET, Xtest1, 58227255Sfsmp SDT_SYS386IGT, SEL_KPL, GSEL(GCODE_SEL, SEL_KPL)); 58327353Sfsmp#endif /** TEST_TEST1 */ 58427353Sfsmp 58525164Speter#endif /* APIC_IO */ 58625164Speter 58727634Sfsmp /* initialize all SMP locks */ 58827634Sfsmp init_locks(); 58927634Sfsmp 59025164Speter /* start each Application Processor */ 59125164Speter start_all_aps(boot_addr); 59226896Stegge 59326896Stegge /* 59426896Stegge * The init process might be started on a different CPU now, 59526896Stegge * and the boot CPU might not call prepare_usermode to get 59626896Stegge * cr0 correctly configured. Thus we initialize cr0 here. 59726896Stegge */ 59826896Stegge load_cr0(rcr0() | CR0_WP | CR0_AM); 59925164Speter} 60025164Speter 60125164Speter 60225164Speter/* 60325164Speter * look for the MP spec signature 60425164Speter */ 60525164Speter 60625164Speter/* string defined by the Intel MP Spec as identifying the MP table */ 60725164Speter#define MP_SIG 0x5f504d5f /* _MP_ */ 60825164Speter#define NEXT(X) ((X) += 4) 60925164Speterstatic int 61025164Spetersearch_for_sig(u_int32_t target, int count) 61125164Speter{ 61225164Speter int x; 61325164Speter u_int32_t *addr = (u_int32_t *) (KERNBASE + target); 61425164Speter 61525164Speter for (x = 0; x < count; NEXT(x)) 61625164Speter if (addr[x] == MP_SIG) 61725164Speter /* make array index a byte index */ 61825164Speter return (target + (x * sizeof(u_int32_t))); 61925164Speter 62025164Speter return -1; 62125164Speter} 62225164Speter 62325164Speter 62425164Speterstatic basetable_entry basetable_entry_types[] = 62525164Speter{ 62625164Speter {0, 20, "Processor"}, 62725164Speter {1, 8, "Bus"}, 62825164Speter {2, 8, "I/O APIC"}, 62925164Speter {3, 8, "I/O INT"}, 63025164Speter {4, 8, "Local INT"} 63125164Speter}; 63225164Speter 63325164Spetertypedef struct BUSDATA { 63425164Speter u_char bus_id; 63525164Speter enum busTypes bus_type; 63625164Speter} bus_datum; 63725164Speter 63825164Spetertypedef struct INTDATA { 63925164Speter u_char int_type; 64025164Speter u_short int_flags; 64125164Speter u_char src_bus_id; 64225164Speter u_char src_bus_irq; 64325164Speter u_char dst_apic_id; 64425164Speter u_char dst_apic_int; 64525164Speter} io_int, local_int; 64625164Speter 64725164Spetertypedef struct BUSTYPENAME { 64825164Speter u_char type; 64925164Speter char name[7]; 65025164Speter} bus_type_name; 65125164Speter 65225164Speterstatic bus_type_name bus_type_table[] = 65325164Speter{ 65425164Speter {CBUS, "CBUS"}, 65525164Speter {CBUSII, "CBUSII"}, 65625164Speter {EISA, "EISA"}, 65725164Speter {UNKNOWN_BUSTYPE, "---"}, 65825164Speter {UNKNOWN_BUSTYPE, "---"}, 65925164Speter {ISA, "ISA"}, 66025164Speter {UNKNOWN_BUSTYPE, "---"}, 66125164Speter {UNKNOWN_BUSTYPE, "---"}, 66225164Speter {UNKNOWN_BUSTYPE, "---"}, 66325164Speter {UNKNOWN_BUSTYPE, "---"}, 66425164Speter {UNKNOWN_BUSTYPE, "---"}, 66525164Speter {UNKNOWN_BUSTYPE, "---"}, 66625164Speter {PCI, "PCI"}, 66725164Speter {UNKNOWN_BUSTYPE, "---"}, 66825164Speter {UNKNOWN_BUSTYPE, "---"}, 66925164Speter {UNKNOWN_BUSTYPE, "---"}, 67025164Speter {UNKNOWN_BUSTYPE, "---"}, 67125164Speter {XPRESS, "XPRESS"}, 67225164Speter {UNKNOWN_BUSTYPE, "---"} 67325164Speter}; 67425164Speter/* from MP spec v1.4, table 5-1 */ 67525164Speterstatic int default_data[7][5] = 67625164Speter{ 67725164Speter/* nbus, id0, type0, id1, type1 */ 67825164Speter {1, 0, ISA, 255, 255}, 67925164Speter {1, 0, EISA, 255, 255}, 68025164Speter {1, 0, EISA, 255, 255}, 68125164Speter {0, 255, 255, 255, 255},/* MCA not supported */ 68225164Speter {2, 0, ISA, 1, PCI}, 68325164Speter {2, 0, EISA, 1, PCI}, 68425164Speter {0, 255, 255, 255, 255} /* MCA not supported */ 68525164Speter}; 68625164Speter 68725164Speter 68825164Speter/* the bus data */ 68925164Speterbus_datum bus_data[NBUS]; 69025164Speter 69125164Speter/* the IO INT data, one entry per possible APIC INTerrupt */ 69225164Speterio_int io_apic_ints[NINTR]; 69325164Speter 69425164Speterstatic int nintrs; 69525164Speter 69626108Sfsmpstatic int processor_entry __P((proc_entry_ptr entry, int cpu)); 69726108Sfsmpstatic int bus_entry __P((bus_entry_ptr entry, int bus)); 69826108Sfsmpstatic int io_apic_entry __P((io_apic_entry_ptr entry, int apic)); 69926108Sfsmpstatic int int_entry __P((int_entry_ptr entry, int intr)); 70026108Sfsmpstatic int lookup_bus_type __P((char *name)); 70125164Speter 70225164Speter 70325164Speter/* 70426155Sfsmp * 1st pass on motherboard's Intel MP specification table. 70526155Sfsmp * 70626155Sfsmp * initializes: 70726155Sfsmp * mp_ncpus = 1 70826155Sfsmp * 70926155Sfsmp * determines: 71026155Sfsmp * cpu_apic_address (common to all CPUs) 71126155Sfsmp * io_apic_address[N] 71226155Sfsmp * mp_naps 71326155Sfsmp * mp_nbusses 71426155Sfsmp * mp_napics 71526155Sfsmp * nintrs 71625164Speter */ 71725164Speterstatic int 71826155Sfsmpmptable_pass1(void) 71925164Speter{ 72026108Sfsmp int x; 72126108Sfsmp mpcth_t cth; 72226108Sfsmp int totalSize; 72326108Sfsmp void* position; 72426108Sfsmp int count; 72526108Sfsmp int type; 72626108Sfsmp int mustpanic; 72725164Speter 72827005Sfsmp POSTCODE(MPTABLE_PASS1_POST); 72927005Sfsmp 73026108Sfsmp mustpanic = 0; 73126108Sfsmp 73226155Sfsmp /* clear various tables */ 73326155Sfsmp for (x = 0; x < NAPICID; ++x) { 73426155Sfsmp io_apic_address[x] = ~0; /* IO APIC address table */ 73526155Sfsmp } 73625164Speter 73726108Sfsmp /* init everything to empty */ 73826108Sfsmp mp_naps = 0; 73926108Sfsmp mp_nbusses = 0; 74026108Sfsmp mp_napics = 0; 74126108Sfsmp nintrs = 0; 74226108Sfsmp 74326108Sfsmp /* check for use of 'default' configuration */ 74428027Sfsmp if (MPFPS_MPFB1 != 0) { 74526108Sfsmp /* use default addresses */ 74626108Sfsmp cpu_apic_address = DEFAULT_APIC_BASE; 74726108Sfsmp io_apic_address[0] = DEFAULT_IO_APIC_BASE; 74826108Sfsmp 74926108Sfsmp /* fill in with defaults */ 75026882Sfsmp mp_naps = 2; /* includes BSP */ 75128027Sfsmp mp_nbusses = default_data[MPFPS_MPFB1 - 1][0]; 75226108Sfsmp#if defined(APIC_IO) 75326108Sfsmp mp_napics = 1; 75426108Sfsmp nintrs = 16; 75526108Sfsmp#endif /* APIC_IO */ 75626108Sfsmp } 75726108Sfsmp else { 75826155Sfsmp if ((cth = mpfps->pap) == 0) 75926108Sfsmp panic("MP Configuration Table Header MISSING!"); 76026108Sfsmp 76126108Sfsmp cpu_apic_address = (vm_offset_t) cth->apic_address; 76226108Sfsmp 76326108Sfsmp /* walk the table, recording info of interest */ 76426108Sfsmp totalSize = cth->base_table_length - sizeof(struct MPCTH); 76526108Sfsmp position = (u_char *) cth + sizeof(struct MPCTH); 76626108Sfsmp count = cth->entry_count; 76726108Sfsmp 76826108Sfsmp while (count--) { 76926108Sfsmp switch (type = *(u_char *) position) { 77026108Sfsmp case 0: /* processor_entry */ 77126108Sfsmp if (((proc_entry_ptr)position)->cpu_flags 77226108Sfsmp & PROCENTRY_FLAG_EN) 77326108Sfsmp ++mp_naps; 77426108Sfsmp break; 77526108Sfsmp case 1: /* bus_entry */ 77626108Sfsmp ++mp_nbusses; 77726108Sfsmp break; 77826108Sfsmp case 2: /* io_apic_entry */ 77926108Sfsmp if (((io_apic_entry_ptr)position)->apic_flags 78026108Sfsmp & IOAPICENTRY_FLAG_EN) 78126108Sfsmp io_apic_address[mp_napics++] = 78226108Sfsmp (vm_offset_t)((io_apic_entry_ptr) 78326108Sfsmp position)->apic_address; 78426108Sfsmp break; 78526108Sfsmp case 3: /* int_entry */ 78626108Sfsmp ++nintrs; 78726108Sfsmp break; 78826108Sfsmp case 4: /* int_entry */ 78926108Sfsmp break; 79026108Sfsmp default: 79126108Sfsmp panic("mpfps Base Table HOSED!"); 79226108Sfsmp /* NOTREACHED */ 79326108Sfsmp } 79426108Sfsmp 79526108Sfsmp totalSize -= basetable_entry_types[type].length; 79626108Sfsmp (u_char*)position += basetable_entry_types[type].length; 79726108Sfsmp } 79826108Sfsmp } 79926108Sfsmp 80026108Sfsmp /* qualify the numbers */ 80126108Sfsmp if (mp_naps > NCPU) 80228041Sfsmp#if 0 /* XXX FIXME: kern/4255 */ 80326108Sfsmp printf("Warning: only using %d of %d available CPUs!\n", 80426108Sfsmp NCPU, mp_naps); 80528041Sfsmp#else 80628041Sfsmp { 80728041Sfsmp printf("NCPU cannot be different than actual CPU count.\n"); 80828041Sfsmp printf(" add 'options NCPU=%d' to your kernel config file,\n", 80928041Sfsmp mp_naps); 81028041Sfsmp printf(" then rerun config & rebuild your SMP kernel\n"); 81126108Sfsmp mustpanic = 1; 81228041Sfsmp } 81328041Sfsmp#endif /* XXX FIXME: kern/4255 */ 81426108Sfsmp if (mp_nbusses > NBUS) { 81526108Sfsmp printf("found %d busses, increase NBUS\n", mp_nbusses); 81626108Sfsmp mustpanic = 1; 81726108Sfsmp } 81826108Sfsmp if (mp_napics > NAPIC) { 81926108Sfsmp printf("found %d apics, increase NAPIC\n", mp_napics); 82026108Sfsmp mustpanic = 1; 82126108Sfsmp } 82226108Sfsmp if (nintrs > NINTR) { 82326108Sfsmp printf("found %d intrs, increase NINTR\n", nintrs); 82426108Sfsmp mustpanic = 1; 82526108Sfsmp } 82626108Sfsmp 82726108Sfsmp /* 82826108Sfsmp * Count the BSP. 82926108Sfsmp * This is also used as a counter while starting the APs. 83026108Sfsmp */ 83126108Sfsmp mp_ncpus = 1; 83226108Sfsmp 83326108Sfsmp --mp_naps; /* subtract the BSP */ 83426108Sfsmp 83526108Sfsmp return mustpanic; 83626108Sfsmp} 83726108Sfsmp 83826108Sfsmp 83926108Sfsmp/* 84026155Sfsmp * 2nd pass on motherboard's Intel MP specification table. 84126155Sfsmp * 84226155Sfsmp * sets: 84326155Sfsmp * boot_cpu_id 84426155Sfsmp * ID_TO_IO(N), phy APIC ID to log CPU/IO table 84526155Sfsmp * CPU_TO_ID(N), logical CPU to APIC ID table 84626155Sfsmp * IO_TO_ID(N), logical IO to APIC ID table 84726155Sfsmp * bus_data[N] 84826155Sfsmp * io_apic_ints[N] 84926108Sfsmp */ 85026108Sfsmpstatic int 85126155Sfsmpmptable_pass2(void) 85226108Sfsmp{ 85326108Sfsmp int x; 85426108Sfsmp mpcth_t cth; 85526108Sfsmp int totalSize; 85626108Sfsmp void* position; 85726108Sfsmp int count; 85826108Sfsmp int type; 85926108Sfsmp int apic, bus, cpu, intr; 86026108Sfsmp 86127005Sfsmp POSTCODE(MPTABLE_PASS2_POST); 86227005Sfsmp 86326155Sfsmp /* clear various tables */ 86426155Sfsmp for (x = 0; x < NAPICID; ++x) { 86526155Sfsmp ID_TO_IO(x) = -1; /* phy APIC ID to log CPU/IO table */ 86626155Sfsmp CPU_TO_ID(x) = -1; /* logical CPU to APIC ID table */ 86726155Sfsmp IO_TO_ID(x) = -1; /* logical IO to APIC ID table */ 86826155Sfsmp } 86926155Sfsmp 87025164Speter /* clear bus data table */ 87125164Speter for (x = 0; x < NBUS; ++x) 87225164Speter bus_data[x].bus_id = 0xff; 87325164Speter 87425164Speter /* clear IO APIC INT table */ 87525164Speter for (x = 0; x < NINTR; ++x) 87625164Speter io_apic_ints[x].int_type = 0xff; 87725164Speter 87825164Speter /* setup the cpu/apic mapping arrays */ 87925164Speter boot_cpu_id = -1; 88025164Speter 88125164Speter /* record whether PIC or virtual-wire mode */ 88226155Sfsmp picmode = (mpfps->mpfb2 & 0x80) ? 1 : 0; 88325164Speter 88425164Speter /* check for use of 'default' configuration */ 88528027Sfsmp if (MPFPS_MPFB1 != 0) 88628027Sfsmp return MPFPS_MPFB1; /* return default configuration type */ 88725164Speter 88826155Sfsmp if ((cth = mpfps->pap) == 0) 88926108Sfsmp panic("MP Configuration Table Header MISSING!"); 89025164Speter 89126108Sfsmp /* walk the table, recording info of interest */ 89225164Speter totalSize = cth->base_table_length - sizeof(struct MPCTH); 89325164Speter position = (u_char *) cth + sizeof(struct MPCTH); 89425164Speter count = cth->entry_count; 89526108Sfsmp apic = bus = intr = 0; 89626108Sfsmp cpu = 1; /* pre-count the BSP */ 89725164Speter 89825164Speter while (count--) { 89925164Speter switch (type = *(u_char *) position) { 90025164Speter case 0: 90126108Sfsmp if (processor_entry(position, cpu)) 90226108Sfsmp ++cpu; 90325164Speter break; 90425164Speter case 1: 90526108Sfsmp if (bus_entry(position, bus)) 90626108Sfsmp ++bus; 90725164Speter break; 90825164Speter case 2: 90926108Sfsmp if (io_apic_entry(position, apic)) 91026108Sfsmp ++apic; 91125164Speter break; 91225164Speter case 3: 91326108Sfsmp if (int_entry(position, intr)) 91426108Sfsmp ++intr; 91525164Speter break; 91625164Speter case 4: 91725164Speter /* int_entry(position); */ 91825164Speter break; 91925164Speter default: 92026108Sfsmp panic("mpfps Base Table HOSED!"); 92125164Speter /* NOTREACHED */ 92225164Speter } 92325164Speter 92425164Speter totalSize -= basetable_entry_types[type].length; 92525164Speter (u_char *) position += basetable_entry_types[type].length; 92625164Speter } 92725164Speter 92826101Sfsmp if (boot_cpu_id == -1) 92926108Sfsmp panic("NO BSP found!"); 93025164Speter 93125164Speter /* report fact that its NOT a default configuration */ 93225164Speter return 0; 93325164Speter} 93425164Speter 93525164Speter 93625164Speter/* 93725164Speter * parse an Intel MP specification table 93825164Speter */ 93925164Speterstatic void 94025164Speterfix_mp_table(void) 94125164Speter{ 94225292Sfsmp int x; 94325292Sfsmp int id; 94425292Sfsmp int bus_0; 94525292Sfsmp int bus_pci; 94625292Sfsmp int num_pci_bus; 94725164Speter 94825164Speter /* 94925164Speter * Fix mis-numbering of the PCI bus and its INT entries if the BIOS 95025164Speter * did it wrong. The MP spec says that when more than 1 PCI bus 95125164Speter * exists the BIOS must begin with bus entries for the PCI bus and use 95225164Speter * actual PCI bus numbering. This implies that when only 1 PCI bus 95325164Speter * exists the BIOS can choose to ignore this ordering, and indeed many 95425164Speter * MP motherboards do ignore it. This causes a problem when the PCI 95525164Speter * sub-system makes requests of the MP sub-system based on PCI bus 95625164Speter * numbers. So here we look for the situation and renumber the 95725164Speter * busses and associated INTs in an effort to "make it right". 95825164Speter */ 95925164Speter 96025292Sfsmp /* find bus 0, PCI bus, count the number of PCI busses */ 96125164Speter for (num_pci_bus = 0, x = 0; x < mp_nbusses; ++x) { 96225292Sfsmp if (bus_data[x].bus_id == 0) { 96325292Sfsmp bus_0 = x; 96425292Sfsmp } 96525292Sfsmp if (bus_data[x].bus_type == PCI) { 96625164Speter ++num_pci_bus; 96725292Sfsmp bus_pci = x; 96825292Sfsmp } 96925164Speter } 97025292Sfsmp /* 97125292Sfsmp * bus_0 == slot of bus with ID of 0 97225292Sfsmp * bus_pci == slot of last PCI bus encountered 97325292Sfsmp */ 97425164Speter 97525164Speter /* check the 1 PCI bus case for sanity */ 97625164Speter if (num_pci_bus == 1) { 97725164Speter 97825292Sfsmp /* if it is number 0 all is well */ 97925292Sfsmp if (bus_data[bus_pci].bus_id == 0) 98025164Speter return; 98125164Speter 98225164Speter /* mis-numbered, swap with whichever bus uses slot 0 */ 98325164Speter 98425292Sfsmp /* swap the bus entry types */ 98525292Sfsmp bus_data[bus_pci].bus_type = bus_data[bus_0].bus_type; 98625292Sfsmp bus_data[bus_0].bus_type = PCI; 98725164Speter 98825164Speter /* swap each relavant INTerrupt entry */ 98925292Sfsmp id = bus_data[bus_pci].bus_id; 99025292Sfsmp for (x = 0; x < nintrs; ++x) { 99125292Sfsmp if (io_apic_ints[x].src_bus_id == id) { 99225292Sfsmp io_apic_ints[x].src_bus_id = 0; 99325292Sfsmp } 99425292Sfsmp else if (io_apic_ints[x].src_bus_id == 0) { 99525292Sfsmp io_apic_ints[x].src_bus_id = id; 99625292Sfsmp } 99725164Speter } 99825164Speter } 99925164Speter /* sanity check if more than 1 PCI bus */ 100025292Sfsmp else if (num_pci_bus > 1) { 100125292Sfsmp for (x = 0; x < mp_nbusses; ++x) { 100225292Sfsmp if (bus_data[x].bus_type != PCI) 100325292Sfsmp continue; 100426812Speter if (bus_data[x].bus_id >= num_pci_bus) 100526108Sfsmp panic("bad PCI bus numbering"); 100625164Speter } 100725292Sfsmp } 100825164Speter} 100925164Speter 101025164Speter 101126108Sfsmpstatic int 101226108Sfsmpprocessor_entry(proc_entry_ptr entry, int cpu) 101325164Speter{ 101425164Speter /* check for usability */ 101526108Sfsmp if ((cpu >= NCPU) || !(entry->cpu_flags & PROCENTRY_FLAG_EN)) 101626108Sfsmp return 0; 101725164Speter 101825164Speter /* check for BSP flag */ 101925164Speter if (entry->cpu_flags & PROCENTRY_FLAG_BP) { 102025164Speter boot_cpu_id = entry->apic_id; 102126108Sfsmp CPU_TO_ID(0) = entry->apic_id; 102226108Sfsmp ID_TO_CPU(entry->apic_id) = 0; 102326108Sfsmp return 0; /* its already been counted */ 102425164Speter } 102525164Speter 102626108Sfsmp /* add another AP to list, if less than max number of CPUs */ 102726108Sfsmp else { 102826108Sfsmp CPU_TO_ID(cpu) = entry->apic_id; 102926108Sfsmp ID_TO_CPU(entry->apic_id) = cpu; 103026108Sfsmp return 1; 103126108Sfsmp } 103225164Speter} 103325164Speter 103425164Speter 103526108Sfsmpstatic int 103626108Sfsmpbus_entry(bus_entry_ptr entry, int bus) 103725164Speter{ 103826108Sfsmp int x; 103926108Sfsmp char c, name[8]; 104025164Speter 104125164Speter /* encode the name into an index */ 104226108Sfsmp for (x = 0; x < 6; ++x) { 104326108Sfsmp if ((c = entry->bus_type[x]) == ' ') 104425164Speter break; 104526108Sfsmp name[x] = c; 104625164Speter } 104726108Sfsmp name[x] = '\0'; 104825164Speter 104926108Sfsmp if ((x = lookup_bus_type(name)) == UNKNOWN_BUSTYPE) 105026108Sfsmp panic("unknown bus type: '%s'", name); 105125164Speter 105226108Sfsmp bus_data[bus].bus_id = entry->bus_id; 105326108Sfsmp bus_data[bus].bus_type = x; 105426108Sfsmp 105526108Sfsmp return 1; 105625164Speter} 105725164Speter 105825164Speter 105926108Sfsmpstatic int 106026108Sfsmpio_apic_entry(io_apic_entry_ptr entry, int apic) 106125164Speter{ 106225164Speter if (!(entry->apic_flags & IOAPICENTRY_FLAG_EN)) 106326108Sfsmp return 0; 106425164Speter 106526108Sfsmp IO_TO_ID(apic) = entry->apic_id; 106626108Sfsmp ID_TO_IO(entry->apic_id) = apic; 106725164Speter 106826108Sfsmp return 1; 106925164Speter} 107025164Speter 107125164Speter 107225164Speterstatic int 107325164Speterlookup_bus_type(char *name) 107425164Speter{ 107525164Speter int x; 107625164Speter 107725164Speter for (x = 0; x < MAX_BUSTYPE; ++x) 107825164Speter if (strcmp(bus_type_table[x].name, name) == 0) 107925164Speter return bus_type_table[x].type; 108025164Speter 108125164Speter return UNKNOWN_BUSTYPE; 108225164Speter} 108325164Speter 108425164Speter 108526108Sfsmpstatic int 108626108Sfsmpint_entry(int_entry_ptr entry, int intr) 108725164Speter{ 108826108Sfsmp io_apic_ints[intr].int_type = entry->int_type; 108926108Sfsmp io_apic_ints[intr].int_flags = entry->int_flags; 109026108Sfsmp io_apic_ints[intr].src_bus_id = entry->src_bus_id; 109126108Sfsmp io_apic_ints[intr].src_bus_irq = entry->src_bus_irq; 109226108Sfsmp io_apic_ints[intr].dst_apic_id = entry->dst_apic_id; 109326108Sfsmp io_apic_ints[intr].dst_apic_int = entry->dst_apic_int; 109425164Speter 109526108Sfsmp return 1; 109625164Speter} 109725164Speter 109825164Speter 109925164Speterstatic int 110025164Speterapic_int_is_bus_type(int intr, int bus_type) 110125164Speter{ 110225164Speter int bus; 110325164Speter 110425164Speter for (bus = 0; bus < mp_nbusses; ++bus) 110525164Speter if ((bus_data[bus].bus_id == io_apic_ints[intr].src_bus_id) 110625164Speter && ((int) bus_data[bus].bus_type == bus_type)) 110725164Speter return 1; 110825164Speter 110925164Speter return 0; 111025164Speter} 111125164Speter 111225164Speter 111325164Speter/* 111426950Sfsmp * Given a traditional ISA INT mask, return an APIC mask. 111525164Speter */ 111625499Sfsmpu_int 111726950Sfsmpisa_apic_mask(u_int isa_mask) 111825419Sfsmp{ 111926950Sfsmp int isa_irq; 112026950Sfsmp int apic_pin; 112125419Sfsmp 112227255Sfsmp#if defined(SKIP_IRQ15_REDIRECT) 112327255Sfsmp if (isa_mask == (1 << 15)) { 112427255Sfsmp printf("skipping ISA IRQ15 redirect\n"); 112527255Sfsmp return isa_mask; 112627255Sfsmp } 112727255Sfsmp#endif /* SKIP_IRQ15_REDIRECT */ 112827255Sfsmp 112926950Sfsmp isa_irq = ffs(isa_mask); /* find its bit position */ 113026950Sfsmp if (isa_irq == 0) /* doesn't exist */ 113125419Sfsmp return 0; 113226950Sfsmp --isa_irq; /* make it zero based */ 113325419Sfsmp 113426950Sfsmp apic_pin = isa_apic_pin(isa_irq); /* look for APIC connection */ 113526950Sfsmp if (apic_pin == -1) 113626950Sfsmp return 0; 113725419Sfsmp 113826950Sfsmp return (1 << apic_pin); /* convert pin# to a mask */ 113925419Sfsmp} 114025419Sfsmp 114125419Sfsmp 114225419Sfsmp/* 114326950Sfsmp * Determine which APIC pin an ISA/EISA INT is attached to. 114425164Speter */ 114526950Sfsmp#define INTTYPE(I) (io_apic_ints[(I)].int_type) 114626950Sfsmp#define INTPIN(I) (io_apic_ints[(I)].dst_apic_int) 114726950Sfsmp 114825164Speter#define SRCBUSIRQ(I) (io_apic_ints[(I)].src_bus_irq) 114925164Speterint 115026950Sfsmpisa_apic_pin(int isa_irq) 115125164Speter{ 115225164Speter int intr; 115325164Speter 115426950Sfsmp for (intr = 0; intr < nintrs; ++intr) { /* check each record */ 115526950Sfsmp if (INTTYPE(intr) == 0) { /* standard INT */ 115626950Sfsmp if (SRCBUSIRQ(intr) == isa_irq) { 115726950Sfsmp if (apic_int_is_bus_type(intr, ISA) || 115826950Sfsmp apic_int_is_bus_type(intr, EISA)) 115926950Sfsmp return INTPIN(intr); /* found */ 116026950Sfsmp } 116126950Sfsmp } 116226950Sfsmp } 116326950Sfsmp return -1; /* NOT found */ 116425164Speter} 116525164Speter#undef SRCBUSIRQ 116625164Speter 116725164Speter 116825164Speter/* 116926950Sfsmp * Determine which APIC pin a PCI INT is attached to. 117025164Speter */ 117125164Speter#define SRCBUSID(I) (io_apic_ints[(I)].src_bus_id) 117225164Speter#define SRCBUSDEVICE(I) ((io_apic_ints[(I)].src_bus_irq >> 2) & 0x1f) 117325164Speter#define SRCBUSLINE(I) (io_apic_ints[(I)].src_bus_irq & 0x03) 117425164Speterint 117526950Sfsmppci_apic_pin(int pciBus, int pciDevice, int pciInt) 117625164Speter{ 117725164Speter int intr; 117825164Speter 117926950Sfsmp --pciInt; /* zero based */ 118025164Speter 118126950Sfsmp for (intr = 0; intr < nintrs; ++intr) /* check each record */ 118226950Sfsmp if ((INTTYPE(intr) == 0) /* standard INT */ 118325164Speter && (SRCBUSID(intr) == pciBus) 118425164Speter && (SRCBUSDEVICE(intr) == pciDevice) 118525164Speter && (SRCBUSLINE(intr) == pciInt)) /* a candidate IRQ */ 118626950Sfsmp if (apic_int_is_bus_type(intr, PCI)) 118725164Speter return INTPIN(intr); /* exact match */ 118825164Speter 118926950Sfsmp return -1; /* NOT found */ 119025164Speter} 119125164Speter#undef SRCBUSLINE 119225164Speter#undef SRCBUSDEVICE 119325164Speter#undef SRCBUSID 119425164Speter 119525164Speter#undef INTPIN 119625164Speter#undef INTTYPE 119725164Speter 119825164Speter 119925499Sfsmp/* 120026950Sfsmp * Reprogram the MB chipset to NOT redirect an ISA INTerrupt. 120126950Sfsmp * 120226950Sfsmp * XXX FIXME: 120326950Sfsmp * Exactly what this means is unclear at this point. It is a solution 120426950Sfsmp * for motherboards that redirect the MBIRQ0 pin. Generically a motherboard 120526950Sfsmp * could route any of the ISA INTs to upper (>15) IRQ values. But most would 120626950Sfsmp * NOT be redirected via MBIRQ0, thus "undirect()ing" them would NOT be an 120726950Sfsmp * option. 120825499Sfsmp */ 120925164Speterint 121026950Sfsmpundirect_isa_irq(int rirq) 121125164Speter{ 121225164Speter#if defined(READY) 121326950Sfsmp printf("Freeing redirected ISA irq %d.\n", rirq); 121425164Speter /** FIXME: tickle the MB redirector chip */ 121525164Speter return ???; 121625164Speter#else 121726950Sfsmp printf("Freeing (NOT implemented) redirected ISA irq %d.\n", rirq); 121825164Speter return 0; 121925499Sfsmp#endif /* READY */ 122025164Speter} 122125164Speter 122225164Speter 122325164Speter/* 122426950Sfsmp * Reprogram the MB chipset to NOT redirect a PCI INTerrupt 122525499Sfsmp */ 122625499Sfsmpint 122726950Sfsmpundirect_pci_irq(int rirq) 122825499Sfsmp{ 122925499Sfsmp#if defined(READY) 123026950Sfsmp if (bootverbose) 123126950Sfsmp printf("Freeing redirected PCI irq %d.\n", rirq); 123226950Sfsmp 123325499Sfsmp /** FIXME: tickle the MB redirector chip */ 123425499Sfsmp return ???; 123525499Sfsmp#else 123626950Sfsmp if (bootverbose) 123726950Sfsmp printf("Freeing (NOT implemented) redirected PCI irq %d.\n", 123826950Sfsmp rirq); 123925499Sfsmp return 0; 124025499Sfsmp#endif /* READY */ 124125499Sfsmp} 124225499Sfsmp 124325499Sfsmp 124425499Sfsmp/* 124525164Speter * given a bus ID, return: 124625164Speter * the bus type if found 124725164Speter * -1 if NOT found 124825164Speter */ 124925164Speterint 125025164Speterapic_bus_type(int id) 125125164Speter{ 125225164Speter int x; 125325164Speter 125425164Speter for (x = 0; x < mp_nbusses; ++x) 125525164Speter if (bus_data[x].bus_id == id) 125625164Speter return bus_data[x].bus_type; 125725164Speter 125825164Speter return -1; 125925164Speter} 126025164Speter 126125164Speter 126225164Speter/* 126325164Speter * given a LOGICAL APIC# and pin#, return: 126425164Speter * the associated src bus ID if found 126525164Speter * -1 if NOT found 126625164Speter */ 126725164Speterint 126825164Speterapic_src_bus_id(int apic, int pin) 126925164Speter{ 127025164Speter int x; 127125164Speter 127225164Speter /* search each of the possible INTerrupt sources */ 127325164Speter for (x = 0; x < nintrs; ++x) 127425164Speter if ((apic == ID_TO_IO(io_apic_ints[x].dst_apic_id)) && 127525164Speter (pin == io_apic_ints[x].dst_apic_int)) 127625164Speter return (io_apic_ints[x].src_bus_id); 127725164Speter 127825164Speter return -1; /* NOT found */ 127925164Speter} 128025164Speter 128125164Speter 128225164Speter/* 128325164Speter * given a LOGICAL APIC# and pin#, return: 128425164Speter * the associated src bus IRQ if found 128525164Speter * -1 if NOT found 128625164Speter */ 128725164Speterint 128825164Speterapic_src_bus_irq(int apic, int pin) 128925164Speter{ 129025164Speter int x; 129125164Speter 129225164Speter for (x = 0; x < nintrs; x++) 129325164Speter if ((apic == ID_TO_IO(io_apic_ints[x].dst_apic_id)) && 129425164Speter (pin == io_apic_ints[x].dst_apic_int)) 129525164Speter return (io_apic_ints[x].src_bus_irq); 129625164Speter 129725164Speter return -1; /* NOT found */ 129825164Speter} 129925164Speter 130025164Speter 130125164Speter/* 130225164Speter * given a LOGICAL APIC# and pin#, return: 130325164Speter * the associated INTerrupt type if found 130425164Speter * -1 if NOT found 130525164Speter */ 130625164Speterint 130725164Speterapic_int_type(int apic, int pin) 130825164Speter{ 130925164Speter int x; 131025164Speter 131125164Speter /* search each of the possible INTerrupt sources */ 131225164Speter for (x = 0; x < nintrs; ++x) 131325164Speter if ((apic == ID_TO_IO(io_apic_ints[x].dst_apic_id)) && 131425164Speter (pin == io_apic_ints[x].dst_apic_int)) 131525164Speter return (io_apic_ints[x].int_type); 131625164Speter 131725164Speter return -1; /* NOT found */ 131825164Speter} 131925164Speter 132025164Speter 132125164Speter/* 132225164Speter * given a LOGICAL APIC# and pin#, return: 132325164Speter * the associated trigger mode if found 132425164Speter * -1 if NOT found 132525164Speter */ 132625164Speterint 132725164Speterapic_trigger(int apic, int pin) 132825164Speter{ 132925164Speter int x; 133025164Speter 133125164Speter /* search each of the possible INTerrupt sources */ 133225164Speter for (x = 0; x < nintrs; ++x) 133325164Speter if ((apic == ID_TO_IO(io_apic_ints[x].dst_apic_id)) && 133425164Speter (pin == io_apic_ints[x].dst_apic_int)) 133525164Speter return ((io_apic_ints[x].int_flags >> 2) & 0x03); 133625164Speter 133725164Speter return -1; /* NOT found */ 133825164Speter} 133925164Speter 134025164Speter 134125164Speter/* 134225164Speter * given a LOGICAL APIC# and pin#, return: 134325164Speter * the associated 'active' level if found 134425164Speter * -1 if NOT found 134525164Speter */ 134625164Speterint 134725164Speterapic_polarity(int apic, int pin) 134825164Speter{ 134925164Speter int x; 135025164Speter 135125164Speter /* search each of the possible INTerrupt sources */ 135225164Speter for (x = 0; x < nintrs; ++x) 135325164Speter if ((apic == ID_TO_IO(io_apic_ints[x].dst_apic_id)) && 135425164Speter (pin == io_apic_ints[x].dst_apic_int)) 135525164Speter return (io_apic_ints[x].int_flags & 0x03); 135625164Speter 135725164Speter return -1; /* NOT found */ 135825164Speter} 135925164Speter 136025164Speter 136125164Speter/* 136225164Speter * set data according to MP defaults 136325164Speter * FIXME: probably not complete yet... 136425164Speter */ 136525164Speterstatic void 136625164Speterdefault_mp_table(int type) 136725164Speter{ 136825164Speter int ap_cpu_id; 136925164Speter#if defined(APIC_IO) 137025164Speter u_int32_t ux; 137125164Speter int io_apic_id; 137225164Speter int pin; 137325164Speter#endif /* APIC_IO */ 137425164Speter 137525164Speter#if 0 137625164Speter printf(" MP default config type: %d\n", type); 137725164Speter switch (type) { 137825164Speter case 1: 137925164Speter printf(" bus: ISA, APIC: 82489DX\n"); 138025164Speter break; 138125164Speter case 2: 138225164Speter printf(" bus: EISA, APIC: 82489DX\n"); 138325164Speter break; 138425164Speter case 3: 138525164Speter printf(" bus: EISA, APIC: 82489DX\n"); 138625164Speter break; 138725164Speter case 4: 138825164Speter printf(" bus: MCA, APIC: 82489DX\n"); 138925164Speter break; 139025164Speter case 5: 139125164Speter printf(" bus: ISA+PCI, APIC: Integrated\n"); 139225164Speter break; 139325164Speter case 6: 139425164Speter printf(" bus: EISA+PCI, APIC: Integrated\n"); 139525164Speter break; 139625164Speter case 7: 139725164Speter printf(" bus: MCA+PCI, APIC: Integrated\n"); 139825164Speter break; 139925164Speter default: 140025164Speter printf(" future type\n"); 140125164Speter break; 140225164Speter /* NOTREACHED */ 140325164Speter } 140425164Speter#endif /* 0 */ 140525164Speter 140626812Speter boot_cpu_id = (lapic.id & APIC_ID_MASK) >> 24; 140725164Speter ap_cpu_id = (boot_cpu_id == 0) ? 1 : 0; 140825164Speter 140925164Speter /* BSP */ 141025164Speter CPU_TO_ID(0) = boot_cpu_id; 141125164Speter ID_TO_CPU(boot_cpu_id) = 0; 141225164Speter 141325164Speter /* one and only AP */ 141425164Speter CPU_TO_ID(1) = ap_cpu_id; 141525164Speter ID_TO_CPU(ap_cpu_id) = 1; 141625164Speter 141726108Sfsmp#if defined(APIC_IO) 141825164Speter /* one and only IO APIC */ 141925164Speter io_apic_id = (io_apic_read(0, IOAPIC_ID) & APIC_ID_MASK) >> 24; 142025164Speter 142125164Speter /* 142225164Speter * sanity check, refer to MP spec section 3.6.6, last paragraph 142325164Speter * necessary as some hardware isn't properly setting up the IO APIC 142425164Speter */ 142525164Speter#if defined(REALLY_ANAL_IOAPICID_VALUE) 142625164Speter if (io_apic_id != 2) { 142725164Speter#else 142825164Speter if ((io_apic_id == 0) || (io_apic_id == 1) || (io_apic_id == 15)) { 142925164Speter#endif /* REALLY_ANAL_IOAPICID_VALUE */ 143025164Speter ux = io_apic_read(0, IOAPIC_ID); /* get current contents */ 143125164Speter ux &= ~APIC_ID_MASK; /* clear the ID field */ 143225164Speter ux |= 0x02000000; /* set it to '2' */ 143325164Speter io_apic_write(0, IOAPIC_ID, ux); /* write new value */ 143425164Speter ux = io_apic_read(0, IOAPIC_ID); /* re-read && test */ 143526101Sfsmp if ((ux & APIC_ID_MASK) != 0x02000000) 143626108Sfsmp panic("can't control IO APIC ID, reg: 0x%08x", ux); 143725164Speter io_apic_id = 2; 143825164Speter } 143925164Speter IO_TO_ID(0) = io_apic_id; 144025164Speter ID_TO_IO(io_apic_id) = 0; 144125164Speter#endif /* APIC_IO */ 144225164Speter 144325164Speter /* fill out bus entries */ 144425164Speter switch (type) { 144525164Speter case 1: 144625164Speter case 2: 144725164Speter case 3: 144825164Speter case 5: 144925164Speter case 6: 145025164Speter bus_data[0].bus_id = default_data[type - 1][1]; 145125164Speter bus_data[0].bus_type = default_data[type - 1][2]; 145225164Speter bus_data[1].bus_id = default_data[type - 1][3]; 145325164Speter bus_data[1].bus_type = default_data[type - 1][4]; 145425164Speter break; 145525164Speter 145625164Speter /* case 4: case 7: MCA NOT supported */ 145725164Speter default: /* illegal/reserved */ 145826108Sfsmp panic("BAD default MP config: %d", type); 145926101Sfsmp /* NOTREACHED */ 146025164Speter } 146125164Speter 146225164Speter#if defined(APIC_IO) 146325164Speter /* general cases from MP v1.4, table 5-2 */ 146425164Speter for (pin = 0; pin < 16; ++pin) { 146525164Speter io_apic_ints[pin].int_type = 0; 146627728Sfsmp io_apic_ints[pin].int_flags = 0x05; /* edge/active-hi */ 146725164Speter io_apic_ints[pin].src_bus_id = 0; 146827728Sfsmp io_apic_ints[pin].src_bus_irq = pin; /* IRQ2 caught below */ 146925164Speter io_apic_ints[pin].dst_apic_id = io_apic_id; 147027728Sfsmp io_apic_ints[pin].dst_apic_int = pin; /* 1-to-1 */ 147125164Speter } 147225164Speter 147325164Speter /* special cases from MP v1.4, table 5-2 */ 147425164Speter if (type == 2) { 147525164Speter io_apic_ints[2].int_type = 0xff; /* N/C */ 147625164Speter io_apic_ints[13].int_type = 0xff; /* N/C */ 147725164Speter#if !defined(APIC_MIXED_MODE) 147825164Speter /** FIXME: ??? */ 147926108Sfsmp panic("sorry, can't support type 2 default yet"); 148025164Speter#endif /* APIC_MIXED_MODE */ 148126019Sfsmp } 148226019Sfsmp else 148325164Speter io_apic_ints[2].src_bus_irq = 0; /* ISA IRQ0 is on APIC INT 2 */ 148425164Speter 148525164Speter if (type == 7) 148625164Speter io_apic_ints[0].int_type = 0xff; /* N/C */ 148725164Speter else 148825164Speter io_apic_ints[0].int_type = 3; /* vectored 8259 */ 148925164Speter#endif /* APIC_IO */ 149025164Speter} 149125164Speter 149225164Speter 149325164Speter/* 149427634Sfsmp * initialize all the SMP locks 149527634Sfsmp */ 149628442Sfsmp 149728487Sfsmp/* critical region around IO APIC, apic_imen */ 149828487Sfsmpstruct simplelock imen_lock; 149928487Sfsmp 150029213Sfsmp/* critical region around splxx(), cpl, cml, cil, ipending */ 150128487Sfsmpstruct simplelock cpl_lock; 150228487Sfsmp 150328487Sfsmp/* Make FAST_INTR() routines sequential */ 150428487Sfsmpstruct simplelock fast_intr_lock; 150528487Sfsmp 150628487Sfsmp/* critical region around INTR() routines */ 150728487Sfsmpstruct simplelock intr_lock; 150828487Sfsmp 150928951Sfsmp/* lock regions protected in UP kernel via cli/sti */ 151028921Sfsmpstruct simplelock mpintr_lock; 151128921Sfsmp 151228951Sfsmp#ifdef USE_COMLOCK 151328951Sfsmp/* locks com (tty) data/hardware accesses: a FASTINTR() */ 151428442Sfsmpstruct simplelock com_lock; 151528951Sfsmp#endif /* USE_COMLOCK */ 151628442Sfsmp 151728999Sfsmp#ifdef USE_CLOCKLOCK 151828999Sfsmp/* lock regions around the clock hardware */ 151928999Sfsmpstruct simplelock clock_lock; 152028999Sfsmp#endif /* USE_CLOCKLOCK */ 152128999Sfsmp 152227634Sfsmpstatic void 152327634Sfsmpinit_locks(void) 152427634Sfsmp{ 152527634Sfsmp /* 152627634Sfsmp * Get the initial mp_lock with a count of 1 for the BSP. 152727634Sfsmp * This uses a LOGICAL cpu ID, ie BSP == 0. 152827634Sfsmp */ 152927634Sfsmp mp_lock = 0x00000001; 153027634Sfsmp 153128442Sfsmp /* ISR uses its own "giant lock" */ 153228921Sfsmp isr_lock = FREE_LOCK; 153328442Sfsmp 153428921Sfsmp s_lock_init((struct simplelock*)&mpintr_lock); 153528999Sfsmp 153628442Sfsmp s_lock_init((struct simplelock*)&fast_intr_lock); 153728442Sfsmp s_lock_init((struct simplelock*)&intr_lock); 153827780Sfsmp s_lock_init((struct simplelock*)&imen_lock); 153928442Sfsmp s_lock_init((struct simplelock*)&cpl_lock); 154028999Sfsmp 154128951Sfsmp#ifdef USE_COMLOCK 154228442Sfsmp s_lock_init((struct simplelock*)&com_lock); 154328951Sfsmp#endif /* USE_COMLOCK */ 154428999Sfsmp#ifdef USE_CLOCKLOCK 154528999Sfsmp s_lock_init((struct simplelock*)&clock_lock); 154628999Sfsmp#endif /* USE_CLOCKLOCK */ 154727634Sfsmp} 154827634Sfsmp 154927634Sfsmp 155027634Sfsmp/* 155125164Speter * start each AP in our list 155225164Speter */ 155325164Speterstatic int 155425164Speterstart_all_aps(u_int boot_addr) 155525164Speter{ 155626812Speter int x, i; 155725164Speter u_char mpbiosreason; 155825164Speter u_long mpbioswarmvec; 155928808Speter pd_entry_t *newptd; 156028808Speter pt_entry_t *newpt; 156129655Sdyson int *newpp; 156229655Sdyson char *stack; 156329655Sdyson pd_entry_t *myPTD; 156425164Speter 156527005Sfsmp POSTCODE(START_ALL_APS_POST); 156627005Sfsmp 156725164Speter /* initialize BSP's local APIC */ 156827289Sfsmp apic_initialize(); 156928669Sfsmp bsp_apic_ready = 1; 157025164Speter 157125164Speter /* install the AP 1st level boot code */ 157225164Speter install_ap_tramp(boot_addr); 157325164Speter 157426812Speter 157525164Speter /* save the current value of the warm-start vector */ 157625164Speter mpbioswarmvec = *((u_long *) WARMBOOT_OFF); 157725164Speter outb(CMOS_REG, BIOS_RESET); 157825164Speter mpbiosreason = inb(CMOS_DATA); 157925164Speter 158027005Sfsmp /* record BSP in CPU map */ 158127005Sfsmp all_cpus = 1; 158227005Sfsmp 158325164Speter /* start each AP */ 158425164Speter for (x = 1; x <= mp_naps; ++x) { 158525164Speter 158628808Speter /* This is a bit verbose, it will go away soon. */ 158726812Speter 158826812Speter /* alloc new page table directory */ 158928808Speter newptd = (pd_entry_t *)(kmem_alloc(kernel_map, PAGE_SIZE)); 159026812Speter 159128808Speter /* Store the virtual PTD address for this CPU */ 159228808Speter IdlePTDS[x] = newptd; 159328808Speter 159426812Speter /* clone currently active one (ie: IdlePTD) */ 159526812Speter bcopy(PTD, newptd, PAGE_SIZE); /* inc prv page pde */ 159626812Speter 159726812Speter /* set up 0 -> 4MB P==V mapping for AP boot */ 159828808Speter newptd[0] = (pd_entry_t) (PG_V | PG_RW | 159928808Speter ((u_long)KPTphys & PG_FRAME)); 160026812Speter 160128808Speter /* store PTD for this AP's boot sequence */ 160229655Sdyson myPTD = (pd_entry_t *)vtophys(newptd); 160326812Speter 160426812Speter /* alloc new page table page */ 160528808Speter newpt = (pt_entry_t *)(kmem_alloc(kernel_map, PAGE_SIZE)); 160626812Speter 160726812Speter /* set the new PTD's private page to point there */ 160828808Speter newptd[MPPTDI] = (pt_entry_t)(PG_V | PG_RW | vtophys(newpt)); 160926812Speter 161026812Speter /* install self referential entry */ 161128808Speter newptd[PTDPTDI] = (pd_entry_t)(PG_V | PG_RW | vtophys(newptd)); 161226812Speter 161328808Speter /* allocate a new private data page */ 161426812Speter newpp = (int *)kmem_alloc(kernel_map, PAGE_SIZE); 161526812Speter 161626812Speter /* wire it into the private page table page */ 161728808Speter newpt[0] = (pt_entry_t)(PG_V | PG_RW | vtophys(newpp)); 161826812Speter 161926812Speter /* wire the ptp into itself for access */ 162028808Speter newpt[1] = (pt_entry_t)(PG_V | PG_RW | vtophys(newpt)); 162126812Speter 162228808Speter /* copy in the pointer to the local apic */ 162326812Speter newpt[2] = SMP_prvpt[2]; 162426812Speter 162526812Speter /* and the IO apic mapping[s] */ 162626812Speter for (i = 16; i < 32; i++) 162726812Speter newpt[i] = SMP_prvpt[i]; 162826812Speter 162928808Speter /* allocate and set up an idle stack data page */ 163029655Sdyson stack = (char *)kmem_alloc(kernel_map, UPAGES*PAGE_SIZE); 163129655Sdyson for (i = 0; i < UPAGES; i++) 163229655Sdyson newpt[i + 3] = (pt_entry_t)(PG_V | PG_RW | vtophys(PAGE_SIZE * i + stack)); 163326812Speter 163431030Stegge newpt[3 + UPAGES] = 0; /* *prv_CMAP1 */ 163531030Stegge newpt[4 + UPAGES] = 0; /* *prv_CMAP2 */ 163631030Stegge newpt[5 + UPAGES] = 0; /* *prv_CMAP3 */ 163726812Speter 163828808Speter /* prime data page for it to use */ 163928808Speter newpp[0] = x; /* cpuid */ 164028808Speter newpp[1] = 0; /* curproc */ 164128808Speter newpp[2] = 0; /* curpcb */ 164228808Speter newpp[3] = 0; /* npxproc */ 164328808Speter newpp[4] = 0; /* runtime.tv_sec */ 164428808Speter newpp[5] = 0; /* runtime.tv_usec */ 164528808Speter newpp[6] = x << 24; /* cpu_lockid */ 164628808Speter newpp[7] = 0; /* other_cpus */ 164729655Sdyson newpp[8] = (int)myPTD; /* my_idlePTD */ 164828808Speter newpp[9] = 0; /* ss_tpr */ 164931030Stegge newpp[10] = (int)&newpt[3 + UPAGES]; /* prv_CMAP1 */ 165031030Stegge newpp[11] = (int)&newpt[4 + UPAGES]; /* prv_CMAP2 */ 165131030Stegge newpp[12] = (int)&newpt[5 + UPAGES]; /* prv_CMAP3 */ 165226812Speter 165325164Speter /* setup a vector to our boot code */ 165425164Speter *((volatile u_short *) WARMBOOT_OFF) = WARMBOOT_TARGET; 165525164Speter *((volatile u_short *) WARMBOOT_SEG) = (boot_addr >> 4); 165625164Speter outb(CMOS_REG, BIOS_RESET); 165725164Speter outb(CMOS_DATA, BIOS_WARM); /* 'warm-start' */ 165825164Speter 165929655Sdyson bootPTD = myPTD; 166025164Speter /* attempt to start the Application Processor */ 166125164Speter CHECK_INIT(99); /* setup checkpoints */ 166225164Speter if (!start_ap(x, boot_addr)) { 166325164Speter printf("AP #%d (PHY# %d) failed!\n", x, CPU_TO_ID(x)); 166425164Speter CHECK_PRINT("trace"); /* show checkpoints */ 166526155Sfsmp /* better panic as the AP may be running loose */ 166626155Sfsmp printf("panic y/n? [y] "); 166726101Sfsmp if (cngetc() != 'n') 166826108Sfsmp panic("bye-bye"); 166925164Speter } 167027005Sfsmp CHECK_PRINT("trace"); /* show checkpoints */ 167125164Speter 167225164Speter /* record its version info */ 167325164Speter cpu_apic_versions[x] = cpu_apic_versions[0]; 167427005Sfsmp 167527005Sfsmp all_cpus |= (1 << x); /* record AP in CPU map */ 167625164Speter } 167725164Speter 167827005Sfsmp /* build our map of 'other' CPUs */ 167927005Sfsmp other_cpus = all_cpus & ~(1 << cpuid); 168027005Sfsmp 168125164Speter /* fill in our (BSP) APIC version */ 168226812Speter cpu_apic_versions[0] = lapic.version; 168325164Speter 168425164Speter /* restore the warmstart vector */ 168525164Speter *(u_long *) WARMBOOT_OFF = mpbioswarmvec; 168625164Speter outb(CMOS_REG, BIOS_RESET); 168725164Speter outb(CMOS_DATA, mpbiosreason); 168825164Speter 168928808Speter /* 169028808Speter * Set up the idle context for the BSP. Similar to above except 169128808Speter * that some was done by locore, some by pmap.c and some is implicit 169228808Speter * because the BSP is cpu#0 and the page is initially zero, and also 169328808Speter * because we can refer to variables by name on the BSP.. 169428808Speter */ 169528808Speter newptd = (pd_entry_t *)(kmem_alloc(kernel_map, PAGE_SIZE)); 169628808Speter 169728808Speter bcopy(PTD, newptd, PAGE_SIZE); /* inc prv page pde */ 169828808Speter IdlePTDS[0] = newptd; 169928808Speter 170028808Speter /* Point PTD[] to this page instead of IdlePTD's physical page */ 170128808Speter newptd[PTDPTDI] = (pd_entry_t)(PG_V | PG_RW | vtophys(newptd)); 170228808Speter 170328808Speter my_idlePTD = (pd_entry_t *)vtophys(newptd); 170428808Speter 170528808Speter /* Allocate and setup BSP idle stack */ 170629655Sdyson stack = (char *)kmem_alloc(kernel_map, UPAGES * PAGE_SIZE); 170729655Sdyson for (i = 0; i < UPAGES; i++) 170829655Sdyson SMP_prvpt[i + 3] = (pt_entry_t)(PG_V | PG_RW | vtophys(PAGE_SIZE * i + stack)); 170928808Speter 171027484Sdyson pmap_set_opt_bsp(); 171127484Sdyson 171229655Sdyson for (i = 0; i < mp_ncpus; i++) { 171329655Sdyson bcopy( (int *) PTD + KPTDI, (int *) IdlePTDS[i] + KPTDI, NKPDE * sizeof (int)); 171429655Sdyson } 171529655Sdyson 171625164Speter /* number of APs actually started */ 171725164Speter return mp_ncpus - 1; 171825164Speter} 171925164Speter 172025164Speter 172125164Speter/* 172225164Speter * load the 1st level AP boot code into base memory. 172325164Speter */ 172425164Speter 172525164Speter/* targets for relocation */ 172625164Speterextern void bigJump(void); 172725164Speterextern void bootCodeSeg(void); 172825164Speterextern void bootDataSeg(void); 172925164Speterextern void MPentry(void); 173025164Speterextern u_int MP_GDT; 173125164Speterextern u_int mp_gdtbase; 173225164Speter 173325164Speterstatic void 173425164Speterinstall_ap_tramp(u_int boot_addr) 173525164Speter{ 173625164Speter int x; 173725164Speter int size = *(int *) ((u_long) & bootMP_size); 173825164Speter u_char *src = (u_char *) ((u_long) bootMP); 173925164Speter u_char *dst = (u_char *) boot_addr + KERNBASE; 174025164Speter u_int boot_base = (u_int) bootMP; 174125164Speter u_int8_t *dst8; 174225164Speter u_int16_t *dst16; 174325164Speter u_int32_t *dst32; 174425164Speter 174527005Sfsmp POSTCODE(INSTALL_AP_TRAMP_POST); 174627005Sfsmp 174725164Speter for (x = 0; x < size; ++x) 174825164Speter *dst++ = *src++; 174925164Speter 175025164Speter /* 175125164Speter * modify addresses in code we just moved to basemem. unfortunately we 175225164Speter * need fairly detailed info about mpboot.s for this to work. changes 175325164Speter * to mpboot.s might require changes here. 175425164Speter */ 175525164Speter 175625164Speter /* boot code is located in KERNEL space */ 175725164Speter dst = (u_char *) boot_addr + KERNBASE; 175825164Speter 175925164Speter /* modify the lgdt arg */ 176025164Speter dst32 = (u_int32_t *) (dst + ((u_int) & mp_gdtbase - boot_base)); 176125164Speter *dst32 = boot_addr + ((u_int) & MP_GDT - boot_base); 176225164Speter 176325164Speter /* modify the ljmp target for MPentry() */ 176425164Speter dst32 = (u_int32_t *) (dst + ((u_int) bigJump - boot_base) + 1); 176525164Speter *dst32 = ((u_int) MPentry - KERNBASE); 176625164Speter 176725164Speter /* modify the target for boot code segment */ 176825164Speter dst16 = (u_int16_t *) (dst + ((u_int) bootCodeSeg - boot_base)); 176925164Speter dst8 = (u_int8_t *) (dst16 + 1); 177025164Speter *dst16 = (u_int) boot_addr & 0xffff; 177125164Speter *dst8 = ((u_int) boot_addr >> 16) & 0xff; 177225164Speter 177325164Speter /* modify the target for boot data segment */ 177425164Speter dst16 = (u_int16_t *) (dst + ((u_int) bootDataSeg - boot_base)); 177525164Speter dst8 = (u_int8_t *) (dst16 + 1); 177625164Speter *dst16 = (u_int) boot_addr & 0xffff; 177725164Speter *dst8 = ((u_int) boot_addr >> 16) & 0xff; 177825164Speter} 177925164Speter 178025164Speter 178125164Speter/* 178225164Speter * this function starts the AP (application processor) identified 178325164Speter * by the APIC ID 'physicalCpu'. It does quite a "song and dance" 178425164Speter * to accomplish this. This is necessary because of the nuances 178525164Speter * of the different hardware we might encounter. It ain't pretty, 178625164Speter * but it seems to work. 178725164Speter */ 178825164Speterstatic int 178925164Speterstart_ap(int logical_cpu, u_int boot_addr) 179025164Speter{ 179125164Speter int physical_cpu; 179225164Speter int vector; 179325164Speter int cpus; 179425164Speter u_long icr_lo, icr_hi; 179525164Speter 179627005Sfsmp POSTCODE(START_AP_POST); 179727005Sfsmp 179825164Speter /* get the PHYSICAL APIC ID# */ 179925164Speter physical_cpu = CPU_TO_ID(logical_cpu); 180025164Speter 180125164Speter /* calculate the vector */ 180225164Speter vector = (boot_addr >> 12) & 0xff; 180325164Speter 180425164Speter /* used as a watchpoint to signal AP startup */ 180525164Speter cpus = mp_ncpus; 180625164Speter 180725164Speter /* 180825164Speter * first we do an INIT/RESET IPI this INIT IPI might be run, reseting 180925164Speter * and running the target CPU. OR this INIT IPI might be latched (P5 181025164Speter * bug), CPU waiting for STARTUP IPI. OR this INIT IPI might be 181125164Speter * ignored. 181225164Speter */ 181325164Speter 181425164Speter /* setup the address for the target AP */ 181526812Speter icr_hi = lapic.icr_hi & ~APIC_ID_MASK; 181625164Speter icr_hi |= (physical_cpu << 24); 181726812Speter lapic.icr_hi = icr_hi; 181825164Speter 181925164Speter /* do an INIT IPI: assert RESET */ 182026812Speter icr_lo = lapic.icr_lo & 0xfff00000; 182126812Speter lapic.icr_lo = icr_lo | 0x0000c500; 182225164Speter 182325164Speter /* wait for pending status end */ 182426812Speter while (lapic.icr_lo & APIC_DELSTAT_MASK) 182525164Speter /* spin */ ; 182625164Speter 182725164Speter /* do an INIT IPI: deassert RESET */ 182826812Speter lapic.icr_lo = icr_lo | 0x00008500; 182925164Speter 183025164Speter /* wait for pending status end */ 183125164Speter u_sleep(10000); /* wait ~10mS */ 183226812Speter while (lapic.icr_lo & APIC_DELSTAT_MASK) 183325164Speter /* spin */ ; 183425164Speter 183525164Speter /* 183625164Speter * next we do a STARTUP IPI: the previous INIT IPI might still be 183725164Speter * latched, (P5 bug) this 1st STARTUP would then terminate 183825164Speter * immediately, and the previously started INIT IPI would continue. OR 183925164Speter * the previous INIT IPI has already run. and this STARTUP IPI will 184025164Speter * run. OR the previous INIT IPI was ignored. and this STARTUP IPI 184125164Speter * will run. 184225164Speter */ 184325164Speter 184425164Speter /* do a STARTUP IPI */ 184526812Speter lapic.icr_lo = icr_lo | 0x00000600 | vector; 184626812Speter while (lapic.icr_lo & APIC_DELSTAT_MASK) 184725164Speter /* spin */ ; 184825164Speter u_sleep(200); /* wait ~200uS */ 184925164Speter 185025164Speter /* 185125164Speter * finally we do a 2nd STARTUP IPI: this 2nd STARTUP IPI should run IF 185225164Speter * the previous STARTUP IPI was cancelled by a latched INIT IPI. OR 185325164Speter * this STARTUP IPI will be ignored, as only ONE STARTUP IPI is 185425164Speter * recognized after hardware RESET or INIT IPI. 185525164Speter */ 185625164Speter 185726812Speter lapic.icr_lo = icr_lo | 0x00000600 | vector; 185826812Speter while (lapic.icr_lo & APIC_DELSTAT_MASK) 185925164Speter /* spin */ ; 186025164Speter u_sleep(200); /* wait ~200uS */ 186125164Speter 186225164Speter /* wait for it to start */ 186325164Speter set_apic_timer(5000000);/* == 5 seconds */ 186425164Speter while (read_apic_timer()) 186525164Speter if (mp_ncpus > cpus) 186625164Speter return 1; /* return SUCCESS */ 186725164Speter 186825164Speter return 0; /* return FAILURE */ 186925164Speter} 187025164Speter 187125164Speter 187225164Speter/* 187325164Speter * Flush the TLB on all other CPU's 187425164Speter * 187525164Speter * XXX: Needs to handshake and wait for completion before proceding. 187625164Speter */ 187725164Spetervoid 187825215Sfsmpsmp_invltlb(void) 187925164Speter{ 188025419Sfsmp#if defined(APIC_IO) 188128809Speter if (smp_started && invltlb_ok) 188227005Sfsmp all_but_self_ipi(XINVLTLB_OFFSET); 188325419Sfsmp#endif /* APIC_IO */ 188425164Speter} 188525164Speter 188625164Spetervoid 188725164Speterinvlpg(u_int addr) 188825164Speter{ 188925164Speter __asm __volatile("invlpg (%0)"::"r"(addr):"memory"); 189025215Sfsmp 189125215Sfsmp /* send a message to the other CPUs */ 189225164Speter smp_invltlb(); 189325164Speter} 189425164Speter 189525164Spetervoid 189625164Speterinvltlb(void) 189725164Speter{ 189825164Speter u_long temp; 189925215Sfsmp 190025164Speter /* 190125164Speter * This should be implemented as load_cr3(rcr3()) when load_cr3() is 190225164Speter * inlined. 190325164Speter */ 190425164Speter __asm __volatile("movl %%cr3, %0; movl %0, %%cr3":"=r"(temp) :: "memory"); 190525215Sfsmp 190625215Sfsmp /* send a message to the other CPUs */ 190725164Speter smp_invltlb(); 190825164Speter} 190927005Sfsmp 191027005Sfsmp 191127005Sfsmp/* 191227005Sfsmp * When called the executing CPU will send an IPI to all other CPUs 191327005Sfsmp * requesting that they halt execution. 191427005Sfsmp * 191527005Sfsmp * Usually (but not necessarily) called with 'other_cpus' as its arg. 191627005Sfsmp * 191727005Sfsmp * - Signals all CPUs in map to stop. 191827005Sfsmp * - Waits for each to stop. 191927005Sfsmp * 192027005Sfsmp * Returns: 192127005Sfsmp * -1: error 192227005Sfsmp * 0: NA 192327005Sfsmp * 1: ok 192427005Sfsmp * 192527005Sfsmp * XXX FIXME: this is not MP-safe, needs a lock to prevent multiple CPUs 192627005Sfsmp * from executing at same time. 192727005Sfsmp */ 192827005Sfsmpint 192928808Speterstop_cpus(u_int map) 193027005Sfsmp{ 193128809Speter if (!smp_started) 193227005Sfsmp return 0; 193327005Sfsmp 193427005Sfsmp /* send IPI to all CPUs in map */ 193527255Sfsmp stopped_cpus = 0; 193627353Sfsmp 193727353Sfsmp /* send the Xcpustop IPI to all CPUs in map */ 193827005Sfsmp selected_apic_ipi(map, XCPUSTOP_OFFSET, APIC_DELMODE_FIXED); 193927005Sfsmp 194027353Sfsmp while (stopped_cpus != map) 194127005Sfsmp /* spin */ ; 194227005Sfsmp 194327005Sfsmp return 1; 194427005Sfsmp} 194527005Sfsmp 194627005Sfsmp 194727005Sfsmp/* 194827005Sfsmp * Called by a CPU to restart stopped CPUs. 194927005Sfsmp * 195027005Sfsmp * Usually (but not necessarily) called with 'stopped_cpus' as its arg. 195127005Sfsmp * 195227005Sfsmp * - Signals all CPUs in map to restart. 195327005Sfsmp * - Waits for each to restart. 195427005Sfsmp * 195527005Sfsmp * Returns: 195627005Sfsmp * -1: error 195727005Sfsmp * 0: NA 195827005Sfsmp * 1: ok 195927005Sfsmp */ 196027005Sfsmpint 196128808Speterrestart_cpus(u_int map) 196227005Sfsmp{ 196328809Speter if (!smp_started) 196427005Sfsmp return 0; 196527005Sfsmp 196627255Sfsmp started_cpus = map; /* signal other cpus to restart */ 196727255Sfsmp 196827005Sfsmp while (started_cpus) /* wait for each to clear its bit */ 196927005Sfsmp /* spin */ ; 197027005Sfsmp 197127005Sfsmp return 1; 197227005Sfsmp} 197328808Speter 197428808Speterint smp_active = 0; /* are the APs allowed to run? */ 197528808SpeterSYSCTL_INT(_machdep, OID_AUTO, smp_active, CTLFLAG_RW, &smp_active, 0, ""); 197628808Speter 197728808Speter/* XXX maybe should be hw.ncpu */ 197828808Speterint smp_cpus = 1; /* how many cpu's running */ 197928808SpeterSYSCTL_INT(_machdep, OID_AUTO, smp_cpus, CTLFLAG_RD, &smp_cpus, 0, ""); 198028808Speter 198128808Speterint invltlb_ok = 0; /* throttle smp_invltlb() till safe */ 198228808SpeterSYSCTL_INT(_machdep, OID_AUTO, invltlb_ok, CTLFLAG_RW, &invltlb_ok, 0, ""); 198328808Speter 198428808Speterint do_page_zero_idle = 0; /* bzero pages for fun and profit in idleloop */ 198528808SpeterSYSCTL_INT(_machdep, OID_AUTO, do_page_zero_idle, CTLFLAG_RW, 198628808Speter &do_page_zero_idle, 0, ""); 198728808Speter 198828808Speter 198928808Speter/* 199028808Speter * This is called once the rest of the system is up and running and we're 199128808Speter * ready to let the AP's out of the pen. 199228808Speter */ 199328808Spetervoid ap_init(void); 199428808Speter 199528808Spetervoid 199628808Speterap_init() 199728808Speter{ 199828808Speter u_int temp; 199928808Speter u_int apic_id; 200028808Speter 200128808Speter smp_cpus++; 200228808Speter 200328808Speter /* Build our map of 'other' CPUs. */ 200428808Speter other_cpus = all_cpus & ~(1 << cpuid); 200528808Speter 200628808Speter printf("SMP: AP CPU #%d Launched!\n", cpuid); 200728808Speter 200828808Speter /* XXX FIXME: i386 specific, and redundant: Setup the FPU. */ 200928808Speter load_cr0((rcr0() & ~CR0_EM) | CR0_MP | CR0_NE | CR0_TS); 201028808Speter 201128808Speter /* A quick check from sanity claus */ 201228808Speter apic_id = (apic_id_to_logical[(lapic.id & 0x0f000000) >> 24]); 201328808Speter if (cpuid != apic_id) { 201428808Speter printf("SMP: cpuid = %d\n", cpuid); 201528808Speter printf("SMP: apic_id = %d\n", apic_id); 201628808Speter printf("PTD[MPPTDI] = %08x\n", PTD[MPPTDI]); 201728808Speter panic("cpuid mismatch! boom!!"); 201828808Speter } 201928808Speter 202028808Speter /* Init local apic for irq's */ 202128808Speter apic_initialize(); 202228808Speter 202328808Speter /* 202428808Speter * Activate smp_invltlb, although strictly speaking, this isn't 202528808Speter * quite correct yet. We should have a bitfield for cpus willing 202628808Speter * to accept TLB flush IPI's or something and sync them. 202728808Speter */ 202828808Speter invltlb_ok = 1; 202928809Speter smp_started = 1; /* enable IPI's, tlb shootdown, freezes etc */ 203028808Speter smp_active = 1; /* historic */ 203128808Speter 203228808Speter curproc = NULL; /* make sure */ 203328808Speter} 203430112Sdyson 203530112Sdysonvoid 203630343Spetergetmtrr() 203730343Speter{ 203830112Sdyson int i; 203930343Speter 204030112Sdyson if (cpu_class == CPUCLASS_686) { 204130343Speter for(i = 0; i < NPPROVMTRR; i++) { 204230112Sdyson PPro_vmtrr[i].base = rdmsr(PPRO_VMTRRphysBase0 + i * 2); 204330112Sdyson PPro_vmtrr[i].mask = rdmsr(PPRO_VMTRRphysMask0 + i * 2); 204430112Sdyson } 204530112Sdyson } 204630112Sdyson} 204730112Sdyson 204830112Sdysonvoid 204930343Speterputmtrr() 205030343Speter{ 205130112Sdyson int i; 205230343Speter 205330112Sdyson if (cpu_class == CPUCLASS_686) { 205430112Sdyson wbinvd(); 205530343Speter for(i = 0; i < NPPROVMTRR; i++) { 205630112Sdyson wrmsr(PPRO_VMTRRphysBase0 + i * 2, PPro_vmtrr[i].base); 205730112Sdyson wrmsr(PPRO_VMTRRphysMask0 + i * 2, PPro_vmtrr[i].mask); 205830112Sdyson } 205930112Sdyson } 206030112Sdyson} 206130112Sdyson 206230112Sdysonvoid 206330343Speterputfmtrr() 206430343Speter{ 206530112Sdyson if (cpu_class == CPUCLASS_686) { 206630112Sdyson wbinvd(); 206730136Sdyson /* 206830136Sdyson * Set memory between 0-640K to be WB 206930136Sdyson */ 207030136Sdyson wrmsr(0x250, 0x0606060606060606LL); 207130136Sdyson wrmsr(0x258, 0x0606060606060606LL); 207230136Sdyson /* 207330136Sdyson * Set normal, PC video memory to be WC 207430136Sdyson */ 207530112Sdyson wrmsr(0x259, 0x0101010101010101LL); 207630112Sdyson } 207730112Sdyson} 207831639Sfsmp 207931639Sfsmp 208031639Sfsmp#ifdef BETTER_CLOCK 208131639Sfsmp 208231639Sfsmp#define CHECKSTATE_USER 0 208331639Sfsmp#define CHECKSTATE_SYS 1 208431639Sfsmp#define CHECKSTATE_INTR 2 208531639Sfsmp 208631639Sfsmpstruct proc* checkstate_curproc[NCPU]; 208731639Sfsmpint checkstate_cpustate[NCPU]; 208831639Sfsmpu_long checkstate_pc[NCPU]; 208931639Sfsmp 209031639Sfsmpextern long cp_time[CPUSTATES]; 209131639Sfsmp 209231639Sfsmp#define PC_TO_INDEX(pc, prof) \ 209331639Sfsmp ((int)(((u_quad_t)((pc) - (prof)->pr_off) * \ 209431639Sfsmp (u_quad_t)((prof)->pr_scale)) >> 16) & ~1) 209531639Sfsmp 209631639Sfsmpstatic void 209731639Sfsmpaddupc_intr_forwarded(struct proc *p, int id, int *astmap) 209831639Sfsmp{ 209931639Sfsmp int i; 210031639Sfsmp struct uprof *prof; 210131639Sfsmp u_long pc; 210231639Sfsmp 210331639Sfsmp pc = checkstate_pc[id]; 210431639Sfsmp prof = &p->p_stats->p_prof; 210531639Sfsmp if (pc >= prof->pr_off && 210631639Sfsmp (i = PC_TO_INDEX(pc, prof)) < prof->pr_size) { 210731639Sfsmp if ((p->p_flag & P_OWEUPC) == 0) { 210831639Sfsmp prof->pr_addr = pc; 210931639Sfsmp prof->pr_ticks = 1; 211031639Sfsmp p->p_flag |= P_OWEUPC; 211131639Sfsmp } 211231639Sfsmp *astmap |= (1 << id); 211331639Sfsmp } 211431639Sfsmp} 211531639Sfsmp 211631639Sfsmpstatic void 211731639Sfsmpforwarded_statclock(int id, int pscnt, int *astmap) 211831639Sfsmp{ 211931639Sfsmp struct pstats *pstats; 212031639Sfsmp long rss; 212131639Sfsmp struct rusage *ru; 212231639Sfsmp struct vmspace *vm; 212331639Sfsmp int cpustate; 212431639Sfsmp struct proc *p; 212531639Sfsmp#ifdef GPROF 212631639Sfsmp register struct gmonparam *g; 212731639Sfsmp int i; 212831639Sfsmp#endif 212931639Sfsmp 213031639Sfsmp p = checkstate_curproc[id]; 213131639Sfsmp cpustate = checkstate_cpustate[id]; 213231639Sfsmp 213331639Sfsmp switch (cpustate) { 213431639Sfsmp case CHECKSTATE_USER: 213531639Sfsmp if (p->p_flag & P_PROFIL) 213631639Sfsmp addupc_intr_forwarded(p, id, astmap); 213731639Sfsmp if (pscnt > 1) 213831639Sfsmp return; 213931639Sfsmp p->p_uticks++; 214031639Sfsmp if (p->p_nice > NZERO) 214131639Sfsmp cp_time[CP_NICE]++; 214231639Sfsmp else 214331639Sfsmp cp_time[CP_USER]++; 214431639Sfsmp break; 214531639Sfsmp case CHECKSTATE_SYS: 214631639Sfsmp#ifdef GPROF 214731639Sfsmp /* 214831639Sfsmp * Kernel statistics are just like addupc_intr, only easier. 214931639Sfsmp */ 215031639Sfsmp g = &_gmonparam; 215131639Sfsmp if (g->state == GMON_PROF_ON) { 215231639Sfsmp i = checkstate_pc[id] - g->lowpc; 215331639Sfsmp if (i < g->textsize) { 215431639Sfsmp i /= HISTFRACTION * sizeof(*g->kcount); 215531639Sfsmp g->kcount[i]++; 215631639Sfsmp } 215731639Sfsmp } 215831639Sfsmp#endif 215931639Sfsmp if (pscnt > 1) 216031639Sfsmp return; 216131639Sfsmp 216231639Sfsmp if (!p) 216331639Sfsmp cp_time[CP_IDLE]++; 216431639Sfsmp else { 216531639Sfsmp p->p_sticks++; 216631639Sfsmp cp_time[CP_SYS]++; 216731639Sfsmp } 216831639Sfsmp break; 216931639Sfsmp case CHECKSTATE_INTR: 217031639Sfsmp default: 217131639Sfsmp#ifdef GPROF 217231639Sfsmp /* 217331639Sfsmp * Kernel statistics are just like addupc_intr, only easier. 217431639Sfsmp */ 217531639Sfsmp g = &_gmonparam; 217631639Sfsmp if (g->state == GMON_PROF_ON) { 217731639Sfsmp i = checkstate_pc[id] - g->lowpc; 217831639Sfsmp if (i < g->textsize) { 217931639Sfsmp i /= HISTFRACTION * sizeof(*g->kcount); 218031639Sfsmp g->kcount[i]++; 218131639Sfsmp } 218231639Sfsmp } 218331639Sfsmp#endif 218431639Sfsmp if (pscnt > 1) 218531639Sfsmp return; 218631639Sfsmp if (p) 218731639Sfsmp p->p_iticks++; 218831639Sfsmp cp_time[CP_INTR]++; 218931639Sfsmp } 219031639Sfsmp if (p != NULL) { 219131639Sfsmp p->p_cpticks++; 219231639Sfsmp if (++p->p_estcpu == 0) 219331639Sfsmp p->p_estcpu--; 219431639Sfsmp if ((p->p_estcpu & 3) == 0) { 219531639Sfsmp resetpriority(p); 219631639Sfsmp if (p->p_priority >= PUSER) 219731639Sfsmp p->p_priority = p->p_usrpri; 219831639Sfsmp } 219931639Sfsmp 220031639Sfsmp /* Update resource usage integrals and maximums. */ 220131639Sfsmp if ((pstats = p->p_stats) != NULL && 220231639Sfsmp (ru = &pstats->p_ru) != NULL && 220331639Sfsmp (vm = p->p_vmspace) != NULL) { 220431639Sfsmp ru->ru_ixrss += vm->vm_tsize * PAGE_SIZE / 1024; 220531639Sfsmp ru->ru_idrss += vm->vm_dsize * PAGE_SIZE / 1024; 220631639Sfsmp ru->ru_isrss += vm->vm_ssize * PAGE_SIZE / 1024; 220731639Sfsmp rss = vm->vm_pmap.pm_stats.resident_count * 220831639Sfsmp PAGE_SIZE / 1024; 220931639Sfsmp if (ru->ru_maxrss < rss) 221031639Sfsmp ru->ru_maxrss = rss; 221131639Sfsmp } 221231639Sfsmp } 221331639Sfsmp} 221431639Sfsmp 221531639Sfsmpvoid 221631639Sfsmpforward_statclock(int pscnt) 221731639Sfsmp{ 221831639Sfsmp int map; 221931639Sfsmp int id; 222031639Sfsmp int i; 222131639Sfsmp 222231639Sfsmp /* Kludge. We don't yet have separate locks for the interrupts 222331639Sfsmp * and the kernel. This means that we cannot let the other processors 222431639Sfsmp * handle complex interrupts while inhibiting them from entering 222531639Sfsmp * the kernel in a non-interrupt context. 222631639Sfsmp * 222731639Sfsmp * What we can do, without changing the locking mechanisms yet, 222831639Sfsmp * is letting the other processors handle a very simple interrupt 222931639Sfsmp * (wich determines the processor states), and do the main 223031639Sfsmp * work ourself. 223131639Sfsmp */ 223231639Sfsmp 223331639Sfsmp if (!smp_started || !invltlb_ok) 223431639Sfsmp return; 223531639Sfsmp 223631639Sfsmp /* Step 1: Probe state (user, cpu, interrupt, spinlock, idle ) */ 223731639Sfsmp 223831639Sfsmp map = other_cpus; 223931639Sfsmp checkstate_probed_cpus = 0; 224031639Sfsmp selected_apic_ipi(map, XCPUCHECKSTATE_OFFSET, APIC_DELMODE_FIXED); 224131639Sfsmp 224231639Sfsmp i = 0; 224331639Sfsmp while (checkstate_probed_cpus != map) { 224431639Sfsmp /* spin */ 224531639Sfsmp i++; 224631639Sfsmp if (i == 1000000) { 224731639Sfsmp printf("forward_statclock: checkstate %x\n", 224831639Sfsmp checkstate_probed_cpus); 224931639Sfsmp } 225031639Sfsmp } 225131639Sfsmp 225231639Sfsmp /* 225331639Sfsmp * Step 2: walk through other processors processes, update ticks and 225431639Sfsmp * profiling info. 225531639Sfsmp */ 225631639Sfsmp 225731639Sfsmp map = 0; 225831639Sfsmp for (id = 0; id < mp_ncpus; id++) { 225931639Sfsmp if (id == cpuid) 226031639Sfsmp continue; 226131639Sfsmp if (((1 << id) & checkstate_probed_cpus) == 0) 226231639Sfsmp panic("state for cpu %d not available", cpuid); 226331639Sfsmp forwarded_statclock(id, pscnt, &map); 226431639Sfsmp } 226531639Sfsmp if (map != 0) { 226631639Sfsmp checkstate_need_ast |= map; 226731639Sfsmp selected_apic_ipi(map, XCPUAST_OFFSET, APIC_DELMODE_FIXED); 226831639Sfsmp i = 0; 226931639Sfsmp while (checkstate_need_ast != 0) { 227031639Sfsmp /* spin */ 227131639Sfsmp i++; 227231639Sfsmp if (i > 1000000) { 227331639Sfsmp printf("forward_statclock: dropped ast 0x%x\n", 227431639Sfsmp checkstate_need_ast); 227531639Sfsmp break; 227631639Sfsmp } 227731639Sfsmp } 227831639Sfsmp } 227931639Sfsmp} 228031639Sfsmp 228131639Sfsmpvoid 228231639Sfsmpforward_hardclock(int pscnt) 228331639Sfsmp{ 228431639Sfsmp int map; 228531639Sfsmp int id; 228631639Sfsmp struct proc *p; 228731639Sfsmp struct pstats *pstats; 228831639Sfsmp int i; 228931639Sfsmp 229031639Sfsmp /* Kludge. We don't yet have separate locks for the interrupts 229131639Sfsmp * and the kernel. This means that we cannot let the other processors 229231639Sfsmp * handle complex interrupts while inhibiting them from entering 229331639Sfsmp * the kernel in a non-interrupt context. 229431639Sfsmp * 229531639Sfsmp * What we can do, without changing the locking mechanisms yet, 229631639Sfsmp * is letting the other processors handle a very simple interrupt 229731639Sfsmp * (wich determines the processor states), and do the main 229831639Sfsmp * work ourself. 229931639Sfsmp */ 230031639Sfsmp 230131639Sfsmp if (!smp_started || !invltlb_ok) 230231639Sfsmp return; 230331639Sfsmp 230431639Sfsmp /* Step 1: Probe state (user, cpu, interrupt, spinlock, idle) */ 230531639Sfsmp 230631639Sfsmp map = other_cpus; 230731639Sfsmp checkstate_probed_cpus = 0; 230831639Sfsmp selected_apic_ipi(map, XCPUCHECKSTATE_OFFSET, APIC_DELMODE_FIXED); 230931639Sfsmp 231031639Sfsmp i = 0; 231131639Sfsmp while (checkstate_probed_cpus != map) { 231231639Sfsmp /* spin */ 231331639Sfsmp i++; 231431639Sfsmp if (i == 1000000) { 231531639Sfsmp printf("forward_hardclock: checkstate %x\n", 231631639Sfsmp checkstate_probed_cpus); 231731639Sfsmp } 231831639Sfsmp } 231931639Sfsmp 232031639Sfsmp /* 232131639Sfsmp * Step 2: walk through other processors processes, update virtual 232231639Sfsmp * timer and profiling timer. If stathz == 0, also update ticks and 232331639Sfsmp * profiling info. 232431639Sfsmp */ 232531639Sfsmp 232631639Sfsmp map = 0; 232731639Sfsmp for (id = 0; id < mp_ncpus; id++) { 232831639Sfsmp if (id == cpuid) 232931639Sfsmp continue; 233031639Sfsmp if (((1 << id) & checkstate_probed_cpus) == 0) 233131639Sfsmp panic("state for cpu %d not available", cpuid); 233231639Sfsmp p = checkstate_curproc[id]; 233331639Sfsmp if (p) { 233431639Sfsmp pstats = p->p_stats; 233531639Sfsmp if (checkstate_cpustate[id] == CHECKSTATE_USER && 233631639Sfsmp timerisset(&pstats->p_timer[ITIMER_VIRTUAL].it_value) && 233731639Sfsmp itimerdecr(&pstats->p_timer[ITIMER_VIRTUAL], tick) == 0) { 233831639Sfsmp psignal(p, SIGVTALRM); 233931639Sfsmp map |= (1 << id); 234031639Sfsmp } 234131639Sfsmp if (timerisset(&pstats->p_timer[ITIMER_PROF].it_value) && 234231639Sfsmp itimerdecr(&pstats->p_timer[ITIMER_PROF], tick) == 0) { 234331639Sfsmp psignal(p, SIGPROF); 234431639Sfsmp map |= (1 << id); 234531639Sfsmp } 234631639Sfsmp } 234731639Sfsmp if (stathz == 0) { 234831639Sfsmp forwarded_statclock( id, pscnt, &map); 234931639Sfsmp } 235031639Sfsmp } 235131639Sfsmp if (map != 0) { 235231639Sfsmp checkstate_need_ast |= map; 235331639Sfsmp selected_apic_ipi(map, XCPUAST_OFFSET, APIC_DELMODE_FIXED); 235431639Sfsmp i = 0; 235531639Sfsmp while (checkstate_need_ast != 0) { 235631639Sfsmp /* spin */ 235731639Sfsmp i++; 235831639Sfsmp if (i > 1000000) { 235931639Sfsmp printf("forward_hardclock: dropped ast 0x%x\n", 236031639Sfsmp checkstate_need_ast); 236131639Sfsmp break; 236231639Sfsmp } 236331639Sfsmp } 236431639Sfsmp } 236531639Sfsmp} 236631639Sfsmp 236731639Sfsmp#endif /* BETTER_CLOCK */ 2368