mp_x86.c revision 30136 
1/*
2 * Copyright (c) 1996, by Steve Passe
3 * All rights reserved.
4 *
5 * Redistribution and use in source and binary forms, with or without
6 * modification, are permitted provided that the following conditions
7 * are met:
8 * 1. Redistributions of source code must retain the above copyright
9 *    notice, this list of conditions and the following disclaimer.
10 * 2. The name of the developer may NOT be used to endorse or promote products
11 *    derived from this software without specific prior written permission.
12 *
13 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
14 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
15 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
16 * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
17 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
18 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
19 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
20 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
21 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
22 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
23 * SUCH DAMAGE.
24 *
25 *	$Id: mp_machdep.c,v 1.54 1997/09/21 15:03:57 peter Exp $
26 */
27
28#include "opt_smp.h"
29
30#include <sys/param.h>
31#include <sys/systm.h>
32#include <sys/kernel.h>
33#include <sys/proc.h>
34#include <sys/sysctl.h>
35
36#include <vm/vm.h>
37#include <vm/vm_param.h>
38#include <vm/pmap.h>
39#include <vm/vm_kern.h>
40#include <vm/vm_extern.h>
41
42#include <machine/smp.h>
43#include <machine/apic.h>
44#include <machine/mpapic.h>
45#include <machine/segments.h>
46#include <machine/smptests.h>	/** TEST_DEFAULT_CONFIG, TEST_TEST1 */
47#include <machine/tss.h>
48#include <machine/specialreg.h>
49#include <machine/cpu.h>
50#include <machine/cputypes.h>
51
52#include <i386/i386/cons.h>	/* cngetc() */
53
54#if defined(APIC_IO)
55#include <machine/md_var.h>		/* setidt() */
56#include <i386/isa/icu.h>		/* IPIs */
57#include <i386/isa/intr_machdep.h>	/* IPIs */
58#endif	/* APIC_IO */
59
60#if defined(TEST_DEFAULT_CONFIG)
61#define MPFPS_MPFB1	TEST_DEFAULT_CONFIG
62#else
63#define MPFPS_MPFB1	mpfps->mpfb1
64#endif  /* TEST_DEFAULT_CONFIG */
65
66#define WARMBOOT_TARGET		0
67#define WARMBOOT_OFF		(KERNBASE + 0x0467)
68#define WARMBOOT_SEG		(KERNBASE + 0x0469)
69
70#define BIOS_BASE		(0xf0000)
71#define BIOS_SIZE		(0x10000)
72#define BIOS_COUNT		(BIOS_SIZE/4)
73
74#define CMOS_REG		(0x70)
75#define CMOS_DATA		(0x71)
76#define BIOS_RESET		(0x0f)
77#define BIOS_WARM		(0x0a)
78
79#define PROCENTRY_FLAG_EN	0x01
80#define PROCENTRY_FLAG_BP	0x02
81#define IOAPICENTRY_FLAG_EN	0x01
82
83
84/* MP Floating Pointer Structure */
85typedef struct MPFPS {
86	char    signature[4];
87	void   *pap;
88	u_char  length;
89	u_char  spec_rev;
90	u_char  checksum;
91	u_char  mpfb1;
92	u_char  mpfb2;
93	u_char  mpfb3;
94	u_char  mpfb4;
95	u_char  mpfb5;
96}      *mpfps_t;
97
98/* MP Configuration Table Header */
99typedef struct MPCTH {
100	char    signature[4];
101	u_short base_table_length;
102	u_char  spec_rev;
103	u_char  checksum;
104	u_char  oem_id[8];
105	u_char  product_id[12];
106	void   *oem_table_pointer;
107	u_short oem_table_size;
108	u_short entry_count;
109	void   *apic_address;
110	u_short extended_table_length;
111	u_char  extended_table_checksum;
112	u_char  reserved;
113}      *mpcth_t;
114
115
116typedef struct PROCENTRY {
117	u_char  type;
118	u_char  apic_id;
119	u_char  apic_version;
120	u_char  cpu_flags;
121	u_long  cpu_signature;
122	u_long  feature_flags;
123	u_long  reserved1;
124	u_long  reserved2;
125}      *proc_entry_ptr;
126
127typedef struct BUSENTRY {
128	u_char  type;
129	u_char  bus_id;
130	char    bus_type[6];
131}      *bus_entry_ptr;
132
133typedef struct IOAPICENTRY {
134	u_char  type;
135	u_char  apic_id;
136	u_char  apic_version;
137	u_char  apic_flags;
138	void   *apic_address;
139}      *io_apic_entry_ptr;
140
141typedef struct INTENTRY {
142	u_char  type;
143	u_char  int_type;
144	u_short int_flags;
145	u_char  src_bus_id;
146	u_char  src_bus_irq;
147	u_char  dst_apic_id;
148	u_char  dst_apic_int;
149}      *int_entry_ptr;
150
151/* descriptions of MP basetable entries */
152typedef struct BASETABLE_ENTRY {
153	u_char  type;
154	u_char  length;
155	char    name[16];
156}       basetable_entry;
157
158/*
159 * this code MUST be enabled here and in mpboot.s.
160 * it follows the very early stages of AP boot by placing values in CMOS ram.
161 * it NORMALLY will never be needed and thus the primitive method for enabling.
162 *
163#define CHECK_POINTS
164 */
165
166#if defined(CHECK_POINTS)
167#define CHECK_READ(A)	 (outb(CMOS_REG, (A)), inb(CMOS_DATA))
168#define CHECK_WRITE(A,D) (outb(CMOS_REG, (A)), outb(CMOS_DATA, (D)))
169
170#define CHECK_INIT(D);				\
171	CHECK_WRITE(0x34, (D));			\
172	CHECK_WRITE(0x35, (D));			\
173	CHECK_WRITE(0x36, (D));			\
174	CHECK_WRITE(0x37, (D));			\
175	CHECK_WRITE(0x38, (D));			\
176	CHECK_WRITE(0x39, (D));
177
178#define CHECK_PRINT(S);				\
179	printf("%s: %d, %d, %d, %d, %d, %d\n",	\
180	   (S),					\
181	   CHECK_READ(0x34),			\
182	   CHECK_READ(0x35),			\
183	   CHECK_READ(0x36),			\
184	   CHECK_READ(0x37),			\
185	   CHECK_READ(0x38),			\
186	   CHECK_READ(0x39));
187
188#else				/* CHECK_POINTS */
189
190#define CHECK_INIT(D)
191#define CHECK_PRINT(S)
192
193#endif				/* CHECK_POINTS */
194
195/*
196 * Values to send to the POST hardware.
197 */
198#define MP_BOOTADDRESS_POST	0x10
199#define MP_PROBE_POST		0x11
200#define MPTABLE_PASS1_POST	0x12
201
202#define MP_START_POST		0x13
203#define MP_ENABLE_POST		0x14
204#define MPTABLE_PASS2_POST	0x15
205
206#define START_ALL_APS_POST	0x16
207#define INSTALL_AP_TRAMP_POST	0x17
208#define START_AP_POST		0x18
209
210#define MP_ANNOUNCE_POST	0x19
211
212
213/** XXX FIXME: where does this really belong, isa.h/isa.c perhaps? */
214int	current_postcode;
215
216/** XXX FIXME: what system files declare these??? */
217extern struct region_descriptor r_gdt, r_idt;
218
219int	bsp_apic_ready = 0;	/* flags useability of BSP apic */
220int	mp_ncpus;		/* # of CPUs, including BSP */
221int	mp_naps;		/* # of Applications processors */
222int	mp_nbusses;		/* # of busses */
223int	mp_napics;		/* # of IO APICs */
224int	boot_cpu_id;		/* designated BSP */
225vm_offset_t cpu_apic_address;
226vm_offset_t io_apic_address[NAPICID];	/* NAPICID is more than enough */
227extern	int nkpt;
228
229u_int32_t cpu_apic_versions[NCPU];
230u_int32_t io_apic_versions[NAPIC];
231
232/*
233 * APIC ID logical/physical mapping structures.
234 * We oversize these to simplify boot-time config.
235 */
236int     cpu_num_to_apic_id[NAPICID];
237int     io_num_to_apic_id[NAPICID];
238int     apic_id_to_logical[NAPICID];
239
240
241#define NPPROVMTRR	8
242#define	PPRO_VMTRRphysBase0 0x200
243#define	PPRO_VMTRRphysMask0 0x201
244static struct {
245	u_int64_t base,
246			  mask;
247} PPro_vmtrr[NPPROVMTRR];
248
249/* Bitmap of all available CPUs */
250u_int	all_cpus;
251
252/* AP uses this PTD during bootstrap */
253pd_entry_t *bootPTD;
254
255/* Hotwire a 0->4MB V==P mapping */
256extern pt_entry_t *KPTphys;
257
258/* Virtual address of per-cpu common_tss */
259extern struct i386tss common_tss;
260#ifdef VM86
261extern struct segment_descriptor common_tssd;
262extern u_int private_tss;		/* flag indicating private tss */
263extern u_int my_tr;
264#endif /* VM86 */
265
266/* IdlePTD per cpu */
267pd_entry_t *IdlePTDS[NCPU];
268
269/* "my" private page table page, for BSP init */
270extern pt_entry_t SMP_prvpt[];
271
272/* Private page pointer to curcpu's PTD, used during BSP init */
273extern pd_entry_t *my_idlePTD;
274
275static int smp_started;		/* has the system started? */
276
277/*
278 * Local data and functions.
279 */
280
281static int	mp_capable;
282static u_int	boot_address;
283static u_int	base_memory;
284
285static int	picmode;		/* 0: virtual wire mode, 1: PIC mode */
286static mpfps_t	mpfps;
287static int	search_for_sig(u_int32_t target, int count);
288static void	mp_enable(u_int boot_addr);
289
290static int	mptable_pass1(void);
291static int	mptable_pass2(void);
292static void	default_mp_table(int type);
293static void	fix_mp_table(void);
294static void	init_locks(void);
295static int	start_all_aps(u_int boot_addr);
296static void	install_ap_tramp(u_int boot_addr);
297static int	start_ap(int logicalCpu, u_int boot_addr);
298static void getmtrr(void) ;
299static void putmtrr(void) ;
300static void putfmtrr(void) ;
301
302
303/*
304 * Calculate usable address in base memory for AP trampoline code.
305 */
306u_int
307mp_bootaddress(u_int basemem)
308{
309	POSTCODE(MP_BOOTADDRESS_POST);
310
311	base_memory = basemem * 1024;	/* convert to bytes */
312
313	boot_address = base_memory & ~0xfff;	/* round down to 4k boundary */
314	if ((base_memory - boot_address) < bootMP_size)
315		boot_address -= 4096;	/* not enough, lower by 4k */
316
317	return boot_address;
318}
319
320
321/*
322 * Look for an Intel MP spec table (ie, SMP capable hardware).
323 */
324int
325mp_probe(void)
326{
327	int     x;
328	u_long  segment;
329	u_int32_t target;
330
331	POSTCODE(MP_PROBE_POST);
332
333	/* see if EBDA exists */
334	if (segment = (u_long) * (u_short *) (KERNBASE + 0x40e)) {
335		/* search first 1K of EBDA */
336		target = (u_int32_t) (segment << 4);
337		if ((x = search_for_sig(target, 1024 / 4)) >= 0)
338			goto found;
339	} else {
340		/* last 1K of base memory, effective 'top of base' passed in */
341		target = (u_int32_t) (base_memory - 0x400);
342		if ((x = search_for_sig(target, 1024 / 4)) >= 0)
343			goto found;
344	}
345
346	/* search the BIOS */
347	target = (u_int32_t) BIOS_BASE;
348	if ((x = search_for_sig(target, BIOS_COUNT)) >= 0)
349		goto found;
350
351	/* nothing found */
352	mpfps = (mpfps_t)0;
353	mp_capable = 0;
354	return 0;
355
356found:
357	/* calculate needed resources */
358	mpfps = (mpfps_t)x;
359	if (mptable_pass1())
360		panic("you must reconfigure your kernel");
361
362	/* flag fact that we are running multiple processors */
363	mp_capable = 1;
364	return 1;
365}
366
367
368/*
369 * Startup the SMP processors.
370 */
371void
372mp_start(void)
373{
374	POSTCODE(MP_START_POST);
375
376	/* look for MP capable motherboard */
377	if (mp_capable)
378		mp_enable(boot_address);
379	else
380		panic("MP hardware not found!");
381}
382
383
384/*
385 * Print various information about the SMP system hardware and setup.
386 */
387void
388mp_announce(void)
389{
390	int     x;
391
392	POSTCODE(MP_ANNOUNCE_POST);
393
394	printf("FreeBSD/SMP: Multiprocessor motherboard\n");
395	printf(" cpu0 (BSP): apic id: %2d", CPU_TO_ID(0));
396	printf(", version: 0x%08x", cpu_apic_versions[0]);
397	printf(", at 0x%08x\n", cpu_apic_address);
398	for (x = 1; x <= mp_naps; ++x) {
399		printf(" cpu%d (AP):  apic id: %2d", x, CPU_TO_ID(x));
400		printf(", version: 0x%08x", cpu_apic_versions[x]);
401		printf(", at 0x%08x\n", cpu_apic_address);
402	}
403
404#if defined(APIC_IO)
405	for (x = 0; x < mp_napics; ++x) {
406		printf(" io%d (APIC): apic id: %2d", x, IO_TO_ID(x));
407		printf(", version: 0x%08x", io_apic_versions[x]);
408		printf(", at 0x%08x\n", io_apic_address[x]);
409	}
410#else
411	printf(" Warning: APIC I/O disabled\n");
412#endif	/* APIC_IO */
413}
414
415/*
416 * AP cpu's call this to sync up protected mode.
417 */
418void
419init_secondary(void)
420{
421	int	gsel_tss;
422#ifndef VM86
423	u_int	my_tr;
424#endif
425
426	r_gdt.rd_limit = sizeof(gdt[0]) * (NGDT + NCPU) - 1;
427	r_gdt.rd_base = (int) gdt;
428	lgdt(&r_gdt);			/* does magic intra-segment return */
429	lidt(&r_idt);
430	lldt(_default_ldt);
431
432	my_tr = NGDT + cpuid;
433	gsel_tss = GSEL(my_tr, SEL_KPL);
434	gdt[my_tr].sd.sd_type = SDT_SYS386TSS;
435	common_tss.tss_esp0 = 0;	/* not used until after switch */
436	common_tss.tss_ss0 = GSEL(GDATA_SEL, SEL_KPL);
437	common_tss.tss_ioopt = (sizeof common_tss) << 16;
438#ifdef VM86
439	common_tssd = gdt[my_tr].sd;
440	private_tss = 0;
441#endif /* VM86 */
442	ltr(gsel_tss);
443
444	load_cr0(0x8005003b);		/* XXX! */
445
446	PTD[0] = 0;
447	pmap_set_opt((unsigned *)PTD);
448
449	putmtrr();
450	putfmtrr();
451
452	invltlb();
453}
454
455
456#if defined(APIC_IO)
457/*
458 * Final configuration of the BSP's local APIC:
459 *  - disable 'pic mode'.
460 *  - disable 'virtual wire mode'.
461 *  - enable NMI.
462 */
463void
464bsp_apic_configure(void)
465{
466	u_char		byte;
467	u_int32_t	temp;
468
469	/* leave 'pic mode' if necessary */
470	if (picmode) {
471		outb(0x22, 0x70);	/* select IMCR */
472		byte = inb(0x23);	/* current contents */
473		byte |= 0x01;		/* mask external INTR */
474		outb(0x23, byte);	/* disconnect 8259s/NMI */
475	}
476
477	/* mask lint0 (the 8259 'virtual wire' connection) */
478	temp = lapic.lvt_lint0;
479	temp |= APIC_LVT_M;		/* set the mask */
480	lapic.lvt_lint0 = temp;
481
482        /* setup lint1 to handle NMI */
483        temp = lapic.lvt_lint1;
484        temp &= ~APIC_LVT_M;		/* clear the mask */
485        lapic.lvt_lint1 = temp;
486
487	if (bootverbose)
488		apic_dump("bsp_apic_configure()");
489}
490#endif  /* APIC_IO */
491
492
493/*******************************************************************
494 * local functions and data
495 */
496
497/*
498 * start the SMP system
499 */
500static void
501mp_enable(u_int boot_addr)
502{
503	int     x;
504#if defined(APIC_IO)
505	int     apic;
506	u_int   ux;
507#endif	/* APIC_IO */
508
509	getmtrr();
510	putfmtrr();
511
512	POSTCODE(MP_ENABLE_POST);
513
514	/* turn on 4MB of V == P addressing so we can get to MP table */
515	*(int *)PTD = PG_V | PG_RW | ((u_long)KPTphys & PG_FRAME);
516	invltlb();
517
518	/* examine the MP table for needed info, uses physical addresses */
519	x = mptable_pass2();
520
521	*(int *)PTD = 0;
522	invltlb();
523
524	/* can't process default configs till the CPU APIC is pmapped */
525	if (x)
526		default_mp_table(x);
527
528	/* post scan cleanup */
529	fix_mp_table();
530
531#if defined(APIC_IO)
532
533	/* fill the LOGICAL io_apic_versions table */
534	for (apic = 0; apic < mp_napics; ++apic) {
535		ux = io_apic_read(apic, IOAPIC_VER);
536		io_apic_versions[apic] = ux;
537	}
538
539	/* program each IO APIC in the system */
540	for (apic = 0; apic < mp_napics; ++apic)
541		if (io_apic_setup(apic) < 0)
542			panic("IO APIC setup failure");
543
544	/* install a 'Spurious INTerrupt' vector */
545	setidt(XSPURIOUSINT_OFFSET, Xspuriousint,
546	       SDT_SYS386IGT, SEL_KPL, GSEL(GCODE_SEL, SEL_KPL));
547
548	/* install an inter-CPU IPI for TLB invalidation */
549	setidt(XINVLTLB_OFFSET, Xinvltlb,
550	       SDT_SYS386IGT, SEL_KPL, GSEL(GCODE_SEL, SEL_KPL));
551
552	/* install an inter-CPU IPI for CPU stop/restart */
553	setidt(XCPUSTOP_OFFSET, Xcpustop,
554	       SDT_SYS386IGT, SEL_KPL, GSEL(GCODE_SEL, SEL_KPL));
555
556#if defined(TEST_TEST1)
557	/* install a "fake hardware INTerrupt" vector */
558	setidt(XTEST1_OFFSET, Xtest1,
559	       SDT_SYS386IGT, SEL_KPL, GSEL(GCODE_SEL, SEL_KPL));
560#endif  /** TEST_TEST1 */
561
562#endif	/* APIC_IO */
563
564	/* initialize all SMP locks */
565	init_locks();
566
567	/* start each Application Processor */
568	start_all_aps(boot_addr);
569
570	/*
571	 * The init process might be started on a different CPU now,
572	 * and the boot CPU might not call prepare_usermode to get
573	 * cr0 correctly configured. Thus we initialize cr0 here.
574	 */
575	load_cr0(rcr0() | CR0_WP | CR0_AM);
576}
577
578
579/*
580 * look for the MP spec signature
581 */
582
583/* string defined by the Intel MP Spec as identifying the MP table */
584#define MP_SIG		0x5f504d5f	/* _MP_ */
585#define NEXT(X)		((X) += 4)
586static int
587search_for_sig(u_int32_t target, int count)
588{
589	int     x;
590	u_int32_t *addr = (u_int32_t *) (KERNBASE + target);
591
592	for (x = 0; x < count; NEXT(x))
593		if (addr[x] == MP_SIG)
594			/* make array index a byte index */
595			return (target + (x * sizeof(u_int32_t)));
596
597	return -1;
598}
599
600
601static basetable_entry basetable_entry_types[] =
602{
603	{0, 20, "Processor"},
604	{1, 8, "Bus"},
605	{2, 8, "I/O APIC"},
606	{3, 8, "I/O INT"},
607	{4, 8, "Local INT"}
608};
609
610typedef struct BUSDATA {
611	u_char  bus_id;
612	enum busTypes bus_type;
613}       bus_datum;
614
615typedef struct INTDATA {
616	u_char  int_type;
617	u_short int_flags;
618	u_char  src_bus_id;
619	u_char  src_bus_irq;
620	u_char  dst_apic_id;
621	u_char  dst_apic_int;
622}       io_int, local_int;
623
624typedef struct BUSTYPENAME {
625	u_char  type;
626	char    name[7];
627}       bus_type_name;
628
629static bus_type_name bus_type_table[] =
630{
631	{CBUS, "CBUS"},
632	{CBUSII, "CBUSII"},
633	{EISA, "EISA"},
634	{UNKNOWN_BUSTYPE, "---"},
635	{UNKNOWN_BUSTYPE, "---"},
636	{ISA, "ISA"},
637	{UNKNOWN_BUSTYPE, "---"},
638	{UNKNOWN_BUSTYPE, "---"},
639	{UNKNOWN_BUSTYPE, "---"},
640	{UNKNOWN_BUSTYPE, "---"},
641	{UNKNOWN_BUSTYPE, "---"},
642	{UNKNOWN_BUSTYPE, "---"},
643	{PCI, "PCI"},
644	{UNKNOWN_BUSTYPE, "---"},
645	{UNKNOWN_BUSTYPE, "---"},
646	{UNKNOWN_BUSTYPE, "---"},
647	{UNKNOWN_BUSTYPE, "---"},
648	{XPRESS, "XPRESS"},
649	{UNKNOWN_BUSTYPE, "---"}
650};
651/* from MP spec v1.4, table 5-1 */
652static int default_data[7][5] =
653{
654/*   nbus, id0, type0, id1, type1 */
655	{1, 0, ISA, 255, 255},
656	{1, 0, EISA, 255, 255},
657	{1, 0, EISA, 255, 255},
658	{0, 255, 255, 255, 255},/* MCA not supported */
659	{2, 0, ISA, 1, PCI},
660	{2, 0, EISA, 1, PCI},
661	{0, 255, 255, 255, 255}	/* MCA not supported */
662};
663
664
665/* the bus data */
666bus_datum bus_data[NBUS];
667
668/* the IO INT data, one entry per possible APIC INTerrupt */
669io_int  io_apic_ints[NINTR];
670
671static int nintrs;
672
673static int processor_entry	__P((proc_entry_ptr entry, int cpu));
674static int bus_entry		__P((bus_entry_ptr entry, int bus));
675static int io_apic_entry	__P((io_apic_entry_ptr entry, int apic));
676static int int_entry		__P((int_entry_ptr entry, int intr));
677static int lookup_bus_type	__P((char *name));
678
679
680/*
681 * 1st pass on motherboard's Intel MP specification table.
682 *
683 * initializes:
684 *	mp_ncpus = 1
685 *
686 * determines:
687 *	cpu_apic_address (common to all CPUs)
688 *	io_apic_address[N]
689 *	mp_naps
690 *	mp_nbusses
691 *	mp_napics
692 *	nintrs
693 */
694static int
695mptable_pass1(void)
696{
697	int	x;
698	mpcth_t	cth;
699	int	totalSize;
700	void*	position;
701	int	count;
702	int	type;
703	int	mustpanic;
704
705	POSTCODE(MPTABLE_PASS1_POST);
706
707	mustpanic = 0;
708
709	/* clear various tables */
710	for (x = 0; x < NAPICID; ++x) {
711		io_apic_address[x] = ~0;	/* IO APIC address table */
712	}
713
714	/* init everything to empty */
715	mp_naps = 0;
716	mp_nbusses = 0;
717	mp_napics = 0;
718	nintrs = 0;
719
720	/* check for use of 'default' configuration */
721	if (MPFPS_MPFB1 != 0) {
722		/* use default addresses */
723		cpu_apic_address = DEFAULT_APIC_BASE;
724		io_apic_address[0] = DEFAULT_IO_APIC_BASE;
725
726		/* fill in with defaults */
727		mp_naps = 2;		/* includes BSP */
728		mp_nbusses = default_data[MPFPS_MPFB1 - 1][0];
729#if defined(APIC_IO)
730		mp_napics = 1;
731		nintrs = 16;
732#endif	/* APIC_IO */
733	}
734	else {
735		if ((cth = mpfps->pap) == 0)
736			panic("MP Configuration Table Header MISSING!");
737
738		cpu_apic_address = (vm_offset_t) cth->apic_address;
739
740		/* walk the table, recording info of interest */
741		totalSize = cth->base_table_length - sizeof(struct MPCTH);
742		position = (u_char *) cth + sizeof(struct MPCTH);
743		count = cth->entry_count;
744
745		while (count--) {
746			switch (type = *(u_char *) position) {
747			case 0: /* processor_entry */
748				if (((proc_entry_ptr)position)->cpu_flags
749					& PROCENTRY_FLAG_EN)
750					++mp_naps;
751				break;
752			case 1: /* bus_entry */
753				++mp_nbusses;
754				break;
755			case 2: /* io_apic_entry */
756				if (((io_apic_entry_ptr)position)->apic_flags
757					& IOAPICENTRY_FLAG_EN)
758					io_apic_address[mp_napics++] =
759					    (vm_offset_t)((io_apic_entry_ptr)
760						position)->apic_address;
761				break;
762			case 3: /* int_entry */
763				++nintrs;
764				break;
765			case 4:	/* int_entry */
766				break;
767			default:
768				panic("mpfps Base Table HOSED!");
769				/* NOTREACHED */
770			}
771
772			totalSize -= basetable_entry_types[type].length;
773			(u_char*)position += basetable_entry_types[type].length;
774		}
775	}
776
777	/* qualify the numbers */
778	if (mp_naps > NCPU)
779#if 0 /* XXX FIXME: kern/4255 */
780		printf("Warning: only using %d of %d available CPUs!\n",
781			NCPU, mp_naps);
782#else
783	{
784		printf("NCPU cannot be different than actual CPU count.\n");
785		printf(" add 'options NCPU=%d' to your kernel config file,\n",
786			mp_naps);
787		printf(" then rerun config & rebuild your SMP kernel\n");
788		mustpanic = 1;
789	}
790#endif /* XXX FIXME: kern/4255 */
791	if (mp_nbusses > NBUS) {
792		printf("found %d busses, increase NBUS\n", mp_nbusses);
793		mustpanic = 1;
794	}
795	if (mp_napics > NAPIC) {
796		printf("found %d apics, increase NAPIC\n", mp_napics);
797		mustpanic = 1;
798	}
799	if (nintrs > NINTR) {
800		printf("found %d intrs, increase NINTR\n", nintrs);
801		mustpanic = 1;
802	}
803
804	/*
805	 * Count the BSP.
806	 * This is also used as a counter while starting the APs.
807	 */
808	mp_ncpus = 1;
809
810	--mp_naps;	/* subtract the BSP */
811
812	return mustpanic;
813}
814
815
816/*
817 * 2nd pass on motherboard's Intel MP specification table.
818 *
819 * sets:
820 *	boot_cpu_id
821 *	ID_TO_IO(N), phy APIC ID to log CPU/IO table
822 *	CPU_TO_ID(N), logical CPU to APIC ID table
823 *	IO_TO_ID(N), logical IO to APIC ID table
824 *	bus_data[N]
825 *	io_apic_ints[N]
826 */
827static int
828mptable_pass2(void)
829{
830	int     x;
831	mpcth_t cth;
832	int     totalSize;
833	void*   position;
834	int     count;
835	int     type;
836	int     apic, bus, cpu, intr;
837
838	POSTCODE(MPTABLE_PASS2_POST);
839
840	/* clear various tables */
841	for (x = 0; x < NAPICID; ++x) {
842		ID_TO_IO(x) = -1;	/* phy APIC ID to log CPU/IO table */
843		CPU_TO_ID(x) = -1;	/* logical CPU to APIC ID table */
844		IO_TO_ID(x) = -1;	/* logical IO to APIC ID table */
845	}
846
847	/* clear bus data table */
848	for (x = 0; x < NBUS; ++x)
849		bus_data[x].bus_id = 0xff;
850
851	/* clear IO APIC INT table */
852	for (x = 0; x < NINTR; ++x)
853		io_apic_ints[x].int_type = 0xff;
854
855	/* setup the cpu/apic mapping arrays */
856	boot_cpu_id = -1;
857
858	/* record whether PIC or virtual-wire mode */
859	picmode = (mpfps->mpfb2 & 0x80) ? 1 : 0;
860
861	/* check for use of 'default' configuration */
862	if (MPFPS_MPFB1 != 0)
863		return MPFPS_MPFB1;	/* return default configuration type */
864
865	if ((cth = mpfps->pap) == 0)
866		panic("MP Configuration Table Header MISSING!");
867
868	/* walk the table, recording info of interest */
869	totalSize = cth->base_table_length - sizeof(struct MPCTH);
870	position = (u_char *) cth + sizeof(struct MPCTH);
871	count = cth->entry_count;
872	apic = bus = intr = 0;
873	cpu = 1;				/* pre-count the BSP */
874
875	while (count--) {
876		switch (type = *(u_char *) position) {
877		case 0:
878			if (processor_entry(position, cpu))
879				++cpu;
880			break;
881		case 1:
882			if (bus_entry(position, bus))
883				++bus;
884			break;
885		case 2:
886			if (io_apic_entry(position, apic))
887				++apic;
888			break;
889		case 3:
890			if (int_entry(position, intr))
891				++intr;
892			break;
893		case 4:
894			/* int_entry(position); */
895			break;
896		default:
897			panic("mpfps Base Table HOSED!");
898			/* NOTREACHED */
899		}
900
901		totalSize -= basetable_entry_types[type].length;
902		(u_char *) position += basetable_entry_types[type].length;
903	}
904
905	if (boot_cpu_id == -1)
906		panic("NO BSP found!");
907
908	/* report fact that its NOT a default configuration */
909	return 0;
910}
911
912
913/*
914 * parse an Intel MP specification table
915 */
916static void
917fix_mp_table(void)
918{
919	int	x;
920	int	id;
921	int	bus_0;
922	int	bus_pci;
923	int	num_pci_bus;
924
925	/*
926	 * Fix mis-numbering of the PCI bus and its INT entries if the BIOS
927	 * did it wrong.  The MP spec says that when more than 1 PCI bus
928	 * exists the BIOS must begin with bus entries for the PCI bus and use
929	 * actual PCI bus numbering.  This implies that when only 1 PCI bus
930	 * exists the BIOS can choose to ignore this ordering, and indeed many
931	 * MP motherboards do ignore it.  This causes a problem when the PCI
932	 * sub-system makes requests of the MP sub-system based on PCI bus
933	 * numbers.	So here we look for the situation and renumber the
934	 * busses and associated INTs in an effort to "make it right".
935	 */
936
937	/* find bus 0, PCI bus, count the number of PCI busses */
938	for (num_pci_bus = 0, x = 0; x < mp_nbusses; ++x) {
939		if (bus_data[x].bus_id == 0) {
940			bus_0 = x;
941		}
942		if (bus_data[x].bus_type == PCI) {
943			++num_pci_bus;
944			bus_pci = x;
945		}
946	}
947	/*
948	 * bus_0 == slot of bus with ID of 0
949	 * bus_pci == slot of last PCI bus encountered
950	 */
951
952	/* check the 1 PCI bus case for sanity */
953	if (num_pci_bus == 1) {
954
955		/* if it is number 0 all is well */
956		if (bus_data[bus_pci].bus_id == 0)
957			return;
958
959		/* mis-numbered, swap with whichever bus uses slot 0 */
960
961		/* swap the bus entry types */
962		bus_data[bus_pci].bus_type = bus_data[bus_0].bus_type;
963		bus_data[bus_0].bus_type = PCI;
964
965		/* swap each relavant INTerrupt entry */
966		id = bus_data[bus_pci].bus_id;
967		for (x = 0; x < nintrs; ++x) {
968			if (io_apic_ints[x].src_bus_id == id) {
969				io_apic_ints[x].src_bus_id = 0;
970			}
971			else if (io_apic_ints[x].src_bus_id == 0) {
972				io_apic_ints[x].src_bus_id = id;
973			}
974		}
975	}
976	/* sanity check if more than 1 PCI bus */
977	else if (num_pci_bus > 1) {
978		for (x = 0; x < mp_nbusses; ++x) {
979			if (bus_data[x].bus_type != PCI)
980				continue;
981			if (bus_data[x].bus_id >= num_pci_bus)
982				panic("bad PCI bus numbering");
983		}
984	}
985}
986
987
988static int
989processor_entry(proc_entry_ptr entry, int cpu)
990{
991	/* check for usability */
992	if ((cpu >= NCPU) || !(entry->cpu_flags & PROCENTRY_FLAG_EN))
993		return 0;
994
995	/* check for BSP flag */
996	if (entry->cpu_flags & PROCENTRY_FLAG_BP) {
997		boot_cpu_id = entry->apic_id;
998		CPU_TO_ID(0) = entry->apic_id;
999		ID_TO_CPU(entry->apic_id) = 0;
1000		return 0;	/* its already been counted */
1001	}
1002
1003	/* add another AP to list, if less than max number of CPUs */
1004	else {
1005		CPU_TO_ID(cpu) = entry->apic_id;
1006		ID_TO_CPU(entry->apic_id) = cpu;
1007		return 1;
1008	}
1009}
1010
1011
1012static int
1013bus_entry(bus_entry_ptr entry, int bus)
1014{
1015	int     x;
1016	char    c, name[8];
1017
1018	/* encode the name into an index */
1019	for (x = 0; x < 6; ++x) {
1020		if ((c = entry->bus_type[x]) == ' ')
1021			break;
1022		name[x] = c;
1023	}
1024	name[x] = '\0';
1025
1026	if ((x = lookup_bus_type(name)) == UNKNOWN_BUSTYPE)
1027		panic("unknown bus type: '%s'", name);
1028
1029	bus_data[bus].bus_id = entry->bus_id;
1030	bus_data[bus].bus_type = x;
1031
1032	return 1;
1033}
1034
1035
1036static int
1037io_apic_entry(io_apic_entry_ptr entry, int apic)
1038{
1039	if (!(entry->apic_flags & IOAPICENTRY_FLAG_EN))
1040		return 0;
1041
1042	IO_TO_ID(apic) = entry->apic_id;
1043	ID_TO_IO(entry->apic_id) = apic;
1044
1045	return 1;
1046}
1047
1048
1049static int
1050lookup_bus_type(char *name)
1051{
1052	int     x;
1053
1054	for (x = 0; x < MAX_BUSTYPE; ++x)
1055		if (strcmp(bus_type_table[x].name, name) == 0)
1056			return bus_type_table[x].type;
1057
1058	return UNKNOWN_BUSTYPE;
1059}
1060
1061
1062static int
1063int_entry(int_entry_ptr entry, int intr)
1064{
1065	io_apic_ints[intr].int_type = entry->int_type;
1066	io_apic_ints[intr].int_flags = entry->int_flags;
1067	io_apic_ints[intr].src_bus_id = entry->src_bus_id;
1068	io_apic_ints[intr].src_bus_irq = entry->src_bus_irq;
1069	io_apic_ints[intr].dst_apic_id = entry->dst_apic_id;
1070	io_apic_ints[intr].dst_apic_int = entry->dst_apic_int;
1071
1072	return 1;
1073}
1074
1075
1076static int
1077apic_int_is_bus_type(int intr, int bus_type)
1078{
1079	int     bus;
1080
1081	for (bus = 0; bus < mp_nbusses; ++bus)
1082		if ((bus_data[bus].bus_id == io_apic_ints[intr].src_bus_id)
1083		    && ((int) bus_data[bus].bus_type == bus_type))
1084			return 1;
1085
1086	return 0;
1087}
1088
1089
1090/*
1091 * Given a traditional ISA INT mask, return an APIC mask.
1092 */
1093u_int
1094isa_apic_mask(u_int isa_mask)
1095{
1096	int isa_irq;
1097	int apic_pin;
1098
1099#if defined(SKIP_IRQ15_REDIRECT)
1100	if (isa_mask == (1 << 15)) {
1101		printf("skipping ISA IRQ15 redirect\n");
1102		return isa_mask;
1103	}
1104#endif  /* SKIP_IRQ15_REDIRECT */
1105
1106	isa_irq = ffs(isa_mask);		/* find its bit position */
1107	if (isa_irq == 0)			/* doesn't exist */
1108		return 0;
1109	--isa_irq;				/* make it zero based */
1110
1111	apic_pin = isa_apic_pin(isa_irq);	/* look for APIC connection */
1112	if (apic_pin == -1)
1113		return 0;
1114
1115	return (1 << apic_pin);			/* convert pin# to a mask */
1116}
1117
1118
1119/*
1120 * Determine which APIC pin an ISA/EISA INT is attached to.
1121 */
1122#define INTTYPE(I)	(io_apic_ints[(I)].int_type)
1123#define INTPIN(I)	(io_apic_ints[(I)].dst_apic_int)
1124
1125#define SRCBUSIRQ(I)	(io_apic_ints[(I)].src_bus_irq)
1126int
1127isa_apic_pin(int isa_irq)
1128{
1129	int     intr;
1130
1131	for (intr = 0; intr < nintrs; ++intr) {		/* check each record */
1132		if (INTTYPE(intr) == 0) {		/* standard INT */
1133			if (SRCBUSIRQ(intr) == isa_irq) {
1134				if (apic_int_is_bus_type(intr, ISA) ||
1135			            apic_int_is_bus_type(intr, EISA))
1136					return INTPIN(intr);	/* found */
1137			}
1138		}
1139	}
1140	return -1;					/* NOT found */
1141}
1142#undef SRCBUSIRQ
1143
1144
1145/*
1146 * Determine which APIC pin a PCI INT is attached to.
1147 */
1148#define SRCBUSID(I)	(io_apic_ints[(I)].src_bus_id)
1149#define SRCBUSDEVICE(I)	((io_apic_ints[(I)].src_bus_irq >> 2) & 0x1f)
1150#define SRCBUSLINE(I)	(io_apic_ints[(I)].src_bus_irq & 0x03)
1151int
1152pci_apic_pin(int pciBus, int pciDevice, int pciInt)
1153{
1154	int     intr;
1155
1156	--pciInt;					/* zero based */
1157
1158	for (intr = 0; intr < nintrs; ++intr)		/* check each record */
1159		if ((INTTYPE(intr) == 0)		/* standard INT */
1160		    && (SRCBUSID(intr) == pciBus)
1161		    && (SRCBUSDEVICE(intr) == pciDevice)
1162		    && (SRCBUSLINE(intr) == pciInt))	/* a candidate IRQ */
1163			if (apic_int_is_bus_type(intr, PCI))
1164				return INTPIN(intr);	/* exact match */
1165
1166	return -1;					/* NOT found */
1167}
1168#undef SRCBUSLINE
1169#undef SRCBUSDEVICE
1170#undef SRCBUSID
1171
1172#undef INTPIN
1173#undef INTTYPE
1174
1175
1176/*
1177 * Reprogram the MB chipset to NOT redirect an ISA INTerrupt.
1178 *
1179 * XXX FIXME:
1180 *  Exactly what this means is unclear at this point.  It is a solution
1181 *  for motherboards that redirect the MBIRQ0 pin.  Generically a motherboard
1182 *  could route any of the ISA INTs to upper (>15) IRQ values.  But most would
1183 *  NOT be redirected via MBIRQ0, thus "undirect()ing" them would NOT be an
1184 *  option.
1185 */
1186int
1187undirect_isa_irq(int rirq)
1188{
1189#if defined(READY)
1190	printf("Freeing redirected ISA irq %d.\n", rirq);
1191	/** FIXME: tickle the MB redirector chip */
1192	return ???;
1193#else
1194	printf("Freeing (NOT implemented) redirected ISA irq %d.\n", rirq);
1195	return 0;
1196#endif  /* READY */
1197}
1198
1199
1200/*
1201 * Reprogram the MB chipset to NOT redirect a PCI INTerrupt
1202 */
1203int
1204undirect_pci_irq(int rirq)
1205{
1206#if defined(READY)
1207	if (bootverbose)
1208		printf("Freeing redirected PCI irq %d.\n", rirq);
1209
1210	/** FIXME: tickle the MB redirector chip */
1211	return ???;
1212#else
1213	if (bootverbose)
1214		printf("Freeing (NOT implemented) redirected PCI irq %d.\n",
1215		       rirq);
1216	return 0;
1217#endif  /* READY */
1218}
1219
1220
1221/*
1222 * given a bus ID, return:
1223 *  the bus type if found
1224 *  -1 if NOT found
1225 */
1226int
1227apic_bus_type(int id)
1228{
1229	int     x;
1230
1231	for (x = 0; x < mp_nbusses; ++x)
1232		if (bus_data[x].bus_id == id)
1233			return bus_data[x].bus_type;
1234
1235	return -1;
1236}
1237
1238
1239/*
1240 * given a LOGICAL APIC# and pin#, return:
1241 *  the associated src bus ID if found
1242 *  -1 if NOT found
1243 */
1244int
1245apic_src_bus_id(int apic, int pin)
1246{
1247	int     x;
1248
1249	/* search each of the possible INTerrupt sources */
1250	for (x = 0; x < nintrs; ++x)
1251		if ((apic == ID_TO_IO(io_apic_ints[x].dst_apic_id)) &&
1252		    (pin == io_apic_ints[x].dst_apic_int))
1253			return (io_apic_ints[x].src_bus_id);
1254
1255	return -1;		/* NOT found */
1256}
1257
1258
1259/*
1260 * given a LOGICAL APIC# and pin#, return:
1261 *  the associated src bus IRQ if found
1262 *  -1 if NOT found
1263 */
1264int
1265apic_src_bus_irq(int apic, int pin)
1266{
1267	int     x;
1268
1269	for (x = 0; x < nintrs; x++)
1270		if ((apic == ID_TO_IO(io_apic_ints[x].dst_apic_id)) &&
1271		    (pin == io_apic_ints[x].dst_apic_int))
1272			return (io_apic_ints[x].src_bus_irq);
1273
1274	return -1;		/* NOT found */
1275}
1276
1277
1278/*
1279 * given a LOGICAL APIC# and pin#, return:
1280 *  the associated INTerrupt type if found
1281 *  -1 if NOT found
1282 */
1283int
1284apic_int_type(int apic, int pin)
1285{
1286	int     x;
1287
1288	/* search each of the possible INTerrupt sources */
1289	for (x = 0; x < nintrs; ++x)
1290		if ((apic == ID_TO_IO(io_apic_ints[x].dst_apic_id)) &&
1291		    (pin == io_apic_ints[x].dst_apic_int))
1292			return (io_apic_ints[x].int_type);
1293
1294	return -1;		/* NOT found */
1295}
1296
1297
1298/*
1299 * given a LOGICAL APIC# and pin#, return:
1300 *  the associated trigger mode if found
1301 *  -1 if NOT found
1302 */
1303int
1304apic_trigger(int apic, int pin)
1305{
1306	int     x;
1307
1308	/* search each of the possible INTerrupt sources */
1309	for (x = 0; x < nintrs; ++x)
1310		if ((apic == ID_TO_IO(io_apic_ints[x].dst_apic_id)) &&
1311		    (pin == io_apic_ints[x].dst_apic_int))
1312			return ((io_apic_ints[x].int_flags >> 2) & 0x03);
1313
1314	return -1;		/* NOT found */
1315}
1316
1317
1318/*
1319 * given a LOGICAL APIC# and pin#, return:
1320 *  the associated 'active' level if found
1321 *  -1 if NOT found
1322 */
1323int
1324apic_polarity(int apic, int pin)
1325{
1326	int     x;
1327
1328	/* search each of the possible INTerrupt sources */
1329	for (x = 0; x < nintrs; ++x)
1330		if ((apic == ID_TO_IO(io_apic_ints[x].dst_apic_id)) &&
1331		    (pin == io_apic_ints[x].dst_apic_int))
1332			return (io_apic_ints[x].int_flags & 0x03);
1333
1334	return -1;		/* NOT found */
1335}
1336
1337
1338/*
1339 * set data according to MP defaults
1340 * FIXME: probably not complete yet...
1341 */
1342static void
1343default_mp_table(int type)
1344{
1345	int     ap_cpu_id;
1346#if defined(APIC_IO)
1347	u_int32_t ux;
1348	int     io_apic_id;
1349	int     pin;
1350#endif	/* APIC_IO */
1351
1352#if 0
1353	printf("  MP default config type: %d\n", type);
1354	switch (type) {
1355	case 1:
1356		printf("   bus: ISA, APIC: 82489DX\n");
1357		break;
1358	case 2:
1359		printf("   bus: EISA, APIC: 82489DX\n");
1360		break;
1361	case 3:
1362		printf("   bus: EISA, APIC: 82489DX\n");
1363		break;
1364	case 4:
1365		printf("   bus: MCA, APIC: 82489DX\n");
1366		break;
1367	case 5:
1368		printf("   bus: ISA+PCI, APIC: Integrated\n");
1369		break;
1370	case 6:
1371		printf("   bus: EISA+PCI, APIC: Integrated\n");
1372		break;
1373	case 7:
1374		printf("   bus: MCA+PCI, APIC: Integrated\n");
1375		break;
1376	default:
1377		printf("   future type\n");
1378		break;
1379		/* NOTREACHED */
1380	}
1381#endif	/* 0 */
1382
1383	boot_cpu_id = (lapic.id & APIC_ID_MASK) >> 24;
1384	ap_cpu_id = (boot_cpu_id == 0) ? 1 : 0;
1385
1386	/* BSP */
1387	CPU_TO_ID(0) = boot_cpu_id;
1388	ID_TO_CPU(boot_cpu_id) = 0;
1389
1390	/* one and only AP */
1391	CPU_TO_ID(1) = ap_cpu_id;
1392	ID_TO_CPU(ap_cpu_id) = 1;
1393
1394#if defined(APIC_IO)
1395	/* one and only IO APIC */
1396	io_apic_id = (io_apic_read(0, IOAPIC_ID) & APIC_ID_MASK) >> 24;
1397
1398	/*
1399	 * sanity check, refer to MP spec section 3.6.6, last paragraph
1400	 * necessary as some hardware isn't properly setting up the IO APIC
1401	 */
1402#if defined(REALLY_ANAL_IOAPICID_VALUE)
1403	if (io_apic_id != 2) {
1404#else
1405	if ((io_apic_id == 0) || (io_apic_id == 1) || (io_apic_id == 15)) {
1406#endif	/* REALLY_ANAL_IOAPICID_VALUE */
1407		ux = io_apic_read(0, IOAPIC_ID);	/* get current contents */
1408		ux &= ~APIC_ID_MASK;	/* clear the ID field */
1409		ux |= 0x02000000;	/* set it to '2' */
1410		io_apic_write(0, IOAPIC_ID, ux);	/* write new value */
1411		ux = io_apic_read(0, IOAPIC_ID);	/* re-read && test */
1412		if ((ux & APIC_ID_MASK) != 0x02000000)
1413			panic("can't control IO APIC ID, reg: 0x%08x", ux);
1414		io_apic_id = 2;
1415	}
1416	IO_TO_ID(0) = io_apic_id;
1417	ID_TO_IO(io_apic_id) = 0;
1418#endif	/* APIC_IO */
1419
1420	/* fill out bus entries */
1421	switch (type) {
1422	case 1:
1423	case 2:
1424	case 3:
1425	case 5:
1426	case 6:
1427		bus_data[0].bus_id = default_data[type - 1][1];
1428		bus_data[0].bus_type = default_data[type - 1][2];
1429		bus_data[1].bus_id = default_data[type - 1][3];
1430		bus_data[1].bus_type = default_data[type - 1][4];
1431		break;
1432
1433	/* case 4: case 7:		   MCA NOT supported */
1434	default:		/* illegal/reserved */
1435		panic("BAD default MP config: %d", type);
1436		/* NOTREACHED */
1437	}
1438
1439#if defined(APIC_IO)
1440	/* general cases from MP v1.4, table 5-2 */
1441	for (pin = 0; pin < 16; ++pin) {
1442		io_apic_ints[pin].int_type = 0;
1443		io_apic_ints[pin].int_flags = 0x05;	/* edge/active-hi */
1444		io_apic_ints[pin].src_bus_id = 0;
1445		io_apic_ints[pin].src_bus_irq = pin;	/* IRQ2 caught below */
1446		io_apic_ints[pin].dst_apic_id = io_apic_id;
1447		io_apic_ints[pin].dst_apic_int = pin;	/* 1-to-1 */
1448	}
1449
1450	/* special cases from MP v1.4, table 5-2 */
1451	if (type == 2) {
1452		io_apic_ints[2].int_type = 0xff;	/* N/C */
1453		io_apic_ints[13].int_type = 0xff;	/* N/C */
1454#if !defined(APIC_MIXED_MODE)
1455		/** FIXME: ??? */
1456		panic("sorry, can't support type 2 default yet");
1457#endif	/* APIC_MIXED_MODE */
1458	}
1459	else
1460		io_apic_ints[2].src_bus_irq = 0;	/* ISA IRQ0 is on APIC INT 2 */
1461
1462	if (type == 7)
1463		io_apic_ints[0].int_type = 0xff;	/* N/C */
1464	else
1465		io_apic_ints[0].int_type = 3;	/* vectored 8259 */
1466#endif	/* APIC_IO */
1467}
1468
1469
1470/*
1471 * initialize all the SMP locks
1472 */
1473
1474/* critical region around IO APIC, apic_imen */
1475struct simplelock	imen_lock;
1476
1477/* critical region around splxx(), cpl, cml, cil, ipending */
1478struct simplelock	cpl_lock;
1479
1480/* Make FAST_INTR() routines sequential */
1481struct simplelock	fast_intr_lock;
1482
1483/* critical region around INTR() routines */
1484struct simplelock	intr_lock;
1485
1486/* lock regions protected in UP kernel via cli/sti */
1487struct simplelock	mpintr_lock;
1488
1489#ifdef USE_COMLOCK
1490/* locks com (tty) data/hardware accesses: a FASTINTR() */
1491struct simplelock	com_lock;
1492#endif /* USE_COMLOCK */
1493
1494#ifdef USE_CLOCKLOCK
1495/* lock regions around the clock hardware */
1496struct simplelock	clock_lock;
1497#endif /* USE_CLOCKLOCK */
1498
1499static void
1500init_locks(void)
1501{
1502	/*
1503	 * Get the initial mp_lock with a count of 1 for the BSP.
1504	 * This uses a LOGICAL cpu ID, ie BSP == 0.
1505	 */
1506	mp_lock = 0x00000001;
1507
1508	/* ISR uses its own "giant lock" */
1509	isr_lock = FREE_LOCK;
1510
1511	s_lock_init((struct simplelock*)&mpintr_lock);
1512
1513	s_lock_init((struct simplelock*)&fast_intr_lock);
1514	s_lock_init((struct simplelock*)&intr_lock);
1515	s_lock_init((struct simplelock*)&imen_lock);
1516	s_lock_init((struct simplelock*)&cpl_lock);
1517
1518#ifdef USE_COMLOCK
1519	s_lock_init((struct simplelock*)&com_lock);
1520#endif /* USE_COMLOCK */
1521#ifdef USE_CLOCKLOCK
1522	s_lock_init((struct simplelock*)&clock_lock);
1523#endif /* USE_CLOCKLOCK */
1524}
1525
1526
1527/*
1528 * start each AP in our list
1529 */
1530static int
1531start_all_aps(u_int boot_addr)
1532{
1533	int     x, i;
1534	u_char  mpbiosreason;
1535	u_long  mpbioswarmvec;
1536	pd_entry_t *newptd;
1537	pt_entry_t *newpt;
1538	int *newpp;
1539	char *stack;
1540	pd_entry_t	*myPTD;
1541
1542	POSTCODE(START_ALL_APS_POST);
1543
1544	/* initialize BSP's local APIC */
1545	apic_initialize();
1546	bsp_apic_ready = 1;
1547
1548	/* install the AP 1st level boot code */
1549	install_ap_tramp(boot_addr);
1550
1551
1552	/* save the current value of the warm-start vector */
1553	mpbioswarmvec = *((u_long *) WARMBOOT_OFF);
1554	outb(CMOS_REG, BIOS_RESET);
1555	mpbiosreason = inb(CMOS_DATA);
1556
1557	/* record BSP in CPU map */
1558	all_cpus = 1;
1559
1560	/* start each AP */
1561	for (x = 1; x <= mp_naps; ++x) {
1562
1563		/* This is a bit verbose, it will go away soon.  */
1564
1565		/* alloc new page table directory */
1566		newptd = (pd_entry_t *)(kmem_alloc(kernel_map, PAGE_SIZE));
1567
1568		/* Store the virtual PTD address for this CPU */
1569		IdlePTDS[x] = newptd;
1570
1571		/* clone currently active one (ie: IdlePTD) */
1572		bcopy(PTD, newptd, PAGE_SIZE);	/* inc prv page pde */
1573
1574		/* set up 0 -> 4MB P==V mapping for AP boot */
1575		newptd[0] = (pd_entry_t) (PG_V | PG_RW |
1576						((u_long)KPTphys & PG_FRAME));
1577
1578		/* store PTD for this AP's boot sequence */
1579		myPTD = (pd_entry_t *)vtophys(newptd);
1580
1581		/* alloc new page table page */
1582		newpt = (pt_entry_t *)(kmem_alloc(kernel_map, PAGE_SIZE));
1583
1584		/* set the new PTD's private page to point there */
1585		newptd[MPPTDI] = (pt_entry_t)(PG_V | PG_RW | vtophys(newpt));
1586
1587		/* install self referential entry */
1588		newptd[PTDPTDI] = (pd_entry_t)(PG_V | PG_RW | vtophys(newptd));
1589
1590		/* allocate a new private data page */
1591		newpp = (int *)kmem_alloc(kernel_map, PAGE_SIZE);
1592
1593		/* wire it into the private page table page */
1594		newpt[0] = (pt_entry_t)(PG_V | PG_RW | vtophys(newpp));
1595
1596		/* wire the ptp into itself for access */
1597		newpt[1] = (pt_entry_t)(PG_V | PG_RW | vtophys(newpt));
1598
1599		/* copy in the pointer to the local apic */
1600		newpt[2] = SMP_prvpt[2];
1601
1602		/* and the IO apic mapping[s] */
1603		for (i = 16; i < 32; i++)
1604			newpt[i] = SMP_prvpt[i];
1605
1606		/* allocate and set up an idle stack data page */
1607		stack = (char *)kmem_alloc(kernel_map, UPAGES*PAGE_SIZE);
1608		for (i = 0; i < UPAGES; i++)
1609			newpt[i + 3] = (pt_entry_t)(PG_V | PG_RW | vtophys(PAGE_SIZE * i + stack));
1610
1611		newpt[5] = 0;			/* *prv_CMAP1 */
1612		newpt[6] = 0;			/* *prv_CMAP2 */
1613		newpt[7] = 0;			/* *prv_CMAP3 */
1614
1615		/* prime data page for it to use */
1616		newpp[0] = x;			/* cpuid */
1617		newpp[1] = 0;			/* curproc */
1618		newpp[2] = 0;			/* curpcb */
1619		newpp[3] = 0;			/* npxproc */
1620		newpp[4] = 0;			/* runtime.tv_sec */
1621		newpp[5] = 0;			/* runtime.tv_usec */
1622		newpp[6] = x << 24;		/* cpu_lockid */
1623		newpp[7] = 0;			/* other_cpus */
1624		newpp[8] = (int)myPTD;		/* my_idlePTD */
1625		newpp[9] = 0;			/* ss_tpr */
1626		newpp[10] = (int)&newpt[5];	/* prv_CMAP1 */
1627		newpp[11] = (int)&newpt[6];	/* prv_CMAP2 */
1628		newpp[12] = (int)&newpt[7];	/* prv_CMAP3 */
1629
1630		/* setup a vector to our boot code */
1631		*((volatile u_short *) WARMBOOT_OFF) = WARMBOOT_TARGET;
1632		*((volatile u_short *) WARMBOOT_SEG) = (boot_addr >> 4);
1633		outb(CMOS_REG, BIOS_RESET);
1634		outb(CMOS_DATA, BIOS_WARM);	/* 'warm-start' */
1635
1636		bootPTD = myPTD;
1637		/* attempt to start the Application Processor */
1638		CHECK_INIT(99);	/* setup checkpoints */
1639		if (!start_ap(x, boot_addr)) {
1640			printf("AP #%d (PHY# %d) failed!\n", x, CPU_TO_ID(x));
1641			CHECK_PRINT("trace");	/* show checkpoints */
1642			/* better panic as the AP may be running loose */
1643			printf("panic y/n? [y] ");
1644			if (cngetc() != 'n')
1645				panic("bye-bye");
1646		}
1647		CHECK_PRINT("trace");		/* show checkpoints */
1648
1649		/* record its version info */
1650		cpu_apic_versions[x] = cpu_apic_versions[0];
1651
1652		all_cpus |= (1 << x);		/* record AP in CPU map */
1653	}
1654
1655	/* build our map of 'other' CPUs */
1656	other_cpus = all_cpus & ~(1 << cpuid);
1657
1658	/* fill in our (BSP) APIC version */
1659	cpu_apic_versions[0] = lapic.version;
1660
1661	/* restore the warmstart vector */
1662	*(u_long *) WARMBOOT_OFF = mpbioswarmvec;
1663	outb(CMOS_REG, BIOS_RESET);
1664	outb(CMOS_DATA, mpbiosreason);
1665
1666	/*
1667	 * Set up the idle context for the BSP.  Similar to above except
1668	 * that some was done by locore, some by pmap.c and some is implicit
1669	 * because the BSP is cpu#0 and the page is initially zero, and also
1670	 * because we can refer to variables by name on the BSP..
1671	 */
1672	newptd = (pd_entry_t *)(kmem_alloc(kernel_map, PAGE_SIZE));
1673
1674	bcopy(PTD, newptd, PAGE_SIZE);	/* inc prv page pde */
1675	IdlePTDS[0] = newptd;
1676
1677	/* Point PTD[] to this page instead of IdlePTD's physical page */
1678	newptd[PTDPTDI] = (pd_entry_t)(PG_V | PG_RW | vtophys(newptd));
1679
1680	my_idlePTD = (pd_entry_t *)vtophys(newptd);
1681
1682	/* Allocate and setup BSP idle stack */
1683	stack = (char *)kmem_alloc(kernel_map, UPAGES * PAGE_SIZE);
1684	for (i = 0; i < UPAGES; i++)
1685		SMP_prvpt[i + 3] = (pt_entry_t)(PG_V | PG_RW | vtophys(PAGE_SIZE * i + stack));
1686
1687	pmap_set_opt_bsp();
1688
1689	for (i = 0; i < mp_ncpus; i++) {
1690		bcopy( (int *) PTD + KPTDI, (int *) IdlePTDS[i] + KPTDI, NKPDE * sizeof (int));
1691	}
1692
1693	/* number of APs actually started */
1694	return mp_ncpus - 1;
1695}
1696
1697
1698/*
1699 * load the 1st level AP boot code into base memory.
1700 */
1701
1702/* targets for relocation */
1703extern void bigJump(void);
1704extern void bootCodeSeg(void);
1705extern void bootDataSeg(void);
1706extern void MPentry(void);
1707extern u_int MP_GDT;
1708extern u_int mp_gdtbase;
1709
1710static void
1711install_ap_tramp(u_int boot_addr)
1712{
1713	int     x;
1714	int     size = *(int *) ((u_long) & bootMP_size);
1715	u_char *src = (u_char *) ((u_long) bootMP);
1716	u_char *dst = (u_char *) boot_addr + KERNBASE;
1717	u_int   boot_base = (u_int) bootMP;
1718	u_int8_t *dst8;
1719	u_int16_t *dst16;
1720	u_int32_t *dst32;
1721
1722	POSTCODE(INSTALL_AP_TRAMP_POST);
1723
1724	for (x = 0; x < size; ++x)
1725		*dst++ = *src++;
1726
1727	/*
1728	 * modify addresses in code we just moved to basemem. unfortunately we
1729	 * need fairly detailed info about mpboot.s for this to work.  changes
1730	 * to mpboot.s might require changes here.
1731	 */
1732
1733	/* boot code is located in KERNEL space */
1734	dst = (u_char *) boot_addr + KERNBASE;
1735
1736	/* modify the lgdt arg */
1737	dst32 = (u_int32_t *) (dst + ((u_int) & mp_gdtbase - boot_base));
1738	*dst32 = boot_addr + ((u_int) & MP_GDT - boot_base);
1739
1740	/* modify the ljmp target for MPentry() */
1741	dst32 = (u_int32_t *) (dst + ((u_int) bigJump - boot_base) + 1);
1742	*dst32 = ((u_int) MPentry - KERNBASE);
1743
1744	/* modify the target for boot code segment */
1745	dst16 = (u_int16_t *) (dst + ((u_int) bootCodeSeg - boot_base));
1746	dst8 = (u_int8_t *) (dst16 + 1);
1747	*dst16 = (u_int) boot_addr & 0xffff;
1748	*dst8 = ((u_int) boot_addr >> 16) & 0xff;
1749
1750	/* modify the target for boot data segment */
1751	dst16 = (u_int16_t *) (dst + ((u_int) bootDataSeg - boot_base));
1752	dst8 = (u_int8_t *) (dst16 + 1);
1753	*dst16 = (u_int) boot_addr & 0xffff;
1754	*dst8 = ((u_int) boot_addr >> 16) & 0xff;
1755}
1756
1757
1758/*
1759 * this function starts the AP (application processor) identified
1760 * by the APIC ID 'physicalCpu'.  It does quite a "song and dance"
1761 * to accomplish this.  This is necessary because of the nuances
1762 * of the different hardware we might encounter.  It ain't pretty,
1763 * but it seems to work.
1764 */
1765static int
1766start_ap(int logical_cpu, u_int boot_addr)
1767{
1768	int     physical_cpu;
1769	int     vector;
1770	int     cpus;
1771	u_long  icr_lo, icr_hi;
1772
1773	POSTCODE(START_AP_POST);
1774
1775	/* get the PHYSICAL APIC ID# */
1776	physical_cpu = CPU_TO_ID(logical_cpu);
1777
1778	/* calculate the vector */
1779	vector = (boot_addr >> 12) & 0xff;
1780
1781	/* used as a watchpoint to signal AP startup */
1782	cpus = mp_ncpus;
1783
1784	/*
1785	 * first we do an INIT/RESET IPI this INIT IPI might be run, reseting
1786	 * and running the target CPU. OR this INIT IPI might be latched (P5
1787	 * bug), CPU waiting for STARTUP IPI. OR this INIT IPI might be
1788	 * ignored.
1789	 */
1790
1791	/* setup the address for the target AP */
1792	icr_hi = lapic.icr_hi & ~APIC_ID_MASK;
1793	icr_hi |= (physical_cpu << 24);
1794	lapic.icr_hi = icr_hi;
1795
1796	/* do an INIT IPI: assert RESET */
1797	icr_lo = lapic.icr_lo & 0xfff00000;
1798	lapic.icr_lo = icr_lo | 0x0000c500;
1799
1800	/* wait for pending status end */
1801	while (lapic.icr_lo & APIC_DELSTAT_MASK)
1802		 /* spin */ ;
1803
1804	/* do an INIT IPI: deassert RESET */
1805	lapic.icr_lo = icr_lo | 0x00008500;
1806
1807	/* wait for pending status end */
1808	u_sleep(10000);		/* wait ~10mS */
1809	while (lapic.icr_lo & APIC_DELSTAT_MASK)
1810		 /* spin */ ;
1811
1812	/*
1813	 * next we do a STARTUP IPI: the previous INIT IPI might still be
1814	 * latched, (P5 bug) this 1st STARTUP would then terminate
1815	 * immediately, and the previously started INIT IPI would continue. OR
1816	 * the previous INIT IPI has already run. and this STARTUP IPI will
1817	 * run. OR the previous INIT IPI was ignored. and this STARTUP IPI
1818	 * will run.
1819	 */
1820
1821	/* do a STARTUP IPI */
1822	lapic.icr_lo = icr_lo | 0x00000600 | vector;
1823	while (lapic.icr_lo & APIC_DELSTAT_MASK)
1824		 /* spin */ ;
1825	u_sleep(200);		/* wait ~200uS */
1826
1827	/*
1828	 * finally we do a 2nd STARTUP IPI: this 2nd STARTUP IPI should run IF
1829	 * the previous STARTUP IPI was cancelled by a latched INIT IPI. OR
1830	 * this STARTUP IPI will be ignored, as only ONE STARTUP IPI is
1831	 * recognized after hardware RESET or INIT IPI.
1832	 */
1833
1834	lapic.icr_lo = icr_lo | 0x00000600 | vector;
1835	while (lapic.icr_lo & APIC_DELSTAT_MASK)
1836		 /* spin */ ;
1837	u_sleep(200);		/* wait ~200uS */
1838
1839	/* wait for it to start */
1840	set_apic_timer(5000000);/* == 5 seconds */
1841	while (read_apic_timer())
1842		if (mp_ncpus > cpus)
1843			return 1;	/* return SUCCESS */
1844
1845	return 0;		/* return FAILURE */
1846}
1847
1848
1849/*
1850 * Flush the TLB on all other CPU's
1851 *
1852 * XXX: Needs to handshake and wait for completion before proceding.
1853 */
1854void
1855smp_invltlb(void)
1856{
1857#if defined(APIC_IO)
1858	if (smp_started && invltlb_ok)
1859		all_but_self_ipi(XINVLTLB_OFFSET);
1860#endif  /* APIC_IO */
1861}
1862
1863void
1864invlpg(u_int addr)
1865{
1866	__asm   __volatile("invlpg (%0)"::"r"(addr):"memory");
1867
1868	/* send a message to the other CPUs */
1869	smp_invltlb();
1870}
1871
1872void
1873invltlb(void)
1874{
1875	u_long  temp;
1876
1877	/*
1878	 * This should be implemented as load_cr3(rcr3()) when load_cr3() is
1879	 * inlined.
1880	 */
1881	__asm __volatile("movl %%cr3, %0; movl %0, %%cr3":"=r"(temp) :: "memory");
1882
1883	/* send a message to the other CPUs */
1884	smp_invltlb();
1885}
1886
1887
1888/*
1889 * When called the executing CPU will send an IPI to all other CPUs
1890 *  requesting that they halt execution.
1891 *
1892 * Usually (but not necessarily) called with 'other_cpus' as its arg.
1893 *
1894 *  - Signals all CPUs in map to stop.
1895 *  - Waits for each to stop.
1896 *
1897 * Returns:
1898 *  -1: error
1899 *   0: NA
1900 *   1: ok
1901 *
1902 * XXX FIXME: this is not MP-safe, needs a lock to prevent multiple CPUs
1903 *            from executing at same time.
1904 */
1905int
1906stop_cpus(u_int map)
1907{
1908	if (!smp_started)
1909		return 0;
1910
1911	/* send IPI to all CPUs in map */
1912	stopped_cpus = 0;
1913
1914	/* send the Xcpustop IPI to all CPUs in map */
1915	selected_apic_ipi(map, XCPUSTOP_OFFSET, APIC_DELMODE_FIXED);
1916
1917	while (stopped_cpus != map)
1918		/* spin */ ;
1919
1920	return 1;
1921}
1922
1923
1924/*
1925 * Called by a CPU to restart stopped CPUs.
1926 *
1927 * Usually (but not necessarily) called with 'stopped_cpus' as its arg.
1928 *
1929 *  - Signals all CPUs in map to restart.
1930 *  - Waits for each to restart.
1931 *
1932 * Returns:
1933 *  -1: error
1934 *   0: NA
1935 *   1: ok
1936 */
1937int
1938restart_cpus(u_int map)
1939{
1940	if (!smp_started)
1941		return 0;
1942
1943	started_cpus = map;		/* signal other cpus to restart */
1944
1945	while (started_cpus)		/* wait for each to clear its bit */
1946		/* spin */ ;
1947
1948	return 1;
1949}
1950
1951int smp_active = 0;	/* are the APs allowed to run? */
1952SYSCTL_INT(_machdep, OID_AUTO, smp_active, CTLFLAG_RW, &smp_active, 0, "");
1953
1954/* XXX maybe should be hw.ncpu */
1955int smp_cpus = 1;	/* how many cpu's running */
1956SYSCTL_INT(_machdep, OID_AUTO, smp_cpus, CTLFLAG_RD, &smp_cpus, 0, "");
1957
1958int invltlb_ok = 0;	/* throttle smp_invltlb() till safe */
1959SYSCTL_INT(_machdep, OID_AUTO, invltlb_ok, CTLFLAG_RW, &invltlb_ok, 0, "");
1960
1961int do_page_zero_idle = 0; /* bzero pages for fun and profit in idleloop */
1962SYSCTL_INT(_machdep, OID_AUTO, do_page_zero_idle, CTLFLAG_RW,
1963	   &do_page_zero_idle, 0, "");
1964
1965
1966/*
1967 * This is called once the rest of the system is up and running and we're
1968 * ready to let the AP's out of the pen.
1969 */
1970void ap_init(void);
1971
1972void
1973ap_init()
1974{
1975	u_int   temp;
1976	u_int	apic_id;
1977
1978	smp_cpus++;
1979
1980	/* Build our map of 'other' CPUs. */
1981	other_cpus = all_cpus & ~(1 << cpuid);
1982
1983	printf("SMP: AP CPU #%d Launched!\n", cpuid);
1984
1985	/* XXX FIXME: i386 specific, and redundant: Setup the FPU. */
1986	load_cr0((rcr0() & ~CR0_EM) | CR0_MP | CR0_NE | CR0_TS);
1987
1988	/* A quick check from sanity claus */
1989	apic_id = (apic_id_to_logical[(lapic.id & 0x0f000000) >> 24]);
1990	if (cpuid != apic_id) {
1991		printf("SMP: cpuid = %d\n", cpuid);
1992		printf("SMP: apic_id = %d\n", apic_id);
1993		printf("PTD[MPPTDI] = %08x\n", PTD[MPPTDI]);
1994		panic("cpuid mismatch! boom!!");
1995	}
1996
1997	/* Init local apic for irq's */
1998	apic_initialize();
1999
2000	/*
2001	 * Activate smp_invltlb, although strictly speaking, this isn't
2002	 * quite correct yet.  We should have a bitfield for cpus willing
2003	 * to accept TLB flush IPI's or something and sync them.
2004	 */
2005	invltlb_ok = 1;
2006	smp_started = 1;	/* enable IPI's, tlb shootdown, freezes etc */
2007	smp_active = 1;		/* historic */
2008
2009	curproc = NULL;		/* make sure */
2010}
2011
2012void
2013getmtrr() {
2014	int i;
2015	if (cpu_class == CPUCLASS_686) {
2016		for(i=0;i<NPPROVMTRR;i++) {
2017			PPro_vmtrr[i].base = rdmsr(PPRO_VMTRRphysBase0 + i * 2);
2018			PPro_vmtrr[i].mask = rdmsr(PPRO_VMTRRphysMask0 + i * 2);
2019		}
2020	}
2021}
2022
2023void
2024putmtrr() {
2025	int i;
2026	if (cpu_class == CPUCLASS_686) {
2027		wbinvd();
2028		for(i=0;i<NPPROVMTRR;i++) {
2029			wrmsr(PPRO_VMTRRphysBase0 + i * 2, PPro_vmtrr[i].base);
2030			wrmsr(PPRO_VMTRRphysMask0 + i * 2, PPro_vmtrr[i].mask);
2031		}
2032	}
2033}
2034
2035void
2036putfmtrr() {
2037	if (cpu_class == CPUCLASS_686) {
2038		wbinvd();
2039		/*
2040		 * Set memory between 0-640K to be WB
2041		 */
2042		wrmsr(0x250, 0x0606060606060606LL);
2043		wrmsr(0x258, 0x0606060606060606LL);
2044		/*
2045		 * Set normal, PC video memory to be WC
2046		 */
2047		wrmsr(0x259, 0x0101010101010101LL);
2048	}
2049}
2050
2051