mp_machdep.c revision 27523 
1/*
2 * Copyright (c) 1996, by Steve Passe
3 * All rights reserved.
4 *
5 * Redistribution and use in source and binary forms, with or without
6 * modification, are permitted provided that the following conditions
7 * are met:
8 * 1. Redistributions of source code must retain the above copyright
9 *    notice, this list of conditions and the following disclaimer.
10 * 2. The name of the developer may NOT be used to endorse or promote products
11 *    derived from this software without specific prior written permission.
12 *
13 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
14 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
15 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
16 * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
17 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
18 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
19 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
20 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
21 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
22 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
23 * SUCH DAMAGE.
24 *
25 *	$Id: mp_machdep.c,v 1.30 1997/07/18 21:27:52 fsmp Exp $
26 */
27
28#include "opt_smp.h"
29
30#include <sys/param.h>		/* for KERNBASE */
31#include <sys/types.h>
32#include <sys/sysproto.h>
33#include <sys/time.h>
34#include <sys/systm.h>
35
36#include <vm/vm.h>		/* for KERNBASE */
37#include <vm/vm_param.h>	/* for KERNBASE */
38#include <vm/pmap.h>		/* for KERNBASE */
39#include <machine/pmap.h>	/* for KERNBASE */
40#include <vm/vm_kern.h>
41#include <vm/vm_extern.h>
42
43#include <machine/smp.h>
44#include <machine/apic.h>
45#include <machine/mpapic.h>
46#include <machine/cpufunc.h>
47#include <machine/segments.h>
48#include <machine/smptests.h>	/** TEST_DEFAULT_CONFIG, APIC_PIN0_TIMER, TEST_TEST1 */
49#include <machine/tss.h>
50#include <machine/specialreg.h>
51
52#include <i386/i386/cons.h>	/* cngetc() */
53
54#if defined(APIC_IO)
55#include <machine/md_var.h>		/* setidt() */
56#include <i386/isa/icu.h>		/* IPIs */
57#include <i386/isa/intr_machdep.h>	/* IPIs */
58#endif	/* APIC_IO */
59
60#define WARMBOOT_TARGET		0
61#define WARMBOOT_OFF		(KERNBASE + 0x0467)
62#define WARMBOOT_SEG		(KERNBASE + 0x0469)
63
64#define BIOS_BASE		(0xf0000)
65#define BIOS_SIZE		(0x10000)
66#define BIOS_COUNT		(BIOS_SIZE/4)
67
68#define CMOS_REG		(0x70)
69#define CMOS_DATA		(0x71)
70#define BIOS_RESET		(0x0f)
71#define BIOS_WARM		(0x0a)
72
73#define PROCENTRY_FLAG_EN	0x01
74#define PROCENTRY_FLAG_BP	0x02
75#define IOAPICENTRY_FLAG_EN	0x01
76
77
78/* MP Floating Pointer Structure */
79typedef struct MPFPS {
80	char    signature[4];
81	void   *pap;
82	u_char  length;
83	u_char  spec_rev;
84	u_char  checksum;
85	u_char  mpfb1;
86	u_char  mpfb2;
87	u_char  mpfb3;
88	u_char  mpfb4;
89	u_char  mpfb5;
90}      *mpfps_t;
91
92/* MP Configuration Table Header */
93typedef struct MPCTH {
94	char    signature[4];
95	u_short base_table_length;
96	u_char  spec_rev;
97	u_char  checksum;
98	u_char  oem_id[8];
99	u_char  product_id[12];
100	void   *oem_table_pointer;
101	u_short oem_table_size;
102	u_short entry_count;
103	void   *apic_address;
104	u_short extended_table_length;
105	u_char  extended_table_checksum;
106	u_char  reserved;
107}      *mpcth_t;
108
109
110typedef struct PROCENTRY {
111	u_char  type;
112	u_char  apic_id;
113	u_char  apic_version;
114	u_char  cpu_flags;
115	u_long  cpu_signature;
116	u_long  feature_flags;
117	u_long  reserved1;
118	u_long  reserved2;
119}      *proc_entry_ptr;
120
121typedef struct BUSENTRY {
122	u_char  type;
123	u_char  bus_id;
124	char    bus_type[6];
125}      *bus_entry_ptr;
126
127typedef struct IOAPICENTRY {
128	u_char  type;
129	u_char  apic_id;
130	u_char  apic_version;
131	u_char  apic_flags;
132	void   *apic_address;
133}      *io_apic_entry_ptr;
134
135typedef struct INTENTRY {
136	u_char  type;
137	u_char  int_type;
138	u_short int_flags;
139	u_char  src_bus_id;
140	u_char  src_bus_irq;
141	u_char  dst_apic_id;
142	u_char  dst_apic_int;
143}      *int_entry_ptr;
144
145/* descriptions of MP basetable entries */
146typedef struct BASETABLE_ENTRY {
147	u_char  type;
148	u_char  length;
149	char    name[16];
150}       basetable_entry;
151
152/*
153 * this code MUST be enabled here and in mpboot.s.
154 * it follows the very early stages of AP boot by placing values in CMOS ram.
155 * it NORMALLY will never be needed and thus the primitive method for enabling.
156 *
157#define CHECK_POINTS
158 */
159
160#if defined(CHECK_POINTS)
161#define CHECK_READ(A)	 (outb(CMOS_REG, (A)), inb(CMOS_DATA))
162#define CHECK_WRITE(A,D) (outb(CMOS_REG, (A)), outb(CMOS_DATA, (D)))
163
164#define CHECK_INIT(D);				\
165	CHECK_WRITE(0x34, (D));			\
166	CHECK_WRITE(0x35, (D));			\
167	CHECK_WRITE(0x36, (D));			\
168	CHECK_WRITE(0x37, (D));			\
169	CHECK_WRITE(0x38, (D));			\
170	CHECK_WRITE(0x39, (D));
171
172#define CHECK_PRINT(S);				\
173	printf("%s: %d, %d, %d, %d, %d, %d\n",	\
174	   (S),					\
175	   CHECK_READ(0x34),			\
176	   CHECK_READ(0x35),			\
177	   CHECK_READ(0x36),			\
178	   CHECK_READ(0x37),			\
179	   CHECK_READ(0x38),			\
180	   CHECK_READ(0x39));
181
182#else				/* CHECK_POINTS */
183
184#define CHECK_INIT(D)
185#define CHECK_PRINT(S)
186
187#endif				/* CHECK_POINTS */
188
189/*
190 * Values to send to the POST hardware.
191 */
192#define MP_BOOTADDRESS_POST	0x10
193#define MP_PROBE_POST		0x11
194#define MP_START_POST		0x12
195#define MP_ANNOUNCE_POST	0x13
196#define MPTABLE_PASS1_POST	0x14
197#define MPTABLE_PASS2_POST	0x15
198#define MP_ENABLE_POST		0x16
199#define START_ALL_APS_POST	0x17
200#define INSTALL_AP_TRAMP_POST	0x18
201#define START_AP_POST		0x19
202
203/** XXX FIXME: where does this really belong, isa.h/isa.c perhaps? */
204int	current_postcode;
205
206/** XXX FIXME: what system files declare these??? */
207extern struct region_descriptor r_gdt, r_idt;
208
209int	mp_ncpus;		/* # of CPUs, including BSP */
210int	mp_naps;		/* # of Applications processors */
211int	mp_nbusses;		/* # of busses */
212int	mp_napics;		/* # of IO APICs */
213int	boot_cpu_id;		/* designated BSP */
214vm_offset_t cpu_apic_address;
215vm_offset_t io_apic_address[NAPICID];	/* NAPICID is more than enough */
216
217u_int32_t cpu_apic_versions[NCPU];
218u_int32_t io_apic_versions[NAPIC];
219
220/*
221 * APIC ID logical/physical mapping structures.
222 * We oversize these to simplify boot-time config.
223 */
224int     cpu_num_to_apic_id[NAPICID];
225int     io_num_to_apic_id[NAPICID];
226int     apic_id_to_logical[NAPICID];
227
228/* Bitmap of all available CPUs */
229u_int	all_cpus;
230
231/* Boot of AP uses this PTD */
232u_int *bootPTD;
233
234/* Hotwire a 0->4MB V==P mapping */
235extern pt_entry_t KPTphys;
236
237/* Virtual address of per-cpu common_tss */
238extern struct i386tss common_tss;
239
240/*
241 * Local data and functions.
242 */
243
244static int	mp_capable;
245static u_int	boot_address;
246static u_int	base_memory;
247
248static int	picmode;		/* 0: virtual wire mode, 1: PIC mode */
249static mpfps_t	mpfps;
250static int	search_for_sig(u_int32_t target, int count);
251static void	mp_enable(u_int boot_addr);
252
253static int	mptable_pass1(void);
254static int	mptable_pass2(void);
255static void	default_mp_table(int type);
256static int	start_all_aps(u_int boot_addr);
257static void	install_ap_tramp(u_int boot_addr);
258static int	start_ap(int logicalCpu, u_int boot_addr);
259
260
261/*
262 * Calculate usable address in base memory for AP trampoline code.
263 */
264u_int
265mp_bootaddress(u_int basemem)
266{
267	POSTCODE(MP_BOOTADDRESS_POST);
268
269	base_memory = basemem * 1024;	/* convert to bytes */
270
271	boot_address = base_memory & ~0xfff;	/* round down to 4k boundary */
272	if ((base_memory - boot_address) < bootMP_size)
273		boot_address -= 4096;	/* not enough, lower by 4k */
274
275	return boot_address;
276}
277
278
279/*
280 * Look for an Intel MP spec table (ie, SMP capable hardware).
281 */
282int
283mp_probe(void)
284{
285	int     x;
286	u_long  segment;
287	u_int32_t target;
288
289	POSTCODE(MP_PROBE_POST);
290
291	/* see if EBDA exists */
292	if (segment = (u_long) * (u_short *) (KERNBASE + 0x40e)) {
293		/* search first 1K of EBDA */
294		target = (u_int32_t) (segment << 4);
295		if ((x = search_for_sig(target, 1024 / 4)) >= 0)
296			goto found;
297	} else {
298		/* last 1K of base memory, effective 'top of base' passed in */
299		target = (u_int32_t) (base_memory - 0x400);
300		if ((x = search_for_sig(target, 1024 / 4)) >= 0)
301			goto found;
302	}
303
304	/* search the BIOS */
305	target = (u_int32_t) BIOS_BASE;
306	if ((x = search_for_sig(target, BIOS_COUNT)) >= 0)
307		goto found;
308
309	/* nothing found */
310	mpfps = (mpfps_t)0;
311	mp_capable = 0;
312	return 0;
313
314found:
315	/* calculate needed resources */
316	mpfps = (mpfps_t)x;
317	if (mptable_pass1())
318		panic("you must reconfigure your kernel");
319
320	/* flag fact that we are running multiple processors */
321	mp_capable = 1;
322	return 1;
323}
324
325
326/*
327 * Startup the SMP processors.
328 */
329void
330mp_start(void)
331{
332	POSTCODE(MP_START_POST);
333
334	/* look for MP capable motherboard */
335	if (mp_capable)
336		mp_enable(boot_address);
337	else
338		panic("MP hardware not found!");
339}
340
341
342/*
343 * Print various information about the SMP system hardware and setup.
344 */
345void
346mp_announce(void)
347{
348	int     x;
349
350	POSTCODE(MP_ANNOUNCE_POST);
351
352	printf("FreeBSD/SMP: Multiprocessor motherboard\n");
353	printf(" cpu0 (BSP): apic id: %2d", CPU_TO_ID(0));
354	printf(", version: 0x%08x", cpu_apic_versions[0]);
355	printf(", at 0x%08x\n", cpu_apic_address);
356	for (x = 1; x <= mp_naps; ++x) {
357		printf(" cpu%d (AP):  apic id: %d", x, CPU_TO_ID(x));
358		printf(", version: 0x%08x", cpu_apic_versions[x]);
359		printf(", at 0x%08x\n", cpu_apic_address);
360	}
361
362#if defined(APIC_IO)
363	for (x = 0; x < mp_napics; ++x) {
364		printf(" io%d (APIC): apic id: %2d", x, IO_TO_ID(x));
365		printf(", version: 0x%08x", io_apic_versions[x]);
366		printf(", at 0x%08x\n", io_apic_address[x]);
367	}
368#else
369	printf(" Warning: APIC I/O disabled\n");
370#endif	/* APIC_IO */
371}
372
373/*
374 * AP cpu's call this to sync up protected mode.
375 */
376void
377init_secondary(void)
378{
379	int     gsel_tss, slot;
380
381	r_gdt.rd_limit = sizeof(gdt[0]) * (NGDT + NCPU) - 1;
382	r_gdt.rd_base = (int) gdt;
383	lgdt(&r_gdt);			/* does magic intra-segment return */
384	lidt(&r_idt);
385	lldt(_default_ldt);
386
387	slot = NGDT + cpuid;
388	gsel_tss = GSEL(slot, SEL_KPL);
389	gdt[slot].sd.sd_type = SDT_SYS386TSS;
390	common_tss.tss_esp0 = 0;	/* not used until after switch */
391	common_tss.tss_ss0 = GSEL(GDATA_SEL, SEL_KPL);
392	common_tss.tss_ioopt = (sizeof common_tss) << 16;
393	ltr(gsel_tss);
394
395	load_cr0(0x8005003b);		/* XXX! */
396
397	PTD[0] = 0;
398	pmap_set_opt((unsigned *)PTD);
399
400	invltlb();
401}
402
403
404#if defined(APIC_IO)
405/*
406 * Final configuration of the BSP's local APIC:
407 *  - disable 'pic mode'.
408 *  - disable 'virtual wire mode'.
409 *  - enable NMI.
410 */
411void
412bsp_apic_configure(void)
413{
414	u_char		byte;
415	u_int32_t	temp;
416
417	/* leave 'pic mode' if necessary */
418	if (picmode) {
419		outb(0x22, 0x70);	/* select IMCR */
420		byte = inb(0x23);	/* current contents */
421		byte |= 0x01;		/* mask external INTR */
422		outb(0x23, byte);	/* disconnect 8259s/NMI */
423	}
424
425	/* mask lint0 (the 8259 'virtual wire' connection) */
426	temp = lapic.lvt_lint0;
427	temp |= APIC_LVT_M;		/* set the mask */
428	lapic.lvt_lint0 = temp;
429
430        /* setup lint1 to handle NMI */
431        temp = lapic.lvt_lint1;
432        temp &= ~APIC_LVT_M;		/* clear the mask */
433        lapic.lvt_lint1 = temp;
434
435	if (bootverbose)
436		apic_dump();
437}
438#endif  /* APIC_IO */
439
440
441/*******************************************************************
442 * local functions and data
443 */
444
445/*
446 * start the SMP system
447 */
448static void
449mp_enable(u_int boot_addr)
450{
451	int     x;
452#if defined(APIC_IO)
453	int     apic;
454	u_int   ux;
455#endif	/* APIC_IO */
456
457	POSTCODE(MP_ENABLE_POST);
458
459	/* turn on 4MB of V == P addressing so we can get to MP table */
460	*(int *)PTD = PG_V | PG_RW | ((u_long)KPTphys & PG_FRAME);
461	invltlb();
462
463	/* examine the MP table for needed info, uses physical addresses */
464	x = mptable_pass2();
465
466	*(int *)PTD = 0;
467	invltlb();
468
469	/* can't process default configs till the CPU APIC is pmapped */
470	if (x)
471		default_mp_table(x);
472
473#if defined(APIC_IO)
474
475	/* fill the LOGICAL io_apic_versions table */
476	for (apic = 0; apic < mp_napics; ++apic) {
477		ux = io_apic_read(apic, IOAPIC_VER);
478		io_apic_versions[apic] = ux;
479	}
480
481	/* program each IO APIC in the system */
482	for (apic = 0; apic < mp_napics; ++apic)
483		if (io_apic_setup(apic) < 0)
484			panic("IO APIC setup failure");
485
486	/* install a 'Spurious INTerrupt' vector */
487	setidt(XSPURIOUSINT_OFFSET, Xspuriousint,
488	       SDT_SYS386IGT, SEL_KPL, GSEL(GCODE_SEL, SEL_KPL));
489
490	/* install an inter-CPU IPI for TLB invalidation */
491	setidt(XINVLTLB_OFFSET, Xinvltlb,
492	       SDT_SYS386IGT, SEL_KPL, GSEL(GCODE_SEL, SEL_KPL));
493
494	/* install an inter-CPU IPI for CPU stop/restart */
495	setidt(XCPUSTOP_OFFSET, Xcpustop,
496	       SDT_SYS386IGT, SEL_KPL, GSEL(GCODE_SEL, SEL_KPL));
497
498#if defined(TEST_TEST1)
499	/* install a "fake hardware INTerrupt" vector */
500	setidt(XTEST1_OFFSET, Xtest1,
501	       SDT_SYS386IGT, SEL_KPL, GSEL(GCODE_SEL, SEL_KPL));
502#endif  /** TEST_TEST1 */
503
504#endif	/* APIC_IO */
505
506	/* start each Application Processor */
507	start_all_aps(boot_addr);
508
509	/*
510	 * The init process might be started on a different CPU now,
511	 * and the boot CPU might not call prepare_usermode to get
512	 * cr0 correctly configured. Thus we initialize cr0 here.
513	 */
514	load_cr0(rcr0() | CR0_WP | CR0_AM);
515}
516
517
518/*
519 * look for the MP spec signature
520 */
521
522/* string defined by the Intel MP Spec as identifying the MP table */
523#define MP_SIG		0x5f504d5f	/* _MP_ */
524#define NEXT(X)		((X) += 4)
525static int
526search_for_sig(u_int32_t target, int count)
527{
528	int     x;
529	u_int32_t *addr = (u_int32_t *) (KERNBASE + target);
530
531	for (x = 0; x < count; NEXT(x))
532		if (addr[x] == MP_SIG)
533			/* make array index a byte index */
534			return (target + (x * sizeof(u_int32_t)));
535
536	return -1;
537}
538
539
540static basetable_entry basetable_entry_types[] =
541{
542	{0, 20, "Processor"},
543	{1, 8, "Bus"},
544	{2, 8, "I/O APIC"},
545	{3, 8, "I/O INT"},
546	{4, 8, "Local INT"}
547};
548
549typedef struct BUSDATA {
550	u_char  bus_id;
551	enum busTypes bus_type;
552}       bus_datum;
553
554typedef struct INTDATA {
555	u_char  int_type;
556	u_short int_flags;
557	u_char  src_bus_id;
558	u_char  src_bus_irq;
559	u_char  dst_apic_id;
560	u_char  dst_apic_int;
561}       io_int, local_int;
562
563typedef struct BUSTYPENAME {
564	u_char  type;
565	char    name[7];
566}       bus_type_name;
567
568static bus_type_name bus_type_table[] =
569{
570	{CBUS, "CBUS"},
571	{CBUSII, "CBUSII"},
572	{EISA, "EISA"},
573	{UNKNOWN_BUSTYPE, "---"},
574	{UNKNOWN_BUSTYPE, "---"},
575	{ISA, "ISA"},
576	{UNKNOWN_BUSTYPE, "---"},
577	{UNKNOWN_BUSTYPE, "---"},
578	{UNKNOWN_BUSTYPE, "---"},
579	{UNKNOWN_BUSTYPE, "---"},
580	{UNKNOWN_BUSTYPE, "---"},
581	{UNKNOWN_BUSTYPE, "---"},
582	{PCI, "PCI"},
583	{UNKNOWN_BUSTYPE, "---"},
584	{UNKNOWN_BUSTYPE, "---"},
585	{UNKNOWN_BUSTYPE, "---"},
586	{UNKNOWN_BUSTYPE, "---"},
587	{XPRESS, "XPRESS"},
588	{UNKNOWN_BUSTYPE, "---"}
589};
590/* from MP spec v1.4, table 5-1 */
591static int default_data[7][5] =
592{
593/*   nbus, id0, type0, id1, type1 */
594	{1, 0, ISA, 255, 255},
595	{1, 0, EISA, 255, 255},
596	{1, 0, EISA, 255, 255},
597	{0, 255, 255, 255, 255},/* MCA not supported */
598	{2, 0, ISA, 1, PCI},
599	{2, 0, EISA, 1, PCI},
600	{0, 255, 255, 255, 255}	/* MCA not supported */
601};
602
603
604/* the bus data */
605bus_datum bus_data[NBUS];
606
607/* the IO INT data, one entry per possible APIC INTerrupt */
608io_int  io_apic_ints[NINTR];
609
610static int nintrs;
611
612static void fix_mp_table	__P((void));
613static int processor_entry	__P((proc_entry_ptr entry, int cpu));
614static int bus_entry		__P((bus_entry_ptr entry, int bus));
615static int io_apic_entry	__P((io_apic_entry_ptr entry, int apic));
616static int int_entry		__P((int_entry_ptr entry, int intr));
617static int lookup_bus_type	__P((char *name));
618
619
620/*
621 * 1st pass on motherboard's Intel MP specification table.
622 *
623 * initializes:
624 *	mp_ncpus = 1
625 *
626 * determines:
627 *	cpu_apic_address (common to all CPUs)
628 *	io_apic_address[N]
629 *	mp_naps
630 *	mp_nbusses
631 *	mp_napics
632 *	nintrs
633 */
634static int
635mptable_pass1(void)
636{
637	int	x;
638	mpcth_t	cth;
639	int	totalSize;
640	void*	position;
641	int	count;
642	int	type;
643	int	mustpanic;
644
645	POSTCODE(MPTABLE_PASS1_POST);
646
647	mustpanic = 0;
648
649	/* clear various tables */
650	for (x = 0; x < NAPICID; ++x) {
651		io_apic_address[x] = ~0;	/* IO APIC address table */
652	}
653
654	/* init everything to empty */
655	mp_naps = 0;
656	mp_nbusses = 0;
657	mp_napics = 0;
658	nintrs = 0;
659
660	/* check for use of 'default' configuration */
661	if (mpfps->mpfb1 != 0) {
662		/* use default addresses */
663		cpu_apic_address = DEFAULT_APIC_BASE;
664		io_apic_address[0] = DEFAULT_IO_APIC_BASE;
665
666		/* fill in with defaults */
667		mp_naps = 2;		/* includes BSP */
668		mp_nbusses = default_data[mpfps->mpfb1 - 1][0];
669#if defined(APIC_IO)
670		mp_napics = 1;
671		nintrs = 16;
672#endif	/* APIC_IO */
673	}
674	else {
675		if ((cth = mpfps->pap) == 0)
676			panic("MP Configuration Table Header MISSING!");
677
678		cpu_apic_address = (vm_offset_t) cth->apic_address;
679
680		/* walk the table, recording info of interest */
681		totalSize = cth->base_table_length - sizeof(struct MPCTH);
682		position = (u_char *) cth + sizeof(struct MPCTH);
683		count = cth->entry_count;
684
685		while (count--) {
686			switch (type = *(u_char *) position) {
687			case 0: /* processor_entry */
688				if (((proc_entry_ptr)position)->cpu_flags
689					& PROCENTRY_FLAG_EN)
690					++mp_naps;
691				break;
692			case 1: /* bus_entry */
693				++mp_nbusses;
694				break;
695			case 2: /* io_apic_entry */
696				if (((io_apic_entry_ptr)position)->apic_flags
697					& IOAPICENTRY_FLAG_EN)
698					io_apic_address[mp_napics++] =
699					    (vm_offset_t)((io_apic_entry_ptr)
700						position)->apic_address;
701				break;
702			case 3: /* int_entry */
703				++nintrs;
704				break;
705			case 4:	/* int_entry */
706				break;
707			default:
708				panic("mpfps Base Table HOSED!");
709				/* NOTREACHED */
710			}
711
712			totalSize -= basetable_entry_types[type].length;
713			(u_char*)position += basetable_entry_types[type].length;
714		}
715	}
716
717	/* qualify the numbers */
718	if (mp_naps > NCPU)
719		printf("Warning: only using %d of %d available CPUs!\n",
720			NCPU, mp_naps);
721#if 0
722		/** XXX we consider this legal now (but should we?) */
723		mustpanic = 1;
724#endif
725	if (mp_nbusses > NBUS) {
726		printf("found %d busses, increase NBUS\n", mp_nbusses);
727		mustpanic = 1;
728	}
729	if (mp_napics > NAPIC) {
730		printf("found %d apics, increase NAPIC\n", mp_napics);
731		mustpanic = 1;
732	}
733	if (nintrs > NINTR) {
734		printf("found %d intrs, increase NINTR\n", nintrs);
735		mustpanic = 1;
736	}
737
738	/*
739	 * Count the BSP.
740	 * This is also used as a counter while starting the APs.
741	 */
742	mp_ncpus = 1;
743
744	--mp_naps;	/* subtract the BSP */
745
746	return mustpanic;
747}
748
749
750/*
751 * 2nd pass on motherboard's Intel MP specification table.
752 *
753 * sets:
754 *	boot_cpu_id
755 *	ID_TO_IO(N), phy APIC ID to log CPU/IO table
756 *	CPU_TO_ID(N), logical CPU to APIC ID table
757 *	IO_TO_ID(N), logical IO to APIC ID table
758 *	bus_data[N]
759 *	io_apic_ints[N]
760 */
761static int
762mptable_pass2(void)
763{
764	int     x;
765	mpcth_t cth;
766	int     totalSize;
767	void*   position;
768	int     count;
769	int     type;
770	int     apic, bus, cpu, intr;
771
772	POSTCODE(MPTABLE_PASS2_POST);
773
774	/* clear various tables */
775	for (x = 0; x < NAPICID; ++x) {
776		ID_TO_IO(x) = -1;	/* phy APIC ID to log CPU/IO table */
777		CPU_TO_ID(x) = -1;	/* logical CPU to APIC ID table */
778		IO_TO_ID(x) = -1;	/* logical IO to APIC ID table */
779	}
780
781	/* clear bus data table */
782	for (x = 0; x < NBUS; ++x)
783		bus_data[x].bus_id = 0xff;
784
785	/* clear IO APIC INT table */
786	for (x = 0; x < NINTR; ++x)
787		io_apic_ints[x].int_type = 0xff;
788
789	/* setup the cpu/apic mapping arrays */
790	boot_cpu_id = -1;
791
792	/* record whether PIC or virtual-wire mode */
793	picmode = (mpfps->mpfb2 & 0x80) ? 1 : 0;
794
795	/* check for use of 'default' configuration */
796#if defined(TEST_DEFAULT_CONFIG)
797	return TEST_DEFAULT_CONFIG;
798#else
799	if (mpfps->mpfb1 != 0)
800		return mpfps->mpfb1;	/* return default configuration type */
801#endif	/* TEST_DEFAULT_CONFIG */
802
803	if ((cth = mpfps->pap) == 0)
804		panic("MP Configuration Table Header MISSING!");
805
806	/* walk the table, recording info of interest */
807	totalSize = cth->base_table_length - sizeof(struct MPCTH);
808	position = (u_char *) cth + sizeof(struct MPCTH);
809	count = cth->entry_count;
810	apic = bus = intr = 0;
811	cpu = 1;				/* pre-count the BSP */
812
813	while (count--) {
814		switch (type = *(u_char *) position) {
815		case 0:
816			if (processor_entry(position, cpu))
817				++cpu;
818			break;
819		case 1:
820			if (bus_entry(position, bus))
821				++bus;
822			break;
823		case 2:
824			if (io_apic_entry(position, apic))
825				++apic;
826			break;
827		case 3:
828			if (int_entry(position, intr))
829				++intr;
830			break;
831		case 4:
832			/* int_entry(position); */
833			break;
834		default:
835			panic("mpfps Base Table HOSED!");
836			/* NOTREACHED */
837		}
838
839		totalSize -= basetable_entry_types[type].length;
840		(u_char *) position += basetable_entry_types[type].length;
841	}
842
843	if (boot_cpu_id == -1)
844		panic("NO BSP found!");
845
846	/* post scan cleanup */
847	fix_mp_table();
848
849	/* report fact that its NOT a default configuration */
850	return 0;
851}
852
853
854/*
855 * parse an Intel MP specification table
856 */
857static void
858fix_mp_table(void)
859{
860	int	x;
861	int	id;
862	int	bus_0;
863	int	bus_pci;
864	int	num_pci_bus;
865
866	/*
867	 * Fix mis-numbering of the PCI bus and its INT entries if the BIOS
868	 * did it wrong.  The MP spec says that when more than 1 PCI bus
869	 * exists the BIOS must begin with bus entries for the PCI bus and use
870	 * actual PCI bus numbering.  This implies that when only 1 PCI bus
871	 * exists the BIOS can choose to ignore this ordering, and indeed many
872	 * MP motherboards do ignore it.  This causes a problem when the PCI
873	 * sub-system makes requests of the MP sub-system based on PCI bus
874	 * numbers.	So here we look for the situation and renumber the
875	 * busses and associated INTs in an effort to "make it right".
876	 */
877
878	/* find bus 0, PCI bus, count the number of PCI busses */
879	for (num_pci_bus = 0, x = 0; x < mp_nbusses; ++x) {
880		if (bus_data[x].bus_id == 0) {
881			bus_0 = x;
882		}
883		if (bus_data[x].bus_type == PCI) {
884			++num_pci_bus;
885			bus_pci = x;
886		}
887	}
888	/*
889	 * bus_0 == slot of bus with ID of 0
890	 * bus_pci == slot of last PCI bus encountered
891	 */
892
893	/* check the 1 PCI bus case for sanity */
894	if (num_pci_bus == 1) {
895
896		/* if it is number 0 all is well */
897		if (bus_data[bus_pci].bus_id == 0)
898			return;
899
900		/* mis-numbered, swap with whichever bus uses slot 0 */
901
902		/* swap the bus entry types */
903		bus_data[bus_pci].bus_type = bus_data[bus_0].bus_type;
904		bus_data[bus_0].bus_type = PCI;
905
906		/* swap each relavant INTerrupt entry */
907		id = bus_data[bus_pci].bus_id;
908		for (x = 0; x < nintrs; ++x) {
909			if (io_apic_ints[x].src_bus_id == id) {
910				io_apic_ints[x].src_bus_id = 0;
911			}
912			else if (io_apic_ints[x].src_bus_id == 0) {
913				io_apic_ints[x].src_bus_id = id;
914			}
915		}
916	}
917	/* sanity check if more than 1 PCI bus */
918	else if (num_pci_bus > 1) {
919		for (x = 0; x < mp_nbusses; ++x) {
920			if (bus_data[x].bus_type != PCI)
921				continue;
922			if (bus_data[x].bus_id >= num_pci_bus)
923				panic("bad PCI bus numbering");
924		}
925	}
926}
927
928
929static int
930processor_entry(proc_entry_ptr entry, int cpu)
931{
932	/* check for usability */
933	if ((cpu >= NCPU) || !(entry->cpu_flags & PROCENTRY_FLAG_EN))
934		return 0;
935
936	/* check for BSP flag */
937	if (entry->cpu_flags & PROCENTRY_FLAG_BP) {
938		boot_cpu_id = entry->apic_id;
939		CPU_TO_ID(0) = entry->apic_id;
940		ID_TO_CPU(entry->apic_id) = 0;
941		return 0;	/* its already been counted */
942	}
943
944	/* add another AP to list, if less than max number of CPUs */
945	else {
946		CPU_TO_ID(cpu) = entry->apic_id;
947		ID_TO_CPU(entry->apic_id) = cpu;
948		return 1;
949	}
950}
951
952
953static int
954bus_entry(bus_entry_ptr entry, int bus)
955{
956	int     x;
957	char    c, name[8];
958
959	/* encode the name into an index */
960	for (x = 0; x < 6; ++x) {
961		if ((c = entry->bus_type[x]) == ' ')
962			break;
963		name[x] = c;
964	}
965	name[x] = '\0';
966
967	if ((x = lookup_bus_type(name)) == UNKNOWN_BUSTYPE)
968		panic("unknown bus type: '%s'", name);
969
970	bus_data[bus].bus_id = entry->bus_id;
971	bus_data[bus].bus_type = x;
972
973	return 1;
974}
975
976
977static int
978io_apic_entry(io_apic_entry_ptr entry, int apic)
979{
980	if (!(entry->apic_flags & IOAPICENTRY_FLAG_EN))
981		return 0;
982
983	IO_TO_ID(apic) = entry->apic_id;
984	ID_TO_IO(entry->apic_id) = apic;
985
986	return 1;
987}
988
989
990static int
991lookup_bus_type(char *name)
992{
993	int     x;
994
995	for (x = 0; x < MAX_BUSTYPE; ++x)
996		if (strcmp(bus_type_table[x].name, name) == 0)
997			return bus_type_table[x].type;
998
999	return UNKNOWN_BUSTYPE;
1000}
1001
1002
1003static int
1004int_entry(int_entry_ptr entry, int intr)
1005{
1006	io_apic_ints[intr].int_type = entry->int_type;
1007	io_apic_ints[intr].int_flags = entry->int_flags;
1008	io_apic_ints[intr].src_bus_id = entry->src_bus_id;
1009	io_apic_ints[intr].src_bus_irq = entry->src_bus_irq;
1010	io_apic_ints[intr].dst_apic_id = entry->dst_apic_id;
1011	io_apic_ints[intr].dst_apic_int = entry->dst_apic_int;
1012
1013	return 1;
1014}
1015
1016
1017static int
1018apic_int_is_bus_type(int intr, int bus_type)
1019{
1020	int     bus;
1021
1022	for (bus = 0; bus < mp_nbusses; ++bus)
1023		if ((bus_data[bus].bus_id == io_apic_ints[intr].src_bus_id)
1024		    && ((int) bus_data[bus].bus_type == bus_type))
1025			return 1;
1026
1027	return 0;
1028}
1029
1030
1031/*
1032 * Given a traditional ISA INT mask, return an APIC mask.
1033 */
1034u_int
1035isa_apic_mask(u_int isa_mask)
1036{
1037	int isa_irq;
1038	int apic_pin;
1039
1040#if defined(SKIP_IRQ15_REDIRECT)
1041	if (isa_mask == (1 << 15)) {
1042		printf("skipping ISA IRQ15 redirect\n");
1043		return isa_mask;
1044	}
1045#endif  /* SKIP_IRQ15_REDIRECT */
1046
1047	isa_irq = ffs(isa_mask);		/* find its bit position */
1048	if (isa_irq == 0)			/* doesn't exist */
1049		return 0;
1050	--isa_irq;				/* make it zero based */
1051
1052	apic_pin = isa_apic_pin(isa_irq);	/* look for APIC connection */
1053	if (apic_pin == -1)
1054		return 0;
1055
1056	return (1 << apic_pin);			/* convert pin# to a mask */
1057}
1058
1059
1060/*
1061 * Determine which APIC pin an ISA/EISA INT is attached to.
1062 */
1063#define INTTYPE(I)	(io_apic_ints[(I)].int_type)
1064#define INTPIN(I)	(io_apic_ints[(I)].dst_apic_int)
1065
1066#define SRCBUSIRQ(I)	(io_apic_ints[(I)].src_bus_irq)
1067int
1068isa_apic_pin(int isa_irq)
1069{
1070	int     intr;
1071
1072#if defined(SMP_TIMER_NC)
1073#if defined(APIC_PIN0_TIMER)
1074#error 'options SMP_TIMER_NC' no longer used, remove & reconfig.
1075#else
1076	if (isa_irq == 0)
1077		return -1;
1078#endif	/* APIC_PIN0_TIMER */
1079#endif	/* SMP_TIMER_NC */
1080
1081	for (intr = 0; intr < nintrs; ++intr) {		/* check each record */
1082		if (INTTYPE(intr) == 0) {		/* standard INT */
1083			if (SRCBUSIRQ(intr) == isa_irq) {
1084				if (apic_int_is_bus_type(intr, ISA) ||
1085			            apic_int_is_bus_type(intr, EISA))
1086					return INTPIN(intr);	/* found */
1087			}
1088		}
1089	}
1090	return -1;					/* NOT found */
1091}
1092#undef SRCBUSIRQ
1093
1094
1095/*
1096 * Determine which APIC pin a PCI INT is attached to.
1097 */
1098#define SRCBUSID(I)	(io_apic_ints[(I)].src_bus_id)
1099#define SRCBUSDEVICE(I)	((io_apic_ints[(I)].src_bus_irq >> 2) & 0x1f)
1100#define SRCBUSLINE(I)	(io_apic_ints[(I)].src_bus_irq & 0x03)
1101int
1102pci_apic_pin(int pciBus, int pciDevice, int pciInt)
1103{
1104	int     intr;
1105
1106	--pciInt;					/* zero based */
1107
1108	for (intr = 0; intr < nintrs; ++intr)		/* check each record */
1109		if ((INTTYPE(intr) == 0)		/* standard INT */
1110		    && (SRCBUSID(intr) == pciBus)
1111		    && (SRCBUSDEVICE(intr) == pciDevice)
1112		    && (SRCBUSLINE(intr) == pciInt))	/* a candidate IRQ */
1113			if (apic_int_is_bus_type(intr, PCI))
1114				return INTPIN(intr);	/* exact match */
1115
1116	return -1;					/* NOT found */
1117}
1118#undef SRCBUSLINE
1119#undef SRCBUSDEVICE
1120#undef SRCBUSID
1121
1122#undef INTPIN
1123#undef INTTYPE
1124
1125
1126/*
1127 * Reprogram the MB chipset to NOT redirect an ISA INTerrupt.
1128 *
1129 * XXX FIXME:
1130 *  Exactly what this means is unclear at this point.  It is a solution
1131 *  for motherboards that redirect the MBIRQ0 pin.  Generically a motherboard
1132 *  could route any of the ISA INTs to upper (>15) IRQ values.  But most would
1133 *  NOT be redirected via MBIRQ0, thus "undirect()ing" them would NOT be an
1134 *  option.
1135 */
1136int
1137undirect_isa_irq(int rirq)
1138{
1139#if defined(READY)
1140	printf("Freeing redirected ISA irq %d.\n", rirq);
1141	/** FIXME: tickle the MB redirector chip */
1142	return ???;
1143#else
1144	printf("Freeing (NOT implemented) redirected ISA irq %d.\n", rirq);
1145	return 0;
1146#endif  /* READY */
1147}
1148
1149
1150/*
1151 * Reprogram the MB chipset to NOT redirect a PCI INTerrupt
1152 */
1153int
1154undirect_pci_irq(int rirq)
1155{
1156#if defined(READY)
1157	if (bootverbose)
1158		printf("Freeing redirected PCI irq %d.\n", rirq);
1159
1160	/** FIXME: tickle the MB redirector chip */
1161	return ???;
1162#else
1163	if (bootverbose)
1164		printf("Freeing (NOT implemented) redirected PCI irq %d.\n",
1165		       rirq);
1166	return 0;
1167#endif  /* READY */
1168}
1169
1170
1171/*
1172 * given a bus ID, return:
1173 *  the bus type if found
1174 *  -1 if NOT found
1175 */
1176int
1177apic_bus_type(int id)
1178{
1179	int     x;
1180
1181	for (x = 0; x < mp_nbusses; ++x)
1182		if (bus_data[x].bus_id == id)
1183			return bus_data[x].bus_type;
1184
1185	return -1;
1186}
1187
1188
1189/*
1190 * given a LOGICAL APIC# and pin#, return:
1191 *  the associated src bus ID if found
1192 *  -1 if NOT found
1193 */
1194int
1195apic_src_bus_id(int apic, int pin)
1196{
1197	int     x;
1198
1199	/* search each of the possible INTerrupt sources */
1200	for (x = 0; x < nintrs; ++x)
1201		if ((apic == ID_TO_IO(io_apic_ints[x].dst_apic_id)) &&
1202		    (pin == io_apic_ints[x].dst_apic_int))
1203			return (io_apic_ints[x].src_bus_id);
1204
1205	return -1;		/* NOT found */
1206}
1207
1208
1209/*
1210 * given a LOGICAL APIC# and pin#, return:
1211 *  the associated src bus IRQ if found
1212 *  -1 if NOT found
1213 */
1214int
1215apic_src_bus_irq(int apic, int pin)
1216{
1217	int     x;
1218
1219	for (x = 0; x < nintrs; x++)
1220		if ((apic == ID_TO_IO(io_apic_ints[x].dst_apic_id)) &&
1221		    (pin == io_apic_ints[x].dst_apic_int))
1222			return (io_apic_ints[x].src_bus_irq);
1223
1224	return -1;		/* NOT found */
1225}
1226
1227
1228/*
1229 * given a LOGICAL APIC# and pin#, return:
1230 *  the associated INTerrupt type if found
1231 *  -1 if NOT found
1232 */
1233int
1234apic_int_type(int apic, int pin)
1235{
1236	int     x;
1237
1238	/* search each of the possible INTerrupt sources */
1239	for (x = 0; x < nintrs; ++x)
1240		if ((apic == ID_TO_IO(io_apic_ints[x].dst_apic_id)) &&
1241		    (pin == io_apic_ints[x].dst_apic_int))
1242			return (io_apic_ints[x].int_type);
1243
1244	return -1;		/* NOT found */
1245}
1246
1247
1248/*
1249 * given a LOGICAL APIC# and pin#, return:
1250 *  the associated trigger mode if found
1251 *  -1 if NOT found
1252 */
1253int
1254apic_trigger(int apic, int pin)
1255{
1256	int     x;
1257
1258	/* search each of the possible INTerrupt sources */
1259	for (x = 0; x < nintrs; ++x)
1260		if ((apic == ID_TO_IO(io_apic_ints[x].dst_apic_id)) &&
1261		    (pin == io_apic_ints[x].dst_apic_int))
1262			return ((io_apic_ints[x].int_flags >> 2) & 0x03);
1263
1264	return -1;		/* NOT found */
1265}
1266
1267
1268/*
1269 * given a LOGICAL APIC# and pin#, return:
1270 *  the associated 'active' level if found
1271 *  -1 if NOT found
1272 */
1273int
1274apic_polarity(int apic, int pin)
1275{
1276	int     x;
1277
1278	/* search each of the possible INTerrupt sources */
1279	for (x = 0; x < nintrs; ++x)
1280		if ((apic == ID_TO_IO(io_apic_ints[x].dst_apic_id)) &&
1281		    (pin == io_apic_ints[x].dst_apic_int))
1282			return (io_apic_ints[x].int_flags & 0x03);
1283
1284	return -1;		/* NOT found */
1285}
1286
1287
1288/*
1289 * set data according to MP defaults
1290 * FIXME: probably not complete yet...
1291 */
1292static void
1293default_mp_table(int type)
1294{
1295	int     ap_cpu_id;
1296#if defined(APIC_IO)
1297	u_int32_t ux;
1298	int     io_apic_id;
1299	int     pin;
1300#endif	/* APIC_IO */
1301
1302#if 0
1303	printf("  MP default config type: %d\n", type);
1304	switch (type) {
1305	case 1:
1306		printf("   bus: ISA, APIC: 82489DX\n");
1307		break;
1308	case 2:
1309		printf("   bus: EISA, APIC: 82489DX\n");
1310		break;
1311	case 3:
1312		printf("   bus: EISA, APIC: 82489DX\n");
1313		break;
1314	case 4:
1315		printf("   bus: MCA, APIC: 82489DX\n");
1316		break;
1317	case 5:
1318		printf("   bus: ISA+PCI, APIC: Integrated\n");
1319		break;
1320	case 6:
1321		printf("   bus: EISA+PCI, APIC: Integrated\n");
1322		break;
1323	case 7:
1324		printf("   bus: MCA+PCI, APIC: Integrated\n");
1325		break;
1326	default:
1327		printf("   future type\n");
1328		break;
1329		/* NOTREACHED */
1330	}
1331#endif	/* 0 */
1332
1333	boot_cpu_id = (lapic.id & APIC_ID_MASK) >> 24;
1334	ap_cpu_id = (boot_cpu_id == 0) ? 1 : 0;
1335
1336	/* BSP */
1337	CPU_TO_ID(0) = boot_cpu_id;
1338	ID_TO_CPU(boot_cpu_id) = 0;
1339
1340	/* one and only AP */
1341	CPU_TO_ID(1) = ap_cpu_id;
1342	ID_TO_CPU(ap_cpu_id) = 1;
1343
1344#if defined(APIC_IO)
1345	/* one and only IO APIC */
1346	io_apic_id = (io_apic_read(0, IOAPIC_ID) & APIC_ID_MASK) >> 24;
1347
1348	/*
1349	 * sanity check, refer to MP spec section 3.6.6, last paragraph
1350	 * necessary as some hardware isn't properly setting up the IO APIC
1351	 */
1352#if defined(REALLY_ANAL_IOAPICID_VALUE)
1353	if (io_apic_id != 2) {
1354#else
1355	if ((io_apic_id == 0) || (io_apic_id == 1) || (io_apic_id == 15)) {
1356#endif	/* REALLY_ANAL_IOAPICID_VALUE */
1357		ux = io_apic_read(0, IOAPIC_ID);	/* get current contents */
1358		ux &= ~APIC_ID_MASK;	/* clear the ID field */
1359		ux |= 0x02000000;	/* set it to '2' */
1360		io_apic_write(0, IOAPIC_ID, ux);	/* write new value */
1361		ux = io_apic_read(0, IOAPIC_ID);	/* re-read && test */
1362		if ((ux & APIC_ID_MASK) != 0x02000000)
1363			panic("can't control IO APIC ID, reg: 0x%08x", ux);
1364		io_apic_id = 2;
1365	}
1366	IO_TO_ID(0) = io_apic_id;
1367	ID_TO_IO(io_apic_id) = 0;
1368#endif	/* APIC_IO */
1369
1370	/* fill out bus entries */
1371	switch (type) {
1372	case 1:
1373	case 2:
1374	case 3:
1375	case 5:
1376	case 6:
1377		bus_data[0].bus_id = default_data[type - 1][1];
1378		bus_data[0].bus_type = default_data[type - 1][2];
1379		bus_data[1].bus_id = default_data[type - 1][3];
1380		bus_data[1].bus_type = default_data[type - 1][4];
1381		break;
1382
1383	/* case 4: case 7:		   MCA NOT supported */
1384	default:		/* illegal/reserved */
1385		panic("BAD default MP config: %d", type);
1386		/* NOTREACHED */
1387	}
1388
1389#if defined(APIC_IO)
1390	/* general cases from MP v1.4, table 5-2 */
1391	for (pin = 0; pin < 16; ++pin) {
1392		io_apic_ints[pin].int_type = 0;
1393		io_apic_ints[pin].int_flags = 0x05;	/* edge-triggered/active-hi */
1394		io_apic_ints[pin].src_bus_id = 0;
1395		io_apic_ints[pin].src_bus_irq = pin;	/* IRQ2 is caught below */
1396		io_apic_ints[pin].dst_apic_id = io_apic_id;
1397		io_apic_ints[pin].dst_apic_int = pin;	/* 1-to-1 correspondence */
1398	}
1399
1400	/* special cases from MP v1.4, table 5-2 */
1401	if (type == 2) {
1402		io_apic_ints[2].int_type = 0xff;	/* N/C */
1403		io_apic_ints[13].int_type = 0xff;	/* N/C */
1404#if !defined(APIC_MIXED_MODE)
1405		/** FIXME: ??? */
1406		panic("sorry, can't support type 2 default yet");
1407#endif	/* APIC_MIXED_MODE */
1408	}
1409	else
1410		io_apic_ints[2].src_bus_irq = 0;	/* ISA IRQ0 is on APIC INT 2 */
1411
1412	if (type == 7)
1413		io_apic_ints[0].int_type = 0xff;	/* N/C */
1414	else
1415		io_apic_ints[0].int_type = 3;	/* vectored 8259 */
1416#endif	/* APIC_IO */
1417}
1418
1419
1420/*
1421 * start each AP in our list
1422 */
1423static int
1424start_all_aps(u_int boot_addr)
1425{
1426	int     x, i;
1427	u_char  mpbiosreason;
1428	u_long  mpbioswarmvec;
1429	pd_entry_t newptd;
1430	pt_entry_t newpt;
1431	int *newpp;
1432
1433	POSTCODE(START_ALL_APS_POST);
1434
1435	/**
1436         * NOTE: this needs further thought:
1437         *        where does it get released?
1438         *        should it be set to empy?
1439         *
1440         * get the initial mp_lock with a count of 1 for the BSP
1441         */
1442	mp_lock = 1;	/* this uses a LOGICAL cpu ID, ie BSP == 0 */
1443
1444	/* initialize BSP's local APIC */
1445	apic_initialize();
1446
1447	/* install the AP 1st level boot code */
1448	install_ap_tramp(boot_addr);
1449
1450
1451	/* save the current value of the warm-start vector */
1452	mpbioswarmvec = *((u_long *) WARMBOOT_OFF);
1453	outb(CMOS_REG, BIOS_RESET);
1454	mpbiosreason = inb(CMOS_DATA);
1455
1456	/* record BSP in CPU map */
1457	all_cpus = 1;
1458
1459	/* start each AP */
1460	for (x = 1; x <= mp_naps; ++x) {
1461
1462		/* HACK HACK HACK !!! */
1463
1464		/* alloc new page table directory */
1465		newptd = (pd_entry_t)(kmem_alloc(kernel_map, PAGE_SIZE));
1466
1467		/* clone currently active one (ie: IdlePTD) */
1468		bcopy(PTD, newptd, PAGE_SIZE);	/* inc prv page pde */
1469
1470		/* set up 0 -> 4MB P==V mapping for AP boot */
1471		newptd[0] = PG_V | PG_RW | ((u_long)KPTphys & PG_FRAME);
1472
1473		/* store PTD for this AP */
1474		bootPTD = (pd_entry_t)vtophys(newptd);
1475
1476		/* alloc new page table page */
1477		newpt = (pt_entry_t)(kmem_alloc(kernel_map, PAGE_SIZE));
1478
1479		/* set the new PTD's private page to point there */
1480		newptd[MPPTDI] = PG_V | PG_RW | vtophys(newpt);
1481
1482		/* install self referential entry */
1483		newptd[PTDPTDI] = PG_V | PG_RW | vtophys(newptd);
1484
1485		/* get a new private data page */
1486		newpp = (int *)kmem_alloc(kernel_map, PAGE_SIZE);
1487
1488		/* wire it into the private page table page */
1489		newpt[0] = PG_V | PG_RW | vtophys(newpp);
1490
1491		/* wire the ptp into itself for access */
1492		newpt[1] = PG_V | PG_RW | vtophys(newpt);
1493
1494		/* and the local apic */
1495		newpt[2] = SMP_prvpt[2];
1496
1497		/* and the IO apic mapping[s] */
1498		for (i = 16; i < 32; i++)
1499			newpt[i] = SMP_prvpt[i];
1500
1501		/* prime data page for it to use */
1502		newpp[0] = x;		/* cpuid */
1503		newpp[1] = 0;		/* curproc */
1504		newpp[2] = 0;		/* curpcb */
1505		newpp[3] = 0;		/* npxproc */
1506		newpp[4] = 0;		/* runtime.tv_sec */
1507		newpp[5] = 0;		/* runtime.tv_usec */
1508		newpp[6] = x << 24;	/* cpu_lockid */
1509
1510		/* XXX NOTE: ABANDON bootPTD for now!!!! */
1511
1512		/* END REVOLTING HACKERY */
1513
1514		/* setup a vector to our boot code */
1515		*((volatile u_short *) WARMBOOT_OFF) = WARMBOOT_TARGET;
1516		*((volatile u_short *) WARMBOOT_SEG) = (boot_addr >> 4);
1517		outb(CMOS_REG, BIOS_RESET);
1518		outb(CMOS_DATA, BIOS_WARM);	/* 'warm-start' */
1519
1520		/* attempt to start the Application Processor */
1521		CHECK_INIT(99);	/* setup checkpoints */
1522		if (!start_ap(x, boot_addr)) {
1523			printf("AP #%d (PHY# %d) failed!\n", x, CPU_TO_ID(x));
1524			CHECK_PRINT("trace");	/* show checkpoints */
1525			/* better panic as the AP may be running loose */
1526			printf("panic y/n? [y] ");
1527			if (cngetc() != 'n')
1528				panic("bye-bye");
1529		}
1530		CHECK_PRINT("trace");		/* show checkpoints */
1531
1532		/* record its version info */
1533		cpu_apic_versions[x] = cpu_apic_versions[0];
1534
1535		all_cpus |= (1 << x);		/* record AP in CPU map */
1536	}
1537
1538	/* build our map of 'other' CPUs */
1539	other_cpus = all_cpus & ~(1 << cpuid);
1540
1541	/* fill in our (BSP) APIC version */
1542	cpu_apic_versions[0] = lapic.version;
1543
1544	/* restore the warmstart vector */
1545	*(u_long *) WARMBOOT_OFF = mpbioswarmvec;
1546	outb(CMOS_REG, BIOS_RESET);
1547	outb(CMOS_DATA, mpbiosreason);
1548
1549	pmap_set_opt_bsp();
1550
1551	/* number of APs actually started */
1552	return mp_ncpus - 1;
1553}
1554
1555
1556/*
1557 * load the 1st level AP boot code into base memory.
1558 */
1559
1560/* targets for relocation */
1561extern void bigJump(void);
1562extern void bootCodeSeg(void);
1563extern void bootDataSeg(void);
1564extern void MPentry(void);
1565extern u_int MP_GDT;
1566extern u_int mp_gdtbase;
1567
1568static void
1569install_ap_tramp(u_int boot_addr)
1570{
1571	int     x;
1572	int     size = *(int *) ((u_long) & bootMP_size);
1573	u_char *src = (u_char *) ((u_long) bootMP);
1574	u_char *dst = (u_char *) boot_addr + KERNBASE;
1575	u_int   boot_base = (u_int) bootMP;
1576	u_int8_t *dst8;
1577	u_int16_t *dst16;
1578	u_int32_t *dst32;
1579
1580	POSTCODE(INSTALL_AP_TRAMP_POST);
1581
1582	for (x = 0; x < size; ++x)
1583		*dst++ = *src++;
1584
1585	/*
1586	 * modify addresses in code we just moved to basemem. unfortunately we
1587	 * need fairly detailed info about mpboot.s for this to work.  changes
1588	 * to mpboot.s might require changes here.
1589	 */
1590
1591	/* boot code is located in KERNEL space */
1592	dst = (u_char *) boot_addr + KERNBASE;
1593
1594	/* modify the lgdt arg */
1595	dst32 = (u_int32_t *) (dst + ((u_int) & mp_gdtbase - boot_base));
1596	*dst32 = boot_addr + ((u_int) & MP_GDT - boot_base);
1597
1598	/* modify the ljmp target for MPentry() */
1599	dst32 = (u_int32_t *) (dst + ((u_int) bigJump - boot_base) + 1);
1600	*dst32 = ((u_int) MPentry - KERNBASE);
1601
1602	/* modify the target for boot code segment */
1603	dst16 = (u_int16_t *) (dst + ((u_int) bootCodeSeg - boot_base));
1604	dst8 = (u_int8_t *) (dst16 + 1);
1605	*dst16 = (u_int) boot_addr & 0xffff;
1606	*dst8 = ((u_int) boot_addr >> 16) & 0xff;
1607
1608	/* modify the target for boot data segment */
1609	dst16 = (u_int16_t *) (dst + ((u_int) bootDataSeg - boot_base));
1610	dst8 = (u_int8_t *) (dst16 + 1);
1611	*dst16 = (u_int) boot_addr & 0xffff;
1612	*dst8 = ((u_int) boot_addr >> 16) & 0xff;
1613}
1614
1615
1616/*
1617 * this function starts the AP (application processor) identified
1618 * by the APIC ID 'physicalCpu'.  It does quite a "song and dance"
1619 * to accomplish this.  This is necessary because of the nuances
1620 * of the different hardware we might encounter.  It ain't pretty,
1621 * but it seems to work.
1622 */
1623static int
1624start_ap(int logical_cpu, u_int boot_addr)
1625{
1626	int     physical_cpu;
1627	int     vector;
1628	int     cpus;
1629	u_long  icr_lo, icr_hi;
1630
1631	POSTCODE(START_AP_POST);
1632
1633	/* get the PHYSICAL APIC ID# */
1634	physical_cpu = CPU_TO_ID(logical_cpu);
1635
1636	/* calculate the vector */
1637	vector = (boot_addr >> 12) & 0xff;
1638
1639	/* used as a watchpoint to signal AP startup */
1640	cpus = mp_ncpus;
1641
1642	/*
1643	 * first we do an INIT/RESET IPI this INIT IPI might be run, reseting
1644	 * and running the target CPU. OR this INIT IPI might be latched (P5
1645	 * bug), CPU waiting for STARTUP IPI. OR this INIT IPI might be
1646	 * ignored.
1647	 */
1648
1649	/* setup the address for the target AP */
1650	icr_hi = lapic.icr_hi & ~APIC_ID_MASK;
1651	icr_hi |= (physical_cpu << 24);
1652	lapic.icr_hi = icr_hi;
1653
1654	/* do an INIT IPI: assert RESET */
1655	icr_lo = lapic.icr_lo & 0xfff00000;
1656	lapic.icr_lo = icr_lo | 0x0000c500;
1657
1658	/* wait for pending status end */
1659	while (lapic.icr_lo & APIC_DELSTAT_MASK)
1660		 /* spin */ ;
1661
1662	/* do an INIT IPI: deassert RESET */
1663	lapic.icr_lo = icr_lo | 0x00008500;
1664
1665	/* wait for pending status end */
1666	u_sleep(10000);		/* wait ~10mS */
1667	while (lapic.icr_lo & APIC_DELSTAT_MASK)
1668		 /* spin */ ;
1669
1670	/*
1671	 * next we do a STARTUP IPI: the previous INIT IPI might still be
1672	 * latched, (P5 bug) this 1st STARTUP would then terminate
1673	 * immediately, and the previously started INIT IPI would continue. OR
1674	 * the previous INIT IPI has already run. and this STARTUP IPI will
1675	 * run. OR the previous INIT IPI was ignored. and this STARTUP IPI
1676	 * will run.
1677	 */
1678
1679	/* do a STARTUP IPI */
1680	lapic.icr_lo = icr_lo | 0x00000600 | vector;
1681	while (lapic.icr_lo & APIC_DELSTAT_MASK)
1682		 /* spin */ ;
1683	u_sleep(200);		/* wait ~200uS */
1684
1685	/*
1686	 * finally we do a 2nd STARTUP IPI: this 2nd STARTUP IPI should run IF
1687	 * the previous STARTUP IPI was cancelled by a latched INIT IPI. OR
1688	 * this STARTUP IPI will be ignored, as only ONE STARTUP IPI is
1689	 * recognized after hardware RESET or INIT IPI.
1690	 */
1691
1692	lapic.icr_lo = icr_lo | 0x00000600 | vector;
1693	while (lapic.icr_lo & APIC_DELSTAT_MASK)
1694		 /* spin */ ;
1695	u_sleep(200);		/* wait ~200uS */
1696
1697	/* wait for it to start */
1698	set_apic_timer(5000000);/* == 5 seconds */
1699	while (read_apic_timer())
1700		if (mp_ncpus > cpus)
1701			return 1;	/* return SUCCESS */
1702
1703	return 0;		/* return FAILURE */
1704}
1705
1706
1707/*
1708 * Flush the TLB on all other CPU's
1709 *
1710 * XXX: Needs to handshake and wait for completion before proceding.
1711 */
1712void
1713smp_invltlb(void)
1714{
1715#if defined(APIC_IO)
1716	if (smp_active && invltlb_ok)
1717		all_but_self_ipi(XINVLTLB_OFFSET);
1718#endif  /* APIC_IO */
1719}
1720
1721void
1722invlpg(u_int addr)
1723{
1724	__asm   __volatile("invlpg (%0)"::"r"(addr):"memory");
1725
1726	/* send a message to the other CPUs */
1727	smp_invltlb();
1728}
1729
1730void
1731invltlb(void)
1732{
1733	u_long  temp;
1734
1735	/*
1736	 * This should be implemented as load_cr3(rcr3()) when load_cr3() is
1737	 * inlined.
1738	 */
1739	__asm __volatile("movl %%cr3, %0; movl %0, %%cr3":"=r"(temp) :: "memory");
1740
1741	/* send a message to the other CPUs */
1742	smp_invltlb();
1743}
1744
1745
1746/*
1747 * When called the executing CPU will send an IPI to all other CPUs
1748 *  requesting that they halt execution.
1749 *
1750 * Usually (but not necessarily) called with 'other_cpus' as its arg.
1751 *
1752 *  - Signals all CPUs in map to stop.
1753 *  - Waits for each to stop.
1754 *
1755 * Returns:
1756 *  -1: error
1757 *   0: NA
1758 *   1: ok
1759 *
1760 * XXX FIXME: this is not MP-safe, needs a lock to prevent multiple CPUs
1761 *            from executing at same time.
1762 */
1763int
1764stop_cpus( u_int map )
1765{
1766	if (!smp_active)
1767		return 0;
1768
1769	/* send IPI to all CPUs in map */
1770	stopped_cpus = 0;
1771
1772	/* send the Xcpustop IPI to all CPUs in map */
1773	selected_apic_ipi(map, XCPUSTOP_OFFSET, APIC_DELMODE_FIXED);
1774
1775	while (stopped_cpus != map)
1776		/* spin */ ;
1777
1778	return 1;
1779}
1780
1781
1782/*
1783 * Called by a CPU to restart stopped CPUs.
1784 *
1785 * Usually (but not necessarily) called with 'stopped_cpus' as its arg.
1786 *
1787 *  - Signals all CPUs in map to restart.
1788 *  - Waits for each to restart.
1789 *
1790 * Returns:
1791 *  -1: error
1792 *   0: NA
1793 *   1: ok
1794 */
1795int
1796restart_cpus( u_int map )
1797{
1798	if (!smp_active)
1799		return 0;
1800
1801	started_cpus = map;		/* signal other cpus to restart */
1802
1803	while (started_cpus)		/* wait for each to clear its bit */
1804		/* spin */ ;
1805
1806	return 1;
1807}
1808