mp_x86.c revision 27612 
1/*
2 * Copyright (c) 1996, by Steve Passe
3 * All rights reserved.
4 *
5 * Redistribution and use in source and binary forms, with or without
6 * modification, are permitted provided that the following conditions
7 * are met:
8 * 1. Redistributions of source code must retain the above copyright
9 *    notice, this list of conditions and the following disclaimer.
10 * 2. The name of the developer may NOT be used to endorse or promote products
11 *    derived from this software without specific prior written permission.
12 *
13 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
14 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
15 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
16 * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
17 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
18 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
19 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
20 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
21 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
22 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
23 * SUCH DAMAGE.
24 *
25 *	$Id: mp_machdep.c,v 1.18 1997/07/20 18:13:01 smp Exp smp $
26 */
27
28#include "opt_smp.h"
29
30#include <sys/param.h>		/* for KERNBASE */
31#include <sys/systm.h>
32
33#include <vm/vm.h>		/* for KERNBASE */
34#include <vm/vm_param.h>	/* for KERNBASE */
35#include <vm/pmap.h>		/* for KERNBASE */
36#include <vm/vm_kern.h>
37#include <vm/vm_extern.h>
38
39#include <machine/smp.h>
40#include <machine/apic.h>
41#include <machine/mpapic.h>
42#include <machine/segments.h>
43#include <machine/smptests.h>	/** TEST_DEFAULT_CONFIG, APIC_PIN0_TIMER, TEST_TEST1 */
44#include <machine/tss.h>
45#include <machine/specialreg.h>
46
47#include <i386/i386/cons.h>	/* cngetc() */
48
49#if defined(APIC_IO)
50#include <machine/md_var.h>		/* setidt() */
51#include <i386/isa/icu.h>		/* IPIs */
52#include <i386/isa/intr_machdep.h>	/* IPIs */
53#endif	/* APIC_IO */
54
55#define WARMBOOT_TARGET		0
56#define WARMBOOT_OFF		(KERNBASE + 0x0467)
57#define WARMBOOT_SEG		(KERNBASE + 0x0469)
58
59#define BIOS_BASE		(0xf0000)
60#define BIOS_SIZE		(0x10000)
61#define BIOS_COUNT		(BIOS_SIZE/4)
62
63#define CMOS_REG		(0x70)
64#define CMOS_DATA		(0x71)
65#define BIOS_RESET		(0x0f)
66#define BIOS_WARM		(0x0a)
67
68#define PROCENTRY_FLAG_EN	0x01
69#define PROCENTRY_FLAG_BP	0x02
70#define IOAPICENTRY_FLAG_EN	0x01
71
72
73/* MP Floating Pointer Structure */
74typedef struct MPFPS {
75	char    signature[4];
76	void   *pap;
77	u_char  length;
78	u_char  spec_rev;
79	u_char  checksum;
80	u_char  mpfb1;
81	u_char  mpfb2;
82	u_char  mpfb3;
83	u_char  mpfb4;
84	u_char  mpfb5;
85}      *mpfps_t;
86
87/* MP Configuration Table Header */
88typedef struct MPCTH {
89	char    signature[4];
90	u_short base_table_length;
91	u_char  spec_rev;
92	u_char  checksum;
93	u_char  oem_id[8];
94	u_char  product_id[12];
95	void   *oem_table_pointer;
96	u_short oem_table_size;
97	u_short entry_count;
98	void   *apic_address;
99	u_short extended_table_length;
100	u_char  extended_table_checksum;
101	u_char  reserved;
102}      *mpcth_t;
103
104
105typedef struct PROCENTRY {
106	u_char  type;
107	u_char  apic_id;
108	u_char  apic_version;
109	u_char  cpu_flags;
110	u_long  cpu_signature;
111	u_long  feature_flags;
112	u_long  reserved1;
113	u_long  reserved2;
114}      *proc_entry_ptr;
115
116typedef struct BUSENTRY {
117	u_char  type;
118	u_char  bus_id;
119	char    bus_type[6];
120}      *bus_entry_ptr;
121
122typedef struct IOAPICENTRY {
123	u_char  type;
124	u_char  apic_id;
125	u_char  apic_version;
126	u_char  apic_flags;
127	void   *apic_address;
128}      *io_apic_entry_ptr;
129
130typedef struct INTENTRY {
131	u_char  type;
132	u_char  int_type;
133	u_short int_flags;
134	u_char  src_bus_id;
135	u_char  src_bus_irq;
136	u_char  dst_apic_id;
137	u_char  dst_apic_int;
138}      *int_entry_ptr;
139
140/* descriptions of MP basetable entries */
141typedef struct BASETABLE_ENTRY {
142	u_char  type;
143	u_char  length;
144	char    name[16];
145}       basetable_entry;
146
147/*
148 * this code MUST be enabled here and in mpboot.s.
149 * it follows the very early stages of AP boot by placing values in CMOS ram.
150 * it NORMALLY will never be needed and thus the primitive method for enabling.
151 *
152#define CHECK_POINTS
153 */
154
155#if defined(CHECK_POINTS)
156#define CHECK_READ(A)	 (outb(CMOS_REG, (A)), inb(CMOS_DATA))
157#define CHECK_WRITE(A,D) (outb(CMOS_REG, (A)), outb(CMOS_DATA, (D)))
158
159#define CHECK_INIT(D);				\
160	CHECK_WRITE(0x34, (D));			\
161	CHECK_WRITE(0x35, (D));			\
162	CHECK_WRITE(0x36, (D));			\
163	CHECK_WRITE(0x37, (D));			\
164	CHECK_WRITE(0x38, (D));			\
165	CHECK_WRITE(0x39, (D));
166
167#define CHECK_PRINT(S);				\
168	printf("%s: %d, %d, %d, %d, %d, %d\n",	\
169	   (S),					\
170	   CHECK_READ(0x34),			\
171	   CHECK_READ(0x35),			\
172	   CHECK_READ(0x36),			\
173	   CHECK_READ(0x37),			\
174	   CHECK_READ(0x38),			\
175	   CHECK_READ(0x39));
176
177#else				/* CHECK_POINTS */
178
179#define CHECK_INIT(D)
180#define CHECK_PRINT(S)
181
182#endif				/* CHECK_POINTS */
183
184/*
185 * Values to send to the POST hardware.
186 */
187#define MP_BOOTADDRESS_POST	0x10
188#define MP_PROBE_POST		0x11
189#define MP_START_POST		0x12
190#define MP_ANNOUNCE_POST	0x13
191#define MPTABLE_PASS1_POST	0x14
192#define MPTABLE_PASS2_POST	0x15
193#define MP_ENABLE_POST		0x16
194#define START_ALL_APS_POST	0x17
195#define INSTALL_AP_TRAMP_POST	0x18
196#define START_AP_POST		0x19
197
198/** XXX FIXME: where does this really belong, isa.h/isa.c perhaps? */
199int	current_postcode;
200
201/** XXX FIXME: what system files declare these??? */
202extern struct region_descriptor r_gdt, r_idt;
203
204int	mp_ncpus;		/* # of CPUs, including BSP */
205int	mp_naps;		/* # of Applications processors */
206int	mp_nbusses;		/* # of busses */
207int	mp_napics;		/* # of IO APICs */
208int	boot_cpu_id;		/* designated BSP */
209vm_offset_t cpu_apic_address;
210vm_offset_t io_apic_address[NAPICID];	/* NAPICID is more than enough */
211
212u_int32_t cpu_apic_versions[NCPU];
213u_int32_t io_apic_versions[NAPIC];
214
215/*
216 * APIC ID logical/physical mapping structures.
217 * We oversize these to simplify boot-time config.
218 */
219int     cpu_num_to_apic_id[NAPICID];
220int     io_num_to_apic_id[NAPICID];
221int     apic_id_to_logical[NAPICID];
222
223/* Bitmap of all available CPUs */
224u_int	all_cpus;
225
226/* Boot of AP uses this PTD */
227u_int *bootPTD;
228
229/* Hotwire a 0->4MB V==P mapping */
230extern pt_entry_t KPTphys;
231
232/* Virtual address of per-cpu common_tss */
233extern struct i386tss common_tss;
234
235/*
236 * Local data and functions.
237 */
238
239static int	mp_capable;
240static u_int	boot_address;
241static u_int	base_memory;
242
243static int	picmode;		/* 0: virtual wire mode, 1: PIC mode */
244static mpfps_t	mpfps;
245static int	search_for_sig(u_int32_t target, int count);
246static void	mp_enable(u_int boot_addr);
247
248static int	mptable_pass1(void);
249static int	mptable_pass2(void);
250static void	default_mp_table(int type);
251static int	start_all_aps(u_int boot_addr);
252static void	install_ap_tramp(u_int boot_addr);
253static int	start_ap(int logicalCpu, u_int boot_addr);
254
255
256/*
257 * Calculate usable address in base memory for AP trampoline code.
258 */
259u_int
260mp_bootaddress(u_int basemem)
261{
262	POSTCODE(MP_BOOTADDRESS_POST);
263
264	base_memory = basemem * 1024;	/* convert to bytes */
265
266	boot_address = base_memory & ~0xfff;	/* round down to 4k boundary */
267	if ((base_memory - boot_address) < bootMP_size)
268		boot_address -= 4096;	/* not enough, lower by 4k */
269
270	return boot_address;
271}
272
273
274/*
275 * Look for an Intel MP spec table (ie, SMP capable hardware).
276 */
277int
278mp_probe(void)
279{
280	int     x;
281	u_long  segment;
282	u_int32_t target;
283
284	POSTCODE(MP_PROBE_POST);
285
286	/* see if EBDA exists */
287	if (segment = (u_long) * (u_short *) (KERNBASE + 0x40e)) {
288		/* search first 1K of EBDA */
289		target = (u_int32_t) (segment << 4);
290		if ((x = search_for_sig(target, 1024 / 4)) >= 0)
291			goto found;
292	} else {
293		/* last 1K of base memory, effective 'top of base' passed in */
294		target = (u_int32_t) (base_memory - 0x400);
295		if ((x = search_for_sig(target, 1024 / 4)) >= 0)
296			goto found;
297	}
298
299	/* search the BIOS */
300	target = (u_int32_t) BIOS_BASE;
301	if ((x = search_for_sig(target, BIOS_COUNT)) >= 0)
302		goto found;
303
304	/* nothing found */
305	mpfps = (mpfps_t)0;
306	mp_capable = 0;
307	return 0;
308
309found:
310	/* calculate needed resources */
311	mpfps = (mpfps_t)x;
312	if (mptable_pass1())
313		panic("you must reconfigure your kernel");
314
315	/* flag fact that we are running multiple processors */
316	mp_capable = 1;
317	return 1;
318}
319
320
321/*
322 * Startup the SMP processors.
323 */
324void
325mp_start(void)
326{
327	POSTCODE(MP_START_POST);
328
329	/* look for MP capable motherboard */
330	if (mp_capable)
331		mp_enable(boot_address);
332	else
333		panic("MP hardware not found!");
334}
335
336
337/*
338 * Print various information about the SMP system hardware and setup.
339 */
340void
341mp_announce(void)
342{
343	int     x;
344
345	POSTCODE(MP_ANNOUNCE_POST);
346
347	printf("FreeBSD/SMP: Multiprocessor motherboard\n");
348	printf(" cpu0 (BSP): apic id: %2d", CPU_TO_ID(0));
349	printf(", version: 0x%08x", cpu_apic_versions[0]);
350	printf(", at 0x%08x\n", cpu_apic_address);
351	for (x = 1; x <= mp_naps; ++x) {
352		printf(" cpu%d (AP):  apic id: %2d", x, CPU_TO_ID(x));
353		printf(", version: 0x%08x", cpu_apic_versions[x]);
354		printf(", at 0x%08x\n", cpu_apic_address);
355	}
356
357#if defined(APIC_IO)
358	for (x = 0; x < mp_napics; ++x) {
359		printf(" io%d (APIC): apic id: %2d", x, IO_TO_ID(x));
360		printf(", version: 0x%08x", io_apic_versions[x]);
361		printf(", at 0x%08x\n", io_apic_address[x]);
362	}
363#else
364	printf(" Warning: APIC I/O disabled\n");
365#endif	/* APIC_IO */
366}
367
368/*
369 * AP cpu's call this to sync up protected mode.
370 */
371void
372init_secondary(void)
373{
374	int     gsel_tss, slot;
375
376	r_gdt.rd_limit = sizeof(gdt[0]) * (NGDT + NCPU) - 1;
377	r_gdt.rd_base = (int) gdt;
378	lgdt(&r_gdt);			/* does magic intra-segment return */
379	lidt(&r_idt);
380	lldt(_default_ldt);
381
382	slot = NGDT + cpuid;
383	gsel_tss = GSEL(slot, SEL_KPL);
384	gdt[slot].sd.sd_type = SDT_SYS386TSS;
385	common_tss.tss_esp0 = 0;	/* not used until after switch */
386	common_tss.tss_ss0 = GSEL(GDATA_SEL, SEL_KPL);
387	common_tss.tss_ioopt = (sizeof common_tss) << 16;
388	ltr(gsel_tss);
389
390	load_cr0(0x8005003b);		/* XXX! */
391
392	PTD[0] = 0;
393	pmap_set_opt((unsigned *)PTD);
394
395	invltlb();
396}
397
398
399#if defined(APIC_IO)
400/*
401 * Final configuration of the BSP's local APIC:
402 *  - disable 'pic mode'.
403 *  - disable 'virtual wire mode'.
404 *  - enable NMI.
405 */
406void
407bsp_apic_configure(void)
408{
409	u_char		byte;
410	u_int32_t	temp;
411
412	/* leave 'pic mode' if necessary */
413	if (picmode) {
414		outb(0x22, 0x70);	/* select IMCR */
415		byte = inb(0x23);	/* current contents */
416		byte |= 0x01;		/* mask external INTR */
417		outb(0x23, byte);	/* disconnect 8259s/NMI */
418	}
419
420	/* mask lint0 (the 8259 'virtual wire' connection) */
421	temp = lapic.lvt_lint0;
422	temp |= APIC_LVT_M;		/* set the mask */
423	lapic.lvt_lint0 = temp;
424
425        /* setup lint1 to handle NMI */
426        temp = lapic.lvt_lint1;
427        temp &= ~APIC_LVT_M;		/* clear the mask */
428        lapic.lvt_lint1 = temp;
429
430	if (bootverbose)
431		apic_dump("bsp_apic_configure()");
432}
433#endif  /* APIC_IO */
434
435
436/*******************************************************************
437 * local functions and data
438 */
439
440/*
441 * start the SMP system
442 */
443static void
444mp_enable(u_int boot_addr)
445{
446	int     x;
447#if defined(APIC_IO)
448	int     apic;
449	u_int   ux;
450#endif	/* APIC_IO */
451
452	POSTCODE(MP_ENABLE_POST);
453
454	/* turn on 4MB of V == P addressing so we can get to MP table */
455	*(int *)PTD = PG_V | PG_RW | ((u_long)KPTphys & PG_FRAME);
456	invltlb();
457
458	/* examine the MP table for needed info, uses physical addresses */
459	x = mptable_pass2();
460
461	*(int *)PTD = 0;
462	invltlb();
463
464	/* can't process default configs till the CPU APIC is pmapped */
465	if (x)
466		default_mp_table(x);
467
468#if defined(APIC_IO)
469
470	/* fill the LOGICAL io_apic_versions table */
471	for (apic = 0; apic < mp_napics; ++apic) {
472		ux = io_apic_read(apic, IOAPIC_VER);
473		io_apic_versions[apic] = ux;
474	}
475
476	/* program each IO APIC in the system */
477	for (apic = 0; apic < mp_napics; ++apic)
478		if (io_apic_setup(apic) < 0)
479			panic("IO APIC setup failure");
480
481	/* install a 'Spurious INTerrupt' vector */
482	setidt(XSPURIOUSINT_OFFSET, Xspuriousint,
483	       SDT_SYS386IGT, SEL_KPL, GSEL(GCODE_SEL, SEL_KPL));
484
485	/* install an inter-CPU IPI for TLB invalidation */
486	setidt(XINVLTLB_OFFSET, Xinvltlb,
487	       SDT_SYS386IGT, SEL_KPL, GSEL(GCODE_SEL, SEL_KPL));
488
489	/* install an inter-CPU IPI for CPU stop/restart */
490	setidt(XCPUSTOP_OFFSET, Xcpustop,
491	       SDT_SYS386IGT, SEL_KPL, GSEL(GCODE_SEL, SEL_KPL));
492
493#if defined(TEST_TEST1)
494	/* install a "fake hardware INTerrupt" vector */
495	setidt(XTEST1_OFFSET, Xtest1,
496	       SDT_SYS386IGT, SEL_KPL, GSEL(GCODE_SEL, SEL_KPL));
497#endif  /** TEST_TEST1 */
498
499#endif	/* APIC_IO */
500
501	/* start each Application Processor */
502	start_all_aps(boot_addr);
503
504	/*
505	 * The init process might be started on a different CPU now,
506	 * and the boot CPU might not call prepare_usermode to get
507	 * cr0 correctly configured. Thus we initialize cr0 here.
508	 */
509	load_cr0(rcr0() | CR0_WP | CR0_AM);
510}
511
512
513/*
514 * look for the MP spec signature
515 */
516
517/* string defined by the Intel MP Spec as identifying the MP table */
518#define MP_SIG		0x5f504d5f	/* _MP_ */
519#define NEXT(X)		((X) += 4)
520static int
521search_for_sig(u_int32_t target, int count)
522{
523	int     x;
524	u_int32_t *addr = (u_int32_t *) (KERNBASE + target);
525
526	for (x = 0; x < count; NEXT(x))
527		if (addr[x] == MP_SIG)
528			/* make array index a byte index */
529			return (target + (x * sizeof(u_int32_t)));
530
531	return -1;
532}
533
534
535static basetable_entry basetable_entry_types[] =
536{
537	{0, 20, "Processor"},
538	{1, 8, "Bus"},
539	{2, 8, "I/O APIC"},
540	{3, 8, "I/O INT"},
541	{4, 8, "Local INT"}
542};
543
544typedef struct BUSDATA {
545	u_char  bus_id;
546	enum busTypes bus_type;
547}       bus_datum;
548
549typedef struct INTDATA {
550	u_char  int_type;
551	u_short int_flags;
552	u_char  src_bus_id;
553	u_char  src_bus_irq;
554	u_char  dst_apic_id;
555	u_char  dst_apic_int;
556}       io_int, local_int;
557
558typedef struct BUSTYPENAME {
559	u_char  type;
560	char    name[7];
561}       bus_type_name;
562
563static bus_type_name bus_type_table[] =
564{
565	{CBUS, "CBUS"},
566	{CBUSII, "CBUSII"},
567	{EISA, "EISA"},
568	{UNKNOWN_BUSTYPE, "---"},
569	{UNKNOWN_BUSTYPE, "---"},
570	{ISA, "ISA"},
571	{UNKNOWN_BUSTYPE, "---"},
572	{UNKNOWN_BUSTYPE, "---"},
573	{UNKNOWN_BUSTYPE, "---"},
574	{UNKNOWN_BUSTYPE, "---"},
575	{UNKNOWN_BUSTYPE, "---"},
576	{UNKNOWN_BUSTYPE, "---"},
577	{PCI, "PCI"},
578	{UNKNOWN_BUSTYPE, "---"},
579	{UNKNOWN_BUSTYPE, "---"},
580	{UNKNOWN_BUSTYPE, "---"},
581	{UNKNOWN_BUSTYPE, "---"},
582	{XPRESS, "XPRESS"},
583	{UNKNOWN_BUSTYPE, "---"}
584};
585/* from MP spec v1.4, table 5-1 */
586static int default_data[7][5] =
587{
588/*   nbus, id0, type0, id1, type1 */
589	{1, 0, ISA, 255, 255},
590	{1, 0, EISA, 255, 255},
591	{1, 0, EISA, 255, 255},
592	{0, 255, 255, 255, 255},/* MCA not supported */
593	{2, 0, ISA, 1, PCI},
594	{2, 0, EISA, 1, PCI},
595	{0, 255, 255, 255, 255}	/* MCA not supported */
596};
597
598
599/* the bus data */
600bus_datum bus_data[NBUS];
601
602/* the IO INT data, one entry per possible APIC INTerrupt */
603io_int  io_apic_ints[NINTR];
604
605static int nintrs;
606
607static void fix_mp_table	__P((void));
608static int processor_entry	__P((proc_entry_ptr entry, int cpu));
609static int bus_entry		__P((bus_entry_ptr entry, int bus));
610static int io_apic_entry	__P((io_apic_entry_ptr entry, int apic));
611static int int_entry		__P((int_entry_ptr entry, int intr));
612static int lookup_bus_type	__P((char *name));
613
614
615/*
616 * 1st pass on motherboard's Intel MP specification table.
617 *
618 * initializes:
619 *	mp_ncpus = 1
620 *
621 * determines:
622 *	cpu_apic_address (common to all CPUs)
623 *	io_apic_address[N]
624 *	mp_naps
625 *	mp_nbusses
626 *	mp_napics
627 *	nintrs
628 */
629static int
630mptable_pass1(void)
631{
632	int	x;
633	mpcth_t	cth;
634	int	totalSize;
635	void*	position;
636	int	count;
637	int	type;
638	int	mustpanic;
639
640	POSTCODE(MPTABLE_PASS1_POST);
641
642	mustpanic = 0;
643
644	/* clear various tables */
645	for (x = 0; x < NAPICID; ++x) {
646		io_apic_address[x] = ~0;	/* IO APIC address table */
647	}
648
649	/* init everything to empty */
650	mp_naps = 0;
651	mp_nbusses = 0;
652	mp_napics = 0;
653	nintrs = 0;
654
655	/* check for use of 'default' configuration */
656	if (mpfps->mpfb1 != 0) {
657		/* use default addresses */
658		cpu_apic_address = DEFAULT_APIC_BASE;
659		io_apic_address[0] = DEFAULT_IO_APIC_BASE;
660
661		/* fill in with defaults */
662		mp_naps = 2;		/* includes BSP */
663		mp_nbusses = default_data[mpfps->mpfb1 - 1][0];
664#if defined(APIC_IO)
665		mp_napics = 1;
666		nintrs = 16;
667#endif	/* APIC_IO */
668	}
669	else {
670		if ((cth = mpfps->pap) == 0)
671			panic("MP Configuration Table Header MISSING!");
672
673		cpu_apic_address = (vm_offset_t) cth->apic_address;
674
675		/* walk the table, recording info of interest */
676		totalSize = cth->base_table_length - sizeof(struct MPCTH);
677		position = (u_char *) cth + sizeof(struct MPCTH);
678		count = cth->entry_count;
679
680		while (count--) {
681			switch (type = *(u_char *) position) {
682			case 0: /* processor_entry */
683				if (((proc_entry_ptr)position)->cpu_flags
684					& PROCENTRY_FLAG_EN)
685					++mp_naps;
686				break;
687			case 1: /* bus_entry */
688				++mp_nbusses;
689				break;
690			case 2: /* io_apic_entry */
691				if (((io_apic_entry_ptr)position)->apic_flags
692					& IOAPICENTRY_FLAG_EN)
693					io_apic_address[mp_napics++] =
694					    (vm_offset_t)((io_apic_entry_ptr)
695						position)->apic_address;
696				break;
697			case 3: /* int_entry */
698				++nintrs;
699				break;
700			case 4:	/* int_entry */
701				break;
702			default:
703				panic("mpfps Base Table HOSED!");
704				/* NOTREACHED */
705			}
706
707			totalSize -= basetable_entry_types[type].length;
708			(u_char*)position += basetable_entry_types[type].length;
709		}
710	}
711
712	/* qualify the numbers */
713	if (mp_naps > NCPU)
714		printf("Warning: only using %d of %d available CPUs!\n",
715			NCPU, mp_naps);
716#if 0
717		/** XXX we consider this legal now (but should we?) */
718		mustpanic = 1;
719#endif
720	if (mp_nbusses > NBUS) {
721		printf("found %d busses, increase NBUS\n", mp_nbusses);
722		mustpanic = 1;
723	}
724	if (mp_napics > NAPIC) {
725		printf("found %d apics, increase NAPIC\n", mp_napics);
726		mustpanic = 1;
727	}
728	if (nintrs > NINTR) {
729		printf("found %d intrs, increase NINTR\n", nintrs);
730		mustpanic = 1;
731	}
732
733	/*
734	 * Count the BSP.
735	 * This is also used as a counter while starting the APs.
736	 */
737	mp_ncpus = 1;
738
739	--mp_naps;	/* subtract the BSP */
740
741	return mustpanic;
742}
743
744
745/*
746 * 2nd pass on motherboard's Intel MP specification table.
747 *
748 * sets:
749 *	boot_cpu_id
750 *	ID_TO_IO(N), phy APIC ID to log CPU/IO table
751 *	CPU_TO_ID(N), logical CPU to APIC ID table
752 *	IO_TO_ID(N), logical IO to APIC ID table
753 *	bus_data[N]
754 *	io_apic_ints[N]
755 */
756static int
757mptable_pass2(void)
758{
759	int     x;
760	mpcth_t cth;
761	int     totalSize;
762	void*   position;
763	int     count;
764	int     type;
765	int     apic, bus, cpu, intr;
766
767	POSTCODE(MPTABLE_PASS2_POST);
768
769	/* clear various tables */
770	for (x = 0; x < NAPICID; ++x) {
771		ID_TO_IO(x) = -1;	/* phy APIC ID to log CPU/IO table */
772		CPU_TO_ID(x) = -1;	/* logical CPU to APIC ID table */
773		IO_TO_ID(x) = -1;	/* logical IO to APIC ID table */
774	}
775
776	/* clear bus data table */
777	for (x = 0; x < NBUS; ++x)
778		bus_data[x].bus_id = 0xff;
779
780	/* clear IO APIC INT table */
781	for (x = 0; x < NINTR; ++x)
782		io_apic_ints[x].int_type = 0xff;
783
784	/* setup the cpu/apic mapping arrays */
785	boot_cpu_id = -1;
786
787	/* record whether PIC or virtual-wire mode */
788	picmode = (mpfps->mpfb2 & 0x80) ? 1 : 0;
789
790	/* check for use of 'default' configuration */
791#if defined(TEST_DEFAULT_CONFIG)
792	return TEST_DEFAULT_CONFIG;
793#else
794	if (mpfps->mpfb1 != 0)
795		return mpfps->mpfb1;	/* return default configuration type */
796#endif	/* TEST_DEFAULT_CONFIG */
797
798	if ((cth = mpfps->pap) == 0)
799		panic("MP Configuration Table Header MISSING!");
800
801	/* walk the table, recording info of interest */
802	totalSize = cth->base_table_length - sizeof(struct MPCTH);
803	position = (u_char *) cth + sizeof(struct MPCTH);
804	count = cth->entry_count;
805	apic = bus = intr = 0;
806	cpu = 1;				/* pre-count the BSP */
807
808	while (count--) {
809		switch (type = *(u_char *) position) {
810		case 0:
811			if (processor_entry(position, cpu))
812				++cpu;
813			break;
814		case 1:
815			if (bus_entry(position, bus))
816				++bus;
817			break;
818		case 2:
819			if (io_apic_entry(position, apic))
820				++apic;
821			break;
822		case 3:
823			if (int_entry(position, intr))
824				++intr;
825			break;
826		case 4:
827			/* int_entry(position); */
828			break;
829		default:
830			panic("mpfps Base Table HOSED!");
831			/* NOTREACHED */
832		}
833
834		totalSize -= basetable_entry_types[type].length;
835		(u_char *) position += basetable_entry_types[type].length;
836	}
837
838	if (boot_cpu_id == -1)
839		panic("NO BSP found!");
840
841	/* post scan cleanup */
842	fix_mp_table();
843
844	/* report fact that its NOT a default configuration */
845	return 0;
846}
847
848
849/*
850 * parse an Intel MP specification table
851 */
852static void
853fix_mp_table(void)
854{
855	int	x;
856	int	id;
857	int	bus_0;
858	int	bus_pci;
859	int	num_pci_bus;
860
861	/*
862	 * Fix mis-numbering of the PCI bus and its INT entries if the BIOS
863	 * did it wrong.  The MP spec says that when more than 1 PCI bus
864	 * exists the BIOS must begin with bus entries for the PCI bus and use
865	 * actual PCI bus numbering.  This implies that when only 1 PCI bus
866	 * exists the BIOS can choose to ignore this ordering, and indeed many
867	 * MP motherboards do ignore it.  This causes a problem when the PCI
868	 * sub-system makes requests of the MP sub-system based on PCI bus
869	 * numbers.	So here we look for the situation and renumber the
870	 * busses and associated INTs in an effort to "make it right".
871	 */
872
873	/* find bus 0, PCI bus, count the number of PCI busses */
874	for (num_pci_bus = 0, x = 0; x < mp_nbusses; ++x) {
875		if (bus_data[x].bus_id == 0) {
876			bus_0 = x;
877		}
878		if (bus_data[x].bus_type == PCI) {
879			++num_pci_bus;
880			bus_pci = x;
881		}
882	}
883	/*
884	 * bus_0 == slot of bus with ID of 0
885	 * bus_pci == slot of last PCI bus encountered
886	 */
887
888	/* check the 1 PCI bus case for sanity */
889	if (num_pci_bus == 1) {
890
891		/* if it is number 0 all is well */
892		if (bus_data[bus_pci].bus_id == 0)
893			return;
894
895		/* mis-numbered, swap with whichever bus uses slot 0 */
896
897		/* swap the bus entry types */
898		bus_data[bus_pci].bus_type = bus_data[bus_0].bus_type;
899		bus_data[bus_0].bus_type = PCI;
900
901		/* swap each relavant INTerrupt entry */
902		id = bus_data[bus_pci].bus_id;
903		for (x = 0; x < nintrs; ++x) {
904			if (io_apic_ints[x].src_bus_id == id) {
905				io_apic_ints[x].src_bus_id = 0;
906			}
907			else if (io_apic_ints[x].src_bus_id == 0) {
908				io_apic_ints[x].src_bus_id = id;
909			}
910		}
911	}
912	/* sanity check if more than 1 PCI bus */
913	else if (num_pci_bus > 1) {
914		for (x = 0; x < mp_nbusses; ++x) {
915			if (bus_data[x].bus_type != PCI)
916				continue;
917			if (bus_data[x].bus_id >= num_pci_bus)
918				panic("bad PCI bus numbering");
919		}
920	}
921}
922
923
924static int
925processor_entry(proc_entry_ptr entry, int cpu)
926{
927	/* check for usability */
928	if ((cpu >= NCPU) || !(entry->cpu_flags & PROCENTRY_FLAG_EN))
929		return 0;
930
931	/* check for BSP flag */
932	if (entry->cpu_flags & PROCENTRY_FLAG_BP) {
933		boot_cpu_id = entry->apic_id;
934		CPU_TO_ID(0) = entry->apic_id;
935		ID_TO_CPU(entry->apic_id) = 0;
936		return 0;	/* its already been counted */
937	}
938
939	/* add another AP to list, if less than max number of CPUs */
940	else {
941		CPU_TO_ID(cpu) = entry->apic_id;
942		ID_TO_CPU(entry->apic_id) = cpu;
943		return 1;
944	}
945}
946
947
948static int
949bus_entry(bus_entry_ptr entry, int bus)
950{
951	int     x;
952	char    c, name[8];
953
954	/* encode the name into an index */
955	for (x = 0; x < 6; ++x) {
956		if ((c = entry->bus_type[x]) == ' ')
957			break;
958		name[x] = c;
959	}
960	name[x] = '\0';
961
962	if ((x = lookup_bus_type(name)) == UNKNOWN_BUSTYPE)
963		panic("unknown bus type: '%s'", name);
964
965	bus_data[bus].bus_id = entry->bus_id;
966	bus_data[bus].bus_type = x;
967
968	return 1;
969}
970
971
972static int
973io_apic_entry(io_apic_entry_ptr entry, int apic)
974{
975	if (!(entry->apic_flags & IOAPICENTRY_FLAG_EN))
976		return 0;
977
978	IO_TO_ID(apic) = entry->apic_id;
979	ID_TO_IO(entry->apic_id) = apic;
980
981	return 1;
982}
983
984
985static int
986lookup_bus_type(char *name)
987{
988	int     x;
989
990	for (x = 0; x < MAX_BUSTYPE; ++x)
991		if (strcmp(bus_type_table[x].name, name) == 0)
992			return bus_type_table[x].type;
993
994	return UNKNOWN_BUSTYPE;
995}
996
997
998static int
999int_entry(int_entry_ptr entry, int intr)
1000{
1001	io_apic_ints[intr].int_type = entry->int_type;
1002	io_apic_ints[intr].int_flags = entry->int_flags;
1003	io_apic_ints[intr].src_bus_id = entry->src_bus_id;
1004	io_apic_ints[intr].src_bus_irq = entry->src_bus_irq;
1005	io_apic_ints[intr].dst_apic_id = entry->dst_apic_id;
1006	io_apic_ints[intr].dst_apic_int = entry->dst_apic_int;
1007
1008	return 1;
1009}
1010
1011
1012static int
1013apic_int_is_bus_type(int intr, int bus_type)
1014{
1015	int     bus;
1016
1017	for (bus = 0; bus < mp_nbusses; ++bus)
1018		if ((bus_data[bus].bus_id == io_apic_ints[intr].src_bus_id)
1019		    && ((int) bus_data[bus].bus_type == bus_type))
1020			return 1;
1021
1022	return 0;
1023}
1024
1025
1026/*
1027 * Given a traditional ISA INT mask, return an APIC mask.
1028 */
1029u_int
1030isa_apic_mask(u_int isa_mask)
1031{
1032	int isa_irq;
1033	int apic_pin;
1034
1035#if defined(SKIP_IRQ15_REDIRECT)
1036	if (isa_mask == (1 << 15)) {
1037		printf("skipping ISA IRQ15 redirect\n");
1038		return isa_mask;
1039	}
1040#endif  /* SKIP_IRQ15_REDIRECT */
1041
1042	isa_irq = ffs(isa_mask);		/* find its bit position */
1043	if (isa_irq == 0)			/* doesn't exist */
1044		return 0;
1045	--isa_irq;				/* make it zero based */
1046
1047	apic_pin = isa_apic_pin(isa_irq);	/* look for APIC connection */
1048	if (apic_pin == -1)
1049		return 0;
1050
1051	return (1 << apic_pin);			/* convert pin# to a mask */
1052}
1053
1054
1055/*
1056 * Determine which APIC pin an ISA/EISA INT is attached to.
1057 */
1058#define INTTYPE(I)	(io_apic_ints[(I)].int_type)
1059#define INTPIN(I)	(io_apic_ints[(I)].dst_apic_int)
1060
1061#define SRCBUSIRQ(I)	(io_apic_ints[(I)].src_bus_irq)
1062int
1063isa_apic_pin(int isa_irq)
1064{
1065	int     intr;
1066
1067#if defined(SMP_TIMER_NC)
1068#if defined(NEW_STRATEGY) || defined(APIC_PIN0_TIMER)
1069#error
1070#error 'options SMP_TIMER_NC' no longer used, remove & reconfig.
1071#error
1072#else
1073	if (isa_irq == 0)
1074		return -1;
1075#endif	/* NEW_STRATEGY || APIC_PIN0_TIMER */
1076#endif	/* SMP_TIMER_NC */
1077
1078	for (intr = 0; intr < nintrs; ++intr) {		/* check each record */
1079		if (INTTYPE(intr) == 0) {		/* standard INT */
1080			if (SRCBUSIRQ(intr) == isa_irq) {
1081				if (apic_int_is_bus_type(intr, ISA) ||
1082			            apic_int_is_bus_type(intr, EISA))
1083					return INTPIN(intr);	/* found */
1084			}
1085		}
1086	}
1087	return -1;					/* NOT found */
1088}
1089#undef SRCBUSIRQ
1090
1091
1092/*
1093 * Determine which APIC pin a PCI INT is attached to.
1094 */
1095#define SRCBUSID(I)	(io_apic_ints[(I)].src_bus_id)
1096#define SRCBUSDEVICE(I)	((io_apic_ints[(I)].src_bus_irq >> 2) & 0x1f)
1097#define SRCBUSLINE(I)	(io_apic_ints[(I)].src_bus_irq & 0x03)
1098int
1099pci_apic_pin(int pciBus, int pciDevice, int pciInt)
1100{
1101	int     intr;
1102
1103	--pciInt;					/* zero based */
1104
1105	for (intr = 0; intr < nintrs; ++intr)		/* check each record */
1106		if ((INTTYPE(intr) == 0)		/* standard INT */
1107		    && (SRCBUSID(intr) == pciBus)
1108		    && (SRCBUSDEVICE(intr) == pciDevice)
1109		    && (SRCBUSLINE(intr) == pciInt))	/* a candidate IRQ */
1110			if (apic_int_is_bus_type(intr, PCI))
1111				return INTPIN(intr);	/* exact match */
1112
1113	return -1;					/* NOT found */
1114}
1115#undef SRCBUSLINE
1116#undef SRCBUSDEVICE
1117#undef SRCBUSID
1118
1119#undef INTPIN
1120#undef INTTYPE
1121
1122
1123/*
1124 * Reprogram the MB chipset to NOT redirect an ISA INTerrupt.
1125 *
1126 * XXX FIXME:
1127 *  Exactly what this means is unclear at this point.  It is a solution
1128 *  for motherboards that redirect the MBIRQ0 pin.  Generically a motherboard
1129 *  could route any of the ISA INTs to upper (>15) IRQ values.  But most would
1130 *  NOT be redirected via MBIRQ0, thus "undirect()ing" them would NOT be an
1131 *  option.
1132 */
1133int
1134undirect_isa_irq(int rirq)
1135{
1136#if defined(READY)
1137	printf("Freeing redirected ISA irq %d.\n", rirq);
1138	/** FIXME: tickle the MB redirector chip */
1139	return ???;
1140#else
1141	printf("Freeing (NOT implemented) redirected ISA irq %d.\n", rirq);
1142	return 0;
1143#endif  /* READY */
1144}
1145
1146
1147/*
1148 * Reprogram the MB chipset to NOT redirect a PCI INTerrupt
1149 */
1150int
1151undirect_pci_irq(int rirq)
1152{
1153#if defined(READY)
1154	if (bootverbose)
1155		printf("Freeing redirected PCI irq %d.\n", rirq);
1156
1157	/** FIXME: tickle the MB redirector chip */
1158	return ???;
1159#else
1160	if (bootverbose)
1161		printf("Freeing (NOT implemented) redirected PCI irq %d.\n",
1162		       rirq);
1163	return 0;
1164#endif  /* READY */
1165}
1166
1167
1168/*
1169 * given a bus ID, return:
1170 *  the bus type if found
1171 *  -1 if NOT found
1172 */
1173int
1174apic_bus_type(int id)
1175{
1176	int     x;
1177
1178	for (x = 0; x < mp_nbusses; ++x)
1179		if (bus_data[x].bus_id == id)
1180			return bus_data[x].bus_type;
1181
1182	return -1;
1183}
1184
1185
1186/*
1187 * given a LOGICAL APIC# and pin#, return:
1188 *  the associated src bus ID if found
1189 *  -1 if NOT found
1190 */
1191int
1192apic_src_bus_id(int apic, int pin)
1193{
1194	int     x;
1195
1196	/* search each of the possible INTerrupt sources */
1197	for (x = 0; x < nintrs; ++x)
1198		if ((apic == ID_TO_IO(io_apic_ints[x].dst_apic_id)) &&
1199		    (pin == io_apic_ints[x].dst_apic_int))
1200			return (io_apic_ints[x].src_bus_id);
1201
1202	return -1;		/* NOT found */
1203}
1204
1205
1206/*
1207 * given a LOGICAL APIC# and pin#, return:
1208 *  the associated src bus IRQ if found
1209 *  -1 if NOT found
1210 */
1211int
1212apic_src_bus_irq(int apic, int pin)
1213{
1214	int     x;
1215
1216	for (x = 0; x < nintrs; x++)
1217		if ((apic == ID_TO_IO(io_apic_ints[x].dst_apic_id)) &&
1218		    (pin == io_apic_ints[x].dst_apic_int))
1219			return (io_apic_ints[x].src_bus_irq);
1220
1221	return -1;		/* NOT found */
1222}
1223
1224
1225/*
1226 * given a LOGICAL APIC# and pin#, return:
1227 *  the associated INTerrupt type if found
1228 *  -1 if NOT found
1229 */
1230int
1231apic_int_type(int apic, int pin)
1232{
1233	int     x;
1234
1235	/* search each of the possible INTerrupt sources */
1236	for (x = 0; x < nintrs; ++x)
1237		if ((apic == ID_TO_IO(io_apic_ints[x].dst_apic_id)) &&
1238		    (pin == io_apic_ints[x].dst_apic_int))
1239			return (io_apic_ints[x].int_type);
1240
1241	return -1;		/* NOT found */
1242}
1243
1244
1245/*
1246 * given a LOGICAL APIC# and pin#, return:
1247 *  the associated trigger mode if found
1248 *  -1 if NOT found
1249 */
1250int
1251apic_trigger(int apic, int pin)
1252{
1253	int     x;
1254
1255	/* search each of the possible INTerrupt sources */
1256	for (x = 0; x < nintrs; ++x)
1257		if ((apic == ID_TO_IO(io_apic_ints[x].dst_apic_id)) &&
1258		    (pin == io_apic_ints[x].dst_apic_int))
1259			return ((io_apic_ints[x].int_flags >> 2) & 0x03);
1260
1261	return -1;		/* NOT found */
1262}
1263
1264
1265/*
1266 * given a LOGICAL APIC# and pin#, return:
1267 *  the associated 'active' level if found
1268 *  -1 if NOT found
1269 */
1270int
1271apic_polarity(int apic, int pin)
1272{
1273	int     x;
1274
1275	/* search each of the possible INTerrupt sources */
1276	for (x = 0; x < nintrs; ++x)
1277		if ((apic == ID_TO_IO(io_apic_ints[x].dst_apic_id)) &&
1278		    (pin == io_apic_ints[x].dst_apic_int))
1279			return (io_apic_ints[x].int_flags & 0x03);
1280
1281	return -1;		/* NOT found */
1282}
1283
1284
1285/*
1286 * set data according to MP defaults
1287 * FIXME: probably not complete yet...
1288 */
1289static void
1290default_mp_table(int type)
1291{
1292	int     ap_cpu_id;
1293#if defined(APIC_IO)
1294	u_int32_t ux;
1295	int     io_apic_id;
1296	int     pin;
1297#endif	/* APIC_IO */
1298
1299#if 0
1300	printf("  MP default config type: %d\n", type);
1301	switch (type) {
1302	case 1:
1303		printf("   bus: ISA, APIC: 82489DX\n");
1304		break;
1305	case 2:
1306		printf("   bus: EISA, APIC: 82489DX\n");
1307		break;
1308	case 3:
1309		printf("   bus: EISA, APIC: 82489DX\n");
1310		break;
1311	case 4:
1312		printf("   bus: MCA, APIC: 82489DX\n");
1313		break;
1314	case 5:
1315		printf("   bus: ISA+PCI, APIC: Integrated\n");
1316		break;
1317	case 6:
1318		printf("   bus: EISA+PCI, APIC: Integrated\n");
1319		break;
1320	case 7:
1321		printf("   bus: MCA+PCI, APIC: Integrated\n");
1322		break;
1323	default:
1324		printf("   future type\n");
1325		break;
1326		/* NOTREACHED */
1327	}
1328#endif	/* 0 */
1329
1330	boot_cpu_id = (lapic.id & APIC_ID_MASK) >> 24;
1331	ap_cpu_id = (boot_cpu_id == 0) ? 1 : 0;
1332
1333	/* BSP */
1334	CPU_TO_ID(0) = boot_cpu_id;
1335	ID_TO_CPU(boot_cpu_id) = 0;
1336
1337	/* one and only AP */
1338	CPU_TO_ID(1) = ap_cpu_id;
1339	ID_TO_CPU(ap_cpu_id) = 1;
1340
1341#if defined(APIC_IO)
1342	/* one and only IO APIC */
1343	io_apic_id = (io_apic_read(0, IOAPIC_ID) & APIC_ID_MASK) >> 24;
1344
1345	/*
1346	 * sanity check, refer to MP spec section 3.6.6, last paragraph
1347	 * necessary as some hardware isn't properly setting up the IO APIC
1348	 */
1349#if defined(REALLY_ANAL_IOAPICID_VALUE)
1350	if (io_apic_id != 2) {
1351#else
1352	if ((io_apic_id == 0) || (io_apic_id == 1) || (io_apic_id == 15)) {
1353#endif	/* REALLY_ANAL_IOAPICID_VALUE */
1354		ux = io_apic_read(0, IOAPIC_ID);	/* get current contents */
1355		ux &= ~APIC_ID_MASK;	/* clear the ID field */
1356		ux |= 0x02000000;	/* set it to '2' */
1357		io_apic_write(0, IOAPIC_ID, ux);	/* write new value */
1358		ux = io_apic_read(0, IOAPIC_ID);	/* re-read && test */
1359		if ((ux & APIC_ID_MASK) != 0x02000000)
1360			panic("can't control IO APIC ID, reg: 0x%08x", ux);
1361		io_apic_id = 2;
1362	}
1363	IO_TO_ID(0) = io_apic_id;
1364	ID_TO_IO(io_apic_id) = 0;
1365#endif	/* APIC_IO */
1366
1367	/* fill out bus entries */
1368	switch (type) {
1369	case 1:
1370	case 2:
1371	case 3:
1372	case 5:
1373	case 6:
1374		bus_data[0].bus_id = default_data[type - 1][1];
1375		bus_data[0].bus_type = default_data[type - 1][2];
1376		bus_data[1].bus_id = default_data[type - 1][3];
1377		bus_data[1].bus_type = default_data[type - 1][4];
1378		break;
1379
1380	/* case 4: case 7:		   MCA NOT supported */
1381	default:		/* illegal/reserved */
1382		panic("BAD default MP config: %d", type);
1383		/* NOTREACHED */
1384	}
1385
1386#if defined(APIC_IO)
1387	/* general cases from MP v1.4, table 5-2 */
1388	for (pin = 0; pin < 16; ++pin) {
1389		io_apic_ints[pin].int_type = 0;
1390		io_apic_ints[pin].int_flags = 0x05;	/* edge-triggered/active-hi */
1391		io_apic_ints[pin].src_bus_id = 0;
1392		io_apic_ints[pin].src_bus_irq = pin;	/* IRQ2 is caught below */
1393		io_apic_ints[pin].dst_apic_id = io_apic_id;
1394		io_apic_ints[pin].dst_apic_int = pin;	/* 1-to-1 correspondence */
1395	}
1396
1397	/* special cases from MP v1.4, table 5-2 */
1398	if (type == 2) {
1399		io_apic_ints[2].int_type = 0xff;	/* N/C */
1400		io_apic_ints[13].int_type = 0xff;	/* N/C */
1401#if !defined(APIC_MIXED_MODE)
1402		/** FIXME: ??? */
1403		panic("sorry, can't support type 2 default yet");
1404#endif	/* APIC_MIXED_MODE */
1405	}
1406	else
1407		io_apic_ints[2].src_bus_irq = 0;	/* ISA IRQ0 is on APIC INT 2 */
1408
1409	if (type == 7)
1410		io_apic_ints[0].int_type = 0xff;	/* N/C */
1411	else
1412		io_apic_ints[0].int_type = 3;	/* vectored 8259 */
1413#endif	/* APIC_IO */
1414}
1415
1416
1417/*
1418 * start each AP in our list
1419 */
1420static int
1421start_all_aps(u_int boot_addr)
1422{
1423	int     x, i;
1424	u_char  mpbiosreason;
1425	u_long  mpbioswarmvec;
1426	pd_entry_t newptd;
1427	pt_entry_t newpt;
1428	int *newpp;
1429
1430	POSTCODE(START_ALL_APS_POST);
1431
1432	/**
1433         * NOTE: this needs further thought:
1434         *        where does it get released?
1435         *        should it be set to empy?
1436         *
1437         * get the initial mp_lock with a count of 1 for the BSP
1438         */
1439	mp_lock = 1;	/* this uses a LOGICAL cpu ID, ie BSP == 0 */
1440
1441	/* initialize BSP's local APIC */
1442	apic_initialize();
1443
1444	/* install the AP 1st level boot code */
1445	install_ap_tramp(boot_addr);
1446
1447
1448	/* save the current value of the warm-start vector */
1449	mpbioswarmvec = *((u_long *) WARMBOOT_OFF);
1450	outb(CMOS_REG, BIOS_RESET);
1451	mpbiosreason = inb(CMOS_DATA);
1452
1453	/* record BSP in CPU map */
1454	all_cpus = 1;
1455
1456	/* start each AP */
1457	for (x = 1; x <= mp_naps; ++x) {
1458
1459		/* HACK HACK HACK !!! */
1460
1461		/* alloc new page table directory */
1462		newptd = (pd_entry_t)(kmem_alloc(kernel_map, PAGE_SIZE));
1463
1464		/* clone currently active one (ie: IdlePTD) */
1465		bcopy(PTD, newptd, PAGE_SIZE);	/* inc prv page pde */
1466
1467		/* set up 0 -> 4MB P==V mapping for AP boot */
1468		newptd[0] = PG_V | PG_RW | ((u_long)KPTphys & PG_FRAME);
1469
1470		/* store PTD for this AP */
1471		bootPTD = (pd_entry_t)vtophys(newptd);
1472
1473		/* alloc new page table page */
1474		newpt = (pt_entry_t)(kmem_alloc(kernel_map, PAGE_SIZE));
1475
1476		/* set the new PTD's private page to point there */
1477		newptd[MPPTDI] = PG_V | PG_RW | vtophys(newpt);
1478
1479		/* install self referential entry */
1480		newptd[PTDPTDI] = PG_V | PG_RW | vtophys(newptd);
1481
1482		/* get a new private data page */
1483		newpp = (int *)kmem_alloc(kernel_map, PAGE_SIZE);
1484
1485		/* wire it into the private page table page */
1486		newpt[0] = PG_V | PG_RW | vtophys(newpp);
1487
1488		/* wire the ptp into itself for access */
1489		newpt[1] = PG_V | PG_RW | vtophys(newpt);
1490
1491		/* and the local apic */
1492		newpt[2] = SMP_prvpt[2];
1493
1494		/* and the IO apic mapping[s] */
1495		for (i = 16; i < 32; i++)
1496			newpt[i] = SMP_prvpt[i];
1497
1498		/* prime data page for it to use */
1499		newpp[0] = x;		/* cpuid */
1500		newpp[1] = 0;		/* curproc */
1501		newpp[2] = 0;		/* curpcb */
1502		newpp[3] = 0;		/* npxproc */
1503		newpp[4] = 0;		/* runtime.tv_sec */
1504		newpp[5] = 0;		/* runtime.tv_usec */
1505		newpp[6] = x << 24;	/* cpu_lockid */
1506
1507		/* XXX NOTE: ABANDON bootPTD for now!!!! */
1508
1509		/* END REVOLTING HACKERY */
1510
1511		/* setup a vector to our boot code */
1512		*((volatile u_short *) WARMBOOT_OFF) = WARMBOOT_TARGET;
1513		*((volatile u_short *) WARMBOOT_SEG) = (boot_addr >> 4);
1514		outb(CMOS_REG, BIOS_RESET);
1515		outb(CMOS_DATA, BIOS_WARM);	/* 'warm-start' */
1516
1517		/* attempt to start the Application Processor */
1518		CHECK_INIT(99);	/* setup checkpoints */
1519		if (!start_ap(x, boot_addr)) {
1520			printf("AP #%d (PHY# %d) failed!\n", x, CPU_TO_ID(x));
1521			CHECK_PRINT("trace");	/* show checkpoints */
1522			/* better panic as the AP may be running loose */
1523			printf("panic y/n? [y] ");
1524			if (cngetc() != 'n')
1525				panic("bye-bye");
1526		}
1527		CHECK_PRINT("trace");		/* show checkpoints */
1528
1529		/* record its version info */
1530		cpu_apic_versions[x] = cpu_apic_versions[0];
1531
1532		all_cpus |= (1 << x);		/* record AP in CPU map */
1533	}
1534
1535	/* build our map of 'other' CPUs */
1536	other_cpus = all_cpus & ~(1 << cpuid);
1537
1538	/* fill in our (BSP) APIC version */
1539	cpu_apic_versions[0] = lapic.version;
1540
1541	/* restore the warmstart vector */
1542	*(u_long *) WARMBOOT_OFF = mpbioswarmvec;
1543	outb(CMOS_REG, BIOS_RESET);
1544	outb(CMOS_DATA, mpbiosreason);
1545
1546	pmap_set_opt_bsp();
1547
1548	/* number of APs actually started */
1549	return mp_ncpus - 1;
1550}
1551
1552
1553/*
1554 * load the 1st level AP boot code into base memory.
1555 */
1556
1557/* targets for relocation */
1558extern void bigJump(void);
1559extern void bootCodeSeg(void);
1560extern void bootDataSeg(void);
1561extern void MPentry(void);
1562extern u_int MP_GDT;
1563extern u_int mp_gdtbase;
1564
1565static void
1566install_ap_tramp(u_int boot_addr)
1567{
1568	int     x;
1569	int     size = *(int *) ((u_long) & bootMP_size);
1570	u_char *src = (u_char *) ((u_long) bootMP);
1571	u_char *dst = (u_char *) boot_addr + KERNBASE;
1572	u_int   boot_base = (u_int) bootMP;
1573	u_int8_t *dst8;
1574	u_int16_t *dst16;
1575	u_int32_t *dst32;
1576
1577	POSTCODE(INSTALL_AP_TRAMP_POST);
1578
1579	for (x = 0; x < size; ++x)
1580		*dst++ = *src++;
1581
1582	/*
1583	 * modify addresses in code we just moved to basemem. unfortunately we
1584	 * need fairly detailed info about mpboot.s for this to work.  changes
1585	 * to mpboot.s might require changes here.
1586	 */
1587
1588	/* boot code is located in KERNEL space */
1589	dst = (u_char *) boot_addr + KERNBASE;
1590
1591	/* modify the lgdt arg */
1592	dst32 = (u_int32_t *) (dst + ((u_int) & mp_gdtbase - boot_base));
1593	*dst32 = boot_addr + ((u_int) & MP_GDT - boot_base);
1594
1595	/* modify the ljmp target for MPentry() */
1596	dst32 = (u_int32_t *) (dst + ((u_int) bigJump - boot_base) + 1);
1597	*dst32 = ((u_int) MPentry - KERNBASE);
1598
1599	/* modify the target for boot code segment */
1600	dst16 = (u_int16_t *) (dst + ((u_int) bootCodeSeg - boot_base));
1601	dst8 = (u_int8_t *) (dst16 + 1);
1602	*dst16 = (u_int) boot_addr & 0xffff;
1603	*dst8 = ((u_int) boot_addr >> 16) & 0xff;
1604
1605	/* modify the target for boot data segment */
1606	dst16 = (u_int16_t *) (dst + ((u_int) bootDataSeg - boot_base));
1607	dst8 = (u_int8_t *) (dst16 + 1);
1608	*dst16 = (u_int) boot_addr & 0xffff;
1609	*dst8 = ((u_int) boot_addr >> 16) & 0xff;
1610}
1611
1612
1613/*
1614 * this function starts the AP (application processor) identified
1615 * by the APIC ID 'physicalCpu'.  It does quite a "song and dance"
1616 * to accomplish this.  This is necessary because of the nuances
1617 * of the different hardware we might encounter.  It ain't pretty,
1618 * but it seems to work.
1619 */
1620static int
1621start_ap(int logical_cpu, u_int boot_addr)
1622{
1623	int     physical_cpu;
1624	int     vector;
1625	int     cpus;
1626	u_long  icr_lo, icr_hi;
1627
1628	POSTCODE(START_AP_POST);
1629
1630	/* get the PHYSICAL APIC ID# */
1631	physical_cpu = CPU_TO_ID(logical_cpu);
1632
1633	/* calculate the vector */
1634	vector = (boot_addr >> 12) & 0xff;
1635
1636	/* used as a watchpoint to signal AP startup */
1637	cpus = mp_ncpus;
1638
1639	/*
1640	 * first we do an INIT/RESET IPI this INIT IPI might be run, reseting
1641	 * and running the target CPU. OR this INIT IPI might be latched (P5
1642	 * bug), CPU waiting for STARTUP IPI. OR this INIT IPI might be
1643	 * ignored.
1644	 */
1645
1646	/* setup the address for the target AP */
1647	icr_hi = lapic.icr_hi & ~APIC_ID_MASK;
1648	icr_hi |= (physical_cpu << 24);
1649	lapic.icr_hi = icr_hi;
1650
1651	/* do an INIT IPI: assert RESET */
1652	icr_lo = lapic.icr_lo & 0xfff00000;
1653	lapic.icr_lo = icr_lo | 0x0000c500;
1654
1655	/* wait for pending status end */
1656	while (lapic.icr_lo & APIC_DELSTAT_MASK)
1657		 /* spin */ ;
1658
1659	/* do an INIT IPI: deassert RESET */
1660	lapic.icr_lo = icr_lo | 0x00008500;
1661
1662	/* wait for pending status end */
1663	u_sleep(10000);		/* wait ~10mS */
1664	while (lapic.icr_lo & APIC_DELSTAT_MASK)
1665		 /* spin */ ;
1666
1667	/*
1668	 * next we do a STARTUP IPI: the previous INIT IPI might still be
1669	 * latched, (P5 bug) this 1st STARTUP would then terminate
1670	 * immediately, and the previously started INIT IPI would continue. OR
1671	 * the previous INIT IPI has already run. and this STARTUP IPI will
1672	 * run. OR the previous INIT IPI was ignored. and this STARTUP IPI
1673	 * will run.
1674	 */
1675
1676	/* do a STARTUP IPI */
1677	lapic.icr_lo = icr_lo | 0x00000600 | vector;
1678	while (lapic.icr_lo & APIC_DELSTAT_MASK)
1679		 /* spin */ ;
1680	u_sleep(200);		/* wait ~200uS */
1681
1682	/*
1683	 * finally we do a 2nd STARTUP IPI: this 2nd STARTUP IPI should run IF
1684	 * the previous STARTUP IPI was cancelled by a latched INIT IPI. OR
1685	 * this STARTUP IPI will be ignored, as only ONE STARTUP IPI is
1686	 * recognized after hardware RESET or INIT IPI.
1687	 */
1688
1689	lapic.icr_lo = icr_lo | 0x00000600 | vector;
1690	while (lapic.icr_lo & APIC_DELSTAT_MASK)
1691		 /* spin */ ;
1692	u_sleep(200);		/* wait ~200uS */
1693
1694	/* wait for it to start */
1695	set_apic_timer(5000000);/* == 5 seconds */
1696	while (read_apic_timer())
1697		if (mp_ncpus > cpus)
1698			return 1;	/* return SUCCESS */
1699
1700	return 0;		/* return FAILURE */
1701}
1702
1703
1704/*
1705 * Flush the TLB on all other CPU's
1706 *
1707 * XXX: Needs to handshake and wait for completion before proceding.
1708 */
1709void
1710smp_invltlb(void)
1711{
1712#if defined(APIC_IO)
1713	if (smp_active && invltlb_ok)
1714		all_but_self_ipi(XINVLTLB_OFFSET);
1715#endif  /* APIC_IO */
1716}
1717
1718void
1719invlpg(u_int addr)
1720{
1721	__asm   __volatile("invlpg (%0)"::"r"(addr):"memory");
1722
1723	/* send a message to the other CPUs */
1724	smp_invltlb();
1725}
1726
1727void
1728invltlb(void)
1729{
1730	u_long  temp;
1731
1732	/*
1733	 * This should be implemented as load_cr3(rcr3()) when load_cr3() is
1734	 * inlined.
1735	 */
1736	__asm __volatile("movl %%cr3, %0; movl %0, %%cr3":"=r"(temp) :: "memory");
1737
1738	/* send a message to the other CPUs */
1739	smp_invltlb();
1740}
1741
1742
1743/*
1744 * When called the executing CPU will send an IPI to all other CPUs
1745 *  requesting that they halt execution.
1746 *
1747 * Usually (but not necessarily) called with 'other_cpus' as its arg.
1748 *
1749 *  - Signals all CPUs in map to stop.
1750 *  - Waits for each to stop.
1751 *
1752 * Returns:
1753 *  -1: error
1754 *   0: NA
1755 *   1: ok
1756 *
1757 * XXX FIXME: this is not MP-safe, needs a lock to prevent multiple CPUs
1758 *            from executing at same time.
1759 */
1760int
1761stop_cpus( u_int map )
1762{
1763	if (!smp_active)
1764		return 0;
1765
1766	/* send IPI to all CPUs in map */
1767	stopped_cpus = 0;
1768
1769	/* send the Xcpustop IPI to all CPUs in map */
1770	selected_apic_ipi(map, XCPUSTOP_OFFSET, APIC_DELMODE_FIXED);
1771
1772	while (stopped_cpus != map)
1773		/* spin */ ;
1774
1775	return 1;
1776}
1777
1778
1779/*
1780 * Called by a CPU to restart stopped CPUs.
1781 *
1782 * Usually (but not necessarily) called with 'stopped_cpus' as its arg.
1783 *
1784 *  - Signals all CPUs in map to restart.
1785 *  - Waits for each to restart.
1786 *
1787 * Returns:
1788 *  -1: error
1789 *   0: NA
1790 *   1: ok
1791 */
1792int
1793restart_cpus( u_int map )
1794{
1795	if (!smp_active)
1796		return 0;
1797
1798	started_cpus = map;		/* signal other cpus to restart */
1799
1800	while (started_cpus)		/* wait for each to clear its bit */
1801		/* spin */ ;
1802
1803	return 1;
1804}
1805