mp_machdep.c revision 28027 
1/*
2 * Copyright (c) 1996, by Steve Passe
3 * All rights reserved.
4 *
5 * Redistribution and use in source and binary forms, with or without
6 * modification, are permitted provided that the following conditions
7 * are met:
8 * 1. Redistributions of source code must retain the above copyright
9 *    notice, this list of conditions and the following disclaimer.
10 * 2. The name of the developer may NOT be used to endorse or promote products
11 *    derived from this software without specific prior written permission.
12 *
13 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
14 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
15 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
16 * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
17 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
18 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
19 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
20 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
21 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
22 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
23 * SUCH DAMAGE.
24 *
25 *	$Id: mp_machdep.c,v 1.26 1997/08/09 22:12:14 smp Exp $
26 */
27
28#include "opt_smp.h"
29
30#include <sys/param.h>		/* for KERNBASE */
31#include <sys/systm.h>
32
33#include <vm/vm.h>		/* for KERNBASE */
34#include <vm/vm_param.h>	/* for KERNBASE */
35#include <vm/pmap.h>		/* for KERNBASE */
36#include <vm/vm_kern.h>
37#include <vm/vm_extern.h>
38
39#include <machine/smp.h>
40#include <machine/apic.h>
41#include <machine/mpapic.h>
42#include <machine/segments.h>
43#include <machine/smptests.h>	/** TEST_DEFAULT_CONFIG, TEST_TEST1 */
44#include <machine/tss.h>
45#include <machine/specialreg.h>
46
47#include <i386/i386/cons.h>	/* cngetc() */
48
49#if defined(APIC_IO)
50#include <machine/md_var.h>		/* setidt() */
51#include <i386/isa/icu.h>		/* IPIs */
52#include <i386/isa/intr_machdep.h>	/* IPIs */
53#endif	/* APIC_IO */
54
55#if defined(TEST_DEFAULT_CONFIG)
56#define MPFPS_MPFB1	TEST_DEFAULT_CONFIG
57#else
58#define MPFPS_MPFB1	mpfps->mpfb1
59#endif  /* TEST_DEFAULT_CONFIG */
60
61#define WARMBOOT_TARGET		0
62#define WARMBOOT_OFF		(KERNBASE + 0x0467)
63#define WARMBOOT_SEG		(KERNBASE + 0x0469)
64
65#define BIOS_BASE		(0xf0000)
66#define BIOS_SIZE		(0x10000)
67#define BIOS_COUNT		(BIOS_SIZE/4)
68
69#define CMOS_REG		(0x70)
70#define CMOS_DATA		(0x71)
71#define BIOS_RESET		(0x0f)
72#define BIOS_WARM		(0x0a)
73
74#define PROCENTRY_FLAG_EN	0x01
75#define PROCENTRY_FLAG_BP	0x02
76#define IOAPICENTRY_FLAG_EN	0x01
77
78
79/* MP Floating Pointer Structure */
80typedef struct MPFPS {
81	char    signature[4];
82	void   *pap;
83	u_char  length;
84	u_char  spec_rev;
85	u_char  checksum;
86	u_char  mpfb1;
87	u_char  mpfb2;
88	u_char  mpfb3;
89	u_char  mpfb4;
90	u_char  mpfb5;
91}      *mpfps_t;
92
93/* MP Configuration Table Header */
94typedef struct MPCTH {
95	char    signature[4];
96	u_short base_table_length;
97	u_char  spec_rev;
98	u_char  checksum;
99	u_char  oem_id[8];
100	u_char  product_id[12];
101	void   *oem_table_pointer;
102	u_short oem_table_size;
103	u_short entry_count;
104	void   *apic_address;
105	u_short extended_table_length;
106	u_char  extended_table_checksum;
107	u_char  reserved;
108}      *mpcth_t;
109
110
111typedef struct PROCENTRY {
112	u_char  type;
113	u_char  apic_id;
114	u_char  apic_version;
115	u_char  cpu_flags;
116	u_long  cpu_signature;
117	u_long  feature_flags;
118	u_long  reserved1;
119	u_long  reserved2;
120}      *proc_entry_ptr;
121
122typedef struct BUSENTRY {
123	u_char  type;
124	u_char  bus_id;
125	char    bus_type[6];
126}      *bus_entry_ptr;
127
128typedef struct IOAPICENTRY {
129	u_char  type;
130	u_char  apic_id;
131	u_char  apic_version;
132	u_char  apic_flags;
133	void   *apic_address;
134}      *io_apic_entry_ptr;
135
136typedef struct INTENTRY {
137	u_char  type;
138	u_char  int_type;
139	u_short int_flags;
140	u_char  src_bus_id;
141	u_char  src_bus_irq;
142	u_char  dst_apic_id;
143	u_char  dst_apic_int;
144}      *int_entry_ptr;
145
146/* descriptions of MP basetable entries */
147typedef struct BASETABLE_ENTRY {
148	u_char  type;
149	u_char  length;
150	char    name[16];
151}       basetable_entry;
152
153/*
154 * this code MUST be enabled here and in mpboot.s.
155 * it follows the very early stages of AP boot by placing values in CMOS ram.
156 * it NORMALLY will never be needed and thus the primitive method for enabling.
157 *
158#define CHECK_POINTS
159 */
160
161#if defined(CHECK_POINTS)
162#define CHECK_READ(A)	 (outb(CMOS_REG, (A)), inb(CMOS_DATA))
163#define CHECK_WRITE(A,D) (outb(CMOS_REG, (A)), outb(CMOS_DATA, (D)))
164
165#define CHECK_INIT(D);				\
166	CHECK_WRITE(0x34, (D));			\
167	CHECK_WRITE(0x35, (D));			\
168	CHECK_WRITE(0x36, (D));			\
169	CHECK_WRITE(0x37, (D));			\
170	CHECK_WRITE(0x38, (D));			\
171	CHECK_WRITE(0x39, (D));
172
173#define CHECK_PRINT(S);				\
174	printf("%s: %d, %d, %d, %d, %d, %d\n",	\
175	   (S),					\
176	   CHECK_READ(0x34),			\
177	   CHECK_READ(0x35),			\
178	   CHECK_READ(0x36),			\
179	   CHECK_READ(0x37),			\
180	   CHECK_READ(0x38),			\
181	   CHECK_READ(0x39));
182
183#else				/* CHECK_POINTS */
184
185#define CHECK_INIT(D)
186#define CHECK_PRINT(S)
187
188#endif				/* CHECK_POINTS */
189
190/*
191 * Values to send to the POST hardware.
192 */
193#define MP_BOOTADDRESS_POST	0x10
194#define MP_PROBE_POST		0x11
195#define MP_START_POST		0x12
196#define MP_ANNOUNCE_POST	0x13
197#define MPTABLE_PASS1_POST	0x14
198#define MPTABLE_PASS2_POST	0x15
199#define MP_ENABLE_POST		0x16
200#define START_ALL_APS_POST	0x17
201#define INSTALL_AP_TRAMP_POST	0x18
202#define START_AP_POST		0x19
203
204/** XXX FIXME: where does this really belong, isa.h/isa.c perhaps? */
205int	current_postcode;
206
207/** XXX FIXME: what system files declare these??? */
208extern struct region_descriptor r_gdt, r_idt;
209
210int	mp_ncpus;		/* # of CPUs, including BSP */
211int	mp_naps;		/* # of Applications processors */
212int	mp_nbusses;		/* # of busses */
213int	mp_napics;		/* # of IO APICs */
214int	boot_cpu_id;		/* designated BSP */
215vm_offset_t cpu_apic_address;
216vm_offset_t io_apic_address[NAPICID];	/* NAPICID is more than enough */
217
218u_int32_t cpu_apic_versions[NCPU];
219u_int32_t io_apic_versions[NAPIC];
220
221/*
222 * APIC ID logical/physical mapping structures.
223 * We oversize these to simplify boot-time config.
224 */
225int     cpu_num_to_apic_id[NAPICID];
226int     io_num_to_apic_id[NAPICID];
227int     apic_id_to_logical[NAPICID];
228
229/* Bitmap of all available CPUs */
230u_int	all_cpus;
231
232/* Boot of AP uses this PTD */
233u_int *bootPTD;
234
235/* Hotwire a 0->4MB V==P mapping */
236extern pt_entry_t KPTphys;
237
238/* Virtual address of per-cpu common_tss */
239extern struct i386tss common_tss;
240
241/*
242 * Local data and functions.
243 */
244
245static int	mp_capable;
246static u_int	boot_address;
247static u_int	base_memory;
248
249static int	picmode;		/* 0: virtual wire mode, 1: PIC mode */
250static mpfps_t	mpfps;
251static int	search_for_sig(u_int32_t target, int count);
252static void	mp_enable(u_int boot_addr);
253
254static int	mptable_pass1(void);
255static int	mptable_pass2(void);
256static void	default_mp_table(int type);
257static void	fix_mp_table(void);
258static void	init_locks(void);
259static int	start_all_aps(u_int boot_addr);
260static void	install_ap_tramp(u_int boot_addr);
261static int	start_ap(int logicalCpu, u_int boot_addr);
262
263
264/*
265 * Calculate usable address in base memory for AP trampoline code.
266 */
267u_int
268mp_bootaddress(u_int basemem)
269{
270	POSTCODE(MP_BOOTADDRESS_POST);
271
272	base_memory = basemem * 1024;	/* convert to bytes */
273
274	boot_address = base_memory & ~0xfff;	/* round down to 4k boundary */
275	if ((base_memory - boot_address) < bootMP_size)
276		boot_address -= 4096;	/* not enough, lower by 4k */
277
278	return boot_address;
279}
280
281
282/*
283 * Look for an Intel MP spec table (ie, SMP capable hardware).
284 */
285int
286mp_probe(void)
287{
288	int     x;
289	u_long  segment;
290	u_int32_t target;
291
292	POSTCODE(MP_PROBE_POST);
293
294	/* see if EBDA exists */
295	if (segment = (u_long) * (u_short *) (KERNBASE + 0x40e)) {
296		/* search first 1K of EBDA */
297		target = (u_int32_t) (segment << 4);
298		if ((x = search_for_sig(target, 1024 / 4)) >= 0)
299			goto found;
300	} else {
301		/* last 1K of base memory, effective 'top of base' passed in */
302		target = (u_int32_t) (base_memory - 0x400);
303		if ((x = search_for_sig(target, 1024 / 4)) >= 0)
304			goto found;
305	}
306
307	/* search the BIOS */
308	target = (u_int32_t) BIOS_BASE;
309	if ((x = search_for_sig(target, BIOS_COUNT)) >= 0)
310		goto found;
311
312	/* nothing found */
313	mpfps = (mpfps_t)0;
314	mp_capable = 0;
315	return 0;
316
317found:
318	/* calculate needed resources */
319	mpfps = (mpfps_t)x;
320	if (mptable_pass1())
321		panic("you must reconfigure your kernel");
322
323	/* flag fact that we are running multiple processors */
324	mp_capable = 1;
325	return 1;
326}
327
328
329/*
330 * Startup the SMP processors.
331 */
332void
333mp_start(void)
334{
335	POSTCODE(MP_START_POST);
336
337	/* look for MP capable motherboard */
338	if (mp_capable)
339		mp_enable(boot_address);
340	else
341		panic("MP hardware not found!");
342}
343
344
345/*
346 * Print various information about the SMP system hardware and setup.
347 */
348void
349mp_announce(void)
350{
351	int     x;
352
353	POSTCODE(MP_ANNOUNCE_POST);
354
355	printf("FreeBSD/SMP: Multiprocessor motherboard\n");
356	printf(" cpu0 (BSP): apic id: %2d", CPU_TO_ID(0));
357	printf(", version: 0x%08x", cpu_apic_versions[0]);
358	printf(", at 0x%08x\n", cpu_apic_address);
359	for (x = 1; x <= mp_naps; ++x) {
360		printf(" cpu%d (AP):  apic id: %2d", x, CPU_TO_ID(x));
361		printf(", version: 0x%08x", cpu_apic_versions[x]);
362		printf(", at 0x%08x\n", cpu_apic_address);
363	}
364
365#if defined(APIC_IO)
366	for (x = 0; x < mp_napics; ++x) {
367		printf(" io%d (APIC): apic id: %2d", x, IO_TO_ID(x));
368		printf(", version: 0x%08x", io_apic_versions[x]);
369		printf(", at 0x%08x\n", io_apic_address[x]);
370	}
371#else
372	printf(" Warning: APIC I/O disabled\n");
373#endif	/* APIC_IO */
374}
375
376/*
377 * AP cpu's call this to sync up protected mode.
378 */
379void
380init_secondary(void)
381{
382	int     gsel_tss, slot;
383
384	r_gdt.rd_limit = sizeof(gdt[0]) * (NGDT + NCPU) - 1;
385	r_gdt.rd_base = (int) gdt;
386	lgdt(&r_gdt);			/* does magic intra-segment return */
387	lidt(&r_idt);
388	lldt(_default_ldt);
389
390	slot = NGDT + cpuid;
391	gsel_tss = GSEL(slot, SEL_KPL);
392	gdt[slot].sd.sd_type = SDT_SYS386TSS;
393	common_tss.tss_esp0 = 0;	/* not used until after switch */
394	common_tss.tss_ss0 = GSEL(GDATA_SEL, SEL_KPL);
395	common_tss.tss_ioopt = (sizeof common_tss) << 16;
396	ltr(gsel_tss);
397
398	load_cr0(0x8005003b);		/* XXX! */
399
400	PTD[0] = 0;
401	pmap_set_opt((unsigned *)PTD);
402
403	invltlb();
404}
405
406
407#if defined(APIC_IO)
408/*
409 * Final configuration of the BSP's local APIC:
410 *  - disable 'pic mode'.
411 *  - disable 'virtual wire mode'.
412 *  - enable NMI.
413 */
414void
415bsp_apic_configure(void)
416{
417	u_char		byte;
418	u_int32_t	temp;
419
420	/* leave 'pic mode' if necessary */
421	if (picmode) {
422		outb(0x22, 0x70);	/* select IMCR */
423		byte = inb(0x23);	/* current contents */
424		byte |= 0x01;		/* mask external INTR */
425		outb(0x23, byte);	/* disconnect 8259s/NMI */
426	}
427
428	/* mask lint0 (the 8259 'virtual wire' connection) */
429	temp = lapic.lvt_lint0;
430	temp |= APIC_LVT_M;		/* set the mask */
431	lapic.lvt_lint0 = temp;
432
433        /* setup lint1 to handle NMI */
434        temp = lapic.lvt_lint1;
435        temp &= ~APIC_LVT_M;		/* clear the mask */
436        lapic.lvt_lint1 = temp;
437
438	if (bootverbose)
439		apic_dump("bsp_apic_configure()");
440}
441#endif  /* APIC_IO */
442
443
444/*******************************************************************
445 * local functions and data
446 */
447
448/*
449 * start the SMP system
450 */
451static void
452mp_enable(u_int boot_addr)
453{
454	int     x;
455#if defined(APIC_IO)
456	int     apic;
457	u_int   ux;
458#endif	/* APIC_IO */
459
460	POSTCODE(MP_ENABLE_POST);
461
462	/* turn on 4MB of V == P addressing so we can get to MP table */
463	*(int *)PTD = PG_V | PG_RW | ((u_long)KPTphys & PG_FRAME);
464	invltlb();
465
466	/* examine the MP table for needed info, uses physical addresses */
467	x = mptable_pass2();
468
469	*(int *)PTD = 0;
470	invltlb();
471
472	/* can't process default configs till the CPU APIC is pmapped */
473	if (x)
474		default_mp_table(x);
475
476	/* post scan cleanup */
477	fix_mp_table();
478
479#if defined(APIC_IO)
480
481	/* fill the LOGICAL io_apic_versions table */
482	for (apic = 0; apic < mp_napics; ++apic) {
483		ux = io_apic_read(apic, IOAPIC_VER);
484		io_apic_versions[apic] = ux;
485	}
486
487	/* program each IO APIC in the system */
488	for (apic = 0; apic < mp_napics; ++apic)
489		if (io_apic_setup(apic) < 0)
490			panic("IO APIC setup failure");
491
492	/* install a 'Spurious INTerrupt' vector */
493	setidt(XSPURIOUSINT_OFFSET, Xspuriousint,
494	       SDT_SYS386IGT, SEL_KPL, GSEL(GCODE_SEL, SEL_KPL));
495
496	/* install an inter-CPU IPI for TLB invalidation */
497	setidt(XINVLTLB_OFFSET, Xinvltlb,
498	       SDT_SYS386IGT, SEL_KPL, GSEL(GCODE_SEL, SEL_KPL));
499
500	/* install an inter-CPU IPI for CPU stop/restart */
501	setidt(XCPUSTOP_OFFSET, Xcpustop,
502	       SDT_SYS386IGT, SEL_KPL, GSEL(GCODE_SEL, SEL_KPL));
503
504#if defined(TEST_TEST1)
505	/* install a "fake hardware INTerrupt" vector */
506	setidt(XTEST1_OFFSET, Xtest1,
507	       SDT_SYS386IGT, SEL_KPL, GSEL(GCODE_SEL, SEL_KPL));
508#endif  /** TEST_TEST1 */
509
510#endif	/* APIC_IO */
511
512	/* initialize all SMP locks */
513	init_locks();
514
515	/* start each Application Processor */
516	start_all_aps(boot_addr);
517
518	/*
519	 * The init process might be started on a different CPU now,
520	 * and the boot CPU might not call prepare_usermode to get
521	 * cr0 correctly configured. Thus we initialize cr0 here.
522	 */
523	load_cr0(rcr0() | CR0_WP | CR0_AM);
524}
525
526
527/*
528 * look for the MP spec signature
529 */
530
531/* string defined by the Intel MP Spec as identifying the MP table */
532#define MP_SIG		0x5f504d5f	/* _MP_ */
533#define NEXT(X)		((X) += 4)
534static int
535search_for_sig(u_int32_t target, int count)
536{
537	int     x;
538	u_int32_t *addr = (u_int32_t *) (KERNBASE + target);
539
540	for (x = 0; x < count; NEXT(x))
541		if (addr[x] == MP_SIG)
542			/* make array index a byte index */
543			return (target + (x * sizeof(u_int32_t)));
544
545	return -1;
546}
547
548
549static basetable_entry basetable_entry_types[] =
550{
551	{0, 20, "Processor"},
552	{1, 8, "Bus"},
553	{2, 8, "I/O APIC"},
554	{3, 8, "I/O INT"},
555	{4, 8, "Local INT"}
556};
557
558typedef struct BUSDATA {
559	u_char  bus_id;
560	enum busTypes bus_type;
561}       bus_datum;
562
563typedef struct INTDATA {
564	u_char  int_type;
565	u_short int_flags;
566	u_char  src_bus_id;
567	u_char  src_bus_irq;
568	u_char  dst_apic_id;
569	u_char  dst_apic_int;
570}       io_int, local_int;
571
572typedef struct BUSTYPENAME {
573	u_char  type;
574	char    name[7];
575}       bus_type_name;
576
577static bus_type_name bus_type_table[] =
578{
579	{CBUS, "CBUS"},
580	{CBUSII, "CBUSII"},
581	{EISA, "EISA"},
582	{UNKNOWN_BUSTYPE, "---"},
583	{UNKNOWN_BUSTYPE, "---"},
584	{ISA, "ISA"},
585	{UNKNOWN_BUSTYPE, "---"},
586	{UNKNOWN_BUSTYPE, "---"},
587	{UNKNOWN_BUSTYPE, "---"},
588	{UNKNOWN_BUSTYPE, "---"},
589	{UNKNOWN_BUSTYPE, "---"},
590	{UNKNOWN_BUSTYPE, "---"},
591	{PCI, "PCI"},
592	{UNKNOWN_BUSTYPE, "---"},
593	{UNKNOWN_BUSTYPE, "---"},
594	{UNKNOWN_BUSTYPE, "---"},
595	{UNKNOWN_BUSTYPE, "---"},
596	{XPRESS, "XPRESS"},
597	{UNKNOWN_BUSTYPE, "---"}
598};
599/* from MP spec v1.4, table 5-1 */
600static int default_data[7][5] =
601{
602/*   nbus, id0, type0, id1, type1 */
603	{1, 0, ISA, 255, 255},
604	{1, 0, EISA, 255, 255},
605	{1, 0, EISA, 255, 255},
606	{0, 255, 255, 255, 255},/* MCA not supported */
607	{2, 0, ISA, 1, PCI},
608	{2, 0, EISA, 1, PCI},
609	{0, 255, 255, 255, 255}	/* MCA not supported */
610};
611
612
613/* the bus data */
614bus_datum bus_data[NBUS];
615
616/* the IO INT data, one entry per possible APIC INTerrupt */
617io_int  io_apic_ints[NINTR];
618
619static int nintrs;
620
621static int processor_entry	__P((proc_entry_ptr entry, int cpu));
622static int bus_entry		__P((bus_entry_ptr entry, int bus));
623static int io_apic_entry	__P((io_apic_entry_ptr entry, int apic));
624static int int_entry		__P((int_entry_ptr entry, int intr));
625static int lookup_bus_type	__P((char *name));
626
627
628/*
629 * 1st pass on motherboard's Intel MP specification table.
630 *
631 * initializes:
632 *	mp_ncpus = 1
633 *
634 * determines:
635 *	cpu_apic_address (common to all CPUs)
636 *	io_apic_address[N]
637 *	mp_naps
638 *	mp_nbusses
639 *	mp_napics
640 *	nintrs
641 */
642static int
643mptable_pass1(void)
644{
645	int	x;
646	mpcth_t	cth;
647	int	totalSize;
648	void*	position;
649	int	count;
650	int	type;
651	int	mustpanic;
652
653	POSTCODE(MPTABLE_PASS1_POST);
654
655	mustpanic = 0;
656
657	/* clear various tables */
658	for (x = 0; x < NAPICID; ++x) {
659		io_apic_address[x] = ~0;	/* IO APIC address table */
660	}
661
662	/* init everything to empty */
663	mp_naps = 0;
664	mp_nbusses = 0;
665	mp_napics = 0;
666	nintrs = 0;
667
668	/* check for use of 'default' configuration */
669	if (MPFPS_MPFB1 != 0) {
670		/* use default addresses */
671		cpu_apic_address = DEFAULT_APIC_BASE;
672		io_apic_address[0] = DEFAULT_IO_APIC_BASE;
673
674		/* fill in with defaults */
675		mp_naps = 2;		/* includes BSP */
676		mp_nbusses = default_data[MPFPS_MPFB1 - 1][0];
677#if defined(APIC_IO)
678		mp_napics = 1;
679		nintrs = 16;
680#endif	/* APIC_IO */
681	}
682	else {
683		if ((cth = mpfps->pap) == 0)
684			panic("MP Configuration Table Header MISSING!");
685
686		cpu_apic_address = (vm_offset_t) cth->apic_address;
687
688		/* walk the table, recording info of interest */
689		totalSize = cth->base_table_length - sizeof(struct MPCTH);
690		position = (u_char *) cth + sizeof(struct MPCTH);
691		count = cth->entry_count;
692
693		while (count--) {
694			switch (type = *(u_char *) position) {
695			case 0: /* processor_entry */
696				if (((proc_entry_ptr)position)->cpu_flags
697					& PROCENTRY_FLAG_EN)
698					++mp_naps;
699				break;
700			case 1: /* bus_entry */
701				++mp_nbusses;
702				break;
703			case 2: /* io_apic_entry */
704				if (((io_apic_entry_ptr)position)->apic_flags
705					& IOAPICENTRY_FLAG_EN)
706					io_apic_address[mp_napics++] =
707					    (vm_offset_t)((io_apic_entry_ptr)
708						position)->apic_address;
709				break;
710			case 3: /* int_entry */
711				++nintrs;
712				break;
713			case 4:	/* int_entry */
714				break;
715			default:
716				panic("mpfps Base Table HOSED!");
717				/* NOTREACHED */
718			}
719
720			totalSize -= basetable_entry_types[type].length;
721			(u_char*)position += basetable_entry_types[type].length;
722		}
723	}
724
725	/* qualify the numbers */
726	if (mp_naps > NCPU)
727		printf("Warning: only using %d of %d available CPUs!\n",
728			NCPU, mp_naps);
729#if 0
730		/** XXX we consider this legal now (but should we?) */
731		mustpanic = 1;
732#endif
733	if (mp_nbusses > NBUS) {
734		printf("found %d busses, increase NBUS\n", mp_nbusses);
735		mustpanic = 1;
736	}
737	if (mp_napics > NAPIC) {
738		printf("found %d apics, increase NAPIC\n", mp_napics);
739		mustpanic = 1;
740	}
741	if (nintrs > NINTR) {
742		printf("found %d intrs, increase NINTR\n", nintrs);
743		mustpanic = 1;
744	}
745
746	/*
747	 * Count the BSP.
748	 * This is also used as a counter while starting the APs.
749	 */
750	mp_ncpus = 1;
751
752	--mp_naps;	/* subtract the BSP */
753
754	return mustpanic;
755}
756
757
758/*
759 * 2nd pass on motherboard's Intel MP specification table.
760 *
761 * sets:
762 *	boot_cpu_id
763 *	ID_TO_IO(N), phy APIC ID to log CPU/IO table
764 *	CPU_TO_ID(N), logical CPU to APIC ID table
765 *	IO_TO_ID(N), logical IO to APIC ID table
766 *	bus_data[N]
767 *	io_apic_ints[N]
768 */
769static int
770mptable_pass2(void)
771{
772	int     x;
773	mpcth_t cth;
774	int     totalSize;
775	void*   position;
776	int     count;
777	int     type;
778	int     apic, bus, cpu, intr;
779
780	POSTCODE(MPTABLE_PASS2_POST);
781
782	/* clear various tables */
783	for (x = 0; x < NAPICID; ++x) {
784		ID_TO_IO(x) = -1;	/* phy APIC ID to log CPU/IO table */
785		CPU_TO_ID(x) = -1;	/* logical CPU to APIC ID table */
786		IO_TO_ID(x) = -1;	/* logical IO to APIC ID table */
787	}
788
789	/* clear bus data table */
790	for (x = 0; x < NBUS; ++x)
791		bus_data[x].bus_id = 0xff;
792
793	/* clear IO APIC INT table */
794	for (x = 0; x < NINTR; ++x)
795		io_apic_ints[x].int_type = 0xff;
796
797	/* setup the cpu/apic mapping arrays */
798	boot_cpu_id = -1;
799
800	/* record whether PIC or virtual-wire mode */
801	picmode = (mpfps->mpfb2 & 0x80) ? 1 : 0;
802
803	/* check for use of 'default' configuration */
804	if (MPFPS_MPFB1 != 0)
805		return MPFPS_MPFB1;	/* return default configuration type */
806
807	if ((cth = mpfps->pap) == 0)
808		panic("MP Configuration Table Header MISSING!");
809
810	/* walk the table, recording info of interest */
811	totalSize = cth->base_table_length - sizeof(struct MPCTH);
812	position = (u_char *) cth + sizeof(struct MPCTH);
813	count = cth->entry_count;
814	apic = bus = intr = 0;
815	cpu = 1;				/* pre-count the BSP */
816
817	while (count--) {
818		switch (type = *(u_char *) position) {
819		case 0:
820			if (processor_entry(position, cpu))
821				++cpu;
822			break;
823		case 1:
824			if (bus_entry(position, bus))
825				++bus;
826			break;
827		case 2:
828			if (io_apic_entry(position, apic))
829				++apic;
830			break;
831		case 3:
832			if (int_entry(position, intr))
833				++intr;
834			break;
835		case 4:
836			/* int_entry(position); */
837			break;
838		default:
839			panic("mpfps Base Table HOSED!");
840			/* NOTREACHED */
841		}
842
843		totalSize -= basetable_entry_types[type].length;
844		(u_char *) position += basetable_entry_types[type].length;
845	}
846
847	if (boot_cpu_id == -1)
848		panic("NO BSP found!");
849
850	/* report fact that its NOT a default configuration */
851	return 0;
852}
853
854
855/*
856 * parse an Intel MP specification table
857 */
858static void
859fix_mp_table(void)
860{
861	int	x;
862	int	id;
863	int	bus_0;
864	int	bus_pci;
865	int	num_pci_bus;
866
867	/*
868	 * Fix mis-numbering of the PCI bus and its INT entries if the BIOS
869	 * did it wrong.  The MP spec says that when more than 1 PCI bus
870	 * exists the BIOS must begin with bus entries for the PCI bus and use
871	 * actual PCI bus numbering.  This implies that when only 1 PCI bus
872	 * exists the BIOS can choose to ignore this ordering, and indeed many
873	 * MP motherboards do ignore it.  This causes a problem when the PCI
874	 * sub-system makes requests of the MP sub-system based on PCI bus
875	 * numbers.	So here we look for the situation and renumber the
876	 * busses and associated INTs in an effort to "make it right".
877	 */
878
879	/* find bus 0, PCI bus, count the number of PCI busses */
880	for (num_pci_bus = 0, x = 0; x < mp_nbusses; ++x) {
881		if (bus_data[x].bus_id == 0) {
882			bus_0 = x;
883		}
884		if (bus_data[x].bus_type == PCI) {
885			++num_pci_bus;
886			bus_pci = x;
887		}
888	}
889	/*
890	 * bus_0 == slot of bus with ID of 0
891	 * bus_pci == slot of last PCI bus encountered
892	 */
893
894	/* check the 1 PCI bus case for sanity */
895	if (num_pci_bus == 1) {
896
897		/* if it is number 0 all is well */
898		if (bus_data[bus_pci].bus_id == 0)
899			return;
900
901		/* mis-numbered, swap with whichever bus uses slot 0 */
902
903		/* swap the bus entry types */
904		bus_data[bus_pci].bus_type = bus_data[bus_0].bus_type;
905		bus_data[bus_0].bus_type = PCI;
906
907		/* swap each relavant INTerrupt entry */
908		id = bus_data[bus_pci].bus_id;
909		for (x = 0; x < nintrs; ++x) {
910			if (io_apic_ints[x].src_bus_id == id) {
911				io_apic_ints[x].src_bus_id = 0;
912			}
913			else if (io_apic_ints[x].src_bus_id == 0) {
914				io_apic_ints[x].src_bus_id = id;
915			}
916		}
917	}
918	/* sanity check if more than 1 PCI bus */
919	else if (num_pci_bus > 1) {
920		for (x = 0; x < mp_nbusses; ++x) {
921			if (bus_data[x].bus_type != PCI)
922				continue;
923			if (bus_data[x].bus_id >= num_pci_bus)
924				panic("bad PCI bus numbering");
925		}
926	}
927}
928
929
930static int
931processor_entry(proc_entry_ptr entry, int cpu)
932{
933	/* check for usability */
934	if ((cpu >= NCPU) || !(entry->cpu_flags & PROCENTRY_FLAG_EN))
935		return 0;
936
937	/* check for BSP flag */
938	if (entry->cpu_flags & PROCENTRY_FLAG_BP) {
939		boot_cpu_id = entry->apic_id;
940		CPU_TO_ID(0) = entry->apic_id;
941		ID_TO_CPU(entry->apic_id) = 0;
942		return 0;	/* its already been counted */
943	}
944
945	/* add another AP to list, if less than max number of CPUs */
946	else {
947		CPU_TO_ID(cpu) = entry->apic_id;
948		ID_TO_CPU(entry->apic_id) = cpu;
949		return 1;
950	}
951}
952
953
954static int
955bus_entry(bus_entry_ptr entry, int bus)
956{
957	int     x;
958	char    c, name[8];
959
960	/* encode the name into an index */
961	for (x = 0; x < 6; ++x) {
962		if ((c = entry->bus_type[x]) == ' ')
963			break;
964		name[x] = c;
965	}
966	name[x] = '\0';
967
968	if ((x = lookup_bus_type(name)) == UNKNOWN_BUSTYPE)
969		panic("unknown bus type: '%s'", name);
970
971	bus_data[bus].bus_id = entry->bus_id;
972	bus_data[bus].bus_type = x;
973
974	return 1;
975}
976
977
978static int
979io_apic_entry(io_apic_entry_ptr entry, int apic)
980{
981	if (!(entry->apic_flags & IOAPICENTRY_FLAG_EN))
982		return 0;
983
984	IO_TO_ID(apic) = entry->apic_id;
985	ID_TO_IO(entry->apic_id) = apic;
986
987	return 1;
988}
989
990
991static int
992lookup_bus_type(char *name)
993{
994	int     x;
995
996	for (x = 0; x < MAX_BUSTYPE; ++x)
997		if (strcmp(bus_type_table[x].name, name) == 0)
998			return bus_type_table[x].type;
999
1000	return UNKNOWN_BUSTYPE;
1001}
1002
1003
1004static int
1005int_entry(int_entry_ptr entry, int intr)
1006{
1007	io_apic_ints[intr].int_type = entry->int_type;
1008	io_apic_ints[intr].int_flags = entry->int_flags;
1009	io_apic_ints[intr].src_bus_id = entry->src_bus_id;
1010	io_apic_ints[intr].src_bus_irq = entry->src_bus_irq;
1011	io_apic_ints[intr].dst_apic_id = entry->dst_apic_id;
1012	io_apic_ints[intr].dst_apic_int = entry->dst_apic_int;
1013
1014	return 1;
1015}
1016
1017
1018static int
1019apic_int_is_bus_type(int intr, int bus_type)
1020{
1021	int     bus;
1022
1023	for (bus = 0; bus < mp_nbusses; ++bus)
1024		if ((bus_data[bus].bus_id == io_apic_ints[intr].src_bus_id)
1025		    && ((int) bus_data[bus].bus_type == bus_type))
1026			return 1;
1027
1028	return 0;
1029}
1030
1031
1032/*
1033 * Given a traditional ISA INT mask, return an APIC mask.
1034 */
1035u_int
1036isa_apic_mask(u_int isa_mask)
1037{
1038	int isa_irq;
1039	int apic_pin;
1040
1041#if defined(SKIP_IRQ15_REDIRECT)
1042	if (isa_mask == (1 << 15)) {
1043		printf("skipping ISA IRQ15 redirect\n");
1044		return isa_mask;
1045	}
1046#endif  /* SKIP_IRQ15_REDIRECT */
1047
1048	isa_irq = ffs(isa_mask);		/* find its bit position */
1049	if (isa_irq == 0)			/* doesn't exist */
1050		return 0;
1051	--isa_irq;				/* make it zero based */
1052
1053	apic_pin = isa_apic_pin(isa_irq);	/* look for APIC connection */
1054	if (apic_pin == -1)
1055		return 0;
1056
1057	return (1 << apic_pin);			/* convert pin# to a mask */
1058}
1059
1060
1061/*
1062 * Determine which APIC pin an ISA/EISA INT is attached to.
1063 */
1064#define INTTYPE(I)	(io_apic_ints[(I)].int_type)
1065#define INTPIN(I)	(io_apic_ints[(I)].dst_apic_int)
1066
1067#define SRCBUSIRQ(I)	(io_apic_ints[(I)].src_bus_irq)
1068int
1069isa_apic_pin(int isa_irq)
1070{
1071	int     intr;
1072
1073	for (intr = 0; intr < nintrs; ++intr) {		/* check each record */
1074		if (INTTYPE(intr) == 0) {		/* standard INT */
1075			if (SRCBUSIRQ(intr) == isa_irq) {
1076				if (apic_int_is_bus_type(intr, ISA) ||
1077			            apic_int_is_bus_type(intr, EISA))
1078					return INTPIN(intr);	/* found */
1079			}
1080		}
1081	}
1082	return -1;					/* NOT found */
1083}
1084#undef SRCBUSIRQ
1085
1086
1087/*
1088 * Determine which APIC pin a PCI INT is attached to.
1089 */
1090#define SRCBUSID(I)	(io_apic_ints[(I)].src_bus_id)
1091#define SRCBUSDEVICE(I)	((io_apic_ints[(I)].src_bus_irq >> 2) & 0x1f)
1092#define SRCBUSLINE(I)	(io_apic_ints[(I)].src_bus_irq & 0x03)
1093int
1094pci_apic_pin(int pciBus, int pciDevice, int pciInt)
1095{
1096	int     intr;
1097
1098	--pciInt;					/* zero based */
1099
1100	for (intr = 0; intr < nintrs; ++intr)		/* check each record */
1101		if ((INTTYPE(intr) == 0)		/* standard INT */
1102		    && (SRCBUSID(intr) == pciBus)
1103		    && (SRCBUSDEVICE(intr) == pciDevice)
1104		    && (SRCBUSLINE(intr) == pciInt))	/* a candidate IRQ */
1105			if (apic_int_is_bus_type(intr, PCI))
1106				return INTPIN(intr);	/* exact match */
1107
1108	return -1;					/* NOT found */
1109}
1110#undef SRCBUSLINE
1111#undef SRCBUSDEVICE
1112#undef SRCBUSID
1113
1114#undef INTPIN
1115#undef INTTYPE
1116
1117
1118/*
1119 * Reprogram the MB chipset to NOT redirect an ISA INTerrupt.
1120 *
1121 * XXX FIXME:
1122 *  Exactly what this means is unclear at this point.  It is a solution
1123 *  for motherboards that redirect the MBIRQ0 pin.  Generically a motherboard
1124 *  could route any of the ISA INTs to upper (>15) IRQ values.  But most would
1125 *  NOT be redirected via MBIRQ0, thus "undirect()ing" them would NOT be an
1126 *  option.
1127 */
1128int
1129undirect_isa_irq(int rirq)
1130{
1131#if defined(READY)
1132	printf("Freeing redirected ISA irq %d.\n", rirq);
1133	/** FIXME: tickle the MB redirector chip */
1134	return ???;
1135#else
1136	printf("Freeing (NOT implemented) redirected ISA irq %d.\n", rirq);
1137	return 0;
1138#endif  /* READY */
1139}
1140
1141
1142/*
1143 * Reprogram the MB chipset to NOT redirect a PCI INTerrupt
1144 */
1145int
1146undirect_pci_irq(int rirq)
1147{
1148#if defined(READY)
1149	if (bootverbose)
1150		printf("Freeing redirected PCI irq %d.\n", rirq);
1151
1152	/** FIXME: tickle the MB redirector chip */
1153	return ???;
1154#else
1155	if (bootverbose)
1156		printf("Freeing (NOT implemented) redirected PCI irq %d.\n",
1157		       rirq);
1158	return 0;
1159#endif  /* READY */
1160}
1161
1162
1163/*
1164 * given a bus ID, return:
1165 *  the bus type if found
1166 *  -1 if NOT found
1167 */
1168int
1169apic_bus_type(int id)
1170{
1171	int     x;
1172
1173	for (x = 0; x < mp_nbusses; ++x)
1174		if (bus_data[x].bus_id == id)
1175			return bus_data[x].bus_type;
1176
1177	return -1;
1178}
1179
1180
1181/*
1182 * given a LOGICAL APIC# and pin#, return:
1183 *  the associated src bus ID if found
1184 *  -1 if NOT found
1185 */
1186int
1187apic_src_bus_id(int apic, int pin)
1188{
1189	int     x;
1190
1191	/* search each of the possible INTerrupt sources */
1192	for (x = 0; x < nintrs; ++x)
1193		if ((apic == ID_TO_IO(io_apic_ints[x].dst_apic_id)) &&
1194		    (pin == io_apic_ints[x].dst_apic_int))
1195			return (io_apic_ints[x].src_bus_id);
1196
1197	return -1;		/* NOT found */
1198}
1199
1200
1201/*
1202 * given a LOGICAL APIC# and pin#, return:
1203 *  the associated src bus IRQ if found
1204 *  -1 if NOT found
1205 */
1206int
1207apic_src_bus_irq(int apic, int pin)
1208{
1209	int     x;
1210
1211	for (x = 0; x < nintrs; x++)
1212		if ((apic == ID_TO_IO(io_apic_ints[x].dst_apic_id)) &&
1213		    (pin == io_apic_ints[x].dst_apic_int))
1214			return (io_apic_ints[x].src_bus_irq);
1215
1216	return -1;		/* NOT found */
1217}
1218
1219
1220/*
1221 * given a LOGICAL APIC# and pin#, return:
1222 *  the associated INTerrupt type if found
1223 *  -1 if NOT found
1224 */
1225int
1226apic_int_type(int apic, int pin)
1227{
1228	int     x;
1229
1230	/* search each of the possible INTerrupt sources */
1231	for (x = 0; x < nintrs; ++x)
1232		if ((apic == ID_TO_IO(io_apic_ints[x].dst_apic_id)) &&
1233		    (pin == io_apic_ints[x].dst_apic_int))
1234			return (io_apic_ints[x].int_type);
1235
1236	return -1;		/* NOT found */
1237}
1238
1239
1240/*
1241 * given a LOGICAL APIC# and pin#, return:
1242 *  the associated trigger mode if found
1243 *  -1 if NOT found
1244 */
1245int
1246apic_trigger(int apic, int pin)
1247{
1248	int     x;
1249
1250	/* search each of the possible INTerrupt sources */
1251	for (x = 0; x < nintrs; ++x)
1252		if ((apic == ID_TO_IO(io_apic_ints[x].dst_apic_id)) &&
1253		    (pin == io_apic_ints[x].dst_apic_int))
1254			return ((io_apic_ints[x].int_flags >> 2) & 0x03);
1255
1256	return -1;		/* NOT found */
1257}
1258
1259
1260/*
1261 * given a LOGICAL APIC# and pin#, return:
1262 *  the associated 'active' level if found
1263 *  -1 if NOT found
1264 */
1265int
1266apic_polarity(int apic, int pin)
1267{
1268	int     x;
1269
1270	/* search each of the possible INTerrupt sources */
1271	for (x = 0; x < nintrs; ++x)
1272		if ((apic == ID_TO_IO(io_apic_ints[x].dst_apic_id)) &&
1273		    (pin == io_apic_ints[x].dst_apic_int))
1274			return (io_apic_ints[x].int_flags & 0x03);
1275
1276	return -1;		/* NOT found */
1277}
1278
1279
1280/*
1281 * set data according to MP defaults
1282 * FIXME: probably not complete yet...
1283 */
1284static void
1285default_mp_table(int type)
1286{
1287	int     ap_cpu_id;
1288#if defined(APIC_IO)
1289	u_int32_t ux;
1290	int     io_apic_id;
1291	int     pin;
1292#endif	/* APIC_IO */
1293
1294#if 0
1295	printf("  MP default config type: %d\n", type);
1296	switch (type) {
1297	case 1:
1298		printf("   bus: ISA, APIC: 82489DX\n");
1299		break;
1300	case 2:
1301		printf("   bus: EISA, APIC: 82489DX\n");
1302		break;
1303	case 3:
1304		printf("   bus: EISA, APIC: 82489DX\n");
1305		break;
1306	case 4:
1307		printf("   bus: MCA, APIC: 82489DX\n");
1308		break;
1309	case 5:
1310		printf("   bus: ISA+PCI, APIC: Integrated\n");
1311		break;
1312	case 6:
1313		printf("   bus: EISA+PCI, APIC: Integrated\n");
1314		break;
1315	case 7:
1316		printf("   bus: MCA+PCI, APIC: Integrated\n");
1317		break;
1318	default:
1319		printf("   future type\n");
1320		break;
1321		/* NOTREACHED */
1322	}
1323#endif	/* 0 */
1324
1325	boot_cpu_id = (lapic.id & APIC_ID_MASK) >> 24;
1326	ap_cpu_id = (boot_cpu_id == 0) ? 1 : 0;
1327
1328	/* BSP */
1329	CPU_TO_ID(0) = boot_cpu_id;
1330	ID_TO_CPU(boot_cpu_id) = 0;
1331
1332	/* one and only AP */
1333	CPU_TO_ID(1) = ap_cpu_id;
1334	ID_TO_CPU(ap_cpu_id) = 1;
1335
1336#if defined(APIC_IO)
1337	/* one and only IO APIC */
1338	io_apic_id = (io_apic_read(0, IOAPIC_ID) & APIC_ID_MASK) >> 24;
1339
1340	/*
1341	 * sanity check, refer to MP spec section 3.6.6, last paragraph
1342	 * necessary as some hardware isn't properly setting up the IO APIC
1343	 */
1344#if defined(REALLY_ANAL_IOAPICID_VALUE)
1345	if (io_apic_id != 2) {
1346#else
1347	if ((io_apic_id == 0) || (io_apic_id == 1) || (io_apic_id == 15)) {
1348#endif	/* REALLY_ANAL_IOAPICID_VALUE */
1349		ux = io_apic_read(0, IOAPIC_ID);	/* get current contents */
1350		ux &= ~APIC_ID_MASK;	/* clear the ID field */
1351		ux |= 0x02000000;	/* set it to '2' */
1352		io_apic_write(0, IOAPIC_ID, ux);	/* write new value */
1353		ux = io_apic_read(0, IOAPIC_ID);	/* re-read && test */
1354		if ((ux & APIC_ID_MASK) != 0x02000000)
1355			panic("can't control IO APIC ID, reg: 0x%08x", ux);
1356		io_apic_id = 2;
1357	}
1358	IO_TO_ID(0) = io_apic_id;
1359	ID_TO_IO(io_apic_id) = 0;
1360#endif	/* APIC_IO */
1361
1362	/* fill out bus entries */
1363	switch (type) {
1364	case 1:
1365	case 2:
1366	case 3:
1367	case 5:
1368	case 6:
1369		bus_data[0].bus_id = default_data[type - 1][1];
1370		bus_data[0].bus_type = default_data[type - 1][2];
1371		bus_data[1].bus_id = default_data[type - 1][3];
1372		bus_data[1].bus_type = default_data[type - 1][4];
1373		break;
1374
1375	/* case 4: case 7:		   MCA NOT supported */
1376	default:		/* illegal/reserved */
1377		panic("BAD default MP config: %d", type);
1378		/* NOTREACHED */
1379	}
1380
1381#if defined(APIC_IO)
1382	/* general cases from MP v1.4, table 5-2 */
1383	for (pin = 0; pin < 16; ++pin) {
1384		io_apic_ints[pin].int_type = 0;
1385		io_apic_ints[pin].int_flags = 0x05;	/* edge/active-hi */
1386		io_apic_ints[pin].src_bus_id = 0;
1387		io_apic_ints[pin].src_bus_irq = pin;	/* IRQ2 caught below */
1388		io_apic_ints[pin].dst_apic_id = io_apic_id;
1389		io_apic_ints[pin].dst_apic_int = pin;	/* 1-to-1 */
1390	}
1391
1392	/* special cases from MP v1.4, table 5-2 */
1393	if (type == 2) {
1394		io_apic_ints[2].int_type = 0xff;	/* N/C */
1395		io_apic_ints[13].int_type = 0xff;	/* N/C */
1396#if !defined(APIC_MIXED_MODE)
1397		/** FIXME: ??? */
1398		panic("sorry, can't support type 2 default yet");
1399#endif	/* APIC_MIXED_MODE */
1400	}
1401	else
1402		io_apic_ints[2].src_bus_irq = 0;	/* ISA IRQ0 is on APIC INT 2 */
1403
1404	if (type == 7)
1405		io_apic_ints[0].int_type = 0xff;	/* N/C */
1406	else
1407		io_apic_ints[0].int_type = 3;	/* vectored 8259 */
1408#endif	/* APIC_IO */
1409}
1410
1411
1412/*
1413 * initialize all the SMP locks
1414 */
1415static void
1416init_locks(void)
1417{
1418	/*
1419	 * Get the initial mp_lock with a count of 1 for the BSP.
1420	 * This uses a LOGICAL cpu ID, ie BSP == 0.
1421	 */
1422	mp_lock = 0x00000001;
1423
1424	/* locks the IO APIC and apic_imen accesses */
1425	s_lock_init((struct simplelock*)&imen_lock);
1426}
1427
1428
1429/*
1430 * start each AP in our list
1431 */
1432static int
1433start_all_aps(u_int boot_addr)
1434{
1435	int     x, i;
1436	u_char  mpbiosreason;
1437	u_long  mpbioswarmvec;
1438	pd_entry_t newptd;
1439	pt_entry_t newpt;
1440	int *newpp;
1441
1442	POSTCODE(START_ALL_APS_POST);
1443
1444	/* initialize BSP's local APIC */
1445	apic_initialize();
1446
1447	/* install the AP 1st level boot code */
1448	install_ap_tramp(boot_addr);
1449
1450
1451	/* save the current value of the warm-start vector */
1452	mpbioswarmvec = *((u_long *) WARMBOOT_OFF);
1453	outb(CMOS_REG, BIOS_RESET);
1454	mpbiosreason = inb(CMOS_DATA);
1455
1456	/* record BSP in CPU map */
1457	all_cpus = 1;
1458
1459	/* start each AP */
1460	for (x = 1; x <= mp_naps; ++x) {
1461
1462		/* HACK HACK HACK !!! */
1463
1464		/* alloc new page table directory */
1465		newptd = (pd_entry_t)(kmem_alloc(kernel_map, PAGE_SIZE));
1466
1467		/* clone currently active one (ie: IdlePTD) */
1468		bcopy(PTD, newptd, PAGE_SIZE);	/* inc prv page pde */
1469
1470		/* set up 0 -> 4MB P==V mapping for AP boot */
1471		newptd[0] = PG_V | PG_RW | ((u_long)KPTphys & PG_FRAME);
1472
1473		/* store PTD for this AP */
1474		bootPTD = (pd_entry_t)vtophys(newptd);
1475
1476		/* alloc new page table page */
1477		newpt = (pt_entry_t)(kmem_alloc(kernel_map, PAGE_SIZE));
1478
1479		/* set the new PTD's private page to point there */
1480		newptd[MPPTDI] = PG_V | PG_RW | vtophys(newpt);
1481
1482		/* install self referential entry */
1483		newptd[PTDPTDI] = PG_V | PG_RW | vtophys(newptd);
1484
1485		/* get a new private data page */
1486		newpp = (int *)kmem_alloc(kernel_map, PAGE_SIZE);
1487
1488		/* wire it into the private page table page */
1489		newpt[0] = PG_V | PG_RW | vtophys(newpp);
1490
1491		/* wire the ptp into itself for access */
1492		newpt[1] = PG_V | PG_RW | vtophys(newpt);
1493
1494		/* and the local apic */
1495		newpt[2] = SMP_prvpt[2];
1496
1497		/* and the IO apic mapping[s] */
1498		for (i = 16; i < 32; i++)
1499			newpt[i] = SMP_prvpt[i];
1500
1501		/* prime data page for it to use */
1502		newpp[0] = x;		/* cpuid */
1503		newpp[1] = 0;		/* curproc */
1504		newpp[2] = 0;		/* curpcb */
1505		newpp[3] = 0;		/* npxproc */
1506		newpp[4] = 0;		/* runtime.tv_sec */
1507		newpp[5] = 0;		/* runtime.tv_usec */
1508		newpp[6] = x << 24;	/* cpu_lockid */
1509
1510		/* XXX NOTE: ABANDON bootPTD for now!!!! */
1511
1512		/* END REVOLTING HACKERY */
1513
1514		/* setup a vector to our boot code */
1515		*((volatile u_short *) WARMBOOT_OFF) = WARMBOOT_TARGET;
1516		*((volatile u_short *) WARMBOOT_SEG) = (boot_addr >> 4);
1517		outb(CMOS_REG, BIOS_RESET);
1518		outb(CMOS_DATA, BIOS_WARM);	/* 'warm-start' */
1519
1520		/* attempt to start the Application Processor */
1521		CHECK_INIT(99);	/* setup checkpoints */
1522		if (!start_ap(x, boot_addr)) {
1523			printf("AP #%d (PHY# %d) failed!\n", x, CPU_TO_ID(x));
1524			CHECK_PRINT("trace");	/* show checkpoints */
1525			/* better panic as the AP may be running loose */
1526			printf("panic y/n? [y] ");
1527			if (cngetc() != 'n')
1528				panic("bye-bye");
1529		}
1530		CHECK_PRINT("trace");		/* show checkpoints */
1531
1532		/* record its version info */
1533		cpu_apic_versions[x] = cpu_apic_versions[0];
1534
1535		all_cpus |= (1 << x);		/* record AP in CPU map */
1536	}
1537
1538	/* build our map of 'other' CPUs */
1539	other_cpus = all_cpus & ~(1 << cpuid);
1540
1541	/* fill in our (BSP) APIC version */
1542	cpu_apic_versions[0] = lapic.version;
1543
1544	/* restore the warmstart vector */
1545	*(u_long *) WARMBOOT_OFF = mpbioswarmvec;
1546	outb(CMOS_REG, BIOS_RESET);
1547	outb(CMOS_DATA, mpbiosreason);
1548
1549	pmap_set_opt_bsp();
1550
1551	/* number of APs actually started */
1552	return mp_ncpus - 1;
1553}
1554
1555
1556/*
1557 * load the 1st level AP boot code into base memory.
1558 */
1559
1560/* targets for relocation */
1561extern void bigJump(void);
1562extern void bootCodeSeg(void);
1563extern void bootDataSeg(void);
1564extern void MPentry(void);
1565extern u_int MP_GDT;
1566extern u_int mp_gdtbase;
1567
1568static void
1569install_ap_tramp(u_int boot_addr)
1570{
1571	int     x;
1572	int     size = *(int *) ((u_long) & bootMP_size);
1573	u_char *src = (u_char *) ((u_long) bootMP);
1574	u_char *dst = (u_char *) boot_addr + KERNBASE;
1575	u_int   boot_base = (u_int) bootMP;
1576	u_int8_t *dst8;
1577	u_int16_t *dst16;
1578	u_int32_t *dst32;
1579
1580	POSTCODE(INSTALL_AP_TRAMP_POST);
1581
1582	for (x = 0; x < size; ++x)
1583		*dst++ = *src++;
1584
1585	/*
1586	 * modify addresses in code we just moved to basemem. unfortunately we
1587	 * need fairly detailed info about mpboot.s for this to work.  changes
1588	 * to mpboot.s might require changes here.
1589	 */
1590
1591	/* boot code is located in KERNEL space */
1592	dst = (u_char *) boot_addr + KERNBASE;
1593
1594	/* modify the lgdt arg */
1595	dst32 = (u_int32_t *) (dst + ((u_int) & mp_gdtbase - boot_base));
1596	*dst32 = boot_addr + ((u_int) & MP_GDT - boot_base);
1597
1598	/* modify the ljmp target for MPentry() */
1599	dst32 = (u_int32_t *) (dst + ((u_int) bigJump - boot_base) + 1);
1600	*dst32 = ((u_int) MPentry - KERNBASE);
1601
1602	/* modify the target for boot code segment */
1603	dst16 = (u_int16_t *) (dst + ((u_int) bootCodeSeg - boot_base));
1604	dst8 = (u_int8_t *) (dst16 + 1);
1605	*dst16 = (u_int) boot_addr & 0xffff;
1606	*dst8 = ((u_int) boot_addr >> 16) & 0xff;
1607
1608	/* modify the target for boot data segment */
1609	dst16 = (u_int16_t *) (dst + ((u_int) bootDataSeg - boot_base));
1610	dst8 = (u_int8_t *) (dst16 + 1);
1611	*dst16 = (u_int) boot_addr & 0xffff;
1612	*dst8 = ((u_int) boot_addr >> 16) & 0xff;
1613}
1614
1615
1616/*
1617 * this function starts the AP (application processor) identified
1618 * by the APIC ID 'physicalCpu'.  It does quite a "song and dance"
1619 * to accomplish this.  This is necessary because of the nuances
1620 * of the different hardware we might encounter.  It ain't pretty,
1621 * but it seems to work.
1622 */
1623static int
1624start_ap(int logical_cpu, u_int boot_addr)
1625{
1626	int     physical_cpu;
1627	int     vector;
1628	int     cpus;
1629	u_long  icr_lo, icr_hi;
1630
1631	POSTCODE(START_AP_POST);
1632
1633	/* get the PHYSICAL APIC ID# */
1634	physical_cpu = CPU_TO_ID(logical_cpu);
1635
1636	/* calculate the vector */
1637	vector = (boot_addr >> 12) & 0xff;
1638
1639	/* used as a watchpoint to signal AP startup */
1640	cpus = mp_ncpus;
1641
1642	/*
1643	 * first we do an INIT/RESET IPI this INIT IPI might be run, reseting
1644	 * and running the target CPU. OR this INIT IPI might be latched (P5
1645	 * bug), CPU waiting for STARTUP IPI. OR this INIT IPI might be
1646	 * ignored.
1647	 */
1648
1649	/* setup the address for the target AP */
1650	icr_hi = lapic.icr_hi & ~APIC_ID_MASK;
1651	icr_hi |= (physical_cpu << 24);
1652	lapic.icr_hi = icr_hi;
1653
1654	/* do an INIT IPI: assert RESET */
1655	icr_lo = lapic.icr_lo & 0xfff00000;
1656	lapic.icr_lo = icr_lo | 0x0000c500;
1657
1658	/* wait for pending status end */
1659	while (lapic.icr_lo & APIC_DELSTAT_MASK)
1660		 /* spin */ ;
1661
1662	/* do an INIT IPI: deassert RESET */
1663	lapic.icr_lo = icr_lo | 0x00008500;
1664
1665	/* wait for pending status end */
1666	u_sleep(10000);		/* wait ~10mS */
1667	while (lapic.icr_lo & APIC_DELSTAT_MASK)
1668		 /* spin */ ;
1669
1670	/*
1671	 * next we do a STARTUP IPI: the previous INIT IPI might still be
1672	 * latched, (P5 bug) this 1st STARTUP would then terminate
1673	 * immediately, and the previously started INIT IPI would continue. OR
1674	 * the previous INIT IPI has already run. and this STARTUP IPI will
1675	 * run. OR the previous INIT IPI was ignored. and this STARTUP IPI
1676	 * will run.
1677	 */
1678
1679	/* do a STARTUP IPI */
1680	lapic.icr_lo = icr_lo | 0x00000600 | vector;
1681	while (lapic.icr_lo & APIC_DELSTAT_MASK)
1682		 /* spin */ ;
1683	u_sleep(200);		/* wait ~200uS */
1684
1685	/*
1686	 * finally we do a 2nd STARTUP IPI: this 2nd STARTUP IPI should run IF
1687	 * the previous STARTUP IPI was cancelled by a latched INIT IPI. OR
1688	 * this STARTUP IPI will be ignored, as only ONE STARTUP IPI is
1689	 * recognized after hardware RESET or INIT IPI.
1690	 */
1691
1692	lapic.icr_lo = icr_lo | 0x00000600 | vector;
1693	while (lapic.icr_lo & APIC_DELSTAT_MASK)
1694		 /* spin */ ;
1695	u_sleep(200);		/* wait ~200uS */
1696
1697	/* wait for it to start */
1698	set_apic_timer(5000000);/* == 5 seconds */
1699	while (read_apic_timer())
1700		if (mp_ncpus > cpus)
1701			return 1;	/* return SUCCESS */
1702
1703	return 0;		/* return FAILURE */
1704}
1705
1706
1707/*
1708 * Flush the TLB on all other CPU's
1709 *
1710 * XXX: Needs to handshake and wait for completion before proceding.
1711 */
1712void
1713smp_invltlb(void)
1714{
1715#if defined(APIC_IO)
1716	if (smp_active && invltlb_ok)
1717		all_but_self_ipi(XINVLTLB_OFFSET);
1718#endif  /* APIC_IO */
1719}
1720
1721void
1722invlpg(u_int addr)
1723{
1724	__asm   __volatile("invlpg (%0)"::"r"(addr):"memory");
1725
1726	/* send a message to the other CPUs */
1727	smp_invltlb();
1728}
1729
1730void
1731invltlb(void)
1732{
1733	u_long  temp;
1734
1735	/*
1736	 * This should be implemented as load_cr3(rcr3()) when load_cr3() is
1737	 * inlined.
1738	 */
1739	__asm __volatile("movl %%cr3, %0; movl %0, %%cr3":"=r"(temp) :: "memory");
1740
1741	/* send a message to the other CPUs */
1742	smp_invltlb();
1743}
1744
1745
1746/*
1747 * When called the executing CPU will send an IPI to all other CPUs
1748 *  requesting that they halt execution.
1749 *
1750 * Usually (but not necessarily) called with 'other_cpus' as its arg.
1751 *
1752 *  - Signals all CPUs in map to stop.
1753 *  - Waits for each to stop.
1754 *
1755 * Returns:
1756 *  -1: error
1757 *   0: NA
1758 *   1: ok
1759 *
1760 * XXX FIXME: this is not MP-safe, needs a lock to prevent multiple CPUs
1761 *            from executing at same time.
1762 */
1763int
1764stop_cpus( u_int map )
1765{
1766	if (!smp_active)
1767		return 0;
1768
1769	/* send IPI to all CPUs in map */
1770	stopped_cpus = 0;
1771
1772	/* send the Xcpustop IPI to all CPUs in map */
1773	selected_apic_ipi(map, XCPUSTOP_OFFSET, APIC_DELMODE_FIXED);
1774
1775	while (stopped_cpus != map)
1776		/* spin */ ;
1777
1778	return 1;
1779}
1780
1781
1782/*
1783 * Called by a CPU to restart stopped CPUs.
1784 *
1785 * Usually (but not necessarily) called with 'stopped_cpus' as its arg.
1786 *
1787 *  - Signals all CPUs in map to restart.
1788 *  - Waits for each to restart.
1789 *
1790 * Returns:
1791 *  -1: error
1792 *   0: NA
1793 *   1: ok
1794 */
1795int
1796restart_cpus( u_int map )
1797{
1798	if (!smp_active)
1799		return 0;
1800
1801	started_cpus = map;		/* signal other cpus to restart */
1802
1803	while (started_cpus)		/* wait for each to clear its bit */
1804		/* spin */ ;
1805
1806	return 1;
1807}
1808