mp_x86.c revision 65597
1/*
2 * Copyright (c) 1996, by Steve Passe
3 * All rights reserved.
4 *
5 * Redistribution and use in source and binary forms, with or without
6 * modification, are permitted provided that the following conditions
7 * are met:
8 * 1. Redistributions of source code must retain the above copyright
9 *    notice, this list of conditions and the following disclaimer.
10 * 2. The name of the developer may NOT be used to endorse or promote products
11 *    derived from this software without specific prior written permission.
12 *
13 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
14 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
15 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
16 * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
17 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
18 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
19 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
20 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
21 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
22 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
23 * SUCH DAMAGE.
24 *
25 * $FreeBSD: head/sys/i386/i386/mp_machdep.c 65597 2000-09-08 03:36:09Z jake $
26 */
27
28#include "opt_smp.h"
29#include "opt_cpu.h"
30#include "opt_user_ldt.h"
31
32#ifdef SMP
33#include <machine/smptests.h>
34#else
35#error
36#endif
37
38#include <sys/param.h>
39#include <sys/bus.h>
40#include <sys/systm.h>
41#include <sys/kernel.h>
42#include <sys/proc.h>
43#include <sys/sysctl.h>
44#include <sys/malloc.h>
45#include <sys/memrange.h>
46#ifdef BETTER_CLOCK
47#include <sys/dkstat.h>
48#endif
49#include <sys/cons.h>	/* cngetc() */
50
51#include <vm/vm.h>
52#include <vm/vm_param.h>
53#include <vm/pmap.h>
54#include <vm/vm_kern.h>
55#include <vm/vm_extern.h>
56#ifdef BETTER_CLOCK
57#include <sys/lock.h>
58#include <vm/vm_map.h>
59#include <sys/user.h>
60#ifdef GPROF
61#include <sys/gmon.h>
62#endif
63#endif
64
65#include <machine/smp.h>
66#include <machine/apic.h>
67#include <machine/atomic.h>
68#include <machine/cpufunc.h>
69#include <machine/mutex.h>
70#include <machine/mpapic.h>
71#include <machine/psl.h>
72#include <machine/segments.h>
73#include <machine/smptests.h>	/** TEST_DEFAULT_CONFIG, TEST_TEST1 */
74#include <machine/tss.h>
75#include <machine/specialreg.h>
76#include <machine/globaldata.h>
77
78#if defined(APIC_IO)
79#include <machine/md_var.h>		/* setidt() */
80#include <i386/isa/icu.h>		/* IPIs */
81#include <i386/isa/intr_machdep.h>	/* IPIs */
82#endif	/* APIC_IO */
83
84#if defined(TEST_DEFAULT_CONFIG)
85#define MPFPS_MPFB1	TEST_DEFAULT_CONFIG
86#else
87#define MPFPS_MPFB1	mpfps->mpfb1
88#endif  /* TEST_DEFAULT_CONFIG */
89
90#define WARMBOOT_TARGET		0
91#define WARMBOOT_OFF		(KERNBASE + 0x0467)
92#define WARMBOOT_SEG		(KERNBASE + 0x0469)
93
94#ifdef PC98
95#define BIOS_BASE		(0xe8000)
96#define BIOS_SIZE		(0x18000)
97#else
98#define BIOS_BASE		(0xf0000)
99#define BIOS_SIZE		(0x10000)
100#endif
101#define BIOS_COUNT		(BIOS_SIZE/4)
102
103#define CMOS_REG		(0x70)
104#define CMOS_DATA		(0x71)
105#define BIOS_RESET		(0x0f)
106#define BIOS_WARM		(0x0a)
107
108#define PROCENTRY_FLAG_EN	0x01
109#define PROCENTRY_FLAG_BP	0x02
110#define IOAPICENTRY_FLAG_EN	0x01
111
112
113/* MP Floating Pointer Structure */
114typedef struct MPFPS {
115	char    signature[4];
116	void   *pap;
117	u_char  length;
118	u_char  spec_rev;
119	u_char  checksum;
120	u_char  mpfb1;
121	u_char  mpfb2;
122	u_char  mpfb3;
123	u_char  mpfb4;
124	u_char  mpfb5;
125}      *mpfps_t;
126
127/* MP Configuration Table Header */
128typedef struct MPCTH {
129	char    signature[4];
130	u_short base_table_length;
131	u_char  spec_rev;
132	u_char  checksum;
133	u_char  oem_id[8];
134	u_char  product_id[12];
135	void   *oem_table_pointer;
136	u_short oem_table_size;
137	u_short entry_count;
138	void   *apic_address;
139	u_short extended_table_length;
140	u_char  extended_table_checksum;
141	u_char  reserved;
142}      *mpcth_t;
143
144
145typedef struct PROCENTRY {
146	u_char  type;
147	u_char  apic_id;
148	u_char  apic_version;
149	u_char  cpu_flags;
150	u_long  cpu_signature;
151	u_long  feature_flags;
152	u_long  reserved1;
153	u_long  reserved2;
154}      *proc_entry_ptr;
155
156typedef struct BUSENTRY {
157	u_char  type;
158	u_char  bus_id;
159	char    bus_type[6];
160}      *bus_entry_ptr;
161
162typedef struct IOAPICENTRY {
163	u_char  type;
164	u_char  apic_id;
165	u_char  apic_version;
166	u_char  apic_flags;
167	void   *apic_address;
168}      *io_apic_entry_ptr;
169
170typedef struct INTENTRY {
171	u_char  type;
172	u_char  int_type;
173	u_short int_flags;
174	u_char  src_bus_id;
175	u_char  src_bus_irq;
176	u_char  dst_apic_id;
177	u_char  dst_apic_int;
178}      *int_entry_ptr;
179
180/* descriptions of MP basetable entries */
181typedef struct BASETABLE_ENTRY {
182	u_char  type;
183	u_char  length;
184	char    name[16];
185}       basetable_entry;
186
187/*
188 * this code MUST be enabled here and in mpboot.s.
189 * it follows the very early stages of AP boot by placing values in CMOS ram.
190 * it NORMALLY will never be needed and thus the primitive method for enabling.
191 *
192#define CHECK_POINTS
193 */
194
195#if defined(CHECK_POINTS) && !defined(PC98)
196#define CHECK_READ(A)	 (outb(CMOS_REG, (A)), inb(CMOS_DATA))
197#define CHECK_WRITE(A,D) (outb(CMOS_REG, (A)), outb(CMOS_DATA, (D)))
198
199#define CHECK_INIT(D);				\
200	CHECK_WRITE(0x34, (D));			\
201	CHECK_WRITE(0x35, (D));			\
202	CHECK_WRITE(0x36, (D));			\
203	CHECK_WRITE(0x37, (D));			\
204	CHECK_WRITE(0x38, (D));			\
205	CHECK_WRITE(0x39, (D));
206
207#define CHECK_PRINT(S);				\
208	printf("%s: %d, %d, %d, %d, %d, %d\n",	\
209	   (S),					\
210	   CHECK_READ(0x34),			\
211	   CHECK_READ(0x35),			\
212	   CHECK_READ(0x36),			\
213	   CHECK_READ(0x37),			\
214	   CHECK_READ(0x38),			\
215	   CHECK_READ(0x39));
216
217#else				/* CHECK_POINTS */
218
219#define CHECK_INIT(D)
220#define CHECK_PRINT(S)
221
222#endif				/* CHECK_POINTS */
223
224/*
225 * Values to send to the POST hardware.
226 */
227#define MP_BOOTADDRESS_POST	0x10
228#define MP_PROBE_POST		0x11
229#define MPTABLE_PASS1_POST	0x12
230
231#define MP_START_POST		0x13
232#define MP_ENABLE_POST		0x14
233#define MPTABLE_PASS2_POST	0x15
234
235#define START_ALL_APS_POST	0x16
236#define INSTALL_AP_TRAMP_POST	0x17
237#define START_AP_POST		0x18
238
239#define MP_ANNOUNCE_POST	0x19
240
241/* used to hold the AP's until we are ready to release them */
242struct simplelock	ap_boot_lock;
243
244/** XXX FIXME: where does this really belong, isa.h/isa.c perhaps? */
245int	current_postcode;
246
247/** XXX FIXME: what system files declare these??? */
248extern struct region_descriptor r_gdt, r_idt;
249
250int	bsp_apic_ready = 0;	/* flags useability of BSP apic */
251int	mp_ncpus;		/* # of CPUs, including BSP */
252int	mp_naps;		/* # of Applications processors */
253int	mp_nbusses;		/* # of busses */
254int	mp_napics;		/* # of IO APICs */
255int	boot_cpu_id;		/* designated BSP */
256vm_offset_t cpu_apic_address;
257vm_offset_t io_apic_address[NAPICID];	/* NAPICID is more than enough */
258extern	int nkpt;
259
260u_int32_t cpu_apic_versions[NCPU];
261u_int32_t io_apic_versions[NAPIC];
262
263#ifdef APIC_INTR_DIAGNOSTIC
264int apic_itrace_enter[32];
265int apic_itrace_tryisrlock[32];
266int apic_itrace_gotisrlock[32];
267int apic_itrace_active[32];
268int apic_itrace_masked[32];
269int apic_itrace_noisrlock[32];
270int apic_itrace_masked2[32];
271int apic_itrace_unmask[32];
272int apic_itrace_noforward[32];
273int apic_itrace_leave[32];
274int apic_itrace_enter2[32];
275int apic_itrace_doreti[32];
276int apic_itrace_splz[32];
277int apic_itrace_eoi[32];
278#ifdef APIC_INTR_DIAGNOSTIC_IRQ
279unsigned short apic_itrace_debugbuffer[32768];
280int apic_itrace_debugbuffer_idx;
281struct simplelock apic_itrace_debuglock;
282#endif
283#endif
284
285#ifdef APIC_INTR_REORDER
286struct {
287	volatile int *location;
288	int bit;
289} apic_isrbit_location[32];
290#endif
291
292struct apic_intmapinfo	int_to_apicintpin[APIC_INTMAPSIZE];
293
294/*
295 * APIC ID logical/physical mapping structures.
296 * We oversize these to simplify boot-time config.
297 */
298int     cpu_num_to_apic_id[NAPICID];
299int     io_num_to_apic_id[NAPICID];
300int     apic_id_to_logical[NAPICID];
301
302
303/* Bitmap of all available CPUs */
304u_int	all_cpus;
305
306/* AP uses this during bootstrap.  Do not staticize.  */
307char *bootSTK;
308static int bootAP;
309
310/* Hotwire a 0->4MB V==P mapping */
311extern pt_entry_t *KPTphys;
312
313/* SMP page table page */
314extern pt_entry_t *SMPpt;
315
316struct pcb stoppcbs[NCPU];
317
318int smp_started;		/* has the system started? */
319
320/*
321 * Local data and functions.
322 */
323
324static int	mp_capable;
325static u_int	boot_address;
326static u_int	base_memory;
327
328static int	picmode;		/* 0: virtual wire mode, 1: PIC mode */
329static mpfps_t	mpfps;
330static int	search_for_sig(u_int32_t target, int count);
331static void	mp_enable(u_int boot_addr);
332
333static int	mptable_pass1(void);
334static int	mptable_pass2(void);
335static void	default_mp_table(int type);
336static void	fix_mp_table(void);
337static void	setup_apic_irq_mapping(void);
338static void	init_locks(void);
339static int	start_all_aps(u_int boot_addr);
340static void	install_ap_tramp(u_int boot_addr);
341static int	start_ap(int logicalCpu, u_int boot_addr);
342static int	apic_int_is_bus_type(int intr, int bus_type);
343static void	release_aps(void *dummy);
344
345/*
346 * Calculate usable address in base memory for AP trampoline code.
347 */
348u_int
349mp_bootaddress(u_int basemem)
350{
351	POSTCODE(MP_BOOTADDRESS_POST);
352
353	base_memory = basemem * 1024;	/* convert to bytes */
354
355	boot_address = base_memory & ~0xfff;	/* round down to 4k boundary */
356	if ((base_memory - boot_address) < bootMP_size)
357		boot_address -= 4096;	/* not enough, lower by 4k */
358
359	return boot_address;
360}
361
362
363/*
364 * Look for an Intel MP spec table (ie, SMP capable hardware).
365 */
366int
367mp_probe(void)
368{
369	int     x;
370	u_long  segment;
371	u_int32_t target;
372
373	POSTCODE(MP_PROBE_POST);
374
375	/* see if EBDA exists */
376	if ((segment = (u_long) * (u_short *) (KERNBASE + 0x40e)) != 0) {
377		/* search first 1K of EBDA */
378		target = (u_int32_t) (segment << 4);
379		if ((x = search_for_sig(target, 1024 / 4)) >= 0)
380			goto found;
381	} else {
382		/* last 1K of base memory, effective 'top of base' passed in */
383		target = (u_int32_t) (base_memory - 0x400);
384		if ((x = search_for_sig(target, 1024 / 4)) >= 0)
385			goto found;
386	}
387
388	/* search the BIOS */
389	target = (u_int32_t) BIOS_BASE;
390	if ((x = search_for_sig(target, BIOS_COUNT)) >= 0)
391		goto found;
392
393	/* nothing found */
394	mpfps = (mpfps_t)0;
395	mp_capable = 0;
396	return 0;
397
398found:
399	/* calculate needed resources */
400	mpfps = (mpfps_t)x;
401	if (mptable_pass1())
402		panic("you must reconfigure your kernel");
403
404	/* flag fact that we are running multiple processors */
405	mp_capable = 1;
406	return 1;
407}
408
409
410/*
411 * Initialize the SMP hardware and the APIC and start up the AP's.
412 */
413void
414mp_start(void)
415{
416	POSTCODE(MP_START_POST);
417
418	/* look for MP capable motherboard */
419	if (mp_capable)
420		mp_enable(boot_address);
421	else
422		panic("MP hardware not found!");
423}
424
425
426/*
427 * Print various information about the SMP system hardware and setup.
428 */
429void
430mp_announce(void)
431{
432	int     x;
433
434	POSTCODE(MP_ANNOUNCE_POST);
435
436	printf("FreeBSD/SMP: Multiprocessor motherboard\n");
437	printf(" cpu0 (BSP): apic id: %2d", CPU_TO_ID(0));
438	printf(", version: 0x%08x", cpu_apic_versions[0]);
439	printf(", at 0x%08x\n", cpu_apic_address);
440	for (x = 1; x <= mp_naps; ++x) {
441		printf(" cpu%d (AP):  apic id: %2d", x, CPU_TO_ID(x));
442		printf(", version: 0x%08x", cpu_apic_versions[x]);
443		printf(", at 0x%08x\n", cpu_apic_address);
444	}
445
446#if defined(APIC_IO)
447	for (x = 0; x < mp_napics; ++x) {
448		printf(" io%d (APIC): apic id: %2d", x, IO_TO_ID(x));
449		printf(", version: 0x%08x", io_apic_versions[x]);
450		printf(", at 0x%08x\n", io_apic_address[x]);
451	}
452#else
453	printf(" Warning: APIC I/O disabled\n");
454#endif	/* APIC_IO */
455}
456
457/*
458 * AP cpu's call this to sync up protected mode.
459 */
460void
461init_secondary(void)
462{
463	int	gsel_tss;
464	int	x, myid = bootAP;
465
466	gdt_segs[GPRIV_SEL].ssd_base = (int) &SMP_prvspace[myid];
467	gdt_segs[GPROC0_SEL].ssd_base =
468		(int) &SMP_prvspace[myid].globaldata.gd_common_tss;
469	SMP_prvspace[myid].globaldata.gd_prvspace = &SMP_prvspace[myid];
470
471	for (x = 0; x < NGDT; x++) {
472		ssdtosd(&gdt_segs[x], &gdt[myid * NGDT + x].sd);
473	}
474
475	r_gdt.rd_limit = NGDT * sizeof(gdt[0]) - 1;
476	r_gdt.rd_base = (int) &gdt[myid * NGDT];
477	lgdt(&r_gdt);			/* does magic intra-segment return */
478
479	lidt(&r_idt);
480
481	lldt(_default_ldt);
482#ifdef USER_LDT
483	PCPU_SET(currentldt, _default_ldt);
484#endif
485
486	gsel_tss = GSEL(GPROC0_SEL, SEL_KPL);
487	gdt[myid * NGDT + GPROC0_SEL].sd.sd_type = SDT_SYS386TSS;
488	common_tss.tss_esp0 = 0;	/* not used until after switch */
489	common_tss.tss_ss0 = GSEL(GDATA_SEL, SEL_KPL);
490	common_tss.tss_ioopt = (sizeof common_tss) << 16;
491	tss_gdt = &gdt[myid * NGDT + GPROC0_SEL].sd;
492	common_tssd = *tss_gdt;
493	ltr(gsel_tss);
494
495	pmap_set_opt();
496}
497
498
499#if defined(APIC_IO)
500/*
501 * Final configuration of the BSP's local APIC:
502 *  - disable 'pic mode'.
503 *  - disable 'virtual wire mode'.
504 *  - enable NMI.
505 */
506void
507bsp_apic_configure(void)
508{
509	u_char		byte;
510	u_int32_t	temp;
511
512	/* leave 'pic mode' if necessary */
513	if (picmode) {
514		outb(0x22, 0x70);	/* select IMCR */
515		byte = inb(0x23);	/* current contents */
516		byte |= 0x01;		/* mask external INTR */
517		outb(0x23, byte);	/* disconnect 8259s/NMI */
518	}
519
520	/* mask lint0 (the 8259 'virtual wire' connection) */
521	temp = lapic.lvt_lint0;
522	temp |= APIC_LVT_M;		/* set the mask */
523	lapic.lvt_lint0 = temp;
524
525        /* setup lint1 to handle NMI */
526        temp = lapic.lvt_lint1;
527        temp &= ~APIC_LVT_M;		/* clear the mask */
528        lapic.lvt_lint1 = temp;
529
530	if (bootverbose)
531		apic_dump("bsp_apic_configure()");
532}
533#endif  /* APIC_IO */
534
535
536/*******************************************************************
537 * local functions and data
538 */
539
540/*
541 * start the SMP system
542 */
543static void
544mp_enable(u_int boot_addr)
545{
546	int     x;
547#if defined(APIC_IO)
548	int     apic;
549	u_int   ux;
550#endif	/* APIC_IO */
551
552	POSTCODE(MP_ENABLE_POST);
553
554	/* turn on 4MB of V == P addressing so we can get to MP table */
555	*(int *)PTD = PG_V | PG_RW | ((uintptr_t)(void *)KPTphys & PG_FRAME);
556	invltlb();
557
558	/* examine the MP table for needed info, uses physical addresses */
559	x = mptable_pass2();
560
561	*(int *)PTD = 0;
562	invltlb();
563
564	/* can't process default configs till the CPU APIC is pmapped */
565	if (x)
566		default_mp_table(x);
567
568	/* post scan cleanup */
569	fix_mp_table();
570	setup_apic_irq_mapping();
571
572#if defined(APIC_IO)
573
574	/* fill the LOGICAL io_apic_versions table */
575	for (apic = 0; apic < mp_napics; ++apic) {
576		ux = io_apic_read(apic, IOAPIC_VER);
577		io_apic_versions[apic] = ux;
578		io_apic_set_id(apic, IO_TO_ID(apic));
579	}
580
581	/* program each IO APIC in the system */
582	for (apic = 0; apic < mp_napics; ++apic)
583		if (io_apic_setup(apic) < 0)
584			panic("IO APIC setup failure");
585
586	/* install a 'Spurious INTerrupt' vector */
587	setidt(XSPURIOUSINT_OFFSET, Xspuriousint,
588	       SDT_SYS386IGT, SEL_KPL, GSEL(GCODE_SEL, SEL_KPL));
589
590	/* install an inter-CPU IPI for TLB invalidation */
591	setidt(XINVLTLB_OFFSET, Xinvltlb,
592	       SDT_SYS386IGT, SEL_KPL, GSEL(GCODE_SEL, SEL_KPL));
593
594#ifdef BETTER_CLOCK
595	/* install an inter-CPU IPI for reading processor state */
596	setidt(XCPUCHECKSTATE_OFFSET, Xcpucheckstate,
597	       SDT_SYS386IGT, SEL_KPL, GSEL(GCODE_SEL, SEL_KPL));
598#endif
599
600	/* install an inter-CPU IPI for all-CPU rendezvous */
601	setidt(XRENDEZVOUS_OFFSET, Xrendezvous,
602	       SDT_SYS386IGT, SEL_KPL, GSEL(GCODE_SEL, SEL_KPL));
603
604	/* install an inter-CPU IPI for forcing an additional software trap */
605	setidt(XCPUAST_OFFSET, Xcpuast,
606	       SDT_SYS386IGT, SEL_KPL, GSEL(GCODE_SEL, SEL_KPL));
607
608	/* install an inter-CPU IPI for interrupt forwarding */
609	setidt(XFORWARD_IRQ_OFFSET, Xforward_irq,
610	       SDT_SYS386IGT, SEL_KPL, GSEL(GCODE_SEL, SEL_KPL));
611
612	/* install an inter-CPU IPI for CPU stop/restart */
613	setidt(XCPUSTOP_OFFSET, Xcpustop,
614	       SDT_SYS386IGT, SEL_KPL, GSEL(GCODE_SEL, SEL_KPL));
615
616#if defined(TEST_TEST1)
617	/* install a "fake hardware INTerrupt" vector */
618	setidt(XTEST1_OFFSET, Xtest1,
619	       SDT_SYS386IGT, SEL_KPL, GSEL(GCODE_SEL, SEL_KPL));
620#endif  /** TEST_TEST1 */
621
622#endif	/* APIC_IO */
623
624	/* initialize all SMP locks */
625	init_locks();
626
627	/* obtain the ap_boot_lock */
628	s_lock(&ap_boot_lock);
629
630	/* start each Application Processor */
631	start_all_aps(boot_addr);
632}
633
634
635/*
636 * look for the MP spec signature
637 */
638
639/* string defined by the Intel MP Spec as identifying the MP table */
640#define MP_SIG		0x5f504d5f	/* _MP_ */
641#define NEXT(X)		((X) += 4)
642static int
643search_for_sig(u_int32_t target, int count)
644{
645	int     x;
646	u_int32_t *addr = (u_int32_t *) (KERNBASE + target);
647
648	for (x = 0; x < count; NEXT(x))
649		if (addr[x] == MP_SIG)
650			/* make array index a byte index */
651			return (target + (x * sizeof(u_int32_t)));
652
653	return -1;
654}
655
656
657static basetable_entry basetable_entry_types[] =
658{
659	{0, 20, "Processor"},
660	{1, 8, "Bus"},
661	{2, 8, "I/O APIC"},
662	{3, 8, "I/O INT"},
663	{4, 8, "Local INT"}
664};
665
666typedef struct BUSDATA {
667	u_char  bus_id;
668	enum busTypes bus_type;
669}       bus_datum;
670
671typedef struct INTDATA {
672	u_char  int_type;
673	u_short int_flags;
674	u_char  src_bus_id;
675	u_char  src_bus_irq;
676	u_char  dst_apic_id;
677	u_char  dst_apic_int;
678	u_char	int_vector;
679}       io_int, local_int;
680
681typedef struct BUSTYPENAME {
682	u_char  type;
683	char    name[7];
684}       bus_type_name;
685
686static bus_type_name bus_type_table[] =
687{
688	{CBUS, "CBUS"},
689	{CBUSII, "CBUSII"},
690	{EISA, "EISA"},
691	{MCA, "MCA"},
692	{UNKNOWN_BUSTYPE, "---"},
693	{ISA, "ISA"},
694	{MCA, "MCA"},
695	{UNKNOWN_BUSTYPE, "---"},
696	{UNKNOWN_BUSTYPE, "---"},
697	{UNKNOWN_BUSTYPE, "---"},
698	{UNKNOWN_BUSTYPE, "---"},
699	{UNKNOWN_BUSTYPE, "---"},
700	{PCI, "PCI"},
701	{UNKNOWN_BUSTYPE, "---"},
702	{UNKNOWN_BUSTYPE, "---"},
703	{UNKNOWN_BUSTYPE, "---"},
704	{UNKNOWN_BUSTYPE, "---"},
705	{XPRESS, "XPRESS"},
706	{UNKNOWN_BUSTYPE, "---"}
707};
708/* from MP spec v1.4, table 5-1 */
709static int default_data[7][5] =
710{
711/*   nbus, id0, type0, id1, type1 */
712	{1, 0, ISA, 255, 255},
713	{1, 0, EISA, 255, 255},
714	{1, 0, EISA, 255, 255},
715	{1, 0, MCA, 255, 255},
716	{2, 0, ISA, 1, PCI},
717	{2, 0, EISA, 1, PCI},
718	{2, 0, MCA, 1, PCI}
719};
720
721
722/* the bus data */
723static bus_datum bus_data[NBUS];
724
725/* the IO INT data, one entry per possible APIC INTerrupt */
726static io_int  io_apic_ints[NINTR];
727
728static int nintrs;
729
730static int processor_entry	__P((proc_entry_ptr entry, int cpu));
731static int bus_entry		__P((bus_entry_ptr entry, int bus));
732static int io_apic_entry	__P((io_apic_entry_ptr entry, int apic));
733static int int_entry		__P((int_entry_ptr entry, int intr));
734static int lookup_bus_type	__P((char *name));
735
736
737/*
738 * 1st pass on motherboard's Intel MP specification table.
739 *
740 * initializes:
741 *	mp_ncpus = 1
742 *
743 * determines:
744 *	cpu_apic_address (common to all CPUs)
745 *	io_apic_address[N]
746 *	mp_naps
747 *	mp_nbusses
748 *	mp_napics
749 *	nintrs
750 */
751static int
752mptable_pass1(void)
753{
754	int	x;
755	mpcth_t	cth;
756	int	totalSize;
757	void*	position;
758	int	count;
759	int	type;
760	int	mustpanic;
761
762	POSTCODE(MPTABLE_PASS1_POST);
763
764	mustpanic = 0;
765
766	/* clear various tables */
767	for (x = 0; x < NAPICID; ++x) {
768		io_apic_address[x] = ~0;	/* IO APIC address table */
769	}
770
771	/* init everything to empty */
772	mp_naps = 0;
773	mp_nbusses = 0;
774	mp_napics = 0;
775	nintrs = 0;
776
777	/* check for use of 'default' configuration */
778	if (MPFPS_MPFB1 != 0) {
779		/* use default addresses */
780		cpu_apic_address = DEFAULT_APIC_BASE;
781		io_apic_address[0] = DEFAULT_IO_APIC_BASE;
782
783		/* fill in with defaults */
784		mp_naps = 2;		/* includes BSP */
785		mp_nbusses = default_data[MPFPS_MPFB1 - 1][0];
786#if defined(APIC_IO)
787		mp_napics = 1;
788		nintrs = 16;
789#endif	/* APIC_IO */
790	}
791	else {
792		if ((cth = mpfps->pap) == 0)
793			panic("MP Configuration Table Header MISSING!");
794
795		cpu_apic_address = (vm_offset_t) cth->apic_address;
796
797		/* walk the table, recording info of interest */
798		totalSize = cth->base_table_length - sizeof(struct MPCTH);
799		position = (u_char *) cth + sizeof(struct MPCTH);
800		count = cth->entry_count;
801
802		while (count--) {
803			switch (type = *(u_char *) position) {
804			case 0: /* processor_entry */
805				if (((proc_entry_ptr)position)->cpu_flags
806					& PROCENTRY_FLAG_EN)
807					++mp_naps;
808				break;
809			case 1: /* bus_entry */
810				++mp_nbusses;
811				break;
812			case 2: /* io_apic_entry */
813				if (((io_apic_entry_ptr)position)->apic_flags
814					& IOAPICENTRY_FLAG_EN)
815					io_apic_address[mp_napics++] =
816					    (vm_offset_t)((io_apic_entry_ptr)
817						position)->apic_address;
818				break;
819			case 3: /* int_entry */
820				++nintrs;
821				break;
822			case 4:	/* int_entry */
823				break;
824			default:
825				panic("mpfps Base Table HOSED!");
826				/* NOTREACHED */
827			}
828
829			totalSize -= basetable_entry_types[type].length;
830			(u_char*)position += basetable_entry_types[type].length;
831		}
832	}
833
834	/* qualify the numbers */
835	if (mp_naps > NCPU) {
836		printf("Warning: only using %d of %d available CPUs!\n",
837			NCPU, mp_naps);
838		mp_naps = NCPU;
839	}
840	if (mp_nbusses > NBUS) {
841		printf("found %d busses, increase NBUS\n", mp_nbusses);
842		mustpanic = 1;
843	}
844	if (mp_napics > NAPIC) {
845		printf("found %d apics, increase NAPIC\n", mp_napics);
846		mustpanic = 1;
847	}
848	if (nintrs > NINTR) {
849		printf("found %d intrs, increase NINTR\n", nintrs);
850		mustpanic = 1;
851	}
852
853	/*
854	 * Count the BSP.
855	 * This is also used as a counter while starting the APs.
856	 */
857	mp_ncpus = 1;
858
859	--mp_naps;	/* subtract the BSP */
860
861	return mustpanic;
862}
863
864
865/*
866 * 2nd pass on motherboard's Intel MP specification table.
867 *
868 * sets:
869 *	boot_cpu_id
870 *	ID_TO_IO(N), phy APIC ID to log CPU/IO table
871 *	CPU_TO_ID(N), logical CPU to APIC ID table
872 *	IO_TO_ID(N), logical IO to APIC ID table
873 *	bus_data[N]
874 *	io_apic_ints[N]
875 */
876static int
877mptable_pass2(void)
878{
879	int     x;
880	mpcth_t cth;
881	int     totalSize;
882	void*   position;
883	int     count;
884	int     type;
885	int     apic, bus, cpu, intr;
886
887	POSTCODE(MPTABLE_PASS2_POST);
888
889	/* clear various tables */
890	for (x = 0; x < NAPICID; ++x) {
891		ID_TO_IO(x) = -1;	/* phy APIC ID to log CPU/IO table */
892		CPU_TO_ID(x) = -1;	/* logical CPU to APIC ID table */
893		IO_TO_ID(x) = -1;	/* logical IO to APIC ID table */
894	}
895
896	/* clear bus data table */
897	for (x = 0; x < NBUS; ++x)
898		bus_data[x].bus_id = 0xff;
899
900	/* clear IO APIC INT table */
901	for (x = 0; x < NINTR; ++x) {
902		io_apic_ints[x].int_type = 0xff;
903		io_apic_ints[x].int_vector = 0xff;
904	}
905
906	/* setup the cpu/apic mapping arrays */
907	boot_cpu_id = -1;
908
909	/* record whether PIC or virtual-wire mode */
910	picmode = (mpfps->mpfb2 & 0x80) ? 1 : 0;
911
912	/* check for use of 'default' configuration */
913	if (MPFPS_MPFB1 != 0)
914		return MPFPS_MPFB1;	/* return default configuration type */
915
916	if ((cth = mpfps->pap) == 0)
917		panic("MP Configuration Table Header MISSING!");
918
919	/* walk the table, recording info of interest */
920	totalSize = cth->base_table_length - sizeof(struct MPCTH);
921	position = (u_char *) cth + sizeof(struct MPCTH);
922	count = cth->entry_count;
923	apic = bus = intr = 0;
924	cpu = 1;				/* pre-count the BSP */
925
926	while (count--) {
927		switch (type = *(u_char *) position) {
928		case 0:
929			if (processor_entry(position, cpu))
930				++cpu;
931			break;
932		case 1:
933			if (bus_entry(position, bus))
934				++bus;
935			break;
936		case 2:
937			if (io_apic_entry(position, apic))
938				++apic;
939			break;
940		case 3:
941			if (int_entry(position, intr))
942				++intr;
943			break;
944		case 4:
945			/* int_entry(position); */
946			break;
947		default:
948			panic("mpfps Base Table HOSED!");
949			/* NOTREACHED */
950		}
951
952		totalSize -= basetable_entry_types[type].length;
953		(u_char *) position += basetable_entry_types[type].length;
954	}
955
956	if (boot_cpu_id == -1)
957		panic("NO BSP found!");
958
959	/* report fact that its NOT a default configuration */
960	return 0;
961}
962
963
964void
965assign_apic_irq(int apic, int intpin, int irq)
966{
967	int x;
968
969	if (int_to_apicintpin[irq].ioapic != -1)
970		panic("assign_apic_irq: inconsistent table");
971
972	int_to_apicintpin[irq].ioapic = apic;
973	int_to_apicintpin[irq].int_pin = intpin;
974	int_to_apicintpin[irq].apic_address = ioapic[apic];
975	int_to_apicintpin[irq].redirindex = IOAPIC_REDTBL + 2 * intpin;
976
977	for (x = 0; x < nintrs; x++) {
978		if ((io_apic_ints[x].int_type == 0 ||
979		     io_apic_ints[x].int_type == 3) &&
980		    io_apic_ints[x].int_vector == 0xff &&
981		    io_apic_ints[x].dst_apic_id == IO_TO_ID(apic) &&
982		    io_apic_ints[x].dst_apic_int == intpin)
983			io_apic_ints[x].int_vector = irq;
984	}
985}
986
987void
988revoke_apic_irq(int irq)
989{
990	int x;
991	int oldapic;
992	int oldintpin;
993
994	if (int_to_apicintpin[irq].ioapic == -1)
995		panic("assign_apic_irq: inconsistent table");
996
997	oldapic = int_to_apicintpin[irq].ioapic;
998	oldintpin = int_to_apicintpin[irq].int_pin;
999
1000	int_to_apicintpin[irq].ioapic = -1;
1001	int_to_apicintpin[irq].int_pin = 0;
1002	int_to_apicintpin[irq].apic_address = NULL;
1003	int_to_apicintpin[irq].redirindex = 0;
1004
1005	for (x = 0; x < nintrs; x++) {
1006		if ((io_apic_ints[x].int_type == 0 ||
1007		     io_apic_ints[x].int_type == 3) &&
1008		    io_apic_ints[x].int_vector == 0xff &&
1009		    io_apic_ints[x].dst_apic_id == IO_TO_ID(oldapic) &&
1010		    io_apic_ints[x].dst_apic_int == oldintpin)
1011			io_apic_ints[x].int_vector = 0xff;
1012	}
1013}
1014
1015
1016
1017static void
1018swap_apic_id(int apic, int oldid, int newid)
1019{
1020	int x;
1021	int oapic;
1022
1023
1024	if (oldid == newid)
1025		return;			/* Nothing to do */
1026
1027	printf("Changing APIC ID for IO APIC #%d from %d to %d in MP table\n",
1028	       apic, oldid, newid);
1029
1030	/* Swap physical APIC IDs in interrupt entries */
1031	for (x = 0; x < nintrs; x++) {
1032		if (io_apic_ints[x].dst_apic_id == oldid)
1033			io_apic_ints[x].dst_apic_id = newid;
1034		else if (io_apic_ints[x].dst_apic_id == newid)
1035			io_apic_ints[x].dst_apic_id = oldid;
1036	}
1037
1038	/* Swap physical APIC IDs in IO_TO_ID mappings */
1039	for (oapic = 0; oapic < mp_napics; oapic++)
1040		if (IO_TO_ID(oapic) == newid)
1041			break;
1042
1043	if (oapic < mp_napics) {
1044		printf("Changing APIC ID for IO APIC #%d from "
1045		       "%d to %d in MP table\n",
1046		       oapic, newid, oldid);
1047		IO_TO_ID(oapic) = oldid;
1048	}
1049	IO_TO_ID(apic) = newid;
1050}
1051
1052
1053static void
1054fix_id_to_io_mapping(void)
1055{
1056	int x;
1057
1058	for (x = 0; x < NAPICID; x++)
1059		ID_TO_IO(x) = -1;
1060
1061	for (x = 0; x <= mp_naps; x++)
1062		if (CPU_TO_ID(x) < NAPICID)
1063			ID_TO_IO(CPU_TO_ID(x)) = x;
1064
1065	for (x = 0; x < mp_napics; x++)
1066		if (IO_TO_ID(x) < NAPICID)
1067			ID_TO_IO(IO_TO_ID(x)) = x;
1068}
1069
1070
1071static int
1072first_free_apic_id(void)
1073{
1074	int freeid, x;
1075
1076	for (freeid = 0; freeid < NAPICID; freeid++) {
1077		for (x = 0; x <= mp_naps; x++)
1078			if (CPU_TO_ID(x) == freeid)
1079				break;
1080		if (x <= mp_naps)
1081			continue;
1082		for (x = 0; x < mp_napics; x++)
1083			if (IO_TO_ID(x) == freeid)
1084				break;
1085		if (x < mp_napics)
1086			continue;
1087		return freeid;
1088	}
1089	return freeid;
1090}
1091
1092
1093static int
1094io_apic_id_acceptable(int apic, int id)
1095{
1096	int cpu;		/* Logical CPU number */
1097	int oapic;		/* Logical IO APIC number for other IO APIC */
1098
1099	if (id >= NAPICID)
1100		return 0;	/* Out of range */
1101
1102	for (cpu = 0; cpu <= mp_naps; cpu++)
1103		if (CPU_TO_ID(cpu) == id)
1104			return 0;	/* Conflict with CPU */
1105
1106	for (oapic = 0; oapic < mp_napics && oapic < apic; oapic++)
1107		if (IO_TO_ID(oapic) == id)
1108			return 0;	/* Conflict with other APIC */
1109
1110	return 1;		/* ID is acceptable for IO APIC */
1111}
1112
1113
1114/*
1115 * parse an Intel MP specification table
1116 */
1117static void
1118fix_mp_table(void)
1119{
1120	int	x;
1121	int	id;
1122	int	bus_0 = 0;	/* Stop GCC warning */
1123	int	bus_pci = 0;	/* Stop GCC warning */
1124	int	num_pci_bus;
1125	int	apic;		/* IO APIC unit number */
1126	int     freeid;		/* Free physical APIC ID */
1127	int	physid;		/* Current physical IO APIC ID */
1128
1129	/*
1130	 * Fix mis-numbering of the PCI bus and its INT entries if the BIOS
1131	 * did it wrong.  The MP spec says that when more than 1 PCI bus
1132	 * exists the BIOS must begin with bus entries for the PCI bus and use
1133	 * actual PCI bus numbering.  This implies that when only 1 PCI bus
1134	 * exists the BIOS can choose to ignore this ordering, and indeed many
1135	 * MP motherboards do ignore it.  This causes a problem when the PCI
1136	 * sub-system makes requests of the MP sub-system based on PCI bus
1137	 * numbers.	So here we look for the situation and renumber the
1138	 * busses and associated INTs in an effort to "make it right".
1139	 */
1140
1141	/* find bus 0, PCI bus, count the number of PCI busses */
1142	for (num_pci_bus = 0, x = 0; x < mp_nbusses; ++x) {
1143		if (bus_data[x].bus_id == 0) {
1144			bus_0 = x;
1145		}
1146		if (bus_data[x].bus_type == PCI) {
1147			++num_pci_bus;
1148			bus_pci = x;
1149		}
1150	}
1151	/*
1152	 * bus_0 == slot of bus with ID of 0
1153	 * bus_pci == slot of last PCI bus encountered
1154	 */
1155
1156	/* check the 1 PCI bus case for sanity */
1157	/* if it is number 0 all is well */
1158	if (num_pci_bus == 1 &&
1159	    bus_data[bus_pci].bus_id != 0) {
1160
1161		/* mis-numbered, swap with whichever bus uses slot 0 */
1162
1163		/* swap the bus entry types */
1164		bus_data[bus_pci].bus_type = bus_data[bus_0].bus_type;
1165		bus_data[bus_0].bus_type = PCI;
1166
1167		/* swap each relavant INTerrupt entry */
1168		id = bus_data[bus_pci].bus_id;
1169		for (x = 0; x < nintrs; ++x) {
1170			if (io_apic_ints[x].src_bus_id == id) {
1171				io_apic_ints[x].src_bus_id = 0;
1172			}
1173			else if (io_apic_ints[x].src_bus_id == 0) {
1174				io_apic_ints[x].src_bus_id = id;
1175			}
1176		}
1177	}
1178
1179	/* Assign IO APIC IDs.
1180	 *
1181	 * First try the existing ID. If a conflict is detected, try
1182	 * the ID in the MP table.  If a conflict is still detected, find
1183	 * a free id.
1184	 *
1185	 * We cannot use the ID_TO_IO table before all conflicts has been
1186	 * resolved and the table has been corrected.
1187	 */
1188	for (apic = 0; apic < mp_napics; ++apic) { /* For all IO APICs */
1189
1190		/* First try to use the value set by the BIOS */
1191		physid = io_apic_get_id(apic);
1192		if (io_apic_id_acceptable(apic, physid)) {
1193			if (IO_TO_ID(apic) != physid)
1194				swap_apic_id(apic, IO_TO_ID(apic), physid);
1195			continue;
1196		}
1197
1198		/* Then check if the value in the MP table is acceptable */
1199		if (io_apic_id_acceptable(apic, IO_TO_ID(apic)))
1200			continue;
1201
1202		/* Last resort, find a free APIC ID and use it */
1203		freeid = first_free_apic_id();
1204		if (freeid >= NAPICID)
1205			panic("No free physical APIC IDs found");
1206
1207		if (io_apic_id_acceptable(apic, freeid)) {
1208			swap_apic_id(apic, IO_TO_ID(apic), freeid);
1209			continue;
1210		}
1211		panic("Free physical APIC ID not usable");
1212	}
1213	fix_id_to_io_mapping();
1214}
1215
1216
1217/* Assign low level interrupt handlers */
1218static void
1219setup_apic_irq_mapping(void)
1220{
1221	int	x;
1222	int	int_vector;
1223
1224	/* Clear array */
1225	for (x = 0; x < APIC_INTMAPSIZE; x++) {
1226		int_to_apicintpin[x].ioapic = -1;
1227		int_to_apicintpin[x].int_pin = 0;
1228		int_to_apicintpin[x].apic_address = NULL;
1229		int_to_apicintpin[x].redirindex = 0;
1230	}
1231
1232	/* First assign ISA/EISA interrupts */
1233	for (x = 0; x < nintrs; x++) {
1234		int_vector = io_apic_ints[x].src_bus_irq;
1235		if (int_vector < APIC_INTMAPSIZE &&
1236		    io_apic_ints[x].int_vector == 0xff &&
1237		    int_to_apicintpin[int_vector].ioapic == -1 &&
1238		    (apic_int_is_bus_type(x, ISA) ||
1239		     apic_int_is_bus_type(x, EISA)) &&
1240		    io_apic_ints[x].int_type == 0) {
1241			assign_apic_irq(ID_TO_IO(io_apic_ints[x].dst_apic_id),
1242					io_apic_ints[x].dst_apic_int,
1243					int_vector);
1244		}
1245	}
1246
1247	/* Assign interrupts on first 24 intpins on IOAPIC #0 */
1248	for (x = 0; x < nintrs; x++) {
1249		int_vector = io_apic_ints[x].dst_apic_int;
1250		if (int_vector < APIC_INTMAPSIZE &&
1251		    io_apic_ints[x].dst_apic_id == IO_TO_ID(0) &&
1252		    io_apic_ints[x].int_vector == 0xff &&
1253		    int_to_apicintpin[int_vector].ioapic == -1 &&
1254		    (io_apic_ints[x].int_type == 0 ||
1255		     io_apic_ints[x].int_type == 3)) {
1256			assign_apic_irq(0,
1257					io_apic_ints[x].dst_apic_int,
1258					int_vector);
1259		}
1260	}
1261	/*
1262	 * Assign interrupts for remaining intpins.
1263	 * Skip IOAPIC #0 intpin 0 if the type is ExtInt, since this indicates
1264	 * that an entry for ISA/EISA irq 0 exist, and a fallback to mixed mode
1265	 * due to 8254 interrupts not being delivered can reuse that low level
1266	 * interrupt handler.
1267	 */
1268	int_vector = 0;
1269	while (int_vector < APIC_INTMAPSIZE &&
1270	       int_to_apicintpin[int_vector].ioapic != -1)
1271		int_vector++;
1272	for (x = 0; x < nintrs && int_vector < APIC_INTMAPSIZE; x++) {
1273		if ((io_apic_ints[x].int_type == 0 ||
1274		     (io_apic_ints[x].int_type == 3 &&
1275		      (io_apic_ints[x].dst_apic_id != IO_TO_ID(0) ||
1276		       io_apic_ints[x].dst_apic_int != 0))) &&
1277		    io_apic_ints[x].int_vector == 0xff) {
1278			assign_apic_irq(ID_TO_IO(io_apic_ints[x].dst_apic_id),
1279					io_apic_ints[x].dst_apic_int,
1280					int_vector);
1281			int_vector++;
1282			while (int_vector < APIC_INTMAPSIZE &&
1283			       int_to_apicintpin[int_vector].ioapic != -1)
1284				int_vector++;
1285		}
1286	}
1287}
1288
1289
1290static int
1291processor_entry(proc_entry_ptr entry, int cpu)
1292{
1293	/* check for usability */
1294	if (!(entry->cpu_flags & PROCENTRY_FLAG_EN))
1295		return 0;
1296
1297	if(entry->apic_id >= NAPICID)
1298		panic("CPU APIC ID out of range (0..%d)", NAPICID - 1);
1299	/* check for BSP flag */
1300	if (entry->cpu_flags & PROCENTRY_FLAG_BP) {
1301		boot_cpu_id = entry->apic_id;
1302		CPU_TO_ID(0) = entry->apic_id;
1303		ID_TO_CPU(entry->apic_id) = 0;
1304		return 0;	/* its already been counted */
1305	}
1306
1307	/* add another AP to list, if less than max number of CPUs */
1308	else if (cpu < NCPU) {
1309		CPU_TO_ID(cpu) = entry->apic_id;
1310		ID_TO_CPU(entry->apic_id) = cpu;
1311		return 1;
1312	}
1313
1314	return 0;
1315}
1316
1317
1318static int
1319bus_entry(bus_entry_ptr entry, int bus)
1320{
1321	int     x;
1322	char    c, name[8];
1323
1324	/* encode the name into an index */
1325	for (x = 0; x < 6; ++x) {
1326		if ((c = entry->bus_type[x]) == ' ')
1327			break;
1328		name[x] = c;
1329	}
1330	name[x] = '\0';
1331
1332	if ((x = lookup_bus_type(name)) == UNKNOWN_BUSTYPE)
1333		panic("unknown bus type: '%s'", name);
1334
1335	bus_data[bus].bus_id = entry->bus_id;
1336	bus_data[bus].bus_type = x;
1337
1338	return 1;
1339}
1340
1341
1342static int
1343io_apic_entry(io_apic_entry_ptr entry, int apic)
1344{
1345	if (!(entry->apic_flags & IOAPICENTRY_FLAG_EN))
1346		return 0;
1347
1348	IO_TO_ID(apic) = entry->apic_id;
1349	if (entry->apic_id < NAPICID)
1350		ID_TO_IO(entry->apic_id) = apic;
1351
1352	return 1;
1353}
1354
1355
1356static int
1357lookup_bus_type(char *name)
1358{
1359	int     x;
1360
1361	for (x = 0; x < MAX_BUSTYPE; ++x)
1362		if (strcmp(bus_type_table[x].name, name) == 0)
1363			return bus_type_table[x].type;
1364
1365	return UNKNOWN_BUSTYPE;
1366}
1367
1368
1369static int
1370int_entry(int_entry_ptr entry, int intr)
1371{
1372	int apic;
1373
1374	io_apic_ints[intr].int_type = entry->int_type;
1375	io_apic_ints[intr].int_flags = entry->int_flags;
1376	io_apic_ints[intr].src_bus_id = entry->src_bus_id;
1377	io_apic_ints[intr].src_bus_irq = entry->src_bus_irq;
1378	if (entry->dst_apic_id == 255) {
1379		/* This signal goes to all IO APICS.  Select an IO APIC
1380		   with sufficient number of interrupt pins */
1381		for (apic = 0; apic < mp_napics; apic++)
1382			if (((io_apic_read(apic, IOAPIC_VER) &
1383			      IOART_VER_MAXREDIR) >> MAXREDIRSHIFT) >=
1384			    entry->dst_apic_int)
1385				break;
1386		if (apic < mp_napics)
1387			io_apic_ints[intr].dst_apic_id = IO_TO_ID(apic);
1388		else
1389			io_apic_ints[intr].dst_apic_id = entry->dst_apic_id;
1390	} else
1391		io_apic_ints[intr].dst_apic_id = entry->dst_apic_id;
1392	io_apic_ints[intr].dst_apic_int = entry->dst_apic_int;
1393
1394	return 1;
1395}
1396
1397
1398static int
1399apic_int_is_bus_type(int intr, int bus_type)
1400{
1401	int     bus;
1402
1403	for (bus = 0; bus < mp_nbusses; ++bus)
1404		if ((bus_data[bus].bus_id == io_apic_ints[intr].src_bus_id)
1405		    && ((int) bus_data[bus].bus_type == bus_type))
1406			return 1;
1407
1408	return 0;
1409}
1410
1411
1412/*
1413 * Given a traditional ISA INT mask, return an APIC mask.
1414 */
1415u_int
1416isa_apic_mask(u_int isa_mask)
1417{
1418	int isa_irq;
1419	int apic_pin;
1420
1421#if defined(SKIP_IRQ15_REDIRECT)
1422	if (isa_mask == (1 << 15)) {
1423		printf("skipping ISA IRQ15 redirect\n");
1424		return isa_mask;
1425	}
1426#endif  /* SKIP_IRQ15_REDIRECT */
1427
1428	isa_irq = ffs(isa_mask);		/* find its bit position */
1429	if (isa_irq == 0)			/* doesn't exist */
1430		return 0;
1431	--isa_irq;				/* make it zero based */
1432
1433	apic_pin = isa_apic_irq(isa_irq);	/* look for APIC connection */
1434	if (apic_pin == -1)
1435		return 0;
1436
1437	return (1 << apic_pin);			/* convert pin# to a mask */
1438}
1439
1440
1441/*
1442 * Determine which APIC pin an ISA/EISA INT is attached to.
1443 */
1444#define INTTYPE(I)	(io_apic_ints[(I)].int_type)
1445#define INTPIN(I)	(io_apic_ints[(I)].dst_apic_int)
1446#define INTIRQ(I)	(io_apic_ints[(I)].int_vector)
1447#define INTAPIC(I)	(ID_TO_IO(io_apic_ints[(I)].dst_apic_id))
1448
1449#define SRCBUSIRQ(I)	(io_apic_ints[(I)].src_bus_irq)
1450int
1451isa_apic_irq(int isa_irq)
1452{
1453	int     intr;
1454
1455	for (intr = 0; intr < nintrs; ++intr) {		/* check each record */
1456		if (INTTYPE(intr) == 0) {		/* standard INT */
1457			if (SRCBUSIRQ(intr) == isa_irq) {
1458				if (apic_int_is_bus_type(intr, ISA) ||
1459			            apic_int_is_bus_type(intr, EISA))
1460					return INTIRQ(intr);	/* found */
1461			}
1462		}
1463	}
1464	return -1;					/* NOT found */
1465}
1466
1467
1468/*
1469 * Determine which APIC pin a PCI INT is attached to.
1470 */
1471#define SRCBUSID(I)	(io_apic_ints[(I)].src_bus_id)
1472#define SRCBUSDEVICE(I)	((io_apic_ints[(I)].src_bus_irq >> 2) & 0x1f)
1473#define SRCBUSLINE(I)	(io_apic_ints[(I)].src_bus_irq & 0x03)
1474int
1475pci_apic_irq(int pciBus, int pciDevice, int pciInt)
1476{
1477	int     intr;
1478
1479	--pciInt;					/* zero based */
1480
1481	for (intr = 0; intr < nintrs; ++intr)		/* check each record */
1482		if ((INTTYPE(intr) == 0)		/* standard INT */
1483		    && (SRCBUSID(intr) == pciBus)
1484		    && (SRCBUSDEVICE(intr) == pciDevice)
1485		    && (SRCBUSLINE(intr) == pciInt))	/* a candidate IRQ */
1486			if (apic_int_is_bus_type(intr, PCI))
1487				return INTIRQ(intr);	/* exact match */
1488
1489	return -1;					/* NOT found */
1490}
1491
1492int
1493next_apic_irq(int irq)
1494{
1495	int intr, ointr;
1496	int bus, bustype;
1497
1498	bus = 0;
1499	bustype = 0;
1500	for (intr = 0; intr < nintrs; intr++) {
1501		if (INTIRQ(intr) != irq || INTTYPE(intr) != 0)
1502			continue;
1503		bus = SRCBUSID(intr);
1504		bustype = apic_bus_type(bus);
1505		if (bustype != ISA &&
1506		    bustype != EISA &&
1507		    bustype != PCI)
1508			continue;
1509		break;
1510	}
1511	if (intr >= nintrs) {
1512		return -1;
1513	}
1514	for (ointr = intr + 1; ointr < nintrs; ointr++) {
1515		if (INTTYPE(ointr) != 0)
1516			continue;
1517		if (bus != SRCBUSID(ointr))
1518			continue;
1519		if (bustype == PCI) {
1520			if (SRCBUSDEVICE(intr) != SRCBUSDEVICE(ointr))
1521				continue;
1522			if (SRCBUSLINE(intr) != SRCBUSLINE(ointr))
1523				continue;
1524		}
1525		if (bustype == ISA || bustype == EISA) {
1526			if (SRCBUSIRQ(intr) != SRCBUSIRQ(ointr))
1527				continue;
1528		}
1529		if (INTPIN(intr) == INTPIN(ointr))
1530			continue;
1531		break;
1532	}
1533	if (ointr >= nintrs) {
1534		return -1;
1535	}
1536	return INTIRQ(ointr);
1537}
1538#undef SRCBUSLINE
1539#undef SRCBUSDEVICE
1540#undef SRCBUSID
1541#undef SRCBUSIRQ
1542
1543#undef INTPIN
1544#undef INTIRQ
1545#undef INTAPIC
1546#undef INTTYPE
1547
1548
1549/*
1550 * Reprogram the MB chipset to NOT redirect an ISA INTerrupt.
1551 *
1552 * XXX FIXME:
1553 *  Exactly what this means is unclear at this point.  It is a solution
1554 *  for motherboards that redirect the MBIRQ0 pin.  Generically a motherboard
1555 *  could route any of the ISA INTs to upper (>15) IRQ values.  But most would
1556 *  NOT be redirected via MBIRQ0, thus "undirect()ing" them would NOT be an
1557 *  option.
1558 */
1559int
1560undirect_isa_irq(int rirq)
1561{
1562#if defined(READY)
1563	if (bootverbose)
1564	    printf("Freeing redirected ISA irq %d.\n", rirq);
1565	/** FIXME: tickle the MB redirector chip */
1566	return ???;
1567#else
1568	if (bootverbose)
1569	    printf("Freeing (NOT implemented) redirected ISA irq %d.\n", rirq);
1570	return 0;
1571#endif  /* READY */
1572}
1573
1574
1575/*
1576 * Reprogram the MB chipset to NOT redirect a PCI INTerrupt
1577 */
1578int
1579undirect_pci_irq(int rirq)
1580{
1581#if defined(READY)
1582	if (bootverbose)
1583		printf("Freeing redirected PCI irq %d.\n", rirq);
1584
1585	/** FIXME: tickle the MB redirector chip */
1586	return ???;
1587#else
1588	if (bootverbose)
1589		printf("Freeing (NOT implemented) redirected PCI irq %d.\n",
1590		       rirq);
1591	return 0;
1592#endif  /* READY */
1593}
1594
1595
1596/*
1597 * given a bus ID, return:
1598 *  the bus type if found
1599 *  -1 if NOT found
1600 */
1601int
1602apic_bus_type(int id)
1603{
1604	int     x;
1605
1606	for (x = 0; x < mp_nbusses; ++x)
1607		if (bus_data[x].bus_id == id)
1608			return bus_data[x].bus_type;
1609
1610	return -1;
1611}
1612
1613
1614/*
1615 * given a LOGICAL APIC# and pin#, return:
1616 *  the associated src bus ID if found
1617 *  -1 if NOT found
1618 */
1619int
1620apic_src_bus_id(int apic, int pin)
1621{
1622	int     x;
1623
1624	/* search each of the possible INTerrupt sources */
1625	for (x = 0; x < nintrs; ++x)
1626		if ((apic == ID_TO_IO(io_apic_ints[x].dst_apic_id)) &&
1627		    (pin == io_apic_ints[x].dst_apic_int))
1628			return (io_apic_ints[x].src_bus_id);
1629
1630	return -1;		/* NOT found */
1631}
1632
1633
1634/*
1635 * given a LOGICAL APIC# and pin#, return:
1636 *  the associated src bus IRQ if found
1637 *  -1 if NOT found
1638 */
1639int
1640apic_src_bus_irq(int apic, int pin)
1641{
1642	int     x;
1643
1644	for (x = 0; x < nintrs; x++)
1645		if ((apic == ID_TO_IO(io_apic_ints[x].dst_apic_id)) &&
1646		    (pin == io_apic_ints[x].dst_apic_int))
1647			return (io_apic_ints[x].src_bus_irq);
1648
1649	return -1;		/* NOT found */
1650}
1651
1652
1653/*
1654 * given a LOGICAL APIC# and pin#, return:
1655 *  the associated INTerrupt type if found
1656 *  -1 if NOT found
1657 */
1658int
1659apic_int_type(int apic, int pin)
1660{
1661	int     x;
1662
1663	/* search each of the possible INTerrupt sources */
1664	for (x = 0; x < nintrs; ++x)
1665		if ((apic == ID_TO_IO(io_apic_ints[x].dst_apic_id)) &&
1666		    (pin == io_apic_ints[x].dst_apic_int))
1667			return (io_apic_ints[x].int_type);
1668
1669	return -1;		/* NOT found */
1670}
1671
1672int
1673apic_irq(int apic, int pin)
1674{
1675	int x;
1676	int res;
1677
1678	for (x = 0; x < nintrs; ++x)
1679		if ((apic == ID_TO_IO(io_apic_ints[x].dst_apic_id)) &&
1680		    (pin == io_apic_ints[x].dst_apic_int)) {
1681			res = io_apic_ints[x].int_vector;
1682			if (res == 0xff)
1683				return -1;
1684			if (apic != int_to_apicintpin[res].ioapic)
1685				panic("apic_irq: inconsistent table");
1686			if (pin != int_to_apicintpin[res].int_pin)
1687				panic("apic_irq inconsistent table (2)");
1688			return res;
1689		}
1690	return -1;
1691}
1692
1693
1694/*
1695 * given a LOGICAL APIC# and pin#, return:
1696 *  the associated trigger mode if found
1697 *  -1 if NOT found
1698 */
1699int
1700apic_trigger(int apic, int pin)
1701{
1702	int     x;
1703
1704	/* search each of the possible INTerrupt sources */
1705	for (x = 0; x < nintrs; ++x)
1706		if ((apic == ID_TO_IO(io_apic_ints[x].dst_apic_id)) &&
1707		    (pin == io_apic_ints[x].dst_apic_int))
1708			return ((io_apic_ints[x].int_flags >> 2) & 0x03);
1709
1710	return -1;		/* NOT found */
1711}
1712
1713
1714/*
1715 * given a LOGICAL APIC# and pin#, return:
1716 *  the associated 'active' level if found
1717 *  -1 if NOT found
1718 */
1719int
1720apic_polarity(int apic, int pin)
1721{
1722	int     x;
1723
1724	/* search each of the possible INTerrupt sources */
1725	for (x = 0; x < nintrs; ++x)
1726		if ((apic == ID_TO_IO(io_apic_ints[x].dst_apic_id)) &&
1727		    (pin == io_apic_ints[x].dst_apic_int))
1728			return (io_apic_ints[x].int_flags & 0x03);
1729
1730	return -1;		/* NOT found */
1731}
1732
1733
1734/*
1735 * set data according to MP defaults
1736 * FIXME: probably not complete yet...
1737 */
1738static void
1739default_mp_table(int type)
1740{
1741	int     ap_cpu_id;
1742#if defined(APIC_IO)
1743	int     io_apic_id;
1744	int     pin;
1745#endif	/* APIC_IO */
1746
1747#if 0
1748	printf("  MP default config type: %d\n", type);
1749	switch (type) {
1750	case 1:
1751		printf("   bus: ISA, APIC: 82489DX\n");
1752		break;
1753	case 2:
1754		printf("   bus: EISA, APIC: 82489DX\n");
1755		break;
1756	case 3:
1757		printf("   bus: EISA, APIC: 82489DX\n");
1758		break;
1759	case 4:
1760		printf("   bus: MCA, APIC: 82489DX\n");
1761		break;
1762	case 5:
1763		printf("   bus: ISA+PCI, APIC: Integrated\n");
1764		break;
1765	case 6:
1766		printf("   bus: EISA+PCI, APIC: Integrated\n");
1767		break;
1768	case 7:
1769		printf("   bus: MCA+PCI, APIC: Integrated\n");
1770		break;
1771	default:
1772		printf("   future type\n");
1773		break;
1774		/* NOTREACHED */
1775	}
1776#endif	/* 0 */
1777
1778	boot_cpu_id = (lapic.id & APIC_ID_MASK) >> 24;
1779	ap_cpu_id = (boot_cpu_id == 0) ? 1 : 0;
1780
1781	/* BSP */
1782	CPU_TO_ID(0) = boot_cpu_id;
1783	ID_TO_CPU(boot_cpu_id) = 0;
1784
1785	/* one and only AP */
1786	CPU_TO_ID(1) = ap_cpu_id;
1787	ID_TO_CPU(ap_cpu_id) = 1;
1788
1789#if defined(APIC_IO)
1790	/* one and only IO APIC */
1791	io_apic_id = (io_apic_read(0, IOAPIC_ID) & APIC_ID_MASK) >> 24;
1792
1793	/*
1794	 * sanity check, refer to MP spec section 3.6.6, last paragraph
1795	 * necessary as some hardware isn't properly setting up the IO APIC
1796	 */
1797#if defined(REALLY_ANAL_IOAPICID_VALUE)
1798	if (io_apic_id != 2) {
1799#else
1800	if ((io_apic_id == 0) || (io_apic_id == 1) || (io_apic_id == 15)) {
1801#endif	/* REALLY_ANAL_IOAPICID_VALUE */
1802		io_apic_set_id(0, 2);
1803		io_apic_id = 2;
1804	}
1805	IO_TO_ID(0) = io_apic_id;
1806	ID_TO_IO(io_apic_id) = 0;
1807#endif	/* APIC_IO */
1808
1809	/* fill out bus entries */
1810	switch (type) {
1811	case 1:
1812	case 2:
1813	case 3:
1814	case 4:
1815	case 5:
1816	case 6:
1817	case 7:
1818		bus_data[0].bus_id = default_data[type - 1][1];
1819		bus_data[0].bus_type = default_data[type - 1][2];
1820		bus_data[1].bus_id = default_data[type - 1][3];
1821		bus_data[1].bus_type = default_data[type - 1][4];
1822		break;
1823
1824	/* case 4: case 7:		   MCA NOT supported */
1825	default:		/* illegal/reserved */
1826		panic("BAD default MP config: %d", type);
1827		/* NOTREACHED */
1828	}
1829
1830#if defined(APIC_IO)
1831	/* general cases from MP v1.4, table 5-2 */
1832	for (pin = 0; pin < 16; ++pin) {
1833		io_apic_ints[pin].int_type = 0;
1834		io_apic_ints[pin].int_flags = 0x05;	/* edge/active-hi */
1835		io_apic_ints[pin].src_bus_id = 0;
1836		io_apic_ints[pin].src_bus_irq = pin;	/* IRQ2 caught below */
1837		io_apic_ints[pin].dst_apic_id = io_apic_id;
1838		io_apic_ints[pin].dst_apic_int = pin;	/* 1-to-1 */
1839	}
1840
1841	/* special cases from MP v1.4, table 5-2 */
1842	if (type == 2) {
1843		io_apic_ints[2].int_type = 0xff;	/* N/C */
1844		io_apic_ints[13].int_type = 0xff;	/* N/C */
1845#if !defined(APIC_MIXED_MODE)
1846		/** FIXME: ??? */
1847		panic("sorry, can't support type 2 default yet");
1848#endif	/* APIC_MIXED_MODE */
1849	}
1850	else
1851		io_apic_ints[2].src_bus_irq = 0;	/* ISA IRQ0 is on APIC INT 2 */
1852
1853	if (type == 7)
1854		io_apic_ints[0].int_type = 0xff;	/* N/C */
1855	else
1856		io_apic_ints[0].int_type = 3;	/* vectored 8259 */
1857#endif	/* APIC_IO */
1858}
1859
1860
1861/*
1862 * initialize all the SMP locks
1863 */
1864
1865/* critical region around IO APIC, apic_imen */
1866struct simplelock	imen_lock;
1867
1868/* critical region around splxx(), cpl, cml, cil, ipending */
1869struct simplelock	cpl_lock;
1870
1871/* Make FAST_INTR() routines sequential */
1872struct simplelock	fast_intr_lock;
1873
1874/* critical region around INTR() routines */
1875struct simplelock	intr_lock;
1876
1877/* lock region used by kernel profiling */
1878struct simplelock	mcount_lock;
1879
1880#ifdef USE_COMLOCK
1881/* locks com (tty) data/hardware accesses: a FASTINTR() */
1882struct simplelock	com_lock;
1883#endif /* USE_COMLOCK */
1884
1885#ifdef USE_CLOCKLOCK
1886/* lock regions around the clock hardware */
1887struct simplelock	clock_lock;
1888#endif /* USE_CLOCKLOCK */
1889
1890/* lock around the MP rendezvous */
1891static struct simplelock smp_rv_lock;
1892
1893/* only 1 CPU can panic at a time :) */
1894struct simplelock	panic_lock;
1895
1896static void
1897init_locks(void)
1898{
1899#if defined(APIC_INTR_DIAGNOSTIC) && defined(APIC_INTR_DIAGNOSTIC_IRQ)
1900	s_lock_init((struct simplelock*)&apic_itrace_debuglock);
1901#endif
1902
1903	s_lock_init((struct simplelock*)&mcount_lock);
1904
1905	s_lock_init((struct simplelock*)&fast_intr_lock);
1906	s_lock_init((struct simplelock*)&intr_lock);
1907	s_lock_init((struct simplelock*)&imen_lock);
1908	s_lock_init((struct simplelock*)&cpl_lock);
1909	s_lock_init(&smp_rv_lock);
1910	s_lock_init(&panic_lock);
1911
1912#ifdef USE_COMLOCK
1913	s_lock_init((struct simplelock*)&com_lock);
1914#endif /* USE_COMLOCK */
1915#ifdef USE_CLOCKLOCK
1916	s_lock_init((struct simplelock*)&clock_lock);
1917#endif /* USE_CLOCKLOCK */
1918
1919	s_lock_init(&ap_boot_lock);
1920}
1921
1922/*
1923 * start each AP in our list
1924 */
1925static int
1926start_all_aps(u_int boot_addr)
1927{
1928	int     x, i, pg;
1929	u_char  mpbiosreason;
1930	u_long  mpbioswarmvec;
1931	struct globaldata *gd;
1932	char *stack;
1933
1934	POSTCODE(START_ALL_APS_POST);
1935
1936	/* initialize BSP's local APIC */
1937	apic_initialize();
1938	bsp_apic_ready = 1;
1939
1940	/* install the AP 1st level boot code */
1941	install_ap_tramp(boot_addr);
1942
1943
1944	/* save the current value of the warm-start vector */
1945	mpbioswarmvec = *((u_long *) WARMBOOT_OFF);
1946#ifndef PC98
1947	outb(CMOS_REG, BIOS_RESET);
1948	mpbiosreason = inb(CMOS_DATA);
1949#endif
1950
1951	/* record BSP in CPU map */
1952	all_cpus = 1;
1953
1954	/* set up 0 -> 4MB P==V mapping for AP boot */
1955	*(int *)PTD = PG_V | PG_RW | ((uintptr_t)(void *)KPTphys & PG_FRAME);
1956	invltlb();
1957
1958	/* start each AP */
1959	for (x = 1; x <= mp_naps; ++x) {
1960
1961		/* This is a bit verbose, it will go away soon.  */
1962
1963		/* first page of AP's private space */
1964		pg = x * i386_btop(sizeof(struct privatespace));
1965
1966		/* allocate a new private data page */
1967		gd = (struct globaldata *)kmem_alloc(kernel_map, PAGE_SIZE);
1968
1969		/* wire it into the private page table page */
1970		SMPpt[pg] = (pt_entry_t)(PG_V | PG_RW | vtophys(gd));
1971
1972		/* allocate and set up an idle stack data page */
1973		stack = (char *)kmem_alloc(kernel_map, UPAGES*PAGE_SIZE);
1974		for (i = 0; i < UPAGES; i++)
1975			SMPpt[pg + 5 + i] = (pt_entry_t)
1976			    (PG_V | PG_RW | vtophys(PAGE_SIZE * i + stack));
1977
1978		SMPpt[pg + 1] = 0;		/* *prv_CMAP1 */
1979		SMPpt[pg + 2] = 0;		/* *prv_CMAP2 */
1980		SMPpt[pg + 3] = 0;		/* *prv_CMAP3 */
1981		SMPpt[pg + 4] = 0;		/* *prv_PMAP1 */
1982
1983		/* prime data page for it to use */
1984		SLIST_INSERT_HEAD(&cpuhead, gd, gd_allcpu);
1985		gd->gd_cpuid = x;
1986		gd->gd_cpu_lockid = x << 24;
1987		gd->gd_prv_CMAP1 = &SMPpt[pg + 1];
1988		gd->gd_prv_CMAP2 = &SMPpt[pg + 2];
1989		gd->gd_prv_CMAP3 = &SMPpt[pg + 3];
1990		gd->gd_prv_PMAP1 = &SMPpt[pg + 4];
1991		gd->gd_prv_CADDR1 = SMP_prvspace[x].CPAGE1;
1992		gd->gd_prv_CADDR2 = SMP_prvspace[x].CPAGE2;
1993		gd->gd_prv_CADDR3 = SMP_prvspace[x].CPAGE3;
1994		gd->gd_prv_PADDR1 = (unsigned *)SMP_prvspace[x].PPAGE1;
1995
1996		/* setup a vector to our boot code */
1997		*((volatile u_short *) WARMBOOT_OFF) = WARMBOOT_TARGET;
1998		*((volatile u_short *) WARMBOOT_SEG) = (boot_addr >> 4);
1999#ifndef PC98
2000		outb(CMOS_REG, BIOS_RESET);
2001		outb(CMOS_DATA, BIOS_WARM);	/* 'warm-start' */
2002#endif
2003
2004		bootSTK = &SMP_prvspace[x].idlestack[UPAGES*PAGE_SIZE];
2005		bootAP = x;
2006
2007		/* attempt to start the Application Processor */
2008		CHECK_INIT(99);	/* setup checkpoints */
2009		if (!start_ap(x, boot_addr)) {
2010			printf("AP #%d (PHY# %d) failed!\n", x, CPU_TO_ID(x));
2011			CHECK_PRINT("trace");	/* show checkpoints */
2012			/* better panic as the AP may be running loose */
2013			printf("panic y/n? [y] ");
2014			if (cngetc() != 'n')
2015				panic("bye-bye");
2016		}
2017		CHECK_PRINT("trace");		/* show checkpoints */
2018
2019		/* record its version info */
2020		cpu_apic_versions[x] = cpu_apic_versions[0];
2021
2022		all_cpus |= (1 << x);		/* record AP in CPU map */
2023	}
2024
2025	/* build our map of 'other' CPUs */
2026	other_cpus = all_cpus & ~(1 << cpuid);
2027
2028	/* fill in our (BSP) APIC version */
2029	cpu_apic_versions[0] = lapic.version;
2030
2031	/* restore the warmstart vector */
2032	*(u_long *) WARMBOOT_OFF = mpbioswarmvec;
2033#ifndef PC98
2034	outb(CMOS_REG, BIOS_RESET);
2035	outb(CMOS_DATA, mpbiosreason);
2036#endif
2037
2038	/*
2039	 * Set up the idle context for the BSP.  Similar to above except
2040	 * that some was done by locore, some by pmap.c and some is implicit
2041	 * because the BSP is cpu#0 and the page is initially zero, and also
2042	 * because we can refer to variables by name on the BSP..
2043	 */
2044
2045	/* Allocate and setup BSP idle stack */
2046	stack = (char *)kmem_alloc(kernel_map, UPAGES * PAGE_SIZE);
2047	for (i = 0; i < UPAGES; i++)
2048		SMPpt[5 + i] = (pt_entry_t)
2049		    (PG_V | PG_RW | vtophys(PAGE_SIZE * i + stack));
2050
2051	*(int *)PTD = 0;
2052	pmap_set_opt();
2053
2054	/* number of APs actually started */
2055	return mp_ncpus - 1;
2056}
2057
2058
2059/*
2060 * load the 1st level AP boot code into base memory.
2061 */
2062
2063/* targets for relocation */
2064extern void bigJump(void);
2065extern void bootCodeSeg(void);
2066extern void bootDataSeg(void);
2067extern void MPentry(void);
2068extern u_int MP_GDT;
2069extern u_int mp_gdtbase;
2070
2071static void
2072install_ap_tramp(u_int boot_addr)
2073{
2074	int     x;
2075	int     size = *(int *) ((u_long) & bootMP_size);
2076	u_char *src = (u_char *) ((u_long) bootMP);
2077	u_char *dst = (u_char *) boot_addr + KERNBASE;
2078	u_int   boot_base = (u_int) bootMP;
2079	u_int8_t *dst8;
2080	u_int16_t *dst16;
2081	u_int32_t *dst32;
2082
2083	POSTCODE(INSTALL_AP_TRAMP_POST);
2084
2085	for (x = 0; x < size; ++x)
2086		*dst++ = *src++;
2087
2088	/*
2089	 * modify addresses in code we just moved to basemem. unfortunately we
2090	 * need fairly detailed info about mpboot.s for this to work.  changes
2091	 * to mpboot.s might require changes here.
2092	 */
2093
2094	/* boot code is located in KERNEL space */
2095	dst = (u_char *) boot_addr + KERNBASE;
2096
2097	/* modify the lgdt arg */
2098	dst32 = (u_int32_t *) (dst + ((u_int) & mp_gdtbase - boot_base));
2099	*dst32 = boot_addr + ((u_int) & MP_GDT - boot_base);
2100
2101	/* modify the ljmp target for MPentry() */
2102	dst32 = (u_int32_t *) (dst + ((u_int) bigJump - boot_base) + 1);
2103	*dst32 = ((u_int) MPentry - KERNBASE);
2104
2105	/* modify the target for boot code segment */
2106	dst16 = (u_int16_t *) (dst + ((u_int) bootCodeSeg - boot_base));
2107	dst8 = (u_int8_t *) (dst16 + 1);
2108	*dst16 = (u_int) boot_addr & 0xffff;
2109	*dst8 = ((u_int) boot_addr >> 16) & 0xff;
2110
2111	/* modify the target for boot data segment */
2112	dst16 = (u_int16_t *) (dst + ((u_int) bootDataSeg - boot_base));
2113	dst8 = (u_int8_t *) (dst16 + 1);
2114	*dst16 = (u_int) boot_addr & 0xffff;
2115	*dst8 = ((u_int) boot_addr >> 16) & 0xff;
2116}
2117
2118
2119/*
2120 * this function starts the AP (application processor) identified
2121 * by the APIC ID 'physicalCpu'.  It does quite a "song and dance"
2122 * to accomplish this.  This is necessary because of the nuances
2123 * of the different hardware we might encounter.  It ain't pretty,
2124 * but it seems to work.
2125 */
2126static int
2127start_ap(int logical_cpu, u_int boot_addr)
2128{
2129	int     physical_cpu;
2130	int     vector;
2131	int     cpus;
2132	u_long  icr_lo, icr_hi;
2133
2134	POSTCODE(START_AP_POST);
2135
2136	/* get the PHYSICAL APIC ID# */
2137	physical_cpu = CPU_TO_ID(logical_cpu);
2138
2139	/* calculate the vector */
2140	vector = (boot_addr >> 12) & 0xff;
2141
2142	/* used as a watchpoint to signal AP startup */
2143	cpus = mp_ncpus;
2144
2145	/*
2146	 * first we do an INIT/RESET IPI this INIT IPI might be run, reseting
2147	 * and running the target CPU. OR this INIT IPI might be latched (P5
2148	 * bug), CPU waiting for STARTUP IPI. OR this INIT IPI might be
2149	 * ignored.
2150	 */
2151
2152	/* setup the address for the target AP */
2153	icr_hi = lapic.icr_hi & ~APIC_ID_MASK;
2154	icr_hi |= (physical_cpu << 24);
2155	lapic.icr_hi = icr_hi;
2156
2157	/* do an INIT IPI: assert RESET */
2158	icr_lo = lapic.icr_lo & 0xfff00000;
2159	lapic.icr_lo = icr_lo | 0x0000c500;
2160
2161	/* wait for pending status end */
2162	while (lapic.icr_lo & APIC_DELSTAT_MASK)
2163		 /* spin */ ;
2164
2165	/* do an INIT IPI: deassert RESET */
2166	lapic.icr_lo = icr_lo | 0x00008500;
2167
2168	/* wait for pending status end */
2169	u_sleep(10000);		/* wait ~10mS */
2170	while (lapic.icr_lo & APIC_DELSTAT_MASK)
2171		 /* spin */ ;
2172
2173	/*
2174	 * next we do a STARTUP IPI: the previous INIT IPI might still be
2175	 * latched, (P5 bug) this 1st STARTUP would then terminate
2176	 * immediately, and the previously started INIT IPI would continue. OR
2177	 * the previous INIT IPI has already run. and this STARTUP IPI will
2178	 * run. OR the previous INIT IPI was ignored. and this STARTUP IPI
2179	 * will run.
2180	 */
2181
2182	/* do a STARTUP IPI */
2183	lapic.icr_lo = icr_lo | 0x00000600 | vector;
2184	while (lapic.icr_lo & APIC_DELSTAT_MASK)
2185		 /* spin */ ;
2186	u_sleep(200);		/* wait ~200uS */
2187
2188	/*
2189	 * finally we do a 2nd STARTUP IPI: this 2nd STARTUP IPI should run IF
2190	 * the previous STARTUP IPI was cancelled by a latched INIT IPI. OR
2191	 * this STARTUP IPI will be ignored, as only ONE STARTUP IPI is
2192	 * recognized after hardware RESET or INIT IPI.
2193	 */
2194
2195	lapic.icr_lo = icr_lo | 0x00000600 | vector;
2196	while (lapic.icr_lo & APIC_DELSTAT_MASK)
2197		 /* spin */ ;
2198	u_sleep(200);		/* wait ~200uS */
2199
2200	/* wait for it to start */
2201	set_apic_timer(5000000);/* == 5 seconds */
2202	while (read_apic_timer())
2203		if (mp_ncpus > cpus)
2204			return 1;	/* return SUCCESS */
2205
2206	return 0;		/* return FAILURE */
2207}
2208
2209/*
2210 * Flush the TLB on all other CPU's
2211 *
2212 * XXX: Needs to handshake and wait for completion before proceding.
2213 */
2214void
2215smp_invltlb(void)
2216{
2217#if defined(APIC_IO)
2218	if (smp_started && invltlb_ok)
2219		all_but_self_ipi(XINVLTLB_OFFSET);
2220#endif  /* APIC_IO */
2221}
2222
2223void
2224invlpg(u_int addr)
2225{
2226	__asm   __volatile("invlpg (%0)"::"r"(addr):"memory");
2227
2228	/* send a message to the other CPUs */
2229	smp_invltlb();
2230}
2231
2232void
2233invltlb(void)
2234{
2235	u_long  temp;
2236
2237	/*
2238	 * This should be implemented as load_cr3(rcr3()) when load_cr3() is
2239	 * inlined.
2240	 */
2241	__asm __volatile("movl %%cr3, %0; movl %0, %%cr3":"=r"(temp) :: "memory");
2242
2243	/* send a message to the other CPUs */
2244	smp_invltlb();
2245}
2246
2247
2248/*
2249 * When called the executing CPU will send an IPI to all other CPUs
2250 *  requesting that they halt execution.
2251 *
2252 * Usually (but not necessarily) called with 'other_cpus' as its arg.
2253 *
2254 *  - Signals all CPUs in map to stop.
2255 *  - Waits for each to stop.
2256 *
2257 * Returns:
2258 *  -1: error
2259 *   0: NA
2260 *   1: ok
2261 *
2262 * XXX FIXME: this is not MP-safe, needs a lock to prevent multiple CPUs
2263 *            from executing at same time.
2264 */
2265int
2266stop_cpus(u_int map)
2267{
2268	if (!smp_started)
2269		return 0;
2270
2271	/* send the Xcpustop IPI to all CPUs in map */
2272	selected_apic_ipi(map, XCPUSTOP_OFFSET, APIC_DELMODE_FIXED);
2273
2274	while ((stopped_cpus & map) != map)
2275		/* spin */ ;
2276
2277	return 1;
2278}
2279
2280
2281/*
2282 * Called by a CPU to restart stopped CPUs.
2283 *
2284 * Usually (but not necessarily) called with 'stopped_cpus' as its arg.
2285 *
2286 *  - Signals all CPUs in map to restart.
2287 *  - Waits for each to restart.
2288 *
2289 * Returns:
2290 *  -1: error
2291 *   0: NA
2292 *   1: ok
2293 */
2294int
2295restart_cpus(u_int map)
2296{
2297	if (!smp_started)
2298		return 0;
2299
2300	started_cpus = map;		/* signal other cpus to restart */
2301
2302	while ((stopped_cpus & map) != 0) /* wait for each to clear its bit */
2303		/* spin */ ;
2304
2305	return 1;
2306}
2307
2308int smp_active = 0;	/* are the APs allowed to run? */
2309SYSCTL_INT(_machdep, OID_AUTO, smp_active, CTLFLAG_RW, &smp_active, 0, "");
2310
2311/* XXX maybe should be hw.ncpu */
2312static int smp_cpus = 1;	/* how many cpu's running */
2313SYSCTL_INT(_machdep, OID_AUTO, smp_cpus, CTLFLAG_RD, &smp_cpus, 0, "");
2314
2315int invltlb_ok = 0;	/* throttle smp_invltlb() till safe */
2316SYSCTL_INT(_machdep, OID_AUTO, invltlb_ok, CTLFLAG_RW, &invltlb_ok, 0, "");
2317
2318/* Warning: Do not staticize.  Used from swtch.s */
2319int do_page_zero_idle = 1; /* bzero pages for fun and profit in idleloop */
2320SYSCTL_INT(_machdep, OID_AUTO, do_page_zero_idle, CTLFLAG_RW,
2321	   &do_page_zero_idle, 0, "");
2322
2323/* Is forwarding of a interrupt to the CPU holding the ISR lock enabled ? */
2324int forward_irq_enabled = 1;
2325SYSCTL_INT(_machdep, OID_AUTO, forward_irq_enabled, CTLFLAG_RW,
2326	   &forward_irq_enabled, 0, "");
2327
2328/* Enable forwarding of a signal to a process running on a different CPU */
2329static int forward_signal_enabled = 1;
2330SYSCTL_INT(_machdep, OID_AUTO, forward_signal_enabled, CTLFLAG_RW,
2331	   &forward_signal_enabled, 0, "");
2332
2333/* Enable forwarding of roundrobin to all other cpus */
2334static int forward_roundrobin_enabled = 1;
2335SYSCTL_INT(_machdep, OID_AUTO, forward_roundrobin_enabled, CTLFLAG_RW,
2336	   &forward_roundrobin_enabled, 0, "");
2337
2338/*
2339 * This is called once the rest of the system is up and running and we're
2340 * ready to let the AP's out of the pen.
2341 */
2342void ap_init(void);
2343
2344void
2345ap_init(void)
2346{
2347	u_int	apic_id;
2348
2349	/* lock against other AP's that are waking up */
2350	s_lock(&ap_boot_lock);
2351
2352	/* BSP may have changed PTD while we're waiting for the lock */
2353	cpu_invltlb();
2354
2355	smp_cpus++;
2356
2357#if defined(I586_CPU) && !defined(NO_F00F_HACK)
2358	lidt(&r_idt);
2359#endif
2360
2361	/* Build our map of 'other' CPUs. */
2362	other_cpus = all_cpus & ~(1 << cpuid);
2363
2364	printf("SMP: AP CPU #%d Launched!\n", cpuid);
2365
2366	/* set up CPU registers and state */
2367	cpu_setregs();
2368
2369	/* set up FPU state on the AP */
2370	npxinit(__INITIAL_NPXCW__);
2371
2372	/* A quick check from sanity claus */
2373	apic_id = (apic_id_to_logical[(lapic.id & 0x0f000000) >> 24]);
2374	if (cpuid != apic_id) {
2375		printf("SMP: cpuid = %d\n", cpuid);
2376		printf("SMP: apic_id = %d\n", apic_id);
2377		printf("PTD[MPPTDI] = %p\n", (void *)PTD[MPPTDI]);
2378		panic("cpuid mismatch! boom!!");
2379	}
2380
2381	/* Init local apic for irq's */
2382	apic_initialize();
2383
2384	/* Set memory range attributes for this CPU to match the BSP */
2385	mem_range_AP_init();
2386
2387	/*
2388	 * Activate smp_invltlb, although strictly speaking, this isn't
2389	 * quite correct yet.  We should have a bitfield for cpus willing
2390	 * to accept TLB flush IPI's or something and sync them.
2391	 */
2392	if (smp_cpus == mp_ncpus) {
2393		invltlb_ok = 1;
2394		smp_started = 1; /* enable IPI's, tlb shootdown, freezes etc */
2395		smp_active = 1;	 /* historic */
2396	}
2397
2398	/* let other AP's wake up now */
2399	s_unlock(&ap_boot_lock);
2400
2401	/* wait until all the AP's are up */
2402	while (smp_started == 0)
2403		; /* nothing */
2404
2405	/*
2406	 * Set curproc to our per-cpu idleproc so that mutexes have
2407	 * something unique to lock with.
2408	 */
2409	PCPU_SET(curproc,idleproc);
2410	PCPU_SET(prevproc,idleproc);
2411
2412	microuptime(&switchtime);
2413	switchticks = ticks;
2414
2415	/* ok, now grab sched_lock and enter the scheduler */
2416	enable_intr();
2417	mtx_enter(&sched_lock, MTX_SPIN);
2418	cpu_throw();	/* doesn't return */
2419
2420	panic("scheduler returned us to ap_init");
2421}
2422
2423#ifdef BETTER_CLOCK
2424
2425#define CHECKSTATE_USER	0
2426#define CHECKSTATE_SYS	1
2427#define CHECKSTATE_INTR	2
2428
2429/* Do not staticize.  Used from apic_vector.s */
2430struct proc*	checkstate_curproc[NCPU];
2431int		checkstate_cpustate[NCPU];
2432u_long		checkstate_pc[NCPU];
2433
2434extern long	cp_time[CPUSTATES];
2435
2436#define PC_TO_INDEX(pc, prof)				\
2437        ((int)(((u_quad_t)((pc) - (prof)->pr_off) *	\
2438            (u_quad_t)((prof)->pr_scale)) >> 16) & ~1)
2439
2440static void
2441addupc_intr_forwarded(struct proc *p, int id, int *astmap)
2442{
2443	int i;
2444	struct uprof *prof;
2445	u_long pc;
2446
2447	pc = checkstate_pc[id];
2448	prof = &p->p_stats->p_prof;
2449	if (pc >= prof->pr_off &&
2450	    (i = PC_TO_INDEX(pc, prof)) < prof->pr_size) {
2451		if ((p->p_flag & P_OWEUPC) == 0) {
2452			prof->pr_addr = pc;
2453			prof->pr_ticks = 1;
2454			p->p_flag |= P_OWEUPC;
2455		}
2456		*astmap |= (1 << id);
2457	}
2458}
2459
2460static void
2461forwarded_statclock(int id, int pscnt, int *astmap)
2462{
2463	struct pstats *pstats;
2464	long rss;
2465	struct rusage *ru;
2466	struct vmspace *vm;
2467	int cpustate;
2468	struct proc *p;
2469#ifdef GPROF
2470	register struct gmonparam *g;
2471	int i;
2472#endif
2473
2474	p = checkstate_curproc[id];
2475	cpustate = checkstate_cpustate[id];
2476
2477	/* XXX */
2478	if (p->p_ithd)
2479		cpustate = CHECKSTATE_INTR;
2480	else if (p == idleproc)
2481		cpustate = CHECKSTATE_SYS;
2482
2483	switch (cpustate) {
2484	case CHECKSTATE_USER:
2485		if (p->p_flag & P_PROFIL)
2486			addupc_intr_forwarded(p, id, astmap);
2487		if (pscnt > 1)
2488			return;
2489		p->p_uticks++;
2490		if (p->p_nice > NZERO)
2491			cp_time[CP_NICE]++;
2492		else
2493			cp_time[CP_USER]++;
2494		break;
2495	case CHECKSTATE_SYS:
2496#ifdef GPROF
2497		/*
2498		 * Kernel statistics are just like addupc_intr, only easier.
2499		 */
2500		g = &_gmonparam;
2501		if (g->state == GMON_PROF_ON) {
2502			i = checkstate_pc[id] - g->lowpc;
2503			if (i < g->textsize) {
2504				i /= HISTFRACTION * sizeof(*g->kcount);
2505				g->kcount[i]++;
2506			}
2507		}
2508#endif
2509		if (pscnt > 1)
2510			return;
2511
2512		if (p == idleproc) {
2513			p->p_sticks++;
2514			cp_time[CP_IDLE]++;
2515		} else {
2516			p->p_sticks++;
2517			cp_time[CP_SYS]++;
2518		}
2519		break;
2520	case CHECKSTATE_INTR:
2521	default:
2522#ifdef GPROF
2523		/*
2524		 * Kernel statistics are just like addupc_intr, only easier.
2525		 */
2526		g = &_gmonparam;
2527		if (g->state == GMON_PROF_ON) {
2528			i = checkstate_pc[id] - g->lowpc;
2529			if (i < g->textsize) {
2530				i /= HISTFRACTION * sizeof(*g->kcount);
2531				g->kcount[i]++;
2532			}
2533		}
2534#endif
2535		if (pscnt > 1)
2536			return;
2537		if (p)
2538			p->p_iticks++;
2539		cp_time[CP_INTR]++;
2540	}
2541	if (p != idleproc) {
2542		schedclock(p);
2543
2544		/* Update resource usage integrals and maximums. */
2545		if ((pstats = p->p_stats) != NULL &&
2546		    (ru = &pstats->p_ru) != NULL &&
2547		    (vm = p->p_vmspace) != NULL) {
2548			ru->ru_ixrss += pgtok(vm->vm_tsize);
2549			ru->ru_idrss += pgtok(vm->vm_dsize);
2550			ru->ru_isrss += pgtok(vm->vm_ssize);
2551			rss = pgtok(vmspace_resident_count(vm));
2552			if (ru->ru_maxrss < rss)
2553				ru->ru_maxrss = rss;
2554        	}
2555	}
2556}
2557
2558void
2559forward_statclock(int pscnt)
2560{
2561	int map;
2562	int id;
2563	int i;
2564
2565	/* Kludge. We don't yet have separate locks for the interrupts
2566	 * and the kernel. This means that we cannot let the other processors
2567	 * handle complex interrupts while inhibiting them from entering
2568	 * the kernel in a non-interrupt context.
2569	 *
2570	 * What we can do, without changing the locking mechanisms yet,
2571	 * is letting the other processors handle a very simple interrupt
2572	 * (wich determines the processor states), and do the main
2573	 * work ourself.
2574	 */
2575
2576	if (!smp_started || !invltlb_ok || cold || panicstr)
2577		return;
2578
2579	/* Step 1: Probe state   (user, cpu, interrupt, spinlock, idle ) */
2580
2581	map = other_cpus & ~stopped_cpus ;
2582	checkstate_probed_cpus = 0;
2583	if (map != 0)
2584		selected_apic_ipi(map,
2585				  XCPUCHECKSTATE_OFFSET, APIC_DELMODE_FIXED);
2586
2587	i = 0;
2588	while (checkstate_probed_cpus != map) {
2589		/* spin */
2590		i++;
2591		if (i == 100000) {
2592#ifdef BETTER_CLOCK_DIAGNOSTIC
2593			printf("forward_statclock: checkstate %x\n",
2594			       checkstate_probed_cpus);
2595#endif
2596			break;
2597		}
2598	}
2599
2600	/*
2601	 * Step 2: walk through other processors processes, update ticks and
2602	 * profiling info.
2603	 */
2604
2605	map = 0;
2606	for (id = 0; id < mp_ncpus; id++) {
2607		if (id == cpuid)
2608			continue;
2609		if (((1 << id) & checkstate_probed_cpus) == 0)
2610			continue;
2611		forwarded_statclock(id, pscnt, &map);
2612	}
2613	if (map != 0) {
2614		checkstate_need_ast |= map;
2615		selected_apic_ipi(map, XCPUAST_OFFSET, APIC_DELMODE_FIXED);
2616		i = 0;
2617		while ((checkstate_need_ast & map) != 0) {
2618			/* spin */
2619			i++;
2620			if (i > 100000) {
2621#ifdef BETTER_CLOCK_DIAGNOSTIC
2622				printf("forward_statclock: dropped ast 0x%x\n",
2623				       checkstate_need_ast & map);
2624#endif
2625				break;
2626			}
2627		}
2628	}
2629}
2630
2631void
2632forward_hardclock(int pscnt)
2633{
2634	int map;
2635	int id;
2636	struct proc *p;
2637	struct pstats *pstats;
2638	int i;
2639
2640	/* Kludge. We don't yet have separate locks for the interrupts
2641	 * and the kernel. This means that we cannot let the other processors
2642	 * handle complex interrupts while inhibiting them from entering
2643	 * the kernel in a non-interrupt context.
2644	 *
2645	 * What we can do, without changing the locking mechanisms yet,
2646	 * is letting the other processors handle a very simple interrupt
2647	 * (wich determines the processor states), and do the main
2648	 * work ourself.
2649	 */
2650
2651	if (!smp_started || !invltlb_ok || cold || panicstr)
2652		return;
2653
2654	/* Step 1: Probe state   (user, cpu, interrupt, spinlock, idle) */
2655
2656	map = other_cpus & ~stopped_cpus ;
2657	checkstate_probed_cpus = 0;
2658	if (map != 0)
2659		selected_apic_ipi(map,
2660				  XCPUCHECKSTATE_OFFSET, APIC_DELMODE_FIXED);
2661
2662	i = 0;
2663	while (checkstate_probed_cpus != map) {
2664		/* spin */
2665		i++;
2666		if (i == 100000) {
2667#ifdef BETTER_CLOCK_DIAGNOSTIC
2668			printf("forward_hardclock: checkstate %x\n",
2669			       checkstate_probed_cpus);
2670#endif
2671			break;
2672		}
2673	}
2674
2675	/*
2676	 * Step 2: walk through other processors processes, update virtual
2677	 * timer and profiling timer. If stathz == 0, also update ticks and
2678	 * profiling info.
2679	 */
2680
2681	map = 0;
2682	for (id = 0; id < mp_ncpus; id++) {
2683		if (id == cpuid)
2684			continue;
2685		if (((1 << id) & checkstate_probed_cpus) == 0)
2686			continue;
2687		p = checkstate_curproc[id];
2688		if (p) {
2689			pstats = p->p_stats;
2690			if (checkstate_cpustate[id] == CHECKSTATE_USER &&
2691			    timevalisset(&pstats->p_timer[ITIMER_VIRTUAL].it_value) &&
2692			    itimerdecr(&pstats->p_timer[ITIMER_VIRTUAL], tick) == 0) {
2693				psignal(p, SIGVTALRM);
2694				map |= (1 << id);
2695			}
2696			if (timevalisset(&pstats->p_timer[ITIMER_PROF].it_value) &&
2697			    itimerdecr(&pstats->p_timer[ITIMER_PROF], tick) == 0) {
2698				psignal(p, SIGPROF);
2699				map |= (1 << id);
2700			}
2701		}
2702		if (stathz == 0) {
2703			forwarded_statclock( id, pscnt, &map);
2704		}
2705	}
2706	if (map != 0) {
2707		checkstate_need_ast |= map;
2708		selected_apic_ipi(map, XCPUAST_OFFSET, APIC_DELMODE_FIXED);
2709		i = 0;
2710		while ((checkstate_need_ast & map) != 0) {
2711			/* spin */
2712			i++;
2713			if (i > 100000) {
2714#ifdef BETTER_CLOCK_DIAGNOSTIC
2715				printf("forward_hardclock: dropped ast 0x%x\n",
2716				       checkstate_need_ast & map);
2717#endif
2718				break;
2719			}
2720		}
2721	}
2722}
2723
2724#endif /* BETTER_CLOCK */
2725
2726void
2727forward_signal(struct proc *p)
2728{
2729	int map;
2730	int id;
2731	int i;
2732
2733	/* Kludge. We don't yet have separate locks for the interrupts
2734	 * and the kernel. This means that we cannot let the other processors
2735	 * handle complex interrupts while inhibiting them from entering
2736	 * the kernel in a non-interrupt context.
2737	 *
2738	 * What we can do, without changing the locking mechanisms yet,
2739	 * is letting the other processors handle a very simple interrupt
2740	 * (wich determines the processor states), and do the main
2741	 * work ourself.
2742	 */
2743
2744	if (!smp_started || !invltlb_ok || cold || panicstr)
2745		return;
2746	if (!forward_signal_enabled)
2747		return;
2748	while (1) {
2749		if (p->p_stat != SRUN)
2750			return;
2751		id = p->p_oncpu;
2752		if (id == 0xff)
2753			return;
2754		map = (1<<id);
2755		checkstate_need_ast |= map;
2756		selected_apic_ipi(map, XCPUAST_OFFSET, APIC_DELMODE_FIXED);
2757		i = 0;
2758		while ((checkstate_need_ast & map) != 0) {
2759			/* spin */
2760			i++;
2761			if (i > 100000) {
2762#if 0
2763				printf("forward_signal: dropped ast 0x%x\n",
2764				       checkstate_need_ast & map);
2765#endif
2766				break;
2767			}
2768		}
2769		if (id == p->p_oncpu)
2770			return;
2771	}
2772}
2773
2774void
2775forward_roundrobin(void)
2776{
2777	u_int map;
2778	int i;
2779
2780	if (!smp_started || !invltlb_ok || cold || panicstr)
2781		return;
2782	if (!forward_roundrobin_enabled)
2783		return;
2784	resched_cpus |= other_cpus;
2785	map = other_cpus & ~stopped_cpus ;
2786#if 1
2787	selected_apic_ipi(map, XCPUAST_OFFSET, APIC_DELMODE_FIXED);
2788#else
2789	(void) all_but_self_ipi(XCPUAST_OFFSET);
2790#endif
2791	i = 0;
2792	while ((checkstate_need_ast & map) != 0) {
2793		/* spin */
2794		i++;
2795		if (i > 100000) {
2796#if 0
2797			printf("forward_roundrobin: dropped ast 0x%x\n",
2798			       checkstate_need_ast & map);
2799#endif
2800			break;
2801		}
2802	}
2803}
2804
2805
2806#ifdef APIC_INTR_REORDER
2807/*
2808 *	Maintain mapping from softintr vector to isr bit in local apic.
2809 */
2810void
2811set_lapic_isrloc(int intr, int vector)
2812{
2813	if (intr < 0 || intr > 32)
2814		panic("set_apic_isrloc: bad intr argument: %d",intr);
2815	if (vector < ICU_OFFSET || vector > 255)
2816		panic("set_apic_isrloc: bad vector argument: %d",vector);
2817	apic_isrbit_location[intr].location = &lapic.isr0 + ((vector>>5)<<2);
2818	apic_isrbit_location[intr].bit = (1<<(vector & 31));
2819}
2820#endif
2821
2822/*
2823 * All-CPU rendezvous.  CPUs are signalled, all execute the setup function
2824 * (if specified), rendezvous, execute the action function (if specified),
2825 * rendezvous again, execute the teardown function (if specified), and then
2826 * resume.
2827 *
2828 * Note that the supplied external functions _must_ be reentrant and aware
2829 * that they are running in parallel and in an unknown lock context.
2830 */
2831static void (*smp_rv_setup_func)(void *arg);
2832static void (*smp_rv_action_func)(void *arg);
2833static void (*smp_rv_teardown_func)(void *arg);
2834static void *smp_rv_func_arg;
2835static volatile int smp_rv_waiters[2];
2836
2837void
2838smp_rendezvous_action(void)
2839{
2840	/* setup function */
2841	if (smp_rv_setup_func != NULL)
2842		smp_rv_setup_func(smp_rv_func_arg);
2843	/* spin on entry rendezvous */
2844	atomic_add_int(&smp_rv_waiters[0], 1);
2845	while (smp_rv_waiters[0] < mp_ncpus)
2846		;
2847	/* action function */
2848	if (smp_rv_action_func != NULL)
2849		smp_rv_action_func(smp_rv_func_arg);
2850	/* spin on exit rendezvous */
2851	atomic_add_int(&smp_rv_waiters[1], 1);
2852	while (smp_rv_waiters[1] < mp_ncpus)
2853		;
2854	/* teardown function */
2855	if (smp_rv_teardown_func != NULL)
2856		smp_rv_teardown_func(smp_rv_func_arg);
2857}
2858
2859void
2860smp_rendezvous(void (* setup_func)(void *),
2861	       void (* action_func)(void *),
2862	       void (* teardown_func)(void *),
2863	       void *arg)
2864{
2865	u_int	efl;
2866
2867	/* obtain rendezvous lock */
2868	s_lock(&smp_rv_lock);		/* XXX sleep here? NOWAIT flag? */
2869
2870	/* set static function pointers */
2871	smp_rv_setup_func = setup_func;
2872	smp_rv_action_func = action_func;
2873	smp_rv_teardown_func = teardown_func;
2874	smp_rv_func_arg = arg;
2875	smp_rv_waiters[0] = 0;
2876	smp_rv_waiters[1] = 0;
2877
2878	/* disable interrupts on this CPU, save interrupt status */
2879	efl = read_eflags();
2880	write_eflags(efl & ~PSL_I);
2881
2882	/* signal other processors, which will enter the IPI with interrupts off */
2883	all_but_self_ipi(XRENDEZVOUS_OFFSET);
2884
2885	/* call executor function */
2886	smp_rendezvous_action();
2887
2888	/* restore interrupt flag */
2889	write_eflags(efl);
2890
2891	/* release lock */
2892	s_unlock(&smp_rv_lock);
2893}
2894
2895void
2896release_aps(void *dummy __unused)
2897{
2898	s_unlock(&ap_boot_lock);
2899}
2900
2901SYSINIT(start_aps, SI_SUB_SMP, SI_ORDER_FIRST, release_aps, NULL);
2902