26 */
27
28#include "opt_smp.h"
29#include "opt_vm86.h"
30#include "opt_cpu.h"
31
32#ifdef SMP
33#include <machine/smptests.h>
34#else
35#error
36#endif
37
38#include <sys/param.h>
39#include <sys/systm.h>
40#include <sys/kernel.h>
41#include <sys/proc.h>
42#include <sys/sysctl.h>
43#ifdef BETTER_CLOCK
44#include <sys/dkstat.h>
45#endif
46
47#include <vm/vm.h>
48#include <vm/vm_param.h>
49#include <vm/pmap.h>
50#include <vm/vm_kern.h>
51#include <vm/vm_extern.h>
52#ifdef BETTER_CLOCK
53#include <sys/lock.h>
54#include <vm/vm_map.h>
55#include <sys/user.h>
56#ifdef GPROF
57#include <sys/gmon.h>
58#endif
59#endif
60
61#include <machine/smp.h>
62#include <machine/apic.h>
63#include <machine/mpapic.h>
64#include <machine/segments.h>
65#include <machine/smptests.h> /** TEST_DEFAULT_CONFIG, TEST_TEST1 */
66#include <machine/tss.h>
67#include <machine/specialreg.h>
68#include <machine/cputypes.h>
69#include <machine/globaldata.h>
70
71#include <i386/i386/cons.h> /* cngetc() */
72
73#if defined(APIC_IO)
74#include <machine/md_var.h> /* setidt() */
75#include <i386/isa/icu.h> /* IPIs */
76#include <i386/isa/intr_machdep.h> /* IPIs */
77#endif /* APIC_IO */
78
79#if defined(TEST_DEFAULT_CONFIG)
80#define MPFPS_MPFB1 TEST_DEFAULT_CONFIG
81#else
82#define MPFPS_MPFB1 mpfps->mpfb1
83#endif /* TEST_DEFAULT_CONFIG */
84
85#define WARMBOOT_TARGET 0
86#define WARMBOOT_OFF (KERNBASE + 0x0467)
87#define WARMBOOT_SEG (KERNBASE + 0x0469)
88
89#define BIOS_BASE (0xf0000)
90#define BIOS_SIZE (0x10000)
91#define BIOS_COUNT (BIOS_SIZE/4)
92
93#define CMOS_REG (0x70)
94#define CMOS_DATA (0x71)
95#define BIOS_RESET (0x0f)
96#define BIOS_WARM (0x0a)
97
98#define PROCENTRY_FLAG_EN 0x01
99#define PROCENTRY_FLAG_BP 0x02
100#define IOAPICENTRY_FLAG_EN 0x01
101
102
103/* MP Floating Pointer Structure */
104typedef struct MPFPS {
105 char signature[4];
106 void *pap;
107 u_char length;
108 u_char spec_rev;
109 u_char checksum;
110 u_char mpfb1;
111 u_char mpfb2;
112 u_char mpfb3;
113 u_char mpfb4;
114 u_char mpfb5;
115} *mpfps_t;
116
117/* MP Configuration Table Header */
118typedef struct MPCTH {
119 char signature[4];
120 u_short base_table_length;
121 u_char spec_rev;
122 u_char checksum;
123 u_char oem_id[8];
124 u_char product_id[12];
125 void *oem_table_pointer;
126 u_short oem_table_size;
127 u_short entry_count;
128 void *apic_address;
129 u_short extended_table_length;
130 u_char extended_table_checksum;
131 u_char reserved;
132} *mpcth_t;
133
134
135typedef struct PROCENTRY {
136 u_char type;
137 u_char apic_id;
138 u_char apic_version;
139 u_char cpu_flags;
140 u_long cpu_signature;
141 u_long feature_flags;
142 u_long reserved1;
143 u_long reserved2;
144} *proc_entry_ptr;
145
146typedef struct BUSENTRY {
147 u_char type;
148 u_char bus_id;
149 char bus_type[6];
150} *bus_entry_ptr;
151
152typedef struct IOAPICENTRY {
153 u_char type;
154 u_char apic_id;
155 u_char apic_version;
156 u_char apic_flags;
157 void *apic_address;
158} *io_apic_entry_ptr;
159
160typedef struct INTENTRY {
161 u_char type;
162 u_char int_type;
163 u_short int_flags;
164 u_char src_bus_id;
165 u_char src_bus_irq;
166 u_char dst_apic_id;
167 u_char dst_apic_int;
168} *int_entry_ptr;
169
170/* descriptions of MP basetable entries */
171typedef struct BASETABLE_ENTRY {
172 u_char type;
173 u_char length;
174 char name[16];
175} basetable_entry;
176
177/*
178 * this code MUST be enabled here and in mpboot.s.
179 * it follows the very early stages of AP boot by placing values in CMOS ram.
180 * it NORMALLY will never be needed and thus the primitive method for enabling.
181 *
182#define CHECK_POINTS
183 */
184
185#if defined(CHECK_POINTS)
186#define CHECK_READ(A) (outb(CMOS_REG, (A)), inb(CMOS_DATA))
187#define CHECK_WRITE(A,D) (outb(CMOS_REG, (A)), outb(CMOS_DATA, (D)))
188
189#define CHECK_INIT(D); \
190 CHECK_WRITE(0x34, (D)); \
191 CHECK_WRITE(0x35, (D)); \
192 CHECK_WRITE(0x36, (D)); \
193 CHECK_WRITE(0x37, (D)); \
194 CHECK_WRITE(0x38, (D)); \
195 CHECK_WRITE(0x39, (D));
196
197#define CHECK_PRINT(S); \
198 printf("%s: %d, %d, %d, %d, %d, %d\n", \
199 (S), \
200 CHECK_READ(0x34), \
201 CHECK_READ(0x35), \
202 CHECK_READ(0x36), \
203 CHECK_READ(0x37), \
204 CHECK_READ(0x38), \
205 CHECK_READ(0x39));
206
207#else /* CHECK_POINTS */
208
209#define CHECK_INIT(D)
210#define CHECK_PRINT(S)
211
212#endif /* CHECK_POINTS */
213
214/*
215 * Values to send to the POST hardware.
216 */
217#define MP_BOOTADDRESS_POST 0x10
218#define MP_PROBE_POST 0x11
219#define MPTABLE_PASS1_POST 0x12
220
221#define MP_START_POST 0x13
222#define MP_ENABLE_POST 0x14
223#define MPTABLE_PASS2_POST 0x15
224
225#define START_ALL_APS_POST 0x16
226#define INSTALL_AP_TRAMP_POST 0x17
227#define START_AP_POST 0x18
228
229#define MP_ANNOUNCE_POST 0x19
230
231
232/** XXX FIXME: where does this really belong, isa.h/isa.c perhaps? */
233int current_postcode;
234
235/** XXX FIXME: what system files declare these??? */
236extern struct region_descriptor r_gdt, r_idt;
237
238int bsp_apic_ready = 0; /* flags useability of BSP apic */
239int mp_ncpus; /* # of CPUs, including BSP */
240int mp_naps; /* # of Applications processors */
241int mp_nbusses; /* # of busses */
242int mp_napics; /* # of IO APICs */
243int boot_cpu_id; /* designated BSP */
244vm_offset_t cpu_apic_address;
245vm_offset_t io_apic_address[NAPICID]; /* NAPICID is more than enough */
246extern int nkpt;
247
248u_int32_t cpu_apic_versions[NCPU];
249u_int32_t io_apic_versions[NAPIC];
250
251#ifdef APIC_INTR_DIAGNOSTIC
252int apic_itrace_enter[32];
253int apic_itrace_tryisrlock[32];
254int apic_itrace_gotisrlock[32];
255int apic_itrace_active[32];
256int apic_itrace_masked[32];
257int apic_itrace_noisrlock[32];
258int apic_itrace_masked2[32];
259int apic_itrace_unmask[32];
260int apic_itrace_noforward[32];
261int apic_itrace_leave[32];
262int apic_itrace_enter2[32];
263int apic_itrace_doreti[32];
264int apic_itrace_splz[32];
265int apic_itrace_eoi[32];
266#ifdef APIC_INTR_DIAGNOSTIC_IRQ
267unsigned short apic_itrace_debugbuffer[32768];
268int apic_itrace_debugbuffer_idx;
269struct simplelock apic_itrace_debuglock;
270#endif
271#endif
272
273#ifdef APIC_INTR_REORDER
274struct {
275 volatile int *location;
276 int bit;
277} apic_isrbit_location[32];
278#endif
279
280/*
281 * APIC ID logical/physical mapping structures.
282 * We oversize these to simplify boot-time config.
283 */
284int cpu_num_to_apic_id[NAPICID];
285int io_num_to_apic_id[NAPICID];
286int apic_id_to_logical[NAPICID];
287
288
289/* Bitmap of all available CPUs */
290u_int all_cpus;
291
292/* AP uses this PTD during bootstrap. Do not staticize. */
293pd_entry_t *bootPTD;
294
295/* Hotwire a 0->4MB V==P mapping */
296extern pt_entry_t *KPTphys;
297
298/* Virtual address of per-cpu common_tss */
299extern struct i386tss common_tss;
300#ifdef VM86
301extern struct segment_descriptor common_tssd;
302extern u_int private_tss; /* flag indicating private tss */
303extern u_int my_tr;
304#endif /* VM86 */
305
306/* IdlePTD per cpu */
307pd_entry_t *IdlePTDS[NCPU];
308
309/* "my" private page table page, for BSP init */
310extern pt_entry_t SMP_prvpt[];
311
312/* Private page pointer to curcpu's PTD, used during BSP init */
313extern pd_entry_t *my_idlePTD;
314
315static int smp_started; /* has the system started? */
316
317/*
318 * Local data and functions.
319 */
320
321static int mp_capable;
322static u_int boot_address;
323static u_int base_memory;
324
325static int picmode; /* 0: virtual wire mode, 1: PIC mode */
326static mpfps_t mpfps;
327static int search_for_sig(u_int32_t target, int count);
328static void mp_enable(u_int boot_addr);
329
330static int mptable_pass1(void);
331static int mptable_pass2(void);
332static void default_mp_table(int type);
333static void fix_mp_table(void);
334static void init_locks(void);
335static int start_all_aps(u_int boot_addr);
336static void install_ap_tramp(u_int boot_addr);
337static int start_ap(int logicalCpu, u_int boot_addr);
338
339/*
340 * Calculate usable address in base memory for AP trampoline code.
341 */
342u_int
343mp_bootaddress(u_int basemem)
344{
345 POSTCODE(MP_BOOTADDRESS_POST);
346
347 base_memory = basemem * 1024; /* convert to bytes */
348
349 boot_address = base_memory & ~0xfff; /* round down to 4k boundary */
350 if ((base_memory - boot_address) < bootMP_size)
351 boot_address -= 4096; /* not enough, lower by 4k */
352
353 return boot_address;
354}
355
356
357/*
358 * Look for an Intel MP spec table (ie, SMP capable hardware).
359 */
360int
361mp_probe(void)
362{
363 int x;
364 u_long segment;
365 u_int32_t target;
366
367 POSTCODE(MP_PROBE_POST);
368
369 /* see if EBDA exists */
370 if (segment = (u_long) * (u_short *) (KERNBASE + 0x40e)) {
371 /* search first 1K of EBDA */
372 target = (u_int32_t) (segment << 4);
373 if ((x = search_for_sig(target, 1024 / 4)) >= 0)
374 goto found;
375 } else {
376 /* last 1K of base memory, effective 'top of base' passed in */
377 target = (u_int32_t) (base_memory - 0x400);
378 if ((x = search_for_sig(target, 1024 / 4)) >= 0)
379 goto found;
380 }
381
382 /* search the BIOS */
383 target = (u_int32_t) BIOS_BASE;
384 if ((x = search_for_sig(target, BIOS_COUNT)) >= 0)
385 goto found;
386
387 /* nothing found */
388 mpfps = (mpfps_t)0;
389 mp_capable = 0;
390 return 0;
391
392found:
393 /* calculate needed resources */
394 mpfps = (mpfps_t)x;
395 if (mptable_pass1())
396 panic("you must reconfigure your kernel");
397
398 /* flag fact that we are running multiple processors */
399 mp_capable = 1;
400 return 1;
401}
402
403
404/*
405 * Startup the SMP processors.
406 */
407void
408mp_start(void)
409{
410 POSTCODE(MP_START_POST);
411
412 /* look for MP capable motherboard */
413 if (mp_capable)
414 mp_enable(boot_address);
415 else
416 panic("MP hardware not found!");
417}
418
419
420/*
421 * Print various information about the SMP system hardware and setup.
422 */
423void
424mp_announce(void)
425{
426 int x;
427
428 POSTCODE(MP_ANNOUNCE_POST);
429
430 printf("FreeBSD/SMP: Multiprocessor motherboard\n");
431 printf(" cpu0 (BSP): apic id: %2d", CPU_TO_ID(0));
432 printf(", version: 0x%08x", cpu_apic_versions[0]);
433 printf(", at 0x%08x\n", cpu_apic_address);
434 for (x = 1; x <= mp_naps; ++x) {
435 printf(" cpu%d (AP): apic id: %2d", x, CPU_TO_ID(x));
436 printf(", version: 0x%08x", cpu_apic_versions[x]);
437 printf(", at 0x%08x\n", cpu_apic_address);
438 }
439
440#if defined(APIC_IO)
441 for (x = 0; x < mp_napics; ++x) {
442 printf(" io%d (APIC): apic id: %2d", x, IO_TO_ID(x));
443 printf(", version: 0x%08x", io_apic_versions[x]);
444 printf(", at 0x%08x\n", io_apic_address[x]);
445 }
446#else
447 printf(" Warning: APIC I/O disabled\n");
448#endif /* APIC_IO */
449}
450
451/*
452 * AP cpu's call this to sync up protected mode.
453 */
454void
455init_secondary(void)
456{
457 int gsel_tss;
458#ifndef VM86
459 u_int my_tr;
460#endif
461
462 r_gdt.rd_limit = sizeof(gdt[0]) * (NGDT + NCPU) - 1;
463 r_gdt.rd_base = (int) gdt;
464 lgdt(&r_gdt); /* does magic intra-segment return */
465 lidt(&r_idt);
466 lldt(_default_ldt);
467
468 my_tr = NGDT + cpuid;
469 gsel_tss = GSEL(my_tr, SEL_KPL);
470 gdt[my_tr].sd.sd_type = SDT_SYS386TSS;
471 common_tss.tss_esp0 = 0; /* not used until after switch */
472 common_tss.tss_ss0 = GSEL(GDATA_SEL, SEL_KPL);
473 common_tss.tss_ioopt = (sizeof common_tss) << 16;
474#ifdef VM86
475 common_tssd = gdt[my_tr].sd;
476 private_tss = 0;
477#endif /* VM86 */
478 ltr(gsel_tss);
479
480 load_cr0(0x8005003b); /* XXX! */
481
482 PTD[0] = 0;
483 pmap_set_opt((unsigned *)PTD);
484
485 putmtrr();
486 pmap_setvidram();
487
488 invltlb();
489}
490
491
492#if defined(APIC_IO)
493/*
494 * Final configuration of the BSP's local APIC:
495 * - disable 'pic mode'.
496 * - disable 'virtual wire mode'.
497 * - enable NMI.
498 */
499void
500bsp_apic_configure(void)
501{
502 u_char byte;
503 u_int32_t temp;
504
505 /* leave 'pic mode' if necessary */
506 if (picmode) {
507 outb(0x22, 0x70); /* select IMCR */
508 byte = inb(0x23); /* current contents */
509 byte |= 0x01; /* mask external INTR */
510 outb(0x23, byte); /* disconnect 8259s/NMI */
511 }
512
513 /* mask lint0 (the 8259 'virtual wire' connection) */
514 temp = lapic.lvt_lint0;
515 temp |= APIC_LVT_M; /* set the mask */
516 lapic.lvt_lint0 = temp;
517
518 /* setup lint1 to handle NMI */
519 temp = lapic.lvt_lint1;
520 temp &= ~APIC_LVT_M; /* clear the mask */
521 lapic.lvt_lint1 = temp;
522
523 if (bootverbose)
524 apic_dump("bsp_apic_configure()");
525}
526#endif /* APIC_IO */
527
528
529/*******************************************************************
530 * local functions and data
531 */
532
533/*
534 * start the SMP system
535 */
536static void
537mp_enable(u_int boot_addr)
538{
539 int x;
540#if defined(APIC_IO)
541 int apic;
542 u_int ux;
543#endif /* APIC_IO */
544
545 getmtrr();
546 pmap_setvidram();
547
548 POSTCODE(MP_ENABLE_POST);
549
550 /* turn on 4MB of V == P addressing so we can get to MP table */
551 *(int *)PTD = PG_V | PG_RW | ((u_long)KPTphys & PG_FRAME);
552 invltlb();
553
554 /* examine the MP table for needed info, uses physical addresses */
555 x = mptable_pass2();
556
557 *(int *)PTD = 0;
558 invltlb();
559
560 /* can't process default configs till the CPU APIC is pmapped */
561 if (x)
562 default_mp_table(x);
563
564 /* post scan cleanup */
565 fix_mp_table();
566
567#if defined(APIC_IO)
568
569 /* fill the LOGICAL io_apic_versions table */
570 for (apic = 0; apic < mp_napics; ++apic) {
571 ux = io_apic_read(apic, IOAPIC_VER);
572 io_apic_versions[apic] = ux;
573 }
574
575 /* program each IO APIC in the system */
576 for (apic = 0; apic < mp_napics; ++apic)
577 if (io_apic_setup(apic) < 0)
578 panic("IO APIC setup failure");
579
580 /* install a 'Spurious INTerrupt' vector */
581 setidt(XSPURIOUSINT_OFFSET, Xspuriousint,
582 SDT_SYS386IGT, SEL_KPL, GSEL(GCODE_SEL, SEL_KPL));
583
584 /* install an inter-CPU IPI for TLB invalidation */
585 setidt(XINVLTLB_OFFSET, Xinvltlb,
586 SDT_SYS386IGT, SEL_KPL, GSEL(GCODE_SEL, SEL_KPL));
587
588#ifdef BETTER_CLOCK
589 /* install an inter-CPU IPI for reading processor state */
590 setidt(XCPUCHECKSTATE_OFFSET, Xcpucheckstate,
591 SDT_SYS386IGT, SEL_KPL, GSEL(GCODE_SEL, SEL_KPL));
592#endif
593
594 /* install an inter-CPU IPI for forcing an additional software trap */
595 setidt(XCPUAST_OFFSET, Xcpuast,
596 SDT_SYS386IGT, SEL_KPL, GSEL(GCODE_SEL, SEL_KPL));
597
598 /* install an inter-CPU IPI for interrupt forwarding */
599 setidt(XFORWARD_IRQ_OFFSET, Xforward_irq,
600 SDT_SYS386IGT, SEL_KPL, GSEL(GCODE_SEL, SEL_KPL));
601
602 /* install an inter-CPU IPI for CPU stop/restart */
603 setidt(XCPUSTOP_OFFSET, Xcpustop,
604 SDT_SYS386IGT, SEL_KPL, GSEL(GCODE_SEL, SEL_KPL));
605
606#if defined(TEST_TEST1)
607 /* install a "fake hardware INTerrupt" vector */
608 setidt(XTEST1_OFFSET, Xtest1,
609 SDT_SYS386IGT, SEL_KPL, GSEL(GCODE_SEL, SEL_KPL));
610#endif /** TEST_TEST1 */
611
612#endif /* APIC_IO */
613
614 /* initialize all SMP locks */
615 init_locks();
616
617 /* start each Application Processor */
618 start_all_aps(boot_addr);
619
620 /*
621 * The init process might be started on a different CPU now,
622 * and the boot CPU might not call prepare_usermode to get
623 * cr0 correctly configured. Thus we initialize cr0 here.
624 */
625 load_cr0(rcr0() | CR0_WP | CR0_AM);
626}
627
628
629/*
630 * look for the MP spec signature
631 */
632
633/* string defined by the Intel MP Spec as identifying the MP table */
634#define MP_SIG 0x5f504d5f /* _MP_ */
635#define NEXT(X) ((X) += 4)
636static int
637search_for_sig(u_int32_t target, int count)
638{
639 int x;
640 u_int32_t *addr = (u_int32_t *) (KERNBASE + target);
641
642 for (x = 0; x < count; NEXT(x))
643 if (addr[x] == MP_SIG)
644 /* make array index a byte index */
645 return (target + (x * sizeof(u_int32_t)));
646
647 return -1;
648}
649
650
651static basetable_entry basetable_entry_types[] =
652{
653 {0, 20, "Processor"},
654 {1, 8, "Bus"},
655 {2, 8, "I/O APIC"},
656 {3, 8, "I/O INT"},
657 {4, 8, "Local INT"}
658};
659
660typedef struct BUSDATA {
661 u_char bus_id;
662 enum busTypes bus_type;
663} bus_datum;
664
665typedef struct INTDATA {
666 u_char int_type;
667 u_short int_flags;
668 u_char src_bus_id;
669 u_char src_bus_irq;
670 u_char dst_apic_id;
671 u_char dst_apic_int;
672} io_int, local_int;
673
674typedef struct BUSTYPENAME {
675 u_char type;
676 char name[7];
677} bus_type_name;
678
679static bus_type_name bus_type_table[] =
680{
681 {CBUS, "CBUS"},
682 {CBUSII, "CBUSII"},
683 {EISA, "EISA"},
684 {UNKNOWN_BUSTYPE, "---"},
685 {UNKNOWN_BUSTYPE, "---"},
686 {ISA, "ISA"},
687 {UNKNOWN_BUSTYPE, "---"},
688 {UNKNOWN_BUSTYPE, "---"},
689 {UNKNOWN_BUSTYPE, "---"},
690 {UNKNOWN_BUSTYPE, "---"},
691 {UNKNOWN_BUSTYPE, "---"},
692 {UNKNOWN_BUSTYPE, "---"},
693 {PCI, "PCI"},
694 {UNKNOWN_BUSTYPE, "---"},
695 {UNKNOWN_BUSTYPE, "---"},
696 {UNKNOWN_BUSTYPE, "---"},
697 {UNKNOWN_BUSTYPE, "---"},
698 {XPRESS, "XPRESS"},
699 {UNKNOWN_BUSTYPE, "---"}
700};
701/* from MP spec v1.4, table 5-1 */
702static int default_data[7][5] =
703{
704/* nbus, id0, type0, id1, type1 */
705 {1, 0, ISA, 255, 255},
706 {1, 0, EISA, 255, 255},
707 {1, 0, EISA, 255, 255},
708 {0, 255, 255, 255, 255},/* MCA not supported */
709 {2, 0, ISA, 1, PCI},
710 {2, 0, EISA, 1, PCI},
711 {0, 255, 255, 255, 255} /* MCA not supported */
712};
713
714
715/* the bus data */
716static bus_datum bus_data[NBUS];
717
718/* the IO INT data, one entry per possible APIC INTerrupt */
719static io_int io_apic_ints[NINTR];
720
721static int nintrs;
722
723static int processor_entry __P((proc_entry_ptr entry, int cpu));
724static int bus_entry __P((bus_entry_ptr entry, int bus));
725static int io_apic_entry __P((io_apic_entry_ptr entry, int apic));
726static int int_entry __P((int_entry_ptr entry, int intr));
727static int lookup_bus_type __P((char *name));
728
729
730/*
731 * 1st pass on motherboard's Intel MP specification table.
732 *
733 * initializes:
734 * mp_ncpus = 1
735 *
736 * determines:
737 * cpu_apic_address (common to all CPUs)
738 * io_apic_address[N]
739 * mp_naps
740 * mp_nbusses
741 * mp_napics
742 * nintrs
743 */
744static int
745mptable_pass1(void)
746{
747 int x;
748 mpcth_t cth;
749 int totalSize;
750 void* position;
751 int count;
752 int type;
753 int mustpanic;
754
755 POSTCODE(MPTABLE_PASS1_POST);
756
757 mustpanic = 0;
758
759 /* clear various tables */
760 for (x = 0; x < NAPICID; ++x) {
761 io_apic_address[x] = ~0; /* IO APIC address table */
762 }
763
764 /* init everything to empty */
765 mp_naps = 0;
766 mp_nbusses = 0;
767 mp_napics = 0;
768 nintrs = 0;
769
770 /* check for use of 'default' configuration */
771 if (MPFPS_MPFB1 != 0) {
772 /* use default addresses */
773 cpu_apic_address = DEFAULT_APIC_BASE;
774 io_apic_address[0] = DEFAULT_IO_APIC_BASE;
775
776 /* fill in with defaults */
777 mp_naps = 2; /* includes BSP */
778 mp_nbusses = default_data[MPFPS_MPFB1 - 1][0];
779#if defined(APIC_IO)
780 mp_napics = 1;
781 nintrs = 16;
782#endif /* APIC_IO */
783 }
784 else {
785 if ((cth = mpfps->pap) == 0)
786 panic("MP Configuration Table Header MISSING!");
787
788 cpu_apic_address = (vm_offset_t) cth->apic_address;
789
790 /* walk the table, recording info of interest */
791 totalSize = cth->base_table_length - sizeof(struct MPCTH);
792 position = (u_char *) cth + sizeof(struct MPCTH);
793 count = cth->entry_count;
794
795 while (count--) {
796 switch (type = *(u_char *) position) {
797 case 0: /* processor_entry */
798 if (((proc_entry_ptr)position)->cpu_flags
799 & PROCENTRY_FLAG_EN)
800 ++mp_naps;
801 break;
802 case 1: /* bus_entry */
803 ++mp_nbusses;
804 break;
805 case 2: /* io_apic_entry */
806 if (((io_apic_entry_ptr)position)->apic_flags
807 & IOAPICENTRY_FLAG_EN)
808 io_apic_address[mp_napics++] =
809 (vm_offset_t)((io_apic_entry_ptr)
810 position)->apic_address;
811 break;
812 case 3: /* int_entry */
813 ++nintrs;
814 break;
815 case 4: /* int_entry */
816 break;
817 default:
818 panic("mpfps Base Table HOSED!");
819 /* NOTREACHED */
820 }
821
822 totalSize -= basetable_entry_types[type].length;
823 (u_char*)position += basetable_entry_types[type].length;
824 }
825 }
826
827 /* qualify the numbers */
828 if (mp_naps > NCPU)
829#if 0 /* XXX FIXME: kern/4255 */
830 printf("Warning: only using %d of %d available CPUs!\n",
831 NCPU, mp_naps);
832#else
833 {
834 printf("NCPU cannot be different than actual CPU count.\n");
835 printf(" add 'options NCPU=%d' to your kernel config file,\n",
836 mp_naps);
837 printf(" then rerun config & rebuild your SMP kernel\n");
838 mustpanic = 1;
839 }
840#endif /* XXX FIXME: kern/4255 */
841 if (mp_nbusses > NBUS) {
842 printf("found %d busses, increase NBUS\n", mp_nbusses);
843 mustpanic = 1;
844 }
845 if (mp_napics > NAPIC) {
846 printf("found %d apics, increase NAPIC\n", mp_napics);
847 mustpanic = 1;
848 }
849 if (nintrs > NINTR) {
850 printf("found %d intrs, increase NINTR\n", nintrs);
851 mustpanic = 1;
852 }
853
854 /*
855 * Count the BSP.
856 * This is also used as a counter while starting the APs.
857 */
858 mp_ncpus = 1;
859
860 --mp_naps; /* subtract the BSP */
861
862 return mustpanic;
863}
864
865
866/*
867 * 2nd pass on motherboard's Intel MP specification table.
868 *
869 * sets:
870 * boot_cpu_id
871 * ID_TO_IO(N), phy APIC ID to log CPU/IO table
872 * CPU_TO_ID(N), logical CPU to APIC ID table
873 * IO_TO_ID(N), logical IO to APIC ID table
874 * bus_data[N]
875 * io_apic_ints[N]
876 */
877static int
878mptable_pass2(void)
879{
880 int x;
881 mpcth_t cth;
882 int totalSize;
883 void* position;
884 int count;
885 int type;
886 int apic, bus, cpu, intr;
887
888 POSTCODE(MPTABLE_PASS2_POST);
889
890 /* clear various tables */
891 for (x = 0; x < NAPICID; ++x) {
892 ID_TO_IO(x) = -1; /* phy APIC ID to log CPU/IO table */
893 CPU_TO_ID(x) = -1; /* logical CPU to APIC ID table */
894 IO_TO_ID(x) = -1; /* logical IO to APIC ID table */
895 }
896
897 /* clear bus data table */
898 for (x = 0; x < NBUS; ++x)
899 bus_data[x].bus_id = 0xff;
900
901 /* clear IO APIC INT table */
902 for (x = 0; x < NINTR; ++x)
903 io_apic_ints[x].int_type = 0xff;
904
905 /* setup the cpu/apic mapping arrays */
906 boot_cpu_id = -1;
907
908 /* record whether PIC or virtual-wire mode */
909 picmode = (mpfps->mpfb2 & 0x80) ? 1 : 0;
910
911 /* check for use of 'default' configuration */
912 if (MPFPS_MPFB1 != 0)
913 return MPFPS_MPFB1; /* return default configuration type */
914
915 if ((cth = mpfps->pap) == 0)
916 panic("MP Configuration Table Header MISSING!");
917
918 /* walk the table, recording info of interest */
919 totalSize = cth->base_table_length - sizeof(struct MPCTH);
920 position = (u_char *) cth + sizeof(struct MPCTH);
921 count = cth->entry_count;
922 apic = bus = intr = 0;
923 cpu = 1; /* pre-count the BSP */
924
925 while (count--) {
926 switch (type = *(u_char *) position) {
927 case 0:
928 if (processor_entry(position, cpu))
929 ++cpu;
930 break;
931 case 1:
932 if (bus_entry(position, bus))
933 ++bus;
934 break;
935 case 2:
936 if (io_apic_entry(position, apic))
937 ++apic;
938 break;
939 case 3:
940 if (int_entry(position, intr))
941 ++intr;
942 break;
943 case 4:
944 /* int_entry(position); */
945 break;
946 default:
947 panic("mpfps Base Table HOSED!");
948 /* NOTREACHED */
949 }
950
951 totalSize -= basetable_entry_types[type].length;
952 (u_char *) position += basetable_entry_types[type].length;
953 }
954
955 if (boot_cpu_id == -1)
956 panic("NO BSP found!");
957
958 /* report fact that its NOT a default configuration */
959 return 0;
960}
961
962
963/*
964 * parse an Intel MP specification table
965 */
966static void
967fix_mp_table(void)
968{
969 int x;
970 int id;
971 int bus_0;
972 int bus_pci;
973 int num_pci_bus;
974
975 /*
976 * Fix mis-numbering of the PCI bus and its INT entries if the BIOS
977 * did it wrong. The MP spec says that when more than 1 PCI bus
978 * exists the BIOS must begin with bus entries for the PCI bus and use
979 * actual PCI bus numbering. This implies that when only 1 PCI bus
980 * exists the BIOS can choose to ignore this ordering, and indeed many
981 * MP motherboards do ignore it. This causes a problem when the PCI
982 * sub-system makes requests of the MP sub-system based on PCI bus
983 * numbers. So here we look for the situation and renumber the
984 * busses and associated INTs in an effort to "make it right".
985 */
986
987 /* find bus 0, PCI bus, count the number of PCI busses */
988 for (num_pci_bus = 0, x = 0; x < mp_nbusses; ++x) {
989 if (bus_data[x].bus_id == 0) {
990 bus_0 = x;
991 }
992 if (bus_data[x].bus_type == PCI) {
993 ++num_pci_bus;
994 bus_pci = x;
995 }
996 }
997 /*
998 * bus_0 == slot of bus with ID of 0
999 * bus_pci == slot of last PCI bus encountered
1000 */
1001
1002 /* check the 1 PCI bus case for sanity */
1003 if (num_pci_bus == 1) {
1004
1005 /* if it is number 0 all is well */
1006 if (bus_data[bus_pci].bus_id == 0)
1007 return;
1008
1009 /* mis-numbered, swap with whichever bus uses slot 0 */
1010
1011 /* swap the bus entry types */
1012 bus_data[bus_pci].bus_type = bus_data[bus_0].bus_type;
1013 bus_data[bus_0].bus_type = PCI;
1014
1015 /* swap each relavant INTerrupt entry */
1016 id = bus_data[bus_pci].bus_id;
1017 for (x = 0; x < nintrs; ++x) {
1018 if (io_apic_ints[x].src_bus_id == id) {
1019 io_apic_ints[x].src_bus_id = 0;
1020 }
1021 else if (io_apic_ints[x].src_bus_id == 0) {
1022 io_apic_ints[x].src_bus_id = id;
1023 }
1024 }
1025 }
1026 /* sanity check if more than 1 PCI bus */
1027 else if (num_pci_bus > 1) {
1028 for (x = 0; x < mp_nbusses; ++x) {
1029 if (bus_data[x].bus_type != PCI)
1030 continue;
1031 if (bus_data[x].bus_id >= num_pci_bus)
1032 panic("bad PCI bus numbering");
1033 }
1034 }
1035}
1036
1037
1038static int
1039processor_entry(proc_entry_ptr entry, int cpu)
1040{
1041 /* check for usability */
1042 if ((cpu >= NCPU) || !(entry->cpu_flags & PROCENTRY_FLAG_EN))
1043 return 0;
1044
1045 /* check for BSP flag */
1046 if (entry->cpu_flags & PROCENTRY_FLAG_BP) {
1047 boot_cpu_id = entry->apic_id;
1048 CPU_TO_ID(0) = entry->apic_id;
1049 ID_TO_CPU(entry->apic_id) = 0;
1050 return 0; /* its already been counted */
1051 }
1052
1053 /* add another AP to list, if less than max number of CPUs */
1054 else {
1055 CPU_TO_ID(cpu) = entry->apic_id;
1056 ID_TO_CPU(entry->apic_id) = cpu;
1057 return 1;
1058 }
1059}
1060
1061
1062static int
1063bus_entry(bus_entry_ptr entry, int bus)
1064{
1065 int x;
1066 char c, name[8];
1067
1068 /* encode the name into an index */
1069 for (x = 0; x < 6; ++x) {
1070 if ((c = entry->bus_type[x]) == ' ')
1071 break;
1072 name[x] = c;
1073 }
1074 name[x] = '\0';
1075
1076 if ((x = lookup_bus_type(name)) == UNKNOWN_BUSTYPE)
1077 panic("unknown bus type: '%s'", name);
1078
1079 bus_data[bus].bus_id = entry->bus_id;
1080 bus_data[bus].bus_type = x;
1081
1082 return 1;
1083}
1084
1085
1086static int
1087io_apic_entry(io_apic_entry_ptr entry, int apic)
1088{
1089 if (!(entry->apic_flags & IOAPICENTRY_FLAG_EN))
1090 return 0;
1091
1092 IO_TO_ID(apic) = entry->apic_id;
1093 ID_TO_IO(entry->apic_id) = apic;
1094
1095 return 1;
1096}
1097
1098
1099static int
1100lookup_bus_type(char *name)
1101{
1102 int x;
1103
1104 for (x = 0; x < MAX_BUSTYPE; ++x)
1105 if (strcmp(bus_type_table[x].name, name) == 0)
1106 return bus_type_table[x].type;
1107
1108 return UNKNOWN_BUSTYPE;
1109}
1110
1111
1112static int
1113int_entry(int_entry_ptr entry, int intr)
1114{
1115 io_apic_ints[intr].int_type = entry->int_type;
1116 io_apic_ints[intr].int_flags = entry->int_flags;
1117 io_apic_ints[intr].src_bus_id = entry->src_bus_id;
1118 io_apic_ints[intr].src_bus_irq = entry->src_bus_irq;
1119 io_apic_ints[intr].dst_apic_id = entry->dst_apic_id;
1120 io_apic_ints[intr].dst_apic_int = entry->dst_apic_int;
1121
1122 return 1;
1123}
1124
1125
1126static int
1127apic_int_is_bus_type(int intr, int bus_type)
1128{
1129 int bus;
1130
1131 for (bus = 0; bus < mp_nbusses; ++bus)
1132 if ((bus_data[bus].bus_id == io_apic_ints[intr].src_bus_id)
1133 && ((int) bus_data[bus].bus_type == bus_type))
1134 return 1;
1135
1136 return 0;
1137}
1138
1139
1140/*
1141 * Given a traditional ISA INT mask, return an APIC mask.
1142 */
1143u_int
1144isa_apic_mask(u_int isa_mask)
1145{
1146 int isa_irq;
1147 int apic_pin;
1148
1149#if defined(SKIP_IRQ15_REDIRECT)
1150 if (isa_mask == (1 << 15)) {
1151 printf("skipping ISA IRQ15 redirect\n");
1152 return isa_mask;
1153 }
1154#endif /* SKIP_IRQ15_REDIRECT */
1155
1156 isa_irq = ffs(isa_mask); /* find its bit position */
1157 if (isa_irq == 0) /* doesn't exist */
1158 return 0;
1159 --isa_irq; /* make it zero based */
1160
1161 apic_pin = isa_apic_pin(isa_irq); /* look for APIC connection */
1162 if (apic_pin == -1)
1163 return 0;
1164
1165 return (1 << apic_pin); /* convert pin# to a mask */
1166}
1167
1168
1169/*
1170 * Determine which APIC pin an ISA/EISA INT is attached to.
1171 */
1172#define INTTYPE(I) (io_apic_ints[(I)].int_type)
1173#define INTPIN(I) (io_apic_ints[(I)].dst_apic_int)
1174
1175#define SRCBUSIRQ(I) (io_apic_ints[(I)].src_bus_irq)
1176int
1177isa_apic_pin(int isa_irq)
1178{
1179 int intr;
1180
1181 for (intr = 0; intr < nintrs; ++intr) { /* check each record */
1182 if (INTTYPE(intr) == 0) { /* standard INT */
1183 if (SRCBUSIRQ(intr) == isa_irq) {
1184 if (apic_int_is_bus_type(intr, ISA) ||
1185 apic_int_is_bus_type(intr, EISA))
1186 return INTPIN(intr); /* found */
1187 }
1188 }
1189 }
1190 return -1; /* NOT found */
1191}
1192
1193
1194/*
1195 * Determine which APIC pin a PCI INT is attached to.
1196 */
1197#define SRCBUSID(I) (io_apic_ints[(I)].src_bus_id)
1198#define SRCBUSDEVICE(I) ((io_apic_ints[(I)].src_bus_irq >> 2) & 0x1f)
1199#define SRCBUSLINE(I) (io_apic_ints[(I)].src_bus_irq & 0x03)
1200int
1201pci_apic_pin(int pciBus, int pciDevice, int pciInt)
1202{
1203 int intr;
1204
1205 --pciInt; /* zero based */
1206
1207 for (intr = 0; intr < nintrs; ++intr) /* check each record */
1208 if ((INTTYPE(intr) == 0) /* standard INT */
1209 && (SRCBUSID(intr) == pciBus)
1210 && (SRCBUSDEVICE(intr) == pciDevice)
1211 && (SRCBUSLINE(intr) == pciInt)) /* a candidate IRQ */
1212 if (apic_int_is_bus_type(intr, PCI))
1213 return INTPIN(intr); /* exact match */
1214
1215 return -1; /* NOT found */
1216}
1217
1218int
1219next_apic_pin(int pin)
1220{
1221 int intr, ointr;
1222 int bus, bustype;
1223
1224 bus = 0;
1225 bustype = 0;
1226 for (intr = 0; intr < nintrs; intr++) {
1227 if (INTPIN(intr) != pin || INTTYPE(intr) != 0)
1228 continue;
1229 bus = SRCBUSID(intr);
1230 bustype = apic_bus_type(bus);
1231 if (bustype != ISA &&
1232 bustype != EISA &&
1233 bustype != PCI)
1234 continue;
1235 break;
1236 }
1237 if (intr >= nintrs) {
1238 return -1;
1239 }
1240 for (ointr = intr + 1; ointr < nintrs; ointr++) {
1241 if (INTTYPE(ointr) != 0)
1242 continue;
1243 if (bus != SRCBUSID(ointr))
1244 continue;
1245 if (bustype == PCI) {
1246 if (SRCBUSDEVICE(intr) != SRCBUSDEVICE(ointr))
1247 continue;
1248 if (SRCBUSLINE(intr) != SRCBUSLINE(ointr))
1249 continue;
1250 }
1251 if (bustype == ISA || bustype == EISA) {
1252 if (SRCBUSIRQ(intr) != SRCBUSIRQ(ointr))
1253 continue;
1254 }
1255 if (INTPIN(intr) == INTPIN(ointr))
1256 continue;
1257 break;
1258 }
1259 if (ointr >= nintrs) {
1260 return -1;
1261 }
1262 return INTPIN(ointr);
1263}
1264#undef SRCBUSLINE
1265#undef SRCBUSDEVICE
1266#undef SRCBUSID
1267#undef SRCBUSIRQ
1268
1269#undef INTPIN
1270#undef INTTYPE
1271
1272
1273/*
1274 * Reprogram the MB chipset to NOT redirect an ISA INTerrupt.
1275 *
1276 * XXX FIXME:
1277 * Exactly what this means is unclear at this point. It is a solution
1278 * for motherboards that redirect the MBIRQ0 pin. Generically a motherboard
1279 * could route any of the ISA INTs to upper (>15) IRQ values. But most would
1280 * NOT be redirected via MBIRQ0, thus "undirect()ing" them would NOT be an
1281 * option.
1282 */
1283int
1284undirect_isa_irq(int rirq)
1285{
1286#if defined(READY)
1287 printf("Freeing redirected ISA irq %d.\n", rirq);
1288 /** FIXME: tickle the MB redirector chip */
1289 return ???;
1290#else
1291 printf("Freeing (NOT implemented) redirected ISA irq %d.\n", rirq);
1292 return 0;
1293#endif /* READY */
1294}
1295
1296
1297/*
1298 * Reprogram the MB chipset to NOT redirect a PCI INTerrupt
1299 */
1300int
1301undirect_pci_irq(int rirq)
1302{
1303#if defined(READY)
1304 if (bootverbose)
1305 printf("Freeing redirected PCI irq %d.\n", rirq);
1306
1307 /** FIXME: tickle the MB redirector chip */
1308 return ???;
1309#else
1310 if (bootverbose)
1311 printf("Freeing (NOT implemented) redirected PCI irq %d.\n",
1312 rirq);
1313 return 0;
1314#endif /* READY */
1315}
1316
1317
1318/*
1319 * given a bus ID, return:
1320 * the bus type if found
1321 * -1 if NOT found
1322 */
1323int
1324apic_bus_type(int id)
1325{
1326 int x;
1327
1328 for (x = 0; x < mp_nbusses; ++x)
1329 if (bus_data[x].bus_id == id)
1330 return bus_data[x].bus_type;
1331
1332 return -1;
1333}
1334
1335
1336/*
1337 * given a LOGICAL APIC# and pin#, return:
1338 * the associated src bus ID if found
1339 * -1 if NOT found
1340 */
1341int
1342apic_src_bus_id(int apic, int pin)
1343{
1344 int x;
1345
1346 /* search each of the possible INTerrupt sources */
1347 for (x = 0; x < nintrs; ++x)
1348 if ((apic == ID_TO_IO(io_apic_ints[x].dst_apic_id)) &&
1349 (pin == io_apic_ints[x].dst_apic_int))
1350 return (io_apic_ints[x].src_bus_id);
1351
1352 return -1; /* NOT found */
1353}
1354
1355
1356/*
1357 * given a LOGICAL APIC# and pin#, return:
1358 * the associated src bus IRQ if found
1359 * -1 if NOT found
1360 */
1361int
1362apic_src_bus_irq(int apic, int pin)
1363{
1364 int x;
1365
1366 for (x = 0; x < nintrs; x++)
1367 if ((apic == ID_TO_IO(io_apic_ints[x].dst_apic_id)) &&
1368 (pin == io_apic_ints[x].dst_apic_int))
1369 return (io_apic_ints[x].src_bus_irq);
1370
1371 return -1; /* NOT found */
1372}
1373
1374
1375/*
1376 * given a LOGICAL APIC# and pin#, return:
1377 * the associated INTerrupt type if found
1378 * -1 if NOT found
1379 */
1380int
1381apic_int_type(int apic, int pin)
1382{
1383 int x;
1384
1385 /* search each of the possible INTerrupt sources */
1386 for (x = 0; x < nintrs; ++x)
1387 if ((apic == ID_TO_IO(io_apic_ints[x].dst_apic_id)) &&
1388 (pin == io_apic_ints[x].dst_apic_int))
1389 return (io_apic_ints[x].int_type);
1390
1391 return -1; /* NOT found */
1392}
1393
1394
1395/*
1396 * given a LOGICAL APIC# and pin#, return:
1397 * the associated trigger mode if found
1398 * -1 if NOT found
1399 */
1400int
1401apic_trigger(int apic, int pin)
1402{
1403 int x;
1404
1405 /* search each of the possible INTerrupt sources */
1406 for (x = 0; x < nintrs; ++x)
1407 if ((apic == ID_TO_IO(io_apic_ints[x].dst_apic_id)) &&
1408 (pin == io_apic_ints[x].dst_apic_int))
1409 return ((io_apic_ints[x].int_flags >> 2) & 0x03);
1410
1411 return -1; /* NOT found */
1412}
1413
1414
1415/*
1416 * given a LOGICAL APIC# and pin#, return:
1417 * the associated 'active' level if found
1418 * -1 if NOT found
1419 */
1420int
1421apic_polarity(int apic, int pin)
1422{
1423 int x;
1424
1425 /* search each of the possible INTerrupt sources */
1426 for (x = 0; x < nintrs; ++x)
1427 if ((apic == ID_TO_IO(io_apic_ints[x].dst_apic_id)) &&
1428 (pin == io_apic_ints[x].dst_apic_int))
1429 return (io_apic_ints[x].int_flags & 0x03);
1430
1431 return -1; /* NOT found */
1432}
1433
1434
1435/*
1436 * set data according to MP defaults
1437 * FIXME: probably not complete yet...
1438 */
1439static void
1440default_mp_table(int type)
1441{
1442 int ap_cpu_id;
1443#if defined(APIC_IO)
1444 u_int32_t ux;
1445 int io_apic_id;
1446 int pin;
1447#endif /* APIC_IO */
1448
1449#if 0
1450 printf(" MP default config type: %d\n", type);
1451 switch (type) {
1452 case 1:
1453 printf(" bus: ISA, APIC: 82489DX\n");
1454 break;
1455 case 2:
1456 printf(" bus: EISA, APIC: 82489DX\n");
1457 break;
1458 case 3:
1459 printf(" bus: EISA, APIC: 82489DX\n");
1460 break;
1461 case 4:
1462 printf(" bus: MCA, APIC: 82489DX\n");
1463 break;
1464 case 5:
1465 printf(" bus: ISA+PCI, APIC: Integrated\n");
1466 break;
1467 case 6:
1468 printf(" bus: EISA+PCI, APIC: Integrated\n");
1469 break;
1470 case 7:
1471 printf(" bus: MCA+PCI, APIC: Integrated\n");
1472 break;
1473 default:
1474 printf(" future type\n");
1475 break;
1476 /* NOTREACHED */
1477 }
1478#endif /* 0 */
1479
1480 boot_cpu_id = (lapic.id & APIC_ID_MASK) >> 24;
1481 ap_cpu_id = (boot_cpu_id == 0) ? 1 : 0;
1482
1483 /* BSP */
1484 CPU_TO_ID(0) = boot_cpu_id;
1485 ID_TO_CPU(boot_cpu_id) = 0;
1486
1487 /* one and only AP */
1488 CPU_TO_ID(1) = ap_cpu_id;
1489 ID_TO_CPU(ap_cpu_id) = 1;
1490
1491#if defined(APIC_IO)
1492 /* one and only IO APIC */
1493 io_apic_id = (io_apic_read(0, IOAPIC_ID) & APIC_ID_MASK) >> 24;
1494
1495 /*
1496 * sanity check, refer to MP spec section 3.6.6, last paragraph
1497 * necessary as some hardware isn't properly setting up the IO APIC
1498 */
1499#if defined(REALLY_ANAL_IOAPICID_VALUE)
1500 if (io_apic_id != 2) {
1501#else
1502 if ((io_apic_id == 0) || (io_apic_id == 1) || (io_apic_id == 15)) {
1503#endif /* REALLY_ANAL_IOAPICID_VALUE */
1504 ux = io_apic_read(0, IOAPIC_ID); /* get current contents */
1505 ux &= ~APIC_ID_MASK; /* clear the ID field */
1506 ux |= 0x02000000; /* set it to '2' */
1507 io_apic_write(0, IOAPIC_ID, ux); /* write new value */
1508 ux = io_apic_read(0, IOAPIC_ID); /* re-read && test */
1509 if ((ux & APIC_ID_MASK) != 0x02000000)
1510 panic("can't control IO APIC ID, reg: 0x%08x", ux);
1511 io_apic_id = 2;
1512 }
1513 IO_TO_ID(0) = io_apic_id;
1514 ID_TO_IO(io_apic_id) = 0;
1515#endif /* APIC_IO */
1516
1517 /* fill out bus entries */
1518 switch (type) {
1519 case 1:
1520 case 2:
1521 case 3:
1522 case 5:
1523 case 6:
1524 bus_data[0].bus_id = default_data[type - 1][1];
1525 bus_data[0].bus_type = default_data[type - 1][2];
1526 bus_data[1].bus_id = default_data[type - 1][3];
1527 bus_data[1].bus_type = default_data[type - 1][4];
1528 break;
1529
1530 /* case 4: case 7: MCA NOT supported */
1531 default: /* illegal/reserved */
1532 panic("BAD default MP config: %d", type);
1533 /* NOTREACHED */
1534 }
1535
1536#if defined(APIC_IO)
1537 /* general cases from MP v1.4, table 5-2 */
1538 for (pin = 0; pin < 16; ++pin) {
1539 io_apic_ints[pin].int_type = 0;
1540 io_apic_ints[pin].int_flags = 0x05; /* edge/active-hi */
1541 io_apic_ints[pin].src_bus_id = 0;
1542 io_apic_ints[pin].src_bus_irq = pin; /* IRQ2 caught below */
1543 io_apic_ints[pin].dst_apic_id = io_apic_id;
1544 io_apic_ints[pin].dst_apic_int = pin; /* 1-to-1 */
1545 }
1546
1547 /* special cases from MP v1.4, table 5-2 */
1548 if (type == 2) {
1549 io_apic_ints[2].int_type = 0xff; /* N/C */
1550 io_apic_ints[13].int_type = 0xff; /* N/C */
1551#if !defined(APIC_MIXED_MODE)
1552 /** FIXME: ??? */
1553 panic("sorry, can't support type 2 default yet");
1554#endif /* APIC_MIXED_MODE */
1555 }
1556 else
1557 io_apic_ints[2].src_bus_irq = 0; /* ISA IRQ0 is on APIC INT 2 */
1558
1559 if (type == 7)
1560 io_apic_ints[0].int_type = 0xff; /* N/C */
1561 else
1562 io_apic_ints[0].int_type = 3; /* vectored 8259 */
1563#endif /* APIC_IO */
1564}
1565
1566
1567/*
1568 * initialize all the SMP locks
1569 */
1570
1571/* critical region around IO APIC, apic_imen */
1572struct simplelock imen_lock;
1573
1574/* critical region around splxx(), cpl, cml, cil, ipending */
1575struct simplelock cpl_lock;
1576
1577/* Make FAST_INTR() routines sequential */
1578struct simplelock fast_intr_lock;
1579
1580/* critical region around INTR() routines */
1581struct simplelock intr_lock;
1582
1583/* lock regions protected in UP kernel via cli/sti */
1584struct simplelock mpintr_lock;
1585
1586/* lock region used by kernel profiling */
1587struct simplelock mcount_lock;
1588
1589#ifdef USE_COMLOCK
1590/* locks com (tty) data/hardware accesses: a FASTINTR() */
1591struct simplelock com_lock;
1592#endif /* USE_COMLOCK */
1593
1594#ifdef USE_CLOCKLOCK
1595/* lock regions around the clock hardware */
1596struct simplelock clock_lock;
1597#endif /* USE_CLOCKLOCK */
1598
1599static void
1600init_locks(void)
1601{
1602 /*
1603 * Get the initial mp_lock with a count of 1 for the BSP.
1604 * This uses a LOGICAL cpu ID, ie BSP == 0.
1605 */
1606 mp_lock = 0x00000001;
1607
1608 /* ISR uses its own "giant lock" */
1609 isr_lock = FREE_LOCK;
1610
1611#if defined(APIC_INTR_DIAGNOSTIC) && defined(APIC_INTR_DIAGNOSTIC_IRQ)
1612 s_lock_init((struct simplelock*)&apic_itrace_debuglock);
1613#endif
1614
1615 s_lock_init((struct simplelock*)&mpintr_lock);
1616
1617 s_lock_init((struct simplelock*)&mcount_lock);
1618
1619 s_lock_init((struct simplelock*)&fast_intr_lock);
1620 s_lock_init((struct simplelock*)&intr_lock);
1621 s_lock_init((struct simplelock*)&imen_lock);
1622 s_lock_init((struct simplelock*)&cpl_lock);
1623
1624#ifdef USE_COMLOCK
1625 s_lock_init((struct simplelock*)&com_lock);
1626#endif /* USE_COMLOCK */
1627#ifdef USE_CLOCKLOCK
1628 s_lock_init((struct simplelock*)&clock_lock);
1629#endif /* USE_CLOCKLOCK */
1630}
1631
1632
1633/*
1634 * start each AP in our list
1635 */
1636static int
1637start_all_aps(u_int boot_addr)
1638{
1639 int x, i;
1640 u_char mpbiosreason;
1641 u_long mpbioswarmvec;
1642 pd_entry_t *newptd;
1643 pt_entry_t *newpt;
1644 struct globaldata *gd;
1645 char *stack;
1646 pd_entry_t *myPTD;
1647
1648 POSTCODE(START_ALL_APS_POST);
1649
1650 /* initialize BSP's local APIC */
1651 apic_initialize();
1652 bsp_apic_ready = 1;
1653
1654 /* install the AP 1st level boot code */
1655 install_ap_tramp(boot_addr);
1656
1657
1658 /* save the current value of the warm-start vector */
1659 mpbioswarmvec = *((u_long *) WARMBOOT_OFF);
1660 outb(CMOS_REG, BIOS_RESET);
1661 mpbiosreason = inb(CMOS_DATA);
1662
1663 /* record BSP in CPU map */
1664 all_cpus = 1;
1665
1666 /* start each AP */
1667 for (x = 1; x <= mp_naps; ++x) {
1668
1669 /* This is a bit verbose, it will go away soon. */
1670
1671 /* alloc new page table directory */
1672 newptd = (pd_entry_t *)(kmem_alloc(kernel_map, PAGE_SIZE));
1673
1674 /* Store the virtual PTD address for this CPU */
1675 IdlePTDS[x] = newptd;
1676
1677 /* clone currently active one (ie: IdlePTD) */
1678 bcopy(PTD, newptd, PAGE_SIZE); /* inc prv page pde */
1679
1680 /* set up 0 -> 4MB P==V mapping for AP boot */
1681 newptd[0] = (pd_entry_t) (PG_V | PG_RW |
1682 ((u_long)KPTphys & PG_FRAME));
1683
1684 /* store PTD for this AP's boot sequence */
1685 myPTD = (pd_entry_t *)vtophys(newptd);
1686
1687 /* alloc new page table page */
1688 newpt = (pt_entry_t *)(kmem_alloc(kernel_map, PAGE_SIZE));
1689
1690 /* set the new PTD's private page to point there */
1691 newptd[MPPTDI] = (pt_entry_t)(PG_V | PG_RW | vtophys(newpt));
1692
1693 /* install self referential entry */
1694 newptd[PTDPTDI] = (pd_entry_t)(PG_V | PG_RW | vtophys(newptd));
1695
1696 /* allocate a new private data page */
1697 gd = (struct globaldata *)kmem_alloc(kernel_map, PAGE_SIZE);
1698
1699 /* wire it into the private page table page */
1700 newpt[0] = (pt_entry_t)(PG_V | PG_RW | vtophys(gd));
1701
1702 /* wire the ptp into itself for access */
1703 newpt[1] = (pt_entry_t)(PG_V | PG_RW | vtophys(newpt));
1704
1705 /* copy in the pointer to the local apic */
1706 newpt[2] = SMP_prvpt[2];
1707
1708 /* and the IO apic mapping[s] */
1709 for (i = 16; i < 32; i++)
1710 newpt[i] = SMP_prvpt[i];
1711
1712 /* allocate and set up an idle stack data page */
1713 stack = (char *)kmem_alloc(kernel_map, UPAGES*PAGE_SIZE);
1714 for (i = 0; i < UPAGES; i++)
1715 newpt[i + 3] = (pt_entry_t)(PG_V | PG_RW | vtophys(PAGE_SIZE * i + stack));
1716
1717 newpt[3 + UPAGES] = 0; /* *prv_CMAP1 */
1718 newpt[4 + UPAGES] = 0; /* *prv_CMAP2 */
1719 newpt[5 + UPAGES] = 0; /* *prv_CMAP3 */
| 26 */
27
28#include "opt_smp.h"
29#include "opt_vm86.h"
30#include "opt_cpu.h"
31
32#ifdef SMP
33#include <machine/smptests.h>
34#else
35#error
36#endif
37
38#include <sys/param.h>
39#include <sys/systm.h>
40#include <sys/kernel.h>
41#include <sys/proc.h>
42#include <sys/sysctl.h>
43#ifdef BETTER_CLOCK
44#include <sys/dkstat.h>
45#endif
46
47#include <vm/vm.h>
48#include <vm/vm_param.h>
49#include <vm/pmap.h>
50#include <vm/vm_kern.h>
51#include <vm/vm_extern.h>
52#ifdef BETTER_CLOCK
53#include <sys/lock.h>
54#include <vm/vm_map.h>
55#include <sys/user.h>
56#ifdef GPROF
57#include <sys/gmon.h>
58#endif
59#endif
60
61#include <machine/smp.h>
62#include <machine/apic.h>
63#include <machine/mpapic.h>
64#include <machine/segments.h>
65#include <machine/smptests.h> /** TEST_DEFAULT_CONFIG, TEST_TEST1 */
66#include <machine/tss.h>
67#include <machine/specialreg.h>
68#include <machine/cputypes.h>
69#include <machine/globaldata.h>
70
71#include <i386/i386/cons.h> /* cngetc() */
72
73#if defined(APIC_IO)
74#include <machine/md_var.h> /* setidt() */
75#include <i386/isa/icu.h> /* IPIs */
76#include <i386/isa/intr_machdep.h> /* IPIs */
77#endif /* APIC_IO */
78
79#if defined(TEST_DEFAULT_CONFIG)
80#define MPFPS_MPFB1 TEST_DEFAULT_CONFIG
81#else
82#define MPFPS_MPFB1 mpfps->mpfb1
83#endif /* TEST_DEFAULT_CONFIG */
84
85#define WARMBOOT_TARGET 0
86#define WARMBOOT_OFF (KERNBASE + 0x0467)
87#define WARMBOOT_SEG (KERNBASE + 0x0469)
88
89#define BIOS_BASE (0xf0000)
90#define BIOS_SIZE (0x10000)
91#define BIOS_COUNT (BIOS_SIZE/4)
92
93#define CMOS_REG (0x70)
94#define CMOS_DATA (0x71)
95#define BIOS_RESET (0x0f)
96#define BIOS_WARM (0x0a)
97
98#define PROCENTRY_FLAG_EN 0x01
99#define PROCENTRY_FLAG_BP 0x02
100#define IOAPICENTRY_FLAG_EN 0x01
101
102
103/* MP Floating Pointer Structure */
104typedef struct MPFPS {
105 char signature[4];
106 void *pap;
107 u_char length;
108 u_char spec_rev;
109 u_char checksum;
110 u_char mpfb1;
111 u_char mpfb2;
112 u_char mpfb3;
113 u_char mpfb4;
114 u_char mpfb5;
115} *mpfps_t;
116
117/* MP Configuration Table Header */
118typedef struct MPCTH {
119 char signature[4];
120 u_short base_table_length;
121 u_char spec_rev;
122 u_char checksum;
123 u_char oem_id[8];
124 u_char product_id[12];
125 void *oem_table_pointer;
126 u_short oem_table_size;
127 u_short entry_count;
128 void *apic_address;
129 u_short extended_table_length;
130 u_char extended_table_checksum;
131 u_char reserved;
132} *mpcth_t;
133
134
135typedef struct PROCENTRY {
136 u_char type;
137 u_char apic_id;
138 u_char apic_version;
139 u_char cpu_flags;
140 u_long cpu_signature;
141 u_long feature_flags;
142 u_long reserved1;
143 u_long reserved2;
144} *proc_entry_ptr;
145
146typedef struct BUSENTRY {
147 u_char type;
148 u_char bus_id;
149 char bus_type[6];
150} *bus_entry_ptr;
151
152typedef struct IOAPICENTRY {
153 u_char type;
154 u_char apic_id;
155 u_char apic_version;
156 u_char apic_flags;
157 void *apic_address;
158} *io_apic_entry_ptr;
159
160typedef struct INTENTRY {
161 u_char type;
162 u_char int_type;
163 u_short int_flags;
164 u_char src_bus_id;
165 u_char src_bus_irq;
166 u_char dst_apic_id;
167 u_char dst_apic_int;
168} *int_entry_ptr;
169
170/* descriptions of MP basetable entries */
171typedef struct BASETABLE_ENTRY {
172 u_char type;
173 u_char length;
174 char name[16];
175} basetable_entry;
176
177/*
178 * this code MUST be enabled here and in mpboot.s.
179 * it follows the very early stages of AP boot by placing values in CMOS ram.
180 * it NORMALLY will never be needed and thus the primitive method for enabling.
181 *
182#define CHECK_POINTS
183 */
184
185#if defined(CHECK_POINTS)
186#define CHECK_READ(A) (outb(CMOS_REG, (A)), inb(CMOS_DATA))
187#define CHECK_WRITE(A,D) (outb(CMOS_REG, (A)), outb(CMOS_DATA, (D)))
188
189#define CHECK_INIT(D); \
190 CHECK_WRITE(0x34, (D)); \
191 CHECK_WRITE(0x35, (D)); \
192 CHECK_WRITE(0x36, (D)); \
193 CHECK_WRITE(0x37, (D)); \
194 CHECK_WRITE(0x38, (D)); \
195 CHECK_WRITE(0x39, (D));
196
197#define CHECK_PRINT(S); \
198 printf("%s: %d, %d, %d, %d, %d, %d\n", \
199 (S), \
200 CHECK_READ(0x34), \
201 CHECK_READ(0x35), \
202 CHECK_READ(0x36), \
203 CHECK_READ(0x37), \
204 CHECK_READ(0x38), \
205 CHECK_READ(0x39));
206
207#else /* CHECK_POINTS */
208
209#define CHECK_INIT(D)
210#define CHECK_PRINT(S)
211
212#endif /* CHECK_POINTS */
213
214/*
215 * Values to send to the POST hardware.
216 */
217#define MP_BOOTADDRESS_POST 0x10
218#define MP_PROBE_POST 0x11
219#define MPTABLE_PASS1_POST 0x12
220
221#define MP_START_POST 0x13
222#define MP_ENABLE_POST 0x14
223#define MPTABLE_PASS2_POST 0x15
224
225#define START_ALL_APS_POST 0x16
226#define INSTALL_AP_TRAMP_POST 0x17
227#define START_AP_POST 0x18
228
229#define MP_ANNOUNCE_POST 0x19
230
231
232/** XXX FIXME: where does this really belong, isa.h/isa.c perhaps? */
233int current_postcode;
234
235/** XXX FIXME: what system files declare these??? */
236extern struct region_descriptor r_gdt, r_idt;
237
238int bsp_apic_ready = 0; /* flags useability of BSP apic */
239int mp_ncpus; /* # of CPUs, including BSP */
240int mp_naps; /* # of Applications processors */
241int mp_nbusses; /* # of busses */
242int mp_napics; /* # of IO APICs */
243int boot_cpu_id; /* designated BSP */
244vm_offset_t cpu_apic_address;
245vm_offset_t io_apic_address[NAPICID]; /* NAPICID is more than enough */
246extern int nkpt;
247
248u_int32_t cpu_apic_versions[NCPU];
249u_int32_t io_apic_versions[NAPIC];
250
251#ifdef APIC_INTR_DIAGNOSTIC
252int apic_itrace_enter[32];
253int apic_itrace_tryisrlock[32];
254int apic_itrace_gotisrlock[32];
255int apic_itrace_active[32];
256int apic_itrace_masked[32];
257int apic_itrace_noisrlock[32];
258int apic_itrace_masked2[32];
259int apic_itrace_unmask[32];
260int apic_itrace_noforward[32];
261int apic_itrace_leave[32];
262int apic_itrace_enter2[32];
263int apic_itrace_doreti[32];
264int apic_itrace_splz[32];
265int apic_itrace_eoi[32];
266#ifdef APIC_INTR_DIAGNOSTIC_IRQ
267unsigned short apic_itrace_debugbuffer[32768];
268int apic_itrace_debugbuffer_idx;
269struct simplelock apic_itrace_debuglock;
270#endif
271#endif
272
273#ifdef APIC_INTR_REORDER
274struct {
275 volatile int *location;
276 int bit;
277} apic_isrbit_location[32];
278#endif
279
280/*
281 * APIC ID logical/physical mapping structures.
282 * We oversize these to simplify boot-time config.
283 */
284int cpu_num_to_apic_id[NAPICID];
285int io_num_to_apic_id[NAPICID];
286int apic_id_to_logical[NAPICID];
287
288
289/* Bitmap of all available CPUs */
290u_int all_cpus;
291
292/* AP uses this PTD during bootstrap. Do not staticize. */
293pd_entry_t *bootPTD;
294
295/* Hotwire a 0->4MB V==P mapping */
296extern pt_entry_t *KPTphys;
297
298/* Virtual address of per-cpu common_tss */
299extern struct i386tss common_tss;
300#ifdef VM86
301extern struct segment_descriptor common_tssd;
302extern u_int private_tss; /* flag indicating private tss */
303extern u_int my_tr;
304#endif /* VM86 */
305
306/* IdlePTD per cpu */
307pd_entry_t *IdlePTDS[NCPU];
308
309/* "my" private page table page, for BSP init */
310extern pt_entry_t SMP_prvpt[];
311
312/* Private page pointer to curcpu's PTD, used during BSP init */
313extern pd_entry_t *my_idlePTD;
314
315static int smp_started; /* has the system started? */
316
317/*
318 * Local data and functions.
319 */
320
321static int mp_capable;
322static u_int boot_address;
323static u_int base_memory;
324
325static int picmode; /* 0: virtual wire mode, 1: PIC mode */
326static mpfps_t mpfps;
327static int search_for_sig(u_int32_t target, int count);
328static void mp_enable(u_int boot_addr);
329
330static int mptable_pass1(void);
331static int mptable_pass2(void);
332static void default_mp_table(int type);
333static void fix_mp_table(void);
334static void init_locks(void);
335static int start_all_aps(u_int boot_addr);
336static void install_ap_tramp(u_int boot_addr);
337static int start_ap(int logicalCpu, u_int boot_addr);
338
339/*
340 * Calculate usable address in base memory for AP trampoline code.
341 */
342u_int
343mp_bootaddress(u_int basemem)
344{
345 POSTCODE(MP_BOOTADDRESS_POST);
346
347 base_memory = basemem * 1024; /* convert to bytes */
348
349 boot_address = base_memory & ~0xfff; /* round down to 4k boundary */
350 if ((base_memory - boot_address) < bootMP_size)
351 boot_address -= 4096; /* not enough, lower by 4k */
352
353 return boot_address;
354}
355
356
357/*
358 * Look for an Intel MP spec table (ie, SMP capable hardware).
359 */
360int
361mp_probe(void)
362{
363 int x;
364 u_long segment;
365 u_int32_t target;
366
367 POSTCODE(MP_PROBE_POST);
368
369 /* see if EBDA exists */
370 if (segment = (u_long) * (u_short *) (KERNBASE + 0x40e)) {
371 /* search first 1K of EBDA */
372 target = (u_int32_t) (segment << 4);
373 if ((x = search_for_sig(target, 1024 / 4)) >= 0)
374 goto found;
375 } else {
376 /* last 1K of base memory, effective 'top of base' passed in */
377 target = (u_int32_t) (base_memory - 0x400);
378 if ((x = search_for_sig(target, 1024 / 4)) >= 0)
379 goto found;
380 }
381
382 /* search the BIOS */
383 target = (u_int32_t) BIOS_BASE;
384 if ((x = search_for_sig(target, BIOS_COUNT)) >= 0)
385 goto found;
386
387 /* nothing found */
388 mpfps = (mpfps_t)0;
389 mp_capable = 0;
390 return 0;
391
392found:
393 /* calculate needed resources */
394 mpfps = (mpfps_t)x;
395 if (mptable_pass1())
396 panic("you must reconfigure your kernel");
397
398 /* flag fact that we are running multiple processors */
399 mp_capable = 1;
400 return 1;
401}
402
403
404/*
405 * Startup the SMP processors.
406 */
407void
408mp_start(void)
409{
410 POSTCODE(MP_START_POST);
411
412 /* look for MP capable motherboard */
413 if (mp_capable)
414 mp_enable(boot_address);
415 else
416 panic("MP hardware not found!");
417}
418
419
420/*
421 * Print various information about the SMP system hardware and setup.
422 */
423void
424mp_announce(void)
425{
426 int x;
427
428 POSTCODE(MP_ANNOUNCE_POST);
429
430 printf("FreeBSD/SMP: Multiprocessor motherboard\n");
431 printf(" cpu0 (BSP): apic id: %2d", CPU_TO_ID(0));
432 printf(", version: 0x%08x", cpu_apic_versions[0]);
433 printf(", at 0x%08x\n", cpu_apic_address);
434 for (x = 1; x <= mp_naps; ++x) {
435 printf(" cpu%d (AP): apic id: %2d", x, CPU_TO_ID(x));
436 printf(", version: 0x%08x", cpu_apic_versions[x]);
437 printf(", at 0x%08x\n", cpu_apic_address);
438 }
439
440#if defined(APIC_IO)
441 for (x = 0; x < mp_napics; ++x) {
442 printf(" io%d (APIC): apic id: %2d", x, IO_TO_ID(x));
443 printf(", version: 0x%08x", io_apic_versions[x]);
444 printf(", at 0x%08x\n", io_apic_address[x]);
445 }
446#else
447 printf(" Warning: APIC I/O disabled\n");
448#endif /* APIC_IO */
449}
450
451/*
452 * AP cpu's call this to sync up protected mode.
453 */
454void
455init_secondary(void)
456{
457 int gsel_tss;
458#ifndef VM86
459 u_int my_tr;
460#endif
461
462 r_gdt.rd_limit = sizeof(gdt[0]) * (NGDT + NCPU) - 1;
463 r_gdt.rd_base = (int) gdt;
464 lgdt(&r_gdt); /* does magic intra-segment return */
465 lidt(&r_idt);
466 lldt(_default_ldt);
467
468 my_tr = NGDT + cpuid;
469 gsel_tss = GSEL(my_tr, SEL_KPL);
470 gdt[my_tr].sd.sd_type = SDT_SYS386TSS;
471 common_tss.tss_esp0 = 0; /* not used until after switch */
472 common_tss.tss_ss0 = GSEL(GDATA_SEL, SEL_KPL);
473 common_tss.tss_ioopt = (sizeof common_tss) << 16;
474#ifdef VM86
475 common_tssd = gdt[my_tr].sd;
476 private_tss = 0;
477#endif /* VM86 */
478 ltr(gsel_tss);
479
480 load_cr0(0x8005003b); /* XXX! */
481
482 PTD[0] = 0;
483 pmap_set_opt((unsigned *)PTD);
484
485 putmtrr();
486 pmap_setvidram();
487
488 invltlb();
489}
490
491
492#if defined(APIC_IO)
493/*
494 * Final configuration of the BSP's local APIC:
495 * - disable 'pic mode'.
496 * - disable 'virtual wire mode'.
497 * - enable NMI.
498 */
499void
500bsp_apic_configure(void)
501{
502 u_char byte;
503 u_int32_t temp;
504
505 /* leave 'pic mode' if necessary */
506 if (picmode) {
507 outb(0x22, 0x70); /* select IMCR */
508 byte = inb(0x23); /* current contents */
509 byte |= 0x01; /* mask external INTR */
510 outb(0x23, byte); /* disconnect 8259s/NMI */
511 }
512
513 /* mask lint0 (the 8259 'virtual wire' connection) */
514 temp = lapic.lvt_lint0;
515 temp |= APIC_LVT_M; /* set the mask */
516 lapic.lvt_lint0 = temp;
517
518 /* setup lint1 to handle NMI */
519 temp = lapic.lvt_lint1;
520 temp &= ~APIC_LVT_M; /* clear the mask */
521 lapic.lvt_lint1 = temp;
522
523 if (bootverbose)
524 apic_dump("bsp_apic_configure()");
525}
526#endif /* APIC_IO */
527
528
529/*******************************************************************
530 * local functions and data
531 */
532
533/*
534 * start the SMP system
535 */
536static void
537mp_enable(u_int boot_addr)
538{
539 int x;
540#if defined(APIC_IO)
541 int apic;
542 u_int ux;
543#endif /* APIC_IO */
544
545 getmtrr();
546 pmap_setvidram();
547
548 POSTCODE(MP_ENABLE_POST);
549
550 /* turn on 4MB of V == P addressing so we can get to MP table */
551 *(int *)PTD = PG_V | PG_RW | ((u_long)KPTphys & PG_FRAME);
552 invltlb();
553
554 /* examine the MP table for needed info, uses physical addresses */
555 x = mptable_pass2();
556
557 *(int *)PTD = 0;
558 invltlb();
559
560 /* can't process default configs till the CPU APIC is pmapped */
561 if (x)
562 default_mp_table(x);
563
564 /* post scan cleanup */
565 fix_mp_table();
566
567#if defined(APIC_IO)
568
569 /* fill the LOGICAL io_apic_versions table */
570 for (apic = 0; apic < mp_napics; ++apic) {
571 ux = io_apic_read(apic, IOAPIC_VER);
572 io_apic_versions[apic] = ux;
573 }
574
575 /* program each IO APIC in the system */
576 for (apic = 0; apic < mp_napics; ++apic)
577 if (io_apic_setup(apic) < 0)
578 panic("IO APIC setup failure");
579
580 /* install a 'Spurious INTerrupt' vector */
581 setidt(XSPURIOUSINT_OFFSET, Xspuriousint,
582 SDT_SYS386IGT, SEL_KPL, GSEL(GCODE_SEL, SEL_KPL));
583
584 /* install an inter-CPU IPI for TLB invalidation */
585 setidt(XINVLTLB_OFFSET, Xinvltlb,
586 SDT_SYS386IGT, SEL_KPL, GSEL(GCODE_SEL, SEL_KPL));
587
588#ifdef BETTER_CLOCK
589 /* install an inter-CPU IPI for reading processor state */
590 setidt(XCPUCHECKSTATE_OFFSET, Xcpucheckstate,
591 SDT_SYS386IGT, SEL_KPL, GSEL(GCODE_SEL, SEL_KPL));
592#endif
593
594 /* install an inter-CPU IPI for forcing an additional software trap */
595 setidt(XCPUAST_OFFSET, Xcpuast,
596 SDT_SYS386IGT, SEL_KPL, GSEL(GCODE_SEL, SEL_KPL));
597
598 /* install an inter-CPU IPI for interrupt forwarding */
599 setidt(XFORWARD_IRQ_OFFSET, Xforward_irq,
600 SDT_SYS386IGT, SEL_KPL, GSEL(GCODE_SEL, SEL_KPL));
601
602 /* install an inter-CPU IPI for CPU stop/restart */
603 setidt(XCPUSTOP_OFFSET, Xcpustop,
604 SDT_SYS386IGT, SEL_KPL, GSEL(GCODE_SEL, SEL_KPL));
605
606#if defined(TEST_TEST1)
607 /* install a "fake hardware INTerrupt" vector */
608 setidt(XTEST1_OFFSET, Xtest1,
609 SDT_SYS386IGT, SEL_KPL, GSEL(GCODE_SEL, SEL_KPL));
610#endif /** TEST_TEST1 */
611
612#endif /* APIC_IO */
613
614 /* initialize all SMP locks */
615 init_locks();
616
617 /* start each Application Processor */
618 start_all_aps(boot_addr);
619
620 /*
621 * The init process might be started on a different CPU now,
622 * and the boot CPU might not call prepare_usermode to get
623 * cr0 correctly configured. Thus we initialize cr0 here.
624 */
625 load_cr0(rcr0() | CR0_WP | CR0_AM);
626}
627
628
629/*
630 * look for the MP spec signature
631 */
632
633/* string defined by the Intel MP Spec as identifying the MP table */
634#define MP_SIG 0x5f504d5f /* _MP_ */
635#define NEXT(X) ((X) += 4)
636static int
637search_for_sig(u_int32_t target, int count)
638{
639 int x;
640 u_int32_t *addr = (u_int32_t *) (KERNBASE + target);
641
642 for (x = 0; x < count; NEXT(x))
643 if (addr[x] == MP_SIG)
644 /* make array index a byte index */
645 return (target + (x * sizeof(u_int32_t)));
646
647 return -1;
648}
649
650
651static basetable_entry basetable_entry_types[] =
652{
653 {0, 20, "Processor"},
654 {1, 8, "Bus"},
655 {2, 8, "I/O APIC"},
656 {3, 8, "I/O INT"},
657 {4, 8, "Local INT"}
658};
659
660typedef struct BUSDATA {
661 u_char bus_id;
662 enum busTypes bus_type;
663} bus_datum;
664
665typedef struct INTDATA {
666 u_char int_type;
667 u_short int_flags;
668 u_char src_bus_id;
669 u_char src_bus_irq;
670 u_char dst_apic_id;
671 u_char dst_apic_int;
672} io_int, local_int;
673
674typedef struct BUSTYPENAME {
675 u_char type;
676 char name[7];
677} bus_type_name;
678
679static bus_type_name bus_type_table[] =
680{
681 {CBUS, "CBUS"},
682 {CBUSII, "CBUSII"},
683 {EISA, "EISA"},
684 {UNKNOWN_BUSTYPE, "---"},
685 {UNKNOWN_BUSTYPE, "---"},
686 {ISA, "ISA"},
687 {UNKNOWN_BUSTYPE, "---"},
688 {UNKNOWN_BUSTYPE, "---"},
689 {UNKNOWN_BUSTYPE, "---"},
690 {UNKNOWN_BUSTYPE, "---"},
691 {UNKNOWN_BUSTYPE, "---"},
692 {UNKNOWN_BUSTYPE, "---"},
693 {PCI, "PCI"},
694 {UNKNOWN_BUSTYPE, "---"},
695 {UNKNOWN_BUSTYPE, "---"},
696 {UNKNOWN_BUSTYPE, "---"},
697 {UNKNOWN_BUSTYPE, "---"},
698 {XPRESS, "XPRESS"},
699 {UNKNOWN_BUSTYPE, "---"}
700};
701/* from MP spec v1.4, table 5-1 */
702static int default_data[7][5] =
703{
704/* nbus, id0, type0, id1, type1 */
705 {1, 0, ISA, 255, 255},
706 {1, 0, EISA, 255, 255},
707 {1, 0, EISA, 255, 255},
708 {0, 255, 255, 255, 255},/* MCA not supported */
709 {2, 0, ISA, 1, PCI},
710 {2, 0, EISA, 1, PCI},
711 {0, 255, 255, 255, 255} /* MCA not supported */
712};
713
714
715/* the bus data */
716static bus_datum bus_data[NBUS];
717
718/* the IO INT data, one entry per possible APIC INTerrupt */
719static io_int io_apic_ints[NINTR];
720
721static int nintrs;
722
723static int processor_entry __P((proc_entry_ptr entry, int cpu));
724static int bus_entry __P((bus_entry_ptr entry, int bus));
725static int io_apic_entry __P((io_apic_entry_ptr entry, int apic));
726static int int_entry __P((int_entry_ptr entry, int intr));
727static int lookup_bus_type __P((char *name));
728
729
730/*
731 * 1st pass on motherboard's Intel MP specification table.
732 *
733 * initializes:
734 * mp_ncpus = 1
735 *
736 * determines:
737 * cpu_apic_address (common to all CPUs)
738 * io_apic_address[N]
739 * mp_naps
740 * mp_nbusses
741 * mp_napics
742 * nintrs
743 */
744static int
745mptable_pass1(void)
746{
747 int x;
748 mpcth_t cth;
749 int totalSize;
750 void* position;
751 int count;
752 int type;
753 int mustpanic;
754
755 POSTCODE(MPTABLE_PASS1_POST);
756
757 mustpanic = 0;
758
759 /* clear various tables */
760 for (x = 0; x < NAPICID; ++x) {
761 io_apic_address[x] = ~0; /* IO APIC address table */
762 }
763
764 /* init everything to empty */
765 mp_naps = 0;
766 mp_nbusses = 0;
767 mp_napics = 0;
768 nintrs = 0;
769
770 /* check for use of 'default' configuration */
771 if (MPFPS_MPFB1 != 0) {
772 /* use default addresses */
773 cpu_apic_address = DEFAULT_APIC_BASE;
774 io_apic_address[0] = DEFAULT_IO_APIC_BASE;
775
776 /* fill in with defaults */
777 mp_naps = 2; /* includes BSP */
778 mp_nbusses = default_data[MPFPS_MPFB1 - 1][0];
779#if defined(APIC_IO)
780 mp_napics = 1;
781 nintrs = 16;
782#endif /* APIC_IO */
783 }
784 else {
785 if ((cth = mpfps->pap) == 0)
786 panic("MP Configuration Table Header MISSING!");
787
788 cpu_apic_address = (vm_offset_t) cth->apic_address;
789
790 /* walk the table, recording info of interest */
791 totalSize = cth->base_table_length - sizeof(struct MPCTH);
792 position = (u_char *) cth + sizeof(struct MPCTH);
793 count = cth->entry_count;
794
795 while (count--) {
796 switch (type = *(u_char *) position) {
797 case 0: /* processor_entry */
798 if (((proc_entry_ptr)position)->cpu_flags
799 & PROCENTRY_FLAG_EN)
800 ++mp_naps;
801 break;
802 case 1: /* bus_entry */
803 ++mp_nbusses;
804 break;
805 case 2: /* io_apic_entry */
806 if (((io_apic_entry_ptr)position)->apic_flags
807 & IOAPICENTRY_FLAG_EN)
808 io_apic_address[mp_napics++] =
809 (vm_offset_t)((io_apic_entry_ptr)
810 position)->apic_address;
811 break;
812 case 3: /* int_entry */
813 ++nintrs;
814 break;
815 case 4: /* int_entry */
816 break;
817 default:
818 panic("mpfps Base Table HOSED!");
819 /* NOTREACHED */
820 }
821
822 totalSize -= basetable_entry_types[type].length;
823 (u_char*)position += basetable_entry_types[type].length;
824 }
825 }
826
827 /* qualify the numbers */
828 if (mp_naps > NCPU)
829#if 0 /* XXX FIXME: kern/4255 */
830 printf("Warning: only using %d of %d available CPUs!\n",
831 NCPU, mp_naps);
832#else
833 {
834 printf("NCPU cannot be different than actual CPU count.\n");
835 printf(" add 'options NCPU=%d' to your kernel config file,\n",
836 mp_naps);
837 printf(" then rerun config & rebuild your SMP kernel\n");
838 mustpanic = 1;
839 }
840#endif /* XXX FIXME: kern/4255 */
841 if (mp_nbusses > NBUS) {
842 printf("found %d busses, increase NBUS\n", mp_nbusses);
843 mustpanic = 1;
844 }
845 if (mp_napics > NAPIC) {
846 printf("found %d apics, increase NAPIC\n", mp_napics);
847 mustpanic = 1;
848 }
849 if (nintrs > NINTR) {
850 printf("found %d intrs, increase NINTR\n", nintrs);
851 mustpanic = 1;
852 }
853
854 /*
855 * Count the BSP.
856 * This is also used as a counter while starting the APs.
857 */
858 mp_ncpus = 1;
859
860 --mp_naps; /* subtract the BSP */
861
862 return mustpanic;
863}
864
865
866/*
867 * 2nd pass on motherboard's Intel MP specification table.
868 *
869 * sets:
870 * boot_cpu_id
871 * ID_TO_IO(N), phy APIC ID to log CPU/IO table
872 * CPU_TO_ID(N), logical CPU to APIC ID table
873 * IO_TO_ID(N), logical IO to APIC ID table
874 * bus_data[N]
875 * io_apic_ints[N]
876 */
877static int
878mptable_pass2(void)
879{
880 int x;
881 mpcth_t cth;
882 int totalSize;
883 void* position;
884 int count;
885 int type;
886 int apic, bus, cpu, intr;
887
888 POSTCODE(MPTABLE_PASS2_POST);
889
890 /* clear various tables */
891 for (x = 0; x < NAPICID; ++x) {
892 ID_TO_IO(x) = -1; /* phy APIC ID to log CPU/IO table */
893 CPU_TO_ID(x) = -1; /* logical CPU to APIC ID table */
894 IO_TO_ID(x) = -1; /* logical IO to APIC ID table */
895 }
896
897 /* clear bus data table */
898 for (x = 0; x < NBUS; ++x)
899 bus_data[x].bus_id = 0xff;
900
901 /* clear IO APIC INT table */
902 for (x = 0; x < NINTR; ++x)
903 io_apic_ints[x].int_type = 0xff;
904
905 /* setup the cpu/apic mapping arrays */
906 boot_cpu_id = -1;
907
908 /* record whether PIC or virtual-wire mode */
909 picmode = (mpfps->mpfb2 & 0x80) ? 1 : 0;
910
911 /* check for use of 'default' configuration */
912 if (MPFPS_MPFB1 != 0)
913 return MPFPS_MPFB1; /* return default configuration type */
914
915 if ((cth = mpfps->pap) == 0)
916 panic("MP Configuration Table Header MISSING!");
917
918 /* walk the table, recording info of interest */
919 totalSize = cth->base_table_length - sizeof(struct MPCTH);
920 position = (u_char *) cth + sizeof(struct MPCTH);
921 count = cth->entry_count;
922 apic = bus = intr = 0;
923 cpu = 1; /* pre-count the BSP */
924
925 while (count--) {
926 switch (type = *(u_char *) position) {
927 case 0:
928 if (processor_entry(position, cpu))
929 ++cpu;
930 break;
931 case 1:
932 if (bus_entry(position, bus))
933 ++bus;
934 break;
935 case 2:
936 if (io_apic_entry(position, apic))
937 ++apic;
938 break;
939 case 3:
940 if (int_entry(position, intr))
941 ++intr;
942 break;
943 case 4:
944 /* int_entry(position); */
945 break;
946 default:
947 panic("mpfps Base Table HOSED!");
948 /* NOTREACHED */
949 }
950
951 totalSize -= basetable_entry_types[type].length;
952 (u_char *) position += basetable_entry_types[type].length;
953 }
954
955 if (boot_cpu_id == -1)
956 panic("NO BSP found!");
957
958 /* report fact that its NOT a default configuration */
959 return 0;
960}
961
962
963/*
964 * parse an Intel MP specification table
965 */
966static void
967fix_mp_table(void)
968{
969 int x;
970 int id;
971 int bus_0;
972 int bus_pci;
973 int num_pci_bus;
974
975 /*
976 * Fix mis-numbering of the PCI bus and its INT entries if the BIOS
977 * did it wrong. The MP spec says that when more than 1 PCI bus
978 * exists the BIOS must begin with bus entries for the PCI bus and use
979 * actual PCI bus numbering. This implies that when only 1 PCI bus
980 * exists the BIOS can choose to ignore this ordering, and indeed many
981 * MP motherboards do ignore it. This causes a problem when the PCI
982 * sub-system makes requests of the MP sub-system based on PCI bus
983 * numbers. So here we look for the situation and renumber the
984 * busses and associated INTs in an effort to "make it right".
985 */
986
987 /* find bus 0, PCI bus, count the number of PCI busses */
988 for (num_pci_bus = 0, x = 0; x < mp_nbusses; ++x) {
989 if (bus_data[x].bus_id == 0) {
990 bus_0 = x;
991 }
992 if (bus_data[x].bus_type == PCI) {
993 ++num_pci_bus;
994 bus_pci = x;
995 }
996 }
997 /*
998 * bus_0 == slot of bus with ID of 0
999 * bus_pci == slot of last PCI bus encountered
1000 */
1001
1002 /* check the 1 PCI bus case for sanity */
1003 if (num_pci_bus == 1) {
1004
1005 /* if it is number 0 all is well */
1006 if (bus_data[bus_pci].bus_id == 0)
1007 return;
1008
1009 /* mis-numbered, swap with whichever bus uses slot 0 */
1010
1011 /* swap the bus entry types */
1012 bus_data[bus_pci].bus_type = bus_data[bus_0].bus_type;
1013 bus_data[bus_0].bus_type = PCI;
1014
1015 /* swap each relavant INTerrupt entry */
1016 id = bus_data[bus_pci].bus_id;
1017 for (x = 0; x < nintrs; ++x) {
1018 if (io_apic_ints[x].src_bus_id == id) {
1019 io_apic_ints[x].src_bus_id = 0;
1020 }
1021 else if (io_apic_ints[x].src_bus_id == 0) {
1022 io_apic_ints[x].src_bus_id = id;
1023 }
1024 }
1025 }
1026 /* sanity check if more than 1 PCI bus */
1027 else if (num_pci_bus > 1) {
1028 for (x = 0; x < mp_nbusses; ++x) {
1029 if (bus_data[x].bus_type != PCI)
1030 continue;
1031 if (bus_data[x].bus_id >= num_pci_bus)
1032 panic("bad PCI bus numbering");
1033 }
1034 }
1035}
1036
1037
1038static int
1039processor_entry(proc_entry_ptr entry, int cpu)
1040{
1041 /* check for usability */
1042 if ((cpu >= NCPU) || !(entry->cpu_flags & PROCENTRY_FLAG_EN))
1043 return 0;
1044
1045 /* check for BSP flag */
1046 if (entry->cpu_flags & PROCENTRY_FLAG_BP) {
1047 boot_cpu_id = entry->apic_id;
1048 CPU_TO_ID(0) = entry->apic_id;
1049 ID_TO_CPU(entry->apic_id) = 0;
1050 return 0; /* its already been counted */
1051 }
1052
1053 /* add another AP to list, if less than max number of CPUs */
1054 else {
1055 CPU_TO_ID(cpu) = entry->apic_id;
1056 ID_TO_CPU(entry->apic_id) = cpu;
1057 return 1;
1058 }
1059}
1060
1061
1062static int
1063bus_entry(bus_entry_ptr entry, int bus)
1064{
1065 int x;
1066 char c, name[8];
1067
1068 /* encode the name into an index */
1069 for (x = 0; x < 6; ++x) {
1070 if ((c = entry->bus_type[x]) == ' ')
1071 break;
1072 name[x] = c;
1073 }
1074 name[x] = '\0';
1075
1076 if ((x = lookup_bus_type(name)) == UNKNOWN_BUSTYPE)
1077 panic("unknown bus type: '%s'", name);
1078
1079 bus_data[bus].bus_id = entry->bus_id;
1080 bus_data[bus].bus_type = x;
1081
1082 return 1;
1083}
1084
1085
1086static int
1087io_apic_entry(io_apic_entry_ptr entry, int apic)
1088{
1089 if (!(entry->apic_flags & IOAPICENTRY_FLAG_EN))
1090 return 0;
1091
1092 IO_TO_ID(apic) = entry->apic_id;
1093 ID_TO_IO(entry->apic_id) = apic;
1094
1095 return 1;
1096}
1097
1098
1099static int
1100lookup_bus_type(char *name)
1101{
1102 int x;
1103
1104 for (x = 0; x < MAX_BUSTYPE; ++x)
1105 if (strcmp(bus_type_table[x].name, name) == 0)
1106 return bus_type_table[x].type;
1107
1108 return UNKNOWN_BUSTYPE;
1109}
1110
1111
1112static int
1113int_entry(int_entry_ptr entry, int intr)
1114{
1115 io_apic_ints[intr].int_type = entry->int_type;
1116 io_apic_ints[intr].int_flags = entry->int_flags;
1117 io_apic_ints[intr].src_bus_id = entry->src_bus_id;
1118 io_apic_ints[intr].src_bus_irq = entry->src_bus_irq;
1119 io_apic_ints[intr].dst_apic_id = entry->dst_apic_id;
1120 io_apic_ints[intr].dst_apic_int = entry->dst_apic_int;
1121
1122 return 1;
1123}
1124
1125
1126static int
1127apic_int_is_bus_type(int intr, int bus_type)
1128{
1129 int bus;
1130
1131 for (bus = 0; bus < mp_nbusses; ++bus)
1132 if ((bus_data[bus].bus_id == io_apic_ints[intr].src_bus_id)
1133 && ((int) bus_data[bus].bus_type == bus_type))
1134 return 1;
1135
1136 return 0;
1137}
1138
1139
1140/*
1141 * Given a traditional ISA INT mask, return an APIC mask.
1142 */
1143u_int
1144isa_apic_mask(u_int isa_mask)
1145{
1146 int isa_irq;
1147 int apic_pin;
1148
1149#if defined(SKIP_IRQ15_REDIRECT)
1150 if (isa_mask == (1 << 15)) {
1151 printf("skipping ISA IRQ15 redirect\n");
1152 return isa_mask;
1153 }
1154#endif /* SKIP_IRQ15_REDIRECT */
1155
1156 isa_irq = ffs(isa_mask); /* find its bit position */
1157 if (isa_irq == 0) /* doesn't exist */
1158 return 0;
1159 --isa_irq; /* make it zero based */
1160
1161 apic_pin = isa_apic_pin(isa_irq); /* look for APIC connection */
1162 if (apic_pin == -1)
1163 return 0;
1164
1165 return (1 << apic_pin); /* convert pin# to a mask */
1166}
1167
1168
1169/*
1170 * Determine which APIC pin an ISA/EISA INT is attached to.
1171 */
1172#define INTTYPE(I) (io_apic_ints[(I)].int_type)
1173#define INTPIN(I) (io_apic_ints[(I)].dst_apic_int)
1174
1175#define SRCBUSIRQ(I) (io_apic_ints[(I)].src_bus_irq)
1176int
1177isa_apic_pin(int isa_irq)
1178{
1179 int intr;
1180
1181 for (intr = 0; intr < nintrs; ++intr) { /* check each record */
1182 if (INTTYPE(intr) == 0) { /* standard INT */
1183 if (SRCBUSIRQ(intr) == isa_irq) {
1184 if (apic_int_is_bus_type(intr, ISA) ||
1185 apic_int_is_bus_type(intr, EISA))
1186 return INTPIN(intr); /* found */
1187 }
1188 }
1189 }
1190 return -1; /* NOT found */
1191}
1192
1193
1194/*
1195 * Determine which APIC pin a PCI INT is attached to.
1196 */
1197#define SRCBUSID(I) (io_apic_ints[(I)].src_bus_id)
1198#define SRCBUSDEVICE(I) ((io_apic_ints[(I)].src_bus_irq >> 2) & 0x1f)
1199#define SRCBUSLINE(I) (io_apic_ints[(I)].src_bus_irq & 0x03)
1200int
1201pci_apic_pin(int pciBus, int pciDevice, int pciInt)
1202{
1203 int intr;
1204
1205 --pciInt; /* zero based */
1206
1207 for (intr = 0; intr < nintrs; ++intr) /* check each record */
1208 if ((INTTYPE(intr) == 0) /* standard INT */
1209 && (SRCBUSID(intr) == pciBus)
1210 && (SRCBUSDEVICE(intr) == pciDevice)
1211 && (SRCBUSLINE(intr) == pciInt)) /* a candidate IRQ */
1212 if (apic_int_is_bus_type(intr, PCI))
1213 return INTPIN(intr); /* exact match */
1214
1215 return -1; /* NOT found */
1216}
1217
1218int
1219next_apic_pin(int pin)
1220{
1221 int intr, ointr;
1222 int bus, bustype;
1223
1224 bus = 0;
1225 bustype = 0;
1226 for (intr = 0; intr < nintrs; intr++) {
1227 if (INTPIN(intr) != pin || INTTYPE(intr) != 0)
1228 continue;
1229 bus = SRCBUSID(intr);
1230 bustype = apic_bus_type(bus);
1231 if (bustype != ISA &&
1232 bustype != EISA &&
1233 bustype != PCI)
1234 continue;
1235 break;
1236 }
1237 if (intr >= nintrs) {
1238 return -1;
1239 }
1240 for (ointr = intr + 1; ointr < nintrs; ointr++) {
1241 if (INTTYPE(ointr) != 0)
1242 continue;
1243 if (bus != SRCBUSID(ointr))
1244 continue;
1245 if (bustype == PCI) {
1246 if (SRCBUSDEVICE(intr) != SRCBUSDEVICE(ointr))
1247 continue;
1248 if (SRCBUSLINE(intr) != SRCBUSLINE(ointr))
1249 continue;
1250 }
1251 if (bustype == ISA || bustype == EISA) {
1252 if (SRCBUSIRQ(intr) != SRCBUSIRQ(ointr))
1253 continue;
1254 }
1255 if (INTPIN(intr) == INTPIN(ointr))
1256 continue;
1257 break;
1258 }
1259 if (ointr >= nintrs) {
1260 return -1;
1261 }
1262 return INTPIN(ointr);
1263}
1264#undef SRCBUSLINE
1265#undef SRCBUSDEVICE
1266#undef SRCBUSID
1267#undef SRCBUSIRQ
1268
1269#undef INTPIN
1270#undef INTTYPE
1271
1272
1273/*
1274 * Reprogram the MB chipset to NOT redirect an ISA INTerrupt.
1275 *
1276 * XXX FIXME:
1277 * Exactly what this means is unclear at this point. It is a solution
1278 * for motherboards that redirect the MBIRQ0 pin. Generically a motherboard
1279 * could route any of the ISA INTs to upper (>15) IRQ values. But most would
1280 * NOT be redirected via MBIRQ0, thus "undirect()ing" them would NOT be an
1281 * option.
1282 */
1283int
1284undirect_isa_irq(int rirq)
1285{
1286#if defined(READY)
1287 printf("Freeing redirected ISA irq %d.\n", rirq);
1288 /** FIXME: tickle the MB redirector chip */
1289 return ???;
1290#else
1291 printf("Freeing (NOT implemented) redirected ISA irq %d.\n", rirq);
1292 return 0;
1293#endif /* READY */
1294}
1295
1296
1297/*
1298 * Reprogram the MB chipset to NOT redirect a PCI INTerrupt
1299 */
1300int
1301undirect_pci_irq(int rirq)
1302{
1303#if defined(READY)
1304 if (bootverbose)
1305 printf("Freeing redirected PCI irq %d.\n", rirq);
1306
1307 /** FIXME: tickle the MB redirector chip */
1308 return ???;
1309#else
1310 if (bootverbose)
1311 printf("Freeing (NOT implemented) redirected PCI irq %d.\n",
1312 rirq);
1313 return 0;
1314#endif /* READY */
1315}
1316
1317
1318/*
1319 * given a bus ID, return:
1320 * the bus type if found
1321 * -1 if NOT found
1322 */
1323int
1324apic_bus_type(int id)
1325{
1326 int x;
1327
1328 for (x = 0; x < mp_nbusses; ++x)
1329 if (bus_data[x].bus_id == id)
1330 return bus_data[x].bus_type;
1331
1332 return -1;
1333}
1334
1335
1336/*
1337 * given a LOGICAL APIC# and pin#, return:
1338 * the associated src bus ID if found
1339 * -1 if NOT found
1340 */
1341int
1342apic_src_bus_id(int apic, int pin)
1343{
1344 int x;
1345
1346 /* search each of the possible INTerrupt sources */
1347 for (x = 0; x < nintrs; ++x)
1348 if ((apic == ID_TO_IO(io_apic_ints[x].dst_apic_id)) &&
1349 (pin == io_apic_ints[x].dst_apic_int))
1350 return (io_apic_ints[x].src_bus_id);
1351
1352 return -1; /* NOT found */
1353}
1354
1355
1356/*
1357 * given a LOGICAL APIC# and pin#, return:
1358 * the associated src bus IRQ if found
1359 * -1 if NOT found
1360 */
1361int
1362apic_src_bus_irq(int apic, int pin)
1363{
1364 int x;
1365
1366 for (x = 0; x < nintrs; x++)
1367 if ((apic == ID_TO_IO(io_apic_ints[x].dst_apic_id)) &&
1368 (pin == io_apic_ints[x].dst_apic_int))
1369 return (io_apic_ints[x].src_bus_irq);
1370
1371 return -1; /* NOT found */
1372}
1373
1374
1375/*
1376 * given a LOGICAL APIC# and pin#, return:
1377 * the associated INTerrupt type if found
1378 * -1 if NOT found
1379 */
1380int
1381apic_int_type(int apic, int pin)
1382{
1383 int x;
1384
1385 /* search each of the possible INTerrupt sources */
1386 for (x = 0; x < nintrs; ++x)
1387 if ((apic == ID_TO_IO(io_apic_ints[x].dst_apic_id)) &&
1388 (pin == io_apic_ints[x].dst_apic_int))
1389 return (io_apic_ints[x].int_type);
1390
1391 return -1; /* NOT found */
1392}
1393
1394
1395/*
1396 * given a LOGICAL APIC# and pin#, return:
1397 * the associated trigger mode if found
1398 * -1 if NOT found
1399 */
1400int
1401apic_trigger(int apic, int pin)
1402{
1403 int x;
1404
1405 /* search each of the possible INTerrupt sources */
1406 for (x = 0; x < nintrs; ++x)
1407 if ((apic == ID_TO_IO(io_apic_ints[x].dst_apic_id)) &&
1408 (pin == io_apic_ints[x].dst_apic_int))
1409 return ((io_apic_ints[x].int_flags >> 2) & 0x03);
1410
1411 return -1; /* NOT found */
1412}
1413
1414
1415/*
1416 * given a LOGICAL APIC# and pin#, return:
1417 * the associated 'active' level if found
1418 * -1 if NOT found
1419 */
1420int
1421apic_polarity(int apic, int pin)
1422{
1423 int x;
1424
1425 /* search each of the possible INTerrupt sources */
1426 for (x = 0; x < nintrs; ++x)
1427 if ((apic == ID_TO_IO(io_apic_ints[x].dst_apic_id)) &&
1428 (pin == io_apic_ints[x].dst_apic_int))
1429 return (io_apic_ints[x].int_flags & 0x03);
1430
1431 return -1; /* NOT found */
1432}
1433
1434
1435/*
1436 * set data according to MP defaults
1437 * FIXME: probably not complete yet...
1438 */
1439static void
1440default_mp_table(int type)
1441{
1442 int ap_cpu_id;
1443#if defined(APIC_IO)
1444 u_int32_t ux;
1445 int io_apic_id;
1446 int pin;
1447#endif /* APIC_IO */
1448
1449#if 0
1450 printf(" MP default config type: %d\n", type);
1451 switch (type) {
1452 case 1:
1453 printf(" bus: ISA, APIC: 82489DX\n");
1454 break;
1455 case 2:
1456 printf(" bus: EISA, APIC: 82489DX\n");
1457 break;
1458 case 3:
1459 printf(" bus: EISA, APIC: 82489DX\n");
1460 break;
1461 case 4:
1462 printf(" bus: MCA, APIC: 82489DX\n");
1463 break;
1464 case 5:
1465 printf(" bus: ISA+PCI, APIC: Integrated\n");
1466 break;
1467 case 6:
1468 printf(" bus: EISA+PCI, APIC: Integrated\n");
1469 break;
1470 case 7:
1471 printf(" bus: MCA+PCI, APIC: Integrated\n");
1472 break;
1473 default:
1474 printf(" future type\n");
1475 break;
1476 /* NOTREACHED */
1477 }
1478#endif /* 0 */
1479
1480 boot_cpu_id = (lapic.id & APIC_ID_MASK) >> 24;
1481 ap_cpu_id = (boot_cpu_id == 0) ? 1 : 0;
1482
1483 /* BSP */
1484 CPU_TO_ID(0) = boot_cpu_id;
1485 ID_TO_CPU(boot_cpu_id) = 0;
1486
1487 /* one and only AP */
1488 CPU_TO_ID(1) = ap_cpu_id;
1489 ID_TO_CPU(ap_cpu_id) = 1;
1490
1491#if defined(APIC_IO)
1492 /* one and only IO APIC */
1493 io_apic_id = (io_apic_read(0, IOAPIC_ID) & APIC_ID_MASK) >> 24;
1494
1495 /*
1496 * sanity check, refer to MP spec section 3.6.6, last paragraph
1497 * necessary as some hardware isn't properly setting up the IO APIC
1498 */
1499#if defined(REALLY_ANAL_IOAPICID_VALUE)
1500 if (io_apic_id != 2) {
1501#else
1502 if ((io_apic_id == 0) || (io_apic_id == 1) || (io_apic_id == 15)) {
1503#endif /* REALLY_ANAL_IOAPICID_VALUE */
1504 ux = io_apic_read(0, IOAPIC_ID); /* get current contents */
1505 ux &= ~APIC_ID_MASK; /* clear the ID field */
1506 ux |= 0x02000000; /* set it to '2' */
1507 io_apic_write(0, IOAPIC_ID, ux); /* write new value */
1508 ux = io_apic_read(0, IOAPIC_ID); /* re-read && test */
1509 if ((ux & APIC_ID_MASK) != 0x02000000)
1510 panic("can't control IO APIC ID, reg: 0x%08x", ux);
1511 io_apic_id = 2;
1512 }
1513 IO_TO_ID(0) = io_apic_id;
1514 ID_TO_IO(io_apic_id) = 0;
1515#endif /* APIC_IO */
1516
1517 /* fill out bus entries */
1518 switch (type) {
1519 case 1:
1520 case 2:
1521 case 3:
1522 case 5:
1523 case 6:
1524 bus_data[0].bus_id = default_data[type - 1][1];
1525 bus_data[0].bus_type = default_data[type - 1][2];
1526 bus_data[1].bus_id = default_data[type - 1][3];
1527 bus_data[1].bus_type = default_data[type - 1][4];
1528 break;
1529
1530 /* case 4: case 7: MCA NOT supported */
1531 default: /* illegal/reserved */
1532 panic("BAD default MP config: %d", type);
1533 /* NOTREACHED */
1534 }
1535
1536#if defined(APIC_IO)
1537 /* general cases from MP v1.4, table 5-2 */
1538 for (pin = 0; pin < 16; ++pin) {
1539 io_apic_ints[pin].int_type = 0;
1540 io_apic_ints[pin].int_flags = 0x05; /* edge/active-hi */
1541 io_apic_ints[pin].src_bus_id = 0;
1542 io_apic_ints[pin].src_bus_irq = pin; /* IRQ2 caught below */
1543 io_apic_ints[pin].dst_apic_id = io_apic_id;
1544 io_apic_ints[pin].dst_apic_int = pin; /* 1-to-1 */
1545 }
1546
1547 /* special cases from MP v1.4, table 5-2 */
1548 if (type == 2) {
1549 io_apic_ints[2].int_type = 0xff; /* N/C */
1550 io_apic_ints[13].int_type = 0xff; /* N/C */
1551#if !defined(APIC_MIXED_MODE)
1552 /** FIXME: ??? */
1553 panic("sorry, can't support type 2 default yet");
1554#endif /* APIC_MIXED_MODE */
1555 }
1556 else
1557 io_apic_ints[2].src_bus_irq = 0; /* ISA IRQ0 is on APIC INT 2 */
1558
1559 if (type == 7)
1560 io_apic_ints[0].int_type = 0xff; /* N/C */
1561 else
1562 io_apic_ints[0].int_type = 3; /* vectored 8259 */
1563#endif /* APIC_IO */
1564}
1565
1566
1567/*
1568 * initialize all the SMP locks
1569 */
1570
1571/* critical region around IO APIC, apic_imen */
1572struct simplelock imen_lock;
1573
1574/* critical region around splxx(), cpl, cml, cil, ipending */
1575struct simplelock cpl_lock;
1576
1577/* Make FAST_INTR() routines sequential */
1578struct simplelock fast_intr_lock;
1579
1580/* critical region around INTR() routines */
1581struct simplelock intr_lock;
1582
1583/* lock regions protected in UP kernel via cli/sti */
1584struct simplelock mpintr_lock;
1585
1586/* lock region used by kernel profiling */
1587struct simplelock mcount_lock;
1588
1589#ifdef USE_COMLOCK
1590/* locks com (tty) data/hardware accesses: a FASTINTR() */
1591struct simplelock com_lock;
1592#endif /* USE_COMLOCK */
1593
1594#ifdef USE_CLOCKLOCK
1595/* lock regions around the clock hardware */
1596struct simplelock clock_lock;
1597#endif /* USE_CLOCKLOCK */
1598
1599static void
1600init_locks(void)
1601{
1602 /*
1603 * Get the initial mp_lock with a count of 1 for the BSP.
1604 * This uses a LOGICAL cpu ID, ie BSP == 0.
1605 */
1606 mp_lock = 0x00000001;
1607
1608 /* ISR uses its own "giant lock" */
1609 isr_lock = FREE_LOCK;
1610
1611#if defined(APIC_INTR_DIAGNOSTIC) && defined(APIC_INTR_DIAGNOSTIC_IRQ)
1612 s_lock_init((struct simplelock*)&apic_itrace_debuglock);
1613#endif
1614
1615 s_lock_init((struct simplelock*)&mpintr_lock);
1616
1617 s_lock_init((struct simplelock*)&mcount_lock);
1618
1619 s_lock_init((struct simplelock*)&fast_intr_lock);
1620 s_lock_init((struct simplelock*)&intr_lock);
1621 s_lock_init((struct simplelock*)&imen_lock);
1622 s_lock_init((struct simplelock*)&cpl_lock);
1623
1624#ifdef USE_COMLOCK
1625 s_lock_init((struct simplelock*)&com_lock);
1626#endif /* USE_COMLOCK */
1627#ifdef USE_CLOCKLOCK
1628 s_lock_init((struct simplelock*)&clock_lock);
1629#endif /* USE_CLOCKLOCK */
1630}
1631
1632
1633/*
1634 * start each AP in our list
1635 */
1636static int
1637start_all_aps(u_int boot_addr)
1638{
1639 int x, i;
1640 u_char mpbiosreason;
1641 u_long mpbioswarmvec;
1642 pd_entry_t *newptd;
1643 pt_entry_t *newpt;
1644 struct globaldata *gd;
1645 char *stack;
1646 pd_entry_t *myPTD;
1647
1648 POSTCODE(START_ALL_APS_POST);
1649
1650 /* initialize BSP's local APIC */
1651 apic_initialize();
1652 bsp_apic_ready = 1;
1653
1654 /* install the AP 1st level boot code */
1655 install_ap_tramp(boot_addr);
1656
1657
1658 /* save the current value of the warm-start vector */
1659 mpbioswarmvec = *((u_long *) WARMBOOT_OFF);
1660 outb(CMOS_REG, BIOS_RESET);
1661 mpbiosreason = inb(CMOS_DATA);
1662
1663 /* record BSP in CPU map */
1664 all_cpus = 1;
1665
1666 /* start each AP */
1667 for (x = 1; x <= mp_naps; ++x) {
1668
1669 /* This is a bit verbose, it will go away soon. */
1670
1671 /* alloc new page table directory */
1672 newptd = (pd_entry_t *)(kmem_alloc(kernel_map, PAGE_SIZE));
1673
1674 /* Store the virtual PTD address for this CPU */
1675 IdlePTDS[x] = newptd;
1676
1677 /* clone currently active one (ie: IdlePTD) */
1678 bcopy(PTD, newptd, PAGE_SIZE); /* inc prv page pde */
1679
1680 /* set up 0 -> 4MB P==V mapping for AP boot */
1681 newptd[0] = (pd_entry_t) (PG_V | PG_RW |
1682 ((u_long)KPTphys & PG_FRAME));
1683
1684 /* store PTD for this AP's boot sequence */
1685 myPTD = (pd_entry_t *)vtophys(newptd);
1686
1687 /* alloc new page table page */
1688 newpt = (pt_entry_t *)(kmem_alloc(kernel_map, PAGE_SIZE));
1689
1690 /* set the new PTD's private page to point there */
1691 newptd[MPPTDI] = (pt_entry_t)(PG_V | PG_RW | vtophys(newpt));
1692
1693 /* install self referential entry */
1694 newptd[PTDPTDI] = (pd_entry_t)(PG_V | PG_RW | vtophys(newptd));
1695
1696 /* allocate a new private data page */
1697 gd = (struct globaldata *)kmem_alloc(kernel_map, PAGE_SIZE);
1698
1699 /* wire it into the private page table page */
1700 newpt[0] = (pt_entry_t)(PG_V | PG_RW | vtophys(gd));
1701
1702 /* wire the ptp into itself for access */
1703 newpt[1] = (pt_entry_t)(PG_V | PG_RW | vtophys(newpt));
1704
1705 /* copy in the pointer to the local apic */
1706 newpt[2] = SMP_prvpt[2];
1707
1708 /* and the IO apic mapping[s] */
1709 for (i = 16; i < 32; i++)
1710 newpt[i] = SMP_prvpt[i];
1711
1712 /* allocate and set up an idle stack data page */
1713 stack = (char *)kmem_alloc(kernel_map, UPAGES*PAGE_SIZE);
1714 for (i = 0; i < UPAGES; i++)
1715 newpt[i + 3] = (pt_entry_t)(PG_V | PG_RW | vtophys(PAGE_SIZE * i + stack));
1716
1717 newpt[3 + UPAGES] = 0; /* *prv_CMAP1 */
1718 newpt[4 + UPAGES] = 0; /* *prv_CMAP2 */
1719 newpt[5 + UPAGES] = 0; /* *prv_CMAP3 */
|
1728
1729 /* setup a vector to our boot code */
1730 *((volatile u_short *) WARMBOOT_OFF) = WARMBOOT_TARGET;
1731 *((volatile u_short *) WARMBOOT_SEG) = (boot_addr >> 4);
1732 outb(CMOS_REG, BIOS_RESET);
1733 outb(CMOS_DATA, BIOS_WARM); /* 'warm-start' */
1734
1735 bootPTD = myPTD;
1736 /* attempt to start the Application Processor */
1737 CHECK_INIT(99); /* setup checkpoints */
1738 if (!start_ap(x, boot_addr)) {
1739 printf("AP #%d (PHY# %d) failed!\n", x, CPU_TO_ID(x));
1740 CHECK_PRINT("trace"); /* show checkpoints */
1741 /* better panic as the AP may be running loose */
1742 printf("panic y/n? [y] ");
1743 if (cngetc() != 'n')
1744 panic("bye-bye");
1745 }
1746 CHECK_PRINT("trace"); /* show checkpoints */
1747
1748 /* record its version info */
1749 cpu_apic_versions[x] = cpu_apic_versions[0];
1750
1751 all_cpus |= (1 << x); /* record AP in CPU map */
1752 }
1753
1754 /* build our map of 'other' CPUs */
1755 other_cpus = all_cpus & ~(1 << cpuid);
1756
1757 /* fill in our (BSP) APIC version */
1758 cpu_apic_versions[0] = lapic.version;
1759
1760 /* restore the warmstart vector */
1761 *(u_long *) WARMBOOT_OFF = mpbioswarmvec;
1762 outb(CMOS_REG, BIOS_RESET);
1763 outb(CMOS_DATA, mpbiosreason);
1764
1765 /*
1766 * Set up the idle context for the BSP. Similar to above except
1767 * that some was done by locore, some by pmap.c and some is implicit
1768 * because the BSP is cpu#0 and the page is initially zero, and also
1769 * because we can refer to variables by name on the BSP..
1770 */
1771 newptd = (pd_entry_t *)(kmem_alloc(kernel_map, PAGE_SIZE));
1772
1773 bcopy(PTD, newptd, PAGE_SIZE); /* inc prv page pde */
1774 IdlePTDS[0] = newptd;
1775
1776 /* Point PTD[] to this page instead of IdlePTD's physical page */
1777 newptd[PTDPTDI] = (pd_entry_t)(PG_V | PG_RW | vtophys(newptd));
1778
1779 my_idlePTD = (pd_entry_t *)vtophys(newptd);
1780
1781 /* Allocate and setup BSP idle stack */
1782 stack = (char *)kmem_alloc(kernel_map, UPAGES * PAGE_SIZE);
1783 for (i = 0; i < UPAGES; i++)
1784 SMP_prvpt[i + 3] = (pt_entry_t)(PG_V | PG_RW | vtophys(PAGE_SIZE * i + stack));
1785
1786 pmap_set_opt_bsp();
1787
1788 for (i = 0; i < mp_ncpus; i++) {
1789 bcopy( (int *) PTD + KPTDI, (int *) IdlePTDS[i] + KPTDI, NKPDE * sizeof (int));
1790 }
1791
1792 /* number of APs actually started */
1793 return mp_ncpus - 1;
1794}
1795
1796
1797/*
1798 * load the 1st level AP boot code into base memory.
1799 */
1800
1801/* targets for relocation */
1802extern void bigJump(void);
1803extern void bootCodeSeg(void);
1804extern void bootDataSeg(void);
1805extern void MPentry(void);
1806extern u_int MP_GDT;
1807extern u_int mp_gdtbase;
1808
1809static void
1810install_ap_tramp(u_int boot_addr)
1811{
1812 int x;
1813 int size = *(int *) ((u_long) & bootMP_size);
1814 u_char *src = (u_char *) ((u_long) bootMP);
1815 u_char *dst = (u_char *) boot_addr + KERNBASE;
1816 u_int boot_base = (u_int) bootMP;
1817 u_int8_t *dst8;
1818 u_int16_t *dst16;
1819 u_int32_t *dst32;
1820
1821 POSTCODE(INSTALL_AP_TRAMP_POST);
1822
1823 for (x = 0; x < size; ++x)
1824 *dst++ = *src++;
1825
1826 /*
1827 * modify addresses in code we just moved to basemem. unfortunately we
1828 * need fairly detailed info about mpboot.s for this to work. changes
1829 * to mpboot.s might require changes here.
1830 */
1831
1832 /* boot code is located in KERNEL space */
1833 dst = (u_char *) boot_addr + KERNBASE;
1834
1835 /* modify the lgdt arg */
1836 dst32 = (u_int32_t *) (dst + ((u_int) & mp_gdtbase - boot_base));
1837 *dst32 = boot_addr + ((u_int) & MP_GDT - boot_base);
1838
1839 /* modify the ljmp target for MPentry() */
1840 dst32 = (u_int32_t *) (dst + ((u_int) bigJump - boot_base) + 1);
1841 *dst32 = ((u_int) MPentry - KERNBASE);
1842
1843 /* modify the target for boot code segment */
1844 dst16 = (u_int16_t *) (dst + ((u_int) bootCodeSeg - boot_base));
1845 dst8 = (u_int8_t *) (dst16 + 1);
1846 *dst16 = (u_int) boot_addr & 0xffff;
1847 *dst8 = ((u_int) boot_addr >> 16) & 0xff;
1848
1849 /* modify the target for boot data segment */
1850 dst16 = (u_int16_t *) (dst + ((u_int) bootDataSeg - boot_base));
1851 dst8 = (u_int8_t *) (dst16 + 1);
1852 *dst16 = (u_int) boot_addr & 0xffff;
1853 *dst8 = ((u_int) boot_addr >> 16) & 0xff;
1854}
1855
1856
1857/*
1858 * this function starts the AP (application processor) identified
1859 * by the APIC ID 'physicalCpu'. It does quite a "song and dance"
1860 * to accomplish this. This is necessary because of the nuances
1861 * of the different hardware we might encounter. It ain't pretty,
1862 * but it seems to work.
1863 */
1864static int
1865start_ap(int logical_cpu, u_int boot_addr)
1866{
1867 int physical_cpu;
1868 int vector;
1869 int cpus;
1870 u_long icr_lo, icr_hi;
1871
1872 POSTCODE(START_AP_POST);
1873
1874 /* get the PHYSICAL APIC ID# */
1875 physical_cpu = CPU_TO_ID(logical_cpu);
1876
1877 /* calculate the vector */
1878 vector = (boot_addr >> 12) & 0xff;
1879
1880 /* used as a watchpoint to signal AP startup */
1881 cpus = mp_ncpus;
1882
1883 /*
1884 * first we do an INIT/RESET IPI this INIT IPI might be run, reseting
1885 * and running the target CPU. OR this INIT IPI might be latched (P5
1886 * bug), CPU waiting for STARTUP IPI. OR this INIT IPI might be
1887 * ignored.
1888 */
1889
1890 /* setup the address for the target AP */
1891 icr_hi = lapic.icr_hi & ~APIC_ID_MASK;
1892 icr_hi |= (physical_cpu << 24);
1893 lapic.icr_hi = icr_hi;
1894
1895 /* do an INIT IPI: assert RESET */
1896 icr_lo = lapic.icr_lo & 0xfff00000;
1897 lapic.icr_lo = icr_lo | 0x0000c500;
1898
1899 /* wait for pending status end */
1900 while (lapic.icr_lo & APIC_DELSTAT_MASK)
1901 /* spin */ ;
1902
1903 /* do an INIT IPI: deassert RESET */
1904 lapic.icr_lo = icr_lo | 0x00008500;
1905
1906 /* wait for pending status end */
1907 u_sleep(10000); /* wait ~10mS */
1908 while (lapic.icr_lo & APIC_DELSTAT_MASK)
1909 /* spin */ ;
1910
1911 /*
1912 * next we do a STARTUP IPI: the previous INIT IPI might still be
1913 * latched, (P5 bug) this 1st STARTUP would then terminate
1914 * immediately, and the previously started INIT IPI would continue. OR
1915 * the previous INIT IPI has already run. and this STARTUP IPI will
1916 * run. OR the previous INIT IPI was ignored. and this STARTUP IPI
1917 * will run.
1918 */
1919
1920 /* do a STARTUP IPI */
1921 lapic.icr_lo = icr_lo | 0x00000600 | vector;
1922 while (lapic.icr_lo & APIC_DELSTAT_MASK)
1923 /* spin */ ;
1924 u_sleep(200); /* wait ~200uS */
1925
1926 /*
1927 * finally we do a 2nd STARTUP IPI: this 2nd STARTUP IPI should run IF
1928 * the previous STARTUP IPI was cancelled by a latched INIT IPI. OR
1929 * this STARTUP IPI will be ignored, as only ONE STARTUP IPI is
1930 * recognized after hardware RESET or INIT IPI.
1931 */
1932
1933 lapic.icr_lo = icr_lo | 0x00000600 | vector;
1934 while (lapic.icr_lo & APIC_DELSTAT_MASK)
1935 /* spin */ ;
1936 u_sleep(200); /* wait ~200uS */
1937
1938 /* wait for it to start */
1939 set_apic_timer(5000000);/* == 5 seconds */
1940 while (read_apic_timer())
1941 if (mp_ncpus > cpus)
1942 return 1; /* return SUCCESS */
1943
1944 return 0; /* return FAILURE */
1945}
1946
1947
1948/*
1949 * Flush the TLB on all other CPU's
1950 *
1951 * XXX: Needs to handshake and wait for completion before proceding.
1952 */
1953void
1954smp_invltlb(void)
1955{
1956#if defined(APIC_IO)
1957 if (smp_started && invltlb_ok)
1958 all_but_self_ipi(XINVLTLB_OFFSET);
1959#endif /* APIC_IO */
1960}
1961
1962void
1963invlpg(u_int addr)
1964{
1965 __asm __volatile("invlpg (%0)"::"r"(addr):"memory");
1966
1967 /* send a message to the other CPUs */
1968 smp_invltlb();
1969}
1970
1971void
1972invltlb(void)
1973{
1974 u_long temp;
1975
1976 /*
1977 * This should be implemented as load_cr3(rcr3()) when load_cr3() is
1978 * inlined.
1979 */
1980 __asm __volatile("movl %%cr3, %0; movl %0, %%cr3":"=r"(temp) :: "memory");
1981
1982 /* send a message to the other CPUs */
1983 smp_invltlb();
1984}
1985
1986
1987/*
1988 * When called the executing CPU will send an IPI to all other CPUs
1989 * requesting that they halt execution.
1990 *
1991 * Usually (but not necessarily) called with 'other_cpus' as its arg.
1992 *
1993 * - Signals all CPUs in map to stop.
1994 * - Waits for each to stop.
1995 *
1996 * Returns:
1997 * -1: error
1998 * 0: NA
1999 * 1: ok
2000 *
2001 * XXX FIXME: this is not MP-safe, needs a lock to prevent multiple CPUs
2002 * from executing at same time.
2003 */
2004int
2005stop_cpus(u_int map)
2006{
2007 if (!smp_started)
2008 return 0;
2009
2010 /* send IPI to all CPUs in map */
2011 stopped_cpus = 0;
2012
2013 /* send the Xcpustop IPI to all CPUs in map */
2014 selected_apic_ipi(map, XCPUSTOP_OFFSET, APIC_DELMODE_FIXED);
2015
2016 while (stopped_cpus != map)
2017 /* spin */ ;
2018
2019 return 1;
2020}
2021
2022
2023/*
2024 * Called by a CPU to restart stopped CPUs.
2025 *
2026 * Usually (but not necessarily) called with 'stopped_cpus' as its arg.
2027 *
2028 * - Signals all CPUs in map to restart.
2029 * - Waits for each to restart.
2030 *
2031 * Returns:
2032 * -1: error
2033 * 0: NA
2034 * 1: ok
2035 */
2036int
2037restart_cpus(u_int map)
2038{
2039 if (!smp_started)
2040 return 0;
2041
2042 started_cpus = map; /* signal other cpus to restart */
2043
2044 while (started_cpus) /* wait for each to clear its bit */
2045 /* spin */ ;
2046 stopped_cpus = 0;
2047
2048 return 1;
2049}
2050
2051int smp_active = 0; /* are the APs allowed to run? */
2052SYSCTL_INT(_machdep, OID_AUTO, smp_active, CTLFLAG_RW, &smp_active, 0, "");
2053
2054/* XXX maybe should be hw.ncpu */
2055static int smp_cpus = 1; /* how many cpu's running */
2056SYSCTL_INT(_machdep, OID_AUTO, smp_cpus, CTLFLAG_RD, &smp_cpus, 0, "");
2057
2058int invltlb_ok = 0; /* throttle smp_invltlb() till safe */
2059SYSCTL_INT(_machdep, OID_AUTO, invltlb_ok, CTLFLAG_RW, &invltlb_ok, 0, "");
2060
2061/* Warning: Do not staticize. Used from swtch.s */
2062int do_page_zero_idle = 1; /* bzero pages for fun and profit in idleloop */
2063SYSCTL_INT(_machdep, OID_AUTO, do_page_zero_idle, CTLFLAG_RW,
2064 &do_page_zero_idle, 0, "");
2065
2066/* Is forwarding of a interrupt to the CPU holding the ISR lock enabled ? */
2067int forward_irq_enabled = 1;
2068SYSCTL_INT(_machdep, OID_AUTO, forward_irq_enabled, CTLFLAG_RW,
2069 &forward_irq_enabled, 0, "");
2070
2071/* Enable forwarding of a signal to a process running on a different CPU */
2072int forward_signal_enabled = 1;
2073SYSCTL_INT(_machdep, OID_AUTO, forward_signal_enabled, CTLFLAG_RW,
2074 &forward_signal_enabled, 0, "");
2075
2076/*
2077 * This is called once the rest of the system is up and running and we're
2078 * ready to let the AP's out of the pen.
2079 */
2080void ap_init(void);
2081
2082void
2083ap_init()
2084{
2085 u_int temp;
2086 u_int apic_id;
2087
2088 smp_cpus++;
2089
2090#if defined(I586_CPU) && !defined(NO_F00F_HACK)
2091 lidt(&r_idt);
2092#endif
2093
2094 /* Build our map of 'other' CPUs. */
2095 other_cpus = all_cpus & ~(1 << cpuid);
2096
2097 printf("SMP: AP CPU #%d Launched!\n", cpuid);
2098
2099 /* XXX FIXME: i386 specific, and redundant: Setup the FPU. */
2100 load_cr0((rcr0() & ~CR0_EM) | CR0_MP | CR0_NE | CR0_TS);
2101
2102 /* A quick check from sanity claus */
2103 apic_id = (apic_id_to_logical[(lapic.id & 0x0f000000) >> 24]);
2104 if (cpuid != apic_id) {
2105 printf("SMP: cpuid = %d\n", cpuid);
2106 printf("SMP: apic_id = %d\n", apic_id);
2107 printf("PTD[MPPTDI] = %08x\n", PTD[MPPTDI]);
2108 panic("cpuid mismatch! boom!!");
2109 }
2110
2111 getmtrr();
2112
2113 /* Init local apic for irq's */
2114 apic_initialize();
2115
2116 /*
2117 * Activate smp_invltlb, although strictly speaking, this isn't
2118 * quite correct yet. We should have a bitfield for cpus willing
2119 * to accept TLB flush IPI's or something and sync them.
2120 */
2121 invltlb_ok = 1;
2122 smp_started = 1; /* enable IPI's, tlb shootdown, freezes etc */
2123 smp_active = 1; /* historic */
2124
2125 curproc = NULL; /* make sure */
2126}
2127
2128#ifdef BETTER_CLOCK
2129
2130#define CHECKSTATE_USER 0
2131#define CHECKSTATE_SYS 1
2132#define CHECKSTATE_INTR 2
2133
2134/* Do not staticize. Used from apic_vector.s */
2135struct proc* checkstate_curproc[NCPU];
2136int checkstate_cpustate[NCPU];
2137u_long checkstate_pc[NCPU];
2138
2139extern long cp_time[CPUSTATES];
2140
2141#define PC_TO_INDEX(pc, prof) \
2142 ((int)(((u_quad_t)((pc) - (prof)->pr_off) * \
2143 (u_quad_t)((prof)->pr_scale)) >> 16) & ~1)
2144
2145static void
2146addupc_intr_forwarded(struct proc *p, int id, int *astmap)
2147{
2148 int i;
2149 struct uprof *prof;
2150 u_long pc;
2151
2152 pc = checkstate_pc[id];
2153 prof = &p->p_stats->p_prof;
2154 if (pc >= prof->pr_off &&
2155 (i = PC_TO_INDEX(pc, prof)) < prof->pr_size) {
2156 if ((p->p_flag & P_OWEUPC) == 0) {
2157 prof->pr_addr = pc;
2158 prof->pr_ticks = 1;
2159 p->p_flag |= P_OWEUPC;
2160 }
2161 *astmap |= (1 << id);
2162 }
2163}
2164
2165static void
2166forwarded_statclock(int id, int pscnt, int *astmap)
2167{
2168 struct pstats *pstats;
2169 long rss;
2170 struct rusage *ru;
2171 struct vmspace *vm;
2172 int cpustate;
2173 struct proc *p;
2174#ifdef GPROF
2175 register struct gmonparam *g;
2176 int i;
2177#endif
2178
2179 p = checkstate_curproc[id];
2180 cpustate = checkstate_cpustate[id];
2181
2182 switch (cpustate) {
2183 case CHECKSTATE_USER:
2184 if (p->p_flag & P_PROFIL)
2185 addupc_intr_forwarded(p, id, astmap);
2186 if (pscnt > 1)
2187 return;
2188 p->p_uticks++;
2189 if (p->p_nice > NZERO)
2190 cp_time[CP_NICE]++;
2191 else
2192 cp_time[CP_USER]++;
2193 break;
2194 case CHECKSTATE_SYS:
2195#ifdef GPROF
2196 /*
2197 * Kernel statistics are just like addupc_intr, only easier.
2198 */
2199 g = &_gmonparam;
2200 if (g->state == GMON_PROF_ON) {
2201 i = checkstate_pc[id] - g->lowpc;
2202 if (i < g->textsize) {
2203 i /= HISTFRACTION * sizeof(*g->kcount);
2204 g->kcount[i]++;
2205 }
2206 }
2207#endif
2208 if (pscnt > 1)
2209 return;
2210
2211 if (!p)
2212 cp_time[CP_IDLE]++;
2213 else {
2214 p->p_sticks++;
2215 cp_time[CP_SYS]++;
2216 }
2217 break;
2218 case CHECKSTATE_INTR:
2219 default:
2220#ifdef GPROF
2221 /*
2222 * Kernel statistics are just like addupc_intr, only easier.
2223 */
2224 g = &_gmonparam;
2225 if (g->state == GMON_PROF_ON) {
2226 i = checkstate_pc[id] - g->lowpc;
2227 if (i < g->textsize) {
2228 i /= HISTFRACTION * sizeof(*g->kcount);
2229 g->kcount[i]++;
2230 }
2231 }
2232#endif
2233 if (pscnt > 1)
2234 return;
2235 if (p)
2236 p->p_iticks++;
2237 cp_time[CP_INTR]++;
2238 }
2239 if (p != NULL) {
2240 p->p_cpticks++;
2241 if (++p->p_estcpu == 0)
2242 p->p_estcpu--;
2243 if ((p->p_estcpu & 3) == 0) {
2244 resetpriority(p);
2245 if (p->p_priority >= PUSER)
2246 p->p_priority = p->p_usrpri;
2247 }
2248
2249 /* Update resource usage integrals and maximums. */
2250 if ((pstats = p->p_stats) != NULL &&
2251 (ru = &pstats->p_ru) != NULL &&
2252 (vm = p->p_vmspace) != NULL) {
2253 ru->ru_ixrss += vm->vm_tsize * PAGE_SIZE / 1024;
2254 ru->ru_idrss += vm->vm_dsize * PAGE_SIZE / 1024;
2255 ru->ru_isrss += vm->vm_ssize * PAGE_SIZE / 1024;
2256 rss = vm->vm_pmap.pm_stats.resident_count *
2257 PAGE_SIZE / 1024;
2258 if (ru->ru_maxrss < rss)
2259 ru->ru_maxrss = rss;
2260 }
2261 }
2262}
2263
2264void
2265forward_statclock(int pscnt)
2266{
2267 int map;
2268 int id;
2269 int i;
2270
2271 /* Kludge. We don't yet have separate locks for the interrupts
2272 * and the kernel. This means that we cannot let the other processors
2273 * handle complex interrupts while inhibiting them from entering
2274 * the kernel in a non-interrupt context.
2275 *
2276 * What we can do, without changing the locking mechanisms yet,
2277 * is letting the other processors handle a very simple interrupt
2278 * (wich determines the processor states), and do the main
2279 * work ourself.
2280 */
2281
2282 if (!smp_started || !invltlb_ok || cold || panicstr)
2283 return;
2284
2285 /* Step 1: Probe state (user, cpu, interrupt, spinlock, idle ) */
2286
2287 map = other_cpus & ~stopped_cpus ;
2288 checkstate_probed_cpus = 0;
2289 if (map != 0)
2290 selected_apic_ipi(map,
2291 XCPUCHECKSTATE_OFFSET, APIC_DELMODE_FIXED);
2292
2293 i = 0;
2294 while (checkstate_probed_cpus != map) {
2295 /* spin */
2296 i++;
2297 if (i == 1000000) {
2298 printf("forward_statclock: checkstate %x\n",
2299 checkstate_probed_cpus);
2300 break;
2301 }
2302 }
2303
2304 /*
2305 * Step 2: walk through other processors processes, update ticks and
2306 * profiling info.
2307 */
2308
2309 map = 0;
2310 for (id = 0; id < mp_ncpus; id++) {
2311 if (id == cpuid)
2312 continue;
2313 if (((1 << id) & checkstate_probed_cpus) == 0)
2314 continue;
2315 forwarded_statclock(id, pscnt, &map);
2316 }
2317 if (map != 0) {
2318 checkstate_need_ast |= map;
2319 selected_apic_ipi(map, XCPUAST_OFFSET, APIC_DELMODE_FIXED);
2320 i = 0;
2321 while ((checkstate_need_ast & map) != 0) {
2322 /* spin */
2323 i++;
2324 if (i > 100000) {
2325#ifdef BETTER_CLOCK_DIAGNOSTIC
2326 printf("forward_statclock: dropped ast 0x%x\n",
2327 checkstate_need_ast & map);
2328#endif
2329 break;
2330 }
2331 }
2332 }
2333}
2334
2335void
2336forward_hardclock(int pscnt)
2337{
2338 int map;
2339 int id;
2340 struct proc *p;
2341 struct pstats *pstats;
2342 int i;
2343
2344 /* Kludge. We don't yet have separate locks for the interrupts
2345 * and the kernel. This means that we cannot let the other processors
2346 * handle complex interrupts while inhibiting them from entering
2347 * the kernel in a non-interrupt context.
2348 *
2349 * What we can do, without changing the locking mechanisms yet,
2350 * is letting the other processors handle a very simple interrupt
2351 * (wich determines the processor states), and do the main
2352 * work ourself.
2353 */
2354
2355 if (!smp_started || !invltlb_ok || cold || panicstr)
2356 return;
2357
2358 /* Step 1: Probe state (user, cpu, interrupt, spinlock, idle) */
2359
2360 map = other_cpus & ~stopped_cpus ;
2361 checkstate_probed_cpus = 0;
2362 if (map != 0)
2363 selected_apic_ipi(map,
2364 XCPUCHECKSTATE_OFFSET, APIC_DELMODE_FIXED);
2365
2366 i = 0;
2367 while (checkstate_probed_cpus != map) {
2368 /* spin */
2369 i++;
2370 if (i == 1000000) {
2371 printf("forward_hardclock: checkstate %x\n",
2372 checkstate_probed_cpus);
2373 break;
2374 }
2375 }
2376
2377 /*
2378 * Step 2: walk through other processors processes, update virtual
2379 * timer and profiling timer. If stathz == 0, also update ticks and
2380 * profiling info.
2381 */
2382
2383 map = 0;
2384 for (id = 0; id < mp_ncpus; id++) {
2385 if (id == cpuid)
2386 continue;
2387 if (((1 << id) & checkstate_probed_cpus) == 0)
2388 continue;
2389 p = checkstate_curproc[id];
2390 if (p) {
2391 pstats = p->p_stats;
2392 if (checkstate_cpustate[id] == CHECKSTATE_USER &&
2393 timevalisset(&pstats->p_timer[ITIMER_VIRTUAL].it_value) &&
2394 itimerdecr(&pstats->p_timer[ITIMER_VIRTUAL], tick) == 0) {
2395 psignal(p, SIGVTALRM);
2396 map |= (1 << id);
2397 }
2398 if (timevalisset(&pstats->p_timer[ITIMER_PROF].it_value) &&
2399 itimerdecr(&pstats->p_timer[ITIMER_PROF], tick) == 0) {
2400 psignal(p, SIGPROF);
2401 map |= (1 << id);
2402 }
2403 }
2404 if (stathz == 0) {
2405 forwarded_statclock( id, pscnt, &map);
2406 }
2407 }
2408 if (map != 0) {
2409 checkstate_need_ast |= map;
2410 selected_apic_ipi(map, XCPUAST_OFFSET, APIC_DELMODE_FIXED);
2411 i = 0;
2412 while ((checkstate_need_ast & map) != 0) {
2413 /* spin */
2414 i++;
2415 if (i > 100000) {
2416#ifdef BETTER_CLOCK_DIAGNOSTIC
2417 printf("forward_hardclock: dropped ast 0x%x\n",
2418 checkstate_need_ast & map);
2419#endif
2420 break;
2421 }
2422 }
2423 }
2424}
2425
2426#endif /* BETTER_CLOCK */
2427
2428void
2429forward_signal(struct proc *p)
2430{
2431 int map;
2432 int id;
2433 int i;
2434
2435 /* Kludge. We don't yet have separate locks for the interrupts
2436 * and the kernel. This means that we cannot let the other processors
2437 * handle complex interrupts while inhibiting them from entering
2438 * the kernel in a non-interrupt context.
2439 *
2440 * What we can do, without changing the locking mechanisms yet,
2441 * is letting the other processors handle a very simple interrupt
2442 * (wich determines the processor states), and do the main
2443 * work ourself.
2444 */
2445
2446 if (!smp_started || !invltlb_ok || cold || panicstr)
2447 return;
2448 if (!forward_signal_enabled)
2449 return;
2450 while (1) {
2451 if (p->p_stat != SRUN)
2452 return;
2453 id = (u_char) p->p_oncpu;
2454 if (id == 0xff)
2455 return;
2456 map = (1<<id);
2457 checkstate_need_ast |= map;
2458 selected_apic_ipi(map, XCPUAST_OFFSET, APIC_DELMODE_FIXED);
2459 i = 0;
2460 while ((checkstate_need_ast & map) != 0) {
2461 /* spin */
2462 i++;
2463 if (i > 100000) {
2464#if 0
2465 printf("forward_signal: dropped ast 0x%x\n",
2466 checkstate_need_ast & map);
2467#endif
2468 break;
2469 }
2470 }
2471 if (id == (u_char) p->p_oncpu)
2472 return;
2473 }
2474}
2475
2476
2477#ifdef APIC_INTR_REORDER
2478/*
2479 * Maintain mapping from softintr vector to isr bit in local apic.
2480 */
2481void
2482set_lapic_isrloc(int intr, int vector)
2483{
2484 if (intr < 0 || intr > 32)
2485 panic("set_apic_isrloc: bad intr argument: %d",intr);
2486 if (vector < ICU_OFFSET || vector > 255)
2487 panic("set_apic_isrloc: bad vector argument: %d",vector);
2488 apic_isrbit_location[intr].location = &lapic.isr0 + ((vector>>5)<<2);
2489 apic_isrbit_location[intr].bit = (1<<(vector & 31));
2490}
2491#endif
| 1730
1731 /* setup a vector to our boot code */
1732 *((volatile u_short *) WARMBOOT_OFF) = WARMBOOT_TARGET;
1733 *((volatile u_short *) WARMBOOT_SEG) = (boot_addr >> 4);
1734 outb(CMOS_REG, BIOS_RESET);
1735 outb(CMOS_DATA, BIOS_WARM); /* 'warm-start' */
1736
1737 bootPTD = myPTD;
1738 /* attempt to start the Application Processor */
1739 CHECK_INIT(99); /* setup checkpoints */
1740 if (!start_ap(x, boot_addr)) {
1741 printf("AP #%d (PHY# %d) failed!\n", x, CPU_TO_ID(x));
1742 CHECK_PRINT("trace"); /* show checkpoints */
1743 /* better panic as the AP may be running loose */
1744 printf("panic y/n? [y] ");
1745 if (cngetc() != 'n')
1746 panic("bye-bye");
1747 }
1748 CHECK_PRINT("trace"); /* show checkpoints */
1749
1750 /* record its version info */
1751 cpu_apic_versions[x] = cpu_apic_versions[0];
1752
1753 all_cpus |= (1 << x); /* record AP in CPU map */
1754 }
1755
1756 /* build our map of 'other' CPUs */
1757 other_cpus = all_cpus & ~(1 << cpuid);
1758
1759 /* fill in our (BSP) APIC version */
1760 cpu_apic_versions[0] = lapic.version;
1761
1762 /* restore the warmstart vector */
1763 *(u_long *) WARMBOOT_OFF = mpbioswarmvec;
1764 outb(CMOS_REG, BIOS_RESET);
1765 outb(CMOS_DATA, mpbiosreason);
1766
1767 /*
1768 * Set up the idle context for the BSP. Similar to above except
1769 * that some was done by locore, some by pmap.c and some is implicit
1770 * because the BSP is cpu#0 and the page is initially zero, and also
1771 * because we can refer to variables by name on the BSP..
1772 */
1773 newptd = (pd_entry_t *)(kmem_alloc(kernel_map, PAGE_SIZE));
1774
1775 bcopy(PTD, newptd, PAGE_SIZE); /* inc prv page pde */
1776 IdlePTDS[0] = newptd;
1777
1778 /* Point PTD[] to this page instead of IdlePTD's physical page */
1779 newptd[PTDPTDI] = (pd_entry_t)(PG_V | PG_RW | vtophys(newptd));
1780
1781 my_idlePTD = (pd_entry_t *)vtophys(newptd);
1782
1783 /* Allocate and setup BSP idle stack */
1784 stack = (char *)kmem_alloc(kernel_map, UPAGES * PAGE_SIZE);
1785 for (i = 0; i < UPAGES; i++)
1786 SMP_prvpt[i + 3] = (pt_entry_t)(PG_V | PG_RW | vtophys(PAGE_SIZE * i + stack));
1787
1788 pmap_set_opt_bsp();
1789
1790 for (i = 0; i < mp_ncpus; i++) {
1791 bcopy( (int *) PTD + KPTDI, (int *) IdlePTDS[i] + KPTDI, NKPDE * sizeof (int));
1792 }
1793
1794 /* number of APs actually started */
1795 return mp_ncpus - 1;
1796}
1797
1798
1799/*
1800 * load the 1st level AP boot code into base memory.
1801 */
1802
1803/* targets for relocation */
1804extern void bigJump(void);
1805extern void bootCodeSeg(void);
1806extern void bootDataSeg(void);
1807extern void MPentry(void);
1808extern u_int MP_GDT;
1809extern u_int mp_gdtbase;
1810
1811static void
1812install_ap_tramp(u_int boot_addr)
1813{
1814 int x;
1815 int size = *(int *) ((u_long) & bootMP_size);
1816 u_char *src = (u_char *) ((u_long) bootMP);
1817 u_char *dst = (u_char *) boot_addr + KERNBASE;
1818 u_int boot_base = (u_int) bootMP;
1819 u_int8_t *dst8;
1820 u_int16_t *dst16;
1821 u_int32_t *dst32;
1822
1823 POSTCODE(INSTALL_AP_TRAMP_POST);
1824
1825 for (x = 0; x < size; ++x)
1826 *dst++ = *src++;
1827
1828 /*
1829 * modify addresses in code we just moved to basemem. unfortunately we
1830 * need fairly detailed info about mpboot.s for this to work. changes
1831 * to mpboot.s might require changes here.
1832 */
1833
1834 /* boot code is located in KERNEL space */
1835 dst = (u_char *) boot_addr + KERNBASE;
1836
1837 /* modify the lgdt arg */
1838 dst32 = (u_int32_t *) (dst + ((u_int) & mp_gdtbase - boot_base));
1839 *dst32 = boot_addr + ((u_int) & MP_GDT - boot_base);
1840
1841 /* modify the ljmp target for MPentry() */
1842 dst32 = (u_int32_t *) (dst + ((u_int) bigJump - boot_base) + 1);
1843 *dst32 = ((u_int) MPentry - KERNBASE);
1844
1845 /* modify the target for boot code segment */
1846 dst16 = (u_int16_t *) (dst + ((u_int) bootCodeSeg - boot_base));
1847 dst8 = (u_int8_t *) (dst16 + 1);
1848 *dst16 = (u_int) boot_addr & 0xffff;
1849 *dst8 = ((u_int) boot_addr >> 16) & 0xff;
1850
1851 /* modify the target for boot data segment */
1852 dst16 = (u_int16_t *) (dst + ((u_int) bootDataSeg - boot_base));
1853 dst8 = (u_int8_t *) (dst16 + 1);
1854 *dst16 = (u_int) boot_addr & 0xffff;
1855 *dst8 = ((u_int) boot_addr >> 16) & 0xff;
1856}
1857
1858
1859/*
1860 * this function starts the AP (application processor) identified
1861 * by the APIC ID 'physicalCpu'. It does quite a "song and dance"
1862 * to accomplish this. This is necessary because of the nuances
1863 * of the different hardware we might encounter. It ain't pretty,
1864 * but it seems to work.
1865 */
1866static int
1867start_ap(int logical_cpu, u_int boot_addr)
1868{
1869 int physical_cpu;
1870 int vector;
1871 int cpus;
1872 u_long icr_lo, icr_hi;
1873
1874 POSTCODE(START_AP_POST);
1875
1876 /* get the PHYSICAL APIC ID# */
1877 physical_cpu = CPU_TO_ID(logical_cpu);
1878
1879 /* calculate the vector */
1880 vector = (boot_addr >> 12) & 0xff;
1881
1882 /* used as a watchpoint to signal AP startup */
1883 cpus = mp_ncpus;
1884
1885 /*
1886 * first we do an INIT/RESET IPI this INIT IPI might be run, reseting
1887 * and running the target CPU. OR this INIT IPI might be latched (P5
1888 * bug), CPU waiting for STARTUP IPI. OR this INIT IPI might be
1889 * ignored.
1890 */
1891
1892 /* setup the address for the target AP */
1893 icr_hi = lapic.icr_hi & ~APIC_ID_MASK;
1894 icr_hi |= (physical_cpu << 24);
1895 lapic.icr_hi = icr_hi;
1896
1897 /* do an INIT IPI: assert RESET */
1898 icr_lo = lapic.icr_lo & 0xfff00000;
1899 lapic.icr_lo = icr_lo | 0x0000c500;
1900
1901 /* wait for pending status end */
1902 while (lapic.icr_lo & APIC_DELSTAT_MASK)
1903 /* spin */ ;
1904
1905 /* do an INIT IPI: deassert RESET */
1906 lapic.icr_lo = icr_lo | 0x00008500;
1907
1908 /* wait for pending status end */
1909 u_sleep(10000); /* wait ~10mS */
1910 while (lapic.icr_lo & APIC_DELSTAT_MASK)
1911 /* spin */ ;
1912
1913 /*
1914 * next we do a STARTUP IPI: the previous INIT IPI might still be
1915 * latched, (P5 bug) this 1st STARTUP would then terminate
1916 * immediately, and the previously started INIT IPI would continue. OR
1917 * the previous INIT IPI has already run. and this STARTUP IPI will
1918 * run. OR the previous INIT IPI was ignored. and this STARTUP IPI
1919 * will run.
1920 */
1921
1922 /* do a STARTUP IPI */
1923 lapic.icr_lo = icr_lo | 0x00000600 | vector;
1924 while (lapic.icr_lo & APIC_DELSTAT_MASK)
1925 /* spin */ ;
1926 u_sleep(200); /* wait ~200uS */
1927
1928 /*
1929 * finally we do a 2nd STARTUP IPI: this 2nd STARTUP IPI should run IF
1930 * the previous STARTUP IPI was cancelled by a latched INIT IPI. OR
1931 * this STARTUP IPI will be ignored, as only ONE STARTUP IPI is
1932 * recognized after hardware RESET or INIT IPI.
1933 */
1934
1935 lapic.icr_lo = icr_lo | 0x00000600 | vector;
1936 while (lapic.icr_lo & APIC_DELSTAT_MASK)
1937 /* spin */ ;
1938 u_sleep(200); /* wait ~200uS */
1939
1940 /* wait for it to start */
1941 set_apic_timer(5000000);/* == 5 seconds */
1942 while (read_apic_timer())
1943 if (mp_ncpus > cpus)
1944 return 1; /* return SUCCESS */
1945
1946 return 0; /* return FAILURE */
1947}
1948
1949
1950/*
1951 * Flush the TLB on all other CPU's
1952 *
1953 * XXX: Needs to handshake and wait for completion before proceding.
1954 */
1955void
1956smp_invltlb(void)
1957{
1958#if defined(APIC_IO)
1959 if (smp_started && invltlb_ok)
1960 all_but_self_ipi(XINVLTLB_OFFSET);
1961#endif /* APIC_IO */
1962}
1963
1964void
1965invlpg(u_int addr)
1966{
1967 __asm __volatile("invlpg (%0)"::"r"(addr):"memory");
1968
1969 /* send a message to the other CPUs */
1970 smp_invltlb();
1971}
1972
1973void
1974invltlb(void)
1975{
1976 u_long temp;
1977
1978 /*
1979 * This should be implemented as load_cr3(rcr3()) when load_cr3() is
1980 * inlined.
1981 */
1982 __asm __volatile("movl %%cr3, %0; movl %0, %%cr3":"=r"(temp) :: "memory");
1983
1984 /* send a message to the other CPUs */
1985 smp_invltlb();
1986}
1987
1988
1989/*
1990 * When called the executing CPU will send an IPI to all other CPUs
1991 * requesting that they halt execution.
1992 *
1993 * Usually (but not necessarily) called with 'other_cpus' as its arg.
1994 *
1995 * - Signals all CPUs in map to stop.
1996 * - Waits for each to stop.
1997 *
1998 * Returns:
1999 * -1: error
2000 * 0: NA
2001 * 1: ok
2002 *
2003 * XXX FIXME: this is not MP-safe, needs a lock to prevent multiple CPUs
2004 * from executing at same time.
2005 */
2006int
2007stop_cpus(u_int map)
2008{
2009 if (!smp_started)
2010 return 0;
2011
2012 /* send IPI to all CPUs in map */
2013 stopped_cpus = 0;
2014
2015 /* send the Xcpustop IPI to all CPUs in map */
2016 selected_apic_ipi(map, XCPUSTOP_OFFSET, APIC_DELMODE_FIXED);
2017
2018 while (stopped_cpus != map)
2019 /* spin */ ;
2020
2021 return 1;
2022}
2023
2024
2025/*
2026 * Called by a CPU to restart stopped CPUs.
2027 *
2028 * Usually (but not necessarily) called with 'stopped_cpus' as its arg.
2029 *
2030 * - Signals all CPUs in map to restart.
2031 * - Waits for each to restart.
2032 *
2033 * Returns:
2034 * -1: error
2035 * 0: NA
2036 * 1: ok
2037 */
2038int
2039restart_cpus(u_int map)
2040{
2041 if (!smp_started)
2042 return 0;
2043
2044 started_cpus = map; /* signal other cpus to restart */
2045
2046 while (started_cpus) /* wait for each to clear its bit */
2047 /* spin */ ;
2048 stopped_cpus = 0;
2049
2050 return 1;
2051}
2052
2053int smp_active = 0; /* are the APs allowed to run? */
2054SYSCTL_INT(_machdep, OID_AUTO, smp_active, CTLFLAG_RW, &smp_active, 0, "");
2055
2056/* XXX maybe should be hw.ncpu */
2057static int smp_cpus = 1; /* how many cpu's running */
2058SYSCTL_INT(_machdep, OID_AUTO, smp_cpus, CTLFLAG_RD, &smp_cpus, 0, "");
2059
2060int invltlb_ok = 0; /* throttle smp_invltlb() till safe */
2061SYSCTL_INT(_machdep, OID_AUTO, invltlb_ok, CTLFLAG_RW, &invltlb_ok, 0, "");
2062
2063/* Warning: Do not staticize. Used from swtch.s */
2064int do_page_zero_idle = 1; /* bzero pages for fun and profit in idleloop */
2065SYSCTL_INT(_machdep, OID_AUTO, do_page_zero_idle, CTLFLAG_RW,
2066 &do_page_zero_idle, 0, "");
2067
2068/* Is forwarding of a interrupt to the CPU holding the ISR lock enabled ? */
2069int forward_irq_enabled = 1;
2070SYSCTL_INT(_machdep, OID_AUTO, forward_irq_enabled, CTLFLAG_RW,
2071 &forward_irq_enabled, 0, "");
2072
2073/* Enable forwarding of a signal to a process running on a different CPU */
2074int forward_signal_enabled = 1;
2075SYSCTL_INT(_machdep, OID_AUTO, forward_signal_enabled, CTLFLAG_RW,
2076 &forward_signal_enabled, 0, "");
2077
2078/*
2079 * This is called once the rest of the system is up and running and we're
2080 * ready to let the AP's out of the pen.
2081 */
2082void ap_init(void);
2083
2084void
2085ap_init()
2086{
2087 u_int temp;
2088 u_int apic_id;
2089
2090 smp_cpus++;
2091
2092#if defined(I586_CPU) && !defined(NO_F00F_HACK)
2093 lidt(&r_idt);
2094#endif
2095
2096 /* Build our map of 'other' CPUs. */
2097 other_cpus = all_cpus & ~(1 << cpuid);
2098
2099 printf("SMP: AP CPU #%d Launched!\n", cpuid);
2100
2101 /* XXX FIXME: i386 specific, and redundant: Setup the FPU. */
2102 load_cr0((rcr0() & ~CR0_EM) | CR0_MP | CR0_NE | CR0_TS);
2103
2104 /* A quick check from sanity claus */
2105 apic_id = (apic_id_to_logical[(lapic.id & 0x0f000000) >> 24]);
2106 if (cpuid != apic_id) {
2107 printf("SMP: cpuid = %d\n", cpuid);
2108 printf("SMP: apic_id = %d\n", apic_id);
2109 printf("PTD[MPPTDI] = %08x\n", PTD[MPPTDI]);
2110 panic("cpuid mismatch! boom!!");
2111 }
2112
2113 getmtrr();
2114
2115 /* Init local apic for irq's */
2116 apic_initialize();
2117
2118 /*
2119 * Activate smp_invltlb, although strictly speaking, this isn't
2120 * quite correct yet. We should have a bitfield for cpus willing
2121 * to accept TLB flush IPI's or something and sync them.
2122 */
2123 invltlb_ok = 1;
2124 smp_started = 1; /* enable IPI's, tlb shootdown, freezes etc */
2125 smp_active = 1; /* historic */
2126
2127 curproc = NULL; /* make sure */
2128}
2129
2130#ifdef BETTER_CLOCK
2131
2132#define CHECKSTATE_USER 0
2133#define CHECKSTATE_SYS 1
2134#define CHECKSTATE_INTR 2
2135
2136/* Do not staticize. Used from apic_vector.s */
2137struct proc* checkstate_curproc[NCPU];
2138int checkstate_cpustate[NCPU];
2139u_long checkstate_pc[NCPU];
2140
2141extern long cp_time[CPUSTATES];
2142
2143#define PC_TO_INDEX(pc, prof) \
2144 ((int)(((u_quad_t)((pc) - (prof)->pr_off) * \
2145 (u_quad_t)((prof)->pr_scale)) >> 16) & ~1)
2146
2147static void
2148addupc_intr_forwarded(struct proc *p, int id, int *astmap)
2149{
2150 int i;
2151 struct uprof *prof;
2152 u_long pc;
2153
2154 pc = checkstate_pc[id];
2155 prof = &p->p_stats->p_prof;
2156 if (pc >= prof->pr_off &&
2157 (i = PC_TO_INDEX(pc, prof)) < prof->pr_size) {
2158 if ((p->p_flag & P_OWEUPC) == 0) {
2159 prof->pr_addr = pc;
2160 prof->pr_ticks = 1;
2161 p->p_flag |= P_OWEUPC;
2162 }
2163 *astmap |= (1 << id);
2164 }
2165}
2166
2167static void
2168forwarded_statclock(int id, int pscnt, int *astmap)
2169{
2170 struct pstats *pstats;
2171 long rss;
2172 struct rusage *ru;
2173 struct vmspace *vm;
2174 int cpustate;
2175 struct proc *p;
2176#ifdef GPROF
2177 register struct gmonparam *g;
2178 int i;
2179#endif
2180
2181 p = checkstate_curproc[id];
2182 cpustate = checkstate_cpustate[id];
2183
2184 switch (cpustate) {
2185 case CHECKSTATE_USER:
2186 if (p->p_flag & P_PROFIL)
2187 addupc_intr_forwarded(p, id, astmap);
2188 if (pscnt > 1)
2189 return;
2190 p->p_uticks++;
2191 if (p->p_nice > NZERO)
2192 cp_time[CP_NICE]++;
2193 else
2194 cp_time[CP_USER]++;
2195 break;
2196 case CHECKSTATE_SYS:
2197#ifdef GPROF
2198 /*
2199 * Kernel statistics are just like addupc_intr, only easier.
2200 */
2201 g = &_gmonparam;
2202 if (g->state == GMON_PROF_ON) {
2203 i = checkstate_pc[id] - g->lowpc;
2204 if (i < g->textsize) {
2205 i /= HISTFRACTION * sizeof(*g->kcount);
2206 g->kcount[i]++;
2207 }
2208 }
2209#endif
2210 if (pscnt > 1)
2211 return;
2212
2213 if (!p)
2214 cp_time[CP_IDLE]++;
2215 else {
2216 p->p_sticks++;
2217 cp_time[CP_SYS]++;
2218 }
2219 break;
2220 case CHECKSTATE_INTR:
2221 default:
2222#ifdef GPROF
2223 /*
2224 * Kernel statistics are just like addupc_intr, only easier.
2225 */
2226 g = &_gmonparam;
2227 if (g->state == GMON_PROF_ON) {
2228 i = checkstate_pc[id] - g->lowpc;
2229 if (i < g->textsize) {
2230 i /= HISTFRACTION * sizeof(*g->kcount);
2231 g->kcount[i]++;
2232 }
2233 }
2234#endif
2235 if (pscnt > 1)
2236 return;
2237 if (p)
2238 p->p_iticks++;
2239 cp_time[CP_INTR]++;
2240 }
2241 if (p != NULL) {
2242 p->p_cpticks++;
2243 if (++p->p_estcpu == 0)
2244 p->p_estcpu--;
2245 if ((p->p_estcpu & 3) == 0) {
2246 resetpriority(p);
2247 if (p->p_priority >= PUSER)
2248 p->p_priority = p->p_usrpri;
2249 }
2250
2251 /* Update resource usage integrals and maximums. */
2252 if ((pstats = p->p_stats) != NULL &&
2253 (ru = &pstats->p_ru) != NULL &&
2254 (vm = p->p_vmspace) != NULL) {
2255 ru->ru_ixrss += vm->vm_tsize * PAGE_SIZE / 1024;
2256 ru->ru_idrss += vm->vm_dsize * PAGE_SIZE / 1024;
2257 ru->ru_isrss += vm->vm_ssize * PAGE_SIZE / 1024;
2258 rss = vm->vm_pmap.pm_stats.resident_count *
2259 PAGE_SIZE / 1024;
2260 if (ru->ru_maxrss < rss)
2261 ru->ru_maxrss = rss;
2262 }
2263 }
2264}
2265
2266void
2267forward_statclock(int pscnt)
2268{
2269 int map;
2270 int id;
2271 int i;
2272
2273 /* Kludge. We don't yet have separate locks for the interrupts
2274 * and the kernel. This means that we cannot let the other processors
2275 * handle complex interrupts while inhibiting them from entering
2276 * the kernel in a non-interrupt context.
2277 *
2278 * What we can do, without changing the locking mechanisms yet,
2279 * is letting the other processors handle a very simple interrupt
2280 * (wich determines the processor states), and do the main
2281 * work ourself.
2282 */
2283
2284 if (!smp_started || !invltlb_ok || cold || panicstr)
2285 return;
2286
2287 /* Step 1: Probe state (user, cpu, interrupt, spinlock, idle ) */
2288
2289 map = other_cpus & ~stopped_cpus ;
2290 checkstate_probed_cpus = 0;
2291 if (map != 0)
2292 selected_apic_ipi(map,
2293 XCPUCHECKSTATE_OFFSET, APIC_DELMODE_FIXED);
2294
2295 i = 0;
2296 while (checkstate_probed_cpus != map) {
2297 /* spin */
2298 i++;
2299 if (i == 1000000) {
2300 printf("forward_statclock: checkstate %x\n",
2301 checkstate_probed_cpus);
2302 break;
2303 }
2304 }
2305
2306 /*
2307 * Step 2: walk through other processors processes, update ticks and
2308 * profiling info.
2309 */
2310
2311 map = 0;
2312 for (id = 0; id < mp_ncpus; id++) {
2313 if (id == cpuid)
2314 continue;
2315 if (((1 << id) & checkstate_probed_cpus) == 0)
2316 continue;
2317 forwarded_statclock(id, pscnt, &map);
2318 }
2319 if (map != 0) {
2320 checkstate_need_ast |= map;
2321 selected_apic_ipi(map, XCPUAST_OFFSET, APIC_DELMODE_FIXED);
2322 i = 0;
2323 while ((checkstate_need_ast & map) != 0) {
2324 /* spin */
2325 i++;
2326 if (i > 100000) {
2327#ifdef BETTER_CLOCK_DIAGNOSTIC
2328 printf("forward_statclock: dropped ast 0x%x\n",
2329 checkstate_need_ast & map);
2330#endif
2331 break;
2332 }
2333 }
2334 }
2335}
2336
2337void
2338forward_hardclock(int pscnt)
2339{
2340 int map;
2341 int id;
2342 struct proc *p;
2343 struct pstats *pstats;
2344 int i;
2345
2346 /* Kludge. We don't yet have separate locks for the interrupts
2347 * and the kernel. This means that we cannot let the other processors
2348 * handle complex interrupts while inhibiting them from entering
2349 * the kernel in a non-interrupt context.
2350 *
2351 * What we can do, without changing the locking mechanisms yet,
2352 * is letting the other processors handle a very simple interrupt
2353 * (wich determines the processor states), and do the main
2354 * work ourself.
2355 */
2356
2357 if (!smp_started || !invltlb_ok || cold || panicstr)
2358 return;
2359
2360 /* Step 1: Probe state (user, cpu, interrupt, spinlock, idle) */
2361
2362 map = other_cpus & ~stopped_cpus ;
2363 checkstate_probed_cpus = 0;
2364 if (map != 0)
2365 selected_apic_ipi(map,
2366 XCPUCHECKSTATE_OFFSET, APIC_DELMODE_FIXED);
2367
2368 i = 0;
2369 while (checkstate_probed_cpus != map) {
2370 /* spin */
2371 i++;
2372 if (i == 1000000) {
2373 printf("forward_hardclock: checkstate %x\n",
2374 checkstate_probed_cpus);
2375 break;
2376 }
2377 }
2378
2379 /*
2380 * Step 2: walk through other processors processes, update virtual
2381 * timer and profiling timer. If stathz == 0, also update ticks and
2382 * profiling info.
2383 */
2384
2385 map = 0;
2386 for (id = 0; id < mp_ncpus; id++) {
2387 if (id == cpuid)
2388 continue;
2389 if (((1 << id) & checkstate_probed_cpus) == 0)
2390 continue;
2391 p = checkstate_curproc[id];
2392 if (p) {
2393 pstats = p->p_stats;
2394 if (checkstate_cpustate[id] == CHECKSTATE_USER &&
2395 timevalisset(&pstats->p_timer[ITIMER_VIRTUAL].it_value) &&
2396 itimerdecr(&pstats->p_timer[ITIMER_VIRTUAL], tick) == 0) {
2397 psignal(p, SIGVTALRM);
2398 map |= (1 << id);
2399 }
2400 if (timevalisset(&pstats->p_timer[ITIMER_PROF].it_value) &&
2401 itimerdecr(&pstats->p_timer[ITIMER_PROF], tick) == 0) {
2402 psignal(p, SIGPROF);
2403 map |= (1 << id);
2404 }
2405 }
2406 if (stathz == 0) {
2407 forwarded_statclock( id, pscnt, &map);
2408 }
2409 }
2410 if (map != 0) {
2411 checkstate_need_ast |= map;
2412 selected_apic_ipi(map, XCPUAST_OFFSET, APIC_DELMODE_FIXED);
2413 i = 0;
2414 while ((checkstate_need_ast & map) != 0) {
2415 /* spin */
2416 i++;
2417 if (i > 100000) {
2418#ifdef BETTER_CLOCK_DIAGNOSTIC
2419 printf("forward_hardclock: dropped ast 0x%x\n",
2420 checkstate_need_ast & map);
2421#endif
2422 break;
2423 }
2424 }
2425 }
2426}
2427
2428#endif /* BETTER_CLOCK */
2429
2430void
2431forward_signal(struct proc *p)
2432{
2433 int map;
2434 int id;
2435 int i;
2436
2437 /* Kludge. We don't yet have separate locks for the interrupts
2438 * and the kernel. This means that we cannot let the other processors
2439 * handle complex interrupts while inhibiting them from entering
2440 * the kernel in a non-interrupt context.
2441 *
2442 * What we can do, without changing the locking mechanisms yet,
2443 * is letting the other processors handle a very simple interrupt
2444 * (wich determines the processor states), and do the main
2445 * work ourself.
2446 */
2447
2448 if (!smp_started || !invltlb_ok || cold || panicstr)
2449 return;
2450 if (!forward_signal_enabled)
2451 return;
2452 while (1) {
2453 if (p->p_stat != SRUN)
2454 return;
2455 id = (u_char) p->p_oncpu;
2456 if (id == 0xff)
2457 return;
2458 map = (1<<id);
2459 checkstate_need_ast |= map;
2460 selected_apic_ipi(map, XCPUAST_OFFSET, APIC_DELMODE_FIXED);
2461 i = 0;
2462 while ((checkstate_need_ast & map) != 0) {
2463 /* spin */
2464 i++;
2465 if (i > 100000) {
2466#if 0
2467 printf("forward_signal: dropped ast 0x%x\n",
2468 checkstate_need_ast & map);
2469#endif
2470 break;
2471 }
2472 }
2473 if (id == (u_char) p->p_oncpu)
2474 return;
2475 }
2476}
2477
2478
2479#ifdef APIC_INTR_REORDER
2480/*
2481 * Maintain mapping from softintr vector to isr bit in local apic.
2482 */
2483void
2484set_lapic_isrloc(int intr, int vector)
2485{
2486 if (intr < 0 || intr > 32)
2487 panic("set_apic_isrloc: bad intr argument: %d",intr);
2488 if (vector < ICU_OFFSET || vector > 255)
2489 panic("set_apic_isrloc: bad vector argument: %d",vector);
2490 apic_isrbit_location[intr].location = &lapic.isr0 + ((vector>>5)<<2);
2491 apic_isrbit_location[intr].bit = (1<<(vector & 31));
2492}
2493#endif
|