Deleted Added
sdiff udiff text old ( 75724 ) new ( 76078 )
full compact
mptable.c (75724) mptable.c (76078)
1/*
2 * Copyright (c) 1996, by Steve Passe
3 * All rights reserved.
4 *
5 * Redistribution and use in source and binary forms, with or without
6 * modification, are permitted provided that the following conditions
7 * are met:
8 * 1. Redistributions of source code must retain the above copyright

--- 8 unchanged lines hidden (view full) ---

17 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
18 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
19 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
20 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
21 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
22 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
23 * SUCH DAMAGE.
24 *
1/*
2 * Copyright (c) 1996, by Steve Passe
3 * All rights reserved.
4 *
5 * Redistribution and use in source and binary forms, with or without
6 * modification, are permitted provided that the following conditions
7 * are met:
8 * 1. Redistributions of source code must retain the above copyright

--- 8 unchanged lines hidden (view full) ---

17 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
18 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
19 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
20 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
21 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
22 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
23 * SUCH DAMAGE.
24 *
25 * $FreeBSD: head/sys/i386/i386/mptable.c 75724 2001-04-20 01:09:05Z jhb $
25 * $FreeBSD: head/sys/i386/i386/mptable.c 76078 2001-04-27 19:28:25Z jhb $
26 */
27
28#include "opt_cpu.h"
29
30#ifdef SMP
31#include <machine/smptests.h>
32#else
33#error
34#endif
35
36#include <sys/param.h>
37#include <sys/bus.h>
38#include <sys/systm.h>
39#include <sys/kernel.h>
40#include <sys/proc.h>
41#include <sys/sysctl.h>
42#include <sys/malloc.h>
43#include <sys/memrange.h>
44#include <sys/mutex.h>
26 */
27
28#include "opt_cpu.h"
29
30#ifdef SMP
31#include <machine/smptests.h>
32#else
33#error
34#endif
35
36#include <sys/param.h>
37#include <sys/bus.h>
38#include <sys/systm.h>
39#include <sys/kernel.h>
40#include <sys/proc.h>
41#include <sys/sysctl.h>
42#include <sys/malloc.h>
43#include <sys/memrange.h>
44#include <sys/mutex.h>
45#include <sys/smp.h>
45#include <sys/dkstat.h>
46#include <sys/cons.h> /* cngetc() */
47
48#include <vm/vm.h>
49#include <vm/vm_param.h>
50#include <vm/pmap.h>
51#include <vm/vm_kern.h>
52#include <vm/vm_extern.h>
53#include <sys/lock.h>
54#include <vm/vm_map.h>
55#include <sys/user.h>
56#ifdef GPROF
57#include <sys/gmon.h>
58#endif
59
46#include <sys/dkstat.h>
47#include <sys/cons.h> /* cngetc() */
48
49#include <vm/vm.h>
50#include <vm/vm_param.h>
51#include <vm/pmap.h>
52#include <vm/vm_kern.h>
53#include <vm/vm_extern.h>
54#include <sys/lock.h>
55#include <vm/vm_map.h>
56#include <sys/user.h>
57#ifdef GPROF
58#include <sys/gmon.h>
59#endif
60
60#include <machine/smp.h>
61#include <machine/apic.h>
62#include <machine/atomic.h>
61#include <machine/apic.h>
62#include <machine/atomic.h>
63#include <machine/cpu.h>
63#include <machine/cpufunc.h>
64#include <machine/ipl.h>
65#include <machine/mpapic.h>
66#include <machine/psl.h>
67#include <machine/segments.h>
68#include <machine/smptests.h> /** TEST_DEFAULT_CONFIG, TEST_TEST1 */
69#include <machine/tss.h>
70#include <machine/specialreg.h>

--- 167 unchanged lines hidden (view full) ---

238
239/** XXX FIXME: where does this really belong, isa.h/isa.c perhaps? */
240int current_postcode;
241
242/** XXX FIXME: what system files declare these??? */
243extern struct region_descriptor r_gdt, r_idt;
244
245int bsp_apic_ready = 0; /* flags useability of BSP apic */
64#include <machine/cpufunc.h>
65#include <machine/ipl.h>
66#include <machine/mpapic.h>
67#include <machine/psl.h>
68#include <machine/segments.h>
69#include <machine/smptests.h> /** TEST_DEFAULT_CONFIG, TEST_TEST1 */
70#include <machine/tss.h>
71#include <machine/specialreg.h>

--- 167 unchanged lines hidden (view full) ---

239
240/** XXX FIXME: where does this really belong, isa.h/isa.c perhaps? */
241int current_postcode;
242
243/** XXX FIXME: what system files declare these??? */
244extern struct region_descriptor r_gdt, r_idt;
245
246int bsp_apic_ready = 0; /* flags useability of BSP apic */
246int mp_ncpus; /* # of CPUs, including BSP */
247int mp_naps; /* # of Applications processors */
248int mp_nbusses; /* # of busses */
249int mp_napics; /* # of IO APICs */
250int boot_cpu_id; /* designated BSP */
251vm_offset_t cpu_apic_address;
252vm_offset_t io_apic_address[NAPICID]; /* NAPICID is more than enough */
253extern int nkpt;
254

--- 13 unchanged lines hidden (view full) ---

268 * APIC ID logical/physical mapping structures.
269 * We oversize these to simplify boot-time config.
270 */
271int cpu_num_to_apic_id[NAPICID];
272int io_num_to_apic_id[NAPICID];
273int apic_id_to_logical[NAPICID];
274
275
247int mp_naps; /* # of Applications processors */
248int mp_nbusses; /* # of busses */
249int mp_napics; /* # of IO APICs */
250int boot_cpu_id; /* designated BSP */
251vm_offset_t cpu_apic_address;
252vm_offset_t io_apic_address[NAPICID]; /* NAPICID is more than enough */
253extern int nkpt;
254

--- 13 unchanged lines hidden (view full) ---

268 * APIC ID logical/physical mapping structures.
269 * We oversize these to simplify boot-time config.
270 */
271int cpu_num_to_apic_id[NAPICID];
272int io_num_to_apic_id[NAPICID];
273int apic_id_to_logical[NAPICID];
274
275
276/* Bitmap of all available CPUs */
277u_int all_cpus;
278
279/* AP uses this during bootstrap. Do not staticize. */
280char *bootSTK;
281static int bootAP;
282
283/* Hotwire a 0->4MB V==P mapping */
284extern pt_entry_t *KPTphys;
285
286/* SMP page table page */
287extern pt_entry_t *SMPpt;
288
289struct pcb stoppcbs[MAXCPU];
290
276/* AP uses this during bootstrap. Do not staticize. */
277char *bootSTK;
278static int bootAP;
279
280/* Hotwire a 0->4MB V==P mapping */
281extern pt_entry_t *KPTphys;
282
283/* SMP page table page */
284extern pt_entry_t *SMPpt;
285
286struct pcb stoppcbs[MAXCPU];
287
291int smp_started; /* has the system started? */
292int smp_active = 0; /* are the APs allowed to run? */
293SYSCTL_INT(_machdep, OID_AUTO, smp_active, CTLFLAG_RW, &smp_active, 0, "");
294
295/* XXX maybe should be hw.ncpu */
296static int smp_cpus = 1; /* how many cpu's running */
297SYSCTL_INT(_machdep, OID_AUTO, smp_cpus, CTLFLAG_RD, &smp_cpus, 0, "");
298
299int invltlb_ok = 0; /* throttle smp_invltlb() till safe */
300SYSCTL_INT(_machdep, OID_AUTO, invltlb_ok, CTLFLAG_RW, &invltlb_ok, 0, "");
301
288int invltlb_ok = 0; /* throttle smp_invltlb() till safe */
289SYSCTL_INT(_machdep, OID_AUTO, invltlb_ok, CTLFLAG_RW, &invltlb_ok, 0, "");
290
302/* Enable forwarding of a signal to a process running on a different CPU */
303static int forward_signal_enabled = 1;
304SYSCTL_INT(_machdep, OID_AUTO, forward_signal_enabled, CTLFLAG_RW,
305 &forward_signal_enabled, 0, "");
306
307/* Enable forwarding of roundrobin to all other cpus */
308static int forward_roundrobin_enabled = 1;
309SYSCTL_INT(_machdep, OID_AUTO, forward_roundrobin_enabled, CTLFLAG_RW,
310 &forward_roundrobin_enabled, 0, "");
311
312
313/*
314 * Local data and functions.
315 */
316
317/* Set to 1 once we're ready to let the APs out of the pen. */
318static volatile int aps_ready = 0;
319
320static int mp_capable;

--- 28 unchanged lines hidden (view full) ---

349/* lock region used by kernel profiling */
350struct mtx mcount_mtx;
351
352#ifdef USE_COMLOCK
353/* locks com (tty) data/hardware accesses: a FASTINTR() */
354struct mtx com_mtx;
355#endif /* USE_COMLOCK */
356
291/*
292 * Local data and functions.
293 */
294
295/* Set to 1 once we're ready to let the APs out of the pen. */
296static volatile int aps_ready = 0;
297
298static int mp_capable;

--- 28 unchanged lines hidden (view full) ---

327/* lock region used by kernel profiling */
328struct mtx mcount_mtx;
329
330#ifdef USE_COMLOCK
331/* locks com (tty) data/hardware accesses: a FASTINTR() */
332struct mtx com_mtx;
333#endif /* USE_COMLOCK */
334
357/* lock around the MP rendezvous */
358static struct mtx smp_rv_mtx;
359
360static void
361init_locks(void)
362{
363 /*
364 * XXX The mcount mutex probably needs to be statically initialized,
365 * since it will be used even in the function calls that get us to this
366 * point.
367 */
368 mtx_init(&mcount_mtx, "mcount", MTX_DEF);
369
335static void
336init_locks(void)
337{
338 /*
339 * XXX The mcount mutex probably needs to be statically initialized,
340 * since it will be used even in the function calls that get us to this
341 * point.
342 */
343 mtx_init(&mcount_mtx, "mcount", MTX_DEF);
344
370 mtx_init(&smp_rv_mtx, "smp rendezvous", MTX_SPIN);
371
372#ifdef USE_COMLOCK
373 mtx_init(&com_mtx, "com", MTX_SPIN);
374#endif /* USE_COMLOCK */
345#ifdef USE_COMLOCK
346 mtx_init(&com_mtx, "com", MTX_SPIN);
347#endif /* USE_COMLOCK */
375
376 mtx_init(&ap_boot_mtx, "ap boot", MTX_SPIN);
377}
378
379/*
380 * Calculate usable address in base memory for AP trampoline code.
381 */
382u_int
383mp_bootaddress(u_int basemem)
384{

--- 7 unchanged lines hidden (view full) ---

392
393 return boot_address;
394}
395
396
397/*
398 * Look for an Intel MP spec table (ie, SMP capable hardware).
399 */
348}
349
350/*
351 * Calculate usable address in base memory for AP trampoline code.
352 */
353u_int
354mp_bootaddress(u_int basemem)
355{

--- 7 unchanged lines hidden (view full) ---

363
364 return boot_address;
365}
366
367
368/*
369 * Look for an Intel MP spec table (ie, SMP capable hardware).
370 */
400int
401mp_probe(void)
371void
372i386_mp_probe(void)
402{
403 int x;
404 u_long segment;
405 u_int32_t target;
406
407 POSTCODE(MP_PROBE_POST);
408
409 /* see if EBDA exists */

--- 12 unchanged lines hidden (view full) ---

422 /* search the BIOS */
423 target = (u_int32_t) BIOS_BASE;
424 if ((x = search_for_sig(target, BIOS_COUNT)) >= 0)
425 goto found;
426
427 /* nothing found */
428 mpfps = (mpfps_t)0;
429 mp_capable = 0;
373{
374 int x;
375 u_long segment;
376 u_int32_t target;
377
378 POSTCODE(MP_PROBE_POST);
379
380 /* see if EBDA exists */

--- 12 unchanged lines hidden (view full) ---

393 /* search the BIOS */
394 target = (u_int32_t) BIOS_BASE;
395 if ((x = search_for_sig(target, BIOS_COUNT)) >= 0)
396 goto found;
397
398 /* nothing found */
399 mpfps = (mpfps_t)0;
400 mp_capable = 0;
430 return 0;
401 return;
431
432found:
433 /* calculate needed resources */
434 mpfps = (mpfps_t)x;
435 mptable_pass1();
436
437 /* flag fact that we are running multiple processors */
438 mp_capable = 1;
402
403found:
404 /* calculate needed resources */
405 mpfps = (mpfps_t)x;
406 mptable_pass1();
407
408 /* flag fact that we are running multiple processors */
409 mp_capable = 1;
439 return 1;
440}
441
410}
411
412int
413cpu_mp_probe(void)
414{
415 return (mp_capable);
416}
442
443/*
444 * Initialize the SMP hardware and the APIC and start up the AP's.
445 */
446void
417
418/*
419 * Initialize the SMP hardware and the APIC and start up the AP's.
420 */
421void
447mp_start(void)
422cpu_mp_start(void)
448{
449 POSTCODE(MP_START_POST);
450
451 /* look for MP capable motherboard */
452 if (mp_capable)
453 mp_enable(boot_address);
454 else
455 panic("MP hardware not found!");
423{
424 POSTCODE(MP_START_POST);
425
426 /* look for MP capable motherboard */
427 if (mp_capable)
428 mp_enable(boot_address);
429 else
430 panic("MP hardware not found!");
431
432 cpu_setregs();
456}
457
458
459/*
460 * Print various information about the SMP system hardware and setup.
461 */
462void
433}
434
435
436/*
437 * Print various information about the SMP system hardware and setup.
438 */
439void
463mp_announce(void)
440cpu_mp_announce(void)
464{
465 int x;
466
467 POSTCODE(MP_ANNOUNCE_POST);
468
441{
442 int x;
443
444 POSTCODE(MP_ANNOUNCE_POST);
445
469 printf("FreeBSD/SMP: Multiprocessor motherboard\n");
470 printf(" cpu0 (BSP): apic id: %2d", CPU_TO_ID(0));
471 printf(", version: 0x%08x", cpu_apic_versions[0]);
472 printf(", at 0x%08x\n", cpu_apic_address);
473 for (x = 1; x <= mp_naps; ++x) {
474 printf(" cpu%d (AP): apic id: %2d", x, CPU_TO_ID(x));
475 printf(", version: 0x%08x", cpu_apic_versions[x]);
476 printf(", at 0x%08x\n", cpu_apic_address);
477 }

--- 140 unchanged lines hidden (view full) ---

618 /* install a 'Spurious INTerrupt' vector */
619 setidt(XSPURIOUSINT_OFFSET, Xspuriousint,
620 SDT_SYS386IGT, SEL_KPL, GSEL(GCODE_SEL, SEL_KPL));
621
622 /* install an inter-CPU IPI for TLB invalidation */
623 setidt(XINVLTLB_OFFSET, Xinvltlb,
624 SDT_SYS386IGT, SEL_KPL, GSEL(GCODE_SEL, SEL_KPL));
625
446 printf(" cpu0 (BSP): apic id: %2d", CPU_TO_ID(0));
447 printf(", version: 0x%08x", cpu_apic_versions[0]);
448 printf(", at 0x%08x\n", cpu_apic_address);
449 for (x = 1; x <= mp_naps; ++x) {
450 printf(" cpu%d (AP): apic id: %2d", x, CPU_TO_ID(x));
451 printf(", version: 0x%08x", cpu_apic_versions[x]);
452 printf(", at 0x%08x\n", cpu_apic_address);
453 }

--- 140 unchanged lines hidden (view full) ---

594 /* install a 'Spurious INTerrupt' vector */
595 setidt(XSPURIOUSINT_OFFSET, Xspuriousint,
596 SDT_SYS386IGT, SEL_KPL, GSEL(GCODE_SEL, SEL_KPL));
597
598 /* install an inter-CPU IPI for TLB invalidation */
599 setidt(XINVLTLB_OFFSET, Xinvltlb,
600 SDT_SYS386IGT, SEL_KPL, GSEL(GCODE_SEL, SEL_KPL));
601
626 /* install an inter-CPU IPI for reading processor state */
627 setidt(XCPUCHECKSTATE_OFFSET, Xcpucheckstate,
602 /* install an inter-CPU IPI for forwarding hardclock() */
603 setidt(XHARDCLOCK_OFFSET, Xhardclock,
628 SDT_SYS386IGT, SEL_KPL, GSEL(GCODE_SEL, SEL_KPL));
629
604 SDT_SYS386IGT, SEL_KPL, GSEL(GCODE_SEL, SEL_KPL));
605
606 /* install an inter-CPU IPI for forwarding statclock() */
607 setidt(XSTATCLOCK_OFFSET, Xstatclock,
608 SDT_SYS386IGT, SEL_KPL, GSEL(GCODE_SEL, SEL_KPL));
609
630 /* install an inter-CPU IPI for all-CPU rendezvous */
631 setidt(XRENDEZVOUS_OFFSET, Xrendezvous,
632 SDT_SYS386IGT, SEL_KPL, GSEL(GCODE_SEL, SEL_KPL));
633
634 /* install an inter-CPU IPI for forcing an additional software trap */
635 setidt(XCPUAST_OFFSET, Xcpuast,
636 SDT_SYS386IGT, SEL_KPL, GSEL(GCODE_SEL, SEL_KPL));
637

--- 1295 unchanged lines hidden (view full) ---

1933 u_char mpbiosreason;
1934 u_long mpbioswarmvec;
1935 struct globaldata *gd;
1936 char *stack;
1937 uintptr_t kptbase;
1938
1939 POSTCODE(START_ALL_APS_POST);
1940
610 /* install an inter-CPU IPI for all-CPU rendezvous */
611 setidt(XRENDEZVOUS_OFFSET, Xrendezvous,
612 SDT_SYS386IGT, SEL_KPL, GSEL(GCODE_SEL, SEL_KPL));
613
614 /* install an inter-CPU IPI for forcing an additional software trap */
615 setidt(XCPUAST_OFFSET, Xcpuast,
616 SDT_SYS386IGT, SEL_KPL, GSEL(GCODE_SEL, SEL_KPL));
617

--- 1295 unchanged lines hidden (view full) ---

1913 u_char mpbiosreason;
1914 u_long mpbioswarmvec;
1915 struct globaldata *gd;
1916 char *stack;
1917 uintptr_t kptbase;
1918
1919 POSTCODE(START_ALL_APS_POST);
1920
1921 mtx_init(&ap_boot_mtx, "ap boot", MTX_SPIN);
1922
1941 /* initialize BSP's local APIC */
1942 apic_initialize();
1943 bsp_apic_ready = 1;
1944
1945 /* install the AP 1st level boot code */
1946 install_ap_tramp(boot_addr);
1947
1948

--- 31 unchanged lines hidden (view full) ---

1980
1981 /* allocate and set up an idle stack data page */
1982 stack = (char *)kmem_alloc(kernel_map, UPAGES*PAGE_SIZE);
1983 for (i = 0; i < UPAGES; i++)
1984 SMPpt[pg + 1 + i] = (pt_entry_t)
1985 (PG_V | PG_RW | vtophys(PAGE_SIZE * i + stack));
1986
1987 /* prime data page for it to use */
1923 /* initialize BSP's local APIC */
1924 apic_initialize();
1925 bsp_apic_ready = 1;
1926
1927 /* install the AP 1st level boot code */
1928 install_ap_tramp(boot_addr);
1929
1930

--- 31 unchanged lines hidden (view full) ---

1962
1963 /* allocate and set up an idle stack data page */
1964 stack = (char *)kmem_alloc(kernel_map, UPAGES*PAGE_SIZE);
1965 for (i = 0; i < UPAGES; i++)
1966 SMPpt[pg + 1 + i] = (pt_entry_t)
1967 (PG_V | PG_RW | vtophys(PAGE_SIZE * i + stack));
1968
1969 /* prime data page for it to use */
1988 SLIST_INSERT_HEAD(&cpuhead, gd, gd_allcpu);
1989 gd->gd_cpuid = x;
1970 gd->gd_cpuid = x;
1971 globaldata_register(gd);
1990
1991 /* setup a vector to our boot code */
1992 *((volatile u_short *) WARMBOOT_OFF) = WARMBOOT_TARGET;
1993 *((volatile u_short *) WARMBOOT_SEG) = (boot_addr >> 4);
1994#ifndef PC98
1995 outb(CMOS_REG, BIOS_RESET);
1996 outb(CMOS_DATA, BIOS_WARM); /* 'warm-start' */
1997#endif

--- 325 unchanged lines hidden (view full) ---

2323 /* ok, now grab sched_lock and enter the scheduler */
2324 enable_intr();
2325 mtx_lock_spin(&sched_lock);
2326 cpu_throw(); /* doesn't return */
2327
2328 panic("scheduler returned us to ap_init");
2329}
2330
1972
1973 /* setup a vector to our boot code */
1974 *((volatile u_short *) WARMBOOT_OFF) = WARMBOOT_TARGET;
1975 *((volatile u_short *) WARMBOOT_SEG) = (boot_addr >> 4);
1976#ifndef PC98
1977 outb(CMOS_REG, BIOS_RESET);
1978 outb(CMOS_DATA, BIOS_WARM); /* 'warm-start' */
1979#endif

--- 325 unchanged lines hidden (view full) ---

2305 /* ok, now grab sched_lock and enter the scheduler */
2306 enable_intr();
2307 mtx_lock_spin(&sched_lock);
2308 cpu_throw(); /* doesn't return */
2309
2310 panic("scheduler returned us to ap_init");
2311}
2312
2331#define CHECKSTATE_USER 0
2332#define CHECKSTATE_SYS 1
2333#define CHECKSTATE_INTR 2
2334
2335/* Do not staticize. Used from apic_vector.s */
2336struct proc* checkstate_curproc[MAXCPU];
2337int checkstate_cpustate[MAXCPU];
2338u_long checkstate_pc[MAXCPU];
2339
2340#define PC_TO_INDEX(pc, prof) \
2341 ((int)(((u_quad_t)((pc) - (prof)->pr_off) * \
2342 (u_quad_t)((prof)->pr_scale)) >> 16) & ~1)
2343
2344static void
2345addupc_intr_forwarded(struct proc *p, int id, int *astmap)
2313/*
2314 * For statclock, we send an IPI to all CPU's to have them call this
2315 * function.
2316 */
2317void
2318forwarded_statclock(struct trapframe frame)
2346{
2319{
2347 int i;
2348 struct uprof *prof;
2349 u_long pc;
2350
2320
2351 pc = checkstate_pc[id];
2352 prof = &p->p_stats->p_prof;
2353 if (pc >= prof->pr_off &&
2354 (i = PC_TO_INDEX(pc, prof)) < prof->pr_size) {
2355 mtx_assert(&sched_lock, MA_OWNED);
2356 if ((p->p_sflag & PS_OWEUPC) == 0) {
2357 prof->pr_addr = pc;
2358 prof->pr_ticks = 1;
2359 p->p_sflag |= PS_OWEUPC;
2360 }
2361 *astmap |= (1 << id);
2362 }
2321 mtx_lock_spin(&sched_lock);
2322 statclock_process(curproc, TRAPF_PC(&frame), TRAPF_USERMODE(&frame));
2323 mtx_unlock_spin(&sched_lock);
2363}
2364
2324}
2325
2365static void
2366forwarded_statclock(int id, int pscnt, int *astmap)
2367{
2368 struct pstats *pstats;
2369 long rss;
2370 struct rusage *ru;
2371 struct vmspace *vm;
2372 int cpustate;
2373 struct proc *p;
2374#ifdef GPROF
2375 register struct gmonparam *g;
2376 int i;
2377#endif
2378
2379 mtx_assert(&sched_lock, MA_OWNED);
2380 p = checkstate_curproc[id];
2381 cpustate = checkstate_cpustate[id];
2382
2383 /* XXX */
2384 if (p->p_ithd)
2385 cpustate = CHECKSTATE_INTR;
2386 else if (p == SMP_prvspace[id].globaldata.gd_idleproc)
2387 cpustate = CHECKSTATE_SYS;
2388
2389 switch (cpustate) {
2390 case CHECKSTATE_USER:
2391 if (p->p_sflag & PS_PROFIL)
2392 addupc_intr_forwarded(p, id, astmap);
2393 if (pscnt > 1)
2394 return;
2395 p->p_uticks++;
2396 if (p->p_nice > NZERO)
2397 cp_time[CP_NICE]++;
2398 else
2399 cp_time[CP_USER]++;
2400 break;
2401 case CHECKSTATE_SYS:
2402#ifdef GPROF
2403 /*
2404 * Kernel statistics are just like addupc_intr, only easier.
2405 */
2406 g = &_gmonparam;
2407 if (g->state == GMON_PROF_ON) {
2408 i = checkstate_pc[id] - g->lowpc;
2409 if (i < g->textsize) {
2410 i /= HISTFRACTION * sizeof(*g->kcount);
2411 g->kcount[i]++;
2412 }
2413 }
2414#endif
2415 if (pscnt > 1)
2416 return;
2417
2418 p->p_sticks++;
2419 if (p == SMP_prvspace[id].globaldata.gd_idleproc)
2420 cp_time[CP_IDLE]++;
2421 else
2422 cp_time[CP_SYS]++;
2423 break;
2424 case CHECKSTATE_INTR:
2425 default:
2426#ifdef GPROF
2427 /*
2428 * Kernel statistics are just like addupc_intr, only easier.
2429 */
2430 g = &_gmonparam;
2431 if (g->state == GMON_PROF_ON) {
2432 i = checkstate_pc[id] - g->lowpc;
2433 if (i < g->textsize) {
2434 i /= HISTFRACTION * sizeof(*g->kcount);
2435 g->kcount[i]++;
2436 }
2437 }
2438#endif
2439 if (pscnt > 1)
2440 return;
2441 KASSERT(p != NULL, ("NULL process in interrupt state"));
2442 p->p_iticks++;
2443 cp_time[CP_INTR]++;
2444 }
2445
2446 schedclock(p);
2447
2448 /* Update resource usage integrals and maximums. */
2449 if ((pstats = p->p_stats) != NULL &&
2450 (ru = &pstats->p_ru) != NULL &&
2451 (vm = p->p_vmspace) != NULL) {
2452 ru->ru_ixrss += pgtok(vm->vm_tsize);
2453 ru->ru_idrss += pgtok(vm->vm_dsize);
2454 ru->ru_isrss += pgtok(vm->vm_ssize);
2455 rss = pgtok(vmspace_resident_count(vm));
2456 if (ru->ru_maxrss < rss)
2457 ru->ru_maxrss = rss;
2458 }
2459}
2460
2461void
2326void
2462forward_statclock(int pscnt)
2327forward_statclock(void)
2463{
2464 int map;
2328{
2329 int map;
2465 int id;
2466 int i;
2467
2330
2468 /* Kludge. We don't yet have separate locks for the interrupts
2469 * and the kernel. This means that we cannot let the other processors
2470 * handle complex interrupts while inhibiting them from entering
2471 * the kernel in a non-interrupt context.
2472 *
2473 * What we can do, without changing the locking mechanisms yet,
2474 * is letting the other processors handle a very simple interrupt
2475 * (wich determines the processor states), and do the main
2476 * work ourself.
2477 */
2331 CTR0(KTR_SMP, "forward_statclock");
2478
2332
2479 CTR1(KTR_SMP, "forward_statclock(%d)", pscnt);
2480
2481 if (!smp_started || !invltlb_ok || cold || panicstr)
2482 return;
2483
2333 if (!smp_started || !invltlb_ok || cold || panicstr)
2334 return;
2335
2484 /* Step 1: Probe state (user, cpu, interrupt, spinlock, idle ) */
2485
2486 map = PCPU_GET(other_cpus) & ~stopped_cpus ;
2336 map = PCPU_GET(other_cpus) & ~stopped_cpus ;
2487 checkstate_probed_cpus = 0;
2488 if (map != 0)
2337 if (map != 0)
2489 ipi_selected(map, IPI_CHECKSTATE);
2490
2491 i = 0;
2492 while (checkstate_probed_cpus != map) {
2493 /* spin */
2494 i++;
2495 if (i == 100000) {
2496#ifdef DIAGNOSTIC
2497 printf("forward_statclock: checkstate %x\n",
2498 checkstate_probed_cpus);
2499#endif
2500 break;
2501 }
2502 }
2503
2504 /*
2505 * Step 2: walk through other processors processes, update ticks and
2506 * profiling info.
2507 */
2508
2509 map = 0;
2510 for (id = 0; id < mp_ncpus; id++) {
2511 if (id == PCPU_GET(cpuid))
2512 continue;
2513 if (((1 << id) & checkstate_probed_cpus) == 0)
2514 continue;
2515 forwarded_statclock(id, pscnt, &map);
2516 }
2517 if (map != 0) {
2518 checkstate_need_ast |= map;
2519 ipi_selected(map, IPI_AST);
2520 i = 0;
2521 while ((checkstate_need_ast & map) != 0) {
2522 /* spin */
2523 i++;
2524 if (i > 100000) {
2525#ifdef DIAGNOSTIC
2526 printf("forward_statclock: dropped ast 0x%x\n",
2527 checkstate_need_ast & map);
2528#endif
2529 break;
2530 }
2531 }
2532 }
2338 ipi_selected(map, IPI_STATCLOCK);
2533}
2534
2339}
2340
2535void
2536forward_hardclock(int pscnt)
2341/*
2342 * For each hardclock(), we send an IPI to all other CPU's to have them
2343 * execute this function. It would be nice to reduce contention on
2344 * sched_lock if we could simply peek at the CPU to determine the user/kernel
2345 * state and call hardclock_process() on the CPU receiving the clock interrupt
2346 * and then just use a simple IPI to handle any ast's if needed.
2347 */
2348void
2349forwarded_hardclock(struct trapframe frame)
2537{
2350{
2538 int map;
2539 int id;
2540 struct proc *p;
2541 struct pstats *pstats;
2542 int i;
2543
2351
2544 /* Kludge. We don't yet have separate locks for the interrupts
2545 * and the kernel. This means that we cannot let the other processors
2546 * handle complex interrupts while inhibiting them from entering
2547 * the kernel in a non-interrupt context.
2548 *
2549 * What we can do, without changing the locking mechanisms yet,
2550 * is letting the other processors handle a very simple interrupt
2551 * (wich determines the processor states), and do the main
2552 * work ourself.
2553 */
2554
2555 CTR1(KTR_SMP, "forward_hardclock(%d)", pscnt);
2556
2557 if (!smp_started || !invltlb_ok || cold || panicstr)
2558 return;
2559
2560 /* Step 1: Probe state (user, cpu, interrupt, spinlock, idle) */
2561
2562 map = PCPU_GET(other_cpus) & ~stopped_cpus ;
2563 checkstate_probed_cpus = 0;
2564 if (map != 0)
2565 ipi_selected(map, IPI_CHECKSTATE);
2566
2567 i = 0;
2568 while (checkstate_probed_cpus != map) {
2569 /* spin */
2570 i++;
2571 if (i == 100000) {
2572#ifdef DIAGNOSTIC
2573 printf("forward_hardclock: checkstate %x\n",
2574 checkstate_probed_cpus);
2575#endif
2576 break;
2577 }
2578 }
2579
2580 /*
2581 * Step 2: walk through other processors processes, update virtual
2582 * timer and profiling timer. If stathz == 0, also update ticks and
2583 * profiling info.
2584 */
2585
2586 map = 0;
2587 for (id = 0; id < mp_ncpus; id++) {
2588 if (id == PCPU_GET(cpuid))
2589 continue;
2590 if (((1 << id) & checkstate_probed_cpus) == 0)
2591 continue;
2592 p = checkstate_curproc[id];
2593 if (p) {
2594 pstats = p->p_stats;
2595 if (checkstate_cpustate[id] == CHECKSTATE_USER &&
2596 timevalisset(&pstats->p_timer[ITIMER_VIRTUAL].it_value) &&
2597 itimerdecr(&pstats->p_timer[ITIMER_VIRTUAL], tick) == 0) {
2598 p->p_sflag |= PS_ALRMPEND;
2599 map |= (1 << id);
2600 }
2601 if (timevalisset(&pstats->p_timer[ITIMER_PROF].it_value) &&
2602 itimerdecr(&pstats->p_timer[ITIMER_PROF], tick) == 0) {
2603 p->p_sflag |= PS_PROFPEND;
2604 map |= (1 << id);
2605 }
2606 }
2607 if (stathz == 0) {
2608 forwarded_statclock( id, pscnt, &map);
2609 }
2610 }
2611 if (map != 0) {
2612 checkstate_need_ast |= map;
2613 ipi_selected(map, IPI_AST);
2614 i = 0;
2615 while ((checkstate_need_ast & map) != 0) {
2616 /* spin */
2617 i++;
2618 if (i > 100000) {
2619#ifdef DIAGNOSTIC
2620 printf("forward_hardclock: dropped ast 0x%x\n",
2621 checkstate_need_ast & map);
2622#endif
2623 break;
2624 }
2625 }
2626 }
2352 mtx_lock_spin(&sched_lock);
2353 hardclock_process(curproc, TRAPF_USERMODE(&frame));
2354 mtx_unlock_spin(&sched_lock);
2627}
2628
2629void
2355}
2356
2357void
2630forward_signal(struct proc *p)
2358forward_hardclock(void)
2631{
2359{
2632 int map;
2633 int id;
2634 int i;
2635
2636 /* Kludge. We don't yet have separate locks for the interrupts
2637 * and the kernel. This means that we cannot let the other processors
2638 * handle complex interrupts while inhibiting them from entering
2639 * the kernel in a non-interrupt context.
2640 *
2641 * What we can do, without changing the locking mechanisms yet,
2642 * is letting the other processors handle a very simple interrupt
2643 * (wich determines the processor states), and do the main
2644 * work ourself.
2645 */
2646
2647 CTR1(KTR_SMP, "forward_signal(%p)", p);
2648
2649 if (!smp_started || !invltlb_ok || cold || panicstr)
2650 return;
2651 if (!forward_signal_enabled)
2652 return;
2653 mtx_lock_spin(&sched_lock);
2654 while (1) {
2655 if (p->p_stat != SRUN) {
2656 mtx_unlock_spin(&sched_lock);
2657 return;
2658 }
2659 id = p->p_oncpu;
2660 mtx_unlock_spin(&sched_lock);
2661 if (id == 0xff)
2662 return;
2663 map = (1<<id);
2664 checkstate_need_ast |= map;
2665 ipi_selected(map, IPI_AST);
2666 i = 0;
2667 while ((checkstate_need_ast & map) != 0) {
2668 /* spin */
2669 i++;
2670 if (i > 100000) {
2671#if 0
2672 printf("forward_signal: dropped ast 0x%x\n",
2673 checkstate_need_ast & map);
2674#endif
2675 break;
2676 }
2677 }
2678 mtx_lock_spin(&sched_lock);
2679 if (id == p->p_oncpu) {
2680 mtx_unlock_spin(&sched_lock);
2681 return;
2682 }
2683 }
2684}
2685
2686void
2687forward_roundrobin(void)
2688{
2689 u_int map;
2360 u_int map;
2690 int i;
2691
2361
2692 CTR0(KTR_SMP, "forward_roundrobin()");
2362 CTR0(KTR_SMP, "forward_hardclock");
2693
2694 if (!smp_started || !invltlb_ok || cold || panicstr)
2695 return;
2363
2364 if (!smp_started || !invltlb_ok || cold || panicstr)
2365 return;
2696 if (!forward_roundrobin_enabled)
2697 return;
2698 resched_cpus |= PCPU_GET(other_cpus);
2366
2699 map = PCPU_GET(other_cpus) & ~stopped_cpus ;
2367 map = PCPU_GET(other_cpus) & ~stopped_cpus ;
2700#if 1
2701 ipi_selected(map, IPI_AST);
2702#else
2703 ipi_all_but_self(IPI_AST);
2704#endif
2705 i = 0;
2706 while ((checkstate_need_ast & map) != 0) {
2707 /* spin */
2708 i++;
2709 if (i > 100000) {
2710#if 0
2711 printf("forward_roundrobin: dropped ast 0x%x\n",
2712 checkstate_need_ast & map);
2713#endif
2714 break;
2715 }
2716 }
2368 if (map != 0)
2369 ipi_selected(map, IPI_HARDCLOCK);
2717}
2718
2370}
2371
2719/*
2720 * When called the executing CPU will send an IPI to all other CPUs
2721 * requesting that they halt execution.
2722 *
2723 * Usually (but not necessarily) called with 'other_cpus' as its arg.
2724 *
2725 * - Signals all CPUs in map to stop.
2726 * - Waits for each to stop.
2727 *
2728 * Returns:
2729 * -1: error
2730 * 0: NA
2731 * 1: ok
2732 *
2733 * XXX FIXME: this is not MP-safe, needs a lock to prevent multiple CPUs
2734 * from executing at same time.
2735 */
2736int
2737stop_cpus(u_int map)
2738{
2739 int count = 0;
2740
2741 if (!smp_started)
2742 return 0;
2743
2744 /* send the Xcpustop IPI to all CPUs in map */
2745 ipi_selected(map, IPI_STOP);
2746
2747 while (count++ < 100000 && (stopped_cpus & map) != map)
2748 /* spin */ ;
2749
2750#ifdef DIAGNOSTIC
2751 if ((stopped_cpus & map) != map)
2752 printf("Warning: CPUs 0x%x did not stop!\n",
2753 (~(stopped_cpus & map)) & map);
2754#endif
2755
2756 return 1;
2757}
2758
2759
2760/*
2761 * Called by a CPU to restart stopped CPUs.
2762 *
2763 * Usually (but not necessarily) called with 'stopped_cpus' as its arg.
2764 *
2765 * - Signals all CPUs in map to restart.
2766 * - Waits for each to restart.
2767 *
2768 * Returns:
2769 * -1: error
2770 * 0: NA
2771 * 1: ok
2772 */
2773int
2774restart_cpus(u_int map)
2775{
2776 int count = 0;
2777
2778 if (!smp_started)
2779 return 0;
2780
2781 started_cpus = map; /* signal other cpus to restart */
2782
2783 /* wait for each to clear its bit */
2784 while (count++ < 100000 && (stopped_cpus & map) != 0)
2785 /* spin */ ;
2786
2787#ifdef DIAGNOSTIC
2788 if ((stopped_cpus & map) != 0)
2789 printf("Warning: CPUs 0x%x did not restart!\n",
2790 (~(stopped_cpus & map)) & map);
2791#endif
2792
2793 return 1;
2794}
2795
2796
2797#ifdef APIC_INTR_REORDER
2798/*
2799 * Maintain mapping from softintr vector to isr bit in local apic.
2800 */
2801void
2802set_lapic_isrloc(int intr, int vector)
2803{
2804 if (intr < 0 || intr > 32)
2805 panic("set_apic_isrloc: bad intr argument: %d",intr);
2806 if (vector < ICU_OFFSET || vector > 255)
2807 panic("set_apic_isrloc: bad vector argument: %d",vector);
2808 apic_isrbit_location[intr].location = &lapic.isr0 + ((vector>>5)<<2);
2809 apic_isrbit_location[intr].bit = (1<<(vector & 31));
2810}
2811#endif
2812
2813/*
2372#ifdef APIC_INTR_REORDER
2373/*
2374 * Maintain mapping from softintr vector to isr bit in local apic.
2375 */
2376void
2377set_lapic_isrloc(int intr, int vector)
2378{
2379 if (intr < 0 || intr > 32)
2380 panic("set_apic_isrloc: bad intr argument: %d",intr);
2381 if (vector < ICU_OFFSET || vector > 255)
2382 panic("set_apic_isrloc: bad vector argument: %d",vector);
2383 apic_isrbit_location[intr].location = &lapic.isr0 + ((vector>>5)<<2);
2384 apic_isrbit_location[intr].bit = (1<<(vector & 31));
2385}
2386#endif
2387
2388/*
2814 * All-CPU rendezvous. CPUs are signalled, all execute the setup function
2815 * (if specified), rendezvous, execute the action function (if specified),
2816 * rendezvous again, execute the teardown function (if specified), and then
2817 * resume.
2818 *
2819 * Note that the supplied external functions _must_ be reentrant and aware
2820 * that they are running in parallel and in an unknown lock context.
2821 */
2822static void (*smp_rv_setup_func)(void *arg);
2823static void (*smp_rv_action_func)(void *arg);
2824static void (*smp_rv_teardown_func)(void *arg);
2825static void *smp_rv_func_arg;
2826static volatile int smp_rv_waiters[2];
2827
2828void
2829smp_rendezvous_action(void)
2830{
2831 /* setup function */
2832 if (smp_rv_setup_func != NULL)
2833 smp_rv_setup_func(smp_rv_func_arg);
2834 /* spin on entry rendezvous */
2835 atomic_add_int(&smp_rv_waiters[0], 1);
2836 while (smp_rv_waiters[0] < mp_ncpus)
2837 ;
2838 /* action function */
2839 if (smp_rv_action_func != NULL)
2840 smp_rv_action_func(smp_rv_func_arg);
2841 /* spin on exit rendezvous */
2842 atomic_add_int(&smp_rv_waiters[1], 1);
2843 while (smp_rv_waiters[1] < mp_ncpus)
2844 ;
2845 /* teardown function */
2846 if (smp_rv_teardown_func != NULL)
2847 smp_rv_teardown_func(smp_rv_func_arg);
2848}
2849
2850void
2851smp_rendezvous(void (* setup_func)(void *),
2852 void (* action_func)(void *),
2853 void (* teardown_func)(void *),
2854 void *arg)
2855{
2856
2857 /* obtain rendezvous lock */
2858 mtx_lock_spin(&smp_rv_mtx);
2859
2860 /* set static function pointers */
2861 smp_rv_setup_func = setup_func;
2862 smp_rv_action_func = action_func;
2863 smp_rv_teardown_func = teardown_func;
2864 smp_rv_func_arg = arg;
2865 smp_rv_waiters[0] = 0;
2866 smp_rv_waiters[1] = 0;
2867
2868 /*
2869 * signal other processors, which will enter the IPI with interrupts off
2870 */
2871 ipi_all_but_self(IPI_RENDEZVOUS);
2872
2873 /* call executor function */
2874 smp_rendezvous_action();
2875
2876 /* release lock */
2877 mtx_unlock_spin(&smp_rv_mtx);
2878}
2879
2880/*
2881 * send an IPI to a set of cpus.
2882 */
2883void
2884ipi_selected(u_int32_t cpus, u_int ipi)
2885{
2886
2887 CTR2(KTR_SMP, __func__ ": cpus: %x ipi: %x", cpus, ipi);
2888 selected_apic_ipi(cpus, ipi, APIC_DELMODE_FIXED);

--- 42 unchanged lines hidden --- 		2389 * send an IPI to a set of cpus.
2390 */
2391void
2392ipi_selected(u_int32_t cpus, u_int ipi)
2393{
2394
2395 CTR2(KTR_SMP, __func__ ": cpus: %x ipi: %x", cpus, ipi);
2396 selected_apic_ipi(cpus, ipi, APIC_DELMODE_FIXED);

--- 42 unchanged lines hidden ---