Deleted Added
sdiff udiff text old ( 99766 ) new ( 99862 )
full compact
mptable.h (99766) mptable.h (99862)
1/*
2 * Copyright (c) 1996, by Steve Passe
3 * All rights reserved.
4 *
5 * Redistribution and use in source and binary forms, with or without
6 * modification, are permitted provided that the following conditions
7 * are met:
8 * 1. Redistributions of source code must retain the above copyright

--- 8 unchanged lines hidden (view full) ---

17 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
18 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
19 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
20 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
21 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
22 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
23 * SUCH DAMAGE.
24 *
1/*
2 * Copyright (c) 1996, by Steve Passe
3 * All rights reserved.
4 *
5 * Redistribution and use in source and binary forms, with or without
6 * modification, are permitted provided that the following conditions
7 * are met:
8 * 1. Redistributions of source code must retain the above copyright

--- 8 unchanged lines hidden (view full) ---

17 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
18 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
19 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
20 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
21 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
22 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
23 * SUCH DAMAGE.
24 *
25 * $FreeBSD: head/sys/i386/include/mptable.h 99766 2002-07-11 08:31:10Z peter $
25 * $FreeBSD: head/sys/i386/include/mptable.h 99862 2002-07-12 07:56:11Z peter $
26 */
27
28#include "opt_cpu.h"
29#include "opt_kstack_pages.h"
30
31#ifdef SMP
32#include <machine/smptests.h>
33#else

--- 249 unchanged lines hidden (view full) ---

283/* Hotwire a 0->4MB V==P mapping */
284extern pt_entry_t *KPTphys;
285
286/* SMP page table page */
287extern pt_entry_t *SMPpt;
288
289struct pcb stoppcbs[MAXCPU];
290
26 */
27
28#include "opt_cpu.h"
29#include "opt_kstack_pages.h"
30
31#ifdef SMP
32#include <machine/smptests.h>
33#else

--- 249 unchanged lines hidden (view full) ---

283/* Hotwire a 0->4MB V==P mapping */
284extern pt_entry_t *KPTphys;
285
286/* SMP page table page */
287extern pt_entry_t *SMPpt;
288
289struct pcb stoppcbs[MAXCPU];
290
291#ifdef APIC_IO
292/* Variables needed for SMP tlb shootdown. */
293vm_offset_t smp_tlb_addr1;
294vm_offset_t smp_tlb_addr2;
295volatile int smp_tlb_wait;
296static struct mtx smp_tlb_mtx;
297#endif
298
291/*
292 * Local data and functions.
293 */
294
295/* Set to 1 once we're ready to let the APs out of the pen. */
296static volatile int aps_ready = 0;
297
298static int mp_capable;

--- 32 unchanged lines hidden (view full) ---

331
332static void
333init_locks(void)
334{
335
336#ifdef USE_COMLOCK
337 mtx_init(&com_mtx, "com", NULL, MTX_SPIN);
338#endif /* USE_COMLOCK */
299/*
300 * Local data and functions.
301 */
302
303/* Set to 1 once we're ready to let the APs out of the pen. */
304static volatile int aps_ready = 0;
305
306static int mp_capable;

--- 32 unchanged lines hidden (view full) ---

339
340static void
341init_locks(void)
342{
343
344#ifdef USE_COMLOCK
345 mtx_init(&com_mtx, "com", NULL, MTX_SPIN);
346#endif /* USE_COMLOCK */
347#ifdef APIC_IO
348 mtx_init(&smp_tlb_mtx, "tlb", NULL, MTX_SPIN);
349#endif
339}
340
341/*
342 * Calculate usable address in base memory for AP trampoline code.
343 */
344u_int
345mp_bootaddress(u_int basemem)
346{

--- 253 unchanged lines hidden (view full) ---

600
601 /* install a 'Spurious INTerrupt' vector */
602 setidt(XSPURIOUSINT_OFFSET, Xspuriousint,
603 SDT_SYS386IGT, SEL_KPL, GSEL(GCODE_SEL, SEL_KPL));
604
605 /* install an inter-CPU IPI for TLB invalidation */
606 setidt(XINVLTLB_OFFSET, Xinvltlb,
607 SDT_SYS386IGT, SEL_KPL, GSEL(GCODE_SEL, SEL_KPL));
350}
351
352/*
353 * Calculate usable address in base memory for AP trampoline code.
354 */
355u_int
356mp_bootaddress(u_int basemem)
357{

--- 253 unchanged lines hidden (view full) ---

611
612 /* install a 'Spurious INTerrupt' vector */
613 setidt(XSPURIOUSINT_OFFSET, Xspuriousint,
614 SDT_SYS386IGT, SEL_KPL, GSEL(GCODE_SEL, SEL_KPL));
615
616 /* install an inter-CPU IPI for TLB invalidation */
617 setidt(XINVLTLB_OFFSET, Xinvltlb,
618 SDT_SYS386IGT, SEL_KPL, GSEL(GCODE_SEL, SEL_KPL));
619 setidt(XINVLPG_OFFSET, Xinvlpg,
620 SDT_SYS386IGT, SEL_KPL, GSEL(GCODE_SEL, SEL_KPL));
621 setidt(XINVLRNG_OFFSET, Xinvlrng,
622 SDT_SYS386IGT, SEL_KPL, GSEL(GCODE_SEL, SEL_KPL));
608
609 /* install an inter-CPU IPI for forwarding hardclock() */
610 setidt(XHARDCLOCK_OFFSET, Xhardclock,
611 SDT_SYS386IGT, SEL_KPL, GSEL(GCODE_SEL, SEL_KPL));
612
613 /* install an inter-CPU IPI for forwarding statclock() */
614 setidt(XSTATCLOCK_OFFSET, Xstatclock,
615 SDT_SYS386IGT, SEL_KPL, GSEL(GCODE_SEL, SEL_KPL));

--- 1569 unchanged lines hidden (view full) ---

2185 set_apic_timer(5000000);/* == 5 seconds */
2186 while (read_apic_timer())
2187 if (mp_ncpus > cpus)
2188 return 1; /* return SUCCESS */
2189
2190 return 0; /* return FAILURE */
2191}
2192
623
624 /* install an inter-CPU IPI for forwarding hardclock() */
625 setidt(XHARDCLOCK_OFFSET, Xhardclock,
626 SDT_SYS386IGT, SEL_KPL, GSEL(GCODE_SEL, SEL_KPL));
627
628 /* install an inter-CPU IPI for forwarding statclock() */
629 setidt(XSTATCLOCK_OFFSET, Xstatclock,
630 SDT_SYS386IGT, SEL_KPL, GSEL(GCODE_SEL, SEL_KPL));

--- 1569 unchanged lines hidden (view full) ---

2200 set_apic_timer(5000000);/* == 5 seconds */
2201 while (read_apic_timer())
2202 if (mp_ncpus > cpus)
2203 return 1; /* return SUCCESS */
2204
2205 return 0; /* return FAILURE */
2206}
2207
2193#if defined(APIC_IO) && defined(COUNT_XINVLTLB_HITS)
2194u_int xhits[MAXCPU];
2195SYSCTL_OPAQUE(_debug, OID_AUTO, xhits, CTLFLAG_RW, &xhits, sizeof(xhits),
2196 "IU", "");
2208#if defined(APIC_IO)
2209
2210#ifdef COUNT_XINVLTLB_HITS
2211u_int xhits_gbl[MAXCPU];
2212u_int xhits_pg[MAXCPU];
2213u_int xhits_rng[MAXCPU];
2214SYSCTL_NODE(_debug, OID_AUTO, xhits, CTLFLAG_RW, 0, "");
2215SYSCTL_OPAQUE(_debug_xhits, OID_AUTO, global, CTLFLAG_RW, &xhits_gbl,
2216 sizeof(xhits_gbl), "IU", "");
2217SYSCTL_OPAQUE(_debug_xhits, OID_AUTO, page, CTLFLAG_RW, &xhits_pg,
2218 sizeof(xhits_pg), "IU", "");
2219SYSCTL_OPAQUE(_debug_xhits, OID_AUTO, range, CTLFLAG_RW, &xhits_rng,
2220 sizeof(xhits_rng), "IU", "");
2221
2222u_int ipi_global;
2223u_int ipi_page;
2224u_int ipi_range;
2225u_int ipi_range_size;
2226SYSCTL_INT(_debug_xhits, OID_AUTO, ipi_global, CTLFLAG_RW, &ipi_global, 0, "");
2227SYSCTL_INT(_debug_xhits, OID_AUTO, ipi_page, CTLFLAG_RW, &ipi_page, 0, "");
2228SYSCTL_INT(_debug_xhits, OID_AUTO, ipi_range, CTLFLAG_RW, &ipi_range, 0, "");
2229SYSCTL_INT(_debug_xhits, OID_AUTO, ipi_range_size, CTLFLAG_RW, &ipi_range_size,
2230 0, "");
2231
2232u_int ipi_masked_global;
2233u_int ipi_masked_page;
2234u_int ipi_masked_range;
2235u_int ipi_masked_range_size;
2236SYSCTL_INT(_debug_xhits, OID_AUTO, ipi_masked_global, CTLFLAG_RW,
2237 &ipi_masked_global, 0, "");
2238SYSCTL_INT(_debug_xhits, OID_AUTO, ipi_masked_page, CTLFLAG_RW,
2239 &ipi_masked_page, 0, "");
2240SYSCTL_INT(_debug_xhits, OID_AUTO, ipi_masked_range, CTLFLAG_RW,
2241 &ipi_masked_range, 0, "");
2242SYSCTL_INT(_debug_xhits, OID_AUTO, ipi_masked_range_size, CTLFLAG_RW,
2243 &ipi_masked_range_size, 0, "");
2197#endif
2198
2199/*
2200 * Flush the TLB on all other CPU's
2244#endif
2245
2246/*
2247 * Flush the TLB on all other CPU's
2248 */
2249static void
2250smp_tlb_shootdown(u_int vector, vm_offset_t addr1, vm_offset_t addr2)
2251{
2252 u_int ncpu;
2253 register_t eflags;
2254
2255 ncpu = mp_ncpus - 1; /* does not shootdown self */
2256 if (ncpu < 1)
2257 return; /* no other cpus */
2258 eflags = read_eflags();
2259 if ((eflags & PSL_I) == 0)
2260 panic("absolutely cannot call smp_ipi_shootdown with interrupts already disabled");
2261 mtx_lock_spin(&smp_tlb_mtx);
2262 smp_tlb_addr1 = addr1;
2263 smp_tlb_addr2 = addr2;
2264 atomic_store_rel_int(&smp_tlb_wait, 0);
2265 ipi_all_but_self(vector);
2266 while (smp_tlb_wait < ncpu)
2267 ia32_pause();
2268 mtx_unlock_spin(&smp_tlb_mtx);
2269}
2270
2271/*
2272 * This is about as magic as it gets. fortune(1) has got similar code
2273 * for reversing bits in a word. Who thinks up this stuff??
2201 *
2274 *
2202 * XXX: Needs to handshake and wait for completion before proceding.
2275 * Yes, it does appear to be consistently faster than:
2276 * while (i = ffs(m)) {
2277 * m >>= i;
2278 * bits++;
2279 * }
2280 * and
2281 * while (lsb = (m & -m)) { // This is magic too
2282 * m &= ~lsb; // or: m ^= lsb
2283 * bits++;
2284 * }
2285 * Both of these latter forms do some very strange things on gcc-3.1 with
2286 * -mcpu=pentiumpro and/or -march=pentiumpro and/or -O or -O2.
2287 * There is probably an SSE or MMX popcnt instruction.
2288 *
2289 * I wonder if this should be in libkern?
2290 *
2291 * XXX Stop the presses! Another one:
2292 * static __inline u_int32_t
2293 * popcnt1(u_int32_t v)
2294 * {
2295 * v -= ((v >> 1) & 0x55555555);
2296 * v = (v & 0x33333333) + ((v >> 2) & 0x33333333);
2297 * v = (v + (v >> 4)) & 0x0F0F0F0F;
2298 * return (v * 0x01010101) >> 24;
2299 * }
2300 * The downside is that it has a multiply. With a pentium3 with
2301 * -mcpu=pentiumpro and -march=pentiumpro then gcc-3.1 will use
2302 * an imull, and in that case it is faster. In most other cases
2303 * it appears slightly slower.
2203 */
2304 */
2305static __inline u_int32_t
2306popcnt(u_int32_t m)
2307{
2308
2309 m = (m & 0x55555555) + ((m & 0xaaaaaaaa) >> 1);
2310 m = (m & 0x33333333) + ((m & 0xcccccccc) >> 2);
2311 m = (m & 0x0f0f0f0f) + ((m & 0xf0f0f0f0) >> 4);
2312 m = (m & 0x00ff00ff) + ((m & 0xff00ff00) >> 8);
2313 m = (m & 0x0000ffff) + ((m & 0xffff0000) >> 16);
2314 return m;
2315}
2316
2317static void
2318smp_targeted_tlb_shootdown(u_int mask, u_int vector, vm_offset_t addr1, vm_offset_t addr2)
2319{
2320 int ncpu, othercpus;
2321 register_t eflags;
2322
2323 othercpus = mp_ncpus - 1;
2324 if (mask == (u_int)-1) {
2325 ncpu = othercpus;
2326 if (ncpu < 1)
2327 return;
2328 } else {
2329 /* XXX there should be a pcpu self mask */
2330 mask &= ~(1 << PCPU_GET(cpuid));
2331 if (mask == 0)
2332 return;
2333 ncpu = popcnt(mask);
2334 if (ncpu > othercpus) {
2335 /* XXX this should be a panic offence */
2336 printf("SMP: tlb shootdown to %d other cpus (only have %d)\n",
2337 ncpu, othercpus);
2338 ncpu = othercpus;
2339 }
2340 /* XXX should be a panic, implied by mask == 0 above */
2341 if (ncpu < 1)
2342 return;
2343 }
2344 eflags = read_eflags();
2345 if ((eflags & PSL_I) == 0)
2346 panic("absolutely cannot call smp_targeted_ipi_shootdown with interrupts already disabled");
2347 mtx_lock_spin(&smp_tlb_mtx);
2348 smp_tlb_addr1 = addr1;
2349 smp_tlb_addr2 = addr2;
2350 atomic_store_rel_int(&smp_tlb_wait, 0);
2351 if (mask == (u_int)-1)
2352 ipi_all_but_self(vector);
2353 else
2354 ipi_selected(mask, vector);
2355 while (smp_tlb_wait < ncpu)
2356 ia32_pause();
2357 mtx_unlock_spin(&smp_tlb_mtx);
2358}
2359#endif
2360
2204void
2205smp_invltlb(void)
2206{
2207#if defined(APIC_IO)
2361void
2362smp_invltlb(void)
2363{
2364#if defined(APIC_IO)
2208 if (smp_started)
2209 ipi_all_but_self(IPI_INVLTLB);
2365 if (smp_started) {
2366 smp_tlb_shootdown(IPI_INVLTLB, 0, 0);
2367#ifdef COUNT_XINVLTLB_HITS
2368 ipi_global++;
2369#endif
2370 }
2210#endif /* APIC_IO */
2211}
2212
2213void
2371#endif /* APIC_IO */
2372}
2373
2374void
2214invlpg(u_int addr)
2375smp_invlpg(vm_offset_t addr)
2215{
2376{
2216 __asm __volatile("invlpg (%0)"::"r"(addr):"memory");
2377#if defined(APIC_IO)
2378 if (smp_started) {
2379 smp_tlb_shootdown(IPI_INVLPG, addr, 0);
2380#ifdef COUNT_XINVLTLB_HITS
2381 ipi_page++;
2382#endif
2383 }
2384#endif /* APIC_IO */
2385}
2217
2386
2218 /* send a message to the other CPUs */
2219 smp_invltlb();
2387void
2388smp_invlpg_range(vm_offset_t addr1, vm_offset_t addr2)
2389{
2390#if defined(APIC_IO)
2391 if (smp_started) {
2392 smp_tlb_shootdown(IPI_INVLRNG, addr1, addr2);
2393#ifdef COUNT_XINVLTLB_HITS
2394 ipi_range++;
2395 ipi_range_size += (addr2 - addr1) / PAGE_SIZE;
2396#endif
2397 }
2398#endif /* APIC_IO */
2220}
2221
2222void
2399}
2400
2401void
2223invltlb(void)
2402smp_masked_invltlb(u_int mask)
2224{
2403{
2225 u_long temp;
2404#if defined(APIC_IO)
2405 if (smp_started) {
2406 smp_targeted_tlb_shootdown(mask, IPI_INVLTLB, 0, 0);
2407#ifdef COUNT_XINVLTLB_HITS
2408 ipi_masked_global++;
2409#endif
2410 }
2411#endif /* APIC_IO */
2412}
2226
2413
2227 /*
2228 * This should be implemented as load_cr3(rcr3()) when load_cr3() is
2229 * inlined.
2230 */
2231 __asm __volatile("movl %%cr3, %0; movl %0, %%cr3":"=r"(temp) :: "memory");
2414void
2415smp_masked_invlpg(u_int mask, vm_offset_t addr)
2416{
2417#if defined(APIC_IO)
2418 if (smp_started) {
2419 smp_targeted_tlb_shootdown(mask, IPI_INVLPG, addr, 0);
2420#ifdef COUNT_XINVLTLB_HITS
2421 ipi_masked_page++;
2422#endif
2423 }
2424#endif /* APIC_IO */
2425}
2232
2426
2233 /* send a message to the other CPUs */
2234 smp_invltlb();
2427void
2428smp_masked_invlpg_range(u_int mask, vm_offset_t addr1, vm_offset_t addr2)
2429{
2430#if defined(APIC_IO)
2431 if (smp_started) {
2432 smp_targeted_tlb_shootdown(mask, IPI_INVLRNG, addr1, addr2);
2433#ifdef COUNT_XINVLTLB_HITS
2434 ipi_masked_range++;
2435 ipi_masked_range_size += (addr2 - addr1) / PAGE_SIZE;
2436#endif
2437 }
2438#endif /* APIC_IO */
2235}
2236
2237
2238/*
2239 * This is called once the rest of the system is up and running and we're
2240 * ready to let the AP's out of the pen.
2241 */
2242extern void enable_sse(void);
2243
2244void
2245ap_init(void)
2246{
2247 u_int apic_id;
2248
2249 /* spin until all the AP's are ready */
2250 while (!aps_ready)
2251 /* spin */ ;
2252
2253 /* BSP may have changed PTD while we were waiting */
2439}
2440
2441
2442/*
2443 * This is called once the rest of the system is up and running and we're
2444 * ready to let the AP's out of the pen.
2445 */
2446extern void enable_sse(void);
2447
2448void
2449ap_init(void)
2450{
2451 u_int apic_id;
2452
2453 /* spin until all the AP's are ready */
2454 while (!aps_ready)
2455 /* spin */ ;
2456
2457 /* BSP may have changed PTD while we were waiting */
2254 cpu_invltlb();
2458 invltlb();
2255
2256#if defined(I586_CPU) && !defined(NO_F00F_HACK)
2257 lidt(&r_idt);
2258#endif
2259
2260 /* set up CPU registers and state */
2261 cpu_setregs();
2262

--- 22 unchanged lines hidden (view full) ---

2285
2286 CTR1(KTR_SMP, "SMP: AP CPU #%d Launched", PCPU_GET(cpuid));
2287
2288 smp_cpus++;
2289
2290 /* Build our map of 'other' CPUs. */
2291 PCPU_SET(other_cpus, all_cpus & ~PCPU_GET(cpumask));
2292
2459
2460#if defined(I586_CPU) && !defined(NO_F00F_HACK)
2461 lidt(&r_idt);
2462#endif
2463
2464 /* set up CPU registers and state */
2465 cpu_setregs();
2466

--- 22 unchanged lines hidden (view full) ---

2489
2490 CTR1(KTR_SMP, "SMP: AP CPU #%d Launched", PCPU_GET(cpuid));
2491
2492 smp_cpus++;
2493
2494 /* Build our map of 'other' CPUs. */
2495 PCPU_SET(other_cpus, all_cpus & ~PCPU_GET(cpumask));
2496
2497 if (bootverbose)
2498 apic_dump("ap_init()");
2499
2293 printf("SMP: AP CPU #%d Launched!\n", PCPU_GET(cpuid));
2294
2295 if (smp_cpus == mp_ncpus) {
2296 smp_started = 1; /* enable IPI's, tlb shootdown, freezes etc */
2297 smp_active = 1; /* historic */
2298 }
2299
2300 mtx_unlock_spin(&ap_boot_mtx);

--- 19 unchanged lines hidden (view full) ---

2320 * WARNING! unpend() will call statclock_process() directly and skip this
2321 * routine.
2322 */
2323void
2324forwarded_statclock(struct trapframe frame)
2325{
2326
2327 mtx_lock_spin(&sched_lock);
2500 printf("SMP: AP CPU #%d Launched!\n", PCPU_GET(cpuid));
2501
2502 if (smp_cpus == mp_ncpus) {
2503 smp_started = 1; /* enable IPI's, tlb shootdown, freezes etc */
2504 smp_active = 1; /* historic */
2505 }
2506
2507 mtx_unlock_spin(&ap_boot_mtx);

--- 19 unchanged lines hidden (view full) ---

2527 * WARNING! unpend() will call statclock_process() directly and skip this
2528 * routine.
2529 */
2530void
2531forwarded_statclock(struct trapframe frame)
2532{
2533
2534 mtx_lock_spin(&sched_lock);
2328 statclock_process(curthread->td_kse, TRAPF_PC(&frame), TRAPF_USERMODE(&frame));
2535 statclock_process(curthread->td_kse, TRAPF_PC(&frame),
2536 TRAPF_USERMODE(&frame));
2329 mtx_unlock_spin(&sched_lock);
2330}
2331
2332void
2333forward_statclock(void)
2334{
2335 int map;
2336

--- 111 unchanged lines hidden --- 		2537 mtx_unlock_spin(&sched_lock);
2538}
2539
2540void
2541forward_statclock(void)
2542{
2543 int map;
2544

--- 111 unchanged lines hidden ---