mp_x86.c (75393) | mp_x86.c (75421) |
---|---|
1/* 2 * Copyright (c) 1996, by Steve Passe 3 * All rights reserved. 4 * 5 * Redistribution and use in source and binary forms, with or without 6 * modification, are permitted provided that the following conditions 7 * are met: 8 * 1. Redistributions of source code must retain the above copyright --- 8 unchanged lines hidden (view full) --- 17 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 18 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 19 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 20 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 21 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 22 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 23 * SUCH DAMAGE. 24 * | 1/* 2 * Copyright (c) 1996, by Steve Passe 3 * All rights reserved. 4 * 5 * Redistribution and use in source and binary forms, with or without 6 * modification, are permitted provided that the following conditions 7 * are met: 8 * 1. Redistributions of source code must retain the above copyright --- 8 unchanged lines hidden (view full) --- 17 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 18 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 19 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 20 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 21 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 22 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 23 * SUCH DAMAGE. 24 * |
25 * $FreeBSD: head/sys/i386/i386/mp_machdep.c 75393 2001-04-10 21:34:13Z jhb $ | 25 * $FreeBSD: head/sys/i386/i386/mp_machdep.c 75421 2001-04-11 17:06:02Z jhb $ |
26 */ 27 28#include "opt_cpu.h" 29 30#ifdef SMP 31#include <machine/smptests.h> 32#else 33#error --- 2177 unchanged lines hidden (view full) --- 2211 * 2212 * XXX: Needs to handshake and wait for completion before proceding. 2213 */ 2214void 2215smp_invltlb(void) 2216{ 2217#if defined(APIC_IO) 2218 if (smp_started && invltlb_ok) | 26 */ 27 28#include "opt_cpu.h" 29 30#ifdef SMP 31#include <machine/smptests.h> 32#else 33#error --- 2177 unchanged lines hidden (view full) --- 2211 * 2212 * XXX: Needs to handshake and wait for completion before proceding. 2213 */ 2214void 2215smp_invltlb(void) 2216{ 2217#if defined(APIC_IO) 2218 if (smp_started && invltlb_ok) |
2219 smp_ipi_all_but_self(IPI_INVLTLB); | 2219 ipi_all_but_self(IPI_INVLTLB); |
2220#endif /* APIC_IO */ 2221} 2222 2223void 2224invlpg(u_int addr) 2225{ 2226 __asm __volatile("invlpg (%0)"::"r"(addr):"memory"); 2227 --- 257 unchanged lines hidden (view full) --- 2485 if (!smp_started || !invltlb_ok || cold || panicstr) 2486 return; 2487 2488 /* Step 1: Probe state (user, cpu, interrupt, spinlock, idle ) */ 2489 2490 map = PCPU_GET(other_cpus) & ~stopped_cpus ; 2491 checkstate_probed_cpus = 0; 2492 if (map != 0) | 2220#endif /* APIC_IO */ 2221} 2222 2223void 2224invlpg(u_int addr) 2225{ 2226 __asm __volatile("invlpg (%0)"::"r"(addr):"memory"); 2227 --- 257 unchanged lines hidden (view full) --- 2485 if (!smp_started || !invltlb_ok || cold || panicstr) 2486 return; 2487 2488 /* Step 1: Probe state (user, cpu, interrupt, spinlock, idle ) */ 2489 2490 map = PCPU_GET(other_cpus) & ~stopped_cpus ; 2491 checkstate_probed_cpus = 0; 2492 if (map != 0) |
2493 smp_ipi_selected(map, IPI_CHECKSTATE); | 2493 ipi_selected(map, IPI_CHECKSTATE); |
2494 2495 i = 0; 2496 while (checkstate_probed_cpus != map) { 2497 /* spin */ 2498 i++; 2499 if (i == 100000) { 2500#ifdef DIAGNOSTIC 2501 printf("forward_statclock: checkstate %x\n", --- 13 unchanged lines hidden (view full) --- 2515 if (id == PCPU_GET(cpuid)) 2516 continue; 2517 if (((1 << id) & checkstate_probed_cpus) == 0) 2518 continue; 2519 forwarded_statclock(id, pscnt, &map); 2520 } 2521 if (map != 0) { 2522 checkstate_need_ast |= map; | 2494 2495 i = 0; 2496 while (checkstate_probed_cpus != map) { 2497 /* spin */ 2498 i++; 2499 if (i == 100000) { 2500#ifdef DIAGNOSTIC 2501 printf("forward_statclock: checkstate %x\n", --- 13 unchanged lines hidden (view full) --- 2515 if (id == PCPU_GET(cpuid)) 2516 continue; 2517 if (((1 << id) & checkstate_probed_cpus) == 0) 2518 continue; 2519 forwarded_statclock(id, pscnt, &map); 2520 } 2521 if (map != 0) { 2522 checkstate_need_ast |= map; |
2523 smp_ipi_selected(map, IPI_AST); | 2523 ipi_selected(map, IPI_AST); |
2524 i = 0; 2525 while ((checkstate_need_ast & map) != 0) { 2526 /* spin */ 2527 i++; 2528 if (i > 100000) { 2529#ifdef DIAGNOSTIC 2530 printf("forward_statclock: dropped ast 0x%x\n", 2531 checkstate_need_ast & map); --- 29 unchanged lines hidden (view full) --- 2561 if (!smp_started || !invltlb_ok || cold || panicstr) 2562 return; 2563 2564 /* Step 1: Probe state (user, cpu, interrupt, spinlock, idle) */ 2565 2566 map = PCPU_GET(other_cpus) & ~stopped_cpus ; 2567 checkstate_probed_cpus = 0; 2568 if (map != 0) | 2524 i = 0; 2525 while ((checkstate_need_ast & map) != 0) { 2526 /* spin */ 2527 i++; 2528 if (i > 100000) { 2529#ifdef DIAGNOSTIC 2530 printf("forward_statclock: dropped ast 0x%x\n", 2531 checkstate_need_ast & map); --- 29 unchanged lines hidden (view full) --- 2561 if (!smp_started || !invltlb_ok || cold || panicstr) 2562 return; 2563 2564 /* Step 1: Probe state (user, cpu, interrupt, spinlock, idle) */ 2565 2566 map = PCPU_GET(other_cpus) & ~stopped_cpus ; 2567 checkstate_probed_cpus = 0; 2568 if (map != 0) |
2569 smp_ipi_selected(map, IPI_CHECKSTATE); | 2569 ipi_selected(map, IPI_CHECKSTATE); |
2570 2571 i = 0; 2572 while (checkstate_probed_cpus != map) { 2573 /* spin */ 2574 i++; 2575 if (i == 100000) { 2576#ifdef DIAGNOSTIC 2577 printf("forward_hardclock: checkstate %x\n", --- 31 unchanged lines hidden (view full) --- 2609 } 2610 } 2611 if (stathz == 0) { 2612 forwarded_statclock( id, pscnt, &map); 2613 } 2614 } 2615 if (map != 0) { 2616 checkstate_need_ast |= map; | 2570 2571 i = 0; 2572 while (checkstate_probed_cpus != map) { 2573 /* spin */ 2574 i++; 2575 if (i == 100000) { 2576#ifdef DIAGNOSTIC 2577 printf("forward_hardclock: checkstate %x\n", --- 31 unchanged lines hidden (view full) --- 2609 } 2610 } 2611 if (stathz == 0) { 2612 forwarded_statclock( id, pscnt, &map); 2613 } 2614 } 2615 if (map != 0) { 2616 checkstate_need_ast |= map; |
2617 smp_ipi_selected(map, IPI_AST); | 2617 ipi_selected(map, IPI_AST); |
2618 i = 0; 2619 while ((checkstate_need_ast & map) != 0) { 2620 /* spin */ 2621 i++; 2622 if (i > 100000) { 2623#ifdef DIAGNOSTIC 2624 printf("forward_hardclock: dropped ast 0x%x\n", 2625 checkstate_need_ast & map); --- 35 unchanged lines hidden (view full) --- 2661 return; 2662 } 2663 id = p->p_oncpu; 2664 mtx_unlock_spin(&sched_lock); 2665 if (id == 0xff) 2666 return; 2667 map = (1<<id); 2668 checkstate_need_ast |= map; | 2618 i = 0; 2619 while ((checkstate_need_ast & map) != 0) { 2620 /* spin */ 2621 i++; 2622 if (i > 100000) { 2623#ifdef DIAGNOSTIC 2624 printf("forward_hardclock: dropped ast 0x%x\n", 2625 checkstate_need_ast & map); --- 35 unchanged lines hidden (view full) --- 2661 return; 2662 } 2663 id = p->p_oncpu; 2664 mtx_unlock_spin(&sched_lock); 2665 if (id == 0xff) 2666 return; 2667 map = (1<<id); 2668 checkstate_need_ast |= map; |
2669 smp_ipi_selected(map, IPI_AST); | 2669 ipi_selected(map, IPI_AST); |
2670 i = 0; 2671 while ((checkstate_need_ast & map) != 0) { 2672 /* spin */ 2673 i++; 2674 if (i > 100000) { 2675#if 0 2676 printf("forward_signal: dropped ast 0x%x\n", 2677 checkstate_need_ast & map); --- 19 unchanged lines hidden (view full) --- 2697 2698 if (!smp_started || !invltlb_ok || cold || panicstr) 2699 return; 2700 if (!forward_roundrobin_enabled) 2701 return; 2702 resched_cpus |= PCPU_GET(other_cpus); 2703 map = PCPU_GET(other_cpus) & ~stopped_cpus ; 2704#if 1 | 2670 i = 0; 2671 while ((checkstate_need_ast & map) != 0) { 2672 /* spin */ 2673 i++; 2674 if (i > 100000) { 2675#if 0 2676 printf("forward_signal: dropped ast 0x%x\n", 2677 checkstate_need_ast & map); --- 19 unchanged lines hidden (view full) --- 2697 2698 if (!smp_started || !invltlb_ok || cold || panicstr) 2699 return; 2700 if (!forward_roundrobin_enabled) 2701 return; 2702 resched_cpus |= PCPU_GET(other_cpus); 2703 map = PCPU_GET(other_cpus) & ~stopped_cpus ; 2704#if 1 |
2705 smp_ipi_selected(map, IPI_AST); | 2705 ipi_selected(map, IPI_AST); |
2706#else | 2706#else |
2707 smp_ipi_all_but_self(IPI_AST); | 2707 ipi_all_but_self(IPI_AST); |
2708#endif 2709 i = 0; 2710 while ((checkstate_need_ast & map) != 0) { 2711 /* spin */ 2712 i++; 2713 if (i > 100000) { 2714#if 0 2715 printf("forward_roundrobin: dropped ast 0x%x\n", --- 25 unchanged lines hidden (view full) --- 2741stop_cpus(u_int map) 2742{ 2743 int count = 0; 2744 2745 if (!smp_started) 2746 return 0; 2747 2748 /* send the Xcpustop IPI to all CPUs in map */ | 2708#endif 2709 i = 0; 2710 while ((checkstate_need_ast & map) != 0) { 2711 /* spin */ 2712 i++; 2713 if (i > 100000) { 2714#if 0 2715 printf("forward_roundrobin: dropped ast 0x%x\n", --- 25 unchanged lines hidden (view full) --- 2741stop_cpus(u_int map) 2742{ 2743 int count = 0; 2744 2745 if (!smp_started) 2746 return 0; 2747 2748 /* send the Xcpustop IPI to all CPUs in map */ |
2749 smp_ipi_selected(map, IPI_STOP); | 2749 ipi_selected(map, IPI_STOP); |
2750 2751 while (count++ < 100000 && (stopped_cpus & map) != map) 2752 /* spin */ ; 2753 2754#ifdef DIAGNOSTIC 2755 if ((stopped_cpus & map) != map) 2756 printf("Warning: CPUs 0x%x did not stop!\n", 2757 (~(stopped_cpus & map)) & map); --- 109 unchanged lines hidden (view full) --- 2867 smp_rv_teardown_func = teardown_func; 2868 smp_rv_func_arg = arg; 2869 smp_rv_waiters[0] = 0; 2870 smp_rv_waiters[1] = 0; 2871 2872 /* 2873 * signal other processors, which will enter the IPI with interrupts off 2874 */ | 2750 2751 while (count++ < 100000 && (stopped_cpus & map) != map) 2752 /* spin */ ; 2753 2754#ifdef DIAGNOSTIC 2755 if ((stopped_cpus & map) != map) 2756 printf("Warning: CPUs 0x%x did not stop!\n", 2757 (~(stopped_cpus & map)) & map); --- 109 unchanged lines hidden (view full) --- 2867 smp_rv_teardown_func = teardown_func; 2868 smp_rv_func_arg = arg; 2869 smp_rv_waiters[0] = 0; 2870 smp_rv_waiters[1] = 0; 2871 2872 /* 2873 * signal other processors, which will enter the IPI with interrupts off 2874 */ |
2875 smp_ipi_all_but_self(IPI_RENDEZVOUS); | 2875 ipi_all_but_self(IPI_RENDEZVOUS); |
2876 2877 /* call executor function */ 2878 smp_rendezvous_action(); 2879 2880 /* release lock */ 2881 mtx_unlock_spin(&smp_rv_mtx); 2882} 2883 2884/* 2885 * send an IPI to a set of cpus. 2886 */ 2887void | 2876 2877 /* call executor function */ 2878 smp_rendezvous_action(); 2879 2880 /* release lock */ 2881 mtx_unlock_spin(&smp_rv_mtx); 2882} 2883 2884/* 2885 * send an IPI to a set of cpus. 2886 */ 2887void |
2888smp_ipi_selected(u_int32_t cpus, u_int ipi) | 2888ipi_selected(u_int32_t cpus, u_int ipi) |
2889{ 2890 2891 CTR2(KTR_SMP, __func__ ": cpus: %x ipi: %x", cpus, ipi); 2892 selected_apic_ipi(cpus, ipi, APIC_DELMODE_FIXED); 2893} 2894 2895/* 2896 * send an IPI INTerrupt containing 'vector' to all CPUs, including myself 2897 */ 2898void | 2889{ 2890 2891 CTR2(KTR_SMP, __func__ ": cpus: %x ipi: %x", cpus, ipi); 2892 selected_apic_ipi(cpus, ipi, APIC_DELMODE_FIXED); 2893} 2894 2895/* 2896 * send an IPI INTerrupt containing 'vector' to all CPUs, including myself 2897 */ 2898void |
2899smp_ipi_all(u_int ipi) | 2899ipi_all(u_int ipi) |
2900{ 2901 2902 CTR1(KTR_SMP, __func__ ": ipi: %x", ipi); 2903 apic_ipi(APIC_DEST_ALLISELF, ipi, APIC_DELMODE_FIXED); 2904} 2905 2906/* 2907 * send an IPI to all CPUs EXCEPT myself 2908 */ 2909void | 2900{ 2901 2902 CTR1(KTR_SMP, __func__ ": ipi: %x", ipi); 2903 apic_ipi(APIC_DEST_ALLISELF, ipi, APIC_DELMODE_FIXED); 2904} 2905 2906/* 2907 * send an IPI to all CPUs EXCEPT myself 2908 */ 2909void |
2910smp_ipi_all_but_self(u_int ipi) | 2910ipi_all_but_self(u_int ipi) |
2911{ 2912 2913 CTR1(KTR_SMP, __func__ ": ipi: %x", ipi); 2914 apic_ipi(APIC_DEST_ALLESELF, ipi, APIC_DELMODE_FIXED); 2915} 2916 2917/* 2918 * send an IPI to myself 2919 */ 2920void | 2911{ 2912 2913 CTR1(KTR_SMP, __func__ ": ipi: %x", ipi); 2914 apic_ipi(APIC_DEST_ALLESELF, ipi, APIC_DELMODE_FIXED); 2915} 2916 2917/* 2918 * send an IPI to myself 2919 */ 2920void |
2921smp_ipi_self(u_int ipi) | 2921ipi_self(u_int ipi) |
2922{ 2923 2924 CTR1(KTR_SMP, __func__ ": ipi: %x", ipi); 2925 apic_ipi(APIC_DEST_SELF, ipi, APIC_DELMODE_FIXED); 2926} 2927 2928void 2929release_aps(void *dummy __unused) 2930{ 2931 atomic_store_rel_int(&aps_ready, 1); 2932} 2933 2934SYSINIT(start_aps, SI_SUB_SMP, SI_ORDER_FIRST, release_aps, NULL); | 2922{ 2923 2924 CTR1(KTR_SMP, __func__ ": ipi: %x", ipi); 2925 apic_ipi(APIC_DEST_SELF, ipi, APIC_DELMODE_FIXED); 2926} 2927 2928void 2929release_aps(void *dummy __unused) 2930{ 2931 atomic_store_rel_int(&aps_ready, 1); 2932} 2933 2934SYSINIT(start_aps, SI_SUB_SMP, SI_ORDER_FIRST, release_aps, NULL); |