Deleted Added
sdiff udiff text old ( 75393 ) new ( 75421 )
full compact
mp_machdep.c (75393) mp_machdep.c (75421)
1/*
2 * Copyright (c) 1996, by Steve Passe
3 * All rights reserved.
4 *
5 * Redistribution and use in source and binary forms, with or without
6 * modification, are permitted provided that the following conditions
7 * are met:
8 * 1. Redistributions of source code must retain the above copyright

--- 8 unchanged lines hidden (view full) ---

17 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
18 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
19 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
20 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
21 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
22 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
23 * SUCH DAMAGE.
24 *
1/*
2 * Copyright (c) 1996, by Steve Passe
3 * All rights reserved.
4 *
5 * Redistribution and use in source and binary forms, with or without
6 * modification, are permitted provided that the following conditions
7 * are met:
8 * 1. Redistributions of source code must retain the above copyright

--- 8 unchanged lines hidden (view full) ---

17 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
18 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
19 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
20 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
21 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
22 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
23 * SUCH DAMAGE.
24 *
25 * $FreeBSD: head/sys/amd64/amd64/mp_machdep.c 75393 2001-04-10 21:34:13Z jhb $
25 * $FreeBSD: head/sys/amd64/amd64/mp_machdep.c 75421 2001-04-11 17:06:02Z jhb $
26 */
27
28#include "opt_cpu.h"
29
30#ifdef SMP
31#include <machine/smptests.h>
32#else
33#error

--- 2177 unchanged lines hidden (view full) ---

2211 *
2212 * XXX: Needs to handshake and wait for completion before proceding.
2213 */
2214void
2215smp_invltlb(void)
2216{
2217#if defined(APIC_IO)
2218 if (smp_started && invltlb_ok)
26 */
27
28#include "opt_cpu.h"
29
30#ifdef SMP
31#include <machine/smptests.h>
32#else
33#error

--- 2177 unchanged lines hidden (view full) ---

2211 *
2212 * XXX: Needs to handshake and wait for completion before proceding.
2213 */
2214void
2215smp_invltlb(void)
2216{
2217#if defined(APIC_IO)
2218 if (smp_started && invltlb_ok)
2219 smp_ipi_all_but_self(IPI_INVLTLB);
2219 ipi_all_but_self(IPI_INVLTLB);
2220#endif /* APIC_IO */
2221}
2222
2223void
2224invlpg(u_int addr)
2225{
2226 __asm __volatile("invlpg (%0)"::"r"(addr):"memory");
2227

--- 257 unchanged lines hidden (view full) ---

2485 if (!smp_started || !invltlb_ok || cold || panicstr)
2486 return;
2487
2488 /* Step 1: Probe state (user, cpu, interrupt, spinlock, idle ) */
2489
2490 map = PCPU_GET(other_cpus) & ~stopped_cpus ;
2491 checkstate_probed_cpus = 0;
2492 if (map != 0)
2220#endif /* APIC_IO */
2221}
2222
2223void
2224invlpg(u_int addr)
2225{
2226 __asm __volatile("invlpg (%0)"::"r"(addr):"memory");
2227

--- 257 unchanged lines hidden (view full) ---

2485 if (!smp_started || !invltlb_ok || cold || panicstr)
2486 return;
2487
2488 /* Step 1: Probe state (user, cpu, interrupt, spinlock, idle ) */
2489
2490 map = PCPU_GET(other_cpus) & ~stopped_cpus ;
2491 checkstate_probed_cpus = 0;
2492 if (map != 0)
2493 smp_ipi_selected(map, IPI_CHECKSTATE);
2493 ipi_selected(map, IPI_CHECKSTATE);
2494
2495 i = 0;
2496 while (checkstate_probed_cpus != map) {
2497 /* spin */
2498 i++;
2499 if (i == 100000) {
2500#ifdef DIAGNOSTIC
2501 printf("forward_statclock: checkstate %x\n",

--- 13 unchanged lines hidden (view full) ---

2515 if (id == PCPU_GET(cpuid))
2516 continue;
2517 if (((1 << id) & checkstate_probed_cpus) == 0)
2518 continue;
2519 forwarded_statclock(id, pscnt, &map);
2520 }
2521 if (map != 0) {
2522 checkstate_need_ast |= map;
2494
2495 i = 0;
2496 while (checkstate_probed_cpus != map) {
2497 /* spin */
2498 i++;
2499 if (i == 100000) {
2500#ifdef DIAGNOSTIC
2501 printf("forward_statclock: checkstate %x\n",

--- 13 unchanged lines hidden (view full) ---

2515 if (id == PCPU_GET(cpuid))
2516 continue;
2517 if (((1 << id) & checkstate_probed_cpus) == 0)
2518 continue;
2519 forwarded_statclock(id, pscnt, &map);
2520 }
2521 if (map != 0) {
2522 checkstate_need_ast |= map;
2523 smp_ipi_selected(map, IPI_AST);
2523 ipi_selected(map, IPI_AST);
2524 i = 0;
2525 while ((checkstate_need_ast & map) != 0) {
2526 /* spin */
2527 i++;
2528 if (i > 100000) {
2529#ifdef DIAGNOSTIC
2530 printf("forward_statclock: dropped ast 0x%x\n",
2531 checkstate_need_ast & map);

--- 29 unchanged lines hidden (view full) ---

2561 if (!smp_started || !invltlb_ok || cold || panicstr)
2562 return;
2563
2564 /* Step 1: Probe state (user, cpu, interrupt, spinlock, idle) */
2565
2566 map = PCPU_GET(other_cpus) & ~stopped_cpus ;
2567 checkstate_probed_cpus = 0;
2568 if (map != 0)
2524 i = 0;
2525 while ((checkstate_need_ast & map) != 0) {
2526 /* spin */
2527 i++;
2528 if (i > 100000) {
2529#ifdef DIAGNOSTIC
2530 printf("forward_statclock: dropped ast 0x%x\n",
2531 checkstate_need_ast & map);

--- 29 unchanged lines hidden (view full) ---

2561 if (!smp_started || !invltlb_ok || cold || panicstr)
2562 return;
2563
2564 /* Step 1: Probe state (user, cpu, interrupt, spinlock, idle) */
2565
2566 map = PCPU_GET(other_cpus) & ~stopped_cpus ;
2567 checkstate_probed_cpus = 0;
2568 if (map != 0)
2569 smp_ipi_selected(map, IPI_CHECKSTATE);
2569 ipi_selected(map, IPI_CHECKSTATE);
2570
2571 i = 0;
2572 while (checkstate_probed_cpus != map) {
2573 /* spin */
2574 i++;
2575 if (i == 100000) {
2576#ifdef DIAGNOSTIC
2577 printf("forward_hardclock: checkstate %x\n",

--- 31 unchanged lines hidden (view full) ---

2609 }
2610 }
2611 if (stathz == 0) {
2612 forwarded_statclock( id, pscnt, &map);
2613 }
2614 }
2615 if (map != 0) {
2616 checkstate_need_ast |= map;
2570
2571 i = 0;
2572 while (checkstate_probed_cpus != map) {
2573 /* spin */
2574 i++;
2575 if (i == 100000) {
2576#ifdef DIAGNOSTIC
2577 printf("forward_hardclock: checkstate %x\n",

--- 31 unchanged lines hidden (view full) ---

2609 }
2610 }
2611 if (stathz == 0) {
2612 forwarded_statclock( id, pscnt, &map);
2613 }
2614 }
2615 if (map != 0) {
2616 checkstate_need_ast |= map;
2617 smp_ipi_selected(map, IPI_AST);
2617 ipi_selected(map, IPI_AST);
2618 i = 0;
2619 while ((checkstate_need_ast & map) != 0) {
2620 /* spin */
2621 i++;
2622 if (i > 100000) {
2623#ifdef DIAGNOSTIC
2624 printf("forward_hardclock: dropped ast 0x%x\n",
2625 checkstate_need_ast & map);

--- 35 unchanged lines hidden (view full) ---

2661 return;
2662 }
2663 id = p->p_oncpu;
2664 mtx_unlock_spin(&sched_lock);
2665 if (id == 0xff)
2666 return;
2667 map = (1<<id);
2668 checkstate_need_ast |= map;
2618 i = 0;
2619 while ((checkstate_need_ast & map) != 0) {
2620 /* spin */
2621 i++;
2622 if (i > 100000) {
2623#ifdef DIAGNOSTIC
2624 printf("forward_hardclock: dropped ast 0x%x\n",
2625 checkstate_need_ast & map);

--- 35 unchanged lines hidden (view full) ---

2661 return;
2662 }
2663 id = p->p_oncpu;
2664 mtx_unlock_spin(&sched_lock);
2665 if (id == 0xff)
2666 return;
2667 map = (1<<id);
2668 checkstate_need_ast |= map;
2669 smp_ipi_selected(map, IPI_AST);
2669 ipi_selected(map, IPI_AST);
2670 i = 0;
2671 while ((checkstate_need_ast & map) != 0) {
2672 /* spin */
2673 i++;
2674 if (i > 100000) {
2675#if 0
2676 printf("forward_signal: dropped ast 0x%x\n",
2677 checkstate_need_ast & map);

--- 19 unchanged lines hidden (view full) ---

2697
2698 if (!smp_started || !invltlb_ok || cold || panicstr)
2699 return;
2700 if (!forward_roundrobin_enabled)
2701 return;
2702 resched_cpus |= PCPU_GET(other_cpus);
2703 map = PCPU_GET(other_cpus) & ~stopped_cpus ;
2704#if 1
2670 i = 0;
2671 while ((checkstate_need_ast & map) != 0) {
2672 /* spin */
2673 i++;
2674 if (i > 100000) {
2675#if 0
2676 printf("forward_signal: dropped ast 0x%x\n",
2677 checkstate_need_ast & map);

--- 19 unchanged lines hidden (view full) ---

2697
2698 if (!smp_started || !invltlb_ok || cold || panicstr)
2699 return;
2700 if (!forward_roundrobin_enabled)
2701 return;
2702 resched_cpus |= PCPU_GET(other_cpus);
2703 map = PCPU_GET(other_cpus) & ~stopped_cpus ;
2704#if 1
2705 smp_ipi_selected(map, IPI_AST);
2705 ipi_selected(map, IPI_AST);
2706#else
2706#else
2707 smp_ipi_all_but_self(IPI_AST);
2707 ipi_all_but_self(IPI_AST);
2708#endif
2709 i = 0;
2710 while ((checkstate_need_ast & map) != 0) {
2711 /* spin */
2712 i++;
2713 if (i > 100000) {
2714#if 0
2715 printf("forward_roundrobin: dropped ast 0x%x\n",

--- 25 unchanged lines hidden (view full) ---

2741stop_cpus(u_int map)
2742{
2743 int count = 0;
2744
2745 if (!smp_started)
2746 return 0;
2747
2748 /* send the Xcpustop IPI to all CPUs in map */
2708#endif
2709 i = 0;
2710 while ((checkstate_need_ast & map) != 0) {
2711 /* spin */
2712 i++;
2713 if (i > 100000) {
2714#if 0
2715 printf("forward_roundrobin: dropped ast 0x%x\n",

--- 25 unchanged lines hidden (view full) ---

2741stop_cpus(u_int map)
2742{
2743 int count = 0;
2744
2745 if (!smp_started)
2746 return 0;
2747
2748 /* send the Xcpustop IPI to all CPUs in map */
2749 smp_ipi_selected(map, IPI_STOP);
2749 ipi_selected(map, IPI_STOP);
2750
2751 while (count++ < 100000 && (stopped_cpus & map) != map)
2752 /* spin */ ;
2753
2754#ifdef DIAGNOSTIC
2755 if ((stopped_cpus & map) != map)
2756 printf("Warning: CPUs 0x%x did not stop!\n",
2757 (~(stopped_cpus & map)) & map);

--- 109 unchanged lines hidden (view full) ---

2867 smp_rv_teardown_func = teardown_func;
2868 smp_rv_func_arg = arg;
2869 smp_rv_waiters[0] = 0;
2870 smp_rv_waiters[1] = 0;
2871
2872 /*
2873 * signal other processors, which will enter the IPI with interrupts off
2874 */
2750
2751 while (count++ < 100000 && (stopped_cpus & map) != map)
2752 /* spin */ ;
2753
2754#ifdef DIAGNOSTIC
2755 if ((stopped_cpus & map) != map)
2756 printf("Warning: CPUs 0x%x did not stop!\n",
2757 (~(stopped_cpus & map)) & map);

--- 109 unchanged lines hidden (view full) ---

2867 smp_rv_teardown_func = teardown_func;
2868 smp_rv_func_arg = arg;
2869 smp_rv_waiters[0] = 0;
2870 smp_rv_waiters[1] = 0;
2871
2872 /*
2873 * signal other processors, which will enter the IPI with interrupts off
2874 */
2875 smp_ipi_all_but_self(IPI_RENDEZVOUS);
2875 ipi_all_but_self(IPI_RENDEZVOUS);
2876
2877 /* call executor function */
2878 smp_rendezvous_action();
2879
2880 /* release lock */
2881 mtx_unlock_spin(&smp_rv_mtx);
2882}
2883
2884/*
2885 * send an IPI to a set of cpus.
2886 */
2887void
2876
2877 /* call executor function */
2878 smp_rendezvous_action();
2879
2880 /* release lock */
2881 mtx_unlock_spin(&smp_rv_mtx);
2882}
2883
2884/*
2885 * send an IPI to a set of cpus.
2886 */
2887void
2888smp_ipi_selected(u_int32_t cpus, u_int ipi)
2888ipi_selected(u_int32_t cpus, u_int ipi)
2889{
2890
2891 CTR2(KTR_SMP, __func__ ": cpus: %x ipi: %x", cpus, ipi);
2892 selected_apic_ipi(cpus, ipi, APIC_DELMODE_FIXED);
2893}
2894
2895/*
2896 * send an IPI INTerrupt containing 'vector' to all CPUs, including myself
2897 */
2898void
2889{
2890
2891 CTR2(KTR_SMP, __func__ ": cpus: %x ipi: %x", cpus, ipi);
2892 selected_apic_ipi(cpus, ipi, APIC_DELMODE_FIXED);
2893}
2894
2895/*
2896 * send an IPI INTerrupt containing 'vector' to all CPUs, including myself
2897 */
2898void
2899smp_ipi_all(u_int ipi)
2899ipi_all(u_int ipi)
2900{
2901
2902 CTR1(KTR_SMP, __func__ ": ipi: %x", ipi);
2903 apic_ipi(APIC_DEST_ALLISELF, ipi, APIC_DELMODE_FIXED);
2904}
2905
2906/*
2907 * send an IPI to all CPUs EXCEPT myself
2908 */
2909void
2900{
2901
2902 CTR1(KTR_SMP, __func__ ": ipi: %x", ipi);
2903 apic_ipi(APIC_DEST_ALLISELF, ipi, APIC_DELMODE_FIXED);
2904}
2905
2906/*
2907 * send an IPI to all CPUs EXCEPT myself
2908 */
2909void
2910smp_ipi_all_but_self(u_int ipi)
2910ipi_all_but_self(u_int ipi)
2911{
2912
2913 CTR1(KTR_SMP, __func__ ": ipi: %x", ipi);
2914 apic_ipi(APIC_DEST_ALLESELF, ipi, APIC_DELMODE_FIXED);
2915}
2916
2917/*
2918 * send an IPI to myself
2919 */
2920void
2911{
2912
2913 CTR1(KTR_SMP, __func__ ": ipi: %x", ipi);
2914 apic_ipi(APIC_DEST_ALLESELF, ipi, APIC_DELMODE_FIXED);
2915}
2916
2917/*
2918 * send an IPI to myself
2919 */
2920void
2921smp_ipi_self(u_int ipi)
2921ipi_self(u_int ipi)
2922{
2923
2924 CTR1(KTR_SMP, __func__ ": ipi: %x", ipi);
2925 apic_ipi(APIC_DEST_SELF, ipi, APIC_DELMODE_FIXED);
2926}
2927
2928void
2929release_aps(void *dummy __unused)
2930{
2931 atomic_store_rel_int(&aps_ready, 1);
2932}
2933
2934SYSINIT(start_aps, SI_SUB_SMP, SI_ORDER_FIRST, release_aps, NULL);
2922{
2923
2924 CTR1(KTR_SMP, __func__ ": ipi: %x", ipi);
2925 apic_ipi(APIC_DEST_SELF, ipi, APIC_DELMODE_FIXED);
2926}
2927
2928void
2929release_aps(void *dummy __unused)
2930{
2931 atomic_store_rel_int(&aps_ready, 1);
2932}
2933
2934SYSINIT(start_aps, SI_SUB_SMP, SI_ORDER_FIRST, release_aps, NULL);