Deleted Added
sdiff udiff text old ( 211924 ) new ( 212541 )
full compact
machdep.c (211924) machdep.c (212541)
1/*-
2 * Copyright (c) 2003 Peter Wemm.
3 * Copyright (c) 1992 Terrence R. Lambert.
4 * Copyright (c) 1982, 1987, 1990 The Regents of the University of California.
5 * All rights reserved.
6 *
7 * This code is derived from software contributed to Berkeley by
8 * William Jolitz.

--- 25 unchanged lines hidden (view full) ---

34 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
35 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
36 * SUCH DAMAGE.
37 *
38 * from: @(#)machdep.c 7.4 (Berkeley) 6/3/91
39 */
40
41#include <sys/cdefs.h>
1/*-
2 * Copyright (c) 2003 Peter Wemm.
3 * Copyright (c) 1992 Terrence R. Lambert.
4 * Copyright (c) 1982, 1987, 1990 The Regents of the University of California.
5 * All rights reserved.
6 *
7 * This code is derived from software contributed to Berkeley by
8 * William Jolitz.

--- 25 unchanged lines hidden (view full) ---

34 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
35 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
36 * SUCH DAMAGE.
37 *
38 * from: @(#)machdep.c 7.4 (Berkeley) 6/3/91
39 */
40
41#include <sys/cdefs.h>
42__FBSDID("$FreeBSD: head/sys/amd64/amd64/machdep.c 211924 2010-08-28 08:03:29Z rpaulo $");
42__FBSDID("$FreeBSD: head/sys/amd64/amd64/machdep.c 212541 2010-09-13 07:25:35Z mav $");
43
44#include "opt_atalk.h"
45#include "opt_atpic.h"
46#include "opt_compat.h"
47#include "opt_cpu.h"
48#include "opt_ddb.h"
49#include "opt_inet.h"
50#include "opt_ipx.h"

--- 529 unchanged lines hidden (view full) ---

580void
581cpu_halt(void)
582{
583 for (;;)
584 __asm__ ("hlt");
585}
586
587void (*cpu_idle_hook)(void) = NULL; /* ACPI idle hook. */
43
44#include "opt_atalk.h"
45#include "opt_atpic.h"
46#include "opt_compat.h"
47#include "opt_cpu.h"
48#include "opt_ddb.h"
49#include "opt_inet.h"
50#include "opt_ipx.h"

--- 529 unchanged lines hidden (view full) ---

580void
581cpu_halt(void)
582{
583 for (;;)
584 __asm__ ("hlt");
585}
586
587void (*cpu_idle_hook)(void) = NULL; /* ACPI idle hook. */
588static int cpu_ident_amdc1e = 0; /* AMD C1E supported. */
589static int idle_mwait = 1; /* Use MONITOR/MWAIT for short idle. */
590TUNABLE_INT("machdep.idle_mwait", &idle_mwait);
591SYSCTL_INT(_machdep, OID_AUTO, idle_mwait, CTLFLAG_RW, &idle_mwait,
592 0, "Use MONITOR/MWAIT for short idle");
588
593
594#define STATE_RUNNING 0x0
595#define STATE_MWAIT 0x1
596#define STATE_SLEEPING 0x2
597
589static void
598static void
590cpu_idle_hlt(int busy)
599cpu_idle_acpi(int busy)
591{
600{
592 /*
593 * we must absolutely guarentee that hlt is the next instruction
594 * after sti or we introduce a timing window.
595 */
601 int *state;
602
603 state = (int *)PCPU_PTR(monitorbuf);
604 *state = STATE_SLEEPING;
596 disable_intr();
605 disable_intr();
597 if (sched_runnable())
606 if (sched_runnable())
598 enable_intr();
607 enable_intr();
608 else if (cpu_idle_hook)
609 cpu_idle_hook();
599 else
600 __asm __volatile("sti; hlt");
610 else
611 __asm __volatile("sti; hlt");
612 *state = STATE_RUNNING;
601}
602
603static void
613}
614
615static void
604cpu_idle_acpi(int busy)
616cpu_idle_hlt(int busy)
605{
617{
618 int *state;
619
620 state = (int *)PCPU_PTR(monitorbuf);
621 *state = STATE_SLEEPING;
622 /*
623 * We must absolutely guarentee that hlt is the next instruction
624 * after sti or we introduce a timing window.
625 */
606 disable_intr();
626 disable_intr();
607 if (sched_runnable())
627 if (sched_runnable())
608 enable_intr();
628 enable_intr();
609 else if (cpu_idle_hook)
610 cpu_idle_hook();
611 else
612 __asm __volatile("sti; hlt");
629 else
630 __asm __volatile("sti; hlt");
631 *state = STATE_RUNNING;
613}
614
632}
633
615static int cpu_ident_amdc1e = 0;
634/*
635 * MWAIT cpu power states. Lower 4 bits are sub-states.
636 */
637#define MWAIT_C0 0xf0
638#define MWAIT_C1 0x00
639#define MWAIT_C2 0x10
640#define MWAIT_C3 0x20
641#define MWAIT_C4 0x30
616
642
617static int
618cpu_probe_amdc1e(void)
643static void
644cpu_idle_mwait(int busy)
619{
645{
620 int i;
646 int *state;
621
647
622 /*
623 * Forget it, if we're not using local APIC timer.
624 */
625 if (resource_disabled("apic", 0) ||
626 (resource_int_value("apic", 0, "clock", &i) == 0 && i == 0))
627 return (0);
628
629 /*
630 * Detect the presence of C1E capability mostly on latest
631 * dual-cores (or future) k8 family.
632 */
633 if (cpu_vendor_id == CPU_VENDOR_AMD &&
634 (cpu_id & 0x00000f00) == 0x00000f00 &&
635 (cpu_id & 0x0fff0000) >= 0x00040000) {
636 cpu_ident_amdc1e = 1;
637 return (1);
648 state = (int *)PCPU_PTR(monitorbuf);
649 *state = STATE_MWAIT;
650 if (!sched_runnable()) {
651 cpu_monitor(state, 0, 0);
652 if (*state == STATE_MWAIT)
653 cpu_mwait(0, MWAIT_C1);
638 }
654 }
655 *state = STATE_RUNNING;
656}
639
657
640 return (0);
658static void
659cpu_idle_spin(int busy)
660{
661 int *state;
662 int i;
663
664 state = (int *)PCPU_PTR(monitorbuf);
665 *state = STATE_RUNNING;
666 for (i = 0; i < 1000; i++) {
667 if (sched_runnable())
668 return;
669 cpu_spinwait();
670 }
641}
642
643/*
644 * C1E renders the local APIC timer dead, so we disable it by
645 * reading the Interrupt Pending Message register and clearing
646 * both C1eOnCmpHalt (bit 28) and SmiOnCmpHalt (bit 27).
647 *
648 * Reference:
649 * "BIOS and Kernel Developer's Guide for AMD NPT Family 0Fh Processors"
650 * #32559 revision 3.00+
651 */
652#define MSR_AMDK8_IPM 0xc0010055
653#define AMDK8_SMIONCMPHALT (1ULL << 27)
654#define AMDK8_C1EONCMPHALT (1ULL << 28)
655#define AMDK8_CMPHALT (AMDK8_SMIONCMPHALT | AMDK8_C1EONCMPHALT)
656
657static void
671}
672
673/*
674 * C1E renders the local APIC timer dead, so we disable it by
675 * reading the Interrupt Pending Message register and clearing
676 * both C1eOnCmpHalt (bit 28) and SmiOnCmpHalt (bit 27).
677 *
678 * Reference:
679 * "BIOS and Kernel Developer's Guide for AMD NPT Family 0Fh Processors"
680 * #32559 revision 3.00+
681 */
682#define MSR_AMDK8_IPM 0xc0010055
683#define AMDK8_SMIONCMPHALT (1ULL << 27)
684#define AMDK8_C1EONCMPHALT (1ULL << 28)
685#define AMDK8_CMPHALT (AMDK8_SMIONCMPHALT | AMDK8_C1EONCMPHALT)
686
687static void
658cpu_idle_amdc1e(int busy)
688cpu_probe_amdc1e(void)
659{
660
689{
690
661 disable_intr();
662 if (sched_runnable())
663 enable_intr();
664 else {
665 uint64_t msr;
666
667 msr = rdmsr(MSR_AMDK8_IPM);
668 if (msr & AMDK8_CMPHALT)
669 wrmsr(MSR_AMDK8_IPM, msr & ~AMDK8_CMPHALT);
670
671 if (cpu_idle_hook)
672 cpu_idle_hook();
673 else
674 __asm __volatile("sti; hlt");
691 /*
692 * Detect the presence of C1E capability mostly on latest
693 * dual-cores (or future) k8 family.
694 */
695 if (cpu_vendor_id == CPU_VENDOR_AMD &&
696 (cpu_id & 0x00000f00) == 0x00000f00 &&
697 (cpu_id & 0x0fff0000) >= 0x00040000) {
698 cpu_ident_amdc1e = 1;
675 }
676}
677
699 }
700}
701
678static void
679cpu_idle_spin(int busy)
680{
681 return;
682}
683
684void (*cpu_idle_fn)(int) = cpu_idle_acpi;
685
686void
687cpu_idle(int busy)
688{
702void (*cpu_idle_fn)(int) = cpu_idle_acpi;
703
704void
705cpu_idle(int busy)
706{
707 uint64_t msr;
708
709 CTR2(KTR_SPARE2, "cpu_idle(%d) at %d",
710 busy, curcpu);
689#ifdef SMP
690 if (mp_grab_cpu_hlt())
691 return;
692#endif
711#ifdef SMP
712 if (mp_grab_cpu_hlt())
713 return;
714#endif
693 cpu_idle_fn(busy);
694}
715 /* If we are busy - try to use fast methods. */
716 if (busy) {
717 if ((cpu_feature2 & CPUID2_MON) && idle_mwait) {
718 cpu_idle_mwait(busy);
719 goto out;
720 }
721 }
695
722
696/*
697 * mwait cpu power states. Lower 4 bits are sub-states.
698 */
699#define MWAIT_C0 0xf0
700#define MWAIT_C1 0x00
701#define MWAIT_C2 0x10
702#define MWAIT_C3 0x20
703#define MWAIT_C4 0x30
723 /* If we have time - switch timers into idle mode. */
724 if (!busy) {
725 critical_enter();
726 cpu_idleclock();
727 }
704
728
705#define MWAIT_DISABLED 0x0
706#define MWAIT_WOKEN 0x1
707#define MWAIT_WAITING 0x2
729 /* Apply AMD APIC timer C1E workaround. */
730 if (cpu_ident_amdc1e && cpu_disable_deep_sleep) {
731 msr = rdmsr(MSR_AMDK8_IPM);
732 if (msr & AMDK8_CMPHALT)
733 wrmsr(MSR_AMDK8_IPM, msr & ~AMDK8_CMPHALT);
734 }
708
735
709static void
710cpu_idle_mwait(int busy)
711{
712 int *mwait;
736 /* Call main idle method. */
737 cpu_idle_fn(busy);
713
738
714 mwait = (int *)PCPU_PTR(monitorbuf);
715 *mwait = MWAIT_WAITING;
716 if (sched_runnable())
717 return;
718 cpu_monitor(mwait, 0, 0);
719 if (*mwait == MWAIT_WAITING)
720 cpu_mwait(0, MWAIT_C1);
721}
722
723static void
724cpu_idle_mwait_hlt(int busy)
725{
726 int *mwait;
727
728 mwait = (int *)PCPU_PTR(monitorbuf);
729 if (busy == 0) {
730 *mwait = MWAIT_DISABLED;
731 cpu_idle_hlt(busy);
732 return;
739 /* Switch timers mack into active mode. */
740 if (!busy) {
741 cpu_activeclock();
742 critical_exit();
733 }
743 }
734 *mwait = MWAIT_WAITING;
735 if (sched_runnable())
736 return;
737 cpu_monitor(mwait, 0, 0);
738 if (*mwait == MWAIT_WAITING)
739 cpu_mwait(0, MWAIT_C1);
744out:
745 CTR2(KTR_SPARE2, "cpu_idle(%d) at %d done",
746 busy, curcpu);
740}
741
742int
743cpu_idle_wakeup(int cpu)
744{
745 struct pcpu *pcpu;
747}
748
749int
750cpu_idle_wakeup(int cpu)
751{
752 struct pcpu *pcpu;
746 int *mwait;
753 int *state;
747
754
748 if (cpu_idle_fn == cpu_idle_spin)
749 return (1);
750 if (cpu_idle_fn != cpu_idle_mwait && cpu_idle_fn != cpu_idle_mwait_hlt)
751 return (0);
752 pcpu = pcpu_find(cpu);
755 pcpu = pcpu_find(cpu);
753 mwait = (int *)pcpu->pc_monitorbuf;
756 state = (int *)pcpu->pc_monitorbuf;
754 /*
755 * This doesn't need to be atomic since missing the race will
756 * simply result in unnecessary IPIs.
757 */
757 /*
758 * This doesn't need to be atomic since missing the race will
759 * simply result in unnecessary IPIs.
760 */
758 if (cpu_idle_fn == cpu_idle_mwait_hlt && *mwait == MWAIT_DISABLED)
761 if (*state == STATE_SLEEPING)
759 return (0);
762 return (0);
760 *mwait = MWAIT_WOKEN;
761
763 if (*state == STATE_MWAIT)
764 *state = STATE_RUNNING;
762 return (1);
763}
764
765/*
766 * Ordered by speed/power consumption.
767 */
768struct {
769 void *id_fn;
770 char *id_name;
771} idle_tbl[] = {
772 { cpu_idle_spin, "spin" },
773 { cpu_idle_mwait, "mwait" },
765 return (1);
766}
767
768/*
769 * Ordered by speed/power consumption.
770 */
771struct {
772 void *id_fn;
773 char *id_name;
774} idle_tbl[] = {
775 { cpu_idle_spin, "spin" },
776 { cpu_idle_mwait, "mwait" },
774 { cpu_idle_mwait_hlt, "mwait_hlt" },
775 { cpu_idle_amdc1e, "amdc1e" },
776 { cpu_idle_hlt, "hlt" },
777 { cpu_idle_acpi, "acpi" },
778 { NULL, NULL }
779};
780
781static int
782idle_sysctl_available(SYSCTL_HANDLER_ARGS)
783{
784 char *avail, *p;
785 int error;
786 int i;
787
788 avail = malloc(256, M_TEMP, M_WAITOK);
789 p = avail;
790 for (i = 0; idle_tbl[i].id_name != NULL; i++) {
791 if (strstr(idle_tbl[i].id_name, "mwait") &&
792 (cpu_feature2 & CPUID2_MON) == 0)
793 continue;
777 { cpu_idle_hlt, "hlt" },
778 { cpu_idle_acpi, "acpi" },
779 { NULL, NULL }
780};
781
782static int
783idle_sysctl_available(SYSCTL_HANDLER_ARGS)
784{
785 char *avail, *p;
786 int error;
787 int i;
788
789 avail = malloc(256, M_TEMP, M_WAITOK);
790 p = avail;
791 for (i = 0; idle_tbl[i].id_name != NULL; i++) {
792 if (strstr(idle_tbl[i].id_name, "mwait") &&
793 (cpu_feature2 & CPUID2_MON) == 0)
794 continue;
794 if (strcmp(idle_tbl[i].id_name, "amdc1e") == 0 &&
795 cpu_ident_amdc1e == 0)
795 if (strcmp(idle_tbl[i].id_name, "acpi") == 0 &&
796 cpu_idle_hook == NULL)
796 continue;
797 p += sprintf(p, "%s, ", idle_tbl[i].id_name);
798 }
799 error = sysctl_handle_string(oidp, avail, 0, req);
800 free(avail, M_TEMP);
801 return (error);
802}
803
797 continue;
798 p += sprintf(p, "%s, ", idle_tbl[i].id_name);
799 }
800 error = sysctl_handle_string(oidp, avail, 0, req);
801 free(avail, M_TEMP);
802 return (error);
803}
804
805SYSCTL_PROC(_machdep, OID_AUTO, idle_available, CTLTYPE_STRING | CTLFLAG_RD,
806 0, 0, idle_sysctl_available, "A", "list of available idle functions");
807
804static int
805idle_sysctl(SYSCTL_HANDLER_ARGS)
806{
807 char buf[16];
808 int error;
809 char *p;
810 int i;
811

--- 7 unchanged lines hidden (view full) ---

819 strncpy(buf, p, sizeof(buf));
820 error = sysctl_handle_string(oidp, buf, sizeof(buf), req);
821 if (error != 0 || req->newptr == NULL)
822 return (error);
823 for (i = 0; idle_tbl[i].id_name != NULL; i++) {
824 if (strstr(idle_tbl[i].id_name, "mwait") &&
825 (cpu_feature2 & CPUID2_MON) == 0)
826 continue;
808static int
809idle_sysctl(SYSCTL_HANDLER_ARGS)
810{
811 char buf[16];
812 int error;
813 char *p;
814 int i;
815

--- 7 unchanged lines hidden (view full) ---

823 strncpy(buf, p, sizeof(buf));
824 error = sysctl_handle_string(oidp, buf, sizeof(buf), req);
825 if (error != 0 || req->newptr == NULL)
826 return (error);
827 for (i = 0; idle_tbl[i].id_name != NULL; i++) {
828 if (strstr(idle_tbl[i].id_name, "mwait") &&
829 (cpu_feature2 & CPUID2_MON) == 0)
830 continue;
827 if (strcmp(idle_tbl[i].id_name, "amdc1e") == 0 &&
828 cpu_ident_amdc1e == 0)
831 if (strcmp(idle_tbl[i].id_name, "acpi") == 0 &&
832 cpu_idle_hook == NULL)
829 continue;
830 if (strcmp(idle_tbl[i].id_name, buf))
831 continue;
832 cpu_idle_fn = idle_tbl[i].id_fn;
833 return (0);
834 }
835 return (EINVAL);
836}
837
833 continue;
834 if (strcmp(idle_tbl[i].id_name, buf))
835 continue;
836 cpu_idle_fn = idle_tbl[i].id_fn;
837 return (0);
838 }
839 return (EINVAL);
840}
841
838SYSCTL_PROC(_machdep, OID_AUTO, idle_available, CTLTYPE_STRING | CTLFLAG_RD,
839 0, 0, idle_sysctl_available, "A", "list of available idle functions");
840
841SYSCTL_PROC(_machdep, OID_AUTO, idle, CTLTYPE_STRING | CTLFLAG_RW, 0, 0,
842 idle_sysctl, "A", "currently selected idle function");
843
844/*
845 * Reset registers to default values on exec.
846 */
847void
848exec_setregs(struct thread *td, struct image_params *imgp, u_long stack)

--- 889 unchanged lines hidden (view full) ---

1738#ifdef XENHVM
1739 if (inw(0x10) == 0x49d2) {
1740 if (bootverbose)
1741 printf("Xen detected: disabling emulated block and network devices\n");
1742 outw(0x10, 3);
1743 }
1744#endif
1745
842SYSCTL_PROC(_machdep, OID_AUTO, idle, CTLTYPE_STRING | CTLFLAG_RW, 0, 0,
843 idle_sysctl, "A", "currently selected idle function");
844
845/*
846 * Reset registers to default values on exec.
847 */
848void
849exec_setregs(struct thread *td, struct image_params *imgp, u_long stack)

--- 889 unchanged lines hidden (view full) ---

1739#ifdef XENHVM
1740 if (inw(0x10) == 0x49d2) {
1741 if (bootverbose)
1742 printf("Xen detected: disabling emulated block and network devices\n");
1743 outw(0x10, 3);
1744 }
1745#endif
1746
1746 if (cpu_probe_amdc1e())
1747 cpu_idle_fn = cpu_idle_amdc1e;
1747 cpu_probe_amdc1e();
1748
1749 /* Location of kernel stack for locore */
1750 return ((u_int64_t)thread0.td_pcb);
1751}
1752
1753void
1754cpu_pcpu_init(struct pcpu *pcpu, int cpuid, size_t size)
1755{

--- 609 unchanged lines hidden --- 		1748
1749 /* Location of kernel stack for locore */
1750 return ((u_int64_t)thread0.td_pcb);
1751}
1752
1753void
1754cpu_pcpu_init(struct pcpu *pcpu, int cpuid, size_t size)
1755{

--- 609 unchanged lines hidden ---