machdep.c (211924) | machdep.c (212541) |
---|---|
1/*- 2 * Copyright (c) 2003 Peter Wemm. 3 * Copyright (c) 1992 Terrence R. Lambert. 4 * Copyright (c) 1982, 1987, 1990 The Regents of the University of California. 5 * All rights reserved. 6 * 7 * This code is derived from software contributed to Berkeley by 8 * William Jolitz. --- 25 unchanged lines hidden (view full) --- 34 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 35 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 36 * SUCH DAMAGE. 37 * 38 * from: @(#)machdep.c 7.4 (Berkeley) 6/3/91 39 */ 40 41#include <sys/cdefs.h> | 1/*- 2 * Copyright (c) 2003 Peter Wemm. 3 * Copyright (c) 1992 Terrence R. Lambert. 4 * Copyright (c) 1982, 1987, 1990 The Regents of the University of California. 5 * All rights reserved. 6 * 7 * This code is derived from software contributed to Berkeley by 8 * William Jolitz. --- 25 unchanged lines hidden (view full) --- 34 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 35 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 36 * SUCH DAMAGE. 37 * 38 * from: @(#)machdep.c 7.4 (Berkeley) 6/3/91 39 */ 40 41#include <sys/cdefs.h> |
42__FBSDID("$FreeBSD: head/sys/amd64/amd64/machdep.c 211924 2010-08-28 08:03:29Z rpaulo $"); | 42__FBSDID("$FreeBSD: head/sys/amd64/amd64/machdep.c 212541 2010-09-13 07:25:35Z mav $"); |
43 44#include "opt_atalk.h" 45#include "opt_atpic.h" 46#include "opt_compat.h" 47#include "opt_cpu.h" 48#include "opt_ddb.h" 49#include "opt_inet.h" 50#include "opt_ipx.h" --- 529 unchanged lines hidden (view full) --- 580void 581cpu_halt(void) 582{ 583 for (;;) 584 __asm__ ("hlt"); 585} 586 587void (*cpu_idle_hook)(void) = NULL; /* ACPI idle hook. */ | 43 44#include "opt_atalk.h" 45#include "opt_atpic.h" 46#include "opt_compat.h" 47#include "opt_cpu.h" 48#include "opt_ddb.h" 49#include "opt_inet.h" 50#include "opt_ipx.h" --- 529 unchanged lines hidden (view full) --- 580void 581cpu_halt(void) 582{ 583 for (;;) 584 __asm__ ("hlt"); 585} 586 587void (*cpu_idle_hook)(void) = NULL; /* ACPI idle hook. */ |
588static int cpu_ident_amdc1e = 0; /* AMD C1E supported. */ 589static int idle_mwait = 1; /* Use MONITOR/MWAIT for short idle. */ 590TUNABLE_INT("machdep.idle_mwait", &idle_mwait); 591SYSCTL_INT(_machdep, OID_AUTO, idle_mwait, CTLFLAG_RW, &idle_mwait, 592 0, "Use MONITOR/MWAIT for short idle"); |
|
588 | 593 |
594#define STATE_RUNNING 0x0 595#define STATE_MWAIT 0x1 596#define STATE_SLEEPING 0x2 597 |
|
589static void | 598static void |
590cpu_idle_hlt(int busy) | 599cpu_idle_acpi(int busy) |
591{ | 600{ |
592 /* 593 * we must absolutely guarentee that hlt is the next instruction 594 * after sti or we introduce a timing window. 595 */ | 601 int *state; 602 603 state = (int *)PCPU_PTR(monitorbuf); 604 *state = STATE_SLEEPING; |
596 disable_intr(); | 605 disable_intr(); |
597 if (sched_runnable()) | 606 if (sched_runnable()) |
598 enable_intr(); | 607 enable_intr(); |
608 else if (cpu_idle_hook) 609 cpu_idle_hook(); |
|
599 else 600 __asm __volatile("sti; hlt"); | 610 else 611 __asm __volatile("sti; hlt"); |
612 *state = STATE_RUNNING; |
|
601} 602 603static void | 613} 614 615static void |
604cpu_idle_acpi(int busy) | 616cpu_idle_hlt(int busy) |
605{ | 617{ |
618 int *state; 619 620 state = (int *)PCPU_PTR(monitorbuf); 621 *state = STATE_SLEEPING; 622 /* 623 * We must absolutely guarentee that hlt is the next instruction 624 * after sti or we introduce a timing window. 625 */ |
|
606 disable_intr(); | 626 disable_intr(); |
607 if (sched_runnable()) | 627 if (sched_runnable()) |
608 enable_intr(); | 628 enable_intr(); |
609 else if (cpu_idle_hook) 610 cpu_idle_hook(); | |
611 else 612 __asm __volatile("sti; hlt"); | 629 else 630 __asm __volatile("sti; hlt"); |
631 *state = STATE_RUNNING; |
|
613} 614 | 632} 633 |
615static int cpu_ident_amdc1e = 0; | 634/* 635 * MWAIT cpu power states. Lower 4 bits are sub-states. 636 */ 637#define MWAIT_C0 0xf0 638#define MWAIT_C1 0x00 639#define MWAIT_C2 0x10 640#define MWAIT_C3 0x20 641#define MWAIT_C4 0x30 |
616 | 642 |
617static int 618cpu_probe_amdc1e(void) | 643static void 644cpu_idle_mwait(int busy) |
619{ | 645{ |
620 int i; | 646 int *state; |
621 | 647 |
622 /* 623 * Forget it, if we're not using local APIC timer. 624 */ 625 if (resource_disabled("apic", 0) || 626 (resource_int_value("apic", 0, "clock", &i) == 0 && i == 0)) 627 return (0); 628 629 /* 630 * Detect the presence of C1E capability mostly on latest 631 * dual-cores (or future) k8 family. 632 */ 633 if (cpu_vendor_id == CPU_VENDOR_AMD && 634 (cpu_id & 0x00000f00) == 0x00000f00 && 635 (cpu_id & 0x0fff0000) >= 0x00040000) { 636 cpu_ident_amdc1e = 1; 637 return (1); | 648 state = (int *)PCPU_PTR(monitorbuf); 649 *state = STATE_MWAIT; 650 if (!sched_runnable()) { 651 cpu_monitor(state, 0, 0); 652 if (*state == STATE_MWAIT) 653 cpu_mwait(0, MWAIT_C1); |
638 } | 654 } |
655 *state = STATE_RUNNING; 656} |
|
639 | 657 |
640 return (0); | 658static void 659cpu_idle_spin(int busy) 660{ 661 int *state; 662 int i; 663 664 state = (int *)PCPU_PTR(monitorbuf); 665 *state = STATE_RUNNING; 666 for (i = 0; i < 1000; i++) { 667 if (sched_runnable()) 668 return; 669 cpu_spinwait(); 670 } |
641} 642 643/* 644 * C1E renders the local APIC timer dead, so we disable it by 645 * reading the Interrupt Pending Message register and clearing 646 * both C1eOnCmpHalt (bit 28) and SmiOnCmpHalt (bit 27). 647 * 648 * Reference: 649 * "BIOS and Kernel Developer's Guide for AMD NPT Family 0Fh Processors" 650 * #32559 revision 3.00+ 651 */ 652#define MSR_AMDK8_IPM 0xc0010055 653#define AMDK8_SMIONCMPHALT (1ULL << 27) 654#define AMDK8_C1EONCMPHALT (1ULL << 28) 655#define AMDK8_CMPHALT (AMDK8_SMIONCMPHALT | AMDK8_C1EONCMPHALT) 656 657static void | 671} 672 673/* 674 * C1E renders the local APIC timer dead, so we disable it by 675 * reading the Interrupt Pending Message register and clearing 676 * both C1eOnCmpHalt (bit 28) and SmiOnCmpHalt (bit 27). 677 * 678 * Reference: 679 * "BIOS and Kernel Developer's Guide for AMD NPT Family 0Fh Processors" 680 * #32559 revision 3.00+ 681 */ 682#define MSR_AMDK8_IPM 0xc0010055 683#define AMDK8_SMIONCMPHALT (1ULL << 27) 684#define AMDK8_C1EONCMPHALT (1ULL << 28) 685#define AMDK8_CMPHALT (AMDK8_SMIONCMPHALT | AMDK8_C1EONCMPHALT) 686 687static void |
658cpu_idle_amdc1e(int busy) | 688cpu_probe_amdc1e(void) |
659{ 660 | 689{ 690 |
661 disable_intr(); 662 if (sched_runnable()) 663 enable_intr(); 664 else { 665 uint64_t msr; 666 667 msr = rdmsr(MSR_AMDK8_IPM); 668 if (msr & AMDK8_CMPHALT) 669 wrmsr(MSR_AMDK8_IPM, msr & ~AMDK8_CMPHALT); 670 671 if (cpu_idle_hook) 672 cpu_idle_hook(); 673 else 674 __asm __volatile("sti; hlt"); | 691 /* 692 * Detect the presence of C1E capability mostly on latest 693 * dual-cores (or future) k8 family. 694 */ 695 if (cpu_vendor_id == CPU_VENDOR_AMD && 696 (cpu_id & 0x00000f00) == 0x00000f00 && 697 (cpu_id & 0x0fff0000) >= 0x00040000) { 698 cpu_ident_amdc1e = 1; |
675 } 676} 677 | 699 } 700} 701 |
678static void 679cpu_idle_spin(int busy) 680{ 681 return; 682} 683 | |
684void (*cpu_idle_fn)(int) = cpu_idle_acpi; 685 686void 687cpu_idle(int busy) 688{ | 702void (*cpu_idle_fn)(int) = cpu_idle_acpi; 703 704void 705cpu_idle(int busy) 706{ |
707 uint64_t msr; 708 709 CTR2(KTR_SPARE2, "cpu_idle(%d) at %d", 710 busy, curcpu); |
|
689#ifdef SMP 690 if (mp_grab_cpu_hlt()) 691 return; 692#endif | 711#ifdef SMP 712 if (mp_grab_cpu_hlt()) 713 return; 714#endif |
693 cpu_idle_fn(busy); 694} | 715 /* If we are busy - try to use fast methods. */ 716 if (busy) { 717 if ((cpu_feature2 & CPUID2_MON) && idle_mwait) { 718 cpu_idle_mwait(busy); 719 goto out; 720 } 721 } |
695 | 722 |
696/* 697 * mwait cpu power states. Lower 4 bits are sub-states. 698 */ 699#define MWAIT_C0 0xf0 700#define MWAIT_C1 0x00 701#define MWAIT_C2 0x10 702#define MWAIT_C3 0x20 703#define MWAIT_C4 0x30 | 723 /* If we have time - switch timers into idle mode. */ 724 if (!busy) { 725 critical_enter(); 726 cpu_idleclock(); 727 } |
704 | 728 |
705#define MWAIT_DISABLED 0x0 706#define MWAIT_WOKEN 0x1 707#define MWAIT_WAITING 0x2 | 729 /* Apply AMD APIC timer C1E workaround. */ 730 if (cpu_ident_amdc1e && cpu_disable_deep_sleep) { 731 msr = rdmsr(MSR_AMDK8_IPM); 732 if (msr & AMDK8_CMPHALT) 733 wrmsr(MSR_AMDK8_IPM, msr & ~AMDK8_CMPHALT); 734 } |
708 | 735 |
709static void 710cpu_idle_mwait(int busy) 711{ 712 int *mwait; | 736 /* Call main idle method. */ 737 cpu_idle_fn(busy); |
713 | 738 |
714 mwait = (int *)PCPU_PTR(monitorbuf); 715 *mwait = MWAIT_WAITING; 716 if (sched_runnable()) 717 return; 718 cpu_monitor(mwait, 0, 0); 719 if (*mwait == MWAIT_WAITING) 720 cpu_mwait(0, MWAIT_C1); 721} 722 723static void 724cpu_idle_mwait_hlt(int busy) 725{ 726 int *mwait; 727 728 mwait = (int *)PCPU_PTR(monitorbuf); 729 if (busy == 0) { 730 *mwait = MWAIT_DISABLED; 731 cpu_idle_hlt(busy); 732 return; | 739 /* Switch timers mack into active mode. */ 740 if (!busy) { 741 cpu_activeclock(); 742 critical_exit(); |
733 } | 743 } |
734 *mwait = MWAIT_WAITING; 735 if (sched_runnable()) 736 return; 737 cpu_monitor(mwait, 0, 0); 738 if (*mwait == MWAIT_WAITING) 739 cpu_mwait(0, MWAIT_C1); | 744out: 745 CTR2(KTR_SPARE2, "cpu_idle(%d) at %d done", 746 busy, curcpu); |
740} 741 742int 743cpu_idle_wakeup(int cpu) 744{ 745 struct pcpu *pcpu; | 747} 748 749int 750cpu_idle_wakeup(int cpu) 751{ 752 struct pcpu *pcpu; |
746 int *mwait; | 753 int *state; |
747 | 754 |
748 if (cpu_idle_fn == cpu_idle_spin) 749 return (1); 750 if (cpu_idle_fn != cpu_idle_mwait && cpu_idle_fn != cpu_idle_mwait_hlt) 751 return (0); | |
752 pcpu = pcpu_find(cpu); | 755 pcpu = pcpu_find(cpu); |
753 mwait = (int *)pcpu->pc_monitorbuf; | 756 state = (int *)pcpu->pc_monitorbuf; |
754 /* 755 * This doesn't need to be atomic since missing the race will 756 * simply result in unnecessary IPIs. 757 */ | 757 /* 758 * This doesn't need to be atomic since missing the race will 759 * simply result in unnecessary IPIs. 760 */ |
758 if (cpu_idle_fn == cpu_idle_mwait_hlt && *mwait == MWAIT_DISABLED) | 761 if (*state == STATE_SLEEPING) |
759 return (0); | 762 return (0); |
760 *mwait = MWAIT_WOKEN; 761 | 763 if (*state == STATE_MWAIT) 764 *state = STATE_RUNNING; |
762 return (1); 763} 764 765/* 766 * Ordered by speed/power consumption. 767 */ 768struct { 769 void *id_fn; 770 char *id_name; 771} idle_tbl[] = { 772 { cpu_idle_spin, "spin" }, 773 { cpu_idle_mwait, "mwait" }, | 765 return (1); 766} 767 768/* 769 * Ordered by speed/power consumption. 770 */ 771struct { 772 void *id_fn; 773 char *id_name; 774} idle_tbl[] = { 775 { cpu_idle_spin, "spin" }, 776 { cpu_idle_mwait, "mwait" }, |
774 { cpu_idle_mwait_hlt, "mwait_hlt" }, 775 { cpu_idle_amdc1e, "amdc1e" }, | |
776 { cpu_idle_hlt, "hlt" }, 777 { cpu_idle_acpi, "acpi" }, 778 { NULL, NULL } 779}; 780 781static int 782idle_sysctl_available(SYSCTL_HANDLER_ARGS) 783{ 784 char *avail, *p; 785 int error; 786 int i; 787 788 avail = malloc(256, M_TEMP, M_WAITOK); 789 p = avail; 790 for (i = 0; idle_tbl[i].id_name != NULL; i++) { 791 if (strstr(idle_tbl[i].id_name, "mwait") && 792 (cpu_feature2 & CPUID2_MON) == 0) 793 continue; | 777 { cpu_idle_hlt, "hlt" }, 778 { cpu_idle_acpi, "acpi" }, 779 { NULL, NULL } 780}; 781 782static int 783idle_sysctl_available(SYSCTL_HANDLER_ARGS) 784{ 785 char *avail, *p; 786 int error; 787 int i; 788 789 avail = malloc(256, M_TEMP, M_WAITOK); 790 p = avail; 791 for (i = 0; idle_tbl[i].id_name != NULL; i++) { 792 if (strstr(idle_tbl[i].id_name, "mwait") && 793 (cpu_feature2 & CPUID2_MON) == 0) 794 continue; |
794 if (strcmp(idle_tbl[i].id_name, "amdc1e") == 0 && 795 cpu_ident_amdc1e == 0) | 795 if (strcmp(idle_tbl[i].id_name, "acpi") == 0 && 796 cpu_idle_hook == NULL) |
796 continue; 797 p += sprintf(p, "%s, ", idle_tbl[i].id_name); 798 } 799 error = sysctl_handle_string(oidp, avail, 0, req); 800 free(avail, M_TEMP); 801 return (error); 802} 803 | 797 continue; 798 p += sprintf(p, "%s, ", idle_tbl[i].id_name); 799 } 800 error = sysctl_handle_string(oidp, avail, 0, req); 801 free(avail, M_TEMP); 802 return (error); 803} 804 |
805SYSCTL_PROC(_machdep, OID_AUTO, idle_available, CTLTYPE_STRING | CTLFLAG_RD, 806 0, 0, idle_sysctl_available, "A", "list of available idle functions"); 807 |
|
804static int 805idle_sysctl(SYSCTL_HANDLER_ARGS) 806{ 807 char buf[16]; 808 int error; 809 char *p; 810 int i; 811 --- 7 unchanged lines hidden (view full) --- 819 strncpy(buf, p, sizeof(buf)); 820 error = sysctl_handle_string(oidp, buf, sizeof(buf), req); 821 if (error != 0 || req->newptr == NULL) 822 return (error); 823 for (i = 0; idle_tbl[i].id_name != NULL; i++) { 824 if (strstr(idle_tbl[i].id_name, "mwait") && 825 (cpu_feature2 & CPUID2_MON) == 0) 826 continue; | 808static int 809idle_sysctl(SYSCTL_HANDLER_ARGS) 810{ 811 char buf[16]; 812 int error; 813 char *p; 814 int i; 815 --- 7 unchanged lines hidden (view full) --- 823 strncpy(buf, p, sizeof(buf)); 824 error = sysctl_handle_string(oidp, buf, sizeof(buf), req); 825 if (error != 0 || req->newptr == NULL) 826 return (error); 827 for (i = 0; idle_tbl[i].id_name != NULL; i++) { 828 if (strstr(idle_tbl[i].id_name, "mwait") && 829 (cpu_feature2 & CPUID2_MON) == 0) 830 continue; |
827 if (strcmp(idle_tbl[i].id_name, "amdc1e") == 0 && 828 cpu_ident_amdc1e == 0) | 831 if (strcmp(idle_tbl[i].id_name, "acpi") == 0 && 832 cpu_idle_hook == NULL) |
829 continue; 830 if (strcmp(idle_tbl[i].id_name, buf)) 831 continue; 832 cpu_idle_fn = idle_tbl[i].id_fn; 833 return (0); 834 } 835 return (EINVAL); 836} 837 | 833 continue; 834 if (strcmp(idle_tbl[i].id_name, buf)) 835 continue; 836 cpu_idle_fn = idle_tbl[i].id_fn; 837 return (0); 838 } 839 return (EINVAL); 840} 841 |
838SYSCTL_PROC(_machdep, OID_AUTO, idle_available, CTLTYPE_STRING | CTLFLAG_RD, 839 0, 0, idle_sysctl_available, "A", "list of available idle functions"); 840 | |
841SYSCTL_PROC(_machdep, OID_AUTO, idle, CTLTYPE_STRING | CTLFLAG_RW, 0, 0, 842 idle_sysctl, "A", "currently selected idle function"); 843 844/* 845 * Reset registers to default values on exec. 846 */ 847void 848exec_setregs(struct thread *td, struct image_params *imgp, u_long stack) --- 889 unchanged lines hidden (view full) --- 1738#ifdef XENHVM 1739 if (inw(0x10) == 0x49d2) { 1740 if (bootverbose) 1741 printf("Xen detected: disabling emulated block and network devices\n"); 1742 outw(0x10, 3); 1743 } 1744#endif 1745 | 842SYSCTL_PROC(_machdep, OID_AUTO, idle, CTLTYPE_STRING | CTLFLAG_RW, 0, 0, 843 idle_sysctl, "A", "currently selected idle function"); 844 845/* 846 * Reset registers to default values on exec. 847 */ 848void 849exec_setregs(struct thread *td, struct image_params *imgp, u_long stack) --- 889 unchanged lines hidden (view full) --- 1739#ifdef XENHVM 1740 if (inw(0x10) == 0x49d2) { 1741 if (bootverbose) 1742 printf("Xen detected: disabling emulated block and network devices\n"); 1743 outw(0x10, 3); 1744 } 1745#endif 1746 |
1746 if (cpu_probe_amdc1e()) 1747 cpu_idle_fn = cpu_idle_amdc1e; | 1747 cpu_probe_amdc1e(); |
1748 1749 /* Location of kernel stack for locore */ 1750 return ((u_int64_t)thread0.td_pcb); 1751} 1752 1753void 1754cpu_pcpu_init(struct pcpu *pcpu, int cpuid, size_t size) 1755{ --- 609 unchanged lines hidden --- | 1748 1749 /* Location of kernel stack for locore */ 1750 return ((u_int64_t)thread0.td_pcb); 1751} 1752 1753void 1754cpu_pcpu_init(struct pcpu *pcpu, int cpuid, size_t size) 1755{ --- 609 unchanged lines hidden --- |