Deleted Added
sdiff udiff text old ( 145615 ) new ( 145774 )
full compact
hwpmc_piv.c (145615) hwpmc_piv.c (145774)
1/*-
2 * Copyright (c) 2003-2005 Joseph Koshy
3 * All rights reserved.
4 *
5 * Redistribution and use in source and binary forms, with or without
6 * modification, are permitted provided that the following conditions
7 * are met:
8 * 1. Redistributions of source code must retain the above copyright

--- 11 unchanged lines hidden (view full) ---

20 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
21 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
22 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
23 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
24 * SUCH DAMAGE.
25 */
26
27#include <sys/cdefs.h>
1/*-
2 * Copyright (c) 2003-2005 Joseph Koshy
3 * All rights reserved.
4 *
5 * Redistribution and use in source and binary forms, with or without
6 * modification, are permitted provided that the following conditions
7 * are met:
8 * 1. Redistributions of source code must retain the above copyright

--- 11 unchanged lines hidden (view full) ---

20 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
21 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
22 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
23 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
24 * SUCH DAMAGE.
25 */
26
27#include <sys/cdefs.h>
28__FBSDID("$FreeBSD: head/sys/dev/hwpmc/hwpmc_piv.c 145615 2005-04-28 08:13:19Z jkoshy $");
28__FBSDID("$FreeBSD: head/sys/dev/hwpmc/hwpmc_piv.c 145774 2005-05-01 14:11:49Z jkoshy $");
29
30#include <sys/param.h>
31#include <sys/lock.h>
32#include <sys/mutex.h>
33#include <sys/pmc.h>
34#include <sys/pmckern.h>
35#include <sys/smp.h>
36#include <sys/systm.h>
37
29
30#include <sys/param.h>
31#include <sys/lock.h>
32#include <sys/mutex.h>
33#include <sys/pmc.h>
34#include <sys/pmckern.h>
35#include <sys/smp.h>
36#include <sys/systm.h>
37
38#include <machine/cputypes.h>
38#include <machine/apicreg.h>
39#include <machine/md_var.h>
40
41/*
42 * PENTIUM 4 SUPPORT
43 *
44 * The P4 has 18 PMCs, divided into 4 groups with 4,4,4 and 6 PMCs
45 * respectively. Each PMC comprises of two model specific registers:
46 * a counter configuration control register (CCCR) and a counter

--- 44 unchanged lines hidden (view full) ---

91 * cpus as independent CPUs and can schedule kernel threads on them
92 * independently. However, the two logical CPUs share the same set of
93 * PMC resources. We need to ensure that:
94 * - PMCs that use the PMC_F_DESCENDANTS semantics are handled correctly,
95 * and,
96 * - Threads of multi-threaded processes that get scheduled on the same
97 * physical CPU are handled correctly.
98 *
39#include <machine/md_var.h>
40
41/*
42 * PENTIUM 4 SUPPORT
43 *
44 * The P4 has 18 PMCs, divided into 4 groups with 4,4,4 and 6 PMCs
45 * respectively. Each PMC comprises of two model specific registers:
46 * a counter configuration control register (CCCR) and a counter

--- 44 unchanged lines hidden (view full) ---

91 * cpus as independent CPUs and can schedule kernel threads on them
92 * independently. However, the two logical CPUs share the same set of
93 * PMC resources. We need to ensure that:
94 * - PMCs that use the PMC_F_DESCENDANTS semantics are handled correctly,
95 * and,
96 * - Threads of multi-threaded processes that get scheduled on the same
97 * physical CPU are handled correctly.
98 *
99 * HTT Detection
100 *
99 * Not all HTT capable systems will have HTT enabled since users may
100 * have turned HTT support off using the appropriate sysctls
101 * Not all HTT capable systems will have HTT enabled since users may
102 * have turned HTT support off using the appropriate sysctls
101 * (machdep.hlt_logical_cpus and machdep.logical_cpus_mask). We
102 * detect the presence of HTT by remembering if an initialization was
103 * done for a logical CPU.
103 * (machdep.hlt_logical_cpus or machdep.logical_cpus_mask). We detect
104 * the presence of HTT by remembering if 'p4_init()' was called for a
105 * logical CPU. Note that hwpmc(4) cannot deal with a change in HTT
106 * status once it is loaded.
104 *
107 *
108 * Handling HTT READ / WRITE / START / STOP
109 *
110 * PMC resources are shared across multiple logical CPUs. In each
111 * physical CPU's state we keep track of a 'runcount' which reflects
112 * the number of PMC-using processes that have been scheduled on the
113 * logical CPUs of this physical CPU. Process-mode PMC operations
114 * will actually 'start' or 'stop' hardware only if these are the
115 * first or last processes respectively to use the hardware. PMC
116 * values written by a 'write' operation are saved and are transferred
117 * to hardware at PMC 'start' time if the runcount is 0. If the
118 * runcount is greater than 0 at the time of a 'start' operation, we
119 * keep track of the actual hardware value at the time of the 'start'
120 * operation and use this to adjust the final readings at PMC 'stop'
121 * or 'read' time.
122 *
123 * Execution sequences:
124 *
125 * Case 1: CPUx +...- (no overlap)
126 * CPUy +...-
127 * RC 0 1 0 1 0
128 *
129 * Case 2: CPUx +........- (partial overlap)
130 * CPUy +........-
131 * RC 0 1 2 1 0
132 *
133 * Case 3: CPUx +..............- (fully overlapped)
134 * CPUy +.....-
135 * RC 0 1 2 1 0
136 *
137 * Here CPUx and CPUy are one of the two logical processors on a HTT CPU.
138 *
139 * Handling HTT CONFIG
140 *
141 * Different processes attached to the same PMC may get scheduled on
142 * the two logical processors in the package. We keep track of config
143 * and de-config operations using the CFGFLAGS fields of the per-physical
144 * cpu state.
105 */
106
107#define P4_PMCS() \
108 P4_PMC(BPU_COUNTER0) \
109 P4_PMC(BPU_COUNTER1) \
110 P4_PMC(BPU_COUNTER2) \
111 P4_PMC(BPU_COUNTER3) \
112 P4_PMC(MS_COUNTER0) \

--- 268 unchanged lines hidden (view full) ---

381/*
382 * Per-CPU data structure for P4 class CPUs
383 *
384 * [common stuff]
385 * [19 struct pmc_hw pointers]
386 * [19 struct pmc_hw structures]
387 * [45 ESCRs status bytes]
388 * [per-cpu spin mutex]
145 */
146
147#define P4_PMCS() \
148 P4_PMC(BPU_COUNTER0) \
149 P4_PMC(BPU_COUNTER1) \
150 P4_PMC(BPU_COUNTER2) \
151 P4_PMC(BPU_COUNTER3) \
152 P4_PMC(MS_COUNTER0) \

--- 268 unchanged lines hidden (view full) ---

421/*
422 * Per-CPU data structure for P4 class CPUs
423 *
424 * [common stuff]
425 * [19 struct pmc_hw pointers]
426 * [19 struct pmc_hw structures]
427 * [45 ESCRs status bytes]
428 * [per-cpu spin mutex]
389 * [19 flags for holding the config count and runcount]
390 * [19*2 saved value fields] (Thread mode PMC support)
391 * [19*2 pmc value fields] (-do-)
429 * [19 flag fields for holding config flags and a runcount]
430 * [19*2 hw value fields] (Thread mode PMC support)
431 * or
432 * [19*2 EIP values] (Sampling mode PMCs)
433 * [19*2 pmc value fields] (Thread mode PMC support))
392 */
393
394struct p4_cpu {
395 struct pmc_cpu pc_common;
396 struct pmc_hw *pc_hwpmcs[P4_NPMCS];
397 struct pmc_hw pc_p4pmcs[P4_NPMCS];
398 char pc_escrs[P4_NESCR];
399 struct mtx pc_mtx; /* spin lock */
400 unsigned char pc_flags[P4_NPMCS]; /* 4 bits each: {cfg,run}count */
434 */
435
436struct p4_cpu {
437 struct pmc_cpu pc_common;
438 struct pmc_hw *pc_hwpmcs[P4_NPMCS];
439 struct pmc_hw pc_p4pmcs[P4_NPMCS];
440 char pc_escrs[P4_NESCR];
441 struct mtx pc_mtx; /* spin lock */
442 unsigned char pc_flags[P4_NPMCS]; /* 4 bits each: {cfg,run}count */
401 pmc_value_t pc_saved[P4_NPMCS * P4_NHTT];
443 union {
444 pmc_value_t pc_hw[P4_NPMCS * P4_NHTT];
445 uintptr_t pc_ip[P4_NPMCS * P4_NHTT];
446 } pc_si;
402 pmc_value_t pc_pmc_values[P4_NPMCS * P4_NHTT];
403};
404
447 pmc_value_t pc_pmc_values[P4_NPMCS * P4_NHTT];
448};
449
405#define P4_PCPU_SAVED_VALUE(PC,RI,CPU) (PC)->pc_saved[(RI)*((CPU) & 1)]
406#define P4_PCPU_PMC_VALUE(P,R,C) (P)->pc_pmc_values[(R)*((C) & 1)]
450#define P4_PCPU_PMC_VALUE(PC,RI,CPU) (PC)->pc_pmc_values[(RI)*((CPU) & 1)]
451#define P4_PCPU_HW_VALUE(PC,RI,CPU) (PC)->pc_si.pc_hw[(RI)*((CPU) & 1)]
452#define P4_PCPU_SAVED_IP(PC,RI,CPU) (PC)->pc_si.pc_ip[(RI)*((CPU) & 1)]
407
408#define P4_PCPU_GET_FLAGS(PC,RI,MASK) ((PC)->pc_flags[(RI)] & (MASK))
409#define P4_PCPU_SET_FLAGS(PC,RI,MASK,VAL) do { \
410 char _tmp; \
411 _tmp = (PC)->pc_flags[(RI)]; \
412 _tmp &= ~(MASK); \
413 _tmp |= (VAL) & (MASK); \
414 (PC)->pc_flags[(RI)] = _tmp; \
415} while (0)
416
417#define P4_PCPU_GET_RUNCOUNT(PC,RI) P4_PCPU_GET_FLAGS(PC,RI,0x0F)
418#define P4_PCPU_SET_RUNCOUNT(PC,RI,V) P4_PCPU_SET_FLAGS(PC,RI,0x0F,V)
419
453
454#define P4_PCPU_GET_FLAGS(PC,RI,MASK) ((PC)->pc_flags[(RI)] & (MASK))
455#define P4_PCPU_SET_FLAGS(PC,RI,MASK,VAL) do { \
456 char _tmp; \
457 _tmp = (PC)->pc_flags[(RI)]; \
458 _tmp &= ~(MASK); \
459 _tmp |= (VAL) & (MASK); \
460 (PC)->pc_flags[(RI)] = _tmp; \
461} while (0)
462
463#define P4_PCPU_GET_RUNCOUNT(PC,RI) P4_PCPU_GET_FLAGS(PC,RI,0x0F)
464#define P4_PCPU_SET_RUNCOUNT(PC,RI,V) P4_PCPU_SET_FLAGS(PC,RI,0x0F,V)
465
420#define P4_PCPU_GET_CFGCOUNT(PC,RI) (P4_PCPU_GET_FLAGS(PC,RI,0xF0) >> 4)
421#define P4_PCPU_SET_CFGCOUNT(PC,RI,C) P4_PCPU_SET_FLAGS(PC,RI,0xF0,((C) <<4))
466#define P4_PCPU_GET_CFGFLAGS(PC,RI) (P4_PCPU_GET_FLAGS(PC,RI,0xF0) >> 4)
467#define P4_PCPU_SET_CFGFLAGS(PC,RI,C) P4_PCPU_SET_FLAGS(PC,RI,0xF0,((C) <<4))
422
468
469#define P4_CPU_TO_FLAG(C) (pmc_cpu_is_logical(cpu) ? 0x2 : 0x1)
470
423/* ESCR row disposition */
424static int p4_escrdisp[P4_NESCR];
425
426#define P4_ESCR_ROW_DISP_IS_THREAD(E) (p4_escrdisp[(E)] > 0)
427#define P4_ESCR_ROW_DISP_IS_STANDALONE(E) (p4_escrdisp[(E)] < 0)
428#define P4_ESCR_ROW_DISP_IS_FREE(E) (p4_escrdisp[(E)] == 0)
429
430#define P4_ESCR_MARK_ROW_STANDALONE(E) do { \

--- 147 unchanged lines hidden (view full) ---

578 */
579
580static int
581p4_switch_in(struct pmc_cpu *pc, struct pmc_process *pp)
582{
583 (void) pc;
584
585 PMCDBG(MDP,SWI,1, "pc=%p pp=%p enable-msr=%d", pc, pp,
471/* ESCR row disposition */
472static int p4_escrdisp[P4_NESCR];
473
474#define P4_ESCR_ROW_DISP_IS_THREAD(E) (p4_escrdisp[(E)] > 0)
475#define P4_ESCR_ROW_DISP_IS_STANDALONE(E) (p4_escrdisp[(E)] < 0)
476#define P4_ESCR_ROW_DISP_IS_FREE(E) (p4_escrdisp[(E)] == 0)
477
478#define P4_ESCR_MARK_ROW_STANDALONE(E) do { \

--- 147 unchanged lines hidden (view full) ---

626 */
627
628static int
629p4_switch_in(struct pmc_cpu *pc, struct pmc_process *pp)
630{
631 (void) pc;
632
633 PMCDBG(MDP,SWI,1, "pc=%p pp=%p enable-msr=%d", pc, pp,
586 (pp->pp_flags & PMC_FLAG_ENABLE_MSR_ACCESS) != 0);
634 (pp->pp_flags & PMC_PP_ENABLE_MSR_ACCESS) != 0);
587
588 /* enable the RDPMC instruction */
635
636 /* enable the RDPMC instruction */
589 if (pp->pp_flags & PMC_FLAG_ENABLE_MSR_ACCESS)
637 if (pp->pp_flags & PMC_PP_ENABLE_MSR_ACCESS)
590 load_cr4(rcr4() | CR4_PCE);
591
592 PMCDBG(MDP,SWI,2, "cr4=0x%x", rcr4());
593
594 return 0;
595}
596
597/*

--- 39 unchanged lines hidden (view full) ---

637 phw = pc->pc_hwpmcs[ri];
638 pd = &p4_pmcdesc[ri];
639 pm = phw->phw_pmc;
640
641 KASSERT(pm != NULL,
642 ("[p4,%d] No owner for HWPMC [cpu%d,pmc%d]", __LINE__,
643 cpu, ri));
644
638 load_cr4(rcr4() | CR4_PCE);
639
640 PMCDBG(MDP,SWI,2, "cr4=0x%x", rcr4());
641
642 return 0;
643}
644
645/*

--- 39 unchanged lines hidden (view full) ---

685 phw = pc->pc_hwpmcs[ri];
686 pd = &p4_pmcdesc[ri];
687 pm = phw->phw_pmc;
688
689 KASSERT(pm != NULL,
690 ("[p4,%d] No owner for HWPMC [cpu%d,pmc%d]", __LINE__,
691 cpu, ri));
692
645 mode = pm->pm_mode;
693 KASSERT(pd->pm_descr.pd_class == PMC_TO_CLASS(pm),
694 ("[p4,%d] class mismatch pd %d != id class %d", __LINE__,
695 pd->pm_descr.pd_class, PMC_TO_CLASS(pm)));
646
696
697 mode = PMC_TO_MODE(pm);
698
647 PMCDBG(MDP,REA,1, "p4-read cpu=%d ri=%d mode=%d", cpu, ri, mode);
648
699 PMCDBG(MDP,REA,1, "p4-read cpu=%d ri=%d mode=%d", cpu, ri, mode);
700
649 if (pd->pm_descr.pd_class == PMC_CLASS_TSC) {
701 if (PMC_TO_CLASS(pm) == PMC_CLASS_TSC) {
650 KASSERT(PMC_IS_COUNTING_MODE(mode),
651 ("[p4,%d] TSC counter in non-counting mode", __LINE__));
652 *v = rdtsc();
653 PMCDBG(MDP,REA,2, "p4-read -> %jx", *v);
654 return 0;
655 }
656
657 KASSERT(pd->pm_descr.pd_class == PMC_CLASS_P4,
658 ("[p4,%d] unknown PMC class %d", __LINE__, pd->pm_descr.pd_class));
659
702 KASSERT(PMC_IS_COUNTING_MODE(mode),
703 ("[p4,%d] TSC counter in non-counting mode", __LINE__));
704 *v = rdtsc();
705 PMCDBG(MDP,REA,2, "p4-read -> %jx", *v);
706 return 0;
707 }
708
709 KASSERT(pd->pm_descr.pd_class == PMC_CLASS_P4,
710 ("[p4,%d] unknown PMC class %d", __LINE__, pd->pm_descr.pd_class));
711
660 if (PMC_IS_SYSTEM_MODE(pm->pm_mode))
661 tmp = rdmsr(p4_pmcdesc[ri].pm_pmc_msr);
662 else
663 tmp = P4_PCPU_PMC_VALUE(pc,ri,cpu);
712 tmp = rdmsr(p4_pmcdesc[ri].pm_pmc_msr);
664
713
665 if (PMC_IS_SAMPLING_MODE(mode))
666 *v = -(tmp + 1); /* undo transformation */
714 if (PMC_IS_VIRTUAL_MODE(mode)) {
715 if (tmp < P4_PCPU_HW_VALUE(pc,ri,cpu)) /* 40 bit overflow */
716 tmp += (P4_PERFCTR_MASK + 1) -
717 P4_PCPU_HW_VALUE(pc,ri,cpu);
718 else
719 tmp -= P4_PCPU_HW_VALUE(pc,ri,cpu);
720 tmp += P4_PCPU_PMC_VALUE(pc,ri,cpu);
721 }
722
723 if (PMC_IS_SAMPLING_MODE(mode)) /* undo transformation */
724 *v = P4_PERFCTR_VALUE_TO_RELOAD_COUNT(tmp);
667 else
668 *v = tmp;
669
670 PMCDBG(MDP,REA,2, "p4-read -> %jx", *v);
671 return 0;
672}
673
674/*
675 * Write a PMC
676 */
677
678static int
679p4_write_pmc(int cpu, int ri, pmc_value_t v)
680{
725 else
726 *v = tmp;
727
728 PMCDBG(MDP,REA,2, "p4-read -> %jx", *v);
729 return 0;
730}
731
732/*
733 * Write a PMC
734 */
735
736static int
737p4_write_pmc(int cpu, int ri, pmc_value_t v)
738{
739 enum pmc_mode mode;
681 struct pmc *pm;
682 struct p4_cpu *pc;
683 const struct pmc_hw *phw;
684 const struct p4pmc_descr *pd;
685
686 KASSERT(cpu >= 0 && cpu < mp_ncpus,
687 ("[amd,%d] illegal CPU value %d", __LINE__, cpu));
688 KASSERT(ri >= 0 && ri < P4_NPMCS,
689 ("[amd,%d] illegal row-index %d", __LINE__, ri));
690
691 pc = (struct p4_cpu *) pmc_pcpu[P4_TO_PHYSICAL_CPU(cpu)];
692 phw = pc->pc_hwpmcs[ri];
693 pm = phw->phw_pmc;
694 pd = &p4_pmcdesc[ri];
695
696 KASSERT(pm != NULL,
697 ("[p4,%d] No owner for HWPMC [cpu%d,pmc%d]", __LINE__,
698 cpu, ri));
699
740 struct pmc *pm;
741 struct p4_cpu *pc;
742 const struct pmc_hw *phw;
743 const struct p4pmc_descr *pd;
744
745 KASSERT(cpu >= 0 && cpu < mp_ncpus,
746 ("[amd,%d] illegal CPU value %d", __LINE__, cpu));
747 KASSERT(ri >= 0 && ri < P4_NPMCS,
748 ("[amd,%d] illegal row-index %d", __LINE__, ri));
749
750 pc = (struct p4_cpu *) pmc_pcpu[P4_TO_PHYSICAL_CPU(cpu)];
751 phw = pc->pc_hwpmcs[ri];
752 pm = phw->phw_pmc;
753 pd = &p4_pmcdesc[ri];
754
755 KASSERT(pm != NULL,
756 ("[p4,%d] No owner for HWPMC [cpu%d,pmc%d]", __LINE__,
757 cpu, ri));
758
759 mode = PMC_TO_MODE(pm);
760
700 PMCDBG(MDP,WRI,1, "p4-write cpu=%d ri=%d mode=%d v=%jx", cpu, ri,
761 PMCDBG(MDP,WRI,1, "p4-write cpu=%d ri=%d mode=%d v=%jx", cpu, ri,
701 pm->pm_mode, v);
762 mode, v);
702
703 /*
704 * The P4's TSC register is writeable, but we don't allow a
705 * write as changing the TSC's value could interfere with
763
764 /*
765 * The P4's TSC register is writeable, but we don't allow a
766 * write as changing the TSC's value could interfere with
706 * other parts of the system.
767 * timekeeping and other system functions.
707 */
768 */
708 if (pd->pm_descr.pd_class == PMC_CLASS_TSC)
769 if (PMC_TO_CLASS(pm) == PMC_CLASS_TSC)
709 return 0;
710
711 /*
712 * write the PMC value to the register/saved value: for
713 * sampling mode PMCs, the value to be programmed into the PMC
714 * counter is -(C+1) where 'C' is the requested sample rate.
715 */
770 return 0;
771
772 /*
773 * write the PMC value to the register/saved value: for
774 * sampling mode PMCs, the value to be programmed into the PMC
775 * counter is -(C+1) where 'C' is the requested sample rate.
776 */
716 if (PMC_IS_SAMPLING_MODE(pm->pm_mode))
717 v = -(v + 1);
777 if (PMC_IS_SAMPLING_MODE(mode))
778 v = P4_RELOAD_COUNT_TO_PERFCTR_VALUE(v);
718
779
719 if (PMC_IS_SYSTEM_MODE(pm->pm_mode))
780 if (PMC_IS_SYSTEM_MODE(mode))
720 wrmsr(pd->pm_pmc_msr, v);
721 else
722 P4_PCPU_PMC_VALUE(pc,ri,cpu) = v;
723
724 return 0;
725}
726
727/*
728 * Configure a PMC 'pm' on the given CPU and row-index.
729 *
730 * 'pm' may be NULL to indicate de-configuration.
731 *
732 * On HTT systems, a PMC may get configured twice, once for each
781 wrmsr(pd->pm_pmc_msr, v);
782 else
783 P4_PCPU_PMC_VALUE(pc,ri,cpu) = v;
784
785 return 0;
786}
787
788/*
789 * Configure a PMC 'pm' on the given CPU and row-index.
790 *
791 * 'pm' may be NULL to indicate de-configuration.
792 *
793 * On HTT systems, a PMC may get configured twice, once for each
733 * "logical" CPU.
794 * "logical" CPU. We track this using the CFGFLAGS field of the
795 * per-cpu state; this field is a bit mask with one bit each for
796 * logical CPUs 0 & 1.
734 */
735
736static int
737p4_config_pmc(int cpu, int ri, struct pmc *pm)
738{
739 struct pmc_hw *phw;
740 struct p4_cpu *pc;
797 */
798
799static int
800p4_config_pmc(int cpu, int ri, struct pmc *pm)
801{
802 struct pmc_hw *phw;
803 struct p4_cpu *pc;
741 int cfgcount;
804 int cfgflags, cpuflag;
742
743 KASSERT(cpu >= 0 && cpu < mp_ncpus,
744 ("[p4,%d] illegal CPU %d", __LINE__, cpu));
745 KASSERT(ri >= 0 && ri < P4_NPMCS,
746 ("[p4,%d] illegal row-index %d", __LINE__, ri));
747
748 pc = (struct p4_cpu *) pmc_pcpu[P4_TO_PHYSICAL_CPU(cpu)];
749 phw = pc->pc_hwpmcs[ri];
750
751 KASSERT(pm == NULL || phw->phw_pmc == NULL ||
752 (p4_system_has_htt && phw->phw_pmc == pm),
753 ("[p4,%d] hwpmc not unconfigured before re-config", __LINE__));
754
755 mtx_lock_spin(&pc->pc_mtx);
805
806 KASSERT(cpu >= 0 && cpu < mp_ncpus,
807 ("[p4,%d] illegal CPU %d", __LINE__, cpu));
808 KASSERT(ri >= 0 && ri < P4_NPMCS,
809 ("[p4,%d] illegal row-index %d", __LINE__, ri));
810
811 pc = (struct p4_cpu *) pmc_pcpu[P4_TO_PHYSICAL_CPU(cpu)];
812 phw = pc->pc_hwpmcs[ri];
813
814 KASSERT(pm == NULL || phw->phw_pmc == NULL ||
815 (p4_system_has_htt && phw->phw_pmc == pm),
816 ("[p4,%d] hwpmc not unconfigured before re-config", __LINE__));
817
818 mtx_lock_spin(&pc->pc_mtx);
756 cfgcount = P4_PCPU_GET_CFGCOUNT(pc,ri);
819 cfgflags = P4_PCPU_GET_CFGFLAGS(pc,ri);
757
820
758 KASSERT(cfgcount >= 0 || cfgcount <= 2,
759 ("[p4,%d] illegal cfgcount cfg=%d on cpu=%d ri=%d", __LINE__,
760 cfgcount, cpu, ri));
821 KASSERT(cfgflags >= 0 || cfgflags <= 3,
822 ("[p4,%d] illegal cfgflags cfg=%d on cpu=%d ri=%d", __LINE__,
823 cfgflags, cpu, ri));
761
824
762 KASSERT(cfgcount == 0 || phw->phw_pmc,
825 KASSERT(cfgflags == 0 || phw->phw_pmc,
763 ("[p4,%d] cpu=%d ri=%d pmc configured with zero cfg count",
764 __LINE__, cpu, ri));
765
826 ("[p4,%d] cpu=%d ri=%d pmc configured with zero cfg count",
827 __LINE__, cpu, ri));
828
766 PMCDBG(MDP,CFG,1, "cpu=%d ri=%d cfg=%d pm=%p", cpu, ri, cfgcount,
829 PMCDBG(MDP,CFG,1, "cpu=%d ri=%d cfg=%d pm=%p", cpu, ri, cfgflags,
767 pm);
768
830 pm);
831
832 cpuflag = P4_CPU_TO_FLAG(cpu);
833
769 if (pm) { /* config */
834 if (pm) { /* config */
770 if (cfgcount == 0)
835 if (cfgflags == 0)
771 phw->phw_pmc = pm;
772
773 KASSERT(phw->phw_pmc == pm,
774 ("[p4,%d] cpu=%d ri=%d config %p != hw %p",
775 __LINE__, cpu, ri, pm, phw->phw_pmc));
776
836 phw->phw_pmc = pm;
837
838 KASSERT(phw->phw_pmc == pm,
839 ("[p4,%d] cpu=%d ri=%d config %p != hw %p",
840 __LINE__, cpu, ri, pm, phw->phw_pmc));
841
777 cfgcount++;
842 cfgflags |= cpuflag;
778 } else { /* unconfig */
843 } else { /* unconfig */
779 --cfgcount;
780 if (cfgcount == 0)
844 cfgflags &= ~cpuflag;
845
846 if (cfgflags == 0)
781 phw->phw_pmc = NULL;
782 }
783
847 phw->phw_pmc = NULL;
848 }
849
784 KASSERT(cfgcount >= 0 || cfgcount <= 2,
850 KASSERT(cfgflags >= 0 || cfgflags <= 3,
785 ("[p4,%d] illegal runcount cfg=%d on cpu=%d ri=%d", __LINE__,
851 ("[p4,%d] illegal runcount cfg=%d on cpu=%d ri=%d", __LINE__,
786 cfgcount, cpu, ri));
852 cfgflags, cpu, ri));
787
853
788 P4_PCPU_SET_CFGCOUNT(pc,ri,cfgcount);
854 P4_PCPU_SET_CFGFLAGS(pc,ri,cfgflags);
789
790 mtx_unlock_spin(&pc->pc_mtx);
791
792 return 0;
793}
794
795/*
855
856 mtx_unlock_spin(&pc->pc_mtx);
857
858 return 0;
859}
860
861/*
862 * Retrieve a configured PMC pointer from hardware state.
863 */
864
865static int
866p4_get_config(int cpu, int ri, struct pmc **ppm)
867{
868 struct p4_cpu *pc;
869 struct pmc_hw *phw;
870 int cfgflags;
871
872 pc = (struct p4_cpu *) pmc_pcpu[P4_TO_PHYSICAL_CPU(cpu)];
873 phw = pc->pc_hwpmcs[ri];
874
875 mtx_lock_spin(&pc->pc_mtx);
876 cfgflags = P4_PCPU_GET_CFGFLAGS(pc,ri);
877 mtx_unlock_spin(&pc->pc_mtx);
878
879 if (cfgflags & P4_CPU_TO_FLAG(cpu))
880 *ppm = phw->phw_pmc; /* PMC config'ed on this CPU */
881 else
882 *ppm = NULL;
883
884 return 0;
885}
886
887/*
796 * Allocate a PMC.
797 *
798 * The allocation strategy differs between HTT and non-HTT systems.
799 *
800 * The non-HTT case:
801 * - Given the desired event and the PMC row-index, lookup the
802 * list of valid ESCRs for the event.
803 * - For each valid ESCR:

--- 36 unchanged lines hidden (view full) ---

840 KASSERT(cpu >= 0 && cpu < mp_ncpus,
841 ("[p4,%d] illegal CPU %d", __LINE__, cpu));
842 KASSERT(ri >= 0 && ri < P4_NPMCS,
843 ("[p4,%d] illegal row-index value %d", __LINE__, ri));
844
845 pd = &p4_pmcdesc[ri];
846
847 PMCDBG(MDP,ALL,1, "p4-allocate ri=%d class=%d pmccaps=0x%x "
888 * Allocate a PMC.
889 *
890 * The allocation strategy differs between HTT and non-HTT systems.
891 *
892 * The non-HTT case:
893 * - Given the desired event and the PMC row-index, lookup the
894 * list of valid ESCRs for the event.
895 * - For each valid ESCR:

--- 36 unchanged lines hidden (view full) ---

932 KASSERT(cpu >= 0 && cpu < mp_ncpus,
933 ("[p4,%d] illegal CPU %d", __LINE__, cpu));
934 KASSERT(ri >= 0 && ri < P4_NPMCS,
935 ("[p4,%d] illegal row-index value %d", __LINE__, ri));
936
937 pd = &p4_pmcdesc[ri];
938
939 PMCDBG(MDP,ALL,1, "p4-allocate ri=%d class=%d pmccaps=0x%x "
848 "reqcaps=0x%x\n", ri, pd->pm_descr.pd_class, pd->pm_descr.pd_caps,
940 "reqcaps=0x%x", ri, pd->pm_descr.pd_class, pd->pm_descr.pd_caps,
849 pm->pm_caps);
850
851 /* check class */
941 pm->pm_caps);
942
943 /* check class */
852 if (pd->pm_descr.pd_class != pm->pm_class)
944 if (pd->pm_descr.pd_class != a->pm_class)
853 return EINVAL;
854
855 /* check requested capabilities */
856 caps = a->pm_caps;
857 if ((pd->pm_descr.pd_caps & caps) != caps)
858 return EPERM;
859
860 if (pd->pm_descr.pd_class == PMC_CLASS_TSC) {

--- 6 unchanged lines hidden (view full) ---

867
868 /*
869 * If the system has HTT enabled, and the desired allocation
870 * mode is process-private, and the PMC row disposition is not
871 * FREE (0), decline the allocation.
872 */
873
874 if (p4_system_has_htt &&
945 return EINVAL;
946
947 /* check requested capabilities */
948 caps = a->pm_caps;
949 if ((pd->pm_descr.pd_caps & caps) != caps)
950 return EPERM;
951
952 if (pd->pm_descr.pd_class == PMC_CLASS_TSC) {

--- 6 unchanged lines hidden (view full) ---

959
960 /*
961 * If the system has HTT enabled, and the desired allocation
962 * mode is process-private, and the PMC row disposition is not
963 * FREE (0), decline the allocation.
964 */
965
966 if (p4_system_has_htt &&
875 PMC_IS_VIRTUAL_MODE(pm->pm_mode) &&
967 PMC_IS_VIRTUAL_MODE(PMC_TO_MODE(pm)) &&
876 pmc_getrowdisp(ri) != 0)
877 return EBUSY;
878
879 KASSERT(pd->pm_descr.pd_class == PMC_CLASS_P4,
880 ("[p4,%d] unknown PMC class %d", __LINE__,
881 pd->pm_descr.pd_class));
882
883 if (pm->pm_event < PMC_EV_P4_FIRST ||

--- 9 unchanged lines hidden (view full) ---

893
894 /*
895 * Some PMC events are 'thread independent'and therefore
896 * cannot be used for process-private modes if HTT is being
897 * used.
898 */
899
900 if (P4_EVENT_IS_TI(pevent) &&
968 pmc_getrowdisp(ri) != 0)
969 return EBUSY;
970
971 KASSERT(pd->pm_descr.pd_class == PMC_CLASS_P4,
972 ("[p4,%d] unknown PMC class %d", __LINE__,
973 pd->pm_descr.pd_class));
974
975 if (pm->pm_event < PMC_EV_P4_FIRST ||

--- 9 unchanged lines hidden (view full) ---

985
986 /*
987 * Some PMC events are 'thread independent'and therefore
988 * cannot be used for process-private modes if HTT is being
989 * used.
990 */
991
992 if (P4_EVENT_IS_TI(pevent) &&
901 PMC_IS_VIRTUAL_MODE(pm->pm_mode) && p4_system_has_htt)
993 PMC_IS_VIRTUAL_MODE(PMC_TO_MODE(pm)) &&
994 p4_system_has_htt)
902 return EINVAL;
903
904 pc = (struct p4_cpu *) pmc_pcpu[P4_TO_PHYSICAL_CPU(cpu)];
905
906 found = 0;
907
908 /* look for a suitable ESCR for this event */
909 for (n = 0; n < P4_MAX_ESCR_PER_EVENT && !found; n++) {
910 if ((escr = pevent->pm_escrs[n]) == P4_ESCR_NONE)
911 break; /* out of ESCRs */
912 /*
913 * Check ESCR row disposition.
914 *
915 * If the request is for a system-mode PMC, then the
916 * ESCR row should not be in process-virtual mode, and
917 * should also be free on the current CPU.
918 */
919
995 return EINVAL;
996
997 pc = (struct p4_cpu *) pmc_pcpu[P4_TO_PHYSICAL_CPU(cpu)];
998
999 found = 0;
1000
1001 /* look for a suitable ESCR for this event */
1002 for (n = 0; n < P4_MAX_ESCR_PER_EVENT && !found; n++) {
1003 if ((escr = pevent->pm_escrs[n]) == P4_ESCR_NONE)
1004 break; /* out of ESCRs */
1005 /*
1006 * Check ESCR row disposition.
1007 *
1008 * If the request is for a system-mode PMC, then the
1009 * ESCR row should not be in process-virtual mode, and
1010 * should also be free on the current CPU.
1011 */
1012
920 if (PMC_IS_SYSTEM_MODE(pm->pm_mode)) {
1013 if (PMC_IS_SYSTEM_MODE(PMC_TO_MODE(pm))) {
921 if (P4_ESCR_ROW_DISP_IS_THREAD(escr) ||
922 pc->pc_escrs[escr] != P4_INVALID_PMC_INDEX)
923 continue;
924 }
925
926 /*
927 * If the request is for a process-virtual PMC, and if
928 * HTT is not enabled, we can use an ESCR row that is
929 * either FREE or already in process mode.
930 *
931 * If HTT is enabled, then we need to ensure that a
932 * given ESCR is never allocated to two PMCS that
933 * could run simultaneously on the two logical CPUs of
934 * a CPU package. We ensure this be only allocating
935 * ESCRs from rows marked as 'FREE'.
936 */
937
1014 if (P4_ESCR_ROW_DISP_IS_THREAD(escr) ||
1015 pc->pc_escrs[escr] != P4_INVALID_PMC_INDEX)
1016 continue;
1017 }
1018
1019 /*
1020 * If the request is for a process-virtual PMC, and if
1021 * HTT is not enabled, we can use an ESCR row that is
1022 * either FREE or already in process mode.
1023 *
1024 * If HTT is enabled, then we need to ensure that a
1025 * given ESCR is never allocated to two PMCS that
1026 * could run simultaneously on the two logical CPUs of
1027 * a CPU package. We ensure this be only allocating
1028 * ESCRs from rows marked as 'FREE'.
1029 */
1030
938 if (PMC_IS_VIRTUAL_MODE(pm->pm_mode)) {
1031 if (PMC_IS_VIRTUAL_MODE(PMC_TO_MODE(pm))) {
939 if (p4_system_has_htt) {
940 if (!P4_ESCR_ROW_DISP_IS_FREE(escr))
941 continue;
942 } else
943 if (P4_ESCR_ROW_DISP_IS_STANDALONE(escr))
944 continue;
945 }
946

--- 11 unchanged lines hidden (view full) ---

958
959 if (found == 0)
960 return ESRCH;
961
962 KASSERT((int) escr >= 0 && escr < P4_NESCR,
963 ("[p4,%d] illegal ESCR value %d", __LINE__, escr));
964
965 /* mark ESCR row mode */
1032 if (p4_system_has_htt) {
1033 if (!P4_ESCR_ROW_DISP_IS_FREE(escr))
1034 continue;
1035 } else
1036 if (P4_ESCR_ROW_DISP_IS_STANDALONE(escr))
1037 continue;
1038 }
1039

--- 11 unchanged lines hidden (view full) ---

1051
1052 if (found == 0)
1053 return ESRCH;
1054
1055 KASSERT((int) escr >= 0 && escr < P4_NESCR,
1056 ("[p4,%d] illegal ESCR value %d", __LINE__, escr));
1057
1058 /* mark ESCR row mode */
966 if (PMC_IS_SYSTEM_MODE(pm->pm_mode)) {
1059 if (PMC_IS_SYSTEM_MODE(PMC_TO_MODE(pm))) {
967 pc->pc_escrs[escr] = ri; /* mark ESCR as in use on this cpu */
968 P4_ESCR_MARK_ROW_STANDALONE(escr);
969 } else {
970 KASSERT(pc->pc_escrs[escr] == P4_INVALID_PMC_INDEX,
971 ("[p4,%d] escr[%d] already in use", __LINE__, escr));
972 P4_ESCR_MARK_ROW_THREAD(escr);
973 }
974

--- 44 unchanged lines hidden (view full) ---

1019 if (tflags == 0)
1020 tflags = (P4_ESCR_T0_OS|P4_ESCR_T0_USR);
1021 escrvalue |= tflags;
1022
1023 pm->pm_md.pm_p4.pm_p4_cccrvalue = cccrvalue;
1024 pm->pm_md.pm_p4.pm_p4_escrvalue = escrvalue;
1025
1026 PMCDBG(MDP,ALL,2, "p4-allocate cccrsel=0x%x cccrval=0x%x "
1060 pc->pc_escrs[escr] = ri; /* mark ESCR as in use on this cpu */
1061 P4_ESCR_MARK_ROW_STANDALONE(escr);
1062 } else {
1063 KASSERT(pc->pc_escrs[escr] == P4_INVALID_PMC_INDEX,
1064 ("[p4,%d] escr[%d] already in use", __LINE__, escr));
1065 P4_ESCR_MARK_ROW_THREAD(escr);
1066 }
1067

--- 44 unchanged lines hidden (view full) ---

1112 if (tflags == 0)
1113 tflags = (P4_ESCR_T0_OS|P4_ESCR_T0_USR);
1114 escrvalue |= tflags;
1115
1116 pm->pm_md.pm_p4.pm_p4_cccrvalue = cccrvalue;
1117 pm->pm_md.pm_p4.pm_p4_escrvalue = escrvalue;
1118
1119 PMCDBG(MDP,ALL,2, "p4-allocate cccrsel=0x%x cccrval=0x%x "
1027 "escr=%d escrmsr=0x%x escrval=0x%x\n", pevent->pm_cccr_select,
1120 "escr=%d escrmsr=0x%x escrval=0x%x", pevent->pm_cccr_select,
1028 cccrvalue, escr, pm->pm_md.pm_p4.pm_p4_escrmsr, escrvalue);
1029
1030 return 0;
1031}
1032
1033/*
1034 * release a PMC.
1035 */

--- 7 unchanged lines hidden (view full) ---

1043
1044 if (p4_pmcdesc[ri].pm_descr.pd_class == PMC_CLASS_TSC)
1045 return 0;
1046
1047 escr = pm->pm_md.pm_p4.pm_p4_escr;
1048
1049 PMCDBG(MDP,REL,1, "p4-release cpu=%d ri=%d escr=%d", cpu, ri, escr);
1050
1121 cccrvalue, escr, pm->pm_md.pm_p4.pm_p4_escrmsr, escrvalue);
1122
1123 return 0;
1124}
1125
1126/*
1127 * release a PMC.
1128 */

--- 7 unchanged lines hidden (view full) ---

1136
1137 if (p4_pmcdesc[ri].pm_descr.pd_class == PMC_CLASS_TSC)
1138 return 0;
1139
1140 escr = pm->pm_md.pm_p4.pm_p4_escr;
1141
1142 PMCDBG(MDP,REL,1, "p4-release cpu=%d ri=%d escr=%d", cpu, ri, escr);
1143
1051 if (PMC_IS_SYSTEM_MODE(pm->pm_mode)) {
1144 if (PMC_IS_SYSTEM_MODE(PMC_TO_MODE(pm))) {
1052 pc = (struct p4_cpu *) pmc_pcpu[P4_TO_PHYSICAL_CPU(cpu)];
1053 phw = pc->pc_hwpmcs[ri];
1054
1055 KASSERT(phw->phw_pmc == NULL,
1056 ("[p4,%d] releasing configured PMC ri=%d", __LINE__, ri));
1057
1058 P4_ESCR_UNMARK_ROW_STANDALONE(escr);
1059 KASSERT(pc->pc_escrs[escr] == ri,

--- 55 unchanged lines hidden (view full) ---

1115 escrvalue &= ~(P4_ESCR_T0_OS|P4_ESCR_T0_USR);
1116
1117 if (pmc_cpu_is_logical(cpu)) { /* shift T0 bits to T1 position */
1118 cccrtbits <<= 1;
1119 escrtbits >>= 2;
1120 }
1121
1122 /* start system mode PMCs directly */
1145 pc = (struct p4_cpu *) pmc_pcpu[P4_TO_PHYSICAL_CPU(cpu)];
1146 phw = pc->pc_hwpmcs[ri];
1147
1148 KASSERT(phw->phw_pmc == NULL,
1149 ("[p4,%d] releasing configured PMC ri=%d", __LINE__, ri));
1150
1151 P4_ESCR_UNMARK_ROW_STANDALONE(escr);
1152 KASSERT(pc->pc_escrs[escr] == ri,

--- 55 unchanged lines hidden (view full) ---

1208 escrvalue &= ~(P4_ESCR_T0_OS|P4_ESCR_T0_USR);
1209
1210 if (pmc_cpu_is_logical(cpu)) { /* shift T0 bits to T1 position */
1211 cccrtbits <<= 1;
1212 escrtbits >>= 2;
1213 }
1214
1215 /* start system mode PMCs directly */
1123 if (PMC_IS_SYSTEM_MODE(pm->pm_mode)) {
1216 if (PMC_IS_SYSTEM_MODE(PMC_TO_MODE(pm))) {
1124 wrmsr(escrmsr, escrvalue | escrtbits);
1125 wrmsr(pd->pm_cccr_msr, cccrvalue | cccrtbits | P4_CCCR_ENABLE);
1126 return 0;
1127 }
1128
1129 /*
1130 * Thread mode PMCs
1131 *

--- 7 unchanged lines hidden (view full) ---

1139 mtx_lock_spin(&pc->pc_mtx);
1140
1141 rc = P4_PCPU_GET_RUNCOUNT(pc,ri);
1142 KASSERT(rc == 0 || rc == 1,
1143 ("[p4,%d] illegal runcount cpu=%d ri=%d rc=%d", __LINE__, cpu, ri,
1144 rc));
1145
1146 if (rc == 0) { /* 1st CPU and the non-HTT case */
1217 wrmsr(escrmsr, escrvalue | escrtbits);
1218 wrmsr(pd->pm_cccr_msr, cccrvalue | cccrtbits | P4_CCCR_ENABLE);
1219 return 0;
1220 }
1221
1222 /*
1223 * Thread mode PMCs
1224 *

--- 7 unchanged lines hidden (view full) ---

1232 mtx_lock_spin(&pc->pc_mtx);
1233
1234 rc = P4_PCPU_GET_RUNCOUNT(pc,ri);
1235 KASSERT(rc == 0 || rc == 1,
1236 ("[p4,%d] illegal runcount cpu=%d ri=%d rc=%d", __LINE__, cpu, ri,
1237 rc));
1238
1239 if (rc == 0) { /* 1st CPU and the non-HTT case */
1147 /*
1148 * Enable the correct bits for this CPU.
1149 */
1150 escrvalue |= escrtbits;
1151 cccrvalue |= cccrtbits | P4_CCCR_ENABLE;
1152
1153 KASSERT(P4_PMC_IS_STOPPED(pd->pm_cccr_msr),
1154 ("[p4,%d] cpu=%d ri=%d cccr=0x%x not stopped", __LINE__,
1155 cpu, ri, pd->pm_cccr_msr));
1156
1157 /* write out the low 40 bits of the saved value to hardware */
1158 wrmsr(pd->pm_pmc_msr,
1159 P4_PCPU_PMC_VALUE(pc,ri,cpu) & P4_PERFCTR_MASK);
1240
1241 KASSERT(P4_PMC_IS_STOPPED(pd->pm_cccr_msr),
1242 ("[p4,%d] cpu=%d ri=%d cccr=0x%x not stopped", __LINE__,
1243 cpu, ri, pd->pm_cccr_msr));
1244
1245 /* write out the low 40 bits of the saved value to hardware */
1246 wrmsr(pd->pm_pmc_msr,
1247 P4_PCPU_PMC_VALUE(pc,ri,cpu) & P4_PERFCTR_MASK);
1160 P4_PCPU_SAVED_VALUE(pc,ri,cpu) = P4_PCPU_PMC_VALUE(pc,ri,cpu) &
1161 P4_PERFCTR_MASK;
1162
1248
1163 /* Program the ESCR and CCCR and start the PMC */
1164 wrmsr(escrmsr, escrvalue);
1165 wrmsr(pd->pm_cccr_msr, cccrvalue);
1166
1167 PMCDBG(MDP,STA,2,"p4-start cpu=%d rc=%d ri=%d escr=%d "
1168 "escrmsr=0x%x escrvalue=0x%x cccr_config=0x%x\n", cpu, rc,
1169 ri, pm->pm_md.pm_p4.pm_p4_escr, escrmsr, escrvalue,
1170 cccrvalue);
1171
1172 } else if (rc == 1) { /* 2nd CPU */
1173
1174 /*
1249 } else if (rc == 1) { /* 2nd CPU */
1250
1251 /*
1175 * Retrieve the CCCR and ESCR values from their MSRs,
1176 * and turn on the addition T[0/1] bits for the 2nd
1177 * CPU. Remember the difference between the saved
1178 * value from the previous 'write()' operation to this
1179 * (PMC,CPU) pair and the current PMC reading; this is
1180 * used at PMCSTOP time to derive the correct
1181 * increment.
1252 * Stop the PMC and retrieve the CCCR and ESCR values
1253 * from their MSRs, and turn on the additional T[0/1]
1254 * bits for the 2nd CPU.
1182 */
1183
1184 cccrvalue = rdmsr(pd->pm_cccr_msr);
1255 */
1256
1257 cccrvalue = rdmsr(pd->pm_cccr_msr);
1258 wrmsr(pd->pm_cccr_msr, cccrvalue & ~P4_CCCR_ENABLE);
1185
1259
1260 /* check that the configuration bits read back match the PMC */
1186 KASSERT((cccrvalue & P4_CCCR_Tx_MASK) ==
1187 (pm->pm_md.pm_p4.pm_p4_cccrvalue & P4_CCCR_Tx_MASK),
1261 KASSERT((cccrvalue & P4_CCCR_Tx_MASK) ==
1262 (pm->pm_md.pm_p4.pm_p4_cccrvalue & P4_CCCR_Tx_MASK),
1188 ("[p4,%d] cpu=%d rc=%d ri=%d CCCR bits 0x%x PMC 0x%x",
1189 __LINE__, cpu, rc, ri, cccrvalue & P4_CCCR_Tx_MASK,
1263 ("[p4,%d] Extra CCCR bits cpu=%d rc=%d ri=%d "
1264 "cccr=0x%x PMC=0x%x", __LINE__, cpu, rc, ri,
1265 cccrvalue & P4_CCCR_Tx_MASK,
1190 pm->pm_md.pm_p4.pm_p4_cccrvalue & P4_CCCR_Tx_MASK));
1191 KASSERT(cccrvalue & P4_CCCR_ENABLE,
1192 ("[p4,%d] 2nd cpu rc=%d cpu=%d ri=%d not running",
1193 __LINE__, rc, cpu, ri));
1194 KASSERT((cccrvalue & cccrtbits) == 0,
1195 ("[p4,%d] CCCR T0/T1 mismatch rc=%d cpu=%d ri=%d"
1196 "cccrvalue=0x%x tbits=0x%x", __LINE__, rc, cpu, ri,
1197 cccrvalue, cccrtbits));
1198
1266 pm->pm_md.pm_p4.pm_p4_cccrvalue & P4_CCCR_Tx_MASK));
1267 KASSERT(cccrvalue & P4_CCCR_ENABLE,
1268 ("[p4,%d] 2nd cpu rc=%d cpu=%d ri=%d not running",
1269 __LINE__, rc, cpu, ri));
1270 KASSERT((cccrvalue & cccrtbits) == 0,
1271 ("[p4,%d] CCCR T0/T1 mismatch rc=%d cpu=%d ri=%d"
1272 "cccrvalue=0x%x tbits=0x%x", __LINE__, rc, cpu, ri,
1273 cccrvalue, cccrtbits));
1274
1199 /* stop PMC */
1200 wrmsr(pd->pm_cccr_msr, cccrvalue & ~P4_CCCR_ENABLE);
1201
1202 escrvalue = rdmsr(escrmsr);
1203
1204 KASSERT((escrvalue & P4_ESCR_Tx_MASK) ==
1205 (pm->pm_md.pm_p4.pm_p4_escrvalue & P4_ESCR_Tx_MASK),
1206 ("[p4,%d] Extra ESCR bits cpu=%d rc=%d ri=%d "
1207 "escr=0x%x pm=0x%x", __LINE__, cpu, rc, ri,
1208 escrvalue & P4_ESCR_Tx_MASK,
1209 pm->pm_md.pm_p4.pm_p4_escrvalue & P4_ESCR_Tx_MASK));
1275 escrvalue = rdmsr(escrmsr);
1276
1277 KASSERT((escrvalue & P4_ESCR_Tx_MASK) ==
1278 (pm->pm_md.pm_p4.pm_p4_escrvalue & P4_ESCR_Tx_MASK),
1279 ("[p4,%d] Extra ESCR bits cpu=%d rc=%d ri=%d "
1280 "escr=0x%x pm=0x%x", __LINE__, cpu, rc, ri,
1281 escrvalue & P4_ESCR_Tx_MASK,
1282 pm->pm_md.pm_p4.pm_p4_escrvalue & P4_ESCR_Tx_MASK));
1210
1211 KASSERT((escrvalue & escrtbits) == 0,
1212 ("[p4,%d] ESCR T0/T1 mismatch rc=%d cpu=%d ri=%d "
1213 "escrmsr=0x%x escrvalue=0x%x tbits=0x%x", __LINE__,
1214 rc, cpu, ri, escrmsr, escrvalue, escrtbits));
1283 KASSERT((escrvalue & escrtbits) == 0,
1284 ("[p4,%d] ESCR T0/T1 mismatch rc=%d cpu=%d ri=%d "
1285 "escrmsr=0x%x escrvalue=0x%x tbits=0x%x", __LINE__,
1286 rc, cpu, ri, escrmsr, escrvalue, escrtbits));
1287 }
1215
1288
1216 /* read current value and save it */
1217 P4_PCPU_SAVED_VALUE(pc,ri,cpu) =
1218 rdmsr(pd->pm_pmc_msr) & P4_PERFCTR_MASK;
1289 /* Enable the correct bits for this CPU. */
1290 escrvalue |= escrtbits;
1291 cccrvalue |= cccrtbits | P4_CCCR_ENABLE;
1219
1292
1220 /*
1221 * program the new bits into the ESCR and CCCR,
1222 * starting the PMC in the process.
1223 */
1293 /* Save HW value at the time of starting hardware */
1294 P4_PCPU_HW_VALUE(pc,ri,cpu) = rdmsr(pd->pm_pmc_msr);
1224
1295
1225 escrvalue |= escrtbits;
1226 cccrvalue |= cccrvalue;
1296 /* Program the ESCR and CCCR and start the PMC */
1297 wrmsr(escrmsr, escrvalue);
1298 wrmsr(pd->pm_cccr_msr, cccrvalue);
1227
1299
1228 wrmsr(escrmsr, escrvalue);
1229 wrmsr(pd->pm_cccr_msr, cccrvalue);
1230
1231 PMCDBG(MDP,STA,2,"p4-start/2 cpu=%d rc=%d ri=%d escr=%d"
1232 "escrmsr=0x%x escrvalue=0x%x cccr_config=0x%x",
1233 cpu, rc, ri, pm->pm_md.pm_p4.pm_p4_escr, escrmsr,
1234 escrvalue, cccrvalue);
1235
1236 } else
1237 panic("invalid runcount %d\n", rc);
1238
1239 ++rc;
1240 P4_PCPU_SET_RUNCOUNT(pc,ri,rc);
1241
1242 mtx_unlock_spin(&pc->pc_mtx);
1243
1300 ++rc;
1301 P4_PCPU_SET_RUNCOUNT(pc,ri,rc);
1302
1303 mtx_unlock_spin(&pc->pc_mtx);
1304
1305 PMCDBG(MDP,STA,2,"p4-start cpu=%d rc=%d ri=%d escr=%d "
1306 "escrmsr=0x%x escrvalue=0x%x cccr_config=0x%x v=%jx", cpu, rc,
1307 ri, pm->pm_md.pm_p4.pm_p4_escr, escrmsr, escrvalue,
1308 cccrvalue, P4_PCPU_HW_VALUE(pc,ri,cpu));
1309
1244 return 0;
1245}
1246
1247/*
1248 * Stop a PMC.
1249 */
1250
1251static int

--- 25 unchanged lines hidden (view full) ---

1277
1278 pm = phw->phw_pmc;
1279
1280 KASSERT(pm != NULL,
1281 ("[p4,%d] null pmc for cpu%d, ri%d", __LINE__, cpu, ri));
1282
1283 PMCDBG(MDP,STO,1, "p4-stop cpu=%d ri=%d", cpu, ri);
1284
1310 return 0;
1311}
1312
1313/*
1314 * Stop a PMC.
1315 */
1316
1317static int

--- 25 unchanged lines hidden (view full) ---

1343
1344 pm = phw->phw_pmc;
1345
1346 KASSERT(pm != NULL,
1347 ("[p4,%d] null pmc for cpu%d, ri%d", __LINE__, cpu, ri));
1348
1349 PMCDBG(MDP,STO,1, "p4-stop cpu=%d ri=%d", cpu, ri);
1350
1285 if (PMC_IS_SYSTEM_MODE(pm->pm_mode)) {
1351 if (PMC_IS_SYSTEM_MODE(PMC_TO_MODE(pm))) {
1286 wrmsr(pd->pm_cccr_msr,
1287 pm->pm_md.pm_p4.pm_p4_cccrvalue & ~P4_CCCR_ENABLE);
1288 return 0;
1289 }
1290
1291 /*
1292 * Thread mode PMCs.
1293 *
1294 * On HTT machines, this PMC may be in use by two threads
1295 * running on two logical CPUS. Thus we look at the
1296 * 'pm_runcount' field and only turn off the appropriate TO/T1
1352 wrmsr(pd->pm_cccr_msr,
1353 pm->pm_md.pm_p4.pm_p4_cccrvalue & ~P4_CCCR_ENABLE);
1354 return 0;
1355 }
1356
1357 /*
1358 * Thread mode PMCs.
1359 *
1360 * On HTT machines, this PMC may be in use by two threads
1361 * running on two logical CPUS. Thus we look at the
1362 * 'pm_runcount' field and only turn off the appropriate TO/T1
1297 * bits (and keep the PMC running).
1363 * bits (and keep the PMC running) if two logical CPUs were
1364 * using the PMC.
1298 *
1365 *
1299 * The 'pc_saved' field has the 'diff' between the value in
1300 * the hardware register at PMCSTART time and the nominal
1301 * start value for the PMC. This diff is added to the current
1302 * PMC reading to derived the correct (absolute) return value.
1303 */
1304
1305 /* bits to mask */
1306 cccrtbits = P4_CCCR_OVF_PMI_T0;
1307 escrtbits = P4_ESCR_T0_OS | P4_ESCR_T0_USR;
1308 if (pmc_cpu_is_logical(cpu)) {
1309 cccrtbits <<= 1;
1310 escrtbits >>= 2;

--- 13 unchanged lines hidden (view full) ---

1324
1325 /* Stop this PMC */
1326 cccrvalue = rdmsr(pd->pm_cccr_msr);
1327 wrmsr(pd->pm_cccr_msr, cccrvalue & ~P4_CCCR_ENABLE);
1328
1329 escrmsr = pm->pm_md.pm_p4.pm_p4_escrmsr;
1330 escrvalue = rdmsr(escrmsr);
1331
1366 */
1367
1368 /* bits to mask */
1369 cccrtbits = P4_CCCR_OVF_PMI_T0;
1370 escrtbits = P4_ESCR_T0_OS | P4_ESCR_T0_USR;
1371 if (pmc_cpu_is_logical(cpu)) {
1372 cccrtbits <<= 1;
1373 escrtbits >>= 2;

--- 13 unchanged lines hidden (view full) ---

1387
1388 /* Stop this PMC */
1389 cccrvalue = rdmsr(pd->pm_cccr_msr);
1390 wrmsr(pd->pm_cccr_msr, cccrvalue & ~P4_CCCR_ENABLE);
1391
1392 escrmsr = pm->pm_md.pm_p4.pm_p4_escrmsr;
1393 escrvalue = rdmsr(escrmsr);
1394
1332 /* get the current PMC reading */
1333 tmp = rdmsr(pd->pm_pmc_msr) & P4_PERFCTR_MASK;
1395 /* The current CPU should be running on this PMC */
1396 KASSERT(escrvalue & escrtbits,
1397 ("[p4,%d] ESCR T0/T1 mismatch cpu=%d rc=%d ri=%d escrmsr=0x%x "
1398 "escrvalue=0x%x tbits=0x%x", __LINE__, cpu, rc, ri, escrmsr,
1399 escrvalue, escrtbits));
1400 KASSERT(PMC_IS_COUNTING_MODE(PMC_TO_MODE(pm)) ||
1401 (cccrvalue & cccrtbits),
1402 ("[p4,%d] CCCR T0/T1 mismatch cpu=%d ri=%d cccrvalue=0x%x "
1403 "tbits=0x%x", __LINE__, cpu, ri, cccrvalue, cccrtbits));
1334
1404
1335 if (rc == 1) { /* need to keep the PMC running */
1405 /* get the current hardware reading */
1406 tmp = rdmsr(pd->pm_pmc_msr);
1336
1407
1337 KASSERT(escrvalue & escrtbits,
1338 ("[p4,%d] ESCR T0/T1 mismatch cpu=%d ri=%d escrmsr=0x%x "
1339 "escrvalue=0x%x tbits=0x%x", __LINE__, cpu, ri, escrmsr,
1340 escrvalue, escrtbits));
1341
1342 KASSERT(PMC_IS_COUNTING_MODE(pm->pm_mode) ||
1343 (cccrvalue & cccrtbits),
1344 ("[p4,%d] CCCR T0/T1 mismatch cpu=%d ri=%d cccrvalue=0x%x "
1345 "tbits=0x%x", __LINE__, cpu, ri, cccrvalue, cccrtbits));
1346
1408 if (rc == 1) { /* need to keep the PMC running */
1347 escrvalue &= ~escrtbits;
1348 cccrvalue &= ~cccrtbits;
1409 escrvalue &= ~escrtbits;
1410 cccrvalue &= ~cccrtbits;
1349
1350 wrmsr(escrmsr, escrvalue);
1351 wrmsr(pd->pm_cccr_msr, cccrvalue);
1411 wrmsr(escrmsr, escrvalue);
1412 wrmsr(pd->pm_cccr_msr, cccrvalue);
1352
1353 }
1354
1413 }
1414
1355 PMCDBG(MDP,STO,2, "p4-stop/2 cpu=%d rc=%d ri=%d escrmsr=0x%x escrval=0x%x "
1356 "cccrval=0x%x", cpu, rc, ri, escrmsr, escrvalue, cccrvalue);
1415 mtx_unlock_spin(&pc->pc_mtx);
1357
1416
1358 /* get the incremental count from this context switch */
1359 tmp -= P4_PCPU_SAVED_VALUE(pc,ri,cpu);
1360 if ((int64_t) tmp < 0) /* counter wrap-around */
1361 tmp = -tmp + 1;
1417 PMCDBG(MDP,STO,2, "p4-stop cpu=%d rc=%d ri=%d escrmsr=0x%x "
1418 "escrval=0x%x cccrval=0x%x v=%jx", cpu, rc, ri, escrmsr,
1419 escrvalue, cccrvalue, tmp);
1362
1420
1421 if (tmp < P4_PCPU_HW_VALUE(pc,ri,cpu)) /* 40 bit counter overflow */
1422 tmp += (P4_PERFCTR_MASK + 1) - P4_PCPU_HW_VALUE(pc,ri,cpu);
1423 else
1424 tmp -= P4_PCPU_HW_VALUE(pc,ri,cpu);
1425
1363 P4_PCPU_PMC_VALUE(pc,ri,cpu) += tmp;
1364
1426 P4_PCPU_PMC_VALUE(pc,ri,cpu) += tmp;
1427
1365 mtx_unlock_spin(&pc->pc_mtx);
1366 return 0;
1367}
1368
1369/*
1370 * Handle an interrupt.
1428 return 0;
1429}
1430
1431/*
1432 * Handle an interrupt.
1433 *
1434 * The hardware sets the CCCR_OVF whenever a counter overflow occurs, so the handler
1435 * examines all the 18 CCCR registers, processing the counters that have overflowed.
1436 *
1437 * On HTT machines, multiple logical CPUs may try to enter the NMI service
1438 * routine at the same time.
1371 */
1372
1439 */
1440
1441extern volatile lapic_t *lapic;
1442
1443static void
1444p4_lapic_enable_pmc_interrupt(void)
1445{
1446 uint32_t value;
1447
1448 value = lapic->lvt_pcint;
1449 value &= ~APIC_LVT_M;
1450 lapic->lvt_pcint = value;
1451}
1452
1453
1373static int
1374p4_intr(int cpu, uintptr_t eip)
1375{
1454static int
1455p4_intr(int cpu, uintptr_t eip)
1456{
1376 (void) cpu;
1457 int i, pmc_interrupted;
1458 uint32_t cccrval, pmi_ovf_mask;
1459 struct p4_cpu *pc;
1460 struct pmc_hw *phw;
1461 struct pmc *pm;
1462 pmc_value_t v;
1463
1377 (void) eip;
1464 (void) eip;
1465 PMCDBG(MDP,INT, 1, "cpu=%d eip=%x pcint=0x%x", cpu, eip,
1466 lapic->lvt_pcint);
1378
1467
1379 return 0;
1468 pmc_interrupted = 0;
1469 pc = (struct p4_cpu *) pmc_pcpu[cpu];
1470
1471 pmi_ovf_mask = pmc_cpu_is_logical(cpu) ?
1472 P4_CCCR_OVF_PMI_T1 : P4_CCCR_OVF_PMI_T0;
1473 pmi_ovf_mask |= P4_CCCR_OVF;
1474
1475 /*
1476 * Loop through all CCCRs, looking for ones that have the
1477 * OVF_PMI bit set for our logical CPU.
1478 */
1479
1480 for (i = 1; i < P4_NPMCS; i++) {
1481 cccrval = rdmsr(P4_CCCR_MSR_FIRST + i - 1);
1482
1483 if ((cccrval & pmi_ovf_mask) != pmi_ovf_mask)
1484 continue;
1485
1486 v = rdmsr(P4_PERFCTR_MSR_FIRST + i - 1);
1487
1488 pmc_interrupted = 1;
1489
1490 PMCDBG(MDP,INT, 2, "ri=%d v=%jx", i, v);
1491
1492 /* Stop the counter, and turn off the overflow bit */
1493 cccrval &= ~(P4_CCCR_OVF | P4_CCCR_ENABLE);
1494 wrmsr(P4_CCCR_MSR_FIRST + i - 1, cccrval);
1495
1496 phw = pc->pc_hwpmcs[i];
1497 pm = phw->phw_pmc;
1498
1499 /*
1500 * Ignore de-configured or stopped PMCs.
1501 * Also ignore counting mode PMCs that may
1502 * have overflowed their counters.
1503 */
1504 if (pm == NULL ||
1505 pm->pm_state != PMC_STATE_RUNNING ||
1506 !PMC_IS_SAMPLING_MODE(PMC_TO_MODE(pm)))
1507 continue;
1508
1509 /*
1510 * If the previous sample hasn't been read yet, the
1511 * sampling interrupt is coming in too fast for the
1512 * rest of the system to cope. Do not re-enable the
1513 * counter.
1514 */
1515
1516 if (P4_PCPU_SAVED_IP(pc,i,cpu)) {
1517 atomic_add_int(&pmc_stats.pm_intr_ignored, 1);
1518 continue;
1519 }
1520
1521 /*
1522 * write the the reload count and restart the
1523 * hardware.
1524 */
1525
1526 v = P4_RELOAD_COUNT_TO_PERFCTR_VALUE(
1527 pm->pm_sc.pm_reloadcount);
1528 wrmsr(P4_PERFCTR_MSR_FIRST + i - 1, v);
1529 wrmsr(P4_CCCR_MSR_FIRST + i - 1,
1530 cccrval | P4_CCCR_ENABLE);
1531 }
1532
1533 if (pmc_interrupted) {
1534
1535 /*
1536 * On Intel CPUs, the PMC 'pcint' entry in the LAPIC
1537 * gets masked when a PMC interrupts the CPU. We need
1538 * to unmask this.
1539 */
1540 p4_lapic_enable_pmc_interrupt();
1541
1542 /* XXX: Invoke helper (non-NMI) interrupt here */
1543 }
1544
1545 return pmc_interrupted;
1380}
1381
1382/*
1383 * Describe a CPU's PMC state.
1384 */
1385
1386static int
1387p4_describe(int cpu, int ri, struct pmc_info *pi,

--- 17 unchanged lines hidden (view full) ---

1405 phw = pmc_pcpu[cpu]->pc_hwpmcs[ri];
1406 pd = &p4_pmcdesc[ri];
1407
1408 if ((error = copystr(pd->pm_descr.pd_name, pi->pm_name,
1409 PMC_NAME_MAX, &copied)) != 0)
1410 return error;
1411
1412 pi->pm_class = pd->pm_descr.pd_class;
1546}
1547
1548/*
1549 * Describe a CPU's PMC state.
1550 */
1551
1552static int
1553p4_describe(int cpu, int ri, struct pmc_info *pi,

--- 17 unchanged lines hidden (view full) ---

1571 phw = pmc_pcpu[cpu]->pc_hwpmcs[ri];
1572 pd = &p4_pmcdesc[ri];
1573
1574 if ((error = copystr(pd->pm_descr.pd_name, pi->pm_name,
1575 PMC_NAME_MAX, &copied)) != 0)
1576 return error;
1577
1578 pi->pm_class = pd->pm_descr.pd_class;
1413 pi->pm_caps = pd->pm_descr.pd_caps;
1414 pi->pm_width = pd->pm_descr.pd_width;
1415
1416 if (phw->phw_state & PMC_PHW_FLAG_IS_ENABLED) {
1417 pi->pm_enabled = TRUE;
1418 *ppmc = phw->phw_pmc;
1419 } else {
1420 pi->pm_enabled = FALSE;
1421 *ppmc = NULL;
1422 }

--- 28 unchanged lines hidden (view full) ---

1451 ("[p4,%d] Initializing non-intel processor", __LINE__));
1452
1453 PMCDBG(MDP,INI,1, "%s", "p4-initialize");
1454
1455 switch (pmc_mdep->pmd_cputype) {
1456 case PMC_CPU_INTEL_PIV:
1457
1458 pmc_mdep->pmd_npmc = P4_NPMCS;
1579
1580 if (phw->phw_state & PMC_PHW_FLAG_IS_ENABLED) {
1581 pi->pm_enabled = TRUE;
1582 *ppmc = phw->phw_pmc;
1583 } else {
1584 pi->pm_enabled = FALSE;
1585 *ppmc = NULL;
1586 }

--- 28 unchanged lines hidden (view full) ---

1615 ("[p4,%d] Initializing non-intel processor", __LINE__));
1616
1617 PMCDBG(MDP,INI,1, "%s", "p4-initialize");
1618
1619 switch (pmc_mdep->pmd_cputype) {
1620 case PMC_CPU_INTEL_PIV:
1621
1622 pmc_mdep->pmd_npmc = P4_NPMCS;
1459 pmc_mdep->pmd_classes[1] = PMC_CLASS_P4;
1623 pmc_mdep->pmd_classes[1].pm_class = PMC_CLASS_P4;
1624 pmc_mdep->pmd_classes[1].pm_caps = P4_PMC_CAPS;
1625 pmc_mdep->pmd_classes[1].pm_width = 40;
1460 pmc_mdep->pmd_nclasspmcs[1] = 18;
1461
1462 pmc_mdep->pmd_init = p4_init;
1463 pmc_mdep->pmd_cleanup = p4_cleanup;
1464 pmc_mdep->pmd_switch_in = p4_switch_in;
1465 pmc_mdep->pmd_switch_out = p4_switch_out;
1466 pmc_mdep->pmd_read_pmc = p4_read_pmc;
1467 pmc_mdep->pmd_write_pmc = p4_write_pmc;
1468 pmc_mdep->pmd_config_pmc = p4_config_pmc;
1626 pmc_mdep->pmd_nclasspmcs[1] = 18;
1627
1628 pmc_mdep->pmd_init = p4_init;
1629 pmc_mdep->pmd_cleanup = p4_cleanup;
1630 pmc_mdep->pmd_switch_in = p4_switch_in;
1631 pmc_mdep->pmd_switch_out = p4_switch_out;
1632 pmc_mdep->pmd_read_pmc = p4_read_pmc;
1633 pmc_mdep->pmd_write_pmc = p4_write_pmc;
1634 pmc_mdep->pmd_config_pmc = p4_config_pmc;
1635 pmc_mdep->pmd_get_config = p4_get_config;
1469 pmc_mdep->pmd_allocate_pmc = p4_allocate_pmc;
1470 pmc_mdep->pmd_release_pmc = p4_release_pmc;
1471 pmc_mdep->pmd_start_pmc = p4_start_pmc;
1472 pmc_mdep->pmd_stop_pmc = p4_stop_pmc;
1473 pmc_mdep->pmd_intr = p4_intr;
1474 pmc_mdep->pmd_describe = p4_describe;
1475 pmc_mdep->pmd_get_msr = p4_get_msr; /* i386 */
1476

--- 22 unchanged lines hidden --- 		1636 pmc_mdep->pmd_allocate_pmc = p4_allocate_pmc;
1637 pmc_mdep->pmd_release_pmc = p4_release_pmc;
1638 pmc_mdep->pmd_start_pmc = p4_start_pmc;
1639 pmc_mdep->pmd_stop_pmc = p4_stop_pmc;
1640 pmc_mdep->pmd_intr = p4_intr;
1641 pmc_mdep->pmd_describe = p4_describe;
1642 pmc_mdep->pmd_get_msr = p4_get_msr; /* i386 */
1643

--- 22 unchanged lines hidden ---