Deleted Added
sdiff udiff text old ( 210012 ) new ( 210621 )
full compact
hwpmc_core.c (210012) hwpmc_core.c (210621)
1/*-
2 * Copyright (c) 2008 Joseph Koshy
3 * All rights reserved.
4 *
5 * Redistribution and use in source and binary forms, with or without
6 * modification, are permitted provided that the following conditions
7 * are met:
8 * 1. Redistributions of source code must retain the above copyright

--- 15 unchanged lines hidden (view full) ---

24 * SUCH DAMAGE.
25 */
26
27/*
28 * Intel Core, Core 2 and Atom PMCs.
29 */
30
31#include <sys/cdefs.h>
1/*-
2 * Copyright (c) 2008 Joseph Koshy
3 * All rights reserved.
4 *
5 * Redistribution and use in source and binary forms, with or without
6 * modification, are permitted provided that the following conditions
7 * are met:
8 * 1. Redistributions of source code must retain the above copyright

--- 15 unchanged lines hidden (view full) ---

24 * SUCH DAMAGE.
25 */
26
27/*
28 * Intel Core, Core 2 and Atom PMCs.
29 */
30
31#include <sys/cdefs.h>
32__FBSDID("$FreeBSD: head/sys/dev/hwpmc/hwpmc_core.c 210012 2010-07-13 19:37:45Z gnn $");
32__FBSDID("$FreeBSD: head/sys/dev/hwpmc/hwpmc_core.c 210621 2010-07-29 17:52:23Z gnn $");
33
34#include <sys/param.h>
35#include <sys/bus.h>
36#include <sys/pmc.h>
37#include <sys/pmckern.h>
38#include <sys/systm.h>
39
40#include <machine/intr_machdep.h>

--- 122 unchanged lines hidden (view full) ---

163
164 KASSERT(pc != NULL, ("[core,%d] NULL per-cpu %d state", __LINE__,
165 cpu));
166
167 npmc = md->pmd_classdep[PMC_MDEP_CLASS_INDEX_IAP].pcd_num;
168 core_ri = md->pmd_classdep[PMC_MDEP_CLASS_INDEX_IAP].pcd_ri;
169
170 for (n = 0; n < npmc; n++) {
33
34#include <sys/param.h>
35#include <sys/bus.h>
36#include <sys/pmc.h>
37#include <sys/pmckern.h>
38#include <sys/systm.h>
39
40#include <machine/intr_machdep.h>

--- 122 unchanged lines hidden (view full) ---

163
164 KASSERT(pc != NULL, ("[core,%d] NULL per-cpu %d state", __LINE__,
165 cpu));
166
167 npmc = md->pmd_classdep[PMC_MDEP_CLASS_INDEX_IAP].pcd_num;
168 core_ri = md->pmd_classdep[PMC_MDEP_CLASS_INDEX_IAP].pcd_ri;
169
170 for (n = 0; n < npmc; n++) {
171 msr = rdmsr(IAP_EVSEL0 + n);
172 wrmsr(IAP_EVSEL0 + n, msr & ~IAP_EVSEL_MASK);
171 msr = rdmsr(IAP_EVSEL0 + n) & ~IAP_EVSEL_MASK;
172 wrmsr(IAP_EVSEL0 + n, msr);
173 }
174
175 if (core_cputype != PMC_CPU_INTEL_CORE) {
173 }
174
175 if (core_cputype != PMC_CPU_INTEL_CORE) {
176 msr = rdmsr(IAF_CTRL);
177 wrmsr(IAF_CTRL, msr & ~IAF_CTRL_MASK);
176 msr = rdmsr(IAF_CTRL) & ~IAF_CTRL_MASK;
177 wrmsr(IAF_CTRL, msr);
178 npmc += md->pmd_classdep[PMC_MDEP_CLASS_INDEX_IAF].pcd_num;
179 }
180
181 for (n = 0; n < npmc; n++)
182 pc->pc_hwpmcs[n + core_ri] = NULL;
183
184 free(cc, M_PMC);
185

--- 201 unchanged lines hidden (view full) ---

387
388 PMCDBG(MDP,STA,1,"iaf-start cpu=%d ri=%d", cpu, ri);
389
390 iafc = core_pcpu[cpu];
391 pm = iafc->pc_corepmcs[ri + core_iaf_ri].phw_pmc;
392
393 iafc->pc_iafctrl |= pm->pm_md.pm_iaf.pm_iaf_ctrl;
394
178 npmc += md->pmd_classdep[PMC_MDEP_CLASS_INDEX_IAF].pcd_num;
179 }
180
181 for (n = 0; n < npmc; n++)
182 pc->pc_hwpmcs[n + core_ri] = NULL;
183
184 free(cc, M_PMC);
185

--- 201 unchanged lines hidden (view full) ---

387
388 PMCDBG(MDP,STA,1,"iaf-start cpu=%d ri=%d", cpu, ri);
389
390 iafc = core_pcpu[cpu];
391 pm = iafc->pc_corepmcs[ri + core_iaf_ri].phw_pmc;
392
393 iafc->pc_iafctrl |= pm->pm_md.pm_iaf.pm_iaf_ctrl;
394
395 msr = rdmsr(IAF_CTRL);
395 msr = rdmsr(IAF_CTRL) & ~IAF_CTRL_MASK;
396 wrmsr(IAF_CTRL, msr | (iafc->pc_iafctrl & IAF_CTRL_MASK));
397
398 do {
399 iafc->pc_resync = 0;
400 iafc->pc_globalctrl |= (1ULL << (ri + IAF_OFFSET));
396 wrmsr(IAF_CTRL, msr | (iafc->pc_iafctrl & IAF_CTRL_MASK));
397
398 do {
399 iafc->pc_resync = 0;
400 iafc->pc_globalctrl |= (1ULL << (ri + IAF_OFFSET));
401 msr = rdmsr(IA_GLOBAL_CTRL);
401 msr = rdmsr(IA_GLOBAL_CTRL) & ~IAF_GLOBAL_CTRL_MASK;
402 wrmsr(IA_GLOBAL_CTRL, msr | (iafc->pc_globalctrl &
403 IAF_GLOBAL_CTRL_MASK));
404 } while (iafc->pc_resync != 0);
405
406 PMCDBG(MDP,STA,1,"iafctrl=%x(%x) globalctrl=%jx(%jx)",
407 iafc->pc_iafctrl, (uint32_t) rdmsr(IAF_CTRL),
408 iafc->pc_globalctrl, rdmsr(IA_GLOBAL_CTRL));
409

--- 19 unchanged lines hidden (view full) ---

429 fc = (IAF_MASK << (ri * 4));
430
431 if (core_cputype != PMC_CPU_INTEL_ATOM)
432 fc &= ~IAF_ANY;
433
434 iafc->pc_iafctrl &= ~fc;
435
436 PMCDBG(MDP,STO,1,"iaf-stop iafctrl=%x", iafc->pc_iafctrl);
402 wrmsr(IA_GLOBAL_CTRL, msr | (iafc->pc_globalctrl &
403 IAF_GLOBAL_CTRL_MASK));
404 } while (iafc->pc_resync != 0);
405
406 PMCDBG(MDP,STA,1,"iafctrl=%x(%x) globalctrl=%jx(%jx)",
407 iafc->pc_iafctrl, (uint32_t) rdmsr(IAF_CTRL),
408 iafc->pc_globalctrl, rdmsr(IA_GLOBAL_CTRL));
409

--- 19 unchanged lines hidden (view full) ---

429 fc = (IAF_MASK << (ri * 4));
430
431 if (core_cputype != PMC_CPU_INTEL_ATOM)
432 fc &= ~IAF_ANY;
433
434 iafc->pc_iafctrl &= ~fc;
435
436 PMCDBG(MDP,STO,1,"iaf-stop iafctrl=%x", iafc->pc_iafctrl);
437 msr = rdmsr(IAF_CTRL);
437 msr = rdmsr(IAF_CTRL) & ~IAF_CTRL_MASK;
438 wrmsr(IAF_CTRL, msr | (iafc->pc_iafctrl & IAF_CTRL_MASK));
439
440 do {
441 iafc->pc_resync = 0;
442 iafc->pc_globalctrl &= ~(1ULL << (ri + IAF_OFFSET));
438 wrmsr(IAF_CTRL, msr | (iafc->pc_iafctrl & IAF_CTRL_MASK));
439
440 do {
441 iafc->pc_resync = 0;
442 iafc->pc_globalctrl &= ~(1ULL << (ri + IAF_OFFSET));
443 msr = rdmsr(IA_GLOBAL_CTRL);
443 msr = rdmsr(IA_GLOBAL_CTRL) & ~IAF_GLOBAL_CTRL_MASK;
444 wrmsr(IA_GLOBAL_CTRL, msr | (iafc->pc_globalctrl &
445 IAF_GLOBAL_CTRL_MASK));
446 } while (iafc->pc_resync != 0);
447
448 PMCDBG(MDP,STO,1,"iafctrl=%x(%x) globalctrl=%jx(%jx)",
449 iafc->pc_iafctrl, (uint32_t) rdmsr(IAF_CTRL),
450 iafc->pc_globalctrl, rdmsr(IA_GLOBAL_CTRL));
451

--- 16 unchanged lines hidden (view full) ---

468 pm = cc->pc_corepmcs[ri + core_iaf_ri].phw_pmc;
469
470 KASSERT(pm,
471 ("[core,%d] cpu %d ri %d pmc not configured", __LINE__, cpu, ri));
472
473 if (PMC_IS_SAMPLING_MODE(PMC_TO_MODE(pm)))
474 v = iaf_reload_count_to_perfctr_value(v);
475
444 wrmsr(IA_GLOBAL_CTRL, msr | (iafc->pc_globalctrl &
445 IAF_GLOBAL_CTRL_MASK));
446 } while (iafc->pc_resync != 0);
447
448 PMCDBG(MDP,STO,1,"iafctrl=%x(%x) globalctrl=%jx(%jx)",
449 iafc->pc_iafctrl, (uint32_t) rdmsr(IAF_CTRL),
450 iafc->pc_globalctrl, rdmsr(IA_GLOBAL_CTRL));
451

--- 16 unchanged lines hidden (view full) ---

468 pm = cc->pc_corepmcs[ri + core_iaf_ri].phw_pmc;
469
470 KASSERT(pm,
471 ("[core,%d] cpu %d ri %d pmc not configured", __LINE__, cpu, ri));
472
473 if (PMC_IS_SAMPLING_MODE(PMC_TO_MODE(pm)))
474 v = iaf_reload_count_to_perfctr_value(v);
475
476 msr = rdmsr(IAF_CTRL);
477 wrmsr(IAF_CTRL, msr & ~IAF_CTRL_MASK);
476 /* Turn off fixed counters */
477 msr = rdmsr(IAF_CTRL) & ~IAF_CTRL_MASK;
478 wrmsr(IAF_CTRL, msr);
479
478 wrmsr(IAF_CTR0 + ri, v & ((1ULL << core_iaf_width) - 1));
480 wrmsr(IAF_CTR0 + ri, v & ((1ULL << core_iaf_width) - 1));
479 msr = rdmsr(IAF_CTRL);
481
482 /* Turn on fixed counters */
483 msr = rdmsr(IAF_CTRL) & ~IAF_CTRL_MASK;
480 wrmsr(IAF_CTRL, msr | (cc->pc_iafctrl & IAF_CTRL_MASK));
481
482 PMCDBG(MDP,WRI,1, "iaf-write cpu=%d ri=%d msr=0x%x v=%jx iafctrl=%jx "
483 "pmc=%jx", cpu, ri, IAF_RI_TO_MSR(ri), v,
484 (uintmax_t) rdmsr(IAF_CTRL),
485 (uintmax_t) rdpmc(IAF_RI_TO_MSR(ri)));
486
487 return (0);

--- 1417 unchanged lines hidden (view full) ---

1905 pm = cc->pc_corepmcs[ri].phw_pmc;
1906
1907 KASSERT(pm,
1908 ("[core,%d] cpu%d ri%d no configured PMC to stop", __LINE__,
1909 cpu, ri));
1910
1911 PMCDBG(MDP,STO,1, "iap-stop cpu=%d ri=%d", cpu, ri);
1912
484 wrmsr(IAF_CTRL, msr | (cc->pc_iafctrl & IAF_CTRL_MASK));
485
486 PMCDBG(MDP,WRI,1, "iaf-write cpu=%d ri=%d msr=0x%x v=%jx iafctrl=%jx "
487 "pmc=%jx", cpu, ri, IAF_RI_TO_MSR(ri), v,
488 (uintmax_t) rdmsr(IAF_CTRL),
489 (uintmax_t) rdpmc(IAF_RI_TO_MSR(ri)));
490
491 return (0);

--- 1417 unchanged lines hidden (view full) ---

1909 pm = cc->pc_corepmcs[ri].phw_pmc;
1910
1911 KASSERT(pm,
1912 ("[core,%d] cpu%d ri%d no configured PMC to stop", __LINE__,
1913 cpu, ri));
1914
1915 PMCDBG(MDP,STO,1, "iap-stop cpu=%d ri=%d", cpu, ri);
1916
1913 msr = rdmsr(IAP_EVSEL0 + ri);
1914 wrmsr(IAP_EVSEL0 + ri, msr & IAP_EVSEL_MASK); /* stop hw */
1917 msr = rdmsr(IAP_EVSEL0 + ri) & ~IAP_EVSEL_MASK;
1918 wrmsr(IAP_EVSEL0 + ri, msr); /* stop hw */
1915
1916 if (core_cputype == PMC_CPU_INTEL_CORE)
1917 return (0);
1918
1919
1920 if (core_cputype == PMC_CPU_INTEL_CORE)
1921 return (0);
1922
1923 msr = 0;
1919 do {
1920 cc->pc_resync = 0;
1921 cc->pc_globalctrl &= ~(1ULL << ri);
1924 do {
1925 cc->pc_resync = 0;
1926 cc->pc_globalctrl &= ~(1ULL << ri);
1927 msr = rdmsr(IA_GLOBAL_CTRL) & ~IA_GLOBAL_CTRL_MASK;
1922 wrmsr(IA_GLOBAL_CTRL, cc->pc_globalctrl);
1923 } while (cc->pc_resync != 0);
1924
1925 return (0);
1926}
1927
1928static int
1929iap_write_pmc(int cpu, int ri, pmc_value_t v)

--- 69 unchanged lines hidden (view full) ---

1999
2000static int
2001core_intr(int cpu, struct trapframe *tf)
2002{
2003 pmc_value_t v;
2004 struct pmc *pm;
2005 struct core_cpu *cc;
2006 int error, found_interrupt, ri;
1928 wrmsr(IA_GLOBAL_CTRL, cc->pc_globalctrl);
1929 } while (cc->pc_resync != 0);
1930
1931 return (0);
1932}
1933
1934static int
1935iap_write_pmc(int cpu, int ri, pmc_value_t v)

--- 69 unchanged lines hidden (view full) ---

2005
2006static int
2007core_intr(int cpu, struct trapframe *tf)
2008{
2009 pmc_value_t v;
2010 struct pmc *pm;
2011 struct core_cpu *cc;
2012 int error, found_interrupt, ri;
2007 uint64_t msr = 0;
2013 uint64_t msr;
2008
2009 PMCDBG(MDP,INT, 1, "cpu=%d tf=0x%p um=%d", cpu, (void *) tf,
2010 TRAPF_USERMODE(tf));
2011
2012 found_interrupt = 0;
2013 cc = core_pcpu[cpu];
2014
2015 for (ri = 0; ri < core_iap_npmc; ri++) {

--- 15 unchanged lines hidden (view full) ---

2031
2032 v = pm->pm_sc.pm_reloadcount;
2033 v = iaf_reload_count_to_perfctr_value(v);
2034
2035 /*
2036 * Stop the counter, reload it but only restart it if
2037 * the PMC is not stalled.
2038 */
2014
2015 PMCDBG(MDP,INT, 1, "cpu=%d tf=0x%p um=%d", cpu, (void *) tf,
2016 TRAPF_USERMODE(tf));
2017
2018 found_interrupt = 0;
2019 cc = core_pcpu[cpu];
2020
2021 for (ri = 0; ri < core_iap_npmc; ri++) {

--- 15 unchanged lines hidden (view full) ---

2037
2038 v = pm->pm_sc.pm_reloadcount;
2039 v = iaf_reload_count_to_perfctr_value(v);
2040
2041 /*
2042 * Stop the counter, reload it but only restart it if
2043 * the PMC is not stalled.
2044 */
2039 msr = rdmsr(IAP_EVSEL0 + ri);
2040 wrmsr(IAP_EVSEL0 + ri, msr & ~IAP_EVSEL_MASK);
2045 msr = rdmsr(IAP_EVSEL0 + ri) & ~IAP_EVSEL_MASK;
2046 wrmsr(IAP_EVSEL0 + ri, msr);
2041 wrmsr(IAP_PMC0 + ri, v);
2042
2043 if (error)
2044 continue;
2045
2047 wrmsr(IAP_PMC0 + ri, v);
2048
2049 if (error)
2050 continue;
2051
2046 wrmsr(IAP_EVSEL0 + ri,
2047 pm->pm_md.pm_iap.pm_iap_evsel | IAP_EN);
2052 wrmsr(IAP_EVSEL0 + ri, msr | (pm->pm_md.pm_iap.pm_iap_evsel |
2053 IAP_EN));
2048 }
2049
2050 if (found_interrupt)
2051 lapic_reenable_pmc();
2052
2053 atomic_add_int(found_interrupt ? &pmc_stats.pm_intr_processed :
2054 &pmc_stats.pm_intr_ignored, 1);
2055
2056 return (found_interrupt);
2057}
2058
2059static int
2060core2_intr(int cpu, struct trapframe *tf)
2061{
2062 int error, found_interrupt, n;
2054 }
2055
2056 if (found_interrupt)
2057 lapic_reenable_pmc();
2058
2059 atomic_add_int(found_interrupt ? &pmc_stats.pm_intr_processed :
2060 &pmc_stats.pm_intr_ignored, 1);
2061
2062 return (found_interrupt);
2063}
2064
2065static int
2066core2_intr(int cpu, struct trapframe *tf)
2067{
2068 int error, found_interrupt, n;
2063 uint64_t flag, intrstatus, intrenable;
2069 uint64_t flag, intrstatus, intrenable, msr;
2064 struct pmc *pm;
2065 struct core_cpu *cc;
2066 pmc_value_t v;
2067
2068 PMCDBG(MDP,INT, 1, "cpu=%d tf=0x%p um=%d", cpu, (void *) tf,
2069 TRAPF_USERMODE(tf));
2070
2071 /*

--- 14 unchanged lines hidden (view full) ---

2086 KASSERT(cc != NULL, ("[core,%d] null pcpu", __LINE__));
2087
2088 cc->pc_globalctrl &= ~intrenable;
2089 cc->pc_resync = 1; /* MSRs now potentially out of sync. */
2090
2091 /*
2092 * Stop PMCs and clear overflow status bits.
2093 */
2070 struct pmc *pm;
2071 struct core_cpu *cc;
2072 pmc_value_t v;
2073
2074 PMCDBG(MDP,INT, 1, "cpu=%d tf=0x%p um=%d", cpu, (void *) tf,
2075 TRAPF_USERMODE(tf));
2076
2077 /*

--- 14 unchanged lines hidden (view full) ---

2092 KASSERT(cc != NULL, ("[core,%d] null pcpu", __LINE__));
2093
2094 cc->pc_globalctrl &= ~intrenable;
2095 cc->pc_resync = 1; /* MSRs now potentially out of sync. */
2096
2097 /*
2098 * Stop PMCs and clear overflow status bits.
2099 */
2094 wrmsr(IA_GLOBAL_CTRL, 0);
2100 msr = rdmsr(IA_GLOBAL_CTRL) & ~IA_GLOBAL_CTRL_MASK;
2101 wrmsr(IA_GLOBAL_CTRL, msr);
2095 wrmsr(IA_GLOBAL_OVF_CTRL, intrenable |
2096 IA_GLOBAL_STATUS_FLAG_OVFBUF |
2097 IA_GLOBAL_STATUS_FLAG_CONDCHG);
2098
2099 /*
2100 * Look for interrupts from fixed function PMCs.
2101 */
2102 for (n = 0, flag = (1ULL << IAF_OFFSET); n < core_iaf_npmc;

--- 54 unchanged lines hidden (view full) ---

2157 /*
2158 * Reenable all non-stalled PMCs.
2159 */
2160 PMCDBG(MDP,INT, 1, "cpu=%d intrenable=%jx", cpu,
2161 (uintmax_t) intrenable);
2162
2163 cc->pc_globalctrl |= intrenable;
2164
2102 wrmsr(IA_GLOBAL_OVF_CTRL, intrenable |
2103 IA_GLOBAL_STATUS_FLAG_OVFBUF |
2104 IA_GLOBAL_STATUS_FLAG_CONDCHG);
2105
2106 /*
2107 * Look for interrupts from fixed function PMCs.
2108 */
2109 for (n = 0, flag = (1ULL << IAF_OFFSET); n < core_iaf_npmc;

--- 54 unchanged lines hidden (view full) ---

2164 /*
2165 * Reenable all non-stalled PMCs.
2166 */
2167 PMCDBG(MDP,INT, 1, "cpu=%d intrenable=%jx", cpu,
2168 (uintmax_t) intrenable);
2169
2170 cc->pc_globalctrl |= intrenable;
2171
2165 wrmsr(IA_GLOBAL_CTRL, cc->pc_globalctrl);
2172 wrmsr(IA_GLOBAL_CTRL, cc->pc_globalctrl & IA_GLOBAL_CTRL_MASK);
2166
2167 PMCDBG(MDP,INT, 1, "cpu=%d fixedctrl=%jx globalctrl=%jx status=%jx "
2168 "ovf=%jx", cpu, (uintmax_t) rdmsr(IAF_CTRL),
2169 (uintmax_t) rdmsr(IA_GLOBAL_CTRL),
2170 (uintmax_t) rdmsr(IA_GLOBAL_STATUS),
2171 (uintmax_t) rdmsr(IA_GLOBAL_OVF_CTRL));
2172
2173 if (found_interrupt)

--- 101 unchanged lines hidden --- 		2173
2174 PMCDBG(MDP,INT, 1, "cpu=%d fixedctrl=%jx globalctrl=%jx status=%jx "
2175 "ovf=%jx", cpu, (uintmax_t) rdmsr(IAF_CTRL),
2176 (uintmax_t) rdmsr(IA_GLOBAL_CTRL),
2177 (uintmax_t) rdmsr(IA_GLOBAL_STATUS),
2178 (uintmax_t) rdmsr(IA_GLOBAL_OVF_CTRL));
2179
2180 if (found_interrupt)

--- 101 unchanged lines hidden ---