hwpmc_amd.c revision 150050
1178479Sjb/*- 2178479Sjb * Copyright (c) 2003-2005 Joseph Koshy 3178479Sjb * All rights reserved. 4178479Sjb * 5178479Sjb * Redistribution and use in source and binary forms, with or without 6178479Sjb * modification, are permitted provided that the following conditions 7178479Sjb * are met: 8178479Sjb * 1. Redistributions of source code must retain the above copyright 9178479Sjb * notice, this list of conditions and the following disclaimer. 10178479Sjb * 2. Redistributions in binary form must reproduce the above copyright 11178479Sjb * notice, this list of conditions and the following disclaimer in the 12178479Sjb * documentation and/or other materials provided with the distribution. 13178479Sjb * 14178479Sjb * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 15178479Sjb * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 16178479Sjb * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 17178479Sjb * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 18178479Sjb * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 19178479Sjb * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 20178479Sjb * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 21178479Sjb * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 22178479Sjb * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 23178479Sjb * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 24178479Sjb * SUCH DAMAGE. 25178479Sjb * 26178479Sjb */ 27178479Sjb 28178479Sjb#include <sys/cdefs.h> 29178479Sjb__FBSDID("$FreeBSD: head/sys/dev/hwpmc/hwpmc_amd.c 150050 2005-09-12 15:55:44Z jkoshy $"); 30178479Sjb 31178479Sjb/* Support for the AMD K7 and later processors */ 32178479Sjb 33178479Sjb#include <sys/param.h> 34178479Sjb#include <sys/lock.h> 35178479Sjb#include <sys/malloc.h> 36178479Sjb#include <sys/mutex.h> 37178479Sjb#include <sys/pmc.h> 38178479Sjb#include <sys/smp.h> 39178479Sjb#include <sys/systm.h> 40178479Sjb 41178479Sjb#include <machine/cpufunc.h> 42178479Sjb#include <machine/md_var.h> 43178479Sjb#include <machine/pmc_mdep.h> 44178479Sjb#include <machine/specialreg.h> 45178479Sjb 46178479Sjb#if DEBUG 47178479Sjbenum pmc_class amd_pmc_class; 48178479Sjb#endif 49178479Sjb 50178479Sjb/* AMD K7 & K8 PMCs */ 51178479Sjbstruct amd_descr { 52178479Sjb struct pmc_descr pm_descr; /* "base class" */ 53178479Sjb uint32_t pm_evsel; /* address of EVSEL register */ 54178479Sjb uint32_t pm_perfctr; /* address of PERFCTR register */ 55178479Sjb}; 56178479Sjb 57178479Sjbstatic struct amd_descr amd_pmcdesc[AMD_NPMCS] = 58178479Sjb{ 59178479Sjb { 60178479Sjb .pm_descr = 61178479Sjb { 62178479Sjb .pd_name = "TSC", 63178479Sjb .pd_class = PMC_CLASS_TSC, 64178479Sjb .pd_caps = PMC_CAP_READ, 65178479Sjb .pd_width = 64 66178479Sjb }, 67178479Sjb .pm_evsel = MSR_TSC, 68178479Sjb .pm_perfctr = 0 /* unused */ 69178479Sjb }, 70178479Sjb 71178479Sjb { 72178479Sjb .pm_descr = 73178479Sjb { 74178479Sjb .pd_name = "", 75178479Sjb .pd_class = -1, 76178479Sjb .pd_caps = AMD_PMC_CAPS, 77178479Sjb .pd_width = 48 78178479Sjb }, 79178479Sjb .pm_evsel = AMD_PMC_EVSEL_0, 80178479Sjb .pm_perfctr = AMD_PMC_PERFCTR_0 81178479Sjb }, 82178479Sjb { 83178479Sjb .pm_descr = 84178479Sjb { 85178479Sjb .pd_name = "", 86178479Sjb .pd_class = -1, 87178479Sjb .pd_caps = AMD_PMC_CAPS, 88178479Sjb .pd_width = 48 89178479Sjb }, 90178479Sjb .pm_evsel = AMD_PMC_EVSEL_1, 91178479Sjb .pm_perfctr = AMD_PMC_PERFCTR_1 92178479Sjb }, 93178479Sjb { 94178479Sjb .pm_descr = 95178479Sjb { 96178479Sjb .pd_name = "", 97178479Sjb .pd_class = -1, 98178479Sjb .pd_caps = AMD_PMC_CAPS, 99178479Sjb .pd_width = 48 100178479Sjb }, 101178479Sjb .pm_evsel = AMD_PMC_EVSEL_2, 102178479Sjb .pm_perfctr = AMD_PMC_PERFCTR_2 103178567Sjb }, 104178479Sjb { 105178567Sjb .pm_descr = 106178479Sjb { 107178479Sjb .pd_name = "", 108178479Sjb .pd_class = -1, 109178479Sjb .pd_caps = AMD_PMC_CAPS, 110178479Sjb .pd_width = 48 111178479Sjb }, 112178479Sjb .pm_evsel = AMD_PMC_EVSEL_3, 113178479Sjb .pm_perfctr = AMD_PMC_PERFCTR_3 114178479Sjb } 115178479Sjb}; 116178479Sjb 117178479Sjbstruct amd_event_code_map { 118178479Sjb enum pmc_event pe_ev; /* enum value */ 119178479Sjb uint8_t pe_code; /* encoded event mask */ 120178479Sjb uint8_t pe_mask; /* bits allowed in unit mask */ 121178479Sjb}; 122178479Sjb 123178479Sjbconst struct amd_event_code_map amd_event_codes[] = { 124178479Sjb#if defined(__i386__) /* 32 bit Athlon (K7) only */ 125178479Sjb { PMC_EV_K7_DC_ACCESSES, 0x40, 0 }, 126178479Sjb { PMC_EV_K7_DC_MISSES, 0x41, 0 }, 127178479Sjb { PMC_EV_K7_DC_REFILLS_FROM_L2, 0x42, AMD_PMC_UNITMASK_MOESI }, 128178479Sjb { PMC_EV_K7_DC_REFILLS_FROM_SYSTEM, 0x43, AMD_PMC_UNITMASK_MOESI }, 129178479Sjb { PMC_EV_K7_DC_WRITEBACKS, 0x44, AMD_PMC_UNITMASK_MOESI }, 130178479Sjb { PMC_EV_K7_L1_DTLB_MISS_AND_L2_DTLB_HITS, 0x45, 0 }, 131178479Sjb { PMC_EV_K7_L1_AND_L2_DTLB_MISSES, 0x46, 0 }, 132178479Sjb { PMC_EV_K7_MISALIGNED_REFERENCES, 0x47, 0 }, 133178479Sjb 134178479Sjb { PMC_EV_K7_IC_FETCHES, 0x80, 0 }, 135178479Sjb { PMC_EV_K7_IC_MISSES, 0x81, 0 }, 136178479Sjb 137178479Sjb { PMC_EV_K7_L1_ITLB_MISSES, 0x84, 0 }, 138178479Sjb { PMC_EV_K7_L1_L2_ITLB_MISSES, 0x85, 0 }, 139178479Sjb 140178479Sjb { PMC_EV_K7_RETIRED_INSTRUCTIONS, 0xC0, 0 }, 141178479Sjb { PMC_EV_K7_RETIRED_OPS, 0xC1, 0 }, 142178479Sjb { PMC_EV_K7_RETIRED_BRANCHES, 0xC2, 0 }, 143178479Sjb { PMC_EV_K7_RETIRED_BRANCHES_MISPREDICTED, 0xC3, 0 }, 144178479Sjb { PMC_EV_K7_RETIRED_TAKEN_BRANCHES, 0xC4, 0 }, 145178479Sjb { PMC_EV_K7_RETIRED_TAKEN_BRANCHES_MISPREDICTED, 0xC5, 0 }, 146178479Sjb { PMC_EV_K7_RETIRED_FAR_CONTROL_TRANSFERS, 0xC6, 0 }, 147178479Sjb { PMC_EV_K7_RETIRED_RESYNC_BRANCHES, 0xC7, 0 }, 148178479Sjb { PMC_EV_K7_INTERRUPTS_MASKED_CYCLES, 0xCD, 0 }, 149178479Sjb { PMC_EV_K7_INTERRUPTS_MASKED_WHILE_PENDING_CYCLES, 0xCE, 0 }, 150178479Sjb { PMC_EV_K7_HARDWARE_INTERRUPTS, 0xCF, 0 }, 151178479Sjb#endif 152178479Sjb 153178479Sjb { PMC_EV_K8_FP_DISPATCHED_FPU_OPS, 0x00, 0x3F }, 154178479Sjb { PMC_EV_K8_FP_CYCLES_WITH_NO_FPU_OPS_RETIRED, 0x01, 0x00 }, 155178479Sjb { PMC_EV_K8_FP_DISPATCHED_FPU_FAST_FLAG_OPS, 0x02, 0x00 }, 156178479Sjb 157178479Sjb { PMC_EV_K8_LS_SEGMENT_REGISTER_LOAD, 0x20, 0x7F }, 158178479Sjb { PMC_EV_K8_LS_MICROARCHITECTURAL_RESYNC_BY_SELF_MODIFYING_CODE, 159178479Sjb 0x21, 0x00 }, 160178479Sjb { PMC_EV_K8_LS_MICROARCHITECTURAL_RESYNC_BY_SNOOP, 0x22, 0x00 }, 161178479Sjb { PMC_EV_K8_LS_BUFFER2_FULL, 0x23, 0x00 }, 162178479Sjb { PMC_EV_K8_LS_LOCKED_OPERATION, 0x24, 0x07 }, 163178479Sjb { PMC_EV_K8_LS_MICROARCHITECTURAL_LATE_CANCEL, 0x25, 0x00 }, 164178479Sjb { PMC_EV_K8_LS_RETIRED_CFLUSH_INSTRUCTIONS, 0x26, 0x00 }, 165178479Sjb { PMC_EV_K8_LS_RETIRED_CPUID_INSTRUCTIONS, 0x27, 0x00 }, 166178479Sjb 167178479Sjb { PMC_EV_K8_DC_ACCESS, 0x40, 0x00 }, 168178479Sjb { PMC_EV_K8_DC_MISS, 0x41, 0x00 }, 169178479Sjb { PMC_EV_K8_DC_REFILL_FROM_L2, 0x42, 0x1F }, 170178479Sjb { PMC_EV_K8_DC_REFILL_FROM_SYSTEM, 0x43, 0x1F }, 171178479Sjb { PMC_EV_K8_DC_COPYBACK, 0x44, 0x1F }, 172178479Sjb { PMC_EV_K8_DC_L1_DTLB_MISS_AND_L2_DTLB_HIT, 0x45, 0x00 }, 173178479Sjb { PMC_EV_K8_DC_L1_DTLB_MISS_AND_L2_DTLB_MISS, 0x46, 0x00 }, 174178479Sjb { PMC_EV_K8_DC_MISALIGNED_DATA_REFERENCE, 0x47, 0x00 }, 175178479Sjb { PMC_EV_K8_DC_MICROARCHITECTURAL_LATE_CANCEL, 0x48, 0x00 }, 176178479Sjb { PMC_EV_K8_DC_MICROARCHITECTURAL_EARLY_CANCEL, 0x49, 0x00 }, 177178479Sjb { PMC_EV_K8_DC_ONE_BIT_ECC_ERROR, 0x4A, 0x03 }, 178178479Sjb { PMC_EV_K8_DC_DISPATCHED_PREFETCH_INSTRUCTIONS, 0x4B, 0x07 }, 179178479Sjb { PMC_EV_K8_DC_DCACHE_ACCESSES_BY_LOCKS, 0x4C, 0x03 }, 180178479Sjb 181178479Sjb { PMC_EV_K8_BU_CPU_CLK_UNHALTED, 0x76, 0x00 }, 182178479Sjb { PMC_EV_K8_BU_INTERNAL_L2_REQUEST, 0x7D, 0x1F }, 183178479Sjb { PMC_EV_K8_BU_FILL_REQUEST_L2_MISS, 0x7E, 0x07 }, 184178479Sjb { PMC_EV_K8_BU_FILL_INTO_L2, 0x7F, 0x03 }, 185178479Sjb 186178479Sjb { PMC_EV_K8_IC_FETCH, 0x80, 0x00 }, 187178479Sjb { PMC_EV_K8_IC_MISS, 0x81, 0x00 }, 188178479Sjb { PMC_EV_K8_IC_REFILL_FROM_L2, 0x82, 0x00 }, 189178479Sjb { PMC_EV_K8_IC_REFILL_FROM_SYSTEM, 0x83, 0x00 }, 190178479Sjb { PMC_EV_K8_IC_L1_ITLB_MISS_AND_L2_ITLB_HIT, 0x84, 0x00 }, 191178479Sjb { PMC_EV_K8_IC_L1_ITLB_MISS_AND_L2_ITLB_MISS, 0x85, 0x00 }, 192178479Sjb { PMC_EV_K8_IC_MICROARCHITECTURAL_RESYNC_BY_SNOOP, 0x86, 0x00 }, 193178479Sjb { PMC_EV_K8_IC_INSTRUCTION_FETCH_STALL, 0x87, 0x00 }, 194178479Sjb { PMC_EV_K8_IC_RETURN_STACK_HIT, 0x88, 0x00 }, 195178479Sjb { PMC_EV_K8_IC_RETURN_STACK_OVERFLOW, 0x89, 0x00 }, 196178479Sjb 197178479Sjb { PMC_EV_K8_FR_RETIRED_X86_INSTRUCTIONS, 0xC0, 0x00 }, 198178479Sjb { PMC_EV_K8_FR_RETIRED_UOPS, 0xC1, 0x00 }, 199178479Sjb { PMC_EV_K8_FR_RETIRED_BRANCHES, 0xC2, 0x00 }, 200178479Sjb { PMC_EV_K8_FR_RETIRED_BRANCHES_MISPREDICTED, 0xC3, 0x00 }, 201178479Sjb { PMC_EV_K8_FR_RETIRED_TAKEN_BRANCHES, 0xC4, 0x00 }, 202178479Sjb { PMC_EV_K8_FR_RETIRED_TAKEN_BRANCHES_MISPREDICTED, 0xC5, 0x00 }, 203178479Sjb { PMC_EV_K8_FR_RETIRED_FAR_CONTROL_TRANSFERS, 0xC6, 0x00 }, 204178479Sjb { PMC_EV_K8_FR_RETIRED_RESYNCS, 0xC7, 0x00 }, 205178479Sjb { PMC_EV_K8_FR_RETIRED_NEAR_RETURNS, 0xC8, 0x00 }, 206178479Sjb { PMC_EV_K8_FR_RETIRED_NEAR_RETURNS_MISPREDICTED, 0xC9, 0x00 }, 207178479Sjb { PMC_EV_K8_FR_RETIRED_TAKEN_BRANCHES_MISPREDICTED_BY_ADDR_MISCOMPARE, 208178479Sjb 0xCA, 0x00 }, 209178479Sjb { PMC_EV_K8_FR_RETIRED_FPU_INSTRUCTIONS, 0xCB, 0x0F }, 210178479Sjb { PMC_EV_K8_FR_RETIRED_FASTPATH_DOUBLE_OP_INSTRUCTIONS, 211178479Sjb 0xCC, 0x07 }, 212178479Sjb { PMC_EV_K8_FR_INTERRUPTS_MASKED_CYCLES, 0xCD, 0x00 }, 213178479Sjb { PMC_EV_K8_FR_INTERRUPTS_MASKED_WHILE_PENDING_CYCLES, 0xCE, 0x00 }, 214178479Sjb { PMC_EV_K8_FR_TAKEN_HARDWARE_INTERRUPTS, 0xCF, 0x00 }, 215178479Sjb 216178479Sjb { PMC_EV_K8_FR_DECODER_EMPTY, 0xD0, 0x00 }, 217178479Sjb { PMC_EV_K8_FR_DISPATCH_STALLS, 0xD1, 0x00 }, 218178479Sjb { PMC_EV_K8_FR_DISPATCH_STALL_FROM_BRANCH_ABORT_TO_RETIRE, 219178479Sjb 0xD2, 0x00 }, 220178479Sjb { PMC_EV_K8_FR_DISPATCH_STALL_FOR_SERIALIZATION, 0xD3, 0x00 }, 221178479Sjb { PMC_EV_K8_FR_DISPATCH_STALL_FOR_SEGMENT_LOAD, 0xD4, 0x00 }, 222178479Sjb { PMC_EV_K8_FR_DISPATCH_STALL_WHEN_REORDER_BUFFER_IS_FULL, 223178479Sjb 0xD5, 0x00 }, 224178479Sjb { PMC_EV_K8_FR_DISPATCH_STALL_WHEN_RESERVATION_STATIONS_ARE_FULL, 225178479Sjb 0xD6, 0x00 }, 226178479Sjb { PMC_EV_K8_FR_DISPATCH_STALL_WHEN_FPU_IS_FULL, 0xD7, 0x00 }, 227178479Sjb { PMC_EV_K8_FR_DISPATCH_STALL_WHEN_LS_IS_FULL, 0xD8, 0x00 }, 228178479Sjb { PMC_EV_K8_FR_DISPATCH_STALL_WHEN_WAITING_FOR_ALL_TO_BE_QUIET, 229178479Sjb 0xD9, 0x00 }, 230178479Sjb { PMC_EV_K8_FR_DISPATCH_STALL_WHEN_FAR_XFER_OR_RESYNC_BRANCH_PENDING, 231178479Sjb 0xDA, 0x00 }, 232178479Sjb { PMC_EV_K8_FR_FPU_EXCEPTIONS, 0xDB, 0x0F }, 233178479Sjb { PMC_EV_K8_FR_NUMBER_OF_BREAKPOINTS_FOR_DR0, 0xDC, 0x00 }, 234178479Sjb { PMC_EV_K8_FR_NUMBER_OF_BREAKPOINTS_FOR_DR1, 0xDD, 0x00 }, 235178479Sjb { PMC_EV_K8_FR_NUMBER_OF_BREAKPOINTS_FOR_DR2, 0xDE, 0x00 }, 236178479Sjb { PMC_EV_K8_FR_NUMBER_OF_BREAKPOINTS_FOR_DR3, 0xDF, 0x00 }, 237178479Sjb 238178479Sjb { PMC_EV_K8_NB_MEMORY_CONTROLLER_PAGE_ACCESS_EVENT, 0xE0, 0x7 }, 239178479Sjb { PMC_EV_K8_NB_MEMORY_CONTROLLER_PAGE_TABLE_OVERFLOW, 0xE1, 0x00 }, 240178479Sjb { PMC_EV_K8_NB_MEMORY_CONTROLLER_DRAM_COMMAND_SLOTS_MISSED, 241178479Sjb 0xE2, 0x00 }, 242178479Sjb { PMC_EV_K8_NB_MEMORY_CONTROLLER_TURNAROUND, 0xE3, 0x07 }, 243178479Sjb { PMC_EV_K8_NB_MEMORY_CONTROLLER_BYPASS_SATURATION, 0xE4, 0x0F }, 244178479Sjb { PMC_EV_K8_NB_SIZED_COMMANDS, 0xEB, 0x7F }, 245178479Sjb { PMC_EV_K8_NB_PROBE_RESULT, 0xEC, 0x0F }, 246178479Sjb { PMC_EV_K8_NB_HT_BUS0_BANDWIDTH, 0xF6, 0x0F }, 247178479Sjb { PMC_EV_K8_NB_HT_BUS1_BANDWIDTH, 0xF7, 0x0F }, 248178479Sjb { PMC_EV_K8_NB_HT_BUS2_BANDWIDTH, 0xF8, 0x0F } 249178479Sjb 250178479Sjb}; 251178479Sjb 252178479Sjbconst int amd_event_codes_size = 253178479Sjb sizeof(amd_event_codes) / sizeof(amd_event_codes[0]); 254178479Sjb 255178479Sjb/* 256178479Sjb * read a pmc register 257178479Sjb */ 258178479Sjb 259178479Sjbstatic int 260178479Sjbamd_read_pmc(int cpu, int ri, pmc_value_t *v) 261178479Sjb{ 262178479Sjb enum pmc_mode mode; 263178479Sjb const struct amd_descr *pd; 264178479Sjb struct pmc *pm; 265178479Sjb const struct pmc_hw *phw; 266178479Sjb pmc_value_t tmp; 267178479Sjb 268178479Sjb KASSERT(cpu >= 0 && cpu < mp_ncpus, 269178479Sjb ("[amd,%d] illegal CPU value %d", __LINE__, cpu)); 270178479Sjb KASSERT(ri >= 0 && ri < AMD_NPMCS, 271178479Sjb ("[amd,%d] illegal row-index %d", __LINE__, ri)); 272178479Sjb 273178479Sjb phw = pmc_pcpu[cpu]->pc_hwpmcs[ri]; 274178479Sjb pd = &amd_pmcdesc[ri]; 275178479Sjb pm = phw->phw_pmc; 276178479Sjb 277178479Sjb KASSERT(pm != NULL, 278178479Sjb ("[amd,%d] No owner for HWPMC [cpu%d,pmc%d]", __LINE__, 279178479Sjb cpu, ri)); 280178479Sjb 281178479Sjb mode = PMC_TO_MODE(pm); 282178479Sjb 283178479Sjb PMCDBG(MDP,REA,1,"amd-read id=%d class=%d", ri, pd->pm_descr.pd_class); 284178479Sjb 285178479Sjb /* Reading the TSC is a special case */ 286178479Sjb if (pd->pm_descr.pd_class == PMC_CLASS_TSC) { 287178479Sjb KASSERT(PMC_IS_COUNTING_MODE(mode), 288178479Sjb ("[amd,%d] TSC counter in non-counting mode", __LINE__)); 289178479Sjb *v = rdtsc(); 290178479Sjb PMCDBG(MDP,REA,2,"amd-read id=%d -> %jd", ri, *v); 291178479Sjb return 0; 292178479Sjb } 293178479Sjb 294178479Sjb#if DEBUG 295178479Sjb KASSERT(pd->pm_descr.pd_class == amd_pmc_class, 296178479Sjb ("[amd,%d] unknown PMC class (%d)", __LINE__, 297178479Sjb pd->pm_descr.pd_class)); 298178479Sjb#endif 299178479Sjb 300178479Sjb tmp = rdmsr(pd->pm_perfctr); /* RDMSR serializes */ 301178479Sjb if (PMC_IS_SAMPLING_MODE(mode)) 302178479Sjb *v = AMD_PERFCTR_VALUE_TO_RELOAD_COUNT(tmp); 303178479Sjb else 304178479Sjb *v = tmp; 305178479Sjb 306178479Sjb PMCDBG(MDP,REA,2,"amd-read id=%d -> %jd", ri, *v); 307178479Sjb 308178479Sjb return 0; 309178479Sjb} 310178479Sjb 311178479Sjb/* 312178479Sjb * Write a PMC MSR. 313178479Sjb */ 314178479Sjb 315178479Sjbstatic int 316178479Sjbamd_write_pmc(int cpu, int ri, pmc_value_t v) 317178479Sjb{ 318178479Sjb const struct amd_descr *pd; 319178479Sjb struct pmc *pm; 320178479Sjb const struct pmc_hw *phw; 321178479Sjb enum pmc_mode mode; 322178479Sjb 323178479Sjb KASSERT(cpu >= 0 && cpu < mp_ncpus, 324178479Sjb ("[amd,%d] illegal CPU value %d", __LINE__, cpu)); 325178479Sjb KASSERT(ri >= 0 && ri < AMD_NPMCS, 326178479Sjb ("[amd,%d] illegal row-index %d", __LINE__, ri)); 327178479Sjb 328178479Sjb phw = pmc_pcpu[cpu]->pc_hwpmcs[ri]; 329178479Sjb pd = &amd_pmcdesc[ri]; 330178479Sjb pm = phw->phw_pmc; 331178479Sjb 332178479Sjb KASSERT(pm != NULL, 333178479Sjb ("[amd,%d] PMC not owned (cpu%d,pmc%d)", __LINE__, 334178479Sjb cpu, ri)); 335178479Sjb 336178479Sjb mode = PMC_TO_MODE(pm); 337178479Sjb 338178479Sjb if (pd->pm_descr.pd_class == PMC_CLASS_TSC) 339178479Sjb return 0; 340178479Sjb 341178479Sjb#if DEBUG 342178479Sjb KASSERT(pd->pm_descr.pd_class == amd_pmc_class, 343178479Sjb ("[amd,%d] unknown PMC class (%d)", __LINE__, 344178479Sjb pd->pm_descr.pd_class)); 345178479Sjb#endif 346178479Sjb 347178479Sjb /* use 2's complement of the count for sampling mode PMCs */ 348178479Sjb if (PMC_IS_SAMPLING_MODE(mode)) 349178479Sjb v = AMD_RELOAD_COUNT_TO_PERFCTR_VALUE(v); 350178479Sjb 351178479Sjb PMCDBG(MDP,WRI,1,"amd-write cpu=%d ri=%d v=%jx", cpu, ri, v); 352178479Sjb 353178479Sjb /* write the PMC value */ 354178479Sjb wrmsr(pd->pm_perfctr, v); 355178479Sjb return 0; 356178479Sjb} 357178479Sjb 358178479Sjb/* 359178479Sjb * configure hardware pmc according to the configuration recorded in 360178479Sjb * pmc 'pm'. 361178479Sjb */ 362178479Sjb 363178479Sjbstatic int 364178479Sjbamd_config_pmc(int cpu, int ri, struct pmc *pm) 365178479Sjb{ 366178479Sjb struct pmc_hw *phw; 367178479Sjb 368178479Sjb PMCDBG(MDP,CFG,1, "cpu=%d ri=%d pm=%p", cpu, ri, pm); 369178479Sjb 370178479Sjb KASSERT(cpu >= 0 && cpu < mp_ncpus, 371178479Sjb ("[amd,%d] illegal CPU value %d", __LINE__, cpu)); 372178479Sjb KASSERT(ri >= 0 && ri < AMD_NPMCS, 373178479Sjb ("[amd,%d] illegal row-index %d", __LINE__, ri)); 374178479Sjb 375178479Sjb phw = pmc_pcpu[cpu]->pc_hwpmcs[ri]; 376178479Sjb 377178479Sjb KASSERT(pm == NULL || phw->phw_pmc == NULL, 378178479Sjb ("[amd,%d] pm=%p phw->pm=%p hwpmc not unconfigured", 379178479Sjb __LINE__, pm, phw->phw_pmc)); 380178479Sjb 381178479Sjb phw->phw_pmc = pm; 382178479Sjb return 0; 383178479Sjb} 384178479Sjb 385178479Sjb/* 386178479Sjb * Retrieve a configured PMC pointer from hardware state. 387178479Sjb */ 388178479Sjb 389178479Sjbstatic int 390178479Sjbamd_get_config(int cpu, int ri, struct pmc **ppm) 391178479Sjb{ 392178479Sjb *ppm = pmc_pcpu[cpu]->pc_hwpmcs[ri]->phw_pmc; 393178479Sjb 394178479Sjb return 0; 395178479Sjb} 396178479Sjb 397178479Sjb/* 398178479Sjb * Machine dependent actions taken during the context switch in of a 399178479Sjb * thread. 400178479Sjb */ 401178479Sjb 402178479Sjbstatic int 403178479Sjbamd_switch_in(struct pmc_cpu *pc, struct pmc_process *pp) 404178479Sjb{ 405178479Sjb (void) pc; 406178479Sjb 407178479Sjb PMCDBG(MDP,SWI,1, "pc=%p pp=%p enable-msr=%d", pc, pp, 408178479Sjb (pp->pp_flags & PMC_PP_ENABLE_MSR_ACCESS) != 0); 409178479Sjb 410178479Sjb /* enable the RDPMC instruction if needed */ 411178479Sjb if (pp->pp_flags & PMC_PP_ENABLE_MSR_ACCESS) 412178479Sjb load_cr4(rcr4() | CR4_PCE); 413178479Sjb 414178479Sjb return 0; 415178479Sjb} 416178479Sjb 417178479Sjb/* 418178479Sjb * Machine dependent actions taken during the context switch out of a 419178479Sjb * thread. 420178479Sjb */ 421178479Sjb 422178479Sjbstatic int 423178479Sjbamd_switch_out(struct pmc_cpu *pc, struct pmc_process *pp) 424178479Sjb{ 425178479Sjb (void) pc; 426178479Sjb (void) pp; /* can be NULL */ 427178479Sjb 428178479Sjb PMCDBG(MDP,SWO,1, "pc=%p pp=%p enable-msr=%d", pc, pp, pp ? 429178479Sjb (pp->pp_flags & PMC_PP_ENABLE_MSR_ACCESS) == 1 : 0); 430178479Sjb 431178479Sjb /* always turn off the RDPMC instruction */ 432178479Sjb load_cr4(rcr4() & ~CR4_PCE); 433178479Sjb 434178479Sjb return 0; 435178479Sjb} 436178479Sjb 437178479Sjb/* 438178479Sjb * Check if a given allocation is feasible. 439178479Sjb */ 440178479Sjb 441178479Sjbstatic int 442178479Sjbamd_allocate_pmc(int cpu, int ri, struct pmc *pm, 443178479Sjb const struct pmc_op_pmcallocate *a) 444178479Sjb{ 445178479Sjb int i; 446178479Sjb uint32_t allowed_unitmask, caps, config, unitmask; 447178479Sjb enum pmc_event pe; 448178479Sjb const struct pmc_descr *pd; 449178479Sjb 450178479Sjb (void) cpu; 451178479Sjb 452178479Sjb KASSERT(cpu >= 0 && cpu < mp_ncpus, 453178479Sjb ("[amd,%d] illegal CPU value %d", __LINE__, cpu)); 454178479Sjb KASSERT(ri >= 0 && ri < AMD_NPMCS, 455178479Sjb ("[amd,%d] illegal row index %d", __LINE__, ri)); 456178479Sjb 457178479Sjb pd = &amd_pmcdesc[ri].pm_descr; 458178479Sjb 459178479Sjb /* check class match */ 460178479Sjb if (pd->pd_class != a->pm_class) 461178479Sjb return EINVAL; 462178479Sjb 463178479Sjb caps = pm->pm_caps; 464178479Sjb 465178479Sjb PMCDBG(MDP,ALL,1,"amd-allocate ri=%d caps=0x%x", ri, caps); 466178479Sjb 467178479Sjb if ((pd->pd_caps & caps) != caps) 468178479Sjb return EPERM; 469178479Sjb if (pd->pd_class == PMC_CLASS_TSC) { 470178479Sjb /* TSC's are always allocated in system-wide counting mode */ 471178479Sjb if (a->pm_ev != PMC_EV_TSC_TSC || 472178479Sjb a->pm_mode != PMC_MODE_SC) 473178479Sjb return EINVAL; 474178479Sjb return 0; 475178479Sjb } 476178479Sjb 477178479Sjb#if DEBUG 478178479Sjb KASSERT(pd->pd_class == amd_pmc_class, 479178479Sjb ("[amd,%d] Unknown PMC class (%d)", __LINE__, pd->pd_class)); 480178479Sjb#endif 481178479Sjb 482178479Sjb pe = a->pm_ev; 483178479Sjb 484178479Sjb /* map ev to the correct event mask code */ 485178479Sjb config = allowed_unitmask = 0; 486178479Sjb for (i = 0; i < amd_event_codes_size; i++) 487178479Sjb if (amd_event_codes[i].pe_ev == pe) { 488178479Sjb config = 489178479Sjb AMD_PMC_TO_EVENTMASK(amd_event_codes[i].pe_code); 490178479Sjb allowed_unitmask = 491178479Sjb AMD_PMC_TO_UNITMASK(amd_event_codes[i].pe_mask); 492178479Sjb break; 493178479Sjb } 494178479Sjb if (i == amd_event_codes_size) 495178479Sjb return EINVAL; 496178479Sjb 497178479Sjb unitmask = a->pm_md.pm_amd.pm_amd_config & AMD_PMC_UNITMASK; 498178479Sjb if (unitmask & ~allowed_unitmask) /* disallow reserved bits */ 499178479Sjb return EINVAL; 500178479Sjb 501178479Sjb if (unitmask && (caps & PMC_CAP_QUALIFIER)) 502178479Sjb config |= unitmask; 503178479Sjb 504178479Sjb if (caps & PMC_CAP_THRESHOLD) 505178479Sjb config |= a->pm_md.pm_amd.pm_amd_config & AMD_PMC_COUNTERMASK; 506178479Sjb 507178479Sjb /* set at least one of the 'usr' or 'os' caps */ 508178479Sjb if (caps & PMC_CAP_USER) 509178479Sjb config |= AMD_PMC_USR; 510178479Sjb if (caps & PMC_CAP_SYSTEM) 511178479Sjb config |= AMD_PMC_OS; 512178479Sjb if ((caps & (PMC_CAP_USER|PMC_CAP_SYSTEM)) == 0) 513178479Sjb config |= (AMD_PMC_USR|AMD_PMC_OS); 514178479Sjb 515178479Sjb if (caps & PMC_CAP_EDGE) 516178479Sjb config |= AMD_PMC_EDGE; 517178479Sjb if (caps & PMC_CAP_INVERT) 518178479Sjb config |= AMD_PMC_INVERT; 519178479Sjb if (caps & PMC_CAP_INTERRUPT) 520178479Sjb config |= AMD_PMC_INT; 521178479Sjb 522178479Sjb pm->pm_md.pm_amd.pm_amd_evsel = config; /* save config value */ 523178479Sjb 524178479Sjb PMCDBG(MDP,ALL,2,"amd-allocate ri=%d -> config=0x%x", ri, config); 525178479Sjb 526178479Sjb return 0; 527178479Sjb} 528178479Sjb 529178479Sjb/* 530178479Sjb * Release machine dependent state associated with a PMC. This is a 531178479Sjb * no-op on this architecture. 532178479Sjb * 533178479Sjb */ 534178479Sjb 535178479Sjb/* ARGSUSED0 */ 536178479Sjbstatic int 537178479Sjbamd_release_pmc(int cpu, int ri, struct pmc *pmc) 538178479Sjb{ 539178479Sjb#if DEBUG 540178479Sjb const struct amd_descr *pd; 541178479Sjb#endif 542178479Sjb struct pmc_hw *phw; 543178479Sjb 544178479Sjb (void) pmc; 545178479Sjb 546178479Sjb KASSERT(cpu >= 0 && cpu < mp_ncpus, 547178479Sjb ("[amd,%d] illegal CPU value %d", __LINE__, cpu)); 548178479Sjb KASSERT(ri >= 0 && ri < AMD_NPMCS, 549178479Sjb ("[amd,%d] illegal row-index %d", __LINE__, ri)); 550178479Sjb 551178479Sjb phw = pmc_pcpu[cpu]->pc_hwpmcs[ri]; 552178479Sjb 553178479Sjb KASSERT(phw->phw_pmc == NULL, 554178479Sjb ("[amd,%d] PHW pmc %p non-NULL", __LINE__, phw->phw_pmc)); 555178479Sjb 556178479Sjb#if DEBUG 557178479Sjb pd = &amd_pmcdesc[ri]; 558178479Sjb if (pd->pm_descr.pd_class == amd_pmc_class) 559178479Sjb KASSERT(AMD_PMC_IS_STOPPED(pd->pm_evsel), 560178479Sjb ("[amd,%d] PMC %d released while active", __LINE__, ri)); 561178479Sjb#endif 562178479Sjb 563178479Sjb return 0; 564178479Sjb} 565178479Sjb 566178479Sjb/* 567178479Sjb * start a PMC. 568178479Sjb */ 569178479Sjb 570178479Sjbstatic int 571178479Sjbamd_start_pmc(int cpu, int ri) 572178479Sjb{ 573178479Sjb uint32_t config; 574178479Sjb struct pmc *pm; 575178479Sjb struct pmc_hw *phw; 576178479Sjb const struct amd_descr *pd; 577178479Sjb 578178479Sjb KASSERT(cpu >= 0 && cpu < mp_ncpus, 579178479Sjb ("[amd,%d] illegal CPU value %d", __LINE__, cpu)); 580178479Sjb KASSERT(ri >= 0 && ri < AMD_NPMCS, 581178479Sjb ("[amd,%d] illegal row-index %d", __LINE__, ri)); 582178479Sjb 583178479Sjb phw = pmc_pcpu[cpu]->pc_hwpmcs[ri]; 584178479Sjb pm = phw->phw_pmc; 585178479Sjb pd = &amd_pmcdesc[ri]; 586178479Sjb 587178479Sjb KASSERT(pm != NULL, 588178479Sjb ("[amd,%d] starting cpu%d,pmc%d with null pmc record", __LINE__, 589178479Sjb cpu, ri)); 590178479Sjb 591178479Sjb PMCDBG(MDP,STA,1,"amd-start cpu=%d ri=%d", cpu, ri); 592178479Sjb 593178479Sjb if (pd->pm_descr.pd_class == PMC_CLASS_TSC) 594178479Sjb return 0; /* TSCs are always running */ 595178479Sjb 596178479Sjb#if DEBUG 597178479Sjb KASSERT(pd->pm_descr.pd_class == amd_pmc_class, 598178479Sjb ("[amd,%d] unknown PMC class (%d)", __LINE__, 599178479Sjb pd->pm_descr.pd_class)); 600178479Sjb#endif 601178479Sjb 602178479Sjb KASSERT(AMD_PMC_IS_STOPPED(pd->pm_evsel), 603178479Sjb ("[amd,%d] pmc%d,cpu%d: Starting active PMC \"%s\"", __LINE__, 604178479Sjb ri, cpu, pd->pm_descr.pd_name)); 605178479Sjb 606178479Sjb /* turn on the PMC ENABLE bit */ 607178479Sjb config = pm->pm_md.pm_amd.pm_amd_evsel | AMD_PMC_ENABLE; 608178479Sjb 609178479Sjb PMCDBG(MDP,STA,2,"amd-start config=0x%x", config); 610178479Sjb 611178479Sjb wrmsr(pd->pm_evsel, config); 612178479Sjb return 0; 613178479Sjb} 614178479Sjb 615178479Sjb/* 616178479Sjb * Stop a PMC. 617178479Sjb */ 618178479Sjb 619178479Sjbstatic int 620178479Sjbamd_stop_pmc(int cpu, int ri) 621178479Sjb{ 622178479Sjb struct pmc *pm; 623178479Sjb struct pmc_hw *phw; 624178479Sjb const struct amd_descr *pd; 625178479Sjb uint64_t config; 626178479Sjb 627178479Sjb KASSERT(cpu >= 0 && cpu < mp_ncpus, 628178479Sjb ("[amd,%d] illegal CPU value %d", __LINE__, cpu)); 629178479Sjb KASSERT(ri >= 0 && ri < AMD_NPMCS, 630178479Sjb ("[amd,%d] illegal row-index %d", __LINE__, ri)); 631178479Sjb 632178479Sjb phw = pmc_pcpu[cpu]->pc_hwpmcs[ri]; 633178479Sjb pm = phw->phw_pmc; 634178479Sjb pd = &amd_pmcdesc[ri]; 635178479Sjb 636178479Sjb KASSERT(pm != NULL, 637178479Sjb ("[amd,%d] cpu%d,pmc%d no PMC to stop", __LINE__, 638178479Sjb cpu, ri)); 639178479Sjb 640178479Sjb /* can't stop a TSC */ 641178479Sjb if (pd->pm_descr.pd_class == PMC_CLASS_TSC) 642178479Sjb return 0; 643178479Sjb 644178479Sjb#if DEBUG 645178479Sjb KASSERT(pd->pm_descr.pd_class == amd_pmc_class, 646178479Sjb ("[amd,%d] unknown PMC class (%d)", __LINE__, 647178479Sjb pd->pm_descr.pd_class)); 648178479Sjb#endif 649178479Sjb 650178479Sjb KASSERT(!AMD_PMC_IS_STOPPED(pd->pm_evsel), 651178479Sjb ("[amd,%d] PMC%d, CPU%d \"%s\" already stopped", 652178479Sjb __LINE__, ri, cpu, pd->pm_descr.pd_name)); 653178479Sjb 654178479Sjb PMCDBG(MDP,STO,1,"amd-stop ri=%d", ri); 655178479Sjb 656178479Sjb /* turn off the PMC ENABLE bit */ 657178479Sjb config = pm->pm_md.pm_amd.pm_amd_evsel & ~AMD_PMC_ENABLE; 658178479Sjb wrmsr(pd->pm_evsel, config); 659178479Sjb return 0; 660178479Sjb} 661178479Sjb 662178479Sjb/* 663178479Sjb * Interrupt handler. This function needs to return '1' if the 664178479Sjb * interrupt was this CPU's PMCs or '0' otherwise. It is not allowed 665178479Sjb * to sleep or do anything a 'fast' interrupt handler is not allowed 666178479Sjb * to do. 667178479Sjb */ 668178479Sjb 669178479Sjbstatic int 670178479Sjbamd_intr(int cpu, uintptr_t eip, int usermode) 671178479Sjb{ 672178479Sjb int i, error, retval, ri; 673178479Sjb uint32_t config, evsel, perfctr; 674178479Sjb struct pmc *pm; 675178479Sjb struct pmc_cpu *pc; 676178479Sjb struct pmc_hw *phw; 677178479Sjb pmc_value_t v; 678178479Sjb 679178479Sjb KASSERT(cpu >= 0 && cpu < mp_ncpus, 680178479Sjb ("[amd,%d] out of range CPU %d", __LINE__, cpu)); 681178479Sjb 682178479Sjb PMCDBG(MDP,INT,1, "cpu=%d eip=%p um=%d", cpu, (void *) eip, 683178479Sjb usermode); 684178479Sjb 685178479Sjb retval = 0; 686178479Sjb 687178479Sjb pc = pmc_pcpu[cpu]; 688178479Sjb 689178479Sjb /* 690178479Sjb * look for all PMCs that have interrupted: 691178479Sjb * - skip over the TSC [PMC#0] 692178479Sjb * - look for a running, sampling PMC which has overflowed 693178479Sjb * and which has a valid 'struct pmc' association 694178479Sjb * 695178479Sjb * If found, we call a helper to process the interrupt. 696178479Sjb * 697178479Sjb * If multiple PMCs interrupt at the same time, the AMD64 698178479Sjb * processor appears to deliver as many NMIs as there are 699178479Sjb * outstanding PMC interrupts. Thus we need to only process 700178479Sjb * one interrupt at a time. 701178479Sjb */ 702178479Sjb 703178479Sjb for (i = 0; retval == 0 && i < AMD_NPMCS-1; i++) { 704178479Sjb 705178479Sjb ri = i + 1; /* row index; TSC is at ri == 0 */ 706178479Sjb 707178479Sjb if (!AMD_PMC_HAS_OVERFLOWED(i)) 708178479Sjb continue; 709178479Sjb 710178479Sjb phw = pc->pc_hwpmcs[ri]; 711178479Sjb 712178479Sjb KASSERT(phw != NULL, ("[amd,%d] null PHW pointer", __LINE__)); 713178479Sjb 714178479Sjb if ((pm = phw->phw_pmc) == NULL || 715178479Sjb pm->pm_state != PMC_STATE_RUNNING || 716178479Sjb !PMC_IS_SAMPLING_MODE(PMC_TO_MODE(pm))) { 717178479Sjb continue; 718178479Sjb } 719178479Sjb 720178479Sjb retval = 1; /* found an interrupting PMC */ 721178479Sjb 722178479Sjb /* stop the PMC, reload count */ 723178479Sjb evsel = AMD_PMC_EVSEL_0 + i; 724178479Sjb perfctr = AMD_PMC_PERFCTR_0 + i; 725178479Sjb v = pm->pm_sc.pm_reloadcount; 726178479Sjb config = rdmsr(evsel); 727178479Sjb 728178479Sjb KASSERT((config & ~AMD_PMC_ENABLE) == 729178479Sjb (pm->pm_md.pm_amd.pm_amd_evsel & ~AMD_PMC_ENABLE), 730178479Sjb ("[amd,%d] config mismatch reg=0x%x pm=0x%x", __LINE__, 731178479Sjb config, pm->pm_md.pm_amd.pm_amd_evsel)); 732178479Sjb 733178479Sjb wrmsr(evsel, config & ~AMD_PMC_ENABLE); 734178479Sjb wrmsr(perfctr, AMD_RELOAD_COUNT_TO_PERFCTR_VALUE(v)); 735178479Sjb 736178479Sjb /* restart the counter if there was no error during logging */ 737178479Sjb error = pmc_process_interrupt(cpu, pm, eip, usermode); 738178479Sjb if (error == 0) 739178479Sjb wrmsr(evsel, config | AMD_PMC_ENABLE); 740178479Sjb } 741178479Sjb 742178479Sjb atomic_add_int(retval ? &pmc_stats.pm_intr_processed : 743178479Sjb &pmc_stats.pm_intr_ignored, 1); 744178479Sjb 745178479Sjb return retval; 746178479Sjb} 747178479Sjb 748178479Sjb/* 749178479Sjb * describe a PMC 750178479Sjb */ 751178479Sjbstatic int 752178479Sjbamd_describe(int cpu, int ri, struct pmc_info *pi, struct pmc **ppmc) 753178479Sjb{ 754178479Sjb int error; 755178479Sjb size_t copied; 756178479Sjb const struct amd_descr *pd; 757178479Sjb struct pmc_hw *phw; 758178479Sjb 759178479Sjb KASSERT(cpu >= 0 && cpu < mp_ncpus, 760178479Sjb ("[amd,%d] illegal CPU %d", __LINE__, cpu)); 761178479Sjb KASSERT(ri >= 0 && ri < AMD_NPMCS, 762178479Sjb ("[amd,%d] row-index %d out of range", __LINE__, ri)); 763178479Sjb 764178479Sjb phw = pmc_pcpu[cpu]->pc_hwpmcs[ri]; 765178479Sjb pd = &amd_pmcdesc[ri]; 766178479Sjb 767178479Sjb if ((error = copystr(pd->pm_descr.pd_name, pi->pm_name, 768178479Sjb PMC_NAME_MAX, &copied)) != 0) 769178479Sjb return error; 770178479Sjb 771178479Sjb pi->pm_class = pd->pm_descr.pd_class; 772178479Sjb 773178479Sjb if (phw->phw_state & PMC_PHW_FLAG_IS_ENABLED) { 774178479Sjb pi->pm_enabled = TRUE; 775178479Sjb *ppmc = phw->phw_pmc; 776178479Sjb } else { 777178479Sjb pi->pm_enabled = FALSE; 778178479Sjb *ppmc = NULL; 779178479Sjb } 780178479Sjb 781178479Sjb return 0; 782178479Sjb} 783178479Sjb 784178479Sjb/* 785178479Sjb * i386 specific entry points 786178479Sjb */ 787178479Sjb 788178479Sjb/* 789178479Sjb * return the MSR address of the given PMC. 790178479Sjb */ 791178479Sjb 792178479Sjbstatic int 793178479Sjbamd_get_msr(int ri, uint32_t *msr) 794178479Sjb{ 795178479Sjb KASSERT(ri >= 0 && ri < AMD_NPMCS, 796178479Sjb ("[amd,%d] ri %d out of range", __LINE__, ri)); 797178479Sjb 798178479Sjb *msr = amd_pmcdesc[ri].pm_perfctr - AMD_PMC_PERFCTR_0; 799178479Sjb return 0; 800178479Sjb} 801178479Sjb 802178479Sjb/* 803178479Sjb * processor dependent initialization. 804178479Sjb */ 805178479Sjb 806178479Sjb/* 807178479Sjb * Per-processor data structure 808178479Sjb * 809178479Sjb * [common stuff] 810178479Sjb * [5 struct pmc_hw pointers] 811178479Sjb * [5 struct pmc_hw structures] 812178479Sjb */ 813178479Sjb 814178479Sjbstruct amd_cpu { 815178479Sjb struct pmc_cpu pc_common; 816178479Sjb struct pmc_hw *pc_hwpmcs[AMD_NPMCS]; 817178479Sjb struct pmc_hw pc_amdpmcs[AMD_NPMCS]; 818178479Sjb}; 819178479Sjb 820178479Sjb 821178479Sjbstatic int 822178479Sjbamd_init(int cpu) 823178479Sjb{ 824178479Sjb int n; 825178479Sjb struct amd_cpu *pcs; 826178479Sjb struct pmc_hw *phw; 827178479Sjb 828178479Sjb KASSERT(cpu >= 0 && cpu < mp_ncpus, 829178479Sjb ("[amd,%d] insane cpu number %d", __LINE__, cpu)); 830178479Sjb 831178479Sjb PMCDBG(MDP,INI,1,"amd-init cpu=%d", cpu); 832178479Sjb 833178479Sjb MALLOC(pcs, struct amd_cpu *, sizeof(struct amd_cpu), M_PMC, 834178479Sjb M_WAITOK|M_ZERO); 835178479Sjb 836178479Sjb phw = &pcs->pc_amdpmcs[0]; 837178479Sjb 838178479Sjb /* 839178479Sjb * Initialize the per-cpu mutex and set the content of the 840178479Sjb * hardware descriptors to a known state. 841178479Sjb */ 842178479Sjb 843178479Sjb for (n = 0; n < AMD_NPMCS; n++, phw++) { 844178479Sjb phw->phw_state = PMC_PHW_FLAG_IS_ENABLED | 845178479Sjb PMC_PHW_CPU_TO_STATE(cpu) | PMC_PHW_INDEX_TO_STATE(n); 846178479Sjb phw->phw_pmc = NULL; 847178479Sjb pcs->pc_hwpmcs[n] = phw; 848178479Sjb } 849178479Sjb 850178479Sjb /* Mark the TSC as shareable */ 851178479Sjb pcs->pc_hwpmcs[0]->phw_state |= PMC_PHW_FLAG_IS_SHAREABLE; 852178479Sjb 853178479Sjb pmc_pcpu[cpu] = (struct pmc_cpu *) pcs; 854178479Sjb 855178479Sjb return 0; 856178479Sjb} 857178479Sjb 858178479Sjb 859178479Sjb/* 860178479Sjb * processor dependent cleanup prior to the KLD 861178479Sjb * being unloaded 862178479Sjb */ 863178479Sjb 864178479Sjbstatic int 865178479Sjbamd_cleanup(int cpu) 866178479Sjb{ 867178479Sjb int i; 868178479Sjb uint32_t evsel; 869178479Sjb struct pmc_cpu *pcs; 870178479Sjb 871178479Sjb KASSERT(cpu >= 0 && cpu < mp_ncpus, 872178479Sjb ("[amd,%d] insane cpu number (%d)", __LINE__, cpu)); 873178479Sjb 874178479Sjb PMCDBG(MDP,INI,1,"amd-cleanup cpu=%d", cpu); 875178479Sjb 876178479Sjb /* 877178479Sjb * First, turn off all PMCs on this CPU. 878178479Sjb */ 879178479Sjb 880178479Sjb for (i = 0; i < 4; i++) { /* XXX this loop is now not needed */ 881178479Sjb evsel = rdmsr(AMD_PMC_EVSEL_0 + i); 882178479Sjb evsel &= ~AMD_PMC_ENABLE; 883178479Sjb wrmsr(AMD_PMC_EVSEL_0 + i, evsel); 884178479Sjb } 885178479Sjb 886178479Sjb /* 887178479Sjb * Next, free up allocated space. 888178479Sjb */ 889178479Sjb 890178479Sjb if ((pcs = pmc_pcpu[cpu]) == NULL) 891178479Sjb return 0; 892178479Sjb 893178479Sjb#if DEBUG 894178479Sjb /* check the TSC */ 895178479Sjb KASSERT(pcs->pc_hwpmcs[0]->phw_pmc == NULL, 896178479Sjb ("[amd,%d] CPU%d,PMC0 still in use", __LINE__, cpu)); 897178479Sjb for (i = 1; i < AMD_NPMCS; i++) { 898178479Sjb KASSERT(pcs->pc_hwpmcs[i]->phw_pmc == NULL, 899178479Sjb ("[amd,%d] CPU%d/PMC%d in use", __LINE__, cpu, i)); 900178479Sjb KASSERT(AMD_PMC_IS_STOPPED(AMD_PMC_EVSEL_0 + (i-1)), 901178479Sjb ("[amd,%d] CPU%d/PMC%d not stopped", __LINE__, cpu, i)); 902178479Sjb } 903178479Sjb#endif 904178479Sjb 905178479Sjb pmc_pcpu[cpu] = NULL; 906178479Sjb FREE(pcs, M_PMC); 907178479Sjb return 0; 908178479Sjb} 909178479Sjb 910178479Sjb/* 911178479Sjb * Initialize ourselves. 912178479Sjb */ 913178479Sjb 914178479Sjbstruct pmc_mdep * 915178479Sjbpmc_amd_initialize(void) 916178479Sjb{ 917178479Sjb enum pmc_cputype cputype; 918178479Sjb enum pmc_class class; 919178479Sjb struct pmc_mdep *pmc_mdep; 920178479Sjb char *name; 921178479Sjb int i; 922178479Sjb 923178479Sjb /* 924178479Sjb * The presence of hardware performance counters on the AMD 925178479Sjb * Athlon, Duron or later processors, is _not_ indicated by 926178479Sjb * any of the processor feature flags set by the 'CPUID' 927178479Sjb * instruction, so we only check the 'instruction family' 928178479Sjb * field returned by CPUID for instruction family >= 6. 929178479Sjb */ 930178479Sjb 931178479Sjb class = cputype = -1; 932178479Sjb name = NULL; 933178479Sjb switch (cpu_id & 0xF00) { 934178479Sjb case 0x600: /* Athlon(tm) processor */ 935178479Sjb cputype = PMC_CPU_AMD_K7; 936178479Sjb class = PMC_CLASS_K7; 937178479Sjb name = "K7"; 938178479Sjb break; 939178479Sjb case 0xF00: /* Athlon64/Opteron processor */ 940178479Sjb cputype = PMC_CPU_AMD_K8; 941178479Sjb class = PMC_CLASS_K8; 942178479Sjb name = "K8"; 943178479Sjb break; 944178479Sjb } 945178479Sjb 946178479Sjb if ((int) cputype == -1) { 947178479Sjb (void) printf("pmc: Unknown AMD CPU.\n"); 948178479Sjb return NULL; 949178479Sjb } 950178479Sjb 951178479Sjb#if DEBUG 952178479Sjb amd_pmc_class = class; 953178479Sjb#endif 954178479Sjb 955178479Sjb MALLOC(pmc_mdep, struct pmc_mdep *, sizeof(struct pmc_mdep), 956178479Sjb M_PMC, M_WAITOK|M_ZERO); 957178479Sjb 958178479Sjb pmc_mdep->pmd_cputype = cputype; 959178479Sjb pmc_mdep->pmd_npmc = AMD_NPMCS; 960178479Sjb 961178479Sjb /* this processor has two classes of usable PMCs */ 962178479Sjb pmc_mdep->pmd_nclass = 2; 963178479Sjb 964178479Sjb /* TSC */ 965178479Sjb pmc_mdep->pmd_classes[0].pm_class = PMC_CLASS_TSC; 966178479Sjb pmc_mdep->pmd_classes[0].pm_caps = PMC_CAP_READ; 967178479Sjb pmc_mdep->pmd_classes[0].pm_width = 64; 968178479Sjb 969178479Sjb /* AMD K7/K8 PMCs */ 970178479Sjb pmc_mdep->pmd_classes[1].pm_class = class; 971178479Sjb pmc_mdep->pmd_classes[1].pm_caps = AMD_PMC_CAPS; 972178479Sjb pmc_mdep->pmd_classes[1].pm_width = 48; 973178479Sjb 974178479Sjb pmc_mdep->pmd_nclasspmcs[0] = 1; 975178479Sjb pmc_mdep->pmd_nclasspmcs[1] = (AMD_NPMCS-1); 976178479Sjb 977178479Sjb /* fill in the correct pmc name and class */ 978178479Sjb for (i = 1; i < AMD_NPMCS; i++) { 979178479Sjb (void) snprintf(amd_pmcdesc[i].pm_descr.pd_name, 980178479Sjb sizeof(amd_pmcdesc[i].pm_descr.pd_name), "%s-%d", 981178479Sjb name, i-1); 982178479Sjb amd_pmcdesc[i].pm_descr.pd_class = class; 983178479Sjb } 984178479Sjb 985178479Sjb pmc_mdep->pmd_init = amd_init; 986178479Sjb pmc_mdep->pmd_cleanup = amd_cleanup; 987178479Sjb pmc_mdep->pmd_switch_in = amd_switch_in; 988178479Sjb pmc_mdep->pmd_switch_out = amd_switch_out; 989178479Sjb pmc_mdep->pmd_read_pmc = amd_read_pmc; 990178479Sjb pmc_mdep->pmd_write_pmc = amd_write_pmc; 991178479Sjb pmc_mdep->pmd_config_pmc = amd_config_pmc; 992178479Sjb pmc_mdep->pmd_get_config = amd_get_config; 993178479Sjb pmc_mdep->pmd_allocate_pmc = amd_allocate_pmc; 994178479Sjb pmc_mdep->pmd_release_pmc = amd_release_pmc; 995178479Sjb pmc_mdep->pmd_start_pmc = amd_start_pmc; 996178479Sjb pmc_mdep->pmd_stop_pmc = amd_stop_pmc; 997178479Sjb pmc_mdep->pmd_intr = amd_intr; 998178479Sjb pmc_mdep->pmd_describe = amd_describe; 999178479Sjb pmc_mdep->pmd_get_msr = amd_get_msr; /* i386 */ 1000178479Sjb 1001178479Sjb PMCDBG(MDP,INI,0,"%s","amd-initialize"); 1002178479Sjb 1003178479Sjb return pmc_mdep; 1004178479Sjb} 1005178479Sjb