hwpmc_amd.c revision 153110
1145256Sjkoshy/*- 2145256Sjkoshy * Copyright (c) 2003-2005 Joseph Koshy 3145256Sjkoshy * All rights reserved. 4145256Sjkoshy * 5145256Sjkoshy * Redistribution and use in source and binary forms, with or without 6145256Sjkoshy * modification, are permitted provided that the following conditions 7145256Sjkoshy * are met: 8145256Sjkoshy * 1. Redistributions of source code must retain the above copyright 9145256Sjkoshy * notice, this list of conditions and the following disclaimer. 10145256Sjkoshy * 2. Redistributions in binary form must reproduce the above copyright 11145256Sjkoshy * notice, this list of conditions and the following disclaimer in the 12145256Sjkoshy * documentation and/or other materials provided with the distribution. 13145256Sjkoshy * 14145256Sjkoshy * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 15145256Sjkoshy * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 16145256Sjkoshy * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 17145256Sjkoshy * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 18145256Sjkoshy * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 19145256Sjkoshy * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 20145256Sjkoshy * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 21145256Sjkoshy * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 22145256Sjkoshy * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 23145256Sjkoshy * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 24145256Sjkoshy * SUCH DAMAGE. 25145256Sjkoshy * 26145256Sjkoshy */ 27145256Sjkoshy 28145256Sjkoshy#include <sys/cdefs.h> 29145256Sjkoshy__FBSDID("$FreeBSD: head/sys/dev/hwpmc/hwpmc_amd.c 153110 2005-12-05 11:58:35Z ru $"); 30145256Sjkoshy 31145256Sjkoshy/* Support for the AMD K7 and later processors */ 32145256Sjkoshy 33145256Sjkoshy#include <sys/param.h> 34145256Sjkoshy#include <sys/lock.h> 35145256Sjkoshy#include <sys/malloc.h> 36145256Sjkoshy#include <sys/mutex.h> 37145338Smarcel#include <sys/pmc.h> 38145256Sjkoshy#include <sys/smp.h> 39145256Sjkoshy#include <sys/systm.h> 40145256Sjkoshy 41147191Sjkoshy#include <machine/cpufunc.h> 42145256Sjkoshy#include <machine/md_var.h> 43147191Sjkoshy#include <machine/pmc_mdep.h> 44147191Sjkoshy#include <machine/specialreg.h> 45145256Sjkoshy 46153110Sru#ifdef DEBUG 47147191Sjkoshyenum pmc_class amd_pmc_class; 48145256Sjkoshy#endif 49145256Sjkoshy 50145256Sjkoshy/* AMD K7 & K8 PMCs */ 51145256Sjkoshystruct amd_descr { 52145256Sjkoshy struct pmc_descr pm_descr; /* "base class" */ 53145256Sjkoshy uint32_t pm_evsel; /* address of EVSEL register */ 54145256Sjkoshy uint32_t pm_perfctr; /* address of PERFCTR register */ 55145256Sjkoshy}; 56145256Sjkoshy 57147191Sjkoshystatic struct amd_descr amd_pmcdesc[AMD_NPMCS] = 58145256Sjkoshy{ 59145256Sjkoshy { 60145256Sjkoshy .pm_descr = 61145256Sjkoshy { 62145256Sjkoshy .pd_name = "TSC", 63145256Sjkoshy .pd_class = PMC_CLASS_TSC, 64145256Sjkoshy .pd_caps = PMC_CAP_READ, 65145256Sjkoshy .pd_width = 64 66145256Sjkoshy }, 67145256Sjkoshy .pm_evsel = MSR_TSC, 68145256Sjkoshy .pm_perfctr = 0 /* unused */ 69145256Sjkoshy }, 70145256Sjkoshy 71145256Sjkoshy { 72145256Sjkoshy .pm_descr = 73145256Sjkoshy { 74147191Sjkoshy .pd_name = "", 75147191Sjkoshy .pd_class = -1, 76145256Sjkoshy .pd_caps = AMD_PMC_CAPS, 77145256Sjkoshy .pd_width = 48 78145256Sjkoshy }, 79145256Sjkoshy .pm_evsel = AMD_PMC_EVSEL_0, 80145256Sjkoshy .pm_perfctr = AMD_PMC_PERFCTR_0 81145256Sjkoshy }, 82145256Sjkoshy { 83145256Sjkoshy .pm_descr = 84145256Sjkoshy { 85147191Sjkoshy .pd_name = "", 86147191Sjkoshy .pd_class = -1, 87145256Sjkoshy .pd_caps = AMD_PMC_CAPS, 88145256Sjkoshy .pd_width = 48 89145256Sjkoshy }, 90145256Sjkoshy .pm_evsel = AMD_PMC_EVSEL_1, 91145256Sjkoshy .pm_perfctr = AMD_PMC_PERFCTR_1 92145256Sjkoshy }, 93145256Sjkoshy { 94145256Sjkoshy .pm_descr = 95145256Sjkoshy { 96147191Sjkoshy .pd_name = "", 97147191Sjkoshy .pd_class = -1, 98145256Sjkoshy .pd_caps = AMD_PMC_CAPS, 99145256Sjkoshy .pd_width = 48 100145256Sjkoshy }, 101145256Sjkoshy .pm_evsel = AMD_PMC_EVSEL_2, 102145256Sjkoshy .pm_perfctr = AMD_PMC_PERFCTR_2 103145256Sjkoshy }, 104145256Sjkoshy { 105145256Sjkoshy .pm_descr = 106145256Sjkoshy { 107147191Sjkoshy .pd_name = "", 108147191Sjkoshy .pd_class = -1, 109145256Sjkoshy .pd_caps = AMD_PMC_CAPS, 110145256Sjkoshy .pd_width = 48 111145256Sjkoshy }, 112145256Sjkoshy .pm_evsel = AMD_PMC_EVSEL_3, 113145256Sjkoshy .pm_perfctr = AMD_PMC_PERFCTR_3 114145256Sjkoshy } 115145256Sjkoshy}; 116145256Sjkoshy 117145256Sjkoshystruct amd_event_code_map { 118145256Sjkoshy enum pmc_event pe_ev; /* enum value */ 119145256Sjkoshy uint8_t pe_code; /* encoded event mask */ 120145256Sjkoshy uint8_t pe_mask; /* bits allowed in unit mask */ 121145256Sjkoshy}; 122145256Sjkoshy 123145256Sjkoshyconst struct amd_event_code_map amd_event_codes[] = { 124147191Sjkoshy#if defined(__i386__) /* 32 bit Athlon (K7) only */ 125145256Sjkoshy { PMC_EV_K7_DC_ACCESSES, 0x40, 0 }, 126145256Sjkoshy { PMC_EV_K7_DC_MISSES, 0x41, 0 }, 127147191Sjkoshy { PMC_EV_K7_DC_REFILLS_FROM_L2, 0x42, AMD_PMC_UNITMASK_MOESI }, 128147191Sjkoshy { PMC_EV_K7_DC_REFILLS_FROM_SYSTEM, 0x43, AMD_PMC_UNITMASK_MOESI }, 129147191Sjkoshy { PMC_EV_K7_DC_WRITEBACKS, 0x44, AMD_PMC_UNITMASK_MOESI }, 130145256Sjkoshy { PMC_EV_K7_L1_DTLB_MISS_AND_L2_DTLB_HITS, 0x45, 0 }, 131145256Sjkoshy { PMC_EV_K7_L1_AND_L2_DTLB_MISSES, 0x46, 0 }, 132145256Sjkoshy { PMC_EV_K7_MISALIGNED_REFERENCES, 0x47, 0 }, 133145256Sjkoshy 134145256Sjkoshy { PMC_EV_K7_IC_FETCHES, 0x80, 0 }, 135145256Sjkoshy { PMC_EV_K7_IC_MISSES, 0x81, 0 }, 136145256Sjkoshy 137145256Sjkoshy { PMC_EV_K7_L1_ITLB_MISSES, 0x84, 0 }, 138145256Sjkoshy { PMC_EV_K7_L1_L2_ITLB_MISSES, 0x85, 0 }, 139145256Sjkoshy 140145256Sjkoshy { PMC_EV_K7_RETIRED_INSTRUCTIONS, 0xC0, 0 }, 141145256Sjkoshy { PMC_EV_K7_RETIRED_OPS, 0xC1, 0 }, 142145256Sjkoshy { PMC_EV_K7_RETIRED_BRANCHES, 0xC2, 0 }, 143145256Sjkoshy { PMC_EV_K7_RETIRED_BRANCHES_MISPREDICTED, 0xC3, 0 }, 144145256Sjkoshy { PMC_EV_K7_RETIRED_TAKEN_BRANCHES, 0xC4, 0 }, 145145256Sjkoshy { PMC_EV_K7_RETIRED_TAKEN_BRANCHES_MISPREDICTED, 0xC5, 0 }, 146145256Sjkoshy { PMC_EV_K7_RETIRED_FAR_CONTROL_TRANSFERS, 0xC6, 0 }, 147145256Sjkoshy { PMC_EV_K7_RETIRED_RESYNC_BRANCHES, 0xC7, 0 }, 148145256Sjkoshy { PMC_EV_K7_INTERRUPTS_MASKED_CYCLES, 0xCD, 0 }, 149145256Sjkoshy { PMC_EV_K7_INTERRUPTS_MASKED_WHILE_PENDING_CYCLES, 0xCE, 0 }, 150147191Sjkoshy { PMC_EV_K7_HARDWARE_INTERRUPTS, 0xCF, 0 }, 151145256Sjkoshy#endif 152145256Sjkoshy 153145256Sjkoshy { PMC_EV_K8_FP_DISPATCHED_FPU_OPS, 0x00, 0x3F }, 154145256Sjkoshy { PMC_EV_K8_FP_CYCLES_WITH_NO_FPU_OPS_RETIRED, 0x01, 0x00 }, 155145256Sjkoshy { PMC_EV_K8_FP_DISPATCHED_FPU_FAST_FLAG_OPS, 0x02, 0x00 }, 156145256Sjkoshy 157145256Sjkoshy { PMC_EV_K8_LS_SEGMENT_REGISTER_LOAD, 0x20, 0x7F }, 158145256Sjkoshy { PMC_EV_K8_LS_MICROARCHITECTURAL_RESYNC_BY_SELF_MODIFYING_CODE, 159145256Sjkoshy 0x21, 0x00 }, 160145256Sjkoshy { PMC_EV_K8_LS_MICROARCHITECTURAL_RESYNC_BY_SNOOP, 0x22, 0x00 }, 161145256Sjkoshy { PMC_EV_K8_LS_BUFFER2_FULL, 0x23, 0x00 }, 162145256Sjkoshy { PMC_EV_K8_LS_LOCKED_OPERATION, 0x24, 0x07 }, 163145256Sjkoshy { PMC_EV_K8_LS_MICROARCHITECTURAL_LATE_CANCEL, 0x25, 0x00 }, 164145256Sjkoshy { PMC_EV_K8_LS_RETIRED_CFLUSH_INSTRUCTIONS, 0x26, 0x00 }, 165145256Sjkoshy { PMC_EV_K8_LS_RETIRED_CPUID_INSTRUCTIONS, 0x27, 0x00 }, 166145256Sjkoshy 167145256Sjkoshy { PMC_EV_K8_DC_ACCESS, 0x40, 0x00 }, 168145256Sjkoshy { PMC_EV_K8_DC_MISS, 0x41, 0x00 }, 169145256Sjkoshy { PMC_EV_K8_DC_REFILL_FROM_L2, 0x42, 0x1F }, 170145256Sjkoshy { PMC_EV_K8_DC_REFILL_FROM_SYSTEM, 0x43, 0x1F }, 171145256Sjkoshy { PMC_EV_K8_DC_COPYBACK, 0x44, 0x1F }, 172145256Sjkoshy { PMC_EV_K8_DC_L1_DTLB_MISS_AND_L2_DTLB_HIT, 0x45, 0x00 }, 173145256Sjkoshy { PMC_EV_K8_DC_L1_DTLB_MISS_AND_L2_DTLB_MISS, 0x46, 0x00 }, 174145256Sjkoshy { PMC_EV_K8_DC_MISALIGNED_DATA_REFERENCE, 0x47, 0x00 }, 175145256Sjkoshy { PMC_EV_K8_DC_MICROARCHITECTURAL_LATE_CANCEL, 0x48, 0x00 }, 176145256Sjkoshy { PMC_EV_K8_DC_MICROARCHITECTURAL_EARLY_CANCEL, 0x49, 0x00 }, 177145256Sjkoshy { PMC_EV_K8_DC_ONE_BIT_ECC_ERROR, 0x4A, 0x03 }, 178145256Sjkoshy { PMC_EV_K8_DC_DISPATCHED_PREFETCH_INSTRUCTIONS, 0x4B, 0x07 }, 179145256Sjkoshy { PMC_EV_K8_DC_DCACHE_ACCESSES_BY_LOCKS, 0x4C, 0x03 }, 180145256Sjkoshy 181145256Sjkoshy { PMC_EV_K8_BU_CPU_CLK_UNHALTED, 0x76, 0x00 }, 182145256Sjkoshy { PMC_EV_K8_BU_INTERNAL_L2_REQUEST, 0x7D, 0x1F }, 183145256Sjkoshy { PMC_EV_K8_BU_FILL_REQUEST_L2_MISS, 0x7E, 0x07 }, 184145256Sjkoshy { PMC_EV_K8_BU_FILL_INTO_L2, 0x7F, 0x03 }, 185145256Sjkoshy 186145256Sjkoshy { PMC_EV_K8_IC_FETCH, 0x80, 0x00 }, 187145256Sjkoshy { PMC_EV_K8_IC_MISS, 0x81, 0x00 }, 188145256Sjkoshy { PMC_EV_K8_IC_REFILL_FROM_L2, 0x82, 0x00 }, 189145256Sjkoshy { PMC_EV_K8_IC_REFILL_FROM_SYSTEM, 0x83, 0x00 }, 190145256Sjkoshy { PMC_EV_K8_IC_L1_ITLB_MISS_AND_L2_ITLB_HIT, 0x84, 0x00 }, 191145256Sjkoshy { PMC_EV_K8_IC_L1_ITLB_MISS_AND_L2_ITLB_MISS, 0x85, 0x00 }, 192145256Sjkoshy { PMC_EV_K8_IC_MICROARCHITECTURAL_RESYNC_BY_SNOOP, 0x86, 0x00 }, 193145256Sjkoshy { PMC_EV_K8_IC_INSTRUCTION_FETCH_STALL, 0x87, 0x00 }, 194145256Sjkoshy { PMC_EV_K8_IC_RETURN_STACK_HIT, 0x88, 0x00 }, 195145256Sjkoshy { PMC_EV_K8_IC_RETURN_STACK_OVERFLOW, 0x89, 0x00 }, 196145256Sjkoshy 197145256Sjkoshy { PMC_EV_K8_FR_RETIRED_X86_INSTRUCTIONS, 0xC0, 0x00 }, 198145256Sjkoshy { PMC_EV_K8_FR_RETIRED_UOPS, 0xC1, 0x00 }, 199145256Sjkoshy { PMC_EV_K8_FR_RETIRED_BRANCHES, 0xC2, 0x00 }, 200145256Sjkoshy { PMC_EV_K8_FR_RETIRED_BRANCHES_MISPREDICTED, 0xC3, 0x00 }, 201145256Sjkoshy { PMC_EV_K8_FR_RETIRED_TAKEN_BRANCHES, 0xC4, 0x00 }, 202145256Sjkoshy { PMC_EV_K8_FR_RETIRED_TAKEN_BRANCHES_MISPREDICTED, 0xC5, 0x00 }, 203145256Sjkoshy { PMC_EV_K8_FR_RETIRED_FAR_CONTROL_TRANSFERS, 0xC6, 0x00 }, 204145256Sjkoshy { PMC_EV_K8_FR_RETIRED_RESYNCS, 0xC7, 0x00 }, 205145256Sjkoshy { PMC_EV_K8_FR_RETIRED_NEAR_RETURNS, 0xC8, 0x00 }, 206145256Sjkoshy { PMC_EV_K8_FR_RETIRED_NEAR_RETURNS_MISPREDICTED, 0xC9, 0x00 }, 207145256Sjkoshy { PMC_EV_K8_FR_RETIRED_TAKEN_BRANCHES_MISPREDICTED_BY_ADDR_MISCOMPARE, 208145256Sjkoshy 0xCA, 0x00 }, 209145256Sjkoshy { PMC_EV_K8_FR_RETIRED_FPU_INSTRUCTIONS, 0xCB, 0x0F }, 210145256Sjkoshy { PMC_EV_K8_FR_RETIRED_FASTPATH_DOUBLE_OP_INSTRUCTIONS, 211145256Sjkoshy 0xCC, 0x07 }, 212145256Sjkoshy { PMC_EV_K8_FR_INTERRUPTS_MASKED_CYCLES, 0xCD, 0x00 }, 213145256Sjkoshy { PMC_EV_K8_FR_INTERRUPTS_MASKED_WHILE_PENDING_CYCLES, 0xCE, 0x00 }, 214145256Sjkoshy { PMC_EV_K8_FR_TAKEN_HARDWARE_INTERRUPTS, 0xCF, 0x00 }, 215145256Sjkoshy 216145256Sjkoshy { PMC_EV_K8_FR_DECODER_EMPTY, 0xD0, 0x00 }, 217145256Sjkoshy { PMC_EV_K8_FR_DISPATCH_STALLS, 0xD1, 0x00 }, 218145256Sjkoshy { PMC_EV_K8_FR_DISPATCH_STALL_FROM_BRANCH_ABORT_TO_RETIRE, 219145256Sjkoshy 0xD2, 0x00 }, 220145256Sjkoshy { PMC_EV_K8_FR_DISPATCH_STALL_FOR_SERIALIZATION, 0xD3, 0x00 }, 221145256Sjkoshy { PMC_EV_K8_FR_DISPATCH_STALL_FOR_SEGMENT_LOAD, 0xD4, 0x00 }, 222145256Sjkoshy { PMC_EV_K8_FR_DISPATCH_STALL_WHEN_REORDER_BUFFER_IS_FULL, 223145256Sjkoshy 0xD5, 0x00 }, 224145256Sjkoshy { PMC_EV_K8_FR_DISPATCH_STALL_WHEN_RESERVATION_STATIONS_ARE_FULL, 225145256Sjkoshy 0xD6, 0x00 }, 226145256Sjkoshy { PMC_EV_K8_FR_DISPATCH_STALL_WHEN_FPU_IS_FULL, 0xD7, 0x00 }, 227145256Sjkoshy { PMC_EV_K8_FR_DISPATCH_STALL_WHEN_LS_IS_FULL, 0xD8, 0x00 }, 228145256Sjkoshy { PMC_EV_K8_FR_DISPATCH_STALL_WHEN_WAITING_FOR_ALL_TO_BE_QUIET, 229145256Sjkoshy 0xD9, 0x00 }, 230145256Sjkoshy { PMC_EV_K8_FR_DISPATCH_STALL_WHEN_FAR_XFER_OR_RESYNC_BRANCH_PENDING, 231145256Sjkoshy 0xDA, 0x00 }, 232145256Sjkoshy { PMC_EV_K8_FR_FPU_EXCEPTIONS, 0xDB, 0x0F }, 233145256Sjkoshy { PMC_EV_K8_FR_NUMBER_OF_BREAKPOINTS_FOR_DR0, 0xDC, 0x00 }, 234145256Sjkoshy { PMC_EV_K8_FR_NUMBER_OF_BREAKPOINTS_FOR_DR1, 0xDD, 0x00 }, 235145256Sjkoshy { PMC_EV_K8_FR_NUMBER_OF_BREAKPOINTS_FOR_DR2, 0xDE, 0x00 }, 236145256Sjkoshy { PMC_EV_K8_FR_NUMBER_OF_BREAKPOINTS_FOR_DR3, 0xDF, 0x00 }, 237145256Sjkoshy 238145256Sjkoshy { PMC_EV_K8_NB_MEMORY_CONTROLLER_PAGE_ACCESS_EVENT, 0xE0, 0x7 }, 239145256Sjkoshy { PMC_EV_K8_NB_MEMORY_CONTROLLER_PAGE_TABLE_OVERFLOW, 0xE1, 0x00 }, 240145256Sjkoshy { PMC_EV_K8_NB_MEMORY_CONTROLLER_DRAM_COMMAND_SLOTS_MISSED, 241145256Sjkoshy 0xE2, 0x00 }, 242145256Sjkoshy { PMC_EV_K8_NB_MEMORY_CONTROLLER_TURNAROUND, 0xE3, 0x07 }, 243145256Sjkoshy { PMC_EV_K8_NB_MEMORY_CONTROLLER_BYPASS_SATURATION, 0xE4, 0x0F }, 244145256Sjkoshy { PMC_EV_K8_NB_SIZED_COMMANDS, 0xEB, 0x7F }, 245145256Sjkoshy { PMC_EV_K8_NB_PROBE_RESULT, 0xEC, 0x0F }, 246145256Sjkoshy { PMC_EV_K8_NB_HT_BUS0_BANDWIDTH, 0xF6, 0x0F }, 247145256Sjkoshy { PMC_EV_K8_NB_HT_BUS1_BANDWIDTH, 0xF7, 0x0F }, 248145256Sjkoshy { PMC_EV_K8_NB_HT_BUS2_BANDWIDTH, 0xF8, 0x0F } 249145256Sjkoshy 250145256Sjkoshy}; 251145256Sjkoshy 252145256Sjkoshyconst int amd_event_codes_size = 253145256Sjkoshy sizeof(amd_event_codes) / sizeof(amd_event_codes[0]); 254145256Sjkoshy 255145256Sjkoshy/* 256145256Sjkoshy * read a pmc register 257145256Sjkoshy */ 258145256Sjkoshy 259145256Sjkoshystatic int 260145256Sjkoshyamd_read_pmc(int cpu, int ri, pmc_value_t *v) 261145256Sjkoshy{ 262145256Sjkoshy enum pmc_mode mode; 263145256Sjkoshy const struct amd_descr *pd; 264145256Sjkoshy struct pmc *pm; 265145256Sjkoshy const struct pmc_hw *phw; 266145256Sjkoshy pmc_value_t tmp; 267145256Sjkoshy 268145256Sjkoshy KASSERT(cpu >= 0 && cpu < mp_ncpus, 269145256Sjkoshy ("[amd,%d] illegal CPU value %d", __LINE__, cpu)); 270145256Sjkoshy KASSERT(ri >= 0 && ri < AMD_NPMCS, 271145256Sjkoshy ("[amd,%d] illegal row-index %d", __LINE__, ri)); 272145256Sjkoshy 273145256Sjkoshy phw = pmc_pcpu[cpu]->pc_hwpmcs[ri]; 274145256Sjkoshy pd = &amd_pmcdesc[ri]; 275145256Sjkoshy pm = phw->phw_pmc; 276145256Sjkoshy 277145256Sjkoshy KASSERT(pm != NULL, 278145256Sjkoshy ("[amd,%d] No owner for HWPMC [cpu%d,pmc%d]", __LINE__, 279145256Sjkoshy cpu, ri)); 280145256Sjkoshy 281145774Sjkoshy mode = PMC_TO_MODE(pm); 282145256Sjkoshy 283145256Sjkoshy PMCDBG(MDP,REA,1,"amd-read id=%d class=%d", ri, pd->pm_descr.pd_class); 284145256Sjkoshy 285145256Sjkoshy /* Reading the TSC is a special case */ 286145256Sjkoshy if (pd->pm_descr.pd_class == PMC_CLASS_TSC) { 287145256Sjkoshy KASSERT(PMC_IS_COUNTING_MODE(mode), 288145256Sjkoshy ("[amd,%d] TSC counter in non-counting mode", __LINE__)); 289145256Sjkoshy *v = rdtsc(); 290145256Sjkoshy PMCDBG(MDP,REA,2,"amd-read id=%d -> %jd", ri, *v); 291145256Sjkoshy return 0; 292145256Sjkoshy } 293145256Sjkoshy 294153110Sru#ifdef DEBUG 295147191Sjkoshy KASSERT(pd->pm_descr.pd_class == amd_pmc_class, 296145256Sjkoshy ("[amd,%d] unknown PMC class (%d)", __LINE__, 297145256Sjkoshy pd->pm_descr.pd_class)); 298147191Sjkoshy#endif 299145256Sjkoshy 300145256Sjkoshy tmp = rdmsr(pd->pm_perfctr); /* RDMSR serializes */ 301145256Sjkoshy if (PMC_IS_SAMPLING_MODE(mode)) 302147191Sjkoshy *v = AMD_PERFCTR_VALUE_TO_RELOAD_COUNT(tmp); 303145256Sjkoshy else 304145256Sjkoshy *v = tmp; 305145256Sjkoshy 306145256Sjkoshy PMCDBG(MDP,REA,2,"amd-read id=%d -> %jd", ri, *v); 307145256Sjkoshy 308145256Sjkoshy return 0; 309145256Sjkoshy} 310145256Sjkoshy 311145256Sjkoshy/* 312145256Sjkoshy * Write a PMC MSR. 313145256Sjkoshy */ 314145256Sjkoshy 315145256Sjkoshystatic int 316145256Sjkoshyamd_write_pmc(int cpu, int ri, pmc_value_t v) 317145256Sjkoshy{ 318145256Sjkoshy const struct amd_descr *pd; 319145256Sjkoshy struct pmc *pm; 320145256Sjkoshy const struct pmc_hw *phw; 321145256Sjkoshy enum pmc_mode mode; 322145256Sjkoshy 323145256Sjkoshy KASSERT(cpu >= 0 && cpu < mp_ncpus, 324145256Sjkoshy ("[amd,%d] illegal CPU value %d", __LINE__, cpu)); 325145256Sjkoshy KASSERT(ri >= 0 && ri < AMD_NPMCS, 326145256Sjkoshy ("[amd,%d] illegal row-index %d", __LINE__, ri)); 327145256Sjkoshy 328145256Sjkoshy phw = pmc_pcpu[cpu]->pc_hwpmcs[ri]; 329145256Sjkoshy pd = &amd_pmcdesc[ri]; 330145256Sjkoshy pm = phw->phw_pmc; 331145256Sjkoshy 332145256Sjkoshy KASSERT(pm != NULL, 333145256Sjkoshy ("[amd,%d] PMC not owned (cpu%d,pmc%d)", __LINE__, 334145256Sjkoshy cpu, ri)); 335145256Sjkoshy 336145774Sjkoshy mode = PMC_TO_MODE(pm); 337145256Sjkoshy 338145256Sjkoshy if (pd->pm_descr.pd_class == PMC_CLASS_TSC) 339145256Sjkoshy return 0; 340145256Sjkoshy 341153110Sru#ifdef DEBUG 342147191Sjkoshy KASSERT(pd->pm_descr.pd_class == amd_pmc_class, 343145256Sjkoshy ("[amd,%d] unknown PMC class (%d)", __LINE__, 344145256Sjkoshy pd->pm_descr.pd_class)); 345147191Sjkoshy#endif 346145256Sjkoshy 347145256Sjkoshy /* use 2's complement of the count for sampling mode PMCs */ 348145256Sjkoshy if (PMC_IS_SAMPLING_MODE(mode)) 349147191Sjkoshy v = AMD_RELOAD_COUNT_TO_PERFCTR_VALUE(v); 350145256Sjkoshy 351145256Sjkoshy PMCDBG(MDP,WRI,1,"amd-write cpu=%d ri=%d v=%jx", cpu, ri, v); 352145256Sjkoshy 353145256Sjkoshy /* write the PMC value */ 354145256Sjkoshy wrmsr(pd->pm_perfctr, v); 355145256Sjkoshy return 0; 356145256Sjkoshy} 357145256Sjkoshy 358145256Sjkoshy/* 359145256Sjkoshy * configure hardware pmc according to the configuration recorded in 360145256Sjkoshy * pmc 'pm'. 361145256Sjkoshy */ 362145256Sjkoshy 363145256Sjkoshystatic int 364145256Sjkoshyamd_config_pmc(int cpu, int ri, struct pmc *pm) 365145256Sjkoshy{ 366145256Sjkoshy struct pmc_hw *phw; 367145256Sjkoshy 368145615Sjkoshy PMCDBG(MDP,CFG,1, "cpu=%d ri=%d pm=%p", cpu, ri, pm); 369145615Sjkoshy 370145256Sjkoshy KASSERT(cpu >= 0 && cpu < mp_ncpus, 371145256Sjkoshy ("[amd,%d] illegal CPU value %d", __LINE__, cpu)); 372145256Sjkoshy KASSERT(ri >= 0 && ri < AMD_NPMCS, 373145256Sjkoshy ("[amd,%d] illegal row-index %d", __LINE__, ri)); 374145256Sjkoshy 375145256Sjkoshy phw = pmc_pcpu[cpu]->pc_hwpmcs[ri]; 376145256Sjkoshy 377145256Sjkoshy KASSERT(pm == NULL || phw->phw_pmc == NULL, 378145615Sjkoshy ("[amd,%d] pm=%p phw->pm=%p hwpmc not unconfigured", 379145615Sjkoshy __LINE__, pm, phw->phw_pmc)); 380145256Sjkoshy 381145256Sjkoshy phw->phw_pmc = pm; 382145256Sjkoshy return 0; 383145256Sjkoshy} 384145256Sjkoshy 385145256Sjkoshy/* 386145774Sjkoshy * Retrieve a configured PMC pointer from hardware state. 387145774Sjkoshy */ 388145774Sjkoshy 389145774Sjkoshystatic int 390145774Sjkoshyamd_get_config(int cpu, int ri, struct pmc **ppm) 391145774Sjkoshy{ 392145774Sjkoshy *ppm = pmc_pcpu[cpu]->pc_hwpmcs[ri]->phw_pmc; 393145774Sjkoshy 394145774Sjkoshy return 0; 395145774Sjkoshy} 396145774Sjkoshy 397145774Sjkoshy/* 398145256Sjkoshy * Machine dependent actions taken during the context switch in of a 399145256Sjkoshy * thread. 400145256Sjkoshy */ 401145256Sjkoshy 402145256Sjkoshystatic int 403145615Sjkoshyamd_switch_in(struct pmc_cpu *pc, struct pmc_process *pp) 404145256Sjkoshy{ 405145256Sjkoshy (void) pc; 406145256Sjkoshy 407145615Sjkoshy PMCDBG(MDP,SWI,1, "pc=%p pp=%p enable-msr=%d", pc, pp, 408145774Sjkoshy (pp->pp_flags & PMC_PP_ENABLE_MSR_ACCESS) != 0); 409145615Sjkoshy 410145615Sjkoshy /* enable the RDPMC instruction if needed */ 411145774Sjkoshy if (pp->pp_flags & PMC_PP_ENABLE_MSR_ACCESS) 412145615Sjkoshy load_cr4(rcr4() | CR4_PCE); 413145615Sjkoshy 414145256Sjkoshy return 0; 415145256Sjkoshy} 416145256Sjkoshy 417145256Sjkoshy/* 418145256Sjkoshy * Machine dependent actions taken during the context switch out of a 419145256Sjkoshy * thread. 420145256Sjkoshy */ 421145256Sjkoshy 422145256Sjkoshystatic int 423145615Sjkoshyamd_switch_out(struct pmc_cpu *pc, struct pmc_process *pp) 424145256Sjkoshy{ 425145256Sjkoshy (void) pc; 426145615Sjkoshy (void) pp; /* can be NULL */ 427145256Sjkoshy 428145615Sjkoshy PMCDBG(MDP,SWO,1, "pc=%p pp=%p enable-msr=%d", pc, pp, pp ? 429145774Sjkoshy (pp->pp_flags & PMC_PP_ENABLE_MSR_ACCESS) == 1 : 0); 430145615Sjkoshy 431145615Sjkoshy /* always turn off the RDPMC instruction */ 432145256Sjkoshy load_cr4(rcr4() & ~CR4_PCE); 433145615Sjkoshy 434145256Sjkoshy return 0; 435145256Sjkoshy} 436145256Sjkoshy 437145256Sjkoshy/* 438145256Sjkoshy * Check if a given allocation is feasible. 439145256Sjkoshy */ 440145256Sjkoshy 441145256Sjkoshystatic int 442145256Sjkoshyamd_allocate_pmc(int cpu, int ri, struct pmc *pm, 443145256Sjkoshy const struct pmc_op_pmcallocate *a) 444145256Sjkoshy{ 445145256Sjkoshy int i; 446145256Sjkoshy uint32_t allowed_unitmask, caps, config, unitmask; 447145256Sjkoshy enum pmc_event pe; 448145256Sjkoshy const struct pmc_descr *pd; 449145256Sjkoshy 450145256Sjkoshy (void) cpu; 451145256Sjkoshy 452145256Sjkoshy KASSERT(cpu >= 0 && cpu < mp_ncpus, 453145256Sjkoshy ("[amd,%d] illegal CPU value %d", __LINE__, cpu)); 454145256Sjkoshy KASSERT(ri >= 0 && ri < AMD_NPMCS, 455145256Sjkoshy ("[amd,%d] illegal row index %d", __LINE__, ri)); 456145256Sjkoshy 457145256Sjkoshy pd = &amd_pmcdesc[ri].pm_descr; 458145256Sjkoshy 459145256Sjkoshy /* check class match */ 460145774Sjkoshy if (pd->pd_class != a->pm_class) 461145256Sjkoshy return EINVAL; 462145256Sjkoshy 463145256Sjkoshy caps = pm->pm_caps; 464145256Sjkoshy 465145256Sjkoshy PMCDBG(MDP,ALL,1,"amd-allocate ri=%d caps=0x%x", ri, caps); 466145256Sjkoshy 467145256Sjkoshy if ((pd->pd_caps & caps) != caps) 468145256Sjkoshy return EPERM; 469145256Sjkoshy if (pd->pd_class == PMC_CLASS_TSC) { 470145256Sjkoshy /* TSC's are always allocated in system-wide counting mode */ 471145256Sjkoshy if (a->pm_ev != PMC_EV_TSC_TSC || 472145256Sjkoshy a->pm_mode != PMC_MODE_SC) 473145256Sjkoshy return EINVAL; 474145256Sjkoshy return 0; 475145256Sjkoshy } 476145256Sjkoshy 477153110Sru#ifdef DEBUG 478147191Sjkoshy KASSERT(pd->pd_class == amd_pmc_class, 479145256Sjkoshy ("[amd,%d] Unknown PMC class (%d)", __LINE__, pd->pd_class)); 480147191Sjkoshy#endif 481145256Sjkoshy 482145256Sjkoshy pe = a->pm_ev; 483145256Sjkoshy 484145256Sjkoshy /* map ev to the correct event mask code */ 485145256Sjkoshy config = allowed_unitmask = 0; 486145256Sjkoshy for (i = 0; i < amd_event_codes_size; i++) 487145256Sjkoshy if (amd_event_codes[i].pe_ev == pe) { 488145256Sjkoshy config = 489145256Sjkoshy AMD_PMC_TO_EVENTMASK(amd_event_codes[i].pe_code); 490145256Sjkoshy allowed_unitmask = 491145256Sjkoshy AMD_PMC_TO_UNITMASK(amd_event_codes[i].pe_mask); 492145256Sjkoshy break; 493145256Sjkoshy } 494145256Sjkoshy if (i == amd_event_codes_size) 495145256Sjkoshy return EINVAL; 496145256Sjkoshy 497147191Sjkoshy unitmask = a->pm_md.pm_amd.pm_amd_config & AMD_PMC_UNITMASK; 498145256Sjkoshy if (unitmask & ~allowed_unitmask) /* disallow reserved bits */ 499145256Sjkoshy return EINVAL; 500145256Sjkoshy 501145256Sjkoshy if (unitmask && (caps & PMC_CAP_QUALIFIER)) 502145256Sjkoshy config |= unitmask; 503145256Sjkoshy 504145256Sjkoshy if (caps & PMC_CAP_THRESHOLD) 505147191Sjkoshy config |= a->pm_md.pm_amd.pm_amd_config & AMD_PMC_COUNTERMASK; 506145256Sjkoshy 507145256Sjkoshy /* set at least one of the 'usr' or 'os' caps */ 508145256Sjkoshy if (caps & PMC_CAP_USER) 509145256Sjkoshy config |= AMD_PMC_USR; 510145256Sjkoshy if (caps & PMC_CAP_SYSTEM) 511145256Sjkoshy config |= AMD_PMC_OS; 512145256Sjkoshy if ((caps & (PMC_CAP_USER|PMC_CAP_SYSTEM)) == 0) 513145256Sjkoshy config |= (AMD_PMC_USR|AMD_PMC_OS); 514145256Sjkoshy 515145256Sjkoshy if (caps & PMC_CAP_EDGE) 516145256Sjkoshy config |= AMD_PMC_EDGE; 517145256Sjkoshy if (caps & PMC_CAP_INVERT) 518145256Sjkoshy config |= AMD_PMC_INVERT; 519145256Sjkoshy if (caps & PMC_CAP_INTERRUPT) 520145256Sjkoshy config |= AMD_PMC_INT; 521145256Sjkoshy 522145256Sjkoshy pm->pm_md.pm_amd.pm_amd_evsel = config; /* save config value */ 523145256Sjkoshy 524145256Sjkoshy PMCDBG(MDP,ALL,2,"amd-allocate ri=%d -> config=0x%x", ri, config); 525145256Sjkoshy 526145256Sjkoshy return 0; 527145256Sjkoshy} 528145256Sjkoshy 529145256Sjkoshy/* 530145256Sjkoshy * Release machine dependent state associated with a PMC. This is a 531145256Sjkoshy * no-op on this architecture. 532145256Sjkoshy * 533145256Sjkoshy */ 534145256Sjkoshy 535145256Sjkoshy/* ARGSUSED0 */ 536145256Sjkoshystatic int 537145256Sjkoshyamd_release_pmc(int cpu, int ri, struct pmc *pmc) 538145256Sjkoshy{ 539153110Sru#ifdef DEBUG 540145256Sjkoshy const struct amd_descr *pd; 541145256Sjkoshy#endif 542145256Sjkoshy struct pmc_hw *phw; 543145256Sjkoshy 544145256Sjkoshy (void) pmc; 545145256Sjkoshy 546145256Sjkoshy KASSERT(cpu >= 0 && cpu < mp_ncpus, 547145256Sjkoshy ("[amd,%d] illegal CPU value %d", __LINE__, cpu)); 548145256Sjkoshy KASSERT(ri >= 0 && ri < AMD_NPMCS, 549145256Sjkoshy ("[amd,%d] illegal row-index %d", __LINE__, ri)); 550145256Sjkoshy 551145256Sjkoshy phw = pmc_pcpu[cpu]->pc_hwpmcs[ri]; 552145256Sjkoshy 553145256Sjkoshy KASSERT(phw->phw_pmc == NULL, 554145256Sjkoshy ("[amd,%d] PHW pmc %p non-NULL", __LINE__, phw->phw_pmc)); 555145256Sjkoshy 556153110Sru#ifdef DEBUG 557145256Sjkoshy pd = &amd_pmcdesc[ri]; 558147191Sjkoshy if (pd->pm_descr.pd_class == amd_pmc_class) 559145256Sjkoshy KASSERT(AMD_PMC_IS_STOPPED(pd->pm_evsel), 560145256Sjkoshy ("[amd,%d] PMC %d released while active", __LINE__, ri)); 561145256Sjkoshy#endif 562145256Sjkoshy 563145256Sjkoshy return 0; 564145256Sjkoshy} 565145256Sjkoshy 566145256Sjkoshy/* 567145256Sjkoshy * start a PMC. 568145256Sjkoshy */ 569145256Sjkoshy 570145256Sjkoshystatic int 571145256Sjkoshyamd_start_pmc(int cpu, int ri) 572145256Sjkoshy{ 573145256Sjkoshy uint32_t config; 574145256Sjkoshy struct pmc *pm; 575145256Sjkoshy struct pmc_hw *phw; 576145256Sjkoshy const struct amd_descr *pd; 577145256Sjkoshy 578145256Sjkoshy KASSERT(cpu >= 0 && cpu < mp_ncpus, 579145256Sjkoshy ("[amd,%d] illegal CPU value %d", __LINE__, cpu)); 580145256Sjkoshy KASSERT(ri >= 0 && ri < AMD_NPMCS, 581145256Sjkoshy ("[amd,%d] illegal row-index %d", __LINE__, ri)); 582145256Sjkoshy 583145256Sjkoshy phw = pmc_pcpu[cpu]->pc_hwpmcs[ri]; 584145256Sjkoshy pm = phw->phw_pmc; 585145256Sjkoshy pd = &amd_pmcdesc[ri]; 586145256Sjkoshy 587145256Sjkoshy KASSERT(pm != NULL, 588145256Sjkoshy ("[amd,%d] starting cpu%d,pmc%d with null pmc record", __LINE__, 589145256Sjkoshy cpu, ri)); 590145256Sjkoshy 591145256Sjkoshy PMCDBG(MDP,STA,1,"amd-start cpu=%d ri=%d", cpu, ri); 592145256Sjkoshy 593145256Sjkoshy if (pd->pm_descr.pd_class == PMC_CLASS_TSC) 594145256Sjkoshy return 0; /* TSCs are always running */ 595145256Sjkoshy 596153110Sru#ifdef DEBUG 597147191Sjkoshy KASSERT(pd->pm_descr.pd_class == amd_pmc_class, 598145256Sjkoshy ("[amd,%d] unknown PMC class (%d)", __LINE__, 599145256Sjkoshy pd->pm_descr.pd_class)); 600147191Sjkoshy#endif 601145256Sjkoshy 602145256Sjkoshy KASSERT(AMD_PMC_IS_STOPPED(pd->pm_evsel), 603145256Sjkoshy ("[amd,%d] pmc%d,cpu%d: Starting active PMC \"%s\"", __LINE__, 604145256Sjkoshy ri, cpu, pd->pm_descr.pd_name)); 605145256Sjkoshy 606145256Sjkoshy /* turn on the PMC ENABLE bit */ 607145256Sjkoshy config = pm->pm_md.pm_amd.pm_amd_evsel | AMD_PMC_ENABLE; 608145256Sjkoshy 609145256Sjkoshy PMCDBG(MDP,STA,2,"amd-start config=0x%x", config); 610145256Sjkoshy 611145256Sjkoshy wrmsr(pd->pm_evsel, config); 612145256Sjkoshy return 0; 613145256Sjkoshy} 614145256Sjkoshy 615145256Sjkoshy/* 616145256Sjkoshy * Stop a PMC. 617145256Sjkoshy */ 618145256Sjkoshy 619145256Sjkoshystatic int 620145256Sjkoshyamd_stop_pmc(int cpu, int ri) 621145256Sjkoshy{ 622145256Sjkoshy struct pmc *pm; 623145256Sjkoshy struct pmc_hw *phw; 624145256Sjkoshy const struct amd_descr *pd; 625145256Sjkoshy uint64_t config; 626145256Sjkoshy 627145256Sjkoshy KASSERT(cpu >= 0 && cpu < mp_ncpus, 628145256Sjkoshy ("[amd,%d] illegal CPU value %d", __LINE__, cpu)); 629145256Sjkoshy KASSERT(ri >= 0 && ri < AMD_NPMCS, 630145256Sjkoshy ("[amd,%d] illegal row-index %d", __LINE__, ri)); 631145256Sjkoshy 632145256Sjkoshy phw = pmc_pcpu[cpu]->pc_hwpmcs[ri]; 633145256Sjkoshy pm = phw->phw_pmc; 634145256Sjkoshy pd = &amd_pmcdesc[ri]; 635145256Sjkoshy 636145256Sjkoshy KASSERT(pm != NULL, 637145256Sjkoshy ("[amd,%d] cpu%d,pmc%d no PMC to stop", __LINE__, 638145256Sjkoshy cpu, ri)); 639145256Sjkoshy 640145256Sjkoshy /* can't stop a TSC */ 641145256Sjkoshy if (pd->pm_descr.pd_class == PMC_CLASS_TSC) 642145256Sjkoshy return 0; 643145256Sjkoshy 644153110Sru#ifdef DEBUG 645147191Sjkoshy KASSERT(pd->pm_descr.pd_class == amd_pmc_class, 646145256Sjkoshy ("[amd,%d] unknown PMC class (%d)", __LINE__, 647145256Sjkoshy pd->pm_descr.pd_class)); 648147191Sjkoshy#endif 649145256Sjkoshy 650145256Sjkoshy KASSERT(!AMD_PMC_IS_STOPPED(pd->pm_evsel), 651145256Sjkoshy ("[amd,%d] PMC%d, CPU%d \"%s\" already stopped", 652145256Sjkoshy __LINE__, ri, cpu, pd->pm_descr.pd_name)); 653145256Sjkoshy 654145256Sjkoshy PMCDBG(MDP,STO,1,"amd-stop ri=%d", ri); 655145256Sjkoshy 656145256Sjkoshy /* turn off the PMC ENABLE bit */ 657145256Sjkoshy config = pm->pm_md.pm_amd.pm_amd_evsel & ~AMD_PMC_ENABLE; 658145256Sjkoshy wrmsr(pd->pm_evsel, config); 659145256Sjkoshy return 0; 660145256Sjkoshy} 661145256Sjkoshy 662145256Sjkoshy/* 663145256Sjkoshy * Interrupt handler. This function needs to return '1' if the 664145256Sjkoshy * interrupt was this CPU's PMCs or '0' otherwise. It is not allowed 665145256Sjkoshy * to sleep or do anything a 'fast' interrupt handler is not allowed 666145256Sjkoshy * to do. 667145256Sjkoshy */ 668145256Sjkoshy 669145256Sjkoshystatic int 670146799Sjkoshyamd_intr(int cpu, uintptr_t eip, int usermode) 671145256Sjkoshy{ 672147191Sjkoshy int i, error, retval, ri; 673147191Sjkoshy uint32_t config, evsel, perfctr; 674145256Sjkoshy struct pmc *pm; 675145256Sjkoshy struct pmc_cpu *pc; 676145256Sjkoshy struct pmc_hw *phw; 677147191Sjkoshy pmc_value_t v; 678145256Sjkoshy 679145256Sjkoshy KASSERT(cpu >= 0 && cpu < mp_ncpus, 680145256Sjkoshy ("[amd,%d] out of range CPU %d", __LINE__, cpu)); 681145256Sjkoshy 682147867Sjkoshy PMCDBG(MDP,INT,1, "cpu=%d eip=%p um=%d", cpu, (void *) eip, 683147867Sjkoshy usermode); 684147191Sjkoshy 685145256Sjkoshy retval = 0; 686145256Sjkoshy 687145256Sjkoshy pc = pmc_pcpu[cpu]; 688145256Sjkoshy 689145256Sjkoshy /* 690145256Sjkoshy * look for all PMCs that have interrupted: 691145256Sjkoshy * - skip over the TSC [PMC#0] 692147191Sjkoshy * - look for a running, sampling PMC which has overflowed 693147191Sjkoshy * and which has a valid 'struct pmc' association 694147191Sjkoshy * 695147191Sjkoshy * If found, we call a helper to process the interrupt. 696150050Sjkoshy * 697150050Sjkoshy * If multiple PMCs interrupt at the same time, the AMD64 698150050Sjkoshy * processor appears to deliver as many NMIs as there are 699150050Sjkoshy * outstanding PMC interrupts. Thus we need to only process 700150050Sjkoshy * one interrupt at a time. 701145256Sjkoshy */ 702145256Sjkoshy 703150050Sjkoshy for (i = 0; retval == 0 && i < AMD_NPMCS-1; i++) { 704145256Sjkoshy 705147191Sjkoshy ri = i + 1; /* row index; TSC is at ri == 0 */ 706147191Sjkoshy 707147191Sjkoshy if (!AMD_PMC_HAS_OVERFLOWED(i)) 708147191Sjkoshy continue; 709147191Sjkoshy 710147191Sjkoshy phw = pc->pc_hwpmcs[ri]; 711147191Sjkoshy 712145256Sjkoshy KASSERT(phw != NULL, ("[amd,%d] null PHW pointer", __LINE__)); 713145256Sjkoshy 714145256Sjkoshy if ((pm = phw->phw_pmc) == NULL || 715147191Sjkoshy pm->pm_state != PMC_STATE_RUNNING || 716147191Sjkoshy !PMC_IS_SAMPLING_MODE(PMC_TO_MODE(pm))) { 717145256Sjkoshy continue; 718145256Sjkoshy } 719145256Sjkoshy 720150050Sjkoshy retval = 1; /* found an interrupting PMC */ 721150050Sjkoshy 722147191Sjkoshy /* stop the PMC, reload count */ 723147191Sjkoshy evsel = AMD_PMC_EVSEL_0 + i; 724147191Sjkoshy perfctr = AMD_PMC_PERFCTR_0 + i; 725147191Sjkoshy v = pm->pm_sc.pm_reloadcount; 726147191Sjkoshy config = rdmsr(evsel); 727147191Sjkoshy 728147191Sjkoshy KASSERT((config & ~AMD_PMC_ENABLE) == 729147191Sjkoshy (pm->pm_md.pm_amd.pm_amd_evsel & ~AMD_PMC_ENABLE), 730147191Sjkoshy ("[amd,%d] config mismatch reg=0x%x pm=0x%x", __LINE__, 731147191Sjkoshy config, pm->pm_md.pm_amd.pm_amd_evsel)); 732147191Sjkoshy 733147191Sjkoshy wrmsr(evsel, config & ~AMD_PMC_ENABLE); 734147191Sjkoshy wrmsr(perfctr, AMD_RELOAD_COUNT_TO_PERFCTR_VALUE(v)); 735147191Sjkoshy 736150050Sjkoshy /* restart the counter if there was no error during logging */ 737147191Sjkoshy error = pmc_process_interrupt(cpu, pm, eip, usermode); 738147191Sjkoshy if (error == 0) 739147191Sjkoshy wrmsr(evsel, config | AMD_PMC_ENABLE); 740145256Sjkoshy } 741147191Sjkoshy 742147867Sjkoshy atomic_add_int(retval ? &pmc_stats.pm_intr_processed : 743147867Sjkoshy &pmc_stats.pm_intr_ignored, 1); 744147867Sjkoshy 745145256Sjkoshy return retval; 746145256Sjkoshy} 747145256Sjkoshy 748145256Sjkoshy/* 749145256Sjkoshy * describe a PMC 750145256Sjkoshy */ 751145256Sjkoshystatic int 752145256Sjkoshyamd_describe(int cpu, int ri, struct pmc_info *pi, struct pmc **ppmc) 753145256Sjkoshy{ 754145256Sjkoshy int error; 755145256Sjkoshy size_t copied; 756145256Sjkoshy const struct amd_descr *pd; 757145256Sjkoshy struct pmc_hw *phw; 758145256Sjkoshy 759145256Sjkoshy KASSERT(cpu >= 0 && cpu < mp_ncpus, 760145256Sjkoshy ("[amd,%d] illegal CPU %d", __LINE__, cpu)); 761145256Sjkoshy KASSERT(ri >= 0 && ri < AMD_NPMCS, 762145256Sjkoshy ("[amd,%d] row-index %d out of range", __LINE__, ri)); 763145256Sjkoshy 764145256Sjkoshy phw = pmc_pcpu[cpu]->pc_hwpmcs[ri]; 765145256Sjkoshy pd = &amd_pmcdesc[ri]; 766145256Sjkoshy 767145256Sjkoshy if ((error = copystr(pd->pm_descr.pd_name, pi->pm_name, 768145256Sjkoshy PMC_NAME_MAX, &copied)) != 0) 769145256Sjkoshy return error; 770145256Sjkoshy 771145256Sjkoshy pi->pm_class = pd->pm_descr.pd_class; 772145256Sjkoshy 773145256Sjkoshy if (phw->phw_state & PMC_PHW_FLAG_IS_ENABLED) { 774145256Sjkoshy pi->pm_enabled = TRUE; 775145256Sjkoshy *ppmc = phw->phw_pmc; 776145256Sjkoshy } else { 777145256Sjkoshy pi->pm_enabled = FALSE; 778145256Sjkoshy *ppmc = NULL; 779145256Sjkoshy } 780145256Sjkoshy 781145256Sjkoshy return 0; 782145256Sjkoshy} 783145256Sjkoshy 784145256Sjkoshy/* 785145256Sjkoshy * i386 specific entry points 786145256Sjkoshy */ 787145256Sjkoshy 788145256Sjkoshy/* 789145256Sjkoshy * return the MSR address of the given PMC. 790145256Sjkoshy */ 791145256Sjkoshy 792145256Sjkoshystatic int 793145256Sjkoshyamd_get_msr(int ri, uint32_t *msr) 794145256Sjkoshy{ 795145256Sjkoshy KASSERT(ri >= 0 && ri < AMD_NPMCS, 796145256Sjkoshy ("[amd,%d] ri %d out of range", __LINE__, ri)); 797145256Sjkoshy 798145615Sjkoshy *msr = amd_pmcdesc[ri].pm_perfctr - AMD_PMC_PERFCTR_0; 799145256Sjkoshy return 0; 800145256Sjkoshy} 801145256Sjkoshy 802145256Sjkoshy/* 803145256Sjkoshy * processor dependent initialization. 804145256Sjkoshy */ 805145256Sjkoshy 806145256Sjkoshy/* 807145256Sjkoshy * Per-processor data structure 808145256Sjkoshy * 809145256Sjkoshy * [common stuff] 810145256Sjkoshy * [5 struct pmc_hw pointers] 811145256Sjkoshy * [5 struct pmc_hw structures] 812145256Sjkoshy */ 813145256Sjkoshy 814145256Sjkoshystruct amd_cpu { 815145256Sjkoshy struct pmc_cpu pc_common; 816145256Sjkoshy struct pmc_hw *pc_hwpmcs[AMD_NPMCS]; 817145256Sjkoshy struct pmc_hw pc_amdpmcs[AMD_NPMCS]; 818145256Sjkoshy}; 819145256Sjkoshy 820145256Sjkoshy 821145256Sjkoshystatic int 822145256Sjkoshyamd_init(int cpu) 823145256Sjkoshy{ 824145256Sjkoshy int n; 825145256Sjkoshy struct amd_cpu *pcs; 826145256Sjkoshy struct pmc_hw *phw; 827145256Sjkoshy 828145256Sjkoshy KASSERT(cpu >= 0 && cpu < mp_ncpus, 829145256Sjkoshy ("[amd,%d] insane cpu number %d", __LINE__, cpu)); 830145256Sjkoshy 831145256Sjkoshy PMCDBG(MDP,INI,1,"amd-init cpu=%d", cpu); 832145256Sjkoshy 833145256Sjkoshy MALLOC(pcs, struct amd_cpu *, sizeof(struct amd_cpu), M_PMC, 834145256Sjkoshy M_WAITOK|M_ZERO); 835145256Sjkoshy 836145256Sjkoshy phw = &pcs->pc_amdpmcs[0]; 837145256Sjkoshy 838145256Sjkoshy /* 839145256Sjkoshy * Initialize the per-cpu mutex and set the content of the 840145256Sjkoshy * hardware descriptors to a known state. 841145256Sjkoshy */ 842145256Sjkoshy 843145256Sjkoshy for (n = 0; n < AMD_NPMCS; n++, phw++) { 844145256Sjkoshy phw->phw_state = PMC_PHW_FLAG_IS_ENABLED | 845145256Sjkoshy PMC_PHW_CPU_TO_STATE(cpu) | PMC_PHW_INDEX_TO_STATE(n); 846145256Sjkoshy phw->phw_pmc = NULL; 847145256Sjkoshy pcs->pc_hwpmcs[n] = phw; 848145256Sjkoshy } 849145256Sjkoshy 850145256Sjkoshy /* Mark the TSC as shareable */ 851145256Sjkoshy pcs->pc_hwpmcs[0]->phw_state |= PMC_PHW_FLAG_IS_SHAREABLE; 852145256Sjkoshy 853145256Sjkoshy pmc_pcpu[cpu] = (struct pmc_cpu *) pcs; 854145256Sjkoshy 855145256Sjkoshy return 0; 856145256Sjkoshy} 857145256Sjkoshy 858145256Sjkoshy 859145256Sjkoshy/* 860145256Sjkoshy * processor dependent cleanup prior to the KLD 861145256Sjkoshy * being unloaded 862145256Sjkoshy */ 863145256Sjkoshy 864145256Sjkoshystatic int 865145256Sjkoshyamd_cleanup(int cpu) 866145256Sjkoshy{ 867145256Sjkoshy int i; 868145256Sjkoshy uint32_t evsel; 869145256Sjkoshy struct pmc_cpu *pcs; 870145256Sjkoshy 871145256Sjkoshy KASSERT(cpu >= 0 && cpu < mp_ncpus, 872145256Sjkoshy ("[amd,%d] insane cpu number (%d)", __LINE__, cpu)); 873145256Sjkoshy 874145256Sjkoshy PMCDBG(MDP,INI,1,"amd-cleanup cpu=%d", cpu); 875145256Sjkoshy 876145256Sjkoshy /* 877145256Sjkoshy * First, turn off all PMCs on this CPU. 878145256Sjkoshy */ 879145256Sjkoshy 880145256Sjkoshy for (i = 0; i < 4; i++) { /* XXX this loop is now not needed */ 881145256Sjkoshy evsel = rdmsr(AMD_PMC_EVSEL_0 + i); 882145256Sjkoshy evsel &= ~AMD_PMC_ENABLE; 883145256Sjkoshy wrmsr(AMD_PMC_EVSEL_0 + i, evsel); 884145256Sjkoshy } 885145256Sjkoshy 886145256Sjkoshy /* 887145256Sjkoshy * Next, free up allocated space. 888145256Sjkoshy */ 889145256Sjkoshy 890147191Sjkoshy if ((pcs = pmc_pcpu[cpu]) == NULL) 891147191Sjkoshy return 0; 892145256Sjkoshy 893153110Sru#ifdef DEBUG 894145256Sjkoshy /* check the TSC */ 895145256Sjkoshy KASSERT(pcs->pc_hwpmcs[0]->phw_pmc == NULL, 896145256Sjkoshy ("[amd,%d] CPU%d,PMC0 still in use", __LINE__, cpu)); 897145256Sjkoshy for (i = 1; i < AMD_NPMCS; i++) { 898145256Sjkoshy KASSERT(pcs->pc_hwpmcs[i]->phw_pmc == NULL, 899145256Sjkoshy ("[amd,%d] CPU%d/PMC%d in use", __LINE__, cpu, i)); 900145256Sjkoshy KASSERT(AMD_PMC_IS_STOPPED(AMD_PMC_EVSEL_0 + (i-1)), 901145256Sjkoshy ("[amd,%d] CPU%d/PMC%d not stopped", __LINE__, cpu, i)); 902145256Sjkoshy } 903145256Sjkoshy#endif 904145256Sjkoshy 905145256Sjkoshy pmc_pcpu[cpu] = NULL; 906145256Sjkoshy FREE(pcs, M_PMC); 907145256Sjkoshy return 0; 908145256Sjkoshy} 909145256Sjkoshy 910145256Sjkoshy/* 911145256Sjkoshy * Initialize ourselves. 912145256Sjkoshy */ 913145256Sjkoshy 914145256Sjkoshystruct pmc_mdep * 915145256Sjkoshypmc_amd_initialize(void) 916145256Sjkoshy{ 917147191Sjkoshy enum pmc_cputype cputype; 918147191Sjkoshy enum pmc_class class; 919145256Sjkoshy struct pmc_mdep *pmc_mdep; 920147191Sjkoshy char *name; 921147191Sjkoshy int i; 922145256Sjkoshy 923147191Sjkoshy /* 924147191Sjkoshy * The presence of hardware performance counters on the AMD 925147191Sjkoshy * Athlon, Duron or later processors, is _not_ indicated by 926147191Sjkoshy * any of the processor feature flags set by the 'CPUID' 927147191Sjkoshy * instruction, so we only check the 'instruction family' 928147191Sjkoshy * field returned by CPUID for instruction family >= 6. 929147191Sjkoshy */ 930145256Sjkoshy 931147510Sjkoshy class = cputype = -1; 932147510Sjkoshy name = NULL; 933147191Sjkoshy switch (cpu_id & 0xF00) { 934147191Sjkoshy case 0x600: /* Athlon(tm) processor */ 935147191Sjkoshy cputype = PMC_CPU_AMD_K7; 936147191Sjkoshy class = PMC_CLASS_K7; 937147191Sjkoshy name = "K7"; 938147191Sjkoshy break; 939147191Sjkoshy case 0xF00: /* Athlon64/Opteron processor */ 940147191Sjkoshy cputype = PMC_CPU_AMD_K8; 941147191Sjkoshy class = PMC_CLASS_K8; 942147191Sjkoshy name = "K8"; 943147191Sjkoshy break; 944147191Sjkoshy } 945147191Sjkoshy 946147191Sjkoshy if ((int) cputype == -1) { 947147191Sjkoshy (void) printf("pmc: Unknown AMD CPU.\n"); 948145256Sjkoshy return NULL; 949147191Sjkoshy } 950145256Sjkoshy 951153110Sru#ifdef DEBUG 952147191Sjkoshy amd_pmc_class = class; 953147191Sjkoshy#endif 954147191Sjkoshy 955145256Sjkoshy MALLOC(pmc_mdep, struct pmc_mdep *, sizeof(struct pmc_mdep), 956145256Sjkoshy M_PMC, M_WAITOK|M_ZERO); 957145256Sjkoshy 958147191Sjkoshy pmc_mdep->pmd_cputype = cputype; 959145256Sjkoshy pmc_mdep->pmd_npmc = AMD_NPMCS; 960145256Sjkoshy 961145256Sjkoshy /* this processor has two classes of usable PMCs */ 962145256Sjkoshy pmc_mdep->pmd_nclass = 2; 963145774Sjkoshy 964145774Sjkoshy /* TSC */ 965145774Sjkoshy pmc_mdep->pmd_classes[0].pm_class = PMC_CLASS_TSC; 966145774Sjkoshy pmc_mdep->pmd_classes[0].pm_caps = PMC_CAP_READ; 967145774Sjkoshy pmc_mdep->pmd_classes[0].pm_width = 64; 968145774Sjkoshy 969145774Sjkoshy /* AMD K7/K8 PMCs */ 970147191Sjkoshy pmc_mdep->pmd_classes[1].pm_class = class; 971145774Sjkoshy pmc_mdep->pmd_classes[1].pm_caps = AMD_PMC_CAPS; 972145774Sjkoshy pmc_mdep->pmd_classes[1].pm_width = 48; 973145774Sjkoshy 974145256Sjkoshy pmc_mdep->pmd_nclasspmcs[0] = 1; 975145256Sjkoshy pmc_mdep->pmd_nclasspmcs[1] = (AMD_NPMCS-1); 976145256Sjkoshy 977147191Sjkoshy /* fill in the correct pmc name and class */ 978147191Sjkoshy for (i = 1; i < AMD_NPMCS; i++) { 979147191Sjkoshy (void) snprintf(amd_pmcdesc[i].pm_descr.pd_name, 980147191Sjkoshy sizeof(amd_pmcdesc[i].pm_descr.pd_name), "%s-%d", 981147191Sjkoshy name, i-1); 982147191Sjkoshy amd_pmcdesc[i].pm_descr.pd_class = class; 983147191Sjkoshy } 984147191Sjkoshy 985145256Sjkoshy pmc_mdep->pmd_init = amd_init; 986145256Sjkoshy pmc_mdep->pmd_cleanup = amd_cleanup; 987145256Sjkoshy pmc_mdep->pmd_switch_in = amd_switch_in; 988145256Sjkoshy pmc_mdep->pmd_switch_out = amd_switch_out; 989145256Sjkoshy pmc_mdep->pmd_read_pmc = amd_read_pmc; 990145256Sjkoshy pmc_mdep->pmd_write_pmc = amd_write_pmc; 991145256Sjkoshy pmc_mdep->pmd_config_pmc = amd_config_pmc; 992145774Sjkoshy pmc_mdep->pmd_get_config = amd_get_config; 993145256Sjkoshy pmc_mdep->pmd_allocate_pmc = amd_allocate_pmc; 994145256Sjkoshy pmc_mdep->pmd_release_pmc = amd_release_pmc; 995145256Sjkoshy pmc_mdep->pmd_start_pmc = amd_start_pmc; 996145256Sjkoshy pmc_mdep->pmd_stop_pmc = amd_stop_pmc; 997145256Sjkoshy pmc_mdep->pmd_intr = amd_intr; 998145256Sjkoshy pmc_mdep->pmd_describe = amd_describe; 999145256Sjkoshy pmc_mdep->pmd_get_msr = amd_get_msr; /* i386 */ 1000145256Sjkoshy 1001145256Sjkoshy PMCDBG(MDP,INI,0,"%s","amd-initialize"); 1002145256Sjkoshy 1003145256Sjkoshy return pmc_mdep; 1004145256Sjkoshy} 1005