hwpmc_amd.c revision 177344
1145256Sjkoshy/*- 2174395Sjkoshy * Copyright (c) 2003-2007 Joseph Koshy 3174395Sjkoshy * Copyright (c) 2007 The FreeBSD Foundation 4145256Sjkoshy * All rights reserved. 5145256Sjkoshy * 6174395Sjkoshy * Portions of this software were developed by A. Joseph Koshy under 7174395Sjkoshy * sponsorship from the FreeBSD Foundation and Google, Inc. 8174395Sjkoshy * 9145256Sjkoshy * Redistribution and use in source and binary forms, with or without 10145256Sjkoshy * modification, are permitted provided that the following conditions 11145256Sjkoshy * are met: 12145256Sjkoshy * 1. Redistributions of source code must retain the above copyright 13145256Sjkoshy * notice, this list of conditions and the following disclaimer. 14145256Sjkoshy * 2. Redistributions in binary form must reproduce the above copyright 15145256Sjkoshy * notice, this list of conditions and the following disclaimer in the 16145256Sjkoshy * documentation and/or other materials provided with the distribution. 17145256Sjkoshy * 18145256Sjkoshy * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 19145256Sjkoshy * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 20145256Sjkoshy * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 21145256Sjkoshy * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 22145256Sjkoshy * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 23145256Sjkoshy * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 24145256Sjkoshy * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 25145256Sjkoshy * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 26145256Sjkoshy * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 27145256Sjkoshy * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 28145256Sjkoshy * SUCH DAMAGE. 29145256Sjkoshy * 30145256Sjkoshy */ 31145256Sjkoshy 32145256Sjkoshy#include <sys/cdefs.h> 33145256Sjkoshy__FBSDID("$FreeBSD: head/sys/dev/hwpmc/hwpmc_amd.c 177344 2008-03-18 08:39:11Z adrian $"); 34145256Sjkoshy 35145256Sjkoshy/* Support for the AMD K7 and later processors */ 36145256Sjkoshy 37145256Sjkoshy#include <sys/param.h> 38145256Sjkoshy#include <sys/lock.h> 39145256Sjkoshy#include <sys/malloc.h> 40145256Sjkoshy#include <sys/mutex.h> 41145338Smarcel#include <sys/pmc.h> 42145256Sjkoshy#include <sys/smp.h> 43145256Sjkoshy#include <sys/systm.h> 44145256Sjkoshy 45174395Sjkoshy#include <machine/cpu.h> 46147191Sjkoshy#include <machine/cpufunc.h> 47145256Sjkoshy#include <machine/md_var.h> 48147191Sjkoshy#include <machine/specialreg.h> 49145256Sjkoshy 50153110Sru#ifdef DEBUG 51147191Sjkoshyenum pmc_class amd_pmc_class; 52145256Sjkoshy#endif 53145256Sjkoshy 54145256Sjkoshy/* AMD K7 & K8 PMCs */ 55145256Sjkoshystruct amd_descr { 56145256Sjkoshy struct pmc_descr pm_descr; /* "base class" */ 57145256Sjkoshy uint32_t pm_evsel; /* address of EVSEL register */ 58145256Sjkoshy uint32_t pm_perfctr; /* address of PERFCTR register */ 59145256Sjkoshy}; 60145256Sjkoshy 61147191Sjkoshystatic struct amd_descr amd_pmcdesc[AMD_NPMCS] = 62145256Sjkoshy{ 63145256Sjkoshy { 64145256Sjkoshy .pm_descr = 65145256Sjkoshy { 66145256Sjkoshy .pd_name = "TSC", 67145256Sjkoshy .pd_class = PMC_CLASS_TSC, 68145256Sjkoshy .pd_caps = PMC_CAP_READ, 69145256Sjkoshy .pd_width = 64 70145256Sjkoshy }, 71145256Sjkoshy .pm_evsel = MSR_TSC, 72145256Sjkoshy .pm_perfctr = 0 /* unused */ 73145256Sjkoshy }, 74145256Sjkoshy 75145256Sjkoshy { 76145256Sjkoshy .pm_descr = 77145256Sjkoshy { 78147191Sjkoshy .pd_name = "", 79147191Sjkoshy .pd_class = -1, 80145256Sjkoshy .pd_caps = AMD_PMC_CAPS, 81145256Sjkoshy .pd_width = 48 82145256Sjkoshy }, 83145256Sjkoshy .pm_evsel = AMD_PMC_EVSEL_0, 84145256Sjkoshy .pm_perfctr = AMD_PMC_PERFCTR_0 85145256Sjkoshy }, 86145256Sjkoshy { 87145256Sjkoshy .pm_descr = 88145256Sjkoshy { 89147191Sjkoshy .pd_name = "", 90147191Sjkoshy .pd_class = -1, 91145256Sjkoshy .pd_caps = AMD_PMC_CAPS, 92145256Sjkoshy .pd_width = 48 93145256Sjkoshy }, 94145256Sjkoshy .pm_evsel = AMD_PMC_EVSEL_1, 95145256Sjkoshy .pm_perfctr = AMD_PMC_PERFCTR_1 96145256Sjkoshy }, 97145256Sjkoshy { 98145256Sjkoshy .pm_descr = 99145256Sjkoshy { 100147191Sjkoshy .pd_name = "", 101147191Sjkoshy .pd_class = -1, 102145256Sjkoshy .pd_caps = AMD_PMC_CAPS, 103145256Sjkoshy .pd_width = 48 104145256Sjkoshy }, 105145256Sjkoshy .pm_evsel = AMD_PMC_EVSEL_2, 106145256Sjkoshy .pm_perfctr = AMD_PMC_PERFCTR_2 107145256Sjkoshy }, 108145256Sjkoshy { 109145256Sjkoshy .pm_descr = 110145256Sjkoshy { 111147191Sjkoshy .pd_name = "", 112147191Sjkoshy .pd_class = -1, 113145256Sjkoshy .pd_caps = AMD_PMC_CAPS, 114145256Sjkoshy .pd_width = 48 115145256Sjkoshy }, 116145256Sjkoshy .pm_evsel = AMD_PMC_EVSEL_3, 117145256Sjkoshy .pm_perfctr = AMD_PMC_PERFCTR_3 118145256Sjkoshy } 119145256Sjkoshy}; 120145256Sjkoshy 121145256Sjkoshystruct amd_event_code_map { 122145256Sjkoshy enum pmc_event pe_ev; /* enum value */ 123145256Sjkoshy uint8_t pe_code; /* encoded event mask */ 124145256Sjkoshy uint8_t pe_mask; /* bits allowed in unit mask */ 125145256Sjkoshy}; 126145256Sjkoshy 127145256Sjkoshyconst struct amd_event_code_map amd_event_codes[] = { 128147191Sjkoshy#if defined(__i386__) /* 32 bit Athlon (K7) only */ 129145256Sjkoshy { PMC_EV_K7_DC_ACCESSES, 0x40, 0 }, 130145256Sjkoshy { PMC_EV_K7_DC_MISSES, 0x41, 0 }, 131147191Sjkoshy { PMC_EV_K7_DC_REFILLS_FROM_L2, 0x42, AMD_PMC_UNITMASK_MOESI }, 132147191Sjkoshy { PMC_EV_K7_DC_REFILLS_FROM_SYSTEM, 0x43, AMD_PMC_UNITMASK_MOESI }, 133147191Sjkoshy { PMC_EV_K7_DC_WRITEBACKS, 0x44, AMD_PMC_UNITMASK_MOESI }, 134145256Sjkoshy { PMC_EV_K7_L1_DTLB_MISS_AND_L2_DTLB_HITS, 0x45, 0 }, 135145256Sjkoshy { PMC_EV_K7_L1_AND_L2_DTLB_MISSES, 0x46, 0 }, 136145256Sjkoshy { PMC_EV_K7_MISALIGNED_REFERENCES, 0x47, 0 }, 137145256Sjkoshy 138145256Sjkoshy { PMC_EV_K7_IC_FETCHES, 0x80, 0 }, 139145256Sjkoshy { PMC_EV_K7_IC_MISSES, 0x81, 0 }, 140145256Sjkoshy 141145256Sjkoshy { PMC_EV_K7_L1_ITLB_MISSES, 0x84, 0 }, 142145256Sjkoshy { PMC_EV_K7_L1_L2_ITLB_MISSES, 0x85, 0 }, 143145256Sjkoshy 144145256Sjkoshy { PMC_EV_K7_RETIRED_INSTRUCTIONS, 0xC0, 0 }, 145145256Sjkoshy { PMC_EV_K7_RETIRED_OPS, 0xC1, 0 }, 146145256Sjkoshy { PMC_EV_K7_RETIRED_BRANCHES, 0xC2, 0 }, 147145256Sjkoshy { PMC_EV_K7_RETIRED_BRANCHES_MISPREDICTED, 0xC3, 0 }, 148145256Sjkoshy { PMC_EV_K7_RETIRED_TAKEN_BRANCHES, 0xC4, 0 }, 149145256Sjkoshy { PMC_EV_K7_RETIRED_TAKEN_BRANCHES_MISPREDICTED, 0xC5, 0 }, 150145256Sjkoshy { PMC_EV_K7_RETIRED_FAR_CONTROL_TRANSFERS, 0xC6, 0 }, 151145256Sjkoshy { PMC_EV_K7_RETIRED_RESYNC_BRANCHES, 0xC7, 0 }, 152145256Sjkoshy { PMC_EV_K7_INTERRUPTS_MASKED_CYCLES, 0xCD, 0 }, 153145256Sjkoshy { PMC_EV_K7_INTERRUPTS_MASKED_WHILE_PENDING_CYCLES, 0xCE, 0 }, 154147191Sjkoshy { PMC_EV_K7_HARDWARE_INTERRUPTS, 0xCF, 0 }, 155145256Sjkoshy#endif 156145256Sjkoshy 157145256Sjkoshy { PMC_EV_K8_FP_DISPATCHED_FPU_OPS, 0x00, 0x3F }, 158145256Sjkoshy { PMC_EV_K8_FP_CYCLES_WITH_NO_FPU_OPS_RETIRED, 0x01, 0x00 }, 159145256Sjkoshy { PMC_EV_K8_FP_DISPATCHED_FPU_FAST_FLAG_OPS, 0x02, 0x00 }, 160145256Sjkoshy 161145256Sjkoshy { PMC_EV_K8_LS_SEGMENT_REGISTER_LOAD, 0x20, 0x7F }, 162145256Sjkoshy { PMC_EV_K8_LS_MICROARCHITECTURAL_RESYNC_BY_SELF_MODIFYING_CODE, 163145256Sjkoshy 0x21, 0x00 }, 164145256Sjkoshy { PMC_EV_K8_LS_MICROARCHITECTURAL_RESYNC_BY_SNOOP, 0x22, 0x00 }, 165145256Sjkoshy { PMC_EV_K8_LS_BUFFER2_FULL, 0x23, 0x00 }, 166145256Sjkoshy { PMC_EV_K8_LS_LOCKED_OPERATION, 0x24, 0x07 }, 167145256Sjkoshy { PMC_EV_K8_LS_MICROARCHITECTURAL_LATE_CANCEL, 0x25, 0x00 }, 168145256Sjkoshy { PMC_EV_K8_LS_RETIRED_CFLUSH_INSTRUCTIONS, 0x26, 0x00 }, 169145256Sjkoshy { PMC_EV_K8_LS_RETIRED_CPUID_INSTRUCTIONS, 0x27, 0x00 }, 170145256Sjkoshy 171145256Sjkoshy { PMC_EV_K8_DC_ACCESS, 0x40, 0x00 }, 172145256Sjkoshy { PMC_EV_K8_DC_MISS, 0x41, 0x00 }, 173145256Sjkoshy { PMC_EV_K8_DC_REFILL_FROM_L2, 0x42, 0x1F }, 174145256Sjkoshy { PMC_EV_K8_DC_REFILL_FROM_SYSTEM, 0x43, 0x1F }, 175145256Sjkoshy { PMC_EV_K8_DC_COPYBACK, 0x44, 0x1F }, 176145256Sjkoshy { PMC_EV_K8_DC_L1_DTLB_MISS_AND_L2_DTLB_HIT, 0x45, 0x00 }, 177145256Sjkoshy { PMC_EV_K8_DC_L1_DTLB_MISS_AND_L2_DTLB_MISS, 0x46, 0x00 }, 178145256Sjkoshy { PMC_EV_K8_DC_MISALIGNED_DATA_REFERENCE, 0x47, 0x00 }, 179145256Sjkoshy { PMC_EV_K8_DC_MICROARCHITECTURAL_LATE_CANCEL, 0x48, 0x00 }, 180145256Sjkoshy { PMC_EV_K8_DC_MICROARCHITECTURAL_EARLY_CANCEL, 0x49, 0x00 }, 181145256Sjkoshy { PMC_EV_K8_DC_ONE_BIT_ECC_ERROR, 0x4A, 0x03 }, 182145256Sjkoshy { PMC_EV_K8_DC_DISPATCHED_PREFETCH_INSTRUCTIONS, 0x4B, 0x07 }, 183145256Sjkoshy { PMC_EV_K8_DC_DCACHE_ACCESSES_BY_LOCKS, 0x4C, 0x03 }, 184145256Sjkoshy 185145256Sjkoshy { PMC_EV_K8_BU_CPU_CLK_UNHALTED, 0x76, 0x00 }, 186145256Sjkoshy { PMC_EV_K8_BU_INTERNAL_L2_REQUEST, 0x7D, 0x1F }, 187145256Sjkoshy { PMC_EV_K8_BU_FILL_REQUEST_L2_MISS, 0x7E, 0x07 }, 188145256Sjkoshy { PMC_EV_K8_BU_FILL_INTO_L2, 0x7F, 0x03 }, 189145256Sjkoshy 190145256Sjkoshy { PMC_EV_K8_IC_FETCH, 0x80, 0x00 }, 191145256Sjkoshy { PMC_EV_K8_IC_MISS, 0x81, 0x00 }, 192145256Sjkoshy { PMC_EV_K8_IC_REFILL_FROM_L2, 0x82, 0x00 }, 193145256Sjkoshy { PMC_EV_K8_IC_REFILL_FROM_SYSTEM, 0x83, 0x00 }, 194145256Sjkoshy { PMC_EV_K8_IC_L1_ITLB_MISS_AND_L2_ITLB_HIT, 0x84, 0x00 }, 195145256Sjkoshy { PMC_EV_K8_IC_L1_ITLB_MISS_AND_L2_ITLB_MISS, 0x85, 0x00 }, 196145256Sjkoshy { PMC_EV_K8_IC_MICROARCHITECTURAL_RESYNC_BY_SNOOP, 0x86, 0x00 }, 197145256Sjkoshy { PMC_EV_K8_IC_INSTRUCTION_FETCH_STALL, 0x87, 0x00 }, 198145256Sjkoshy { PMC_EV_K8_IC_RETURN_STACK_HIT, 0x88, 0x00 }, 199145256Sjkoshy { PMC_EV_K8_IC_RETURN_STACK_OVERFLOW, 0x89, 0x00 }, 200145256Sjkoshy 201145256Sjkoshy { PMC_EV_K8_FR_RETIRED_X86_INSTRUCTIONS, 0xC0, 0x00 }, 202145256Sjkoshy { PMC_EV_K8_FR_RETIRED_UOPS, 0xC1, 0x00 }, 203145256Sjkoshy { PMC_EV_K8_FR_RETIRED_BRANCHES, 0xC2, 0x00 }, 204145256Sjkoshy { PMC_EV_K8_FR_RETIRED_BRANCHES_MISPREDICTED, 0xC3, 0x00 }, 205145256Sjkoshy { PMC_EV_K8_FR_RETIRED_TAKEN_BRANCHES, 0xC4, 0x00 }, 206145256Sjkoshy { PMC_EV_K8_FR_RETIRED_TAKEN_BRANCHES_MISPREDICTED, 0xC5, 0x00 }, 207145256Sjkoshy { PMC_EV_K8_FR_RETIRED_FAR_CONTROL_TRANSFERS, 0xC6, 0x00 }, 208145256Sjkoshy { PMC_EV_K8_FR_RETIRED_RESYNCS, 0xC7, 0x00 }, 209145256Sjkoshy { PMC_EV_K8_FR_RETIRED_NEAR_RETURNS, 0xC8, 0x00 }, 210145256Sjkoshy { PMC_EV_K8_FR_RETIRED_NEAR_RETURNS_MISPREDICTED, 0xC9, 0x00 }, 211145256Sjkoshy { PMC_EV_K8_FR_RETIRED_TAKEN_BRANCHES_MISPREDICTED_BY_ADDR_MISCOMPARE, 212145256Sjkoshy 0xCA, 0x00 }, 213145256Sjkoshy { PMC_EV_K8_FR_RETIRED_FPU_INSTRUCTIONS, 0xCB, 0x0F }, 214145256Sjkoshy { PMC_EV_K8_FR_RETIRED_FASTPATH_DOUBLE_OP_INSTRUCTIONS, 215145256Sjkoshy 0xCC, 0x07 }, 216145256Sjkoshy { PMC_EV_K8_FR_INTERRUPTS_MASKED_CYCLES, 0xCD, 0x00 }, 217145256Sjkoshy { PMC_EV_K8_FR_INTERRUPTS_MASKED_WHILE_PENDING_CYCLES, 0xCE, 0x00 }, 218145256Sjkoshy { PMC_EV_K8_FR_TAKEN_HARDWARE_INTERRUPTS, 0xCF, 0x00 }, 219145256Sjkoshy 220145256Sjkoshy { PMC_EV_K8_FR_DECODER_EMPTY, 0xD0, 0x00 }, 221145256Sjkoshy { PMC_EV_K8_FR_DISPATCH_STALLS, 0xD1, 0x00 }, 222145256Sjkoshy { PMC_EV_K8_FR_DISPATCH_STALL_FROM_BRANCH_ABORT_TO_RETIRE, 223145256Sjkoshy 0xD2, 0x00 }, 224145256Sjkoshy { PMC_EV_K8_FR_DISPATCH_STALL_FOR_SERIALIZATION, 0xD3, 0x00 }, 225145256Sjkoshy { PMC_EV_K8_FR_DISPATCH_STALL_FOR_SEGMENT_LOAD, 0xD4, 0x00 }, 226145256Sjkoshy { PMC_EV_K8_FR_DISPATCH_STALL_WHEN_REORDER_BUFFER_IS_FULL, 227145256Sjkoshy 0xD5, 0x00 }, 228145256Sjkoshy { PMC_EV_K8_FR_DISPATCH_STALL_WHEN_RESERVATION_STATIONS_ARE_FULL, 229145256Sjkoshy 0xD6, 0x00 }, 230145256Sjkoshy { PMC_EV_K8_FR_DISPATCH_STALL_WHEN_FPU_IS_FULL, 0xD7, 0x00 }, 231145256Sjkoshy { PMC_EV_K8_FR_DISPATCH_STALL_WHEN_LS_IS_FULL, 0xD8, 0x00 }, 232145256Sjkoshy { PMC_EV_K8_FR_DISPATCH_STALL_WHEN_WAITING_FOR_ALL_TO_BE_QUIET, 233145256Sjkoshy 0xD9, 0x00 }, 234145256Sjkoshy { PMC_EV_K8_FR_DISPATCH_STALL_WHEN_FAR_XFER_OR_RESYNC_BRANCH_PENDING, 235145256Sjkoshy 0xDA, 0x00 }, 236145256Sjkoshy { PMC_EV_K8_FR_FPU_EXCEPTIONS, 0xDB, 0x0F }, 237145256Sjkoshy { PMC_EV_K8_FR_NUMBER_OF_BREAKPOINTS_FOR_DR0, 0xDC, 0x00 }, 238145256Sjkoshy { PMC_EV_K8_FR_NUMBER_OF_BREAKPOINTS_FOR_DR1, 0xDD, 0x00 }, 239145256Sjkoshy { PMC_EV_K8_FR_NUMBER_OF_BREAKPOINTS_FOR_DR2, 0xDE, 0x00 }, 240145256Sjkoshy { PMC_EV_K8_FR_NUMBER_OF_BREAKPOINTS_FOR_DR3, 0xDF, 0x00 }, 241145256Sjkoshy 242145256Sjkoshy { PMC_EV_K8_NB_MEMORY_CONTROLLER_PAGE_ACCESS_EVENT, 0xE0, 0x7 }, 243145256Sjkoshy { PMC_EV_K8_NB_MEMORY_CONTROLLER_PAGE_TABLE_OVERFLOW, 0xE1, 0x00 }, 244145256Sjkoshy { PMC_EV_K8_NB_MEMORY_CONTROLLER_DRAM_COMMAND_SLOTS_MISSED, 245145256Sjkoshy 0xE2, 0x00 }, 246145256Sjkoshy { PMC_EV_K8_NB_MEMORY_CONTROLLER_TURNAROUND, 0xE3, 0x07 }, 247145256Sjkoshy { PMC_EV_K8_NB_MEMORY_CONTROLLER_BYPASS_SATURATION, 0xE4, 0x0F }, 248145256Sjkoshy { PMC_EV_K8_NB_SIZED_COMMANDS, 0xEB, 0x7F }, 249145256Sjkoshy { PMC_EV_K8_NB_PROBE_RESULT, 0xEC, 0x0F }, 250145256Sjkoshy { PMC_EV_K8_NB_HT_BUS0_BANDWIDTH, 0xF6, 0x0F }, 251145256Sjkoshy { PMC_EV_K8_NB_HT_BUS1_BANDWIDTH, 0xF7, 0x0F }, 252145256Sjkoshy { PMC_EV_K8_NB_HT_BUS2_BANDWIDTH, 0xF8, 0x0F } 253145256Sjkoshy 254145256Sjkoshy}; 255145256Sjkoshy 256145256Sjkoshyconst int amd_event_codes_size = 257145256Sjkoshy sizeof(amd_event_codes) / sizeof(amd_event_codes[0]); 258145256Sjkoshy 259145256Sjkoshy/* 260145256Sjkoshy * read a pmc register 261145256Sjkoshy */ 262145256Sjkoshy 263145256Sjkoshystatic int 264145256Sjkoshyamd_read_pmc(int cpu, int ri, pmc_value_t *v) 265145256Sjkoshy{ 266145256Sjkoshy enum pmc_mode mode; 267145256Sjkoshy const struct amd_descr *pd; 268145256Sjkoshy struct pmc *pm; 269145256Sjkoshy const struct pmc_hw *phw; 270145256Sjkoshy pmc_value_t tmp; 271145256Sjkoshy 272145256Sjkoshy KASSERT(cpu >= 0 && cpu < mp_ncpus, 273145256Sjkoshy ("[amd,%d] illegal CPU value %d", __LINE__, cpu)); 274145256Sjkoshy KASSERT(ri >= 0 && ri < AMD_NPMCS, 275145256Sjkoshy ("[amd,%d] illegal row-index %d", __LINE__, ri)); 276145256Sjkoshy 277145256Sjkoshy phw = pmc_pcpu[cpu]->pc_hwpmcs[ri]; 278145256Sjkoshy pd = &amd_pmcdesc[ri]; 279145256Sjkoshy pm = phw->phw_pmc; 280145256Sjkoshy 281145256Sjkoshy KASSERT(pm != NULL, 282145256Sjkoshy ("[amd,%d] No owner for HWPMC [cpu%d,pmc%d]", __LINE__, 283145256Sjkoshy cpu, ri)); 284145256Sjkoshy 285145774Sjkoshy mode = PMC_TO_MODE(pm); 286145256Sjkoshy 287145256Sjkoshy PMCDBG(MDP,REA,1,"amd-read id=%d class=%d", ri, pd->pm_descr.pd_class); 288145256Sjkoshy 289145256Sjkoshy /* Reading the TSC is a special case */ 290145256Sjkoshy if (pd->pm_descr.pd_class == PMC_CLASS_TSC) { 291145256Sjkoshy KASSERT(PMC_IS_COUNTING_MODE(mode), 292145256Sjkoshy ("[amd,%d] TSC counter in non-counting mode", __LINE__)); 293145256Sjkoshy *v = rdtsc(); 294145256Sjkoshy PMCDBG(MDP,REA,2,"amd-read id=%d -> %jd", ri, *v); 295145256Sjkoshy return 0; 296145256Sjkoshy } 297145256Sjkoshy 298153110Sru#ifdef DEBUG 299147191Sjkoshy KASSERT(pd->pm_descr.pd_class == amd_pmc_class, 300145256Sjkoshy ("[amd,%d] unknown PMC class (%d)", __LINE__, 301145256Sjkoshy pd->pm_descr.pd_class)); 302147191Sjkoshy#endif 303145256Sjkoshy 304145256Sjkoshy tmp = rdmsr(pd->pm_perfctr); /* RDMSR serializes */ 305177344Sadrian PMCDBG(MDP,REA,2,"amd-read (pre-munge) id=%d -> %jd", ri, tmp); 306177344Sadrian if (PMC_IS_SAMPLING_MODE(mode)) { 307177344Sadrian /* Sign extend 48 bit value to 64 bits. */ 308177344Sadrian tmp = (pmc_value_t) (((int64_t) tmp << 16) >> 16); 309177344Sadrian tmp = AMD_PERFCTR_VALUE_TO_RELOAD_COUNT(tmp); 310177344Sadrian } 311177344Sadrian *v = tmp; 312145256Sjkoshy 313177344Sadrian PMCDBG(MDP,REA,2,"amd-read (post-munge) id=%d -> %jd", ri, *v); 314145256Sjkoshy 315145256Sjkoshy return 0; 316145256Sjkoshy} 317145256Sjkoshy 318145256Sjkoshy/* 319145256Sjkoshy * Write a PMC MSR. 320145256Sjkoshy */ 321145256Sjkoshy 322145256Sjkoshystatic int 323145256Sjkoshyamd_write_pmc(int cpu, int ri, pmc_value_t v) 324145256Sjkoshy{ 325145256Sjkoshy const struct amd_descr *pd; 326145256Sjkoshy struct pmc *pm; 327145256Sjkoshy const struct pmc_hw *phw; 328145256Sjkoshy enum pmc_mode mode; 329145256Sjkoshy 330145256Sjkoshy KASSERT(cpu >= 0 && cpu < mp_ncpus, 331145256Sjkoshy ("[amd,%d] illegal CPU value %d", __LINE__, cpu)); 332145256Sjkoshy KASSERT(ri >= 0 && ri < AMD_NPMCS, 333145256Sjkoshy ("[amd,%d] illegal row-index %d", __LINE__, ri)); 334145256Sjkoshy 335145256Sjkoshy phw = pmc_pcpu[cpu]->pc_hwpmcs[ri]; 336145256Sjkoshy pd = &amd_pmcdesc[ri]; 337145256Sjkoshy pm = phw->phw_pmc; 338145256Sjkoshy 339145256Sjkoshy KASSERT(pm != NULL, 340145256Sjkoshy ("[amd,%d] PMC not owned (cpu%d,pmc%d)", __LINE__, 341145256Sjkoshy cpu, ri)); 342145256Sjkoshy 343145774Sjkoshy mode = PMC_TO_MODE(pm); 344145256Sjkoshy 345145256Sjkoshy if (pd->pm_descr.pd_class == PMC_CLASS_TSC) 346145256Sjkoshy return 0; 347145256Sjkoshy 348153110Sru#ifdef DEBUG 349147191Sjkoshy KASSERT(pd->pm_descr.pd_class == amd_pmc_class, 350145256Sjkoshy ("[amd,%d] unknown PMC class (%d)", __LINE__, 351145256Sjkoshy pd->pm_descr.pd_class)); 352147191Sjkoshy#endif 353145256Sjkoshy 354145256Sjkoshy /* use 2's complement of the count for sampling mode PMCs */ 355145256Sjkoshy if (PMC_IS_SAMPLING_MODE(mode)) 356147191Sjkoshy v = AMD_RELOAD_COUNT_TO_PERFCTR_VALUE(v); 357145256Sjkoshy 358145256Sjkoshy PMCDBG(MDP,WRI,1,"amd-write cpu=%d ri=%d v=%jx", cpu, ri, v); 359145256Sjkoshy 360145256Sjkoshy /* write the PMC value */ 361145256Sjkoshy wrmsr(pd->pm_perfctr, v); 362145256Sjkoshy return 0; 363145256Sjkoshy} 364145256Sjkoshy 365145256Sjkoshy/* 366145256Sjkoshy * configure hardware pmc according to the configuration recorded in 367145256Sjkoshy * pmc 'pm'. 368145256Sjkoshy */ 369145256Sjkoshy 370145256Sjkoshystatic int 371145256Sjkoshyamd_config_pmc(int cpu, int ri, struct pmc *pm) 372145256Sjkoshy{ 373145256Sjkoshy struct pmc_hw *phw; 374145256Sjkoshy 375145615Sjkoshy PMCDBG(MDP,CFG,1, "cpu=%d ri=%d pm=%p", cpu, ri, pm); 376145615Sjkoshy 377145256Sjkoshy KASSERT(cpu >= 0 && cpu < mp_ncpus, 378145256Sjkoshy ("[amd,%d] illegal CPU value %d", __LINE__, cpu)); 379145256Sjkoshy KASSERT(ri >= 0 && ri < AMD_NPMCS, 380145256Sjkoshy ("[amd,%d] illegal row-index %d", __LINE__, ri)); 381145256Sjkoshy 382145256Sjkoshy phw = pmc_pcpu[cpu]->pc_hwpmcs[ri]; 383145256Sjkoshy 384145256Sjkoshy KASSERT(pm == NULL || phw->phw_pmc == NULL, 385145615Sjkoshy ("[amd,%d] pm=%p phw->pm=%p hwpmc not unconfigured", 386145615Sjkoshy __LINE__, pm, phw->phw_pmc)); 387145256Sjkoshy 388145256Sjkoshy phw->phw_pmc = pm; 389145256Sjkoshy return 0; 390145256Sjkoshy} 391145256Sjkoshy 392145256Sjkoshy/* 393145774Sjkoshy * Retrieve a configured PMC pointer from hardware state. 394145774Sjkoshy */ 395145774Sjkoshy 396145774Sjkoshystatic int 397145774Sjkoshyamd_get_config(int cpu, int ri, struct pmc **ppm) 398145774Sjkoshy{ 399145774Sjkoshy *ppm = pmc_pcpu[cpu]->pc_hwpmcs[ri]->phw_pmc; 400145774Sjkoshy 401145774Sjkoshy return 0; 402145774Sjkoshy} 403145774Sjkoshy 404145774Sjkoshy/* 405145256Sjkoshy * Machine dependent actions taken during the context switch in of a 406145256Sjkoshy * thread. 407145256Sjkoshy */ 408145256Sjkoshy 409145256Sjkoshystatic int 410145615Sjkoshyamd_switch_in(struct pmc_cpu *pc, struct pmc_process *pp) 411145256Sjkoshy{ 412145256Sjkoshy (void) pc; 413145256Sjkoshy 414145615Sjkoshy PMCDBG(MDP,SWI,1, "pc=%p pp=%p enable-msr=%d", pc, pp, 415145774Sjkoshy (pp->pp_flags & PMC_PP_ENABLE_MSR_ACCESS) != 0); 416145615Sjkoshy 417145615Sjkoshy /* enable the RDPMC instruction if needed */ 418145774Sjkoshy if (pp->pp_flags & PMC_PP_ENABLE_MSR_ACCESS) 419145615Sjkoshy load_cr4(rcr4() | CR4_PCE); 420145615Sjkoshy 421145256Sjkoshy return 0; 422145256Sjkoshy} 423145256Sjkoshy 424145256Sjkoshy/* 425145256Sjkoshy * Machine dependent actions taken during the context switch out of a 426145256Sjkoshy * thread. 427145256Sjkoshy */ 428145256Sjkoshy 429145256Sjkoshystatic int 430145615Sjkoshyamd_switch_out(struct pmc_cpu *pc, struct pmc_process *pp) 431145256Sjkoshy{ 432145256Sjkoshy (void) pc; 433145615Sjkoshy (void) pp; /* can be NULL */ 434145256Sjkoshy 435145615Sjkoshy PMCDBG(MDP,SWO,1, "pc=%p pp=%p enable-msr=%d", pc, pp, pp ? 436145774Sjkoshy (pp->pp_flags & PMC_PP_ENABLE_MSR_ACCESS) == 1 : 0); 437145615Sjkoshy 438145615Sjkoshy /* always turn off the RDPMC instruction */ 439145256Sjkoshy load_cr4(rcr4() & ~CR4_PCE); 440145615Sjkoshy 441145256Sjkoshy return 0; 442145256Sjkoshy} 443145256Sjkoshy 444145256Sjkoshy/* 445145256Sjkoshy * Check if a given allocation is feasible. 446145256Sjkoshy */ 447145256Sjkoshy 448145256Sjkoshystatic int 449145256Sjkoshyamd_allocate_pmc(int cpu, int ri, struct pmc *pm, 450145256Sjkoshy const struct pmc_op_pmcallocate *a) 451145256Sjkoshy{ 452145256Sjkoshy int i; 453145256Sjkoshy uint32_t allowed_unitmask, caps, config, unitmask; 454145256Sjkoshy enum pmc_event pe; 455145256Sjkoshy const struct pmc_descr *pd; 456145256Sjkoshy 457145256Sjkoshy (void) cpu; 458145256Sjkoshy 459145256Sjkoshy KASSERT(cpu >= 0 && cpu < mp_ncpus, 460145256Sjkoshy ("[amd,%d] illegal CPU value %d", __LINE__, cpu)); 461145256Sjkoshy KASSERT(ri >= 0 && ri < AMD_NPMCS, 462145256Sjkoshy ("[amd,%d] illegal row index %d", __LINE__, ri)); 463145256Sjkoshy 464145256Sjkoshy pd = &amd_pmcdesc[ri].pm_descr; 465145256Sjkoshy 466145256Sjkoshy /* check class match */ 467145774Sjkoshy if (pd->pd_class != a->pm_class) 468145256Sjkoshy return EINVAL; 469145256Sjkoshy 470145256Sjkoshy caps = pm->pm_caps; 471145256Sjkoshy 472145256Sjkoshy PMCDBG(MDP,ALL,1,"amd-allocate ri=%d caps=0x%x", ri, caps); 473145256Sjkoshy 474145256Sjkoshy if ((pd->pd_caps & caps) != caps) 475145256Sjkoshy return EPERM; 476145256Sjkoshy if (pd->pd_class == PMC_CLASS_TSC) { 477145256Sjkoshy /* TSC's are always allocated in system-wide counting mode */ 478145256Sjkoshy if (a->pm_ev != PMC_EV_TSC_TSC || 479145256Sjkoshy a->pm_mode != PMC_MODE_SC) 480145256Sjkoshy return EINVAL; 481145256Sjkoshy return 0; 482145256Sjkoshy } 483145256Sjkoshy 484153110Sru#ifdef DEBUG 485147191Sjkoshy KASSERT(pd->pd_class == amd_pmc_class, 486145256Sjkoshy ("[amd,%d] Unknown PMC class (%d)", __LINE__, pd->pd_class)); 487147191Sjkoshy#endif 488145256Sjkoshy 489145256Sjkoshy pe = a->pm_ev; 490145256Sjkoshy 491145256Sjkoshy /* map ev to the correct event mask code */ 492145256Sjkoshy config = allowed_unitmask = 0; 493145256Sjkoshy for (i = 0; i < amd_event_codes_size; i++) 494145256Sjkoshy if (amd_event_codes[i].pe_ev == pe) { 495145256Sjkoshy config = 496145256Sjkoshy AMD_PMC_TO_EVENTMASK(amd_event_codes[i].pe_code); 497145256Sjkoshy allowed_unitmask = 498145256Sjkoshy AMD_PMC_TO_UNITMASK(amd_event_codes[i].pe_mask); 499145256Sjkoshy break; 500145256Sjkoshy } 501145256Sjkoshy if (i == amd_event_codes_size) 502145256Sjkoshy return EINVAL; 503145256Sjkoshy 504147191Sjkoshy unitmask = a->pm_md.pm_amd.pm_amd_config & AMD_PMC_UNITMASK; 505145256Sjkoshy if (unitmask & ~allowed_unitmask) /* disallow reserved bits */ 506145256Sjkoshy return EINVAL; 507145256Sjkoshy 508145256Sjkoshy if (unitmask && (caps & PMC_CAP_QUALIFIER)) 509145256Sjkoshy config |= unitmask; 510145256Sjkoshy 511145256Sjkoshy if (caps & PMC_CAP_THRESHOLD) 512147191Sjkoshy config |= a->pm_md.pm_amd.pm_amd_config & AMD_PMC_COUNTERMASK; 513145256Sjkoshy 514145256Sjkoshy /* set at least one of the 'usr' or 'os' caps */ 515145256Sjkoshy if (caps & PMC_CAP_USER) 516145256Sjkoshy config |= AMD_PMC_USR; 517145256Sjkoshy if (caps & PMC_CAP_SYSTEM) 518145256Sjkoshy config |= AMD_PMC_OS; 519145256Sjkoshy if ((caps & (PMC_CAP_USER|PMC_CAP_SYSTEM)) == 0) 520145256Sjkoshy config |= (AMD_PMC_USR|AMD_PMC_OS); 521145256Sjkoshy 522145256Sjkoshy if (caps & PMC_CAP_EDGE) 523145256Sjkoshy config |= AMD_PMC_EDGE; 524145256Sjkoshy if (caps & PMC_CAP_INVERT) 525145256Sjkoshy config |= AMD_PMC_INVERT; 526145256Sjkoshy if (caps & PMC_CAP_INTERRUPT) 527145256Sjkoshy config |= AMD_PMC_INT; 528145256Sjkoshy 529145256Sjkoshy pm->pm_md.pm_amd.pm_amd_evsel = config; /* save config value */ 530145256Sjkoshy 531145256Sjkoshy PMCDBG(MDP,ALL,2,"amd-allocate ri=%d -> config=0x%x", ri, config); 532145256Sjkoshy 533145256Sjkoshy return 0; 534145256Sjkoshy} 535145256Sjkoshy 536145256Sjkoshy/* 537145256Sjkoshy * Release machine dependent state associated with a PMC. This is a 538145256Sjkoshy * no-op on this architecture. 539145256Sjkoshy * 540145256Sjkoshy */ 541145256Sjkoshy 542145256Sjkoshy/* ARGSUSED0 */ 543145256Sjkoshystatic int 544145256Sjkoshyamd_release_pmc(int cpu, int ri, struct pmc *pmc) 545145256Sjkoshy{ 546153110Sru#ifdef DEBUG 547145256Sjkoshy const struct amd_descr *pd; 548145256Sjkoshy#endif 549145256Sjkoshy struct pmc_hw *phw; 550145256Sjkoshy 551145256Sjkoshy (void) pmc; 552145256Sjkoshy 553145256Sjkoshy KASSERT(cpu >= 0 && cpu < mp_ncpus, 554145256Sjkoshy ("[amd,%d] illegal CPU value %d", __LINE__, cpu)); 555145256Sjkoshy KASSERT(ri >= 0 && ri < AMD_NPMCS, 556145256Sjkoshy ("[amd,%d] illegal row-index %d", __LINE__, ri)); 557145256Sjkoshy 558145256Sjkoshy phw = pmc_pcpu[cpu]->pc_hwpmcs[ri]; 559145256Sjkoshy 560145256Sjkoshy KASSERT(phw->phw_pmc == NULL, 561145256Sjkoshy ("[amd,%d] PHW pmc %p non-NULL", __LINE__, phw->phw_pmc)); 562145256Sjkoshy 563153110Sru#ifdef DEBUG 564145256Sjkoshy pd = &amd_pmcdesc[ri]; 565147191Sjkoshy if (pd->pm_descr.pd_class == amd_pmc_class) 566145256Sjkoshy KASSERT(AMD_PMC_IS_STOPPED(pd->pm_evsel), 567145256Sjkoshy ("[amd,%d] PMC %d released while active", __LINE__, ri)); 568145256Sjkoshy#endif 569145256Sjkoshy 570145256Sjkoshy return 0; 571145256Sjkoshy} 572145256Sjkoshy 573145256Sjkoshy/* 574145256Sjkoshy * start a PMC. 575145256Sjkoshy */ 576145256Sjkoshy 577145256Sjkoshystatic int 578145256Sjkoshyamd_start_pmc(int cpu, int ri) 579145256Sjkoshy{ 580145256Sjkoshy uint32_t config; 581145256Sjkoshy struct pmc *pm; 582145256Sjkoshy struct pmc_hw *phw; 583145256Sjkoshy const struct amd_descr *pd; 584145256Sjkoshy 585145256Sjkoshy KASSERT(cpu >= 0 && cpu < mp_ncpus, 586145256Sjkoshy ("[amd,%d] illegal CPU value %d", __LINE__, cpu)); 587145256Sjkoshy KASSERT(ri >= 0 && ri < AMD_NPMCS, 588145256Sjkoshy ("[amd,%d] illegal row-index %d", __LINE__, ri)); 589145256Sjkoshy 590145256Sjkoshy phw = pmc_pcpu[cpu]->pc_hwpmcs[ri]; 591145256Sjkoshy pm = phw->phw_pmc; 592145256Sjkoshy pd = &amd_pmcdesc[ri]; 593145256Sjkoshy 594145256Sjkoshy KASSERT(pm != NULL, 595145256Sjkoshy ("[amd,%d] starting cpu%d,pmc%d with null pmc record", __LINE__, 596145256Sjkoshy cpu, ri)); 597145256Sjkoshy 598145256Sjkoshy PMCDBG(MDP,STA,1,"amd-start cpu=%d ri=%d", cpu, ri); 599145256Sjkoshy 600145256Sjkoshy if (pd->pm_descr.pd_class == PMC_CLASS_TSC) 601145256Sjkoshy return 0; /* TSCs are always running */ 602145256Sjkoshy 603153110Sru#ifdef DEBUG 604147191Sjkoshy KASSERT(pd->pm_descr.pd_class == amd_pmc_class, 605145256Sjkoshy ("[amd,%d] unknown PMC class (%d)", __LINE__, 606145256Sjkoshy pd->pm_descr.pd_class)); 607147191Sjkoshy#endif 608145256Sjkoshy 609145256Sjkoshy KASSERT(AMD_PMC_IS_STOPPED(pd->pm_evsel), 610145256Sjkoshy ("[amd,%d] pmc%d,cpu%d: Starting active PMC \"%s\"", __LINE__, 611145256Sjkoshy ri, cpu, pd->pm_descr.pd_name)); 612145256Sjkoshy 613145256Sjkoshy /* turn on the PMC ENABLE bit */ 614145256Sjkoshy config = pm->pm_md.pm_amd.pm_amd_evsel | AMD_PMC_ENABLE; 615145256Sjkoshy 616145256Sjkoshy PMCDBG(MDP,STA,2,"amd-start config=0x%x", config); 617145256Sjkoshy 618145256Sjkoshy wrmsr(pd->pm_evsel, config); 619145256Sjkoshy return 0; 620145256Sjkoshy} 621145256Sjkoshy 622145256Sjkoshy/* 623145256Sjkoshy * Stop a PMC. 624145256Sjkoshy */ 625145256Sjkoshy 626145256Sjkoshystatic int 627145256Sjkoshyamd_stop_pmc(int cpu, int ri) 628145256Sjkoshy{ 629145256Sjkoshy struct pmc *pm; 630145256Sjkoshy struct pmc_hw *phw; 631145256Sjkoshy const struct amd_descr *pd; 632145256Sjkoshy uint64_t config; 633145256Sjkoshy 634145256Sjkoshy KASSERT(cpu >= 0 && cpu < mp_ncpus, 635145256Sjkoshy ("[amd,%d] illegal CPU value %d", __LINE__, cpu)); 636145256Sjkoshy KASSERT(ri >= 0 && ri < AMD_NPMCS, 637145256Sjkoshy ("[amd,%d] illegal row-index %d", __LINE__, ri)); 638145256Sjkoshy 639145256Sjkoshy phw = pmc_pcpu[cpu]->pc_hwpmcs[ri]; 640145256Sjkoshy pm = phw->phw_pmc; 641145256Sjkoshy pd = &amd_pmcdesc[ri]; 642145256Sjkoshy 643145256Sjkoshy KASSERT(pm != NULL, 644145256Sjkoshy ("[amd,%d] cpu%d,pmc%d no PMC to stop", __LINE__, 645145256Sjkoshy cpu, ri)); 646145256Sjkoshy 647145256Sjkoshy /* can't stop a TSC */ 648145256Sjkoshy if (pd->pm_descr.pd_class == PMC_CLASS_TSC) 649145256Sjkoshy return 0; 650145256Sjkoshy 651153110Sru#ifdef DEBUG 652147191Sjkoshy KASSERT(pd->pm_descr.pd_class == amd_pmc_class, 653145256Sjkoshy ("[amd,%d] unknown PMC class (%d)", __LINE__, 654145256Sjkoshy pd->pm_descr.pd_class)); 655147191Sjkoshy#endif 656145256Sjkoshy 657145256Sjkoshy KASSERT(!AMD_PMC_IS_STOPPED(pd->pm_evsel), 658145256Sjkoshy ("[amd,%d] PMC%d, CPU%d \"%s\" already stopped", 659145256Sjkoshy __LINE__, ri, cpu, pd->pm_descr.pd_name)); 660145256Sjkoshy 661145256Sjkoshy PMCDBG(MDP,STO,1,"amd-stop ri=%d", ri); 662145256Sjkoshy 663145256Sjkoshy /* turn off the PMC ENABLE bit */ 664145256Sjkoshy config = pm->pm_md.pm_amd.pm_amd_evsel & ~AMD_PMC_ENABLE; 665145256Sjkoshy wrmsr(pd->pm_evsel, config); 666145256Sjkoshy return 0; 667145256Sjkoshy} 668145256Sjkoshy 669145256Sjkoshy/* 670145256Sjkoshy * Interrupt handler. This function needs to return '1' if the 671145256Sjkoshy * interrupt was this CPU's PMCs or '0' otherwise. It is not allowed 672145256Sjkoshy * to sleep or do anything a 'fast' interrupt handler is not allowed 673145256Sjkoshy * to do. 674145256Sjkoshy */ 675145256Sjkoshy 676145256Sjkoshystatic int 677174395Sjkoshyamd_intr(int cpu, struct trapframe *tf) 678145256Sjkoshy{ 679147191Sjkoshy int i, error, retval, ri; 680147191Sjkoshy uint32_t config, evsel, perfctr; 681145256Sjkoshy struct pmc *pm; 682145256Sjkoshy struct pmc_cpu *pc; 683145256Sjkoshy struct pmc_hw *phw; 684147191Sjkoshy pmc_value_t v; 685145256Sjkoshy 686145256Sjkoshy KASSERT(cpu >= 0 && cpu < mp_ncpus, 687145256Sjkoshy ("[amd,%d] out of range CPU %d", __LINE__, cpu)); 688145256Sjkoshy 689177343Sadrian PMCDBG(MDP,INT,1, "cpu=%d tf=%p um=%d", cpu, (void *) tf, 690174395Sjkoshy TRAPF_USERMODE(tf)); 691147191Sjkoshy 692145256Sjkoshy retval = 0; 693145256Sjkoshy 694145256Sjkoshy pc = pmc_pcpu[cpu]; 695145256Sjkoshy 696145256Sjkoshy /* 697145256Sjkoshy * look for all PMCs that have interrupted: 698145256Sjkoshy * - skip over the TSC [PMC#0] 699147191Sjkoshy * - look for a running, sampling PMC which has overflowed 700147191Sjkoshy * and which has a valid 'struct pmc' association 701147191Sjkoshy * 702147191Sjkoshy * If found, we call a helper to process the interrupt. 703150050Sjkoshy * 704150050Sjkoshy * If multiple PMCs interrupt at the same time, the AMD64 705150050Sjkoshy * processor appears to deliver as many NMIs as there are 706174395Sjkoshy * outstanding PMC interrupts. So we process only one NMI 707174395Sjkoshy * interrupt at a time. 708145256Sjkoshy */ 709145256Sjkoshy 710150050Sjkoshy for (i = 0; retval == 0 && i < AMD_NPMCS-1; i++) { 711145256Sjkoshy 712147191Sjkoshy ri = i + 1; /* row index; TSC is at ri == 0 */ 713147191Sjkoshy 714147191Sjkoshy if (!AMD_PMC_HAS_OVERFLOWED(i)) 715147191Sjkoshy continue; 716147191Sjkoshy 717147191Sjkoshy phw = pc->pc_hwpmcs[ri]; 718147191Sjkoshy 719145256Sjkoshy KASSERT(phw != NULL, ("[amd,%d] null PHW pointer", __LINE__)); 720145256Sjkoshy 721145256Sjkoshy if ((pm = phw->phw_pmc) == NULL || 722147191Sjkoshy pm->pm_state != PMC_STATE_RUNNING || 723147191Sjkoshy !PMC_IS_SAMPLING_MODE(PMC_TO_MODE(pm))) { 724145256Sjkoshy continue; 725145256Sjkoshy } 726145256Sjkoshy 727174395Sjkoshy retval = 1; /* Found an interrupting PMC. */ 728150050Sjkoshy 729174395Sjkoshy /* Stop the PMC, reload count. */ 730147191Sjkoshy evsel = AMD_PMC_EVSEL_0 + i; 731147191Sjkoshy perfctr = AMD_PMC_PERFCTR_0 + i; 732147191Sjkoshy v = pm->pm_sc.pm_reloadcount; 733147191Sjkoshy config = rdmsr(evsel); 734147191Sjkoshy 735147191Sjkoshy KASSERT((config & ~AMD_PMC_ENABLE) == 736147191Sjkoshy (pm->pm_md.pm_amd.pm_amd_evsel & ~AMD_PMC_ENABLE), 737147191Sjkoshy ("[amd,%d] config mismatch reg=0x%x pm=0x%x", __LINE__, 738147191Sjkoshy config, pm->pm_md.pm_amd.pm_amd_evsel)); 739147191Sjkoshy 740147191Sjkoshy wrmsr(evsel, config & ~AMD_PMC_ENABLE); 741147191Sjkoshy wrmsr(perfctr, AMD_RELOAD_COUNT_TO_PERFCTR_VALUE(v)); 742147191Sjkoshy 743174395Sjkoshy /* Restart the counter if logging succeeded. */ 744174395Sjkoshy error = pmc_process_interrupt(cpu, pm, tf, TRAPF_USERMODE(tf)); 745147191Sjkoshy if (error == 0) 746147191Sjkoshy wrmsr(evsel, config | AMD_PMC_ENABLE); 747145256Sjkoshy } 748147191Sjkoshy 749147867Sjkoshy atomic_add_int(retval ? &pmc_stats.pm_intr_processed : 750147867Sjkoshy &pmc_stats.pm_intr_ignored, 1); 751147867Sjkoshy 752174395Sjkoshy return (retval); 753145256Sjkoshy} 754145256Sjkoshy 755145256Sjkoshy/* 756145256Sjkoshy * describe a PMC 757145256Sjkoshy */ 758145256Sjkoshystatic int 759145256Sjkoshyamd_describe(int cpu, int ri, struct pmc_info *pi, struct pmc **ppmc) 760145256Sjkoshy{ 761145256Sjkoshy int error; 762145256Sjkoshy size_t copied; 763145256Sjkoshy const struct amd_descr *pd; 764145256Sjkoshy struct pmc_hw *phw; 765145256Sjkoshy 766145256Sjkoshy KASSERT(cpu >= 0 && cpu < mp_ncpus, 767145256Sjkoshy ("[amd,%d] illegal CPU %d", __LINE__, cpu)); 768145256Sjkoshy KASSERT(ri >= 0 && ri < AMD_NPMCS, 769145256Sjkoshy ("[amd,%d] row-index %d out of range", __LINE__, ri)); 770145256Sjkoshy 771145256Sjkoshy phw = pmc_pcpu[cpu]->pc_hwpmcs[ri]; 772145256Sjkoshy pd = &amd_pmcdesc[ri]; 773145256Sjkoshy 774145256Sjkoshy if ((error = copystr(pd->pm_descr.pd_name, pi->pm_name, 775145256Sjkoshy PMC_NAME_MAX, &copied)) != 0) 776145256Sjkoshy return error; 777145256Sjkoshy 778145256Sjkoshy pi->pm_class = pd->pm_descr.pd_class; 779145256Sjkoshy 780145256Sjkoshy if (phw->phw_state & PMC_PHW_FLAG_IS_ENABLED) { 781145256Sjkoshy pi->pm_enabled = TRUE; 782145256Sjkoshy *ppmc = phw->phw_pmc; 783145256Sjkoshy } else { 784145256Sjkoshy pi->pm_enabled = FALSE; 785145256Sjkoshy *ppmc = NULL; 786145256Sjkoshy } 787145256Sjkoshy 788145256Sjkoshy return 0; 789145256Sjkoshy} 790145256Sjkoshy 791145256Sjkoshy/* 792145256Sjkoshy * i386 specific entry points 793145256Sjkoshy */ 794145256Sjkoshy 795145256Sjkoshy/* 796145256Sjkoshy * return the MSR address of the given PMC. 797145256Sjkoshy */ 798145256Sjkoshy 799145256Sjkoshystatic int 800145256Sjkoshyamd_get_msr(int ri, uint32_t *msr) 801145256Sjkoshy{ 802145256Sjkoshy KASSERT(ri >= 0 && ri < AMD_NPMCS, 803145256Sjkoshy ("[amd,%d] ri %d out of range", __LINE__, ri)); 804145256Sjkoshy 805145615Sjkoshy *msr = amd_pmcdesc[ri].pm_perfctr - AMD_PMC_PERFCTR_0; 806145256Sjkoshy return 0; 807145256Sjkoshy} 808145256Sjkoshy 809145256Sjkoshy/* 810145256Sjkoshy * processor dependent initialization. 811145256Sjkoshy */ 812145256Sjkoshy 813145256Sjkoshy/* 814145256Sjkoshy * Per-processor data structure 815145256Sjkoshy * 816145256Sjkoshy * [common stuff] 817145256Sjkoshy * [5 struct pmc_hw pointers] 818145256Sjkoshy * [5 struct pmc_hw structures] 819145256Sjkoshy */ 820145256Sjkoshy 821145256Sjkoshystruct amd_cpu { 822145256Sjkoshy struct pmc_cpu pc_common; 823145256Sjkoshy struct pmc_hw *pc_hwpmcs[AMD_NPMCS]; 824145256Sjkoshy struct pmc_hw pc_amdpmcs[AMD_NPMCS]; 825145256Sjkoshy}; 826145256Sjkoshy 827145256Sjkoshy 828145256Sjkoshystatic int 829145256Sjkoshyamd_init(int cpu) 830145256Sjkoshy{ 831145256Sjkoshy int n; 832145256Sjkoshy struct amd_cpu *pcs; 833145256Sjkoshy struct pmc_hw *phw; 834145256Sjkoshy 835145256Sjkoshy KASSERT(cpu >= 0 && cpu < mp_ncpus, 836145256Sjkoshy ("[amd,%d] insane cpu number %d", __LINE__, cpu)); 837145256Sjkoshy 838145256Sjkoshy PMCDBG(MDP,INI,1,"amd-init cpu=%d", cpu); 839145256Sjkoshy 840145256Sjkoshy MALLOC(pcs, struct amd_cpu *, sizeof(struct amd_cpu), M_PMC, 841145256Sjkoshy M_WAITOK|M_ZERO); 842145256Sjkoshy 843145256Sjkoshy phw = &pcs->pc_amdpmcs[0]; 844145256Sjkoshy 845145256Sjkoshy /* 846145256Sjkoshy * Initialize the per-cpu mutex and set the content of the 847145256Sjkoshy * hardware descriptors to a known state. 848145256Sjkoshy */ 849145256Sjkoshy 850145256Sjkoshy for (n = 0; n < AMD_NPMCS; n++, phw++) { 851145256Sjkoshy phw->phw_state = PMC_PHW_FLAG_IS_ENABLED | 852145256Sjkoshy PMC_PHW_CPU_TO_STATE(cpu) | PMC_PHW_INDEX_TO_STATE(n); 853145256Sjkoshy phw->phw_pmc = NULL; 854145256Sjkoshy pcs->pc_hwpmcs[n] = phw; 855145256Sjkoshy } 856145256Sjkoshy 857145256Sjkoshy /* Mark the TSC as shareable */ 858145256Sjkoshy pcs->pc_hwpmcs[0]->phw_state |= PMC_PHW_FLAG_IS_SHAREABLE; 859145256Sjkoshy 860145256Sjkoshy pmc_pcpu[cpu] = (struct pmc_cpu *) pcs; 861145256Sjkoshy 862145256Sjkoshy return 0; 863145256Sjkoshy} 864145256Sjkoshy 865145256Sjkoshy 866145256Sjkoshy/* 867145256Sjkoshy * processor dependent cleanup prior to the KLD 868145256Sjkoshy * being unloaded 869145256Sjkoshy */ 870145256Sjkoshy 871145256Sjkoshystatic int 872145256Sjkoshyamd_cleanup(int cpu) 873145256Sjkoshy{ 874145256Sjkoshy int i; 875145256Sjkoshy uint32_t evsel; 876145256Sjkoshy struct pmc_cpu *pcs; 877145256Sjkoshy 878145256Sjkoshy KASSERT(cpu >= 0 && cpu < mp_ncpus, 879145256Sjkoshy ("[amd,%d] insane cpu number (%d)", __LINE__, cpu)); 880145256Sjkoshy 881145256Sjkoshy PMCDBG(MDP,INI,1,"amd-cleanup cpu=%d", cpu); 882145256Sjkoshy 883145256Sjkoshy /* 884145256Sjkoshy * First, turn off all PMCs on this CPU. 885145256Sjkoshy */ 886145256Sjkoshy 887145256Sjkoshy for (i = 0; i < 4; i++) { /* XXX this loop is now not needed */ 888145256Sjkoshy evsel = rdmsr(AMD_PMC_EVSEL_0 + i); 889145256Sjkoshy evsel &= ~AMD_PMC_ENABLE; 890145256Sjkoshy wrmsr(AMD_PMC_EVSEL_0 + i, evsel); 891145256Sjkoshy } 892145256Sjkoshy 893145256Sjkoshy /* 894145256Sjkoshy * Next, free up allocated space. 895145256Sjkoshy */ 896145256Sjkoshy 897147191Sjkoshy if ((pcs = pmc_pcpu[cpu]) == NULL) 898147191Sjkoshy return 0; 899145256Sjkoshy 900153110Sru#ifdef DEBUG 901145256Sjkoshy /* check the TSC */ 902145256Sjkoshy KASSERT(pcs->pc_hwpmcs[0]->phw_pmc == NULL, 903145256Sjkoshy ("[amd,%d] CPU%d,PMC0 still in use", __LINE__, cpu)); 904145256Sjkoshy for (i = 1; i < AMD_NPMCS; i++) { 905145256Sjkoshy KASSERT(pcs->pc_hwpmcs[i]->phw_pmc == NULL, 906145256Sjkoshy ("[amd,%d] CPU%d/PMC%d in use", __LINE__, cpu, i)); 907145256Sjkoshy KASSERT(AMD_PMC_IS_STOPPED(AMD_PMC_EVSEL_0 + (i-1)), 908145256Sjkoshy ("[amd,%d] CPU%d/PMC%d not stopped", __LINE__, cpu, i)); 909145256Sjkoshy } 910145256Sjkoshy#endif 911145256Sjkoshy 912145256Sjkoshy pmc_pcpu[cpu] = NULL; 913145256Sjkoshy FREE(pcs, M_PMC); 914145256Sjkoshy return 0; 915145256Sjkoshy} 916145256Sjkoshy 917145256Sjkoshy/* 918145256Sjkoshy * Initialize ourselves. 919145256Sjkoshy */ 920145256Sjkoshy 921145256Sjkoshystruct pmc_mdep * 922145256Sjkoshypmc_amd_initialize(void) 923145256Sjkoshy{ 924147191Sjkoshy enum pmc_cputype cputype; 925147191Sjkoshy enum pmc_class class; 926145256Sjkoshy struct pmc_mdep *pmc_mdep; 927147191Sjkoshy char *name; 928147191Sjkoshy int i; 929145256Sjkoshy 930147191Sjkoshy /* 931147191Sjkoshy * The presence of hardware performance counters on the AMD 932147191Sjkoshy * Athlon, Duron or later processors, is _not_ indicated by 933147191Sjkoshy * any of the processor feature flags set by the 'CPUID' 934147191Sjkoshy * instruction, so we only check the 'instruction family' 935147191Sjkoshy * field returned by CPUID for instruction family >= 6. 936147191Sjkoshy */ 937145256Sjkoshy 938147510Sjkoshy class = cputype = -1; 939147510Sjkoshy name = NULL; 940147191Sjkoshy switch (cpu_id & 0xF00) { 941147191Sjkoshy case 0x600: /* Athlon(tm) processor */ 942147191Sjkoshy cputype = PMC_CPU_AMD_K7; 943147191Sjkoshy class = PMC_CLASS_K7; 944147191Sjkoshy name = "K7"; 945147191Sjkoshy break; 946147191Sjkoshy case 0xF00: /* Athlon64/Opteron processor */ 947147191Sjkoshy cputype = PMC_CPU_AMD_K8; 948147191Sjkoshy class = PMC_CLASS_K8; 949147191Sjkoshy name = "K8"; 950147191Sjkoshy break; 951147191Sjkoshy } 952147191Sjkoshy 953147191Sjkoshy if ((int) cputype == -1) { 954147191Sjkoshy (void) printf("pmc: Unknown AMD CPU.\n"); 955145256Sjkoshy return NULL; 956147191Sjkoshy } 957145256Sjkoshy 958153110Sru#ifdef DEBUG 959147191Sjkoshy amd_pmc_class = class; 960147191Sjkoshy#endif 961147191Sjkoshy 962145256Sjkoshy MALLOC(pmc_mdep, struct pmc_mdep *, sizeof(struct pmc_mdep), 963145256Sjkoshy M_PMC, M_WAITOK|M_ZERO); 964145256Sjkoshy 965147191Sjkoshy pmc_mdep->pmd_cputype = cputype; 966145256Sjkoshy pmc_mdep->pmd_npmc = AMD_NPMCS; 967145256Sjkoshy 968145256Sjkoshy /* this processor has two classes of usable PMCs */ 969145256Sjkoshy pmc_mdep->pmd_nclass = 2; 970145774Sjkoshy 971145774Sjkoshy /* TSC */ 972145774Sjkoshy pmc_mdep->pmd_classes[0].pm_class = PMC_CLASS_TSC; 973145774Sjkoshy pmc_mdep->pmd_classes[0].pm_caps = PMC_CAP_READ; 974145774Sjkoshy pmc_mdep->pmd_classes[0].pm_width = 64; 975145774Sjkoshy 976145774Sjkoshy /* AMD K7/K8 PMCs */ 977147191Sjkoshy pmc_mdep->pmd_classes[1].pm_class = class; 978145774Sjkoshy pmc_mdep->pmd_classes[1].pm_caps = AMD_PMC_CAPS; 979145774Sjkoshy pmc_mdep->pmd_classes[1].pm_width = 48; 980145774Sjkoshy 981145256Sjkoshy pmc_mdep->pmd_nclasspmcs[0] = 1; 982145256Sjkoshy pmc_mdep->pmd_nclasspmcs[1] = (AMD_NPMCS-1); 983145256Sjkoshy 984147191Sjkoshy /* fill in the correct pmc name and class */ 985147191Sjkoshy for (i = 1; i < AMD_NPMCS; i++) { 986147191Sjkoshy (void) snprintf(amd_pmcdesc[i].pm_descr.pd_name, 987147191Sjkoshy sizeof(amd_pmcdesc[i].pm_descr.pd_name), "%s-%d", 988147191Sjkoshy name, i-1); 989147191Sjkoshy amd_pmcdesc[i].pm_descr.pd_class = class; 990147191Sjkoshy } 991147191Sjkoshy 992145256Sjkoshy pmc_mdep->pmd_init = amd_init; 993145256Sjkoshy pmc_mdep->pmd_cleanup = amd_cleanup; 994145256Sjkoshy pmc_mdep->pmd_switch_in = amd_switch_in; 995145256Sjkoshy pmc_mdep->pmd_switch_out = amd_switch_out; 996145256Sjkoshy pmc_mdep->pmd_read_pmc = amd_read_pmc; 997145256Sjkoshy pmc_mdep->pmd_write_pmc = amd_write_pmc; 998145256Sjkoshy pmc_mdep->pmd_config_pmc = amd_config_pmc; 999145774Sjkoshy pmc_mdep->pmd_get_config = amd_get_config; 1000145256Sjkoshy pmc_mdep->pmd_allocate_pmc = amd_allocate_pmc; 1001145256Sjkoshy pmc_mdep->pmd_release_pmc = amd_release_pmc; 1002145256Sjkoshy pmc_mdep->pmd_start_pmc = amd_start_pmc; 1003145256Sjkoshy pmc_mdep->pmd_stop_pmc = amd_stop_pmc; 1004145256Sjkoshy pmc_mdep->pmd_intr = amd_intr; 1005145256Sjkoshy pmc_mdep->pmd_describe = amd_describe; 1006145256Sjkoshy pmc_mdep->pmd_get_msr = amd_get_msr; /* i386 */ 1007145256Sjkoshy 1008145256Sjkoshy PMCDBG(MDP,INI,0,"%s","amd-initialize"); 1009145256Sjkoshy 1010145256Sjkoshy return pmc_mdep; 1011145256Sjkoshy} 1012