hwpmc_amd.c revision 177343
1145256Sjkoshy/*- 2174395Sjkoshy * Copyright (c) 2003-2007 Joseph Koshy 3174395Sjkoshy * Copyright (c) 2007 The FreeBSD Foundation 4145256Sjkoshy * All rights reserved. 5145256Sjkoshy * 6174395Sjkoshy * Portions of this software were developed by A. Joseph Koshy under 7174395Sjkoshy * sponsorship from the FreeBSD Foundation and Google, Inc. 8174395Sjkoshy * 9145256Sjkoshy * Redistribution and use in source and binary forms, with or without 10145256Sjkoshy * modification, are permitted provided that the following conditions 11145256Sjkoshy * are met: 12145256Sjkoshy * 1. Redistributions of source code must retain the above copyright 13145256Sjkoshy * notice, this list of conditions and the following disclaimer. 14145256Sjkoshy * 2. Redistributions in binary form must reproduce the above copyright 15145256Sjkoshy * notice, this list of conditions and the following disclaimer in the 16145256Sjkoshy * documentation and/or other materials provided with the distribution. 17145256Sjkoshy * 18145256Sjkoshy * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 19145256Sjkoshy * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 20145256Sjkoshy * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 21145256Sjkoshy * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 22145256Sjkoshy * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 23145256Sjkoshy * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 24145256Sjkoshy * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 25145256Sjkoshy * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 26145256Sjkoshy * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 27145256Sjkoshy * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 28145256Sjkoshy * SUCH DAMAGE. 29145256Sjkoshy * 30145256Sjkoshy */ 31145256Sjkoshy 32145256Sjkoshy#include <sys/cdefs.h> 33145256Sjkoshy__FBSDID("$FreeBSD: head/sys/dev/hwpmc/hwpmc_amd.c 177343 2008-03-18 08:36:19Z adrian $"); 34145256Sjkoshy 35145256Sjkoshy/* Support for the AMD K7 and later processors */ 36145256Sjkoshy 37145256Sjkoshy#include <sys/param.h> 38145256Sjkoshy#include <sys/lock.h> 39145256Sjkoshy#include <sys/malloc.h> 40145256Sjkoshy#include <sys/mutex.h> 41145338Smarcel#include <sys/pmc.h> 42145256Sjkoshy#include <sys/smp.h> 43145256Sjkoshy#include <sys/systm.h> 44145256Sjkoshy 45174395Sjkoshy#include <machine/cpu.h> 46147191Sjkoshy#include <machine/cpufunc.h> 47145256Sjkoshy#include <machine/md_var.h> 48147191Sjkoshy#include <machine/specialreg.h> 49145256Sjkoshy 50153110Sru#ifdef DEBUG 51147191Sjkoshyenum pmc_class amd_pmc_class; 52145256Sjkoshy#endif 53145256Sjkoshy 54145256Sjkoshy/* AMD K7 & K8 PMCs */ 55145256Sjkoshystruct amd_descr { 56145256Sjkoshy struct pmc_descr pm_descr; /* "base class" */ 57145256Sjkoshy uint32_t pm_evsel; /* address of EVSEL register */ 58145256Sjkoshy uint32_t pm_perfctr; /* address of PERFCTR register */ 59145256Sjkoshy}; 60145256Sjkoshy 61147191Sjkoshystatic struct amd_descr amd_pmcdesc[AMD_NPMCS] = 62145256Sjkoshy{ 63145256Sjkoshy { 64145256Sjkoshy .pm_descr = 65145256Sjkoshy { 66145256Sjkoshy .pd_name = "TSC", 67145256Sjkoshy .pd_class = PMC_CLASS_TSC, 68145256Sjkoshy .pd_caps = PMC_CAP_READ, 69145256Sjkoshy .pd_width = 64 70145256Sjkoshy }, 71145256Sjkoshy .pm_evsel = MSR_TSC, 72145256Sjkoshy .pm_perfctr = 0 /* unused */ 73145256Sjkoshy }, 74145256Sjkoshy 75145256Sjkoshy { 76145256Sjkoshy .pm_descr = 77145256Sjkoshy { 78147191Sjkoshy .pd_name = "", 79147191Sjkoshy .pd_class = -1, 80145256Sjkoshy .pd_caps = AMD_PMC_CAPS, 81145256Sjkoshy .pd_width = 48 82145256Sjkoshy }, 83145256Sjkoshy .pm_evsel = AMD_PMC_EVSEL_0, 84145256Sjkoshy .pm_perfctr = AMD_PMC_PERFCTR_0 85145256Sjkoshy }, 86145256Sjkoshy { 87145256Sjkoshy .pm_descr = 88145256Sjkoshy { 89147191Sjkoshy .pd_name = "", 90147191Sjkoshy .pd_class = -1, 91145256Sjkoshy .pd_caps = AMD_PMC_CAPS, 92145256Sjkoshy .pd_width = 48 93145256Sjkoshy }, 94145256Sjkoshy .pm_evsel = AMD_PMC_EVSEL_1, 95145256Sjkoshy .pm_perfctr = AMD_PMC_PERFCTR_1 96145256Sjkoshy }, 97145256Sjkoshy { 98145256Sjkoshy .pm_descr = 99145256Sjkoshy { 100147191Sjkoshy .pd_name = "", 101147191Sjkoshy .pd_class = -1, 102145256Sjkoshy .pd_caps = AMD_PMC_CAPS, 103145256Sjkoshy .pd_width = 48 104145256Sjkoshy }, 105145256Sjkoshy .pm_evsel = AMD_PMC_EVSEL_2, 106145256Sjkoshy .pm_perfctr = AMD_PMC_PERFCTR_2 107145256Sjkoshy }, 108145256Sjkoshy { 109145256Sjkoshy .pm_descr = 110145256Sjkoshy { 111147191Sjkoshy .pd_name = "", 112147191Sjkoshy .pd_class = -1, 113145256Sjkoshy .pd_caps = AMD_PMC_CAPS, 114145256Sjkoshy .pd_width = 48 115145256Sjkoshy }, 116145256Sjkoshy .pm_evsel = AMD_PMC_EVSEL_3, 117145256Sjkoshy .pm_perfctr = AMD_PMC_PERFCTR_3 118145256Sjkoshy } 119145256Sjkoshy}; 120145256Sjkoshy 121145256Sjkoshystruct amd_event_code_map { 122145256Sjkoshy enum pmc_event pe_ev; /* enum value */ 123145256Sjkoshy uint8_t pe_code; /* encoded event mask */ 124145256Sjkoshy uint8_t pe_mask; /* bits allowed in unit mask */ 125145256Sjkoshy}; 126145256Sjkoshy 127145256Sjkoshyconst struct amd_event_code_map amd_event_codes[] = { 128147191Sjkoshy#if defined(__i386__) /* 32 bit Athlon (K7) only */ 129145256Sjkoshy { PMC_EV_K7_DC_ACCESSES, 0x40, 0 }, 130145256Sjkoshy { PMC_EV_K7_DC_MISSES, 0x41, 0 }, 131147191Sjkoshy { PMC_EV_K7_DC_REFILLS_FROM_L2, 0x42, AMD_PMC_UNITMASK_MOESI }, 132147191Sjkoshy { PMC_EV_K7_DC_REFILLS_FROM_SYSTEM, 0x43, AMD_PMC_UNITMASK_MOESI }, 133147191Sjkoshy { PMC_EV_K7_DC_WRITEBACKS, 0x44, AMD_PMC_UNITMASK_MOESI }, 134145256Sjkoshy { PMC_EV_K7_L1_DTLB_MISS_AND_L2_DTLB_HITS, 0x45, 0 }, 135145256Sjkoshy { PMC_EV_K7_L1_AND_L2_DTLB_MISSES, 0x46, 0 }, 136145256Sjkoshy { PMC_EV_K7_MISALIGNED_REFERENCES, 0x47, 0 }, 137145256Sjkoshy 138145256Sjkoshy { PMC_EV_K7_IC_FETCHES, 0x80, 0 }, 139145256Sjkoshy { PMC_EV_K7_IC_MISSES, 0x81, 0 }, 140145256Sjkoshy 141145256Sjkoshy { PMC_EV_K7_L1_ITLB_MISSES, 0x84, 0 }, 142145256Sjkoshy { PMC_EV_K7_L1_L2_ITLB_MISSES, 0x85, 0 }, 143145256Sjkoshy 144145256Sjkoshy { PMC_EV_K7_RETIRED_INSTRUCTIONS, 0xC0, 0 }, 145145256Sjkoshy { PMC_EV_K7_RETIRED_OPS, 0xC1, 0 }, 146145256Sjkoshy { PMC_EV_K7_RETIRED_BRANCHES, 0xC2, 0 }, 147145256Sjkoshy { PMC_EV_K7_RETIRED_BRANCHES_MISPREDICTED, 0xC3, 0 }, 148145256Sjkoshy { PMC_EV_K7_RETIRED_TAKEN_BRANCHES, 0xC4, 0 }, 149145256Sjkoshy { PMC_EV_K7_RETIRED_TAKEN_BRANCHES_MISPREDICTED, 0xC5, 0 }, 150145256Sjkoshy { PMC_EV_K7_RETIRED_FAR_CONTROL_TRANSFERS, 0xC6, 0 }, 151145256Sjkoshy { PMC_EV_K7_RETIRED_RESYNC_BRANCHES, 0xC7, 0 }, 152145256Sjkoshy { PMC_EV_K7_INTERRUPTS_MASKED_CYCLES, 0xCD, 0 }, 153145256Sjkoshy { PMC_EV_K7_INTERRUPTS_MASKED_WHILE_PENDING_CYCLES, 0xCE, 0 }, 154147191Sjkoshy { PMC_EV_K7_HARDWARE_INTERRUPTS, 0xCF, 0 }, 155145256Sjkoshy#endif 156145256Sjkoshy 157145256Sjkoshy { PMC_EV_K8_FP_DISPATCHED_FPU_OPS, 0x00, 0x3F }, 158145256Sjkoshy { PMC_EV_K8_FP_CYCLES_WITH_NO_FPU_OPS_RETIRED, 0x01, 0x00 }, 159145256Sjkoshy { PMC_EV_K8_FP_DISPATCHED_FPU_FAST_FLAG_OPS, 0x02, 0x00 }, 160145256Sjkoshy 161145256Sjkoshy { PMC_EV_K8_LS_SEGMENT_REGISTER_LOAD, 0x20, 0x7F }, 162145256Sjkoshy { PMC_EV_K8_LS_MICROARCHITECTURAL_RESYNC_BY_SELF_MODIFYING_CODE, 163145256Sjkoshy 0x21, 0x00 }, 164145256Sjkoshy { PMC_EV_K8_LS_MICROARCHITECTURAL_RESYNC_BY_SNOOP, 0x22, 0x00 }, 165145256Sjkoshy { PMC_EV_K8_LS_BUFFER2_FULL, 0x23, 0x00 }, 166145256Sjkoshy { PMC_EV_K8_LS_LOCKED_OPERATION, 0x24, 0x07 }, 167145256Sjkoshy { PMC_EV_K8_LS_MICROARCHITECTURAL_LATE_CANCEL, 0x25, 0x00 }, 168145256Sjkoshy { PMC_EV_K8_LS_RETIRED_CFLUSH_INSTRUCTIONS, 0x26, 0x00 }, 169145256Sjkoshy { PMC_EV_K8_LS_RETIRED_CPUID_INSTRUCTIONS, 0x27, 0x00 }, 170145256Sjkoshy 171145256Sjkoshy { PMC_EV_K8_DC_ACCESS, 0x40, 0x00 }, 172145256Sjkoshy { PMC_EV_K8_DC_MISS, 0x41, 0x00 }, 173145256Sjkoshy { PMC_EV_K8_DC_REFILL_FROM_L2, 0x42, 0x1F }, 174145256Sjkoshy { PMC_EV_K8_DC_REFILL_FROM_SYSTEM, 0x43, 0x1F }, 175145256Sjkoshy { PMC_EV_K8_DC_COPYBACK, 0x44, 0x1F }, 176145256Sjkoshy { PMC_EV_K8_DC_L1_DTLB_MISS_AND_L2_DTLB_HIT, 0x45, 0x00 }, 177145256Sjkoshy { PMC_EV_K8_DC_L1_DTLB_MISS_AND_L2_DTLB_MISS, 0x46, 0x00 }, 178145256Sjkoshy { PMC_EV_K8_DC_MISALIGNED_DATA_REFERENCE, 0x47, 0x00 }, 179145256Sjkoshy { PMC_EV_K8_DC_MICROARCHITECTURAL_LATE_CANCEL, 0x48, 0x00 }, 180145256Sjkoshy { PMC_EV_K8_DC_MICROARCHITECTURAL_EARLY_CANCEL, 0x49, 0x00 }, 181145256Sjkoshy { PMC_EV_K8_DC_ONE_BIT_ECC_ERROR, 0x4A, 0x03 }, 182145256Sjkoshy { PMC_EV_K8_DC_DISPATCHED_PREFETCH_INSTRUCTIONS, 0x4B, 0x07 }, 183145256Sjkoshy { PMC_EV_K8_DC_DCACHE_ACCESSES_BY_LOCKS, 0x4C, 0x03 }, 184145256Sjkoshy 185145256Sjkoshy { PMC_EV_K8_BU_CPU_CLK_UNHALTED, 0x76, 0x00 }, 186145256Sjkoshy { PMC_EV_K8_BU_INTERNAL_L2_REQUEST, 0x7D, 0x1F }, 187145256Sjkoshy { PMC_EV_K8_BU_FILL_REQUEST_L2_MISS, 0x7E, 0x07 }, 188145256Sjkoshy { PMC_EV_K8_BU_FILL_INTO_L2, 0x7F, 0x03 }, 189145256Sjkoshy 190145256Sjkoshy { PMC_EV_K8_IC_FETCH, 0x80, 0x00 }, 191145256Sjkoshy { PMC_EV_K8_IC_MISS, 0x81, 0x00 }, 192145256Sjkoshy { PMC_EV_K8_IC_REFILL_FROM_L2, 0x82, 0x00 }, 193145256Sjkoshy { PMC_EV_K8_IC_REFILL_FROM_SYSTEM, 0x83, 0x00 }, 194145256Sjkoshy { PMC_EV_K8_IC_L1_ITLB_MISS_AND_L2_ITLB_HIT, 0x84, 0x00 }, 195145256Sjkoshy { PMC_EV_K8_IC_L1_ITLB_MISS_AND_L2_ITLB_MISS, 0x85, 0x00 }, 196145256Sjkoshy { PMC_EV_K8_IC_MICROARCHITECTURAL_RESYNC_BY_SNOOP, 0x86, 0x00 }, 197145256Sjkoshy { PMC_EV_K8_IC_INSTRUCTION_FETCH_STALL, 0x87, 0x00 }, 198145256Sjkoshy { PMC_EV_K8_IC_RETURN_STACK_HIT, 0x88, 0x00 }, 199145256Sjkoshy { PMC_EV_K8_IC_RETURN_STACK_OVERFLOW, 0x89, 0x00 }, 200145256Sjkoshy 201145256Sjkoshy { PMC_EV_K8_FR_RETIRED_X86_INSTRUCTIONS, 0xC0, 0x00 }, 202145256Sjkoshy { PMC_EV_K8_FR_RETIRED_UOPS, 0xC1, 0x00 }, 203145256Sjkoshy { PMC_EV_K8_FR_RETIRED_BRANCHES, 0xC2, 0x00 }, 204145256Sjkoshy { PMC_EV_K8_FR_RETIRED_BRANCHES_MISPREDICTED, 0xC3, 0x00 }, 205145256Sjkoshy { PMC_EV_K8_FR_RETIRED_TAKEN_BRANCHES, 0xC4, 0x00 }, 206145256Sjkoshy { PMC_EV_K8_FR_RETIRED_TAKEN_BRANCHES_MISPREDICTED, 0xC5, 0x00 }, 207145256Sjkoshy { PMC_EV_K8_FR_RETIRED_FAR_CONTROL_TRANSFERS, 0xC6, 0x00 }, 208145256Sjkoshy { PMC_EV_K8_FR_RETIRED_RESYNCS, 0xC7, 0x00 }, 209145256Sjkoshy { PMC_EV_K8_FR_RETIRED_NEAR_RETURNS, 0xC8, 0x00 }, 210145256Sjkoshy { PMC_EV_K8_FR_RETIRED_NEAR_RETURNS_MISPREDICTED, 0xC9, 0x00 }, 211145256Sjkoshy { PMC_EV_K8_FR_RETIRED_TAKEN_BRANCHES_MISPREDICTED_BY_ADDR_MISCOMPARE, 212145256Sjkoshy 0xCA, 0x00 }, 213145256Sjkoshy { PMC_EV_K8_FR_RETIRED_FPU_INSTRUCTIONS, 0xCB, 0x0F }, 214145256Sjkoshy { PMC_EV_K8_FR_RETIRED_FASTPATH_DOUBLE_OP_INSTRUCTIONS, 215145256Sjkoshy 0xCC, 0x07 }, 216145256Sjkoshy { PMC_EV_K8_FR_INTERRUPTS_MASKED_CYCLES, 0xCD, 0x00 }, 217145256Sjkoshy { PMC_EV_K8_FR_INTERRUPTS_MASKED_WHILE_PENDING_CYCLES, 0xCE, 0x00 }, 218145256Sjkoshy { PMC_EV_K8_FR_TAKEN_HARDWARE_INTERRUPTS, 0xCF, 0x00 }, 219145256Sjkoshy 220145256Sjkoshy { PMC_EV_K8_FR_DECODER_EMPTY, 0xD0, 0x00 }, 221145256Sjkoshy { PMC_EV_K8_FR_DISPATCH_STALLS, 0xD1, 0x00 }, 222145256Sjkoshy { PMC_EV_K8_FR_DISPATCH_STALL_FROM_BRANCH_ABORT_TO_RETIRE, 223145256Sjkoshy 0xD2, 0x00 }, 224145256Sjkoshy { PMC_EV_K8_FR_DISPATCH_STALL_FOR_SERIALIZATION, 0xD3, 0x00 }, 225145256Sjkoshy { PMC_EV_K8_FR_DISPATCH_STALL_FOR_SEGMENT_LOAD, 0xD4, 0x00 }, 226145256Sjkoshy { PMC_EV_K8_FR_DISPATCH_STALL_WHEN_REORDER_BUFFER_IS_FULL, 227145256Sjkoshy 0xD5, 0x00 }, 228145256Sjkoshy { PMC_EV_K8_FR_DISPATCH_STALL_WHEN_RESERVATION_STATIONS_ARE_FULL, 229145256Sjkoshy 0xD6, 0x00 }, 230145256Sjkoshy { PMC_EV_K8_FR_DISPATCH_STALL_WHEN_FPU_IS_FULL, 0xD7, 0x00 }, 231145256Sjkoshy { PMC_EV_K8_FR_DISPATCH_STALL_WHEN_LS_IS_FULL, 0xD8, 0x00 }, 232145256Sjkoshy { PMC_EV_K8_FR_DISPATCH_STALL_WHEN_WAITING_FOR_ALL_TO_BE_QUIET, 233145256Sjkoshy 0xD9, 0x00 }, 234145256Sjkoshy { PMC_EV_K8_FR_DISPATCH_STALL_WHEN_FAR_XFER_OR_RESYNC_BRANCH_PENDING, 235145256Sjkoshy 0xDA, 0x00 }, 236145256Sjkoshy { PMC_EV_K8_FR_FPU_EXCEPTIONS, 0xDB, 0x0F }, 237145256Sjkoshy { PMC_EV_K8_FR_NUMBER_OF_BREAKPOINTS_FOR_DR0, 0xDC, 0x00 }, 238145256Sjkoshy { PMC_EV_K8_FR_NUMBER_OF_BREAKPOINTS_FOR_DR1, 0xDD, 0x00 }, 239145256Sjkoshy { PMC_EV_K8_FR_NUMBER_OF_BREAKPOINTS_FOR_DR2, 0xDE, 0x00 }, 240145256Sjkoshy { PMC_EV_K8_FR_NUMBER_OF_BREAKPOINTS_FOR_DR3, 0xDF, 0x00 }, 241145256Sjkoshy 242145256Sjkoshy { PMC_EV_K8_NB_MEMORY_CONTROLLER_PAGE_ACCESS_EVENT, 0xE0, 0x7 }, 243145256Sjkoshy { PMC_EV_K8_NB_MEMORY_CONTROLLER_PAGE_TABLE_OVERFLOW, 0xE1, 0x00 }, 244145256Sjkoshy { PMC_EV_K8_NB_MEMORY_CONTROLLER_DRAM_COMMAND_SLOTS_MISSED, 245145256Sjkoshy 0xE2, 0x00 }, 246145256Sjkoshy { PMC_EV_K8_NB_MEMORY_CONTROLLER_TURNAROUND, 0xE3, 0x07 }, 247145256Sjkoshy { PMC_EV_K8_NB_MEMORY_CONTROLLER_BYPASS_SATURATION, 0xE4, 0x0F }, 248145256Sjkoshy { PMC_EV_K8_NB_SIZED_COMMANDS, 0xEB, 0x7F }, 249145256Sjkoshy { PMC_EV_K8_NB_PROBE_RESULT, 0xEC, 0x0F }, 250145256Sjkoshy { PMC_EV_K8_NB_HT_BUS0_BANDWIDTH, 0xF6, 0x0F }, 251145256Sjkoshy { PMC_EV_K8_NB_HT_BUS1_BANDWIDTH, 0xF7, 0x0F }, 252145256Sjkoshy { PMC_EV_K8_NB_HT_BUS2_BANDWIDTH, 0xF8, 0x0F } 253145256Sjkoshy 254145256Sjkoshy}; 255145256Sjkoshy 256145256Sjkoshyconst int amd_event_codes_size = 257145256Sjkoshy sizeof(amd_event_codes) / sizeof(amd_event_codes[0]); 258145256Sjkoshy 259145256Sjkoshy/* 260145256Sjkoshy * read a pmc register 261145256Sjkoshy */ 262145256Sjkoshy 263145256Sjkoshystatic int 264145256Sjkoshyamd_read_pmc(int cpu, int ri, pmc_value_t *v) 265145256Sjkoshy{ 266145256Sjkoshy enum pmc_mode mode; 267145256Sjkoshy const struct amd_descr *pd; 268145256Sjkoshy struct pmc *pm; 269145256Sjkoshy const struct pmc_hw *phw; 270145256Sjkoshy pmc_value_t tmp; 271145256Sjkoshy 272145256Sjkoshy KASSERT(cpu >= 0 && cpu < mp_ncpus, 273145256Sjkoshy ("[amd,%d] illegal CPU value %d", __LINE__, cpu)); 274145256Sjkoshy KASSERT(ri >= 0 && ri < AMD_NPMCS, 275145256Sjkoshy ("[amd,%d] illegal row-index %d", __LINE__, ri)); 276145256Sjkoshy 277145256Sjkoshy phw = pmc_pcpu[cpu]->pc_hwpmcs[ri]; 278145256Sjkoshy pd = &amd_pmcdesc[ri]; 279145256Sjkoshy pm = phw->phw_pmc; 280145256Sjkoshy 281145256Sjkoshy KASSERT(pm != NULL, 282145256Sjkoshy ("[amd,%d] No owner for HWPMC [cpu%d,pmc%d]", __LINE__, 283145256Sjkoshy cpu, ri)); 284145256Sjkoshy 285145774Sjkoshy mode = PMC_TO_MODE(pm); 286145256Sjkoshy 287145256Sjkoshy PMCDBG(MDP,REA,1,"amd-read id=%d class=%d", ri, pd->pm_descr.pd_class); 288145256Sjkoshy 289145256Sjkoshy /* Reading the TSC is a special case */ 290145256Sjkoshy if (pd->pm_descr.pd_class == PMC_CLASS_TSC) { 291145256Sjkoshy KASSERT(PMC_IS_COUNTING_MODE(mode), 292145256Sjkoshy ("[amd,%d] TSC counter in non-counting mode", __LINE__)); 293145256Sjkoshy *v = rdtsc(); 294145256Sjkoshy PMCDBG(MDP,REA,2,"amd-read id=%d -> %jd", ri, *v); 295145256Sjkoshy return 0; 296145256Sjkoshy } 297145256Sjkoshy 298153110Sru#ifdef DEBUG 299147191Sjkoshy KASSERT(pd->pm_descr.pd_class == amd_pmc_class, 300145256Sjkoshy ("[amd,%d] unknown PMC class (%d)", __LINE__, 301145256Sjkoshy pd->pm_descr.pd_class)); 302147191Sjkoshy#endif 303145256Sjkoshy 304145256Sjkoshy tmp = rdmsr(pd->pm_perfctr); /* RDMSR serializes */ 305145256Sjkoshy if (PMC_IS_SAMPLING_MODE(mode)) 306147191Sjkoshy *v = AMD_PERFCTR_VALUE_TO_RELOAD_COUNT(tmp); 307145256Sjkoshy else 308145256Sjkoshy *v = tmp; 309145256Sjkoshy 310145256Sjkoshy PMCDBG(MDP,REA,2,"amd-read id=%d -> %jd", ri, *v); 311145256Sjkoshy 312145256Sjkoshy return 0; 313145256Sjkoshy} 314145256Sjkoshy 315145256Sjkoshy/* 316145256Sjkoshy * Write a PMC MSR. 317145256Sjkoshy */ 318145256Sjkoshy 319145256Sjkoshystatic int 320145256Sjkoshyamd_write_pmc(int cpu, int ri, pmc_value_t v) 321145256Sjkoshy{ 322145256Sjkoshy const struct amd_descr *pd; 323145256Sjkoshy struct pmc *pm; 324145256Sjkoshy const struct pmc_hw *phw; 325145256Sjkoshy enum pmc_mode mode; 326145256Sjkoshy 327145256Sjkoshy KASSERT(cpu >= 0 && cpu < mp_ncpus, 328145256Sjkoshy ("[amd,%d] illegal CPU value %d", __LINE__, cpu)); 329145256Sjkoshy KASSERT(ri >= 0 && ri < AMD_NPMCS, 330145256Sjkoshy ("[amd,%d] illegal row-index %d", __LINE__, ri)); 331145256Sjkoshy 332145256Sjkoshy phw = pmc_pcpu[cpu]->pc_hwpmcs[ri]; 333145256Sjkoshy pd = &amd_pmcdesc[ri]; 334145256Sjkoshy pm = phw->phw_pmc; 335145256Sjkoshy 336145256Sjkoshy KASSERT(pm != NULL, 337145256Sjkoshy ("[amd,%d] PMC not owned (cpu%d,pmc%d)", __LINE__, 338145256Sjkoshy cpu, ri)); 339145256Sjkoshy 340145774Sjkoshy mode = PMC_TO_MODE(pm); 341145256Sjkoshy 342145256Sjkoshy if (pd->pm_descr.pd_class == PMC_CLASS_TSC) 343145256Sjkoshy return 0; 344145256Sjkoshy 345153110Sru#ifdef DEBUG 346147191Sjkoshy KASSERT(pd->pm_descr.pd_class == amd_pmc_class, 347145256Sjkoshy ("[amd,%d] unknown PMC class (%d)", __LINE__, 348145256Sjkoshy pd->pm_descr.pd_class)); 349147191Sjkoshy#endif 350145256Sjkoshy 351145256Sjkoshy /* use 2's complement of the count for sampling mode PMCs */ 352145256Sjkoshy if (PMC_IS_SAMPLING_MODE(mode)) 353147191Sjkoshy v = AMD_RELOAD_COUNT_TO_PERFCTR_VALUE(v); 354145256Sjkoshy 355145256Sjkoshy PMCDBG(MDP,WRI,1,"amd-write cpu=%d ri=%d v=%jx", cpu, ri, v); 356145256Sjkoshy 357145256Sjkoshy /* write the PMC value */ 358145256Sjkoshy wrmsr(pd->pm_perfctr, v); 359145256Sjkoshy return 0; 360145256Sjkoshy} 361145256Sjkoshy 362145256Sjkoshy/* 363145256Sjkoshy * configure hardware pmc according to the configuration recorded in 364145256Sjkoshy * pmc 'pm'. 365145256Sjkoshy */ 366145256Sjkoshy 367145256Sjkoshystatic int 368145256Sjkoshyamd_config_pmc(int cpu, int ri, struct pmc *pm) 369145256Sjkoshy{ 370145256Sjkoshy struct pmc_hw *phw; 371145256Sjkoshy 372145615Sjkoshy PMCDBG(MDP,CFG,1, "cpu=%d ri=%d pm=%p", cpu, ri, pm); 373145615Sjkoshy 374145256Sjkoshy KASSERT(cpu >= 0 && cpu < mp_ncpus, 375145256Sjkoshy ("[amd,%d] illegal CPU value %d", __LINE__, cpu)); 376145256Sjkoshy KASSERT(ri >= 0 && ri < AMD_NPMCS, 377145256Sjkoshy ("[amd,%d] illegal row-index %d", __LINE__, ri)); 378145256Sjkoshy 379145256Sjkoshy phw = pmc_pcpu[cpu]->pc_hwpmcs[ri]; 380145256Sjkoshy 381145256Sjkoshy KASSERT(pm == NULL || phw->phw_pmc == NULL, 382145615Sjkoshy ("[amd,%d] pm=%p phw->pm=%p hwpmc not unconfigured", 383145615Sjkoshy __LINE__, pm, phw->phw_pmc)); 384145256Sjkoshy 385145256Sjkoshy phw->phw_pmc = pm; 386145256Sjkoshy return 0; 387145256Sjkoshy} 388145256Sjkoshy 389145256Sjkoshy/* 390145774Sjkoshy * Retrieve a configured PMC pointer from hardware state. 391145774Sjkoshy */ 392145774Sjkoshy 393145774Sjkoshystatic int 394145774Sjkoshyamd_get_config(int cpu, int ri, struct pmc **ppm) 395145774Sjkoshy{ 396145774Sjkoshy *ppm = pmc_pcpu[cpu]->pc_hwpmcs[ri]->phw_pmc; 397145774Sjkoshy 398145774Sjkoshy return 0; 399145774Sjkoshy} 400145774Sjkoshy 401145774Sjkoshy/* 402145256Sjkoshy * Machine dependent actions taken during the context switch in of a 403145256Sjkoshy * thread. 404145256Sjkoshy */ 405145256Sjkoshy 406145256Sjkoshystatic int 407145615Sjkoshyamd_switch_in(struct pmc_cpu *pc, struct pmc_process *pp) 408145256Sjkoshy{ 409145256Sjkoshy (void) pc; 410145256Sjkoshy 411145615Sjkoshy PMCDBG(MDP,SWI,1, "pc=%p pp=%p enable-msr=%d", pc, pp, 412145774Sjkoshy (pp->pp_flags & PMC_PP_ENABLE_MSR_ACCESS) != 0); 413145615Sjkoshy 414145615Sjkoshy /* enable the RDPMC instruction if needed */ 415145774Sjkoshy if (pp->pp_flags & PMC_PP_ENABLE_MSR_ACCESS) 416145615Sjkoshy load_cr4(rcr4() | CR4_PCE); 417145615Sjkoshy 418145256Sjkoshy return 0; 419145256Sjkoshy} 420145256Sjkoshy 421145256Sjkoshy/* 422145256Sjkoshy * Machine dependent actions taken during the context switch out of a 423145256Sjkoshy * thread. 424145256Sjkoshy */ 425145256Sjkoshy 426145256Sjkoshystatic int 427145615Sjkoshyamd_switch_out(struct pmc_cpu *pc, struct pmc_process *pp) 428145256Sjkoshy{ 429145256Sjkoshy (void) pc; 430145615Sjkoshy (void) pp; /* can be NULL */ 431145256Sjkoshy 432145615Sjkoshy PMCDBG(MDP,SWO,1, "pc=%p pp=%p enable-msr=%d", pc, pp, pp ? 433145774Sjkoshy (pp->pp_flags & PMC_PP_ENABLE_MSR_ACCESS) == 1 : 0); 434145615Sjkoshy 435145615Sjkoshy /* always turn off the RDPMC instruction */ 436145256Sjkoshy load_cr4(rcr4() & ~CR4_PCE); 437145615Sjkoshy 438145256Sjkoshy return 0; 439145256Sjkoshy} 440145256Sjkoshy 441145256Sjkoshy/* 442145256Sjkoshy * Check if a given allocation is feasible. 443145256Sjkoshy */ 444145256Sjkoshy 445145256Sjkoshystatic int 446145256Sjkoshyamd_allocate_pmc(int cpu, int ri, struct pmc *pm, 447145256Sjkoshy const struct pmc_op_pmcallocate *a) 448145256Sjkoshy{ 449145256Sjkoshy int i; 450145256Sjkoshy uint32_t allowed_unitmask, caps, config, unitmask; 451145256Sjkoshy enum pmc_event pe; 452145256Sjkoshy const struct pmc_descr *pd; 453145256Sjkoshy 454145256Sjkoshy (void) cpu; 455145256Sjkoshy 456145256Sjkoshy KASSERT(cpu >= 0 && cpu < mp_ncpus, 457145256Sjkoshy ("[amd,%d] illegal CPU value %d", __LINE__, cpu)); 458145256Sjkoshy KASSERT(ri >= 0 && ri < AMD_NPMCS, 459145256Sjkoshy ("[amd,%d] illegal row index %d", __LINE__, ri)); 460145256Sjkoshy 461145256Sjkoshy pd = &amd_pmcdesc[ri].pm_descr; 462145256Sjkoshy 463145256Sjkoshy /* check class match */ 464145774Sjkoshy if (pd->pd_class != a->pm_class) 465145256Sjkoshy return EINVAL; 466145256Sjkoshy 467145256Sjkoshy caps = pm->pm_caps; 468145256Sjkoshy 469145256Sjkoshy PMCDBG(MDP,ALL,1,"amd-allocate ri=%d caps=0x%x", ri, caps); 470145256Sjkoshy 471145256Sjkoshy if ((pd->pd_caps & caps) != caps) 472145256Sjkoshy return EPERM; 473145256Sjkoshy if (pd->pd_class == PMC_CLASS_TSC) { 474145256Sjkoshy /* TSC's are always allocated in system-wide counting mode */ 475145256Sjkoshy if (a->pm_ev != PMC_EV_TSC_TSC || 476145256Sjkoshy a->pm_mode != PMC_MODE_SC) 477145256Sjkoshy return EINVAL; 478145256Sjkoshy return 0; 479145256Sjkoshy } 480145256Sjkoshy 481153110Sru#ifdef DEBUG 482147191Sjkoshy KASSERT(pd->pd_class == amd_pmc_class, 483145256Sjkoshy ("[amd,%d] Unknown PMC class (%d)", __LINE__, pd->pd_class)); 484147191Sjkoshy#endif 485145256Sjkoshy 486145256Sjkoshy pe = a->pm_ev; 487145256Sjkoshy 488145256Sjkoshy /* map ev to the correct event mask code */ 489145256Sjkoshy config = allowed_unitmask = 0; 490145256Sjkoshy for (i = 0; i < amd_event_codes_size; i++) 491145256Sjkoshy if (amd_event_codes[i].pe_ev == pe) { 492145256Sjkoshy config = 493145256Sjkoshy AMD_PMC_TO_EVENTMASK(amd_event_codes[i].pe_code); 494145256Sjkoshy allowed_unitmask = 495145256Sjkoshy AMD_PMC_TO_UNITMASK(amd_event_codes[i].pe_mask); 496145256Sjkoshy break; 497145256Sjkoshy } 498145256Sjkoshy if (i == amd_event_codes_size) 499145256Sjkoshy return EINVAL; 500145256Sjkoshy 501147191Sjkoshy unitmask = a->pm_md.pm_amd.pm_amd_config & AMD_PMC_UNITMASK; 502145256Sjkoshy if (unitmask & ~allowed_unitmask) /* disallow reserved bits */ 503145256Sjkoshy return EINVAL; 504145256Sjkoshy 505145256Sjkoshy if (unitmask && (caps & PMC_CAP_QUALIFIER)) 506145256Sjkoshy config |= unitmask; 507145256Sjkoshy 508145256Sjkoshy if (caps & PMC_CAP_THRESHOLD) 509147191Sjkoshy config |= a->pm_md.pm_amd.pm_amd_config & AMD_PMC_COUNTERMASK; 510145256Sjkoshy 511145256Sjkoshy /* set at least one of the 'usr' or 'os' caps */ 512145256Sjkoshy if (caps & PMC_CAP_USER) 513145256Sjkoshy config |= AMD_PMC_USR; 514145256Sjkoshy if (caps & PMC_CAP_SYSTEM) 515145256Sjkoshy config |= AMD_PMC_OS; 516145256Sjkoshy if ((caps & (PMC_CAP_USER|PMC_CAP_SYSTEM)) == 0) 517145256Sjkoshy config |= (AMD_PMC_USR|AMD_PMC_OS); 518145256Sjkoshy 519145256Sjkoshy if (caps & PMC_CAP_EDGE) 520145256Sjkoshy config |= AMD_PMC_EDGE; 521145256Sjkoshy if (caps & PMC_CAP_INVERT) 522145256Sjkoshy config |= AMD_PMC_INVERT; 523145256Sjkoshy if (caps & PMC_CAP_INTERRUPT) 524145256Sjkoshy config |= AMD_PMC_INT; 525145256Sjkoshy 526145256Sjkoshy pm->pm_md.pm_amd.pm_amd_evsel = config; /* save config value */ 527145256Sjkoshy 528145256Sjkoshy PMCDBG(MDP,ALL,2,"amd-allocate ri=%d -> config=0x%x", ri, config); 529145256Sjkoshy 530145256Sjkoshy return 0; 531145256Sjkoshy} 532145256Sjkoshy 533145256Sjkoshy/* 534145256Sjkoshy * Release machine dependent state associated with a PMC. This is a 535145256Sjkoshy * no-op on this architecture. 536145256Sjkoshy * 537145256Sjkoshy */ 538145256Sjkoshy 539145256Sjkoshy/* ARGSUSED0 */ 540145256Sjkoshystatic int 541145256Sjkoshyamd_release_pmc(int cpu, int ri, struct pmc *pmc) 542145256Sjkoshy{ 543153110Sru#ifdef DEBUG 544145256Sjkoshy const struct amd_descr *pd; 545145256Sjkoshy#endif 546145256Sjkoshy struct pmc_hw *phw; 547145256Sjkoshy 548145256Sjkoshy (void) pmc; 549145256Sjkoshy 550145256Sjkoshy KASSERT(cpu >= 0 && cpu < mp_ncpus, 551145256Sjkoshy ("[amd,%d] illegal CPU value %d", __LINE__, cpu)); 552145256Sjkoshy KASSERT(ri >= 0 && ri < AMD_NPMCS, 553145256Sjkoshy ("[amd,%d] illegal row-index %d", __LINE__, ri)); 554145256Sjkoshy 555145256Sjkoshy phw = pmc_pcpu[cpu]->pc_hwpmcs[ri]; 556145256Sjkoshy 557145256Sjkoshy KASSERT(phw->phw_pmc == NULL, 558145256Sjkoshy ("[amd,%d] PHW pmc %p non-NULL", __LINE__, phw->phw_pmc)); 559145256Sjkoshy 560153110Sru#ifdef DEBUG 561145256Sjkoshy pd = &amd_pmcdesc[ri]; 562147191Sjkoshy if (pd->pm_descr.pd_class == amd_pmc_class) 563145256Sjkoshy KASSERT(AMD_PMC_IS_STOPPED(pd->pm_evsel), 564145256Sjkoshy ("[amd,%d] PMC %d released while active", __LINE__, ri)); 565145256Sjkoshy#endif 566145256Sjkoshy 567145256Sjkoshy return 0; 568145256Sjkoshy} 569145256Sjkoshy 570145256Sjkoshy/* 571145256Sjkoshy * start a PMC. 572145256Sjkoshy */ 573145256Sjkoshy 574145256Sjkoshystatic int 575145256Sjkoshyamd_start_pmc(int cpu, int ri) 576145256Sjkoshy{ 577145256Sjkoshy uint32_t config; 578145256Sjkoshy struct pmc *pm; 579145256Sjkoshy struct pmc_hw *phw; 580145256Sjkoshy const struct amd_descr *pd; 581145256Sjkoshy 582145256Sjkoshy KASSERT(cpu >= 0 && cpu < mp_ncpus, 583145256Sjkoshy ("[amd,%d] illegal CPU value %d", __LINE__, cpu)); 584145256Sjkoshy KASSERT(ri >= 0 && ri < AMD_NPMCS, 585145256Sjkoshy ("[amd,%d] illegal row-index %d", __LINE__, ri)); 586145256Sjkoshy 587145256Sjkoshy phw = pmc_pcpu[cpu]->pc_hwpmcs[ri]; 588145256Sjkoshy pm = phw->phw_pmc; 589145256Sjkoshy pd = &amd_pmcdesc[ri]; 590145256Sjkoshy 591145256Sjkoshy KASSERT(pm != NULL, 592145256Sjkoshy ("[amd,%d] starting cpu%d,pmc%d with null pmc record", __LINE__, 593145256Sjkoshy cpu, ri)); 594145256Sjkoshy 595145256Sjkoshy PMCDBG(MDP,STA,1,"amd-start cpu=%d ri=%d", cpu, ri); 596145256Sjkoshy 597145256Sjkoshy if (pd->pm_descr.pd_class == PMC_CLASS_TSC) 598145256Sjkoshy return 0; /* TSCs are always running */ 599145256Sjkoshy 600153110Sru#ifdef DEBUG 601147191Sjkoshy KASSERT(pd->pm_descr.pd_class == amd_pmc_class, 602145256Sjkoshy ("[amd,%d] unknown PMC class (%d)", __LINE__, 603145256Sjkoshy pd->pm_descr.pd_class)); 604147191Sjkoshy#endif 605145256Sjkoshy 606145256Sjkoshy KASSERT(AMD_PMC_IS_STOPPED(pd->pm_evsel), 607145256Sjkoshy ("[amd,%d] pmc%d,cpu%d: Starting active PMC \"%s\"", __LINE__, 608145256Sjkoshy ri, cpu, pd->pm_descr.pd_name)); 609145256Sjkoshy 610145256Sjkoshy /* turn on the PMC ENABLE bit */ 611145256Sjkoshy config = pm->pm_md.pm_amd.pm_amd_evsel | AMD_PMC_ENABLE; 612145256Sjkoshy 613145256Sjkoshy PMCDBG(MDP,STA,2,"amd-start config=0x%x", config); 614145256Sjkoshy 615145256Sjkoshy wrmsr(pd->pm_evsel, config); 616145256Sjkoshy return 0; 617145256Sjkoshy} 618145256Sjkoshy 619145256Sjkoshy/* 620145256Sjkoshy * Stop a PMC. 621145256Sjkoshy */ 622145256Sjkoshy 623145256Sjkoshystatic int 624145256Sjkoshyamd_stop_pmc(int cpu, int ri) 625145256Sjkoshy{ 626145256Sjkoshy struct pmc *pm; 627145256Sjkoshy struct pmc_hw *phw; 628145256Sjkoshy const struct amd_descr *pd; 629145256Sjkoshy uint64_t config; 630145256Sjkoshy 631145256Sjkoshy KASSERT(cpu >= 0 && cpu < mp_ncpus, 632145256Sjkoshy ("[amd,%d] illegal CPU value %d", __LINE__, cpu)); 633145256Sjkoshy KASSERT(ri >= 0 && ri < AMD_NPMCS, 634145256Sjkoshy ("[amd,%d] illegal row-index %d", __LINE__, ri)); 635145256Sjkoshy 636145256Sjkoshy phw = pmc_pcpu[cpu]->pc_hwpmcs[ri]; 637145256Sjkoshy pm = phw->phw_pmc; 638145256Sjkoshy pd = &amd_pmcdesc[ri]; 639145256Sjkoshy 640145256Sjkoshy KASSERT(pm != NULL, 641145256Sjkoshy ("[amd,%d] cpu%d,pmc%d no PMC to stop", __LINE__, 642145256Sjkoshy cpu, ri)); 643145256Sjkoshy 644145256Sjkoshy /* can't stop a TSC */ 645145256Sjkoshy if (pd->pm_descr.pd_class == PMC_CLASS_TSC) 646145256Sjkoshy return 0; 647145256Sjkoshy 648153110Sru#ifdef DEBUG 649147191Sjkoshy KASSERT(pd->pm_descr.pd_class == amd_pmc_class, 650145256Sjkoshy ("[amd,%d] unknown PMC class (%d)", __LINE__, 651145256Sjkoshy pd->pm_descr.pd_class)); 652147191Sjkoshy#endif 653145256Sjkoshy 654145256Sjkoshy KASSERT(!AMD_PMC_IS_STOPPED(pd->pm_evsel), 655145256Sjkoshy ("[amd,%d] PMC%d, CPU%d \"%s\" already stopped", 656145256Sjkoshy __LINE__, ri, cpu, pd->pm_descr.pd_name)); 657145256Sjkoshy 658145256Sjkoshy PMCDBG(MDP,STO,1,"amd-stop ri=%d", ri); 659145256Sjkoshy 660145256Sjkoshy /* turn off the PMC ENABLE bit */ 661145256Sjkoshy config = pm->pm_md.pm_amd.pm_amd_evsel & ~AMD_PMC_ENABLE; 662145256Sjkoshy wrmsr(pd->pm_evsel, config); 663145256Sjkoshy return 0; 664145256Sjkoshy} 665145256Sjkoshy 666145256Sjkoshy/* 667145256Sjkoshy * Interrupt handler. This function needs to return '1' if the 668145256Sjkoshy * interrupt was this CPU's PMCs or '0' otherwise. It is not allowed 669145256Sjkoshy * to sleep or do anything a 'fast' interrupt handler is not allowed 670145256Sjkoshy * to do. 671145256Sjkoshy */ 672145256Sjkoshy 673145256Sjkoshystatic int 674174395Sjkoshyamd_intr(int cpu, struct trapframe *tf) 675145256Sjkoshy{ 676147191Sjkoshy int i, error, retval, ri; 677147191Sjkoshy uint32_t config, evsel, perfctr; 678145256Sjkoshy struct pmc *pm; 679145256Sjkoshy struct pmc_cpu *pc; 680145256Sjkoshy struct pmc_hw *phw; 681147191Sjkoshy pmc_value_t v; 682145256Sjkoshy 683145256Sjkoshy KASSERT(cpu >= 0 && cpu < mp_ncpus, 684145256Sjkoshy ("[amd,%d] out of range CPU %d", __LINE__, cpu)); 685145256Sjkoshy 686177343Sadrian PMCDBG(MDP,INT,1, "cpu=%d tf=%p um=%d", cpu, (void *) tf, 687174395Sjkoshy TRAPF_USERMODE(tf)); 688147191Sjkoshy 689145256Sjkoshy retval = 0; 690145256Sjkoshy 691145256Sjkoshy pc = pmc_pcpu[cpu]; 692145256Sjkoshy 693145256Sjkoshy /* 694145256Sjkoshy * look for all PMCs that have interrupted: 695145256Sjkoshy * - skip over the TSC [PMC#0] 696147191Sjkoshy * - look for a running, sampling PMC which has overflowed 697147191Sjkoshy * and which has a valid 'struct pmc' association 698147191Sjkoshy * 699147191Sjkoshy * If found, we call a helper to process the interrupt. 700150050Sjkoshy * 701150050Sjkoshy * If multiple PMCs interrupt at the same time, the AMD64 702150050Sjkoshy * processor appears to deliver as many NMIs as there are 703174395Sjkoshy * outstanding PMC interrupts. So we process only one NMI 704174395Sjkoshy * interrupt at a time. 705145256Sjkoshy */ 706145256Sjkoshy 707150050Sjkoshy for (i = 0; retval == 0 && i < AMD_NPMCS-1; i++) { 708145256Sjkoshy 709147191Sjkoshy ri = i + 1; /* row index; TSC is at ri == 0 */ 710147191Sjkoshy 711147191Sjkoshy if (!AMD_PMC_HAS_OVERFLOWED(i)) 712147191Sjkoshy continue; 713147191Sjkoshy 714147191Sjkoshy phw = pc->pc_hwpmcs[ri]; 715147191Sjkoshy 716145256Sjkoshy KASSERT(phw != NULL, ("[amd,%d] null PHW pointer", __LINE__)); 717145256Sjkoshy 718145256Sjkoshy if ((pm = phw->phw_pmc) == NULL || 719147191Sjkoshy pm->pm_state != PMC_STATE_RUNNING || 720147191Sjkoshy !PMC_IS_SAMPLING_MODE(PMC_TO_MODE(pm))) { 721145256Sjkoshy continue; 722145256Sjkoshy } 723145256Sjkoshy 724174395Sjkoshy retval = 1; /* Found an interrupting PMC. */ 725150050Sjkoshy 726174395Sjkoshy /* Stop the PMC, reload count. */ 727147191Sjkoshy evsel = AMD_PMC_EVSEL_0 + i; 728147191Sjkoshy perfctr = AMD_PMC_PERFCTR_0 + i; 729147191Sjkoshy v = pm->pm_sc.pm_reloadcount; 730147191Sjkoshy config = rdmsr(evsel); 731147191Sjkoshy 732147191Sjkoshy KASSERT((config & ~AMD_PMC_ENABLE) == 733147191Sjkoshy (pm->pm_md.pm_amd.pm_amd_evsel & ~AMD_PMC_ENABLE), 734147191Sjkoshy ("[amd,%d] config mismatch reg=0x%x pm=0x%x", __LINE__, 735147191Sjkoshy config, pm->pm_md.pm_amd.pm_amd_evsel)); 736147191Sjkoshy 737147191Sjkoshy wrmsr(evsel, config & ~AMD_PMC_ENABLE); 738147191Sjkoshy wrmsr(perfctr, AMD_RELOAD_COUNT_TO_PERFCTR_VALUE(v)); 739147191Sjkoshy 740174395Sjkoshy /* Restart the counter if logging succeeded. */ 741174395Sjkoshy error = pmc_process_interrupt(cpu, pm, tf, TRAPF_USERMODE(tf)); 742147191Sjkoshy if (error == 0) 743147191Sjkoshy wrmsr(evsel, config | AMD_PMC_ENABLE); 744145256Sjkoshy } 745147191Sjkoshy 746147867Sjkoshy atomic_add_int(retval ? &pmc_stats.pm_intr_processed : 747147867Sjkoshy &pmc_stats.pm_intr_ignored, 1); 748147867Sjkoshy 749174395Sjkoshy return (retval); 750145256Sjkoshy} 751145256Sjkoshy 752145256Sjkoshy/* 753145256Sjkoshy * describe a PMC 754145256Sjkoshy */ 755145256Sjkoshystatic int 756145256Sjkoshyamd_describe(int cpu, int ri, struct pmc_info *pi, struct pmc **ppmc) 757145256Sjkoshy{ 758145256Sjkoshy int error; 759145256Sjkoshy size_t copied; 760145256Sjkoshy const struct amd_descr *pd; 761145256Sjkoshy struct pmc_hw *phw; 762145256Sjkoshy 763145256Sjkoshy KASSERT(cpu >= 0 && cpu < mp_ncpus, 764145256Sjkoshy ("[amd,%d] illegal CPU %d", __LINE__, cpu)); 765145256Sjkoshy KASSERT(ri >= 0 && ri < AMD_NPMCS, 766145256Sjkoshy ("[amd,%d] row-index %d out of range", __LINE__, ri)); 767145256Sjkoshy 768145256Sjkoshy phw = pmc_pcpu[cpu]->pc_hwpmcs[ri]; 769145256Sjkoshy pd = &amd_pmcdesc[ri]; 770145256Sjkoshy 771145256Sjkoshy if ((error = copystr(pd->pm_descr.pd_name, pi->pm_name, 772145256Sjkoshy PMC_NAME_MAX, &copied)) != 0) 773145256Sjkoshy return error; 774145256Sjkoshy 775145256Sjkoshy pi->pm_class = pd->pm_descr.pd_class; 776145256Sjkoshy 777145256Sjkoshy if (phw->phw_state & PMC_PHW_FLAG_IS_ENABLED) { 778145256Sjkoshy pi->pm_enabled = TRUE; 779145256Sjkoshy *ppmc = phw->phw_pmc; 780145256Sjkoshy } else { 781145256Sjkoshy pi->pm_enabled = FALSE; 782145256Sjkoshy *ppmc = NULL; 783145256Sjkoshy } 784145256Sjkoshy 785145256Sjkoshy return 0; 786145256Sjkoshy} 787145256Sjkoshy 788145256Sjkoshy/* 789145256Sjkoshy * i386 specific entry points 790145256Sjkoshy */ 791145256Sjkoshy 792145256Sjkoshy/* 793145256Sjkoshy * return the MSR address of the given PMC. 794145256Sjkoshy */ 795145256Sjkoshy 796145256Sjkoshystatic int 797145256Sjkoshyamd_get_msr(int ri, uint32_t *msr) 798145256Sjkoshy{ 799145256Sjkoshy KASSERT(ri >= 0 && ri < AMD_NPMCS, 800145256Sjkoshy ("[amd,%d] ri %d out of range", __LINE__, ri)); 801145256Sjkoshy 802145615Sjkoshy *msr = amd_pmcdesc[ri].pm_perfctr - AMD_PMC_PERFCTR_0; 803145256Sjkoshy return 0; 804145256Sjkoshy} 805145256Sjkoshy 806145256Sjkoshy/* 807145256Sjkoshy * processor dependent initialization. 808145256Sjkoshy */ 809145256Sjkoshy 810145256Sjkoshy/* 811145256Sjkoshy * Per-processor data structure 812145256Sjkoshy * 813145256Sjkoshy * [common stuff] 814145256Sjkoshy * [5 struct pmc_hw pointers] 815145256Sjkoshy * [5 struct pmc_hw structures] 816145256Sjkoshy */ 817145256Sjkoshy 818145256Sjkoshystruct amd_cpu { 819145256Sjkoshy struct pmc_cpu pc_common; 820145256Sjkoshy struct pmc_hw *pc_hwpmcs[AMD_NPMCS]; 821145256Sjkoshy struct pmc_hw pc_amdpmcs[AMD_NPMCS]; 822145256Sjkoshy}; 823145256Sjkoshy 824145256Sjkoshy 825145256Sjkoshystatic int 826145256Sjkoshyamd_init(int cpu) 827145256Sjkoshy{ 828145256Sjkoshy int n; 829145256Sjkoshy struct amd_cpu *pcs; 830145256Sjkoshy struct pmc_hw *phw; 831145256Sjkoshy 832145256Sjkoshy KASSERT(cpu >= 0 && cpu < mp_ncpus, 833145256Sjkoshy ("[amd,%d] insane cpu number %d", __LINE__, cpu)); 834145256Sjkoshy 835145256Sjkoshy PMCDBG(MDP,INI,1,"amd-init cpu=%d", cpu); 836145256Sjkoshy 837145256Sjkoshy MALLOC(pcs, struct amd_cpu *, sizeof(struct amd_cpu), M_PMC, 838145256Sjkoshy M_WAITOK|M_ZERO); 839145256Sjkoshy 840145256Sjkoshy phw = &pcs->pc_amdpmcs[0]; 841145256Sjkoshy 842145256Sjkoshy /* 843145256Sjkoshy * Initialize the per-cpu mutex and set the content of the 844145256Sjkoshy * hardware descriptors to a known state. 845145256Sjkoshy */ 846145256Sjkoshy 847145256Sjkoshy for (n = 0; n < AMD_NPMCS; n++, phw++) { 848145256Sjkoshy phw->phw_state = PMC_PHW_FLAG_IS_ENABLED | 849145256Sjkoshy PMC_PHW_CPU_TO_STATE(cpu) | PMC_PHW_INDEX_TO_STATE(n); 850145256Sjkoshy phw->phw_pmc = NULL; 851145256Sjkoshy pcs->pc_hwpmcs[n] = phw; 852145256Sjkoshy } 853145256Sjkoshy 854145256Sjkoshy /* Mark the TSC as shareable */ 855145256Sjkoshy pcs->pc_hwpmcs[0]->phw_state |= PMC_PHW_FLAG_IS_SHAREABLE; 856145256Sjkoshy 857145256Sjkoshy pmc_pcpu[cpu] = (struct pmc_cpu *) pcs; 858145256Sjkoshy 859145256Sjkoshy return 0; 860145256Sjkoshy} 861145256Sjkoshy 862145256Sjkoshy 863145256Sjkoshy/* 864145256Sjkoshy * processor dependent cleanup prior to the KLD 865145256Sjkoshy * being unloaded 866145256Sjkoshy */ 867145256Sjkoshy 868145256Sjkoshystatic int 869145256Sjkoshyamd_cleanup(int cpu) 870145256Sjkoshy{ 871145256Sjkoshy int i; 872145256Sjkoshy uint32_t evsel; 873145256Sjkoshy struct pmc_cpu *pcs; 874145256Sjkoshy 875145256Sjkoshy KASSERT(cpu >= 0 && cpu < mp_ncpus, 876145256Sjkoshy ("[amd,%d] insane cpu number (%d)", __LINE__, cpu)); 877145256Sjkoshy 878145256Sjkoshy PMCDBG(MDP,INI,1,"amd-cleanup cpu=%d", cpu); 879145256Sjkoshy 880145256Sjkoshy /* 881145256Sjkoshy * First, turn off all PMCs on this CPU. 882145256Sjkoshy */ 883145256Sjkoshy 884145256Sjkoshy for (i = 0; i < 4; i++) { /* XXX this loop is now not needed */ 885145256Sjkoshy evsel = rdmsr(AMD_PMC_EVSEL_0 + i); 886145256Sjkoshy evsel &= ~AMD_PMC_ENABLE; 887145256Sjkoshy wrmsr(AMD_PMC_EVSEL_0 + i, evsel); 888145256Sjkoshy } 889145256Sjkoshy 890145256Sjkoshy /* 891145256Sjkoshy * Next, free up allocated space. 892145256Sjkoshy */ 893145256Sjkoshy 894147191Sjkoshy if ((pcs = pmc_pcpu[cpu]) == NULL) 895147191Sjkoshy return 0; 896145256Sjkoshy 897153110Sru#ifdef DEBUG 898145256Sjkoshy /* check the TSC */ 899145256Sjkoshy KASSERT(pcs->pc_hwpmcs[0]->phw_pmc == NULL, 900145256Sjkoshy ("[amd,%d] CPU%d,PMC0 still in use", __LINE__, cpu)); 901145256Sjkoshy for (i = 1; i < AMD_NPMCS; i++) { 902145256Sjkoshy KASSERT(pcs->pc_hwpmcs[i]->phw_pmc == NULL, 903145256Sjkoshy ("[amd,%d] CPU%d/PMC%d in use", __LINE__, cpu, i)); 904145256Sjkoshy KASSERT(AMD_PMC_IS_STOPPED(AMD_PMC_EVSEL_0 + (i-1)), 905145256Sjkoshy ("[amd,%d] CPU%d/PMC%d not stopped", __LINE__, cpu, i)); 906145256Sjkoshy } 907145256Sjkoshy#endif 908145256Sjkoshy 909145256Sjkoshy pmc_pcpu[cpu] = NULL; 910145256Sjkoshy FREE(pcs, M_PMC); 911145256Sjkoshy return 0; 912145256Sjkoshy} 913145256Sjkoshy 914145256Sjkoshy/* 915145256Sjkoshy * Initialize ourselves. 916145256Sjkoshy */ 917145256Sjkoshy 918145256Sjkoshystruct pmc_mdep * 919145256Sjkoshypmc_amd_initialize(void) 920145256Sjkoshy{ 921147191Sjkoshy enum pmc_cputype cputype; 922147191Sjkoshy enum pmc_class class; 923145256Sjkoshy struct pmc_mdep *pmc_mdep; 924147191Sjkoshy char *name; 925147191Sjkoshy int i; 926145256Sjkoshy 927147191Sjkoshy /* 928147191Sjkoshy * The presence of hardware performance counters on the AMD 929147191Sjkoshy * Athlon, Duron or later processors, is _not_ indicated by 930147191Sjkoshy * any of the processor feature flags set by the 'CPUID' 931147191Sjkoshy * instruction, so we only check the 'instruction family' 932147191Sjkoshy * field returned by CPUID for instruction family >= 6. 933147191Sjkoshy */ 934145256Sjkoshy 935147510Sjkoshy class = cputype = -1; 936147510Sjkoshy name = NULL; 937147191Sjkoshy switch (cpu_id & 0xF00) { 938147191Sjkoshy case 0x600: /* Athlon(tm) processor */ 939147191Sjkoshy cputype = PMC_CPU_AMD_K7; 940147191Sjkoshy class = PMC_CLASS_K7; 941147191Sjkoshy name = "K7"; 942147191Sjkoshy break; 943147191Sjkoshy case 0xF00: /* Athlon64/Opteron processor */ 944147191Sjkoshy cputype = PMC_CPU_AMD_K8; 945147191Sjkoshy class = PMC_CLASS_K8; 946147191Sjkoshy name = "K8"; 947147191Sjkoshy break; 948147191Sjkoshy } 949147191Sjkoshy 950147191Sjkoshy if ((int) cputype == -1) { 951147191Sjkoshy (void) printf("pmc: Unknown AMD CPU.\n"); 952145256Sjkoshy return NULL; 953147191Sjkoshy } 954145256Sjkoshy 955153110Sru#ifdef DEBUG 956147191Sjkoshy amd_pmc_class = class; 957147191Sjkoshy#endif 958147191Sjkoshy 959145256Sjkoshy MALLOC(pmc_mdep, struct pmc_mdep *, sizeof(struct pmc_mdep), 960145256Sjkoshy M_PMC, M_WAITOK|M_ZERO); 961145256Sjkoshy 962147191Sjkoshy pmc_mdep->pmd_cputype = cputype; 963145256Sjkoshy pmc_mdep->pmd_npmc = AMD_NPMCS; 964145256Sjkoshy 965145256Sjkoshy /* this processor has two classes of usable PMCs */ 966145256Sjkoshy pmc_mdep->pmd_nclass = 2; 967145774Sjkoshy 968145774Sjkoshy /* TSC */ 969145774Sjkoshy pmc_mdep->pmd_classes[0].pm_class = PMC_CLASS_TSC; 970145774Sjkoshy pmc_mdep->pmd_classes[0].pm_caps = PMC_CAP_READ; 971145774Sjkoshy pmc_mdep->pmd_classes[0].pm_width = 64; 972145774Sjkoshy 973145774Sjkoshy /* AMD K7/K8 PMCs */ 974147191Sjkoshy pmc_mdep->pmd_classes[1].pm_class = class; 975145774Sjkoshy pmc_mdep->pmd_classes[1].pm_caps = AMD_PMC_CAPS; 976145774Sjkoshy pmc_mdep->pmd_classes[1].pm_width = 48; 977145774Sjkoshy 978145256Sjkoshy pmc_mdep->pmd_nclasspmcs[0] = 1; 979145256Sjkoshy pmc_mdep->pmd_nclasspmcs[1] = (AMD_NPMCS-1); 980145256Sjkoshy 981147191Sjkoshy /* fill in the correct pmc name and class */ 982147191Sjkoshy for (i = 1; i < AMD_NPMCS; i++) { 983147191Sjkoshy (void) snprintf(amd_pmcdesc[i].pm_descr.pd_name, 984147191Sjkoshy sizeof(amd_pmcdesc[i].pm_descr.pd_name), "%s-%d", 985147191Sjkoshy name, i-1); 986147191Sjkoshy amd_pmcdesc[i].pm_descr.pd_class = class; 987147191Sjkoshy } 988147191Sjkoshy 989145256Sjkoshy pmc_mdep->pmd_init = amd_init; 990145256Sjkoshy pmc_mdep->pmd_cleanup = amd_cleanup; 991145256Sjkoshy pmc_mdep->pmd_switch_in = amd_switch_in; 992145256Sjkoshy pmc_mdep->pmd_switch_out = amd_switch_out; 993145256Sjkoshy pmc_mdep->pmd_read_pmc = amd_read_pmc; 994145256Sjkoshy pmc_mdep->pmd_write_pmc = amd_write_pmc; 995145256Sjkoshy pmc_mdep->pmd_config_pmc = amd_config_pmc; 996145774Sjkoshy pmc_mdep->pmd_get_config = amd_get_config; 997145256Sjkoshy pmc_mdep->pmd_allocate_pmc = amd_allocate_pmc; 998145256Sjkoshy pmc_mdep->pmd_release_pmc = amd_release_pmc; 999145256Sjkoshy pmc_mdep->pmd_start_pmc = amd_start_pmc; 1000145256Sjkoshy pmc_mdep->pmd_stop_pmc = amd_stop_pmc; 1001145256Sjkoshy pmc_mdep->pmd_intr = amd_intr; 1002145256Sjkoshy pmc_mdep->pmd_describe = amd_describe; 1003145256Sjkoshy pmc_mdep->pmd_get_msr = amd_get_msr; /* i386 */ 1004145256Sjkoshy 1005145256Sjkoshy PMCDBG(MDP,INI,0,"%s","amd-initialize"); 1006145256Sjkoshy 1007145256Sjkoshy return pmc_mdep; 1008145256Sjkoshy} 1009