hwpmc_amd.c revision 145338
1145256Sjkoshy/*- 2145256Sjkoshy * Copyright (c) 2003-2005 Joseph Koshy 3145256Sjkoshy * All rights reserved. 4145256Sjkoshy * 5145256Sjkoshy * Redistribution and use in source and binary forms, with or without 6145256Sjkoshy * modification, are permitted provided that the following conditions 7145256Sjkoshy * are met: 8145256Sjkoshy * 1. Redistributions of source code must retain the above copyright 9145256Sjkoshy * notice, this list of conditions and the following disclaimer. 10145256Sjkoshy * 2. Redistributions in binary form must reproduce the above copyright 11145256Sjkoshy * notice, this list of conditions and the following disclaimer in the 12145256Sjkoshy * documentation and/or other materials provided with the distribution. 13145256Sjkoshy * 14145256Sjkoshy * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 15145256Sjkoshy * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 16145256Sjkoshy * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 17145256Sjkoshy * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 18145256Sjkoshy * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 19145256Sjkoshy * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 20145256Sjkoshy * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 21145256Sjkoshy * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 22145256Sjkoshy * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 23145256Sjkoshy * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 24145256Sjkoshy * SUCH DAMAGE. 25145256Sjkoshy * 26145256Sjkoshy */ 27145256Sjkoshy 28145256Sjkoshy#include <sys/cdefs.h> 29145256Sjkoshy__FBSDID("$FreeBSD: head/sys/dev/hwpmc/hwpmc_amd.c 145338 2005-04-20 20:26:39Z marcel $"); 30145256Sjkoshy 31145256Sjkoshy/* Support for the AMD K7 and later processors */ 32145256Sjkoshy 33145256Sjkoshy#include <sys/param.h> 34145256Sjkoshy#include <sys/lock.h> 35145256Sjkoshy#include <sys/malloc.h> 36145256Sjkoshy#include <sys/mutex.h> 37145338Smarcel#include <sys/pmc.h> 38145256Sjkoshy#include <sys/smp.h> 39145256Sjkoshy#include <sys/systm.h> 40145256Sjkoshy 41145256Sjkoshy#include <machine/md_var.h> 42145256Sjkoshy 43145256Sjkoshy/* AMD K7 and K8 PMCs */ 44145256Sjkoshy 45145256Sjkoshy#define AMD_PMC_EVSEL_0 0xC0010000 46145256Sjkoshy#define AMD_PMC_EVSEL_1 0xC0010001 47145256Sjkoshy#define AMD_PMC_EVSEL_2 0xC0010002 48145256Sjkoshy#define AMD_PMC_EVSEL_3 0xC0010003 49145256Sjkoshy 50145256Sjkoshy#define AMD_PMC_PERFCTR_0 0xC0010004 51145256Sjkoshy#define AMD_PMC_PERFCTR_1 0xC0010005 52145256Sjkoshy#define AMD_PMC_PERFCTR_2 0xC0010006 53145256Sjkoshy#define AMD_PMC_PERFCTR_3 0xC0010007 54145256Sjkoshy 55145256Sjkoshy#define K7_VALID_EVENT_CODE(c) (((c) >= 0x40 && (c) <= 0x47) || \ 56145256Sjkoshy ((c) >= 0x80 && (c) <= 0x85) || ((c) >= 0xC0 && (c) <= 0xC7) || \ 57145256Sjkoshy ((c) >= 0xCD && (c) <= 0xCF)) 58145256Sjkoshy 59145256Sjkoshy#define AMD_PMC_CAPS (PMC_CAP_INTERRUPT | PMC_CAP_USER | \ 60145256Sjkoshy PMC_CAP_SYSTEM | PMC_CAP_EDGE | PMC_CAP_THRESHOLD | \ 61145256Sjkoshy PMC_CAP_READ | PMC_CAP_WRITE | PMC_CAP_INVERT | PMC_CAP_QUALIFIER) 62145256Sjkoshy 63145256Sjkoshy/* reserved bits include bit 21 and the top two bits of the unit mask */ 64145256Sjkoshy#define K7_PMC_RESERVED ((1 << 21) | (3 << 13)) 65145256Sjkoshy 66145256Sjkoshy#define K8_PMC_RESERVED (1 << 21) 67145256Sjkoshy 68145256Sjkoshy#define AMD_PMC_IS_STOPPED(evsel) ((rdmsr((evsel)) & AMD_PMC_ENABLE) == 0) 69145256Sjkoshy#define AMD_PMC_HAS_OVERFLOWED(pmc) ((rdpmc(pmc) & (1ULL << 47)) == 0) 70145256Sjkoshy 71145256Sjkoshy#if __i386__ 72145256Sjkoshy#define AMD_NPMCS K7_NPMCS 73145256Sjkoshy#define AMD_PMC_CLASS PMC_CLASS_K7 74145256Sjkoshy#define AMD_PMC_COUNTERMASK K7_PMC_COUNTERMASK 75145256Sjkoshy#define AMD_PMC_TO_COUNTER(x) K7_PMC_TO_COUNTER(x) 76145256Sjkoshy#define AMD_PMC_INVERT K7_PMC_INVERT 77145256Sjkoshy#define AMD_PMC_ENABLE K7_PMC_ENABLE 78145256Sjkoshy#define AMD_PMC_INT K7_PMC_INT 79145256Sjkoshy#define AMD_PMC_PC K7_PMC_PC 80145256Sjkoshy#define AMD_PMC_EDGE K7_PMC_EDGE 81145256Sjkoshy#define AMD_PMC_OS K7_PMC_OS 82145256Sjkoshy#define AMD_PMC_USR K7_PMC_USR 83145256Sjkoshy 84145256Sjkoshy#define AMD_PMC_UNITMASK_M K7_PMC_UNITMASK_M 85145256Sjkoshy#define AMD_PMC_UNITMASK_O K7_PMC_UNITMASK_O 86145256Sjkoshy#define AMD_PMC_UNITMASK_E K7_PMC_UNITMASK_E 87145256Sjkoshy#define AMD_PMC_UNITMASK_S K7_PMC_UNITMASK_S 88145256Sjkoshy#define AMD_PMC_UNITMASK_I K7_PMC_UNITMASK_I 89145256Sjkoshy 90145256Sjkoshy#define AMD_PMC_UNITMASK K7_PMC_UNITMASK 91145256Sjkoshy#define AMD_PMC_EVENTMASK K7_PMC_EVENTMASK 92145256Sjkoshy#define AMD_PMC_TO_UNITMASK(x) K7_PMC_TO_UNITMASK(x) 93145256Sjkoshy#define AMD_PMC_TO_EVENTMASK(x) K7_PMC_TO_EVENTMASK(x) 94145256Sjkoshy#define AMD_VALID_BITS K7_VALID_BITS 95145256Sjkoshy 96145256Sjkoshy#define AMD_PMC_CLASS_NAME "K7-" 97145256Sjkoshy 98145256Sjkoshy#elif __amd64__ 99145256Sjkoshy 100145256Sjkoshy#define AMD_NPMCS K8_NPMCS 101145256Sjkoshy#define AMD_PMC_CLASS PMC_CLASS_K8 102145256Sjkoshy#define AMD_PMC_COUNTERMASK K8_PMC_COUNTERMASK 103145256Sjkoshy#define AMD_PMC_TO_COUNTER(x) K8_PMC_TO_COUNTER(x) 104145256Sjkoshy#define AMD_PMC_INVERT K8_PMC_INVERT 105145256Sjkoshy#define AMD_PMC_ENABLE K8_PMC_ENABLE 106145256Sjkoshy#define AMD_PMC_INT K8_PMC_INT 107145256Sjkoshy#define AMD_PMC_PC K8_PMC_PC 108145256Sjkoshy#define AMD_PMC_EDGE K8_PMC_EDGE 109145256Sjkoshy#define AMD_PMC_OS K8_PMC_OS 110145256Sjkoshy#define AMD_PMC_USR K8_PMC_USR 111145256Sjkoshy 112145256Sjkoshy#define AMD_PMC_UNITMASK_M K8_PMC_UNITMASK_M 113145256Sjkoshy#define AMD_PMC_UNITMASK_O K8_PMC_UNITMASK_O 114145256Sjkoshy#define AMD_PMC_UNITMASK_E K8_PMC_UNITMASK_E 115145256Sjkoshy#define AMD_PMC_UNITMASK_S K8_PMC_UNITMASK_S 116145256Sjkoshy#define AMD_PMC_UNITMASK_I K8_PMC_UNITMASK_I 117145256Sjkoshy 118145256Sjkoshy#define AMD_PMC_UNITMASK K8_PMC_UNITMASK 119145256Sjkoshy#define AMD_PMC_EVENTMASK K8_PMC_EVENTMASK 120145256Sjkoshy#define AMD_PMC_TO_UNITMASK(x) K8_PMC_TO_UNITMASK(x) 121145256Sjkoshy#define AMD_PMC_TO_EVENTMASK(x) K8_PMC_TO_EVENTMASK(x) 122145256Sjkoshy#define AMD_VALID_BITS K8_VALID_BITS 123145256Sjkoshy 124145256Sjkoshy#define AMD_PMC_CLASS_NAME "K8-" 125145256Sjkoshy 126145256Sjkoshy#else 127145256Sjkoshy#error Unsupported architecture. 128145256Sjkoshy#endif 129145256Sjkoshy 130145256Sjkoshy/* AMD K7 & K8 PMCs */ 131145256Sjkoshystruct amd_descr { 132145256Sjkoshy struct pmc_descr pm_descr; /* "base class" */ 133145256Sjkoshy uint32_t pm_evsel; /* address of EVSEL register */ 134145256Sjkoshy uint32_t pm_perfctr; /* address of PERFCTR register */ 135145256Sjkoshy}; 136145256Sjkoshy 137145256Sjkoshystatic const struct amd_descr amd_pmcdesc[AMD_NPMCS] = 138145256Sjkoshy{ 139145256Sjkoshy { 140145256Sjkoshy .pm_descr = 141145256Sjkoshy { 142145256Sjkoshy .pd_name = "TSC", 143145256Sjkoshy .pd_class = PMC_CLASS_TSC, 144145256Sjkoshy .pd_caps = PMC_CAP_READ, 145145256Sjkoshy .pd_width = 64 146145256Sjkoshy }, 147145256Sjkoshy .pm_evsel = MSR_TSC, 148145256Sjkoshy .pm_perfctr = 0 /* unused */ 149145256Sjkoshy }, 150145256Sjkoshy 151145256Sjkoshy { 152145256Sjkoshy .pm_descr = 153145256Sjkoshy { 154145256Sjkoshy .pd_name = AMD_PMC_CLASS_NAME "0", 155145256Sjkoshy .pd_class = AMD_PMC_CLASS, 156145256Sjkoshy .pd_caps = AMD_PMC_CAPS, 157145256Sjkoshy .pd_width = 48 158145256Sjkoshy }, 159145256Sjkoshy .pm_evsel = AMD_PMC_EVSEL_0, 160145256Sjkoshy .pm_perfctr = AMD_PMC_PERFCTR_0 161145256Sjkoshy }, 162145256Sjkoshy { 163145256Sjkoshy .pm_descr = 164145256Sjkoshy { 165145256Sjkoshy .pd_name = AMD_PMC_CLASS_NAME "1", 166145256Sjkoshy .pd_class = AMD_PMC_CLASS, 167145256Sjkoshy .pd_caps = AMD_PMC_CAPS, 168145256Sjkoshy .pd_width = 48 169145256Sjkoshy }, 170145256Sjkoshy .pm_evsel = AMD_PMC_EVSEL_1, 171145256Sjkoshy .pm_perfctr = AMD_PMC_PERFCTR_1 172145256Sjkoshy }, 173145256Sjkoshy { 174145256Sjkoshy .pm_descr = 175145256Sjkoshy { 176145256Sjkoshy .pd_name = AMD_PMC_CLASS_NAME "2", 177145256Sjkoshy .pd_class = AMD_PMC_CLASS, 178145256Sjkoshy .pd_caps = AMD_PMC_CAPS, 179145256Sjkoshy .pd_width = 48 180145256Sjkoshy }, 181145256Sjkoshy .pm_evsel = AMD_PMC_EVSEL_2, 182145256Sjkoshy .pm_perfctr = AMD_PMC_PERFCTR_2 183145256Sjkoshy }, 184145256Sjkoshy { 185145256Sjkoshy .pm_descr = 186145256Sjkoshy { 187145256Sjkoshy .pd_name = AMD_PMC_CLASS_NAME "3", 188145256Sjkoshy .pd_class = AMD_PMC_CLASS, 189145256Sjkoshy .pd_caps = AMD_PMC_CAPS, 190145256Sjkoshy .pd_width = 48 191145256Sjkoshy }, 192145256Sjkoshy .pm_evsel = AMD_PMC_EVSEL_3, 193145256Sjkoshy .pm_perfctr = AMD_PMC_PERFCTR_3 194145256Sjkoshy } 195145256Sjkoshy}; 196145256Sjkoshy 197145256Sjkoshystruct amd_event_code_map { 198145256Sjkoshy enum pmc_event pe_ev; /* enum value */ 199145256Sjkoshy uint8_t pe_code; /* encoded event mask */ 200145256Sjkoshy uint8_t pe_mask; /* bits allowed in unit mask */ 201145256Sjkoshy}; 202145256Sjkoshy 203145256Sjkoshyconst struct amd_event_code_map amd_event_codes[] = { 204145256Sjkoshy#if __i386__ 205145256Sjkoshy { PMC_EV_K7_DC_ACCESSES, 0x40, 0 }, 206145256Sjkoshy { PMC_EV_K7_DC_MISSES, 0x41, 0 }, 207145256Sjkoshy { PMC_EV_K7_DC_REFILLS_FROM_L2, 0x42, K7_PMC_UNITMASK_MOESI }, 208145256Sjkoshy { PMC_EV_K7_DC_REFILLS_FROM_SYSTEM, 0x43, K7_PMC_UNITMASK_MOESI }, 209145256Sjkoshy { PMC_EV_K7_DC_WRITEBACKS, 0x44, K7_PMC_UNITMASK_MOESI }, 210145256Sjkoshy { PMC_EV_K7_L1_DTLB_MISS_AND_L2_DTLB_HITS, 0x45, 0 }, 211145256Sjkoshy { PMC_EV_K7_L1_AND_L2_DTLB_MISSES, 0x46, 0 }, 212145256Sjkoshy { PMC_EV_K7_MISALIGNED_REFERENCES, 0x47, 0 }, 213145256Sjkoshy 214145256Sjkoshy { PMC_EV_K7_IC_FETCHES, 0x80, 0 }, 215145256Sjkoshy { PMC_EV_K7_IC_MISSES, 0x81, 0 }, 216145256Sjkoshy 217145256Sjkoshy { PMC_EV_K7_L1_ITLB_MISSES, 0x84, 0 }, 218145256Sjkoshy { PMC_EV_K7_L1_L2_ITLB_MISSES, 0x85, 0 }, 219145256Sjkoshy 220145256Sjkoshy { PMC_EV_K7_RETIRED_INSTRUCTIONS, 0xC0, 0 }, 221145256Sjkoshy { PMC_EV_K7_RETIRED_OPS, 0xC1, 0 }, 222145256Sjkoshy { PMC_EV_K7_RETIRED_BRANCHES, 0xC2, 0 }, 223145256Sjkoshy { PMC_EV_K7_RETIRED_BRANCHES_MISPREDICTED, 0xC3, 0 }, 224145256Sjkoshy { PMC_EV_K7_RETIRED_TAKEN_BRANCHES, 0xC4, 0 }, 225145256Sjkoshy { PMC_EV_K7_RETIRED_TAKEN_BRANCHES_MISPREDICTED, 0xC5, 0 }, 226145256Sjkoshy { PMC_EV_K7_RETIRED_FAR_CONTROL_TRANSFERS, 0xC6, 0 }, 227145256Sjkoshy { PMC_EV_K7_RETIRED_RESYNC_BRANCHES, 0xC7, 0 }, 228145256Sjkoshy { PMC_EV_K7_INTERRUPTS_MASKED_CYCLES, 0xCD, 0 }, 229145256Sjkoshy { PMC_EV_K7_INTERRUPTS_MASKED_WHILE_PENDING_CYCLES, 0xCE, 0 }, 230145256Sjkoshy { PMC_EV_K7_HARDWARE_INTERRUPTS, 0xCF, 0 } 231145256Sjkoshy#endif 232145256Sjkoshy 233145256Sjkoshy#if __amd64__ 234145256Sjkoshy { PMC_EV_K8_FP_DISPATCHED_FPU_OPS, 0x00, 0x3F }, 235145256Sjkoshy { PMC_EV_K8_FP_CYCLES_WITH_NO_FPU_OPS_RETIRED, 0x01, 0x00 }, 236145256Sjkoshy { PMC_EV_K8_FP_DISPATCHED_FPU_FAST_FLAG_OPS, 0x02, 0x00 }, 237145256Sjkoshy 238145256Sjkoshy { PMC_EV_K8_LS_SEGMENT_REGISTER_LOAD, 0x20, 0x7F }, 239145256Sjkoshy { PMC_EV_K8_LS_MICROARCHITECTURAL_RESYNC_BY_SELF_MODIFYING_CODE, 240145256Sjkoshy 0x21, 0x00 }, 241145256Sjkoshy { PMC_EV_K8_LS_MICROARCHITECTURAL_RESYNC_BY_SNOOP, 0x22, 0x00 }, 242145256Sjkoshy { PMC_EV_K8_LS_BUFFER2_FULL, 0x23, 0x00 }, 243145256Sjkoshy { PMC_EV_K8_LS_LOCKED_OPERATION, 0x24, 0x07 }, 244145256Sjkoshy { PMC_EV_K8_LS_MICROARCHITECTURAL_LATE_CANCEL, 0x25, 0x00 }, 245145256Sjkoshy { PMC_EV_K8_LS_RETIRED_CFLUSH_INSTRUCTIONS, 0x26, 0x00 }, 246145256Sjkoshy { PMC_EV_K8_LS_RETIRED_CPUID_INSTRUCTIONS, 0x27, 0x00 }, 247145256Sjkoshy 248145256Sjkoshy { PMC_EV_K8_DC_ACCESS, 0x40, 0x00 }, 249145256Sjkoshy { PMC_EV_K8_DC_MISS, 0x41, 0x00 }, 250145256Sjkoshy { PMC_EV_K8_DC_REFILL_FROM_L2, 0x42, 0x1F }, 251145256Sjkoshy { PMC_EV_K8_DC_REFILL_FROM_SYSTEM, 0x43, 0x1F }, 252145256Sjkoshy { PMC_EV_K8_DC_COPYBACK, 0x44, 0x1F }, 253145256Sjkoshy { PMC_EV_K8_DC_L1_DTLB_MISS_AND_L2_DTLB_HIT, 0x45, 0x00 }, 254145256Sjkoshy { PMC_EV_K8_DC_L1_DTLB_MISS_AND_L2_DTLB_MISS, 0x46, 0x00 }, 255145256Sjkoshy { PMC_EV_K8_DC_MISALIGNED_DATA_REFERENCE, 0x47, 0x00 }, 256145256Sjkoshy { PMC_EV_K8_DC_MICROARCHITECTURAL_LATE_CANCEL, 0x48, 0x00 }, 257145256Sjkoshy { PMC_EV_K8_DC_MICROARCHITECTURAL_EARLY_CANCEL, 0x49, 0x00 }, 258145256Sjkoshy { PMC_EV_K8_DC_ONE_BIT_ECC_ERROR, 0x4A, 0x03 }, 259145256Sjkoshy { PMC_EV_K8_DC_DISPATCHED_PREFETCH_INSTRUCTIONS, 0x4B, 0x07 }, 260145256Sjkoshy { PMC_EV_K8_DC_DCACHE_ACCESSES_BY_LOCKS, 0x4C, 0x03 }, 261145256Sjkoshy 262145256Sjkoshy { PMC_EV_K8_BU_CPU_CLK_UNHALTED, 0x76, 0x00 }, 263145256Sjkoshy { PMC_EV_K8_BU_INTERNAL_L2_REQUEST, 0x7D, 0x1F }, 264145256Sjkoshy { PMC_EV_K8_BU_FILL_REQUEST_L2_MISS, 0x7E, 0x07 }, 265145256Sjkoshy { PMC_EV_K8_BU_FILL_INTO_L2, 0x7F, 0x03 }, 266145256Sjkoshy 267145256Sjkoshy { PMC_EV_K8_IC_FETCH, 0x80, 0x00 }, 268145256Sjkoshy { PMC_EV_K8_IC_MISS, 0x81, 0x00 }, 269145256Sjkoshy { PMC_EV_K8_IC_REFILL_FROM_L2, 0x82, 0x00 }, 270145256Sjkoshy { PMC_EV_K8_IC_REFILL_FROM_SYSTEM, 0x83, 0x00 }, 271145256Sjkoshy { PMC_EV_K8_IC_L1_ITLB_MISS_AND_L2_ITLB_HIT, 0x84, 0x00 }, 272145256Sjkoshy { PMC_EV_K8_IC_L1_ITLB_MISS_AND_L2_ITLB_MISS, 0x85, 0x00 }, 273145256Sjkoshy { PMC_EV_K8_IC_MICROARCHITECTURAL_RESYNC_BY_SNOOP, 0x86, 0x00 }, 274145256Sjkoshy { PMC_EV_K8_IC_INSTRUCTION_FETCH_STALL, 0x87, 0x00 }, 275145256Sjkoshy { PMC_EV_K8_IC_RETURN_STACK_HIT, 0x88, 0x00 }, 276145256Sjkoshy { PMC_EV_K8_IC_RETURN_STACK_OVERFLOW, 0x89, 0x00 }, 277145256Sjkoshy 278145256Sjkoshy { PMC_EV_K8_FR_RETIRED_X86_INSTRUCTIONS, 0xC0, 0x00 }, 279145256Sjkoshy { PMC_EV_K8_FR_RETIRED_UOPS, 0xC1, 0x00 }, 280145256Sjkoshy { PMC_EV_K8_FR_RETIRED_BRANCHES, 0xC2, 0x00 }, 281145256Sjkoshy { PMC_EV_K8_FR_RETIRED_BRANCHES_MISPREDICTED, 0xC3, 0x00 }, 282145256Sjkoshy { PMC_EV_K8_FR_RETIRED_TAKEN_BRANCHES, 0xC4, 0x00 }, 283145256Sjkoshy { PMC_EV_K8_FR_RETIRED_TAKEN_BRANCHES_MISPREDICTED, 0xC5, 0x00 }, 284145256Sjkoshy { PMC_EV_K8_FR_RETIRED_FAR_CONTROL_TRANSFERS, 0xC6, 0x00 }, 285145256Sjkoshy { PMC_EV_K8_FR_RETIRED_RESYNCS, 0xC7, 0x00 }, 286145256Sjkoshy { PMC_EV_K8_FR_RETIRED_NEAR_RETURNS, 0xC8, 0x00 }, 287145256Sjkoshy { PMC_EV_K8_FR_RETIRED_NEAR_RETURNS_MISPREDICTED, 0xC9, 0x00 }, 288145256Sjkoshy { PMC_EV_K8_FR_RETIRED_TAKEN_BRANCHES_MISPREDICTED_BY_ADDR_MISCOMPARE, 289145256Sjkoshy 0xCA, 0x00 }, 290145256Sjkoshy { PMC_EV_K8_FR_RETIRED_FPU_INSTRUCTIONS, 0xCB, 0x0F }, 291145256Sjkoshy { PMC_EV_K8_FR_RETIRED_FASTPATH_DOUBLE_OP_INSTRUCTIONS, 292145256Sjkoshy 0xCC, 0x07 }, 293145256Sjkoshy { PMC_EV_K8_FR_INTERRUPTS_MASKED_CYCLES, 0xCD, 0x00 }, 294145256Sjkoshy { PMC_EV_K8_FR_INTERRUPTS_MASKED_WHILE_PENDING_CYCLES, 0xCE, 0x00 }, 295145256Sjkoshy { PMC_EV_K8_FR_TAKEN_HARDWARE_INTERRUPTS, 0xCF, 0x00 }, 296145256Sjkoshy 297145256Sjkoshy { PMC_EV_K8_FR_DECODER_EMPTY, 0xD0, 0x00 }, 298145256Sjkoshy { PMC_EV_K8_FR_DISPATCH_STALLS, 0xD1, 0x00 }, 299145256Sjkoshy { PMC_EV_K8_FR_DISPATCH_STALL_FROM_BRANCH_ABORT_TO_RETIRE, 300145256Sjkoshy 0xD2, 0x00 }, 301145256Sjkoshy { PMC_EV_K8_FR_DISPATCH_STALL_FOR_SERIALIZATION, 0xD3, 0x00 }, 302145256Sjkoshy { PMC_EV_K8_FR_DISPATCH_STALL_FOR_SEGMENT_LOAD, 0xD4, 0x00 }, 303145256Sjkoshy { PMC_EV_K8_FR_DISPATCH_STALL_WHEN_REORDER_BUFFER_IS_FULL, 304145256Sjkoshy 0xD5, 0x00 }, 305145256Sjkoshy { PMC_EV_K8_FR_DISPATCH_STALL_WHEN_RESERVATION_STATIONS_ARE_FULL, 306145256Sjkoshy 0xD6, 0x00 }, 307145256Sjkoshy { PMC_EV_K8_FR_DISPATCH_STALL_WHEN_FPU_IS_FULL, 0xD7, 0x00 }, 308145256Sjkoshy { PMC_EV_K8_FR_DISPATCH_STALL_WHEN_LS_IS_FULL, 0xD8, 0x00 }, 309145256Sjkoshy { PMC_EV_K8_FR_DISPATCH_STALL_WHEN_WAITING_FOR_ALL_TO_BE_QUIET, 310145256Sjkoshy 0xD9, 0x00 }, 311145256Sjkoshy { PMC_EV_K8_FR_DISPATCH_STALL_WHEN_FAR_XFER_OR_RESYNC_BRANCH_PENDING, 312145256Sjkoshy 0xDA, 0x00 }, 313145256Sjkoshy { PMC_EV_K8_FR_FPU_EXCEPTIONS, 0xDB, 0x0F }, 314145256Sjkoshy { PMC_EV_K8_FR_NUMBER_OF_BREAKPOINTS_FOR_DR0, 0xDC, 0x00 }, 315145256Sjkoshy { PMC_EV_K8_FR_NUMBER_OF_BREAKPOINTS_FOR_DR1, 0xDD, 0x00 }, 316145256Sjkoshy { PMC_EV_K8_FR_NUMBER_OF_BREAKPOINTS_FOR_DR2, 0xDE, 0x00 }, 317145256Sjkoshy { PMC_EV_K8_FR_NUMBER_OF_BREAKPOINTS_FOR_DR3, 0xDF, 0x00 }, 318145256Sjkoshy 319145256Sjkoshy { PMC_EV_K8_NB_MEMORY_CONTROLLER_PAGE_ACCESS_EVENT, 0xE0, 0x7 }, 320145256Sjkoshy { PMC_EV_K8_NB_MEMORY_CONTROLLER_PAGE_TABLE_OVERFLOW, 0xE1, 0x00 }, 321145256Sjkoshy { PMC_EV_K8_NB_MEMORY_CONTROLLER_DRAM_COMMAND_SLOTS_MISSED, 322145256Sjkoshy 0xE2, 0x00 }, 323145256Sjkoshy { PMC_EV_K8_NB_MEMORY_CONTROLLER_TURNAROUND, 0xE3, 0x07 }, 324145256Sjkoshy { PMC_EV_K8_NB_MEMORY_CONTROLLER_BYPASS_SATURATION, 0xE4, 0x0F }, 325145256Sjkoshy { PMC_EV_K8_NB_SIZED_COMMANDS, 0xEB, 0x7F }, 326145256Sjkoshy { PMC_EV_K8_NB_PROBE_RESULT, 0xEC, 0x0F }, 327145256Sjkoshy { PMC_EV_K8_NB_HT_BUS0_BANDWIDTH, 0xF6, 0x0F }, 328145256Sjkoshy { PMC_EV_K8_NB_HT_BUS1_BANDWIDTH, 0xF7, 0x0F }, 329145256Sjkoshy { PMC_EV_K8_NB_HT_BUS2_BANDWIDTH, 0xF8, 0x0F } 330145256Sjkoshy#endif 331145256Sjkoshy 332145256Sjkoshy}; 333145256Sjkoshy 334145256Sjkoshyconst int amd_event_codes_size = 335145256Sjkoshy sizeof(amd_event_codes) / sizeof(amd_event_codes[0]); 336145256Sjkoshy 337145256Sjkoshy/* 338145256Sjkoshy * read a pmc register 339145256Sjkoshy */ 340145256Sjkoshy 341145256Sjkoshystatic int 342145256Sjkoshyamd_read_pmc(int cpu, int ri, pmc_value_t *v) 343145256Sjkoshy{ 344145256Sjkoshy enum pmc_mode mode; 345145256Sjkoshy const struct amd_descr *pd; 346145256Sjkoshy struct pmc *pm; 347145256Sjkoshy const struct pmc_hw *phw; 348145256Sjkoshy pmc_value_t tmp; 349145256Sjkoshy 350145256Sjkoshy KASSERT(cpu >= 0 && cpu < mp_ncpus, 351145256Sjkoshy ("[amd,%d] illegal CPU value %d", __LINE__, cpu)); 352145256Sjkoshy KASSERT(ri >= 0 && ri < AMD_NPMCS, 353145256Sjkoshy ("[amd,%d] illegal row-index %d", __LINE__, ri)); 354145256Sjkoshy 355145256Sjkoshy phw = pmc_pcpu[cpu]->pc_hwpmcs[ri]; 356145256Sjkoshy pd = &amd_pmcdesc[ri]; 357145256Sjkoshy pm = phw->phw_pmc; 358145256Sjkoshy 359145256Sjkoshy KASSERT(pm != NULL, 360145256Sjkoshy ("[amd,%d] No owner for HWPMC [cpu%d,pmc%d]", __LINE__, 361145256Sjkoshy cpu, ri)); 362145256Sjkoshy 363145256Sjkoshy mode = pm->pm_mode; 364145256Sjkoshy 365145256Sjkoshy PMCDBG(MDP,REA,1,"amd-read id=%d class=%d", ri, pd->pm_descr.pd_class); 366145256Sjkoshy 367145256Sjkoshy /* Reading the TSC is a special case */ 368145256Sjkoshy if (pd->pm_descr.pd_class == PMC_CLASS_TSC) { 369145256Sjkoshy KASSERT(PMC_IS_COUNTING_MODE(mode), 370145256Sjkoshy ("[amd,%d] TSC counter in non-counting mode", __LINE__)); 371145256Sjkoshy *v = rdtsc(); 372145256Sjkoshy PMCDBG(MDP,REA,2,"amd-read id=%d -> %jd", ri, *v); 373145256Sjkoshy return 0; 374145256Sjkoshy } 375145256Sjkoshy 376145256Sjkoshy KASSERT(pd->pm_descr.pd_class == AMD_PMC_CLASS, 377145256Sjkoshy ("[amd,%d] unknown PMC class (%d)", __LINE__, 378145256Sjkoshy pd->pm_descr.pd_class)); 379145256Sjkoshy 380145256Sjkoshy tmp = rdmsr(pd->pm_perfctr); /* RDMSR serializes */ 381145256Sjkoshy if (PMC_IS_SAMPLING_MODE(mode)) 382145256Sjkoshy *v = -tmp; 383145256Sjkoshy else 384145256Sjkoshy *v = tmp; 385145256Sjkoshy 386145256Sjkoshy PMCDBG(MDP,REA,2,"amd-read id=%d -> %jd", ri, *v); 387145256Sjkoshy 388145256Sjkoshy return 0; 389145256Sjkoshy} 390145256Sjkoshy 391145256Sjkoshy/* 392145256Sjkoshy * Write a PMC MSR. 393145256Sjkoshy */ 394145256Sjkoshy 395145256Sjkoshystatic int 396145256Sjkoshyamd_write_pmc(int cpu, int ri, pmc_value_t v) 397145256Sjkoshy{ 398145256Sjkoshy const struct amd_descr *pd; 399145256Sjkoshy struct pmc *pm; 400145256Sjkoshy const struct pmc_hw *phw; 401145256Sjkoshy enum pmc_mode mode; 402145256Sjkoshy 403145256Sjkoshy KASSERT(cpu >= 0 && cpu < mp_ncpus, 404145256Sjkoshy ("[amd,%d] illegal CPU value %d", __LINE__, cpu)); 405145256Sjkoshy KASSERT(ri >= 0 && ri < AMD_NPMCS, 406145256Sjkoshy ("[amd,%d] illegal row-index %d", __LINE__, ri)); 407145256Sjkoshy 408145256Sjkoshy phw = pmc_pcpu[cpu]->pc_hwpmcs[ri]; 409145256Sjkoshy pd = &amd_pmcdesc[ri]; 410145256Sjkoshy pm = phw->phw_pmc; 411145256Sjkoshy 412145256Sjkoshy KASSERT(pm != NULL, 413145256Sjkoshy ("[amd,%d] PMC not owned (cpu%d,pmc%d)", __LINE__, 414145256Sjkoshy cpu, ri)); 415145256Sjkoshy 416145256Sjkoshy mode = pm->pm_mode; 417145256Sjkoshy 418145256Sjkoshy if (pd->pm_descr.pd_class == PMC_CLASS_TSC) 419145256Sjkoshy return 0; 420145256Sjkoshy 421145256Sjkoshy KASSERT(pd->pm_descr.pd_class == AMD_PMC_CLASS, 422145256Sjkoshy ("[amd,%d] unknown PMC class (%d)", __LINE__, 423145256Sjkoshy pd->pm_descr.pd_class)); 424145256Sjkoshy 425145256Sjkoshy /* use 2's complement of the count for sampling mode PMCs */ 426145256Sjkoshy if (PMC_IS_SAMPLING_MODE(mode)) 427145256Sjkoshy v = -v; 428145256Sjkoshy 429145256Sjkoshy PMCDBG(MDP,WRI,1,"amd-write cpu=%d ri=%d v=%jx", cpu, ri, v); 430145256Sjkoshy 431145256Sjkoshy /* write the PMC value */ 432145256Sjkoshy wrmsr(pd->pm_perfctr, v); 433145256Sjkoshy return 0; 434145256Sjkoshy} 435145256Sjkoshy 436145256Sjkoshy/* 437145256Sjkoshy * configure hardware pmc according to the configuration recorded in 438145256Sjkoshy * pmc 'pm'. 439145256Sjkoshy */ 440145256Sjkoshy 441145256Sjkoshystatic int 442145256Sjkoshyamd_config_pmc(int cpu, int ri, struct pmc *pm) 443145256Sjkoshy{ 444145256Sjkoshy struct pmc_hw *phw; 445145256Sjkoshy 446145256Sjkoshy KASSERT(cpu >= 0 && cpu < mp_ncpus, 447145256Sjkoshy ("[amd,%d] illegal CPU value %d", __LINE__, cpu)); 448145256Sjkoshy KASSERT(ri >= 0 && ri < AMD_NPMCS, 449145256Sjkoshy ("[amd,%d] illegal row-index %d", __LINE__, ri)); 450145256Sjkoshy 451145256Sjkoshy phw = pmc_pcpu[cpu]->pc_hwpmcs[ri]; 452145256Sjkoshy 453145256Sjkoshy KASSERT(pm == NULL || phw->phw_pmc == NULL, 454145256Sjkoshy ("[amd,%d] hwpmc not unconfigured before re-config", __LINE__)); 455145256Sjkoshy 456145256Sjkoshy phw->phw_pmc = pm; 457145256Sjkoshy return 0; 458145256Sjkoshy} 459145256Sjkoshy 460145256Sjkoshy/* 461145256Sjkoshy * Machine dependent actions taken during the context switch in of a 462145256Sjkoshy * thread. 463145256Sjkoshy */ 464145256Sjkoshy 465145256Sjkoshystatic int 466145256Sjkoshyamd_switch_in(struct pmc_cpu *pc) 467145256Sjkoshy{ 468145256Sjkoshy (void) pc; 469145256Sjkoshy 470145256Sjkoshy /* enable the RDPMC instruction */ 471145256Sjkoshy load_cr4(rcr4() | CR4_PCE); 472145256Sjkoshy return 0; 473145256Sjkoshy} 474145256Sjkoshy 475145256Sjkoshy/* 476145256Sjkoshy * Machine dependent actions taken during the context switch out of a 477145256Sjkoshy * thread. 478145256Sjkoshy */ 479145256Sjkoshy 480145256Sjkoshystatic int 481145256Sjkoshyamd_switch_out(struct pmc_cpu *pc) 482145256Sjkoshy{ 483145256Sjkoshy (void) pc; 484145256Sjkoshy 485145256Sjkoshy /* disallow RDPMC instruction */ 486145256Sjkoshy load_cr4(rcr4() & ~CR4_PCE); 487145256Sjkoshy return 0; 488145256Sjkoshy} 489145256Sjkoshy 490145256Sjkoshy/* 491145256Sjkoshy * Check if a given allocation is feasible. 492145256Sjkoshy */ 493145256Sjkoshy 494145256Sjkoshystatic int 495145256Sjkoshyamd_allocate_pmc(int cpu, int ri, struct pmc *pm, 496145256Sjkoshy const struct pmc_op_pmcallocate *a) 497145256Sjkoshy{ 498145256Sjkoshy int i; 499145256Sjkoshy uint32_t allowed_unitmask, caps, config, unitmask; 500145256Sjkoshy enum pmc_event pe; 501145256Sjkoshy const struct pmc_descr *pd; 502145256Sjkoshy 503145256Sjkoshy (void) cpu; 504145256Sjkoshy 505145256Sjkoshy KASSERT(cpu >= 0 && cpu < mp_ncpus, 506145256Sjkoshy ("[amd,%d] illegal CPU value %d", __LINE__, cpu)); 507145256Sjkoshy KASSERT(ri >= 0 && ri < AMD_NPMCS, 508145256Sjkoshy ("[amd,%d] illegal row index %d", __LINE__, ri)); 509145256Sjkoshy 510145256Sjkoshy pd = &amd_pmcdesc[ri].pm_descr; 511145256Sjkoshy 512145256Sjkoshy /* check class match */ 513145256Sjkoshy if (pd->pd_class != pm->pm_class) 514145256Sjkoshy return EINVAL; 515145256Sjkoshy 516145256Sjkoshy caps = pm->pm_caps; 517145256Sjkoshy 518145256Sjkoshy PMCDBG(MDP,ALL,1,"amd-allocate ri=%d caps=0x%x", ri, caps); 519145256Sjkoshy 520145256Sjkoshy if ((pd->pd_caps & caps) != caps) 521145256Sjkoshy return EPERM; 522145256Sjkoshy if (pd->pd_class == PMC_CLASS_TSC) { 523145256Sjkoshy /* TSC's are always allocated in system-wide counting mode */ 524145256Sjkoshy if (a->pm_ev != PMC_EV_TSC_TSC || 525145256Sjkoshy a->pm_mode != PMC_MODE_SC) 526145256Sjkoshy return EINVAL; 527145256Sjkoshy return 0; 528145256Sjkoshy } 529145256Sjkoshy 530145256Sjkoshy KASSERT(pd->pd_class == AMD_PMC_CLASS, 531145256Sjkoshy ("[amd,%d] Unknown PMC class (%d)", __LINE__, pd->pd_class)); 532145256Sjkoshy 533145256Sjkoshy pe = a->pm_ev; 534145256Sjkoshy 535145256Sjkoshy /* map ev to the correct event mask code */ 536145256Sjkoshy config = allowed_unitmask = 0; 537145256Sjkoshy for (i = 0; i < amd_event_codes_size; i++) 538145256Sjkoshy if (amd_event_codes[i].pe_ev == pe) { 539145256Sjkoshy config = 540145256Sjkoshy AMD_PMC_TO_EVENTMASK(amd_event_codes[i].pe_code); 541145256Sjkoshy allowed_unitmask = 542145256Sjkoshy AMD_PMC_TO_UNITMASK(amd_event_codes[i].pe_mask); 543145256Sjkoshy break; 544145256Sjkoshy } 545145256Sjkoshy if (i == amd_event_codes_size) 546145256Sjkoshy return EINVAL; 547145256Sjkoshy 548145256Sjkoshy unitmask = a->pm_amd_config & AMD_PMC_UNITMASK; 549145256Sjkoshy if (unitmask & ~allowed_unitmask) /* disallow reserved bits */ 550145256Sjkoshy return EINVAL; 551145256Sjkoshy 552145256Sjkoshy if (unitmask && (caps & PMC_CAP_QUALIFIER)) 553145256Sjkoshy config |= unitmask; 554145256Sjkoshy 555145256Sjkoshy if (caps & PMC_CAP_THRESHOLD) 556145256Sjkoshy config |= a->pm_amd_config & AMD_PMC_COUNTERMASK; 557145256Sjkoshy 558145256Sjkoshy /* set at least one of the 'usr' or 'os' caps */ 559145256Sjkoshy if (caps & PMC_CAP_USER) 560145256Sjkoshy config |= AMD_PMC_USR; 561145256Sjkoshy if (caps & PMC_CAP_SYSTEM) 562145256Sjkoshy config |= AMD_PMC_OS; 563145256Sjkoshy if ((caps & (PMC_CAP_USER|PMC_CAP_SYSTEM)) == 0) 564145256Sjkoshy config |= (AMD_PMC_USR|AMD_PMC_OS); 565145256Sjkoshy 566145256Sjkoshy if (caps & PMC_CAP_EDGE) 567145256Sjkoshy config |= AMD_PMC_EDGE; 568145256Sjkoshy if (caps & PMC_CAP_INVERT) 569145256Sjkoshy config |= AMD_PMC_INVERT; 570145256Sjkoshy if (caps & PMC_CAP_INTERRUPT) 571145256Sjkoshy config |= AMD_PMC_INT; 572145256Sjkoshy 573145256Sjkoshy pm->pm_md.pm_amd.pm_amd_evsel = config; /* save config value */ 574145256Sjkoshy 575145256Sjkoshy PMCDBG(MDP,ALL,2,"amd-allocate ri=%d -> config=0x%x", ri, config); 576145256Sjkoshy 577145256Sjkoshy return 0; 578145256Sjkoshy} 579145256Sjkoshy 580145256Sjkoshy/* 581145256Sjkoshy * Release machine dependent state associated with a PMC. This is a 582145256Sjkoshy * no-op on this architecture. 583145256Sjkoshy * 584145256Sjkoshy */ 585145256Sjkoshy 586145256Sjkoshy/* ARGSUSED0 */ 587145256Sjkoshystatic int 588145256Sjkoshyamd_release_pmc(int cpu, int ri, struct pmc *pmc) 589145256Sjkoshy{ 590145256Sjkoshy#if DEBUG 591145256Sjkoshy const struct amd_descr *pd; 592145256Sjkoshy#endif 593145256Sjkoshy struct pmc_hw *phw; 594145256Sjkoshy 595145256Sjkoshy (void) pmc; 596145256Sjkoshy 597145256Sjkoshy KASSERT(cpu >= 0 && cpu < mp_ncpus, 598145256Sjkoshy ("[amd,%d] illegal CPU value %d", __LINE__, cpu)); 599145256Sjkoshy KASSERT(ri >= 0 && ri < AMD_NPMCS, 600145256Sjkoshy ("[amd,%d] illegal row-index %d", __LINE__, ri)); 601145256Sjkoshy 602145256Sjkoshy phw = pmc_pcpu[cpu]->pc_hwpmcs[ri]; 603145256Sjkoshy 604145256Sjkoshy KASSERT(phw->phw_pmc == NULL, 605145256Sjkoshy ("[amd,%d] PHW pmc %p non-NULL", __LINE__, phw->phw_pmc)); 606145256Sjkoshy 607145256Sjkoshy#if DEBUG 608145256Sjkoshy pd = &amd_pmcdesc[ri]; 609145256Sjkoshy if (pd->pm_descr.pd_class == AMD_PMC_CLASS) 610145256Sjkoshy KASSERT(AMD_PMC_IS_STOPPED(pd->pm_evsel), 611145256Sjkoshy ("[amd,%d] PMC %d released while active", __LINE__, ri)); 612145256Sjkoshy#endif 613145256Sjkoshy 614145256Sjkoshy return 0; 615145256Sjkoshy} 616145256Sjkoshy 617145256Sjkoshy/* 618145256Sjkoshy * start a PMC. 619145256Sjkoshy */ 620145256Sjkoshy 621145256Sjkoshystatic int 622145256Sjkoshyamd_start_pmc(int cpu, int ri) 623145256Sjkoshy{ 624145256Sjkoshy uint32_t config; 625145256Sjkoshy struct pmc *pm; 626145256Sjkoshy struct pmc_hw *phw; 627145256Sjkoshy const struct amd_descr *pd; 628145256Sjkoshy 629145256Sjkoshy KASSERT(cpu >= 0 && cpu < mp_ncpus, 630145256Sjkoshy ("[amd,%d] illegal CPU value %d", __LINE__, cpu)); 631145256Sjkoshy KASSERT(ri >= 0 && ri < AMD_NPMCS, 632145256Sjkoshy ("[amd,%d] illegal row-index %d", __LINE__, ri)); 633145256Sjkoshy 634145256Sjkoshy phw = pmc_pcpu[cpu]->pc_hwpmcs[ri]; 635145256Sjkoshy pm = phw->phw_pmc; 636145256Sjkoshy pd = &amd_pmcdesc[ri]; 637145256Sjkoshy 638145256Sjkoshy KASSERT(pm != NULL, 639145256Sjkoshy ("[amd,%d] starting cpu%d,pmc%d with null pmc record", __LINE__, 640145256Sjkoshy cpu, ri)); 641145256Sjkoshy 642145256Sjkoshy PMCDBG(MDP,STA,1,"amd-start cpu=%d ri=%d", cpu, ri); 643145256Sjkoshy 644145256Sjkoshy if (pd->pm_descr.pd_class == PMC_CLASS_TSC) 645145256Sjkoshy return 0; /* TSCs are always running */ 646145256Sjkoshy 647145256Sjkoshy KASSERT(pd->pm_descr.pd_class == AMD_PMC_CLASS, 648145256Sjkoshy ("[amd,%d] unknown PMC class (%d)", __LINE__, 649145256Sjkoshy pd->pm_descr.pd_class)); 650145256Sjkoshy 651145256Sjkoshy KASSERT(AMD_PMC_IS_STOPPED(pd->pm_evsel), 652145256Sjkoshy ("[amd,%d] pmc%d,cpu%d: Starting active PMC \"%s\"", __LINE__, 653145256Sjkoshy ri, cpu, pd->pm_descr.pd_name)); 654145256Sjkoshy 655145256Sjkoshy /* turn on the PMC ENABLE bit */ 656145256Sjkoshy config = pm->pm_md.pm_amd.pm_amd_evsel | AMD_PMC_ENABLE; 657145256Sjkoshy 658145256Sjkoshy PMCDBG(MDP,STA,2,"amd-start config=0x%x", config); 659145256Sjkoshy 660145256Sjkoshy wrmsr(pd->pm_evsel, config); 661145256Sjkoshy return 0; 662145256Sjkoshy} 663145256Sjkoshy 664145256Sjkoshy/* 665145256Sjkoshy * Stop a PMC. 666145256Sjkoshy */ 667145256Sjkoshy 668145256Sjkoshystatic int 669145256Sjkoshyamd_stop_pmc(int cpu, int ri) 670145256Sjkoshy{ 671145256Sjkoshy struct pmc *pm; 672145256Sjkoshy struct pmc_hw *phw; 673145256Sjkoshy const struct amd_descr *pd; 674145256Sjkoshy uint64_t config; 675145256Sjkoshy 676145256Sjkoshy KASSERT(cpu >= 0 && cpu < mp_ncpus, 677145256Sjkoshy ("[amd,%d] illegal CPU value %d", __LINE__, cpu)); 678145256Sjkoshy KASSERT(ri >= 0 && ri < AMD_NPMCS, 679145256Sjkoshy ("[amd,%d] illegal row-index %d", __LINE__, ri)); 680145256Sjkoshy 681145256Sjkoshy phw = pmc_pcpu[cpu]->pc_hwpmcs[ri]; 682145256Sjkoshy pm = phw->phw_pmc; 683145256Sjkoshy pd = &amd_pmcdesc[ri]; 684145256Sjkoshy 685145256Sjkoshy KASSERT(pm != NULL, 686145256Sjkoshy ("[amd,%d] cpu%d,pmc%d no PMC to stop", __LINE__, 687145256Sjkoshy cpu, ri)); 688145256Sjkoshy 689145256Sjkoshy /* can't stop a TSC */ 690145256Sjkoshy if (pd->pm_descr.pd_class == PMC_CLASS_TSC) 691145256Sjkoshy return 0; 692145256Sjkoshy 693145256Sjkoshy KASSERT(pd->pm_descr.pd_class == AMD_PMC_CLASS, 694145256Sjkoshy ("[amd,%d] unknown PMC class (%d)", __LINE__, 695145256Sjkoshy pd->pm_descr.pd_class)); 696145256Sjkoshy 697145256Sjkoshy KASSERT(!AMD_PMC_IS_STOPPED(pd->pm_evsel), 698145256Sjkoshy ("[amd,%d] PMC%d, CPU%d \"%s\" already stopped", 699145256Sjkoshy __LINE__, ri, cpu, pd->pm_descr.pd_name)); 700145256Sjkoshy 701145256Sjkoshy PMCDBG(MDP,STO,1,"amd-stop ri=%d", ri); 702145256Sjkoshy 703145256Sjkoshy /* turn off the PMC ENABLE bit */ 704145256Sjkoshy config = pm->pm_md.pm_amd.pm_amd_evsel & ~AMD_PMC_ENABLE; 705145256Sjkoshy wrmsr(pd->pm_evsel, config); 706145256Sjkoshy return 0; 707145256Sjkoshy} 708145256Sjkoshy 709145256Sjkoshy/* 710145256Sjkoshy * Interrupt handler. This function needs to return '1' if the 711145256Sjkoshy * interrupt was this CPU's PMCs or '0' otherwise. It is not allowed 712145256Sjkoshy * to sleep or do anything a 'fast' interrupt handler is not allowed 713145256Sjkoshy * to do. 714145256Sjkoshy */ 715145256Sjkoshy 716145256Sjkoshystatic int 717145256Sjkoshyamd_intr(int cpu, uintptr_t eip) 718145256Sjkoshy{ 719145256Sjkoshy int i, retval; 720145256Sjkoshy enum pmc_mode mode; 721145256Sjkoshy uint32_t perfctr; 722145256Sjkoshy struct pmc *pm; 723145256Sjkoshy struct pmc_cpu *pc; 724145256Sjkoshy struct pmc_hw *phw; 725145256Sjkoshy 726145256Sjkoshy KASSERT(cpu >= 0 && cpu < mp_ncpus, 727145256Sjkoshy ("[amd,%d] out of range CPU %d", __LINE__, cpu)); 728145256Sjkoshy 729145256Sjkoshy retval = 0; 730145256Sjkoshy 731145256Sjkoshy pc = pmc_pcpu[cpu]; 732145256Sjkoshy 733145256Sjkoshy /* 734145256Sjkoshy * look for all PMCs that have interrupted: 735145256Sjkoshy * - skip over the TSC [PMC#0] 736145256Sjkoshy * - look for a PMC with a valid 'struct pmc' association 737145256Sjkoshy * - look for a PMC in (a) sampling mode and (b) which has 738145256Sjkoshy * overflowed. If found, we update the process's 739145256Sjkoshy * histogram or send it a profiling signal by calling 740145256Sjkoshy * the appropriate helper function. 741145256Sjkoshy */ 742145256Sjkoshy 743145256Sjkoshy for (i = 1; i < AMD_NPMCS; i++) { 744145256Sjkoshy 745145256Sjkoshy phw = pc->pc_hwpmcs[i]; 746145256Sjkoshy perfctr = amd_pmcdesc[i].pm_perfctr; 747145256Sjkoshy KASSERT(phw != NULL, ("[amd,%d] null PHW pointer", __LINE__)); 748145256Sjkoshy 749145256Sjkoshy if ((pm = phw->phw_pmc) == NULL || 750145256Sjkoshy pm->pm_state != PMC_STATE_RUNNING) { 751145256Sjkoshy atomic_add_int(&pmc_stats.pm_intr_ignored, 1); 752145256Sjkoshy continue; 753145256Sjkoshy } 754145256Sjkoshy 755145256Sjkoshy mode = pm->pm_mode; 756145256Sjkoshy if (PMC_IS_SAMPLING_MODE(mode) && 757145256Sjkoshy AMD_PMC_HAS_OVERFLOWED(perfctr)) { 758145256Sjkoshy atomic_add_int(&pmc_stats.pm_intr_processed, 1); 759145256Sjkoshy if (PMC_IS_SYSTEM_MODE(mode)) 760145256Sjkoshy pmc_update_histogram(phw, eip); 761145256Sjkoshy else if (PMC_IS_VIRTUAL_MODE(mode)) 762145256Sjkoshy pmc_send_signal(pm); 763145256Sjkoshy retval = 1; 764145256Sjkoshy } 765145256Sjkoshy } 766145256Sjkoshy return retval; 767145256Sjkoshy} 768145256Sjkoshy 769145256Sjkoshy/* 770145256Sjkoshy * describe a PMC 771145256Sjkoshy */ 772145256Sjkoshystatic int 773145256Sjkoshyamd_describe(int cpu, int ri, struct pmc_info *pi, struct pmc **ppmc) 774145256Sjkoshy{ 775145256Sjkoshy int error; 776145256Sjkoshy size_t copied; 777145256Sjkoshy const struct amd_descr *pd; 778145256Sjkoshy struct pmc_hw *phw; 779145256Sjkoshy 780145256Sjkoshy KASSERT(cpu >= 0 && cpu < mp_ncpus, 781145256Sjkoshy ("[amd,%d] illegal CPU %d", __LINE__, cpu)); 782145256Sjkoshy KASSERT(ri >= 0 && ri < AMD_NPMCS, 783145256Sjkoshy ("[amd,%d] row-index %d out of range", __LINE__, ri)); 784145256Sjkoshy 785145256Sjkoshy phw = pmc_pcpu[cpu]->pc_hwpmcs[ri]; 786145256Sjkoshy pd = &amd_pmcdesc[ri]; 787145256Sjkoshy 788145256Sjkoshy if ((error = copystr(pd->pm_descr.pd_name, pi->pm_name, 789145256Sjkoshy PMC_NAME_MAX, &copied)) != 0) 790145256Sjkoshy return error; 791145256Sjkoshy 792145256Sjkoshy pi->pm_class = pd->pm_descr.pd_class; 793145256Sjkoshy pi->pm_caps = pd->pm_descr.pd_caps; 794145256Sjkoshy pi->pm_width = pd->pm_descr.pd_width; 795145256Sjkoshy 796145256Sjkoshy if (phw->phw_state & PMC_PHW_FLAG_IS_ENABLED) { 797145256Sjkoshy pi->pm_enabled = TRUE; 798145256Sjkoshy *ppmc = phw->phw_pmc; 799145256Sjkoshy } else { 800145256Sjkoshy pi->pm_enabled = FALSE; 801145256Sjkoshy *ppmc = NULL; 802145256Sjkoshy } 803145256Sjkoshy 804145256Sjkoshy return 0; 805145256Sjkoshy} 806145256Sjkoshy 807145256Sjkoshy/* 808145256Sjkoshy * i386 specific entry points 809145256Sjkoshy */ 810145256Sjkoshy 811145256Sjkoshy/* 812145256Sjkoshy * return the MSR address of the given PMC. 813145256Sjkoshy */ 814145256Sjkoshy 815145256Sjkoshystatic int 816145256Sjkoshyamd_get_msr(int ri, uint32_t *msr) 817145256Sjkoshy{ 818145256Sjkoshy KASSERT(ri >= 0 && ri < AMD_NPMCS, 819145256Sjkoshy ("[amd,%d] ri %d out of range", __LINE__, ri)); 820145256Sjkoshy 821145256Sjkoshy *msr = amd_pmcdesc[ri].pm_perfctr; 822145256Sjkoshy return 0; 823145256Sjkoshy} 824145256Sjkoshy 825145256Sjkoshy/* 826145256Sjkoshy * processor dependent initialization. 827145256Sjkoshy */ 828145256Sjkoshy 829145256Sjkoshy/* 830145256Sjkoshy * Per-processor data structure 831145256Sjkoshy * 832145256Sjkoshy * [common stuff] 833145256Sjkoshy * [5 struct pmc_hw pointers] 834145256Sjkoshy * [5 struct pmc_hw structures] 835145256Sjkoshy */ 836145256Sjkoshy 837145256Sjkoshystruct amd_cpu { 838145256Sjkoshy struct pmc_cpu pc_common; 839145256Sjkoshy struct pmc_hw *pc_hwpmcs[AMD_NPMCS]; 840145256Sjkoshy struct pmc_hw pc_amdpmcs[AMD_NPMCS]; 841145256Sjkoshy}; 842145256Sjkoshy 843145256Sjkoshy 844145256Sjkoshystatic int 845145256Sjkoshyamd_init(int cpu) 846145256Sjkoshy{ 847145256Sjkoshy int n; 848145256Sjkoshy struct amd_cpu *pcs; 849145256Sjkoshy struct pmc_hw *phw; 850145256Sjkoshy 851145256Sjkoshy KASSERT(cpu >= 0 && cpu < mp_ncpus, 852145256Sjkoshy ("[amd,%d] insane cpu number %d", __LINE__, cpu)); 853145256Sjkoshy 854145256Sjkoshy PMCDBG(MDP,INI,1,"amd-init cpu=%d", cpu); 855145256Sjkoshy 856145256Sjkoshy MALLOC(pcs, struct amd_cpu *, sizeof(struct amd_cpu), M_PMC, 857145256Sjkoshy M_WAITOK|M_ZERO); 858145256Sjkoshy 859145256Sjkoshy if (pcs == NULL) 860145256Sjkoshy return ENOMEM; 861145256Sjkoshy 862145256Sjkoshy phw = &pcs->pc_amdpmcs[0]; 863145256Sjkoshy 864145256Sjkoshy /* 865145256Sjkoshy * Initialize the per-cpu mutex and set the content of the 866145256Sjkoshy * hardware descriptors to a known state. 867145256Sjkoshy */ 868145256Sjkoshy 869145256Sjkoshy for (n = 0; n < AMD_NPMCS; n++, phw++) { 870145256Sjkoshy phw->phw_state = PMC_PHW_FLAG_IS_ENABLED | 871145256Sjkoshy PMC_PHW_CPU_TO_STATE(cpu) | PMC_PHW_INDEX_TO_STATE(n); 872145256Sjkoshy phw->phw_pmc = NULL; 873145256Sjkoshy pcs->pc_hwpmcs[n] = phw; 874145256Sjkoshy } 875145256Sjkoshy 876145256Sjkoshy /* Mark the TSC as shareable */ 877145256Sjkoshy pcs->pc_hwpmcs[0]->phw_state |= PMC_PHW_FLAG_IS_SHAREABLE; 878145256Sjkoshy 879145256Sjkoshy pmc_pcpu[cpu] = (struct pmc_cpu *) pcs; 880145256Sjkoshy 881145256Sjkoshy return 0; 882145256Sjkoshy} 883145256Sjkoshy 884145256Sjkoshy 885145256Sjkoshy/* 886145256Sjkoshy * processor dependent cleanup prior to the KLD 887145256Sjkoshy * being unloaded 888145256Sjkoshy */ 889145256Sjkoshy 890145256Sjkoshystatic int 891145256Sjkoshyamd_cleanup(int cpu) 892145256Sjkoshy{ 893145256Sjkoshy int i; 894145256Sjkoshy uint32_t evsel; 895145256Sjkoshy struct pmc_cpu *pcs; 896145256Sjkoshy 897145256Sjkoshy KASSERT(cpu >= 0 && cpu < mp_ncpus, 898145256Sjkoshy ("[amd,%d] insane cpu number (%d)", __LINE__, cpu)); 899145256Sjkoshy 900145256Sjkoshy PMCDBG(MDP,INI,1,"amd-cleanup cpu=%d", cpu); 901145256Sjkoshy 902145256Sjkoshy /* 903145256Sjkoshy * First, turn off all PMCs on this CPU. 904145256Sjkoshy */ 905145256Sjkoshy 906145256Sjkoshy for (i = 0; i < 4; i++) { /* XXX this loop is now not needed */ 907145256Sjkoshy evsel = rdmsr(AMD_PMC_EVSEL_0 + i); 908145256Sjkoshy evsel &= ~AMD_PMC_ENABLE; 909145256Sjkoshy wrmsr(AMD_PMC_EVSEL_0 + i, evsel); 910145256Sjkoshy } 911145256Sjkoshy 912145256Sjkoshy /* 913145256Sjkoshy * Next, free up allocated space. 914145256Sjkoshy */ 915145256Sjkoshy 916145256Sjkoshy pcs = pmc_pcpu[cpu]; 917145256Sjkoshy 918145256Sjkoshy#if DEBUG 919145256Sjkoshy /* check the TSC */ 920145256Sjkoshy KASSERT(pcs->pc_hwpmcs[0]->phw_pmc == NULL, 921145256Sjkoshy ("[amd,%d] CPU%d,PMC0 still in use", __LINE__, cpu)); 922145256Sjkoshy for (i = 1; i < AMD_NPMCS; i++) { 923145256Sjkoshy KASSERT(pcs->pc_hwpmcs[i]->phw_pmc == NULL, 924145256Sjkoshy ("[amd,%d] CPU%d/PMC%d in use", __LINE__, cpu, i)); 925145256Sjkoshy KASSERT(AMD_PMC_IS_STOPPED(AMD_PMC_EVSEL_0 + (i-1)), 926145256Sjkoshy ("[amd,%d] CPU%d/PMC%d not stopped", __LINE__, cpu, i)); 927145256Sjkoshy } 928145256Sjkoshy#endif 929145256Sjkoshy KASSERT(pcs != NULL, 930145256Sjkoshy ("[amd,%d] null per-cpu state pointer (cpu%d)", __LINE__, cpu)); 931145256Sjkoshy 932145256Sjkoshy pmc_pcpu[cpu] = NULL; 933145256Sjkoshy FREE(pcs, M_PMC); 934145256Sjkoshy return 0; 935145256Sjkoshy} 936145256Sjkoshy 937145256Sjkoshy/* 938145256Sjkoshy * Initialize ourselves. 939145256Sjkoshy */ 940145256Sjkoshy 941145256Sjkoshystruct pmc_mdep * 942145256Sjkoshypmc_amd_initialize(void) 943145256Sjkoshy{ 944145256Sjkoshy 945145256Sjkoshy struct pmc_mdep *pmc_mdep; 946145256Sjkoshy 947145256Sjkoshy /* The presence of hardware performance counters on the AMD 948145256Sjkoshy Athlon, Duron or later processors, is _not_ indicated by 949145256Sjkoshy any of the processor feature flags set by the 'CPUID' 950145256Sjkoshy instruction, so we only check the 'instruction family' 951145256Sjkoshy field returned by CPUID for instruction family >= 6. This 952145256Sjkoshy test needs to be be refined. */ 953145256Sjkoshy 954145256Sjkoshy if ((cpu_id & 0xF00) < 0x600) 955145256Sjkoshy return NULL; 956145256Sjkoshy 957145256Sjkoshy MALLOC(pmc_mdep, struct pmc_mdep *, sizeof(struct pmc_mdep), 958145256Sjkoshy M_PMC, M_WAITOK|M_ZERO); 959145256Sjkoshy 960145256Sjkoshy#if __i386__ 961145256Sjkoshy pmc_mdep->pmd_cputype = PMC_CPU_AMD_K7; 962145256Sjkoshy#elif __amd64__ 963145256Sjkoshy pmc_mdep->pmd_cputype = PMC_CPU_AMD_K8; 964145256Sjkoshy#else 965145256Sjkoshy#error Unknown AMD CPU type. 966145256Sjkoshy#endif 967145256Sjkoshy 968145256Sjkoshy pmc_mdep->pmd_npmc = AMD_NPMCS; 969145256Sjkoshy 970145256Sjkoshy /* this processor has two classes of usable PMCs */ 971145256Sjkoshy pmc_mdep->pmd_nclass = 2; 972145256Sjkoshy pmc_mdep->pmd_classes[0] = PMC_CLASS_TSC; 973145256Sjkoshy pmc_mdep->pmd_classes[1] = AMD_PMC_CLASS; 974145256Sjkoshy pmc_mdep->pmd_nclasspmcs[0] = 1; 975145256Sjkoshy pmc_mdep->pmd_nclasspmcs[1] = (AMD_NPMCS-1); 976145256Sjkoshy 977145256Sjkoshy pmc_mdep->pmd_init = amd_init; 978145256Sjkoshy pmc_mdep->pmd_cleanup = amd_cleanup; 979145256Sjkoshy pmc_mdep->pmd_switch_in = amd_switch_in; 980145256Sjkoshy pmc_mdep->pmd_switch_out = amd_switch_out; 981145256Sjkoshy pmc_mdep->pmd_read_pmc = amd_read_pmc; 982145256Sjkoshy pmc_mdep->pmd_write_pmc = amd_write_pmc; 983145256Sjkoshy pmc_mdep->pmd_config_pmc = amd_config_pmc; 984145256Sjkoshy pmc_mdep->pmd_allocate_pmc = amd_allocate_pmc; 985145256Sjkoshy pmc_mdep->pmd_release_pmc = amd_release_pmc; 986145256Sjkoshy pmc_mdep->pmd_start_pmc = amd_start_pmc; 987145256Sjkoshy pmc_mdep->pmd_stop_pmc = amd_stop_pmc; 988145256Sjkoshy pmc_mdep->pmd_intr = amd_intr; 989145256Sjkoshy pmc_mdep->pmd_describe = amd_describe; 990145256Sjkoshy pmc_mdep->pmd_get_msr = amd_get_msr; /* i386 */ 991145256Sjkoshy 992145256Sjkoshy PMCDBG(MDP,INI,0,"%s","amd-initialize"); 993145256Sjkoshy 994145256Sjkoshy return pmc_mdep; 995145256Sjkoshy} 996