hwpmc_amd.c revision 146799
1145256Sjkoshy/*- 2145256Sjkoshy * Copyright (c) 2003-2005 Joseph Koshy 3145256Sjkoshy * All rights reserved. 4145256Sjkoshy * 5145256Sjkoshy * Redistribution and use in source and binary forms, with or without 6145256Sjkoshy * modification, are permitted provided that the following conditions 7145256Sjkoshy * are met: 8145256Sjkoshy * 1. Redistributions of source code must retain the above copyright 9145256Sjkoshy * notice, this list of conditions and the following disclaimer. 10145256Sjkoshy * 2. Redistributions in binary form must reproduce the above copyright 11145256Sjkoshy * notice, this list of conditions and the following disclaimer in the 12145256Sjkoshy * documentation and/or other materials provided with the distribution. 13145256Sjkoshy * 14145256Sjkoshy * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 15145256Sjkoshy * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 16145256Sjkoshy * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 17145256Sjkoshy * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 18145256Sjkoshy * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 19145256Sjkoshy * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 20145256Sjkoshy * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 21145256Sjkoshy * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 22145256Sjkoshy * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 23145256Sjkoshy * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 24145256Sjkoshy * SUCH DAMAGE. 25145256Sjkoshy * 26145256Sjkoshy */ 27145256Sjkoshy 28145256Sjkoshy#include <sys/cdefs.h> 29145256Sjkoshy__FBSDID("$FreeBSD: head/sys/dev/hwpmc/hwpmc_amd.c 146799 2005-05-30 06:29:29Z jkoshy $"); 30145256Sjkoshy 31145256Sjkoshy/* Support for the AMD K7 and later processors */ 32145256Sjkoshy 33145256Sjkoshy#include <sys/param.h> 34145256Sjkoshy#include <sys/lock.h> 35145256Sjkoshy#include <sys/malloc.h> 36145256Sjkoshy#include <sys/mutex.h> 37145338Smarcel#include <sys/pmc.h> 38145256Sjkoshy#include <sys/smp.h> 39145256Sjkoshy#include <sys/systm.h> 40145256Sjkoshy 41145256Sjkoshy#include <machine/md_var.h> 42145256Sjkoshy 43145256Sjkoshy/* AMD K7 and K8 PMCs */ 44145256Sjkoshy 45145256Sjkoshy#define AMD_PMC_EVSEL_0 0xC0010000 46145256Sjkoshy#define AMD_PMC_EVSEL_1 0xC0010001 47145256Sjkoshy#define AMD_PMC_EVSEL_2 0xC0010002 48145256Sjkoshy#define AMD_PMC_EVSEL_3 0xC0010003 49145256Sjkoshy 50145256Sjkoshy#define AMD_PMC_PERFCTR_0 0xC0010004 51145256Sjkoshy#define AMD_PMC_PERFCTR_1 0xC0010005 52145256Sjkoshy#define AMD_PMC_PERFCTR_2 0xC0010006 53145256Sjkoshy#define AMD_PMC_PERFCTR_3 0xC0010007 54145256Sjkoshy 55145256Sjkoshy#define K7_VALID_EVENT_CODE(c) (((c) >= 0x40 && (c) <= 0x47) || \ 56145256Sjkoshy ((c) >= 0x80 && (c) <= 0x85) || ((c) >= 0xC0 && (c) <= 0xC7) || \ 57145256Sjkoshy ((c) >= 0xCD && (c) <= 0xCF)) 58145256Sjkoshy 59145256Sjkoshy#define AMD_PMC_CAPS (PMC_CAP_INTERRUPT | PMC_CAP_USER | \ 60145256Sjkoshy PMC_CAP_SYSTEM | PMC_CAP_EDGE | PMC_CAP_THRESHOLD | \ 61145256Sjkoshy PMC_CAP_READ | PMC_CAP_WRITE | PMC_CAP_INVERT | PMC_CAP_QUALIFIER) 62145256Sjkoshy 63145256Sjkoshy/* reserved bits include bit 21 and the top two bits of the unit mask */ 64145256Sjkoshy#define K7_PMC_RESERVED ((1 << 21) | (3 << 13)) 65145256Sjkoshy 66145256Sjkoshy#define K8_PMC_RESERVED (1 << 21) 67145256Sjkoshy 68145256Sjkoshy#define AMD_PMC_IS_STOPPED(evsel) ((rdmsr((evsel)) & AMD_PMC_ENABLE) == 0) 69145256Sjkoshy#define AMD_PMC_HAS_OVERFLOWED(pmc) ((rdpmc(pmc) & (1ULL << 47)) == 0) 70145256Sjkoshy 71145256Sjkoshy#if __i386__ 72145256Sjkoshy#define AMD_NPMCS K7_NPMCS 73145256Sjkoshy#define AMD_PMC_CLASS PMC_CLASS_K7 74145256Sjkoshy#define AMD_PMC_COUNTERMASK K7_PMC_COUNTERMASK 75145256Sjkoshy#define AMD_PMC_TO_COUNTER(x) K7_PMC_TO_COUNTER(x) 76145256Sjkoshy#define AMD_PMC_INVERT K7_PMC_INVERT 77145256Sjkoshy#define AMD_PMC_ENABLE K7_PMC_ENABLE 78145256Sjkoshy#define AMD_PMC_INT K7_PMC_INT 79145256Sjkoshy#define AMD_PMC_PC K7_PMC_PC 80145256Sjkoshy#define AMD_PMC_EDGE K7_PMC_EDGE 81145256Sjkoshy#define AMD_PMC_OS K7_PMC_OS 82145256Sjkoshy#define AMD_PMC_USR K7_PMC_USR 83145256Sjkoshy 84145256Sjkoshy#define AMD_PMC_UNITMASK_M K7_PMC_UNITMASK_M 85145256Sjkoshy#define AMD_PMC_UNITMASK_O K7_PMC_UNITMASK_O 86145256Sjkoshy#define AMD_PMC_UNITMASK_E K7_PMC_UNITMASK_E 87145256Sjkoshy#define AMD_PMC_UNITMASK_S K7_PMC_UNITMASK_S 88145256Sjkoshy#define AMD_PMC_UNITMASK_I K7_PMC_UNITMASK_I 89145256Sjkoshy 90145256Sjkoshy#define AMD_PMC_UNITMASK K7_PMC_UNITMASK 91145256Sjkoshy#define AMD_PMC_EVENTMASK K7_PMC_EVENTMASK 92145256Sjkoshy#define AMD_PMC_TO_UNITMASK(x) K7_PMC_TO_UNITMASK(x) 93145256Sjkoshy#define AMD_PMC_TO_EVENTMASK(x) K7_PMC_TO_EVENTMASK(x) 94145256Sjkoshy#define AMD_VALID_BITS K7_VALID_BITS 95145256Sjkoshy 96145256Sjkoshy#define AMD_PMC_CLASS_NAME "K7-" 97145256Sjkoshy 98145256Sjkoshy#elif __amd64__ 99145256Sjkoshy 100145256Sjkoshy#define AMD_NPMCS K8_NPMCS 101145256Sjkoshy#define AMD_PMC_CLASS PMC_CLASS_K8 102145256Sjkoshy#define AMD_PMC_COUNTERMASK K8_PMC_COUNTERMASK 103145256Sjkoshy#define AMD_PMC_TO_COUNTER(x) K8_PMC_TO_COUNTER(x) 104145256Sjkoshy#define AMD_PMC_INVERT K8_PMC_INVERT 105145256Sjkoshy#define AMD_PMC_ENABLE K8_PMC_ENABLE 106145256Sjkoshy#define AMD_PMC_INT K8_PMC_INT 107145256Sjkoshy#define AMD_PMC_PC K8_PMC_PC 108145256Sjkoshy#define AMD_PMC_EDGE K8_PMC_EDGE 109145256Sjkoshy#define AMD_PMC_OS K8_PMC_OS 110145256Sjkoshy#define AMD_PMC_USR K8_PMC_USR 111145256Sjkoshy 112145256Sjkoshy#define AMD_PMC_UNITMASK_M K8_PMC_UNITMASK_M 113145256Sjkoshy#define AMD_PMC_UNITMASK_O K8_PMC_UNITMASK_O 114145256Sjkoshy#define AMD_PMC_UNITMASK_E K8_PMC_UNITMASK_E 115145256Sjkoshy#define AMD_PMC_UNITMASK_S K8_PMC_UNITMASK_S 116145256Sjkoshy#define AMD_PMC_UNITMASK_I K8_PMC_UNITMASK_I 117145256Sjkoshy 118145256Sjkoshy#define AMD_PMC_UNITMASK K8_PMC_UNITMASK 119145256Sjkoshy#define AMD_PMC_EVENTMASK K8_PMC_EVENTMASK 120145256Sjkoshy#define AMD_PMC_TO_UNITMASK(x) K8_PMC_TO_UNITMASK(x) 121145256Sjkoshy#define AMD_PMC_TO_EVENTMASK(x) K8_PMC_TO_EVENTMASK(x) 122145256Sjkoshy#define AMD_VALID_BITS K8_VALID_BITS 123145256Sjkoshy 124145256Sjkoshy#define AMD_PMC_CLASS_NAME "K8-" 125145256Sjkoshy 126145256Sjkoshy#else 127145256Sjkoshy#error Unsupported architecture. 128145256Sjkoshy#endif 129145256Sjkoshy 130145256Sjkoshy/* AMD K7 & K8 PMCs */ 131145256Sjkoshystruct amd_descr { 132145256Sjkoshy struct pmc_descr pm_descr; /* "base class" */ 133145256Sjkoshy uint32_t pm_evsel; /* address of EVSEL register */ 134145256Sjkoshy uint32_t pm_perfctr; /* address of PERFCTR register */ 135145256Sjkoshy}; 136145256Sjkoshy 137145256Sjkoshystatic const struct amd_descr amd_pmcdesc[AMD_NPMCS] = 138145256Sjkoshy{ 139145256Sjkoshy { 140145256Sjkoshy .pm_descr = 141145256Sjkoshy { 142145256Sjkoshy .pd_name = "TSC", 143145256Sjkoshy .pd_class = PMC_CLASS_TSC, 144145256Sjkoshy .pd_caps = PMC_CAP_READ, 145145256Sjkoshy .pd_width = 64 146145256Sjkoshy }, 147145256Sjkoshy .pm_evsel = MSR_TSC, 148145256Sjkoshy .pm_perfctr = 0 /* unused */ 149145256Sjkoshy }, 150145256Sjkoshy 151145256Sjkoshy { 152145256Sjkoshy .pm_descr = 153145256Sjkoshy { 154145256Sjkoshy .pd_name = AMD_PMC_CLASS_NAME "0", 155145256Sjkoshy .pd_class = AMD_PMC_CLASS, 156145256Sjkoshy .pd_caps = AMD_PMC_CAPS, 157145256Sjkoshy .pd_width = 48 158145256Sjkoshy }, 159145256Sjkoshy .pm_evsel = AMD_PMC_EVSEL_0, 160145256Sjkoshy .pm_perfctr = AMD_PMC_PERFCTR_0 161145256Sjkoshy }, 162145256Sjkoshy { 163145256Sjkoshy .pm_descr = 164145256Sjkoshy { 165145256Sjkoshy .pd_name = AMD_PMC_CLASS_NAME "1", 166145256Sjkoshy .pd_class = AMD_PMC_CLASS, 167145256Sjkoshy .pd_caps = AMD_PMC_CAPS, 168145256Sjkoshy .pd_width = 48 169145256Sjkoshy }, 170145256Sjkoshy .pm_evsel = AMD_PMC_EVSEL_1, 171145256Sjkoshy .pm_perfctr = AMD_PMC_PERFCTR_1 172145256Sjkoshy }, 173145256Sjkoshy { 174145256Sjkoshy .pm_descr = 175145256Sjkoshy { 176145256Sjkoshy .pd_name = AMD_PMC_CLASS_NAME "2", 177145256Sjkoshy .pd_class = AMD_PMC_CLASS, 178145256Sjkoshy .pd_caps = AMD_PMC_CAPS, 179145256Sjkoshy .pd_width = 48 180145256Sjkoshy }, 181145256Sjkoshy .pm_evsel = AMD_PMC_EVSEL_2, 182145256Sjkoshy .pm_perfctr = AMD_PMC_PERFCTR_2 183145256Sjkoshy }, 184145256Sjkoshy { 185145256Sjkoshy .pm_descr = 186145256Sjkoshy { 187145256Sjkoshy .pd_name = AMD_PMC_CLASS_NAME "3", 188145256Sjkoshy .pd_class = AMD_PMC_CLASS, 189145256Sjkoshy .pd_caps = AMD_PMC_CAPS, 190145256Sjkoshy .pd_width = 48 191145256Sjkoshy }, 192145256Sjkoshy .pm_evsel = AMD_PMC_EVSEL_3, 193145256Sjkoshy .pm_perfctr = AMD_PMC_PERFCTR_3 194145256Sjkoshy } 195145256Sjkoshy}; 196145256Sjkoshy 197145256Sjkoshystruct amd_event_code_map { 198145256Sjkoshy enum pmc_event pe_ev; /* enum value */ 199145256Sjkoshy uint8_t pe_code; /* encoded event mask */ 200145256Sjkoshy uint8_t pe_mask; /* bits allowed in unit mask */ 201145256Sjkoshy}; 202145256Sjkoshy 203145256Sjkoshyconst struct amd_event_code_map amd_event_codes[] = { 204145256Sjkoshy#if __i386__ 205145256Sjkoshy { PMC_EV_K7_DC_ACCESSES, 0x40, 0 }, 206145256Sjkoshy { PMC_EV_K7_DC_MISSES, 0x41, 0 }, 207145256Sjkoshy { PMC_EV_K7_DC_REFILLS_FROM_L2, 0x42, K7_PMC_UNITMASK_MOESI }, 208145256Sjkoshy { PMC_EV_K7_DC_REFILLS_FROM_SYSTEM, 0x43, K7_PMC_UNITMASK_MOESI }, 209145256Sjkoshy { PMC_EV_K7_DC_WRITEBACKS, 0x44, K7_PMC_UNITMASK_MOESI }, 210145256Sjkoshy { PMC_EV_K7_L1_DTLB_MISS_AND_L2_DTLB_HITS, 0x45, 0 }, 211145256Sjkoshy { PMC_EV_K7_L1_AND_L2_DTLB_MISSES, 0x46, 0 }, 212145256Sjkoshy { PMC_EV_K7_MISALIGNED_REFERENCES, 0x47, 0 }, 213145256Sjkoshy 214145256Sjkoshy { PMC_EV_K7_IC_FETCHES, 0x80, 0 }, 215145256Sjkoshy { PMC_EV_K7_IC_MISSES, 0x81, 0 }, 216145256Sjkoshy 217145256Sjkoshy { PMC_EV_K7_L1_ITLB_MISSES, 0x84, 0 }, 218145256Sjkoshy { PMC_EV_K7_L1_L2_ITLB_MISSES, 0x85, 0 }, 219145256Sjkoshy 220145256Sjkoshy { PMC_EV_K7_RETIRED_INSTRUCTIONS, 0xC0, 0 }, 221145256Sjkoshy { PMC_EV_K7_RETIRED_OPS, 0xC1, 0 }, 222145256Sjkoshy { PMC_EV_K7_RETIRED_BRANCHES, 0xC2, 0 }, 223145256Sjkoshy { PMC_EV_K7_RETIRED_BRANCHES_MISPREDICTED, 0xC3, 0 }, 224145256Sjkoshy { PMC_EV_K7_RETIRED_TAKEN_BRANCHES, 0xC4, 0 }, 225145256Sjkoshy { PMC_EV_K7_RETIRED_TAKEN_BRANCHES_MISPREDICTED, 0xC5, 0 }, 226145256Sjkoshy { PMC_EV_K7_RETIRED_FAR_CONTROL_TRANSFERS, 0xC6, 0 }, 227145256Sjkoshy { PMC_EV_K7_RETIRED_RESYNC_BRANCHES, 0xC7, 0 }, 228145256Sjkoshy { PMC_EV_K7_INTERRUPTS_MASKED_CYCLES, 0xCD, 0 }, 229145256Sjkoshy { PMC_EV_K7_INTERRUPTS_MASKED_WHILE_PENDING_CYCLES, 0xCE, 0 }, 230145256Sjkoshy { PMC_EV_K7_HARDWARE_INTERRUPTS, 0xCF, 0 } 231145256Sjkoshy#endif 232145256Sjkoshy 233145256Sjkoshy#if __amd64__ 234145256Sjkoshy { PMC_EV_K8_FP_DISPATCHED_FPU_OPS, 0x00, 0x3F }, 235145256Sjkoshy { PMC_EV_K8_FP_CYCLES_WITH_NO_FPU_OPS_RETIRED, 0x01, 0x00 }, 236145256Sjkoshy { PMC_EV_K8_FP_DISPATCHED_FPU_FAST_FLAG_OPS, 0x02, 0x00 }, 237145256Sjkoshy 238145256Sjkoshy { PMC_EV_K8_LS_SEGMENT_REGISTER_LOAD, 0x20, 0x7F }, 239145256Sjkoshy { PMC_EV_K8_LS_MICROARCHITECTURAL_RESYNC_BY_SELF_MODIFYING_CODE, 240145256Sjkoshy 0x21, 0x00 }, 241145256Sjkoshy { PMC_EV_K8_LS_MICROARCHITECTURAL_RESYNC_BY_SNOOP, 0x22, 0x00 }, 242145256Sjkoshy { PMC_EV_K8_LS_BUFFER2_FULL, 0x23, 0x00 }, 243145256Sjkoshy { PMC_EV_K8_LS_LOCKED_OPERATION, 0x24, 0x07 }, 244145256Sjkoshy { PMC_EV_K8_LS_MICROARCHITECTURAL_LATE_CANCEL, 0x25, 0x00 }, 245145256Sjkoshy { PMC_EV_K8_LS_RETIRED_CFLUSH_INSTRUCTIONS, 0x26, 0x00 }, 246145256Sjkoshy { PMC_EV_K8_LS_RETIRED_CPUID_INSTRUCTIONS, 0x27, 0x00 }, 247145256Sjkoshy 248145256Sjkoshy { PMC_EV_K8_DC_ACCESS, 0x40, 0x00 }, 249145256Sjkoshy { PMC_EV_K8_DC_MISS, 0x41, 0x00 }, 250145256Sjkoshy { PMC_EV_K8_DC_REFILL_FROM_L2, 0x42, 0x1F }, 251145256Sjkoshy { PMC_EV_K8_DC_REFILL_FROM_SYSTEM, 0x43, 0x1F }, 252145256Sjkoshy { PMC_EV_K8_DC_COPYBACK, 0x44, 0x1F }, 253145256Sjkoshy { PMC_EV_K8_DC_L1_DTLB_MISS_AND_L2_DTLB_HIT, 0x45, 0x00 }, 254145256Sjkoshy { PMC_EV_K8_DC_L1_DTLB_MISS_AND_L2_DTLB_MISS, 0x46, 0x00 }, 255145256Sjkoshy { PMC_EV_K8_DC_MISALIGNED_DATA_REFERENCE, 0x47, 0x00 }, 256145256Sjkoshy { PMC_EV_K8_DC_MICROARCHITECTURAL_LATE_CANCEL, 0x48, 0x00 }, 257145256Sjkoshy { PMC_EV_K8_DC_MICROARCHITECTURAL_EARLY_CANCEL, 0x49, 0x00 }, 258145256Sjkoshy { PMC_EV_K8_DC_ONE_BIT_ECC_ERROR, 0x4A, 0x03 }, 259145256Sjkoshy { PMC_EV_K8_DC_DISPATCHED_PREFETCH_INSTRUCTIONS, 0x4B, 0x07 }, 260145256Sjkoshy { PMC_EV_K8_DC_DCACHE_ACCESSES_BY_LOCKS, 0x4C, 0x03 }, 261145256Sjkoshy 262145256Sjkoshy { PMC_EV_K8_BU_CPU_CLK_UNHALTED, 0x76, 0x00 }, 263145256Sjkoshy { PMC_EV_K8_BU_INTERNAL_L2_REQUEST, 0x7D, 0x1F }, 264145256Sjkoshy { PMC_EV_K8_BU_FILL_REQUEST_L2_MISS, 0x7E, 0x07 }, 265145256Sjkoshy { PMC_EV_K8_BU_FILL_INTO_L2, 0x7F, 0x03 }, 266145256Sjkoshy 267145256Sjkoshy { PMC_EV_K8_IC_FETCH, 0x80, 0x00 }, 268145256Sjkoshy { PMC_EV_K8_IC_MISS, 0x81, 0x00 }, 269145256Sjkoshy { PMC_EV_K8_IC_REFILL_FROM_L2, 0x82, 0x00 }, 270145256Sjkoshy { PMC_EV_K8_IC_REFILL_FROM_SYSTEM, 0x83, 0x00 }, 271145256Sjkoshy { PMC_EV_K8_IC_L1_ITLB_MISS_AND_L2_ITLB_HIT, 0x84, 0x00 }, 272145256Sjkoshy { PMC_EV_K8_IC_L1_ITLB_MISS_AND_L2_ITLB_MISS, 0x85, 0x00 }, 273145256Sjkoshy { PMC_EV_K8_IC_MICROARCHITECTURAL_RESYNC_BY_SNOOP, 0x86, 0x00 }, 274145256Sjkoshy { PMC_EV_K8_IC_INSTRUCTION_FETCH_STALL, 0x87, 0x00 }, 275145256Sjkoshy { PMC_EV_K8_IC_RETURN_STACK_HIT, 0x88, 0x00 }, 276145256Sjkoshy { PMC_EV_K8_IC_RETURN_STACK_OVERFLOW, 0x89, 0x00 }, 277145256Sjkoshy 278145256Sjkoshy { PMC_EV_K8_FR_RETIRED_X86_INSTRUCTIONS, 0xC0, 0x00 }, 279145256Sjkoshy { PMC_EV_K8_FR_RETIRED_UOPS, 0xC1, 0x00 }, 280145256Sjkoshy { PMC_EV_K8_FR_RETIRED_BRANCHES, 0xC2, 0x00 }, 281145256Sjkoshy { PMC_EV_K8_FR_RETIRED_BRANCHES_MISPREDICTED, 0xC3, 0x00 }, 282145256Sjkoshy { PMC_EV_K8_FR_RETIRED_TAKEN_BRANCHES, 0xC4, 0x00 }, 283145256Sjkoshy { PMC_EV_K8_FR_RETIRED_TAKEN_BRANCHES_MISPREDICTED, 0xC5, 0x00 }, 284145256Sjkoshy { PMC_EV_K8_FR_RETIRED_FAR_CONTROL_TRANSFERS, 0xC6, 0x00 }, 285145256Sjkoshy { PMC_EV_K8_FR_RETIRED_RESYNCS, 0xC7, 0x00 }, 286145256Sjkoshy { PMC_EV_K8_FR_RETIRED_NEAR_RETURNS, 0xC8, 0x00 }, 287145256Sjkoshy { PMC_EV_K8_FR_RETIRED_NEAR_RETURNS_MISPREDICTED, 0xC9, 0x00 }, 288145256Sjkoshy { PMC_EV_K8_FR_RETIRED_TAKEN_BRANCHES_MISPREDICTED_BY_ADDR_MISCOMPARE, 289145256Sjkoshy 0xCA, 0x00 }, 290145256Sjkoshy { PMC_EV_K8_FR_RETIRED_FPU_INSTRUCTIONS, 0xCB, 0x0F }, 291145256Sjkoshy { PMC_EV_K8_FR_RETIRED_FASTPATH_DOUBLE_OP_INSTRUCTIONS, 292145256Sjkoshy 0xCC, 0x07 }, 293145256Sjkoshy { PMC_EV_K8_FR_INTERRUPTS_MASKED_CYCLES, 0xCD, 0x00 }, 294145256Sjkoshy { PMC_EV_K8_FR_INTERRUPTS_MASKED_WHILE_PENDING_CYCLES, 0xCE, 0x00 }, 295145256Sjkoshy { PMC_EV_K8_FR_TAKEN_HARDWARE_INTERRUPTS, 0xCF, 0x00 }, 296145256Sjkoshy 297145256Sjkoshy { PMC_EV_K8_FR_DECODER_EMPTY, 0xD0, 0x00 }, 298145256Sjkoshy { PMC_EV_K8_FR_DISPATCH_STALLS, 0xD1, 0x00 }, 299145256Sjkoshy { PMC_EV_K8_FR_DISPATCH_STALL_FROM_BRANCH_ABORT_TO_RETIRE, 300145256Sjkoshy 0xD2, 0x00 }, 301145256Sjkoshy { PMC_EV_K8_FR_DISPATCH_STALL_FOR_SERIALIZATION, 0xD3, 0x00 }, 302145256Sjkoshy { PMC_EV_K8_FR_DISPATCH_STALL_FOR_SEGMENT_LOAD, 0xD4, 0x00 }, 303145256Sjkoshy { PMC_EV_K8_FR_DISPATCH_STALL_WHEN_REORDER_BUFFER_IS_FULL, 304145256Sjkoshy 0xD5, 0x00 }, 305145256Sjkoshy { PMC_EV_K8_FR_DISPATCH_STALL_WHEN_RESERVATION_STATIONS_ARE_FULL, 306145256Sjkoshy 0xD6, 0x00 }, 307145256Sjkoshy { PMC_EV_K8_FR_DISPATCH_STALL_WHEN_FPU_IS_FULL, 0xD7, 0x00 }, 308145256Sjkoshy { PMC_EV_K8_FR_DISPATCH_STALL_WHEN_LS_IS_FULL, 0xD8, 0x00 }, 309145256Sjkoshy { PMC_EV_K8_FR_DISPATCH_STALL_WHEN_WAITING_FOR_ALL_TO_BE_QUIET, 310145256Sjkoshy 0xD9, 0x00 }, 311145256Sjkoshy { PMC_EV_K8_FR_DISPATCH_STALL_WHEN_FAR_XFER_OR_RESYNC_BRANCH_PENDING, 312145256Sjkoshy 0xDA, 0x00 }, 313145256Sjkoshy { PMC_EV_K8_FR_FPU_EXCEPTIONS, 0xDB, 0x0F }, 314145256Sjkoshy { PMC_EV_K8_FR_NUMBER_OF_BREAKPOINTS_FOR_DR0, 0xDC, 0x00 }, 315145256Sjkoshy { PMC_EV_K8_FR_NUMBER_OF_BREAKPOINTS_FOR_DR1, 0xDD, 0x00 }, 316145256Sjkoshy { PMC_EV_K8_FR_NUMBER_OF_BREAKPOINTS_FOR_DR2, 0xDE, 0x00 }, 317145256Sjkoshy { PMC_EV_K8_FR_NUMBER_OF_BREAKPOINTS_FOR_DR3, 0xDF, 0x00 }, 318145256Sjkoshy 319145256Sjkoshy { PMC_EV_K8_NB_MEMORY_CONTROLLER_PAGE_ACCESS_EVENT, 0xE0, 0x7 }, 320145256Sjkoshy { PMC_EV_K8_NB_MEMORY_CONTROLLER_PAGE_TABLE_OVERFLOW, 0xE1, 0x00 }, 321145256Sjkoshy { PMC_EV_K8_NB_MEMORY_CONTROLLER_DRAM_COMMAND_SLOTS_MISSED, 322145256Sjkoshy 0xE2, 0x00 }, 323145256Sjkoshy { PMC_EV_K8_NB_MEMORY_CONTROLLER_TURNAROUND, 0xE3, 0x07 }, 324145256Sjkoshy { PMC_EV_K8_NB_MEMORY_CONTROLLER_BYPASS_SATURATION, 0xE4, 0x0F }, 325145256Sjkoshy { PMC_EV_K8_NB_SIZED_COMMANDS, 0xEB, 0x7F }, 326145256Sjkoshy { PMC_EV_K8_NB_PROBE_RESULT, 0xEC, 0x0F }, 327145256Sjkoshy { PMC_EV_K8_NB_HT_BUS0_BANDWIDTH, 0xF6, 0x0F }, 328145256Sjkoshy { PMC_EV_K8_NB_HT_BUS1_BANDWIDTH, 0xF7, 0x0F }, 329145256Sjkoshy { PMC_EV_K8_NB_HT_BUS2_BANDWIDTH, 0xF8, 0x0F } 330145256Sjkoshy#endif 331145256Sjkoshy 332145256Sjkoshy}; 333145256Sjkoshy 334145256Sjkoshyconst int amd_event_codes_size = 335145256Sjkoshy sizeof(amd_event_codes) / sizeof(amd_event_codes[0]); 336145256Sjkoshy 337145256Sjkoshy/* 338145256Sjkoshy * read a pmc register 339145256Sjkoshy */ 340145256Sjkoshy 341145256Sjkoshystatic int 342145256Sjkoshyamd_read_pmc(int cpu, int ri, pmc_value_t *v) 343145256Sjkoshy{ 344145256Sjkoshy enum pmc_mode mode; 345145256Sjkoshy const struct amd_descr *pd; 346145256Sjkoshy struct pmc *pm; 347145256Sjkoshy const struct pmc_hw *phw; 348145256Sjkoshy pmc_value_t tmp; 349145256Sjkoshy 350145256Sjkoshy KASSERT(cpu >= 0 && cpu < mp_ncpus, 351145256Sjkoshy ("[amd,%d] illegal CPU value %d", __LINE__, cpu)); 352145256Sjkoshy KASSERT(ri >= 0 && ri < AMD_NPMCS, 353145256Sjkoshy ("[amd,%d] illegal row-index %d", __LINE__, ri)); 354145256Sjkoshy 355145256Sjkoshy phw = pmc_pcpu[cpu]->pc_hwpmcs[ri]; 356145256Sjkoshy pd = &amd_pmcdesc[ri]; 357145256Sjkoshy pm = phw->phw_pmc; 358145256Sjkoshy 359145256Sjkoshy KASSERT(pm != NULL, 360145256Sjkoshy ("[amd,%d] No owner for HWPMC [cpu%d,pmc%d]", __LINE__, 361145256Sjkoshy cpu, ri)); 362145256Sjkoshy 363145774Sjkoshy mode = PMC_TO_MODE(pm); 364145256Sjkoshy 365145256Sjkoshy PMCDBG(MDP,REA,1,"amd-read id=%d class=%d", ri, pd->pm_descr.pd_class); 366145256Sjkoshy 367145256Sjkoshy /* Reading the TSC is a special case */ 368145256Sjkoshy if (pd->pm_descr.pd_class == PMC_CLASS_TSC) { 369145256Sjkoshy KASSERT(PMC_IS_COUNTING_MODE(mode), 370145256Sjkoshy ("[amd,%d] TSC counter in non-counting mode", __LINE__)); 371145256Sjkoshy *v = rdtsc(); 372145256Sjkoshy PMCDBG(MDP,REA,2,"amd-read id=%d -> %jd", ri, *v); 373145256Sjkoshy return 0; 374145256Sjkoshy } 375145256Sjkoshy 376145256Sjkoshy KASSERT(pd->pm_descr.pd_class == AMD_PMC_CLASS, 377145256Sjkoshy ("[amd,%d] unknown PMC class (%d)", __LINE__, 378145256Sjkoshy pd->pm_descr.pd_class)); 379145256Sjkoshy 380145256Sjkoshy tmp = rdmsr(pd->pm_perfctr); /* RDMSR serializes */ 381145256Sjkoshy if (PMC_IS_SAMPLING_MODE(mode)) 382145256Sjkoshy *v = -tmp; 383145256Sjkoshy else 384145256Sjkoshy *v = tmp; 385145256Sjkoshy 386145256Sjkoshy PMCDBG(MDP,REA,2,"amd-read id=%d -> %jd", ri, *v); 387145256Sjkoshy 388145256Sjkoshy return 0; 389145256Sjkoshy} 390145256Sjkoshy 391145256Sjkoshy/* 392145256Sjkoshy * Write a PMC MSR. 393145256Sjkoshy */ 394145256Sjkoshy 395145256Sjkoshystatic int 396145256Sjkoshyamd_write_pmc(int cpu, int ri, pmc_value_t v) 397145256Sjkoshy{ 398145256Sjkoshy const struct amd_descr *pd; 399145256Sjkoshy struct pmc *pm; 400145256Sjkoshy const struct pmc_hw *phw; 401145256Sjkoshy enum pmc_mode mode; 402145256Sjkoshy 403145256Sjkoshy KASSERT(cpu >= 0 && cpu < mp_ncpus, 404145256Sjkoshy ("[amd,%d] illegal CPU value %d", __LINE__, cpu)); 405145256Sjkoshy KASSERT(ri >= 0 && ri < AMD_NPMCS, 406145256Sjkoshy ("[amd,%d] illegal row-index %d", __LINE__, ri)); 407145256Sjkoshy 408145256Sjkoshy phw = pmc_pcpu[cpu]->pc_hwpmcs[ri]; 409145256Sjkoshy pd = &amd_pmcdesc[ri]; 410145256Sjkoshy pm = phw->phw_pmc; 411145256Sjkoshy 412145256Sjkoshy KASSERT(pm != NULL, 413145256Sjkoshy ("[amd,%d] PMC not owned (cpu%d,pmc%d)", __LINE__, 414145256Sjkoshy cpu, ri)); 415145256Sjkoshy 416145774Sjkoshy mode = PMC_TO_MODE(pm); 417145256Sjkoshy 418145256Sjkoshy if (pd->pm_descr.pd_class == PMC_CLASS_TSC) 419145256Sjkoshy return 0; 420145256Sjkoshy 421145256Sjkoshy KASSERT(pd->pm_descr.pd_class == AMD_PMC_CLASS, 422145256Sjkoshy ("[amd,%d] unknown PMC class (%d)", __LINE__, 423145256Sjkoshy pd->pm_descr.pd_class)); 424145256Sjkoshy 425145256Sjkoshy /* use 2's complement of the count for sampling mode PMCs */ 426145256Sjkoshy if (PMC_IS_SAMPLING_MODE(mode)) 427145256Sjkoshy v = -v; 428145256Sjkoshy 429145256Sjkoshy PMCDBG(MDP,WRI,1,"amd-write cpu=%d ri=%d v=%jx", cpu, ri, v); 430145256Sjkoshy 431145256Sjkoshy /* write the PMC value */ 432145256Sjkoshy wrmsr(pd->pm_perfctr, v); 433145256Sjkoshy return 0; 434145256Sjkoshy} 435145256Sjkoshy 436145256Sjkoshy/* 437145256Sjkoshy * configure hardware pmc according to the configuration recorded in 438145256Sjkoshy * pmc 'pm'. 439145256Sjkoshy */ 440145256Sjkoshy 441145256Sjkoshystatic int 442145256Sjkoshyamd_config_pmc(int cpu, int ri, struct pmc *pm) 443145256Sjkoshy{ 444145256Sjkoshy struct pmc_hw *phw; 445145256Sjkoshy 446145615Sjkoshy PMCDBG(MDP,CFG,1, "cpu=%d ri=%d pm=%p", cpu, ri, pm); 447145615Sjkoshy 448145256Sjkoshy KASSERT(cpu >= 0 && cpu < mp_ncpus, 449145256Sjkoshy ("[amd,%d] illegal CPU value %d", __LINE__, cpu)); 450145256Sjkoshy KASSERT(ri >= 0 && ri < AMD_NPMCS, 451145256Sjkoshy ("[amd,%d] illegal row-index %d", __LINE__, ri)); 452145256Sjkoshy 453145256Sjkoshy phw = pmc_pcpu[cpu]->pc_hwpmcs[ri]; 454145256Sjkoshy 455145256Sjkoshy KASSERT(pm == NULL || phw->phw_pmc == NULL, 456145615Sjkoshy ("[amd,%d] pm=%p phw->pm=%p hwpmc not unconfigured", 457145615Sjkoshy __LINE__, pm, phw->phw_pmc)); 458145256Sjkoshy 459145256Sjkoshy phw->phw_pmc = pm; 460145256Sjkoshy return 0; 461145256Sjkoshy} 462145256Sjkoshy 463145256Sjkoshy/* 464145774Sjkoshy * Retrieve a configured PMC pointer from hardware state. 465145774Sjkoshy */ 466145774Sjkoshy 467145774Sjkoshystatic int 468145774Sjkoshyamd_get_config(int cpu, int ri, struct pmc **ppm) 469145774Sjkoshy{ 470145774Sjkoshy *ppm = pmc_pcpu[cpu]->pc_hwpmcs[ri]->phw_pmc; 471145774Sjkoshy 472145774Sjkoshy return 0; 473145774Sjkoshy} 474145774Sjkoshy 475145774Sjkoshy/* 476145256Sjkoshy * Machine dependent actions taken during the context switch in of a 477145256Sjkoshy * thread. 478145256Sjkoshy */ 479145256Sjkoshy 480145256Sjkoshystatic int 481145615Sjkoshyamd_switch_in(struct pmc_cpu *pc, struct pmc_process *pp) 482145256Sjkoshy{ 483145256Sjkoshy (void) pc; 484145256Sjkoshy 485145615Sjkoshy PMCDBG(MDP,SWI,1, "pc=%p pp=%p enable-msr=%d", pc, pp, 486145774Sjkoshy (pp->pp_flags & PMC_PP_ENABLE_MSR_ACCESS) != 0); 487145615Sjkoshy 488145615Sjkoshy /* enable the RDPMC instruction if needed */ 489145774Sjkoshy if (pp->pp_flags & PMC_PP_ENABLE_MSR_ACCESS) 490145615Sjkoshy load_cr4(rcr4() | CR4_PCE); 491145615Sjkoshy 492145256Sjkoshy return 0; 493145256Sjkoshy} 494145256Sjkoshy 495145256Sjkoshy/* 496145256Sjkoshy * Machine dependent actions taken during the context switch out of a 497145256Sjkoshy * thread. 498145256Sjkoshy */ 499145256Sjkoshy 500145256Sjkoshystatic int 501145615Sjkoshyamd_switch_out(struct pmc_cpu *pc, struct pmc_process *pp) 502145256Sjkoshy{ 503145256Sjkoshy (void) pc; 504145615Sjkoshy (void) pp; /* can be NULL */ 505145256Sjkoshy 506145615Sjkoshy PMCDBG(MDP,SWO,1, "pc=%p pp=%p enable-msr=%d", pc, pp, pp ? 507145774Sjkoshy (pp->pp_flags & PMC_PP_ENABLE_MSR_ACCESS) == 1 : 0); 508145615Sjkoshy 509145615Sjkoshy /* always turn off the RDPMC instruction */ 510145256Sjkoshy load_cr4(rcr4() & ~CR4_PCE); 511145615Sjkoshy 512145256Sjkoshy return 0; 513145256Sjkoshy} 514145256Sjkoshy 515145256Sjkoshy/* 516145256Sjkoshy * Check if a given allocation is feasible. 517145256Sjkoshy */ 518145256Sjkoshy 519145256Sjkoshystatic int 520145256Sjkoshyamd_allocate_pmc(int cpu, int ri, struct pmc *pm, 521145256Sjkoshy const struct pmc_op_pmcallocate *a) 522145256Sjkoshy{ 523145256Sjkoshy int i; 524145256Sjkoshy uint32_t allowed_unitmask, caps, config, unitmask; 525145256Sjkoshy enum pmc_event pe; 526145256Sjkoshy const struct pmc_descr *pd; 527145256Sjkoshy 528145256Sjkoshy (void) cpu; 529145256Sjkoshy 530145256Sjkoshy KASSERT(cpu >= 0 && cpu < mp_ncpus, 531145256Sjkoshy ("[amd,%d] illegal CPU value %d", __LINE__, cpu)); 532145256Sjkoshy KASSERT(ri >= 0 && ri < AMD_NPMCS, 533145256Sjkoshy ("[amd,%d] illegal row index %d", __LINE__, ri)); 534145256Sjkoshy 535145256Sjkoshy pd = &amd_pmcdesc[ri].pm_descr; 536145256Sjkoshy 537145256Sjkoshy /* check class match */ 538145774Sjkoshy if (pd->pd_class != a->pm_class) 539145256Sjkoshy return EINVAL; 540145256Sjkoshy 541145256Sjkoshy caps = pm->pm_caps; 542145256Sjkoshy 543145256Sjkoshy PMCDBG(MDP,ALL,1,"amd-allocate ri=%d caps=0x%x", ri, caps); 544145256Sjkoshy 545145256Sjkoshy if ((pd->pd_caps & caps) != caps) 546145256Sjkoshy return EPERM; 547145256Sjkoshy if (pd->pd_class == PMC_CLASS_TSC) { 548145256Sjkoshy /* TSC's are always allocated in system-wide counting mode */ 549145256Sjkoshy if (a->pm_ev != PMC_EV_TSC_TSC || 550145256Sjkoshy a->pm_mode != PMC_MODE_SC) 551145256Sjkoshy return EINVAL; 552145256Sjkoshy return 0; 553145256Sjkoshy } 554145256Sjkoshy 555145256Sjkoshy KASSERT(pd->pd_class == AMD_PMC_CLASS, 556145256Sjkoshy ("[amd,%d] Unknown PMC class (%d)", __LINE__, pd->pd_class)); 557145256Sjkoshy 558145256Sjkoshy pe = a->pm_ev; 559145256Sjkoshy 560145256Sjkoshy /* map ev to the correct event mask code */ 561145256Sjkoshy config = allowed_unitmask = 0; 562145256Sjkoshy for (i = 0; i < amd_event_codes_size; i++) 563145256Sjkoshy if (amd_event_codes[i].pe_ev == pe) { 564145256Sjkoshy config = 565145256Sjkoshy AMD_PMC_TO_EVENTMASK(amd_event_codes[i].pe_code); 566145256Sjkoshy allowed_unitmask = 567145256Sjkoshy AMD_PMC_TO_UNITMASK(amd_event_codes[i].pe_mask); 568145256Sjkoshy break; 569145256Sjkoshy } 570145256Sjkoshy if (i == amd_event_codes_size) 571145256Sjkoshy return EINVAL; 572145256Sjkoshy 573145256Sjkoshy unitmask = a->pm_amd_config & AMD_PMC_UNITMASK; 574145256Sjkoshy if (unitmask & ~allowed_unitmask) /* disallow reserved bits */ 575145256Sjkoshy return EINVAL; 576145256Sjkoshy 577145256Sjkoshy if (unitmask && (caps & PMC_CAP_QUALIFIER)) 578145256Sjkoshy config |= unitmask; 579145256Sjkoshy 580145256Sjkoshy if (caps & PMC_CAP_THRESHOLD) 581145256Sjkoshy config |= a->pm_amd_config & AMD_PMC_COUNTERMASK; 582145256Sjkoshy 583145256Sjkoshy /* set at least one of the 'usr' or 'os' caps */ 584145256Sjkoshy if (caps & PMC_CAP_USER) 585145256Sjkoshy config |= AMD_PMC_USR; 586145256Sjkoshy if (caps & PMC_CAP_SYSTEM) 587145256Sjkoshy config |= AMD_PMC_OS; 588145256Sjkoshy if ((caps & (PMC_CAP_USER|PMC_CAP_SYSTEM)) == 0) 589145256Sjkoshy config |= (AMD_PMC_USR|AMD_PMC_OS); 590145256Sjkoshy 591145256Sjkoshy if (caps & PMC_CAP_EDGE) 592145256Sjkoshy config |= AMD_PMC_EDGE; 593145256Sjkoshy if (caps & PMC_CAP_INVERT) 594145256Sjkoshy config |= AMD_PMC_INVERT; 595145256Sjkoshy if (caps & PMC_CAP_INTERRUPT) 596145256Sjkoshy config |= AMD_PMC_INT; 597145256Sjkoshy 598145256Sjkoshy pm->pm_md.pm_amd.pm_amd_evsel = config; /* save config value */ 599145256Sjkoshy 600145256Sjkoshy PMCDBG(MDP,ALL,2,"amd-allocate ri=%d -> config=0x%x", ri, config); 601145256Sjkoshy 602145256Sjkoshy return 0; 603145256Sjkoshy} 604145256Sjkoshy 605145256Sjkoshy/* 606145256Sjkoshy * Release machine dependent state associated with a PMC. This is a 607145256Sjkoshy * no-op on this architecture. 608145256Sjkoshy * 609145256Sjkoshy */ 610145256Sjkoshy 611145256Sjkoshy/* ARGSUSED0 */ 612145256Sjkoshystatic int 613145256Sjkoshyamd_release_pmc(int cpu, int ri, struct pmc *pmc) 614145256Sjkoshy{ 615145256Sjkoshy#if DEBUG 616145256Sjkoshy const struct amd_descr *pd; 617145256Sjkoshy#endif 618145256Sjkoshy struct pmc_hw *phw; 619145256Sjkoshy 620145256Sjkoshy (void) pmc; 621145256Sjkoshy 622145256Sjkoshy KASSERT(cpu >= 0 && cpu < mp_ncpus, 623145256Sjkoshy ("[amd,%d] illegal CPU value %d", __LINE__, cpu)); 624145256Sjkoshy KASSERT(ri >= 0 && ri < AMD_NPMCS, 625145256Sjkoshy ("[amd,%d] illegal row-index %d", __LINE__, ri)); 626145256Sjkoshy 627145256Sjkoshy phw = pmc_pcpu[cpu]->pc_hwpmcs[ri]; 628145256Sjkoshy 629145256Sjkoshy KASSERT(phw->phw_pmc == NULL, 630145256Sjkoshy ("[amd,%d] PHW pmc %p non-NULL", __LINE__, phw->phw_pmc)); 631145256Sjkoshy 632145256Sjkoshy#if DEBUG 633145256Sjkoshy pd = &amd_pmcdesc[ri]; 634145256Sjkoshy if (pd->pm_descr.pd_class == AMD_PMC_CLASS) 635145256Sjkoshy KASSERT(AMD_PMC_IS_STOPPED(pd->pm_evsel), 636145256Sjkoshy ("[amd,%d] PMC %d released while active", __LINE__, ri)); 637145256Sjkoshy#endif 638145256Sjkoshy 639145256Sjkoshy return 0; 640145256Sjkoshy} 641145256Sjkoshy 642145256Sjkoshy/* 643145256Sjkoshy * start a PMC. 644145256Sjkoshy */ 645145256Sjkoshy 646145256Sjkoshystatic int 647145256Sjkoshyamd_start_pmc(int cpu, int ri) 648145256Sjkoshy{ 649145256Sjkoshy uint32_t config; 650145256Sjkoshy struct pmc *pm; 651145256Sjkoshy struct pmc_hw *phw; 652145256Sjkoshy const struct amd_descr *pd; 653145256Sjkoshy 654145256Sjkoshy KASSERT(cpu >= 0 && cpu < mp_ncpus, 655145256Sjkoshy ("[amd,%d] illegal CPU value %d", __LINE__, cpu)); 656145256Sjkoshy KASSERT(ri >= 0 && ri < AMD_NPMCS, 657145256Sjkoshy ("[amd,%d] illegal row-index %d", __LINE__, ri)); 658145256Sjkoshy 659145256Sjkoshy phw = pmc_pcpu[cpu]->pc_hwpmcs[ri]; 660145256Sjkoshy pm = phw->phw_pmc; 661145256Sjkoshy pd = &amd_pmcdesc[ri]; 662145256Sjkoshy 663145256Sjkoshy KASSERT(pm != NULL, 664145256Sjkoshy ("[amd,%d] starting cpu%d,pmc%d with null pmc record", __LINE__, 665145256Sjkoshy cpu, ri)); 666145256Sjkoshy 667145256Sjkoshy PMCDBG(MDP,STA,1,"amd-start cpu=%d ri=%d", cpu, ri); 668145256Sjkoshy 669145256Sjkoshy if (pd->pm_descr.pd_class == PMC_CLASS_TSC) 670145256Sjkoshy return 0; /* TSCs are always running */ 671145256Sjkoshy 672145256Sjkoshy KASSERT(pd->pm_descr.pd_class == AMD_PMC_CLASS, 673145256Sjkoshy ("[amd,%d] unknown PMC class (%d)", __LINE__, 674145256Sjkoshy pd->pm_descr.pd_class)); 675145256Sjkoshy 676145256Sjkoshy KASSERT(AMD_PMC_IS_STOPPED(pd->pm_evsel), 677145256Sjkoshy ("[amd,%d] pmc%d,cpu%d: Starting active PMC \"%s\"", __LINE__, 678145256Sjkoshy ri, cpu, pd->pm_descr.pd_name)); 679145256Sjkoshy 680145256Sjkoshy /* turn on the PMC ENABLE bit */ 681145256Sjkoshy config = pm->pm_md.pm_amd.pm_amd_evsel | AMD_PMC_ENABLE; 682145256Sjkoshy 683145256Sjkoshy PMCDBG(MDP,STA,2,"amd-start config=0x%x", config); 684145256Sjkoshy 685145256Sjkoshy wrmsr(pd->pm_evsel, config); 686145256Sjkoshy return 0; 687145256Sjkoshy} 688145256Sjkoshy 689145256Sjkoshy/* 690145256Sjkoshy * Stop a PMC. 691145256Sjkoshy */ 692145256Sjkoshy 693145256Sjkoshystatic int 694145256Sjkoshyamd_stop_pmc(int cpu, int ri) 695145256Sjkoshy{ 696145256Sjkoshy struct pmc *pm; 697145256Sjkoshy struct pmc_hw *phw; 698145256Sjkoshy const struct amd_descr *pd; 699145256Sjkoshy uint64_t config; 700145256Sjkoshy 701145256Sjkoshy KASSERT(cpu >= 0 && cpu < mp_ncpus, 702145256Sjkoshy ("[amd,%d] illegal CPU value %d", __LINE__, cpu)); 703145256Sjkoshy KASSERT(ri >= 0 && ri < AMD_NPMCS, 704145256Sjkoshy ("[amd,%d] illegal row-index %d", __LINE__, ri)); 705145256Sjkoshy 706145256Sjkoshy phw = pmc_pcpu[cpu]->pc_hwpmcs[ri]; 707145256Sjkoshy pm = phw->phw_pmc; 708145256Sjkoshy pd = &amd_pmcdesc[ri]; 709145256Sjkoshy 710145256Sjkoshy KASSERT(pm != NULL, 711145256Sjkoshy ("[amd,%d] cpu%d,pmc%d no PMC to stop", __LINE__, 712145256Sjkoshy cpu, ri)); 713145256Sjkoshy 714145256Sjkoshy /* can't stop a TSC */ 715145256Sjkoshy if (pd->pm_descr.pd_class == PMC_CLASS_TSC) 716145256Sjkoshy return 0; 717145256Sjkoshy 718145256Sjkoshy KASSERT(pd->pm_descr.pd_class == AMD_PMC_CLASS, 719145256Sjkoshy ("[amd,%d] unknown PMC class (%d)", __LINE__, 720145256Sjkoshy pd->pm_descr.pd_class)); 721145256Sjkoshy 722145256Sjkoshy KASSERT(!AMD_PMC_IS_STOPPED(pd->pm_evsel), 723145256Sjkoshy ("[amd,%d] PMC%d, CPU%d \"%s\" already stopped", 724145256Sjkoshy __LINE__, ri, cpu, pd->pm_descr.pd_name)); 725145256Sjkoshy 726145256Sjkoshy PMCDBG(MDP,STO,1,"amd-stop ri=%d", ri); 727145256Sjkoshy 728145256Sjkoshy /* turn off the PMC ENABLE bit */ 729145256Sjkoshy config = pm->pm_md.pm_amd.pm_amd_evsel & ~AMD_PMC_ENABLE; 730145256Sjkoshy wrmsr(pd->pm_evsel, config); 731145256Sjkoshy return 0; 732145256Sjkoshy} 733145256Sjkoshy 734145256Sjkoshy/* 735145256Sjkoshy * Interrupt handler. This function needs to return '1' if the 736145256Sjkoshy * interrupt was this CPU's PMCs or '0' otherwise. It is not allowed 737145256Sjkoshy * to sleep or do anything a 'fast' interrupt handler is not allowed 738145256Sjkoshy * to do. 739145256Sjkoshy */ 740145256Sjkoshy 741145256Sjkoshystatic int 742146799Sjkoshyamd_intr(int cpu, uintptr_t eip, int usermode) 743145256Sjkoshy{ 744145256Sjkoshy int i, retval; 745145256Sjkoshy enum pmc_mode mode; 746145256Sjkoshy uint32_t perfctr; 747145256Sjkoshy struct pmc *pm; 748145256Sjkoshy struct pmc_cpu *pc; 749145256Sjkoshy struct pmc_hw *phw; 750145256Sjkoshy 751146799Sjkoshy (void) usermode; 752146799Sjkoshy 753145256Sjkoshy KASSERT(cpu >= 0 && cpu < mp_ncpus, 754145256Sjkoshy ("[amd,%d] out of range CPU %d", __LINE__, cpu)); 755145256Sjkoshy 756145256Sjkoshy retval = 0; 757145256Sjkoshy 758145256Sjkoshy pc = pmc_pcpu[cpu]; 759145256Sjkoshy 760145256Sjkoshy /* 761145256Sjkoshy * look for all PMCs that have interrupted: 762145256Sjkoshy * - skip over the TSC [PMC#0] 763145256Sjkoshy * - look for a PMC with a valid 'struct pmc' association 764145256Sjkoshy * - look for a PMC in (a) sampling mode and (b) which has 765145256Sjkoshy * overflowed. If found, we update the process's 766145256Sjkoshy * histogram or send it a profiling signal by calling 767145256Sjkoshy * the appropriate helper function. 768145256Sjkoshy */ 769145256Sjkoshy 770145256Sjkoshy for (i = 1; i < AMD_NPMCS; i++) { 771145256Sjkoshy 772145256Sjkoshy phw = pc->pc_hwpmcs[i]; 773145256Sjkoshy perfctr = amd_pmcdesc[i].pm_perfctr; 774145256Sjkoshy KASSERT(phw != NULL, ("[amd,%d] null PHW pointer", __LINE__)); 775145256Sjkoshy 776145256Sjkoshy if ((pm = phw->phw_pmc) == NULL || 777145256Sjkoshy pm->pm_state != PMC_STATE_RUNNING) { 778145256Sjkoshy atomic_add_int(&pmc_stats.pm_intr_ignored, 1); 779145256Sjkoshy continue; 780145256Sjkoshy } 781145256Sjkoshy 782145774Sjkoshy mode = PMC_TO_MODE(pm); 783145256Sjkoshy if (PMC_IS_SAMPLING_MODE(mode) && 784145256Sjkoshy AMD_PMC_HAS_OVERFLOWED(perfctr)) { 785145256Sjkoshy atomic_add_int(&pmc_stats.pm_intr_processed, 1); 786145256Sjkoshy if (PMC_IS_SYSTEM_MODE(mode)) 787145256Sjkoshy pmc_update_histogram(phw, eip); 788145256Sjkoshy else if (PMC_IS_VIRTUAL_MODE(mode)) 789145256Sjkoshy pmc_send_signal(pm); 790145256Sjkoshy retval = 1; 791145256Sjkoshy } 792145256Sjkoshy } 793145256Sjkoshy return retval; 794145256Sjkoshy} 795145256Sjkoshy 796145256Sjkoshy/* 797145256Sjkoshy * describe a PMC 798145256Sjkoshy */ 799145256Sjkoshystatic int 800145256Sjkoshyamd_describe(int cpu, int ri, struct pmc_info *pi, struct pmc **ppmc) 801145256Sjkoshy{ 802145256Sjkoshy int error; 803145256Sjkoshy size_t copied; 804145256Sjkoshy const struct amd_descr *pd; 805145256Sjkoshy struct pmc_hw *phw; 806145256Sjkoshy 807145256Sjkoshy KASSERT(cpu >= 0 && cpu < mp_ncpus, 808145256Sjkoshy ("[amd,%d] illegal CPU %d", __LINE__, cpu)); 809145256Sjkoshy KASSERT(ri >= 0 && ri < AMD_NPMCS, 810145256Sjkoshy ("[amd,%d] row-index %d out of range", __LINE__, ri)); 811145256Sjkoshy 812145256Sjkoshy phw = pmc_pcpu[cpu]->pc_hwpmcs[ri]; 813145256Sjkoshy pd = &amd_pmcdesc[ri]; 814145256Sjkoshy 815145256Sjkoshy if ((error = copystr(pd->pm_descr.pd_name, pi->pm_name, 816145256Sjkoshy PMC_NAME_MAX, &copied)) != 0) 817145256Sjkoshy return error; 818145256Sjkoshy 819145256Sjkoshy pi->pm_class = pd->pm_descr.pd_class; 820145256Sjkoshy 821145256Sjkoshy if (phw->phw_state & PMC_PHW_FLAG_IS_ENABLED) { 822145256Sjkoshy pi->pm_enabled = TRUE; 823145256Sjkoshy *ppmc = phw->phw_pmc; 824145256Sjkoshy } else { 825145256Sjkoshy pi->pm_enabled = FALSE; 826145256Sjkoshy *ppmc = NULL; 827145256Sjkoshy } 828145256Sjkoshy 829145256Sjkoshy return 0; 830145256Sjkoshy} 831145256Sjkoshy 832145256Sjkoshy/* 833145256Sjkoshy * i386 specific entry points 834145256Sjkoshy */ 835145256Sjkoshy 836145256Sjkoshy/* 837145256Sjkoshy * return the MSR address of the given PMC. 838145256Sjkoshy */ 839145256Sjkoshy 840145256Sjkoshystatic int 841145256Sjkoshyamd_get_msr(int ri, uint32_t *msr) 842145256Sjkoshy{ 843145256Sjkoshy KASSERT(ri >= 0 && ri < AMD_NPMCS, 844145256Sjkoshy ("[amd,%d] ri %d out of range", __LINE__, ri)); 845145256Sjkoshy 846145615Sjkoshy *msr = amd_pmcdesc[ri].pm_perfctr - AMD_PMC_PERFCTR_0; 847145256Sjkoshy return 0; 848145256Sjkoshy} 849145256Sjkoshy 850145256Sjkoshy/* 851145256Sjkoshy * processor dependent initialization. 852145256Sjkoshy */ 853145256Sjkoshy 854145256Sjkoshy/* 855145256Sjkoshy * Per-processor data structure 856145256Sjkoshy * 857145256Sjkoshy * [common stuff] 858145256Sjkoshy * [5 struct pmc_hw pointers] 859145256Sjkoshy * [5 struct pmc_hw structures] 860145256Sjkoshy */ 861145256Sjkoshy 862145256Sjkoshystruct amd_cpu { 863145256Sjkoshy struct pmc_cpu pc_common; 864145256Sjkoshy struct pmc_hw *pc_hwpmcs[AMD_NPMCS]; 865145256Sjkoshy struct pmc_hw pc_amdpmcs[AMD_NPMCS]; 866145256Sjkoshy}; 867145256Sjkoshy 868145256Sjkoshy 869145256Sjkoshystatic int 870145256Sjkoshyamd_init(int cpu) 871145256Sjkoshy{ 872145256Sjkoshy int n; 873145256Sjkoshy struct amd_cpu *pcs; 874145256Sjkoshy struct pmc_hw *phw; 875145256Sjkoshy 876145256Sjkoshy KASSERT(cpu >= 0 && cpu < mp_ncpus, 877145256Sjkoshy ("[amd,%d] insane cpu number %d", __LINE__, cpu)); 878145256Sjkoshy 879145256Sjkoshy PMCDBG(MDP,INI,1,"amd-init cpu=%d", cpu); 880145256Sjkoshy 881145256Sjkoshy MALLOC(pcs, struct amd_cpu *, sizeof(struct amd_cpu), M_PMC, 882145256Sjkoshy M_WAITOK|M_ZERO); 883145256Sjkoshy 884145256Sjkoshy if (pcs == NULL) 885145256Sjkoshy return ENOMEM; 886145256Sjkoshy 887145256Sjkoshy phw = &pcs->pc_amdpmcs[0]; 888145256Sjkoshy 889145256Sjkoshy /* 890145256Sjkoshy * Initialize the per-cpu mutex and set the content of the 891145256Sjkoshy * hardware descriptors to a known state. 892145256Sjkoshy */ 893145256Sjkoshy 894145256Sjkoshy for (n = 0; n < AMD_NPMCS; n++, phw++) { 895145256Sjkoshy phw->phw_state = PMC_PHW_FLAG_IS_ENABLED | 896145256Sjkoshy PMC_PHW_CPU_TO_STATE(cpu) | PMC_PHW_INDEX_TO_STATE(n); 897145256Sjkoshy phw->phw_pmc = NULL; 898145256Sjkoshy pcs->pc_hwpmcs[n] = phw; 899145256Sjkoshy } 900145256Sjkoshy 901145256Sjkoshy /* Mark the TSC as shareable */ 902145256Sjkoshy pcs->pc_hwpmcs[0]->phw_state |= PMC_PHW_FLAG_IS_SHAREABLE; 903145256Sjkoshy 904145256Sjkoshy pmc_pcpu[cpu] = (struct pmc_cpu *) pcs; 905145256Sjkoshy 906145256Sjkoshy return 0; 907145256Sjkoshy} 908145256Sjkoshy 909145256Sjkoshy 910145256Sjkoshy/* 911145256Sjkoshy * processor dependent cleanup prior to the KLD 912145256Sjkoshy * being unloaded 913145256Sjkoshy */ 914145256Sjkoshy 915145256Sjkoshystatic int 916145256Sjkoshyamd_cleanup(int cpu) 917145256Sjkoshy{ 918145256Sjkoshy int i; 919145256Sjkoshy uint32_t evsel; 920145256Sjkoshy struct pmc_cpu *pcs; 921145256Sjkoshy 922145256Sjkoshy KASSERT(cpu >= 0 && cpu < mp_ncpus, 923145256Sjkoshy ("[amd,%d] insane cpu number (%d)", __LINE__, cpu)); 924145256Sjkoshy 925145256Sjkoshy PMCDBG(MDP,INI,1,"amd-cleanup cpu=%d", cpu); 926145256Sjkoshy 927145256Sjkoshy /* 928145256Sjkoshy * First, turn off all PMCs on this CPU. 929145256Sjkoshy */ 930145256Sjkoshy 931145256Sjkoshy for (i = 0; i < 4; i++) { /* XXX this loop is now not needed */ 932145256Sjkoshy evsel = rdmsr(AMD_PMC_EVSEL_0 + i); 933145256Sjkoshy evsel &= ~AMD_PMC_ENABLE; 934145256Sjkoshy wrmsr(AMD_PMC_EVSEL_0 + i, evsel); 935145256Sjkoshy } 936145256Sjkoshy 937145256Sjkoshy /* 938145256Sjkoshy * Next, free up allocated space. 939145256Sjkoshy */ 940145256Sjkoshy 941145256Sjkoshy pcs = pmc_pcpu[cpu]; 942145256Sjkoshy 943145256Sjkoshy#if DEBUG 944145256Sjkoshy /* check the TSC */ 945145256Sjkoshy KASSERT(pcs->pc_hwpmcs[0]->phw_pmc == NULL, 946145256Sjkoshy ("[amd,%d] CPU%d,PMC0 still in use", __LINE__, cpu)); 947145256Sjkoshy for (i = 1; i < AMD_NPMCS; i++) { 948145256Sjkoshy KASSERT(pcs->pc_hwpmcs[i]->phw_pmc == NULL, 949145256Sjkoshy ("[amd,%d] CPU%d/PMC%d in use", __LINE__, cpu, i)); 950145256Sjkoshy KASSERT(AMD_PMC_IS_STOPPED(AMD_PMC_EVSEL_0 + (i-1)), 951145256Sjkoshy ("[amd,%d] CPU%d/PMC%d not stopped", __LINE__, cpu, i)); 952145256Sjkoshy } 953145256Sjkoshy#endif 954145256Sjkoshy KASSERT(pcs != NULL, 955145256Sjkoshy ("[amd,%d] null per-cpu state pointer (cpu%d)", __LINE__, cpu)); 956145256Sjkoshy 957145256Sjkoshy pmc_pcpu[cpu] = NULL; 958145256Sjkoshy FREE(pcs, M_PMC); 959145256Sjkoshy return 0; 960145256Sjkoshy} 961145256Sjkoshy 962145256Sjkoshy/* 963145256Sjkoshy * Initialize ourselves. 964145256Sjkoshy */ 965145256Sjkoshy 966145256Sjkoshystruct pmc_mdep * 967145256Sjkoshypmc_amd_initialize(void) 968145256Sjkoshy{ 969145256Sjkoshy 970145256Sjkoshy struct pmc_mdep *pmc_mdep; 971145256Sjkoshy 972145256Sjkoshy /* The presence of hardware performance counters on the AMD 973145256Sjkoshy Athlon, Duron or later processors, is _not_ indicated by 974145256Sjkoshy any of the processor feature flags set by the 'CPUID' 975145256Sjkoshy instruction, so we only check the 'instruction family' 976145256Sjkoshy field returned by CPUID for instruction family >= 6. This 977145256Sjkoshy test needs to be be refined. */ 978145256Sjkoshy 979145256Sjkoshy if ((cpu_id & 0xF00) < 0x600) 980145256Sjkoshy return NULL; 981145256Sjkoshy 982145256Sjkoshy MALLOC(pmc_mdep, struct pmc_mdep *, sizeof(struct pmc_mdep), 983145256Sjkoshy M_PMC, M_WAITOK|M_ZERO); 984145256Sjkoshy 985145256Sjkoshy#if __i386__ 986145256Sjkoshy pmc_mdep->pmd_cputype = PMC_CPU_AMD_K7; 987145256Sjkoshy#elif __amd64__ 988145256Sjkoshy pmc_mdep->pmd_cputype = PMC_CPU_AMD_K8; 989145256Sjkoshy#else 990145256Sjkoshy#error Unknown AMD CPU type. 991145256Sjkoshy#endif 992145256Sjkoshy 993145256Sjkoshy pmc_mdep->pmd_npmc = AMD_NPMCS; 994145256Sjkoshy 995145256Sjkoshy /* this processor has two classes of usable PMCs */ 996145256Sjkoshy pmc_mdep->pmd_nclass = 2; 997145774Sjkoshy 998145774Sjkoshy /* TSC */ 999145774Sjkoshy pmc_mdep->pmd_classes[0].pm_class = PMC_CLASS_TSC; 1000145774Sjkoshy pmc_mdep->pmd_classes[0].pm_caps = PMC_CAP_READ; 1001145774Sjkoshy pmc_mdep->pmd_classes[0].pm_width = 64; 1002145774Sjkoshy 1003145774Sjkoshy /* AMD K7/K8 PMCs */ 1004145774Sjkoshy pmc_mdep->pmd_classes[1].pm_class = AMD_PMC_CLASS; 1005145774Sjkoshy pmc_mdep->pmd_classes[1].pm_caps = AMD_PMC_CAPS; 1006145774Sjkoshy pmc_mdep->pmd_classes[1].pm_width = 48; 1007145774Sjkoshy 1008145256Sjkoshy pmc_mdep->pmd_nclasspmcs[0] = 1; 1009145256Sjkoshy pmc_mdep->pmd_nclasspmcs[1] = (AMD_NPMCS-1); 1010145256Sjkoshy 1011145256Sjkoshy pmc_mdep->pmd_init = amd_init; 1012145256Sjkoshy pmc_mdep->pmd_cleanup = amd_cleanup; 1013145256Sjkoshy pmc_mdep->pmd_switch_in = amd_switch_in; 1014145256Sjkoshy pmc_mdep->pmd_switch_out = amd_switch_out; 1015145256Sjkoshy pmc_mdep->pmd_read_pmc = amd_read_pmc; 1016145256Sjkoshy pmc_mdep->pmd_write_pmc = amd_write_pmc; 1017145256Sjkoshy pmc_mdep->pmd_config_pmc = amd_config_pmc; 1018145774Sjkoshy pmc_mdep->pmd_get_config = amd_get_config; 1019145256Sjkoshy pmc_mdep->pmd_allocate_pmc = amd_allocate_pmc; 1020145256Sjkoshy pmc_mdep->pmd_release_pmc = amd_release_pmc; 1021145256Sjkoshy pmc_mdep->pmd_start_pmc = amd_start_pmc; 1022145256Sjkoshy pmc_mdep->pmd_stop_pmc = amd_stop_pmc; 1023145256Sjkoshy pmc_mdep->pmd_intr = amd_intr; 1024145256Sjkoshy pmc_mdep->pmd_describe = amd_describe; 1025145256Sjkoshy pmc_mdep->pmd_get_msr = amd_get_msr; /* i386 */ 1026145256Sjkoshy 1027145256Sjkoshy PMCDBG(MDP,INI,0,"%s","amd-initialize"); 1028145256Sjkoshy 1029145256Sjkoshy return pmc_mdep; 1030145256Sjkoshy} 1031