hwpmc_amd.c revision 145615
1145256Sjkoshy/*- 2145256Sjkoshy * Copyright (c) 2003-2005 Joseph Koshy 3145256Sjkoshy * All rights reserved. 4145256Sjkoshy * 5145256Sjkoshy * Redistribution and use in source and binary forms, with or without 6145256Sjkoshy * modification, are permitted provided that the following conditions 7145256Sjkoshy * are met: 8145256Sjkoshy * 1. Redistributions of source code must retain the above copyright 9145256Sjkoshy * notice, this list of conditions and the following disclaimer. 10145256Sjkoshy * 2. Redistributions in binary form must reproduce the above copyright 11145256Sjkoshy * notice, this list of conditions and the following disclaimer in the 12145256Sjkoshy * documentation and/or other materials provided with the distribution. 13145256Sjkoshy * 14145256Sjkoshy * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 15145256Sjkoshy * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 16145256Sjkoshy * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 17145256Sjkoshy * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 18145256Sjkoshy * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 19145256Sjkoshy * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 20145256Sjkoshy * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 21145256Sjkoshy * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 22145256Sjkoshy * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 23145256Sjkoshy * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 24145256Sjkoshy * SUCH DAMAGE. 25145256Sjkoshy * 26145256Sjkoshy */ 27145256Sjkoshy 28145256Sjkoshy#include <sys/cdefs.h> 29145256Sjkoshy__FBSDID("$FreeBSD: head/sys/dev/hwpmc/hwpmc_amd.c 145615 2005-04-28 08:13:19Z jkoshy $"); 30145256Sjkoshy 31145256Sjkoshy/* Support for the AMD K7 and later processors */ 32145256Sjkoshy 33145256Sjkoshy#include <sys/param.h> 34145256Sjkoshy#include <sys/lock.h> 35145256Sjkoshy#include <sys/malloc.h> 36145256Sjkoshy#include <sys/mutex.h> 37145338Smarcel#include <sys/pmc.h> 38145256Sjkoshy#include <sys/smp.h> 39145256Sjkoshy#include <sys/systm.h> 40145256Sjkoshy 41145256Sjkoshy#include <machine/md_var.h> 42145256Sjkoshy 43145256Sjkoshy/* AMD K7 and K8 PMCs */ 44145256Sjkoshy 45145256Sjkoshy#define AMD_PMC_EVSEL_0 0xC0010000 46145256Sjkoshy#define AMD_PMC_EVSEL_1 0xC0010001 47145256Sjkoshy#define AMD_PMC_EVSEL_2 0xC0010002 48145256Sjkoshy#define AMD_PMC_EVSEL_3 0xC0010003 49145256Sjkoshy 50145256Sjkoshy#define AMD_PMC_PERFCTR_0 0xC0010004 51145256Sjkoshy#define AMD_PMC_PERFCTR_1 0xC0010005 52145256Sjkoshy#define AMD_PMC_PERFCTR_2 0xC0010006 53145256Sjkoshy#define AMD_PMC_PERFCTR_3 0xC0010007 54145256Sjkoshy 55145256Sjkoshy#define K7_VALID_EVENT_CODE(c) (((c) >= 0x40 && (c) <= 0x47) || \ 56145256Sjkoshy ((c) >= 0x80 && (c) <= 0x85) || ((c) >= 0xC0 && (c) <= 0xC7) || \ 57145256Sjkoshy ((c) >= 0xCD && (c) <= 0xCF)) 58145256Sjkoshy 59145256Sjkoshy#define AMD_PMC_CAPS (PMC_CAP_INTERRUPT | PMC_CAP_USER | \ 60145256Sjkoshy PMC_CAP_SYSTEM | PMC_CAP_EDGE | PMC_CAP_THRESHOLD | \ 61145256Sjkoshy PMC_CAP_READ | PMC_CAP_WRITE | PMC_CAP_INVERT | PMC_CAP_QUALIFIER) 62145256Sjkoshy 63145256Sjkoshy/* reserved bits include bit 21 and the top two bits of the unit mask */ 64145256Sjkoshy#define K7_PMC_RESERVED ((1 << 21) | (3 << 13)) 65145256Sjkoshy 66145256Sjkoshy#define K8_PMC_RESERVED (1 << 21) 67145256Sjkoshy 68145256Sjkoshy#define AMD_PMC_IS_STOPPED(evsel) ((rdmsr((evsel)) & AMD_PMC_ENABLE) == 0) 69145256Sjkoshy#define AMD_PMC_HAS_OVERFLOWED(pmc) ((rdpmc(pmc) & (1ULL << 47)) == 0) 70145256Sjkoshy 71145256Sjkoshy#if __i386__ 72145256Sjkoshy#define AMD_NPMCS K7_NPMCS 73145256Sjkoshy#define AMD_PMC_CLASS PMC_CLASS_K7 74145256Sjkoshy#define AMD_PMC_COUNTERMASK K7_PMC_COUNTERMASK 75145256Sjkoshy#define AMD_PMC_TO_COUNTER(x) K7_PMC_TO_COUNTER(x) 76145256Sjkoshy#define AMD_PMC_INVERT K7_PMC_INVERT 77145256Sjkoshy#define AMD_PMC_ENABLE K7_PMC_ENABLE 78145256Sjkoshy#define AMD_PMC_INT K7_PMC_INT 79145256Sjkoshy#define AMD_PMC_PC K7_PMC_PC 80145256Sjkoshy#define AMD_PMC_EDGE K7_PMC_EDGE 81145256Sjkoshy#define AMD_PMC_OS K7_PMC_OS 82145256Sjkoshy#define AMD_PMC_USR K7_PMC_USR 83145256Sjkoshy 84145256Sjkoshy#define AMD_PMC_UNITMASK_M K7_PMC_UNITMASK_M 85145256Sjkoshy#define AMD_PMC_UNITMASK_O K7_PMC_UNITMASK_O 86145256Sjkoshy#define AMD_PMC_UNITMASK_E K7_PMC_UNITMASK_E 87145256Sjkoshy#define AMD_PMC_UNITMASK_S K7_PMC_UNITMASK_S 88145256Sjkoshy#define AMD_PMC_UNITMASK_I K7_PMC_UNITMASK_I 89145256Sjkoshy 90145256Sjkoshy#define AMD_PMC_UNITMASK K7_PMC_UNITMASK 91145256Sjkoshy#define AMD_PMC_EVENTMASK K7_PMC_EVENTMASK 92145256Sjkoshy#define AMD_PMC_TO_UNITMASK(x) K7_PMC_TO_UNITMASK(x) 93145256Sjkoshy#define AMD_PMC_TO_EVENTMASK(x) K7_PMC_TO_EVENTMASK(x) 94145256Sjkoshy#define AMD_VALID_BITS K7_VALID_BITS 95145256Sjkoshy 96145256Sjkoshy#define AMD_PMC_CLASS_NAME "K7-" 97145256Sjkoshy 98145256Sjkoshy#elif __amd64__ 99145256Sjkoshy 100145256Sjkoshy#define AMD_NPMCS K8_NPMCS 101145256Sjkoshy#define AMD_PMC_CLASS PMC_CLASS_K8 102145256Sjkoshy#define AMD_PMC_COUNTERMASK K8_PMC_COUNTERMASK 103145256Sjkoshy#define AMD_PMC_TO_COUNTER(x) K8_PMC_TO_COUNTER(x) 104145256Sjkoshy#define AMD_PMC_INVERT K8_PMC_INVERT 105145256Sjkoshy#define AMD_PMC_ENABLE K8_PMC_ENABLE 106145256Sjkoshy#define AMD_PMC_INT K8_PMC_INT 107145256Sjkoshy#define AMD_PMC_PC K8_PMC_PC 108145256Sjkoshy#define AMD_PMC_EDGE K8_PMC_EDGE 109145256Sjkoshy#define AMD_PMC_OS K8_PMC_OS 110145256Sjkoshy#define AMD_PMC_USR K8_PMC_USR 111145256Sjkoshy 112145256Sjkoshy#define AMD_PMC_UNITMASK_M K8_PMC_UNITMASK_M 113145256Sjkoshy#define AMD_PMC_UNITMASK_O K8_PMC_UNITMASK_O 114145256Sjkoshy#define AMD_PMC_UNITMASK_E K8_PMC_UNITMASK_E 115145256Sjkoshy#define AMD_PMC_UNITMASK_S K8_PMC_UNITMASK_S 116145256Sjkoshy#define AMD_PMC_UNITMASK_I K8_PMC_UNITMASK_I 117145256Sjkoshy 118145256Sjkoshy#define AMD_PMC_UNITMASK K8_PMC_UNITMASK 119145256Sjkoshy#define AMD_PMC_EVENTMASK K8_PMC_EVENTMASK 120145256Sjkoshy#define AMD_PMC_TO_UNITMASK(x) K8_PMC_TO_UNITMASK(x) 121145256Sjkoshy#define AMD_PMC_TO_EVENTMASK(x) K8_PMC_TO_EVENTMASK(x) 122145256Sjkoshy#define AMD_VALID_BITS K8_VALID_BITS 123145256Sjkoshy 124145256Sjkoshy#define AMD_PMC_CLASS_NAME "K8-" 125145256Sjkoshy 126145256Sjkoshy#else 127145256Sjkoshy#error Unsupported architecture. 128145256Sjkoshy#endif 129145256Sjkoshy 130145256Sjkoshy/* AMD K7 & K8 PMCs */ 131145256Sjkoshystruct amd_descr { 132145256Sjkoshy struct pmc_descr pm_descr; /* "base class" */ 133145256Sjkoshy uint32_t pm_evsel; /* address of EVSEL register */ 134145256Sjkoshy uint32_t pm_perfctr; /* address of PERFCTR register */ 135145256Sjkoshy}; 136145256Sjkoshy 137145256Sjkoshystatic const struct amd_descr amd_pmcdesc[AMD_NPMCS] = 138145256Sjkoshy{ 139145256Sjkoshy { 140145256Sjkoshy .pm_descr = 141145256Sjkoshy { 142145256Sjkoshy .pd_name = "TSC", 143145256Sjkoshy .pd_class = PMC_CLASS_TSC, 144145256Sjkoshy .pd_caps = PMC_CAP_READ, 145145256Sjkoshy .pd_width = 64 146145256Sjkoshy }, 147145256Sjkoshy .pm_evsel = MSR_TSC, 148145256Sjkoshy .pm_perfctr = 0 /* unused */ 149145256Sjkoshy }, 150145256Sjkoshy 151145256Sjkoshy { 152145256Sjkoshy .pm_descr = 153145256Sjkoshy { 154145256Sjkoshy .pd_name = AMD_PMC_CLASS_NAME "0", 155145256Sjkoshy .pd_class = AMD_PMC_CLASS, 156145256Sjkoshy .pd_caps = AMD_PMC_CAPS, 157145256Sjkoshy .pd_width = 48 158145256Sjkoshy }, 159145256Sjkoshy .pm_evsel = AMD_PMC_EVSEL_0, 160145256Sjkoshy .pm_perfctr = AMD_PMC_PERFCTR_0 161145256Sjkoshy }, 162145256Sjkoshy { 163145256Sjkoshy .pm_descr = 164145256Sjkoshy { 165145256Sjkoshy .pd_name = AMD_PMC_CLASS_NAME "1", 166145256Sjkoshy .pd_class = AMD_PMC_CLASS, 167145256Sjkoshy .pd_caps = AMD_PMC_CAPS, 168145256Sjkoshy .pd_width = 48 169145256Sjkoshy }, 170145256Sjkoshy .pm_evsel = AMD_PMC_EVSEL_1, 171145256Sjkoshy .pm_perfctr = AMD_PMC_PERFCTR_1 172145256Sjkoshy }, 173145256Sjkoshy { 174145256Sjkoshy .pm_descr = 175145256Sjkoshy { 176145256Sjkoshy .pd_name = AMD_PMC_CLASS_NAME "2", 177145256Sjkoshy .pd_class = AMD_PMC_CLASS, 178145256Sjkoshy .pd_caps = AMD_PMC_CAPS, 179145256Sjkoshy .pd_width = 48 180145256Sjkoshy }, 181145256Sjkoshy .pm_evsel = AMD_PMC_EVSEL_2, 182145256Sjkoshy .pm_perfctr = AMD_PMC_PERFCTR_2 183145256Sjkoshy }, 184145256Sjkoshy { 185145256Sjkoshy .pm_descr = 186145256Sjkoshy { 187145256Sjkoshy .pd_name = AMD_PMC_CLASS_NAME "3", 188145256Sjkoshy .pd_class = AMD_PMC_CLASS, 189145256Sjkoshy .pd_caps = AMD_PMC_CAPS, 190145256Sjkoshy .pd_width = 48 191145256Sjkoshy }, 192145256Sjkoshy .pm_evsel = AMD_PMC_EVSEL_3, 193145256Sjkoshy .pm_perfctr = AMD_PMC_PERFCTR_3 194145256Sjkoshy } 195145256Sjkoshy}; 196145256Sjkoshy 197145256Sjkoshystruct amd_event_code_map { 198145256Sjkoshy enum pmc_event pe_ev; /* enum value */ 199145256Sjkoshy uint8_t pe_code; /* encoded event mask */ 200145256Sjkoshy uint8_t pe_mask; /* bits allowed in unit mask */ 201145256Sjkoshy}; 202145256Sjkoshy 203145256Sjkoshyconst struct amd_event_code_map amd_event_codes[] = { 204145256Sjkoshy#if __i386__ 205145256Sjkoshy { PMC_EV_K7_DC_ACCESSES, 0x40, 0 }, 206145256Sjkoshy { PMC_EV_K7_DC_MISSES, 0x41, 0 }, 207145256Sjkoshy { PMC_EV_K7_DC_REFILLS_FROM_L2, 0x42, K7_PMC_UNITMASK_MOESI }, 208145256Sjkoshy { PMC_EV_K7_DC_REFILLS_FROM_SYSTEM, 0x43, K7_PMC_UNITMASK_MOESI }, 209145256Sjkoshy { PMC_EV_K7_DC_WRITEBACKS, 0x44, K7_PMC_UNITMASK_MOESI }, 210145256Sjkoshy { PMC_EV_K7_L1_DTLB_MISS_AND_L2_DTLB_HITS, 0x45, 0 }, 211145256Sjkoshy { PMC_EV_K7_L1_AND_L2_DTLB_MISSES, 0x46, 0 }, 212145256Sjkoshy { PMC_EV_K7_MISALIGNED_REFERENCES, 0x47, 0 }, 213145256Sjkoshy 214145256Sjkoshy { PMC_EV_K7_IC_FETCHES, 0x80, 0 }, 215145256Sjkoshy { PMC_EV_K7_IC_MISSES, 0x81, 0 }, 216145256Sjkoshy 217145256Sjkoshy { PMC_EV_K7_L1_ITLB_MISSES, 0x84, 0 }, 218145256Sjkoshy { PMC_EV_K7_L1_L2_ITLB_MISSES, 0x85, 0 }, 219145256Sjkoshy 220145256Sjkoshy { PMC_EV_K7_RETIRED_INSTRUCTIONS, 0xC0, 0 }, 221145256Sjkoshy { PMC_EV_K7_RETIRED_OPS, 0xC1, 0 }, 222145256Sjkoshy { PMC_EV_K7_RETIRED_BRANCHES, 0xC2, 0 }, 223145256Sjkoshy { PMC_EV_K7_RETIRED_BRANCHES_MISPREDICTED, 0xC3, 0 }, 224145256Sjkoshy { PMC_EV_K7_RETIRED_TAKEN_BRANCHES, 0xC4, 0 }, 225145256Sjkoshy { PMC_EV_K7_RETIRED_TAKEN_BRANCHES_MISPREDICTED, 0xC5, 0 }, 226145256Sjkoshy { PMC_EV_K7_RETIRED_FAR_CONTROL_TRANSFERS, 0xC6, 0 }, 227145256Sjkoshy { PMC_EV_K7_RETIRED_RESYNC_BRANCHES, 0xC7, 0 }, 228145256Sjkoshy { PMC_EV_K7_INTERRUPTS_MASKED_CYCLES, 0xCD, 0 }, 229145256Sjkoshy { PMC_EV_K7_INTERRUPTS_MASKED_WHILE_PENDING_CYCLES, 0xCE, 0 }, 230145256Sjkoshy { PMC_EV_K7_HARDWARE_INTERRUPTS, 0xCF, 0 } 231145256Sjkoshy#endif 232145256Sjkoshy 233145256Sjkoshy#if __amd64__ 234145256Sjkoshy { PMC_EV_K8_FP_DISPATCHED_FPU_OPS, 0x00, 0x3F }, 235145256Sjkoshy { PMC_EV_K8_FP_CYCLES_WITH_NO_FPU_OPS_RETIRED, 0x01, 0x00 }, 236145256Sjkoshy { PMC_EV_K8_FP_DISPATCHED_FPU_FAST_FLAG_OPS, 0x02, 0x00 }, 237145256Sjkoshy 238145256Sjkoshy { PMC_EV_K8_LS_SEGMENT_REGISTER_LOAD, 0x20, 0x7F }, 239145256Sjkoshy { PMC_EV_K8_LS_MICROARCHITECTURAL_RESYNC_BY_SELF_MODIFYING_CODE, 240145256Sjkoshy 0x21, 0x00 }, 241145256Sjkoshy { PMC_EV_K8_LS_MICROARCHITECTURAL_RESYNC_BY_SNOOP, 0x22, 0x00 }, 242145256Sjkoshy { PMC_EV_K8_LS_BUFFER2_FULL, 0x23, 0x00 }, 243145256Sjkoshy { PMC_EV_K8_LS_LOCKED_OPERATION, 0x24, 0x07 }, 244145256Sjkoshy { PMC_EV_K8_LS_MICROARCHITECTURAL_LATE_CANCEL, 0x25, 0x00 }, 245145256Sjkoshy { PMC_EV_K8_LS_RETIRED_CFLUSH_INSTRUCTIONS, 0x26, 0x00 }, 246145256Sjkoshy { PMC_EV_K8_LS_RETIRED_CPUID_INSTRUCTIONS, 0x27, 0x00 }, 247145256Sjkoshy 248145256Sjkoshy { PMC_EV_K8_DC_ACCESS, 0x40, 0x00 }, 249145256Sjkoshy { PMC_EV_K8_DC_MISS, 0x41, 0x00 }, 250145256Sjkoshy { PMC_EV_K8_DC_REFILL_FROM_L2, 0x42, 0x1F }, 251145256Sjkoshy { PMC_EV_K8_DC_REFILL_FROM_SYSTEM, 0x43, 0x1F }, 252145256Sjkoshy { PMC_EV_K8_DC_COPYBACK, 0x44, 0x1F }, 253145256Sjkoshy { PMC_EV_K8_DC_L1_DTLB_MISS_AND_L2_DTLB_HIT, 0x45, 0x00 }, 254145256Sjkoshy { PMC_EV_K8_DC_L1_DTLB_MISS_AND_L2_DTLB_MISS, 0x46, 0x00 }, 255145256Sjkoshy { PMC_EV_K8_DC_MISALIGNED_DATA_REFERENCE, 0x47, 0x00 }, 256145256Sjkoshy { PMC_EV_K8_DC_MICROARCHITECTURAL_LATE_CANCEL, 0x48, 0x00 }, 257145256Sjkoshy { PMC_EV_K8_DC_MICROARCHITECTURAL_EARLY_CANCEL, 0x49, 0x00 }, 258145256Sjkoshy { PMC_EV_K8_DC_ONE_BIT_ECC_ERROR, 0x4A, 0x03 }, 259145256Sjkoshy { PMC_EV_K8_DC_DISPATCHED_PREFETCH_INSTRUCTIONS, 0x4B, 0x07 }, 260145256Sjkoshy { PMC_EV_K8_DC_DCACHE_ACCESSES_BY_LOCKS, 0x4C, 0x03 }, 261145256Sjkoshy 262145256Sjkoshy { PMC_EV_K8_BU_CPU_CLK_UNHALTED, 0x76, 0x00 }, 263145256Sjkoshy { PMC_EV_K8_BU_INTERNAL_L2_REQUEST, 0x7D, 0x1F }, 264145256Sjkoshy { PMC_EV_K8_BU_FILL_REQUEST_L2_MISS, 0x7E, 0x07 }, 265145256Sjkoshy { PMC_EV_K8_BU_FILL_INTO_L2, 0x7F, 0x03 }, 266145256Sjkoshy 267145256Sjkoshy { PMC_EV_K8_IC_FETCH, 0x80, 0x00 }, 268145256Sjkoshy { PMC_EV_K8_IC_MISS, 0x81, 0x00 }, 269145256Sjkoshy { PMC_EV_K8_IC_REFILL_FROM_L2, 0x82, 0x00 }, 270145256Sjkoshy { PMC_EV_K8_IC_REFILL_FROM_SYSTEM, 0x83, 0x00 }, 271145256Sjkoshy { PMC_EV_K8_IC_L1_ITLB_MISS_AND_L2_ITLB_HIT, 0x84, 0x00 }, 272145256Sjkoshy { PMC_EV_K8_IC_L1_ITLB_MISS_AND_L2_ITLB_MISS, 0x85, 0x00 }, 273145256Sjkoshy { PMC_EV_K8_IC_MICROARCHITECTURAL_RESYNC_BY_SNOOP, 0x86, 0x00 }, 274145256Sjkoshy { PMC_EV_K8_IC_INSTRUCTION_FETCH_STALL, 0x87, 0x00 }, 275145256Sjkoshy { PMC_EV_K8_IC_RETURN_STACK_HIT, 0x88, 0x00 }, 276145256Sjkoshy { PMC_EV_K8_IC_RETURN_STACK_OVERFLOW, 0x89, 0x00 }, 277145256Sjkoshy 278145256Sjkoshy { PMC_EV_K8_FR_RETIRED_X86_INSTRUCTIONS, 0xC0, 0x00 }, 279145256Sjkoshy { PMC_EV_K8_FR_RETIRED_UOPS, 0xC1, 0x00 }, 280145256Sjkoshy { PMC_EV_K8_FR_RETIRED_BRANCHES, 0xC2, 0x00 }, 281145256Sjkoshy { PMC_EV_K8_FR_RETIRED_BRANCHES_MISPREDICTED, 0xC3, 0x00 }, 282145256Sjkoshy { PMC_EV_K8_FR_RETIRED_TAKEN_BRANCHES, 0xC4, 0x00 }, 283145256Sjkoshy { PMC_EV_K8_FR_RETIRED_TAKEN_BRANCHES_MISPREDICTED, 0xC5, 0x00 }, 284145256Sjkoshy { PMC_EV_K8_FR_RETIRED_FAR_CONTROL_TRANSFERS, 0xC6, 0x00 }, 285145256Sjkoshy { PMC_EV_K8_FR_RETIRED_RESYNCS, 0xC7, 0x00 }, 286145256Sjkoshy { PMC_EV_K8_FR_RETIRED_NEAR_RETURNS, 0xC8, 0x00 }, 287145256Sjkoshy { PMC_EV_K8_FR_RETIRED_NEAR_RETURNS_MISPREDICTED, 0xC9, 0x00 }, 288145256Sjkoshy { PMC_EV_K8_FR_RETIRED_TAKEN_BRANCHES_MISPREDICTED_BY_ADDR_MISCOMPARE, 289145256Sjkoshy 0xCA, 0x00 }, 290145256Sjkoshy { PMC_EV_K8_FR_RETIRED_FPU_INSTRUCTIONS, 0xCB, 0x0F }, 291145256Sjkoshy { PMC_EV_K8_FR_RETIRED_FASTPATH_DOUBLE_OP_INSTRUCTIONS, 292145256Sjkoshy 0xCC, 0x07 }, 293145256Sjkoshy { PMC_EV_K8_FR_INTERRUPTS_MASKED_CYCLES, 0xCD, 0x00 }, 294145256Sjkoshy { PMC_EV_K8_FR_INTERRUPTS_MASKED_WHILE_PENDING_CYCLES, 0xCE, 0x00 }, 295145256Sjkoshy { PMC_EV_K8_FR_TAKEN_HARDWARE_INTERRUPTS, 0xCF, 0x00 }, 296145256Sjkoshy 297145256Sjkoshy { PMC_EV_K8_FR_DECODER_EMPTY, 0xD0, 0x00 }, 298145256Sjkoshy { PMC_EV_K8_FR_DISPATCH_STALLS, 0xD1, 0x00 }, 299145256Sjkoshy { PMC_EV_K8_FR_DISPATCH_STALL_FROM_BRANCH_ABORT_TO_RETIRE, 300145256Sjkoshy 0xD2, 0x00 }, 301145256Sjkoshy { PMC_EV_K8_FR_DISPATCH_STALL_FOR_SERIALIZATION, 0xD3, 0x00 }, 302145256Sjkoshy { PMC_EV_K8_FR_DISPATCH_STALL_FOR_SEGMENT_LOAD, 0xD4, 0x00 }, 303145256Sjkoshy { PMC_EV_K8_FR_DISPATCH_STALL_WHEN_REORDER_BUFFER_IS_FULL, 304145256Sjkoshy 0xD5, 0x00 }, 305145256Sjkoshy { PMC_EV_K8_FR_DISPATCH_STALL_WHEN_RESERVATION_STATIONS_ARE_FULL, 306145256Sjkoshy 0xD6, 0x00 }, 307145256Sjkoshy { PMC_EV_K8_FR_DISPATCH_STALL_WHEN_FPU_IS_FULL, 0xD7, 0x00 }, 308145256Sjkoshy { PMC_EV_K8_FR_DISPATCH_STALL_WHEN_LS_IS_FULL, 0xD8, 0x00 }, 309145256Sjkoshy { PMC_EV_K8_FR_DISPATCH_STALL_WHEN_WAITING_FOR_ALL_TO_BE_QUIET, 310145256Sjkoshy 0xD9, 0x00 }, 311145256Sjkoshy { PMC_EV_K8_FR_DISPATCH_STALL_WHEN_FAR_XFER_OR_RESYNC_BRANCH_PENDING, 312145256Sjkoshy 0xDA, 0x00 }, 313145256Sjkoshy { PMC_EV_K8_FR_FPU_EXCEPTIONS, 0xDB, 0x0F }, 314145256Sjkoshy { PMC_EV_K8_FR_NUMBER_OF_BREAKPOINTS_FOR_DR0, 0xDC, 0x00 }, 315145256Sjkoshy { PMC_EV_K8_FR_NUMBER_OF_BREAKPOINTS_FOR_DR1, 0xDD, 0x00 }, 316145256Sjkoshy { PMC_EV_K8_FR_NUMBER_OF_BREAKPOINTS_FOR_DR2, 0xDE, 0x00 }, 317145256Sjkoshy { PMC_EV_K8_FR_NUMBER_OF_BREAKPOINTS_FOR_DR3, 0xDF, 0x00 }, 318145256Sjkoshy 319145256Sjkoshy { PMC_EV_K8_NB_MEMORY_CONTROLLER_PAGE_ACCESS_EVENT, 0xE0, 0x7 }, 320145256Sjkoshy { PMC_EV_K8_NB_MEMORY_CONTROLLER_PAGE_TABLE_OVERFLOW, 0xE1, 0x00 }, 321145256Sjkoshy { PMC_EV_K8_NB_MEMORY_CONTROLLER_DRAM_COMMAND_SLOTS_MISSED, 322145256Sjkoshy 0xE2, 0x00 }, 323145256Sjkoshy { PMC_EV_K8_NB_MEMORY_CONTROLLER_TURNAROUND, 0xE3, 0x07 }, 324145256Sjkoshy { PMC_EV_K8_NB_MEMORY_CONTROLLER_BYPASS_SATURATION, 0xE4, 0x0F }, 325145256Sjkoshy { PMC_EV_K8_NB_SIZED_COMMANDS, 0xEB, 0x7F }, 326145256Sjkoshy { PMC_EV_K8_NB_PROBE_RESULT, 0xEC, 0x0F }, 327145256Sjkoshy { PMC_EV_K8_NB_HT_BUS0_BANDWIDTH, 0xF6, 0x0F }, 328145256Sjkoshy { PMC_EV_K8_NB_HT_BUS1_BANDWIDTH, 0xF7, 0x0F }, 329145256Sjkoshy { PMC_EV_K8_NB_HT_BUS2_BANDWIDTH, 0xF8, 0x0F } 330145256Sjkoshy#endif 331145256Sjkoshy 332145256Sjkoshy}; 333145256Sjkoshy 334145256Sjkoshyconst int amd_event_codes_size = 335145256Sjkoshy sizeof(amd_event_codes) / sizeof(amd_event_codes[0]); 336145256Sjkoshy 337145256Sjkoshy/* 338145256Sjkoshy * read a pmc register 339145256Sjkoshy */ 340145256Sjkoshy 341145256Sjkoshystatic int 342145256Sjkoshyamd_read_pmc(int cpu, int ri, pmc_value_t *v) 343145256Sjkoshy{ 344145256Sjkoshy enum pmc_mode mode; 345145256Sjkoshy const struct amd_descr *pd; 346145256Sjkoshy struct pmc *pm; 347145256Sjkoshy const struct pmc_hw *phw; 348145256Sjkoshy pmc_value_t tmp; 349145256Sjkoshy 350145256Sjkoshy KASSERT(cpu >= 0 && cpu < mp_ncpus, 351145256Sjkoshy ("[amd,%d] illegal CPU value %d", __LINE__, cpu)); 352145256Sjkoshy KASSERT(ri >= 0 && ri < AMD_NPMCS, 353145256Sjkoshy ("[amd,%d] illegal row-index %d", __LINE__, ri)); 354145256Sjkoshy 355145256Sjkoshy phw = pmc_pcpu[cpu]->pc_hwpmcs[ri]; 356145256Sjkoshy pd = &amd_pmcdesc[ri]; 357145256Sjkoshy pm = phw->phw_pmc; 358145256Sjkoshy 359145256Sjkoshy KASSERT(pm != NULL, 360145256Sjkoshy ("[amd,%d] No owner for HWPMC [cpu%d,pmc%d]", __LINE__, 361145256Sjkoshy cpu, ri)); 362145256Sjkoshy 363145256Sjkoshy mode = pm->pm_mode; 364145256Sjkoshy 365145256Sjkoshy PMCDBG(MDP,REA,1,"amd-read id=%d class=%d", ri, pd->pm_descr.pd_class); 366145256Sjkoshy 367145256Sjkoshy /* Reading the TSC is a special case */ 368145256Sjkoshy if (pd->pm_descr.pd_class == PMC_CLASS_TSC) { 369145256Sjkoshy KASSERT(PMC_IS_COUNTING_MODE(mode), 370145256Sjkoshy ("[amd,%d] TSC counter in non-counting mode", __LINE__)); 371145256Sjkoshy *v = rdtsc(); 372145256Sjkoshy PMCDBG(MDP,REA,2,"amd-read id=%d -> %jd", ri, *v); 373145256Sjkoshy return 0; 374145256Sjkoshy } 375145256Sjkoshy 376145256Sjkoshy KASSERT(pd->pm_descr.pd_class == AMD_PMC_CLASS, 377145256Sjkoshy ("[amd,%d] unknown PMC class (%d)", __LINE__, 378145256Sjkoshy pd->pm_descr.pd_class)); 379145256Sjkoshy 380145256Sjkoshy tmp = rdmsr(pd->pm_perfctr); /* RDMSR serializes */ 381145256Sjkoshy if (PMC_IS_SAMPLING_MODE(mode)) 382145256Sjkoshy *v = -tmp; 383145256Sjkoshy else 384145256Sjkoshy *v = tmp; 385145256Sjkoshy 386145256Sjkoshy PMCDBG(MDP,REA,2,"amd-read id=%d -> %jd", ri, *v); 387145256Sjkoshy 388145256Sjkoshy return 0; 389145256Sjkoshy} 390145256Sjkoshy 391145256Sjkoshy/* 392145256Sjkoshy * Write a PMC MSR. 393145256Sjkoshy */ 394145256Sjkoshy 395145256Sjkoshystatic int 396145256Sjkoshyamd_write_pmc(int cpu, int ri, pmc_value_t v) 397145256Sjkoshy{ 398145256Sjkoshy const struct amd_descr *pd; 399145256Sjkoshy struct pmc *pm; 400145256Sjkoshy const struct pmc_hw *phw; 401145256Sjkoshy enum pmc_mode mode; 402145256Sjkoshy 403145256Sjkoshy KASSERT(cpu >= 0 && cpu < mp_ncpus, 404145256Sjkoshy ("[amd,%d] illegal CPU value %d", __LINE__, cpu)); 405145256Sjkoshy KASSERT(ri >= 0 && ri < AMD_NPMCS, 406145256Sjkoshy ("[amd,%d] illegal row-index %d", __LINE__, ri)); 407145256Sjkoshy 408145256Sjkoshy phw = pmc_pcpu[cpu]->pc_hwpmcs[ri]; 409145256Sjkoshy pd = &amd_pmcdesc[ri]; 410145256Sjkoshy pm = phw->phw_pmc; 411145256Sjkoshy 412145256Sjkoshy KASSERT(pm != NULL, 413145256Sjkoshy ("[amd,%d] PMC not owned (cpu%d,pmc%d)", __LINE__, 414145256Sjkoshy cpu, ri)); 415145256Sjkoshy 416145256Sjkoshy mode = pm->pm_mode; 417145256Sjkoshy 418145256Sjkoshy if (pd->pm_descr.pd_class == PMC_CLASS_TSC) 419145256Sjkoshy return 0; 420145256Sjkoshy 421145256Sjkoshy KASSERT(pd->pm_descr.pd_class == AMD_PMC_CLASS, 422145256Sjkoshy ("[amd,%d] unknown PMC class (%d)", __LINE__, 423145256Sjkoshy pd->pm_descr.pd_class)); 424145256Sjkoshy 425145256Sjkoshy /* use 2's complement of the count for sampling mode PMCs */ 426145256Sjkoshy if (PMC_IS_SAMPLING_MODE(mode)) 427145256Sjkoshy v = -v; 428145256Sjkoshy 429145256Sjkoshy PMCDBG(MDP,WRI,1,"amd-write cpu=%d ri=%d v=%jx", cpu, ri, v); 430145256Sjkoshy 431145256Sjkoshy /* write the PMC value */ 432145256Sjkoshy wrmsr(pd->pm_perfctr, v); 433145256Sjkoshy return 0; 434145256Sjkoshy} 435145256Sjkoshy 436145256Sjkoshy/* 437145256Sjkoshy * configure hardware pmc according to the configuration recorded in 438145256Sjkoshy * pmc 'pm'. 439145256Sjkoshy */ 440145256Sjkoshy 441145256Sjkoshystatic int 442145256Sjkoshyamd_config_pmc(int cpu, int ri, struct pmc *pm) 443145256Sjkoshy{ 444145256Sjkoshy struct pmc_hw *phw; 445145256Sjkoshy 446145615Sjkoshy PMCDBG(MDP,CFG,1, "cpu=%d ri=%d pm=%p", cpu, ri, pm); 447145615Sjkoshy 448145256Sjkoshy KASSERT(cpu >= 0 && cpu < mp_ncpus, 449145256Sjkoshy ("[amd,%d] illegal CPU value %d", __LINE__, cpu)); 450145256Sjkoshy KASSERT(ri >= 0 && ri < AMD_NPMCS, 451145256Sjkoshy ("[amd,%d] illegal row-index %d", __LINE__, ri)); 452145256Sjkoshy 453145256Sjkoshy phw = pmc_pcpu[cpu]->pc_hwpmcs[ri]; 454145256Sjkoshy 455145256Sjkoshy KASSERT(pm == NULL || phw->phw_pmc == NULL, 456145615Sjkoshy ("[amd,%d] pm=%p phw->pm=%p hwpmc not unconfigured", 457145615Sjkoshy __LINE__, pm, phw->phw_pmc)); 458145256Sjkoshy 459145256Sjkoshy phw->phw_pmc = pm; 460145256Sjkoshy return 0; 461145256Sjkoshy} 462145256Sjkoshy 463145256Sjkoshy/* 464145256Sjkoshy * Machine dependent actions taken during the context switch in of a 465145256Sjkoshy * thread. 466145256Sjkoshy */ 467145256Sjkoshy 468145256Sjkoshystatic int 469145615Sjkoshyamd_switch_in(struct pmc_cpu *pc, struct pmc_process *pp) 470145256Sjkoshy{ 471145256Sjkoshy (void) pc; 472145256Sjkoshy 473145615Sjkoshy PMCDBG(MDP,SWI,1, "pc=%p pp=%p enable-msr=%d", pc, pp, 474145615Sjkoshy (pp->pp_flags & PMC_FLAG_ENABLE_MSR_ACCESS) != 0); 475145615Sjkoshy 476145615Sjkoshy /* enable the RDPMC instruction if needed */ 477145615Sjkoshy if (pp->pp_flags & PMC_FLAG_ENABLE_MSR_ACCESS) 478145615Sjkoshy load_cr4(rcr4() | CR4_PCE); 479145615Sjkoshy 480145256Sjkoshy return 0; 481145256Sjkoshy} 482145256Sjkoshy 483145256Sjkoshy/* 484145256Sjkoshy * Machine dependent actions taken during the context switch out of a 485145256Sjkoshy * thread. 486145256Sjkoshy */ 487145256Sjkoshy 488145256Sjkoshystatic int 489145615Sjkoshyamd_switch_out(struct pmc_cpu *pc, struct pmc_process *pp) 490145256Sjkoshy{ 491145256Sjkoshy (void) pc; 492145615Sjkoshy (void) pp; /* can be NULL */ 493145256Sjkoshy 494145615Sjkoshy PMCDBG(MDP,SWO,1, "pc=%p pp=%p enable-msr=%d", pc, pp, pp ? 495145615Sjkoshy (pp->pp_flags & PMC_FLAG_ENABLE_MSR_ACCESS) == 1 : 0); 496145615Sjkoshy 497145615Sjkoshy /* always turn off the RDPMC instruction */ 498145256Sjkoshy load_cr4(rcr4() & ~CR4_PCE); 499145615Sjkoshy 500145256Sjkoshy return 0; 501145256Sjkoshy} 502145256Sjkoshy 503145256Sjkoshy/* 504145256Sjkoshy * Check if a given allocation is feasible. 505145256Sjkoshy */ 506145256Sjkoshy 507145256Sjkoshystatic int 508145256Sjkoshyamd_allocate_pmc(int cpu, int ri, struct pmc *pm, 509145256Sjkoshy const struct pmc_op_pmcallocate *a) 510145256Sjkoshy{ 511145256Sjkoshy int i; 512145256Sjkoshy uint32_t allowed_unitmask, caps, config, unitmask; 513145256Sjkoshy enum pmc_event pe; 514145256Sjkoshy const struct pmc_descr *pd; 515145256Sjkoshy 516145256Sjkoshy (void) cpu; 517145256Sjkoshy 518145256Sjkoshy KASSERT(cpu >= 0 && cpu < mp_ncpus, 519145256Sjkoshy ("[amd,%d] illegal CPU value %d", __LINE__, cpu)); 520145256Sjkoshy KASSERT(ri >= 0 && ri < AMD_NPMCS, 521145256Sjkoshy ("[amd,%d] illegal row index %d", __LINE__, ri)); 522145256Sjkoshy 523145256Sjkoshy pd = &amd_pmcdesc[ri].pm_descr; 524145256Sjkoshy 525145256Sjkoshy /* check class match */ 526145256Sjkoshy if (pd->pd_class != pm->pm_class) 527145256Sjkoshy return EINVAL; 528145256Sjkoshy 529145256Sjkoshy caps = pm->pm_caps; 530145256Sjkoshy 531145256Sjkoshy PMCDBG(MDP,ALL,1,"amd-allocate ri=%d caps=0x%x", ri, caps); 532145256Sjkoshy 533145256Sjkoshy if ((pd->pd_caps & caps) != caps) 534145256Sjkoshy return EPERM; 535145256Sjkoshy if (pd->pd_class == PMC_CLASS_TSC) { 536145256Sjkoshy /* TSC's are always allocated in system-wide counting mode */ 537145256Sjkoshy if (a->pm_ev != PMC_EV_TSC_TSC || 538145256Sjkoshy a->pm_mode != PMC_MODE_SC) 539145256Sjkoshy return EINVAL; 540145256Sjkoshy return 0; 541145256Sjkoshy } 542145256Sjkoshy 543145256Sjkoshy KASSERT(pd->pd_class == AMD_PMC_CLASS, 544145256Sjkoshy ("[amd,%d] Unknown PMC class (%d)", __LINE__, pd->pd_class)); 545145256Sjkoshy 546145256Sjkoshy pe = a->pm_ev; 547145256Sjkoshy 548145256Sjkoshy /* map ev to the correct event mask code */ 549145256Sjkoshy config = allowed_unitmask = 0; 550145256Sjkoshy for (i = 0; i < amd_event_codes_size; i++) 551145256Sjkoshy if (amd_event_codes[i].pe_ev == pe) { 552145256Sjkoshy config = 553145256Sjkoshy AMD_PMC_TO_EVENTMASK(amd_event_codes[i].pe_code); 554145256Sjkoshy allowed_unitmask = 555145256Sjkoshy AMD_PMC_TO_UNITMASK(amd_event_codes[i].pe_mask); 556145256Sjkoshy break; 557145256Sjkoshy } 558145256Sjkoshy if (i == amd_event_codes_size) 559145256Sjkoshy return EINVAL; 560145256Sjkoshy 561145256Sjkoshy unitmask = a->pm_amd_config & AMD_PMC_UNITMASK; 562145256Sjkoshy if (unitmask & ~allowed_unitmask) /* disallow reserved bits */ 563145256Sjkoshy return EINVAL; 564145256Sjkoshy 565145256Sjkoshy if (unitmask && (caps & PMC_CAP_QUALIFIER)) 566145256Sjkoshy config |= unitmask; 567145256Sjkoshy 568145256Sjkoshy if (caps & PMC_CAP_THRESHOLD) 569145256Sjkoshy config |= a->pm_amd_config & AMD_PMC_COUNTERMASK; 570145256Sjkoshy 571145256Sjkoshy /* set at least one of the 'usr' or 'os' caps */ 572145256Sjkoshy if (caps & PMC_CAP_USER) 573145256Sjkoshy config |= AMD_PMC_USR; 574145256Sjkoshy if (caps & PMC_CAP_SYSTEM) 575145256Sjkoshy config |= AMD_PMC_OS; 576145256Sjkoshy if ((caps & (PMC_CAP_USER|PMC_CAP_SYSTEM)) == 0) 577145256Sjkoshy config |= (AMD_PMC_USR|AMD_PMC_OS); 578145256Sjkoshy 579145256Sjkoshy if (caps & PMC_CAP_EDGE) 580145256Sjkoshy config |= AMD_PMC_EDGE; 581145256Sjkoshy if (caps & PMC_CAP_INVERT) 582145256Sjkoshy config |= AMD_PMC_INVERT; 583145256Sjkoshy if (caps & PMC_CAP_INTERRUPT) 584145256Sjkoshy config |= AMD_PMC_INT; 585145256Sjkoshy 586145256Sjkoshy pm->pm_md.pm_amd.pm_amd_evsel = config; /* save config value */ 587145256Sjkoshy 588145256Sjkoshy PMCDBG(MDP,ALL,2,"amd-allocate ri=%d -> config=0x%x", ri, config); 589145256Sjkoshy 590145256Sjkoshy return 0; 591145256Sjkoshy} 592145256Sjkoshy 593145256Sjkoshy/* 594145256Sjkoshy * Release machine dependent state associated with a PMC. This is a 595145256Sjkoshy * no-op on this architecture. 596145256Sjkoshy * 597145256Sjkoshy */ 598145256Sjkoshy 599145256Sjkoshy/* ARGSUSED0 */ 600145256Sjkoshystatic int 601145256Sjkoshyamd_release_pmc(int cpu, int ri, struct pmc *pmc) 602145256Sjkoshy{ 603145256Sjkoshy#if DEBUG 604145256Sjkoshy const struct amd_descr *pd; 605145256Sjkoshy#endif 606145256Sjkoshy struct pmc_hw *phw; 607145256Sjkoshy 608145256Sjkoshy (void) pmc; 609145256Sjkoshy 610145256Sjkoshy KASSERT(cpu >= 0 && cpu < mp_ncpus, 611145256Sjkoshy ("[amd,%d] illegal CPU value %d", __LINE__, cpu)); 612145256Sjkoshy KASSERT(ri >= 0 && ri < AMD_NPMCS, 613145256Sjkoshy ("[amd,%d] illegal row-index %d", __LINE__, ri)); 614145256Sjkoshy 615145256Sjkoshy phw = pmc_pcpu[cpu]->pc_hwpmcs[ri]; 616145256Sjkoshy 617145256Sjkoshy KASSERT(phw->phw_pmc == NULL, 618145256Sjkoshy ("[amd,%d] PHW pmc %p non-NULL", __LINE__, phw->phw_pmc)); 619145256Sjkoshy 620145256Sjkoshy#if DEBUG 621145256Sjkoshy pd = &amd_pmcdesc[ri]; 622145256Sjkoshy if (pd->pm_descr.pd_class == AMD_PMC_CLASS) 623145256Sjkoshy KASSERT(AMD_PMC_IS_STOPPED(pd->pm_evsel), 624145256Sjkoshy ("[amd,%d] PMC %d released while active", __LINE__, ri)); 625145256Sjkoshy#endif 626145256Sjkoshy 627145256Sjkoshy return 0; 628145256Sjkoshy} 629145256Sjkoshy 630145256Sjkoshy/* 631145256Sjkoshy * start a PMC. 632145256Sjkoshy */ 633145256Sjkoshy 634145256Sjkoshystatic int 635145256Sjkoshyamd_start_pmc(int cpu, int ri) 636145256Sjkoshy{ 637145256Sjkoshy uint32_t config; 638145256Sjkoshy struct pmc *pm; 639145256Sjkoshy struct pmc_hw *phw; 640145256Sjkoshy const struct amd_descr *pd; 641145256Sjkoshy 642145256Sjkoshy KASSERT(cpu >= 0 && cpu < mp_ncpus, 643145256Sjkoshy ("[amd,%d] illegal CPU value %d", __LINE__, cpu)); 644145256Sjkoshy KASSERT(ri >= 0 && ri < AMD_NPMCS, 645145256Sjkoshy ("[amd,%d] illegal row-index %d", __LINE__, ri)); 646145256Sjkoshy 647145256Sjkoshy phw = pmc_pcpu[cpu]->pc_hwpmcs[ri]; 648145256Sjkoshy pm = phw->phw_pmc; 649145256Sjkoshy pd = &amd_pmcdesc[ri]; 650145256Sjkoshy 651145256Sjkoshy KASSERT(pm != NULL, 652145256Sjkoshy ("[amd,%d] starting cpu%d,pmc%d with null pmc record", __LINE__, 653145256Sjkoshy cpu, ri)); 654145256Sjkoshy 655145256Sjkoshy PMCDBG(MDP,STA,1,"amd-start cpu=%d ri=%d", cpu, ri); 656145256Sjkoshy 657145256Sjkoshy if (pd->pm_descr.pd_class == PMC_CLASS_TSC) 658145256Sjkoshy return 0; /* TSCs are always running */ 659145256Sjkoshy 660145256Sjkoshy KASSERT(pd->pm_descr.pd_class == AMD_PMC_CLASS, 661145256Sjkoshy ("[amd,%d] unknown PMC class (%d)", __LINE__, 662145256Sjkoshy pd->pm_descr.pd_class)); 663145256Sjkoshy 664145256Sjkoshy KASSERT(AMD_PMC_IS_STOPPED(pd->pm_evsel), 665145256Sjkoshy ("[amd,%d] pmc%d,cpu%d: Starting active PMC \"%s\"", __LINE__, 666145256Sjkoshy ri, cpu, pd->pm_descr.pd_name)); 667145256Sjkoshy 668145256Sjkoshy /* turn on the PMC ENABLE bit */ 669145256Sjkoshy config = pm->pm_md.pm_amd.pm_amd_evsel | AMD_PMC_ENABLE; 670145256Sjkoshy 671145256Sjkoshy PMCDBG(MDP,STA,2,"amd-start config=0x%x", config); 672145256Sjkoshy 673145256Sjkoshy wrmsr(pd->pm_evsel, config); 674145256Sjkoshy return 0; 675145256Sjkoshy} 676145256Sjkoshy 677145256Sjkoshy/* 678145256Sjkoshy * Stop a PMC. 679145256Sjkoshy */ 680145256Sjkoshy 681145256Sjkoshystatic int 682145256Sjkoshyamd_stop_pmc(int cpu, int ri) 683145256Sjkoshy{ 684145256Sjkoshy struct pmc *pm; 685145256Sjkoshy struct pmc_hw *phw; 686145256Sjkoshy const struct amd_descr *pd; 687145256Sjkoshy uint64_t config; 688145256Sjkoshy 689145256Sjkoshy KASSERT(cpu >= 0 && cpu < mp_ncpus, 690145256Sjkoshy ("[amd,%d] illegal CPU value %d", __LINE__, cpu)); 691145256Sjkoshy KASSERT(ri >= 0 && ri < AMD_NPMCS, 692145256Sjkoshy ("[amd,%d] illegal row-index %d", __LINE__, ri)); 693145256Sjkoshy 694145256Sjkoshy phw = pmc_pcpu[cpu]->pc_hwpmcs[ri]; 695145256Sjkoshy pm = phw->phw_pmc; 696145256Sjkoshy pd = &amd_pmcdesc[ri]; 697145256Sjkoshy 698145256Sjkoshy KASSERT(pm != NULL, 699145256Sjkoshy ("[amd,%d] cpu%d,pmc%d no PMC to stop", __LINE__, 700145256Sjkoshy cpu, ri)); 701145256Sjkoshy 702145256Sjkoshy /* can't stop a TSC */ 703145256Sjkoshy if (pd->pm_descr.pd_class == PMC_CLASS_TSC) 704145256Sjkoshy return 0; 705145256Sjkoshy 706145256Sjkoshy KASSERT(pd->pm_descr.pd_class == AMD_PMC_CLASS, 707145256Sjkoshy ("[amd,%d] unknown PMC class (%d)", __LINE__, 708145256Sjkoshy pd->pm_descr.pd_class)); 709145256Sjkoshy 710145256Sjkoshy KASSERT(!AMD_PMC_IS_STOPPED(pd->pm_evsel), 711145256Sjkoshy ("[amd,%d] PMC%d, CPU%d \"%s\" already stopped", 712145256Sjkoshy __LINE__, ri, cpu, pd->pm_descr.pd_name)); 713145256Sjkoshy 714145256Sjkoshy PMCDBG(MDP,STO,1,"amd-stop ri=%d", ri); 715145256Sjkoshy 716145256Sjkoshy /* turn off the PMC ENABLE bit */ 717145256Sjkoshy config = pm->pm_md.pm_amd.pm_amd_evsel & ~AMD_PMC_ENABLE; 718145256Sjkoshy wrmsr(pd->pm_evsel, config); 719145256Sjkoshy return 0; 720145256Sjkoshy} 721145256Sjkoshy 722145256Sjkoshy/* 723145256Sjkoshy * Interrupt handler. This function needs to return '1' if the 724145256Sjkoshy * interrupt was this CPU's PMCs or '0' otherwise. It is not allowed 725145256Sjkoshy * to sleep or do anything a 'fast' interrupt handler is not allowed 726145256Sjkoshy * to do. 727145256Sjkoshy */ 728145256Sjkoshy 729145256Sjkoshystatic int 730145256Sjkoshyamd_intr(int cpu, uintptr_t eip) 731145256Sjkoshy{ 732145256Sjkoshy int i, retval; 733145256Sjkoshy enum pmc_mode mode; 734145256Sjkoshy uint32_t perfctr; 735145256Sjkoshy struct pmc *pm; 736145256Sjkoshy struct pmc_cpu *pc; 737145256Sjkoshy struct pmc_hw *phw; 738145256Sjkoshy 739145256Sjkoshy KASSERT(cpu >= 0 && cpu < mp_ncpus, 740145256Sjkoshy ("[amd,%d] out of range CPU %d", __LINE__, cpu)); 741145256Sjkoshy 742145256Sjkoshy retval = 0; 743145256Sjkoshy 744145256Sjkoshy pc = pmc_pcpu[cpu]; 745145256Sjkoshy 746145256Sjkoshy /* 747145256Sjkoshy * look for all PMCs that have interrupted: 748145256Sjkoshy * - skip over the TSC [PMC#0] 749145256Sjkoshy * - look for a PMC with a valid 'struct pmc' association 750145256Sjkoshy * - look for a PMC in (a) sampling mode and (b) which has 751145256Sjkoshy * overflowed. If found, we update the process's 752145256Sjkoshy * histogram or send it a profiling signal by calling 753145256Sjkoshy * the appropriate helper function. 754145256Sjkoshy */ 755145256Sjkoshy 756145256Sjkoshy for (i = 1; i < AMD_NPMCS; i++) { 757145256Sjkoshy 758145256Sjkoshy phw = pc->pc_hwpmcs[i]; 759145256Sjkoshy perfctr = amd_pmcdesc[i].pm_perfctr; 760145256Sjkoshy KASSERT(phw != NULL, ("[amd,%d] null PHW pointer", __LINE__)); 761145256Sjkoshy 762145256Sjkoshy if ((pm = phw->phw_pmc) == NULL || 763145256Sjkoshy pm->pm_state != PMC_STATE_RUNNING) { 764145256Sjkoshy atomic_add_int(&pmc_stats.pm_intr_ignored, 1); 765145256Sjkoshy continue; 766145256Sjkoshy } 767145256Sjkoshy 768145256Sjkoshy mode = pm->pm_mode; 769145256Sjkoshy if (PMC_IS_SAMPLING_MODE(mode) && 770145256Sjkoshy AMD_PMC_HAS_OVERFLOWED(perfctr)) { 771145256Sjkoshy atomic_add_int(&pmc_stats.pm_intr_processed, 1); 772145256Sjkoshy if (PMC_IS_SYSTEM_MODE(mode)) 773145256Sjkoshy pmc_update_histogram(phw, eip); 774145256Sjkoshy else if (PMC_IS_VIRTUAL_MODE(mode)) 775145256Sjkoshy pmc_send_signal(pm); 776145256Sjkoshy retval = 1; 777145256Sjkoshy } 778145256Sjkoshy } 779145256Sjkoshy return retval; 780145256Sjkoshy} 781145256Sjkoshy 782145256Sjkoshy/* 783145256Sjkoshy * describe a PMC 784145256Sjkoshy */ 785145256Sjkoshystatic int 786145256Sjkoshyamd_describe(int cpu, int ri, struct pmc_info *pi, struct pmc **ppmc) 787145256Sjkoshy{ 788145256Sjkoshy int error; 789145256Sjkoshy size_t copied; 790145256Sjkoshy const struct amd_descr *pd; 791145256Sjkoshy struct pmc_hw *phw; 792145256Sjkoshy 793145256Sjkoshy KASSERT(cpu >= 0 && cpu < mp_ncpus, 794145256Sjkoshy ("[amd,%d] illegal CPU %d", __LINE__, cpu)); 795145256Sjkoshy KASSERT(ri >= 0 && ri < AMD_NPMCS, 796145256Sjkoshy ("[amd,%d] row-index %d out of range", __LINE__, ri)); 797145256Sjkoshy 798145256Sjkoshy phw = pmc_pcpu[cpu]->pc_hwpmcs[ri]; 799145256Sjkoshy pd = &amd_pmcdesc[ri]; 800145256Sjkoshy 801145256Sjkoshy if ((error = copystr(pd->pm_descr.pd_name, pi->pm_name, 802145256Sjkoshy PMC_NAME_MAX, &copied)) != 0) 803145256Sjkoshy return error; 804145256Sjkoshy 805145256Sjkoshy pi->pm_class = pd->pm_descr.pd_class; 806145256Sjkoshy pi->pm_caps = pd->pm_descr.pd_caps; 807145256Sjkoshy pi->pm_width = pd->pm_descr.pd_width; 808145256Sjkoshy 809145256Sjkoshy if (phw->phw_state & PMC_PHW_FLAG_IS_ENABLED) { 810145256Sjkoshy pi->pm_enabled = TRUE; 811145256Sjkoshy *ppmc = phw->phw_pmc; 812145256Sjkoshy } else { 813145256Sjkoshy pi->pm_enabled = FALSE; 814145256Sjkoshy *ppmc = NULL; 815145256Sjkoshy } 816145256Sjkoshy 817145256Sjkoshy return 0; 818145256Sjkoshy} 819145256Sjkoshy 820145256Sjkoshy/* 821145256Sjkoshy * i386 specific entry points 822145256Sjkoshy */ 823145256Sjkoshy 824145256Sjkoshy/* 825145256Sjkoshy * return the MSR address of the given PMC. 826145256Sjkoshy */ 827145256Sjkoshy 828145256Sjkoshystatic int 829145256Sjkoshyamd_get_msr(int ri, uint32_t *msr) 830145256Sjkoshy{ 831145256Sjkoshy KASSERT(ri >= 0 && ri < AMD_NPMCS, 832145256Sjkoshy ("[amd,%d] ri %d out of range", __LINE__, ri)); 833145256Sjkoshy 834145615Sjkoshy *msr = amd_pmcdesc[ri].pm_perfctr - AMD_PMC_PERFCTR_0; 835145256Sjkoshy return 0; 836145256Sjkoshy} 837145256Sjkoshy 838145256Sjkoshy/* 839145256Sjkoshy * processor dependent initialization. 840145256Sjkoshy */ 841145256Sjkoshy 842145256Sjkoshy/* 843145256Sjkoshy * Per-processor data structure 844145256Sjkoshy * 845145256Sjkoshy * [common stuff] 846145256Sjkoshy * [5 struct pmc_hw pointers] 847145256Sjkoshy * [5 struct pmc_hw structures] 848145256Sjkoshy */ 849145256Sjkoshy 850145256Sjkoshystruct amd_cpu { 851145256Sjkoshy struct pmc_cpu pc_common; 852145256Sjkoshy struct pmc_hw *pc_hwpmcs[AMD_NPMCS]; 853145256Sjkoshy struct pmc_hw pc_amdpmcs[AMD_NPMCS]; 854145256Sjkoshy}; 855145256Sjkoshy 856145256Sjkoshy 857145256Sjkoshystatic int 858145256Sjkoshyamd_init(int cpu) 859145256Sjkoshy{ 860145256Sjkoshy int n; 861145256Sjkoshy struct amd_cpu *pcs; 862145256Sjkoshy struct pmc_hw *phw; 863145256Sjkoshy 864145256Sjkoshy KASSERT(cpu >= 0 && cpu < mp_ncpus, 865145256Sjkoshy ("[amd,%d] insane cpu number %d", __LINE__, cpu)); 866145256Sjkoshy 867145256Sjkoshy PMCDBG(MDP,INI,1,"amd-init cpu=%d", cpu); 868145256Sjkoshy 869145256Sjkoshy MALLOC(pcs, struct amd_cpu *, sizeof(struct amd_cpu), M_PMC, 870145256Sjkoshy M_WAITOK|M_ZERO); 871145256Sjkoshy 872145256Sjkoshy if (pcs == NULL) 873145256Sjkoshy return ENOMEM; 874145256Sjkoshy 875145256Sjkoshy phw = &pcs->pc_amdpmcs[0]; 876145256Sjkoshy 877145256Sjkoshy /* 878145256Sjkoshy * Initialize the per-cpu mutex and set the content of the 879145256Sjkoshy * hardware descriptors to a known state. 880145256Sjkoshy */ 881145256Sjkoshy 882145256Sjkoshy for (n = 0; n < AMD_NPMCS; n++, phw++) { 883145256Sjkoshy phw->phw_state = PMC_PHW_FLAG_IS_ENABLED | 884145256Sjkoshy PMC_PHW_CPU_TO_STATE(cpu) | PMC_PHW_INDEX_TO_STATE(n); 885145256Sjkoshy phw->phw_pmc = NULL; 886145256Sjkoshy pcs->pc_hwpmcs[n] = phw; 887145256Sjkoshy } 888145256Sjkoshy 889145256Sjkoshy /* Mark the TSC as shareable */ 890145256Sjkoshy pcs->pc_hwpmcs[0]->phw_state |= PMC_PHW_FLAG_IS_SHAREABLE; 891145256Sjkoshy 892145256Sjkoshy pmc_pcpu[cpu] = (struct pmc_cpu *) pcs; 893145256Sjkoshy 894145256Sjkoshy return 0; 895145256Sjkoshy} 896145256Sjkoshy 897145256Sjkoshy 898145256Sjkoshy/* 899145256Sjkoshy * processor dependent cleanup prior to the KLD 900145256Sjkoshy * being unloaded 901145256Sjkoshy */ 902145256Sjkoshy 903145256Sjkoshystatic int 904145256Sjkoshyamd_cleanup(int cpu) 905145256Sjkoshy{ 906145256Sjkoshy int i; 907145256Sjkoshy uint32_t evsel; 908145256Sjkoshy struct pmc_cpu *pcs; 909145256Sjkoshy 910145256Sjkoshy KASSERT(cpu >= 0 && cpu < mp_ncpus, 911145256Sjkoshy ("[amd,%d] insane cpu number (%d)", __LINE__, cpu)); 912145256Sjkoshy 913145256Sjkoshy PMCDBG(MDP,INI,1,"amd-cleanup cpu=%d", cpu); 914145256Sjkoshy 915145256Sjkoshy /* 916145256Sjkoshy * First, turn off all PMCs on this CPU. 917145256Sjkoshy */ 918145256Sjkoshy 919145256Sjkoshy for (i = 0; i < 4; i++) { /* XXX this loop is now not needed */ 920145256Sjkoshy evsel = rdmsr(AMD_PMC_EVSEL_0 + i); 921145256Sjkoshy evsel &= ~AMD_PMC_ENABLE; 922145256Sjkoshy wrmsr(AMD_PMC_EVSEL_0 + i, evsel); 923145256Sjkoshy } 924145256Sjkoshy 925145256Sjkoshy /* 926145256Sjkoshy * Next, free up allocated space. 927145256Sjkoshy */ 928145256Sjkoshy 929145256Sjkoshy pcs = pmc_pcpu[cpu]; 930145256Sjkoshy 931145256Sjkoshy#if DEBUG 932145256Sjkoshy /* check the TSC */ 933145256Sjkoshy KASSERT(pcs->pc_hwpmcs[0]->phw_pmc == NULL, 934145256Sjkoshy ("[amd,%d] CPU%d,PMC0 still in use", __LINE__, cpu)); 935145256Sjkoshy for (i = 1; i < AMD_NPMCS; i++) { 936145256Sjkoshy KASSERT(pcs->pc_hwpmcs[i]->phw_pmc == NULL, 937145256Sjkoshy ("[amd,%d] CPU%d/PMC%d in use", __LINE__, cpu, i)); 938145256Sjkoshy KASSERT(AMD_PMC_IS_STOPPED(AMD_PMC_EVSEL_0 + (i-1)), 939145256Sjkoshy ("[amd,%d] CPU%d/PMC%d not stopped", __LINE__, cpu, i)); 940145256Sjkoshy } 941145256Sjkoshy#endif 942145256Sjkoshy KASSERT(pcs != NULL, 943145256Sjkoshy ("[amd,%d] null per-cpu state pointer (cpu%d)", __LINE__, cpu)); 944145256Sjkoshy 945145256Sjkoshy pmc_pcpu[cpu] = NULL; 946145256Sjkoshy FREE(pcs, M_PMC); 947145256Sjkoshy return 0; 948145256Sjkoshy} 949145256Sjkoshy 950145256Sjkoshy/* 951145256Sjkoshy * Initialize ourselves. 952145256Sjkoshy */ 953145256Sjkoshy 954145256Sjkoshystruct pmc_mdep * 955145256Sjkoshypmc_amd_initialize(void) 956145256Sjkoshy{ 957145256Sjkoshy 958145256Sjkoshy struct pmc_mdep *pmc_mdep; 959145256Sjkoshy 960145256Sjkoshy /* The presence of hardware performance counters on the AMD 961145256Sjkoshy Athlon, Duron or later processors, is _not_ indicated by 962145256Sjkoshy any of the processor feature flags set by the 'CPUID' 963145256Sjkoshy instruction, so we only check the 'instruction family' 964145256Sjkoshy field returned by CPUID for instruction family >= 6. This 965145256Sjkoshy test needs to be be refined. */ 966145256Sjkoshy 967145256Sjkoshy if ((cpu_id & 0xF00) < 0x600) 968145256Sjkoshy return NULL; 969145256Sjkoshy 970145256Sjkoshy MALLOC(pmc_mdep, struct pmc_mdep *, sizeof(struct pmc_mdep), 971145256Sjkoshy M_PMC, M_WAITOK|M_ZERO); 972145256Sjkoshy 973145256Sjkoshy#if __i386__ 974145256Sjkoshy pmc_mdep->pmd_cputype = PMC_CPU_AMD_K7; 975145256Sjkoshy#elif __amd64__ 976145256Sjkoshy pmc_mdep->pmd_cputype = PMC_CPU_AMD_K8; 977145256Sjkoshy#else 978145256Sjkoshy#error Unknown AMD CPU type. 979145256Sjkoshy#endif 980145256Sjkoshy 981145256Sjkoshy pmc_mdep->pmd_npmc = AMD_NPMCS; 982145256Sjkoshy 983145256Sjkoshy /* this processor has two classes of usable PMCs */ 984145256Sjkoshy pmc_mdep->pmd_nclass = 2; 985145256Sjkoshy pmc_mdep->pmd_classes[0] = PMC_CLASS_TSC; 986145256Sjkoshy pmc_mdep->pmd_classes[1] = AMD_PMC_CLASS; 987145256Sjkoshy pmc_mdep->pmd_nclasspmcs[0] = 1; 988145256Sjkoshy pmc_mdep->pmd_nclasspmcs[1] = (AMD_NPMCS-1); 989145256Sjkoshy 990145256Sjkoshy pmc_mdep->pmd_init = amd_init; 991145256Sjkoshy pmc_mdep->pmd_cleanup = amd_cleanup; 992145256Sjkoshy pmc_mdep->pmd_switch_in = amd_switch_in; 993145256Sjkoshy pmc_mdep->pmd_switch_out = amd_switch_out; 994145256Sjkoshy pmc_mdep->pmd_read_pmc = amd_read_pmc; 995145256Sjkoshy pmc_mdep->pmd_write_pmc = amd_write_pmc; 996145256Sjkoshy pmc_mdep->pmd_config_pmc = amd_config_pmc; 997145256Sjkoshy pmc_mdep->pmd_allocate_pmc = amd_allocate_pmc; 998145256Sjkoshy pmc_mdep->pmd_release_pmc = amd_release_pmc; 999145256Sjkoshy pmc_mdep->pmd_start_pmc = amd_start_pmc; 1000145256Sjkoshy pmc_mdep->pmd_stop_pmc = amd_stop_pmc; 1001145256Sjkoshy pmc_mdep->pmd_intr = amd_intr; 1002145256Sjkoshy pmc_mdep->pmd_describe = amd_describe; 1003145256Sjkoshy pmc_mdep->pmd_get_msr = amd_get_msr; /* i386 */ 1004145256Sjkoshy 1005145256Sjkoshy PMCDBG(MDP,INI,0,"%s","amd-initialize"); 1006145256Sjkoshy 1007145256Sjkoshy return pmc_mdep; 1008145256Sjkoshy} 1009