hwpmc_amd.c revision 145774
1/*- 2 * Copyright (c) 2003-2005 Joseph Koshy 3 * All rights reserved. 4 * 5 * Redistribution and use in source and binary forms, with or without 6 * modification, are permitted provided that the following conditions 7 * are met: 8 * 1. Redistributions of source code must retain the above copyright 9 * notice, this list of conditions and the following disclaimer. 10 * 2. Redistributions in binary form must reproduce the above copyright 11 * notice, this list of conditions and the following disclaimer in the 12 * documentation and/or other materials provided with the distribution. 13 * 14 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 15 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 16 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 17 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 18 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 19 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 20 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 21 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 22 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 23 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 24 * SUCH DAMAGE. 25 * 26 */ 27 28#include <sys/cdefs.h> 29__FBSDID("$FreeBSD: head/sys/dev/hwpmc/hwpmc_amd.c 145774 2005-05-01 14:11:49Z jkoshy $"); 30 31/* Support for the AMD K7 and later processors */ 32 33#include <sys/param.h> 34#include <sys/lock.h> 35#include <sys/malloc.h> 36#include <sys/mutex.h> 37#include <sys/pmc.h> 38#include <sys/smp.h> 39#include <sys/systm.h> 40 41#include <machine/md_var.h> 42 43/* AMD K7 and K8 PMCs */ 44 45#define AMD_PMC_EVSEL_0 0xC0010000 46#define AMD_PMC_EVSEL_1 0xC0010001 47#define AMD_PMC_EVSEL_2 0xC0010002 48#define AMD_PMC_EVSEL_3 0xC0010003 49 50#define AMD_PMC_PERFCTR_0 0xC0010004 51#define AMD_PMC_PERFCTR_1 0xC0010005 52#define AMD_PMC_PERFCTR_2 0xC0010006 53#define AMD_PMC_PERFCTR_3 0xC0010007 54 55#define K7_VALID_EVENT_CODE(c) (((c) >= 0x40 && (c) <= 0x47) || \ 56 ((c) >= 0x80 && (c) <= 0x85) || ((c) >= 0xC0 && (c) <= 0xC7) || \ 57 ((c) >= 0xCD && (c) <= 0xCF)) 58 59#define AMD_PMC_CAPS (PMC_CAP_INTERRUPT | PMC_CAP_USER | \ 60 PMC_CAP_SYSTEM | PMC_CAP_EDGE | PMC_CAP_THRESHOLD | \ 61 PMC_CAP_READ | PMC_CAP_WRITE | PMC_CAP_INVERT | PMC_CAP_QUALIFIER) 62 63/* reserved bits include bit 21 and the top two bits of the unit mask */ 64#define K7_PMC_RESERVED ((1 << 21) | (3 << 13)) 65 66#define K8_PMC_RESERVED (1 << 21) 67 68#define AMD_PMC_IS_STOPPED(evsel) ((rdmsr((evsel)) & AMD_PMC_ENABLE) == 0) 69#define AMD_PMC_HAS_OVERFLOWED(pmc) ((rdpmc(pmc) & (1ULL << 47)) == 0) 70 71#if __i386__ 72#define AMD_NPMCS K7_NPMCS 73#define AMD_PMC_CLASS PMC_CLASS_K7 74#define AMD_PMC_COUNTERMASK K7_PMC_COUNTERMASK 75#define AMD_PMC_TO_COUNTER(x) K7_PMC_TO_COUNTER(x) 76#define AMD_PMC_INVERT K7_PMC_INVERT 77#define AMD_PMC_ENABLE K7_PMC_ENABLE 78#define AMD_PMC_INT K7_PMC_INT 79#define AMD_PMC_PC K7_PMC_PC 80#define AMD_PMC_EDGE K7_PMC_EDGE 81#define AMD_PMC_OS K7_PMC_OS 82#define AMD_PMC_USR K7_PMC_USR 83 84#define AMD_PMC_UNITMASK_M K7_PMC_UNITMASK_M 85#define AMD_PMC_UNITMASK_O K7_PMC_UNITMASK_O 86#define AMD_PMC_UNITMASK_E K7_PMC_UNITMASK_E 87#define AMD_PMC_UNITMASK_S K7_PMC_UNITMASK_S 88#define AMD_PMC_UNITMASK_I K7_PMC_UNITMASK_I 89 90#define AMD_PMC_UNITMASK K7_PMC_UNITMASK 91#define AMD_PMC_EVENTMASK K7_PMC_EVENTMASK 92#define AMD_PMC_TO_UNITMASK(x) K7_PMC_TO_UNITMASK(x) 93#define AMD_PMC_TO_EVENTMASK(x) K7_PMC_TO_EVENTMASK(x) 94#define AMD_VALID_BITS K7_VALID_BITS 95 96#define AMD_PMC_CLASS_NAME "K7-" 97 98#elif __amd64__ 99 100#define AMD_NPMCS K8_NPMCS 101#define AMD_PMC_CLASS PMC_CLASS_K8 102#define AMD_PMC_COUNTERMASK K8_PMC_COUNTERMASK 103#define AMD_PMC_TO_COUNTER(x) K8_PMC_TO_COUNTER(x) 104#define AMD_PMC_INVERT K8_PMC_INVERT 105#define AMD_PMC_ENABLE K8_PMC_ENABLE 106#define AMD_PMC_INT K8_PMC_INT 107#define AMD_PMC_PC K8_PMC_PC 108#define AMD_PMC_EDGE K8_PMC_EDGE 109#define AMD_PMC_OS K8_PMC_OS 110#define AMD_PMC_USR K8_PMC_USR 111 112#define AMD_PMC_UNITMASK_M K8_PMC_UNITMASK_M 113#define AMD_PMC_UNITMASK_O K8_PMC_UNITMASK_O 114#define AMD_PMC_UNITMASK_E K8_PMC_UNITMASK_E 115#define AMD_PMC_UNITMASK_S K8_PMC_UNITMASK_S 116#define AMD_PMC_UNITMASK_I K8_PMC_UNITMASK_I 117 118#define AMD_PMC_UNITMASK K8_PMC_UNITMASK 119#define AMD_PMC_EVENTMASK K8_PMC_EVENTMASK 120#define AMD_PMC_TO_UNITMASK(x) K8_PMC_TO_UNITMASK(x) 121#define AMD_PMC_TO_EVENTMASK(x) K8_PMC_TO_EVENTMASK(x) 122#define AMD_VALID_BITS K8_VALID_BITS 123 124#define AMD_PMC_CLASS_NAME "K8-" 125 126#else 127#error Unsupported architecture. 128#endif 129 130/* AMD K7 & K8 PMCs */ 131struct amd_descr { 132 struct pmc_descr pm_descr; /* "base class" */ 133 uint32_t pm_evsel; /* address of EVSEL register */ 134 uint32_t pm_perfctr; /* address of PERFCTR register */ 135}; 136 137static const struct amd_descr amd_pmcdesc[AMD_NPMCS] = 138{ 139 { 140 .pm_descr = 141 { 142 .pd_name = "TSC", 143 .pd_class = PMC_CLASS_TSC, 144 .pd_caps = PMC_CAP_READ, 145 .pd_width = 64 146 }, 147 .pm_evsel = MSR_TSC, 148 .pm_perfctr = 0 /* unused */ 149 }, 150 151 { 152 .pm_descr = 153 { 154 .pd_name = AMD_PMC_CLASS_NAME "0", 155 .pd_class = AMD_PMC_CLASS, 156 .pd_caps = AMD_PMC_CAPS, 157 .pd_width = 48 158 }, 159 .pm_evsel = AMD_PMC_EVSEL_0, 160 .pm_perfctr = AMD_PMC_PERFCTR_0 161 }, 162 { 163 .pm_descr = 164 { 165 .pd_name = AMD_PMC_CLASS_NAME "1", 166 .pd_class = AMD_PMC_CLASS, 167 .pd_caps = AMD_PMC_CAPS, 168 .pd_width = 48 169 }, 170 .pm_evsel = AMD_PMC_EVSEL_1, 171 .pm_perfctr = AMD_PMC_PERFCTR_1 172 }, 173 { 174 .pm_descr = 175 { 176 .pd_name = AMD_PMC_CLASS_NAME "2", 177 .pd_class = AMD_PMC_CLASS, 178 .pd_caps = AMD_PMC_CAPS, 179 .pd_width = 48 180 }, 181 .pm_evsel = AMD_PMC_EVSEL_2, 182 .pm_perfctr = AMD_PMC_PERFCTR_2 183 }, 184 { 185 .pm_descr = 186 { 187 .pd_name = AMD_PMC_CLASS_NAME "3", 188 .pd_class = AMD_PMC_CLASS, 189 .pd_caps = AMD_PMC_CAPS, 190 .pd_width = 48 191 }, 192 .pm_evsel = AMD_PMC_EVSEL_3, 193 .pm_perfctr = AMD_PMC_PERFCTR_3 194 } 195}; 196 197struct amd_event_code_map { 198 enum pmc_event pe_ev; /* enum value */ 199 uint8_t pe_code; /* encoded event mask */ 200 uint8_t pe_mask; /* bits allowed in unit mask */ 201}; 202 203const struct amd_event_code_map amd_event_codes[] = { 204#if __i386__ 205 { PMC_EV_K7_DC_ACCESSES, 0x40, 0 }, 206 { PMC_EV_K7_DC_MISSES, 0x41, 0 }, 207 { PMC_EV_K7_DC_REFILLS_FROM_L2, 0x42, K7_PMC_UNITMASK_MOESI }, 208 { PMC_EV_K7_DC_REFILLS_FROM_SYSTEM, 0x43, K7_PMC_UNITMASK_MOESI }, 209 { PMC_EV_K7_DC_WRITEBACKS, 0x44, K7_PMC_UNITMASK_MOESI }, 210 { PMC_EV_K7_L1_DTLB_MISS_AND_L2_DTLB_HITS, 0x45, 0 }, 211 { PMC_EV_K7_L1_AND_L2_DTLB_MISSES, 0x46, 0 }, 212 { PMC_EV_K7_MISALIGNED_REFERENCES, 0x47, 0 }, 213 214 { PMC_EV_K7_IC_FETCHES, 0x80, 0 }, 215 { PMC_EV_K7_IC_MISSES, 0x81, 0 }, 216 217 { PMC_EV_K7_L1_ITLB_MISSES, 0x84, 0 }, 218 { PMC_EV_K7_L1_L2_ITLB_MISSES, 0x85, 0 }, 219 220 { PMC_EV_K7_RETIRED_INSTRUCTIONS, 0xC0, 0 }, 221 { PMC_EV_K7_RETIRED_OPS, 0xC1, 0 }, 222 { PMC_EV_K7_RETIRED_BRANCHES, 0xC2, 0 }, 223 { PMC_EV_K7_RETIRED_BRANCHES_MISPREDICTED, 0xC3, 0 }, 224 { PMC_EV_K7_RETIRED_TAKEN_BRANCHES, 0xC4, 0 }, 225 { PMC_EV_K7_RETIRED_TAKEN_BRANCHES_MISPREDICTED, 0xC5, 0 }, 226 { PMC_EV_K7_RETIRED_FAR_CONTROL_TRANSFERS, 0xC6, 0 }, 227 { PMC_EV_K7_RETIRED_RESYNC_BRANCHES, 0xC7, 0 }, 228 { PMC_EV_K7_INTERRUPTS_MASKED_CYCLES, 0xCD, 0 }, 229 { PMC_EV_K7_INTERRUPTS_MASKED_WHILE_PENDING_CYCLES, 0xCE, 0 }, 230 { PMC_EV_K7_HARDWARE_INTERRUPTS, 0xCF, 0 } 231#endif 232 233#if __amd64__ 234 { PMC_EV_K8_FP_DISPATCHED_FPU_OPS, 0x00, 0x3F }, 235 { PMC_EV_K8_FP_CYCLES_WITH_NO_FPU_OPS_RETIRED, 0x01, 0x00 }, 236 { PMC_EV_K8_FP_DISPATCHED_FPU_FAST_FLAG_OPS, 0x02, 0x00 }, 237 238 { PMC_EV_K8_LS_SEGMENT_REGISTER_LOAD, 0x20, 0x7F }, 239 { PMC_EV_K8_LS_MICROARCHITECTURAL_RESYNC_BY_SELF_MODIFYING_CODE, 240 0x21, 0x00 }, 241 { PMC_EV_K8_LS_MICROARCHITECTURAL_RESYNC_BY_SNOOP, 0x22, 0x00 }, 242 { PMC_EV_K8_LS_BUFFER2_FULL, 0x23, 0x00 }, 243 { PMC_EV_K8_LS_LOCKED_OPERATION, 0x24, 0x07 }, 244 { PMC_EV_K8_LS_MICROARCHITECTURAL_LATE_CANCEL, 0x25, 0x00 }, 245 { PMC_EV_K8_LS_RETIRED_CFLUSH_INSTRUCTIONS, 0x26, 0x00 }, 246 { PMC_EV_K8_LS_RETIRED_CPUID_INSTRUCTIONS, 0x27, 0x00 }, 247 248 { PMC_EV_K8_DC_ACCESS, 0x40, 0x00 }, 249 { PMC_EV_K8_DC_MISS, 0x41, 0x00 }, 250 { PMC_EV_K8_DC_REFILL_FROM_L2, 0x42, 0x1F }, 251 { PMC_EV_K8_DC_REFILL_FROM_SYSTEM, 0x43, 0x1F }, 252 { PMC_EV_K8_DC_COPYBACK, 0x44, 0x1F }, 253 { PMC_EV_K8_DC_L1_DTLB_MISS_AND_L2_DTLB_HIT, 0x45, 0x00 }, 254 { PMC_EV_K8_DC_L1_DTLB_MISS_AND_L2_DTLB_MISS, 0x46, 0x00 }, 255 { PMC_EV_K8_DC_MISALIGNED_DATA_REFERENCE, 0x47, 0x00 }, 256 { PMC_EV_K8_DC_MICROARCHITECTURAL_LATE_CANCEL, 0x48, 0x00 }, 257 { PMC_EV_K8_DC_MICROARCHITECTURAL_EARLY_CANCEL, 0x49, 0x00 }, 258 { PMC_EV_K8_DC_ONE_BIT_ECC_ERROR, 0x4A, 0x03 }, 259 { PMC_EV_K8_DC_DISPATCHED_PREFETCH_INSTRUCTIONS, 0x4B, 0x07 }, 260 { PMC_EV_K8_DC_DCACHE_ACCESSES_BY_LOCKS, 0x4C, 0x03 }, 261 262 { PMC_EV_K8_BU_CPU_CLK_UNHALTED, 0x76, 0x00 }, 263 { PMC_EV_K8_BU_INTERNAL_L2_REQUEST, 0x7D, 0x1F }, 264 { PMC_EV_K8_BU_FILL_REQUEST_L2_MISS, 0x7E, 0x07 }, 265 { PMC_EV_K8_BU_FILL_INTO_L2, 0x7F, 0x03 }, 266 267 { PMC_EV_K8_IC_FETCH, 0x80, 0x00 }, 268 { PMC_EV_K8_IC_MISS, 0x81, 0x00 }, 269 { PMC_EV_K8_IC_REFILL_FROM_L2, 0x82, 0x00 }, 270 { PMC_EV_K8_IC_REFILL_FROM_SYSTEM, 0x83, 0x00 }, 271 { PMC_EV_K8_IC_L1_ITLB_MISS_AND_L2_ITLB_HIT, 0x84, 0x00 }, 272 { PMC_EV_K8_IC_L1_ITLB_MISS_AND_L2_ITLB_MISS, 0x85, 0x00 }, 273 { PMC_EV_K8_IC_MICROARCHITECTURAL_RESYNC_BY_SNOOP, 0x86, 0x00 }, 274 { PMC_EV_K8_IC_INSTRUCTION_FETCH_STALL, 0x87, 0x00 }, 275 { PMC_EV_K8_IC_RETURN_STACK_HIT, 0x88, 0x00 }, 276 { PMC_EV_K8_IC_RETURN_STACK_OVERFLOW, 0x89, 0x00 }, 277 278 { PMC_EV_K8_FR_RETIRED_X86_INSTRUCTIONS, 0xC0, 0x00 }, 279 { PMC_EV_K8_FR_RETIRED_UOPS, 0xC1, 0x00 }, 280 { PMC_EV_K8_FR_RETIRED_BRANCHES, 0xC2, 0x00 }, 281 { PMC_EV_K8_FR_RETIRED_BRANCHES_MISPREDICTED, 0xC3, 0x00 }, 282 { PMC_EV_K8_FR_RETIRED_TAKEN_BRANCHES, 0xC4, 0x00 }, 283 { PMC_EV_K8_FR_RETIRED_TAKEN_BRANCHES_MISPREDICTED, 0xC5, 0x00 }, 284 { PMC_EV_K8_FR_RETIRED_FAR_CONTROL_TRANSFERS, 0xC6, 0x00 }, 285 { PMC_EV_K8_FR_RETIRED_RESYNCS, 0xC7, 0x00 }, 286 { PMC_EV_K8_FR_RETIRED_NEAR_RETURNS, 0xC8, 0x00 }, 287 { PMC_EV_K8_FR_RETIRED_NEAR_RETURNS_MISPREDICTED, 0xC9, 0x00 }, 288 { PMC_EV_K8_FR_RETIRED_TAKEN_BRANCHES_MISPREDICTED_BY_ADDR_MISCOMPARE, 289 0xCA, 0x00 }, 290 { PMC_EV_K8_FR_RETIRED_FPU_INSTRUCTIONS, 0xCB, 0x0F }, 291 { PMC_EV_K8_FR_RETIRED_FASTPATH_DOUBLE_OP_INSTRUCTIONS, 292 0xCC, 0x07 }, 293 { PMC_EV_K8_FR_INTERRUPTS_MASKED_CYCLES, 0xCD, 0x00 }, 294 { PMC_EV_K8_FR_INTERRUPTS_MASKED_WHILE_PENDING_CYCLES, 0xCE, 0x00 }, 295 { PMC_EV_K8_FR_TAKEN_HARDWARE_INTERRUPTS, 0xCF, 0x00 }, 296 297 { PMC_EV_K8_FR_DECODER_EMPTY, 0xD0, 0x00 }, 298 { PMC_EV_K8_FR_DISPATCH_STALLS, 0xD1, 0x00 }, 299 { PMC_EV_K8_FR_DISPATCH_STALL_FROM_BRANCH_ABORT_TO_RETIRE, 300 0xD2, 0x00 }, 301 { PMC_EV_K8_FR_DISPATCH_STALL_FOR_SERIALIZATION, 0xD3, 0x00 }, 302 { PMC_EV_K8_FR_DISPATCH_STALL_FOR_SEGMENT_LOAD, 0xD4, 0x00 }, 303 { PMC_EV_K8_FR_DISPATCH_STALL_WHEN_REORDER_BUFFER_IS_FULL, 304 0xD5, 0x00 }, 305 { PMC_EV_K8_FR_DISPATCH_STALL_WHEN_RESERVATION_STATIONS_ARE_FULL, 306 0xD6, 0x00 }, 307 { PMC_EV_K8_FR_DISPATCH_STALL_WHEN_FPU_IS_FULL, 0xD7, 0x00 }, 308 { PMC_EV_K8_FR_DISPATCH_STALL_WHEN_LS_IS_FULL, 0xD8, 0x00 }, 309 { PMC_EV_K8_FR_DISPATCH_STALL_WHEN_WAITING_FOR_ALL_TO_BE_QUIET, 310 0xD9, 0x00 }, 311 { PMC_EV_K8_FR_DISPATCH_STALL_WHEN_FAR_XFER_OR_RESYNC_BRANCH_PENDING, 312 0xDA, 0x00 }, 313 { PMC_EV_K8_FR_FPU_EXCEPTIONS, 0xDB, 0x0F }, 314 { PMC_EV_K8_FR_NUMBER_OF_BREAKPOINTS_FOR_DR0, 0xDC, 0x00 }, 315 { PMC_EV_K8_FR_NUMBER_OF_BREAKPOINTS_FOR_DR1, 0xDD, 0x00 }, 316 { PMC_EV_K8_FR_NUMBER_OF_BREAKPOINTS_FOR_DR2, 0xDE, 0x00 }, 317 { PMC_EV_K8_FR_NUMBER_OF_BREAKPOINTS_FOR_DR3, 0xDF, 0x00 }, 318 319 { PMC_EV_K8_NB_MEMORY_CONTROLLER_PAGE_ACCESS_EVENT, 0xE0, 0x7 }, 320 { PMC_EV_K8_NB_MEMORY_CONTROLLER_PAGE_TABLE_OVERFLOW, 0xE1, 0x00 }, 321 { PMC_EV_K8_NB_MEMORY_CONTROLLER_DRAM_COMMAND_SLOTS_MISSED, 322 0xE2, 0x00 }, 323 { PMC_EV_K8_NB_MEMORY_CONTROLLER_TURNAROUND, 0xE3, 0x07 }, 324 { PMC_EV_K8_NB_MEMORY_CONTROLLER_BYPASS_SATURATION, 0xE4, 0x0F }, 325 { PMC_EV_K8_NB_SIZED_COMMANDS, 0xEB, 0x7F }, 326 { PMC_EV_K8_NB_PROBE_RESULT, 0xEC, 0x0F }, 327 { PMC_EV_K8_NB_HT_BUS0_BANDWIDTH, 0xF6, 0x0F }, 328 { PMC_EV_K8_NB_HT_BUS1_BANDWIDTH, 0xF7, 0x0F }, 329 { PMC_EV_K8_NB_HT_BUS2_BANDWIDTH, 0xF8, 0x0F } 330#endif 331 332}; 333 334const int amd_event_codes_size = 335 sizeof(amd_event_codes) / sizeof(amd_event_codes[0]); 336 337/* 338 * read a pmc register 339 */ 340 341static int 342amd_read_pmc(int cpu, int ri, pmc_value_t *v) 343{ 344 enum pmc_mode mode; 345 const struct amd_descr *pd; 346 struct pmc *pm; 347 const struct pmc_hw *phw; 348 pmc_value_t tmp; 349 350 KASSERT(cpu >= 0 && cpu < mp_ncpus, 351 ("[amd,%d] illegal CPU value %d", __LINE__, cpu)); 352 KASSERT(ri >= 0 && ri < AMD_NPMCS, 353 ("[amd,%d] illegal row-index %d", __LINE__, ri)); 354 355 phw = pmc_pcpu[cpu]->pc_hwpmcs[ri]; 356 pd = &amd_pmcdesc[ri]; 357 pm = phw->phw_pmc; 358 359 KASSERT(pm != NULL, 360 ("[amd,%d] No owner for HWPMC [cpu%d,pmc%d]", __LINE__, 361 cpu, ri)); 362 363 mode = PMC_TO_MODE(pm); 364 365 PMCDBG(MDP,REA,1,"amd-read id=%d class=%d", ri, pd->pm_descr.pd_class); 366 367 /* Reading the TSC is a special case */ 368 if (pd->pm_descr.pd_class == PMC_CLASS_TSC) { 369 KASSERT(PMC_IS_COUNTING_MODE(mode), 370 ("[amd,%d] TSC counter in non-counting mode", __LINE__)); 371 *v = rdtsc(); 372 PMCDBG(MDP,REA,2,"amd-read id=%d -> %jd", ri, *v); 373 return 0; 374 } 375 376 KASSERT(pd->pm_descr.pd_class == AMD_PMC_CLASS, 377 ("[amd,%d] unknown PMC class (%d)", __LINE__, 378 pd->pm_descr.pd_class)); 379 380 tmp = rdmsr(pd->pm_perfctr); /* RDMSR serializes */ 381 if (PMC_IS_SAMPLING_MODE(mode)) 382 *v = -tmp; 383 else 384 *v = tmp; 385 386 PMCDBG(MDP,REA,2,"amd-read id=%d -> %jd", ri, *v); 387 388 return 0; 389} 390 391/* 392 * Write a PMC MSR. 393 */ 394 395static int 396amd_write_pmc(int cpu, int ri, pmc_value_t v) 397{ 398 const struct amd_descr *pd; 399 struct pmc *pm; 400 const struct pmc_hw *phw; 401 enum pmc_mode mode; 402 403 KASSERT(cpu >= 0 && cpu < mp_ncpus, 404 ("[amd,%d] illegal CPU value %d", __LINE__, cpu)); 405 KASSERT(ri >= 0 && ri < AMD_NPMCS, 406 ("[amd,%d] illegal row-index %d", __LINE__, ri)); 407 408 phw = pmc_pcpu[cpu]->pc_hwpmcs[ri]; 409 pd = &amd_pmcdesc[ri]; 410 pm = phw->phw_pmc; 411 412 KASSERT(pm != NULL, 413 ("[amd,%d] PMC not owned (cpu%d,pmc%d)", __LINE__, 414 cpu, ri)); 415 416 mode = PMC_TO_MODE(pm); 417 418 if (pd->pm_descr.pd_class == PMC_CLASS_TSC) 419 return 0; 420 421 KASSERT(pd->pm_descr.pd_class == AMD_PMC_CLASS, 422 ("[amd,%d] unknown PMC class (%d)", __LINE__, 423 pd->pm_descr.pd_class)); 424 425 /* use 2's complement of the count for sampling mode PMCs */ 426 if (PMC_IS_SAMPLING_MODE(mode)) 427 v = -v; 428 429 PMCDBG(MDP,WRI,1,"amd-write cpu=%d ri=%d v=%jx", cpu, ri, v); 430 431 /* write the PMC value */ 432 wrmsr(pd->pm_perfctr, v); 433 return 0; 434} 435 436/* 437 * configure hardware pmc according to the configuration recorded in 438 * pmc 'pm'. 439 */ 440 441static int 442amd_config_pmc(int cpu, int ri, struct pmc *pm) 443{ 444 struct pmc_hw *phw; 445 446 PMCDBG(MDP,CFG,1, "cpu=%d ri=%d pm=%p", cpu, ri, pm); 447 448 KASSERT(cpu >= 0 && cpu < mp_ncpus, 449 ("[amd,%d] illegal CPU value %d", __LINE__, cpu)); 450 KASSERT(ri >= 0 && ri < AMD_NPMCS, 451 ("[amd,%d] illegal row-index %d", __LINE__, ri)); 452 453 phw = pmc_pcpu[cpu]->pc_hwpmcs[ri]; 454 455 KASSERT(pm == NULL || phw->phw_pmc == NULL, 456 ("[amd,%d] pm=%p phw->pm=%p hwpmc not unconfigured", 457 __LINE__, pm, phw->phw_pmc)); 458 459 phw->phw_pmc = pm; 460 return 0; 461} 462 463/* 464 * Retrieve a configured PMC pointer from hardware state. 465 */ 466 467static int 468amd_get_config(int cpu, int ri, struct pmc **ppm) 469{ 470 *ppm = pmc_pcpu[cpu]->pc_hwpmcs[ri]->phw_pmc; 471 472 return 0; 473} 474 475/* 476 * Machine dependent actions taken during the context switch in of a 477 * thread. 478 */ 479 480static int 481amd_switch_in(struct pmc_cpu *pc, struct pmc_process *pp) 482{ 483 (void) pc; 484 485 PMCDBG(MDP,SWI,1, "pc=%p pp=%p enable-msr=%d", pc, pp, 486 (pp->pp_flags & PMC_PP_ENABLE_MSR_ACCESS) != 0); 487 488 /* enable the RDPMC instruction if needed */ 489 if (pp->pp_flags & PMC_PP_ENABLE_MSR_ACCESS) 490 load_cr4(rcr4() | CR4_PCE); 491 492 return 0; 493} 494 495/* 496 * Machine dependent actions taken during the context switch out of a 497 * thread. 498 */ 499 500static int 501amd_switch_out(struct pmc_cpu *pc, struct pmc_process *pp) 502{ 503 (void) pc; 504 (void) pp; /* can be NULL */ 505 506 PMCDBG(MDP,SWO,1, "pc=%p pp=%p enable-msr=%d", pc, pp, pp ? 507 (pp->pp_flags & PMC_PP_ENABLE_MSR_ACCESS) == 1 : 0); 508 509 /* always turn off the RDPMC instruction */ 510 load_cr4(rcr4() & ~CR4_PCE); 511 512 return 0; 513} 514 515/* 516 * Check if a given allocation is feasible. 517 */ 518 519static int 520amd_allocate_pmc(int cpu, int ri, struct pmc *pm, 521 const struct pmc_op_pmcallocate *a) 522{ 523 int i; 524 uint32_t allowed_unitmask, caps, config, unitmask; 525 enum pmc_event pe; 526 const struct pmc_descr *pd; 527 528 (void) cpu; 529 530 KASSERT(cpu >= 0 && cpu < mp_ncpus, 531 ("[amd,%d] illegal CPU value %d", __LINE__, cpu)); 532 KASSERT(ri >= 0 && ri < AMD_NPMCS, 533 ("[amd,%d] illegal row index %d", __LINE__, ri)); 534 535 pd = &amd_pmcdesc[ri].pm_descr; 536 537 /* check class match */ 538 if (pd->pd_class != a->pm_class) 539 return EINVAL; 540 541 caps = pm->pm_caps; 542 543 PMCDBG(MDP,ALL,1,"amd-allocate ri=%d caps=0x%x", ri, caps); 544 545 if ((pd->pd_caps & caps) != caps) 546 return EPERM; 547 if (pd->pd_class == PMC_CLASS_TSC) { 548 /* TSC's are always allocated in system-wide counting mode */ 549 if (a->pm_ev != PMC_EV_TSC_TSC || 550 a->pm_mode != PMC_MODE_SC) 551 return EINVAL; 552 return 0; 553 } 554 555 KASSERT(pd->pd_class == AMD_PMC_CLASS, 556 ("[amd,%d] Unknown PMC class (%d)", __LINE__, pd->pd_class)); 557 558 pe = a->pm_ev; 559 560 /* map ev to the correct event mask code */ 561 config = allowed_unitmask = 0; 562 for (i = 0; i < amd_event_codes_size; i++) 563 if (amd_event_codes[i].pe_ev == pe) { 564 config = 565 AMD_PMC_TO_EVENTMASK(amd_event_codes[i].pe_code); 566 allowed_unitmask = 567 AMD_PMC_TO_UNITMASK(amd_event_codes[i].pe_mask); 568 break; 569 } 570 if (i == amd_event_codes_size) 571 return EINVAL; 572 573 unitmask = a->pm_amd_config & AMD_PMC_UNITMASK; 574 if (unitmask & ~allowed_unitmask) /* disallow reserved bits */ 575 return EINVAL; 576 577 if (unitmask && (caps & PMC_CAP_QUALIFIER)) 578 config |= unitmask; 579 580 if (caps & PMC_CAP_THRESHOLD) 581 config |= a->pm_amd_config & AMD_PMC_COUNTERMASK; 582 583 /* set at least one of the 'usr' or 'os' caps */ 584 if (caps & PMC_CAP_USER) 585 config |= AMD_PMC_USR; 586 if (caps & PMC_CAP_SYSTEM) 587 config |= AMD_PMC_OS; 588 if ((caps & (PMC_CAP_USER|PMC_CAP_SYSTEM)) == 0) 589 config |= (AMD_PMC_USR|AMD_PMC_OS); 590 591 if (caps & PMC_CAP_EDGE) 592 config |= AMD_PMC_EDGE; 593 if (caps & PMC_CAP_INVERT) 594 config |= AMD_PMC_INVERT; 595 if (caps & PMC_CAP_INTERRUPT) 596 config |= AMD_PMC_INT; 597 598 pm->pm_md.pm_amd.pm_amd_evsel = config; /* save config value */ 599 600 PMCDBG(MDP,ALL,2,"amd-allocate ri=%d -> config=0x%x", ri, config); 601 602 return 0; 603} 604 605/* 606 * Release machine dependent state associated with a PMC. This is a 607 * no-op on this architecture. 608 * 609 */ 610 611/* ARGSUSED0 */ 612static int 613amd_release_pmc(int cpu, int ri, struct pmc *pmc) 614{ 615#if DEBUG 616 const struct amd_descr *pd; 617#endif 618 struct pmc_hw *phw; 619 620 (void) pmc; 621 622 KASSERT(cpu >= 0 && cpu < mp_ncpus, 623 ("[amd,%d] illegal CPU value %d", __LINE__, cpu)); 624 KASSERT(ri >= 0 && ri < AMD_NPMCS, 625 ("[amd,%d] illegal row-index %d", __LINE__, ri)); 626 627 phw = pmc_pcpu[cpu]->pc_hwpmcs[ri]; 628 629 KASSERT(phw->phw_pmc == NULL, 630 ("[amd,%d] PHW pmc %p non-NULL", __LINE__, phw->phw_pmc)); 631 632#if DEBUG 633 pd = &amd_pmcdesc[ri]; 634 if (pd->pm_descr.pd_class == AMD_PMC_CLASS) 635 KASSERT(AMD_PMC_IS_STOPPED(pd->pm_evsel), 636 ("[amd,%d] PMC %d released while active", __LINE__, ri)); 637#endif 638 639 return 0; 640} 641 642/* 643 * start a PMC. 644 */ 645 646static int 647amd_start_pmc(int cpu, int ri) 648{ 649 uint32_t config; 650 struct pmc *pm; 651 struct pmc_hw *phw; 652 const struct amd_descr *pd; 653 654 KASSERT(cpu >= 0 && cpu < mp_ncpus, 655 ("[amd,%d] illegal CPU value %d", __LINE__, cpu)); 656 KASSERT(ri >= 0 && ri < AMD_NPMCS, 657 ("[amd,%d] illegal row-index %d", __LINE__, ri)); 658 659 phw = pmc_pcpu[cpu]->pc_hwpmcs[ri]; 660 pm = phw->phw_pmc; 661 pd = &amd_pmcdesc[ri]; 662 663 KASSERT(pm != NULL, 664 ("[amd,%d] starting cpu%d,pmc%d with null pmc record", __LINE__, 665 cpu, ri)); 666 667 PMCDBG(MDP,STA,1,"amd-start cpu=%d ri=%d", cpu, ri); 668 669 if (pd->pm_descr.pd_class == PMC_CLASS_TSC) 670 return 0; /* TSCs are always running */ 671 672 KASSERT(pd->pm_descr.pd_class == AMD_PMC_CLASS, 673 ("[amd,%d] unknown PMC class (%d)", __LINE__, 674 pd->pm_descr.pd_class)); 675 676 KASSERT(AMD_PMC_IS_STOPPED(pd->pm_evsel), 677 ("[amd,%d] pmc%d,cpu%d: Starting active PMC \"%s\"", __LINE__, 678 ri, cpu, pd->pm_descr.pd_name)); 679 680 /* turn on the PMC ENABLE bit */ 681 config = pm->pm_md.pm_amd.pm_amd_evsel | AMD_PMC_ENABLE; 682 683 PMCDBG(MDP,STA,2,"amd-start config=0x%x", config); 684 685 wrmsr(pd->pm_evsel, config); 686 return 0; 687} 688 689/* 690 * Stop a PMC. 691 */ 692 693static int 694amd_stop_pmc(int cpu, int ri) 695{ 696 struct pmc *pm; 697 struct pmc_hw *phw; 698 const struct amd_descr *pd; 699 uint64_t config; 700 701 KASSERT(cpu >= 0 && cpu < mp_ncpus, 702 ("[amd,%d] illegal CPU value %d", __LINE__, cpu)); 703 KASSERT(ri >= 0 && ri < AMD_NPMCS, 704 ("[amd,%d] illegal row-index %d", __LINE__, ri)); 705 706 phw = pmc_pcpu[cpu]->pc_hwpmcs[ri]; 707 pm = phw->phw_pmc; 708 pd = &amd_pmcdesc[ri]; 709 710 KASSERT(pm != NULL, 711 ("[amd,%d] cpu%d,pmc%d no PMC to stop", __LINE__, 712 cpu, ri)); 713 714 /* can't stop a TSC */ 715 if (pd->pm_descr.pd_class == PMC_CLASS_TSC) 716 return 0; 717 718 KASSERT(pd->pm_descr.pd_class == AMD_PMC_CLASS, 719 ("[amd,%d] unknown PMC class (%d)", __LINE__, 720 pd->pm_descr.pd_class)); 721 722 KASSERT(!AMD_PMC_IS_STOPPED(pd->pm_evsel), 723 ("[amd,%d] PMC%d, CPU%d \"%s\" already stopped", 724 __LINE__, ri, cpu, pd->pm_descr.pd_name)); 725 726 PMCDBG(MDP,STO,1,"amd-stop ri=%d", ri); 727 728 /* turn off the PMC ENABLE bit */ 729 config = pm->pm_md.pm_amd.pm_amd_evsel & ~AMD_PMC_ENABLE; 730 wrmsr(pd->pm_evsel, config); 731 return 0; 732} 733 734/* 735 * Interrupt handler. This function needs to return '1' if the 736 * interrupt was this CPU's PMCs or '0' otherwise. It is not allowed 737 * to sleep or do anything a 'fast' interrupt handler is not allowed 738 * to do. 739 */ 740 741static int 742amd_intr(int cpu, uintptr_t eip) 743{ 744 int i, retval; 745 enum pmc_mode mode; 746 uint32_t perfctr; 747 struct pmc *pm; 748 struct pmc_cpu *pc; 749 struct pmc_hw *phw; 750 751 KASSERT(cpu >= 0 && cpu < mp_ncpus, 752 ("[amd,%d] out of range CPU %d", __LINE__, cpu)); 753 754 retval = 0; 755 756 pc = pmc_pcpu[cpu]; 757 758 /* 759 * look for all PMCs that have interrupted: 760 * - skip over the TSC [PMC#0] 761 * - look for a PMC with a valid 'struct pmc' association 762 * - look for a PMC in (a) sampling mode and (b) which has 763 * overflowed. If found, we update the process's 764 * histogram or send it a profiling signal by calling 765 * the appropriate helper function. 766 */ 767 768 for (i = 1; i < AMD_NPMCS; i++) { 769 770 phw = pc->pc_hwpmcs[i]; 771 perfctr = amd_pmcdesc[i].pm_perfctr; 772 KASSERT(phw != NULL, ("[amd,%d] null PHW pointer", __LINE__)); 773 774 if ((pm = phw->phw_pmc) == NULL || 775 pm->pm_state != PMC_STATE_RUNNING) { 776 atomic_add_int(&pmc_stats.pm_intr_ignored, 1); 777 continue; 778 } 779 780 mode = PMC_TO_MODE(pm); 781 if (PMC_IS_SAMPLING_MODE(mode) && 782 AMD_PMC_HAS_OVERFLOWED(perfctr)) { 783 atomic_add_int(&pmc_stats.pm_intr_processed, 1); 784 if (PMC_IS_SYSTEM_MODE(mode)) 785 pmc_update_histogram(phw, eip); 786 else if (PMC_IS_VIRTUAL_MODE(mode)) 787 pmc_send_signal(pm); 788 retval = 1; 789 } 790 } 791 return retval; 792} 793 794/* 795 * describe a PMC 796 */ 797static int 798amd_describe(int cpu, int ri, struct pmc_info *pi, struct pmc **ppmc) 799{ 800 int error; 801 size_t copied; 802 const struct amd_descr *pd; 803 struct pmc_hw *phw; 804 805 KASSERT(cpu >= 0 && cpu < mp_ncpus, 806 ("[amd,%d] illegal CPU %d", __LINE__, cpu)); 807 KASSERT(ri >= 0 && ri < AMD_NPMCS, 808 ("[amd,%d] row-index %d out of range", __LINE__, ri)); 809 810 phw = pmc_pcpu[cpu]->pc_hwpmcs[ri]; 811 pd = &amd_pmcdesc[ri]; 812 813 if ((error = copystr(pd->pm_descr.pd_name, pi->pm_name, 814 PMC_NAME_MAX, &copied)) != 0) 815 return error; 816 817 pi->pm_class = pd->pm_descr.pd_class; 818 819 if (phw->phw_state & PMC_PHW_FLAG_IS_ENABLED) { 820 pi->pm_enabled = TRUE; 821 *ppmc = phw->phw_pmc; 822 } else { 823 pi->pm_enabled = FALSE; 824 *ppmc = NULL; 825 } 826 827 return 0; 828} 829 830/* 831 * i386 specific entry points 832 */ 833 834/* 835 * return the MSR address of the given PMC. 836 */ 837 838static int 839amd_get_msr(int ri, uint32_t *msr) 840{ 841 KASSERT(ri >= 0 && ri < AMD_NPMCS, 842 ("[amd,%d] ri %d out of range", __LINE__, ri)); 843 844 *msr = amd_pmcdesc[ri].pm_perfctr - AMD_PMC_PERFCTR_0; 845 return 0; 846} 847 848/* 849 * processor dependent initialization. 850 */ 851 852/* 853 * Per-processor data structure 854 * 855 * [common stuff] 856 * [5 struct pmc_hw pointers] 857 * [5 struct pmc_hw structures] 858 */ 859 860struct amd_cpu { 861 struct pmc_cpu pc_common; 862 struct pmc_hw *pc_hwpmcs[AMD_NPMCS]; 863 struct pmc_hw pc_amdpmcs[AMD_NPMCS]; 864}; 865 866 867static int 868amd_init(int cpu) 869{ 870 int n; 871 struct amd_cpu *pcs; 872 struct pmc_hw *phw; 873 874 KASSERT(cpu >= 0 && cpu < mp_ncpus, 875 ("[amd,%d] insane cpu number %d", __LINE__, cpu)); 876 877 PMCDBG(MDP,INI,1,"amd-init cpu=%d", cpu); 878 879 MALLOC(pcs, struct amd_cpu *, sizeof(struct amd_cpu), M_PMC, 880 M_WAITOK|M_ZERO); 881 882 if (pcs == NULL) 883 return ENOMEM; 884 885 phw = &pcs->pc_amdpmcs[0]; 886 887 /* 888 * Initialize the per-cpu mutex and set the content of the 889 * hardware descriptors to a known state. 890 */ 891 892 for (n = 0; n < AMD_NPMCS; n++, phw++) { 893 phw->phw_state = PMC_PHW_FLAG_IS_ENABLED | 894 PMC_PHW_CPU_TO_STATE(cpu) | PMC_PHW_INDEX_TO_STATE(n); 895 phw->phw_pmc = NULL; 896 pcs->pc_hwpmcs[n] = phw; 897 } 898 899 /* Mark the TSC as shareable */ 900 pcs->pc_hwpmcs[0]->phw_state |= PMC_PHW_FLAG_IS_SHAREABLE; 901 902 pmc_pcpu[cpu] = (struct pmc_cpu *) pcs; 903 904 return 0; 905} 906 907 908/* 909 * processor dependent cleanup prior to the KLD 910 * being unloaded 911 */ 912 913static int 914amd_cleanup(int cpu) 915{ 916 int i; 917 uint32_t evsel; 918 struct pmc_cpu *pcs; 919 920 KASSERT(cpu >= 0 && cpu < mp_ncpus, 921 ("[amd,%d] insane cpu number (%d)", __LINE__, cpu)); 922 923 PMCDBG(MDP,INI,1,"amd-cleanup cpu=%d", cpu); 924 925 /* 926 * First, turn off all PMCs on this CPU. 927 */ 928 929 for (i = 0; i < 4; i++) { /* XXX this loop is now not needed */ 930 evsel = rdmsr(AMD_PMC_EVSEL_0 + i); 931 evsel &= ~AMD_PMC_ENABLE; 932 wrmsr(AMD_PMC_EVSEL_0 + i, evsel); 933 } 934 935 /* 936 * Next, free up allocated space. 937 */ 938 939 pcs = pmc_pcpu[cpu]; 940 941#if DEBUG 942 /* check the TSC */ 943 KASSERT(pcs->pc_hwpmcs[0]->phw_pmc == NULL, 944 ("[amd,%d] CPU%d,PMC0 still in use", __LINE__, cpu)); 945 for (i = 1; i < AMD_NPMCS; i++) { 946 KASSERT(pcs->pc_hwpmcs[i]->phw_pmc == NULL, 947 ("[amd,%d] CPU%d/PMC%d in use", __LINE__, cpu, i)); 948 KASSERT(AMD_PMC_IS_STOPPED(AMD_PMC_EVSEL_0 + (i-1)), 949 ("[amd,%d] CPU%d/PMC%d not stopped", __LINE__, cpu, i)); 950 } 951#endif 952 KASSERT(pcs != NULL, 953 ("[amd,%d] null per-cpu state pointer (cpu%d)", __LINE__, cpu)); 954 955 pmc_pcpu[cpu] = NULL; 956 FREE(pcs, M_PMC); 957 return 0; 958} 959 960/* 961 * Initialize ourselves. 962 */ 963 964struct pmc_mdep * 965pmc_amd_initialize(void) 966{ 967 968 struct pmc_mdep *pmc_mdep; 969 970 /* The presence of hardware performance counters on the AMD 971 Athlon, Duron or later processors, is _not_ indicated by 972 any of the processor feature flags set by the 'CPUID' 973 instruction, so we only check the 'instruction family' 974 field returned by CPUID for instruction family >= 6. This 975 test needs to be be refined. */ 976 977 if ((cpu_id & 0xF00) < 0x600) 978 return NULL; 979 980 MALLOC(pmc_mdep, struct pmc_mdep *, sizeof(struct pmc_mdep), 981 M_PMC, M_WAITOK|M_ZERO); 982 983#if __i386__ 984 pmc_mdep->pmd_cputype = PMC_CPU_AMD_K7; 985#elif __amd64__ 986 pmc_mdep->pmd_cputype = PMC_CPU_AMD_K8; 987#else 988#error Unknown AMD CPU type. 989#endif 990 991 pmc_mdep->pmd_npmc = AMD_NPMCS; 992 993 /* this processor has two classes of usable PMCs */ 994 pmc_mdep->pmd_nclass = 2; 995 996 /* TSC */ 997 pmc_mdep->pmd_classes[0].pm_class = PMC_CLASS_TSC; 998 pmc_mdep->pmd_classes[0].pm_caps = PMC_CAP_READ; 999 pmc_mdep->pmd_classes[0].pm_width = 64; 1000 1001 /* AMD K7/K8 PMCs */ 1002 pmc_mdep->pmd_classes[1].pm_class = AMD_PMC_CLASS; 1003 pmc_mdep->pmd_classes[1].pm_caps = AMD_PMC_CAPS; 1004 pmc_mdep->pmd_classes[1].pm_width = 48; 1005 1006 pmc_mdep->pmd_nclasspmcs[0] = 1; 1007 pmc_mdep->pmd_nclasspmcs[1] = (AMD_NPMCS-1); 1008 1009 pmc_mdep->pmd_init = amd_init; 1010 pmc_mdep->pmd_cleanup = amd_cleanup; 1011 pmc_mdep->pmd_switch_in = amd_switch_in; 1012 pmc_mdep->pmd_switch_out = amd_switch_out; 1013 pmc_mdep->pmd_read_pmc = amd_read_pmc; 1014 pmc_mdep->pmd_write_pmc = amd_write_pmc; 1015 pmc_mdep->pmd_config_pmc = amd_config_pmc; 1016 pmc_mdep->pmd_get_config = amd_get_config; 1017 pmc_mdep->pmd_allocate_pmc = amd_allocate_pmc; 1018 pmc_mdep->pmd_release_pmc = amd_release_pmc; 1019 pmc_mdep->pmd_start_pmc = amd_start_pmc; 1020 pmc_mdep->pmd_stop_pmc = amd_stop_pmc; 1021 pmc_mdep->pmd_intr = amd_intr; 1022 pmc_mdep->pmd_describe = amd_describe; 1023 pmc_mdep->pmd_get_msr = amd_get_msr; /* i386 */ 1024 1025 PMCDBG(MDP,INI,0,"%s","amd-initialize"); 1026 1027 return pmc_mdep; 1028} 1029