hwpmc_amd.c revision 145615
1/*- 2 * Copyright (c) 2003-2005 Joseph Koshy 3 * All rights reserved. 4 * 5 * Redistribution and use in source and binary forms, with or without 6 * modification, are permitted provided that the following conditions 7 * are met: 8 * 1. Redistributions of source code must retain the above copyright 9 * notice, this list of conditions and the following disclaimer. 10 * 2. Redistributions in binary form must reproduce the above copyright 11 * notice, this list of conditions and the following disclaimer in the 12 * documentation and/or other materials provided with the distribution. 13 * 14 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 15 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 16 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 17 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 18 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 19 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 20 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 21 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 22 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 23 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 24 * SUCH DAMAGE. 25 * 26 */ 27 28#include <sys/cdefs.h> 29__FBSDID("$FreeBSD: head/sys/dev/hwpmc/hwpmc_amd.c 145615 2005-04-28 08:13:19Z jkoshy $"); 30 31/* Support for the AMD K7 and later processors */ 32 33#include <sys/param.h> 34#include <sys/lock.h> 35#include <sys/malloc.h> 36#include <sys/mutex.h> 37#include <sys/pmc.h> 38#include <sys/smp.h> 39#include <sys/systm.h> 40 41#include <machine/md_var.h> 42 43/* AMD K7 and K8 PMCs */ 44 45#define AMD_PMC_EVSEL_0 0xC0010000 46#define AMD_PMC_EVSEL_1 0xC0010001 47#define AMD_PMC_EVSEL_2 0xC0010002 48#define AMD_PMC_EVSEL_3 0xC0010003 49 50#define AMD_PMC_PERFCTR_0 0xC0010004 51#define AMD_PMC_PERFCTR_1 0xC0010005 52#define AMD_PMC_PERFCTR_2 0xC0010006 53#define AMD_PMC_PERFCTR_3 0xC0010007 54 55#define K7_VALID_EVENT_CODE(c) (((c) >= 0x40 && (c) <= 0x47) || \ 56 ((c) >= 0x80 && (c) <= 0x85) || ((c) >= 0xC0 && (c) <= 0xC7) || \ 57 ((c) >= 0xCD && (c) <= 0xCF)) 58 59#define AMD_PMC_CAPS (PMC_CAP_INTERRUPT | PMC_CAP_USER | \ 60 PMC_CAP_SYSTEM | PMC_CAP_EDGE | PMC_CAP_THRESHOLD | \ 61 PMC_CAP_READ | PMC_CAP_WRITE | PMC_CAP_INVERT | PMC_CAP_QUALIFIER) 62 63/* reserved bits include bit 21 and the top two bits of the unit mask */ 64#define K7_PMC_RESERVED ((1 << 21) | (3 << 13)) 65 66#define K8_PMC_RESERVED (1 << 21) 67 68#define AMD_PMC_IS_STOPPED(evsel) ((rdmsr((evsel)) & AMD_PMC_ENABLE) == 0) 69#define AMD_PMC_HAS_OVERFLOWED(pmc) ((rdpmc(pmc) & (1ULL << 47)) == 0) 70 71#if __i386__ 72#define AMD_NPMCS K7_NPMCS 73#define AMD_PMC_CLASS PMC_CLASS_K7 74#define AMD_PMC_COUNTERMASK K7_PMC_COUNTERMASK 75#define AMD_PMC_TO_COUNTER(x) K7_PMC_TO_COUNTER(x) 76#define AMD_PMC_INVERT K7_PMC_INVERT 77#define AMD_PMC_ENABLE K7_PMC_ENABLE 78#define AMD_PMC_INT K7_PMC_INT 79#define AMD_PMC_PC K7_PMC_PC 80#define AMD_PMC_EDGE K7_PMC_EDGE 81#define AMD_PMC_OS K7_PMC_OS 82#define AMD_PMC_USR K7_PMC_USR 83 84#define AMD_PMC_UNITMASK_M K7_PMC_UNITMASK_M 85#define AMD_PMC_UNITMASK_O K7_PMC_UNITMASK_O 86#define AMD_PMC_UNITMASK_E K7_PMC_UNITMASK_E 87#define AMD_PMC_UNITMASK_S K7_PMC_UNITMASK_S 88#define AMD_PMC_UNITMASK_I K7_PMC_UNITMASK_I 89 90#define AMD_PMC_UNITMASK K7_PMC_UNITMASK 91#define AMD_PMC_EVENTMASK K7_PMC_EVENTMASK 92#define AMD_PMC_TO_UNITMASK(x) K7_PMC_TO_UNITMASK(x) 93#define AMD_PMC_TO_EVENTMASK(x) K7_PMC_TO_EVENTMASK(x) 94#define AMD_VALID_BITS K7_VALID_BITS 95 96#define AMD_PMC_CLASS_NAME "K7-" 97 98#elif __amd64__ 99 100#define AMD_NPMCS K8_NPMCS 101#define AMD_PMC_CLASS PMC_CLASS_K8 102#define AMD_PMC_COUNTERMASK K8_PMC_COUNTERMASK 103#define AMD_PMC_TO_COUNTER(x) K8_PMC_TO_COUNTER(x) 104#define AMD_PMC_INVERT K8_PMC_INVERT 105#define AMD_PMC_ENABLE K8_PMC_ENABLE 106#define AMD_PMC_INT K8_PMC_INT 107#define AMD_PMC_PC K8_PMC_PC 108#define AMD_PMC_EDGE K8_PMC_EDGE 109#define AMD_PMC_OS K8_PMC_OS 110#define AMD_PMC_USR K8_PMC_USR 111 112#define AMD_PMC_UNITMASK_M K8_PMC_UNITMASK_M 113#define AMD_PMC_UNITMASK_O K8_PMC_UNITMASK_O 114#define AMD_PMC_UNITMASK_E K8_PMC_UNITMASK_E 115#define AMD_PMC_UNITMASK_S K8_PMC_UNITMASK_S 116#define AMD_PMC_UNITMASK_I K8_PMC_UNITMASK_I 117 118#define AMD_PMC_UNITMASK K8_PMC_UNITMASK 119#define AMD_PMC_EVENTMASK K8_PMC_EVENTMASK 120#define AMD_PMC_TO_UNITMASK(x) K8_PMC_TO_UNITMASK(x) 121#define AMD_PMC_TO_EVENTMASK(x) K8_PMC_TO_EVENTMASK(x) 122#define AMD_VALID_BITS K8_VALID_BITS 123 124#define AMD_PMC_CLASS_NAME "K8-" 125 126#else 127#error Unsupported architecture. 128#endif 129 130/* AMD K7 & K8 PMCs */ 131struct amd_descr { 132 struct pmc_descr pm_descr; /* "base class" */ 133 uint32_t pm_evsel; /* address of EVSEL register */ 134 uint32_t pm_perfctr; /* address of PERFCTR register */ 135}; 136 137static const struct amd_descr amd_pmcdesc[AMD_NPMCS] = 138{ 139 { 140 .pm_descr = 141 { 142 .pd_name = "TSC", 143 .pd_class = PMC_CLASS_TSC, 144 .pd_caps = PMC_CAP_READ, 145 .pd_width = 64 146 }, 147 .pm_evsel = MSR_TSC, 148 .pm_perfctr = 0 /* unused */ 149 }, 150 151 { 152 .pm_descr = 153 { 154 .pd_name = AMD_PMC_CLASS_NAME "0", 155 .pd_class = AMD_PMC_CLASS, 156 .pd_caps = AMD_PMC_CAPS, 157 .pd_width = 48 158 }, 159 .pm_evsel = AMD_PMC_EVSEL_0, 160 .pm_perfctr = AMD_PMC_PERFCTR_0 161 }, 162 { 163 .pm_descr = 164 { 165 .pd_name = AMD_PMC_CLASS_NAME "1", 166 .pd_class = AMD_PMC_CLASS, 167 .pd_caps = AMD_PMC_CAPS, 168 .pd_width = 48 169 }, 170 .pm_evsel = AMD_PMC_EVSEL_1, 171 .pm_perfctr = AMD_PMC_PERFCTR_1 172 }, 173 { 174 .pm_descr = 175 { 176 .pd_name = AMD_PMC_CLASS_NAME "2", 177 .pd_class = AMD_PMC_CLASS, 178 .pd_caps = AMD_PMC_CAPS, 179 .pd_width = 48 180 }, 181 .pm_evsel = AMD_PMC_EVSEL_2, 182 .pm_perfctr = AMD_PMC_PERFCTR_2 183 }, 184 { 185 .pm_descr = 186 { 187 .pd_name = AMD_PMC_CLASS_NAME "3", 188 .pd_class = AMD_PMC_CLASS, 189 .pd_caps = AMD_PMC_CAPS, 190 .pd_width = 48 191 }, 192 .pm_evsel = AMD_PMC_EVSEL_3, 193 .pm_perfctr = AMD_PMC_PERFCTR_3 194 } 195}; 196 197struct amd_event_code_map { 198 enum pmc_event pe_ev; /* enum value */ 199 uint8_t pe_code; /* encoded event mask */ 200 uint8_t pe_mask; /* bits allowed in unit mask */ 201}; 202 203const struct amd_event_code_map amd_event_codes[] = { 204#if __i386__ 205 { PMC_EV_K7_DC_ACCESSES, 0x40, 0 }, 206 { PMC_EV_K7_DC_MISSES, 0x41, 0 }, 207 { PMC_EV_K7_DC_REFILLS_FROM_L2, 0x42, K7_PMC_UNITMASK_MOESI }, 208 { PMC_EV_K7_DC_REFILLS_FROM_SYSTEM, 0x43, K7_PMC_UNITMASK_MOESI }, 209 { PMC_EV_K7_DC_WRITEBACKS, 0x44, K7_PMC_UNITMASK_MOESI }, 210 { PMC_EV_K7_L1_DTLB_MISS_AND_L2_DTLB_HITS, 0x45, 0 }, 211 { PMC_EV_K7_L1_AND_L2_DTLB_MISSES, 0x46, 0 }, 212 { PMC_EV_K7_MISALIGNED_REFERENCES, 0x47, 0 }, 213 214 { PMC_EV_K7_IC_FETCHES, 0x80, 0 }, 215 { PMC_EV_K7_IC_MISSES, 0x81, 0 }, 216 217 { PMC_EV_K7_L1_ITLB_MISSES, 0x84, 0 }, 218 { PMC_EV_K7_L1_L2_ITLB_MISSES, 0x85, 0 }, 219 220 { PMC_EV_K7_RETIRED_INSTRUCTIONS, 0xC0, 0 }, 221 { PMC_EV_K7_RETIRED_OPS, 0xC1, 0 }, 222 { PMC_EV_K7_RETIRED_BRANCHES, 0xC2, 0 }, 223 { PMC_EV_K7_RETIRED_BRANCHES_MISPREDICTED, 0xC3, 0 }, 224 { PMC_EV_K7_RETIRED_TAKEN_BRANCHES, 0xC4, 0 }, 225 { PMC_EV_K7_RETIRED_TAKEN_BRANCHES_MISPREDICTED, 0xC5, 0 }, 226 { PMC_EV_K7_RETIRED_FAR_CONTROL_TRANSFERS, 0xC6, 0 }, 227 { PMC_EV_K7_RETIRED_RESYNC_BRANCHES, 0xC7, 0 }, 228 { PMC_EV_K7_INTERRUPTS_MASKED_CYCLES, 0xCD, 0 }, 229 { PMC_EV_K7_INTERRUPTS_MASKED_WHILE_PENDING_CYCLES, 0xCE, 0 }, 230 { PMC_EV_K7_HARDWARE_INTERRUPTS, 0xCF, 0 } 231#endif 232 233#if __amd64__ 234 { PMC_EV_K8_FP_DISPATCHED_FPU_OPS, 0x00, 0x3F }, 235 { PMC_EV_K8_FP_CYCLES_WITH_NO_FPU_OPS_RETIRED, 0x01, 0x00 }, 236 { PMC_EV_K8_FP_DISPATCHED_FPU_FAST_FLAG_OPS, 0x02, 0x00 }, 237 238 { PMC_EV_K8_LS_SEGMENT_REGISTER_LOAD, 0x20, 0x7F }, 239 { PMC_EV_K8_LS_MICROARCHITECTURAL_RESYNC_BY_SELF_MODIFYING_CODE, 240 0x21, 0x00 }, 241 { PMC_EV_K8_LS_MICROARCHITECTURAL_RESYNC_BY_SNOOP, 0x22, 0x00 }, 242 { PMC_EV_K8_LS_BUFFER2_FULL, 0x23, 0x00 }, 243 { PMC_EV_K8_LS_LOCKED_OPERATION, 0x24, 0x07 }, 244 { PMC_EV_K8_LS_MICROARCHITECTURAL_LATE_CANCEL, 0x25, 0x00 }, 245 { PMC_EV_K8_LS_RETIRED_CFLUSH_INSTRUCTIONS, 0x26, 0x00 }, 246 { PMC_EV_K8_LS_RETIRED_CPUID_INSTRUCTIONS, 0x27, 0x00 }, 247 248 { PMC_EV_K8_DC_ACCESS, 0x40, 0x00 }, 249 { PMC_EV_K8_DC_MISS, 0x41, 0x00 }, 250 { PMC_EV_K8_DC_REFILL_FROM_L2, 0x42, 0x1F }, 251 { PMC_EV_K8_DC_REFILL_FROM_SYSTEM, 0x43, 0x1F }, 252 { PMC_EV_K8_DC_COPYBACK, 0x44, 0x1F }, 253 { PMC_EV_K8_DC_L1_DTLB_MISS_AND_L2_DTLB_HIT, 0x45, 0x00 }, 254 { PMC_EV_K8_DC_L1_DTLB_MISS_AND_L2_DTLB_MISS, 0x46, 0x00 }, 255 { PMC_EV_K8_DC_MISALIGNED_DATA_REFERENCE, 0x47, 0x00 }, 256 { PMC_EV_K8_DC_MICROARCHITECTURAL_LATE_CANCEL, 0x48, 0x00 }, 257 { PMC_EV_K8_DC_MICROARCHITECTURAL_EARLY_CANCEL, 0x49, 0x00 }, 258 { PMC_EV_K8_DC_ONE_BIT_ECC_ERROR, 0x4A, 0x03 }, 259 { PMC_EV_K8_DC_DISPATCHED_PREFETCH_INSTRUCTIONS, 0x4B, 0x07 }, 260 { PMC_EV_K8_DC_DCACHE_ACCESSES_BY_LOCKS, 0x4C, 0x03 }, 261 262 { PMC_EV_K8_BU_CPU_CLK_UNHALTED, 0x76, 0x00 }, 263 { PMC_EV_K8_BU_INTERNAL_L2_REQUEST, 0x7D, 0x1F }, 264 { PMC_EV_K8_BU_FILL_REQUEST_L2_MISS, 0x7E, 0x07 }, 265 { PMC_EV_K8_BU_FILL_INTO_L2, 0x7F, 0x03 }, 266 267 { PMC_EV_K8_IC_FETCH, 0x80, 0x00 }, 268 { PMC_EV_K8_IC_MISS, 0x81, 0x00 }, 269 { PMC_EV_K8_IC_REFILL_FROM_L2, 0x82, 0x00 }, 270 { PMC_EV_K8_IC_REFILL_FROM_SYSTEM, 0x83, 0x00 }, 271 { PMC_EV_K8_IC_L1_ITLB_MISS_AND_L2_ITLB_HIT, 0x84, 0x00 }, 272 { PMC_EV_K8_IC_L1_ITLB_MISS_AND_L2_ITLB_MISS, 0x85, 0x00 }, 273 { PMC_EV_K8_IC_MICROARCHITECTURAL_RESYNC_BY_SNOOP, 0x86, 0x00 }, 274 { PMC_EV_K8_IC_INSTRUCTION_FETCH_STALL, 0x87, 0x00 }, 275 { PMC_EV_K8_IC_RETURN_STACK_HIT, 0x88, 0x00 }, 276 { PMC_EV_K8_IC_RETURN_STACK_OVERFLOW, 0x89, 0x00 }, 277 278 { PMC_EV_K8_FR_RETIRED_X86_INSTRUCTIONS, 0xC0, 0x00 }, 279 { PMC_EV_K8_FR_RETIRED_UOPS, 0xC1, 0x00 }, 280 { PMC_EV_K8_FR_RETIRED_BRANCHES, 0xC2, 0x00 }, 281 { PMC_EV_K8_FR_RETIRED_BRANCHES_MISPREDICTED, 0xC3, 0x00 }, 282 { PMC_EV_K8_FR_RETIRED_TAKEN_BRANCHES, 0xC4, 0x00 }, 283 { PMC_EV_K8_FR_RETIRED_TAKEN_BRANCHES_MISPREDICTED, 0xC5, 0x00 }, 284 { PMC_EV_K8_FR_RETIRED_FAR_CONTROL_TRANSFERS, 0xC6, 0x00 }, 285 { PMC_EV_K8_FR_RETIRED_RESYNCS, 0xC7, 0x00 }, 286 { PMC_EV_K8_FR_RETIRED_NEAR_RETURNS, 0xC8, 0x00 }, 287 { PMC_EV_K8_FR_RETIRED_NEAR_RETURNS_MISPREDICTED, 0xC9, 0x00 }, 288 { PMC_EV_K8_FR_RETIRED_TAKEN_BRANCHES_MISPREDICTED_BY_ADDR_MISCOMPARE, 289 0xCA, 0x00 }, 290 { PMC_EV_K8_FR_RETIRED_FPU_INSTRUCTIONS, 0xCB, 0x0F }, 291 { PMC_EV_K8_FR_RETIRED_FASTPATH_DOUBLE_OP_INSTRUCTIONS, 292 0xCC, 0x07 }, 293 { PMC_EV_K8_FR_INTERRUPTS_MASKED_CYCLES, 0xCD, 0x00 }, 294 { PMC_EV_K8_FR_INTERRUPTS_MASKED_WHILE_PENDING_CYCLES, 0xCE, 0x00 }, 295 { PMC_EV_K8_FR_TAKEN_HARDWARE_INTERRUPTS, 0xCF, 0x00 }, 296 297 { PMC_EV_K8_FR_DECODER_EMPTY, 0xD0, 0x00 }, 298 { PMC_EV_K8_FR_DISPATCH_STALLS, 0xD1, 0x00 }, 299 { PMC_EV_K8_FR_DISPATCH_STALL_FROM_BRANCH_ABORT_TO_RETIRE, 300 0xD2, 0x00 }, 301 { PMC_EV_K8_FR_DISPATCH_STALL_FOR_SERIALIZATION, 0xD3, 0x00 }, 302 { PMC_EV_K8_FR_DISPATCH_STALL_FOR_SEGMENT_LOAD, 0xD4, 0x00 }, 303 { PMC_EV_K8_FR_DISPATCH_STALL_WHEN_REORDER_BUFFER_IS_FULL, 304 0xD5, 0x00 }, 305 { PMC_EV_K8_FR_DISPATCH_STALL_WHEN_RESERVATION_STATIONS_ARE_FULL, 306 0xD6, 0x00 }, 307 { PMC_EV_K8_FR_DISPATCH_STALL_WHEN_FPU_IS_FULL, 0xD7, 0x00 }, 308 { PMC_EV_K8_FR_DISPATCH_STALL_WHEN_LS_IS_FULL, 0xD8, 0x00 }, 309 { PMC_EV_K8_FR_DISPATCH_STALL_WHEN_WAITING_FOR_ALL_TO_BE_QUIET, 310 0xD9, 0x00 }, 311 { PMC_EV_K8_FR_DISPATCH_STALL_WHEN_FAR_XFER_OR_RESYNC_BRANCH_PENDING, 312 0xDA, 0x00 }, 313 { PMC_EV_K8_FR_FPU_EXCEPTIONS, 0xDB, 0x0F }, 314 { PMC_EV_K8_FR_NUMBER_OF_BREAKPOINTS_FOR_DR0, 0xDC, 0x00 }, 315 { PMC_EV_K8_FR_NUMBER_OF_BREAKPOINTS_FOR_DR1, 0xDD, 0x00 }, 316 { PMC_EV_K8_FR_NUMBER_OF_BREAKPOINTS_FOR_DR2, 0xDE, 0x00 }, 317 { PMC_EV_K8_FR_NUMBER_OF_BREAKPOINTS_FOR_DR3, 0xDF, 0x00 }, 318 319 { PMC_EV_K8_NB_MEMORY_CONTROLLER_PAGE_ACCESS_EVENT, 0xE0, 0x7 }, 320 { PMC_EV_K8_NB_MEMORY_CONTROLLER_PAGE_TABLE_OVERFLOW, 0xE1, 0x00 }, 321 { PMC_EV_K8_NB_MEMORY_CONTROLLER_DRAM_COMMAND_SLOTS_MISSED, 322 0xE2, 0x00 }, 323 { PMC_EV_K8_NB_MEMORY_CONTROLLER_TURNAROUND, 0xE3, 0x07 }, 324 { PMC_EV_K8_NB_MEMORY_CONTROLLER_BYPASS_SATURATION, 0xE4, 0x0F }, 325 { PMC_EV_K8_NB_SIZED_COMMANDS, 0xEB, 0x7F }, 326 { PMC_EV_K8_NB_PROBE_RESULT, 0xEC, 0x0F }, 327 { PMC_EV_K8_NB_HT_BUS0_BANDWIDTH, 0xF6, 0x0F }, 328 { PMC_EV_K8_NB_HT_BUS1_BANDWIDTH, 0xF7, 0x0F }, 329 { PMC_EV_K8_NB_HT_BUS2_BANDWIDTH, 0xF8, 0x0F } 330#endif 331 332}; 333 334const int amd_event_codes_size = 335 sizeof(amd_event_codes) / sizeof(amd_event_codes[0]); 336 337/* 338 * read a pmc register 339 */ 340 341static int 342amd_read_pmc(int cpu, int ri, pmc_value_t *v) 343{ 344 enum pmc_mode mode; 345 const struct amd_descr *pd; 346 struct pmc *pm; 347 const struct pmc_hw *phw; 348 pmc_value_t tmp; 349 350 KASSERT(cpu >= 0 && cpu < mp_ncpus, 351 ("[amd,%d] illegal CPU value %d", __LINE__, cpu)); 352 KASSERT(ri >= 0 && ri < AMD_NPMCS, 353 ("[amd,%d] illegal row-index %d", __LINE__, ri)); 354 355 phw = pmc_pcpu[cpu]->pc_hwpmcs[ri]; 356 pd = &amd_pmcdesc[ri]; 357 pm = phw->phw_pmc; 358 359 KASSERT(pm != NULL, 360 ("[amd,%d] No owner for HWPMC [cpu%d,pmc%d]", __LINE__, 361 cpu, ri)); 362 363 mode = pm->pm_mode; 364 365 PMCDBG(MDP,REA,1,"amd-read id=%d class=%d", ri, pd->pm_descr.pd_class); 366 367 /* Reading the TSC is a special case */ 368 if (pd->pm_descr.pd_class == PMC_CLASS_TSC) { 369 KASSERT(PMC_IS_COUNTING_MODE(mode), 370 ("[amd,%d] TSC counter in non-counting mode", __LINE__)); 371 *v = rdtsc(); 372 PMCDBG(MDP,REA,2,"amd-read id=%d -> %jd", ri, *v); 373 return 0; 374 } 375 376 KASSERT(pd->pm_descr.pd_class == AMD_PMC_CLASS, 377 ("[amd,%d] unknown PMC class (%d)", __LINE__, 378 pd->pm_descr.pd_class)); 379 380 tmp = rdmsr(pd->pm_perfctr); /* RDMSR serializes */ 381 if (PMC_IS_SAMPLING_MODE(mode)) 382 *v = -tmp; 383 else 384 *v = tmp; 385 386 PMCDBG(MDP,REA,2,"amd-read id=%d -> %jd", ri, *v); 387 388 return 0; 389} 390 391/* 392 * Write a PMC MSR. 393 */ 394 395static int 396amd_write_pmc(int cpu, int ri, pmc_value_t v) 397{ 398 const struct amd_descr *pd; 399 struct pmc *pm; 400 const struct pmc_hw *phw; 401 enum pmc_mode mode; 402 403 KASSERT(cpu >= 0 && cpu < mp_ncpus, 404 ("[amd,%d] illegal CPU value %d", __LINE__, cpu)); 405 KASSERT(ri >= 0 && ri < AMD_NPMCS, 406 ("[amd,%d] illegal row-index %d", __LINE__, ri)); 407 408 phw = pmc_pcpu[cpu]->pc_hwpmcs[ri]; 409 pd = &amd_pmcdesc[ri]; 410 pm = phw->phw_pmc; 411 412 KASSERT(pm != NULL, 413 ("[amd,%d] PMC not owned (cpu%d,pmc%d)", __LINE__, 414 cpu, ri)); 415 416 mode = pm->pm_mode; 417 418 if (pd->pm_descr.pd_class == PMC_CLASS_TSC) 419 return 0; 420 421 KASSERT(pd->pm_descr.pd_class == AMD_PMC_CLASS, 422 ("[amd,%d] unknown PMC class (%d)", __LINE__, 423 pd->pm_descr.pd_class)); 424 425 /* use 2's complement of the count for sampling mode PMCs */ 426 if (PMC_IS_SAMPLING_MODE(mode)) 427 v = -v; 428 429 PMCDBG(MDP,WRI,1,"amd-write cpu=%d ri=%d v=%jx", cpu, ri, v); 430 431 /* write the PMC value */ 432 wrmsr(pd->pm_perfctr, v); 433 return 0; 434} 435 436/* 437 * configure hardware pmc according to the configuration recorded in 438 * pmc 'pm'. 439 */ 440 441static int 442amd_config_pmc(int cpu, int ri, struct pmc *pm) 443{ 444 struct pmc_hw *phw; 445 446 PMCDBG(MDP,CFG,1, "cpu=%d ri=%d pm=%p", cpu, ri, pm); 447 448 KASSERT(cpu >= 0 && cpu < mp_ncpus, 449 ("[amd,%d] illegal CPU value %d", __LINE__, cpu)); 450 KASSERT(ri >= 0 && ri < AMD_NPMCS, 451 ("[amd,%d] illegal row-index %d", __LINE__, ri)); 452 453 phw = pmc_pcpu[cpu]->pc_hwpmcs[ri]; 454 455 KASSERT(pm == NULL || phw->phw_pmc == NULL, 456 ("[amd,%d] pm=%p phw->pm=%p hwpmc not unconfigured", 457 __LINE__, pm, phw->phw_pmc)); 458 459 phw->phw_pmc = pm; 460 return 0; 461} 462 463/* 464 * Machine dependent actions taken during the context switch in of a 465 * thread. 466 */ 467 468static int 469amd_switch_in(struct pmc_cpu *pc, struct pmc_process *pp) 470{ 471 (void) pc; 472 473 PMCDBG(MDP,SWI,1, "pc=%p pp=%p enable-msr=%d", pc, pp, 474 (pp->pp_flags & PMC_FLAG_ENABLE_MSR_ACCESS) != 0); 475 476 /* enable the RDPMC instruction if needed */ 477 if (pp->pp_flags & PMC_FLAG_ENABLE_MSR_ACCESS) 478 load_cr4(rcr4() | CR4_PCE); 479 480 return 0; 481} 482 483/* 484 * Machine dependent actions taken during the context switch out of a 485 * thread. 486 */ 487 488static int 489amd_switch_out(struct pmc_cpu *pc, struct pmc_process *pp) 490{ 491 (void) pc; 492 (void) pp; /* can be NULL */ 493 494 PMCDBG(MDP,SWO,1, "pc=%p pp=%p enable-msr=%d", pc, pp, pp ? 495 (pp->pp_flags & PMC_FLAG_ENABLE_MSR_ACCESS) == 1 : 0); 496 497 /* always turn off the RDPMC instruction */ 498 load_cr4(rcr4() & ~CR4_PCE); 499 500 return 0; 501} 502 503/* 504 * Check if a given allocation is feasible. 505 */ 506 507static int 508amd_allocate_pmc(int cpu, int ri, struct pmc *pm, 509 const struct pmc_op_pmcallocate *a) 510{ 511 int i; 512 uint32_t allowed_unitmask, caps, config, unitmask; 513 enum pmc_event pe; 514 const struct pmc_descr *pd; 515 516 (void) cpu; 517 518 KASSERT(cpu >= 0 && cpu < mp_ncpus, 519 ("[amd,%d] illegal CPU value %d", __LINE__, cpu)); 520 KASSERT(ri >= 0 && ri < AMD_NPMCS, 521 ("[amd,%d] illegal row index %d", __LINE__, ri)); 522 523 pd = &amd_pmcdesc[ri].pm_descr; 524 525 /* check class match */ 526 if (pd->pd_class != pm->pm_class) 527 return EINVAL; 528 529 caps = pm->pm_caps; 530 531 PMCDBG(MDP,ALL,1,"amd-allocate ri=%d caps=0x%x", ri, caps); 532 533 if ((pd->pd_caps & caps) != caps) 534 return EPERM; 535 if (pd->pd_class == PMC_CLASS_TSC) { 536 /* TSC's are always allocated in system-wide counting mode */ 537 if (a->pm_ev != PMC_EV_TSC_TSC || 538 a->pm_mode != PMC_MODE_SC) 539 return EINVAL; 540 return 0; 541 } 542 543 KASSERT(pd->pd_class == AMD_PMC_CLASS, 544 ("[amd,%d] Unknown PMC class (%d)", __LINE__, pd->pd_class)); 545 546 pe = a->pm_ev; 547 548 /* map ev to the correct event mask code */ 549 config = allowed_unitmask = 0; 550 for (i = 0; i < amd_event_codes_size; i++) 551 if (amd_event_codes[i].pe_ev == pe) { 552 config = 553 AMD_PMC_TO_EVENTMASK(amd_event_codes[i].pe_code); 554 allowed_unitmask = 555 AMD_PMC_TO_UNITMASK(amd_event_codes[i].pe_mask); 556 break; 557 } 558 if (i == amd_event_codes_size) 559 return EINVAL; 560 561 unitmask = a->pm_amd_config & AMD_PMC_UNITMASK; 562 if (unitmask & ~allowed_unitmask) /* disallow reserved bits */ 563 return EINVAL; 564 565 if (unitmask && (caps & PMC_CAP_QUALIFIER)) 566 config |= unitmask; 567 568 if (caps & PMC_CAP_THRESHOLD) 569 config |= a->pm_amd_config & AMD_PMC_COUNTERMASK; 570 571 /* set at least one of the 'usr' or 'os' caps */ 572 if (caps & PMC_CAP_USER) 573 config |= AMD_PMC_USR; 574 if (caps & PMC_CAP_SYSTEM) 575 config |= AMD_PMC_OS; 576 if ((caps & (PMC_CAP_USER|PMC_CAP_SYSTEM)) == 0) 577 config |= (AMD_PMC_USR|AMD_PMC_OS); 578 579 if (caps & PMC_CAP_EDGE) 580 config |= AMD_PMC_EDGE; 581 if (caps & PMC_CAP_INVERT) 582 config |= AMD_PMC_INVERT; 583 if (caps & PMC_CAP_INTERRUPT) 584 config |= AMD_PMC_INT; 585 586 pm->pm_md.pm_amd.pm_amd_evsel = config; /* save config value */ 587 588 PMCDBG(MDP,ALL,2,"amd-allocate ri=%d -> config=0x%x", ri, config); 589 590 return 0; 591} 592 593/* 594 * Release machine dependent state associated with a PMC. This is a 595 * no-op on this architecture. 596 * 597 */ 598 599/* ARGSUSED0 */ 600static int 601amd_release_pmc(int cpu, int ri, struct pmc *pmc) 602{ 603#if DEBUG 604 const struct amd_descr *pd; 605#endif 606 struct pmc_hw *phw; 607 608 (void) pmc; 609 610 KASSERT(cpu >= 0 && cpu < mp_ncpus, 611 ("[amd,%d] illegal CPU value %d", __LINE__, cpu)); 612 KASSERT(ri >= 0 && ri < AMD_NPMCS, 613 ("[amd,%d] illegal row-index %d", __LINE__, ri)); 614 615 phw = pmc_pcpu[cpu]->pc_hwpmcs[ri]; 616 617 KASSERT(phw->phw_pmc == NULL, 618 ("[amd,%d] PHW pmc %p non-NULL", __LINE__, phw->phw_pmc)); 619 620#if DEBUG 621 pd = &amd_pmcdesc[ri]; 622 if (pd->pm_descr.pd_class == AMD_PMC_CLASS) 623 KASSERT(AMD_PMC_IS_STOPPED(pd->pm_evsel), 624 ("[amd,%d] PMC %d released while active", __LINE__, ri)); 625#endif 626 627 return 0; 628} 629 630/* 631 * start a PMC. 632 */ 633 634static int 635amd_start_pmc(int cpu, int ri) 636{ 637 uint32_t config; 638 struct pmc *pm; 639 struct pmc_hw *phw; 640 const struct amd_descr *pd; 641 642 KASSERT(cpu >= 0 && cpu < mp_ncpus, 643 ("[amd,%d] illegal CPU value %d", __LINE__, cpu)); 644 KASSERT(ri >= 0 && ri < AMD_NPMCS, 645 ("[amd,%d] illegal row-index %d", __LINE__, ri)); 646 647 phw = pmc_pcpu[cpu]->pc_hwpmcs[ri]; 648 pm = phw->phw_pmc; 649 pd = &amd_pmcdesc[ri]; 650 651 KASSERT(pm != NULL, 652 ("[amd,%d] starting cpu%d,pmc%d with null pmc record", __LINE__, 653 cpu, ri)); 654 655 PMCDBG(MDP,STA,1,"amd-start cpu=%d ri=%d", cpu, ri); 656 657 if (pd->pm_descr.pd_class == PMC_CLASS_TSC) 658 return 0; /* TSCs are always running */ 659 660 KASSERT(pd->pm_descr.pd_class == AMD_PMC_CLASS, 661 ("[amd,%d] unknown PMC class (%d)", __LINE__, 662 pd->pm_descr.pd_class)); 663 664 KASSERT(AMD_PMC_IS_STOPPED(pd->pm_evsel), 665 ("[amd,%d] pmc%d,cpu%d: Starting active PMC \"%s\"", __LINE__, 666 ri, cpu, pd->pm_descr.pd_name)); 667 668 /* turn on the PMC ENABLE bit */ 669 config = pm->pm_md.pm_amd.pm_amd_evsel | AMD_PMC_ENABLE; 670 671 PMCDBG(MDP,STA,2,"amd-start config=0x%x", config); 672 673 wrmsr(pd->pm_evsel, config); 674 return 0; 675} 676 677/* 678 * Stop a PMC. 679 */ 680 681static int 682amd_stop_pmc(int cpu, int ri) 683{ 684 struct pmc *pm; 685 struct pmc_hw *phw; 686 const struct amd_descr *pd; 687 uint64_t config; 688 689 KASSERT(cpu >= 0 && cpu < mp_ncpus, 690 ("[amd,%d] illegal CPU value %d", __LINE__, cpu)); 691 KASSERT(ri >= 0 && ri < AMD_NPMCS, 692 ("[amd,%d] illegal row-index %d", __LINE__, ri)); 693 694 phw = pmc_pcpu[cpu]->pc_hwpmcs[ri]; 695 pm = phw->phw_pmc; 696 pd = &amd_pmcdesc[ri]; 697 698 KASSERT(pm != NULL, 699 ("[amd,%d] cpu%d,pmc%d no PMC to stop", __LINE__, 700 cpu, ri)); 701 702 /* can't stop a TSC */ 703 if (pd->pm_descr.pd_class == PMC_CLASS_TSC) 704 return 0; 705 706 KASSERT(pd->pm_descr.pd_class == AMD_PMC_CLASS, 707 ("[amd,%d] unknown PMC class (%d)", __LINE__, 708 pd->pm_descr.pd_class)); 709 710 KASSERT(!AMD_PMC_IS_STOPPED(pd->pm_evsel), 711 ("[amd,%d] PMC%d, CPU%d \"%s\" already stopped", 712 __LINE__, ri, cpu, pd->pm_descr.pd_name)); 713 714 PMCDBG(MDP,STO,1,"amd-stop ri=%d", ri); 715 716 /* turn off the PMC ENABLE bit */ 717 config = pm->pm_md.pm_amd.pm_amd_evsel & ~AMD_PMC_ENABLE; 718 wrmsr(pd->pm_evsel, config); 719 return 0; 720} 721 722/* 723 * Interrupt handler. This function needs to return '1' if the 724 * interrupt was this CPU's PMCs or '0' otherwise. It is not allowed 725 * to sleep or do anything a 'fast' interrupt handler is not allowed 726 * to do. 727 */ 728 729static int 730amd_intr(int cpu, uintptr_t eip) 731{ 732 int i, retval; 733 enum pmc_mode mode; 734 uint32_t perfctr; 735 struct pmc *pm; 736 struct pmc_cpu *pc; 737 struct pmc_hw *phw; 738 739 KASSERT(cpu >= 0 && cpu < mp_ncpus, 740 ("[amd,%d] out of range CPU %d", __LINE__, cpu)); 741 742 retval = 0; 743 744 pc = pmc_pcpu[cpu]; 745 746 /* 747 * look for all PMCs that have interrupted: 748 * - skip over the TSC [PMC#0] 749 * - look for a PMC with a valid 'struct pmc' association 750 * - look for a PMC in (a) sampling mode and (b) which has 751 * overflowed. If found, we update the process's 752 * histogram or send it a profiling signal by calling 753 * the appropriate helper function. 754 */ 755 756 for (i = 1; i < AMD_NPMCS; i++) { 757 758 phw = pc->pc_hwpmcs[i]; 759 perfctr = amd_pmcdesc[i].pm_perfctr; 760 KASSERT(phw != NULL, ("[amd,%d] null PHW pointer", __LINE__)); 761 762 if ((pm = phw->phw_pmc) == NULL || 763 pm->pm_state != PMC_STATE_RUNNING) { 764 atomic_add_int(&pmc_stats.pm_intr_ignored, 1); 765 continue; 766 } 767 768 mode = pm->pm_mode; 769 if (PMC_IS_SAMPLING_MODE(mode) && 770 AMD_PMC_HAS_OVERFLOWED(perfctr)) { 771 atomic_add_int(&pmc_stats.pm_intr_processed, 1); 772 if (PMC_IS_SYSTEM_MODE(mode)) 773 pmc_update_histogram(phw, eip); 774 else if (PMC_IS_VIRTUAL_MODE(mode)) 775 pmc_send_signal(pm); 776 retval = 1; 777 } 778 } 779 return retval; 780} 781 782/* 783 * describe a PMC 784 */ 785static int 786amd_describe(int cpu, int ri, struct pmc_info *pi, struct pmc **ppmc) 787{ 788 int error; 789 size_t copied; 790 const struct amd_descr *pd; 791 struct pmc_hw *phw; 792 793 KASSERT(cpu >= 0 && cpu < mp_ncpus, 794 ("[amd,%d] illegal CPU %d", __LINE__, cpu)); 795 KASSERT(ri >= 0 && ri < AMD_NPMCS, 796 ("[amd,%d] row-index %d out of range", __LINE__, ri)); 797 798 phw = pmc_pcpu[cpu]->pc_hwpmcs[ri]; 799 pd = &amd_pmcdesc[ri]; 800 801 if ((error = copystr(pd->pm_descr.pd_name, pi->pm_name, 802 PMC_NAME_MAX, &copied)) != 0) 803 return error; 804 805 pi->pm_class = pd->pm_descr.pd_class; 806 pi->pm_caps = pd->pm_descr.pd_caps; 807 pi->pm_width = pd->pm_descr.pd_width; 808 809 if (phw->phw_state & PMC_PHW_FLAG_IS_ENABLED) { 810 pi->pm_enabled = TRUE; 811 *ppmc = phw->phw_pmc; 812 } else { 813 pi->pm_enabled = FALSE; 814 *ppmc = NULL; 815 } 816 817 return 0; 818} 819 820/* 821 * i386 specific entry points 822 */ 823 824/* 825 * return the MSR address of the given PMC. 826 */ 827 828static int 829amd_get_msr(int ri, uint32_t *msr) 830{ 831 KASSERT(ri >= 0 && ri < AMD_NPMCS, 832 ("[amd,%d] ri %d out of range", __LINE__, ri)); 833 834 *msr = amd_pmcdesc[ri].pm_perfctr - AMD_PMC_PERFCTR_0; 835 return 0; 836} 837 838/* 839 * processor dependent initialization. 840 */ 841 842/* 843 * Per-processor data structure 844 * 845 * [common stuff] 846 * [5 struct pmc_hw pointers] 847 * [5 struct pmc_hw structures] 848 */ 849 850struct amd_cpu { 851 struct pmc_cpu pc_common; 852 struct pmc_hw *pc_hwpmcs[AMD_NPMCS]; 853 struct pmc_hw pc_amdpmcs[AMD_NPMCS]; 854}; 855 856 857static int 858amd_init(int cpu) 859{ 860 int n; 861 struct amd_cpu *pcs; 862 struct pmc_hw *phw; 863 864 KASSERT(cpu >= 0 && cpu < mp_ncpus, 865 ("[amd,%d] insane cpu number %d", __LINE__, cpu)); 866 867 PMCDBG(MDP,INI,1,"amd-init cpu=%d", cpu); 868 869 MALLOC(pcs, struct amd_cpu *, sizeof(struct amd_cpu), M_PMC, 870 M_WAITOK|M_ZERO); 871 872 if (pcs == NULL) 873 return ENOMEM; 874 875 phw = &pcs->pc_amdpmcs[0]; 876 877 /* 878 * Initialize the per-cpu mutex and set the content of the 879 * hardware descriptors to a known state. 880 */ 881 882 for (n = 0; n < AMD_NPMCS; n++, phw++) { 883 phw->phw_state = PMC_PHW_FLAG_IS_ENABLED | 884 PMC_PHW_CPU_TO_STATE(cpu) | PMC_PHW_INDEX_TO_STATE(n); 885 phw->phw_pmc = NULL; 886 pcs->pc_hwpmcs[n] = phw; 887 } 888 889 /* Mark the TSC as shareable */ 890 pcs->pc_hwpmcs[0]->phw_state |= PMC_PHW_FLAG_IS_SHAREABLE; 891 892 pmc_pcpu[cpu] = (struct pmc_cpu *) pcs; 893 894 return 0; 895} 896 897 898/* 899 * processor dependent cleanup prior to the KLD 900 * being unloaded 901 */ 902 903static int 904amd_cleanup(int cpu) 905{ 906 int i; 907 uint32_t evsel; 908 struct pmc_cpu *pcs; 909 910 KASSERT(cpu >= 0 && cpu < mp_ncpus, 911 ("[amd,%d] insane cpu number (%d)", __LINE__, cpu)); 912 913 PMCDBG(MDP,INI,1,"amd-cleanup cpu=%d", cpu); 914 915 /* 916 * First, turn off all PMCs on this CPU. 917 */ 918 919 for (i = 0; i < 4; i++) { /* XXX this loop is now not needed */ 920 evsel = rdmsr(AMD_PMC_EVSEL_0 + i); 921 evsel &= ~AMD_PMC_ENABLE; 922 wrmsr(AMD_PMC_EVSEL_0 + i, evsel); 923 } 924 925 /* 926 * Next, free up allocated space. 927 */ 928 929 pcs = pmc_pcpu[cpu]; 930 931#if DEBUG 932 /* check the TSC */ 933 KASSERT(pcs->pc_hwpmcs[0]->phw_pmc == NULL, 934 ("[amd,%d] CPU%d,PMC0 still in use", __LINE__, cpu)); 935 for (i = 1; i < AMD_NPMCS; i++) { 936 KASSERT(pcs->pc_hwpmcs[i]->phw_pmc == NULL, 937 ("[amd,%d] CPU%d/PMC%d in use", __LINE__, cpu, i)); 938 KASSERT(AMD_PMC_IS_STOPPED(AMD_PMC_EVSEL_0 + (i-1)), 939 ("[amd,%d] CPU%d/PMC%d not stopped", __LINE__, cpu, i)); 940 } 941#endif 942 KASSERT(pcs != NULL, 943 ("[amd,%d] null per-cpu state pointer (cpu%d)", __LINE__, cpu)); 944 945 pmc_pcpu[cpu] = NULL; 946 FREE(pcs, M_PMC); 947 return 0; 948} 949 950/* 951 * Initialize ourselves. 952 */ 953 954struct pmc_mdep * 955pmc_amd_initialize(void) 956{ 957 958 struct pmc_mdep *pmc_mdep; 959 960 /* The presence of hardware performance counters on the AMD 961 Athlon, Duron or later processors, is _not_ indicated by 962 any of the processor feature flags set by the 'CPUID' 963 instruction, so we only check the 'instruction family' 964 field returned by CPUID for instruction family >= 6. This 965 test needs to be be refined. */ 966 967 if ((cpu_id & 0xF00) < 0x600) 968 return NULL; 969 970 MALLOC(pmc_mdep, struct pmc_mdep *, sizeof(struct pmc_mdep), 971 M_PMC, M_WAITOK|M_ZERO); 972 973#if __i386__ 974 pmc_mdep->pmd_cputype = PMC_CPU_AMD_K7; 975#elif __amd64__ 976 pmc_mdep->pmd_cputype = PMC_CPU_AMD_K8; 977#else 978#error Unknown AMD CPU type. 979#endif 980 981 pmc_mdep->pmd_npmc = AMD_NPMCS; 982 983 /* this processor has two classes of usable PMCs */ 984 pmc_mdep->pmd_nclass = 2; 985 pmc_mdep->pmd_classes[0] = PMC_CLASS_TSC; 986 pmc_mdep->pmd_classes[1] = AMD_PMC_CLASS; 987 pmc_mdep->pmd_nclasspmcs[0] = 1; 988 pmc_mdep->pmd_nclasspmcs[1] = (AMD_NPMCS-1); 989 990 pmc_mdep->pmd_init = amd_init; 991 pmc_mdep->pmd_cleanup = amd_cleanup; 992 pmc_mdep->pmd_switch_in = amd_switch_in; 993 pmc_mdep->pmd_switch_out = amd_switch_out; 994 pmc_mdep->pmd_read_pmc = amd_read_pmc; 995 pmc_mdep->pmd_write_pmc = amd_write_pmc; 996 pmc_mdep->pmd_config_pmc = amd_config_pmc; 997 pmc_mdep->pmd_allocate_pmc = amd_allocate_pmc; 998 pmc_mdep->pmd_release_pmc = amd_release_pmc; 999 pmc_mdep->pmd_start_pmc = amd_start_pmc; 1000 pmc_mdep->pmd_stop_pmc = amd_stop_pmc; 1001 pmc_mdep->pmd_intr = amd_intr; 1002 pmc_mdep->pmd_describe = amd_describe; 1003 pmc_mdep->pmd_get_msr = amd_get_msr; /* i386 */ 1004 1005 PMCDBG(MDP,INI,0,"%s","amd-initialize"); 1006 1007 return pmc_mdep; 1008} 1009