hwpmc_amd.c revision 177344
1/*- 2 * Copyright (c) 2003-2007 Joseph Koshy 3 * Copyright (c) 2007 The FreeBSD Foundation 4 * All rights reserved. 5 * 6 * Portions of this software were developed by A. Joseph Koshy under 7 * sponsorship from the FreeBSD Foundation and Google, Inc. 8 * 9 * Redistribution and use in source and binary forms, with or without 10 * modification, are permitted provided that the following conditions 11 * are met: 12 * 1. Redistributions of source code must retain the above copyright 13 * notice, this list of conditions and the following disclaimer. 14 * 2. Redistributions in binary form must reproduce the above copyright 15 * notice, this list of conditions and the following disclaimer in the 16 * documentation and/or other materials provided with the distribution. 17 * 18 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 19 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 20 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 21 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 22 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 23 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 24 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 25 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 26 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 27 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 28 * SUCH DAMAGE. 29 * 30 */ 31 32#include <sys/cdefs.h> 33__FBSDID("$FreeBSD: head/sys/dev/hwpmc/hwpmc_amd.c 177344 2008-03-18 08:39:11Z adrian $"); 34 35/* Support for the AMD K7 and later processors */ 36 37#include <sys/param.h> 38#include <sys/lock.h> 39#include <sys/malloc.h> 40#include <sys/mutex.h> 41#include <sys/pmc.h> 42#include <sys/smp.h> 43#include <sys/systm.h> 44 45#include <machine/cpu.h> 46#include <machine/cpufunc.h> 47#include <machine/md_var.h> 48#include <machine/specialreg.h> 49 50#ifdef DEBUG 51enum pmc_class amd_pmc_class; 52#endif 53 54/* AMD K7 & K8 PMCs */ 55struct amd_descr { 56 struct pmc_descr pm_descr; /* "base class" */ 57 uint32_t pm_evsel; /* address of EVSEL register */ 58 uint32_t pm_perfctr; /* address of PERFCTR register */ 59}; 60 61static struct amd_descr amd_pmcdesc[AMD_NPMCS] = 62{ 63 { 64 .pm_descr = 65 { 66 .pd_name = "TSC", 67 .pd_class = PMC_CLASS_TSC, 68 .pd_caps = PMC_CAP_READ, 69 .pd_width = 64 70 }, 71 .pm_evsel = MSR_TSC, 72 .pm_perfctr = 0 /* unused */ 73 }, 74 75 { 76 .pm_descr = 77 { 78 .pd_name = "", 79 .pd_class = -1, 80 .pd_caps = AMD_PMC_CAPS, 81 .pd_width = 48 82 }, 83 .pm_evsel = AMD_PMC_EVSEL_0, 84 .pm_perfctr = AMD_PMC_PERFCTR_0 85 }, 86 { 87 .pm_descr = 88 { 89 .pd_name = "", 90 .pd_class = -1, 91 .pd_caps = AMD_PMC_CAPS, 92 .pd_width = 48 93 }, 94 .pm_evsel = AMD_PMC_EVSEL_1, 95 .pm_perfctr = AMD_PMC_PERFCTR_1 96 }, 97 { 98 .pm_descr = 99 { 100 .pd_name = "", 101 .pd_class = -1, 102 .pd_caps = AMD_PMC_CAPS, 103 .pd_width = 48 104 }, 105 .pm_evsel = AMD_PMC_EVSEL_2, 106 .pm_perfctr = AMD_PMC_PERFCTR_2 107 }, 108 { 109 .pm_descr = 110 { 111 .pd_name = "", 112 .pd_class = -1, 113 .pd_caps = AMD_PMC_CAPS, 114 .pd_width = 48 115 }, 116 .pm_evsel = AMD_PMC_EVSEL_3, 117 .pm_perfctr = AMD_PMC_PERFCTR_3 118 } 119}; 120 121struct amd_event_code_map { 122 enum pmc_event pe_ev; /* enum value */ 123 uint8_t pe_code; /* encoded event mask */ 124 uint8_t pe_mask; /* bits allowed in unit mask */ 125}; 126 127const struct amd_event_code_map amd_event_codes[] = { 128#if defined(__i386__) /* 32 bit Athlon (K7) only */ 129 { PMC_EV_K7_DC_ACCESSES, 0x40, 0 }, 130 { PMC_EV_K7_DC_MISSES, 0x41, 0 }, 131 { PMC_EV_K7_DC_REFILLS_FROM_L2, 0x42, AMD_PMC_UNITMASK_MOESI }, 132 { PMC_EV_K7_DC_REFILLS_FROM_SYSTEM, 0x43, AMD_PMC_UNITMASK_MOESI }, 133 { PMC_EV_K7_DC_WRITEBACKS, 0x44, AMD_PMC_UNITMASK_MOESI }, 134 { PMC_EV_K7_L1_DTLB_MISS_AND_L2_DTLB_HITS, 0x45, 0 }, 135 { PMC_EV_K7_L1_AND_L2_DTLB_MISSES, 0x46, 0 }, 136 { PMC_EV_K7_MISALIGNED_REFERENCES, 0x47, 0 }, 137 138 { PMC_EV_K7_IC_FETCHES, 0x80, 0 }, 139 { PMC_EV_K7_IC_MISSES, 0x81, 0 }, 140 141 { PMC_EV_K7_L1_ITLB_MISSES, 0x84, 0 }, 142 { PMC_EV_K7_L1_L2_ITLB_MISSES, 0x85, 0 }, 143 144 { PMC_EV_K7_RETIRED_INSTRUCTIONS, 0xC0, 0 }, 145 { PMC_EV_K7_RETIRED_OPS, 0xC1, 0 }, 146 { PMC_EV_K7_RETIRED_BRANCHES, 0xC2, 0 }, 147 { PMC_EV_K7_RETIRED_BRANCHES_MISPREDICTED, 0xC3, 0 }, 148 { PMC_EV_K7_RETIRED_TAKEN_BRANCHES, 0xC4, 0 }, 149 { PMC_EV_K7_RETIRED_TAKEN_BRANCHES_MISPREDICTED, 0xC5, 0 }, 150 { PMC_EV_K7_RETIRED_FAR_CONTROL_TRANSFERS, 0xC6, 0 }, 151 { PMC_EV_K7_RETIRED_RESYNC_BRANCHES, 0xC7, 0 }, 152 { PMC_EV_K7_INTERRUPTS_MASKED_CYCLES, 0xCD, 0 }, 153 { PMC_EV_K7_INTERRUPTS_MASKED_WHILE_PENDING_CYCLES, 0xCE, 0 }, 154 { PMC_EV_K7_HARDWARE_INTERRUPTS, 0xCF, 0 }, 155#endif 156 157 { PMC_EV_K8_FP_DISPATCHED_FPU_OPS, 0x00, 0x3F }, 158 { PMC_EV_K8_FP_CYCLES_WITH_NO_FPU_OPS_RETIRED, 0x01, 0x00 }, 159 { PMC_EV_K8_FP_DISPATCHED_FPU_FAST_FLAG_OPS, 0x02, 0x00 }, 160 161 { PMC_EV_K8_LS_SEGMENT_REGISTER_LOAD, 0x20, 0x7F }, 162 { PMC_EV_K8_LS_MICROARCHITECTURAL_RESYNC_BY_SELF_MODIFYING_CODE, 163 0x21, 0x00 }, 164 { PMC_EV_K8_LS_MICROARCHITECTURAL_RESYNC_BY_SNOOP, 0x22, 0x00 }, 165 { PMC_EV_K8_LS_BUFFER2_FULL, 0x23, 0x00 }, 166 { PMC_EV_K8_LS_LOCKED_OPERATION, 0x24, 0x07 }, 167 { PMC_EV_K8_LS_MICROARCHITECTURAL_LATE_CANCEL, 0x25, 0x00 }, 168 { PMC_EV_K8_LS_RETIRED_CFLUSH_INSTRUCTIONS, 0x26, 0x00 }, 169 { PMC_EV_K8_LS_RETIRED_CPUID_INSTRUCTIONS, 0x27, 0x00 }, 170 171 { PMC_EV_K8_DC_ACCESS, 0x40, 0x00 }, 172 { PMC_EV_K8_DC_MISS, 0x41, 0x00 }, 173 { PMC_EV_K8_DC_REFILL_FROM_L2, 0x42, 0x1F }, 174 { PMC_EV_K8_DC_REFILL_FROM_SYSTEM, 0x43, 0x1F }, 175 { PMC_EV_K8_DC_COPYBACK, 0x44, 0x1F }, 176 { PMC_EV_K8_DC_L1_DTLB_MISS_AND_L2_DTLB_HIT, 0x45, 0x00 }, 177 { PMC_EV_K8_DC_L1_DTLB_MISS_AND_L2_DTLB_MISS, 0x46, 0x00 }, 178 { PMC_EV_K8_DC_MISALIGNED_DATA_REFERENCE, 0x47, 0x00 }, 179 { PMC_EV_K8_DC_MICROARCHITECTURAL_LATE_CANCEL, 0x48, 0x00 }, 180 { PMC_EV_K8_DC_MICROARCHITECTURAL_EARLY_CANCEL, 0x49, 0x00 }, 181 { PMC_EV_K8_DC_ONE_BIT_ECC_ERROR, 0x4A, 0x03 }, 182 { PMC_EV_K8_DC_DISPATCHED_PREFETCH_INSTRUCTIONS, 0x4B, 0x07 }, 183 { PMC_EV_K8_DC_DCACHE_ACCESSES_BY_LOCKS, 0x4C, 0x03 }, 184 185 { PMC_EV_K8_BU_CPU_CLK_UNHALTED, 0x76, 0x00 }, 186 { PMC_EV_K8_BU_INTERNAL_L2_REQUEST, 0x7D, 0x1F }, 187 { PMC_EV_K8_BU_FILL_REQUEST_L2_MISS, 0x7E, 0x07 }, 188 { PMC_EV_K8_BU_FILL_INTO_L2, 0x7F, 0x03 }, 189 190 { PMC_EV_K8_IC_FETCH, 0x80, 0x00 }, 191 { PMC_EV_K8_IC_MISS, 0x81, 0x00 }, 192 { PMC_EV_K8_IC_REFILL_FROM_L2, 0x82, 0x00 }, 193 { PMC_EV_K8_IC_REFILL_FROM_SYSTEM, 0x83, 0x00 }, 194 { PMC_EV_K8_IC_L1_ITLB_MISS_AND_L2_ITLB_HIT, 0x84, 0x00 }, 195 { PMC_EV_K8_IC_L1_ITLB_MISS_AND_L2_ITLB_MISS, 0x85, 0x00 }, 196 { PMC_EV_K8_IC_MICROARCHITECTURAL_RESYNC_BY_SNOOP, 0x86, 0x00 }, 197 { PMC_EV_K8_IC_INSTRUCTION_FETCH_STALL, 0x87, 0x00 }, 198 { PMC_EV_K8_IC_RETURN_STACK_HIT, 0x88, 0x00 }, 199 { PMC_EV_K8_IC_RETURN_STACK_OVERFLOW, 0x89, 0x00 }, 200 201 { PMC_EV_K8_FR_RETIRED_X86_INSTRUCTIONS, 0xC0, 0x00 }, 202 { PMC_EV_K8_FR_RETIRED_UOPS, 0xC1, 0x00 }, 203 { PMC_EV_K8_FR_RETIRED_BRANCHES, 0xC2, 0x00 }, 204 { PMC_EV_K8_FR_RETIRED_BRANCHES_MISPREDICTED, 0xC3, 0x00 }, 205 { PMC_EV_K8_FR_RETIRED_TAKEN_BRANCHES, 0xC4, 0x00 }, 206 { PMC_EV_K8_FR_RETIRED_TAKEN_BRANCHES_MISPREDICTED, 0xC5, 0x00 }, 207 { PMC_EV_K8_FR_RETIRED_FAR_CONTROL_TRANSFERS, 0xC6, 0x00 }, 208 { PMC_EV_K8_FR_RETIRED_RESYNCS, 0xC7, 0x00 }, 209 { PMC_EV_K8_FR_RETIRED_NEAR_RETURNS, 0xC8, 0x00 }, 210 { PMC_EV_K8_FR_RETIRED_NEAR_RETURNS_MISPREDICTED, 0xC9, 0x00 }, 211 { PMC_EV_K8_FR_RETIRED_TAKEN_BRANCHES_MISPREDICTED_BY_ADDR_MISCOMPARE, 212 0xCA, 0x00 }, 213 { PMC_EV_K8_FR_RETIRED_FPU_INSTRUCTIONS, 0xCB, 0x0F }, 214 { PMC_EV_K8_FR_RETIRED_FASTPATH_DOUBLE_OP_INSTRUCTIONS, 215 0xCC, 0x07 }, 216 { PMC_EV_K8_FR_INTERRUPTS_MASKED_CYCLES, 0xCD, 0x00 }, 217 { PMC_EV_K8_FR_INTERRUPTS_MASKED_WHILE_PENDING_CYCLES, 0xCE, 0x00 }, 218 { PMC_EV_K8_FR_TAKEN_HARDWARE_INTERRUPTS, 0xCF, 0x00 }, 219 220 { PMC_EV_K8_FR_DECODER_EMPTY, 0xD0, 0x00 }, 221 { PMC_EV_K8_FR_DISPATCH_STALLS, 0xD1, 0x00 }, 222 { PMC_EV_K8_FR_DISPATCH_STALL_FROM_BRANCH_ABORT_TO_RETIRE, 223 0xD2, 0x00 }, 224 { PMC_EV_K8_FR_DISPATCH_STALL_FOR_SERIALIZATION, 0xD3, 0x00 }, 225 { PMC_EV_K8_FR_DISPATCH_STALL_FOR_SEGMENT_LOAD, 0xD4, 0x00 }, 226 { PMC_EV_K8_FR_DISPATCH_STALL_WHEN_REORDER_BUFFER_IS_FULL, 227 0xD5, 0x00 }, 228 { PMC_EV_K8_FR_DISPATCH_STALL_WHEN_RESERVATION_STATIONS_ARE_FULL, 229 0xD6, 0x00 }, 230 { PMC_EV_K8_FR_DISPATCH_STALL_WHEN_FPU_IS_FULL, 0xD7, 0x00 }, 231 { PMC_EV_K8_FR_DISPATCH_STALL_WHEN_LS_IS_FULL, 0xD8, 0x00 }, 232 { PMC_EV_K8_FR_DISPATCH_STALL_WHEN_WAITING_FOR_ALL_TO_BE_QUIET, 233 0xD9, 0x00 }, 234 { PMC_EV_K8_FR_DISPATCH_STALL_WHEN_FAR_XFER_OR_RESYNC_BRANCH_PENDING, 235 0xDA, 0x00 }, 236 { PMC_EV_K8_FR_FPU_EXCEPTIONS, 0xDB, 0x0F }, 237 { PMC_EV_K8_FR_NUMBER_OF_BREAKPOINTS_FOR_DR0, 0xDC, 0x00 }, 238 { PMC_EV_K8_FR_NUMBER_OF_BREAKPOINTS_FOR_DR1, 0xDD, 0x00 }, 239 { PMC_EV_K8_FR_NUMBER_OF_BREAKPOINTS_FOR_DR2, 0xDE, 0x00 }, 240 { PMC_EV_K8_FR_NUMBER_OF_BREAKPOINTS_FOR_DR3, 0xDF, 0x00 }, 241 242 { PMC_EV_K8_NB_MEMORY_CONTROLLER_PAGE_ACCESS_EVENT, 0xE0, 0x7 }, 243 { PMC_EV_K8_NB_MEMORY_CONTROLLER_PAGE_TABLE_OVERFLOW, 0xE1, 0x00 }, 244 { PMC_EV_K8_NB_MEMORY_CONTROLLER_DRAM_COMMAND_SLOTS_MISSED, 245 0xE2, 0x00 }, 246 { PMC_EV_K8_NB_MEMORY_CONTROLLER_TURNAROUND, 0xE3, 0x07 }, 247 { PMC_EV_K8_NB_MEMORY_CONTROLLER_BYPASS_SATURATION, 0xE4, 0x0F }, 248 { PMC_EV_K8_NB_SIZED_COMMANDS, 0xEB, 0x7F }, 249 { PMC_EV_K8_NB_PROBE_RESULT, 0xEC, 0x0F }, 250 { PMC_EV_K8_NB_HT_BUS0_BANDWIDTH, 0xF6, 0x0F }, 251 { PMC_EV_K8_NB_HT_BUS1_BANDWIDTH, 0xF7, 0x0F }, 252 { PMC_EV_K8_NB_HT_BUS2_BANDWIDTH, 0xF8, 0x0F } 253 254}; 255 256const int amd_event_codes_size = 257 sizeof(amd_event_codes) / sizeof(amd_event_codes[0]); 258 259/* 260 * read a pmc register 261 */ 262 263static int 264amd_read_pmc(int cpu, int ri, pmc_value_t *v) 265{ 266 enum pmc_mode mode; 267 const struct amd_descr *pd; 268 struct pmc *pm; 269 const struct pmc_hw *phw; 270 pmc_value_t tmp; 271 272 KASSERT(cpu >= 0 && cpu < mp_ncpus, 273 ("[amd,%d] illegal CPU value %d", __LINE__, cpu)); 274 KASSERT(ri >= 0 && ri < AMD_NPMCS, 275 ("[amd,%d] illegal row-index %d", __LINE__, ri)); 276 277 phw = pmc_pcpu[cpu]->pc_hwpmcs[ri]; 278 pd = &amd_pmcdesc[ri]; 279 pm = phw->phw_pmc; 280 281 KASSERT(pm != NULL, 282 ("[amd,%d] No owner for HWPMC [cpu%d,pmc%d]", __LINE__, 283 cpu, ri)); 284 285 mode = PMC_TO_MODE(pm); 286 287 PMCDBG(MDP,REA,1,"amd-read id=%d class=%d", ri, pd->pm_descr.pd_class); 288 289 /* Reading the TSC is a special case */ 290 if (pd->pm_descr.pd_class == PMC_CLASS_TSC) { 291 KASSERT(PMC_IS_COUNTING_MODE(mode), 292 ("[amd,%d] TSC counter in non-counting mode", __LINE__)); 293 *v = rdtsc(); 294 PMCDBG(MDP,REA,2,"amd-read id=%d -> %jd", ri, *v); 295 return 0; 296 } 297 298#ifdef DEBUG 299 KASSERT(pd->pm_descr.pd_class == amd_pmc_class, 300 ("[amd,%d] unknown PMC class (%d)", __LINE__, 301 pd->pm_descr.pd_class)); 302#endif 303 304 tmp = rdmsr(pd->pm_perfctr); /* RDMSR serializes */ 305 PMCDBG(MDP,REA,2,"amd-read (pre-munge) id=%d -> %jd", ri, tmp); 306 if (PMC_IS_SAMPLING_MODE(mode)) { 307 /* Sign extend 48 bit value to 64 bits. */ 308 tmp = (pmc_value_t) (((int64_t) tmp << 16) >> 16); 309 tmp = AMD_PERFCTR_VALUE_TO_RELOAD_COUNT(tmp); 310 } 311 *v = tmp; 312 313 PMCDBG(MDP,REA,2,"amd-read (post-munge) id=%d -> %jd", ri, *v); 314 315 return 0; 316} 317 318/* 319 * Write a PMC MSR. 320 */ 321 322static int 323amd_write_pmc(int cpu, int ri, pmc_value_t v) 324{ 325 const struct amd_descr *pd; 326 struct pmc *pm; 327 const struct pmc_hw *phw; 328 enum pmc_mode mode; 329 330 KASSERT(cpu >= 0 && cpu < mp_ncpus, 331 ("[amd,%d] illegal CPU value %d", __LINE__, cpu)); 332 KASSERT(ri >= 0 && ri < AMD_NPMCS, 333 ("[amd,%d] illegal row-index %d", __LINE__, ri)); 334 335 phw = pmc_pcpu[cpu]->pc_hwpmcs[ri]; 336 pd = &amd_pmcdesc[ri]; 337 pm = phw->phw_pmc; 338 339 KASSERT(pm != NULL, 340 ("[amd,%d] PMC not owned (cpu%d,pmc%d)", __LINE__, 341 cpu, ri)); 342 343 mode = PMC_TO_MODE(pm); 344 345 if (pd->pm_descr.pd_class == PMC_CLASS_TSC) 346 return 0; 347 348#ifdef DEBUG 349 KASSERT(pd->pm_descr.pd_class == amd_pmc_class, 350 ("[amd,%d] unknown PMC class (%d)", __LINE__, 351 pd->pm_descr.pd_class)); 352#endif 353 354 /* use 2's complement of the count for sampling mode PMCs */ 355 if (PMC_IS_SAMPLING_MODE(mode)) 356 v = AMD_RELOAD_COUNT_TO_PERFCTR_VALUE(v); 357 358 PMCDBG(MDP,WRI,1,"amd-write cpu=%d ri=%d v=%jx", cpu, ri, v); 359 360 /* write the PMC value */ 361 wrmsr(pd->pm_perfctr, v); 362 return 0; 363} 364 365/* 366 * configure hardware pmc according to the configuration recorded in 367 * pmc 'pm'. 368 */ 369 370static int 371amd_config_pmc(int cpu, int ri, struct pmc *pm) 372{ 373 struct pmc_hw *phw; 374 375 PMCDBG(MDP,CFG,1, "cpu=%d ri=%d pm=%p", cpu, ri, pm); 376 377 KASSERT(cpu >= 0 && cpu < mp_ncpus, 378 ("[amd,%d] illegal CPU value %d", __LINE__, cpu)); 379 KASSERT(ri >= 0 && ri < AMD_NPMCS, 380 ("[amd,%d] illegal row-index %d", __LINE__, ri)); 381 382 phw = pmc_pcpu[cpu]->pc_hwpmcs[ri]; 383 384 KASSERT(pm == NULL || phw->phw_pmc == NULL, 385 ("[amd,%d] pm=%p phw->pm=%p hwpmc not unconfigured", 386 __LINE__, pm, phw->phw_pmc)); 387 388 phw->phw_pmc = pm; 389 return 0; 390} 391 392/* 393 * Retrieve a configured PMC pointer from hardware state. 394 */ 395 396static int 397amd_get_config(int cpu, int ri, struct pmc **ppm) 398{ 399 *ppm = pmc_pcpu[cpu]->pc_hwpmcs[ri]->phw_pmc; 400 401 return 0; 402} 403 404/* 405 * Machine dependent actions taken during the context switch in of a 406 * thread. 407 */ 408 409static int 410amd_switch_in(struct pmc_cpu *pc, struct pmc_process *pp) 411{ 412 (void) pc; 413 414 PMCDBG(MDP,SWI,1, "pc=%p pp=%p enable-msr=%d", pc, pp, 415 (pp->pp_flags & PMC_PP_ENABLE_MSR_ACCESS) != 0); 416 417 /* enable the RDPMC instruction if needed */ 418 if (pp->pp_flags & PMC_PP_ENABLE_MSR_ACCESS) 419 load_cr4(rcr4() | CR4_PCE); 420 421 return 0; 422} 423 424/* 425 * Machine dependent actions taken during the context switch out of a 426 * thread. 427 */ 428 429static int 430amd_switch_out(struct pmc_cpu *pc, struct pmc_process *pp) 431{ 432 (void) pc; 433 (void) pp; /* can be NULL */ 434 435 PMCDBG(MDP,SWO,1, "pc=%p pp=%p enable-msr=%d", pc, pp, pp ? 436 (pp->pp_flags & PMC_PP_ENABLE_MSR_ACCESS) == 1 : 0); 437 438 /* always turn off the RDPMC instruction */ 439 load_cr4(rcr4() & ~CR4_PCE); 440 441 return 0; 442} 443 444/* 445 * Check if a given allocation is feasible. 446 */ 447 448static int 449amd_allocate_pmc(int cpu, int ri, struct pmc *pm, 450 const struct pmc_op_pmcallocate *a) 451{ 452 int i; 453 uint32_t allowed_unitmask, caps, config, unitmask; 454 enum pmc_event pe; 455 const struct pmc_descr *pd; 456 457 (void) cpu; 458 459 KASSERT(cpu >= 0 && cpu < mp_ncpus, 460 ("[amd,%d] illegal CPU value %d", __LINE__, cpu)); 461 KASSERT(ri >= 0 && ri < AMD_NPMCS, 462 ("[amd,%d] illegal row index %d", __LINE__, ri)); 463 464 pd = &amd_pmcdesc[ri].pm_descr; 465 466 /* check class match */ 467 if (pd->pd_class != a->pm_class) 468 return EINVAL; 469 470 caps = pm->pm_caps; 471 472 PMCDBG(MDP,ALL,1,"amd-allocate ri=%d caps=0x%x", ri, caps); 473 474 if ((pd->pd_caps & caps) != caps) 475 return EPERM; 476 if (pd->pd_class == PMC_CLASS_TSC) { 477 /* TSC's are always allocated in system-wide counting mode */ 478 if (a->pm_ev != PMC_EV_TSC_TSC || 479 a->pm_mode != PMC_MODE_SC) 480 return EINVAL; 481 return 0; 482 } 483 484#ifdef DEBUG 485 KASSERT(pd->pd_class == amd_pmc_class, 486 ("[amd,%d] Unknown PMC class (%d)", __LINE__, pd->pd_class)); 487#endif 488 489 pe = a->pm_ev; 490 491 /* map ev to the correct event mask code */ 492 config = allowed_unitmask = 0; 493 for (i = 0; i < amd_event_codes_size; i++) 494 if (amd_event_codes[i].pe_ev == pe) { 495 config = 496 AMD_PMC_TO_EVENTMASK(amd_event_codes[i].pe_code); 497 allowed_unitmask = 498 AMD_PMC_TO_UNITMASK(amd_event_codes[i].pe_mask); 499 break; 500 } 501 if (i == amd_event_codes_size) 502 return EINVAL; 503 504 unitmask = a->pm_md.pm_amd.pm_amd_config & AMD_PMC_UNITMASK; 505 if (unitmask & ~allowed_unitmask) /* disallow reserved bits */ 506 return EINVAL; 507 508 if (unitmask && (caps & PMC_CAP_QUALIFIER)) 509 config |= unitmask; 510 511 if (caps & PMC_CAP_THRESHOLD) 512 config |= a->pm_md.pm_amd.pm_amd_config & AMD_PMC_COUNTERMASK; 513 514 /* set at least one of the 'usr' or 'os' caps */ 515 if (caps & PMC_CAP_USER) 516 config |= AMD_PMC_USR; 517 if (caps & PMC_CAP_SYSTEM) 518 config |= AMD_PMC_OS; 519 if ((caps & (PMC_CAP_USER|PMC_CAP_SYSTEM)) == 0) 520 config |= (AMD_PMC_USR|AMD_PMC_OS); 521 522 if (caps & PMC_CAP_EDGE) 523 config |= AMD_PMC_EDGE; 524 if (caps & PMC_CAP_INVERT) 525 config |= AMD_PMC_INVERT; 526 if (caps & PMC_CAP_INTERRUPT) 527 config |= AMD_PMC_INT; 528 529 pm->pm_md.pm_amd.pm_amd_evsel = config; /* save config value */ 530 531 PMCDBG(MDP,ALL,2,"amd-allocate ri=%d -> config=0x%x", ri, config); 532 533 return 0; 534} 535 536/* 537 * Release machine dependent state associated with a PMC. This is a 538 * no-op on this architecture. 539 * 540 */ 541 542/* ARGSUSED0 */ 543static int 544amd_release_pmc(int cpu, int ri, struct pmc *pmc) 545{ 546#ifdef DEBUG 547 const struct amd_descr *pd; 548#endif 549 struct pmc_hw *phw; 550 551 (void) pmc; 552 553 KASSERT(cpu >= 0 && cpu < mp_ncpus, 554 ("[amd,%d] illegal CPU value %d", __LINE__, cpu)); 555 KASSERT(ri >= 0 && ri < AMD_NPMCS, 556 ("[amd,%d] illegal row-index %d", __LINE__, ri)); 557 558 phw = pmc_pcpu[cpu]->pc_hwpmcs[ri]; 559 560 KASSERT(phw->phw_pmc == NULL, 561 ("[amd,%d] PHW pmc %p non-NULL", __LINE__, phw->phw_pmc)); 562 563#ifdef DEBUG 564 pd = &amd_pmcdesc[ri]; 565 if (pd->pm_descr.pd_class == amd_pmc_class) 566 KASSERT(AMD_PMC_IS_STOPPED(pd->pm_evsel), 567 ("[amd,%d] PMC %d released while active", __LINE__, ri)); 568#endif 569 570 return 0; 571} 572 573/* 574 * start a PMC. 575 */ 576 577static int 578amd_start_pmc(int cpu, int ri) 579{ 580 uint32_t config; 581 struct pmc *pm; 582 struct pmc_hw *phw; 583 const struct amd_descr *pd; 584 585 KASSERT(cpu >= 0 && cpu < mp_ncpus, 586 ("[amd,%d] illegal CPU value %d", __LINE__, cpu)); 587 KASSERT(ri >= 0 && ri < AMD_NPMCS, 588 ("[amd,%d] illegal row-index %d", __LINE__, ri)); 589 590 phw = pmc_pcpu[cpu]->pc_hwpmcs[ri]; 591 pm = phw->phw_pmc; 592 pd = &amd_pmcdesc[ri]; 593 594 KASSERT(pm != NULL, 595 ("[amd,%d] starting cpu%d,pmc%d with null pmc record", __LINE__, 596 cpu, ri)); 597 598 PMCDBG(MDP,STA,1,"amd-start cpu=%d ri=%d", cpu, ri); 599 600 if (pd->pm_descr.pd_class == PMC_CLASS_TSC) 601 return 0; /* TSCs are always running */ 602 603#ifdef DEBUG 604 KASSERT(pd->pm_descr.pd_class == amd_pmc_class, 605 ("[amd,%d] unknown PMC class (%d)", __LINE__, 606 pd->pm_descr.pd_class)); 607#endif 608 609 KASSERT(AMD_PMC_IS_STOPPED(pd->pm_evsel), 610 ("[amd,%d] pmc%d,cpu%d: Starting active PMC \"%s\"", __LINE__, 611 ri, cpu, pd->pm_descr.pd_name)); 612 613 /* turn on the PMC ENABLE bit */ 614 config = pm->pm_md.pm_amd.pm_amd_evsel | AMD_PMC_ENABLE; 615 616 PMCDBG(MDP,STA,2,"amd-start config=0x%x", config); 617 618 wrmsr(pd->pm_evsel, config); 619 return 0; 620} 621 622/* 623 * Stop a PMC. 624 */ 625 626static int 627amd_stop_pmc(int cpu, int ri) 628{ 629 struct pmc *pm; 630 struct pmc_hw *phw; 631 const struct amd_descr *pd; 632 uint64_t config; 633 634 KASSERT(cpu >= 0 && cpu < mp_ncpus, 635 ("[amd,%d] illegal CPU value %d", __LINE__, cpu)); 636 KASSERT(ri >= 0 && ri < AMD_NPMCS, 637 ("[amd,%d] illegal row-index %d", __LINE__, ri)); 638 639 phw = pmc_pcpu[cpu]->pc_hwpmcs[ri]; 640 pm = phw->phw_pmc; 641 pd = &amd_pmcdesc[ri]; 642 643 KASSERT(pm != NULL, 644 ("[amd,%d] cpu%d,pmc%d no PMC to stop", __LINE__, 645 cpu, ri)); 646 647 /* can't stop a TSC */ 648 if (pd->pm_descr.pd_class == PMC_CLASS_TSC) 649 return 0; 650 651#ifdef DEBUG 652 KASSERT(pd->pm_descr.pd_class == amd_pmc_class, 653 ("[amd,%d] unknown PMC class (%d)", __LINE__, 654 pd->pm_descr.pd_class)); 655#endif 656 657 KASSERT(!AMD_PMC_IS_STOPPED(pd->pm_evsel), 658 ("[amd,%d] PMC%d, CPU%d \"%s\" already stopped", 659 __LINE__, ri, cpu, pd->pm_descr.pd_name)); 660 661 PMCDBG(MDP,STO,1,"amd-stop ri=%d", ri); 662 663 /* turn off the PMC ENABLE bit */ 664 config = pm->pm_md.pm_amd.pm_amd_evsel & ~AMD_PMC_ENABLE; 665 wrmsr(pd->pm_evsel, config); 666 return 0; 667} 668 669/* 670 * Interrupt handler. This function needs to return '1' if the 671 * interrupt was this CPU's PMCs or '0' otherwise. It is not allowed 672 * to sleep or do anything a 'fast' interrupt handler is not allowed 673 * to do. 674 */ 675 676static int 677amd_intr(int cpu, struct trapframe *tf) 678{ 679 int i, error, retval, ri; 680 uint32_t config, evsel, perfctr; 681 struct pmc *pm; 682 struct pmc_cpu *pc; 683 struct pmc_hw *phw; 684 pmc_value_t v; 685 686 KASSERT(cpu >= 0 && cpu < mp_ncpus, 687 ("[amd,%d] out of range CPU %d", __LINE__, cpu)); 688 689 PMCDBG(MDP,INT,1, "cpu=%d tf=%p um=%d", cpu, (void *) tf, 690 TRAPF_USERMODE(tf)); 691 692 retval = 0; 693 694 pc = pmc_pcpu[cpu]; 695 696 /* 697 * look for all PMCs that have interrupted: 698 * - skip over the TSC [PMC#0] 699 * - look for a running, sampling PMC which has overflowed 700 * and which has a valid 'struct pmc' association 701 * 702 * If found, we call a helper to process the interrupt. 703 * 704 * If multiple PMCs interrupt at the same time, the AMD64 705 * processor appears to deliver as many NMIs as there are 706 * outstanding PMC interrupts. So we process only one NMI 707 * interrupt at a time. 708 */ 709 710 for (i = 0; retval == 0 && i < AMD_NPMCS-1; i++) { 711 712 ri = i + 1; /* row index; TSC is at ri == 0 */ 713 714 if (!AMD_PMC_HAS_OVERFLOWED(i)) 715 continue; 716 717 phw = pc->pc_hwpmcs[ri]; 718 719 KASSERT(phw != NULL, ("[amd,%d] null PHW pointer", __LINE__)); 720 721 if ((pm = phw->phw_pmc) == NULL || 722 pm->pm_state != PMC_STATE_RUNNING || 723 !PMC_IS_SAMPLING_MODE(PMC_TO_MODE(pm))) { 724 continue; 725 } 726 727 retval = 1; /* Found an interrupting PMC. */ 728 729 /* Stop the PMC, reload count. */ 730 evsel = AMD_PMC_EVSEL_0 + i; 731 perfctr = AMD_PMC_PERFCTR_0 + i; 732 v = pm->pm_sc.pm_reloadcount; 733 config = rdmsr(evsel); 734 735 KASSERT((config & ~AMD_PMC_ENABLE) == 736 (pm->pm_md.pm_amd.pm_amd_evsel & ~AMD_PMC_ENABLE), 737 ("[amd,%d] config mismatch reg=0x%x pm=0x%x", __LINE__, 738 config, pm->pm_md.pm_amd.pm_amd_evsel)); 739 740 wrmsr(evsel, config & ~AMD_PMC_ENABLE); 741 wrmsr(perfctr, AMD_RELOAD_COUNT_TO_PERFCTR_VALUE(v)); 742 743 /* Restart the counter if logging succeeded. */ 744 error = pmc_process_interrupt(cpu, pm, tf, TRAPF_USERMODE(tf)); 745 if (error == 0) 746 wrmsr(evsel, config | AMD_PMC_ENABLE); 747 } 748 749 atomic_add_int(retval ? &pmc_stats.pm_intr_processed : 750 &pmc_stats.pm_intr_ignored, 1); 751 752 return (retval); 753} 754 755/* 756 * describe a PMC 757 */ 758static int 759amd_describe(int cpu, int ri, struct pmc_info *pi, struct pmc **ppmc) 760{ 761 int error; 762 size_t copied; 763 const struct amd_descr *pd; 764 struct pmc_hw *phw; 765 766 KASSERT(cpu >= 0 && cpu < mp_ncpus, 767 ("[amd,%d] illegal CPU %d", __LINE__, cpu)); 768 KASSERT(ri >= 0 && ri < AMD_NPMCS, 769 ("[amd,%d] row-index %d out of range", __LINE__, ri)); 770 771 phw = pmc_pcpu[cpu]->pc_hwpmcs[ri]; 772 pd = &amd_pmcdesc[ri]; 773 774 if ((error = copystr(pd->pm_descr.pd_name, pi->pm_name, 775 PMC_NAME_MAX, &copied)) != 0) 776 return error; 777 778 pi->pm_class = pd->pm_descr.pd_class; 779 780 if (phw->phw_state & PMC_PHW_FLAG_IS_ENABLED) { 781 pi->pm_enabled = TRUE; 782 *ppmc = phw->phw_pmc; 783 } else { 784 pi->pm_enabled = FALSE; 785 *ppmc = NULL; 786 } 787 788 return 0; 789} 790 791/* 792 * i386 specific entry points 793 */ 794 795/* 796 * return the MSR address of the given PMC. 797 */ 798 799static int 800amd_get_msr(int ri, uint32_t *msr) 801{ 802 KASSERT(ri >= 0 && ri < AMD_NPMCS, 803 ("[amd,%d] ri %d out of range", __LINE__, ri)); 804 805 *msr = amd_pmcdesc[ri].pm_perfctr - AMD_PMC_PERFCTR_0; 806 return 0; 807} 808 809/* 810 * processor dependent initialization. 811 */ 812 813/* 814 * Per-processor data structure 815 * 816 * [common stuff] 817 * [5 struct pmc_hw pointers] 818 * [5 struct pmc_hw structures] 819 */ 820 821struct amd_cpu { 822 struct pmc_cpu pc_common; 823 struct pmc_hw *pc_hwpmcs[AMD_NPMCS]; 824 struct pmc_hw pc_amdpmcs[AMD_NPMCS]; 825}; 826 827 828static int 829amd_init(int cpu) 830{ 831 int n; 832 struct amd_cpu *pcs; 833 struct pmc_hw *phw; 834 835 KASSERT(cpu >= 0 && cpu < mp_ncpus, 836 ("[amd,%d] insane cpu number %d", __LINE__, cpu)); 837 838 PMCDBG(MDP,INI,1,"amd-init cpu=%d", cpu); 839 840 MALLOC(pcs, struct amd_cpu *, sizeof(struct amd_cpu), M_PMC, 841 M_WAITOK|M_ZERO); 842 843 phw = &pcs->pc_amdpmcs[0]; 844 845 /* 846 * Initialize the per-cpu mutex and set the content of the 847 * hardware descriptors to a known state. 848 */ 849 850 for (n = 0; n < AMD_NPMCS; n++, phw++) { 851 phw->phw_state = PMC_PHW_FLAG_IS_ENABLED | 852 PMC_PHW_CPU_TO_STATE(cpu) | PMC_PHW_INDEX_TO_STATE(n); 853 phw->phw_pmc = NULL; 854 pcs->pc_hwpmcs[n] = phw; 855 } 856 857 /* Mark the TSC as shareable */ 858 pcs->pc_hwpmcs[0]->phw_state |= PMC_PHW_FLAG_IS_SHAREABLE; 859 860 pmc_pcpu[cpu] = (struct pmc_cpu *) pcs; 861 862 return 0; 863} 864 865 866/* 867 * processor dependent cleanup prior to the KLD 868 * being unloaded 869 */ 870 871static int 872amd_cleanup(int cpu) 873{ 874 int i; 875 uint32_t evsel; 876 struct pmc_cpu *pcs; 877 878 KASSERT(cpu >= 0 && cpu < mp_ncpus, 879 ("[amd,%d] insane cpu number (%d)", __LINE__, cpu)); 880 881 PMCDBG(MDP,INI,1,"amd-cleanup cpu=%d", cpu); 882 883 /* 884 * First, turn off all PMCs on this CPU. 885 */ 886 887 for (i = 0; i < 4; i++) { /* XXX this loop is now not needed */ 888 evsel = rdmsr(AMD_PMC_EVSEL_0 + i); 889 evsel &= ~AMD_PMC_ENABLE; 890 wrmsr(AMD_PMC_EVSEL_0 + i, evsel); 891 } 892 893 /* 894 * Next, free up allocated space. 895 */ 896 897 if ((pcs = pmc_pcpu[cpu]) == NULL) 898 return 0; 899 900#ifdef DEBUG 901 /* check the TSC */ 902 KASSERT(pcs->pc_hwpmcs[0]->phw_pmc == NULL, 903 ("[amd,%d] CPU%d,PMC0 still in use", __LINE__, cpu)); 904 for (i = 1; i < AMD_NPMCS; i++) { 905 KASSERT(pcs->pc_hwpmcs[i]->phw_pmc == NULL, 906 ("[amd,%d] CPU%d/PMC%d in use", __LINE__, cpu, i)); 907 KASSERT(AMD_PMC_IS_STOPPED(AMD_PMC_EVSEL_0 + (i-1)), 908 ("[amd,%d] CPU%d/PMC%d not stopped", __LINE__, cpu, i)); 909 } 910#endif 911 912 pmc_pcpu[cpu] = NULL; 913 FREE(pcs, M_PMC); 914 return 0; 915} 916 917/* 918 * Initialize ourselves. 919 */ 920 921struct pmc_mdep * 922pmc_amd_initialize(void) 923{ 924 enum pmc_cputype cputype; 925 enum pmc_class class; 926 struct pmc_mdep *pmc_mdep; 927 char *name; 928 int i; 929 930 /* 931 * The presence of hardware performance counters on the AMD 932 * Athlon, Duron or later processors, is _not_ indicated by 933 * any of the processor feature flags set by the 'CPUID' 934 * instruction, so we only check the 'instruction family' 935 * field returned by CPUID for instruction family >= 6. 936 */ 937 938 class = cputype = -1; 939 name = NULL; 940 switch (cpu_id & 0xF00) { 941 case 0x600: /* Athlon(tm) processor */ 942 cputype = PMC_CPU_AMD_K7; 943 class = PMC_CLASS_K7; 944 name = "K7"; 945 break; 946 case 0xF00: /* Athlon64/Opteron processor */ 947 cputype = PMC_CPU_AMD_K8; 948 class = PMC_CLASS_K8; 949 name = "K8"; 950 break; 951 } 952 953 if ((int) cputype == -1) { 954 (void) printf("pmc: Unknown AMD CPU.\n"); 955 return NULL; 956 } 957 958#ifdef DEBUG 959 amd_pmc_class = class; 960#endif 961 962 MALLOC(pmc_mdep, struct pmc_mdep *, sizeof(struct pmc_mdep), 963 M_PMC, M_WAITOK|M_ZERO); 964 965 pmc_mdep->pmd_cputype = cputype; 966 pmc_mdep->pmd_npmc = AMD_NPMCS; 967 968 /* this processor has two classes of usable PMCs */ 969 pmc_mdep->pmd_nclass = 2; 970 971 /* TSC */ 972 pmc_mdep->pmd_classes[0].pm_class = PMC_CLASS_TSC; 973 pmc_mdep->pmd_classes[0].pm_caps = PMC_CAP_READ; 974 pmc_mdep->pmd_classes[0].pm_width = 64; 975 976 /* AMD K7/K8 PMCs */ 977 pmc_mdep->pmd_classes[1].pm_class = class; 978 pmc_mdep->pmd_classes[1].pm_caps = AMD_PMC_CAPS; 979 pmc_mdep->pmd_classes[1].pm_width = 48; 980 981 pmc_mdep->pmd_nclasspmcs[0] = 1; 982 pmc_mdep->pmd_nclasspmcs[1] = (AMD_NPMCS-1); 983 984 /* fill in the correct pmc name and class */ 985 for (i = 1; i < AMD_NPMCS; i++) { 986 (void) snprintf(amd_pmcdesc[i].pm_descr.pd_name, 987 sizeof(amd_pmcdesc[i].pm_descr.pd_name), "%s-%d", 988 name, i-1); 989 amd_pmcdesc[i].pm_descr.pd_class = class; 990 } 991 992 pmc_mdep->pmd_init = amd_init; 993 pmc_mdep->pmd_cleanup = amd_cleanup; 994 pmc_mdep->pmd_switch_in = amd_switch_in; 995 pmc_mdep->pmd_switch_out = amd_switch_out; 996 pmc_mdep->pmd_read_pmc = amd_read_pmc; 997 pmc_mdep->pmd_write_pmc = amd_write_pmc; 998 pmc_mdep->pmd_config_pmc = amd_config_pmc; 999 pmc_mdep->pmd_get_config = amd_get_config; 1000 pmc_mdep->pmd_allocate_pmc = amd_allocate_pmc; 1001 pmc_mdep->pmd_release_pmc = amd_release_pmc; 1002 pmc_mdep->pmd_start_pmc = amd_start_pmc; 1003 pmc_mdep->pmd_stop_pmc = amd_stop_pmc; 1004 pmc_mdep->pmd_intr = amd_intr; 1005 pmc_mdep->pmd_describe = amd_describe; 1006 pmc_mdep->pmd_get_msr = amd_get_msr; /* i386 */ 1007 1008 PMCDBG(MDP,INI,0,"%s","amd-initialize"); 1009 1010 return pmc_mdep; 1011} 1012