hwpmc_amd.c revision 146799
1/*- 2 * Copyright (c) 2003-2005 Joseph Koshy 3 * All rights reserved. 4 * 5 * Redistribution and use in source and binary forms, with or without 6 * modification, are permitted provided that the following conditions 7 * are met: 8 * 1. Redistributions of source code must retain the above copyright 9 * notice, this list of conditions and the following disclaimer. 10 * 2. Redistributions in binary form must reproduce the above copyright 11 * notice, this list of conditions and the following disclaimer in the 12 * documentation and/or other materials provided with the distribution. 13 * 14 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 15 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 16 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 17 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 18 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 19 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 20 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 21 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 22 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 23 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 24 * SUCH DAMAGE. 25 * 26 */ 27 28#include <sys/cdefs.h> 29__FBSDID("$FreeBSD: head/sys/dev/hwpmc/hwpmc_amd.c 146799 2005-05-30 06:29:29Z jkoshy $"); 30 31/* Support for the AMD K7 and later processors */ 32 33#include <sys/param.h> 34#include <sys/lock.h> 35#include <sys/malloc.h> 36#include <sys/mutex.h> 37#include <sys/pmc.h> 38#include <sys/smp.h> 39#include <sys/systm.h> 40 41#include <machine/md_var.h> 42 43/* AMD K7 and K8 PMCs */ 44 45#define AMD_PMC_EVSEL_0 0xC0010000 46#define AMD_PMC_EVSEL_1 0xC0010001 47#define AMD_PMC_EVSEL_2 0xC0010002 48#define AMD_PMC_EVSEL_3 0xC0010003 49 50#define AMD_PMC_PERFCTR_0 0xC0010004 51#define AMD_PMC_PERFCTR_1 0xC0010005 52#define AMD_PMC_PERFCTR_2 0xC0010006 53#define AMD_PMC_PERFCTR_3 0xC0010007 54 55#define K7_VALID_EVENT_CODE(c) (((c) >= 0x40 && (c) <= 0x47) || \ 56 ((c) >= 0x80 && (c) <= 0x85) || ((c) >= 0xC0 && (c) <= 0xC7) || \ 57 ((c) >= 0xCD && (c) <= 0xCF)) 58 59#define AMD_PMC_CAPS (PMC_CAP_INTERRUPT | PMC_CAP_USER | \ 60 PMC_CAP_SYSTEM | PMC_CAP_EDGE | PMC_CAP_THRESHOLD | \ 61 PMC_CAP_READ | PMC_CAP_WRITE | PMC_CAP_INVERT | PMC_CAP_QUALIFIER) 62 63/* reserved bits include bit 21 and the top two bits of the unit mask */ 64#define K7_PMC_RESERVED ((1 << 21) | (3 << 13)) 65 66#define K8_PMC_RESERVED (1 << 21) 67 68#define AMD_PMC_IS_STOPPED(evsel) ((rdmsr((evsel)) & AMD_PMC_ENABLE) == 0) 69#define AMD_PMC_HAS_OVERFLOWED(pmc) ((rdpmc(pmc) & (1ULL << 47)) == 0) 70 71#if __i386__ 72#define AMD_NPMCS K7_NPMCS 73#define AMD_PMC_CLASS PMC_CLASS_K7 74#define AMD_PMC_COUNTERMASK K7_PMC_COUNTERMASK 75#define AMD_PMC_TO_COUNTER(x) K7_PMC_TO_COUNTER(x) 76#define AMD_PMC_INVERT K7_PMC_INVERT 77#define AMD_PMC_ENABLE K7_PMC_ENABLE 78#define AMD_PMC_INT K7_PMC_INT 79#define AMD_PMC_PC K7_PMC_PC 80#define AMD_PMC_EDGE K7_PMC_EDGE 81#define AMD_PMC_OS K7_PMC_OS 82#define AMD_PMC_USR K7_PMC_USR 83 84#define AMD_PMC_UNITMASK_M K7_PMC_UNITMASK_M 85#define AMD_PMC_UNITMASK_O K7_PMC_UNITMASK_O 86#define AMD_PMC_UNITMASK_E K7_PMC_UNITMASK_E 87#define AMD_PMC_UNITMASK_S K7_PMC_UNITMASK_S 88#define AMD_PMC_UNITMASK_I K7_PMC_UNITMASK_I 89 90#define AMD_PMC_UNITMASK K7_PMC_UNITMASK 91#define AMD_PMC_EVENTMASK K7_PMC_EVENTMASK 92#define AMD_PMC_TO_UNITMASK(x) K7_PMC_TO_UNITMASK(x) 93#define AMD_PMC_TO_EVENTMASK(x) K7_PMC_TO_EVENTMASK(x) 94#define AMD_VALID_BITS K7_VALID_BITS 95 96#define AMD_PMC_CLASS_NAME "K7-" 97 98#elif __amd64__ 99 100#define AMD_NPMCS K8_NPMCS 101#define AMD_PMC_CLASS PMC_CLASS_K8 102#define AMD_PMC_COUNTERMASK K8_PMC_COUNTERMASK 103#define AMD_PMC_TO_COUNTER(x) K8_PMC_TO_COUNTER(x) 104#define AMD_PMC_INVERT K8_PMC_INVERT 105#define AMD_PMC_ENABLE K8_PMC_ENABLE 106#define AMD_PMC_INT K8_PMC_INT 107#define AMD_PMC_PC K8_PMC_PC 108#define AMD_PMC_EDGE K8_PMC_EDGE 109#define AMD_PMC_OS K8_PMC_OS 110#define AMD_PMC_USR K8_PMC_USR 111 112#define AMD_PMC_UNITMASK_M K8_PMC_UNITMASK_M 113#define AMD_PMC_UNITMASK_O K8_PMC_UNITMASK_O 114#define AMD_PMC_UNITMASK_E K8_PMC_UNITMASK_E 115#define AMD_PMC_UNITMASK_S K8_PMC_UNITMASK_S 116#define AMD_PMC_UNITMASK_I K8_PMC_UNITMASK_I 117 118#define AMD_PMC_UNITMASK K8_PMC_UNITMASK 119#define AMD_PMC_EVENTMASK K8_PMC_EVENTMASK 120#define AMD_PMC_TO_UNITMASK(x) K8_PMC_TO_UNITMASK(x) 121#define AMD_PMC_TO_EVENTMASK(x) K8_PMC_TO_EVENTMASK(x) 122#define AMD_VALID_BITS K8_VALID_BITS 123 124#define AMD_PMC_CLASS_NAME "K8-" 125 126#else 127#error Unsupported architecture. 128#endif 129 130/* AMD K7 & K8 PMCs */ 131struct amd_descr { 132 struct pmc_descr pm_descr; /* "base class" */ 133 uint32_t pm_evsel; /* address of EVSEL register */ 134 uint32_t pm_perfctr; /* address of PERFCTR register */ 135}; 136 137static const struct amd_descr amd_pmcdesc[AMD_NPMCS] = 138{ 139 { 140 .pm_descr = 141 { 142 .pd_name = "TSC", 143 .pd_class = PMC_CLASS_TSC, 144 .pd_caps = PMC_CAP_READ, 145 .pd_width = 64 146 }, 147 .pm_evsel = MSR_TSC, 148 .pm_perfctr = 0 /* unused */ 149 }, 150 151 { 152 .pm_descr = 153 { 154 .pd_name = AMD_PMC_CLASS_NAME "0", 155 .pd_class = AMD_PMC_CLASS, 156 .pd_caps = AMD_PMC_CAPS, 157 .pd_width = 48 158 }, 159 .pm_evsel = AMD_PMC_EVSEL_0, 160 .pm_perfctr = AMD_PMC_PERFCTR_0 161 }, 162 { 163 .pm_descr = 164 { 165 .pd_name = AMD_PMC_CLASS_NAME "1", 166 .pd_class = AMD_PMC_CLASS, 167 .pd_caps = AMD_PMC_CAPS, 168 .pd_width = 48 169 }, 170 .pm_evsel = AMD_PMC_EVSEL_1, 171 .pm_perfctr = AMD_PMC_PERFCTR_1 172 }, 173 { 174 .pm_descr = 175 { 176 .pd_name = AMD_PMC_CLASS_NAME "2", 177 .pd_class = AMD_PMC_CLASS, 178 .pd_caps = AMD_PMC_CAPS, 179 .pd_width = 48 180 }, 181 .pm_evsel = AMD_PMC_EVSEL_2, 182 .pm_perfctr = AMD_PMC_PERFCTR_2 183 }, 184 { 185 .pm_descr = 186 { 187 .pd_name = AMD_PMC_CLASS_NAME "3", 188 .pd_class = AMD_PMC_CLASS, 189 .pd_caps = AMD_PMC_CAPS, 190 .pd_width = 48 191 }, 192 .pm_evsel = AMD_PMC_EVSEL_3, 193 .pm_perfctr = AMD_PMC_PERFCTR_3 194 } 195}; 196 197struct amd_event_code_map { 198 enum pmc_event pe_ev; /* enum value */ 199 uint8_t pe_code; /* encoded event mask */ 200 uint8_t pe_mask; /* bits allowed in unit mask */ 201}; 202 203const struct amd_event_code_map amd_event_codes[] = { 204#if __i386__ 205 { PMC_EV_K7_DC_ACCESSES, 0x40, 0 }, 206 { PMC_EV_K7_DC_MISSES, 0x41, 0 }, 207 { PMC_EV_K7_DC_REFILLS_FROM_L2, 0x42, K7_PMC_UNITMASK_MOESI }, 208 { PMC_EV_K7_DC_REFILLS_FROM_SYSTEM, 0x43, K7_PMC_UNITMASK_MOESI }, 209 { PMC_EV_K7_DC_WRITEBACKS, 0x44, K7_PMC_UNITMASK_MOESI }, 210 { PMC_EV_K7_L1_DTLB_MISS_AND_L2_DTLB_HITS, 0x45, 0 }, 211 { PMC_EV_K7_L1_AND_L2_DTLB_MISSES, 0x46, 0 }, 212 { PMC_EV_K7_MISALIGNED_REFERENCES, 0x47, 0 }, 213 214 { PMC_EV_K7_IC_FETCHES, 0x80, 0 }, 215 { PMC_EV_K7_IC_MISSES, 0x81, 0 }, 216 217 { PMC_EV_K7_L1_ITLB_MISSES, 0x84, 0 }, 218 { PMC_EV_K7_L1_L2_ITLB_MISSES, 0x85, 0 }, 219 220 { PMC_EV_K7_RETIRED_INSTRUCTIONS, 0xC0, 0 }, 221 { PMC_EV_K7_RETIRED_OPS, 0xC1, 0 }, 222 { PMC_EV_K7_RETIRED_BRANCHES, 0xC2, 0 }, 223 { PMC_EV_K7_RETIRED_BRANCHES_MISPREDICTED, 0xC3, 0 }, 224 { PMC_EV_K7_RETIRED_TAKEN_BRANCHES, 0xC4, 0 }, 225 { PMC_EV_K7_RETIRED_TAKEN_BRANCHES_MISPREDICTED, 0xC5, 0 }, 226 { PMC_EV_K7_RETIRED_FAR_CONTROL_TRANSFERS, 0xC6, 0 }, 227 { PMC_EV_K7_RETIRED_RESYNC_BRANCHES, 0xC7, 0 }, 228 { PMC_EV_K7_INTERRUPTS_MASKED_CYCLES, 0xCD, 0 }, 229 { PMC_EV_K7_INTERRUPTS_MASKED_WHILE_PENDING_CYCLES, 0xCE, 0 }, 230 { PMC_EV_K7_HARDWARE_INTERRUPTS, 0xCF, 0 } 231#endif 232 233#if __amd64__ 234 { PMC_EV_K8_FP_DISPATCHED_FPU_OPS, 0x00, 0x3F }, 235 { PMC_EV_K8_FP_CYCLES_WITH_NO_FPU_OPS_RETIRED, 0x01, 0x00 }, 236 { PMC_EV_K8_FP_DISPATCHED_FPU_FAST_FLAG_OPS, 0x02, 0x00 }, 237 238 { PMC_EV_K8_LS_SEGMENT_REGISTER_LOAD, 0x20, 0x7F }, 239 { PMC_EV_K8_LS_MICROARCHITECTURAL_RESYNC_BY_SELF_MODIFYING_CODE, 240 0x21, 0x00 }, 241 { PMC_EV_K8_LS_MICROARCHITECTURAL_RESYNC_BY_SNOOP, 0x22, 0x00 }, 242 { PMC_EV_K8_LS_BUFFER2_FULL, 0x23, 0x00 }, 243 { PMC_EV_K8_LS_LOCKED_OPERATION, 0x24, 0x07 }, 244 { PMC_EV_K8_LS_MICROARCHITECTURAL_LATE_CANCEL, 0x25, 0x00 }, 245 { PMC_EV_K8_LS_RETIRED_CFLUSH_INSTRUCTIONS, 0x26, 0x00 }, 246 { PMC_EV_K8_LS_RETIRED_CPUID_INSTRUCTIONS, 0x27, 0x00 }, 247 248 { PMC_EV_K8_DC_ACCESS, 0x40, 0x00 }, 249 { PMC_EV_K8_DC_MISS, 0x41, 0x00 }, 250 { PMC_EV_K8_DC_REFILL_FROM_L2, 0x42, 0x1F }, 251 { PMC_EV_K8_DC_REFILL_FROM_SYSTEM, 0x43, 0x1F }, 252 { PMC_EV_K8_DC_COPYBACK, 0x44, 0x1F }, 253 { PMC_EV_K8_DC_L1_DTLB_MISS_AND_L2_DTLB_HIT, 0x45, 0x00 }, 254 { PMC_EV_K8_DC_L1_DTLB_MISS_AND_L2_DTLB_MISS, 0x46, 0x00 }, 255 { PMC_EV_K8_DC_MISALIGNED_DATA_REFERENCE, 0x47, 0x00 }, 256 { PMC_EV_K8_DC_MICROARCHITECTURAL_LATE_CANCEL, 0x48, 0x00 }, 257 { PMC_EV_K8_DC_MICROARCHITECTURAL_EARLY_CANCEL, 0x49, 0x00 }, 258 { PMC_EV_K8_DC_ONE_BIT_ECC_ERROR, 0x4A, 0x03 }, 259 { PMC_EV_K8_DC_DISPATCHED_PREFETCH_INSTRUCTIONS, 0x4B, 0x07 }, 260 { PMC_EV_K8_DC_DCACHE_ACCESSES_BY_LOCKS, 0x4C, 0x03 }, 261 262 { PMC_EV_K8_BU_CPU_CLK_UNHALTED, 0x76, 0x00 }, 263 { PMC_EV_K8_BU_INTERNAL_L2_REQUEST, 0x7D, 0x1F }, 264 { PMC_EV_K8_BU_FILL_REQUEST_L2_MISS, 0x7E, 0x07 }, 265 { PMC_EV_K8_BU_FILL_INTO_L2, 0x7F, 0x03 }, 266 267 { PMC_EV_K8_IC_FETCH, 0x80, 0x00 }, 268 { PMC_EV_K8_IC_MISS, 0x81, 0x00 }, 269 { PMC_EV_K8_IC_REFILL_FROM_L2, 0x82, 0x00 }, 270 { PMC_EV_K8_IC_REFILL_FROM_SYSTEM, 0x83, 0x00 }, 271 { PMC_EV_K8_IC_L1_ITLB_MISS_AND_L2_ITLB_HIT, 0x84, 0x00 }, 272 { PMC_EV_K8_IC_L1_ITLB_MISS_AND_L2_ITLB_MISS, 0x85, 0x00 }, 273 { PMC_EV_K8_IC_MICROARCHITECTURAL_RESYNC_BY_SNOOP, 0x86, 0x00 }, 274 { PMC_EV_K8_IC_INSTRUCTION_FETCH_STALL, 0x87, 0x00 }, 275 { PMC_EV_K8_IC_RETURN_STACK_HIT, 0x88, 0x00 }, 276 { PMC_EV_K8_IC_RETURN_STACK_OVERFLOW, 0x89, 0x00 }, 277 278 { PMC_EV_K8_FR_RETIRED_X86_INSTRUCTIONS, 0xC0, 0x00 }, 279 { PMC_EV_K8_FR_RETIRED_UOPS, 0xC1, 0x00 }, 280 { PMC_EV_K8_FR_RETIRED_BRANCHES, 0xC2, 0x00 }, 281 { PMC_EV_K8_FR_RETIRED_BRANCHES_MISPREDICTED, 0xC3, 0x00 }, 282 { PMC_EV_K8_FR_RETIRED_TAKEN_BRANCHES, 0xC4, 0x00 }, 283 { PMC_EV_K8_FR_RETIRED_TAKEN_BRANCHES_MISPREDICTED, 0xC5, 0x00 }, 284 { PMC_EV_K8_FR_RETIRED_FAR_CONTROL_TRANSFERS, 0xC6, 0x00 }, 285 { PMC_EV_K8_FR_RETIRED_RESYNCS, 0xC7, 0x00 }, 286 { PMC_EV_K8_FR_RETIRED_NEAR_RETURNS, 0xC8, 0x00 }, 287 { PMC_EV_K8_FR_RETIRED_NEAR_RETURNS_MISPREDICTED, 0xC9, 0x00 }, 288 { PMC_EV_K8_FR_RETIRED_TAKEN_BRANCHES_MISPREDICTED_BY_ADDR_MISCOMPARE, 289 0xCA, 0x00 }, 290 { PMC_EV_K8_FR_RETIRED_FPU_INSTRUCTIONS, 0xCB, 0x0F }, 291 { PMC_EV_K8_FR_RETIRED_FASTPATH_DOUBLE_OP_INSTRUCTIONS, 292 0xCC, 0x07 }, 293 { PMC_EV_K8_FR_INTERRUPTS_MASKED_CYCLES, 0xCD, 0x00 }, 294 { PMC_EV_K8_FR_INTERRUPTS_MASKED_WHILE_PENDING_CYCLES, 0xCE, 0x00 }, 295 { PMC_EV_K8_FR_TAKEN_HARDWARE_INTERRUPTS, 0xCF, 0x00 }, 296 297 { PMC_EV_K8_FR_DECODER_EMPTY, 0xD0, 0x00 }, 298 { PMC_EV_K8_FR_DISPATCH_STALLS, 0xD1, 0x00 }, 299 { PMC_EV_K8_FR_DISPATCH_STALL_FROM_BRANCH_ABORT_TO_RETIRE, 300 0xD2, 0x00 }, 301 { PMC_EV_K8_FR_DISPATCH_STALL_FOR_SERIALIZATION, 0xD3, 0x00 }, 302 { PMC_EV_K8_FR_DISPATCH_STALL_FOR_SEGMENT_LOAD, 0xD4, 0x00 }, 303 { PMC_EV_K8_FR_DISPATCH_STALL_WHEN_REORDER_BUFFER_IS_FULL, 304 0xD5, 0x00 }, 305 { PMC_EV_K8_FR_DISPATCH_STALL_WHEN_RESERVATION_STATIONS_ARE_FULL, 306 0xD6, 0x00 }, 307 { PMC_EV_K8_FR_DISPATCH_STALL_WHEN_FPU_IS_FULL, 0xD7, 0x00 }, 308 { PMC_EV_K8_FR_DISPATCH_STALL_WHEN_LS_IS_FULL, 0xD8, 0x00 }, 309 { PMC_EV_K8_FR_DISPATCH_STALL_WHEN_WAITING_FOR_ALL_TO_BE_QUIET, 310 0xD9, 0x00 }, 311 { PMC_EV_K8_FR_DISPATCH_STALL_WHEN_FAR_XFER_OR_RESYNC_BRANCH_PENDING, 312 0xDA, 0x00 }, 313 { PMC_EV_K8_FR_FPU_EXCEPTIONS, 0xDB, 0x0F }, 314 { PMC_EV_K8_FR_NUMBER_OF_BREAKPOINTS_FOR_DR0, 0xDC, 0x00 }, 315 { PMC_EV_K8_FR_NUMBER_OF_BREAKPOINTS_FOR_DR1, 0xDD, 0x00 }, 316 { PMC_EV_K8_FR_NUMBER_OF_BREAKPOINTS_FOR_DR2, 0xDE, 0x00 }, 317 { PMC_EV_K8_FR_NUMBER_OF_BREAKPOINTS_FOR_DR3, 0xDF, 0x00 }, 318 319 { PMC_EV_K8_NB_MEMORY_CONTROLLER_PAGE_ACCESS_EVENT, 0xE0, 0x7 }, 320 { PMC_EV_K8_NB_MEMORY_CONTROLLER_PAGE_TABLE_OVERFLOW, 0xE1, 0x00 }, 321 { PMC_EV_K8_NB_MEMORY_CONTROLLER_DRAM_COMMAND_SLOTS_MISSED, 322 0xE2, 0x00 }, 323 { PMC_EV_K8_NB_MEMORY_CONTROLLER_TURNAROUND, 0xE3, 0x07 }, 324 { PMC_EV_K8_NB_MEMORY_CONTROLLER_BYPASS_SATURATION, 0xE4, 0x0F }, 325 { PMC_EV_K8_NB_SIZED_COMMANDS, 0xEB, 0x7F }, 326 { PMC_EV_K8_NB_PROBE_RESULT, 0xEC, 0x0F }, 327 { PMC_EV_K8_NB_HT_BUS0_BANDWIDTH, 0xF6, 0x0F }, 328 { PMC_EV_K8_NB_HT_BUS1_BANDWIDTH, 0xF7, 0x0F }, 329 { PMC_EV_K8_NB_HT_BUS2_BANDWIDTH, 0xF8, 0x0F } 330#endif 331 332}; 333 334const int amd_event_codes_size = 335 sizeof(amd_event_codes) / sizeof(amd_event_codes[0]); 336 337/* 338 * read a pmc register 339 */ 340 341static int 342amd_read_pmc(int cpu, int ri, pmc_value_t *v) 343{ 344 enum pmc_mode mode; 345 const struct amd_descr *pd; 346 struct pmc *pm; 347 const struct pmc_hw *phw; 348 pmc_value_t tmp; 349 350 KASSERT(cpu >= 0 && cpu < mp_ncpus, 351 ("[amd,%d] illegal CPU value %d", __LINE__, cpu)); 352 KASSERT(ri >= 0 && ri < AMD_NPMCS, 353 ("[amd,%d] illegal row-index %d", __LINE__, ri)); 354 355 phw = pmc_pcpu[cpu]->pc_hwpmcs[ri]; 356 pd = &amd_pmcdesc[ri]; 357 pm = phw->phw_pmc; 358 359 KASSERT(pm != NULL, 360 ("[amd,%d] No owner for HWPMC [cpu%d,pmc%d]", __LINE__, 361 cpu, ri)); 362 363 mode = PMC_TO_MODE(pm); 364 365 PMCDBG(MDP,REA,1,"amd-read id=%d class=%d", ri, pd->pm_descr.pd_class); 366 367 /* Reading the TSC is a special case */ 368 if (pd->pm_descr.pd_class == PMC_CLASS_TSC) { 369 KASSERT(PMC_IS_COUNTING_MODE(mode), 370 ("[amd,%d] TSC counter in non-counting mode", __LINE__)); 371 *v = rdtsc(); 372 PMCDBG(MDP,REA,2,"amd-read id=%d -> %jd", ri, *v); 373 return 0; 374 } 375 376 KASSERT(pd->pm_descr.pd_class == AMD_PMC_CLASS, 377 ("[amd,%d] unknown PMC class (%d)", __LINE__, 378 pd->pm_descr.pd_class)); 379 380 tmp = rdmsr(pd->pm_perfctr); /* RDMSR serializes */ 381 if (PMC_IS_SAMPLING_MODE(mode)) 382 *v = -tmp; 383 else 384 *v = tmp; 385 386 PMCDBG(MDP,REA,2,"amd-read id=%d -> %jd", ri, *v); 387 388 return 0; 389} 390 391/* 392 * Write a PMC MSR. 393 */ 394 395static int 396amd_write_pmc(int cpu, int ri, pmc_value_t v) 397{ 398 const struct amd_descr *pd; 399 struct pmc *pm; 400 const struct pmc_hw *phw; 401 enum pmc_mode mode; 402 403 KASSERT(cpu >= 0 && cpu < mp_ncpus, 404 ("[amd,%d] illegal CPU value %d", __LINE__, cpu)); 405 KASSERT(ri >= 0 && ri < AMD_NPMCS, 406 ("[amd,%d] illegal row-index %d", __LINE__, ri)); 407 408 phw = pmc_pcpu[cpu]->pc_hwpmcs[ri]; 409 pd = &amd_pmcdesc[ri]; 410 pm = phw->phw_pmc; 411 412 KASSERT(pm != NULL, 413 ("[amd,%d] PMC not owned (cpu%d,pmc%d)", __LINE__, 414 cpu, ri)); 415 416 mode = PMC_TO_MODE(pm); 417 418 if (pd->pm_descr.pd_class == PMC_CLASS_TSC) 419 return 0; 420 421 KASSERT(pd->pm_descr.pd_class == AMD_PMC_CLASS, 422 ("[amd,%d] unknown PMC class (%d)", __LINE__, 423 pd->pm_descr.pd_class)); 424 425 /* use 2's complement of the count for sampling mode PMCs */ 426 if (PMC_IS_SAMPLING_MODE(mode)) 427 v = -v; 428 429 PMCDBG(MDP,WRI,1,"amd-write cpu=%d ri=%d v=%jx", cpu, ri, v); 430 431 /* write the PMC value */ 432 wrmsr(pd->pm_perfctr, v); 433 return 0; 434} 435 436/* 437 * configure hardware pmc according to the configuration recorded in 438 * pmc 'pm'. 439 */ 440 441static int 442amd_config_pmc(int cpu, int ri, struct pmc *pm) 443{ 444 struct pmc_hw *phw; 445 446 PMCDBG(MDP,CFG,1, "cpu=%d ri=%d pm=%p", cpu, ri, pm); 447 448 KASSERT(cpu >= 0 && cpu < mp_ncpus, 449 ("[amd,%d] illegal CPU value %d", __LINE__, cpu)); 450 KASSERT(ri >= 0 && ri < AMD_NPMCS, 451 ("[amd,%d] illegal row-index %d", __LINE__, ri)); 452 453 phw = pmc_pcpu[cpu]->pc_hwpmcs[ri]; 454 455 KASSERT(pm == NULL || phw->phw_pmc == NULL, 456 ("[amd,%d] pm=%p phw->pm=%p hwpmc not unconfigured", 457 __LINE__, pm, phw->phw_pmc)); 458 459 phw->phw_pmc = pm; 460 return 0; 461} 462 463/* 464 * Retrieve a configured PMC pointer from hardware state. 465 */ 466 467static int 468amd_get_config(int cpu, int ri, struct pmc **ppm) 469{ 470 *ppm = pmc_pcpu[cpu]->pc_hwpmcs[ri]->phw_pmc; 471 472 return 0; 473} 474 475/* 476 * Machine dependent actions taken during the context switch in of a 477 * thread. 478 */ 479 480static int 481amd_switch_in(struct pmc_cpu *pc, struct pmc_process *pp) 482{ 483 (void) pc; 484 485 PMCDBG(MDP,SWI,1, "pc=%p pp=%p enable-msr=%d", pc, pp, 486 (pp->pp_flags & PMC_PP_ENABLE_MSR_ACCESS) != 0); 487 488 /* enable the RDPMC instruction if needed */ 489 if (pp->pp_flags & PMC_PP_ENABLE_MSR_ACCESS) 490 load_cr4(rcr4() | CR4_PCE); 491 492 return 0; 493} 494 495/* 496 * Machine dependent actions taken during the context switch out of a 497 * thread. 498 */ 499 500static int 501amd_switch_out(struct pmc_cpu *pc, struct pmc_process *pp) 502{ 503 (void) pc; 504 (void) pp; /* can be NULL */ 505 506 PMCDBG(MDP,SWO,1, "pc=%p pp=%p enable-msr=%d", pc, pp, pp ? 507 (pp->pp_flags & PMC_PP_ENABLE_MSR_ACCESS) == 1 : 0); 508 509 /* always turn off the RDPMC instruction */ 510 load_cr4(rcr4() & ~CR4_PCE); 511 512 return 0; 513} 514 515/* 516 * Check if a given allocation is feasible. 517 */ 518 519static int 520amd_allocate_pmc(int cpu, int ri, struct pmc *pm, 521 const struct pmc_op_pmcallocate *a) 522{ 523 int i; 524 uint32_t allowed_unitmask, caps, config, unitmask; 525 enum pmc_event pe; 526 const struct pmc_descr *pd; 527 528 (void) cpu; 529 530 KASSERT(cpu >= 0 && cpu < mp_ncpus, 531 ("[amd,%d] illegal CPU value %d", __LINE__, cpu)); 532 KASSERT(ri >= 0 && ri < AMD_NPMCS, 533 ("[amd,%d] illegal row index %d", __LINE__, ri)); 534 535 pd = &amd_pmcdesc[ri].pm_descr; 536 537 /* check class match */ 538 if (pd->pd_class != a->pm_class) 539 return EINVAL; 540 541 caps = pm->pm_caps; 542 543 PMCDBG(MDP,ALL,1,"amd-allocate ri=%d caps=0x%x", ri, caps); 544 545 if ((pd->pd_caps & caps) != caps) 546 return EPERM; 547 if (pd->pd_class == PMC_CLASS_TSC) { 548 /* TSC's are always allocated in system-wide counting mode */ 549 if (a->pm_ev != PMC_EV_TSC_TSC || 550 a->pm_mode != PMC_MODE_SC) 551 return EINVAL; 552 return 0; 553 } 554 555 KASSERT(pd->pd_class == AMD_PMC_CLASS, 556 ("[amd,%d] Unknown PMC class (%d)", __LINE__, pd->pd_class)); 557 558 pe = a->pm_ev; 559 560 /* map ev to the correct event mask code */ 561 config = allowed_unitmask = 0; 562 for (i = 0; i < amd_event_codes_size; i++) 563 if (amd_event_codes[i].pe_ev == pe) { 564 config = 565 AMD_PMC_TO_EVENTMASK(amd_event_codes[i].pe_code); 566 allowed_unitmask = 567 AMD_PMC_TO_UNITMASK(amd_event_codes[i].pe_mask); 568 break; 569 } 570 if (i == amd_event_codes_size) 571 return EINVAL; 572 573 unitmask = a->pm_amd_config & AMD_PMC_UNITMASK; 574 if (unitmask & ~allowed_unitmask) /* disallow reserved bits */ 575 return EINVAL; 576 577 if (unitmask && (caps & PMC_CAP_QUALIFIER)) 578 config |= unitmask; 579 580 if (caps & PMC_CAP_THRESHOLD) 581 config |= a->pm_amd_config & AMD_PMC_COUNTERMASK; 582 583 /* set at least one of the 'usr' or 'os' caps */ 584 if (caps & PMC_CAP_USER) 585 config |= AMD_PMC_USR; 586 if (caps & PMC_CAP_SYSTEM) 587 config |= AMD_PMC_OS; 588 if ((caps & (PMC_CAP_USER|PMC_CAP_SYSTEM)) == 0) 589 config |= (AMD_PMC_USR|AMD_PMC_OS); 590 591 if (caps & PMC_CAP_EDGE) 592 config |= AMD_PMC_EDGE; 593 if (caps & PMC_CAP_INVERT) 594 config |= AMD_PMC_INVERT; 595 if (caps & PMC_CAP_INTERRUPT) 596 config |= AMD_PMC_INT; 597 598 pm->pm_md.pm_amd.pm_amd_evsel = config; /* save config value */ 599 600 PMCDBG(MDP,ALL,2,"amd-allocate ri=%d -> config=0x%x", ri, config); 601 602 return 0; 603} 604 605/* 606 * Release machine dependent state associated with a PMC. This is a 607 * no-op on this architecture. 608 * 609 */ 610 611/* ARGSUSED0 */ 612static int 613amd_release_pmc(int cpu, int ri, struct pmc *pmc) 614{ 615#if DEBUG 616 const struct amd_descr *pd; 617#endif 618 struct pmc_hw *phw; 619 620 (void) pmc; 621 622 KASSERT(cpu >= 0 && cpu < mp_ncpus, 623 ("[amd,%d] illegal CPU value %d", __LINE__, cpu)); 624 KASSERT(ri >= 0 && ri < AMD_NPMCS, 625 ("[amd,%d] illegal row-index %d", __LINE__, ri)); 626 627 phw = pmc_pcpu[cpu]->pc_hwpmcs[ri]; 628 629 KASSERT(phw->phw_pmc == NULL, 630 ("[amd,%d] PHW pmc %p non-NULL", __LINE__, phw->phw_pmc)); 631 632#if DEBUG 633 pd = &amd_pmcdesc[ri]; 634 if (pd->pm_descr.pd_class == AMD_PMC_CLASS) 635 KASSERT(AMD_PMC_IS_STOPPED(pd->pm_evsel), 636 ("[amd,%d] PMC %d released while active", __LINE__, ri)); 637#endif 638 639 return 0; 640} 641 642/* 643 * start a PMC. 644 */ 645 646static int 647amd_start_pmc(int cpu, int ri) 648{ 649 uint32_t config; 650 struct pmc *pm; 651 struct pmc_hw *phw; 652 const struct amd_descr *pd; 653 654 KASSERT(cpu >= 0 && cpu < mp_ncpus, 655 ("[amd,%d] illegal CPU value %d", __LINE__, cpu)); 656 KASSERT(ri >= 0 && ri < AMD_NPMCS, 657 ("[amd,%d] illegal row-index %d", __LINE__, ri)); 658 659 phw = pmc_pcpu[cpu]->pc_hwpmcs[ri]; 660 pm = phw->phw_pmc; 661 pd = &amd_pmcdesc[ri]; 662 663 KASSERT(pm != NULL, 664 ("[amd,%d] starting cpu%d,pmc%d with null pmc record", __LINE__, 665 cpu, ri)); 666 667 PMCDBG(MDP,STA,1,"amd-start cpu=%d ri=%d", cpu, ri); 668 669 if (pd->pm_descr.pd_class == PMC_CLASS_TSC) 670 return 0; /* TSCs are always running */ 671 672 KASSERT(pd->pm_descr.pd_class == AMD_PMC_CLASS, 673 ("[amd,%d] unknown PMC class (%d)", __LINE__, 674 pd->pm_descr.pd_class)); 675 676 KASSERT(AMD_PMC_IS_STOPPED(pd->pm_evsel), 677 ("[amd,%d] pmc%d,cpu%d: Starting active PMC \"%s\"", __LINE__, 678 ri, cpu, pd->pm_descr.pd_name)); 679 680 /* turn on the PMC ENABLE bit */ 681 config = pm->pm_md.pm_amd.pm_amd_evsel | AMD_PMC_ENABLE; 682 683 PMCDBG(MDP,STA,2,"amd-start config=0x%x", config); 684 685 wrmsr(pd->pm_evsel, config); 686 return 0; 687} 688 689/* 690 * Stop a PMC. 691 */ 692 693static int 694amd_stop_pmc(int cpu, int ri) 695{ 696 struct pmc *pm; 697 struct pmc_hw *phw; 698 const struct amd_descr *pd; 699 uint64_t config; 700 701 KASSERT(cpu >= 0 && cpu < mp_ncpus, 702 ("[amd,%d] illegal CPU value %d", __LINE__, cpu)); 703 KASSERT(ri >= 0 && ri < AMD_NPMCS, 704 ("[amd,%d] illegal row-index %d", __LINE__, ri)); 705 706 phw = pmc_pcpu[cpu]->pc_hwpmcs[ri]; 707 pm = phw->phw_pmc; 708 pd = &amd_pmcdesc[ri]; 709 710 KASSERT(pm != NULL, 711 ("[amd,%d] cpu%d,pmc%d no PMC to stop", __LINE__, 712 cpu, ri)); 713 714 /* can't stop a TSC */ 715 if (pd->pm_descr.pd_class == PMC_CLASS_TSC) 716 return 0; 717 718 KASSERT(pd->pm_descr.pd_class == AMD_PMC_CLASS, 719 ("[amd,%d] unknown PMC class (%d)", __LINE__, 720 pd->pm_descr.pd_class)); 721 722 KASSERT(!AMD_PMC_IS_STOPPED(pd->pm_evsel), 723 ("[amd,%d] PMC%d, CPU%d \"%s\" already stopped", 724 __LINE__, ri, cpu, pd->pm_descr.pd_name)); 725 726 PMCDBG(MDP,STO,1,"amd-stop ri=%d", ri); 727 728 /* turn off the PMC ENABLE bit */ 729 config = pm->pm_md.pm_amd.pm_amd_evsel & ~AMD_PMC_ENABLE; 730 wrmsr(pd->pm_evsel, config); 731 return 0; 732} 733 734/* 735 * Interrupt handler. This function needs to return '1' if the 736 * interrupt was this CPU's PMCs or '0' otherwise. It is not allowed 737 * to sleep or do anything a 'fast' interrupt handler is not allowed 738 * to do. 739 */ 740 741static int 742amd_intr(int cpu, uintptr_t eip, int usermode) 743{ 744 int i, retval; 745 enum pmc_mode mode; 746 uint32_t perfctr; 747 struct pmc *pm; 748 struct pmc_cpu *pc; 749 struct pmc_hw *phw; 750 751 (void) usermode; 752 753 KASSERT(cpu >= 0 && cpu < mp_ncpus, 754 ("[amd,%d] out of range CPU %d", __LINE__, cpu)); 755 756 retval = 0; 757 758 pc = pmc_pcpu[cpu]; 759 760 /* 761 * look for all PMCs that have interrupted: 762 * - skip over the TSC [PMC#0] 763 * - look for a PMC with a valid 'struct pmc' association 764 * - look for a PMC in (a) sampling mode and (b) which has 765 * overflowed. If found, we update the process's 766 * histogram or send it a profiling signal by calling 767 * the appropriate helper function. 768 */ 769 770 for (i = 1; i < AMD_NPMCS; i++) { 771 772 phw = pc->pc_hwpmcs[i]; 773 perfctr = amd_pmcdesc[i].pm_perfctr; 774 KASSERT(phw != NULL, ("[amd,%d] null PHW pointer", __LINE__)); 775 776 if ((pm = phw->phw_pmc) == NULL || 777 pm->pm_state != PMC_STATE_RUNNING) { 778 atomic_add_int(&pmc_stats.pm_intr_ignored, 1); 779 continue; 780 } 781 782 mode = PMC_TO_MODE(pm); 783 if (PMC_IS_SAMPLING_MODE(mode) && 784 AMD_PMC_HAS_OVERFLOWED(perfctr)) { 785 atomic_add_int(&pmc_stats.pm_intr_processed, 1); 786 if (PMC_IS_SYSTEM_MODE(mode)) 787 pmc_update_histogram(phw, eip); 788 else if (PMC_IS_VIRTUAL_MODE(mode)) 789 pmc_send_signal(pm); 790 retval = 1; 791 } 792 } 793 return retval; 794} 795 796/* 797 * describe a PMC 798 */ 799static int 800amd_describe(int cpu, int ri, struct pmc_info *pi, struct pmc **ppmc) 801{ 802 int error; 803 size_t copied; 804 const struct amd_descr *pd; 805 struct pmc_hw *phw; 806 807 KASSERT(cpu >= 0 && cpu < mp_ncpus, 808 ("[amd,%d] illegal CPU %d", __LINE__, cpu)); 809 KASSERT(ri >= 0 && ri < AMD_NPMCS, 810 ("[amd,%d] row-index %d out of range", __LINE__, ri)); 811 812 phw = pmc_pcpu[cpu]->pc_hwpmcs[ri]; 813 pd = &amd_pmcdesc[ri]; 814 815 if ((error = copystr(pd->pm_descr.pd_name, pi->pm_name, 816 PMC_NAME_MAX, &copied)) != 0) 817 return error; 818 819 pi->pm_class = pd->pm_descr.pd_class; 820 821 if (phw->phw_state & PMC_PHW_FLAG_IS_ENABLED) { 822 pi->pm_enabled = TRUE; 823 *ppmc = phw->phw_pmc; 824 } else { 825 pi->pm_enabled = FALSE; 826 *ppmc = NULL; 827 } 828 829 return 0; 830} 831 832/* 833 * i386 specific entry points 834 */ 835 836/* 837 * return the MSR address of the given PMC. 838 */ 839 840static int 841amd_get_msr(int ri, uint32_t *msr) 842{ 843 KASSERT(ri >= 0 && ri < AMD_NPMCS, 844 ("[amd,%d] ri %d out of range", __LINE__, ri)); 845 846 *msr = amd_pmcdesc[ri].pm_perfctr - AMD_PMC_PERFCTR_0; 847 return 0; 848} 849 850/* 851 * processor dependent initialization. 852 */ 853 854/* 855 * Per-processor data structure 856 * 857 * [common stuff] 858 * [5 struct pmc_hw pointers] 859 * [5 struct pmc_hw structures] 860 */ 861 862struct amd_cpu { 863 struct pmc_cpu pc_common; 864 struct pmc_hw *pc_hwpmcs[AMD_NPMCS]; 865 struct pmc_hw pc_amdpmcs[AMD_NPMCS]; 866}; 867 868 869static int 870amd_init(int cpu) 871{ 872 int n; 873 struct amd_cpu *pcs; 874 struct pmc_hw *phw; 875 876 KASSERT(cpu >= 0 && cpu < mp_ncpus, 877 ("[amd,%d] insane cpu number %d", __LINE__, cpu)); 878 879 PMCDBG(MDP,INI,1,"amd-init cpu=%d", cpu); 880 881 MALLOC(pcs, struct amd_cpu *, sizeof(struct amd_cpu), M_PMC, 882 M_WAITOK|M_ZERO); 883 884 if (pcs == NULL) 885 return ENOMEM; 886 887 phw = &pcs->pc_amdpmcs[0]; 888 889 /* 890 * Initialize the per-cpu mutex and set the content of the 891 * hardware descriptors to a known state. 892 */ 893 894 for (n = 0; n < AMD_NPMCS; n++, phw++) { 895 phw->phw_state = PMC_PHW_FLAG_IS_ENABLED | 896 PMC_PHW_CPU_TO_STATE(cpu) | PMC_PHW_INDEX_TO_STATE(n); 897 phw->phw_pmc = NULL; 898 pcs->pc_hwpmcs[n] = phw; 899 } 900 901 /* Mark the TSC as shareable */ 902 pcs->pc_hwpmcs[0]->phw_state |= PMC_PHW_FLAG_IS_SHAREABLE; 903 904 pmc_pcpu[cpu] = (struct pmc_cpu *) pcs; 905 906 return 0; 907} 908 909 910/* 911 * processor dependent cleanup prior to the KLD 912 * being unloaded 913 */ 914 915static int 916amd_cleanup(int cpu) 917{ 918 int i; 919 uint32_t evsel; 920 struct pmc_cpu *pcs; 921 922 KASSERT(cpu >= 0 && cpu < mp_ncpus, 923 ("[amd,%d] insane cpu number (%d)", __LINE__, cpu)); 924 925 PMCDBG(MDP,INI,1,"amd-cleanup cpu=%d", cpu); 926 927 /* 928 * First, turn off all PMCs on this CPU. 929 */ 930 931 for (i = 0; i < 4; i++) { /* XXX this loop is now not needed */ 932 evsel = rdmsr(AMD_PMC_EVSEL_0 + i); 933 evsel &= ~AMD_PMC_ENABLE; 934 wrmsr(AMD_PMC_EVSEL_0 + i, evsel); 935 } 936 937 /* 938 * Next, free up allocated space. 939 */ 940 941 pcs = pmc_pcpu[cpu]; 942 943#if DEBUG 944 /* check the TSC */ 945 KASSERT(pcs->pc_hwpmcs[0]->phw_pmc == NULL, 946 ("[amd,%d] CPU%d,PMC0 still in use", __LINE__, cpu)); 947 for (i = 1; i < AMD_NPMCS; i++) { 948 KASSERT(pcs->pc_hwpmcs[i]->phw_pmc == NULL, 949 ("[amd,%d] CPU%d/PMC%d in use", __LINE__, cpu, i)); 950 KASSERT(AMD_PMC_IS_STOPPED(AMD_PMC_EVSEL_0 + (i-1)), 951 ("[amd,%d] CPU%d/PMC%d not stopped", __LINE__, cpu, i)); 952 } 953#endif 954 KASSERT(pcs != NULL, 955 ("[amd,%d] null per-cpu state pointer (cpu%d)", __LINE__, cpu)); 956 957 pmc_pcpu[cpu] = NULL; 958 FREE(pcs, M_PMC); 959 return 0; 960} 961 962/* 963 * Initialize ourselves. 964 */ 965 966struct pmc_mdep * 967pmc_amd_initialize(void) 968{ 969 970 struct pmc_mdep *pmc_mdep; 971 972 /* The presence of hardware performance counters on the AMD 973 Athlon, Duron or later processors, is _not_ indicated by 974 any of the processor feature flags set by the 'CPUID' 975 instruction, so we only check the 'instruction family' 976 field returned by CPUID for instruction family >= 6. This 977 test needs to be be refined. */ 978 979 if ((cpu_id & 0xF00) < 0x600) 980 return NULL; 981 982 MALLOC(pmc_mdep, struct pmc_mdep *, sizeof(struct pmc_mdep), 983 M_PMC, M_WAITOK|M_ZERO); 984 985#if __i386__ 986 pmc_mdep->pmd_cputype = PMC_CPU_AMD_K7; 987#elif __amd64__ 988 pmc_mdep->pmd_cputype = PMC_CPU_AMD_K8; 989#else 990#error Unknown AMD CPU type. 991#endif 992 993 pmc_mdep->pmd_npmc = AMD_NPMCS; 994 995 /* this processor has two classes of usable PMCs */ 996 pmc_mdep->pmd_nclass = 2; 997 998 /* TSC */ 999 pmc_mdep->pmd_classes[0].pm_class = PMC_CLASS_TSC; 1000 pmc_mdep->pmd_classes[0].pm_caps = PMC_CAP_READ; 1001 pmc_mdep->pmd_classes[0].pm_width = 64; 1002 1003 /* AMD K7/K8 PMCs */ 1004 pmc_mdep->pmd_classes[1].pm_class = AMD_PMC_CLASS; 1005 pmc_mdep->pmd_classes[1].pm_caps = AMD_PMC_CAPS; 1006 pmc_mdep->pmd_classes[1].pm_width = 48; 1007 1008 pmc_mdep->pmd_nclasspmcs[0] = 1; 1009 pmc_mdep->pmd_nclasspmcs[1] = (AMD_NPMCS-1); 1010 1011 pmc_mdep->pmd_init = amd_init; 1012 pmc_mdep->pmd_cleanup = amd_cleanup; 1013 pmc_mdep->pmd_switch_in = amd_switch_in; 1014 pmc_mdep->pmd_switch_out = amd_switch_out; 1015 pmc_mdep->pmd_read_pmc = amd_read_pmc; 1016 pmc_mdep->pmd_write_pmc = amd_write_pmc; 1017 pmc_mdep->pmd_config_pmc = amd_config_pmc; 1018 pmc_mdep->pmd_get_config = amd_get_config; 1019 pmc_mdep->pmd_allocate_pmc = amd_allocate_pmc; 1020 pmc_mdep->pmd_release_pmc = amd_release_pmc; 1021 pmc_mdep->pmd_start_pmc = amd_start_pmc; 1022 pmc_mdep->pmd_stop_pmc = amd_stop_pmc; 1023 pmc_mdep->pmd_intr = amd_intr; 1024 pmc_mdep->pmd_describe = amd_describe; 1025 pmc_mdep->pmd_get_msr = amd_get_msr; /* i386 */ 1026 1027 PMCDBG(MDP,INI,0,"%s","amd-initialize"); 1028 1029 return pmc_mdep; 1030} 1031