hwpmc_amd.c revision 145581
1/*- 2 * Copyright (c) 2003-2005 Joseph Koshy 3 * All rights reserved. 4 * 5 * Redistribution and use in source and binary forms, with or without 6 * modification, are permitted provided that the following conditions 7 * are met: 8 * 1. Redistributions of source code must retain the above copyright 9 * notice, this list of conditions and the following disclaimer. 10 * 2. Redistributions in binary form must reproduce the above copyright 11 * notice, this list of conditions and the following disclaimer in the 12 * documentation and/or other materials provided with the distribution. 13 * 14 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 15 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 16 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 17 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 18 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 19 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 20 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 21 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 22 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 23 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 24 * SUCH DAMAGE. 25 * 26 */ 27 28#include <sys/cdefs.h> 29__FBSDID("$FreeBSD: head/sys/dev/hwpmc/hwpmc_amd.c 145338 2005-04-20 20:26:39Z marcel $"); 30 31/* Support for the AMD K7 and later processors */ 32 33#include <sys/param.h> 34#include <sys/lock.h> 35#include <sys/malloc.h> 36#include <sys/mutex.h> 37#include <sys/pmc.h> 38#include <sys/smp.h> 39#include <sys/systm.h> 40 41#include <machine/md_var.h> 42 43/* AMD K7 and K8 PMCs */ 44 45#define AMD_PMC_EVSEL_0 0xC0010000 46#define AMD_PMC_EVSEL_1 0xC0010001 47#define AMD_PMC_EVSEL_2 0xC0010002 48#define AMD_PMC_EVSEL_3 0xC0010003 49 50#define AMD_PMC_PERFCTR_0 0xC0010004 51#define AMD_PMC_PERFCTR_1 0xC0010005 52#define AMD_PMC_PERFCTR_2 0xC0010006 53#define AMD_PMC_PERFCTR_3 0xC0010007 54 55#define K7_VALID_EVENT_CODE(c) (((c) >= 0x40 && (c) <= 0x47) || \ 56 ((c) >= 0x80 && (c) <= 0x85) || ((c) >= 0xC0 && (c) <= 0xC7) || \ 57 ((c) >= 0xCD && (c) <= 0xCF)) 58 59#define AMD_PMC_CAPS (PMC_CAP_INTERRUPT | PMC_CAP_USER | \ 60 PMC_CAP_SYSTEM | PMC_CAP_EDGE | PMC_CAP_THRESHOLD | \ 61 PMC_CAP_READ | PMC_CAP_WRITE | PMC_CAP_INVERT | PMC_CAP_QUALIFIER) 62 63/* reserved bits include bit 21 and the top two bits of the unit mask */ 64#define K7_PMC_RESERVED ((1 << 21) | (3 << 13)) 65 66#define K8_PMC_RESERVED (1 << 21) 67 68#define AMD_PMC_IS_STOPPED(evsel) ((rdmsr((evsel)) & AMD_PMC_ENABLE) == 0) 69#define AMD_PMC_HAS_OVERFLOWED(pmc) ((rdpmc(pmc) & (1ULL << 47)) == 0) 70 71#if __i386__ 72#define AMD_NPMCS K7_NPMCS 73#define AMD_PMC_CLASS PMC_CLASS_K7 74#define AMD_PMC_COUNTERMASK K7_PMC_COUNTERMASK 75#define AMD_PMC_TO_COUNTER(x) K7_PMC_TO_COUNTER(x) 76#define AMD_PMC_INVERT K7_PMC_INVERT 77#define AMD_PMC_ENABLE K7_PMC_ENABLE 78#define AMD_PMC_INT K7_PMC_INT 79#define AMD_PMC_PC K7_PMC_PC 80#define AMD_PMC_EDGE K7_PMC_EDGE 81#define AMD_PMC_OS K7_PMC_OS 82#define AMD_PMC_USR K7_PMC_USR 83 84#define AMD_PMC_UNITMASK_M K7_PMC_UNITMASK_M 85#define AMD_PMC_UNITMASK_O K7_PMC_UNITMASK_O 86#define AMD_PMC_UNITMASK_E K7_PMC_UNITMASK_E 87#define AMD_PMC_UNITMASK_S K7_PMC_UNITMASK_S 88#define AMD_PMC_UNITMASK_I K7_PMC_UNITMASK_I 89 90#define AMD_PMC_UNITMASK K7_PMC_UNITMASK 91#define AMD_PMC_EVENTMASK K7_PMC_EVENTMASK 92#define AMD_PMC_TO_UNITMASK(x) K7_PMC_TO_UNITMASK(x) 93#define AMD_PMC_TO_EVENTMASK(x) K7_PMC_TO_EVENTMASK(x) 94#define AMD_VALID_BITS K7_VALID_BITS 95 96#define AMD_PMC_CLASS_NAME "K7-" 97 98#elif __amd64__ 99 100#define AMD_NPMCS K8_NPMCS 101#define AMD_PMC_CLASS PMC_CLASS_K8 102#define AMD_PMC_COUNTERMASK K8_PMC_COUNTERMASK 103#define AMD_PMC_TO_COUNTER(x) K8_PMC_TO_COUNTER(x) 104#define AMD_PMC_INVERT K8_PMC_INVERT 105#define AMD_PMC_ENABLE K8_PMC_ENABLE 106#define AMD_PMC_INT K8_PMC_INT 107#define AMD_PMC_PC K8_PMC_PC 108#define AMD_PMC_EDGE K8_PMC_EDGE 109#define AMD_PMC_OS K8_PMC_OS 110#define AMD_PMC_USR K8_PMC_USR 111 112#define AMD_PMC_UNITMASK_M K8_PMC_UNITMASK_M 113#define AMD_PMC_UNITMASK_O K8_PMC_UNITMASK_O 114#define AMD_PMC_UNITMASK_E K8_PMC_UNITMASK_E 115#define AMD_PMC_UNITMASK_S K8_PMC_UNITMASK_S 116#define AMD_PMC_UNITMASK_I K8_PMC_UNITMASK_I 117 118#define AMD_PMC_UNITMASK K8_PMC_UNITMASK 119#define AMD_PMC_EVENTMASK K8_PMC_EVENTMASK 120#define AMD_PMC_TO_UNITMASK(x) K8_PMC_TO_UNITMASK(x) 121#define AMD_PMC_TO_EVENTMASK(x) K8_PMC_TO_EVENTMASK(x) 122#define AMD_VALID_BITS K8_VALID_BITS 123 124#define AMD_PMC_CLASS_NAME "K8-" 125 126#else 127#error Unsupported architecture. 128#endif 129 130/* AMD K7 & K8 PMCs */ 131struct amd_descr { 132 struct pmc_descr pm_descr; /* "base class" */ 133 uint32_t pm_evsel; /* address of EVSEL register */ 134 uint32_t pm_perfctr; /* address of PERFCTR register */ 135}; 136 137static const struct amd_descr amd_pmcdesc[AMD_NPMCS] = 138{ 139 { 140 .pm_descr = 141 { 142 .pd_name = "TSC", 143 .pd_class = PMC_CLASS_TSC, 144 .pd_caps = PMC_CAP_READ, 145 .pd_width = 64 146 }, 147 .pm_evsel = MSR_TSC, 148 .pm_perfctr = 0 /* unused */ 149 }, 150 151 { 152 .pm_descr = 153 { 154 .pd_name = AMD_PMC_CLASS_NAME "0", 155 .pd_class = AMD_PMC_CLASS, 156 .pd_caps = AMD_PMC_CAPS, 157 .pd_width = 48 158 }, 159 .pm_evsel = AMD_PMC_EVSEL_0, 160 .pm_perfctr = AMD_PMC_PERFCTR_0 161 }, 162 { 163 .pm_descr = 164 { 165 .pd_name = AMD_PMC_CLASS_NAME "1", 166 .pd_class = AMD_PMC_CLASS, 167 .pd_caps = AMD_PMC_CAPS, 168 .pd_width = 48 169 }, 170 .pm_evsel = AMD_PMC_EVSEL_1, 171 .pm_perfctr = AMD_PMC_PERFCTR_1 172 }, 173 { 174 .pm_descr = 175 { 176 .pd_name = AMD_PMC_CLASS_NAME "2", 177 .pd_class = AMD_PMC_CLASS, 178 .pd_caps = AMD_PMC_CAPS, 179 .pd_width = 48 180 }, 181 .pm_evsel = AMD_PMC_EVSEL_2, 182 .pm_perfctr = AMD_PMC_PERFCTR_2 183 }, 184 { 185 .pm_descr = 186 { 187 .pd_name = AMD_PMC_CLASS_NAME "3", 188 .pd_class = AMD_PMC_CLASS, 189 .pd_caps = AMD_PMC_CAPS, 190 .pd_width = 48 191 }, 192 .pm_evsel = AMD_PMC_EVSEL_3, 193 .pm_perfctr = AMD_PMC_PERFCTR_3 194 } 195}; 196 197struct amd_event_code_map { 198 enum pmc_event pe_ev; /* enum value */ 199 uint8_t pe_code; /* encoded event mask */ 200 uint8_t pe_mask; /* bits allowed in unit mask */ 201}; 202 203const struct amd_event_code_map amd_event_codes[] = { 204#if __i386__ 205 { PMC_EV_K7_DC_ACCESSES, 0x40, 0 }, 206 { PMC_EV_K7_DC_MISSES, 0x41, 0 }, 207 { PMC_EV_K7_DC_REFILLS_FROM_L2, 0x42, K7_PMC_UNITMASK_MOESI }, 208 { PMC_EV_K7_DC_REFILLS_FROM_SYSTEM, 0x43, K7_PMC_UNITMASK_MOESI }, 209 { PMC_EV_K7_DC_WRITEBACKS, 0x44, K7_PMC_UNITMASK_MOESI }, 210 { PMC_EV_K7_L1_DTLB_MISS_AND_L2_DTLB_HITS, 0x45, 0 }, 211 { PMC_EV_K7_L1_AND_L2_DTLB_MISSES, 0x46, 0 }, 212 { PMC_EV_K7_MISALIGNED_REFERENCES, 0x47, 0 }, 213 214 { PMC_EV_K7_IC_FETCHES, 0x80, 0 }, 215 { PMC_EV_K7_IC_MISSES, 0x81, 0 }, 216 217 { PMC_EV_K7_L1_ITLB_MISSES, 0x84, 0 }, 218 { PMC_EV_K7_L1_L2_ITLB_MISSES, 0x85, 0 }, 219 220 { PMC_EV_K7_RETIRED_INSTRUCTIONS, 0xC0, 0 }, 221 { PMC_EV_K7_RETIRED_OPS, 0xC1, 0 }, 222 { PMC_EV_K7_RETIRED_BRANCHES, 0xC2, 0 }, 223 { PMC_EV_K7_RETIRED_BRANCHES_MISPREDICTED, 0xC3, 0 }, 224 { PMC_EV_K7_RETIRED_TAKEN_BRANCHES, 0xC4, 0 }, 225 { PMC_EV_K7_RETIRED_TAKEN_BRANCHES_MISPREDICTED, 0xC5, 0 }, 226 { PMC_EV_K7_RETIRED_FAR_CONTROL_TRANSFERS, 0xC6, 0 }, 227 { PMC_EV_K7_RETIRED_RESYNC_BRANCHES, 0xC7, 0 }, 228 { PMC_EV_K7_INTERRUPTS_MASKED_CYCLES, 0xCD, 0 }, 229 { PMC_EV_K7_INTERRUPTS_MASKED_WHILE_PENDING_CYCLES, 0xCE, 0 }, 230 { PMC_EV_K7_HARDWARE_INTERRUPTS, 0xCF, 0 } 231#endif 232 233#if __amd64__ 234 { PMC_EV_K8_FP_DISPATCHED_FPU_OPS, 0x00, 0x3F }, 235 { PMC_EV_K8_FP_CYCLES_WITH_NO_FPU_OPS_RETIRED, 0x01, 0x00 }, 236 { PMC_EV_K8_FP_DISPATCHED_FPU_FAST_FLAG_OPS, 0x02, 0x00 }, 237 238 { PMC_EV_K8_LS_SEGMENT_REGISTER_LOAD, 0x20, 0x7F }, 239 { PMC_EV_K8_LS_MICROARCHITECTURAL_RESYNC_BY_SELF_MODIFYING_CODE, 240 0x21, 0x00 }, 241 { PMC_EV_K8_LS_MICROARCHITECTURAL_RESYNC_BY_SNOOP, 0x22, 0x00 }, 242 { PMC_EV_K8_LS_BUFFER2_FULL, 0x23, 0x00 }, 243 { PMC_EV_K8_LS_LOCKED_OPERATION, 0x24, 0x07 }, 244 { PMC_EV_K8_LS_MICROARCHITECTURAL_LATE_CANCEL, 0x25, 0x00 }, 245 { PMC_EV_K8_LS_RETIRED_CFLUSH_INSTRUCTIONS, 0x26, 0x00 }, 246 { PMC_EV_K8_LS_RETIRED_CPUID_INSTRUCTIONS, 0x27, 0x00 }, 247 248 { PMC_EV_K8_DC_ACCESS, 0x40, 0x00 }, 249 { PMC_EV_K8_DC_MISS, 0x41, 0x00 }, 250 { PMC_EV_K8_DC_REFILL_FROM_L2, 0x42, 0x1F }, 251 { PMC_EV_K8_DC_REFILL_FROM_SYSTEM, 0x43, 0x1F }, 252 { PMC_EV_K8_DC_COPYBACK, 0x44, 0x1F }, 253 { PMC_EV_K8_DC_L1_DTLB_MISS_AND_L2_DTLB_HIT, 0x45, 0x00 }, 254 { PMC_EV_K8_DC_L1_DTLB_MISS_AND_L2_DTLB_MISS, 0x46, 0x00 }, 255 { PMC_EV_K8_DC_MISALIGNED_DATA_REFERENCE, 0x47, 0x00 }, 256 { PMC_EV_K8_DC_MICROARCHITECTURAL_LATE_CANCEL, 0x48, 0x00 }, 257 { PMC_EV_K8_DC_MICROARCHITECTURAL_EARLY_CANCEL, 0x49, 0x00 }, 258 { PMC_EV_K8_DC_ONE_BIT_ECC_ERROR, 0x4A, 0x03 }, 259 { PMC_EV_K8_DC_DISPATCHED_PREFETCH_INSTRUCTIONS, 0x4B, 0x07 }, 260 { PMC_EV_K8_DC_DCACHE_ACCESSES_BY_LOCKS, 0x4C, 0x03 }, 261 262 { PMC_EV_K8_BU_CPU_CLK_UNHALTED, 0x76, 0x00 }, 263 { PMC_EV_K8_BU_INTERNAL_L2_REQUEST, 0x7D, 0x1F }, 264 { PMC_EV_K8_BU_FILL_REQUEST_L2_MISS, 0x7E, 0x07 }, 265 { PMC_EV_K8_BU_FILL_INTO_L2, 0x7F, 0x03 }, 266 267 { PMC_EV_K8_IC_FETCH, 0x80, 0x00 }, 268 { PMC_EV_K8_IC_MISS, 0x81, 0x00 }, 269 { PMC_EV_K8_IC_REFILL_FROM_L2, 0x82, 0x00 }, 270 { PMC_EV_K8_IC_REFILL_FROM_SYSTEM, 0x83, 0x00 }, 271 { PMC_EV_K8_IC_L1_ITLB_MISS_AND_L2_ITLB_HIT, 0x84, 0x00 }, 272 { PMC_EV_K8_IC_L1_ITLB_MISS_AND_L2_ITLB_MISS, 0x85, 0x00 }, 273 { PMC_EV_K8_IC_MICROARCHITECTURAL_RESYNC_BY_SNOOP, 0x86, 0x00 }, 274 { PMC_EV_K8_IC_INSTRUCTION_FETCH_STALL, 0x87, 0x00 }, 275 { PMC_EV_K8_IC_RETURN_STACK_HIT, 0x88, 0x00 }, 276 { PMC_EV_K8_IC_RETURN_STACK_OVERFLOW, 0x89, 0x00 }, 277 278 { PMC_EV_K8_FR_RETIRED_X86_INSTRUCTIONS, 0xC0, 0x00 }, 279 { PMC_EV_K8_FR_RETIRED_UOPS, 0xC1, 0x00 }, 280 { PMC_EV_K8_FR_RETIRED_BRANCHES, 0xC2, 0x00 }, 281 { PMC_EV_K8_FR_RETIRED_BRANCHES_MISPREDICTED, 0xC3, 0x00 }, 282 { PMC_EV_K8_FR_RETIRED_TAKEN_BRANCHES, 0xC4, 0x00 }, 283 { PMC_EV_K8_FR_RETIRED_TAKEN_BRANCHES_MISPREDICTED, 0xC5, 0x00 }, 284 { PMC_EV_K8_FR_RETIRED_FAR_CONTROL_TRANSFERS, 0xC6, 0x00 }, 285 { PMC_EV_K8_FR_RETIRED_RESYNCS, 0xC7, 0x00 }, 286 { PMC_EV_K8_FR_RETIRED_NEAR_RETURNS, 0xC8, 0x00 }, 287 { PMC_EV_K8_FR_RETIRED_NEAR_RETURNS_MISPREDICTED, 0xC9, 0x00 }, 288 { PMC_EV_K8_FR_RETIRED_TAKEN_BRANCHES_MISPREDICTED_BY_ADDR_MISCOMPARE, 289 0xCA, 0x00 }, 290 { PMC_EV_K8_FR_RETIRED_FPU_INSTRUCTIONS, 0xCB, 0x0F }, 291 { PMC_EV_K8_FR_RETIRED_FASTPATH_DOUBLE_OP_INSTRUCTIONS, 292 0xCC, 0x07 }, 293 { PMC_EV_K8_FR_INTERRUPTS_MASKED_CYCLES, 0xCD, 0x00 }, 294 { PMC_EV_K8_FR_INTERRUPTS_MASKED_WHILE_PENDING_CYCLES, 0xCE, 0x00 }, 295 { PMC_EV_K8_FR_TAKEN_HARDWARE_INTERRUPTS, 0xCF, 0x00 }, 296 297 { PMC_EV_K8_FR_DECODER_EMPTY, 0xD0, 0x00 }, 298 { PMC_EV_K8_FR_DISPATCH_STALLS, 0xD1, 0x00 }, 299 { PMC_EV_K8_FR_DISPATCH_STALL_FROM_BRANCH_ABORT_TO_RETIRE, 300 0xD2, 0x00 }, 301 { PMC_EV_K8_FR_DISPATCH_STALL_FOR_SERIALIZATION, 0xD3, 0x00 }, 302 { PMC_EV_K8_FR_DISPATCH_STALL_FOR_SEGMENT_LOAD, 0xD4, 0x00 }, 303 { PMC_EV_K8_FR_DISPATCH_STALL_WHEN_REORDER_BUFFER_IS_FULL, 304 0xD5, 0x00 }, 305 { PMC_EV_K8_FR_DISPATCH_STALL_WHEN_RESERVATION_STATIONS_ARE_FULL, 306 0xD6, 0x00 }, 307 { PMC_EV_K8_FR_DISPATCH_STALL_WHEN_FPU_IS_FULL, 0xD7, 0x00 }, 308 { PMC_EV_K8_FR_DISPATCH_STALL_WHEN_LS_IS_FULL, 0xD8, 0x00 }, 309 { PMC_EV_K8_FR_DISPATCH_STALL_WHEN_WAITING_FOR_ALL_TO_BE_QUIET, 310 0xD9, 0x00 }, 311 { PMC_EV_K8_FR_DISPATCH_STALL_WHEN_FAR_XFER_OR_RESYNC_BRANCH_PENDING, 312 0xDA, 0x00 }, 313 { PMC_EV_K8_FR_FPU_EXCEPTIONS, 0xDB, 0x0F }, 314 { PMC_EV_K8_FR_NUMBER_OF_BREAKPOINTS_FOR_DR0, 0xDC, 0x00 }, 315 { PMC_EV_K8_FR_NUMBER_OF_BREAKPOINTS_FOR_DR1, 0xDD, 0x00 }, 316 { PMC_EV_K8_FR_NUMBER_OF_BREAKPOINTS_FOR_DR2, 0xDE, 0x00 }, 317 { PMC_EV_K8_FR_NUMBER_OF_BREAKPOINTS_FOR_DR3, 0xDF, 0x00 }, 318 319 { PMC_EV_K8_NB_MEMORY_CONTROLLER_PAGE_ACCESS_EVENT, 0xE0, 0x7 }, 320 { PMC_EV_K8_NB_MEMORY_CONTROLLER_PAGE_TABLE_OVERFLOW, 0xE1, 0x00 }, 321 { PMC_EV_K8_NB_MEMORY_CONTROLLER_DRAM_COMMAND_SLOTS_MISSED, 322 0xE2, 0x00 }, 323 { PMC_EV_K8_NB_MEMORY_CONTROLLER_TURNAROUND, 0xE3, 0x07 }, 324 { PMC_EV_K8_NB_MEMORY_CONTROLLER_BYPASS_SATURATION, 0xE4, 0x0F }, 325 { PMC_EV_K8_NB_SIZED_COMMANDS, 0xEB, 0x7F }, 326 { PMC_EV_K8_NB_PROBE_RESULT, 0xEC, 0x0F }, 327 { PMC_EV_K8_NB_HT_BUS0_BANDWIDTH, 0xF6, 0x0F }, 328 { PMC_EV_K8_NB_HT_BUS1_BANDWIDTH, 0xF7, 0x0F }, 329 { PMC_EV_K8_NB_HT_BUS2_BANDWIDTH, 0xF8, 0x0F } 330#endif 331 332}; 333 334const int amd_event_codes_size = 335 sizeof(amd_event_codes) / sizeof(amd_event_codes[0]); 336 337/* 338 * read a pmc register 339 */ 340 341static int 342amd_read_pmc(int cpu, int ri, pmc_value_t *v) 343{ 344 enum pmc_mode mode; 345 const struct amd_descr *pd; 346 struct pmc *pm; 347 const struct pmc_hw *phw; 348 pmc_value_t tmp; 349 350 KASSERT(cpu >= 0 && cpu < mp_ncpus, 351 ("[amd,%d] illegal CPU value %d", __LINE__, cpu)); 352 KASSERT(ri >= 0 && ri < AMD_NPMCS, 353 ("[amd,%d] illegal row-index %d", __LINE__, ri)); 354 355 phw = pmc_pcpu[cpu]->pc_hwpmcs[ri]; 356 pd = &amd_pmcdesc[ri]; 357 pm = phw->phw_pmc; 358 359 KASSERT(pm != NULL, 360 ("[amd,%d] No owner for HWPMC [cpu%d,pmc%d]", __LINE__, 361 cpu, ri)); 362 363 mode = pm->pm_mode; 364 365 PMCDBG(MDP,REA,1,"amd-read id=%d class=%d", ri, pd->pm_descr.pd_class); 366 367 /* Reading the TSC is a special case */ 368 if (pd->pm_descr.pd_class == PMC_CLASS_TSC) { 369 KASSERT(PMC_IS_COUNTING_MODE(mode), 370 ("[amd,%d] TSC counter in non-counting mode", __LINE__)); 371 *v = rdtsc(); 372 PMCDBG(MDP,REA,2,"amd-read id=%d -> %jd", ri, *v); 373 return 0; 374 } 375 376 KASSERT(pd->pm_descr.pd_class == AMD_PMC_CLASS, 377 ("[amd,%d] unknown PMC class (%d)", __LINE__, 378 pd->pm_descr.pd_class)); 379 380 tmp = rdmsr(pd->pm_perfctr); /* RDMSR serializes */ 381 if (PMC_IS_SAMPLING_MODE(mode)) 382 *v = -tmp; 383 else 384 *v = tmp; 385 386 PMCDBG(MDP,REA,2,"amd-read id=%d -> %jd", ri, *v); 387 388 return 0; 389} 390 391/* 392 * Write a PMC MSR. 393 */ 394 395static int 396amd_write_pmc(int cpu, int ri, pmc_value_t v) 397{ 398 const struct amd_descr *pd; 399 struct pmc *pm; 400 const struct pmc_hw *phw; 401 enum pmc_mode mode; 402 403 KASSERT(cpu >= 0 && cpu < mp_ncpus, 404 ("[amd,%d] illegal CPU value %d", __LINE__, cpu)); 405 KASSERT(ri >= 0 && ri < AMD_NPMCS, 406 ("[amd,%d] illegal row-index %d", __LINE__, ri)); 407 408 phw = pmc_pcpu[cpu]->pc_hwpmcs[ri]; 409 pd = &amd_pmcdesc[ri]; 410 pm = phw->phw_pmc; 411 412 KASSERT(pm != NULL, 413 ("[amd,%d] PMC not owned (cpu%d,pmc%d)", __LINE__, 414 cpu, ri)); 415 416 mode = pm->pm_mode; 417 418 if (pd->pm_descr.pd_class == PMC_CLASS_TSC) 419 return 0; 420 421 KASSERT(pd->pm_descr.pd_class == AMD_PMC_CLASS, 422 ("[amd,%d] unknown PMC class (%d)", __LINE__, 423 pd->pm_descr.pd_class)); 424 425 /* use 2's complement of the count for sampling mode PMCs */ 426 if (PMC_IS_SAMPLING_MODE(mode)) 427 v = -v; 428 429 PMCDBG(MDP,WRI,1,"amd-write cpu=%d ri=%d v=%jx", cpu, ri, v); 430 431 /* write the PMC value */ 432 wrmsr(pd->pm_perfctr, v); 433 return 0; 434} 435 436/* 437 * configure hardware pmc according to the configuration recorded in 438 * pmc 'pm'. 439 */ 440 441static int 442amd_config_pmc(int cpu, int ri, struct pmc *pm) 443{ 444 struct pmc_hw *phw; 445 446 KASSERT(cpu >= 0 && cpu < mp_ncpus, 447 ("[amd,%d] illegal CPU value %d", __LINE__, cpu)); 448 KASSERT(ri >= 0 && ri < AMD_NPMCS, 449 ("[amd,%d] illegal row-index %d", __LINE__, ri)); 450 451 phw = pmc_pcpu[cpu]->pc_hwpmcs[ri]; 452 453 KASSERT(pm == NULL || phw->phw_pmc == NULL, 454 ("[amd,%d] hwpmc not unconfigured before re-config", __LINE__)); 455 456 phw->phw_pmc = pm; 457 return 0; 458} 459 460/* 461 * Machine dependent actions taken during the context switch in of a 462 * thread. 463 */ 464 465static int 466amd_switch_in(struct pmc_cpu *pc) 467{ 468 (void) pc; 469 470 /* enable the RDPMC instruction */ 471 load_cr4(rcr4() | CR4_PCE); 472 return 0; 473} 474 475/* 476 * Machine dependent actions taken during the context switch out of a 477 * thread. 478 */ 479 480static int 481amd_switch_out(struct pmc_cpu *pc) 482{ 483 (void) pc; 484 485 /* disallow RDPMC instruction */ 486 load_cr4(rcr4() & ~CR4_PCE); 487 return 0; 488} 489 490/* 491 * Check if a given allocation is feasible. 492 */ 493 494static int 495amd_allocate_pmc(int cpu, int ri, struct pmc *pm, 496 const struct pmc_op_pmcallocate *a) 497{ 498 int i; 499 uint32_t allowed_unitmask, caps, config, unitmask; 500 enum pmc_event pe; 501 const struct pmc_descr *pd; 502 503 (void) cpu; 504 505 KASSERT(cpu >= 0 && cpu < mp_ncpus, 506 ("[amd,%d] illegal CPU value %d", __LINE__, cpu)); 507 KASSERT(ri >= 0 && ri < AMD_NPMCS, 508 ("[amd,%d] illegal row index %d", __LINE__, ri)); 509 510 pd = &amd_pmcdesc[ri].pm_descr; 511 512 /* check class match */ 513 if (pd->pd_class != pm->pm_class) 514 return EINVAL; 515 516 caps = pm->pm_caps; 517 518 PMCDBG(MDP,ALL,1,"amd-allocate ri=%d caps=0x%x", ri, caps); 519 520 if ((pd->pd_caps & caps) != caps) 521 return EPERM; 522 if (pd->pd_class == PMC_CLASS_TSC) { 523 /* TSC's are always allocated in system-wide counting mode */ 524 if (a->pm_ev != PMC_EV_TSC_TSC || 525 a->pm_mode != PMC_MODE_SC) 526 return EINVAL; 527 return 0; 528 } 529 530 KASSERT(pd->pd_class == AMD_PMC_CLASS, 531 ("[amd,%d] Unknown PMC class (%d)", __LINE__, pd->pd_class)); 532 533 pe = a->pm_ev; 534 535 /* map ev to the correct event mask code */ 536 config = allowed_unitmask = 0; 537 for (i = 0; i < amd_event_codes_size; i++) 538 if (amd_event_codes[i].pe_ev == pe) { 539 config = 540 AMD_PMC_TO_EVENTMASK(amd_event_codes[i].pe_code); 541 allowed_unitmask = 542 AMD_PMC_TO_UNITMASK(amd_event_codes[i].pe_mask); 543 break; 544 } 545 if (i == amd_event_codes_size) 546 return EINVAL; 547 548 unitmask = a->pm_amd_config & AMD_PMC_UNITMASK; 549 if (unitmask & ~allowed_unitmask) /* disallow reserved bits */ 550 return EINVAL; 551 552 if (unitmask && (caps & PMC_CAP_QUALIFIER)) 553 config |= unitmask; 554 555 if (caps & PMC_CAP_THRESHOLD) 556 config |= a->pm_amd_config & AMD_PMC_COUNTERMASK; 557 558 /* set at least one of the 'usr' or 'os' caps */ 559 if (caps & PMC_CAP_USER) 560 config |= AMD_PMC_USR; 561 if (caps & PMC_CAP_SYSTEM) 562 config |= AMD_PMC_OS; 563 if ((caps & (PMC_CAP_USER|PMC_CAP_SYSTEM)) == 0) 564 config |= (AMD_PMC_USR|AMD_PMC_OS); 565 566 if (caps & PMC_CAP_EDGE) 567 config |= AMD_PMC_EDGE; 568 if (caps & PMC_CAP_INVERT) 569 config |= AMD_PMC_INVERT; 570 if (caps & PMC_CAP_INTERRUPT) 571 config |= AMD_PMC_INT; 572 573 pm->pm_md.pm_amd.pm_amd_evsel = config; /* save config value */ 574 575 PMCDBG(MDP,ALL,2,"amd-allocate ri=%d -> config=0x%x", ri, config); 576 577 return 0; 578} 579 580/* 581 * Release machine dependent state associated with a PMC. This is a 582 * no-op on this architecture. 583 * 584 */ 585 586/* ARGSUSED0 */ 587static int 588amd_release_pmc(int cpu, int ri, struct pmc *pmc) 589{ 590#if DEBUG 591 const struct amd_descr *pd; 592#endif 593 struct pmc_hw *phw; 594 595 (void) pmc; 596 597 KASSERT(cpu >= 0 && cpu < mp_ncpus, 598 ("[amd,%d] illegal CPU value %d", __LINE__, cpu)); 599 KASSERT(ri >= 0 && ri < AMD_NPMCS, 600 ("[amd,%d] illegal row-index %d", __LINE__, ri)); 601 602 phw = pmc_pcpu[cpu]->pc_hwpmcs[ri]; 603 604 KASSERT(phw->phw_pmc == NULL, 605 ("[amd,%d] PHW pmc %p non-NULL", __LINE__, phw->phw_pmc)); 606 607#if DEBUG 608 pd = &amd_pmcdesc[ri]; 609 if (pd->pm_descr.pd_class == AMD_PMC_CLASS) 610 KASSERT(AMD_PMC_IS_STOPPED(pd->pm_evsel), 611 ("[amd,%d] PMC %d released while active", __LINE__, ri)); 612#endif 613 614 return 0; 615} 616 617/* 618 * start a PMC. 619 */ 620 621static int 622amd_start_pmc(int cpu, int ri) 623{ 624 uint32_t config; 625 struct pmc *pm; 626 struct pmc_hw *phw; 627 const struct amd_descr *pd; 628 629 KASSERT(cpu >= 0 && cpu < mp_ncpus, 630 ("[amd,%d] illegal CPU value %d", __LINE__, cpu)); 631 KASSERT(ri >= 0 && ri < AMD_NPMCS, 632 ("[amd,%d] illegal row-index %d", __LINE__, ri)); 633 634 phw = pmc_pcpu[cpu]->pc_hwpmcs[ri]; 635 pm = phw->phw_pmc; 636 pd = &amd_pmcdesc[ri]; 637 638 KASSERT(pm != NULL, 639 ("[amd,%d] starting cpu%d,pmc%d with null pmc record", __LINE__, 640 cpu, ri)); 641 642 PMCDBG(MDP,STA,1,"amd-start cpu=%d ri=%d", cpu, ri); 643 644 if (pd->pm_descr.pd_class == PMC_CLASS_TSC) 645 return 0; /* TSCs are always running */ 646 647 KASSERT(pd->pm_descr.pd_class == AMD_PMC_CLASS, 648 ("[amd,%d] unknown PMC class (%d)", __LINE__, 649 pd->pm_descr.pd_class)); 650 651 KASSERT(AMD_PMC_IS_STOPPED(pd->pm_evsel), 652 ("[amd,%d] pmc%d,cpu%d: Starting active PMC \"%s\"", __LINE__, 653 ri, cpu, pd->pm_descr.pd_name)); 654 655 /* turn on the PMC ENABLE bit */ 656 config = pm->pm_md.pm_amd.pm_amd_evsel | AMD_PMC_ENABLE; 657 658 PMCDBG(MDP,STA,2,"amd-start config=0x%x", config); 659 660 wrmsr(pd->pm_evsel, config); 661 return 0; 662} 663 664/* 665 * Stop a PMC. 666 */ 667 668static int 669amd_stop_pmc(int cpu, int ri) 670{ 671 struct pmc *pm; 672 struct pmc_hw *phw; 673 const struct amd_descr *pd; 674 uint64_t config; 675 676 KASSERT(cpu >= 0 && cpu < mp_ncpus, 677 ("[amd,%d] illegal CPU value %d", __LINE__, cpu)); 678 KASSERT(ri >= 0 && ri < AMD_NPMCS, 679 ("[amd,%d] illegal row-index %d", __LINE__, ri)); 680 681 phw = pmc_pcpu[cpu]->pc_hwpmcs[ri]; 682 pm = phw->phw_pmc; 683 pd = &amd_pmcdesc[ri]; 684 685 KASSERT(pm != NULL, 686 ("[amd,%d] cpu%d,pmc%d no PMC to stop", __LINE__, 687 cpu, ri)); 688 689 /* can't stop a TSC */ 690 if (pd->pm_descr.pd_class == PMC_CLASS_TSC) 691 return 0; 692 693 KASSERT(pd->pm_descr.pd_class == AMD_PMC_CLASS, 694 ("[amd,%d] unknown PMC class (%d)", __LINE__, 695 pd->pm_descr.pd_class)); 696 697 KASSERT(!AMD_PMC_IS_STOPPED(pd->pm_evsel), 698 ("[amd,%d] PMC%d, CPU%d \"%s\" already stopped", 699 __LINE__, ri, cpu, pd->pm_descr.pd_name)); 700 701 PMCDBG(MDP,STO,1,"amd-stop ri=%d", ri); 702 703 /* turn off the PMC ENABLE bit */ 704 config = pm->pm_md.pm_amd.pm_amd_evsel & ~AMD_PMC_ENABLE; 705 wrmsr(pd->pm_evsel, config); 706 return 0; 707} 708 709/* 710 * Interrupt handler. This function needs to return '1' if the 711 * interrupt was this CPU's PMCs or '0' otherwise. It is not allowed 712 * to sleep or do anything a 'fast' interrupt handler is not allowed 713 * to do. 714 */ 715 716static int 717amd_intr(int cpu, uintptr_t eip) 718{ 719 int i, retval; 720 enum pmc_mode mode; 721 uint32_t perfctr; 722 struct pmc *pm; 723 struct pmc_cpu *pc; 724 struct pmc_hw *phw; 725 726 KASSERT(cpu >= 0 && cpu < mp_ncpus, 727 ("[amd,%d] out of range CPU %d", __LINE__, cpu)); 728 729 retval = 0; 730 731 pc = pmc_pcpu[cpu]; 732 733 /* 734 * look for all PMCs that have interrupted: 735 * - skip over the TSC [PMC#0] 736 * - look for a PMC with a valid 'struct pmc' association 737 * - look for a PMC in (a) sampling mode and (b) which has 738 * overflowed. If found, we update the process's 739 * histogram or send it a profiling signal by calling 740 * the appropriate helper function. 741 */ 742 743 for (i = 1; i < AMD_NPMCS; i++) { 744 745 phw = pc->pc_hwpmcs[i]; 746 perfctr = amd_pmcdesc[i].pm_perfctr; 747 KASSERT(phw != NULL, ("[amd,%d] null PHW pointer", __LINE__)); 748 749 if ((pm = phw->phw_pmc) == NULL || 750 pm->pm_state != PMC_STATE_RUNNING) { 751 atomic_add_int(&pmc_stats.pm_intr_ignored, 1); 752 continue; 753 } 754 755 mode = pm->pm_mode; 756 if (PMC_IS_SAMPLING_MODE(mode) && 757 AMD_PMC_HAS_OVERFLOWED(perfctr)) { 758 atomic_add_int(&pmc_stats.pm_intr_processed, 1); 759 if (PMC_IS_SYSTEM_MODE(mode)) 760 pmc_update_histogram(phw, eip); 761 else if (PMC_IS_VIRTUAL_MODE(mode)) 762 pmc_send_signal(pm); 763 retval = 1; 764 } 765 } 766 return retval; 767} 768 769/* 770 * describe a PMC 771 */ 772static int 773amd_describe(int cpu, int ri, struct pmc_info *pi, struct pmc **ppmc) 774{ 775 int error; 776 size_t copied; 777 const struct amd_descr *pd; 778 struct pmc_hw *phw; 779 780 KASSERT(cpu >= 0 && cpu < mp_ncpus, 781 ("[amd,%d] illegal CPU %d", __LINE__, cpu)); 782 KASSERT(ri >= 0 && ri < AMD_NPMCS, 783 ("[amd,%d] row-index %d out of range", __LINE__, ri)); 784 785 phw = pmc_pcpu[cpu]->pc_hwpmcs[ri]; 786 pd = &amd_pmcdesc[ri]; 787 788 if ((error = copystr(pd->pm_descr.pd_name, pi->pm_name, 789 PMC_NAME_MAX, &copied)) != 0) 790 return error; 791 792 pi->pm_class = pd->pm_descr.pd_class; 793 pi->pm_caps = pd->pm_descr.pd_caps; 794 pi->pm_width = pd->pm_descr.pd_width; 795 796 if (phw->phw_state & PMC_PHW_FLAG_IS_ENABLED) { 797 pi->pm_enabled = TRUE; 798 *ppmc = phw->phw_pmc; 799 } else { 800 pi->pm_enabled = FALSE; 801 *ppmc = NULL; 802 } 803 804 return 0; 805} 806 807/* 808 * i386 specific entry points 809 */ 810 811/* 812 * return the MSR address of the given PMC. 813 */ 814 815static int 816amd_get_msr(int ri, uint32_t *msr) 817{ 818 KASSERT(ri >= 0 && ri < AMD_NPMCS, 819 ("[amd,%d] ri %d out of range", __LINE__, ri)); 820 821 *msr = amd_pmcdesc[ri].pm_perfctr; 822 return 0; 823} 824 825/* 826 * processor dependent initialization. 827 */ 828 829/* 830 * Per-processor data structure 831 * 832 * [common stuff] 833 * [5 struct pmc_hw pointers] 834 * [5 struct pmc_hw structures] 835 */ 836 837struct amd_cpu { 838 struct pmc_cpu pc_common; 839 struct pmc_hw *pc_hwpmcs[AMD_NPMCS]; 840 struct pmc_hw pc_amdpmcs[AMD_NPMCS]; 841}; 842 843 844static int 845amd_init(int cpu) 846{ 847 int n; 848 struct amd_cpu *pcs; 849 struct pmc_hw *phw; 850 851 KASSERT(cpu >= 0 && cpu < mp_ncpus, 852 ("[amd,%d] insane cpu number %d", __LINE__, cpu)); 853 854 PMCDBG(MDP,INI,1,"amd-init cpu=%d", cpu); 855 856 MALLOC(pcs, struct amd_cpu *, sizeof(struct amd_cpu), M_PMC, 857 M_WAITOK|M_ZERO); 858 859 if (pcs == NULL) 860 return ENOMEM; 861 862 phw = &pcs->pc_amdpmcs[0]; 863 864 /* 865 * Initialize the per-cpu mutex and set the content of the 866 * hardware descriptors to a known state. 867 */ 868 869 for (n = 0; n < AMD_NPMCS; n++, phw++) { 870 phw->phw_state = PMC_PHW_FLAG_IS_ENABLED | 871 PMC_PHW_CPU_TO_STATE(cpu) | PMC_PHW_INDEX_TO_STATE(n); 872 phw->phw_pmc = NULL; 873 pcs->pc_hwpmcs[n] = phw; 874 } 875 876 /* Mark the TSC as shareable */ 877 pcs->pc_hwpmcs[0]->phw_state |= PMC_PHW_FLAG_IS_SHAREABLE; 878 879 pmc_pcpu[cpu] = (struct pmc_cpu *) pcs; 880 881 return 0; 882} 883 884 885/* 886 * processor dependent cleanup prior to the KLD 887 * being unloaded 888 */ 889 890static int 891amd_cleanup(int cpu) 892{ 893 int i; 894 uint32_t evsel; 895 struct pmc_cpu *pcs; 896 897 KASSERT(cpu >= 0 && cpu < mp_ncpus, 898 ("[amd,%d] insane cpu number (%d)", __LINE__, cpu)); 899 900 PMCDBG(MDP,INI,1,"amd-cleanup cpu=%d", cpu); 901 902 /* 903 * First, turn off all PMCs on this CPU. 904 */ 905 906 for (i = 0; i < 4; i++) { /* XXX this loop is now not needed */ 907 evsel = rdmsr(AMD_PMC_EVSEL_0 + i); 908 evsel &= ~AMD_PMC_ENABLE; 909 wrmsr(AMD_PMC_EVSEL_0 + i, evsel); 910 } 911 912 /* 913 * Next, free up allocated space. 914 */ 915 916 pcs = pmc_pcpu[cpu]; 917 918#if DEBUG 919 /* check the TSC */ 920 KASSERT(pcs->pc_hwpmcs[0]->phw_pmc == NULL, 921 ("[amd,%d] CPU%d,PMC0 still in use", __LINE__, cpu)); 922 for (i = 1; i < AMD_NPMCS; i++) { 923 KASSERT(pcs->pc_hwpmcs[i]->phw_pmc == NULL, 924 ("[amd,%d] CPU%d/PMC%d in use", __LINE__, cpu, i)); 925 KASSERT(AMD_PMC_IS_STOPPED(AMD_PMC_EVSEL_0 + (i-1)), 926 ("[amd,%d] CPU%d/PMC%d not stopped", __LINE__, cpu, i)); 927 } 928#endif 929 KASSERT(pcs != NULL, 930 ("[amd,%d] null per-cpu state pointer (cpu%d)", __LINE__, cpu)); 931 932 pmc_pcpu[cpu] = NULL; 933 FREE(pcs, M_PMC); 934 return 0; 935} 936 937/* 938 * Initialize ourselves. 939 */ 940 941struct pmc_mdep * 942pmc_amd_initialize(void) 943{ 944 945 struct pmc_mdep *pmc_mdep; 946 947 /* The presence of hardware performance counters on the AMD 948 Athlon, Duron or later processors, is _not_ indicated by 949 any of the processor feature flags set by the 'CPUID' 950 instruction, so we only check the 'instruction family' 951 field returned by CPUID for instruction family >= 6. This 952 test needs to be be refined. */ 953 954 if ((cpu_id & 0xF00) < 0x600) 955 return NULL; 956 957 MALLOC(pmc_mdep, struct pmc_mdep *, sizeof(struct pmc_mdep), 958 M_PMC, M_WAITOK|M_ZERO); 959 960#if __i386__ 961 pmc_mdep->pmd_cputype = PMC_CPU_AMD_K7; 962#elif __amd64__ 963 pmc_mdep->pmd_cputype = PMC_CPU_AMD_K8; 964#else 965#error Unknown AMD CPU type. 966#endif 967 968 pmc_mdep->pmd_npmc = AMD_NPMCS; 969 970 /* this processor has two classes of usable PMCs */ 971 pmc_mdep->pmd_nclass = 2; 972 pmc_mdep->pmd_classes[0] = PMC_CLASS_TSC; 973 pmc_mdep->pmd_classes[1] = AMD_PMC_CLASS; 974 pmc_mdep->pmd_nclasspmcs[0] = 1; 975 pmc_mdep->pmd_nclasspmcs[1] = (AMD_NPMCS-1); 976 977 pmc_mdep->pmd_init = amd_init; 978 pmc_mdep->pmd_cleanup = amd_cleanup; 979 pmc_mdep->pmd_switch_in = amd_switch_in; 980 pmc_mdep->pmd_switch_out = amd_switch_out; 981 pmc_mdep->pmd_read_pmc = amd_read_pmc; 982 pmc_mdep->pmd_write_pmc = amd_write_pmc; 983 pmc_mdep->pmd_config_pmc = amd_config_pmc; 984 pmc_mdep->pmd_allocate_pmc = amd_allocate_pmc; 985 pmc_mdep->pmd_release_pmc = amd_release_pmc; 986 pmc_mdep->pmd_start_pmc = amd_start_pmc; 987 pmc_mdep->pmd_stop_pmc = amd_stop_pmc; 988 pmc_mdep->pmd_intr = amd_intr; 989 pmc_mdep->pmd_describe = amd_describe; 990 pmc_mdep->pmd_get_msr = amd_get_msr; /* i386 */ 991 992 PMCDBG(MDP,INI,0,"%s","amd-initialize"); 993 994 return pmc_mdep; 995} 996