1/*- 2 * Copyright (c) 2003-2005,2008 Joseph Koshy 3 * Copyright (c) 2007 The FreeBSD Foundation 4 * All rights reserved. 5 * 6 * Portions of this software were developed by A. Joseph Koshy under 7 * sponsorship from the FreeBSD Foundation and Google, Inc. 8 * 9 * Redistribution and use in source and binary forms, with or without 10 * modification, are permitted provided that the following conditions 11 * are met: 12 * 1. Redistributions of source code must retain the above copyright 13 * notice, this list of conditions and the following disclaimer. 14 * 2. Redistributions in binary form must reproduce the above copyright 15 * notice, this list of conditions and the following disclaimer in the 16 * documentation and/or other materials provided with the distribution. 17 * 18 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 19 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 20 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 21 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 22 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 23 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 24 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 25 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 26 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 27 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 28 * SUCH DAMAGE. 29 */ 30 31#include <sys/cdefs.h> 32__FBSDID("$FreeBSD$"); 33 34#include <sys/param.h> 35#include <sys/bus.h> 36#include <sys/lock.h> 37#include <sys/mutex.h> 38#include <sys/pmc.h> 39#include <sys/pmckern.h> 40#include <sys/smp.h> 41#include <sys/systm.h> 42 43#include <machine/intr_machdep.h> 44#include <machine/apicvar.h> 45#include <machine/cpu.h> 46#include <machine/cpufunc.h> 47#include <machine/cputypes.h> 48#include <machine/md_var.h> 49#include <machine/pmc_mdep.h> 50#include <machine/specialreg.h> 51 52/* 53 * PENTIUM PRO SUPPORT 54 * 55 * Quirks: 56 * 57 * - Both PMCs are enabled by a single bit P6_EVSEL_EN in performance 58 * counter '0'. This bit needs to be '1' if any of the two 59 * performance counters are in use. Perf counters can also be 60 * switched off by writing zeros to their EVSEL register. 61 * 62 * - While the width of these counters is 40 bits, we do not appear to 63 * have a way of writing 40 bits to the counter MSRs. A WRMSR 64 * instruction will sign extend bit 31 of the value being written to 65 * the perf counter -- a value of 0x80000000 written to an perf 66 * counter register will be sign extended to 0xFF80000000. 67 * 68 * This quirk primarily affects thread-mode PMCs in counting mode, as 69 * these PMCs read and write PMC registers at every context switch. 70 */ 71 72struct p6pmc_descr { 73 struct pmc_descr pm_descr; /* common information */ 74 uint32_t pm_pmc_msr; 75 uint32_t pm_evsel_msr; 76}; 77 78static struct p6pmc_descr p6_pmcdesc[P6_NPMCS] = { 79 80#define P6_PMC_CAPS (PMC_CAP_INTERRUPT | PMC_CAP_USER | PMC_CAP_SYSTEM | \ 81 PMC_CAP_EDGE | PMC_CAP_THRESHOLD | PMC_CAP_READ | PMC_CAP_WRITE | \ 82 PMC_CAP_INVERT | PMC_CAP_QUALIFIER) 83 84 /* PMC 0 */ 85 { 86 .pm_descr = 87 { 88 .pd_name ="P6-0", 89 .pd_class = PMC_CLASS_P6, 90 .pd_caps = P6_PMC_CAPS, 91 .pd_width = 40 92 }, 93 .pm_pmc_msr = P6_MSR_PERFCTR0, 94 .pm_evsel_msr = P6_MSR_EVSEL0 95 }, 96 97 /* PMC 1 */ 98 { 99 .pm_descr = 100 { 101 .pd_name ="P6-1", 102 .pd_class = PMC_CLASS_P6, 103 .pd_caps = P6_PMC_CAPS, 104 .pd_width = 40 105 }, 106 .pm_pmc_msr = P6_MSR_PERFCTR1, 107 .pm_evsel_msr = P6_MSR_EVSEL1 108 } 109}; 110 111static enum pmc_cputype p6_cputype; 112 113/* 114 * P6 Event descriptor 115 * 116 * The 'pm_flags' field has the following structure: 117 * - The upper 4 bits are used to track which counter an event is valid on. 118 * - The lower bits form a bitmask of flags indicating support for the event 119 * on a given CPU. 120 */ 121 122struct p6_event_descr { 123 const enum pmc_event pm_event; 124 uint32_t pm_evsel; 125 uint32_t pm_flags; 126 uint32_t pm_unitmask; 127}; 128 129#define P6F_CTR(C) (1 << (28 + (C))) 130#define P6F_CTR0 P6F_CTR(0) 131#define P6F_CTR1 P6F_CTR(1) 132#define P6F(CPU) (1 << ((CPU) - PMC_CPU_INTEL_P6)) 133#define _P6F(C) P6F(PMC_CPU_INTEL_##C) 134#define P6F_P6 _P6F(P6) 135#define P6F_CL _P6F(CL) 136#define P6F_PII _P6F(PII) 137#define P6F_PIII _P6F(PIII) 138#define P6F_PM _P6F(PM) 139#define P6F_ALL_CPUS (P6F_P6 | P6F_PII | P6F_CL | P6F_PIII | P6F_PM) 140#define P6F_ALL_CTRS (P6F_CTR0 | P6F_CTR1) 141#define P6F_ALL (P6F_ALL_CPUS | P6F_ALL_CTRS) 142 143#define P6_EVENT_VALID_FOR_CPU(P,CPU) ((P)->pm_flags & P6F(CPU)) 144#define P6_EVENT_VALID_FOR_CTR(P,CTR) ((P)->pm_flags & P6F_CTR(CTR)) 145 146static const struct p6_event_descr p6_events[] = { 147 148#define P6_EVDESCR(NAME, EVSEL, FLAGS, UMASK) \ 149 { \ 150 .pm_event = PMC_EV_P6_##NAME, \ 151 .pm_evsel = (EVSEL), \ 152 .pm_flags = (FLAGS), \ 153 .pm_unitmask = (UMASK) \ 154 } 155 156P6_EVDESCR(DATA_MEM_REFS, 0x43, P6F_ALL, 0x00), 157P6_EVDESCR(DCU_LINES_IN, 0x45, P6F_ALL, 0x00), 158P6_EVDESCR(DCU_M_LINES_IN, 0x46, P6F_ALL, 0x00), 159P6_EVDESCR(DCU_M_LINES_OUT, 0x47, P6F_ALL, 0x00), 160P6_EVDESCR(DCU_MISS_OUTSTANDING, 0x47, P6F_ALL, 0x00), 161P6_EVDESCR(IFU_FETCH, 0x80, P6F_ALL, 0x00), 162P6_EVDESCR(IFU_FETCH_MISS, 0x81, P6F_ALL, 0x00), 163P6_EVDESCR(ITLB_MISS, 0x85, P6F_ALL, 0x00), 164P6_EVDESCR(IFU_MEM_STALL, 0x86, P6F_ALL, 0x00), 165P6_EVDESCR(ILD_STALL, 0x87, P6F_ALL, 0x00), 166P6_EVDESCR(L2_IFETCH, 0x28, P6F_ALL, 0x0F), 167P6_EVDESCR(L2_LD, 0x29, P6F_ALL, 0x0F), 168P6_EVDESCR(L2_ST, 0x2A, P6F_ALL, 0x0F), 169P6_EVDESCR(L2_LINES_IN, 0x24, P6F_ALL, 0x0F), 170P6_EVDESCR(L2_LINES_OUT, 0x26, P6F_ALL, 0x0F), 171P6_EVDESCR(L2_M_LINES_INM, 0x25, P6F_ALL, 0x00), 172P6_EVDESCR(L2_M_LINES_OUTM, 0x27, P6F_ALL, 0x0F), 173P6_EVDESCR(L2_RQSTS, 0x2E, P6F_ALL, 0x0F), 174P6_EVDESCR(L2_ADS, 0x21, P6F_ALL, 0x00), 175P6_EVDESCR(L2_DBUS_BUSY, 0x22, P6F_ALL, 0x00), 176P6_EVDESCR(L2_DBUS_BUSY_RD, 0x23, P6F_ALL, 0x00), 177P6_EVDESCR(BUS_DRDY_CLOCKS, 0x62, P6F_ALL, 0x20), 178P6_EVDESCR(BUS_LOCK_CLOCKS, 0x63, P6F_ALL, 0x20), 179P6_EVDESCR(BUS_REQ_OUTSTANDING, 0x60, P6F_ALL, 0x00), 180P6_EVDESCR(BUS_TRAN_BRD, 0x65, P6F_ALL, 0x20), 181P6_EVDESCR(BUS_TRAN_RFO, 0x66, P6F_ALL, 0x20), 182P6_EVDESCR(BUS_TRANS_WB, 0x67, P6F_ALL, 0x20), 183P6_EVDESCR(BUS_TRAN_IFETCH, 0x68, P6F_ALL, 0x20), 184P6_EVDESCR(BUS_TRAN_INVAL, 0x69, P6F_ALL, 0x20), 185P6_EVDESCR(BUS_TRAN_PWR, 0x6A, P6F_ALL, 0x20), 186P6_EVDESCR(BUS_TRANS_P, 0x6B, P6F_ALL, 0x20), 187P6_EVDESCR(BUS_TRANS_IO, 0x6C, P6F_ALL, 0x20), 188P6_EVDESCR(BUS_TRAN_DEF, 0x6D, P6F_ALL, 0x20), 189P6_EVDESCR(BUS_TRAN_BURST, 0x6E, P6F_ALL, 0x20), 190P6_EVDESCR(BUS_TRAN_ANY, 0x70, P6F_ALL, 0x20), 191P6_EVDESCR(BUS_TRAN_MEM, 0x6F, P6F_ALL, 0x20), 192P6_EVDESCR(BUS_DATA_RCV, 0x64, P6F_ALL, 0x00), 193P6_EVDESCR(BUS_BNR_DRV, 0x61, P6F_ALL, 0x00), 194P6_EVDESCR(BUS_HIT_DRV, 0x7A, P6F_ALL, 0x00), 195P6_EVDESCR(BUS_HITM_DRV, 0x7B, P6F_ALL, 0x00), 196P6_EVDESCR(BUS_SNOOP_STALL, 0x7E, P6F_ALL, 0x00), 197P6_EVDESCR(FLOPS, 0xC1, P6F_ALL_CPUS | P6F_CTR0, 0x00), 198P6_EVDESCR(FP_COMPS_OPS_EXE, 0x10, P6F_ALL_CPUS | P6F_CTR0, 0x00), 199P6_EVDESCR(FP_ASSIST, 0x11, P6F_ALL_CPUS | P6F_CTR1, 0x00), 200P6_EVDESCR(MUL, 0x12, P6F_ALL_CPUS | P6F_CTR1, 0x00), 201P6_EVDESCR(DIV, 0x13, P6F_ALL_CPUS | P6F_CTR1, 0x00), 202P6_EVDESCR(CYCLES_DIV_BUSY, 0x14, P6F_ALL_CPUS | P6F_CTR0, 0x00), 203P6_EVDESCR(LD_BLOCKS, 0x03, P6F_ALL, 0x00), 204P6_EVDESCR(SB_DRAINS, 0x04, P6F_ALL, 0x00), 205P6_EVDESCR(MISALIGN_MEM_REF, 0x05, P6F_ALL, 0x00), 206P6_EVDESCR(EMON_KNI_PREF_DISPATCHED, 0x07, P6F_PIII | P6F_ALL_CTRS, 0x03), 207P6_EVDESCR(EMON_KNI_PREF_MISS, 0x4B, P6F_PIII | P6F_ALL_CTRS, 0x03), 208P6_EVDESCR(INST_RETIRED, 0xC0, P6F_ALL, 0x00), 209P6_EVDESCR(UOPS_RETIRED, 0xC2, P6F_ALL, 0x00), 210P6_EVDESCR(INST_DECODED, 0xD0, P6F_ALL, 0x00), 211P6_EVDESCR(EMON_KNI_INST_RETIRED, 0xD8, P6F_PIII | P6F_ALL_CTRS, 0x01), 212P6_EVDESCR(EMON_KNI_COMP_INST_RET, 0xD9, P6F_PIII | P6F_ALL_CTRS, 0x01), 213P6_EVDESCR(HW_INT_RX, 0xC8, P6F_ALL, 0x00), 214P6_EVDESCR(CYCLES_INT_MASKED, 0xC6, P6F_ALL, 0x00), 215P6_EVDESCR(CYCLES_INT_PENDING_AND_MASKED, 0xC7, P6F_ALL, 0x00), 216P6_EVDESCR(BR_INST_RETIRED, 0xC4, P6F_ALL, 0x00), 217P6_EVDESCR(BR_MISS_PRED_RETIRED, 0xC5, P6F_ALL, 0x00), 218P6_EVDESCR(BR_TAKEN_RETIRED, 0xC9, P6F_ALL, 0x00), 219P6_EVDESCR(BR_MISS_PRED_TAKEN_RET, 0xCA, P6F_ALL, 0x00), 220P6_EVDESCR(BR_INST_DECODED, 0xE0, P6F_ALL, 0x00), 221P6_EVDESCR(BTB_MISSES, 0xE2, P6F_ALL, 0x00), 222P6_EVDESCR(BR_BOGUS, 0xE4, P6F_ALL, 0x00), 223P6_EVDESCR(BACLEARS, 0xE6, P6F_ALL, 0x00), 224P6_EVDESCR(RESOURCE_STALLS, 0xA2, P6F_ALL, 0x00), 225P6_EVDESCR(PARTIAL_RAT_STALLS, 0xD2, P6F_ALL, 0x00), 226P6_EVDESCR(SEGMENT_REG_LOADS, 0x06, P6F_ALL, 0x00), 227P6_EVDESCR(CPU_CLK_UNHALTED, 0x79, P6F_ALL, 0x00), 228P6_EVDESCR(MMX_INSTR_EXEC, 0xB0, 229 P6F_ALL_CTRS | P6F_CL | P6F_PII, 0x00), 230P6_EVDESCR(MMX_SAT_INSTR_EXEC, 0xB1, 231 P6F_ALL_CTRS | P6F_PII | P6F_PIII, 0x00), 232P6_EVDESCR(MMX_UOPS_EXEC, 0xB2, 233 P6F_ALL_CTRS | P6F_PII | P6F_PIII, 0x0F), 234P6_EVDESCR(MMX_INSTR_TYPE_EXEC, 0xB3, 235 P6F_ALL_CTRS | P6F_PII | P6F_PIII, 0x3F), 236P6_EVDESCR(FP_MMX_TRANS, 0xCC, 237 P6F_ALL_CTRS | P6F_PII | P6F_PIII, 0x01), 238P6_EVDESCR(MMX_ASSIST, 0xCD, 239 P6F_ALL_CTRS | P6F_PII | P6F_PIII, 0x00), 240P6_EVDESCR(MMX_INSTR_RET, 0xCE, P6F_ALL_CTRS | P6F_PII, 0x00), 241P6_EVDESCR(SEG_RENAME_STALLS, 0xD4, 242 P6F_ALL_CTRS | P6F_PII | P6F_PIII, 0x0F), 243P6_EVDESCR(SEG_REG_RENAMES, 0xD5, 244 P6F_ALL_CTRS | P6F_PII | P6F_PIII, 0x0F), 245P6_EVDESCR(RET_SEG_RENAMES, 0xD6, 246 P6F_ALL_CTRS | P6F_PII | P6F_PIII, 0x00), 247P6_EVDESCR(EMON_EST_TRANS, 0x58, P6F_ALL_CTRS | P6F_PM, 0x02), 248P6_EVDESCR(EMON_THERMAL_TRIP, 0x59, P6F_ALL_CTRS | P6F_PM, 0x00), 249P6_EVDESCR(BR_INST_EXEC, 0x88, P6F_ALL_CTRS | P6F_PM, 0x00), 250P6_EVDESCR(BR_MISSP_EXEC, 0x89, P6F_ALL_CTRS | P6F_PM, 0x00), 251P6_EVDESCR(BR_BAC_MISSP_EXEC, 0x8A, P6F_ALL_CTRS | P6F_PM, 0x00), 252P6_EVDESCR(BR_CND_EXEC, 0x8B, P6F_ALL_CTRS | P6F_PM, 0x00), 253P6_EVDESCR(BR_CND_MISSP_EXEC, 0x8C, P6F_ALL_CTRS | P6F_PM, 0x00), 254P6_EVDESCR(BR_IND_EXEC, 0x8D, P6F_ALL_CTRS | P6F_PM, 0x00), 255P6_EVDESCR(BR_IND_MISSP_EXEC, 0x8E, P6F_ALL_CTRS | P6F_PM, 0x00), 256P6_EVDESCR(BR_RET_EXEC, 0x8F, P6F_ALL_CTRS | P6F_PM, 0x00), 257P6_EVDESCR(BR_RET_MISSP_EXEC, 0x90, P6F_ALL_CTRS | P6F_PM, 0x00), 258P6_EVDESCR(BR_RET_BAC_MISSP_EXEC, 0x91, P6F_ALL_CTRS | P6F_PM, 0x00), 259P6_EVDESCR(BR_CALL_EXEC, 0x92, P6F_ALL_CTRS | P6F_PM, 0x00), 260P6_EVDESCR(BR_CALL_MISSP_EXEC, 0x93, P6F_ALL_CTRS | P6F_PM, 0x00), 261P6_EVDESCR(BR_IND_CALL_EXEC, 0x94, P6F_ALL_CTRS | P6F_PM, 0x00), 262P6_EVDESCR(EMON_SIMD_INSTR_RETIRED, 0xCE, P6F_ALL_CTRS | P6F_PM, 0x00), 263P6_EVDESCR(EMON_SYNCH_UOPS, 0xD3, P6F_ALL_CTRS | P6F_PM, 0x00), 264P6_EVDESCR(EMON_ESP_UOPS, 0xD7, P6F_ALL_CTRS | P6F_PM, 0x00), 265P6_EVDESCR(EMON_FUSED_UOPS_RET, 0xDA, P6F_ALL_CTRS | P6F_PM, 0x03), 266P6_EVDESCR(EMON_UNFUSION, 0xDB, P6F_ALL_CTRS | P6F_PM, 0x00), 267P6_EVDESCR(EMON_PREF_RQSTS_UP, 0xF0, P6F_ALL_CTRS | P6F_PM, 0x00), 268P6_EVDESCR(EMON_PREF_RQSTS_DN, 0xD8, P6F_ALL_CTRS | P6F_PM, 0x00), 269P6_EVDESCR(EMON_SSE_SSE2_INST_RETIRED, 0xD8, P6F_ALL_CTRS | P6F_PM, 0x03), 270P6_EVDESCR(EMON_SSE_SSE2_COMP_INST_RETIRED, 0xD9, P6F_ALL_CTRS | P6F_PM, 0x03) 271 272#undef P6_EVDESCR 273}; 274 275#define P6_NEVENTS (PMC_EV_P6_LAST - PMC_EV_P6_FIRST + 1) 276 277static const struct p6_event_descr * 278p6_find_event(enum pmc_event ev) 279{ 280 int n; 281 282 for (n = 0; n < P6_NEVENTS; n++) 283 if (p6_events[n].pm_event == ev) 284 break; 285 if (n == P6_NEVENTS) 286 return NULL; 287 return &p6_events[n]; 288} 289 290/* 291 * Per-CPU data structure for P6 class CPUs 292 * 293 * [common stuff] 294 * [flags for maintaining PMC start/stop state] 295 * [3 struct pmc_hw pointers] 296 * [3 struct pmc_hw structures] 297 */ 298 299struct p6_cpu { 300 struct pmc_hw pc_p6pmcs[P6_NPMCS]; 301 uint32_t pc_state; 302}; 303 304static struct p6_cpu **p6_pcpu; 305 306/* 307 * If CTR1 is active, we need to keep the 'EN' bit if CTR0 set, 308 * with the rest of CTR0 being zero'ed out. 309 */ 310#define P6_SYNC_CTR_STATE(PC) do { \ 311 uint32_t _config, _enable; \ 312 _enable = 0; \ 313 if ((PC)->pc_state & 0x02) \ 314 _enable |= P6_EVSEL_EN; \ 315 if ((PC)->pc_state & 0x01) \ 316 _config = rdmsr(P6_MSR_EVSEL0) | \ 317 P6_EVSEL_EN; \ 318 else \ 319 _config = 0; \ 320 wrmsr(P6_MSR_EVSEL0, _config | _enable); \ 321 } while (0) 322 323#define P6_MARK_STARTED(PC,RI) do { \ 324 (PC)->pc_state |= (1 << ((RI)-1)); \ 325 } while (0) 326 327#define P6_MARK_STOPPED(PC,RI) do { \ 328 (PC)->pc_state &= ~(1<< ((RI)-1)); \ 329 } while (0) 330 331static int 332p6_pcpu_init(struct pmc_mdep *md, int cpu) 333{ 334 int first_ri, n; 335 struct p6_cpu *p6c; 336 struct pmc_cpu *pc; 337 struct pmc_hw *phw; 338 339 KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), 340 ("[p6,%d] bad cpu %d", __LINE__, cpu)); 341 342 PMCDBG(MDP,INI,0,"p6-init cpu=%d", cpu); 343 344 p6c = malloc(sizeof (struct p6_cpu), M_PMC, M_WAITOK|M_ZERO); 345 pc = pmc_pcpu[cpu]; 346 347 KASSERT(pc != NULL, ("[p6,%d] cpu %d null per-cpu", __LINE__, cpu)); 348 349 phw = p6c->pc_p6pmcs; 350 p6_pcpu[cpu] = p6c; 351 352 first_ri = md->pmd_classdep[PMC_MDEP_CLASS_INDEX_P6].pcd_ri; 353 354 for (n = 0; n < P6_NPMCS; n++, phw++) { 355 phw->phw_state = PMC_PHW_FLAG_IS_ENABLED | 356 PMC_PHW_CPU_TO_STATE(cpu) | PMC_PHW_INDEX_TO_STATE(n); 357 phw->phw_pmc = NULL; 358 pc->pc_hwpmcs[n + first_ri] = phw; 359 } 360 361 return (0); 362} 363 364static int 365p6_pcpu_fini(struct pmc_mdep *md, int cpu) 366{ 367 int first_ri, n; 368 struct p6_cpu *p6c; 369 struct pmc_cpu *pc; 370 371 KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), 372 ("[p6,%d] bad cpu %d", __LINE__, cpu)); 373 374 PMCDBG(MDP,INI,0,"p6-cleanup cpu=%d", cpu); 375 376 p6c = p6_pcpu[cpu]; 377 p6_pcpu[cpu] = NULL; 378 379 KASSERT(p6c != NULL, ("[p6,%d] null pcpu", __LINE__)); 380 381 free(p6c, M_PMC); 382 383 first_ri = md->pmd_classdep[PMC_MDEP_CLASS_INDEX_P6].pcd_ri; 384 pc = pmc_pcpu[cpu]; 385 for (n = 0; n < P6_NPMCS; n++) 386 pc->pc_hwpmcs[n + first_ri] = NULL; 387 388 return (0); 389} 390 391static int 392p6_read_pmc(int cpu, int ri, pmc_value_t *v) 393{ 394 struct pmc *pm; 395 struct p6pmc_descr *pd; 396 pmc_value_t tmp; 397 398 KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), 399 ("[p6,%d] illegal cpu value %d", __LINE__, cpu)); 400 KASSERT(ri >= 0 && ri < P6_NPMCS, 401 ("[p6,%d] illegal row-index %d", __LINE__, ri)); 402 403 pm = p6_pcpu[cpu]->pc_p6pmcs[ri].phw_pmc; 404 pd = &p6_pmcdesc[ri]; 405 406 KASSERT(pm, 407 ("[p6,%d] cpu %d ri %d pmc not configured", __LINE__, cpu, ri)); 408 409 tmp = rdmsr(pd->pm_pmc_msr) & P6_PERFCTR_READ_MASK; 410 if (PMC_IS_SAMPLING_MODE(PMC_TO_MODE(pm))) 411 *v = P6_PERFCTR_VALUE_TO_RELOAD_COUNT(tmp); 412 else 413 *v = tmp; 414 415 PMCDBG(MDP,REA,1, "p6-read cpu=%d ri=%d msr=0x%x -> v=%jx", cpu, ri, 416 pd->pm_pmc_msr, *v); 417 418 return (0); 419} 420 421static int 422p6_write_pmc(int cpu, int ri, pmc_value_t v) 423{ 424 struct pmc *pm; 425 struct p6pmc_descr *pd; 426 427 KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), 428 ("[p6,%d] illegal cpu value %d", __LINE__, cpu)); 429 KASSERT(ri >= 0 && ri < P6_NPMCS, 430 ("[p6,%d] illegal row-index %d", __LINE__, ri)); 431 432 pm = p6_pcpu[cpu]->pc_p6pmcs[ri].phw_pmc; 433 pd = &p6_pmcdesc[ri]; 434 435 KASSERT(pm, 436 ("[p6,%d] cpu %d ri %d pmc not configured", __LINE__, cpu, ri)); 437 438 PMCDBG(MDP,WRI,1, "p6-write cpu=%d ri=%d msr=0x%x v=%jx", cpu, ri, 439 pd->pm_pmc_msr, v); 440 441 if (PMC_IS_SAMPLING_MODE(PMC_TO_MODE(pm))) 442 v = P6_RELOAD_COUNT_TO_PERFCTR_VALUE(v); 443 444 wrmsr(pd->pm_pmc_msr, v & P6_PERFCTR_WRITE_MASK); 445 446 return (0); 447} 448 449static int 450p6_config_pmc(int cpu, int ri, struct pmc *pm) 451{ 452 KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), 453 ("[p6,%d] illegal CPU %d", __LINE__, cpu)); 454 455 KASSERT(ri >= 0 && ri < P6_NPMCS, 456 ("[p6,%d] illegal row-index %d", __LINE__, ri)); 457 458 PMCDBG(MDP,CFG,1, "p6-config cpu=%d ri=%d pm=%p", cpu, ri, pm); 459 460 KASSERT(p6_pcpu[cpu] != NULL, ("[p6,%d] null per-cpu %d", __LINE__, 461 cpu)); 462 463 p6_pcpu[cpu]->pc_p6pmcs[ri].phw_pmc = pm; 464 465 return (0); 466} 467 468/* 469 * Retrieve a configured PMC pointer from hardware state. 470 */ 471 472static int 473p6_get_config(int cpu, int ri, struct pmc **ppm) 474{ 475 476 KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), 477 ("[p6,%d] illegal CPU %d", __LINE__, cpu)); 478 KASSERT(ri >= 0 && ri < P6_NPMCS, 479 ("[p6,%d] illegal row-index %d", __LINE__, ri)); 480 481 *ppm = p6_pcpu[cpu]->pc_p6pmcs[ri].phw_pmc; 482 483 return (0); 484} 485 486 487/* 488 * A pmc may be allocated to a given row index if: 489 * - the event is valid for this CPU 490 * - the event is valid for this counter index 491 */ 492 493static int 494p6_allocate_pmc(int cpu, int ri, struct pmc *pm, 495 const struct pmc_op_pmcallocate *a) 496{ 497 uint32_t allowed_unitmask, caps, config, unitmask; 498 const struct p6pmc_descr *pd; 499 const struct p6_event_descr *pevent; 500 enum pmc_event ev; 501 502 (void) cpu; 503 504 KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), 505 ("[p6,%d] illegal CPU %d", __LINE__, cpu)); 506 KASSERT(ri >= 0 && ri < P6_NPMCS, 507 ("[p6,%d] illegal row-index value %d", __LINE__, ri)); 508 509 pd = &p6_pmcdesc[ri]; 510 511 PMCDBG(MDP,ALL,1, "p6-allocate ri=%d class=%d pmccaps=0x%x " 512 "reqcaps=0x%x", ri, pd->pm_descr.pd_class, pd->pm_descr.pd_caps, 513 pm->pm_caps); 514 515 /* check class */ 516 if (pd->pm_descr.pd_class != a->pm_class) 517 return (EINVAL); 518 519 /* check requested capabilities */ 520 caps = a->pm_caps; 521 if ((pd->pm_descr.pd_caps & caps) != caps) 522 return (EPERM); 523 524 ev = pm->pm_event; 525 526 if (ev < PMC_EV_P6_FIRST || ev > PMC_EV_P6_LAST) 527 return (EINVAL); 528 529 if ((pevent = p6_find_event(ev)) == NULL) 530 return (ESRCH); 531 532 if (!P6_EVENT_VALID_FOR_CPU(pevent, p6_cputype) || 533 !P6_EVENT_VALID_FOR_CTR(pevent, (ri-1))) 534 return (EINVAL); 535 536 /* For certain events, Pentium M differs from the stock P6 */ 537 allowed_unitmask = 0; 538 if (p6_cputype == PMC_CPU_INTEL_PM) { 539 if (ev == PMC_EV_P6_L2_LD || ev == PMC_EV_P6_L2_LINES_IN || 540 ev == PMC_EV_P6_L2_LINES_OUT) 541 allowed_unitmask = P6_EVSEL_TO_UMASK(0x3F); 542 else if (ev == PMC_EV_P6_L2_M_LINES_OUTM) 543 allowed_unitmask = P6_EVSEL_TO_UMASK(0x30); 544 } else 545 allowed_unitmask = P6_EVSEL_TO_UMASK(pevent->pm_unitmask); 546 547 unitmask = a->pm_md.pm_ppro.pm_ppro_config & P6_EVSEL_UMASK_MASK; 548 if (unitmask & ~allowed_unitmask) /* disallow reserved bits */ 549 return (EINVAL); 550 551 if (ev == PMC_EV_P6_MMX_UOPS_EXEC) /* hardcoded mask */ 552 unitmask = P6_EVSEL_TO_UMASK(0x0F); 553 554 config = 0; 555 556 config |= P6_EVSEL_EVENT_SELECT(pevent->pm_evsel); 557 558 if (unitmask & (caps & PMC_CAP_QUALIFIER)) 559 config |= unitmask; 560 561 if (caps & PMC_CAP_THRESHOLD) 562 config |= a->pm_md.pm_ppro.pm_ppro_config & 563 P6_EVSEL_CMASK_MASK; 564 565 /* set at least one of the 'usr' or 'os' caps */ 566 if (caps & PMC_CAP_USER) 567 config |= P6_EVSEL_USR; 568 if (caps & PMC_CAP_SYSTEM) 569 config |= P6_EVSEL_OS; 570 if ((caps & (PMC_CAP_USER|PMC_CAP_SYSTEM)) == 0) 571 config |= (P6_EVSEL_USR|P6_EVSEL_OS); 572 573 if (caps & PMC_CAP_EDGE) 574 config |= P6_EVSEL_E; 575 if (caps & PMC_CAP_INVERT) 576 config |= P6_EVSEL_INV; 577 if (caps & PMC_CAP_INTERRUPT) 578 config |= P6_EVSEL_INT; 579 580 pm->pm_md.pm_ppro.pm_ppro_evsel = config; 581 582 PMCDBG(MDP,ALL,2, "p6-allocate config=0x%x", config); 583 584 return (0); 585} 586 587static int 588p6_release_pmc(int cpu, int ri, struct pmc *pm) 589{ 590 (void) pm; 591 592 PMCDBG(MDP,REL,1, "p6-release cpu=%d ri=%d pm=%p", cpu, ri, pm); 593 594 KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), 595 ("[p6,%d] illegal CPU value %d", __LINE__, cpu)); 596 KASSERT(ri >= 0 && ri < P6_NPMCS, 597 ("[p6,%d] illegal row-index %d", __LINE__, ri)); 598 599 KASSERT(p6_pcpu[cpu]->pc_p6pmcs[ri].phw_pmc == NULL, 600 ("[p6,%d] PHW pmc non-NULL", __LINE__)); 601 602 return (0); 603} 604 605static int 606p6_start_pmc(int cpu, int ri) 607{ 608 uint32_t config; 609 struct pmc *pm; 610 struct p6_cpu *pc; 611 const struct p6pmc_descr *pd; 612 613 KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), 614 ("[p6,%d] illegal CPU value %d", __LINE__, cpu)); 615 KASSERT(ri >= 0 && ri < P6_NPMCS, 616 ("[p6,%d] illegal row-index %d", __LINE__, ri)); 617 618 pc = p6_pcpu[cpu]; 619 pm = pc->pc_p6pmcs[ri].phw_pmc; 620 pd = &p6_pmcdesc[ri]; 621 622 KASSERT(pm, 623 ("[p6,%d] starting cpu%d,ri%d with no pmc configured", 624 __LINE__, cpu, ri)); 625 626 PMCDBG(MDP,STA,1, "p6-start cpu=%d ri=%d", cpu, ri); 627 628 config = pm->pm_md.pm_ppro.pm_ppro_evsel; 629 630 PMCDBG(MDP,STA,2, "p6-start/2 cpu=%d ri=%d evselmsr=0x%x config=0x%x", 631 cpu, ri, pd->pm_evsel_msr, config); 632 633 P6_MARK_STARTED(pc, ri); 634 wrmsr(pd->pm_evsel_msr, config); 635 636 P6_SYNC_CTR_STATE(pc); 637 638 return (0); 639} 640 641static int 642p6_stop_pmc(int cpu, int ri) 643{ 644 struct pmc *pm; 645 struct p6_cpu *pc; 646 struct p6pmc_descr *pd; 647 648 KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), 649 ("[p6,%d] illegal cpu value %d", __LINE__, cpu)); 650 KASSERT(ri >= 0 && ri < P6_NPMCS, 651 ("[p6,%d] illegal row index %d", __LINE__, ri)); 652 653 pc = p6_pcpu[cpu]; 654 pm = pc->pc_p6pmcs[ri].phw_pmc; 655 pd = &p6_pmcdesc[ri]; 656 657 KASSERT(pm, 658 ("[p6,%d] cpu%d ri%d no configured PMC to stop", __LINE__, 659 cpu, ri)); 660 661 PMCDBG(MDP,STO,1, "p6-stop cpu=%d ri=%d", cpu, ri); 662 663 wrmsr(pd->pm_evsel_msr, 0); /* stop hw */ 664 P6_MARK_STOPPED(pc, ri); /* update software state */ 665 666 P6_SYNC_CTR_STATE(pc); /* restart CTR1 if need be */ 667 668 PMCDBG(MDP,STO,2, "p6-stop/2 cpu=%d ri=%d", cpu, ri); 669 670 return (0); 671} 672 673static int 674p6_intr(int cpu, struct trapframe *tf) 675{ 676 int error, retval, ri; 677 uint32_t perf0cfg; 678 struct pmc *pm; 679 struct p6_cpu *pc; 680 pmc_value_t v; 681 682 KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), 683 ("[p6,%d] CPU %d out of range", __LINE__, cpu)); 684 685 retval = 0; 686 pc = p6_pcpu[cpu]; 687 688 /* stop both PMCs */ 689 perf0cfg = rdmsr(P6_MSR_EVSEL0); 690 wrmsr(P6_MSR_EVSEL0, perf0cfg & ~P6_EVSEL_EN); 691 692 for (ri = 0; ri < P6_NPMCS; ri++) { 693 694 if ((pm = pc->pc_p6pmcs[ri].phw_pmc) == NULL || 695 !PMC_IS_SAMPLING_MODE(PMC_TO_MODE(pm))) { 696 continue; 697 } 698 699 if (!P6_PMC_HAS_OVERFLOWED(ri)) 700 continue; 701 702 retval = 1; 703 704 if (pm->pm_state != PMC_STATE_RUNNING) 705 continue; 706 707 error = pmc_process_interrupt(cpu, PMC_HR, pm, tf, 708 TRAPF_USERMODE(tf)); 709 if (error) 710 P6_MARK_STOPPED(pc,ri); 711 712 /* reload sampling count */ 713 v = pm->pm_sc.pm_reloadcount; 714 wrmsr(P6_MSR_PERFCTR0 + ri, 715 P6_RELOAD_COUNT_TO_PERFCTR_VALUE(v)); 716 717 } 718 719 /* 720 * On P6 processors, the LAPIC needs to have its PMC interrupt 721 * unmasked after a PMC interrupt. 722 */ 723 if (retval) 724 lapic_reenable_pmc(); 725 726 atomic_add_int(retval ? &pmc_stats.pm_intr_processed : 727 &pmc_stats.pm_intr_ignored, 1); 728 729 /* restart counters that can be restarted */ 730 P6_SYNC_CTR_STATE(pc); 731 732 return (retval); 733} 734 735static int 736p6_describe(int cpu, int ri, struct pmc_info *pi, 737 struct pmc **ppmc) 738{ 739 int error; 740 size_t copied; 741 struct pmc_hw *phw; 742 struct p6pmc_descr *pd; 743 744 KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), 745 ("[p6,%d] illegal CPU %d", __LINE__, cpu)); 746 KASSERT(ri >= 0 && ri < P6_NPMCS, 747 ("[p6,%d] row-index %d out of range", __LINE__, ri)); 748 749 phw = pmc_pcpu[cpu]->pc_hwpmcs[ri]; 750 pd = &p6_pmcdesc[ri]; 751 752 KASSERT(phw == &p6_pcpu[cpu]->pc_p6pmcs[ri], 753 ("[p6,%d] phw mismatch", __LINE__)); 754 755 if ((error = copystr(pd->pm_descr.pd_name, pi->pm_name, 756 PMC_NAME_MAX, &copied)) != 0) 757 return (error); 758 759 pi->pm_class = pd->pm_descr.pd_class; 760 761 if (phw->phw_state & PMC_PHW_FLAG_IS_ENABLED) { 762 pi->pm_enabled = TRUE; 763 *ppmc = phw->phw_pmc; 764 } else { 765 pi->pm_enabled = FALSE; 766 *ppmc = NULL; 767 } 768 769 return (0); 770} 771 772static int 773p6_get_msr(int ri, uint32_t *msr) 774{ 775 KASSERT(ri >= 0 && ri < P6_NPMCS, 776 ("[p6,%d ri %d out of range", __LINE__, ri)); 777 778 *msr = p6_pmcdesc[ri].pm_pmc_msr - P6_MSR_PERFCTR0; 779 780 return (0); 781} 782 783int 784pmc_p6_initialize(struct pmc_mdep *md, int ncpus) 785{ 786 struct pmc_classdep *pcd; 787 788 KASSERT(cpu_vendor_id == CPU_VENDOR_INTEL, 789 ("[p6,%d] Initializing non-intel processor", __LINE__)); 790 791 PMCDBG(MDP,INI,1, "%s", "p6-initialize"); 792 793 /* Allocate space for pointers to per-cpu descriptors. */ 794 p6_pcpu = malloc(sizeof(struct p6_cpu **) * ncpus, M_PMC, 795 M_ZERO|M_WAITOK); 796 797 /* Fill in the class dependent descriptor. */ 798 pcd = &md->pmd_classdep[PMC_MDEP_CLASS_INDEX_P6]; 799 800 switch (md->pmd_cputype) { 801 802 /* 803 * P6 Family Processors 804 */ 805 case PMC_CPU_INTEL_P6: 806 case PMC_CPU_INTEL_CL: 807 case PMC_CPU_INTEL_PII: 808 case PMC_CPU_INTEL_PIII: 809 case PMC_CPU_INTEL_PM: 810 811 p6_cputype = md->pmd_cputype; 812 813 pcd->pcd_caps = P6_PMC_CAPS; 814 pcd->pcd_class = PMC_CLASS_P6; 815 pcd->pcd_num = P6_NPMCS; 816 pcd->pcd_ri = md->pmd_npmc; 817 pcd->pcd_width = 40; 818 819 pcd->pcd_allocate_pmc = p6_allocate_pmc; 820 pcd->pcd_config_pmc = p6_config_pmc; 821 pcd->pcd_describe = p6_describe; 822 pcd->pcd_get_config = p6_get_config; 823 pcd->pcd_get_msr = p6_get_msr; 824 pcd->pcd_pcpu_fini = p6_pcpu_fini; 825 pcd->pcd_pcpu_init = p6_pcpu_init; 826 pcd->pcd_read_pmc = p6_read_pmc; 827 pcd->pcd_release_pmc = p6_release_pmc; 828 pcd->pcd_start_pmc = p6_start_pmc; 829 pcd->pcd_stop_pmc = p6_stop_pmc; 830 pcd->pcd_write_pmc = p6_write_pmc; 831 832 md->pmd_pcpu_fini = NULL; 833 md->pmd_pcpu_init = NULL; 834 md->pmd_intr = p6_intr; 835 836 md->pmd_npmc += P6_NPMCS; 837 838 break; 839 840 default: 841 KASSERT(0,("[p6,%d] Unknown CPU type", __LINE__)); 842 return ENOSYS; 843 } 844 845 return (0); 846} 847 848void 849pmc_p6_finalize(struct pmc_mdep *md) 850{ 851#if defined(INVARIANTS) 852 int i, ncpus; 853#endif 854 855 KASSERT(p6_pcpu != NULL, ("[p6,%d] NULL p6_pcpu", __LINE__)); 856 857#if defined(INVARIANTS) 858 ncpus = pmc_cpu_max(); 859 for (i = 0; i < ncpus; i++) 860 KASSERT(p6_pcpu[i] == NULL, ("[p6,%d] non-null pcpu %d", 861 __LINE__, i)); 862#endif 863 864 free(p6_pcpu, M_PMC); 865 p6_pcpu = NULL; 866} 867