1/* 2 * Copyright (c) 2012 Apple Inc. All rights reserved. 3 * 4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ 5 * 6 * This file contains Original Code and/or Modifications of Original Code 7 * as defined in and that are subject to the Apple Public Source License 8 * Version 2.0 (the 'License'). You may not use this file except in 9 * compliance with the License. The rights granted to you under the License 10 * may not be used to create, or enable the creation or redistribution of, 11 * unlawful or unlicensed copies of an Apple operating system, or to 12 * circumvent, violate, or enable the circumvention or violation of, any 13 * terms of an Apple operating system software license agreement. 14 * 15 * Please obtain a copy of the License at 16 * http://www.opensource.apple.com/apsl/ and read it before using this file. 17 * 18 * The Original Code and all software distributed under the License are 19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER 20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, 21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, 22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. 23 * Please see the License for the specific language governing rights and 24 * limitations under the License. 25 * 26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ 27 */ 28 29#include <mach/mach_types.h> 30#include <machine/machine_routines.h> 31#include <kern/processor.h> 32#include <kern/kalloc.h> 33#include <i386/cpuid.h> 34#include <i386/proc_reg.h> 35#include <i386/mp.h> 36#include <i386/lapic.h> 37#include <sys/errno.h> 38#include <kperf/buffer.h> 39 40#include <kern/kpc.h> 41 42#include <kperf/kperf.h> 43#include <kperf/sample.h> 44#include <kperf/context.h> 45#include <kperf/action.h> 46 47#include <chud/chud_xnu.h> 48 49 50 51/* Fixed counter mask -- three counters, each with OS and USER */ 52#define IA32_FIXED_CTR_ENABLE_ALL_CTRS_ALL_RINGS (0x333) 53#define IA32_FIXED_CTR_ENABLE_ALL_PMI (0x888) 54 55#define IA32_PERFEVTSEL_PMI (1ull << 20) 56#define IA32_PERFEVTSEL_EN (1ull << 22) 57 58/* Non-serialising */ 59#define USE_RDPMC 60 61#define RDPMC_FIXED_COUNTER_SELECTOR (1ULL<<30) 62 63/* track the last config we enabled */ 64static uint32_t kpc_running = 0; 65 66/* PMC / MSR accesses */ 67 68static uint64_t 69IA32_FIXED_CTR_CTRL(void) 70{ 71 return rdmsr64( MSR_IA32_PERF_FIXED_CTR_CTRL ); 72} 73 74static uint64_t 75IA32_FIXED_CTRx(uint32_t ctr) 76{ 77#ifdef USE_RDPMC 78 return rdpmc64(RDPMC_FIXED_COUNTER_SELECTOR | ctr); 79#else /* !USE_RDPMC */ 80 return rdmsr64(MSR_IA32_PERF_FIXED_CTR0 + ctr); 81#endif /* !USE_RDPMC */ 82} 83 84#ifdef FIXED_COUNTER_RELOAD 85static void 86wrIA32_FIXED_CTRx(uint32_t ctr, uint64_t value) 87{ 88 return wrmsr64(MSR_IA32_PERF_FIXED_CTR0 + ctr, value); 89} 90#endif 91 92static uint64_t 93IA32_PMCx(uint32_t ctr) 94{ 95#ifdef USE_RDPMC 96 return rdpmc64(ctr); 97#else /* !USE_RDPMC */ 98 return rdmsr64(MSR_IA32_PERFCTR0 + ctr); 99#endif /* !USE_RDPMC */ 100} 101 102static void 103wrIA32_PMCx(uint32_t ctr, uint64_t value) 104{ 105 return wrmsr64(MSR_IA32_PERFCTR0 + ctr, value); 106} 107 108static uint64_t 109IA32_PERFEVTSELx(uint32_t ctr) 110{ 111 return rdmsr64(MSR_IA32_EVNTSEL0 + ctr); 112} 113 114static void 115wrIA32_PERFEVTSELx(uint32_t ctr, uint64_t value) 116{ 117 wrmsr64(MSR_IA32_EVNTSEL0 + ctr, value); 118} 119 120 121/* internal functions */ 122 123boolean_t 124kpc_is_running_fixed(void) 125{ 126 return (kpc_running & KPC_CLASS_FIXED_MASK) == KPC_CLASS_FIXED_MASK; 127} 128 129boolean_t 130kpc_is_running_configurable(void) 131{ 132 return (kpc_running & KPC_CLASS_CONFIGURABLE_MASK) == KPC_CLASS_CONFIGURABLE_MASK; 133} 134 135uint32_t 136kpc_fixed_count(void) 137{ 138 i386_cpu_info_t *info = NULL; 139 140 info = cpuid_info(); 141 142 return info->cpuid_arch_perf_leaf.fixed_number; 143} 144 145uint32_t 146kpc_configurable_count(void) 147{ 148 i386_cpu_info_t *info = NULL; 149 150 info = cpuid_info(); 151 152 return info->cpuid_arch_perf_leaf.number; 153} 154 155uint32_t 156kpc_fixed_config_count(void) 157{ 158 return KPC_X86_64_FIXED_CONFIGS; 159} 160 161uint32_t 162kpc_configurable_config_count(void) 163{ 164 return kpc_configurable_count(); 165} 166 167uint32_t 168kpc_rawpmu_config_count(void) 169{ 170 // RAW PMU access not implemented. 171 return 0; 172} 173 174int 175kpc_get_rawpmu_config(__unused kpc_config_t *configv) 176{ 177 return 0; 178} 179 180static uint8_t 181kpc_fixed_width(void) 182{ 183 i386_cpu_info_t *info = NULL; 184 185 info = cpuid_info(); 186 187 return info->cpuid_arch_perf_leaf.fixed_width; 188} 189 190static uint8_t 191kpc_configurable_width(void) 192{ 193 i386_cpu_info_t *info = NULL; 194 195 info = cpuid_info(); 196 197 return info->cpuid_arch_perf_leaf.width; 198} 199 200uint64_t 201kpc_fixed_max(void) 202{ 203 return (1ULL << kpc_fixed_width()) - 1; 204} 205 206uint64_t 207kpc_configurable_max(void) 208{ 209 return (1ULL << kpc_configurable_width()) - 1; 210} 211 212#ifdef FIXED_COUNTER_SHADOW 213static uint64_t 214kpc_reload_fixed(int ctr) 215{ 216 uint64_t old = IA32_FIXED_CTRx(ctr); 217 wrIA32_FIXED_CTRx(ctr, FIXED_RELOAD(ctr)); 218 return old; 219} 220#endif 221 222static uint64_t 223kpc_reload_configurable(int ctr) 224{ 225 uint64_t cfg = IA32_PERFEVTSELx(ctr); 226 227 /* counters must be disabled before they can be written to */ 228 uint64_t old = IA32_PMCx(ctr); 229 wrIA32_PERFEVTSELx(ctr, cfg & ~IA32_PERFEVTSEL_EN); 230 wrIA32_PMCx(ctr, CONFIGURABLE_RELOAD(ctr)); 231 wrIA32_PERFEVTSELx(ctr, cfg); 232 return old; 233} 234 235void kpc_pmi_handler(x86_saved_state_t *state); 236 237static void 238set_running_fixed(boolean_t on) 239{ 240 uint64_t global = 0, mask = 0, fixed_ctrl = 0; 241 int i; 242 boolean_t enabled; 243 244 if( on ) 245 /* these are per-thread in SMT */ 246 fixed_ctrl = IA32_FIXED_CTR_ENABLE_ALL_CTRS_ALL_RINGS | IA32_FIXED_CTR_ENABLE_ALL_PMI; 247 else 248 /* don't allow disabling fixed counters */ 249 return; 250 251 wrmsr64( MSR_IA32_PERF_FIXED_CTR_CTRL, fixed_ctrl ); 252 253 enabled = ml_set_interrupts_enabled(FALSE); 254 255 /* rmw the global control */ 256 global = rdmsr64(MSR_IA32_PERF_GLOBAL_CTRL); 257 for( i = 0; i < (int) kpc_fixed_count(); i++ ) 258 mask |= (1ULL<<(32+i)); 259 260 if( on ) 261 global |= mask; 262 else 263 global &= ~mask; 264 265 wrmsr64(MSR_IA32_PERF_GLOBAL_CTRL, global); 266 267 ml_set_interrupts_enabled(enabled); 268} 269 270static void 271set_running_configurable(boolean_t on) 272{ 273 uint64_t global = 0, mask = 0; 274 uint64_t cfg, save; 275 int i; 276 boolean_t enabled; 277 int ncnt = (int) kpc_get_counter_count(KPC_CLASS_CONFIGURABLE_MASK); 278 279 enabled = ml_set_interrupts_enabled(FALSE); 280 281 /* rmw the global control */ 282 global = rdmsr64(MSR_IA32_PERF_GLOBAL_CTRL); 283 for( i = 0; i < ncnt; i++ ) { 284 mask |= (1ULL<<i); 285 286 /* need to save and restore counter since it resets when reconfigured */ 287 cfg = IA32_PERFEVTSELx(i); 288 save = IA32_PMCx(i); 289 wrIA32_PERFEVTSELx(i, cfg | IA32_PERFEVTSEL_PMI | IA32_PERFEVTSEL_EN); 290 wrIA32_PMCx(i, save); 291 } 292 293 if( on ) 294 global |= mask; 295 else 296 global &= ~mask; 297 298 wrmsr64(MSR_IA32_PERF_GLOBAL_CTRL, global); 299 300 ml_set_interrupts_enabled(enabled); 301} 302 303static void 304kpc_set_running_mp_call( void *vstate ) 305{ 306 uint32_t new_state = *(uint32_t*)vstate; 307 308 set_running_fixed((new_state & KPC_CLASS_FIXED_MASK) != 0); 309 set_running_configurable((new_state & KPC_CLASS_CONFIGURABLE_MASK) != 0); 310} 311 312int 313kpc_get_fixed_config(kpc_config_t *configv) 314{ 315 configv[0] = IA32_FIXED_CTR_CTRL(); 316 317 return 0; 318} 319 320static int 321kpc_set_fixed_config(kpc_config_t *configv) 322{ 323 (void) configv; 324 325 /* NYI */ 326 return -1; 327} 328 329int 330kpc_get_fixed_counters(uint64_t *counterv) 331{ 332 int i, n = kpc_fixed_count(); 333 334#ifdef FIXED_COUNTER_SHADOW 335 uint64_t status; 336 337 /* snap the counters */ 338 for( i = 0; i < n; i++ ) { 339 counterv[i] = FIXED_SHADOW(ctr) + 340 (IA32_FIXED_CTRx(i) - FIXED_RELOAD(ctr)); 341 } 342 343 /* Grab the overflow bits */ 344 status = rdmsr64(MSR_IA32_PERF_GLOBAL_STATUS); 345 346 /* If the overflow bit is set for a counter, our previous read may or may not have been 347 * before the counter overflowed. Re-read any counter with it's overflow bit set so 348 * we know for sure that it has overflowed. The reason this matters is that the math 349 * is different for a counter that has overflowed. */ 350 for( i = 0; i < n; i++ ) { 351 if ((1ull << (i + 32)) & status) 352 counterv[i] = FIXED_SHADOW(ctr) + 353 (kpc_fixed_max() - FIXED_RELOAD(ctr) + 1 /* Wrap */) + IA32_FIXED_CTRx(i); 354 } 355#else 356 for( i = 0; i < n; i++ ) 357 counterv[i] = IA32_FIXED_CTRx(i); 358#endif 359 360 return 0; 361} 362 363int 364kpc_get_configurable_config(kpc_config_t *configv) 365{ 366 int i, n = kpc_get_config_count(KPC_CLASS_CONFIGURABLE_MASK); 367 368 for( i = 0; i < n; i++ ) 369 configv[i] = IA32_PERFEVTSELx(i); 370 371 return 0; 372} 373 374static int 375kpc_set_configurable_config(kpc_config_t *configv) 376{ 377 int i, n = kpc_get_config_count(KPC_CLASS_CONFIGURABLE_MASK); 378 uint64_t save; 379 380 for( i = 0; i < n; i++ ) { 381 /* need to save and restore counter since it resets when reconfigured */ 382 save = IA32_PMCx(i); 383 /* 384 * Some bits are not safe to set from user space. 385 * Allow these bits to be set: 386 * 387 * 0-7 Event select 388 * 8-15 UMASK 389 * 16 USR 390 * 17 OS 391 * 18 E 392 * 22 EN 393 * 23 INV 394 * 24-31 CMASK 395 * 396 * Excluding: 397 * 398 * 19 PC 399 * 20 INT 400 * 21 AnyThread 401 * 32 IN_TX 402 * 33 IN_TXCP 403 * 34-63 Reserved 404 */ 405 wrIA32_PERFEVTSELx(i, configv[i] & 0xffc7ffffull); 406 wrIA32_PMCx(i, save); 407 } 408 409 return 0; 410} 411 412int 413kpc_get_configurable_counters(uint64_t *counterv) 414{ 415 int i, n = kpc_get_config_count(KPC_CLASS_CONFIGURABLE_MASK); 416 uint64_t status; 417 418 /* snap the counters */ 419 for( i = 0; i < n; i++ ) { 420 counterv[i] = CONFIGURABLE_SHADOW(i) + 421 (IA32_PMCx(i) - CONFIGURABLE_RELOAD(i)); 422 } 423 424 /* Grab the overflow bits */ 425 status = rdmsr64(MSR_IA32_PERF_GLOBAL_STATUS); 426 427 /* If the overflow bit is set for a counter, our previous read may or may not have been 428 * before the counter overflowed. Re-read any counter with it's overflow bit set so 429 * we know for sure that it has overflowed. The reason this matters is that the math 430 * is different for a counter that has overflowed. */ 431 for( i = 0; i < n; i++ ) { 432 if ((1ull << i) & status) { 433 counterv[i] = CONFIGURABLE_SHADOW(i) + 434 (kpc_configurable_max() - CONFIGURABLE_RELOAD(i)) + IA32_PMCx(i); 435 } 436 } 437 438 return 0; 439} 440 441static void 442kpc_set_config_mp_call(void *vmp_config) 443{ 444 struct kpc_config_remote *mp_config = vmp_config; 445 uint32_t classes = mp_config->classes; 446 kpc_config_t *new_config = mp_config->configv; 447 int count = 0; 448 boolean_t enabled; 449 450 enabled = ml_set_interrupts_enabled(FALSE); 451 452 if( classes & KPC_CLASS_FIXED_MASK ) 453 { 454 kpc_set_fixed_config(&new_config[count]); 455 count += kpc_get_config_count(KPC_CLASS_FIXED_MASK); 456 } 457 458 if( classes & KPC_CLASS_CONFIGURABLE_MASK ) 459 { 460 kpc_set_configurable_config(&new_config[count]); 461 count += kpc_get_config_count(KPC_CLASS_CONFIGURABLE_MASK); 462 } 463 464 ml_set_interrupts_enabled(enabled); 465} 466 467static void 468kpc_set_reload_mp_call(void *vmp_config) 469{ 470 struct kpc_config_remote *mp_config = vmp_config; 471 uint64_t max = kpc_configurable_max(); 472 uint32_t i, count = kpc_get_counter_count(KPC_CLASS_CONFIGURABLE_MASK); 473 uint64_t *new_period; 474 uint64_t classes; 475 int enabled; 476 477 classes = mp_config->classes; 478 new_period = mp_config->configv; 479 480 if (classes & KPC_CLASS_CONFIGURABLE_MASK) { 481 enabled = ml_set_interrupts_enabled(FALSE); 482 483 kpc_get_configurable_counters(&CONFIGURABLE_SHADOW(0)); 484 485 for (i = 0; i < count; i++) { 486 if (new_period[i] == 0) 487 new_period[i] = kpc_configurable_max(); 488 489 CONFIGURABLE_RELOAD(i) = max - new_period[i]; 490 491 kpc_reload_configurable(i); 492 493 /* clear overflow bit just in case */ 494 wrmsr64(MSR_IA32_PERF_GLOBAL_OVF_CTRL, 1ull << i); 495 } 496 497 ml_set_interrupts_enabled(enabled); 498 } 499} 500 501int 502kpc_set_period_arch( struct kpc_config_remote *mp_config ) 503{ 504 mp_cpus_call( CPUMASK_ALL, ASYNC, kpc_set_reload_mp_call, mp_config ); 505 506 return 0; 507} 508 509 510/* interface functions */ 511 512void 513kpc_arch_init(void) 514{ 515 /* No-op */ 516} 517 518uint32_t 519kpc_get_classes(void) 520{ 521 return KPC_CLASS_FIXED_MASK | KPC_CLASS_CONFIGURABLE_MASK; 522} 523 524int 525kpc_set_running(uint32_t new_state) 526{ 527 lapic_set_pmi_func((i386_intr_func_t)kpc_pmi_handler); 528 529 /* dispatch to all CPUs */ 530 mp_cpus_call( CPUMASK_ALL, ASYNC, kpc_set_running_mp_call, &new_state ); 531 532 kpc_running = new_state; 533 534 return 0; 535} 536 537int 538kpc_set_config_arch(struct kpc_config_remote *mp_config) 539{ 540 mp_cpus_call( CPUMASK_ALL, ASYNC, kpc_set_config_mp_call, mp_config ); 541 542 return 0; 543} 544 545/* PMI stuff */ 546void kpc_pmi_handler(__unused x86_saved_state_t *state) 547{ 548 uint64_t status, extra; 549 uint32_t ctr; 550 int enabled; 551 552 enabled = ml_set_interrupts_enabled(FALSE); 553 554 status = rdmsr64(MSR_IA32_PERF_GLOBAL_STATUS); 555 556#ifdef FIXED_COUNTER_SHADOW 557 for (ctr = 0; ctr < kpc_fixed_count(); ctr++) { 558 if ((1ULL << (ctr + 32)) & status) { 559 extra = kpc_reload_fixed(ctr); 560 561 FIXED_SHADOW(ctr) 562 += (kpc_fixed_max() - FIXED_RELOAD(ctr) + 1 /* Wrap */) + extra; 563 564 BUF_INFO(PERF_KPC_FCOUNTER, ctr, FIXED_SHADOW(ctr), extra, FIXED_ACTIONID(ctr)); 565 566 if (FIXED_ACTIONID(ctr)) 567 kpc_sample_kperf(FIXED_ACTIONID(ctr)); 568 } 569 } 570#endif 571 572 for (ctr = 0; ctr < kpc_configurable_count(); ctr++) { 573 if ((1ULL << ctr) & status) { 574 extra = kpc_reload_configurable(ctr); 575 576 CONFIGURABLE_SHADOW(ctr) 577 += kpc_configurable_max() - CONFIGURABLE_RELOAD(ctr) + extra; 578 579 /* kperf can grab the PMCs when it samples so we need to make sure the overflow 580 * bits are in the correct state before the call to kperf_sample */ 581 wrmsr64(MSR_IA32_PERF_GLOBAL_OVF_CTRL, 1ull << ctr); 582 583 BUF_INFO(PERF_KPC_COUNTER, ctr, CONFIGURABLE_SHADOW(ctr), extra, CONFIGURABLE_ACTIONID(ctr)); 584 585 if (CONFIGURABLE_ACTIONID(ctr)) 586 kpc_sample_kperf(CONFIGURABLE_ACTIONID(ctr)); 587 } 588 } 589 590 ml_set_interrupts_enabled(enabled); 591} 592 593int 594kpc_force_all_ctrs_arch( task_t task __unused, int val __unused ) 595{ 596 /* TODO: reclaim counters ownership from XCPM */ 597 return 0; 598} 599 600int 601kpc_set_sw_inc( uint32_t mask __unused ) 602{ 603 return ENOTSUP; 604} 605