1/* 2 * Copyright (c) 2011 Apple Computer, Inc. All rights reserved. 3 * 4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ 5 * 6 * This file contains Original Code and/or Modifications of Original Code 7 * as defined in and that are subject to the Apple Public Source License 8 * Version 2.0 (the 'License'). You may not use this file except in 9 * compliance with the License. The rights granted to you under the License 10 * may not be used to create, or enable the creation or redistribution of, 11 * unlawful or unlicensed copies of an Apple operating system, or to 12 * circumvent, violate, or enable the circumvention or violation of, any 13 * terms of an Apple operating system software license agreement. 14 * 15 * Please obtain a copy of the License at 16 * http://www.opensource.apple.com/apsl/ and read it before using this file. 17 * 18 * The Original Code and all software distributed under the License are 19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER 20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, 21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, 22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. 23 * Please see the License for the specific language governing rights and 24 * limitations under the License. 25 * 26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ 27 */ 28 29/* 30 * Called from a trigger. Actually takes the data from the different 31 * modules and puts them in a buffer 32 */ 33 34#include <mach/mach_types.h> 35#include <machine/machine_routines.h> 36// #include <libkern/libkern.h> 37#include <kern/kalloc.h> 38#include <kern/debug.h> /* panic */ 39#include <kern/thread.h> 40#include <sys/errno.h> 41 42#include <chud/chud_xnu.h> 43#include <kperf/kperf.h> 44 45#include <kperf/buffer.h> 46#include <kperf/timetrigger.h> 47#include <kperf/threadinfo.h> 48#include <kperf/callstack.h> 49#include <kperf/sample.h> 50#include <kperf/action.h> 51#include <kperf/context.h> 52#include <kperf/ast.h> 53 54#define ACTION_MAX 32 55 56/* the list of different actions to take */ 57struct action 58{ 59 uint32_t sample; 60 uint32_t userdata; 61 int pid_filter; 62}; 63 64/* the list of actions */ 65static unsigned actionc = 0; 66static struct action *actionv = NULL; 67 68/* whether to record callstacks on kdebug events */ 69static int kdebug_callstack_action = 0; 70 71/* whether we get a callback on a thread switch */ 72int kperf_cswitch_hook = 0; 73 74/* indirect hooks to play nice with CHUD for the transition to kperf */ 75kern_return_t chudxnu_kdebug_callback_enter(chudxnu_kdebug_callback_func_t fn); 76kern_return_t chudxnu_kdebug_callback_cancel(void); 77 78/* Do the real work! */ 79/* this can be called in any context ... right? */ 80static kern_return_t 81kperf_sample_internal( struct kperf_sample *sbuf, 82 struct kperf_context *context, 83 unsigned sample_what, unsigned sample_flags, 84 unsigned actionid ) 85{ 86 boolean_t enabled; 87 int did_ucallstack = 0, did_tinfo_extra = 0; 88 uint32_t userdata; 89 90 /* not much point continuing here, but what to do ? return 91 * Shutdown? cut a tracepoint and continue? 92 */ 93 if( sample_what == 0 ) 94 return SAMPLE_CONTINUE; 95 96 int is_kernel = (context->cur_pid == 0); 97 98 sbuf->kcallstack.nframes = 0; 99 sbuf->kcallstack.flags = CALLSTACK_VALID; 100 sbuf->ucallstack.nframes = 0; 101 sbuf->ucallstack.flags = CALLSTACK_VALID; 102 103 /* an event occurred. Sample everything and dump it in a 104 * buffer. 105 */ 106 107 /* collect data from samplers */ 108 if( sample_what & SAMPLER_TINFO ) { 109 kperf_threadinfo_sample( &sbuf->threadinfo, context ); 110 111 /* See if we should drop idle thread samples */ 112 if( !(sample_flags & SAMPLE_FLAG_IDLE_THREADS) ) 113 if (sbuf->threadinfo.runmode & 0x40) 114 return SAMPLE_CONTINUE; 115 } 116 117 if( (sample_what & SAMPLER_KSTACK) && !(sample_flags & SAMPLE_FLAG_EMPTY_CALLSTACK) ) 118 kperf_kcallstack_sample( &sbuf->kcallstack, context ); 119 120 /* sensitive ones */ 121 if ( !is_kernel ) { 122 if( sample_flags & SAMPLE_FLAG_PEND_USER ) 123 { 124 if( (sample_what & SAMPLER_USTACK) && !(sample_flags & SAMPLE_FLAG_EMPTY_CALLSTACK) ) 125 did_ucallstack = kperf_ucallstack_pend( context ); 126 127 if( sample_what & SAMPLER_TINFOEX ) 128 did_tinfo_extra = kperf_threadinfo_extra_pend( context ); 129 } 130 else 131 { 132 if( (sample_what & SAMPLER_USTACK) && !(sample_flags & SAMPLE_FLAG_EMPTY_CALLSTACK) ) 133 kperf_ucallstack_sample( &sbuf->ucallstack, context ); 134 135 if( sample_what & SAMPLER_TINFOEX ) 136 kperf_threadinfo_extra_sample( &sbuf->tinfo_ex, 137 context ); 138 } 139 } 140 141#if KPC 142 if ( sample_what & SAMPLER_PMC_CPU ) 143 kperf_kpc_cpu_sample( &sbuf->kpcdata, 144 (sample_what & SAMPLER_PMC_CPU) != 0 ); 145#endif 146 147 /* lookup the user tag, if any */ 148 if( actionid 149 && (actionid <= actionc) ) 150 userdata = actionv[actionid-1].userdata; 151 else 152 userdata = actionid; 153 154 /* stash the data into the buffer 155 * interrupts off to ensure we don't get split 156 */ 157 enabled = ml_set_interrupts_enabled(FALSE); 158 159 BUF_DATA( PERF_GEN_EVENT | DBG_FUNC_START, sample_what, 160 actionid, userdata, sample_flags ); 161 162 /* dump threadinfo */ 163 if( sample_what & SAMPLER_TINFO ) 164 kperf_threadinfo_log( &sbuf->threadinfo ); 165 166 /* dump kcallstack */ 167 if( sample_what & SAMPLER_KSTACK ) 168 kperf_kcallstack_log( &sbuf->kcallstack ); 169 170 171 /* dump user stuff */ 172 if ( !is_kernel ) { 173 if ( sample_flags & SAMPLE_FLAG_PEND_USER ) 174 { 175 if ( did_ucallstack ) 176 BUF_INFO1( PERF_CS_UPEND, 0 ); 177 178 if ( did_tinfo_extra ) 179 BUF_INFO1( PERF_TI_XPEND, 0 ); 180 } 181 else 182 { 183 if( sample_what & SAMPLER_USTACK ) 184 kperf_ucallstack_log( &sbuf->ucallstack ); 185 186 if( sample_what & SAMPLER_TINFOEX ) 187 kperf_threadinfo_extra_log( &sbuf->tinfo_ex ); 188 } 189 } 190 191#if KPC 192 if ( sample_what & SAMPLER_PMC_CPU ) 193 kperf_kpc_cpu_log( &sbuf->kpcdata ); 194 195#endif 196 197 BUF_DATA1( PERF_GEN_EVENT | DBG_FUNC_END, sample_what ); 198 199 /* intrs back on */ 200 ml_set_interrupts_enabled(enabled); 201 202 return SAMPLE_CONTINUE; 203} 204 205/* Translate actionid into sample bits and take a sample */ 206kern_return_t 207kperf_sample( struct kperf_sample *sbuf, 208 struct kperf_context *context, 209 unsigned actionid, unsigned sample_flags ) 210{ 211 unsigned sample_what = 0; 212 int pid_filter; 213 214 /* work out what to sample, if anything */ 215 if( (actionid > actionc) || (actionid == 0) ) 216 return SAMPLE_SHUTDOWN; 217 218 /* check the pid filter against the context's current pid. 219 * filter pid == -1 means any pid 220 */ 221 pid_filter = actionv[actionid-1].pid_filter; 222 if( (pid_filter != -1) 223 && (pid_filter != context->cur_pid) ) 224 return SAMPLE_CONTINUE; 225 226 /* the samplers to run */ 227 sample_what = actionv[actionid-1].sample; 228 229 /* do the actual sample operation */ 230 return kperf_sample_internal( sbuf, context, sample_what, 231 sample_flags, actionid ); 232} 233 234/* ast callback on a thread */ 235void 236kperf_thread_ast_handler( thread_t thread ) 237{ 238 int r; 239 uint32_t t_chud; 240 unsigned sample_what = 0; 241 /* we know we're on a thread, so let's do stuff */ 242 task_t task = NULL; 243 244 BUF_INFO1(PERF_AST_HNDLR | DBG_FUNC_START, thread); 245 246 /* use ~2kb of the stack for the sample, should be ok since we're in the ast */ 247 struct kperf_sample sbuf; 248 bzero(&sbuf, sizeof(struct kperf_sample)); 249 250 /* make a context, take a sample */ 251 struct kperf_context ctx; 252 ctx.cur_thread = thread; 253 ctx.cur_pid = -1; 254 255 task = chudxnu_task_for_thread(thread); 256 if(task) 257 ctx.cur_pid = chudxnu_pid_for_task(task); 258 259 /* decode the chud bits so we know what to sample */ 260 t_chud = kperf_get_thread_bits(thread); 261 262 if (t_chud & T_AST_NAME) 263 sample_what |= SAMPLER_TINFOEX; 264 265 if (t_chud & T_AST_CALLSTACK) 266 { 267 sample_what |= SAMPLER_USTACK; 268 sample_what |= SAMPLER_TINFO; 269 } 270 271 /* do the sample, just of the user stuff */ 272 r = kperf_sample_internal( &sbuf, &ctx, sample_what, 0, 0 ); 273 274 BUF_INFO1(PERF_AST_HNDLR | DBG_FUNC_END, r); 275} 276 277/* register AST bits */ 278int 279kperf_ast_pend( thread_t cur_thread, uint32_t check_bits, 280 uint32_t set_bits ) 281{ 282 /* pend on the thread */ 283 uint32_t t_chud, set_done = 0; 284 285 /* can only pend on the current thread */ 286 if( cur_thread != chudxnu_current_thread() ) 287 panic("pending to non-current thread"); 288 289 /* get our current bits */ 290 t_chud = kperf_get_thread_bits(cur_thread); 291 292 /* see if it's already been done or pended */ 293 if( !(t_chud & check_bits ) ) 294 { 295 /* set the bit on the thread */ 296 t_chud |= set_bits; 297 kperf_set_thread_bits(cur_thread, t_chud); 298 299 /* set the actual AST */ 300 kperf_set_thread_ast( cur_thread ); 301 302 set_done = 1; 303 } 304 305 return set_done; 306 307// BUF_INFO3( dbg_code, (uintptr_t)cur_thread, t_chud, set_done ); 308} 309 310/* 311 * kdebug callback & stack management 312 */ 313 314#define IS_END(debugid) ((debugid & 3) == DBG_FUNC_END) 315#define IS_MIG(debugid) (IS_END(debugid) && ((debugid & 0xff000000U) == KDBG_CLASS_ENCODE((unsigned)DBG_MIG, 0U))) 316#define IS_MACH_SYSCALL(debugid) (IS_END(debugid) && (KDBG_CLASS_DECODE(debugid) == KDBG_CLASS_ENCODE(DBG_MACH, DBG_MACH_EXCP_SC))) 317#define IS_VM_FAULT(debugid) (IS_END(debugid) && (KDBG_CLASS_DECODE(debugid) == KDBG_CLASS_ENCODE(DBG_MACH, DBG_MACH_VM))) 318#define IS_BSD_SYSCTLL(debugid) (IS_END(debugid) && (KDBG_CLASS_DECODE(debugid) == KDBG_CLASS_ENCODE(DBG_BSD, DBG_BSD_EXCP_SC))) 319#define IS_APPS_SIGNPOST(debugid) (IS_END(debugid) && (KDBG_CLASS_DECODE(debugid) == KDBG_CLASS_ENCODE(DBG_APPS, DBG_MACH_CHUD))) 320#define IS_MACH_SIGNPOST(debugid) (IS_END(debugid) && (KDBG_CLASS_DECODE(debugid) == KDBG_CLASS_ENCODE(DBG_MACH, DBG_MACH_CHUD))) 321 322void 323kperf_kdebug_callback(uint32_t debugid) 324{ 325 int cur_pid = 0; 326 task_t task = NULL; 327 328 /* if we're not doing kperf callback stacks, return */ 329 if( !kdebug_callstack_action ) 330 return; 331 332 /* if we're looking at a kperf tracepoint, don't recurse */ 333 if( (debugid & 0xff000000) == KDBG_CLASS_ENCODE(DBG_PERF, 0) ) 334 return; 335 336 /* ensure interrupts are already off thanks to kdebug */ 337 if( ml_get_interrupts_enabled() ) 338 return; 339 340 /* make sure we're not being called recursively. */ 341#if NOTYET 342 if( kperf_kdbg_recurse(KPERF_RECURSE_IN) ) 343 return; 344#endif 345 346 /* check the happy list of trace codes */ 347 if( !( IS_MIG(debugid) 348 || IS_MACH_SYSCALL(debugid) 349 || IS_VM_FAULT(debugid) 350 || IS_BSD_SYSCTLL(debugid) 351 || IS_MACH_SIGNPOST(debugid) 352 || IS_APPS_SIGNPOST(debugid) ) ) 353 return; 354 355 /* check for kernel */ 356 thread_t thread = chudxnu_current_thread(); 357 task = chudxnu_task_for_thread(thread); 358 if(task) 359 cur_pid = chudxnu_pid_for_task(task); 360 if( !cur_pid ) 361 return; 362 363#if NOTYET 364 /* setup a context */ 365 struct kperf_context ctx; 366 struct kperf_sample *intbuf = NULL; 367 368 ctx.cur_thread = thread; 369 ctx.cur_pid = cur_pid; 370 ctx.trigger_type = TRIGGER_TYPE_TRACE; 371 ctx.trigger_id = 0; 372 373 /* CPU sample buffer -- only valid with interrupts off (above) 374 * Technically this isn't true -- tracepoints can, and often 375 * are, cut from interrupt handlers, but none of those tracepoints 376 * should make it this far. 377 */ 378 intbuf = kperf_intr_sample_buffer(); 379 380 /* do the sample */ 381 kperf_sample( intbuf, &ctx, kdebug_callstack_action, SAMPLE_FLAG_PEND_USER ); 382 383 /* no longer recursive */ 384 kperf_kdbg_recurse(KPERF_RECURSE_OUT); 385#else 386 /* dicing with death */ 387 BUF_INFO2(PERF_KDBG_HNDLR, debugid, cur_pid); 388 389 /* pend the AST */ 390 kperf_ast_pend( thread, T_AST_CALLSTACK, T_AST_CALLSTACK ); 391#endif 392 393} 394 395int 396kperf_kdbg_get_stacks(void) 397{ 398 return kdebug_callstack_action; 399} 400 401int 402kperf_kdbg_set_stacks(int newval) 403{ 404 /* set the value */ 405 kdebug_callstack_action = newval; 406 407 /* enable the callback from kdebug */ 408 if( newval ) 409 chudxnu_kdebug_callback_enter(NULL); 410 else 411 chudxnu_kdebug_callback_cancel(); 412 413 return 0; 414} 415 416/* 417 * Thread switch 418 */ 419 420/* called from context switch handler */ 421void 422kperf_switch_context( __unused thread_t old, thread_t new ) 423{ 424 task_t task = get_threadtask(new); 425 int pid = chudxnu_pid_for_task(task); 426 427 /* cut a tracepoint to tell us what the new thread's PID is 428 * for Instruments 429 */ 430 BUF_DATA2( PERF_TI_CSWITCH, thread_tid(new), pid ); 431} 432 433/* 434 * Action configuration 435 */ 436unsigned 437kperf_action_get_count(void) 438{ 439 return actionc; 440} 441 442int 443kperf_action_set_samplers( unsigned actionid, uint32_t samplers ) 444{ 445 if( (actionid > actionc) || (actionid == 0) ) 446 return EINVAL; 447 448 actionv[actionid-1].sample = samplers; 449 450 return 0; 451} 452 453int 454kperf_action_get_samplers( unsigned actionid, uint32_t *samplers_out ) 455{ 456 if( (actionid > actionc) ) 457 return EINVAL; 458 459 if( actionid == 0 ) 460 *samplers_out = 0; /* "NULL" action */ 461 else 462 *samplers_out = actionv[actionid-1].sample; 463 464 return 0; 465} 466 467int 468kperf_action_set_userdata( unsigned actionid, uint32_t userdata ) 469{ 470 if( (actionid > actionc) || (actionid == 0) ) 471 return EINVAL; 472 473 actionv[actionid-1].userdata = userdata; 474 475 return 0; 476} 477 478int 479kperf_action_get_userdata( unsigned actionid, uint32_t *userdata_out ) 480{ 481 if( (actionid > actionc) ) 482 return EINVAL; 483 484 if( actionid == 0 ) 485 *userdata_out = 0; /* "NULL" action */ 486 else 487 *userdata_out = actionv[actionid-1].userdata; 488 489 return 0; 490} 491 492int 493kperf_action_set_filter( unsigned actionid, 494 int pid ) 495{ 496 if( (actionid > actionc) || (actionid == 0) ) 497 return EINVAL; 498 499 actionv[actionid-1].pid_filter = pid; 500 501 return 0; 502} 503 504int 505kperf_action_get_filter( unsigned actionid, 506 int *pid_out ) 507{ 508 if( (actionid > actionc) ) 509 return EINVAL; 510 511 if( actionid == 0 ) 512 *pid_out = -1; /* "NULL" action */ 513 else 514 *pid_out = actionv[actionid-1].pid_filter; 515 516 return 0; 517} 518 519int 520kperf_action_set_count(unsigned count) 521{ 522 struct action *new_actionv = NULL, *old_actionv = NULL; 523 unsigned old_count, i; 524 525 /* easy no-op */ 526 if( count == actionc ) 527 return 0; 528 529 /* TODO: allow shrinking? */ 530 if( count < actionc ) 531 return EINVAL; 532 533 /* cap it for good measure */ 534 if( count > ACTION_MAX ) 535 return EINVAL; 536 537 /* creating the action arror for the first time. create a few 538 * more things, too. 539 */ 540 if( actionc == 0 ) 541 { 542 int r; 543 r = kperf_init(); 544 545 if( r != 0 ) 546 return r; 547 } 548 549 /* create a new array */ 550 new_actionv = kalloc( count * sizeof(*new_actionv) ); 551 if( new_actionv == NULL ) 552 return ENOMEM; 553 554 old_actionv = actionv; 555 old_count = actionc; 556 557 if( old_actionv != NULL ) 558 bcopy( actionv, new_actionv, actionc * sizeof(*actionv) ); 559 560 bzero( &new_actionv[actionc], (count - old_count) * sizeof(*actionv) ); 561 562 for( i = old_count; i < count; i++ ) 563 new_actionv[i].pid_filter = -1; 564 565 actionv = new_actionv; 566 actionc = count; 567 568 if( old_actionv != NULL ) 569 kfree( old_actionv, old_count * sizeof(*actionv) ); 570 571 return 0; 572} 573