1/*
2 * Copyright (c) 2011 Apple Computer, Inc. All rights reserved.
3 *
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
14 *
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
25 *
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27 */
28
29/*
30 * Called from a trigger. Actually takes the data from the different
31 * modules and puts them in a buffer
32 */
33
34#include <mach/mach_types.h>
35#include <machine/machine_routines.h>
36// #include <libkern/libkern.h>
37#include <kern/kalloc.h>
38#include <kern/debug.h> /* panic */
39#include <kern/thread.h>
40#include <sys/errno.h>
41
42#include <chud/chud_xnu.h>
43#include <kperf/kperf.h>
44
45#include <kperf/buffer.h>
46#include <kperf/timetrigger.h>
47#include <kperf/threadinfo.h>
48#include <kperf/callstack.h>
49#include <kperf/sample.h>
50#include <kperf/action.h>
51#include <kperf/context.h>
52#include <kperf/ast.h>
53
54#define ACTION_MAX 32
55
56/* the list of different actions to take */
57struct action
58{
59	uint32_t sample;
60	uint32_t userdata;
61	int pid_filter;
62};
63
64/* the list of actions */
65static unsigned actionc = 0;
66static struct action *actionv = NULL;
67
68/* whether to record callstacks on kdebug events */
69static int kdebug_callstack_action = 0;
70
71/* whether we get a callback on a thread switch */
72int  kperf_cswitch_hook = 0;
73
74/* indirect hooks to play nice with CHUD for the transition to kperf */
75kern_return_t chudxnu_kdebug_callback_enter(chudxnu_kdebug_callback_func_t fn);
76kern_return_t chudxnu_kdebug_callback_cancel(void);
77
78/* Do the real work! */
79/* this can be called in any context ... right? */
80static kern_return_t
81kperf_sample_internal( struct kperf_sample *sbuf,
82                       struct kperf_context *context,
83                       unsigned sample_what, unsigned sample_flags,
84                       unsigned actionid )
85{
86	boolean_t enabled;
87	int did_ucallstack = 0, did_tinfo_extra = 0;
88	uint32_t userdata;
89
90	/* not much point continuing here, but what to do ? return
91	 * Shutdown? cut a tracepoint and continue?
92	 */
93	if( sample_what == 0 )
94		return SAMPLE_CONTINUE;
95
96	int is_kernel = (context->cur_pid == 0);
97
98	sbuf->kcallstack.nframes = 0;
99	sbuf->kcallstack.flags = CALLSTACK_VALID;
100	sbuf->ucallstack.nframes = 0;
101	sbuf->ucallstack.flags = CALLSTACK_VALID;
102
103	/*  an event occurred. Sample everything and dump it in a
104	 *  buffer.
105	 */
106
107	/* collect data from samplers */
108	if( sample_what & SAMPLER_TINFO ) {
109		kperf_threadinfo_sample( &sbuf->threadinfo, context );
110
111		/* See if we should drop idle thread samples */
112		if( !(sample_flags & SAMPLE_FLAG_IDLE_THREADS) )
113			if (sbuf->threadinfo.runmode & 0x40)
114				return SAMPLE_CONTINUE;
115	}
116
117	if( (sample_what & SAMPLER_KSTACK) && !(sample_flags & SAMPLE_FLAG_EMPTY_CALLSTACK) )
118		kperf_kcallstack_sample( &sbuf->kcallstack, context );
119
120	/* sensitive ones */
121	if ( !is_kernel ) {
122		if( sample_flags & SAMPLE_FLAG_PEND_USER )
123		{
124			if( (sample_what & SAMPLER_USTACK) && !(sample_flags & SAMPLE_FLAG_EMPTY_CALLSTACK) )
125				did_ucallstack = kperf_ucallstack_pend( context );
126
127			if( sample_what & SAMPLER_TINFOEX )
128				did_tinfo_extra = kperf_threadinfo_extra_pend( context );
129		}
130		else
131		{
132			if( (sample_what & SAMPLER_USTACK) && !(sample_flags & SAMPLE_FLAG_EMPTY_CALLSTACK) )
133				kperf_ucallstack_sample( &sbuf->ucallstack, context );
134
135			if( sample_what & SAMPLER_TINFOEX )
136				kperf_threadinfo_extra_sample( &sbuf->tinfo_ex,
137							       context );
138		}
139	}
140
141#if KPC
142	if ( sample_what & SAMPLER_PMC_CPU )
143		kperf_kpc_cpu_sample( &sbuf->kpcdata,
144		                      (sample_what & SAMPLER_PMC_CPU) != 0 );
145#endif
146
147	/* lookup the user tag, if any */
148	if( actionid
149	    && (actionid <= actionc) )
150		userdata = actionv[actionid-1].userdata;
151	else
152		userdata = actionid;
153
154	/* stash the data into the buffer
155	 * interrupts off to ensure we don't get split
156	 */
157	enabled = ml_set_interrupts_enabled(FALSE);
158
159	BUF_DATA( PERF_GEN_EVENT | DBG_FUNC_START, sample_what,
160	           actionid, userdata, sample_flags );
161
162	/* dump threadinfo */
163	if( sample_what & SAMPLER_TINFO )
164		kperf_threadinfo_log( &sbuf->threadinfo );
165
166	/* dump kcallstack */
167	if( sample_what & SAMPLER_KSTACK )
168		kperf_kcallstack_log( &sbuf->kcallstack );
169
170
171	/* dump user stuff */
172	if ( !is_kernel ) {
173		if ( sample_flags & SAMPLE_FLAG_PEND_USER )
174		{
175			if ( did_ucallstack )
176				BUF_INFO1( PERF_CS_UPEND, 0 );
177
178			if ( did_tinfo_extra )
179				BUF_INFO1( PERF_TI_XPEND, 0 );
180		}
181		else
182		{
183			if( sample_what & SAMPLER_USTACK )
184				kperf_ucallstack_log( &sbuf->ucallstack );
185
186			if( sample_what & SAMPLER_TINFOEX )
187				kperf_threadinfo_extra_log( &sbuf->tinfo_ex );
188		}
189	}
190
191#if KPC
192	if ( sample_what & SAMPLER_PMC_CPU )
193		kperf_kpc_cpu_log( &sbuf->kpcdata );
194
195#endif
196
197	BUF_DATA1( PERF_GEN_EVENT | DBG_FUNC_END, sample_what );
198
199	/* intrs back on */
200	ml_set_interrupts_enabled(enabled);
201
202	return SAMPLE_CONTINUE;
203}
204
205/* Translate actionid into sample bits and take a sample */
206kern_return_t
207kperf_sample( struct kperf_sample *sbuf,
208	      struct kperf_context *context,
209              unsigned actionid, unsigned sample_flags )
210{
211	unsigned sample_what = 0;
212	int pid_filter;
213
214	/* work out what to sample, if anything */
215	if( (actionid > actionc) || (actionid == 0) )
216		return SAMPLE_SHUTDOWN;
217
218	/* check the pid filter against the context's current pid.
219	 * filter pid == -1 means any pid
220	 */
221	pid_filter = actionv[actionid-1].pid_filter;
222	if( (pid_filter != -1)
223	    && (pid_filter != context->cur_pid) )
224		return SAMPLE_CONTINUE;
225
226	/* the samplers to run */
227	sample_what = actionv[actionid-1].sample;
228
229	/* do the actual sample operation */
230	return kperf_sample_internal( sbuf, context, sample_what,
231	                              sample_flags, actionid );
232}
233
234/* ast callback on a thread */
235void
236kperf_thread_ast_handler( thread_t thread )
237{
238	int r;
239	uint32_t t_chud;
240	unsigned sample_what = 0;
241	/* we know we're on a thread, so let's do stuff */
242	task_t task = NULL;
243
244	BUF_INFO1(PERF_AST_HNDLR | DBG_FUNC_START, thread);
245
246	/* use ~2kb of the stack for the sample, should be ok since we're in the ast */
247	struct kperf_sample sbuf;
248	bzero(&sbuf, sizeof(struct kperf_sample));
249
250	/* make a context, take a sample */
251	struct kperf_context ctx;
252	ctx.cur_thread = thread;
253	ctx.cur_pid = -1;
254
255	task = chudxnu_task_for_thread(thread);
256	if(task)
257		ctx.cur_pid = chudxnu_pid_for_task(task);
258
259	/* decode the chud bits so we know what to sample */
260	t_chud = kperf_get_thread_bits(thread);
261
262	if (t_chud & T_AST_NAME)
263		sample_what |= SAMPLER_TINFOEX;
264
265	if (t_chud & T_AST_CALLSTACK)
266	{
267		sample_what |= SAMPLER_USTACK;
268		sample_what |= SAMPLER_TINFO;
269	}
270
271	/* do the sample, just of the user stuff */
272	r = kperf_sample_internal( &sbuf, &ctx, sample_what, 0, 0 );
273
274	BUF_INFO1(PERF_AST_HNDLR | DBG_FUNC_END, r);
275}
276
277/* register AST bits */
278int
279kperf_ast_pend( thread_t cur_thread, uint32_t check_bits,
280		uint32_t set_bits )
281{
282	/* pend on the thread */
283	uint32_t t_chud, set_done = 0;
284
285	/* can only pend on the current thread */
286	if( cur_thread != chudxnu_current_thread() )
287		panic("pending to non-current thread");
288
289	/* get our current bits */
290	t_chud = kperf_get_thread_bits(cur_thread);
291
292	/* see if it's already been done or pended */
293	if( !(t_chud & check_bits ) )
294	{
295		/* set the bit on the thread */
296		t_chud |= set_bits;
297		kperf_set_thread_bits(cur_thread, t_chud);
298
299		/* set the actual AST */
300		kperf_set_thread_ast( cur_thread );
301
302		set_done = 1;
303	}
304
305	return set_done;
306
307//	BUF_INFO3( dbg_code, (uintptr_t)cur_thread, t_chud, set_done );
308}
309
310/*
311 * kdebug callback & stack management
312 */
313
314#define IS_END(debugid)           ((debugid & 3) == DBG_FUNC_END)
315#define IS_MIG(debugid)           (IS_END(debugid) && ((debugid & 0xff000000U) == KDBG_CLASS_ENCODE((unsigned)DBG_MIG, 0U)))
316#define IS_MACH_SYSCALL(debugid)  (IS_END(debugid) && (KDBG_CLASS_DECODE(debugid) == KDBG_CLASS_ENCODE(DBG_MACH, DBG_MACH_EXCP_SC)))
317#define IS_VM_FAULT(debugid)      (IS_END(debugid) && (KDBG_CLASS_DECODE(debugid) == KDBG_CLASS_ENCODE(DBG_MACH, DBG_MACH_VM)))
318#define IS_BSD_SYSCTLL(debugid)   (IS_END(debugid) && (KDBG_CLASS_DECODE(debugid) == KDBG_CLASS_ENCODE(DBG_BSD, DBG_BSD_EXCP_SC)))
319#define IS_APPS_SIGNPOST(debugid) (IS_END(debugid) && (KDBG_CLASS_DECODE(debugid) == KDBG_CLASS_ENCODE(DBG_APPS, DBG_MACH_CHUD)))
320#define IS_MACH_SIGNPOST(debugid) (IS_END(debugid) && (KDBG_CLASS_DECODE(debugid) == KDBG_CLASS_ENCODE(DBG_MACH, DBG_MACH_CHUD)))
321
322void
323kperf_kdebug_callback(uint32_t debugid)
324{
325	int cur_pid = 0;
326	task_t task = NULL;
327
328	/* if we're not doing kperf callback stacks, return */
329	if( !kdebug_callstack_action )
330		return;
331
332	/* if we're looking at a kperf tracepoint, don't recurse */
333	if( (debugid & 0xff000000) == KDBG_CLASS_ENCODE(DBG_PERF, 0) )
334		return;
335
336	/* ensure interrupts are already off thanks to kdebug */
337	if( ml_get_interrupts_enabled() )
338		return;
339
340	/* make sure we're not being called recursively.  */
341#if NOTYET
342	if( kperf_kdbg_recurse(KPERF_RECURSE_IN) )
343		return;
344#endif
345
346	/* check the happy list of trace codes */
347	if( !( IS_MIG(debugid)
348	       || IS_MACH_SYSCALL(debugid)
349	       || IS_VM_FAULT(debugid)
350	       || IS_BSD_SYSCTLL(debugid)
351	       || IS_MACH_SIGNPOST(debugid)
352	       || IS_APPS_SIGNPOST(debugid) ) )
353		return;
354
355	/* check for kernel */
356	thread_t thread = chudxnu_current_thread();
357	task = chudxnu_task_for_thread(thread);
358	if(task)
359		cur_pid = chudxnu_pid_for_task(task);
360	if( !cur_pid )
361		return;
362
363#if NOTYET
364	/* setup a context */
365	struct kperf_context ctx;
366	struct kperf_sample *intbuf = NULL;
367
368	ctx.cur_thread = thread;
369	ctx.cur_pid = cur_pid;
370	ctx.trigger_type = TRIGGER_TYPE_TRACE;
371	ctx.trigger_id = 0;
372
373	/* CPU sample buffer -- only valid with interrupts off (above)
374	 * Technically this isn't true -- tracepoints can, and often
375	 * are, cut from interrupt handlers, but none of those tracepoints
376	 * should make it this far.
377	 */
378	intbuf = kperf_intr_sample_buffer();
379
380	/* do the sample */
381	kperf_sample( intbuf, &ctx, kdebug_callstack_action, SAMPLE_FLAG_PEND_USER );
382
383	/* no longer recursive */
384	kperf_kdbg_recurse(KPERF_RECURSE_OUT);
385#else
386	/* dicing with death */
387	BUF_INFO2(PERF_KDBG_HNDLR, debugid, cur_pid);
388
389	/* pend the AST */
390	kperf_ast_pend( thread, T_AST_CALLSTACK, T_AST_CALLSTACK );
391#endif
392
393}
394
395int
396kperf_kdbg_get_stacks(void)
397{
398	return kdebug_callstack_action;
399}
400
401int
402kperf_kdbg_set_stacks(int newval)
403{
404	/* set the value */
405	kdebug_callstack_action = newval;
406
407	/* enable the callback from kdebug */
408	if( newval )
409		chudxnu_kdebug_callback_enter(NULL);
410	else
411		chudxnu_kdebug_callback_cancel();
412
413	return 0;
414}
415
416/*
417 * Thread switch
418 */
419
420/* called from context switch handler */
421void
422kperf_switch_context( __unused thread_t old, thread_t new )
423{
424	task_t task = get_threadtask(new);
425	int pid = chudxnu_pid_for_task(task);
426
427	/* cut a tracepoint to tell us what the new thread's PID is
428	 * for Instruments
429	 */
430	BUF_DATA2( PERF_TI_CSWITCH, thread_tid(new), pid );
431}
432
433/*
434 * Action configuration
435 */
436unsigned
437kperf_action_get_count(void)
438{
439	return actionc;
440}
441
442int
443kperf_action_set_samplers( unsigned actionid, uint32_t samplers )
444{
445	if( (actionid > actionc) || (actionid == 0) )
446		return EINVAL;
447
448	actionv[actionid-1].sample = samplers;
449
450	return 0;
451}
452
453int
454kperf_action_get_samplers( unsigned actionid, uint32_t *samplers_out )
455{
456	if( (actionid > actionc) )
457		return EINVAL;
458
459	if( actionid == 0 )
460		*samplers_out = 0; /* "NULL" action */
461	else
462		*samplers_out = actionv[actionid-1].sample;
463
464	return 0;
465}
466
467int
468kperf_action_set_userdata( unsigned actionid, uint32_t userdata )
469{
470	if( (actionid > actionc) || (actionid == 0) )
471		return EINVAL;
472
473	actionv[actionid-1].userdata = userdata;
474
475	return 0;
476}
477
478int
479kperf_action_get_userdata( unsigned actionid, uint32_t *userdata_out )
480{
481	if( (actionid > actionc) )
482		return EINVAL;
483
484	if( actionid == 0 )
485		*userdata_out = 0; /* "NULL" action */
486	else
487		*userdata_out = actionv[actionid-1].userdata;
488
489	return 0;
490}
491
492int
493kperf_action_set_filter( unsigned actionid,
494			 int pid )
495{
496	if( (actionid > actionc) || (actionid == 0) )
497		return EINVAL;
498
499	actionv[actionid-1].pid_filter = pid;
500
501	return 0;
502}
503
504int
505kperf_action_get_filter( unsigned actionid,
506			 int *pid_out )
507{
508	if( (actionid > actionc) )
509		return EINVAL;
510
511	if( actionid == 0 )
512		*pid_out = -1; /* "NULL" action */
513	else
514		*pid_out = actionv[actionid-1].pid_filter;
515
516	return 0;
517}
518
519int
520kperf_action_set_count(unsigned count)
521{
522	struct action *new_actionv = NULL, *old_actionv = NULL;
523	unsigned old_count, i;
524
525	/* easy no-op */
526	if( count == actionc )
527		return 0;
528
529	/* TODO: allow shrinking? */
530	if( count < actionc )
531		return EINVAL;
532
533	/* cap it for good measure */
534	if( count > ACTION_MAX )
535		return EINVAL;
536
537	/* creating the action arror for the first time. create a few
538	 * more things, too.
539	 */
540       	if( actionc == 0 )
541	{
542		int r;
543		r = kperf_init();
544
545		if( r != 0 )
546			return r;
547	}
548
549	/* create a new array */
550	new_actionv = kalloc( count * sizeof(*new_actionv) );
551	if( new_actionv == NULL )
552		return ENOMEM;
553
554	old_actionv = actionv;
555	old_count = actionc;
556
557	if( old_actionv != NULL )
558		bcopy( actionv, new_actionv, actionc * sizeof(*actionv) );
559
560	bzero( &new_actionv[actionc], (count - old_count) * sizeof(*actionv) );
561
562	for( i = old_count; i < count; i++ )
563		new_actionv[i].pid_filter = -1;
564
565	actionv = new_actionv;
566	actionc = count;
567
568	if( old_actionv != NULL )
569		kfree( old_actionv, old_count * sizeof(*actionv) );
570
571	return 0;
572}
573