1/*
2 * CDDL HEADER START
3 *
4 * The contents of this file are subject to the terms of the
5 * Common Development and Distribution License (the "License").
6 * You may not use this file except in compliance with the License.
7 *
8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 * or http://www.opensolaris.org/os/licensing.
10 * See the License for the specific language governing permissions
11 * and limitations under the License.
12 *
13 * When distributing Covered Code, include this CDDL HEADER in each
14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 * If applicable, add the following below this CDDL HEADER, with the
16 * fields enclosed by brackets "[]" replaced with your own identifying
17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 *
19 * CDDL HEADER END
20 *
21 * Portions Copyright 2006-2008 John Birrell jb@freebsd.org
22 *
23 * $FreeBSD$
24 *
25 */
26
27/*
28 * Copyright 2006 Sun Microsystems, Inc.  All rights reserved.
29 * Use is subject to license terms.
30 */
31
32#include <sys/cdefs.h>
33#include <sys/param.h>
34#include <sys/systm.h>
35#include <sys/conf.h>
36#include <sys/cpuvar.h>
37#include <sys/endian.h>
38#include <sys/fcntl.h>
39#include <sys/filio.h>
40#include <sys/kdb.h>
41#include <sys/kernel.h>
42#include <sys/kmem.h>
43#include <sys/kthread.h>
44#include <sys/limits.h>
45#include <sys/linker.h>
46#include <sys/lock.h>
47#include <sys/malloc.h>
48#include <sys/module.h>
49#include <sys/mutex.h>
50#include <sys/poll.h>
51#include <sys/proc.h>
52#include <sys/selinfo.h>
53#include <sys/smp.h>
54#include <sys/sysctl.h>
55#include <sys/uio.h>
56#include <sys/unistd.h>
57#include <machine/cpu.h>
58#include <machine/stdarg.h>
59
60#include <sys/dtrace.h>
61#include <sys/dtrace_bsd.h>
62
63#define	PROF_NAMELEN		15
64
65#define	PROF_PROFILE		0
66#define	PROF_TICK		1
67#define	PROF_PREFIX_PROFILE	"profile-"
68#define	PROF_PREFIX_TICK	"tick-"
69
70/*
71 * Regardless of platform, there are five artificial frames in the case of the
72 * profile provider:
73 *
74 *	profile_fire
75 *	cyclic_expire
76 *	cyclic_fire
77 *	[ cbe ]
78 *	[ locore ]
79 *
80 * On amd64, there are two frames associated with locore:  one in locore, and
81 * another in common interrupt dispatch code.  (i386 has not been modified to
82 * use this common layer.)  Further, on i386, the interrupted instruction
83 * appears as its own stack frame.  All of this means that we need to add one
84 * frame for amd64, and then take one away for both amd64 and i386.
85 *
86 * On SPARC, the picture is further complicated because the compiler
87 * optimizes away tail-calls -- so the following frames are optimized away:
88 *
89 * 	profile_fire
90 *	cyclic_expire
91 *
92 * This gives three frames.  However, on DEBUG kernels, the cyclic_expire
93 * frame cannot be tail-call eliminated, yielding four frames in this case.
94 *
95 * All of the above constraints lead to the mess below.  Yes, the profile
96 * provider should ideally figure this out on-the-fly by hiting one of its own
97 * probes and then walking its own stack trace.  This is complicated, however,
98 * and the static definition doesn't seem to be overly brittle.  Still, we
99 * allow for a manual override in case we get it completely wrong.
100 */
101#ifdef __amd64
102#define	PROF_ARTIFICIAL_FRAMES	10
103#else
104#ifdef __i386
105#define	PROF_ARTIFICIAL_FRAMES	6
106#else
107#ifdef __sparc
108#ifdef DEBUG
109#define	PROF_ARTIFICIAL_FRAMES	4
110#else
111#define	PROF_ARTIFICIAL_FRAMES	3
112#endif
113#endif
114#endif
115#endif
116
117#ifdef __mips
118/*
119 * This value is bogus just to make module compilable on mips
120 */
121#define	PROF_ARTIFICIAL_FRAMES	3
122#endif
123
124#ifdef __powerpc__
125/*
126 * This value is bogus just to make module compilable on powerpc
127 */
128#define	PROF_ARTIFICIAL_FRAMES	3
129#endif
130
131struct profile_probe_percpu;
132
133#ifdef __mips
134/* bogus */
135#define	PROF_ARTIFICIAL_FRAMES	3
136#endif
137
138#ifdef __arm__
139#define	PROF_ARTIFICIAL_FRAMES	3
140#endif
141
142#ifdef __aarch64__
143#define	PROF_ARTIFICIAL_FRAMES	12
144#endif
145
146#ifdef __riscv
147/* TODO: verify */
148#define	PROF_ARTIFICIAL_FRAMES	10
149#endif
150
151typedef struct profile_probe {
152	char		prof_name[PROF_NAMELEN];
153	dtrace_id_t	prof_id;
154	int		prof_kind;
155#ifdef illumos
156	hrtime_t	prof_interval;
157	cyclic_id_t	prof_cyclic;
158#else
159	sbintime_t	prof_interval;
160	struct callout	prof_cyclic;
161	sbintime_t	prof_expected;
162	struct profile_probe_percpu **prof_pcpus;
163#endif
164} profile_probe_t;
165
166typedef struct profile_probe_percpu {
167	hrtime_t	profc_expected;
168	hrtime_t	profc_interval;
169	profile_probe_t	*profc_probe;
170#ifdef __FreeBSD__
171	struct callout	profc_cyclic;
172#endif
173} profile_probe_percpu_t;
174
175static d_open_t	profile_open;
176static int	profile_unload(void);
177static void	profile_create(hrtime_t, char *, int);
178static void	profile_destroy(void *, dtrace_id_t, void *);
179static void	profile_enable(void *, dtrace_id_t, void *);
180static void	profile_disable(void *, dtrace_id_t, void *);
181static void	profile_load(void *);
182static void	profile_provide(void *, dtrace_probedesc_t *);
183
184static int profile_rates[] = {
185    97, 199, 499, 997, 1999,
186    4001, 4999, 0, 0, 0,
187    0, 0, 0, 0, 0,
188    0, 0, 0, 0, 0
189};
190
191static int profile_ticks[] = {
192    1, 10, 100, 500, 1000,
193    5000, 0, 0, 0, 0,
194    0, 0, 0, 0, 0
195};
196
197/*
198 * profile_max defines the upper bound on the number of profile probes that
199 * can exist (this is to prevent malicious or clumsy users from exhausing
200 * system resources by creating a slew of profile probes). At mod load time,
201 * this gets its value from PROFILE_MAX_DEFAULT or profile-max-probes if it's
202 * present in the profile.conf file.
203 */
204#define	PROFILE_MAX_DEFAULT	1000	/* default max. number of probes */
205static uint32_t profile_max = PROFILE_MAX_DEFAULT;
206					/* maximum number of profile probes */
207static uint32_t profile_total;		/* current number of profile probes */
208
209static struct cdevsw profile_cdevsw = {
210	.d_version	= D_VERSION,
211	.d_open		= profile_open,
212	.d_name		= "profile",
213};
214
215static dtrace_pattr_t profile_attr = {
216{ DTRACE_STABILITY_EVOLVING, DTRACE_STABILITY_EVOLVING, DTRACE_CLASS_COMMON },
217{ DTRACE_STABILITY_PRIVATE, DTRACE_STABILITY_PRIVATE, DTRACE_CLASS_UNKNOWN },
218{ DTRACE_STABILITY_PRIVATE, DTRACE_STABILITY_PRIVATE, DTRACE_CLASS_ISA },
219{ DTRACE_STABILITY_EVOLVING, DTRACE_STABILITY_EVOLVING, DTRACE_CLASS_COMMON },
220{ DTRACE_STABILITY_PRIVATE, DTRACE_STABILITY_PRIVATE, DTRACE_CLASS_ISA },
221};
222
223static dtrace_pops_t profile_pops = {
224	.dtps_provide =		profile_provide,
225	.dtps_provide_module =	NULL,
226	.dtps_enable =		profile_enable,
227	.dtps_disable =		profile_disable,
228	.dtps_suspend =		NULL,
229	.dtps_resume =		NULL,
230	.dtps_getargdesc =	NULL,
231	.dtps_getargval =	NULL,
232	.dtps_usermode =	NULL,
233	.dtps_destroy =		profile_destroy
234};
235
236static struct cdev		*profile_cdev;
237static dtrace_provider_id_t	profile_id;
238static hrtime_t			profile_interval_min = NANOSEC / 5000;	/* 5000 hz */
239static int			profile_aframes = PROF_ARTIFICIAL_FRAMES;
240
241SYSCTL_DECL(_kern_dtrace);
242SYSCTL_NODE(_kern_dtrace, OID_AUTO, profile, CTLFLAG_RD | CTLFLAG_MPSAFE, 0,
243    "DTrace profile parameters");
244SYSCTL_INT(_kern_dtrace_profile, OID_AUTO, aframes, CTLFLAG_RW, &profile_aframes,
245    0, "Skipped frames for profile provider");
246
247static sbintime_t
248nsec_to_sbt(hrtime_t nsec)
249{
250	time_t sec;
251
252	/*
253	 * We need to calculate nsec * 2^32 / 10^9
254	 * Seconds and nanoseconds are split to avoid overflow.
255	 */
256	sec = nsec / NANOSEC;
257	nsec = nsec % NANOSEC;
258	return (((sbintime_t)sec << 32) | ((sbintime_t)nsec << 32) / NANOSEC);
259}
260
261static hrtime_t
262sbt_to_nsec(sbintime_t sbt)
263{
264
265	return ((sbt >> 32) * NANOSEC +
266	    (((uint32_t)sbt * (hrtime_t)NANOSEC) >> 32));
267}
268
269static void
270profile_probe(profile_probe_t *prof, hrtime_t late)
271{
272	struct thread *td;
273	struct trapframe *frame;
274	uintfptr_t pc, upc;
275
276	td = curthread;
277	pc = upc = 0;
278
279	/*
280	 * td_intr_frame can be unset if this is a catch-up event upon waking up
281	 * from idle sleep. This can only happen on a CPU idle thread. Use a
282	 * representative arg0 value in this case so that one of the probe
283	 * arguments is non-zero.
284	 */
285	frame = td->td_intr_frame;
286	if (frame != NULL) {
287		if (TRAPF_USERMODE(frame))
288			upc = TRAPF_PC(frame);
289		else
290			pc = TRAPF_PC(frame);
291	} else if (TD_IS_IDLETHREAD(td))
292		pc = (uintfptr_t)&cpu_idle;
293
294	dtrace_probe(prof->prof_id, pc, upc, late, 0, 0);
295}
296
297static void
298profile_fire(void *arg)
299{
300	profile_probe_percpu_t *pcpu = arg;
301	profile_probe_t *prof = pcpu->profc_probe;
302	hrtime_t late;
303
304	late = sbt_to_nsec(sbinuptime() - pcpu->profc_expected);
305
306	profile_probe(prof, late);
307	pcpu->profc_expected += pcpu->profc_interval;
308	callout_schedule_sbt_curcpu(&pcpu->profc_cyclic,
309	    pcpu->profc_expected, 0, C_DIRECT_EXEC | C_ABSOLUTE);
310}
311
312static void
313profile_tick(void *arg)
314{
315	profile_probe_t *prof = arg;
316
317	profile_probe(prof, 0);
318	prof->prof_expected += prof->prof_interval;
319	callout_schedule_sbt(&prof->prof_cyclic,
320	    prof->prof_expected, 0, C_DIRECT_EXEC | C_ABSOLUTE);
321}
322
323static void
324profile_create(hrtime_t interval, char *name, int kind)
325{
326	profile_probe_t *prof;
327
328	if (interval < profile_interval_min)
329		return;
330
331	if (dtrace_probe_lookup(profile_id, NULL, NULL, name) != 0)
332		return;
333
334	atomic_add_32(&profile_total, 1);
335	if (profile_total > profile_max) {
336		atomic_add_32(&profile_total, -1);
337		return;
338	}
339
340	prof = kmem_zalloc(sizeof (profile_probe_t), KM_SLEEP);
341	(void) strcpy(prof->prof_name, name);
342#ifdef illumos
343	prof->prof_interval = interval;
344	prof->prof_cyclic = CYCLIC_NONE;
345#else
346	prof->prof_interval = nsec_to_sbt(interval);
347	callout_init(&prof->prof_cyclic, 1);
348#endif
349	prof->prof_kind = kind;
350	prof->prof_id = dtrace_probe_create(profile_id,
351	    NULL, NULL, name,
352	    profile_aframes, prof);
353}
354
355/*ARGSUSED*/
356static void
357profile_provide(void *arg, dtrace_probedesc_t *desc)
358{
359	int i, j, rate, kind;
360	hrtime_t val = 0, mult = 1, len = 0;
361	char *name, *suffix = NULL;
362
363	const struct {
364		char *prefix;
365		int kind;
366	} types[] = {
367		{ PROF_PREFIX_PROFILE, PROF_PROFILE },
368		{ PROF_PREFIX_TICK, PROF_TICK },
369		{ 0, 0 }
370	};
371
372	const struct {
373		char *name;
374		hrtime_t mult;
375	} suffixes[] = {
376		{ "ns", 	NANOSEC / NANOSEC },
377		{ "nsec",	NANOSEC / NANOSEC },
378		{ "us",		NANOSEC / MICROSEC },
379		{ "usec",	NANOSEC / MICROSEC },
380		{ "ms",		NANOSEC / MILLISEC },
381		{ "msec",	NANOSEC / MILLISEC },
382		{ "s",		NANOSEC / SEC },
383		{ "sec",	NANOSEC / SEC },
384		{ "m",		NANOSEC * (hrtime_t)60 },
385		{ "min",	NANOSEC * (hrtime_t)60 },
386		{ "h",		NANOSEC * (hrtime_t)(60 * 60) },
387		{ "hour",	NANOSEC * (hrtime_t)(60 * 60) },
388		{ "d",		NANOSEC * (hrtime_t)(24 * 60 * 60) },
389		{ "day",	NANOSEC * (hrtime_t)(24 * 60 * 60) },
390		{ "hz",		0 },
391		{ NULL }
392	};
393
394	if (desc == NULL) {
395		char n[PROF_NAMELEN];
396
397		/*
398		 * If no description was provided, provide all of our probes.
399		 */
400		for (i = 0; i < sizeof (profile_rates) / sizeof (int); i++) {
401			if ((rate = profile_rates[i]) == 0)
402				continue;
403
404			(void) snprintf(n, PROF_NAMELEN, "%s%d",
405			    PROF_PREFIX_PROFILE, rate);
406			profile_create(NANOSEC / rate, n, PROF_PROFILE);
407		}
408
409		for (i = 0; i < sizeof (profile_ticks) / sizeof (int); i++) {
410			if ((rate = profile_ticks[i]) == 0)
411				continue;
412
413			(void) snprintf(n, PROF_NAMELEN, "%s%d",
414			    PROF_PREFIX_TICK, rate);
415			profile_create(NANOSEC / rate, n, PROF_TICK);
416		}
417
418		return;
419	}
420
421	name = desc->dtpd_name;
422
423	for (i = 0; types[i].prefix != NULL; i++) {
424		len = strlen(types[i].prefix);
425
426		if (strncmp(name, types[i].prefix, len) != 0)
427			continue;
428		break;
429	}
430
431	if (types[i].prefix == NULL)
432		return;
433
434	kind = types[i].kind;
435	j = strlen(name) - len;
436
437	/*
438	 * We need to start before any time suffix.
439	 */
440	for (j = strlen(name); j >= len; j--) {
441		if (name[j] >= '0' && name[j] <= '9')
442			break;
443		suffix = &name[j];
444	}
445
446	ASSERT(suffix != NULL);
447
448	/*
449	 * Now determine the numerical value present in the probe name.
450	 */
451	for (; j >= len; j--) {
452		if (name[j] < '0' || name[j] > '9')
453			return;
454
455		val += (name[j] - '0') * mult;
456		mult *= (hrtime_t)10;
457	}
458
459	if (val == 0)
460		return;
461
462	/*
463	 * Look-up the suffix to determine the multiplier.
464	 */
465	for (i = 0, mult = 0; suffixes[i].name != NULL; i++) {
466		if (strcasecmp(suffixes[i].name, suffix) == 0) {
467			mult = suffixes[i].mult;
468			break;
469		}
470	}
471
472	if (suffixes[i].name == NULL && *suffix != '\0')
473		return;
474
475	if (mult == 0) {
476		/*
477		 * The default is frequency-per-second.
478		 */
479		val = NANOSEC / val;
480	} else {
481		val *= mult;
482	}
483
484	profile_create(val, name, kind);
485}
486
487/* ARGSUSED */
488static void
489profile_destroy(void *arg, dtrace_id_t id, void *parg)
490{
491	profile_probe_t *prof = parg;
492
493#ifdef illumos
494	ASSERT(prof->prof_cyclic == CYCLIC_NONE);
495#else
496	ASSERT(!callout_active(&prof->prof_cyclic) && prof->prof_pcpus == NULL);
497#endif
498	kmem_free(prof, sizeof (profile_probe_t));
499
500	ASSERT(profile_total >= 1);
501	atomic_add_32(&profile_total, -1);
502}
503
504#ifdef illumos
505/*ARGSUSED*/
506static void
507profile_online(void *arg, cpu_t *cpu, cyc_handler_t *hdlr, cyc_time_t *when)
508{
509	profile_probe_t *prof = arg;
510	profile_probe_percpu_t *pcpu;
511
512	pcpu = kmem_zalloc(sizeof (profile_probe_percpu_t), KM_SLEEP);
513	pcpu->profc_probe = prof;
514
515	hdlr->cyh_func = profile_fire;
516	hdlr->cyh_arg = pcpu;
517
518	when->cyt_interval = prof->prof_interval;
519	when->cyt_when = gethrtime() + when->cyt_interval;
520
521	pcpu->profc_expected = when->cyt_when;
522	pcpu->profc_interval = when->cyt_interval;
523}
524
525/*ARGSUSED*/
526static void
527profile_offline(void *arg, cpu_t *cpu, void *oarg)
528{
529	profile_probe_percpu_t *pcpu = oarg;
530
531	ASSERT(pcpu->profc_probe == arg);
532	kmem_free(pcpu, sizeof (profile_probe_percpu_t));
533}
534
535/* ARGSUSED */
536static void
537profile_enable(void *arg, dtrace_id_t id, void *parg)
538{
539	profile_probe_t *prof = parg;
540	cyc_omni_handler_t omni;
541	cyc_handler_t hdlr;
542	cyc_time_t when;
543
544	ASSERT(prof->prof_interval != 0);
545	ASSERT(MUTEX_HELD(&cpu_lock));
546
547	if (prof->prof_kind == PROF_TICK) {
548		hdlr.cyh_func = profile_tick;
549		hdlr.cyh_arg = prof;
550
551		when.cyt_interval = prof->prof_interval;
552		when.cyt_when = gethrtime() + when.cyt_interval;
553	} else {
554		ASSERT(prof->prof_kind == PROF_PROFILE);
555		omni.cyo_online = profile_online;
556		omni.cyo_offline = profile_offline;
557		omni.cyo_arg = prof;
558	}
559
560	if (prof->prof_kind == PROF_TICK) {
561		prof->prof_cyclic = cyclic_add(&hdlr, &when);
562	} else {
563		prof->prof_cyclic = cyclic_add_omni(&omni);
564	}
565}
566
567/* ARGSUSED */
568static void
569profile_disable(void *arg, dtrace_id_t id, void *parg)
570{
571	profile_probe_t *prof = parg;
572
573	ASSERT(prof->prof_cyclic != CYCLIC_NONE);
574	ASSERT(MUTEX_HELD(&cpu_lock));
575
576	cyclic_remove(prof->prof_cyclic);
577	prof->prof_cyclic = CYCLIC_NONE;
578}
579
580#else
581
582static void
583profile_enable_omni(profile_probe_t *prof)
584{
585	profile_probe_percpu_t *pcpu;
586	int cpu;
587
588	prof->prof_pcpus = kmem_zalloc((mp_maxid + 1) * sizeof(pcpu), KM_SLEEP);
589	CPU_FOREACH(cpu) {
590		pcpu = kmem_zalloc(sizeof(profile_probe_percpu_t), KM_SLEEP);
591		prof->prof_pcpus[cpu] = pcpu;
592		pcpu->profc_probe = prof;
593		pcpu->profc_expected = sbinuptime() + prof->prof_interval;
594		pcpu->profc_interval = prof->prof_interval;
595		callout_init(&pcpu->profc_cyclic, 1);
596		callout_reset_sbt_on(&pcpu->profc_cyclic,
597		    pcpu->profc_expected, 0, profile_fire, pcpu,
598		    cpu, C_DIRECT_EXEC | C_ABSOLUTE);
599	}
600}
601
602static void
603profile_disable_omni(profile_probe_t *prof)
604{
605	profile_probe_percpu_t *pcpu;
606	int cpu;
607
608	ASSERT(prof->prof_pcpus != NULL);
609	CPU_FOREACH(cpu) {
610		pcpu = prof->prof_pcpus[cpu];
611		ASSERT(pcpu->profc_probe == prof);
612		ASSERT(callout_active(&pcpu->profc_cyclic));
613		callout_stop(&pcpu->profc_cyclic);
614		callout_drain(&pcpu->profc_cyclic);
615		kmem_free(pcpu, sizeof(profile_probe_percpu_t));
616	}
617	kmem_free(prof->prof_pcpus, (mp_maxid + 1) * sizeof(pcpu));
618	prof->prof_pcpus = NULL;
619}
620
621/* ARGSUSED */
622static void
623profile_enable(void *arg, dtrace_id_t id, void *parg)
624{
625	profile_probe_t *prof = parg;
626
627	if (prof->prof_kind == PROF_TICK) {
628		prof->prof_expected = sbinuptime() + prof->prof_interval;
629		callout_reset_sbt(&prof->prof_cyclic,
630		    prof->prof_expected, 0, profile_tick, prof,
631		    C_DIRECT_EXEC | C_ABSOLUTE);
632	} else {
633		ASSERT(prof->prof_kind == PROF_PROFILE);
634		profile_enable_omni(prof);
635	}
636}
637
638/* ARGSUSED */
639static void
640profile_disable(void *arg, dtrace_id_t id, void *parg)
641{
642	profile_probe_t *prof = parg;
643
644	if (prof->prof_kind == PROF_TICK) {
645		ASSERT(callout_active(&prof->prof_cyclic));
646		callout_stop(&prof->prof_cyclic);
647		callout_drain(&prof->prof_cyclic);
648	} else {
649		ASSERT(prof->prof_kind == PROF_PROFILE);
650		profile_disable_omni(prof);
651	}
652}
653#endif
654
655static void
656profile_load(void *dummy)
657{
658	/* Create the /dev/dtrace/profile entry. */
659	profile_cdev = make_dev(&profile_cdevsw, 0, UID_ROOT, GID_WHEEL, 0600,
660	    "dtrace/profile");
661
662	if (dtrace_register("profile", &profile_attr, DTRACE_PRIV_USER,
663	    NULL, &profile_pops, NULL, &profile_id) != 0)
664		return;
665}
666
667
668static int
669profile_unload()
670{
671	int error = 0;
672
673	if ((error = dtrace_unregister(profile_id)) != 0)
674		return (error);
675
676	destroy_dev(profile_cdev);
677
678	return (error);
679}
680
681/* ARGSUSED */
682static int
683profile_modevent(module_t mod __unused, int type, void *data __unused)
684{
685	int error = 0;
686
687	switch (type) {
688	case MOD_LOAD:
689		break;
690
691	case MOD_UNLOAD:
692		break;
693
694	case MOD_SHUTDOWN:
695		break;
696
697	default:
698		error = EOPNOTSUPP;
699		break;
700
701	}
702	return (error);
703}
704
705/* ARGSUSED */
706static int
707profile_open(struct cdev *dev __unused, int oflags __unused, int devtype __unused, struct thread *td __unused)
708{
709	return (0);
710}
711
712SYSINIT(profile_load, SI_SUB_DTRACE_PROVIDER, SI_ORDER_ANY, profile_load, NULL);
713SYSUNINIT(profile_unload, SI_SUB_DTRACE_PROVIDER, SI_ORDER_ANY, profile_unload, NULL);
714
715DEV_MODULE(profile, profile_modevent, NULL);
716MODULE_VERSION(profile, 1);
717MODULE_DEPEND(profile, dtrace, 1, 1, 1);
718MODULE_DEPEND(profile, opensolaris, 1, 1, 1);
719