1/*
2 * CDDL HEADER START
3 *
4 * The contents of this file are subject to the terms of the
5 * Common Development and Distribution License (the "License").
6 * You may not use this file except in compliance with the License.
7 *
8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 * or http://www.opensolaris.org/os/licensing.
10 * See the License for the specific language governing permissions
11 * and limitations under the License.
12 *
13 * When distributing Covered Code, include this CDDL HEADER in each
14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 * If applicable, add the following below this CDDL HEADER, with the
16 * fields enclosed by brackets "[]" replaced with your own identifying
17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 *
19 * CDDL HEADER END
20 */
21/*
22 * Copyright 2009 Sun Microsystems, Inc.  All rights reserved.
23 * Use is subject to license terms.
24 */
25
26
27#include <sys/modctl.h>
28#include <sys/sunddi.h>
29#include <sys/dtrace.h>
30#include <sys/kobj.h>
31#include <sys/stat.h>
32#include <sys/conf.h>
33#include <vm/seg_kmem.h>
34#include <sys/stack.h>
35#include <sys/frame.h>
36#include <sys/dtrace_impl.h>
37#include <sys/cmn_err.h>
38#include <sys/sysmacros.h>
39#include <sys/privregs.h>
40#include <sys/sdt_impl.h>
41
42#define	SDT_PATCHVAL	0xf0
43#define	SDT_ADDR2NDX(addr)	((((uintptr_t)(addr)) >> 4) & sdt_probetab_mask)
44#define	SDT_PROBETAB_SIZE	0x1000		/* 4k entries -- 16K total */
45
46static dev_info_t		*sdt_devi;
47static int			sdt_verbose = 0;
48static sdt_probe_t		**sdt_probetab;
49static int			sdt_probetab_size;
50static int			sdt_probetab_mask;
51
52/*ARGSUSED*/
53static int
54sdt_invop(uintptr_t addr, uintptr_t *stack, uintptr_t eax)
55{
56	uintptr_t stack0, stack1, stack2, stack3, stack4;
57	int i = 0;
58	sdt_probe_t *sdt = sdt_probetab[SDT_ADDR2NDX(addr)];
59
60#ifdef __amd64
61	/*
62	 * On amd64, stack[0] contains the dereferenced stack pointer,
63	 * stack[1] contains savfp, stack[2] contains savpc.  We want
64	 * to step over these entries.
65	 */
66	i += 3;
67#endif
68
69	for (; sdt != NULL; sdt = sdt->sdp_hashnext) {
70		if ((uintptr_t)sdt->sdp_patchpoint == addr) {
71			/*
72			 * When accessing the arguments on the stack, we must
73			 * protect against accessing beyond the stack.  We can
74			 * safely set NOFAULT here -- we know that interrupts
75			 * are already disabled.
76			 */
77			DTRACE_CPUFLAG_SET(CPU_DTRACE_NOFAULT);
78			stack0 = stack[i++];
79			stack1 = stack[i++];
80			stack2 = stack[i++];
81			stack3 = stack[i++];
82			stack4 = stack[i++];
83			DTRACE_CPUFLAG_CLEAR(CPU_DTRACE_NOFAULT |
84			    CPU_DTRACE_BADADDR);
85
86			dtrace_probe(sdt->sdp_id, stack0, stack1,
87			    stack2, stack3, stack4);
88
89			return (DTRACE_INVOP_NOP);
90		}
91	}
92
93	return (0);
94}
95
96/*ARGSUSED*/
97static void
98sdt_provide_module(void *arg, struct modctl *ctl)
99{
100	struct module *mp = ctl->mod_mp;
101	char *modname = ctl->mod_modname;
102	sdt_probedesc_t *sdpd;
103	sdt_probe_t *sdp, *old;
104	sdt_provider_t *prov;
105	int len;
106
107	/*
108	 * One for all, and all for one:  if we haven't yet registered all of
109	 * our providers, we'll refuse to provide anything.
110	 */
111	for (prov = sdt_providers; prov->sdtp_name != NULL; prov++) {
112		if (prov->sdtp_id == DTRACE_PROVNONE)
113			return;
114	}
115
116	if (mp->sdt_nprobes != 0 || (sdpd = mp->sdt_probes) == NULL)
117		return;
118
119	for (sdpd = mp->sdt_probes; sdpd != NULL; sdpd = sdpd->sdpd_next) {
120		char *name = sdpd->sdpd_name, *func, *nname;
121		int i, j;
122		sdt_provider_t *prov;
123		ulong_t offs;
124		dtrace_id_t id;
125
126		for (prov = sdt_providers; prov->sdtp_prefix != NULL; prov++) {
127			char *prefix = prov->sdtp_prefix;
128
129			if (strncmp(name, prefix, strlen(prefix)) == 0) {
130				name += strlen(prefix);
131				break;
132			}
133		}
134
135		nname = kmem_alloc(len = strlen(name) + 1, KM_SLEEP);
136
137		for (i = 0, j = 0; name[j] != '\0'; i++) {
138			if (name[j] == '_' && name[j + 1] == '_') {
139				nname[i] = '-';
140				j += 2;
141			} else {
142				nname[i] = name[j++];
143			}
144		}
145
146		nname[i] = '\0';
147
148		sdp = kmem_zalloc(sizeof (sdt_probe_t), KM_SLEEP);
149		sdp->sdp_loadcnt = ctl->mod_loadcnt;
150		sdp->sdp_ctl = ctl;
151		sdp->sdp_name = nname;
152		sdp->sdp_namelen = len;
153		sdp->sdp_provider = prov;
154
155		func = kobj_searchsym(mp, sdpd->sdpd_offset, &offs);
156
157		if (func == NULL)
158			func = "<unknown>";
159
160		/*
161		 * We have our provider.  Now create the probe.
162		 */
163		if ((id = dtrace_probe_lookup(prov->sdtp_id, modname,
164		    func, nname)) != DTRACE_IDNONE) {
165			old = dtrace_probe_arg(prov->sdtp_id, id);
166			ASSERT(old != NULL);
167
168			sdp->sdp_next = old->sdp_next;
169			sdp->sdp_id = id;
170			old->sdp_next = sdp;
171		} else {
172			sdp->sdp_id = dtrace_probe_create(prov->sdtp_id,
173			    modname, func, nname, 3, sdp);
174
175			mp->sdt_nprobes++;
176		}
177
178		sdp->sdp_hashnext =
179		    sdt_probetab[SDT_ADDR2NDX(sdpd->sdpd_offset)];
180		sdt_probetab[SDT_ADDR2NDX(sdpd->sdpd_offset)] = sdp;
181
182		sdp->sdp_patchval = SDT_PATCHVAL;
183		sdp->sdp_patchpoint = (uint8_t *)sdpd->sdpd_offset;
184		sdp->sdp_savedval = *sdp->sdp_patchpoint;
185	}
186}
187
188/*ARGSUSED*/
189static void
190sdt_destroy(void *arg, dtrace_id_t id, void *parg)
191{
192	sdt_probe_t *sdp = parg, *old, *last, *hash;
193	struct modctl *ctl = sdp->sdp_ctl;
194	int ndx;
195
196	if (ctl != NULL && ctl->mod_loadcnt == sdp->sdp_loadcnt) {
197		if ((ctl->mod_loadcnt == sdp->sdp_loadcnt &&
198		    ctl->mod_loaded)) {
199			((struct module *)(ctl->mod_mp))->sdt_nprobes--;
200		}
201	}
202
203	while (sdp != NULL) {
204		old = sdp;
205
206		/*
207		 * Now we need to remove this probe from the sdt_probetab.
208		 */
209		ndx = SDT_ADDR2NDX(sdp->sdp_patchpoint);
210		last = NULL;
211		hash = sdt_probetab[ndx];
212
213		while (hash != sdp) {
214			ASSERT(hash != NULL);
215			last = hash;
216			hash = hash->sdp_hashnext;
217		}
218
219		if (last != NULL) {
220			last->sdp_hashnext = sdp->sdp_hashnext;
221		} else {
222			sdt_probetab[ndx] = sdp->sdp_hashnext;
223		}
224
225		kmem_free(sdp->sdp_name, sdp->sdp_namelen);
226		sdp = sdp->sdp_next;
227		kmem_free(old, sizeof (sdt_probe_t));
228	}
229}
230
231/*ARGSUSED*/
232static int
233sdt_enable(void *arg, dtrace_id_t id, void *parg)
234{
235	sdt_probe_t *sdp = parg;
236	struct modctl *ctl = sdp->sdp_ctl;
237
238	ctl->mod_nenabled++;
239
240	/*
241	 * If this module has disappeared since we discovered its probes,
242	 * refuse to enable it.
243	 */
244	if (!ctl->mod_loaded) {
245		if (sdt_verbose) {
246			cmn_err(CE_NOTE, "sdt is failing for probe %s "
247			    "(module %s unloaded)",
248			    sdp->sdp_name, ctl->mod_modname);
249		}
250		goto err;
251	}
252
253	/*
254	 * Now check that our modctl has the expected load count.  If it
255	 * doesn't, this module must have been unloaded and reloaded -- and
256	 * we're not going to touch it.
257	 */
258	if (ctl->mod_loadcnt != sdp->sdp_loadcnt) {
259		if (sdt_verbose) {
260			cmn_err(CE_NOTE, "sdt is failing for probe %s "
261			    "(module %s reloaded)",
262			    sdp->sdp_name, ctl->mod_modname);
263		}
264		goto err;
265	}
266
267	while (sdp != NULL) {
268		*sdp->sdp_patchpoint = sdp->sdp_patchval;
269		sdp = sdp->sdp_next;
270	}
271err:
272	return (0);
273}
274
275/*ARGSUSED*/
276static void
277sdt_disable(void *arg, dtrace_id_t id, void *parg)
278{
279	sdt_probe_t *sdp = parg;
280	struct modctl *ctl = sdp->sdp_ctl;
281
282	ctl->mod_nenabled--;
283
284	if (!ctl->mod_loaded || ctl->mod_loadcnt != sdp->sdp_loadcnt)
285		goto err;
286
287	while (sdp != NULL) {
288		*sdp->sdp_patchpoint = sdp->sdp_savedval;
289		sdp = sdp->sdp_next;
290	}
291
292err:
293	;
294}
295
296/*ARGSUSED*/
297uint64_t
298sdt_getarg(void *arg, dtrace_id_t id, void *parg, int argno, int aframes)
299{
300	uintptr_t val;
301	struct frame *fp = (struct frame *)dtrace_getfp();
302	uintptr_t *stack;
303	int i;
304#if defined(__amd64)
305	/*
306	 * A total of 6 arguments are passed via registers; any argument with
307	 * index of 5 or lower is therefore in a register.
308	 */
309	int inreg = 5;
310#endif
311
312	for (i = 1; i <= aframes; i++) {
313		fp = (struct frame *)(fp->fr_savfp);
314
315		if (fp->fr_savpc == (pc_t)dtrace_invop_callsite) {
316#if !defined(__amd64)
317			/*
318			 * If we pass through the invalid op handler, we will
319			 * use the pointer that it passed to the stack as the
320			 * second argument to dtrace_invop() as the pointer to
321			 * the stack.
322			 */
323			stack = ((uintptr_t **)&fp[1])[1];
324#else
325			/*
326			 * In the case of amd64, we will use the pointer to the
327			 * regs structure that was pushed when we took the
328			 * trap.  To get this structure, we must increment
329			 * beyond the frame structure.  If the argument that
330			 * we're seeking is passed on the stack, we'll pull
331			 * the true stack pointer out of the saved registers
332			 * and decrement our argument by the number of
333			 * arguments passed in registers; if the argument
334			 * we're seeking is passed in regsiters, we can just
335			 * load it directly.
336			 */
337			struct regs *rp = (struct regs *)((uintptr_t)&fp[1] +
338			    sizeof (uintptr_t));
339
340			if (argno <= inreg) {
341				stack = (uintptr_t *)&rp->r_rdi;
342			} else {
343				stack = (uintptr_t *)(rp->r_rsp);
344				argno -= (inreg + 1);
345			}
346#endif
347			goto load;
348		}
349	}
350
351	/*
352	 * We know that we did not come through a trap to get into
353	 * dtrace_probe() -- the provider simply called dtrace_probe()
354	 * directly.  As this is the case, we need to shift the argument
355	 * that we're looking for:  the probe ID is the first argument to
356	 * dtrace_probe(), so the argument n will actually be found where
357	 * one would expect to find argument (n + 1).
358	 */
359	argno++;
360
361#if defined(__amd64)
362	if (argno <= inreg) {
363		/*
364		 * This shouldn't happen.  If the argument is passed in a
365		 * register then it should have been, well, passed in a
366		 * register...
367		 */
368		DTRACE_CPUFLAG_SET(CPU_DTRACE_ILLOP);
369		return (0);
370	}
371
372	argno -= (inreg + 1);
373#endif
374	stack = (uintptr_t *)&fp[1];
375
376load:
377	DTRACE_CPUFLAG_SET(CPU_DTRACE_NOFAULT);
378	val = stack[argno];
379	DTRACE_CPUFLAG_CLEAR(CPU_DTRACE_NOFAULT);
380
381	return (val);
382}
383
384static dtrace_pops_t sdt_pops = {
385	NULL,
386	sdt_provide_module,
387	sdt_enable,
388	sdt_disable,
389	NULL,
390	NULL,
391	sdt_getargdesc,
392	sdt_getarg,
393	NULL,
394	sdt_destroy
395};
396
397/*ARGSUSED*/
398static int
399sdt_attach(dev_info_t *devi, ddi_attach_cmd_t cmd)
400{
401	sdt_provider_t *prov;
402
403	if (ddi_create_minor_node(devi, "sdt", S_IFCHR,
404	    0, DDI_PSEUDO, NULL) == DDI_FAILURE) {
405		cmn_err(CE_NOTE, "/dev/sdt couldn't create minor node");
406		ddi_remove_minor_node(devi, NULL);
407		return (DDI_FAILURE);
408	}
409
410	ddi_report_dev(devi);
411	sdt_devi = devi;
412
413	if (sdt_probetab_size == 0)
414		sdt_probetab_size = SDT_PROBETAB_SIZE;
415
416	sdt_probetab_mask = sdt_probetab_size - 1;
417	sdt_probetab =
418	    kmem_zalloc(sdt_probetab_size * sizeof (sdt_probe_t *), KM_SLEEP);
419	dtrace_invop_add(sdt_invop);
420
421	for (prov = sdt_providers; prov->sdtp_name != NULL; prov++) {
422		if (dtrace_register(prov->sdtp_name, prov->sdtp_attr,
423		    DTRACE_PRIV_KERNEL, NULL,
424		    &sdt_pops, prov, &prov->sdtp_id) != 0) {
425			cmn_err(CE_WARN, "failed to register sdt provider %s",
426			    prov->sdtp_name);
427		}
428	}
429
430	return (DDI_SUCCESS);
431}
432
433/*ARGSUSED*/
434static int
435sdt_detach(dev_info_t *dip, ddi_detach_cmd_t cmd)
436{
437	sdt_provider_t *prov;
438
439	switch (cmd) {
440	case DDI_DETACH:
441		break;
442
443	case DDI_SUSPEND:
444		return (DDI_SUCCESS);
445
446	default:
447		return (DDI_FAILURE);
448	}
449
450	for (prov = sdt_providers; prov->sdtp_name != NULL; prov++) {
451		if (prov->sdtp_id != DTRACE_PROVNONE) {
452			if (dtrace_unregister(prov->sdtp_id) != 0)
453				return (DDI_FAILURE);
454
455			prov->sdtp_id = DTRACE_PROVNONE;
456		}
457	}
458
459	dtrace_invop_remove(sdt_invop);
460	kmem_free(sdt_probetab, sdt_probetab_size * sizeof (sdt_probe_t *));
461
462	return (DDI_SUCCESS);
463}
464
465/*ARGSUSED*/
466static int
467sdt_info(dev_info_t *dip, ddi_info_cmd_t infocmd, void *arg, void **result)
468{
469	int error;
470
471	switch (infocmd) {
472	case DDI_INFO_DEVT2DEVINFO:
473		*result = (void *)sdt_devi;
474		error = DDI_SUCCESS;
475		break;
476	case DDI_INFO_DEVT2INSTANCE:
477		*result = (void *)0;
478		error = DDI_SUCCESS;
479		break;
480	default:
481		error = DDI_FAILURE;
482	}
483	return (error);
484}
485
486/*ARGSUSED*/
487static int
488sdt_open(dev_t *devp, int flag, int otyp, cred_t *cred_p)
489{
490	return (0);
491}
492
493static struct cb_ops sdt_cb_ops = {
494	sdt_open,		/* open */
495	nodev,			/* close */
496	nulldev,		/* strategy */
497	nulldev,		/* print */
498	nodev,			/* dump */
499	nodev,			/* read */
500	nodev,			/* write */
501	nodev,			/* ioctl */
502	nodev,			/* devmap */
503	nodev,			/* mmap */
504	nodev,			/* segmap */
505	nochpoll,		/* poll */
506	ddi_prop_op,		/* cb_prop_op */
507	0,			/* streamtab  */
508	D_NEW | D_MP		/* Driver compatibility flag */
509};
510
511static struct dev_ops sdt_ops = {
512	DEVO_REV,		/* devo_rev, */
513	0,			/* refcnt  */
514	sdt_info,		/* get_dev_info */
515	nulldev,		/* identify */
516	nulldev,		/* probe */
517	sdt_attach,		/* attach */
518	sdt_detach,		/* detach */
519	nodev,			/* reset */
520	&sdt_cb_ops,		/* driver operations */
521	NULL,			/* bus operations */
522	nodev,			/* dev power */
523	ddi_quiesce_not_needed,		/* quiesce */
524};
525
526/*
527 * Module linkage information for the kernel.
528 */
529static struct modldrv modldrv = {
530	&mod_driverops,		/* module type (this is a pseudo driver) */
531	"Statically Defined Tracing",	/* name of module */
532	&sdt_ops,		/* driver ops */
533};
534
535static struct modlinkage modlinkage = {
536	MODREV_1,
537	(void *)&modldrv,
538	NULL
539};
540
541int
542_init(void)
543{
544	return (mod_install(&modlinkage));
545}
546
547int
548_info(struct modinfo *modinfop)
549{
550	return (mod_info(&modlinkage, modinfop));
551}
552
553int
554_fini(void)
555{
556	return (mod_remove(&modlinkage));
557}
558