1/*
2 * CDDL HEADER START
3 *
4 * The contents of this file are subject to the terms of the
5 * Common Development and Distribution License (the "License").
6 * You may not use this file except in compliance with the License.
7 *
8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 * or http://www.opensolaris.org/os/licensing.
10 * See the License for the specific language governing permissions
11 * and limitations under the License.
12 *
13 * When distributing Covered Code, include this CDDL HEADER in each
14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 * If applicable, add the following below this CDDL HEADER, with the
16 * fields enclosed by brackets "[]" replaced with your own identifying
17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 *
19 * CDDL HEADER END
20 *
21 * Portions Copyright 2010 The FreeBSD Foundation
22 *
23 * $FreeBSD$
24 */
25
26/*
27 * Copyright 2008 Sun Microsystems, Inc.  All rights reserved.
28 * Use is subject to license terms.
29 */
30
31#if defined(sun)
32#pragma ident	"%Z%%M%	%I%	%E% SMI"
33#endif
34
35#include <sys/atomic.h>
36#include <sys/errno.h>
37#include <sys/stat.h>
38#include <sys/modctl.h>
39#include <sys/conf.h>
40#include <sys/systm.h>
41#if defined(sun)
42#include <sys/ddi.h>
43#endif
44#include <sys/sunddi.h>
45#include <sys/cpuvar.h>
46#include <sys/kmem.h>
47#if defined(sun)
48#include <sys/strsubr.h>
49#endif
50#include <sys/fasttrap.h>
51#include <sys/fasttrap_impl.h>
52#include <sys/fasttrap_isa.h>
53#include <sys/dtrace.h>
54#include <sys/dtrace_impl.h>
55#include <sys/sysmacros.h>
56#include <sys/proc.h>
57#include <sys/policy.h>
58#if defined(sun)
59#include <util/qsort.h>
60#endif
61#include <sys/mutex.h>
62#include <sys/kernel.h>
63#if !defined(sun)
64#include <sys/user.h>
65#include <sys/dtrace_bsd.h>
66#include <cddl/dev/dtrace/dtrace_cddl.h>
67#endif
68
69/*
70 * User-Land Trap-Based Tracing
71 * ----------------------------
72 *
73 * The fasttrap provider allows DTrace consumers to instrument any user-level
74 * instruction to gather data; this includes probes with semantic
75 * signifigance like entry and return as well as simple offsets into the
76 * function. While the specific techniques used are very ISA specific, the
77 * methodology is generalizable to any architecture.
78 *
79 *
80 * The General Methodology
81 * -----------------------
82 *
83 * With the primary goal of tracing every user-land instruction and the
84 * limitation that we can't trust user space so don't want to rely on much
85 * information there, we begin by replacing the instructions we want to trace
86 * with trap instructions. Each instruction we overwrite is saved into a hash
87 * table keyed by process ID and pc address. When we enter the kernel due to
88 * this trap instruction, we need the effects of the replaced instruction to
89 * appear to have occurred before we proceed with the user thread's
90 * execution.
91 *
92 * Each user level thread is represented by a ulwp_t structure which is
93 * always easily accessible through a register. The most basic way to produce
94 * the effects of the instruction we replaced is to copy that instruction out
95 * to a bit of scratch space reserved in the user thread's ulwp_t structure
96 * (a sort of kernel-private thread local storage), set the PC to that
97 * scratch space and single step. When we reenter the kernel after single
98 * stepping the instruction we must then adjust the PC to point to what would
99 * normally be the next instruction. Of course, special care must be taken
100 * for branches and jumps, but these represent such a small fraction of any
101 * instruction set that writing the code to emulate these in the kernel is
102 * not too difficult.
103 *
104 * Return probes may require several tracepoints to trace every return site,
105 * and, conversely, each tracepoint may activate several probes (the entry
106 * and offset 0 probes, for example). To solve this muliplexing problem,
107 * tracepoints contain lists of probes to activate and probes contain lists
108 * of tracepoints to enable. If a probe is activated, it adds its ID to
109 * existing tracepoints or creates new ones as necessary.
110 *
111 * Most probes are activated _before_ the instruction is executed, but return
112 * probes are activated _after_ the effects of the last instruction of the
113 * function are visible. Return probes must be fired _after_ we have
114 * single-stepped the instruction whereas all other probes are fired
115 * beforehand.
116 *
117 *
118 * Lock Ordering
119 * -------------
120 *
121 * The lock ordering below -- both internally and with respect to the DTrace
122 * framework -- is a little tricky and bears some explanation. Each provider
123 * has a lock (ftp_mtx) that protects its members including reference counts
124 * for enabled probes (ftp_rcount), consumers actively creating probes
125 * (ftp_ccount) and USDT consumers (ftp_mcount); all three prevent a provider
126 * from being freed. A provider is looked up by taking the bucket lock for the
127 * provider hash table, and is returned with its lock held. The provider lock
128 * may be taken in functions invoked by the DTrace framework, but may not be
129 * held while calling functions in the DTrace framework.
130 *
131 * To ensure consistency over multiple calls to the DTrace framework, the
132 * creation lock (ftp_cmtx) should be held. Naturally, the creation lock may
133 * not be taken when holding the provider lock as that would create a cyclic
134 * lock ordering. In situations where one would naturally take the provider
135 * lock and then the creation lock, we instead up a reference count to prevent
136 * the provider from disappearing, drop the provider lock, and acquire the
137 * creation lock.
138 *
139 * Briefly:
140 * 	bucket lock before provider lock
141 *	DTrace before provider lock
142 *	creation lock before DTrace
143 *	never hold the provider lock and creation lock simultaneously
144 */
145
146static d_open_t fasttrap_open;
147static d_ioctl_t fasttrap_ioctl;
148
149static struct cdevsw fasttrap_cdevsw = {
150	.d_version	= D_VERSION,
151	.d_open		= fasttrap_open,
152	.d_ioctl	= fasttrap_ioctl,
153	.d_name		= "fasttrap",
154};
155static struct cdev *fasttrap_cdev;
156static dtrace_meta_provider_id_t fasttrap_meta_id;
157
158static struct proc *fasttrap_cleanup_proc;
159static struct mtx fasttrap_cleanup_mtx;
160static uint_t fasttrap_cleanup_work, fasttrap_cleanup_drain, fasttrap_cleanup_cv;
161
162/*
163 * Generation count on modifications to the global tracepoint lookup table.
164 */
165static volatile uint64_t fasttrap_mod_gen;
166
167/*
168 * When the fasttrap provider is loaded, fasttrap_max is set to either
169 * FASTTRAP_MAX_DEFAULT or the value for fasttrap-max-probes in the
170 * fasttrap.conf file. Each time a probe is created, fasttrap_total is
171 * incremented by the number of tracepoints that may be associated with that
172 * probe; fasttrap_total is capped at fasttrap_max.
173 */
174#define	FASTTRAP_MAX_DEFAULT		250000
175static uint32_t fasttrap_max;
176static uint32_t fasttrap_total;
177
178/*
179 * Copyright (c) 2011, Joyent, Inc. All rights reserved.
180 */
181
182#define	FASTTRAP_TPOINTS_DEFAULT_SIZE	0x4000
183#define	FASTTRAP_PROVIDERS_DEFAULT_SIZE	0x100
184#define	FASTTRAP_PROCS_DEFAULT_SIZE	0x100
185
186#define	FASTTRAP_PID_NAME		"pid"
187
188fasttrap_hash_t			fasttrap_tpoints;
189static fasttrap_hash_t		fasttrap_provs;
190static fasttrap_hash_t		fasttrap_procs;
191
192static uint64_t			fasttrap_pid_count;	/* pid ref count */
193static kmutex_t			fasttrap_count_mtx;	/* lock on ref count */
194
195#define	FASTTRAP_ENABLE_FAIL	1
196#define	FASTTRAP_ENABLE_PARTIAL	2
197
198static int fasttrap_tracepoint_enable(proc_t *, fasttrap_probe_t *, uint_t);
199static void fasttrap_tracepoint_disable(proc_t *, fasttrap_probe_t *, uint_t);
200
201static fasttrap_provider_t *fasttrap_provider_lookup(pid_t, const char *,
202    const dtrace_pattr_t *);
203static void fasttrap_provider_retire(pid_t, const char *, int);
204static void fasttrap_provider_free(fasttrap_provider_t *);
205
206static fasttrap_proc_t *fasttrap_proc_lookup(pid_t);
207static void fasttrap_proc_release(fasttrap_proc_t *);
208
209#define	FASTTRAP_PROVS_INDEX(pid, name) \
210	((fasttrap_hash_str(name) + (pid)) & fasttrap_provs.fth_mask)
211
212#define	FASTTRAP_PROCS_INDEX(pid) ((pid) & fasttrap_procs.fth_mask)
213
214#if !defined(sun)
215static kmutex_t fasttrap_cpuc_pid_lock[MAXCPU];
216#endif
217
218static int
219fasttrap_highbit(ulong_t i)
220{
221	int h = 1;
222
223	if (i == 0)
224		return (0);
225#ifdef _LP64
226	if (i & 0xffffffff00000000ul) {
227		h += 32; i >>= 32;
228	}
229#endif
230	if (i & 0xffff0000) {
231		h += 16; i >>= 16;
232	}
233	if (i & 0xff00) {
234		h += 8; i >>= 8;
235	}
236	if (i & 0xf0) {
237		h += 4; i >>= 4;
238	}
239	if (i & 0xc) {
240		h += 2; i >>= 2;
241	}
242	if (i & 0x2) {
243		h += 1;
244	}
245	return (h);
246}
247
248static uint_t
249fasttrap_hash_str(const char *p)
250{
251	unsigned int g;
252	uint_t hval = 0;
253
254	while (*p) {
255		hval = (hval << 4) + *p++;
256		if ((g = (hval & 0xf0000000)) != 0)
257			hval ^= g >> 24;
258		hval &= ~g;
259	}
260	return (hval);
261}
262
263void
264fasttrap_sigtrap(proc_t *p, kthread_t *t, uintptr_t pc)
265{
266#if defined(sun)
267	sigqueue_t *sqp = kmem_zalloc(sizeof (sigqueue_t), KM_SLEEP);
268
269	sqp->sq_info.si_signo = SIGTRAP;
270	sqp->sq_info.si_code = TRAP_DTRACE;
271	sqp->sq_info.si_addr = (caddr_t)pc;
272
273	mutex_enter(&p->p_lock);
274	sigaddqa(p, t, sqp);
275	mutex_exit(&p->p_lock);
276
277	if (t != NULL)
278		aston(t);
279#else
280	ksiginfo_t *ksi = kmem_zalloc(sizeof (ksiginfo_t), KM_SLEEP);
281
282	ksiginfo_init(ksi);
283	ksi->ksi_signo = SIGTRAP;
284	ksi->ksi_code = TRAP_DTRACE;
285	ksi->ksi_addr = (caddr_t)pc;
286	PROC_LOCK(p);
287	(void) tdksignal(t, SIGTRAP, ksi);
288	PROC_UNLOCK(p);
289#endif
290}
291
292/*
293 * This function ensures that no threads are actively using the memory
294 * associated with probes that were formerly live.
295 */
296static void
297fasttrap_mod_barrier(uint64_t gen)
298{
299	int i;
300
301	if (gen < fasttrap_mod_gen)
302		return;
303
304	fasttrap_mod_gen++;
305
306	CPU_FOREACH(i) {
307		mutex_enter(&fasttrap_cpuc_pid_lock[i]);
308		mutex_exit(&fasttrap_cpuc_pid_lock[i]);
309	}
310}
311
312/*
313 * This function performs asynchronous cleanup of fasttrap providers. The
314 * Solaris implementation of this mechanism use a timeout that's activated in
315 * fasttrap_pid_cleanup(), but this doesn't work in FreeBSD: one may sleep while
316 * holding the DTrace mutexes, but it is unsafe to sleep in a callout handler.
317 * Thus we use a dedicated process to perform the cleanup when requested.
318 */
319/*ARGSUSED*/
320static void
321fasttrap_pid_cleanup_cb(void *data)
322{
323	fasttrap_provider_t **fpp, *fp;
324	fasttrap_bucket_t *bucket;
325	dtrace_provider_id_t provid;
326	int i, later = 0, rval;
327
328	mtx_lock(&fasttrap_cleanup_mtx);
329	while (!fasttrap_cleanup_drain || later > 0) {
330		fasttrap_cleanup_work = 0;
331		mtx_unlock(&fasttrap_cleanup_mtx);
332
333		later = 0;
334
335		/*
336		 * Iterate over all the providers trying to remove the marked
337		 * ones. If a provider is marked but not retired, we just
338		 * have to take a crack at removing it -- it's no big deal if
339		 * we can't.
340		 */
341		for (i = 0; i < fasttrap_provs.fth_nent; i++) {
342			bucket = &fasttrap_provs.fth_table[i];
343			mutex_enter(&bucket->ftb_mtx);
344			fpp = (fasttrap_provider_t **)&bucket->ftb_data;
345
346			while ((fp = *fpp) != NULL) {
347				if (!fp->ftp_marked) {
348					fpp = &fp->ftp_next;
349					continue;
350				}
351
352				mutex_enter(&fp->ftp_mtx);
353
354				/*
355				 * If this provider has consumers actively
356				 * creating probes (ftp_ccount) or is a USDT
357				 * provider (ftp_mcount), we can't unregister
358				 * or even condense.
359				 */
360				if (fp->ftp_ccount != 0 ||
361				    fp->ftp_mcount != 0) {
362					mutex_exit(&fp->ftp_mtx);
363					fp->ftp_marked = 0;
364					continue;
365				}
366
367				if (!fp->ftp_retired || fp->ftp_rcount != 0)
368					fp->ftp_marked = 0;
369
370				mutex_exit(&fp->ftp_mtx);
371
372				/*
373				 * If we successfully unregister this
374				 * provider we can remove it from the hash
375				 * chain and free the memory. If our attempt
376				 * to unregister fails and this is a retired
377				 * provider, increment our flag to try again
378				 * pretty soon. If we've consumed more than
379				 * half of our total permitted number of
380				 * probes call dtrace_condense() to try to
381				 * clean out the unenabled probes.
382				 */
383				provid = fp->ftp_provid;
384				if ((rval = dtrace_unregister(provid)) != 0) {
385					if (fasttrap_total > fasttrap_max / 2)
386						(void) dtrace_condense(provid);
387
388					if (rval == EAGAIN)
389						fp->ftp_marked = 1;
390
391					later += fp->ftp_marked;
392					fpp = &fp->ftp_next;
393				} else {
394					*fpp = fp->ftp_next;
395					fasttrap_provider_free(fp);
396				}
397			}
398			mutex_exit(&bucket->ftb_mtx);
399		}
400		mtx_lock(&fasttrap_cleanup_mtx);
401
402		/*
403		 * If we were unable to retire a provider, try again after a
404		 * second. This situation can occur in certain circumstances
405		 * where providers cannot be unregistered even though they have
406		 * no probes enabled because of an execution of dtrace -l or
407		 * something similar.
408		 */
409		if (later > 0 || fasttrap_cleanup_work ||
410		    fasttrap_cleanup_drain) {
411			mtx_unlock(&fasttrap_cleanup_mtx);
412			pause("ftclean", hz);
413			mtx_lock(&fasttrap_cleanup_mtx);
414		} else
415			mtx_sleep(&fasttrap_cleanup_cv, &fasttrap_cleanup_mtx,
416			    0, "ftcl", 0);
417	}
418
419	/*
420	 * Wake up the thread in fasttrap_unload() now that we're done.
421	 */
422	wakeup(&fasttrap_cleanup_drain);
423	mtx_unlock(&fasttrap_cleanup_mtx);
424
425	kthread_exit();
426}
427
428/*
429 * Activates the asynchronous cleanup mechanism.
430 */
431static void
432fasttrap_pid_cleanup(void)
433{
434
435	mtx_lock(&fasttrap_cleanup_mtx);
436	if (!fasttrap_cleanup_work) {
437		fasttrap_cleanup_work = 1;
438		wakeup(&fasttrap_cleanup_cv);
439	}
440	mtx_unlock(&fasttrap_cleanup_mtx);
441}
442
443/*
444 * This is called from cfork() via dtrace_fasttrap_fork(). The child
445 * process's address space is (roughly) a copy of the parent process's so
446 * we have to remove all the instrumentation we had previously enabled in the
447 * parent.
448 */
449static void
450fasttrap_fork(proc_t *p, proc_t *cp)
451{
452	pid_t ppid = p->p_pid;
453	int i;
454
455#if defined(sun)
456	ASSERT(curproc == p);
457	ASSERT(p->p_proc_flag & P_PR_LOCK);
458#else
459	PROC_LOCK_ASSERT(p, MA_OWNED);
460#endif
461#if defined(sun)
462	ASSERT(p->p_dtrace_count > 0);
463#else
464	if (p->p_dtrace_helpers) {
465		/*
466		 * dtrace_helpers_duplicate() allocates memory.
467		 */
468		_PHOLD(cp);
469		PROC_UNLOCK(p);
470		PROC_UNLOCK(cp);
471		dtrace_helpers_duplicate(p, cp);
472		PROC_LOCK(cp);
473		PROC_LOCK(p);
474		_PRELE(cp);
475	}
476	/*
477	 * This check is purposely here instead of in kern_fork.c because,
478	 * for legal resons, we cannot include the dtrace_cddl.h header
479	 * inside kern_fork.c and insert if-clause there.
480	 */
481	if (p->p_dtrace_count == 0)
482		return;
483#endif
484	ASSERT(cp->p_dtrace_count == 0);
485
486	/*
487	 * This would be simpler and faster if we maintained per-process
488	 * hash tables of enabled tracepoints. It could, however, potentially
489	 * slow down execution of a tracepoint since we'd need to go
490	 * through two levels of indirection. In the future, we should
491	 * consider either maintaining per-process ancillary lists of
492	 * enabled tracepoints or hanging a pointer to a per-process hash
493	 * table of enabled tracepoints off the proc structure.
494	 */
495
496	/*
497	 * We don't have to worry about the child process disappearing
498	 * because we're in fork().
499	 */
500#if defined(sun)
501	mtx_lock_spin(&cp->p_slock);
502	sprlock_proc(cp);
503	mtx_unlock_spin(&cp->p_slock);
504#else
505	_PHOLD(cp);
506#endif
507
508	/*
509	 * Iterate over every tracepoint looking for ones that belong to the
510	 * parent process, and remove each from the child process.
511	 */
512	for (i = 0; i < fasttrap_tpoints.fth_nent; i++) {
513		fasttrap_tracepoint_t *tp;
514		fasttrap_bucket_t *bucket = &fasttrap_tpoints.fth_table[i];
515
516		mutex_enter(&bucket->ftb_mtx);
517		for (tp = bucket->ftb_data; tp != NULL; tp = tp->ftt_next) {
518			if (tp->ftt_pid == ppid &&
519			    tp->ftt_proc->ftpc_acount != 0) {
520				int ret = fasttrap_tracepoint_remove(cp, tp);
521				ASSERT(ret == 0);
522
523				/*
524				 * The count of active providers can only be
525				 * decremented (i.e. to zero) during exec,
526				 * exit, and removal of a meta provider so it
527				 * should be impossible to drop the count
528				 * mid-fork.
529				 */
530				ASSERT(tp->ftt_proc->ftpc_acount != 0);
531			}
532		}
533		mutex_exit(&bucket->ftb_mtx);
534	}
535
536#if defined(sun)
537	mutex_enter(&cp->p_lock);
538	sprunlock(cp);
539#else
540	_PRELE(cp);
541#endif
542}
543
544/*
545 * This is called from proc_exit() or from exec_common() if p_dtrace_probes
546 * is set on the proc structure to indicate that there is a pid provider
547 * associated with this process.
548 */
549static void
550fasttrap_exec_exit(proc_t *p)
551{
552#if defined(sun)
553	ASSERT(p == curproc);
554#endif
555	PROC_LOCK_ASSERT(p, MA_OWNED);
556	_PHOLD(p);
557	PROC_UNLOCK(p);
558
559	/*
560	 * We clean up the pid provider for this process here; user-land
561	 * static probes are handled by the meta-provider remove entry point.
562	 */
563	fasttrap_provider_retire(p->p_pid, FASTTRAP_PID_NAME, 0);
564#if !defined(sun)
565	if (p->p_dtrace_helpers)
566		dtrace_helpers_destroy(p);
567#endif
568	PROC_LOCK(p);
569	_PRELE(p);
570}
571
572
573/*ARGSUSED*/
574static void
575fasttrap_pid_provide(void *arg, dtrace_probedesc_t *desc)
576{
577	/*
578	 * There are no "default" pid probes.
579	 */
580}
581
582static int
583fasttrap_tracepoint_enable(proc_t *p, fasttrap_probe_t *probe, uint_t index)
584{
585	fasttrap_tracepoint_t *tp, *new_tp = NULL;
586	fasttrap_bucket_t *bucket;
587	fasttrap_id_t *id;
588	pid_t pid;
589	uintptr_t pc;
590
591	ASSERT(index < probe->ftp_ntps);
592
593	pid = probe->ftp_pid;
594	pc = probe->ftp_tps[index].fit_tp->ftt_pc;
595	id = &probe->ftp_tps[index].fit_id;
596
597	ASSERT(probe->ftp_tps[index].fit_tp->ftt_pid == pid);
598
599#if defined(sun)
600	ASSERT(!(p->p_flag & SVFORK));
601#endif
602
603	/*
604	 * Before we make any modifications, make sure we've imposed a barrier
605	 * on the generation in which this probe was last modified.
606	 */
607	fasttrap_mod_barrier(probe->ftp_gen);
608
609	bucket = &fasttrap_tpoints.fth_table[FASTTRAP_TPOINTS_INDEX(pid, pc)];
610
611	/*
612	 * If the tracepoint has already been enabled, just add our id to the
613	 * list of interested probes. This may be our second time through
614	 * this path in which case we'll have constructed the tracepoint we'd
615	 * like to install. If we can't find a match, and have an allocated
616	 * tracepoint ready to go, enable that one now.
617	 *
618	 * A tracepoint whose process is defunct is also considered defunct.
619	 */
620again:
621	mutex_enter(&bucket->ftb_mtx);
622	for (tp = bucket->ftb_data; tp != NULL; tp = tp->ftt_next) {
623		/*
624		 * Note that it's safe to access the active count on the
625		 * associated proc structure because we know that at least one
626		 * provider (this one) will still be around throughout this
627		 * operation.
628		 */
629		if (tp->ftt_pid != pid || tp->ftt_pc != pc ||
630		    tp->ftt_proc->ftpc_acount == 0)
631			continue;
632
633		/*
634		 * Now that we've found a matching tracepoint, it would be
635		 * a decent idea to confirm that the tracepoint is still
636		 * enabled and the trap instruction hasn't been overwritten.
637		 * Since this is a little hairy, we'll punt for now.
638		 */
639
640		/*
641		 * This can't be the first interested probe. We don't have
642		 * to worry about another thread being in the midst of
643		 * deleting this tracepoint (which would be the only valid
644		 * reason for a tracepoint to have no interested probes)
645		 * since we're holding P_PR_LOCK for this process.
646		 */
647		ASSERT(tp->ftt_ids != NULL || tp->ftt_retids != NULL);
648
649		switch (id->fti_ptype) {
650		case DTFTP_ENTRY:
651		case DTFTP_OFFSETS:
652		case DTFTP_IS_ENABLED:
653			id->fti_next = tp->ftt_ids;
654			membar_producer();
655			tp->ftt_ids = id;
656			membar_producer();
657			break;
658
659		case DTFTP_RETURN:
660		case DTFTP_POST_OFFSETS:
661			id->fti_next = tp->ftt_retids;
662			membar_producer();
663			tp->ftt_retids = id;
664			membar_producer();
665			break;
666
667		default:
668			ASSERT(0);
669		}
670
671		mutex_exit(&bucket->ftb_mtx);
672
673		if (new_tp != NULL) {
674			new_tp->ftt_ids = NULL;
675			new_tp->ftt_retids = NULL;
676		}
677
678		return (0);
679	}
680
681	/*
682	 * If we have a good tracepoint ready to go, install it now while
683	 * we have the lock held and no one can screw with us.
684	 */
685	if (new_tp != NULL) {
686		int rc = 0;
687
688		new_tp->ftt_next = bucket->ftb_data;
689		membar_producer();
690		bucket->ftb_data = new_tp;
691		membar_producer();
692		mutex_exit(&bucket->ftb_mtx);
693
694		/*
695		 * Activate the tracepoint in the ISA-specific manner.
696		 * If this fails, we need to report the failure, but
697		 * indicate that this tracepoint must still be disabled
698		 * by calling fasttrap_tracepoint_disable().
699		 */
700		if (fasttrap_tracepoint_install(p, new_tp) != 0)
701			rc = FASTTRAP_ENABLE_PARTIAL;
702
703		/*
704		 * Increment the count of the number of tracepoints active in
705		 * the victim process.
706		 */
707#if defined(sun)
708		ASSERT(p->p_proc_flag & P_PR_LOCK);
709#endif
710		p->p_dtrace_count++;
711
712		return (rc);
713	}
714
715	mutex_exit(&bucket->ftb_mtx);
716
717	/*
718	 * Initialize the tracepoint that's been preallocated with the probe.
719	 */
720	new_tp = probe->ftp_tps[index].fit_tp;
721
722	ASSERT(new_tp->ftt_pid == pid);
723	ASSERT(new_tp->ftt_pc == pc);
724	ASSERT(new_tp->ftt_proc == probe->ftp_prov->ftp_proc);
725	ASSERT(new_tp->ftt_ids == NULL);
726	ASSERT(new_tp->ftt_retids == NULL);
727
728	switch (id->fti_ptype) {
729	case DTFTP_ENTRY:
730	case DTFTP_OFFSETS:
731	case DTFTP_IS_ENABLED:
732		id->fti_next = NULL;
733		new_tp->ftt_ids = id;
734		break;
735
736	case DTFTP_RETURN:
737	case DTFTP_POST_OFFSETS:
738		id->fti_next = NULL;
739		new_tp->ftt_retids = id;
740		break;
741
742	default:
743		ASSERT(0);
744	}
745
746	/*
747	 * If the ISA-dependent initialization goes to plan, go back to the
748	 * beginning and try to install this freshly made tracepoint.
749	 */
750	if (fasttrap_tracepoint_init(p, new_tp, pc, id->fti_ptype) == 0)
751		goto again;
752
753	new_tp->ftt_ids = NULL;
754	new_tp->ftt_retids = NULL;
755
756	return (FASTTRAP_ENABLE_FAIL);
757}
758
759static void
760fasttrap_tracepoint_disable(proc_t *p, fasttrap_probe_t *probe, uint_t index)
761{
762	fasttrap_bucket_t *bucket;
763	fasttrap_provider_t *provider = probe->ftp_prov;
764	fasttrap_tracepoint_t **pp, *tp;
765	fasttrap_id_t *id, **idp = NULL;
766	pid_t pid;
767	uintptr_t pc;
768
769	ASSERT(index < probe->ftp_ntps);
770
771	pid = probe->ftp_pid;
772	pc = probe->ftp_tps[index].fit_tp->ftt_pc;
773	id = &probe->ftp_tps[index].fit_id;
774
775	ASSERT(probe->ftp_tps[index].fit_tp->ftt_pid == pid);
776
777	/*
778	 * Find the tracepoint and make sure that our id is one of the
779	 * ones registered with it.
780	 */
781	bucket = &fasttrap_tpoints.fth_table[FASTTRAP_TPOINTS_INDEX(pid, pc)];
782	mutex_enter(&bucket->ftb_mtx);
783	for (tp = bucket->ftb_data; tp != NULL; tp = tp->ftt_next) {
784		if (tp->ftt_pid == pid && tp->ftt_pc == pc &&
785		    tp->ftt_proc == provider->ftp_proc)
786			break;
787	}
788
789	/*
790	 * If we somehow lost this tracepoint, we're in a world of hurt.
791	 */
792	ASSERT(tp != NULL);
793
794	switch (id->fti_ptype) {
795	case DTFTP_ENTRY:
796	case DTFTP_OFFSETS:
797	case DTFTP_IS_ENABLED:
798		ASSERT(tp->ftt_ids != NULL);
799		idp = &tp->ftt_ids;
800		break;
801
802	case DTFTP_RETURN:
803	case DTFTP_POST_OFFSETS:
804		ASSERT(tp->ftt_retids != NULL);
805		idp = &tp->ftt_retids;
806		break;
807
808	default:
809		ASSERT(0);
810	}
811
812	while ((*idp)->fti_probe != probe) {
813		idp = &(*idp)->fti_next;
814		ASSERT(*idp != NULL);
815	}
816
817	id = *idp;
818	*idp = id->fti_next;
819	membar_producer();
820
821	ASSERT(id->fti_probe == probe);
822
823	/*
824	 * If there are other registered enablings of this tracepoint, we're
825	 * all done, but if this was the last probe assocated with this
826	 * this tracepoint, we need to remove and free it.
827	 */
828	if (tp->ftt_ids != NULL || tp->ftt_retids != NULL) {
829
830		/*
831		 * If the current probe's tracepoint is in use, swap it
832		 * for an unused tracepoint.
833		 */
834		if (tp == probe->ftp_tps[index].fit_tp) {
835			fasttrap_probe_t *tmp_probe;
836			fasttrap_tracepoint_t **tmp_tp;
837			uint_t tmp_index;
838
839			if (tp->ftt_ids != NULL) {
840				tmp_probe = tp->ftt_ids->fti_probe;
841				/* LINTED - alignment */
842				tmp_index = FASTTRAP_ID_INDEX(tp->ftt_ids);
843				tmp_tp = &tmp_probe->ftp_tps[tmp_index].fit_tp;
844			} else {
845				tmp_probe = tp->ftt_retids->fti_probe;
846				/* LINTED - alignment */
847				tmp_index = FASTTRAP_ID_INDEX(tp->ftt_retids);
848				tmp_tp = &tmp_probe->ftp_tps[tmp_index].fit_tp;
849			}
850
851			ASSERT(*tmp_tp != NULL);
852			ASSERT(*tmp_tp != probe->ftp_tps[index].fit_tp);
853			ASSERT((*tmp_tp)->ftt_ids == NULL);
854			ASSERT((*tmp_tp)->ftt_retids == NULL);
855
856			probe->ftp_tps[index].fit_tp = *tmp_tp;
857			*tmp_tp = tp;
858		}
859
860		mutex_exit(&bucket->ftb_mtx);
861
862		/*
863		 * Tag the modified probe with the generation in which it was
864		 * changed.
865		 */
866		probe->ftp_gen = fasttrap_mod_gen;
867		return;
868	}
869
870	mutex_exit(&bucket->ftb_mtx);
871
872	/*
873	 * We can't safely remove the tracepoint from the set of active
874	 * tracepoints until we've actually removed the fasttrap instruction
875	 * from the process's text. We can, however, operate on this
876	 * tracepoint secure in the knowledge that no other thread is going to
877	 * be looking at it since we hold P_PR_LOCK on the process if it's
878	 * live or we hold the provider lock on the process if it's dead and
879	 * gone.
880	 */
881
882	/*
883	 * We only need to remove the actual instruction if we're looking
884	 * at an existing process
885	 */
886	if (p != NULL) {
887		/*
888		 * If we fail to restore the instruction we need to kill
889		 * this process since it's in a completely unrecoverable
890		 * state.
891		 */
892		if (fasttrap_tracepoint_remove(p, tp) != 0)
893			fasttrap_sigtrap(p, NULL, pc);
894
895		/*
896		 * Decrement the count of the number of tracepoints active
897		 * in the victim process.
898		 */
899#if defined(sun)
900		ASSERT(p->p_proc_flag & P_PR_LOCK);
901#endif
902		p->p_dtrace_count--;
903	}
904
905	/*
906	 * Remove the probe from the hash table of active tracepoints.
907	 */
908	mutex_enter(&bucket->ftb_mtx);
909	pp = (fasttrap_tracepoint_t **)&bucket->ftb_data;
910	ASSERT(*pp != NULL);
911	while (*pp != tp) {
912		pp = &(*pp)->ftt_next;
913		ASSERT(*pp != NULL);
914	}
915
916	*pp = tp->ftt_next;
917	membar_producer();
918
919	mutex_exit(&bucket->ftb_mtx);
920
921	/*
922	 * Tag the modified probe with the generation in which it was changed.
923	 */
924	probe->ftp_gen = fasttrap_mod_gen;
925}
926
927static void
928fasttrap_enable_callbacks(void)
929{
930	/*
931	 * We don't have to play the rw lock game here because we're
932	 * providing something rather than taking something away --
933	 * we can be sure that no threads have tried to follow this
934	 * function pointer yet.
935	 */
936	mutex_enter(&fasttrap_count_mtx);
937	if (fasttrap_pid_count == 0) {
938		ASSERT(dtrace_pid_probe_ptr == NULL);
939		ASSERT(dtrace_return_probe_ptr == NULL);
940		dtrace_pid_probe_ptr = &fasttrap_pid_probe;
941		dtrace_return_probe_ptr = &fasttrap_return_probe;
942	}
943	ASSERT(dtrace_pid_probe_ptr == &fasttrap_pid_probe);
944	ASSERT(dtrace_return_probe_ptr == &fasttrap_return_probe);
945	fasttrap_pid_count++;
946	mutex_exit(&fasttrap_count_mtx);
947}
948
949static void
950fasttrap_disable_callbacks(void)
951{
952#if defined(sun)
953	ASSERT(MUTEX_HELD(&cpu_lock));
954#endif
955
956
957	mutex_enter(&fasttrap_count_mtx);
958	ASSERT(fasttrap_pid_count > 0);
959	fasttrap_pid_count--;
960	if (fasttrap_pid_count == 0) {
961#if defined(sun)
962		cpu_t *cur, *cpu = CPU;
963
964		for (cur = cpu->cpu_next_onln; cur != cpu;
965		    cur = cur->cpu_next_onln) {
966			rw_enter(&cur->cpu_ft_lock, RW_WRITER);
967		}
968#endif
969		dtrace_pid_probe_ptr = NULL;
970		dtrace_return_probe_ptr = NULL;
971#if defined(sun)
972		for (cur = cpu->cpu_next_onln; cur != cpu;
973		    cur = cur->cpu_next_onln) {
974			rw_exit(&cur->cpu_ft_lock);
975		}
976#endif
977	}
978	mutex_exit(&fasttrap_count_mtx);
979}
980
981/*ARGSUSED*/
982static void
983fasttrap_pid_enable(void *arg, dtrace_id_t id, void *parg)
984{
985	fasttrap_probe_t *probe = parg;
986	proc_t *p = NULL;
987	int i, rc;
988
989	ASSERT(probe != NULL);
990	ASSERT(!probe->ftp_enabled);
991	ASSERT(id == probe->ftp_id);
992#if defined(sun)
993	ASSERT(MUTEX_HELD(&cpu_lock));
994#endif
995
996	/*
997	 * Increment the count of enabled probes on this probe's provider;
998	 * the provider can't go away while the probe still exists. We
999	 * must increment this even if we aren't able to properly enable
1000	 * this probe.
1001	 */
1002	mutex_enter(&probe->ftp_prov->ftp_mtx);
1003	probe->ftp_prov->ftp_rcount++;
1004	mutex_exit(&probe->ftp_prov->ftp_mtx);
1005
1006	/*
1007	 * If this probe's provider is retired (meaning it was valid in a
1008	 * previously exec'ed incarnation of this address space), bail out. The
1009	 * provider can't go away while we're in this code path.
1010	 */
1011	if (probe->ftp_prov->ftp_retired)
1012		return;
1013
1014	/*
1015	 * If we can't find the process, it may be that we're in the context of
1016	 * a fork in which the traced process is being born and we're copying
1017	 * USDT probes. Otherwise, the process is gone so bail.
1018	 */
1019#if defined(sun)
1020	if ((p = sprlock(probe->ftp_pid)) == NULL) {
1021		if ((curproc->p_flag & SFORKING) == 0)
1022			return;
1023
1024		mutex_enter(&pidlock);
1025		p = prfind(probe->ftp_pid);
1026
1027		/*
1028		 * Confirm that curproc is indeed forking the process in which
1029		 * we're trying to enable probes.
1030		 */
1031		ASSERT(p != NULL);
1032		ASSERT(p->p_parent == curproc);
1033		ASSERT(p->p_stat == SIDL);
1034
1035		mutex_enter(&p->p_lock);
1036		mutex_exit(&pidlock);
1037
1038		sprlock_proc(p);
1039	}
1040
1041	ASSERT(!(p->p_flag & SVFORK));
1042	mutex_exit(&p->p_lock);
1043#else
1044	if ((p = pfind(probe->ftp_pid)) == NULL)
1045		return;
1046#endif
1047
1048	/*
1049	 * We have to enable the trap entry point before any user threads have
1050	 * the chance to execute the trap instruction we're about to place
1051	 * in their process's text.
1052	 */
1053#ifdef __FreeBSD__
1054	/*
1055	 * pfind() returns a locked process.
1056	 */
1057	_PHOLD(p);
1058	PROC_UNLOCK(p);
1059#endif
1060	fasttrap_enable_callbacks();
1061
1062	/*
1063	 * Enable all the tracepoints and add this probe's id to each
1064	 * tracepoint's list of active probes.
1065	 */
1066	for (i = 0; i < probe->ftp_ntps; i++) {
1067		if ((rc = fasttrap_tracepoint_enable(p, probe, i)) != 0) {
1068			/*
1069			 * If enabling the tracepoint failed completely,
1070			 * we don't have to disable it; if the failure
1071			 * was only partial we must disable it.
1072			 */
1073			if (rc == FASTTRAP_ENABLE_FAIL)
1074				i--;
1075			else
1076				ASSERT(rc == FASTTRAP_ENABLE_PARTIAL);
1077
1078			/*
1079			 * Back up and pull out all the tracepoints we've
1080			 * created so far for this probe.
1081			 */
1082			while (i >= 0) {
1083				fasttrap_tracepoint_disable(p, probe, i);
1084				i--;
1085			}
1086
1087#if defined(sun)
1088			mutex_enter(&p->p_lock);
1089			sprunlock(p);
1090#else
1091			PRELE(p);
1092#endif
1093
1094			/*
1095			 * Since we're not actually enabling this probe,
1096			 * drop our reference on the trap table entry.
1097			 */
1098			fasttrap_disable_callbacks();
1099			return;
1100		}
1101	}
1102#if defined(sun)
1103	mutex_enter(&p->p_lock);
1104	sprunlock(p);
1105#else
1106	PRELE(p);
1107#endif
1108
1109	probe->ftp_enabled = 1;
1110}
1111
1112/*ARGSUSED*/
1113static void
1114fasttrap_pid_disable(void *arg, dtrace_id_t id, void *parg)
1115{
1116	fasttrap_probe_t *probe = parg;
1117	fasttrap_provider_t *provider = probe->ftp_prov;
1118	proc_t *p;
1119	int i, whack = 0;
1120
1121	ASSERT(id == probe->ftp_id);
1122
1123	mutex_enter(&provider->ftp_mtx);
1124
1125	/*
1126	 * We won't be able to acquire a /proc-esque lock on the process
1127	 * iff the process is dead and gone. In this case, we rely on the
1128	 * provider lock as a point of mutual exclusion to prevent other
1129	 * DTrace consumers from disabling this probe.
1130	 */
1131	if ((p = pfind(probe->ftp_pid)) != NULL) {
1132#ifdef __FreeBSD__
1133		_PHOLD(p);
1134		PROC_UNLOCK(p);
1135#endif
1136	}
1137
1138	/*
1139	 * Disable all the associated tracepoints (for fully enabled probes).
1140	 */
1141	if (probe->ftp_enabled) {
1142		for (i = 0; i < probe->ftp_ntps; i++) {
1143			fasttrap_tracepoint_disable(p, probe, i);
1144		}
1145	}
1146
1147	ASSERT(provider->ftp_rcount > 0);
1148	provider->ftp_rcount--;
1149
1150	if (p != NULL) {
1151		/*
1152		 * Even though we may not be able to remove it entirely, we
1153		 * mark this retired provider to get a chance to remove some
1154		 * of the associated probes.
1155		 */
1156		if (provider->ftp_retired && !provider->ftp_marked)
1157			whack = provider->ftp_marked = 1;
1158		mutex_exit(&provider->ftp_mtx);
1159	} else {
1160		/*
1161		 * If the process is dead, we're just waiting for the
1162		 * last probe to be disabled to be able to free it.
1163		 */
1164		if (provider->ftp_rcount == 0 && !provider->ftp_marked)
1165			whack = provider->ftp_marked = 1;
1166		mutex_exit(&provider->ftp_mtx);
1167	}
1168
1169	if (whack)
1170		fasttrap_pid_cleanup();
1171
1172#ifdef __FreeBSD__
1173	if (p != NULL)
1174		PRELE(p);
1175#endif
1176	if (!probe->ftp_enabled)
1177		return;
1178
1179	probe->ftp_enabled = 0;
1180
1181#if defined(sun)
1182	ASSERT(MUTEX_HELD(&cpu_lock));
1183#endif
1184	fasttrap_disable_callbacks();
1185}
1186
1187/*ARGSUSED*/
1188static void
1189fasttrap_pid_getargdesc(void *arg, dtrace_id_t id, void *parg,
1190    dtrace_argdesc_t *desc)
1191{
1192	fasttrap_probe_t *probe = parg;
1193	char *str;
1194	int i, ndx;
1195
1196	desc->dtargd_native[0] = '\0';
1197	desc->dtargd_xlate[0] = '\0';
1198
1199	if (probe->ftp_prov->ftp_retired != 0 ||
1200	    desc->dtargd_ndx >= probe->ftp_nargs) {
1201		desc->dtargd_ndx = DTRACE_ARGNONE;
1202		return;
1203	}
1204
1205	ndx = (probe->ftp_argmap != NULL) ?
1206	    probe->ftp_argmap[desc->dtargd_ndx] : desc->dtargd_ndx;
1207
1208	str = probe->ftp_ntypes;
1209	for (i = 0; i < ndx; i++) {
1210		str += strlen(str) + 1;
1211	}
1212
1213	ASSERT(strlen(str + 1) < sizeof (desc->dtargd_native));
1214	(void) strcpy(desc->dtargd_native, str);
1215
1216	if (probe->ftp_xtypes == NULL)
1217		return;
1218
1219	str = probe->ftp_xtypes;
1220	for (i = 0; i < desc->dtargd_ndx; i++) {
1221		str += strlen(str) + 1;
1222	}
1223
1224	ASSERT(strlen(str + 1) < sizeof (desc->dtargd_xlate));
1225	(void) strcpy(desc->dtargd_xlate, str);
1226}
1227
1228/*ARGSUSED*/
1229static void
1230fasttrap_pid_destroy(void *arg, dtrace_id_t id, void *parg)
1231{
1232	fasttrap_probe_t *probe = parg;
1233	int i;
1234	size_t size;
1235
1236	ASSERT(probe != NULL);
1237	ASSERT(!probe->ftp_enabled);
1238	ASSERT(fasttrap_total >= probe->ftp_ntps);
1239
1240	atomic_add_32(&fasttrap_total, -probe->ftp_ntps);
1241	size = offsetof(fasttrap_probe_t, ftp_tps[probe->ftp_ntps]);
1242
1243	if (probe->ftp_gen + 1 >= fasttrap_mod_gen)
1244		fasttrap_mod_barrier(probe->ftp_gen);
1245
1246	for (i = 0; i < probe->ftp_ntps; i++) {
1247		kmem_free(probe->ftp_tps[i].fit_tp,
1248		    sizeof (fasttrap_tracepoint_t));
1249	}
1250
1251	kmem_free(probe, size);
1252}
1253
1254
1255static const dtrace_pattr_t pid_attr = {
1256{ DTRACE_STABILITY_EVOLVING, DTRACE_STABILITY_EVOLVING, DTRACE_CLASS_ISA },
1257{ DTRACE_STABILITY_PRIVATE, DTRACE_STABILITY_PRIVATE, DTRACE_CLASS_UNKNOWN },
1258{ DTRACE_STABILITY_PRIVATE, DTRACE_STABILITY_PRIVATE, DTRACE_CLASS_UNKNOWN },
1259{ DTRACE_STABILITY_EVOLVING, DTRACE_STABILITY_EVOLVING, DTRACE_CLASS_ISA },
1260{ DTRACE_STABILITY_PRIVATE, DTRACE_STABILITY_PRIVATE, DTRACE_CLASS_UNKNOWN },
1261};
1262
1263static dtrace_pops_t pid_pops = {
1264	fasttrap_pid_provide,
1265	NULL,
1266	fasttrap_pid_enable,
1267	fasttrap_pid_disable,
1268	NULL,
1269	NULL,
1270	fasttrap_pid_getargdesc,
1271	fasttrap_pid_getarg,
1272	NULL,
1273	fasttrap_pid_destroy
1274};
1275
1276static dtrace_pops_t usdt_pops = {
1277	fasttrap_pid_provide,
1278	NULL,
1279	fasttrap_pid_enable,
1280	fasttrap_pid_disable,
1281	NULL,
1282	NULL,
1283	fasttrap_pid_getargdesc,
1284	fasttrap_usdt_getarg,
1285	NULL,
1286	fasttrap_pid_destroy
1287};
1288
1289static fasttrap_proc_t *
1290fasttrap_proc_lookup(pid_t pid)
1291{
1292	fasttrap_bucket_t *bucket;
1293	fasttrap_proc_t *fprc, *new_fprc;
1294
1295
1296	bucket = &fasttrap_procs.fth_table[FASTTRAP_PROCS_INDEX(pid)];
1297	mutex_enter(&bucket->ftb_mtx);
1298
1299	for (fprc = bucket->ftb_data; fprc != NULL; fprc = fprc->ftpc_next) {
1300		if (fprc->ftpc_pid == pid && fprc->ftpc_acount != 0) {
1301			mutex_enter(&fprc->ftpc_mtx);
1302			mutex_exit(&bucket->ftb_mtx);
1303			fprc->ftpc_rcount++;
1304			atomic_add_64(&fprc->ftpc_acount, 1);
1305			ASSERT(fprc->ftpc_acount <= fprc->ftpc_rcount);
1306			mutex_exit(&fprc->ftpc_mtx);
1307
1308			return (fprc);
1309		}
1310	}
1311
1312	/*
1313	 * Drop the bucket lock so we don't try to perform a sleeping
1314	 * allocation under it.
1315	 */
1316	mutex_exit(&bucket->ftb_mtx);
1317
1318	new_fprc = kmem_zalloc(sizeof (fasttrap_proc_t), KM_SLEEP);
1319	new_fprc->ftpc_pid = pid;
1320	new_fprc->ftpc_rcount = 1;
1321	new_fprc->ftpc_acount = 1;
1322#if !defined(sun)
1323	mutex_init(&new_fprc->ftpc_mtx, "fasttrap proc mtx", MUTEX_DEFAULT,
1324	    NULL);
1325#endif
1326
1327	mutex_enter(&bucket->ftb_mtx);
1328
1329	/*
1330	 * Take another lap through the list to make sure a proc hasn't
1331	 * been created for this pid while we weren't under the bucket lock.
1332	 */
1333	for (fprc = bucket->ftb_data; fprc != NULL; fprc = fprc->ftpc_next) {
1334		if (fprc->ftpc_pid == pid && fprc->ftpc_acount != 0) {
1335			mutex_enter(&fprc->ftpc_mtx);
1336			mutex_exit(&bucket->ftb_mtx);
1337			fprc->ftpc_rcount++;
1338			atomic_add_64(&fprc->ftpc_acount, 1);
1339			ASSERT(fprc->ftpc_acount <= fprc->ftpc_rcount);
1340			mutex_exit(&fprc->ftpc_mtx);
1341
1342			kmem_free(new_fprc, sizeof (fasttrap_proc_t));
1343
1344			return (fprc);
1345		}
1346	}
1347
1348	new_fprc->ftpc_next = bucket->ftb_data;
1349	bucket->ftb_data = new_fprc;
1350
1351	mutex_exit(&bucket->ftb_mtx);
1352
1353	return (new_fprc);
1354}
1355
1356static void
1357fasttrap_proc_release(fasttrap_proc_t *proc)
1358{
1359	fasttrap_bucket_t *bucket;
1360	fasttrap_proc_t *fprc, **fprcp;
1361	pid_t pid = proc->ftpc_pid;
1362
1363	mutex_enter(&proc->ftpc_mtx);
1364
1365	ASSERT(proc->ftpc_rcount != 0);
1366	ASSERT(proc->ftpc_acount <= proc->ftpc_rcount);
1367
1368	if (--proc->ftpc_rcount != 0) {
1369		mutex_exit(&proc->ftpc_mtx);
1370		return;
1371	}
1372
1373	mutex_exit(&proc->ftpc_mtx);
1374
1375	/*
1376	 * There should definitely be no live providers associated with this
1377	 * process at this point.
1378	 */
1379	ASSERT(proc->ftpc_acount == 0);
1380
1381	bucket = &fasttrap_procs.fth_table[FASTTRAP_PROCS_INDEX(pid)];
1382	mutex_enter(&bucket->ftb_mtx);
1383
1384	fprcp = (fasttrap_proc_t **)&bucket->ftb_data;
1385	while ((fprc = *fprcp) != NULL) {
1386		if (fprc == proc)
1387			break;
1388
1389		fprcp = &fprc->ftpc_next;
1390	}
1391
1392	/*
1393	 * Something strange has happened if we can't find the proc.
1394	 */
1395	ASSERT(fprc != NULL);
1396
1397	*fprcp = fprc->ftpc_next;
1398
1399	mutex_exit(&bucket->ftb_mtx);
1400
1401	kmem_free(fprc, sizeof (fasttrap_proc_t));
1402}
1403
1404/*
1405 * Lookup a fasttrap-managed provider based on its name and associated pid.
1406 * If the pattr argument is non-NULL, this function instantiates the provider
1407 * if it doesn't exist otherwise it returns NULL. The provider is returned
1408 * with its lock held.
1409 */
1410static fasttrap_provider_t *
1411fasttrap_provider_lookup(pid_t pid, const char *name,
1412    const dtrace_pattr_t *pattr)
1413{
1414	fasttrap_provider_t *fp, *new_fp = NULL;
1415	fasttrap_bucket_t *bucket;
1416	char provname[DTRACE_PROVNAMELEN];
1417	proc_t *p;
1418	cred_t *cred;
1419
1420	ASSERT(strlen(name) < sizeof (fp->ftp_name));
1421	ASSERT(pattr != NULL);
1422
1423	bucket = &fasttrap_provs.fth_table[FASTTRAP_PROVS_INDEX(pid, name)];
1424	mutex_enter(&bucket->ftb_mtx);
1425
1426	/*
1427	 * Take a lap through the list and return the match if we find it.
1428	 */
1429	for (fp = bucket->ftb_data; fp != NULL; fp = fp->ftp_next) {
1430		if (fp->ftp_pid == pid && strcmp(fp->ftp_name, name) == 0 &&
1431		    !fp->ftp_retired) {
1432			mutex_enter(&fp->ftp_mtx);
1433			mutex_exit(&bucket->ftb_mtx);
1434			return (fp);
1435		}
1436	}
1437
1438	/*
1439	 * Drop the bucket lock so we don't try to perform a sleeping
1440	 * allocation under it.
1441	 */
1442	mutex_exit(&bucket->ftb_mtx);
1443
1444	/*
1445	 * Make sure the process exists, isn't a child created as the result
1446	 * of a vfork(2), and isn't a zombie (but may be in fork).
1447	 */
1448	if ((p = pfind(pid)) == NULL)
1449		return (NULL);
1450
1451	/*
1452	 * Increment p_dtrace_probes so that the process knows to inform us
1453	 * when it exits or execs. fasttrap_provider_free() decrements this
1454	 * when we're done with this provider.
1455	 */
1456	p->p_dtrace_probes++;
1457
1458	/*
1459	 * Grab the credentials for this process so we have
1460	 * something to pass to dtrace_register().
1461	 */
1462	PROC_LOCK_ASSERT(p, MA_OWNED);
1463	crhold(p->p_ucred);
1464	cred = p->p_ucred;
1465	PROC_UNLOCK(p);
1466
1467	new_fp = kmem_zalloc(sizeof (fasttrap_provider_t), KM_SLEEP);
1468	new_fp->ftp_pid = pid;
1469	new_fp->ftp_proc = fasttrap_proc_lookup(pid);
1470#if !defined(sun)
1471	mutex_init(&new_fp->ftp_mtx, "provider mtx", MUTEX_DEFAULT, NULL);
1472	mutex_init(&new_fp->ftp_cmtx, "lock on creating", MUTEX_DEFAULT, NULL);
1473#endif
1474
1475	ASSERT(new_fp->ftp_proc != NULL);
1476
1477	mutex_enter(&bucket->ftb_mtx);
1478
1479	/*
1480	 * Take another lap through the list to make sure a provider hasn't
1481	 * been created for this pid while we weren't under the bucket lock.
1482	 */
1483	for (fp = bucket->ftb_data; fp != NULL; fp = fp->ftp_next) {
1484		if (fp->ftp_pid == pid && strcmp(fp->ftp_name, name) == 0 &&
1485		    !fp->ftp_retired) {
1486			mutex_enter(&fp->ftp_mtx);
1487			mutex_exit(&bucket->ftb_mtx);
1488			fasttrap_provider_free(new_fp);
1489			crfree(cred);
1490			return (fp);
1491		}
1492	}
1493
1494	(void) strcpy(new_fp->ftp_name, name);
1495
1496	/*
1497	 * Fail and return NULL if either the provider name is too long
1498	 * or we fail to register this new provider with the DTrace
1499	 * framework. Note that this is the only place we ever construct
1500	 * the full provider name -- we keep it in pieces in the provider
1501	 * structure.
1502	 */
1503	if (snprintf(provname, sizeof (provname), "%s%u", name, (uint_t)pid) >=
1504	    sizeof (provname) ||
1505	    dtrace_register(provname, pattr,
1506	    DTRACE_PRIV_PROC | DTRACE_PRIV_OWNER | DTRACE_PRIV_ZONEOWNER, cred,
1507	    pattr == &pid_attr ? &pid_pops : &usdt_pops, new_fp,
1508	    &new_fp->ftp_provid) != 0) {
1509		mutex_exit(&bucket->ftb_mtx);
1510		fasttrap_provider_free(new_fp);
1511		crfree(cred);
1512		return (NULL);
1513	}
1514
1515	new_fp->ftp_next = bucket->ftb_data;
1516	bucket->ftb_data = new_fp;
1517
1518	mutex_enter(&new_fp->ftp_mtx);
1519	mutex_exit(&bucket->ftb_mtx);
1520
1521	crfree(cred);
1522	return (new_fp);
1523}
1524
1525static void
1526fasttrap_provider_free(fasttrap_provider_t *provider)
1527{
1528	pid_t pid = provider->ftp_pid;
1529	proc_t *p;
1530
1531	/*
1532	 * There need to be no associated enabled probes, no consumers
1533	 * creating probes, and no meta providers referencing this provider.
1534	 */
1535	ASSERT(provider->ftp_rcount == 0);
1536	ASSERT(provider->ftp_ccount == 0);
1537	ASSERT(provider->ftp_mcount == 0);
1538
1539	/*
1540	 * If this provider hasn't been retired, we need to explicitly drop the
1541	 * count of active providers on the associated process structure.
1542	 */
1543	if (!provider->ftp_retired) {
1544		atomic_add_64(&provider->ftp_proc->ftpc_acount, -1);
1545		ASSERT(provider->ftp_proc->ftpc_acount <
1546		    provider->ftp_proc->ftpc_rcount);
1547	}
1548
1549	fasttrap_proc_release(provider->ftp_proc);
1550
1551#if !defined(sun)
1552	mutex_destroy(&provider->ftp_mtx);
1553	mutex_destroy(&provider->ftp_cmtx);
1554#endif
1555	kmem_free(provider, sizeof (fasttrap_provider_t));
1556
1557	/*
1558	 * Decrement p_dtrace_probes on the process whose provider we're
1559	 * freeing. We don't have to worry about clobbering somone else's
1560	 * modifications to it because we have locked the bucket that
1561	 * corresponds to this process's hash chain in the provider hash
1562	 * table. Don't sweat it if we can't find the process.
1563	 */
1564	if ((p = pfind(pid)) == NULL) {
1565		return;
1566	}
1567
1568	p->p_dtrace_probes--;
1569#if !defined(sun)
1570	PROC_UNLOCK(p);
1571#endif
1572}
1573
1574static void
1575fasttrap_provider_retire(pid_t pid, const char *name, int mprov)
1576{
1577	fasttrap_provider_t *fp;
1578	fasttrap_bucket_t *bucket;
1579	dtrace_provider_id_t provid;
1580
1581	ASSERT(strlen(name) < sizeof (fp->ftp_name));
1582
1583	bucket = &fasttrap_provs.fth_table[FASTTRAP_PROVS_INDEX(pid, name)];
1584	mutex_enter(&bucket->ftb_mtx);
1585
1586	for (fp = bucket->ftb_data; fp != NULL; fp = fp->ftp_next) {
1587		if (fp->ftp_pid == pid && strcmp(fp->ftp_name, name) == 0 &&
1588		    !fp->ftp_retired)
1589			break;
1590	}
1591
1592	if (fp == NULL) {
1593		mutex_exit(&bucket->ftb_mtx);
1594		return;
1595	}
1596
1597	mutex_enter(&fp->ftp_mtx);
1598	ASSERT(!mprov || fp->ftp_mcount > 0);
1599	if (mprov && --fp->ftp_mcount != 0)  {
1600		mutex_exit(&fp->ftp_mtx);
1601		mutex_exit(&bucket->ftb_mtx);
1602		return;
1603	}
1604
1605	/*
1606	 * Mark the provider to be removed in our post-processing step, mark it
1607	 * retired, and drop the active count on its proc. Marking it indicates
1608	 * that we should try to remove it; setting the retired flag indicates
1609	 * that we're done with this provider; dropping the active the proc
1610	 * releases our hold, and when this reaches zero (as it will during
1611	 * exit or exec) the proc and associated providers become defunct.
1612	 *
1613	 * We obviously need to take the bucket lock before the provider lock
1614	 * to perform the lookup, but we need to drop the provider lock
1615	 * before calling into the DTrace framework since we acquire the
1616	 * provider lock in callbacks invoked from the DTrace framework. The
1617	 * bucket lock therefore protects the integrity of the provider hash
1618	 * table.
1619	 */
1620	atomic_add_64(&fp->ftp_proc->ftpc_acount, -1);
1621	ASSERT(fp->ftp_proc->ftpc_acount < fp->ftp_proc->ftpc_rcount);
1622
1623	fp->ftp_retired = 1;
1624	fp->ftp_marked = 1;
1625	provid = fp->ftp_provid;
1626	mutex_exit(&fp->ftp_mtx);
1627
1628	/*
1629	 * We don't have to worry about invalidating the same provider twice
1630	 * since fasttrap_provider_lookup() will ignore provider that have
1631	 * been marked as retired.
1632	 */
1633	dtrace_invalidate(provid);
1634
1635	mutex_exit(&bucket->ftb_mtx);
1636
1637	fasttrap_pid_cleanup();
1638}
1639
1640static int
1641fasttrap_uint32_cmp(const void *ap, const void *bp)
1642{
1643	return (*(const uint32_t *)ap - *(const uint32_t *)bp);
1644}
1645
1646static int
1647fasttrap_uint64_cmp(const void *ap, const void *bp)
1648{
1649	return (*(const uint64_t *)ap - *(const uint64_t *)bp);
1650}
1651
1652static int
1653fasttrap_add_probe(fasttrap_probe_spec_t *pdata)
1654{
1655	fasttrap_provider_t *provider;
1656	fasttrap_probe_t *pp;
1657	fasttrap_tracepoint_t *tp;
1658	char *name;
1659	int i, aframes = 0, whack;
1660
1661	/*
1662	 * There needs to be at least one desired trace point.
1663	 */
1664	if (pdata->ftps_noffs == 0)
1665		return (EINVAL);
1666
1667	switch (pdata->ftps_type) {
1668	case DTFTP_ENTRY:
1669		name = "entry";
1670		aframes = FASTTRAP_ENTRY_AFRAMES;
1671		break;
1672	case DTFTP_RETURN:
1673		name = "return";
1674		aframes = FASTTRAP_RETURN_AFRAMES;
1675		break;
1676	case DTFTP_OFFSETS:
1677		name = NULL;
1678		break;
1679	default:
1680		return (EINVAL);
1681	}
1682
1683	if ((provider = fasttrap_provider_lookup(pdata->ftps_pid,
1684	    FASTTRAP_PID_NAME, &pid_attr)) == NULL)
1685		return (ESRCH);
1686
1687	/*
1688	 * Increment this reference count to indicate that a consumer is
1689	 * actively adding a new probe associated with this provider. This
1690	 * prevents the provider from being deleted -- we'll need to check
1691	 * for pending deletions when we drop this reference count.
1692	 */
1693	provider->ftp_ccount++;
1694	mutex_exit(&provider->ftp_mtx);
1695
1696	/*
1697	 * Grab the creation lock to ensure consistency between calls to
1698	 * dtrace_probe_lookup() and dtrace_probe_create() in the face of
1699	 * other threads creating probes. We must drop the provider lock
1700	 * before taking this lock to avoid a three-way deadlock with the
1701	 * DTrace framework.
1702	 */
1703	mutex_enter(&provider->ftp_cmtx);
1704
1705	if (name == NULL) {
1706		for (i = 0; i < pdata->ftps_noffs; i++) {
1707			char name_str[17];
1708
1709			(void) sprintf(name_str, "%llx",
1710			    (unsigned long long)pdata->ftps_offs[i]);
1711
1712			if (dtrace_probe_lookup(provider->ftp_provid,
1713			    pdata->ftps_mod, pdata->ftps_func, name_str) != 0)
1714				continue;
1715
1716			atomic_add_32(&fasttrap_total, 1);
1717
1718			if (fasttrap_total > fasttrap_max) {
1719				atomic_add_32(&fasttrap_total, -1);
1720				goto no_mem;
1721			}
1722
1723			pp = kmem_zalloc(sizeof (fasttrap_probe_t), KM_SLEEP);
1724
1725			pp->ftp_prov = provider;
1726			pp->ftp_faddr = pdata->ftps_pc;
1727			pp->ftp_fsize = pdata->ftps_size;
1728			pp->ftp_pid = pdata->ftps_pid;
1729			pp->ftp_ntps = 1;
1730
1731			tp = kmem_zalloc(sizeof (fasttrap_tracepoint_t),
1732			    KM_SLEEP);
1733
1734			tp->ftt_proc = provider->ftp_proc;
1735			tp->ftt_pc = pdata->ftps_offs[i] + pdata->ftps_pc;
1736			tp->ftt_pid = pdata->ftps_pid;
1737
1738			pp->ftp_tps[0].fit_tp = tp;
1739			pp->ftp_tps[0].fit_id.fti_probe = pp;
1740			pp->ftp_tps[0].fit_id.fti_ptype = pdata->ftps_type;
1741
1742			pp->ftp_id = dtrace_probe_create(provider->ftp_provid,
1743			    pdata->ftps_mod, pdata->ftps_func, name_str,
1744			    FASTTRAP_OFFSET_AFRAMES, pp);
1745		}
1746
1747	} else if (dtrace_probe_lookup(provider->ftp_provid, pdata->ftps_mod,
1748	    pdata->ftps_func, name) == 0) {
1749		atomic_add_32(&fasttrap_total, pdata->ftps_noffs);
1750
1751		if (fasttrap_total > fasttrap_max) {
1752			atomic_add_32(&fasttrap_total, -pdata->ftps_noffs);
1753			goto no_mem;
1754		}
1755
1756		/*
1757		 * Make sure all tracepoint program counter values are unique.
1758		 * We later assume that each probe has exactly one tracepoint
1759		 * for a given pc.
1760		 */
1761		qsort(pdata->ftps_offs, pdata->ftps_noffs,
1762		    sizeof (uint64_t), fasttrap_uint64_cmp);
1763		for (i = 1; i < pdata->ftps_noffs; i++) {
1764			if (pdata->ftps_offs[i] > pdata->ftps_offs[i - 1])
1765				continue;
1766
1767			atomic_add_32(&fasttrap_total, -pdata->ftps_noffs);
1768			goto no_mem;
1769		}
1770
1771		ASSERT(pdata->ftps_noffs > 0);
1772		pp = kmem_zalloc(offsetof(fasttrap_probe_t,
1773		    ftp_tps[pdata->ftps_noffs]), KM_SLEEP);
1774
1775		pp->ftp_prov = provider;
1776		pp->ftp_faddr = pdata->ftps_pc;
1777		pp->ftp_fsize = pdata->ftps_size;
1778		pp->ftp_pid = pdata->ftps_pid;
1779		pp->ftp_ntps = pdata->ftps_noffs;
1780
1781		for (i = 0; i < pdata->ftps_noffs; i++) {
1782			tp = kmem_zalloc(sizeof (fasttrap_tracepoint_t),
1783			    KM_SLEEP);
1784
1785			tp->ftt_proc = provider->ftp_proc;
1786			tp->ftt_pc = pdata->ftps_offs[i] + pdata->ftps_pc;
1787			tp->ftt_pid = pdata->ftps_pid;
1788
1789			pp->ftp_tps[i].fit_tp = tp;
1790			pp->ftp_tps[i].fit_id.fti_probe = pp;
1791			pp->ftp_tps[i].fit_id.fti_ptype = pdata->ftps_type;
1792		}
1793
1794		pp->ftp_id = dtrace_probe_create(provider->ftp_provid,
1795		    pdata->ftps_mod, pdata->ftps_func, name, aframes, pp);
1796	}
1797
1798	mutex_exit(&provider->ftp_cmtx);
1799
1800	/*
1801	 * We know that the provider is still valid since we incremented the
1802	 * creation reference count. If someone tried to clean up this provider
1803	 * while we were using it (e.g. because the process called exec(2) or
1804	 * exit(2)), take note of that and try to clean it up now.
1805	 */
1806	mutex_enter(&provider->ftp_mtx);
1807	provider->ftp_ccount--;
1808	whack = provider->ftp_retired;
1809	mutex_exit(&provider->ftp_mtx);
1810
1811	if (whack)
1812		fasttrap_pid_cleanup();
1813
1814	return (0);
1815
1816no_mem:
1817	/*
1818	 * If we've exhausted the allowable resources, we'll try to remove
1819	 * this provider to free some up. This is to cover the case where
1820	 * the user has accidentally created many more probes than was
1821	 * intended (e.g. pid123:::).
1822	 */
1823	mutex_exit(&provider->ftp_cmtx);
1824	mutex_enter(&provider->ftp_mtx);
1825	provider->ftp_ccount--;
1826	provider->ftp_marked = 1;
1827	mutex_exit(&provider->ftp_mtx);
1828
1829	fasttrap_pid_cleanup();
1830
1831	return (ENOMEM);
1832}
1833
1834/*ARGSUSED*/
1835static void *
1836fasttrap_meta_provide(void *arg, dtrace_helper_provdesc_t *dhpv, pid_t pid)
1837{
1838	fasttrap_provider_t *provider;
1839
1840	/*
1841	 * A 32-bit unsigned integer (like a pid for example) can be
1842	 * expressed in 10 or fewer decimal digits. Make sure that we'll
1843	 * have enough space for the provider name.
1844	 */
1845	if (strlen(dhpv->dthpv_provname) + 10 >=
1846	    sizeof (provider->ftp_name)) {
1847		printf("failed to instantiate provider %s: "
1848		    "name too long to accomodate pid", dhpv->dthpv_provname);
1849		return (NULL);
1850	}
1851
1852	/*
1853	 * Don't let folks spoof the true pid provider.
1854	 */
1855	if (strcmp(dhpv->dthpv_provname, FASTTRAP_PID_NAME) == 0) {
1856		printf("failed to instantiate provider %s: "
1857		    "%s is an invalid name", dhpv->dthpv_provname,
1858		    FASTTRAP_PID_NAME);
1859		return (NULL);
1860	}
1861
1862	/*
1863	 * The highest stability class that fasttrap supports is ISA; cap
1864	 * the stability of the new provider accordingly.
1865	 */
1866	if (dhpv->dthpv_pattr.dtpa_provider.dtat_class > DTRACE_CLASS_ISA)
1867		dhpv->dthpv_pattr.dtpa_provider.dtat_class = DTRACE_CLASS_ISA;
1868	if (dhpv->dthpv_pattr.dtpa_mod.dtat_class > DTRACE_CLASS_ISA)
1869		dhpv->dthpv_pattr.dtpa_mod.dtat_class = DTRACE_CLASS_ISA;
1870	if (dhpv->dthpv_pattr.dtpa_func.dtat_class > DTRACE_CLASS_ISA)
1871		dhpv->dthpv_pattr.dtpa_func.dtat_class = DTRACE_CLASS_ISA;
1872	if (dhpv->dthpv_pattr.dtpa_name.dtat_class > DTRACE_CLASS_ISA)
1873		dhpv->dthpv_pattr.dtpa_name.dtat_class = DTRACE_CLASS_ISA;
1874	if (dhpv->dthpv_pattr.dtpa_args.dtat_class > DTRACE_CLASS_ISA)
1875		dhpv->dthpv_pattr.dtpa_args.dtat_class = DTRACE_CLASS_ISA;
1876
1877	if ((provider = fasttrap_provider_lookup(pid, dhpv->dthpv_provname,
1878	    &dhpv->dthpv_pattr)) == NULL) {
1879		printf("failed to instantiate provider %s for "
1880		    "process %u",  dhpv->dthpv_provname, (uint_t)pid);
1881		return (NULL);
1882	}
1883
1884	/*
1885	 * Up the meta provider count so this provider isn't removed until
1886	 * the meta provider has been told to remove it.
1887	 */
1888	provider->ftp_mcount++;
1889
1890	mutex_exit(&provider->ftp_mtx);
1891
1892	return (provider);
1893}
1894
1895/*ARGSUSED*/
1896static void
1897fasttrap_meta_create_probe(void *arg, void *parg,
1898    dtrace_helper_probedesc_t *dhpb)
1899{
1900	fasttrap_provider_t *provider = parg;
1901	fasttrap_probe_t *pp;
1902	fasttrap_tracepoint_t *tp;
1903	int i, j;
1904	uint32_t ntps;
1905
1906	/*
1907	 * Since the meta provider count is non-zero we don't have to worry
1908	 * about this provider disappearing.
1909	 */
1910	ASSERT(provider->ftp_mcount > 0);
1911
1912	/*
1913	 * The offsets must be unique.
1914	 */
1915	qsort(dhpb->dthpb_offs, dhpb->dthpb_noffs, sizeof (uint32_t),
1916	    fasttrap_uint32_cmp);
1917	for (i = 1; i < dhpb->dthpb_noffs; i++) {
1918		if (dhpb->dthpb_base + dhpb->dthpb_offs[i] <=
1919		    dhpb->dthpb_base + dhpb->dthpb_offs[i - 1])
1920			return;
1921	}
1922
1923	qsort(dhpb->dthpb_enoffs, dhpb->dthpb_nenoffs, sizeof (uint32_t),
1924	    fasttrap_uint32_cmp);
1925	for (i = 1; i < dhpb->dthpb_nenoffs; i++) {
1926		if (dhpb->dthpb_base + dhpb->dthpb_enoffs[i] <=
1927		    dhpb->dthpb_base + dhpb->dthpb_enoffs[i - 1])
1928			return;
1929	}
1930
1931	/*
1932	 * Grab the creation lock to ensure consistency between calls to
1933	 * dtrace_probe_lookup() and dtrace_probe_create() in the face of
1934	 * other threads creating probes.
1935	 */
1936	mutex_enter(&provider->ftp_cmtx);
1937
1938	if (dtrace_probe_lookup(provider->ftp_provid, dhpb->dthpb_mod,
1939	    dhpb->dthpb_func, dhpb->dthpb_name) != 0) {
1940		mutex_exit(&provider->ftp_cmtx);
1941		return;
1942	}
1943
1944	ntps = dhpb->dthpb_noffs + dhpb->dthpb_nenoffs;
1945	ASSERT(ntps > 0);
1946
1947	atomic_add_32(&fasttrap_total, ntps);
1948
1949	if (fasttrap_total > fasttrap_max) {
1950		atomic_add_32(&fasttrap_total, -ntps);
1951		mutex_exit(&provider->ftp_cmtx);
1952		return;
1953	}
1954
1955	pp = kmem_zalloc(offsetof(fasttrap_probe_t, ftp_tps[ntps]), KM_SLEEP);
1956
1957	pp->ftp_prov = provider;
1958	pp->ftp_pid = provider->ftp_pid;
1959	pp->ftp_ntps = ntps;
1960	pp->ftp_nargs = dhpb->dthpb_xargc;
1961	pp->ftp_xtypes = dhpb->dthpb_xtypes;
1962	pp->ftp_ntypes = dhpb->dthpb_ntypes;
1963
1964	/*
1965	 * First create a tracepoint for each actual point of interest.
1966	 */
1967	for (i = 0; i < dhpb->dthpb_noffs; i++) {
1968		tp = kmem_zalloc(sizeof (fasttrap_tracepoint_t), KM_SLEEP);
1969
1970		tp->ftt_proc = provider->ftp_proc;
1971		tp->ftt_pc = dhpb->dthpb_base + dhpb->dthpb_offs[i];
1972		tp->ftt_pid = provider->ftp_pid;
1973
1974		pp->ftp_tps[i].fit_tp = tp;
1975		pp->ftp_tps[i].fit_id.fti_probe = pp;
1976#ifdef __sparc
1977		pp->ftp_tps[i].fit_id.fti_ptype = DTFTP_POST_OFFSETS;
1978#else
1979		pp->ftp_tps[i].fit_id.fti_ptype = DTFTP_OFFSETS;
1980#endif
1981	}
1982
1983	/*
1984	 * Then create a tracepoint for each is-enabled point.
1985	 */
1986	for (j = 0; i < ntps; i++, j++) {
1987		tp = kmem_zalloc(sizeof (fasttrap_tracepoint_t), KM_SLEEP);
1988
1989		tp->ftt_proc = provider->ftp_proc;
1990		tp->ftt_pc = dhpb->dthpb_base + dhpb->dthpb_enoffs[j];
1991		tp->ftt_pid = provider->ftp_pid;
1992
1993		pp->ftp_tps[i].fit_tp = tp;
1994		pp->ftp_tps[i].fit_id.fti_probe = pp;
1995		pp->ftp_tps[i].fit_id.fti_ptype = DTFTP_IS_ENABLED;
1996	}
1997
1998	/*
1999	 * If the arguments are shuffled around we set the argument remapping
2000	 * table. Later, when the probe fires, we only remap the arguments
2001	 * if the table is non-NULL.
2002	 */
2003	for (i = 0; i < dhpb->dthpb_xargc; i++) {
2004		if (dhpb->dthpb_args[i] != i) {
2005			pp->ftp_argmap = dhpb->dthpb_args;
2006			break;
2007		}
2008	}
2009
2010	/*
2011	 * The probe is fully constructed -- register it with DTrace.
2012	 */
2013	pp->ftp_id = dtrace_probe_create(provider->ftp_provid, dhpb->dthpb_mod,
2014	    dhpb->dthpb_func, dhpb->dthpb_name, FASTTRAP_OFFSET_AFRAMES, pp);
2015
2016	mutex_exit(&provider->ftp_cmtx);
2017}
2018
2019/*ARGSUSED*/
2020static void
2021fasttrap_meta_remove(void *arg, dtrace_helper_provdesc_t *dhpv, pid_t pid)
2022{
2023	/*
2024	 * Clean up the USDT provider. There may be active consumers of the
2025	 * provider busy adding probes, no damage will actually befall the
2026	 * provider until that count has dropped to zero. This just puts
2027	 * the provider on death row.
2028	 */
2029	fasttrap_provider_retire(pid, dhpv->dthpv_provname, 1);
2030}
2031
2032static dtrace_mops_t fasttrap_mops = {
2033	fasttrap_meta_create_probe,
2034	fasttrap_meta_provide,
2035	fasttrap_meta_remove
2036};
2037
2038/*ARGSUSED*/
2039static int
2040fasttrap_open(struct cdev *dev __unused, int oflags __unused,
2041    int devtype __unused, struct thread *td __unused)
2042{
2043	return (0);
2044}
2045
2046/*ARGSUSED*/
2047static int
2048fasttrap_ioctl(struct cdev *dev, u_long cmd, caddr_t arg, int fflag,
2049    struct thread *td)
2050{
2051#ifdef notyet
2052	struct kinfo_proc kp;
2053	const cred_t *cr = td->td_ucred;
2054#endif
2055	if (!dtrace_attached())
2056		return (EAGAIN);
2057
2058	if (cmd == FASTTRAPIOC_MAKEPROBE) {
2059		fasttrap_probe_spec_t *uprobe = *(fasttrap_probe_spec_t **)arg;
2060		fasttrap_probe_spec_t *probe;
2061		uint64_t noffs;
2062		size_t size;
2063		int ret;
2064		char *c;
2065
2066		if (copyin(&uprobe->ftps_noffs, &noffs,
2067		    sizeof (uprobe->ftps_noffs)))
2068			return (EFAULT);
2069
2070		/*
2071		 * Probes must have at least one tracepoint.
2072		 */
2073		if (noffs == 0)
2074			return (EINVAL);
2075
2076		size = sizeof (fasttrap_probe_spec_t) +
2077		    sizeof (probe->ftps_offs[0]) * (noffs - 1);
2078
2079		if (size > 1024 * 1024)
2080			return (ENOMEM);
2081
2082		probe = kmem_alloc(size, KM_SLEEP);
2083
2084		if (copyin(uprobe, probe, size) != 0) {
2085			kmem_free(probe, size);
2086			return (EFAULT);
2087		}
2088
2089		/*
2090		 * Verify that the function and module strings contain no
2091		 * funny characters.
2092		 */
2093		for (c = &probe->ftps_func[0]; *c != '\0'; c++) {
2094			if (*c < 0x20 || 0x7f <= *c) {
2095				ret = EINVAL;
2096				goto err;
2097			}
2098		}
2099
2100		for (c = &probe->ftps_mod[0]; *c != '\0'; c++) {
2101			if (*c < 0x20 || 0x7f <= *c) {
2102				ret = EINVAL;
2103				goto err;
2104			}
2105		}
2106
2107#ifdef notyet
2108		if (!PRIV_POLICY_CHOICE(cr, PRIV_ALL, B_FALSE)) {
2109			proc_t *p;
2110			pid_t pid = probe->ftps_pid;
2111
2112#if defined(sun)
2113			mutex_enter(&pidlock);
2114#endif
2115			/*
2116			 * Report an error if the process doesn't exist
2117			 * or is actively being birthed.
2118			 */
2119			p = pfind(pid);
2120			if (p)
2121				fill_kinfo_proc(p, &kp);
2122			if (p == NULL || kp.ki_stat == SIDL) {
2123#if defined(sun)
2124				mutex_exit(&pidlock);
2125#endif
2126				return (ESRCH);
2127			}
2128#if defined(sun)
2129			mutex_enter(&p->p_lock);
2130			mutex_exit(&pidlock);
2131#else
2132			PROC_LOCK_ASSERT(p, MA_OWNED);
2133#endif
2134
2135#ifdef notyet
2136			if ((ret = priv_proc_cred_perm(cr, p, NULL,
2137			    VREAD | VWRITE)) != 0) {
2138#if defined(sun)
2139				mutex_exit(&p->p_lock);
2140#else
2141				PROC_UNLOCK(p);
2142#endif
2143				return (ret);
2144			}
2145#endif /* notyet */
2146#if defined(sun)
2147			mutex_exit(&p->p_lock);
2148#else
2149			PROC_UNLOCK(p);
2150#endif
2151		}
2152#endif /* notyet */
2153
2154		ret = fasttrap_add_probe(probe);
2155err:
2156		kmem_free(probe, size);
2157
2158		return (ret);
2159
2160	} else if (cmd == FASTTRAPIOC_GETINSTR) {
2161		fasttrap_instr_query_t instr;
2162		fasttrap_tracepoint_t *tp;
2163		uint_t index;
2164#if defined(sun)
2165		int ret;
2166#endif
2167
2168#if defined(sun)
2169		if (copyin((void *)arg, &instr, sizeof (instr)) != 0)
2170			return (EFAULT);
2171#endif
2172
2173#ifdef notyet
2174		if (!PRIV_POLICY_CHOICE(cr, PRIV_ALL, B_FALSE)) {
2175			proc_t *p;
2176			pid_t pid = instr.ftiq_pid;
2177
2178#if defined(sun)
2179			mutex_enter(&pidlock);
2180#endif
2181			/*
2182			 * Report an error if the process doesn't exist
2183			 * or is actively being birthed.
2184			 */
2185			p = pfind(pid);
2186			if (p)
2187				fill_kinfo_proc(p, &kp);
2188			if (p == NULL || kp.ki_stat == SIDL) {
2189#if defined(sun)
2190				mutex_exit(&pidlock);
2191#endif
2192				return (ESRCH);
2193			}
2194#if defined(sun)
2195			mutex_enter(&p->p_lock);
2196			mutex_exit(&pidlock);
2197#else
2198			PROC_LOCK_ASSERT(p, MA_OWNED);
2199#endif
2200
2201#ifdef notyet
2202			if ((ret = priv_proc_cred_perm(cr, p, NULL,
2203			    VREAD)) != 0) {
2204#if defined(sun)
2205				mutex_exit(&p->p_lock);
2206#else
2207				PROC_UNLOCK(p);
2208#endif
2209				return (ret);
2210			}
2211#endif /* notyet */
2212
2213#if defined(sun)
2214			mutex_exit(&p->p_lock);
2215#else
2216			PROC_UNLOCK(p);
2217#endif
2218		}
2219#endif /* notyet */
2220
2221		index = FASTTRAP_TPOINTS_INDEX(instr.ftiq_pid, instr.ftiq_pc);
2222
2223		mutex_enter(&fasttrap_tpoints.fth_table[index].ftb_mtx);
2224		tp = fasttrap_tpoints.fth_table[index].ftb_data;
2225		while (tp != NULL) {
2226			if (instr.ftiq_pid == tp->ftt_pid &&
2227			    instr.ftiq_pc == tp->ftt_pc &&
2228			    tp->ftt_proc->ftpc_acount != 0)
2229				break;
2230
2231			tp = tp->ftt_next;
2232		}
2233
2234		if (tp == NULL) {
2235			mutex_exit(&fasttrap_tpoints.fth_table[index].ftb_mtx);
2236			return (ENOENT);
2237		}
2238
2239		bcopy(&tp->ftt_instr, &instr.ftiq_instr,
2240		    sizeof (instr.ftiq_instr));
2241		mutex_exit(&fasttrap_tpoints.fth_table[index].ftb_mtx);
2242
2243		if (copyout(&instr, (void *)arg, sizeof (instr)) != 0)
2244			return (EFAULT);
2245
2246		return (0);
2247	}
2248
2249	return (EINVAL);
2250}
2251
2252static int
2253fasttrap_load(void)
2254{
2255	ulong_t nent;
2256	int i, ret;
2257
2258        /* Create the /dev/dtrace/fasttrap entry. */
2259        fasttrap_cdev = make_dev(&fasttrap_cdevsw, 0, UID_ROOT, GID_WHEEL, 0600,
2260            "dtrace/fasttrap");
2261
2262	mtx_init(&fasttrap_cleanup_mtx, "fasttrap clean", "dtrace", MTX_DEF);
2263	mutex_init(&fasttrap_count_mtx, "fasttrap count mtx", MUTEX_DEFAULT,
2264	    NULL);
2265
2266#if defined(sun)
2267	fasttrap_max = ddi_getprop(DDI_DEV_T_ANY, devi, DDI_PROP_DONTPASS,
2268	    "fasttrap-max-probes", FASTTRAP_MAX_DEFAULT);
2269#else
2270	fasttrap_max = FASTTRAP_MAX_DEFAULT;
2271#endif
2272	fasttrap_total = 0;
2273
2274	/*
2275	 * Conjure up the tracepoints hashtable...
2276	 */
2277#if defined(sun)
2278	nent = ddi_getprop(DDI_DEV_T_ANY, devi, DDI_PROP_DONTPASS,
2279	    "fasttrap-hash-size", FASTTRAP_TPOINTS_DEFAULT_SIZE);
2280#else
2281	nent = FASTTRAP_TPOINTS_DEFAULT_SIZE;
2282#endif
2283
2284	if (nent == 0 || nent > 0x1000000)
2285		nent = FASTTRAP_TPOINTS_DEFAULT_SIZE;
2286
2287	if ((nent & (nent - 1)) == 0)
2288		fasttrap_tpoints.fth_nent = nent;
2289	else
2290		fasttrap_tpoints.fth_nent = 1 << fasttrap_highbit(nent);
2291	ASSERT(fasttrap_tpoints.fth_nent > 0);
2292	fasttrap_tpoints.fth_mask = fasttrap_tpoints.fth_nent - 1;
2293	fasttrap_tpoints.fth_table = kmem_zalloc(fasttrap_tpoints.fth_nent *
2294	    sizeof (fasttrap_bucket_t), KM_SLEEP);
2295#if !defined(sun)
2296	for (i = 0; i < fasttrap_tpoints.fth_nent; i++)
2297		mutex_init(&fasttrap_tpoints.fth_table[i].ftb_mtx,
2298		    "tracepoints bucket mtx", MUTEX_DEFAULT, NULL);
2299#endif
2300
2301	/*
2302	 * ... and the providers hash table...
2303	 */
2304	nent = FASTTRAP_PROVIDERS_DEFAULT_SIZE;
2305	if ((nent & (nent - 1)) == 0)
2306		fasttrap_provs.fth_nent = nent;
2307	else
2308		fasttrap_provs.fth_nent = 1 << fasttrap_highbit(nent);
2309	ASSERT(fasttrap_provs.fth_nent > 0);
2310	fasttrap_provs.fth_mask = fasttrap_provs.fth_nent - 1;
2311	fasttrap_provs.fth_table = kmem_zalloc(fasttrap_provs.fth_nent *
2312	    sizeof (fasttrap_bucket_t), KM_SLEEP);
2313#if !defined(sun)
2314	for (i = 0; i < fasttrap_provs.fth_nent; i++)
2315		mutex_init(&fasttrap_provs.fth_table[i].ftb_mtx,
2316		    "providers bucket mtx", MUTEX_DEFAULT, NULL);
2317#endif
2318
2319	ret = kproc_create(fasttrap_pid_cleanup_cb, NULL,
2320	    &fasttrap_cleanup_proc, 0, 0, "ftcleanup");
2321	if (ret != 0) {
2322		destroy_dev(fasttrap_cdev);
2323#if !defined(sun)
2324		for (i = 0; i < fasttrap_provs.fth_nent; i++)
2325			mutex_destroy(&fasttrap_provs.fth_table[i].ftb_mtx);
2326		for (i = 0; i < fasttrap_tpoints.fth_nent; i++)
2327			mutex_destroy(&fasttrap_tpoints.fth_table[i].ftb_mtx);
2328#endif
2329		kmem_free(fasttrap_provs.fth_table, fasttrap_provs.fth_nent *
2330		    sizeof (fasttrap_bucket_t));
2331		mtx_destroy(&fasttrap_cleanup_mtx);
2332		mutex_destroy(&fasttrap_count_mtx);
2333		return (ret);
2334	}
2335
2336
2337	/*
2338	 * ... and the procs hash table.
2339	 */
2340	nent = FASTTRAP_PROCS_DEFAULT_SIZE;
2341	if ((nent & (nent - 1)) == 0)
2342		fasttrap_procs.fth_nent = nent;
2343	else
2344		fasttrap_procs.fth_nent = 1 << fasttrap_highbit(nent);
2345	ASSERT(fasttrap_procs.fth_nent > 0);
2346	fasttrap_procs.fth_mask = fasttrap_procs.fth_nent - 1;
2347	fasttrap_procs.fth_table = kmem_zalloc(fasttrap_procs.fth_nent *
2348	    sizeof (fasttrap_bucket_t), KM_SLEEP);
2349#if !defined(sun)
2350	for (i = 0; i < fasttrap_procs.fth_nent; i++)
2351		mutex_init(&fasttrap_procs.fth_table[i].ftb_mtx,
2352		    "processes bucket mtx", MUTEX_DEFAULT, NULL);
2353
2354	CPU_FOREACH(i) {
2355		mutex_init(&fasttrap_cpuc_pid_lock[i], "fasttrap barrier",
2356		    MUTEX_DEFAULT, NULL);
2357	}
2358#endif
2359
2360	/*
2361	 * Install our hooks into fork(2), exec(2), and exit(2).
2362	 */
2363	dtrace_fasttrap_fork = &fasttrap_fork;
2364	dtrace_fasttrap_exit = &fasttrap_exec_exit;
2365	dtrace_fasttrap_exec = &fasttrap_exec_exit;
2366
2367	(void) dtrace_meta_register("fasttrap", &fasttrap_mops, NULL,
2368	    &fasttrap_meta_id);
2369
2370	return (0);
2371}
2372
2373static int
2374fasttrap_unload(void)
2375{
2376	int i, fail = 0;
2377
2378	/*
2379	 * Unregister the meta-provider to make sure no new fasttrap-
2380	 * managed providers come along while we're trying to close up
2381	 * shop. If we fail to detach, we'll need to re-register as a
2382	 * meta-provider. We can fail to unregister as a meta-provider
2383	 * if providers we manage still exist.
2384	 */
2385	if (fasttrap_meta_id != DTRACE_METAPROVNONE &&
2386	    dtrace_meta_unregister(fasttrap_meta_id) != 0)
2387		return (-1);
2388
2389	/*
2390	 * Iterate over all of our providers. If there's still a process
2391	 * that corresponds to that pid, fail to detach.
2392	 */
2393	for (i = 0; i < fasttrap_provs.fth_nent; i++) {
2394		fasttrap_provider_t **fpp, *fp;
2395		fasttrap_bucket_t *bucket = &fasttrap_provs.fth_table[i];
2396
2397		mutex_enter(&bucket->ftb_mtx);
2398		fpp = (fasttrap_provider_t **)&bucket->ftb_data;
2399		while ((fp = *fpp) != NULL) {
2400			/*
2401			 * Acquire and release the lock as a simple way of
2402			 * waiting for any other consumer to finish with
2403			 * this provider. A thread must first acquire the
2404			 * bucket lock so there's no chance of another thread
2405			 * blocking on the provider's lock.
2406			 */
2407			mutex_enter(&fp->ftp_mtx);
2408			mutex_exit(&fp->ftp_mtx);
2409
2410			if (dtrace_unregister(fp->ftp_provid) != 0) {
2411				fail = 1;
2412				fpp = &fp->ftp_next;
2413			} else {
2414				*fpp = fp->ftp_next;
2415				fasttrap_provider_free(fp);
2416			}
2417		}
2418
2419		mutex_exit(&bucket->ftb_mtx);
2420	}
2421
2422	if (fail) {
2423		(void) dtrace_meta_register("fasttrap", &fasttrap_mops, NULL,
2424		    &fasttrap_meta_id);
2425
2426		return (-1);
2427	}
2428
2429	/*
2430	 * Stop new processes from entering these hooks now, before the
2431	 * fasttrap_cleanup thread runs.  That way all processes will hopefully
2432	 * be out of these hooks before we free fasttrap_provs.fth_table
2433	 */
2434	ASSERT(dtrace_fasttrap_fork == &fasttrap_fork);
2435	dtrace_fasttrap_fork = NULL;
2436
2437	ASSERT(dtrace_fasttrap_exec == &fasttrap_exec_exit);
2438	dtrace_fasttrap_exec = NULL;
2439
2440	ASSERT(dtrace_fasttrap_exit == &fasttrap_exec_exit);
2441	dtrace_fasttrap_exit = NULL;
2442
2443	mtx_lock(&fasttrap_cleanup_mtx);
2444	fasttrap_cleanup_drain = 1;
2445	/* Wait for the cleanup thread to finish up and signal us. */
2446	wakeup(&fasttrap_cleanup_cv);
2447	mtx_sleep(&fasttrap_cleanup_drain, &fasttrap_cleanup_mtx, 0, "ftcld",
2448	    0);
2449	fasttrap_cleanup_proc = NULL;
2450	mtx_destroy(&fasttrap_cleanup_mtx);
2451
2452#ifdef DEBUG
2453	mutex_enter(&fasttrap_count_mtx);
2454	ASSERT(fasttrap_pid_count == 0);
2455	mutex_exit(&fasttrap_count_mtx);
2456#endif
2457
2458#if !defined(sun)
2459	for (i = 0; i < fasttrap_tpoints.fth_nent; i++)
2460		mutex_destroy(&fasttrap_tpoints.fth_table[i].ftb_mtx);
2461	for (i = 0; i < fasttrap_provs.fth_nent; i++)
2462		mutex_destroy(&fasttrap_provs.fth_table[i].ftb_mtx);
2463	for (i = 0; i < fasttrap_procs.fth_nent; i++)
2464		mutex_destroy(&fasttrap_procs.fth_table[i].ftb_mtx);
2465#endif
2466	kmem_free(fasttrap_tpoints.fth_table,
2467	    fasttrap_tpoints.fth_nent * sizeof (fasttrap_bucket_t));
2468	fasttrap_tpoints.fth_nent = 0;
2469
2470	kmem_free(fasttrap_provs.fth_table,
2471	    fasttrap_provs.fth_nent * sizeof (fasttrap_bucket_t));
2472	fasttrap_provs.fth_nent = 0;
2473
2474	kmem_free(fasttrap_procs.fth_table,
2475	    fasttrap_procs.fth_nent * sizeof (fasttrap_bucket_t));
2476	fasttrap_procs.fth_nent = 0;
2477
2478#if !defined(sun)
2479	destroy_dev(fasttrap_cdev);
2480	mutex_destroy(&fasttrap_count_mtx);
2481	CPU_FOREACH(i) {
2482		mutex_destroy(&fasttrap_cpuc_pid_lock[i]);
2483	}
2484#endif
2485
2486	return (0);
2487}
2488
2489/* ARGSUSED */
2490static int
2491fasttrap_modevent(module_t mod __unused, int type, void *data __unused)
2492{
2493	int error = 0;
2494
2495	switch (type) {
2496	case MOD_LOAD:
2497		break;
2498
2499	case MOD_UNLOAD:
2500		break;
2501
2502	case MOD_SHUTDOWN:
2503		break;
2504
2505	default:
2506		error = EOPNOTSUPP;
2507		break;
2508	}
2509	return (error);
2510}
2511
2512SYSINIT(fasttrap_load, SI_SUB_DTRACE_PROVIDER, SI_ORDER_ANY, fasttrap_load,
2513    NULL);
2514SYSUNINIT(fasttrap_unload, SI_SUB_DTRACE_PROVIDER, SI_ORDER_ANY,
2515    fasttrap_unload, NULL);
2516
2517DEV_MODULE(fasttrap, fasttrap_modevent, NULL);
2518MODULE_VERSION(fasttrap, 1);
2519MODULE_DEPEND(fasttrap, dtrace, 1, 1, 1);
2520MODULE_DEPEND(fasttrap, opensolaris, 1, 1, 1);
2521