1/*
2 * CDDL HEADER START
3 *
4 * The contents of this file are subject to the terms of the
5 * Common Development and Distribution License (the "License").
6 * You may not use this file except in compliance with the License.
7 *
8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 * or http://www.opensolaris.org/os/licensing.
10 * See the License for the specific language governing permissions
11 * and limitations under the License.
12 *
13 * When distributing Covered Code, include this CDDL HEADER in each
14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 * If applicable, add the following below this CDDL HEADER, with the
16 * fields enclosed by brackets "[]" replaced with your own identifying
17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 *
19 * CDDL HEADER END
20 *
21 * Portions Copyright 2010 The FreeBSD Foundation
22 *
| 1/*
2 * CDDL HEADER START
3 *
4 * The contents of this file are subject to the terms of the
5 * Common Development and Distribution License (the "License").
6 * You may not use this file except in compliance with the License.
7 *
8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 * or http://www.opensolaris.org/os/licensing.
10 * See the License for the specific language governing permissions
11 * and limitations under the License.
12 *
13 * When distributing Covered Code, include this CDDL HEADER in each
14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 * If applicable, add the following below this CDDL HEADER, with the
16 * fields enclosed by brackets "[]" replaced with your own identifying
17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 *
19 * CDDL HEADER END
20 *
21 * Portions Copyright 2010 The FreeBSD Foundation
22 *
|
23 * $FreeBSD: head/sys/cddl/contrib/opensolaris/uts/common/dtrace/fasttrap.c 258311 2013-11-18 16:51:56Z asomers $
| 23 * $FreeBSD: head/sys/cddl/contrib/opensolaris/uts/common/dtrace/fasttrap.c 264434 2014-04-14 00:22:42Z markj $
|
24 */
25
26/*
27 * Copyright 2008 Sun Microsystems, Inc. All rights reserved.
28 * Use is subject to license terms.
29 */
30
31#if defined(sun)
32#pragma ident "%Z%%M% %I% %E% SMI"
33#endif
34
35#include <sys/atomic.h>
36#include <sys/errno.h>
37#include <sys/stat.h>
38#include <sys/modctl.h>
39#include <sys/conf.h>
40#include <sys/systm.h>
41#if defined(sun)
42#include <sys/ddi.h>
43#endif
44#include <sys/sunddi.h>
45#include <sys/cpuvar.h>
46#include <sys/kmem.h>
47#if defined(sun)
48#include <sys/strsubr.h>
49#endif
50#include <sys/fasttrap.h>
51#include <sys/fasttrap_impl.h>
52#include <sys/fasttrap_isa.h>
53#include <sys/dtrace.h>
54#include <sys/dtrace_impl.h>
55#include <sys/sysmacros.h>
56#include <sys/proc.h>
57#include <sys/policy.h>
58#if defined(sun)
59#include <util/qsort.h>
60#endif
61#include <sys/mutex.h>
62#include <sys/kernel.h>
63#if !defined(sun)
| 24 */
25
26/*
27 * Copyright 2008 Sun Microsystems, Inc. All rights reserved.
28 * Use is subject to license terms.
29 */
30
31#if defined(sun)
32#pragma ident "%Z%%M% %I% %E% SMI"
33#endif
34
35#include <sys/atomic.h>
36#include <sys/errno.h>
37#include <sys/stat.h>
38#include <sys/modctl.h>
39#include <sys/conf.h>
40#include <sys/systm.h>
41#if defined(sun)
42#include <sys/ddi.h>
43#endif
44#include <sys/sunddi.h>
45#include <sys/cpuvar.h>
46#include <sys/kmem.h>
47#if defined(sun)
48#include <sys/strsubr.h>
49#endif
50#include <sys/fasttrap.h>
51#include <sys/fasttrap_impl.h>
52#include <sys/fasttrap_isa.h>
53#include <sys/dtrace.h>
54#include <sys/dtrace_impl.h>
55#include <sys/sysmacros.h>
56#include <sys/proc.h>
57#include <sys/policy.h>
58#if defined(sun)
59#include <util/qsort.h>
60#endif
61#include <sys/mutex.h>
62#include <sys/kernel.h>
63#if !defined(sun)
|
64#include <sys/user.h>
| |
65#include <sys/dtrace_bsd.h>
| 64#include <sys/dtrace_bsd.h>
|
| 65#include <sys/eventhandler.h>
66#include <sys/user.h>
67#include <vm/vm.h>
68#include <vm/pmap.h>
69#include <vm/vm_map.h>
70#include <vm/vm_param.h>
|
66#include <cddl/dev/dtrace/dtrace_cddl.h>
67#endif
68
69/*
70 * User-Land Trap-Based Tracing
71 * ----------------------------
72 *
73 * The fasttrap provider allows DTrace consumers to instrument any user-level
74 * instruction to gather data; this includes probes with semantic
75 * signifigance like entry and return as well as simple offsets into the
76 * function. While the specific techniques used are very ISA specific, the
77 * methodology is generalizable to any architecture.
78 *
79 *
80 * The General Methodology
81 * -----------------------
82 *
83 * With the primary goal of tracing every user-land instruction and the
84 * limitation that we can't trust user space so don't want to rely on much
85 * information there, we begin by replacing the instructions we want to trace
86 * with trap instructions. Each instruction we overwrite is saved into a hash
87 * table keyed by process ID and pc address. When we enter the kernel due to
88 * this trap instruction, we need the effects of the replaced instruction to
89 * appear to have occurred before we proceed with the user thread's
90 * execution.
91 *
92 * Each user level thread is represented by a ulwp_t structure which is
93 * always easily accessible through a register. The most basic way to produce
94 * the effects of the instruction we replaced is to copy that instruction out
95 * to a bit of scratch space reserved in the user thread's ulwp_t structure
96 * (a sort of kernel-private thread local storage), set the PC to that
97 * scratch space and single step. When we reenter the kernel after single
98 * stepping the instruction we must then adjust the PC to point to what would
99 * normally be the next instruction. Of course, special care must be taken
100 * for branches and jumps, but these represent such a small fraction of any
101 * instruction set that writing the code to emulate these in the kernel is
102 * not too difficult.
103 *
104 * Return probes may require several tracepoints to trace every return site,
105 * and, conversely, each tracepoint may activate several probes (the entry
106 * and offset 0 probes, for example). To solve this muliplexing problem,
107 * tracepoints contain lists of probes to activate and probes contain lists
108 * of tracepoints to enable. If a probe is activated, it adds its ID to
109 * existing tracepoints or creates new ones as necessary.
110 *
111 * Most probes are activated _before_ the instruction is executed, but return
112 * probes are activated _after_ the effects of the last instruction of the
113 * function are visible. Return probes must be fired _after_ we have
114 * single-stepped the instruction whereas all other probes are fired
115 * beforehand.
116 *
117 *
118 * Lock Ordering
119 * -------------
120 *
121 * The lock ordering below -- both internally and with respect to the DTrace
122 * framework -- is a little tricky and bears some explanation. Each provider
123 * has a lock (ftp_mtx) that protects its members including reference counts
124 * for enabled probes (ftp_rcount), consumers actively creating probes
125 * (ftp_ccount) and USDT consumers (ftp_mcount); all three prevent a provider
126 * from being freed. A provider is looked up by taking the bucket lock for the
127 * provider hash table, and is returned with its lock held. The provider lock
128 * may be taken in functions invoked by the DTrace framework, but may not be
129 * held while calling functions in the DTrace framework.
130 *
131 * To ensure consistency over multiple calls to the DTrace framework, the
132 * creation lock (ftp_cmtx) should be held. Naturally, the creation lock may
133 * not be taken when holding the provider lock as that would create a cyclic
134 * lock ordering. In situations where one would naturally take the provider
135 * lock and then the creation lock, we instead up a reference count to prevent
136 * the provider from disappearing, drop the provider lock, and acquire the
137 * creation lock.
138 *
139 * Briefly:
140 * bucket lock before provider lock
141 * DTrace before provider lock
142 * creation lock before DTrace
143 * never hold the provider lock and creation lock simultaneously
144 */
145
146static d_open_t fasttrap_open;
147static d_ioctl_t fasttrap_ioctl;
148
149static struct cdevsw fasttrap_cdevsw = {
150 .d_version = D_VERSION,
151 .d_open = fasttrap_open,
152 .d_ioctl = fasttrap_ioctl,
153 .d_name = "fasttrap",
154};
155static struct cdev *fasttrap_cdev;
156static dtrace_meta_provider_id_t fasttrap_meta_id;
157
158static struct proc *fasttrap_cleanup_proc;
159static struct mtx fasttrap_cleanup_mtx;
160static uint_t fasttrap_cleanup_work, fasttrap_cleanup_drain, fasttrap_cleanup_cv;
161
162/*
163 * Generation count on modifications to the global tracepoint lookup table.
164 */
165static volatile uint64_t fasttrap_mod_gen;
166
167/*
168 * When the fasttrap provider is loaded, fasttrap_max is set to either
169 * FASTTRAP_MAX_DEFAULT or the value for fasttrap-max-probes in the
170 * fasttrap.conf file. Each time a probe is created, fasttrap_total is
171 * incremented by the number of tracepoints that may be associated with that
172 * probe; fasttrap_total is capped at fasttrap_max.
173 */
174#define FASTTRAP_MAX_DEFAULT 250000
175static uint32_t fasttrap_max;
176static uint32_t fasttrap_total;
177
178/*
179 * Copyright (c) 2011, Joyent, Inc. All rights reserved.
180 */
181
182#define FASTTRAP_TPOINTS_DEFAULT_SIZE 0x4000
183#define FASTTRAP_PROVIDERS_DEFAULT_SIZE 0x100
184#define FASTTRAP_PROCS_DEFAULT_SIZE 0x100
185
186#define FASTTRAP_PID_NAME "pid"
187
188fasttrap_hash_t fasttrap_tpoints;
189static fasttrap_hash_t fasttrap_provs;
190static fasttrap_hash_t fasttrap_procs;
191
192static uint64_t fasttrap_pid_count; /* pid ref count */
193static kmutex_t fasttrap_count_mtx; /* lock on ref count */
194
195#define FASTTRAP_ENABLE_FAIL 1
196#define FASTTRAP_ENABLE_PARTIAL 2
197
198static int fasttrap_tracepoint_enable(proc_t *, fasttrap_probe_t *, uint_t);
199static void fasttrap_tracepoint_disable(proc_t *, fasttrap_probe_t *, uint_t);
200
201static fasttrap_provider_t *fasttrap_provider_lookup(pid_t, const char *,
202 const dtrace_pattr_t *);
203static void fasttrap_provider_retire(pid_t, const char *, int);
204static void fasttrap_provider_free(fasttrap_provider_t *);
205
206static fasttrap_proc_t *fasttrap_proc_lookup(pid_t);
207static void fasttrap_proc_release(fasttrap_proc_t *);
208
| 71#include <cddl/dev/dtrace/dtrace_cddl.h>
72#endif
73
74/*
75 * User-Land Trap-Based Tracing
76 * ----------------------------
77 *
78 * The fasttrap provider allows DTrace consumers to instrument any user-level
79 * instruction to gather data; this includes probes with semantic
80 * signifigance like entry and return as well as simple offsets into the
81 * function. While the specific techniques used are very ISA specific, the
82 * methodology is generalizable to any architecture.
83 *
84 *
85 * The General Methodology
86 * -----------------------
87 *
88 * With the primary goal of tracing every user-land instruction and the
89 * limitation that we can't trust user space so don't want to rely on much
90 * information there, we begin by replacing the instructions we want to trace
91 * with trap instructions. Each instruction we overwrite is saved into a hash
92 * table keyed by process ID and pc address. When we enter the kernel due to
93 * this trap instruction, we need the effects of the replaced instruction to
94 * appear to have occurred before we proceed with the user thread's
95 * execution.
96 *
97 * Each user level thread is represented by a ulwp_t structure which is
98 * always easily accessible through a register. The most basic way to produce
99 * the effects of the instruction we replaced is to copy that instruction out
100 * to a bit of scratch space reserved in the user thread's ulwp_t structure
101 * (a sort of kernel-private thread local storage), set the PC to that
102 * scratch space and single step. When we reenter the kernel after single
103 * stepping the instruction we must then adjust the PC to point to what would
104 * normally be the next instruction. Of course, special care must be taken
105 * for branches and jumps, but these represent such a small fraction of any
106 * instruction set that writing the code to emulate these in the kernel is
107 * not too difficult.
108 *
109 * Return probes may require several tracepoints to trace every return site,
110 * and, conversely, each tracepoint may activate several probes (the entry
111 * and offset 0 probes, for example). To solve this muliplexing problem,
112 * tracepoints contain lists of probes to activate and probes contain lists
113 * of tracepoints to enable. If a probe is activated, it adds its ID to
114 * existing tracepoints or creates new ones as necessary.
115 *
116 * Most probes are activated _before_ the instruction is executed, but return
117 * probes are activated _after_ the effects of the last instruction of the
118 * function are visible. Return probes must be fired _after_ we have
119 * single-stepped the instruction whereas all other probes are fired
120 * beforehand.
121 *
122 *
123 * Lock Ordering
124 * -------------
125 *
126 * The lock ordering below -- both internally and with respect to the DTrace
127 * framework -- is a little tricky and bears some explanation. Each provider
128 * has a lock (ftp_mtx) that protects its members including reference counts
129 * for enabled probes (ftp_rcount), consumers actively creating probes
130 * (ftp_ccount) and USDT consumers (ftp_mcount); all three prevent a provider
131 * from being freed. A provider is looked up by taking the bucket lock for the
132 * provider hash table, and is returned with its lock held. The provider lock
133 * may be taken in functions invoked by the DTrace framework, but may not be
134 * held while calling functions in the DTrace framework.
135 *
136 * To ensure consistency over multiple calls to the DTrace framework, the
137 * creation lock (ftp_cmtx) should be held. Naturally, the creation lock may
138 * not be taken when holding the provider lock as that would create a cyclic
139 * lock ordering. In situations where one would naturally take the provider
140 * lock and then the creation lock, we instead up a reference count to prevent
141 * the provider from disappearing, drop the provider lock, and acquire the
142 * creation lock.
143 *
144 * Briefly:
145 * bucket lock before provider lock
146 * DTrace before provider lock
147 * creation lock before DTrace
148 * never hold the provider lock and creation lock simultaneously
149 */
150
151static d_open_t fasttrap_open;
152static d_ioctl_t fasttrap_ioctl;
153
154static struct cdevsw fasttrap_cdevsw = {
155 .d_version = D_VERSION,
156 .d_open = fasttrap_open,
157 .d_ioctl = fasttrap_ioctl,
158 .d_name = "fasttrap",
159};
160static struct cdev *fasttrap_cdev;
161static dtrace_meta_provider_id_t fasttrap_meta_id;
162
163static struct proc *fasttrap_cleanup_proc;
164static struct mtx fasttrap_cleanup_mtx;
165static uint_t fasttrap_cleanup_work, fasttrap_cleanup_drain, fasttrap_cleanup_cv;
166
167/*
168 * Generation count on modifications to the global tracepoint lookup table.
169 */
170static volatile uint64_t fasttrap_mod_gen;
171
172/*
173 * When the fasttrap provider is loaded, fasttrap_max is set to either
174 * FASTTRAP_MAX_DEFAULT or the value for fasttrap-max-probes in the
175 * fasttrap.conf file. Each time a probe is created, fasttrap_total is
176 * incremented by the number of tracepoints that may be associated with that
177 * probe; fasttrap_total is capped at fasttrap_max.
178 */
179#define FASTTRAP_MAX_DEFAULT 250000
180static uint32_t fasttrap_max;
181static uint32_t fasttrap_total;
182
183/*
184 * Copyright (c) 2011, Joyent, Inc. All rights reserved.
185 */
186
187#define FASTTRAP_TPOINTS_DEFAULT_SIZE 0x4000
188#define FASTTRAP_PROVIDERS_DEFAULT_SIZE 0x100
189#define FASTTRAP_PROCS_DEFAULT_SIZE 0x100
190
191#define FASTTRAP_PID_NAME "pid"
192
193fasttrap_hash_t fasttrap_tpoints;
194static fasttrap_hash_t fasttrap_provs;
195static fasttrap_hash_t fasttrap_procs;
196
197static uint64_t fasttrap_pid_count; /* pid ref count */
198static kmutex_t fasttrap_count_mtx; /* lock on ref count */
199
200#define FASTTRAP_ENABLE_FAIL 1
201#define FASTTRAP_ENABLE_PARTIAL 2
202
203static int fasttrap_tracepoint_enable(proc_t *, fasttrap_probe_t *, uint_t);
204static void fasttrap_tracepoint_disable(proc_t *, fasttrap_probe_t *, uint_t);
205
206static fasttrap_provider_t *fasttrap_provider_lookup(pid_t, const char *,
207 const dtrace_pattr_t *);
208static void fasttrap_provider_retire(pid_t, const char *, int);
209static void fasttrap_provider_free(fasttrap_provider_t *);
210
211static fasttrap_proc_t *fasttrap_proc_lookup(pid_t);
212static void fasttrap_proc_release(fasttrap_proc_t *);
213
|
| 214#if !defined(sun)
215static void fasttrap_thread_dtor(void *, struct thread *);
216#endif
217
|
209#define FASTTRAP_PROVS_INDEX(pid, name) \
210 ((fasttrap_hash_str(name) + (pid)) & fasttrap_provs.fth_mask)
211
212#define FASTTRAP_PROCS_INDEX(pid) ((pid) & fasttrap_procs.fth_mask)
213
214#if !defined(sun)
215static kmutex_t fasttrap_cpuc_pid_lock[MAXCPU];
| 218#define FASTTRAP_PROVS_INDEX(pid, name) \
219 ((fasttrap_hash_str(name) + (pid)) & fasttrap_provs.fth_mask)
220
221#define FASTTRAP_PROCS_INDEX(pid) ((pid) & fasttrap_procs.fth_mask)
222
223#if !defined(sun)
224static kmutex_t fasttrap_cpuc_pid_lock[MAXCPU];
|
| 225static eventhandler_tag fasttrap_thread_dtor_tag;
|
216#endif
217
218static int
219fasttrap_highbit(ulong_t i)
220{
221 int h = 1;
222
223 if (i == 0)
224 return (0);
225#ifdef _LP64
226 if (i & 0xffffffff00000000ul) {
227 h += 32; i >>= 32;
228 }
229#endif
230 if (i & 0xffff0000) {
231 h += 16; i >>= 16;
232 }
233 if (i & 0xff00) {
234 h += 8; i >>= 8;
235 }
236 if (i & 0xf0) {
237 h += 4; i >>= 4;
238 }
239 if (i & 0xc) {
240 h += 2; i >>= 2;
241 }
242 if (i & 0x2) {
243 h += 1;
244 }
245 return (h);
246}
247
248static uint_t
249fasttrap_hash_str(const char *p)
250{
251 unsigned int g;
252 uint_t hval = 0;
253
254 while (*p) {
255 hval = (hval << 4) + *p++;
256 if ((g = (hval & 0xf0000000)) != 0)
257 hval ^= g >> 24;
258 hval &= ~g;
259 }
260 return (hval);
261}
262
263void
264fasttrap_sigtrap(proc_t *p, kthread_t *t, uintptr_t pc)
265{
266#if defined(sun)
267 sigqueue_t *sqp = kmem_zalloc(sizeof (sigqueue_t), KM_SLEEP);
268
269 sqp->sq_info.si_signo = SIGTRAP;
270 sqp->sq_info.si_code = TRAP_DTRACE;
271 sqp->sq_info.si_addr = (caddr_t)pc;
272
273 mutex_enter(&p->p_lock);
274 sigaddqa(p, t, sqp);
275 mutex_exit(&p->p_lock);
276
277 if (t != NULL)
278 aston(t);
279#else
280 ksiginfo_t *ksi = kmem_zalloc(sizeof (ksiginfo_t), KM_SLEEP);
281
282 ksiginfo_init(ksi);
283 ksi->ksi_signo = SIGTRAP;
284 ksi->ksi_code = TRAP_DTRACE;
285 ksi->ksi_addr = (caddr_t)pc;
286 PROC_LOCK(p);
287 (void) tdksignal(t, SIGTRAP, ksi);
288 PROC_UNLOCK(p);
289#endif
290}
291
| 226#endif
227
228static int
229fasttrap_highbit(ulong_t i)
230{
231 int h = 1;
232
233 if (i == 0)
234 return (0);
235#ifdef _LP64
236 if (i & 0xffffffff00000000ul) {
237 h += 32; i >>= 32;
238 }
239#endif
240 if (i & 0xffff0000) {
241 h += 16; i >>= 16;
242 }
243 if (i & 0xff00) {
244 h += 8; i >>= 8;
245 }
246 if (i & 0xf0) {
247 h += 4; i >>= 4;
248 }
249 if (i & 0xc) {
250 h += 2; i >>= 2;
251 }
252 if (i & 0x2) {
253 h += 1;
254 }
255 return (h);
256}
257
258static uint_t
259fasttrap_hash_str(const char *p)
260{
261 unsigned int g;
262 uint_t hval = 0;
263
264 while (*p) {
265 hval = (hval << 4) + *p++;
266 if ((g = (hval & 0xf0000000)) != 0)
267 hval ^= g >> 24;
268 hval &= ~g;
269 }
270 return (hval);
271}
272
273void
274fasttrap_sigtrap(proc_t *p, kthread_t *t, uintptr_t pc)
275{
276#if defined(sun)
277 sigqueue_t *sqp = kmem_zalloc(sizeof (sigqueue_t), KM_SLEEP);
278
279 sqp->sq_info.si_signo = SIGTRAP;
280 sqp->sq_info.si_code = TRAP_DTRACE;
281 sqp->sq_info.si_addr = (caddr_t)pc;
282
283 mutex_enter(&p->p_lock);
284 sigaddqa(p, t, sqp);
285 mutex_exit(&p->p_lock);
286
287 if (t != NULL)
288 aston(t);
289#else
290 ksiginfo_t *ksi = kmem_zalloc(sizeof (ksiginfo_t), KM_SLEEP);
291
292 ksiginfo_init(ksi);
293 ksi->ksi_signo = SIGTRAP;
294 ksi->ksi_code = TRAP_DTRACE;
295 ksi->ksi_addr = (caddr_t)pc;
296 PROC_LOCK(p);
297 (void) tdksignal(t, SIGTRAP, ksi);
298 PROC_UNLOCK(p);
299#endif
300}
301
|
| 302#if !defined(sun)
|
292/*
| 303/*
|
| 304 * Obtain a chunk of scratch space in the address space of the target process.
305 */
306fasttrap_scrspace_t *
307fasttrap_scraddr(struct thread *td, fasttrap_proc_t *fprc)
308{
309 fasttrap_scrblock_t *scrblk;
310 fasttrap_scrspace_t *scrspc;
311 struct proc *p;
312 vm_offset_t addr;
313 int error, i;
314
315 scrspc = NULL;
316 if (td->t_dtrace_sscr != NULL) {
317 /* If the thread already has scratch space, we're done. */
318 scrspc = (fasttrap_scrspace_t *)td->t_dtrace_sscr;
319 return (scrspc);
320 }
321
322 p = td->td_proc;
323
324 mutex_enter(&fprc->ftpc_mtx);
325 if (LIST_EMPTY(&fprc->ftpc_fscr)) {
326 /*
327 * No scratch space is available, so we'll map a new scratch
328 * space block into the traced process' address space.
329 */
330 addr = 0;
331 error = vm_map_find(&p->p_vmspace->vm_map, NULL, 0, &addr,
332 FASTTRAP_SCRBLOCK_SIZE, 0, VMFS_ANY_SPACE, VM_PROT_ALL,
333 VM_PROT_ALL, 0);
334 if (error != KERN_SUCCESS)
335 goto done;
336
337 scrblk = malloc(sizeof(*scrblk), M_SOLARIS, M_WAITOK);
338 scrblk->ftsb_addr = addr;
339 LIST_INSERT_HEAD(&fprc->ftpc_scrblks, scrblk, ftsb_next);
340
341 /*
342 * Carve the block up into chunks and put them on the free list.
343 */
344 for (i = 0;
345 i < FASTTRAP_SCRBLOCK_SIZE / FASTTRAP_SCRSPACE_SIZE; i++) {
346 scrspc = malloc(sizeof(*scrspc), M_SOLARIS, M_WAITOK);
347 scrspc->ftss_addr = addr +
348 i * FASTTRAP_SCRSPACE_SIZE;
349 LIST_INSERT_HEAD(&fprc->ftpc_fscr, scrspc,
350 ftss_next);
351 }
352 }
353
354 /*
355 * Take the first scratch chunk off the free list, put it on the
356 * allocated list, and return its address.
357 */
358 scrspc = LIST_FIRST(&fprc->ftpc_fscr);
359 LIST_REMOVE(scrspc, ftss_next);
360 LIST_INSERT_HEAD(&fprc->ftpc_ascr, scrspc, ftss_next);
361
362 /*
363 * This scratch space is reserved for use by td until the thread exits.
364 */
365 td->t_dtrace_sscr = scrspc;
366
367done:
368 mutex_exit(&fprc->ftpc_mtx);
369
370 return (scrspc);
371}
372
373/*
374 * Return any allocated per-thread scratch space chunks back to the process'
375 * free list.
376 */
377static void
378fasttrap_thread_dtor(void *arg __unused, struct thread *td)
379{
380 fasttrap_bucket_t *bucket;
381 fasttrap_proc_t *fprc;
382 fasttrap_scrspace_t *scrspc;
383 pid_t pid;
384
385 if (td->t_dtrace_sscr == NULL)
386 return;
387
388 pid = td->td_proc->p_pid;
389 bucket = &fasttrap_procs.fth_table[FASTTRAP_PROCS_INDEX(pid)];
390 fprc = NULL;
391
392 /* Look up the fasttrap process handle for this process. */
393 mutex_enter(&bucket->ftb_mtx);
394 for (fprc = bucket->ftb_data; fprc != NULL; fprc = fprc->ftpc_next) {
395 if (fprc->ftpc_pid == pid) {
396 mutex_enter(&fprc->ftpc_mtx);
397 mutex_exit(&bucket->ftb_mtx);
398 break;
399 }
400 }
401 if (fprc == NULL) {
402 mutex_exit(&bucket->ftb_mtx);
403 return;
404 }
405
406 scrspc = (fasttrap_scrspace_t *)td->t_dtrace_sscr;
407 LIST_REMOVE(scrspc, ftss_next);
408 LIST_INSERT_HEAD(&fprc->ftpc_fscr, scrspc, ftss_next);
409
410 mutex_exit(&fprc->ftpc_mtx);
411}
412#endif
413
414/*
|
293 * This function ensures that no threads are actively using the memory
294 * associated with probes that were formerly live.
295 */
296static void
297fasttrap_mod_barrier(uint64_t gen)
298{
299 int i;
300
301 if (gen < fasttrap_mod_gen)
302 return;
303
304 fasttrap_mod_gen++;
305
306 CPU_FOREACH(i) {
307 mutex_enter(&fasttrap_cpuc_pid_lock[i]);
308 mutex_exit(&fasttrap_cpuc_pid_lock[i]);
309 }
310}
311
312/*
313 * This function performs asynchronous cleanup of fasttrap providers. The
314 * Solaris implementation of this mechanism use a timeout that's activated in
315 * fasttrap_pid_cleanup(), but this doesn't work in FreeBSD: one may sleep while
316 * holding the DTrace mutexes, but it is unsafe to sleep in a callout handler.
317 * Thus we use a dedicated process to perform the cleanup when requested.
318 */
319/*ARGSUSED*/
320static void
321fasttrap_pid_cleanup_cb(void *data)
322{
323 fasttrap_provider_t **fpp, *fp;
324 fasttrap_bucket_t *bucket;
325 dtrace_provider_id_t provid;
326 int i, later = 0, rval;
327
328 mtx_lock(&fasttrap_cleanup_mtx);
329 while (!fasttrap_cleanup_drain || later > 0) {
330 fasttrap_cleanup_work = 0;
331 mtx_unlock(&fasttrap_cleanup_mtx);
332
333 later = 0;
334
335 /*
336 * Iterate over all the providers trying to remove the marked
337 * ones. If a provider is marked but not retired, we just
338 * have to take a crack at removing it -- it's no big deal if
339 * we can't.
340 */
341 for (i = 0; i < fasttrap_provs.fth_nent; i++) {
342 bucket = &fasttrap_provs.fth_table[i];
343 mutex_enter(&bucket->ftb_mtx);
344 fpp = (fasttrap_provider_t **)&bucket->ftb_data;
345
346 while ((fp = *fpp) != NULL) {
347 if (!fp->ftp_marked) {
348 fpp = &fp->ftp_next;
349 continue;
350 }
351
352 mutex_enter(&fp->ftp_mtx);
353
354 /*
355 * If this provider has consumers actively
356 * creating probes (ftp_ccount) or is a USDT
357 * provider (ftp_mcount), we can't unregister
358 * or even condense.
359 */
360 if (fp->ftp_ccount != 0 ||
361 fp->ftp_mcount != 0) {
362 mutex_exit(&fp->ftp_mtx);
363 fp->ftp_marked = 0;
364 continue;
365 }
366
367 if (!fp->ftp_retired || fp->ftp_rcount != 0)
368 fp->ftp_marked = 0;
369
370 mutex_exit(&fp->ftp_mtx);
371
372 /*
373 * If we successfully unregister this
374 * provider we can remove it from the hash
375 * chain and free the memory. If our attempt
376 * to unregister fails and this is a retired
377 * provider, increment our flag to try again
378 * pretty soon. If we've consumed more than
379 * half of our total permitted number of
380 * probes call dtrace_condense() to try to
381 * clean out the unenabled probes.
382 */
383 provid = fp->ftp_provid;
384 if ((rval = dtrace_unregister(provid)) != 0) {
385 if (fasttrap_total > fasttrap_max / 2)
386 (void) dtrace_condense(provid);
387
388 if (rval == EAGAIN)
389 fp->ftp_marked = 1;
390
391 later += fp->ftp_marked;
392 fpp = &fp->ftp_next;
393 } else {
394 *fpp = fp->ftp_next;
395 fasttrap_provider_free(fp);
396 }
397 }
398 mutex_exit(&bucket->ftb_mtx);
399 }
400 mtx_lock(&fasttrap_cleanup_mtx);
401
402 /*
403 * If we were unable to retire a provider, try again after a
404 * second. This situation can occur in certain circumstances
405 * where providers cannot be unregistered even though they have
406 * no probes enabled because of an execution of dtrace -l or
407 * something similar.
408 */
409 if (later > 0 || fasttrap_cleanup_work ||
410 fasttrap_cleanup_drain) {
411 mtx_unlock(&fasttrap_cleanup_mtx);
412 pause("ftclean", hz);
413 mtx_lock(&fasttrap_cleanup_mtx);
414 } else
415 mtx_sleep(&fasttrap_cleanup_cv, &fasttrap_cleanup_mtx,
416 0, "ftcl", 0);
417 }
418
419 /*
420 * Wake up the thread in fasttrap_unload() now that we're done.
421 */
422 wakeup(&fasttrap_cleanup_drain);
423 mtx_unlock(&fasttrap_cleanup_mtx);
424
425 kthread_exit();
426}
427
428/*
429 * Activates the asynchronous cleanup mechanism.
430 */
431static void
432fasttrap_pid_cleanup(void)
433{
434
435 mtx_lock(&fasttrap_cleanup_mtx);
436 if (!fasttrap_cleanup_work) {
437 fasttrap_cleanup_work = 1;
438 wakeup(&fasttrap_cleanup_cv);
439 }
440 mtx_unlock(&fasttrap_cleanup_mtx);
441}
442
443/*
444 * This is called from cfork() via dtrace_fasttrap_fork(). The child
445 * process's address space is (roughly) a copy of the parent process's so
446 * we have to remove all the instrumentation we had previously enabled in the
447 * parent.
448 */
449static void
450fasttrap_fork(proc_t *p, proc_t *cp)
451{
| 415 * This function ensures that no threads are actively using the memory
416 * associated with probes that were formerly live.
417 */
418static void
419fasttrap_mod_barrier(uint64_t gen)
420{
421 int i;
422
423 if (gen < fasttrap_mod_gen)
424 return;
425
426 fasttrap_mod_gen++;
427
428 CPU_FOREACH(i) {
429 mutex_enter(&fasttrap_cpuc_pid_lock[i]);
430 mutex_exit(&fasttrap_cpuc_pid_lock[i]);
431 }
432}
433
434/*
435 * This function performs asynchronous cleanup of fasttrap providers. The
436 * Solaris implementation of this mechanism use a timeout that's activated in
437 * fasttrap_pid_cleanup(), but this doesn't work in FreeBSD: one may sleep while
438 * holding the DTrace mutexes, but it is unsafe to sleep in a callout handler.
439 * Thus we use a dedicated process to perform the cleanup when requested.
440 */
441/*ARGSUSED*/
442static void
443fasttrap_pid_cleanup_cb(void *data)
444{
445 fasttrap_provider_t **fpp, *fp;
446 fasttrap_bucket_t *bucket;
447 dtrace_provider_id_t provid;
448 int i, later = 0, rval;
449
450 mtx_lock(&fasttrap_cleanup_mtx);
451 while (!fasttrap_cleanup_drain || later > 0) {
452 fasttrap_cleanup_work = 0;
453 mtx_unlock(&fasttrap_cleanup_mtx);
454
455 later = 0;
456
457 /*
458 * Iterate over all the providers trying to remove the marked
459 * ones. If a provider is marked but not retired, we just
460 * have to take a crack at removing it -- it's no big deal if
461 * we can't.
462 */
463 for (i = 0; i < fasttrap_provs.fth_nent; i++) {
464 bucket = &fasttrap_provs.fth_table[i];
465 mutex_enter(&bucket->ftb_mtx);
466 fpp = (fasttrap_provider_t **)&bucket->ftb_data;
467
468 while ((fp = *fpp) != NULL) {
469 if (!fp->ftp_marked) {
470 fpp = &fp->ftp_next;
471 continue;
472 }
473
474 mutex_enter(&fp->ftp_mtx);
475
476 /*
477 * If this provider has consumers actively
478 * creating probes (ftp_ccount) or is a USDT
479 * provider (ftp_mcount), we can't unregister
480 * or even condense.
481 */
482 if (fp->ftp_ccount != 0 ||
483 fp->ftp_mcount != 0) {
484 mutex_exit(&fp->ftp_mtx);
485 fp->ftp_marked = 0;
486 continue;
487 }
488
489 if (!fp->ftp_retired || fp->ftp_rcount != 0)
490 fp->ftp_marked = 0;
491
492 mutex_exit(&fp->ftp_mtx);
493
494 /*
495 * If we successfully unregister this
496 * provider we can remove it from the hash
497 * chain and free the memory. If our attempt
498 * to unregister fails and this is a retired
499 * provider, increment our flag to try again
500 * pretty soon. If we've consumed more than
501 * half of our total permitted number of
502 * probes call dtrace_condense() to try to
503 * clean out the unenabled probes.
504 */
505 provid = fp->ftp_provid;
506 if ((rval = dtrace_unregister(provid)) != 0) {
507 if (fasttrap_total > fasttrap_max / 2)
508 (void) dtrace_condense(provid);
509
510 if (rval == EAGAIN)
511 fp->ftp_marked = 1;
512
513 later += fp->ftp_marked;
514 fpp = &fp->ftp_next;
515 } else {
516 *fpp = fp->ftp_next;
517 fasttrap_provider_free(fp);
518 }
519 }
520 mutex_exit(&bucket->ftb_mtx);
521 }
522 mtx_lock(&fasttrap_cleanup_mtx);
523
524 /*
525 * If we were unable to retire a provider, try again after a
526 * second. This situation can occur in certain circumstances
527 * where providers cannot be unregistered even though they have
528 * no probes enabled because of an execution of dtrace -l or
529 * something similar.
530 */
531 if (later > 0 || fasttrap_cleanup_work ||
532 fasttrap_cleanup_drain) {
533 mtx_unlock(&fasttrap_cleanup_mtx);
534 pause("ftclean", hz);
535 mtx_lock(&fasttrap_cleanup_mtx);
536 } else
537 mtx_sleep(&fasttrap_cleanup_cv, &fasttrap_cleanup_mtx,
538 0, "ftcl", 0);
539 }
540
541 /*
542 * Wake up the thread in fasttrap_unload() now that we're done.
543 */
544 wakeup(&fasttrap_cleanup_drain);
545 mtx_unlock(&fasttrap_cleanup_mtx);
546
547 kthread_exit();
548}
549
550/*
551 * Activates the asynchronous cleanup mechanism.
552 */
553static void
554fasttrap_pid_cleanup(void)
555{
556
557 mtx_lock(&fasttrap_cleanup_mtx);
558 if (!fasttrap_cleanup_work) {
559 fasttrap_cleanup_work = 1;
560 wakeup(&fasttrap_cleanup_cv);
561 }
562 mtx_unlock(&fasttrap_cleanup_mtx);
563}
564
565/*
566 * This is called from cfork() via dtrace_fasttrap_fork(). The child
567 * process's address space is (roughly) a copy of the parent process's so
568 * we have to remove all the instrumentation we had previously enabled in the
569 * parent.
570 */
571static void
572fasttrap_fork(proc_t *p, proc_t *cp)
573{
|
| 574#if !defined(sun)
575 fasttrap_scrblock_t *scrblk;
576 fasttrap_proc_t *fprc = NULL;
577#endif
|
452 pid_t ppid = p->p_pid;
453 int i;
454
455#if defined(sun)
456 ASSERT(curproc == p);
457 ASSERT(p->p_proc_flag & P_PR_LOCK);
458#else
459 PROC_LOCK_ASSERT(p, MA_OWNED);
460#endif
461#if defined(sun)
462 ASSERT(p->p_dtrace_count > 0);
463#else
464 if (p->p_dtrace_helpers) {
465 /*
466 * dtrace_helpers_duplicate() allocates memory.
467 */
468 _PHOLD(cp);
469 PROC_UNLOCK(p);
470 PROC_UNLOCK(cp);
471 dtrace_helpers_duplicate(p, cp);
472 PROC_LOCK(cp);
473 PROC_LOCK(p);
474 _PRELE(cp);
475 }
476 /*
477 * This check is purposely here instead of in kern_fork.c because,
478 * for legal resons, we cannot include the dtrace_cddl.h header
479 * inside kern_fork.c and insert if-clause there.
480 */
481 if (p->p_dtrace_count == 0)
482 return;
483#endif
484 ASSERT(cp->p_dtrace_count == 0);
485
486 /*
487 * This would be simpler and faster if we maintained per-process
488 * hash tables of enabled tracepoints. It could, however, potentially
489 * slow down execution of a tracepoint since we'd need to go
490 * through two levels of indirection. In the future, we should
491 * consider either maintaining per-process ancillary lists of
492 * enabled tracepoints or hanging a pointer to a per-process hash
493 * table of enabled tracepoints off the proc structure.
494 */
495
496 /*
497 * We don't have to worry about the child process disappearing
498 * because we're in fork().
499 */
500#if defined(sun)
501 mtx_lock_spin(&cp->p_slock);
502 sprlock_proc(cp);
503 mtx_unlock_spin(&cp->p_slock);
504#else
505 /*
506 * fasttrap_tracepoint_remove() expects the child process to be
507 * unlocked and the VM then expects curproc to be unlocked.
508 */
509 _PHOLD(cp);
510 PROC_UNLOCK(cp);
511 PROC_UNLOCK(p);
512#endif
513
514 /*
515 * Iterate over every tracepoint looking for ones that belong to the
516 * parent process, and remove each from the child process.
517 */
518 for (i = 0; i < fasttrap_tpoints.fth_nent; i++) {
519 fasttrap_tracepoint_t *tp;
520 fasttrap_bucket_t *bucket = &fasttrap_tpoints.fth_table[i];
521
522 mutex_enter(&bucket->ftb_mtx);
523 for (tp = bucket->ftb_data; tp != NULL; tp = tp->ftt_next) {
524 if (tp->ftt_pid == ppid &&
525 tp->ftt_proc->ftpc_acount != 0) {
526 int ret = fasttrap_tracepoint_remove(cp, tp);
527 ASSERT(ret == 0);
528
529 /*
530 * The count of active providers can only be
531 * decremented (i.e. to zero) during exec,
532 * exit, and removal of a meta provider so it
533 * should be impossible to drop the count
534 * mid-fork.
535 */
536 ASSERT(tp->ftt_proc->ftpc_acount != 0);
| 578 pid_t ppid = p->p_pid;
579 int i;
580
581#if defined(sun)
582 ASSERT(curproc == p);
583 ASSERT(p->p_proc_flag & P_PR_LOCK);
584#else
585 PROC_LOCK_ASSERT(p, MA_OWNED);
586#endif
587#if defined(sun)
588 ASSERT(p->p_dtrace_count > 0);
589#else
590 if (p->p_dtrace_helpers) {
591 /*
592 * dtrace_helpers_duplicate() allocates memory.
593 */
594 _PHOLD(cp);
595 PROC_UNLOCK(p);
596 PROC_UNLOCK(cp);
597 dtrace_helpers_duplicate(p, cp);
598 PROC_LOCK(cp);
599 PROC_LOCK(p);
600 _PRELE(cp);
601 }
602 /*
603 * This check is purposely here instead of in kern_fork.c because,
604 * for legal resons, we cannot include the dtrace_cddl.h header
605 * inside kern_fork.c and insert if-clause there.
606 */
607 if (p->p_dtrace_count == 0)
608 return;
609#endif
610 ASSERT(cp->p_dtrace_count == 0);
611
612 /*
613 * This would be simpler and faster if we maintained per-process
614 * hash tables of enabled tracepoints. It could, however, potentially
615 * slow down execution of a tracepoint since we'd need to go
616 * through two levels of indirection. In the future, we should
617 * consider either maintaining per-process ancillary lists of
618 * enabled tracepoints or hanging a pointer to a per-process hash
619 * table of enabled tracepoints off the proc structure.
620 */
621
622 /*
623 * We don't have to worry about the child process disappearing
624 * because we're in fork().
625 */
626#if defined(sun)
627 mtx_lock_spin(&cp->p_slock);
628 sprlock_proc(cp);
629 mtx_unlock_spin(&cp->p_slock);
630#else
631 /*
632 * fasttrap_tracepoint_remove() expects the child process to be
633 * unlocked and the VM then expects curproc to be unlocked.
634 */
635 _PHOLD(cp);
636 PROC_UNLOCK(cp);
637 PROC_UNLOCK(p);
638#endif
639
640 /*
641 * Iterate over every tracepoint looking for ones that belong to the
642 * parent process, and remove each from the child process.
643 */
644 for (i = 0; i < fasttrap_tpoints.fth_nent; i++) {
645 fasttrap_tracepoint_t *tp;
646 fasttrap_bucket_t *bucket = &fasttrap_tpoints.fth_table[i];
647
648 mutex_enter(&bucket->ftb_mtx);
649 for (tp = bucket->ftb_data; tp != NULL; tp = tp->ftt_next) {
650 if (tp->ftt_pid == ppid &&
651 tp->ftt_proc->ftpc_acount != 0) {
652 int ret = fasttrap_tracepoint_remove(cp, tp);
653 ASSERT(ret == 0);
654
655 /*
656 * The count of active providers can only be
657 * decremented (i.e. to zero) during exec,
658 * exit, and removal of a meta provider so it
659 * should be impossible to drop the count
660 * mid-fork.
661 */
662 ASSERT(tp->ftt_proc->ftpc_acount != 0);
|
| 663#if !defined(sun)
664 fprc = tp->ftt_proc;
665#endif
|
537 }
538 }
539 mutex_exit(&bucket->ftb_mtx);
| 666 }
667 }
668 mutex_exit(&bucket->ftb_mtx);
|
| 669
670#if !defined(sun)
671 /*
672 * Unmap any scratch space inherited from the parent's address
673 * space.
674 */
675 if (fprc != NULL) {
676 mutex_enter(&fprc->ftpc_mtx);
677 LIST_FOREACH(scrblk, &fprc->ftpc_scrblks, ftsb_next) {
678 vm_map_remove(&cp->p_vmspace->vm_map,
679 scrblk->ftsb_addr,
680 scrblk->ftsb_addr + FASTTRAP_SCRBLOCK_SIZE);
681 }
682 mutex_exit(&fprc->ftpc_mtx);
683 }
684#endif
|
540 }
541
542#if defined(sun)
543 mutex_enter(&cp->p_lock);
544 sprunlock(cp);
545#else
546 PROC_LOCK(p);
547 PROC_LOCK(cp);
548 _PRELE(cp);
549#endif
550}
551
552/*
553 * This is called from proc_exit() or from exec_common() if p_dtrace_probes
554 * is set on the proc structure to indicate that there is a pid provider
555 * associated with this process.
556 */
557static void
558fasttrap_exec_exit(proc_t *p)
559{
| 685 }
686
687#if defined(sun)
688 mutex_enter(&cp->p_lock);
689 sprunlock(cp);
690#else
691 PROC_LOCK(p);
692 PROC_LOCK(cp);
693 _PRELE(cp);
694#endif
695}
696
697/*
698 * This is called from proc_exit() or from exec_common() if p_dtrace_probes
699 * is set on the proc structure to indicate that there is a pid provider
700 * associated with this process.
701 */
702static void
703fasttrap_exec_exit(proc_t *p)
704{
|
| 705#if !defined(sun)
706 struct thread *td;
707#endif
708
|
560#if defined(sun)
561 ASSERT(p == curproc);
| 709#if defined(sun)
710 ASSERT(p == curproc);
|
562#endif
| 711#else
|
563 PROC_LOCK_ASSERT(p, MA_OWNED);
564 _PHOLD(p);
| 712 PROC_LOCK_ASSERT(p, MA_OWNED);
713 _PHOLD(p);
|
| 714 /*
715 * Since struct threads may be recycled, we cannot rely on t_dtrace_sscr
716 * fields to be zeroed by kdtrace_thread_ctor. Thus we must zero it
717 * ourselves when a process exits.
718 */
719 FOREACH_THREAD_IN_PROC(p, td)
720 td->t_dtrace_sscr = NULL;
|
565 PROC_UNLOCK(p);
| 721 PROC_UNLOCK(p);
|
| 722#endif
|
566
567 /*
568 * We clean up the pid provider for this process here; user-land
569 * static probes are handled by the meta-provider remove entry point.
570 */
571 fasttrap_provider_retire(p->p_pid, FASTTRAP_PID_NAME, 0);
572#if !defined(sun)
573 if (p->p_dtrace_helpers)
574 dtrace_helpers_destroy(p);
| 723
724 /*
725 * We clean up the pid provider for this process here; user-land
726 * static probes are handled by the meta-provider remove entry point.
727 */
728 fasttrap_provider_retire(p->p_pid, FASTTRAP_PID_NAME, 0);
729#if !defined(sun)
730 if (p->p_dtrace_helpers)
731 dtrace_helpers_destroy(p);
|
575#endif
| |
576 PROC_LOCK(p);
577 _PRELE(p);
| 732 PROC_LOCK(p);
733 _PRELE(p);
|
| 734#endif
|
578}
579
580
581/*ARGSUSED*/
582static void
583fasttrap_pid_provide(void *arg, dtrace_probedesc_t *desc)
584{
585 /*
586 * There are no "default" pid probes.
587 */
588}
589
590static int
591fasttrap_tracepoint_enable(proc_t *p, fasttrap_probe_t *probe, uint_t index)
592{
593 fasttrap_tracepoint_t *tp, *new_tp = NULL;
594 fasttrap_bucket_t *bucket;
595 fasttrap_id_t *id;
596 pid_t pid;
597 uintptr_t pc;
598
599 ASSERT(index < probe->ftp_ntps);
600
601 pid = probe->ftp_pid;
602 pc = probe->ftp_tps[index].fit_tp->ftt_pc;
603 id = &probe->ftp_tps[index].fit_id;
604
605 ASSERT(probe->ftp_tps[index].fit_tp->ftt_pid == pid);
606
607#if defined(sun)
608 ASSERT(!(p->p_flag & SVFORK));
609#endif
610
611 /*
612 * Before we make any modifications, make sure we've imposed a barrier
613 * on the generation in which this probe was last modified.
614 */
615 fasttrap_mod_barrier(probe->ftp_gen);
616
617 bucket = &fasttrap_tpoints.fth_table[FASTTRAP_TPOINTS_INDEX(pid, pc)];
618
619 /*
620 * If the tracepoint has already been enabled, just add our id to the
621 * list of interested probes. This may be our second time through
622 * this path in which case we'll have constructed the tracepoint we'd
623 * like to install. If we can't find a match, and have an allocated
624 * tracepoint ready to go, enable that one now.
625 *
626 * A tracepoint whose process is defunct is also considered defunct.
627 */
628again:
629 mutex_enter(&bucket->ftb_mtx);
630 for (tp = bucket->ftb_data; tp != NULL; tp = tp->ftt_next) {
631 /*
632 * Note that it's safe to access the active count on the
633 * associated proc structure because we know that at least one
634 * provider (this one) will still be around throughout this
635 * operation.
636 */
637 if (tp->ftt_pid != pid || tp->ftt_pc != pc ||
638 tp->ftt_proc->ftpc_acount == 0)
639 continue;
640
641 /*
642 * Now that we've found a matching tracepoint, it would be
643 * a decent idea to confirm that the tracepoint is still
644 * enabled and the trap instruction hasn't been overwritten.
645 * Since this is a little hairy, we'll punt for now.
646 */
647
648 /*
649 * This can't be the first interested probe. We don't have
650 * to worry about another thread being in the midst of
651 * deleting this tracepoint (which would be the only valid
652 * reason for a tracepoint to have no interested probes)
653 * since we're holding P_PR_LOCK for this process.
654 */
655 ASSERT(tp->ftt_ids != NULL || tp->ftt_retids != NULL);
656
657 switch (id->fti_ptype) {
658 case DTFTP_ENTRY:
659 case DTFTP_OFFSETS:
660 case DTFTP_IS_ENABLED:
661 id->fti_next = tp->ftt_ids;
662 membar_producer();
663 tp->ftt_ids = id;
664 membar_producer();
665 break;
666
667 case DTFTP_RETURN:
668 case DTFTP_POST_OFFSETS:
669 id->fti_next = tp->ftt_retids;
670 membar_producer();
671 tp->ftt_retids = id;
672 membar_producer();
673 break;
674
675 default:
676 ASSERT(0);
677 }
678
679 mutex_exit(&bucket->ftb_mtx);
680
681 if (new_tp != NULL) {
682 new_tp->ftt_ids = NULL;
683 new_tp->ftt_retids = NULL;
684 }
685
686 return (0);
687 }
688
689 /*
690 * If we have a good tracepoint ready to go, install it now while
691 * we have the lock held and no one can screw with us.
692 */
693 if (new_tp != NULL) {
694 int rc = 0;
695
696 new_tp->ftt_next = bucket->ftb_data;
697 membar_producer();
698 bucket->ftb_data = new_tp;
699 membar_producer();
700 mutex_exit(&bucket->ftb_mtx);
701
702 /*
703 * Activate the tracepoint in the ISA-specific manner.
704 * If this fails, we need to report the failure, but
705 * indicate that this tracepoint must still be disabled
706 * by calling fasttrap_tracepoint_disable().
707 */
708 if (fasttrap_tracepoint_install(p, new_tp) != 0)
709 rc = FASTTRAP_ENABLE_PARTIAL;
710
711 /*
712 * Increment the count of the number of tracepoints active in
713 * the victim process.
714 */
715#if defined(sun)
716 ASSERT(p->p_proc_flag & P_PR_LOCK);
717#endif
718 p->p_dtrace_count++;
719
720 return (rc);
721 }
722
723 mutex_exit(&bucket->ftb_mtx);
724
725 /*
726 * Initialize the tracepoint that's been preallocated with the probe.
727 */
728 new_tp = probe->ftp_tps[index].fit_tp;
729
730 ASSERT(new_tp->ftt_pid == pid);
731 ASSERT(new_tp->ftt_pc == pc);
732 ASSERT(new_tp->ftt_proc == probe->ftp_prov->ftp_proc);
733 ASSERT(new_tp->ftt_ids == NULL);
734 ASSERT(new_tp->ftt_retids == NULL);
735
736 switch (id->fti_ptype) {
737 case DTFTP_ENTRY:
738 case DTFTP_OFFSETS:
739 case DTFTP_IS_ENABLED:
740 id->fti_next = NULL;
741 new_tp->ftt_ids = id;
742 break;
743
744 case DTFTP_RETURN:
745 case DTFTP_POST_OFFSETS:
746 id->fti_next = NULL;
747 new_tp->ftt_retids = id;
748 break;
749
750 default:
751 ASSERT(0);
752 }
753
754 /*
755 * If the ISA-dependent initialization goes to plan, go back to the
756 * beginning and try to install this freshly made tracepoint.
757 */
758 if (fasttrap_tracepoint_init(p, new_tp, pc, id->fti_ptype) == 0)
759 goto again;
760
761 new_tp->ftt_ids = NULL;
762 new_tp->ftt_retids = NULL;
763
764 return (FASTTRAP_ENABLE_FAIL);
765}
766
767static void
768fasttrap_tracepoint_disable(proc_t *p, fasttrap_probe_t *probe, uint_t index)
769{
770 fasttrap_bucket_t *bucket;
771 fasttrap_provider_t *provider = probe->ftp_prov;
772 fasttrap_tracepoint_t **pp, *tp;
773 fasttrap_id_t *id, **idp = NULL;
774 pid_t pid;
775 uintptr_t pc;
776
777 ASSERT(index < probe->ftp_ntps);
778
779 pid = probe->ftp_pid;
780 pc = probe->ftp_tps[index].fit_tp->ftt_pc;
781 id = &probe->ftp_tps[index].fit_id;
782
783 ASSERT(probe->ftp_tps[index].fit_tp->ftt_pid == pid);
784
785 /*
786 * Find the tracepoint and make sure that our id is one of the
787 * ones registered with it.
788 */
789 bucket = &fasttrap_tpoints.fth_table[FASTTRAP_TPOINTS_INDEX(pid, pc)];
790 mutex_enter(&bucket->ftb_mtx);
791 for (tp = bucket->ftb_data; tp != NULL; tp = tp->ftt_next) {
792 if (tp->ftt_pid == pid && tp->ftt_pc == pc &&
793 tp->ftt_proc == provider->ftp_proc)
794 break;
795 }
796
797 /*
798 * If we somehow lost this tracepoint, we're in a world of hurt.
799 */
800 ASSERT(tp != NULL);
801
802 switch (id->fti_ptype) {
803 case DTFTP_ENTRY:
804 case DTFTP_OFFSETS:
805 case DTFTP_IS_ENABLED:
806 ASSERT(tp->ftt_ids != NULL);
807 idp = &tp->ftt_ids;
808 break;
809
810 case DTFTP_RETURN:
811 case DTFTP_POST_OFFSETS:
812 ASSERT(tp->ftt_retids != NULL);
813 idp = &tp->ftt_retids;
814 break;
815
816 default:
817 ASSERT(0);
818 }
819
820 while ((*idp)->fti_probe != probe) {
821 idp = &(*idp)->fti_next;
822 ASSERT(*idp != NULL);
823 }
824
825 id = *idp;
826 *idp = id->fti_next;
827 membar_producer();
828
829 ASSERT(id->fti_probe == probe);
830
831 /*
832 * If there are other registered enablings of this tracepoint, we're
833 * all done, but if this was the last probe assocated with this
834 * this tracepoint, we need to remove and free it.
835 */
836 if (tp->ftt_ids != NULL || tp->ftt_retids != NULL) {
837
838 /*
839 * If the current probe's tracepoint is in use, swap it
840 * for an unused tracepoint.
841 */
842 if (tp == probe->ftp_tps[index].fit_tp) {
843 fasttrap_probe_t *tmp_probe;
844 fasttrap_tracepoint_t **tmp_tp;
845 uint_t tmp_index;
846
847 if (tp->ftt_ids != NULL) {
848 tmp_probe = tp->ftt_ids->fti_probe;
849 /* LINTED - alignment */
850 tmp_index = FASTTRAP_ID_INDEX(tp->ftt_ids);
851 tmp_tp = &tmp_probe->ftp_tps[tmp_index].fit_tp;
852 } else {
853 tmp_probe = tp->ftt_retids->fti_probe;
854 /* LINTED - alignment */
855 tmp_index = FASTTRAP_ID_INDEX(tp->ftt_retids);
856 tmp_tp = &tmp_probe->ftp_tps[tmp_index].fit_tp;
857 }
858
859 ASSERT(*tmp_tp != NULL);
860 ASSERT(*tmp_tp != probe->ftp_tps[index].fit_tp);
861 ASSERT((*tmp_tp)->ftt_ids == NULL);
862 ASSERT((*tmp_tp)->ftt_retids == NULL);
863
864 probe->ftp_tps[index].fit_tp = *tmp_tp;
865 *tmp_tp = tp;
866 }
867
868 mutex_exit(&bucket->ftb_mtx);
869
870 /*
871 * Tag the modified probe with the generation in which it was
872 * changed.
873 */
874 probe->ftp_gen = fasttrap_mod_gen;
875 return;
876 }
877
878 mutex_exit(&bucket->ftb_mtx);
879
880 /*
881 * We can't safely remove the tracepoint from the set of active
882 * tracepoints until we've actually removed the fasttrap instruction
883 * from the process's text. We can, however, operate on this
884 * tracepoint secure in the knowledge that no other thread is going to
885 * be looking at it since we hold P_PR_LOCK on the process if it's
886 * live or we hold the provider lock on the process if it's dead and
887 * gone.
888 */
889
890 /*
891 * We only need to remove the actual instruction if we're looking
892 * at an existing process
893 */
894 if (p != NULL) {
895 /*
896 * If we fail to restore the instruction we need to kill
897 * this process since it's in a completely unrecoverable
898 * state.
899 */
900 if (fasttrap_tracepoint_remove(p, tp) != 0)
901 fasttrap_sigtrap(p, NULL, pc);
902
903 /*
904 * Decrement the count of the number of tracepoints active
905 * in the victim process.
906 */
907#if defined(sun)
908 ASSERT(p->p_proc_flag & P_PR_LOCK);
909#endif
910 p->p_dtrace_count--;
911 }
912
913 /*
914 * Remove the probe from the hash table of active tracepoints.
915 */
916 mutex_enter(&bucket->ftb_mtx);
917 pp = (fasttrap_tracepoint_t **)&bucket->ftb_data;
918 ASSERT(*pp != NULL);
919 while (*pp != tp) {
920 pp = &(*pp)->ftt_next;
921 ASSERT(*pp != NULL);
922 }
923
924 *pp = tp->ftt_next;
925 membar_producer();
926
927 mutex_exit(&bucket->ftb_mtx);
928
929 /*
930 * Tag the modified probe with the generation in which it was changed.
931 */
932 probe->ftp_gen = fasttrap_mod_gen;
933}
934
935static void
936fasttrap_enable_callbacks(void)
937{
938 /*
939 * We don't have to play the rw lock game here because we're
940 * providing something rather than taking something away --
941 * we can be sure that no threads have tried to follow this
942 * function pointer yet.
943 */
944 mutex_enter(&fasttrap_count_mtx);
945 if (fasttrap_pid_count == 0) {
946 ASSERT(dtrace_pid_probe_ptr == NULL);
947 ASSERT(dtrace_return_probe_ptr == NULL);
948 dtrace_pid_probe_ptr = &fasttrap_pid_probe;
949 dtrace_return_probe_ptr = &fasttrap_return_probe;
950 }
951 ASSERT(dtrace_pid_probe_ptr == &fasttrap_pid_probe);
952 ASSERT(dtrace_return_probe_ptr == &fasttrap_return_probe);
953 fasttrap_pid_count++;
954 mutex_exit(&fasttrap_count_mtx);
955}
956
957static void
958fasttrap_disable_callbacks(void)
959{
960#if defined(sun)
961 ASSERT(MUTEX_HELD(&cpu_lock));
962#endif
963
964
965 mutex_enter(&fasttrap_count_mtx);
966 ASSERT(fasttrap_pid_count > 0);
967 fasttrap_pid_count--;
968 if (fasttrap_pid_count == 0) {
969#if defined(sun)
970 cpu_t *cur, *cpu = CPU;
971
972 for (cur = cpu->cpu_next_onln; cur != cpu;
973 cur = cur->cpu_next_onln) {
974 rw_enter(&cur->cpu_ft_lock, RW_WRITER);
975 }
976#endif
977 dtrace_pid_probe_ptr = NULL;
978 dtrace_return_probe_ptr = NULL;
979#if defined(sun)
980 for (cur = cpu->cpu_next_onln; cur != cpu;
981 cur = cur->cpu_next_onln) {
982 rw_exit(&cur->cpu_ft_lock);
983 }
984#endif
985 }
986 mutex_exit(&fasttrap_count_mtx);
987}
988
989/*ARGSUSED*/
990static void
991fasttrap_pid_enable(void *arg, dtrace_id_t id, void *parg)
992{
993 fasttrap_probe_t *probe = parg;
994 proc_t *p = NULL;
995 int i, rc;
996
997 ASSERT(probe != NULL);
998 ASSERT(!probe->ftp_enabled);
999 ASSERT(id == probe->ftp_id);
1000#if defined(sun)
1001 ASSERT(MUTEX_HELD(&cpu_lock));
1002#endif
1003
1004 /*
1005 * Increment the count of enabled probes on this probe's provider;
1006 * the provider can't go away while the probe still exists. We
1007 * must increment this even if we aren't able to properly enable
1008 * this probe.
1009 */
1010 mutex_enter(&probe->ftp_prov->ftp_mtx);
1011 probe->ftp_prov->ftp_rcount++;
1012 mutex_exit(&probe->ftp_prov->ftp_mtx);
1013
1014 /*
1015 * If this probe's provider is retired (meaning it was valid in a
1016 * previously exec'ed incarnation of this address space), bail out. The
1017 * provider can't go away while we're in this code path.
1018 */
1019 if (probe->ftp_prov->ftp_retired)
1020 return;
1021
1022 /*
1023 * If we can't find the process, it may be that we're in the context of
1024 * a fork in which the traced process is being born and we're copying
1025 * USDT probes. Otherwise, the process is gone so bail.
1026 */
1027#if defined(sun)
1028 if ((p = sprlock(probe->ftp_pid)) == NULL) {
1029 if ((curproc->p_flag & SFORKING) == 0)
1030 return;
1031
1032 mutex_enter(&pidlock);
1033 p = prfind(probe->ftp_pid);
1034
1035 /*
1036 * Confirm that curproc is indeed forking the process in which
1037 * we're trying to enable probes.
1038 */
1039 ASSERT(p != NULL);
1040 ASSERT(p->p_parent == curproc);
1041 ASSERT(p->p_stat == SIDL);
1042
1043 mutex_enter(&p->p_lock);
1044 mutex_exit(&pidlock);
1045
1046 sprlock_proc(p);
1047 }
1048
1049 ASSERT(!(p->p_flag & SVFORK));
1050 mutex_exit(&p->p_lock);
1051#else
1052 if ((p = pfind(probe->ftp_pid)) == NULL)
1053 return;
1054#endif
1055
1056 /*
1057 * We have to enable the trap entry point before any user threads have
1058 * the chance to execute the trap instruction we're about to place
1059 * in their process's text.
1060 */
1061#ifdef __FreeBSD__
1062 /*
1063 * pfind() returns a locked process.
1064 */
1065 _PHOLD(p);
1066 PROC_UNLOCK(p);
1067#endif
1068 fasttrap_enable_callbacks();
1069
1070 /*
1071 * Enable all the tracepoints and add this probe's id to each
1072 * tracepoint's list of active probes.
1073 */
1074 for (i = 0; i < probe->ftp_ntps; i++) {
1075 if ((rc = fasttrap_tracepoint_enable(p, probe, i)) != 0) {
1076 /*
1077 * If enabling the tracepoint failed completely,
1078 * we don't have to disable it; if the failure
1079 * was only partial we must disable it.
1080 */
1081 if (rc == FASTTRAP_ENABLE_FAIL)
1082 i--;
1083 else
1084 ASSERT(rc == FASTTRAP_ENABLE_PARTIAL);
1085
1086 /*
1087 * Back up and pull out all the tracepoints we've
1088 * created so far for this probe.
1089 */
1090 while (i >= 0) {
1091 fasttrap_tracepoint_disable(p, probe, i);
1092 i--;
1093 }
1094
1095#if defined(sun)
1096 mutex_enter(&p->p_lock);
1097 sprunlock(p);
1098#else
1099 PRELE(p);
1100#endif
1101
1102 /*
1103 * Since we're not actually enabling this probe,
1104 * drop our reference on the trap table entry.
1105 */
1106 fasttrap_disable_callbacks();
1107 return;
1108 }
1109 }
1110#if defined(sun)
1111 mutex_enter(&p->p_lock);
1112 sprunlock(p);
1113#else
1114 PRELE(p);
1115#endif
1116
1117 probe->ftp_enabled = 1;
1118}
1119
1120/*ARGSUSED*/
1121static void
1122fasttrap_pid_disable(void *arg, dtrace_id_t id, void *parg)
1123{
1124 fasttrap_probe_t *probe = parg;
1125 fasttrap_provider_t *provider = probe->ftp_prov;
1126 proc_t *p;
1127 int i, whack = 0;
1128
1129 ASSERT(id == probe->ftp_id);
1130
1131 mutex_enter(&provider->ftp_mtx);
1132
1133 /*
1134 * We won't be able to acquire a /proc-esque lock on the process
1135 * iff the process is dead and gone. In this case, we rely on the
1136 * provider lock as a point of mutual exclusion to prevent other
1137 * DTrace consumers from disabling this probe.
1138 */
1139 if ((p = pfind(probe->ftp_pid)) != NULL) {
1140#ifdef __FreeBSD__
1141 _PHOLD(p);
1142 PROC_UNLOCK(p);
1143#endif
1144 }
1145
1146 /*
1147 * Disable all the associated tracepoints (for fully enabled probes).
1148 */
1149 if (probe->ftp_enabled) {
1150 for (i = 0; i < probe->ftp_ntps; i++) {
1151 fasttrap_tracepoint_disable(p, probe, i);
1152 }
1153 }
1154
1155 ASSERT(provider->ftp_rcount > 0);
1156 provider->ftp_rcount--;
1157
1158 if (p != NULL) {
1159 /*
1160 * Even though we may not be able to remove it entirely, we
1161 * mark this retired provider to get a chance to remove some
1162 * of the associated probes.
1163 */
1164 if (provider->ftp_retired && !provider->ftp_marked)
1165 whack = provider->ftp_marked = 1;
1166 mutex_exit(&provider->ftp_mtx);
1167 } else {
1168 /*
1169 * If the process is dead, we're just waiting for the
1170 * last probe to be disabled to be able to free it.
1171 */
1172 if (provider->ftp_rcount == 0 && !provider->ftp_marked)
1173 whack = provider->ftp_marked = 1;
1174 mutex_exit(&provider->ftp_mtx);
1175 }
1176
1177 if (whack)
1178 fasttrap_pid_cleanup();
1179
1180#ifdef __FreeBSD__
1181 if (p != NULL)
1182 PRELE(p);
1183#endif
1184 if (!probe->ftp_enabled)
1185 return;
1186
1187 probe->ftp_enabled = 0;
1188
1189#if defined(sun)
1190 ASSERT(MUTEX_HELD(&cpu_lock));
1191#endif
1192 fasttrap_disable_callbacks();
1193}
1194
1195/*ARGSUSED*/
1196static void
1197fasttrap_pid_getargdesc(void *arg, dtrace_id_t id, void *parg,
1198 dtrace_argdesc_t *desc)
1199{
1200 fasttrap_probe_t *probe = parg;
1201 char *str;
1202 int i, ndx;
1203
1204 desc->dtargd_native[0] = '\0';
1205 desc->dtargd_xlate[0] = '\0';
1206
1207 if (probe->ftp_prov->ftp_retired != 0 ||
1208 desc->dtargd_ndx >= probe->ftp_nargs) {
1209 desc->dtargd_ndx = DTRACE_ARGNONE;
1210 return;
1211 }
1212
1213 ndx = (probe->ftp_argmap != NULL) ?
1214 probe->ftp_argmap[desc->dtargd_ndx] : desc->dtargd_ndx;
1215
1216 str = probe->ftp_ntypes;
1217 for (i = 0; i < ndx; i++) {
1218 str += strlen(str) + 1;
1219 }
1220
1221 ASSERT(strlen(str + 1) < sizeof (desc->dtargd_native));
1222 (void) strcpy(desc->dtargd_native, str);
1223
1224 if (probe->ftp_xtypes == NULL)
1225 return;
1226
1227 str = probe->ftp_xtypes;
1228 for (i = 0; i < desc->dtargd_ndx; i++) {
1229 str += strlen(str) + 1;
1230 }
1231
1232 ASSERT(strlen(str + 1) < sizeof (desc->dtargd_xlate));
1233 (void) strcpy(desc->dtargd_xlate, str);
1234}
1235
1236/*ARGSUSED*/
1237static void
1238fasttrap_pid_destroy(void *arg, dtrace_id_t id, void *parg)
1239{
1240 fasttrap_probe_t *probe = parg;
1241 int i;
1242 size_t size;
1243
1244 ASSERT(probe != NULL);
1245 ASSERT(!probe->ftp_enabled);
1246 ASSERT(fasttrap_total >= probe->ftp_ntps);
1247
1248 atomic_add_32(&fasttrap_total, -probe->ftp_ntps);
1249 size = offsetof(fasttrap_probe_t, ftp_tps[probe->ftp_ntps]);
1250
1251 if (probe->ftp_gen + 1 >= fasttrap_mod_gen)
1252 fasttrap_mod_barrier(probe->ftp_gen);
1253
1254 for (i = 0; i < probe->ftp_ntps; i++) {
1255 kmem_free(probe->ftp_tps[i].fit_tp,
1256 sizeof (fasttrap_tracepoint_t));
1257 }
1258
1259 kmem_free(probe, size);
1260}
1261
1262
1263static const dtrace_pattr_t pid_attr = {
1264{ DTRACE_STABILITY_EVOLVING, DTRACE_STABILITY_EVOLVING, DTRACE_CLASS_ISA },
1265{ DTRACE_STABILITY_PRIVATE, DTRACE_STABILITY_PRIVATE, DTRACE_CLASS_UNKNOWN },
1266{ DTRACE_STABILITY_PRIVATE, DTRACE_STABILITY_PRIVATE, DTRACE_CLASS_UNKNOWN },
1267{ DTRACE_STABILITY_EVOLVING, DTRACE_STABILITY_EVOLVING, DTRACE_CLASS_ISA },
1268{ DTRACE_STABILITY_PRIVATE, DTRACE_STABILITY_PRIVATE, DTRACE_CLASS_UNKNOWN },
1269};
1270
1271static dtrace_pops_t pid_pops = {
1272 fasttrap_pid_provide,
1273 NULL,
1274 fasttrap_pid_enable,
1275 fasttrap_pid_disable,
1276 NULL,
1277 NULL,
1278 fasttrap_pid_getargdesc,
1279 fasttrap_pid_getarg,
1280 NULL,
1281 fasttrap_pid_destroy
1282};
1283
1284static dtrace_pops_t usdt_pops = {
1285 fasttrap_pid_provide,
1286 NULL,
1287 fasttrap_pid_enable,
1288 fasttrap_pid_disable,
1289 NULL,
1290 NULL,
1291 fasttrap_pid_getargdesc,
1292 fasttrap_usdt_getarg,
1293 NULL,
1294 fasttrap_pid_destroy
1295};
1296
1297static fasttrap_proc_t *
1298fasttrap_proc_lookup(pid_t pid)
1299{
1300 fasttrap_bucket_t *bucket;
1301 fasttrap_proc_t *fprc, *new_fprc;
1302
1303
1304 bucket = &fasttrap_procs.fth_table[FASTTRAP_PROCS_INDEX(pid)];
1305 mutex_enter(&bucket->ftb_mtx);
1306
1307 for (fprc = bucket->ftb_data; fprc != NULL; fprc = fprc->ftpc_next) {
1308 if (fprc->ftpc_pid == pid && fprc->ftpc_acount != 0) {
1309 mutex_enter(&fprc->ftpc_mtx);
1310 mutex_exit(&bucket->ftb_mtx);
1311 fprc->ftpc_rcount++;
1312 atomic_add_64(&fprc->ftpc_acount, 1);
1313 ASSERT(fprc->ftpc_acount <= fprc->ftpc_rcount);
1314 mutex_exit(&fprc->ftpc_mtx);
1315
1316 return (fprc);
1317 }
1318 }
1319
1320 /*
1321 * Drop the bucket lock so we don't try to perform a sleeping
1322 * allocation under it.
1323 */
1324 mutex_exit(&bucket->ftb_mtx);
1325
1326 new_fprc = kmem_zalloc(sizeof (fasttrap_proc_t), KM_SLEEP);
1327 new_fprc->ftpc_pid = pid;
1328 new_fprc->ftpc_rcount = 1;
1329 new_fprc->ftpc_acount = 1;
1330#if !defined(sun)
1331 mutex_init(&new_fprc->ftpc_mtx, "fasttrap proc mtx", MUTEX_DEFAULT,
1332 NULL);
1333#endif
1334
1335 mutex_enter(&bucket->ftb_mtx);
1336
1337 /*
1338 * Take another lap through the list to make sure a proc hasn't
1339 * been created for this pid while we weren't under the bucket lock.
1340 */
1341 for (fprc = bucket->ftb_data; fprc != NULL; fprc = fprc->ftpc_next) {
1342 if (fprc->ftpc_pid == pid && fprc->ftpc_acount != 0) {
1343 mutex_enter(&fprc->ftpc_mtx);
1344 mutex_exit(&bucket->ftb_mtx);
1345 fprc->ftpc_rcount++;
1346 atomic_add_64(&fprc->ftpc_acount, 1);
1347 ASSERT(fprc->ftpc_acount <= fprc->ftpc_rcount);
1348 mutex_exit(&fprc->ftpc_mtx);
1349
1350 kmem_free(new_fprc, sizeof (fasttrap_proc_t));
1351
1352 return (fprc);
1353 }
1354 }
1355
1356 new_fprc->ftpc_next = bucket->ftb_data;
1357 bucket->ftb_data = new_fprc;
1358
1359 mutex_exit(&bucket->ftb_mtx);
1360
1361 return (new_fprc);
1362}
1363
1364static void
1365fasttrap_proc_release(fasttrap_proc_t *proc)
1366{
1367 fasttrap_bucket_t *bucket;
1368 fasttrap_proc_t *fprc, **fprcp;
1369 pid_t pid = proc->ftpc_pid;
| 735}
736
737
738/*ARGSUSED*/
739static void
740fasttrap_pid_provide(void *arg, dtrace_probedesc_t *desc)
741{
742 /*
743 * There are no "default" pid probes.
744 */
745}
746
747static int
748fasttrap_tracepoint_enable(proc_t *p, fasttrap_probe_t *probe, uint_t index)
749{
750 fasttrap_tracepoint_t *tp, *new_tp = NULL;
751 fasttrap_bucket_t *bucket;
752 fasttrap_id_t *id;
753 pid_t pid;
754 uintptr_t pc;
755
756 ASSERT(index < probe->ftp_ntps);
757
758 pid = probe->ftp_pid;
759 pc = probe->ftp_tps[index].fit_tp->ftt_pc;
760 id = &probe->ftp_tps[index].fit_id;
761
762 ASSERT(probe->ftp_tps[index].fit_tp->ftt_pid == pid);
763
764#if defined(sun)
765 ASSERT(!(p->p_flag & SVFORK));
766#endif
767
768 /*
769 * Before we make any modifications, make sure we've imposed a barrier
770 * on the generation in which this probe was last modified.
771 */
772 fasttrap_mod_barrier(probe->ftp_gen);
773
774 bucket = &fasttrap_tpoints.fth_table[FASTTRAP_TPOINTS_INDEX(pid, pc)];
775
776 /*
777 * If the tracepoint has already been enabled, just add our id to the
778 * list of interested probes. This may be our second time through
779 * this path in which case we'll have constructed the tracepoint we'd
780 * like to install. If we can't find a match, and have an allocated
781 * tracepoint ready to go, enable that one now.
782 *
783 * A tracepoint whose process is defunct is also considered defunct.
784 */
785again:
786 mutex_enter(&bucket->ftb_mtx);
787 for (tp = bucket->ftb_data; tp != NULL; tp = tp->ftt_next) {
788 /*
789 * Note that it's safe to access the active count on the
790 * associated proc structure because we know that at least one
791 * provider (this one) will still be around throughout this
792 * operation.
793 */
794 if (tp->ftt_pid != pid || tp->ftt_pc != pc ||
795 tp->ftt_proc->ftpc_acount == 0)
796 continue;
797
798 /*
799 * Now that we've found a matching tracepoint, it would be
800 * a decent idea to confirm that the tracepoint is still
801 * enabled and the trap instruction hasn't been overwritten.
802 * Since this is a little hairy, we'll punt for now.
803 */
804
805 /*
806 * This can't be the first interested probe. We don't have
807 * to worry about another thread being in the midst of
808 * deleting this tracepoint (which would be the only valid
809 * reason for a tracepoint to have no interested probes)
810 * since we're holding P_PR_LOCK for this process.
811 */
812 ASSERT(tp->ftt_ids != NULL || tp->ftt_retids != NULL);
813
814 switch (id->fti_ptype) {
815 case DTFTP_ENTRY:
816 case DTFTP_OFFSETS:
817 case DTFTP_IS_ENABLED:
818 id->fti_next = tp->ftt_ids;
819 membar_producer();
820 tp->ftt_ids = id;
821 membar_producer();
822 break;
823
824 case DTFTP_RETURN:
825 case DTFTP_POST_OFFSETS:
826 id->fti_next = tp->ftt_retids;
827 membar_producer();
828 tp->ftt_retids = id;
829 membar_producer();
830 break;
831
832 default:
833 ASSERT(0);
834 }
835
836 mutex_exit(&bucket->ftb_mtx);
837
838 if (new_tp != NULL) {
839 new_tp->ftt_ids = NULL;
840 new_tp->ftt_retids = NULL;
841 }
842
843 return (0);
844 }
845
846 /*
847 * If we have a good tracepoint ready to go, install it now while
848 * we have the lock held and no one can screw with us.
849 */
850 if (new_tp != NULL) {
851 int rc = 0;
852
853 new_tp->ftt_next = bucket->ftb_data;
854 membar_producer();
855 bucket->ftb_data = new_tp;
856 membar_producer();
857 mutex_exit(&bucket->ftb_mtx);
858
859 /*
860 * Activate the tracepoint in the ISA-specific manner.
861 * If this fails, we need to report the failure, but
862 * indicate that this tracepoint must still be disabled
863 * by calling fasttrap_tracepoint_disable().
864 */
865 if (fasttrap_tracepoint_install(p, new_tp) != 0)
866 rc = FASTTRAP_ENABLE_PARTIAL;
867
868 /*
869 * Increment the count of the number of tracepoints active in
870 * the victim process.
871 */
872#if defined(sun)
873 ASSERT(p->p_proc_flag & P_PR_LOCK);
874#endif
875 p->p_dtrace_count++;
876
877 return (rc);
878 }
879
880 mutex_exit(&bucket->ftb_mtx);
881
882 /*
883 * Initialize the tracepoint that's been preallocated with the probe.
884 */
885 new_tp = probe->ftp_tps[index].fit_tp;
886
887 ASSERT(new_tp->ftt_pid == pid);
888 ASSERT(new_tp->ftt_pc == pc);
889 ASSERT(new_tp->ftt_proc == probe->ftp_prov->ftp_proc);
890 ASSERT(new_tp->ftt_ids == NULL);
891 ASSERT(new_tp->ftt_retids == NULL);
892
893 switch (id->fti_ptype) {
894 case DTFTP_ENTRY:
895 case DTFTP_OFFSETS:
896 case DTFTP_IS_ENABLED:
897 id->fti_next = NULL;
898 new_tp->ftt_ids = id;
899 break;
900
901 case DTFTP_RETURN:
902 case DTFTP_POST_OFFSETS:
903 id->fti_next = NULL;
904 new_tp->ftt_retids = id;
905 break;
906
907 default:
908 ASSERT(0);
909 }
910
911 /*
912 * If the ISA-dependent initialization goes to plan, go back to the
913 * beginning and try to install this freshly made tracepoint.
914 */
915 if (fasttrap_tracepoint_init(p, new_tp, pc, id->fti_ptype) == 0)
916 goto again;
917
918 new_tp->ftt_ids = NULL;
919 new_tp->ftt_retids = NULL;
920
921 return (FASTTRAP_ENABLE_FAIL);
922}
923
924static void
925fasttrap_tracepoint_disable(proc_t *p, fasttrap_probe_t *probe, uint_t index)
926{
927 fasttrap_bucket_t *bucket;
928 fasttrap_provider_t *provider = probe->ftp_prov;
929 fasttrap_tracepoint_t **pp, *tp;
930 fasttrap_id_t *id, **idp = NULL;
931 pid_t pid;
932 uintptr_t pc;
933
934 ASSERT(index < probe->ftp_ntps);
935
936 pid = probe->ftp_pid;
937 pc = probe->ftp_tps[index].fit_tp->ftt_pc;
938 id = &probe->ftp_tps[index].fit_id;
939
940 ASSERT(probe->ftp_tps[index].fit_tp->ftt_pid == pid);
941
942 /*
943 * Find the tracepoint and make sure that our id is one of the
944 * ones registered with it.
945 */
946 bucket = &fasttrap_tpoints.fth_table[FASTTRAP_TPOINTS_INDEX(pid, pc)];
947 mutex_enter(&bucket->ftb_mtx);
948 for (tp = bucket->ftb_data; tp != NULL; tp = tp->ftt_next) {
949 if (tp->ftt_pid == pid && tp->ftt_pc == pc &&
950 tp->ftt_proc == provider->ftp_proc)
951 break;
952 }
953
954 /*
955 * If we somehow lost this tracepoint, we're in a world of hurt.
956 */
957 ASSERT(tp != NULL);
958
959 switch (id->fti_ptype) {
960 case DTFTP_ENTRY:
961 case DTFTP_OFFSETS:
962 case DTFTP_IS_ENABLED:
963 ASSERT(tp->ftt_ids != NULL);
964 idp = &tp->ftt_ids;
965 break;
966
967 case DTFTP_RETURN:
968 case DTFTP_POST_OFFSETS:
969 ASSERT(tp->ftt_retids != NULL);
970 idp = &tp->ftt_retids;
971 break;
972
973 default:
974 ASSERT(0);
975 }
976
977 while ((*idp)->fti_probe != probe) {
978 idp = &(*idp)->fti_next;
979 ASSERT(*idp != NULL);
980 }
981
982 id = *idp;
983 *idp = id->fti_next;
984 membar_producer();
985
986 ASSERT(id->fti_probe == probe);
987
988 /*
989 * If there are other registered enablings of this tracepoint, we're
990 * all done, but if this was the last probe assocated with this
991 * this tracepoint, we need to remove and free it.
992 */
993 if (tp->ftt_ids != NULL || tp->ftt_retids != NULL) {
994
995 /*
996 * If the current probe's tracepoint is in use, swap it
997 * for an unused tracepoint.
998 */
999 if (tp == probe->ftp_tps[index].fit_tp) {
1000 fasttrap_probe_t *tmp_probe;
1001 fasttrap_tracepoint_t **tmp_tp;
1002 uint_t tmp_index;
1003
1004 if (tp->ftt_ids != NULL) {
1005 tmp_probe = tp->ftt_ids->fti_probe;
1006 /* LINTED - alignment */
1007 tmp_index = FASTTRAP_ID_INDEX(tp->ftt_ids);
1008 tmp_tp = &tmp_probe->ftp_tps[tmp_index].fit_tp;
1009 } else {
1010 tmp_probe = tp->ftt_retids->fti_probe;
1011 /* LINTED - alignment */
1012 tmp_index = FASTTRAP_ID_INDEX(tp->ftt_retids);
1013 tmp_tp = &tmp_probe->ftp_tps[tmp_index].fit_tp;
1014 }
1015
1016 ASSERT(*tmp_tp != NULL);
1017 ASSERT(*tmp_tp != probe->ftp_tps[index].fit_tp);
1018 ASSERT((*tmp_tp)->ftt_ids == NULL);
1019 ASSERT((*tmp_tp)->ftt_retids == NULL);
1020
1021 probe->ftp_tps[index].fit_tp = *tmp_tp;
1022 *tmp_tp = tp;
1023 }
1024
1025 mutex_exit(&bucket->ftb_mtx);
1026
1027 /*
1028 * Tag the modified probe with the generation in which it was
1029 * changed.
1030 */
1031 probe->ftp_gen = fasttrap_mod_gen;
1032 return;
1033 }
1034
1035 mutex_exit(&bucket->ftb_mtx);
1036
1037 /*
1038 * We can't safely remove the tracepoint from the set of active
1039 * tracepoints until we've actually removed the fasttrap instruction
1040 * from the process's text. We can, however, operate on this
1041 * tracepoint secure in the knowledge that no other thread is going to
1042 * be looking at it since we hold P_PR_LOCK on the process if it's
1043 * live or we hold the provider lock on the process if it's dead and
1044 * gone.
1045 */
1046
1047 /*
1048 * We only need to remove the actual instruction if we're looking
1049 * at an existing process
1050 */
1051 if (p != NULL) {
1052 /*
1053 * If we fail to restore the instruction we need to kill
1054 * this process since it's in a completely unrecoverable
1055 * state.
1056 */
1057 if (fasttrap_tracepoint_remove(p, tp) != 0)
1058 fasttrap_sigtrap(p, NULL, pc);
1059
1060 /*
1061 * Decrement the count of the number of tracepoints active
1062 * in the victim process.
1063 */
1064#if defined(sun)
1065 ASSERT(p->p_proc_flag & P_PR_LOCK);
1066#endif
1067 p->p_dtrace_count--;
1068 }
1069
1070 /*
1071 * Remove the probe from the hash table of active tracepoints.
1072 */
1073 mutex_enter(&bucket->ftb_mtx);
1074 pp = (fasttrap_tracepoint_t **)&bucket->ftb_data;
1075 ASSERT(*pp != NULL);
1076 while (*pp != tp) {
1077 pp = &(*pp)->ftt_next;
1078 ASSERT(*pp != NULL);
1079 }
1080
1081 *pp = tp->ftt_next;
1082 membar_producer();
1083
1084 mutex_exit(&bucket->ftb_mtx);
1085
1086 /*
1087 * Tag the modified probe with the generation in which it was changed.
1088 */
1089 probe->ftp_gen = fasttrap_mod_gen;
1090}
1091
1092static void
1093fasttrap_enable_callbacks(void)
1094{
1095 /*
1096 * We don't have to play the rw lock game here because we're
1097 * providing something rather than taking something away --
1098 * we can be sure that no threads have tried to follow this
1099 * function pointer yet.
1100 */
1101 mutex_enter(&fasttrap_count_mtx);
1102 if (fasttrap_pid_count == 0) {
1103 ASSERT(dtrace_pid_probe_ptr == NULL);
1104 ASSERT(dtrace_return_probe_ptr == NULL);
1105 dtrace_pid_probe_ptr = &fasttrap_pid_probe;
1106 dtrace_return_probe_ptr = &fasttrap_return_probe;
1107 }
1108 ASSERT(dtrace_pid_probe_ptr == &fasttrap_pid_probe);
1109 ASSERT(dtrace_return_probe_ptr == &fasttrap_return_probe);
1110 fasttrap_pid_count++;
1111 mutex_exit(&fasttrap_count_mtx);
1112}
1113
1114static void
1115fasttrap_disable_callbacks(void)
1116{
1117#if defined(sun)
1118 ASSERT(MUTEX_HELD(&cpu_lock));
1119#endif
1120
1121
1122 mutex_enter(&fasttrap_count_mtx);
1123 ASSERT(fasttrap_pid_count > 0);
1124 fasttrap_pid_count--;
1125 if (fasttrap_pid_count == 0) {
1126#if defined(sun)
1127 cpu_t *cur, *cpu = CPU;
1128
1129 for (cur = cpu->cpu_next_onln; cur != cpu;
1130 cur = cur->cpu_next_onln) {
1131 rw_enter(&cur->cpu_ft_lock, RW_WRITER);
1132 }
1133#endif
1134 dtrace_pid_probe_ptr = NULL;
1135 dtrace_return_probe_ptr = NULL;
1136#if defined(sun)
1137 for (cur = cpu->cpu_next_onln; cur != cpu;
1138 cur = cur->cpu_next_onln) {
1139 rw_exit(&cur->cpu_ft_lock);
1140 }
1141#endif
1142 }
1143 mutex_exit(&fasttrap_count_mtx);
1144}
1145
1146/*ARGSUSED*/
1147static void
1148fasttrap_pid_enable(void *arg, dtrace_id_t id, void *parg)
1149{
1150 fasttrap_probe_t *probe = parg;
1151 proc_t *p = NULL;
1152 int i, rc;
1153
1154 ASSERT(probe != NULL);
1155 ASSERT(!probe->ftp_enabled);
1156 ASSERT(id == probe->ftp_id);
1157#if defined(sun)
1158 ASSERT(MUTEX_HELD(&cpu_lock));
1159#endif
1160
1161 /*
1162 * Increment the count of enabled probes on this probe's provider;
1163 * the provider can't go away while the probe still exists. We
1164 * must increment this even if we aren't able to properly enable
1165 * this probe.
1166 */
1167 mutex_enter(&probe->ftp_prov->ftp_mtx);
1168 probe->ftp_prov->ftp_rcount++;
1169 mutex_exit(&probe->ftp_prov->ftp_mtx);
1170
1171 /*
1172 * If this probe's provider is retired (meaning it was valid in a
1173 * previously exec'ed incarnation of this address space), bail out. The
1174 * provider can't go away while we're in this code path.
1175 */
1176 if (probe->ftp_prov->ftp_retired)
1177 return;
1178
1179 /*
1180 * If we can't find the process, it may be that we're in the context of
1181 * a fork in which the traced process is being born and we're copying
1182 * USDT probes. Otherwise, the process is gone so bail.
1183 */
1184#if defined(sun)
1185 if ((p = sprlock(probe->ftp_pid)) == NULL) {
1186 if ((curproc->p_flag & SFORKING) == 0)
1187 return;
1188
1189 mutex_enter(&pidlock);
1190 p = prfind(probe->ftp_pid);
1191
1192 /*
1193 * Confirm that curproc is indeed forking the process in which
1194 * we're trying to enable probes.
1195 */
1196 ASSERT(p != NULL);
1197 ASSERT(p->p_parent == curproc);
1198 ASSERT(p->p_stat == SIDL);
1199
1200 mutex_enter(&p->p_lock);
1201 mutex_exit(&pidlock);
1202
1203 sprlock_proc(p);
1204 }
1205
1206 ASSERT(!(p->p_flag & SVFORK));
1207 mutex_exit(&p->p_lock);
1208#else
1209 if ((p = pfind(probe->ftp_pid)) == NULL)
1210 return;
1211#endif
1212
1213 /*
1214 * We have to enable the trap entry point before any user threads have
1215 * the chance to execute the trap instruction we're about to place
1216 * in their process's text.
1217 */
1218#ifdef __FreeBSD__
1219 /*
1220 * pfind() returns a locked process.
1221 */
1222 _PHOLD(p);
1223 PROC_UNLOCK(p);
1224#endif
1225 fasttrap_enable_callbacks();
1226
1227 /*
1228 * Enable all the tracepoints and add this probe's id to each
1229 * tracepoint's list of active probes.
1230 */
1231 for (i = 0; i < probe->ftp_ntps; i++) {
1232 if ((rc = fasttrap_tracepoint_enable(p, probe, i)) != 0) {
1233 /*
1234 * If enabling the tracepoint failed completely,
1235 * we don't have to disable it; if the failure
1236 * was only partial we must disable it.
1237 */
1238 if (rc == FASTTRAP_ENABLE_FAIL)
1239 i--;
1240 else
1241 ASSERT(rc == FASTTRAP_ENABLE_PARTIAL);
1242
1243 /*
1244 * Back up and pull out all the tracepoints we've
1245 * created so far for this probe.
1246 */
1247 while (i >= 0) {
1248 fasttrap_tracepoint_disable(p, probe, i);
1249 i--;
1250 }
1251
1252#if defined(sun)
1253 mutex_enter(&p->p_lock);
1254 sprunlock(p);
1255#else
1256 PRELE(p);
1257#endif
1258
1259 /*
1260 * Since we're not actually enabling this probe,
1261 * drop our reference on the trap table entry.
1262 */
1263 fasttrap_disable_callbacks();
1264 return;
1265 }
1266 }
1267#if defined(sun)
1268 mutex_enter(&p->p_lock);
1269 sprunlock(p);
1270#else
1271 PRELE(p);
1272#endif
1273
1274 probe->ftp_enabled = 1;
1275}
1276
1277/*ARGSUSED*/
1278static void
1279fasttrap_pid_disable(void *arg, dtrace_id_t id, void *parg)
1280{
1281 fasttrap_probe_t *probe = parg;
1282 fasttrap_provider_t *provider = probe->ftp_prov;
1283 proc_t *p;
1284 int i, whack = 0;
1285
1286 ASSERT(id == probe->ftp_id);
1287
1288 mutex_enter(&provider->ftp_mtx);
1289
1290 /*
1291 * We won't be able to acquire a /proc-esque lock on the process
1292 * iff the process is dead and gone. In this case, we rely on the
1293 * provider lock as a point of mutual exclusion to prevent other
1294 * DTrace consumers from disabling this probe.
1295 */
1296 if ((p = pfind(probe->ftp_pid)) != NULL) {
1297#ifdef __FreeBSD__
1298 _PHOLD(p);
1299 PROC_UNLOCK(p);
1300#endif
1301 }
1302
1303 /*
1304 * Disable all the associated tracepoints (for fully enabled probes).
1305 */
1306 if (probe->ftp_enabled) {
1307 for (i = 0; i < probe->ftp_ntps; i++) {
1308 fasttrap_tracepoint_disable(p, probe, i);
1309 }
1310 }
1311
1312 ASSERT(provider->ftp_rcount > 0);
1313 provider->ftp_rcount--;
1314
1315 if (p != NULL) {
1316 /*
1317 * Even though we may not be able to remove it entirely, we
1318 * mark this retired provider to get a chance to remove some
1319 * of the associated probes.
1320 */
1321 if (provider->ftp_retired && !provider->ftp_marked)
1322 whack = provider->ftp_marked = 1;
1323 mutex_exit(&provider->ftp_mtx);
1324 } else {
1325 /*
1326 * If the process is dead, we're just waiting for the
1327 * last probe to be disabled to be able to free it.
1328 */
1329 if (provider->ftp_rcount == 0 && !provider->ftp_marked)
1330 whack = provider->ftp_marked = 1;
1331 mutex_exit(&provider->ftp_mtx);
1332 }
1333
1334 if (whack)
1335 fasttrap_pid_cleanup();
1336
1337#ifdef __FreeBSD__
1338 if (p != NULL)
1339 PRELE(p);
1340#endif
1341 if (!probe->ftp_enabled)
1342 return;
1343
1344 probe->ftp_enabled = 0;
1345
1346#if defined(sun)
1347 ASSERT(MUTEX_HELD(&cpu_lock));
1348#endif
1349 fasttrap_disable_callbacks();
1350}
1351
1352/*ARGSUSED*/
1353static void
1354fasttrap_pid_getargdesc(void *arg, dtrace_id_t id, void *parg,
1355 dtrace_argdesc_t *desc)
1356{
1357 fasttrap_probe_t *probe = parg;
1358 char *str;
1359 int i, ndx;
1360
1361 desc->dtargd_native[0] = '\0';
1362 desc->dtargd_xlate[0] = '\0';
1363
1364 if (probe->ftp_prov->ftp_retired != 0 ||
1365 desc->dtargd_ndx >= probe->ftp_nargs) {
1366 desc->dtargd_ndx = DTRACE_ARGNONE;
1367 return;
1368 }
1369
1370 ndx = (probe->ftp_argmap != NULL) ?
1371 probe->ftp_argmap[desc->dtargd_ndx] : desc->dtargd_ndx;
1372
1373 str = probe->ftp_ntypes;
1374 for (i = 0; i < ndx; i++) {
1375 str += strlen(str) + 1;
1376 }
1377
1378 ASSERT(strlen(str + 1) < sizeof (desc->dtargd_native));
1379 (void) strcpy(desc->dtargd_native, str);
1380
1381 if (probe->ftp_xtypes == NULL)
1382 return;
1383
1384 str = probe->ftp_xtypes;
1385 for (i = 0; i < desc->dtargd_ndx; i++) {
1386 str += strlen(str) + 1;
1387 }
1388
1389 ASSERT(strlen(str + 1) < sizeof (desc->dtargd_xlate));
1390 (void) strcpy(desc->dtargd_xlate, str);
1391}
1392
1393/*ARGSUSED*/
1394static void
1395fasttrap_pid_destroy(void *arg, dtrace_id_t id, void *parg)
1396{
1397 fasttrap_probe_t *probe = parg;
1398 int i;
1399 size_t size;
1400
1401 ASSERT(probe != NULL);
1402 ASSERT(!probe->ftp_enabled);
1403 ASSERT(fasttrap_total >= probe->ftp_ntps);
1404
1405 atomic_add_32(&fasttrap_total, -probe->ftp_ntps);
1406 size = offsetof(fasttrap_probe_t, ftp_tps[probe->ftp_ntps]);
1407
1408 if (probe->ftp_gen + 1 >= fasttrap_mod_gen)
1409 fasttrap_mod_barrier(probe->ftp_gen);
1410
1411 for (i = 0; i < probe->ftp_ntps; i++) {
1412 kmem_free(probe->ftp_tps[i].fit_tp,
1413 sizeof (fasttrap_tracepoint_t));
1414 }
1415
1416 kmem_free(probe, size);
1417}
1418
1419
1420static const dtrace_pattr_t pid_attr = {
1421{ DTRACE_STABILITY_EVOLVING, DTRACE_STABILITY_EVOLVING, DTRACE_CLASS_ISA },
1422{ DTRACE_STABILITY_PRIVATE, DTRACE_STABILITY_PRIVATE, DTRACE_CLASS_UNKNOWN },
1423{ DTRACE_STABILITY_PRIVATE, DTRACE_STABILITY_PRIVATE, DTRACE_CLASS_UNKNOWN },
1424{ DTRACE_STABILITY_EVOLVING, DTRACE_STABILITY_EVOLVING, DTRACE_CLASS_ISA },
1425{ DTRACE_STABILITY_PRIVATE, DTRACE_STABILITY_PRIVATE, DTRACE_CLASS_UNKNOWN },
1426};
1427
1428static dtrace_pops_t pid_pops = {
1429 fasttrap_pid_provide,
1430 NULL,
1431 fasttrap_pid_enable,
1432 fasttrap_pid_disable,
1433 NULL,
1434 NULL,
1435 fasttrap_pid_getargdesc,
1436 fasttrap_pid_getarg,
1437 NULL,
1438 fasttrap_pid_destroy
1439};
1440
1441static dtrace_pops_t usdt_pops = {
1442 fasttrap_pid_provide,
1443 NULL,
1444 fasttrap_pid_enable,
1445 fasttrap_pid_disable,
1446 NULL,
1447 NULL,
1448 fasttrap_pid_getargdesc,
1449 fasttrap_usdt_getarg,
1450 NULL,
1451 fasttrap_pid_destroy
1452};
1453
1454static fasttrap_proc_t *
1455fasttrap_proc_lookup(pid_t pid)
1456{
1457 fasttrap_bucket_t *bucket;
1458 fasttrap_proc_t *fprc, *new_fprc;
1459
1460
1461 bucket = &fasttrap_procs.fth_table[FASTTRAP_PROCS_INDEX(pid)];
1462 mutex_enter(&bucket->ftb_mtx);
1463
1464 for (fprc = bucket->ftb_data; fprc != NULL; fprc = fprc->ftpc_next) {
1465 if (fprc->ftpc_pid == pid && fprc->ftpc_acount != 0) {
1466 mutex_enter(&fprc->ftpc_mtx);
1467 mutex_exit(&bucket->ftb_mtx);
1468 fprc->ftpc_rcount++;
1469 atomic_add_64(&fprc->ftpc_acount, 1);
1470 ASSERT(fprc->ftpc_acount <= fprc->ftpc_rcount);
1471 mutex_exit(&fprc->ftpc_mtx);
1472
1473 return (fprc);
1474 }
1475 }
1476
1477 /*
1478 * Drop the bucket lock so we don't try to perform a sleeping
1479 * allocation under it.
1480 */
1481 mutex_exit(&bucket->ftb_mtx);
1482
1483 new_fprc = kmem_zalloc(sizeof (fasttrap_proc_t), KM_SLEEP);
1484 new_fprc->ftpc_pid = pid;
1485 new_fprc->ftpc_rcount = 1;
1486 new_fprc->ftpc_acount = 1;
1487#if !defined(sun)
1488 mutex_init(&new_fprc->ftpc_mtx, "fasttrap proc mtx", MUTEX_DEFAULT,
1489 NULL);
1490#endif
1491
1492 mutex_enter(&bucket->ftb_mtx);
1493
1494 /*
1495 * Take another lap through the list to make sure a proc hasn't
1496 * been created for this pid while we weren't under the bucket lock.
1497 */
1498 for (fprc = bucket->ftb_data; fprc != NULL; fprc = fprc->ftpc_next) {
1499 if (fprc->ftpc_pid == pid && fprc->ftpc_acount != 0) {
1500 mutex_enter(&fprc->ftpc_mtx);
1501 mutex_exit(&bucket->ftb_mtx);
1502 fprc->ftpc_rcount++;
1503 atomic_add_64(&fprc->ftpc_acount, 1);
1504 ASSERT(fprc->ftpc_acount <= fprc->ftpc_rcount);
1505 mutex_exit(&fprc->ftpc_mtx);
1506
1507 kmem_free(new_fprc, sizeof (fasttrap_proc_t));
1508
1509 return (fprc);
1510 }
1511 }
1512
1513 new_fprc->ftpc_next = bucket->ftb_data;
1514 bucket->ftb_data = new_fprc;
1515
1516 mutex_exit(&bucket->ftb_mtx);
1517
1518 return (new_fprc);
1519}
1520
1521static void
1522fasttrap_proc_release(fasttrap_proc_t *proc)
1523{
1524 fasttrap_bucket_t *bucket;
1525 fasttrap_proc_t *fprc, **fprcp;
1526 pid_t pid = proc->ftpc_pid;
|
| 1527#if !defined(sun)
1528 fasttrap_scrblock_t *scrblk, *scrblktmp;
1529 fasttrap_scrspace_t *scrspc, *scrspctmp;
1530 struct proc *p;
1531 struct thread *td;
1532#endif
|
1370
1371 mutex_enter(&proc->ftpc_mtx);
1372
1373 ASSERT(proc->ftpc_rcount != 0);
1374 ASSERT(proc->ftpc_acount <= proc->ftpc_rcount);
1375
1376 if (--proc->ftpc_rcount != 0) {
1377 mutex_exit(&proc->ftpc_mtx);
1378 return;
1379 }
1380
| 1533
1534 mutex_enter(&proc->ftpc_mtx);
1535
1536 ASSERT(proc->ftpc_rcount != 0);
1537 ASSERT(proc->ftpc_acount <= proc->ftpc_rcount);
1538
1539 if (--proc->ftpc_rcount != 0) {
1540 mutex_exit(&proc->ftpc_mtx);
1541 return;
1542 }
1543
|
| 1544#if !defined(sun)
1545 /*
1546 * Free all structures used to manage per-thread scratch space.
1547 */
1548 LIST_FOREACH_SAFE(scrblk, &proc->ftpc_scrblks, ftsb_next,
1549 scrblktmp) {
1550 LIST_REMOVE(scrblk, ftsb_next);
1551 free(scrblk, M_SOLARIS);
1552 }
1553 LIST_FOREACH_SAFE(scrspc, &proc->ftpc_fscr, ftss_next, scrspctmp) {
1554 LIST_REMOVE(scrspc, ftss_next);
1555 free(scrspc, M_SOLARIS);
1556 }
1557 LIST_FOREACH_SAFE(scrspc, &proc->ftpc_ascr, ftss_next, scrspctmp) {
1558 LIST_REMOVE(scrspc, ftss_next);
1559 free(scrspc, M_SOLARIS);
1560 }
1561
1562 if ((p = pfind(pid)) != NULL) {
1563 FOREACH_THREAD_IN_PROC(p, td)
1564 td->t_dtrace_sscr = NULL;
1565 PROC_UNLOCK(p);
1566 }
1567#endif
1568
|
1381 mutex_exit(&proc->ftpc_mtx);
1382
1383 /*
1384 * There should definitely be no live providers associated with this
1385 * process at this point.
1386 */
1387 ASSERT(proc->ftpc_acount == 0);
1388
1389 bucket = &fasttrap_procs.fth_table[FASTTRAP_PROCS_INDEX(pid)];
1390 mutex_enter(&bucket->ftb_mtx);
1391
1392 fprcp = (fasttrap_proc_t **)&bucket->ftb_data;
1393 while ((fprc = *fprcp) != NULL) {
1394 if (fprc == proc)
1395 break;
1396
1397 fprcp = &fprc->ftpc_next;
1398 }
1399
1400 /*
1401 * Something strange has happened if we can't find the proc.
1402 */
1403 ASSERT(fprc != NULL);
1404
1405 *fprcp = fprc->ftpc_next;
1406
1407 mutex_exit(&bucket->ftb_mtx);
1408
1409 kmem_free(fprc, sizeof (fasttrap_proc_t));
1410}
1411
1412/*
1413 * Lookup a fasttrap-managed provider based on its name and associated pid.
1414 * If the pattr argument is non-NULL, this function instantiates the provider
1415 * if it doesn't exist otherwise it returns NULL. The provider is returned
1416 * with its lock held.
1417 */
1418static fasttrap_provider_t *
1419fasttrap_provider_lookup(pid_t pid, const char *name,
1420 const dtrace_pattr_t *pattr)
1421{
1422 fasttrap_provider_t *fp, *new_fp = NULL;
1423 fasttrap_bucket_t *bucket;
1424 char provname[DTRACE_PROVNAMELEN];
1425 proc_t *p;
1426 cred_t *cred;
1427
1428 ASSERT(strlen(name) < sizeof (fp->ftp_name));
1429 ASSERT(pattr != NULL);
1430
1431 bucket = &fasttrap_provs.fth_table[FASTTRAP_PROVS_INDEX(pid, name)];
1432 mutex_enter(&bucket->ftb_mtx);
1433
1434 /*
1435 * Take a lap through the list and return the match if we find it.
1436 */
1437 for (fp = bucket->ftb_data; fp != NULL; fp = fp->ftp_next) {
1438 if (fp->ftp_pid == pid && strcmp(fp->ftp_name, name) == 0 &&
1439 !fp->ftp_retired) {
1440 mutex_enter(&fp->ftp_mtx);
1441 mutex_exit(&bucket->ftb_mtx);
1442 return (fp);
1443 }
1444 }
1445
1446 /*
1447 * Drop the bucket lock so we don't try to perform a sleeping
1448 * allocation under it.
1449 */
1450 mutex_exit(&bucket->ftb_mtx);
1451
1452 /*
1453 * Make sure the process exists, isn't a child created as the result
1454 * of a vfork(2), and isn't a zombie (but may be in fork).
1455 */
1456 if ((p = pfind(pid)) == NULL)
1457 return (NULL);
1458
1459 /*
1460 * Increment p_dtrace_probes so that the process knows to inform us
1461 * when it exits or execs. fasttrap_provider_free() decrements this
1462 * when we're done with this provider.
1463 */
1464 p->p_dtrace_probes++;
1465
1466 /*
1467 * Grab the credentials for this process so we have
1468 * something to pass to dtrace_register().
1469 */
1470 PROC_LOCK_ASSERT(p, MA_OWNED);
1471 crhold(p->p_ucred);
1472 cred = p->p_ucred;
1473 PROC_UNLOCK(p);
1474
1475 new_fp = kmem_zalloc(sizeof (fasttrap_provider_t), KM_SLEEP);
1476 new_fp->ftp_pid = pid;
1477 new_fp->ftp_proc = fasttrap_proc_lookup(pid);
1478#if !defined(sun)
1479 mutex_init(&new_fp->ftp_mtx, "provider mtx", MUTEX_DEFAULT, NULL);
1480 mutex_init(&new_fp->ftp_cmtx, "lock on creating", MUTEX_DEFAULT, NULL);
1481#endif
1482
1483 ASSERT(new_fp->ftp_proc != NULL);
1484
1485 mutex_enter(&bucket->ftb_mtx);
1486
1487 /*
1488 * Take another lap through the list to make sure a provider hasn't
1489 * been created for this pid while we weren't under the bucket lock.
1490 */
1491 for (fp = bucket->ftb_data; fp != NULL; fp = fp->ftp_next) {
1492 if (fp->ftp_pid == pid && strcmp(fp->ftp_name, name) == 0 &&
1493 !fp->ftp_retired) {
1494 mutex_enter(&fp->ftp_mtx);
1495 mutex_exit(&bucket->ftb_mtx);
1496 fasttrap_provider_free(new_fp);
1497 crfree(cred);
1498 return (fp);
1499 }
1500 }
1501
1502 (void) strcpy(new_fp->ftp_name, name);
1503
1504 /*
1505 * Fail and return NULL if either the provider name is too long
1506 * or we fail to register this new provider with the DTrace
1507 * framework. Note that this is the only place we ever construct
1508 * the full provider name -- we keep it in pieces in the provider
1509 * structure.
1510 */
1511 if (snprintf(provname, sizeof (provname), "%s%u", name, (uint_t)pid) >=
1512 sizeof (provname) ||
1513 dtrace_register(provname, pattr,
1514 DTRACE_PRIV_PROC | DTRACE_PRIV_OWNER | DTRACE_PRIV_ZONEOWNER, cred,
1515 pattr == &pid_attr ? &pid_pops : &usdt_pops, new_fp,
1516 &new_fp->ftp_provid) != 0) {
1517 mutex_exit(&bucket->ftb_mtx);
1518 fasttrap_provider_free(new_fp);
1519 crfree(cred);
1520 return (NULL);
1521 }
1522
1523 new_fp->ftp_next = bucket->ftb_data;
1524 bucket->ftb_data = new_fp;
1525
1526 mutex_enter(&new_fp->ftp_mtx);
1527 mutex_exit(&bucket->ftb_mtx);
1528
1529 crfree(cred);
1530 return (new_fp);
1531}
1532
1533static void
1534fasttrap_provider_free(fasttrap_provider_t *provider)
1535{
1536 pid_t pid = provider->ftp_pid;
1537 proc_t *p;
1538
1539 /*
1540 * There need to be no associated enabled probes, no consumers
1541 * creating probes, and no meta providers referencing this provider.
1542 */
1543 ASSERT(provider->ftp_rcount == 0);
1544 ASSERT(provider->ftp_ccount == 0);
1545 ASSERT(provider->ftp_mcount == 0);
1546
1547 /*
1548 * If this provider hasn't been retired, we need to explicitly drop the
1549 * count of active providers on the associated process structure.
1550 */
1551 if (!provider->ftp_retired) {
1552 atomic_add_64(&provider->ftp_proc->ftpc_acount, -1);
1553 ASSERT(provider->ftp_proc->ftpc_acount <
1554 provider->ftp_proc->ftpc_rcount);
1555 }
1556
1557 fasttrap_proc_release(provider->ftp_proc);
1558
1559#if !defined(sun)
1560 mutex_destroy(&provider->ftp_mtx);
1561 mutex_destroy(&provider->ftp_cmtx);
1562#endif
1563 kmem_free(provider, sizeof (fasttrap_provider_t));
1564
1565 /*
1566 * Decrement p_dtrace_probes on the process whose provider we're
1567 * freeing. We don't have to worry about clobbering somone else's
1568 * modifications to it because we have locked the bucket that
1569 * corresponds to this process's hash chain in the provider hash
1570 * table. Don't sweat it if we can't find the process.
1571 */
1572 if ((p = pfind(pid)) == NULL) {
1573 return;
1574 }
1575
1576 p->p_dtrace_probes--;
1577#if !defined(sun)
1578 PROC_UNLOCK(p);
1579#endif
1580}
1581
1582static void
1583fasttrap_provider_retire(pid_t pid, const char *name, int mprov)
1584{
1585 fasttrap_provider_t *fp;
1586 fasttrap_bucket_t *bucket;
1587 dtrace_provider_id_t provid;
1588
1589 ASSERT(strlen(name) < sizeof (fp->ftp_name));
1590
1591 bucket = &fasttrap_provs.fth_table[FASTTRAP_PROVS_INDEX(pid, name)];
1592 mutex_enter(&bucket->ftb_mtx);
1593
1594 for (fp = bucket->ftb_data; fp != NULL; fp = fp->ftp_next) {
1595 if (fp->ftp_pid == pid && strcmp(fp->ftp_name, name) == 0 &&
1596 !fp->ftp_retired)
1597 break;
1598 }
1599
1600 if (fp == NULL) {
1601 mutex_exit(&bucket->ftb_mtx);
1602 return;
1603 }
1604
1605 mutex_enter(&fp->ftp_mtx);
1606 ASSERT(!mprov || fp->ftp_mcount > 0);
1607 if (mprov && --fp->ftp_mcount != 0) {
1608 mutex_exit(&fp->ftp_mtx);
1609 mutex_exit(&bucket->ftb_mtx);
1610 return;
1611 }
1612
1613 /*
1614 * Mark the provider to be removed in our post-processing step, mark it
1615 * retired, and drop the active count on its proc. Marking it indicates
1616 * that we should try to remove it; setting the retired flag indicates
1617 * that we're done with this provider; dropping the active the proc
1618 * releases our hold, and when this reaches zero (as it will during
1619 * exit or exec) the proc and associated providers become defunct.
1620 *
1621 * We obviously need to take the bucket lock before the provider lock
1622 * to perform the lookup, but we need to drop the provider lock
1623 * before calling into the DTrace framework since we acquire the
1624 * provider lock in callbacks invoked from the DTrace framework. The
1625 * bucket lock therefore protects the integrity of the provider hash
1626 * table.
1627 */
1628 atomic_add_64(&fp->ftp_proc->ftpc_acount, -1);
1629 ASSERT(fp->ftp_proc->ftpc_acount < fp->ftp_proc->ftpc_rcount);
1630
1631 fp->ftp_retired = 1;
1632 fp->ftp_marked = 1;
1633 provid = fp->ftp_provid;
1634 mutex_exit(&fp->ftp_mtx);
1635
1636 /*
1637 * We don't have to worry about invalidating the same provider twice
1638 * since fasttrap_provider_lookup() will ignore provider that have
1639 * been marked as retired.
1640 */
1641 dtrace_invalidate(provid);
1642
1643 mutex_exit(&bucket->ftb_mtx);
1644
1645 fasttrap_pid_cleanup();
1646}
1647
1648static int
1649fasttrap_uint32_cmp(const void *ap, const void *bp)
1650{
1651 return (*(const uint32_t *)ap - *(const uint32_t *)bp);
1652}
1653
1654static int
1655fasttrap_uint64_cmp(const void *ap, const void *bp)
1656{
1657 return (*(const uint64_t *)ap - *(const uint64_t *)bp);
1658}
1659
1660static int
1661fasttrap_add_probe(fasttrap_probe_spec_t *pdata)
1662{
1663 fasttrap_provider_t *provider;
1664 fasttrap_probe_t *pp;
1665 fasttrap_tracepoint_t *tp;
1666 char *name;
1667 int i, aframes = 0, whack;
1668
1669 /*
1670 * There needs to be at least one desired trace point.
1671 */
1672 if (pdata->ftps_noffs == 0)
1673 return (EINVAL);
1674
1675 switch (pdata->ftps_type) {
1676 case DTFTP_ENTRY:
1677 name = "entry";
1678 aframes = FASTTRAP_ENTRY_AFRAMES;
1679 break;
1680 case DTFTP_RETURN:
1681 name = "return";
1682 aframes = FASTTRAP_RETURN_AFRAMES;
1683 break;
1684 case DTFTP_OFFSETS:
1685 name = NULL;
1686 break;
1687 default:
1688 return (EINVAL);
1689 }
1690
1691 if ((provider = fasttrap_provider_lookup(pdata->ftps_pid,
1692 FASTTRAP_PID_NAME, &pid_attr)) == NULL)
1693 return (ESRCH);
1694
1695 /*
1696 * Increment this reference count to indicate that a consumer is
1697 * actively adding a new probe associated with this provider. This
1698 * prevents the provider from being deleted -- we'll need to check
1699 * for pending deletions when we drop this reference count.
1700 */
1701 provider->ftp_ccount++;
1702 mutex_exit(&provider->ftp_mtx);
1703
1704 /*
1705 * Grab the creation lock to ensure consistency between calls to
1706 * dtrace_probe_lookup() and dtrace_probe_create() in the face of
1707 * other threads creating probes. We must drop the provider lock
1708 * before taking this lock to avoid a three-way deadlock with the
1709 * DTrace framework.
1710 */
1711 mutex_enter(&provider->ftp_cmtx);
1712
1713 if (name == NULL) {
1714 for (i = 0; i < pdata->ftps_noffs; i++) {
1715 char name_str[17];
1716
1717 (void) sprintf(name_str, "%llx",
1718 (unsigned long long)pdata->ftps_offs[i]);
1719
1720 if (dtrace_probe_lookup(provider->ftp_provid,
1721 pdata->ftps_mod, pdata->ftps_func, name_str) != 0)
1722 continue;
1723
1724 atomic_add_32(&fasttrap_total, 1);
1725
1726 if (fasttrap_total > fasttrap_max) {
1727 atomic_add_32(&fasttrap_total, -1);
1728 goto no_mem;
1729 }
1730
1731 pp = kmem_zalloc(sizeof (fasttrap_probe_t), KM_SLEEP);
1732
1733 pp->ftp_prov = provider;
1734 pp->ftp_faddr = pdata->ftps_pc;
1735 pp->ftp_fsize = pdata->ftps_size;
1736 pp->ftp_pid = pdata->ftps_pid;
1737 pp->ftp_ntps = 1;
1738
1739 tp = kmem_zalloc(sizeof (fasttrap_tracepoint_t),
1740 KM_SLEEP);
1741
1742 tp->ftt_proc = provider->ftp_proc;
1743 tp->ftt_pc = pdata->ftps_offs[i] + pdata->ftps_pc;
1744 tp->ftt_pid = pdata->ftps_pid;
1745
1746 pp->ftp_tps[0].fit_tp = tp;
1747 pp->ftp_tps[0].fit_id.fti_probe = pp;
1748 pp->ftp_tps[0].fit_id.fti_ptype = pdata->ftps_type;
1749
1750 pp->ftp_id = dtrace_probe_create(provider->ftp_provid,
1751 pdata->ftps_mod, pdata->ftps_func, name_str,
1752 FASTTRAP_OFFSET_AFRAMES, pp);
1753 }
1754
1755 } else if (dtrace_probe_lookup(provider->ftp_provid, pdata->ftps_mod,
1756 pdata->ftps_func, name) == 0) {
1757 atomic_add_32(&fasttrap_total, pdata->ftps_noffs);
1758
1759 if (fasttrap_total > fasttrap_max) {
1760 atomic_add_32(&fasttrap_total, -pdata->ftps_noffs);
1761 goto no_mem;
1762 }
1763
1764 /*
1765 * Make sure all tracepoint program counter values are unique.
1766 * We later assume that each probe has exactly one tracepoint
1767 * for a given pc.
1768 */
1769 qsort(pdata->ftps_offs, pdata->ftps_noffs,
1770 sizeof (uint64_t), fasttrap_uint64_cmp);
1771 for (i = 1; i < pdata->ftps_noffs; i++) {
1772 if (pdata->ftps_offs[i] > pdata->ftps_offs[i - 1])
1773 continue;
1774
1775 atomic_add_32(&fasttrap_total, -pdata->ftps_noffs);
1776 goto no_mem;
1777 }
1778
1779 ASSERT(pdata->ftps_noffs > 0);
1780 pp = kmem_zalloc(offsetof(fasttrap_probe_t,
1781 ftp_tps[pdata->ftps_noffs]), KM_SLEEP);
1782
1783 pp->ftp_prov = provider;
1784 pp->ftp_faddr = pdata->ftps_pc;
1785 pp->ftp_fsize = pdata->ftps_size;
1786 pp->ftp_pid = pdata->ftps_pid;
1787 pp->ftp_ntps = pdata->ftps_noffs;
1788
1789 for (i = 0; i < pdata->ftps_noffs; i++) {
1790 tp = kmem_zalloc(sizeof (fasttrap_tracepoint_t),
1791 KM_SLEEP);
1792
1793 tp->ftt_proc = provider->ftp_proc;
1794 tp->ftt_pc = pdata->ftps_offs[i] + pdata->ftps_pc;
1795 tp->ftt_pid = pdata->ftps_pid;
1796
1797 pp->ftp_tps[i].fit_tp = tp;
1798 pp->ftp_tps[i].fit_id.fti_probe = pp;
1799 pp->ftp_tps[i].fit_id.fti_ptype = pdata->ftps_type;
1800 }
1801
1802 pp->ftp_id = dtrace_probe_create(provider->ftp_provid,
1803 pdata->ftps_mod, pdata->ftps_func, name, aframes, pp);
1804 }
1805
1806 mutex_exit(&provider->ftp_cmtx);
1807
1808 /*
1809 * We know that the provider is still valid since we incremented the
1810 * creation reference count. If someone tried to clean up this provider
1811 * while we were using it (e.g. because the process called exec(2) or
1812 * exit(2)), take note of that and try to clean it up now.
1813 */
1814 mutex_enter(&provider->ftp_mtx);
1815 provider->ftp_ccount--;
1816 whack = provider->ftp_retired;
1817 mutex_exit(&provider->ftp_mtx);
1818
1819 if (whack)
1820 fasttrap_pid_cleanup();
1821
1822 return (0);
1823
1824no_mem:
1825 /*
1826 * If we've exhausted the allowable resources, we'll try to remove
1827 * this provider to free some up. This is to cover the case where
1828 * the user has accidentally created many more probes than was
1829 * intended (e.g. pid123:::).
1830 */
1831 mutex_exit(&provider->ftp_cmtx);
1832 mutex_enter(&provider->ftp_mtx);
1833 provider->ftp_ccount--;
1834 provider->ftp_marked = 1;
1835 mutex_exit(&provider->ftp_mtx);
1836
1837 fasttrap_pid_cleanup();
1838
1839 return (ENOMEM);
1840}
1841
1842/*ARGSUSED*/
1843static void *
1844fasttrap_meta_provide(void *arg, dtrace_helper_provdesc_t *dhpv, pid_t pid)
1845{
1846 fasttrap_provider_t *provider;
1847
1848 /*
1849 * A 32-bit unsigned integer (like a pid for example) can be
1850 * expressed in 10 or fewer decimal digits. Make sure that we'll
1851 * have enough space for the provider name.
1852 */
1853 if (strlen(dhpv->dthpv_provname) + 10 >=
1854 sizeof (provider->ftp_name)) {
1855 printf("failed to instantiate provider %s: "
1856 "name too long to accomodate pid", dhpv->dthpv_provname);
1857 return (NULL);
1858 }
1859
1860 /*
1861 * Don't let folks spoof the true pid provider.
1862 */
1863 if (strcmp(dhpv->dthpv_provname, FASTTRAP_PID_NAME) == 0) {
1864 printf("failed to instantiate provider %s: "
1865 "%s is an invalid name", dhpv->dthpv_provname,
1866 FASTTRAP_PID_NAME);
1867 return (NULL);
1868 }
1869
1870 /*
1871 * The highest stability class that fasttrap supports is ISA; cap
1872 * the stability of the new provider accordingly.
1873 */
1874 if (dhpv->dthpv_pattr.dtpa_provider.dtat_class > DTRACE_CLASS_ISA)
1875 dhpv->dthpv_pattr.dtpa_provider.dtat_class = DTRACE_CLASS_ISA;
1876 if (dhpv->dthpv_pattr.dtpa_mod.dtat_class > DTRACE_CLASS_ISA)
1877 dhpv->dthpv_pattr.dtpa_mod.dtat_class = DTRACE_CLASS_ISA;
1878 if (dhpv->dthpv_pattr.dtpa_func.dtat_class > DTRACE_CLASS_ISA)
1879 dhpv->dthpv_pattr.dtpa_func.dtat_class = DTRACE_CLASS_ISA;
1880 if (dhpv->dthpv_pattr.dtpa_name.dtat_class > DTRACE_CLASS_ISA)
1881 dhpv->dthpv_pattr.dtpa_name.dtat_class = DTRACE_CLASS_ISA;
1882 if (dhpv->dthpv_pattr.dtpa_args.dtat_class > DTRACE_CLASS_ISA)
1883 dhpv->dthpv_pattr.dtpa_args.dtat_class = DTRACE_CLASS_ISA;
1884
1885 if ((provider = fasttrap_provider_lookup(pid, dhpv->dthpv_provname,
1886 &dhpv->dthpv_pattr)) == NULL) {
1887 printf("failed to instantiate provider %s for "
1888 "process %u", dhpv->dthpv_provname, (uint_t)pid);
1889 return (NULL);
1890 }
1891
1892 /*
1893 * Up the meta provider count so this provider isn't removed until
1894 * the meta provider has been told to remove it.
1895 */
1896 provider->ftp_mcount++;
1897
1898 mutex_exit(&provider->ftp_mtx);
1899
1900 return (provider);
1901}
1902
1903/*ARGSUSED*/
1904static void
1905fasttrap_meta_create_probe(void *arg, void *parg,
1906 dtrace_helper_probedesc_t *dhpb)
1907{
1908 fasttrap_provider_t *provider = parg;
1909 fasttrap_probe_t *pp;
1910 fasttrap_tracepoint_t *tp;
1911 int i, j;
1912 uint32_t ntps;
1913
1914 /*
1915 * Since the meta provider count is non-zero we don't have to worry
1916 * about this provider disappearing.
1917 */
1918 ASSERT(provider->ftp_mcount > 0);
1919
1920 /*
1921 * The offsets must be unique.
1922 */
1923 qsort(dhpb->dthpb_offs, dhpb->dthpb_noffs, sizeof (uint32_t),
1924 fasttrap_uint32_cmp);
1925 for (i = 1; i < dhpb->dthpb_noffs; i++) {
1926 if (dhpb->dthpb_base + dhpb->dthpb_offs[i] <=
1927 dhpb->dthpb_base + dhpb->dthpb_offs[i - 1])
1928 return;
1929 }
1930
1931 qsort(dhpb->dthpb_enoffs, dhpb->dthpb_nenoffs, sizeof (uint32_t),
1932 fasttrap_uint32_cmp);
1933 for (i = 1; i < dhpb->dthpb_nenoffs; i++) {
1934 if (dhpb->dthpb_base + dhpb->dthpb_enoffs[i] <=
1935 dhpb->dthpb_base + dhpb->dthpb_enoffs[i - 1])
1936 return;
1937 }
1938
1939 /*
1940 * Grab the creation lock to ensure consistency between calls to
1941 * dtrace_probe_lookup() and dtrace_probe_create() in the face of
1942 * other threads creating probes.
1943 */
1944 mutex_enter(&provider->ftp_cmtx);
1945
1946 if (dtrace_probe_lookup(provider->ftp_provid, dhpb->dthpb_mod,
1947 dhpb->dthpb_func, dhpb->dthpb_name) != 0) {
1948 mutex_exit(&provider->ftp_cmtx);
1949 return;
1950 }
1951
1952 ntps = dhpb->dthpb_noffs + dhpb->dthpb_nenoffs;
1953 ASSERT(ntps > 0);
1954
1955 atomic_add_32(&fasttrap_total, ntps);
1956
1957 if (fasttrap_total > fasttrap_max) {
1958 atomic_add_32(&fasttrap_total, -ntps);
1959 mutex_exit(&provider->ftp_cmtx);
1960 return;
1961 }
1962
1963 pp = kmem_zalloc(offsetof(fasttrap_probe_t, ftp_tps[ntps]), KM_SLEEP);
1964
1965 pp->ftp_prov = provider;
1966 pp->ftp_pid = provider->ftp_pid;
1967 pp->ftp_ntps = ntps;
1968 pp->ftp_nargs = dhpb->dthpb_xargc;
1969 pp->ftp_xtypes = dhpb->dthpb_xtypes;
1970 pp->ftp_ntypes = dhpb->dthpb_ntypes;
1971
1972 /*
1973 * First create a tracepoint for each actual point of interest.
1974 */
1975 for (i = 0; i < dhpb->dthpb_noffs; i++) {
1976 tp = kmem_zalloc(sizeof (fasttrap_tracepoint_t), KM_SLEEP);
1977
1978 tp->ftt_proc = provider->ftp_proc;
1979 tp->ftt_pc = dhpb->dthpb_base + dhpb->dthpb_offs[i];
1980 tp->ftt_pid = provider->ftp_pid;
1981
1982 pp->ftp_tps[i].fit_tp = tp;
1983 pp->ftp_tps[i].fit_id.fti_probe = pp;
1984#ifdef __sparc
1985 pp->ftp_tps[i].fit_id.fti_ptype = DTFTP_POST_OFFSETS;
1986#else
1987 pp->ftp_tps[i].fit_id.fti_ptype = DTFTP_OFFSETS;
1988#endif
1989 }
1990
1991 /*
1992 * Then create a tracepoint for each is-enabled point.
1993 */
1994 for (j = 0; i < ntps; i++, j++) {
1995 tp = kmem_zalloc(sizeof (fasttrap_tracepoint_t), KM_SLEEP);
1996
1997 tp->ftt_proc = provider->ftp_proc;
1998 tp->ftt_pc = dhpb->dthpb_base + dhpb->dthpb_enoffs[j];
1999 tp->ftt_pid = provider->ftp_pid;
2000
2001 pp->ftp_tps[i].fit_tp = tp;
2002 pp->ftp_tps[i].fit_id.fti_probe = pp;
2003 pp->ftp_tps[i].fit_id.fti_ptype = DTFTP_IS_ENABLED;
2004 }
2005
2006 /*
2007 * If the arguments are shuffled around we set the argument remapping
2008 * table. Later, when the probe fires, we only remap the arguments
2009 * if the table is non-NULL.
2010 */
2011 for (i = 0; i < dhpb->dthpb_xargc; i++) {
2012 if (dhpb->dthpb_args[i] != i) {
2013 pp->ftp_argmap = dhpb->dthpb_args;
2014 break;
2015 }
2016 }
2017
2018 /*
2019 * The probe is fully constructed -- register it with DTrace.
2020 */
2021 pp->ftp_id = dtrace_probe_create(provider->ftp_provid, dhpb->dthpb_mod,
2022 dhpb->dthpb_func, dhpb->dthpb_name, FASTTRAP_OFFSET_AFRAMES, pp);
2023
2024 mutex_exit(&provider->ftp_cmtx);
2025}
2026
2027/*ARGSUSED*/
2028static void
2029fasttrap_meta_remove(void *arg, dtrace_helper_provdesc_t *dhpv, pid_t pid)
2030{
2031 /*
2032 * Clean up the USDT provider. There may be active consumers of the
2033 * provider busy adding probes, no damage will actually befall the
2034 * provider until that count has dropped to zero. This just puts
2035 * the provider on death row.
2036 */
2037 fasttrap_provider_retire(pid, dhpv->dthpv_provname, 1);
2038}
2039
2040static dtrace_mops_t fasttrap_mops = {
2041 fasttrap_meta_create_probe,
2042 fasttrap_meta_provide,
2043 fasttrap_meta_remove
2044};
2045
2046/*ARGSUSED*/
2047static int
2048fasttrap_open(struct cdev *dev __unused, int oflags __unused,
2049 int devtype __unused, struct thread *td __unused)
2050{
2051 return (0);
2052}
2053
2054/*ARGSUSED*/
2055static int
2056fasttrap_ioctl(struct cdev *dev, u_long cmd, caddr_t arg, int fflag,
2057 struct thread *td)
2058{
2059#ifdef notyet
2060 struct kinfo_proc kp;
2061 const cred_t *cr = td->td_ucred;
2062#endif
2063 if (!dtrace_attached())
2064 return (EAGAIN);
2065
2066 if (cmd == FASTTRAPIOC_MAKEPROBE) {
2067 fasttrap_probe_spec_t *uprobe = *(fasttrap_probe_spec_t **)arg;
2068 fasttrap_probe_spec_t *probe;
2069 uint64_t noffs;
2070 size_t size;
2071 int ret;
2072 char *c;
2073
2074 if (copyin(&uprobe->ftps_noffs, &noffs,
2075 sizeof (uprobe->ftps_noffs)))
2076 return (EFAULT);
2077
2078 /*
2079 * Probes must have at least one tracepoint.
2080 */
2081 if (noffs == 0)
2082 return (EINVAL);
2083
2084 size = sizeof (fasttrap_probe_spec_t) +
2085 sizeof (probe->ftps_offs[0]) * (noffs - 1);
2086
2087 if (size > 1024 * 1024)
2088 return (ENOMEM);
2089
2090 probe = kmem_alloc(size, KM_SLEEP);
2091
2092 if (copyin(uprobe, probe, size) != 0) {
2093 kmem_free(probe, size);
2094 return (EFAULT);
2095 }
2096
2097 /*
2098 * Verify that the function and module strings contain no
2099 * funny characters.
2100 */
2101 for (c = &probe->ftps_func[0]; *c != '\0'; c++) {
2102 if (*c < 0x20 || 0x7f <= *c) {
2103 ret = EINVAL;
2104 goto err;
2105 }
2106 }
2107
2108 for (c = &probe->ftps_mod[0]; *c != '\0'; c++) {
2109 if (*c < 0x20 || 0x7f <= *c) {
2110 ret = EINVAL;
2111 goto err;
2112 }
2113 }
2114
2115#ifdef notyet
2116 if (!PRIV_POLICY_CHOICE(cr, PRIV_ALL, B_FALSE)) {
2117 proc_t *p;
2118 pid_t pid = probe->ftps_pid;
2119
2120#if defined(sun)
2121 mutex_enter(&pidlock);
2122#endif
2123 /*
2124 * Report an error if the process doesn't exist
2125 * or is actively being birthed.
2126 */
2127 p = pfind(pid);
2128 if (p)
2129 fill_kinfo_proc(p, &kp);
2130 if (p == NULL || kp.ki_stat == SIDL) {
2131#if defined(sun)
2132 mutex_exit(&pidlock);
2133#endif
2134 return (ESRCH);
2135 }
2136#if defined(sun)
2137 mutex_enter(&p->p_lock);
2138 mutex_exit(&pidlock);
2139#else
2140 PROC_LOCK_ASSERT(p, MA_OWNED);
2141#endif
2142
2143#ifdef notyet
2144 if ((ret = priv_proc_cred_perm(cr, p, NULL,
2145 VREAD | VWRITE)) != 0) {
2146#if defined(sun)
2147 mutex_exit(&p->p_lock);
2148#else
2149 PROC_UNLOCK(p);
2150#endif
2151 return (ret);
2152 }
2153#endif /* notyet */
2154#if defined(sun)
2155 mutex_exit(&p->p_lock);
2156#else
2157 PROC_UNLOCK(p);
2158#endif
2159 }
2160#endif /* notyet */
2161
2162 ret = fasttrap_add_probe(probe);
2163err:
2164 kmem_free(probe, size);
2165
2166 return (ret);
2167
2168 } else if (cmd == FASTTRAPIOC_GETINSTR) {
2169 fasttrap_instr_query_t instr;
2170 fasttrap_tracepoint_t *tp;
2171 uint_t index;
2172#if defined(sun)
2173 int ret;
2174#endif
2175
2176#if defined(sun)
2177 if (copyin((void *)arg, &instr, sizeof (instr)) != 0)
2178 return (EFAULT);
2179#endif
2180
2181#ifdef notyet
2182 if (!PRIV_POLICY_CHOICE(cr, PRIV_ALL, B_FALSE)) {
2183 proc_t *p;
2184 pid_t pid = instr.ftiq_pid;
2185
2186#if defined(sun)
2187 mutex_enter(&pidlock);
2188#endif
2189 /*
2190 * Report an error if the process doesn't exist
2191 * or is actively being birthed.
2192 */
2193 p = pfind(pid);
2194 if (p)
2195 fill_kinfo_proc(p, &kp);
2196 if (p == NULL || kp.ki_stat == SIDL) {
2197#if defined(sun)
2198 mutex_exit(&pidlock);
2199#endif
2200 return (ESRCH);
2201 }
2202#if defined(sun)
2203 mutex_enter(&p->p_lock);
2204 mutex_exit(&pidlock);
2205#else
2206 PROC_LOCK_ASSERT(p, MA_OWNED);
2207#endif
2208
2209#ifdef notyet
2210 if ((ret = priv_proc_cred_perm(cr, p, NULL,
2211 VREAD)) != 0) {
2212#if defined(sun)
2213 mutex_exit(&p->p_lock);
2214#else
2215 PROC_UNLOCK(p);
2216#endif
2217 return (ret);
2218 }
2219#endif /* notyet */
2220
2221#if defined(sun)
2222 mutex_exit(&p->p_lock);
2223#else
2224 PROC_UNLOCK(p);
2225#endif
2226 }
2227#endif /* notyet */
2228
2229 index = FASTTRAP_TPOINTS_INDEX(instr.ftiq_pid, instr.ftiq_pc);
2230
2231 mutex_enter(&fasttrap_tpoints.fth_table[index].ftb_mtx);
2232 tp = fasttrap_tpoints.fth_table[index].ftb_data;
2233 while (tp != NULL) {
2234 if (instr.ftiq_pid == tp->ftt_pid &&
2235 instr.ftiq_pc == tp->ftt_pc &&
2236 tp->ftt_proc->ftpc_acount != 0)
2237 break;
2238
2239 tp = tp->ftt_next;
2240 }
2241
2242 if (tp == NULL) {
2243 mutex_exit(&fasttrap_tpoints.fth_table[index].ftb_mtx);
2244 return (ENOENT);
2245 }
2246
2247 bcopy(&tp->ftt_instr, &instr.ftiq_instr,
2248 sizeof (instr.ftiq_instr));
2249 mutex_exit(&fasttrap_tpoints.fth_table[index].ftb_mtx);
2250
2251 if (copyout(&instr, (void *)arg, sizeof (instr)) != 0)
2252 return (EFAULT);
2253
2254 return (0);
2255 }
2256
2257 return (EINVAL);
2258}
2259
2260static int
2261fasttrap_load(void)
2262{
2263 ulong_t nent;
2264 int i, ret;
2265
2266 /* Create the /dev/dtrace/fasttrap entry. */
2267 fasttrap_cdev = make_dev(&fasttrap_cdevsw, 0, UID_ROOT, GID_WHEEL, 0600,
2268 "dtrace/fasttrap");
2269
2270 mtx_init(&fasttrap_cleanup_mtx, "fasttrap clean", "dtrace", MTX_DEF);
2271 mutex_init(&fasttrap_count_mtx, "fasttrap count mtx", MUTEX_DEFAULT,
2272 NULL);
2273
2274#if defined(sun)
2275 fasttrap_max = ddi_getprop(DDI_DEV_T_ANY, devi, DDI_PROP_DONTPASS,
2276 "fasttrap-max-probes", FASTTRAP_MAX_DEFAULT);
2277#else
2278 fasttrap_max = FASTTRAP_MAX_DEFAULT;
2279#endif
2280 fasttrap_total = 0;
2281
2282 /*
2283 * Conjure up the tracepoints hashtable...
2284 */
2285#if defined(sun)
2286 nent = ddi_getprop(DDI_DEV_T_ANY, devi, DDI_PROP_DONTPASS,
2287 "fasttrap-hash-size", FASTTRAP_TPOINTS_DEFAULT_SIZE);
2288#else
2289 nent = FASTTRAP_TPOINTS_DEFAULT_SIZE;
2290#endif
2291
2292 if (nent == 0 || nent > 0x1000000)
2293 nent = FASTTRAP_TPOINTS_DEFAULT_SIZE;
2294
2295 if ((nent & (nent - 1)) == 0)
2296 fasttrap_tpoints.fth_nent = nent;
2297 else
2298 fasttrap_tpoints.fth_nent = 1 << fasttrap_highbit(nent);
2299 ASSERT(fasttrap_tpoints.fth_nent > 0);
2300 fasttrap_tpoints.fth_mask = fasttrap_tpoints.fth_nent - 1;
2301 fasttrap_tpoints.fth_table = kmem_zalloc(fasttrap_tpoints.fth_nent *
2302 sizeof (fasttrap_bucket_t), KM_SLEEP);
2303#if !defined(sun)
2304 for (i = 0; i < fasttrap_tpoints.fth_nent; i++)
2305 mutex_init(&fasttrap_tpoints.fth_table[i].ftb_mtx,
2306 "tracepoints bucket mtx", MUTEX_DEFAULT, NULL);
2307#endif
2308
2309 /*
2310 * ... and the providers hash table...
2311 */
2312 nent = FASTTRAP_PROVIDERS_DEFAULT_SIZE;
2313 if ((nent & (nent - 1)) == 0)
2314 fasttrap_provs.fth_nent = nent;
2315 else
2316 fasttrap_provs.fth_nent = 1 << fasttrap_highbit(nent);
2317 ASSERT(fasttrap_provs.fth_nent > 0);
2318 fasttrap_provs.fth_mask = fasttrap_provs.fth_nent - 1;
2319 fasttrap_provs.fth_table = kmem_zalloc(fasttrap_provs.fth_nent *
2320 sizeof (fasttrap_bucket_t), KM_SLEEP);
2321#if !defined(sun)
2322 for (i = 0; i < fasttrap_provs.fth_nent; i++)
2323 mutex_init(&fasttrap_provs.fth_table[i].ftb_mtx,
2324 "providers bucket mtx", MUTEX_DEFAULT, NULL);
2325#endif
2326
2327 ret = kproc_create(fasttrap_pid_cleanup_cb, NULL,
2328 &fasttrap_cleanup_proc, 0, 0, "ftcleanup");
2329 if (ret != 0) {
2330 destroy_dev(fasttrap_cdev);
2331#if !defined(sun)
2332 for (i = 0; i < fasttrap_provs.fth_nent; i++)
2333 mutex_destroy(&fasttrap_provs.fth_table[i].ftb_mtx);
2334 for (i = 0; i < fasttrap_tpoints.fth_nent; i++)
2335 mutex_destroy(&fasttrap_tpoints.fth_table[i].ftb_mtx);
2336#endif
2337 kmem_free(fasttrap_provs.fth_table, fasttrap_provs.fth_nent *
2338 sizeof (fasttrap_bucket_t));
2339 mtx_destroy(&fasttrap_cleanup_mtx);
2340 mutex_destroy(&fasttrap_count_mtx);
2341 return (ret);
2342 }
2343
2344
2345 /*
2346 * ... and the procs hash table.
2347 */
2348 nent = FASTTRAP_PROCS_DEFAULT_SIZE;
2349 if ((nent & (nent - 1)) == 0)
2350 fasttrap_procs.fth_nent = nent;
2351 else
2352 fasttrap_procs.fth_nent = 1 << fasttrap_highbit(nent);
2353 ASSERT(fasttrap_procs.fth_nent > 0);
2354 fasttrap_procs.fth_mask = fasttrap_procs.fth_nent - 1;
2355 fasttrap_procs.fth_table = kmem_zalloc(fasttrap_procs.fth_nent *
2356 sizeof (fasttrap_bucket_t), KM_SLEEP);
2357#if !defined(sun)
2358 for (i = 0; i < fasttrap_procs.fth_nent; i++)
2359 mutex_init(&fasttrap_procs.fth_table[i].ftb_mtx,
2360 "processes bucket mtx", MUTEX_DEFAULT, NULL);
2361
2362 CPU_FOREACH(i) {
2363 mutex_init(&fasttrap_cpuc_pid_lock[i], "fasttrap barrier",
2364 MUTEX_DEFAULT, NULL);
2365 }
| 1569 mutex_exit(&proc->ftpc_mtx);
1570
1571 /*
1572 * There should definitely be no live providers associated with this
1573 * process at this point.
1574 */
1575 ASSERT(proc->ftpc_acount == 0);
1576
1577 bucket = &fasttrap_procs.fth_table[FASTTRAP_PROCS_INDEX(pid)];
1578 mutex_enter(&bucket->ftb_mtx);
1579
1580 fprcp = (fasttrap_proc_t **)&bucket->ftb_data;
1581 while ((fprc = *fprcp) != NULL) {
1582 if (fprc == proc)
1583 break;
1584
1585 fprcp = &fprc->ftpc_next;
1586 }
1587
1588 /*
1589 * Something strange has happened if we can't find the proc.
1590 */
1591 ASSERT(fprc != NULL);
1592
1593 *fprcp = fprc->ftpc_next;
1594
1595 mutex_exit(&bucket->ftb_mtx);
1596
1597 kmem_free(fprc, sizeof (fasttrap_proc_t));
1598}
1599
1600/*
1601 * Lookup a fasttrap-managed provider based on its name and associated pid.
1602 * If the pattr argument is non-NULL, this function instantiates the provider
1603 * if it doesn't exist otherwise it returns NULL. The provider is returned
1604 * with its lock held.
1605 */
1606static fasttrap_provider_t *
1607fasttrap_provider_lookup(pid_t pid, const char *name,
1608 const dtrace_pattr_t *pattr)
1609{
1610 fasttrap_provider_t *fp, *new_fp = NULL;
1611 fasttrap_bucket_t *bucket;
1612 char provname[DTRACE_PROVNAMELEN];
1613 proc_t *p;
1614 cred_t *cred;
1615
1616 ASSERT(strlen(name) < sizeof (fp->ftp_name));
1617 ASSERT(pattr != NULL);
1618
1619 bucket = &fasttrap_provs.fth_table[FASTTRAP_PROVS_INDEX(pid, name)];
1620 mutex_enter(&bucket->ftb_mtx);
1621
1622 /*
1623 * Take a lap through the list and return the match if we find it.
1624 */
1625 for (fp = bucket->ftb_data; fp != NULL; fp = fp->ftp_next) {
1626 if (fp->ftp_pid == pid && strcmp(fp->ftp_name, name) == 0 &&
1627 !fp->ftp_retired) {
1628 mutex_enter(&fp->ftp_mtx);
1629 mutex_exit(&bucket->ftb_mtx);
1630 return (fp);
1631 }
1632 }
1633
1634 /*
1635 * Drop the bucket lock so we don't try to perform a sleeping
1636 * allocation under it.
1637 */
1638 mutex_exit(&bucket->ftb_mtx);
1639
1640 /*
1641 * Make sure the process exists, isn't a child created as the result
1642 * of a vfork(2), and isn't a zombie (but may be in fork).
1643 */
1644 if ((p = pfind(pid)) == NULL)
1645 return (NULL);
1646
1647 /*
1648 * Increment p_dtrace_probes so that the process knows to inform us
1649 * when it exits or execs. fasttrap_provider_free() decrements this
1650 * when we're done with this provider.
1651 */
1652 p->p_dtrace_probes++;
1653
1654 /*
1655 * Grab the credentials for this process so we have
1656 * something to pass to dtrace_register().
1657 */
1658 PROC_LOCK_ASSERT(p, MA_OWNED);
1659 crhold(p->p_ucred);
1660 cred = p->p_ucred;
1661 PROC_UNLOCK(p);
1662
1663 new_fp = kmem_zalloc(sizeof (fasttrap_provider_t), KM_SLEEP);
1664 new_fp->ftp_pid = pid;
1665 new_fp->ftp_proc = fasttrap_proc_lookup(pid);
1666#if !defined(sun)
1667 mutex_init(&new_fp->ftp_mtx, "provider mtx", MUTEX_DEFAULT, NULL);
1668 mutex_init(&new_fp->ftp_cmtx, "lock on creating", MUTEX_DEFAULT, NULL);
1669#endif
1670
1671 ASSERT(new_fp->ftp_proc != NULL);
1672
1673 mutex_enter(&bucket->ftb_mtx);
1674
1675 /*
1676 * Take another lap through the list to make sure a provider hasn't
1677 * been created for this pid while we weren't under the bucket lock.
1678 */
1679 for (fp = bucket->ftb_data; fp != NULL; fp = fp->ftp_next) {
1680 if (fp->ftp_pid == pid && strcmp(fp->ftp_name, name) == 0 &&
1681 !fp->ftp_retired) {
1682 mutex_enter(&fp->ftp_mtx);
1683 mutex_exit(&bucket->ftb_mtx);
1684 fasttrap_provider_free(new_fp);
1685 crfree(cred);
1686 return (fp);
1687 }
1688 }
1689
1690 (void) strcpy(new_fp->ftp_name, name);
1691
1692 /*
1693 * Fail and return NULL if either the provider name is too long
1694 * or we fail to register this new provider with the DTrace
1695 * framework. Note that this is the only place we ever construct
1696 * the full provider name -- we keep it in pieces in the provider
1697 * structure.
1698 */
1699 if (snprintf(provname, sizeof (provname), "%s%u", name, (uint_t)pid) >=
1700 sizeof (provname) ||
1701 dtrace_register(provname, pattr,
1702 DTRACE_PRIV_PROC | DTRACE_PRIV_OWNER | DTRACE_PRIV_ZONEOWNER, cred,
1703 pattr == &pid_attr ? &pid_pops : &usdt_pops, new_fp,
1704 &new_fp->ftp_provid) != 0) {
1705 mutex_exit(&bucket->ftb_mtx);
1706 fasttrap_provider_free(new_fp);
1707 crfree(cred);
1708 return (NULL);
1709 }
1710
1711 new_fp->ftp_next = bucket->ftb_data;
1712 bucket->ftb_data = new_fp;
1713
1714 mutex_enter(&new_fp->ftp_mtx);
1715 mutex_exit(&bucket->ftb_mtx);
1716
1717 crfree(cred);
1718 return (new_fp);
1719}
1720
1721static void
1722fasttrap_provider_free(fasttrap_provider_t *provider)
1723{
1724 pid_t pid = provider->ftp_pid;
1725 proc_t *p;
1726
1727 /*
1728 * There need to be no associated enabled probes, no consumers
1729 * creating probes, and no meta providers referencing this provider.
1730 */
1731 ASSERT(provider->ftp_rcount == 0);
1732 ASSERT(provider->ftp_ccount == 0);
1733 ASSERT(provider->ftp_mcount == 0);
1734
1735 /*
1736 * If this provider hasn't been retired, we need to explicitly drop the
1737 * count of active providers on the associated process structure.
1738 */
1739 if (!provider->ftp_retired) {
1740 atomic_add_64(&provider->ftp_proc->ftpc_acount, -1);
1741 ASSERT(provider->ftp_proc->ftpc_acount <
1742 provider->ftp_proc->ftpc_rcount);
1743 }
1744
1745 fasttrap_proc_release(provider->ftp_proc);
1746
1747#if !defined(sun)
1748 mutex_destroy(&provider->ftp_mtx);
1749 mutex_destroy(&provider->ftp_cmtx);
1750#endif
1751 kmem_free(provider, sizeof (fasttrap_provider_t));
1752
1753 /*
1754 * Decrement p_dtrace_probes on the process whose provider we're
1755 * freeing. We don't have to worry about clobbering somone else's
1756 * modifications to it because we have locked the bucket that
1757 * corresponds to this process's hash chain in the provider hash
1758 * table. Don't sweat it if we can't find the process.
1759 */
1760 if ((p = pfind(pid)) == NULL) {
1761 return;
1762 }
1763
1764 p->p_dtrace_probes--;
1765#if !defined(sun)
1766 PROC_UNLOCK(p);
1767#endif
1768}
1769
1770static void
1771fasttrap_provider_retire(pid_t pid, const char *name, int mprov)
1772{
1773 fasttrap_provider_t *fp;
1774 fasttrap_bucket_t *bucket;
1775 dtrace_provider_id_t provid;
1776
1777 ASSERT(strlen(name) < sizeof (fp->ftp_name));
1778
1779 bucket = &fasttrap_provs.fth_table[FASTTRAP_PROVS_INDEX(pid, name)];
1780 mutex_enter(&bucket->ftb_mtx);
1781
1782 for (fp = bucket->ftb_data; fp != NULL; fp = fp->ftp_next) {
1783 if (fp->ftp_pid == pid && strcmp(fp->ftp_name, name) == 0 &&
1784 !fp->ftp_retired)
1785 break;
1786 }
1787
1788 if (fp == NULL) {
1789 mutex_exit(&bucket->ftb_mtx);
1790 return;
1791 }
1792
1793 mutex_enter(&fp->ftp_mtx);
1794 ASSERT(!mprov || fp->ftp_mcount > 0);
1795 if (mprov && --fp->ftp_mcount != 0) {
1796 mutex_exit(&fp->ftp_mtx);
1797 mutex_exit(&bucket->ftb_mtx);
1798 return;
1799 }
1800
1801 /*
1802 * Mark the provider to be removed in our post-processing step, mark it
1803 * retired, and drop the active count on its proc. Marking it indicates
1804 * that we should try to remove it; setting the retired flag indicates
1805 * that we're done with this provider; dropping the active the proc
1806 * releases our hold, and when this reaches zero (as it will during
1807 * exit or exec) the proc and associated providers become defunct.
1808 *
1809 * We obviously need to take the bucket lock before the provider lock
1810 * to perform the lookup, but we need to drop the provider lock
1811 * before calling into the DTrace framework since we acquire the
1812 * provider lock in callbacks invoked from the DTrace framework. The
1813 * bucket lock therefore protects the integrity of the provider hash
1814 * table.
1815 */
1816 atomic_add_64(&fp->ftp_proc->ftpc_acount, -1);
1817 ASSERT(fp->ftp_proc->ftpc_acount < fp->ftp_proc->ftpc_rcount);
1818
1819 fp->ftp_retired = 1;
1820 fp->ftp_marked = 1;
1821 provid = fp->ftp_provid;
1822 mutex_exit(&fp->ftp_mtx);
1823
1824 /*
1825 * We don't have to worry about invalidating the same provider twice
1826 * since fasttrap_provider_lookup() will ignore provider that have
1827 * been marked as retired.
1828 */
1829 dtrace_invalidate(provid);
1830
1831 mutex_exit(&bucket->ftb_mtx);
1832
1833 fasttrap_pid_cleanup();
1834}
1835
1836static int
1837fasttrap_uint32_cmp(const void *ap, const void *bp)
1838{
1839 return (*(const uint32_t *)ap - *(const uint32_t *)bp);
1840}
1841
1842static int
1843fasttrap_uint64_cmp(const void *ap, const void *bp)
1844{
1845 return (*(const uint64_t *)ap - *(const uint64_t *)bp);
1846}
1847
1848static int
1849fasttrap_add_probe(fasttrap_probe_spec_t *pdata)
1850{
1851 fasttrap_provider_t *provider;
1852 fasttrap_probe_t *pp;
1853 fasttrap_tracepoint_t *tp;
1854 char *name;
1855 int i, aframes = 0, whack;
1856
1857 /*
1858 * There needs to be at least one desired trace point.
1859 */
1860 if (pdata->ftps_noffs == 0)
1861 return (EINVAL);
1862
1863 switch (pdata->ftps_type) {
1864 case DTFTP_ENTRY:
1865 name = "entry";
1866 aframes = FASTTRAP_ENTRY_AFRAMES;
1867 break;
1868 case DTFTP_RETURN:
1869 name = "return";
1870 aframes = FASTTRAP_RETURN_AFRAMES;
1871 break;
1872 case DTFTP_OFFSETS:
1873 name = NULL;
1874 break;
1875 default:
1876 return (EINVAL);
1877 }
1878
1879 if ((provider = fasttrap_provider_lookup(pdata->ftps_pid,
1880 FASTTRAP_PID_NAME, &pid_attr)) == NULL)
1881 return (ESRCH);
1882
1883 /*
1884 * Increment this reference count to indicate that a consumer is
1885 * actively adding a new probe associated with this provider. This
1886 * prevents the provider from being deleted -- we'll need to check
1887 * for pending deletions when we drop this reference count.
1888 */
1889 provider->ftp_ccount++;
1890 mutex_exit(&provider->ftp_mtx);
1891
1892 /*
1893 * Grab the creation lock to ensure consistency between calls to
1894 * dtrace_probe_lookup() and dtrace_probe_create() in the face of
1895 * other threads creating probes. We must drop the provider lock
1896 * before taking this lock to avoid a three-way deadlock with the
1897 * DTrace framework.
1898 */
1899 mutex_enter(&provider->ftp_cmtx);
1900
1901 if (name == NULL) {
1902 for (i = 0; i < pdata->ftps_noffs; i++) {
1903 char name_str[17];
1904
1905 (void) sprintf(name_str, "%llx",
1906 (unsigned long long)pdata->ftps_offs[i]);
1907
1908 if (dtrace_probe_lookup(provider->ftp_provid,
1909 pdata->ftps_mod, pdata->ftps_func, name_str) != 0)
1910 continue;
1911
1912 atomic_add_32(&fasttrap_total, 1);
1913
1914 if (fasttrap_total > fasttrap_max) {
1915 atomic_add_32(&fasttrap_total, -1);
1916 goto no_mem;
1917 }
1918
1919 pp = kmem_zalloc(sizeof (fasttrap_probe_t), KM_SLEEP);
1920
1921 pp->ftp_prov = provider;
1922 pp->ftp_faddr = pdata->ftps_pc;
1923 pp->ftp_fsize = pdata->ftps_size;
1924 pp->ftp_pid = pdata->ftps_pid;
1925 pp->ftp_ntps = 1;
1926
1927 tp = kmem_zalloc(sizeof (fasttrap_tracepoint_t),
1928 KM_SLEEP);
1929
1930 tp->ftt_proc = provider->ftp_proc;
1931 tp->ftt_pc = pdata->ftps_offs[i] + pdata->ftps_pc;
1932 tp->ftt_pid = pdata->ftps_pid;
1933
1934 pp->ftp_tps[0].fit_tp = tp;
1935 pp->ftp_tps[0].fit_id.fti_probe = pp;
1936 pp->ftp_tps[0].fit_id.fti_ptype = pdata->ftps_type;
1937
1938 pp->ftp_id = dtrace_probe_create(provider->ftp_provid,
1939 pdata->ftps_mod, pdata->ftps_func, name_str,
1940 FASTTRAP_OFFSET_AFRAMES, pp);
1941 }
1942
1943 } else if (dtrace_probe_lookup(provider->ftp_provid, pdata->ftps_mod,
1944 pdata->ftps_func, name) == 0) {
1945 atomic_add_32(&fasttrap_total, pdata->ftps_noffs);
1946
1947 if (fasttrap_total > fasttrap_max) {
1948 atomic_add_32(&fasttrap_total, -pdata->ftps_noffs);
1949 goto no_mem;
1950 }
1951
1952 /*
1953 * Make sure all tracepoint program counter values are unique.
1954 * We later assume that each probe has exactly one tracepoint
1955 * for a given pc.
1956 */
1957 qsort(pdata->ftps_offs, pdata->ftps_noffs,
1958 sizeof (uint64_t), fasttrap_uint64_cmp);
1959 for (i = 1; i < pdata->ftps_noffs; i++) {
1960 if (pdata->ftps_offs[i] > pdata->ftps_offs[i - 1])
1961 continue;
1962
1963 atomic_add_32(&fasttrap_total, -pdata->ftps_noffs);
1964 goto no_mem;
1965 }
1966
1967 ASSERT(pdata->ftps_noffs > 0);
1968 pp = kmem_zalloc(offsetof(fasttrap_probe_t,
1969 ftp_tps[pdata->ftps_noffs]), KM_SLEEP);
1970
1971 pp->ftp_prov = provider;
1972 pp->ftp_faddr = pdata->ftps_pc;
1973 pp->ftp_fsize = pdata->ftps_size;
1974 pp->ftp_pid = pdata->ftps_pid;
1975 pp->ftp_ntps = pdata->ftps_noffs;
1976
1977 for (i = 0; i < pdata->ftps_noffs; i++) {
1978 tp = kmem_zalloc(sizeof (fasttrap_tracepoint_t),
1979 KM_SLEEP);
1980
1981 tp->ftt_proc = provider->ftp_proc;
1982 tp->ftt_pc = pdata->ftps_offs[i] + pdata->ftps_pc;
1983 tp->ftt_pid = pdata->ftps_pid;
1984
1985 pp->ftp_tps[i].fit_tp = tp;
1986 pp->ftp_tps[i].fit_id.fti_probe = pp;
1987 pp->ftp_tps[i].fit_id.fti_ptype = pdata->ftps_type;
1988 }
1989
1990 pp->ftp_id = dtrace_probe_create(provider->ftp_provid,
1991 pdata->ftps_mod, pdata->ftps_func, name, aframes, pp);
1992 }
1993
1994 mutex_exit(&provider->ftp_cmtx);
1995
1996 /*
1997 * We know that the provider is still valid since we incremented the
1998 * creation reference count. If someone tried to clean up this provider
1999 * while we were using it (e.g. because the process called exec(2) or
2000 * exit(2)), take note of that and try to clean it up now.
2001 */
2002 mutex_enter(&provider->ftp_mtx);
2003 provider->ftp_ccount--;
2004 whack = provider->ftp_retired;
2005 mutex_exit(&provider->ftp_mtx);
2006
2007 if (whack)
2008 fasttrap_pid_cleanup();
2009
2010 return (0);
2011
2012no_mem:
2013 /*
2014 * If we've exhausted the allowable resources, we'll try to remove
2015 * this provider to free some up. This is to cover the case where
2016 * the user has accidentally created many more probes than was
2017 * intended (e.g. pid123:::).
2018 */
2019 mutex_exit(&provider->ftp_cmtx);
2020 mutex_enter(&provider->ftp_mtx);
2021 provider->ftp_ccount--;
2022 provider->ftp_marked = 1;
2023 mutex_exit(&provider->ftp_mtx);
2024
2025 fasttrap_pid_cleanup();
2026
2027 return (ENOMEM);
2028}
2029
2030/*ARGSUSED*/
2031static void *
2032fasttrap_meta_provide(void *arg, dtrace_helper_provdesc_t *dhpv, pid_t pid)
2033{
2034 fasttrap_provider_t *provider;
2035
2036 /*
2037 * A 32-bit unsigned integer (like a pid for example) can be
2038 * expressed in 10 or fewer decimal digits. Make sure that we'll
2039 * have enough space for the provider name.
2040 */
2041 if (strlen(dhpv->dthpv_provname) + 10 >=
2042 sizeof (provider->ftp_name)) {
2043 printf("failed to instantiate provider %s: "
2044 "name too long to accomodate pid", dhpv->dthpv_provname);
2045 return (NULL);
2046 }
2047
2048 /*
2049 * Don't let folks spoof the true pid provider.
2050 */
2051 if (strcmp(dhpv->dthpv_provname, FASTTRAP_PID_NAME) == 0) {
2052 printf("failed to instantiate provider %s: "
2053 "%s is an invalid name", dhpv->dthpv_provname,
2054 FASTTRAP_PID_NAME);
2055 return (NULL);
2056 }
2057
2058 /*
2059 * The highest stability class that fasttrap supports is ISA; cap
2060 * the stability of the new provider accordingly.
2061 */
2062 if (dhpv->dthpv_pattr.dtpa_provider.dtat_class > DTRACE_CLASS_ISA)
2063 dhpv->dthpv_pattr.dtpa_provider.dtat_class = DTRACE_CLASS_ISA;
2064 if (dhpv->dthpv_pattr.dtpa_mod.dtat_class > DTRACE_CLASS_ISA)
2065 dhpv->dthpv_pattr.dtpa_mod.dtat_class = DTRACE_CLASS_ISA;
2066 if (dhpv->dthpv_pattr.dtpa_func.dtat_class > DTRACE_CLASS_ISA)
2067 dhpv->dthpv_pattr.dtpa_func.dtat_class = DTRACE_CLASS_ISA;
2068 if (dhpv->dthpv_pattr.dtpa_name.dtat_class > DTRACE_CLASS_ISA)
2069 dhpv->dthpv_pattr.dtpa_name.dtat_class = DTRACE_CLASS_ISA;
2070 if (dhpv->dthpv_pattr.dtpa_args.dtat_class > DTRACE_CLASS_ISA)
2071 dhpv->dthpv_pattr.dtpa_args.dtat_class = DTRACE_CLASS_ISA;
2072
2073 if ((provider = fasttrap_provider_lookup(pid, dhpv->dthpv_provname,
2074 &dhpv->dthpv_pattr)) == NULL) {
2075 printf("failed to instantiate provider %s for "
2076 "process %u", dhpv->dthpv_provname, (uint_t)pid);
2077 return (NULL);
2078 }
2079
2080 /*
2081 * Up the meta provider count so this provider isn't removed until
2082 * the meta provider has been told to remove it.
2083 */
2084 provider->ftp_mcount++;
2085
2086 mutex_exit(&provider->ftp_mtx);
2087
2088 return (provider);
2089}
2090
2091/*ARGSUSED*/
2092static void
2093fasttrap_meta_create_probe(void *arg, void *parg,
2094 dtrace_helper_probedesc_t *dhpb)
2095{
2096 fasttrap_provider_t *provider = parg;
2097 fasttrap_probe_t *pp;
2098 fasttrap_tracepoint_t *tp;
2099 int i, j;
2100 uint32_t ntps;
2101
2102 /*
2103 * Since the meta provider count is non-zero we don't have to worry
2104 * about this provider disappearing.
2105 */
2106 ASSERT(provider->ftp_mcount > 0);
2107
2108 /*
2109 * The offsets must be unique.
2110 */
2111 qsort(dhpb->dthpb_offs, dhpb->dthpb_noffs, sizeof (uint32_t),
2112 fasttrap_uint32_cmp);
2113 for (i = 1; i < dhpb->dthpb_noffs; i++) {
2114 if (dhpb->dthpb_base + dhpb->dthpb_offs[i] <=
2115 dhpb->dthpb_base + dhpb->dthpb_offs[i - 1])
2116 return;
2117 }
2118
2119 qsort(dhpb->dthpb_enoffs, dhpb->dthpb_nenoffs, sizeof (uint32_t),
2120 fasttrap_uint32_cmp);
2121 for (i = 1; i < dhpb->dthpb_nenoffs; i++) {
2122 if (dhpb->dthpb_base + dhpb->dthpb_enoffs[i] <=
2123 dhpb->dthpb_base + dhpb->dthpb_enoffs[i - 1])
2124 return;
2125 }
2126
2127 /*
2128 * Grab the creation lock to ensure consistency between calls to
2129 * dtrace_probe_lookup() and dtrace_probe_create() in the face of
2130 * other threads creating probes.
2131 */
2132 mutex_enter(&provider->ftp_cmtx);
2133
2134 if (dtrace_probe_lookup(provider->ftp_provid, dhpb->dthpb_mod,
2135 dhpb->dthpb_func, dhpb->dthpb_name) != 0) {
2136 mutex_exit(&provider->ftp_cmtx);
2137 return;
2138 }
2139
2140 ntps = dhpb->dthpb_noffs + dhpb->dthpb_nenoffs;
2141 ASSERT(ntps > 0);
2142
2143 atomic_add_32(&fasttrap_total, ntps);
2144
2145 if (fasttrap_total > fasttrap_max) {
2146 atomic_add_32(&fasttrap_total, -ntps);
2147 mutex_exit(&provider->ftp_cmtx);
2148 return;
2149 }
2150
2151 pp = kmem_zalloc(offsetof(fasttrap_probe_t, ftp_tps[ntps]), KM_SLEEP);
2152
2153 pp->ftp_prov = provider;
2154 pp->ftp_pid = provider->ftp_pid;
2155 pp->ftp_ntps = ntps;
2156 pp->ftp_nargs = dhpb->dthpb_xargc;
2157 pp->ftp_xtypes = dhpb->dthpb_xtypes;
2158 pp->ftp_ntypes = dhpb->dthpb_ntypes;
2159
2160 /*
2161 * First create a tracepoint for each actual point of interest.
2162 */
2163 for (i = 0; i < dhpb->dthpb_noffs; i++) {
2164 tp = kmem_zalloc(sizeof (fasttrap_tracepoint_t), KM_SLEEP);
2165
2166 tp->ftt_proc = provider->ftp_proc;
2167 tp->ftt_pc = dhpb->dthpb_base + dhpb->dthpb_offs[i];
2168 tp->ftt_pid = provider->ftp_pid;
2169
2170 pp->ftp_tps[i].fit_tp = tp;
2171 pp->ftp_tps[i].fit_id.fti_probe = pp;
2172#ifdef __sparc
2173 pp->ftp_tps[i].fit_id.fti_ptype = DTFTP_POST_OFFSETS;
2174#else
2175 pp->ftp_tps[i].fit_id.fti_ptype = DTFTP_OFFSETS;
2176#endif
2177 }
2178
2179 /*
2180 * Then create a tracepoint for each is-enabled point.
2181 */
2182 for (j = 0; i < ntps; i++, j++) {
2183 tp = kmem_zalloc(sizeof (fasttrap_tracepoint_t), KM_SLEEP);
2184
2185 tp->ftt_proc = provider->ftp_proc;
2186 tp->ftt_pc = dhpb->dthpb_base + dhpb->dthpb_enoffs[j];
2187 tp->ftt_pid = provider->ftp_pid;
2188
2189 pp->ftp_tps[i].fit_tp = tp;
2190 pp->ftp_tps[i].fit_id.fti_probe = pp;
2191 pp->ftp_tps[i].fit_id.fti_ptype = DTFTP_IS_ENABLED;
2192 }
2193
2194 /*
2195 * If the arguments are shuffled around we set the argument remapping
2196 * table. Later, when the probe fires, we only remap the arguments
2197 * if the table is non-NULL.
2198 */
2199 for (i = 0; i < dhpb->dthpb_xargc; i++) {
2200 if (dhpb->dthpb_args[i] != i) {
2201 pp->ftp_argmap = dhpb->dthpb_args;
2202 break;
2203 }
2204 }
2205
2206 /*
2207 * The probe is fully constructed -- register it with DTrace.
2208 */
2209 pp->ftp_id = dtrace_probe_create(provider->ftp_provid, dhpb->dthpb_mod,
2210 dhpb->dthpb_func, dhpb->dthpb_name, FASTTRAP_OFFSET_AFRAMES, pp);
2211
2212 mutex_exit(&provider->ftp_cmtx);
2213}
2214
2215/*ARGSUSED*/
2216static void
2217fasttrap_meta_remove(void *arg, dtrace_helper_provdesc_t *dhpv, pid_t pid)
2218{
2219 /*
2220 * Clean up the USDT provider. There may be active consumers of the
2221 * provider busy adding probes, no damage will actually befall the
2222 * provider until that count has dropped to zero. This just puts
2223 * the provider on death row.
2224 */
2225 fasttrap_provider_retire(pid, dhpv->dthpv_provname, 1);
2226}
2227
2228static dtrace_mops_t fasttrap_mops = {
2229 fasttrap_meta_create_probe,
2230 fasttrap_meta_provide,
2231 fasttrap_meta_remove
2232};
2233
2234/*ARGSUSED*/
2235static int
2236fasttrap_open(struct cdev *dev __unused, int oflags __unused,
2237 int devtype __unused, struct thread *td __unused)
2238{
2239 return (0);
2240}
2241
2242/*ARGSUSED*/
2243static int
2244fasttrap_ioctl(struct cdev *dev, u_long cmd, caddr_t arg, int fflag,
2245 struct thread *td)
2246{
2247#ifdef notyet
2248 struct kinfo_proc kp;
2249 const cred_t *cr = td->td_ucred;
2250#endif
2251 if (!dtrace_attached())
2252 return (EAGAIN);
2253
2254 if (cmd == FASTTRAPIOC_MAKEPROBE) {
2255 fasttrap_probe_spec_t *uprobe = *(fasttrap_probe_spec_t **)arg;
2256 fasttrap_probe_spec_t *probe;
2257 uint64_t noffs;
2258 size_t size;
2259 int ret;
2260 char *c;
2261
2262 if (copyin(&uprobe->ftps_noffs, &noffs,
2263 sizeof (uprobe->ftps_noffs)))
2264 return (EFAULT);
2265
2266 /*
2267 * Probes must have at least one tracepoint.
2268 */
2269 if (noffs == 0)
2270 return (EINVAL);
2271
2272 size = sizeof (fasttrap_probe_spec_t) +
2273 sizeof (probe->ftps_offs[0]) * (noffs - 1);
2274
2275 if (size > 1024 * 1024)
2276 return (ENOMEM);
2277
2278 probe = kmem_alloc(size, KM_SLEEP);
2279
2280 if (copyin(uprobe, probe, size) != 0) {
2281 kmem_free(probe, size);
2282 return (EFAULT);
2283 }
2284
2285 /*
2286 * Verify that the function and module strings contain no
2287 * funny characters.
2288 */
2289 for (c = &probe->ftps_func[0]; *c != '\0'; c++) {
2290 if (*c < 0x20 || 0x7f <= *c) {
2291 ret = EINVAL;
2292 goto err;
2293 }
2294 }
2295
2296 for (c = &probe->ftps_mod[0]; *c != '\0'; c++) {
2297 if (*c < 0x20 || 0x7f <= *c) {
2298 ret = EINVAL;
2299 goto err;
2300 }
2301 }
2302
2303#ifdef notyet
2304 if (!PRIV_POLICY_CHOICE(cr, PRIV_ALL, B_FALSE)) {
2305 proc_t *p;
2306 pid_t pid = probe->ftps_pid;
2307
2308#if defined(sun)
2309 mutex_enter(&pidlock);
2310#endif
2311 /*
2312 * Report an error if the process doesn't exist
2313 * or is actively being birthed.
2314 */
2315 p = pfind(pid);
2316 if (p)
2317 fill_kinfo_proc(p, &kp);
2318 if (p == NULL || kp.ki_stat == SIDL) {
2319#if defined(sun)
2320 mutex_exit(&pidlock);
2321#endif
2322 return (ESRCH);
2323 }
2324#if defined(sun)
2325 mutex_enter(&p->p_lock);
2326 mutex_exit(&pidlock);
2327#else
2328 PROC_LOCK_ASSERT(p, MA_OWNED);
2329#endif
2330
2331#ifdef notyet
2332 if ((ret = priv_proc_cred_perm(cr, p, NULL,
2333 VREAD | VWRITE)) != 0) {
2334#if defined(sun)
2335 mutex_exit(&p->p_lock);
2336#else
2337 PROC_UNLOCK(p);
2338#endif
2339 return (ret);
2340 }
2341#endif /* notyet */
2342#if defined(sun)
2343 mutex_exit(&p->p_lock);
2344#else
2345 PROC_UNLOCK(p);
2346#endif
2347 }
2348#endif /* notyet */
2349
2350 ret = fasttrap_add_probe(probe);
2351err:
2352 kmem_free(probe, size);
2353
2354 return (ret);
2355
2356 } else if (cmd == FASTTRAPIOC_GETINSTR) {
2357 fasttrap_instr_query_t instr;
2358 fasttrap_tracepoint_t *tp;
2359 uint_t index;
2360#if defined(sun)
2361 int ret;
2362#endif
2363
2364#if defined(sun)
2365 if (copyin((void *)arg, &instr, sizeof (instr)) != 0)
2366 return (EFAULT);
2367#endif
2368
2369#ifdef notyet
2370 if (!PRIV_POLICY_CHOICE(cr, PRIV_ALL, B_FALSE)) {
2371 proc_t *p;
2372 pid_t pid = instr.ftiq_pid;
2373
2374#if defined(sun)
2375 mutex_enter(&pidlock);
2376#endif
2377 /*
2378 * Report an error if the process doesn't exist
2379 * or is actively being birthed.
2380 */
2381 p = pfind(pid);
2382 if (p)
2383 fill_kinfo_proc(p, &kp);
2384 if (p == NULL || kp.ki_stat == SIDL) {
2385#if defined(sun)
2386 mutex_exit(&pidlock);
2387#endif
2388 return (ESRCH);
2389 }
2390#if defined(sun)
2391 mutex_enter(&p->p_lock);
2392 mutex_exit(&pidlock);
2393#else
2394 PROC_LOCK_ASSERT(p, MA_OWNED);
2395#endif
2396
2397#ifdef notyet
2398 if ((ret = priv_proc_cred_perm(cr, p, NULL,
2399 VREAD)) != 0) {
2400#if defined(sun)
2401 mutex_exit(&p->p_lock);
2402#else
2403 PROC_UNLOCK(p);
2404#endif
2405 return (ret);
2406 }
2407#endif /* notyet */
2408
2409#if defined(sun)
2410 mutex_exit(&p->p_lock);
2411#else
2412 PROC_UNLOCK(p);
2413#endif
2414 }
2415#endif /* notyet */
2416
2417 index = FASTTRAP_TPOINTS_INDEX(instr.ftiq_pid, instr.ftiq_pc);
2418
2419 mutex_enter(&fasttrap_tpoints.fth_table[index].ftb_mtx);
2420 tp = fasttrap_tpoints.fth_table[index].ftb_data;
2421 while (tp != NULL) {
2422 if (instr.ftiq_pid == tp->ftt_pid &&
2423 instr.ftiq_pc == tp->ftt_pc &&
2424 tp->ftt_proc->ftpc_acount != 0)
2425 break;
2426
2427 tp = tp->ftt_next;
2428 }
2429
2430 if (tp == NULL) {
2431 mutex_exit(&fasttrap_tpoints.fth_table[index].ftb_mtx);
2432 return (ENOENT);
2433 }
2434
2435 bcopy(&tp->ftt_instr, &instr.ftiq_instr,
2436 sizeof (instr.ftiq_instr));
2437 mutex_exit(&fasttrap_tpoints.fth_table[index].ftb_mtx);
2438
2439 if (copyout(&instr, (void *)arg, sizeof (instr)) != 0)
2440 return (EFAULT);
2441
2442 return (0);
2443 }
2444
2445 return (EINVAL);
2446}
2447
2448static int
2449fasttrap_load(void)
2450{
2451 ulong_t nent;
2452 int i, ret;
2453
2454 /* Create the /dev/dtrace/fasttrap entry. */
2455 fasttrap_cdev = make_dev(&fasttrap_cdevsw, 0, UID_ROOT, GID_WHEEL, 0600,
2456 "dtrace/fasttrap");
2457
2458 mtx_init(&fasttrap_cleanup_mtx, "fasttrap clean", "dtrace", MTX_DEF);
2459 mutex_init(&fasttrap_count_mtx, "fasttrap count mtx", MUTEX_DEFAULT,
2460 NULL);
2461
2462#if defined(sun)
2463 fasttrap_max = ddi_getprop(DDI_DEV_T_ANY, devi, DDI_PROP_DONTPASS,
2464 "fasttrap-max-probes", FASTTRAP_MAX_DEFAULT);
2465#else
2466 fasttrap_max = FASTTRAP_MAX_DEFAULT;
2467#endif
2468 fasttrap_total = 0;
2469
2470 /*
2471 * Conjure up the tracepoints hashtable...
2472 */
2473#if defined(sun)
2474 nent = ddi_getprop(DDI_DEV_T_ANY, devi, DDI_PROP_DONTPASS,
2475 "fasttrap-hash-size", FASTTRAP_TPOINTS_DEFAULT_SIZE);
2476#else
2477 nent = FASTTRAP_TPOINTS_DEFAULT_SIZE;
2478#endif
2479
2480 if (nent == 0 || nent > 0x1000000)
2481 nent = FASTTRAP_TPOINTS_DEFAULT_SIZE;
2482
2483 if ((nent & (nent - 1)) == 0)
2484 fasttrap_tpoints.fth_nent = nent;
2485 else
2486 fasttrap_tpoints.fth_nent = 1 << fasttrap_highbit(nent);
2487 ASSERT(fasttrap_tpoints.fth_nent > 0);
2488 fasttrap_tpoints.fth_mask = fasttrap_tpoints.fth_nent - 1;
2489 fasttrap_tpoints.fth_table = kmem_zalloc(fasttrap_tpoints.fth_nent *
2490 sizeof (fasttrap_bucket_t), KM_SLEEP);
2491#if !defined(sun)
2492 for (i = 0; i < fasttrap_tpoints.fth_nent; i++)
2493 mutex_init(&fasttrap_tpoints.fth_table[i].ftb_mtx,
2494 "tracepoints bucket mtx", MUTEX_DEFAULT, NULL);
2495#endif
2496
2497 /*
2498 * ... and the providers hash table...
2499 */
2500 nent = FASTTRAP_PROVIDERS_DEFAULT_SIZE;
2501 if ((nent & (nent - 1)) == 0)
2502 fasttrap_provs.fth_nent = nent;
2503 else
2504 fasttrap_provs.fth_nent = 1 << fasttrap_highbit(nent);
2505 ASSERT(fasttrap_provs.fth_nent > 0);
2506 fasttrap_provs.fth_mask = fasttrap_provs.fth_nent - 1;
2507 fasttrap_provs.fth_table = kmem_zalloc(fasttrap_provs.fth_nent *
2508 sizeof (fasttrap_bucket_t), KM_SLEEP);
2509#if !defined(sun)
2510 for (i = 0; i < fasttrap_provs.fth_nent; i++)
2511 mutex_init(&fasttrap_provs.fth_table[i].ftb_mtx,
2512 "providers bucket mtx", MUTEX_DEFAULT, NULL);
2513#endif
2514
2515 ret = kproc_create(fasttrap_pid_cleanup_cb, NULL,
2516 &fasttrap_cleanup_proc, 0, 0, "ftcleanup");
2517 if (ret != 0) {
2518 destroy_dev(fasttrap_cdev);
2519#if !defined(sun)
2520 for (i = 0; i < fasttrap_provs.fth_nent; i++)
2521 mutex_destroy(&fasttrap_provs.fth_table[i].ftb_mtx);
2522 for (i = 0; i < fasttrap_tpoints.fth_nent; i++)
2523 mutex_destroy(&fasttrap_tpoints.fth_table[i].ftb_mtx);
2524#endif
2525 kmem_free(fasttrap_provs.fth_table, fasttrap_provs.fth_nent *
2526 sizeof (fasttrap_bucket_t));
2527 mtx_destroy(&fasttrap_cleanup_mtx);
2528 mutex_destroy(&fasttrap_count_mtx);
2529 return (ret);
2530 }
2531
2532
2533 /*
2534 * ... and the procs hash table.
2535 */
2536 nent = FASTTRAP_PROCS_DEFAULT_SIZE;
2537 if ((nent & (nent - 1)) == 0)
2538 fasttrap_procs.fth_nent = nent;
2539 else
2540 fasttrap_procs.fth_nent = 1 << fasttrap_highbit(nent);
2541 ASSERT(fasttrap_procs.fth_nent > 0);
2542 fasttrap_procs.fth_mask = fasttrap_procs.fth_nent - 1;
2543 fasttrap_procs.fth_table = kmem_zalloc(fasttrap_procs.fth_nent *
2544 sizeof (fasttrap_bucket_t), KM_SLEEP);
2545#if !defined(sun)
2546 for (i = 0; i < fasttrap_procs.fth_nent; i++)
2547 mutex_init(&fasttrap_procs.fth_table[i].ftb_mtx,
2548 "processes bucket mtx", MUTEX_DEFAULT, NULL);
2549
2550 CPU_FOREACH(i) {
2551 mutex_init(&fasttrap_cpuc_pid_lock[i], "fasttrap barrier",
2552 MUTEX_DEFAULT, NULL);
2553 }
|
| 2554
2555 /*
2556 * This event handler must run before kdtrace_thread_dtor() since it
2557 * accesses the thread's struct kdtrace_thread.
2558 */
2559 fasttrap_thread_dtor_tag = EVENTHANDLER_REGISTER(thread_dtor,
2560 fasttrap_thread_dtor, NULL, EVENTHANDLER_PRI_FIRST);
|
2366#endif
2367
2368 /*
2369 * Install our hooks into fork(2), exec(2), and exit(2).
2370 */
2371 dtrace_fasttrap_fork = &fasttrap_fork;
2372 dtrace_fasttrap_exit = &fasttrap_exec_exit;
2373 dtrace_fasttrap_exec = &fasttrap_exec_exit;
2374
2375 (void) dtrace_meta_register("fasttrap", &fasttrap_mops, NULL,
2376 &fasttrap_meta_id);
2377
2378 return (0);
2379}
2380
2381static int
2382fasttrap_unload(void)
2383{
2384 int i, fail = 0;
2385
2386 /*
2387 * Unregister the meta-provider to make sure no new fasttrap-
2388 * managed providers come along while we're trying to close up
2389 * shop. If we fail to detach, we'll need to re-register as a
2390 * meta-provider. We can fail to unregister as a meta-provider
2391 * if providers we manage still exist.
2392 */
2393 if (fasttrap_meta_id != DTRACE_METAPROVNONE &&
2394 dtrace_meta_unregister(fasttrap_meta_id) != 0)
2395 return (-1);
2396
2397 /*
2398 * Iterate over all of our providers. If there's still a process
2399 * that corresponds to that pid, fail to detach.
2400 */
2401 for (i = 0; i < fasttrap_provs.fth_nent; i++) {
2402 fasttrap_provider_t **fpp, *fp;
2403 fasttrap_bucket_t *bucket = &fasttrap_provs.fth_table[i];
2404
2405 mutex_enter(&bucket->ftb_mtx);
2406 fpp = (fasttrap_provider_t **)&bucket->ftb_data;
2407 while ((fp = *fpp) != NULL) {
2408 /*
2409 * Acquire and release the lock as a simple way of
2410 * waiting for any other consumer to finish with
2411 * this provider. A thread must first acquire the
2412 * bucket lock so there's no chance of another thread
2413 * blocking on the provider's lock.
2414 */
2415 mutex_enter(&fp->ftp_mtx);
2416 mutex_exit(&fp->ftp_mtx);
2417
2418 if (dtrace_unregister(fp->ftp_provid) != 0) {
2419 fail = 1;
2420 fpp = &fp->ftp_next;
2421 } else {
2422 *fpp = fp->ftp_next;
2423 fasttrap_provider_free(fp);
2424 }
2425 }
2426
2427 mutex_exit(&bucket->ftb_mtx);
2428 }
2429
2430 if (fail) {
2431 (void) dtrace_meta_register("fasttrap", &fasttrap_mops, NULL,
2432 &fasttrap_meta_id);
2433
2434 return (-1);
2435 }
2436
2437 /*
2438 * Stop new processes from entering these hooks now, before the
2439 * fasttrap_cleanup thread runs. That way all processes will hopefully
2440 * be out of these hooks before we free fasttrap_provs.fth_table
2441 */
2442 ASSERT(dtrace_fasttrap_fork == &fasttrap_fork);
2443 dtrace_fasttrap_fork = NULL;
2444
2445 ASSERT(dtrace_fasttrap_exec == &fasttrap_exec_exit);
2446 dtrace_fasttrap_exec = NULL;
2447
2448 ASSERT(dtrace_fasttrap_exit == &fasttrap_exec_exit);
2449 dtrace_fasttrap_exit = NULL;
2450
2451 mtx_lock(&fasttrap_cleanup_mtx);
2452 fasttrap_cleanup_drain = 1;
2453 /* Wait for the cleanup thread to finish up and signal us. */
2454 wakeup(&fasttrap_cleanup_cv);
2455 mtx_sleep(&fasttrap_cleanup_drain, &fasttrap_cleanup_mtx, 0, "ftcld",
2456 0);
2457 fasttrap_cleanup_proc = NULL;
2458 mtx_destroy(&fasttrap_cleanup_mtx);
2459
2460#ifdef DEBUG
2461 mutex_enter(&fasttrap_count_mtx);
2462 ASSERT(fasttrap_pid_count == 0);
2463 mutex_exit(&fasttrap_count_mtx);
2464#endif
2465
2466#if !defined(sun)
| 2561#endif
2562
2563 /*
2564 * Install our hooks into fork(2), exec(2), and exit(2).
2565 */
2566 dtrace_fasttrap_fork = &fasttrap_fork;
2567 dtrace_fasttrap_exit = &fasttrap_exec_exit;
2568 dtrace_fasttrap_exec = &fasttrap_exec_exit;
2569
2570 (void) dtrace_meta_register("fasttrap", &fasttrap_mops, NULL,
2571 &fasttrap_meta_id);
2572
2573 return (0);
2574}
2575
2576static int
2577fasttrap_unload(void)
2578{
2579 int i, fail = 0;
2580
2581 /*
2582 * Unregister the meta-provider to make sure no new fasttrap-
2583 * managed providers come along while we're trying to close up
2584 * shop. If we fail to detach, we'll need to re-register as a
2585 * meta-provider. We can fail to unregister as a meta-provider
2586 * if providers we manage still exist.
2587 */
2588 if (fasttrap_meta_id != DTRACE_METAPROVNONE &&
2589 dtrace_meta_unregister(fasttrap_meta_id) != 0)
2590 return (-1);
2591
2592 /*
2593 * Iterate over all of our providers. If there's still a process
2594 * that corresponds to that pid, fail to detach.
2595 */
2596 for (i = 0; i < fasttrap_provs.fth_nent; i++) {
2597 fasttrap_provider_t **fpp, *fp;
2598 fasttrap_bucket_t *bucket = &fasttrap_provs.fth_table[i];
2599
2600 mutex_enter(&bucket->ftb_mtx);
2601 fpp = (fasttrap_provider_t **)&bucket->ftb_data;
2602 while ((fp = *fpp) != NULL) {
2603 /*
2604 * Acquire and release the lock as a simple way of
2605 * waiting for any other consumer to finish with
2606 * this provider. A thread must first acquire the
2607 * bucket lock so there's no chance of another thread
2608 * blocking on the provider's lock.
2609 */
2610 mutex_enter(&fp->ftp_mtx);
2611 mutex_exit(&fp->ftp_mtx);
2612
2613 if (dtrace_unregister(fp->ftp_provid) != 0) {
2614 fail = 1;
2615 fpp = &fp->ftp_next;
2616 } else {
2617 *fpp = fp->ftp_next;
2618 fasttrap_provider_free(fp);
2619 }
2620 }
2621
2622 mutex_exit(&bucket->ftb_mtx);
2623 }
2624
2625 if (fail) {
2626 (void) dtrace_meta_register("fasttrap", &fasttrap_mops, NULL,
2627 &fasttrap_meta_id);
2628
2629 return (-1);
2630 }
2631
2632 /*
2633 * Stop new processes from entering these hooks now, before the
2634 * fasttrap_cleanup thread runs. That way all processes will hopefully
2635 * be out of these hooks before we free fasttrap_provs.fth_table
2636 */
2637 ASSERT(dtrace_fasttrap_fork == &fasttrap_fork);
2638 dtrace_fasttrap_fork = NULL;
2639
2640 ASSERT(dtrace_fasttrap_exec == &fasttrap_exec_exit);
2641 dtrace_fasttrap_exec = NULL;
2642
2643 ASSERT(dtrace_fasttrap_exit == &fasttrap_exec_exit);
2644 dtrace_fasttrap_exit = NULL;
2645
2646 mtx_lock(&fasttrap_cleanup_mtx);
2647 fasttrap_cleanup_drain = 1;
2648 /* Wait for the cleanup thread to finish up and signal us. */
2649 wakeup(&fasttrap_cleanup_cv);
2650 mtx_sleep(&fasttrap_cleanup_drain, &fasttrap_cleanup_mtx, 0, "ftcld",
2651 0);
2652 fasttrap_cleanup_proc = NULL;
2653 mtx_destroy(&fasttrap_cleanup_mtx);
2654
2655#ifdef DEBUG
2656 mutex_enter(&fasttrap_count_mtx);
2657 ASSERT(fasttrap_pid_count == 0);
2658 mutex_exit(&fasttrap_count_mtx);
2659#endif
2660
2661#if !defined(sun)
|
| 2662 EVENTHANDLER_DEREGISTER(thread_dtor, fasttrap_thread_dtor_tag);
2663
|
2467 for (i = 0; i < fasttrap_tpoints.fth_nent; i++)
2468 mutex_destroy(&fasttrap_tpoints.fth_table[i].ftb_mtx);
2469 for (i = 0; i < fasttrap_provs.fth_nent; i++)
2470 mutex_destroy(&fasttrap_provs.fth_table[i].ftb_mtx);
2471 for (i = 0; i < fasttrap_procs.fth_nent; i++)
2472 mutex_destroy(&fasttrap_procs.fth_table[i].ftb_mtx);
2473#endif
2474 kmem_free(fasttrap_tpoints.fth_table,
2475 fasttrap_tpoints.fth_nent * sizeof (fasttrap_bucket_t));
2476 fasttrap_tpoints.fth_nent = 0;
2477
2478 kmem_free(fasttrap_provs.fth_table,
2479 fasttrap_provs.fth_nent * sizeof (fasttrap_bucket_t));
2480 fasttrap_provs.fth_nent = 0;
2481
2482 kmem_free(fasttrap_procs.fth_table,
2483 fasttrap_procs.fth_nent * sizeof (fasttrap_bucket_t));
2484 fasttrap_procs.fth_nent = 0;
2485
2486#if !defined(sun)
2487 destroy_dev(fasttrap_cdev);
2488 mutex_destroy(&fasttrap_count_mtx);
2489 CPU_FOREACH(i) {
2490 mutex_destroy(&fasttrap_cpuc_pid_lock[i]);
2491 }
2492#endif
2493
2494 return (0);
2495}
2496
2497/* ARGSUSED */
2498static int
2499fasttrap_modevent(module_t mod __unused, int type, void *data __unused)
2500{
2501 int error = 0;
2502
2503 switch (type) {
2504 case MOD_LOAD:
2505 break;
2506
2507 case MOD_UNLOAD:
2508 break;
2509
2510 case MOD_SHUTDOWN:
2511 break;
2512
2513 default:
2514 error = EOPNOTSUPP;
2515 break;
2516 }
2517 return (error);
2518}
2519
2520SYSINIT(fasttrap_load, SI_SUB_DTRACE_PROVIDER, SI_ORDER_ANY, fasttrap_load,
2521 NULL);
2522SYSUNINIT(fasttrap_unload, SI_SUB_DTRACE_PROVIDER, SI_ORDER_ANY,
2523 fasttrap_unload, NULL);
2524
2525DEV_MODULE(fasttrap, fasttrap_modevent, NULL);
2526MODULE_VERSION(fasttrap, 1);
2527MODULE_DEPEND(fasttrap, dtrace, 1, 1, 1);
2528MODULE_DEPEND(fasttrap, opensolaris, 1, 1, 1);
| 2664 for (i = 0; i < fasttrap_tpoints.fth_nent; i++)
2665 mutex_destroy(&fasttrap_tpoints.fth_table[i].ftb_mtx);
2666 for (i = 0; i < fasttrap_provs.fth_nent; i++)
2667 mutex_destroy(&fasttrap_provs.fth_table[i].ftb_mtx);
2668 for (i = 0; i < fasttrap_procs.fth_nent; i++)
2669 mutex_destroy(&fasttrap_procs.fth_table[i].ftb_mtx);
2670#endif
2671 kmem_free(fasttrap_tpoints.fth_table,
2672 fasttrap_tpoints.fth_nent * sizeof (fasttrap_bucket_t));
2673 fasttrap_tpoints.fth_nent = 0;
2674
2675 kmem_free(fasttrap_provs.fth_table,
2676 fasttrap_provs.fth_nent * sizeof (fasttrap_bucket_t));
2677 fasttrap_provs.fth_nent = 0;
2678
2679 kmem_free(fasttrap_procs.fth_table,
2680 fasttrap_procs.fth_nent * sizeof (fasttrap_bucket_t));
2681 fasttrap_procs.fth_nent = 0;
2682
2683#if !defined(sun)
2684 destroy_dev(fasttrap_cdev);
2685 mutex_destroy(&fasttrap_count_mtx);
2686 CPU_FOREACH(i) {
2687 mutex_destroy(&fasttrap_cpuc_pid_lock[i]);
2688 }
2689#endif
2690
2691 return (0);
2692}
2693
2694/* ARGSUSED */
2695static int
2696fasttrap_modevent(module_t mod __unused, int type, void *data __unused)
2697{
2698 int error = 0;
2699
2700 switch (type) {
2701 case MOD_LOAD:
2702 break;
2703
2704 case MOD_UNLOAD:
2705 break;
2706
2707 case MOD_SHUTDOWN:
2708 break;
2709
2710 default:
2711 error = EOPNOTSUPP;
2712 break;
2713 }
2714 return (error);
2715}
2716
2717SYSINIT(fasttrap_load, SI_SUB_DTRACE_PROVIDER, SI_ORDER_ANY, fasttrap_load,
2718 NULL);
2719SYSUNINIT(fasttrap_unload, SI_SUB_DTRACE_PROVIDER, SI_ORDER_ANY,
2720 fasttrap_unload, NULL);
2721
2722DEV_MODULE(fasttrap, fasttrap_modevent, NULL);
2723MODULE_VERSION(fasttrap, 1);
2724MODULE_DEPEND(fasttrap, dtrace, 1, 1, 1);
2725MODULE_DEPEND(fasttrap, opensolaris, 1, 1, 1);
|