24 */
25
26/*
27 * Copyright 2008 Sun Microsystems, Inc. All rights reserved.
28 * Use is subject to license terms.
29 */
30
31#if defined(sun)
32#pragma ident "%Z%%M% %I% %E% SMI"
33#endif
34
35#include <sys/atomic.h>
36#include <sys/errno.h>
37#include <sys/stat.h>
38#include <sys/modctl.h>
39#include <sys/conf.h>
40#include <sys/systm.h>
41#if defined(sun)
42#include <sys/ddi.h>
43#endif
44#include <sys/sunddi.h>
45#include <sys/cpuvar.h>
46#include <sys/kmem.h>
47#if defined(sun)
48#include <sys/strsubr.h>
49#endif
50#include <sys/fasttrap.h>
51#include <sys/fasttrap_impl.h>
52#include <sys/fasttrap_isa.h>
53#include <sys/dtrace.h>
54#include <sys/dtrace_impl.h>
55#include <sys/sysmacros.h>
56#include <sys/proc.h>
57#include <sys/policy.h>
58#if defined(sun)
59#include <util/qsort.h>
60#endif
61#include <sys/mutex.h>
62#include <sys/kernel.h>
63#if !defined(sun)
64#include <sys/user.h>
65#include <sys/dtrace_bsd.h>
66#include <cddl/dev/dtrace/dtrace_cddl.h>
67#endif
68
69/*
70 * User-Land Trap-Based Tracing
71 * ----------------------------
72 *
73 * The fasttrap provider allows DTrace consumers to instrument any user-level
74 * instruction to gather data; this includes probes with semantic
75 * signifigance like entry and return as well as simple offsets into the
76 * function. While the specific techniques used are very ISA specific, the
77 * methodology is generalizable to any architecture.
78 *
79 *
80 * The General Methodology
81 * -----------------------
82 *
83 * With the primary goal of tracing every user-land instruction and the
84 * limitation that we can't trust user space so don't want to rely on much
85 * information there, we begin by replacing the instructions we want to trace
86 * with trap instructions. Each instruction we overwrite is saved into a hash
87 * table keyed by process ID and pc address. When we enter the kernel due to
88 * this trap instruction, we need the effects of the replaced instruction to
89 * appear to have occurred before we proceed with the user thread's
90 * execution.
91 *
92 * Each user level thread is represented by a ulwp_t structure which is
93 * always easily accessible through a register. The most basic way to produce
94 * the effects of the instruction we replaced is to copy that instruction out
95 * to a bit of scratch space reserved in the user thread's ulwp_t structure
96 * (a sort of kernel-private thread local storage), set the PC to that
97 * scratch space and single step. When we reenter the kernel after single
98 * stepping the instruction we must then adjust the PC to point to what would
99 * normally be the next instruction. Of course, special care must be taken
100 * for branches and jumps, but these represent such a small fraction of any
101 * instruction set that writing the code to emulate these in the kernel is
102 * not too difficult.
103 *
104 * Return probes may require several tracepoints to trace every return site,
105 * and, conversely, each tracepoint may activate several probes (the entry
106 * and offset 0 probes, for example). To solve this muliplexing problem,
107 * tracepoints contain lists of probes to activate and probes contain lists
108 * of tracepoints to enable. If a probe is activated, it adds its ID to
109 * existing tracepoints or creates new ones as necessary.
110 *
111 * Most probes are activated _before_ the instruction is executed, but return
112 * probes are activated _after_ the effects of the last instruction of the
113 * function are visible. Return probes must be fired _after_ we have
114 * single-stepped the instruction whereas all other probes are fired
115 * beforehand.
116 *
117 *
118 * Lock Ordering
119 * -------------
120 *
121 * The lock ordering below -- both internally and with respect to the DTrace
122 * framework -- is a little tricky and bears some explanation. Each provider
123 * has a lock (ftp_mtx) that protects its members including reference counts
124 * for enabled probes (ftp_rcount), consumers actively creating probes
125 * (ftp_ccount) and USDT consumers (ftp_mcount); all three prevent a provider
126 * from being freed. A provider is looked up by taking the bucket lock for the
127 * provider hash table, and is returned with its lock held. The provider lock
128 * may be taken in functions invoked by the DTrace framework, but may not be
129 * held while calling functions in the DTrace framework.
130 *
131 * To ensure consistency over multiple calls to the DTrace framework, the
132 * creation lock (ftp_cmtx) should be held. Naturally, the creation lock may
133 * not be taken when holding the provider lock as that would create a cyclic
134 * lock ordering. In situations where one would naturally take the provider
135 * lock and then the creation lock, we instead up a reference count to prevent
136 * the provider from disappearing, drop the provider lock, and acquire the
137 * creation lock.
138 *
139 * Briefly:
140 * bucket lock before provider lock
141 * DTrace before provider lock
142 * creation lock before DTrace
143 * never hold the provider lock and creation lock simultaneously
144 */
145
146static d_open_t fasttrap_open;
147static d_ioctl_t fasttrap_ioctl;
148
149static struct cdevsw fasttrap_cdevsw = {
150 .d_version = D_VERSION,
151 .d_open = fasttrap_open,
152 .d_ioctl = fasttrap_ioctl,
153 .d_name = "fasttrap",
154};
155static struct cdev *fasttrap_cdev;
156static dtrace_meta_provider_id_t fasttrap_meta_id;
157
158static struct proc *fasttrap_cleanup_proc;
159static struct mtx fasttrap_cleanup_mtx;
160static uint_t fasttrap_cleanup_work, fasttrap_cleanup_drain, fasttrap_cleanup_cv;
161
162/*
163 * Generation count on modifications to the global tracepoint lookup table.
164 */
165static volatile uint64_t fasttrap_mod_gen;
166
167/*
168 * When the fasttrap provider is loaded, fasttrap_max is set to either
169 * FASTTRAP_MAX_DEFAULT or the value for fasttrap-max-probes in the
170 * fasttrap.conf file. Each time a probe is created, fasttrap_total is
171 * incremented by the number of tracepoints that may be associated with that
172 * probe; fasttrap_total is capped at fasttrap_max.
173 */
174#define FASTTRAP_MAX_DEFAULT 250000
175static uint32_t fasttrap_max;
176static uint32_t fasttrap_total;
177
178/*
179 * Copyright (c) 2011, Joyent, Inc. All rights reserved.
180 */
181
182#define FASTTRAP_TPOINTS_DEFAULT_SIZE 0x4000
183#define FASTTRAP_PROVIDERS_DEFAULT_SIZE 0x100
184#define FASTTRAP_PROCS_DEFAULT_SIZE 0x100
185
186#define FASTTRAP_PID_NAME "pid"
187
188fasttrap_hash_t fasttrap_tpoints;
189static fasttrap_hash_t fasttrap_provs;
190static fasttrap_hash_t fasttrap_procs;
191
192static uint64_t fasttrap_pid_count; /* pid ref count */
193static kmutex_t fasttrap_count_mtx; /* lock on ref count */
194
195#define FASTTRAP_ENABLE_FAIL 1
196#define FASTTRAP_ENABLE_PARTIAL 2
197
198static int fasttrap_tracepoint_enable(proc_t *, fasttrap_probe_t *, uint_t);
199static void fasttrap_tracepoint_disable(proc_t *, fasttrap_probe_t *, uint_t);
200
201static fasttrap_provider_t *fasttrap_provider_lookup(pid_t, const char *,
202 const dtrace_pattr_t *);
203static void fasttrap_provider_retire(pid_t, const char *, int);
204static void fasttrap_provider_free(fasttrap_provider_t *);
205
206static fasttrap_proc_t *fasttrap_proc_lookup(pid_t);
207static void fasttrap_proc_release(fasttrap_proc_t *);
208
209#define FASTTRAP_PROVS_INDEX(pid, name) \
210 ((fasttrap_hash_str(name) + (pid)) & fasttrap_provs.fth_mask)
211
212#define FASTTRAP_PROCS_INDEX(pid) ((pid) & fasttrap_procs.fth_mask)
213
214#if !defined(sun)
215static kmutex_t fasttrap_cpuc_pid_lock[MAXCPU];
216#endif
217
218static int
219fasttrap_highbit(ulong_t i)
220{
221 int h = 1;
222
223 if (i == 0)
224 return (0);
225#ifdef _LP64
226 if (i & 0xffffffff00000000ul) {
227 h += 32; i >>= 32;
228 }
229#endif
230 if (i & 0xffff0000) {
231 h += 16; i >>= 16;
232 }
233 if (i & 0xff00) {
234 h += 8; i >>= 8;
235 }
236 if (i & 0xf0) {
237 h += 4; i >>= 4;
238 }
239 if (i & 0xc) {
240 h += 2; i >>= 2;
241 }
242 if (i & 0x2) {
243 h += 1;
244 }
245 return (h);
246}
247
248static uint_t
249fasttrap_hash_str(const char *p)
250{
251 unsigned int g;
252 uint_t hval = 0;
253
254 while (*p) {
255 hval = (hval << 4) + *p++;
256 if ((g = (hval & 0xf0000000)) != 0)
257 hval ^= g >> 24;
258 hval &= ~g;
259 }
260 return (hval);
261}
262
263void
264fasttrap_sigtrap(proc_t *p, kthread_t *t, uintptr_t pc)
265{
266#if defined(sun)
267 sigqueue_t *sqp = kmem_zalloc(sizeof (sigqueue_t), KM_SLEEP);
268
269 sqp->sq_info.si_signo = SIGTRAP;
270 sqp->sq_info.si_code = TRAP_DTRACE;
271 sqp->sq_info.si_addr = (caddr_t)pc;
272
273 mutex_enter(&p->p_lock);
274 sigaddqa(p, t, sqp);
275 mutex_exit(&p->p_lock);
276
277 if (t != NULL)
278 aston(t);
279#else
280 ksiginfo_t *ksi = kmem_zalloc(sizeof (ksiginfo_t), KM_SLEEP);
281
282 ksiginfo_init(ksi);
283 ksi->ksi_signo = SIGTRAP;
284 ksi->ksi_code = TRAP_DTRACE;
285 ksi->ksi_addr = (caddr_t)pc;
286 PROC_LOCK(p);
287 (void) tdksignal(t, SIGTRAP, ksi);
288 PROC_UNLOCK(p);
289#endif
290}
291
292/*
293 * This function ensures that no threads are actively using the memory
294 * associated with probes that were formerly live.
295 */
296static void
297fasttrap_mod_barrier(uint64_t gen)
298{
299 int i;
300
301 if (gen < fasttrap_mod_gen)
302 return;
303
304 fasttrap_mod_gen++;
305
306 CPU_FOREACH(i) {
307 mutex_enter(&fasttrap_cpuc_pid_lock[i]);
308 mutex_exit(&fasttrap_cpuc_pid_lock[i]);
309 }
310}
311
312/*
313 * This function performs asynchronous cleanup of fasttrap providers. The
314 * Solaris implementation of this mechanism use a timeout that's activated in
315 * fasttrap_pid_cleanup(), but this doesn't work in FreeBSD: one may sleep while
316 * holding the DTrace mutexes, but it is unsafe to sleep in a callout handler.
317 * Thus we use a dedicated process to perform the cleanup when requested.
318 */
319/*ARGSUSED*/
320static void
321fasttrap_pid_cleanup_cb(void *data)
322{
323 fasttrap_provider_t **fpp, *fp;
324 fasttrap_bucket_t *bucket;
325 dtrace_provider_id_t provid;
326 int i, later = 0, rval;
327
328 mtx_lock(&fasttrap_cleanup_mtx);
329 while (!fasttrap_cleanup_drain || later > 0) {
330 fasttrap_cleanup_work = 0;
331 mtx_unlock(&fasttrap_cleanup_mtx);
332
333 later = 0;
334
335 /*
336 * Iterate over all the providers trying to remove the marked
337 * ones. If a provider is marked but not retired, we just
338 * have to take a crack at removing it -- it's no big deal if
339 * we can't.
340 */
341 for (i = 0; i < fasttrap_provs.fth_nent; i++) {
342 bucket = &fasttrap_provs.fth_table[i];
343 mutex_enter(&bucket->ftb_mtx);
344 fpp = (fasttrap_provider_t **)&bucket->ftb_data;
345
346 while ((fp = *fpp) != NULL) {
347 if (!fp->ftp_marked) {
348 fpp = &fp->ftp_next;
349 continue;
350 }
351
352 mutex_enter(&fp->ftp_mtx);
353
354 /*
355 * If this provider has consumers actively
356 * creating probes (ftp_ccount) or is a USDT
357 * provider (ftp_mcount), we can't unregister
358 * or even condense.
359 */
360 if (fp->ftp_ccount != 0 ||
361 fp->ftp_mcount != 0) {
362 mutex_exit(&fp->ftp_mtx);
363 fp->ftp_marked = 0;
364 continue;
365 }
366
367 if (!fp->ftp_retired || fp->ftp_rcount != 0)
368 fp->ftp_marked = 0;
369
370 mutex_exit(&fp->ftp_mtx);
371
372 /*
373 * If we successfully unregister this
374 * provider we can remove it from the hash
375 * chain and free the memory. If our attempt
376 * to unregister fails and this is a retired
377 * provider, increment our flag to try again
378 * pretty soon. If we've consumed more than
379 * half of our total permitted number of
380 * probes call dtrace_condense() to try to
381 * clean out the unenabled probes.
382 */
383 provid = fp->ftp_provid;
384 if ((rval = dtrace_unregister(provid)) != 0) {
385 if (fasttrap_total > fasttrap_max / 2)
386 (void) dtrace_condense(provid);
387
388 if (rval == EAGAIN)
389 fp->ftp_marked = 1;
390
391 later += fp->ftp_marked;
392 fpp = &fp->ftp_next;
393 } else {
394 *fpp = fp->ftp_next;
395 fasttrap_provider_free(fp);
396 }
397 }
398 mutex_exit(&bucket->ftb_mtx);
399 }
400 mtx_lock(&fasttrap_cleanup_mtx);
401
402 /*
403 * If we were unable to retire a provider, try again after a
404 * second. This situation can occur in certain circumstances
405 * where providers cannot be unregistered even though they have
406 * no probes enabled because of an execution of dtrace -l or
407 * something similar.
408 */
409 if (later > 0 || fasttrap_cleanup_work ||
410 fasttrap_cleanup_drain) {
411 mtx_unlock(&fasttrap_cleanup_mtx);
412 pause("ftclean", hz);
413 mtx_lock(&fasttrap_cleanup_mtx);
414 } else
415 mtx_sleep(&fasttrap_cleanup_cv, &fasttrap_cleanup_mtx,
416 0, "ftcl", 0);
417 }
418
419 /*
420 * Wake up the thread in fasttrap_unload() now that we're done.
421 */
422 wakeup(&fasttrap_cleanup_drain);
423 mtx_unlock(&fasttrap_cleanup_mtx);
424
425 kthread_exit();
426}
427
428/*
429 * Activates the asynchronous cleanup mechanism.
430 */
431static void
432fasttrap_pid_cleanup(void)
433{
434
435 mtx_lock(&fasttrap_cleanup_mtx);
436 if (!fasttrap_cleanup_work) {
437 fasttrap_cleanup_work = 1;
438 wakeup(&fasttrap_cleanup_cv);
439 }
440 mtx_unlock(&fasttrap_cleanup_mtx);
441}
442
443/*
444 * This is called from cfork() via dtrace_fasttrap_fork(). The child
445 * process's address space is (roughly) a copy of the parent process's so
446 * we have to remove all the instrumentation we had previously enabled in the
447 * parent.
448 */
449static void
450fasttrap_fork(proc_t *p, proc_t *cp)
451{
452 pid_t ppid = p->p_pid;
453 int i;
454
455#if defined(sun)
456 ASSERT(curproc == p);
457 ASSERT(p->p_proc_flag & P_PR_LOCK);
458#else
459 PROC_LOCK_ASSERT(p, MA_OWNED);
460#endif
461#if defined(sun)
462 ASSERT(p->p_dtrace_count > 0);
463#else
464 if (p->p_dtrace_helpers) {
465 /*
466 * dtrace_helpers_duplicate() allocates memory.
467 */
468 _PHOLD(cp);
469 PROC_UNLOCK(p);
470 PROC_UNLOCK(cp);
471 dtrace_helpers_duplicate(p, cp);
472 PROC_LOCK(cp);
473 PROC_LOCK(p);
474 _PRELE(cp);
475 }
476 /*
477 * This check is purposely here instead of in kern_fork.c because,
478 * for legal resons, we cannot include the dtrace_cddl.h header
479 * inside kern_fork.c and insert if-clause there.
480 */
481 if (p->p_dtrace_count == 0)
482 return;
483#endif
484 ASSERT(cp->p_dtrace_count == 0);
485
486 /*
487 * This would be simpler and faster if we maintained per-process
488 * hash tables of enabled tracepoints. It could, however, potentially
489 * slow down execution of a tracepoint since we'd need to go
490 * through two levels of indirection. In the future, we should
491 * consider either maintaining per-process ancillary lists of
492 * enabled tracepoints or hanging a pointer to a per-process hash
493 * table of enabled tracepoints off the proc structure.
494 */
495
496 /*
497 * We don't have to worry about the child process disappearing
498 * because we're in fork().
499 */
500#if defined(sun)
501 mtx_lock_spin(&cp->p_slock);
502 sprlock_proc(cp);
503 mtx_unlock_spin(&cp->p_slock);
504#else
505 _PHOLD(cp);
506#endif
507
508 /*
509 * Iterate over every tracepoint looking for ones that belong to the
510 * parent process, and remove each from the child process.
511 */
512 for (i = 0; i < fasttrap_tpoints.fth_nent; i++) {
513 fasttrap_tracepoint_t *tp;
514 fasttrap_bucket_t *bucket = &fasttrap_tpoints.fth_table[i];
515
516 mutex_enter(&bucket->ftb_mtx);
517 for (tp = bucket->ftb_data; tp != NULL; tp = tp->ftt_next) {
518 if (tp->ftt_pid == ppid &&
519 tp->ftt_proc->ftpc_acount != 0) {
520 int ret = fasttrap_tracepoint_remove(cp, tp);
521 ASSERT(ret == 0);
522
523 /*
524 * The count of active providers can only be
525 * decremented (i.e. to zero) during exec,
526 * exit, and removal of a meta provider so it
527 * should be impossible to drop the count
528 * mid-fork.
529 */
530 ASSERT(tp->ftt_proc->ftpc_acount != 0);
531 }
532 }
533 mutex_exit(&bucket->ftb_mtx);
534 }
535
536#if defined(sun)
537 mutex_enter(&cp->p_lock);
538 sprunlock(cp);
539#else
540 _PRELE(cp);
541#endif
542}
543
544/*
545 * This is called from proc_exit() or from exec_common() if p_dtrace_probes
546 * is set on the proc structure to indicate that there is a pid provider
547 * associated with this process.
548 */
549static void
550fasttrap_exec_exit(proc_t *p)
551{
552#if defined(sun)
553 ASSERT(p == curproc);
554#endif
555 PROC_LOCK_ASSERT(p, MA_OWNED);
556 _PHOLD(p);
557 PROC_UNLOCK(p);
558
559 /*
560 * We clean up the pid provider for this process here; user-land
561 * static probes are handled by the meta-provider remove entry point.
562 */
563 fasttrap_provider_retire(p->p_pid, FASTTRAP_PID_NAME, 0);
564#if !defined(sun)
565 if (p->p_dtrace_helpers)
566 dtrace_helpers_destroy(p);
567#endif
568 PROC_LOCK(p);
569 _PRELE(p);
570}
571
572
573/*ARGSUSED*/
574static void
575fasttrap_pid_provide(void *arg, dtrace_probedesc_t *desc)
576{
577 /*
578 * There are no "default" pid probes.
579 */
580}
581
582static int
583fasttrap_tracepoint_enable(proc_t *p, fasttrap_probe_t *probe, uint_t index)
584{
585 fasttrap_tracepoint_t *tp, *new_tp = NULL;
586 fasttrap_bucket_t *bucket;
587 fasttrap_id_t *id;
588 pid_t pid;
589 uintptr_t pc;
590
591 ASSERT(index < probe->ftp_ntps);
592
593 pid = probe->ftp_pid;
594 pc = probe->ftp_tps[index].fit_tp->ftt_pc;
595 id = &probe->ftp_tps[index].fit_id;
596
597 ASSERT(probe->ftp_tps[index].fit_tp->ftt_pid == pid);
598
599#if defined(sun)
600 ASSERT(!(p->p_flag & SVFORK));
601#endif
602
603 /*
604 * Before we make any modifications, make sure we've imposed a barrier
605 * on the generation in which this probe was last modified.
606 */
607 fasttrap_mod_barrier(probe->ftp_gen);
608
609 bucket = &fasttrap_tpoints.fth_table[FASTTRAP_TPOINTS_INDEX(pid, pc)];
610
611 /*
612 * If the tracepoint has already been enabled, just add our id to the
613 * list of interested probes. This may be our second time through
614 * this path in which case we'll have constructed the tracepoint we'd
615 * like to install. If we can't find a match, and have an allocated
616 * tracepoint ready to go, enable that one now.
617 *
618 * A tracepoint whose process is defunct is also considered defunct.
619 */
620again:
621 mutex_enter(&bucket->ftb_mtx);
622 for (tp = bucket->ftb_data; tp != NULL; tp = tp->ftt_next) {
623 /*
624 * Note that it's safe to access the active count on the
625 * associated proc structure because we know that at least one
626 * provider (this one) will still be around throughout this
627 * operation.
628 */
629 if (tp->ftt_pid != pid || tp->ftt_pc != pc ||
630 tp->ftt_proc->ftpc_acount == 0)
631 continue;
632
633 /*
634 * Now that we've found a matching tracepoint, it would be
635 * a decent idea to confirm that the tracepoint is still
636 * enabled and the trap instruction hasn't been overwritten.
637 * Since this is a little hairy, we'll punt for now.
638 */
639
640 /*
641 * This can't be the first interested probe. We don't have
642 * to worry about another thread being in the midst of
643 * deleting this tracepoint (which would be the only valid
644 * reason for a tracepoint to have no interested probes)
645 * since we're holding P_PR_LOCK for this process.
646 */
647 ASSERT(tp->ftt_ids != NULL || tp->ftt_retids != NULL);
648
649 switch (id->fti_ptype) {
650 case DTFTP_ENTRY:
651 case DTFTP_OFFSETS:
652 case DTFTP_IS_ENABLED:
653 id->fti_next = tp->ftt_ids;
654 membar_producer();
655 tp->ftt_ids = id;
656 membar_producer();
657 break;
658
659 case DTFTP_RETURN:
660 case DTFTP_POST_OFFSETS:
661 id->fti_next = tp->ftt_retids;
662 membar_producer();
663 tp->ftt_retids = id;
664 membar_producer();
665 break;
666
667 default:
668 ASSERT(0);
669 }
670
671 mutex_exit(&bucket->ftb_mtx);
672
673 if (new_tp != NULL) {
674 new_tp->ftt_ids = NULL;
675 new_tp->ftt_retids = NULL;
676 }
677
678 return (0);
679 }
680
681 /*
682 * If we have a good tracepoint ready to go, install it now while
683 * we have the lock held and no one can screw with us.
684 */
685 if (new_tp != NULL) {
686 int rc = 0;
687
688 new_tp->ftt_next = bucket->ftb_data;
689 membar_producer();
690 bucket->ftb_data = new_tp;
691 membar_producer();
692 mutex_exit(&bucket->ftb_mtx);
693
694 /*
695 * Activate the tracepoint in the ISA-specific manner.
696 * If this fails, we need to report the failure, but
697 * indicate that this tracepoint must still be disabled
698 * by calling fasttrap_tracepoint_disable().
699 */
700 if (fasttrap_tracepoint_install(p, new_tp) != 0)
701 rc = FASTTRAP_ENABLE_PARTIAL;
702
703 /*
704 * Increment the count of the number of tracepoints active in
705 * the victim process.
706 */
707#if defined(sun)
708 ASSERT(p->p_proc_flag & P_PR_LOCK);
709#endif
710 p->p_dtrace_count++;
711
712 return (rc);
713 }
714
715 mutex_exit(&bucket->ftb_mtx);
716
717 /*
718 * Initialize the tracepoint that's been preallocated with the probe.
719 */
720 new_tp = probe->ftp_tps[index].fit_tp;
721
722 ASSERT(new_tp->ftt_pid == pid);
723 ASSERT(new_tp->ftt_pc == pc);
724 ASSERT(new_tp->ftt_proc == probe->ftp_prov->ftp_proc);
725 ASSERT(new_tp->ftt_ids == NULL);
726 ASSERT(new_tp->ftt_retids == NULL);
727
728 switch (id->fti_ptype) {
729 case DTFTP_ENTRY:
730 case DTFTP_OFFSETS:
731 case DTFTP_IS_ENABLED:
732 id->fti_next = NULL;
733 new_tp->ftt_ids = id;
734 break;
735
736 case DTFTP_RETURN:
737 case DTFTP_POST_OFFSETS:
738 id->fti_next = NULL;
739 new_tp->ftt_retids = id;
740 break;
741
742 default:
743 ASSERT(0);
744 }
745
746 /*
747 * If the ISA-dependent initialization goes to plan, go back to the
748 * beginning and try to install this freshly made tracepoint.
749 */
750 if (fasttrap_tracepoint_init(p, new_tp, pc, id->fti_ptype) == 0)
751 goto again;
752
753 new_tp->ftt_ids = NULL;
754 new_tp->ftt_retids = NULL;
755
756 return (FASTTRAP_ENABLE_FAIL);
757}
758
759static void
760fasttrap_tracepoint_disable(proc_t *p, fasttrap_probe_t *probe, uint_t index)
761{
762 fasttrap_bucket_t *bucket;
763 fasttrap_provider_t *provider = probe->ftp_prov;
764 fasttrap_tracepoint_t **pp, *tp;
765 fasttrap_id_t *id, **idp = NULL;
766 pid_t pid;
767 uintptr_t pc;
768
769 ASSERT(index < probe->ftp_ntps);
770
771 pid = probe->ftp_pid;
772 pc = probe->ftp_tps[index].fit_tp->ftt_pc;
773 id = &probe->ftp_tps[index].fit_id;
774
775 ASSERT(probe->ftp_tps[index].fit_tp->ftt_pid == pid);
776
777 /*
778 * Find the tracepoint and make sure that our id is one of the
779 * ones registered with it.
780 */
781 bucket = &fasttrap_tpoints.fth_table[FASTTRAP_TPOINTS_INDEX(pid, pc)];
782 mutex_enter(&bucket->ftb_mtx);
783 for (tp = bucket->ftb_data; tp != NULL; tp = tp->ftt_next) {
784 if (tp->ftt_pid == pid && tp->ftt_pc == pc &&
785 tp->ftt_proc == provider->ftp_proc)
786 break;
787 }
788
789 /*
790 * If we somehow lost this tracepoint, we're in a world of hurt.
791 */
792 ASSERT(tp != NULL);
793
794 switch (id->fti_ptype) {
795 case DTFTP_ENTRY:
796 case DTFTP_OFFSETS:
797 case DTFTP_IS_ENABLED:
798 ASSERT(tp->ftt_ids != NULL);
799 idp = &tp->ftt_ids;
800 break;
801
802 case DTFTP_RETURN:
803 case DTFTP_POST_OFFSETS:
804 ASSERT(tp->ftt_retids != NULL);
805 idp = &tp->ftt_retids;
806 break;
807
808 default:
809 ASSERT(0);
810 }
811
812 while ((*idp)->fti_probe != probe) {
813 idp = &(*idp)->fti_next;
814 ASSERT(*idp != NULL);
815 }
816
817 id = *idp;
818 *idp = id->fti_next;
819 membar_producer();
820
821 ASSERT(id->fti_probe == probe);
822
823 /*
824 * If there are other registered enablings of this tracepoint, we're
825 * all done, but if this was the last probe assocated with this
826 * this tracepoint, we need to remove and free it.
827 */
828 if (tp->ftt_ids != NULL || tp->ftt_retids != NULL) {
829
830 /*
831 * If the current probe's tracepoint is in use, swap it
832 * for an unused tracepoint.
833 */
834 if (tp == probe->ftp_tps[index].fit_tp) {
835 fasttrap_probe_t *tmp_probe;
836 fasttrap_tracepoint_t **tmp_tp;
837 uint_t tmp_index;
838
839 if (tp->ftt_ids != NULL) {
840 tmp_probe = tp->ftt_ids->fti_probe;
841 /* LINTED - alignment */
842 tmp_index = FASTTRAP_ID_INDEX(tp->ftt_ids);
843 tmp_tp = &tmp_probe->ftp_tps[tmp_index].fit_tp;
844 } else {
845 tmp_probe = tp->ftt_retids->fti_probe;
846 /* LINTED - alignment */
847 tmp_index = FASTTRAP_ID_INDEX(tp->ftt_retids);
848 tmp_tp = &tmp_probe->ftp_tps[tmp_index].fit_tp;
849 }
850
851 ASSERT(*tmp_tp != NULL);
852 ASSERT(*tmp_tp != probe->ftp_tps[index].fit_tp);
853 ASSERT((*tmp_tp)->ftt_ids == NULL);
854 ASSERT((*tmp_tp)->ftt_retids == NULL);
855
856 probe->ftp_tps[index].fit_tp = *tmp_tp;
857 *tmp_tp = tp;
858 }
859
860 mutex_exit(&bucket->ftb_mtx);
861
862 /*
863 * Tag the modified probe with the generation in which it was
864 * changed.
865 */
866 probe->ftp_gen = fasttrap_mod_gen;
867 return;
868 }
869
870 mutex_exit(&bucket->ftb_mtx);
871
872 /*
873 * We can't safely remove the tracepoint from the set of active
874 * tracepoints until we've actually removed the fasttrap instruction
875 * from the process's text. We can, however, operate on this
876 * tracepoint secure in the knowledge that no other thread is going to
877 * be looking at it since we hold P_PR_LOCK on the process if it's
878 * live or we hold the provider lock on the process if it's dead and
879 * gone.
880 */
881
882 /*
883 * We only need to remove the actual instruction if we're looking
884 * at an existing process
885 */
886 if (p != NULL) {
887 /*
888 * If we fail to restore the instruction we need to kill
889 * this process since it's in a completely unrecoverable
890 * state.
891 */
892 if (fasttrap_tracepoint_remove(p, tp) != 0)
893 fasttrap_sigtrap(p, NULL, pc);
894
895 /*
896 * Decrement the count of the number of tracepoints active
897 * in the victim process.
898 */
899#if defined(sun)
900 ASSERT(p->p_proc_flag & P_PR_LOCK);
901#endif
902 p->p_dtrace_count--;
903 }
904
905 /*
906 * Remove the probe from the hash table of active tracepoints.
907 */
908 mutex_enter(&bucket->ftb_mtx);
909 pp = (fasttrap_tracepoint_t **)&bucket->ftb_data;
910 ASSERT(*pp != NULL);
911 while (*pp != tp) {
912 pp = &(*pp)->ftt_next;
913 ASSERT(*pp != NULL);
914 }
915
916 *pp = tp->ftt_next;
917 membar_producer();
918
919 mutex_exit(&bucket->ftb_mtx);
920
921 /*
922 * Tag the modified probe with the generation in which it was changed.
923 */
924 probe->ftp_gen = fasttrap_mod_gen;
925}
926
927static void
928fasttrap_enable_callbacks(void)
929{
930 /*
931 * We don't have to play the rw lock game here because we're
932 * providing something rather than taking something away --
933 * we can be sure that no threads have tried to follow this
934 * function pointer yet.
935 */
936 mutex_enter(&fasttrap_count_mtx);
937 if (fasttrap_pid_count == 0) {
938 ASSERT(dtrace_pid_probe_ptr == NULL);
939 ASSERT(dtrace_return_probe_ptr == NULL);
940 dtrace_pid_probe_ptr = &fasttrap_pid_probe;
941 dtrace_return_probe_ptr = &fasttrap_return_probe;
942 }
943 ASSERT(dtrace_pid_probe_ptr == &fasttrap_pid_probe);
944 ASSERT(dtrace_return_probe_ptr == &fasttrap_return_probe);
945 fasttrap_pid_count++;
946 mutex_exit(&fasttrap_count_mtx);
947}
948
949static void
950fasttrap_disable_callbacks(void)
951{
952#if defined(sun)
953 ASSERT(MUTEX_HELD(&cpu_lock));
954#endif
955
956
957 mutex_enter(&fasttrap_count_mtx);
958 ASSERT(fasttrap_pid_count > 0);
959 fasttrap_pid_count--;
960 if (fasttrap_pid_count == 0) {
961#if defined(sun)
962 cpu_t *cur, *cpu = CPU;
963
964 for (cur = cpu->cpu_next_onln; cur != cpu;
965 cur = cur->cpu_next_onln) {
966 rw_enter(&cur->cpu_ft_lock, RW_WRITER);
967 }
968#endif
969 dtrace_pid_probe_ptr = NULL;
970 dtrace_return_probe_ptr = NULL;
971#if defined(sun)
972 for (cur = cpu->cpu_next_onln; cur != cpu;
973 cur = cur->cpu_next_onln) {
974 rw_exit(&cur->cpu_ft_lock);
975 }
976#endif
977 }
978 mutex_exit(&fasttrap_count_mtx);
979}
980
981/*ARGSUSED*/
982static void
983fasttrap_pid_enable(void *arg, dtrace_id_t id, void *parg)
984{
985 fasttrap_probe_t *probe = parg;
986 proc_t *p = NULL;
987 int i, rc;
988
989 ASSERT(probe != NULL);
990 ASSERT(!probe->ftp_enabled);
991 ASSERT(id == probe->ftp_id);
992#if defined(sun)
993 ASSERT(MUTEX_HELD(&cpu_lock));
994#endif
995
996 /*
997 * Increment the count of enabled probes on this probe's provider;
998 * the provider can't go away while the probe still exists. We
999 * must increment this even if we aren't able to properly enable
1000 * this probe.
1001 */
1002 mutex_enter(&probe->ftp_prov->ftp_mtx);
1003 probe->ftp_prov->ftp_rcount++;
1004 mutex_exit(&probe->ftp_prov->ftp_mtx);
1005
1006 /*
1007 * If this probe's provider is retired (meaning it was valid in a
1008 * previously exec'ed incarnation of this address space), bail out. The
1009 * provider can't go away while we're in this code path.
1010 */
1011 if (probe->ftp_prov->ftp_retired)
1012 return;
1013
1014 /*
1015 * If we can't find the process, it may be that we're in the context of
1016 * a fork in which the traced process is being born and we're copying
1017 * USDT probes. Otherwise, the process is gone so bail.
1018 */
1019#if defined(sun)
1020 if ((p = sprlock(probe->ftp_pid)) == NULL) {
1021 if ((curproc->p_flag & SFORKING) == 0)
1022 return;
1023
1024 mutex_enter(&pidlock);
1025 p = prfind(probe->ftp_pid);
1026
1027 /*
1028 * Confirm that curproc is indeed forking the process in which
1029 * we're trying to enable probes.
1030 */
1031 ASSERT(p != NULL);
1032 ASSERT(p->p_parent == curproc);
1033 ASSERT(p->p_stat == SIDL);
1034
1035 mutex_enter(&p->p_lock);
1036 mutex_exit(&pidlock);
1037
1038 sprlock_proc(p);
1039 }
1040
1041 ASSERT(!(p->p_flag & SVFORK));
1042 mutex_exit(&p->p_lock);
1043#else
1044 if ((p = pfind(probe->ftp_pid)) == NULL)
1045 return;
1046#endif
1047
1048 /*
1049 * We have to enable the trap entry point before any user threads have
1050 * the chance to execute the trap instruction we're about to place
1051 * in their process's text.
1052 */
1053#ifdef __FreeBSD__
1054 /*
1055 * pfind() returns a locked process.
1056 */
1057 _PHOLD(p);
1058 PROC_UNLOCK(p);
1059#endif
1060 fasttrap_enable_callbacks();
1061
1062 /*
1063 * Enable all the tracepoints and add this probe's id to each
1064 * tracepoint's list of active probes.
1065 */
1066 for (i = 0; i < probe->ftp_ntps; i++) {
1067 if ((rc = fasttrap_tracepoint_enable(p, probe, i)) != 0) {
1068 /*
1069 * If enabling the tracepoint failed completely,
1070 * we don't have to disable it; if the failure
1071 * was only partial we must disable it.
1072 */
1073 if (rc == FASTTRAP_ENABLE_FAIL)
1074 i--;
1075 else
1076 ASSERT(rc == FASTTRAP_ENABLE_PARTIAL);
1077
1078 /*
1079 * Back up and pull out all the tracepoints we've
1080 * created so far for this probe.
1081 */
1082 while (i >= 0) {
1083 fasttrap_tracepoint_disable(p, probe, i);
1084 i--;
1085 }
1086
1087#if defined(sun)
1088 mutex_enter(&p->p_lock);
1089 sprunlock(p);
1090#else
1091 PRELE(p);
1092#endif
1093
1094 /*
1095 * Since we're not actually enabling this probe,
1096 * drop our reference on the trap table entry.
1097 */
1098 fasttrap_disable_callbacks();
1099 return;
1100 }
1101 }
1102#if defined(sun)
1103 mutex_enter(&p->p_lock);
1104 sprunlock(p);
1105#else
1106 PRELE(p);
1107#endif
1108
1109 probe->ftp_enabled = 1;
1110}
1111
1112/*ARGSUSED*/
1113static void
1114fasttrap_pid_disable(void *arg, dtrace_id_t id, void *parg)
1115{
1116 fasttrap_probe_t *probe = parg;
1117 fasttrap_provider_t *provider = probe->ftp_prov;
1118 proc_t *p;
1119 int i, whack = 0;
1120
1121 ASSERT(id == probe->ftp_id);
1122
1123 mutex_enter(&provider->ftp_mtx);
1124
1125 /*
1126 * We won't be able to acquire a /proc-esque lock on the process
1127 * iff the process is dead and gone. In this case, we rely on the
1128 * provider lock as a point of mutual exclusion to prevent other
1129 * DTrace consumers from disabling this probe.
1130 */
1131 if ((p = pfind(probe->ftp_pid)) != NULL) {
1132#ifdef __FreeBSD__
1133 _PHOLD(p);
1134 PROC_UNLOCK(p);
1135#endif
1136 }
1137
1138 /*
1139 * Disable all the associated tracepoints (for fully enabled probes).
1140 */
1141 if (probe->ftp_enabled) {
1142 for (i = 0; i < probe->ftp_ntps; i++) {
1143 fasttrap_tracepoint_disable(p, probe, i);
1144 }
1145 }
1146
1147 ASSERT(provider->ftp_rcount > 0);
1148 provider->ftp_rcount--;
1149
1150 if (p != NULL) {
1151 /*
1152 * Even though we may not be able to remove it entirely, we
1153 * mark this retired provider to get a chance to remove some
1154 * of the associated probes.
1155 */
1156 if (provider->ftp_retired && !provider->ftp_marked)
1157 whack = provider->ftp_marked = 1;
1158 mutex_exit(&provider->ftp_mtx);
1159 } else {
1160 /*
1161 * If the process is dead, we're just waiting for the
1162 * last probe to be disabled to be able to free it.
1163 */
1164 if (provider->ftp_rcount == 0 && !provider->ftp_marked)
1165 whack = provider->ftp_marked = 1;
1166 mutex_exit(&provider->ftp_mtx);
1167 }
1168
1169 if (whack)
1170 fasttrap_pid_cleanup();
1171
1172#ifdef __FreeBSD__
1173 if (p != NULL)
1174 PRELE(p);
1175#endif
1176 if (!probe->ftp_enabled)
1177 return;
1178
1179 probe->ftp_enabled = 0;
1180
1181#if defined(sun)
1182 ASSERT(MUTEX_HELD(&cpu_lock));
1183#endif
1184 fasttrap_disable_callbacks();
1185}
1186
1187/*ARGSUSED*/
1188static void
1189fasttrap_pid_getargdesc(void *arg, dtrace_id_t id, void *parg,
1190 dtrace_argdesc_t *desc)
1191{
1192 fasttrap_probe_t *probe = parg;
1193 char *str;
1194 int i, ndx;
1195
1196 desc->dtargd_native[0] = '\0';
1197 desc->dtargd_xlate[0] = '\0';
1198
1199 if (probe->ftp_prov->ftp_retired != 0 ||
1200 desc->dtargd_ndx >= probe->ftp_nargs) {
1201 desc->dtargd_ndx = DTRACE_ARGNONE;
1202 return;
1203 }
1204
1205 ndx = (probe->ftp_argmap != NULL) ?
1206 probe->ftp_argmap[desc->dtargd_ndx] : desc->dtargd_ndx;
1207
1208 str = probe->ftp_ntypes;
1209 for (i = 0; i < ndx; i++) {
1210 str += strlen(str) + 1;
1211 }
1212
1213 ASSERT(strlen(str + 1) < sizeof (desc->dtargd_native));
1214 (void) strcpy(desc->dtargd_native, str);
1215
1216 if (probe->ftp_xtypes == NULL)
1217 return;
1218
1219 str = probe->ftp_xtypes;
1220 for (i = 0; i < desc->dtargd_ndx; i++) {
1221 str += strlen(str) + 1;
1222 }
1223
1224 ASSERT(strlen(str + 1) < sizeof (desc->dtargd_xlate));
1225 (void) strcpy(desc->dtargd_xlate, str);
1226}
1227
1228/*ARGSUSED*/
1229static void
1230fasttrap_pid_destroy(void *arg, dtrace_id_t id, void *parg)
1231{
1232 fasttrap_probe_t *probe = parg;
1233 int i;
1234 size_t size;
1235
1236 ASSERT(probe != NULL);
1237 ASSERT(!probe->ftp_enabled);
1238 ASSERT(fasttrap_total >= probe->ftp_ntps);
1239
1240 atomic_add_32(&fasttrap_total, -probe->ftp_ntps);
1241 size = offsetof(fasttrap_probe_t, ftp_tps[probe->ftp_ntps]);
1242
1243 if (probe->ftp_gen + 1 >= fasttrap_mod_gen)
1244 fasttrap_mod_barrier(probe->ftp_gen);
1245
1246 for (i = 0; i < probe->ftp_ntps; i++) {
1247 kmem_free(probe->ftp_tps[i].fit_tp,
1248 sizeof (fasttrap_tracepoint_t));
1249 }
1250
1251 kmem_free(probe, size);
1252}
1253
1254
1255static const dtrace_pattr_t pid_attr = {
1256{ DTRACE_STABILITY_EVOLVING, DTRACE_STABILITY_EVOLVING, DTRACE_CLASS_ISA },
1257{ DTRACE_STABILITY_PRIVATE, DTRACE_STABILITY_PRIVATE, DTRACE_CLASS_UNKNOWN },
1258{ DTRACE_STABILITY_PRIVATE, DTRACE_STABILITY_PRIVATE, DTRACE_CLASS_UNKNOWN },
1259{ DTRACE_STABILITY_EVOLVING, DTRACE_STABILITY_EVOLVING, DTRACE_CLASS_ISA },
1260{ DTRACE_STABILITY_PRIVATE, DTRACE_STABILITY_PRIVATE, DTRACE_CLASS_UNKNOWN },
1261};
1262
1263static dtrace_pops_t pid_pops = {
1264 fasttrap_pid_provide,
1265 NULL,
1266 fasttrap_pid_enable,
1267 fasttrap_pid_disable,
1268 NULL,
1269 NULL,
1270 fasttrap_pid_getargdesc,
1271 fasttrap_pid_getarg,
1272 NULL,
1273 fasttrap_pid_destroy
1274};
1275
1276static dtrace_pops_t usdt_pops = {
1277 fasttrap_pid_provide,
1278 NULL,
1279 fasttrap_pid_enable,
1280 fasttrap_pid_disable,
1281 NULL,
1282 NULL,
1283 fasttrap_pid_getargdesc,
1284 fasttrap_usdt_getarg,
1285 NULL,
1286 fasttrap_pid_destroy
1287};
1288
1289static fasttrap_proc_t *
1290fasttrap_proc_lookup(pid_t pid)
1291{
1292 fasttrap_bucket_t *bucket;
1293 fasttrap_proc_t *fprc, *new_fprc;
1294
1295
1296 bucket = &fasttrap_procs.fth_table[FASTTRAP_PROCS_INDEX(pid)];
1297 mutex_enter(&bucket->ftb_mtx);
1298
1299 for (fprc = bucket->ftb_data; fprc != NULL; fprc = fprc->ftpc_next) {
1300 if (fprc->ftpc_pid == pid && fprc->ftpc_acount != 0) {
1301 mutex_enter(&fprc->ftpc_mtx);
1302 mutex_exit(&bucket->ftb_mtx);
1303 fprc->ftpc_rcount++;
1304 atomic_add_64(&fprc->ftpc_acount, 1);
1305 ASSERT(fprc->ftpc_acount <= fprc->ftpc_rcount);
1306 mutex_exit(&fprc->ftpc_mtx);
1307
1308 return (fprc);
1309 }
1310 }
1311
1312 /*
1313 * Drop the bucket lock so we don't try to perform a sleeping
1314 * allocation under it.
1315 */
1316 mutex_exit(&bucket->ftb_mtx);
1317
1318 new_fprc = kmem_zalloc(sizeof (fasttrap_proc_t), KM_SLEEP);
1319 new_fprc->ftpc_pid = pid;
1320 new_fprc->ftpc_rcount = 1;
1321 new_fprc->ftpc_acount = 1;
1322#if !defined(sun)
1323 mutex_init(&new_fprc->ftpc_mtx, "fasttrap proc mtx", MUTEX_DEFAULT,
1324 NULL);
1325#endif
1326
1327 mutex_enter(&bucket->ftb_mtx);
1328
1329 /*
1330 * Take another lap through the list to make sure a proc hasn't
1331 * been created for this pid while we weren't under the bucket lock.
1332 */
1333 for (fprc = bucket->ftb_data; fprc != NULL; fprc = fprc->ftpc_next) {
1334 if (fprc->ftpc_pid == pid && fprc->ftpc_acount != 0) {
1335 mutex_enter(&fprc->ftpc_mtx);
1336 mutex_exit(&bucket->ftb_mtx);
1337 fprc->ftpc_rcount++;
1338 atomic_add_64(&fprc->ftpc_acount, 1);
1339 ASSERT(fprc->ftpc_acount <= fprc->ftpc_rcount);
1340 mutex_exit(&fprc->ftpc_mtx);
1341
1342 kmem_free(new_fprc, sizeof (fasttrap_proc_t));
1343
1344 return (fprc);
1345 }
1346 }
1347
1348 new_fprc->ftpc_next = bucket->ftb_data;
1349 bucket->ftb_data = new_fprc;
1350
1351 mutex_exit(&bucket->ftb_mtx);
1352
1353 return (new_fprc);
1354}
1355
1356static void
1357fasttrap_proc_release(fasttrap_proc_t *proc)
1358{
1359 fasttrap_bucket_t *bucket;
1360 fasttrap_proc_t *fprc, **fprcp;
1361 pid_t pid = proc->ftpc_pid;
1362
1363 mutex_enter(&proc->ftpc_mtx);
1364
1365 ASSERT(proc->ftpc_rcount != 0);
1366 ASSERT(proc->ftpc_acount <= proc->ftpc_rcount);
1367
1368 if (--proc->ftpc_rcount != 0) {
1369 mutex_exit(&proc->ftpc_mtx);
1370 return;
1371 }
1372
1373 mutex_exit(&proc->ftpc_mtx);
1374
1375 /*
1376 * There should definitely be no live providers associated with this
1377 * process at this point.
1378 */
1379 ASSERT(proc->ftpc_acount == 0);
1380
1381 bucket = &fasttrap_procs.fth_table[FASTTRAP_PROCS_INDEX(pid)];
1382 mutex_enter(&bucket->ftb_mtx);
1383
1384 fprcp = (fasttrap_proc_t **)&bucket->ftb_data;
1385 while ((fprc = *fprcp) != NULL) {
1386 if (fprc == proc)
1387 break;
1388
1389 fprcp = &fprc->ftpc_next;
1390 }
1391
1392 /*
1393 * Something strange has happened if we can't find the proc.
1394 */
1395 ASSERT(fprc != NULL);
1396
1397 *fprcp = fprc->ftpc_next;
1398
1399 mutex_exit(&bucket->ftb_mtx);
1400
1401 kmem_free(fprc, sizeof (fasttrap_proc_t));
1402}
1403
1404/*
1405 * Lookup a fasttrap-managed provider based on its name and associated pid.
1406 * If the pattr argument is non-NULL, this function instantiates the provider
1407 * if it doesn't exist otherwise it returns NULL. The provider is returned
1408 * with its lock held.
1409 */
1410static fasttrap_provider_t *
1411fasttrap_provider_lookup(pid_t pid, const char *name,
1412 const dtrace_pattr_t *pattr)
1413{
1414 fasttrap_provider_t *fp, *new_fp = NULL;
1415 fasttrap_bucket_t *bucket;
1416 char provname[DTRACE_PROVNAMELEN];
1417 proc_t *p;
1418 cred_t *cred;
1419
1420 ASSERT(strlen(name) < sizeof (fp->ftp_name));
1421 ASSERT(pattr != NULL);
1422
1423 bucket = &fasttrap_provs.fth_table[FASTTRAP_PROVS_INDEX(pid, name)];
1424 mutex_enter(&bucket->ftb_mtx);
1425
1426 /*
1427 * Take a lap through the list and return the match if we find it.
1428 */
1429 for (fp = bucket->ftb_data; fp != NULL; fp = fp->ftp_next) {
1430 if (fp->ftp_pid == pid && strcmp(fp->ftp_name, name) == 0 &&
1431 !fp->ftp_retired) {
1432 mutex_enter(&fp->ftp_mtx);
1433 mutex_exit(&bucket->ftb_mtx);
1434 return (fp);
1435 }
1436 }
1437
1438 /*
1439 * Drop the bucket lock so we don't try to perform a sleeping
1440 * allocation under it.
1441 */
1442 mutex_exit(&bucket->ftb_mtx);
1443
1444 /*
1445 * Make sure the process exists, isn't a child created as the result
1446 * of a vfork(2), and isn't a zombie (but may be in fork).
1447 */
1448 if ((p = pfind(pid)) == NULL)
1449 return (NULL);
1450
1451 /*
1452 * Increment p_dtrace_probes so that the process knows to inform us
1453 * when it exits or execs. fasttrap_provider_free() decrements this
1454 * when we're done with this provider.
1455 */
1456 p->p_dtrace_probes++;
1457
1458 /*
1459 * Grab the credentials for this process so we have
1460 * something to pass to dtrace_register().
1461 */
1462 PROC_LOCK_ASSERT(p, MA_OWNED);
1463 crhold(p->p_ucred);
1464 cred = p->p_ucred;
1465 PROC_UNLOCK(p);
1466
1467 new_fp = kmem_zalloc(sizeof (fasttrap_provider_t), KM_SLEEP);
1468 new_fp->ftp_pid = pid;
1469 new_fp->ftp_proc = fasttrap_proc_lookup(pid);
1470#if !defined(sun)
1471 mutex_init(&new_fp->ftp_mtx, "provider mtx", MUTEX_DEFAULT, NULL);
1472 mutex_init(&new_fp->ftp_cmtx, "lock on creating", MUTEX_DEFAULT, NULL);
1473#endif
1474
1475 ASSERT(new_fp->ftp_proc != NULL);
1476
1477 mutex_enter(&bucket->ftb_mtx);
1478
1479 /*
1480 * Take another lap through the list to make sure a provider hasn't
1481 * been created for this pid while we weren't under the bucket lock.
1482 */
1483 for (fp = bucket->ftb_data; fp != NULL; fp = fp->ftp_next) {
1484 if (fp->ftp_pid == pid && strcmp(fp->ftp_name, name) == 0 &&
1485 !fp->ftp_retired) {
1486 mutex_enter(&fp->ftp_mtx);
1487 mutex_exit(&bucket->ftb_mtx);
1488 fasttrap_provider_free(new_fp);
1489 crfree(cred);
1490 return (fp);
1491 }
1492 }
1493
1494 (void) strcpy(new_fp->ftp_name, name);
1495
1496 /*
1497 * Fail and return NULL if either the provider name is too long
1498 * or we fail to register this new provider with the DTrace
1499 * framework. Note that this is the only place we ever construct
1500 * the full provider name -- we keep it in pieces in the provider
1501 * structure.
1502 */
1503 if (snprintf(provname, sizeof (provname), "%s%u", name, (uint_t)pid) >=
1504 sizeof (provname) ||
1505 dtrace_register(provname, pattr,
1506 DTRACE_PRIV_PROC | DTRACE_PRIV_OWNER | DTRACE_PRIV_ZONEOWNER, cred,
1507 pattr == &pid_attr ? &pid_pops : &usdt_pops, new_fp,
1508 &new_fp->ftp_provid) != 0) {
1509 mutex_exit(&bucket->ftb_mtx);
1510 fasttrap_provider_free(new_fp);
1511 crfree(cred);
1512 return (NULL);
1513 }
1514
1515 new_fp->ftp_next = bucket->ftb_data;
1516 bucket->ftb_data = new_fp;
1517
1518 mutex_enter(&new_fp->ftp_mtx);
1519 mutex_exit(&bucket->ftb_mtx);
1520
1521 crfree(cred);
1522 return (new_fp);
1523}
1524
1525static void
1526fasttrap_provider_free(fasttrap_provider_t *provider)
1527{
1528 pid_t pid = provider->ftp_pid;
1529 proc_t *p;
1530
1531 /*
1532 * There need to be no associated enabled probes, no consumers
1533 * creating probes, and no meta providers referencing this provider.
1534 */
1535 ASSERT(provider->ftp_rcount == 0);
1536 ASSERT(provider->ftp_ccount == 0);
1537 ASSERT(provider->ftp_mcount == 0);
1538
1539 /*
1540 * If this provider hasn't been retired, we need to explicitly drop the
1541 * count of active providers on the associated process structure.
1542 */
1543 if (!provider->ftp_retired) {
1544 atomic_add_64(&provider->ftp_proc->ftpc_acount, -1);
1545 ASSERT(provider->ftp_proc->ftpc_acount <
1546 provider->ftp_proc->ftpc_rcount);
1547 }
1548
1549 fasttrap_proc_release(provider->ftp_proc);
1550
1551#if !defined(sun)
1552 mutex_destroy(&provider->ftp_mtx);
1553 mutex_destroy(&provider->ftp_cmtx);
1554#endif
1555 kmem_free(provider, sizeof (fasttrap_provider_t));
1556
1557 /*
1558 * Decrement p_dtrace_probes on the process whose provider we're
1559 * freeing. We don't have to worry about clobbering somone else's
1560 * modifications to it because we have locked the bucket that
1561 * corresponds to this process's hash chain in the provider hash
1562 * table. Don't sweat it if we can't find the process.
1563 */
1564 if ((p = pfind(pid)) == NULL) {
1565 return;
1566 }
1567
1568 p->p_dtrace_probes--;
1569#if !defined(sun)
1570 PROC_UNLOCK(p);
1571#endif
1572}
1573
1574static void
1575fasttrap_provider_retire(pid_t pid, const char *name, int mprov)
1576{
1577 fasttrap_provider_t *fp;
1578 fasttrap_bucket_t *bucket;
1579 dtrace_provider_id_t provid;
1580
1581 ASSERT(strlen(name) < sizeof (fp->ftp_name));
1582
1583 bucket = &fasttrap_provs.fth_table[FASTTRAP_PROVS_INDEX(pid, name)];
1584 mutex_enter(&bucket->ftb_mtx);
1585
1586 for (fp = bucket->ftb_data; fp != NULL; fp = fp->ftp_next) {
1587 if (fp->ftp_pid == pid && strcmp(fp->ftp_name, name) == 0 &&
1588 !fp->ftp_retired)
1589 break;
1590 }
1591
1592 if (fp == NULL) {
1593 mutex_exit(&bucket->ftb_mtx);
1594 return;
1595 }
1596
1597 mutex_enter(&fp->ftp_mtx);
1598 ASSERT(!mprov || fp->ftp_mcount > 0);
1599 if (mprov && --fp->ftp_mcount != 0) {
1600 mutex_exit(&fp->ftp_mtx);
1601 mutex_exit(&bucket->ftb_mtx);
1602 return;
1603 }
1604
1605 /*
1606 * Mark the provider to be removed in our post-processing step, mark it
1607 * retired, and drop the active count on its proc. Marking it indicates
1608 * that we should try to remove it; setting the retired flag indicates
1609 * that we're done with this provider; dropping the active the proc
1610 * releases our hold, and when this reaches zero (as it will during
1611 * exit or exec) the proc and associated providers become defunct.
1612 *
1613 * We obviously need to take the bucket lock before the provider lock
1614 * to perform the lookup, but we need to drop the provider lock
1615 * before calling into the DTrace framework since we acquire the
1616 * provider lock in callbacks invoked from the DTrace framework. The
1617 * bucket lock therefore protects the integrity of the provider hash
1618 * table.
1619 */
1620 atomic_add_64(&fp->ftp_proc->ftpc_acount, -1);
1621 ASSERT(fp->ftp_proc->ftpc_acount < fp->ftp_proc->ftpc_rcount);
1622
1623 fp->ftp_retired = 1;
1624 fp->ftp_marked = 1;
1625 provid = fp->ftp_provid;
1626 mutex_exit(&fp->ftp_mtx);
1627
1628 /*
1629 * We don't have to worry about invalidating the same provider twice
1630 * since fasttrap_provider_lookup() will ignore provider that have
1631 * been marked as retired.
1632 */
1633 dtrace_invalidate(provid);
1634
1635 mutex_exit(&bucket->ftb_mtx);
1636
1637 fasttrap_pid_cleanup();
1638}
1639
1640static int
1641fasttrap_uint32_cmp(const void *ap, const void *bp)
1642{
1643 return (*(const uint32_t *)ap - *(const uint32_t *)bp);
1644}
1645
1646static int
1647fasttrap_uint64_cmp(const void *ap, const void *bp)
1648{
1649 return (*(const uint64_t *)ap - *(const uint64_t *)bp);
1650}
1651
1652static int
1653fasttrap_add_probe(fasttrap_probe_spec_t *pdata)
1654{
1655 fasttrap_provider_t *provider;
1656 fasttrap_probe_t *pp;
1657 fasttrap_tracepoint_t *tp;
1658 char *name;
1659 int i, aframes = 0, whack;
1660
1661 /*
1662 * There needs to be at least one desired trace point.
1663 */
1664 if (pdata->ftps_noffs == 0)
1665 return (EINVAL);
1666
1667 switch (pdata->ftps_type) {
1668 case DTFTP_ENTRY:
1669 name = "entry";
1670 aframes = FASTTRAP_ENTRY_AFRAMES;
1671 break;
1672 case DTFTP_RETURN:
1673 name = "return";
1674 aframes = FASTTRAP_RETURN_AFRAMES;
1675 break;
1676 case DTFTP_OFFSETS:
1677 name = NULL;
1678 break;
1679 default:
1680 return (EINVAL);
1681 }
1682
1683 if ((provider = fasttrap_provider_lookup(pdata->ftps_pid,
1684 FASTTRAP_PID_NAME, &pid_attr)) == NULL)
1685 return (ESRCH);
1686
1687 /*
1688 * Increment this reference count to indicate that a consumer is
1689 * actively adding a new probe associated with this provider. This
1690 * prevents the provider from being deleted -- we'll need to check
1691 * for pending deletions when we drop this reference count.
1692 */
1693 provider->ftp_ccount++;
1694 mutex_exit(&provider->ftp_mtx);
1695
1696 /*
1697 * Grab the creation lock to ensure consistency between calls to
1698 * dtrace_probe_lookup() and dtrace_probe_create() in the face of
1699 * other threads creating probes. We must drop the provider lock
1700 * before taking this lock to avoid a three-way deadlock with the
1701 * DTrace framework.
1702 */
1703 mutex_enter(&provider->ftp_cmtx);
1704
1705 if (name == NULL) {
1706 for (i = 0; i < pdata->ftps_noffs; i++) {
1707 char name_str[17];
1708
1709 (void) sprintf(name_str, "%llx",
1710 (unsigned long long)pdata->ftps_offs[i]);
1711
1712 if (dtrace_probe_lookup(provider->ftp_provid,
1713 pdata->ftps_mod, pdata->ftps_func, name_str) != 0)
1714 continue;
1715
1716 atomic_add_32(&fasttrap_total, 1);
1717
1718 if (fasttrap_total > fasttrap_max) {
1719 atomic_add_32(&fasttrap_total, -1);
1720 goto no_mem;
1721 }
1722
1723 pp = kmem_zalloc(sizeof (fasttrap_probe_t), KM_SLEEP);
1724
1725 pp->ftp_prov = provider;
1726 pp->ftp_faddr = pdata->ftps_pc;
1727 pp->ftp_fsize = pdata->ftps_size;
1728 pp->ftp_pid = pdata->ftps_pid;
1729 pp->ftp_ntps = 1;
1730
1731 tp = kmem_zalloc(sizeof (fasttrap_tracepoint_t),
1732 KM_SLEEP);
1733
1734 tp->ftt_proc = provider->ftp_proc;
1735 tp->ftt_pc = pdata->ftps_offs[i] + pdata->ftps_pc;
1736 tp->ftt_pid = pdata->ftps_pid;
1737
1738 pp->ftp_tps[0].fit_tp = tp;
1739 pp->ftp_tps[0].fit_id.fti_probe = pp;
1740 pp->ftp_tps[0].fit_id.fti_ptype = pdata->ftps_type;
1741
1742 pp->ftp_id = dtrace_probe_create(provider->ftp_provid,
1743 pdata->ftps_mod, pdata->ftps_func, name_str,
1744 FASTTRAP_OFFSET_AFRAMES, pp);
1745 }
1746
1747 } else if (dtrace_probe_lookup(provider->ftp_provid, pdata->ftps_mod,
1748 pdata->ftps_func, name) == 0) {
1749 atomic_add_32(&fasttrap_total, pdata->ftps_noffs);
1750
1751 if (fasttrap_total > fasttrap_max) {
1752 atomic_add_32(&fasttrap_total, -pdata->ftps_noffs);
1753 goto no_mem;
1754 }
1755
1756 /*
1757 * Make sure all tracepoint program counter values are unique.
1758 * We later assume that each probe has exactly one tracepoint
1759 * for a given pc.
1760 */
1761 qsort(pdata->ftps_offs, pdata->ftps_noffs,
1762 sizeof (uint64_t), fasttrap_uint64_cmp);
1763 for (i = 1; i < pdata->ftps_noffs; i++) {
1764 if (pdata->ftps_offs[i] > pdata->ftps_offs[i - 1])
1765 continue;
1766
1767 atomic_add_32(&fasttrap_total, -pdata->ftps_noffs);
1768 goto no_mem;
1769 }
1770
1771 ASSERT(pdata->ftps_noffs > 0);
1772 pp = kmem_zalloc(offsetof(fasttrap_probe_t,
1773 ftp_tps[pdata->ftps_noffs]), KM_SLEEP);
1774
1775 pp->ftp_prov = provider;
1776 pp->ftp_faddr = pdata->ftps_pc;
1777 pp->ftp_fsize = pdata->ftps_size;
1778 pp->ftp_pid = pdata->ftps_pid;
1779 pp->ftp_ntps = pdata->ftps_noffs;
1780
1781 for (i = 0; i < pdata->ftps_noffs; i++) {
1782 tp = kmem_zalloc(sizeof (fasttrap_tracepoint_t),
1783 KM_SLEEP);
1784
1785 tp->ftt_proc = provider->ftp_proc;
1786 tp->ftt_pc = pdata->ftps_offs[i] + pdata->ftps_pc;
1787 tp->ftt_pid = pdata->ftps_pid;
1788
1789 pp->ftp_tps[i].fit_tp = tp;
1790 pp->ftp_tps[i].fit_id.fti_probe = pp;
1791 pp->ftp_tps[i].fit_id.fti_ptype = pdata->ftps_type;
1792 }
1793
1794 pp->ftp_id = dtrace_probe_create(provider->ftp_provid,
1795 pdata->ftps_mod, pdata->ftps_func, name, aframes, pp);
1796 }
1797
1798 mutex_exit(&provider->ftp_cmtx);
1799
1800 /*
1801 * We know that the provider is still valid since we incremented the
1802 * creation reference count. If someone tried to clean up this provider
1803 * while we were using it (e.g. because the process called exec(2) or
1804 * exit(2)), take note of that and try to clean it up now.
1805 */
1806 mutex_enter(&provider->ftp_mtx);
1807 provider->ftp_ccount--;
1808 whack = provider->ftp_retired;
1809 mutex_exit(&provider->ftp_mtx);
1810
1811 if (whack)
1812 fasttrap_pid_cleanup();
1813
1814 return (0);
1815
1816no_mem:
1817 /*
1818 * If we've exhausted the allowable resources, we'll try to remove
1819 * this provider to free some up. This is to cover the case where
1820 * the user has accidentally created many more probes than was
1821 * intended (e.g. pid123:::).
1822 */
1823 mutex_exit(&provider->ftp_cmtx);
1824 mutex_enter(&provider->ftp_mtx);
1825 provider->ftp_ccount--;
1826 provider->ftp_marked = 1;
1827 mutex_exit(&provider->ftp_mtx);
1828
1829 fasttrap_pid_cleanup();
1830
1831 return (ENOMEM);
1832}
1833
1834/*ARGSUSED*/
1835static void *
1836fasttrap_meta_provide(void *arg, dtrace_helper_provdesc_t *dhpv, pid_t pid)
1837{
1838 fasttrap_provider_t *provider;
1839
1840 /*
1841 * A 32-bit unsigned integer (like a pid for example) can be
1842 * expressed in 10 or fewer decimal digits. Make sure that we'll
1843 * have enough space for the provider name.
1844 */
1845 if (strlen(dhpv->dthpv_provname) + 10 >=
1846 sizeof (provider->ftp_name)) {
1847 printf("failed to instantiate provider %s: "
1848 "name too long to accomodate pid", dhpv->dthpv_provname);
1849 return (NULL);
1850 }
1851
1852 /*
1853 * Don't let folks spoof the true pid provider.
1854 */
1855 if (strcmp(dhpv->dthpv_provname, FASTTRAP_PID_NAME) == 0) {
1856 printf("failed to instantiate provider %s: "
1857 "%s is an invalid name", dhpv->dthpv_provname,
1858 FASTTRAP_PID_NAME);
1859 return (NULL);
1860 }
1861
1862 /*
1863 * The highest stability class that fasttrap supports is ISA; cap
1864 * the stability of the new provider accordingly.
1865 */
1866 if (dhpv->dthpv_pattr.dtpa_provider.dtat_class > DTRACE_CLASS_ISA)
1867 dhpv->dthpv_pattr.dtpa_provider.dtat_class = DTRACE_CLASS_ISA;
1868 if (dhpv->dthpv_pattr.dtpa_mod.dtat_class > DTRACE_CLASS_ISA)
1869 dhpv->dthpv_pattr.dtpa_mod.dtat_class = DTRACE_CLASS_ISA;
1870 if (dhpv->dthpv_pattr.dtpa_func.dtat_class > DTRACE_CLASS_ISA)
1871 dhpv->dthpv_pattr.dtpa_func.dtat_class = DTRACE_CLASS_ISA;
1872 if (dhpv->dthpv_pattr.dtpa_name.dtat_class > DTRACE_CLASS_ISA)
1873 dhpv->dthpv_pattr.dtpa_name.dtat_class = DTRACE_CLASS_ISA;
1874 if (dhpv->dthpv_pattr.dtpa_args.dtat_class > DTRACE_CLASS_ISA)
1875 dhpv->dthpv_pattr.dtpa_args.dtat_class = DTRACE_CLASS_ISA;
1876
1877 if ((provider = fasttrap_provider_lookup(pid, dhpv->dthpv_provname,
1878 &dhpv->dthpv_pattr)) == NULL) {
1879 printf("failed to instantiate provider %s for "
1880 "process %u", dhpv->dthpv_provname, (uint_t)pid);
1881 return (NULL);
1882 }
1883
1884 /*
1885 * Up the meta provider count so this provider isn't removed until
1886 * the meta provider has been told to remove it.
1887 */
1888 provider->ftp_mcount++;
1889
1890 mutex_exit(&provider->ftp_mtx);
1891
1892 return (provider);
1893}
1894
1895/*ARGSUSED*/
1896static void
1897fasttrap_meta_create_probe(void *arg, void *parg,
1898 dtrace_helper_probedesc_t *dhpb)
1899{
1900 fasttrap_provider_t *provider = parg;
1901 fasttrap_probe_t *pp;
1902 fasttrap_tracepoint_t *tp;
1903 int i, j;
1904 uint32_t ntps;
1905
1906 /*
1907 * Since the meta provider count is non-zero we don't have to worry
1908 * about this provider disappearing.
1909 */
1910 ASSERT(provider->ftp_mcount > 0);
1911
1912 /*
1913 * The offsets must be unique.
1914 */
1915 qsort(dhpb->dthpb_offs, dhpb->dthpb_noffs, sizeof (uint32_t),
1916 fasttrap_uint32_cmp);
1917 for (i = 1; i < dhpb->dthpb_noffs; i++) {
1918 if (dhpb->dthpb_base + dhpb->dthpb_offs[i] <=
1919 dhpb->dthpb_base + dhpb->dthpb_offs[i - 1])
1920 return;
1921 }
1922
1923 qsort(dhpb->dthpb_enoffs, dhpb->dthpb_nenoffs, sizeof (uint32_t),
1924 fasttrap_uint32_cmp);
1925 for (i = 1; i < dhpb->dthpb_nenoffs; i++) {
1926 if (dhpb->dthpb_base + dhpb->dthpb_enoffs[i] <=
1927 dhpb->dthpb_base + dhpb->dthpb_enoffs[i - 1])
1928 return;
1929 }
1930
1931 /*
1932 * Grab the creation lock to ensure consistency between calls to
1933 * dtrace_probe_lookup() and dtrace_probe_create() in the face of
1934 * other threads creating probes.
1935 */
1936 mutex_enter(&provider->ftp_cmtx);
1937
1938 if (dtrace_probe_lookup(provider->ftp_provid, dhpb->dthpb_mod,
1939 dhpb->dthpb_func, dhpb->dthpb_name) != 0) {
1940 mutex_exit(&provider->ftp_cmtx);
1941 return;
1942 }
1943
1944 ntps = dhpb->dthpb_noffs + dhpb->dthpb_nenoffs;
1945 ASSERT(ntps > 0);
1946
1947 atomic_add_32(&fasttrap_total, ntps);
1948
1949 if (fasttrap_total > fasttrap_max) {
1950 atomic_add_32(&fasttrap_total, -ntps);
1951 mutex_exit(&provider->ftp_cmtx);
1952 return;
1953 }
1954
1955 pp = kmem_zalloc(offsetof(fasttrap_probe_t, ftp_tps[ntps]), KM_SLEEP);
1956
1957 pp->ftp_prov = provider;
1958 pp->ftp_pid = provider->ftp_pid;
1959 pp->ftp_ntps = ntps;
1960 pp->ftp_nargs = dhpb->dthpb_xargc;
1961 pp->ftp_xtypes = dhpb->dthpb_xtypes;
1962 pp->ftp_ntypes = dhpb->dthpb_ntypes;
1963
1964 /*
1965 * First create a tracepoint for each actual point of interest.
1966 */
1967 for (i = 0; i < dhpb->dthpb_noffs; i++) {
1968 tp = kmem_zalloc(sizeof (fasttrap_tracepoint_t), KM_SLEEP);
1969
1970 tp->ftt_proc = provider->ftp_proc;
1971 tp->ftt_pc = dhpb->dthpb_base + dhpb->dthpb_offs[i];
1972 tp->ftt_pid = provider->ftp_pid;
1973
1974 pp->ftp_tps[i].fit_tp = tp;
1975 pp->ftp_tps[i].fit_id.fti_probe = pp;
1976#ifdef __sparc
1977 pp->ftp_tps[i].fit_id.fti_ptype = DTFTP_POST_OFFSETS;
1978#else
1979 pp->ftp_tps[i].fit_id.fti_ptype = DTFTP_OFFSETS;
1980#endif
1981 }
1982
1983 /*
1984 * Then create a tracepoint for each is-enabled point.
1985 */
1986 for (j = 0; i < ntps; i++, j++) {
1987 tp = kmem_zalloc(sizeof (fasttrap_tracepoint_t), KM_SLEEP);
1988
1989 tp->ftt_proc = provider->ftp_proc;
1990 tp->ftt_pc = dhpb->dthpb_base + dhpb->dthpb_enoffs[j];
1991 tp->ftt_pid = provider->ftp_pid;
1992
1993 pp->ftp_tps[i].fit_tp = tp;
1994 pp->ftp_tps[i].fit_id.fti_probe = pp;
1995 pp->ftp_tps[i].fit_id.fti_ptype = DTFTP_IS_ENABLED;
1996 }
1997
1998 /*
1999 * If the arguments are shuffled around we set the argument remapping
2000 * table. Later, when the probe fires, we only remap the arguments
2001 * if the table is non-NULL.
2002 */
2003 for (i = 0; i < dhpb->dthpb_xargc; i++) {
2004 if (dhpb->dthpb_args[i] != i) {
2005 pp->ftp_argmap = dhpb->dthpb_args;
2006 break;
2007 }
2008 }
2009
2010 /*
2011 * The probe is fully constructed -- register it with DTrace.
2012 */
2013 pp->ftp_id = dtrace_probe_create(provider->ftp_provid, dhpb->dthpb_mod,
2014 dhpb->dthpb_func, dhpb->dthpb_name, FASTTRAP_OFFSET_AFRAMES, pp);
2015
2016 mutex_exit(&provider->ftp_cmtx);
2017}
2018
2019/*ARGSUSED*/
2020static void
2021fasttrap_meta_remove(void *arg, dtrace_helper_provdesc_t *dhpv, pid_t pid)
2022{
2023 /*
2024 * Clean up the USDT provider. There may be active consumers of the
2025 * provider busy adding probes, no damage will actually befall the
2026 * provider until that count has dropped to zero. This just puts
2027 * the provider on death row.
2028 */
2029 fasttrap_provider_retire(pid, dhpv->dthpv_provname, 1);
2030}
2031
2032static dtrace_mops_t fasttrap_mops = {
2033 fasttrap_meta_create_probe,
2034 fasttrap_meta_provide,
2035 fasttrap_meta_remove
2036};
2037
2038/*ARGSUSED*/
2039static int
2040fasttrap_open(struct cdev *dev __unused, int oflags __unused,
2041 int devtype __unused, struct thread *td __unused)
2042{
2043 return (0);
2044}
2045
2046/*ARGSUSED*/
2047static int
2048fasttrap_ioctl(struct cdev *dev, u_long cmd, caddr_t arg, int fflag,
2049 struct thread *td)
2050{
2051#ifdef notyet
2052 struct kinfo_proc kp;
2053 const cred_t *cr = td->td_ucred;
2054#endif
2055 if (!dtrace_attached())
2056 return (EAGAIN);
2057
2058 if (cmd == FASTTRAPIOC_MAKEPROBE) {
2059 fasttrap_probe_spec_t *uprobe = (void *)arg;
2060 fasttrap_probe_spec_t *probe;
2061 uint64_t noffs;
2062 size_t size;
2063 int ret;
2064 char *c;
2065
2066#if defined(sun)
2067 if (copyin(&uprobe->ftps_noffs, &noffs,
2068 sizeof (uprobe->ftps_noffs)))
2069 return (EFAULT);
2070#else
2071 noffs = uprobe->ftps_noffs;
2072#endif
2073
2074 /*
2075 * Probes must have at least one tracepoint.
2076 */
2077 if (noffs == 0)
2078 return (EINVAL);
2079
2080 size = sizeof (fasttrap_probe_spec_t) +
2081 sizeof (probe->ftps_offs[0]) * (noffs - 1);
2082
2083 if (size > 1024 * 1024)
2084 return (ENOMEM);
2085
2086 probe = kmem_alloc(size, KM_SLEEP);
2087
2088#if defined(sun)
2089 if (copyin(uprobe, probe, size) != 0) {
2090 kmem_free(probe, size);
2091 return (EFAULT);
2092 }
2093#else
2094 memcpy(probe, uprobe, sizeof(*probe));
2095 if (noffs > 1 && copyin(uprobe + 1, probe + 1, size) != 0) {
2096 kmem_free(probe, size);
2097 return (EFAULT);
2098 }
2099#endif
2100
2101
2102 /*
2103 * Verify that the function and module strings contain no
2104 * funny characters.
2105 */
2106 for (c = &probe->ftps_func[0]; *c != '\0'; c++) {
2107 if (*c < 0x20 || 0x7f <= *c) {
2108 ret = EINVAL;
2109 goto err;
2110 }
2111 }
2112
2113 for (c = &probe->ftps_mod[0]; *c != '\0'; c++) {
2114 if (*c < 0x20 || 0x7f <= *c) {
2115 ret = EINVAL;
2116 goto err;
2117 }
2118 }
2119
2120#ifdef notyet
2121 if (!PRIV_POLICY_CHOICE(cr, PRIV_ALL, B_FALSE)) {
2122 proc_t *p;
2123 pid_t pid = probe->ftps_pid;
2124
2125#if defined(sun)
2126 mutex_enter(&pidlock);
2127#endif
2128 /*
2129 * Report an error if the process doesn't exist
2130 * or is actively being birthed.
2131 */
2132 p = pfind(pid);
2133 if (p)
2134 fill_kinfo_proc(p, &kp);
2135 if (p == NULL || kp.ki_stat == SIDL) {
2136#if defined(sun)
2137 mutex_exit(&pidlock);
2138#endif
2139 return (ESRCH);
2140 }
2141#if defined(sun)
2142 mutex_enter(&p->p_lock);
2143 mutex_exit(&pidlock);
2144#else
2145 PROC_LOCK_ASSERT(p, MA_OWNED);
2146#endif
2147
2148#ifdef notyet
2149 if ((ret = priv_proc_cred_perm(cr, p, NULL,
2150 VREAD | VWRITE)) != 0) {
2151#if defined(sun)
2152 mutex_exit(&p->p_lock);
2153#else
2154 PROC_UNLOCK(p);
2155#endif
2156 return (ret);
2157 }
2158#endif /* notyet */
2159#if defined(sun)
2160 mutex_exit(&p->p_lock);
2161#else
2162 PROC_UNLOCK(p);
2163#endif
2164 }
2165#endif /* notyet */
2166
2167 ret = fasttrap_add_probe(probe);
2168err:
2169 kmem_free(probe, size);
2170
2171 return (ret);
2172
2173 } else if (cmd == FASTTRAPIOC_GETINSTR) {
2174 fasttrap_instr_query_t instr;
2175 fasttrap_tracepoint_t *tp;
2176 uint_t index;
2177#if defined(sun)
2178 int ret;
2179#endif
2180
2181#if defined(sun)
2182 if (copyin((void *)arg, &instr, sizeof (instr)) != 0)
2183 return (EFAULT);
2184#endif
2185
2186#ifdef notyet
2187 if (!PRIV_POLICY_CHOICE(cr, PRIV_ALL, B_FALSE)) {
2188 proc_t *p;
2189 pid_t pid = instr.ftiq_pid;
2190
2191#if defined(sun)
2192 mutex_enter(&pidlock);
2193#endif
2194 /*
2195 * Report an error if the process doesn't exist
2196 * or is actively being birthed.
2197 */
2198 p = pfind(pid);
2199 if (p)
2200 fill_kinfo_proc(p, &kp);
2201 if (p == NULL || kp.ki_stat == SIDL) {
2202#if defined(sun)
2203 mutex_exit(&pidlock);
2204#endif
2205 return (ESRCH);
2206 }
2207#if defined(sun)
2208 mutex_enter(&p->p_lock);
2209 mutex_exit(&pidlock);
2210#else
2211 PROC_LOCK_ASSERT(p, MA_OWNED);
2212#endif
2213
2214#ifdef notyet
2215 if ((ret = priv_proc_cred_perm(cr, p, NULL,
2216 VREAD)) != 0) {
2217#if defined(sun)
2218 mutex_exit(&p->p_lock);
2219#else
2220 PROC_UNLOCK(p);
2221#endif
2222 return (ret);
2223 }
2224#endif /* notyet */
2225
2226#if defined(sun)
2227 mutex_exit(&p->p_lock);
2228#else
2229 PROC_UNLOCK(p);
2230#endif
2231 }
2232#endif /* notyet */
2233
2234 index = FASTTRAP_TPOINTS_INDEX(instr.ftiq_pid, instr.ftiq_pc);
2235
2236 mutex_enter(&fasttrap_tpoints.fth_table[index].ftb_mtx);
2237 tp = fasttrap_tpoints.fth_table[index].ftb_data;
2238 while (tp != NULL) {
2239 if (instr.ftiq_pid == tp->ftt_pid &&
2240 instr.ftiq_pc == tp->ftt_pc &&
2241 tp->ftt_proc->ftpc_acount != 0)
2242 break;
2243
2244 tp = tp->ftt_next;
2245 }
2246
2247 if (tp == NULL) {
2248 mutex_exit(&fasttrap_tpoints.fth_table[index].ftb_mtx);
2249 return (ENOENT);
2250 }
2251
2252 bcopy(&tp->ftt_instr, &instr.ftiq_instr,
2253 sizeof (instr.ftiq_instr));
2254 mutex_exit(&fasttrap_tpoints.fth_table[index].ftb_mtx);
2255
2256 if (copyout(&instr, (void *)arg, sizeof (instr)) != 0)
2257 return (EFAULT);
2258
2259 return (0);
2260 }
2261
2262 return (EINVAL);
2263}
2264
2265static int
2266fasttrap_load(void)
2267{
2268 ulong_t nent;
2269 int i, ret;
2270
2271 /* Create the /dev/dtrace/fasttrap entry. */
2272 fasttrap_cdev = make_dev(&fasttrap_cdevsw, 0, UID_ROOT, GID_WHEEL, 0600,
2273 "dtrace/fasttrap");
2274
2275 mtx_init(&fasttrap_cleanup_mtx, "fasttrap clean", "dtrace", MTX_DEF);
2276 mutex_init(&fasttrap_count_mtx, "fasttrap count mtx", MUTEX_DEFAULT,
2277 NULL);
2278
2279 ret = kproc_create(fasttrap_pid_cleanup_cb, NULL,
2280 &fasttrap_cleanup_proc, 0, 0, "ftcleanup");
2281 if (ret != 0) {
2282 destroy_dev(fasttrap_cdev);
2283 return (ret);
2284 }
2285
2286 /*
2287 * Install our hooks into fork(2), exec(2), and exit(2).
2288 */
2289 dtrace_fasttrap_fork = &fasttrap_fork;
2290 dtrace_fasttrap_exit = &fasttrap_exec_exit;
2291 dtrace_fasttrap_exec = &fasttrap_exec_exit;
2292
2293#if defined(sun)
2294 fasttrap_max = ddi_getprop(DDI_DEV_T_ANY, devi, DDI_PROP_DONTPASS,
2295 "fasttrap-max-probes", FASTTRAP_MAX_DEFAULT);
2296#else
2297 fasttrap_max = FASTTRAP_MAX_DEFAULT;
2298#endif
2299 fasttrap_total = 0;
2300
2301 /*
2302 * Conjure up the tracepoints hashtable...
2303 */
2304#if defined(sun)
2305 nent = ddi_getprop(DDI_DEV_T_ANY, devi, DDI_PROP_DONTPASS,
2306 "fasttrap-hash-size", FASTTRAP_TPOINTS_DEFAULT_SIZE);
2307#else
2308 nent = FASTTRAP_TPOINTS_DEFAULT_SIZE;
2309#endif
2310
2311 if (nent == 0 || nent > 0x1000000)
2312 nent = FASTTRAP_TPOINTS_DEFAULT_SIZE;
2313
2314 if ((nent & (nent - 1)) == 0)
2315 fasttrap_tpoints.fth_nent = nent;
2316 else
2317 fasttrap_tpoints.fth_nent = 1 << fasttrap_highbit(nent);
2318 ASSERT(fasttrap_tpoints.fth_nent > 0);
2319 fasttrap_tpoints.fth_mask = fasttrap_tpoints.fth_nent - 1;
2320 fasttrap_tpoints.fth_table = kmem_zalloc(fasttrap_tpoints.fth_nent *
2321 sizeof (fasttrap_bucket_t), KM_SLEEP);
2322#if !defined(sun)
2323 for (i = 0; i < fasttrap_tpoints.fth_nent; i++)
2324 mutex_init(&fasttrap_tpoints.fth_table[i].ftb_mtx,
2325 "tracepoints bucket mtx", MUTEX_DEFAULT, NULL);
2326#endif
2327
2328 /*
2329 * ... and the providers hash table...
2330 */
2331 nent = FASTTRAP_PROVIDERS_DEFAULT_SIZE;
2332 if ((nent & (nent - 1)) == 0)
2333 fasttrap_provs.fth_nent = nent;
2334 else
2335 fasttrap_provs.fth_nent = 1 << fasttrap_highbit(nent);
2336 ASSERT(fasttrap_provs.fth_nent > 0);
2337 fasttrap_provs.fth_mask = fasttrap_provs.fth_nent - 1;
2338 fasttrap_provs.fth_table = kmem_zalloc(fasttrap_provs.fth_nent *
2339 sizeof (fasttrap_bucket_t), KM_SLEEP);
2340#if !defined(sun)
2341 for (i = 0; i < fasttrap_provs.fth_nent; i++)
2342 mutex_init(&fasttrap_provs.fth_table[i].ftb_mtx,
2343 "providers bucket mtx", MUTEX_DEFAULT, NULL);
2344#endif
2345
2346 /*
2347 * ... and the procs hash table.
2348 */
2349 nent = FASTTRAP_PROCS_DEFAULT_SIZE;
2350 if ((nent & (nent - 1)) == 0)
2351 fasttrap_procs.fth_nent = nent;
2352 else
2353 fasttrap_procs.fth_nent = 1 << fasttrap_highbit(nent);
2354 ASSERT(fasttrap_procs.fth_nent > 0);
2355 fasttrap_procs.fth_mask = fasttrap_procs.fth_nent - 1;
2356 fasttrap_procs.fth_table = kmem_zalloc(fasttrap_procs.fth_nent *
2357 sizeof (fasttrap_bucket_t), KM_SLEEP);
2358#if !defined(sun)
2359 for (i = 0; i < fasttrap_procs.fth_nent; i++)
2360 mutex_init(&fasttrap_procs.fth_table[i].ftb_mtx,
2361 "processes bucket mtx", MUTEX_DEFAULT, NULL);
2362
2363 CPU_FOREACH(i) {
2364 mutex_init(&fasttrap_cpuc_pid_lock[i], "fasttrap barrier",
2365 MUTEX_DEFAULT, NULL);
2366 }
2367#endif
2368
2369 (void) dtrace_meta_register("fasttrap", &fasttrap_mops, NULL,
2370 &fasttrap_meta_id);
2371
2372 return (0);
2373}
2374
2375static int
2376fasttrap_unload(void)
2377{
2378 int i, fail = 0;
2379
2380 /*
2381 * Unregister the meta-provider to make sure no new fasttrap-
2382 * managed providers come along while we're trying to close up
2383 * shop. If we fail to detach, we'll need to re-register as a
2384 * meta-provider. We can fail to unregister as a meta-provider
2385 * if providers we manage still exist.
2386 */
2387 if (fasttrap_meta_id != DTRACE_METAPROVNONE &&
2388 dtrace_meta_unregister(fasttrap_meta_id) != 0)
2389 return (-1);
2390
2391 /*
2392 * Iterate over all of our providers. If there's still a process
2393 * that corresponds to that pid, fail to detach.
2394 */
2395 for (i = 0; i < fasttrap_provs.fth_nent; i++) {
2396 fasttrap_provider_t **fpp, *fp;
2397 fasttrap_bucket_t *bucket = &fasttrap_provs.fth_table[i];
2398
2399 mutex_enter(&bucket->ftb_mtx);
2400 fpp = (fasttrap_provider_t **)&bucket->ftb_data;
2401 while ((fp = *fpp) != NULL) {
2402 /*
2403 * Acquire and release the lock as a simple way of
2404 * waiting for any other consumer to finish with
2405 * this provider. A thread must first acquire the
2406 * bucket lock so there's no chance of another thread
2407 * blocking on the provider's lock.
2408 */
2409 mutex_enter(&fp->ftp_mtx);
2410 mutex_exit(&fp->ftp_mtx);
2411
2412 if (dtrace_unregister(fp->ftp_provid) != 0) {
2413 fail = 1;
2414 fpp = &fp->ftp_next;
2415 } else {
2416 *fpp = fp->ftp_next;
2417 fasttrap_provider_free(fp);
2418 }
2419 }
2420
2421 mutex_exit(&bucket->ftb_mtx);
2422 }
2423
2424 if (fail) {
2425 (void) dtrace_meta_register("fasttrap", &fasttrap_mops, NULL,
2426 &fasttrap_meta_id);
2427
2428 return (-1);
2429 }
2430
2431 mtx_lock(&fasttrap_cleanup_mtx);
2432 fasttrap_cleanup_drain = 1;
2433 /* Wait for the cleanup thread to finish up and signal us. */
2434 wakeup(&fasttrap_cleanup_cv);
2435 mtx_sleep(&fasttrap_cleanup_drain, &fasttrap_cleanup_mtx, 0, "ftcld",
2436 0);
2437 fasttrap_cleanup_proc = NULL;
| 24 */
25
26/*
27 * Copyright 2008 Sun Microsystems, Inc. All rights reserved.
28 * Use is subject to license terms.
29 */
30
31#if defined(sun)
32#pragma ident "%Z%%M% %I% %E% SMI"
33#endif
34
35#include <sys/atomic.h>
36#include <sys/errno.h>
37#include <sys/stat.h>
38#include <sys/modctl.h>
39#include <sys/conf.h>
40#include <sys/systm.h>
41#if defined(sun)
42#include <sys/ddi.h>
43#endif
44#include <sys/sunddi.h>
45#include <sys/cpuvar.h>
46#include <sys/kmem.h>
47#if defined(sun)
48#include <sys/strsubr.h>
49#endif
50#include <sys/fasttrap.h>
51#include <sys/fasttrap_impl.h>
52#include <sys/fasttrap_isa.h>
53#include <sys/dtrace.h>
54#include <sys/dtrace_impl.h>
55#include <sys/sysmacros.h>
56#include <sys/proc.h>
57#include <sys/policy.h>
58#if defined(sun)
59#include <util/qsort.h>
60#endif
61#include <sys/mutex.h>
62#include <sys/kernel.h>
63#if !defined(sun)
64#include <sys/user.h>
65#include <sys/dtrace_bsd.h>
66#include <cddl/dev/dtrace/dtrace_cddl.h>
67#endif
68
69/*
70 * User-Land Trap-Based Tracing
71 * ----------------------------
72 *
73 * The fasttrap provider allows DTrace consumers to instrument any user-level
74 * instruction to gather data; this includes probes with semantic
75 * signifigance like entry and return as well as simple offsets into the
76 * function. While the specific techniques used are very ISA specific, the
77 * methodology is generalizable to any architecture.
78 *
79 *
80 * The General Methodology
81 * -----------------------
82 *
83 * With the primary goal of tracing every user-land instruction and the
84 * limitation that we can't trust user space so don't want to rely on much
85 * information there, we begin by replacing the instructions we want to trace
86 * with trap instructions. Each instruction we overwrite is saved into a hash
87 * table keyed by process ID and pc address. When we enter the kernel due to
88 * this trap instruction, we need the effects of the replaced instruction to
89 * appear to have occurred before we proceed with the user thread's
90 * execution.
91 *
92 * Each user level thread is represented by a ulwp_t structure which is
93 * always easily accessible through a register. The most basic way to produce
94 * the effects of the instruction we replaced is to copy that instruction out
95 * to a bit of scratch space reserved in the user thread's ulwp_t structure
96 * (a sort of kernel-private thread local storage), set the PC to that
97 * scratch space and single step. When we reenter the kernel after single
98 * stepping the instruction we must then adjust the PC to point to what would
99 * normally be the next instruction. Of course, special care must be taken
100 * for branches and jumps, but these represent such a small fraction of any
101 * instruction set that writing the code to emulate these in the kernel is
102 * not too difficult.
103 *
104 * Return probes may require several tracepoints to trace every return site,
105 * and, conversely, each tracepoint may activate several probes (the entry
106 * and offset 0 probes, for example). To solve this muliplexing problem,
107 * tracepoints contain lists of probes to activate and probes contain lists
108 * of tracepoints to enable. If a probe is activated, it adds its ID to
109 * existing tracepoints or creates new ones as necessary.
110 *
111 * Most probes are activated _before_ the instruction is executed, but return
112 * probes are activated _after_ the effects of the last instruction of the
113 * function are visible. Return probes must be fired _after_ we have
114 * single-stepped the instruction whereas all other probes are fired
115 * beforehand.
116 *
117 *
118 * Lock Ordering
119 * -------------
120 *
121 * The lock ordering below -- both internally and with respect to the DTrace
122 * framework -- is a little tricky and bears some explanation. Each provider
123 * has a lock (ftp_mtx) that protects its members including reference counts
124 * for enabled probes (ftp_rcount), consumers actively creating probes
125 * (ftp_ccount) and USDT consumers (ftp_mcount); all three prevent a provider
126 * from being freed. A provider is looked up by taking the bucket lock for the
127 * provider hash table, and is returned with its lock held. The provider lock
128 * may be taken in functions invoked by the DTrace framework, but may not be
129 * held while calling functions in the DTrace framework.
130 *
131 * To ensure consistency over multiple calls to the DTrace framework, the
132 * creation lock (ftp_cmtx) should be held. Naturally, the creation lock may
133 * not be taken when holding the provider lock as that would create a cyclic
134 * lock ordering. In situations where one would naturally take the provider
135 * lock and then the creation lock, we instead up a reference count to prevent
136 * the provider from disappearing, drop the provider lock, and acquire the
137 * creation lock.
138 *
139 * Briefly:
140 * bucket lock before provider lock
141 * DTrace before provider lock
142 * creation lock before DTrace
143 * never hold the provider lock and creation lock simultaneously
144 */
145
146static d_open_t fasttrap_open;
147static d_ioctl_t fasttrap_ioctl;
148
149static struct cdevsw fasttrap_cdevsw = {
150 .d_version = D_VERSION,
151 .d_open = fasttrap_open,
152 .d_ioctl = fasttrap_ioctl,
153 .d_name = "fasttrap",
154};
155static struct cdev *fasttrap_cdev;
156static dtrace_meta_provider_id_t fasttrap_meta_id;
157
158static struct proc *fasttrap_cleanup_proc;
159static struct mtx fasttrap_cleanup_mtx;
160static uint_t fasttrap_cleanup_work, fasttrap_cleanup_drain, fasttrap_cleanup_cv;
161
162/*
163 * Generation count on modifications to the global tracepoint lookup table.
164 */
165static volatile uint64_t fasttrap_mod_gen;
166
167/*
168 * When the fasttrap provider is loaded, fasttrap_max is set to either
169 * FASTTRAP_MAX_DEFAULT or the value for fasttrap-max-probes in the
170 * fasttrap.conf file. Each time a probe is created, fasttrap_total is
171 * incremented by the number of tracepoints that may be associated with that
172 * probe; fasttrap_total is capped at fasttrap_max.
173 */
174#define FASTTRAP_MAX_DEFAULT 250000
175static uint32_t fasttrap_max;
176static uint32_t fasttrap_total;
177
178/*
179 * Copyright (c) 2011, Joyent, Inc. All rights reserved.
180 */
181
182#define FASTTRAP_TPOINTS_DEFAULT_SIZE 0x4000
183#define FASTTRAP_PROVIDERS_DEFAULT_SIZE 0x100
184#define FASTTRAP_PROCS_DEFAULT_SIZE 0x100
185
186#define FASTTRAP_PID_NAME "pid"
187
188fasttrap_hash_t fasttrap_tpoints;
189static fasttrap_hash_t fasttrap_provs;
190static fasttrap_hash_t fasttrap_procs;
191
192static uint64_t fasttrap_pid_count; /* pid ref count */
193static kmutex_t fasttrap_count_mtx; /* lock on ref count */
194
195#define FASTTRAP_ENABLE_FAIL 1
196#define FASTTRAP_ENABLE_PARTIAL 2
197
198static int fasttrap_tracepoint_enable(proc_t *, fasttrap_probe_t *, uint_t);
199static void fasttrap_tracepoint_disable(proc_t *, fasttrap_probe_t *, uint_t);
200
201static fasttrap_provider_t *fasttrap_provider_lookup(pid_t, const char *,
202 const dtrace_pattr_t *);
203static void fasttrap_provider_retire(pid_t, const char *, int);
204static void fasttrap_provider_free(fasttrap_provider_t *);
205
206static fasttrap_proc_t *fasttrap_proc_lookup(pid_t);
207static void fasttrap_proc_release(fasttrap_proc_t *);
208
209#define FASTTRAP_PROVS_INDEX(pid, name) \
210 ((fasttrap_hash_str(name) + (pid)) & fasttrap_provs.fth_mask)
211
212#define FASTTRAP_PROCS_INDEX(pid) ((pid) & fasttrap_procs.fth_mask)
213
214#if !defined(sun)
215static kmutex_t fasttrap_cpuc_pid_lock[MAXCPU];
216#endif
217
218static int
219fasttrap_highbit(ulong_t i)
220{
221 int h = 1;
222
223 if (i == 0)
224 return (0);
225#ifdef _LP64
226 if (i & 0xffffffff00000000ul) {
227 h += 32; i >>= 32;
228 }
229#endif
230 if (i & 0xffff0000) {
231 h += 16; i >>= 16;
232 }
233 if (i & 0xff00) {
234 h += 8; i >>= 8;
235 }
236 if (i & 0xf0) {
237 h += 4; i >>= 4;
238 }
239 if (i & 0xc) {
240 h += 2; i >>= 2;
241 }
242 if (i & 0x2) {
243 h += 1;
244 }
245 return (h);
246}
247
248static uint_t
249fasttrap_hash_str(const char *p)
250{
251 unsigned int g;
252 uint_t hval = 0;
253
254 while (*p) {
255 hval = (hval << 4) + *p++;
256 if ((g = (hval & 0xf0000000)) != 0)
257 hval ^= g >> 24;
258 hval &= ~g;
259 }
260 return (hval);
261}
262
263void
264fasttrap_sigtrap(proc_t *p, kthread_t *t, uintptr_t pc)
265{
266#if defined(sun)
267 sigqueue_t *sqp = kmem_zalloc(sizeof (sigqueue_t), KM_SLEEP);
268
269 sqp->sq_info.si_signo = SIGTRAP;
270 sqp->sq_info.si_code = TRAP_DTRACE;
271 sqp->sq_info.si_addr = (caddr_t)pc;
272
273 mutex_enter(&p->p_lock);
274 sigaddqa(p, t, sqp);
275 mutex_exit(&p->p_lock);
276
277 if (t != NULL)
278 aston(t);
279#else
280 ksiginfo_t *ksi = kmem_zalloc(sizeof (ksiginfo_t), KM_SLEEP);
281
282 ksiginfo_init(ksi);
283 ksi->ksi_signo = SIGTRAP;
284 ksi->ksi_code = TRAP_DTRACE;
285 ksi->ksi_addr = (caddr_t)pc;
286 PROC_LOCK(p);
287 (void) tdksignal(t, SIGTRAP, ksi);
288 PROC_UNLOCK(p);
289#endif
290}
291
292/*
293 * This function ensures that no threads are actively using the memory
294 * associated with probes that were formerly live.
295 */
296static void
297fasttrap_mod_barrier(uint64_t gen)
298{
299 int i;
300
301 if (gen < fasttrap_mod_gen)
302 return;
303
304 fasttrap_mod_gen++;
305
306 CPU_FOREACH(i) {
307 mutex_enter(&fasttrap_cpuc_pid_lock[i]);
308 mutex_exit(&fasttrap_cpuc_pid_lock[i]);
309 }
310}
311
312/*
313 * This function performs asynchronous cleanup of fasttrap providers. The
314 * Solaris implementation of this mechanism use a timeout that's activated in
315 * fasttrap_pid_cleanup(), but this doesn't work in FreeBSD: one may sleep while
316 * holding the DTrace mutexes, but it is unsafe to sleep in a callout handler.
317 * Thus we use a dedicated process to perform the cleanup when requested.
318 */
319/*ARGSUSED*/
320static void
321fasttrap_pid_cleanup_cb(void *data)
322{
323 fasttrap_provider_t **fpp, *fp;
324 fasttrap_bucket_t *bucket;
325 dtrace_provider_id_t provid;
326 int i, later = 0, rval;
327
328 mtx_lock(&fasttrap_cleanup_mtx);
329 while (!fasttrap_cleanup_drain || later > 0) {
330 fasttrap_cleanup_work = 0;
331 mtx_unlock(&fasttrap_cleanup_mtx);
332
333 later = 0;
334
335 /*
336 * Iterate over all the providers trying to remove the marked
337 * ones. If a provider is marked but not retired, we just
338 * have to take a crack at removing it -- it's no big deal if
339 * we can't.
340 */
341 for (i = 0; i < fasttrap_provs.fth_nent; i++) {
342 bucket = &fasttrap_provs.fth_table[i];
343 mutex_enter(&bucket->ftb_mtx);
344 fpp = (fasttrap_provider_t **)&bucket->ftb_data;
345
346 while ((fp = *fpp) != NULL) {
347 if (!fp->ftp_marked) {
348 fpp = &fp->ftp_next;
349 continue;
350 }
351
352 mutex_enter(&fp->ftp_mtx);
353
354 /*
355 * If this provider has consumers actively
356 * creating probes (ftp_ccount) or is a USDT
357 * provider (ftp_mcount), we can't unregister
358 * or even condense.
359 */
360 if (fp->ftp_ccount != 0 ||
361 fp->ftp_mcount != 0) {
362 mutex_exit(&fp->ftp_mtx);
363 fp->ftp_marked = 0;
364 continue;
365 }
366
367 if (!fp->ftp_retired || fp->ftp_rcount != 0)
368 fp->ftp_marked = 0;
369
370 mutex_exit(&fp->ftp_mtx);
371
372 /*
373 * If we successfully unregister this
374 * provider we can remove it from the hash
375 * chain and free the memory. If our attempt
376 * to unregister fails and this is a retired
377 * provider, increment our flag to try again
378 * pretty soon. If we've consumed more than
379 * half of our total permitted number of
380 * probes call dtrace_condense() to try to
381 * clean out the unenabled probes.
382 */
383 provid = fp->ftp_provid;
384 if ((rval = dtrace_unregister(provid)) != 0) {
385 if (fasttrap_total > fasttrap_max / 2)
386 (void) dtrace_condense(provid);
387
388 if (rval == EAGAIN)
389 fp->ftp_marked = 1;
390
391 later += fp->ftp_marked;
392 fpp = &fp->ftp_next;
393 } else {
394 *fpp = fp->ftp_next;
395 fasttrap_provider_free(fp);
396 }
397 }
398 mutex_exit(&bucket->ftb_mtx);
399 }
400 mtx_lock(&fasttrap_cleanup_mtx);
401
402 /*
403 * If we were unable to retire a provider, try again after a
404 * second. This situation can occur in certain circumstances
405 * where providers cannot be unregistered even though they have
406 * no probes enabled because of an execution of dtrace -l or
407 * something similar.
408 */
409 if (later > 0 || fasttrap_cleanup_work ||
410 fasttrap_cleanup_drain) {
411 mtx_unlock(&fasttrap_cleanup_mtx);
412 pause("ftclean", hz);
413 mtx_lock(&fasttrap_cleanup_mtx);
414 } else
415 mtx_sleep(&fasttrap_cleanup_cv, &fasttrap_cleanup_mtx,
416 0, "ftcl", 0);
417 }
418
419 /*
420 * Wake up the thread in fasttrap_unload() now that we're done.
421 */
422 wakeup(&fasttrap_cleanup_drain);
423 mtx_unlock(&fasttrap_cleanup_mtx);
424
425 kthread_exit();
426}
427
428/*
429 * Activates the asynchronous cleanup mechanism.
430 */
431static void
432fasttrap_pid_cleanup(void)
433{
434
435 mtx_lock(&fasttrap_cleanup_mtx);
436 if (!fasttrap_cleanup_work) {
437 fasttrap_cleanup_work = 1;
438 wakeup(&fasttrap_cleanup_cv);
439 }
440 mtx_unlock(&fasttrap_cleanup_mtx);
441}
442
443/*
444 * This is called from cfork() via dtrace_fasttrap_fork(). The child
445 * process's address space is (roughly) a copy of the parent process's so
446 * we have to remove all the instrumentation we had previously enabled in the
447 * parent.
448 */
449static void
450fasttrap_fork(proc_t *p, proc_t *cp)
451{
452 pid_t ppid = p->p_pid;
453 int i;
454
455#if defined(sun)
456 ASSERT(curproc == p);
457 ASSERT(p->p_proc_flag & P_PR_LOCK);
458#else
459 PROC_LOCK_ASSERT(p, MA_OWNED);
460#endif
461#if defined(sun)
462 ASSERT(p->p_dtrace_count > 0);
463#else
464 if (p->p_dtrace_helpers) {
465 /*
466 * dtrace_helpers_duplicate() allocates memory.
467 */
468 _PHOLD(cp);
469 PROC_UNLOCK(p);
470 PROC_UNLOCK(cp);
471 dtrace_helpers_duplicate(p, cp);
472 PROC_LOCK(cp);
473 PROC_LOCK(p);
474 _PRELE(cp);
475 }
476 /*
477 * This check is purposely here instead of in kern_fork.c because,
478 * for legal resons, we cannot include the dtrace_cddl.h header
479 * inside kern_fork.c and insert if-clause there.
480 */
481 if (p->p_dtrace_count == 0)
482 return;
483#endif
484 ASSERT(cp->p_dtrace_count == 0);
485
486 /*
487 * This would be simpler and faster if we maintained per-process
488 * hash tables of enabled tracepoints. It could, however, potentially
489 * slow down execution of a tracepoint since we'd need to go
490 * through two levels of indirection. In the future, we should
491 * consider either maintaining per-process ancillary lists of
492 * enabled tracepoints or hanging a pointer to a per-process hash
493 * table of enabled tracepoints off the proc structure.
494 */
495
496 /*
497 * We don't have to worry about the child process disappearing
498 * because we're in fork().
499 */
500#if defined(sun)
501 mtx_lock_spin(&cp->p_slock);
502 sprlock_proc(cp);
503 mtx_unlock_spin(&cp->p_slock);
504#else
505 _PHOLD(cp);
506#endif
507
508 /*
509 * Iterate over every tracepoint looking for ones that belong to the
510 * parent process, and remove each from the child process.
511 */
512 for (i = 0; i < fasttrap_tpoints.fth_nent; i++) {
513 fasttrap_tracepoint_t *tp;
514 fasttrap_bucket_t *bucket = &fasttrap_tpoints.fth_table[i];
515
516 mutex_enter(&bucket->ftb_mtx);
517 for (tp = bucket->ftb_data; tp != NULL; tp = tp->ftt_next) {
518 if (tp->ftt_pid == ppid &&
519 tp->ftt_proc->ftpc_acount != 0) {
520 int ret = fasttrap_tracepoint_remove(cp, tp);
521 ASSERT(ret == 0);
522
523 /*
524 * The count of active providers can only be
525 * decremented (i.e. to zero) during exec,
526 * exit, and removal of a meta provider so it
527 * should be impossible to drop the count
528 * mid-fork.
529 */
530 ASSERT(tp->ftt_proc->ftpc_acount != 0);
531 }
532 }
533 mutex_exit(&bucket->ftb_mtx);
534 }
535
536#if defined(sun)
537 mutex_enter(&cp->p_lock);
538 sprunlock(cp);
539#else
540 _PRELE(cp);
541#endif
542}
543
544/*
545 * This is called from proc_exit() or from exec_common() if p_dtrace_probes
546 * is set on the proc structure to indicate that there is a pid provider
547 * associated with this process.
548 */
549static void
550fasttrap_exec_exit(proc_t *p)
551{
552#if defined(sun)
553 ASSERT(p == curproc);
554#endif
555 PROC_LOCK_ASSERT(p, MA_OWNED);
556 _PHOLD(p);
557 PROC_UNLOCK(p);
558
559 /*
560 * We clean up the pid provider for this process here; user-land
561 * static probes are handled by the meta-provider remove entry point.
562 */
563 fasttrap_provider_retire(p->p_pid, FASTTRAP_PID_NAME, 0);
564#if !defined(sun)
565 if (p->p_dtrace_helpers)
566 dtrace_helpers_destroy(p);
567#endif
568 PROC_LOCK(p);
569 _PRELE(p);
570}
571
572
573/*ARGSUSED*/
574static void
575fasttrap_pid_provide(void *arg, dtrace_probedesc_t *desc)
576{
577 /*
578 * There are no "default" pid probes.
579 */
580}
581
582static int
583fasttrap_tracepoint_enable(proc_t *p, fasttrap_probe_t *probe, uint_t index)
584{
585 fasttrap_tracepoint_t *tp, *new_tp = NULL;
586 fasttrap_bucket_t *bucket;
587 fasttrap_id_t *id;
588 pid_t pid;
589 uintptr_t pc;
590
591 ASSERT(index < probe->ftp_ntps);
592
593 pid = probe->ftp_pid;
594 pc = probe->ftp_tps[index].fit_tp->ftt_pc;
595 id = &probe->ftp_tps[index].fit_id;
596
597 ASSERT(probe->ftp_tps[index].fit_tp->ftt_pid == pid);
598
599#if defined(sun)
600 ASSERT(!(p->p_flag & SVFORK));
601#endif
602
603 /*
604 * Before we make any modifications, make sure we've imposed a barrier
605 * on the generation in which this probe was last modified.
606 */
607 fasttrap_mod_barrier(probe->ftp_gen);
608
609 bucket = &fasttrap_tpoints.fth_table[FASTTRAP_TPOINTS_INDEX(pid, pc)];
610
611 /*
612 * If the tracepoint has already been enabled, just add our id to the
613 * list of interested probes. This may be our second time through
614 * this path in which case we'll have constructed the tracepoint we'd
615 * like to install. If we can't find a match, and have an allocated
616 * tracepoint ready to go, enable that one now.
617 *
618 * A tracepoint whose process is defunct is also considered defunct.
619 */
620again:
621 mutex_enter(&bucket->ftb_mtx);
622 for (tp = bucket->ftb_data; tp != NULL; tp = tp->ftt_next) {
623 /*
624 * Note that it's safe to access the active count on the
625 * associated proc structure because we know that at least one
626 * provider (this one) will still be around throughout this
627 * operation.
628 */
629 if (tp->ftt_pid != pid || tp->ftt_pc != pc ||
630 tp->ftt_proc->ftpc_acount == 0)
631 continue;
632
633 /*
634 * Now that we've found a matching tracepoint, it would be
635 * a decent idea to confirm that the tracepoint is still
636 * enabled and the trap instruction hasn't been overwritten.
637 * Since this is a little hairy, we'll punt for now.
638 */
639
640 /*
641 * This can't be the first interested probe. We don't have
642 * to worry about another thread being in the midst of
643 * deleting this tracepoint (which would be the only valid
644 * reason for a tracepoint to have no interested probes)
645 * since we're holding P_PR_LOCK for this process.
646 */
647 ASSERT(tp->ftt_ids != NULL || tp->ftt_retids != NULL);
648
649 switch (id->fti_ptype) {
650 case DTFTP_ENTRY:
651 case DTFTP_OFFSETS:
652 case DTFTP_IS_ENABLED:
653 id->fti_next = tp->ftt_ids;
654 membar_producer();
655 tp->ftt_ids = id;
656 membar_producer();
657 break;
658
659 case DTFTP_RETURN:
660 case DTFTP_POST_OFFSETS:
661 id->fti_next = tp->ftt_retids;
662 membar_producer();
663 tp->ftt_retids = id;
664 membar_producer();
665 break;
666
667 default:
668 ASSERT(0);
669 }
670
671 mutex_exit(&bucket->ftb_mtx);
672
673 if (new_tp != NULL) {
674 new_tp->ftt_ids = NULL;
675 new_tp->ftt_retids = NULL;
676 }
677
678 return (0);
679 }
680
681 /*
682 * If we have a good tracepoint ready to go, install it now while
683 * we have the lock held and no one can screw with us.
684 */
685 if (new_tp != NULL) {
686 int rc = 0;
687
688 new_tp->ftt_next = bucket->ftb_data;
689 membar_producer();
690 bucket->ftb_data = new_tp;
691 membar_producer();
692 mutex_exit(&bucket->ftb_mtx);
693
694 /*
695 * Activate the tracepoint in the ISA-specific manner.
696 * If this fails, we need to report the failure, but
697 * indicate that this tracepoint must still be disabled
698 * by calling fasttrap_tracepoint_disable().
699 */
700 if (fasttrap_tracepoint_install(p, new_tp) != 0)
701 rc = FASTTRAP_ENABLE_PARTIAL;
702
703 /*
704 * Increment the count of the number of tracepoints active in
705 * the victim process.
706 */
707#if defined(sun)
708 ASSERT(p->p_proc_flag & P_PR_LOCK);
709#endif
710 p->p_dtrace_count++;
711
712 return (rc);
713 }
714
715 mutex_exit(&bucket->ftb_mtx);
716
717 /*
718 * Initialize the tracepoint that's been preallocated with the probe.
719 */
720 new_tp = probe->ftp_tps[index].fit_tp;
721
722 ASSERT(new_tp->ftt_pid == pid);
723 ASSERT(new_tp->ftt_pc == pc);
724 ASSERT(new_tp->ftt_proc == probe->ftp_prov->ftp_proc);
725 ASSERT(new_tp->ftt_ids == NULL);
726 ASSERT(new_tp->ftt_retids == NULL);
727
728 switch (id->fti_ptype) {
729 case DTFTP_ENTRY:
730 case DTFTP_OFFSETS:
731 case DTFTP_IS_ENABLED:
732 id->fti_next = NULL;
733 new_tp->ftt_ids = id;
734 break;
735
736 case DTFTP_RETURN:
737 case DTFTP_POST_OFFSETS:
738 id->fti_next = NULL;
739 new_tp->ftt_retids = id;
740 break;
741
742 default:
743 ASSERT(0);
744 }
745
746 /*
747 * If the ISA-dependent initialization goes to plan, go back to the
748 * beginning and try to install this freshly made tracepoint.
749 */
750 if (fasttrap_tracepoint_init(p, new_tp, pc, id->fti_ptype) == 0)
751 goto again;
752
753 new_tp->ftt_ids = NULL;
754 new_tp->ftt_retids = NULL;
755
756 return (FASTTRAP_ENABLE_FAIL);
757}
758
759static void
760fasttrap_tracepoint_disable(proc_t *p, fasttrap_probe_t *probe, uint_t index)
761{
762 fasttrap_bucket_t *bucket;
763 fasttrap_provider_t *provider = probe->ftp_prov;
764 fasttrap_tracepoint_t **pp, *tp;
765 fasttrap_id_t *id, **idp = NULL;
766 pid_t pid;
767 uintptr_t pc;
768
769 ASSERT(index < probe->ftp_ntps);
770
771 pid = probe->ftp_pid;
772 pc = probe->ftp_tps[index].fit_tp->ftt_pc;
773 id = &probe->ftp_tps[index].fit_id;
774
775 ASSERT(probe->ftp_tps[index].fit_tp->ftt_pid == pid);
776
777 /*
778 * Find the tracepoint and make sure that our id is one of the
779 * ones registered with it.
780 */
781 bucket = &fasttrap_tpoints.fth_table[FASTTRAP_TPOINTS_INDEX(pid, pc)];
782 mutex_enter(&bucket->ftb_mtx);
783 for (tp = bucket->ftb_data; tp != NULL; tp = tp->ftt_next) {
784 if (tp->ftt_pid == pid && tp->ftt_pc == pc &&
785 tp->ftt_proc == provider->ftp_proc)
786 break;
787 }
788
789 /*
790 * If we somehow lost this tracepoint, we're in a world of hurt.
791 */
792 ASSERT(tp != NULL);
793
794 switch (id->fti_ptype) {
795 case DTFTP_ENTRY:
796 case DTFTP_OFFSETS:
797 case DTFTP_IS_ENABLED:
798 ASSERT(tp->ftt_ids != NULL);
799 idp = &tp->ftt_ids;
800 break;
801
802 case DTFTP_RETURN:
803 case DTFTP_POST_OFFSETS:
804 ASSERT(tp->ftt_retids != NULL);
805 idp = &tp->ftt_retids;
806 break;
807
808 default:
809 ASSERT(0);
810 }
811
812 while ((*idp)->fti_probe != probe) {
813 idp = &(*idp)->fti_next;
814 ASSERT(*idp != NULL);
815 }
816
817 id = *idp;
818 *idp = id->fti_next;
819 membar_producer();
820
821 ASSERT(id->fti_probe == probe);
822
823 /*
824 * If there are other registered enablings of this tracepoint, we're
825 * all done, but if this was the last probe assocated with this
826 * this tracepoint, we need to remove and free it.
827 */
828 if (tp->ftt_ids != NULL || tp->ftt_retids != NULL) {
829
830 /*
831 * If the current probe's tracepoint is in use, swap it
832 * for an unused tracepoint.
833 */
834 if (tp == probe->ftp_tps[index].fit_tp) {
835 fasttrap_probe_t *tmp_probe;
836 fasttrap_tracepoint_t **tmp_tp;
837 uint_t tmp_index;
838
839 if (tp->ftt_ids != NULL) {
840 tmp_probe = tp->ftt_ids->fti_probe;
841 /* LINTED - alignment */
842 tmp_index = FASTTRAP_ID_INDEX(tp->ftt_ids);
843 tmp_tp = &tmp_probe->ftp_tps[tmp_index].fit_tp;
844 } else {
845 tmp_probe = tp->ftt_retids->fti_probe;
846 /* LINTED - alignment */
847 tmp_index = FASTTRAP_ID_INDEX(tp->ftt_retids);
848 tmp_tp = &tmp_probe->ftp_tps[tmp_index].fit_tp;
849 }
850
851 ASSERT(*tmp_tp != NULL);
852 ASSERT(*tmp_tp != probe->ftp_tps[index].fit_tp);
853 ASSERT((*tmp_tp)->ftt_ids == NULL);
854 ASSERT((*tmp_tp)->ftt_retids == NULL);
855
856 probe->ftp_tps[index].fit_tp = *tmp_tp;
857 *tmp_tp = tp;
858 }
859
860 mutex_exit(&bucket->ftb_mtx);
861
862 /*
863 * Tag the modified probe with the generation in which it was
864 * changed.
865 */
866 probe->ftp_gen = fasttrap_mod_gen;
867 return;
868 }
869
870 mutex_exit(&bucket->ftb_mtx);
871
872 /*
873 * We can't safely remove the tracepoint from the set of active
874 * tracepoints until we've actually removed the fasttrap instruction
875 * from the process's text. We can, however, operate on this
876 * tracepoint secure in the knowledge that no other thread is going to
877 * be looking at it since we hold P_PR_LOCK on the process if it's
878 * live or we hold the provider lock on the process if it's dead and
879 * gone.
880 */
881
882 /*
883 * We only need to remove the actual instruction if we're looking
884 * at an existing process
885 */
886 if (p != NULL) {
887 /*
888 * If we fail to restore the instruction we need to kill
889 * this process since it's in a completely unrecoverable
890 * state.
891 */
892 if (fasttrap_tracepoint_remove(p, tp) != 0)
893 fasttrap_sigtrap(p, NULL, pc);
894
895 /*
896 * Decrement the count of the number of tracepoints active
897 * in the victim process.
898 */
899#if defined(sun)
900 ASSERT(p->p_proc_flag & P_PR_LOCK);
901#endif
902 p->p_dtrace_count--;
903 }
904
905 /*
906 * Remove the probe from the hash table of active tracepoints.
907 */
908 mutex_enter(&bucket->ftb_mtx);
909 pp = (fasttrap_tracepoint_t **)&bucket->ftb_data;
910 ASSERT(*pp != NULL);
911 while (*pp != tp) {
912 pp = &(*pp)->ftt_next;
913 ASSERT(*pp != NULL);
914 }
915
916 *pp = tp->ftt_next;
917 membar_producer();
918
919 mutex_exit(&bucket->ftb_mtx);
920
921 /*
922 * Tag the modified probe with the generation in which it was changed.
923 */
924 probe->ftp_gen = fasttrap_mod_gen;
925}
926
927static void
928fasttrap_enable_callbacks(void)
929{
930 /*
931 * We don't have to play the rw lock game here because we're
932 * providing something rather than taking something away --
933 * we can be sure that no threads have tried to follow this
934 * function pointer yet.
935 */
936 mutex_enter(&fasttrap_count_mtx);
937 if (fasttrap_pid_count == 0) {
938 ASSERT(dtrace_pid_probe_ptr == NULL);
939 ASSERT(dtrace_return_probe_ptr == NULL);
940 dtrace_pid_probe_ptr = &fasttrap_pid_probe;
941 dtrace_return_probe_ptr = &fasttrap_return_probe;
942 }
943 ASSERT(dtrace_pid_probe_ptr == &fasttrap_pid_probe);
944 ASSERT(dtrace_return_probe_ptr == &fasttrap_return_probe);
945 fasttrap_pid_count++;
946 mutex_exit(&fasttrap_count_mtx);
947}
948
949static void
950fasttrap_disable_callbacks(void)
951{
952#if defined(sun)
953 ASSERT(MUTEX_HELD(&cpu_lock));
954#endif
955
956
957 mutex_enter(&fasttrap_count_mtx);
958 ASSERT(fasttrap_pid_count > 0);
959 fasttrap_pid_count--;
960 if (fasttrap_pid_count == 0) {
961#if defined(sun)
962 cpu_t *cur, *cpu = CPU;
963
964 for (cur = cpu->cpu_next_onln; cur != cpu;
965 cur = cur->cpu_next_onln) {
966 rw_enter(&cur->cpu_ft_lock, RW_WRITER);
967 }
968#endif
969 dtrace_pid_probe_ptr = NULL;
970 dtrace_return_probe_ptr = NULL;
971#if defined(sun)
972 for (cur = cpu->cpu_next_onln; cur != cpu;
973 cur = cur->cpu_next_onln) {
974 rw_exit(&cur->cpu_ft_lock);
975 }
976#endif
977 }
978 mutex_exit(&fasttrap_count_mtx);
979}
980
981/*ARGSUSED*/
982static void
983fasttrap_pid_enable(void *arg, dtrace_id_t id, void *parg)
984{
985 fasttrap_probe_t *probe = parg;
986 proc_t *p = NULL;
987 int i, rc;
988
989 ASSERT(probe != NULL);
990 ASSERT(!probe->ftp_enabled);
991 ASSERT(id == probe->ftp_id);
992#if defined(sun)
993 ASSERT(MUTEX_HELD(&cpu_lock));
994#endif
995
996 /*
997 * Increment the count of enabled probes on this probe's provider;
998 * the provider can't go away while the probe still exists. We
999 * must increment this even if we aren't able to properly enable
1000 * this probe.
1001 */
1002 mutex_enter(&probe->ftp_prov->ftp_mtx);
1003 probe->ftp_prov->ftp_rcount++;
1004 mutex_exit(&probe->ftp_prov->ftp_mtx);
1005
1006 /*
1007 * If this probe's provider is retired (meaning it was valid in a
1008 * previously exec'ed incarnation of this address space), bail out. The
1009 * provider can't go away while we're in this code path.
1010 */
1011 if (probe->ftp_prov->ftp_retired)
1012 return;
1013
1014 /*
1015 * If we can't find the process, it may be that we're in the context of
1016 * a fork in which the traced process is being born and we're copying
1017 * USDT probes. Otherwise, the process is gone so bail.
1018 */
1019#if defined(sun)
1020 if ((p = sprlock(probe->ftp_pid)) == NULL) {
1021 if ((curproc->p_flag & SFORKING) == 0)
1022 return;
1023
1024 mutex_enter(&pidlock);
1025 p = prfind(probe->ftp_pid);
1026
1027 /*
1028 * Confirm that curproc is indeed forking the process in which
1029 * we're trying to enable probes.
1030 */
1031 ASSERT(p != NULL);
1032 ASSERT(p->p_parent == curproc);
1033 ASSERT(p->p_stat == SIDL);
1034
1035 mutex_enter(&p->p_lock);
1036 mutex_exit(&pidlock);
1037
1038 sprlock_proc(p);
1039 }
1040
1041 ASSERT(!(p->p_flag & SVFORK));
1042 mutex_exit(&p->p_lock);
1043#else
1044 if ((p = pfind(probe->ftp_pid)) == NULL)
1045 return;
1046#endif
1047
1048 /*
1049 * We have to enable the trap entry point before any user threads have
1050 * the chance to execute the trap instruction we're about to place
1051 * in their process's text.
1052 */
1053#ifdef __FreeBSD__
1054 /*
1055 * pfind() returns a locked process.
1056 */
1057 _PHOLD(p);
1058 PROC_UNLOCK(p);
1059#endif
1060 fasttrap_enable_callbacks();
1061
1062 /*
1063 * Enable all the tracepoints and add this probe's id to each
1064 * tracepoint's list of active probes.
1065 */
1066 for (i = 0; i < probe->ftp_ntps; i++) {
1067 if ((rc = fasttrap_tracepoint_enable(p, probe, i)) != 0) {
1068 /*
1069 * If enabling the tracepoint failed completely,
1070 * we don't have to disable it; if the failure
1071 * was only partial we must disable it.
1072 */
1073 if (rc == FASTTRAP_ENABLE_FAIL)
1074 i--;
1075 else
1076 ASSERT(rc == FASTTRAP_ENABLE_PARTIAL);
1077
1078 /*
1079 * Back up and pull out all the tracepoints we've
1080 * created so far for this probe.
1081 */
1082 while (i >= 0) {
1083 fasttrap_tracepoint_disable(p, probe, i);
1084 i--;
1085 }
1086
1087#if defined(sun)
1088 mutex_enter(&p->p_lock);
1089 sprunlock(p);
1090#else
1091 PRELE(p);
1092#endif
1093
1094 /*
1095 * Since we're not actually enabling this probe,
1096 * drop our reference on the trap table entry.
1097 */
1098 fasttrap_disable_callbacks();
1099 return;
1100 }
1101 }
1102#if defined(sun)
1103 mutex_enter(&p->p_lock);
1104 sprunlock(p);
1105#else
1106 PRELE(p);
1107#endif
1108
1109 probe->ftp_enabled = 1;
1110}
1111
1112/*ARGSUSED*/
1113static void
1114fasttrap_pid_disable(void *arg, dtrace_id_t id, void *parg)
1115{
1116 fasttrap_probe_t *probe = parg;
1117 fasttrap_provider_t *provider = probe->ftp_prov;
1118 proc_t *p;
1119 int i, whack = 0;
1120
1121 ASSERT(id == probe->ftp_id);
1122
1123 mutex_enter(&provider->ftp_mtx);
1124
1125 /*
1126 * We won't be able to acquire a /proc-esque lock on the process
1127 * iff the process is dead and gone. In this case, we rely on the
1128 * provider lock as a point of mutual exclusion to prevent other
1129 * DTrace consumers from disabling this probe.
1130 */
1131 if ((p = pfind(probe->ftp_pid)) != NULL) {
1132#ifdef __FreeBSD__
1133 _PHOLD(p);
1134 PROC_UNLOCK(p);
1135#endif
1136 }
1137
1138 /*
1139 * Disable all the associated tracepoints (for fully enabled probes).
1140 */
1141 if (probe->ftp_enabled) {
1142 for (i = 0; i < probe->ftp_ntps; i++) {
1143 fasttrap_tracepoint_disable(p, probe, i);
1144 }
1145 }
1146
1147 ASSERT(provider->ftp_rcount > 0);
1148 provider->ftp_rcount--;
1149
1150 if (p != NULL) {
1151 /*
1152 * Even though we may not be able to remove it entirely, we
1153 * mark this retired provider to get a chance to remove some
1154 * of the associated probes.
1155 */
1156 if (provider->ftp_retired && !provider->ftp_marked)
1157 whack = provider->ftp_marked = 1;
1158 mutex_exit(&provider->ftp_mtx);
1159 } else {
1160 /*
1161 * If the process is dead, we're just waiting for the
1162 * last probe to be disabled to be able to free it.
1163 */
1164 if (provider->ftp_rcount == 0 && !provider->ftp_marked)
1165 whack = provider->ftp_marked = 1;
1166 mutex_exit(&provider->ftp_mtx);
1167 }
1168
1169 if (whack)
1170 fasttrap_pid_cleanup();
1171
1172#ifdef __FreeBSD__
1173 if (p != NULL)
1174 PRELE(p);
1175#endif
1176 if (!probe->ftp_enabled)
1177 return;
1178
1179 probe->ftp_enabled = 0;
1180
1181#if defined(sun)
1182 ASSERT(MUTEX_HELD(&cpu_lock));
1183#endif
1184 fasttrap_disable_callbacks();
1185}
1186
1187/*ARGSUSED*/
1188static void
1189fasttrap_pid_getargdesc(void *arg, dtrace_id_t id, void *parg,
1190 dtrace_argdesc_t *desc)
1191{
1192 fasttrap_probe_t *probe = parg;
1193 char *str;
1194 int i, ndx;
1195
1196 desc->dtargd_native[0] = '\0';
1197 desc->dtargd_xlate[0] = '\0';
1198
1199 if (probe->ftp_prov->ftp_retired != 0 ||
1200 desc->dtargd_ndx >= probe->ftp_nargs) {
1201 desc->dtargd_ndx = DTRACE_ARGNONE;
1202 return;
1203 }
1204
1205 ndx = (probe->ftp_argmap != NULL) ?
1206 probe->ftp_argmap[desc->dtargd_ndx] : desc->dtargd_ndx;
1207
1208 str = probe->ftp_ntypes;
1209 for (i = 0; i < ndx; i++) {
1210 str += strlen(str) + 1;
1211 }
1212
1213 ASSERT(strlen(str + 1) < sizeof (desc->dtargd_native));
1214 (void) strcpy(desc->dtargd_native, str);
1215
1216 if (probe->ftp_xtypes == NULL)
1217 return;
1218
1219 str = probe->ftp_xtypes;
1220 for (i = 0; i < desc->dtargd_ndx; i++) {
1221 str += strlen(str) + 1;
1222 }
1223
1224 ASSERT(strlen(str + 1) < sizeof (desc->dtargd_xlate));
1225 (void) strcpy(desc->dtargd_xlate, str);
1226}
1227
1228/*ARGSUSED*/
1229static void
1230fasttrap_pid_destroy(void *arg, dtrace_id_t id, void *parg)
1231{
1232 fasttrap_probe_t *probe = parg;
1233 int i;
1234 size_t size;
1235
1236 ASSERT(probe != NULL);
1237 ASSERT(!probe->ftp_enabled);
1238 ASSERT(fasttrap_total >= probe->ftp_ntps);
1239
1240 atomic_add_32(&fasttrap_total, -probe->ftp_ntps);
1241 size = offsetof(fasttrap_probe_t, ftp_tps[probe->ftp_ntps]);
1242
1243 if (probe->ftp_gen + 1 >= fasttrap_mod_gen)
1244 fasttrap_mod_barrier(probe->ftp_gen);
1245
1246 for (i = 0; i < probe->ftp_ntps; i++) {
1247 kmem_free(probe->ftp_tps[i].fit_tp,
1248 sizeof (fasttrap_tracepoint_t));
1249 }
1250
1251 kmem_free(probe, size);
1252}
1253
1254
1255static const dtrace_pattr_t pid_attr = {
1256{ DTRACE_STABILITY_EVOLVING, DTRACE_STABILITY_EVOLVING, DTRACE_CLASS_ISA },
1257{ DTRACE_STABILITY_PRIVATE, DTRACE_STABILITY_PRIVATE, DTRACE_CLASS_UNKNOWN },
1258{ DTRACE_STABILITY_PRIVATE, DTRACE_STABILITY_PRIVATE, DTRACE_CLASS_UNKNOWN },
1259{ DTRACE_STABILITY_EVOLVING, DTRACE_STABILITY_EVOLVING, DTRACE_CLASS_ISA },
1260{ DTRACE_STABILITY_PRIVATE, DTRACE_STABILITY_PRIVATE, DTRACE_CLASS_UNKNOWN },
1261};
1262
1263static dtrace_pops_t pid_pops = {
1264 fasttrap_pid_provide,
1265 NULL,
1266 fasttrap_pid_enable,
1267 fasttrap_pid_disable,
1268 NULL,
1269 NULL,
1270 fasttrap_pid_getargdesc,
1271 fasttrap_pid_getarg,
1272 NULL,
1273 fasttrap_pid_destroy
1274};
1275
1276static dtrace_pops_t usdt_pops = {
1277 fasttrap_pid_provide,
1278 NULL,
1279 fasttrap_pid_enable,
1280 fasttrap_pid_disable,
1281 NULL,
1282 NULL,
1283 fasttrap_pid_getargdesc,
1284 fasttrap_usdt_getarg,
1285 NULL,
1286 fasttrap_pid_destroy
1287};
1288
1289static fasttrap_proc_t *
1290fasttrap_proc_lookup(pid_t pid)
1291{
1292 fasttrap_bucket_t *bucket;
1293 fasttrap_proc_t *fprc, *new_fprc;
1294
1295
1296 bucket = &fasttrap_procs.fth_table[FASTTRAP_PROCS_INDEX(pid)];
1297 mutex_enter(&bucket->ftb_mtx);
1298
1299 for (fprc = bucket->ftb_data; fprc != NULL; fprc = fprc->ftpc_next) {
1300 if (fprc->ftpc_pid == pid && fprc->ftpc_acount != 0) {
1301 mutex_enter(&fprc->ftpc_mtx);
1302 mutex_exit(&bucket->ftb_mtx);
1303 fprc->ftpc_rcount++;
1304 atomic_add_64(&fprc->ftpc_acount, 1);
1305 ASSERT(fprc->ftpc_acount <= fprc->ftpc_rcount);
1306 mutex_exit(&fprc->ftpc_mtx);
1307
1308 return (fprc);
1309 }
1310 }
1311
1312 /*
1313 * Drop the bucket lock so we don't try to perform a sleeping
1314 * allocation under it.
1315 */
1316 mutex_exit(&bucket->ftb_mtx);
1317
1318 new_fprc = kmem_zalloc(sizeof (fasttrap_proc_t), KM_SLEEP);
1319 new_fprc->ftpc_pid = pid;
1320 new_fprc->ftpc_rcount = 1;
1321 new_fprc->ftpc_acount = 1;
1322#if !defined(sun)
1323 mutex_init(&new_fprc->ftpc_mtx, "fasttrap proc mtx", MUTEX_DEFAULT,
1324 NULL);
1325#endif
1326
1327 mutex_enter(&bucket->ftb_mtx);
1328
1329 /*
1330 * Take another lap through the list to make sure a proc hasn't
1331 * been created for this pid while we weren't under the bucket lock.
1332 */
1333 for (fprc = bucket->ftb_data; fprc != NULL; fprc = fprc->ftpc_next) {
1334 if (fprc->ftpc_pid == pid && fprc->ftpc_acount != 0) {
1335 mutex_enter(&fprc->ftpc_mtx);
1336 mutex_exit(&bucket->ftb_mtx);
1337 fprc->ftpc_rcount++;
1338 atomic_add_64(&fprc->ftpc_acount, 1);
1339 ASSERT(fprc->ftpc_acount <= fprc->ftpc_rcount);
1340 mutex_exit(&fprc->ftpc_mtx);
1341
1342 kmem_free(new_fprc, sizeof (fasttrap_proc_t));
1343
1344 return (fprc);
1345 }
1346 }
1347
1348 new_fprc->ftpc_next = bucket->ftb_data;
1349 bucket->ftb_data = new_fprc;
1350
1351 mutex_exit(&bucket->ftb_mtx);
1352
1353 return (new_fprc);
1354}
1355
1356static void
1357fasttrap_proc_release(fasttrap_proc_t *proc)
1358{
1359 fasttrap_bucket_t *bucket;
1360 fasttrap_proc_t *fprc, **fprcp;
1361 pid_t pid = proc->ftpc_pid;
1362
1363 mutex_enter(&proc->ftpc_mtx);
1364
1365 ASSERT(proc->ftpc_rcount != 0);
1366 ASSERT(proc->ftpc_acount <= proc->ftpc_rcount);
1367
1368 if (--proc->ftpc_rcount != 0) {
1369 mutex_exit(&proc->ftpc_mtx);
1370 return;
1371 }
1372
1373 mutex_exit(&proc->ftpc_mtx);
1374
1375 /*
1376 * There should definitely be no live providers associated with this
1377 * process at this point.
1378 */
1379 ASSERT(proc->ftpc_acount == 0);
1380
1381 bucket = &fasttrap_procs.fth_table[FASTTRAP_PROCS_INDEX(pid)];
1382 mutex_enter(&bucket->ftb_mtx);
1383
1384 fprcp = (fasttrap_proc_t **)&bucket->ftb_data;
1385 while ((fprc = *fprcp) != NULL) {
1386 if (fprc == proc)
1387 break;
1388
1389 fprcp = &fprc->ftpc_next;
1390 }
1391
1392 /*
1393 * Something strange has happened if we can't find the proc.
1394 */
1395 ASSERT(fprc != NULL);
1396
1397 *fprcp = fprc->ftpc_next;
1398
1399 mutex_exit(&bucket->ftb_mtx);
1400
1401 kmem_free(fprc, sizeof (fasttrap_proc_t));
1402}
1403
1404/*
1405 * Lookup a fasttrap-managed provider based on its name and associated pid.
1406 * If the pattr argument is non-NULL, this function instantiates the provider
1407 * if it doesn't exist otherwise it returns NULL. The provider is returned
1408 * with its lock held.
1409 */
1410static fasttrap_provider_t *
1411fasttrap_provider_lookup(pid_t pid, const char *name,
1412 const dtrace_pattr_t *pattr)
1413{
1414 fasttrap_provider_t *fp, *new_fp = NULL;
1415 fasttrap_bucket_t *bucket;
1416 char provname[DTRACE_PROVNAMELEN];
1417 proc_t *p;
1418 cred_t *cred;
1419
1420 ASSERT(strlen(name) < sizeof (fp->ftp_name));
1421 ASSERT(pattr != NULL);
1422
1423 bucket = &fasttrap_provs.fth_table[FASTTRAP_PROVS_INDEX(pid, name)];
1424 mutex_enter(&bucket->ftb_mtx);
1425
1426 /*
1427 * Take a lap through the list and return the match if we find it.
1428 */
1429 for (fp = bucket->ftb_data; fp != NULL; fp = fp->ftp_next) {
1430 if (fp->ftp_pid == pid && strcmp(fp->ftp_name, name) == 0 &&
1431 !fp->ftp_retired) {
1432 mutex_enter(&fp->ftp_mtx);
1433 mutex_exit(&bucket->ftb_mtx);
1434 return (fp);
1435 }
1436 }
1437
1438 /*
1439 * Drop the bucket lock so we don't try to perform a sleeping
1440 * allocation under it.
1441 */
1442 mutex_exit(&bucket->ftb_mtx);
1443
1444 /*
1445 * Make sure the process exists, isn't a child created as the result
1446 * of a vfork(2), and isn't a zombie (but may be in fork).
1447 */
1448 if ((p = pfind(pid)) == NULL)
1449 return (NULL);
1450
1451 /*
1452 * Increment p_dtrace_probes so that the process knows to inform us
1453 * when it exits or execs. fasttrap_provider_free() decrements this
1454 * when we're done with this provider.
1455 */
1456 p->p_dtrace_probes++;
1457
1458 /*
1459 * Grab the credentials for this process so we have
1460 * something to pass to dtrace_register().
1461 */
1462 PROC_LOCK_ASSERT(p, MA_OWNED);
1463 crhold(p->p_ucred);
1464 cred = p->p_ucred;
1465 PROC_UNLOCK(p);
1466
1467 new_fp = kmem_zalloc(sizeof (fasttrap_provider_t), KM_SLEEP);
1468 new_fp->ftp_pid = pid;
1469 new_fp->ftp_proc = fasttrap_proc_lookup(pid);
1470#if !defined(sun)
1471 mutex_init(&new_fp->ftp_mtx, "provider mtx", MUTEX_DEFAULT, NULL);
1472 mutex_init(&new_fp->ftp_cmtx, "lock on creating", MUTEX_DEFAULT, NULL);
1473#endif
1474
1475 ASSERT(new_fp->ftp_proc != NULL);
1476
1477 mutex_enter(&bucket->ftb_mtx);
1478
1479 /*
1480 * Take another lap through the list to make sure a provider hasn't
1481 * been created for this pid while we weren't under the bucket lock.
1482 */
1483 for (fp = bucket->ftb_data; fp != NULL; fp = fp->ftp_next) {
1484 if (fp->ftp_pid == pid && strcmp(fp->ftp_name, name) == 0 &&
1485 !fp->ftp_retired) {
1486 mutex_enter(&fp->ftp_mtx);
1487 mutex_exit(&bucket->ftb_mtx);
1488 fasttrap_provider_free(new_fp);
1489 crfree(cred);
1490 return (fp);
1491 }
1492 }
1493
1494 (void) strcpy(new_fp->ftp_name, name);
1495
1496 /*
1497 * Fail and return NULL if either the provider name is too long
1498 * or we fail to register this new provider with the DTrace
1499 * framework. Note that this is the only place we ever construct
1500 * the full provider name -- we keep it in pieces in the provider
1501 * structure.
1502 */
1503 if (snprintf(provname, sizeof (provname), "%s%u", name, (uint_t)pid) >=
1504 sizeof (provname) ||
1505 dtrace_register(provname, pattr,
1506 DTRACE_PRIV_PROC | DTRACE_PRIV_OWNER | DTRACE_PRIV_ZONEOWNER, cred,
1507 pattr == &pid_attr ? &pid_pops : &usdt_pops, new_fp,
1508 &new_fp->ftp_provid) != 0) {
1509 mutex_exit(&bucket->ftb_mtx);
1510 fasttrap_provider_free(new_fp);
1511 crfree(cred);
1512 return (NULL);
1513 }
1514
1515 new_fp->ftp_next = bucket->ftb_data;
1516 bucket->ftb_data = new_fp;
1517
1518 mutex_enter(&new_fp->ftp_mtx);
1519 mutex_exit(&bucket->ftb_mtx);
1520
1521 crfree(cred);
1522 return (new_fp);
1523}
1524
1525static void
1526fasttrap_provider_free(fasttrap_provider_t *provider)
1527{
1528 pid_t pid = provider->ftp_pid;
1529 proc_t *p;
1530
1531 /*
1532 * There need to be no associated enabled probes, no consumers
1533 * creating probes, and no meta providers referencing this provider.
1534 */
1535 ASSERT(provider->ftp_rcount == 0);
1536 ASSERT(provider->ftp_ccount == 0);
1537 ASSERT(provider->ftp_mcount == 0);
1538
1539 /*
1540 * If this provider hasn't been retired, we need to explicitly drop the
1541 * count of active providers on the associated process structure.
1542 */
1543 if (!provider->ftp_retired) {
1544 atomic_add_64(&provider->ftp_proc->ftpc_acount, -1);
1545 ASSERT(provider->ftp_proc->ftpc_acount <
1546 provider->ftp_proc->ftpc_rcount);
1547 }
1548
1549 fasttrap_proc_release(provider->ftp_proc);
1550
1551#if !defined(sun)
1552 mutex_destroy(&provider->ftp_mtx);
1553 mutex_destroy(&provider->ftp_cmtx);
1554#endif
1555 kmem_free(provider, sizeof (fasttrap_provider_t));
1556
1557 /*
1558 * Decrement p_dtrace_probes on the process whose provider we're
1559 * freeing. We don't have to worry about clobbering somone else's
1560 * modifications to it because we have locked the bucket that
1561 * corresponds to this process's hash chain in the provider hash
1562 * table. Don't sweat it if we can't find the process.
1563 */
1564 if ((p = pfind(pid)) == NULL) {
1565 return;
1566 }
1567
1568 p->p_dtrace_probes--;
1569#if !defined(sun)
1570 PROC_UNLOCK(p);
1571#endif
1572}
1573
1574static void
1575fasttrap_provider_retire(pid_t pid, const char *name, int mprov)
1576{
1577 fasttrap_provider_t *fp;
1578 fasttrap_bucket_t *bucket;
1579 dtrace_provider_id_t provid;
1580
1581 ASSERT(strlen(name) < sizeof (fp->ftp_name));
1582
1583 bucket = &fasttrap_provs.fth_table[FASTTRAP_PROVS_INDEX(pid, name)];
1584 mutex_enter(&bucket->ftb_mtx);
1585
1586 for (fp = bucket->ftb_data; fp != NULL; fp = fp->ftp_next) {
1587 if (fp->ftp_pid == pid && strcmp(fp->ftp_name, name) == 0 &&
1588 !fp->ftp_retired)
1589 break;
1590 }
1591
1592 if (fp == NULL) {
1593 mutex_exit(&bucket->ftb_mtx);
1594 return;
1595 }
1596
1597 mutex_enter(&fp->ftp_mtx);
1598 ASSERT(!mprov || fp->ftp_mcount > 0);
1599 if (mprov && --fp->ftp_mcount != 0) {
1600 mutex_exit(&fp->ftp_mtx);
1601 mutex_exit(&bucket->ftb_mtx);
1602 return;
1603 }
1604
1605 /*
1606 * Mark the provider to be removed in our post-processing step, mark it
1607 * retired, and drop the active count on its proc. Marking it indicates
1608 * that we should try to remove it; setting the retired flag indicates
1609 * that we're done with this provider; dropping the active the proc
1610 * releases our hold, and when this reaches zero (as it will during
1611 * exit or exec) the proc and associated providers become defunct.
1612 *
1613 * We obviously need to take the bucket lock before the provider lock
1614 * to perform the lookup, but we need to drop the provider lock
1615 * before calling into the DTrace framework since we acquire the
1616 * provider lock in callbacks invoked from the DTrace framework. The
1617 * bucket lock therefore protects the integrity of the provider hash
1618 * table.
1619 */
1620 atomic_add_64(&fp->ftp_proc->ftpc_acount, -1);
1621 ASSERT(fp->ftp_proc->ftpc_acount < fp->ftp_proc->ftpc_rcount);
1622
1623 fp->ftp_retired = 1;
1624 fp->ftp_marked = 1;
1625 provid = fp->ftp_provid;
1626 mutex_exit(&fp->ftp_mtx);
1627
1628 /*
1629 * We don't have to worry about invalidating the same provider twice
1630 * since fasttrap_provider_lookup() will ignore provider that have
1631 * been marked as retired.
1632 */
1633 dtrace_invalidate(provid);
1634
1635 mutex_exit(&bucket->ftb_mtx);
1636
1637 fasttrap_pid_cleanup();
1638}
1639
1640static int
1641fasttrap_uint32_cmp(const void *ap, const void *bp)
1642{
1643 return (*(const uint32_t *)ap - *(const uint32_t *)bp);
1644}
1645
1646static int
1647fasttrap_uint64_cmp(const void *ap, const void *bp)
1648{
1649 return (*(const uint64_t *)ap - *(const uint64_t *)bp);
1650}
1651
1652static int
1653fasttrap_add_probe(fasttrap_probe_spec_t *pdata)
1654{
1655 fasttrap_provider_t *provider;
1656 fasttrap_probe_t *pp;
1657 fasttrap_tracepoint_t *tp;
1658 char *name;
1659 int i, aframes = 0, whack;
1660
1661 /*
1662 * There needs to be at least one desired trace point.
1663 */
1664 if (pdata->ftps_noffs == 0)
1665 return (EINVAL);
1666
1667 switch (pdata->ftps_type) {
1668 case DTFTP_ENTRY:
1669 name = "entry";
1670 aframes = FASTTRAP_ENTRY_AFRAMES;
1671 break;
1672 case DTFTP_RETURN:
1673 name = "return";
1674 aframes = FASTTRAP_RETURN_AFRAMES;
1675 break;
1676 case DTFTP_OFFSETS:
1677 name = NULL;
1678 break;
1679 default:
1680 return (EINVAL);
1681 }
1682
1683 if ((provider = fasttrap_provider_lookup(pdata->ftps_pid,
1684 FASTTRAP_PID_NAME, &pid_attr)) == NULL)
1685 return (ESRCH);
1686
1687 /*
1688 * Increment this reference count to indicate that a consumer is
1689 * actively adding a new probe associated with this provider. This
1690 * prevents the provider from being deleted -- we'll need to check
1691 * for pending deletions when we drop this reference count.
1692 */
1693 provider->ftp_ccount++;
1694 mutex_exit(&provider->ftp_mtx);
1695
1696 /*
1697 * Grab the creation lock to ensure consistency between calls to
1698 * dtrace_probe_lookup() and dtrace_probe_create() in the face of
1699 * other threads creating probes. We must drop the provider lock
1700 * before taking this lock to avoid a three-way deadlock with the
1701 * DTrace framework.
1702 */
1703 mutex_enter(&provider->ftp_cmtx);
1704
1705 if (name == NULL) {
1706 for (i = 0; i < pdata->ftps_noffs; i++) {
1707 char name_str[17];
1708
1709 (void) sprintf(name_str, "%llx",
1710 (unsigned long long)pdata->ftps_offs[i]);
1711
1712 if (dtrace_probe_lookup(provider->ftp_provid,
1713 pdata->ftps_mod, pdata->ftps_func, name_str) != 0)
1714 continue;
1715
1716 atomic_add_32(&fasttrap_total, 1);
1717
1718 if (fasttrap_total > fasttrap_max) {
1719 atomic_add_32(&fasttrap_total, -1);
1720 goto no_mem;
1721 }
1722
1723 pp = kmem_zalloc(sizeof (fasttrap_probe_t), KM_SLEEP);
1724
1725 pp->ftp_prov = provider;
1726 pp->ftp_faddr = pdata->ftps_pc;
1727 pp->ftp_fsize = pdata->ftps_size;
1728 pp->ftp_pid = pdata->ftps_pid;
1729 pp->ftp_ntps = 1;
1730
1731 tp = kmem_zalloc(sizeof (fasttrap_tracepoint_t),
1732 KM_SLEEP);
1733
1734 tp->ftt_proc = provider->ftp_proc;
1735 tp->ftt_pc = pdata->ftps_offs[i] + pdata->ftps_pc;
1736 tp->ftt_pid = pdata->ftps_pid;
1737
1738 pp->ftp_tps[0].fit_tp = tp;
1739 pp->ftp_tps[0].fit_id.fti_probe = pp;
1740 pp->ftp_tps[0].fit_id.fti_ptype = pdata->ftps_type;
1741
1742 pp->ftp_id = dtrace_probe_create(provider->ftp_provid,
1743 pdata->ftps_mod, pdata->ftps_func, name_str,
1744 FASTTRAP_OFFSET_AFRAMES, pp);
1745 }
1746
1747 } else if (dtrace_probe_lookup(provider->ftp_provid, pdata->ftps_mod,
1748 pdata->ftps_func, name) == 0) {
1749 atomic_add_32(&fasttrap_total, pdata->ftps_noffs);
1750
1751 if (fasttrap_total > fasttrap_max) {
1752 atomic_add_32(&fasttrap_total, -pdata->ftps_noffs);
1753 goto no_mem;
1754 }
1755
1756 /*
1757 * Make sure all tracepoint program counter values are unique.
1758 * We later assume that each probe has exactly one tracepoint
1759 * for a given pc.
1760 */
1761 qsort(pdata->ftps_offs, pdata->ftps_noffs,
1762 sizeof (uint64_t), fasttrap_uint64_cmp);
1763 for (i = 1; i < pdata->ftps_noffs; i++) {
1764 if (pdata->ftps_offs[i] > pdata->ftps_offs[i - 1])
1765 continue;
1766
1767 atomic_add_32(&fasttrap_total, -pdata->ftps_noffs);
1768 goto no_mem;
1769 }
1770
1771 ASSERT(pdata->ftps_noffs > 0);
1772 pp = kmem_zalloc(offsetof(fasttrap_probe_t,
1773 ftp_tps[pdata->ftps_noffs]), KM_SLEEP);
1774
1775 pp->ftp_prov = provider;
1776 pp->ftp_faddr = pdata->ftps_pc;
1777 pp->ftp_fsize = pdata->ftps_size;
1778 pp->ftp_pid = pdata->ftps_pid;
1779 pp->ftp_ntps = pdata->ftps_noffs;
1780
1781 for (i = 0; i < pdata->ftps_noffs; i++) {
1782 tp = kmem_zalloc(sizeof (fasttrap_tracepoint_t),
1783 KM_SLEEP);
1784
1785 tp->ftt_proc = provider->ftp_proc;
1786 tp->ftt_pc = pdata->ftps_offs[i] + pdata->ftps_pc;
1787 tp->ftt_pid = pdata->ftps_pid;
1788
1789 pp->ftp_tps[i].fit_tp = tp;
1790 pp->ftp_tps[i].fit_id.fti_probe = pp;
1791 pp->ftp_tps[i].fit_id.fti_ptype = pdata->ftps_type;
1792 }
1793
1794 pp->ftp_id = dtrace_probe_create(provider->ftp_provid,
1795 pdata->ftps_mod, pdata->ftps_func, name, aframes, pp);
1796 }
1797
1798 mutex_exit(&provider->ftp_cmtx);
1799
1800 /*
1801 * We know that the provider is still valid since we incremented the
1802 * creation reference count. If someone tried to clean up this provider
1803 * while we were using it (e.g. because the process called exec(2) or
1804 * exit(2)), take note of that and try to clean it up now.
1805 */
1806 mutex_enter(&provider->ftp_mtx);
1807 provider->ftp_ccount--;
1808 whack = provider->ftp_retired;
1809 mutex_exit(&provider->ftp_mtx);
1810
1811 if (whack)
1812 fasttrap_pid_cleanup();
1813
1814 return (0);
1815
1816no_mem:
1817 /*
1818 * If we've exhausted the allowable resources, we'll try to remove
1819 * this provider to free some up. This is to cover the case where
1820 * the user has accidentally created many more probes than was
1821 * intended (e.g. pid123:::).
1822 */
1823 mutex_exit(&provider->ftp_cmtx);
1824 mutex_enter(&provider->ftp_mtx);
1825 provider->ftp_ccount--;
1826 provider->ftp_marked = 1;
1827 mutex_exit(&provider->ftp_mtx);
1828
1829 fasttrap_pid_cleanup();
1830
1831 return (ENOMEM);
1832}
1833
1834/*ARGSUSED*/
1835static void *
1836fasttrap_meta_provide(void *arg, dtrace_helper_provdesc_t *dhpv, pid_t pid)
1837{
1838 fasttrap_provider_t *provider;
1839
1840 /*
1841 * A 32-bit unsigned integer (like a pid for example) can be
1842 * expressed in 10 or fewer decimal digits. Make sure that we'll
1843 * have enough space for the provider name.
1844 */
1845 if (strlen(dhpv->dthpv_provname) + 10 >=
1846 sizeof (provider->ftp_name)) {
1847 printf("failed to instantiate provider %s: "
1848 "name too long to accomodate pid", dhpv->dthpv_provname);
1849 return (NULL);
1850 }
1851
1852 /*
1853 * Don't let folks spoof the true pid provider.
1854 */
1855 if (strcmp(dhpv->dthpv_provname, FASTTRAP_PID_NAME) == 0) {
1856 printf("failed to instantiate provider %s: "
1857 "%s is an invalid name", dhpv->dthpv_provname,
1858 FASTTRAP_PID_NAME);
1859 return (NULL);
1860 }
1861
1862 /*
1863 * The highest stability class that fasttrap supports is ISA; cap
1864 * the stability of the new provider accordingly.
1865 */
1866 if (dhpv->dthpv_pattr.dtpa_provider.dtat_class > DTRACE_CLASS_ISA)
1867 dhpv->dthpv_pattr.dtpa_provider.dtat_class = DTRACE_CLASS_ISA;
1868 if (dhpv->dthpv_pattr.dtpa_mod.dtat_class > DTRACE_CLASS_ISA)
1869 dhpv->dthpv_pattr.dtpa_mod.dtat_class = DTRACE_CLASS_ISA;
1870 if (dhpv->dthpv_pattr.dtpa_func.dtat_class > DTRACE_CLASS_ISA)
1871 dhpv->dthpv_pattr.dtpa_func.dtat_class = DTRACE_CLASS_ISA;
1872 if (dhpv->dthpv_pattr.dtpa_name.dtat_class > DTRACE_CLASS_ISA)
1873 dhpv->dthpv_pattr.dtpa_name.dtat_class = DTRACE_CLASS_ISA;
1874 if (dhpv->dthpv_pattr.dtpa_args.dtat_class > DTRACE_CLASS_ISA)
1875 dhpv->dthpv_pattr.dtpa_args.dtat_class = DTRACE_CLASS_ISA;
1876
1877 if ((provider = fasttrap_provider_lookup(pid, dhpv->dthpv_provname,
1878 &dhpv->dthpv_pattr)) == NULL) {
1879 printf("failed to instantiate provider %s for "
1880 "process %u", dhpv->dthpv_provname, (uint_t)pid);
1881 return (NULL);
1882 }
1883
1884 /*
1885 * Up the meta provider count so this provider isn't removed until
1886 * the meta provider has been told to remove it.
1887 */
1888 provider->ftp_mcount++;
1889
1890 mutex_exit(&provider->ftp_mtx);
1891
1892 return (provider);
1893}
1894
1895/*ARGSUSED*/
1896static void
1897fasttrap_meta_create_probe(void *arg, void *parg,
1898 dtrace_helper_probedesc_t *dhpb)
1899{
1900 fasttrap_provider_t *provider = parg;
1901 fasttrap_probe_t *pp;
1902 fasttrap_tracepoint_t *tp;
1903 int i, j;
1904 uint32_t ntps;
1905
1906 /*
1907 * Since the meta provider count is non-zero we don't have to worry
1908 * about this provider disappearing.
1909 */
1910 ASSERT(provider->ftp_mcount > 0);
1911
1912 /*
1913 * The offsets must be unique.
1914 */
1915 qsort(dhpb->dthpb_offs, dhpb->dthpb_noffs, sizeof (uint32_t),
1916 fasttrap_uint32_cmp);
1917 for (i = 1; i < dhpb->dthpb_noffs; i++) {
1918 if (dhpb->dthpb_base + dhpb->dthpb_offs[i] <=
1919 dhpb->dthpb_base + dhpb->dthpb_offs[i - 1])
1920 return;
1921 }
1922
1923 qsort(dhpb->dthpb_enoffs, dhpb->dthpb_nenoffs, sizeof (uint32_t),
1924 fasttrap_uint32_cmp);
1925 for (i = 1; i < dhpb->dthpb_nenoffs; i++) {
1926 if (dhpb->dthpb_base + dhpb->dthpb_enoffs[i] <=
1927 dhpb->dthpb_base + dhpb->dthpb_enoffs[i - 1])
1928 return;
1929 }
1930
1931 /*
1932 * Grab the creation lock to ensure consistency between calls to
1933 * dtrace_probe_lookup() and dtrace_probe_create() in the face of
1934 * other threads creating probes.
1935 */
1936 mutex_enter(&provider->ftp_cmtx);
1937
1938 if (dtrace_probe_lookup(provider->ftp_provid, dhpb->dthpb_mod,
1939 dhpb->dthpb_func, dhpb->dthpb_name) != 0) {
1940 mutex_exit(&provider->ftp_cmtx);
1941 return;
1942 }
1943
1944 ntps = dhpb->dthpb_noffs + dhpb->dthpb_nenoffs;
1945 ASSERT(ntps > 0);
1946
1947 atomic_add_32(&fasttrap_total, ntps);
1948
1949 if (fasttrap_total > fasttrap_max) {
1950 atomic_add_32(&fasttrap_total, -ntps);
1951 mutex_exit(&provider->ftp_cmtx);
1952 return;
1953 }
1954
1955 pp = kmem_zalloc(offsetof(fasttrap_probe_t, ftp_tps[ntps]), KM_SLEEP);
1956
1957 pp->ftp_prov = provider;
1958 pp->ftp_pid = provider->ftp_pid;
1959 pp->ftp_ntps = ntps;
1960 pp->ftp_nargs = dhpb->dthpb_xargc;
1961 pp->ftp_xtypes = dhpb->dthpb_xtypes;
1962 pp->ftp_ntypes = dhpb->dthpb_ntypes;
1963
1964 /*
1965 * First create a tracepoint for each actual point of interest.
1966 */
1967 for (i = 0; i < dhpb->dthpb_noffs; i++) {
1968 tp = kmem_zalloc(sizeof (fasttrap_tracepoint_t), KM_SLEEP);
1969
1970 tp->ftt_proc = provider->ftp_proc;
1971 tp->ftt_pc = dhpb->dthpb_base + dhpb->dthpb_offs[i];
1972 tp->ftt_pid = provider->ftp_pid;
1973
1974 pp->ftp_tps[i].fit_tp = tp;
1975 pp->ftp_tps[i].fit_id.fti_probe = pp;
1976#ifdef __sparc
1977 pp->ftp_tps[i].fit_id.fti_ptype = DTFTP_POST_OFFSETS;
1978#else
1979 pp->ftp_tps[i].fit_id.fti_ptype = DTFTP_OFFSETS;
1980#endif
1981 }
1982
1983 /*
1984 * Then create a tracepoint for each is-enabled point.
1985 */
1986 for (j = 0; i < ntps; i++, j++) {
1987 tp = kmem_zalloc(sizeof (fasttrap_tracepoint_t), KM_SLEEP);
1988
1989 tp->ftt_proc = provider->ftp_proc;
1990 tp->ftt_pc = dhpb->dthpb_base + dhpb->dthpb_enoffs[j];
1991 tp->ftt_pid = provider->ftp_pid;
1992
1993 pp->ftp_tps[i].fit_tp = tp;
1994 pp->ftp_tps[i].fit_id.fti_probe = pp;
1995 pp->ftp_tps[i].fit_id.fti_ptype = DTFTP_IS_ENABLED;
1996 }
1997
1998 /*
1999 * If the arguments are shuffled around we set the argument remapping
2000 * table. Later, when the probe fires, we only remap the arguments
2001 * if the table is non-NULL.
2002 */
2003 for (i = 0; i < dhpb->dthpb_xargc; i++) {
2004 if (dhpb->dthpb_args[i] != i) {
2005 pp->ftp_argmap = dhpb->dthpb_args;
2006 break;
2007 }
2008 }
2009
2010 /*
2011 * The probe is fully constructed -- register it with DTrace.
2012 */
2013 pp->ftp_id = dtrace_probe_create(provider->ftp_provid, dhpb->dthpb_mod,
2014 dhpb->dthpb_func, dhpb->dthpb_name, FASTTRAP_OFFSET_AFRAMES, pp);
2015
2016 mutex_exit(&provider->ftp_cmtx);
2017}
2018
2019/*ARGSUSED*/
2020static void
2021fasttrap_meta_remove(void *arg, dtrace_helper_provdesc_t *dhpv, pid_t pid)
2022{
2023 /*
2024 * Clean up the USDT provider. There may be active consumers of the
2025 * provider busy adding probes, no damage will actually befall the
2026 * provider until that count has dropped to zero. This just puts
2027 * the provider on death row.
2028 */
2029 fasttrap_provider_retire(pid, dhpv->dthpv_provname, 1);
2030}
2031
2032static dtrace_mops_t fasttrap_mops = {
2033 fasttrap_meta_create_probe,
2034 fasttrap_meta_provide,
2035 fasttrap_meta_remove
2036};
2037
2038/*ARGSUSED*/
2039static int
2040fasttrap_open(struct cdev *dev __unused, int oflags __unused,
2041 int devtype __unused, struct thread *td __unused)
2042{
2043 return (0);
2044}
2045
2046/*ARGSUSED*/
2047static int
2048fasttrap_ioctl(struct cdev *dev, u_long cmd, caddr_t arg, int fflag,
2049 struct thread *td)
2050{
2051#ifdef notyet
2052 struct kinfo_proc kp;
2053 const cred_t *cr = td->td_ucred;
2054#endif
2055 if (!dtrace_attached())
2056 return (EAGAIN);
2057
2058 if (cmd == FASTTRAPIOC_MAKEPROBE) {
2059 fasttrap_probe_spec_t *uprobe = (void *)arg;
2060 fasttrap_probe_spec_t *probe;
2061 uint64_t noffs;
2062 size_t size;
2063 int ret;
2064 char *c;
2065
2066#if defined(sun)
2067 if (copyin(&uprobe->ftps_noffs, &noffs,
2068 sizeof (uprobe->ftps_noffs)))
2069 return (EFAULT);
2070#else
2071 noffs = uprobe->ftps_noffs;
2072#endif
2073
2074 /*
2075 * Probes must have at least one tracepoint.
2076 */
2077 if (noffs == 0)
2078 return (EINVAL);
2079
2080 size = sizeof (fasttrap_probe_spec_t) +
2081 sizeof (probe->ftps_offs[0]) * (noffs - 1);
2082
2083 if (size > 1024 * 1024)
2084 return (ENOMEM);
2085
2086 probe = kmem_alloc(size, KM_SLEEP);
2087
2088#if defined(sun)
2089 if (copyin(uprobe, probe, size) != 0) {
2090 kmem_free(probe, size);
2091 return (EFAULT);
2092 }
2093#else
2094 memcpy(probe, uprobe, sizeof(*probe));
2095 if (noffs > 1 && copyin(uprobe + 1, probe + 1, size) != 0) {
2096 kmem_free(probe, size);
2097 return (EFAULT);
2098 }
2099#endif
2100
2101
2102 /*
2103 * Verify that the function and module strings contain no
2104 * funny characters.
2105 */
2106 for (c = &probe->ftps_func[0]; *c != '\0'; c++) {
2107 if (*c < 0x20 || 0x7f <= *c) {
2108 ret = EINVAL;
2109 goto err;
2110 }
2111 }
2112
2113 for (c = &probe->ftps_mod[0]; *c != '\0'; c++) {
2114 if (*c < 0x20 || 0x7f <= *c) {
2115 ret = EINVAL;
2116 goto err;
2117 }
2118 }
2119
2120#ifdef notyet
2121 if (!PRIV_POLICY_CHOICE(cr, PRIV_ALL, B_FALSE)) {
2122 proc_t *p;
2123 pid_t pid = probe->ftps_pid;
2124
2125#if defined(sun)
2126 mutex_enter(&pidlock);
2127#endif
2128 /*
2129 * Report an error if the process doesn't exist
2130 * or is actively being birthed.
2131 */
2132 p = pfind(pid);
2133 if (p)
2134 fill_kinfo_proc(p, &kp);
2135 if (p == NULL || kp.ki_stat == SIDL) {
2136#if defined(sun)
2137 mutex_exit(&pidlock);
2138#endif
2139 return (ESRCH);
2140 }
2141#if defined(sun)
2142 mutex_enter(&p->p_lock);
2143 mutex_exit(&pidlock);
2144#else
2145 PROC_LOCK_ASSERT(p, MA_OWNED);
2146#endif
2147
2148#ifdef notyet
2149 if ((ret = priv_proc_cred_perm(cr, p, NULL,
2150 VREAD | VWRITE)) != 0) {
2151#if defined(sun)
2152 mutex_exit(&p->p_lock);
2153#else
2154 PROC_UNLOCK(p);
2155#endif
2156 return (ret);
2157 }
2158#endif /* notyet */
2159#if defined(sun)
2160 mutex_exit(&p->p_lock);
2161#else
2162 PROC_UNLOCK(p);
2163#endif
2164 }
2165#endif /* notyet */
2166
2167 ret = fasttrap_add_probe(probe);
2168err:
2169 kmem_free(probe, size);
2170
2171 return (ret);
2172
2173 } else if (cmd == FASTTRAPIOC_GETINSTR) {
2174 fasttrap_instr_query_t instr;
2175 fasttrap_tracepoint_t *tp;
2176 uint_t index;
2177#if defined(sun)
2178 int ret;
2179#endif
2180
2181#if defined(sun)
2182 if (copyin((void *)arg, &instr, sizeof (instr)) != 0)
2183 return (EFAULT);
2184#endif
2185
2186#ifdef notyet
2187 if (!PRIV_POLICY_CHOICE(cr, PRIV_ALL, B_FALSE)) {
2188 proc_t *p;
2189 pid_t pid = instr.ftiq_pid;
2190
2191#if defined(sun)
2192 mutex_enter(&pidlock);
2193#endif
2194 /*
2195 * Report an error if the process doesn't exist
2196 * or is actively being birthed.
2197 */
2198 p = pfind(pid);
2199 if (p)
2200 fill_kinfo_proc(p, &kp);
2201 if (p == NULL || kp.ki_stat == SIDL) {
2202#if defined(sun)
2203 mutex_exit(&pidlock);
2204#endif
2205 return (ESRCH);
2206 }
2207#if defined(sun)
2208 mutex_enter(&p->p_lock);
2209 mutex_exit(&pidlock);
2210#else
2211 PROC_LOCK_ASSERT(p, MA_OWNED);
2212#endif
2213
2214#ifdef notyet
2215 if ((ret = priv_proc_cred_perm(cr, p, NULL,
2216 VREAD)) != 0) {
2217#if defined(sun)
2218 mutex_exit(&p->p_lock);
2219#else
2220 PROC_UNLOCK(p);
2221#endif
2222 return (ret);
2223 }
2224#endif /* notyet */
2225
2226#if defined(sun)
2227 mutex_exit(&p->p_lock);
2228#else
2229 PROC_UNLOCK(p);
2230#endif
2231 }
2232#endif /* notyet */
2233
2234 index = FASTTRAP_TPOINTS_INDEX(instr.ftiq_pid, instr.ftiq_pc);
2235
2236 mutex_enter(&fasttrap_tpoints.fth_table[index].ftb_mtx);
2237 tp = fasttrap_tpoints.fth_table[index].ftb_data;
2238 while (tp != NULL) {
2239 if (instr.ftiq_pid == tp->ftt_pid &&
2240 instr.ftiq_pc == tp->ftt_pc &&
2241 tp->ftt_proc->ftpc_acount != 0)
2242 break;
2243
2244 tp = tp->ftt_next;
2245 }
2246
2247 if (tp == NULL) {
2248 mutex_exit(&fasttrap_tpoints.fth_table[index].ftb_mtx);
2249 return (ENOENT);
2250 }
2251
2252 bcopy(&tp->ftt_instr, &instr.ftiq_instr,
2253 sizeof (instr.ftiq_instr));
2254 mutex_exit(&fasttrap_tpoints.fth_table[index].ftb_mtx);
2255
2256 if (copyout(&instr, (void *)arg, sizeof (instr)) != 0)
2257 return (EFAULT);
2258
2259 return (0);
2260 }
2261
2262 return (EINVAL);
2263}
2264
2265static int
2266fasttrap_load(void)
2267{
2268 ulong_t nent;
2269 int i, ret;
2270
2271 /* Create the /dev/dtrace/fasttrap entry. */
2272 fasttrap_cdev = make_dev(&fasttrap_cdevsw, 0, UID_ROOT, GID_WHEEL, 0600,
2273 "dtrace/fasttrap");
2274
2275 mtx_init(&fasttrap_cleanup_mtx, "fasttrap clean", "dtrace", MTX_DEF);
2276 mutex_init(&fasttrap_count_mtx, "fasttrap count mtx", MUTEX_DEFAULT,
2277 NULL);
2278
2279 ret = kproc_create(fasttrap_pid_cleanup_cb, NULL,
2280 &fasttrap_cleanup_proc, 0, 0, "ftcleanup");
2281 if (ret != 0) {
2282 destroy_dev(fasttrap_cdev);
2283 return (ret);
2284 }
2285
2286 /*
2287 * Install our hooks into fork(2), exec(2), and exit(2).
2288 */
2289 dtrace_fasttrap_fork = &fasttrap_fork;
2290 dtrace_fasttrap_exit = &fasttrap_exec_exit;
2291 dtrace_fasttrap_exec = &fasttrap_exec_exit;
2292
2293#if defined(sun)
2294 fasttrap_max = ddi_getprop(DDI_DEV_T_ANY, devi, DDI_PROP_DONTPASS,
2295 "fasttrap-max-probes", FASTTRAP_MAX_DEFAULT);
2296#else
2297 fasttrap_max = FASTTRAP_MAX_DEFAULT;
2298#endif
2299 fasttrap_total = 0;
2300
2301 /*
2302 * Conjure up the tracepoints hashtable...
2303 */
2304#if defined(sun)
2305 nent = ddi_getprop(DDI_DEV_T_ANY, devi, DDI_PROP_DONTPASS,
2306 "fasttrap-hash-size", FASTTRAP_TPOINTS_DEFAULT_SIZE);
2307#else
2308 nent = FASTTRAP_TPOINTS_DEFAULT_SIZE;
2309#endif
2310
2311 if (nent == 0 || nent > 0x1000000)
2312 nent = FASTTRAP_TPOINTS_DEFAULT_SIZE;
2313
2314 if ((nent & (nent - 1)) == 0)
2315 fasttrap_tpoints.fth_nent = nent;
2316 else
2317 fasttrap_tpoints.fth_nent = 1 << fasttrap_highbit(nent);
2318 ASSERT(fasttrap_tpoints.fth_nent > 0);
2319 fasttrap_tpoints.fth_mask = fasttrap_tpoints.fth_nent - 1;
2320 fasttrap_tpoints.fth_table = kmem_zalloc(fasttrap_tpoints.fth_nent *
2321 sizeof (fasttrap_bucket_t), KM_SLEEP);
2322#if !defined(sun)
2323 for (i = 0; i < fasttrap_tpoints.fth_nent; i++)
2324 mutex_init(&fasttrap_tpoints.fth_table[i].ftb_mtx,
2325 "tracepoints bucket mtx", MUTEX_DEFAULT, NULL);
2326#endif
2327
2328 /*
2329 * ... and the providers hash table...
2330 */
2331 nent = FASTTRAP_PROVIDERS_DEFAULT_SIZE;
2332 if ((nent & (nent - 1)) == 0)
2333 fasttrap_provs.fth_nent = nent;
2334 else
2335 fasttrap_provs.fth_nent = 1 << fasttrap_highbit(nent);
2336 ASSERT(fasttrap_provs.fth_nent > 0);
2337 fasttrap_provs.fth_mask = fasttrap_provs.fth_nent - 1;
2338 fasttrap_provs.fth_table = kmem_zalloc(fasttrap_provs.fth_nent *
2339 sizeof (fasttrap_bucket_t), KM_SLEEP);
2340#if !defined(sun)
2341 for (i = 0; i < fasttrap_provs.fth_nent; i++)
2342 mutex_init(&fasttrap_provs.fth_table[i].ftb_mtx,
2343 "providers bucket mtx", MUTEX_DEFAULT, NULL);
2344#endif
2345
2346 /*
2347 * ... and the procs hash table.
2348 */
2349 nent = FASTTRAP_PROCS_DEFAULT_SIZE;
2350 if ((nent & (nent - 1)) == 0)
2351 fasttrap_procs.fth_nent = nent;
2352 else
2353 fasttrap_procs.fth_nent = 1 << fasttrap_highbit(nent);
2354 ASSERT(fasttrap_procs.fth_nent > 0);
2355 fasttrap_procs.fth_mask = fasttrap_procs.fth_nent - 1;
2356 fasttrap_procs.fth_table = kmem_zalloc(fasttrap_procs.fth_nent *
2357 sizeof (fasttrap_bucket_t), KM_SLEEP);
2358#if !defined(sun)
2359 for (i = 0; i < fasttrap_procs.fth_nent; i++)
2360 mutex_init(&fasttrap_procs.fth_table[i].ftb_mtx,
2361 "processes bucket mtx", MUTEX_DEFAULT, NULL);
2362
2363 CPU_FOREACH(i) {
2364 mutex_init(&fasttrap_cpuc_pid_lock[i], "fasttrap barrier",
2365 MUTEX_DEFAULT, NULL);
2366 }
2367#endif
2368
2369 (void) dtrace_meta_register("fasttrap", &fasttrap_mops, NULL,
2370 &fasttrap_meta_id);
2371
2372 return (0);
2373}
2374
2375static int
2376fasttrap_unload(void)
2377{
2378 int i, fail = 0;
2379
2380 /*
2381 * Unregister the meta-provider to make sure no new fasttrap-
2382 * managed providers come along while we're trying to close up
2383 * shop. If we fail to detach, we'll need to re-register as a
2384 * meta-provider. We can fail to unregister as a meta-provider
2385 * if providers we manage still exist.
2386 */
2387 if (fasttrap_meta_id != DTRACE_METAPROVNONE &&
2388 dtrace_meta_unregister(fasttrap_meta_id) != 0)
2389 return (-1);
2390
2391 /*
2392 * Iterate over all of our providers. If there's still a process
2393 * that corresponds to that pid, fail to detach.
2394 */
2395 for (i = 0; i < fasttrap_provs.fth_nent; i++) {
2396 fasttrap_provider_t **fpp, *fp;
2397 fasttrap_bucket_t *bucket = &fasttrap_provs.fth_table[i];
2398
2399 mutex_enter(&bucket->ftb_mtx);
2400 fpp = (fasttrap_provider_t **)&bucket->ftb_data;
2401 while ((fp = *fpp) != NULL) {
2402 /*
2403 * Acquire and release the lock as a simple way of
2404 * waiting for any other consumer to finish with
2405 * this provider. A thread must first acquire the
2406 * bucket lock so there's no chance of another thread
2407 * blocking on the provider's lock.
2408 */
2409 mutex_enter(&fp->ftp_mtx);
2410 mutex_exit(&fp->ftp_mtx);
2411
2412 if (dtrace_unregister(fp->ftp_provid) != 0) {
2413 fail = 1;
2414 fpp = &fp->ftp_next;
2415 } else {
2416 *fpp = fp->ftp_next;
2417 fasttrap_provider_free(fp);
2418 }
2419 }
2420
2421 mutex_exit(&bucket->ftb_mtx);
2422 }
2423
2424 if (fail) {
2425 (void) dtrace_meta_register("fasttrap", &fasttrap_mops, NULL,
2426 &fasttrap_meta_id);
2427
2428 return (-1);
2429 }
2430
2431 mtx_lock(&fasttrap_cleanup_mtx);
2432 fasttrap_cleanup_drain = 1;
2433 /* Wait for the cleanup thread to finish up and signal us. */
2434 wakeup(&fasttrap_cleanup_cv);
2435 mtx_sleep(&fasttrap_cleanup_drain, &fasttrap_cleanup_mtx, 0, "ftcld",
2436 0);
2437 fasttrap_cleanup_proc = NULL;
|