1/* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License (the "License"). 6 * You may not use this file except in compliance with the License. 7 * 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9 * or http://www.opensolaris.org/os/licensing. 10 * See the License for the specific language governing permissions 11 * and limitations under the License. 12 * 13 * When distributing Covered Code, include this CDDL HEADER in each 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15 * If applicable, add the following below this CDDL HEADER, with the 16 * fields enclosed by brackets "[]" replaced with your own identifying 17 * information: Portions Copyright [yyyy] [name of copyright owner] 18 * 19 * CDDL HEADER END 20 * 21 * Portions Copyright 2010 The FreeBSD Foundation 22 * 23 * $FreeBSD$ 24 */ 25 26/* 27 * Copyright 2008 Sun Microsystems, Inc. All rights reserved. 28 * Use is subject to license terms. 29 */ 30 31/* 32 * Copyright (c) 2015, Joyent, Inc. All rights reserved. 33 */ 34 35#include <sys/atomic.h> 36#include <sys/errno.h> 37#include <sys/stat.h> 38#include <sys/endian.h> 39#include <sys/modctl.h> 40#include <sys/conf.h> 41#include <sys/systm.h> 42#ifdef illumos 43#include <sys/ddi.h> 44#endif 45#include <sys/sunddi.h> 46#include <sys/cpuvar.h> 47#include <sys/kmem.h> 48#ifdef illumos 49#include <sys/strsubr.h> 50#endif 51#include <sys/fasttrap.h> 52#include <sys/fasttrap_impl.h> 53#include <sys/fasttrap_isa.h> 54#include <sys/dtrace.h> 55#include <sys/dtrace_impl.h> 56#include <sys/sysmacros.h> 57#include <sys/proc.h> 58#undef AT_UID 59#undef AT_GID 60#include <sys/policy.h> 61#ifdef illumos 62#include <util/qsort.h> 63#endif 64#include <sys/mutex.h> 65#include <sys/kernel.h> 66#ifndef illumos 67#include <sys/dtrace_bsd.h> 68#include <sys/eventhandler.h> 69#include <sys/rmlock.h> 70#include <sys/sysent.h> 71#include <sys/sysctl.h> 72#include <sys/u8_textprep.h> 73#include <sys/user.h> 74 75#include <vm/vm.h> 76#include <vm/pmap.h> 77#include <vm/vm_map.h> 78#include <vm/vm_param.h> 79 80#include <cddl/dev/dtrace/dtrace_cddl.h> 81#endif 82 83/* 84 * User-Land Trap-Based Tracing 85 * ---------------------------- 86 * 87 * The fasttrap provider allows DTrace consumers to instrument any user-level 88 * instruction to gather data; this includes probes with semantic 89 * signifigance like entry and return as well as simple offsets into the 90 * function. While the specific techniques used are very ISA specific, the 91 * methodology is generalizable to any architecture. 92 * 93 * 94 * The General Methodology 95 * ----------------------- 96 * 97 * With the primary goal of tracing every user-land instruction and the 98 * limitation that we can't trust user space so don't want to rely on much 99 * information there, we begin by replacing the instructions we want to trace 100 * with trap instructions. Each instruction we overwrite is saved into a hash 101 * table keyed by process ID and pc address. When we enter the kernel due to 102 * this trap instruction, we need the effects of the replaced instruction to 103 * appear to have occurred before we proceed with the user thread's 104 * execution. 105 * 106 * Each user level thread is represented by a ulwp_t structure which is 107 * always easily accessible through a register. The most basic way to produce 108 * the effects of the instruction we replaced is to copy that instruction out 109 * to a bit of scratch space reserved in the user thread's ulwp_t structure 110 * (a sort of kernel-private thread local storage), set the PC to that 111 * scratch space and single step. When we reenter the kernel after single 112 * stepping the instruction we must then adjust the PC to point to what would 113 * normally be the next instruction. Of course, special care must be taken 114 * for branches and jumps, but these represent such a small fraction of any 115 * instruction set that writing the code to emulate these in the kernel is 116 * not too difficult. 117 * 118 * Return probes may require several tracepoints to trace every return site, 119 * and, conversely, each tracepoint may activate several probes (the entry 120 * and offset 0 probes, for example). To solve this muliplexing problem, 121 * tracepoints contain lists of probes to activate and probes contain lists 122 * of tracepoints to enable. If a probe is activated, it adds its ID to 123 * existing tracepoints or creates new ones as necessary. 124 * 125 * Most probes are activated _before_ the instruction is executed, but return 126 * probes are activated _after_ the effects of the last instruction of the 127 * function are visible. Return probes must be fired _after_ we have 128 * single-stepped the instruction whereas all other probes are fired 129 * beforehand. 130 * 131 * 132 * Lock Ordering 133 * ------------- 134 * 135 * The lock ordering below -- both internally and with respect to the DTrace 136 * framework -- is a little tricky and bears some explanation. Each provider 137 * has a lock (ftp_mtx) that protects its members including reference counts 138 * for enabled probes (ftp_rcount), consumers actively creating probes 139 * (ftp_ccount) and USDT consumers (ftp_mcount); all three prevent a provider 140 * from being freed. A provider is looked up by taking the bucket lock for the 141 * provider hash table, and is returned with its lock held. The provider lock 142 * may be taken in functions invoked by the DTrace framework, but may not be 143 * held while calling functions in the DTrace framework. 144 * 145 * To ensure consistency over multiple calls to the DTrace framework, the 146 * creation lock (ftp_cmtx) should be held. Naturally, the creation lock may 147 * not be taken when holding the provider lock as that would create a cyclic 148 * lock ordering. In situations where one would naturally take the provider 149 * lock and then the creation lock, we instead up a reference count to prevent 150 * the provider from disappearing, drop the provider lock, and acquire the 151 * creation lock. 152 * 153 * Briefly: 154 * bucket lock before provider lock 155 * DTrace before provider lock 156 * creation lock before DTrace 157 * never hold the provider lock and creation lock simultaneously 158 */ 159 160static d_open_t fasttrap_open; 161static d_ioctl_t fasttrap_ioctl; 162 163static struct cdevsw fasttrap_cdevsw = { 164 .d_version = D_VERSION, 165 .d_open = fasttrap_open, 166 .d_ioctl = fasttrap_ioctl, 167 .d_name = "fasttrap", 168}; 169static struct cdev *fasttrap_cdev; 170static dtrace_meta_provider_id_t fasttrap_meta_id; 171 172static struct proc *fasttrap_cleanup_proc; 173static struct mtx fasttrap_cleanup_mtx; 174static uint_t fasttrap_cleanup_work, fasttrap_cleanup_drain, fasttrap_cleanup_cv; 175 176/* 177 * Generation count on modifications to the global tracepoint lookup table. 178 */ 179static volatile uint64_t fasttrap_mod_gen; 180 181/* 182 * When the fasttrap provider is loaded, fasttrap_max is set to either 183 * FASTTRAP_MAX_DEFAULT, or the value for fasttrap-max-probes in the 184 * fasttrap.conf file (Illumos), or the value provied in the loader.conf (FreeBSD). 185 * Each time a probe is created, fasttrap_total is incremented by the number 186 * of tracepoints that may be associated with that probe; fasttrap_total is capped 187 * at fasttrap_max. 188 */ 189#define FASTTRAP_MAX_DEFAULT 250000 190static uint32_t fasttrap_max = FASTTRAP_MAX_DEFAULT; 191static uint32_t fasttrap_total; 192 193/* 194 * Copyright (c) 2011, Joyent, Inc. All rights reserved. 195 */ 196 197#define FASTTRAP_TPOINTS_DEFAULT_SIZE 0x4000 198#define FASTTRAP_PROVIDERS_DEFAULT_SIZE 0x100 199#define FASTTRAP_PROCS_DEFAULT_SIZE 0x100 200 201#define FASTTRAP_PID_NAME "pid" 202 203fasttrap_hash_t fasttrap_tpoints; 204static fasttrap_hash_t fasttrap_provs; 205static fasttrap_hash_t fasttrap_procs; 206 207static uint64_t fasttrap_pid_count; /* pid ref count */ 208static kmutex_t fasttrap_count_mtx; /* lock on ref count */ 209 210#define FASTTRAP_ENABLE_FAIL 1 211#define FASTTRAP_ENABLE_PARTIAL 2 212 213static int fasttrap_tracepoint_enable(proc_t *, fasttrap_probe_t *, uint_t); 214static void fasttrap_tracepoint_disable(proc_t *, fasttrap_probe_t *, uint_t); 215 216static fasttrap_provider_t *fasttrap_provider_lookup(pid_t, const char *, 217 const dtrace_pattr_t *); 218static void fasttrap_provider_retire(pid_t, const char *, int); 219static void fasttrap_provider_free(fasttrap_provider_t *); 220 221static fasttrap_proc_t *fasttrap_proc_lookup(pid_t); 222static void fasttrap_proc_release(fasttrap_proc_t *); 223 224#ifndef illumos 225static void fasttrap_thread_dtor(void *, struct thread *); 226#endif 227 228#define FASTTRAP_PROVS_INDEX(pid, name) \ 229 ((fasttrap_hash_str(name) + (pid)) & fasttrap_provs.fth_mask) 230 231#define FASTTRAP_PROCS_INDEX(pid) ((pid) & fasttrap_procs.fth_mask) 232 233#ifndef illumos 234struct rmlock fasttrap_tp_lock; 235static eventhandler_tag fasttrap_thread_dtor_tag; 236#endif 237 238static unsigned long tpoints_hash_size = FASTTRAP_TPOINTS_DEFAULT_SIZE; 239 240#ifdef __FreeBSD__ 241SYSCTL_DECL(_kern_dtrace); 242SYSCTL_NODE(_kern_dtrace, OID_AUTO, fasttrap, CTLFLAG_RD | CTLFLAG_MPSAFE, 0, 243 "DTrace fasttrap parameters"); 244SYSCTL_UINT(_kern_dtrace_fasttrap, OID_AUTO, max_probes, CTLFLAG_RWTUN, &fasttrap_max, 245 FASTTRAP_MAX_DEFAULT, "Maximum number of fasttrap probes"); 246SYSCTL_ULONG(_kern_dtrace_fasttrap, OID_AUTO, tpoints_hash_size, CTLFLAG_RDTUN, &tpoints_hash_size, 247 FASTTRAP_TPOINTS_DEFAULT_SIZE, "Size of the tracepoint hash table"); 248#endif 249 250static int 251fasttrap_highbit(ulong_t i) 252{ 253 int h = 1; 254 255 if (i == 0) 256 return (0); 257#ifdef _LP64 258 if (i & 0xffffffff00000000ul) { 259 h += 32; i >>= 32; 260 } 261#endif 262 if (i & 0xffff0000) { 263 h += 16; i >>= 16; 264 } 265 if (i & 0xff00) { 266 h += 8; i >>= 8; 267 } 268 if (i & 0xf0) { 269 h += 4; i >>= 4; 270 } 271 if (i & 0xc) { 272 h += 2; i >>= 2; 273 } 274 if (i & 0x2) { 275 h += 1; 276 } 277 return (h); 278} 279 280static uint_t 281fasttrap_hash_str(const char *p) 282{ 283 unsigned int g; 284 uint_t hval = 0; 285 286 while (*p) { 287 hval = (hval << 4) + *p++; 288 if ((g = (hval & 0xf0000000)) != 0) 289 hval ^= g >> 24; 290 hval &= ~g; 291 } 292 return (hval); 293} 294 295void 296fasttrap_sigtrap(proc_t *p, kthread_t *t, uintptr_t pc) 297{ 298 ksiginfo_t ksi; 299 300 ksiginfo_init(&ksi); 301 ksi.ksi_signo = SIGTRAP; 302 ksi.ksi_code = TRAP_DTRACE; 303 ksi.ksi_addr = (caddr_t)pc; 304 PROC_LOCK(p); 305 (void)tdsendsignal(p, t, SIGTRAP, &ksi); 306 PROC_UNLOCK(p); 307} 308 309#ifndef illumos 310/* 311 * Obtain a chunk of scratch space in the address space of the target process. 312 */ 313fasttrap_scrspace_t * 314fasttrap_scraddr(struct thread *td, fasttrap_proc_t *fprc) 315{ 316 fasttrap_scrblock_t *scrblk; 317 fasttrap_scrspace_t *scrspc; 318 struct proc *p; 319 vm_offset_t addr; 320 int error, i; 321 322 scrspc = NULL; 323 if (td->t_dtrace_sscr != NULL) { 324 /* If the thread already has scratch space, we're done. */ 325 scrspc = (fasttrap_scrspace_t *)td->t_dtrace_sscr; 326 return (scrspc); 327 } 328 329 p = td->td_proc; 330 331 mutex_enter(&fprc->ftpc_mtx); 332 if (LIST_EMPTY(&fprc->ftpc_fscr)) { 333 /* 334 * No scratch space is available, so we'll map a new scratch 335 * space block into the traced process' address space. 336 */ 337 addr = 0; 338 error = vm_map_find(&p->p_vmspace->vm_map, NULL, 0, &addr, 339 FASTTRAP_SCRBLOCK_SIZE, 0, VMFS_ANY_SPACE, VM_PROT_ALL, 340 VM_PROT_ALL, 0); 341 if (error != KERN_SUCCESS) 342 goto done; 343 344 scrblk = malloc(sizeof(*scrblk), M_SOLARIS, M_WAITOK); 345 scrblk->ftsb_addr = addr; 346 LIST_INSERT_HEAD(&fprc->ftpc_scrblks, scrblk, ftsb_next); 347 348 /* 349 * Carve the block up into chunks and put them on the free list. 350 */ 351 for (i = 0; 352 i < FASTTRAP_SCRBLOCK_SIZE / FASTTRAP_SCRSPACE_SIZE; i++) { 353 scrspc = malloc(sizeof(*scrspc), M_SOLARIS, M_WAITOK); 354 scrspc->ftss_addr = addr + 355 i * FASTTRAP_SCRSPACE_SIZE; 356 LIST_INSERT_HEAD(&fprc->ftpc_fscr, scrspc, 357 ftss_next); 358 } 359 } 360 361 /* 362 * Take the first scratch chunk off the free list, put it on the 363 * allocated list, and return its address. 364 */ 365 scrspc = LIST_FIRST(&fprc->ftpc_fscr); 366 LIST_REMOVE(scrspc, ftss_next); 367 LIST_INSERT_HEAD(&fprc->ftpc_ascr, scrspc, ftss_next); 368 369 /* 370 * This scratch space is reserved for use by td until the thread exits. 371 */ 372 td->t_dtrace_sscr = scrspc; 373 374done: 375 mutex_exit(&fprc->ftpc_mtx); 376 377 return (scrspc); 378} 379 380/* 381 * Return any allocated per-thread scratch space chunks back to the process' 382 * free list. 383 */ 384static void 385fasttrap_thread_dtor(void *arg __unused, struct thread *td) 386{ 387 fasttrap_bucket_t *bucket; 388 fasttrap_proc_t *fprc; 389 fasttrap_scrspace_t *scrspc; 390 pid_t pid; 391 392 if (td->t_dtrace_sscr == NULL) 393 return; 394 395 pid = td->td_proc->p_pid; 396 bucket = &fasttrap_procs.fth_table[FASTTRAP_PROCS_INDEX(pid)]; 397 fprc = NULL; 398 399 /* Look up the fasttrap process handle for this process. */ 400 mutex_enter(&bucket->ftb_mtx); 401 for (fprc = bucket->ftb_data; fprc != NULL; fprc = fprc->ftpc_next) { 402 if (fprc->ftpc_pid == pid) { 403 mutex_enter(&fprc->ftpc_mtx); 404 mutex_exit(&bucket->ftb_mtx); 405 break; 406 } 407 } 408 if (fprc == NULL) { 409 mutex_exit(&bucket->ftb_mtx); 410 return; 411 } 412 413 scrspc = (fasttrap_scrspace_t *)td->t_dtrace_sscr; 414 LIST_REMOVE(scrspc, ftss_next); 415 LIST_INSERT_HEAD(&fprc->ftpc_fscr, scrspc, ftss_next); 416 417 mutex_exit(&fprc->ftpc_mtx); 418} 419#endif 420 421/* 422 * This function ensures that no threads are actively using the memory 423 * associated with probes that were formerly live. 424 */ 425static void 426fasttrap_mod_barrier(uint64_t gen) 427{ 428 int i; 429 430 if (gen < fasttrap_mod_gen) 431 return; 432 433 fasttrap_mod_gen++; 434 435#ifdef illumos 436 CPU_FOREACH(i) { 437 mutex_enter(&fasttrap_cpuc_pid_lock[i]); 438 mutex_exit(&fasttrap_cpuc_pid_lock[i]); 439 } 440#else 441 rm_wlock(&fasttrap_tp_lock); 442 rm_wunlock(&fasttrap_tp_lock); 443#endif 444} 445 446/* 447 * This function performs asynchronous cleanup of fasttrap providers. The 448 * Solaris implementation of this mechanism use a timeout that's activated in 449 * fasttrap_pid_cleanup(), but this doesn't work in FreeBSD: one may sleep while 450 * holding the DTrace mutexes, but it is unsafe to sleep in a callout handler. 451 * Thus we use a dedicated process to perform the cleanup when requested. 452 */ 453/*ARGSUSED*/ 454static void 455fasttrap_pid_cleanup_cb(void *data) 456{ 457 fasttrap_provider_t **fpp, *fp; 458 fasttrap_bucket_t *bucket; 459 dtrace_provider_id_t provid; 460 int i, later = 0, rval; 461 462 mtx_lock(&fasttrap_cleanup_mtx); 463 while (!fasttrap_cleanup_drain || later > 0) { 464 fasttrap_cleanup_work = 0; 465 mtx_unlock(&fasttrap_cleanup_mtx); 466 467 later = 0; 468 469 /* 470 * Iterate over all the providers trying to remove the marked 471 * ones. If a provider is marked but not retired, we just 472 * have to take a crack at removing it -- it's no big deal if 473 * we can't. 474 */ 475 for (i = 0; i < fasttrap_provs.fth_nent; i++) { 476 bucket = &fasttrap_provs.fth_table[i]; 477 mutex_enter(&bucket->ftb_mtx); 478 fpp = (fasttrap_provider_t **)&bucket->ftb_data; 479 480 while ((fp = *fpp) != NULL) { 481 if (!fp->ftp_marked) { 482 fpp = &fp->ftp_next; 483 continue; 484 } 485 486 mutex_enter(&fp->ftp_mtx); 487 488 /* 489 * If this provider has consumers actively 490 * creating probes (ftp_ccount) or is a USDT 491 * provider (ftp_mcount), we can't unregister 492 * or even condense. 493 */ 494 if (fp->ftp_ccount != 0 || 495 fp->ftp_mcount != 0) { 496 mutex_exit(&fp->ftp_mtx); 497 fp->ftp_marked = 0; 498 continue; 499 } 500 501 if (!fp->ftp_retired || fp->ftp_rcount != 0) 502 fp->ftp_marked = 0; 503 504 mutex_exit(&fp->ftp_mtx); 505 506 /* 507 * If we successfully unregister this 508 * provider we can remove it from the hash 509 * chain and free the memory. If our attempt 510 * to unregister fails and this is a retired 511 * provider, increment our flag to try again 512 * pretty soon. If we've consumed more than 513 * half of our total permitted number of 514 * probes call dtrace_condense() to try to 515 * clean out the unenabled probes. 516 */ 517 provid = fp->ftp_provid; 518 if ((rval = dtrace_unregister(provid)) != 0) { 519 if (fasttrap_total > fasttrap_max / 2) 520 (void) dtrace_condense(provid); 521 522 if (rval == EAGAIN) 523 fp->ftp_marked = 1; 524 525 later += fp->ftp_marked; 526 fpp = &fp->ftp_next; 527 } else { 528 *fpp = fp->ftp_next; 529 fasttrap_provider_free(fp); 530 } 531 } 532 mutex_exit(&bucket->ftb_mtx); 533 } 534 mtx_lock(&fasttrap_cleanup_mtx); 535 536 /* 537 * If we were unable to retire a provider, try again after a 538 * second. This situation can occur in certain circumstances 539 * where providers cannot be unregistered even though they have 540 * no probes enabled because of an execution of dtrace -l or 541 * something similar. 542 */ 543 if (later > 0 || fasttrap_cleanup_work || 544 fasttrap_cleanup_drain) { 545 mtx_unlock(&fasttrap_cleanup_mtx); 546 pause("ftclean", hz); 547 mtx_lock(&fasttrap_cleanup_mtx); 548 } else 549 mtx_sleep(&fasttrap_cleanup_cv, &fasttrap_cleanup_mtx, 550 0, "ftcl", 0); 551 } 552 553 /* 554 * Wake up the thread in fasttrap_unload() now that we're done. 555 */ 556 wakeup(&fasttrap_cleanup_drain); 557 mtx_unlock(&fasttrap_cleanup_mtx); 558 559 kthread_exit(); 560} 561 562/* 563 * Activates the asynchronous cleanup mechanism. 564 */ 565static void 566fasttrap_pid_cleanup(void) 567{ 568 569 mtx_lock(&fasttrap_cleanup_mtx); 570 if (!fasttrap_cleanup_work) { 571 fasttrap_cleanup_work = 1; 572 wakeup(&fasttrap_cleanup_cv); 573 } 574 mtx_unlock(&fasttrap_cleanup_mtx); 575} 576 577/* 578 * This is called from cfork() via dtrace_fasttrap_fork(). The child 579 * process's address space is (roughly) a copy of the parent process's so 580 * we have to remove all the instrumentation we had previously enabled in the 581 * parent. 582 */ 583static void 584fasttrap_fork(proc_t *p, proc_t *cp) 585{ 586#ifndef illumos 587 fasttrap_scrblock_t *scrblk; 588 fasttrap_proc_t *fprc = NULL; 589#endif 590 pid_t ppid = p->p_pid; 591 int i; 592 593 ASSERT(curproc == p); 594#ifdef illumos 595 ASSERT(p->p_proc_flag & P_PR_LOCK); 596#else 597 PROC_LOCK_ASSERT(p, MA_OWNED); 598#endif 599#ifdef illumos 600 ASSERT(p->p_dtrace_count > 0); 601#else 602 /* 603 * This check is purposely here instead of in kern_fork.c because, 604 * for legal resons, we cannot include the dtrace_cddl.h header 605 * inside kern_fork.c and insert if-clause there. 606 */ 607 if (p->p_dtrace_count == 0 && p->p_dtrace_helpers == NULL) 608 return; 609#endif 610 611 ASSERT(cp->p_dtrace_count == 0); 612 613 /* 614 * This would be simpler and faster if we maintained per-process 615 * hash tables of enabled tracepoints. It could, however, potentially 616 * slow down execution of a tracepoint since we'd need to go 617 * through two levels of indirection. In the future, we should 618 * consider either maintaining per-process ancillary lists of 619 * enabled tracepoints or hanging a pointer to a per-process hash 620 * table of enabled tracepoints off the proc structure. 621 */ 622 623 /* 624 * We don't have to worry about the child process disappearing 625 * because we're in fork(). 626 */ 627#ifdef illumos 628 mtx_lock_spin(&cp->p_slock); 629 sprlock_proc(cp); 630 mtx_unlock_spin(&cp->p_slock); 631#else 632 /* 633 * fasttrap_tracepoint_remove() expects the child process to be 634 * unlocked and the VM then expects curproc to be unlocked. 635 */ 636 _PHOLD(cp); 637 PROC_UNLOCK(cp); 638 PROC_UNLOCK(p); 639 if (p->p_dtrace_count == 0) 640 goto dup_helpers; 641#endif 642 643 /* 644 * Iterate over every tracepoint looking for ones that belong to the 645 * parent process, and remove each from the child process. 646 */ 647 for (i = 0; i < fasttrap_tpoints.fth_nent; i++) { 648 fasttrap_tracepoint_t *tp; 649 fasttrap_bucket_t *bucket = &fasttrap_tpoints.fth_table[i]; 650 651 mutex_enter(&bucket->ftb_mtx); 652 for (tp = bucket->ftb_data; tp != NULL; tp = tp->ftt_next) { 653 if (tp->ftt_pid == ppid && 654 tp->ftt_proc->ftpc_acount != 0) { 655 int ret = fasttrap_tracepoint_remove(cp, tp); 656 ASSERT(ret == 0); 657 658 /* 659 * The count of active providers can only be 660 * decremented (i.e. to zero) during exec, 661 * exit, and removal of a meta provider so it 662 * should be impossible to drop the count 663 * mid-fork. 664 */ 665 ASSERT(tp->ftt_proc->ftpc_acount != 0); 666#ifndef illumos 667 fprc = tp->ftt_proc; 668#endif 669 } 670 } 671 mutex_exit(&bucket->ftb_mtx); 672 673#ifndef illumos 674 /* 675 * Unmap any scratch space inherited from the parent's address 676 * space. 677 */ 678 if (fprc != NULL) { 679 mutex_enter(&fprc->ftpc_mtx); 680 LIST_FOREACH(scrblk, &fprc->ftpc_scrblks, ftsb_next) { 681 vm_map_remove(&cp->p_vmspace->vm_map, 682 scrblk->ftsb_addr, 683 scrblk->ftsb_addr + FASTTRAP_SCRBLOCK_SIZE); 684 } 685 mutex_exit(&fprc->ftpc_mtx); 686 } 687#endif 688 } 689 690#ifdef illumos 691 mutex_enter(&cp->p_lock); 692 sprunlock(cp); 693#else 694dup_helpers: 695 if (p->p_dtrace_helpers != NULL) 696 dtrace_helpers_duplicate(p, cp); 697 PROC_LOCK(p); 698 PROC_LOCK(cp); 699 _PRELE(cp); 700#endif 701} 702 703/* 704 * This is called from proc_exit() or from exec_common() if p_dtrace_probes 705 * is set on the proc structure to indicate that there is a pid provider 706 * associated with this process. 707 */ 708static void 709fasttrap_exec_exit(proc_t *p) 710{ 711#ifndef illumos 712 struct thread *td; 713#endif 714 715#ifdef illumos 716 ASSERT(p == curproc); 717#else 718 PROC_LOCK_ASSERT(p, MA_OWNED); 719 _PHOLD(p); 720 /* 721 * Since struct threads may be recycled, we cannot rely on t_dtrace_sscr 722 * fields to be zeroed by kdtrace_thread_ctor. Thus we must zero it 723 * ourselves when a process exits. 724 */ 725 FOREACH_THREAD_IN_PROC(p, td) 726 td->t_dtrace_sscr = NULL; 727 PROC_UNLOCK(p); 728#endif 729 730 /* 731 * We clean up the pid provider for this process here; user-land 732 * static probes are handled by the meta-provider remove entry point. 733 */ 734 fasttrap_provider_retire(p->p_pid, FASTTRAP_PID_NAME, 0); 735#ifndef illumos 736 if (p->p_dtrace_helpers) 737 dtrace_helpers_destroy(p); 738 PROC_LOCK(p); 739 _PRELE(p); 740#endif 741} 742 743 744/*ARGSUSED*/ 745static void 746fasttrap_pid_provide(void *arg, dtrace_probedesc_t *desc) 747{ 748 /* 749 * There are no "default" pid probes. 750 */ 751} 752 753static int 754fasttrap_tracepoint_enable(proc_t *p, fasttrap_probe_t *probe, uint_t index) 755{ 756 fasttrap_tracepoint_t *tp, *new_tp = NULL; 757 fasttrap_bucket_t *bucket; 758 fasttrap_id_t *id; 759 pid_t pid; 760 uintptr_t pc; 761 762 ASSERT(index < probe->ftp_ntps); 763 764 pid = probe->ftp_pid; 765 pc = probe->ftp_tps[index].fit_tp->ftt_pc; 766 id = &probe->ftp_tps[index].fit_id; 767 768 ASSERT(probe->ftp_tps[index].fit_tp->ftt_pid == pid); 769 770#ifdef illumos 771 ASSERT(!(p->p_flag & SVFORK)); 772#endif 773 774 /* 775 * Before we make any modifications, make sure we've imposed a barrier 776 * on the generation in which this probe was last modified. 777 */ 778 fasttrap_mod_barrier(probe->ftp_gen); 779 780 bucket = &fasttrap_tpoints.fth_table[FASTTRAP_TPOINTS_INDEX(pid, pc)]; 781 782 /* 783 * If the tracepoint has already been enabled, just add our id to the 784 * list of interested probes. This may be our second time through 785 * this path in which case we'll have constructed the tracepoint we'd 786 * like to install. If we can't find a match, and have an allocated 787 * tracepoint ready to go, enable that one now. 788 * 789 * A tracepoint whose process is defunct is also considered defunct. 790 */ 791again: 792 mutex_enter(&bucket->ftb_mtx); 793 for (tp = bucket->ftb_data; tp != NULL; tp = tp->ftt_next) { 794 /* 795 * Note that it's safe to access the active count on the 796 * associated proc structure because we know that at least one 797 * provider (this one) will still be around throughout this 798 * operation. 799 */ 800 if (tp->ftt_pid != pid || tp->ftt_pc != pc || 801 tp->ftt_proc->ftpc_acount == 0) 802 continue; 803 804 /* 805 * Now that we've found a matching tracepoint, it would be 806 * a decent idea to confirm that the tracepoint is still 807 * enabled and the trap instruction hasn't been overwritten. 808 * Since this is a little hairy, we'll punt for now. 809 */ 810 811 /* 812 * This can't be the first interested probe. We don't have 813 * to worry about another thread being in the midst of 814 * deleting this tracepoint (which would be the only valid 815 * reason for a tracepoint to have no interested probes) 816 * since we're holding P_PR_LOCK for this process. 817 */ 818 ASSERT(tp->ftt_ids != NULL || tp->ftt_retids != NULL); 819 820 switch (id->fti_ptype) { 821 case DTFTP_ENTRY: 822 case DTFTP_OFFSETS: 823 case DTFTP_IS_ENABLED: 824 id->fti_next = tp->ftt_ids; 825 membar_producer(); 826 tp->ftt_ids = id; 827 membar_producer(); 828 break; 829 830 case DTFTP_RETURN: 831 case DTFTP_POST_OFFSETS: 832 id->fti_next = tp->ftt_retids; 833 membar_producer(); 834 tp->ftt_retids = id; 835 membar_producer(); 836 break; 837 838 default: 839 ASSERT(0); 840 } 841 842 mutex_exit(&bucket->ftb_mtx); 843 844 if (new_tp != NULL) { 845 new_tp->ftt_ids = NULL; 846 new_tp->ftt_retids = NULL; 847 } 848 849 return (0); 850 } 851 852 /* 853 * If we have a good tracepoint ready to go, install it now while 854 * we have the lock held and no one can screw with us. 855 */ 856 if (new_tp != NULL) { 857 int rc = 0; 858 859 new_tp->ftt_next = bucket->ftb_data; 860 membar_producer(); 861 bucket->ftb_data = new_tp; 862 membar_producer(); 863 mutex_exit(&bucket->ftb_mtx); 864 865 /* 866 * Activate the tracepoint in the ISA-specific manner. 867 * If this fails, we need to report the failure, but 868 * indicate that this tracepoint must still be disabled 869 * by calling fasttrap_tracepoint_disable(). 870 */ 871 if (fasttrap_tracepoint_install(p, new_tp) != 0) 872 rc = FASTTRAP_ENABLE_PARTIAL; 873 874 /* 875 * Increment the count of the number of tracepoints active in 876 * the victim process. 877 */ 878#ifdef illumos 879 ASSERT(p->p_proc_flag & P_PR_LOCK); 880#endif 881 p->p_dtrace_count++; 882 883 return (rc); 884 } 885 886 mutex_exit(&bucket->ftb_mtx); 887 888 /* 889 * Initialize the tracepoint that's been preallocated with the probe. 890 */ 891 new_tp = probe->ftp_tps[index].fit_tp; 892 893 ASSERT(new_tp->ftt_pid == pid); 894 ASSERT(new_tp->ftt_pc == pc); 895 ASSERT(new_tp->ftt_proc == probe->ftp_prov->ftp_proc); 896 ASSERT(new_tp->ftt_ids == NULL); 897 ASSERT(new_tp->ftt_retids == NULL); 898 899 switch (id->fti_ptype) { 900 case DTFTP_ENTRY: 901 case DTFTP_OFFSETS: 902 case DTFTP_IS_ENABLED: 903 id->fti_next = NULL; 904 new_tp->ftt_ids = id; 905 break; 906 907 case DTFTP_RETURN: 908 case DTFTP_POST_OFFSETS: 909 id->fti_next = NULL; 910 new_tp->ftt_retids = id; 911 break; 912 913 default: 914 ASSERT(0); 915 } 916 917#ifdef __FreeBSD__ 918 if (SV_PROC_FLAG(p, SV_LP64)) 919 p->p_model = DATAMODEL_LP64; 920 else 921 p->p_model = DATAMODEL_ILP32; 922#endif 923 924 /* 925 * If the ISA-dependent initialization goes to plan, go back to the 926 * beginning and try to install this freshly made tracepoint. 927 */ 928 if (fasttrap_tracepoint_init(p, new_tp, pc, id->fti_ptype) == 0) 929 goto again; 930 931 new_tp->ftt_ids = NULL; 932 new_tp->ftt_retids = NULL; 933 934 return (FASTTRAP_ENABLE_FAIL); 935} 936 937static void 938fasttrap_tracepoint_disable(proc_t *p, fasttrap_probe_t *probe, uint_t index) 939{ 940 fasttrap_bucket_t *bucket; 941 fasttrap_provider_t *provider = probe->ftp_prov; 942 fasttrap_tracepoint_t **pp, *tp; 943 fasttrap_id_t *id, **idp = NULL; 944 pid_t pid; 945 uintptr_t pc; 946 947 ASSERT(index < probe->ftp_ntps); 948 949 pid = probe->ftp_pid; 950 pc = probe->ftp_tps[index].fit_tp->ftt_pc; 951 id = &probe->ftp_tps[index].fit_id; 952 953 ASSERT(probe->ftp_tps[index].fit_tp->ftt_pid == pid); 954 955 /* 956 * Find the tracepoint and make sure that our id is one of the 957 * ones registered with it. 958 */ 959 bucket = &fasttrap_tpoints.fth_table[FASTTRAP_TPOINTS_INDEX(pid, pc)]; 960 mutex_enter(&bucket->ftb_mtx); 961 for (tp = bucket->ftb_data; tp != NULL; tp = tp->ftt_next) { 962 if (tp->ftt_pid == pid && tp->ftt_pc == pc && 963 tp->ftt_proc == provider->ftp_proc) 964 break; 965 } 966 967 /* 968 * If we somehow lost this tracepoint, we're in a world of hurt. 969 */ 970 ASSERT(tp != NULL); 971 972 switch (id->fti_ptype) { 973 case DTFTP_ENTRY: 974 case DTFTP_OFFSETS: 975 case DTFTP_IS_ENABLED: 976 ASSERT(tp->ftt_ids != NULL); 977 idp = &tp->ftt_ids; 978 break; 979 980 case DTFTP_RETURN: 981 case DTFTP_POST_OFFSETS: 982 ASSERT(tp->ftt_retids != NULL); 983 idp = &tp->ftt_retids; 984 break; 985 986 default: 987 ASSERT(0); 988 } 989 990 while ((*idp)->fti_probe != probe) { 991 idp = &(*idp)->fti_next; 992 ASSERT(*idp != NULL); 993 } 994 995 id = *idp; 996 *idp = id->fti_next; 997 membar_producer(); 998 999 ASSERT(id->fti_probe == probe); 1000 1001 /* 1002 * If there are other registered enablings of this tracepoint, we're 1003 * all done, but if this was the last probe assocated with this 1004 * this tracepoint, we need to remove and free it. 1005 */ 1006 if (tp->ftt_ids != NULL || tp->ftt_retids != NULL) { 1007 1008 /* 1009 * If the current probe's tracepoint is in use, swap it 1010 * for an unused tracepoint. 1011 */ 1012 if (tp == probe->ftp_tps[index].fit_tp) { 1013 fasttrap_probe_t *tmp_probe; 1014 fasttrap_tracepoint_t **tmp_tp; 1015 uint_t tmp_index; 1016 1017 if (tp->ftt_ids != NULL) { 1018 tmp_probe = tp->ftt_ids->fti_probe; 1019 /* LINTED - alignment */ 1020 tmp_index = FASTTRAP_ID_INDEX(tp->ftt_ids); 1021 tmp_tp = &tmp_probe->ftp_tps[tmp_index].fit_tp; 1022 } else { 1023 tmp_probe = tp->ftt_retids->fti_probe; 1024 /* LINTED - alignment */ 1025 tmp_index = FASTTRAP_ID_INDEX(tp->ftt_retids); 1026 tmp_tp = &tmp_probe->ftp_tps[tmp_index].fit_tp; 1027 } 1028 1029 ASSERT(*tmp_tp != NULL); 1030 ASSERT(*tmp_tp != probe->ftp_tps[index].fit_tp); 1031 ASSERT((*tmp_tp)->ftt_ids == NULL); 1032 ASSERT((*tmp_tp)->ftt_retids == NULL); 1033 1034 probe->ftp_tps[index].fit_tp = *tmp_tp; 1035 *tmp_tp = tp; 1036 } 1037 1038 mutex_exit(&bucket->ftb_mtx); 1039 1040 /* 1041 * Tag the modified probe with the generation in which it was 1042 * changed. 1043 */ 1044 probe->ftp_gen = fasttrap_mod_gen; 1045 return; 1046 } 1047 1048 mutex_exit(&bucket->ftb_mtx); 1049 1050 /* 1051 * We can't safely remove the tracepoint from the set of active 1052 * tracepoints until we've actually removed the fasttrap instruction 1053 * from the process's text. We can, however, operate on this 1054 * tracepoint secure in the knowledge that no other thread is going to 1055 * be looking at it since we hold P_PR_LOCK on the process if it's 1056 * live or we hold the provider lock on the process if it's dead and 1057 * gone. 1058 */ 1059 1060 /* 1061 * We only need to remove the actual instruction if we're looking 1062 * at an existing process 1063 */ 1064 if (p != NULL) { 1065 /* 1066 * If we fail to restore the instruction we need to kill 1067 * this process since it's in a completely unrecoverable 1068 * state. 1069 */ 1070 if (fasttrap_tracepoint_remove(p, tp) != 0) 1071 fasttrap_sigtrap(p, NULL, pc); 1072 1073 /* 1074 * Decrement the count of the number of tracepoints active 1075 * in the victim process. 1076 */ 1077#ifdef illumos 1078 ASSERT(p->p_proc_flag & P_PR_LOCK); 1079#endif 1080 p->p_dtrace_count--; 1081 1082 atomic_add_rel_64(&p->p_fasttrap_tp_gen, 1); 1083 } 1084 1085 /* 1086 * Remove the probe from the hash table of active tracepoints. 1087 */ 1088 mutex_enter(&bucket->ftb_mtx); 1089 pp = (fasttrap_tracepoint_t **)&bucket->ftb_data; 1090 ASSERT(*pp != NULL); 1091 while (*pp != tp) { 1092 pp = &(*pp)->ftt_next; 1093 ASSERT(*pp != NULL); 1094 } 1095 1096 *pp = tp->ftt_next; 1097 membar_producer(); 1098 1099 mutex_exit(&bucket->ftb_mtx); 1100 1101 /* 1102 * Tag the modified probe with the generation in which it was changed. 1103 */ 1104 probe->ftp_gen = fasttrap_mod_gen; 1105} 1106 1107static void 1108fasttrap_enable_callbacks(void) 1109{ 1110 /* 1111 * We don't have to play the rw lock game here because we're 1112 * providing something rather than taking something away -- 1113 * we can be sure that no threads have tried to follow this 1114 * function pointer yet. 1115 */ 1116 mutex_enter(&fasttrap_count_mtx); 1117 if (fasttrap_pid_count == 0) { 1118 ASSERT(dtrace_pid_probe_ptr == NULL); 1119 ASSERT(dtrace_return_probe_ptr == NULL); 1120 dtrace_pid_probe_ptr = &fasttrap_pid_probe; 1121 dtrace_return_probe_ptr = &fasttrap_return_probe; 1122 } 1123 ASSERT(dtrace_pid_probe_ptr == &fasttrap_pid_probe); 1124 ASSERT(dtrace_return_probe_ptr == &fasttrap_return_probe); 1125 fasttrap_pid_count++; 1126 mutex_exit(&fasttrap_count_mtx); 1127} 1128 1129static void 1130fasttrap_disable_callbacks(void) 1131{ 1132 mutex_enter(&fasttrap_count_mtx); 1133 ASSERT(fasttrap_pid_count > 0); 1134 fasttrap_pid_count--; 1135 if (fasttrap_pid_count == 0) { 1136 /* 1137 * Synchronize with the breakpoint handler, which is careful to 1138 * enable interrupts only after loading the hook pointer. 1139 */ 1140 dtrace_sync(); 1141 dtrace_pid_probe_ptr = NULL; 1142 dtrace_return_probe_ptr = NULL; 1143 } 1144 mutex_exit(&fasttrap_count_mtx); 1145} 1146 1147/*ARGSUSED*/ 1148static void 1149fasttrap_pid_enable(void *arg, dtrace_id_t id, void *parg) 1150{ 1151 fasttrap_probe_t *probe = parg; 1152 proc_t *p = NULL; 1153 int i, rc; 1154 1155 ASSERT(probe != NULL); 1156 ASSERT(!probe->ftp_enabled); 1157 ASSERT(id == probe->ftp_id); 1158#ifdef illumos 1159 ASSERT(MUTEX_HELD(&cpu_lock)); 1160#endif 1161 1162 /* 1163 * Increment the count of enabled probes on this probe's provider; 1164 * the provider can't go away while the probe still exists. We 1165 * must increment this even if we aren't able to properly enable 1166 * this probe. 1167 */ 1168 mutex_enter(&probe->ftp_prov->ftp_mtx); 1169 probe->ftp_prov->ftp_rcount++; 1170 mutex_exit(&probe->ftp_prov->ftp_mtx); 1171 1172 /* 1173 * If this probe's provider is retired (meaning it was valid in a 1174 * previously exec'ed incarnation of this address space), bail out. The 1175 * provider can't go away while we're in this code path. 1176 */ 1177 if (probe->ftp_prov->ftp_retired) 1178 return; 1179 1180 /* 1181 * If we can't find the process, it may be that we're in the context of 1182 * a fork in which the traced process is being born and we're copying 1183 * USDT probes. Otherwise, the process is gone so bail. 1184 */ 1185#ifdef illumos 1186 if ((p = sprlock(probe->ftp_pid)) == NULL) { 1187 if ((curproc->p_flag & SFORKING) == 0) 1188 return; 1189 1190 mutex_enter(&pidlock); 1191 p = prfind(probe->ftp_pid); 1192 1193 if (p == NULL) { 1194 /* 1195 * So it's not that the target process is being born, 1196 * it's that it isn't there at all (and we simply 1197 * happen to be forking). Anyway, we know that the 1198 * target is definitely gone, so bail out. 1199 */ 1200 mutex_exit(&pidlock); 1201 return (0); 1202 } 1203 1204 /* 1205 * Confirm that curproc is indeed forking the process in which 1206 * we're trying to enable probes. 1207 */ 1208 ASSERT(p->p_parent == curproc); 1209 ASSERT(p->p_stat == SIDL); 1210 1211 mutex_enter(&p->p_lock); 1212 mutex_exit(&pidlock); 1213 1214 sprlock_proc(p); 1215 } 1216 1217 ASSERT(!(p->p_flag & SVFORK)); 1218 mutex_exit(&p->p_lock); 1219#else 1220 if (pget(probe->ftp_pid, PGET_HOLD | PGET_NOTWEXIT, &p) != 0) 1221 return; 1222#endif 1223 1224 /* 1225 * We have to enable the trap entry point before any user threads have 1226 * the chance to execute the trap instruction we're about to place 1227 * in their process's text. 1228 */ 1229 fasttrap_enable_callbacks(); 1230 1231 /* 1232 * Enable all the tracepoints and add this probe's id to each 1233 * tracepoint's list of active probes. 1234 */ 1235 for (i = 0; i < probe->ftp_ntps; i++) { 1236 if ((rc = fasttrap_tracepoint_enable(p, probe, i)) != 0) { 1237 /* 1238 * If enabling the tracepoint failed completely, 1239 * we don't have to disable it; if the failure 1240 * was only partial we must disable it. 1241 */ 1242 if (rc == FASTTRAP_ENABLE_FAIL) 1243 i--; 1244 else 1245 ASSERT(rc == FASTTRAP_ENABLE_PARTIAL); 1246 1247 /* 1248 * Back up and pull out all the tracepoints we've 1249 * created so far for this probe. 1250 */ 1251 while (i >= 0) { 1252 fasttrap_tracepoint_disable(p, probe, i); 1253 i--; 1254 } 1255 1256#ifdef illumos 1257 mutex_enter(&p->p_lock); 1258 sprunlock(p); 1259#else 1260 PRELE(p); 1261#endif 1262 1263 /* 1264 * Since we're not actually enabling this probe, 1265 * drop our reference on the trap table entry. 1266 */ 1267 fasttrap_disable_callbacks(); 1268 return; 1269 } 1270 } 1271#ifdef illumos 1272 mutex_enter(&p->p_lock); 1273 sprunlock(p); 1274#else 1275 PRELE(p); 1276#endif 1277 1278 probe->ftp_enabled = 1; 1279} 1280 1281/*ARGSUSED*/ 1282static void 1283fasttrap_pid_disable(void *arg, dtrace_id_t id, void *parg) 1284{ 1285 fasttrap_probe_t *probe = parg; 1286 fasttrap_provider_t *provider = probe->ftp_prov; 1287 proc_t *p; 1288 int i, whack = 0; 1289 1290 ASSERT(id == probe->ftp_id); 1291 1292 mutex_enter(&provider->ftp_mtx); 1293 1294 /* 1295 * We won't be able to acquire a /proc-esque lock on the process 1296 * iff the process is dead and gone. In this case, we rely on the 1297 * provider lock as a point of mutual exclusion to prevent other 1298 * DTrace consumers from disabling this probe. 1299 */ 1300 if (pget(probe->ftp_pid, PGET_HOLD | PGET_NOTWEXIT, &p) != 0) 1301 p = NULL; 1302 1303 /* 1304 * Disable all the associated tracepoints (for fully enabled probes). 1305 */ 1306 if (probe->ftp_enabled) { 1307 for (i = 0; i < probe->ftp_ntps; i++) { 1308 fasttrap_tracepoint_disable(p, probe, i); 1309 } 1310 } 1311 1312 ASSERT(provider->ftp_rcount > 0); 1313 provider->ftp_rcount--; 1314 1315 if (p != NULL) { 1316 /* 1317 * Even though we may not be able to remove it entirely, we 1318 * mark this retired provider to get a chance to remove some 1319 * of the associated probes. 1320 */ 1321 if (provider->ftp_retired && !provider->ftp_marked) 1322 whack = provider->ftp_marked = 1; 1323 mutex_exit(&provider->ftp_mtx); 1324 } else { 1325 /* 1326 * If the process is dead, we're just waiting for the 1327 * last probe to be disabled to be able to free it. 1328 */ 1329 if (provider->ftp_rcount == 0 && !provider->ftp_marked) 1330 whack = provider->ftp_marked = 1; 1331 mutex_exit(&provider->ftp_mtx); 1332 } 1333 1334 if (whack) 1335 fasttrap_pid_cleanup(); 1336 1337#ifdef __FreeBSD__ 1338 if (p != NULL) 1339 PRELE(p); 1340#endif 1341 if (!probe->ftp_enabled) 1342 return; 1343 1344 probe->ftp_enabled = 0; 1345 1346#ifdef illumos 1347 ASSERT(MUTEX_HELD(&cpu_lock)); 1348#endif 1349 fasttrap_disable_callbacks(); 1350} 1351 1352/*ARGSUSED*/ 1353static void 1354fasttrap_pid_getargdesc(void *arg, dtrace_id_t id, void *parg, 1355 dtrace_argdesc_t *desc) 1356{ 1357 fasttrap_probe_t *probe = parg; 1358 char *str; 1359 int i, ndx; 1360 1361 desc->dtargd_native[0] = '\0'; 1362 desc->dtargd_xlate[0] = '\0'; 1363 1364 if (probe->ftp_prov->ftp_retired != 0 || 1365 desc->dtargd_ndx >= probe->ftp_nargs) { 1366 desc->dtargd_ndx = DTRACE_ARGNONE; 1367 return; 1368 } 1369 1370 ndx = (probe->ftp_argmap != NULL) ? 1371 probe->ftp_argmap[desc->dtargd_ndx] : desc->dtargd_ndx; 1372 1373 str = probe->ftp_ntypes; 1374 for (i = 0; i < ndx; i++) { 1375 str += strlen(str) + 1; 1376 } 1377 1378 ASSERT(strlen(str + 1) < sizeof (desc->dtargd_native)); 1379 (void) strcpy(desc->dtargd_native, str); 1380 1381 if (probe->ftp_xtypes == NULL) 1382 return; 1383 1384 str = probe->ftp_xtypes; 1385 for (i = 0; i < desc->dtargd_ndx; i++) { 1386 str += strlen(str) + 1; 1387 } 1388 1389 ASSERT(strlen(str + 1) < sizeof (desc->dtargd_xlate)); 1390 (void) strcpy(desc->dtargd_xlate, str); 1391} 1392 1393/*ARGSUSED*/ 1394static void 1395fasttrap_pid_destroy(void *arg, dtrace_id_t id, void *parg) 1396{ 1397 fasttrap_probe_t *probe = parg; 1398 int i; 1399 size_t size; 1400 1401 ASSERT(probe != NULL); 1402 ASSERT(!probe->ftp_enabled); 1403 ASSERT(fasttrap_total >= probe->ftp_ntps); 1404 1405 atomic_add_32(&fasttrap_total, -probe->ftp_ntps); 1406 size = offsetof(fasttrap_probe_t, ftp_tps[probe->ftp_ntps]); 1407 1408 if (probe->ftp_gen + 1 >= fasttrap_mod_gen) 1409 fasttrap_mod_barrier(probe->ftp_gen); 1410 1411 for (i = 0; i < probe->ftp_ntps; i++) { 1412 kmem_free(probe->ftp_tps[i].fit_tp, 1413 sizeof (fasttrap_tracepoint_t)); 1414 } 1415 1416 kmem_free(probe, size); 1417} 1418 1419 1420static const dtrace_pattr_t pid_attr = { 1421{ DTRACE_STABILITY_EVOLVING, DTRACE_STABILITY_EVOLVING, DTRACE_CLASS_ISA }, 1422{ DTRACE_STABILITY_PRIVATE, DTRACE_STABILITY_PRIVATE, DTRACE_CLASS_UNKNOWN }, 1423{ DTRACE_STABILITY_PRIVATE, DTRACE_STABILITY_PRIVATE, DTRACE_CLASS_UNKNOWN }, 1424{ DTRACE_STABILITY_EVOLVING, DTRACE_STABILITY_EVOLVING, DTRACE_CLASS_ISA }, 1425{ DTRACE_STABILITY_PRIVATE, DTRACE_STABILITY_PRIVATE, DTRACE_CLASS_UNKNOWN }, 1426}; 1427 1428static dtrace_pops_t pid_pops = { 1429 .dtps_provide = fasttrap_pid_provide, 1430 .dtps_provide_module = NULL, 1431 .dtps_enable = fasttrap_pid_enable, 1432 .dtps_disable = fasttrap_pid_disable, 1433 .dtps_suspend = NULL, 1434 .dtps_resume = NULL, 1435 .dtps_getargdesc = fasttrap_pid_getargdesc, 1436 .dtps_getargval = fasttrap_pid_getarg, 1437 .dtps_usermode = NULL, 1438 .dtps_destroy = fasttrap_pid_destroy 1439}; 1440 1441static dtrace_pops_t usdt_pops = { 1442 .dtps_provide = fasttrap_pid_provide, 1443 .dtps_provide_module = NULL, 1444 .dtps_enable = fasttrap_pid_enable, 1445 .dtps_disable = fasttrap_pid_disable, 1446 .dtps_suspend = NULL, 1447 .dtps_resume = NULL, 1448 .dtps_getargdesc = fasttrap_pid_getargdesc, 1449 .dtps_getargval = fasttrap_usdt_getarg, 1450 .dtps_usermode = NULL, 1451 .dtps_destroy = fasttrap_pid_destroy 1452}; 1453 1454static fasttrap_proc_t * 1455fasttrap_proc_lookup(pid_t pid) 1456{ 1457 fasttrap_bucket_t *bucket; 1458 fasttrap_proc_t *fprc, *new_fprc; 1459 1460 1461 bucket = &fasttrap_procs.fth_table[FASTTRAP_PROCS_INDEX(pid)]; 1462 mutex_enter(&bucket->ftb_mtx); 1463 1464 for (fprc = bucket->ftb_data; fprc != NULL; fprc = fprc->ftpc_next) { 1465 if (fprc->ftpc_pid == pid && fprc->ftpc_acount != 0) { 1466 mutex_enter(&fprc->ftpc_mtx); 1467 mutex_exit(&bucket->ftb_mtx); 1468 fprc->ftpc_rcount++; 1469 atomic_inc_64(&fprc->ftpc_acount); 1470 ASSERT(fprc->ftpc_acount <= fprc->ftpc_rcount); 1471 mutex_exit(&fprc->ftpc_mtx); 1472 1473 return (fprc); 1474 } 1475 } 1476 1477 /* 1478 * Drop the bucket lock so we don't try to perform a sleeping 1479 * allocation under it. 1480 */ 1481 mutex_exit(&bucket->ftb_mtx); 1482 1483 new_fprc = kmem_zalloc(sizeof (fasttrap_proc_t), KM_SLEEP); 1484 new_fprc->ftpc_pid = pid; 1485 new_fprc->ftpc_rcount = 1; 1486 new_fprc->ftpc_acount = 1; 1487#ifndef illumos 1488 mutex_init(&new_fprc->ftpc_mtx, "fasttrap proc mtx", MUTEX_DEFAULT, 1489 NULL); 1490#endif 1491 1492 mutex_enter(&bucket->ftb_mtx); 1493 1494 /* 1495 * Take another lap through the list to make sure a proc hasn't 1496 * been created for this pid while we weren't under the bucket lock. 1497 */ 1498 for (fprc = bucket->ftb_data; fprc != NULL; fprc = fprc->ftpc_next) { 1499 if (fprc->ftpc_pid == pid && fprc->ftpc_acount != 0) { 1500 mutex_enter(&fprc->ftpc_mtx); 1501 mutex_exit(&bucket->ftb_mtx); 1502 fprc->ftpc_rcount++; 1503 atomic_inc_64(&fprc->ftpc_acount); 1504 ASSERT(fprc->ftpc_acount <= fprc->ftpc_rcount); 1505 mutex_exit(&fprc->ftpc_mtx); 1506 1507 kmem_free(new_fprc, sizeof (fasttrap_proc_t)); 1508 1509 return (fprc); 1510 } 1511 } 1512 1513 new_fprc->ftpc_next = bucket->ftb_data; 1514 bucket->ftb_data = new_fprc; 1515 1516 mutex_exit(&bucket->ftb_mtx); 1517 1518 return (new_fprc); 1519} 1520 1521static void 1522fasttrap_proc_release(fasttrap_proc_t *proc) 1523{ 1524 fasttrap_bucket_t *bucket; 1525 fasttrap_proc_t *fprc, **fprcp; 1526 pid_t pid = proc->ftpc_pid; 1527#ifndef illumos 1528 fasttrap_scrblock_t *scrblk, *scrblktmp; 1529 fasttrap_scrspace_t *scrspc, *scrspctmp; 1530 struct proc *p; 1531 struct thread *td; 1532#endif 1533 1534 mutex_enter(&proc->ftpc_mtx); 1535 1536 ASSERT(proc->ftpc_rcount != 0); 1537 ASSERT(proc->ftpc_acount <= proc->ftpc_rcount); 1538 1539 if (--proc->ftpc_rcount != 0) { 1540 mutex_exit(&proc->ftpc_mtx); 1541 return; 1542 } 1543 1544#ifndef illumos 1545 /* 1546 * Free all structures used to manage per-thread scratch space. 1547 */ 1548 LIST_FOREACH_SAFE(scrblk, &proc->ftpc_scrblks, ftsb_next, 1549 scrblktmp) { 1550 LIST_REMOVE(scrblk, ftsb_next); 1551 free(scrblk, M_SOLARIS); 1552 } 1553 LIST_FOREACH_SAFE(scrspc, &proc->ftpc_fscr, ftss_next, scrspctmp) { 1554 LIST_REMOVE(scrspc, ftss_next); 1555 free(scrspc, M_SOLARIS); 1556 } 1557 LIST_FOREACH_SAFE(scrspc, &proc->ftpc_ascr, ftss_next, scrspctmp) { 1558 LIST_REMOVE(scrspc, ftss_next); 1559 free(scrspc, M_SOLARIS); 1560 } 1561 1562 if ((p = pfind(pid)) != NULL) { 1563 FOREACH_THREAD_IN_PROC(p, td) 1564 td->t_dtrace_sscr = NULL; 1565 PROC_UNLOCK(p); 1566 } 1567#endif 1568 1569 mutex_exit(&proc->ftpc_mtx); 1570 1571 /* 1572 * There should definitely be no live providers associated with this 1573 * process at this point. 1574 */ 1575 ASSERT(proc->ftpc_acount == 0); 1576 1577 bucket = &fasttrap_procs.fth_table[FASTTRAP_PROCS_INDEX(pid)]; 1578 mutex_enter(&bucket->ftb_mtx); 1579 1580 fprcp = (fasttrap_proc_t **)&bucket->ftb_data; 1581 while ((fprc = *fprcp) != NULL) { 1582 if (fprc == proc) 1583 break; 1584 1585 fprcp = &fprc->ftpc_next; 1586 } 1587 1588 /* 1589 * Something strange has happened if we can't find the proc. 1590 */ 1591 ASSERT(fprc != NULL); 1592 1593 *fprcp = fprc->ftpc_next; 1594 1595 mutex_exit(&bucket->ftb_mtx); 1596 1597 kmem_free(fprc, sizeof (fasttrap_proc_t)); 1598} 1599 1600/* 1601 * Lookup a fasttrap-managed provider based on its name and associated pid. 1602 * If the pattr argument is non-NULL, this function instantiates the provider 1603 * if it doesn't exist otherwise it returns NULL. The provider is returned 1604 * with its lock held. 1605 */ 1606static fasttrap_provider_t * 1607fasttrap_provider_lookup(pid_t pid, const char *name, 1608 const dtrace_pattr_t *pattr) 1609{ 1610 fasttrap_provider_t *fp, *new_fp = NULL; 1611 fasttrap_bucket_t *bucket; 1612 char provname[DTRACE_PROVNAMELEN]; 1613 proc_t *p; 1614 cred_t *cred; 1615 1616 ASSERT(strlen(name) < sizeof (fp->ftp_name)); 1617 ASSERT(pattr != NULL); 1618 1619 bucket = &fasttrap_provs.fth_table[FASTTRAP_PROVS_INDEX(pid, name)]; 1620 mutex_enter(&bucket->ftb_mtx); 1621 1622 /* 1623 * Take a lap through the list and return the match if we find it. 1624 */ 1625 for (fp = bucket->ftb_data; fp != NULL; fp = fp->ftp_next) { 1626 if (fp->ftp_pid == pid && strcmp(fp->ftp_name, name) == 0 && 1627 !fp->ftp_retired) { 1628 mutex_enter(&fp->ftp_mtx); 1629 mutex_exit(&bucket->ftb_mtx); 1630 return (fp); 1631 } 1632 } 1633 1634 /* 1635 * Drop the bucket lock so we don't try to perform a sleeping 1636 * allocation under it. 1637 */ 1638 mutex_exit(&bucket->ftb_mtx); 1639 1640 /* 1641 * Make sure the process exists, isn't a child created as the result 1642 * of a vfork(2), and isn't a zombie (but may be in fork). 1643 */ 1644 if ((p = pfind(pid)) == NULL) 1645 return (NULL); 1646 1647 /* 1648 * Increment p_dtrace_probes so that the process knows to inform us 1649 * when it exits or execs. fasttrap_provider_free() decrements this 1650 * when we're done with this provider. 1651 */ 1652 p->p_dtrace_probes++; 1653 1654 /* 1655 * Grab the credentials for this process so we have 1656 * something to pass to dtrace_register(). 1657 */ 1658 PROC_LOCK_ASSERT(p, MA_OWNED); 1659 crhold(p->p_ucred); 1660 cred = p->p_ucred; 1661 PROC_UNLOCK(p); 1662 1663 new_fp = kmem_zalloc(sizeof (fasttrap_provider_t), KM_SLEEP); 1664 new_fp->ftp_pid = pid; 1665 new_fp->ftp_proc = fasttrap_proc_lookup(pid); 1666#ifndef illumos 1667 mutex_init(&new_fp->ftp_mtx, "provider mtx", MUTEX_DEFAULT, NULL); 1668 mutex_init(&new_fp->ftp_cmtx, "lock on creating", MUTEX_DEFAULT, NULL); 1669#endif 1670 1671 ASSERT(new_fp->ftp_proc != NULL); 1672 1673 mutex_enter(&bucket->ftb_mtx); 1674 1675 /* 1676 * Take another lap through the list to make sure a provider hasn't 1677 * been created for this pid while we weren't under the bucket lock. 1678 */ 1679 for (fp = bucket->ftb_data; fp != NULL; fp = fp->ftp_next) { 1680 if (fp->ftp_pid == pid && strcmp(fp->ftp_name, name) == 0 && 1681 !fp->ftp_retired) { 1682 mutex_enter(&fp->ftp_mtx); 1683 mutex_exit(&bucket->ftb_mtx); 1684 fasttrap_provider_free(new_fp); 1685 crfree(cred); 1686 return (fp); 1687 } 1688 } 1689 1690 (void) strcpy(new_fp->ftp_name, name); 1691 1692 /* 1693 * Fail and return NULL if either the provider name is too long 1694 * or we fail to register this new provider with the DTrace 1695 * framework. Note that this is the only place we ever construct 1696 * the full provider name -- we keep it in pieces in the provider 1697 * structure. 1698 */ 1699 if (snprintf(provname, sizeof (provname), "%s%u", name, (uint_t)pid) >= 1700 sizeof (provname) || 1701 dtrace_register(provname, pattr, 1702 DTRACE_PRIV_PROC | DTRACE_PRIV_OWNER | DTRACE_PRIV_ZONEOWNER, cred, 1703 pattr == &pid_attr ? &pid_pops : &usdt_pops, new_fp, 1704 &new_fp->ftp_provid) != 0) { 1705 mutex_exit(&bucket->ftb_mtx); 1706 fasttrap_provider_free(new_fp); 1707 crfree(cred); 1708 return (NULL); 1709 } 1710 1711 new_fp->ftp_next = bucket->ftb_data; 1712 bucket->ftb_data = new_fp; 1713 1714 mutex_enter(&new_fp->ftp_mtx); 1715 mutex_exit(&bucket->ftb_mtx); 1716 1717 crfree(cred); 1718 return (new_fp); 1719} 1720 1721static void 1722fasttrap_provider_free(fasttrap_provider_t *provider) 1723{ 1724 pid_t pid = provider->ftp_pid; 1725 proc_t *p; 1726 1727 /* 1728 * There need to be no associated enabled probes, no consumers 1729 * creating probes, and no meta providers referencing this provider. 1730 */ 1731 ASSERT(provider->ftp_rcount == 0); 1732 ASSERT(provider->ftp_ccount == 0); 1733 ASSERT(provider->ftp_mcount == 0); 1734 1735 /* 1736 * If this provider hasn't been retired, we need to explicitly drop the 1737 * count of active providers on the associated process structure. 1738 */ 1739 if (!provider->ftp_retired) { 1740 atomic_dec_64(&provider->ftp_proc->ftpc_acount); 1741 ASSERT(provider->ftp_proc->ftpc_acount < 1742 provider->ftp_proc->ftpc_rcount); 1743 } 1744 1745 fasttrap_proc_release(provider->ftp_proc); 1746 1747#ifndef illumos 1748 mutex_destroy(&provider->ftp_mtx); 1749 mutex_destroy(&provider->ftp_cmtx); 1750#endif 1751 kmem_free(provider, sizeof (fasttrap_provider_t)); 1752 1753 /* 1754 * Decrement p_dtrace_probes on the process whose provider we're 1755 * freeing. We don't have to worry about clobbering somone else's 1756 * modifications to it because we have locked the bucket that 1757 * corresponds to this process's hash chain in the provider hash 1758 * table. Don't sweat it if we can't find the process. 1759 */ 1760 if ((p = pfind(pid)) == NULL) { 1761 return; 1762 } 1763 1764 p->p_dtrace_probes--; 1765#ifndef illumos 1766 PROC_UNLOCK(p); 1767#endif 1768} 1769 1770static void 1771fasttrap_provider_retire(pid_t pid, const char *name, int mprov) 1772{ 1773 fasttrap_provider_t *fp; 1774 fasttrap_bucket_t *bucket; 1775 dtrace_provider_id_t provid; 1776 1777 ASSERT(strlen(name) < sizeof (fp->ftp_name)); 1778 1779 bucket = &fasttrap_provs.fth_table[FASTTRAP_PROVS_INDEX(pid, name)]; 1780 mutex_enter(&bucket->ftb_mtx); 1781 1782 for (fp = bucket->ftb_data; fp != NULL; fp = fp->ftp_next) { 1783 if (fp->ftp_pid == pid && strcmp(fp->ftp_name, name) == 0 && 1784 !fp->ftp_retired) 1785 break; 1786 } 1787 1788 if (fp == NULL) { 1789 mutex_exit(&bucket->ftb_mtx); 1790 return; 1791 } 1792 1793 mutex_enter(&fp->ftp_mtx); 1794 ASSERT(!mprov || fp->ftp_mcount > 0); 1795 if (mprov && --fp->ftp_mcount != 0) { 1796 mutex_exit(&fp->ftp_mtx); 1797 mutex_exit(&bucket->ftb_mtx); 1798 return; 1799 } 1800 1801 /* 1802 * Mark the provider to be removed in our post-processing step, mark it 1803 * retired, and drop the active count on its proc. Marking it indicates 1804 * that we should try to remove it; setting the retired flag indicates 1805 * that we're done with this provider; dropping the active the proc 1806 * releases our hold, and when this reaches zero (as it will during 1807 * exit or exec) the proc and associated providers become defunct. 1808 * 1809 * We obviously need to take the bucket lock before the provider lock 1810 * to perform the lookup, but we need to drop the provider lock 1811 * before calling into the DTrace framework since we acquire the 1812 * provider lock in callbacks invoked from the DTrace framework. The 1813 * bucket lock therefore protects the integrity of the provider hash 1814 * table. 1815 */ 1816 atomic_dec_64(&fp->ftp_proc->ftpc_acount); 1817 ASSERT(fp->ftp_proc->ftpc_acount < fp->ftp_proc->ftpc_rcount); 1818 1819 fp->ftp_retired = 1; 1820 fp->ftp_marked = 1; 1821 provid = fp->ftp_provid; 1822 mutex_exit(&fp->ftp_mtx); 1823 1824 /* 1825 * We don't have to worry about invalidating the same provider twice 1826 * since fasttrap_provider_lookup() will ignore provider that have 1827 * been marked as retired. 1828 */ 1829 dtrace_invalidate(provid); 1830 1831 mutex_exit(&bucket->ftb_mtx); 1832 1833 fasttrap_pid_cleanup(); 1834} 1835 1836static int 1837fasttrap_uint32_cmp(const void *ap, const void *bp) 1838{ 1839 return (*(const uint32_t *)ap - *(const uint32_t *)bp); 1840} 1841 1842static int 1843fasttrap_uint64_cmp(const void *ap, const void *bp) 1844{ 1845 return (*(const uint64_t *)ap - *(const uint64_t *)bp); 1846} 1847 1848static int 1849fasttrap_add_probe(fasttrap_probe_spec_t *pdata) 1850{ 1851 fasttrap_provider_t *provider; 1852 fasttrap_probe_t *pp; 1853 fasttrap_tracepoint_t *tp; 1854 char *name; 1855 int i, aframes = 0, whack; 1856 1857 /* 1858 * There needs to be at least one desired trace point. 1859 */ 1860 if (pdata->ftps_noffs == 0) 1861 return (EINVAL); 1862 1863 switch (pdata->ftps_type) { 1864 case DTFTP_ENTRY: 1865 name = "entry"; 1866 aframes = FASTTRAP_ENTRY_AFRAMES; 1867 break; 1868 case DTFTP_RETURN: 1869 name = "return"; 1870 aframes = FASTTRAP_RETURN_AFRAMES; 1871 break; 1872 case DTFTP_OFFSETS: 1873 name = NULL; 1874 break; 1875 default: 1876 return (EINVAL); 1877 } 1878 1879 if ((provider = fasttrap_provider_lookup(pdata->ftps_pid, 1880 FASTTRAP_PID_NAME, &pid_attr)) == NULL) 1881 return (ESRCH); 1882 1883 /* 1884 * Increment this reference count to indicate that a consumer is 1885 * actively adding a new probe associated with this provider. This 1886 * prevents the provider from being deleted -- we'll need to check 1887 * for pending deletions when we drop this reference count. 1888 */ 1889 provider->ftp_ccount++; 1890 mutex_exit(&provider->ftp_mtx); 1891 1892 /* 1893 * Grab the creation lock to ensure consistency between calls to 1894 * dtrace_probe_lookup() and dtrace_probe_create() in the face of 1895 * other threads creating probes. We must drop the provider lock 1896 * before taking this lock to avoid a three-way deadlock with the 1897 * DTrace framework. 1898 */ 1899 mutex_enter(&provider->ftp_cmtx); 1900 1901 if (name == NULL) { 1902 for (i = 0; i < pdata->ftps_noffs; i++) { 1903 char name_str[17]; 1904 1905 (void) sprintf(name_str, "%llx", 1906 (unsigned long long)pdata->ftps_offs[i]); 1907 1908 if (dtrace_probe_lookup(provider->ftp_provid, 1909 pdata->ftps_mod, pdata->ftps_func, name_str) != 0) 1910 continue; 1911 1912 atomic_inc_32(&fasttrap_total); 1913 1914 if (fasttrap_total > fasttrap_max) { 1915 atomic_dec_32(&fasttrap_total); 1916 goto no_mem; 1917 } 1918 1919 pp = kmem_zalloc(sizeof (fasttrap_probe_t), KM_SLEEP); 1920 1921 pp->ftp_prov = provider; 1922 pp->ftp_faddr = pdata->ftps_pc; 1923 pp->ftp_fsize = pdata->ftps_size; 1924 pp->ftp_pid = pdata->ftps_pid; 1925 pp->ftp_ntps = 1; 1926 1927 tp = kmem_zalloc(sizeof (fasttrap_tracepoint_t), 1928 KM_SLEEP); 1929 1930 tp->ftt_proc = provider->ftp_proc; 1931 tp->ftt_pc = pdata->ftps_offs[i] + pdata->ftps_pc; 1932 tp->ftt_pid = pdata->ftps_pid; 1933 1934 pp->ftp_tps[0].fit_tp = tp; 1935 pp->ftp_tps[0].fit_id.fti_probe = pp; 1936 pp->ftp_tps[0].fit_id.fti_ptype = pdata->ftps_type; 1937 1938 pp->ftp_id = dtrace_probe_create(provider->ftp_provid, 1939 pdata->ftps_mod, pdata->ftps_func, name_str, 1940 FASTTRAP_OFFSET_AFRAMES, pp); 1941 } 1942 1943 } else if (dtrace_probe_lookup(provider->ftp_provid, pdata->ftps_mod, 1944 pdata->ftps_func, name) == 0) { 1945 atomic_add_32(&fasttrap_total, pdata->ftps_noffs); 1946 1947 if (fasttrap_total > fasttrap_max) { 1948 atomic_add_32(&fasttrap_total, -pdata->ftps_noffs); 1949 goto no_mem; 1950 } 1951 1952 /* 1953 * Make sure all tracepoint program counter values are unique. 1954 * We later assume that each probe has exactly one tracepoint 1955 * for a given pc. 1956 */ 1957 qsort(pdata->ftps_offs, pdata->ftps_noffs, 1958 sizeof (uint64_t), fasttrap_uint64_cmp); 1959 for (i = 1; i < pdata->ftps_noffs; i++) { 1960 if (pdata->ftps_offs[i] > pdata->ftps_offs[i - 1]) 1961 continue; 1962 1963 atomic_add_32(&fasttrap_total, -pdata->ftps_noffs); 1964 goto no_mem; 1965 } 1966 1967 ASSERT(pdata->ftps_noffs > 0); 1968 pp = kmem_zalloc(offsetof(fasttrap_probe_t, 1969 ftp_tps[pdata->ftps_noffs]), KM_SLEEP); 1970 1971 pp->ftp_prov = provider; 1972 pp->ftp_faddr = pdata->ftps_pc; 1973 pp->ftp_fsize = pdata->ftps_size; 1974 pp->ftp_pid = pdata->ftps_pid; 1975 pp->ftp_ntps = pdata->ftps_noffs; 1976 1977 for (i = 0; i < pdata->ftps_noffs; i++) { 1978 tp = kmem_zalloc(sizeof (fasttrap_tracepoint_t), 1979 KM_SLEEP); 1980 1981 tp->ftt_proc = provider->ftp_proc; 1982 tp->ftt_pc = pdata->ftps_offs[i] + pdata->ftps_pc; 1983 tp->ftt_pid = pdata->ftps_pid; 1984 1985 pp->ftp_tps[i].fit_tp = tp; 1986 pp->ftp_tps[i].fit_id.fti_probe = pp; 1987 pp->ftp_tps[i].fit_id.fti_ptype = pdata->ftps_type; 1988 } 1989 1990 pp->ftp_id = dtrace_probe_create(provider->ftp_provid, 1991 pdata->ftps_mod, pdata->ftps_func, name, aframes, pp); 1992 } 1993 1994 mutex_exit(&provider->ftp_cmtx); 1995 1996 /* 1997 * We know that the provider is still valid since we incremented the 1998 * creation reference count. If someone tried to clean up this provider 1999 * while we were using it (e.g. because the process called exec(2) or 2000 * exit(2)), take note of that and try to clean it up now. 2001 */ 2002 mutex_enter(&provider->ftp_mtx); 2003 provider->ftp_ccount--; 2004 whack = provider->ftp_retired; 2005 mutex_exit(&provider->ftp_mtx); 2006 2007 if (whack) 2008 fasttrap_pid_cleanup(); 2009 2010 return (0); 2011 2012no_mem: 2013 /* 2014 * If we've exhausted the allowable resources, we'll try to remove 2015 * this provider to free some up. This is to cover the case where 2016 * the user has accidentally created many more probes than was 2017 * intended (e.g. pid123:::). 2018 */ 2019 mutex_exit(&provider->ftp_cmtx); 2020 mutex_enter(&provider->ftp_mtx); 2021 provider->ftp_ccount--; 2022 provider->ftp_marked = 1; 2023 mutex_exit(&provider->ftp_mtx); 2024 2025 fasttrap_pid_cleanup(); 2026 2027 return (ENOMEM); 2028} 2029 2030/*ARGSUSED*/ 2031static void * 2032fasttrap_meta_provide(void *arg, dtrace_helper_provdesc_t *dhpv, pid_t pid) 2033{ 2034 fasttrap_provider_t *provider; 2035 2036 /* 2037 * A 32-bit unsigned integer (like a pid for example) can be 2038 * expressed in 10 or fewer decimal digits. Make sure that we'll 2039 * have enough space for the provider name. 2040 */ 2041 if (strlen(dhpv->dthpv_provname) + 10 >= 2042 sizeof (provider->ftp_name)) { 2043 printf("failed to instantiate provider %s: " 2044 "name too long to accomodate pid", dhpv->dthpv_provname); 2045 return (NULL); 2046 } 2047 2048 /* 2049 * Don't let folks spoof the true pid provider. 2050 */ 2051 if (strcmp(dhpv->dthpv_provname, FASTTRAP_PID_NAME) == 0) { 2052 printf("failed to instantiate provider %s: " 2053 "%s is an invalid name", dhpv->dthpv_provname, 2054 FASTTRAP_PID_NAME); 2055 return (NULL); 2056 } 2057 2058 /* 2059 * The highest stability class that fasttrap supports is ISA; cap 2060 * the stability of the new provider accordingly. 2061 */ 2062 if (dhpv->dthpv_pattr.dtpa_provider.dtat_class > DTRACE_CLASS_ISA) 2063 dhpv->dthpv_pattr.dtpa_provider.dtat_class = DTRACE_CLASS_ISA; 2064 if (dhpv->dthpv_pattr.dtpa_mod.dtat_class > DTRACE_CLASS_ISA) 2065 dhpv->dthpv_pattr.dtpa_mod.dtat_class = DTRACE_CLASS_ISA; 2066 if (dhpv->dthpv_pattr.dtpa_func.dtat_class > DTRACE_CLASS_ISA) 2067 dhpv->dthpv_pattr.dtpa_func.dtat_class = DTRACE_CLASS_ISA; 2068 if (dhpv->dthpv_pattr.dtpa_name.dtat_class > DTRACE_CLASS_ISA) 2069 dhpv->dthpv_pattr.dtpa_name.dtat_class = DTRACE_CLASS_ISA; 2070 if (dhpv->dthpv_pattr.dtpa_args.dtat_class > DTRACE_CLASS_ISA) 2071 dhpv->dthpv_pattr.dtpa_args.dtat_class = DTRACE_CLASS_ISA; 2072 2073 if ((provider = fasttrap_provider_lookup(pid, dhpv->dthpv_provname, 2074 &dhpv->dthpv_pattr)) == NULL) { 2075 printf("failed to instantiate provider %s for " 2076 "process %u", dhpv->dthpv_provname, (uint_t)pid); 2077 return (NULL); 2078 } 2079 2080 /* 2081 * Up the meta provider count so this provider isn't removed until 2082 * the meta provider has been told to remove it. 2083 */ 2084 provider->ftp_mcount++; 2085 2086 mutex_exit(&provider->ftp_mtx); 2087 2088 return (provider); 2089} 2090 2091/* 2092 * We know a few things about our context here: we know that the probe being 2093 * created doesn't already exist (DTrace won't load DOF at the same address 2094 * twice, even if explicitly told to do so) and we know that we are 2095 * single-threaded with respect to the meta provider machinery. Knowing that 2096 * this is a new probe and that there is no way for us to race with another 2097 * operation on this provider allows us an important optimization: we need not 2098 * lookup a probe before adding it. Saving this lookup is important because 2099 * this code is in the fork path for processes with USDT probes, and lookups 2100 * here are potentially very expensive because of long hash conflicts on 2101 * module, function and name (DTrace doesn't hash on provider name). 2102 */ 2103/*ARGSUSED*/ 2104static void 2105fasttrap_meta_create_probe(void *arg, void *parg, 2106 dtrace_helper_probedesc_t *dhpb) 2107{ 2108 fasttrap_provider_t *provider = parg; 2109 fasttrap_probe_t *pp; 2110 fasttrap_tracepoint_t *tp; 2111 int i, j; 2112 uint32_t ntps; 2113 2114 /* 2115 * Since the meta provider count is non-zero we don't have to worry 2116 * about this provider disappearing. 2117 */ 2118 ASSERT(provider->ftp_mcount > 0); 2119 2120 /* 2121 * The offsets must be unique. 2122 */ 2123 qsort(dhpb->dthpb_offs, dhpb->dthpb_noffs, sizeof (uint32_t), 2124 fasttrap_uint32_cmp); 2125 for (i = 1; i < dhpb->dthpb_noffs; i++) { 2126 if (dhpb->dthpb_base + dhpb->dthpb_offs[i] <= 2127 dhpb->dthpb_base + dhpb->dthpb_offs[i - 1]) 2128 return; 2129 } 2130 2131 qsort(dhpb->dthpb_enoffs, dhpb->dthpb_nenoffs, sizeof (uint32_t), 2132 fasttrap_uint32_cmp); 2133 for (i = 1; i < dhpb->dthpb_nenoffs; i++) { 2134 if (dhpb->dthpb_base + dhpb->dthpb_enoffs[i] <= 2135 dhpb->dthpb_base + dhpb->dthpb_enoffs[i - 1]) 2136 return; 2137 } 2138 2139 ntps = dhpb->dthpb_noffs + dhpb->dthpb_nenoffs; 2140 ASSERT(ntps > 0); 2141 2142 atomic_add_32(&fasttrap_total, ntps); 2143 2144 if (fasttrap_total > fasttrap_max) { 2145 atomic_add_32(&fasttrap_total, -ntps); 2146 return; 2147 } 2148 2149 pp = kmem_zalloc(offsetof(fasttrap_probe_t, ftp_tps[ntps]), KM_SLEEP); 2150 2151 pp->ftp_prov = provider; 2152 pp->ftp_pid = provider->ftp_pid; 2153 pp->ftp_ntps = ntps; 2154 pp->ftp_nargs = dhpb->dthpb_xargc; 2155 pp->ftp_xtypes = dhpb->dthpb_xtypes; 2156 pp->ftp_ntypes = dhpb->dthpb_ntypes; 2157 2158 /* 2159 * First create a tracepoint for each actual point of interest. 2160 */ 2161 for (i = 0; i < dhpb->dthpb_noffs; i++) { 2162 tp = kmem_zalloc(sizeof (fasttrap_tracepoint_t), KM_SLEEP); 2163 2164 tp->ftt_proc = provider->ftp_proc; 2165 tp->ftt_pc = dhpb->dthpb_base + dhpb->dthpb_offs[i]; 2166 tp->ftt_pid = provider->ftp_pid; 2167 2168 pp->ftp_tps[i].fit_tp = tp; 2169 pp->ftp_tps[i].fit_id.fti_probe = pp; 2170#ifdef __sparc 2171 pp->ftp_tps[i].fit_id.fti_ptype = DTFTP_POST_OFFSETS; 2172#else 2173 pp->ftp_tps[i].fit_id.fti_ptype = DTFTP_OFFSETS; 2174#endif 2175 } 2176 2177 /* 2178 * Then create a tracepoint for each is-enabled point. 2179 */ 2180 for (j = 0; i < ntps; i++, j++) { 2181 tp = kmem_zalloc(sizeof (fasttrap_tracepoint_t), KM_SLEEP); 2182 2183 tp->ftt_proc = provider->ftp_proc; 2184 tp->ftt_pc = dhpb->dthpb_base + dhpb->dthpb_enoffs[j]; 2185 tp->ftt_pid = provider->ftp_pid; 2186 2187 pp->ftp_tps[i].fit_tp = tp; 2188 pp->ftp_tps[i].fit_id.fti_probe = pp; 2189 pp->ftp_tps[i].fit_id.fti_ptype = DTFTP_IS_ENABLED; 2190 } 2191 2192 /* 2193 * If the arguments are shuffled around we set the argument remapping 2194 * table. Later, when the probe fires, we only remap the arguments 2195 * if the table is non-NULL. 2196 */ 2197 for (i = 0; i < dhpb->dthpb_xargc; i++) { 2198 if (dhpb->dthpb_args[i] != i) { 2199 pp->ftp_argmap = dhpb->dthpb_args; 2200 break; 2201 } 2202 } 2203 2204 /* 2205 * The probe is fully constructed -- register it with DTrace. 2206 */ 2207 pp->ftp_id = dtrace_probe_create(provider->ftp_provid, dhpb->dthpb_mod, 2208 dhpb->dthpb_func, dhpb->dthpb_name, FASTTRAP_OFFSET_AFRAMES, pp); 2209} 2210 2211/*ARGSUSED*/ 2212static void 2213fasttrap_meta_remove(void *arg, dtrace_helper_provdesc_t *dhpv, pid_t pid) 2214{ 2215 /* 2216 * Clean up the USDT provider. There may be active consumers of the 2217 * provider busy adding probes, no damage will actually befall the 2218 * provider until that count has dropped to zero. This just puts 2219 * the provider on death row. 2220 */ 2221 fasttrap_provider_retire(pid, dhpv->dthpv_provname, 1); 2222} 2223 2224static dtrace_mops_t fasttrap_mops = { 2225 .dtms_create_probe = fasttrap_meta_create_probe, 2226 .dtms_provide_pid = fasttrap_meta_provide, 2227 .dtms_remove_pid = fasttrap_meta_remove 2228}; 2229 2230/*ARGSUSED*/ 2231static int 2232fasttrap_open(struct cdev *dev __unused, int oflags __unused, 2233 int devtype __unused, struct thread *td __unused) 2234{ 2235 return (0); 2236} 2237 2238/*ARGSUSED*/ 2239static int 2240fasttrap_ioctl(struct cdev *dev, u_long cmd, caddr_t arg, int fflag, 2241 struct thread *td) 2242{ 2243 if (!dtrace_attached()) 2244 return (EAGAIN); 2245 2246 if (cmd == FASTTRAPIOC_MAKEPROBE) { 2247 fasttrap_probe_spec_t *uprobe = *(fasttrap_probe_spec_t **)arg; 2248 fasttrap_probe_spec_t *probe; 2249 uint64_t noffs; 2250 size_t size; 2251 int ret, err; 2252 2253 if (copyin(&uprobe->ftps_noffs, &noffs, 2254 sizeof (uprobe->ftps_noffs))) 2255 return (EFAULT); 2256 2257 /* 2258 * Probes must have at least one tracepoint. 2259 */ 2260 if (noffs == 0) 2261 return (EINVAL); 2262 2263 size = sizeof (fasttrap_probe_spec_t) + 2264 sizeof (probe->ftps_offs[0]) * (noffs - 1); 2265 2266 if (size > 1024 * 1024) 2267 return (ENOMEM); 2268 2269 probe = kmem_alloc(size, KM_SLEEP); 2270 2271 if (copyin(uprobe, probe, size) != 0 || 2272 probe->ftps_noffs != noffs) { 2273 kmem_free(probe, size); 2274 return (EFAULT); 2275 } 2276 2277 /* 2278 * Verify that the function and module strings contain no 2279 * funny characters. 2280 */ 2281 if (u8_validate(probe->ftps_func, strlen(probe->ftps_func), 2282 NULL, U8_VALIDATE_ENTIRE, &err) < 0) { 2283 ret = EINVAL; 2284 goto err; 2285 } 2286 2287 if (u8_validate(probe->ftps_mod, strlen(probe->ftps_mod), 2288 NULL, U8_VALIDATE_ENTIRE, &err) < 0) { 2289 ret = EINVAL; 2290 goto err; 2291 } 2292 2293#ifdef notyet 2294 if (!PRIV_POLICY_CHOICE(cr, PRIV_ALL, B_FALSE)) { 2295 proc_t *p; 2296 pid_t pid = probe->ftps_pid; 2297 2298 mutex_enter(&pidlock); 2299 /* 2300 * Report an error if the process doesn't exist 2301 * or is actively being birthed. 2302 */ 2303 if ((p = pfind(pid)) == NULL || p->p_stat == SIDL) { 2304 mutex_exit(&pidlock); 2305 return (ESRCH); 2306 } 2307 mutex_enter(&p->p_lock); 2308 mutex_exit(&pidlock); 2309 2310 if ((ret = priv_proc_cred_perm(cr, p, NULL, 2311 VREAD | VWRITE)) != 0) { 2312 mutex_exit(&p->p_lock); 2313 return (ret); 2314 } 2315 mutex_exit(&p->p_lock); 2316 } 2317#endif /* notyet */ 2318 2319 ret = fasttrap_add_probe(probe); 2320err: 2321 kmem_free(probe, size); 2322 2323 return (ret); 2324 2325 } else if (cmd == FASTTRAPIOC_GETINSTR) { 2326 fasttrap_instr_query_t instr; 2327 fasttrap_tracepoint_t *tp; 2328 uint_t index; 2329#ifdef notyet 2330 int ret; 2331#endif 2332 2333 if (copyin((void *)arg, &instr, sizeof (instr)) != 0) 2334 return (EFAULT); 2335 2336#ifdef notyet 2337 if (!PRIV_POLICY_CHOICE(cr, PRIV_ALL, B_FALSE)) { 2338 proc_t *p; 2339 pid_t pid = instr.ftiq_pid; 2340 2341 mutex_enter(&pidlock); 2342 /* 2343 * Report an error if the process doesn't exist 2344 * or is actively being birthed. 2345 */ 2346 if ((p == pfind(pid)) == NULL || p->p_stat == SIDL) { 2347 mutex_exit(&pidlock); 2348 return (ESRCH); 2349 } 2350 mutex_enter(&p->p_lock); 2351 mutex_exit(&pidlock); 2352 2353 if ((ret = priv_proc_cred_perm(cr, p, NULL, 2354 VREAD)) != 0) { 2355 mutex_exit(&p->p_lock); 2356 return (ret); 2357 } 2358 2359 mutex_exit(&p->p_lock); 2360 } 2361#endif /* notyet */ 2362 2363 index = FASTTRAP_TPOINTS_INDEX(instr.ftiq_pid, instr.ftiq_pc); 2364 2365 mutex_enter(&fasttrap_tpoints.fth_table[index].ftb_mtx); 2366 tp = fasttrap_tpoints.fth_table[index].ftb_data; 2367 while (tp != NULL) { 2368 if (instr.ftiq_pid == tp->ftt_pid && 2369 instr.ftiq_pc == tp->ftt_pc && 2370 tp->ftt_proc->ftpc_acount != 0) 2371 break; 2372 2373 tp = tp->ftt_next; 2374 } 2375 2376 if (tp == NULL) { 2377 mutex_exit(&fasttrap_tpoints.fth_table[index].ftb_mtx); 2378 return (ENOENT); 2379 } 2380 2381 bcopy(&tp->ftt_instr, &instr.ftiq_instr, 2382 sizeof (instr.ftiq_instr)); 2383 mutex_exit(&fasttrap_tpoints.fth_table[index].ftb_mtx); 2384 2385 if (copyout(&instr, (void *)arg, sizeof (instr)) != 0) 2386 return (EFAULT); 2387 2388 return (0); 2389 } 2390 2391 return (EINVAL); 2392} 2393 2394static int 2395fasttrap_load(void) 2396{ 2397 ulong_t nent; 2398 int i, ret; 2399 2400 /* Create the /dev/dtrace/fasttrap entry. */ 2401 fasttrap_cdev = make_dev(&fasttrap_cdevsw, 0, UID_ROOT, GID_WHEEL, 0600, 2402 "dtrace/fasttrap"); 2403 2404 mtx_init(&fasttrap_cleanup_mtx, "fasttrap clean", "dtrace", MTX_DEF); 2405 mutex_init(&fasttrap_count_mtx, "fasttrap count mtx", MUTEX_DEFAULT, 2406 NULL); 2407 2408#ifdef illumos 2409 fasttrap_max = ddi_getprop(DDI_DEV_T_ANY, devi, DDI_PROP_DONTPASS, 2410 "fasttrap-max-probes", FASTTRAP_MAX_DEFAULT); 2411#endif 2412 fasttrap_total = 0; 2413 2414 /* 2415 * Conjure up the tracepoints hashtable... 2416 */ 2417#ifdef illumos 2418 nent = ddi_getprop(DDI_DEV_T_ANY, devi, DDI_PROP_DONTPASS, 2419 "fasttrap-hash-size", FASTTRAP_TPOINTS_DEFAULT_SIZE); 2420#else 2421 nent = tpoints_hash_size; 2422#endif 2423 2424 if (nent == 0 || nent > 0x1000000) 2425 nent = FASTTRAP_TPOINTS_DEFAULT_SIZE; 2426 2427 tpoints_hash_size = nent; 2428 2429 if (ISP2(nent)) 2430 fasttrap_tpoints.fth_nent = nent; 2431 else 2432 fasttrap_tpoints.fth_nent = 1 << fasttrap_highbit(nent); 2433 ASSERT(fasttrap_tpoints.fth_nent > 0); 2434 fasttrap_tpoints.fth_mask = fasttrap_tpoints.fth_nent - 1; 2435 fasttrap_tpoints.fth_table = kmem_zalloc(fasttrap_tpoints.fth_nent * 2436 sizeof (fasttrap_bucket_t), KM_SLEEP); 2437#ifndef illumos 2438 for (i = 0; i < fasttrap_tpoints.fth_nent; i++) 2439 mutex_init(&fasttrap_tpoints.fth_table[i].ftb_mtx, 2440 "tracepoints bucket mtx", MUTEX_DEFAULT, NULL); 2441#endif 2442 2443 /* 2444 * ... and the providers hash table... 2445 */ 2446 nent = FASTTRAP_PROVIDERS_DEFAULT_SIZE; 2447 if (ISP2(nent)) 2448 fasttrap_provs.fth_nent = nent; 2449 else 2450 fasttrap_provs.fth_nent = 1 << fasttrap_highbit(nent); 2451 ASSERT(fasttrap_provs.fth_nent > 0); 2452 fasttrap_provs.fth_mask = fasttrap_provs.fth_nent - 1; 2453 fasttrap_provs.fth_table = kmem_zalloc(fasttrap_provs.fth_nent * 2454 sizeof (fasttrap_bucket_t), KM_SLEEP); 2455#ifndef illumos 2456 for (i = 0; i < fasttrap_provs.fth_nent; i++) 2457 mutex_init(&fasttrap_provs.fth_table[i].ftb_mtx, 2458 "providers bucket mtx", MUTEX_DEFAULT, NULL); 2459#endif 2460 2461 ret = kproc_create(fasttrap_pid_cleanup_cb, NULL, 2462 &fasttrap_cleanup_proc, 0, 0, "ftcleanup"); 2463 if (ret != 0) { 2464 destroy_dev(fasttrap_cdev); 2465#ifndef illumos 2466 for (i = 0; i < fasttrap_provs.fth_nent; i++) 2467 mutex_destroy(&fasttrap_provs.fth_table[i].ftb_mtx); 2468 for (i = 0; i < fasttrap_tpoints.fth_nent; i++) 2469 mutex_destroy(&fasttrap_tpoints.fth_table[i].ftb_mtx); 2470#endif 2471 kmem_free(fasttrap_provs.fth_table, fasttrap_provs.fth_nent * 2472 sizeof (fasttrap_bucket_t)); 2473 mtx_destroy(&fasttrap_cleanup_mtx); 2474 mutex_destroy(&fasttrap_count_mtx); 2475 return (ret); 2476 } 2477 2478 2479 /* 2480 * ... and the procs hash table. 2481 */ 2482 nent = FASTTRAP_PROCS_DEFAULT_SIZE; 2483 if (ISP2(nent)) 2484 fasttrap_procs.fth_nent = nent; 2485 else 2486 fasttrap_procs.fth_nent = 1 << fasttrap_highbit(nent); 2487 ASSERT(fasttrap_procs.fth_nent > 0); 2488 fasttrap_procs.fth_mask = fasttrap_procs.fth_nent - 1; 2489 fasttrap_procs.fth_table = kmem_zalloc(fasttrap_procs.fth_nent * 2490 sizeof (fasttrap_bucket_t), KM_SLEEP); 2491#ifndef illumos 2492 for (i = 0; i < fasttrap_procs.fth_nent; i++) 2493 mutex_init(&fasttrap_procs.fth_table[i].ftb_mtx, 2494 "processes bucket mtx", MUTEX_DEFAULT, NULL); 2495 2496 rm_init(&fasttrap_tp_lock, "fasttrap tracepoint"); 2497 2498 /* 2499 * This event handler must run before kdtrace_thread_dtor() since it 2500 * accesses the thread's struct kdtrace_thread. 2501 */ 2502 fasttrap_thread_dtor_tag = EVENTHANDLER_REGISTER(thread_dtor, 2503 fasttrap_thread_dtor, NULL, EVENTHANDLER_PRI_FIRST); 2504#endif 2505 2506 /* 2507 * Install our hooks into fork(2), exec(2), and exit(2). 2508 */ 2509 dtrace_fasttrap_fork = &fasttrap_fork; 2510 dtrace_fasttrap_exit = &fasttrap_exec_exit; 2511 dtrace_fasttrap_exec = &fasttrap_exec_exit; 2512 2513 (void) dtrace_meta_register("fasttrap", &fasttrap_mops, NULL, 2514 &fasttrap_meta_id); 2515 2516 return (0); 2517} 2518 2519static int 2520fasttrap_unload(void) 2521{ 2522 int i, fail = 0; 2523 2524 /* 2525 * Unregister the meta-provider to make sure no new fasttrap- 2526 * managed providers come along while we're trying to close up 2527 * shop. If we fail to detach, we'll need to re-register as a 2528 * meta-provider. We can fail to unregister as a meta-provider 2529 * if providers we manage still exist. 2530 */ 2531 if (fasttrap_meta_id != DTRACE_METAPROVNONE && 2532 dtrace_meta_unregister(fasttrap_meta_id) != 0) 2533 return (-1); 2534 2535 /* 2536 * Iterate over all of our providers. If there's still a process 2537 * that corresponds to that pid, fail to detach. 2538 */ 2539 for (i = 0; i < fasttrap_provs.fth_nent; i++) { 2540 fasttrap_provider_t **fpp, *fp; 2541 fasttrap_bucket_t *bucket = &fasttrap_provs.fth_table[i]; 2542 2543 mutex_enter(&bucket->ftb_mtx); 2544 fpp = (fasttrap_provider_t **)&bucket->ftb_data; 2545 while ((fp = *fpp) != NULL) { 2546 /* 2547 * Acquire and release the lock as a simple way of 2548 * waiting for any other consumer to finish with 2549 * this provider. A thread must first acquire the 2550 * bucket lock so there's no chance of another thread 2551 * blocking on the provider's lock. 2552 */ 2553 mutex_enter(&fp->ftp_mtx); 2554 mutex_exit(&fp->ftp_mtx); 2555 2556 if (dtrace_unregister(fp->ftp_provid) != 0) { 2557 fail = 1; 2558 fpp = &fp->ftp_next; 2559 } else { 2560 *fpp = fp->ftp_next; 2561 fasttrap_provider_free(fp); 2562 } 2563 } 2564 2565 mutex_exit(&bucket->ftb_mtx); 2566 } 2567 2568 if (fail) { 2569 (void) dtrace_meta_register("fasttrap", &fasttrap_mops, NULL, 2570 &fasttrap_meta_id); 2571 2572 return (-1); 2573 } 2574 2575 /* 2576 * Stop new processes from entering these hooks now, before the 2577 * fasttrap_cleanup thread runs. That way all processes will hopefully 2578 * be out of these hooks before we free fasttrap_provs.fth_table 2579 */ 2580 ASSERT(dtrace_fasttrap_fork == &fasttrap_fork); 2581 dtrace_fasttrap_fork = NULL; 2582 2583 ASSERT(dtrace_fasttrap_exec == &fasttrap_exec_exit); 2584 dtrace_fasttrap_exec = NULL; 2585 2586 ASSERT(dtrace_fasttrap_exit == &fasttrap_exec_exit); 2587 dtrace_fasttrap_exit = NULL; 2588 2589 mtx_lock(&fasttrap_cleanup_mtx); 2590 fasttrap_cleanup_drain = 1; 2591 /* Wait for the cleanup thread to finish up and signal us. */ 2592 wakeup(&fasttrap_cleanup_cv); 2593 mtx_sleep(&fasttrap_cleanup_drain, &fasttrap_cleanup_mtx, 0, "ftcld", 2594 0); 2595 fasttrap_cleanup_proc = NULL; 2596 mtx_destroy(&fasttrap_cleanup_mtx); 2597 2598#ifdef DEBUG 2599 mutex_enter(&fasttrap_count_mtx); 2600 ASSERT(fasttrap_pid_count == 0); 2601 mutex_exit(&fasttrap_count_mtx); 2602#endif 2603 2604#ifndef illumos 2605 EVENTHANDLER_DEREGISTER(thread_dtor, fasttrap_thread_dtor_tag); 2606 2607 for (i = 0; i < fasttrap_tpoints.fth_nent; i++) 2608 mutex_destroy(&fasttrap_tpoints.fth_table[i].ftb_mtx); 2609 for (i = 0; i < fasttrap_provs.fth_nent; i++) 2610 mutex_destroy(&fasttrap_provs.fth_table[i].ftb_mtx); 2611 for (i = 0; i < fasttrap_procs.fth_nent; i++) 2612 mutex_destroy(&fasttrap_procs.fth_table[i].ftb_mtx); 2613#endif 2614 kmem_free(fasttrap_tpoints.fth_table, 2615 fasttrap_tpoints.fth_nent * sizeof (fasttrap_bucket_t)); 2616 fasttrap_tpoints.fth_nent = 0; 2617 2618 kmem_free(fasttrap_provs.fth_table, 2619 fasttrap_provs.fth_nent * sizeof (fasttrap_bucket_t)); 2620 fasttrap_provs.fth_nent = 0; 2621 2622 kmem_free(fasttrap_procs.fth_table, 2623 fasttrap_procs.fth_nent * sizeof (fasttrap_bucket_t)); 2624 fasttrap_procs.fth_nent = 0; 2625 2626#ifndef illumos 2627 destroy_dev(fasttrap_cdev); 2628 mutex_destroy(&fasttrap_count_mtx); 2629 rm_destroy(&fasttrap_tp_lock); 2630#endif 2631 2632 return (0); 2633} 2634 2635/* ARGSUSED */ 2636static int 2637fasttrap_modevent(module_t mod __unused, int type, void *data __unused) 2638{ 2639 int error = 0; 2640 2641 switch (type) { 2642 case MOD_LOAD: 2643 break; 2644 2645 case MOD_UNLOAD: 2646 break; 2647 2648 case MOD_SHUTDOWN: 2649 break; 2650 2651 default: 2652 error = EOPNOTSUPP; 2653 break; 2654 } 2655 return (error); 2656} 2657 2658SYSINIT(fasttrap_load, SI_SUB_DTRACE_PROVIDER, SI_ORDER_ANY, fasttrap_load, 2659 NULL); 2660SYSUNINIT(fasttrap_unload, SI_SUB_DTRACE_PROVIDER, SI_ORDER_ANY, 2661 fasttrap_unload, NULL); 2662 2663DEV_MODULE(fasttrap, fasttrap_modevent, NULL); 2664MODULE_VERSION(fasttrap, 1); 2665MODULE_DEPEND(fasttrap, dtrace, 1, 1, 1); 2666MODULE_DEPEND(fasttrap, opensolaris, 1, 1, 1); 2667