1/* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License (the "License"). 6 * You may not use this file except in compliance with the License. 7 * 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9 * or http://www.opensolaris.org/os/licensing. 10 * See the License for the specific language governing permissions 11 * and limitations under the License. 12 * 13 * When distributing Covered Code, include this CDDL HEADER in each 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15 * If applicable, add the following below this CDDL HEADER, with the 16 * fields enclosed by brackets "[]" replaced with your own identifying 17 * information: Portions Copyright [yyyy] [name of copyright owner] 18 * 19 * CDDL HEADER END 20 * 21 * Portions Copyright 2010 The FreeBSD Foundation 22 * 23 * $FreeBSD: stable/11/sys/cddl/contrib/opensolaris/uts/common/dtrace/fasttrap.c 326701 2017-12-08 18:04:43Z markj $ 24 */ 25 26/* 27 * Copyright 2008 Sun Microsystems, Inc. All rights reserved. 28 * Use is subject to license terms. 29 */ 30 31/* 32 * Copyright (c) 2015, Joyent, Inc. All rights reserved. 33 */ 34 35#include <sys/atomic.h> 36#include <sys/errno.h> 37#include <sys/stat.h> 38#include <sys/modctl.h> 39#include <sys/conf.h> 40#include <sys/systm.h> 41#ifdef illumos 42#include <sys/ddi.h> 43#endif 44#include <sys/sunddi.h> 45#include <sys/cpuvar.h> 46#include <sys/kmem.h> 47#ifdef illumos 48#include <sys/strsubr.h> 49#endif 50#include <sys/fasttrap.h> 51#include <sys/fasttrap_impl.h> 52#include <sys/fasttrap_isa.h> 53#include <sys/dtrace.h> 54#include <sys/dtrace_impl.h> 55#include <sys/sysmacros.h> 56#include <sys/proc.h> 57#include <sys/policy.h> 58#ifdef illumos 59#include <util/qsort.h> 60#endif 61#include <sys/mutex.h> 62#include <sys/kernel.h> 63#ifndef illumos 64#include <sys/dtrace_bsd.h> 65#include <sys/eventhandler.h> 66#include <sys/rmlock.h> 67#include <sys/sysctl.h> 68#include <sys/u8_textprep.h> 69#include <sys/user.h> 70 71#include <vm/vm.h> 72#include <vm/pmap.h> 73#include <vm/vm_map.h> 74#include <vm/vm_param.h> 75 76#include <cddl/dev/dtrace/dtrace_cddl.h> 77#endif 78 79/* 80 * User-Land Trap-Based Tracing 81 * ---------------------------- 82 * 83 * The fasttrap provider allows DTrace consumers to instrument any user-level 84 * instruction to gather data; this includes probes with semantic 85 * signifigance like entry and return as well as simple offsets into the 86 * function. While the specific techniques used are very ISA specific, the 87 * methodology is generalizable to any architecture. 88 * 89 * 90 * The General Methodology 91 * ----------------------- 92 * 93 * With the primary goal of tracing every user-land instruction and the 94 * limitation that we can't trust user space so don't want to rely on much 95 * information there, we begin by replacing the instructions we want to trace 96 * with trap instructions. Each instruction we overwrite is saved into a hash 97 * table keyed by process ID and pc address. When we enter the kernel due to 98 * this trap instruction, we need the effects of the replaced instruction to 99 * appear to have occurred before we proceed with the user thread's 100 * execution. 101 * 102 * Each user level thread is represented by a ulwp_t structure which is 103 * always easily accessible through a register. The most basic way to produce 104 * the effects of the instruction we replaced is to copy that instruction out 105 * to a bit of scratch space reserved in the user thread's ulwp_t structure 106 * (a sort of kernel-private thread local storage), set the PC to that 107 * scratch space and single step. When we reenter the kernel after single 108 * stepping the instruction we must then adjust the PC to point to what would 109 * normally be the next instruction. Of course, special care must be taken 110 * for branches and jumps, but these represent such a small fraction of any 111 * instruction set that writing the code to emulate these in the kernel is 112 * not too difficult. 113 * 114 * Return probes may require several tracepoints to trace every return site, 115 * and, conversely, each tracepoint may activate several probes (the entry 116 * and offset 0 probes, for example). To solve this muliplexing problem, 117 * tracepoints contain lists of probes to activate and probes contain lists 118 * of tracepoints to enable. If a probe is activated, it adds its ID to 119 * existing tracepoints or creates new ones as necessary. 120 * 121 * Most probes are activated _before_ the instruction is executed, but return 122 * probes are activated _after_ the effects of the last instruction of the 123 * function are visible. Return probes must be fired _after_ we have 124 * single-stepped the instruction whereas all other probes are fired 125 * beforehand. 126 * 127 * 128 * Lock Ordering 129 * ------------- 130 * 131 * The lock ordering below -- both internally and with respect to the DTrace 132 * framework -- is a little tricky and bears some explanation. Each provider 133 * has a lock (ftp_mtx) that protects its members including reference counts 134 * for enabled probes (ftp_rcount), consumers actively creating probes 135 * (ftp_ccount) and USDT consumers (ftp_mcount); all three prevent a provider 136 * from being freed. A provider is looked up by taking the bucket lock for the 137 * provider hash table, and is returned with its lock held. The provider lock 138 * may be taken in functions invoked by the DTrace framework, but may not be 139 * held while calling functions in the DTrace framework. 140 * 141 * To ensure consistency over multiple calls to the DTrace framework, the 142 * creation lock (ftp_cmtx) should be held. Naturally, the creation lock may 143 * not be taken when holding the provider lock as that would create a cyclic 144 * lock ordering. In situations where one would naturally take the provider 145 * lock and then the creation lock, we instead up a reference count to prevent 146 * the provider from disappearing, drop the provider lock, and acquire the 147 * creation lock. 148 * 149 * Briefly: 150 * bucket lock before provider lock 151 * DTrace before provider lock 152 * creation lock before DTrace 153 * never hold the provider lock and creation lock simultaneously 154 */ 155 156static d_open_t fasttrap_open; 157static d_ioctl_t fasttrap_ioctl; 158 159static struct cdevsw fasttrap_cdevsw = { 160 .d_version = D_VERSION, 161 .d_open = fasttrap_open, 162 .d_ioctl = fasttrap_ioctl, 163 .d_name = "fasttrap", 164}; 165static struct cdev *fasttrap_cdev; 166static dtrace_meta_provider_id_t fasttrap_meta_id; 167 168static struct proc *fasttrap_cleanup_proc; 169static struct mtx fasttrap_cleanup_mtx; 170static uint_t fasttrap_cleanup_work, fasttrap_cleanup_drain, fasttrap_cleanup_cv; 171 172/* 173 * Generation count on modifications to the global tracepoint lookup table. 174 */ 175static volatile uint64_t fasttrap_mod_gen; 176 177/* 178 * When the fasttrap provider is loaded, fasttrap_max is set to either 179 * FASTTRAP_MAX_DEFAULT, or the value for fasttrap-max-probes in the 180 * fasttrap.conf file (Illumos), or the value provied in the loader.conf (FreeBSD). 181 * Each time a probe is created, fasttrap_total is incremented by the number 182 * of tracepoints that may be associated with that probe; fasttrap_total is capped 183 * at fasttrap_max. 184 */ 185#define FASTTRAP_MAX_DEFAULT 250000 186static uint32_t fasttrap_max = FASTTRAP_MAX_DEFAULT; 187static uint32_t fasttrap_total; 188 189/* 190 * Copyright (c) 2011, Joyent, Inc. All rights reserved. 191 */ 192 193#define FASTTRAP_TPOINTS_DEFAULT_SIZE 0x4000 194#define FASTTRAP_PROVIDERS_DEFAULT_SIZE 0x100 195#define FASTTRAP_PROCS_DEFAULT_SIZE 0x100 196 197#define FASTTRAP_PID_NAME "pid" 198 199fasttrap_hash_t fasttrap_tpoints; 200static fasttrap_hash_t fasttrap_provs; 201static fasttrap_hash_t fasttrap_procs; 202 203static uint64_t fasttrap_pid_count; /* pid ref count */ 204static kmutex_t fasttrap_count_mtx; /* lock on ref count */ 205 206#define FASTTRAP_ENABLE_FAIL 1 207#define FASTTRAP_ENABLE_PARTIAL 2 208 209static int fasttrap_tracepoint_enable(proc_t *, fasttrap_probe_t *, uint_t); 210static void fasttrap_tracepoint_disable(proc_t *, fasttrap_probe_t *, uint_t); 211 212static fasttrap_provider_t *fasttrap_provider_lookup(pid_t, const char *, 213 const dtrace_pattr_t *); 214static void fasttrap_provider_retire(pid_t, const char *, int); 215static void fasttrap_provider_free(fasttrap_provider_t *); 216 217static fasttrap_proc_t *fasttrap_proc_lookup(pid_t); 218static void fasttrap_proc_release(fasttrap_proc_t *); 219 220#ifndef illumos 221static void fasttrap_thread_dtor(void *, struct thread *); 222#endif 223 224#define FASTTRAP_PROVS_INDEX(pid, name) \ 225 ((fasttrap_hash_str(name) + (pid)) & fasttrap_provs.fth_mask) 226 227#define FASTTRAP_PROCS_INDEX(pid) ((pid) & fasttrap_procs.fth_mask) 228 229#ifndef illumos 230struct rmlock fasttrap_tp_lock; 231static eventhandler_tag fasttrap_thread_dtor_tag; 232#endif 233 234static unsigned long tpoints_hash_size = FASTTRAP_TPOINTS_DEFAULT_SIZE; 235 236#ifdef __FreeBSD__ 237SYSCTL_DECL(_kern_dtrace); 238SYSCTL_NODE(_kern_dtrace, OID_AUTO, fasttrap, CTLFLAG_RD, 0, "DTrace fasttrap parameters"); 239SYSCTL_UINT(_kern_dtrace_fasttrap, OID_AUTO, max_probes, CTLFLAG_RWTUN, &fasttrap_max, 240 FASTTRAP_MAX_DEFAULT, "Maximum number of fasttrap probes"); 241SYSCTL_ULONG(_kern_dtrace_fasttrap, OID_AUTO, tpoints_hash_size, CTLFLAG_RDTUN, &tpoints_hash_size, 242 FASTTRAP_TPOINTS_DEFAULT_SIZE, "Size of the tracepoint hash table"); 243#endif 244 245static int 246fasttrap_highbit(ulong_t i) 247{ 248 int h = 1; 249 250 if (i == 0) 251 return (0); 252#ifdef _LP64 253 if (i & 0xffffffff00000000ul) { 254 h += 32; i >>= 32; 255 } 256#endif 257 if (i & 0xffff0000) { 258 h += 16; i >>= 16; 259 } 260 if (i & 0xff00) { 261 h += 8; i >>= 8; 262 } 263 if (i & 0xf0) { 264 h += 4; i >>= 4; 265 } 266 if (i & 0xc) { 267 h += 2; i >>= 2; 268 } 269 if (i & 0x2) { 270 h += 1; 271 } 272 return (h); 273} 274 275static uint_t 276fasttrap_hash_str(const char *p) 277{ 278 unsigned int g; 279 uint_t hval = 0; 280 281 while (*p) { 282 hval = (hval << 4) + *p++; 283 if ((g = (hval & 0xf0000000)) != 0) 284 hval ^= g >> 24; 285 hval &= ~g; 286 } 287 return (hval); 288} 289 290void 291fasttrap_sigtrap(proc_t *p, kthread_t *t, uintptr_t pc) 292{ 293#ifdef illumos 294 sigqueue_t *sqp = kmem_zalloc(sizeof (sigqueue_t), KM_SLEEP); 295 296 sqp->sq_info.si_signo = SIGTRAP; 297 sqp->sq_info.si_code = TRAP_DTRACE; 298 sqp->sq_info.si_addr = (caddr_t)pc; 299 300 mutex_enter(&p->p_lock); 301 sigaddqa(p, t, sqp); 302 mutex_exit(&p->p_lock); 303 304 if (t != NULL) 305 aston(t); 306#else 307 ksiginfo_t *ksi = kmem_zalloc(sizeof (ksiginfo_t), KM_SLEEP); 308 309 ksiginfo_init(ksi); 310 ksi->ksi_signo = SIGTRAP; 311 ksi->ksi_code = TRAP_DTRACE; 312 ksi->ksi_addr = (caddr_t)pc; 313 PROC_LOCK(p); 314 (void) tdsendsignal(p, t, SIGTRAP, ksi); 315 PROC_UNLOCK(p); 316#endif 317} 318 319#ifndef illumos 320/* 321 * Obtain a chunk of scratch space in the address space of the target process. 322 */ 323fasttrap_scrspace_t * 324fasttrap_scraddr(struct thread *td, fasttrap_proc_t *fprc) 325{ 326 fasttrap_scrblock_t *scrblk; 327 fasttrap_scrspace_t *scrspc; 328 struct proc *p; 329 vm_offset_t addr; 330 int error, i; 331 332 scrspc = NULL; 333 if (td->t_dtrace_sscr != NULL) { 334 /* If the thread already has scratch space, we're done. */ 335 scrspc = (fasttrap_scrspace_t *)td->t_dtrace_sscr; 336 return (scrspc); 337 } 338 339 p = td->td_proc; 340 341 mutex_enter(&fprc->ftpc_mtx); 342 if (LIST_EMPTY(&fprc->ftpc_fscr)) { 343 /* 344 * No scratch space is available, so we'll map a new scratch 345 * space block into the traced process' address space. 346 */ 347 addr = 0; 348 error = vm_map_find(&p->p_vmspace->vm_map, NULL, 0, &addr, 349 FASTTRAP_SCRBLOCK_SIZE, 0, VMFS_ANY_SPACE, VM_PROT_ALL, 350 VM_PROT_ALL, 0); 351 if (error != KERN_SUCCESS) 352 goto done; 353 354 scrblk = malloc(sizeof(*scrblk), M_SOLARIS, M_WAITOK); 355 scrblk->ftsb_addr = addr; 356 LIST_INSERT_HEAD(&fprc->ftpc_scrblks, scrblk, ftsb_next); 357 358 /* 359 * Carve the block up into chunks and put them on the free list. 360 */ 361 for (i = 0; 362 i < FASTTRAP_SCRBLOCK_SIZE / FASTTRAP_SCRSPACE_SIZE; i++) { 363 scrspc = malloc(sizeof(*scrspc), M_SOLARIS, M_WAITOK); 364 scrspc->ftss_addr = addr + 365 i * FASTTRAP_SCRSPACE_SIZE; 366 LIST_INSERT_HEAD(&fprc->ftpc_fscr, scrspc, 367 ftss_next); 368 } 369 } 370 371 /* 372 * Take the first scratch chunk off the free list, put it on the 373 * allocated list, and return its address. 374 */ 375 scrspc = LIST_FIRST(&fprc->ftpc_fscr); 376 LIST_REMOVE(scrspc, ftss_next); 377 LIST_INSERT_HEAD(&fprc->ftpc_ascr, scrspc, ftss_next); 378 379 /* 380 * This scratch space is reserved for use by td until the thread exits. 381 */ 382 td->t_dtrace_sscr = scrspc; 383 384done: 385 mutex_exit(&fprc->ftpc_mtx); 386 387 return (scrspc); 388} 389 390/* 391 * Return any allocated per-thread scratch space chunks back to the process' 392 * free list. 393 */ 394static void 395fasttrap_thread_dtor(void *arg __unused, struct thread *td) 396{ 397 fasttrap_bucket_t *bucket; 398 fasttrap_proc_t *fprc; 399 fasttrap_scrspace_t *scrspc; 400 pid_t pid; 401 402 if (td->t_dtrace_sscr == NULL) 403 return; 404 405 pid = td->td_proc->p_pid; 406 bucket = &fasttrap_procs.fth_table[FASTTRAP_PROCS_INDEX(pid)]; 407 fprc = NULL; 408 409 /* Look up the fasttrap process handle for this process. */ 410 mutex_enter(&bucket->ftb_mtx); 411 for (fprc = bucket->ftb_data; fprc != NULL; fprc = fprc->ftpc_next) { 412 if (fprc->ftpc_pid == pid) { 413 mutex_enter(&fprc->ftpc_mtx); 414 mutex_exit(&bucket->ftb_mtx); 415 break; 416 } 417 } 418 if (fprc == NULL) { 419 mutex_exit(&bucket->ftb_mtx); 420 return; 421 } 422 423 scrspc = (fasttrap_scrspace_t *)td->t_dtrace_sscr; 424 LIST_REMOVE(scrspc, ftss_next); 425 LIST_INSERT_HEAD(&fprc->ftpc_fscr, scrspc, ftss_next); 426 427 mutex_exit(&fprc->ftpc_mtx); 428} 429#endif 430 431/* 432 * This function ensures that no threads are actively using the memory 433 * associated with probes that were formerly live. 434 */ 435static void 436fasttrap_mod_barrier(uint64_t gen) 437{ 438 int i; 439 440 if (gen < fasttrap_mod_gen) 441 return; 442 443 fasttrap_mod_gen++; 444 445#ifdef illumos 446 CPU_FOREACH(i) { 447 mutex_enter(&fasttrap_cpuc_pid_lock[i]); 448 mutex_exit(&fasttrap_cpuc_pid_lock[i]); 449 } 450#else 451 rm_wlock(&fasttrap_tp_lock); 452 rm_wunlock(&fasttrap_tp_lock); 453#endif 454} 455 456/* 457 * This function performs asynchronous cleanup of fasttrap providers. The 458 * Solaris implementation of this mechanism use a timeout that's activated in 459 * fasttrap_pid_cleanup(), but this doesn't work in FreeBSD: one may sleep while 460 * holding the DTrace mutexes, but it is unsafe to sleep in a callout handler. 461 * Thus we use a dedicated process to perform the cleanup when requested. 462 */ 463/*ARGSUSED*/ 464static void 465fasttrap_pid_cleanup_cb(void *data) 466{ 467 fasttrap_provider_t **fpp, *fp; 468 fasttrap_bucket_t *bucket; 469 dtrace_provider_id_t provid; 470 int i, later = 0, rval; 471 472 mtx_lock(&fasttrap_cleanup_mtx); 473 while (!fasttrap_cleanup_drain || later > 0) { 474 fasttrap_cleanup_work = 0; 475 mtx_unlock(&fasttrap_cleanup_mtx); 476 477 later = 0; 478 479 /* 480 * Iterate over all the providers trying to remove the marked 481 * ones. If a provider is marked but not retired, we just 482 * have to take a crack at removing it -- it's no big deal if 483 * we can't. 484 */ 485 for (i = 0; i < fasttrap_provs.fth_nent; i++) { 486 bucket = &fasttrap_provs.fth_table[i]; 487 mutex_enter(&bucket->ftb_mtx); 488 fpp = (fasttrap_provider_t **)&bucket->ftb_data; 489 490 while ((fp = *fpp) != NULL) { 491 if (!fp->ftp_marked) { 492 fpp = &fp->ftp_next; 493 continue; 494 } 495 496 mutex_enter(&fp->ftp_mtx); 497 498 /* 499 * If this provider has consumers actively 500 * creating probes (ftp_ccount) or is a USDT 501 * provider (ftp_mcount), we can't unregister 502 * or even condense. 503 */ 504 if (fp->ftp_ccount != 0 || 505 fp->ftp_mcount != 0) { 506 mutex_exit(&fp->ftp_mtx); 507 fp->ftp_marked = 0; 508 continue; 509 } 510 511 if (!fp->ftp_retired || fp->ftp_rcount != 0) 512 fp->ftp_marked = 0; 513 514 mutex_exit(&fp->ftp_mtx); 515 516 /* 517 * If we successfully unregister this 518 * provider we can remove it from the hash 519 * chain and free the memory. If our attempt 520 * to unregister fails and this is a retired 521 * provider, increment our flag to try again 522 * pretty soon. If we've consumed more than 523 * half of our total permitted number of 524 * probes call dtrace_condense() to try to 525 * clean out the unenabled probes. 526 */ 527 provid = fp->ftp_provid; 528 if ((rval = dtrace_unregister(provid)) != 0) { 529 if (fasttrap_total > fasttrap_max / 2) 530 (void) dtrace_condense(provid); 531 532 if (rval == EAGAIN) 533 fp->ftp_marked = 1; 534 535 later += fp->ftp_marked; 536 fpp = &fp->ftp_next; 537 } else { 538 *fpp = fp->ftp_next; 539 fasttrap_provider_free(fp); 540 } 541 } 542 mutex_exit(&bucket->ftb_mtx); 543 } 544 mtx_lock(&fasttrap_cleanup_mtx); 545 546 /* 547 * If we were unable to retire a provider, try again after a 548 * second. This situation can occur in certain circumstances 549 * where providers cannot be unregistered even though they have 550 * no probes enabled because of an execution of dtrace -l or 551 * something similar. 552 */ 553 if (later > 0 || fasttrap_cleanup_work || 554 fasttrap_cleanup_drain) { 555 mtx_unlock(&fasttrap_cleanup_mtx); 556 pause("ftclean", hz); 557 mtx_lock(&fasttrap_cleanup_mtx); 558 } else 559 mtx_sleep(&fasttrap_cleanup_cv, &fasttrap_cleanup_mtx, 560 0, "ftcl", 0); 561 } 562 563 /* 564 * Wake up the thread in fasttrap_unload() now that we're done. 565 */ 566 wakeup(&fasttrap_cleanup_drain); 567 mtx_unlock(&fasttrap_cleanup_mtx); 568 569 kthread_exit(); 570} 571 572/* 573 * Activates the asynchronous cleanup mechanism. 574 */ 575static void 576fasttrap_pid_cleanup(void) 577{ 578 579 mtx_lock(&fasttrap_cleanup_mtx); 580 if (!fasttrap_cleanup_work) { 581 fasttrap_cleanup_work = 1; 582 wakeup(&fasttrap_cleanup_cv); 583 } 584 mtx_unlock(&fasttrap_cleanup_mtx); 585} 586 587/* 588 * This is called from cfork() via dtrace_fasttrap_fork(). The child 589 * process's address space is (roughly) a copy of the parent process's so 590 * we have to remove all the instrumentation we had previously enabled in the 591 * parent. 592 */ 593static void 594fasttrap_fork(proc_t *p, proc_t *cp) 595{ 596#ifndef illumos 597 fasttrap_scrblock_t *scrblk; 598 fasttrap_proc_t *fprc = NULL; 599#endif 600 pid_t ppid = p->p_pid; 601 int i; 602 603 ASSERT(curproc == p); 604#ifdef illumos 605 ASSERT(p->p_proc_flag & P_PR_LOCK); 606#else 607 PROC_LOCK_ASSERT(p, MA_OWNED); 608#endif 609#ifdef illumos 610 ASSERT(p->p_dtrace_count > 0); 611#else 612 /* 613 * This check is purposely here instead of in kern_fork.c because, 614 * for legal resons, we cannot include the dtrace_cddl.h header 615 * inside kern_fork.c and insert if-clause there. 616 */ 617 if (p->p_dtrace_count == 0 && p->p_dtrace_helpers == NULL) 618 return; 619#endif 620 621 ASSERT(cp->p_dtrace_count == 0); 622 623 /* 624 * This would be simpler and faster if we maintained per-process 625 * hash tables of enabled tracepoints. It could, however, potentially 626 * slow down execution of a tracepoint since we'd need to go 627 * through two levels of indirection. In the future, we should 628 * consider either maintaining per-process ancillary lists of 629 * enabled tracepoints or hanging a pointer to a per-process hash 630 * table of enabled tracepoints off the proc structure. 631 */ 632 633 /* 634 * We don't have to worry about the child process disappearing 635 * because we're in fork(). 636 */ 637#ifdef illumos 638 mtx_lock_spin(&cp->p_slock); 639 sprlock_proc(cp); 640 mtx_unlock_spin(&cp->p_slock); 641#else 642 /* 643 * fasttrap_tracepoint_remove() expects the child process to be 644 * unlocked and the VM then expects curproc to be unlocked. 645 */ 646 _PHOLD(cp); 647 PROC_UNLOCK(cp); 648 PROC_UNLOCK(p); 649 if (p->p_dtrace_count == 0) 650 goto dup_helpers; 651#endif 652 653 /* 654 * Iterate over every tracepoint looking for ones that belong to the 655 * parent process, and remove each from the child process. 656 */ 657 for (i = 0; i < fasttrap_tpoints.fth_nent; i++) { 658 fasttrap_tracepoint_t *tp; 659 fasttrap_bucket_t *bucket = &fasttrap_tpoints.fth_table[i]; 660 661 mutex_enter(&bucket->ftb_mtx); 662 for (tp = bucket->ftb_data; tp != NULL; tp = tp->ftt_next) { 663 if (tp->ftt_pid == ppid && 664 tp->ftt_proc->ftpc_acount != 0) { 665 int ret = fasttrap_tracepoint_remove(cp, tp); 666 ASSERT(ret == 0); 667 668 /* 669 * The count of active providers can only be 670 * decremented (i.e. to zero) during exec, 671 * exit, and removal of a meta provider so it 672 * should be impossible to drop the count 673 * mid-fork. 674 */ 675 ASSERT(tp->ftt_proc->ftpc_acount != 0); 676#ifndef illumos 677 fprc = tp->ftt_proc; 678#endif 679 } 680 } 681 mutex_exit(&bucket->ftb_mtx); 682 683#ifndef illumos 684 /* 685 * Unmap any scratch space inherited from the parent's address 686 * space. 687 */ 688 if (fprc != NULL) { 689 mutex_enter(&fprc->ftpc_mtx); 690 LIST_FOREACH(scrblk, &fprc->ftpc_scrblks, ftsb_next) { 691 vm_map_remove(&cp->p_vmspace->vm_map, 692 scrblk->ftsb_addr, 693 scrblk->ftsb_addr + FASTTRAP_SCRBLOCK_SIZE); 694 } 695 mutex_exit(&fprc->ftpc_mtx); 696 } 697#endif 698 } 699 700#ifdef illumos 701 mutex_enter(&cp->p_lock); 702 sprunlock(cp); 703#else 704dup_helpers: 705 if (p->p_dtrace_helpers != NULL) 706 dtrace_helpers_duplicate(p, cp); 707 PROC_LOCK(p); 708 PROC_LOCK(cp); 709 _PRELE(cp); 710#endif 711} 712 713/* 714 * This is called from proc_exit() or from exec_common() if p_dtrace_probes 715 * is set on the proc structure to indicate that there is a pid provider 716 * associated with this process. 717 */ 718static void 719fasttrap_exec_exit(proc_t *p) 720{ 721#ifndef illumos 722 struct thread *td; 723#endif 724 725#ifdef illumos 726 ASSERT(p == curproc); 727#else 728 PROC_LOCK_ASSERT(p, MA_OWNED); 729 _PHOLD(p); 730 /* 731 * Since struct threads may be recycled, we cannot rely on t_dtrace_sscr 732 * fields to be zeroed by kdtrace_thread_ctor. Thus we must zero it 733 * ourselves when a process exits. 734 */ 735 FOREACH_THREAD_IN_PROC(p, td) 736 td->t_dtrace_sscr = NULL; 737 PROC_UNLOCK(p); 738#endif 739 740 /* 741 * We clean up the pid provider for this process here; user-land 742 * static probes are handled by the meta-provider remove entry point. 743 */ 744 fasttrap_provider_retire(p->p_pid, FASTTRAP_PID_NAME, 0); 745#ifndef illumos 746 if (p->p_dtrace_helpers) 747 dtrace_helpers_destroy(p); 748 PROC_LOCK(p); 749 _PRELE(p); 750#endif 751} 752 753 754/*ARGSUSED*/ 755static void 756fasttrap_pid_provide(void *arg, dtrace_probedesc_t *desc) 757{ 758 /* 759 * There are no "default" pid probes. 760 */ 761} 762 763static int 764fasttrap_tracepoint_enable(proc_t *p, fasttrap_probe_t *probe, uint_t index) 765{ 766 fasttrap_tracepoint_t *tp, *new_tp = NULL; 767 fasttrap_bucket_t *bucket; 768 fasttrap_id_t *id; 769 pid_t pid; 770 uintptr_t pc; 771 772 ASSERT(index < probe->ftp_ntps); 773 774 pid = probe->ftp_pid; 775 pc = probe->ftp_tps[index].fit_tp->ftt_pc; 776 id = &probe->ftp_tps[index].fit_id; 777 778 ASSERT(probe->ftp_tps[index].fit_tp->ftt_pid == pid); 779 780#ifdef illumos 781 ASSERT(!(p->p_flag & SVFORK)); 782#endif 783 784 /* 785 * Before we make any modifications, make sure we've imposed a barrier 786 * on the generation in which this probe was last modified. 787 */ 788 fasttrap_mod_barrier(probe->ftp_gen); 789 790 bucket = &fasttrap_tpoints.fth_table[FASTTRAP_TPOINTS_INDEX(pid, pc)]; 791 792 /* 793 * If the tracepoint has already been enabled, just add our id to the 794 * list of interested probes. This may be our second time through 795 * this path in which case we'll have constructed the tracepoint we'd 796 * like to install. If we can't find a match, and have an allocated 797 * tracepoint ready to go, enable that one now. 798 * 799 * A tracepoint whose process is defunct is also considered defunct. 800 */ 801again: 802 mutex_enter(&bucket->ftb_mtx); 803 for (tp = bucket->ftb_data; tp != NULL; tp = tp->ftt_next) { 804 /* 805 * Note that it's safe to access the active count on the 806 * associated proc structure because we know that at least one 807 * provider (this one) will still be around throughout this 808 * operation. 809 */ 810 if (tp->ftt_pid != pid || tp->ftt_pc != pc || 811 tp->ftt_proc->ftpc_acount == 0) 812 continue; 813 814 /* 815 * Now that we've found a matching tracepoint, it would be 816 * a decent idea to confirm that the tracepoint is still 817 * enabled and the trap instruction hasn't been overwritten. 818 * Since this is a little hairy, we'll punt for now. 819 */ 820 821 /* 822 * This can't be the first interested probe. We don't have 823 * to worry about another thread being in the midst of 824 * deleting this tracepoint (which would be the only valid 825 * reason for a tracepoint to have no interested probes) 826 * since we're holding P_PR_LOCK for this process. 827 */ 828 ASSERT(tp->ftt_ids != NULL || tp->ftt_retids != NULL); 829 830 switch (id->fti_ptype) { 831 case DTFTP_ENTRY: 832 case DTFTP_OFFSETS: 833 case DTFTP_IS_ENABLED: 834 id->fti_next = tp->ftt_ids; 835 membar_producer(); 836 tp->ftt_ids = id; 837 membar_producer(); 838 break; 839 840 case DTFTP_RETURN: 841 case DTFTP_POST_OFFSETS: 842 id->fti_next = tp->ftt_retids; 843 membar_producer(); 844 tp->ftt_retids = id; 845 membar_producer(); 846 break; 847 848 default: 849 ASSERT(0); 850 } 851 852 mutex_exit(&bucket->ftb_mtx); 853 854 if (new_tp != NULL) { 855 new_tp->ftt_ids = NULL; 856 new_tp->ftt_retids = NULL; 857 } 858 859 return (0); 860 } 861 862 /* 863 * If we have a good tracepoint ready to go, install it now while 864 * we have the lock held and no one can screw with us. 865 */ 866 if (new_tp != NULL) { 867 int rc = 0; 868 869 new_tp->ftt_next = bucket->ftb_data; 870 membar_producer(); 871 bucket->ftb_data = new_tp; 872 membar_producer(); 873 mutex_exit(&bucket->ftb_mtx); 874 875 /* 876 * Activate the tracepoint in the ISA-specific manner. 877 * If this fails, we need to report the failure, but 878 * indicate that this tracepoint must still be disabled 879 * by calling fasttrap_tracepoint_disable(). 880 */ 881 if (fasttrap_tracepoint_install(p, new_tp) != 0) 882 rc = FASTTRAP_ENABLE_PARTIAL; 883 884 /* 885 * Increment the count of the number of tracepoints active in 886 * the victim process. 887 */ 888#ifdef illumos 889 ASSERT(p->p_proc_flag & P_PR_LOCK); 890#endif 891 p->p_dtrace_count++; 892 893 return (rc); 894 } 895 896 mutex_exit(&bucket->ftb_mtx); 897 898 /* 899 * Initialize the tracepoint that's been preallocated with the probe. 900 */ 901 new_tp = probe->ftp_tps[index].fit_tp; 902 903 ASSERT(new_tp->ftt_pid == pid); 904 ASSERT(new_tp->ftt_pc == pc); 905 ASSERT(new_tp->ftt_proc == probe->ftp_prov->ftp_proc); 906 ASSERT(new_tp->ftt_ids == NULL); 907 ASSERT(new_tp->ftt_retids == NULL); 908 909 switch (id->fti_ptype) { 910 case DTFTP_ENTRY: 911 case DTFTP_OFFSETS: 912 case DTFTP_IS_ENABLED: 913 id->fti_next = NULL; 914 new_tp->ftt_ids = id; 915 break; 916 917 case DTFTP_RETURN: 918 case DTFTP_POST_OFFSETS: 919 id->fti_next = NULL; 920 new_tp->ftt_retids = id; 921 break; 922 923 default: 924 ASSERT(0); 925 } 926 927 /* 928 * If the ISA-dependent initialization goes to plan, go back to the 929 * beginning and try to install this freshly made tracepoint. 930 */ 931 if (fasttrap_tracepoint_init(p, new_tp, pc, id->fti_ptype) == 0) 932 goto again; 933 934 new_tp->ftt_ids = NULL; 935 new_tp->ftt_retids = NULL; 936 937 return (FASTTRAP_ENABLE_FAIL); 938} 939 940static void 941fasttrap_tracepoint_disable(proc_t *p, fasttrap_probe_t *probe, uint_t index) 942{ 943 fasttrap_bucket_t *bucket; 944 fasttrap_provider_t *provider = probe->ftp_prov; 945 fasttrap_tracepoint_t **pp, *tp; 946 fasttrap_id_t *id, **idp = NULL; 947 pid_t pid; 948 uintptr_t pc; 949 950 ASSERT(index < probe->ftp_ntps); 951 952 pid = probe->ftp_pid; 953 pc = probe->ftp_tps[index].fit_tp->ftt_pc; 954 id = &probe->ftp_tps[index].fit_id; 955 956 ASSERT(probe->ftp_tps[index].fit_tp->ftt_pid == pid); 957 958 /* 959 * Find the tracepoint and make sure that our id is one of the 960 * ones registered with it. 961 */ 962 bucket = &fasttrap_tpoints.fth_table[FASTTRAP_TPOINTS_INDEX(pid, pc)]; 963 mutex_enter(&bucket->ftb_mtx); 964 for (tp = bucket->ftb_data; tp != NULL; tp = tp->ftt_next) { 965 if (tp->ftt_pid == pid && tp->ftt_pc == pc && 966 tp->ftt_proc == provider->ftp_proc) 967 break; 968 } 969 970 /* 971 * If we somehow lost this tracepoint, we're in a world of hurt. 972 */ 973 ASSERT(tp != NULL); 974 975 switch (id->fti_ptype) { 976 case DTFTP_ENTRY: 977 case DTFTP_OFFSETS: 978 case DTFTP_IS_ENABLED: 979 ASSERT(tp->ftt_ids != NULL); 980 idp = &tp->ftt_ids; 981 break; 982 983 case DTFTP_RETURN: 984 case DTFTP_POST_OFFSETS: 985 ASSERT(tp->ftt_retids != NULL); 986 idp = &tp->ftt_retids; 987 break; 988 989 default: 990 ASSERT(0); 991 } 992 993 while ((*idp)->fti_probe != probe) { 994 idp = &(*idp)->fti_next; 995 ASSERT(*idp != NULL); 996 } 997 998 id = *idp; 999 *idp = id->fti_next; 1000 membar_producer(); 1001 1002 ASSERT(id->fti_probe == probe); 1003 1004 /* 1005 * If there are other registered enablings of this tracepoint, we're 1006 * all done, but if this was the last probe assocated with this 1007 * this tracepoint, we need to remove and free it. 1008 */ 1009 if (tp->ftt_ids != NULL || tp->ftt_retids != NULL) { 1010 1011 /* 1012 * If the current probe's tracepoint is in use, swap it 1013 * for an unused tracepoint. 1014 */ 1015 if (tp == probe->ftp_tps[index].fit_tp) { 1016 fasttrap_probe_t *tmp_probe; 1017 fasttrap_tracepoint_t **tmp_tp; 1018 uint_t tmp_index; 1019 1020 if (tp->ftt_ids != NULL) { 1021 tmp_probe = tp->ftt_ids->fti_probe; 1022 /* LINTED - alignment */ 1023 tmp_index = FASTTRAP_ID_INDEX(tp->ftt_ids); 1024 tmp_tp = &tmp_probe->ftp_tps[tmp_index].fit_tp; 1025 } else { 1026 tmp_probe = tp->ftt_retids->fti_probe; 1027 /* LINTED - alignment */ 1028 tmp_index = FASTTRAP_ID_INDEX(tp->ftt_retids); 1029 tmp_tp = &tmp_probe->ftp_tps[tmp_index].fit_tp; 1030 } 1031 1032 ASSERT(*tmp_tp != NULL); 1033 ASSERT(*tmp_tp != probe->ftp_tps[index].fit_tp); 1034 ASSERT((*tmp_tp)->ftt_ids == NULL); 1035 ASSERT((*tmp_tp)->ftt_retids == NULL); 1036 1037 probe->ftp_tps[index].fit_tp = *tmp_tp; 1038 *tmp_tp = tp; 1039 } 1040 1041 mutex_exit(&bucket->ftb_mtx); 1042 1043 /* 1044 * Tag the modified probe with the generation in which it was 1045 * changed. 1046 */ 1047 probe->ftp_gen = fasttrap_mod_gen; 1048 return; 1049 } 1050 1051 mutex_exit(&bucket->ftb_mtx); 1052 1053 /* 1054 * We can't safely remove the tracepoint from the set of active 1055 * tracepoints until we've actually removed the fasttrap instruction 1056 * from the process's text. We can, however, operate on this 1057 * tracepoint secure in the knowledge that no other thread is going to 1058 * be looking at it since we hold P_PR_LOCK on the process if it's 1059 * live or we hold the provider lock on the process if it's dead and 1060 * gone. 1061 */ 1062 1063 /* 1064 * We only need to remove the actual instruction if we're looking 1065 * at an existing process 1066 */ 1067 if (p != NULL) { 1068 /* 1069 * If we fail to restore the instruction we need to kill 1070 * this process since it's in a completely unrecoverable 1071 * state. 1072 */ 1073 if (fasttrap_tracepoint_remove(p, tp) != 0) 1074 fasttrap_sigtrap(p, NULL, pc); 1075 1076 /* 1077 * Decrement the count of the number of tracepoints active 1078 * in the victim process. 1079 */ 1080#ifdef illumos 1081 ASSERT(p->p_proc_flag & P_PR_LOCK); 1082#endif 1083 p->p_dtrace_count--; 1084 } 1085 1086 /* 1087 * Remove the probe from the hash table of active tracepoints. 1088 */ 1089 mutex_enter(&bucket->ftb_mtx); 1090 pp = (fasttrap_tracepoint_t **)&bucket->ftb_data; 1091 ASSERT(*pp != NULL); 1092 while (*pp != tp) { 1093 pp = &(*pp)->ftt_next; 1094 ASSERT(*pp != NULL); 1095 } 1096 1097 *pp = tp->ftt_next; 1098 membar_producer(); 1099 1100 mutex_exit(&bucket->ftb_mtx); 1101 1102 /* 1103 * Tag the modified probe with the generation in which it was changed. 1104 */ 1105 probe->ftp_gen = fasttrap_mod_gen; 1106} 1107 1108static void 1109fasttrap_enable_callbacks(void) 1110{ 1111 /* 1112 * We don't have to play the rw lock game here because we're 1113 * providing something rather than taking something away -- 1114 * we can be sure that no threads have tried to follow this 1115 * function pointer yet. 1116 */ 1117 mutex_enter(&fasttrap_count_mtx); 1118 if (fasttrap_pid_count == 0) { 1119 ASSERT(dtrace_pid_probe_ptr == NULL); 1120 ASSERT(dtrace_return_probe_ptr == NULL); 1121 dtrace_pid_probe_ptr = &fasttrap_pid_probe; 1122 dtrace_return_probe_ptr = &fasttrap_return_probe; 1123 } 1124 ASSERT(dtrace_pid_probe_ptr == &fasttrap_pid_probe); 1125 ASSERT(dtrace_return_probe_ptr == &fasttrap_return_probe); 1126 fasttrap_pid_count++; 1127 mutex_exit(&fasttrap_count_mtx); 1128} 1129 1130static void 1131fasttrap_disable_callbacks(void) 1132{ 1133#ifdef illumos 1134 ASSERT(MUTEX_HELD(&cpu_lock)); 1135#endif 1136 1137 1138 mutex_enter(&fasttrap_count_mtx); 1139 ASSERT(fasttrap_pid_count > 0); 1140 fasttrap_pid_count--; 1141 if (fasttrap_pid_count == 0) { 1142#ifdef illumos 1143 cpu_t *cur, *cpu = CPU; 1144 1145 for (cur = cpu->cpu_next_onln; cur != cpu; 1146 cur = cur->cpu_next_onln) { 1147 rw_enter(&cur->cpu_ft_lock, RW_WRITER); 1148 } 1149#endif 1150 dtrace_pid_probe_ptr = NULL; 1151 dtrace_return_probe_ptr = NULL; 1152#ifdef illumos 1153 for (cur = cpu->cpu_next_onln; cur != cpu; 1154 cur = cur->cpu_next_onln) { 1155 rw_exit(&cur->cpu_ft_lock); 1156 } 1157#endif 1158 } 1159 mutex_exit(&fasttrap_count_mtx); 1160} 1161 1162/*ARGSUSED*/ 1163static void 1164fasttrap_pid_enable(void *arg, dtrace_id_t id, void *parg) 1165{ 1166 fasttrap_probe_t *probe = parg; 1167 proc_t *p = NULL; 1168 int i, rc; 1169 1170 ASSERT(probe != NULL); 1171 ASSERT(!probe->ftp_enabled); 1172 ASSERT(id == probe->ftp_id); 1173#ifdef illumos 1174 ASSERT(MUTEX_HELD(&cpu_lock)); 1175#endif 1176 1177 /* 1178 * Increment the count of enabled probes on this probe's provider; 1179 * the provider can't go away while the probe still exists. We 1180 * must increment this even if we aren't able to properly enable 1181 * this probe. 1182 */ 1183 mutex_enter(&probe->ftp_prov->ftp_mtx); 1184 probe->ftp_prov->ftp_rcount++; 1185 mutex_exit(&probe->ftp_prov->ftp_mtx); 1186 1187 /* 1188 * If this probe's provider is retired (meaning it was valid in a 1189 * previously exec'ed incarnation of this address space), bail out. The 1190 * provider can't go away while we're in this code path. 1191 */ 1192 if (probe->ftp_prov->ftp_retired) 1193 return; 1194 1195 /* 1196 * If we can't find the process, it may be that we're in the context of 1197 * a fork in which the traced process is being born and we're copying 1198 * USDT probes. Otherwise, the process is gone so bail. 1199 */ 1200#ifdef illumos 1201 if ((p = sprlock(probe->ftp_pid)) == NULL) { 1202 if ((curproc->p_flag & SFORKING) == 0) 1203 return; 1204 1205 mutex_enter(&pidlock); 1206 p = prfind(probe->ftp_pid); 1207 1208 if (p == NULL) { 1209 /* 1210 * So it's not that the target process is being born, 1211 * it's that it isn't there at all (and we simply 1212 * happen to be forking). Anyway, we know that the 1213 * target is definitely gone, so bail out. 1214 */ 1215 mutex_exit(&pidlock); 1216 return (0); 1217 } 1218 1219 /* 1220 * Confirm that curproc is indeed forking the process in which 1221 * we're trying to enable probes. 1222 */ 1223 ASSERT(p->p_parent == curproc); 1224 ASSERT(p->p_stat == SIDL); 1225 1226 mutex_enter(&p->p_lock); 1227 mutex_exit(&pidlock); 1228 1229 sprlock_proc(p); 1230 } 1231 1232 ASSERT(!(p->p_flag & SVFORK)); 1233 mutex_exit(&p->p_lock); 1234#else 1235 if ((p = pfind(probe->ftp_pid)) == NULL) 1236 return; 1237#endif 1238 1239 /* 1240 * We have to enable the trap entry point before any user threads have 1241 * the chance to execute the trap instruction we're about to place 1242 * in their process's text. 1243 */ 1244#ifdef __FreeBSD__ 1245 /* 1246 * pfind() returns a locked process. 1247 */ 1248 _PHOLD(p); 1249 PROC_UNLOCK(p); 1250#endif 1251 fasttrap_enable_callbacks(); 1252 1253 /* 1254 * Enable all the tracepoints and add this probe's id to each 1255 * tracepoint's list of active probes. 1256 */ 1257 for (i = 0; i < probe->ftp_ntps; i++) { 1258 if ((rc = fasttrap_tracepoint_enable(p, probe, i)) != 0) { 1259 /* 1260 * If enabling the tracepoint failed completely, 1261 * we don't have to disable it; if the failure 1262 * was only partial we must disable it. 1263 */ 1264 if (rc == FASTTRAP_ENABLE_FAIL) 1265 i--; 1266 else 1267 ASSERT(rc == FASTTRAP_ENABLE_PARTIAL); 1268 1269 /* 1270 * Back up and pull out all the tracepoints we've 1271 * created so far for this probe. 1272 */ 1273 while (i >= 0) { 1274 fasttrap_tracepoint_disable(p, probe, i); 1275 i--; 1276 } 1277 1278#ifdef illumos 1279 mutex_enter(&p->p_lock); 1280 sprunlock(p); 1281#else 1282 PRELE(p); 1283#endif 1284 1285 /* 1286 * Since we're not actually enabling this probe, 1287 * drop our reference on the trap table entry. 1288 */ 1289 fasttrap_disable_callbacks(); 1290 return; 1291 } 1292 } 1293#ifdef illumos 1294 mutex_enter(&p->p_lock); 1295 sprunlock(p); 1296#else 1297 PRELE(p); 1298#endif 1299 1300 probe->ftp_enabled = 1; 1301} 1302 1303/*ARGSUSED*/ 1304static void 1305fasttrap_pid_disable(void *arg, dtrace_id_t id, void *parg) 1306{ 1307 fasttrap_probe_t *probe = parg; 1308 fasttrap_provider_t *provider = probe->ftp_prov; 1309 proc_t *p; 1310 int i, whack = 0; 1311 1312 ASSERT(id == probe->ftp_id); 1313 1314 mutex_enter(&provider->ftp_mtx); 1315 1316 /* 1317 * We won't be able to acquire a /proc-esque lock on the process 1318 * iff the process is dead and gone. In this case, we rely on the 1319 * provider lock as a point of mutual exclusion to prevent other 1320 * DTrace consumers from disabling this probe. 1321 */ 1322 if ((p = pfind(probe->ftp_pid)) != NULL) { 1323#ifdef __FreeBSD__ 1324 if (p->p_flag & P_WEXIT) { 1325 PROC_UNLOCK(p); 1326 p = NULL; 1327 } else { 1328 _PHOLD(p); 1329 PROC_UNLOCK(p); 1330 } 1331#endif 1332 } 1333 1334 /* 1335 * Disable all the associated tracepoints (for fully enabled probes). 1336 */ 1337 if (probe->ftp_enabled) { 1338 for (i = 0; i < probe->ftp_ntps; i++) { 1339 fasttrap_tracepoint_disable(p, probe, i); 1340 } 1341 } 1342 1343 ASSERT(provider->ftp_rcount > 0); 1344 provider->ftp_rcount--; 1345 1346 if (p != NULL) { 1347 /* 1348 * Even though we may not be able to remove it entirely, we 1349 * mark this retired provider to get a chance to remove some 1350 * of the associated probes. 1351 */ 1352 if (provider->ftp_retired && !provider->ftp_marked) 1353 whack = provider->ftp_marked = 1; 1354 mutex_exit(&provider->ftp_mtx); 1355 } else { 1356 /* 1357 * If the process is dead, we're just waiting for the 1358 * last probe to be disabled to be able to free it. 1359 */ 1360 if (provider->ftp_rcount == 0 && !provider->ftp_marked) 1361 whack = provider->ftp_marked = 1; 1362 mutex_exit(&provider->ftp_mtx); 1363 } 1364 1365 if (whack) 1366 fasttrap_pid_cleanup(); 1367 1368#ifdef __FreeBSD__ 1369 if (p != NULL) 1370 PRELE(p); 1371#endif 1372 if (!probe->ftp_enabled) 1373 return; 1374 1375 probe->ftp_enabled = 0; 1376 1377#ifdef illumos 1378 ASSERT(MUTEX_HELD(&cpu_lock)); 1379#endif 1380 fasttrap_disable_callbacks(); 1381} 1382 1383/*ARGSUSED*/ 1384static void 1385fasttrap_pid_getargdesc(void *arg, dtrace_id_t id, void *parg, 1386 dtrace_argdesc_t *desc) 1387{ 1388 fasttrap_probe_t *probe = parg; 1389 char *str; 1390 int i, ndx; 1391 1392 desc->dtargd_native[0] = '\0'; 1393 desc->dtargd_xlate[0] = '\0'; 1394 1395 if (probe->ftp_prov->ftp_retired != 0 || 1396 desc->dtargd_ndx >= probe->ftp_nargs) { 1397 desc->dtargd_ndx = DTRACE_ARGNONE; 1398 return; 1399 } 1400 1401 ndx = (probe->ftp_argmap != NULL) ? 1402 probe->ftp_argmap[desc->dtargd_ndx] : desc->dtargd_ndx; 1403 1404 str = probe->ftp_ntypes; 1405 for (i = 0; i < ndx; i++) { 1406 str += strlen(str) + 1; 1407 } 1408 1409 ASSERT(strlen(str + 1) < sizeof (desc->dtargd_native)); 1410 (void) strcpy(desc->dtargd_native, str); 1411 1412 if (probe->ftp_xtypes == NULL) 1413 return; 1414 1415 str = probe->ftp_xtypes; 1416 for (i = 0; i < desc->dtargd_ndx; i++) { 1417 str += strlen(str) + 1; 1418 } 1419 1420 ASSERT(strlen(str + 1) < sizeof (desc->dtargd_xlate)); 1421 (void) strcpy(desc->dtargd_xlate, str); 1422} 1423 1424/*ARGSUSED*/ 1425static void 1426fasttrap_pid_destroy(void *arg, dtrace_id_t id, void *parg) 1427{ 1428 fasttrap_probe_t *probe = parg; 1429 int i; 1430 size_t size; 1431 1432 ASSERT(probe != NULL); 1433 ASSERT(!probe->ftp_enabled); 1434 ASSERT(fasttrap_total >= probe->ftp_ntps); 1435 1436 atomic_add_32(&fasttrap_total, -probe->ftp_ntps); 1437 size = offsetof(fasttrap_probe_t, ftp_tps[probe->ftp_ntps]); 1438 1439 if (probe->ftp_gen + 1 >= fasttrap_mod_gen) 1440 fasttrap_mod_barrier(probe->ftp_gen); 1441 1442 for (i = 0; i < probe->ftp_ntps; i++) { 1443 kmem_free(probe->ftp_tps[i].fit_tp, 1444 sizeof (fasttrap_tracepoint_t)); 1445 } 1446 1447 kmem_free(probe, size); 1448} 1449 1450 1451static const dtrace_pattr_t pid_attr = { 1452{ DTRACE_STABILITY_EVOLVING, DTRACE_STABILITY_EVOLVING, DTRACE_CLASS_ISA }, 1453{ DTRACE_STABILITY_PRIVATE, DTRACE_STABILITY_PRIVATE, DTRACE_CLASS_UNKNOWN }, 1454{ DTRACE_STABILITY_PRIVATE, DTRACE_STABILITY_PRIVATE, DTRACE_CLASS_UNKNOWN }, 1455{ DTRACE_STABILITY_EVOLVING, DTRACE_STABILITY_EVOLVING, DTRACE_CLASS_ISA }, 1456{ DTRACE_STABILITY_PRIVATE, DTRACE_STABILITY_PRIVATE, DTRACE_CLASS_UNKNOWN }, 1457}; 1458 1459static dtrace_pops_t pid_pops = { 1460 .dtps_provide = fasttrap_pid_provide, 1461 .dtps_provide_module = NULL, 1462 .dtps_enable = fasttrap_pid_enable, 1463 .dtps_disable = fasttrap_pid_disable, 1464 .dtps_suspend = NULL, 1465 .dtps_resume = NULL, 1466 .dtps_getargdesc = fasttrap_pid_getargdesc, 1467 .dtps_getargval = fasttrap_pid_getarg, 1468 .dtps_usermode = NULL, 1469 .dtps_destroy = fasttrap_pid_destroy 1470}; 1471 1472static dtrace_pops_t usdt_pops = { 1473 .dtps_provide = fasttrap_pid_provide, 1474 .dtps_provide_module = NULL, 1475 .dtps_enable = fasttrap_pid_enable, 1476 .dtps_disable = fasttrap_pid_disable, 1477 .dtps_suspend = NULL, 1478 .dtps_resume = NULL, 1479 .dtps_getargdesc = fasttrap_pid_getargdesc, 1480 .dtps_getargval = fasttrap_usdt_getarg, 1481 .dtps_usermode = NULL, 1482 .dtps_destroy = fasttrap_pid_destroy 1483}; 1484 1485static fasttrap_proc_t * 1486fasttrap_proc_lookup(pid_t pid) 1487{ 1488 fasttrap_bucket_t *bucket; 1489 fasttrap_proc_t *fprc, *new_fprc; 1490 1491 1492 bucket = &fasttrap_procs.fth_table[FASTTRAP_PROCS_INDEX(pid)]; 1493 mutex_enter(&bucket->ftb_mtx); 1494 1495 for (fprc = bucket->ftb_data; fprc != NULL; fprc = fprc->ftpc_next) { 1496 if (fprc->ftpc_pid == pid && fprc->ftpc_acount != 0) { 1497 mutex_enter(&fprc->ftpc_mtx); 1498 mutex_exit(&bucket->ftb_mtx); 1499 fprc->ftpc_rcount++; 1500 atomic_inc_64(&fprc->ftpc_acount); 1501 ASSERT(fprc->ftpc_acount <= fprc->ftpc_rcount); 1502 mutex_exit(&fprc->ftpc_mtx); 1503 1504 return (fprc); 1505 } 1506 } 1507 1508 /* 1509 * Drop the bucket lock so we don't try to perform a sleeping 1510 * allocation under it. 1511 */ 1512 mutex_exit(&bucket->ftb_mtx); 1513 1514 new_fprc = kmem_zalloc(sizeof (fasttrap_proc_t), KM_SLEEP); 1515 new_fprc->ftpc_pid = pid; 1516 new_fprc->ftpc_rcount = 1; 1517 new_fprc->ftpc_acount = 1; 1518#ifndef illumos 1519 mutex_init(&new_fprc->ftpc_mtx, "fasttrap proc mtx", MUTEX_DEFAULT, 1520 NULL); 1521#endif 1522 1523 mutex_enter(&bucket->ftb_mtx); 1524 1525 /* 1526 * Take another lap through the list to make sure a proc hasn't 1527 * been created for this pid while we weren't under the bucket lock. 1528 */ 1529 for (fprc = bucket->ftb_data; fprc != NULL; fprc = fprc->ftpc_next) { 1530 if (fprc->ftpc_pid == pid && fprc->ftpc_acount != 0) { 1531 mutex_enter(&fprc->ftpc_mtx); 1532 mutex_exit(&bucket->ftb_mtx); 1533 fprc->ftpc_rcount++; 1534 atomic_inc_64(&fprc->ftpc_acount); 1535 ASSERT(fprc->ftpc_acount <= fprc->ftpc_rcount); 1536 mutex_exit(&fprc->ftpc_mtx); 1537 1538 kmem_free(new_fprc, sizeof (fasttrap_proc_t)); 1539 1540 return (fprc); 1541 } 1542 } 1543 1544 new_fprc->ftpc_next = bucket->ftb_data; 1545 bucket->ftb_data = new_fprc; 1546 1547 mutex_exit(&bucket->ftb_mtx); 1548 1549 return (new_fprc); 1550} 1551 1552static void 1553fasttrap_proc_release(fasttrap_proc_t *proc) 1554{ 1555 fasttrap_bucket_t *bucket; 1556 fasttrap_proc_t *fprc, **fprcp; 1557 pid_t pid = proc->ftpc_pid; 1558#ifndef illumos 1559 fasttrap_scrblock_t *scrblk, *scrblktmp; 1560 fasttrap_scrspace_t *scrspc, *scrspctmp; 1561 struct proc *p; 1562 struct thread *td; 1563#endif 1564 1565 mutex_enter(&proc->ftpc_mtx); 1566 1567 ASSERT(proc->ftpc_rcount != 0); 1568 ASSERT(proc->ftpc_acount <= proc->ftpc_rcount); 1569 1570 if (--proc->ftpc_rcount != 0) { 1571 mutex_exit(&proc->ftpc_mtx); 1572 return; 1573 } 1574 1575#ifndef illumos 1576 /* 1577 * Free all structures used to manage per-thread scratch space. 1578 */ 1579 LIST_FOREACH_SAFE(scrblk, &proc->ftpc_scrblks, ftsb_next, 1580 scrblktmp) { 1581 LIST_REMOVE(scrblk, ftsb_next); 1582 free(scrblk, M_SOLARIS); 1583 } 1584 LIST_FOREACH_SAFE(scrspc, &proc->ftpc_fscr, ftss_next, scrspctmp) { 1585 LIST_REMOVE(scrspc, ftss_next); 1586 free(scrspc, M_SOLARIS); 1587 } 1588 LIST_FOREACH_SAFE(scrspc, &proc->ftpc_ascr, ftss_next, scrspctmp) { 1589 LIST_REMOVE(scrspc, ftss_next); 1590 free(scrspc, M_SOLARIS); 1591 } 1592 1593 if ((p = pfind(pid)) != NULL) { 1594 FOREACH_THREAD_IN_PROC(p, td) 1595 td->t_dtrace_sscr = NULL; 1596 PROC_UNLOCK(p); 1597 } 1598#endif 1599 1600 mutex_exit(&proc->ftpc_mtx); 1601 1602 /* 1603 * There should definitely be no live providers associated with this 1604 * process at this point. 1605 */ 1606 ASSERT(proc->ftpc_acount == 0); 1607 1608 bucket = &fasttrap_procs.fth_table[FASTTRAP_PROCS_INDEX(pid)]; 1609 mutex_enter(&bucket->ftb_mtx); 1610 1611 fprcp = (fasttrap_proc_t **)&bucket->ftb_data; 1612 while ((fprc = *fprcp) != NULL) { 1613 if (fprc == proc) 1614 break; 1615 1616 fprcp = &fprc->ftpc_next; 1617 } 1618 1619 /* 1620 * Something strange has happened if we can't find the proc. 1621 */ 1622 ASSERT(fprc != NULL); 1623 1624 *fprcp = fprc->ftpc_next; 1625 1626 mutex_exit(&bucket->ftb_mtx); 1627 1628 kmem_free(fprc, sizeof (fasttrap_proc_t)); 1629} 1630 1631/* 1632 * Lookup a fasttrap-managed provider based on its name and associated pid. 1633 * If the pattr argument is non-NULL, this function instantiates the provider 1634 * if it doesn't exist otherwise it returns NULL. The provider is returned 1635 * with its lock held. 1636 */ 1637static fasttrap_provider_t * 1638fasttrap_provider_lookup(pid_t pid, const char *name, 1639 const dtrace_pattr_t *pattr) 1640{ 1641 fasttrap_provider_t *fp, *new_fp = NULL; 1642 fasttrap_bucket_t *bucket; 1643 char provname[DTRACE_PROVNAMELEN]; 1644 proc_t *p; 1645 cred_t *cred; 1646 1647 ASSERT(strlen(name) < sizeof (fp->ftp_name)); 1648 ASSERT(pattr != NULL); 1649 1650 bucket = &fasttrap_provs.fth_table[FASTTRAP_PROVS_INDEX(pid, name)]; 1651 mutex_enter(&bucket->ftb_mtx); 1652 1653 /* 1654 * Take a lap through the list and return the match if we find it. 1655 */ 1656 for (fp = bucket->ftb_data; fp != NULL; fp = fp->ftp_next) { 1657 if (fp->ftp_pid == pid && strcmp(fp->ftp_name, name) == 0 && 1658 !fp->ftp_retired) { 1659 mutex_enter(&fp->ftp_mtx); 1660 mutex_exit(&bucket->ftb_mtx); 1661 return (fp); 1662 } 1663 } 1664 1665 /* 1666 * Drop the bucket lock so we don't try to perform a sleeping 1667 * allocation under it. 1668 */ 1669 mutex_exit(&bucket->ftb_mtx); 1670 1671 /* 1672 * Make sure the process exists, isn't a child created as the result 1673 * of a vfork(2), and isn't a zombie (but may be in fork). 1674 */ 1675 if ((p = pfind(pid)) == NULL) 1676 return (NULL); 1677 1678 /* 1679 * Increment p_dtrace_probes so that the process knows to inform us 1680 * when it exits or execs. fasttrap_provider_free() decrements this 1681 * when we're done with this provider. 1682 */ 1683 p->p_dtrace_probes++; 1684 1685 /* 1686 * Grab the credentials for this process so we have 1687 * something to pass to dtrace_register(). 1688 */ 1689 PROC_LOCK_ASSERT(p, MA_OWNED); 1690 crhold(p->p_ucred); 1691 cred = p->p_ucred; 1692 PROC_UNLOCK(p); 1693 1694 new_fp = kmem_zalloc(sizeof (fasttrap_provider_t), KM_SLEEP); 1695 new_fp->ftp_pid = pid; 1696 new_fp->ftp_proc = fasttrap_proc_lookup(pid); 1697#ifndef illumos 1698 mutex_init(&new_fp->ftp_mtx, "provider mtx", MUTEX_DEFAULT, NULL); 1699 mutex_init(&new_fp->ftp_cmtx, "lock on creating", MUTEX_DEFAULT, NULL); 1700#endif 1701 1702 ASSERT(new_fp->ftp_proc != NULL); 1703 1704 mutex_enter(&bucket->ftb_mtx); 1705 1706 /* 1707 * Take another lap through the list to make sure a provider hasn't 1708 * been created for this pid while we weren't under the bucket lock. 1709 */ 1710 for (fp = bucket->ftb_data; fp != NULL; fp = fp->ftp_next) { 1711 if (fp->ftp_pid == pid && strcmp(fp->ftp_name, name) == 0 && 1712 !fp->ftp_retired) { 1713 mutex_enter(&fp->ftp_mtx); 1714 mutex_exit(&bucket->ftb_mtx); 1715 fasttrap_provider_free(new_fp); 1716 crfree(cred); 1717 return (fp); 1718 } 1719 } 1720 1721 (void) strcpy(new_fp->ftp_name, name); 1722 1723 /* 1724 * Fail and return NULL if either the provider name is too long 1725 * or we fail to register this new provider with the DTrace 1726 * framework. Note that this is the only place we ever construct 1727 * the full provider name -- we keep it in pieces in the provider 1728 * structure. 1729 */ 1730 if (snprintf(provname, sizeof (provname), "%s%u", name, (uint_t)pid) >= 1731 sizeof (provname) || 1732 dtrace_register(provname, pattr, 1733 DTRACE_PRIV_PROC | DTRACE_PRIV_OWNER | DTRACE_PRIV_ZONEOWNER, cred, 1734 pattr == &pid_attr ? &pid_pops : &usdt_pops, new_fp, 1735 &new_fp->ftp_provid) != 0) { 1736 mutex_exit(&bucket->ftb_mtx); 1737 fasttrap_provider_free(new_fp); 1738 crfree(cred); 1739 return (NULL); 1740 } 1741 1742 new_fp->ftp_next = bucket->ftb_data; 1743 bucket->ftb_data = new_fp; 1744 1745 mutex_enter(&new_fp->ftp_mtx); 1746 mutex_exit(&bucket->ftb_mtx); 1747 1748 crfree(cred); 1749 return (new_fp); 1750} 1751 1752static void 1753fasttrap_provider_free(fasttrap_provider_t *provider) 1754{ 1755 pid_t pid = provider->ftp_pid; 1756 proc_t *p; 1757 1758 /* 1759 * There need to be no associated enabled probes, no consumers 1760 * creating probes, and no meta providers referencing this provider. 1761 */ 1762 ASSERT(provider->ftp_rcount == 0); 1763 ASSERT(provider->ftp_ccount == 0); 1764 ASSERT(provider->ftp_mcount == 0); 1765 1766 /* 1767 * If this provider hasn't been retired, we need to explicitly drop the 1768 * count of active providers on the associated process structure. 1769 */ 1770 if (!provider->ftp_retired) { 1771 atomic_dec_64(&provider->ftp_proc->ftpc_acount); 1772 ASSERT(provider->ftp_proc->ftpc_acount < 1773 provider->ftp_proc->ftpc_rcount); 1774 } 1775 1776 fasttrap_proc_release(provider->ftp_proc); 1777 1778#ifndef illumos 1779 mutex_destroy(&provider->ftp_mtx); 1780 mutex_destroy(&provider->ftp_cmtx); 1781#endif 1782 kmem_free(provider, sizeof (fasttrap_provider_t)); 1783 1784 /* 1785 * Decrement p_dtrace_probes on the process whose provider we're 1786 * freeing. We don't have to worry about clobbering somone else's 1787 * modifications to it because we have locked the bucket that 1788 * corresponds to this process's hash chain in the provider hash 1789 * table. Don't sweat it if we can't find the process. 1790 */ 1791 if ((p = pfind(pid)) == NULL) { 1792 return; 1793 } 1794 1795 p->p_dtrace_probes--; 1796#ifndef illumos 1797 PROC_UNLOCK(p); 1798#endif 1799} 1800 1801static void 1802fasttrap_provider_retire(pid_t pid, const char *name, int mprov) 1803{ 1804 fasttrap_provider_t *fp; 1805 fasttrap_bucket_t *bucket; 1806 dtrace_provider_id_t provid; 1807 1808 ASSERT(strlen(name) < sizeof (fp->ftp_name)); 1809 1810 bucket = &fasttrap_provs.fth_table[FASTTRAP_PROVS_INDEX(pid, name)]; 1811 mutex_enter(&bucket->ftb_mtx); 1812 1813 for (fp = bucket->ftb_data; fp != NULL; fp = fp->ftp_next) { 1814 if (fp->ftp_pid == pid && strcmp(fp->ftp_name, name) == 0 && 1815 !fp->ftp_retired) 1816 break; 1817 } 1818 1819 if (fp == NULL) { 1820 mutex_exit(&bucket->ftb_mtx); 1821 return; 1822 } 1823 1824 mutex_enter(&fp->ftp_mtx); 1825 ASSERT(!mprov || fp->ftp_mcount > 0); 1826 if (mprov && --fp->ftp_mcount != 0) { 1827 mutex_exit(&fp->ftp_mtx); 1828 mutex_exit(&bucket->ftb_mtx); 1829 return; 1830 } 1831 1832 /* 1833 * Mark the provider to be removed in our post-processing step, mark it 1834 * retired, and drop the active count on its proc. Marking it indicates 1835 * that we should try to remove it; setting the retired flag indicates 1836 * that we're done with this provider; dropping the active the proc 1837 * releases our hold, and when this reaches zero (as it will during 1838 * exit or exec) the proc and associated providers become defunct. 1839 * 1840 * We obviously need to take the bucket lock before the provider lock 1841 * to perform the lookup, but we need to drop the provider lock 1842 * before calling into the DTrace framework since we acquire the 1843 * provider lock in callbacks invoked from the DTrace framework. The 1844 * bucket lock therefore protects the integrity of the provider hash 1845 * table. 1846 */ 1847 atomic_dec_64(&fp->ftp_proc->ftpc_acount); 1848 ASSERT(fp->ftp_proc->ftpc_acount < fp->ftp_proc->ftpc_rcount); 1849 1850 fp->ftp_retired = 1; 1851 fp->ftp_marked = 1; 1852 provid = fp->ftp_provid; 1853 mutex_exit(&fp->ftp_mtx); 1854 1855 /* 1856 * We don't have to worry about invalidating the same provider twice 1857 * since fasttrap_provider_lookup() will ignore provider that have 1858 * been marked as retired. 1859 */ 1860 dtrace_invalidate(provid); 1861 1862 mutex_exit(&bucket->ftb_mtx); 1863 1864 fasttrap_pid_cleanup(); 1865} 1866 1867static int 1868fasttrap_uint32_cmp(const void *ap, const void *bp) 1869{ 1870 return (*(const uint32_t *)ap - *(const uint32_t *)bp); 1871} 1872 1873static int 1874fasttrap_uint64_cmp(const void *ap, const void *bp) 1875{ 1876 return (*(const uint64_t *)ap - *(const uint64_t *)bp); 1877} 1878 1879static int 1880fasttrap_add_probe(fasttrap_probe_spec_t *pdata) 1881{ 1882 fasttrap_provider_t *provider; 1883 fasttrap_probe_t *pp; 1884 fasttrap_tracepoint_t *tp; 1885 char *name; 1886 int i, aframes = 0, whack; 1887 1888 /* 1889 * There needs to be at least one desired trace point. 1890 */ 1891 if (pdata->ftps_noffs == 0) 1892 return (EINVAL); 1893 1894 switch (pdata->ftps_type) { 1895 case DTFTP_ENTRY: 1896 name = "entry"; 1897 aframes = FASTTRAP_ENTRY_AFRAMES; 1898 break; 1899 case DTFTP_RETURN: 1900 name = "return"; 1901 aframes = FASTTRAP_RETURN_AFRAMES; 1902 break; 1903 case DTFTP_OFFSETS: 1904 name = NULL; 1905 break; 1906 default: 1907 return (EINVAL); 1908 } 1909 1910 if ((provider = fasttrap_provider_lookup(pdata->ftps_pid, 1911 FASTTRAP_PID_NAME, &pid_attr)) == NULL) 1912 return (ESRCH); 1913 1914 /* 1915 * Increment this reference count to indicate that a consumer is 1916 * actively adding a new probe associated with this provider. This 1917 * prevents the provider from being deleted -- we'll need to check 1918 * for pending deletions when we drop this reference count. 1919 */ 1920 provider->ftp_ccount++; 1921 mutex_exit(&provider->ftp_mtx); 1922 1923 /* 1924 * Grab the creation lock to ensure consistency between calls to 1925 * dtrace_probe_lookup() and dtrace_probe_create() in the face of 1926 * other threads creating probes. We must drop the provider lock 1927 * before taking this lock to avoid a three-way deadlock with the 1928 * DTrace framework. 1929 */ 1930 mutex_enter(&provider->ftp_cmtx); 1931 1932 if (name == NULL) { 1933 for (i = 0; i < pdata->ftps_noffs; i++) { 1934 char name_str[17]; 1935 1936 (void) sprintf(name_str, "%llx", 1937 (unsigned long long)pdata->ftps_offs[i]); 1938 1939 if (dtrace_probe_lookup(provider->ftp_provid, 1940 pdata->ftps_mod, pdata->ftps_func, name_str) != 0) 1941 continue; 1942 1943 atomic_inc_32(&fasttrap_total); 1944 1945 if (fasttrap_total > fasttrap_max) { 1946 atomic_dec_32(&fasttrap_total); 1947 goto no_mem; 1948 } 1949 1950 pp = kmem_zalloc(sizeof (fasttrap_probe_t), KM_SLEEP); 1951 1952 pp->ftp_prov = provider; 1953 pp->ftp_faddr = pdata->ftps_pc; 1954 pp->ftp_fsize = pdata->ftps_size; 1955 pp->ftp_pid = pdata->ftps_pid; 1956 pp->ftp_ntps = 1; 1957 1958 tp = kmem_zalloc(sizeof (fasttrap_tracepoint_t), 1959 KM_SLEEP); 1960 1961 tp->ftt_proc = provider->ftp_proc; 1962 tp->ftt_pc = pdata->ftps_offs[i] + pdata->ftps_pc; 1963 tp->ftt_pid = pdata->ftps_pid; 1964 1965 pp->ftp_tps[0].fit_tp = tp; 1966 pp->ftp_tps[0].fit_id.fti_probe = pp; 1967 pp->ftp_tps[0].fit_id.fti_ptype = pdata->ftps_type; 1968 1969 pp->ftp_id = dtrace_probe_create(provider->ftp_provid, 1970 pdata->ftps_mod, pdata->ftps_func, name_str, 1971 FASTTRAP_OFFSET_AFRAMES, pp); 1972 } 1973 1974 } else if (dtrace_probe_lookup(provider->ftp_provid, pdata->ftps_mod, 1975 pdata->ftps_func, name) == 0) { 1976 atomic_add_32(&fasttrap_total, pdata->ftps_noffs); 1977 1978 if (fasttrap_total > fasttrap_max) { 1979 atomic_add_32(&fasttrap_total, -pdata->ftps_noffs); 1980 goto no_mem; 1981 } 1982 1983 /* 1984 * Make sure all tracepoint program counter values are unique. 1985 * We later assume that each probe has exactly one tracepoint 1986 * for a given pc. 1987 */ 1988 qsort(pdata->ftps_offs, pdata->ftps_noffs, 1989 sizeof (uint64_t), fasttrap_uint64_cmp); 1990 for (i = 1; i < pdata->ftps_noffs; i++) { 1991 if (pdata->ftps_offs[i] > pdata->ftps_offs[i - 1]) 1992 continue; 1993 1994 atomic_add_32(&fasttrap_total, -pdata->ftps_noffs); 1995 goto no_mem; 1996 } 1997 1998 ASSERT(pdata->ftps_noffs > 0); 1999 pp = kmem_zalloc(offsetof(fasttrap_probe_t, 2000 ftp_tps[pdata->ftps_noffs]), KM_SLEEP); 2001 2002 pp->ftp_prov = provider; 2003 pp->ftp_faddr = pdata->ftps_pc; 2004 pp->ftp_fsize = pdata->ftps_size; 2005 pp->ftp_pid = pdata->ftps_pid; 2006 pp->ftp_ntps = pdata->ftps_noffs; 2007 2008 for (i = 0; i < pdata->ftps_noffs; i++) { 2009 tp = kmem_zalloc(sizeof (fasttrap_tracepoint_t), 2010 KM_SLEEP); 2011 2012 tp->ftt_proc = provider->ftp_proc; 2013 tp->ftt_pc = pdata->ftps_offs[i] + pdata->ftps_pc; 2014 tp->ftt_pid = pdata->ftps_pid; 2015 2016 pp->ftp_tps[i].fit_tp = tp; 2017 pp->ftp_tps[i].fit_id.fti_probe = pp; 2018 pp->ftp_tps[i].fit_id.fti_ptype = pdata->ftps_type; 2019 } 2020 2021 pp->ftp_id = dtrace_probe_create(provider->ftp_provid, 2022 pdata->ftps_mod, pdata->ftps_func, name, aframes, pp); 2023 } 2024 2025 mutex_exit(&provider->ftp_cmtx); 2026 2027 /* 2028 * We know that the provider is still valid since we incremented the 2029 * creation reference count. If someone tried to clean up this provider 2030 * while we were using it (e.g. because the process called exec(2) or 2031 * exit(2)), take note of that and try to clean it up now. 2032 */ 2033 mutex_enter(&provider->ftp_mtx); 2034 provider->ftp_ccount--; 2035 whack = provider->ftp_retired; 2036 mutex_exit(&provider->ftp_mtx); 2037 2038 if (whack) 2039 fasttrap_pid_cleanup(); 2040 2041 return (0); 2042 2043no_mem: 2044 /* 2045 * If we've exhausted the allowable resources, we'll try to remove 2046 * this provider to free some up. This is to cover the case where 2047 * the user has accidentally created many more probes than was 2048 * intended (e.g. pid123:::). 2049 */ 2050 mutex_exit(&provider->ftp_cmtx); 2051 mutex_enter(&provider->ftp_mtx); 2052 provider->ftp_ccount--; 2053 provider->ftp_marked = 1; 2054 mutex_exit(&provider->ftp_mtx); 2055 2056 fasttrap_pid_cleanup(); 2057 2058 return (ENOMEM); 2059} 2060 2061/*ARGSUSED*/ 2062static void * 2063fasttrap_meta_provide(void *arg, dtrace_helper_provdesc_t *dhpv, pid_t pid) 2064{ 2065 fasttrap_provider_t *provider; 2066 2067 /* 2068 * A 32-bit unsigned integer (like a pid for example) can be 2069 * expressed in 10 or fewer decimal digits. Make sure that we'll 2070 * have enough space for the provider name. 2071 */ 2072 if (strlen(dhpv->dthpv_provname) + 10 >= 2073 sizeof (provider->ftp_name)) { 2074 printf("failed to instantiate provider %s: " 2075 "name too long to accomodate pid", dhpv->dthpv_provname); 2076 return (NULL); 2077 } 2078 2079 /* 2080 * Don't let folks spoof the true pid provider. 2081 */ 2082 if (strcmp(dhpv->dthpv_provname, FASTTRAP_PID_NAME) == 0) { 2083 printf("failed to instantiate provider %s: " 2084 "%s is an invalid name", dhpv->dthpv_provname, 2085 FASTTRAP_PID_NAME); 2086 return (NULL); 2087 } 2088 2089 /* 2090 * The highest stability class that fasttrap supports is ISA; cap 2091 * the stability of the new provider accordingly. 2092 */ 2093 if (dhpv->dthpv_pattr.dtpa_provider.dtat_class > DTRACE_CLASS_ISA) 2094 dhpv->dthpv_pattr.dtpa_provider.dtat_class = DTRACE_CLASS_ISA; 2095 if (dhpv->dthpv_pattr.dtpa_mod.dtat_class > DTRACE_CLASS_ISA) 2096 dhpv->dthpv_pattr.dtpa_mod.dtat_class = DTRACE_CLASS_ISA; 2097 if (dhpv->dthpv_pattr.dtpa_func.dtat_class > DTRACE_CLASS_ISA) 2098 dhpv->dthpv_pattr.dtpa_func.dtat_class = DTRACE_CLASS_ISA; 2099 if (dhpv->dthpv_pattr.dtpa_name.dtat_class > DTRACE_CLASS_ISA) 2100 dhpv->dthpv_pattr.dtpa_name.dtat_class = DTRACE_CLASS_ISA; 2101 if (dhpv->dthpv_pattr.dtpa_args.dtat_class > DTRACE_CLASS_ISA) 2102 dhpv->dthpv_pattr.dtpa_args.dtat_class = DTRACE_CLASS_ISA; 2103 2104 if ((provider = fasttrap_provider_lookup(pid, dhpv->dthpv_provname, 2105 &dhpv->dthpv_pattr)) == NULL) { 2106 printf("failed to instantiate provider %s for " 2107 "process %u", dhpv->dthpv_provname, (uint_t)pid); 2108 return (NULL); 2109 } 2110 2111 /* 2112 * Up the meta provider count so this provider isn't removed until 2113 * the meta provider has been told to remove it. 2114 */ 2115 provider->ftp_mcount++; 2116 2117 mutex_exit(&provider->ftp_mtx); 2118 2119 return (provider); 2120} 2121 2122/* 2123 * We know a few things about our context here: we know that the probe being 2124 * created doesn't already exist (DTrace won't load DOF at the same address 2125 * twice, even if explicitly told to do so) and we know that we are 2126 * single-threaded with respect to the meta provider machinery. Knowing that 2127 * this is a new probe and that there is no way for us to race with another 2128 * operation on this provider allows us an important optimization: we need not 2129 * lookup a probe before adding it. Saving this lookup is important because 2130 * this code is in the fork path for processes with USDT probes, and lookups 2131 * here are potentially very expensive because of long hash conflicts on 2132 * module, function and name (DTrace doesn't hash on provider name). 2133 */ 2134/*ARGSUSED*/ 2135static void 2136fasttrap_meta_create_probe(void *arg, void *parg, 2137 dtrace_helper_probedesc_t *dhpb) 2138{ 2139 fasttrap_provider_t *provider = parg; 2140 fasttrap_probe_t *pp; 2141 fasttrap_tracepoint_t *tp; 2142 int i, j; 2143 uint32_t ntps; 2144 2145 /* 2146 * Since the meta provider count is non-zero we don't have to worry 2147 * about this provider disappearing. 2148 */ 2149 ASSERT(provider->ftp_mcount > 0); 2150 2151 /* 2152 * The offsets must be unique. 2153 */ 2154 qsort(dhpb->dthpb_offs, dhpb->dthpb_noffs, sizeof (uint32_t), 2155 fasttrap_uint32_cmp); 2156 for (i = 1; i < dhpb->dthpb_noffs; i++) { 2157 if (dhpb->dthpb_base + dhpb->dthpb_offs[i] <= 2158 dhpb->dthpb_base + dhpb->dthpb_offs[i - 1]) 2159 return; 2160 } 2161 2162 qsort(dhpb->dthpb_enoffs, dhpb->dthpb_nenoffs, sizeof (uint32_t), 2163 fasttrap_uint32_cmp); 2164 for (i = 1; i < dhpb->dthpb_nenoffs; i++) { 2165 if (dhpb->dthpb_base + dhpb->dthpb_enoffs[i] <= 2166 dhpb->dthpb_base + dhpb->dthpb_enoffs[i - 1]) 2167 return; 2168 } 2169 2170 ntps = dhpb->dthpb_noffs + dhpb->dthpb_nenoffs; 2171 ASSERT(ntps > 0); 2172 2173 atomic_add_32(&fasttrap_total, ntps); 2174 2175 if (fasttrap_total > fasttrap_max) { 2176 atomic_add_32(&fasttrap_total, -ntps); 2177 return; 2178 } 2179 2180 pp = kmem_zalloc(offsetof(fasttrap_probe_t, ftp_tps[ntps]), KM_SLEEP); 2181 2182 pp->ftp_prov = provider; 2183 pp->ftp_pid = provider->ftp_pid; 2184 pp->ftp_ntps = ntps; 2185 pp->ftp_nargs = dhpb->dthpb_xargc; 2186 pp->ftp_xtypes = dhpb->dthpb_xtypes; 2187 pp->ftp_ntypes = dhpb->dthpb_ntypes; 2188 2189 /* 2190 * First create a tracepoint for each actual point of interest. 2191 */ 2192 for (i = 0; i < dhpb->dthpb_noffs; i++) { 2193 tp = kmem_zalloc(sizeof (fasttrap_tracepoint_t), KM_SLEEP); 2194 2195 tp->ftt_proc = provider->ftp_proc; 2196 tp->ftt_pc = dhpb->dthpb_base + dhpb->dthpb_offs[i]; 2197 tp->ftt_pid = provider->ftp_pid; 2198 2199 pp->ftp_tps[i].fit_tp = tp; 2200 pp->ftp_tps[i].fit_id.fti_probe = pp; 2201#ifdef __sparc 2202 pp->ftp_tps[i].fit_id.fti_ptype = DTFTP_POST_OFFSETS; 2203#else 2204 pp->ftp_tps[i].fit_id.fti_ptype = DTFTP_OFFSETS; 2205#endif 2206 } 2207 2208 /* 2209 * Then create a tracepoint for each is-enabled point. 2210 */ 2211 for (j = 0; i < ntps; i++, j++) { 2212 tp = kmem_zalloc(sizeof (fasttrap_tracepoint_t), KM_SLEEP); 2213 2214 tp->ftt_proc = provider->ftp_proc; 2215 tp->ftt_pc = dhpb->dthpb_base + dhpb->dthpb_enoffs[j]; 2216 tp->ftt_pid = provider->ftp_pid; 2217 2218 pp->ftp_tps[i].fit_tp = tp; 2219 pp->ftp_tps[i].fit_id.fti_probe = pp; 2220 pp->ftp_tps[i].fit_id.fti_ptype = DTFTP_IS_ENABLED; 2221 } 2222 2223 /* 2224 * If the arguments are shuffled around we set the argument remapping 2225 * table. Later, when the probe fires, we only remap the arguments 2226 * if the table is non-NULL. 2227 */ 2228 for (i = 0; i < dhpb->dthpb_xargc; i++) { 2229 if (dhpb->dthpb_args[i] != i) { 2230 pp->ftp_argmap = dhpb->dthpb_args; 2231 break; 2232 } 2233 } 2234 2235 /* 2236 * The probe is fully constructed -- register it with DTrace. 2237 */ 2238 pp->ftp_id = dtrace_probe_create(provider->ftp_provid, dhpb->dthpb_mod, 2239 dhpb->dthpb_func, dhpb->dthpb_name, FASTTRAP_OFFSET_AFRAMES, pp); 2240} 2241 2242/*ARGSUSED*/ 2243static void 2244fasttrap_meta_remove(void *arg, dtrace_helper_provdesc_t *dhpv, pid_t pid) 2245{ 2246 /* 2247 * Clean up the USDT provider. There may be active consumers of the 2248 * provider busy adding probes, no damage will actually befall the 2249 * provider until that count has dropped to zero. This just puts 2250 * the provider on death row. 2251 */ 2252 fasttrap_provider_retire(pid, dhpv->dthpv_provname, 1); 2253} 2254 2255static dtrace_mops_t fasttrap_mops = { 2256 .dtms_create_probe = fasttrap_meta_create_probe, 2257 .dtms_provide_pid = fasttrap_meta_provide, 2258 .dtms_remove_pid = fasttrap_meta_remove 2259}; 2260 2261/*ARGSUSED*/ 2262static int 2263fasttrap_open(struct cdev *dev __unused, int oflags __unused, 2264 int devtype __unused, struct thread *td __unused) 2265{ 2266 return (0); 2267} 2268 2269/*ARGSUSED*/ 2270static int 2271fasttrap_ioctl(struct cdev *dev, u_long cmd, caddr_t arg, int fflag, 2272 struct thread *td) 2273{ 2274#ifdef notyet 2275 struct kinfo_proc kp; 2276 const cred_t *cr = td->td_ucred; 2277#endif 2278 if (!dtrace_attached()) 2279 return (EAGAIN); 2280 2281 if (cmd == FASTTRAPIOC_MAKEPROBE) { 2282 fasttrap_probe_spec_t *uprobe = *(fasttrap_probe_spec_t **)arg; 2283 fasttrap_probe_spec_t *probe; 2284 uint64_t noffs; 2285 size_t size; 2286 int ret, err; 2287 2288 if (copyin(&uprobe->ftps_noffs, &noffs, 2289 sizeof (uprobe->ftps_noffs))) 2290 return (EFAULT); 2291 2292 /* 2293 * Probes must have at least one tracepoint. 2294 */ 2295 if (noffs == 0) 2296 return (EINVAL); 2297 2298 size = sizeof (fasttrap_probe_spec_t) + 2299 sizeof (probe->ftps_offs[0]) * (noffs - 1); 2300 2301 if (size > 1024 * 1024) 2302 return (ENOMEM); 2303 2304 probe = kmem_alloc(size, KM_SLEEP); 2305 2306 if (copyin(uprobe, probe, size) != 0 || 2307 probe->ftps_noffs != noffs) { 2308 kmem_free(probe, size); 2309 return (EFAULT); 2310 } 2311 2312 /* 2313 * Verify that the function and module strings contain no 2314 * funny characters. 2315 */ 2316 if (u8_validate(probe->ftps_func, strlen(probe->ftps_func), 2317 NULL, U8_VALIDATE_ENTIRE, &err) < 0) { 2318 ret = EINVAL; 2319 goto err; 2320 } 2321 2322 if (u8_validate(probe->ftps_mod, strlen(probe->ftps_mod), 2323 NULL, U8_VALIDATE_ENTIRE, &err) < 0) { 2324 ret = EINVAL; 2325 goto err; 2326 } 2327 2328#ifdef notyet 2329 if (!PRIV_POLICY_CHOICE(cr, PRIV_ALL, B_FALSE)) { 2330 proc_t *p; 2331 pid_t pid = probe->ftps_pid; 2332 2333#ifdef illumos 2334 mutex_enter(&pidlock); 2335#endif 2336 /* 2337 * Report an error if the process doesn't exist 2338 * or is actively being birthed. 2339 */ 2340 sx_slock(&proctree_lock); 2341 p = pfind(pid); 2342 if (p) 2343 fill_kinfo_proc(p, &kp); 2344 sx_sunlock(&proctree_lock); 2345 if (p == NULL || kp.ki_stat == SIDL) { 2346#ifdef illumos 2347 mutex_exit(&pidlock); 2348#endif 2349 return (ESRCH); 2350 } 2351#ifdef illumos 2352 mutex_enter(&p->p_lock); 2353 mutex_exit(&pidlock); 2354#else 2355 PROC_LOCK_ASSERT(p, MA_OWNED); 2356#endif 2357 2358#ifdef notyet 2359 if ((ret = priv_proc_cred_perm(cr, p, NULL, 2360 VREAD | VWRITE)) != 0) { 2361#ifdef illumos 2362 mutex_exit(&p->p_lock); 2363#else 2364 PROC_UNLOCK(p); 2365#endif 2366 return (ret); 2367 } 2368#endif /* notyet */ 2369#ifdef illumos 2370 mutex_exit(&p->p_lock); 2371#else 2372 PROC_UNLOCK(p); 2373#endif 2374 } 2375#endif /* notyet */ 2376 2377 ret = fasttrap_add_probe(probe); 2378err: 2379 kmem_free(probe, size); 2380 2381 return (ret); 2382 2383 } else if (cmd == FASTTRAPIOC_GETINSTR) { 2384 fasttrap_instr_query_t instr; 2385 fasttrap_tracepoint_t *tp; 2386 uint_t index; 2387#ifdef illumos 2388 int ret; 2389#endif 2390 2391#ifdef illumos 2392 if (copyin((void *)arg, &instr, sizeof (instr)) != 0) 2393 return (EFAULT); 2394#endif 2395 2396#ifdef notyet 2397 if (!PRIV_POLICY_CHOICE(cr, PRIV_ALL, B_FALSE)) { 2398 proc_t *p; 2399 pid_t pid = instr.ftiq_pid; 2400 2401#ifdef illumos 2402 mutex_enter(&pidlock); 2403#endif 2404 /* 2405 * Report an error if the process doesn't exist 2406 * or is actively being birthed. 2407 */ 2408 sx_slock(&proctree_lock); 2409 p = pfind(pid); 2410 if (p) 2411 fill_kinfo_proc(p, &kp); 2412 sx_sunlock(&proctree_lock); 2413 if (p == NULL || kp.ki_stat == SIDL) { 2414#ifdef illumos 2415 mutex_exit(&pidlock); 2416#endif 2417 return (ESRCH); 2418 } 2419#ifdef illumos 2420 mutex_enter(&p->p_lock); 2421 mutex_exit(&pidlock); 2422#else 2423 PROC_LOCK_ASSERT(p, MA_OWNED); 2424#endif 2425 2426#ifdef notyet 2427 if ((ret = priv_proc_cred_perm(cr, p, NULL, 2428 VREAD)) != 0) { 2429#ifdef illumos 2430 mutex_exit(&p->p_lock); 2431#else 2432 PROC_UNLOCK(p); 2433#endif 2434 return (ret); 2435 } 2436#endif /* notyet */ 2437 2438#ifdef illumos 2439 mutex_exit(&p->p_lock); 2440#else 2441 PROC_UNLOCK(p); 2442#endif 2443 } 2444#endif /* notyet */ 2445 2446 index = FASTTRAP_TPOINTS_INDEX(instr.ftiq_pid, instr.ftiq_pc); 2447 2448 mutex_enter(&fasttrap_tpoints.fth_table[index].ftb_mtx); 2449 tp = fasttrap_tpoints.fth_table[index].ftb_data; 2450 while (tp != NULL) { 2451 if (instr.ftiq_pid == tp->ftt_pid && 2452 instr.ftiq_pc == tp->ftt_pc && 2453 tp->ftt_proc->ftpc_acount != 0) 2454 break; 2455 2456 tp = tp->ftt_next; 2457 } 2458 2459 if (tp == NULL) { 2460 mutex_exit(&fasttrap_tpoints.fth_table[index].ftb_mtx); 2461 return (ENOENT); 2462 } 2463 2464 bcopy(&tp->ftt_instr, &instr.ftiq_instr, 2465 sizeof (instr.ftiq_instr)); 2466 mutex_exit(&fasttrap_tpoints.fth_table[index].ftb_mtx); 2467 2468 if (copyout(&instr, (void *)arg, sizeof (instr)) != 0) 2469 return (EFAULT); 2470 2471 return (0); 2472 } 2473 2474 return (EINVAL); 2475} 2476 2477static int 2478fasttrap_load(void) 2479{ 2480 ulong_t nent; 2481 int i, ret; 2482 2483 /* Create the /dev/dtrace/fasttrap entry. */ 2484 fasttrap_cdev = make_dev(&fasttrap_cdevsw, 0, UID_ROOT, GID_WHEEL, 0600, 2485 "dtrace/fasttrap"); 2486 2487 mtx_init(&fasttrap_cleanup_mtx, "fasttrap clean", "dtrace", MTX_DEF); 2488 mutex_init(&fasttrap_count_mtx, "fasttrap count mtx", MUTEX_DEFAULT, 2489 NULL); 2490 2491#ifdef illumos 2492 fasttrap_max = ddi_getprop(DDI_DEV_T_ANY, devi, DDI_PROP_DONTPASS, 2493 "fasttrap-max-probes", FASTTRAP_MAX_DEFAULT); 2494#endif 2495 fasttrap_total = 0; 2496 2497 /* 2498 * Conjure up the tracepoints hashtable... 2499 */ 2500#ifdef illumos 2501 nent = ddi_getprop(DDI_DEV_T_ANY, devi, DDI_PROP_DONTPASS, 2502 "fasttrap-hash-size", FASTTRAP_TPOINTS_DEFAULT_SIZE); 2503#else 2504 nent = tpoints_hash_size; 2505#endif 2506 2507 if (nent == 0 || nent > 0x1000000) 2508 nent = FASTTRAP_TPOINTS_DEFAULT_SIZE; 2509 2510 tpoints_hash_size = nent; 2511 2512 if (ISP2(nent)) 2513 fasttrap_tpoints.fth_nent = nent; 2514 else 2515 fasttrap_tpoints.fth_nent = 1 << fasttrap_highbit(nent); 2516 ASSERT(fasttrap_tpoints.fth_nent > 0); 2517 fasttrap_tpoints.fth_mask = fasttrap_tpoints.fth_nent - 1; 2518 fasttrap_tpoints.fth_table = kmem_zalloc(fasttrap_tpoints.fth_nent * 2519 sizeof (fasttrap_bucket_t), KM_SLEEP); 2520#ifndef illumos 2521 for (i = 0; i < fasttrap_tpoints.fth_nent; i++) 2522 mutex_init(&fasttrap_tpoints.fth_table[i].ftb_mtx, 2523 "tracepoints bucket mtx", MUTEX_DEFAULT, NULL); 2524#endif 2525 2526 /* 2527 * ... and the providers hash table... 2528 */ 2529 nent = FASTTRAP_PROVIDERS_DEFAULT_SIZE; 2530 if (ISP2(nent)) 2531 fasttrap_provs.fth_nent = nent; 2532 else 2533 fasttrap_provs.fth_nent = 1 << fasttrap_highbit(nent); 2534 ASSERT(fasttrap_provs.fth_nent > 0); 2535 fasttrap_provs.fth_mask = fasttrap_provs.fth_nent - 1; 2536 fasttrap_provs.fth_table = kmem_zalloc(fasttrap_provs.fth_nent * 2537 sizeof (fasttrap_bucket_t), KM_SLEEP); 2538#ifndef illumos 2539 for (i = 0; i < fasttrap_provs.fth_nent; i++) 2540 mutex_init(&fasttrap_provs.fth_table[i].ftb_mtx, 2541 "providers bucket mtx", MUTEX_DEFAULT, NULL); 2542#endif 2543 2544 ret = kproc_create(fasttrap_pid_cleanup_cb, NULL, 2545 &fasttrap_cleanup_proc, 0, 0, "ftcleanup"); 2546 if (ret != 0) { 2547 destroy_dev(fasttrap_cdev); 2548#ifndef illumos 2549 for (i = 0; i < fasttrap_provs.fth_nent; i++) 2550 mutex_destroy(&fasttrap_provs.fth_table[i].ftb_mtx); 2551 for (i = 0; i < fasttrap_tpoints.fth_nent; i++) 2552 mutex_destroy(&fasttrap_tpoints.fth_table[i].ftb_mtx); 2553#endif 2554 kmem_free(fasttrap_provs.fth_table, fasttrap_provs.fth_nent * 2555 sizeof (fasttrap_bucket_t)); 2556 mtx_destroy(&fasttrap_cleanup_mtx); 2557 mutex_destroy(&fasttrap_count_mtx); 2558 return (ret); 2559 } 2560 2561 2562 /* 2563 * ... and the procs hash table. 2564 */ 2565 nent = FASTTRAP_PROCS_DEFAULT_SIZE; 2566 if (ISP2(nent)) 2567 fasttrap_procs.fth_nent = nent; 2568 else 2569 fasttrap_procs.fth_nent = 1 << fasttrap_highbit(nent); 2570 ASSERT(fasttrap_procs.fth_nent > 0); 2571 fasttrap_procs.fth_mask = fasttrap_procs.fth_nent - 1; 2572 fasttrap_procs.fth_table = kmem_zalloc(fasttrap_procs.fth_nent * 2573 sizeof (fasttrap_bucket_t), KM_SLEEP); 2574#ifndef illumos 2575 for (i = 0; i < fasttrap_procs.fth_nent; i++) 2576 mutex_init(&fasttrap_procs.fth_table[i].ftb_mtx, 2577 "processes bucket mtx", MUTEX_DEFAULT, NULL); 2578 2579 rm_init(&fasttrap_tp_lock, "fasttrap tracepoint"); 2580 2581 /* 2582 * This event handler must run before kdtrace_thread_dtor() since it 2583 * accesses the thread's struct kdtrace_thread. 2584 */ 2585 fasttrap_thread_dtor_tag = EVENTHANDLER_REGISTER(thread_dtor, 2586 fasttrap_thread_dtor, NULL, EVENTHANDLER_PRI_FIRST); 2587#endif 2588 2589 /* 2590 * Install our hooks into fork(2), exec(2), and exit(2). 2591 */ 2592 dtrace_fasttrap_fork = &fasttrap_fork; 2593 dtrace_fasttrap_exit = &fasttrap_exec_exit; 2594 dtrace_fasttrap_exec = &fasttrap_exec_exit; 2595 2596 (void) dtrace_meta_register("fasttrap", &fasttrap_mops, NULL, 2597 &fasttrap_meta_id); 2598 2599 return (0); 2600} 2601 2602static int 2603fasttrap_unload(void) 2604{ 2605 int i, fail = 0; 2606 2607 /* 2608 * Unregister the meta-provider to make sure no new fasttrap- 2609 * managed providers come along while we're trying to close up 2610 * shop. If we fail to detach, we'll need to re-register as a 2611 * meta-provider. We can fail to unregister as a meta-provider 2612 * if providers we manage still exist. 2613 */ 2614 if (fasttrap_meta_id != DTRACE_METAPROVNONE && 2615 dtrace_meta_unregister(fasttrap_meta_id) != 0) 2616 return (-1); 2617 2618 /* 2619 * Iterate over all of our providers. If there's still a process 2620 * that corresponds to that pid, fail to detach. 2621 */ 2622 for (i = 0; i < fasttrap_provs.fth_nent; i++) { 2623 fasttrap_provider_t **fpp, *fp; 2624 fasttrap_bucket_t *bucket = &fasttrap_provs.fth_table[i]; 2625 2626 mutex_enter(&bucket->ftb_mtx); 2627 fpp = (fasttrap_provider_t **)&bucket->ftb_data; 2628 while ((fp = *fpp) != NULL) { 2629 /* 2630 * Acquire and release the lock as a simple way of 2631 * waiting for any other consumer to finish with 2632 * this provider. A thread must first acquire the 2633 * bucket lock so there's no chance of another thread 2634 * blocking on the provider's lock. 2635 */ 2636 mutex_enter(&fp->ftp_mtx); 2637 mutex_exit(&fp->ftp_mtx); 2638 2639 if (dtrace_unregister(fp->ftp_provid) != 0) { 2640 fail = 1; 2641 fpp = &fp->ftp_next; 2642 } else { 2643 *fpp = fp->ftp_next; 2644 fasttrap_provider_free(fp); 2645 } 2646 } 2647 2648 mutex_exit(&bucket->ftb_mtx); 2649 } 2650 2651 if (fail) { 2652 (void) dtrace_meta_register("fasttrap", &fasttrap_mops, NULL, 2653 &fasttrap_meta_id); 2654 2655 return (-1); 2656 } 2657 2658 /* 2659 * Stop new processes from entering these hooks now, before the 2660 * fasttrap_cleanup thread runs. That way all processes will hopefully 2661 * be out of these hooks before we free fasttrap_provs.fth_table 2662 */ 2663 ASSERT(dtrace_fasttrap_fork == &fasttrap_fork); 2664 dtrace_fasttrap_fork = NULL; 2665 2666 ASSERT(dtrace_fasttrap_exec == &fasttrap_exec_exit); 2667 dtrace_fasttrap_exec = NULL; 2668 2669 ASSERT(dtrace_fasttrap_exit == &fasttrap_exec_exit); 2670 dtrace_fasttrap_exit = NULL; 2671 2672 mtx_lock(&fasttrap_cleanup_mtx); 2673 fasttrap_cleanup_drain = 1; 2674 /* Wait for the cleanup thread to finish up and signal us. */ 2675 wakeup(&fasttrap_cleanup_cv); 2676 mtx_sleep(&fasttrap_cleanup_drain, &fasttrap_cleanup_mtx, 0, "ftcld", 2677 0); 2678 fasttrap_cleanup_proc = NULL; 2679 mtx_destroy(&fasttrap_cleanup_mtx); 2680 2681#ifdef DEBUG 2682 mutex_enter(&fasttrap_count_mtx); 2683 ASSERT(fasttrap_pid_count == 0); 2684 mutex_exit(&fasttrap_count_mtx); 2685#endif 2686 2687#ifndef illumos 2688 EVENTHANDLER_DEREGISTER(thread_dtor, fasttrap_thread_dtor_tag); 2689 2690 for (i = 0; i < fasttrap_tpoints.fth_nent; i++) 2691 mutex_destroy(&fasttrap_tpoints.fth_table[i].ftb_mtx); 2692 for (i = 0; i < fasttrap_provs.fth_nent; i++) 2693 mutex_destroy(&fasttrap_provs.fth_table[i].ftb_mtx); 2694 for (i = 0; i < fasttrap_procs.fth_nent; i++) 2695 mutex_destroy(&fasttrap_procs.fth_table[i].ftb_mtx); 2696#endif 2697 kmem_free(fasttrap_tpoints.fth_table, 2698 fasttrap_tpoints.fth_nent * sizeof (fasttrap_bucket_t)); 2699 fasttrap_tpoints.fth_nent = 0; 2700 2701 kmem_free(fasttrap_provs.fth_table, 2702 fasttrap_provs.fth_nent * sizeof (fasttrap_bucket_t)); 2703 fasttrap_provs.fth_nent = 0; 2704 2705 kmem_free(fasttrap_procs.fth_table, 2706 fasttrap_procs.fth_nent * sizeof (fasttrap_bucket_t)); 2707 fasttrap_procs.fth_nent = 0; 2708 2709#ifndef illumos 2710 destroy_dev(fasttrap_cdev); 2711 mutex_destroy(&fasttrap_count_mtx); 2712 rm_destroy(&fasttrap_tp_lock); 2713#endif 2714 2715 return (0); 2716} 2717 2718/* ARGSUSED */ 2719static int 2720fasttrap_modevent(module_t mod __unused, int type, void *data __unused) 2721{ 2722 int error = 0; 2723 2724 switch (type) { 2725 case MOD_LOAD: 2726 break; 2727 2728 case MOD_UNLOAD: 2729 break; 2730 2731 case MOD_SHUTDOWN: 2732 break; 2733 2734 default: 2735 error = EOPNOTSUPP; 2736 break; 2737 } 2738 return (error); 2739} 2740 2741SYSINIT(fasttrap_load, SI_SUB_DTRACE_PROVIDER, SI_ORDER_ANY, fasttrap_load, 2742 NULL); 2743SYSUNINIT(fasttrap_unload, SI_SUB_DTRACE_PROVIDER, SI_ORDER_ANY, 2744 fasttrap_unload, NULL); 2745 2746DEV_MODULE(fasttrap, fasttrap_modevent, NULL); 2747MODULE_VERSION(fasttrap, 1); 2748MODULE_DEPEND(fasttrap, dtrace, 1, 1, 1); 2749MODULE_DEPEND(fasttrap, opensolaris, 1, 1, 1); 2750