1179193Sjb/* 2179193Sjb * CDDL HEADER START 3179193Sjb * 4179193Sjb * The contents of this file are subject to the terms of the 5179193Sjb * Common Development and Distribution License (the "License"). 6179193Sjb * You may not use this file except in compliance with the License. 7179193Sjb * 8179193Sjb * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9179193Sjb * or http://www.opensolaris.org/os/licensing. 10179193Sjb * See the License for the specific language governing permissions 11179193Sjb * and limitations under the License. 12179193Sjb * 13179193Sjb * When distributing Covered Code, include this CDDL HEADER in each 14179193Sjb * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15179193Sjb * If applicable, add the following below this CDDL HEADER, with the 16179193Sjb * fields enclosed by brackets "[]" replaced with your own identifying 17179193Sjb * information: Portions Copyright [yyyy] [name of copyright owner] 18179193Sjb * 19179193Sjb * CDDL HEADER END 20211738Srpaulo * 21211738Srpaulo * Portions Copyright 2010 The FreeBSD Foundation 22211738Srpaulo * 23211738Srpaulo * $FreeBSD$ 24179193Sjb */ 25179193Sjb 26179193Sjb/* 27179198Sjb * Copyright 2008 Sun Microsystems, Inc. All rights reserved. 28179193Sjb * Use is subject to license terms. 29179193Sjb */ 30179193Sjb 31211738Srpaulo#if defined(sun) 32179193Sjb#pragma ident "%Z%%M% %I% %E% SMI" 33211738Srpaulo#endif 34179193Sjb 35179193Sjb#include <sys/atomic.h> 36179193Sjb#include <sys/errno.h> 37179193Sjb#include <sys/stat.h> 38179193Sjb#include <sys/modctl.h> 39179193Sjb#include <sys/conf.h> 40179193Sjb#include <sys/systm.h> 41211738Srpaulo#if defined(sun) 42179193Sjb#include <sys/ddi.h> 43211738Srpaulo#endif 44179193Sjb#include <sys/sunddi.h> 45179193Sjb#include <sys/cpuvar.h> 46179193Sjb#include <sys/kmem.h> 47211738Srpaulo#if defined(sun) 48179193Sjb#include <sys/strsubr.h> 49211738Srpaulo#endif 50179193Sjb#include <sys/fasttrap.h> 51179193Sjb#include <sys/fasttrap_impl.h> 52179193Sjb#include <sys/fasttrap_isa.h> 53179193Sjb#include <sys/dtrace.h> 54179193Sjb#include <sys/dtrace_impl.h> 55179193Sjb#include <sys/sysmacros.h> 56179193Sjb#include <sys/proc.h> 57179193Sjb#include <sys/policy.h> 58211738Srpaulo#if defined(sun) 59179193Sjb#include <util/qsort.h> 60211738Srpaulo#endif 61211738Srpaulo#include <sys/mutex.h> 62211738Srpaulo#include <sys/kernel.h> 63211738Srpaulo#if !defined(sun) 64211738Srpaulo#include <sys/user.h> 65211738Srpaulo#include <sys/dtrace_bsd.h> 66211738Srpaulo#include <cddl/dev/dtrace/dtrace_cddl.h> 67211738Srpaulo#endif 68179193Sjb 69179193Sjb/* 70179193Sjb * User-Land Trap-Based Tracing 71179193Sjb * ---------------------------- 72179193Sjb * 73179193Sjb * The fasttrap provider allows DTrace consumers to instrument any user-level 74179193Sjb * instruction to gather data; this includes probes with semantic 75179193Sjb * signifigance like entry and return as well as simple offsets into the 76179193Sjb * function. While the specific techniques used are very ISA specific, the 77179193Sjb * methodology is generalizable to any architecture. 78179193Sjb * 79179193Sjb * 80179193Sjb * The General Methodology 81179193Sjb * ----------------------- 82179193Sjb * 83179193Sjb * With the primary goal of tracing every user-land instruction and the 84179193Sjb * limitation that we can't trust user space so don't want to rely on much 85179193Sjb * information there, we begin by replacing the instructions we want to trace 86179193Sjb * with trap instructions. Each instruction we overwrite is saved into a hash 87179193Sjb * table keyed by process ID and pc address. When we enter the kernel due to 88179193Sjb * this trap instruction, we need the effects of the replaced instruction to 89179193Sjb * appear to have occurred before we proceed with the user thread's 90179193Sjb * execution. 91179193Sjb * 92179193Sjb * Each user level thread is represented by a ulwp_t structure which is 93179193Sjb * always easily accessible through a register. The most basic way to produce 94179193Sjb * the effects of the instruction we replaced is to copy that instruction out 95179193Sjb * to a bit of scratch space reserved in the user thread's ulwp_t structure 96179193Sjb * (a sort of kernel-private thread local storage), set the PC to that 97179193Sjb * scratch space and single step. When we reenter the kernel after single 98179193Sjb * stepping the instruction we must then adjust the PC to point to what would 99179193Sjb * normally be the next instruction. Of course, special care must be taken 100179193Sjb * for branches and jumps, but these represent such a small fraction of any 101179193Sjb * instruction set that writing the code to emulate these in the kernel is 102179193Sjb * not too difficult. 103179193Sjb * 104179193Sjb * Return probes may require several tracepoints to trace every return site, 105179193Sjb * and, conversely, each tracepoint may activate several probes (the entry 106179193Sjb * and offset 0 probes, for example). To solve this muliplexing problem, 107179193Sjb * tracepoints contain lists of probes to activate and probes contain lists 108179193Sjb * of tracepoints to enable. If a probe is activated, it adds its ID to 109179193Sjb * existing tracepoints or creates new ones as necessary. 110179193Sjb * 111179193Sjb * Most probes are activated _before_ the instruction is executed, but return 112179193Sjb * probes are activated _after_ the effects of the last instruction of the 113179193Sjb * function are visible. Return probes must be fired _after_ we have 114179193Sjb * single-stepped the instruction whereas all other probes are fired 115179193Sjb * beforehand. 116179193Sjb * 117179193Sjb * 118179193Sjb * Lock Ordering 119179193Sjb * ------------- 120179193Sjb * 121179193Sjb * The lock ordering below -- both internally and with respect to the DTrace 122179193Sjb * framework -- is a little tricky and bears some explanation. Each provider 123179193Sjb * has a lock (ftp_mtx) that protects its members including reference counts 124179193Sjb * for enabled probes (ftp_rcount), consumers actively creating probes 125179193Sjb * (ftp_ccount) and USDT consumers (ftp_mcount); all three prevent a provider 126179193Sjb * from being freed. A provider is looked up by taking the bucket lock for the 127179193Sjb * provider hash table, and is returned with its lock held. The provider lock 128179193Sjb * may be taken in functions invoked by the DTrace framework, but may not be 129179193Sjb * held while calling functions in the DTrace framework. 130179193Sjb * 131179193Sjb * To ensure consistency over multiple calls to the DTrace framework, the 132179193Sjb * creation lock (ftp_cmtx) should be held. Naturally, the creation lock may 133179193Sjb * not be taken when holding the provider lock as that would create a cyclic 134179193Sjb * lock ordering. In situations where one would naturally take the provider 135179193Sjb * lock and then the creation lock, we instead up a reference count to prevent 136179193Sjb * the provider from disappearing, drop the provider lock, and acquire the 137179193Sjb * creation lock. 138179193Sjb * 139179193Sjb * Briefly: 140179193Sjb * bucket lock before provider lock 141179193Sjb * DTrace before provider lock 142179193Sjb * creation lock before DTrace 143179193Sjb * never hold the provider lock and creation lock simultaneously 144179193Sjb */ 145179193Sjb 146211738Srpaulostatic d_open_t fasttrap_open; 147211738Srpaulostatic d_ioctl_t fasttrap_ioctl; 148211738Srpaulo 149211738Srpaulostatic struct cdevsw fasttrap_cdevsw = { 150211738Srpaulo .d_version = D_VERSION, 151211738Srpaulo .d_open = fasttrap_open, 152211738Srpaulo .d_ioctl = fasttrap_ioctl, 153211738Srpaulo .d_name = "fasttrap", 154211738Srpaulo}; 155211738Srpaulostatic struct cdev *fasttrap_cdev; 156179193Sjbstatic dtrace_meta_provider_id_t fasttrap_meta_id; 157179193Sjb 158255749Smarkjstatic struct proc *fasttrap_cleanup_proc; 159211738Srpaulostatic struct mtx fasttrap_cleanup_mtx; 160255749Smarkjstatic uint_t fasttrap_cleanup_work, fasttrap_cleanup_drain, fasttrap_cleanup_cv; 161179193Sjb 162179193Sjb/* 163179193Sjb * Generation count on modifications to the global tracepoint lookup table. 164179193Sjb */ 165179193Sjbstatic volatile uint64_t fasttrap_mod_gen; 166179193Sjb 167179193Sjb/* 168179193Sjb * When the fasttrap provider is loaded, fasttrap_max is set to either 169179193Sjb * FASTTRAP_MAX_DEFAULT or the value for fasttrap-max-probes in the 170179193Sjb * fasttrap.conf file. Each time a probe is created, fasttrap_total is 171179193Sjb * incremented by the number of tracepoints that may be associated with that 172179193Sjb * probe; fasttrap_total is capped at fasttrap_max. 173179193Sjb */ 174179193Sjb#define FASTTRAP_MAX_DEFAULT 250000 175179193Sjbstatic uint32_t fasttrap_max; 176179193Sjbstatic uint32_t fasttrap_total; 177179193Sjb 178250484Spfg/* 179250484Spfg * Copyright (c) 2011, Joyent, Inc. All rights reserved. 180250484Spfg */ 181179193Sjb 182179193Sjb#define FASTTRAP_TPOINTS_DEFAULT_SIZE 0x4000 183179193Sjb#define FASTTRAP_PROVIDERS_DEFAULT_SIZE 0x100 184179193Sjb#define FASTTRAP_PROCS_DEFAULT_SIZE 0x100 185179193Sjb 186179193Sjb#define FASTTRAP_PID_NAME "pid" 187179193Sjb 188179193Sjbfasttrap_hash_t fasttrap_tpoints; 189179193Sjbstatic fasttrap_hash_t fasttrap_provs; 190179193Sjbstatic fasttrap_hash_t fasttrap_procs; 191179193Sjb 192179193Sjbstatic uint64_t fasttrap_pid_count; /* pid ref count */ 193179193Sjbstatic kmutex_t fasttrap_count_mtx; /* lock on ref count */ 194179193Sjb 195179193Sjb#define FASTTRAP_ENABLE_FAIL 1 196179193Sjb#define FASTTRAP_ENABLE_PARTIAL 2 197179193Sjb 198179193Sjbstatic int fasttrap_tracepoint_enable(proc_t *, fasttrap_probe_t *, uint_t); 199179193Sjbstatic void fasttrap_tracepoint_disable(proc_t *, fasttrap_probe_t *, uint_t); 200179193Sjb 201179193Sjbstatic fasttrap_provider_t *fasttrap_provider_lookup(pid_t, const char *, 202179193Sjb const dtrace_pattr_t *); 203179193Sjbstatic void fasttrap_provider_retire(pid_t, const char *, int); 204179193Sjbstatic void fasttrap_provider_free(fasttrap_provider_t *); 205179193Sjb 206179193Sjbstatic fasttrap_proc_t *fasttrap_proc_lookup(pid_t); 207179193Sjbstatic void fasttrap_proc_release(fasttrap_proc_t *); 208179193Sjb 209179193Sjb#define FASTTRAP_PROVS_INDEX(pid, name) \ 210179193Sjb ((fasttrap_hash_str(name) + (pid)) & fasttrap_provs.fth_mask) 211179193Sjb 212179193Sjb#define FASTTRAP_PROCS_INDEX(pid) ((pid) & fasttrap_procs.fth_mask) 213179193Sjb 214211925Srpaulo#if !defined(sun) 215211925Srpaulostatic kmutex_t fasttrap_cpuc_pid_lock[MAXCPU]; 216211925Srpaulo#endif 217211925Srpaulo 218179193Sjbstatic int 219179193Sjbfasttrap_highbit(ulong_t i) 220179193Sjb{ 221179193Sjb int h = 1; 222179193Sjb 223179193Sjb if (i == 0) 224179193Sjb return (0); 225179193Sjb#ifdef _LP64 226179193Sjb if (i & 0xffffffff00000000ul) { 227179193Sjb h += 32; i >>= 32; 228179193Sjb } 229179193Sjb#endif 230179193Sjb if (i & 0xffff0000) { 231179193Sjb h += 16; i >>= 16; 232179193Sjb } 233179193Sjb if (i & 0xff00) { 234179193Sjb h += 8; i >>= 8; 235179193Sjb } 236179193Sjb if (i & 0xf0) { 237179193Sjb h += 4; i >>= 4; 238179193Sjb } 239179193Sjb if (i & 0xc) { 240179193Sjb h += 2; i >>= 2; 241179193Sjb } 242179193Sjb if (i & 0x2) { 243179193Sjb h += 1; 244179193Sjb } 245179193Sjb return (h); 246179193Sjb} 247179193Sjb 248179193Sjbstatic uint_t 249179193Sjbfasttrap_hash_str(const char *p) 250179193Sjb{ 251179193Sjb unsigned int g; 252179193Sjb uint_t hval = 0; 253179193Sjb 254179193Sjb while (*p) { 255179193Sjb hval = (hval << 4) + *p++; 256179193Sjb if ((g = (hval & 0xf0000000)) != 0) 257179193Sjb hval ^= g >> 24; 258179193Sjb hval &= ~g; 259179193Sjb } 260179193Sjb return (hval); 261179193Sjb} 262179193Sjb 263179193Sjbvoid 264179193Sjbfasttrap_sigtrap(proc_t *p, kthread_t *t, uintptr_t pc) 265179193Sjb{ 266211738Srpaulo#if defined(sun) 267179193Sjb sigqueue_t *sqp = kmem_zalloc(sizeof (sigqueue_t), KM_SLEEP); 268179193Sjb 269179193Sjb sqp->sq_info.si_signo = SIGTRAP; 270179193Sjb sqp->sq_info.si_code = TRAP_DTRACE; 271179193Sjb sqp->sq_info.si_addr = (caddr_t)pc; 272179193Sjb 273179193Sjb mutex_enter(&p->p_lock); 274179193Sjb sigaddqa(p, t, sqp); 275179193Sjb mutex_exit(&p->p_lock); 276179193Sjb 277179193Sjb if (t != NULL) 278179193Sjb aston(t); 279211738Srpaulo#else 280211738Srpaulo ksiginfo_t *ksi = kmem_zalloc(sizeof (ksiginfo_t), KM_SLEEP); 281211738Srpaulo 282211738Srpaulo ksiginfo_init(ksi); 283211738Srpaulo ksi->ksi_signo = SIGTRAP; 284211738Srpaulo ksi->ksi_code = TRAP_DTRACE; 285211738Srpaulo ksi->ksi_addr = (caddr_t)pc; 286211738Srpaulo PROC_LOCK(p); 287211745Srpaulo (void) tdksignal(t, SIGTRAP, ksi); 288211738Srpaulo PROC_UNLOCK(p); 289211738Srpaulo#endif 290179193Sjb} 291179193Sjb 292179193Sjb/* 293179193Sjb * This function ensures that no threads are actively using the memory 294179193Sjb * associated with probes that were formerly live. 295179193Sjb */ 296179193Sjbstatic void 297179193Sjbfasttrap_mod_barrier(uint64_t gen) 298179193Sjb{ 299179193Sjb int i; 300179193Sjb 301179193Sjb if (gen < fasttrap_mod_gen) 302179193Sjb return; 303179193Sjb 304179193Sjb fasttrap_mod_gen++; 305179193Sjb 306211925Srpaulo CPU_FOREACH(i) { 307211925Srpaulo mutex_enter(&fasttrap_cpuc_pid_lock[i]); 308211925Srpaulo mutex_exit(&fasttrap_cpuc_pid_lock[i]); 309179193Sjb } 310179193Sjb} 311179193Sjb 312179193Sjb/* 313255749Smarkj * This function performs asynchronous cleanup of fasttrap providers. The 314255749Smarkj * Solaris implementation of this mechanism use a timeout that's activated in 315255749Smarkj * fasttrap_pid_cleanup(), but this doesn't work in FreeBSD: one may sleep while 316255749Smarkj * holding the DTrace mutexes, but it is unsafe to sleep in a callout handler. 317255749Smarkj * Thus we use a dedicated process to perform the cleanup when requested. 318179193Sjb */ 319179193Sjb/*ARGSUSED*/ 320179193Sjbstatic void 321179193Sjbfasttrap_pid_cleanup_cb(void *data) 322179193Sjb{ 323179193Sjb fasttrap_provider_t **fpp, *fp; 324179193Sjb fasttrap_bucket_t *bucket; 325179193Sjb dtrace_provider_id_t provid; 326250484Spfg int i, later = 0, rval; 327179193Sjb 328255749Smarkj mtx_lock(&fasttrap_cleanup_mtx); 329255749Smarkj while (!fasttrap_cleanup_drain || later > 0) { 330179193Sjb fasttrap_cleanup_work = 0; 331211738Srpaulo mtx_unlock(&fasttrap_cleanup_mtx); 332179193Sjb 333179193Sjb later = 0; 334179193Sjb 335179193Sjb /* 336179193Sjb * Iterate over all the providers trying to remove the marked 337179193Sjb * ones. If a provider is marked but not retired, we just 338179193Sjb * have to take a crack at removing it -- it's no big deal if 339179193Sjb * we can't. 340179193Sjb */ 341179193Sjb for (i = 0; i < fasttrap_provs.fth_nent; i++) { 342179193Sjb bucket = &fasttrap_provs.fth_table[i]; 343179193Sjb mutex_enter(&bucket->ftb_mtx); 344179193Sjb fpp = (fasttrap_provider_t **)&bucket->ftb_data; 345179193Sjb 346179193Sjb while ((fp = *fpp) != NULL) { 347179193Sjb if (!fp->ftp_marked) { 348179193Sjb fpp = &fp->ftp_next; 349179193Sjb continue; 350179193Sjb } 351179193Sjb 352179193Sjb mutex_enter(&fp->ftp_mtx); 353179193Sjb 354179193Sjb /* 355179193Sjb * If this provider has consumers actively 356179193Sjb * creating probes (ftp_ccount) or is a USDT 357179193Sjb * provider (ftp_mcount), we can't unregister 358179193Sjb * or even condense. 359179193Sjb */ 360179193Sjb if (fp->ftp_ccount != 0 || 361179193Sjb fp->ftp_mcount != 0) { 362179193Sjb mutex_exit(&fp->ftp_mtx); 363179193Sjb fp->ftp_marked = 0; 364179193Sjb continue; 365179193Sjb } 366179193Sjb 367179193Sjb if (!fp->ftp_retired || fp->ftp_rcount != 0) 368179193Sjb fp->ftp_marked = 0; 369179193Sjb 370179193Sjb mutex_exit(&fp->ftp_mtx); 371179193Sjb 372179193Sjb /* 373179193Sjb * If we successfully unregister this 374179193Sjb * provider we can remove it from the hash 375179193Sjb * chain and free the memory. If our attempt 376179193Sjb * to unregister fails and this is a retired 377179193Sjb * provider, increment our flag to try again 378179193Sjb * pretty soon. If we've consumed more than 379179193Sjb * half of our total permitted number of 380179193Sjb * probes call dtrace_condense() to try to 381179193Sjb * clean out the unenabled probes. 382179193Sjb */ 383179193Sjb provid = fp->ftp_provid; 384250484Spfg if ((rval = dtrace_unregister(provid)) != 0) { 385179193Sjb if (fasttrap_total > fasttrap_max / 2) 386179193Sjb (void) dtrace_condense(provid); 387250484Spfg 388250484Spfg if (rval == EAGAIN) 389250484Spfg fp->ftp_marked = 1; 390250484Spfg 391179193Sjb later += fp->ftp_marked; 392179193Sjb fpp = &fp->ftp_next; 393179193Sjb } else { 394179193Sjb *fpp = fp->ftp_next; 395179193Sjb fasttrap_provider_free(fp); 396179193Sjb } 397179193Sjb } 398179193Sjb mutex_exit(&bucket->ftb_mtx); 399179193Sjb } 400255749Smarkj mtx_lock(&fasttrap_cleanup_mtx); 401179193Sjb 402255749Smarkj /* 403255749Smarkj * If we were unable to retire a provider, try again after a 404255749Smarkj * second. This situation can occur in certain circumstances 405255749Smarkj * where providers cannot be unregistered even though they have 406255749Smarkj * no probes enabled because of an execution of dtrace -l or 407255749Smarkj * something similar. 408255749Smarkj */ 409255749Smarkj if (later > 0 || fasttrap_cleanup_work || 410255749Smarkj fasttrap_cleanup_drain) { 411255749Smarkj mtx_unlock(&fasttrap_cleanup_mtx); 412255749Smarkj pause("ftclean", hz); 413255749Smarkj mtx_lock(&fasttrap_cleanup_mtx); 414255749Smarkj } else 415255749Smarkj mtx_sleep(&fasttrap_cleanup_cv, &fasttrap_cleanup_mtx, 416255749Smarkj 0, "ftcl", 0); 417179193Sjb } 418179193Sjb 419179193Sjb /* 420255749Smarkj * Wake up the thread in fasttrap_unload() now that we're done. 421179193Sjb */ 422255749Smarkj wakeup(&fasttrap_cleanup_drain); 423255749Smarkj mtx_unlock(&fasttrap_cleanup_mtx); 424179193Sjb 425255749Smarkj kthread_exit(); 426179193Sjb} 427179193Sjb 428179193Sjb/* 429179193Sjb * Activates the asynchronous cleanup mechanism. 430179193Sjb */ 431179193Sjbstatic void 432179193Sjbfasttrap_pid_cleanup(void) 433179193Sjb{ 434211738Srpaulo 435211738Srpaulo mtx_lock(&fasttrap_cleanup_mtx); 436255749Smarkj if (!fasttrap_cleanup_work) { 437255749Smarkj fasttrap_cleanup_work = 1; 438255749Smarkj wakeup(&fasttrap_cleanup_cv); 439255749Smarkj } 440211738Srpaulo mtx_unlock(&fasttrap_cleanup_mtx); 441179193Sjb} 442179193Sjb 443179193Sjb/* 444179193Sjb * This is called from cfork() via dtrace_fasttrap_fork(). The child 445179198Sjb * process's address space is (roughly) a copy of the parent process's so 446179193Sjb * we have to remove all the instrumentation we had previously enabled in the 447179193Sjb * parent. 448179193Sjb */ 449179193Sjbstatic void 450179193Sjbfasttrap_fork(proc_t *p, proc_t *cp) 451179193Sjb{ 452179193Sjb pid_t ppid = p->p_pid; 453179193Sjb int i; 454179193Sjb 455211738Srpaulo#if defined(sun) 456179193Sjb ASSERT(curproc == p); 457179193Sjb ASSERT(p->p_proc_flag & P_PR_LOCK); 458211738Srpaulo#else 459211738Srpaulo PROC_LOCK_ASSERT(p, MA_OWNED); 460211738Srpaulo#endif 461211738Srpaulo#if defined(sun) 462179193Sjb ASSERT(p->p_dtrace_count > 0); 463211738Srpaulo#else 464212357Srpaulo if (p->p_dtrace_helpers) { 465212357Srpaulo /* 466212357Srpaulo * dtrace_helpers_duplicate() allocates memory. 467212357Srpaulo */ 468212494Srpaulo _PHOLD(cp); 469212357Srpaulo PROC_UNLOCK(p); 470212357Srpaulo PROC_UNLOCK(cp); 471212357Srpaulo dtrace_helpers_duplicate(p, cp); 472212357Srpaulo PROC_LOCK(cp); 473212357Srpaulo PROC_LOCK(p); 474212494Srpaulo _PRELE(cp); 475212357Srpaulo } 476211738Srpaulo /* 477211738Srpaulo * This check is purposely here instead of in kern_fork.c because, 478211738Srpaulo * for legal resons, we cannot include the dtrace_cddl.h header 479211738Srpaulo * inside kern_fork.c and insert if-clause there. 480211738Srpaulo */ 481211738Srpaulo if (p->p_dtrace_count == 0) 482211738Srpaulo return; 483211738Srpaulo#endif 484179193Sjb ASSERT(cp->p_dtrace_count == 0); 485179193Sjb 486179193Sjb /* 487179193Sjb * This would be simpler and faster if we maintained per-process 488179193Sjb * hash tables of enabled tracepoints. It could, however, potentially 489179193Sjb * slow down execution of a tracepoint since we'd need to go 490179193Sjb * through two levels of indirection. In the future, we should 491179193Sjb * consider either maintaining per-process ancillary lists of 492179193Sjb * enabled tracepoints or hanging a pointer to a per-process hash 493179193Sjb * table of enabled tracepoints off the proc structure. 494179193Sjb */ 495179193Sjb 496179193Sjb /* 497179193Sjb * We don't have to worry about the child process disappearing 498179193Sjb * because we're in fork(). 499179193Sjb */ 500211738Srpaulo#if defined(sun) 501211738Srpaulo mtx_lock_spin(&cp->p_slock); 502179193Sjb sprlock_proc(cp); 503211738Srpaulo mtx_unlock_spin(&cp->p_slock); 504212494Srpaulo#else 505212494Srpaulo _PHOLD(cp); 506211738Srpaulo#endif 507179193Sjb 508179193Sjb /* 509179193Sjb * Iterate over every tracepoint looking for ones that belong to the 510179193Sjb * parent process, and remove each from the child process. 511179193Sjb */ 512179193Sjb for (i = 0; i < fasttrap_tpoints.fth_nent; i++) { 513179193Sjb fasttrap_tracepoint_t *tp; 514179193Sjb fasttrap_bucket_t *bucket = &fasttrap_tpoints.fth_table[i]; 515179193Sjb 516179193Sjb mutex_enter(&bucket->ftb_mtx); 517179193Sjb for (tp = bucket->ftb_data; tp != NULL; tp = tp->ftt_next) { 518179193Sjb if (tp->ftt_pid == ppid && 519179193Sjb tp->ftt_proc->ftpc_acount != 0) { 520179193Sjb int ret = fasttrap_tracepoint_remove(cp, tp); 521179193Sjb ASSERT(ret == 0); 522179198Sjb 523179198Sjb /* 524179198Sjb * The count of active providers can only be 525179198Sjb * decremented (i.e. to zero) during exec, 526179198Sjb * exit, and removal of a meta provider so it 527179198Sjb * should be impossible to drop the count 528179198Sjb * mid-fork. 529179198Sjb */ 530179198Sjb ASSERT(tp->ftt_proc->ftpc_acount != 0); 531179193Sjb } 532179193Sjb } 533179193Sjb mutex_exit(&bucket->ftb_mtx); 534179193Sjb } 535179193Sjb 536211738Srpaulo#if defined(sun) 537179193Sjb mutex_enter(&cp->p_lock); 538179193Sjb sprunlock(cp); 539212494Srpaulo#else 540212494Srpaulo _PRELE(cp); 541211738Srpaulo#endif 542179193Sjb} 543179193Sjb 544179193Sjb/* 545179193Sjb * This is called from proc_exit() or from exec_common() if p_dtrace_probes 546179193Sjb * is set on the proc structure to indicate that there is a pid provider 547179193Sjb * associated with this process. 548179193Sjb */ 549179193Sjbstatic void 550179193Sjbfasttrap_exec_exit(proc_t *p) 551179193Sjb{ 552211738Srpaulo#if defined(sun) 553179193Sjb ASSERT(p == curproc); 554211738Srpaulo#endif 555211738Srpaulo PROC_LOCK_ASSERT(p, MA_OWNED); 556212494Srpaulo _PHOLD(p); 557211738Srpaulo PROC_UNLOCK(p); 558179193Sjb 559179193Sjb /* 560179193Sjb * We clean up the pid provider for this process here; user-land 561179193Sjb * static probes are handled by the meta-provider remove entry point. 562179193Sjb */ 563179193Sjb fasttrap_provider_retire(p->p_pid, FASTTRAP_PID_NAME, 0); 564212357Srpaulo#if !defined(sun) 565212357Srpaulo if (p->p_dtrace_helpers) 566212357Srpaulo dtrace_helpers_destroy(p); 567212357Srpaulo#endif 568211738Srpaulo PROC_LOCK(p); 569212494Srpaulo _PRELE(p); 570179193Sjb} 571179193Sjb 572179193Sjb 573179193Sjb/*ARGSUSED*/ 574179193Sjbstatic void 575211738Srpaulofasttrap_pid_provide(void *arg, dtrace_probedesc_t *desc) 576179193Sjb{ 577179193Sjb /* 578179193Sjb * There are no "default" pid probes. 579179193Sjb */ 580179193Sjb} 581179193Sjb 582179193Sjbstatic int 583179193Sjbfasttrap_tracepoint_enable(proc_t *p, fasttrap_probe_t *probe, uint_t index) 584179193Sjb{ 585179193Sjb fasttrap_tracepoint_t *tp, *new_tp = NULL; 586179193Sjb fasttrap_bucket_t *bucket; 587179193Sjb fasttrap_id_t *id; 588179193Sjb pid_t pid; 589179193Sjb uintptr_t pc; 590179193Sjb 591179193Sjb ASSERT(index < probe->ftp_ntps); 592179193Sjb 593179193Sjb pid = probe->ftp_pid; 594179193Sjb pc = probe->ftp_tps[index].fit_tp->ftt_pc; 595179193Sjb id = &probe->ftp_tps[index].fit_id; 596179193Sjb 597179193Sjb ASSERT(probe->ftp_tps[index].fit_tp->ftt_pid == pid); 598179193Sjb 599211738Srpaulo#if defined(sun) 600179193Sjb ASSERT(!(p->p_flag & SVFORK)); 601211738Srpaulo#endif 602179193Sjb 603179193Sjb /* 604179193Sjb * Before we make any modifications, make sure we've imposed a barrier 605179193Sjb * on the generation in which this probe was last modified. 606179193Sjb */ 607179193Sjb fasttrap_mod_barrier(probe->ftp_gen); 608179193Sjb 609179193Sjb bucket = &fasttrap_tpoints.fth_table[FASTTRAP_TPOINTS_INDEX(pid, pc)]; 610179193Sjb 611179193Sjb /* 612179193Sjb * If the tracepoint has already been enabled, just add our id to the 613179193Sjb * list of interested probes. This may be our second time through 614179193Sjb * this path in which case we'll have constructed the tracepoint we'd 615179193Sjb * like to install. If we can't find a match, and have an allocated 616179193Sjb * tracepoint ready to go, enable that one now. 617179193Sjb * 618179193Sjb * A tracepoint whose process is defunct is also considered defunct. 619179193Sjb */ 620179193Sjbagain: 621179193Sjb mutex_enter(&bucket->ftb_mtx); 622179193Sjb for (tp = bucket->ftb_data; tp != NULL; tp = tp->ftt_next) { 623179198Sjb /* 624179198Sjb * Note that it's safe to access the active count on the 625179198Sjb * associated proc structure because we know that at least one 626179198Sjb * provider (this one) will still be around throughout this 627179198Sjb * operation. 628179198Sjb */ 629179193Sjb if (tp->ftt_pid != pid || tp->ftt_pc != pc || 630179193Sjb tp->ftt_proc->ftpc_acount == 0) 631179193Sjb continue; 632179193Sjb 633179193Sjb /* 634179193Sjb * Now that we've found a matching tracepoint, it would be 635179193Sjb * a decent idea to confirm that the tracepoint is still 636179193Sjb * enabled and the trap instruction hasn't been overwritten. 637179193Sjb * Since this is a little hairy, we'll punt for now. 638179193Sjb */ 639179193Sjb 640179193Sjb /* 641179193Sjb * This can't be the first interested probe. We don't have 642179193Sjb * to worry about another thread being in the midst of 643179193Sjb * deleting this tracepoint (which would be the only valid 644179193Sjb * reason for a tracepoint to have no interested probes) 645179193Sjb * since we're holding P_PR_LOCK for this process. 646179193Sjb */ 647179193Sjb ASSERT(tp->ftt_ids != NULL || tp->ftt_retids != NULL); 648179193Sjb 649179193Sjb switch (id->fti_ptype) { 650179193Sjb case DTFTP_ENTRY: 651179193Sjb case DTFTP_OFFSETS: 652179193Sjb case DTFTP_IS_ENABLED: 653179193Sjb id->fti_next = tp->ftt_ids; 654179193Sjb membar_producer(); 655179193Sjb tp->ftt_ids = id; 656179193Sjb membar_producer(); 657179193Sjb break; 658179193Sjb 659179193Sjb case DTFTP_RETURN: 660179193Sjb case DTFTP_POST_OFFSETS: 661179193Sjb id->fti_next = tp->ftt_retids; 662179193Sjb membar_producer(); 663179193Sjb tp->ftt_retids = id; 664179193Sjb membar_producer(); 665179193Sjb break; 666179193Sjb 667179193Sjb default: 668179193Sjb ASSERT(0); 669179193Sjb } 670179193Sjb 671179193Sjb mutex_exit(&bucket->ftb_mtx); 672179193Sjb 673179193Sjb if (new_tp != NULL) { 674179193Sjb new_tp->ftt_ids = NULL; 675179193Sjb new_tp->ftt_retids = NULL; 676179193Sjb } 677179193Sjb 678179193Sjb return (0); 679179193Sjb } 680179193Sjb 681179193Sjb /* 682179193Sjb * If we have a good tracepoint ready to go, install it now while 683179193Sjb * we have the lock held and no one can screw with us. 684179193Sjb */ 685179193Sjb if (new_tp != NULL) { 686179193Sjb int rc = 0; 687179193Sjb 688179193Sjb new_tp->ftt_next = bucket->ftb_data; 689179193Sjb membar_producer(); 690179193Sjb bucket->ftb_data = new_tp; 691179193Sjb membar_producer(); 692179193Sjb mutex_exit(&bucket->ftb_mtx); 693179193Sjb 694179193Sjb /* 695179193Sjb * Activate the tracepoint in the ISA-specific manner. 696179193Sjb * If this fails, we need to report the failure, but 697179193Sjb * indicate that this tracepoint must still be disabled 698179193Sjb * by calling fasttrap_tracepoint_disable(). 699179193Sjb */ 700179193Sjb if (fasttrap_tracepoint_install(p, new_tp) != 0) 701179193Sjb rc = FASTTRAP_ENABLE_PARTIAL; 702179193Sjb 703179193Sjb /* 704179193Sjb * Increment the count of the number of tracepoints active in 705179193Sjb * the victim process. 706179193Sjb */ 707211738Srpaulo#if defined(sun) 708179193Sjb ASSERT(p->p_proc_flag & P_PR_LOCK); 709211738Srpaulo#endif 710179193Sjb p->p_dtrace_count++; 711179193Sjb 712179193Sjb return (rc); 713179193Sjb } 714179193Sjb 715179193Sjb mutex_exit(&bucket->ftb_mtx); 716179193Sjb 717179193Sjb /* 718179193Sjb * Initialize the tracepoint that's been preallocated with the probe. 719179193Sjb */ 720179193Sjb new_tp = probe->ftp_tps[index].fit_tp; 721179193Sjb 722179193Sjb ASSERT(new_tp->ftt_pid == pid); 723179193Sjb ASSERT(new_tp->ftt_pc == pc); 724179193Sjb ASSERT(new_tp->ftt_proc == probe->ftp_prov->ftp_proc); 725179193Sjb ASSERT(new_tp->ftt_ids == NULL); 726179193Sjb ASSERT(new_tp->ftt_retids == NULL); 727179193Sjb 728179193Sjb switch (id->fti_ptype) { 729179193Sjb case DTFTP_ENTRY: 730179193Sjb case DTFTP_OFFSETS: 731179193Sjb case DTFTP_IS_ENABLED: 732179193Sjb id->fti_next = NULL; 733179193Sjb new_tp->ftt_ids = id; 734179193Sjb break; 735179193Sjb 736179193Sjb case DTFTP_RETURN: 737179193Sjb case DTFTP_POST_OFFSETS: 738179193Sjb id->fti_next = NULL; 739179193Sjb new_tp->ftt_retids = id; 740179193Sjb break; 741179193Sjb 742179193Sjb default: 743179193Sjb ASSERT(0); 744179193Sjb } 745179193Sjb 746179193Sjb /* 747179193Sjb * If the ISA-dependent initialization goes to plan, go back to the 748179193Sjb * beginning and try to install this freshly made tracepoint. 749179193Sjb */ 750179193Sjb if (fasttrap_tracepoint_init(p, new_tp, pc, id->fti_ptype) == 0) 751179193Sjb goto again; 752179193Sjb 753179193Sjb new_tp->ftt_ids = NULL; 754179193Sjb new_tp->ftt_retids = NULL; 755179193Sjb 756179193Sjb return (FASTTRAP_ENABLE_FAIL); 757179193Sjb} 758179193Sjb 759179193Sjbstatic void 760179193Sjbfasttrap_tracepoint_disable(proc_t *p, fasttrap_probe_t *probe, uint_t index) 761179193Sjb{ 762179193Sjb fasttrap_bucket_t *bucket; 763179193Sjb fasttrap_provider_t *provider = probe->ftp_prov; 764179193Sjb fasttrap_tracepoint_t **pp, *tp; 765211738Srpaulo fasttrap_id_t *id, **idp = NULL; 766179193Sjb pid_t pid; 767179193Sjb uintptr_t pc; 768179193Sjb 769179193Sjb ASSERT(index < probe->ftp_ntps); 770179193Sjb 771179193Sjb pid = probe->ftp_pid; 772179193Sjb pc = probe->ftp_tps[index].fit_tp->ftt_pc; 773179193Sjb id = &probe->ftp_tps[index].fit_id; 774179193Sjb 775179193Sjb ASSERT(probe->ftp_tps[index].fit_tp->ftt_pid == pid); 776179193Sjb 777179193Sjb /* 778179193Sjb * Find the tracepoint and make sure that our id is one of the 779179193Sjb * ones registered with it. 780179193Sjb */ 781179193Sjb bucket = &fasttrap_tpoints.fth_table[FASTTRAP_TPOINTS_INDEX(pid, pc)]; 782179193Sjb mutex_enter(&bucket->ftb_mtx); 783179193Sjb for (tp = bucket->ftb_data; tp != NULL; tp = tp->ftt_next) { 784179193Sjb if (tp->ftt_pid == pid && tp->ftt_pc == pc && 785179193Sjb tp->ftt_proc == provider->ftp_proc) 786179193Sjb break; 787179193Sjb } 788179193Sjb 789179193Sjb /* 790179193Sjb * If we somehow lost this tracepoint, we're in a world of hurt. 791179193Sjb */ 792179193Sjb ASSERT(tp != NULL); 793179193Sjb 794179193Sjb switch (id->fti_ptype) { 795179193Sjb case DTFTP_ENTRY: 796179193Sjb case DTFTP_OFFSETS: 797179193Sjb case DTFTP_IS_ENABLED: 798179193Sjb ASSERT(tp->ftt_ids != NULL); 799179193Sjb idp = &tp->ftt_ids; 800179193Sjb break; 801179193Sjb 802179193Sjb case DTFTP_RETURN: 803179193Sjb case DTFTP_POST_OFFSETS: 804179193Sjb ASSERT(tp->ftt_retids != NULL); 805179193Sjb idp = &tp->ftt_retids; 806179193Sjb break; 807179193Sjb 808179193Sjb default: 809179193Sjb ASSERT(0); 810179193Sjb } 811179193Sjb 812179193Sjb while ((*idp)->fti_probe != probe) { 813179193Sjb idp = &(*idp)->fti_next; 814179193Sjb ASSERT(*idp != NULL); 815179193Sjb } 816179193Sjb 817179193Sjb id = *idp; 818179193Sjb *idp = id->fti_next; 819179193Sjb membar_producer(); 820179193Sjb 821179193Sjb ASSERT(id->fti_probe == probe); 822179193Sjb 823179193Sjb /* 824179193Sjb * If there are other registered enablings of this tracepoint, we're 825179193Sjb * all done, but if this was the last probe assocated with this 826179193Sjb * this tracepoint, we need to remove and free it. 827179193Sjb */ 828179193Sjb if (tp->ftt_ids != NULL || tp->ftt_retids != NULL) { 829179193Sjb 830179193Sjb /* 831179193Sjb * If the current probe's tracepoint is in use, swap it 832179193Sjb * for an unused tracepoint. 833179193Sjb */ 834179193Sjb if (tp == probe->ftp_tps[index].fit_tp) { 835179193Sjb fasttrap_probe_t *tmp_probe; 836179193Sjb fasttrap_tracepoint_t **tmp_tp; 837179193Sjb uint_t tmp_index; 838179193Sjb 839179193Sjb if (tp->ftt_ids != NULL) { 840179193Sjb tmp_probe = tp->ftt_ids->fti_probe; 841179193Sjb /* LINTED - alignment */ 842179193Sjb tmp_index = FASTTRAP_ID_INDEX(tp->ftt_ids); 843179193Sjb tmp_tp = &tmp_probe->ftp_tps[tmp_index].fit_tp; 844179193Sjb } else { 845179193Sjb tmp_probe = tp->ftt_retids->fti_probe; 846179193Sjb /* LINTED - alignment */ 847179193Sjb tmp_index = FASTTRAP_ID_INDEX(tp->ftt_retids); 848179193Sjb tmp_tp = &tmp_probe->ftp_tps[tmp_index].fit_tp; 849179193Sjb } 850179193Sjb 851179193Sjb ASSERT(*tmp_tp != NULL); 852179193Sjb ASSERT(*tmp_tp != probe->ftp_tps[index].fit_tp); 853179193Sjb ASSERT((*tmp_tp)->ftt_ids == NULL); 854179193Sjb ASSERT((*tmp_tp)->ftt_retids == NULL); 855179193Sjb 856179193Sjb probe->ftp_tps[index].fit_tp = *tmp_tp; 857179193Sjb *tmp_tp = tp; 858179193Sjb } 859179193Sjb 860179193Sjb mutex_exit(&bucket->ftb_mtx); 861179193Sjb 862179193Sjb /* 863179193Sjb * Tag the modified probe with the generation in which it was 864179193Sjb * changed. 865179193Sjb */ 866179193Sjb probe->ftp_gen = fasttrap_mod_gen; 867179193Sjb return; 868179193Sjb } 869179193Sjb 870179193Sjb mutex_exit(&bucket->ftb_mtx); 871179193Sjb 872179193Sjb /* 873179193Sjb * We can't safely remove the tracepoint from the set of active 874179193Sjb * tracepoints until we've actually removed the fasttrap instruction 875179193Sjb * from the process's text. We can, however, operate on this 876179193Sjb * tracepoint secure in the knowledge that no other thread is going to 877179193Sjb * be looking at it since we hold P_PR_LOCK on the process if it's 878179193Sjb * live or we hold the provider lock on the process if it's dead and 879179193Sjb * gone. 880179193Sjb */ 881179193Sjb 882179193Sjb /* 883179193Sjb * We only need to remove the actual instruction if we're looking 884179193Sjb * at an existing process 885179193Sjb */ 886179193Sjb if (p != NULL) { 887179193Sjb /* 888179193Sjb * If we fail to restore the instruction we need to kill 889179193Sjb * this process since it's in a completely unrecoverable 890179193Sjb * state. 891179193Sjb */ 892179193Sjb if (fasttrap_tracepoint_remove(p, tp) != 0) 893179193Sjb fasttrap_sigtrap(p, NULL, pc); 894179193Sjb 895179193Sjb /* 896179193Sjb * Decrement the count of the number of tracepoints active 897179193Sjb * in the victim process. 898179193Sjb */ 899211738Srpaulo#if defined(sun) 900179193Sjb ASSERT(p->p_proc_flag & P_PR_LOCK); 901211738Srpaulo#endif 902179193Sjb p->p_dtrace_count--; 903179193Sjb } 904179193Sjb 905179193Sjb /* 906179193Sjb * Remove the probe from the hash table of active tracepoints. 907179193Sjb */ 908179193Sjb mutex_enter(&bucket->ftb_mtx); 909179193Sjb pp = (fasttrap_tracepoint_t **)&bucket->ftb_data; 910179193Sjb ASSERT(*pp != NULL); 911179193Sjb while (*pp != tp) { 912179193Sjb pp = &(*pp)->ftt_next; 913179193Sjb ASSERT(*pp != NULL); 914179193Sjb } 915179193Sjb 916179193Sjb *pp = tp->ftt_next; 917179193Sjb membar_producer(); 918179193Sjb 919179193Sjb mutex_exit(&bucket->ftb_mtx); 920179193Sjb 921179193Sjb /* 922179193Sjb * Tag the modified probe with the generation in which it was changed. 923179193Sjb */ 924179193Sjb probe->ftp_gen = fasttrap_mod_gen; 925179193Sjb} 926179193Sjb 927179193Sjbstatic void 928179193Sjbfasttrap_enable_callbacks(void) 929179193Sjb{ 930179193Sjb /* 931179193Sjb * We don't have to play the rw lock game here because we're 932179193Sjb * providing something rather than taking something away -- 933179193Sjb * we can be sure that no threads have tried to follow this 934179193Sjb * function pointer yet. 935179193Sjb */ 936179193Sjb mutex_enter(&fasttrap_count_mtx); 937179193Sjb if (fasttrap_pid_count == 0) { 938179193Sjb ASSERT(dtrace_pid_probe_ptr == NULL); 939179193Sjb ASSERT(dtrace_return_probe_ptr == NULL); 940179193Sjb dtrace_pid_probe_ptr = &fasttrap_pid_probe; 941179193Sjb dtrace_return_probe_ptr = &fasttrap_return_probe; 942179193Sjb } 943179193Sjb ASSERT(dtrace_pid_probe_ptr == &fasttrap_pid_probe); 944179193Sjb ASSERT(dtrace_return_probe_ptr == &fasttrap_return_probe); 945179193Sjb fasttrap_pid_count++; 946179193Sjb mutex_exit(&fasttrap_count_mtx); 947179193Sjb} 948179193Sjb 949179193Sjbstatic void 950179193Sjbfasttrap_disable_callbacks(void) 951179193Sjb{ 952211738Srpaulo#if defined(sun) 953179193Sjb ASSERT(MUTEX_HELD(&cpu_lock)); 954211738Srpaulo#endif 955179193Sjb 956211738Srpaulo 957179193Sjb mutex_enter(&fasttrap_count_mtx); 958179193Sjb ASSERT(fasttrap_pid_count > 0); 959179193Sjb fasttrap_pid_count--; 960179193Sjb if (fasttrap_pid_count == 0) { 961211738Srpaulo#if defined(sun) 962179193Sjb cpu_t *cur, *cpu = CPU; 963179193Sjb 964179193Sjb for (cur = cpu->cpu_next_onln; cur != cpu; 965179193Sjb cur = cur->cpu_next_onln) { 966179193Sjb rw_enter(&cur->cpu_ft_lock, RW_WRITER); 967179193Sjb } 968211738Srpaulo#endif 969179193Sjb dtrace_pid_probe_ptr = NULL; 970179193Sjb dtrace_return_probe_ptr = NULL; 971211738Srpaulo#if defined(sun) 972179193Sjb for (cur = cpu->cpu_next_onln; cur != cpu; 973179193Sjb cur = cur->cpu_next_onln) { 974179193Sjb rw_exit(&cur->cpu_ft_lock); 975179193Sjb } 976211738Srpaulo#endif 977179193Sjb } 978179193Sjb mutex_exit(&fasttrap_count_mtx); 979179193Sjb} 980179193Sjb 981179193Sjb/*ARGSUSED*/ 982179193Sjbstatic void 983179193Sjbfasttrap_pid_enable(void *arg, dtrace_id_t id, void *parg) 984179193Sjb{ 985179193Sjb fasttrap_probe_t *probe = parg; 986211738Srpaulo proc_t *p = NULL; 987179193Sjb int i, rc; 988179193Sjb 989179193Sjb ASSERT(probe != NULL); 990179193Sjb ASSERT(!probe->ftp_enabled); 991179193Sjb ASSERT(id == probe->ftp_id); 992211738Srpaulo#if defined(sun) 993179193Sjb ASSERT(MUTEX_HELD(&cpu_lock)); 994211738Srpaulo#endif 995179193Sjb 996179193Sjb /* 997179193Sjb * Increment the count of enabled probes on this probe's provider; 998179193Sjb * the provider can't go away while the probe still exists. We 999179193Sjb * must increment this even if we aren't able to properly enable 1000179193Sjb * this probe. 1001179193Sjb */ 1002179193Sjb mutex_enter(&probe->ftp_prov->ftp_mtx); 1003179193Sjb probe->ftp_prov->ftp_rcount++; 1004179193Sjb mutex_exit(&probe->ftp_prov->ftp_mtx); 1005179193Sjb 1006179193Sjb /* 1007179193Sjb * If this probe's provider is retired (meaning it was valid in a 1008179193Sjb * previously exec'ed incarnation of this address space), bail out. The 1009179193Sjb * provider can't go away while we're in this code path. 1010179193Sjb */ 1011179193Sjb if (probe->ftp_prov->ftp_retired) 1012179193Sjb return; 1013179193Sjb 1014179193Sjb /* 1015179193Sjb * If we can't find the process, it may be that we're in the context of 1016179193Sjb * a fork in which the traced process is being born and we're copying 1017179193Sjb * USDT probes. Otherwise, the process is gone so bail. 1018179193Sjb */ 1019211738Srpaulo#if defined(sun) 1020179193Sjb if ((p = sprlock(probe->ftp_pid)) == NULL) { 1021179193Sjb if ((curproc->p_flag & SFORKING) == 0) 1022179193Sjb return; 1023179193Sjb 1024179193Sjb mutex_enter(&pidlock); 1025179193Sjb p = prfind(probe->ftp_pid); 1026179193Sjb 1027179193Sjb /* 1028179193Sjb * Confirm that curproc is indeed forking the process in which 1029179193Sjb * we're trying to enable probes. 1030179193Sjb */ 1031179193Sjb ASSERT(p != NULL); 1032179193Sjb ASSERT(p->p_parent == curproc); 1033179193Sjb ASSERT(p->p_stat == SIDL); 1034179193Sjb 1035179193Sjb mutex_enter(&p->p_lock); 1036179193Sjb mutex_exit(&pidlock); 1037179193Sjb 1038179193Sjb sprlock_proc(p); 1039179193Sjb } 1040179193Sjb 1041179193Sjb ASSERT(!(p->p_flag & SVFORK)); 1042179193Sjb mutex_exit(&p->p_lock); 1043211738Srpaulo#else 1044211738Srpaulo if ((p = pfind(probe->ftp_pid)) == NULL) 1045211738Srpaulo return; 1046211738Srpaulo#endif 1047179193Sjb 1048179193Sjb /* 1049179193Sjb * We have to enable the trap entry point before any user threads have 1050179193Sjb * the chance to execute the trap instruction we're about to place 1051179193Sjb * in their process's text. 1052179193Sjb */ 1053212494Srpaulo#ifdef __FreeBSD__ 1054212494Srpaulo /* 1055212494Srpaulo * pfind() returns a locked process. 1056212494Srpaulo */ 1057212494Srpaulo _PHOLD(p); 1058211738Srpaulo PROC_UNLOCK(p); 1059212494Srpaulo#endif 1060179193Sjb fasttrap_enable_callbacks(); 1061179193Sjb 1062179193Sjb /* 1063179193Sjb * Enable all the tracepoints and add this probe's id to each 1064179193Sjb * tracepoint's list of active probes. 1065179193Sjb */ 1066179193Sjb for (i = 0; i < probe->ftp_ntps; i++) { 1067179193Sjb if ((rc = fasttrap_tracepoint_enable(p, probe, i)) != 0) { 1068179193Sjb /* 1069179193Sjb * If enabling the tracepoint failed completely, 1070179193Sjb * we don't have to disable it; if the failure 1071179193Sjb * was only partial we must disable it. 1072179193Sjb */ 1073179193Sjb if (rc == FASTTRAP_ENABLE_FAIL) 1074179193Sjb i--; 1075179193Sjb else 1076179193Sjb ASSERT(rc == FASTTRAP_ENABLE_PARTIAL); 1077179193Sjb 1078179193Sjb /* 1079179193Sjb * Back up and pull out all the tracepoints we've 1080179193Sjb * created so far for this probe. 1081179193Sjb */ 1082179193Sjb while (i >= 0) { 1083179193Sjb fasttrap_tracepoint_disable(p, probe, i); 1084179193Sjb i--; 1085179193Sjb } 1086179193Sjb 1087211738Srpaulo#if defined(sun) 1088179193Sjb mutex_enter(&p->p_lock); 1089179193Sjb sprunlock(p); 1090211738Srpaulo#else 1091212494Srpaulo PRELE(p); 1092211738Srpaulo#endif 1093179193Sjb 1094179193Sjb /* 1095179193Sjb * Since we're not actually enabling this probe, 1096179193Sjb * drop our reference on the trap table entry. 1097179193Sjb */ 1098179193Sjb fasttrap_disable_callbacks(); 1099179193Sjb return; 1100179193Sjb } 1101179193Sjb } 1102211738Srpaulo#if defined(sun) 1103179193Sjb mutex_enter(&p->p_lock); 1104179193Sjb sprunlock(p); 1105211738Srpaulo#else 1106212494Srpaulo PRELE(p); 1107211738Srpaulo#endif 1108179193Sjb 1109179193Sjb probe->ftp_enabled = 1; 1110179193Sjb} 1111179193Sjb 1112179193Sjb/*ARGSUSED*/ 1113179193Sjbstatic void 1114179193Sjbfasttrap_pid_disable(void *arg, dtrace_id_t id, void *parg) 1115179193Sjb{ 1116179193Sjb fasttrap_probe_t *probe = parg; 1117179193Sjb fasttrap_provider_t *provider = probe->ftp_prov; 1118179193Sjb proc_t *p; 1119179193Sjb int i, whack = 0; 1120179193Sjb 1121179193Sjb ASSERT(id == probe->ftp_id); 1122179193Sjb 1123211738Srpaulo mutex_enter(&provider->ftp_mtx); 1124211738Srpaulo 1125179193Sjb /* 1126179193Sjb * We won't be able to acquire a /proc-esque lock on the process 1127179193Sjb * iff the process is dead and gone. In this case, we rely on the 1128179193Sjb * provider lock as a point of mutual exclusion to prevent other 1129179193Sjb * DTrace consumers from disabling this probe. 1130179193Sjb */ 1131248248Sgibbs if ((p = pfind(probe->ftp_pid)) != NULL) { 1132212494Srpaulo#ifdef __FreeBSD__ 1133248248Sgibbs _PHOLD(p); 1134248248Sgibbs PROC_UNLOCK(p); 1135212494Srpaulo#endif 1136248248Sgibbs } 1137179193Sjb 1138179193Sjb /* 1139179193Sjb * Disable all the associated tracepoints (for fully enabled probes). 1140179193Sjb */ 1141179193Sjb if (probe->ftp_enabled) { 1142179193Sjb for (i = 0; i < probe->ftp_ntps; i++) { 1143179193Sjb fasttrap_tracepoint_disable(p, probe, i); 1144179193Sjb } 1145179193Sjb } 1146179193Sjb 1147179193Sjb ASSERT(provider->ftp_rcount > 0); 1148179193Sjb provider->ftp_rcount--; 1149179193Sjb 1150179193Sjb if (p != NULL) { 1151179193Sjb /* 1152179193Sjb * Even though we may not be able to remove it entirely, we 1153179193Sjb * mark this retired provider to get a chance to remove some 1154179193Sjb * of the associated probes. 1155179193Sjb */ 1156179193Sjb if (provider->ftp_retired && !provider->ftp_marked) 1157179193Sjb whack = provider->ftp_marked = 1; 1158179193Sjb mutex_exit(&provider->ftp_mtx); 1159179193Sjb } else { 1160179193Sjb /* 1161179193Sjb * If the process is dead, we're just waiting for the 1162179193Sjb * last probe to be disabled to be able to free it. 1163179193Sjb */ 1164179193Sjb if (provider->ftp_rcount == 0 && !provider->ftp_marked) 1165179193Sjb whack = provider->ftp_marked = 1; 1166179193Sjb mutex_exit(&provider->ftp_mtx); 1167179193Sjb } 1168179193Sjb 1169179193Sjb if (whack) 1170179193Sjb fasttrap_pid_cleanup(); 1171179193Sjb 1172212494Srpaulo#ifdef __FreeBSD__ 1173248248Sgibbs if (p != NULL) 1174248248Sgibbs PRELE(p); 1175212494Srpaulo#endif 1176179193Sjb if (!probe->ftp_enabled) 1177179193Sjb return; 1178179193Sjb 1179179193Sjb probe->ftp_enabled = 0; 1180179193Sjb 1181211738Srpaulo#if defined(sun) 1182179193Sjb ASSERT(MUTEX_HELD(&cpu_lock)); 1183211738Srpaulo#endif 1184179193Sjb fasttrap_disable_callbacks(); 1185179193Sjb} 1186179193Sjb 1187179193Sjb/*ARGSUSED*/ 1188179193Sjbstatic void 1189179193Sjbfasttrap_pid_getargdesc(void *arg, dtrace_id_t id, void *parg, 1190179193Sjb dtrace_argdesc_t *desc) 1191179193Sjb{ 1192179193Sjb fasttrap_probe_t *probe = parg; 1193179193Sjb char *str; 1194179193Sjb int i, ndx; 1195179193Sjb 1196179193Sjb desc->dtargd_native[0] = '\0'; 1197179193Sjb desc->dtargd_xlate[0] = '\0'; 1198179193Sjb 1199179193Sjb if (probe->ftp_prov->ftp_retired != 0 || 1200179193Sjb desc->dtargd_ndx >= probe->ftp_nargs) { 1201179193Sjb desc->dtargd_ndx = DTRACE_ARGNONE; 1202179193Sjb return; 1203179193Sjb } 1204179193Sjb 1205179193Sjb ndx = (probe->ftp_argmap != NULL) ? 1206179193Sjb probe->ftp_argmap[desc->dtargd_ndx] : desc->dtargd_ndx; 1207179193Sjb 1208179193Sjb str = probe->ftp_ntypes; 1209179193Sjb for (i = 0; i < ndx; i++) { 1210179193Sjb str += strlen(str) + 1; 1211179193Sjb } 1212179193Sjb 1213179193Sjb ASSERT(strlen(str + 1) < sizeof (desc->dtargd_native)); 1214179193Sjb (void) strcpy(desc->dtargd_native, str); 1215179193Sjb 1216179193Sjb if (probe->ftp_xtypes == NULL) 1217179193Sjb return; 1218179193Sjb 1219179193Sjb str = probe->ftp_xtypes; 1220179193Sjb for (i = 0; i < desc->dtargd_ndx; i++) { 1221179193Sjb str += strlen(str) + 1; 1222179193Sjb } 1223179193Sjb 1224179193Sjb ASSERT(strlen(str + 1) < sizeof (desc->dtargd_xlate)); 1225179193Sjb (void) strcpy(desc->dtargd_xlate, str); 1226179193Sjb} 1227179193Sjb 1228179193Sjb/*ARGSUSED*/ 1229179193Sjbstatic void 1230179193Sjbfasttrap_pid_destroy(void *arg, dtrace_id_t id, void *parg) 1231179193Sjb{ 1232179193Sjb fasttrap_probe_t *probe = parg; 1233179193Sjb int i; 1234179193Sjb size_t size; 1235179193Sjb 1236179193Sjb ASSERT(probe != NULL); 1237179193Sjb ASSERT(!probe->ftp_enabled); 1238179193Sjb ASSERT(fasttrap_total >= probe->ftp_ntps); 1239179193Sjb 1240179193Sjb atomic_add_32(&fasttrap_total, -probe->ftp_ntps); 1241179193Sjb size = offsetof(fasttrap_probe_t, ftp_tps[probe->ftp_ntps]); 1242179193Sjb 1243179193Sjb if (probe->ftp_gen + 1 >= fasttrap_mod_gen) 1244179193Sjb fasttrap_mod_barrier(probe->ftp_gen); 1245179193Sjb 1246179193Sjb for (i = 0; i < probe->ftp_ntps; i++) { 1247179193Sjb kmem_free(probe->ftp_tps[i].fit_tp, 1248179193Sjb sizeof (fasttrap_tracepoint_t)); 1249179193Sjb } 1250179193Sjb 1251179193Sjb kmem_free(probe, size); 1252179193Sjb} 1253179193Sjb 1254179193Sjb 1255179193Sjbstatic const dtrace_pattr_t pid_attr = { 1256179193Sjb{ DTRACE_STABILITY_EVOLVING, DTRACE_STABILITY_EVOLVING, DTRACE_CLASS_ISA }, 1257179193Sjb{ DTRACE_STABILITY_PRIVATE, DTRACE_STABILITY_PRIVATE, DTRACE_CLASS_UNKNOWN }, 1258179193Sjb{ DTRACE_STABILITY_PRIVATE, DTRACE_STABILITY_PRIVATE, DTRACE_CLASS_UNKNOWN }, 1259179193Sjb{ DTRACE_STABILITY_EVOLVING, DTRACE_STABILITY_EVOLVING, DTRACE_CLASS_ISA }, 1260179193Sjb{ DTRACE_STABILITY_PRIVATE, DTRACE_STABILITY_PRIVATE, DTRACE_CLASS_UNKNOWN }, 1261179193Sjb}; 1262179193Sjb 1263179193Sjbstatic dtrace_pops_t pid_pops = { 1264179193Sjb fasttrap_pid_provide, 1265179193Sjb NULL, 1266179193Sjb fasttrap_pid_enable, 1267179193Sjb fasttrap_pid_disable, 1268179193Sjb NULL, 1269179193Sjb NULL, 1270179193Sjb fasttrap_pid_getargdesc, 1271179193Sjb fasttrap_pid_getarg, 1272179193Sjb NULL, 1273179193Sjb fasttrap_pid_destroy 1274179193Sjb}; 1275179193Sjb 1276179193Sjbstatic dtrace_pops_t usdt_pops = { 1277179193Sjb fasttrap_pid_provide, 1278179193Sjb NULL, 1279179193Sjb fasttrap_pid_enable, 1280179193Sjb fasttrap_pid_disable, 1281179193Sjb NULL, 1282179193Sjb NULL, 1283179193Sjb fasttrap_pid_getargdesc, 1284179193Sjb fasttrap_usdt_getarg, 1285179193Sjb NULL, 1286179193Sjb fasttrap_pid_destroy 1287179193Sjb}; 1288179193Sjb 1289179193Sjbstatic fasttrap_proc_t * 1290179193Sjbfasttrap_proc_lookup(pid_t pid) 1291179193Sjb{ 1292179193Sjb fasttrap_bucket_t *bucket; 1293179193Sjb fasttrap_proc_t *fprc, *new_fprc; 1294179193Sjb 1295211738Srpaulo 1296179193Sjb bucket = &fasttrap_procs.fth_table[FASTTRAP_PROCS_INDEX(pid)]; 1297179193Sjb mutex_enter(&bucket->ftb_mtx); 1298179193Sjb 1299179193Sjb for (fprc = bucket->ftb_data; fprc != NULL; fprc = fprc->ftpc_next) { 1300179193Sjb if (fprc->ftpc_pid == pid && fprc->ftpc_acount != 0) { 1301179193Sjb mutex_enter(&fprc->ftpc_mtx); 1302179193Sjb mutex_exit(&bucket->ftb_mtx); 1303179193Sjb fprc->ftpc_rcount++; 1304179193Sjb atomic_add_64(&fprc->ftpc_acount, 1); 1305179198Sjb ASSERT(fprc->ftpc_acount <= fprc->ftpc_rcount); 1306179193Sjb mutex_exit(&fprc->ftpc_mtx); 1307179193Sjb 1308179193Sjb return (fprc); 1309179193Sjb } 1310179193Sjb } 1311179193Sjb 1312179193Sjb /* 1313179193Sjb * Drop the bucket lock so we don't try to perform a sleeping 1314179193Sjb * allocation under it. 1315179193Sjb */ 1316179193Sjb mutex_exit(&bucket->ftb_mtx); 1317179193Sjb 1318179193Sjb new_fprc = kmem_zalloc(sizeof (fasttrap_proc_t), KM_SLEEP); 1319179193Sjb new_fprc->ftpc_pid = pid; 1320179193Sjb new_fprc->ftpc_rcount = 1; 1321179193Sjb new_fprc->ftpc_acount = 1; 1322211738Srpaulo#if !defined(sun) 1323211738Srpaulo mutex_init(&new_fprc->ftpc_mtx, "fasttrap proc mtx", MUTEX_DEFAULT, 1324211738Srpaulo NULL); 1325211738Srpaulo#endif 1326179193Sjb 1327179193Sjb mutex_enter(&bucket->ftb_mtx); 1328179193Sjb 1329179193Sjb /* 1330179193Sjb * Take another lap through the list to make sure a proc hasn't 1331179193Sjb * been created for this pid while we weren't under the bucket lock. 1332179193Sjb */ 1333179193Sjb for (fprc = bucket->ftb_data; fprc != NULL; fprc = fprc->ftpc_next) { 1334179193Sjb if (fprc->ftpc_pid == pid && fprc->ftpc_acount != 0) { 1335179193Sjb mutex_enter(&fprc->ftpc_mtx); 1336179193Sjb mutex_exit(&bucket->ftb_mtx); 1337179193Sjb fprc->ftpc_rcount++; 1338179193Sjb atomic_add_64(&fprc->ftpc_acount, 1); 1339179198Sjb ASSERT(fprc->ftpc_acount <= fprc->ftpc_rcount); 1340179193Sjb mutex_exit(&fprc->ftpc_mtx); 1341179193Sjb 1342179193Sjb kmem_free(new_fprc, sizeof (fasttrap_proc_t)); 1343179193Sjb 1344179193Sjb return (fprc); 1345179193Sjb } 1346179193Sjb } 1347179193Sjb 1348179193Sjb new_fprc->ftpc_next = bucket->ftb_data; 1349179193Sjb bucket->ftb_data = new_fprc; 1350179193Sjb 1351179193Sjb mutex_exit(&bucket->ftb_mtx); 1352179193Sjb 1353179193Sjb return (new_fprc); 1354179193Sjb} 1355179193Sjb 1356179193Sjbstatic void 1357179193Sjbfasttrap_proc_release(fasttrap_proc_t *proc) 1358179193Sjb{ 1359179193Sjb fasttrap_bucket_t *bucket; 1360179193Sjb fasttrap_proc_t *fprc, **fprcp; 1361179193Sjb pid_t pid = proc->ftpc_pid; 1362179193Sjb 1363179193Sjb mutex_enter(&proc->ftpc_mtx); 1364179193Sjb 1365179193Sjb ASSERT(proc->ftpc_rcount != 0); 1366179198Sjb ASSERT(proc->ftpc_acount <= proc->ftpc_rcount); 1367179193Sjb 1368179193Sjb if (--proc->ftpc_rcount != 0) { 1369179193Sjb mutex_exit(&proc->ftpc_mtx); 1370179193Sjb return; 1371179193Sjb } 1372179193Sjb 1373179193Sjb mutex_exit(&proc->ftpc_mtx); 1374179193Sjb 1375179193Sjb /* 1376179193Sjb * There should definitely be no live providers associated with this 1377179193Sjb * process at this point. 1378179193Sjb */ 1379179193Sjb ASSERT(proc->ftpc_acount == 0); 1380179193Sjb 1381179193Sjb bucket = &fasttrap_procs.fth_table[FASTTRAP_PROCS_INDEX(pid)]; 1382179193Sjb mutex_enter(&bucket->ftb_mtx); 1383179193Sjb 1384179193Sjb fprcp = (fasttrap_proc_t **)&bucket->ftb_data; 1385179193Sjb while ((fprc = *fprcp) != NULL) { 1386179193Sjb if (fprc == proc) 1387179193Sjb break; 1388179193Sjb 1389179193Sjb fprcp = &fprc->ftpc_next; 1390179193Sjb } 1391179193Sjb 1392179193Sjb /* 1393179193Sjb * Something strange has happened if we can't find the proc. 1394179193Sjb */ 1395179193Sjb ASSERT(fprc != NULL); 1396179193Sjb 1397179193Sjb *fprcp = fprc->ftpc_next; 1398179193Sjb 1399179193Sjb mutex_exit(&bucket->ftb_mtx); 1400179193Sjb 1401179193Sjb kmem_free(fprc, sizeof (fasttrap_proc_t)); 1402179193Sjb} 1403179193Sjb 1404179193Sjb/* 1405179193Sjb * Lookup a fasttrap-managed provider based on its name and associated pid. 1406179193Sjb * If the pattr argument is non-NULL, this function instantiates the provider 1407179193Sjb * if it doesn't exist otherwise it returns NULL. The provider is returned 1408179193Sjb * with its lock held. 1409179193Sjb */ 1410179193Sjbstatic fasttrap_provider_t * 1411179193Sjbfasttrap_provider_lookup(pid_t pid, const char *name, 1412179193Sjb const dtrace_pattr_t *pattr) 1413179193Sjb{ 1414179193Sjb fasttrap_provider_t *fp, *new_fp = NULL; 1415179193Sjb fasttrap_bucket_t *bucket; 1416179193Sjb char provname[DTRACE_PROVNAMELEN]; 1417179193Sjb proc_t *p; 1418179193Sjb cred_t *cred; 1419179193Sjb 1420179193Sjb ASSERT(strlen(name) < sizeof (fp->ftp_name)); 1421179193Sjb ASSERT(pattr != NULL); 1422179193Sjb 1423179193Sjb bucket = &fasttrap_provs.fth_table[FASTTRAP_PROVS_INDEX(pid, name)]; 1424179193Sjb mutex_enter(&bucket->ftb_mtx); 1425179193Sjb 1426179193Sjb /* 1427179193Sjb * Take a lap through the list and return the match if we find it. 1428179193Sjb */ 1429179193Sjb for (fp = bucket->ftb_data; fp != NULL; fp = fp->ftp_next) { 1430179193Sjb if (fp->ftp_pid == pid && strcmp(fp->ftp_name, name) == 0 && 1431179193Sjb !fp->ftp_retired) { 1432179193Sjb mutex_enter(&fp->ftp_mtx); 1433179193Sjb mutex_exit(&bucket->ftb_mtx); 1434179193Sjb return (fp); 1435179193Sjb } 1436179193Sjb } 1437179193Sjb 1438179193Sjb /* 1439179193Sjb * Drop the bucket lock so we don't try to perform a sleeping 1440179193Sjb * allocation under it. 1441179193Sjb */ 1442179193Sjb mutex_exit(&bucket->ftb_mtx); 1443179193Sjb 1444179193Sjb /* 1445179193Sjb * Make sure the process exists, isn't a child created as the result 1446179193Sjb * of a vfork(2), and isn't a zombie (but may be in fork). 1447179193Sjb */ 1448211738Srpaulo if ((p = pfind(pid)) == NULL) 1449179193Sjb return (NULL); 1450179193Sjb 1451179193Sjb /* 1452179193Sjb * Increment p_dtrace_probes so that the process knows to inform us 1453179193Sjb * when it exits or execs. fasttrap_provider_free() decrements this 1454179193Sjb * when we're done with this provider. 1455179193Sjb */ 1456179193Sjb p->p_dtrace_probes++; 1457179193Sjb 1458179193Sjb /* 1459179193Sjb * Grab the credentials for this process so we have 1460179193Sjb * something to pass to dtrace_register(). 1461179193Sjb */ 1462211738Srpaulo PROC_LOCK_ASSERT(p, MA_OWNED); 1463211738Srpaulo crhold(p->p_ucred); 1464211738Srpaulo cred = p->p_ucred; 1465211738Srpaulo PROC_UNLOCK(p); 1466179193Sjb 1467179193Sjb new_fp = kmem_zalloc(sizeof (fasttrap_provider_t), KM_SLEEP); 1468179193Sjb new_fp->ftp_pid = pid; 1469179193Sjb new_fp->ftp_proc = fasttrap_proc_lookup(pid); 1470211738Srpaulo#if !defined(sun) 1471211738Srpaulo mutex_init(&new_fp->ftp_mtx, "provider mtx", MUTEX_DEFAULT, NULL); 1472211738Srpaulo mutex_init(&new_fp->ftp_cmtx, "lock on creating", MUTEX_DEFAULT, NULL); 1473211738Srpaulo#endif 1474179193Sjb 1475179193Sjb ASSERT(new_fp->ftp_proc != NULL); 1476179193Sjb 1477179193Sjb mutex_enter(&bucket->ftb_mtx); 1478179193Sjb 1479179193Sjb /* 1480179193Sjb * Take another lap through the list to make sure a provider hasn't 1481179193Sjb * been created for this pid while we weren't under the bucket lock. 1482179193Sjb */ 1483179193Sjb for (fp = bucket->ftb_data; fp != NULL; fp = fp->ftp_next) { 1484179193Sjb if (fp->ftp_pid == pid && strcmp(fp->ftp_name, name) == 0 && 1485179193Sjb !fp->ftp_retired) { 1486179193Sjb mutex_enter(&fp->ftp_mtx); 1487179193Sjb mutex_exit(&bucket->ftb_mtx); 1488179193Sjb fasttrap_provider_free(new_fp); 1489179193Sjb crfree(cred); 1490179193Sjb return (fp); 1491179193Sjb } 1492179193Sjb } 1493179193Sjb 1494179193Sjb (void) strcpy(new_fp->ftp_name, name); 1495179193Sjb 1496179193Sjb /* 1497179193Sjb * Fail and return NULL if either the provider name is too long 1498179193Sjb * or we fail to register this new provider with the DTrace 1499179193Sjb * framework. Note that this is the only place we ever construct 1500179193Sjb * the full provider name -- we keep it in pieces in the provider 1501179193Sjb * structure. 1502179193Sjb */ 1503179193Sjb if (snprintf(provname, sizeof (provname), "%s%u", name, (uint_t)pid) >= 1504179193Sjb sizeof (provname) || 1505179193Sjb dtrace_register(provname, pattr, 1506179193Sjb DTRACE_PRIV_PROC | DTRACE_PRIV_OWNER | DTRACE_PRIV_ZONEOWNER, cred, 1507179193Sjb pattr == &pid_attr ? &pid_pops : &usdt_pops, new_fp, 1508179193Sjb &new_fp->ftp_provid) != 0) { 1509179193Sjb mutex_exit(&bucket->ftb_mtx); 1510179193Sjb fasttrap_provider_free(new_fp); 1511179193Sjb crfree(cred); 1512179193Sjb return (NULL); 1513179193Sjb } 1514179193Sjb 1515179193Sjb new_fp->ftp_next = bucket->ftb_data; 1516179193Sjb bucket->ftb_data = new_fp; 1517179193Sjb 1518179193Sjb mutex_enter(&new_fp->ftp_mtx); 1519179193Sjb mutex_exit(&bucket->ftb_mtx); 1520179193Sjb 1521179193Sjb crfree(cred); 1522179193Sjb return (new_fp); 1523179193Sjb} 1524179193Sjb 1525179193Sjbstatic void 1526179193Sjbfasttrap_provider_free(fasttrap_provider_t *provider) 1527179193Sjb{ 1528179193Sjb pid_t pid = provider->ftp_pid; 1529179193Sjb proc_t *p; 1530179193Sjb 1531179193Sjb /* 1532179193Sjb * There need to be no associated enabled probes, no consumers 1533179193Sjb * creating probes, and no meta providers referencing this provider. 1534179193Sjb */ 1535179193Sjb ASSERT(provider->ftp_rcount == 0); 1536179193Sjb ASSERT(provider->ftp_ccount == 0); 1537179193Sjb ASSERT(provider->ftp_mcount == 0); 1538179193Sjb 1539179198Sjb /* 1540179198Sjb * If this provider hasn't been retired, we need to explicitly drop the 1541179198Sjb * count of active providers on the associated process structure. 1542179198Sjb */ 1543179198Sjb if (!provider->ftp_retired) { 1544179198Sjb atomic_add_64(&provider->ftp_proc->ftpc_acount, -1); 1545179198Sjb ASSERT(provider->ftp_proc->ftpc_acount < 1546179198Sjb provider->ftp_proc->ftpc_rcount); 1547179198Sjb } 1548179198Sjb 1549179193Sjb fasttrap_proc_release(provider->ftp_proc); 1550179193Sjb 1551211738Srpaulo#if !defined(sun) 1552211738Srpaulo mutex_destroy(&provider->ftp_mtx); 1553211738Srpaulo mutex_destroy(&provider->ftp_cmtx); 1554211738Srpaulo#endif 1555179193Sjb kmem_free(provider, sizeof (fasttrap_provider_t)); 1556179193Sjb 1557179193Sjb /* 1558179193Sjb * Decrement p_dtrace_probes on the process whose provider we're 1559179193Sjb * freeing. We don't have to worry about clobbering somone else's 1560179193Sjb * modifications to it because we have locked the bucket that 1561179193Sjb * corresponds to this process's hash chain in the provider hash 1562179193Sjb * table. Don't sweat it if we can't find the process. 1563179193Sjb */ 1564211738Srpaulo if ((p = pfind(pid)) == NULL) { 1565179193Sjb return; 1566179193Sjb } 1567179193Sjb 1568179193Sjb p->p_dtrace_probes--; 1569211738Srpaulo#if !defined(sun) 1570211738Srpaulo PROC_UNLOCK(p); 1571211738Srpaulo#endif 1572179193Sjb} 1573179193Sjb 1574179193Sjbstatic void 1575179193Sjbfasttrap_provider_retire(pid_t pid, const char *name, int mprov) 1576179193Sjb{ 1577179193Sjb fasttrap_provider_t *fp; 1578179193Sjb fasttrap_bucket_t *bucket; 1579179193Sjb dtrace_provider_id_t provid; 1580179193Sjb 1581179193Sjb ASSERT(strlen(name) < sizeof (fp->ftp_name)); 1582179193Sjb 1583179193Sjb bucket = &fasttrap_provs.fth_table[FASTTRAP_PROVS_INDEX(pid, name)]; 1584179193Sjb mutex_enter(&bucket->ftb_mtx); 1585179193Sjb 1586179193Sjb for (fp = bucket->ftb_data; fp != NULL; fp = fp->ftp_next) { 1587179193Sjb if (fp->ftp_pid == pid && strcmp(fp->ftp_name, name) == 0 && 1588179193Sjb !fp->ftp_retired) 1589179193Sjb break; 1590179193Sjb } 1591179193Sjb 1592179193Sjb if (fp == NULL) { 1593179193Sjb mutex_exit(&bucket->ftb_mtx); 1594179193Sjb return; 1595179193Sjb } 1596179193Sjb 1597179193Sjb mutex_enter(&fp->ftp_mtx); 1598179193Sjb ASSERT(!mprov || fp->ftp_mcount > 0); 1599179193Sjb if (mprov && --fp->ftp_mcount != 0) { 1600179193Sjb mutex_exit(&fp->ftp_mtx); 1601179193Sjb mutex_exit(&bucket->ftb_mtx); 1602179193Sjb return; 1603179193Sjb } 1604179193Sjb 1605179193Sjb /* 1606179193Sjb * Mark the provider to be removed in our post-processing step, mark it 1607179193Sjb * retired, and drop the active count on its proc. Marking it indicates 1608179193Sjb * that we should try to remove it; setting the retired flag indicates 1609179193Sjb * that we're done with this provider; dropping the active the proc 1610179193Sjb * releases our hold, and when this reaches zero (as it will during 1611179193Sjb * exit or exec) the proc and associated providers become defunct. 1612179193Sjb * 1613179193Sjb * We obviously need to take the bucket lock before the provider lock 1614179193Sjb * to perform the lookup, but we need to drop the provider lock 1615179193Sjb * before calling into the DTrace framework since we acquire the 1616179193Sjb * provider lock in callbacks invoked from the DTrace framework. The 1617179193Sjb * bucket lock therefore protects the integrity of the provider hash 1618179193Sjb * table. 1619179193Sjb */ 1620179193Sjb atomic_add_64(&fp->ftp_proc->ftpc_acount, -1); 1621179198Sjb ASSERT(fp->ftp_proc->ftpc_acount < fp->ftp_proc->ftpc_rcount); 1622179198Sjb 1623179193Sjb fp->ftp_retired = 1; 1624179193Sjb fp->ftp_marked = 1; 1625179193Sjb provid = fp->ftp_provid; 1626179193Sjb mutex_exit(&fp->ftp_mtx); 1627179193Sjb 1628179193Sjb /* 1629179193Sjb * We don't have to worry about invalidating the same provider twice 1630179193Sjb * since fasttrap_provider_lookup() will ignore provider that have 1631179193Sjb * been marked as retired. 1632179193Sjb */ 1633179193Sjb dtrace_invalidate(provid); 1634179193Sjb 1635179193Sjb mutex_exit(&bucket->ftb_mtx); 1636179193Sjb 1637179193Sjb fasttrap_pid_cleanup(); 1638179193Sjb} 1639179193Sjb 1640179193Sjbstatic int 1641179193Sjbfasttrap_uint32_cmp(const void *ap, const void *bp) 1642179193Sjb{ 1643179193Sjb return (*(const uint32_t *)ap - *(const uint32_t *)bp); 1644179193Sjb} 1645179193Sjb 1646179193Sjbstatic int 1647179193Sjbfasttrap_uint64_cmp(const void *ap, const void *bp) 1648179193Sjb{ 1649179193Sjb return (*(const uint64_t *)ap - *(const uint64_t *)bp); 1650179193Sjb} 1651179193Sjb 1652179193Sjbstatic int 1653179193Sjbfasttrap_add_probe(fasttrap_probe_spec_t *pdata) 1654179193Sjb{ 1655179193Sjb fasttrap_provider_t *provider; 1656179193Sjb fasttrap_probe_t *pp; 1657179193Sjb fasttrap_tracepoint_t *tp; 1658179193Sjb char *name; 1659211738Srpaulo int i, aframes = 0, whack; 1660179193Sjb 1661179193Sjb /* 1662179193Sjb * There needs to be at least one desired trace point. 1663179193Sjb */ 1664179193Sjb if (pdata->ftps_noffs == 0) 1665179193Sjb return (EINVAL); 1666179193Sjb 1667179193Sjb switch (pdata->ftps_type) { 1668179193Sjb case DTFTP_ENTRY: 1669179193Sjb name = "entry"; 1670179193Sjb aframes = FASTTRAP_ENTRY_AFRAMES; 1671179193Sjb break; 1672179193Sjb case DTFTP_RETURN: 1673179193Sjb name = "return"; 1674179193Sjb aframes = FASTTRAP_RETURN_AFRAMES; 1675179193Sjb break; 1676179193Sjb case DTFTP_OFFSETS: 1677179193Sjb name = NULL; 1678179193Sjb break; 1679179193Sjb default: 1680179193Sjb return (EINVAL); 1681179193Sjb } 1682179193Sjb 1683179193Sjb if ((provider = fasttrap_provider_lookup(pdata->ftps_pid, 1684179193Sjb FASTTRAP_PID_NAME, &pid_attr)) == NULL) 1685179193Sjb return (ESRCH); 1686179193Sjb 1687179193Sjb /* 1688179193Sjb * Increment this reference count to indicate that a consumer is 1689179193Sjb * actively adding a new probe associated with this provider. This 1690179193Sjb * prevents the provider from being deleted -- we'll need to check 1691179193Sjb * for pending deletions when we drop this reference count. 1692179193Sjb */ 1693179193Sjb provider->ftp_ccount++; 1694179193Sjb mutex_exit(&provider->ftp_mtx); 1695179193Sjb 1696179193Sjb /* 1697179193Sjb * Grab the creation lock to ensure consistency between calls to 1698179193Sjb * dtrace_probe_lookup() and dtrace_probe_create() in the face of 1699179193Sjb * other threads creating probes. We must drop the provider lock 1700179193Sjb * before taking this lock to avoid a three-way deadlock with the 1701179193Sjb * DTrace framework. 1702179193Sjb */ 1703179193Sjb mutex_enter(&provider->ftp_cmtx); 1704179193Sjb 1705179193Sjb if (name == NULL) { 1706179193Sjb for (i = 0; i < pdata->ftps_noffs; i++) { 1707179193Sjb char name_str[17]; 1708179193Sjb 1709179193Sjb (void) sprintf(name_str, "%llx", 1710179193Sjb (unsigned long long)pdata->ftps_offs[i]); 1711179193Sjb 1712179193Sjb if (dtrace_probe_lookup(provider->ftp_provid, 1713179193Sjb pdata->ftps_mod, pdata->ftps_func, name_str) != 0) 1714179193Sjb continue; 1715179193Sjb 1716179193Sjb atomic_add_32(&fasttrap_total, 1); 1717179193Sjb 1718179193Sjb if (fasttrap_total > fasttrap_max) { 1719179193Sjb atomic_add_32(&fasttrap_total, -1); 1720179193Sjb goto no_mem; 1721179193Sjb } 1722179193Sjb 1723179193Sjb pp = kmem_zalloc(sizeof (fasttrap_probe_t), KM_SLEEP); 1724179193Sjb 1725179193Sjb pp->ftp_prov = provider; 1726179193Sjb pp->ftp_faddr = pdata->ftps_pc; 1727179193Sjb pp->ftp_fsize = pdata->ftps_size; 1728179193Sjb pp->ftp_pid = pdata->ftps_pid; 1729179193Sjb pp->ftp_ntps = 1; 1730179193Sjb 1731179193Sjb tp = kmem_zalloc(sizeof (fasttrap_tracepoint_t), 1732179193Sjb KM_SLEEP); 1733179193Sjb 1734179193Sjb tp->ftt_proc = provider->ftp_proc; 1735179193Sjb tp->ftt_pc = pdata->ftps_offs[i] + pdata->ftps_pc; 1736179193Sjb tp->ftt_pid = pdata->ftps_pid; 1737179193Sjb 1738179193Sjb pp->ftp_tps[0].fit_tp = tp; 1739179193Sjb pp->ftp_tps[0].fit_id.fti_probe = pp; 1740179193Sjb pp->ftp_tps[0].fit_id.fti_ptype = pdata->ftps_type; 1741179193Sjb 1742179193Sjb pp->ftp_id = dtrace_probe_create(provider->ftp_provid, 1743179193Sjb pdata->ftps_mod, pdata->ftps_func, name_str, 1744179193Sjb FASTTRAP_OFFSET_AFRAMES, pp); 1745179193Sjb } 1746179193Sjb 1747179193Sjb } else if (dtrace_probe_lookup(provider->ftp_provid, pdata->ftps_mod, 1748179193Sjb pdata->ftps_func, name) == 0) { 1749179193Sjb atomic_add_32(&fasttrap_total, pdata->ftps_noffs); 1750179193Sjb 1751179193Sjb if (fasttrap_total > fasttrap_max) { 1752179193Sjb atomic_add_32(&fasttrap_total, -pdata->ftps_noffs); 1753179193Sjb goto no_mem; 1754179193Sjb } 1755179193Sjb 1756179193Sjb /* 1757179193Sjb * Make sure all tracepoint program counter values are unique. 1758179193Sjb * We later assume that each probe has exactly one tracepoint 1759179193Sjb * for a given pc. 1760179193Sjb */ 1761179193Sjb qsort(pdata->ftps_offs, pdata->ftps_noffs, 1762179193Sjb sizeof (uint64_t), fasttrap_uint64_cmp); 1763179193Sjb for (i = 1; i < pdata->ftps_noffs; i++) { 1764179193Sjb if (pdata->ftps_offs[i] > pdata->ftps_offs[i - 1]) 1765179193Sjb continue; 1766179193Sjb 1767179193Sjb atomic_add_32(&fasttrap_total, -pdata->ftps_noffs); 1768179193Sjb goto no_mem; 1769179193Sjb } 1770179193Sjb 1771179193Sjb ASSERT(pdata->ftps_noffs > 0); 1772179193Sjb pp = kmem_zalloc(offsetof(fasttrap_probe_t, 1773179193Sjb ftp_tps[pdata->ftps_noffs]), KM_SLEEP); 1774179193Sjb 1775179193Sjb pp->ftp_prov = provider; 1776179193Sjb pp->ftp_faddr = pdata->ftps_pc; 1777179193Sjb pp->ftp_fsize = pdata->ftps_size; 1778179193Sjb pp->ftp_pid = pdata->ftps_pid; 1779179193Sjb pp->ftp_ntps = pdata->ftps_noffs; 1780179193Sjb 1781179193Sjb for (i = 0; i < pdata->ftps_noffs; i++) { 1782179193Sjb tp = kmem_zalloc(sizeof (fasttrap_tracepoint_t), 1783179193Sjb KM_SLEEP); 1784179193Sjb 1785179193Sjb tp->ftt_proc = provider->ftp_proc; 1786179193Sjb tp->ftt_pc = pdata->ftps_offs[i] + pdata->ftps_pc; 1787179193Sjb tp->ftt_pid = pdata->ftps_pid; 1788179193Sjb 1789179193Sjb pp->ftp_tps[i].fit_tp = tp; 1790179193Sjb pp->ftp_tps[i].fit_id.fti_probe = pp; 1791179193Sjb pp->ftp_tps[i].fit_id.fti_ptype = pdata->ftps_type; 1792179193Sjb } 1793179193Sjb 1794179193Sjb pp->ftp_id = dtrace_probe_create(provider->ftp_provid, 1795179193Sjb pdata->ftps_mod, pdata->ftps_func, name, aframes, pp); 1796179193Sjb } 1797179193Sjb 1798179193Sjb mutex_exit(&provider->ftp_cmtx); 1799179193Sjb 1800179193Sjb /* 1801179193Sjb * We know that the provider is still valid since we incremented the 1802179193Sjb * creation reference count. If someone tried to clean up this provider 1803179193Sjb * while we were using it (e.g. because the process called exec(2) or 1804179193Sjb * exit(2)), take note of that and try to clean it up now. 1805179193Sjb */ 1806179193Sjb mutex_enter(&provider->ftp_mtx); 1807179193Sjb provider->ftp_ccount--; 1808179193Sjb whack = provider->ftp_retired; 1809179193Sjb mutex_exit(&provider->ftp_mtx); 1810179193Sjb 1811179193Sjb if (whack) 1812179193Sjb fasttrap_pid_cleanup(); 1813179193Sjb 1814179193Sjb return (0); 1815179193Sjb 1816179193Sjbno_mem: 1817179193Sjb /* 1818179193Sjb * If we've exhausted the allowable resources, we'll try to remove 1819179193Sjb * this provider to free some up. This is to cover the case where 1820179193Sjb * the user has accidentally created many more probes than was 1821179193Sjb * intended (e.g. pid123:::). 1822179193Sjb */ 1823179193Sjb mutex_exit(&provider->ftp_cmtx); 1824179193Sjb mutex_enter(&provider->ftp_mtx); 1825179193Sjb provider->ftp_ccount--; 1826179193Sjb provider->ftp_marked = 1; 1827179193Sjb mutex_exit(&provider->ftp_mtx); 1828179193Sjb 1829179193Sjb fasttrap_pid_cleanup(); 1830179193Sjb 1831179193Sjb return (ENOMEM); 1832179193Sjb} 1833179193Sjb 1834179193Sjb/*ARGSUSED*/ 1835179193Sjbstatic void * 1836179193Sjbfasttrap_meta_provide(void *arg, dtrace_helper_provdesc_t *dhpv, pid_t pid) 1837179193Sjb{ 1838179193Sjb fasttrap_provider_t *provider; 1839179193Sjb 1840179193Sjb /* 1841179193Sjb * A 32-bit unsigned integer (like a pid for example) can be 1842179193Sjb * expressed in 10 or fewer decimal digits. Make sure that we'll 1843179193Sjb * have enough space for the provider name. 1844179193Sjb */ 1845179193Sjb if (strlen(dhpv->dthpv_provname) + 10 >= 1846179193Sjb sizeof (provider->ftp_name)) { 1847211738Srpaulo printf("failed to instantiate provider %s: " 1848179193Sjb "name too long to accomodate pid", dhpv->dthpv_provname); 1849179193Sjb return (NULL); 1850179193Sjb } 1851179193Sjb 1852179193Sjb /* 1853179193Sjb * Don't let folks spoof the true pid provider. 1854179193Sjb */ 1855179193Sjb if (strcmp(dhpv->dthpv_provname, FASTTRAP_PID_NAME) == 0) { 1856211738Srpaulo printf("failed to instantiate provider %s: " 1857179193Sjb "%s is an invalid name", dhpv->dthpv_provname, 1858179193Sjb FASTTRAP_PID_NAME); 1859179193Sjb return (NULL); 1860179193Sjb } 1861179193Sjb 1862179193Sjb /* 1863179193Sjb * The highest stability class that fasttrap supports is ISA; cap 1864179193Sjb * the stability of the new provider accordingly. 1865179193Sjb */ 1866179193Sjb if (dhpv->dthpv_pattr.dtpa_provider.dtat_class > DTRACE_CLASS_ISA) 1867179193Sjb dhpv->dthpv_pattr.dtpa_provider.dtat_class = DTRACE_CLASS_ISA; 1868179193Sjb if (dhpv->dthpv_pattr.dtpa_mod.dtat_class > DTRACE_CLASS_ISA) 1869179193Sjb dhpv->dthpv_pattr.dtpa_mod.dtat_class = DTRACE_CLASS_ISA; 1870179193Sjb if (dhpv->dthpv_pattr.dtpa_func.dtat_class > DTRACE_CLASS_ISA) 1871179193Sjb dhpv->dthpv_pattr.dtpa_func.dtat_class = DTRACE_CLASS_ISA; 1872179193Sjb if (dhpv->dthpv_pattr.dtpa_name.dtat_class > DTRACE_CLASS_ISA) 1873179193Sjb dhpv->dthpv_pattr.dtpa_name.dtat_class = DTRACE_CLASS_ISA; 1874179193Sjb if (dhpv->dthpv_pattr.dtpa_args.dtat_class > DTRACE_CLASS_ISA) 1875179193Sjb dhpv->dthpv_pattr.dtpa_args.dtat_class = DTRACE_CLASS_ISA; 1876179193Sjb 1877179193Sjb if ((provider = fasttrap_provider_lookup(pid, dhpv->dthpv_provname, 1878179193Sjb &dhpv->dthpv_pattr)) == NULL) { 1879211738Srpaulo printf("failed to instantiate provider %s for " 1880179193Sjb "process %u", dhpv->dthpv_provname, (uint_t)pid); 1881179193Sjb return (NULL); 1882179193Sjb } 1883179193Sjb 1884179193Sjb /* 1885179193Sjb * Up the meta provider count so this provider isn't removed until 1886179193Sjb * the meta provider has been told to remove it. 1887179193Sjb */ 1888179193Sjb provider->ftp_mcount++; 1889179193Sjb 1890179193Sjb mutex_exit(&provider->ftp_mtx); 1891179193Sjb 1892179193Sjb return (provider); 1893179193Sjb} 1894179193Sjb 1895179193Sjb/*ARGSUSED*/ 1896179193Sjbstatic void 1897179193Sjbfasttrap_meta_create_probe(void *arg, void *parg, 1898179193Sjb dtrace_helper_probedesc_t *dhpb) 1899179193Sjb{ 1900179193Sjb fasttrap_provider_t *provider = parg; 1901179193Sjb fasttrap_probe_t *pp; 1902179193Sjb fasttrap_tracepoint_t *tp; 1903179193Sjb int i, j; 1904179193Sjb uint32_t ntps; 1905179193Sjb 1906179193Sjb /* 1907179193Sjb * Since the meta provider count is non-zero we don't have to worry 1908179193Sjb * about this provider disappearing. 1909179193Sjb */ 1910179193Sjb ASSERT(provider->ftp_mcount > 0); 1911179193Sjb 1912179193Sjb /* 1913179193Sjb * The offsets must be unique. 1914179193Sjb */ 1915179193Sjb qsort(dhpb->dthpb_offs, dhpb->dthpb_noffs, sizeof (uint32_t), 1916179193Sjb fasttrap_uint32_cmp); 1917179193Sjb for (i = 1; i < dhpb->dthpb_noffs; i++) { 1918179193Sjb if (dhpb->dthpb_base + dhpb->dthpb_offs[i] <= 1919179193Sjb dhpb->dthpb_base + dhpb->dthpb_offs[i - 1]) 1920179193Sjb return; 1921179193Sjb } 1922179193Sjb 1923179193Sjb qsort(dhpb->dthpb_enoffs, dhpb->dthpb_nenoffs, sizeof (uint32_t), 1924179193Sjb fasttrap_uint32_cmp); 1925179193Sjb for (i = 1; i < dhpb->dthpb_nenoffs; i++) { 1926179193Sjb if (dhpb->dthpb_base + dhpb->dthpb_enoffs[i] <= 1927179193Sjb dhpb->dthpb_base + dhpb->dthpb_enoffs[i - 1]) 1928179193Sjb return; 1929179193Sjb } 1930179193Sjb 1931179193Sjb /* 1932179193Sjb * Grab the creation lock to ensure consistency between calls to 1933179193Sjb * dtrace_probe_lookup() and dtrace_probe_create() in the face of 1934179193Sjb * other threads creating probes. 1935179193Sjb */ 1936179193Sjb mutex_enter(&provider->ftp_cmtx); 1937179193Sjb 1938179193Sjb if (dtrace_probe_lookup(provider->ftp_provid, dhpb->dthpb_mod, 1939179193Sjb dhpb->dthpb_func, dhpb->dthpb_name) != 0) { 1940179193Sjb mutex_exit(&provider->ftp_cmtx); 1941179193Sjb return; 1942179193Sjb } 1943179193Sjb 1944179193Sjb ntps = dhpb->dthpb_noffs + dhpb->dthpb_nenoffs; 1945179193Sjb ASSERT(ntps > 0); 1946179193Sjb 1947179193Sjb atomic_add_32(&fasttrap_total, ntps); 1948179193Sjb 1949179193Sjb if (fasttrap_total > fasttrap_max) { 1950179193Sjb atomic_add_32(&fasttrap_total, -ntps); 1951179193Sjb mutex_exit(&provider->ftp_cmtx); 1952179193Sjb return; 1953179193Sjb } 1954179193Sjb 1955179193Sjb pp = kmem_zalloc(offsetof(fasttrap_probe_t, ftp_tps[ntps]), KM_SLEEP); 1956179193Sjb 1957179193Sjb pp->ftp_prov = provider; 1958179193Sjb pp->ftp_pid = provider->ftp_pid; 1959179193Sjb pp->ftp_ntps = ntps; 1960179193Sjb pp->ftp_nargs = dhpb->dthpb_xargc; 1961179193Sjb pp->ftp_xtypes = dhpb->dthpb_xtypes; 1962179193Sjb pp->ftp_ntypes = dhpb->dthpb_ntypes; 1963179193Sjb 1964179193Sjb /* 1965179193Sjb * First create a tracepoint for each actual point of interest. 1966179193Sjb */ 1967179193Sjb for (i = 0; i < dhpb->dthpb_noffs; i++) { 1968179193Sjb tp = kmem_zalloc(sizeof (fasttrap_tracepoint_t), KM_SLEEP); 1969179193Sjb 1970179193Sjb tp->ftt_proc = provider->ftp_proc; 1971179193Sjb tp->ftt_pc = dhpb->dthpb_base + dhpb->dthpb_offs[i]; 1972179193Sjb tp->ftt_pid = provider->ftp_pid; 1973179193Sjb 1974179193Sjb pp->ftp_tps[i].fit_tp = tp; 1975179193Sjb pp->ftp_tps[i].fit_id.fti_probe = pp; 1976179193Sjb#ifdef __sparc 1977179193Sjb pp->ftp_tps[i].fit_id.fti_ptype = DTFTP_POST_OFFSETS; 1978179193Sjb#else 1979179193Sjb pp->ftp_tps[i].fit_id.fti_ptype = DTFTP_OFFSETS; 1980179193Sjb#endif 1981179193Sjb } 1982179193Sjb 1983179193Sjb /* 1984179193Sjb * Then create a tracepoint for each is-enabled point. 1985179193Sjb */ 1986179193Sjb for (j = 0; i < ntps; i++, j++) { 1987179193Sjb tp = kmem_zalloc(sizeof (fasttrap_tracepoint_t), KM_SLEEP); 1988179193Sjb 1989179193Sjb tp->ftt_proc = provider->ftp_proc; 1990179193Sjb tp->ftt_pc = dhpb->dthpb_base + dhpb->dthpb_enoffs[j]; 1991179193Sjb tp->ftt_pid = provider->ftp_pid; 1992179193Sjb 1993179193Sjb pp->ftp_tps[i].fit_tp = tp; 1994179193Sjb pp->ftp_tps[i].fit_id.fti_probe = pp; 1995179193Sjb pp->ftp_tps[i].fit_id.fti_ptype = DTFTP_IS_ENABLED; 1996179193Sjb } 1997179193Sjb 1998179193Sjb /* 1999179193Sjb * If the arguments are shuffled around we set the argument remapping 2000179193Sjb * table. Later, when the probe fires, we only remap the arguments 2001179193Sjb * if the table is non-NULL. 2002179193Sjb */ 2003179193Sjb for (i = 0; i < dhpb->dthpb_xargc; i++) { 2004179193Sjb if (dhpb->dthpb_args[i] != i) { 2005179193Sjb pp->ftp_argmap = dhpb->dthpb_args; 2006179193Sjb break; 2007179193Sjb } 2008179193Sjb } 2009179193Sjb 2010179193Sjb /* 2011179193Sjb * The probe is fully constructed -- register it with DTrace. 2012179193Sjb */ 2013179193Sjb pp->ftp_id = dtrace_probe_create(provider->ftp_provid, dhpb->dthpb_mod, 2014179193Sjb dhpb->dthpb_func, dhpb->dthpb_name, FASTTRAP_OFFSET_AFRAMES, pp); 2015179193Sjb 2016179193Sjb mutex_exit(&provider->ftp_cmtx); 2017179193Sjb} 2018179193Sjb 2019179193Sjb/*ARGSUSED*/ 2020179193Sjbstatic void 2021179193Sjbfasttrap_meta_remove(void *arg, dtrace_helper_provdesc_t *dhpv, pid_t pid) 2022179193Sjb{ 2023179193Sjb /* 2024179193Sjb * Clean up the USDT provider. There may be active consumers of the 2025179193Sjb * provider busy adding probes, no damage will actually befall the 2026179193Sjb * provider until that count has dropped to zero. This just puts 2027179193Sjb * the provider on death row. 2028179193Sjb */ 2029179193Sjb fasttrap_provider_retire(pid, dhpv->dthpv_provname, 1); 2030179193Sjb} 2031179193Sjb 2032179193Sjbstatic dtrace_mops_t fasttrap_mops = { 2033179193Sjb fasttrap_meta_create_probe, 2034179193Sjb fasttrap_meta_provide, 2035179193Sjb fasttrap_meta_remove 2036179193Sjb}; 2037179193Sjb 2038179193Sjb/*ARGSUSED*/ 2039179193Sjbstatic int 2040211738Srpaulofasttrap_open(struct cdev *dev __unused, int oflags __unused, 2041211738Srpaulo int devtype __unused, struct thread *td __unused) 2042179193Sjb{ 2043179193Sjb return (0); 2044179193Sjb} 2045179193Sjb 2046179193Sjb/*ARGSUSED*/ 2047179193Sjbstatic int 2048211738Srpaulofasttrap_ioctl(struct cdev *dev, u_long cmd, caddr_t arg, int fflag, 2049211738Srpaulo struct thread *td) 2050179193Sjb{ 2051211738Srpaulo#ifdef notyet 2052211738Srpaulo struct kinfo_proc kp; 2053211738Srpaulo const cred_t *cr = td->td_ucred; 2054211738Srpaulo#endif 2055179193Sjb if (!dtrace_attached()) 2056179193Sjb return (EAGAIN); 2057179193Sjb 2058179193Sjb if (cmd == FASTTRAPIOC_MAKEPROBE) { 2059262049Savg fasttrap_probe_spec_t *uprobe = *(fasttrap_probe_spec_t **)arg; 2060179193Sjb fasttrap_probe_spec_t *probe; 2061179193Sjb uint64_t noffs; 2062179193Sjb size_t size; 2063179193Sjb int ret; 2064179193Sjb char *c; 2065179193Sjb 2066179193Sjb if (copyin(&uprobe->ftps_noffs, &noffs, 2067179193Sjb sizeof (uprobe->ftps_noffs))) 2068179193Sjb return (EFAULT); 2069179193Sjb 2070179193Sjb /* 2071179193Sjb * Probes must have at least one tracepoint. 2072179193Sjb */ 2073179193Sjb if (noffs == 0) 2074179193Sjb return (EINVAL); 2075179193Sjb 2076179193Sjb size = sizeof (fasttrap_probe_spec_t) + 2077179193Sjb sizeof (probe->ftps_offs[0]) * (noffs - 1); 2078179193Sjb 2079179193Sjb if (size > 1024 * 1024) 2080179193Sjb return (ENOMEM); 2081179193Sjb 2082179193Sjb probe = kmem_alloc(size, KM_SLEEP); 2083179193Sjb 2084179193Sjb if (copyin(uprobe, probe, size) != 0) { 2085179193Sjb kmem_free(probe, size); 2086179193Sjb return (EFAULT); 2087179193Sjb } 2088179193Sjb 2089179193Sjb /* 2090179193Sjb * Verify that the function and module strings contain no 2091179193Sjb * funny characters. 2092179193Sjb */ 2093179193Sjb for (c = &probe->ftps_func[0]; *c != '\0'; c++) { 2094179193Sjb if (*c < 0x20 || 0x7f <= *c) { 2095179193Sjb ret = EINVAL; 2096179193Sjb goto err; 2097179193Sjb } 2098179193Sjb } 2099179193Sjb 2100179193Sjb for (c = &probe->ftps_mod[0]; *c != '\0'; c++) { 2101179193Sjb if (*c < 0x20 || 0x7f <= *c) { 2102179193Sjb ret = EINVAL; 2103179193Sjb goto err; 2104179193Sjb } 2105179193Sjb } 2106179193Sjb 2107211738Srpaulo#ifdef notyet 2108179193Sjb if (!PRIV_POLICY_CHOICE(cr, PRIV_ALL, B_FALSE)) { 2109179193Sjb proc_t *p; 2110179193Sjb pid_t pid = probe->ftps_pid; 2111179193Sjb 2112211738Srpaulo#if defined(sun) 2113179193Sjb mutex_enter(&pidlock); 2114211738Srpaulo#endif 2115179193Sjb /* 2116179193Sjb * Report an error if the process doesn't exist 2117179193Sjb * or is actively being birthed. 2118179193Sjb */ 2119211738Srpaulo p = pfind(pid); 2120211738Srpaulo if (p) 2121211738Srpaulo fill_kinfo_proc(p, &kp); 2122211738Srpaulo if (p == NULL || kp.ki_stat == SIDL) { 2123211738Srpaulo#if defined(sun) 2124179193Sjb mutex_exit(&pidlock); 2125211738Srpaulo#endif 2126179193Sjb return (ESRCH); 2127179193Sjb } 2128211738Srpaulo#if defined(sun) 2129179193Sjb mutex_enter(&p->p_lock); 2130179193Sjb mutex_exit(&pidlock); 2131211738Srpaulo#else 2132211738Srpaulo PROC_LOCK_ASSERT(p, MA_OWNED); 2133211738Srpaulo#endif 2134179193Sjb 2135211738Srpaulo#ifdef notyet 2136179193Sjb if ((ret = priv_proc_cred_perm(cr, p, NULL, 2137179193Sjb VREAD | VWRITE)) != 0) { 2138211738Srpaulo#if defined(sun) 2139179193Sjb mutex_exit(&p->p_lock); 2140211738Srpaulo#else 2141211738Srpaulo PROC_UNLOCK(p); 2142211738Srpaulo#endif 2143179193Sjb return (ret); 2144179193Sjb } 2145211738Srpaulo#endif /* notyet */ 2146211738Srpaulo#if defined(sun) 2147179193Sjb mutex_exit(&p->p_lock); 2148211738Srpaulo#else 2149211738Srpaulo PROC_UNLOCK(p); 2150211738Srpaulo#endif 2151179193Sjb } 2152211738Srpaulo#endif /* notyet */ 2153179193Sjb 2154179193Sjb ret = fasttrap_add_probe(probe); 2155179193Sjberr: 2156179193Sjb kmem_free(probe, size); 2157179193Sjb 2158179193Sjb return (ret); 2159179193Sjb 2160179193Sjb } else if (cmd == FASTTRAPIOC_GETINSTR) { 2161179193Sjb fasttrap_instr_query_t instr; 2162179193Sjb fasttrap_tracepoint_t *tp; 2163179193Sjb uint_t index; 2164211738Srpaulo#if defined(sun) 2165179193Sjb int ret; 2166211738Srpaulo#endif 2167179193Sjb 2168211738Srpaulo#if defined(sun) 2169179193Sjb if (copyin((void *)arg, &instr, sizeof (instr)) != 0) 2170179193Sjb return (EFAULT); 2171211738Srpaulo#endif 2172179193Sjb 2173211738Srpaulo#ifdef notyet 2174179193Sjb if (!PRIV_POLICY_CHOICE(cr, PRIV_ALL, B_FALSE)) { 2175179193Sjb proc_t *p; 2176179193Sjb pid_t pid = instr.ftiq_pid; 2177179193Sjb 2178211738Srpaulo#if defined(sun) 2179179193Sjb mutex_enter(&pidlock); 2180211738Srpaulo#endif 2181179193Sjb /* 2182179193Sjb * Report an error if the process doesn't exist 2183179193Sjb * or is actively being birthed. 2184179193Sjb */ 2185211738Srpaulo p = pfind(pid); 2186211738Srpaulo if (p) 2187211738Srpaulo fill_kinfo_proc(p, &kp); 2188211738Srpaulo if (p == NULL || kp.ki_stat == SIDL) { 2189211738Srpaulo#if defined(sun) 2190179193Sjb mutex_exit(&pidlock); 2191211738Srpaulo#endif 2192179193Sjb return (ESRCH); 2193179193Sjb } 2194211738Srpaulo#if defined(sun) 2195179193Sjb mutex_enter(&p->p_lock); 2196179193Sjb mutex_exit(&pidlock); 2197211738Srpaulo#else 2198211738Srpaulo PROC_LOCK_ASSERT(p, MA_OWNED); 2199211738Srpaulo#endif 2200179193Sjb 2201211738Srpaulo#ifdef notyet 2202179193Sjb if ((ret = priv_proc_cred_perm(cr, p, NULL, 2203179193Sjb VREAD)) != 0) { 2204211738Srpaulo#if defined(sun) 2205179193Sjb mutex_exit(&p->p_lock); 2206211738Srpaulo#else 2207211738Srpaulo PROC_UNLOCK(p); 2208211738Srpaulo#endif 2209179193Sjb return (ret); 2210179193Sjb } 2211211738Srpaulo#endif /* notyet */ 2212179193Sjb 2213211738Srpaulo#if defined(sun) 2214179193Sjb mutex_exit(&p->p_lock); 2215211738Srpaulo#else 2216211738Srpaulo PROC_UNLOCK(p); 2217211738Srpaulo#endif 2218179193Sjb } 2219211738Srpaulo#endif /* notyet */ 2220179193Sjb 2221179193Sjb index = FASTTRAP_TPOINTS_INDEX(instr.ftiq_pid, instr.ftiq_pc); 2222179193Sjb 2223179193Sjb mutex_enter(&fasttrap_tpoints.fth_table[index].ftb_mtx); 2224179193Sjb tp = fasttrap_tpoints.fth_table[index].ftb_data; 2225179193Sjb while (tp != NULL) { 2226179193Sjb if (instr.ftiq_pid == tp->ftt_pid && 2227179193Sjb instr.ftiq_pc == tp->ftt_pc && 2228179193Sjb tp->ftt_proc->ftpc_acount != 0) 2229179193Sjb break; 2230179193Sjb 2231179193Sjb tp = tp->ftt_next; 2232179193Sjb } 2233179193Sjb 2234179193Sjb if (tp == NULL) { 2235179193Sjb mutex_exit(&fasttrap_tpoints.fth_table[index].ftb_mtx); 2236179193Sjb return (ENOENT); 2237179193Sjb } 2238179193Sjb 2239179193Sjb bcopy(&tp->ftt_instr, &instr.ftiq_instr, 2240179193Sjb sizeof (instr.ftiq_instr)); 2241179193Sjb mutex_exit(&fasttrap_tpoints.fth_table[index].ftb_mtx); 2242179193Sjb 2243179193Sjb if (copyout(&instr, (void *)arg, sizeof (instr)) != 0) 2244179193Sjb return (EFAULT); 2245179193Sjb 2246179193Sjb return (0); 2247179193Sjb } 2248179193Sjb 2249179193Sjb return (EINVAL); 2250179193Sjb} 2251179193Sjb 2252179193Sjbstatic int 2253211738Srpaulofasttrap_load(void) 2254179193Sjb{ 2255179193Sjb ulong_t nent; 2256255749Smarkj int i, ret; 2257179193Sjb 2258211738Srpaulo /* Create the /dev/dtrace/fasttrap entry. */ 2259211738Srpaulo fasttrap_cdev = make_dev(&fasttrap_cdevsw, 0, UID_ROOT, GID_WHEEL, 0600, 2260211738Srpaulo "dtrace/fasttrap"); 2261179193Sjb 2262211738Srpaulo mtx_init(&fasttrap_cleanup_mtx, "fasttrap clean", "dtrace", MTX_DEF); 2263211738Srpaulo mutex_init(&fasttrap_count_mtx, "fasttrap count mtx", MUTEX_DEFAULT, 2264211738Srpaulo NULL); 2265179193Sjb 2266211738Srpaulo#if defined(sun) 2267179193Sjb fasttrap_max = ddi_getprop(DDI_DEV_T_ANY, devi, DDI_PROP_DONTPASS, 2268179193Sjb "fasttrap-max-probes", FASTTRAP_MAX_DEFAULT); 2269211738Srpaulo#else 2270211738Srpaulo fasttrap_max = FASTTRAP_MAX_DEFAULT; 2271211738Srpaulo#endif 2272179193Sjb fasttrap_total = 0; 2273179193Sjb 2274179193Sjb /* 2275179193Sjb * Conjure up the tracepoints hashtable... 2276179193Sjb */ 2277211738Srpaulo#if defined(sun) 2278179193Sjb nent = ddi_getprop(DDI_DEV_T_ANY, devi, DDI_PROP_DONTPASS, 2279179193Sjb "fasttrap-hash-size", FASTTRAP_TPOINTS_DEFAULT_SIZE); 2280211738Srpaulo#else 2281211738Srpaulo nent = FASTTRAP_TPOINTS_DEFAULT_SIZE; 2282211738Srpaulo#endif 2283179193Sjb 2284179193Sjb if (nent == 0 || nent > 0x1000000) 2285179193Sjb nent = FASTTRAP_TPOINTS_DEFAULT_SIZE; 2286179193Sjb 2287179193Sjb if ((nent & (nent - 1)) == 0) 2288179193Sjb fasttrap_tpoints.fth_nent = nent; 2289179193Sjb else 2290179193Sjb fasttrap_tpoints.fth_nent = 1 << fasttrap_highbit(nent); 2291179193Sjb ASSERT(fasttrap_tpoints.fth_nent > 0); 2292179193Sjb fasttrap_tpoints.fth_mask = fasttrap_tpoints.fth_nent - 1; 2293179193Sjb fasttrap_tpoints.fth_table = kmem_zalloc(fasttrap_tpoints.fth_nent * 2294179193Sjb sizeof (fasttrap_bucket_t), KM_SLEEP); 2295211738Srpaulo#if !defined(sun) 2296211738Srpaulo for (i = 0; i < fasttrap_tpoints.fth_nent; i++) 2297211738Srpaulo mutex_init(&fasttrap_tpoints.fth_table[i].ftb_mtx, 2298211738Srpaulo "tracepoints bucket mtx", MUTEX_DEFAULT, NULL); 2299211738Srpaulo#endif 2300179193Sjb 2301179193Sjb /* 2302179193Sjb * ... and the providers hash table... 2303179193Sjb */ 2304179193Sjb nent = FASTTRAP_PROVIDERS_DEFAULT_SIZE; 2305179193Sjb if ((nent & (nent - 1)) == 0) 2306179193Sjb fasttrap_provs.fth_nent = nent; 2307179193Sjb else 2308179193Sjb fasttrap_provs.fth_nent = 1 << fasttrap_highbit(nent); 2309179193Sjb ASSERT(fasttrap_provs.fth_nent > 0); 2310179193Sjb fasttrap_provs.fth_mask = fasttrap_provs.fth_nent - 1; 2311179193Sjb fasttrap_provs.fth_table = kmem_zalloc(fasttrap_provs.fth_nent * 2312179193Sjb sizeof (fasttrap_bucket_t), KM_SLEEP); 2313211738Srpaulo#if !defined(sun) 2314211738Srpaulo for (i = 0; i < fasttrap_provs.fth_nent; i++) 2315211738Srpaulo mutex_init(&fasttrap_provs.fth_table[i].ftb_mtx, 2316211738Srpaulo "providers bucket mtx", MUTEX_DEFAULT, NULL); 2317211738Srpaulo#endif 2318179193Sjb 2319259519Sasomers ret = kproc_create(fasttrap_pid_cleanup_cb, NULL, 2320259519Sasomers &fasttrap_cleanup_proc, 0, 0, "ftcleanup"); 2321259519Sasomers if (ret != 0) { 2322259519Sasomers destroy_dev(fasttrap_cdev); 2323259519Sasomers#if !defined(sun) 2324259519Sasomers for (i = 0; i < fasttrap_provs.fth_nent; i++) 2325259519Sasomers mutex_destroy(&fasttrap_provs.fth_table[i].ftb_mtx); 2326259519Sasomers for (i = 0; i < fasttrap_tpoints.fth_nent; i++) 2327259519Sasomers mutex_destroy(&fasttrap_tpoints.fth_table[i].ftb_mtx); 2328259519Sasomers#endif 2329259519Sasomers kmem_free(fasttrap_provs.fth_table, fasttrap_provs.fth_nent * 2330259519Sasomers sizeof (fasttrap_bucket_t)); 2331259519Sasomers mtx_destroy(&fasttrap_cleanup_mtx); 2332259519Sasomers mutex_destroy(&fasttrap_count_mtx); 2333259519Sasomers return (ret); 2334259519Sasomers } 2335259519Sasomers 2336259519Sasomers 2337179193Sjb /* 2338179193Sjb * ... and the procs hash table. 2339179193Sjb */ 2340179193Sjb nent = FASTTRAP_PROCS_DEFAULT_SIZE; 2341179193Sjb if ((nent & (nent - 1)) == 0) 2342179193Sjb fasttrap_procs.fth_nent = nent; 2343179193Sjb else 2344179193Sjb fasttrap_procs.fth_nent = 1 << fasttrap_highbit(nent); 2345179193Sjb ASSERT(fasttrap_procs.fth_nent > 0); 2346179193Sjb fasttrap_procs.fth_mask = fasttrap_procs.fth_nent - 1; 2347179193Sjb fasttrap_procs.fth_table = kmem_zalloc(fasttrap_procs.fth_nent * 2348179193Sjb sizeof (fasttrap_bucket_t), KM_SLEEP); 2349211738Srpaulo#if !defined(sun) 2350211738Srpaulo for (i = 0; i < fasttrap_procs.fth_nent; i++) 2351211738Srpaulo mutex_init(&fasttrap_procs.fth_table[i].ftb_mtx, 2352211738Srpaulo "processes bucket mtx", MUTEX_DEFAULT, NULL); 2353211925Srpaulo 2354211925Srpaulo CPU_FOREACH(i) { 2355211925Srpaulo mutex_init(&fasttrap_cpuc_pid_lock[i], "fasttrap barrier", 2356211925Srpaulo MUTEX_DEFAULT, NULL); 2357211925Srpaulo } 2358211738Srpaulo#endif 2359179193Sjb 2360253394Savg /* 2361253394Savg * Install our hooks into fork(2), exec(2), and exit(2). 2362253394Savg */ 2363253394Savg dtrace_fasttrap_fork = &fasttrap_fork; 2364253394Savg dtrace_fasttrap_exit = &fasttrap_exec_exit; 2365253394Savg dtrace_fasttrap_exec = &fasttrap_exec_exit; 2366253394Savg 2367179193Sjb (void) dtrace_meta_register("fasttrap", &fasttrap_mops, NULL, 2368179193Sjb &fasttrap_meta_id); 2369179193Sjb 2370211738Srpaulo return (0); 2371179193Sjb} 2372179193Sjb 2373179193Sjbstatic int 2374211738Srpaulofasttrap_unload(void) 2375179193Sjb{ 2376179193Sjb int i, fail = 0; 2377179193Sjb 2378179193Sjb /* 2379179193Sjb * Unregister the meta-provider to make sure no new fasttrap- 2380179193Sjb * managed providers come along while we're trying to close up 2381179193Sjb * shop. If we fail to detach, we'll need to re-register as a 2382179193Sjb * meta-provider. We can fail to unregister as a meta-provider 2383179193Sjb * if providers we manage still exist. 2384179193Sjb */ 2385179193Sjb if (fasttrap_meta_id != DTRACE_METAPROVNONE && 2386179193Sjb dtrace_meta_unregister(fasttrap_meta_id) != 0) 2387211738Srpaulo return (-1); 2388179193Sjb 2389179193Sjb /* 2390179193Sjb * Iterate over all of our providers. If there's still a process 2391179193Sjb * that corresponds to that pid, fail to detach. 2392179193Sjb */ 2393179193Sjb for (i = 0; i < fasttrap_provs.fth_nent; i++) { 2394179193Sjb fasttrap_provider_t **fpp, *fp; 2395179193Sjb fasttrap_bucket_t *bucket = &fasttrap_provs.fth_table[i]; 2396179193Sjb 2397179193Sjb mutex_enter(&bucket->ftb_mtx); 2398179193Sjb fpp = (fasttrap_provider_t **)&bucket->ftb_data; 2399179193Sjb while ((fp = *fpp) != NULL) { 2400179193Sjb /* 2401179193Sjb * Acquire and release the lock as a simple way of 2402179193Sjb * waiting for any other consumer to finish with 2403179193Sjb * this provider. A thread must first acquire the 2404179193Sjb * bucket lock so there's no chance of another thread 2405179193Sjb * blocking on the provider's lock. 2406179193Sjb */ 2407179193Sjb mutex_enter(&fp->ftp_mtx); 2408179193Sjb mutex_exit(&fp->ftp_mtx); 2409179193Sjb 2410179193Sjb if (dtrace_unregister(fp->ftp_provid) != 0) { 2411179193Sjb fail = 1; 2412179193Sjb fpp = &fp->ftp_next; 2413179193Sjb } else { 2414179193Sjb *fpp = fp->ftp_next; 2415179193Sjb fasttrap_provider_free(fp); 2416179193Sjb } 2417179193Sjb } 2418179193Sjb 2419179193Sjb mutex_exit(&bucket->ftb_mtx); 2420179193Sjb } 2421179193Sjb 2422179193Sjb if (fail) { 2423179193Sjb (void) dtrace_meta_register("fasttrap", &fasttrap_mops, NULL, 2424179193Sjb &fasttrap_meta_id); 2425179193Sjb 2426211738Srpaulo return (-1); 2427179193Sjb } 2428179193Sjb 2429259519Sasomers /* 2430259519Sasomers * Stop new processes from entering these hooks now, before the 2431259519Sasomers * fasttrap_cleanup thread runs. That way all processes will hopefully 2432259519Sasomers * be out of these hooks before we free fasttrap_provs.fth_table 2433259519Sasomers */ 2434259519Sasomers ASSERT(dtrace_fasttrap_fork == &fasttrap_fork); 2435259519Sasomers dtrace_fasttrap_fork = NULL; 2436259519Sasomers 2437259519Sasomers ASSERT(dtrace_fasttrap_exec == &fasttrap_exec_exit); 2438259519Sasomers dtrace_fasttrap_exec = NULL; 2439259519Sasomers 2440259519Sasomers ASSERT(dtrace_fasttrap_exit == &fasttrap_exec_exit); 2441259519Sasomers dtrace_fasttrap_exit = NULL; 2442259519Sasomers 2443255749Smarkj mtx_lock(&fasttrap_cleanup_mtx); 2444255749Smarkj fasttrap_cleanup_drain = 1; 2445255749Smarkj /* Wait for the cleanup thread to finish up and signal us. */ 2446255749Smarkj wakeup(&fasttrap_cleanup_cv); 2447255749Smarkj mtx_sleep(&fasttrap_cleanup_drain, &fasttrap_cleanup_mtx, 0, "ftcld", 2448255749Smarkj 0); 2449255749Smarkj fasttrap_cleanup_proc = NULL; 2450255749Smarkj mtx_destroy(&fasttrap_cleanup_mtx); 2451255749Smarkj 2452179193Sjb#ifdef DEBUG 2453179193Sjb mutex_enter(&fasttrap_count_mtx); 2454179193Sjb ASSERT(fasttrap_pid_count == 0); 2455179193Sjb mutex_exit(&fasttrap_count_mtx); 2456179193Sjb#endif 2457179193Sjb 2458259519Sasomers#if !defined(sun) 2459259519Sasomers for (i = 0; i < fasttrap_tpoints.fth_nent; i++) 2460259519Sasomers mutex_destroy(&fasttrap_tpoints.fth_table[i].ftb_mtx); 2461259519Sasomers for (i = 0; i < fasttrap_provs.fth_nent; i++) 2462259519Sasomers mutex_destroy(&fasttrap_provs.fth_table[i].ftb_mtx); 2463259519Sasomers for (i = 0; i < fasttrap_procs.fth_nent; i++) 2464259519Sasomers mutex_destroy(&fasttrap_procs.fth_table[i].ftb_mtx); 2465259519Sasomers#endif 2466179193Sjb kmem_free(fasttrap_tpoints.fth_table, 2467179193Sjb fasttrap_tpoints.fth_nent * sizeof (fasttrap_bucket_t)); 2468179193Sjb fasttrap_tpoints.fth_nent = 0; 2469179193Sjb 2470179193Sjb kmem_free(fasttrap_provs.fth_table, 2471179193Sjb fasttrap_provs.fth_nent * sizeof (fasttrap_bucket_t)); 2472179193Sjb fasttrap_provs.fth_nent = 0; 2473179193Sjb 2474179193Sjb kmem_free(fasttrap_procs.fth_table, 2475179193Sjb fasttrap_procs.fth_nent * sizeof (fasttrap_bucket_t)); 2476179193Sjb fasttrap_procs.fth_nent = 0; 2477179193Sjb 2478211738Srpaulo#if !defined(sun) 2479211738Srpaulo destroy_dev(fasttrap_cdev); 2480211738Srpaulo mutex_destroy(&fasttrap_count_mtx); 2481211925Srpaulo CPU_FOREACH(i) { 2482211925Srpaulo mutex_destroy(&fasttrap_cpuc_pid_lock[i]); 2483211925Srpaulo } 2484211738Srpaulo#endif 2485179193Sjb 2486211738Srpaulo return (0); 2487179193Sjb} 2488179193Sjb 2489211738Srpaulo/* ARGSUSED */ 2490211738Srpaulostatic int 2491211738Srpaulofasttrap_modevent(module_t mod __unused, int type, void *data __unused) 2492211738Srpaulo{ 2493211738Srpaulo int error = 0; 2494179193Sjb 2495211738Srpaulo switch (type) { 2496211738Srpaulo case MOD_LOAD: 2497211738Srpaulo break; 2498179193Sjb 2499211738Srpaulo case MOD_UNLOAD: 2500211738Srpaulo break; 2501179193Sjb 2502211738Srpaulo case MOD_SHUTDOWN: 2503211738Srpaulo break; 2504179193Sjb 2505211738Srpaulo default: 2506211738Srpaulo error = EOPNOTSUPP; 2507211738Srpaulo break; 2508211738Srpaulo } 2509211738Srpaulo return (error); 2510179193Sjb} 2511179193Sjb 2512211738SrpauloSYSINIT(fasttrap_load, SI_SUB_DTRACE_PROVIDER, SI_ORDER_ANY, fasttrap_load, 2513211738Srpaulo NULL); 2514211738SrpauloSYSUNINIT(fasttrap_unload, SI_SUB_DTRACE_PROVIDER, SI_ORDER_ANY, 2515211738Srpaulo fasttrap_unload, NULL); 2516211738Srpaulo 2517211738SrpauloDEV_MODULE(fasttrap, fasttrap_modevent, NULL); 2518211738SrpauloMODULE_VERSION(fasttrap, 1); 2519211738SrpauloMODULE_DEPEND(fasttrap, dtrace, 1, 1, 1); 2520211738SrpauloMODULE_DEPEND(fasttrap, opensolaris, 1, 1, 1); 2521