fasttrap.c revision 179198
1179193Sjb/* 2179193Sjb * CDDL HEADER START 3179193Sjb * 4179193Sjb * The contents of this file are subject to the terms of the 5179193Sjb * Common Development and Distribution License (the "License"). 6179193Sjb * You may not use this file except in compliance with the License. 7179193Sjb * 8179193Sjb * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9179193Sjb * or http://www.opensolaris.org/os/licensing. 10179193Sjb * See the License for the specific language governing permissions 11179193Sjb * and limitations under the License. 12179193Sjb * 13179193Sjb * When distributing Covered Code, include this CDDL HEADER in each 14179193Sjb * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15179193Sjb * If applicable, add the following below this CDDL HEADER, with the 16179193Sjb * fields enclosed by brackets "[]" replaced with your own identifying 17179193Sjb * information: Portions Copyright [yyyy] [name of copyright owner] 18179193Sjb * 19179193Sjb * CDDL HEADER END 20179193Sjb */ 21179193Sjb 22179193Sjb/* 23179198Sjb * Copyright 2008 Sun Microsystems, Inc. All rights reserved. 24179193Sjb * Use is subject to license terms. 25179193Sjb */ 26179193Sjb 27179193Sjb#pragma ident "%Z%%M% %I% %E% SMI" 28179193Sjb 29179193Sjb#include <sys/atomic.h> 30179193Sjb#include <sys/errno.h> 31179193Sjb#include <sys/stat.h> 32179193Sjb#include <sys/modctl.h> 33179193Sjb#include <sys/conf.h> 34179193Sjb#include <sys/systm.h> 35179193Sjb#include <sys/ddi.h> 36179193Sjb#include <sys/sunddi.h> 37179193Sjb#include <sys/cpuvar.h> 38179193Sjb#include <sys/kmem.h> 39179193Sjb#include <sys/strsubr.h> 40179193Sjb#include <sys/fasttrap.h> 41179193Sjb#include <sys/fasttrap_impl.h> 42179193Sjb#include <sys/fasttrap_isa.h> 43179193Sjb#include <sys/dtrace.h> 44179193Sjb#include <sys/dtrace_impl.h> 45179193Sjb#include <sys/sysmacros.h> 46179193Sjb#include <sys/proc.h> 47179193Sjb#include <sys/priv.h> 48179193Sjb#include <sys/policy.h> 49179193Sjb#include <util/qsort.h> 50179193Sjb 51179193Sjb/* 52179193Sjb * User-Land Trap-Based Tracing 53179193Sjb * ---------------------------- 54179193Sjb * 55179193Sjb * The fasttrap provider allows DTrace consumers to instrument any user-level 56179193Sjb * instruction to gather data; this includes probes with semantic 57179193Sjb * signifigance like entry and return as well as simple offsets into the 58179193Sjb * function. While the specific techniques used are very ISA specific, the 59179193Sjb * methodology is generalizable to any architecture. 60179193Sjb * 61179193Sjb * 62179193Sjb * The General Methodology 63179193Sjb * ----------------------- 64179193Sjb * 65179193Sjb * With the primary goal of tracing every user-land instruction and the 66179193Sjb * limitation that we can't trust user space so don't want to rely on much 67179193Sjb * information there, we begin by replacing the instructions we want to trace 68179193Sjb * with trap instructions. Each instruction we overwrite is saved into a hash 69179193Sjb * table keyed by process ID and pc address. When we enter the kernel due to 70179193Sjb * this trap instruction, we need the effects of the replaced instruction to 71179193Sjb * appear to have occurred before we proceed with the user thread's 72179193Sjb * execution. 73179193Sjb * 74179193Sjb * Each user level thread is represented by a ulwp_t structure which is 75179193Sjb * always easily accessible through a register. The most basic way to produce 76179193Sjb * the effects of the instruction we replaced is to copy that instruction out 77179193Sjb * to a bit of scratch space reserved in the user thread's ulwp_t structure 78179193Sjb * (a sort of kernel-private thread local storage), set the PC to that 79179193Sjb * scratch space and single step. When we reenter the kernel after single 80179193Sjb * stepping the instruction we must then adjust the PC to point to what would 81179193Sjb * normally be the next instruction. Of course, special care must be taken 82179193Sjb * for branches and jumps, but these represent such a small fraction of any 83179193Sjb * instruction set that writing the code to emulate these in the kernel is 84179193Sjb * not too difficult. 85179193Sjb * 86179193Sjb * Return probes may require several tracepoints to trace every return site, 87179193Sjb * and, conversely, each tracepoint may activate several probes (the entry 88179193Sjb * and offset 0 probes, for example). To solve this muliplexing problem, 89179193Sjb * tracepoints contain lists of probes to activate and probes contain lists 90179193Sjb * of tracepoints to enable. If a probe is activated, it adds its ID to 91179193Sjb * existing tracepoints or creates new ones as necessary. 92179193Sjb * 93179193Sjb * Most probes are activated _before_ the instruction is executed, but return 94179193Sjb * probes are activated _after_ the effects of the last instruction of the 95179193Sjb * function are visible. Return probes must be fired _after_ we have 96179193Sjb * single-stepped the instruction whereas all other probes are fired 97179193Sjb * beforehand. 98179193Sjb * 99179193Sjb * 100179193Sjb * Lock Ordering 101179193Sjb * ------------- 102179193Sjb * 103179193Sjb * The lock ordering below -- both internally and with respect to the DTrace 104179193Sjb * framework -- is a little tricky and bears some explanation. Each provider 105179193Sjb * has a lock (ftp_mtx) that protects its members including reference counts 106179193Sjb * for enabled probes (ftp_rcount), consumers actively creating probes 107179193Sjb * (ftp_ccount) and USDT consumers (ftp_mcount); all three prevent a provider 108179193Sjb * from being freed. A provider is looked up by taking the bucket lock for the 109179193Sjb * provider hash table, and is returned with its lock held. The provider lock 110179193Sjb * may be taken in functions invoked by the DTrace framework, but may not be 111179193Sjb * held while calling functions in the DTrace framework. 112179193Sjb * 113179193Sjb * To ensure consistency over multiple calls to the DTrace framework, the 114179193Sjb * creation lock (ftp_cmtx) should be held. Naturally, the creation lock may 115179193Sjb * not be taken when holding the provider lock as that would create a cyclic 116179193Sjb * lock ordering. In situations where one would naturally take the provider 117179193Sjb * lock and then the creation lock, we instead up a reference count to prevent 118179193Sjb * the provider from disappearing, drop the provider lock, and acquire the 119179193Sjb * creation lock. 120179193Sjb * 121179193Sjb * Briefly: 122179193Sjb * bucket lock before provider lock 123179193Sjb * DTrace before provider lock 124179193Sjb * creation lock before DTrace 125179193Sjb * never hold the provider lock and creation lock simultaneously 126179193Sjb */ 127179193Sjb 128179193Sjbstatic dev_info_t *fasttrap_devi; 129179193Sjbstatic dtrace_meta_provider_id_t fasttrap_meta_id; 130179193Sjb 131179193Sjbstatic timeout_id_t fasttrap_timeout; 132179193Sjbstatic kmutex_t fasttrap_cleanup_mtx; 133179193Sjbstatic uint_t fasttrap_cleanup_work; 134179193Sjb 135179193Sjb/* 136179193Sjb * Generation count on modifications to the global tracepoint lookup table. 137179193Sjb */ 138179193Sjbstatic volatile uint64_t fasttrap_mod_gen; 139179193Sjb 140179193Sjb/* 141179193Sjb * When the fasttrap provider is loaded, fasttrap_max is set to either 142179193Sjb * FASTTRAP_MAX_DEFAULT or the value for fasttrap-max-probes in the 143179193Sjb * fasttrap.conf file. Each time a probe is created, fasttrap_total is 144179193Sjb * incremented by the number of tracepoints that may be associated with that 145179193Sjb * probe; fasttrap_total is capped at fasttrap_max. 146179193Sjb */ 147179193Sjb#define FASTTRAP_MAX_DEFAULT 250000 148179193Sjbstatic uint32_t fasttrap_max; 149179193Sjbstatic uint32_t fasttrap_total; 150179193Sjb 151179193Sjb 152179193Sjb#define FASTTRAP_TPOINTS_DEFAULT_SIZE 0x4000 153179193Sjb#define FASTTRAP_PROVIDERS_DEFAULT_SIZE 0x100 154179193Sjb#define FASTTRAP_PROCS_DEFAULT_SIZE 0x100 155179193Sjb 156179193Sjb#define FASTTRAP_PID_NAME "pid" 157179193Sjb 158179193Sjbfasttrap_hash_t fasttrap_tpoints; 159179193Sjbstatic fasttrap_hash_t fasttrap_provs; 160179193Sjbstatic fasttrap_hash_t fasttrap_procs; 161179193Sjb 162179193Sjbstatic uint64_t fasttrap_pid_count; /* pid ref count */ 163179193Sjbstatic kmutex_t fasttrap_count_mtx; /* lock on ref count */ 164179193Sjb 165179193Sjb#define FASTTRAP_ENABLE_FAIL 1 166179193Sjb#define FASTTRAP_ENABLE_PARTIAL 2 167179193Sjb 168179193Sjbstatic int fasttrap_tracepoint_enable(proc_t *, fasttrap_probe_t *, uint_t); 169179193Sjbstatic void fasttrap_tracepoint_disable(proc_t *, fasttrap_probe_t *, uint_t); 170179193Sjb 171179193Sjbstatic fasttrap_provider_t *fasttrap_provider_lookup(pid_t, const char *, 172179193Sjb const dtrace_pattr_t *); 173179193Sjbstatic void fasttrap_provider_retire(pid_t, const char *, int); 174179193Sjbstatic void fasttrap_provider_free(fasttrap_provider_t *); 175179193Sjb 176179193Sjbstatic fasttrap_proc_t *fasttrap_proc_lookup(pid_t); 177179193Sjbstatic void fasttrap_proc_release(fasttrap_proc_t *); 178179193Sjb 179179193Sjb#define FASTTRAP_PROVS_INDEX(pid, name) \ 180179193Sjb ((fasttrap_hash_str(name) + (pid)) & fasttrap_provs.fth_mask) 181179193Sjb 182179193Sjb#define FASTTRAP_PROCS_INDEX(pid) ((pid) & fasttrap_procs.fth_mask) 183179193Sjb 184179193Sjbstatic int 185179193Sjbfasttrap_highbit(ulong_t i) 186179193Sjb{ 187179193Sjb int h = 1; 188179193Sjb 189179193Sjb if (i == 0) 190179193Sjb return (0); 191179193Sjb#ifdef _LP64 192179193Sjb if (i & 0xffffffff00000000ul) { 193179193Sjb h += 32; i >>= 32; 194179193Sjb } 195179193Sjb#endif 196179193Sjb if (i & 0xffff0000) { 197179193Sjb h += 16; i >>= 16; 198179193Sjb } 199179193Sjb if (i & 0xff00) { 200179193Sjb h += 8; i >>= 8; 201179193Sjb } 202179193Sjb if (i & 0xf0) { 203179193Sjb h += 4; i >>= 4; 204179193Sjb } 205179193Sjb if (i & 0xc) { 206179193Sjb h += 2; i >>= 2; 207179193Sjb } 208179193Sjb if (i & 0x2) { 209179193Sjb h += 1; 210179193Sjb } 211179193Sjb return (h); 212179193Sjb} 213179193Sjb 214179193Sjbstatic uint_t 215179193Sjbfasttrap_hash_str(const char *p) 216179193Sjb{ 217179193Sjb unsigned int g; 218179193Sjb uint_t hval = 0; 219179193Sjb 220179193Sjb while (*p) { 221179193Sjb hval = (hval << 4) + *p++; 222179193Sjb if ((g = (hval & 0xf0000000)) != 0) 223179193Sjb hval ^= g >> 24; 224179193Sjb hval &= ~g; 225179193Sjb } 226179193Sjb return (hval); 227179193Sjb} 228179193Sjb 229179193Sjbvoid 230179193Sjbfasttrap_sigtrap(proc_t *p, kthread_t *t, uintptr_t pc) 231179193Sjb{ 232179193Sjb sigqueue_t *sqp = kmem_zalloc(sizeof (sigqueue_t), KM_SLEEP); 233179193Sjb 234179193Sjb sqp->sq_info.si_signo = SIGTRAP; 235179193Sjb sqp->sq_info.si_code = TRAP_DTRACE; 236179193Sjb sqp->sq_info.si_addr = (caddr_t)pc; 237179193Sjb 238179193Sjb mutex_enter(&p->p_lock); 239179193Sjb sigaddqa(p, t, sqp); 240179193Sjb mutex_exit(&p->p_lock); 241179193Sjb 242179193Sjb if (t != NULL) 243179193Sjb aston(t); 244179193Sjb} 245179193Sjb 246179193Sjb/* 247179193Sjb * This function ensures that no threads are actively using the memory 248179193Sjb * associated with probes that were formerly live. 249179193Sjb */ 250179193Sjbstatic void 251179193Sjbfasttrap_mod_barrier(uint64_t gen) 252179193Sjb{ 253179193Sjb int i; 254179193Sjb 255179193Sjb if (gen < fasttrap_mod_gen) 256179193Sjb return; 257179193Sjb 258179193Sjb fasttrap_mod_gen++; 259179193Sjb 260179193Sjb for (i = 0; i < NCPU; i++) { 261179193Sjb mutex_enter(&cpu_core[i].cpuc_pid_lock); 262179193Sjb mutex_exit(&cpu_core[i].cpuc_pid_lock); 263179193Sjb } 264179193Sjb} 265179193Sjb 266179193Sjb/* 267179193Sjb * This is the timeout's callback for cleaning up the providers and their 268179193Sjb * probes. 269179193Sjb */ 270179193Sjb/*ARGSUSED*/ 271179193Sjbstatic void 272179193Sjbfasttrap_pid_cleanup_cb(void *data) 273179193Sjb{ 274179193Sjb fasttrap_provider_t **fpp, *fp; 275179193Sjb fasttrap_bucket_t *bucket; 276179193Sjb dtrace_provider_id_t provid; 277179193Sjb int i, later; 278179193Sjb 279179193Sjb static volatile int in = 0; 280179193Sjb ASSERT(in == 0); 281179193Sjb in = 1; 282179193Sjb 283179193Sjb mutex_enter(&fasttrap_cleanup_mtx); 284179193Sjb while (fasttrap_cleanup_work) { 285179193Sjb fasttrap_cleanup_work = 0; 286179193Sjb mutex_exit(&fasttrap_cleanup_mtx); 287179193Sjb 288179193Sjb later = 0; 289179193Sjb 290179193Sjb /* 291179193Sjb * Iterate over all the providers trying to remove the marked 292179193Sjb * ones. If a provider is marked but not retired, we just 293179193Sjb * have to take a crack at removing it -- it's no big deal if 294179193Sjb * we can't. 295179193Sjb */ 296179193Sjb for (i = 0; i < fasttrap_provs.fth_nent; i++) { 297179193Sjb bucket = &fasttrap_provs.fth_table[i]; 298179193Sjb mutex_enter(&bucket->ftb_mtx); 299179193Sjb fpp = (fasttrap_provider_t **)&bucket->ftb_data; 300179193Sjb 301179193Sjb while ((fp = *fpp) != NULL) { 302179193Sjb if (!fp->ftp_marked) { 303179193Sjb fpp = &fp->ftp_next; 304179193Sjb continue; 305179193Sjb } 306179193Sjb 307179193Sjb mutex_enter(&fp->ftp_mtx); 308179193Sjb 309179193Sjb /* 310179193Sjb * If this provider has consumers actively 311179193Sjb * creating probes (ftp_ccount) or is a USDT 312179193Sjb * provider (ftp_mcount), we can't unregister 313179193Sjb * or even condense. 314179193Sjb */ 315179193Sjb if (fp->ftp_ccount != 0 || 316179193Sjb fp->ftp_mcount != 0) { 317179193Sjb mutex_exit(&fp->ftp_mtx); 318179193Sjb fp->ftp_marked = 0; 319179193Sjb continue; 320179193Sjb } 321179193Sjb 322179193Sjb if (!fp->ftp_retired || fp->ftp_rcount != 0) 323179193Sjb fp->ftp_marked = 0; 324179193Sjb 325179193Sjb mutex_exit(&fp->ftp_mtx); 326179193Sjb 327179193Sjb /* 328179193Sjb * If we successfully unregister this 329179193Sjb * provider we can remove it from the hash 330179193Sjb * chain and free the memory. If our attempt 331179193Sjb * to unregister fails and this is a retired 332179193Sjb * provider, increment our flag to try again 333179193Sjb * pretty soon. If we've consumed more than 334179193Sjb * half of our total permitted number of 335179193Sjb * probes call dtrace_condense() to try to 336179193Sjb * clean out the unenabled probes. 337179193Sjb */ 338179193Sjb provid = fp->ftp_provid; 339179193Sjb if (dtrace_unregister(provid) != 0) { 340179193Sjb if (fasttrap_total > fasttrap_max / 2) 341179193Sjb (void) dtrace_condense(provid); 342179193Sjb later += fp->ftp_marked; 343179193Sjb fpp = &fp->ftp_next; 344179193Sjb } else { 345179193Sjb *fpp = fp->ftp_next; 346179193Sjb fasttrap_provider_free(fp); 347179193Sjb } 348179193Sjb } 349179193Sjb mutex_exit(&bucket->ftb_mtx); 350179193Sjb } 351179193Sjb 352179193Sjb mutex_enter(&fasttrap_cleanup_mtx); 353179193Sjb } 354179193Sjb 355179193Sjb ASSERT(fasttrap_timeout != 0); 356179193Sjb 357179193Sjb /* 358179193Sjb * If we were unable to remove a retired provider, try again after 359179193Sjb * a second. This situation can occur in certain circumstances where 360179193Sjb * providers cannot be unregistered even though they have no probes 361179193Sjb * enabled because of an execution of dtrace -l or something similar. 362179193Sjb * If the timeout has been disabled (set to 1 because we're trying 363179193Sjb * to detach), we set fasttrap_cleanup_work to ensure that we'll 364179193Sjb * get a chance to do that work if and when the timeout is reenabled 365179193Sjb * (if detach fails). 366179193Sjb */ 367179193Sjb if (later > 0 && fasttrap_timeout != (timeout_id_t)1) 368179193Sjb fasttrap_timeout = timeout(&fasttrap_pid_cleanup_cb, NULL, hz); 369179193Sjb else if (later > 0) 370179193Sjb fasttrap_cleanup_work = 1; 371179193Sjb else 372179193Sjb fasttrap_timeout = 0; 373179193Sjb 374179193Sjb mutex_exit(&fasttrap_cleanup_mtx); 375179193Sjb in = 0; 376179193Sjb} 377179193Sjb 378179193Sjb/* 379179193Sjb * Activates the asynchronous cleanup mechanism. 380179193Sjb */ 381179193Sjbstatic void 382179193Sjbfasttrap_pid_cleanup(void) 383179193Sjb{ 384179193Sjb mutex_enter(&fasttrap_cleanup_mtx); 385179193Sjb fasttrap_cleanup_work = 1; 386179193Sjb if (fasttrap_timeout == 0) 387179193Sjb fasttrap_timeout = timeout(&fasttrap_pid_cleanup_cb, NULL, 1); 388179193Sjb mutex_exit(&fasttrap_cleanup_mtx); 389179193Sjb} 390179193Sjb 391179193Sjb/* 392179193Sjb * This is called from cfork() via dtrace_fasttrap_fork(). The child 393179198Sjb * process's address space is (roughly) a copy of the parent process's so 394179193Sjb * we have to remove all the instrumentation we had previously enabled in the 395179193Sjb * parent. 396179193Sjb */ 397179193Sjbstatic void 398179193Sjbfasttrap_fork(proc_t *p, proc_t *cp) 399179193Sjb{ 400179193Sjb pid_t ppid = p->p_pid; 401179193Sjb int i; 402179193Sjb 403179193Sjb ASSERT(curproc == p); 404179193Sjb ASSERT(p->p_proc_flag & P_PR_LOCK); 405179193Sjb ASSERT(p->p_dtrace_count > 0); 406179193Sjb ASSERT(cp->p_dtrace_count == 0); 407179193Sjb 408179193Sjb /* 409179193Sjb * This would be simpler and faster if we maintained per-process 410179193Sjb * hash tables of enabled tracepoints. It could, however, potentially 411179193Sjb * slow down execution of a tracepoint since we'd need to go 412179193Sjb * through two levels of indirection. In the future, we should 413179193Sjb * consider either maintaining per-process ancillary lists of 414179193Sjb * enabled tracepoints or hanging a pointer to a per-process hash 415179193Sjb * table of enabled tracepoints off the proc structure. 416179193Sjb */ 417179193Sjb 418179193Sjb /* 419179193Sjb * We don't have to worry about the child process disappearing 420179193Sjb * because we're in fork(). 421179193Sjb */ 422179193Sjb mutex_enter(&cp->p_lock); 423179193Sjb sprlock_proc(cp); 424179193Sjb mutex_exit(&cp->p_lock); 425179193Sjb 426179193Sjb /* 427179193Sjb * Iterate over every tracepoint looking for ones that belong to the 428179193Sjb * parent process, and remove each from the child process. 429179193Sjb */ 430179193Sjb for (i = 0; i < fasttrap_tpoints.fth_nent; i++) { 431179193Sjb fasttrap_tracepoint_t *tp; 432179193Sjb fasttrap_bucket_t *bucket = &fasttrap_tpoints.fth_table[i]; 433179193Sjb 434179193Sjb mutex_enter(&bucket->ftb_mtx); 435179193Sjb for (tp = bucket->ftb_data; tp != NULL; tp = tp->ftt_next) { 436179193Sjb if (tp->ftt_pid == ppid && 437179193Sjb tp->ftt_proc->ftpc_acount != 0) { 438179193Sjb int ret = fasttrap_tracepoint_remove(cp, tp); 439179193Sjb ASSERT(ret == 0); 440179198Sjb 441179198Sjb /* 442179198Sjb * The count of active providers can only be 443179198Sjb * decremented (i.e. to zero) during exec, 444179198Sjb * exit, and removal of a meta provider so it 445179198Sjb * should be impossible to drop the count 446179198Sjb * mid-fork. 447179198Sjb */ 448179198Sjb ASSERT(tp->ftt_proc->ftpc_acount != 0); 449179193Sjb } 450179193Sjb } 451179193Sjb mutex_exit(&bucket->ftb_mtx); 452179193Sjb } 453179193Sjb 454179193Sjb mutex_enter(&cp->p_lock); 455179193Sjb sprunlock(cp); 456179193Sjb} 457179193Sjb 458179193Sjb/* 459179193Sjb * This is called from proc_exit() or from exec_common() if p_dtrace_probes 460179193Sjb * is set on the proc structure to indicate that there is a pid provider 461179193Sjb * associated with this process. 462179193Sjb */ 463179193Sjbstatic void 464179193Sjbfasttrap_exec_exit(proc_t *p) 465179193Sjb{ 466179193Sjb ASSERT(p == curproc); 467179193Sjb ASSERT(MUTEX_HELD(&p->p_lock)); 468179193Sjb 469179193Sjb mutex_exit(&p->p_lock); 470179193Sjb 471179193Sjb /* 472179193Sjb * We clean up the pid provider for this process here; user-land 473179193Sjb * static probes are handled by the meta-provider remove entry point. 474179193Sjb */ 475179193Sjb fasttrap_provider_retire(p->p_pid, FASTTRAP_PID_NAME, 0); 476179193Sjb 477179193Sjb mutex_enter(&p->p_lock); 478179193Sjb} 479179193Sjb 480179193Sjb 481179193Sjb/*ARGSUSED*/ 482179193Sjbstatic void 483179193Sjbfasttrap_pid_provide(void *arg, const dtrace_probedesc_t *desc) 484179193Sjb{ 485179193Sjb /* 486179193Sjb * There are no "default" pid probes. 487179193Sjb */ 488179193Sjb} 489179193Sjb 490179193Sjbstatic int 491179193Sjbfasttrap_tracepoint_enable(proc_t *p, fasttrap_probe_t *probe, uint_t index) 492179193Sjb{ 493179193Sjb fasttrap_tracepoint_t *tp, *new_tp = NULL; 494179193Sjb fasttrap_bucket_t *bucket; 495179193Sjb fasttrap_id_t *id; 496179193Sjb pid_t pid; 497179193Sjb uintptr_t pc; 498179193Sjb 499179193Sjb ASSERT(index < probe->ftp_ntps); 500179193Sjb 501179193Sjb pid = probe->ftp_pid; 502179193Sjb pc = probe->ftp_tps[index].fit_tp->ftt_pc; 503179193Sjb id = &probe->ftp_tps[index].fit_id; 504179193Sjb 505179193Sjb ASSERT(probe->ftp_tps[index].fit_tp->ftt_pid == pid); 506179193Sjb 507179193Sjb ASSERT(!(p->p_flag & SVFORK)); 508179193Sjb 509179193Sjb /* 510179193Sjb * Before we make any modifications, make sure we've imposed a barrier 511179193Sjb * on the generation in which this probe was last modified. 512179193Sjb */ 513179193Sjb fasttrap_mod_barrier(probe->ftp_gen); 514179193Sjb 515179193Sjb bucket = &fasttrap_tpoints.fth_table[FASTTRAP_TPOINTS_INDEX(pid, pc)]; 516179193Sjb 517179193Sjb /* 518179193Sjb * If the tracepoint has already been enabled, just add our id to the 519179193Sjb * list of interested probes. This may be our second time through 520179193Sjb * this path in which case we'll have constructed the tracepoint we'd 521179193Sjb * like to install. If we can't find a match, and have an allocated 522179193Sjb * tracepoint ready to go, enable that one now. 523179193Sjb * 524179193Sjb * A tracepoint whose process is defunct is also considered defunct. 525179193Sjb */ 526179193Sjbagain: 527179193Sjb mutex_enter(&bucket->ftb_mtx); 528179193Sjb for (tp = bucket->ftb_data; tp != NULL; tp = tp->ftt_next) { 529179198Sjb /* 530179198Sjb * Note that it's safe to access the active count on the 531179198Sjb * associated proc structure because we know that at least one 532179198Sjb * provider (this one) will still be around throughout this 533179198Sjb * operation. 534179198Sjb */ 535179193Sjb if (tp->ftt_pid != pid || tp->ftt_pc != pc || 536179193Sjb tp->ftt_proc->ftpc_acount == 0) 537179193Sjb continue; 538179193Sjb 539179193Sjb /* 540179193Sjb * Now that we've found a matching tracepoint, it would be 541179193Sjb * a decent idea to confirm that the tracepoint is still 542179193Sjb * enabled and the trap instruction hasn't been overwritten. 543179193Sjb * Since this is a little hairy, we'll punt for now. 544179193Sjb */ 545179193Sjb 546179193Sjb /* 547179193Sjb * This can't be the first interested probe. We don't have 548179193Sjb * to worry about another thread being in the midst of 549179193Sjb * deleting this tracepoint (which would be the only valid 550179193Sjb * reason for a tracepoint to have no interested probes) 551179193Sjb * since we're holding P_PR_LOCK for this process. 552179193Sjb */ 553179193Sjb ASSERT(tp->ftt_ids != NULL || tp->ftt_retids != NULL); 554179193Sjb 555179193Sjb switch (id->fti_ptype) { 556179193Sjb case DTFTP_ENTRY: 557179193Sjb case DTFTP_OFFSETS: 558179193Sjb case DTFTP_IS_ENABLED: 559179193Sjb id->fti_next = tp->ftt_ids; 560179193Sjb membar_producer(); 561179193Sjb tp->ftt_ids = id; 562179193Sjb membar_producer(); 563179193Sjb break; 564179193Sjb 565179193Sjb case DTFTP_RETURN: 566179193Sjb case DTFTP_POST_OFFSETS: 567179193Sjb id->fti_next = tp->ftt_retids; 568179193Sjb membar_producer(); 569179193Sjb tp->ftt_retids = id; 570179193Sjb membar_producer(); 571179193Sjb break; 572179193Sjb 573179193Sjb default: 574179193Sjb ASSERT(0); 575179193Sjb } 576179193Sjb 577179193Sjb mutex_exit(&bucket->ftb_mtx); 578179193Sjb 579179193Sjb if (new_tp != NULL) { 580179193Sjb new_tp->ftt_ids = NULL; 581179193Sjb new_tp->ftt_retids = NULL; 582179193Sjb } 583179193Sjb 584179193Sjb return (0); 585179193Sjb } 586179193Sjb 587179193Sjb /* 588179193Sjb * If we have a good tracepoint ready to go, install it now while 589179193Sjb * we have the lock held and no one can screw with us. 590179193Sjb */ 591179193Sjb if (new_tp != NULL) { 592179193Sjb int rc = 0; 593179193Sjb 594179193Sjb new_tp->ftt_next = bucket->ftb_data; 595179193Sjb membar_producer(); 596179193Sjb bucket->ftb_data = new_tp; 597179193Sjb membar_producer(); 598179193Sjb mutex_exit(&bucket->ftb_mtx); 599179193Sjb 600179193Sjb /* 601179193Sjb * Activate the tracepoint in the ISA-specific manner. 602179193Sjb * If this fails, we need to report the failure, but 603179193Sjb * indicate that this tracepoint must still be disabled 604179193Sjb * by calling fasttrap_tracepoint_disable(). 605179193Sjb */ 606179193Sjb if (fasttrap_tracepoint_install(p, new_tp) != 0) 607179193Sjb rc = FASTTRAP_ENABLE_PARTIAL; 608179193Sjb 609179193Sjb /* 610179193Sjb * Increment the count of the number of tracepoints active in 611179193Sjb * the victim process. 612179193Sjb */ 613179193Sjb ASSERT(p->p_proc_flag & P_PR_LOCK); 614179193Sjb p->p_dtrace_count++; 615179193Sjb 616179193Sjb return (rc); 617179193Sjb } 618179193Sjb 619179193Sjb mutex_exit(&bucket->ftb_mtx); 620179193Sjb 621179193Sjb /* 622179193Sjb * Initialize the tracepoint that's been preallocated with the probe. 623179193Sjb */ 624179193Sjb new_tp = probe->ftp_tps[index].fit_tp; 625179193Sjb 626179193Sjb ASSERT(new_tp->ftt_pid == pid); 627179193Sjb ASSERT(new_tp->ftt_pc == pc); 628179193Sjb ASSERT(new_tp->ftt_proc == probe->ftp_prov->ftp_proc); 629179193Sjb ASSERT(new_tp->ftt_ids == NULL); 630179193Sjb ASSERT(new_tp->ftt_retids == NULL); 631179193Sjb 632179193Sjb switch (id->fti_ptype) { 633179193Sjb case DTFTP_ENTRY: 634179193Sjb case DTFTP_OFFSETS: 635179193Sjb case DTFTP_IS_ENABLED: 636179193Sjb id->fti_next = NULL; 637179193Sjb new_tp->ftt_ids = id; 638179193Sjb break; 639179193Sjb 640179193Sjb case DTFTP_RETURN: 641179193Sjb case DTFTP_POST_OFFSETS: 642179193Sjb id->fti_next = NULL; 643179193Sjb new_tp->ftt_retids = id; 644179193Sjb break; 645179193Sjb 646179193Sjb default: 647179193Sjb ASSERT(0); 648179193Sjb } 649179193Sjb 650179193Sjb /* 651179193Sjb * If the ISA-dependent initialization goes to plan, go back to the 652179193Sjb * beginning and try to install this freshly made tracepoint. 653179193Sjb */ 654179193Sjb if (fasttrap_tracepoint_init(p, new_tp, pc, id->fti_ptype) == 0) 655179193Sjb goto again; 656179193Sjb 657179193Sjb new_tp->ftt_ids = NULL; 658179193Sjb new_tp->ftt_retids = NULL; 659179193Sjb 660179193Sjb return (FASTTRAP_ENABLE_FAIL); 661179193Sjb} 662179193Sjb 663179193Sjbstatic void 664179193Sjbfasttrap_tracepoint_disable(proc_t *p, fasttrap_probe_t *probe, uint_t index) 665179193Sjb{ 666179193Sjb fasttrap_bucket_t *bucket; 667179193Sjb fasttrap_provider_t *provider = probe->ftp_prov; 668179193Sjb fasttrap_tracepoint_t **pp, *tp; 669179193Sjb fasttrap_id_t *id, **idp; 670179193Sjb pid_t pid; 671179193Sjb uintptr_t pc; 672179193Sjb 673179193Sjb ASSERT(index < probe->ftp_ntps); 674179193Sjb 675179193Sjb pid = probe->ftp_pid; 676179193Sjb pc = probe->ftp_tps[index].fit_tp->ftt_pc; 677179193Sjb id = &probe->ftp_tps[index].fit_id; 678179193Sjb 679179193Sjb ASSERT(probe->ftp_tps[index].fit_tp->ftt_pid == pid); 680179193Sjb 681179193Sjb /* 682179193Sjb * Find the tracepoint and make sure that our id is one of the 683179193Sjb * ones registered with it. 684179193Sjb */ 685179193Sjb bucket = &fasttrap_tpoints.fth_table[FASTTRAP_TPOINTS_INDEX(pid, pc)]; 686179193Sjb mutex_enter(&bucket->ftb_mtx); 687179193Sjb for (tp = bucket->ftb_data; tp != NULL; tp = tp->ftt_next) { 688179193Sjb if (tp->ftt_pid == pid && tp->ftt_pc == pc && 689179193Sjb tp->ftt_proc == provider->ftp_proc) 690179193Sjb break; 691179193Sjb } 692179193Sjb 693179193Sjb /* 694179193Sjb * If we somehow lost this tracepoint, we're in a world of hurt. 695179193Sjb */ 696179193Sjb ASSERT(tp != NULL); 697179193Sjb 698179193Sjb switch (id->fti_ptype) { 699179193Sjb case DTFTP_ENTRY: 700179193Sjb case DTFTP_OFFSETS: 701179193Sjb case DTFTP_IS_ENABLED: 702179193Sjb ASSERT(tp->ftt_ids != NULL); 703179193Sjb idp = &tp->ftt_ids; 704179193Sjb break; 705179193Sjb 706179193Sjb case DTFTP_RETURN: 707179193Sjb case DTFTP_POST_OFFSETS: 708179193Sjb ASSERT(tp->ftt_retids != NULL); 709179193Sjb idp = &tp->ftt_retids; 710179193Sjb break; 711179193Sjb 712179193Sjb default: 713179193Sjb ASSERT(0); 714179193Sjb } 715179193Sjb 716179193Sjb while ((*idp)->fti_probe != probe) { 717179193Sjb idp = &(*idp)->fti_next; 718179193Sjb ASSERT(*idp != NULL); 719179193Sjb } 720179193Sjb 721179193Sjb id = *idp; 722179193Sjb *idp = id->fti_next; 723179193Sjb membar_producer(); 724179193Sjb 725179193Sjb ASSERT(id->fti_probe == probe); 726179193Sjb 727179193Sjb /* 728179193Sjb * If there are other registered enablings of this tracepoint, we're 729179193Sjb * all done, but if this was the last probe assocated with this 730179193Sjb * this tracepoint, we need to remove and free it. 731179193Sjb */ 732179193Sjb if (tp->ftt_ids != NULL || tp->ftt_retids != NULL) { 733179193Sjb 734179193Sjb /* 735179193Sjb * If the current probe's tracepoint is in use, swap it 736179193Sjb * for an unused tracepoint. 737179193Sjb */ 738179193Sjb if (tp == probe->ftp_tps[index].fit_tp) { 739179193Sjb fasttrap_probe_t *tmp_probe; 740179193Sjb fasttrap_tracepoint_t **tmp_tp; 741179193Sjb uint_t tmp_index; 742179193Sjb 743179193Sjb if (tp->ftt_ids != NULL) { 744179193Sjb tmp_probe = tp->ftt_ids->fti_probe; 745179193Sjb /* LINTED - alignment */ 746179193Sjb tmp_index = FASTTRAP_ID_INDEX(tp->ftt_ids); 747179193Sjb tmp_tp = &tmp_probe->ftp_tps[tmp_index].fit_tp; 748179193Sjb } else { 749179193Sjb tmp_probe = tp->ftt_retids->fti_probe; 750179193Sjb /* LINTED - alignment */ 751179193Sjb tmp_index = FASTTRAP_ID_INDEX(tp->ftt_retids); 752179193Sjb tmp_tp = &tmp_probe->ftp_tps[tmp_index].fit_tp; 753179193Sjb } 754179193Sjb 755179193Sjb ASSERT(*tmp_tp != NULL); 756179193Sjb ASSERT(*tmp_tp != probe->ftp_tps[index].fit_tp); 757179193Sjb ASSERT((*tmp_tp)->ftt_ids == NULL); 758179193Sjb ASSERT((*tmp_tp)->ftt_retids == NULL); 759179193Sjb 760179193Sjb probe->ftp_tps[index].fit_tp = *tmp_tp; 761179193Sjb *tmp_tp = tp; 762179193Sjb } 763179193Sjb 764179193Sjb mutex_exit(&bucket->ftb_mtx); 765179193Sjb 766179193Sjb /* 767179193Sjb * Tag the modified probe with the generation in which it was 768179193Sjb * changed. 769179193Sjb */ 770179193Sjb probe->ftp_gen = fasttrap_mod_gen; 771179193Sjb return; 772179193Sjb } 773179193Sjb 774179193Sjb mutex_exit(&bucket->ftb_mtx); 775179193Sjb 776179193Sjb /* 777179193Sjb * We can't safely remove the tracepoint from the set of active 778179193Sjb * tracepoints until we've actually removed the fasttrap instruction 779179193Sjb * from the process's text. We can, however, operate on this 780179193Sjb * tracepoint secure in the knowledge that no other thread is going to 781179193Sjb * be looking at it since we hold P_PR_LOCK on the process if it's 782179193Sjb * live or we hold the provider lock on the process if it's dead and 783179193Sjb * gone. 784179193Sjb */ 785179193Sjb 786179193Sjb /* 787179193Sjb * We only need to remove the actual instruction if we're looking 788179193Sjb * at an existing process 789179193Sjb */ 790179193Sjb if (p != NULL) { 791179193Sjb /* 792179193Sjb * If we fail to restore the instruction we need to kill 793179193Sjb * this process since it's in a completely unrecoverable 794179193Sjb * state. 795179193Sjb */ 796179193Sjb if (fasttrap_tracepoint_remove(p, tp) != 0) 797179193Sjb fasttrap_sigtrap(p, NULL, pc); 798179193Sjb 799179193Sjb /* 800179193Sjb * Decrement the count of the number of tracepoints active 801179193Sjb * in the victim process. 802179193Sjb */ 803179193Sjb ASSERT(p->p_proc_flag & P_PR_LOCK); 804179193Sjb p->p_dtrace_count--; 805179193Sjb } 806179193Sjb 807179193Sjb /* 808179193Sjb * Remove the probe from the hash table of active tracepoints. 809179193Sjb */ 810179193Sjb mutex_enter(&bucket->ftb_mtx); 811179193Sjb pp = (fasttrap_tracepoint_t **)&bucket->ftb_data; 812179193Sjb ASSERT(*pp != NULL); 813179193Sjb while (*pp != tp) { 814179193Sjb pp = &(*pp)->ftt_next; 815179193Sjb ASSERT(*pp != NULL); 816179193Sjb } 817179193Sjb 818179193Sjb *pp = tp->ftt_next; 819179193Sjb membar_producer(); 820179193Sjb 821179193Sjb mutex_exit(&bucket->ftb_mtx); 822179193Sjb 823179193Sjb /* 824179193Sjb * Tag the modified probe with the generation in which it was changed. 825179193Sjb */ 826179193Sjb probe->ftp_gen = fasttrap_mod_gen; 827179193Sjb} 828179193Sjb 829179193Sjbstatic void 830179193Sjbfasttrap_enable_callbacks(void) 831179193Sjb{ 832179193Sjb /* 833179193Sjb * We don't have to play the rw lock game here because we're 834179193Sjb * providing something rather than taking something away -- 835179193Sjb * we can be sure that no threads have tried to follow this 836179193Sjb * function pointer yet. 837179193Sjb */ 838179193Sjb mutex_enter(&fasttrap_count_mtx); 839179193Sjb if (fasttrap_pid_count == 0) { 840179193Sjb ASSERT(dtrace_pid_probe_ptr == NULL); 841179193Sjb ASSERT(dtrace_return_probe_ptr == NULL); 842179193Sjb dtrace_pid_probe_ptr = &fasttrap_pid_probe; 843179193Sjb dtrace_return_probe_ptr = &fasttrap_return_probe; 844179193Sjb } 845179193Sjb ASSERT(dtrace_pid_probe_ptr == &fasttrap_pid_probe); 846179193Sjb ASSERT(dtrace_return_probe_ptr == &fasttrap_return_probe); 847179193Sjb fasttrap_pid_count++; 848179193Sjb mutex_exit(&fasttrap_count_mtx); 849179193Sjb} 850179193Sjb 851179193Sjbstatic void 852179193Sjbfasttrap_disable_callbacks(void) 853179193Sjb{ 854179193Sjb ASSERT(MUTEX_HELD(&cpu_lock)); 855179193Sjb 856179193Sjb mutex_enter(&fasttrap_count_mtx); 857179193Sjb ASSERT(fasttrap_pid_count > 0); 858179193Sjb fasttrap_pid_count--; 859179193Sjb if (fasttrap_pid_count == 0) { 860179193Sjb cpu_t *cur, *cpu = CPU; 861179193Sjb 862179193Sjb for (cur = cpu->cpu_next_onln; cur != cpu; 863179193Sjb cur = cur->cpu_next_onln) { 864179193Sjb rw_enter(&cur->cpu_ft_lock, RW_WRITER); 865179193Sjb } 866179193Sjb 867179193Sjb dtrace_pid_probe_ptr = NULL; 868179193Sjb dtrace_return_probe_ptr = NULL; 869179193Sjb 870179193Sjb for (cur = cpu->cpu_next_onln; cur != cpu; 871179193Sjb cur = cur->cpu_next_onln) { 872179193Sjb rw_exit(&cur->cpu_ft_lock); 873179193Sjb } 874179193Sjb } 875179193Sjb mutex_exit(&fasttrap_count_mtx); 876179193Sjb} 877179193Sjb 878179193Sjb/*ARGSUSED*/ 879179193Sjbstatic void 880179193Sjbfasttrap_pid_enable(void *arg, dtrace_id_t id, void *parg) 881179193Sjb{ 882179193Sjb fasttrap_probe_t *probe = parg; 883179193Sjb proc_t *p; 884179193Sjb int i, rc; 885179193Sjb 886179193Sjb ASSERT(probe != NULL); 887179193Sjb ASSERT(!probe->ftp_enabled); 888179193Sjb ASSERT(id == probe->ftp_id); 889179193Sjb ASSERT(MUTEX_HELD(&cpu_lock)); 890179193Sjb 891179193Sjb /* 892179193Sjb * Increment the count of enabled probes on this probe's provider; 893179193Sjb * the provider can't go away while the probe still exists. We 894179193Sjb * must increment this even if we aren't able to properly enable 895179193Sjb * this probe. 896179193Sjb */ 897179193Sjb mutex_enter(&probe->ftp_prov->ftp_mtx); 898179193Sjb probe->ftp_prov->ftp_rcount++; 899179193Sjb mutex_exit(&probe->ftp_prov->ftp_mtx); 900179193Sjb 901179193Sjb /* 902179193Sjb * If this probe's provider is retired (meaning it was valid in a 903179193Sjb * previously exec'ed incarnation of this address space), bail out. The 904179193Sjb * provider can't go away while we're in this code path. 905179193Sjb */ 906179193Sjb if (probe->ftp_prov->ftp_retired) 907179193Sjb return; 908179193Sjb 909179193Sjb /* 910179193Sjb * If we can't find the process, it may be that we're in the context of 911179193Sjb * a fork in which the traced process is being born and we're copying 912179193Sjb * USDT probes. Otherwise, the process is gone so bail. 913179193Sjb */ 914179193Sjb if ((p = sprlock(probe->ftp_pid)) == NULL) { 915179193Sjb if ((curproc->p_flag & SFORKING) == 0) 916179193Sjb return; 917179193Sjb 918179193Sjb mutex_enter(&pidlock); 919179193Sjb p = prfind(probe->ftp_pid); 920179193Sjb 921179193Sjb /* 922179193Sjb * Confirm that curproc is indeed forking the process in which 923179193Sjb * we're trying to enable probes. 924179193Sjb */ 925179193Sjb ASSERT(p != NULL); 926179193Sjb ASSERT(p->p_parent == curproc); 927179193Sjb ASSERT(p->p_stat == SIDL); 928179193Sjb 929179193Sjb mutex_enter(&p->p_lock); 930179193Sjb mutex_exit(&pidlock); 931179193Sjb 932179193Sjb sprlock_proc(p); 933179193Sjb } 934179193Sjb 935179193Sjb ASSERT(!(p->p_flag & SVFORK)); 936179193Sjb mutex_exit(&p->p_lock); 937179193Sjb 938179193Sjb /* 939179193Sjb * We have to enable the trap entry point before any user threads have 940179193Sjb * the chance to execute the trap instruction we're about to place 941179193Sjb * in their process's text. 942179193Sjb */ 943179193Sjb fasttrap_enable_callbacks(); 944179193Sjb 945179193Sjb /* 946179193Sjb * Enable all the tracepoints and add this probe's id to each 947179193Sjb * tracepoint's list of active probes. 948179193Sjb */ 949179193Sjb for (i = 0; i < probe->ftp_ntps; i++) { 950179193Sjb if ((rc = fasttrap_tracepoint_enable(p, probe, i)) != 0) { 951179193Sjb /* 952179193Sjb * If enabling the tracepoint failed completely, 953179193Sjb * we don't have to disable it; if the failure 954179193Sjb * was only partial we must disable it. 955179193Sjb */ 956179193Sjb if (rc == FASTTRAP_ENABLE_FAIL) 957179193Sjb i--; 958179193Sjb else 959179193Sjb ASSERT(rc == FASTTRAP_ENABLE_PARTIAL); 960179193Sjb 961179193Sjb /* 962179193Sjb * Back up and pull out all the tracepoints we've 963179193Sjb * created so far for this probe. 964179193Sjb */ 965179193Sjb while (i >= 0) { 966179193Sjb fasttrap_tracepoint_disable(p, probe, i); 967179193Sjb i--; 968179193Sjb } 969179193Sjb 970179193Sjb mutex_enter(&p->p_lock); 971179193Sjb sprunlock(p); 972179193Sjb 973179193Sjb /* 974179193Sjb * Since we're not actually enabling this probe, 975179193Sjb * drop our reference on the trap table entry. 976179193Sjb */ 977179193Sjb fasttrap_disable_callbacks(); 978179193Sjb return; 979179193Sjb } 980179193Sjb } 981179193Sjb 982179193Sjb mutex_enter(&p->p_lock); 983179193Sjb sprunlock(p); 984179193Sjb 985179193Sjb probe->ftp_enabled = 1; 986179193Sjb} 987179193Sjb 988179193Sjb/*ARGSUSED*/ 989179193Sjbstatic void 990179193Sjbfasttrap_pid_disable(void *arg, dtrace_id_t id, void *parg) 991179193Sjb{ 992179193Sjb fasttrap_probe_t *probe = parg; 993179193Sjb fasttrap_provider_t *provider = probe->ftp_prov; 994179193Sjb proc_t *p; 995179193Sjb int i, whack = 0; 996179193Sjb 997179193Sjb ASSERT(id == probe->ftp_id); 998179193Sjb 999179193Sjb /* 1000179193Sjb * We won't be able to acquire a /proc-esque lock on the process 1001179193Sjb * iff the process is dead and gone. In this case, we rely on the 1002179193Sjb * provider lock as a point of mutual exclusion to prevent other 1003179193Sjb * DTrace consumers from disabling this probe. 1004179193Sjb */ 1005179193Sjb if ((p = sprlock(probe->ftp_pid)) != NULL) { 1006179193Sjb ASSERT(!(p->p_flag & SVFORK)); 1007179193Sjb mutex_exit(&p->p_lock); 1008179193Sjb } 1009179193Sjb 1010179193Sjb mutex_enter(&provider->ftp_mtx); 1011179193Sjb 1012179193Sjb /* 1013179193Sjb * Disable all the associated tracepoints (for fully enabled probes). 1014179193Sjb */ 1015179193Sjb if (probe->ftp_enabled) { 1016179193Sjb for (i = 0; i < probe->ftp_ntps; i++) { 1017179193Sjb fasttrap_tracepoint_disable(p, probe, i); 1018179193Sjb } 1019179193Sjb } 1020179193Sjb 1021179193Sjb ASSERT(provider->ftp_rcount > 0); 1022179193Sjb provider->ftp_rcount--; 1023179193Sjb 1024179193Sjb if (p != NULL) { 1025179193Sjb /* 1026179193Sjb * Even though we may not be able to remove it entirely, we 1027179193Sjb * mark this retired provider to get a chance to remove some 1028179193Sjb * of the associated probes. 1029179193Sjb */ 1030179193Sjb if (provider->ftp_retired && !provider->ftp_marked) 1031179193Sjb whack = provider->ftp_marked = 1; 1032179193Sjb mutex_exit(&provider->ftp_mtx); 1033179193Sjb 1034179193Sjb mutex_enter(&p->p_lock); 1035179193Sjb sprunlock(p); 1036179193Sjb } else { 1037179193Sjb /* 1038179193Sjb * If the process is dead, we're just waiting for the 1039179193Sjb * last probe to be disabled to be able to free it. 1040179193Sjb */ 1041179193Sjb if (provider->ftp_rcount == 0 && !provider->ftp_marked) 1042179193Sjb whack = provider->ftp_marked = 1; 1043179193Sjb mutex_exit(&provider->ftp_mtx); 1044179193Sjb } 1045179193Sjb 1046179193Sjb if (whack) 1047179193Sjb fasttrap_pid_cleanup(); 1048179193Sjb 1049179193Sjb if (!probe->ftp_enabled) 1050179193Sjb return; 1051179193Sjb 1052179193Sjb probe->ftp_enabled = 0; 1053179193Sjb 1054179193Sjb ASSERT(MUTEX_HELD(&cpu_lock)); 1055179193Sjb fasttrap_disable_callbacks(); 1056179193Sjb} 1057179193Sjb 1058179193Sjb/*ARGSUSED*/ 1059179193Sjbstatic void 1060179193Sjbfasttrap_pid_getargdesc(void *arg, dtrace_id_t id, void *parg, 1061179193Sjb dtrace_argdesc_t *desc) 1062179193Sjb{ 1063179193Sjb fasttrap_probe_t *probe = parg; 1064179193Sjb char *str; 1065179193Sjb int i, ndx; 1066179193Sjb 1067179193Sjb desc->dtargd_native[0] = '\0'; 1068179193Sjb desc->dtargd_xlate[0] = '\0'; 1069179193Sjb 1070179193Sjb if (probe->ftp_prov->ftp_retired != 0 || 1071179193Sjb desc->dtargd_ndx >= probe->ftp_nargs) { 1072179193Sjb desc->dtargd_ndx = DTRACE_ARGNONE; 1073179193Sjb return; 1074179193Sjb } 1075179193Sjb 1076179193Sjb ndx = (probe->ftp_argmap != NULL) ? 1077179193Sjb probe->ftp_argmap[desc->dtargd_ndx] : desc->dtargd_ndx; 1078179193Sjb 1079179193Sjb str = probe->ftp_ntypes; 1080179193Sjb for (i = 0; i < ndx; i++) { 1081179193Sjb str += strlen(str) + 1; 1082179193Sjb } 1083179193Sjb 1084179193Sjb ASSERT(strlen(str + 1) < sizeof (desc->dtargd_native)); 1085179193Sjb (void) strcpy(desc->dtargd_native, str); 1086179193Sjb 1087179193Sjb if (probe->ftp_xtypes == NULL) 1088179193Sjb return; 1089179193Sjb 1090179193Sjb str = probe->ftp_xtypes; 1091179193Sjb for (i = 0; i < desc->dtargd_ndx; i++) { 1092179193Sjb str += strlen(str) + 1; 1093179193Sjb } 1094179193Sjb 1095179193Sjb ASSERT(strlen(str + 1) < sizeof (desc->dtargd_xlate)); 1096179193Sjb (void) strcpy(desc->dtargd_xlate, str); 1097179193Sjb} 1098179193Sjb 1099179193Sjb/*ARGSUSED*/ 1100179193Sjbstatic void 1101179193Sjbfasttrap_pid_destroy(void *arg, dtrace_id_t id, void *parg) 1102179193Sjb{ 1103179193Sjb fasttrap_probe_t *probe = parg; 1104179193Sjb int i; 1105179193Sjb size_t size; 1106179193Sjb 1107179193Sjb ASSERT(probe != NULL); 1108179193Sjb ASSERT(!probe->ftp_enabled); 1109179193Sjb ASSERT(fasttrap_total >= probe->ftp_ntps); 1110179193Sjb 1111179193Sjb atomic_add_32(&fasttrap_total, -probe->ftp_ntps); 1112179193Sjb size = offsetof(fasttrap_probe_t, ftp_tps[probe->ftp_ntps]); 1113179193Sjb 1114179193Sjb if (probe->ftp_gen + 1 >= fasttrap_mod_gen) 1115179193Sjb fasttrap_mod_barrier(probe->ftp_gen); 1116179193Sjb 1117179193Sjb for (i = 0; i < probe->ftp_ntps; i++) { 1118179193Sjb kmem_free(probe->ftp_tps[i].fit_tp, 1119179193Sjb sizeof (fasttrap_tracepoint_t)); 1120179193Sjb } 1121179193Sjb 1122179193Sjb kmem_free(probe, size); 1123179193Sjb} 1124179193Sjb 1125179193Sjb 1126179193Sjbstatic const dtrace_pattr_t pid_attr = { 1127179193Sjb{ DTRACE_STABILITY_EVOLVING, DTRACE_STABILITY_EVOLVING, DTRACE_CLASS_ISA }, 1128179193Sjb{ DTRACE_STABILITY_PRIVATE, DTRACE_STABILITY_PRIVATE, DTRACE_CLASS_UNKNOWN }, 1129179193Sjb{ DTRACE_STABILITY_PRIVATE, DTRACE_STABILITY_PRIVATE, DTRACE_CLASS_UNKNOWN }, 1130179193Sjb{ DTRACE_STABILITY_EVOLVING, DTRACE_STABILITY_EVOLVING, DTRACE_CLASS_ISA }, 1131179193Sjb{ DTRACE_STABILITY_PRIVATE, DTRACE_STABILITY_PRIVATE, DTRACE_CLASS_UNKNOWN }, 1132179193Sjb}; 1133179193Sjb 1134179193Sjbstatic dtrace_pops_t pid_pops = { 1135179193Sjb fasttrap_pid_provide, 1136179193Sjb NULL, 1137179193Sjb fasttrap_pid_enable, 1138179193Sjb fasttrap_pid_disable, 1139179193Sjb NULL, 1140179193Sjb NULL, 1141179193Sjb fasttrap_pid_getargdesc, 1142179193Sjb fasttrap_pid_getarg, 1143179193Sjb NULL, 1144179193Sjb fasttrap_pid_destroy 1145179193Sjb}; 1146179193Sjb 1147179193Sjbstatic dtrace_pops_t usdt_pops = { 1148179193Sjb fasttrap_pid_provide, 1149179193Sjb NULL, 1150179193Sjb fasttrap_pid_enable, 1151179193Sjb fasttrap_pid_disable, 1152179193Sjb NULL, 1153179193Sjb NULL, 1154179193Sjb fasttrap_pid_getargdesc, 1155179193Sjb fasttrap_usdt_getarg, 1156179193Sjb NULL, 1157179193Sjb fasttrap_pid_destroy 1158179193Sjb}; 1159179193Sjb 1160179193Sjbstatic fasttrap_proc_t * 1161179193Sjbfasttrap_proc_lookup(pid_t pid) 1162179193Sjb{ 1163179193Sjb fasttrap_bucket_t *bucket; 1164179193Sjb fasttrap_proc_t *fprc, *new_fprc; 1165179193Sjb 1166179193Sjb bucket = &fasttrap_procs.fth_table[FASTTRAP_PROCS_INDEX(pid)]; 1167179193Sjb mutex_enter(&bucket->ftb_mtx); 1168179193Sjb 1169179193Sjb for (fprc = bucket->ftb_data; fprc != NULL; fprc = fprc->ftpc_next) { 1170179193Sjb if (fprc->ftpc_pid == pid && fprc->ftpc_acount != 0) { 1171179193Sjb mutex_enter(&fprc->ftpc_mtx); 1172179193Sjb mutex_exit(&bucket->ftb_mtx); 1173179193Sjb fprc->ftpc_rcount++; 1174179193Sjb atomic_add_64(&fprc->ftpc_acount, 1); 1175179198Sjb ASSERT(fprc->ftpc_acount <= fprc->ftpc_rcount); 1176179193Sjb mutex_exit(&fprc->ftpc_mtx); 1177179193Sjb 1178179193Sjb return (fprc); 1179179193Sjb } 1180179193Sjb } 1181179193Sjb 1182179193Sjb /* 1183179193Sjb * Drop the bucket lock so we don't try to perform a sleeping 1184179193Sjb * allocation under it. 1185179193Sjb */ 1186179193Sjb mutex_exit(&bucket->ftb_mtx); 1187179193Sjb 1188179193Sjb new_fprc = kmem_zalloc(sizeof (fasttrap_proc_t), KM_SLEEP); 1189179193Sjb new_fprc->ftpc_pid = pid; 1190179193Sjb new_fprc->ftpc_rcount = 1; 1191179193Sjb new_fprc->ftpc_acount = 1; 1192179193Sjb 1193179193Sjb mutex_enter(&bucket->ftb_mtx); 1194179193Sjb 1195179193Sjb /* 1196179193Sjb * Take another lap through the list to make sure a proc hasn't 1197179193Sjb * been created for this pid while we weren't under the bucket lock. 1198179193Sjb */ 1199179193Sjb for (fprc = bucket->ftb_data; fprc != NULL; fprc = fprc->ftpc_next) { 1200179193Sjb if (fprc->ftpc_pid == pid && fprc->ftpc_acount != 0) { 1201179193Sjb mutex_enter(&fprc->ftpc_mtx); 1202179193Sjb mutex_exit(&bucket->ftb_mtx); 1203179193Sjb fprc->ftpc_rcount++; 1204179193Sjb atomic_add_64(&fprc->ftpc_acount, 1); 1205179198Sjb ASSERT(fprc->ftpc_acount <= fprc->ftpc_rcount); 1206179193Sjb mutex_exit(&fprc->ftpc_mtx); 1207179193Sjb 1208179193Sjb kmem_free(new_fprc, sizeof (fasttrap_proc_t)); 1209179193Sjb 1210179193Sjb return (fprc); 1211179193Sjb } 1212179193Sjb } 1213179193Sjb 1214179193Sjb new_fprc->ftpc_next = bucket->ftb_data; 1215179193Sjb bucket->ftb_data = new_fprc; 1216179193Sjb 1217179193Sjb mutex_exit(&bucket->ftb_mtx); 1218179193Sjb 1219179193Sjb return (new_fprc); 1220179193Sjb} 1221179193Sjb 1222179193Sjbstatic void 1223179193Sjbfasttrap_proc_release(fasttrap_proc_t *proc) 1224179193Sjb{ 1225179193Sjb fasttrap_bucket_t *bucket; 1226179193Sjb fasttrap_proc_t *fprc, **fprcp; 1227179193Sjb pid_t pid = proc->ftpc_pid; 1228179193Sjb 1229179193Sjb mutex_enter(&proc->ftpc_mtx); 1230179193Sjb 1231179193Sjb ASSERT(proc->ftpc_rcount != 0); 1232179198Sjb ASSERT(proc->ftpc_acount <= proc->ftpc_rcount); 1233179193Sjb 1234179193Sjb if (--proc->ftpc_rcount != 0) { 1235179193Sjb mutex_exit(&proc->ftpc_mtx); 1236179193Sjb return; 1237179193Sjb } 1238179193Sjb 1239179193Sjb mutex_exit(&proc->ftpc_mtx); 1240179193Sjb 1241179193Sjb /* 1242179193Sjb * There should definitely be no live providers associated with this 1243179193Sjb * process at this point. 1244179193Sjb */ 1245179193Sjb ASSERT(proc->ftpc_acount == 0); 1246179193Sjb 1247179193Sjb bucket = &fasttrap_procs.fth_table[FASTTRAP_PROCS_INDEX(pid)]; 1248179193Sjb mutex_enter(&bucket->ftb_mtx); 1249179193Sjb 1250179193Sjb fprcp = (fasttrap_proc_t **)&bucket->ftb_data; 1251179193Sjb while ((fprc = *fprcp) != NULL) { 1252179193Sjb if (fprc == proc) 1253179193Sjb break; 1254179193Sjb 1255179193Sjb fprcp = &fprc->ftpc_next; 1256179193Sjb } 1257179193Sjb 1258179193Sjb /* 1259179193Sjb * Something strange has happened if we can't find the proc. 1260179193Sjb */ 1261179193Sjb ASSERT(fprc != NULL); 1262179193Sjb 1263179193Sjb *fprcp = fprc->ftpc_next; 1264179193Sjb 1265179193Sjb mutex_exit(&bucket->ftb_mtx); 1266179193Sjb 1267179193Sjb kmem_free(fprc, sizeof (fasttrap_proc_t)); 1268179193Sjb} 1269179193Sjb 1270179193Sjb/* 1271179193Sjb * Lookup a fasttrap-managed provider based on its name and associated pid. 1272179193Sjb * If the pattr argument is non-NULL, this function instantiates the provider 1273179193Sjb * if it doesn't exist otherwise it returns NULL. The provider is returned 1274179193Sjb * with its lock held. 1275179193Sjb */ 1276179193Sjbstatic fasttrap_provider_t * 1277179193Sjbfasttrap_provider_lookup(pid_t pid, const char *name, 1278179193Sjb const dtrace_pattr_t *pattr) 1279179193Sjb{ 1280179193Sjb fasttrap_provider_t *fp, *new_fp = NULL; 1281179193Sjb fasttrap_bucket_t *bucket; 1282179193Sjb char provname[DTRACE_PROVNAMELEN]; 1283179193Sjb proc_t *p; 1284179193Sjb cred_t *cred; 1285179193Sjb 1286179193Sjb ASSERT(strlen(name) < sizeof (fp->ftp_name)); 1287179193Sjb ASSERT(pattr != NULL); 1288179193Sjb 1289179193Sjb bucket = &fasttrap_provs.fth_table[FASTTRAP_PROVS_INDEX(pid, name)]; 1290179193Sjb mutex_enter(&bucket->ftb_mtx); 1291179193Sjb 1292179193Sjb /* 1293179193Sjb * Take a lap through the list and return the match if we find it. 1294179193Sjb */ 1295179193Sjb for (fp = bucket->ftb_data; fp != NULL; fp = fp->ftp_next) { 1296179193Sjb if (fp->ftp_pid == pid && strcmp(fp->ftp_name, name) == 0 && 1297179193Sjb !fp->ftp_retired) { 1298179193Sjb mutex_enter(&fp->ftp_mtx); 1299179193Sjb mutex_exit(&bucket->ftb_mtx); 1300179193Sjb return (fp); 1301179193Sjb } 1302179193Sjb } 1303179193Sjb 1304179193Sjb /* 1305179193Sjb * Drop the bucket lock so we don't try to perform a sleeping 1306179193Sjb * allocation under it. 1307179193Sjb */ 1308179193Sjb mutex_exit(&bucket->ftb_mtx); 1309179193Sjb 1310179193Sjb /* 1311179193Sjb * Make sure the process exists, isn't a child created as the result 1312179193Sjb * of a vfork(2), and isn't a zombie (but may be in fork). 1313179193Sjb */ 1314179193Sjb mutex_enter(&pidlock); 1315179193Sjb if ((p = prfind(pid)) == NULL) { 1316179193Sjb mutex_exit(&pidlock); 1317179193Sjb return (NULL); 1318179193Sjb } 1319179193Sjb mutex_enter(&p->p_lock); 1320179193Sjb mutex_exit(&pidlock); 1321179193Sjb if (p->p_flag & (SVFORK | SEXITING)) { 1322179193Sjb mutex_exit(&p->p_lock); 1323179193Sjb return (NULL); 1324179193Sjb } 1325179193Sjb 1326179193Sjb /* 1327179193Sjb * Increment p_dtrace_probes so that the process knows to inform us 1328179193Sjb * when it exits or execs. fasttrap_provider_free() decrements this 1329179193Sjb * when we're done with this provider. 1330179193Sjb */ 1331179193Sjb p->p_dtrace_probes++; 1332179193Sjb 1333179193Sjb /* 1334179193Sjb * Grab the credentials for this process so we have 1335179193Sjb * something to pass to dtrace_register(). 1336179193Sjb */ 1337179193Sjb mutex_enter(&p->p_crlock); 1338179193Sjb crhold(p->p_cred); 1339179193Sjb cred = p->p_cred; 1340179193Sjb mutex_exit(&p->p_crlock); 1341179193Sjb mutex_exit(&p->p_lock); 1342179193Sjb 1343179193Sjb new_fp = kmem_zalloc(sizeof (fasttrap_provider_t), KM_SLEEP); 1344179193Sjb new_fp->ftp_pid = pid; 1345179193Sjb new_fp->ftp_proc = fasttrap_proc_lookup(pid); 1346179193Sjb 1347179193Sjb ASSERT(new_fp->ftp_proc != NULL); 1348179193Sjb 1349179193Sjb mutex_enter(&bucket->ftb_mtx); 1350179193Sjb 1351179193Sjb /* 1352179193Sjb * Take another lap through the list to make sure a provider hasn't 1353179193Sjb * been created for this pid while we weren't under the bucket lock. 1354179193Sjb */ 1355179193Sjb for (fp = bucket->ftb_data; fp != NULL; fp = fp->ftp_next) { 1356179193Sjb if (fp->ftp_pid == pid && strcmp(fp->ftp_name, name) == 0 && 1357179193Sjb !fp->ftp_retired) { 1358179193Sjb mutex_enter(&fp->ftp_mtx); 1359179193Sjb mutex_exit(&bucket->ftb_mtx); 1360179193Sjb fasttrap_provider_free(new_fp); 1361179193Sjb crfree(cred); 1362179193Sjb return (fp); 1363179193Sjb } 1364179193Sjb } 1365179193Sjb 1366179193Sjb (void) strcpy(new_fp->ftp_name, name); 1367179193Sjb 1368179193Sjb /* 1369179193Sjb * Fail and return NULL if either the provider name is too long 1370179193Sjb * or we fail to register this new provider with the DTrace 1371179193Sjb * framework. Note that this is the only place we ever construct 1372179193Sjb * the full provider name -- we keep it in pieces in the provider 1373179193Sjb * structure. 1374179193Sjb */ 1375179193Sjb if (snprintf(provname, sizeof (provname), "%s%u", name, (uint_t)pid) >= 1376179193Sjb sizeof (provname) || 1377179193Sjb dtrace_register(provname, pattr, 1378179193Sjb DTRACE_PRIV_PROC | DTRACE_PRIV_OWNER | DTRACE_PRIV_ZONEOWNER, cred, 1379179193Sjb pattr == &pid_attr ? &pid_pops : &usdt_pops, new_fp, 1380179193Sjb &new_fp->ftp_provid) != 0) { 1381179193Sjb mutex_exit(&bucket->ftb_mtx); 1382179193Sjb fasttrap_provider_free(new_fp); 1383179193Sjb crfree(cred); 1384179193Sjb return (NULL); 1385179193Sjb } 1386179193Sjb 1387179193Sjb new_fp->ftp_next = bucket->ftb_data; 1388179193Sjb bucket->ftb_data = new_fp; 1389179193Sjb 1390179193Sjb mutex_enter(&new_fp->ftp_mtx); 1391179193Sjb mutex_exit(&bucket->ftb_mtx); 1392179193Sjb 1393179193Sjb crfree(cred); 1394179193Sjb return (new_fp); 1395179193Sjb} 1396179193Sjb 1397179193Sjbstatic void 1398179193Sjbfasttrap_provider_free(fasttrap_provider_t *provider) 1399179193Sjb{ 1400179193Sjb pid_t pid = provider->ftp_pid; 1401179193Sjb proc_t *p; 1402179193Sjb 1403179193Sjb /* 1404179193Sjb * There need to be no associated enabled probes, no consumers 1405179193Sjb * creating probes, and no meta providers referencing this provider. 1406179193Sjb */ 1407179193Sjb ASSERT(provider->ftp_rcount == 0); 1408179193Sjb ASSERT(provider->ftp_ccount == 0); 1409179193Sjb ASSERT(provider->ftp_mcount == 0); 1410179193Sjb 1411179198Sjb /* 1412179198Sjb * If this provider hasn't been retired, we need to explicitly drop the 1413179198Sjb * count of active providers on the associated process structure. 1414179198Sjb */ 1415179198Sjb if (!provider->ftp_retired) { 1416179198Sjb atomic_add_64(&provider->ftp_proc->ftpc_acount, -1); 1417179198Sjb ASSERT(provider->ftp_proc->ftpc_acount < 1418179198Sjb provider->ftp_proc->ftpc_rcount); 1419179198Sjb } 1420179198Sjb 1421179193Sjb fasttrap_proc_release(provider->ftp_proc); 1422179193Sjb 1423179193Sjb kmem_free(provider, sizeof (fasttrap_provider_t)); 1424179193Sjb 1425179193Sjb /* 1426179193Sjb * Decrement p_dtrace_probes on the process whose provider we're 1427179193Sjb * freeing. We don't have to worry about clobbering somone else's 1428179193Sjb * modifications to it because we have locked the bucket that 1429179193Sjb * corresponds to this process's hash chain in the provider hash 1430179193Sjb * table. Don't sweat it if we can't find the process. 1431179193Sjb */ 1432179193Sjb mutex_enter(&pidlock); 1433179193Sjb if ((p = prfind(pid)) == NULL) { 1434179193Sjb mutex_exit(&pidlock); 1435179193Sjb return; 1436179193Sjb } 1437179193Sjb 1438179193Sjb mutex_enter(&p->p_lock); 1439179193Sjb mutex_exit(&pidlock); 1440179193Sjb 1441179193Sjb p->p_dtrace_probes--; 1442179193Sjb mutex_exit(&p->p_lock); 1443179193Sjb} 1444179193Sjb 1445179193Sjbstatic void 1446179193Sjbfasttrap_provider_retire(pid_t pid, const char *name, int mprov) 1447179193Sjb{ 1448179193Sjb fasttrap_provider_t *fp; 1449179193Sjb fasttrap_bucket_t *bucket; 1450179193Sjb dtrace_provider_id_t provid; 1451179193Sjb 1452179193Sjb ASSERT(strlen(name) < sizeof (fp->ftp_name)); 1453179193Sjb 1454179193Sjb bucket = &fasttrap_provs.fth_table[FASTTRAP_PROVS_INDEX(pid, name)]; 1455179193Sjb mutex_enter(&bucket->ftb_mtx); 1456179193Sjb 1457179193Sjb for (fp = bucket->ftb_data; fp != NULL; fp = fp->ftp_next) { 1458179193Sjb if (fp->ftp_pid == pid && strcmp(fp->ftp_name, name) == 0 && 1459179193Sjb !fp->ftp_retired) 1460179193Sjb break; 1461179193Sjb } 1462179193Sjb 1463179193Sjb if (fp == NULL) { 1464179193Sjb mutex_exit(&bucket->ftb_mtx); 1465179193Sjb return; 1466179193Sjb } 1467179193Sjb 1468179193Sjb mutex_enter(&fp->ftp_mtx); 1469179193Sjb ASSERT(!mprov || fp->ftp_mcount > 0); 1470179193Sjb if (mprov && --fp->ftp_mcount != 0) { 1471179193Sjb mutex_exit(&fp->ftp_mtx); 1472179193Sjb mutex_exit(&bucket->ftb_mtx); 1473179193Sjb return; 1474179193Sjb } 1475179193Sjb 1476179193Sjb /* 1477179193Sjb * Mark the provider to be removed in our post-processing step, mark it 1478179193Sjb * retired, and drop the active count on its proc. Marking it indicates 1479179193Sjb * that we should try to remove it; setting the retired flag indicates 1480179193Sjb * that we're done with this provider; dropping the active the proc 1481179193Sjb * releases our hold, and when this reaches zero (as it will during 1482179193Sjb * exit or exec) the proc and associated providers become defunct. 1483179193Sjb * 1484179193Sjb * We obviously need to take the bucket lock before the provider lock 1485179193Sjb * to perform the lookup, but we need to drop the provider lock 1486179193Sjb * before calling into the DTrace framework since we acquire the 1487179193Sjb * provider lock in callbacks invoked from the DTrace framework. The 1488179193Sjb * bucket lock therefore protects the integrity of the provider hash 1489179193Sjb * table. 1490179193Sjb */ 1491179193Sjb atomic_add_64(&fp->ftp_proc->ftpc_acount, -1); 1492179198Sjb ASSERT(fp->ftp_proc->ftpc_acount < fp->ftp_proc->ftpc_rcount); 1493179198Sjb 1494179193Sjb fp->ftp_retired = 1; 1495179193Sjb fp->ftp_marked = 1; 1496179193Sjb provid = fp->ftp_provid; 1497179193Sjb mutex_exit(&fp->ftp_mtx); 1498179193Sjb 1499179193Sjb /* 1500179193Sjb * We don't have to worry about invalidating the same provider twice 1501179193Sjb * since fasttrap_provider_lookup() will ignore provider that have 1502179193Sjb * been marked as retired. 1503179193Sjb */ 1504179193Sjb dtrace_invalidate(provid); 1505179193Sjb 1506179193Sjb mutex_exit(&bucket->ftb_mtx); 1507179193Sjb 1508179193Sjb fasttrap_pid_cleanup(); 1509179193Sjb} 1510179193Sjb 1511179193Sjbstatic int 1512179193Sjbfasttrap_uint32_cmp(const void *ap, const void *bp) 1513179193Sjb{ 1514179193Sjb return (*(const uint32_t *)ap - *(const uint32_t *)bp); 1515179193Sjb} 1516179193Sjb 1517179193Sjbstatic int 1518179193Sjbfasttrap_uint64_cmp(const void *ap, const void *bp) 1519179193Sjb{ 1520179193Sjb return (*(const uint64_t *)ap - *(const uint64_t *)bp); 1521179193Sjb} 1522179193Sjb 1523179193Sjbstatic int 1524179193Sjbfasttrap_add_probe(fasttrap_probe_spec_t *pdata) 1525179193Sjb{ 1526179193Sjb fasttrap_provider_t *provider; 1527179193Sjb fasttrap_probe_t *pp; 1528179193Sjb fasttrap_tracepoint_t *tp; 1529179193Sjb char *name; 1530179193Sjb int i, aframes, whack; 1531179193Sjb 1532179193Sjb /* 1533179193Sjb * There needs to be at least one desired trace point. 1534179193Sjb */ 1535179193Sjb if (pdata->ftps_noffs == 0) 1536179193Sjb return (EINVAL); 1537179193Sjb 1538179193Sjb switch (pdata->ftps_type) { 1539179193Sjb case DTFTP_ENTRY: 1540179193Sjb name = "entry"; 1541179193Sjb aframes = FASTTRAP_ENTRY_AFRAMES; 1542179193Sjb break; 1543179193Sjb case DTFTP_RETURN: 1544179193Sjb name = "return"; 1545179193Sjb aframes = FASTTRAP_RETURN_AFRAMES; 1546179193Sjb break; 1547179193Sjb case DTFTP_OFFSETS: 1548179193Sjb name = NULL; 1549179193Sjb break; 1550179193Sjb default: 1551179193Sjb return (EINVAL); 1552179193Sjb } 1553179193Sjb 1554179193Sjb if ((provider = fasttrap_provider_lookup(pdata->ftps_pid, 1555179193Sjb FASTTRAP_PID_NAME, &pid_attr)) == NULL) 1556179193Sjb return (ESRCH); 1557179193Sjb 1558179193Sjb /* 1559179193Sjb * Increment this reference count to indicate that a consumer is 1560179193Sjb * actively adding a new probe associated with this provider. This 1561179193Sjb * prevents the provider from being deleted -- we'll need to check 1562179193Sjb * for pending deletions when we drop this reference count. 1563179193Sjb */ 1564179193Sjb provider->ftp_ccount++; 1565179193Sjb mutex_exit(&provider->ftp_mtx); 1566179193Sjb 1567179193Sjb /* 1568179193Sjb * Grab the creation lock to ensure consistency between calls to 1569179193Sjb * dtrace_probe_lookup() and dtrace_probe_create() in the face of 1570179193Sjb * other threads creating probes. We must drop the provider lock 1571179193Sjb * before taking this lock to avoid a three-way deadlock with the 1572179193Sjb * DTrace framework. 1573179193Sjb */ 1574179193Sjb mutex_enter(&provider->ftp_cmtx); 1575179193Sjb 1576179193Sjb if (name == NULL) { 1577179193Sjb for (i = 0; i < pdata->ftps_noffs; i++) { 1578179193Sjb char name_str[17]; 1579179193Sjb 1580179193Sjb (void) sprintf(name_str, "%llx", 1581179193Sjb (unsigned long long)pdata->ftps_offs[i]); 1582179193Sjb 1583179193Sjb if (dtrace_probe_lookup(provider->ftp_provid, 1584179193Sjb pdata->ftps_mod, pdata->ftps_func, name_str) != 0) 1585179193Sjb continue; 1586179193Sjb 1587179193Sjb atomic_add_32(&fasttrap_total, 1); 1588179193Sjb 1589179193Sjb if (fasttrap_total > fasttrap_max) { 1590179193Sjb atomic_add_32(&fasttrap_total, -1); 1591179193Sjb goto no_mem; 1592179193Sjb } 1593179193Sjb 1594179193Sjb pp = kmem_zalloc(sizeof (fasttrap_probe_t), KM_SLEEP); 1595179193Sjb 1596179193Sjb pp->ftp_prov = provider; 1597179193Sjb pp->ftp_faddr = pdata->ftps_pc; 1598179193Sjb pp->ftp_fsize = pdata->ftps_size; 1599179193Sjb pp->ftp_pid = pdata->ftps_pid; 1600179193Sjb pp->ftp_ntps = 1; 1601179193Sjb 1602179193Sjb tp = kmem_zalloc(sizeof (fasttrap_tracepoint_t), 1603179193Sjb KM_SLEEP); 1604179193Sjb 1605179193Sjb tp->ftt_proc = provider->ftp_proc; 1606179193Sjb tp->ftt_pc = pdata->ftps_offs[i] + pdata->ftps_pc; 1607179193Sjb tp->ftt_pid = pdata->ftps_pid; 1608179193Sjb 1609179193Sjb pp->ftp_tps[0].fit_tp = tp; 1610179193Sjb pp->ftp_tps[0].fit_id.fti_probe = pp; 1611179193Sjb pp->ftp_tps[0].fit_id.fti_ptype = pdata->ftps_type; 1612179193Sjb 1613179193Sjb pp->ftp_id = dtrace_probe_create(provider->ftp_provid, 1614179193Sjb pdata->ftps_mod, pdata->ftps_func, name_str, 1615179193Sjb FASTTRAP_OFFSET_AFRAMES, pp); 1616179193Sjb } 1617179193Sjb 1618179193Sjb } else if (dtrace_probe_lookup(provider->ftp_provid, pdata->ftps_mod, 1619179193Sjb pdata->ftps_func, name) == 0) { 1620179193Sjb atomic_add_32(&fasttrap_total, pdata->ftps_noffs); 1621179193Sjb 1622179193Sjb if (fasttrap_total > fasttrap_max) { 1623179193Sjb atomic_add_32(&fasttrap_total, -pdata->ftps_noffs); 1624179193Sjb goto no_mem; 1625179193Sjb } 1626179193Sjb 1627179193Sjb /* 1628179193Sjb * Make sure all tracepoint program counter values are unique. 1629179193Sjb * We later assume that each probe has exactly one tracepoint 1630179193Sjb * for a given pc. 1631179193Sjb */ 1632179193Sjb qsort(pdata->ftps_offs, pdata->ftps_noffs, 1633179193Sjb sizeof (uint64_t), fasttrap_uint64_cmp); 1634179193Sjb for (i = 1; i < pdata->ftps_noffs; i++) { 1635179193Sjb if (pdata->ftps_offs[i] > pdata->ftps_offs[i - 1]) 1636179193Sjb continue; 1637179193Sjb 1638179193Sjb atomic_add_32(&fasttrap_total, -pdata->ftps_noffs); 1639179193Sjb goto no_mem; 1640179193Sjb } 1641179193Sjb 1642179193Sjb ASSERT(pdata->ftps_noffs > 0); 1643179193Sjb pp = kmem_zalloc(offsetof(fasttrap_probe_t, 1644179193Sjb ftp_tps[pdata->ftps_noffs]), KM_SLEEP); 1645179193Sjb 1646179193Sjb pp->ftp_prov = provider; 1647179193Sjb pp->ftp_faddr = pdata->ftps_pc; 1648179193Sjb pp->ftp_fsize = pdata->ftps_size; 1649179193Sjb pp->ftp_pid = pdata->ftps_pid; 1650179193Sjb pp->ftp_ntps = pdata->ftps_noffs; 1651179193Sjb 1652179193Sjb for (i = 0; i < pdata->ftps_noffs; i++) { 1653179193Sjb tp = kmem_zalloc(sizeof (fasttrap_tracepoint_t), 1654179193Sjb KM_SLEEP); 1655179193Sjb 1656179193Sjb tp->ftt_proc = provider->ftp_proc; 1657179193Sjb tp->ftt_pc = pdata->ftps_offs[i] + pdata->ftps_pc; 1658179193Sjb tp->ftt_pid = pdata->ftps_pid; 1659179193Sjb 1660179193Sjb pp->ftp_tps[i].fit_tp = tp; 1661179193Sjb pp->ftp_tps[i].fit_id.fti_probe = pp; 1662179193Sjb pp->ftp_tps[i].fit_id.fti_ptype = pdata->ftps_type; 1663179193Sjb } 1664179193Sjb 1665179193Sjb pp->ftp_id = dtrace_probe_create(provider->ftp_provid, 1666179193Sjb pdata->ftps_mod, pdata->ftps_func, name, aframes, pp); 1667179193Sjb } 1668179193Sjb 1669179193Sjb mutex_exit(&provider->ftp_cmtx); 1670179193Sjb 1671179193Sjb /* 1672179193Sjb * We know that the provider is still valid since we incremented the 1673179193Sjb * creation reference count. If someone tried to clean up this provider 1674179193Sjb * while we were using it (e.g. because the process called exec(2) or 1675179193Sjb * exit(2)), take note of that and try to clean it up now. 1676179193Sjb */ 1677179193Sjb mutex_enter(&provider->ftp_mtx); 1678179193Sjb provider->ftp_ccount--; 1679179193Sjb whack = provider->ftp_retired; 1680179193Sjb mutex_exit(&provider->ftp_mtx); 1681179193Sjb 1682179193Sjb if (whack) 1683179193Sjb fasttrap_pid_cleanup(); 1684179193Sjb 1685179193Sjb return (0); 1686179193Sjb 1687179193Sjbno_mem: 1688179193Sjb /* 1689179193Sjb * If we've exhausted the allowable resources, we'll try to remove 1690179193Sjb * this provider to free some up. This is to cover the case where 1691179193Sjb * the user has accidentally created many more probes than was 1692179193Sjb * intended (e.g. pid123:::). 1693179193Sjb */ 1694179193Sjb mutex_exit(&provider->ftp_cmtx); 1695179193Sjb mutex_enter(&provider->ftp_mtx); 1696179193Sjb provider->ftp_ccount--; 1697179193Sjb provider->ftp_marked = 1; 1698179193Sjb mutex_exit(&provider->ftp_mtx); 1699179193Sjb 1700179193Sjb fasttrap_pid_cleanup(); 1701179193Sjb 1702179193Sjb return (ENOMEM); 1703179193Sjb} 1704179193Sjb 1705179193Sjb/*ARGSUSED*/ 1706179193Sjbstatic void * 1707179193Sjbfasttrap_meta_provide(void *arg, dtrace_helper_provdesc_t *dhpv, pid_t pid) 1708179193Sjb{ 1709179193Sjb fasttrap_provider_t *provider; 1710179193Sjb 1711179193Sjb /* 1712179193Sjb * A 32-bit unsigned integer (like a pid for example) can be 1713179193Sjb * expressed in 10 or fewer decimal digits. Make sure that we'll 1714179193Sjb * have enough space for the provider name. 1715179193Sjb */ 1716179193Sjb if (strlen(dhpv->dthpv_provname) + 10 >= 1717179193Sjb sizeof (provider->ftp_name)) { 1718179193Sjb cmn_err(CE_WARN, "failed to instantiate provider %s: " 1719179193Sjb "name too long to accomodate pid", dhpv->dthpv_provname); 1720179193Sjb return (NULL); 1721179193Sjb } 1722179193Sjb 1723179193Sjb /* 1724179193Sjb * Don't let folks spoof the true pid provider. 1725179193Sjb */ 1726179193Sjb if (strcmp(dhpv->dthpv_provname, FASTTRAP_PID_NAME) == 0) { 1727179193Sjb cmn_err(CE_WARN, "failed to instantiate provider %s: " 1728179193Sjb "%s is an invalid name", dhpv->dthpv_provname, 1729179193Sjb FASTTRAP_PID_NAME); 1730179193Sjb return (NULL); 1731179193Sjb } 1732179193Sjb 1733179193Sjb /* 1734179193Sjb * The highest stability class that fasttrap supports is ISA; cap 1735179193Sjb * the stability of the new provider accordingly. 1736179193Sjb */ 1737179193Sjb if (dhpv->dthpv_pattr.dtpa_provider.dtat_class > DTRACE_CLASS_ISA) 1738179193Sjb dhpv->dthpv_pattr.dtpa_provider.dtat_class = DTRACE_CLASS_ISA; 1739179193Sjb if (dhpv->dthpv_pattr.dtpa_mod.dtat_class > DTRACE_CLASS_ISA) 1740179193Sjb dhpv->dthpv_pattr.dtpa_mod.dtat_class = DTRACE_CLASS_ISA; 1741179193Sjb if (dhpv->dthpv_pattr.dtpa_func.dtat_class > DTRACE_CLASS_ISA) 1742179193Sjb dhpv->dthpv_pattr.dtpa_func.dtat_class = DTRACE_CLASS_ISA; 1743179193Sjb if (dhpv->dthpv_pattr.dtpa_name.dtat_class > DTRACE_CLASS_ISA) 1744179193Sjb dhpv->dthpv_pattr.dtpa_name.dtat_class = DTRACE_CLASS_ISA; 1745179193Sjb if (dhpv->dthpv_pattr.dtpa_args.dtat_class > DTRACE_CLASS_ISA) 1746179193Sjb dhpv->dthpv_pattr.dtpa_args.dtat_class = DTRACE_CLASS_ISA; 1747179193Sjb 1748179193Sjb if ((provider = fasttrap_provider_lookup(pid, dhpv->dthpv_provname, 1749179193Sjb &dhpv->dthpv_pattr)) == NULL) { 1750179193Sjb cmn_err(CE_WARN, "failed to instantiate provider %s for " 1751179193Sjb "process %u", dhpv->dthpv_provname, (uint_t)pid); 1752179193Sjb return (NULL); 1753179193Sjb } 1754179193Sjb 1755179193Sjb /* 1756179193Sjb * Up the meta provider count so this provider isn't removed until 1757179193Sjb * the meta provider has been told to remove it. 1758179193Sjb */ 1759179193Sjb provider->ftp_mcount++; 1760179193Sjb 1761179193Sjb mutex_exit(&provider->ftp_mtx); 1762179193Sjb 1763179193Sjb return (provider); 1764179193Sjb} 1765179193Sjb 1766179193Sjb/*ARGSUSED*/ 1767179193Sjbstatic void 1768179193Sjbfasttrap_meta_create_probe(void *arg, void *parg, 1769179193Sjb dtrace_helper_probedesc_t *dhpb) 1770179193Sjb{ 1771179193Sjb fasttrap_provider_t *provider = parg; 1772179193Sjb fasttrap_probe_t *pp; 1773179193Sjb fasttrap_tracepoint_t *tp; 1774179193Sjb int i, j; 1775179193Sjb uint32_t ntps; 1776179193Sjb 1777179193Sjb /* 1778179193Sjb * Since the meta provider count is non-zero we don't have to worry 1779179193Sjb * about this provider disappearing. 1780179193Sjb */ 1781179193Sjb ASSERT(provider->ftp_mcount > 0); 1782179193Sjb 1783179193Sjb /* 1784179193Sjb * The offsets must be unique. 1785179193Sjb */ 1786179193Sjb qsort(dhpb->dthpb_offs, dhpb->dthpb_noffs, sizeof (uint32_t), 1787179193Sjb fasttrap_uint32_cmp); 1788179193Sjb for (i = 1; i < dhpb->dthpb_noffs; i++) { 1789179193Sjb if (dhpb->dthpb_base + dhpb->dthpb_offs[i] <= 1790179193Sjb dhpb->dthpb_base + dhpb->dthpb_offs[i - 1]) 1791179193Sjb return; 1792179193Sjb } 1793179193Sjb 1794179193Sjb qsort(dhpb->dthpb_enoffs, dhpb->dthpb_nenoffs, sizeof (uint32_t), 1795179193Sjb fasttrap_uint32_cmp); 1796179193Sjb for (i = 1; i < dhpb->dthpb_nenoffs; i++) { 1797179193Sjb if (dhpb->dthpb_base + dhpb->dthpb_enoffs[i] <= 1798179193Sjb dhpb->dthpb_base + dhpb->dthpb_enoffs[i - 1]) 1799179193Sjb return; 1800179193Sjb } 1801179193Sjb 1802179193Sjb /* 1803179193Sjb * Grab the creation lock to ensure consistency between calls to 1804179193Sjb * dtrace_probe_lookup() and dtrace_probe_create() in the face of 1805179193Sjb * other threads creating probes. 1806179193Sjb */ 1807179193Sjb mutex_enter(&provider->ftp_cmtx); 1808179193Sjb 1809179193Sjb if (dtrace_probe_lookup(provider->ftp_provid, dhpb->dthpb_mod, 1810179193Sjb dhpb->dthpb_func, dhpb->dthpb_name) != 0) { 1811179193Sjb mutex_exit(&provider->ftp_cmtx); 1812179193Sjb return; 1813179193Sjb } 1814179193Sjb 1815179193Sjb ntps = dhpb->dthpb_noffs + dhpb->dthpb_nenoffs; 1816179193Sjb ASSERT(ntps > 0); 1817179193Sjb 1818179193Sjb atomic_add_32(&fasttrap_total, ntps); 1819179193Sjb 1820179193Sjb if (fasttrap_total > fasttrap_max) { 1821179193Sjb atomic_add_32(&fasttrap_total, -ntps); 1822179193Sjb mutex_exit(&provider->ftp_cmtx); 1823179193Sjb return; 1824179193Sjb } 1825179193Sjb 1826179193Sjb pp = kmem_zalloc(offsetof(fasttrap_probe_t, ftp_tps[ntps]), KM_SLEEP); 1827179193Sjb 1828179193Sjb pp->ftp_prov = provider; 1829179193Sjb pp->ftp_pid = provider->ftp_pid; 1830179193Sjb pp->ftp_ntps = ntps; 1831179193Sjb pp->ftp_nargs = dhpb->dthpb_xargc; 1832179193Sjb pp->ftp_xtypes = dhpb->dthpb_xtypes; 1833179193Sjb pp->ftp_ntypes = dhpb->dthpb_ntypes; 1834179193Sjb 1835179193Sjb /* 1836179193Sjb * First create a tracepoint for each actual point of interest. 1837179193Sjb */ 1838179193Sjb for (i = 0; i < dhpb->dthpb_noffs; i++) { 1839179193Sjb tp = kmem_zalloc(sizeof (fasttrap_tracepoint_t), KM_SLEEP); 1840179193Sjb 1841179193Sjb tp->ftt_proc = provider->ftp_proc; 1842179193Sjb tp->ftt_pc = dhpb->dthpb_base + dhpb->dthpb_offs[i]; 1843179193Sjb tp->ftt_pid = provider->ftp_pid; 1844179193Sjb 1845179193Sjb pp->ftp_tps[i].fit_tp = tp; 1846179193Sjb pp->ftp_tps[i].fit_id.fti_probe = pp; 1847179193Sjb#ifdef __sparc 1848179193Sjb pp->ftp_tps[i].fit_id.fti_ptype = DTFTP_POST_OFFSETS; 1849179193Sjb#else 1850179193Sjb pp->ftp_tps[i].fit_id.fti_ptype = DTFTP_OFFSETS; 1851179193Sjb#endif 1852179193Sjb } 1853179193Sjb 1854179193Sjb /* 1855179193Sjb * Then create a tracepoint for each is-enabled point. 1856179193Sjb */ 1857179193Sjb for (j = 0; i < ntps; i++, j++) { 1858179193Sjb tp = kmem_zalloc(sizeof (fasttrap_tracepoint_t), KM_SLEEP); 1859179193Sjb 1860179193Sjb tp->ftt_proc = provider->ftp_proc; 1861179193Sjb tp->ftt_pc = dhpb->dthpb_base + dhpb->dthpb_enoffs[j]; 1862179193Sjb tp->ftt_pid = provider->ftp_pid; 1863179193Sjb 1864179193Sjb pp->ftp_tps[i].fit_tp = tp; 1865179193Sjb pp->ftp_tps[i].fit_id.fti_probe = pp; 1866179193Sjb pp->ftp_tps[i].fit_id.fti_ptype = DTFTP_IS_ENABLED; 1867179193Sjb } 1868179193Sjb 1869179193Sjb /* 1870179193Sjb * If the arguments are shuffled around we set the argument remapping 1871179193Sjb * table. Later, when the probe fires, we only remap the arguments 1872179193Sjb * if the table is non-NULL. 1873179193Sjb */ 1874179193Sjb for (i = 0; i < dhpb->dthpb_xargc; i++) { 1875179193Sjb if (dhpb->dthpb_args[i] != i) { 1876179193Sjb pp->ftp_argmap = dhpb->dthpb_args; 1877179193Sjb break; 1878179193Sjb } 1879179193Sjb } 1880179193Sjb 1881179193Sjb /* 1882179193Sjb * The probe is fully constructed -- register it with DTrace. 1883179193Sjb */ 1884179193Sjb pp->ftp_id = dtrace_probe_create(provider->ftp_provid, dhpb->dthpb_mod, 1885179193Sjb dhpb->dthpb_func, dhpb->dthpb_name, FASTTRAP_OFFSET_AFRAMES, pp); 1886179193Sjb 1887179193Sjb mutex_exit(&provider->ftp_cmtx); 1888179193Sjb} 1889179193Sjb 1890179193Sjb/*ARGSUSED*/ 1891179193Sjbstatic void 1892179193Sjbfasttrap_meta_remove(void *arg, dtrace_helper_provdesc_t *dhpv, pid_t pid) 1893179193Sjb{ 1894179193Sjb /* 1895179193Sjb * Clean up the USDT provider. There may be active consumers of the 1896179193Sjb * provider busy adding probes, no damage will actually befall the 1897179193Sjb * provider until that count has dropped to zero. This just puts 1898179193Sjb * the provider on death row. 1899179193Sjb */ 1900179193Sjb fasttrap_provider_retire(pid, dhpv->dthpv_provname, 1); 1901179193Sjb} 1902179193Sjb 1903179193Sjbstatic dtrace_mops_t fasttrap_mops = { 1904179193Sjb fasttrap_meta_create_probe, 1905179193Sjb fasttrap_meta_provide, 1906179193Sjb fasttrap_meta_remove 1907179193Sjb}; 1908179193Sjb 1909179193Sjb/*ARGSUSED*/ 1910179193Sjbstatic int 1911179193Sjbfasttrap_open(dev_t *devp, int flag, int otyp, cred_t *cred_p) 1912179193Sjb{ 1913179193Sjb return (0); 1914179193Sjb} 1915179193Sjb 1916179193Sjb/*ARGSUSED*/ 1917179193Sjbstatic int 1918179193Sjbfasttrap_ioctl(dev_t dev, int cmd, intptr_t arg, int md, cred_t *cr, int *rv) 1919179193Sjb{ 1920179193Sjb if (!dtrace_attached()) 1921179193Sjb return (EAGAIN); 1922179193Sjb 1923179193Sjb if (cmd == FASTTRAPIOC_MAKEPROBE) { 1924179193Sjb fasttrap_probe_spec_t *uprobe = (void *)arg; 1925179193Sjb fasttrap_probe_spec_t *probe; 1926179193Sjb uint64_t noffs; 1927179193Sjb size_t size; 1928179193Sjb int ret; 1929179193Sjb char *c; 1930179193Sjb 1931179193Sjb if (copyin(&uprobe->ftps_noffs, &noffs, 1932179193Sjb sizeof (uprobe->ftps_noffs))) 1933179193Sjb return (EFAULT); 1934179193Sjb 1935179193Sjb /* 1936179193Sjb * Probes must have at least one tracepoint. 1937179193Sjb */ 1938179193Sjb if (noffs == 0) 1939179193Sjb return (EINVAL); 1940179193Sjb 1941179193Sjb size = sizeof (fasttrap_probe_spec_t) + 1942179193Sjb sizeof (probe->ftps_offs[0]) * (noffs - 1); 1943179193Sjb 1944179193Sjb if (size > 1024 * 1024) 1945179193Sjb return (ENOMEM); 1946179193Sjb 1947179193Sjb probe = kmem_alloc(size, KM_SLEEP); 1948179193Sjb 1949179193Sjb if (copyin(uprobe, probe, size) != 0) { 1950179193Sjb kmem_free(probe, size); 1951179193Sjb return (EFAULT); 1952179193Sjb } 1953179193Sjb 1954179193Sjb /* 1955179193Sjb * Verify that the function and module strings contain no 1956179193Sjb * funny characters. 1957179193Sjb */ 1958179193Sjb for (c = &probe->ftps_func[0]; *c != '\0'; c++) { 1959179193Sjb if (*c < 0x20 || 0x7f <= *c) { 1960179193Sjb ret = EINVAL; 1961179193Sjb goto err; 1962179193Sjb } 1963179193Sjb } 1964179193Sjb 1965179193Sjb for (c = &probe->ftps_mod[0]; *c != '\0'; c++) { 1966179193Sjb if (*c < 0x20 || 0x7f <= *c) { 1967179193Sjb ret = EINVAL; 1968179193Sjb goto err; 1969179193Sjb } 1970179193Sjb } 1971179193Sjb 1972179193Sjb if (!PRIV_POLICY_CHOICE(cr, PRIV_ALL, B_FALSE)) { 1973179193Sjb proc_t *p; 1974179193Sjb pid_t pid = probe->ftps_pid; 1975179193Sjb 1976179193Sjb mutex_enter(&pidlock); 1977179193Sjb /* 1978179193Sjb * Report an error if the process doesn't exist 1979179193Sjb * or is actively being birthed. 1980179193Sjb */ 1981179193Sjb if ((p = prfind(pid)) == NULL || p->p_stat == SIDL) { 1982179193Sjb mutex_exit(&pidlock); 1983179193Sjb return (ESRCH); 1984179193Sjb } 1985179193Sjb mutex_enter(&p->p_lock); 1986179193Sjb mutex_exit(&pidlock); 1987179193Sjb 1988179193Sjb if ((ret = priv_proc_cred_perm(cr, p, NULL, 1989179193Sjb VREAD | VWRITE)) != 0) { 1990179193Sjb mutex_exit(&p->p_lock); 1991179193Sjb return (ret); 1992179193Sjb } 1993179193Sjb 1994179193Sjb mutex_exit(&p->p_lock); 1995179193Sjb } 1996179193Sjb 1997179193Sjb ret = fasttrap_add_probe(probe); 1998179193Sjberr: 1999179193Sjb kmem_free(probe, size); 2000179193Sjb 2001179193Sjb return (ret); 2002179193Sjb 2003179193Sjb } else if (cmd == FASTTRAPIOC_GETINSTR) { 2004179193Sjb fasttrap_instr_query_t instr; 2005179193Sjb fasttrap_tracepoint_t *tp; 2006179193Sjb uint_t index; 2007179193Sjb int ret; 2008179193Sjb 2009179193Sjb if (copyin((void *)arg, &instr, sizeof (instr)) != 0) 2010179193Sjb return (EFAULT); 2011179193Sjb 2012179193Sjb if (!PRIV_POLICY_CHOICE(cr, PRIV_ALL, B_FALSE)) { 2013179193Sjb proc_t *p; 2014179193Sjb pid_t pid = instr.ftiq_pid; 2015179193Sjb 2016179193Sjb mutex_enter(&pidlock); 2017179193Sjb /* 2018179193Sjb * Report an error if the process doesn't exist 2019179193Sjb * or is actively being birthed. 2020179193Sjb */ 2021179193Sjb if ((p = prfind(pid)) == NULL || p->p_stat == SIDL) { 2022179193Sjb mutex_exit(&pidlock); 2023179193Sjb return (ESRCH); 2024179193Sjb } 2025179193Sjb mutex_enter(&p->p_lock); 2026179193Sjb mutex_exit(&pidlock); 2027179193Sjb 2028179193Sjb if ((ret = priv_proc_cred_perm(cr, p, NULL, 2029179193Sjb VREAD)) != 0) { 2030179193Sjb mutex_exit(&p->p_lock); 2031179193Sjb return (ret); 2032179193Sjb } 2033179193Sjb 2034179193Sjb mutex_exit(&p->p_lock); 2035179193Sjb } 2036179193Sjb 2037179193Sjb index = FASTTRAP_TPOINTS_INDEX(instr.ftiq_pid, instr.ftiq_pc); 2038179193Sjb 2039179193Sjb mutex_enter(&fasttrap_tpoints.fth_table[index].ftb_mtx); 2040179193Sjb tp = fasttrap_tpoints.fth_table[index].ftb_data; 2041179193Sjb while (tp != NULL) { 2042179193Sjb if (instr.ftiq_pid == tp->ftt_pid && 2043179193Sjb instr.ftiq_pc == tp->ftt_pc && 2044179193Sjb tp->ftt_proc->ftpc_acount != 0) 2045179193Sjb break; 2046179193Sjb 2047179198Sjb /* 2048179198Sjb * The count of active providers can only be 2049179198Sjb * decremented (i.e. to zero) during exec, exit, and 2050179198Sjb * removal of a meta provider so it should be 2051179198Sjb * impossible to drop the count during this operation(). 2052179198Sjb */ 2053179198Sjb ASSERT(tp->ftt_proc->ftpc_acount != 0); 2054179193Sjb tp = tp->ftt_next; 2055179193Sjb } 2056179193Sjb 2057179193Sjb if (tp == NULL) { 2058179193Sjb mutex_exit(&fasttrap_tpoints.fth_table[index].ftb_mtx); 2059179193Sjb return (ENOENT); 2060179193Sjb } 2061179193Sjb 2062179193Sjb bcopy(&tp->ftt_instr, &instr.ftiq_instr, 2063179193Sjb sizeof (instr.ftiq_instr)); 2064179193Sjb mutex_exit(&fasttrap_tpoints.fth_table[index].ftb_mtx); 2065179193Sjb 2066179193Sjb if (copyout(&instr, (void *)arg, sizeof (instr)) != 0) 2067179193Sjb return (EFAULT); 2068179193Sjb 2069179193Sjb return (0); 2070179193Sjb } 2071179193Sjb 2072179193Sjb return (EINVAL); 2073179193Sjb} 2074179193Sjb 2075179193Sjbstatic struct cb_ops fasttrap_cb_ops = { 2076179193Sjb fasttrap_open, /* open */ 2077179193Sjb nodev, /* close */ 2078179193Sjb nulldev, /* strategy */ 2079179193Sjb nulldev, /* print */ 2080179193Sjb nodev, /* dump */ 2081179193Sjb nodev, /* read */ 2082179193Sjb nodev, /* write */ 2083179193Sjb fasttrap_ioctl, /* ioctl */ 2084179193Sjb nodev, /* devmap */ 2085179193Sjb nodev, /* mmap */ 2086179193Sjb nodev, /* segmap */ 2087179193Sjb nochpoll, /* poll */ 2088179193Sjb ddi_prop_op, /* cb_prop_op */ 2089179193Sjb 0, /* streamtab */ 2090179193Sjb D_NEW | D_MP /* Driver compatibility flag */ 2091179193Sjb}; 2092179193Sjb 2093179193Sjb/*ARGSUSED*/ 2094179193Sjbstatic int 2095179193Sjbfasttrap_info(dev_info_t *dip, ddi_info_cmd_t infocmd, void *arg, void **result) 2096179193Sjb{ 2097179193Sjb int error; 2098179193Sjb 2099179193Sjb switch (infocmd) { 2100179193Sjb case DDI_INFO_DEVT2DEVINFO: 2101179193Sjb *result = (void *)fasttrap_devi; 2102179193Sjb error = DDI_SUCCESS; 2103179193Sjb break; 2104179193Sjb case DDI_INFO_DEVT2INSTANCE: 2105179193Sjb *result = (void *)0; 2106179193Sjb error = DDI_SUCCESS; 2107179193Sjb break; 2108179193Sjb default: 2109179193Sjb error = DDI_FAILURE; 2110179193Sjb } 2111179193Sjb return (error); 2112179193Sjb} 2113179193Sjb 2114179193Sjbstatic int 2115179193Sjbfasttrap_attach(dev_info_t *devi, ddi_attach_cmd_t cmd) 2116179193Sjb{ 2117179193Sjb ulong_t nent; 2118179193Sjb 2119179193Sjb switch (cmd) { 2120179193Sjb case DDI_ATTACH: 2121179193Sjb break; 2122179193Sjb case DDI_RESUME: 2123179193Sjb return (DDI_SUCCESS); 2124179193Sjb default: 2125179193Sjb return (DDI_FAILURE); 2126179193Sjb } 2127179193Sjb 2128179193Sjb if (ddi_create_minor_node(devi, "fasttrap", S_IFCHR, 0, 2129179193Sjb DDI_PSEUDO, NULL) == DDI_FAILURE) { 2130179193Sjb ddi_remove_minor_node(devi, NULL); 2131179193Sjb return (DDI_FAILURE); 2132179193Sjb } 2133179193Sjb 2134179193Sjb ddi_report_dev(devi); 2135179193Sjb fasttrap_devi = devi; 2136179193Sjb 2137179193Sjb /* 2138179193Sjb * Install our hooks into fork(2), exec(2), and exit(2). 2139179193Sjb */ 2140179193Sjb dtrace_fasttrap_fork_ptr = &fasttrap_fork; 2141179193Sjb dtrace_fasttrap_exit_ptr = &fasttrap_exec_exit; 2142179193Sjb dtrace_fasttrap_exec_ptr = &fasttrap_exec_exit; 2143179193Sjb 2144179193Sjb fasttrap_max = ddi_getprop(DDI_DEV_T_ANY, devi, DDI_PROP_DONTPASS, 2145179193Sjb "fasttrap-max-probes", FASTTRAP_MAX_DEFAULT); 2146179193Sjb fasttrap_total = 0; 2147179193Sjb 2148179193Sjb /* 2149179193Sjb * Conjure up the tracepoints hashtable... 2150179193Sjb */ 2151179193Sjb nent = ddi_getprop(DDI_DEV_T_ANY, devi, DDI_PROP_DONTPASS, 2152179193Sjb "fasttrap-hash-size", FASTTRAP_TPOINTS_DEFAULT_SIZE); 2153179193Sjb 2154179193Sjb if (nent == 0 || nent > 0x1000000) 2155179193Sjb nent = FASTTRAP_TPOINTS_DEFAULT_SIZE; 2156179193Sjb 2157179193Sjb if ((nent & (nent - 1)) == 0) 2158179193Sjb fasttrap_tpoints.fth_nent = nent; 2159179193Sjb else 2160179193Sjb fasttrap_tpoints.fth_nent = 1 << fasttrap_highbit(nent); 2161179193Sjb ASSERT(fasttrap_tpoints.fth_nent > 0); 2162179193Sjb fasttrap_tpoints.fth_mask = fasttrap_tpoints.fth_nent - 1; 2163179193Sjb fasttrap_tpoints.fth_table = kmem_zalloc(fasttrap_tpoints.fth_nent * 2164179193Sjb sizeof (fasttrap_bucket_t), KM_SLEEP); 2165179193Sjb 2166179193Sjb /* 2167179193Sjb * ... and the providers hash table... 2168179193Sjb */ 2169179193Sjb nent = FASTTRAP_PROVIDERS_DEFAULT_SIZE; 2170179193Sjb if ((nent & (nent - 1)) == 0) 2171179193Sjb fasttrap_provs.fth_nent = nent; 2172179193Sjb else 2173179193Sjb fasttrap_provs.fth_nent = 1 << fasttrap_highbit(nent); 2174179193Sjb ASSERT(fasttrap_provs.fth_nent > 0); 2175179193Sjb fasttrap_provs.fth_mask = fasttrap_provs.fth_nent - 1; 2176179193Sjb fasttrap_provs.fth_table = kmem_zalloc(fasttrap_provs.fth_nent * 2177179193Sjb sizeof (fasttrap_bucket_t), KM_SLEEP); 2178179193Sjb 2179179193Sjb /* 2180179193Sjb * ... and the procs hash table. 2181179193Sjb */ 2182179193Sjb nent = FASTTRAP_PROCS_DEFAULT_SIZE; 2183179193Sjb if ((nent & (nent - 1)) == 0) 2184179193Sjb fasttrap_procs.fth_nent = nent; 2185179193Sjb else 2186179193Sjb fasttrap_procs.fth_nent = 1 << fasttrap_highbit(nent); 2187179193Sjb ASSERT(fasttrap_procs.fth_nent > 0); 2188179193Sjb fasttrap_procs.fth_mask = fasttrap_procs.fth_nent - 1; 2189179193Sjb fasttrap_procs.fth_table = kmem_zalloc(fasttrap_procs.fth_nent * 2190179193Sjb sizeof (fasttrap_bucket_t), KM_SLEEP); 2191179193Sjb 2192179193Sjb (void) dtrace_meta_register("fasttrap", &fasttrap_mops, NULL, 2193179193Sjb &fasttrap_meta_id); 2194179193Sjb 2195179193Sjb return (DDI_SUCCESS); 2196179193Sjb} 2197179193Sjb 2198179193Sjbstatic int 2199179193Sjbfasttrap_detach(dev_info_t *devi, ddi_detach_cmd_t cmd) 2200179193Sjb{ 2201179193Sjb int i, fail = 0; 2202179193Sjb timeout_id_t tmp; 2203179193Sjb 2204179193Sjb switch (cmd) { 2205179193Sjb case DDI_DETACH: 2206179193Sjb break; 2207179193Sjb case DDI_SUSPEND: 2208179193Sjb return (DDI_SUCCESS); 2209179193Sjb default: 2210179193Sjb return (DDI_FAILURE); 2211179193Sjb } 2212179193Sjb 2213179193Sjb /* 2214179193Sjb * Unregister the meta-provider to make sure no new fasttrap- 2215179193Sjb * managed providers come along while we're trying to close up 2216179193Sjb * shop. If we fail to detach, we'll need to re-register as a 2217179193Sjb * meta-provider. We can fail to unregister as a meta-provider 2218179193Sjb * if providers we manage still exist. 2219179193Sjb */ 2220179193Sjb if (fasttrap_meta_id != DTRACE_METAPROVNONE && 2221179193Sjb dtrace_meta_unregister(fasttrap_meta_id) != 0) 2222179193Sjb return (DDI_FAILURE); 2223179193Sjb 2224179193Sjb /* 2225179193Sjb * Prevent any new timeouts from running by setting fasttrap_timeout 2226179193Sjb * to a non-zero value, and wait for the current timeout to complete. 2227179193Sjb */ 2228179193Sjb mutex_enter(&fasttrap_cleanup_mtx); 2229179193Sjb fasttrap_cleanup_work = 0; 2230179193Sjb 2231179193Sjb while (fasttrap_timeout != (timeout_id_t)1) { 2232179193Sjb tmp = fasttrap_timeout; 2233179193Sjb fasttrap_timeout = (timeout_id_t)1; 2234179193Sjb 2235179193Sjb if (tmp != 0) { 2236179193Sjb mutex_exit(&fasttrap_cleanup_mtx); 2237179193Sjb (void) untimeout(tmp); 2238179193Sjb mutex_enter(&fasttrap_cleanup_mtx); 2239179193Sjb } 2240179193Sjb } 2241179193Sjb 2242179193Sjb fasttrap_cleanup_work = 0; 2243179193Sjb mutex_exit(&fasttrap_cleanup_mtx); 2244179193Sjb 2245179193Sjb /* 2246179193Sjb * Iterate over all of our providers. If there's still a process 2247179193Sjb * that corresponds to that pid, fail to detach. 2248179193Sjb */ 2249179193Sjb for (i = 0; i < fasttrap_provs.fth_nent; i++) { 2250179193Sjb fasttrap_provider_t **fpp, *fp; 2251179193Sjb fasttrap_bucket_t *bucket = &fasttrap_provs.fth_table[i]; 2252179193Sjb 2253179193Sjb mutex_enter(&bucket->ftb_mtx); 2254179193Sjb fpp = (fasttrap_provider_t **)&bucket->ftb_data; 2255179193Sjb while ((fp = *fpp) != NULL) { 2256179193Sjb /* 2257179193Sjb * Acquire and release the lock as a simple way of 2258179193Sjb * waiting for any other consumer to finish with 2259179193Sjb * this provider. A thread must first acquire the 2260179193Sjb * bucket lock so there's no chance of another thread 2261179193Sjb * blocking on the provider's lock. 2262179193Sjb */ 2263179193Sjb mutex_enter(&fp->ftp_mtx); 2264179193Sjb mutex_exit(&fp->ftp_mtx); 2265179193Sjb 2266179193Sjb if (dtrace_unregister(fp->ftp_provid) != 0) { 2267179193Sjb fail = 1; 2268179193Sjb fpp = &fp->ftp_next; 2269179193Sjb } else { 2270179193Sjb *fpp = fp->ftp_next; 2271179193Sjb fasttrap_provider_free(fp); 2272179193Sjb } 2273179193Sjb } 2274179193Sjb 2275179193Sjb mutex_exit(&bucket->ftb_mtx); 2276179193Sjb } 2277179193Sjb 2278179193Sjb if (fail) { 2279179193Sjb uint_t work; 2280179193Sjb /* 2281179193Sjb * If we're failing to detach, we need to unblock timeouts 2282179193Sjb * and start a new timeout if any work has accumulated while 2283179193Sjb * we've been unsuccessfully trying to detach. 2284179193Sjb */ 2285179193Sjb mutex_enter(&fasttrap_cleanup_mtx); 2286179193Sjb fasttrap_timeout = 0; 2287179193Sjb work = fasttrap_cleanup_work; 2288179193Sjb mutex_exit(&fasttrap_cleanup_mtx); 2289179193Sjb 2290179193Sjb if (work) 2291179193Sjb fasttrap_pid_cleanup(); 2292179193Sjb 2293179193Sjb (void) dtrace_meta_register("fasttrap", &fasttrap_mops, NULL, 2294179193Sjb &fasttrap_meta_id); 2295179193Sjb 2296179193Sjb return (DDI_FAILURE); 2297179193Sjb } 2298179193Sjb 2299179193Sjb#ifdef DEBUG 2300179193Sjb mutex_enter(&fasttrap_count_mtx); 2301179193Sjb ASSERT(fasttrap_pid_count == 0); 2302179193Sjb mutex_exit(&fasttrap_count_mtx); 2303179193Sjb#endif 2304179193Sjb 2305179193Sjb kmem_free(fasttrap_tpoints.fth_table, 2306179193Sjb fasttrap_tpoints.fth_nent * sizeof (fasttrap_bucket_t)); 2307179193Sjb fasttrap_tpoints.fth_nent = 0; 2308179193Sjb 2309179193Sjb kmem_free(fasttrap_provs.fth_table, 2310179193Sjb fasttrap_provs.fth_nent * sizeof (fasttrap_bucket_t)); 2311179193Sjb fasttrap_provs.fth_nent = 0; 2312179193Sjb 2313179193Sjb kmem_free(fasttrap_procs.fth_table, 2314179193Sjb fasttrap_procs.fth_nent * sizeof (fasttrap_bucket_t)); 2315179193Sjb fasttrap_procs.fth_nent = 0; 2316179193Sjb 2317179193Sjb /* 2318179193Sjb * We know there are no tracepoints in any process anywhere in 2319179193Sjb * the system so there is no process which has its p_dtrace_count 2320179193Sjb * greater than zero, therefore we know that no thread can actively 2321179193Sjb * be executing code in fasttrap_fork(). Similarly for p_dtrace_probes 2322179193Sjb * and fasttrap_exec() and fasttrap_exit(). 2323179193Sjb */ 2324179193Sjb ASSERT(dtrace_fasttrap_fork_ptr == &fasttrap_fork); 2325179193Sjb dtrace_fasttrap_fork_ptr = NULL; 2326179193Sjb 2327179193Sjb ASSERT(dtrace_fasttrap_exec_ptr == &fasttrap_exec_exit); 2328179193Sjb dtrace_fasttrap_exec_ptr = NULL; 2329179193Sjb 2330179193Sjb ASSERT(dtrace_fasttrap_exit_ptr == &fasttrap_exec_exit); 2331179193Sjb dtrace_fasttrap_exit_ptr = NULL; 2332179193Sjb 2333179193Sjb ddi_remove_minor_node(devi, NULL); 2334179193Sjb 2335179193Sjb return (DDI_SUCCESS); 2336179193Sjb} 2337179193Sjb 2338179193Sjbstatic struct dev_ops fasttrap_ops = { 2339179193Sjb DEVO_REV, /* devo_rev */ 2340179193Sjb 0, /* refcnt */ 2341179193Sjb fasttrap_info, /* get_dev_info */ 2342179193Sjb nulldev, /* identify */ 2343179193Sjb nulldev, /* probe */ 2344179193Sjb fasttrap_attach, /* attach */ 2345179193Sjb fasttrap_detach, /* detach */ 2346179193Sjb nodev, /* reset */ 2347179193Sjb &fasttrap_cb_ops, /* driver operations */ 2348179193Sjb NULL, /* bus operations */ 2349179193Sjb nodev /* dev power */ 2350179193Sjb}; 2351179193Sjb 2352179193Sjb/* 2353179193Sjb * Module linkage information for the kernel. 2354179193Sjb */ 2355179193Sjbstatic struct modldrv modldrv = { 2356179193Sjb &mod_driverops, /* module type (this is a pseudo driver) */ 2357179193Sjb "Fasttrap Tracing", /* name of module */ 2358179193Sjb &fasttrap_ops, /* driver ops */ 2359179193Sjb}; 2360179193Sjb 2361179193Sjbstatic struct modlinkage modlinkage = { 2362179193Sjb MODREV_1, 2363179193Sjb (void *)&modldrv, 2364179193Sjb NULL 2365179193Sjb}; 2366179193Sjb 2367179193Sjbint 2368179193Sjb_init(void) 2369179193Sjb{ 2370179193Sjb return (mod_install(&modlinkage)); 2371179193Sjb} 2372179193Sjb 2373179193Sjbint 2374179193Sjb_info(struct modinfo *modinfop) 2375179193Sjb{ 2376179193Sjb return (mod_info(&modlinkage, modinfop)); 2377179193Sjb} 2378179193Sjb 2379179193Sjbint 2380179193Sjb_fini(void) 2381179193Sjb{ 2382179193Sjb return (mod_remove(&modlinkage)); 2383179193Sjb} 2384