1179193Sjb/* 2179193Sjb * CDDL HEADER START 3179193Sjb * 4179193Sjb * The contents of this file are subject to the terms of the 5179193Sjb * Common Development and Distribution License (the "License"). 6179193Sjb * You may not use this file except in compliance with the License. 7179193Sjb * 8179193Sjb * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9179193Sjb * or http://www.opensolaris.org/os/licensing. 10179193Sjb * See the License for the specific language governing permissions 11179193Sjb * and limitations under the License. 12179193Sjb * 13179193Sjb * When distributing Covered Code, include this CDDL HEADER in each 14179193Sjb * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15179193Sjb * If applicable, add the following below this CDDL HEADER, with the 16179193Sjb * fields enclosed by brackets "[]" replaced with your own identifying 17179193Sjb * information: Portions Copyright [yyyy] [name of copyright owner] 18179193Sjb * 19179193Sjb * CDDL HEADER END 20211738Srpaulo * 21211738Srpaulo * Portions Copyright 2010 The FreeBSD Foundation 22211738Srpaulo * 23211738Srpaulo * $FreeBSD: releng/11.0/sys/cddl/contrib/opensolaris/uts/common/dtrace/fasttrap.c 296479 2016-03-08 00:43:03Z markj $ 24179193Sjb */ 25179193Sjb 26179193Sjb/* 27179198Sjb * Copyright 2008 Sun Microsystems, Inc. All rights reserved. 28179193Sjb * Use is subject to license terms. 29179193Sjb */ 30179193Sjb 31268097Spfg/* 32287642Smarkj * Copyright (c) 2015, Joyent, Inc. All rights reserved. 33268097Spfg */ 34179193Sjb 35179193Sjb#include <sys/atomic.h> 36179193Sjb#include <sys/errno.h> 37179193Sjb#include <sys/stat.h> 38179193Sjb#include <sys/modctl.h> 39179193Sjb#include <sys/conf.h> 40179193Sjb#include <sys/systm.h> 41277300Ssmh#ifdef illumos 42179193Sjb#include <sys/ddi.h> 43211738Srpaulo#endif 44179193Sjb#include <sys/sunddi.h> 45179193Sjb#include <sys/cpuvar.h> 46179193Sjb#include <sys/kmem.h> 47277300Ssmh#ifdef illumos 48179193Sjb#include <sys/strsubr.h> 49211738Srpaulo#endif 50179193Sjb#include <sys/fasttrap.h> 51179193Sjb#include <sys/fasttrap_impl.h> 52179193Sjb#include <sys/fasttrap_isa.h> 53179193Sjb#include <sys/dtrace.h> 54179193Sjb#include <sys/dtrace_impl.h> 55179193Sjb#include <sys/sysmacros.h> 56179193Sjb#include <sys/proc.h> 57179193Sjb#include <sys/policy.h> 58277300Ssmh#ifdef illumos 59179193Sjb#include <util/qsort.h> 60211738Srpaulo#endif 61211738Srpaulo#include <sys/mutex.h> 62211738Srpaulo#include <sys/kernel.h> 63277300Ssmh#ifndef illumos 64264434Smarkj#include <sys/dtrace_bsd.h> 65264434Smarkj#include <sys/eventhandler.h> 66296479Smarkj#include <sys/rmlock.h> 67291545Sstas#include <sys/sysctl.h> 68268097Spfg#include <sys/u8_textprep.h> 69211738Srpaulo#include <sys/user.h> 70296479Smarkj 71264434Smarkj#include <vm/vm.h> 72264434Smarkj#include <vm/pmap.h> 73264434Smarkj#include <vm/vm_map.h> 74264434Smarkj#include <vm/vm_param.h> 75296479Smarkj 76211738Srpaulo#include <cddl/dev/dtrace/dtrace_cddl.h> 77211738Srpaulo#endif 78179193Sjb 79179193Sjb/* 80179193Sjb * User-Land Trap-Based Tracing 81179193Sjb * ---------------------------- 82179193Sjb * 83179193Sjb * The fasttrap provider allows DTrace consumers to instrument any user-level 84179193Sjb * instruction to gather data; this includes probes with semantic 85179193Sjb * signifigance like entry and return as well as simple offsets into the 86179193Sjb * function. While the specific techniques used are very ISA specific, the 87179193Sjb * methodology is generalizable to any architecture. 88179193Sjb * 89179193Sjb * 90179193Sjb * The General Methodology 91179193Sjb * ----------------------- 92179193Sjb * 93179193Sjb * With the primary goal of tracing every user-land instruction and the 94179193Sjb * limitation that we can't trust user space so don't want to rely on much 95179193Sjb * information there, we begin by replacing the instructions we want to trace 96179193Sjb * with trap instructions. Each instruction we overwrite is saved into a hash 97179193Sjb * table keyed by process ID and pc address. When we enter the kernel due to 98179193Sjb * this trap instruction, we need the effects of the replaced instruction to 99179193Sjb * appear to have occurred before we proceed with the user thread's 100179193Sjb * execution. 101179193Sjb * 102179193Sjb * Each user level thread is represented by a ulwp_t structure which is 103179193Sjb * always easily accessible through a register. The most basic way to produce 104179193Sjb * the effects of the instruction we replaced is to copy that instruction out 105179193Sjb * to a bit of scratch space reserved in the user thread's ulwp_t structure 106179193Sjb * (a sort of kernel-private thread local storage), set the PC to that 107179193Sjb * scratch space and single step. When we reenter the kernel after single 108179193Sjb * stepping the instruction we must then adjust the PC to point to what would 109179193Sjb * normally be the next instruction. Of course, special care must be taken 110179193Sjb * for branches and jumps, but these represent such a small fraction of any 111179193Sjb * instruction set that writing the code to emulate these in the kernel is 112179193Sjb * not too difficult. 113179193Sjb * 114179193Sjb * Return probes may require several tracepoints to trace every return site, 115179193Sjb * and, conversely, each tracepoint may activate several probes (the entry 116179193Sjb * and offset 0 probes, for example). To solve this muliplexing problem, 117179193Sjb * tracepoints contain lists of probes to activate and probes contain lists 118179193Sjb * of tracepoints to enable. If a probe is activated, it adds its ID to 119179193Sjb * existing tracepoints or creates new ones as necessary. 120179193Sjb * 121179193Sjb * Most probes are activated _before_ the instruction is executed, but return 122179193Sjb * probes are activated _after_ the effects of the last instruction of the 123179193Sjb * function are visible. Return probes must be fired _after_ we have 124179193Sjb * single-stepped the instruction whereas all other probes are fired 125179193Sjb * beforehand. 126179193Sjb * 127179193Sjb * 128179193Sjb * Lock Ordering 129179193Sjb * ------------- 130179193Sjb * 131179193Sjb * The lock ordering below -- both internally and with respect to the DTrace 132179193Sjb * framework -- is a little tricky and bears some explanation. Each provider 133179193Sjb * has a lock (ftp_mtx) that protects its members including reference counts 134179193Sjb * for enabled probes (ftp_rcount), consumers actively creating probes 135179193Sjb * (ftp_ccount) and USDT consumers (ftp_mcount); all three prevent a provider 136179193Sjb * from being freed. A provider is looked up by taking the bucket lock for the 137179193Sjb * provider hash table, and is returned with its lock held. The provider lock 138179193Sjb * may be taken in functions invoked by the DTrace framework, but may not be 139179193Sjb * held while calling functions in the DTrace framework. 140179193Sjb * 141179193Sjb * To ensure consistency over multiple calls to the DTrace framework, the 142179193Sjb * creation lock (ftp_cmtx) should be held. Naturally, the creation lock may 143179193Sjb * not be taken when holding the provider lock as that would create a cyclic 144179193Sjb * lock ordering. In situations where one would naturally take the provider 145179193Sjb * lock and then the creation lock, we instead up a reference count to prevent 146179193Sjb * the provider from disappearing, drop the provider lock, and acquire the 147179193Sjb * creation lock. 148179193Sjb * 149179193Sjb * Briefly: 150179193Sjb * bucket lock before provider lock 151179193Sjb * DTrace before provider lock 152179193Sjb * creation lock before DTrace 153179193Sjb * never hold the provider lock and creation lock simultaneously 154179193Sjb */ 155179193Sjb 156211738Srpaulostatic d_open_t fasttrap_open; 157211738Srpaulostatic d_ioctl_t fasttrap_ioctl; 158211738Srpaulo 159211738Srpaulostatic struct cdevsw fasttrap_cdevsw = { 160211738Srpaulo .d_version = D_VERSION, 161211738Srpaulo .d_open = fasttrap_open, 162211738Srpaulo .d_ioctl = fasttrap_ioctl, 163211738Srpaulo .d_name = "fasttrap", 164211738Srpaulo}; 165211738Srpaulostatic struct cdev *fasttrap_cdev; 166179193Sjbstatic dtrace_meta_provider_id_t fasttrap_meta_id; 167179193Sjb 168250953Smarkjstatic struct proc *fasttrap_cleanup_proc; 169211738Srpaulostatic struct mtx fasttrap_cleanup_mtx; 170250953Smarkjstatic uint_t fasttrap_cleanup_work, fasttrap_cleanup_drain, fasttrap_cleanup_cv; 171179193Sjb 172179193Sjb/* 173179193Sjb * Generation count on modifications to the global tracepoint lookup table. 174179193Sjb */ 175179193Sjbstatic volatile uint64_t fasttrap_mod_gen; 176179193Sjb 177179193Sjb/* 178179193Sjb * When the fasttrap provider is loaded, fasttrap_max is set to either 179291545Sstas * FASTTRAP_MAX_DEFAULT, or the value for fasttrap-max-probes in the 180291545Sstas * fasttrap.conf file (Illumos), or the value provied in the loader.conf (FreeBSD). 181291545Sstas * Each time a probe is created, fasttrap_total is incremented by the number 182291545Sstas * of tracepoints that may be associated with that probe; fasttrap_total is capped 183291545Sstas * at fasttrap_max. 184179193Sjb */ 185179193Sjb#define FASTTRAP_MAX_DEFAULT 250000 186291545Sstasstatic uint32_t fasttrap_max = FASTTRAP_MAX_DEFAULT; 187179193Sjbstatic uint32_t fasttrap_total; 188179193Sjb 189248983Spfg/* 190248983Spfg * Copyright (c) 2011, Joyent, Inc. All rights reserved. 191248983Spfg */ 192179193Sjb 193179193Sjb#define FASTTRAP_TPOINTS_DEFAULT_SIZE 0x4000 194179193Sjb#define FASTTRAP_PROVIDERS_DEFAULT_SIZE 0x100 195179193Sjb#define FASTTRAP_PROCS_DEFAULT_SIZE 0x100 196179193Sjb 197179193Sjb#define FASTTRAP_PID_NAME "pid" 198179193Sjb 199179193Sjbfasttrap_hash_t fasttrap_tpoints; 200179193Sjbstatic fasttrap_hash_t fasttrap_provs; 201179193Sjbstatic fasttrap_hash_t fasttrap_procs; 202179193Sjb 203179193Sjbstatic uint64_t fasttrap_pid_count; /* pid ref count */ 204179193Sjbstatic kmutex_t fasttrap_count_mtx; /* lock on ref count */ 205179193Sjb 206179193Sjb#define FASTTRAP_ENABLE_FAIL 1 207179193Sjb#define FASTTRAP_ENABLE_PARTIAL 2 208179193Sjb 209179193Sjbstatic int fasttrap_tracepoint_enable(proc_t *, fasttrap_probe_t *, uint_t); 210179193Sjbstatic void fasttrap_tracepoint_disable(proc_t *, fasttrap_probe_t *, uint_t); 211179193Sjb 212179193Sjbstatic fasttrap_provider_t *fasttrap_provider_lookup(pid_t, const char *, 213179193Sjb const dtrace_pattr_t *); 214179193Sjbstatic void fasttrap_provider_retire(pid_t, const char *, int); 215179193Sjbstatic void fasttrap_provider_free(fasttrap_provider_t *); 216179193Sjb 217179193Sjbstatic fasttrap_proc_t *fasttrap_proc_lookup(pid_t); 218179193Sjbstatic void fasttrap_proc_release(fasttrap_proc_t *); 219179193Sjb 220277300Ssmh#ifndef illumos 221264434Smarkjstatic void fasttrap_thread_dtor(void *, struct thread *); 222264434Smarkj#endif 223264434Smarkj 224179193Sjb#define FASTTRAP_PROVS_INDEX(pid, name) \ 225179193Sjb ((fasttrap_hash_str(name) + (pid)) & fasttrap_provs.fth_mask) 226179193Sjb 227179193Sjb#define FASTTRAP_PROCS_INDEX(pid) ((pid) & fasttrap_procs.fth_mask) 228179193Sjb 229277300Ssmh#ifndef illumos 230296479Smarkjstruct rmlock fasttrap_tp_lock; 231264434Smarkjstatic eventhandler_tag fasttrap_thread_dtor_tag; 232211925Srpaulo#endif 233211925Srpaulo 234291545Sstasstatic unsigned long tpoints_hash_size = FASTTRAP_TPOINTS_DEFAULT_SIZE; 235291545Sstas 236291545Sstas#ifdef __FreeBSD__ 237291545SstasSYSCTL_DECL(_kern_dtrace); 238291545SstasSYSCTL_NODE(_kern_dtrace, OID_AUTO, fasttrap, CTLFLAG_RD, 0, "DTrace fasttrap parameters"); 239291545SstasSYSCTL_UINT(_kern_dtrace_fasttrap, OID_AUTO, max_probes, CTLFLAG_RWTUN, &fasttrap_max, 240291545Sstas FASTTRAP_MAX_DEFAULT, "Maximum number of fasttrap probes"); 241291545SstasSYSCTL_ULONG(_kern_dtrace_fasttrap, OID_AUTO, tpoints_hash_size, CTLFLAG_RDTUN, &tpoints_hash_size, 242291545Sstas FASTTRAP_TPOINTS_DEFAULT_SIZE, "Size of the tracepoint hash table"); 243291545Sstas#endif 244291545Sstas 245179193Sjbstatic int 246179193Sjbfasttrap_highbit(ulong_t i) 247179193Sjb{ 248179193Sjb int h = 1; 249179193Sjb 250179193Sjb if (i == 0) 251179193Sjb return (0); 252179193Sjb#ifdef _LP64 253179193Sjb if (i & 0xffffffff00000000ul) { 254179193Sjb h += 32; i >>= 32; 255179193Sjb } 256179193Sjb#endif 257179193Sjb if (i & 0xffff0000) { 258179193Sjb h += 16; i >>= 16; 259179193Sjb } 260179193Sjb if (i & 0xff00) { 261179193Sjb h += 8; i >>= 8; 262179193Sjb } 263179193Sjb if (i & 0xf0) { 264179193Sjb h += 4; i >>= 4; 265179193Sjb } 266179193Sjb if (i & 0xc) { 267179193Sjb h += 2; i >>= 2; 268179193Sjb } 269179193Sjb if (i & 0x2) { 270179193Sjb h += 1; 271179193Sjb } 272179193Sjb return (h); 273179193Sjb} 274179193Sjb 275179193Sjbstatic uint_t 276179193Sjbfasttrap_hash_str(const char *p) 277179193Sjb{ 278179193Sjb unsigned int g; 279179193Sjb uint_t hval = 0; 280179193Sjb 281179193Sjb while (*p) { 282179193Sjb hval = (hval << 4) + *p++; 283179193Sjb if ((g = (hval & 0xf0000000)) != 0) 284179193Sjb hval ^= g >> 24; 285179193Sjb hval &= ~g; 286179193Sjb } 287179193Sjb return (hval); 288179193Sjb} 289179193Sjb 290179193Sjbvoid 291179193Sjbfasttrap_sigtrap(proc_t *p, kthread_t *t, uintptr_t pc) 292179193Sjb{ 293277300Ssmh#ifdef illumos 294179193Sjb sigqueue_t *sqp = kmem_zalloc(sizeof (sigqueue_t), KM_SLEEP); 295179193Sjb 296179193Sjb sqp->sq_info.si_signo = SIGTRAP; 297179193Sjb sqp->sq_info.si_code = TRAP_DTRACE; 298179193Sjb sqp->sq_info.si_addr = (caddr_t)pc; 299179193Sjb 300179193Sjb mutex_enter(&p->p_lock); 301179193Sjb sigaddqa(p, t, sqp); 302179193Sjb mutex_exit(&p->p_lock); 303179193Sjb 304179193Sjb if (t != NULL) 305179193Sjb aston(t); 306211738Srpaulo#else 307211738Srpaulo ksiginfo_t *ksi = kmem_zalloc(sizeof (ksiginfo_t), KM_SLEEP); 308211738Srpaulo 309211738Srpaulo ksiginfo_init(ksi); 310211738Srpaulo ksi->ksi_signo = SIGTRAP; 311211738Srpaulo ksi->ksi_code = TRAP_DTRACE; 312211738Srpaulo ksi->ksi_addr = (caddr_t)pc; 313211738Srpaulo PROC_LOCK(p); 314277914Smarkj (void) tdsendsignal(p, t, SIGTRAP, ksi); 315211738Srpaulo PROC_UNLOCK(p); 316211738Srpaulo#endif 317179193Sjb} 318179193Sjb 319277300Ssmh#ifndef illumos 320179193Sjb/* 321264434Smarkj * Obtain a chunk of scratch space in the address space of the target process. 322264434Smarkj */ 323264434Smarkjfasttrap_scrspace_t * 324264434Smarkjfasttrap_scraddr(struct thread *td, fasttrap_proc_t *fprc) 325264434Smarkj{ 326264434Smarkj fasttrap_scrblock_t *scrblk; 327264434Smarkj fasttrap_scrspace_t *scrspc; 328264434Smarkj struct proc *p; 329264434Smarkj vm_offset_t addr; 330264434Smarkj int error, i; 331264434Smarkj 332264434Smarkj scrspc = NULL; 333264434Smarkj if (td->t_dtrace_sscr != NULL) { 334264434Smarkj /* If the thread already has scratch space, we're done. */ 335264434Smarkj scrspc = (fasttrap_scrspace_t *)td->t_dtrace_sscr; 336264434Smarkj return (scrspc); 337264434Smarkj } 338264434Smarkj 339264434Smarkj p = td->td_proc; 340264434Smarkj 341264434Smarkj mutex_enter(&fprc->ftpc_mtx); 342264434Smarkj if (LIST_EMPTY(&fprc->ftpc_fscr)) { 343264434Smarkj /* 344264434Smarkj * No scratch space is available, so we'll map a new scratch 345264434Smarkj * space block into the traced process' address space. 346264434Smarkj */ 347264434Smarkj addr = 0; 348264434Smarkj error = vm_map_find(&p->p_vmspace->vm_map, NULL, 0, &addr, 349264434Smarkj FASTTRAP_SCRBLOCK_SIZE, 0, VMFS_ANY_SPACE, VM_PROT_ALL, 350264434Smarkj VM_PROT_ALL, 0); 351264434Smarkj if (error != KERN_SUCCESS) 352264434Smarkj goto done; 353264434Smarkj 354264434Smarkj scrblk = malloc(sizeof(*scrblk), M_SOLARIS, M_WAITOK); 355264434Smarkj scrblk->ftsb_addr = addr; 356264434Smarkj LIST_INSERT_HEAD(&fprc->ftpc_scrblks, scrblk, ftsb_next); 357264434Smarkj 358264434Smarkj /* 359264434Smarkj * Carve the block up into chunks and put them on the free list. 360264434Smarkj */ 361264434Smarkj for (i = 0; 362264434Smarkj i < FASTTRAP_SCRBLOCK_SIZE / FASTTRAP_SCRSPACE_SIZE; i++) { 363264434Smarkj scrspc = malloc(sizeof(*scrspc), M_SOLARIS, M_WAITOK); 364264434Smarkj scrspc->ftss_addr = addr + 365264434Smarkj i * FASTTRAP_SCRSPACE_SIZE; 366264434Smarkj LIST_INSERT_HEAD(&fprc->ftpc_fscr, scrspc, 367264434Smarkj ftss_next); 368264434Smarkj } 369264434Smarkj } 370264434Smarkj 371264434Smarkj /* 372264434Smarkj * Take the first scratch chunk off the free list, put it on the 373264434Smarkj * allocated list, and return its address. 374264434Smarkj */ 375264434Smarkj scrspc = LIST_FIRST(&fprc->ftpc_fscr); 376264434Smarkj LIST_REMOVE(scrspc, ftss_next); 377264434Smarkj LIST_INSERT_HEAD(&fprc->ftpc_ascr, scrspc, ftss_next); 378264434Smarkj 379264434Smarkj /* 380264434Smarkj * This scratch space is reserved for use by td until the thread exits. 381264434Smarkj */ 382264434Smarkj td->t_dtrace_sscr = scrspc; 383264434Smarkj 384264434Smarkjdone: 385264434Smarkj mutex_exit(&fprc->ftpc_mtx); 386264434Smarkj 387264434Smarkj return (scrspc); 388264434Smarkj} 389264434Smarkj 390264434Smarkj/* 391264434Smarkj * Return any allocated per-thread scratch space chunks back to the process' 392264434Smarkj * free list. 393264434Smarkj */ 394264434Smarkjstatic void 395264434Smarkjfasttrap_thread_dtor(void *arg __unused, struct thread *td) 396264434Smarkj{ 397264434Smarkj fasttrap_bucket_t *bucket; 398264434Smarkj fasttrap_proc_t *fprc; 399264434Smarkj fasttrap_scrspace_t *scrspc; 400264434Smarkj pid_t pid; 401264434Smarkj 402264434Smarkj if (td->t_dtrace_sscr == NULL) 403264434Smarkj return; 404264434Smarkj 405264434Smarkj pid = td->td_proc->p_pid; 406264434Smarkj bucket = &fasttrap_procs.fth_table[FASTTRAP_PROCS_INDEX(pid)]; 407264434Smarkj fprc = NULL; 408264434Smarkj 409264434Smarkj /* Look up the fasttrap process handle for this process. */ 410264434Smarkj mutex_enter(&bucket->ftb_mtx); 411264434Smarkj for (fprc = bucket->ftb_data; fprc != NULL; fprc = fprc->ftpc_next) { 412264434Smarkj if (fprc->ftpc_pid == pid) { 413264434Smarkj mutex_enter(&fprc->ftpc_mtx); 414264434Smarkj mutex_exit(&bucket->ftb_mtx); 415264434Smarkj break; 416264434Smarkj } 417264434Smarkj } 418264434Smarkj if (fprc == NULL) { 419264434Smarkj mutex_exit(&bucket->ftb_mtx); 420264434Smarkj return; 421264434Smarkj } 422264434Smarkj 423264434Smarkj scrspc = (fasttrap_scrspace_t *)td->t_dtrace_sscr; 424264434Smarkj LIST_REMOVE(scrspc, ftss_next); 425264434Smarkj LIST_INSERT_HEAD(&fprc->ftpc_fscr, scrspc, ftss_next); 426264434Smarkj 427264434Smarkj mutex_exit(&fprc->ftpc_mtx); 428264434Smarkj} 429264434Smarkj#endif 430264434Smarkj 431264434Smarkj/* 432179193Sjb * This function ensures that no threads are actively using the memory 433179193Sjb * associated with probes that were formerly live. 434179193Sjb */ 435179193Sjbstatic void 436179193Sjbfasttrap_mod_barrier(uint64_t gen) 437179193Sjb{ 438179193Sjb int i; 439179193Sjb 440179193Sjb if (gen < fasttrap_mod_gen) 441179193Sjb return; 442179193Sjb 443179193Sjb fasttrap_mod_gen++; 444179193Sjb 445296479Smarkj#ifdef illumos 446211925Srpaulo CPU_FOREACH(i) { 447211925Srpaulo mutex_enter(&fasttrap_cpuc_pid_lock[i]); 448211925Srpaulo mutex_exit(&fasttrap_cpuc_pid_lock[i]); 449179193Sjb } 450296479Smarkj#else 451296479Smarkj rm_wlock(&fasttrap_tp_lock); 452296479Smarkj rm_wunlock(&fasttrap_tp_lock); 453296479Smarkj#endif 454179193Sjb} 455179193Sjb 456179193Sjb/* 457250953Smarkj * This function performs asynchronous cleanup of fasttrap providers. The 458250953Smarkj * Solaris implementation of this mechanism use a timeout that's activated in 459250953Smarkj * fasttrap_pid_cleanup(), but this doesn't work in FreeBSD: one may sleep while 460250953Smarkj * holding the DTrace mutexes, but it is unsafe to sleep in a callout handler. 461250953Smarkj * Thus we use a dedicated process to perform the cleanup when requested. 462179193Sjb */ 463179193Sjb/*ARGSUSED*/ 464179193Sjbstatic void 465179193Sjbfasttrap_pid_cleanup_cb(void *data) 466179193Sjb{ 467179193Sjb fasttrap_provider_t **fpp, *fp; 468179193Sjb fasttrap_bucket_t *bucket; 469179193Sjb dtrace_provider_id_t provid; 470248983Spfg int i, later = 0, rval; 471179193Sjb 472250953Smarkj mtx_lock(&fasttrap_cleanup_mtx); 473250953Smarkj while (!fasttrap_cleanup_drain || later > 0) { 474179193Sjb fasttrap_cleanup_work = 0; 475211738Srpaulo mtx_unlock(&fasttrap_cleanup_mtx); 476179193Sjb 477179193Sjb later = 0; 478179193Sjb 479179193Sjb /* 480179193Sjb * Iterate over all the providers trying to remove the marked 481179193Sjb * ones. If a provider is marked but not retired, we just 482179193Sjb * have to take a crack at removing it -- it's no big deal if 483179193Sjb * we can't. 484179193Sjb */ 485179193Sjb for (i = 0; i < fasttrap_provs.fth_nent; i++) { 486179193Sjb bucket = &fasttrap_provs.fth_table[i]; 487179193Sjb mutex_enter(&bucket->ftb_mtx); 488179193Sjb fpp = (fasttrap_provider_t **)&bucket->ftb_data; 489179193Sjb 490179193Sjb while ((fp = *fpp) != NULL) { 491179193Sjb if (!fp->ftp_marked) { 492179193Sjb fpp = &fp->ftp_next; 493179193Sjb continue; 494179193Sjb } 495179193Sjb 496179193Sjb mutex_enter(&fp->ftp_mtx); 497179193Sjb 498179193Sjb /* 499179193Sjb * If this provider has consumers actively 500179193Sjb * creating probes (ftp_ccount) or is a USDT 501179193Sjb * provider (ftp_mcount), we can't unregister 502179193Sjb * or even condense. 503179193Sjb */ 504179193Sjb if (fp->ftp_ccount != 0 || 505179193Sjb fp->ftp_mcount != 0) { 506179193Sjb mutex_exit(&fp->ftp_mtx); 507179193Sjb fp->ftp_marked = 0; 508179193Sjb continue; 509179193Sjb } 510179193Sjb 511179193Sjb if (!fp->ftp_retired || fp->ftp_rcount != 0) 512179193Sjb fp->ftp_marked = 0; 513179193Sjb 514179193Sjb mutex_exit(&fp->ftp_mtx); 515179193Sjb 516179193Sjb /* 517179193Sjb * If we successfully unregister this 518179193Sjb * provider we can remove it from the hash 519179193Sjb * chain and free the memory. If our attempt 520179193Sjb * to unregister fails and this is a retired 521179193Sjb * provider, increment our flag to try again 522179193Sjb * pretty soon. If we've consumed more than 523179193Sjb * half of our total permitted number of 524179193Sjb * probes call dtrace_condense() to try to 525179193Sjb * clean out the unenabled probes. 526179193Sjb */ 527179193Sjb provid = fp->ftp_provid; 528248983Spfg if ((rval = dtrace_unregister(provid)) != 0) { 529179193Sjb if (fasttrap_total > fasttrap_max / 2) 530179193Sjb (void) dtrace_condense(provid); 531248983Spfg 532248983Spfg if (rval == EAGAIN) 533248983Spfg fp->ftp_marked = 1; 534248983Spfg 535179193Sjb later += fp->ftp_marked; 536179193Sjb fpp = &fp->ftp_next; 537179193Sjb } else { 538179193Sjb *fpp = fp->ftp_next; 539179193Sjb fasttrap_provider_free(fp); 540179193Sjb } 541179193Sjb } 542179193Sjb mutex_exit(&bucket->ftb_mtx); 543179193Sjb } 544250953Smarkj mtx_lock(&fasttrap_cleanup_mtx); 545179193Sjb 546250953Smarkj /* 547250953Smarkj * If we were unable to retire a provider, try again after a 548250953Smarkj * second. This situation can occur in certain circumstances 549250953Smarkj * where providers cannot be unregistered even though they have 550250953Smarkj * no probes enabled because of an execution of dtrace -l or 551250953Smarkj * something similar. 552250953Smarkj */ 553250953Smarkj if (later > 0 || fasttrap_cleanup_work || 554250953Smarkj fasttrap_cleanup_drain) { 555250953Smarkj mtx_unlock(&fasttrap_cleanup_mtx); 556250953Smarkj pause("ftclean", hz); 557250953Smarkj mtx_lock(&fasttrap_cleanup_mtx); 558250953Smarkj } else 559250953Smarkj mtx_sleep(&fasttrap_cleanup_cv, &fasttrap_cleanup_mtx, 560250953Smarkj 0, "ftcl", 0); 561179193Sjb } 562179193Sjb 563179193Sjb /* 564250953Smarkj * Wake up the thread in fasttrap_unload() now that we're done. 565179193Sjb */ 566250953Smarkj wakeup(&fasttrap_cleanup_drain); 567250953Smarkj mtx_unlock(&fasttrap_cleanup_mtx); 568179193Sjb 569250953Smarkj kthread_exit(); 570179193Sjb} 571179193Sjb 572179193Sjb/* 573179193Sjb * Activates the asynchronous cleanup mechanism. 574179193Sjb */ 575179193Sjbstatic void 576179193Sjbfasttrap_pid_cleanup(void) 577179193Sjb{ 578211738Srpaulo 579211738Srpaulo mtx_lock(&fasttrap_cleanup_mtx); 580250953Smarkj if (!fasttrap_cleanup_work) { 581250953Smarkj fasttrap_cleanup_work = 1; 582250953Smarkj wakeup(&fasttrap_cleanup_cv); 583250953Smarkj } 584211738Srpaulo mtx_unlock(&fasttrap_cleanup_mtx); 585179193Sjb} 586179193Sjb 587179193Sjb/* 588179193Sjb * This is called from cfork() via dtrace_fasttrap_fork(). The child 589179198Sjb * process's address space is (roughly) a copy of the parent process's so 590179193Sjb * we have to remove all the instrumentation we had previously enabled in the 591179193Sjb * parent. 592179193Sjb */ 593179193Sjbstatic void 594179193Sjbfasttrap_fork(proc_t *p, proc_t *cp) 595179193Sjb{ 596277300Ssmh#ifndef illumos 597264434Smarkj fasttrap_scrblock_t *scrblk; 598264434Smarkj fasttrap_proc_t *fprc = NULL; 599264434Smarkj#endif 600179193Sjb pid_t ppid = p->p_pid; 601179193Sjb int i; 602179193Sjb 603277300Ssmh#ifdef illumos 604179193Sjb ASSERT(curproc == p); 605179193Sjb ASSERT(p->p_proc_flag & P_PR_LOCK); 606211738Srpaulo#else 607211738Srpaulo PROC_LOCK_ASSERT(p, MA_OWNED); 608211738Srpaulo#endif 609277300Ssmh#ifdef illumos 610179193Sjb ASSERT(p->p_dtrace_count > 0); 611211738Srpaulo#else 612212357Srpaulo if (p->p_dtrace_helpers) { 613212357Srpaulo /* 614212357Srpaulo * dtrace_helpers_duplicate() allocates memory. 615212357Srpaulo */ 616212494Srpaulo _PHOLD(cp); 617212357Srpaulo PROC_UNLOCK(p); 618212357Srpaulo PROC_UNLOCK(cp); 619212357Srpaulo dtrace_helpers_duplicate(p, cp); 620212357Srpaulo PROC_LOCK(cp); 621212357Srpaulo PROC_LOCK(p); 622212494Srpaulo _PRELE(cp); 623212357Srpaulo } 624211738Srpaulo /* 625211738Srpaulo * This check is purposely here instead of in kern_fork.c because, 626211738Srpaulo * for legal resons, we cannot include the dtrace_cddl.h header 627211738Srpaulo * inside kern_fork.c and insert if-clause there. 628211738Srpaulo */ 629211738Srpaulo if (p->p_dtrace_count == 0) 630211738Srpaulo return; 631211738Srpaulo#endif 632179193Sjb ASSERT(cp->p_dtrace_count == 0); 633179193Sjb 634179193Sjb /* 635179193Sjb * This would be simpler and faster if we maintained per-process 636179193Sjb * hash tables of enabled tracepoints. It could, however, potentially 637179193Sjb * slow down execution of a tracepoint since we'd need to go 638179193Sjb * through two levels of indirection. In the future, we should 639179193Sjb * consider either maintaining per-process ancillary lists of 640179193Sjb * enabled tracepoints or hanging a pointer to a per-process hash 641179193Sjb * table of enabled tracepoints off the proc structure. 642179193Sjb */ 643179193Sjb 644179193Sjb /* 645179193Sjb * We don't have to worry about the child process disappearing 646179193Sjb * because we're in fork(). 647179193Sjb */ 648277300Ssmh#ifdef illumos 649211738Srpaulo mtx_lock_spin(&cp->p_slock); 650179193Sjb sprlock_proc(cp); 651211738Srpaulo mtx_unlock_spin(&cp->p_slock); 652212494Srpaulo#else 653254198Srpaulo /* 654254198Srpaulo * fasttrap_tracepoint_remove() expects the child process to be 655254198Srpaulo * unlocked and the VM then expects curproc to be unlocked. 656254198Srpaulo */ 657212494Srpaulo _PHOLD(cp); 658254198Srpaulo PROC_UNLOCK(cp); 659254198Srpaulo PROC_UNLOCK(p); 660211738Srpaulo#endif 661179193Sjb 662179193Sjb /* 663179193Sjb * Iterate over every tracepoint looking for ones that belong to the 664179193Sjb * parent process, and remove each from the child process. 665179193Sjb */ 666179193Sjb for (i = 0; i < fasttrap_tpoints.fth_nent; i++) { 667179193Sjb fasttrap_tracepoint_t *tp; 668179193Sjb fasttrap_bucket_t *bucket = &fasttrap_tpoints.fth_table[i]; 669179193Sjb 670179193Sjb mutex_enter(&bucket->ftb_mtx); 671179193Sjb for (tp = bucket->ftb_data; tp != NULL; tp = tp->ftt_next) { 672179193Sjb if (tp->ftt_pid == ppid && 673179193Sjb tp->ftt_proc->ftpc_acount != 0) { 674179193Sjb int ret = fasttrap_tracepoint_remove(cp, tp); 675179193Sjb ASSERT(ret == 0); 676179198Sjb 677179198Sjb /* 678179198Sjb * The count of active providers can only be 679179198Sjb * decremented (i.e. to zero) during exec, 680179198Sjb * exit, and removal of a meta provider so it 681179198Sjb * should be impossible to drop the count 682179198Sjb * mid-fork. 683179198Sjb */ 684179198Sjb ASSERT(tp->ftt_proc->ftpc_acount != 0); 685277300Ssmh#ifndef illumos 686264434Smarkj fprc = tp->ftt_proc; 687264434Smarkj#endif 688179193Sjb } 689179193Sjb } 690179193Sjb mutex_exit(&bucket->ftb_mtx); 691264434Smarkj 692277300Ssmh#ifndef illumos 693264434Smarkj /* 694264434Smarkj * Unmap any scratch space inherited from the parent's address 695264434Smarkj * space. 696264434Smarkj */ 697264434Smarkj if (fprc != NULL) { 698264434Smarkj mutex_enter(&fprc->ftpc_mtx); 699264434Smarkj LIST_FOREACH(scrblk, &fprc->ftpc_scrblks, ftsb_next) { 700264434Smarkj vm_map_remove(&cp->p_vmspace->vm_map, 701264434Smarkj scrblk->ftsb_addr, 702264434Smarkj scrblk->ftsb_addr + FASTTRAP_SCRBLOCK_SIZE); 703264434Smarkj } 704264434Smarkj mutex_exit(&fprc->ftpc_mtx); 705264434Smarkj } 706264434Smarkj#endif 707179193Sjb } 708179193Sjb 709277300Ssmh#ifdef illumos 710179193Sjb mutex_enter(&cp->p_lock); 711179193Sjb sprunlock(cp); 712212494Srpaulo#else 713254198Srpaulo PROC_LOCK(p); 714254198Srpaulo PROC_LOCK(cp); 715212494Srpaulo _PRELE(cp); 716211738Srpaulo#endif 717179193Sjb} 718179193Sjb 719179193Sjb/* 720179193Sjb * This is called from proc_exit() or from exec_common() if p_dtrace_probes 721179193Sjb * is set on the proc structure to indicate that there is a pid provider 722179193Sjb * associated with this process. 723179193Sjb */ 724179193Sjbstatic void 725179193Sjbfasttrap_exec_exit(proc_t *p) 726179193Sjb{ 727277300Ssmh#ifndef illumos 728264434Smarkj struct thread *td; 729264434Smarkj#endif 730264434Smarkj 731277300Ssmh#ifdef illumos 732179193Sjb ASSERT(p == curproc); 733264434Smarkj#else 734211738Srpaulo PROC_LOCK_ASSERT(p, MA_OWNED); 735212494Srpaulo _PHOLD(p); 736264434Smarkj /* 737264434Smarkj * Since struct threads may be recycled, we cannot rely on t_dtrace_sscr 738264434Smarkj * fields to be zeroed by kdtrace_thread_ctor. Thus we must zero it 739264434Smarkj * ourselves when a process exits. 740264434Smarkj */ 741264434Smarkj FOREACH_THREAD_IN_PROC(p, td) 742264434Smarkj td->t_dtrace_sscr = NULL; 743211738Srpaulo PROC_UNLOCK(p); 744264434Smarkj#endif 745179193Sjb 746179193Sjb /* 747179193Sjb * We clean up the pid provider for this process here; user-land 748179193Sjb * static probes are handled by the meta-provider remove entry point. 749179193Sjb */ 750179193Sjb fasttrap_provider_retire(p->p_pid, FASTTRAP_PID_NAME, 0); 751277300Ssmh#ifndef illumos 752212357Srpaulo if (p->p_dtrace_helpers) 753212357Srpaulo dtrace_helpers_destroy(p); 754211738Srpaulo PROC_LOCK(p); 755212494Srpaulo _PRELE(p); 756264434Smarkj#endif 757179193Sjb} 758179193Sjb 759179193Sjb 760179193Sjb/*ARGSUSED*/ 761179193Sjbstatic void 762211738Srpaulofasttrap_pid_provide(void *arg, dtrace_probedesc_t *desc) 763179193Sjb{ 764179193Sjb /* 765179193Sjb * There are no "default" pid probes. 766179193Sjb */ 767179193Sjb} 768179193Sjb 769179193Sjbstatic int 770179193Sjbfasttrap_tracepoint_enable(proc_t *p, fasttrap_probe_t *probe, uint_t index) 771179193Sjb{ 772179193Sjb fasttrap_tracepoint_t *tp, *new_tp = NULL; 773179193Sjb fasttrap_bucket_t *bucket; 774179193Sjb fasttrap_id_t *id; 775179193Sjb pid_t pid; 776179193Sjb uintptr_t pc; 777179193Sjb 778179193Sjb ASSERT(index < probe->ftp_ntps); 779179193Sjb 780179193Sjb pid = probe->ftp_pid; 781179193Sjb pc = probe->ftp_tps[index].fit_tp->ftt_pc; 782179193Sjb id = &probe->ftp_tps[index].fit_id; 783179193Sjb 784179193Sjb ASSERT(probe->ftp_tps[index].fit_tp->ftt_pid == pid); 785179193Sjb 786277300Ssmh#ifdef illumos 787179193Sjb ASSERT(!(p->p_flag & SVFORK)); 788211738Srpaulo#endif 789179193Sjb 790179193Sjb /* 791179193Sjb * Before we make any modifications, make sure we've imposed a barrier 792179193Sjb * on the generation in which this probe was last modified. 793179193Sjb */ 794179193Sjb fasttrap_mod_barrier(probe->ftp_gen); 795179193Sjb 796179193Sjb bucket = &fasttrap_tpoints.fth_table[FASTTRAP_TPOINTS_INDEX(pid, pc)]; 797179193Sjb 798179193Sjb /* 799179193Sjb * If the tracepoint has already been enabled, just add our id to the 800179193Sjb * list of interested probes. This may be our second time through 801179193Sjb * this path in which case we'll have constructed the tracepoint we'd 802179193Sjb * like to install. If we can't find a match, and have an allocated 803179193Sjb * tracepoint ready to go, enable that one now. 804179193Sjb * 805179193Sjb * A tracepoint whose process is defunct is also considered defunct. 806179193Sjb */ 807179193Sjbagain: 808179193Sjb mutex_enter(&bucket->ftb_mtx); 809179193Sjb for (tp = bucket->ftb_data; tp != NULL; tp = tp->ftt_next) { 810179198Sjb /* 811179198Sjb * Note that it's safe to access the active count on the 812179198Sjb * associated proc structure because we know that at least one 813179198Sjb * provider (this one) will still be around throughout this 814179198Sjb * operation. 815179198Sjb */ 816179193Sjb if (tp->ftt_pid != pid || tp->ftt_pc != pc || 817179193Sjb tp->ftt_proc->ftpc_acount == 0) 818179193Sjb continue; 819179193Sjb 820179193Sjb /* 821179193Sjb * Now that we've found a matching tracepoint, it would be 822179193Sjb * a decent idea to confirm that the tracepoint is still 823179193Sjb * enabled and the trap instruction hasn't been overwritten. 824179193Sjb * Since this is a little hairy, we'll punt for now. 825179193Sjb */ 826179193Sjb 827179193Sjb /* 828179193Sjb * This can't be the first interested probe. We don't have 829179193Sjb * to worry about another thread being in the midst of 830179193Sjb * deleting this tracepoint (which would be the only valid 831179193Sjb * reason for a tracepoint to have no interested probes) 832179193Sjb * since we're holding P_PR_LOCK for this process. 833179193Sjb */ 834179193Sjb ASSERT(tp->ftt_ids != NULL || tp->ftt_retids != NULL); 835179193Sjb 836179193Sjb switch (id->fti_ptype) { 837179193Sjb case DTFTP_ENTRY: 838179193Sjb case DTFTP_OFFSETS: 839179193Sjb case DTFTP_IS_ENABLED: 840179193Sjb id->fti_next = tp->ftt_ids; 841179193Sjb membar_producer(); 842179193Sjb tp->ftt_ids = id; 843179193Sjb membar_producer(); 844179193Sjb break; 845179193Sjb 846179193Sjb case DTFTP_RETURN: 847179193Sjb case DTFTP_POST_OFFSETS: 848179193Sjb id->fti_next = tp->ftt_retids; 849179193Sjb membar_producer(); 850179193Sjb tp->ftt_retids = id; 851179193Sjb membar_producer(); 852179193Sjb break; 853179193Sjb 854179193Sjb default: 855179193Sjb ASSERT(0); 856179193Sjb } 857179193Sjb 858179193Sjb mutex_exit(&bucket->ftb_mtx); 859179193Sjb 860179193Sjb if (new_tp != NULL) { 861179193Sjb new_tp->ftt_ids = NULL; 862179193Sjb new_tp->ftt_retids = NULL; 863179193Sjb } 864179193Sjb 865179193Sjb return (0); 866179193Sjb } 867179193Sjb 868179193Sjb /* 869179193Sjb * If we have a good tracepoint ready to go, install it now while 870179193Sjb * we have the lock held and no one can screw with us. 871179193Sjb */ 872179193Sjb if (new_tp != NULL) { 873179193Sjb int rc = 0; 874179193Sjb 875179193Sjb new_tp->ftt_next = bucket->ftb_data; 876179193Sjb membar_producer(); 877179193Sjb bucket->ftb_data = new_tp; 878179193Sjb membar_producer(); 879179193Sjb mutex_exit(&bucket->ftb_mtx); 880179193Sjb 881179193Sjb /* 882179193Sjb * Activate the tracepoint in the ISA-specific manner. 883179193Sjb * If this fails, we need to report the failure, but 884179193Sjb * indicate that this tracepoint must still be disabled 885179193Sjb * by calling fasttrap_tracepoint_disable(). 886179193Sjb */ 887179193Sjb if (fasttrap_tracepoint_install(p, new_tp) != 0) 888179193Sjb rc = FASTTRAP_ENABLE_PARTIAL; 889179193Sjb 890179193Sjb /* 891179193Sjb * Increment the count of the number of tracepoints active in 892179193Sjb * the victim process. 893179193Sjb */ 894277300Ssmh#ifdef illumos 895179193Sjb ASSERT(p->p_proc_flag & P_PR_LOCK); 896211738Srpaulo#endif 897179193Sjb p->p_dtrace_count++; 898179193Sjb 899179193Sjb return (rc); 900179193Sjb } 901179193Sjb 902179193Sjb mutex_exit(&bucket->ftb_mtx); 903179193Sjb 904179193Sjb /* 905179193Sjb * Initialize the tracepoint that's been preallocated with the probe. 906179193Sjb */ 907179193Sjb new_tp = probe->ftp_tps[index].fit_tp; 908179193Sjb 909179193Sjb ASSERT(new_tp->ftt_pid == pid); 910179193Sjb ASSERT(new_tp->ftt_pc == pc); 911179193Sjb ASSERT(new_tp->ftt_proc == probe->ftp_prov->ftp_proc); 912179193Sjb ASSERT(new_tp->ftt_ids == NULL); 913179193Sjb ASSERT(new_tp->ftt_retids == NULL); 914179193Sjb 915179193Sjb switch (id->fti_ptype) { 916179193Sjb case DTFTP_ENTRY: 917179193Sjb case DTFTP_OFFSETS: 918179193Sjb case DTFTP_IS_ENABLED: 919179193Sjb id->fti_next = NULL; 920179193Sjb new_tp->ftt_ids = id; 921179193Sjb break; 922179193Sjb 923179193Sjb case DTFTP_RETURN: 924179193Sjb case DTFTP_POST_OFFSETS: 925179193Sjb id->fti_next = NULL; 926179193Sjb new_tp->ftt_retids = id; 927179193Sjb break; 928179193Sjb 929179193Sjb default: 930179193Sjb ASSERT(0); 931179193Sjb } 932179193Sjb 933179193Sjb /* 934179193Sjb * If the ISA-dependent initialization goes to plan, go back to the 935179193Sjb * beginning and try to install this freshly made tracepoint. 936179193Sjb */ 937179193Sjb if (fasttrap_tracepoint_init(p, new_tp, pc, id->fti_ptype) == 0) 938179193Sjb goto again; 939179193Sjb 940179193Sjb new_tp->ftt_ids = NULL; 941179193Sjb new_tp->ftt_retids = NULL; 942179193Sjb 943179193Sjb return (FASTTRAP_ENABLE_FAIL); 944179193Sjb} 945179193Sjb 946179193Sjbstatic void 947179193Sjbfasttrap_tracepoint_disable(proc_t *p, fasttrap_probe_t *probe, uint_t index) 948179193Sjb{ 949179193Sjb fasttrap_bucket_t *bucket; 950179193Sjb fasttrap_provider_t *provider = probe->ftp_prov; 951179193Sjb fasttrap_tracepoint_t **pp, *tp; 952211738Srpaulo fasttrap_id_t *id, **idp = NULL; 953179193Sjb pid_t pid; 954179193Sjb uintptr_t pc; 955179193Sjb 956179193Sjb ASSERT(index < probe->ftp_ntps); 957179193Sjb 958179193Sjb pid = probe->ftp_pid; 959179193Sjb pc = probe->ftp_tps[index].fit_tp->ftt_pc; 960179193Sjb id = &probe->ftp_tps[index].fit_id; 961179193Sjb 962179193Sjb ASSERT(probe->ftp_tps[index].fit_tp->ftt_pid == pid); 963179193Sjb 964179193Sjb /* 965179193Sjb * Find the tracepoint and make sure that our id is one of the 966179193Sjb * ones registered with it. 967179193Sjb */ 968179193Sjb bucket = &fasttrap_tpoints.fth_table[FASTTRAP_TPOINTS_INDEX(pid, pc)]; 969179193Sjb mutex_enter(&bucket->ftb_mtx); 970179193Sjb for (tp = bucket->ftb_data; tp != NULL; tp = tp->ftt_next) { 971179193Sjb if (tp->ftt_pid == pid && tp->ftt_pc == pc && 972179193Sjb tp->ftt_proc == provider->ftp_proc) 973179193Sjb break; 974179193Sjb } 975179193Sjb 976179193Sjb /* 977179193Sjb * If we somehow lost this tracepoint, we're in a world of hurt. 978179193Sjb */ 979179193Sjb ASSERT(tp != NULL); 980179193Sjb 981179193Sjb switch (id->fti_ptype) { 982179193Sjb case DTFTP_ENTRY: 983179193Sjb case DTFTP_OFFSETS: 984179193Sjb case DTFTP_IS_ENABLED: 985179193Sjb ASSERT(tp->ftt_ids != NULL); 986179193Sjb idp = &tp->ftt_ids; 987179193Sjb break; 988179193Sjb 989179193Sjb case DTFTP_RETURN: 990179193Sjb case DTFTP_POST_OFFSETS: 991179193Sjb ASSERT(tp->ftt_retids != NULL); 992179193Sjb idp = &tp->ftt_retids; 993179193Sjb break; 994179193Sjb 995179193Sjb default: 996179193Sjb ASSERT(0); 997179193Sjb } 998179193Sjb 999179193Sjb while ((*idp)->fti_probe != probe) { 1000179193Sjb idp = &(*idp)->fti_next; 1001179193Sjb ASSERT(*idp != NULL); 1002179193Sjb } 1003179193Sjb 1004179193Sjb id = *idp; 1005179193Sjb *idp = id->fti_next; 1006179193Sjb membar_producer(); 1007179193Sjb 1008179193Sjb ASSERT(id->fti_probe == probe); 1009179193Sjb 1010179193Sjb /* 1011179193Sjb * If there are other registered enablings of this tracepoint, we're 1012179193Sjb * all done, but if this was the last probe assocated with this 1013179193Sjb * this tracepoint, we need to remove and free it. 1014179193Sjb */ 1015179193Sjb if (tp->ftt_ids != NULL || tp->ftt_retids != NULL) { 1016179193Sjb 1017179193Sjb /* 1018179193Sjb * If the current probe's tracepoint is in use, swap it 1019179193Sjb * for an unused tracepoint. 1020179193Sjb */ 1021179193Sjb if (tp == probe->ftp_tps[index].fit_tp) { 1022179193Sjb fasttrap_probe_t *tmp_probe; 1023179193Sjb fasttrap_tracepoint_t **tmp_tp; 1024179193Sjb uint_t tmp_index; 1025179193Sjb 1026179193Sjb if (tp->ftt_ids != NULL) { 1027179193Sjb tmp_probe = tp->ftt_ids->fti_probe; 1028179193Sjb /* LINTED - alignment */ 1029179193Sjb tmp_index = FASTTRAP_ID_INDEX(tp->ftt_ids); 1030179193Sjb tmp_tp = &tmp_probe->ftp_tps[tmp_index].fit_tp; 1031179193Sjb } else { 1032179193Sjb tmp_probe = tp->ftt_retids->fti_probe; 1033179193Sjb /* LINTED - alignment */ 1034179193Sjb tmp_index = FASTTRAP_ID_INDEX(tp->ftt_retids); 1035179193Sjb tmp_tp = &tmp_probe->ftp_tps[tmp_index].fit_tp; 1036179193Sjb } 1037179193Sjb 1038179193Sjb ASSERT(*tmp_tp != NULL); 1039179193Sjb ASSERT(*tmp_tp != probe->ftp_tps[index].fit_tp); 1040179193Sjb ASSERT((*tmp_tp)->ftt_ids == NULL); 1041179193Sjb ASSERT((*tmp_tp)->ftt_retids == NULL); 1042179193Sjb 1043179193Sjb probe->ftp_tps[index].fit_tp = *tmp_tp; 1044179193Sjb *tmp_tp = tp; 1045179193Sjb } 1046179193Sjb 1047179193Sjb mutex_exit(&bucket->ftb_mtx); 1048179193Sjb 1049179193Sjb /* 1050179193Sjb * Tag the modified probe with the generation in which it was 1051179193Sjb * changed. 1052179193Sjb */ 1053179193Sjb probe->ftp_gen = fasttrap_mod_gen; 1054179193Sjb return; 1055179193Sjb } 1056179193Sjb 1057179193Sjb mutex_exit(&bucket->ftb_mtx); 1058179193Sjb 1059179193Sjb /* 1060179193Sjb * We can't safely remove the tracepoint from the set of active 1061179193Sjb * tracepoints until we've actually removed the fasttrap instruction 1062179193Sjb * from the process's text. We can, however, operate on this 1063179193Sjb * tracepoint secure in the knowledge that no other thread is going to 1064179193Sjb * be looking at it since we hold P_PR_LOCK on the process if it's 1065179193Sjb * live or we hold the provider lock on the process if it's dead and 1066179193Sjb * gone. 1067179193Sjb */ 1068179193Sjb 1069179193Sjb /* 1070179193Sjb * We only need to remove the actual instruction if we're looking 1071179193Sjb * at an existing process 1072179193Sjb */ 1073179193Sjb if (p != NULL) { 1074179193Sjb /* 1075179193Sjb * If we fail to restore the instruction we need to kill 1076179193Sjb * this process since it's in a completely unrecoverable 1077179193Sjb * state. 1078179193Sjb */ 1079179193Sjb if (fasttrap_tracepoint_remove(p, tp) != 0) 1080179193Sjb fasttrap_sigtrap(p, NULL, pc); 1081179193Sjb 1082179193Sjb /* 1083179193Sjb * Decrement the count of the number of tracepoints active 1084179193Sjb * in the victim process. 1085179193Sjb */ 1086277300Ssmh#ifdef illumos 1087179193Sjb ASSERT(p->p_proc_flag & P_PR_LOCK); 1088211738Srpaulo#endif 1089179193Sjb p->p_dtrace_count--; 1090179193Sjb } 1091179193Sjb 1092179193Sjb /* 1093179193Sjb * Remove the probe from the hash table of active tracepoints. 1094179193Sjb */ 1095179193Sjb mutex_enter(&bucket->ftb_mtx); 1096179193Sjb pp = (fasttrap_tracepoint_t **)&bucket->ftb_data; 1097179193Sjb ASSERT(*pp != NULL); 1098179193Sjb while (*pp != tp) { 1099179193Sjb pp = &(*pp)->ftt_next; 1100179193Sjb ASSERT(*pp != NULL); 1101179193Sjb } 1102179193Sjb 1103179193Sjb *pp = tp->ftt_next; 1104179193Sjb membar_producer(); 1105179193Sjb 1106179193Sjb mutex_exit(&bucket->ftb_mtx); 1107179193Sjb 1108179193Sjb /* 1109179193Sjb * Tag the modified probe with the generation in which it was changed. 1110179193Sjb */ 1111179193Sjb probe->ftp_gen = fasttrap_mod_gen; 1112179193Sjb} 1113179193Sjb 1114179193Sjbstatic void 1115179193Sjbfasttrap_enable_callbacks(void) 1116179193Sjb{ 1117179193Sjb /* 1118179193Sjb * We don't have to play the rw lock game here because we're 1119179193Sjb * providing something rather than taking something away -- 1120179193Sjb * we can be sure that no threads have tried to follow this 1121179193Sjb * function pointer yet. 1122179193Sjb */ 1123179193Sjb mutex_enter(&fasttrap_count_mtx); 1124179193Sjb if (fasttrap_pid_count == 0) { 1125179193Sjb ASSERT(dtrace_pid_probe_ptr == NULL); 1126179193Sjb ASSERT(dtrace_return_probe_ptr == NULL); 1127179193Sjb dtrace_pid_probe_ptr = &fasttrap_pid_probe; 1128179193Sjb dtrace_return_probe_ptr = &fasttrap_return_probe; 1129179193Sjb } 1130179193Sjb ASSERT(dtrace_pid_probe_ptr == &fasttrap_pid_probe); 1131179193Sjb ASSERT(dtrace_return_probe_ptr == &fasttrap_return_probe); 1132179193Sjb fasttrap_pid_count++; 1133179193Sjb mutex_exit(&fasttrap_count_mtx); 1134179193Sjb} 1135179193Sjb 1136179193Sjbstatic void 1137179193Sjbfasttrap_disable_callbacks(void) 1138179193Sjb{ 1139277300Ssmh#ifdef illumos 1140179193Sjb ASSERT(MUTEX_HELD(&cpu_lock)); 1141211738Srpaulo#endif 1142179193Sjb 1143211738Srpaulo 1144179193Sjb mutex_enter(&fasttrap_count_mtx); 1145179193Sjb ASSERT(fasttrap_pid_count > 0); 1146179193Sjb fasttrap_pid_count--; 1147179193Sjb if (fasttrap_pid_count == 0) { 1148277300Ssmh#ifdef illumos 1149179193Sjb cpu_t *cur, *cpu = CPU; 1150179193Sjb 1151179193Sjb for (cur = cpu->cpu_next_onln; cur != cpu; 1152179193Sjb cur = cur->cpu_next_onln) { 1153179193Sjb rw_enter(&cur->cpu_ft_lock, RW_WRITER); 1154179193Sjb } 1155211738Srpaulo#endif 1156179193Sjb dtrace_pid_probe_ptr = NULL; 1157179193Sjb dtrace_return_probe_ptr = NULL; 1158277300Ssmh#ifdef illumos 1159179193Sjb for (cur = cpu->cpu_next_onln; cur != cpu; 1160179193Sjb cur = cur->cpu_next_onln) { 1161179193Sjb rw_exit(&cur->cpu_ft_lock); 1162179193Sjb } 1163211738Srpaulo#endif 1164179193Sjb } 1165179193Sjb mutex_exit(&fasttrap_count_mtx); 1166179193Sjb} 1167179193Sjb 1168179193Sjb/*ARGSUSED*/ 1169179193Sjbstatic void 1170179193Sjbfasttrap_pid_enable(void *arg, dtrace_id_t id, void *parg) 1171179193Sjb{ 1172179193Sjb fasttrap_probe_t *probe = parg; 1173211738Srpaulo proc_t *p = NULL; 1174179193Sjb int i, rc; 1175179193Sjb 1176179193Sjb ASSERT(probe != NULL); 1177179193Sjb ASSERT(!probe->ftp_enabled); 1178179193Sjb ASSERT(id == probe->ftp_id); 1179277300Ssmh#ifdef illumos 1180179193Sjb ASSERT(MUTEX_HELD(&cpu_lock)); 1181211738Srpaulo#endif 1182179193Sjb 1183179193Sjb /* 1184179193Sjb * Increment the count of enabled probes on this probe's provider; 1185179193Sjb * the provider can't go away while the probe still exists. We 1186179193Sjb * must increment this even if we aren't able to properly enable 1187179193Sjb * this probe. 1188179193Sjb */ 1189179193Sjb mutex_enter(&probe->ftp_prov->ftp_mtx); 1190179193Sjb probe->ftp_prov->ftp_rcount++; 1191179193Sjb mutex_exit(&probe->ftp_prov->ftp_mtx); 1192179193Sjb 1193179193Sjb /* 1194179193Sjb * If this probe's provider is retired (meaning it was valid in a 1195179193Sjb * previously exec'ed incarnation of this address space), bail out. The 1196179193Sjb * provider can't go away while we're in this code path. 1197179193Sjb */ 1198179193Sjb if (probe->ftp_prov->ftp_retired) 1199179193Sjb return; 1200179193Sjb 1201179193Sjb /* 1202179193Sjb * If we can't find the process, it may be that we're in the context of 1203179193Sjb * a fork in which the traced process is being born and we're copying 1204179193Sjb * USDT probes. Otherwise, the process is gone so bail. 1205179193Sjb */ 1206277300Ssmh#ifdef illumos 1207179193Sjb if ((p = sprlock(probe->ftp_pid)) == NULL) { 1208179193Sjb if ((curproc->p_flag & SFORKING) == 0) 1209179193Sjb return; 1210179193Sjb 1211179193Sjb mutex_enter(&pidlock); 1212179193Sjb p = prfind(probe->ftp_pid); 1213179193Sjb 1214287642Smarkj if (p == NULL) { 1215287642Smarkj /* 1216287642Smarkj * So it's not that the target process is being born, 1217287642Smarkj * it's that it isn't there at all (and we simply 1218287642Smarkj * happen to be forking). Anyway, we know that the 1219287642Smarkj * target is definitely gone, so bail out. 1220287642Smarkj */ 1221287642Smarkj mutex_exit(&pidlock); 1222287642Smarkj return (0); 1223287642Smarkj } 1224287642Smarkj 1225179193Sjb /* 1226179193Sjb * Confirm that curproc is indeed forking the process in which 1227179193Sjb * we're trying to enable probes. 1228179193Sjb */ 1229179193Sjb ASSERT(p->p_parent == curproc); 1230179193Sjb ASSERT(p->p_stat == SIDL); 1231179193Sjb 1232179193Sjb mutex_enter(&p->p_lock); 1233179193Sjb mutex_exit(&pidlock); 1234179193Sjb 1235179193Sjb sprlock_proc(p); 1236179193Sjb } 1237179193Sjb 1238179193Sjb ASSERT(!(p->p_flag & SVFORK)); 1239179193Sjb mutex_exit(&p->p_lock); 1240211738Srpaulo#else 1241211738Srpaulo if ((p = pfind(probe->ftp_pid)) == NULL) 1242211738Srpaulo return; 1243211738Srpaulo#endif 1244179193Sjb 1245179193Sjb /* 1246179193Sjb * We have to enable the trap entry point before any user threads have 1247179193Sjb * the chance to execute the trap instruction we're about to place 1248179193Sjb * in their process's text. 1249179193Sjb */ 1250212494Srpaulo#ifdef __FreeBSD__ 1251212494Srpaulo /* 1252212494Srpaulo * pfind() returns a locked process. 1253212494Srpaulo */ 1254212494Srpaulo _PHOLD(p); 1255211738Srpaulo PROC_UNLOCK(p); 1256212494Srpaulo#endif 1257179193Sjb fasttrap_enable_callbacks(); 1258179193Sjb 1259179193Sjb /* 1260179193Sjb * Enable all the tracepoints and add this probe's id to each 1261179193Sjb * tracepoint's list of active probes. 1262179193Sjb */ 1263179193Sjb for (i = 0; i < probe->ftp_ntps; i++) { 1264179193Sjb if ((rc = fasttrap_tracepoint_enable(p, probe, i)) != 0) { 1265179193Sjb /* 1266179193Sjb * If enabling the tracepoint failed completely, 1267179193Sjb * we don't have to disable it; if the failure 1268179193Sjb * was only partial we must disable it. 1269179193Sjb */ 1270179193Sjb if (rc == FASTTRAP_ENABLE_FAIL) 1271179193Sjb i--; 1272179193Sjb else 1273179193Sjb ASSERT(rc == FASTTRAP_ENABLE_PARTIAL); 1274179193Sjb 1275179193Sjb /* 1276179193Sjb * Back up and pull out all the tracepoints we've 1277179193Sjb * created so far for this probe. 1278179193Sjb */ 1279179193Sjb while (i >= 0) { 1280179193Sjb fasttrap_tracepoint_disable(p, probe, i); 1281179193Sjb i--; 1282179193Sjb } 1283179193Sjb 1284277300Ssmh#ifdef illumos 1285179193Sjb mutex_enter(&p->p_lock); 1286179193Sjb sprunlock(p); 1287211738Srpaulo#else 1288212494Srpaulo PRELE(p); 1289211738Srpaulo#endif 1290179193Sjb 1291179193Sjb /* 1292179193Sjb * Since we're not actually enabling this probe, 1293179193Sjb * drop our reference on the trap table entry. 1294179193Sjb */ 1295179193Sjb fasttrap_disable_callbacks(); 1296179193Sjb return; 1297179193Sjb } 1298179193Sjb } 1299277300Ssmh#ifdef illumos 1300179193Sjb mutex_enter(&p->p_lock); 1301179193Sjb sprunlock(p); 1302211738Srpaulo#else 1303212494Srpaulo PRELE(p); 1304211738Srpaulo#endif 1305179193Sjb 1306179193Sjb probe->ftp_enabled = 1; 1307179193Sjb} 1308179193Sjb 1309179193Sjb/*ARGSUSED*/ 1310179193Sjbstatic void 1311179193Sjbfasttrap_pid_disable(void *arg, dtrace_id_t id, void *parg) 1312179193Sjb{ 1313179193Sjb fasttrap_probe_t *probe = parg; 1314179193Sjb fasttrap_provider_t *provider = probe->ftp_prov; 1315179193Sjb proc_t *p; 1316179193Sjb int i, whack = 0; 1317179193Sjb 1318179193Sjb ASSERT(id == probe->ftp_id); 1319179193Sjb 1320211738Srpaulo mutex_enter(&provider->ftp_mtx); 1321211738Srpaulo 1322179193Sjb /* 1323179193Sjb * We won't be able to acquire a /proc-esque lock on the process 1324179193Sjb * iff the process is dead and gone. In this case, we rely on the 1325179193Sjb * provider lock as a point of mutual exclusion to prevent other 1326179193Sjb * DTrace consumers from disabling this probe. 1327179193Sjb */ 1328247049Sgibbs if ((p = pfind(probe->ftp_pid)) != NULL) { 1329212494Srpaulo#ifdef __FreeBSD__ 1330277915Smarkj if (p->p_flag & P_WEXIT) { 1331277915Smarkj PROC_UNLOCK(p); 1332277915Smarkj p = NULL; 1333277915Smarkj } else { 1334277915Smarkj _PHOLD(p); 1335277915Smarkj PROC_UNLOCK(p); 1336277915Smarkj } 1337212494Srpaulo#endif 1338247049Sgibbs } 1339179193Sjb 1340179193Sjb /* 1341179193Sjb * Disable all the associated tracepoints (for fully enabled probes). 1342179193Sjb */ 1343179193Sjb if (probe->ftp_enabled) { 1344179193Sjb for (i = 0; i < probe->ftp_ntps; i++) { 1345179193Sjb fasttrap_tracepoint_disable(p, probe, i); 1346179193Sjb } 1347179193Sjb } 1348179193Sjb 1349179193Sjb ASSERT(provider->ftp_rcount > 0); 1350179193Sjb provider->ftp_rcount--; 1351179193Sjb 1352179193Sjb if (p != NULL) { 1353179193Sjb /* 1354179193Sjb * Even though we may not be able to remove it entirely, we 1355179193Sjb * mark this retired provider to get a chance to remove some 1356179193Sjb * of the associated probes. 1357179193Sjb */ 1358179193Sjb if (provider->ftp_retired && !provider->ftp_marked) 1359179193Sjb whack = provider->ftp_marked = 1; 1360179193Sjb mutex_exit(&provider->ftp_mtx); 1361179193Sjb } else { 1362179193Sjb /* 1363179193Sjb * If the process is dead, we're just waiting for the 1364179193Sjb * last probe to be disabled to be able to free it. 1365179193Sjb */ 1366179193Sjb if (provider->ftp_rcount == 0 && !provider->ftp_marked) 1367179193Sjb whack = provider->ftp_marked = 1; 1368179193Sjb mutex_exit(&provider->ftp_mtx); 1369179193Sjb } 1370179193Sjb 1371179193Sjb if (whack) 1372179193Sjb fasttrap_pid_cleanup(); 1373179193Sjb 1374212494Srpaulo#ifdef __FreeBSD__ 1375247049Sgibbs if (p != NULL) 1376247049Sgibbs PRELE(p); 1377212494Srpaulo#endif 1378179193Sjb if (!probe->ftp_enabled) 1379179193Sjb return; 1380179193Sjb 1381179193Sjb probe->ftp_enabled = 0; 1382179193Sjb 1383277300Ssmh#ifdef illumos 1384179193Sjb ASSERT(MUTEX_HELD(&cpu_lock)); 1385211738Srpaulo#endif 1386179193Sjb fasttrap_disable_callbacks(); 1387179193Sjb} 1388179193Sjb 1389179193Sjb/*ARGSUSED*/ 1390179193Sjbstatic void 1391179193Sjbfasttrap_pid_getargdesc(void *arg, dtrace_id_t id, void *parg, 1392179193Sjb dtrace_argdesc_t *desc) 1393179193Sjb{ 1394179193Sjb fasttrap_probe_t *probe = parg; 1395179193Sjb char *str; 1396179193Sjb int i, ndx; 1397179193Sjb 1398179193Sjb desc->dtargd_native[0] = '\0'; 1399179193Sjb desc->dtargd_xlate[0] = '\0'; 1400179193Sjb 1401179193Sjb if (probe->ftp_prov->ftp_retired != 0 || 1402179193Sjb desc->dtargd_ndx >= probe->ftp_nargs) { 1403179193Sjb desc->dtargd_ndx = DTRACE_ARGNONE; 1404179193Sjb return; 1405179193Sjb } 1406179193Sjb 1407179193Sjb ndx = (probe->ftp_argmap != NULL) ? 1408179193Sjb probe->ftp_argmap[desc->dtargd_ndx] : desc->dtargd_ndx; 1409179193Sjb 1410179193Sjb str = probe->ftp_ntypes; 1411179193Sjb for (i = 0; i < ndx; i++) { 1412179193Sjb str += strlen(str) + 1; 1413179193Sjb } 1414179193Sjb 1415179193Sjb ASSERT(strlen(str + 1) < sizeof (desc->dtargd_native)); 1416179193Sjb (void) strcpy(desc->dtargd_native, str); 1417179193Sjb 1418179193Sjb if (probe->ftp_xtypes == NULL) 1419179193Sjb return; 1420179193Sjb 1421179193Sjb str = probe->ftp_xtypes; 1422179193Sjb for (i = 0; i < desc->dtargd_ndx; i++) { 1423179193Sjb str += strlen(str) + 1; 1424179193Sjb } 1425179193Sjb 1426179193Sjb ASSERT(strlen(str + 1) < sizeof (desc->dtargd_xlate)); 1427179193Sjb (void) strcpy(desc->dtargd_xlate, str); 1428179193Sjb} 1429179193Sjb 1430179193Sjb/*ARGSUSED*/ 1431179193Sjbstatic void 1432179193Sjbfasttrap_pid_destroy(void *arg, dtrace_id_t id, void *parg) 1433179193Sjb{ 1434179193Sjb fasttrap_probe_t *probe = parg; 1435179193Sjb int i; 1436179193Sjb size_t size; 1437179193Sjb 1438179193Sjb ASSERT(probe != NULL); 1439179193Sjb ASSERT(!probe->ftp_enabled); 1440179193Sjb ASSERT(fasttrap_total >= probe->ftp_ntps); 1441179193Sjb 1442179193Sjb atomic_add_32(&fasttrap_total, -probe->ftp_ntps); 1443179193Sjb size = offsetof(fasttrap_probe_t, ftp_tps[probe->ftp_ntps]); 1444179193Sjb 1445179193Sjb if (probe->ftp_gen + 1 >= fasttrap_mod_gen) 1446179193Sjb fasttrap_mod_barrier(probe->ftp_gen); 1447179193Sjb 1448179193Sjb for (i = 0; i < probe->ftp_ntps; i++) { 1449179193Sjb kmem_free(probe->ftp_tps[i].fit_tp, 1450179193Sjb sizeof (fasttrap_tracepoint_t)); 1451179193Sjb } 1452179193Sjb 1453179193Sjb kmem_free(probe, size); 1454179193Sjb} 1455179193Sjb 1456179193Sjb 1457179193Sjbstatic const dtrace_pattr_t pid_attr = { 1458179193Sjb{ DTRACE_STABILITY_EVOLVING, DTRACE_STABILITY_EVOLVING, DTRACE_CLASS_ISA }, 1459179193Sjb{ DTRACE_STABILITY_PRIVATE, DTRACE_STABILITY_PRIVATE, DTRACE_CLASS_UNKNOWN }, 1460179193Sjb{ DTRACE_STABILITY_PRIVATE, DTRACE_STABILITY_PRIVATE, DTRACE_CLASS_UNKNOWN }, 1461179193Sjb{ DTRACE_STABILITY_EVOLVING, DTRACE_STABILITY_EVOLVING, DTRACE_CLASS_ISA }, 1462179193Sjb{ DTRACE_STABILITY_PRIVATE, DTRACE_STABILITY_PRIVATE, DTRACE_CLASS_UNKNOWN }, 1463179193Sjb}; 1464179193Sjb 1465179193Sjbstatic dtrace_pops_t pid_pops = { 1466179193Sjb fasttrap_pid_provide, 1467179193Sjb NULL, 1468179193Sjb fasttrap_pid_enable, 1469179193Sjb fasttrap_pid_disable, 1470179193Sjb NULL, 1471179193Sjb NULL, 1472179193Sjb fasttrap_pid_getargdesc, 1473179193Sjb fasttrap_pid_getarg, 1474179193Sjb NULL, 1475179193Sjb fasttrap_pid_destroy 1476179193Sjb}; 1477179193Sjb 1478179193Sjbstatic dtrace_pops_t usdt_pops = { 1479179193Sjb fasttrap_pid_provide, 1480179193Sjb NULL, 1481179193Sjb fasttrap_pid_enable, 1482179193Sjb fasttrap_pid_disable, 1483179193Sjb NULL, 1484179193Sjb NULL, 1485179193Sjb fasttrap_pid_getargdesc, 1486179193Sjb fasttrap_usdt_getarg, 1487179193Sjb NULL, 1488179193Sjb fasttrap_pid_destroy 1489179193Sjb}; 1490179193Sjb 1491179193Sjbstatic fasttrap_proc_t * 1492179193Sjbfasttrap_proc_lookup(pid_t pid) 1493179193Sjb{ 1494179193Sjb fasttrap_bucket_t *bucket; 1495179193Sjb fasttrap_proc_t *fprc, *new_fprc; 1496179193Sjb 1497211738Srpaulo 1498179193Sjb bucket = &fasttrap_procs.fth_table[FASTTRAP_PROCS_INDEX(pid)]; 1499179193Sjb mutex_enter(&bucket->ftb_mtx); 1500179193Sjb 1501179193Sjb for (fprc = bucket->ftb_data; fprc != NULL; fprc = fprc->ftpc_next) { 1502179193Sjb if (fprc->ftpc_pid == pid && fprc->ftpc_acount != 0) { 1503179193Sjb mutex_enter(&fprc->ftpc_mtx); 1504179193Sjb mutex_exit(&bucket->ftb_mtx); 1505179193Sjb fprc->ftpc_rcount++; 1506270247Sdelphij atomic_inc_64(&fprc->ftpc_acount); 1507179198Sjb ASSERT(fprc->ftpc_acount <= fprc->ftpc_rcount); 1508179193Sjb mutex_exit(&fprc->ftpc_mtx); 1509179193Sjb 1510179193Sjb return (fprc); 1511179193Sjb } 1512179193Sjb } 1513179193Sjb 1514179193Sjb /* 1515179193Sjb * Drop the bucket lock so we don't try to perform a sleeping 1516179193Sjb * allocation under it. 1517179193Sjb */ 1518179193Sjb mutex_exit(&bucket->ftb_mtx); 1519179193Sjb 1520179193Sjb new_fprc = kmem_zalloc(sizeof (fasttrap_proc_t), KM_SLEEP); 1521179193Sjb new_fprc->ftpc_pid = pid; 1522179193Sjb new_fprc->ftpc_rcount = 1; 1523179193Sjb new_fprc->ftpc_acount = 1; 1524277300Ssmh#ifndef illumos 1525211738Srpaulo mutex_init(&new_fprc->ftpc_mtx, "fasttrap proc mtx", MUTEX_DEFAULT, 1526211738Srpaulo NULL); 1527211738Srpaulo#endif 1528179193Sjb 1529179193Sjb mutex_enter(&bucket->ftb_mtx); 1530179193Sjb 1531179193Sjb /* 1532179193Sjb * Take another lap through the list to make sure a proc hasn't 1533179193Sjb * been created for this pid while we weren't under the bucket lock. 1534179193Sjb */ 1535179193Sjb for (fprc = bucket->ftb_data; fprc != NULL; fprc = fprc->ftpc_next) { 1536179193Sjb if (fprc->ftpc_pid == pid && fprc->ftpc_acount != 0) { 1537179193Sjb mutex_enter(&fprc->ftpc_mtx); 1538179193Sjb mutex_exit(&bucket->ftb_mtx); 1539179193Sjb fprc->ftpc_rcount++; 1540270247Sdelphij atomic_inc_64(&fprc->ftpc_acount); 1541179198Sjb ASSERT(fprc->ftpc_acount <= fprc->ftpc_rcount); 1542179193Sjb mutex_exit(&fprc->ftpc_mtx); 1543179193Sjb 1544179193Sjb kmem_free(new_fprc, sizeof (fasttrap_proc_t)); 1545179193Sjb 1546179193Sjb return (fprc); 1547179193Sjb } 1548179193Sjb } 1549179193Sjb 1550179193Sjb new_fprc->ftpc_next = bucket->ftb_data; 1551179193Sjb bucket->ftb_data = new_fprc; 1552179193Sjb 1553179193Sjb mutex_exit(&bucket->ftb_mtx); 1554179193Sjb 1555179193Sjb return (new_fprc); 1556179193Sjb} 1557179193Sjb 1558179193Sjbstatic void 1559179193Sjbfasttrap_proc_release(fasttrap_proc_t *proc) 1560179193Sjb{ 1561179193Sjb fasttrap_bucket_t *bucket; 1562179193Sjb fasttrap_proc_t *fprc, **fprcp; 1563179193Sjb pid_t pid = proc->ftpc_pid; 1564277300Ssmh#ifndef illumos 1565264434Smarkj fasttrap_scrblock_t *scrblk, *scrblktmp; 1566264434Smarkj fasttrap_scrspace_t *scrspc, *scrspctmp; 1567264434Smarkj struct proc *p; 1568264434Smarkj struct thread *td; 1569264434Smarkj#endif 1570179193Sjb 1571179193Sjb mutex_enter(&proc->ftpc_mtx); 1572179193Sjb 1573179193Sjb ASSERT(proc->ftpc_rcount != 0); 1574179198Sjb ASSERT(proc->ftpc_acount <= proc->ftpc_rcount); 1575179193Sjb 1576179193Sjb if (--proc->ftpc_rcount != 0) { 1577179193Sjb mutex_exit(&proc->ftpc_mtx); 1578179193Sjb return; 1579179193Sjb } 1580179193Sjb 1581277300Ssmh#ifndef illumos 1582264434Smarkj /* 1583264434Smarkj * Free all structures used to manage per-thread scratch space. 1584264434Smarkj */ 1585264434Smarkj LIST_FOREACH_SAFE(scrblk, &proc->ftpc_scrblks, ftsb_next, 1586264434Smarkj scrblktmp) { 1587264434Smarkj LIST_REMOVE(scrblk, ftsb_next); 1588264434Smarkj free(scrblk, M_SOLARIS); 1589264434Smarkj } 1590264434Smarkj LIST_FOREACH_SAFE(scrspc, &proc->ftpc_fscr, ftss_next, scrspctmp) { 1591264434Smarkj LIST_REMOVE(scrspc, ftss_next); 1592264434Smarkj free(scrspc, M_SOLARIS); 1593264434Smarkj } 1594264434Smarkj LIST_FOREACH_SAFE(scrspc, &proc->ftpc_ascr, ftss_next, scrspctmp) { 1595264434Smarkj LIST_REMOVE(scrspc, ftss_next); 1596264434Smarkj free(scrspc, M_SOLARIS); 1597264434Smarkj } 1598264434Smarkj 1599264434Smarkj if ((p = pfind(pid)) != NULL) { 1600264434Smarkj FOREACH_THREAD_IN_PROC(p, td) 1601264434Smarkj td->t_dtrace_sscr = NULL; 1602264434Smarkj PROC_UNLOCK(p); 1603264434Smarkj } 1604264434Smarkj#endif 1605264434Smarkj 1606179193Sjb mutex_exit(&proc->ftpc_mtx); 1607179193Sjb 1608179193Sjb /* 1609179193Sjb * There should definitely be no live providers associated with this 1610179193Sjb * process at this point. 1611179193Sjb */ 1612179193Sjb ASSERT(proc->ftpc_acount == 0); 1613179193Sjb 1614179193Sjb bucket = &fasttrap_procs.fth_table[FASTTRAP_PROCS_INDEX(pid)]; 1615179193Sjb mutex_enter(&bucket->ftb_mtx); 1616179193Sjb 1617179193Sjb fprcp = (fasttrap_proc_t **)&bucket->ftb_data; 1618179193Sjb while ((fprc = *fprcp) != NULL) { 1619179193Sjb if (fprc == proc) 1620179193Sjb break; 1621179193Sjb 1622179193Sjb fprcp = &fprc->ftpc_next; 1623179193Sjb } 1624179193Sjb 1625179193Sjb /* 1626179193Sjb * Something strange has happened if we can't find the proc. 1627179193Sjb */ 1628179193Sjb ASSERT(fprc != NULL); 1629179193Sjb 1630179193Sjb *fprcp = fprc->ftpc_next; 1631179193Sjb 1632179193Sjb mutex_exit(&bucket->ftb_mtx); 1633179193Sjb 1634179193Sjb kmem_free(fprc, sizeof (fasttrap_proc_t)); 1635179193Sjb} 1636179193Sjb 1637179193Sjb/* 1638179193Sjb * Lookup a fasttrap-managed provider based on its name and associated pid. 1639179193Sjb * If the pattr argument is non-NULL, this function instantiates the provider 1640179193Sjb * if it doesn't exist otherwise it returns NULL. The provider is returned 1641179193Sjb * with its lock held. 1642179193Sjb */ 1643179193Sjbstatic fasttrap_provider_t * 1644179193Sjbfasttrap_provider_lookup(pid_t pid, const char *name, 1645179193Sjb const dtrace_pattr_t *pattr) 1646179193Sjb{ 1647179193Sjb fasttrap_provider_t *fp, *new_fp = NULL; 1648179193Sjb fasttrap_bucket_t *bucket; 1649179193Sjb char provname[DTRACE_PROVNAMELEN]; 1650179193Sjb proc_t *p; 1651179193Sjb cred_t *cred; 1652179193Sjb 1653179193Sjb ASSERT(strlen(name) < sizeof (fp->ftp_name)); 1654179193Sjb ASSERT(pattr != NULL); 1655179193Sjb 1656179193Sjb bucket = &fasttrap_provs.fth_table[FASTTRAP_PROVS_INDEX(pid, name)]; 1657179193Sjb mutex_enter(&bucket->ftb_mtx); 1658179193Sjb 1659179193Sjb /* 1660179193Sjb * Take a lap through the list and return the match if we find it. 1661179193Sjb */ 1662179193Sjb for (fp = bucket->ftb_data; fp != NULL; fp = fp->ftp_next) { 1663179193Sjb if (fp->ftp_pid == pid && strcmp(fp->ftp_name, name) == 0 && 1664179193Sjb !fp->ftp_retired) { 1665179193Sjb mutex_enter(&fp->ftp_mtx); 1666179193Sjb mutex_exit(&bucket->ftb_mtx); 1667179193Sjb return (fp); 1668179193Sjb } 1669179193Sjb } 1670179193Sjb 1671179193Sjb /* 1672179193Sjb * Drop the bucket lock so we don't try to perform a sleeping 1673179193Sjb * allocation under it. 1674179193Sjb */ 1675179193Sjb mutex_exit(&bucket->ftb_mtx); 1676179193Sjb 1677179193Sjb /* 1678179193Sjb * Make sure the process exists, isn't a child created as the result 1679179193Sjb * of a vfork(2), and isn't a zombie (but may be in fork). 1680179193Sjb */ 1681211738Srpaulo if ((p = pfind(pid)) == NULL) 1682179193Sjb return (NULL); 1683179193Sjb 1684179193Sjb /* 1685179193Sjb * Increment p_dtrace_probes so that the process knows to inform us 1686179193Sjb * when it exits or execs. fasttrap_provider_free() decrements this 1687179193Sjb * when we're done with this provider. 1688179193Sjb */ 1689179193Sjb p->p_dtrace_probes++; 1690179193Sjb 1691179193Sjb /* 1692179193Sjb * Grab the credentials for this process so we have 1693179193Sjb * something to pass to dtrace_register(). 1694179193Sjb */ 1695211738Srpaulo PROC_LOCK_ASSERT(p, MA_OWNED); 1696211738Srpaulo crhold(p->p_ucred); 1697211738Srpaulo cred = p->p_ucred; 1698211738Srpaulo PROC_UNLOCK(p); 1699179193Sjb 1700179193Sjb new_fp = kmem_zalloc(sizeof (fasttrap_provider_t), KM_SLEEP); 1701179193Sjb new_fp->ftp_pid = pid; 1702179193Sjb new_fp->ftp_proc = fasttrap_proc_lookup(pid); 1703277300Ssmh#ifndef illumos 1704211738Srpaulo mutex_init(&new_fp->ftp_mtx, "provider mtx", MUTEX_DEFAULT, NULL); 1705211738Srpaulo mutex_init(&new_fp->ftp_cmtx, "lock on creating", MUTEX_DEFAULT, NULL); 1706211738Srpaulo#endif 1707179193Sjb 1708179193Sjb ASSERT(new_fp->ftp_proc != NULL); 1709179193Sjb 1710179193Sjb mutex_enter(&bucket->ftb_mtx); 1711179193Sjb 1712179193Sjb /* 1713179193Sjb * Take another lap through the list to make sure a provider hasn't 1714179193Sjb * been created for this pid while we weren't under the bucket lock. 1715179193Sjb */ 1716179193Sjb for (fp = bucket->ftb_data; fp != NULL; fp = fp->ftp_next) { 1717179193Sjb if (fp->ftp_pid == pid && strcmp(fp->ftp_name, name) == 0 && 1718179193Sjb !fp->ftp_retired) { 1719179193Sjb mutex_enter(&fp->ftp_mtx); 1720179193Sjb mutex_exit(&bucket->ftb_mtx); 1721179193Sjb fasttrap_provider_free(new_fp); 1722179193Sjb crfree(cred); 1723179193Sjb return (fp); 1724179193Sjb } 1725179193Sjb } 1726179193Sjb 1727179193Sjb (void) strcpy(new_fp->ftp_name, name); 1728179193Sjb 1729179193Sjb /* 1730179193Sjb * Fail and return NULL if either the provider name is too long 1731179193Sjb * or we fail to register this new provider with the DTrace 1732179193Sjb * framework. Note that this is the only place we ever construct 1733179193Sjb * the full provider name -- we keep it in pieces in the provider 1734179193Sjb * structure. 1735179193Sjb */ 1736179193Sjb if (snprintf(provname, sizeof (provname), "%s%u", name, (uint_t)pid) >= 1737179193Sjb sizeof (provname) || 1738179193Sjb dtrace_register(provname, pattr, 1739179193Sjb DTRACE_PRIV_PROC | DTRACE_PRIV_OWNER | DTRACE_PRIV_ZONEOWNER, cred, 1740179193Sjb pattr == &pid_attr ? &pid_pops : &usdt_pops, new_fp, 1741179193Sjb &new_fp->ftp_provid) != 0) { 1742179193Sjb mutex_exit(&bucket->ftb_mtx); 1743179193Sjb fasttrap_provider_free(new_fp); 1744179193Sjb crfree(cred); 1745179193Sjb return (NULL); 1746179193Sjb } 1747179193Sjb 1748179193Sjb new_fp->ftp_next = bucket->ftb_data; 1749179193Sjb bucket->ftb_data = new_fp; 1750179193Sjb 1751179193Sjb mutex_enter(&new_fp->ftp_mtx); 1752179193Sjb mutex_exit(&bucket->ftb_mtx); 1753179193Sjb 1754179193Sjb crfree(cred); 1755179193Sjb return (new_fp); 1756179193Sjb} 1757179193Sjb 1758179193Sjbstatic void 1759179193Sjbfasttrap_provider_free(fasttrap_provider_t *provider) 1760179193Sjb{ 1761179193Sjb pid_t pid = provider->ftp_pid; 1762179193Sjb proc_t *p; 1763179193Sjb 1764179193Sjb /* 1765179193Sjb * There need to be no associated enabled probes, no consumers 1766179193Sjb * creating probes, and no meta providers referencing this provider. 1767179193Sjb */ 1768179193Sjb ASSERT(provider->ftp_rcount == 0); 1769179193Sjb ASSERT(provider->ftp_ccount == 0); 1770179193Sjb ASSERT(provider->ftp_mcount == 0); 1771179193Sjb 1772179198Sjb /* 1773179198Sjb * If this provider hasn't been retired, we need to explicitly drop the 1774179198Sjb * count of active providers on the associated process structure. 1775179198Sjb */ 1776179198Sjb if (!provider->ftp_retired) { 1777270247Sdelphij atomic_dec_64(&provider->ftp_proc->ftpc_acount); 1778179198Sjb ASSERT(provider->ftp_proc->ftpc_acount < 1779179198Sjb provider->ftp_proc->ftpc_rcount); 1780179198Sjb } 1781179198Sjb 1782179193Sjb fasttrap_proc_release(provider->ftp_proc); 1783179193Sjb 1784277300Ssmh#ifndef illumos 1785211738Srpaulo mutex_destroy(&provider->ftp_mtx); 1786211738Srpaulo mutex_destroy(&provider->ftp_cmtx); 1787211738Srpaulo#endif 1788179193Sjb kmem_free(provider, sizeof (fasttrap_provider_t)); 1789179193Sjb 1790179193Sjb /* 1791179193Sjb * Decrement p_dtrace_probes on the process whose provider we're 1792179193Sjb * freeing. We don't have to worry about clobbering somone else's 1793179193Sjb * modifications to it because we have locked the bucket that 1794179193Sjb * corresponds to this process's hash chain in the provider hash 1795179193Sjb * table. Don't sweat it if we can't find the process. 1796179193Sjb */ 1797211738Srpaulo if ((p = pfind(pid)) == NULL) { 1798179193Sjb return; 1799179193Sjb } 1800179193Sjb 1801179193Sjb p->p_dtrace_probes--; 1802277300Ssmh#ifndef illumos 1803211738Srpaulo PROC_UNLOCK(p); 1804211738Srpaulo#endif 1805179193Sjb} 1806179193Sjb 1807179193Sjbstatic void 1808179193Sjbfasttrap_provider_retire(pid_t pid, const char *name, int mprov) 1809179193Sjb{ 1810179193Sjb fasttrap_provider_t *fp; 1811179193Sjb fasttrap_bucket_t *bucket; 1812179193Sjb dtrace_provider_id_t provid; 1813179193Sjb 1814179193Sjb ASSERT(strlen(name) < sizeof (fp->ftp_name)); 1815179193Sjb 1816179193Sjb bucket = &fasttrap_provs.fth_table[FASTTRAP_PROVS_INDEX(pid, name)]; 1817179193Sjb mutex_enter(&bucket->ftb_mtx); 1818179193Sjb 1819179193Sjb for (fp = bucket->ftb_data; fp != NULL; fp = fp->ftp_next) { 1820179193Sjb if (fp->ftp_pid == pid && strcmp(fp->ftp_name, name) == 0 && 1821179193Sjb !fp->ftp_retired) 1822179193Sjb break; 1823179193Sjb } 1824179193Sjb 1825179193Sjb if (fp == NULL) { 1826179193Sjb mutex_exit(&bucket->ftb_mtx); 1827179193Sjb return; 1828179193Sjb } 1829179193Sjb 1830179193Sjb mutex_enter(&fp->ftp_mtx); 1831179193Sjb ASSERT(!mprov || fp->ftp_mcount > 0); 1832179193Sjb if (mprov && --fp->ftp_mcount != 0) { 1833179193Sjb mutex_exit(&fp->ftp_mtx); 1834179193Sjb mutex_exit(&bucket->ftb_mtx); 1835179193Sjb return; 1836179193Sjb } 1837179193Sjb 1838179193Sjb /* 1839179193Sjb * Mark the provider to be removed in our post-processing step, mark it 1840179193Sjb * retired, and drop the active count on its proc. Marking it indicates 1841179193Sjb * that we should try to remove it; setting the retired flag indicates 1842179193Sjb * that we're done with this provider; dropping the active the proc 1843179193Sjb * releases our hold, and when this reaches zero (as it will during 1844179193Sjb * exit or exec) the proc and associated providers become defunct. 1845179193Sjb * 1846179193Sjb * We obviously need to take the bucket lock before the provider lock 1847179193Sjb * to perform the lookup, but we need to drop the provider lock 1848179193Sjb * before calling into the DTrace framework since we acquire the 1849179193Sjb * provider lock in callbacks invoked from the DTrace framework. The 1850179193Sjb * bucket lock therefore protects the integrity of the provider hash 1851179193Sjb * table. 1852179193Sjb */ 1853270247Sdelphij atomic_dec_64(&fp->ftp_proc->ftpc_acount); 1854179198Sjb ASSERT(fp->ftp_proc->ftpc_acount < fp->ftp_proc->ftpc_rcount); 1855179198Sjb 1856179193Sjb fp->ftp_retired = 1; 1857179193Sjb fp->ftp_marked = 1; 1858179193Sjb provid = fp->ftp_provid; 1859179193Sjb mutex_exit(&fp->ftp_mtx); 1860179193Sjb 1861179193Sjb /* 1862179193Sjb * We don't have to worry about invalidating the same provider twice 1863179193Sjb * since fasttrap_provider_lookup() will ignore provider that have 1864179193Sjb * been marked as retired. 1865179193Sjb */ 1866179193Sjb dtrace_invalidate(provid); 1867179193Sjb 1868179193Sjb mutex_exit(&bucket->ftb_mtx); 1869179193Sjb 1870179193Sjb fasttrap_pid_cleanup(); 1871179193Sjb} 1872179193Sjb 1873179193Sjbstatic int 1874179193Sjbfasttrap_uint32_cmp(const void *ap, const void *bp) 1875179193Sjb{ 1876179193Sjb return (*(const uint32_t *)ap - *(const uint32_t *)bp); 1877179193Sjb} 1878179193Sjb 1879179193Sjbstatic int 1880179193Sjbfasttrap_uint64_cmp(const void *ap, const void *bp) 1881179193Sjb{ 1882179193Sjb return (*(const uint64_t *)ap - *(const uint64_t *)bp); 1883179193Sjb} 1884179193Sjb 1885179193Sjbstatic int 1886179193Sjbfasttrap_add_probe(fasttrap_probe_spec_t *pdata) 1887179193Sjb{ 1888179193Sjb fasttrap_provider_t *provider; 1889179193Sjb fasttrap_probe_t *pp; 1890179193Sjb fasttrap_tracepoint_t *tp; 1891179193Sjb char *name; 1892211738Srpaulo int i, aframes = 0, whack; 1893179193Sjb 1894179193Sjb /* 1895179193Sjb * There needs to be at least one desired trace point. 1896179193Sjb */ 1897179193Sjb if (pdata->ftps_noffs == 0) 1898179193Sjb return (EINVAL); 1899179193Sjb 1900179193Sjb switch (pdata->ftps_type) { 1901179193Sjb case DTFTP_ENTRY: 1902179193Sjb name = "entry"; 1903179193Sjb aframes = FASTTRAP_ENTRY_AFRAMES; 1904179193Sjb break; 1905179193Sjb case DTFTP_RETURN: 1906179193Sjb name = "return"; 1907179193Sjb aframes = FASTTRAP_RETURN_AFRAMES; 1908179193Sjb break; 1909179193Sjb case DTFTP_OFFSETS: 1910179193Sjb name = NULL; 1911179193Sjb break; 1912179193Sjb default: 1913179193Sjb return (EINVAL); 1914179193Sjb } 1915179193Sjb 1916179193Sjb if ((provider = fasttrap_provider_lookup(pdata->ftps_pid, 1917179193Sjb FASTTRAP_PID_NAME, &pid_attr)) == NULL) 1918179193Sjb return (ESRCH); 1919179193Sjb 1920179193Sjb /* 1921179193Sjb * Increment this reference count to indicate that a consumer is 1922179193Sjb * actively adding a new probe associated with this provider. This 1923179193Sjb * prevents the provider from being deleted -- we'll need to check 1924179193Sjb * for pending deletions when we drop this reference count. 1925179193Sjb */ 1926179193Sjb provider->ftp_ccount++; 1927179193Sjb mutex_exit(&provider->ftp_mtx); 1928179193Sjb 1929179193Sjb /* 1930179193Sjb * Grab the creation lock to ensure consistency between calls to 1931179193Sjb * dtrace_probe_lookup() and dtrace_probe_create() in the face of 1932179193Sjb * other threads creating probes. We must drop the provider lock 1933179193Sjb * before taking this lock to avoid a three-way deadlock with the 1934179193Sjb * DTrace framework. 1935179193Sjb */ 1936179193Sjb mutex_enter(&provider->ftp_cmtx); 1937179193Sjb 1938179193Sjb if (name == NULL) { 1939179193Sjb for (i = 0; i < pdata->ftps_noffs; i++) { 1940179193Sjb char name_str[17]; 1941179193Sjb 1942179193Sjb (void) sprintf(name_str, "%llx", 1943179193Sjb (unsigned long long)pdata->ftps_offs[i]); 1944179193Sjb 1945179193Sjb if (dtrace_probe_lookup(provider->ftp_provid, 1946179193Sjb pdata->ftps_mod, pdata->ftps_func, name_str) != 0) 1947179193Sjb continue; 1948179193Sjb 1949270247Sdelphij atomic_inc_32(&fasttrap_total); 1950179193Sjb 1951179193Sjb if (fasttrap_total > fasttrap_max) { 1952270247Sdelphij atomic_dec_32(&fasttrap_total); 1953179193Sjb goto no_mem; 1954179193Sjb } 1955179193Sjb 1956179193Sjb pp = kmem_zalloc(sizeof (fasttrap_probe_t), KM_SLEEP); 1957179193Sjb 1958179193Sjb pp->ftp_prov = provider; 1959179193Sjb pp->ftp_faddr = pdata->ftps_pc; 1960179193Sjb pp->ftp_fsize = pdata->ftps_size; 1961179193Sjb pp->ftp_pid = pdata->ftps_pid; 1962179193Sjb pp->ftp_ntps = 1; 1963179193Sjb 1964179193Sjb tp = kmem_zalloc(sizeof (fasttrap_tracepoint_t), 1965179193Sjb KM_SLEEP); 1966179193Sjb 1967179193Sjb tp->ftt_proc = provider->ftp_proc; 1968179193Sjb tp->ftt_pc = pdata->ftps_offs[i] + pdata->ftps_pc; 1969179193Sjb tp->ftt_pid = pdata->ftps_pid; 1970179193Sjb 1971179193Sjb pp->ftp_tps[0].fit_tp = tp; 1972179193Sjb pp->ftp_tps[0].fit_id.fti_probe = pp; 1973179193Sjb pp->ftp_tps[0].fit_id.fti_ptype = pdata->ftps_type; 1974179193Sjb 1975179193Sjb pp->ftp_id = dtrace_probe_create(provider->ftp_provid, 1976179193Sjb pdata->ftps_mod, pdata->ftps_func, name_str, 1977179193Sjb FASTTRAP_OFFSET_AFRAMES, pp); 1978179193Sjb } 1979179193Sjb 1980179193Sjb } else if (dtrace_probe_lookup(provider->ftp_provid, pdata->ftps_mod, 1981179193Sjb pdata->ftps_func, name) == 0) { 1982179193Sjb atomic_add_32(&fasttrap_total, pdata->ftps_noffs); 1983179193Sjb 1984179193Sjb if (fasttrap_total > fasttrap_max) { 1985179193Sjb atomic_add_32(&fasttrap_total, -pdata->ftps_noffs); 1986179193Sjb goto no_mem; 1987179193Sjb } 1988179193Sjb 1989179193Sjb /* 1990179193Sjb * Make sure all tracepoint program counter values are unique. 1991179193Sjb * We later assume that each probe has exactly one tracepoint 1992179193Sjb * for a given pc. 1993179193Sjb */ 1994179193Sjb qsort(pdata->ftps_offs, pdata->ftps_noffs, 1995179193Sjb sizeof (uint64_t), fasttrap_uint64_cmp); 1996179193Sjb for (i = 1; i < pdata->ftps_noffs; i++) { 1997179193Sjb if (pdata->ftps_offs[i] > pdata->ftps_offs[i - 1]) 1998179193Sjb continue; 1999179193Sjb 2000179193Sjb atomic_add_32(&fasttrap_total, -pdata->ftps_noffs); 2001179193Sjb goto no_mem; 2002179193Sjb } 2003179193Sjb 2004179193Sjb ASSERT(pdata->ftps_noffs > 0); 2005179193Sjb pp = kmem_zalloc(offsetof(fasttrap_probe_t, 2006179193Sjb ftp_tps[pdata->ftps_noffs]), KM_SLEEP); 2007179193Sjb 2008179193Sjb pp->ftp_prov = provider; 2009179193Sjb pp->ftp_faddr = pdata->ftps_pc; 2010179193Sjb pp->ftp_fsize = pdata->ftps_size; 2011179193Sjb pp->ftp_pid = pdata->ftps_pid; 2012179193Sjb pp->ftp_ntps = pdata->ftps_noffs; 2013179193Sjb 2014179193Sjb for (i = 0; i < pdata->ftps_noffs; i++) { 2015179193Sjb tp = kmem_zalloc(sizeof (fasttrap_tracepoint_t), 2016179193Sjb KM_SLEEP); 2017179193Sjb 2018179193Sjb tp->ftt_proc = provider->ftp_proc; 2019179193Sjb tp->ftt_pc = pdata->ftps_offs[i] + pdata->ftps_pc; 2020179193Sjb tp->ftt_pid = pdata->ftps_pid; 2021179193Sjb 2022179193Sjb pp->ftp_tps[i].fit_tp = tp; 2023179193Sjb pp->ftp_tps[i].fit_id.fti_probe = pp; 2024179193Sjb pp->ftp_tps[i].fit_id.fti_ptype = pdata->ftps_type; 2025179193Sjb } 2026179193Sjb 2027179193Sjb pp->ftp_id = dtrace_probe_create(provider->ftp_provid, 2028179193Sjb pdata->ftps_mod, pdata->ftps_func, name, aframes, pp); 2029179193Sjb } 2030179193Sjb 2031179193Sjb mutex_exit(&provider->ftp_cmtx); 2032179193Sjb 2033179193Sjb /* 2034179193Sjb * We know that the provider is still valid since we incremented the 2035179193Sjb * creation reference count. If someone tried to clean up this provider 2036179193Sjb * while we were using it (e.g. because the process called exec(2) or 2037179193Sjb * exit(2)), take note of that and try to clean it up now. 2038179193Sjb */ 2039179193Sjb mutex_enter(&provider->ftp_mtx); 2040179193Sjb provider->ftp_ccount--; 2041179193Sjb whack = provider->ftp_retired; 2042179193Sjb mutex_exit(&provider->ftp_mtx); 2043179193Sjb 2044179193Sjb if (whack) 2045179193Sjb fasttrap_pid_cleanup(); 2046179193Sjb 2047179193Sjb return (0); 2048179193Sjb 2049179193Sjbno_mem: 2050179193Sjb /* 2051179193Sjb * If we've exhausted the allowable resources, we'll try to remove 2052179193Sjb * this provider to free some up. This is to cover the case where 2053179193Sjb * the user has accidentally created many more probes than was 2054179193Sjb * intended (e.g. pid123:::). 2055179193Sjb */ 2056179193Sjb mutex_exit(&provider->ftp_cmtx); 2057179193Sjb mutex_enter(&provider->ftp_mtx); 2058179193Sjb provider->ftp_ccount--; 2059179193Sjb provider->ftp_marked = 1; 2060179193Sjb mutex_exit(&provider->ftp_mtx); 2061179193Sjb 2062179193Sjb fasttrap_pid_cleanup(); 2063179193Sjb 2064179193Sjb return (ENOMEM); 2065179193Sjb} 2066179193Sjb 2067179193Sjb/*ARGSUSED*/ 2068179193Sjbstatic void * 2069179193Sjbfasttrap_meta_provide(void *arg, dtrace_helper_provdesc_t *dhpv, pid_t pid) 2070179193Sjb{ 2071179193Sjb fasttrap_provider_t *provider; 2072179193Sjb 2073179193Sjb /* 2074179193Sjb * A 32-bit unsigned integer (like a pid for example) can be 2075179193Sjb * expressed in 10 or fewer decimal digits. Make sure that we'll 2076179193Sjb * have enough space for the provider name. 2077179193Sjb */ 2078179193Sjb if (strlen(dhpv->dthpv_provname) + 10 >= 2079179193Sjb sizeof (provider->ftp_name)) { 2080211738Srpaulo printf("failed to instantiate provider %s: " 2081179193Sjb "name too long to accomodate pid", dhpv->dthpv_provname); 2082179193Sjb return (NULL); 2083179193Sjb } 2084179193Sjb 2085179193Sjb /* 2086179193Sjb * Don't let folks spoof the true pid provider. 2087179193Sjb */ 2088179193Sjb if (strcmp(dhpv->dthpv_provname, FASTTRAP_PID_NAME) == 0) { 2089211738Srpaulo printf("failed to instantiate provider %s: " 2090179193Sjb "%s is an invalid name", dhpv->dthpv_provname, 2091179193Sjb FASTTRAP_PID_NAME); 2092179193Sjb return (NULL); 2093179193Sjb } 2094179193Sjb 2095179193Sjb /* 2096179193Sjb * The highest stability class that fasttrap supports is ISA; cap 2097179193Sjb * the stability of the new provider accordingly. 2098179193Sjb */ 2099179193Sjb if (dhpv->dthpv_pattr.dtpa_provider.dtat_class > DTRACE_CLASS_ISA) 2100179193Sjb dhpv->dthpv_pattr.dtpa_provider.dtat_class = DTRACE_CLASS_ISA; 2101179193Sjb if (dhpv->dthpv_pattr.dtpa_mod.dtat_class > DTRACE_CLASS_ISA) 2102179193Sjb dhpv->dthpv_pattr.dtpa_mod.dtat_class = DTRACE_CLASS_ISA; 2103179193Sjb if (dhpv->dthpv_pattr.dtpa_func.dtat_class > DTRACE_CLASS_ISA) 2104179193Sjb dhpv->dthpv_pattr.dtpa_func.dtat_class = DTRACE_CLASS_ISA; 2105179193Sjb if (dhpv->dthpv_pattr.dtpa_name.dtat_class > DTRACE_CLASS_ISA) 2106179193Sjb dhpv->dthpv_pattr.dtpa_name.dtat_class = DTRACE_CLASS_ISA; 2107179193Sjb if (dhpv->dthpv_pattr.dtpa_args.dtat_class > DTRACE_CLASS_ISA) 2108179193Sjb dhpv->dthpv_pattr.dtpa_args.dtat_class = DTRACE_CLASS_ISA; 2109179193Sjb 2110179193Sjb if ((provider = fasttrap_provider_lookup(pid, dhpv->dthpv_provname, 2111179193Sjb &dhpv->dthpv_pattr)) == NULL) { 2112211738Srpaulo printf("failed to instantiate provider %s for " 2113179193Sjb "process %u", dhpv->dthpv_provname, (uint_t)pid); 2114179193Sjb return (NULL); 2115179193Sjb } 2116179193Sjb 2117179193Sjb /* 2118179193Sjb * Up the meta provider count so this provider isn't removed until 2119179193Sjb * the meta provider has been told to remove it. 2120179193Sjb */ 2121179193Sjb provider->ftp_mcount++; 2122179193Sjb 2123179193Sjb mutex_exit(&provider->ftp_mtx); 2124179193Sjb 2125179193Sjb return (provider); 2126179193Sjb} 2127179193Sjb 2128291963Smarkj/* 2129291963Smarkj * We know a few things about our context here: we know that the probe being 2130291963Smarkj * created doesn't already exist (DTrace won't load DOF at the same address 2131291963Smarkj * twice, even if explicitly told to do so) and we know that we are 2132291963Smarkj * single-threaded with respect to the meta provider machinery. Knowing that 2133291963Smarkj * this is a new probe and that there is no way for us to race with another 2134291963Smarkj * operation on this provider allows us an important optimization: we need not 2135291963Smarkj * lookup a probe before adding it. Saving this lookup is important because 2136291963Smarkj * this code is in the fork path for processes with USDT probes, and lookups 2137291963Smarkj * here are potentially very expensive because of long hash conflicts on 2138291963Smarkj * module, function and name (DTrace doesn't hash on provider name). 2139291963Smarkj */ 2140179193Sjb/*ARGSUSED*/ 2141179193Sjbstatic void 2142179193Sjbfasttrap_meta_create_probe(void *arg, void *parg, 2143179193Sjb dtrace_helper_probedesc_t *dhpb) 2144179193Sjb{ 2145179193Sjb fasttrap_provider_t *provider = parg; 2146179193Sjb fasttrap_probe_t *pp; 2147179193Sjb fasttrap_tracepoint_t *tp; 2148179193Sjb int i, j; 2149179193Sjb uint32_t ntps; 2150179193Sjb 2151179193Sjb /* 2152179193Sjb * Since the meta provider count is non-zero we don't have to worry 2153179193Sjb * about this provider disappearing. 2154179193Sjb */ 2155179193Sjb ASSERT(provider->ftp_mcount > 0); 2156179193Sjb 2157179193Sjb /* 2158179193Sjb * The offsets must be unique. 2159179193Sjb */ 2160179193Sjb qsort(dhpb->dthpb_offs, dhpb->dthpb_noffs, sizeof (uint32_t), 2161179193Sjb fasttrap_uint32_cmp); 2162179193Sjb for (i = 1; i < dhpb->dthpb_noffs; i++) { 2163179193Sjb if (dhpb->dthpb_base + dhpb->dthpb_offs[i] <= 2164179193Sjb dhpb->dthpb_base + dhpb->dthpb_offs[i - 1]) 2165179193Sjb return; 2166179193Sjb } 2167179193Sjb 2168179193Sjb qsort(dhpb->dthpb_enoffs, dhpb->dthpb_nenoffs, sizeof (uint32_t), 2169179193Sjb fasttrap_uint32_cmp); 2170179193Sjb for (i = 1; i < dhpb->dthpb_nenoffs; i++) { 2171179193Sjb if (dhpb->dthpb_base + dhpb->dthpb_enoffs[i] <= 2172179193Sjb dhpb->dthpb_base + dhpb->dthpb_enoffs[i - 1]) 2173179193Sjb return; 2174179193Sjb } 2175179193Sjb 2176179193Sjb ntps = dhpb->dthpb_noffs + dhpb->dthpb_nenoffs; 2177179193Sjb ASSERT(ntps > 0); 2178179193Sjb 2179179193Sjb atomic_add_32(&fasttrap_total, ntps); 2180179193Sjb 2181179193Sjb if (fasttrap_total > fasttrap_max) { 2182179193Sjb atomic_add_32(&fasttrap_total, -ntps); 2183179193Sjb return; 2184179193Sjb } 2185179193Sjb 2186179193Sjb pp = kmem_zalloc(offsetof(fasttrap_probe_t, ftp_tps[ntps]), KM_SLEEP); 2187179193Sjb 2188179193Sjb pp->ftp_prov = provider; 2189179193Sjb pp->ftp_pid = provider->ftp_pid; 2190179193Sjb pp->ftp_ntps = ntps; 2191179193Sjb pp->ftp_nargs = dhpb->dthpb_xargc; 2192179193Sjb pp->ftp_xtypes = dhpb->dthpb_xtypes; 2193179193Sjb pp->ftp_ntypes = dhpb->dthpb_ntypes; 2194179193Sjb 2195179193Sjb /* 2196179193Sjb * First create a tracepoint for each actual point of interest. 2197179193Sjb */ 2198179193Sjb for (i = 0; i < dhpb->dthpb_noffs; i++) { 2199179193Sjb tp = kmem_zalloc(sizeof (fasttrap_tracepoint_t), KM_SLEEP); 2200179193Sjb 2201179193Sjb tp->ftt_proc = provider->ftp_proc; 2202179193Sjb tp->ftt_pc = dhpb->dthpb_base + dhpb->dthpb_offs[i]; 2203179193Sjb tp->ftt_pid = provider->ftp_pid; 2204179193Sjb 2205179193Sjb pp->ftp_tps[i].fit_tp = tp; 2206179193Sjb pp->ftp_tps[i].fit_id.fti_probe = pp; 2207179193Sjb#ifdef __sparc 2208179193Sjb pp->ftp_tps[i].fit_id.fti_ptype = DTFTP_POST_OFFSETS; 2209179193Sjb#else 2210179193Sjb pp->ftp_tps[i].fit_id.fti_ptype = DTFTP_OFFSETS; 2211179193Sjb#endif 2212179193Sjb } 2213179193Sjb 2214179193Sjb /* 2215179193Sjb * Then create a tracepoint for each is-enabled point. 2216179193Sjb */ 2217179193Sjb for (j = 0; i < ntps; i++, j++) { 2218179193Sjb tp = kmem_zalloc(sizeof (fasttrap_tracepoint_t), KM_SLEEP); 2219179193Sjb 2220179193Sjb tp->ftt_proc = provider->ftp_proc; 2221179193Sjb tp->ftt_pc = dhpb->dthpb_base + dhpb->dthpb_enoffs[j]; 2222179193Sjb tp->ftt_pid = provider->ftp_pid; 2223179193Sjb 2224179193Sjb pp->ftp_tps[i].fit_tp = tp; 2225179193Sjb pp->ftp_tps[i].fit_id.fti_probe = pp; 2226179193Sjb pp->ftp_tps[i].fit_id.fti_ptype = DTFTP_IS_ENABLED; 2227179193Sjb } 2228179193Sjb 2229179193Sjb /* 2230179193Sjb * If the arguments are shuffled around we set the argument remapping 2231179193Sjb * table. Later, when the probe fires, we only remap the arguments 2232179193Sjb * if the table is non-NULL. 2233179193Sjb */ 2234179193Sjb for (i = 0; i < dhpb->dthpb_xargc; i++) { 2235179193Sjb if (dhpb->dthpb_args[i] != i) { 2236179193Sjb pp->ftp_argmap = dhpb->dthpb_args; 2237179193Sjb break; 2238179193Sjb } 2239179193Sjb } 2240179193Sjb 2241179193Sjb /* 2242179193Sjb * The probe is fully constructed -- register it with DTrace. 2243179193Sjb */ 2244179193Sjb pp->ftp_id = dtrace_probe_create(provider->ftp_provid, dhpb->dthpb_mod, 2245179193Sjb dhpb->dthpb_func, dhpb->dthpb_name, FASTTRAP_OFFSET_AFRAMES, pp); 2246179193Sjb} 2247179193Sjb 2248179193Sjb/*ARGSUSED*/ 2249179193Sjbstatic void 2250179193Sjbfasttrap_meta_remove(void *arg, dtrace_helper_provdesc_t *dhpv, pid_t pid) 2251179193Sjb{ 2252179193Sjb /* 2253179193Sjb * Clean up the USDT provider. There may be active consumers of the 2254179193Sjb * provider busy adding probes, no damage will actually befall the 2255179193Sjb * provider until that count has dropped to zero. This just puts 2256179193Sjb * the provider on death row. 2257179193Sjb */ 2258179193Sjb fasttrap_provider_retire(pid, dhpv->dthpv_provname, 1); 2259179193Sjb} 2260179193Sjb 2261179193Sjbstatic dtrace_mops_t fasttrap_mops = { 2262179193Sjb fasttrap_meta_create_probe, 2263179193Sjb fasttrap_meta_provide, 2264179193Sjb fasttrap_meta_remove 2265179193Sjb}; 2266179193Sjb 2267179193Sjb/*ARGSUSED*/ 2268179193Sjbstatic int 2269211738Srpaulofasttrap_open(struct cdev *dev __unused, int oflags __unused, 2270211738Srpaulo int devtype __unused, struct thread *td __unused) 2271179193Sjb{ 2272179193Sjb return (0); 2273179193Sjb} 2274179193Sjb 2275179193Sjb/*ARGSUSED*/ 2276179193Sjbstatic int 2277211738Srpaulofasttrap_ioctl(struct cdev *dev, u_long cmd, caddr_t arg, int fflag, 2278211738Srpaulo struct thread *td) 2279179193Sjb{ 2280211738Srpaulo#ifdef notyet 2281211738Srpaulo struct kinfo_proc kp; 2282211738Srpaulo const cred_t *cr = td->td_ucred; 2283211738Srpaulo#endif 2284179193Sjb if (!dtrace_attached()) 2285179193Sjb return (EAGAIN); 2286179193Sjb 2287179193Sjb if (cmd == FASTTRAPIOC_MAKEPROBE) { 2288258291Smarkj fasttrap_probe_spec_t *uprobe = *(fasttrap_probe_spec_t **)arg; 2289179193Sjb fasttrap_probe_spec_t *probe; 2290179193Sjb uint64_t noffs; 2291179193Sjb size_t size; 2292268097Spfg int ret, err; 2293179193Sjb 2294179193Sjb if (copyin(&uprobe->ftps_noffs, &noffs, 2295179193Sjb sizeof (uprobe->ftps_noffs))) 2296179193Sjb return (EFAULT); 2297179193Sjb 2298179193Sjb /* 2299179193Sjb * Probes must have at least one tracepoint. 2300179193Sjb */ 2301179193Sjb if (noffs == 0) 2302179193Sjb return (EINVAL); 2303179193Sjb 2304179193Sjb size = sizeof (fasttrap_probe_spec_t) + 2305179193Sjb sizeof (probe->ftps_offs[0]) * (noffs - 1); 2306179193Sjb 2307179193Sjb if (size > 1024 * 1024) 2308179193Sjb return (ENOMEM); 2309179193Sjb 2310179193Sjb probe = kmem_alloc(size, KM_SLEEP); 2311179193Sjb 2312268230Spfg if (copyin(uprobe, probe, size) != 0 || 2313268230Spfg probe->ftps_noffs != noffs) { 2314179193Sjb kmem_free(probe, size); 2315179193Sjb return (EFAULT); 2316179193Sjb } 2317179193Sjb 2318179193Sjb /* 2319179193Sjb * Verify that the function and module strings contain no 2320179193Sjb * funny characters. 2321179193Sjb */ 2322268097Spfg if (u8_validate(probe->ftps_func, strlen(probe->ftps_func), 2323268097Spfg NULL, U8_VALIDATE_ENTIRE, &err) < 0) { 2324268097Spfg ret = EINVAL; 2325268097Spfg goto err; 2326179193Sjb } 2327179193Sjb 2328268097Spfg if (u8_validate(probe->ftps_mod, strlen(probe->ftps_mod), 2329268097Spfg NULL, U8_VALIDATE_ENTIRE, &err) < 0) { 2330268097Spfg ret = EINVAL; 2331268097Spfg goto err; 2332179193Sjb } 2333179193Sjb 2334211738Srpaulo#ifdef notyet 2335179193Sjb if (!PRIV_POLICY_CHOICE(cr, PRIV_ALL, B_FALSE)) { 2336179193Sjb proc_t *p; 2337179193Sjb pid_t pid = probe->ftps_pid; 2338179193Sjb 2339277300Ssmh#ifdef illumos 2340179193Sjb mutex_enter(&pidlock); 2341211738Srpaulo#endif 2342179193Sjb /* 2343179193Sjb * Report an error if the process doesn't exist 2344179193Sjb * or is actively being birthed. 2345179193Sjb */ 2346270834Smjg sx_slock(&proctree_lock); 2347211738Srpaulo p = pfind(pid); 2348211738Srpaulo if (p) 2349211738Srpaulo fill_kinfo_proc(p, &kp); 2350270834Smjg sx_sunlock(&proctree_lock); 2351211738Srpaulo if (p == NULL || kp.ki_stat == SIDL) { 2352277300Ssmh#ifdef illumos 2353179193Sjb mutex_exit(&pidlock); 2354211738Srpaulo#endif 2355179193Sjb return (ESRCH); 2356179193Sjb } 2357277300Ssmh#ifdef illumos 2358179193Sjb mutex_enter(&p->p_lock); 2359179193Sjb mutex_exit(&pidlock); 2360211738Srpaulo#else 2361211738Srpaulo PROC_LOCK_ASSERT(p, MA_OWNED); 2362211738Srpaulo#endif 2363179193Sjb 2364211738Srpaulo#ifdef notyet 2365179193Sjb if ((ret = priv_proc_cred_perm(cr, p, NULL, 2366179193Sjb VREAD | VWRITE)) != 0) { 2367277300Ssmh#ifdef illumos 2368179193Sjb mutex_exit(&p->p_lock); 2369211738Srpaulo#else 2370211738Srpaulo PROC_UNLOCK(p); 2371211738Srpaulo#endif 2372179193Sjb return (ret); 2373179193Sjb } 2374211738Srpaulo#endif /* notyet */ 2375277300Ssmh#ifdef illumos 2376179193Sjb mutex_exit(&p->p_lock); 2377211738Srpaulo#else 2378211738Srpaulo PROC_UNLOCK(p); 2379211738Srpaulo#endif 2380179193Sjb } 2381211738Srpaulo#endif /* notyet */ 2382179193Sjb 2383179193Sjb ret = fasttrap_add_probe(probe); 2384179193Sjberr: 2385179193Sjb kmem_free(probe, size); 2386179193Sjb 2387179193Sjb return (ret); 2388179193Sjb 2389179193Sjb } else if (cmd == FASTTRAPIOC_GETINSTR) { 2390179193Sjb fasttrap_instr_query_t instr; 2391179193Sjb fasttrap_tracepoint_t *tp; 2392179193Sjb uint_t index; 2393277300Ssmh#ifdef illumos 2394179193Sjb int ret; 2395211738Srpaulo#endif 2396179193Sjb 2397277300Ssmh#ifdef illumos 2398179193Sjb if (copyin((void *)arg, &instr, sizeof (instr)) != 0) 2399179193Sjb return (EFAULT); 2400211738Srpaulo#endif 2401179193Sjb 2402211738Srpaulo#ifdef notyet 2403179193Sjb if (!PRIV_POLICY_CHOICE(cr, PRIV_ALL, B_FALSE)) { 2404179193Sjb proc_t *p; 2405179193Sjb pid_t pid = instr.ftiq_pid; 2406179193Sjb 2407277300Ssmh#ifdef illumos 2408179193Sjb mutex_enter(&pidlock); 2409211738Srpaulo#endif 2410179193Sjb /* 2411179193Sjb * Report an error if the process doesn't exist 2412179193Sjb * or is actively being birthed. 2413179193Sjb */ 2414270834Smjg sx_slock(&proctree_lock); 2415211738Srpaulo p = pfind(pid); 2416211738Srpaulo if (p) 2417211738Srpaulo fill_kinfo_proc(p, &kp); 2418270834Smjg sx_sunlock(&proctree_lock); 2419211738Srpaulo if (p == NULL || kp.ki_stat == SIDL) { 2420277300Ssmh#ifdef illumos 2421179193Sjb mutex_exit(&pidlock); 2422211738Srpaulo#endif 2423179193Sjb return (ESRCH); 2424179193Sjb } 2425277300Ssmh#ifdef illumos 2426179193Sjb mutex_enter(&p->p_lock); 2427179193Sjb mutex_exit(&pidlock); 2428211738Srpaulo#else 2429211738Srpaulo PROC_LOCK_ASSERT(p, MA_OWNED); 2430211738Srpaulo#endif 2431179193Sjb 2432211738Srpaulo#ifdef notyet 2433179193Sjb if ((ret = priv_proc_cred_perm(cr, p, NULL, 2434179193Sjb VREAD)) != 0) { 2435277300Ssmh#ifdef illumos 2436179193Sjb mutex_exit(&p->p_lock); 2437211738Srpaulo#else 2438211738Srpaulo PROC_UNLOCK(p); 2439211738Srpaulo#endif 2440179193Sjb return (ret); 2441179193Sjb } 2442211738Srpaulo#endif /* notyet */ 2443179193Sjb 2444277300Ssmh#ifdef illumos 2445179193Sjb mutex_exit(&p->p_lock); 2446211738Srpaulo#else 2447211738Srpaulo PROC_UNLOCK(p); 2448211738Srpaulo#endif 2449179193Sjb } 2450211738Srpaulo#endif /* notyet */ 2451179193Sjb 2452179193Sjb index = FASTTRAP_TPOINTS_INDEX(instr.ftiq_pid, instr.ftiq_pc); 2453179193Sjb 2454179193Sjb mutex_enter(&fasttrap_tpoints.fth_table[index].ftb_mtx); 2455179193Sjb tp = fasttrap_tpoints.fth_table[index].ftb_data; 2456179193Sjb while (tp != NULL) { 2457179193Sjb if (instr.ftiq_pid == tp->ftt_pid && 2458179193Sjb instr.ftiq_pc == tp->ftt_pc && 2459179193Sjb tp->ftt_proc->ftpc_acount != 0) 2460179193Sjb break; 2461179193Sjb 2462179193Sjb tp = tp->ftt_next; 2463179193Sjb } 2464179193Sjb 2465179193Sjb if (tp == NULL) { 2466179193Sjb mutex_exit(&fasttrap_tpoints.fth_table[index].ftb_mtx); 2467179193Sjb return (ENOENT); 2468179193Sjb } 2469179193Sjb 2470179193Sjb bcopy(&tp->ftt_instr, &instr.ftiq_instr, 2471179193Sjb sizeof (instr.ftiq_instr)); 2472179193Sjb mutex_exit(&fasttrap_tpoints.fth_table[index].ftb_mtx); 2473179193Sjb 2474179193Sjb if (copyout(&instr, (void *)arg, sizeof (instr)) != 0) 2475179193Sjb return (EFAULT); 2476179193Sjb 2477179193Sjb return (0); 2478179193Sjb } 2479179193Sjb 2480179193Sjb return (EINVAL); 2481179193Sjb} 2482179193Sjb 2483179193Sjbstatic int 2484211738Srpaulofasttrap_load(void) 2485179193Sjb{ 2486179193Sjb ulong_t nent; 2487250953Smarkj int i, ret; 2488179193Sjb 2489211738Srpaulo /* Create the /dev/dtrace/fasttrap entry. */ 2490211738Srpaulo fasttrap_cdev = make_dev(&fasttrap_cdevsw, 0, UID_ROOT, GID_WHEEL, 0600, 2491211738Srpaulo "dtrace/fasttrap"); 2492179193Sjb 2493211738Srpaulo mtx_init(&fasttrap_cleanup_mtx, "fasttrap clean", "dtrace", MTX_DEF); 2494211738Srpaulo mutex_init(&fasttrap_count_mtx, "fasttrap count mtx", MUTEX_DEFAULT, 2495211738Srpaulo NULL); 2496179193Sjb 2497277300Ssmh#ifdef illumos 2498179193Sjb fasttrap_max = ddi_getprop(DDI_DEV_T_ANY, devi, DDI_PROP_DONTPASS, 2499179193Sjb "fasttrap-max-probes", FASTTRAP_MAX_DEFAULT); 2500211738Srpaulo#endif 2501179193Sjb fasttrap_total = 0; 2502179193Sjb 2503179193Sjb /* 2504179193Sjb * Conjure up the tracepoints hashtable... 2505179193Sjb */ 2506277300Ssmh#ifdef illumos 2507179193Sjb nent = ddi_getprop(DDI_DEV_T_ANY, devi, DDI_PROP_DONTPASS, 2508179193Sjb "fasttrap-hash-size", FASTTRAP_TPOINTS_DEFAULT_SIZE); 2509211738Srpaulo#else 2510291545Sstas nent = tpoints_hash_size; 2511211738Srpaulo#endif 2512179193Sjb 2513179193Sjb if (nent == 0 || nent > 0x1000000) 2514179193Sjb nent = FASTTRAP_TPOINTS_DEFAULT_SIZE; 2515179193Sjb 2516291545Sstas tpoints_hash_size = nent; 2517291545Sstas 2518275562Sdelphij if (ISP2(nent)) 2519179193Sjb fasttrap_tpoints.fth_nent = nent; 2520179193Sjb else 2521179193Sjb fasttrap_tpoints.fth_nent = 1 << fasttrap_highbit(nent); 2522179193Sjb ASSERT(fasttrap_tpoints.fth_nent > 0); 2523179193Sjb fasttrap_tpoints.fth_mask = fasttrap_tpoints.fth_nent - 1; 2524179193Sjb fasttrap_tpoints.fth_table = kmem_zalloc(fasttrap_tpoints.fth_nent * 2525179193Sjb sizeof (fasttrap_bucket_t), KM_SLEEP); 2526277300Ssmh#ifndef illumos 2527211738Srpaulo for (i = 0; i < fasttrap_tpoints.fth_nent; i++) 2528211738Srpaulo mutex_init(&fasttrap_tpoints.fth_table[i].ftb_mtx, 2529211738Srpaulo "tracepoints bucket mtx", MUTEX_DEFAULT, NULL); 2530211738Srpaulo#endif 2531179193Sjb 2532179193Sjb /* 2533179193Sjb * ... and the providers hash table... 2534179193Sjb */ 2535179193Sjb nent = FASTTRAP_PROVIDERS_DEFAULT_SIZE; 2536275562Sdelphij if (ISP2(nent)) 2537179193Sjb fasttrap_provs.fth_nent = nent; 2538179193Sjb else 2539179193Sjb fasttrap_provs.fth_nent = 1 << fasttrap_highbit(nent); 2540179193Sjb ASSERT(fasttrap_provs.fth_nent > 0); 2541179193Sjb fasttrap_provs.fth_mask = fasttrap_provs.fth_nent - 1; 2542179193Sjb fasttrap_provs.fth_table = kmem_zalloc(fasttrap_provs.fth_nent * 2543179193Sjb sizeof (fasttrap_bucket_t), KM_SLEEP); 2544277300Ssmh#ifndef illumos 2545211738Srpaulo for (i = 0; i < fasttrap_provs.fth_nent; i++) 2546211738Srpaulo mutex_init(&fasttrap_provs.fth_table[i].ftb_mtx, 2547211738Srpaulo "providers bucket mtx", MUTEX_DEFAULT, NULL); 2548211738Srpaulo#endif 2549179193Sjb 2550258311Sasomers ret = kproc_create(fasttrap_pid_cleanup_cb, NULL, 2551258311Sasomers &fasttrap_cleanup_proc, 0, 0, "ftcleanup"); 2552258311Sasomers if (ret != 0) { 2553258311Sasomers destroy_dev(fasttrap_cdev); 2554277300Ssmh#ifndef illumos 2555258311Sasomers for (i = 0; i < fasttrap_provs.fth_nent; i++) 2556258311Sasomers mutex_destroy(&fasttrap_provs.fth_table[i].ftb_mtx); 2557258311Sasomers for (i = 0; i < fasttrap_tpoints.fth_nent; i++) 2558258311Sasomers mutex_destroy(&fasttrap_tpoints.fth_table[i].ftb_mtx); 2559258311Sasomers#endif 2560258311Sasomers kmem_free(fasttrap_provs.fth_table, fasttrap_provs.fth_nent * 2561258311Sasomers sizeof (fasttrap_bucket_t)); 2562258311Sasomers mtx_destroy(&fasttrap_cleanup_mtx); 2563258311Sasomers mutex_destroy(&fasttrap_count_mtx); 2564258311Sasomers return (ret); 2565258311Sasomers } 2566258311Sasomers 2567258311Sasomers 2568179193Sjb /* 2569179193Sjb * ... and the procs hash table. 2570179193Sjb */ 2571179193Sjb nent = FASTTRAP_PROCS_DEFAULT_SIZE; 2572275562Sdelphij if (ISP2(nent)) 2573179193Sjb fasttrap_procs.fth_nent = nent; 2574179193Sjb else 2575179193Sjb fasttrap_procs.fth_nent = 1 << fasttrap_highbit(nent); 2576179193Sjb ASSERT(fasttrap_procs.fth_nent > 0); 2577179193Sjb fasttrap_procs.fth_mask = fasttrap_procs.fth_nent - 1; 2578179193Sjb fasttrap_procs.fth_table = kmem_zalloc(fasttrap_procs.fth_nent * 2579179193Sjb sizeof (fasttrap_bucket_t), KM_SLEEP); 2580277300Ssmh#ifndef illumos 2581211738Srpaulo for (i = 0; i < fasttrap_procs.fth_nent; i++) 2582211738Srpaulo mutex_init(&fasttrap_procs.fth_table[i].ftb_mtx, 2583211738Srpaulo "processes bucket mtx", MUTEX_DEFAULT, NULL); 2584211925Srpaulo 2585296479Smarkj rm_init(&fasttrap_tp_lock, "fasttrap tracepoint"); 2586264434Smarkj 2587264434Smarkj /* 2588264434Smarkj * This event handler must run before kdtrace_thread_dtor() since it 2589264434Smarkj * accesses the thread's struct kdtrace_thread. 2590264434Smarkj */ 2591264434Smarkj fasttrap_thread_dtor_tag = EVENTHANDLER_REGISTER(thread_dtor, 2592264434Smarkj fasttrap_thread_dtor, NULL, EVENTHANDLER_PRI_FIRST); 2593211738Srpaulo#endif 2594179193Sjb 2595253079Savg /* 2596253079Savg * Install our hooks into fork(2), exec(2), and exit(2). 2597253079Savg */ 2598253079Savg dtrace_fasttrap_fork = &fasttrap_fork; 2599253079Savg dtrace_fasttrap_exit = &fasttrap_exec_exit; 2600253079Savg dtrace_fasttrap_exec = &fasttrap_exec_exit; 2601253079Savg 2602179193Sjb (void) dtrace_meta_register("fasttrap", &fasttrap_mops, NULL, 2603179193Sjb &fasttrap_meta_id); 2604179193Sjb 2605211738Srpaulo return (0); 2606179193Sjb} 2607179193Sjb 2608179193Sjbstatic int 2609211738Srpaulofasttrap_unload(void) 2610179193Sjb{ 2611179193Sjb int i, fail = 0; 2612179193Sjb 2613179193Sjb /* 2614179193Sjb * Unregister the meta-provider to make sure no new fasttrap- 2615179193Sjb * managed providers come along while we're trying to close up 2616179193Sjb * shop. If we fail to detach, we'll need to re-register as a 2617179193Sjb * meta-provider. We can fail to unregister as a meta-provider 2618179193Sjb * if providers we manage still exist. 2619179193Sjb */ 2620179193Sjb if (fasttrap_meta_id != DTRACE_METAPROVNONE && 2621179193Sjb dtrace_meta_unregister(fasttrap_meta_id) != 0) 2622211738Srpaulo return (-1); 2623179193Sjb 2624179193Sjb /* 2625179193Sjb * Iterate over all of our providers. If there's still a process 2626179193Sjb * that corresponds to that pid, fail to detach. 2627179193Sjb */ 2628179193Sjb for (i = 0; i < fasttrap_provs.fth_nent; i++) { 2629179193Sjb fasttrap_provider_t **fpp, *fp; 2630179193Sjb fasttrap_bucket_t *bucket = &fasttrap_provs.fth_table[i]; 2631179193Sjb 2632179193Sjb mutex_enter(&bucket->ftb_mtx); 2633179193Sjb fpp = (fasttrap_provider_t **)&bucket->ftb_data; 2634179193Sjb while ((fp = *fpp) != NULL) { 2635179193Sjb /* 2636179193Sjb * Acquire and release the lock as a simple way of 2637179193Sjb * waiting for any other consumer to finish with 2638179193Sjb * this provider. A thread must first acquire the 2639179193Sjb * bucket lock so there's no chance of another thread 2640179193Sjb * blocking on the provider's lock. 2641179193Sjb */ 2642179193Sjb mutex_enter(&fp->ftp_mtx); 2643179193Sjb mutex_exit(&fp->ftp_mtx); 2644179193Sjb 2645179193Sjb if (dtrace_unregister(fp->ftp_provid) != 0) { 2646179193Sjb fail = 1; 2647179193Sjb fpp = &fp->ftp_next; 2648179193Sjb } else { 2649179193Sjb *fpp = fp->ftp_next; 2650179193Sjb fasttrap_provider_free(fp); 2651179193Sjb } 2652179193Sjb } 2653179193Sjb 2654179193Sjb mutex_exit(&bucket->ftb_mtx); 2655179193Sjb } 2656179193Sjb 2657179193Sjb if (fail) { 2658179193Sjb (void) dtrace_meta_register("fasttrap", &fasttrap_mops, NULL, 2659179193Sjb &fasttrap_meta_id); 2660179193Sjb 2661211738Srpaulo return (-1); 2662179193Sjb } 2663179193Sjb 2664258311Sasomers /* 2665258311Sasomers * Stop new processes from entering these hooks now, before the 2666258311Sasomers * fasttrap_cleanup thread runs. That way all processes will hopefully 2667258311Sasomers * be out of these hooks before we free fasttrap_provs.fth_table 2668258311Sasomers */ 2669258311Sasomers ASSERT(dtrace_fasttrap_fork == &fasttrap_fork); 2670258311Sasomers dtrace_fasttrap_fork = NULL; 2671258311Sasomers 2672258311Sasomers ASSERT(dtrace_fasttrap_exec == &fasttrap_exec_exit); 2673258311Sasomers dtrace_fasttrap_exec = NULL; 2674258311Sasomers 2675258311Sasomers ASSERT(dtrace_fasttrap_exit == &fasttrap_exec_exit); 2676258311Sasomers dtrace_fasttrap_exit = NULL; 2677258311Sasomers 2678250953Smarkj mtx_lock(&fasttrap_cleanup_mtx); 2679250953Smarkj fasttrap_cleanup_drain = 1; 2680250953Smarkj /* Wait for the cleanup thread to finish up and signal us. */ 2681250953Smarkj wakeup(&fasttrap_cleanup_cv); 2682250953Smarkj mtx_sleep(&fasttrap_cleanup_drain, &fasttrap_cleanup_mtx, 0, "ftcld", 2683250953Smarkj 0); 2684250953Smarkj fasttrap_cleanup_proc = NULL; 2685252493Smarkj mtx_destroy(&fasttrap_cleanup_mtx); 2686250953Smarkj 2687179193Sjb#ifdef DEBUG 2688179193Sjb mutex_enter(&fasttrap_count_mtx); 2689179193Sjb ASSERT(fasttrap_pid_count == 0); 2690179193Sjb mutex_exit(&fasttrap_count_mtx); 2691179193Sjb#endif 2692179193Sjb 2693277300Ssmh#ifndef illumos 2694264434Smarkj EVENTHANDLER_DEREGISTER(thread_dtor, fasttrap_thread_dtor_tag); 2695264434Smarkj 2696258311Sasomers for (i = 0; i < fasttrap_tpoints.fth_nent; i++) 2697258311Sasomers mutex_destroy(&fasttrap_tpoints.fth_table[i].ftb_mtx); 2698258311Sasomers for (i = 0; i < fasttrap_provs.fth_nent; i++) 2699258311Sasomers mutex_destroy(&fasttrap_provs.fth_table[i].ftb_mtx); 2700258311Sasomers for (i = 0; i < fasttrap_procs.fth_nent; i++) 2701258311Sasomers mutex_destroy(&fasttrap_procs.fth_table[i].ftb_mtx); 2702258311Sasomers#endif 2703179193Sjb kmem_free(fasttrap_tpoints.fth_table, 2704179193Sjb fasttrap_tpoints.fth_nent * sizeof (fasttrap_bucket_t)); 2705179193Sjb fasttrap_tpoints.fth_nent = 0; 2706179193Sjb 2707179193Sjb kmem_free(fasttrap_provs.fth_table, 2708179193Sjb fasttrap_provs.fth_nent * sizeof (fasttrap_bucket_t)); 2709179193Sjb fasttrap_provs.fth_nent = 0; 2710179193Sjb 2711179193Sjb kmem_free(fasttrap_procs.fth_table, 2712179193Sjb fasttrap_procs.fth_nent * sizeof (fasttrap_bucket_t)); 2713179193Sjb fasttrap_procs.fth_nent = 0; 2714179193Sjb 2715277300Ssmh#ifndef illumos 2716211738Srpaulo destroy_dev(fasttrap_cdev); 2717211738Srpaulo mutex_destroy(&fasttrap_count_mtx); 2718296479Smarkj rm_destroy(&fasttrap_tp_lock); 2719211738Srpaulo#endif 2720179193Sjb 2721211738Srpaulo return (0); 2722179193Sjb} 2723179193Sjb 2724211738Srpaulo/* ARGSUSED */ 2725211738Srpaulostatic int 2726211738Srpaulofasttrap_modevent(module_t mod __unused, int type, void *data __unused) 2727211738Srpaulo{ 2728211738Srpaulo int error = 0; 2729179193Sjb 2730211738Srpaulo switch (type) { 2731211738Srpaulo case MOD_LOAD: 2732211738Srpaulo break; 2733179193Sjb 2734211738Srpaulo case MOD_UNLOAD: 2735211738Srpaulo break; 2736179193Sjb 2737211738Srpaulo case MOD_SHUTDOWN: 2738211738Srpaulo break; 2739179193Sjb 2740211738Srpaulo default: 2741211738Srpaulo error = EOPNOTSUPP; 2742211738Srpaulo break; 2743211738Srpaulo } 2744211738Srpaulo return (error); 2745179193Sjb} 2746179193Sjb 2747211738SrpauloSYSINIT(fasttrap_load, SI_SUB_DTRACE_PROVIDER, SI_ORDER_ANY, fasttrap_load, 2748211738Srpaulo NULL); 2749211738SrpauloSYSUNINIT(fasttrap_unload, SI_SUB_DTRACE_PROVIDER, SI_ORDER_ANY, 2750211738Srpaulo fasttrap_unload, NULL); 2751211738Srpaulo 2752211738SrpauloDEV_MODULE(fasttrap, fasttrap_modevent, NULL); 2753211738SrpauloMODULE_VERSION(fasttrap, 1); 2754211738SrpauloMODULE_DEPEND(fasttrap, dtrace, 1, 1, 1); 2755211738SrpauloMODULE_DEPEND(fasttrap, opensolaris, 1, 1, 1); 2756