1179193Sjb/* 2179193Sjb * CDDL HEADER START 3179193Sjb * 4179193Sjb * The contents of this file are subject to the terms of the 5179193Sjb * Common Development and Distribution License (the "License"). 6179193Sjb * You may not use this file except in compliance with the License. 7179193Sjb * 8179193Sjb * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9179193Sjb * or http://www.opensolaris.org/os/licensing. 10179193Sjb * See the License for the specific language governing permissions 11179193Sjb * and limitations under the License. 12179193Sjb * 13179193Sjb * When distributing Covered Code, include this CDDL HEADER in each 14179193Sjb * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15179193Sjb * If applicable, add the following below this CDDL HEADER, with the 16179193Sjb * fields enclosed by brackets "[]" replaced with your own identifying 17179193Sjb * information: Portions Copyright [yyyy] [name of copyright owner] 18179193Sjb * 19179193Sjb * CDDL HEADER END 20211738Srpaulo * 21211738Srpaulo * Portions Copyright 2010 The FreeBSD Foundation 22211738Srpaulo * 23211738Srpaulo * $FreeBSD: releng/10.3/sys/cddl/contrib/opensolaris/uts/common/dtrace/fasttrap.c 293413 2016-01-08 03:45:28Z stas $ 24179193Sjb */ 25179193Sjb 26179193Sjb/* 27179198Sjb * Copyright 2008 Sun Microsystems, Inc. All rights reserved. 28179193Sjb * Use is subject to license terms. 29179193Sjb */ 30179193Sjb 31268734Spfg/* 32268734Spfg * Copyright (c) 2013, Joyent, Inc. All rights reserved. 33268734Spfg */ 34179193Sjb 35179193Sjb#include <sys/atomic.h> 36179193Sjb#include <sys/errno.h> 37179193Sjb#include <sys/stat.h> 38179193Sjb#include <sys/modctl.h> 39179193Sjb#include <sys/conf.h> 40179193Sjb#include <sys/systm.h> 41211738Srpaulo#if defined(sun) 42179193Sjb#include <sys/ddi.h> 43211738Srpaulo#endif 44179193Sjb#include <sys/sunddi.h> 45179193Sjb#include <sys/cpuvar.h> 46179193Sjb#include <sys/kmem.h> 47211738Srpaulo#if defined(sun) 48179193Sjb#include <sys/strsubr.h> 49211738Srpaulo#endif 50179193Sjb#include <sys/fasttrap.h> 51179193Sjb#include <sys/fasttrap_impl.h> 52179193Sjb#include <sys/fasttrap_isa.h> 53179193Sjb#include <sys/dtrace.h> 54179193Sjb#include <sys/dtrace_impl.h> 55179193Sjb#include <sys/sysmacros.h> 56179193Sjb#include <sys/proc.h> 57179193Sjb#include <sys/policy.h> 58211738Srpaulo#if defined(sun) 59179193Sjb#include <util/qsort.h> 60211738Srpaulo#endif 61211738Srpaulo#include <sys/mutex.h> 62211738Srpaulo#include <sys/kernel.h> 63211738Srpaulo#if !defined(sun) 64269342Smarkj#include <sys/dtrace_bsd.h> 65269342Smarkj#include <sys/eventhandler.h> 66293413Sstas#include <sys/sysctl.h> 67293413Sstas#include <sys/u8_textprep.h> 68211738Srpaulo#include <sys/user.h> 69269342Smarkj#include <vm/vm.h> 70269342Smarkj#include <vm/pmap.h> 71269342Smarkj#include <vm/vm_map.h> 72269342Smarkj#include <vm/vm_param.h> 73268734Spfg#include <sys/u8_textprep.h> 74211738Srpaulo#include <cddl/dev/dtrace/dtrace_cddl.h> 75211738Srpaulo#endif 76179193Sjb 77179193Sjb/* 78179193Sjb * User-Land Trap-Based Tracing 79179193Sjb * ---------------------------- 80179193Sjb * 81179193Sjb * The fasttrap provider allows DTrace consumers to instrument any user-level 82179193Sjb * instruction to gather data; this includes probes with semantic 83179193Sjb * signifigance like entry and return as well as simple offsets into the 84179193Sjb * function. While the specific techniques used are very ISA specific, the 85179193Sjb * methodology is generalizable to any architecture. 86179193Sjb * 87179193Sjb * 88179193Sjb * The General Methodology 89179193Sjb * ----------------------- 90179193Sjb * 91179193Sjb * With the primary goal of tracing every user-land instruction and the 92179193Sjb * limitation that we can't trust user space so don't want to rely on much 93179193Sjb * information there, we begin by replacing the instructions we want to trace 94179193Sjb * with trap instructions. Each instruction we overwrite is saved into a hash 95179193Sjb * table keyed by process ID and pc address. When we enter the kernel due to 96179193Sjb * this trap instruction, we need the effects of the replaced instruction to 97179193Sjb * appear to have occurred before we proceed with the user thread's 98179193Sjb * execution. 99179193Sjb * 100179193Sjb * Each user level thread is represented by a ulwp_t structure which is 101179193Sjb * always easily accessible through a register. The most basic way to produce 102179193Sjb * the effects of the instruction we replaced is to copy that instruction out 103179193Sjb * to a bit of scratch space reserved in the user thread's ulwp_t structure 104179193Sjb * (a sort of kernel-private thread local storage), set the PC to that 105179193Sjb * scratch space and single step. When we reenter the kernel after single 106179193Sjb * stepping the instruction we must then adjust the PC to point to what would 107179193Sjb * normally be the next instruction. Of course, special care must be taken 108179193Sjb * for branches and jumps, but these represent such a small fraction of any 109179193Sjb * instruction set that writing the code to emulate these in the kernel is 110179193Sjb * not too difficult. 111179193Sjb * 112179193Sjb * Return probes may require several tracepoints to trace every return site, 113179193Sjb * and, conversely, each tracepoint may activate several probes (the entry 114179193Sjb * and offset 0 probes, for example). To solve this muliplexing problem, 115179193Sjb * tracepoints contain lists of probes to activate and probes contain lists 116179193Sjb * of tracepoints to enable. If a probe is activated, it adds its ID to 117179193Sjb * existing tracepoints or creates new ones as necessary. 118179193Sjb * 119179193Sjb * Most probes are activated _before_ the instruction is executed, but return 120179193Sjb * probes are activated _after_ the effects of the last instruction of the 121179193Sjb * function are visible. Return probes must be fired _after_ we have 122179193Sjb * single-stepped the instruction whereas all other probes are fired 123179193Sjb * beforehand. 124179193Sjb * 125179193Sjb * 126179193Sjb * Lock Ordering 127179193Sjb * ------------- 128179193Sjb * 129179193Sjb * The lock ordering below -- both internally and with respect to the DTrace 130179193Sjb * framework -- is a little tricky and bears some explanation. Each provider 131179193Sjb * has a lock (ftp_mtx) that protects its members including reference counts 132179193Sjb * for enabled probes (ftp_rcount), consumers actively creating probes 133179193Sjb * (ftp_ccount) and USDT consumers (ftp_mcount); all three prevent a provider 134179193Sjb * from being freed. A provider is looked up by taking the bucket lock for the 135179193Sjb * provider hash table, and is returned with its lock held. The provider lock 136179193Sjb * may be taken in functions invoked by the DTrace framework, but may not be 137179193Sjb * held while calling functions in the DTrace framework. 138179193Sjb * 139179193Sjb * To ensure consistency over multiple calls to the DTrace framework, the 140179193Sjb * creation lock (ftp_cmtx) should be held. Naturally, the creation lock may 141179193Sjb * not be taken when holding the provider lock as that would create a cyclic 142179193Sjb * lock ordering. In situations where one would naturally take the provider 143179193Sjb * lock and then the creation lock, we instead up a reference count to prevent 144179193Sjb * the provider from disappearing, drop the provider lock, and acquire the 145179193Sjb * creation lock. 146179193Sjb * 147179193Sjb * Briefly: 148179193Sjb * bucket lock before provider lock 149179193Sjb * DTrace before provider lock 150179193Sjb * creation lock before DTrace 151179193Sjb * never hold the provider lock and creation lock simultaneously 152179193Sjb */ 153179193Sjb 154211738Srpaulostatic d_open_t fasttrap_open; 155211738Srpaulostatic d_ioctl_t fasttrap_ioctl; 156211738Srpaulo 157211738Srpaulostatic struct cdevsw fasttrap_cdevsw = { 158211738Srpaulo .d_version = D_VERSION, 159211738Srpaulo .d_open = fasttrap_open, 160211738Srpaulo .d_ioctl = fasttrap_ioctl, 161211738Srpaulo .d_name = "fasttrap", 162211738Srpaulo}; 163211738Srpaulostatic struct cdev *fasttrap_cdev; 164179193Sjbstatic dtrace_meta_provider_id_t fasttrap_meta_id; 165179193Sjb 166250953Smarkjstatic struct proc *fasttrap_cleanup_proc; 167211738Srpaulostatic struct mtx fasttrap_cleanup_mtx; 168250953Smarkjstatic uint_t fasttrap_cleanup_work, fasttrap_cleanup_drain, fasttrap_cleanup_cv; 169179193Sjb 170179193Sjb/* 171179193Sjb * Generation count on modifications to the global tracepoint lookup table. 172179193Sjb */ 173179193Sjbstatic volatile uint64_t fasttrap_mod_gen; 174179193Sjb 175179193Sjb/* 176179193Sjb * When the fasttrap provider is loaded, fasttrap_max is set to either 177293413Sstas * FASTTRAP_MAX_DEFAULT, or the value for fasttrap-max-probes in the 178293413Sstas * fasttrap.conf file (Illumos), or the value provied in the loader.conf (FreeBSD). 179293413Sstas * Each time a probe is created, fasttrap_total is incremented by the number 180293413Sstas * of tracepoints that may be associated with that probe; fasttrap_total is capped 181293413Sstas * at fasttrap_max. 182179193Sjb */ 183179193Sjb#define FASTTRAP_MAX_DEFAULT 250000 184293413Sstasstatic uint32_t fasttrap_max = FASTTRAP_MAX_DEFAULT; 185179193Sjbstatic uint32_t fasttrap_total; 186179193Sjb 187248983Spfg/* 188248983Spfg * Copyright (c) 2011, Joyent, Inc. All rights reserved. 189248983Spfg */ 190179193Sjb 191179193Sjb#define FASTTRAP_TPOINTS_DEFAULT_SIZE 0x4000 192179193Sjb#define FASTTRAP_PROVIDERS_DEFAULT_SIZE 0x100 193179193Sjb#define FASTTRAP_PROCS_DEFAULT_SIZE 0x100 194179193Sjb 195179193Sjb#define FASTTRAP_PID_NAME "pid" 196179193Sjb 197179193Sjbfasttrap_hash_t fasttrap_tpoints; 198179193Sjbstatic fasttrap_hash_t fasttrap_provs; 199179193Sjbstatic fasttrap_hash_t fasttrap_procs; 200179193Sjb 201179193Sjbstatic uint64_t fasttrap_pid_count; /* pid ref count */ 202179193Sjbstatic kmutex_t fasttrap_count_mtx; /* lock on ref count */ 203179193Sjb 204179193Sjb#define FASTTRAP_ENABLE_FAIL 1 205179193Sjb#define FASTTRAP_ENABLE_PARTIAL 2 206179193Sjb 207179193Sjbstatic int fasttrap_tracepoint_enable(proc_t *, fasttrap_probe_t *, uint_t); 208179193Sjbstatic void fasttrap_tracepoint_disable(proc_t *, fasttrap_probe_t *, uint_t); 209179193Sjb 210179193Sjbstatic fasttrap_provider_t *fasttrap_provider_lookup(pid_t, const char *, 211179193Sjb const dtrace_pattr_t *); 212179193Sjbstatic void fasttrap_provider_retire(pid_t, const char *, int); 213179193Sjbstatic void fasttrap_provider_free(fasttrap_provider_t *); 214179193Sjb 215179193Sjbstatic fasttrap_proc_t *fasttrap_proc_lookup(pid_t); 216179193Sjbstatic void fasttrap_proc_release(fasttrap_proc_t *); 217179193Sjb 218269342Smarkj#if !defined(sun) 219269342Smarkjstatic void fasttrap_thread_dtor(void *, struct thread *); 220269342Smarkj#endif 221269342Smarkj 222179193Sjb#define FASTTRAP_PROVS_INDEX(pid, name) \ 223179193Sjb ((fasttrap_hash_str(name) + (pid)) & fasttrap_provs.fth_mask) 224179193Sjb 225179193Sjb#define FASTTRAP_PROCS_INDEX(pid) ((pid) & fasttrap_procs.fth_mask) 226179193Sjb 227211925Srpaulo#if !defined(sun) 228211925Srpaulostatic kmutex_t fasttrap_cpuc_pid_lock[MAXCPU]; 229269342Smarkjstatic eventhandler_tag fasttrap_thread_dtor_tag; 230211925Srpaulo#endif 231211925Srpaulo 232293413Sstasstatic unsigned long tpoints_hash_size = FASTTRAP_TPOINTS_DEFAULT_SIZE; 233293413Sstas 234293413Sstas#ifdef __FreeBSD__ 235293413SstasSYSCTL_DECL(_kern_dtrace); 236293413SstasSYSCTL_NODE(_kern_dtrace, OID_AUTO, fasttrap, CTLFLAG_RD, 0, "DTrace fasttrap parameters"); 237293413SstasSYSCTL_UINT(_kern_dtrace_fasttrap, OID_AUTO, max_probes, CTLFLAG_RWTUN, &fasttrap_max, 238293413Sstas FASTTRAP_MAX_DEFAULT, "Maximum number of fasttrap probes"); 239293413SstasSYSCTL_ULONG(_kern_dtrace_fasttrap, OID_AUTO, tpoints_hash_size, CTLFLAG_RDTUN, &tpoints_hash_size, 240293413Sstas FASTTRAP_TPOINTS_DEFAULT_SIZE, "Size of the tracepoint hash table"); 241293413Sstas#endif 242293413Sstas 243179193Sjbstatic int 244179193Sjbfasttrap_highbit(ulong_t i) 245179193Sjb{ 246179193Sjb int h = 1; 247179193Sjb 248179193Sjb if (i == 0) 249179193Sjb return (0); 250179193Sjb#ifdef _LP64 251179193Sjb if (i & 0xffffffff00000000ul) { 252179193Sjb h += 32; i >>= 32; 253179193Sjb } 254179193Sjb#endif 255179193Sjb if (i & 0xffff0000) { 256179193Sjb h += 16; i >>= 16; 257179193Sjb } 258179193Sjb if (i & 0xff00) { 259179193Sjb h += 8; i >>= 8; 260179193Sjb } 261179193Sjb if (i & 0xf0) { 262179193Sjb h += 4; i >>= 4; 263179193Sjb } 264179193Sjb if (i & 0xc) { 265179193Sjb h += 2; i >>= 2; 266179193Sjb } 267179193Sjb if (i & 0x2) { 268179193Sjb h += 1; 269179193Sjb } 270179193Sjb return (h); 271179193Sjb} 272179193Sjb 273179193Sjbstatic uint_t 274179193Sjbfasttrap_hash_str(const char *p) 275179193Sjb{ 276179193Sjb unsigned int g; 277179193Sjb uint_t hval = 0; 278179193Sjb 279179193Sjb while (*p) { 280179193Sjb hval = (hval << 4) + *p++; 281179193Sjb if ((g = (hval & 0xf0000000)) != 0) 282179193Sjb hval ^= g >> 24; 283179193Sjb hval &= ~g; 284179193Sjb } 285179193Sjb return (hval); 286179193Sjb} 287179193Sjb 288179193Sjbvoid 289179193Sjbfasttrap_sigtrap(proc_t *p, kthread_t *t, uintptr_t pc) 290179193Sjb{ 291211738Srpaulo#if defined(sun) 292179193Sjb sigqueue_t *sqp = kmem_zalloc(sizeof (sigqueue_t), KM_SLEEP); 293179193Sjb 294179193Sjb sqp->sq_info.si_signo = SIGTRAP; 295179193Sjb sqp->sq_info.si_code = TRAP_DTRACE; 296179193Sjb sqp->sq_info.si_addr = (caddr_t)pc; 297179193Sjb 298179193Sjb mutex_enter(&p->p_lock); 299179193Sjb sigaddqa(p, t, sqp); 300179193Sjb mutex_exit(&p->p_lock); 301179193Sjb 302179193Sjb if (t != NULL) 303179193Sjb aston(t); 304211738Srpaulo#else 305211738Srpaulo ksiginfo_t *ksi = kmem_zalloc(sizeof (ksiginfo_t), KM_SLEEP); 306211738Srpaulo 307211738Srpaulo ksiginfo_init(ksi); 308211738Srpaulo ksi->ksi_signo = SIGTRAP; 309211738Srpaulo ksi->ksi_code = TRAP_DTRACE; 310211738Srpaulo ksi->ksi_addr = (caddr_t)pc; 311211738Srpaulo PROC_LOCK(p); 312283677Smarkj (void) tdsendsignal(p, t, SIGTRAP, ksi); 313211738Srpaulo PROC_UNLOCK(p); 314211738Srpaulo#endif 315179193Sjb} 316179193Sjb 317269342Smarkj#if !defined(sun) 318179193Sjb/* 319269342Smarkj * Obtain a chunk of scratch space in the address space of the target process. 320269342Smarkj */ 321269342Smarkjfasttrap_scrspace_t * 322269342Smarkjfasttrap_scraddr(struct thread *td, fasttrap_proc_t *fprc) 323269342Smarkj{ 324269342Smarkj fasttrap_scrblock_t *scrblk; 325269342Smarkj fasttrap_scrspace_t *scrspc; 326269342Smarkj struct proc *p; 327269342Smarkj vm_offset_t addr; 328269342Smarkj int error, i; 329269342Smarkj 330269342Smarkj scrspc = NULL; 331269342Smarkj if (td->t_dtrace_sscr != NULL) { 332269342Smarkj /* If the thread already has scratch space, we're done. */ 333269342Smarkj scrspc = (fasttrap_scrspace_t *)td->t_dtrace_sscr; 334269342Smarkj return (scrspc); 335269342Smarkj } 336269342Smarkj 337269342Smarkj p = td->td_proc; 338269342Smarkj 339269342Smarkj mutex_enter(&fprc->ftpc_mtx); 340269342Smarkj if (LIST_EMPTY(&fprc->ftpc_fscr)) { 341269342Smarkj /* 342269342Smarkj * No scratch space is available, so we'll map a new scratch 343269342Smarkj * space block into the traced process' address space. 344269342Smarkj */ 345269342Smarkj addr = 0; 346269342Smarkj error = vm_map_find(&p->p_vmspace->vm_map, NULL, 0, &addr, 347269342Smarkj FASTTRAP_SCRBLOCK_SIZE, 0, VMFS_ANY_SPACE, VM_PROT_ALL, 348269342Smarkj VM_PROT_ALL, 0); 349269342Smarkj if (error != KERN_SUCCESS) 350269342Smarkj goto done; 351269342Smarkj 352269342Smarkj scrblk = malloc(sizeof(*scrblk), M_SOLARIS, M_WAITOK); 353269342Smarkj scrblk->ftsb_addr = addr; 354269342Smarkj LIST_INSERT_HEAD(&fprc->ftpc_scrblks, scrblk, ftsb_next); 355269342Smarkj 356269342Smarkj /* 357269342Smarkj * Carve the block up into chunks and put them on the free list. 358269342Smarkj */ 359269342Smarkj for (i = 0; 360269342Smarkj i < FASTTRAP_SCRBLOCK_SIZE / FASTTRAP_SCRSPACE_SIZE; i++) { 361269342Smarkj scrspc = malloc(sizeof(*scrspc), M_SOLARIS, M_WAITOK); 362269342Smarkj scrspc->ftss_addr = addr + 363269342Smarkj i * FASTTRAP_SCRSPACE_SIZE; 364269342Smarkj LIST_INSERT_HEAD(&fprc->ftpc_fscr, scrspc, 365269342Smarkj ftss_next); 366269342Smarkj } 367269342Smarkj } 368269342Smarkj 369269342Smarkj /* 370269342Smarkj * Take the first scratch chunk off the free list, put it on the 371269342Smarkj * allocated list, and return its address. 372269342Smarkj */ 373269342Smarkj scrspc = LIST_FIRST(&fprc->ftpc_fscr); 374269342Smarkj LIST_REMOVE(scrspc, ftss_next); 375269342Smarkj LIST_INSERT_HEAD(&fprc->ftpc_ascr, scrspc, ftss_next); 376269342Smarkj 377269342Smarkj /* 378269342Smarkj * This scratch space is reserved for use by td until the thread exits. 379269342Smarkj */ 380269342Smarkj td->t_dtrace_sscr = scrspc; 381269342Smarkj 382269342Smarkjdone: 383269342Smarkj mutex_exit(&fprc->ftpc_mtx); 384269342Smarkj 385269342Smarkj return (scrspc); 386269342Smarkj} 387269342Smarkj 388269342Smarkj/* 389269342Smarkj * Return any allocated per-thread scratch space chunks back to the process' 390269342Smarkj * free list. 391269342Smarkj */ 392269342Smarkjstatic void 393269342Smarkjfasttrap_thread_dtor(void *arg __unused, struct thread *td) 394269342Smarkj{ 395269342Smarkj fasttrap_bucket_t *bucket; 396269342Smarkj fasttrap_proc_t *fprc; 397269342Smarkj fasttrap_scrspace_t *scrspc; 398269342Smarkj pid_t pid; 399269342Smarkj 400269342Smarkj if (td->t_dtrace_sscr == NULL) 401269342Smarkj return; 402269342Smarkj 403269342Smarkj pid = td->td_proc->p_pid; 404269342Smarkj bucket = &fasttrap_procs.fth_table[FASTTRAP_PROCS_INDEX(pid)]; 405269342Smarkj fprc = NULL; 406269342Smarkj 407269342Smarkj /* Look up the fasttrap process handle for this process. */ 408269342Smarkj mutex_enter(&bucket->ftb_mtx); 409269342Smarkj for (fprc = bucket->ftb_data; fprc != NULL; fprc = fprc->ftpc_next) { 410269342Smarkj if (fprc->ftpc_pid == pid) { 411269342Smarkj mutex_enter(&fprc->ftpc_mtx); 412269342Smarkj mutex_exit(&bucket->ftb_mtx); 413269342Smarkj break; 414269342Smarkj } 415269342Smarkj } 416269342Smarkj if (fprc == NULL) { 417269342Smarkj mutex_exit(&bucket->ftb_mtx); 418269342Smarkj return; 419269342Smarkj } 420269342Smarkj 421269342Smarkj scrspc = (fasttrap_scrspace_t *)td->t_dtrace_sscr; 422269342Smarkj LIST_REMOVE(scrspc, ftss_next); 423269342Smarkj LIST_INSERT_HEAD(&fprc->ftpc_fscr, scrspc, ftss_next); 424269342Smarkj 425269342Smarkj mutex_exit(&fprc->ftpc_mtx); 426269342Smarkj} 427269342Smarkj#endif 428269342Smarkj 429269342Smarkj/* 430179193Sjb * This function ensures that no threads are actively using the memory 431179193Sjb * associated with probes that were formerly live. 432179193Sjb */ 433179193Sjbstatic void 434179193Sjbfasttrap_mod_barrier(uint64_t gen) 435179193Sjb{ 436179193Sjb int i; 437179193Sjb 438179193Sjb if (gen < fasttrap_mod_gen) 439179193Sjb return; 440179193Sjb 441179193Sjb fasttrap_mod_gen++; 442179193Sjb 443211925Srpaulo CPU_FOREACH(i) { 444211925Srpaulo mutex_enter(&fasttrap_cpuc_pid_lock[i]); 445211925Srpaulo mutex_exit(&fasttrap_cpuc_pid_lock[i]); 446179193Sjb } 447179193Sjb} 448179193Sjb 449179193Sjb/* 450250953Smarkj * This function performs asynchronous cleanup of fasttrap providers. The 451250953Smarkj * Solaris implementation of this mechanism use a timeout that's activated in 452250953Smarkj * fasttrap_pid_cleanup(), but this doesn't work in FreeBSD: one may sleep while 453250953Smarkj * holding the DTrace mutexes, but it is unsafe to sleep in a callout handler. 454250953Smarkj * Thus we use a dedicated process to perform the cleanup when requested. 455179193Sjb */ 456179193Sjb/*ARGSUSED*/ 457179193Sjbstatic void 458179193Sjbfasttrap_pid_cleanup_cb(void *data) 459179193Sjb{ 460179193Sjb fasttrap_provider_t **fpp, *fp; 461179193Sjb fasttrap_bucket_t *bucket; 462179193Sjb dtrace_provider_id_t provid; 463248983Spfg int i, later = 0, rval; 464179193Sjb 465250953Smarkj mtx_lock(&fasttrap_cleanup_mtx); 466250953Smarkj while (!fasttrap_cleanup_drain || later > 0) { 467179193Sjb fasttrap_cleanup_work = 0; 468211738Srpaulo mtx_unlock(&fasttrap_cleanup_mtx); 469179193Sjb 470179193Sjb later = 0; 471179193Sjb 472179193Sjb /* 473179193Sjb * Iterate over all the providers trying to remove the marked 474179193Sjb * ones. If a provider is marked but not retired, we just 475179193Sjb * have to take a crack at removing it -- it's no big deal if 476179193Sjb * we can't. 477179193Sjb */ 478179193Sjb for (i = 0; i < fasttrap_provs.fth_nent; i++) { 479179193Sjb bucket = &fasttrap_provs.fth_table[i]; 480179193Sjb mutex_enter(&bucket->ftb_mtx); 481179193Sjb fpp = (fasttrap_provider_t **)&bucket->ftb_data; 482179193Sjb 483179193Sjb while ((fp = *fpp) != NULL) { 484179193Sjb if (!fp->ftp_marked) { 485179193Sjb fpp = &fp->ftp_next; 486179193Sjb continue; 487179193Sjb } 488179193Sjb 489179193Sjb mutex_enter(&fp->ftp_mtx); 490179193Sjb 491179193Sjb /* 492179193Sjb * If this provider has consumers actively 493179193Sjb * creating probes (ftp_ccount) or is a USDT 494179193Sjb * provider (ftp_mcount), we can't unregister 495179193Sjb * or even condense. 496179193Sjb */ 497179193Sjb if (fp->ftp_ccount != 0 || 498179193Sjb fp->ftp_mcount != 0) { 499179193Sjb mutex_exit(&fp->ftp_mtx); 500179193Sjb fp->ftp_marked = 0; 501179193Sjb continue; 502179193Sjb } 503179193Sjb 504179193Sjb if (!fp->ftp_retired || fp->ftp_rcount != 0) 505179193Sjb fp->ftp_marked = 0; 506179193Sjb 507179193Sjb mutex_exit(&fp->ftp_mtx); 508179193Sjb 509179193Sjb /* 510179193Sjb * If we successfully unregister this 511179193Sjb * provider we can remove it from the hash 512179193Sjb * chain and free the memory. If our attempt 513179193Sjb * to unregister fails and this is a retired 514179193Sjb * provider, increment our flag to try again 515179193Sjb * pretty soon. If we've consumed more than 516179193Sjb * half of our total permitted number of 517179193Sjb * probes call dtrace_condense() to try to 518179193Sjb * clean out the unenabled probes. 519179193Sjb */ 520179193Sjb provid = fp->ftp_provid; 521248983Spfg if ((rval = dtrace_unregister(provid)) != 0) { 522179193Sjb if (fasttrap_total > fasttrap_max / 2) 523179193Sjb (void) dtrace_condense(provid); 524248983Spfg 525248983Spfg if (rval == EAGAIN) 526248983Spfg fp->ftp_marked = 1; 527248983Spfg 528179193Sjb later += fp->ftp_marked; 529179193Sjb fpp = &fp->ftp_next; 530179193Sjb } else { 531179193Sjb *fpp = fp->ftp_next; 532179193Sjb fasttrap_provider_free(fp); 533179193Sjb } 534179193Sjb } 535179193Sjb mutex_exit(&bucket->ftb_mtx); 536179193Sjb } 537250953Smarkj mtx_lock(&fasttrap_cleanup_mtx); 538179193Sjb 539250953Smarkj /* 540250953Smarkj * If we were unable to retire a provider, try again after a 541250953Smarkj * second. This situation can occur in certain circumstances 542250953Smarkj * where providers cannot be unregistered even though they have 543250953Smarkj * no probes enabled because of an execution of dtrace -l or 544250953Smarkj * something similar. 545250953Smarkj */ 546250953Smarkj if (later > 0 || fasttrap_cleanup_work || 547250953Smarkj fasttrap_cleanup_drain) { 548250953Smarkj mtx_unlock(&fasttrap_cleanup_mtx); 549250953Smarkj pause("ftclean", hz); 550250953Smarkj mtx_lock(&fasttrap_cleanup_mtx); 551250953Smarkj } else 552250953Smarkj mtx_sleep(&fasttrap_cleanup_cv, &fasttrap_cleanup_mtx, 553250953Smarkj 0, "ftcl", 0); 554179193Sjb } 555179193Sjb 556179193Sjb /* 557250953Smarkj * Wake up the thread in fasttrap_unload() now that we're done. 558179193Sjb */ 559250953Smarkj wakeup(&fasttrap_cleanup_drain); 560250953Smarkj mtx_unlock(&fasttrap_cleanup_mtx); 561179193Sjb 562250953Smarkj kthread_exit(); 563179193Sjb} 564179193Sjb 565179193Sjb/* 566179193Sjb * Activates the asynchronous cleanup mechanism. 567179193Sjb */ 568179193Sjbstatic void 569179193Sjbfasttrap_pid_cleanup(void) 570179193Sjb{ 571211738Srpaulo 572211738Srpaulo mtx_lock(&fasttrap_cleanup_mtx); 573250953Smarkj if (!fasttrap_cleanup_work) { 574250953Smarkj fasttrap_cleanup_work = 1; 575250953Smarkj wakeup(&fasttrap_cleanup_cv); 576250953Smarkj } 577211738Srpaulo mtx_unlock(&fasttrap_cleanup_mtx); 578179193Sjb} 579179193Sjb 580179193Sjb/* 581179193Sjb * This is called from cfork() via dtrace_fasttrap_fork(). The child 582179198Sjb * process's address space is (roughly) a copy of the parent process's so 583179193Sjb * we have to remove all the instrumentation we had previously enabled in the 584179193Sjb * parent. 585179193Sjb */ 586179193Sjbstatic void 587179193Sjbfasttrap_fork(proc_t *p, proc_t *cp) 588179193Sjb{ 589269342Smarkj#if !defined(sun) 590269342Smarkj fasttrap_scrblock_t *scrblk; 591269342Smarkj fasttrap_proc_t *fprc = NULL; 592269342Smarkj#endif 593179193Sjb pid_t ppid = p->p_pid; 594179193Sjb int i; 595179193Sjb 596211738Srpaulo#if defined(sun) 597179193Sjb ASSERT(curproc == p); 598179193Sjb ASSERT(p->p_proc_flag & P_PR_LOCK); 599211738Srpaulo#else 600211738Srpaulo PROC_LOCK_ASSERT(p, MA_OWNED); 601211738Srpaulo#endif 602211738Srpaulo#if defined(sun) 603179193Sjb ASSERT(p->p_dtrace_count > 0); 604211738Srpaulo#else 605212357Srpaulo if (p->p_dtrace_helpers) { 606212357Srpaulo /* 607212357Srpaulo * dtrace_helpers_duplicate() allocates memory. 608212357Srpaulo */ 609212494Srpaulo _PHOLD(cp); 610212357Srpaulo PROC_UNLOCK(p); 611212357Srpaulo PROC_UNLOCK(cp); 612212357Srpaulo dtrace_helpers_duplicate(p, cp); 613212357Srpaulo PROC_LOCK(cp); 614212357Srpaulo PROC_LOCK(p); 615212494Srpaulo _PRELE(cp); 616212357Srpaulo } 617211738Srpaulo /* 618211738Srpaulo * This check is purposely here instead of in kern_fork.c because, 619211738Srpaulo * for legal resons, we cannot include the dtrace_cddl.h header 620211738Srpaulo * inside kern_fork.c and insert if-clause there. 621211738Srpaulo */ 622211738Srpaulo if (p->p_dtrace_count == 0) 623211738Srpaulo return; 624211738Srpaulo#endif 625179193Sjb ASSERT(cp->p_dtrace_count == 0); 626179193Sjb 627179193Sjb /* 628179193Sjb * This would be simpler and faster if we maintained per-process 629179193Sjb * hash tables of enabled tracepoints. It could, however, potentially 630179193Sjb * slow down execution of a tracepoint since we'd need to go 631179193Sjb * through two levels of indirection. In the future, we should 632179193Sjb * consider either maintaining per-process ancillary lists of 633179193Sjb * enabled tracepoints or hanging a pointer to a per-process hash 634179193Sjb * table of enabled tracepoints off the proc structure. 635179193Sjb */ 636179193Sjb 637179193Sjb /* 638179193Sjb * We don't have to worry about the child process disappearing 639179193Sjb * because we're in fork(). 640179193Sjb */ 641211738Srpaulo#if defined(sun) 642211738Srpaulo mtx_lock_spin(&cp->p_slock); 643179193Sjb sprlock_proc(cp); 644211738Srpaulo mtx_unlock_spin(&cp->p_slock); 645212494Srpaulo#else 646254198Srpaulo /* 647254198Srpaulo * fasttrap_tracepoint_remove() expects the child process to be 648254198Srpaulo * unlocked and the VM then expects curproc to be unlocked. 649254198Srpaulo */ 650212494Srpaulo _PHOLD(cp); 651254198Srpaulo PROC_UNLOCK(cp); 652254198Srpaulo PROC_UNLOCK(p); 653211738Srpaulo#endif 654179193Sjb 655179193Sjb /* 656179193Sjb * Iterate over every tracepoint looking for ones that belong to the 657179193Sjb * parent process, and remove each from the child process. 658179193Sjb */ 659179193Sjb for (i = 0; i < fasttrap_tpoints.fth_nent; i++) { 660179193Sjb fasttrap_tracepoint_t *tp; 661179193Sjb fasttrap_bucket_t *bucket = &fasttrap_tpoints.fth_table[i]; 662179193Sjb 663179193Sjb mutex_enter(&bucket->ftb_mtx); 664179193Sjb for (tp = bucket->ftb_data; tp != NULL; tp = tp->ftt_next) { 665179193Sjb if (tp->ftt_pid == ppid && 666179193Sjb tp->ftt_proc->ftpc_acount != 0) { 667179193Sjb int ret = fasttrap_tracepoint_remove(cp, tp); 668179193Sjb ASSERT(ret == 0); 669179198Sjb 670179198Sjb /* 671179198Sjb * The count of active providers can only be 672179198Sjb * decremented (i.e. to zero) during exec, 673179198Sjb * exit, and removal of a meta provider so it 674179198Sjb * should be impossible to drop the count 675179198Sjb * mid-fork. 676179198Sjb */ 677179198Sjb ASSERT(tp->ftt_proc->ftpc_acount != 0); 678269342Smarkj#if !defined(sun) 679269342Smarkj fprc = tp->ftt_proc; 680269342Smarkj#endif 681179193Sjb } 682179193Sjb } 683179193Sjb mutex_exit(&bucket->ftb_mtx); 684269342Smarkj 685269342Smarkj#if !defined(sun) 686269342Smarkj /* 687269342Smarkj * Unmap any scratch space inherited from the parent's address 688269342Smarkj * space. 689269342Smarkj */ 690269342Smarkj if (fprc != NULL) { 691269342Smarkj mutex_enter(&fprc->ftpc_mtx); 692269342Smarkj LIST_FOREACH(scrblk, &fprc->ftpc_scrblks, ftsb_next) { 693269342Smarkj vm_map_remove(&cp->p_vmspace->vm_map, 694269342Smarkj scrblk->ftsb_addr, 695269342Smarkj scrblk->ftsb_addr + FASTTRAP_SCRBLOCK_SIZE); 696269342Smarkj } 697269342Smarkj mutex_exit(&fprc->ftpc_mtx); 698269342Smarkj } 699269342Smarkj#endif 700179193Sjb } 701179193Sjb 702211738Srpaulo#if defined(sun) 703179193Sjb mutex_enter(&cp->p_lock); 704179193Sjb sprunlock(cp); 705212494Srpaulo#else 706254198Srpaulo PROC_LOCK(p); 707254198Srpaulo PROC_LOCK(cp); 708212494Srpaulo _PRELE(cp); 709211738Srpaulo#endif 710179193Sjb} 711179193Sjb 712179193Sjb/* 713179193Sjb * This is called from proc_exit() or from exec_common() if p_dtrace_probes 714179193Sjb * is set on the proc structure to indicate that there is a pid provider 715179193Sjb * associated with this process. 716179193Sjb */ 717179193Sjbstatic void 718179193Sjbfasttrap_exec_exit(proc_t *p) 719179193Sjb{ 720269342Smarkj#if !defined(sun) 721269342Smarkj struct thread *td; 722269342Smarkj#endif 723269342Smarkj 724211738Srpaulo#if defined(sun) 725179193Sjb ASSERT(p == curproc); 726269342Smarkj#else 727211738Srpaulo PROC_LOCK_ASSERT(p, MA_OWNED); 728212494Srpaulo _PHOLD(p); 729269342Smarkj /* 730269342Smarkj * Since struct threads may be recycled, we cannot rely on t_dtrace_sscr 731269342Smarkj * fields to be zeroed by kdtrace_thread_ctor. Thus we must zero it 732269342Smarkj * ourselves when a process exits. 733269342Smarkj */ 734269342Smarkj FOREACH_THREAD_IN_PROC(p, td) 735269342Smarkj td->t_dtrace_sscr = NULL; 736211738Srpaulo PROC_UNLOCK(p); 737269342Smarkj#endif 738179193Sjb 739179193Sjb /* 740179193Sjb * We clean up the pid provider for this process here; user-land 741179193Sjb * static probes are handled by the meta-provider remove entry point. 742179193Sjb */ 743179193Sjb fasttrap_provider_retire(p->p_pid, FASTTRAP_PID_NAME, 0); 744212357Srpaulo#if !defined(sun) 745212357Srpaulo if (p->p_dtrace_helpers) 746212357Srpaulo dtrace_helpers_destroy(p); 747211738Srpaulo PROC_LOCK(p); 748212494Srpaulo _PRELE(p); 749269342Smarkj#endif 750179193Sjb} 751179193Sjb 752179193Sjb 753179193Sjb/*ARGSUSED*/ 754179193Sjbstatic void 755211738Srpaulofasttrap_pid_provide(void *arg, dtrace_probedesc_t *desc) 756179193Sjb{ 757179193Sjb /* 758179193Sjb * There are no "default" pid probes. 759179193Sjb */ 760179193Sjb} 761179193Sjb 762179193Sjbstatic int 763179193Sjbfasttrap_tracepoint_enable(proc_t *p, fasttrap_probe_t *probe, uint_t index) 764179193Sjb{ 765179193Sjb fasttrap_tracepoint_t *tp, *new_tp = NULL; 766179193Sjb fasttrap_bucket_t *bucket; 767179193Sjb fasttrap_id_t *id; 768179193Sjb pid_t pid; 769179193Sjb uintptr_t pc; 770179193Sjb 771179193Sjb ASSERT(index < probe->ftp_ntps); 772179193Sjb 773179193Sjb pid = probe->ftp_pid; 774179193Sjb pc = probe->ftp_tps[index].fit_tp->ftt_pc; 775179193Sjb id = &probe->ftp_tps[index].fit_id; 776179193Sjb 777179193Sjb ASSERT(probe->ftp_tps[index].fit_tp->ftt_pid == pid); 778179193Sjb 779211738Srpaulo#if defined(sun) 780179193Sjb ASSERT(!(p->p_flag & SVFORK)); 781211738Srpaulo#endif 782179193Sjb 783179193Sjb /* 784179193Sjb * Before we make any modifications, make sure we've imposed a barrier 785179193Sjb * on the generation in which this probe was last modified. 786179193Sjb */ 787179193Sjb fasttrap_mod_barrier(probe->ftp_gen); 788179193Sjb 789179193Sjb bucket = &fasttrap_tpoints.fth_table[FASTTRAP_TPOINTS_INDEX(pid, pc)]; 790179193Sjb 791179193Sjb /* 792179193Sjb * If the tracepoint has already been enabled, just add our id to the 793179193Sjb * list of interested probes. This may be our second time through 794179193Sjb * this path in which case we'll have constructed the tracepoint we'd 795179193Sjb * like to install. If we can't find a match, and have an allocated 796179193Sjb * tracepoint ready to go, enable that one now. 797179193Sjb * 798179193Sjb * A tracepoint whose process is defunct is also considered defunct. 799179193Sjb */ 800179193Sjbagain: 801179193Sjb mutex_enter(&bucket->ftb_mtx); 802179193Sjb for (tp = bucket->ftb_data; tp != NULL; tp = tp->ftt_next) { 803179198Sjb /* 804179198Sjb * Note that it's safe to access the active count on the 805179198Sjb * associated proc structure because we know that at least one 806179198Sjb * provider (this one) will still be around throughout this 807179198Sjb * operation. 808179198Sjb */ 809179193Sjb if (tp->ftt_pid != pid || tp->ftt_pc != pc || 810179193Sjb tp->ftt_proc->ftpc_acount == 0) 811179193Sjb continue; 812179193Sjb 813179193Sjb /* 814179193Sjb * Now that we've found a matching tracepoint, it would be 815179193Sjb * a decent idea to confirm that the tracepoint is still 816179193Sjb * enabled and the trap instruction hasn't been overwritten. 817179193Sjb * Since this is a little hairy, we'll punt for now. 818179193Sjb */ 819179193Sjb 820179193Sjb /* 821179193Sjb * This can't be the first interested probe. We don't have 822179193Sjb * to worry about another thread being in the midst of 823179193Sjb * deleting this tracepoint (which would be the only valid 824179193Sjb * reason for a tracepoint to have no interested probes) 825179193Sjb * since we're holding P_PR_LOCK for this process. 826179193Sjb */ 827179193Sjb ASSERT(tp->ftt_ids != NULL || tp->ftt_retids != NULL); 828179193Sjb 829179193Sjb switch (id->fti_ptype) { 830179193Sjb case DTFTP_ENTRY: 831179193Sjb case DTFTP_OFFSETS: 832179193Sjb case DTFTP_IS_ENABLED: 833179193Sjb id->fti_next = tp->ftt_ids; 834179193Sjb membar_producer(); 835179193Sjb tp->ftt_ids = id; 836179193Sjb membar_producer(); 837179193Sjb break; 838179193Sjb 839179193Sjb case DTFTP_RETURN: 840179193Sjb case DTFTP_POST_OFFSETS: 841179193Sjb id->fti_next = tp->ftt_retids; 842179193Sjb membar_producer(); 843179193Sjb tp->ftt_retids = id; 844179193Sjb membar_producer(); 845179193Sjb break; 846179193Sjb 847179193Sjb default: 848179193Sjb ASSERT(0); 849179193Sjb } 850179193Sjb 851179193Sjb mutex_exit(&bucket->ftb_mtx); 852179193Sjb 853179193Sjb if (new_tp != NULL) { 854179193Sjb new_tp->ftt_ids = NULL; 855179193Sjb new_tp->ftt_retids = NULL; 856179193Sjb } 857179193Sjb 858179193Sjb return (0); 859179193Sjb } 860179193Sjb 861179193Sjb /* 862179193Sjb * If we have a good tracepoint ready to go, install it now while 863179193Sjb * we have the lock held and no one can screw with us. 864179193Sjb */ 865179193Sjb if (new_tp != NULL) { 866179193Sjb int rc = 0; 867179193Sjb 868179193Sjb new_tp->ftt_next = bucket->ftb_data; 869179193Sjb membar_producer(); 870179193Sjb bucket->ftb_data = new_tp; 871179193Sjb membar_producer(); 872179193Sjb mutex_exit(&bucket->ftb_mtx); 873179193Sjb 874179193Sjb /* 875179193Sjb * Activate the tracepoint in the ISA-specific manner. 876179193Sjb * If this fails, we need to report the failure, but 877179193Sjb * indicate that this tracepoint must still be disabled 878179193Sjb * by calling fasttrap_tracepoint_disable(). 879179193Sjb */ 880179193Sjb if (fasttrap_tracepoint_install(p, new_tp) != 0) 881179193Sjb rc = FASTTRAP_ENABLE_PARTIAL; 882179193Sjb 883179193Sjb /* 884179193Sjb * Increment the count of the number of tracepoints active in 885179193Sjb * the victim process. 886179193Sjb */ 887211738Srpaulo#if defined(sun) 888179193Sjb ASSERT(p->p_proc_flag & P_PR_LOCK); 889211738Srpaulo#endif 890179193Sjb p->p_dtrace_count++; 891179193Sjb 892179193Sjb return (rc); 893179193Sjb } 894179193Sjb 895179193Sjb mutex_exit(&bucket->ftb_mtx); 896179193Sjb 897179193Sjb /* 898179193Sjb * Initialize the tracepoint that's been preallocated with the probe. 899179193Sjb */ 900179193Sjb new_tp = probe->ftp_tps[index].fit_tp; 901179193Sjb 902179193Sjb ASSERT(new_tp->ftt_pid == pid); 903179193Sjb ASSERT(new_tp->ftt_pc == pc); 904179193Sjb ASSERT(new_tp->ftt_proc == probe->ftp_prov->ftp_proc); 905179193Sjb ASSERT(new_tp->ftt_ids == NULL); 906179193Sjb ASSERT(new_tp->ftt_retids == NULL); 907179193Sjb 908179193Sjb switch (id->fti_ptype) { 909179193Sjb case DTFTP_ENTRY: 910179193Sjb case DTFTP_OFFSETS: 911179193Sjb case DTFTP_IS_ENABLED: 912179193Sjb id->fti_next = NULL; 913179193Sjb new_tp->ftt_ids = id; 914179193Sjb break; 915179193Sjb 916179193Sjb case DTFTP_RETURN: 917179193Sjb case DTFTP_POST_OFFSETS: 918179193Sjb id->fti_next = NULL; 919179193Sjb new_tp->ftt_retids = id; 920179193Sjb break; 921179193Sjb 922179193Sjb default: 923179193Sjb ASSERT(0); 924179193Sjb } 925179193Sjb 926179193Sjb /* 927179193Sjb * If the ISA-dependent initialization goes to plan, go back to the 928179193Sjb * beginning and try to install this freshly made tracepoint. 929179193Sjb */ 930179193Sjb if (fasttrap_tracepoint_init(p, new_tp, pc, id->fti_ptype) == 0) 931179193Sjb goto again; 932179193Sjb 933179193Sjb new_tp->ftt_ids = NULL; 934179193Sjb new_tp->ftt_retids = NULL; 935179193Sjb 936179193Sjb return (FASTTRAP_ENABLE_FAIL); 937179193Sjb} 938179193Sjb 939179193Sjbstatic void 940179193Sjbfasttrap_tracepoint_disable(proc_t *p, fasttrap_probe_t *probe, uint_t index) 941179193Sjb{ 942179193Sjb fasttrap_bucket_t *bucket; 943179193Sjb fasttrap_provider_t *provider = probe->ftp_prov; 944179193Sjb fasttrap_tracepoint_t **pp, *tp; 945211738Srpaulo fasttrap_id_t *id, **idp = NULL; 946179193Sjb pid_t pid; 947179193Sjb uintptr_t pc; 948179193Sjb 949179193Sjb ASSERT(index < probe->ftp_ntps); 950179193Sjb 951179193Sjb pid = probe->ftp_pid; 952179193Sjb pc = probe->ftp_tps[index].fit_tp->ftt_pc; 953179193Sjb id = &probe->ftp_tps[index].fit_id; 954179193Sjb 955179193Sjb ASSERT(probe->ftp_tps[index].fit_tp->ftt_pid == pid); 956179193Sjb 957179193Sjb /* 958179193Sjb * Find the tracepoint and make sure that our id is one of the 959179193Sjb * ones registered with it. 960179193Sjb */ 961179193Sjb bucket = &fasttrap_tpoints.fth_table[FASTTRAP_TPOINTS_INDEX(pid, pc)]; 962179193Sjb mutex_enter(&bucket->ftb_mtx); 963179193Sjb for (tp = bucket->ftb_data; tp != NULL; tp = tp->ftt_next) { 964179193Sjb if (tp->ftt_pid == pid && tp->ftt_pc == pc && 965179193Sjb tp->ftt_proc == provider->ftp_proc) 966179193Sjb break; 967179193Sjb } 968179193Sjb 969179193Sjb /* 970179193Sjb * If we somehow lost this tracepoint, we're in a world of hurt. 971179193Sjb */ 972179193Sjb ASSERT(tp != NULL); 973179193Sjb 974179193Sjb switch (id->fti_ptype) { 975179193Sjb case DTFTP_ENTRY: 976179193Sjb case DTFTP_OFFSETS: 977179193Sjb case DTFTP_IS_ENABLED: 978179193Sjb ASSERT(tp->ftt_ids != NULL); 979179193Sjb idp = &tp->ftt_ids; 980179193Sjb break; 981179193Sjb 982179193Sjb case DTFTP_RETURN: 983179193Sjb case DTFTP_POST_OFFSETS: 984179193Sjb ASSERT(tp->ftt_retids != NULL); 985179193Sjb idp = &tp->ftt_retids; 986179193Sjb break; 987179193Sjb 988179193Sjb default: 989179193Sjb ASSERT(0); 990179193Sjb } 991179193Sjb 992179193Sjb while ((*idp)->fti_probe != probe) { 993179193Sjb idp = &(*idp)->fti_next; 994179193Sjb ASSERT(*idp != NULL); 995179193Sjb } 996179193Sjb 997179193Sjb id = *idp; 998179193Sjb *idp = id->fti_next; 999179193Sjb membar_producer(); 1000179193Sjb 1001179193Sjb ASSERT(id->fti_probe == probe); 1002179193Sjb 1003179193Sjb /* 1004179193Sjb * If there are other registered enablings of this tracepoint, we're 1005179193Sjb * all done, but if this was the last probe assocated with this 1006179193Sjb * this tracepoint, we need to remove and free it. 1007179193Sjb */ 1008179193Sjb if (tp->ftt_ids != NULL || tp->ftt_retids != NULL) { 1009179193Sjb 1010179193Sjb /* 1011179193Sjb * If the current probe's tracepoint is in use, swap it 1012179193Sjb * for an unused tracepoint. 1013179193Sjb */ 1014179193Sjb if (tp == probe->ftp_tps[index].fit_tp) { 1015179193Sjb fasttrap_probe_t *tmp_probe; 1016179193Sjb fasttrap_tracepoint_t **tmp_tp; 1017179193Sjb uint_t tmp_index; 1018179193Sjb 1019179193Sjb if (tp->ftt_ids != NULL) { 1020179193Sjb tmp_probe = tp->ftt_ids->fti_probe; 1021179193Sjb /* LINTED - alignment */ 1022179193Sjb tmp_index = FASTTRAP_ID_INDEX(tp->ftt_ids); 1023179193Sjb tmp_tp = &tmp_probe->ftp_tps[tmp_index].fit_tp; 1024179193Sjb } else { 1025179193Sjb tmp_probe = tp->ftt_retids->fti_probe; 1026179193Sjb /* LINTED - alignment */ 1027179193Sjb tmp_index = FASTTRAP_ID_INDEX(tp->ftt_retids); 1028179193Sjb tmp_tp = &tmp_probe->ftp_tps[tmp_index].fit_tp; 1029179193Sjb } 1030179193Sjb 1031179193Sjb ASSERT(*tmp_tp != NULL); 1032179193Sjb ASSERT(*tmp_tp != probe->ftp_tps[index].fit_tp); 1033179193Sjb ASSERT((*tmp_tp)->ftt_ids == NULL); 1034179193Sjb ASSERT((*tmp_tp)->ftt_retids == NULL); 1035179193Sjb 1036179193Sjb probe->ftp_tps[index].fit_tp = *tmp_tp; 1037179193Sjb *tmp_tp = tp; 1038179193Sjb } 1039179193Sjb 1040179193Sjb mutex_exit(&bucket->ftb_mtx); 1041179193Sjb 1042179193Sjb /* 1043179193Sjb * Tag the modified probe with the generation in which it was 1044179193Sjb * changed. 1045179193Sjb */ 1046179193Sjb probe->ftp_gen = fasttrap_mod_gen; 1047179193Sjb return; 1048179193Sjb } 1049179193Sjb 1050179193Sjb mutex_exit(&bucket->ftb_mtx); 1051179193Sjb 1052179193Sjb /* 1053179193Sjb * We can't safely remove the tracepoint from the set of active 1054179193Sjb * tracepoints until we've actually removed the fasttrap instruction 1055179193Sjb * from the process's text. We can, however, operate on this 1056179193Sjb * tracepoint secure in the knowledge that no other thread is going to 1057179193Sjb * be looking at it since we hold P_PR_LOCK on the process if it's 1058179193Sjb * live or we hold the provider lock on the process if it's dead and 1059179193Sjb * gone. 1060179193Sjb */ 1061179193Sjb 1062179193Sjb /* 1063179193Sjb * We only need to remove the actual instruction if we're looking 1064179193Sjb * at an existing process 1065179193Sjb */ 1066179193Sjb if (p != NULL) { 1067179193Sjb /* 1068179193Sjb * If we fail to restore the instruction we need to kill 1069179193Sjb * this process since it's in a completely unrecoverable 1070179193Sjb * state. 1071179193Sjb */ 1072179193Sjb if (fasttrap_tracepoint_remove(p, tp) != 0) 1073179193Sjb fasttrap_sigtrap(p, NULL, pc); 1074179193Sjb 1075179193Sjb /* 1076179193Sjb * Decrement the count of the number of tracepoints active 1077179193Sjb * in the victim process. 1078179193Sjb */ 1079211738Srpaulo#if defined(sun) 1080179193Sjb ASSERT(p->p_proc_flag & P_PR_LOCK); 1081211738Srpaulo#endif 1082179193Sjb p->p_dtrace_count--; 1083179193Sjb } 1084179193Sjb 1085179193Sjb /* 1086179193Sjb * Remove the probe from the hash table of active tracepoints. 1087179193Sjb */ 1088179193Sjb mutex_enter(&bucket->ftb_mtx); 1089179193Sjb pp = (fasttrap_tracepoint_t **)&bucket->ftb_data; 1090179193Sjb ASSERT(*pp != NULL); 1091179193Sjb while (*pp != tp) { 1092179193Sjb pp = &(*pp)->ftt_next; 1093179193Sjb ASSERT(*pp != NULL); 1094179193Sjb } 1095179193Sjb 1096179193Sjb *pp = tp->ftt_next; 1097179193Sjb membar_producer(); 1098179193Sjb 1099179193Sjb mutex_exit(&bucket->ftb_mtx); 1100179193Sjb 1101179193Sjb /* 1102179193Sjb * Tag the modified probe with the generation in which it was changed. 1103179193Sjb */ 1104179193Sjb probe->ftp_gen = fasttrap_mod_gen; 1105179193Sjb} 1106179193Sjb 1107179193Sjbstatic void 1108179193Sjbfasttrap_enable_callbacks(void) 1109179193Sjb{ 1110179193Sjb /* 1111179193Sjb * We don't have to play the rw lock game here because we're 1112179193Sjb * providing something rather than taking something away -- 1113179193Sjb * we can be sure that no threads have tried to follow this 1114179193Sjb * function pointer yet. 1115179193Sjb */ 1116179193Sjb mutex_enter(&fasttrap_count_mtx); 1117179193Sjb if (fasttrap_pid_count == 0) { 1118179193Sjb ASSERT(dtrace_pid_probe_ptr == NULL); 1119179193Sjb ASSERT(dtrace_return_probe_ptr == NULL); 1120179193Sjb dtrace_pid_probe_ptr = &fasttrap_pid_probe; 1121179193Sjb dtrace_return_probe_ptr = &fasttrap_return_probe; 1122179193Sjb } 1123179193Sjb ASSERT(dtrace_pid_probe_ptr == &fasttrap_pid_probe); 1124179193Sjb ASSERT(dtrace_return_probe_ptr == &fasttrap_return_probe); 1125179193Sjb fasttrap_pid_count++; 1126179193Sjb mutex_exit(&fasttrap_count_mtx); 1127179193Sjb} 1128179193Sjb 1129179193Sjbstatic void 1130179193Sjbfasttrap_disable_callbacks(void) 1131179193Sjb{ 1132211738Srpaulo#if defined(sun) 1133179193Sjb ASSERT(MUTEX_HELD(&cpu_lock)); 1134211738Srpaulo#endif 1135179193Sjb 1136211738Srpaulo 1137179193Sjb mutex_enter(&fasttrap_count_mtx); 1138179193Sjb ASSERT(fasttrap_pid_count > 0); 1139179193Sjb fasttrap_pid_count--; 1140179193Sjb if (fasttrap_pid_count == 0) { 1141211738Srpaulo#if defined(sun) 1142179193Sjb cpu_t *cur, *cpu = CPU; 1143179193Sjb 1144179193Sjb for (cur = cpu->cpu_next_onln; cur != cpu; 1145179193Sjb cur = cur->cpu_next_onln) { 1146179193Sjb rw_enter(&cur->cpu_ft_lock, RW_WRITER); 1147179193Sjb } 1148211738Srpaulo#endif 1149179193Sjb dtrace_pid_probe_ptr = NULL; 1150179193Sjb dtrace_return_probe_ptr = NULL; 1151211738Srpaulo#if defined(sun) 1152179193Sjb for (cur = cpu->cpu_next_onln; cur != cpu; 1153179193Sjb cur = cur->cpu_next_onln) { 1154179193Sjb rw_exit(&cur->cpu_ft_lock); 1155179193Sjb } 1156211738Srpaulo#endif 1157179193Sjb } 1158179193Sjb mutex_exit(&fasttrap_count_mtx); 1159179193Sjb} 1160179193Sjb 1161179193Sjb/*ARGSUSED*/ 1162179193Sjbstatic void 1163179193Sjbfasttrap_pid_enable(void *arg, dtrace_id_t id, void *parg) 1164179193Sjb{ 1165179193Sjb fasttrap_probe_t *probe = parg; 1166211738Srpaulo proc_t *p = NULL; 1167179193Sjb int i, rc; 1168179193Sjb 1169179193Sjb ASSERT(probe != NULL); 1170179193Sjb ASSERT(!probe->ftp_enabled); 1171179193Sjb ASSERT(id == probe->ftp_id); 1172211738Srpaulo#if defined(sun) 1173179193Sjb ASSERT(MUTEX_HELD(&cpu_lock)); 1174211738Srpaulo#endif 1175179193Sjb 1176179193Sjb /* 1177179193Sjb * Increment the count of enabled probes on this probe's provider; 1178179193Sjb * the provider can't go away while the probe still exists. We 1179179193Sjb * must increment this even if we aren't able to properly enable 1180179193Sjb * this probe. 1181179193Sjb */ 1182179193Sjb mutex_enter(&probe->ftp_prov->ftp_mtx); 1183179193Sjb probe->ftp_prov->ftp_rcount++; 1184179193Sjb mutex_exit(&probe->ftp_prov->ftp_mtx); 1185179193Sjb 1186179193Sjb /* 1187179193Sjb * If this probe's provider is retired (meaning it was valid in a 1188179193Sjb * previously exec'ed incarnation of this address space), bail out. The 1189179193Sjb * provider can't go away while we're in this code path. 1190179193Sjb */ 1191179193Sjb if (probe->ftp_prov->ftp_retired) 1192179193Sjb return; 1193179193Sjb 1194179193Sjb /* 1195179193Sjb * If we can't find the process, it may be that we're in the context of 1196179193Sjb * a fork in which the traced process is being born and we're copying 1197179193Sjb * USDT probes. Otherwise, the process is gone so bail. 1198179193Sjb */ 1199211738Srpaulo#if defined(sun) 1200179193Sjb if ((p = sprlock(probe->ftp_pid)) == NULL) { 1201179193Sjb if ((curproc->p_flag & SFORKING) == 0) 1202179193Sjb return; 1203179193Sjb 1204179193Sjb mutex_enter(&pidlock); 1205179193Sjb p = prfind(probe->ftp_pid); 1206179193Sjb 1207179193Sjb /* 1208179193Sjb * Confirm that curproc is indeed forking the process in which 1209179193Sjb * we're trying to enable probes. 1210179193Sjb */ 1211179193Sjb ASSERT(p != NULL); 1212179193Sjb ASSERT(p->p_parent == curproc); 1213179193Sjb ASSERT(p->p_stat == SIDL); 1214179193Sjb 1215179193Sjb mutex_enter(&p->p_lock); 1216179193Sjb mutex_exit(&pidlock); 1217179193Sjb 1218179193Sjb sprlock_proc(p); 1219179193Sjb } 1220179193Sjb 1221179193Sjb ASSERT(!(p->p_flag & SVFORK)); 1222179193Sjb mutex_exit(&p->p_lock); 1223211738Srpaulo#else 1224211738Srpaulo if ((p = pfind(probe->ftp_pid)) == NULL) 1225211738Srpaulo return; 1226211738Srpaulo#endif 1227179193Sjb 1228179193Sjb /* 1229179193Sjb * We have to enable the trap entry point before any user threads have 1230179193Sjb * the chance to execute the trap instruction we're about to place 1231179193Sjb * in their process's text. 1232179193Sjb */ 1233212494Srpaulo#ifdef __FreeBSD__ 1234212494Srpaulo /* 1235212494Srpaulo * pfind() returns a locked process. 1236212494Srpaulo */ 1237212494Srpaulo _PHOLD(p); 1238211738Srpaulo PROC_UNLOCK(p); 1239212494Srpaulo#endif 1240179193Sjb fasttrap_enable_callbacks(); 1241179193Sjb 1242179193Sjb /* 1243179193Sjb * Enable all the tracepoints and add this probe's id to each 1244179193Sjb * tracepoint's list of active probes. 1245179193Sjb */ 1246179193Sjb for (i = 0; i < probe->ftp_ntps; i++) { 1247179193Sjb if ((rc = fasttrap_tracepoint_enable(p, probe, i)) != 0) { 1248179193Sjb /* 1249179193Sjb * If enabling the tracepoint failed completely, 1250179193Sjb * we don't have to disable it; if the failure 1251179193Sjb * was only partial we must disable it. 1252179193Sjb */ 1253179193Sjb if (rc == FASTTRAP_ENABLE_FAIL) 1254179193Sjb i--; 1255179193Sjb else 1256179193Sjb ASSERT(rc == FASTTRAP_ENABLE_PARTIAL); 1257179193Sjb 1258179193Sjb /* 1259179193Sjb * Back up and pull out all the tracepoints we've 1260179193Sjb * created so far for this probe. 1261179193Sjb */ 1262179193Sjb while (i >= 0) { 1263179193Sjb fasttrap_tracepoint_disable(p, probe, i); 1264179193Sjb i--; 1265179193Sjb } 1266179193Sjb 1267211738Srpaulo#if defined(sun) 1268179193Sjb mutex_enter(&p->p_lock); 1269179193Sjb sprunlock(p); 1270211738Srpaulo#else 1271212494Srpaulo PRELE(p); 1272211738Srpaulo#endif 1273179193Sjb 1274179193Sjb /* 1275179193Sjb * Since we're not actually enabling this probe, 1276179193Sjb * drop our reference on the trap table entry. 1277179193Sjb */ 1278179193Sjb fasttrap_disable_callbacks(); 1279179193Sjb return; 1280179193Sjb } 1281179193Sjb } 1282211738Srpaulo#if defined(sun) 1283179193Sjb mutex_enter(&p->p_lock); 1284179193Sjb sprunlock(p); 1285211738Srpaulo#else 1286212494Srpaulo PRELE(p); 1287211738Srpaulo#endif 1288179193Sjb 1289179193Sjb probe->ftp_enabled = 1; 1290179193Sjb} 1291179193Sjb 1292179193Sjb/*ARGSUSED*/ 1293179193Sjbstatic void 1294179193Sjbfasttrap_pid_disable(void *arg, dtrace_id_t id, void *parg) 1295179193Sjb{ 1296179193Sjb fasttrap_probe_t *probe = parg; 1297179193Sjb fasttrap_provider_t *provider = probe->ftp_prov; 1298179193Sjb proc_t *p; 1299179193Sjb int i, whack = 0; 1300179193Sjb 1301179193Sjb ASSERT(id == probe->ftp_id); 1302179193Sjb 1303211738Srpaulo mutex_enter(&provider->ftp_mtx); 1304211738Srpaulo 1305179193Sjb /* 1306179193Sjb * We won't be able to acquire a /proc-esque lock on the process 1307179193Sjb * iff the process is dead and gone. In this case, we rely on the 1308179193Sjb * provider lock as a point of mutual exclusion to prevent other 1309179193Sjb * DTrace consumers from disabling this probe. 1310179193Sjb */ 1311247049Sgibbs if ((p = pfind(probe->ftp_pid)) != NULL) { 1312212494Srpaulo#ifdef __FreeBSD__ 1313283677Smarkj if (p->p_flag & P_WEXIT) { 1314283677Smarkj PROC_UNLOCK(p); 1315283677Smarkj p = NULL; 1316283677Smarkj } else { 1317283677Smarkj _PHOLD(p); 1318283677Smarkj PROC_UNLOCK(p); 1319283677Smarkj } 1320212494Srpaulo#endif 1321247049Sgibbs } 1322179193Sjb 1323179193Sjb /* 1324179193Sjb * Disable all the associated tracepoints (for fully enabled probes). 1325179193Sjb */ 1326179193Sjb if (probe->ftp_enabled) { 1327179193Sjb for (i = 0; i < probe->ftp_ntps; i++) { 1328179193Sjb fasttrap_tracepoint_disable(p, probe, i); 1329179193Sjb } 1330179193Sjb } 1331179193Sjb 1332179193Sjb ASSERT(provider->ftp_rcount > 0); 1333179193Sjb provider->ftp_rcount--; 1334179193Sjb 1335179193Sjb if (p != NULL) { 1336179193Sjb /* 1337179193Sjb * Even though we may not be able to remove it entirely, we 1338179193Sjb * mark this retired provider to get a chance to remove some 1339179193Sjb * of the associated probes. 1340179193Sjb */ 1341179193Sjb if (provider->ftp_retired && !provider->ftp_marked) 1342179193Sjb whack = provider->ftp_marked = 1; 1343179193Sjb mutex_exit(&provider->ftp_mtx); 1344179193Sjb } else { 1345179193Sjb /* 1346179193Sjb * If the process is dead, we're just waiting for the 1347179193Sjb * last probe to be disabled to be able to free it. 1348179193Sjb */ 1349179193Sjb if (provider->ftp_rcount == 0 && !provider->ftp_marked) 1350179193Sjb whack = provider->ftp_marked = 1; 1351179193Sjb mutex_exit(&provider->ftp_mtx); 1352179193Sjb } 1353179193Sjb 1354179193Sjb if (whack) 1355179193Sjb fasttrap_pid_cleanup(); 1356179193Sjb 1357212494Srpaulo#ifdef __FreeBSD__ 1358247049Sgibbs if (p != NULL) 1359247049Sgibbs PRELE(p); 1360212494Srpaulo#endif 1361179193Sjb if (!probe->ftp_enabled) 1362179193Sjb return; 1363179193Sjb 1364179193Sjb probe->ftp_enabled = 0; 1365179193Sjb 1366211738Srpaulo#if defined(sun) 1367179193Sjb ASSERT(MUTEX_HELD(&cpu_lock)); 1368211738Srpaulo#endif 1369179193Sjb fasttrap_disable_callbacks(); 1370179193Sjb} 1371179193Sjb 1372179193Sjb/*ARGSUSED*/ 1373179193Sjbstatic void 1374179193Sjbfasttrap_pid_getargdesc(void *arg, dtrace_id_t id, void *parg, 1375179193Sjb dtrace_argdesc_t *desc) 1376179193Sjb{ 1377179193Sjb fasttrap_probe_t *probe = parg; 1378179193Sjb char *str; 1379179193Sjb int i, ndx; 1380179193Sjb 1381179193Sjb desc->dtargd_native[0] = '\0'; 1382179193Sjb desc->dtargd_xlate[0] = '\0'; 1383179193Sjb 1384179193Sjb if (probe->ftp_prov->ftp_retired != 0 || 1385179193Sjb desc->dtargd_ndx >= probe->ftp_nargs) { 1386179193Sjb desc->dtargd_ndx = DTRACE_ARGNONE; 1387179193Sjb return; 1388179193Sjb } 1389179193Sjb 1390179193Sjb ndx = (probe->ftp_argmap != NULL) ? 1391179193Sjb probe->ftp_argmap[desc->dtargd_ndx] : desc->dtargd_ndx; 1392179193Sjb 1393179193Sjb str = probe->ftp_ntypes; 1394179193Sjb for (i = 0; i < ndx; i++) { 1395179193Sjb str += strlen(str) + 1; 1396179193Sjb } 1397179193Sjb 1398179193Sjb ASSERT(strlen(str + 1) < sizeof (desc->dtargd_native)); 1399179193Sjb (void) strcpy(desc->dtargd_native, str); 1400179193Sjb 1401179193Sjb if (probe->ftp_xtypes == NULL) 1402179193Sjb return; 1403179193Sjb 1404179193Sjb str = probe->ftp_xtypes; 1405179193Sjb for (i = 0; i < desc->dtargd_ndx; i++) { 1406179193Sjb str += strlen(str) + 1; 1407179193Sjb } 1408179193Sjb 1409179193Sjb ASSERT(strlen(str + 1) < sizeof (desc->dtargd_xlate)); 1410179193Sjb (void) strcpy(desc->dtargd_xlate, str); 1411179193Sjb} 1412179193Sjb 1413179193Sjb/*ARGSUSED*/ 1414179193Sjbstatic void 1415179193Sjbfasttrap_pid_destroy(void *arg, dtrace_id_t id, void *parg) 1416179193Sjb{ 1417179193Sjb fasttrap_probe_t *probe = parg; 1418179193Sjb int i; 1419179193Sjb size_t size; 1420179193Sjb 1421179193Sjb ASSERT(probe != NULL); 1422179193Sjb ASSERT(!probe->ftp_enabled); 1423179193Sjb ASSERT(fasttrap_total >= probe->ftp_ntps); 1424179193Sjb 1425179193Sjb atomic_add_32(&fasttrap_total, -probe->ftp_ntps); 1426179193Sjb size = offsetof(fasttrap_probe_t, ftp_tps[probe->ftp_ntps]); 1427179193Sjb 1428179193Sjb if (probe->ftp_gen + 1 >= fasttrap_mod_gen) 1429179193Sjb fasttrap_mod_barrier(probe->ftp_gen); 1430179193Sjb 1431179193Sjb for (i = 0; i < probe->ftp_ntps; i++) { 1432179193Sjb kmem_free(probe->ftp_tps[i].fit_tp, 1433179193Sjb sizeof (fasttrap_tracepoint_t)); 1434179193Sjb } 1435179193Sjb 1436179193Sjb kmem_free(probe, size); 1437179193Sjb} 1438179193Sjb 1439179193Sjb 1440179193Sjbstatic const dtrace_pattr_t pid_attr = { 1441179193Sjb{ DTRACE_STABILITY_EVOLVING, DTRACE_STABILITY_EVOLVING, DTRACE_CLASS_ISA }, 1442179193Sjb{ DTRACE_STABILITY_PRIVATE, DTRACE_STABILITY_PRIVATE, DTRACE_CLASS_UNKNOWN }, 1443179193Sjb{ DTRACE_STABILITY_PRIVATE, DTRACE_STABILITY_PRIVATE, DTRACE_CLASS_UNKNOWN }, 1444179193Sjb{ DTRACE_STABILITY_EVOLVING, DTRACE_STABILITY_EVOLVING, DTRACE_CLASS_ISA }, 1445179193Sjb{ DTRACE_STABILITY_PRIVATE, DTRACE_STABILITY_PRIVATE, DTRACE_CLASS_UNKNOWN }, 1446179193Sjb}; 1447179193Sjb 1448179193Sjbstatic dtrace_pops_t pid_pops = { 1449179193Sjb fasttrap_pid_provide, 1450179193Sjb NULL, 1451179193Sjb fasttrap_pid_enable, 1452179193Sjb fasttrap_pid_disable, 1453179193Sjb NULL, 1454179193Sjb NULL, 1455179193Sjb fasttrap_pid_getargdesc, 1456179193Sjb fasttrap_pid_getarg, 1457179193Sjb NULL, 1458179193Sjb fasttrap_pid_destroy 1459179193Sjb}; 1460179193Sjb 1461179193Sjbstatic dtrace_pops_t usdt_pops = { 1462179193Sjb fasttrap_pid_provide, 1463179193Sjb NULL, 1464179193Sjb fasttrap_pid_enable, 1465179193Sjb fasttrap_pid_disable, 1466179193Sjb NULL, 1467179193Sjb NULL, 1468179193Sjb fasttrap_pid_getargdesc, 1469179193Sjb fasttrap_usdt_getarg, 1470179193Sjb NULL, 1471179193Sjb fasttrap_pid_destroy 1472179193Sjb}; 1473179193Sjb 1474179193Sjbstatic fasttrap_proc_t * 1475179193Sjbfasttrap_proc_lookup(pid_t pid) 1476179193Sjb{ 1477179193Sjb fasttrap_bucket_t *bucket; 1478179193Sjb fasttrap_proc_t *fprc, *new_fprc; 1479179193Sjb 1480211738Srpaulo 1481179193Sjb bucket = &fasttrap_procs.fth_table[FASTTRAP_PROCS_INDEX(pid)]; 1482179193Sjb mutex_enter(&bucket->ftb_mtx); 1483179193Sjb 1484179193Sjb for (fprc = bucket->ftb_data; fprc != NULL; fprc = fprc->ftpc_next) { 1485179193Sjb if (fprc->ftpc_pid == pid && fprc->ftpc_acount != 0) { 1486179193Sjb mutex_enter(&fprc->ftpc_mtx); 1487179193Sjb mutex_exit(&bucket->ftb_mtx); 1488179193Sjb fprc->ftpc_rcount++; 1489271001Sdelphij atomic_inc_64(&fprc->ftpc_acount); 1490179198Sjb ASSERT(fprc->ftpc_acount <= fprc->ftpc_rcount); 1491179193Sjb mutex_exit(&fprc->ftpc_mtx); 1492179193Sjb 1493179193Sjb return (fprc); 1494179193Sjb } 1495179193Sjb } 1496179193Sjb 1497179193Sjb /* 1498179193Sjb * Drop the bucket lock so we don't try to perform a sleeping 1499179193Sjb * allocation under it. 1500179193Sjb */ 1501179193Sjb mutex_exit(&bucket->ftb_mtx); 1502179193Sjb 1503179193Sjb new_fprc = kmem_zalloc(sizeof (fasttrap_proc_t), KM_SLEEP); 1504179193Sjb new_fprc->ftpc_pid = pid; 1505179193Sjb new_fprc->ftpc_rcount = 1; 1506179193Sjb new_fprc->ftpc_acount = 1; 1507211738Srpaulo#if !defined(sun) 1508211738Srpaulo mutex_init(&new_fprc->ftpc_mtx, "fasttrap proc mtx", MUTEX_DEFAULT, 1509211738Srpaulo NULL); 1510211738Srpaulo#endif 1511179193Sjb 1512179193Sjb mutex_enter(&bucket->ftb_mtx); 1513179193Sjb 1514179193Sjb /* 1515179193Sjb * Take another lap through the list to make sure a proc hasn't 1516179193Sjb * been created for this pid while we weren't under the bucket lock. 1517179193Sjb */ 1518179193Sjb for (fprc = bucket->ftb_data; fprc != NULL; fprc = fprc->ftpc_next) { 1519179193Sjb if (fprc->ftpc_pid == pid && fprc->ftpc_acount != 0) { 1520179193Sjb mutex_enter(&fprc->ftpc_mtx); 1521179193Sjb mutex_exit(&bucket->ftb_mtx); 1522179193Sjb fprc->ftpc_rcount++; 1523271001Sdelphij atomic_inc_64(&fprc->ftpc_acount); 1524179198Sjb ASSERT(fprc->ftpc_acount <= fprc->ftpc_rcount); 1525179193Sjb mutex_exit(&fprc->ftpc_mtx); 1526179193Sjb 1527179193Sjb kmem_free(new_fprc, sizeof (fasttrap_proc_t)); 1528179193Sjb 1529179193Sjb return (fprc); 1530179193Sjb } 1531179193Sjb } 1532179193Sjb 1533179193Sjb new_fprc->ftpc_next = bucket->ftb_data; 1534179193Sjb bucket->ftb_data = new_fprc; 1535179193Sjb 1536179193Sjb mutex_exit(&bucket->ftb_mtx); 1537179193Sjb 1538179193Sjb return (new_fprc); 1539179193Sjb} 1540179193Sjb 1541179193Sjbstatic void 1542179193Sjbfasttrap_proc_release(fasttrap_proc_t *proc) 1543179193Sjb{ 1544179193Sjb fasttrap_bucket_t *bucket; 1545179193Sjb fasttrap_proc_t *fprc, **fprcp; 1546179193Sjb pid_t pid = proc->ftpc_pid; 1547269342Smarkj#if !defined(sun) 1548269342Smarkj fasttrap_scrblock_t *scrblk, *scrblktmp; 1549269342Smarkj fasttrap_scrspace_t *scrspc, *scrspctmp; 1550269342Smarkj struct proc *p; 1551269342Smarkj struct thread *td; 1552269342Smarkj#endif 1553179193Sjb 1554179193Sjb mutex_enter(&proc->ftpc_mtx); 1555179193Sjb 1556179193Sjb ASSERT(proc->ftpc_rcount != 0); 1557179198Sjb ASSERT(proc->ftpc_acount <= proc->ftpc_rcount); 1558179193Sjb 1559179193Sjb if (--proc->ftpc_rcount != 0) { 1560179193Sjb mutex_exit(&proc->ftpc_mtx); 1561179193Sjb return; 1562179193Sjb } 1563179193Sjb 1564269342Smarkj#if !defined(sun) 1565269342Smarkj /* 1566269342Smarkj * Free all structures used to manage per-thread scratch space. 1567269342Smarkj */ 1568269342Smarkj LIST_FOREACH_SAFE(scrblk, &proc->ftpc_scrblks, ftsb_next, 1569269342Smarkj scrblktmp) { 1570269342Smarkj LIST_REMOVE(scrblk, ftsb_next); 1571269342Smarkj free(scrblk, M_SOLARIS); 1572269342Smarkj } 1573269342Smarkj LIST_FOREACH_SAFE(scrspc, &proc->ftpc_fscr, ftss_next, scrspctmp) { 1574269342Smarkj LIST_REMOVE(scrspc, ftss_next); 1575269342Smarkj free(scrspc, M_SOLARIS); 1576269342Smarkj } 1577269342Smarkj LIST_FOREACH_SAFE(scrspc, &proc->ftpc_ascr, ftss_next, scrspctmp) { 1578269342Smarkj LIST_REMOVE(scrspc, ftss_next); 1579269342Smarkj free(scrspc, M_SOLARIS); 1580269342Smarkj } 1581269342Smarkj 1582269342Smarkj if ((p = pfind(pid)) != NULL) { 1583269342Smarkj FOREACH_THREAD_IN_PROC(p, td) 1584269342Smarkj td->t_dtrace_sscr = NULL; 1585269342Smarkj PROC_UNLOCK(p); 1586269342Smarkj } 1587269342Smarkj#endif 1588269342Smarkj 1589179193Sjb mutex_exit(&proc->ftpc_mtx); 1590179193Sjb 1591179193Sjb /* 1592179193Sjb * There should definitely be no live providers associated with this 1593179193Sjb * process at this point. 1594179193Sjb */ 1595179193Sjb ASSERT(proc->ftpc_acount == 0); 1596179193Sjb 1597179193Sjb bucket = &fasttrap_procs.fth_table[FASTTRAP_PROCS_INDEX(pid)]; 1598179193Sjb mutex_enter(&bucket->ftb_mtx); 1599179193Sjb 1600179193Sjb fprcp = (fasttrap_proc_t **)&bucket->ftb_data; 1601179193Sjb while ((fprc = *fprcp) != NULL) { 1602179193Sjb if (fprc == proc) 1603179193Sjb break; 1604179193Sjb 1605179193Sjb fprcp = &fprc->ftpc_next; 1606179193Sjb } 1607179193Sjb 1608179193Sjb /* 1609179193Sjb * Something strange has happened if we can't find the proc. 1610179193Sjb */ 1611179193Sjb ASSERT(fprc != NULL); 1612179193Sjb 1613179193Sjb *fprcp = fprc->ftpc_next; 1614179193Sjb 1615179193Sjb mutex_exit(&bucket->ftb_mtx); 1616179193Sjb 1617179193Sjb kmem_free(fprc, sizeof (fasttrap_proc_t)); 1618179193Sjb} 1619179193Sjb 1620179193Sjb/* 1621179193Sjb * Lookup a fasttrap-managed provider based on its name and associated pid. 1622179193Sjb * If the pattr argument is non-NULL, this function instantiates the provider 1623179193Sjb * if it doesn't exist otherwise it returns NULL. The provider is returned 1624179193Sjb * with its lock held. 1625179193Sjb */ 1626179193Sjbstatic fasttrap_provider_t * 1627179193Sjbfasttrap_provider_lookup(pid_t pid, const char *name, 1628179193Sjb const dtrace_pattr_t *pattr) 1629179193Sjb{ 1630179193Sjb fasttrap_provider_t *fp, *new_fp = NULL; 1631179193Sjb fasttrap_bucket_t *bucket; 1632179193Sjb char provname[DTRACE_PROVNAMELEN]; 1633179193Sjb proc_t *p; 1634179193Sjb cred_t *cred; 1635179193Sjb 1636179193Sjb ASSERT(strlen(name) < sizeof (fp->ftp_name)); 1637179193Sjb ASSERT(pattr != NULL); 1638179193Sjb 1639179193Sjb bucket = &fasttrap_provs.fth_table[FASTTRAP_PROVS_INDEX(pid, name)]; 1640179193Sjb mutex_enter(&bucket->ftb_mtx); 1641179193Sjb 1642179193Sjb /* 1643179193Sjb * Take a lap through the list and return the match if we find it. 1644179193Sjb */ 1645179193Sjb for (fp = bucket->ftb_data; fp != NULL; fp = fp->ftp_next) { 1646179193Sjb if (fp->ftp_pid == pid && strcmp(fp->ftp_name, name) == 0 && 1647179193Sjb !fp->ftp_retired) { 1648179193Sjb mutex_enter(&fp->ftp_mtx); 1649179193Sjb mutex_exit(&bucket->ftb_mtx); 1650179193Sjb return (fp); 1651179193Sjb } 1652179193Sjb } 1653179193Sjb 1654179193Sjb /* 1655179193Sjb * Drop the bucket lock so we don't try to perform a sleeping 1656179193Sjb * allocation under it. 1657179193Sjb */ 1658179193Sjb mutex_exit(&bucket->ftb_mtx); 1659179193Sjb 1660179193Sjb /* 1661179193Sjb * Make sure the process exists, isn't a child created as the result 1662179193Sjb * of a vfork(2), and isn't a zombie (but may be in fork). 1663179193Sjb */ 1664211738Srpaulo if ((p = pfind(pid)) == NULL) 1665179193Sjb return (NULL); 1666179193Sjb 1667179193Sjb /* 1668179193Sjb * Increment p_dtrace_probes so that the process knows to inform us 1669179193Sjb * when it exits or execs. fasttrap_provider_free() decrements this 1670179193Sjb * when we're done with this provider. 1671179193Sjb */ 1672179193Sjb p->p_dtrace_probes++; 1673179193Sjb 1674179193Sjb /* 1675179193Sjb * Grab the credentials for this process so we have 1676179193Sjb * something to pass to dtrace_register(). 1677179193Sjb */ 1678211738Srpaulo PROC_LOCK_ASSERT(p, MA_OWNED); 1679211738Srpaulo crhold(p->p_ucred); 1680211738Srpaulo cred = p->p_ucred; 1681211738Srpaulo PROC_UNLOCK(p); 1682179193Sjb 1683179193Sjb new_fp = kmem_zalloc(sizeof (fasttrap_provider_t), KM_SLEEP); 1684179193Sjb new_fp->ftp_pid = pid; 1685179193Sjb new_fp->ftp_proc = fasttrap_proc_lookup(pid); 1686211738Srpaulo#if !defined(sun) 1687211738Srpaulo mutex_init(&new_fp->ftp_mtx, "provider mtx", MUTEX_DEFAULT, NULL); 1688211738Srpaulo mutex_init(&new_fp->ftp_cmtx, "lock on creating", MUTEX_DEFAULT, NULL); 1689211738Srpaulo#endif 1690179193Sjb 1691179193Sjb ASSERT(new_fp->ftp_proc != NULL); 1692179193Sjb 1693179193Sjb mutex_enter(&bucket->ftb_mtx); 1694179193Sjb 1695179193Sjb /* 1696179193Sjb * Take another lap through the list to make sure a provider hasn't 1697179193Sjb * been created for this pid while we weren't under the bucket lock. 1698179193Sjb */ 1699179193Sjb for (fp = bucket->ftb_data; fp != NULL; fp = fp->ftp_next) { 1700179193Sjb if (fp->ftp_pid == pid && strcmp(fp->ftp_name, name) == 0 && 1701179193Sjb !fp->ftp_retired) { 1702179193Sjb mutex_enter(&fp->ftp_mtx); 1703179193Sjb mutex_exit(&bucket->ftb_mtx); 1704179193Sjb fasttrap_provider_free(new_fp); 1705179193Sjb crfree(cred); 1706179193Sjb return (fp); 1707179193Sjb } 1708179193Sjb } 1709179193Sjb 1710179193Sjb (void) strcpy(new_fp->ftp_name, name); 1711179193Sjb 1712179193Sjb /* 1713179193Sjb * Fail and return NULL if either the provider name is too long 1714179193Sjb * or we fail to register this new provider with the DTrace 1715179193Sjb * framework. Note that this is the only place we ever construct 1716179193Sjb * the full provider name -- we keep it in pieces in the provider 1717179193Sjb * structure. 1718179193Sjb */ 1719179193Sjb if (snprintf(provname, sizeof (provname), "%s%u", name, (uint_t)pid) >= 1720179193Sjb sizeof (provname) || 1721179193Sjb dtrace_register(provname, pattr, 1722179193Sjb DTRACE_PRIV_PROC | DTRACE_PRIV_OWNER | DTRACE_PRIV_ZONEOWNER, cred, 1723179193Sjb pattr == &pid_attr ? &pid_pops : &usdt_pops, new_fp, 1724179193Sjb &new_fp->ftp_provid) != 0) { 1725179193Sjb mutex_exit(&bucket->ftb_mtx); 1726179193Sjb fasttrap_provider_free(new_fp); 1727179193Sjb crfree(cred); 1728179193Sjb return (NULL); 1729179193Sjb } 1730179193Sjb 1731179193Sjb new_fp->ftp_next = bucket->ftb_data; 1732179193Sjb bucket->ftb_data = new_fp; 1733179193Sjb 1734179193Sjb mutex_enter(&new_fp->ftp_mtx); 1735179193Sjb mutex_exit(&bucket->ftb_mtx); 1736179193Sjb 1737179193Sjb crfree(cred); 1738179193Sjb return (new_fp); 1739179193Sjb} 1740179193Sjb 1741179193Sjbstatic void 1742179193Sjbfasttrap_provider_free(fasttrap_provider_t *provider) 1743179193Sjb{ 1744179193Sjb pid_t pid = provider->ftp_pid; 1745179193Sjb proc_t *p; 1746179193Sjb 1747179193Sjb /* 1748179193Sjb * There need to be no associated enabled probes, no consumers 1749179193Sjb * creating probes, and no meta providers referencing this provider. 1750179193Sjb */ 1751179193Sjb ASSERT(provider->ftp_rcount == 0); 1752179193Sjb ASSERT(provider->ftp_ccount == 0); 1753179193Sjb ASSERT(provider->ftp_mcount == 0); 1754179193Sjb 1755179198Sjb /* 1756179198Sjb * If this provider hasn't been retired, we need to explicitly drop the 1757179198Sjb * count of active providers on the associated process structure. 1758179198Sjb */ 1759179198Sjb if (!provider->ftp_retired) { 1760271001Sdelphij atomic_dec_64(&provider->ftp_proc->ftpc_acount); 1761179198Sjb ASSERT(provider->ftp_proc->ftpc_acount < 1762179198Sjb provider->ftp_proc->ftpc_rcount); 1763179198Sjb } 1764179198Sjb 1765179193Sjb fasttrap_proc_release(provider->ftp_proc); 1766179193Sjb 1767211738Srpaulo#if !defined(sun) 1768211738Srpaulo mutex_destroy(&provider->ftp_mtx); 1769211738Srpaulo mutex_destroy(&provider->ftp_cmtx); 1770211738Srpaulo#endif 1771179193Sjb kmem_free(provider, sizeof (fasttrap_provider_t)); 1772179193Sjb 1773179193Sjb /* 1774179193Sjb * Decrement p_dtrace_probes on the process whose provider we're 1775179193Sjb * freeing. We don't have to worry about clobbering somone else's 1776179193Sjb * modifications to it because we have locked the bucket that 1777179193Sjb * corresponds to this process's hash chain in the provider hash 1778179193Sjb * table. Don't sweat it if we can't find the process. 1779179193Sjb */ 1780211738Srpaulo if ((p = pfind(pid)) == NULL) { 1781179193Sjb return; 1782179193Sjb } 1783179193Sjb 1784179193Sjb p->p_dtrace_probes--; 1785211738Srpaulo#if !defined(sun) 1786211738Srpaulo PROC_UNLOCK(p); 1787211738Srpaulo#endif 1788179193Sjb} 1789179193Sjb 1790179193Sjbstatic void 1791179193Sjbfasttrap_provider_retire(pid_t pid, const char *name, int mprov) 1792179193Sjb{ 1793179193Sjb fasttrap_provider_t *fp; 1794179193Sjb fasttrap_bucket_t *bucket; 1795179193Sjb dtrace_provider_id_t provid; 1796179193Sjb 1797179193Sjb ASSERT(strlen(name) < sizeof (fp->ftp_name)); 1798179193Sjb 1799179193Sjb bucket = &fasttrap_provs.fth_table[FASTTRAP_PROVS_INDEX(pid, name)]; 1800179193Sjb mutex_enter(&bucket->ftb_mtx); 1801179193Sjb 1802179193Sjb for (fp = bucket->ftb_data; fp != NULL; fp = fp->ftp_next) { 1803179193Sjb if (fp->ftp_pid == pid && strcmp(fp->ftp_name, name) == 0 && 1804179193Sjb !fp->ftp_retired) 1805179193Sjb break; 1806179193Sjb } 1807179193Sjb 1808179193Sjb if (fp == NULL) { 1809179193Sjb mutex_exit(&bucket->ftb_mtx); 1810179193Sjb return; 1811179193Sjb } 1812179193Sjb 1813179193Sjb mutex_enter(&fp->ftp_mtx); 1814179193Sjb ASSERT(!mprov || fp->ftp_mcount > 0); 1815179193Sjb if (mprov && --fp->ftp_mcount != 0) { 1816179193Sjb mutex_exit(&fp->ftp_mtx); 1817179193Sjb mutex_exit(&bucket->ftb_mtx); 1818179193Sjb return; 1819179193Sjb } 1820179193Sjb 1821179193Sjb /* 1822179193Sjb * Mark the provider to be removed in our post-processing step, mark it 1823179193Sjb * retired, and drop the active count on its proc. Marking it indicates 1824179193Sjb * that we should try to remove it; setting the retired flag indicates 1825179193Sjb * that we're done with this provider; dropping the active the proc 1826179193Sjb * releases our hold, and when this reaches zero (as it will during 1827179193Sjb * exit or exec) the proc and associated providers become defunct. 1828179193Sjb * 1829179193Sjb * We obviously need to take the bucket lock before the provider lock 1830179193Sjb * to perform the lookup, but we need to drop the provider lock 1831179193Sjb * before calling into the DTrace framework since we acquire the 1832179193Sjb * provider lock in callbacks invoked from the DTrace framework. The 1833179193Sjb * bucket lock therefore protects the integrity of the provider hash 1834179193Sjb * table. 1835179193Sjb */ 1836271001Sdelphij atomic_dec_64(&fp->ftp_proc->ftpc_acount); 1837179198Sjb ASSERT(fp->ftp_proc->ftpc_acount < fp->ftp_proc->ftpc_rcount); 1838179198Sjb 1839179193Sjb fp->ftp_retired = 1; 1840179193Sjb fp->ftp_marked = 1; 1841179193Sjb provid = fp->ftp_provid; 1842179193Sjb mutex_exit(&fp->ftp_mtx); 1843179193Sjb 1844179193Sjb /* 1845179193Sjb * We don't have to worry about invalidating the same provider twice 1846179193Sjb * since fasttrap_provider_lookup() will ignore provider that have 1847179193Sjb * been marked as retired. 1848179193Sjb */ 1849179193Sjb dtrace_invalidate(provid); 1850179193Sjb 1851179193Sjb mutex_exit(&bucket->ftb_mtx); 1852179193Sjb 1853179193Sjb fasttrap_pid_cleanup(); 1854179193Sjb} 1855179193Sjb 1856179193Sjbstatic int 1857179193Sjbfasttrap_uint32_cmp(const void *ap, const void *bp) 1858179193Sjb{ 1859179193Sjb return (*(const uint32_t *)ap - *(const uint32_t *)bp); 1860179193Sjb} 1861179193Sjb 1862179193Sjbstatic int 1863179193Sjbfasttrap_uint64_cmp(const void *ap, const void *bp) 1864179193Sjb{ 1865179193Sjb return (*(const uint64_t *)ap - *(const uint64_t *)bp); 1866179193Sjb} 1867179193Sjb 1868179193Sjbstatic int 1869179193Sjbfasttrap_add_probe(fasttrap_probe_spec_t *pdata) 1870179193Sjb{ 1871179193Sjb fasttrap_provider_t *provider; 1872179193Sjb fasttrap_probe_t *pp; 1873179193Sjb fasttrap_tracepoint_t *tp; 1874179193Sjb char *name; 1875211738Srpaulo int i, aframes = 0, whack; 1876179193Sjb 1877179193Sjb /* 1878179193Sjb * There needs to be at least one desired trace point. 1879179193Sjb */ 1880179193Sjb if (pdata->ftps_noffs == 0) 1881179193Sjb return (EINVAL); 1882179193Sjb 1883179193Sjb switch (pdata->ftps_type) { 1884179193Sjb case DTFTP_ENTRY: 1885179193Sjb name = "entry"; 1886179193Sjb aframes = FASTTRAP_ENTRY_AFRAMES; 1887179193Sjb break; 1888179193Sjb case DTFTP_RETURN: 1889179193Sjb name = "return"; 1890179193Sjb aframes = FASTTRAP_RETURN_AFRAMES; 1891179193Sjb break; 1892179193Sjb case DTFTP_OFFSETS: 1893179193Sjb name = NULL; 1894179193Sjb break; 1895179193Sjb default: 1896179193Sjb return (EINVAL); 1897179193Sjb } 1898179193Sjb 1899179193Sjb if ((provider = fasttrap_provider_lookup(pdata->ftps_pid, 1900179193Sjb FASTTRAP_PID_NAME, &pid_attr)) == NULL) 1901179193Sjb return (ESRCH); 1902179193Sjb 1903179193Sjb /* 1904179193Sjb * Increment this reference count to indicate that a consumer is 1905179193Sjb * actively adding a new probe associated with this provider. This 1906179193Sjb * prevents the provider from being deleted -- we'll need to check 1907179193Sjb * for pending deletions when we drop this reference count. 1908179193Sjb */ 1909179193Sjb provider->ftp_ccount++; 1910179193Sjb mutex_exit(&provider->ftp_mtx); 1911179193Sjb 1912179193Sjb /* 1913179193Sjb * Grab the creation lock to ensure consistency between calls to 1914179193Sjb * dtrace_probe_lookup() and dtrace_probe_create() in the face of 1915179193Sjb * other threads creating probes. We must drop the provider lock 1916179193Sjb * before taking this lock to avoid a three-way deadlock with the 1917179193Sjb * DTrace framework. 1918179193Sjb */ 1919179193Sjb mutex_enter(&provider->ftp_cmtx); 1920179193Sjb 1921179193Sjb if (name == NULL) { 1922179193Sjb for (i = 0; i < pdata->ftps_noffs; i++) { 1923179193Sjb char name_str[17]; 1924179193Sjb 1925179193Sjb (void) sprintf(name_str, "%llx", 1926179193Sjb (unsigned long long)pdata->ftps_offs[i]); 1927179193Sjb 1928179193Sjb if (dtrace_probe_lookup(provider->ftp_provid, 1929179193Sjb pdata->ftps_mod, pdata->ftps_func, name_str) != 0) 1930179193Sjb continue; 1931179193Sjb 1932271001Sdelphij atomic_inc_32(&fasttrap_total); 1933179193Sjb 1934179193Sjb if (fasttrap_total > fasttrap_max) { 1935271001Sdelphij atomic_dec_32(&fasttrap_total); 1936179193Sjb goto no_mem; 1937179193Sjb } 1938179193Sjb 1939179193Sjb pp = kmem_zalloc(sizeof (fasttrap_probe_t), KM_SLEEP); 1940179193Sjb 1941179193Sjb pp->ftp_prov = provider; 1942179193Sjb pp->ftp_faddr = pdata->ftps_pc; 1943179193Sjb pp->ftp_fsize = pdata->ftps_size; 1944179193Sjb pp->ftp_pid = pdata->ftps_pid; 1945179193Sjb pp->ftp_ntps = 1; 1946179193Sjb 1947179193Sjb tp = kmem_zalloc(sizeof (fasttrap_tracepoint_t), 1948179193Sjb KM_SLEEP); 1949179193Sjb 1950179193Sjb tp->ftt_proc = provider->ftp_proc; 1951179193Sjb tp->ftt_pc = pdata->ftps_offs[i] + pdata->ftps_pc; 1952179193Sjb tp->ftt_pid = pdata->ftps_pid; 1953179193Sjb 1954179193Sjb pp->ftp_tps[0].fit_tp = tp; 1955179193Sjb pp->ftp_tps[0].fit_id.fti_probe = pp; 1956179193Sjb pp->ftp_tps[0].fit_id.fti_ptype = pdata->ftps_type; 1957179193Sjb 1958179193Sjb pp->ftp_id = dtrace_probe_create(provider->ftp_provid, 1959179193Sjb pdata->ftps_mod, pdata->ftps_func, name_str, 1960179193Sjb FASTTRAP_OFFSET_AFRAMES, pp); 1961179193Sjb } 1962179193Sjb 1963179193Sjb } else if (dtrace_probe_lookup(provider->ftp_provid, pdata->ftps_mod, 1964179193Sjb pdata->ftps_func, name) == 0) { 1965179193Sjb atomic_add_32(&fasttrap_total, pdata->ftps_noffs); 1966179193Sjb 1967179193Sjb if (fasttrap_total > fasttrap_max) { 1968179193Sjb atomic_add_32(&fasttrap_total, -pdata->ftps_noffs); 1969179193Sjb goto no_mem; 1970179193Sjb } 1971179193Sjb 1972179193Sjb /* 1973179193Sjb * Make sure all tracepoint program counter values are unique. 1974179193Sjb * We later assume that each probe has exactly one tracepoint 1975179193Sjb * for a given pc. 1976179193Sjb */ 1977179193Sjb qsort(pdata->ftps_offs, pdata->ftps_noffs, 1978179193Sjb sizeof (uint64_t), fasttrap_uint64_cmp); 1979179193Sjb for (i = 1; i < pdata->ftps_noffs; i++) { 1980179193Sjb if (pdata->ftps_offs[i] > pdata->ftps_offs[i - 1]) 1981179193Sjb continue; 1982179193Sjb 1983179193Sjb atomic_add_32(&fasttrap_total, -pdata->ftps_noffs); 1984179193Sjb goto no_mem; 1985179193Sjb } 1986179193Sjb 1987179193Sjb ASSERT(pdata->ftps_noffs > 0); 1988179193Sjb pp = kmem_zalloc(offsetof(fasttrap_probe_t, 1989179193Sjb ftp_tps[pdata->ftps_noffs]), KM_SLEEP); 1990179193Sjb 1991179193Sjb pp->ftp_prov = provider; 1992179193Sjb pp->ftp_faddr = pdata->ftps_pc; 1993179193Sjb pp->ftp_fsize = pdata->ftps_size; 1994179193Sjb pp->ftp_pid = pdata->ftps_pid; 1995179193Sjb pp->ftp_ntps = pdata->ftps_noffs; 1996179193Sjb 1997179193Sjb for (i = 0; i < pdata->ftps_noffs; i++) { 1998179193Sjb tp = kmem_zalloc(sizeof (fasttrap_tracepoint_t), 1999179193Sjb KM_SLEEP); 2000179193Sjb 2001179193Sjb tp->ftt_proc = provider->ftp_proc; 2002179193Sjb tp->ftt_pc = pdata->ftps_offs[i] + pdata->ftps_pc; 2003179193Sjb tp->ftt_pid = pdata->ftps_pid; 2004179193Sjb 2005179193Sjb pp->ftp_tps[i].fit_tp = tp; 2006179193Sjb pp->ftp_tps[i].fit_id.fti_probe = pp; 2007179193Sjb pp->ftp_tps[i].fit_id.fti_ptype = pdata->ftps_type; 2008179193Sjb } 2009179193Sjb 2010179193Sjb pp->ftp_id = dtrace_probe_create(provider->ftp_provid, 2011179193Sjb pdata->ftps_mod, pdata->ftps_func, name, aframes, pp); 2012179193Sjb } 2013179193Sjb 2014179193Sjb mutex_exit(&provider->ftp_cmtx); 2015179193Sjb 2016179193Sjb /* 2017179193Sjb * We know that the provider is still valid since we incremented the 2018179193Sjb * creation reference count. If someone tried to clean up this provider 2019179193Sjb * while we were using it (e.g. because the process called exec(2) or 2020179193Sjb * exit(2)), take note of that and try to clean it up now. 2021179193Sjb */ 2022179193Sjb mutex_enter(&provider->ftp_mtx); 2023179193Sjb provider->ftp_ccount--; 2024179193Sjb whack = provider->ftp_retired; 2025179193Sjb mutex_exit(&provider->ftp_mtx); 2026179193Sjb 2027179193Sjb if (whack) 2028179193Sjb fasttrap_pid_cleanup(); 2029179193Sjb 2030179193Sjb return (0); 2031179193Sjb 2032179193Sjbno_mem: 2033179193Sjb /* 2034179193Sjb * If we've exhausted the allowable resources, we'll try to remove 2035179193Sjb * this provider to free some up. This is to cover the case where 2036179193Sjb * the user has accidentally created many more probes than was 2037179193Sjb * intended (e.g. pid123:::). 2038179193Sjb */ 2039179193Sjb mutex_exit(&provider->ftp_cmtx); 2040179193Sjb mutex_enter(&provider->ftp_mtx); 2041179193Sjb provider->ftp_ccount--; 2042179193Sjb provider->ftp_marked = 1; 2043179193Sjb mutex_exit(&provider->ftp_mtx); 2044179193Sjb 2045179193Sjb fasttrap_pid_cleanup(); 2046179193Sjb 2047179193Sjb return (ENOMEM); 2048179193Sjb} 2049179193Sjb 2050179193Sjb/*ARGSUSED*/ 2051179193Sjbstatic void * 2052179193Sjbfasttrap_meta_provide(void *arg, dtrace_helper_provdesc_t *dhpv, pid_t pid) 2053179193Sjb{ 2054179193Sjb fasttrap_provider_t *provider; 2055179193Sjb 2056179193Sjb /* 2057179193Sjb * A 32-bit unsigned integer (like a pid for example) can be 2058179193Sjb * expressed in 10 or fewer decimal digits. Make sure that we'll 2059179193Sjb * have enough space for the provider name. 2060179193Sjb */ 2061179193Sjb if (strlen(dhpv->dthpv_provname) + 10 >= 2062179193Sjb sizeof (provider->ftp_name)) { 2063211738Srpaulo printf("failed to instantiate provider %s: " 2064179193Sjb "name too long to accomodate pid", dhpv->dthpv_provname); 2065179193Sjb return (NULL); 2066179193Sjb } 2067179193Sjb 2068179193Sjb /* 2069179193Sjb * Don't let folks spoof the true pid provider. 2070179193Sjb */ 2071179193Sjb if (strcmp(dhpv->dthpv_provname, FASTTRAP_PID_NAME) == 0) { 2072211738Srpaulo printf("failed to instantiate provider %s: " 2073179193Sjb "%s is an invalid name", dhpv->dthpv_provname, 2074179193Sjb FASTTRAP_PID_NAME); 2075179193Sjb return (NULL); 2076179193Sjb } 2077179193Sjb 2078179193Sjb /* 2079179193Sjb * The highest stability class that fasttrap supports is ISA; cap 2080179193Sjb * the stability of the new provider accordingly. 2081179193Sjb */ 2082179193Sjb if (dhpv->dthpv_pattr.dtpa_provider.dtat_class > DTRACE_CLASS_ISA) 2083179193Sjb dhpv->dthpv_pattr.dtpa_provider.dtat_class = DTRACE_CLASS_ISA; 2084179193Sjb if (dhpv->dthpv_pattr.dtpa_mod.dtat_class > DTRACE_CLASS_ISA) 2085179193Sjb dhpv->dthpv_pattr.dtpa_mod.dtat_class = DTRACE_CLASS_ISA; 2086179193Sjb if (dhpv->dthpv_pattr.dtpa_func.dtat_class > DTRACE_CLASS_ISA) 2087179193Sjb dhpv->dthpv_pattr.dtpa_func.dtat_class = DTRACE_CLASS_ISA; 2088179193Sjb if (dhpv->dthpv_pattr.dtpa_name.dtat_class > DTRACE_CLASS_ISA) 2089179193Sjb dhpv->dthpv_pattr.dtpa_name.dtat_class = DTRACE_CLASS_ISA; 2090179193Sjb if (dhpv->dthpv_pattr.dtpa_args.dtat_class > DTRACE_CLASS_ISA) 2091179193Sjb dhpv->dthpv_pattr.dtpa_args.dtat_class = DTRACE_CLASS_ISA; 2092179193Sjb 2093179193Sjb if ((provider = fasttrap_provider_lookup(pid, dhpv->dthpv_provname, 2094179193Sjb &dhpv->dthpv_pattr)) == NULL) { 2095211738Srpaulo printf("failed to instantiate provider %s for " 2096179193Sjb "process %u", dhpv->dthpv_provname, (uint_t)pid); 2097179193Sjb return (NULL); 2098179193Sjb } 2099179193Sjb 2100179193Sjb /* 2101179193Sjb * Up the meta provider count so this provider isn't removed until 2102179193Sjb * the meta provider has been told to remove it. 2103179193Sjb */ 2104179193Sjb provider->ftp_mcount++; 2105179193Sjb 2106179193Sjb mutex_exit(&provider->ftp_mtx); 2107179193Sjb 2108179193Sjb return (provider); 2109179193Sjb} 2110179193Sjb 2111179193Sjb/*ARGSUSED*/ 2112179193Sjbstatic void 2113179193Sjbfasttrap_meta_create_probe(void *arg, void *parg, 2114179193Sjb dtrace_helper_probedesc_t *dhpb) 2115179193Sjb{ 2116179193Sjb fasttrap_provider_t *provider = parg; 2117179193Sjb fasttrap_probe_t *pp; 2118179193Sjb fasttrap_tracepoint_t *tp; 2119179193Sjb int i, j; 2120179193Sjb uint32_t ntps; 2121179193Sjb 2122179193Sjb /* 2123179193Sjb * Since the meta provider count is non-zero we don't have to worry 2124179193Sjb * about this provider disappearing. 2125179193Sjb */ 2126179193Sjb ASSERT(provider->ftp_mcount > 0); 2127179193Sjb 2128179193Sjb /* 2129179193Sjb * The offsets must be unique. 2130179193Sjb */ 2131179193Sjb qsort(dhpb->dthpb_offs, dhpb->dthpb_noffs, sizeof (uint32_t), 2132179193Sjb fasttrap_uint32_cmp); 2133179193Sjb for (i = 1; i < dhpb->dthpb_noffs; i++) { 2134179193Sjb if (dhpb->dthpb_base + dhpb->dthpb_offs[i] <= 2135179193Sjb dhpb->dthpb_base + dhpb->dthpb_offs[i - 1]) 2136179193Sjb return; 2137179193Sjb } 2138179193Sjb 2139179193Sjb qsort(dhpb->dthpb_enoffs, dhpb->dthpb_nenoffs, sizeof (uint32_t), 2140179193Sjb fasttrap_uint32_cmp); 2141179193Sjb for (i = 1; i < dhpb->dthpb_nenoffs; i++) { 2142179193Sjb if (dhpb->dthpb_base + dhpb->dthpb_enoffs[i] <= 2143179193Sjb dhpb->dthpb_base + dhpb->dthpb_enoffs[i - 1]) 2144179193Sjb return; 2145179193Sjb } 2146179193Sjb 2147179193Sjb /* 2148179193Sjb * Grab the creation lock to ensure consistency between calls to 2149179193Sjb * dtrace_probe_lookup() and dtrace_probe_create() in the face of 2150179193Sjb * other threads creating probes. 2151179193Sjb */ 2152179193Sjb mutex_enter(&provider->ftp_cmtx); 2153179193Sjb 2154179193Sjb if (dtrace_probe_lookup(provider->ftp_provid, dhpb->dthpb_mod, 2155179193Sjb dhpb->dthpb_func, dhpb->dthpb_name) != 0) { 2156179193Sjb mutex_exit(&provider->ftp_cmtx); 2157179193Sjb return; 2158179193Sjb } 2159179193Sjb 2160179193Sjb ntps = dhpb->dthpb_noffs + dhpb->dthpb_nenoffs; 2161179193Sjb ASSERT(ntps > 0); 2162179193Sjb 2163179193Sjb atomic_add_32(&fasttrap_total, ntps); 2164179193Sjb 2165179193Sjb if (fasttrap_total > fasttrap_max) { 2166179193Sjb atomic_add_32(&fasttrap_total, -ntps); 2167179193Sjb mutex_exit(&provider->ftp_cmtx); 2168179193Sjb return; 2169179193Sjb } 2170179193Sjb 2171179193Sjb pp = kmem_zalloc(offsetof(fasttrap_probe_t, ftp_tps[ntps]), KM_SLEEP); 2172179193Sjb 2173179193Sjb pp->ftp_prov = provider; 2174179193Sjb pp->ftp_pid = provider->ftp_pid; 2175179193Sjb pp->ftp_ntps = ntps; 2176179193Sjb pp->ftp_nargs = dhpb->dthpb_xargc; 2177179193Sjb pp->ftp_xtypes = dhpb->dthpb_xtypes; 2178179193Sjb pp->ftp_ntypes = dhpb->dthpb_ntypes; 2179179193Sjb 2180179193Sjb /* 2181179193Sjb * First create a tracepoint for each actual point of interest. 2182179193Sjb */ 2183179193Sjb for (i = 0; i < dhpb->dthpb_noffs; i++) { 2184179193Sjb tp = kmem_zalloc(sizeof (fasttrap_tracepoint_t), KM_SLEEP); 2185179193Sjb 2186179193Sjb tp->ftt_proc = provider->ftp_proc; 2187179193Sjb tp->ftt_pc = dhpb->dthpb_base + dhpb->dthpb_offs[i]; 2188179193Sjb tp->ftt_pid = provider->ftp_pid; 2189179193Sjb 2190179193Sjb pp->ftp_tps[i].fit_tp = tp; 2191179193Sjb pp->ftp_tps[i].fit_id.fti_probe = pp; 2192179193Sjb#ifdef __sparc 2193179193Sjb pp->ftp_tps[i].fit_id.fti_ptype = DTFTP_POST_OFFSETS; 2194179193Sjb#else 2195179193Sjb pp->ftp_tps[i].fit_id.fti_ptype = DTFTP_OFFSETS; 2196179193Sjb#endif 2197179193Sjb } 2198179193Sjb 2199179193Sjb /* 2200179193Sjb * Then create a tracepoint for each is-enabled point. 2201179193Sjb */ 2202179193Sjb for (j = 0; i < ntps; i++, j++) { 2203179193Sjb tp = kmem_zalloc(sizeof (fasttrap_tracepoint_t), KM_SLEEP); 2204179193Sjb 2205179193Sjb tp->ftt_proc = provider->ftp_proc; 2206179193Sjb tp->ftt_pc = dhpb->dthpb_base + dhpb->dthpb_enoffs[j]; 2207179193Sjb tp->ftt_pid = provider->ftp_pid; 2208179193Sjb 2209179193Sjb pp->ftp_tps[i].fit_tp = tp; 2210179193Sjb pp->ftp_tps[i].fit_id.fti_probe = pp; 2211179193Sjb pp->ftp_tps[i].fit_id.fti_ptype = DTFTP_IS_ENABLED; 2212179193Sjb } 2213179193Sjb 2214179193Sjb /* 2215179193Sjb * If the arguments are shuffled around we set the argument remapping 2216179193Sjb * table. Later, when the probe fires, we only remap the arguments 2217179193Sjb * if the table is non-NULL. 2218179193Sjb */ 2219179193Sjb for (i = 0; i < dhpb->dthpb_xargc; i++) { 2220179193Sjb if (dhpb->dthpb_args[i] != i) { 2221179193Sjb pp->ftp_argmap = dhpb->dthpb_args; 2222179193Sjb break; 2223179193Sjb } 2224179193Sjb } 2225179193Sjb 2226179193Sjb /* 2227179193Sjb * The probe is fully constructed -- register it with DTrace. 2228179193Sjb */ 2229179193Sjb pp->ftp_id = dtrace_probe_create(provider->ftp_provid, dhpb->dthpb_mod, 2230179193Sjb dhpb->dthpb_func, dhpb->dthpb_name, FASTTRAP_OFFSET_AFRAMES, pp); 2231179193Sjb 2232179193Sjb mutex_exit(&provider->ftp_cmtx); 2233179193Sjb} 2234179193Sjb 2235179193Sjb/*ARGSUSED*/ 2236179193Sjbstatic void 2237179193Sjbfasttrap_meta_remove(void *arg, dtrace_helper_provdesc_t *dhpv, pid_t pid) 2238179193Sjb{ 2239179193Sjb /* 2240179193Sjb * Clean up the USDT provider. There may be active consumers of the 2241179193Sjb * provider busy adding probes, no damage will actually befall the 2242179193Sjb * provider until that count has dropped to zero. This just puts 2243179193Sjb * the provider on death row. 2244179193Sjb */ 2245179193Sjb fasttrap_provider_retire(pid, dhpv->dthpv_provname, 1); 2246179193Sjb} 2247179193Sjb 2248179193Sjbstatic dtrace_mops_t fasttrap_mops = { 2249179193Sjb fasttrap_meta_create_probe, 2250179193Sjb fasttrap_meta_provide, 2251179193Sjb fasttrap_meta_remove 2252179193Sjb}; 2253179193Sjb 2254179193Sjb/*ARGSUSED*/ 2255179193Sjbstatic int 2256211738Srpaulofasttrap_open(struct cdev *dev __unused, int oflags __unused, 2257211738Srpaulo int devtype __unused, struct thread *td __unused) 2258179193Sjb{ 2259179193Sjb return (0); 2260179193Sjb} 2261179193Sjb 2262179193Sjb/*ARGSUSED*/ 2263179193Sjbstatic int 2264211738Srpaulofasttrap_ioctl(struct cdev *dev, u_long cmd, caddr_t arg, int fflag, 2265211738Srpaulo struct thread *td) 2266179193Sjb{ 2267211738Srpaulo#ifdef notyet 2268211738Srpaulo struct kinfo_proc kp; 2269211738Srpaulo const cred_t *cr = td->td_ucred; 2270211738Srpaulo#endif 2271179193Sjb if (!dtrace_attached()) 2272179193Sjb return (EAGAIN); 2273179193Sjb 2274179193Sjb if (cmd == FASTTRAPIOC_MAKEPROBE) { 2275262048Savg fasttrap_probe_spec_t *uprobe = *(fasttrap_probe_spec_t **)arg; 2276179193Sjb fasttrap_probe_spec_t *probe; 2277179193Sjb uint64_t noffs; 2278179193Sjb size_t size; 2279268734Spfg int ret, err; 2280179193Sjb 2281179193Sjb if (copyin(&uprobe->ftps_noffs, &noffs, 2282179193Sjb sizeof (uprobe->ftps_noffs))) 2283179193Sjb return (EFAULT); 2284179193Sjb 2285179193Sjb /* 2286179193Sjb * Probes must have at least one tracepoint. 2287179193Sjb */ 2288179193Sjb if (noffs == 0) 2289179193Sjb return (EINVAL); 2290179193Sjb 2291179193Sjb size = sizeof (fasttrap_probe_spec_t) + 2292179193Sjb sizeof (probe->ftps_offs[0]) * (noffs - 1); 2293179193Sjb 2294179193Sjb if (size > 1024 * 1024) 2295179193Sjb return (ENOMEM); 2296179193Sjb 2297179193Sjb probe = kmem_alloc(size, KM_SLEEP); 2298179193Sjb 2299268572Spfg if (copyin(uprobe, probe, size) != 0 || 2300268572Spfg probe->ftps_noffs != noffs) { 2301179193Sjb kmem_free(probe, size); 2302179193Sjb return (EFAULT); 2303179193Sjb } 2304179193Sjb 2305179193Sjb /* 2306179193Sjb * Verify that the function and module strings contain no 2307179193Sjb * funny characters. 2308179193Sjb */ 2309268734Spfg if (u8_validate(probe->ftps_func, strlen(probe->ftps_func), 2310268734Spfg NULL, U8_VALIDATE_ENTIRE, &err) < 0) { 2311268734Spfg ret = EINVAL; 2312268734Spfg goto err; 2313179193Sjb } 2314179193Sjb 2315268734Spfg if (u8_validate(probe->ftps_mod, strlen(probe->ftps_mod), 2316268734Spfg NULL, U8_VALIDATE_ENTIRE, &err) < 0) { 2317268734Spfg ret = EINVAL; 2318268734Spfg goto err; 2319179193Sjb } 2320179193Sjb 2321211738Srpaulo#ifdef notyet 2322179193Sjb if (!PRIV_POLICY_CHOICE(cr, PRIV_ALL, B_FALSE)) { 2323179193Sjb proc_t *p; 2324179193Sjb pid_t pid = probe->ftps_pid; 2325179193Sjb 2326211738Srpaulo#if defined(sun) 2327179193Sjb mutex_enter(&pidlock); 2328211738Srpaulo#endif 2329179193Sjb /* 2330179193Sjb * Report an error if the process doesn't exist 2331179193Sjb * or is actively being birthed. 2332179193Sjb */ 2333211738Srpaulo p = pfind(pid); 2334211738Srpaulo if (p) 2335211738Srpaulo fill_kinfo_proc(p, &kp); 2336211738Srpaulo if (p == NULL || kp.ki_stat == SIDL) { 2337211738Srpaulo#if defined(sun) 2338179193Sjb mutex_exit(&pidlock); 2339211738Srpaulo#endif 2340179193Sjb return (ESRCH); 2341179193Sjb } 2342211738Srpaulo#if defined(sun) 2343179193Sjb mutex_enter(&p->p_lock); 2344179193Sjb mutex_exit(&pidlock); 2345211738Srpaulo#else 2346211738Srpaulo PROC_LOCK_ASSERT(p, MA_OWNED); 2347211738Srpaulo#endif 2348179193Sjb 2349211738Srpaulo#ifdef notyet 2350179193Sjb if ((ret = priv_proc_cred_perm(cr, p, NULL, 2351179193Sjb VREAD | VWRITE)) != 0) { 2352211738Srpaulo#if defined(sun) 2353179193Sjb mutex_exit(&p->p_lock); 2354211738Srpaulo#else 2355211738Srpaulo PROC_UNLOCK(p); 2356211738Srpaulo#endif 2357179193Sjb return (ret); 2358179193Sjb } 2359211738Srpaulo#endif /* notyet */ 2360211738Srpaulo#if defined(sun) 2361179193Sjb mutex_exit(&p->p_lock); 2362211738Srpaulo#else 2363211738Srpaulo PROC_UNLOCK(p); 2364211738Srpaulo#endif 2365179193Sjb } 2366211738Srpaulo#endif /* notyet */ 2367179193Sjb 2368179193Sjb ret = fasttrap_add_probe(probe); 2369179193Sjberr: 2370179193Sjb kmem_free(probe, size); 2371179193Sjb 2372179193Sjb return (ret); 2373179193Sjb 2374179193Sjb } else if (cmd == FASTTRAPIOC_GETINSTR) { 2375179193Sjb fasttrap_instr_query_t instr; 2376179193Sjb fasttrap_tracepoint_t *tp; 2377179193Sjb uint_t index; 2378211738Srpaulo#if defined(sun) 2379179193Sjb int ret; 2380211738Srpaulo#endif 2381179193Sjb 2382211738Srpaulo#if defined(sun) 2383179193Sjb if (copyin((void *)arg, &instr, sizeof (instr)) != 0) 2384179193Sjb return (EFAULT); 2385211738Srpaulo#endif 2386179193Sjb 2387211738Srpaulo#ifdef notyet 2388179193Sjb if (!PRIV_POLICY_CHOICE(cr, PRIV_ALL, B_FALSE)) { 2389179193Sjb proc_t *p; 2390179193Sjb pid_t pid = instr.ftiq_pid; 2391179193Sjb 2392211738Srpaulo#if defined(sun) 2393179193Sjb mutex_enter(&pidlock); 2394211738Srpaulo#endif 2395179193Sjb /* 2396179193Sjb * Report an error if the process doesn't exist 2397179193Sjb * or is actively being birthed. 2398179193Sjb */ 2399211738Srpaulo p = pfind(pid); 2400211738Srpaulo if (p) 2401211738Srpaulo fill_kinfo_proc(p, &kp); 2402211738Srpaulo if (p == NULL || kp.ki_stat == SIDL) { 2403211738Srpaulo#if defined(sun) 2404179193Sjb mutex_exit(&pidlock); 2405211738Srpaulo#endif 2406179193Sjb return (ESRCH); 2407179193Sjb } 2408211738Srpaulo#if defined(sun) 2409179193Sjb mutex_enter(&p->p_lock); 2410179193Sjb mutex_exit(&pidlock); 2411211738Srpaulo#else 2412211738Srpaulo PROC_LOCK_ASSERT(p, MA_OWNED); 2413211738Srpaulo#endif 2414179193Sjb 2415211738Srpaulo#ifdef notyet 2416179193Sjb if ((ret = priv_proc_cred_perm(cr, p, NULL, 2417179193Sjb VREAD)) != 0) { 2418211738Srpaulo#if defined(sun) 2419179193Sjb mutex_exit(&p->p_lock); 2420211738Srpaulo#else 2421211738Srpaulo PROC_UNLOCK(p); 2422211738Srpaulo#endif 2423179193Sjb return (ret); 2424179193Sjb } 2425211738Srpaulo#endif /* notyet */ 2426179193Sjb 2427211738Srpaulo#if defined(sun) 2428179193Sjb mutex_exit(&p->p_lock); 2429211738Srpaulo#else 2430211738Srpaulo PROC_UNLOCK(p); 2431211738Srpaulo#endif 2432179193Sjb } 2433211738Srpaulo#endif /* notyet */ 2434179193Sjb 2435179193Sjb index = FASTTRAP_TPOINTS_INDEX(instr.ftiq_pid, instr.ftiq_pc); 2436179193Sjb 2437179193Sjb mutex_enter(&fasttrap_tpoints.fth_table[index].ftb_mtx); 2438179193Sjb tp = fasttrap_tpoints.fth_table[index].ftb_data; 2439179193Sjb while (tp != NULL) { 2440179193Sjb if (instr.ftiq_pid == tp->ftt_pid && 2441179193Sjb instr.ftiq_pc == tp->ftt_pc && 2442179193Sjb tp->ftt_proc->ftpc_acount != 0) 2443179193Sjb break; 2444179193Sjb 2445179193Sjb tp = tp->ftt_next; 2446179193Sjb } 2447179193Sjb 2448179193Sjb if (tp == NULL) { 2449179193Sjb mutex_exit(&fasttrap_tpoints.fth_table[index].ftb_mtx); 2450179193Sjb return (ENOENT); 2451179193Sjb } 2452179193Sjb 2453179193Sjb bcopy(&tp->ftt_instr, &instr.ftiq_instr, 2454179193Sjb sizeof (instr.ftiq_instr)); 2455179193Sjb mutex_exit(&fasttrap_tpoints.fth_table[index].ftb_mtx); 2456179193Sjb 2457179193Sjb if (copyout(&instr, (void *)arg, sizeof (instr)) != 0) 2458179193Sjb return (EFAULT); 2459179193Sjb 2460179193Sjb return (0); 2461179193Sjb } 2462179193Sjb 2463179193Sjb return (EINVAL); 2464179193Sjb} 2465179193Sjb 2466179193Sjbstatic int 2467211738Srpaulofasttrap_load(void) 2468179193Sjb{ 2469179193Sjb ulong_t nent; 2470250953Smarkj int i, ret; 2471179193Sjb 2472211738Srpaulo /* Create the /dev/dtrace/fasttrap entry. */ 2473211738Srpaulo fasttrap_cdev = make_dev(&fasttrap_cdevsw, 0, UID_ROOT, GID_WHEEL, 0600, 2474211738Srpaulo "dtrace/fasttrap"); 2475179193Sjb 2476211738Srpaulo mtx_init(&fasttrap_cleanup_mtx, "fasttrap clean", "dtrace", MTX_DEF); 2477211738Srpaulo mutex_init(&fasttrap_count_mtx, "fasttrap count mtx", MUTEX_DEFAULT, 2478211738Srpaulo NULL); 2479179193Sjb 2480211738Srpaulo#if defined(sun) 2481179193Sjb fasttrap_max = ddi_getprop(DDI_DEV_T_ANY, devi, DDI_PROP_DONTPASS, 2482179193Sjb "fasttrap-max-probes", FASTTRAP_MAX_DEFAULT); 2483211738Srpaulo#endif 2484179193Sjb fasttrap_total = 0; 2485179193Sjb 2486179193Sjb /* 2487179193Sjb * Conjure up the tracepoints hashtable... 2488179193Sjb */ 2489211738Srpaulo#if defined(sun) 2490179193Sjb nent = ddi_getprop(DDI_DEV_T_ANY, devi, DDI_PROP_DONTPASS, 2491179193Sjb "fasttrap-hash-size", FASTTRAP_TPOINTS_DEFAULT_SIZE); 2492211738Srpaulo#else 2493293413Sstas nent = tpoints_hash_size; 2494211738Srpaulo#endif 2495179193Sjb 2496179193Sjb if (nent == 0 || nent > 0x1000000) 2497179193Sjb nent = FASTTRAP_TPOINTS_DEFAULT_SIZE; 2498179193Sjb 2499293413Sstas tpoints_hash_size = nent; 2500293413Sstas 2501277547Sdelphij if (ISP2(nent)) 2502179193Sjb fasttrap_tpoints.fth_nent = nent; 2503179193Sjb else 2504179193Sjb fasttrap_tpoints.fth_nent = 1 << fasttrap_highbit(nent); 2505179193Sjb ASSERT(fasttrap_tpoints.fth_nent > 0); 2506179193Sjb fasttrap_tpoints.fth_mask = fasttrap_tpoints.fth_nent - 1; 2507179193Sjb fasttrap_tpoints.fth_table = kmem_zalloc(fasttrap_tpoints.fth_nent * 2508179193Sjb sizeof (fasttrap_bucket_t), KM_SLEEP); 2509211738Srpaulo#if !defined(sun) 2510211738Srpaulo for (i = 0; i < fasttrap_tpoints.fth_nent; i++) 2511211738Srpaulo mutex_init(&fasttrap_tpoints.fth_table[i].ftb_mtx, 2512211738Srpaulo "tracepoints bucket mtx", MUTEX_DEFAULT, NULL); 2513211738Srpaulo#endif 2514179193Sjb 2515179193Sjb /* 2516179193Sjb * ... and the providers hash table... 2517179193Sjb */ 2518179193Sjb nent = FASTTRAP_PROVIDERS_DEFAULT_SIZE; 2519277547Sdelphij if (ISP2(nent)) 2520179193Sjb fasttrap_provs.fth_nent = nent; 2521179193Sjb else 2522179193Sjb fasttrap_provs.fth_nent = 1 << fasttrap_highbit(nent); 2523179193Sjb ASSERT(fasttrap_provs.fth_nent > 0); 2524179193Sjb fasttrap_provs.fth_mask = fasttrap_provs.fth_nent - 1; 2525179193Sjb fasttrap_provs.fth_table = kmem_zalloc(fasttrap_provs.fth_nent * 2526179193Sjb sizeof (fasttrap_bucket_t), KM_SLEEP); 2527211738Srpaulo#if !defined(sun) 2528211738Srpaulo for (i = 0; i < fasttrap_provs.fth_nent; i++) 2529211738Srpaulo mutex_init(&fasttrap_provs.fth_table[i].ftb_mtx, 2530211738Srpaulo "providers bucket mtx", MUTEX_DEFAULT, NULL); 2531211738Srpaulo#endif 2532179193Sjb 2533259483Sasomers ret = kproc_create(fasttrap_pid_cleanup_cb, NULL, 2534259483Sasomers &fasttrap_cleanup_proc, 0, 0, "ftcleanup"); 2535259483Sasomers if (ret != 0) { 2536259483Sasomers destroy_dev(fasttrap_cdev); 2537259483Sasomers#if !defined(sun) 2538259483Sasomers for (i = 0; i < fasttrap_provs.fth_nent; i++) 2539259483Sasomers mutex_destroy(&fasttrap_provs.fth_table[i].ftb_mtx); 2540259483Sasomers for (i = 0; i < fasttrap_tpoints.fth_nent; i++) 2541259483Sasomers mutex_destroy(&fasttrap_tpoints.fth_table[i].ftb_mtx); 2542259483Sasomers#endif 2543259483Sasomers kmem_free(fasttrap_provs.fth_table, fasttrap_provs.fth_nent * 2544259483Sasomers sizeof (fasttrap_bucket_t)); 2545259483Sasomers mtx_destroy(&fasttrap_cleanup_mtx); 2546259483Sasomers mutex_destroy(&fasttrap_count_mtx); 2547259483Sasomers return (ret); 2548259483Sasomers } 2549259483Sasomers 2550259483Sasomers 2551179193Sjb /* 2552179193Sjb * ... and the procs hash table. 2553179193Sjb */ 2554179193Sjb nent = FASTTRAP_PROCS_DEFAULT_SIZE; 2555277547Sdelphij if (ISP2(nent)) 2556179193Sjb fasttrap_procs.fth_nent = nent; 2557179193Sjb else 2558179193Sjb fasttrap_procs.fth_nent = 1 << fasttrap_highbit(nent); 2559179193Sjb ASSERT(fasttrap_procs.fth_nent > 0); 2560179193Sjb fasttrap_procs.fth_mask = fasttrap_procs.fth_nent - 1; 2561179193Sjb fasttrap_procs.fth_table = kmem_zalloc(fasttrap_procs.fth_nent * 2562179193Sjb sizeof (fasttrap_bucket_t), KM_SLEEP); 2563211738Srpaulo#if !defined(sun) 2564211738Srpaulo for (i = 0; i < fasttrap_procs.fth_nent; i++) 2565211738Srpaulo mutex_init(&fasttrap_procs.fth_table[i].ftb_mtx, 2566211738Srpaulo "processes bucket mtx", MUTEX_DEFAULT, NULL); 2567211925Srpaulo 2568211925Srpaulo CPU_FOREACH(i) { 2569211925Srpaulo mutex_init(&fasttrap_cpuc_pid_lock[i], "fasttrap barrier", 2570211925Srpaulo MUTEX_DEFAULT, NULL); 2571211925Srpaulo } 2572269342Smarkj 2573269342Smarkj /* 2574269342Smarkj * This event handler must run before kdtrace_thread_dtor() since it 2575269342Smarkj * accesses the thread's struct kdtrace_thread. 2576269342Smarkj */ 2577269342Smarkj fasttrap_thread_dtor_tag = EVENTHANDLER_REGISTER(thread_dtor, 2578269342Smarkj fasttrap_thread_dtor, NULL, EVENTHANDLER_PRI_FIRST); 2579211738Srpaulo#endif 2580179193Sjb 2581253079Savg /* 2582253079Savg * Install our hooks into fork(2), exec(2), and exit(2). 2583253079Savg */ 2584253079Savg dtrace_fasttrap_fork = &fasttrap_fork; 2585253079Savg dtrace_fasttrap_exit = &fasttrap_exec_exit; 2586253079Savg dtrace_fasttrap_exec = &fasttrap_exec_exit; 2587253079Savg 2588179193Sjb (void) dtrace_meta_register("fasttrap", &fasttrap_mops, NULL, 2589179193Sjb &fasttrap_meta_id); 2590179193Sjb 2591211738Srpaulo return (0); 2592179193Sjb} 2593179193Sjb 2594179193Sjbstatic int 2595211738Srpaulofasttrap_unload(void) 2596179193Sjb{ 2597179193Sjb int i, fail = 0; 2598179193Sjb 2599179193Sjb /* 2600179193Sjb * Unregister the meta-provider to make sure no new fasttrap- 2601179193Sjb * managed providers come along while we're trying to close up 2602179193Sjb * shop. If we fail to detach, we'll need to re-register as a 2603179193Sjb * meta-provider. We can fail to unregister as a meta-provider 2604179193Sjb * if providers we manage still exist. 2605179193Sjb */ 2606179193Sjb if (fasttrap_meta_id != DTRACE_METAPROVNONE && 2607179193Sjb dtrace_meta_unregister(fasttrap_meta_id) != 0) 2608211738Srpaulo return (-1); 2609179193Sjb 2610179193Sjb /* 2611179193Sjb * Iterate over all of our providers. If there's still a process 2612179193Sjb * that corresponds to that pid, fail to detach. 2613179193Sjb */ 2614179193Sjb for (i = 0; i < fasttrap_provs.fth_nent; i++) { 2615179193Sjb fasttrap_provider_t **fpp, *fp; 2616179193Sjb fasttrap_bucket_t *bucket = &fasttrap_provs.fth_table[i]; 2617179193Sjb 2618179193Sjb mutex_enter(&bucket->ftb_mtx); 2619179193Sjb fpp = (fasttrap_provider_t **)&bucket->ftb_data; 2620179193Sjb while ((fp = *fpp) != NULL) { 2621179193Sjb /* 2622179193Sjb * Acquire and release the lock as a simple way of 2623179193Sjb * waiting for any other consumer to finish with 2624179193Sjb * this provider. A thread must first acquire the 2625179193Sjb * bucket lock so there's no chance of another thread 2626179193Sjb * blocking on the provider's lock. 2627179193Sjb */ 2628179193Sjb mutex_enter(&fp->ftp_mtx); 2629179193Sjb mutex_exit(&fp->ftp_mtx); 2630179193Sjb 2631179193Sjb if (dtrace_unregister(fp->ftp_provid) != 0) { 2632179193Sjb fail = 1; 2633179193Sjb fpp = &fp->ftp_next; 2634179193Sjb } else { 2635179193Sjb *fpp = fp->ftp_next; 2636179193Sjb fasttrap_provider_free(fp); 2637179193Sjb } 2638179193Sjb } 2639179193Sjb 2640179193Sjb mutex_exit(&bucket->ftb_mtx); 2641179193Sjb } 2642179193Sjb 2643179193Sjb if (fail) { 2644179193Sjb (void) dtrace_meta_register("fasttrap", &fasttrap_mops, NULL, 2645179193Sjb &fasttrap_meta_id); 2646179193Sjb 2647211738Srpaulo return (-1); 2648179193Sjb } 2649179193Sjb 2650259483Sasomers /* 2651259483Sasomers * Stop new processes from entering these hooks now, before the 2652259483Sasomers * fasttrap_cleanup thread runs. That way all processes will hopefully 2653259483Sasomers * be out of these hooks before we free fasttrap_provs.fth_table 2654259483Sasomers */ 2655259483Sasomers ASSERT(dtrace_fasttrap_fork == &fasttrap_fork); 2656259483Sasomers dtrace_fasttrap_fork = NULL; 2657259483Sasomers 2658259483Sasomers ASSERT(dtrace_fasttrap_exec == &fasttrap_exec_exit); 2659259483Sasomers dtrace_fasttrap_exec = NULL; 2660259483Sasomers 2661259483Sasomers ASSERT(dtrace_fasttrap_exit == &fasttrap_exec_exit); 2662259483Sasomers dtrace_fasttrap_exit = NULL; 2663259483Sasomers 2664250953Smarkj mtx_lock(&fasttrap_cleanup_mtx); 2665250953Smarkj fasttrap_cleanup_drain = 1; 2666250953Smarkj /* Wait for the cleanup thread to finish up and signal us. */ 2667250953Smarkj wakeup(&fasttrap_cleanup_cv); 2668250953Smarkj mtx_sleep(&fasttrap_cleanup_drain, &fasttrap_cleanup_mtx, 0, "ftcld", 2669250953Smarkj 0); 2670250953Smarkj fasttrap_cleanup_proc = NULL; 2671252493Smarkj mtx_destroy(&fasttrap_cleanup_mtx); 2672250953Smarkj 2673179193Sjb#ifdef DEBUG 2674179193Sjb mutex_enter(&fasttrap_count_mtx); 2675179193Sjb ASSERT(fasttrap_pid_count == 0); 2676179193Sjb mutex_exit(&fasttrap_count_mtx); 2677179193Sjb#endif 2678179193Sjb 2679259483Sasomers#if !defined(sun) 2680269342Smarkj EVENTHANDLER_DEREGISTER(thread_dtor, fasttrap_thread_dtor_tag); 2681269342Smarkj 2682259483Sasomers for (i = 0; i < fasttrap_tpoints.fth_nent; i++) 2683259483Sasomers mutex_destroy(&fasttrap_tpoints.fth_table[i].ftb_mtx); 2684259483Sasomers for (i = 0; i < fasttrap_provs.fth_nent; i++) 2685259483Sasomers mutex_destroy(&fasttrap_provs.fth_table[i].ftb_mtx); 2686259483Sasomers for (i = 0; i < fasttrap_procs.fth_nent; i++) 2687259483Sasomers mutex_destroy(&fasttrap_procs.fth_table[i].ftb_mtx); 2688259483Sasomers#endif 2689179193Sjb kmem_free(fasttrap_tpoints.fth_table, 2690179193Sjb fasttrap_tpoints.fth_nent * sizeof (fasttrap_bucket_t)); 2691179193Sjb fasttrap_tpoints.fth_nent = 0; 2692179193Sjb 2693179193Sjb kmem_free(fasttrap_provs.fth_table, 2694179193Sjb fasttrap_provs.fth_nent * sizeof (fasttrap_bucket_t)); 2695179193Sjb fasttrap_provs.fth_nent = 0; 2696179193Sjb 2697179193Sjb kmem_free(fasttrap_procs.fth_table, 2698179193Sjb fasttrap_procs.fth_nent * sizeof (fasttrap_bucket_t)); 2699179193Sjb fasttrap_procs.fth_nent = 0; 2700179193Sjb 2701211738Srpaulo#if !defined(sun) 2702211738Srpaulo destroy_dev(fasttrap_cdev); 2703211738Srpaulo mutex_destroy(&fasttrap_count_mtx); 2704211925Srpaulo CPU_FOREACH(i) { 2705211925Srpaulo mutex_destroy(&fasttrap_cpuc_pid_lock[i]); 2706211925Srpaulo } 2707211738Srpaulo#endif 2708179193Sjb 2709211738Srpaulo return (0); 2710179193Sjb} 2711179193Sjb 2712211738Srpaulo/* ARGSUSED */ 2713211738Srpaulostatic int 2714211738Srpaulofasttrap_modevent(module_t mod __unused, int type, void *data __unused) 2715211738Srpaulo{ 2716211738Srpaulo int error = 0; 2717179193Sjb 2718211738Srpaulo switch (type) { 2719211738Srpaulo case MOD_LOAD: 2720211738Srpaulo break; 2721179193Sjb 2722211738Srpaulo case MOD_UNLOAD: 2723211738Srpaulo break; 2724179193Sjb 2725211738Srpaulo case MOD_SHUTDOWN: 2726211738Srpaulo break; 2727179193Sjb 2728211738Srpaulo default: 2729211738Srpaulo error = EOPNOTSUPP; 2730211738Srpaulo break; 2731211738Srpaulo } 2732211738Srpaulo return (error); 2733179193Sjb} 2734179193Sjb 2735211738SrpauloSYSINIT(fasttrap_load, SI_SUB_DTRACE_PROVIDER, SI_ORDER_ANY, fasttrap_load, 2736211738Srpaulo NULL); 2737211738SrpauloSYSUNINIT(fasttrap_unload, SI_SUB_DTRACE_PROVIDER, SI_ORDER_ANY, 2738211738Srpaulo fasttrap_unload, NULL); 2739211738Srpaulo 2740211738SrpauloDEV_MODULE(fasttrap, fasttrap_modevent, NULL); 2741211738SrpauloMODULE_VERSION(fasttrap, 1); 2742211738SrpauloMODULE_DEPEND(fasttrap, dtrace, 1, 1, 1); 2743211738SrpauloMODULE_DEPEND(fasttrap, opensolaris, 1, 1, 1); 2744