1/* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License (the "License"). 6 * You may not use this file except in compliance with the License. 7 * 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9 * or http://www.opensolaris.org/os/licensing. 10 * See the License for the specific language governing permissions 11 * and limitations under the License. 12 * 13 * When distributing Covered Code, include this CDDL HEADER in each 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15 * If applicable, add the following below this CDDL HEADER, with the 16 * fields enclosed by brackets "[]" replaced with your own identifying 17 * information: Portions Copyright [yyyy] [name of copyright owner] 18 * 19 * CDDL HEADER END 20 * 21 * Portions Copyright 2010 The FreeBSD Foundation 22 *
| 1/* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License (the "License"). 6 * You may not use this file except in compliance with the License. 7 * 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9 * or http://www.opensolaris.org/os/licensing. 10 * See the License for the specific language governing permissions 11 * and limitations under the License. 12 * 13 * When distributing Covered Code, include this CDDL HEADER in each 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15 * If applicable, add the following below this CDDL HEADER, with the 16 * fields enclosed by brackets "[]" replaced with your own identifying 17 * information: Portions Copyright [yyyy] [name of copyright owner] 18 * 19 * CDDL HEADER END 20 * 21 * Portions Copyright 2010 The FreeBSD Foundation 22 *
|
23 * $FreeBSD: head/sys/cddl/contrib/opensolaris/uts/common/dtrace/fasttrap.c 258311 2013-11-18 16:51:56Z asomers $
| 23 * $FreeBSD: head/sys/cddl/contrib/opensolaris/uts/common/dtrace/fasttrap.c 264434 2014-04-14 00:22:42Z markj $
|
24 */ 25 26/* 27 * Copyright 2008 Sun Microsystems, Inc. All rights reserved. 28 * Use is subject to license terms. 29 */ 30 31#if defined(sun) 32#pragma ident "%Z%%M% %I% %E% SMI" 33#endif 34 35#include <sys/atomic.h> 36#include <sys/errno.h> 37#include <sys/stat.h> 38#include <sys/modctl.h> 39#include <sys/conf.h> 40#include <sys/systm.h> 41#if defined(sun) 42#include <sys/ddi.h> 43#endif 44#include <sys/sunddi.h> 45#include <sys/cpuvar.h> 46#include <sys/kmem.h> 47#if defined(sun) 48#include <sys/strsubr.h> 49#endif 50#include <sys/fasttrap.h> 51#include <sys/fasttrap_impl.h> 52#include <sys/fasttrap_isa.h> 53#include <sys/dtrace.h> 54#include <sys/dtrace_impl.h> 55#include <sys/sysmacros.h> 56#include <sys/proc.h> 57#include <sys/policy.h> 58#if defined(sun) 59#include <util/qsort.h> 60#endif 61#include <sys/mutex.h> 62#include <sys/kernel.h> 63#if !defined(sun)
| 24 */ 25 26/* 27 * Copyright 2008 Sun Microsystems, Inc. All rights reserved. 28 * Use is subject to license terms. 29 */ 30 31#if defined(sun) 32#pragma ident "%Z%%M% %I% %E% SMI" 33#endif 34 35#include <sys/atomic.h> 36#include <sys/errno.h> 37#include <sys/stat.h> 38#include <sys/modctl.h> 39#include <sys/conf.h> 40#include <sys/systm.h> 41#if defined(sun) 42#include <sys/ddi.h> 43#endif 44#include <sys/sunddi.h> 45#include <sys/cpuvar.h> 46#include <sys/kmem.h> 47#if defined(sun) 48#include <sys/strsubr.h> 49#endif 50#include <sys/fasttrap.h> 51#include <sys/fasttrap_impl.h> 52#include <sys/fasttrap_isa.h> 53#include <sys/dtrace.h> 54#include <sys/dtrace_impl.h> 55#include <sys/sysmacros.h> 56#include <sys/proc.h> 57#include <sys/policy.h> 58#if defined(sun) 59#include <util/qsort.h> 60#endif 61#include <sys/mutex.h> 62#include <sys/kernel.h> 63#if !defined(sun)
|
64#include <sys/user.h>
| |
65#include <sys/dtrace_bsd.h>
| 64#include <sys/dtrace_bsd.h>
|
| 65#include <sys/eventhandler.h> 66#include <sys/user.h> 67#include <vm/vm.h> 68#include <vm/pmap.h> 69#include <vm/vm_map.h> 70#include <vm/vm_param.h>
|
66#include <cddl/dev/dtrace/dtrace_cddl.h> 67#endif 68 69/* 70 * User-Land Trap-Based Tracing 71 * ---------------------------- 72 * 73 * The fasttrap provider allows DTrace consumers to instrument any user-level 74 * instruction to gather data; this includes probes with semantic 75 * signifigance like entry and return as well as simple offsets into the 76 * function. While the specific techniques used are very ISA specific, the 77 * methodology is generalizable to any architecture. 78 * 79 * 80 * The General Methodology 81 * ----------------------- 82 * 83 * With the primary goal of tracing every user-land instruction and the 84 * limitation that we can't trust user space so don't want to rely on much 85 * information there, we begin by replacing the instructions we want to trace 86 * with trap instructions. Each instruction we overwrite is saved into a hash 87 * table keyed by process ID and pc address. When we enter the kernel due to 88 * this trap instruction, we need the effects of the replaced instruction to 89 * appear to have occurred before we proceed with the user thread's 90 * execution. 91 * 92 * Each user level thread is represented by a ulwp_t structure which is 93 * always easily accessible through a register. The most basic way to produce 94 * the effects of the instruction we replaced is to copy that instruction out 95 * to a bit of scratch space reserved in the user thread's ulwp_t structure 96 * (a sort of kernel-private thread local storage), set the PC to that 97 * scratch space and single step. When we reenter the kernel after single 98 * stepping the instruction we must then adjust the PC to point to what would 99 * normally be the next instruction. Of course, special care must be taken 100 * for branches and jumps, but these represent such a small fraction of any 101 * instruction set that writing the code to emulate these in the kernel is 102 * not too difficult. 103 * 104 * Return probes may require several tracepoints to trace every return site, 105 * and, conversely, each tracepoint may activate several probes (the entry 106 * and offset 0 probes, for example). To solve this muliplexing problem, 107 * tracepoints contain lists of probes to activate and probes contain lists 108 * of tracepoints to enable. If a probe is activated, it adds its ID to 109 * existing tracepoints or creates new ones as necessary. 110 * 111 * Most probes are activated _before_ the instruction is executed, but return 112 * probes are activated _after_ the effects of the last instruction of the 113 * function are visible. Return probes must be fired _after_ we have 114 * single-stepped the instruction whereas all other probes are fired 115 * beforehand. 116 * 117 * 118 * Lock Ordering 119 * ------------- 120 * 121 * The lock ordering below -- both internally and with respect to the DTrace 122 * framework -- is a little tricky and bears some explanation. Each provider 123 * has a lock (ftp_mtx) that protects its members including reference counts 124 * for enabled probes (ftp_rcount), consumers actively creating probes 125 * (ftp_ccount) and USDT consumers (ftp_mcount); all three prevent a provider 126 * from being freed. A provider is looked up by taking the bucket lock for the 127 * provider hash table, and is returned with its lock held. The provider lock 128 * may be taken in functions invoked by the DTrace framework, but may not be 129 * held while calling functions in the DTrace framework. 130 * 131 * To ensure consistency over multiple calls to the DTrace framework, the 132 * creation lock (ftp_cmtx) should be held. Naturally, the creation lock may 133 * not be taken when holding the provider lock as that would create a cyclic 134 * lock ordering. In situations where one would naturally take the provider 135 * lock and then the creation lock, we instead up a reference count to prevent 136 * the provider from disappearing, drop the provider lock, and acquire the 137 * creation lock. 138 * 139 * Briefly: 140 * bucket lock before provider lock 141 * DTrace before provider lock 142 * creation lock before DTrace 143 * never hold the provider lock and creation lock simultaneously 144 */ 145 146static d_open_t fasttrap_open; 147static d_ioctl_t fasttrap_ioctl; 148 149static struct cdevsw fasttrap_cdevsw = { 150 .d_version = D_VERSION, 151 .d_open = fasttrap_open, 152 .d_ioctl = fasttrap_ioctl, 153 .d_name = "fasttrap", 154}; 155static struct cdev *fasttrap_cdev; 156static dtrace_meta_provider_id_t fasttrap_meta_id; 157 158static struct proc *fasttrap_cleanup_proc; 159static struct mtx fasttrap_cleanup_mtx; 160static uint_t fasttrap_cleanup_work, fasttrap_cleanup_drain, fasttrap_cleanup_cv; 161 162/* 163 * Generation count on modifications to the global tracepoint lookup table. 164 */ 165static volatile uint64_t fasttrap_mod_gen; 166 167/* 168 * When the fasttrap provider is loaded, fasttrap_max is set to either 169 * FASTTRAP_MAX_DEFAULT or the value for fasttrap-max-probes in the 170 * fasttrap.conf file. Each time a probe is created, fasttrap_total is 171 * incremented by the number of tracepoints that may be associated with that 172 * probe; fasttrap_total is capped at fasttrap_max. 173 */ 174#define FASTTRAP_MAX_DEFAULT 250000 175static uint32_t fasttrap_max; 176static uint32_t fasttrap_total; 177 178/* 179 * Copyright (c) 2011, Joyent, Inc. All rights reserved. 180 */ 181 182#define FASTTRAP_TPOINTS_DEFAULT_SIZE 0x4000 183#define FASTTRAP_PROVIDERS_DEFAULT_SIZE 0x100 184#define FASTTRAP_PROCS_DEFAULT_SIZE 0x100 185 186#define FASTTRAP_PID_NAME "pid" 187 188fasttrap_hash_t fasttrap_tpoints; 189static fasttrap_hash_t fasttrap_provs; 190static fasttrap_hash_t fasttrap_procs; 191 192static uint64_t fasttrap_pid_count; /* pid ref count */ 193static kmutex_t fasttrap_count_mtx; /* lock on ref count */ 194 195#define FASTTRAP_ENABLE_FAIL 1 196#define FASTTRAP_ENABLE_PARTIAL 2 197 198static int fasttrap_tracepoint_enable(proc_t *, fasttrap_probe_t *, uint_t); 199static void fasttrap_tracepoint_disable(proc_t *, fasttrap_probe_t *, uint_t); 200 201static fasttrap_provider_t *fasttrap_provider_lookup(pid_t, const char *, 202 const dtrace_pattr_t *); 203static void fasttrap_provider_retire(pid_t, const char *, int); 204static void fasttrap_provider_free(fasttrap_provider_t *); 205 206static fasttrap_proc_t *fasttrap_proc_lookup(pid_t); 207static void fasttrap_proc_release(fasttrap_proc_t *); 208
| 71#include <cddl/dev/dtrace/dtrace_cddl.h> 72#endif 73 74/* 75 * User-Land Trap-Based Tracing 76 * ---------------------------- 77 * 78 * The fasttrap provider allows DTrace consumers to instrument any user-level 79 * instruction to gather data; this includes probes with semantic 80 * signifigance like entry and return as well as simple offsets into the 81 * function. While the specific techniques used are very ISA specific, the 82 * methodology is generalizable to any architecture. 83 * 84 * 85 * The General Methodology 86 * ----------------------- 87 * 88 * With the primary goal of tracing every user-land instruction and the 89 * limitation that we can't trust user space so don't want to rely on much 90 * information there, we begin by replacing the instructions we want to trace 91 * with trap instructions. Each instruction we overwrite is saved into a hash 92 * table keyed by process ID and pc address. When we enter the kernel due to 93 * this trap instruction, we need the effects of the replaced instruction to 94 * appear to have occurred before we proceed with the user thread's 95 * execution. 96 * 97 * Each user level thread is represented by a ulwp_t structure which is 98 * always easily accessible through a register. The most basic way to produce 99 * the effects of the instruction we replaced is to copy that instruction out 100 * to a bit of scratch space reserved in the user thread's ulwp_t structure 101 * (a sort of kernel-private thread local storage), set the PC to that 102 * scratch space and single step. When we reenter the kernel after single 103 * stepping the instruction we must then adjust the PC to point to what would 104 * normally be the next instruction. Of course, special care must be taken 105 * for branches and jumps, but these represent such a small fraction of any 106 * instruction set that writing the code to emulate these in the kernel is 107 * not too difficult. 108 * 109 * Return probes may require several tracepoints to trace every return site, 110 * and, conversely, each tracepoint may activate several probes (the entry 111 * and offset 0 probes, for example). To solve this muliplexing problem, 112 * tracepoints contain lists of probes to activate and probes contain lists 113 * of tracepoints to enable. If a probe is activated, it adds its ID to 114 * existing tracepoints or creates new ones as necessary. 115 * 116 * Most probes are activated _before_ the instruction is executed, but return 117 * probes are activated _after_ the effects of the last instruction of the 118 * function are visible. Return probes must be fired _after_ we have 119 * single-stepped the instruction whereas all other probes are fired 120 * beforehand. 121 * 122 * 123 * Lock Ordering 124 * ------------- 125 * 126 * The lock ordering below -- both internally and with respect to the DTrace 127 * framework -- is a little tricky and bears some explanation. Each provider 128 * has a lock (ftp_mtx) that protects its members including reference counts 129 * for enabled probes (ftp_rcount), consumers actively creating probes 130 * (ftp_ccount) and USDT consumers (ftp_mcount); all three prevent a provider 131 * from being freed. A provider is looked up by taking the bucket lock for the 132 * provider hash table, and is returned with its lock held. The provider lock 133 * may be taken in functions invoked by the DTrace framework, but may not be 134 * held while calling functions in the DTrace framework. 135 * 136 * To ensure consistency over multiple calls to the DTrace framework, the 137 * creation lock (ftp_cmtx) should be held. Naturally, the creation lock may 138 * not be taken when holding the provider lock as that would create a cyclic 139 * lock ordering. In situations where one would naturally take the provider 140 * lock and then the creation lock, we instead up a reference count to prevent 141 * the provider from disappearing, drop the provider lock, and acquire the 142 * creation lock. 143 * 144 * Briefly: 145 * bucket lock before provider lock 146 * DTrace before provider lock 147 * creation lock before DTrace 148 * never hold the provider lock and creation lock simultaneously 149 */ 150 151static d_open_t fasttrap_open; 152static d_ioctl_t fasttrap_ioctl; 153 154static struct cdevsw fasttrap_cdevsw = { 155 .d_version = D_VERSION, 156 .d_open = fasttrap_open, 157 .d_ioctl = fasttrap_ioctl, 158 .d_name = "fasttrap", 159}; 160static struct cdev *fasttrap_cdev; 161static dtrace_meta_provider_id_t fasttrap_meta_id; 162 163static struct proc *fasttrap_cleanup_proc; 164static struct mtx fasttrap_cleanup_mtx; 165static uint_t fasttrap_cleanup_work, fasttrap_cleanup_drain, fasttrap_cleanup_cv; 166 167/* 168 * Generation count on modifications to the global tracepoint lookup table. 169 */ 170static volatile uint64_t fasttrap_mod_gen; 171 172/* 173 * When the fasttrap provider is loaded, fasttrap_max is set to either 174 * FASTTRAP_MAX_DEFAULT or the value for fasttrap-max-probes in the 175 * fasttrap.conf file. Each time a probe is created, fasttrap_total is 176 * incremented by the number of tracepoints that may be associated with that 177 * probe; fasttrap_total is capped at fasttrap_max. 178 */ 179#define FASTTRAP_MAX_DEFAULT 250000 180static uint32_t fasttrap_max; 181static uint32_t fasttrap_total; 182 183/* 184 * Copyright (c) 2011, Joyent, Inc. All rights reserved. 185 */ 186 187#define FASTTRAP_TPOINTS_DEFAULT_SIZE 0x4000 188#define FASTTRAP_PROVIDERS_DEFAULT_SIZE 0x100 189#define FASTTRAP_PROCS_DEFAULT_SIZE 0x100 190 191#define FASTTRAP_PID_NAME "pid" 192 193fasttrap_hash_t fasttrap_tpoints; 194static fasttrap_hash_t fasttrap_provs; 195static fasttrap_hash_t fasttrap_procs; 196 197static uint64_t fasttrap_pid_count; /* pid ref count */ 198static kmutex_t fasttrap_count_mtx; /* lock on ref count */ 199 200#define FASTTRAP_ENABLE_FAIL 1 201#define FASTTRAP_ENABLE_PARTIAL 2 202 203static int fasttrap_tracepoint_enable(proc_t *, fasttrap_probe_t *, uint_t); 204static void fasttrap_tracepoint_disable(proc_t *, fasttrap_probe_t *, uint_t); 205 206static fasttrap_provider_t *fasttrap_provider_lookup(pid_t, const char *, 207 const dtrace_pattr_t *); 208static void fasttrap_provider_retire(pid_t, const char *, int); 209static void fasttrap_provider_free(fasttrap_provider_t *); 210 211static fasttrap_proc_t *fasttrap_proc_lookup(pid_t); 212static void fasttrap_proc_release(fasttrap_proc_t *); 213
|
| 214#if !defined(sun) 215static void fasttrap_thread_dtor(void *, struct thread *); 216#endif 217
|
209#define FASTTRAP_PROVS_INDEX(pid, name) \ 210 ((fasttrap_hash_str(name) + (pid)) & fasttrap_provs.fth_mask) 211 212#define FASTTRAP_PROCS_INDEX(pid) ((pid) & fasttrap_procs.fth_mask) 213 214#if !defined(sun) 215static kmutex_t fasttrap_cpuc_pid_lock[MAXCPU];
| 218#define FASTTRAP_PROVS_INDEX(pid, name) \ 219 ((fasttrap_hash_str(name) + (pid)) & fasttrap_provs.fth_mask) 220 221#define FASTTRAP_PROCS_INDEX(pid) ((pid) & fasttrap_procs.fth_mask) 222 223#if !defined(sun) 224static kmutex_t fasttrap_cpuc_pid_lock[MAXCPU];
|
| 225static eventhandler_tag fasttrap_thread_dtor_tag;
|
216#endif 217 218static int 219fasttrap_highbit(ulong_t i) 220{ 221 int h = 1; 222 223 if (i == 0) 224 return (0); 225#ifdef _LP64 226 if (i & 0xffffffff00000000ul) { 227 h += 32; i >>= 32; 228 } 229#endif 230 if (i & 0xffff0000) { 231 h += 16; i >>= 16; 232 } 233 if (i & 0xff00) { 234 h += 8; i >>= 8; 235 } 236 if (i & 0xf0) { 237 h += 4; i >>= 4; 238 } 239 if (i & 0xc) { 240 h += 2; i >>= 2; 241 } 242 if (i & 0x2) { 243 h += 1; 244 } 245 return (h); 246} 247 248static uint_t 249fasttrap_hash_str(const char *p) 250{ 251 unsigned int g; 252 uint_t hval = 0; 253 254 while (*p) { 255 hval = (hval << 4) + *p++; 256 if ((g = (hval & 0xf0000000)) != 0) 257 hval ^= g >> 24; 258 hval &= ~g; 259 } 260 return (hval); 261} 262 263void 264fasttrap_sigtrap(proc_t *p, kthread_t *t, uintptr_t pc) 265{ 266#if defined(sun) 267 sigqueue_t *sqp = kmem_zalloc(sizeof (sigqueue_t), KM_SLEEP); 268 269 sqp->sq_info.si_signo = SIGTRAP; 270 sqp->sq_info.si_code = TRAP_DTRACE; 271 sqp->sq_info.si_addr = (caddr_t)pc; 272 273 mutex_enter(&p->p_lock); 274 sigaddqa(p, t, sqp); 275 mutex_exit(&p->p_lock); 276 277 if (t != NULL) 278 aston(t); 279#else 280 ksiginfo_t *ksi = kmem_zalloc(sizeof (ksiginfo_t), KM_SLEEP); 281 282 ksiginfo_init(ksi); 283 ksi->ksi_signo = SIGTRAP; 284 ksi->ksi_code = TRAP_DTRACE; 285 ksi->ksi_addr = (caddr_t)pc; 286 PROC_LOCK(p); 287 (void) tdksignal(t, SIGTRAP, ksi); 288 PROC_UNLOCK(p); 289#endif 290} 291
| 226#endif 227 228static int 229fasttrap_highbit(ulong_t i) 230{ 231 int h = 1; 232 233 if (i == 0) 234 return (0); 235#ifdef _LP64 236 if (i & 0xffffffff00000000ul) { 237 h += 32; i >>= 32; 238 } 239#endif 240 if (i & 0xffff0000) { 241 h += 16; i >>= 16; 242 } 243 if (i & 0xff00) { 244 h += 8; i >>= 8; 245 } 246 if (i & 0xf0) { 247 h += 4; i >>= 4; 248 } 249 if (i & 0xc) { 250 h += 2; i >>= 2; 251 } 252 if (i & 0x2) { 253 h += 1; 254 } 255 return (h); 256} 257 258static uint_t 259fasttrap_hash_str(const char *p) 260{ 261 unsigned int g; 262 uint_t hval = 0; 263 264 while (*p) { 265 hval = (hval << 4) + *p++; 266 if ((g = (hval & 0xf0000000)) != 0) 267 hval ^= g >> 24; 268 hval &= ~g; 269 } 270 return (hval); 271} 272 273void 274fasttrap_sigtrap(proc_t *p, kthread_t *t, uintptr_t pc) 275{ 276#if defined(sun) 277 sigqueue_t *sqp = kmem_zalloc(sizeof (sigqueue_t), KM_SLEEP); 278 279 sqp->sq_info.si_signo = SIGTRAP; 280 sqp->sq_info.si_code = TRAP_DTRACE; 281 sqp->sq_info.si_addr = (caddr_t)pc; 282 283 mutex_enter(&p->p_lock); 284 sigaddqa(p, t, sqp); 285 mutex_exit(&p->p_lock); 286 287 if (t != NULL) 288 aston(t); 289#else 290 ksiginfo_t *ksi = kmem_zalloc(sizeof (ksiginfo_t), KM_SLEEP); 291 292 ksiginfo_init(ksi); 293 ksi->ksi_signo = SIGTRAP; 294 ksi->ksi_code = TRAP_DTRACE; 295 ksi->ksi_addr = (caddr_t)pc; 296 PROC_LOCK(p); 297 (void) tdksignal(t, SIGTRAP, ksi); 298 PROC_UNLOCK(p); 299#endif 300} 301
|
| 302#if !defined(sun)
|
292/*
| 303/*
|
| 304 * Obtain a chunk of scratch space in the address space of the target process. 305 */ 306fasttrap_scrspace_t * 307fasttrap_scraddr(struct thread *td, fasttrap_proc_t *fprc) 308{ 309 fasttrap_scrblock_t *scrblk; 310 fasttrap_scrspace_t *scrspc; 311 struct proc *p; 312 vm_offset_t addr; 313 int error, i; 314 315 scrspc = NULL; 316 if (td->t_dtrace_sscr != NULL) { 317 /* If the thread already has scratch space, we're done. */ 318 scrspc = (fasttrap_scrspace_t *)td->t_dtrace_sscr; 319 return (scrspc); 320 } 321 322 p = td->td_proc; 323 324 mutex_enter(&fprc->ftpc_mtx); 325 if (LIST_EMPTY(&fprc->ftpc_fscr)) { 326 /* 327 * No scratch space is available, so we'll map a new scratch 328 * space block into the traced process' address space. 329 */ 330 addr = 0; 331 error = vm_map_find(&p->p_vmspace->vm_map, NULL, 0, &addr, 332 FASTTRAP_SCRBLOCK_SIZE, 0, VMFS_ANY_SPACE, VM_PROT_ALL, 333 VM_PROT_ALL, 0); 334 if (error != KERN_SUCCESS) 335 goto done; 336 337 scrblk = malloc(sizeof(*scrblk), M_SOLARIS, M_WAITOK); 338 scrblk->ftsb_addr = addr; 339 LIST_INSERT_HEAD(&fprc->ftpc_scrblks, scrblk, ftsb_next); 340 341 /* 342 * Carve the block up into chunks and put them on the free list. 343 */ 344 for (i = 0; 345 i < FASTTRAP_SCRBLOCK_SIZE / FASTTRAP_SCRSPACE_SIZE; i++) { 346 scrspc = malloc(sizeof(*scrspc), M_SOLARIS, M_WAITOK); 347 scrspc->ftss_addr = addr + 348 i * FASTTRAP_SCRSPACE_SIZE; 349 LIST_INSERT_HEAD(&fprc->ftpc_fscr, scrspc, 350 ftss_next); 351 } 352 } 353 354 /* 355 * Take the first scratch chunk off the free list, put it on the 356 * allocated list, and return its address. 357 */ 358 scrspc = LIST_FIRST(&fprc->ftpc_fscr); 359 LIST_REMOVE(scrspc, ftss_next); 360 LIST_INSERT_HEAD(&fprc->ftpc_ascr, scrspc, ftss_next); 361 362 /* 363 * This scratch space is reserved for use by td until the thread exits. 364 */ 365 td->t_dtrace_sscr = scrspc; 366 367done: 368 mutex_exit(&fprc->ftpc_mtx); 369 370 return (scrspc); 371} 372 373/* 374 * Return any allocated per-thread scratch space chunks back to the process' 375 * free list. 376 */ 377static void 378fasttrap_thread_dtor(void *arg __unused, struct thread *td) 379{ 380 fasttrap_bucket_t *bucket; 381 fasttrap_proc_t *fprc; 382 fasttrap_scrspace_t *scrspc; 383 pid_t pid; 384 385 if (td->t_dtrace_sscr == NULL) 386 return; 387 388 pid = td->td_proc->p_pid; 389 bucket = &fasttrap_procs.fth_table[FASTTRAP_PROCS_INDEX(pid)]; 390 fprc = NULL; 391 392 /* Look up the fasttrap process handle for this process. */ 393 mutex_enter(&bucket->ftb_mtx); 394 for (fprc = bucket->ftb_data; fprc != NULL; fprc = fprc->ftpc_next) { 395 if (fprc->ftpc_pid == pid) { 396 mutex_enter(&fprc->ftpc_mtx); 397 mutex_exit(&bucket->ftb_mtx); 398 break; 399 } 400 } 401 if (fprc == NULL) { 402 mutex_exit(&bucket->ftb_mtx); 403 return; 404 } 405 406 scrspc = (fasttrap_scrspace_t *)td->t_dtrace_sscr; 407 LIST_REMOVE(scrspc, ftss_next); 408 LIST_INSERT_HEAD(&fprc->ftpc_fscr, scrspc, ftss_next); 409 410 mutex_exit(&fprc->ftpc_mtx); 411} 412#endif 413 414/*
|
293 * This function ensures that no threads are actively using the memory 294 * associated with probes that were formerly live. 295 */ 296static void 297fasttrap_mod_barrier(uint64_t gen) 298{ 299 int i; 300 301 if (gen < fasttrap_mod_gen) 302 return; 303 304 fasttrap_mod_gen++; 305 306 CPU_FOREACH(i) { 307 mutex_enter(&fasttrap_cpuc_pid_lock[i]); 308 mutex_exit(&fasttrap_cpuc_pid_lock[i]); 309 } 310} 311 312/* 313 * This function performs asynchronous cleanup of fasttrap providers. The 314 * Solaris implementation of this mechanism use a timeout that's activated in 315 * fasttrap_pid_cleanup(), but this doesn't work in FreeBSD: one may sleep while 316 * holding the DTrace mutexes, but it is unsafe to sleep in a callout handler. 317 * Thus we use a dedicated process to perform the cleanup when requested. 318 */ 319/*ARGSUSED*/ 320static void 321fasttrap_pid_cleanup_cb(void *data) 322{ 323 fasttrap_provider_t **fpp, *fp; 324 fasttrap_bucket_t *bucket; 325 dtrace_provider_id_t provid; 326 int i, later = 0, rval; 327 328 mtx_lock(&fasttrap_cleanup_mtx); 329 while (!fasttrap_cleanup_drain || later > 0) { 330 fasttrap_cleanup_work = 0; 331 mtx_unlock(&fasttrap_cleanup_mtx); 332 333 later = 0; 334 335 /* 336 * Iterate over all the providers trying to remove the marked 337 * ones. If a provider is marked but not retired, we just 338 * have to take a crack at removing it -- it's no big deal if 339 * we can't. 340 */ 341 for (i = 0; i < fasttrap_provs.fth_nent; i++) { 342 bucket = &fasttrap_provs.fth_table[i]; 343 mutex_enter(&bucket->ftb_mtx); 344 fpp = (fasttrap_provider_t **)&bucket->ftb_data; 345 346 while ((fp = *fpp) != NULL) { 347 if (!fp->ftp_marked) { 348 fpp = &fp->ftp_next; 349 continue; 350 } 351 352 mutex_enter(&fp->ftp_mtx); 353 354 /* 355 * If this provider has consumers actively 356 * creating probes (ftp_ccount) or is a USDT 357 * provider (ftp_mcount), we can't unregister 358 * or even condense. 359 */ 360 if (fp->ftp_ccount != 0 || 361 fp->ftp_mcount != 0) { 362 mutex_exit(&fp->ftp_mtx); 363 fp->ftp_marked = 0; 364 continue; 365 } 366 367 if (!fp->ftp_retired || fp->ftp_rcount != 0) 368 fp->ftp_marked = 0; 369 370 mutex_exit(&fp->ftp_mtx); 371 372 /* 373 * If we successfully unregister this 374 * provider we can remove it from the hash 375 * chain and free the memory. If our attempt 376 * to unregister fails and this is a retired 377 * provider, increment our flag to try again 378 * pretty soon. If we've consumed more than 379 * half of our total permitted number of 380 * probes call dtrace_condense() to try to 381 * clean out the unenabled probes. 382 */ 383 provid = fp->ftp_provid; 384 if ((rval = dtrace_unregister(provid)) != 0) { 385 if (fasttrap_total > fasttrap_max / 2) 386 (void) dtrace_condense(provid); 387 388 if (rval == EAGAIN) 389 fp->ftp_marked = 1; 390 391 later += fp->ftp_marked; 392 fpp = &fp->ftp_next; 393 } else { 394 *fpp = fp->ftp_next; 395 fasttrap_provider_free(fp); 396 } 397 } 398 mutex_exit(&bucket->ftb_mtx); 399 } 400 mtx_lock(&fasttrap_cleanup_mtx); 401 402 /* 403 * If we were unable to retire a provider, try again after a 404 * second. This situation can occur in certain circumstances 405 * where providers cannot be unregistered even though they have 406 * no probes enabled because of an execution of dtrace -l or 407 * something similar. 408 */ 409 if (later > 0 || fasttrap_cleanup_work || 410 fasttrap_cleanup_drain) { 411 mtx_unlock(&fasttrap_cleanup_mtx); 412 pause("ftclean", hz); 413 mtx_lock(&fasttrap_cleanup_mtx); 414 } else 415 mtx_sleep(&fasttrap_cleanup_cv, &fasttrap_cleanup_mtx, 416 0, "ftcl", 0); 417 } 418 419 /* 420 * Wake up the thread in fasttrap_unload() now that we're done. 421 */ 422 wakeup(&fasttrap_cleanup_drain); 423 mtx_unlock(&fasttrap_cleanup_mtx); 424 425 kthread_exit(); 426} 427 428/* 429 * Activates the asynchronous cleanup mechanism. 430 */ 431static void 432fasttrap_pid_cleanup(void) 433{ 434 435 mtx_lock(&fasttrap_cleanup_mtx); 436 if (!fasttrap_cleanup_work) { 437 fasttrap_cleanup_work = 1; 438 wakeup(&fasttrap_cleanup_cv); 439 } 440 mtx_unlock(&fasttrap_cleanup_mtx); 441} 442 443/* 444 * This is called from cfork() via dtrace_fasttrap_fork(). The child 445 * process's address space is (roughly) a copy of the parent process's so 446 * we have to remove all the instrumentation we had previously enabled in the 447 * parent. 448 */ 449static void 450fasttrap_fork(proc_t *p, proc_t *cp) 451{
| 415 * This function ensures that no threads are actively using the memory 416 * associated with probes that were formerly live. 417 */ 418static void 419fasttrap_mod_barrier(uint64_t gen) 420{ 421 int i; 422 423 if (gen < fasttrap_mod_gen) 424 return; 425 426 fasttrap_mod_gen++; 427 428 CPU_FOREACH(i) { 429 mutex_enter(&fasttrap_cpuc_pid_lock[i]); 430 mutex_exit(&fasttrap_cpuc_pid_lock[i]); 431 } 432} 433 434/* 435 * This function performs asynchronous cleanup of fasttrap providers. The 436 * Solaris implementation of this mechanism use a timeout that's activated in 437 * fasttrap_pid_cleanup(), but this doesn't work in FreeBSD: one may sleep while 438 * holding the DTrace mutexes, but it is unsafe to sleep in a callout handler. 439 * Thus we use a dedicated process to perform the cleanup when requested. 440 */ 441/*ARGSUSED*/ 442static void 443fasttrap_pid_cleanup_cb(void *data) 444{ 445 fasttrap_provider_t **fpp, *fp; 446 fasttrap_bucket_t *bucket; 447 dtrace_provider_id_t provid; 448 int i, later = 0, rval; 449 450 mtx_lock(&fasttrap_cleanup_mtx); 451 while (!fasttrap_cleanup_drain || later > 0) { 452 fasttrap_cleanup_work = 0; 453 mtx_unlock(&fasttrap_cleanup_mtx); 454 455 later = 0; 456 457 /* 458 * Iterate over all the providers trying to remove the marked 459 * ones. If a provider is marked but not retired, we just 460 * have to take a crack at removing it -- it's no big deal if 461 * we can't. 462 */ 463 for (i = 0; i < fasttrap_provs.fth_nent; i++) { 464 bucket = &fasttrap_provs.fth_table[i]; 465 mutex_enter(&bucket->ftb_mtx); 466 fpp = (fasttrap_provider_t **)&bucket->ftb_data; 467 468 while ((fp = *fpp) != NULL) { 469 if (!fp->ftp_marked) { 470 fpp = &fp->ftp_next; 471 continue; 472 } 473 474 mutex_enter(&fp->ftp_mtx); 475 476 /* 477 * If this provider has consumers actively 478 * creating probes (ftp_ccount) or is a USDT 479 * provider (ftp_mcount), we can't unregister 480 * or even condense. 481 */ 482 if (fp->ftp_ccount != 0 || 483 fp->ftp_mcount != 0) { 484 mutex_exit(&fp->ftp_mtx); 485 fp->ftp_marked = 0; 486 continue; 487 } 488 489 if (!fp->ftp_retired || fp->ftp_rcount != 0) 490 fp->ftp_marked = 0; 491 492 mutex_exit(&fp->ftp_mtx); 493 494 /* 495 * If we successfully unregister this 496 * provider we can remove it from the hash 497 * chain and free the memory. If our attempt 498 * to unregister fails and this is a retired 499 * provider, increment our flag to try again 500 * pretty soon. If we've consumed more than 501 * half of our total permitted number of 502 * probes call dtrace_condense() to try to 503 * clean out the unenabled probes. 504 */ 505 provid = fp->ftp_provid; 506 if ((rval = dtrace_unregister(provid)) != 0) { 507 if (fasttrap_total > fasttrap_max / 2) 508 (void) dtrace_condense(provid); 509 510 if (rval == EAGAIN) 511 fp->ftp_marked = 1; 512 513 later += fp->ftp_marked; 514 fpp = &fp->ftp_next; 515 } else { 516 *fpp = fp->ftp_next; 517 fasttrap_provider_free(fp); 518 } 519 } 520 mutex_exit(&bucket->ftb_mtx); 521 } 522 mtx_lock(&fasttrap_cleanup_mtx); 523 524 /* 525 * If we were unable to retire a provider, try again after a 526 * second. This situation can occur in certain circumstances 527 * where providers cannot be unregistered even though they have 528 * no probes enabled because of an execution of dtrace -l or 529 * something similar. 530 */ 531 if (later > 0 || fasttrap_cleanup_work || 532 fasttrap_cleanup_drain) { 533 mtx_unlock(&fasttrap_cleanup_mtx); 534 pause("ftclean", hz); 535 mtx_lock(&fasttrap_cleanup_mtx); 536 } else 537 mtx_sleep(&fasttrap_cleanup_cv, &fasttrap_cleanup_mtx, 538 0, "ftcl", 0); 539 } 540 541 /* 542 * Wake up the thread in fasttrap_unload() now that we're done. 543 */ 544 wakeup(&fasttrap_cleanup_drain); 545 mtx_unlock(&fasttrap_cleanup_mtx); 546 547 kthread_exit(); 548} 549 550/* 551 * Activates the asynchronous cleanup mechanism. 552 */ 553static void 554fasttrap_pid_cleanup(void) 555{ 556 557 mtx_lock(&fasttrap_cleanup_mtx); 558 if (!fasttrap_cleanup_work) { 559 fasttrap_cleanup_work = 1; 560 wakeup(&fasttrap_cleanup_cv); 561 } 562 mtx_unlock(&fasttrap_cleanup_mtx); 563} 564 565/* 566 * This is called from cfork() via dtrace_fasttrap_fork(). The child 567 * process's address space is (roughly) a copy of the parent process's so 568 * we have to remove all the instrumentation we had previously enabled in the 569 * parent. 570 */ 571static void 572fasttrap_fork(proc_t *p, proc_t *cp) 573{
|
| 574#if !defined(sun) 575 fasttrap_scrblock_t *scrblk; 576 fasttrap_proc_t *fprc = NULL; 577#endif
|
452 pid_t ppid = p->p_pid; 453 int i; 454 455#if defined(sun) 456 ASSERT(curproc == p); 457 ASSERT(p->p_proc_flag & P_PR_LOCK); 458#else 459 PROC_LOCK_ASSERT(p, MA_OWNED); 460#endif 461#if defined(sun) 462 ASSERT(p->p_dtrace_count > 0); 463#else 464 if (p->p_dtrace_helpers) { 465 /* 466 * dtrace_helpers_duplicate() allocates memory. 467 */ 468 _PHOLD(cp); 469 PROC_UNLOCK(p); 470 PROC_UNLOCK(cp); 471 dtrace_helpers_duplicate(p, cp); 472 PROC_LOCK(cp); 473 PROC_LOCK(p); 474 _PRELE(cp); 475 } 476 /* 477 * This check is purposely here instead of in kern_fork.c because, 478 * for legal resons, we cannot include the dtrace_cddl.h header 479 * inside kern_fork.c and insert if-clause there. 480 */ 481 if (p->p_dtrace_count == 0) 482 return; 483#endif 484 ASSERT(cp->p_dtrace_count == 0); 485 486 /* 487 * This would be simpler and faster if we maintained per-process 488 * hash tables of enabled tracepoints. It could, however, potentially 489 * slow down execution of a tracepoint since we'd need to go 490 * through two levels of indirection. In the future, we should 491 * consider either maintaining per-process ancillary lists of 492 * enabled tracepoints or hanging a pointer to a per-process hash 493 * table of enabled tracepoints off the proc structure. 494 */ 495 496 /* 497 * We don't have to worry about the child process disappearing 498 * because we're in fork(). 499 */ 500#if defined(sun) 501 mtx_lock_spin(&cp->p_slock); 502 sprlock_proc(cp); 503 mtx_unlock_spin(&cp->p_slock); 504#else 505 /* 506 * fasttrap_tracepoint_remove() expects the child process to be 507 * unlocked and the VM then expects curproc to be unlocked. 508 */ 509 _PHOLD(cp); 510 PROC_UNLOCK(cp); 511 PROC_UNLOCK(p); 512#endif 513 514 /* 515 * Iterate over every tracepoint looking for ones that belong to the 516 * parent process, and remove each from the child process. 517 */ 518 for (i = 0; i < fasttrap_tpoints.fth_nent; i++) { 519 fasttrap_tracepoint_t *tp; 520 fasttrap_bucket_t *bucket = &fasttrap_tpoints.fth_table[i]; 521 522 mutex_enter(&bucket->ftb_mtx); 523 for (tp = bucket->ftb_data; tp != NULL; tp = tp->ftt_next) { 524 if (tp->ftt_pid == ppid && 525 tp->ftt_proc->ftpc_acount != 0) { 526 int ret = fasttrap_tracepoint_remove(cp, tp); 527 ASSERT(ret == 0); 528 529 /* 530 * The count of active providers can only be 531 * decremented (i.e. to zero) during exec, 532 * exit, and removal of a meta provider so it 533 * should be impossible to drop the count 534 * mid-fork. 535 */ 536 ASSERT(tp->ftt_proc->ftpc_acount != 0);
| 578 pid_t ppid = p->p_pid; 579 int i; 580 581#if defined(sun) 582 ASSERT(curproc == p); 583 ASSERT(p->p_proc_flag & P_PR_LOCK); 584#else 585 PROC_LOCK_ASSERT(p, MA_OWNED); 586#endif 587#if defined(sun) 588 ASSERT(p->p_dtrace_count > 0); 589#else 590 if (p->p_dtrace_helpers) { 591 /* 592 * dtrace_helpers_duplicate() allocates memory. 593 */ 594 _PHOLD(cp); 595 PROC_UNLOCK(p); 596 PROC_UNLOCK(cp); 597 dtrace_helpers_duplicate(p, cp); 598 PROC_LOCK(cp); 599 PROC_LOCK(p); 600 _PRELE(cp); 601 } 602 /* 603 * This check is purposely here instead of in kern_fork.c because, 604 * for legal resons, we cannot include the dtrace_cddl.h header 605 * inside kern_fork.c and insert if-clause there. 606 */ 607 if (p->p_dtrace_count == 0) 608 return; 609#endif 610 ASSERT(cp->p_dtrace_count == 0); 611 612 /* 613 * This would be simpler and faster if we maintained per-process 614 * hash tables of enabled tracepoints. It could, however, potentially 615 * slow down execution of a tracepoint since we'd need to go 616 * through two levels of indirection. In the future, we should 617 * consider either maintaining per-process ancillary lists of 618 * enabled tracepoints or hanging a pointer to a per-process hash 619 * table of enabled tracepoints off the proc structure. 620 */ 621 622 /* 623 * We don't have to worry about the child process disappearing 624 * because we're in fork(). 625 */ 626#if defined(sun) 627 mtx_lock_spin(&cp->p_slock); 628 sprlock_proc(cp); 629 mtx_unlock_spin(&cp->p_slock); 630#else 631 /* 632 * fasttrap_tracepoint_remove() expects the child process to be 633 * unlocked and the VM then expects curproc to be unlocked. 634 */ 635 _PHOLD(cp); 636 PROC_UNLOCK(cp); 637 PROC_UNLOCK(p); 638#endif 639 640 /* 641 * Iterate over every tracepoint looking for ones that belong to the 642 * parent process, and remove each from the child process. 643 */ 644 for (i = 0; i < fasttrap_tpoints.fth_nent; i++) { 645 fasttrap_tracepoint_t *tp; 646 fasttrap_bucket_t *bucket = &fasttrap_tpoints.fth_table[i]; 647 648 mutex_enter(&bucket->ftb_mtx); 649 for (tp = bucket->ftb_data; tp != NULL; tp = tp->ftt_next) { 650 if (tp->ftt_pid == ppid && 651 tp->ftt_proc->ftpc_acount != 0) { 652 int ret = fasttrap_tracepoint_remove(cp, tp); 653 ASSERT(ret == 0); 654 655 /* 656 * The count of active providers can only be 657 * decremented (i.e. to zero) during exec, 658 * exit, and removal of a meta provider so it 659 * should be impossible to drop the count 660 * mid-fork. 661 */ 662 ASSERT(tp->ftt_proc->ftpc_acount != 0);
|
| 663#if !defined(sun) 664 fprc = tp->ftt_proc; 665#endif
|
537 } 538 } 539 mutex_exit(&bucket->ftb_mtx);
| 666 } 667 } 668 mutex_exit(&bucket->ftb_mtx);
|
| 669 670#if !defined(sun) 671 /* 672 * Unmap any scratch space inherited from the parent's address 673 * space. 674 */ 675 if (fprc != NULL) { 676 mutex_enter(&fprc->ftpc_mtx); 677 LIST_FOREACH(scrblk, &fprc->ftpc_scrblks, ftsb_next) { 678 vm_map_remove(&cp->p_vmspace->vm_map, 679 scrblk->ftsb_addr, 680 scrblk->ftsb_addr + FASTTRAP_SCRBLOCK_SIZE); 681 } 682 mutex_exit(&fprc->ftpc_mtx); 683 } 684#endif
|
540 } 541 542#if defined(sun) 543 mutex_enter(&cp->p_lock); 544 sprunlock(cp); 545#else 546 PROC_LOCK(p); 547 PROC_LOCK(cp); 548 _PRELE(cp); 549#endif 550} 551 552/* 553 * This is called from proc_exit() or from exec_common() if p_dtrace_probes 554 * is set on the proc structure to indicate that there is a pid provider 555 * associated with this process. 556 */ 557static void 558fasttrap_exec_exit(proc_t *p) 559{
| 685 } 686 687#if defined(sun) 688 mutex_enter(&cp->p_lock); 689 sprunlock(cp); 690#else 691 PROC_LOCK(p); 692 PROC_LOCK(cp); 693 _PRELE(cp); 694#endif 695} 696 697/* 698 * This is called from proc_exit() or from exec_common() if p_dtrace_probes 699 * is set on the proc structure to indicate that there is a pid provider 700 * associated with this process. 701 */ 702static void 703fasttrap_exec_exit(proc_t *p) 704{
|
| 705#if !defined(sun) 706 struct thread *td; 707#endif 708
|
560#if defined(sun) 561 ASSERT(p == curproc);
| 709#if defined(sun) 710 ASSERT(p == curproc);
|
562#endif
| 711#else
|
563 PROC_LOCK_ASSERT(p, MA_OWNED); 564 _PHOLD(p);
| 712 PROC_LOCK_ASSERT(p, MA_OWNED); 713 _PHOLD(p);
|
| 714 /* 715 * Since struct threads may be recycled, we cannot rely on t_dtrace_sscr 716 * fields to be zeroed by kdtrace_thread_ctor. Thus we must zero it 717 * ourselves when a process exits. 718 */ 719 FOREACH_THREAD_IN_PROC(p, td) 720 td->t_dtrace_sscr = NULL;
|
565 PROC_UNLOCK(p);
| 721 PROC_UNLOCK(p);
|
| 722#endif
|
566 567 /* 568 * We clean up the pid provider for this process here; user-land 569 * static probes are handled by the meta-provider remove entry point. 570 */ 571 fasttrap_provider_retire(p->p_pid, FASTTRAP_PID_NAME, 0); 572#if !defined(sun) 573 if (p->p_dtrace_helpers) 574 dtrace_helpers_destroy(p);
| 723 724 /* 725 * We clean up the pid provider for this process here; user-land 726 * static probes are handled by the meta-provider remove entry point. 727 */ 728 fasttrap_provider_retire(p->p_pid, FASTTRAP_PID_NAME, 0); 729#if !defined(sun) 730 if (p->p_dtrace_helpers) 731 dtrace_helpers_destroy(p);
|
575#endif
| |
576 PROC_LOCK(p); 577 _PRELE(p);
| 732 PROC_LOCK(p); 733 _PRELE(p);
|
| 734#endif
|
578} 579 580 581/*ARGSUSED*/ 582static void 583fasttrap_pid_provide(void *arg, dtrace_probedesc_t *desc) 584{ 585 /* 586 * There are no "default" pid probes. 587 */ 588} 589 590static int 591fasttrap_tracepoint_enable(proc_t *p, fasttrap_probe_t *probe, uint_t index) 592{ 593 fasttrap_tracepoint_t *tp, *new_tp = NULL; 594 fasttrap_bucket_t *bucket; 595 fasttrap_id_t *id; 596 pid_t pid; 597 uintptr_t pc; 598 599 ASSERT(index < probe->ftp_ntps); 600 601 pid = probe->ftp_pid; 602 pc = probe->ftp_tps[index].fit_tp->ftt_pc; 603 id = &probe->ftp_tps[index].fit_id; 604 605 ASSERT(probe->ftp_tps[index].fit_tp->ftt_pid == pid); 606 607#if defined(sun) 608 ASSERT(!(p->p_flag & SVFORK)); 609#endif 610 611 /* 612 * Before we make any modifications, make sure we've imposed a barrier 613 * on the generation in which this probe was last modified. 614 */ 615 fasttrap_mod_barrier(probe->ftp_gen); 616 617 bucket = &fasttrap_tpoints.fth_table[FASTTRAP_TPOINTS_INDEX(pid, pc)]; 618 619 /* 620 * If the tracepoint has already been enabled, just add our id to the 621 * list of interested probes. This may be our second time through 622 * this path in which case we'll have constructed the tracepoint we'd 623 * like to install. If we can't find a match, and have an allocated 624 * tracepoint ready to go, enable that one now. 625 * 626 * A tracepoint whose process is defunct is also considered defunct. 627 */ 628again: 629 mutex_enter(&bucket->ftb_mtx); 630 for (tp = bucket->ftb_data; tp != NULL; tp = tp->ftt_next) { 631 /* 632 * Note that it's safe to access the active count on the 633 * associated proc structure because we know that at least one 634 * provider (this one) will still be around throughout this 635 * operation. 636 */ 637 if (tp->ftt_pid != pid || tp->ftt_pc != pc || 638 tp->ftt_proc->ftpc_acount == 0) 639 continue; 640 641 /* 642 * Now that we've found a matching tracepoint, it would be 643 * a decent idea to confirm that the tracepoint is still 644 * enabled and the trap instruction hasn't been overwritten. 645 * Since this is a little hairy, we'll punt for now. 646 */ 647 648 /* 649 * This can't be the first interested probe. We don't have 650 * to worry about another thread being in the midst of 651 * deleting this tracepoint (which would be the only valid 652 * reason for a tracepoint to have no interested probes) 653 * since we're holding P_PR_LOCK for this process. 654 */ 655 ASSERT(tp->ftt_ids != NULL || tp->ftt_retids != NULL); 656 657 switch (id->fti_ptype) { 658 case DTFTP_ENTRY: 659 case DTFTP_OFFSETS: 660 case DTFTP_IS_ENABLED: 661 id->fti_next = tp->ftt_ids; 662 membar_producer(); 663 tp->ftt_ids = id; 664 membar_producer(); 665 break; 666 667 case DTFTP_RETURN: 668 case DTFTP_POST_OFFSETS: 669 id->fti_next = tp->ftt_retids; 670 membar_producer(); 671 tp->ftt_retids = id; 672 membar_producer(); 673 break; 674 675 default: 676 ASSERT(0); 677 } 678 679 mutex_exit(&bucket->ftb_mtx); 680 681 if (new_tp != NULL) { 682 new_tp->ftt_ids = NULL; 683 new_tp->ftt_retids = NULL; 684 } 685 686 return (0); 687 } 688 689 /* 690 * If we have a good tracepoint ready to go, install it now while 691 * we have the lock held and no one can screw with us. 692 */ 693 if (new_tp != NULL) { 694 int rc = 0; 695 696 new_tp->ftt_next = bucket->ftb_data; 697 membar_producer(); 698 bucket->ftb_data = new_tp; 699 membar_producer(); 700 mutex_exit(&bucket->ftb_mtx); 701 702 /* 703 * Activate the tracepoint in the ISA-specific manner. 704 * If this fails, we need to report the failure, but 705 * indicate that this tracepoint must still be disabled 706 * by calling fasttrap_tracepoint_disable(). 707 */ 708 if (fasttrap_tracepoint_install(p, new_tp) != 0) 709 rc = FASTTRAP_ENABLE_PARTIAL; 710 711 /* 712 * Increment the count of the number of tracepoints active in 713 * the victim process. 714 */ 715#if defined(sun) 716 ASSERT(p->p_proc_flag & P_PR_LOCK); 717#endif 718 p->p_dtrace_count++; 719 720 return (rc); 721 } 722 723 mutex_exit(&bucket->ftb_mtx); 724 725 /* 726 * Initialize the tracepoint that's been preallocated with the probe. 727 */ 728 new_tp = probe->ftp_tps[index].fit_tp; 729 730 ASSERT(new_tp->ftt_pid == pid); 731 ASSERT(new_tp->ftt_pc == pc); 732 ASSERT(new_tp->ftt_proc == probe->ftp_prov->ftp_proc); 733 ASSERT(new_tp->ftt_ids == NULL); 734 ASSERT(new_tp->ftt_retids == NULL); 735 736 switch (id->fti_ptype) { 737 case DTFTP_ENTRY: 738 case DTFTP_OFFSETS: 739 case DTFTP_IS_ENABLED: 740 id->fti_next = NULL; 741 new_tp->ftt_ids = id; 742 break; 743 744 case DTFTP_RETURN: 745 case DTFTP_POST_OFFSETS: 746 id->fti_next = NULL; 747 new_tp->ftt_retids = id; 748 break; 749 750 default: 751 ASSERT(0); 752 } 753 754 /* 755 * If the ISA-dependent initialization goes to plan, go back to the 756 * beginning and try to install this freshly made tracepoint. 757 */ 758 if (fasttrap_tracepoint_init(p, new_tp, pc, id->fti_ptype) == 0) 759 goto again; 760 761 new_tp->ftt_ids = NULL; 762 new_tp->ftt_retids = NULL; 763 764 return (FASTTRAP_ENABLE_FAIL); 765} 766 767static void 768fasttrap_tracepoint_disable(proc_t *p, fasttrap_probe_t *probe, uint_t index) 769{ 770 fasttrap_bucket_t *bucket; 771 fasttrap_provider_t *provider = probe->ftp_prov; 772 fasttrap_tracepoint_t **pp, *tp; 773 fasttrap_id_t *id, **idp = NULL; 774 pid_t pid; 775 uintptr_t pc; 776 777 ASSERT(index < probe->ftp_ntps); 778 779 pid = probe->ftp_pid; 780 pc = probe->ftp_tps[index].fit_tp->ftt_pc; 781 id = &probe->ftp_tps[index].fit_id; 782 783 ASSERT(probe->ftp_tps[index].fit_tp->ftt_pid == pid); 784 785 /* 786 * Find the tracepoint and make sure that our id is one of the 787 * ones registered with it. 788 */ 789 bucket = &fasttrap_tpoints.fth_table[FASTTRAP_TPOINTS_INDEX(pid, pc)]; 790 mutex_enter(&bucket->ftb_mtx); 791 for (tp = bucket->ftb_data; tp != NULL; tp = tp->ftt_next) { 792 if (tp->ftt_pid == pid && tp->ftt_pc == pc && 793 tp->ftt_proc == provider->ftp_proc) 794 break; 795 } 796 797 /* 798 * If we somehow lost this tracepoint, we're in a world of hurt. 799 */ 800 ASSERT(tp != NULL); 801 802 switch (id->fti_ptype) { 803 case DTFTP_ENTRY: 804 case DTFTP_OFFSETS: 805 case DTFTP_IS_ENABLED: 806 ASSERT(tp->ftt_ids != NULL); 807 idp = &tp->ftt_ids; 808 break; 809 810 case DTFTP_RETURN: 811 case DTFTP_POST_OFFSETS: 812 ASSERT(tp->ftt_retids != NULL); 813 idp = &tp->ftt_retids; 814 break; 815 816 default: 817 ASSERT(0); 818 } 819 820 while ((*idp)->fti_probe != probe) { 821 idp = &(*idp)->fti_next; 822 ASSERT(*idp != NULL); 823 } 824 825 id = *idp; 826 *idp = id->fti_next; 827 membar_producer(); 828 829 ASSERT(id->fti_probe == probe); 830 831 /* 832 * If there are other registered enablings of this tracepoint, we're 833 * all done, but if this was the last probe assocated with this 834 * this tracepoint, we need to remove and free it. 835 */ 836 if (tp->ftt_ids != NULL || tp->ftt_retids != NULL) { 837 838 /* 839 * If the current probe's tracepoint is in use, swap it 840 * for an unused tracepoint. 841 */ 842 if (tp == probe->ftp_tps[index].fit_tp) { 843 fasttrap_probe_t *tmp_probe; 844 fasttrap_tracepoint_t **tmp_tp; 845 uint_t tmp_index; 846 847 if (tp->ftt_ids != NULL) { 848 tmp_probe = tp->ftt_ids->fti_probe; 849 /* LINTED - alignment */ 850 tmp_index = FASTTRAP_ID_INDEX(tp->ftt_ids); 851 tmp_tp = &tmp_probe->ftp_tps[tmp_index].fit_tp; 852 } else { 853 tmp_probe = tp->ftt_retids->fti_probe; 854 /* LINTED - alignment */ 855 tmp_index = FASTTRAP_ID_INDEX(tp->ftt_retids); 856 tmp_tp = &tmp_probe->ftp_tps[tmp_index].fit_tp; 857 } 858 859 ASSERT(*tmp_tp != NULL); 860 ASSERT(*tmp_tp != probe->ftp_tps[index].fit_tp); 861 ASSERT((*tmp_tp)->ftt_ids == NULL); 862 ASSERT((*tmp_tp)->ftt_retids == NULL); 863 864 probe->ftp_tps[index].fit_tp = *tmp_tp; 865 *tmp_tp = tp; 866 } 867 868 mutex_exit(&bucket->ftb_mtx); 869 870 /* 871 * Tag the modified probe with the generation in which it was 872 * changed. 873 */ 874 probe->ftp_gen = fasttrap_mod_gen; 875 return; 876 } 877 878 mutex_exit(&bucket->ftb_mtx); 879 880 /* 881 * We can't safely remove the tracepoint from the set of active 882 * tracepoints until we've actually removed the fasttrap instruction 883 * from the process's text. We can, however, operate on this 884 * tracepoint secure in the knowledge that no other thread is going to 885 * be looking at it since we hold P_PR_LOCK on the process if it's 886 * live or we hold the provider lock on the process if it's dead and 887 * gone. 888 */ 889 890 /* 891 * We only need to remove the actual instruction if we're looking 892 * at an existing process 893 */ 894 if (p != NULL) { 895 /* 896 * If we fail to restore the instruction we need to kill 897 * this process since it's in a completely unrecoverable 898 * state. 899 */ 900 if (fasttrap_tracepoint_remove(p, tp) != 0) 901 fasttrap_sigtrap(p, NULL, pc); 902 903 /* 904 * Decrement the count of the number of tracepoints active 905 * in the victim process. 906 */ 907#if defined(sun) 908 ASSERT(p->p_proc_flag & P_PR_LOCK); 909#endif 910 p->p_dtrace_count--; 911 } 912 913 /* 914 * Remove the probe from the hash table of active tracepoints. 915 */ 916 mutex_enter(&bucket->ftb_mtx); 917 pp = (fasttrap_tracepoint_t **)&bucket->ftb_data; 918 ASSERT(*pp != NULL); 919 while (*pp != tp) { 920 pp = &(*pp)->ftt_next; 921 ASSERT(*pp != NULL); 922 } 923 924 *pp = tp->ftt_next; 925 membar_producer(); 926 927 mutex_exit(&bucket->ftb_mtx); 928 929 /* 930 * Tag the modified probe with the generation in which it was changed. 931 */ 932 probe->ftp_gen = fasttrap_mod_gen; 933} 934 935static void 936fasttrap_enable_callbacks(void) 937{ 938 /* 939 * We don't have to play the rw lock game here because we're 940 * providing something rather than taking something away -- 941 * we can be sure that no threads have tried to follow this 942 * function pointer yet. 943 */ 944 mutex_enter(&fasttrap_count_mtx); 945 if (fasttrap_pid_count == 0) { 946 ASSERT(dtrace_pid_probe_ptr == NULL); 947 ASSERT(dtrace_return_probe_ptr == NULL); 948 dtrace_pid_probe_ptr = &fasttrap_pid_probe; 949 dtrace_return_probe_ptr = &fasttrap_return_probe; 950 } 951 ASSERT(dtrace_pid_probe_ptr == &fasttrap_pid_probe); 952 ASSERT(dtrace_return_probe_ptr == &fasttrap_return_probe); 953 fasttrap_pid_count++; 954 mutex_exit(&fasttrap_count_mtx); 955} 956 957static void 958fasttrap_disable_callbacks(void) 959{ 960#if defined(sun) 961 ASSERT(MUTEX_HELD(&cpu_lock)); 962#endif 963 964 965 mutex_enter(&fasttrap_count_mtx); 966 ASSERT(fasttrap_pid_count > 0); 967 fasttrap_pid_count--; 968 if (fasttrap_pid_count == 0) { 969#if defined(sun) 970 cpu_t *cur, *cpu = CPU; 971 972 for (cur = cpu->cpu_next_onln; cur != cpu; 973 cur = cur->cpu_next_onln) { 974 rw_enter(&cur->cpu_ft_lock, RW_WRITER); 975 } 976#endif 977 dtrace_pid_probe_ptr = NULL; 978 dtrace_return_probe_ptr = NULL; 979#if defined(sun) 980 for (cur = cpu->cpu_next_onln; cur != cpu; 981 cur = cur->cpu_next_onln) { 982 rw_exit(&cur->cpu_ft_lock); 983 } 984#endif 985 } 986 mutex_exit(&fasttrap_count_mtx); 987} 988 989/*ARGSUSED*/ 990static void 991fasttrap_pid_enable(void *arg, dtrace_id_t id, void *parg) 992{ 993 fasttrap_probe_t *probe = parg; 994 proc_t *p = NULL; 995 int i, rc; 996 997 ASSERT(probe != NULL); 998 ASSERT(!probe->ftp_enabled); 999 ASSERT(id == probe->ftp_id); 1000#if defined(sun) 1001 ASSERT(MUTEX_HELD(&cpu_lock)); 1002#endif 1003 1004 /* 1005 * Increment the count of enabled probes on this probe's provider; 1006 * the provider can't go away while the probe still exists. We 1007 * must increment this even if we aren't able to properly enable 1008 * this probe. 1009 */ 1010 mutex_enter(&probe->ftp_prov->ftp_mtx); 1011 probe->ftp_prov->ftp_rcount++; 1012 mutex_exit(&probe->ftp_prov->ftp_mtx); 1013 1014 /* 1015 * If this probe's provider is retired (meaning it was valid in a 1016 * previously exec'ed incarnation of this address space), bail out. The 1017 * provider can't go away while we're in this code path. 1018 */ 1019 if (probe->ftp_prov->ftp_retired) 1020 return; 1021 1022 /* 1023 * If we can't find the process, it may be that we're in the context of 1024 * a fork in which the traced process is being born and we're copying 1025 * USDT probes. Otherwise, the process is gone so bail. 1026 */ 1027#if defined(sun) 1028 if ((p = sprlock(probe->ftp_pid)) == NULL) { 1029 if ((curproc->p_flag & SFORKING) == 0) 1030 return; 1031 1032 mutex_enter(&pidlock); 1033 p = prfind(probe->ftp_pid); 1034 1035 /* 1036 * Confirm that curproc is indeed forking the process in which 1037 * we're trying to enable probes. 1038 */ 1039 ASSERT(p != NULL); 1040 ASSERT(p->p_parent == curproc); 1041 ASSERT(p->p_stat == SIDL); 1042 1043 mutex_enter(&p->p_lock); 1044 mutex_exit(&pidlock); 1045 1046 sprlock_proc(p); 1047 } 1048 1049 ASSERT(!(p->p_flag & SVFORK)); 1050 mutex_exit(&p->p_lock); 1051#else 1052 if ((p = pfind(probe->ftp_pid)) == NULL) 1053 return; 1054#endif 1055 1056 /* 1057 * We have to enable the trap entry point before any user threads have 1058 * the chance to execute the trap instruction we're about to place 1059 * in their process's text. 1060 */ 1061#ifdef __FreeBSD__ 1062 /* 1063 * pfind() returns a locked process. 1064 */ 1065 _PHOLD(p); 1066 PROC_UNLOCK(p); 1067#endif 1068 fasttrap_enable_callbacks(); 1069 1070 /* 1071 * Enable all the tracepoints and add this probe's id to each 1072 * tracepoint's list of active probes. 1073 */ 1074 for (i = 0; i < probe->ftp_ntps; i++) { 1075 if ((rc = fasttrap_tracepoint_enable(p, probe, i)) != 0) { 1076 /* 1077 * If enabling the tracepoint failed completely, 1078 * we don't have to disable it; if the failure 1079 * was only partial we must disable it. 1080 */ 1081 if (rc == FASTTRAP_ENABLE_FAIL) 1082 i--; 1083 else 1084 ASSERT(rc == FASTTRAP_ENABLE_PARTIAL); 1085 1086 /* 1087 * Back up and pull out all the tracepoints we've 1088 * created so far for this probe. 1089 */ 1090 while (i >= 0) { 1091 fasttrap_tracepoint_disable(p, probe, i); 1092 i--; 1093 } 1094 1095#if defined(sun) 1096 mutex_enter(&p->p_lock); 1097 sprunlock(p); 1098#else 1099 PRELE(p); 1100#endif 1101 1102 /* 1103 * Since we're not actually enabling this probe, 1104 * drop our reference on the trap table entry. 1105 */ 1106 fasttrap_disable_callbacks(); 1107 return; 1108 } 1109 } 1110#if defined(sun) 1111 mutex_enter(&p->p_lock); 1112 sprunlock(p); 1113#else 1114 PRELE(p); 1115#endif 1116 1117 probe->ftp_enabled = 1; 1118} 1119 1120/*ARGSUSED*/ 1121static void 1122fasttrap_pid_disable(void *arg, dtrace_id_t id, void *parg) 1123{ 1124 fasttrap_probe_t *probe = parg; 1125 fasttrap_provider_t *provider = probe->ftp_prov; 1126 proc_t *p; 1127 int i, whack = 0; 1128 1129 ASSERT(id == probe->ftp_id); 1130 1131 mutex_enter(&provider->ftp_mtx); 1132 1133 /* 1134 * We won't be able to acquire a /proc-esque lock on the process 1135 * iff the process is dead and gone. In this case, we rely on the 1136 * provider lock as a point of mutual exclusion to prevent other 1137 * DTrace consumers from disabling this probe. 1138 */ 1139 if ((p = pfind(probe->ftp_pid)) != NULL) { 1140#ifdef __FreeBSD__ 1141 _PHOLD(p); 1142 PROC_UNLOCK(p); 1143#endif 1144 } 1145 1146 /* 1147 * Disable all the associated tracepoints (for fully enabled probes). 1148 */ 1149 if (probe->ftp_enabled) { 1150 for (i = 0; i < probe->ftp_ntps; i++) { 1151 fasttrap_tracepoint_disable(p, probe, i); 1152 } 1153 } 1154 1155 ASSERT(provider->ftp_rcount > 0); 1156 provider->ftp_rcount--; 1157 1158 if (p != NULL) { 1159 /* 1160 * Even though we may not be able to remove it entirely, we 1161 * mark this retired provider to get a chance to remove some 1162 * of the associated probes. 1163 */ 1164 if (provider->ftp_retired && !provider->ftp_marked) 1165 whack = provider->ftp_marked = 1; 1166 mutex_exit(&provider->ftp_mtx); 1167 } else { 1168 /* 1169 * If the process is dead, we're just waiting for the 1170 * last probe to be disabled to be able to free it. 1171 */ 1172 if (provider->ftp_rcount == 0 && !provider->ftp_marked) 1173 whack = provider->ftp_marked = 1; 1174 mutex_exit(&provider->ftp_mtx); 1175 } 1176 1177 if (whack) 1178 fasttrap_pid_cleanup(); 1179 1180#ifdef __FreeBSD__ 1181 if (p != NULL) 1182 PRELE(p); 1183#endif 1184 if (!probe->ftp_enabled) 1185 return; 1186 1187 probe->ftp_enabled = 0; 1188 1189#if defined(sun) 1190 ASSERT(MUTEX_HELD(&cpu_lock)); 1191#endif 1192 fasttrap_disable_callbacks(); 1193} 1194 1195/*ARGSUSED*/ 1196static void 1197fasttrap_pid_getargdesc(void *arg, dtrace_id_t id, void *parg, 1198 dtrace_argdesc_t *desc) 1199{ 1200 fasttrap_probe_t *probe = parg; 1201 char *str; 1202 int i, ndx; 1203 1204 desc->dtargd_native[0] = '\0'; 1205 desc->dtargd_xlate[0] = '\0'; 1206 1207 if (probe->ftp_prov->ftp_retired != 0 || 1208 desc->dtargd_ndx >= probe->ftp_nargs) { 1209 desc->dtargd_ndx = DTRACE_ARGNONE; 1210 return; 1211 } 1212 1213 ndx = (probe->ftp_argmap != NULL) ? 1214 probe->ftp_argmap[desc->dtargd_ndx] : desc->dtargd_ndx; 1215 1216 str = probe->ftp_ntypes; 1217 for (i = 0; i < ndx; i++) { 1218 str += strlen(str) + 1; 1219 } 1220 1221 ASSERT(strlen(str + 1) < sizeof (desc->dtargd_native)); 1222 (void) strcpy(desc->dtargd_native, str); 1223 1224 if (probe->ftp_xtypes == NULL) 1225 return; 1226 1227 str = probe->ftp_xtypes; 1228 for (i = 0; i < desc->dtargd_ndx; i++) { 1229 str += strlen(str) + 1; 1230 } 1231 1232 ASSERT(strlen(str + 1) < sizeof (desc->dtargd_xlate)); 1233 (void) strcpy(desc->dtargd_xlate, str); 1234} 1235 1236/*ARGSUSED*/ 1237static void 1238fasttrap_pid_destroy(void *arg, dtrace_id_t id, void *parg) 1239{ 1240 fasttrap_probe_t *probe = parg; 1241 int i; 1242 size_t size; 1243 1244 ASSERT(probe != NULL); 1245 ASSERT(!probe->ftp_enabled); 1246 ASSERT(fasttrap_total >= probe->ftp_ntps); 1247 1248 atomic_add_32(&fasttrap_total, -probe->ftp_ntps); 1249 size = offsetof(fasttrap_probe_t, ftp_tps[probe->ftp_ntps]); 1250 1251 if (probe->ftp_gen + 1 >= fasttrap_mod_gen) 1252 fasttrap_mod_barrier(probe->ftp_gen); 1253 1254 for (i = 0; i < probe->ftp_ntps; i++) { 1255 kmem_free(probe->ftp_tps[i].fit_tp, 1256 sizeof (fasttrap_tracepoint_t)); 1257 } 1258 1259 kmem_free(probe, size); 1260} 1261 1262 1263static const dtrace_pattr_t pid_attr = { 1264{ DTRACE_STABILITY_EVOLVING, DTRACE_STABILITY_EVOLVING, DTRACE_CLASS_ISA }, 1265{ DTRACE_STABILITY_PRIVATE, DTRACE_STABILITY_PRIVATE, DTRACE_CLASS_UNKNOWN }, 1266{ DTRACE_STABILITY_PRIVATE, DTRACE_STABILITY_PRIVATE, DTRACE_CLASS_UNKNOWN }, 1267{ DTRACE_STABILITY_EVOLVING, DTRACE_STABILITY_EVOLVING, DTRACE_CLASS_ISA }, 1268{ DTRACE_STABILITY_PRIVATE, DTRACE_STABILITY_PRIVATE, DTRACE_CLASS_UNKNOWN }, 1269}; 1270 1271static dtrace_pops_t pid_pops = { 1272 fasttrap_pid_provide, 1273 NULL, 1274 fasttrap_pid_enable, 1275 fasttrap_pid_disable, 1276 NULL, 1277 NULL, 1278 fasttrap_pid_getargdesc, 1279 fasttrap_pid_getarg, 1280 NULL, 1281 fasttrap_pid_destroy 1282}; 1283 1284static dtrace_pops_t usdt_pops = { 1285 fasttrap_pid_provide, 1286 NULL, 1287 fasttrap_pid_enable, 1288 fasttrap_pid_disable, 1289 NULL, 1290 NULL, 1291 fasttrap_pid_getargdesc, 1292 fasttrap_usdt_getarg, 1293 NULL, 1294 fasttrap_pid_destroy 1295}; 1296 1297static fasttrap_proc_t * 1298fasttrap_proc_lookup(pid_t pid) 1299{ 1300 fasttrap_bucket_t *bucket; 1301 fasttrap_proc_t *fprc, *new_fprc; 1302 1303 1304 bucket = &fasttrap_procs.fth_table[FASTTRAP_PROCS_INDEX(pid)]; 1305 mutex_enter(&bucket->ftb_mtx); 1306 1307 for (fprc = bucket->ftb_data; fprc != NULL; fprc = fprc->ftpc_next) { 1308 if (fprc->ftpc_pid == pid && fprc->ftpc_acount != 0) { 1309 mutex_enter(&fprc->ftpc_mtx); 1310 mutex_exit(&bucket->ftb_mtx); 1311 fprc->ftpc_rcount++; 1312 atomic_add_64(&fprc->ftpc_acount, 1); 1313 ASSERT(fprc->ftpc_acount <= fprc->ftpc_rcount); 1314 mutex_exit(&fprc->ftpc_mtx); 1315 1316 return (fprc); 1317 } 1318 } 1319 1320 /* 1321 * Drop the bucket lock so we don't try to perform a sleeping 1322 * allocation under it. 1323 */ 1324 mutex_exit(&bucket->ftb_mtx); 1325 1326 new_fprc = kmem_zalloc(sizeof (fasttrap_proc_t), KM_SLEEP); 1327 new_fprc->ftpc_pid = pid; 1328 new_fprc->ftpc_rcount = 1; 1329 new_fprc->ftpc_acount = 1; 1330#if !defined(sun) 1331 mutex_init(&new_fprc->ftpc_mtx, "fasttrap proc mtx", MUTEX_DEFAULT, 1332 NULL); 1333#endif 1334 1335 mutex_enter(&bucket->ftb_mtx); 1336 1337 /* 1338 * Take another lap through the list to make sure a proc hasn't 1339 * been created for this pid while we weren't under the bucket lock. 1340 */ 1341 for (fprc = bucket->ftb_data; fprc != NULL; fprc = fprc->ftpc_next) { 1342 if (fprc->ftpc_pid == pid && fprc->ftpc_acount != 0) { 1343 mutex_enter(&fprc->ftpc_mtx); 1344 mutex_exit(&bucket->ftb_mtx); 1345 fprc->ftpc_rcount++; 1346 atomic_add_64(&fprc->ftpc_acount, 1); 1347 ASSERT(fprc->ftpc_acount <= fprc->ftpc_rcount); 1348 mutex_exit(&fprc->ftpc_mtx); 1349 1350 kmem_free(new_fprc, sizeof (fasttrap_proc_t)); 1351 1352 return (fprc); 1353 } 1354 } 1355 1356 new_fprc->ftpc_next = bucket->ftb_data; 1357 bucket->ftb_data = new_fprc; 1358 1359 mutex_exit(&bucket->ftb_mtx); 1360 1361 return (new_fprc); 1362} 1363 1364static void 1365fasttrap_proc_release(fasttrap_proc_t *proc) 1366{ 1367 fasttrap_bucket_t *bucket; 1368 fasttrap_proc_t *fprc, **fprcp; 1369 pid_t pid = proc->ftpc_pid;
| 735} 736 737 738/*ARGSUSED*/ 739static void 740fasttrap_pid_provide(void *arg, dtrace_probedesc_t *desc) 741{ 742 /* 743 * There are no "default" pid probes. 744 */ 745} 746 747static int 748fasttrap_tracepoint_enable(proc_t *p, fasttrap_probe_t *probe, uint_t index) 749{ 750 fasttrap_tracepoint_t *tp, *new_tp = NULL; 751 fasttrap_bucket_t *bucket; 752 fasttrap_id_t *id; 753 pid_t pid; 754 uintptr_t pc; 755 756 ASSERT(index < probe->ftp_ntps); 757 758 pid = probe->ftp_pid; 759 pc = probe->ftp_tps[index].fit_tp->ftt_pc; 760 id = &probe->ftp_tps[index].fit_id; 761 762 ASSERT(probe->ftp_tps[index].fit_tp->ftt_pid == pid); 763 764#if defined(sun) 765 ASSERT(!(p->p_flag & SVFORK)); 766#endif 767 768 /* 769 * Before we make any modifications, make sure we've imposed a barrier 770 * on the generation in which this probe was last modified. 771 */ 772 fasttrap_mod_barrier(probe->ftp_gen); 773 774 bucket = &fasttrap_tpoints.fth_table[FASTTRAP_TPOINTS_INDEX(pid, pc)]; 775 776 /* 777 * If the tracepoint has already been enabled, just add our id to the 778 * list of interested probes. This may be our second time through 779 * this path in which case we'll have constructed the tracepoint we'd 780 * like to install. If we can't find a match, and have an allocated 781 * tracepoint ready to go, enable that one now. 782 * 783 * A tracepoint whose process is defunct is also considered defunct. 784 */ 785again: 786 mutex_enter(&bucket->ftb_mtx); 787 for (tp = bucket->ftb_data; tp != NULL; tp = tp->ftt_next) { 788 /* 789 * Note that it's safe to access the active count on the 790 * associated proc structure because we know that at least one 791 * provider (this one) will still be around throughout this 792 * operation. 793 */ 794 if (tp->ftt_pid != pid || tp->ftt_pc != pc || 795 tp->ftt_proc->ftpc_acount == 0) 796 continue; 797 798 /* 799 * Now that we've found a matching tracepoint, it would be 800 * a decent idea to confirm that the tracepoint is still 801 * enabled and the trap instruction hasn't been overwritten. 802 * Since this is a little hairy, we'll punt for now. 803 */ 804 805 /* 806 * This can't be the first interested probe. We don't have 807 * to worry about another thread being in the midst of 808 * deleting this tracepoint (which would be the only valid 809 * reason for a tracepoint to have no interested probes) 810 * since we're holding P_PR_LOCK for this process. 811 */ 812 ASSERT(tp->ftt_ids != NULL || tp->ftt_retids != NULL); 813 814 switch (id->fti_ptype) { 815 case DTFTP_ENTRY: 816 case DTFTP_OFFSETS: 817 case DTFTP_IS_ENABLED: 818 id->fti_next = tp->ftt_ids; 819 membar_producer(); 820 tp->ftt_ids = id; 821 membar_producer(); 822 break; 823 824 case DTFTP_RETURN: 825 case DTFTP_POST_OFFSETS: 826 id->fti_next = tp->ftt_retids; 827 membar_producer(); 828 tp->ftt_retids = id; 829 membar_producer(); 830 break; 831 832 default: 833 ASSERT(0); 834 } 835 836 mutex_exit(&bucket->ftb_mtx); 837 838 if (new_tp != NULL) { 839 new_tp->ftt_ids = NULL; 840 new_tp->ftt_retids = NULL; 841 } 842 843 return (0); 844 } 845 846 /* 847 * If we have a good tracepoint ready to go, install it now while 848 * we have the lock held and no one can screw with us. 849 */ 850 if (new_tp != NULL) { 851 int rc = 0; 852 853 new_tp->ftt_next = bucket->ftb_data; 854 membar_producer(); 855 bucket->ftb_data = new_tp; 856 membar_producer(); 857 mutex_exit(&bucket->ftb_mtx); 858 859 /* 860 * Activate the tracepoint in the ISA-specific manner. 861 * If this fails, we need to report the failure, but 862 * indicate that this tracepoint must still be disabled 863 * by calling fasttrap_tracepoint_disable(). 864 */ 865 if (fasttrap_tracepoint_install(p, new_tp) != 0) 866 rc = FASTTRAP_ENABLE_PARTIAL; 867 868 /* 869 * Increment the count of the number of tracepoints active in 870 * the victim process. 871 */ 872#if defined(sun) 873 ASSERT(p->p_proc_flag & P_PR_LOCK); 874#endif 875 p->p_dtrace_count++; 876 877 return (rc); 878 } 879 880 mutex_exit(&bucket->ftb_mtx); 881 882 /* 883 * Initialize the tracepoint that's been preallocated with the probe. 884 */ 885 new_tp = probe->ftp_tps[index].fit_tp; 886 887 ASSERT(new_tp->ftt_pid == pid); 888 ASSERT(new_tp->ftt_pc == pc); 889 ASSERT(new_tp->ftt_proc == probe->ftp_prov->ftp_proc); 890 ASSERT(new_tp->ftt_ids == NULL); 891 ASSERT(new_tp->ftt_retids == NULL); 892 893 switch (id->fti_ptype) { 894 case DTFTP_ENTRY: 895 case DTFTP_OFFSETS: 896 case DTFTP_IS_ENABLED: 897 id->fti_next = NULL; 898 new_tp->ftt_ids = id; 899 break; 900 901 case DTFTP_RETURN: 902 case DTFTP_POST_OFFSETS: 903 id->fti_next = NULL; 904 new_tp->ftt_retids = id; 905 break; 906 907 default: 908 ASSERT(0); 909 } 910 911 /* 912 * If the ISA-dependent initialization goes to plan, go back to the 913 * beginning and try to install this freshly made tracepoint. 914 */ 915 if (fasttrap_tracepoint_init(p, new_tp, pc, id->fti_ptype) == 0) 916 goto again; 917 918 new_tp->ftt_ids = NULL; 919 new_tp->ftt_retids = NULL; 920 921 return (FASTTRAP_ENABLE_FAIL); 922} 923 924static void 925fasttrap_tracepoint_disable(proc_t *p, fasttrap_probe_t *probe, uint_t index) 926{ 927 fasttrap_bucket_t *bucket; 928 fasttrap_provider_t *provider = probe->ftp_prov; 929 fasttrap_tracepoint_t **pp, *tp; 930 fasttrap_id_t *id, **idp = NULL; 931 pid_t pid; 932 uintptr_t pc; 933 934 ASSERT(index < probe->ftp_ntps); 935 936 pid = probe->ftp_pid; 937 pc = probe->ftp_tps[index].fit_tp->ftt_pc; 938 id = &probe->ftp_tps[index].fit_id; 939 940 ASSERT(probe->ftp_tps[index].fit_tp->ftt_pid == pid); 941 942 /* 943 * Find the tracepoint and make sure that our id is one of the 944 * ones registered with it. 945 */ 946 bucket = &fasttrap_tpoints.fth_table[FASTTRAP_TPOINTS_INDEX(pid, pc)]; 947 mutex_enter(&bucket->ftb_mtx); 948 for (tp = bucket->ftb_data; tp != NULL; tp = tp->ftt_next) { 949 if (tp->ftt_pid == pid && tp->ftt_pc == pc && 950 tp->ftt_proc == provider->ftp_proc) 951 break; 952 } 953 954 /* 955 * If we somehow lost this tracepoint, we're in a world of hurt. 956 */ 957 ASSERT(tp != NULL); 958 959 switch (id->fti_ptype) { 960 case DTFTP_ENTRY: 961 case DTFTP_OFFSETS: 962 case DTFTP_IS_ENABLED: 963 ASSERT(tp->ftt_ids != NULL); 964 idp = &tp->ftt_ids; 965 break; 966 967 case DTFTP_RETURN: 968 case DTFTP_POST_OFFSETS: 969 ASSERT(tp->ftt_retids != NULL); 970 idp = &tp->ftt_retids; 971 break; 972 973 default: 974 ASSERT(0); 975 } 976 977 while ((*idp)->fti_probe != probe) { 978 idp = &(*idp)->fti_next; 979 ASSERT(*idp != NULL); 980 } 981 982 id = *idp; 983 *idp = id->fti_next; 984 membar_producer(); 985 986 ASSERT(id->fti_probe == probe); 987 988 /* 989 * If there are other registered enablings of this tracepoint, we're 990 * all done, but if this was the last probe assocated with this 991 * this tracepoint, we need to remove and free it. 992 */ 993 if (tp->ftt_ids != NULL || tp->ftt_retids != NULL) { 994 995 /* 996 * If the current probe's tracepoint is in use, swap it 997 * for an unused tracepoint. 998 */ 999 if (tp == probe->ftp_tps[index].fit_tp) { 1000 fasttrap_probe_t *tmp_probe; 1001 fasttrap_tracepoint_t **tmp_tp; 1002 uint_t tmp_index; 1003 1004 if (tp->ftt_ids != NULL) { 1005 tmp_probe = tp->ftt_ids->fti_probe; 1006 /* LINTED - alignment */ 1007 tmp_index = FASTTRAP_ID_INDEX(tp->ftt_ids); 1008 tmp_tp = &tmp_probe->ftp_tps[tmp_index].fit_tp; 1009 } else { 1010 tmp_probe = tp->ftt_retids->fti_probe; 1011 /* LINTED - alignment */ 1012 tmp_index = FASTTRAP_ID_INDEX(tp->ftt_retids); 1013 tmp_tp = &tmp_probe->ftp_tps[tmp_index].fit_tp; 1014 } 1015 1016 ASSERT(*tmp_tp != NULL); 1017 ASSERT(*tmp_tp != probe->ftp_tps[index].fit_tp); 1018 ASSERT((*tmp_tp)->ftt_ids == NULL); 1019 ASSERT((*tmp_tp)->ftt_retids == NULL); 1020 1021 probe->ftp_tps[index].fit_tp = *tmp_tp; 1022 *tmp_tp = tp; 1023 } 1024 1025 mutex_exit(&bucket->ftb_mtx); 1026 1027 /* 1028 * Tag the modified probe with the generation in which it was 1029 * changed. 1030 */ 1031 probe->ftp_gen = fasttrap_mod_gen; 1032 return; 1033 } 1034 1035 mutex_exit(&bucket->ftb_mtx); 1036 1037 /* 1038 * We can't safely remove the tracepoint from the set of active 1039 * tracepoints until we've actually removed the fasttrap instruction 1040 * from the process's text. We can, however, operate on this 1041 * tracepoint secure in the knowledge that no other thread is going to 1042 * be looking at it since we hold P_PR_LOCK on the process if it's 1043 * live or we hold the provider lock on the process if it's dead and 1044 * gone. 1045 */ 1046 1047 /* 1048 * We only need to remove the actual instruction if we're looking 1049 * at an existing process 1050 */ 1051 if (p != NULL) { 1052 /* 1053 * If we fail to restore the instruction we need to kill 1054 * this process since it's in a completely unrecoverable 1055 * state. 1056 */ 1057 if (fasttrap_tracepoint_remove(p, tp) != 0) 1058 fasttrap_sigtrap(p, NULL, pc); 1059 1060 /* 1061 * Decrement the count of the number of tracepoints active 1062 * in the victim process. 1063 */ 1064#if defined(sun) 1065 ASSERT(p->p_proc_flag & P_PR_LOCK); 1066#endif 1067 p->p_dtrace_count--; 1068 } 1069 1070 /* 1071 * Remove the probe from the hash table of active tracepoints. 1072 */ 1073 mutex_enter(&bucket->ftb_mtx); 1074 pp = (fasttrap_tracepoint_t **)&bucket->ftb_data; 1075 ASSERT(*pp != NULL); 1076 while (*pp != tp) { 1077 pp = &(*pp)->ftt_next; 1078 ASSERT(*pp != NULL); 1079 } 1080 1081 *pp = tp->ftt_next; 1082 membar_producer(); 1083 1084 mutex_exit(&bucket->ftb_mtx); 1085 1086 /* 1087 * Tag the modified probe with the generation in which it was changed. 1088 */ 1089 probe->ftp_gen = fasttrap_mod_gen; 1090} 1091 1092static void 1093fasttrap_enable_callbacks(void) 1094{ 1095 /* 1096 * We don't have to play the rw lock game here because we're 1097 * providing something rather than taking something away -- 1098 * we can be sure that no threads have tried to follow this 1099 * function pointer yet. 1100 */ 1101 mutex_enter(&fasttrap_count_mtx); 1102 if (fasttrap_pid_count == 0) { 1103 ASSERT(dtrace_pid_probe_ptr == NULL); 1104 ASSERT(dtrace_return_probe_ptr == NULL); 1105 dtrace_pid_probe_ptr = &fasttrap_pid_probe; 1106 dtrace_return_probe_ptr = &fasttrap_return_probe; 1107 } 1108 ASSERT(dtrace_pid_probe_ptr == &fasttrap_pid_probe); 1109 ASSERT(dtrace_return_probe_ptr == &fasttrap_return_probe); 1110 fasttrap_pid_count++; 1111 mutex_exit(&fasttrap_count_mtx); 1112} 1113 1114static void 1115fasttrap_disable_callbacks(void) 1116{ 1117#if defined(sun) 1118 ASSERT(MUTEX_HELD(&cpu_lock)); 1119#endif 1120 1121 1122 mutex_enter(&fasttrap_count_mtx); 1123 ASSERT(fasttrap_pid_count > 0); 1124 fasttrap_pid_count--; 1125 if (fasttrap_pid_count == 0) { 1126#if defined(sun) 1127 cpu_t *cur, *cpu = CPU; 1128 1129 for (cur = cpu->cpu_next_onln; cur != cpu; 1130 cur = cur->cpu_next_onln) { 1131 rw_enter(&cur->cpu_ft_lock, RW_WRITER); 1132 } 1133#endif 1134 dtrace_pid_probe_ptr = NULL; 1135 dtrace_return_probe_ptr = NULL; 1136#if defined(sun) 1137 for (cur = cpu->cpu_next_onln; cur != cpu; 1138 cur = cur->cpu_next_onln) { 1139 rw_exit(&cur->cpu_ft_lock); 1140 } 1141#endif 1142 } 1143 mutex_exit(&fasttrap_count_mtx); 1144} 1145 1146/*ARGSUSED*/ 1147static void 1148fasttrap_pid_enable(void *arg, dtrace_id_t id, void *parg) 1149{ 1150 fasttrap_probe_t *probe = parg; 1151 proc_t *p = NULL; 1152 int i, rc; 1153 1154 ASSERT(probe != NULL); 1155 ASSERT(!probe->ftp_enabled); 1156 ASSERT(id == probe->ftp_id); 1157#if defined(sun) 1158 ASSERT(MUTEX_HELD(&cpu_lock)); 1159#endif 1160 1161 /* 1162 * Increment the count of enabled probes on this probe's provider; 1163 * the provider can't go away while the probe still exists. We 1164 * must increment this even if we aren't able to properly enable 1165 * this probe. 1166 */ 1167 mutex_enter(&probe->ftp_prov->ftp_mtx); 1168 probe->ftp_prov->ftp_rcount++; 1169 mutex_exit(&probe->ftp_prov->ftp_mtx); 1170 1171 /* 1172 * If this probe's provider is retired (meaning it was valid in a 1173 * previously exec'ed incarnation of this address space), bail out. The 1174 * provider can't go away while we're in this code path. 1175 */ 1176 if (probe->ftp_prov->ftp_retired) 1177 return; 1178 1179 /* 1180 * If we can't find the process, it may be that we're in the context of 1181 * a fork in which the traced process is being born and we're copying 1182 * USDT probes. Otherwise, the process is gone so bail. 1183 */ 1184#if defined(sun) 1185 if ((p = sprlock(probe->ftp_pid)) == NULL) { 1186 if ((curproc->p_flag & SFORKING) == 0) 1187 return; 1188 1189 mutex_enter(&pidlock); 1190 p = prfind(probe->ftp_pid); 1191 1192 /* 1193 * Confirm that curproc is indeed forking the process in which 1194 * we're trying to enable probes. 1195 */ 1196 ASSERT(p != NULL); 1197 ASSERT(p->p_parent == curproc); 1198 ASSERT(p->p_stat == SIDL); 1199 1200 mutex_enter(&p->p_lock); 1201 mutex_exit(&pidlock); 1202 1203 sprlock_proc(p); 1204 } 1205 1206 ASSERT(!(p->p_flag & SVFORK)); 1207 mutex_exit(&p->p_lock); 1208#else 1209 if ((p = pfind(probe->ftp_pid)) == NULL) 1210 return; 1211#endif 1212 1213 /* 1214 * We have to enable the trap entry point before any user threads have 1215 * the chance to execute the trap instruction we're about to place 1216 * in their process's text. 1217 */ 1218#ifdef __FreeBSD__ 1219 /* 1220 * pfind() returns a locked process. 1221 */ 1222 _PHOLD(p); 1223 PROC_UNLOCK(p); 1224#endif 1225 fasttrap_enable_callbacks(); 1226 1227 /* 1228 * Enable all the tracepoints and add this probe's id to each 1229 * tracepoint's list of active probes. 1230 */ 1231 for (i = 0; i < probe->ftp_ntps; i++) { 1232 if ((rc = fasttrap_tracepoint_enable(p, probe, i)) != 0) { 1233 /* 1234 * If enabling the tracepoint failed completely, 1235 * we don't have to disable it; if the failure 1236 * was only partial we must disable it. 1237 */ 1238 if (rc == FASTTRAP_ENABLE_FAIL) 1239 i--; 1240 else 1241 ASSERT(rc == FASTTRAP_ENABLE_PARTIAL); 1242 1243 /* 1244 * Back up and pull out all the tracepoints we've 1245 * created so far for this probe. 1246 */ 1247 while (i >= 0) { 1248 fasttrap_tracepoint_disable(p, probe, i); 1249 i--; 1250 } 1251 1252#if defined(sun) 1253 mutex_enter(&p->p_lock); 1254 sprunlock(p); 1255#else 1256 PRELE(p); 1257#endif 1258 1259 /* 1260 * Since we're not actually enabling this probe, 1261 * drop our reference on the trap table entry. 1262 */ 1263 fasttrap_disable_callbacks(); 1264 return; 1265 } 1266 } 1267#if defined(sun) 1268 mutex_enter(&p->p_lock); 1269 sprunlock(p); 1270#else 1271 PRELE(p); 1272#endif 1273 1274 probe->ftp_enabled = 1; 1275} 1276 1277/*ARGSUSED*/ 1278static void 1279fasttrap_pid_disable(void *arg, dtrace_id_t id, void *parg) 1280{ 1281 fasttrap_probe_t *probe = parg; 1282 fasttrap_provider_t *provider = probe->ftp_prov; 1283 proc_t *p; 1284 int i, whack = 0; 1285 1286 ASSERT(id == probe->ftp_id); 1287 1288 mutex_enter(&provider->ftp_mtx); 1289 1290 /* 1291 * We won't be able to acquire a /proc-esque lock on the process 1292 * iff the process is dead and gone. In this case, we rely on the 1293 * provider lock as a point of mutual exclusion to prevent other 1294 * DTrace consumers from disabling this probe. 1295 */ 1296 if ((p = pfind(probe->ftp_pid)) != NULL) { 1297#ifdef __FreeBSD__ 1298 _PHOLD(p); 1299 PROC_UNLOCK(p); 1300#endif 1301 } 1302 1303 /* 1304 * Disable all the associated tracepoints (for fully enabled probes). 1305 */ 1306 if (probe->ftp_enabled) { 1307 for (i = 0; i < probe->ftp_ntps; i++) { 1308 fasttrap_tracepoint_disable(p, probe, i); 1309 } 1310 } 1311 1312 ASSERT(provider->ftp_rcount > 0); 1313 provider->ftp_rcount--; 1314 1315 if (p != NULL) { 1316 /* 1317 * Even though we may not be able to remove it entirely, we 1318 * mark this retired provider to get a chance to remove some 1319 * of the associated probes. 1320 */ 1321 if (provider->ftp_retired && !provider->ftp_marked) 1322 whack = provider->ftp_marked = 1; 1323 mutex_exit(&provider->ftp_mtx); 1324 } else { 1325 /* 1326 * If the process is dead, we're just waiting for the 1327 * last probe to be disabled to be able to free it. 1328 */ 1329 if (provider->ftp_rcount == 0 && !provider->ftp_marked) 1330 whack = provider->ftp_marked = 1; 1331 mutex_exit(&provider->ftp_mtx); 1332 } 1333 1334 if (whack) 1335 fasttrap_pid_cleanup(); 1336 1337#ifdef __FreeBSD__ 1338 if (p != NULL) 1339 PRELE(p); 1340#endif 1341 if (!probe->ftp_enabled) 1342 return; 1343 1344 probe->ftp_enabled = 0; 1345 1346#if defined(sun) 1347 ASSERT(MUTEX_HELD(&cpu_lock)); 1348#endif 1349 fasttrap_disable_callbacks(); 1350} 1351 1352/*ARGSUSED*/ 1353static void 1354fasttrap_pid_getargdesc(void *arg, dtrace_id_t id, void *parg, 1355 dtrace_argdesc_t *desc) 1356{ 1357 fasttrap_probe_t *probe = parg; 1358 char *str; 1359 int i, ndx; 1360 1361 desc->dtargd_native[0] = '\0'; 1362 desc->dtargd_xlate[0] = '\0'; 1363 1364 if (probe->ftp_prov->ftp_retired != 0 || 1365 desc->dtargd_ndx >= probe->ftp_nargs) { 1366 desc->dtargd_ndx = DTRACE_ARGNONE; 1367 return; 1368 } 1369 1370 ndx = (probe->ftp_argmap != NULL) ? 1371 probe->ftp_argmap[desc->dtargd_ndx] : desc->dtargd_ndx; 1372 1373 str = probe->ftp_ntypes; 1374 for (i = 0; i < ndx; i++) { 1375 str += strlen(str) + 1; 1376 } 1377 1378 ASSERT(strlen(str + 1) < sizeof (desc->dtargd_native)); 1379 (void) strcpy(desc->dtargd_native, str); 1380 1381 if (probe->ftp_xtypes == NULL) 1382 return; 1383 1384 str = probe->ftp_xtypes; 1385 for (i = 0; i < desc->dtargd_ndx; i++) { 1386 str += strlen(str) + 1; 1387 } 1388 1389 ASSERT(strlen(str + 1) < sizeof (desc->dtargd_xlate)); 1390 (void) strcpy(desc->dtargd_xlate, str); 1391} 1392 1393/*ARGSUSED*/ 1394static void 1395fasttrap_pid_destroy(void *arg, dtrace_id_t id, void *parg) 1396{ 1397 fasttrap_probe_t *probe = parg; 1398 int i; 1399 size_t size; 1400 1401 ASSERT(probe != NULL); 1402 ASSERT(!probe->ftp_enabled); 1403 ASSERT(fasttrap_total >= probe->ftp_ntps); 1404 1405 atomic_add_32(&fasttrap_total, -probe->ftp_ntps); 1406 size = offsetof(fasttrap_probe_t, ftp_tps[probe->ftp_ntps]); 1407 1408 if (probe->ftp_gen + 1 >= fasttrap_mod_gen) 1409 fasttrap_mod_barrier(probe->ftp_gen); 1410 1411 for (i = 0; i < probe->ftp_ntps; i++) { 1412 kmem_free(probe->ftp_tps[i].fit_tp, 1413 sizeof (fasttrap_tracepoint_t)); 1414 } 1415 1416 kmem_free(probe, size); 1417} 1418 1419 1420static const dtrace_pattr_t pid_attr = { 1421{ DTRACE_STABILITY_EVOLVING, DTRACE_STABILITY_EVOLVING, DTRACE_CLASS_ISA }, 1422{ DTRACE_STABILITY_PRIVATE, DTRACE_STABILITY_PRIVATE, DTRACE_CLASS_UNKNOWN }, 1423{ DTRACE_STABILITY_PRIVATE, DTRACE_STABILITY_PRIVATE, DTRACE_CLASS_UNKNOWN }, 1424{ DTRACE_STABILITY_EVOLVING, DTRACE_STABILITY_EVOLVING, DTRACE_CLASS_ISA }, 1425{ DTRACE_STABILITY_PRIVATE, DTRACE_STABILITY_PRIVATE, DTRACE_CLASS_UNKNOWN }, 1426}; 1427 1428static dtrace_pops_t pid_pops = { 1429 fasttrap_pid_provide, 1430 NULL, 1431 fasttrap_pid_enable, 1432 fasttrap_pid_disable, 1433 NULL, 1434 NULL, 1435 fasttrap_pid_getargdesc, 1436 fasttrap_pid_getarg, 1437 NULL, 1438 fasttrap_pid_destroy 1439}; 1440 1441static dtrace_pops_t usdt_pops = { 1442 fasttrap_pid_provide, 1443 NULL, 1444 fasttrap_pid_enable, 1445 fasttrap_pid_disable, 1446 NULL, 1447 NULL, 1448 fasttrap_pid_getargdesc, 1449 fasttrap_usdt_getarg, 1450 NULL, 1451 fasttrap_pid_destroy 1452}; 1453 1454static fasttrap_proc_t * 1455fasttrap_proc_lookup(pid_t pid) 1456{ 1457 fasttrap_bucket_t *bucket; 1458 fasttrap_proc_t *fprc, *new_fprc; 1459 1460 1461 bucket = &fasttrap_procs.fth_table[FASTTRAP_PROCS_INDEX(pid)]; 1462 mutex_enter(&bucket->ftb_mtx); 1463 1464 for (fprc = bucket->ftb_data; fprc != NULL; fprc = fprc->ftpc_next) { 1465 if (fprc->ftpc_pid == pid && fprc->ftpc_acount != 0) { 1466 mutex_enter(&fprc->ftpc_mtx); 1467 mutex_exit(&bucket->ftb_mtx); 1468 fprc->ftpc_rcount++; 1469 atomic_add_64(&fprc->ftpc_acount, 1); 1470 ASSERT(fprc->ftpc_acount <= fprc->ftpc_rcount); 1471 mutex_exit(&fprc->ftpc_mtx); 1472 1473 return (fprc); 1474 } 1475 } 1476 1477 /* 1478 * Drop the bucket lock so we don't try to perform a sleeping 1479 * allocation under it. 1480 */ 1481 mutex_exit(&bucket->ftb_mtx); 1482 1483 new_fprc = kmem_zalloc(sizeof (fasttrap_proc_t), KM_SLEEP); 1484 new_fprc->ftpc_pid = pid; 1485 new_fprc->ftpc_rcount = 1; 1486 new_fprc->ftpc_acount = 1; 1487#if !defined(sun) 1488 mutex_init(&new_fprc->ftpc_mtx, "fasttrap proc mtx", MUTEX_DEFAULT, 1489 NULL); 1490#endif 1491 1492 mutex_enter(&bucket->ftb_mtx); 1493 1494 /* 1495 * Take another lap through the list to make sure a proc hasn't 1496 * been created for this pid while we weren't under the bucket lock. 1497 */ 1498 for (fprc = bucket->ftb_data; fprc != NULL; fprc = fprc->ftpc_next) { 1499 if (fprc->ftpc_pid == pid && fprc->ftpc_acount != 0) { 1500 mutex_enter(&fprc->ftpc_mtx); 1501 mutex_exit(&bucket->ftb_mtx); 1502 fprc->ftpc_rcount++; 1503 atomic_add_64(&fprc->ftpc_acount, 1); 1504 ASSERT(fprc->ftpc_acount <= fprc->ftpc_rcount); 1505 mutex_exit(&fprc->ftpc_mtx); 1506 1507 kmem_free(new_fprc, sizeof (fasttrap_proc_t)); 1508 1509 return (fprc); 1510 } 1511 } 1512 1513 new_fprc->ftpc_next = bucket->ftb_data; 1514 bucket->ftb_data = new_fprc; 1515 1516 mutex_exit(&bucket->ftb_mtx); 1517 1518 return (new_fprc); 1519} 1520 1521static void 1522fasttrap_proc_release(fasttrap_proc_t *proc) 1523{ 1524 fasttrap_bucket_t *bucket; 1525 fasttrap_proc_t *fprc, **fprcp; 1526 pid_t pid = proc->ftpc_pid;
|
| 1527#if !defined(sun) 1528 fasttrap_scrblock_t *scrblk, *scrblktmp; 1529 fasttrap_scrspace_t *scrspc, *scrspctmp; 1530 struct proc *p; 1531 struct thread *td; 1532#endif
|
1370 1371 mutex_enter(&proc->ftpc_mtx); 1372 1373 ASSERT(proc->ftpc_rcount != 0); 1374 ASSERT(proc->ftpc_acount <= proc->ftpc_rcount); 1375 1376 if (--proc->ftpc_rcount != 0) { 1377 mutex_exit(&proc->ftpc_mtx); 1378 return; 1379 } 1380
| 1533 1534 mutex_enter(&proc->ftpc_mtx); 1535 1536 ASSERT(proc->ftpc_rcount != 0); 1537 ASSERT(proc->ftpc_acount <= proc->ftpc_rcount); 1538 1539 if (--proc->ftpc_rcount != 0) { 1540 mutex_exit(&proc->ftpc_mtx); 1541 return; 1542 } 1543
|
| 1544#if !defined(sun) 1545 /* 1546 * Free all structures used to manage per-thread scratch space. 1547 */ 1548 LIST_FOREACH_SAFE(scrblk, &proc->ftpc_scrblks, ftsb_next, 1549 scrblktmp) { 1550 LIST_REMOVE(scrblk, ftsb_next); 1551 free(scrblk, M_SOLARIS); 1552 } 1553 LIST_FOREACH_SAFE(scrspc, &proc->ftpc_fscr, ftss_next, scrspctmp) { 1554 LIST_REMOVE(scrspc, ftss_next); 1555 free(scrspc, M_SOLARIS); 1556 } 1557 LIST_FOREACH_SAFE(scrspc, &proc->ftpc_ascr, ftss_next, scrspctmp) { 1558 LIST_REMOVE(scrspc, ftss_next); 1559 free(scrspc, M_SOLARIS); 1560 } 1561 1562 if ((p = pfind(pid)) != NULL) { 1563 FOREACH_THREAD_IN_PROC(p, td) 1564 td->t_dtrace_sscr = NULL; 1565 PROC_UNLOCK(p); 1566 } 1567#endif 1568
|
1381 mutex_exit(&proc->ftpc_mtx); 1382 1383 /* 1384 * There should definitely be no live providers associated with this 1385 * process at this point. 1386 */ 1387 ASSERT(proc->ftpc_acount == 0); 1388 1389 bucket = &fasttrap_procs.fth_table[FASTTRAP_PROCS_INDEX(pid)]; 1390 mutex_enter(&bucket->ftb_mtx); 1391 1392 fprcp = (fasttrap_proc_t **)&bucket->ftb_data; 1393 while ((fprc = *fprcp) != NULL) { 1394 if (fprc == proc) 1395 break; 1396 1397 fprcp = &fprc->ftpc_next; 1398 } 1399 1400 /* 1401 * Something strange has happened if we can't find the proc. 1402 */ 1403 ASSERT(fprc != NULL); 1404 1405 *fprcp = fprc->ftpc_next; 1406 1407 mutex_exit(&bucket->ftb_mtx); 1408 1409 kmem_free(fprc, sizeof (fasttrap_proc_t)); 1410} 1411 1412/* 1413 * Lookup a fasttrap-managed provider based on its name and associated pid. 1414 * If the pattr argument is non-NULL, this function instantiates the provider 1415 * if it doesn't exist otherwise it returns NULL. The provider is returned 1416 * with its lock held. 1417 */ 1418static fasttrap_provider_t * 1419fasttrap_provider_lookup(pid_t pid, const char *name, 1420 const dtrace_pattr_t *pattr) 1421{ 1422 fasttrap_provider_t *fp, *new_fp = NULL; 1423 fasttrap_bucket_t *bucket; 1424 char provname[DTRACE_PROVNAMELEN]; 1425 proc_t *p; 1426 cred_t *cred; 1427 1428 ASSERT(strlen(name) < sizeof (fp->ftp_name)); 1429 ASSERT(pattr != NULL); 1430 1431 bucket = &fasttrap_provs.fth_table[FASTTRAP_PROVS_INDEX(pid, name)]; 1432 mutex_enter(&bucket->ftb_mtx); 1433 1434 /* 1435 * Take a lap through the list and return the match if we find it. 1436 */ 1437 for (fp = bucket->ftb_data; fp != NULL; fp = fp->ftp_next) { 1438 if (fp->ftp_pid == pid && strcmp(fp->ftp_name, name) == 0 && 1439 !fp->ftp_retired) { 1440 mutex_enter(&fp->ftp_mtx); 1441 mutex_exit(&bucket->ftb_mtx); 1442 return (fp); 1443 } 1444 } 1445 1446 /* 1447 * Drop the bucket lock so we don't try to perform a sleeping 1448 * allocation under it. 1449 */ 1450 mutex_exit(&bucket->ftb_mtx); 1451 1452 /* 1453 * Make sure the process exists, isn't a child created as the result 1454 * of a vfork(2), and isn't a zombie (but may be in fork). 1455 */ 1456 if ((p = pfind(pid)) == NULL) 1457 return (NULL); 1458 1459 /* 1460 * Increment p_dtrace_probes so that the process knows to inform us 1461 * when it exits or execs. fasttrap_provider_free() decrements this 1462 * when we're done with this provider. 1463 */ 1464 p->p_dtrace_probes++; 1465 1466 /* 1467 * Grab the credentials for this process so we have 1468 * something to pass to dtrace_register(). 1469 */ 1470 PROC_LOCK_ASSERT(p, MA_OWNED); 1471 crhold(p->p_ucred); 1472 cred = p->p_ucred; 1473 PROC_UNLOCK(p); 1474 1475 new_fp = kmem_zalloc(sizeof (fasttrap_provider_t), KM_SLEEP); 1476 new_fp->ftp_pid = pid; 1477 new_fp->ftp_proc = fasttrap_proc_lookup(pid); 1478#if !defined(sun) 1479 mutex_init(&new_fp->ftp_mtx, "provider mtx", MUTEX_DEFAULT, NULL); 1480 mutex_init(&new_fp->ftp_cmtx, "lock on creating", MUTEX_DEFAULT, NULL); 1481#endif 1482 1483 ASSERT(new_fp->ftp_proc != NULL); 1484 1485 mutex_enter(&bucket->ftb_mtx); 1486 1487 /* 1488 * Take another lap through the list to make sure a provider hasn't 1489 * been created for this pid while we weren't under the bucket lock. 1490 */ 1491 for (fp = bucket->ftb_data; fp != NULL; fp = fp->ftp_next) { 1492 if (fp->ftp_pid == pid && strcmp(fp->ftp_name, name) == 0 && 1493 !fp->ftp_retired) { 1494 mutex_enter(&fp->ftp_mtx); 1495 mutex_exit(&bucket->ftb_mtx); 1496 fasttrap_provider_free(new_fp); 1497 crfree(cred); 1498 return (fp); 1499 } 1500 } 1501 1502 (void) strcpy(new_fp->ftp_name, name); 1503 1504 /* 1505 * Fail and return NULL if either the provider name is too long 1506 * or we fail to register this new provider with the DTrace 1507 * framework. Note that this is the only place we ever construct 1508 * the full provider name -- we keep it in pieces in the provider 1509 * structure. 1510 */ 1511 if (snprintf(provname, sizeof (provname), "%s%u", name, (uint_t)pid) >= 1512 sizeof (provname) || 1513 dtrace_register(provname, pattr, 1514 DTRACE_PRIV_PROC | DTRACE_PRIV_OWNER | DTRACE_PRIV_ZONEOWNER, cred, 1515 pattr == &pid_attr ? &pid_pops : &usdt_pops, new_fp, 1516 &new_fp->ftp_provid) != 0) { 1517 mutex_exit(&bucket->ftb_mtx); 1518 fasttrap_provider_free(new_fp); 1519 crfree(cred); 1520 return (NULL); 1521 } 1522 1523 new_fp->ftp_next = bucket->ftb_data; 1524 bucket->ftb_data = new_fp; 1525 1526 mutex_enter(&new_fp->ftp_mtx); 1527 mutex_exit(&bucket->ftb_mtx); 1528 1529 crfree(cred); 1530 return (new_fp); 1531} 1532 1533static void 1534fasttrap_provider_free(fasttrap_provider_t *provider) 1535{ 1536 pid_t pid = provider->ftp_pid; 1537 proc_t *p; 1538 1539 /* 1540 * There need to be no associated enabled probes, no consumers 1541 * creating probes, and no meta providers referencing this provider. 1542 */ 1543 ASSERT(provider->ftp_rcount == 0); 1544 ASSERT(provider->ftp_ccount == 0); 1545 ASSERT(provider->ftp_mcount == 0); 1546 1547 /* 1548 * If this provider hasn't been retired, we need to explicitly drop the 1549 * count of active providers on the associated process structure. 1550 */ 1551 if (!provider->ftp_retired) { 1552 atomic_add_64(&provider->ftp_proc->ftpc_acount, -1); 1553 ASSERT(provider->ftp_proc->ftpc_acount < 1554 provider->ftp_proc->ftpc_rcount); 1555 } 1556 1557 fasttrap_proc_release(provider->ftp_proc); 1558 1559#if !defined(sun) 1560 mutex_destroy(&provider->ftp_mtx); 1561 mutex_destroy(&provider->ftp_cmtx); 1562#endif 1563 kmem_free(provider, sizeof (fasttrap_provider_t)); 1564 1565 /* 1566 * Decrement p_dtrace_probes on the process whose provider we're 1567 * freeing. We don't have to worry about clobbering somone else's 1568 * modifications to it because we have locked the bucket that 1569 * corresponds to this process's hash chain in the provider hash 1570 * table. Don't sweat it if we can't find the process. 1571 */ 1572 if ((p = pfind(pid)) == NULL) { 1573 return; 1574 } 1575 1576 p->p_dtrace_probes--; 1577#if !defined(sun) 1578 PROC_UNLOCK(p); 1579#endif 1580} 1581 1582static void 1583fasttrap_provider_retire(pid_t pid, const char *name, int mprov) 1584{ 1585 fasttrap_provider_t *fp; 1586 fasttrap_bucket_t *bucket; 1587 dtrace_provider_id_t provid; 1588 1589 ASSERT(strlen(name) < sizeof (fp->ftp_name)); 1590 1591 bucket = &fasttrap_provs.fth_table[FASTTRAP_PROVS_INDEX(pid, name)]; 1592 mutex_enter(&bucket->ftb_mtx); 1593 1594 for (fp = bucket->ftb_data; fp != NULL; fp = fp->ftp_next) { 1595 if (fp->ftp_pid == pid && strcmp(fp->ftp_name, name) == 0 && 1596 !fp->ftp_retired) 1597 break; 1598 } 1599 1600 if (fp == NULL) { 1601 mutex_exit(&bucket->ftb_mtx); 1602 return; 1603 } 1604 1605 mutex_enter(&fp->ftp_mtx); 1606 ASSERT(!mprov || fp->ftp_mcount > 0); 1607 if (mprov && --fp->ftp_mcount != 0) { 1608 mutex_exit(&fp->ftp_mtx); 1609 mutex_exit(&bucket->ftb_mtx); 1610 return; 1611 } 1612 1613 /* 1614 * Mark the provider to be removed in our post-processing step, mark it 1615 * retired, and drop the active count on its proc. Marking it indicates 1616 * that we should try to remove it; setting the retired flag indicates 1617 * that we're done with this provider; dropping the active the proc 1618 * releases our hold, and when this reaches zero (as it will during 1619 * exit or exec) the proc and associated providers become defunct. 1620 * 1621 * We obviously need to take the bucket lock before the provider lock 1622 * to perform the lookup, but we need to drop the provider lock 1623 * before calling into the DTrace framework since we acquire the 1624 * provider lock in callbacks invoked from the DTrace framework. The 1625 * bucket lock therefore protects the integrity of the provider hash 1626 * table. 1627 */ 1628 atomic_add_64(&fp->ftp_proc->ftpc_acount, -1); 1629 ASSERT(fp->ftp_proc->ftpc_acount < fp->ftp_proc->ftpc_rcount); 1630 1631 fp->ftp_retired = 1; 1632 fp->ftp_marked = 1; 1633 provid = fp->ftp_provid; 1634 mutex_exit(&fp->ftp_mtx); 1635 1636 /* 1637 * We don't have to worry about invalidating the same provider twice 1638 * since fasttrap_provider_lookup() will ignore provider that have 1639 * been marked as retired. 1640 */ 1641 dtrace_invalidate(provid); 1642 1643 mutex_exit(&bucket->ftb_mtx); 1644 1645 fasttrap_pid_cleanup(); 1646} 1647 1648static int 1649fasttrap_uint32_cmp(const void *ap, const void *bp) 1650{ 1651 return (*(const uint32_t *)ap - *(const uint32_t *)bp); 1652} 1653 1654static int 1655fasttrap_uint64_cmp(const void *ap, const void *bp) 1656{ 1657 return (*(const uint64_t *)ap - *(const uint64_t *)bp); 1658} 1659 1660static int 1661fasttrap_add_probe(fasttrap_probe_spec_t *pdata) 1662{ 1663 fasttrap_provider_t *provider; 1664 fasttrap_probe_t *pp; 1665 fasttrap_tracepoint_t *tp; 1666 char *name; 1667 int i, aframes = 0, whack; 1668 1669 /* 1670 * There needs to be at least one desired trace point. 1671 */ 1672 if (pdata->ftps_noffs == 0) 1673 return (EINVAL); 1674 1675 switch (pdata->ftps_type) { 1676 case DTFTP_ENTRY: 1677 name = "entry"; 1678 aframes = FASTTRAP_ENTRY_AFRAMES; 1679 break; 1680 case DTFTP_RETURN: 1681 name = "return"; 1682 aframes = FASTTRAP_RETURN_AFRAMES; 1683 break; 1684 case DTFTP_OFFSETS: 1685 name = NULL; 1686 break; 1687 default: 1688 return (EINVAL); 1689 } 1690 1691 if ((provider = fasttrap_provider_lookup(pdata->ftps_pid, 1692 FASTTRAP_PID_NAME, &pid_attr)) == NULL) 1693 return (ESRCH); 1694 1695 /* 1696 * Increment this reference count to indicate that a consumer is 1697 * actively adding a new probe associated with this provider. This 1698 * prevents the provider from being deleted -- we'll need to check 1699 * for pending deletions when we drop this reference count. 1700 */ 1701 provider->ftp_ccount++; 1702 mutex_exit(&provider->ftp_mtx); 1703 1704 /* 1705 * Grab the creation lock to ensure consistency between calls to 1706 * dtrace_probe_lookup() and dtrace_probe_create() in the face of 1707 * other threads creating probes. We must drop the provider lock 1708 * before taking this lock to avoid a three-way deadlock with the 1709 * DTrace framework. 1710 */ 1711 mutex_enter(&provider->ftp_cmtx); 1712 1713 if (name == NULL) { 1714 for (i = 0; i < pdata->ftps_noffs; i++) { 1715 char name_str[17]; 1716 1717 (void) sprintf(name_str, "%llx", 1718 (unsigned long long)pdata->ftps_offs[i]); 1719 1720 if (dtrace_probe_lookup(provider->ftp_provid, 1721 pdata->ftps_mod, pdata->ftps_func, name_str) != 0) 1722 continue; 1723 1724 atomic_add_32(&fasttrap_total, 1); 1725 1726 if (fasttrap_total > fasttrap_max) { 1727 atomic_add_32(&fasttrap_total, -1); 1728 goto no_mem; 1729 } 1730 1731 pp = kmem_zalloc(sizeof (fasttrap_probe_t), KM_SLEEP); 1732 1733 pp->ftp_prov = provider; 1734 pp->ftp_faddr = pdata->ftps_pc; 1735 pp->ftp_fsize = pdata->ftps_size; 1736 pp->ftp_pid = pdata->ftps_pid; 1737 pp->ftp_ntps = 1; 1738 1739 tp = kmem_zalloc(sizeof (fasttrap_tracepoint_t), 1740 KM_SLEEP); 1741 1742 tp->ftt_proc = provider->ftp_proc; 1743 tp->ftt_pc = pdata->ftps_offs[i] + pdata->ftps_pc; 1744 tp->ftt_pid = pdata->ftps_pid; 1745 1746 pp->ftp_tps[0].fit_tp = tp; 1747 pp->ftp_tps[0].fit_id.fti_probe = pp; 1748 pp->ftp_tps[0].fit_id.fti_ptype = pdata->ftps_type; 1749 1750 pp->ftp_id = dtrace_probe_create(provider->ftp_provid, 1751 pdata->ftps_mod, pdata->ftps_func, name_str, 1752 FASTTRAP_OFFSET_AFRAMES, pp); 1753 } 1754 1755 } else if (dtrace_probe_lookup(provider->ftp_provid, pdata->ftps_mod, 1756 pdata->ftps_func, name) == 0) { 1757 atomic_add_32(&fasttrap_total, pdata->ftps_noffs); 1758 1759 if (fasttrap_total > fasttrap_max) { 1760 atomic_add_32(&fasttrap_total, -pdata->ftps_noffs); 1761 goto no_mem; 1762 } 1763 1764 /* 1765 * Make sure all tracepoint program counter values are unique. 1766 * We later assume that each probe has exactly one tracepoint 1767 * for a given pc. 1768 */ 1769 qsort(pdata->ftps_offs, pdata->ftps_noffs, 1770 sizeof (uint64_t), fasttrap_uint64_cmp); 1771 for (i = 1; i < pdata->ftps_noffs; i++) { 1772 if (pdata->ftps_offs[i] > pdata->ftps_offs[i - 1]) 1773 continue; 1774 1775 atomic_add_32(&fasttrap_total, -pdata->ftps_noffs); 1776 goto no_mem; 1777 } 1778 1779 ASSERT(pdata->ftps_noffs > 0); 1780 pp = kmem_zalloc(offsetof(fasttrap_probe_t, 1781 ftp_tps[pdata->ftps_noffs]), KM_SLEEP); 1782 1783 pp->ftp_prov = provider; 1784 pp->ftp_faddr = pdata->ftps_pc; 1785 pp->ftp_fsize = pdata->ftps_size; 1786 pp->ftp_pid = pdata->ftps_pid; 1787 pp->ftp_ntps = pdata->ftps_noffs; 1788 1789 for (i = 0; i < pdata->ftps_noffs; i++) { 1790 tp = kmem_zalloc(sizeof (fasttrap_tracepoint_t), 1791 KM_SLEEP); 1792 1793 tp->ftt_proc = provider->ftp_proc; 1794 tp->ftt_pc = pdata->ftps_offs[i] + pdata->ftps_pc; 1795 tp->ftt_pid = pdata->ftps_pid; 1796 1797 pp->ftp_tps[i].fit_tp = tp; 1798 pp->ftp_tps[i].fit_id.fti_probe = pp; 1799 pp->ftp_tps[i].fit_id.fti_ptype = pdata->ftps_type; 1800 } 1801 1802 pp->ftp_id = dtrace_probe_create(provider->ftp_provid, 1803 pdata->ftps_mod, pdata->ftps_func, name, aframes, pp); 1804 } 1805 1806 mutex_exit(&provider->ftp_cmtx); 1807 1808 /* 1809 * We know that the provider is still valid since we incremented the 1810 * creation reference count. If someone tried to clean up this provider 1811 * while we were using it (e.g. because the process called exec(2) or 1812 * exit(2)), take note of that and try to clean it up now. 1813 */ 1814 mutex_enter(&provider->ftp_mtx); 1815 provider->ftp_ccount--; 1816 whack = provider->ftp_retired; 1817 mutex_exit(&provider->ftp_mtx); 1818 1819 if (whack) 1820 fasttrap_pid_cleanup(); 1821 1822 return (0); 1823 1824no_mem: 1825 /* 1826 * If we've exhausted the allowable resources, we'll try to remove 1827 * this provider to free some up. This is to cover the case where 1828 * the user has accidentally created many more probes than was 1829 * intended (e.g. pid123:::). 1830 */ 1831 mutex_exit(&provider->ftp_cmtx); 1832 mutex_enter(&provider->ftp_mtx); 1833 provider->ftp_ccount--; 1834 provider->ftp_marked = 1; 1835 mutex_exit(&provider->ftp_mtx); 1836 1837 fasttrap_pid_cleanup(); 1838 1839 return (ENOMEM); 1840} 1841 1842/*ARGSUSED*/ 1843static void * 1844fasttrap_meta_provide(void *arg, dtrace_helper_provdesc_t *dhpv, pid_t pid) 1845{ 1846 fasttrap_provider_t *provider; 1847 1848 /* 1849 * A 32-bit unsigned integer (like a pid for example) can be 1850 * expressed in 10 or fewer decimal digits. Make sure that we'll 1851 * have enough space for the provider name. 1852 */ 1853 if (strlen(dhpv->dthpv_provname) + 10 >= 1854 sizeof (provider->ftp_name)) { 1855 printf("failed to instantiate provider %s: " 1856 "name too long to accomodate pid", dhpv->dthpv_provname); 1857 return (NULL); 1858 } 1859 1860 /* 1861 * Don't let folks spoof the true pid provider. 1862 */ 1863 if (strcmp(dhpv->dthpv_provname, FASTTRAP_PID_NAME) == 0) { 1864 printf("failed to instantiate provider %s: " 1865 "%s is an invalid name", dhpv->dthpv_provname, 1866 FASTTRAP_PID_NAME); 1867 return (NULL); 1868 } 1869 1870 /* 1871 * The highest stability class that fasttrap supports is ISA; cap 1872 * the stability of the new provider accordingly. 1873 */ 1874 if (dhpv->dthpv_pattr.dtpa_provider.dtat_class > DTRACE_CLASS_ISA) 1875 dhpv->dthpv_pattr.dtpa_provider.dtat_class = DTRACE_CLASS_ISA; 1876 if (dhpv->dthpv_pattr.dtpa_mod.dtat_class > DTRACE_CLASS_ISA) 1877 dhpv->dthpv_pattr.dtpa_mod.dtat_class = DTRACE_CLASS_ISA; 1878 if (dhpv->dthpv_pattr.dtpa_func.dtat_class > DTRACE_CLASS_ISA) 1879 dhpv->dthpv_pattr.dtpa_func.dtat_class = DTRACE_CLASS_ISA; 1880 if (dhpv->dthpv_pattr.dtpa_name.dtat_class > DTRACE_CLASS_ISA) 1881 dhpv->dthpv_pattr.dtpa_name.dtat_class = DTRACE_CLASS_ISA; 1882 if (dhpv->dthpv_pattr.dtpa_args.dtat_class > DTRACE_CLASS_ISA) 1883 dhpv->dthpv_pattr.dtpa_args.dtat_class = DTRACE_CLASS_ISA; 1884 1885 if ((provider = fasttrap_provider_lookup(pid, dhpv->dthpv_provname, 1886 &dhpv->dthpv_pattr)) == NULL) { 1887 printf("failed to instantiate provider %s for " 1888 "process %u", dhpv->dthpv_provname, (uint_t)pid); 1889 return (NULL); 1890 } 1891 1892 /* 1893 * Up the meta provider count so this provider isn't removed until 1894 * the meta provider has been told to remove it. 1895 */ 1896 provider->ftp_mcount++; 1897 1898 mutex_exit(&provider->ftp_mtx); 1899 1900 return (provider); 1901} 1902 1903/*ARGSUSED*/ 1904static void 1905fasttrap_meta_create_probe(void *arg, void *parg, 1906 dtrace_helper_probedesc_t *dhpb) 1907{ 1908 fasttrap_provider_t *provider = parg; 1909 fasttrap_probe_t *pp; 1910 fasttrap_tracepoint_t *tp; 1911 int i, j; 1912 uint32_t ntps; 1913 1914 /* 1915 * Since the meta provider count is non-zero we don't have to worry 1916 * about this provider disappearing. 1917 */ 1918 ASSERT(provider->ftp_mcount > 0); 1919 1920 /* 1921 * The offsets must be unique. 1922 */ 1923 qsort(dhpb->dthpb_offs, dhpb->dthpb_noffs, sizeof (uint32_t), 1924 fasttrap_uint32_cmp); 1925 for (i = 1; i < dhpb->dthpb_noffs; i++) { 1926 if (dhpb->dthpb_base + dhpb->dthpb_offs[i] <= 1927 dhpb->dthpb_base + dhpb->dthpb_offs[i - 1]) 1928 return; 1929 } 1930 1931 qsort(dhpb->dthpb_enoffs, dhpb->dthpb_nenoffs, sizeof (uint32_t), 1932 fasttrap_uint32_cmp); 1933 for (i = 1; i < dhpb->dthpb_nenoffs; i++) { 1934 if (dhpb->dthpb_base + dhpb->dthpb_enoffs[i] <= 1935 dhpb->dthpb_base + dhpb->dthpb_enoffs[i - 1]) 1936 return; 1937 } 1938 1939 /* 1940 * Grab the creation lock to ensure consistency between calls to 1941 * dtrace_probe_lookup() and dtrace_probe_create() in the face of 1942 * other threads creating probes. 1943 */ 1944 mutex_enter(&provider->ftp_cmtx); 1945 1946 if (dtrace_probe_lookup(provider->ftp_provid, dhpb->dthpb_mod, 1947 dhpb->dthpb_func, dhpb->dthpb_name) != 0) { 1948 mutex_exit(&provider->ftp_cmtx); 1949 return; 1950 } 1951 1952 ntps = dhpb->dthpb_noffs + dhpb->dthpb_nenoffs; 1953 ASSERT(ntps > 0); 1954 1955 atomic_add_32(&fasttrap_total, ntps); 1956 1957 if (fasttrap_total > fasttrap_max) { 1958 atomic_add_32(&fasttrap_total, -ntps); 1959 mutex_exit(&provider->ftp_cmtx); 1960 return; 1961 } 1962 1963 pp = kmem_zalloc(offsetof(fasttrap_probe_t, ftp_tps[ntps]), KM_SLEEP); 1964 1965 pp->ftp_prov = provider; 1966 pp->ftp_pid = provider->ftp_pid; 1967 pp->ftp_ntps = ntps; 1968 pp->ftp_nargs = dhpb->dthpb_xargc; 1969 pp->ftp_xtypes = dhpb->dthpb_xtypes; 1970 pp->ftp_ntypes = dhpb->dthpb_ntypes; 1971 1972 /* 1973 * First create a tracepoint for each actual point of interest. 1974 */ 1975 for (i = 0; i < dhpb->dthpb_noffs; i++) { 1976 tp = kmem_zalloc(sizeof (fasttrap_tracepoint_t), KM_SLEEP); 1977 1978 tp->ftt_proc = provider->ftp_proc; 1979 tp->ftt_pc = dhpb->dthpb_base + dhpb->dthpb_offs[i]; 1980 tp->ftt_pid = provider->ftp_pid; 1981 1982 pp->ftp_tps[i].fit_tp = tp; 1983 pp->ftp_tps[i].fit_id.fti_probe = pp; 1984#ifdef __sparc 1985 pp->ftp_tps[i].fit_id.fti_ptype = DTFTP_POST_OFFSETS; 1986#else 1987 pp->ftp_tps[i].fit_id.fti_ptype = DTFTP_OFFSETS; 1988#endif 1989 } 1990 1991 /* 1992 * Then create a tracepoint for each is-enabled point. 1993 */ 1994 for (j = 0; i < ntps; i++, j++) { 1995 tp = kmem_zalloc(sizeof (fasttrap_tracepoint_t), KM_SLEEP); 1996 1997 tp->ftt_proc = provider->ftp_proc; 1998 tp->ftt_pc = dhpb->dthpb_base + dhpb->dthpb_enoffs[j]; 1999 tp->ftt_pid = provider->ftp_pid; 2000 2001 pp->ftp_tps[i].fit_tp = tp; 2002 pp->ftp_tps[i].fit_id.fti_probe = pp; 2003 pp->ftp_tps[i].fit_id.fti_ptype = DTFTP_IS_ENABLED; 2004 } 2005 2006 /* 2007 * If the arguments are shuffled around we set the argument remapping 2008 * table. Later, when the probe fires, we only remap the arguments 2009 * if the table is non-NULL. 2010 */ 2011 for (i = 0; i < dhpb->dthpb_xargc; i++) { 2012 if (dhpb->dthpb_args[i] != i) { 2013 pp->ftp_argmap = dhpb->dthpb_args; 2014 break; 2015 } 2016 } 2017 2018 /* 2019 * The probe is fully constructed -- register it with DTrace. 2020 */ 2021 pp->ftp_id = dtrace_probe_create(provider->ftp_provid, dhpb->dthpb_mod, 2022 dhpb->dthpb_func, dhpb->dthpb_name, FASTTRAP_OFFSET_AFRAMES, pp); 2023 2024 mutex_exit(&provider->ftp_cmtx); 2025} 2026 2027/*ARGSUSED*/ 2028static void 2029fasttrap_meta_remove(void *arg, dtrace_helper_provdesc_t *dhpv, pid_t pid) 2030{ 2031 /* 2032 * Clean up the USDT provider. There may be active consumers of the 2033 * provider busy adding probes, no damage will actually befall the 2034 * provider until that count has dropped to zero. This just puts 2035 * the provider on death row. 2036 */ 2037 fasttrap_provider_retire(pid, dhpv->dthpv_provname, 1); 2038} 2039 2040static dtrace_mops_t fasttrap_mops = { 2041 fasttrap_meta_create_probe, 2042 fasttrap_meta_provide, 2043 fasttrap_meta_remove 2044}; 2045 2046/*ARGSUSED*/ 2047static int 2048fasttrap_open(struct cdev *dev __unused, int oflags __unused, 2049 int devtype __unused, struct thread *td __unused) 2050{ 2051 return (0); 2052} 2053 2054/*ARGSUSED*/ 2055static int 2056fasttrap_ioctl(struct cdev *dev, u_long cmd, caddr_t arg, int fflag, 2057 struct thread *td) 2058{ 2059#ifdef notyet 2060 struct kinfo_proc kp; 2061 const cred_t *cr = td->td_ucred; 2062#endif 2063 if (!dtrace_attached()) 2064 return (EAGAIN); 2065 2066 if (cmd == FASTTRAPIOC_MAKEPROBE) { 2067 fasttrap_probe_spec_t *uprobe = *(fasttrap_probe_spec_t **)arg; 2068 fasttrap_probe_spec_t *probe; 2069 uint64_t noffs; 2070 size_t size; 2071 int ret; 2072 char *c; 2073 2074 if (copyin(&uprobe->ftps_noffs, &noffs, 2075 sizeof (uprobe->ftps_noffs))) 2076 return (EFAULT); 2077 2078 /* 2079 * Probes must have at least one tracepoint. 2080 */ 2081 if (noffs == 0) 2082 return (EINVAL); 2083 2084 size = sizeof (fasttrap_probe_spec_t) + 2085 sizeof (probe->ftps_offs[0]) * (noffs - 1); 2086 2087 if (size > 1024 * 1024) 2088 return (ENOMEM); 2089 2090 probe = kmem_alloc(size, KM_SLEEP); 2091 2092 if (copyin(uprobe, probe, size) != 0) { 2093 kmem_free(probe, size); 2094 return (EFAULT); 2095 } 2096 2097 /* 2098 * Verify that the function and module strings contain no 2099 * funny characters. 2100 */ 2101 for (c = &probe->ftps_func[0]; *c != '\0'; c++) { 2102 if (*c < 0x20 || 0x7f <= *c) { 2103 ret = EINVAL; 2104 goto err; 2105 } 2106 } 2107 2108 for (c = &probe->ftps_mod[0]; *c != '\0'; c++) { 2109 if (*c < 0x20 || 0x7f <= *c) { 2110 ret = EINVAL; 2111 goto err; 2112 } 2113 } 2114 2115#ifdef notyet 2116 if (!PRIV_POLICY_CHOICE(cr, PRIV_ALL, B_FALSE)) { 2117 proc_t *p; 2118 pid_t pid = probe->ftps_pid; 2119 2120#if defined(sun) 2121 mutex_enter(&pidlock); 2122#endif 2123 /* 2124 * Report an error if the process doesn't exist 2125 * or is actively being birthed. 2126 */ 2127 p = pfind(pid); 2128 if (p) 2129 fill_kinfo_proc(p, &kp); 2130 if (p == NULL || kp.ki_stat == SIDL) { 2131#if defined(sun) 2132 mutex_exit(&pidlock); 2133#endif 2134 return (ESRCH); 2135 } 2136#if defined(sun) 2137 mutex_enter(&p->p_lock); 2138 mutex_exit(&pidlock); 2139#else 2140 PROC_LOCK_ASSERT(p, MA_OWNED); 2141#endif 2142 2143#ifdef notyet 2144 if ((ret = priv_proc_cred_perm(cr, p, NULL, 2145 VREAD | VWRITE)) != 0) { 2146#if defined(sun) 2147 mutex_exit(&p->p_lock); 2148#else 2149 PROC_UNLOCK(p); 2150#endif 2151 return (ret); 2152 } 2153#endif /* notyet */ 2154#if defined(sun) 2155 mutex_exit(&p->p_lock); 2156#else 2157 PROC_UNLOCK(p); 2158#endif 2159 } 2160#endif /* notyet */ 2161 2162 ret = fasttrap_add_probe(probe); 2163err: 2164 kmem_free(probe, size); 2165 2166 return (ret); 2167 2168 } else if (cmd == FASTTRAPIOC_GETINSTR) { 2169 fasttrap_instr_query_t instr; 2170 fasttrap_tracepoint_t *tp; 2171 uint_t index; 2172#if defined(sun) 2173 int ret; 2174#endif 2175 2176#if defined(sun) 2177 if (copyin((void *)arg, &instr, sizeof (instr)) != 0) 2178 return (EFAULT); 2179#endif 2180 2181#ifdef notyet 2182 if (!PRIV_POLICY_CHOICE(cr, PRIV_ALL, B_FALSE)) { 2183 proc_t *p; 2184 pid_t pid = instr.ftiq_pid; 2185 2186#if defined(sun) 2187 mutex_enter(&pidlock); 2188#endif 2189 /* 2190 * Report an error if the process doesn't exist 2191 * or is actively being birthed. 2192 */ 2193 p = pfind(pid); 2194 if (p) 2195 fill_kinfo_proc(p, &kp); 2196 if (p == NULL || kp.ki_stat == SIDL) { 2197#if defined(sun) 2198 mutex_exit(&pidlock); 2199#endif 2200 return (ESRCH); 2201 } 2202#if defined(sun) 2203 mutex_enter(&p->p_lock); 2204 mutex_exit(&pidlock); 2205#else 2206 PROC_LOCK_ASSERT(p, MA_OWNED); 2207#endif 2208 2209#ifdef notyet 2210 if ((ret = priv_proc_cred_perm(cr, p, NULL, 2211 VREAD)) != 0) { 2212#if defined(sun) 2213 mutex_exit(&p->p_lock); 2214#else 2215 PROC_UNLOCK(p); 2216#endif 2217 return (ret); 2218 } 2219#endif /* notyet */ 2220 2221#if defined(sun) 2222 mutex_exit(&p->p_lock); 2223#else 2224 PROC_UNLOCK(p); 2225#endif 2226 } 2227#endif /* notyet */ 2228 2229 index = FASTTRAP_TPOINTS_INDEX(instr.ftiq_pid, instr.ftiq_pc); 2230 2231 mutex_enter(&fasttrap_tpoints.fth_table[index].ftb_mtx); 2232 tp = fasttrap_tpoints.fth_table[index].ftb_data; 2233 while (tp != NULL) { 2234 if (instr.ftiq_pid == tp->ftt_pid && 2235 instr.ftiq_pc == tp->ftt_pc && 2236 tp->ftt_proc->ftpc_acount != 0) 2237 break; 2238 2239 tp = tp->ftt_next; 2240 } 2241 2242 if (tp == NULL) { 2243 mutex_exit(&fasttrap_tpoints.fth_table[index].ftb_mtx); 2244 return (ENOENT); 2245 } 2246 2247 bcopy(&tp->ftt_instr, &instr.ftiq_instr, 2248 sizeof (instr.ftiq_instr)); 2249 mutex_exit(&fasttrap_tpoints.fth_table[index].ftb_mtx); 2250 2251 if (copyout(&instr, (void *)arg, sizeof (instr)) != 0) 2252 return (EFAULT); 2253 2254 return (0); 2255 } 2256 2257 return (EINVAL); 2258} 2259 2260static int 2261fasttrap_load(void) 2262{ 2263 ulong_t nent; 2264 int i, ret; 2265 2266 /* Create the /dev/dtrace/fasttrap entry. */ 2267 fasttrap_cdev = make_dev(&fasttrap_cdevsw, 0, UID_ROOT, GID_WHEEL, 0600, 2268 "dtrace/fasttrap"); 2269 2270 mtx_init(&fasttrap_cleanup_mtx, "fasttrap clean", "dtrace", MTX_DEF); 2271 mutex_init(&fasttrap_count_mtx, "fasttrap count mtx", MUTEX_DEFAULT, 2272 NULL); 2273 2274#if defined(sun) 2275 fasttrap_max = ddi_getprop(DDI_DEV_T_ANY, devi, DDI_PROP_DONTPASS, 2276 "fasttrap-max-probes", FASTTRAP_MAX_DEFAULT); 2277#else 2278 fasttrap_max = FASTTRAP_MAX_DEFAULT; 2279#endif 2280 fasttrap_total = 0; 2281 2282 /* 2283 * Conjure up the tracepoints hashtable... 2284 */ 2285#if defined(sun) 2286 nent = ddi_getprop(DDI_DEV_T_ANY, devi, DDI_PROP_DONTPASS, 2287 "fasttrap-hash-size", FASTTRAP_TPOINTS_DEFAULT_SIZE); 2288#else 2289 nent = FASTTRAP_TPOINTS_DEFAULT_SIZE; 2290#endif 2291 2292 if (nent == 0 || nent > 0x1000000) 2293 nent = FASTTRAP_TPOINTS_DEFAULT_SIZE; 2294 2295 if ((nent & (nent - 1)) == 0) 2296 fasttrap_tpoints.fth_nent = nent; 2297 else 2298 fasttrap_tpoints.fth_nent = 1 << fasttrap_highbit(nent); 2299 ASSERT(fasttrap_tpoints.fth_nent > 0); 2300 fasttrap_tpoints.fth_mask = fasttrap_tpoints.fth_nent - 1; 2301 fasttrap_tpoints.fth_table = kmem_zalloc(fasttrap_tpoints.fth_nent * 2302 sizeof (fasttrap_bucket_t), KM_SLEEP); 2303#if !defined(sun) 2304 for (i = 0; i < fasttrap_tpoints.fth_nent; i++) 2305 mutex_init(&fasttrap_tpoints.fth_table[i].ftb_mtx, 2306 "tracepoints bucket mtx", MUTEX_DEFAULT, NULL); 2307#endif 2308 2309 /* 2310 * ... and the providers hash table... 2311 */ 2312 nent = FASTTRAP_PROVIDERS_DEFAULT_SIZE; 2313 if ((nent & (nent - 1)) == 0) 2314 fasttrap_provs.fth_nent = nent; 2315 else 2316 fasttrap_provs.fth_nent = 1 << fasttrap_highbit(nent); 2317 ASSERT(fasttrap_provs.fth_nent > 0); 2318 fasttrap_provs.fth_mask = fasttrap_provs.fth_nent - 1; 2319 fasttrap_provs.fth_table = kmem_zalloc(fasttrap_provs.fth_nent * 2320 sizeof (fasttrap_bucket_t), KM_SLEEP); 2321#if !defined(sun) 2322 for (i = 0; i < fasttrap_provs.fth_nent; i++) 2323 mutex_init(&fasttrap_provs.fth_table[i].ftb_mtx, 2324 "providers bucket mtx", MUTEX_DEFAULT, NULL); 2325#endif 2326 2327 ret = kproc_create(fasttrap_pid_cleanup_cb, NULL, 2328 &fasttrap_cleanup_proc, 0, 0, "ftcleanup"); 2329 if (ret != 0) { 2330 destroy_dev(fasttrap_cdev); 2331#if !defined(sun) 2332 for (i = 0; i < fasttrap_provs.fth_nent; i++) 2333 mutex_destroy(&fasttrap_provs.fth_table[i].ftb_mtx); 2334 for (i = 0; i < fasttrap_tpoints.fth_nent; i++) 2335 mutex_destroy(&fasttrap_tpoints.fth_table[i].ftb_mtx); 2336#endif 2337 kmem_free(fasttrap_provs.fth_table, fasttrap_provs.fth_nent * 2338 sizeof (fasttrap_bucket_t)); 2339 mtx_destroy(&fasttrap_cleanup_mtx); 2340 mutex_destroy(&fasttrap_count_mtx); 2341 return (ret); 2342 } 2343 2344 2345 /* 2346 * ... and the procs hash table. 2347 */ 2348 nent = FASTTRAP_PROCS_DEFAULT_SIZE; 2349 if ((nent & (nent - 1)) == 0) 2350 fasttrap_procs.fth_nent = nent; 2351 else 2352 fasttrap_procs.fth_nent = 1 << fasttrap_highbit(nent); 2353 ASSERT(fasttrap_procs.fth_nent > 0); 2354 fasttrap_procs.fth_mask = fasttrap_procs.fth_nent - 1; 2355 fasttrap_procs.fth_table = kmem_zalloc(fasttrap_procs.fth_nent * 2356 sizeof (fasttrap_bucket_t), KM_SLEEP); 2357#if !defined(sun) 2358 for (i = 0; i < fasttrap_procs.fth_nent; i++) 2359 mutex_init(&fasttrap_procs.fth_table[i].ftb_mtx, 2360 "processes bucket mtx", MUTEX_DEFAULT, NULL); 2361 2362 CPU_FOREACH(i) { 2363 mutex_init(&fasttrap_cpuc_pid_lock[i], "fasttrap barrier", 2364 MUTEX_DEFAULT, NULL); 2365 }
| 1569 mutex_exit(&proc->ftpc_mtx); 1570 1571 /* 1572 * There should definitely be no live providers associated with this 1573 * process at this point. 1574 */ 1575 ASSERT(proc->ftpc_acount == 0); 1576 1577 bucket = &fasttrap_procs.fth_table[FASTTRAP_PROCS_INDEX(pid)]; 1578 mutex_enter(&bucket->ftb_mtx); 1579 1580 fprcp = (fasttrap_proc_t **)&bucket->ftb_data; 1581 while ((fprc = *fprcp) != NULL) { 1582 if (fprc == proc) 1583 break; 1584 1585 fprcp = &fprc->ftpc_next; 1586 } 1587 1588 /* 1589 * Something strange has happened if we can't find the proc. 1590 */ 1591 ASSERT(fprc != NULL); 1592 1593 *fprcp = fprc->ftpc_next; 1594 1595 mutex_exit(&bucket->ftb_mtx); 1596 1597 kmem_free(fprc, sizeof (fasttrap_proc_t)); 1598} 1599 1600/* 1601 * Lookup a fasttrap-managed provider based on its name and associated pid. 1602 * If the pattr argument is non-NULL, this function instantiates the provider 1603 * if it doesn't exist otherwise it returns NULL. The provider is returned 1604 * with its lock held. 1605 */ 1606static fasttrap_provider_t * 1607fasttrap_provider_lookup(pid_t pid, const char *name, 1608 const dtrace_pattr_t *pattr) 1609{ 1610 fasttrap_provider_t *fp, *new_fp = NULL; 1611 fasttrap_bucket_t *bucket; 1612 char provname[DTRACE_PROVNAMELEN]; 1613 proc_t *p; 1614 cred_t *cred; 1615 1616 ASSERT(strlen(name) < sizeof (fp->ftp_name)); 1617 ASSERT(pattr != NULL); 1618 1619 bucket = &fasttrap_provs.fth_table[FASTTRAP_PROVS_INDEX(pid, name)]; 1620 mutex_enter(&bucket->ftb_mtx); 1621 1622 /* 1623 * Take a lap through the list and return the match if we find it. 1624 */ 1625 for (fp = bucket->ftb_data; fp != NULL; fp = fp->ftp_next) { 1626 if (fp->ftp_pid == pid && strcmp(fp->ftp_name, name) == 0 && 1627 !fp->ftp_retired) { 1628 mutex_enter(&fp->ftp_mtx); 1629 mutex_exit(&bucket->ftb_mtx); 1630 return (fp); 1631 } 1632 } 1633 1634 /* 1635 * Drop the bucket lock so we don't try to perform a sleeping 1636 * allocation under it. 1637 */ 1638 mutex_exit(&bucket->ftb_mtx); 1639 1640 /* 1641 * Make sure the process exists, isn't a child created as the result 1642 * of a vfork(2), and isn't a zombie (but may be in fork). 1643 */ 1644 if ((p = pfind(pid)) == NULL) 1645 return (NULL); 1646 1647 /* 1648 * Increment p_dtrace_probes so that the process knows to inform us 1649 * when it exits or execs. fasttrap_provider_free() decrements this 1650 * when we're done with this provider. 1651 */ 1652 p->p_dtrace_probes++; 1653 1654 /* 1655 * Grab the credentials for this process so we have 1656 * something to pass to dtrace_register(). 1657 */ 1658 PROC_LOCK_ASSERT(p, MA_OWNED); 1659 crhold(p->p_ucred); 1660 cred = p->p_ucred; 1661 PROC_UNLOCK(p); 1662 1663 new_fp = kmem_zalloc(sizeof (fasttrap_provider_t), KM_SLEEP); 1664 new_fp->ftp_pid = pid; 1665 new_fp->ftp_proc = fasttrap_proc_lookup(pid); 1666#if !defined(sun) 1667 mutex_init(&new_fp->ftp_mtx, "provider mtx", MUTEX_DEFAULT, NULL); 1668 mutex_init(&new_fp->ftp_cmtx, "lock on creating", MUTEX_DEFAULT, NULL); 1669#endif 1670 1671 ASSERT(new_fp->ftp_proc != NULL); 1672 1673 mutex_enter(&bucket->ftb_mtx); 1674 1675 /* 1676 * Take another lap through the list to make sure a provider hasn't 1677 * been created for this pid while we weren't under the bucket lock. 1678 */ 1679 for (fp = bucket->ftb_data; fp != NULL; fp = fp->ftp_next) { 1680 if (fp->ftp_pid == pid && strcmp(fp->ftp_name, name) == 0 && 1681 !fp->ftp_retired) { 1682 mutex_enter(&fp->ftp_mtx); 1683 mutex_exit(&bucket->ftb_mtx); 1684 fasttrap_provider_free(new_fp); 1685 crfree(cred); 1686 return (fp); 1687 } 1688 } 1689 1690 (void) strcpy(new_fp->ftp_name, name); 1691 1692 /* 1693 * Fail and return NULL if either the provider name is too long 1694 * or we fail to register this new provider with the DTrace 1695 * framework. Note that this is the only place we ever construct 1696 * the full provider name -- we keep it in pieces in the provider 1697 * structure. 1698 */ 1699 if (snprintf(provname, sizeof (provname), "%s%u", name, (uint_t)pid) >= 1700 sizeof (provname) || 1701 dtrace_register(provname, pattr, 1702 DTRACE_PRIV_PROC | DTRACE_PRIV_OWNER | DTRACE_PRIV_ZONEOWNER, cred, 1703 pattr == &pid_attr ? &pid_pops : &usdt_pops, new_fp, 1704 &new_fp->ftp_provid) != 0) { 1705 mutex_exit(&bucket->ftb_mtx); 1706 fasttrap_provider_free(new_fp); 1707 crfree(cred); 1708 return (NULL); 1709 } 1710 1711 new_fp->ftp_next = bucket->ftb_data; 1712 bucket->ftb_data = new_fp; 1713 1714 mutex_enter(&new_fp->ftp_mtx); 1715 mutex_exit(&bucket->ftb_mtx); 1716 1717 crfree(cred); 1718 return (new_fp); 1719} 1720 1721static void 1722fasttrap_provider_free(fasttrap_provider_t *provider) 1723{ 1724 pid_t pid = provider->ftp_pid; 1725 proc_t *p; 1726 1727 /* 1728 * There need to be no associated enabled probes, no consumers 1729 * creating probes, and no meta providers referencing this provider. 1730 */ 1731 ASSERT(provider->ftp_rcount == 0); 1732 ASSERT(provider->ftp_ccount == 0); 1733 ASSERT(provider->ftp_mcount == 0); 1734 1735 /* 1736 * If this provider hasn't been retired, we need to explicitly drop the 1737 * count of active providers on the associated process structure. 1738 */ 1739 if (!provider->ftp_retired) { 1740 atomic_add_64(&provider->ftp_proc->ftpc_acount, -1); 1741 ASSERT(provider->ftp_proc->ftpc_acount < 1742 provider->ftp_proc->ftpc_rcount); 1743 } 1744 1745 fasttrap_proc_release(provider->ftp_proc); 1746 1747#if !defined(sun) 1748 mutex_destroy(&provider->ftp_mtx); 1749 mutex_destroy(&provider->ftp_cmtx); 1750#endif 1751 kmem_free(provider, sizeof (fasttrap_provider_t)); 1752 1753 /* 1754 * Decrement p_dtrace_probes on the process whose provider we're 1755 * freeing. We don't have to worry about clobbering somone else's 1756 * modifications to it because we have locked the bucket that 1757 * corresponds to this process's hash chain in the provider hash 1758 * table. Don't sweat it if we can't find the process. 1759 */ 1760 if ((p = pfind(pid)) == NULL) { 1761 return; 1762 } 1763 1764 p->p_dtrace_probes--; 1765#if !defined(sun) 1766 PROC_UNLOCK(p); 1767#endif 1768} 1769 1770static void 1771fasttrap_provider_retire(pid_t pid, const char *name, int mprov) 1772{ 1773 fasttrap_provider_t *fp; 1774 fasttrap_bucket_t *bucket; 1775 dtrace_provider_id_t provid; 1776 1777 ASSERT(strlen(name) < sizeof (fp->ftp_name)); 1778 1779 bucket = &fasttrap_provs.fth_table[FASTTRAP_PROVS_INDEX(pid, name)]; 1780 mutex_enter(&bucket->ftb_mtx); 1781 1782 for (fp = bucket->ftb_data; fp != NULL; fp = fp->ftp_next) { 1783 if (fp->ftp_pid == pid && strcmp(fp->ftp_name, name) == 0 && 1784 !fp->ftp_retired) 1785 break; 1786 } 1787 1788 if (fp == NULL) { 1789 mutex_exit(&bucket->ftb_mtx); 1790 return; 1791 } 1792 1793 mutex_enter(&fp->ftp_mtx); 1794 ASSERT(!mprov || fp->ftp_mcount > 0); 1795 if (mprov && --fp->ftp_mcount != 0) { 1796 mutex_exit(&fp->ftp_mtx); 1797 mutex_exit(&bucket->ftb_mtx); 1798 return; 1799 } 1800 1801 /* 1802 * Mark the provider to be removed in our post-processing step, mark it 1803 * retired, and drop the active count on its proc. Marking it indicates 1804 * that we should try to remove it; setting the retired flag indicates 1805 * that we're done with this provider; dropping the active the proc 1806 * releases our hold, and when this reaches zero (as it will during 1807 * exit or exec) the proc and associated providers become defunct. 1808 * 1809 * We obviously need to take the bucket lock before the provider lock 1810 * to perform the lookup, but we need to drop the provider lock 1811 * before calling into the DTrace framework since we acquire the 1812 * provider lock in callbacks invoked from the DTrace framework. The 1813 * bucket lock therefore protects the integrity of the provider hash 1814 * table. 1815 */ 1816 atomic_add_64(&fp->ftp_proc->ftpc_acount, -1); 1817 ASSERT(fp->ftp_proc->ftpc_acount < fp->ftp_proc->ftpc_rcount); 1818 1819 fp->ftp_retired = 1; 1820 fp->ftp_marked = 1; 1821 provid = fp->ftp_provid; 1822 mutex_exit(&fp->ftp_mtx); 1823 1824 /* 1825 * We don't have to worry about invalidating the same provider twice 1826 * since fasttrap_provider_lookup() will ignore provider that have 1827 * been marked as retired. 1828 */ 1829 dtrace_invalidate(provid); 1830 1831 mutex_exit(&bucket->ftb_mtx); 1832 1833 fasttrap_pid_cleanup(); 1834} 1835 1836static int 1837fasttrap_uint32_cmp(const void *ap, const void *bp) 1838{ 1839 return (*(const uint32_t *)ap - *(const uint32_t *)bp); 1840} 1841 1842static int 1843fasttrap_uint64_cmp(const void *ap, const void *bp) 1844{ 1845 return (*(const uint64_t *)ap - *(const uint64_t *)bp); 1846} 1847 1848static int 1849fasttrap_add_probe(fasttrap_probe_spec_t *pdata) 1850{ 1851 fasttrap_provider_t *provider; 1852 fasttrap_probe_t *pp; 1853 fasttrap_tracepoint_t *tp; 1854 char *name; 1855 int i, aframes = 0, whack; 1856 1857 /* 1858 * There needs to be at least one desired trace point. 1859 */ 1860 if (pdata->ftps_noffs == 0) 1861 return (EINVAL); 1862 1863 switch (pdata->ftps_type) { 1864 case DTFTP_ENTRY: 1865 name = "entry"; 1866 aframes = FASTTRAP_ENTRY_AFRAMES; 1867 break; 1868 case DTFTP_RETURN: 1869 name = "return"; 1870 aframes = FASTTRAP_RETURN_AFRAMES; 1871 break; 1872 case DTFTP_OFFSETS: 1873 name = NULL; 1874 break; 1875 default: 1876 return (EINVAL); 1877 } 1878 1879 if ((provider = fasttrap_provider_lookup(pdata->ftps_pid, 1880 FASTTRAP_PID_NAME, &pid_attr)) == NULL) 1881 return (ESRCH); 1882 1883 /* 1884 * Increment this reference count to indicate that a consumer is 1885 * actively adding a new probe associated with this provider. This 1886 * prevents the provider from being deleted -- we'll need to check 1887 * for pending deletions when we drop this reference count. 1888 */ 1889 provider->ftp_ccount++; 1890 mutex_exit(&provider->ftp_mtx); 1891 1892 /* 1893 * Grab the creation lock to ensure consistency between calls to 1894 * dtrace_probe_lookup() and dtrace_probe_create() in the face of 1895 * other threads creating probes. We must drop the provider lock 1896 * before taking this lock to avoid a three-way deadlock with the 1897 * DTrace framework. 1898 */ 1899 mutex_enter(&provider->ftp_cmtx); 1900 1901 if (name == NULL) { 1902 for (i = 0; i < pdata->ftps_noffs; i++) { 1903 char name_str[17]; 1904 1905 (void) sprintf(name_str, "%llx", 1906 (unsigned long long)pdata->ftps_offs[i]); 1907 1908 if (dtrace_probe_lookup(provider->ftp_provid, 1909 pdata->ftps_mod, pdata->ftps_func, name_str) != 0) 1910 continue; 1911 1912 atomic_add_32(&fasttrap_total, 1); 1913 1914 if (fasttrap_total > fasttrap_max) { 1915 atomic_add_32(&fasttrap_total, -1); 1916 goto no_mem; 1917 } 1918 1919 pp = kmem_zalloc(sizeof (fasttrap_probe_t), KM_SLEEP); 1920 1921 pp->ftp_prov = provider; 1922 pp->ftp_faddr = pdata->ftps_pc; 1923 pp->ftp_fsize = pdata->ftps_size; 1924 pp->ftp_pid = pdata->ftps_pid; 1925 pp->ftp_ntps = 1; 1926 1927 tp = kmem_zalloc(sizeof (fasttrap_tracepoint_t), 1928 KM_SLEEP); 1929 1930 tp->ftt_proc = provider->ftp_proc; 1931 tp->ftt_pc = pdata->ftps_offs[i] + pdata->ftps_pc; 1932 tp->ftt_pid = pdata->ftps_pid; 1933 1934 pp->ftp_tps[0].fit_tp = tp; 1935 pp->ftp_tps[0].fit_id.fti_probe = pp; 1936 pp->ftp_tps[0].fit_id.fti_ptype = pdata->ftps_type; 1937 1938 pp->ftp_id = dtrace_probe_create(provider->ftp_provid, 1939 pdata->ftps_mod, pdata->ftps_func, name_str, 1940 FASTTRAP_OFFSET_AFRAMES, pp); 1941 } 1942 1943 } else if (dtrace_probe_lookup(provider->ftp_provid, pdata->ftps_mod, 1944 pdata->ftps_func, name) == 0) { 1945 atomic_add_32(&fasttrap_total, pdata->ftps_noffs); 1946 1947 if (fasttrap_total > fasttrap_max) { 1948 atomic_add_32(&fasttrap_total, -pdata->ftps_noffs); 1949 goto no_mem; 1950 } 1951 1952 /* 1953 * Make sure all tracepoint program counter values are unique. 1954 * We later assume that each probe has exactly one tracepoint 1955 * for a given pc. 1956 */ 1957 qsort(pdata->ftps_offs, pdata->ftps_noffs, 1958 sizeof (uint64_t), fasttrap_uint64_cmp); 1959 for (i = 1; i < pdata->ftps_noffs; i++) { 1960 if (pdata->ftps_offs[i] > pdata->ftps_offs[i - 1]) 1961 continue; 1962 1963 atomic_add_32(&fasttrap_total, -pdata->ftps_noffs); 1964 goto no_mem; 1965 } 1966 1967 ASSERT(pdata->ftps_noffs > 0); 1968 pp = kmem_zalloc(offsetof(fasttrap_probe_t, 1969 ftp_tps[pdata->ftps_noffs]), KM_SLEEP); 1970 1971 pp->ftp_prov = provider; 1972 pp->ftp_faddr = pdata->ftps_pc; 1973 pp->ftp_fsize = pdata->ftps_size; 1974 pp->ftp_pid = pdata->ftps_pid; 1975 pp->ftp_ntps = pdata->ftps_noffs; 1976 1977 for (i = 0; i < pdata->ftps_noffs; i++) { 1978 tp = kmem_zalloc(sizeof (fasttrap_tracepoint_t), 1979 KM_SLEEP); 1980 1981 tp->ftt_proc = provider->ftp_proc; 1982 tp->ftt_pc = pdata->ftps_offs[i] + pdata->ftps_pc; 1983 tp->ftt_pid = pdata->ftps_pid; 1984 1985 pp->ftp_tps[i].fit_tp = tp; 1986 pp->ftp_tps[i].fit_id.fti_probe = pp; 1987 pp->ftp_tps[i].fit_id.fti_ptype = pdata->ftps_type; 1988 } 1989 1990 pp->ftp_id = dtrace_probe_create(provider->ftp_provid, 1991 pdata->ftps_mod, pdata->ftps_func, name, aframes, pp); 1992 } 1993 1994 mutex_exit(&provider->ftp_cmtx); 1995 1996 /* 1997 * We know that the provider is still valid since we incremented the 1998 * creation reference count. If someone tried to clean up this provider 1999 * while we were using it (e.g. because the process called exec(2) or 2000 * exit(2)), take note of that and try to clean it up now. 2001 */ 2002 mutex_enter(&provider->ftp_mtx); 2003 provider->ftp_ccount--; 2004 whack = provider->ftp_retired; 2005 mutex_exit(&provider->ftp_mtx); 2006 2007 if (whack) 2008 fasttrap_pid_cleanup(); 2009 2010 return (0); 2011 2012no_mem: 2013 /* 2014 * If we've exhausted the allowable resources, we'll try to remove 2015 * this provider to free some up. This is to cover the case where 2016 * the user has accidentally created many more probes than was 2017 * intended (e.g. pid123:::). 2018 */ 2019 mutex_exit(&provider->ftp_cmtx); 2020 mutex_enter(&provider->ftp_mtx); 2021 provider->ftp_ccount--; 2022 provider->ftp_marked = 1; 2023 mutex_exit(&provider->ftp_mtx); 2024 2025 fasttrap_pid_cleanup(); 2026 2027 return (ENOMEM); 2028} 2029 2030/*ARGSUSED*/ 2031static void * 2032fasttrap_meta_provide(void *arg, dtrace_helper_provdesc_t *dhpv, pid_t pid) 2033{ 2034 fasttrap_provider_t *provider; 2035 2036 /* 2037 * A 32-bit unsigned integer (like a pid for example) can be 2038 * expressed in 10 or fewer decimal digits. Make sure that we'll 2039 * have enough space for the provider name. 2040 */ 2041 if (strlen(dhpv->dthpv_provname) + 10 >= 2042 sizeof (provider->ftp_name)) { 2043 printf("failed to instantiate provider %s: " 2044 "name too long to accomodate pid", dhpv->dthpv_provname); 2045 return (NULL); 2046 } 2047 2048 /* 2049 * Don't let folks spoof the true pid provider. 2050 */ 2051 if (strcmp(dhpv->dthpv_provname, FASTTRAP_PID_NAME) == 0) { 2052 printf("failed to instantiate provider %s: " 2053 "%s is an invalid name", dhpv->dthpv_provname, 2054 FASTTRAP_PID_NAME); 2055 return (NULL); 2056 } 2057 2058 /* 2059 * The highest stability class that fasttrap supports is ISA; cap 2060 * the stability of the new provider accordingly. 2061 */ 2062 if (dhpv->dthpv_pattr.dtpa_provider.dtat_class > DTRACE_CLASS_ISA) 2063 dhpv->dthpv_pattr.dtpa_provider.dtat_class = DTRACE_CLASS_ISA; 2064 if (dhpv->dthpv_pattr.dtpa_mod.dtat_class > DTRACE_CLASS_ISA) 2065 dhpv->dthpv_pattr.dtpa_mod.dtat_class = DTRACE_CLASS_ISA; 2066 if (dhpv->dthpv_pattr.dtpa_func.dtat_class > DTRACE_CLASS_ISA) 2067 dhpv->dthpv_pattr.dtpa_func.dtat_class = DTRACE_CLASS_ISA; 2068 if (dhpv->dthpv_pattr.dtpa_name.dtat_class > DTRACE_CLASS_ISA) 2069 dhpv->dthpv_pattr.dtpa_name.dtat_class = DTRACE_CLASS_ISA; 2070 if (dhpv->dthpv_pattr.dtpa_args.dtat_class > DTRACE_CLASS_ISA) 2071 dhpv->dthpv_pattr.dtpa_args.dtat_class = DTRACE_CLASS_ISA; 2072 2073 if ((provider = fasttrap_provider_lookup(pid, dhpv->dthpv_provname, 2074 &dhpv->dthpv_pattr)) == NULL) { 2075 printf("failed to instantiate provider %s for " 2076 "process %u", dhpv->dthpv_provname, (uint_t)pid); 2077 return (NULL); 2078 } 2079 2080 /* 2081 * Up the meta provider count so this provider isn't removed until 2082 * the meta provider has been told to remove it. 2083 */ 2084 provider->ftp_mcount++; 2085 2086 mutex_exit(&provider->ftp_mtx); 2087 2088 return (provider); 2089} 2090 2091/*ARGSUSED*/ 2092static void 2093fasttrap_meta_create_probe(void *arg, void *parg, 2094 dtrace_helper_probedesc_t *dhpb) 2095{ 2096 fasttrap_provider_t *provider = parg; 2097 fasttrap_probe_t *pp; 2098 fasttrap_tracepoint_t *tp; 2099 int i, j; 2100 uint32_t ntps; 2101 2102 /* 2103 * Since the meta provider count is non-zero we don't have to worry 2104 * about this provider disappearing. 2105 */ 2106 ASSERT(provider->ftp_mcount > 0); 2107 2108 /* 2109 * The offsets must be unique. 2110 */ 2111 qsort(dhpb->dthpb_offs, dhpb->dthpb_noffs, sizeof (uint32_t), 2112 fasttrap_uint32_cmp); 2113 for (i = 1; i < dhpb->dthpb_noffs; i++) { 2114 if (dhpb->dthpb_base + dhpb->dthpb_offs[i] <= 2115 dhpb->dthpb_base + dhpb->dthpb_offs[i - 1]) 2116 return; 2117 } 2118 2119 qsort(dhpb->dthpb_enoffs, dhpb->dthpb_nenoffs, sizeof (uint32_t), 2120 fasttrap_uint32_cmp); 2121 for (i = 1; i < dhpb->dthpb_nenoffs; i++) { 2122 if (dhpb->dthpb_base + dhpb->dthpb_enoffs[i] <= 2123 dhpb->dthpb_base + dhpb->dthpb_enoffs[i - 1]) 2124 return; 2125 } 2126 2127 /* 2128 * Grab the creation lock to ensure consistency between calls to 2129 * dtrace_probe_lookup() and dtrace_probe_create() in the face of 2130 * other threads creating probes. 2131 */ 2132 mutex_enter(&provider->ftp_cmtx); 2133 2134 if (dtrace_probe_lookup(provider->ftp_provid, dhpb->dthpb_mod, 2135 dhpb->dthpb_func, dhpb->dthpb_name) != 0) { 2136 mutex_exit(&provider->ftp_cmtx); 2137 return; 2138 } 2139 2140 ntps = dhpb->dthpb_noffs + dhpb->dthpb_nenoffs; 2141 ASSERT(ntps > 0); 2142 2143 atomic_add_32(&fasttrap_total, ntps); 2144 2145 if (fasttrap_total > fasttrap_max) { 2146 atomic_add_32(&fasttrap_total, -ntps); 2147 mutex_exit(&provider->ftp_cmtx); 2148 return; 2149 } 2150 2151 pp = kmem_zalloc(offsetof(fasttrap_probe_t, ftp_tps[ntps]), KM_SLEEP); 2152 2153 pp->ftp_prov = provider; 2154 pp->ftp_pid = provider->ftp_pid; 2155 pp->ftp_ntps = ntps; 2156 pp->ftp_nargs = dhpb->dthpb_xargc; 2157 pp->ftp_xtypes = dhpb->dthpb_xtypes; 2158 pp->ftp_ntypes = dhpb->dthpb_ntypes; 2159 2160 /* 2161 * First create a tracepoint for each actual point of interest. 2162 */ 2163 for (i = 0; i < dhpb->dthpb_noffs; i++) { 2164 tp = kmem_zalloc(sizeof (fasttrap_tracepoint_t), KM_SLEEP); 2165 2166 tp->ftt_proc = provider->ftp_proc; 2167 tp->ftt_pc = dhpb->dthpb_base + dhpb->dthpb_offs[i]; 2168 tp->ftt_pid = provider->ftp_pid; 2169 2170 pp->ftp_tps[i].fit_tp = tp; 2171 pp->ftp_tps[i].fit_id.fti_probe = pp; 2172#ifdef __sparc 2173 pp->ftp_tps[i].fit_id.fti_ptype = DTFTP_POST_OFFSETS; 2174#else 2175 pp->ftp_tps[i].fit_id.fti_ptype = DTFTP_OFFSETS; 2176#endif 2177 } 2178 2179 /* 2180 * Then create a tracepoint for each is-enabled point. 2181 */ 2182 for (j = 0; i < ntps; i++, j++) { 2183 tp = kmem_zalloc(sizeof (fasttrap_tracepoint_t), KM_SLEEP); 2184 2185 tp->ftt_proc = provider->ftp_proc; 2186 tp->ftt_pc = dhpb->dthpb_base + dhpb->dthpb_enoffs[j]; 2187 tp->ftt_pid = provider->ftp_pid; 2188 2189 pp->ftp_tps[i].fit_tp = tp; 2190 pp->ftp_tps[i].fit_id.fti_probe = pp; 2191 pp->ftp_tps[i].fit_id.fti_ptype = DTFTP_IS_ENABLED; 2192 } 2193 2194 /* 2195 * If the arguments are shuffled around we set the argument remapping 2196 * table. Later, when the probe fires, we only remap the arguments 2197 * if the table is non-NULL. 2198 */ 2199 for (i = 0; i < dhpb->dthpb_xargc; i++) { 2200 if (dhpb->dthpb_args[i] != i) { 2201 pp->ftp_argmap = dhpb->dthpb_args; 2202 break; 2203 } 2204 } 2205 2206 /* 2207 * The probe is fully constructed -- register it with DTrace. 2208 */ 2209 pp->ftp_id = dtrace_probe_create(provider->ftp_provid, dhpb->dthpb_mod, 2210 dhpb->dthpb_func, dhpb->dthpb_name, FASTTRAP_OFFSET_AFRAMES, pp); 2211 2212 mutex_exit(&provider->ftp_cmtx); 2213} 2214 2215/*ARGSUSED*/ 2216static void 2217fasttrap_meta_remove(void *arg, dtrace_helper_provdesc_t *dhpv, pid_t pid) 2218{ 2219 /* 2220 * Clean up the USDT provider. There may be active consumers of the 2221 * provider busy adding probes, no damage will actually befall the 2222 * provider until that count has dropped to zero. This just puts 2223 * the provider on death row. 2224 */ 2225 fasttrap_provider_retire(pid, dhpv->dthpv_provname, 1); 2226} 2227 2228static dtrace_mops_t fasttrap_mops = { 2229 fasttrap_meta_create_probe, 2230 fasttrap_meta_provide, 2231 fasttrap_meta_remove 2232}; 2233 2234/*ARGSUSED*/ 2235static int 2236fasttrap_open(struct cdev *dev __unused, int oflags __unused, 2237 int devtype __unused, struct thread *td __unused) 2238{ 2239 return (0); 2240} 2241 2242/*ARGSUSED*/ 2243static int 2244fasttrap_ioctl(struct cdev *dev, u_long cmd, caddr_t arg, int fflag, 2245 struct thread *td) 2246{ 2247#ifdef notyet 2248 struct kinfo_proc kp; 2249 const cred_t *cr = td->td_ucred; 2250#endif 2251 if (!dtrace_attached()) 2252 return (EAGAIN); 2253 2254 if (cmd == FASTTRAPIOC_MAKEPROBE) { 2255 fasttrap_probe_spec_t *uprobe = *(fasttrap_probe_spec_t **)arg; 2256 fasttrap_probe_spec_t *probe; 2257 uint64_t noffs; 2258 size_t size; 2259 int ret; 2260 char *c; 2261 2262 if (copyin(&uprobe->ftps_noffs, &noffs, 2263 sizeof (uprobe->ftps_noffs))) 2264 return (EFAULT); 2265 2266 /* 2267 * Probes must have at least one tracepoint. 2268 */ 2269 if (noffs == 0) 2270 return (EINVAL); 2271 2272 size = sizeof (fasttrap_probe_spec_t) + 2273 sizeof (probe->ftps_offs[0]) * (noffs - 1); 2274 2275 if (size > 1024 * 1024) 2276 return (ENOMEM); 2277 2278 probe = kmem_alloc(size, KM_SLEEP); 2279 2280 if (copyin(uprobe, probe, size) != 0) { 2281 kmem_free(probe, size); 2282 return (EFAULT); 2283 } 2284 2285 /* 2286 * Verify that the function and module strings contain no 2287 * funny characters. 2288 */ 2289 for (c = &probe->ftps_func[0]; *c != '\0'; c++) { 2290 if (*c < 0x20 || 0x7f <= *c) { 2291 ret = EINVAL; 2292 goto err; 2293 } 2294 } 2295 2296 for (c = &probe->ftps_mod[0]; *c != '\0'; c++) { 2297 if (*c < 0x20 || 0x7f <= *c) { 2298 ret = EINVAL; 2299 goto err; 2300 } 2301 } 2302 2303#ifdef notyet 2304 if (!PRIV_POLICY_CHOICE(cr, PRIV_ALL, B_FALSE)) { 2305 proc_t *p; 2306 pid_t pid = probe->ftps_pid; 2307 2308#if defined(sun) 2309 mutex_enter(&pidlock); 2310#endif 2311 /* 2312 * Report an error if the process doesn't exist 2313 * or is actively being birthed. 2314 */ 2315 p = pfind(pid); 2316 if (p) 2317 fill_kinfo_proc(p, &kp); 2318 if (p == NULL || kp.ki_stat == SIDL) { 2319#if defined(sun) 2320 mutex_exit(&pidlock); 2321#endif 2322 return (ESRCH); 2323 } 2324#if defined(sun) 2325 mutex_enter(&p->p_lock); 2326 mutex_exit(&pidlock); 2327#else 2328 PROC_LOCK_ASSERT(p, MA_OWNED); 2329#endif 2330 2331#ifdef notyet 2332 if ((ret = priv_proc_cred_perm(cr, p, NULL, 2333 VREAD | VWRITE)) != 0) { 2334#if defined(sun) 2335 mutex_exit(&p->p_lock); 2336#else 2337 PROC_UNLOCK(p); 2338#endif 2339 return (ret); 2340 } 2341#endif /* notyet */ 2342#if defined(sun) 2343 mutex_exit(&p->p_lock); 2344#else 2345 PROC_UNLOCK(p); 2346#endif 2347 } 2348#endif /* notyet */ 2349 2350 ret = fasttrap_add_probe(probe); 2351err: 2352 kmem_free(probe, size); 2353 2354 return (ret); 2355 2356 } else if (cmd == FASTTRAPIOC_GETINSTR) { 2357 fasttrap_instr_query_t instr; 2358 fasttrap_tracepoint_t *tp; 2359 uint_t index; 2360#if defined(sun) 2361 int ret; 2362#endif 2363 2364#if defined(sun) 2365 if (copyin((void *)arg, &instr, sizeof (instr)) != 0) 2366 return (EFAULT); 2367#endif 2368 2369#ifdef notyet 2370 if (!PRIV_POLICY_CHOICE(cr, PRIV_ALL, B_FALSE)) { 2371 proc_t *p; 2372 pid_t pid = instr.ftiq_pid; 2373 2374#if defined(sun) 2375 mutex_enter(&pidlock); 2376#endif 2377 /* 2378 * Report an error if the process doesn't exist 2379 * or is actively being birthed. 2380 */ 2381 p = pfind(pid); 2382 if (p) 2383 fill_kinfo_proc(p, &kp); 2384 if (p == NULL || kp.ki_stat == SIDL) { 2385#if defined(sun) 2386 mutex_exit(&pidlock); 2387#endif 2388 return (ESRCH); 2389 } 2390#if defined(sun) 2391 mutex_enter(&p->p_lock); 2392 mutex_exit(&pidlock); 2393#else 2394 PROC_LOCK_ASSERT(p, MA_OWNED); 2395#endif 2396 2397#ifdef notyet 2398 if ((ret = priv_proc_cred_perm(cr, p, NULL, 2399 VREAD)) != 0) { 2400#if defined(sun) 2401 mutex_exit(&p->p_lock); 2402#else 2403 PROC_UNLOCK(p); 2404#endif 2405 return (ret); 2406 } 2407#endif /* notyet */ 2408 2409#if defined(sun) 2410 mutex_exit(&p->p_lock); 2411#else 2412 PROC_UNLOCK(p); 2413#endif 2414 } 2415#endif /* notyet */ 2416 2417 index = FASTTRAP_TPOINTS_INDEX(instr.ftiq_pid, instr.ftiq_pc); 2418 2419 mutex_enter(&fasttrap_tpoints.fth_table[index].ftb_mtx); 2420 tp = fasttrap_tpoints.fth_table[index].ftb_data; 2421 while (tp != NULL) { 2422 if (instr.ftiq_pid == tp->ftt_pid && 2423 instr.ftiq_pc == tp->ftt_pc && 2424 tp->ftt_proc->ftpc_acount != 0) 2425 break; 2426 2427 tp = tp->ftt_next; 2428 } 2429 2430 if (tp == NULL) { 2431 mutex_exit(&fasttrap_tpoints.fth_table[index].ftb_mtx); 2432 return (ENOENT); 2433 } 2434 2435 bcopy(&tp->ftt_instr, &instr.ftiq_instr, 2436 sizeof (instr.ftiq_instr)); 2437 mutex_exit(&fasttrap_tpoints.fth_table[index].ftb_mtx); 2438 2439 if (copyout(&instr, (void *)arg, sizeof (instr)) != 0) 2440 return (EFAULT); 2441 2442 return (0); 2443 } 2444 2445 return (EINVAL); 2446} 2447 2448static int 2449fasttrap_load(void) 2450{ 2451 ulong_t nent; 2452 int i, ret; 2453 2454 /* Create the /dev/dtrace/fasttrap entry. */ 2455 fasttrap_cdev = make_dev(&fasttrap_cdevsw, 0, UID_ROOT, GID_WHEEL, 0600, 2456 "dtrace/fasttrap"); 2457 2458 mtx_init(&fasttrap_cleanup_mtx, "fasttrap clean", "dtrace", MTX_DEF); 2459 mutex_init(&fasttrap_count_mtx, "fasttrap count mtx", MUTEX_DEFAULT, 2460 NULL); 2461 2462#if defined(sun) 2463 fasttrap_max = ddi_getprop(DDI_DEV_T_ANY, devi, DDI_PROP_DONTPASS, 2464 "fasttrap-max-probes", FASTTRAP_MAX_DEFAULT); 2465#else 2466 fasttrap_max = FASTTRAP_MAX_DEFAULT; 2467#endif 2468 fasttrap_total = 0; 2469 2470 /* 2471 * Conjure up the tracepoints hashtable... 2472 */ 2473#if defined(sun) 2474 nent = ddi_getprop(DDI_DEV_T_ANY, devi, DDI_PROP_DONTPASS, 2475 "fasttrap-hash-size", FASTTRAP_TPOINTS_DEFAULT_SIZE); 2476#else 2477 nent = FASTTRAP_TPOINTS_DEFAULT_SIZE; 2478#endif 2479 2480 if (nent == 0 || nent > 0x1000000) 2481 nent = FASTTRAP_TPOINTS_DEFAULT_SIZE; 2482 2483 if ((nent & (nent - 1)) == 0) 2484 fasttrap_tpoints.fth_nent = nent; 2485 else 2486 fasttrap_tpoints.fth_nent = 1 << fasttrap_highbit(nent); 2487 ASSERT(fasttrap_tpoints.fth_nent > 0); 2488 fasttrap_tpoints.fth_mask = fasttrap_tpoints.fth_nent - 1; 2489 fasttrap_tpoints.fth_table = kmem_zalloc(fasttrap_tpoints.fth_nent * 2490 sizeof (fasttrap_bucket_t), KM_SLEEP); 2491#if !defined(sun) 2492 for (i = 0; i < fasttrap_tpoints.fth_nent; i++) 2493 mutex_init(&fasttrap_tpoints.fth_table[i].ftb_mtx, 2494 "tracepoints bucket mtx", MUTEX_DEFAULT, NULL); 2495#endif 2496 2497 /* 2498 * ... and the providers hash table... 2499 */ 2500 nent = FASTTRAP_PROVIDERS_DEFAULT_SIZE; 2501 if ((nent & (nent - 1)) == 0) 2502 fasttrap_provs.fth_nent = nent; 2503 else 2504 fasttrap_provs.fth_nent = 1 << fasttrap_highbit(nent); 2505 ASSERT(fasttrap_provs.fth_nent > 0); 2506 fasttrap_provs.fth_mask = fasttrap_provs.fth_nent - 1; 2507 fasttrap_provs.fth_table = kmem_zalloc(fasttrap_provs.fth_nent * 2508 sizeof (fasttrap_bucket_t), KM_SLEEP); 2509#if !defined(sun) 2510 for (i = 0; i < fasttrap_provs.fth_nent; i++) 2511 mutex_init(&fasttrap_provs.fth_table[i].ftb_mtx, 2512 "providers bucket mtx", MUTEX_DEFAULT, NULL); 2513#endif 2514 2515 ret = kproc_create(fasttrap_pid_cleanup_cb, NULL, 2516 &fasttrap_cleanup_proc, 0, 0, "ftcleanup"); 2517 if (ret != 0) { 2518 destroy_dev(fasttrap_cdev); 2519#if !defined(sun) 2520 for (i = 0; i < fasttrap_provs.fth_nent; i++) 2521 mutex_destroy(&fasttrap_provs.fth_table[i].ftb_mtx); 2522 for (i = 0; i < fasttrap_tpoints.fth_nent; i++) 2523 mutex_destroy(&fasttrap_tpoints.fth_table[i].ftb_mtx); 2524#endif 2525 kmem_free(fasttrap_provs.fth_table, fasttrap_provs.fth_nent * 2526 sizeof (fasttrap_bucket_t)); 2527 mtx_destroy(&fasttrap_cleanup_mtx); 2528 mutex_destroy(&fasttrap_count_mtx); 2529 return (ret); 2530 } 2531 2532 2533 /* 2534 * ... and the procs hash table. 2535 */ 2536 nent = FASTTRAP_PROCS_DEFAULT_SIZE; 2537 if ((nent & (nent - 1)) == 0) 2538 fasttrap_procs.fth_nent = nent; 2539 else 2540 fasttrap_procs.fth_nent = 1 << fasttrap_highbit(nent); 2541 ASSERT(fasttrap_procs.fth_nent > 0); 2542 fasttrap_procs.fth_mask = fasttrap_procs.fth_nent - 1; 2543 fasttrap_procs.fth_table = kmem_zalloc(fasttrap_procs.fth_nent * 2544 sizeof (fasttrap_bucket_t), KM_SLEEP); 2545#if !defined(sun) 2546 for (i = 0; i < fasttrap_procs.fth_nent; i++) 2547 mutex_init(&fasttrap_procs.fth_table[i].ftb_mtx, 2548 "processes bucket mtx", MUTEX_DEFAULT, NULL); 2549 2550 CPU_FOREACH(i) { 2551 mutex_init(&fasttrap_cpuc_pid_lock[i], "fasttrap barrier", 2552 MUTEX_DEFAULT, NULL); 2553 }
|
| 2554 2555 /* 2556 * This event handler must run before kdtrace_thread_dtor() since it 2557 * accesses the thread's struct kdtrace_thread. 2558 */ 2559 fasttrap_thread_dtor_tag = EVENTHANDLER_REGISTER(thread_dtor, 2560 fasttrap_thread_dtor, NULL, EVENTHANDLER_PRI_FIRST);
|
2366#endif 2367 2368 /* 2369 * Install our hooks into fork(2), exec(2), and exit(2). 2370 */ 2371 dtrace_fasttrap_fork = &fasttrap_fork; 2372 dtrace_fasttrap_exit = &fasttrap_exec_exit; 2373 dtrace_fasttrap_exec = &fasttrap_exec_exit; 2374 2375 (void) dtrace_meta_register("fasttrap", &fasttrap_mops, NULL, 2376 &fasttrap_meta_id); 2377 2378 return (0); 2379} 2380 2381static int 2382fasttrap_unload(void) 2383{ 2384 int i, fail = 0; 2385 2386 /* 2387 * Unregister the meta-provider to make sure no new fasttrap- 2388 * managed providers come along while we're trying to close up 2389 * shop. If we fail to detach, we'll need to re-register as a 2390 * meta-provider. We can fail to unregister as a meta-provider 2391 * if providers we manage still exist. 2392 */ 2393 if (fasttrap_meta_id != DTRACE_METAPROVNONE && 2394 dtrace_meta_unregister(fasttrap_meta_id) != 0) 2395 return (-1); 2396 2397 /* 2398 * Iterate over all of our providers. If there's still a process 2399 * that corresponds to that pid, fail to detach. 2400 */ 2401 for (i = 0; i < fasttrap_provs.fth_nent; i++) { 2402 fasttrap_provider_t **fpp, *fp; 2403 fasttrap_bucket_t *bucket = &fasttrap_provs.fth_table[i]; 2404 2405 mutex_enter(&bucket->ftb_mtx); 2406 fpp = (fasttrap_provider_t **)&bucket->ftb_data; 2407 while ((fp = *fpp) != NULL) { 2408 /* 2409 * Acquire and release the lock as a simple way of 2410 * waiting for any other consumer to finish with 2411 * this provider. A thread must first acquire the 2412 * bucket lock so there's no chance of another thread 2413 * blocking on the provider's lock. 2414 */ 2415 mutex_enter(&fp->ftp_mtx); 2416 mutex_exit(&fp->ftp_mtx); 2417 2418 if (dtrace_unregister(fp->ftp_provid) != 0) { 2419 fail = 1; 2420 fpp = &fp->ftp_next; 2421 } else { 2422 *fpp = fp->ftp_next; 2423 fasttrap_provider_free(fp); 2424 } 2425 } 2426 2427 mutex_exit(&bucket->ftb_mtx); 2428 } 2429 2430 if (fail) { 2431 (void) dtrace_meta_register("fasttrap", &fasttrap_mops, NULL, 2432 &fasttrap_meta_id); 2433 2434 return (-1); 2435 } 2436 2437 /* 2438 * Stop new processes from entering these hooks now, before the 2439 * fasttrap_cleanup thread runs. That way all processes will hopefully 2440 * be out of these hooks before we free fasttrap_provs.fth_table 2441 */ 2442 ASSERT(dtrace_fasttrap_fork == &fasttrap_fork); 2443 dtrace_fasttrap_fork = NULL; 2444 2445 ASSERT(dtrace_fasttrap_exec == &fasttrap_exec_exit); 2446 dtrace_fasttrap_exec = NULL; 2447 2448 ASSERT(dtrace_fasttrap_exit == &fasttrap_exec_exit); 2449 dtrace_fasttrap_exit = NULL; 2450 2451 mtx_lock(&fasttrap_cleanup_mtx); 2452 fasttrap_cleanup_drain = 1; 2453 /* Wait for the cleanup thread to finish up and signal us. */ 2454 wakeup(&fasttrap_cleanup_cv); 2455 mtx_sleep(&fasttrap_cleanup_drain, &fasttrap_cleanup_mtx, 0, "ftcld", 2456 0); 2457 fasttrap_cleanup_proc = NULL; 2458 mtx_destroy(&fasttrap_cleanup_mtx); 2459 2460#ifdef DEBUG 2461 mutex_enter(&fasttrap_count_mtx); 2462 ASSERT(fasttrap_pid_count == 0); 2463 mutex_exit(&fasttrap_count_mtx); 2464#endif 2465 2466#if !defined(sun)
| 2561#endif 2562 2563 /* 2564 * Install our hooks into fork(2), exec(2), and exit(2). 2565 */ 2566 dtrace_fasttrap_fork = &fasttrap_fork; 2567 dtrace_fasttrap_exit = &fasttrap_exec_exit; 2568 dtrace_fasttrap_exec = &fasttrap_exec_exit; 2569 2570 (void) dtrace_meta_register("fasttrap", &fasttrap_mops, NULL, 2571 &fasttrap_meta_id); 2572 2573 return (0); 2574} 2575 2576static int 2577fasttrap_unload(void) 2578{ 2579 int i, fail = 0; 2580 2581 /* 2582 * Unregister the meta-provider to make sure no new fasttrap- 2583 * managed providers come along while we're trying to close up 2584 * shop. If we fail to detach, we'll need to re-register as a 2585 * meta-provider. We can fail to unregister as a meta-provider 2586 * if providers we manage still exist. 2587 */ 2588 if (fasttrap_meta_id != DTRACE_METAPROVNONE && 2589 dtrace_meta_unregister(fasttrap_meta_id) != 0) 2590 return (-1); 2591 2592 /* 2593 * Iterate over all of our providers. If there's still a process 2594 * that corresponds to that pid, fail to detach. 2595 */ 2596 for (i = 0; i < fasttrap_provs.fth_nent; i++) { 2597 fasttrap_provider_t **fpp, *fp; 2598 fasttrap_bucket_t *bucket = &fasttrap_provs.fth_table[i]; 2599 2600 mutex_enter(&bucket->ftb_mtx); 2601 fpp = (fasttrap_provider_t **)&bucket->ftb_data; 2602 while ((fp = *fpp) != NULL) { 2603 /* 2604 * Acquire and release the lock as a simple way of 2605 * waiting for any other consumer to finish with 2606 * this provider. A thread must first acquire the 2607 * bucket lock so there's no chance of another thread 2608 * blocking on the provider's lock. 2609 */ 2610 mutex_enter(&fp->ftp_mtx); 2611 mutex_exit(&fp->ftp_mtx); 2612 2613 if (dtrace_unregister(fp->ftp_provid) != 0) { 2614 fail = 1; 2615 fpp = &fp->ftp_next; 2616 } else { 2617 *fpp = fp->ftp_next; 2618 fasttrap_provider_free(fp); 2619 } 2620 } 2621 2622 mutex_exit(&bucket->ftb_mtx); 2623 } 2624 2625 if (fail) { 2626 (void) dtrace_meta_register("fasttrap", &fasttrap_mops, NULL, 2627 &fasttrap_meta_id); 2628 2629 return (-1); 2630 } 2631 2632 /* 2633 * Stop new processes from entering these hooks now, before the 2634 * fasttrap_cleanup thread runs. That way all processes will hopefully 2635 * be out of these hooks before we free fasttrap_provs.fth_table 2636 */ 2637 ASSERT(dtrace_fasttrap_fork == &fasttrap_fork); 2638 dtrace_fasttrap_fork = NULL; 2639 2640 ASSERT(dtrace_fasttrap_exec == &fasttrap_exec_exit); 2641 dtrace_fasttrap_exec = NULL; 2642 2643 ASSERT(dtrace_fasttrap_exit == &fasttrap_exec_exit); 2644 dtrace_fasttrap_exit = NULL; 2645 2646 mtx_lock(&fasttrap_cleanup_mtx); 2647 fasttrap_cleanup_drain = 1; 2648 /* Wait for the cleanup thread to finish up and signal us. */ 2649 wakeup(&fasttrap_cleanup_cv); 2650 mtx_sleep(&fasttrap_cleanup_drain, &fasttrap_cleanup_mtx, 0, "ftcld", 2651 0); 2652 fasttrap_cleanup_proc = NULL; 2653 mtx_destroy(&fasttrap_cleanup_mtx); 2654 2655#ifdef DEBUG 2656 mutex_enter(&fasttrap_count_mtx); 2657 ASSERT(fasttrap_pid_count == 0); 2658 mutex_exit(&fasttrap_count_mtx); 2659#endif 2660 2661#if !defined(sun)
|
| 2662 EVENTHANDLER_DEREGISTER(thread_dtor, fasttrap_thread_dtor_tag); 2663
|
2467 for (i = 0; i < fasttrap_tpoints.fth_nent; i++) 2468 mutex_destroy(&fasttrap_tpoints.fth_table[i].ftb_mtx); 2469 for (i = 0; i < fasttrap_provs.fth_nent; i++) 2470 mutex_destroy(&fasttrap_provs.fth_table[i].ftb_mtx); 2471 for (i = 0; i < fasttrap_procs.fth_nent; i++) 2472 mutex_destroy(&fasttrap_procs.fth_table[i].ftb_mtx); 2473#endif 2474 kmem_free(fasttrap_tpoints.fth_table, 2475 fasttrap_tpoints.fth_nent * sizeof (fasttrap_bucket_t)); 2476 fasttrap_tpoints.fth_nent = 0; 2477 2478 kmem_free(fasttrap_provs.fth_table, 2479 fasttrap_provs.fth_nent * sizeof (fasttrap_bucket_t)); 2480 fasttrap_provs.fth_nent = 0; 2481 2482 kmem_free(fasttrap_procs.fth_table, 2483 fasttrap_procs.fth_nent * sizeof (fasttrap_bucket_t)); 2484 fasttrap_procs.fth_nent = 0; 2485 2486#if !defined(sun) 2487 destroy_dev(fasttrap_cdev); 2488 mutex_destroy(&fasttrap_count_mtx); 2489 CPU_FOREACH(i) { 2490 mutex_destroy(&fasttrap_cpuc_pid_lock[i]); 2491 } 2492#endif 2493 2494 return (0); 2495} 2496 2497/* ARGSUSED */ 2498static int 2499fasttrap_modevent(module_t mod __unused, int type, void *data __unused) 2500{ 2501 int error = 0; 2502 2503 switch (type) { 2504 case MOD_LOAD: 2505 break; 2506 2507 case MOD_UNLOAD: 2508 break; 2509 2510 case MOD_SHUTDOWN: 2511 break; 2512 2513 default: 2514 error = EOPNOTSUPP; 2515 break; 2516 } 2517 return (error); 2518} 2519 2520SYSINIT(fasttrap_load, SI_SUB_DTRACE_PROVIDER, SI_ORDER_ANY, fasttrap_load, 2521 NULL); 2522SYSUNINIT(fasttrap_unload, SI_SUB_DTRACE_PROVIDER, SI_ORDER_ANY, 2523 fasttrap_unload, NULL); 2524 2525DEV_MODULE(fasttrap, fasttrap_modevent, NULL); 2526MODULE_VERSION(fasttrap, 1); 2527MODULE_DEPEND(fasttrap, dtrace, 1, 1, 1); 2528MODULE_DEPEND(fasttrap, opensolaris, 1, 1, 1);
| 2664 for (i = 0; i < fasttrap_tpoints.fth_nent; i++) 2665 mutex_destroy(&fasttrap_tpoints.fth_table[i].ftb_mtx); 2666 for (i = 0; i < fasttrap_provs.fth_nent; i++) 2667 mutex_destroy(&fasttrap_provs.fth_table[i].ftb_mtx); 2668 for (i = 0; i < fasttrap_procs.fth_nent; i++) 2669 mutex_destroy(&fasttrap_procs.fth_table[i].ftb_mtx); 2670#endif 2671 kmem_free(fasttrap_tpoints.fth_table, 2672 fasttrap_tpoints.fth_nent * sizeof (fasttrap_bucket_t)); 2673 fasttrap_tpoints.fth_nent = 0; 2674 2675 kmem_free(fasttrap_provs.fth_table, 2676 fasttrap_provs.fth_nent * sizeof (fasttrap_bucket_t)); 2677 fasttrap_provs.fth_nent = 0; 2678 2679 kmem_free(fasttrap_procs.fth_table, 2680 fasttrap_procs.fth_nent * sizeof (fasttrap_bucket_t)); 2681 fasttrap_procs.fth_nent = 0; 2682 2683#if !defined(sun) 2684 destroy_dev(fasttrap_cdev); 2685 mutex_destroy(&fasttrap_count_mtx); 2686 CPU_FOREACH(i) { 2687 mutex_destroy(&fasttrap_cpuc_pid_lock[i]); 2688 } 2689#endif 2690 2691 return (0); 2692} 2693 2694/* ARGSUSED */ 2695static int 2696fasttrap_modevent(module_t mod __unused, int type, void *data __unused) 2697{ 2698 int error = 0; 2699 2700 switch (type) { 2701 case MOD_LOAD: 2702 break; 2703 2704 case MOD_UNLOAD: 2705 break; 2706 2707 case MOD_SHUTDOWN: 2708 break; 2709 2710 default: 2711 error = EOPNOTSUPP; 2712 break; 2713 } 2714 return (error); 2715} 2716 2717SYSINIT(fasttrap_load, SI_SUB_DTRACE_PROVIDER, SI_ORDER_ANY, fasttrap_load, 2718 NULL); 2719SYSUNINIT(fasttrap_unload, SI_SUB_DTRACE_PROVIDER, SI_ORDER_ANY, 2720 fasttrap_unload, NULL); 2721 2722DEV_MODULE(fasttrap, fasttrap_modevent, NULL); 2723MODULE_VERSION(fasttrap, 1); 2724MODULE_DEPEND(fasttrap, dtrace, 1, 1, 1); 2725MODULE_DEPEND(fasttrap, opensolaris, 1, 1, 1);
|