Cross Reference: /freebsd-11-stable/sys/cddl/contrib/opensolaris/uts/common/dtrace/fasttrap.c

Deleted Added

sdiff udiff text old ( 179198 ) new ( 184698 )

full compact

fasttrap.c (179198)	fasttrap.c (184698)
1/* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License (the "License"). 6 * You may not use this file except in compliance with the License. 7 * 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9 * or http://www.opensolaris.org/os/licensing. 10 * See the License for the specific language governing permissions 11 * and limitations under the License. 12 * 13 * When distributing Covered Code, include this CDDL HEADER in each 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15 * If applicable, add the following below this CDDL HEADER, with the 16 * fields enclosed by brackets "[]" replaced with your own identifying 17 * information: Portions Copyright [yyyy] [name of copyright owner] 18 * 19 * CDDL HEADER END 20 / 21 22/ 23 * Copyright 2008 Sun Microsystems, Inc. All rights reserved. 24 * Use is subject to license terms. 25 / 26 27#pragma ident "%Z%%M% %I% %E% SMI" 28 29#include <sys/atomic.h> 30#include <sys/errno.h> 31#include <sys/stat.h> 32#include <sys/modctl.h> 33#include <sys/conf.h> 34#include <sys/systm.h> 35#include <sys/ddi.h> 36#include <sys/sunddi.h> 37#include <sys/cpuvar.h> 38#include <sys/kmem.h> 39#include <sys/strsubr.h> 40#include <sys/fasttrap.h> 41#include <sys/fasttrap_impl.h> 42#include <sys/fasttrap_isa.h> 43#include <sys/dtrace.h> 44#include <sys/dtrace_impl.h> 45#include <sys/sysmacros.h> 46#include <sys/proc.h> 47#include <sys/priv.h> 48#include <sys/policy.h> 49#include <util/qsort.h> 50 51/ 52 * User-Land Trap-Based Tracing 53 * ---------------------------- 54 * 55 * The fasttrap provider allows DTrace consumers to instrument any user-level 56 * instruction to gather data; this includes probes with semantic 57 * signifigance like entry and return as well as simple offsets into the 58 * function. While the specific techniques used are very ISA specific, the 59 * methodology is generalizable to any architecture. 60 * 61 * 62 * The General Methodology 63 * ----------------------- 64 * 65 * With the primary goal of tracing every user-land instruction and the 66 * limitation that we can't trust user space so don't want to rely on much 67 * information there, we begin by replacing the instructions we want to trace 68 * with trap instructions. Each instruction we overwrite is saved into a hash 69 * table keyed by process ID and pc address. When we enter the kernel due to 70 * this trap instruction, we need the effects of the replaced instruction to 71 * appear to have occurred before we proceed with the user thread's 72 * execution. 73 * 74 * Each user level thread is represented by a ulwp_t structure which is 75 * always easily accessible through a register. The most basic way to produce 76 * the effects of the instruction we replaced is to copy that instruction out 77 * to a bit of scratch space reserved in the user thread's ulwp_t structure 78 * (a sort of kernel-private thread local storage), set the PC to that 79 * scratch space and single step. When we reenter the kernel after single 80 * stepping the instruction we must then adjust the PC to point to what would 81 * normally be the next instruction. Of course, special care must be taken 82 * for branches and jumps, but these represent such a small fraction of any 83 * instruction set that writing the code to emulate these in the kernel is 84 * not too difficult. 85 * 86 * Return probes may require several tracepoints to trace every return site, 87 * and, conversely, each tracepoint may activate several probes (the entry 88 * and offset 0 probes, for example). To solve this muliplexing problem, 89 * tracepoints contain lists of probes to activate and probes contain lists 90 * of tracepoints to enable. If a probe is activated, it adds its ID to 91 * existing tracepoints or creates new ones as necessary. 92 * 93 * Most probes are activated _before_ the instruction is executed, but return 94 * probes are activated _after_ the effects of the last instruction of the 95 * function are visible. Return probes must be fired _after_ we have 96 * single-stepped the instruction whereas all other probes are fired 97 * beforehand. 98 * 99 * 100 * Lock Ordering 101 * ------------- 102 * 103 * The lock ordering below -- both internally and with respect to the DTrace 104 * framework -- is a little tricky and bears some explanation. Each provider 105 * has a lock (ftp_mtx) that protects its members including reference counts 106 * for enabled probes (ftp_rcount), consumers actively creating probes 107 * (ftp_ccount) and USDT consumers (ftp_mcount); all three prevent a provider 108 * from being freed. A provider is looked up by taking the bucket lock for the 109 * provider hash table, and is returned with its lock held. The provider lock 110 * may be taken in functions invoked by the DTrace framework, but may not be 111 * held while calling functions in the DTrace framework. 112 * 113 * To ensure consistency over multiple calls to the DTrace framework, the 114 * creation lock (ftp_cmtx) should be held. Naturally, the creation lock may 115 * not be taken when holding the provider lock as that would create a cyclic 116 * lock ordering. In situations where one would naturally take the provider 117 * lock and then the creation lock, we instead up a reference count to prevent 118 * the provider from disappearing, drop the provider lock, and acquire the 119 * creation lock. 120 * 121 * Briefly: 122 * bucket lock before provider lock 123 * DTrace before provider lock 124 * creation lock before DTrace 125 * never hold the provider lock and creation lock simultaneously 126 / 127* 128static dev_info_t fasttrap_devi; 129static dtrace_meta_provider_id_t fasttrap_meta_id; 130* 131static timeout_id_t fasttrap_timeout; 132static kmutex_t fasttrap_cleanup_mtx; 133static uint_t fasttrap_cleanup_work; 134 135/* 136 * Generation count on modifications to the global tracepoint lookup table. 137 / 138static volatile uint64_t fasttrap_mod_gen; 139* 140/* 141 * When the fasttrap provider is loaded, fasttrap_max is set to either 142 * FASTTRAP_MAX_DEFAULT or the value for fasttrap-max-probes in the 143 * fasttrap.conf file. Each time a probe is created, fasttrap_total is 144 * incremented by the number of tracepoints that may be associated with that 145 * probe; fasttrap_total is capped at fasttrap_max. 146 / 147#define FASTTRAP_MAX_DEFAULT 250000 148static uint32_t fasttrap_max; 149static uint32_t fasttrap_total; 150* 151 152#define FASTTRAP_TPOINTS_DEFAULT_SIZE 0x4000 153#define FASTTRAP_PROVIDERS_DEFAULT_SIZE 0x100 154#define FASTTRAP_PROCS_DEFAULT_SIZE 0x100 155 156#define FASTTRAP_PID_NAME "pid" 157 158fasttrap_hash_t fasttrap_tpoints; 159static fasttrap_hash_t fasttrap_provs; 160static fasttrap_hash_t fasttrap_procs; 161 162static uint64_t fasttrap_pid_count; /* pid ref count / 163static kmutex_t fasttrap_count_mtx; / lock on ref count / 164* 165#define FASTTRAP_ENABLE_FAIL 1 166#define FASTTRAP_ENABLE_PARTIAL 2 167 168static int fasttrap_tracepoint_enable(proc_t , fasttrap_probe_t , uint_t); 169static void fasttrap_tracepoint_disable(proc_t , fasttrap_probe_t , uint_t); 170 171static fasttrap_provider_t fasttrap_provider_lookup(pid_t, const char , 172 const dtrace_pattr_t ); 173static void fasttrap_provider_retire(pid_t, const char , int); 174static void fasttrap_provider_free(fasttrap_provider_t ); 175* 176static fasttrap_proc_t fasttrap_proc_lookup(pid_t); 177static void fasttrap_proc_release(fasttrap_proc_t ); 178 179#define FASTTRAP_PROVS_INDEX(pid, name) \ 180 ((fasttrap_hash_str(name) + (pid)) & fasttrap_provs.fth_mask) 181 182#define FASTTRAP_PROCS_INDEX(pid) ((pid) & fasttrap_procs.fth_mask) 183 184static int 185fasttrap_highbit(ulong_t i) 186{ 187 int h = 1; 188 189 if (i == 0) 190 return (0); 191#ifdef _LP64 192 if (i & 0xffffffff00000000ul) { 193 h += 32; i >>= 32; 194 } 195#endif 196 if (i & 0xffff0000) { 197 h += 16; i >>= 16; 198 } 199 if (i & 0xff00) { 200 h += 8; i >>= 8; 201 } 202 if (i & 0xf0) { 203 h += 4; i >>= 4; 204 } 205 if (i & 0xc) { 206 h += 2; i >>= 2; 207 } 208 if (i & 0x2) { 209 h += 1; 210 } 211 return (h); 212} 213 214static uint_t 215fasttrap_hash_str(const char p) 216{ 217* unsigned int g; 218 uint_t hval = 0; 219 220 while (p) { 221* hval = (hval << 4) + p++; 222* if ((g = (hval & 0xf0000000)) != 0) 223 hval ^= g >> 24; 224 hval &= ~g; 225 } 226 return (hval); 227} 228 229void 230fasttrap_sigtrap(proc_t p, kthread_t t, uintptr_t pc) 231{ 232 sigqueue_t sqp = kmem_zalloc(sizeof (sigqueue_t), KM_SLEEP); 233* 234 sqp->sq_info.si_signo = SIGTRAP; 235 sqp->sq_info.si_code = TRAP_DTRACE; 236 sqp->sq_info.si_addr = (caddr_t)pc; 237 238 mutex_enter(&p->p_lock); 239 sigaddqa(p, t, sqp); 240 mutex_exit(&p->p_lock); 241 242 if (t != NULL) 243 aston(t); 244} 245 246/* 247 * This function ensures that no threads are actively using the memory 248 * associated with probes that were formerly live. 249 / 250static void 251fasttrap_mod_barrier(uint64_t gen) 252{ 253* int i; 254 255 if (gen < fasttrap_mod_gen) 256 return; 257 258 fasttrap_mod_gen++; 259 260 for (i = 0; i < NCPU; i++) { 261 mutex_enter(&cpu_core[i].cpuc_pid_lock); 262 mutex_exit(&cpu_core[i].cpuc_pid_lock); 263 } 264} 265 266/* 267 * This is the timeout's callback for cleaning up the providers and their 268 * probes. 269 / 270/ARGSUSED/ 271static void 272fasttrap_pid_cleanup_cb(void data) 273{ 274 fasttrap_provider_t *fpp, fp; 275 fasttrap_bucket_t bucket; 276* dtrace_provider_id_t provid; 277 int i, later; 278 279 static volatile int in = 0; 280 ASSERT(in == 0); 281 in = 1; 282 283 mutex_enter(&fasttrap_cleanup_mtx); 284 while (fasttrap_cleanup_work) { 285 fasttrap_cleanup_work = 0; 286 mutex_exit(&fasttrap_cleanup_mtx); 287 288 later = 0; 289 290 /* 291 * Iterate over all the providers trying to remove the marked 292 * ones. If a provider is marked but not retired, we just 293 * have to take a crack at removing it -- it's no big deal if 294 * we can't. 295 / 296* for (i = 0; i < fasttrap_provs.fth_nent; i++) { 297 bucket = &fasttrap_provs.fth_table[i]; 298 mutex_enter(&bucket->ftb_mtx); 299 fpp = (fasttrap_provider_t *)&bucket->ftb_data; 300* 301 while ((fp = fpp) != NULL) { 302* if (!fp->ftp_marked) { 303 fpp = &fp->ftp_next; 304 continue; 305 } 306 307 mutex_enter(&fp->ftp_mtx); 308 309 /* 310 * If this provider has consumers actively 311 * creating probes (ftp_ccount) or is a USDT 312 * provider (ftp_mcount), we can't unregister 313 * or even condense. 314 / 315* if (fp->ftp_ccount != 0 \|\| 316 fp->ftp_mcount != 0) { 317 mutex_exit(&fp->ftp_mtx); 318 fp->ftp_marked = 0; 319 continue; 320 } 321 322 if (!fp->ftp_retired \|\| fp->ftp_rcount != 0) 323 fp->ftp_marked = 0; 324 325 mutex_exit(&fp->ftp_mtx); 326 327 /* 328 * If we successfully unregister this 329 * provider we can remove it from the hash 330 * chain and free the memory. If our attempt 331 * to unregister fails and this is a retired 332 * provider, increment our flag to try again 333 * pretty soon. If we've consumed more than 334 * half of our total permitted number of 335 * probes call dtrace_condense() to try to 336 * clean out the unenabled probes. 337 / 338* provid = fp->ftp_provid; 339 if (dtrace_unregister(provid) != 0) { 340 if (fasttrap_total > fasttrap_max / 2) 341 (void) dtrace_condense(provid); 342 later += fp->ftp_marked; 343 fpp = &fp->ftp_next; 344 } else { 345 fpp = fp->ftp_next; 346* fasttrap_provider_free(fp); 347 } 348 } 349 mutex_exit(&bucket->ftb_mtx); 350 } 351 352 mutex_enter(&fasttrap_cleanup_mtx); 353 } 354 355 ASSERT(fasttrap_timeout != 0); 356 357 /* 358 * If we were unable to remove a retired provider, try again after 359 * a second. This situation can occur in certain circumstances where 360 * providers cannot be unregistered even though they have no probes 361 * enabled because of an execution of dtrace -l or something similar. 362 * If the timeout has been disabled (set to 1 because we're trying 363 * to detach), we set fasttrap_cleanup_work to ensure that we'll 364 * get a chance to do that work if and when the timeout is reenabled 365 * (if detach fails). 366 / 367* if (later > 0 && fasttrap_timeout != (timeout_id_t)1) 368 fasttrap_timeout = timeout(&fasttrap_pid_cleanup_cb, NULL, hz); 369 else if (later > 0) 370 fasttrap_cleanup_work = 1; 371 else 372 fasttrap_timeout = 0; 373 374 mutex_exit(&fasttrap_cleanup_mtx); 375 in = 0; 376} 377 378/* 379 * Activates the asynchronous cleanup mechanism. 380 / 381static void 382fasttrap_pid_cleanup(void) 383{ 384* mutex_enter(&fasttrap_cleanup_mtx); 385 fasttrap_cleanup_work = 1; 386 if (fasttrap_timeout == 0) 387 fasttrap_timeout = timeout(&fasttrap_pid_cleanup_cb, NULL, 1); 388 mutex_exit(&fasttrap_cleanup_mtx); 389} 390 391/* 392 * This is called from cfork() via dtrace_fasttrap_fork(). The child 393 * process's address space is (roughly) a copy of the parent process's so 394 * we have to remove all the instrumentation we had previously enabled in the 395 * parent. 396 / 397static void 398fasttrap_fork(proc_t p, proc_t cp) 399{ 400* pid_t ppid = p->p_pid; 401 int i; 402 403 ASSERT(curproc == p); 404 ASSERT(p->p_proc_flag & P_PR_LOCK); 405 ASSERT(p->p_dtrace_count > 0); 406 ASSERT(cp->p_dtrace_count == 0); 407 408 /* 409 * This would be simpler and faster if we maintained per-process 410 * hash tables of enabled tracepoints. It could, however, potentially 411 * slow down execution of a tracepoint since we'd need to go 412 * through two levels of indirection. In the future, we should 413 * consider either maintaining per-process ancillary lists of 414 * enabled tracepoints or hanging a pointer to a per-process hash 415 * table of enabled tracepoints off the proc structure. 416 / 417* 418 /* 419 * We don't have to worry about the child process disappearing 420 * because we're in fork(). 421 / 422* mutex_enter(&cp->p_lock); 423 sprlock_proc(cp); 424 mutex_exit(&cp->p_lock); 425 426 /* 427 * Iterate over every tracepoint looking for ones that belong to the 428 * parent process, and remove each from the child process. 429 / 430* for (i = 0; i < fasttrap_tpoints.fth_nent; i++) { 431 fasttrap_tracepoint_t tp; 432* fasttrap_bucket_t bucket = &fasttrap_tpoints.fth_table[i]; 433* 434 mutex_enter(&bucket->ftb_mtx); 435 for (tp = bucket->ftb_data; tp != NULL; tp = tp->ftt_next) { 436 if (tp->ftt_pid == ppid && 437 tp->ftt_proc->ftpc_acount != 0) { 438 int ret = fasttrap_tracepoint_remove(cp, tp); 439 ASSERT(ret == 0); 440 441 /* 442 * The count of active providers can only be 443 * decremented (i.e. to zero) during exec, 444 * exit, and removal of a meta provider so it 445 * should be impossible to drop the count 446 * mid-fork. 447 / 448* ASSERT(tp->ftt_proc->ftpc_acount != 0); 449 } 450 } 451 mutex_exit(&bucket->ftb_mtx); 452 } 453 454 mutex_enter(&cp->p_lock); 455 sprunlock(cp); 456} 457 458/* 459 * This is called from proc_exit() or from exec_common() if p_dtrace_probes 460 * is set on the proc structure to indicate that there is a pid provider 461 * associated with this process. 462 / 463static void 464fasttrap_exec_exit(proc_t p) 465{ 466 ASSERT(p == curproc); 467 ASSERT(MUTEX_HELD(&p->p_lock)); 468 469 mutex_exit(&p->p_lock); 470 471 /* 472 * We clean up the pid provider for this process here; user-land 473 * static probes are handled by the meta-provider remove entry point. 474 / 475* fasttrap_provider_retire(p->p_pid, FASTTRAP_PID_NAME, 0); 476 477 mutex_enter(&p->p_lock); 478} 479 480 481/ARGSUSED/ 482static void 483fasttrap_pid_provide(void arg, const dtrace_probedesc_t desc) 484{ 485 /* 486 * There are no "default" pid probes. 487 / 488} 489* 490static int 491fasttrap_tracepoint_enable(proc_t p, fasttrap_probe_t probe, uint_t index) 492{ 493 fasttrap_tracepoint_t tp, new_tp = NULL; 494 fasttrap_bucket_t bucket; 495* fasttrap_id_t id; 496* pid_t pid; 497 uintptr_t pc; 498 499 ASSERT(index < probe->ftp_ntps); 500 501 pid = probe->ftp_pid; 502 pc = probe->ftp_tps[index].fit_tp->ftt_pc; 503 id = &probe->ftp_tps[index].fit_id; 504 505 ASSERT(probe->ftp_tps[index].fit_tp->ftt_pid == pid); 506 507 ASSERT(!(p->p_flag & SVFORK)); 508 509 /* 510 * Before we make any modifications, make sure we've imposed a barrier 511 * on the generation in which this probe was last modified. 512 / 513* fasttrap_mod_barrier(probe->ftp_gen); 514 515 bucket = &fasttrap_tpoints.fth_table[FASTTRAP_TPOINTS_INDEX(pid, pc)]; 516 517 /* 518 * If the tracepoint has already been enabled, just add our id to the 519 * list of interested probes. This may be our second time through 520 * this path in which case we'll have constructed the tracepoint we'd 521 * like to install. If we can't find a match, and have an allocated 522 * tracepoint ready to go, enable that one now. 523 * 524 * A tracepoint whose process is defunct is also considered defunct. 525 / 526again: 527* mutex_enter(&bucket->ftb_mtx); 528 for (tp = bucket->ftb_data; tp != NULL; tp = tp->ftt_next) { 529 /* 530 * Note that it's safe to access the active count on the 531 * associated proc structure because we know that at least one 532 * provider (this one) will still be around throughout this 533 * operation. 534 / 535* if (tp->ftt_pid != pid \|\| tp->ftt_pc != pc \|\| 536 tp->ftt_proc->ftpc_acount == 0) 537 continue; 538 539 /* 540 * Now that we've found a matching tracepoint, it would be 541 * a decent idea to confirm that the tracepoint is still 542 * enabled and the trap instruction hasn't been overwritten. 543 * Since this is a little hairy, we'll punt for now. 544 / 545* 546 /* 547 * This can't be the first interested probe. We don't have 548 * to worry about another thread being in the midst of 549 * deleting this tracepoint (which would be the only valid 550 * reason for a tracepoint to have no interested probes) 551 * since we're holding P_PR_LOCK for this process. 552 / 553* ASSERT(tp->ftt_ids != NULL \|\| tp->ftt_retids != NULL); 554 555 switch (id->fti_ptype) { 556 case DTFTP_ENTRY: 557 case DTFTP_OFFSETS: 558 case DTFTP_IS_ENABLED: 559 id->fti_next = tp->ftt_ids; 560 membar_producer(); 561 tp->ftt_ids = id; 562 membar_producer(); 563 break; 564 565 case DTFTP_RETURN: 566 case DTFTP_POST_OFFSETS: 567 id->fti_next = tp->ftt_retids; 568 membar_producer(); 569 tp->ftt_retids = id; 570 membar_producer(); 571 break; 572 573 default: 574 ASSERT(0); 575 } 576 577 mutex_exit(&bucket->ftb_mtx); 578 579 if (new_tp != NULL) { 580 new_tp->ftt_ids = NULL; 581 new_tp->ftt_retids = NULL; 582 } 583 584 return (0); 585 } 586 587 /* 588 * If we have a good tracepoint ready to go, install it now while 589 * we have the lock held and no one can screw with us. 590 / 591* if (new_tp != NULL) { 592 int rc = 0; 593 594 new_tp->ftt_next = bucket->ftb_data; 595 membar_producer(); 596 bucket->ftb_data = new_tp; 597 membar_producer(); 598 mutex_exit(&bucket->ftb_mtx); 599 600 /* 601 * Activate the tracepoint in the ISA-specific manner. 602 * If this fails, we need to report the failure, but 603 * indicate that this tracepoint must still be disabled 604 * by calling fasttrap_tracepoint_disable(). 605 / 606* if (fasttrap_tracepoint_install(p, new_tp) != 0) 607 rc = FASTTRAP_ENABLE_PARTIAL; 608 609 /* 610 * Increment the count of the number of tracepoints active in 611 * the victim process. 612 / 613* ASSERT(p->p_proc_flag & P_PR_LOCK); 614 p->p_dtrace_count++; 615 616 return (rc); 617 } 618 619 mutex_exit(&bucket->ftb_mtx); 620 621 /* 622 * Initialize the tracepoint that's been preallocated with the probe. 623 / 624* new_tp = probe->ftp_tps[index].fit_tp; 625 626 ASSERT(new_tp->ftt_pid == pid); 627 ASSERT(new_tp->ftt_pc == pc); 628 ASSERT(new_tp->ftt_proc == probe->ftp_prov->ftp_proc); 629 ASSERT(new_tp->ftt_ids == NULL); 630 ASSERT(new_tp->ftt_retids == NULL); 631 632 switch (id->fti_ptype) { 633 case DTFTP_ENTRY: 634 case DTFTP_OFFSETS: 635 case DTFTP_IS_ENABLED: 636 id->fti_next = NULL; 637 new_tp->ftt_ids = id; 638 break; 639 640 case DTFTP_RETURN: 641 case DTFTP_POST_OFFSETS: 642 id->fti_next = NULL; 643 new_tp->ftt_retids = id; 644 break; 645 646 default: 647 ASSERT(0); 648 } 649 650 /* 651 * If the ISA-dependent initialization goes to plan, go back to the 652 * beginning and try to install this freshly made tracepoint. 653 / 654* if (fasttrap_tracepoint_init(p, new_tp, pc, id->fti_ptype) == 0) 655 goto again; 656 657 new_tp->ftt_ids = NULL; 658 new_tp->ftt_retids = NULL; 659 660 return (FASTTRAP_ENABLE_FAIL); 661} 662 663static void 664fasttrap_tracepoint_disable(proc_t p, fasttrap_probe_t probe, uint_t index) 665{ 666 fasttrap_bucket_t bucket; 667* fasttrap_provider_t provider = probe->ftp_prov; 668* fasttrap_tracepoint_t *pp, tp; 669 fasttrap_id_t id, idp; 670* pid_t pid; 671 uintptr_t pc; 672 673 ASSERT(index < probe->ftp_ntps); 674 675 pid = probe->ftp_pid; 676 pc = probe->ftp_tps[index].fit_tp->ftt_pc; 677 id = &probe->ftp_tps[index].fit_id; 678 679 ASSERT(probe->ftp_tps[index].fit_tp->ftt_pid == pid); 680 681 /* 682 * Find the tracepoint and make sure that our id is one of the 683 * ones registered with it. 684 / 685* bucket = &fasttrap_tpoints.fth_table[FASTTRAP_TPOINTS_INDEX(pid, pc)]; 686 mutex_enter(&bucket->ftb_mtx); 687 for (tp = bucket->ftb_data; tp != NULL; tp = tp->ftt_next) { 688 if (tp->ftt_pid == pid && tp->ftt_pc == pc && 689 tp->ftt_proc == provider->ftp_proc) 690 break; 691 } 692 693 /* 694 * If we somehow lost this tracepoint, we're in a world of hurt. 695 / 696* ASSERT(tp != NULL); 697 698 switch (id->fti_ptype) { 699 case DTFTP_ENTRY: 700 case DTFTP_OFFSETS: 701 case DTFTP_IS_ENABLED: 702 ASSERT(tp->ftt_ids != NULL); 703 idp = &tp->ftt_ids; 704 break; 705 706 case DTFTP_RETURN: 707 case DTFTP_POST_OFFSETS: 708 ASSERT(tp->ftt_retids != NULL); 709 idp = &tp->ftt_retids; 710 break; 711 712 default: 713 ASSERT(0); 714 } 715 716 while ((idp)->fti_probe != probe) { 717* idp = &(idp)->fti_next; 718* ASSERT(idp != NULL); 719* } 720 721 id = idp; 722* idp = id->fti_next; 723* membar_producer(); 724 725 ASSERT(id->fti_probe == probe); 726 727 /* 728 * If there are other registered enablings of this tracepoint, we're 729 * all done, but if this was the last probe assocated with this 730 * this tracepoint, we need to remove and free it. 731 / 732* if (tp->ftt_ids != NULL \|\| tp->ftt_retids != NULL) { 733 734 /* 735 * If the current probe's tracepoint is in use, swap it 736 * for an unused tracepoint. 737 / 738* if (tp == probe->ftp_tps[index].fit_tp) { 739 fasttrap_probe_t tmp_probe; 740* fasttrap_tracepoint_t *tmp_tp; 741* uint_t tmp_index; 742 743 if (tp->ftt_ids != NULL) { 744 tmp_probe = tp->ftt_ids->fti_probe; 745 /* LINTED - alignment / 746* tmp_index = FASTTRAP_ID_INDEX(tp->ftt_ids); 747 tmp_tp = &tmp_probe->ftp_tps[tmp_index].fit_tp; 748 } else { 749 tmp_probe = tp->ftt_retids->fti_probe; 750 /* LINTED - alignment / 751* tmp_index = FASTTRAP_ID_INDEX(tp->ftt_retids); 752 tmp_tp = &tmp_probe->ftp_tps[tmp_index].fit_tp; 753 } 754 755 ASSERT(tmp_tp != NULL); 756* ASSERT(tmp_tp != probe->ftp_tps[index].fit_tp); 757* ASSERT((tmp_tp)->ftt_ids == NULL); 758* ASSERT((tmp_tp)->ftt_retids == NULL); 759* 760 probe->ftp_tps[index].fit_tp = tmp_tp; 761* tmp_tp = tp; 762* } 763 764 mutex_exit(&bucket->ftb_mtx); 765 766 /* 767 * Tag the modified probe with the generation in which it was 768 * changed. 769 / 770* probe->ftp_gen = fasttrap_mod_gen; 771 return; 772 } 773 774 mutex_exit(&bucket->ftb_mtx); 775 776 /* 777 * We can't safely remove the tracepoint from the set of active 778 * tracepoints until we've actually removed the fasttrap instruction 779 * from the process's text. We can, however, operate on this 780 * tracepoint secure in the knowledge that no other thread is going to 781 * be looking at it since we hold P_PR_LOCK on the process if it's 782 * live or we hold the provider lock on the process if it's dead and 783 * gone. 784 / 785* 786 /* 787 * We only need to remove the actual instruction if we're looking 788 * at an existing process 789 / 790* if (p != NULL) { 791 /* 792 * If we fail to restore the instruction we need to kill 793 * this process since it's in a completely unrecoverable 794 * state. 795 / 796* if (fasttrap_tracepoint_remove(p, tp) != 0) 797 fasttrap_sigtrap(p, NULL, pc); 798 799 /* 800 * Decrement the count of the number of tracepoints active 801 * in the victim process. 802 / 803* ASSERT(p->p_proc_flag & P_PR_LOCK); 804 p->p_dtrace_count--; 805 } 806 807 /* 808 * Remove the probe from the hash table of active tracepoints. 809 / 810* mutex_enter(&bucket->ftb_mtx); 811 pp = (fasttrap_tracepoint_t *)&bucket->ftb_data; 812* ASSERT(pp != NULL); 813* while (pp != tp) { 814* pp = &(pp)->ftt_next; 815* ASSERT(pp != NULL); 816* } 817 818 pp = tp->ftt_next; 819* membar_producer(); 820 821 mutex_exit(&bucket->ftb_mtx); 822 823 /* 824 * Tag the modified probe with the generation in which it was changed. 825 / 826* probe->ftp_gen = fasttrap_mod_gen; 827} 828 829static void 830fasttrap_enable_callbacks(void) 831{ 832 /* 833 * We don't have to play the rw lock game here because we're 834 * providing something rather than taking something away -- 835 * we can be sure that no threads have tried to follow this 836 * function pointer yet. 837 / 838* mutex_enter(&fasttrap_count_mtx); 839 if (fasttrap_pid_count == 0) { 840 ASSERT(dtrace_pid_probe_ptr == NULL); 841 ASSERT(dtrace_return_probe_ptr == NULL); 842 dtrace_pid_probe_ptr = &fasttrap_pid_probe; 843 dtrace_return_probe_ptr = &fasttrap_return_probe; 844 } 845 ASSERT(dtrace_pid_probe_ptr == &fasttrap_pid_probe); 846 ASSERT(dtrace_return_probe_ptr == &fasttrap_return_probe); 847 fasttrap_pid_count++; 848 mutex_exit(&fasttrap_count_mtx); 849} 850 851static void 852fasttrap_disable_callbacks(void) 853{ 854 ASSERT(MUTEX_HELD(&cpu_lock)); 855 856 mutex_enter(&fasttrap_count_mtx); 857 ASSERT(fasttrap_pid_count > 0); 858 fasttrap_pid_count--; 859 if (fasttrap_pid_count == 0) { 860 cpu_t cur, cpu = CPU; 861 862 for (cur = cpu->cpu_next_onln; cur != cpu; 863 cur = cur->cpu_next_onln) { 864 rw_enter(&cur->cpu_ft_lock, RW_WRITER); 865 } 866 867 dtrace_pid_probe_ptr = NULL; 868 dtrace_return_probe_ptr = NULL; 869 870 for (cur = cpu->cpu_next_onln; cur != cpu; 871 cur = cur->cpu_next_onln) { 872 rw_exit(&cur->cpu_ft_lock); 873 } 874 } 875 mutex_exit(&fasttrap_count_mtx); 876} 877 878/ARGSUSED/ 879static void 880fasttrap_pid_enable(void arg, dtrace_id_t id, void parg) 881{ 882 fasttrap_probe_t probe = parg; 883* proc_t p; 884* int i, rc; 885 886 ASSERT(probe != NULL); 887 ASSERT(!probe->ftp_enabled); 888 ASSERT(id == probe->ftp_id); 889 ASSERT(MUTEX_HELD(&cpu_lock)); 890 891 /* 892 * Increment the count of enabled probes on this probe's provider; 893 * the provider can't go away while the probe still exists. We 894 * must increment this even if we aren't able to properly enable 895 * this probe. 896 / 897* mutex_enter(&probe->ftp_prov->ftp_mtx); 898 probe->ftp_prov->ftp_rcount++; 899 mutex_exit(&probe->ftp_prov->ftp_mtx); 900 901 /* 902 * If this probe's provider is retired (meaning it was valid in a 903 * previously exec'ed incarnation of this address space), bail out. The 904 * provider can't go away while we're in this code path. 905 / 906* if (probe->ftp_prov->ftp_retired) 907 return; 908 909 /* 910 * If we can't find the process, it may be that we're in the context of 911 * a fork in which the traced process is being born and we're copying 912 * USDT probes. Otherwise, the process is gone so bail. 913 / 914* if ((p = sprlock(probe->ftp_pid)) == NULL) { 915 if ((curproc->p_flag & SFORKING) == 0) 916 return; 917 918 mutex_enter(&pidlock); 919 p = prfind(probe->ftp_pid); 920 921 /* 922 * Confirm that curproc is indeed forking the process in which 923 * we're trying to enable probes. 924 / 925* ASSERT(p != NULL); 926 ASSERT(p->p_parent == curproc); 927 ASSERT(p->p_stat == SIDL); 928 929 mutex_enter(&p->p_lock); 930 mutex_exit(&pidlock); 931 932 sprlock_proc(p); 933 } 934 935 ASSERT(!(p->p_flag & SVFORK)); 936 mutex_exit(&p->p_lock); 937 938 /* 939 * We have to enable the trap entry point before any user threads have 940 * the chance to execute the trap instruction we're about to place 941 * in their process's text. 942 / 943* fasttrap_enable_callbacks(); 944 945 /* 946 * Enable all the tracepoints and add this probe's id to each 947 * tracepoint's list of active probes. 948 / 949* for (i = 0; i < probe->ftp_ntps; i++) { 950 if ((rc = fasttrap_tracepoint_enable(p, probe, i)) != 0) { 951 /* 952 * If enabling the tracepoint failed completely, 953 * we don't have to disable it; if the failure 954 * was only partial we must disable it. 955 / 956* if (rc == FASTTRAP_ENABLE_FAIL) 957 i--; 958 else 959 ASSERT(rc == FASTTRAP_ENABLE_PARTIAL); 960 961 /* 962 * Back up and pull out all the tracepoints we've 963 * created so far for this probe. 964 / 965* while (i >= 0) { 966 fasttrap_tracepoint_disable(p, probe, i); 967 i--; 968 } 969 970 mutex_enter(&p->p_lock); 971 sprunlock(p); 972 973 /* 974 * Since we're not actually enabling this probe, 975 * drop our reference on the trap table entry. 976 / 977* fasttrap_disable_callbacks(); 978 return; 979 } 980 } 981 982 mutex_enter(&p->p_lock); 983 sprunlock(p); 984 985 probe->ftp_enabled = 1; 986} 987 988/ARGSUSED/ 989static void 990fasttrap_pid_disable(void arg, dtrace_id_t id, void parg) 991{ 992 fasttrap_probe_t probe = parg; 993* fasttrap_provider_t provider = probe->ftp_prov; 994* proc_t p; 995* int i, whack = 0; 996 997 ASSERT(id == probe->ftp_id); 998 999 /* 1000 * We won't be able to acquire a /proc-esque lock on the process 1001 * iff the process is dead and gone. In this case, we rely on the 1002 * provider lock as a point of mutual exclusion to prevent other 1003 * DTrace consumers from disabling this probe. 1004 / 1005* if ((p = sprlock(probe->ftp_pid)) != NULL) { 1006 ASSERT(!(p->p_flag & SVFORK)); 1007 mutex_exit(&p->p_lock); 1008 } 1009 1010 mutex_enter(&provider->ftp_mtx); 1011 1012 /* 1013 * Disable all the associated tracepoints (for fully enabled probes). 1014 / 1015* if (probe->ftp_enabled) { 1016 for (i = 0; i < probe->ftp_ntps; i++) { 1017 fasttrap_tracepoint_disable(p, probe, i); 1018 } 1019 } 1020 1021 ASSERT(provider->ftp_rcount > 0); 1022 provider->ftp_rcount--; 1023 1024 if (p != NULL) { 1025 /* 1026 * Even though we may not be able to remove it entirely, we 1027 * mark this retired provider to get a chance to remove some 1028 * of the associated probes. 1029 / 1030* if (provider->ftp_retired && !provider->ftp_marked) 1031 whack = provider->ftp_marked = 1; 1032 mutex_exit(&provider->ftp_mtx); 1033 1034 mutex_enter(&p->p_lock); 1035 sprunlock(p); 1036 } else { 1037 /* 1038 * If the process is dead, we're just waiting for the 1039 * last probe to be disabled to be able to free it. 1040 / 1041* if (provider->ftp_rcount == 0 && !provider->ftp_marked) 1042 whack = provider->ftp_marked = 1; 1043 mutex_exit(&provider->ftp_mtx); 1044 } 1045 1046 if (whack) 1047 fasttrap_pid_cleanup(); 1048 1049 if (!probe->ftp_enabled) 1050 return; 1051 1052 probe->ftp_enabled = 0; 1053 1054 ASSERT(MUTEX_HELD(&cpu_lock)); 1055 fasttrap_disable_callbacks(); 1056} 1057 1058/ARGSUSED/ 1059static void 1060fasttrap_pid_getargdesc(void arg, dtrace_id_t id, void parg, 1061 dtrace_argdesc_t desc) 1062{ 1063* fasttrap_probe_t probe = parg; 1064* char str; 1065* int i, ndx; 1066 1067 desc->dtargd_native[0] = '\0'; 1068 desc->dtargd_xlate[0] = '\0'; 1069 1070 if (probe->ftp_prov->ftp_retired != 0 \|\| 1071 desc->dtargd_ndx >= probe->ftp_nargs) { 1072 desc->dtargd_ndx = DTRACE_ARGNONE; 1073 return; 1074 } 1075 1076 ndx = (probe->ftp_argmap != NULL) ? 1077 probe->ftp_argmap[desc->dtargd_ndx] : desc->dtargd_ndx; 1078 1079 str = probe->ftp_ntypes; 1080 for (i = 0; i < ndx; i++) { 1081 str += strlen(str) + 1; 1082 } 1083 1084 ASSERT(strlen(str + 1) < sizeof (desc->dtargd_native)); 1085 (void) strcpy(desc->dtargd_native, str); 1086 1087 if (probe->ftp_xtypes == NULL) 1088 return; 1089 1090 str = probe->ftp_xtypes; 1091 for (i = 0; i < desc->dtargd_ndx; i++) { 1092 str += strlen(str) + 1; 1093 } 1094 1095 ASSERT(strlen(str + 1) < sizeof (desc->dtargd_xlate)); 1096 (void) strcpy(desc->dtargd_xlate, str); 1097} 1098 1099/ARGSUSED/ 1100static void 1101fasttrap_pid_destroy(void arg, dtrace_id_t id, void parg) 1102{ 1103 fasttrap_probe_t probe = parg; 1104* int i; 1105 size_t size; 1106 1107 ASSERT(probe != NULL); 1108 ASSERT(!probe->ftp_enabled); 1109 ASSERT(fasttrap_total >= probe->ftp_ntps); 1110 1111 atomic_add_32(&fasttrap_total, -probe->ftp_ntps); 1112 size = offsetof(fasttrap_probe_t, ftp_tps[probe->ftp_ntps]); 1113 1114 if (probe->ftp_gen + 1 >= fasttrap_mod_gen) 1115 fasttrap_mod_barrier(probe->ftp_gen); 1116 1117 for (i = 0; i < probe->ftp_ntps; i++) { 1118 kmem_free(probe->ftp_tps[i].fit_tp, 1119 sizeof (fasttrap_tracepoint_t)); 1120 } 1121 1122 kmem_free(probe, size); 1123} 1124 1125 1126static const dtrace_pattr_t pid_attr = { 1127{ DTRACE_STABILITY_EVOLVING, DTRACE_STABILITY_EVOLVING, DTRACE_CLASS_ISA }, 1128{ DTRACE_STABILITY_PRIVATE, DTRACE_STABILITY_PRIVATE, DTRACE_CLASS_UNKNOWN }, 1129{ DTRACE_STABILITY_PRIVATE, DTRACE_STABILITY_PRIVATE, DTRACE_CLASS_UNKNOWN }, 1130{ DTRACE_STABILITY_EVOLVING, DTRACE_STABILITY_EVOLVING, DTRACE_CLASS_ISA }, 1131{ DTRACE_STABILITY_PRIVATE, DTRACE_STABILITY_PRIVATE, DTRACE_CLASS_UNKNOWN }, 1132}; 1133 1134static dtrace_pops_t pid_pops = { 1135 fasttrap_pid_provide, 1136 NULL, 1137 fasttrap_pid_enable, 1138 fasttrap_pid_disable, 1139 NULL, 1140 NULL, 1141 fasttrap_pid_getargdesc, 1142 fasttrap_pid_getarg, 1143 NULL, 1144 fasttrap_pid_destroy 1145}; 1146 1147static dtrace_pops_t usdt_pops = { 1148 fasttrap_pid_provide, 1149 NULL, 1150 fasttrap_pid_enable, 1151 fasttrap_pid_disable, 1152 NULL, 1153 NULL, 1154 fasttrap_pid_getargdesc, 1155 fasttrap_usdt_getarg, 1156 NULL, 1157 fasttrap_pid_destroy 1158}; 1159 1160static fasttrap_proc_t * 1161fasttrap_proc_lookup(pid_t pid) 1162{ 1163 fasttrap_bucket_t bucket; 1164* fasttrap_proc_t fprc, new_fprc; 1165 1166 bucket = &fasttrap_procs.fth_table[FASTTRAP_PROCS_INDEX(pid)]; 1167 mutex_enter(&bucket->ftb_mtx); 1168 1169 for (fprc = bucket->ftb_data; fprc != NULL; fprc = fprc->ftpc_next) { 1170 if (fprc->ftpc_pid == pid && fprc->ftpc_acount != 0) { 1171 mutex_enter(&fprc->ftpc_mtx); 1172 mutex_exit(&bucket->ftb_mtx); 1173 fprc->ftpc_rcount++; 1174 atomic_add_64(&fprc->ftpc_acount, 1); 1175 ASSERT(fprc->ftpc_acount <= fprc->ftpc_rcount); 1176 mutex_exit(&fprc->ftpc_mtx); 1177 1178 return (fprc); 1179 } 1180 } 1181 1182 /* 1183 * Drop the bucket lock so we don't try to perform a sleeping 1184 * allocation under it. 1185 / 1186* mutex_exit(&bucket->ftb_mtx); 1187 1188 new_fprc = kmem_zalloc(sizeof (fasttrap_proc_t), KM_SLEEP); 1189 new_fprc->ftpc_pid = pid; 1190 new_fprc->ftpc_rcount = 1; 1191 new_fprc->ftpc_acount = 1; 1192 1193 mutex_enter(&bucket->ftb_mtx); 1194 1195 /* 1196 * Take another lap through the list to make sure a proc hasn't 1197 * been created for this pid while we weren't under the bucket lock. 1198 / 1199* for (fprc = bucket->ftb_data; fprc != NULL; fprc = fprc->ftpc_next) { 1200 if (fprc->ftpc_pid == pid && fprc->ftpc_acount != 0) { 1201 mutex_enter(&fprc->ftpc_mtx); 1202 mutex_exit(&bucket->ftb_mtx); 1203 fprc->ftpc_rcount++; 1204 atomic_add_64(&fprc->ftpc_acount, 1); 1205 ASSERT(fprc->ftpc_acount <= fprc->ftpc_rcount); 1206 mutex_exit(&fprc->ftpc_mtx); 1207 1208 kmem_free(new_fprc, sizeof (fasttrap_proc_t)); 1209 1210 return (fprc); 1211 } 1212 } 1213 1214 new_fprc->ftpc_next = bucket->ftb_data; 1215 bucket->ftb_data = new_fprc; 1216 1217 mutex_exit(&bucket->ftb_mtx); 1218 1219 return (new_fprc); 1220} 1221 1222static void 1223fasttrap_proc_release(fasttrap_proc_t proc) 1224{ 1225* fasttrap_bucket_t bucket; 1226* fasttrap_proc_t fprc, fprcp; 1227* pid_t pid = proc->ftpc_pid; 1228 1229 mutex_enter(&proc->ftpc_mtx); 1230 1231 ASSERT(proc->ftpc_rcount != 0); 1232 ASSERT(proc->ftpc_acount <= proc->ftpc_rcount); 1233 1234 if (--proc->ftpc_rcount != 0) { 1235 mutex_exit(&proc->ftpc_mtx); 1236 return; 1237 } 1238 1239 mutex_exit(&proc->ftpc_mtx); 1240 1241 /* 1242 * There should definitely be no live providers associated with this 1243 * process at this point. 1244 / 1245* ASSERT(proc->ftpc_acount == 0); 1246 1247 bucket = &fasttrap_procs.fth_table[FASTTRAP_PROCS_INDEX(pid)]; 1248 mutex_enter(&bucket->ftb_mtx); 1249 1250 fprcp = (fasttrap_proc_t *)&bucket->ftb_data; 1251* while ((fprc = fprcp) != NULL) { 1252* if (fprc == proc) 1253 break; 1254 1255 fprcp = &fprc->ftpc_next; 1256 } 1257 1258 /* 1259 * Something strange has happened if we can't find the proc. 1260 / 1261* ASSERT(fprc != NULL); 1262 1263 fprcp = fprc->ftpc_next; 1264* 1265 mutex_exit(&bucket->ftb_mtx); 1266 1267 kmem_free(fprc, sizeof (fasttrap_proc_t)); 1268} 1269 1270/* 1271 * Lookup a fasttrap-managed provider based on its name and associated pid. 1272 * If the pattr argument is non-NULL, this function instantiates the provider 1273 * if it doesn't exist otherwise it returns NULL. The provider is returned 1274 * with its lock held. 1275 / 1276static fasttrap_provider_t 1277fasttrap_provider_lookup(pid_t pid, const char name, 1278* const dtrace_pattr_t pattr) 1279{ 1280* fasttrap_provider_t fp, new_fp = NULL; 1281 fasttrap_bucket_t bucket; 1282* char provname[DTRACE_PROVNAMELEN]; 1283 proc_t p; 1284* cred_t cred; 1285* 1286 ASSERT(strlen(name) < sizeof (fp->ftp_name)); 1287 ASSERT(pattr != NULL); 1288 1289 bucket = &fasttrap_provs.fth_table[FASTTRAP_PROVS_INDEX(pid, name)]; 1290 mutex_enter(&bucket->ftb_mtx); 1291 1292 /* 1293 * Take a lap through the list and return the match if we find it. 1294 / 1295* for (fp = bucket->ftb_data; fp != NULL; fp = fp->ftp_next) { 1296 if (fp->ftp_pid == pid && strcmp(fp->ftp_name, name) == 0 && 1297 !fp->ftp_retired) { 1298 mutex_enter(&fp->ftp_mtx); 1299 mutex_exit(&bucket->ftb_mtx); 1300 return (fp); 1301 } 1302 } 1303 1304 /* 1305 * Drop the bucket lock so we don't try to perform a sleeping 1306 * allocation under it. 1307 / 1308* mutex_exit(&bucket->ftb_mtx); 1309 1310 /* 1311 * Make sure the process exists, isn't a child created as the result 1312 * of a vfork(2), and isn't a zombie (but may be in fork). 1313 / 1314* mutex_enter(&pidlock); 1315 if ((p = prfind(pid)) == NULL) { 1316 mutex_exit(&pidlock); 1317 return (NULL); 1318 } 1319 mutex_enter(&p->p_lock); 1320 mutex_exit(&pidlock); 1321 if (p->p_flag & (SVFORK \| SEXITING)) { 1322 mutex_exit(&p->p_lock); 1323 return (NULL); 1324 } 1325 1326 /* 1327 * Increment p_dtrace_probes so that the process knows to inform us 1328 * when it exits or execs. fasttrap_provider_free() decrements this 1329 * when we're done with this provider. 1330 / 1331* p->p_dtrace_probes++; 1332 1333 /* 1334 * Grab the credentials for this process so we have 1335 * something to pass to dtrace_register(). 1336 / 1337* mutex_enter(&p->p_crlock); 1338 crhold(p->p_cred); 1339 cred = p->p_cred; 1340 mutex_exit(&p->p_crlock); 1341 mutex_exit(&p->p_lock); 1342 1343 new_fp = kmem_zalloc(sizeof (fasttrap_provider_t), KM_SLEEP); 1344 new_fp->ftp_pid = pid; 1345 new_fp->ftp_proc = fasttrap_proc_lookup(pid); 1346 1347 ASSERT(new_fp->ftp_proc != NULL); 1348 1349 mutex_enter(&bucket->ftb_mtx); 1350 1351 /* 1352 * Take another lap through the list to make sure a provider hasn't 1353 * been created for this pid while we weren't under the bucket lock. 1354 / 1355* for (fp = bucket->ftb_data; fp != NULL; fp = fp->ftp_next) { 1356 if (fp->ftp_pid == pid && strcmp(fp->ftp_name, name) == 0 && 1357 !fp->ftp_retired) { 1358 mutex_enter(&fp->ftp_mtx); 1359 mutex_exit(&bucket->ftb_mtx); 1360 fasttrap_provider_free(new_fp); 1361 crfree(cred); 1362 return (fp); 1363 } 1364 } 1365 1366 (void) strcpy(new_fp->ftp_name, name); 1367 1368 /* 1369 * Fail and return NULL if either the provider name is too long 1370 * or we fail to register this new provider with the DTrace 1371 * framework. Note that this is the only place we ever construct 1372 * the full provider name -- we keep it in pieces in the provider 1373 * structure. 1374 / 1375* if (snprintf(provname, sizeof (provname), "%s%u", name, (uint_t)pid) >= 1376 sizeof (provname) \|\| 1377 dtrace_register(provname, pattr, 1378 DTRACE_PRIV_PROC \| DTRACE_PRIV_OWNER \| DTRACE_PRIV_ZONEOWNER, cred, 1379 pattr == &pid_attr ? &pid_pops : &usdt_pops, new_fp, 1380 &new_fp->ftp_provid) != 0) { 1381 mutex_exit(&bucket->ftb_mtx); 1382 fasttrap_provider_free(new_fp); 1383 crfree(cred); 1384 return (NULL); 1385 } 1386 1387 new_fp->ftp_next = bucket->ftb_data; 1388 bucket->ftb_data = new_fp; 1389 1390 mutex_enter(&new_fp->ftp_mtx); 1391 mutex_exit(&bucket->ftb_mtx); 1392 1393 crfree(cred); 1394 return (new_fp); 1395} 1396 1397static void 1398fasttrap_provider_free(fasttrap_provider_t provider) 1399{ 1400* pid_t pid = provider->ftp_pid; 1401 proc_t p; 1402* 1403 /* 1404 * There need to be no associated enabled probes, no consumers 1405 * creating probes, and no meta providers referencing this provider. 1406 / 1407* ASSERT(provider->ftp_rcount == 0); 1408 ASSERT(provider->ftp_ccount == 0); 1409 ASSERT(provider->ftp_mcount == 0); 1410 1411 /* 1412 * If this provider hasn't been retired, we need to explicitly drop the 1413 * count of active providers on the associated process structure. 1414 / 1415* if (!provider->ftp_retired) { 1416 atomic_add_64(&provider->ftp_proc->ftpc_acount, -1); 1417 ASSERT(provider->ftp_proc->ftpc_acount < 1418 provider->ftp_proc->ftpc_rcount); 1419 } 1420 1421 fasttrap_proc_release(provider->ftp_proc); 1422 1423 kmem_free(provider, sizeof (fasttrap_provider_t)); 1424 1425 /* 1426 * Decrement p_dtrace_probes on the process whose provider we're 1427 * freeing. We don't have to worry about clobbering somone else's 1428 * modifications to it because we have locked the bucket that 1429 * corresponds to this process's hash chain in the provider hash 1430 * table. Don't sweat it if we can't find the process. 1431 / 1432* mutex_enter(&pidlock); 1433 if ((p = prfind(pid)) == NULL) { 1434 mutex_exit(&pidlock); 1435 return; 1436 } 1437 1438 mutex_enter(&p->p_lock); 1439 mutex_exit(&pidlock); 1440 1441 p->p_dtrace_probes--; 1442 mutex_exit(&p->p_lock); 1443} 1444 1445static void 1446fasttrap_provider_retire(pid_t pid, const char name, int mprov) 1447{ 1448* fasttrap_provider_t fp; 1449* fasttrap_bucket_t bucket; 1450* dtrace_provider_id_t provid; 1451 1452 ASSERT(strlen(name) < sizeof (fp->ftp_name)); 1453 1454 bucket = &fasttrap_provs.fth_table[FASTTRAP_PROVS_INDEX(pid, name)]; 1455 mutex_enter(&bucket->ftb_mtx); 1456 1457 for (fp = bucket->ftb_data; fp != NULL; fp = fp->ftp_next) { 1458 if (fp->ftp_pid == pid && strcmp(fp->ftp_name, name) == 0 && 1459 !fp->ftp_retired) 1460 break; 1461 } 1462 1463 if (fp == NULL) { 1464 mutex_exit(&bucket->ftb_mtx); 1465 return; 1466 } 1467 1468 mutex_enter(&fp->ftp_mtx); 1469 ASSERT(!mprov \|\| fp->ftp_mcount > 0); 1470 if (mprov && --fp->ftp_mcount != 0) { 1471 mutex_exit(&fp->ftp_mtx); 1472 mutex_exit(&bucket->ftb_mtx); 1473 return; 1474 } 1475 1476 /* 1477 * Mark the provider to be removed in our post-processing step, mark it 1478 * retired, and drop the active count on its proc. Marking it indicates 1479 * that we should try to remove it; setting the retired flag indicates 1480 * that we're done with this provider; dropping the active the proc 1481 * releases our hold, and when this reaches zero (as it will during 1482 * exit or exec) the proc and associated providers become defunct. 1483 * 1484 * We obviously need to take the bucket lock before the provider lock 1485 * to perform the lookup, but we need to drop the provider lock 1486 * before calling into the DTrace framework since we acquire the 1487 * provider lock in callbacks invoked from the DTrace framework. The 1488 * bucket lock therefore protects the integrity of the provider hash 1489 * table. 1490 / 1491* atomic_add_64(&fp->ftp_proc->ftpc_acount, -1); 1492 ASSERT(fp->ftp_proc->ftpc_acount < fp->ftp_proc->ftpc_rcount); 1493 1494 fp->ftp_retired = 1; 1495 fp->ftp_marked = 1; 1496 provid = fp->ftp_provid; 1497 mutex_exit(&fp->ftp_mtx); 1498 1499 /* 1500 * We don't have to worry about invalidating the same provider twice 1501 * since fasttrap_provider_lookup() will ignore provider that have 1502 * been marked as retired. 1503 / 1504* dtrace_invalidate(provid); 1505 1506 mutex_exit(&bucket->ftb_mtx); 1507 1508 fasttrap_pid_cleanup(); 1509} 1510 1511static int 1512fasttrap_uint32_cmp(const void ap, const void bp) 1513{ 1514 return ((const uint32_t )ap - (const uint32_t )bp); 1515} 1516 1517static int 1518fasttrap_uint64_cmp(const void ap, const void bp) 1519{ 1520 return ((const uint64_t )ap - (const uint64_t )bp); 1521} 1522 1523static int 1524fasttrap_add_probe(fasttrap_probe_spec_t pdata) 1525{ 1526* fasttrap_provider_t provider; 1527* fasttrap_probe_t pp; 1528* fasttrap_tracepoint_t tp; 1529* char name; 1530* int i, aframes, whack; 1531 1532 /* 1533 * There needs to be at least one desired trace point. 1534 / 1535* if (pdata->ftps_noffs == 0) 1536 return (EINVAL); 1537 1538 switch (pdata->ftps_type) { 1539 case DTFTP_ENTRY: 1540 name = "entry"; 1541 aframes = FASTTRAP_ENTRY_AFRAMES; 1542 break; 1543 case DTFTP_RETURN: 1544 name = "return"; 1545 aframes = FASTTRAP_RETURN_AFRAMES; 1546 break; 1547 case DTFTP_OFFSETS: 1548 name = NULL; 1549 break; 1550 default: 1551 return (EINVAL); 1552 } 1553 1554 if ((provider = fasttrap_provider_lookup(pdata->ftps_pid, 1555 FASTTRAP_PID_NAME, &pid_attr)) == NULL) 1556 return (ESRCH); 1557 1558 /* 1559 * Increment this reference count to indicate that a consumer is 1560 * actively adding a new probe associated with this provider. This 1561 * prevents the provider from being deleted -- we'll need to check 1562 * for pending deletions when we drop this reference count. 1563 / 1564* provider->ftp_ccount++; 1565 mutex_exit(&provider->ftp_mtx); 1566 1567 /* 1568 * Grab the creation lock to ensure consistency between calls to 1569 * dtrace_probe_lookup() and dtrace_probe_create() in the face of 1570 * other threads creating probes. We must drop the provider lock 1571 * before taking this lock to avoid a three-way deadlock with the 1572 * DTrace framework. 1573 / 1574* mutex_enter(&provider->ftp_cmtx); 1575 1576 if (name == NULL) { 1577 for (i = 0; i < pdata->ftps_noffs; i++) { 1578 char name_str[17]; 1579 1580 (void) sprintf(name_str, "%llx", 1581 (unsigned long long)pdata->ftps_offs[i]); 1582 1583 if (dtrace_probe_lookup(provider->ftp_provid, 1584 pdata->ftps_mod, pdata->ftps_func, name_str) != 0) 1585 continue; 1586 1587 atomic_add_32(&fasttrap_total, 1); 1588 1589 if (fasttrap_total > fasttrap_max) { 1590 atomic_add_32(&fasttrap_total, -1); 1591 goto no_mem; 1592 } 1593 1594 pp = kmem_zalloc(sizeof (fasttrap_probe_t), KM_SLEEP); 1595 1596 pp->ftp_prov = provider; 1597 pp->ftp_faddr = pdata->ftps_pc; 1598 pp->ftp_fsize = pdata->ftps_size; 1599 pp->ftp_pid = pdata->ftps_pid; 1600 pp->ftp_ntps = 1; 1601 1602 tp = kmem_zalloc(sizeof (fasttrap_tracepoint_t), 1603 KM_SLEEP); 1604 1605 tp->ftt_proc = provider->ftp_proc; 1606 tp->ftt_pc = pdata->ftps_offs[i] + pdata->ftps_pc; 1607 tp->ftt_pid = pdata->ftps_pid; 1608 1609 pp->ftp_tps[0].fit_tp = tp; 1610 pp->ftp_tps[0].fit_id.fti_probe = pp; 1611 pp->ftp_tps[0].fit_id.fti_ptype = pdata->ftps_type; 1612 1613 pp->ftp_id = dtrace_probe_create(provider->ftp_provid, 1614 pdata->ftps_mod, pdata->ftps_func, name_str, 1615 FASTTRAP_OFFSET_AFRAMES, pp); 1616 } 1617 1618 } else if (dtrace_probe_lookup(provider->ftp_provid, pdata->ftps_mod, 1619 pdata->ftps_func, name) == 0) { 1620 atomic_add_32(&fasttrap_total, pdata->ftps_noffs); 1621 1622 if (fasttrap_total > fasttrap_max) { 1623 atomic_add_32(&fasttrap_total, -pdata->ftps_noffs); 1624 goto no_mem; 1625 } 1626 1627 /* 1628 * Make sure all tracepoint program counter values are unique. 1629 * We later assume that each probe has exactly one tracepoint 1630 * for a given pc. 1631 / 1632* qsort(pdata->ftps_offs, pdata->ftps_noffs, 1633 sizeof (uint64_t), fasttrap_uint64_cmp); 1634 for (i = 1; i < pdata->ftps_noffs; i++) { 1635 if (pdata->ftps_offs[i] > pdata->ftps_offs[i - 1]) 1636 continue; 1637 1638 atomic_add_32(&fasttrap_total, -pdata->ftps_noffs); 1639 goto no_mem; 1640 } 1641 1642 ASSERT(pdata->ftps_noffs > 0); 1643 pp = kmem_zalloc(offsetof(fasttrap_probe_t, 1644 ftp_tps[pdata->ftps_noffs]), KM_SLEEP); 1645 1646 pp->ftp_prov = provider; 1647 pp->ftp_faddr = pdata->ftps_pc; 1648 pp->ftp_fsize = pdata->ftps_size; 1649 pp->ftp_pid = pdata->ftps_pid; 1650 pp->ftp_ntps = pdata->ftps_noffs; 1651 1652 for (i = 0; i < pdata->ftps_noffs; i++) { 1653 tp = kmem_zalloc(sizeof (fasttrap_tracepoint_t), 1654 KM_SLEEP); 1655 1656 tp->ftt_proc = provider->ftp_proc; 1657 tp->ftt_pc = pdata->ftps_offs[i] + pdata->ftps_pc; 1658 tp->ftt_pid = pdata->ftps_pid; 1659 1660 pp->ftp_tps[i].fit_tp = tp; 1661 pp->ftp_tps[i].fit_id.fti_probe = pp; 1662 pp->ftp_tps[i].fit_id.fti_ptype = pdata->ftps_type; 1663 } 1664 1665 pp->ftp_id = dtrace_probe_create(provider->ftp_provid, 1666 pdata->ftps_mod, pdata->ftps_func, name, aframes, pp); 1667 } 1668 1669 mutex_exit(&provider->ftp_cmtx); 1670 1671 /* 1672 * We know that the provider is still valid since we incremented the 1673 * creation reference count. If someone tried to clean up this provider 1674 * while we were using it (e.g. because the process called exec(2) or 1675 * exit(2)), take note of that and try to clean it up now. 1676 / 1677* mutex_enter(&provider->ftp_mtx); 1678 provider->ftp_ccount--; 1679 whack = provider->ftp_retired; 1680 mutex_exit(&provider->ftp_mtx); 1681 1682 if (whack) 1683 fasttrap_pid_cleanup(); 1684 1685 return (0); 1686 1687no_mem: 1688 /* 1689 * If we've exhausted the allowable resources, we'll try to remove 1690 * this provider to free some up. This is to cover the case where 1691 * the user has accidentally created many more probes than was 1692 * intended (e.g. pid123:::). 1693 / 1694* mutex_exit(&provider->ftp_cmtx); 1695 mutex_enter(&provider->ftp_mtx); 1696 provider->ftp_ccount--; 1697 provider->ftp_marked = 1; 1698 mutex_exit(&provider->ftp_mtx); 1699 1700 fasttrap_pid_cleanup(); 1701 1702 return (ENOMEM); 1703} 1704 1705/ARGSUSED/ 1706static void * 1707fasttrap_meta_provide(void arg, dtrace_helper_provdesc_t dhpv, pid_t pid) 1708{ 1709 fasttrap_provider_t provider; 1710* 1711 /* 1712 * A 32-bit unsigned integer (like a pid for example) can be 1713 * expressed in 10 or fewer decimal digits. Make sure that we'll 1714 * have enough space for the provider name. 1715 / 1716* if (strlen(dhpv->dthpv_provname) + 10 >= 1717 sizeof (provider->ftp_name)) { 1718 cmn_err(CE_WARN, "failed to instantiate provider %s: " 1719 "name too long to accomodate pid", dhpv->dthpv_provname); 1720 return (NULL); 1721 } 1722 1723 /* 1724 * Don't let folks spoof the true pid provider. 1725 / 1726* if (strcmp(dhpv->dthpv_provname, FASTTRAP_PID_NAME) == 0) { 1727 cmn_err(CE_WARN, "failed to instantiate provider %s: " 1728 "%s is an invalid name", dhpv->dthpv_provname, 1729 FASTTRAP_PID_NAME); 1730 return (NULL); 1731 } 1732 1733 /* 1734 * The highest stability class that fasttrap supports is ISA; cap 1735 * the stability of the new provider accordingly. 1736 / 1737* if (dhpv->dthpv_pattr.dtpa_provider.dtat_class > DTRACE_CLASS_ISA) 1738 dhpv->dthpv_pattr.dtpa_provider.dtat_class = DTRACE_CLASS_ISA; 1739 if (dhpv->dthpv_pattr.dtpa_mod.dtat_class > DTRACE_CLASS_ISA) 1740 dhpv->dthpv_pattr.dtpa_mod.dtat_class = DTRACE_CLASS_ISA; 1741 if (dhpv->dthpv_pattr.dtpa_func.dtat_class > DTRACE_CLASS_ISA) 1742 dhpv->dthpv_pattr.dtpa_func.dtat_class = DTRACE_CLASS_ISA; 1743 if (dhpv->dthpv_pattr.dtpa_name.dtat_class > DTRACE_CLASS_ISA) 1744 dhpv->dthpv_pattr.dtpa_name.dtat_class = DTRACE_CLASS_ISA; 1745 if (dhpv->dthpv_pattr.dtpa_args.dtat_class > DTRACE_CLASS_ISA) 1746 dhpv->dthpv_pattr.dtpa_args.dtat_class = DTRACE_CLASS_ISA; 1747 1748 if ((provider = fasttrap_provider_lookup(pid, dhpv->dthpv_provname, 1749 &dhpv->dthpv_pattr)) == NULL) { 1750 cmn_err(CE_WARN, "failed to instantiate provider %s for " 1751 "process %u", dhpv->dthpv_provname, (uint_t)pid); 1752 return (NULL); 1753 } 1754 1755 /* 1756 * Up the meta provider count so this provider isn't removed until 1757 * the meta provider has been told to remove it. 1758 / 1759* provider->ftp_mcount++; 1760 1761 mutex_exit(&provider->ftp_mtx); 1762 1763 return (provider); 1764} 1765 1766/ARGSUSED/ 1767static void 1768fasttrap_meta_create_probe(void arg, void parg, 1769 dtrace_helper_probedesc_t dhpb) 1770{ 1771* fasttrap_provider_t provider = parg; 1772* fasttrap_probe_t pp; 1773* fasttrap_tracepoint_t tp; 1774* int i, j; 1775 uint32_t ntps; 1776 1777 /* 1778 * Since the meta provider count is non-zero we don't have to worry 1779 * about this provider disappearing. 1780 / 1781* ASSERT(provider->ftp_mcount > 0); 1782 1783 /* 1784 * The offsets must be unique. 1785 / 1786* qsort(dhpb->dthpb_offs, dhpb->dthpb_noffs, sizeof (uint32_t), 1787 fasttrap_uint32_cmp); 1788 for (i = 1; i < dhpb->dthpb_noffs; i++) { 1789 if (dhpb->dthpb_base + dhpb->dthpb_offs[i] <= 1790 dhpb->dthpb_base + dhpb->dthpb_offs[i - 1]) 1791 return; 1792 } 1793 1794 qsort(dhpb->dthpb_enoffs, dhpb->dthpb_nenoffs, sizeof (uint32_t), 1795 fasttrap_uint32_cmp); 1796 for (i = 1; i < dhpb->dthpb_nenoffs; i++) { 1797 if (dhpb->dthpb_base + dhpb->dthpb_enoffs[i] <= 1798 dhpb->dthpb_base + dhpb->dthpb_enoffs[i - 1]) 1799 return; 1800 } 1801 1802 /* 1803 * Grab the creation lock to ensure consistency between calls to 1804 * dtrace_probe_lookup() and dtrace_probe_create() in the face of 1805 * other threads creating probes. 1806 / 1807* mutex_enter(&provider->ftp_cmtx); 1808 1809 if (dtrace_probe_lookup(provider->ftp_provid, dhpb->dthpb_mod, 1810 dhpb->dthpb_func, dhpb->dthpb_name) != 0) { 1811 mutex_exit(&provider->ftp_cmtx); 1812 return; 1813 } 1814 1815 ntps = dhpb->dthpb_noffs + dhpb->dthpb_nenoffs; 1816 ASSERT(ntps > 0); 1817 1818 atomic_add_32(&fasttrap_total, ntps); 1819 1820 if (fasttrap_total > fasttrap_max) { 1821 atomic_add_32(&fasttrap_total, -ntps); 1822 mutex_exit(&provider->ftp_cmtx); 1823 return; 1824 } 1825 1826 pp = kmem_zalloc(offsetof(fasttrap_probe_t, ftp_tps[ntps]), KM_SLEEP); 1827 1828 pp->ftp_prov = provider; 1829 pp->ftp_pid = provider->ftp_pid; 1830 pp->ftp_ntps = ntps; 1831 pp->ftp_nargs = dhpb->dthpb_xargc; 1832 pp->ftp_xtypes = dhpb->dthpb_xtypes; 1833 pp->ftp_ntypes = dhpb->dthpb_ntypes; 1834 1835 /* 1836 * First create a tracepoint for each actual point of interest. 1837 / 1838* for (i = 0; i < dhpb->dthpb_noffs; i++) { 1839 tp = kmem_zalloc(sizeof (fasttrap_tracepoint_t), KM_SLEEP); 1840 1841 tp->ftt_proc = provider->ftp_proc; 1842 tp->ftt_pc = dhpb->dthpb_base + dhpb->dthpb_offs[i]; 1843 tp->ftt_pid = provider->ftp_pid; 1844 1845 pp->ftp_tps[i].fit_tp = tp; 1846 pp->ftp_tps[i].fit_id.fti_probe = pp; 1847#ifdef __sparc 1848 pp->ftp_tps[i].fit_id.fti_ptype = DTFTP_POST_OFFSETS; 1849#else 1850 pp->ftp_tps[i].fit_id.fti_ptype = DTFTP_OFFSETS; 1851#endif 1852 } 1853 1854 /* 1855 * Then create a tracepoint for each is-enabled point. 1856 / 1857* for (j = 0; i < ntps; i++, j++) { 1858 tp = kmem_zalloc(sizeof (fasttrap_tracepoint_t), KM_SLEEP); 1859 1860 tp->ftt_proc = provider->ftp_proc; 1861 tp->ftt_pc = dhpb->dthpb_base + dhpb->dthpb_enoffs[j]; 1862 tp->ftt_pid = provider->ftp_pid; 1863 1864 pp->ftp_tps[i].fit_tp = tp; 1865 pp->ftp_tps[i].fit_id.fti_probe = pp; 1866 pp->ftp_tps[i].fit_id.fti_ptype = DTFTP_IS_ENABLED; 1867 } 1868 1869 /* 1870 * If the arguments are shuffled around we set the argument remapping 1871 * table. Later, when the probe fires, we only remap the arguments 1872 * if the table is non-NULL. 1873 / 1874* for (i = 0; i < dhpb->dthpb_xargc; i++) { 1875 if (dhpb->dthpb_args[i] != i) { 1876 pp->ftp_argmap = dhpb->dthpb_args; 1877 break; 1878 } 1879 } 1880 1881 /* 1882 * The probe is fully constructed -- register it with DTrace. 1883 / 1884* pp->ftp_id = dtrace_probe_create(provider->ftp_provid, dhpb->dthpb_mod, 1885 dhpb->dthpb_func, dhpb->dthpb_name, FASTTRAP_OFFSET_AFRAMES, pp); 1886 1887 mutex_exit(&provider->ftp_cmtx); 1888} 1889 1890/ARGSUSED/ 1891static void 1892fasttrap_meta_remove(void arg, dtrace_helper_provdesc_t dhpv, pid_t pid) 1893{ 1894 /* 1895 * Clean up the USDT provider. There may be active consumers of the 1896 * provider busy adding probes, no damage will actually befall the 1897 * provider until that count has dropped to zero. This just puts 1898 * the provider on death row. 1899 / 1900* fasttrap_provider_retire(pid, dhpv->dthpv_provname, 1); 1901} 1902 1903static dtrace_mops_t fasttrap_mops = { 1904 fasttrap_meta_create_probe, 1905 fasttrap_meta_provide, 1906 fasttrap_meta_remove 1907}; 1908 1909/ARGSUSED/ 1910static int 1911fasttrap_open(dev_t devp, int flag, int otyp, cred_t cred_p) 1912{ 1913 return (0); 1914} 1915 1916/ARGSUSED/ 1917static int 1918fasttrap_ioctl(dev_t dev, int cmd, intptr_t arg, int md, cred_t cr, int rv) 1919{ 1920 if (!dtrace_attached()) 1921 return (EAGAIN); 1922 1923 if (cmd == FASTTRAPIOC_MAKEPROBE) { 1924 fasttrap_probe_spec_t uprobe = (void )arg; 1925 fasttrap_probe_spec_t probe; 1926* uint64_t noffs; 1927 size_t size; 1928 int ret; 1929 char c; 1930* 1931 if (copyin(&uprobe->ftps_noffs, &noffs, 1932 sizeof (uprobe->ftps_noffs))) 1933 return (EFAULT); 1934 1935 /* 1936 * Probes must have at least one tracepoint. 1937 / 1938* if (noffs == 0) 1939 return (EINVAL); 1940 1941 size = sizeof (fasttrap_probe_spec_t) + 1942 sizeof (probe->ftps_offs[0]) * (noffs - 1); 1943 1944 if (size > 1024 * 1024) 1945 return (ENOMEM); 1946 1947 probe = kmem_alloc(size, KM_SLEEP); 1948 1949 if (copyin(uprobe, probe, size) != 0) { 1950 kmem_free(probe, size); 1951 return (EFAULT); 1952 } 1953 1954 /* 1955 * Verify that the function and module strings contain no 1956 * funny characters. 1957 / 1958* for (c = &probe->ftps_func[0]; c != '\0'; c++) { 1959* if (c < 0x20 \|\| 0x7f <= c) { 1960 ret = EINVAL; 1961 goto err; 1962 } 1963 } 1964 1965 for (c = &probe->ftps_mod[0]; c != '\0'; c++) { 1966* if (c < 0x20 \|\| 0x7f <= c) { 1967 ret = EINVAL; 1968 goto err; 1969 } 1970 } 1971 1972 if (!PRIV_POLICY_CHOICE(cr, PRIV_ALL, B_FALSE)) { 1973 proc_t p; 1974* pid_t pid = probe->ftps_pid; 1975 1976 mutex_enter(&pidlock); 1977 /* 1978 * Report an error if the process doesn't exist 1979 * or is actively being birthed. 1980 / 1981* if ((p = prfind(pid)) == NULL \|\| p->p_stat == SIDL) { 1982 mutex_exit(&pidlock); 1983 return (ESRCH); 1984 } 1985 mutex_enter(&p->p_lock); 1986 mutex_exit(&pidlock); 1987 1988 if ((ret = priv_proc_cred_perm(cr, p, NULL, 1989 VREAD \| VWRITE)) != 0) { 1990 mutex_exit(&p->p_lock); 1991 return (ret); 1992 } 1993 1994 mutex_exit(&p->p_lock); 1995 } 1996 1997 ret = fasttrap_add_probe(probe); 1998err: 1999 kmem_free(probe, size); 2000 2001 return (ret); 2002 2003 } else if (cmd == FASTTRAPIOC_GETINSTR) { 2004 fasttrap_instr_query_t instr; 2005 fasttrap_tracepoint_t tp; 2006* uint_t index; 2007 int ret; 2008 2009 if (copyin((void )arg, &instr, sizeof (instr)) != 0) 2010* return (EFAULT); 2011 2012 if (!PRIV_POLICY_CHOICE(cr, PRIV_ALL, B_FALSE)) { 2013 proc_t p; 2014* pid_t pid = instr.ftiq_pid; 2015 2016 mutex_enter(&pidlock); 2017 /* 2018 * Report an error if the process doesn't exist 2019 * or is actively being birthed. 2020 / 2021* if ((p = prfind(pid)) == NULL \|\| p->p_stat == SIDL) { 2022 mutex_exit(&pidlock); 2023 return (ESRCH); 2024 } 2025 mutex_enter(&p->p_lock); 2026 mutex_exit(&pidlock); 2027 2028 if ((ret = priv_proc_cred_perm(cr, p, NULL, 2029 VREAD)) != 0) { 2030 mutex_exit(&p->p_lock); 2031 return (ret); 2032 } 2033 2034 mutex_exit(&p->p_lock); 2035 } 2036 2037 index = FASTTRAP_TPOINTS_INDEX(instr.ftiq_pid, instr.ftiq_pc); 2038 2039 mutex_enter(&fasttrap_tpoints.fth_table[index].ftb_mtx); 2040 tp = fasttrap_tpoints.fth_table[index].ftb_data; 2041 while (tp != NULL) { 2042 if (instr.ftiq_pid == tp->ftt_pid && 2043 instr.ftiq_pc == tp->ftt_pc && 2044 tp->ftt_proc->ftpc_acount != 0) 2045 break; 2046	1/* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License (the "License"). 6 * You may not use this file except in compliance with the License. 7 * 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9 * or http://www.opensolaris.org/os/licensing. 10 * See the License for the specific language governing permissions 11 * and limitations under the License. 12 * 13 * When distributing Covered Code, include this CDDL HEADER in each 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15 * If applicable, add the following below this CDDL HEADER, with the 16 * fields enclosed by brackets "[]" replaced with your own identifying 17 * information: Portions Copyright [yyyy] [name of copyright owner] 18 * 19 * CDDL HEADER END 20 / 21 22/ 23 * Copyright 2008 Sun Microsystems, Inc. All rights reserved. 24 * Use is subject to license terms. 25 / 26 27#pragma ident "%Z%%M% %I% %E% SMI" 28 29#include <sys/atomic.h> 30#include <sys/errno.h> 31#include <sys/stat.h> 32#include <sys/modctl.h> 33#include <sys/conf.h> 34#include <sys/systm.h> 35#include <sys/ddi.h> 36#include <sys/sunddi.h> 37#include <sys/cpuvar.h> 38#include <sys/kmem.h> 39#include <sys/strsubr.h> 40#include <sys/fasttrap.h> 41#include <sys/fasttrap_impl.h> 42#include <sys/fasttrap_isa.h> 43#include <sys/dtrace.h> 44#include <sys/dtrace_impl.h> 45#include <sys/sysmacros.h> 46#include <sys/proc.h> 47#include <sys/priv.h> 48#include <sys/policy.h> 49#include <util/qsort.h> 50 51/ 52 * User-Land Trap-Based Tracing 53 * ---------------------------- 54 * 55 * The fasttrap provider allows DTrace consumers to instrument any user-level 56 * instruction to gather data; this includes probes with semantic 57 * signifigance like entry and return as well as simple offsets into the 58 * function. While the specific techniques used are very ISA specific, the 59 * methodology is generalizable to any architecture. 60 * 61 * 62 * The General Methodology 63 * ----------------------- 64 * 65 * With the primary goal of tracing every user-land instruction and the 66 * limitation that we can't trust user space so don't want to rely on much 67 * information there, we begin by replacing the instructions we want to trace 68 * with trap instructions. Each instruction we overwrite is saved into a hash 69 * table keyed by process ID and pc address. When we enter the kernel due to 70 * this trap instruction, we need the effects of the replaced instruction to 71 * appear to have occurred before we proceed with the user thread's 72 * execution. 73 * 74 * Each user level thread is represented by a ulwp_t structure which is 75 * always easily accessible through a register. The most basic way to produce 76 * the effects of the instruction we replaced is to copy that instruction out 77 * to a bit of scratch space reserved in the user thread's ulwp_t structure 78 * (a sort of kernel-private thread local storage), set the PC to that 79 * scratch space and single step. When we reenter the kernel after single 80 * stepping the instruction we must then adjust the PC to point to what would 81 * normally be the next instruction. Of course, special care must be taken 82 * for branches and jumps, but these represent such a small fraction of any 83 * instruction set that writing the code to emulate these in the kernel is 84 * not too difficult. 85 * 86 * Return probes may require several tracepoints to trace every return site, 87 * and, conversely, each tracepoint may activate several probes (the entry 88 * and offset 0 probes, for example). To solve this muliplexing problem, 89 * tracepoints contain lists of probes to activate and probes contain lists 90 * of tracepoints to enable. If a probe is activated, it adds its ID to 91 * existing tracepoints or creates new ones as necessary. 92 * 93 * Most probes are activated _before_ the instruction is executed, but return 94 * probes are activated _after_ the effects of the last instruction of the 95 * function are visible. Return probes must be fired _after_ we have 96 * single-stepped the instruction whereas all other probes are fired 97 * beforehand. 98 * 99 * 100 * Lock Ordering 101 * ------------- 102 * 103 * The lock ordering below -- both internally and with respect to the DTrace 104 * framework -- is a little tricky and bears some explanation. Each provider 105 * has a lock (ftp_mtx) that protects its members including reference counts 106 * for enabled probes (ftp_rcount), consumers actively creating probes 107 * (ftp_ccount) and USDT consumers (ftp_mcount); all three prevent a provider 108 * from being freed. A provider is looked up by taking the bucket lock for the 109 * provider hash table, and is returned with its lock held. The provider lock 110 * may be taken in functions invoked by the DTrace framework, but may not be 111 * held while calling functions in the DTrace framework. 112 * 113 * To ensure consistency over multiple calls to the DTrace framework, the 114 * creation lock (ftp_cmtx) should be held. Naturally, the creation lock may 115 * not be taken when holding the provider lock as that would create a cyclic 116 * lock ordering. In situations where one would naturally take the provider 117 * lock and then the creation lock, we instead up a reference count to prevent 118 * the provider from disappearing, drop the provider lock, and acquire the 119 * creation lock. 120 * 121 * Briefly: 122 * bucket lock before provider lock 123 * DTrace before provider lock 124 * creation lock before DTrace 125 * never hold the provider lock and creation lock simultaneously 126 / 127* 128static dev_info_t fasttrap_devi; 129static dtrace_meta_provider_id_t fasttrap_meta_id; 130* 131static timeout_id_t fasttrap_timeout; 132static kmutex_t fasttrap_cleanup_mtx; 133static uint_t fasttrap_cleanup_work; 134 135/* 136 * Generation count on modifications to the global tracepoint lookup table. 137 / 138static volatile uint64_t fasttrap_mod_gen; 139* 140/* 141 * When the fasttrap provider is loaded, fasttrap_max is set to either 142 * FASTTRAP_MAX_DEFAULT or the value for fasttrap-max-probes in the 143 * fasttrap.conf file. Each time a probe is created, fasttrap_total is 144 * incremented by the number of tracepoints that may be associated with that 145 * probe; fasttrap_total is capped at fasttrap_max. 146 / 147#define FASTTRAP_MAX_DEFAULT 250000 148static uint32_t fasttrap_max; 149static uint32_t fasttrap_total; 150* 151 152#define FASTTRAP_TPOINTS_DEFAULT_SIZE 0x4000 153#define FASTTRAP_PROVIDERS_DEFAULT_SIZE 0x100 154#define FASTTRAP_PROCS_DEFAULT_SIZE 0x100 155 156#define FASTTRAP_PID_NAME "pid" 157 158fasttrap_hash_t fasttrap_tpoints; 159static fasttrap_hash_t fasttrap_provs; 160static fasttrap_hash_t fasttrap_procs; 161 162static uint64_t fasttrap_pid_count; /* pid ref count / 163static kmutex_t fasttrap_count_mtx; / lock on ref count / 164* 165#define FASTTRAP_ENABLE_FAIL 1 166#define FASTTRAP_ENABLE_PARTIAL 2 167 168static int fasttrap_tracepoint_enable(proc_t , fasttrap_probe_t , uint_t); 169static void fasttrap_tracepoint_disable(proc_t , fasttrap_probe_t , uint_t); 170 171static fasttrap_provider_t fasttrap_provider_lookup(pid_t, const char , 172 const dtrace_pattr_t ); 173static void fasttrap_provider_retire(pid_t, const char , int); 174static void fasttrap_provider_free(fasttrap_provider_t ); 175* 176static fasttrap_proc_t fasttrap_proc_lookup(pid_t); 177static void fasttrap_proc_release(fasttrap_proc_t ); 178 179#define FASTTRAP_PROVS_INDEX(pid, name) \ 180 ((fasttrap_hash_str(name) + (pid)) & fasttrap_provs.fth_mask) 181 182#define FASTTRAP_PROCS_INDEX(pid) ((pid) & fasttrap_procs.fth_mask) 183 184static int 185fasttrap_highbit(ulong_t i) 186{ 187 int h = 1; 188 189 if (i == 0) 190 return (0); 191#ifdef _LP64 192 if (i & 0xffffffff00000000ul) { 193 h += 32; i >>= 32; 194 } 195#endif 196 if (i & 0xffff0000) { 197 h += 16; i >>= 16; 198 } 199 if (i & 0xff00) { 200 h += 8; i >>= 8; 201 } 202 if (i & 0xf0) { 203 h += 4; i >>= 4; 204 } 205 if (i & 0xc) { 206 h += 2; i >>= 2; 207 } 208 if (i & 0x2) { 209 h += 1; 210 } 211 return (h); 212} 213 214static uint_t 215fasttrap_hash_str(const char p) 216{ 217* unsigned int g; 218 uint_t hval = 0; 219 220 while (p) { 221* hval = (hval << 4) + p++; 222* if ((g = (hval & 0xf0000000)) != 0) 223 hval ^= g >> 24; 224 hval &= ~g; 225 } 226 return (hval); 227} 228 229void 230fasttrap_sigtrap(proc_t p, kthread_t t, uintptr_t pc) 231{ 232 sigqueue_t sqp = kmem_zalloc(sizeof (sigqueue_t), KM_SLEEP); 233* 234 sqp->sq_info.si_signo = SIGTRAP; 235 sqp->sq_info.si_code = TRAP_DTRACE; 236 sqp->sq_info.si_addr = (caddr_t)pc; 237 238 mutex_enter(&p->p_lock); 239 sigaddqa(p, t, sqp); 240 mutex_exit(&p->p_lock); 241 242 if (t != NULL) 243 aston(t); 244} 245 246/* 247 * This function ensures that no threads are actively using the memory 248 * associated with probes that were formerly live. 249 / 250static void 251fasttrap_mod_barrier(uint64_t gen) 252{ 253* int i; 254 255 if (gen < fasttrap_mod_gen) 256 return; 257 258 fasttrap_mod_gen++; 259 260 for (i = 0; i < NCPU; i++) { 261 mutex_enter(&cpu_core[i].cpuc_pid_lock); 262 mutex_exit(&cpu_core[i].cpuc_pid_lock); 263 } 264} 265 266/* 267 * This is the timeout's callback for cleaning up the providers and their 268 * probes. 269 / 270/ARGSUSED/ 271static void 272fasttrap_pid_cleanup_cb(void data) 273{ 274 fasttrap_provider_t *fpp, fp; 275 fasttrap_bucket_t bucket; 276* dtrace_provider_id_t provid; 277 int i, later; 278 279 static volatile int in = 0; 280 ASSERT(in == 0); 281 in = 1; 282 283 mutex_enter(&fasttrap_cleanup_mtx); 284 while (fasttrap_cleanup_work) { 285 fasttrap_cleanup_work = 0; 286 mutex_exit(&fasttrap_cleanup_mtx); 287 288 later = 0; 289 290 /* 291 * Iterate over all the providers trying to remove the marked 292 * ones. If a provider is marked but not retired, we just 293 * have to take a crack at removing it -- it's no big deal if 294 * we can't. 295 / 296* for (i = 0; i < fasttrap_provs.fth_nent; i++) { 297 bucket = &fasttrap_provs.fth_table[i]; 298 mutex_enter(&bucket->ftb_mtx); 299 fpp = (fasttrap_provider_t *)&bucket->ftb_data; 300* 301 while ((fp = fpp) != NULL) { 302* if (!fp->ftp_marked) { 303 fpp = &fp->ftp_next; 304 continue; 305 } 306 307 mutex_enter(&fp->ftp_mtx); 308 309 /* 310 * If this provider has consumers actively 311 * creating probes (ftp_ccount) or is a USDT 312 * provider (ftp_mcount), we can't unregister 313 * or even condense. 314 / 315* if (fp->ftp_ccount != 0 \|\| 316 fp->ftp_mcount != 0) { 317 mutex_exit(&fp->ftp_mtx); 318 fp->ftp_marked = 0; 319 continue; 320 } 321 322 if (!fp->ftp_retired \|\| fp->ftp_rcount != 0) 323 fp->ftp_marked = 0; 324 325 mutex_exit(&fp->ftp_mtx); 326 327 /* 328 * If we successfully unregister this 329 * provider we can remove it from the hash 330 * chain and free the memory. If our attempt 331 * to unregister fails and this is a retired 332 * provider, increment our flag to try again 333 * pretty soon. If we've consumed more than 334 * half of our total permitted number of 335 * probes call dtrace_condense() to try to 336 * clean out the unenabled probes. 337 / 338* provid = fp->ftp_provid; 339 if (dtrace_unregister(provid) != 0) { 340 if (fasttrap_total > fasttrap_max / 2) 341 (void) dtrace_condense(provid); 342 later += fp->ftp_marked; 343 fpp = &fp->ftp_next; 344 } else { 345 fpp = fp->ftp_next; 346* fasttrap_provider_free(fp); 347 } 348 } 349 mutex_exit(&bucket->ftb_mtx); 350 } 351 352 mutex_enter(&fasttrap_cleanup_mtx); 353 } 354 355 ASSERT(fasttrap_timeout != 0); 356 357 /* 358 * If we were unable to remove a retired provider, try again after 359 * a second. This situation can occur in certain circumstances where 360 * providers cannot be unregistered even though they have no probes 361 * enabled because of an execution of dtrace -l or something similar. 362 * If the timeout has been disabled (set to 1 because we're trying 363 * to detach), we set fasttrap_cleanup_work to ensure that we'll 364 * get a chance to do that work if and when the timeout is reenabled 365 * (if detach fails). 366 / 367* if (later > 0 && fasttrap_timeout != (timeout_id_t)1) 368 fasttrap_timeout = timeout(&fasttrap_pid_cleanup_cb, NULL, hz); 369 else if (later > 0) 370 fasttrap_cleanup_work = 1; 371 else 372 fasttrap_timeout = 0; 373 374 mutex_exit(&fasttrap_cleanup_mtx); 375 in = 0; 376} 377 378/* 379 * Activates the asynchronous cleanup mechanism. 380 / 381static void 382fasttrap_pid_cleanup(void) 383{ 384* mutex_enter(&fasttrap_cleanup_mtx); 385 fasttrap_cleanup_work = 1; 386 if (fasttrap_timeout == 0) 387 fasttrap_timeout = timeout(&fasttrap_pid_cleanup_cb, NULL, 1); 388 mutex_exit(&fasttrap_cleanup_mtx); 389} 390 391/* 392 * This is called from cfork() via dtrace_fasttrap_fork(). The child 393 * process's address space is (roughly) a copy of the parent process's so 394 * we have to remove all the instrumentation we had previously enabled in the 395 * parent. 396 / 397static void 398fasttrap_fork(proc_t p, proc_t cp) 399{ 400* pid_t ppid = p->p_pid; 401 int i; 402 403 ASSERT(curproc == p); 404 ASSERT(p->p_proc_flag & P_PR_LOCK); 405 ASSERT(p->p_dtrace_count > 0); 406 ASSERT(cp->p_dtrace_count == 0); 407 408 /* 409 * This would be simpler and faster if we maintained per-process 410 * hash tables of enabled tracepoints. It could, however, potentially 411 * slow down execution of a tracepoint since we'd need to go 412 * through two levels of indirection. In the future, we should 413 * consider either maintaining per-process ancillary lists of 414 * enabled tracepoints or hanging a pointer to a per-process hash 415 * table of enabled tracepoints off the proc structure. 416 / 417* 418 /* 419 * We don't have to worry about the child process disappearing 420 * because we're in fork(). 421 / 422* mutex_enter(&cp->p_lock); 423 sprlock_proc(cp); 424 mutex_exit(&cp->p_lock); 425 426 /* 427 * Iterate over every tracepoint looking for ones that belong to the 428 * parent process, and remove each from the child process. 429 / 430* for (i = 0; i < fasttrap_tpoints.fth_nent; i++) { 431 fasttrap_tracepoint_t tp; 432* fasttrap_bucket_t bucket = &fasttrap_tpoints.fth_table[i]; 433* 434 mutex_enter(&bucket->ftb_mtx); 435 for (tp = bucket->ftb_data; tp != NULL; tp = tp->ftt_next) { 436 if (tp->ftt_pid == ppid && 437 tp->ftt_proc->ftpc_acount != 0) { 438 int ret = fasttrap_tracepoint_remove(cp, tp); 439 ASSERT(ret == 0); 440 441 /* 442 * The count of active providers can only be 443 * decremented (i.e. to zero) during exec, 444 * exit, and removal of a meta provider so it 445 * should be impossible to drop the count 446 * mid-fork. 447 / 448* ASSERT(tp->ftt_proc->ftpc_acount != 0); 449 } 450 } 451 mutex_exit(&bucket->ftb_mtx); 452 } 453 454 mutex_enter(&cp->p_lock); 455 sprunlock(cp); 456} 457 458/* 459 * This is called from proc_exit() or from exec_common() if p_dtrace_probes 460 * is set on the proc structure to indicate that there is a pid provider 461 * associated with this process. 462 / 463static void 464fasttrap_exec_exit(proc_t p) 465{ 466 ASSERT(p == curproc); 467 ASSERT(MUTEX_HELD(&p->p_lock)); 468 469 mutex_exit(&p->p_lock); 470 471 /* 472 * We clean up the pid provider for this process here; user-land 473 * static probes are handled by the meta-provider remove entry point. 474 / 475* fasttrap_provider_retire(p->p_pid, FASTTRAP_PID_NAME, 0); 476 477 mutex_enter(&p->p_lock); 478} 479 480 481/ARGSUSED/ 482static void 483fasttrap_pid_provide(void arg, const dtrace_probedesc_t desc) 484{ 485 /* 486 * There are no "default" pid probes. 487 / 488} 489* 490static int 491fasttrap_tracepoint_enable(proc_t p, fasttrap_probe_t probe, uint_t index) 492{ 493 fasttrap_tracepoint_t tp, new_tp = NULL; 494 fasttrap_bucket_t bucket; 495* fasttrap_id_t id; 496* pid_t pid; 497 uintptr_t pc; 498 499 ASSERT(index < probe->ftp_ntps); 500 501 pid = probe->ftp_pid; 502 pc = probe->ftp_tps[index].fit_tp->ftt_pc; 503 id = &probe->ftp_tps[index].fit_id; 504 505 ASSERT(probe->ftp_tps[index].fit_tp->ftt_pid == pid); 506 507 ASSERT(!(p->p_flag & SVFORK)); 508 509 /* 510 * Before we make any modifications, make sure we've imposed a barrier 511 * on the generation in which this probe was last modified. 512 / 513* fasttrap_mod_barrier(probe->ftp_gen); 514 515 bucket = &fasttrap_tpoints.fth_table[FASTTRAP_TPOINTS_INDEX(pid, pc)]; 516 517 /* 518 * If the tracepoint has already been enabled, just add our id to the 519 * list of interested probes. This may be our second time through 520 * this path in which case we'll have constructed the tracepoint we'd 521 * like to install. If we can't find a match, and have an allocated 522 * tracepoint ready to go, enable that one now. 523 * 524 * A tracepoint whose process is defunct is also considered defunct. 525 / 526again: 527* mutex_enter(&bucket->ftb_mtx); 528 for (tp = bucket->ftb_data; tp != NULL; tp = tp->ftt_next) { 529 /* 530 * Note that it's safe to access the active count on the 531 * associated proc structure because we know that at least one 532 * provider (this one) will still be around throughout this 533 * operation. 534 / 535* if (tp->ftt_pid != pid \|\| tp->ftt_pc != pc \|\| 536 tp->ftt_proc->ftpc_acount == 0) 537 continue; 538 539 /* 540 * Now that we've found a matching tracepoint, it would be 541 * a decent idea to confirm that the tracepoint is still 542 * enabled and the trap instruction hasn't been overwritten. 543 * Since this is a little hairy, we'll punt for now. 544 / 545* 546 /* 547 * This can't be the first interested probe. We don't have 548 * to worry about another thread being in the midst of 549 * deleting this tracepoint (which would be the only valid 550 * reason for a tracepoint to have no interested probes) 551 * since we're holding P_PR_LOCK for this process. 552 / 553* ASSERT(tp->ftt_ids != NULL \|\| tp->ftt_retids != NULL); 554 555 switch (id->fti_ptype) { 556 case DTFTP_ENTRY: 557 case DTFTP_OFFSETS: 558 case DTFTP_IS_ENABLED: 559 id->fti_next = tp->ftt_ids; 560 membar_producer(); 561 tp->ftt_ids = id; 562 membar_producer(); 563 break; 564 565 case DTFTP_RETURN: 566 case DTFTP_POST_OFFSETS: 567 id->fti_next = tp->ftt_retids; 568 membar_producer(); 569 tp->ftt_retids = id; 570 membar_producer(); 571 break; 572 573 default: 574 ASSERT(0); 575 } 576 577 mutex_exit(&bucket->ftb_mtx); 578 579 if (new_tp != NULL) { 580 new_tp->ftt_ids = NULL; 581 new_tp->ftt_retids = NULL; 582 } 583 584 return (0); 585 } 586 587 /* 588 * If we have a good tracepoint ready to go, install it now while 589 * we have the lock held and no one can screw with us. 590 / 591* if (new_tp != NULL) { 592 int rc = 0; 593 594 new_tp->ftt_next = bucket->ftb_data; 595 membar_producer(); 596 bucket->ftb_data = new_tp; 597 membar_producer(); 598 mutex_exit(&bucket->ftb_mtx); 599 600 /* 601 * Activate the tracepoint in the ISA-specific manner. 602 * If this fails, we need to report the failure, but 603 * indicate that this tracepoint must still be disabled 604 * by calling fasttrap_tracepoint_disable(). 605 / 606* if (fasttrap_tracepoint_install(p, new_tp) != 0) 607 rc = FASTTRAP_ENABLE_PARTIAL; 608 609 /* 610 * Increment the count of the number of tracepoints active in 611 * the victim process. 612 / 613* ASSERT(p->p_proc_flag & P_PR_LOCK); 614 p->p_dtrace_count++; 615 616 return (rc); 617 } 618 619 mutex_exit(&bucket->ftb_mtx); 620 621 /* 622 * Initialize the tracepoint that's been preallocated with the probe. 623 / 624* new_tp = probe->ftp_tps[index].fit_tp; 625 626 ASSERT(new_tp->ftt_pid == pid); 627 ASSERT(new_tp->ftt_pc == pc); 628 ASSERT(new_tp->ftt_proc == probe->ftp_prov->ftp_proc); 629 ASSERT(new_tp->ftt_ids == NULL); 630 ASSERT(new_tp->ftt_retids == NULL); 631 632 switch (id->fti_ptype) { 633 case DTFTP_ENTRY: 634 case DTFTP_OFFSETS: 635 case DTFTP_IS_ENABLED: 636 id->fti_next = NULL; 637 new_tp->ftt_ids = id; 638 break; 639 640 case DTFTP_RETURN: 641 case DTFTP_POST_OFFSETS: 642 id->fti_next = NULL; 643 new_tp->ftt_retids = id; 644 break; 645 646 default: 647 ASSERT(0); 648 } 649 650 /* 651 * If the ISA-dependent initialization goes to plan, go back to the 652 * beginning and try to install this freshly made tracepoint. 653 / 654* if (fasttrap_tracepoint_init(p, new_tp, pc, id->fti_ptype) == 0) 655 goto again; 656 657 new_tp->ftt_ids = NULL; 658 new_tp->ftt_retids = NULL; 659 660 return (FASTTRAP_ENABLE_FAIL); 661} 662 663static void 664fasttrap_tracepoint_disable(proc_t p, fasttrap_probe_t probe, uint_t index) 665{ 666 fasttrap_bucket_t bucket; 667* fasttrap_provider_t provider = probe->ftp_prov; 668* fasttrap_tracepoint_t *pp, tp; 669 fasttrap_id_t id, idp; 670* pid_t pid; 671 uintptr_t pc; 672 673 ASSERT(index < probe->ftp_ntps); 674 675 pid = probe->ftp_pid; 676 pc = probe->ftp_tps[index].fit_tp->ftt_pc; 677 id = &probe->ftp_tps[index].fit_id; 678 679 ASSERT(probe->ftp_tps[index].fit_tp->ftt_pid == pid); 680 681 /* 682 * Find the tracepoint and make sure that our id is one of the 683 * ones registered with it. 684 / 685* bucket = &fasttrap_tpoints.fth_table[FASTTRAP_TPOINTS_INDEX(pid, pc)]; 686 mutex_enter(&bucket->ftb_mtx); 687 for (tp = bucket->ftb_data; tp != NULL; tp = tp->ftt_next) { 688 if (tp->ftt_pid == pid && tp->ftt_pc == pc && 689 tp->ftt_proc == provider->ftp_proc) 690 break; 691 } 692 693 /* 694 * If we somehow lost this tracepoint, we're in a world of hurt. 695 / 696* ASSERT(tp != NULL); 697 698 switch (id->fti_ptype) { 699 case DTFTP_ENTRY: 700 case DTFTP_OFFSETS: 701 case DTFTP_IS_ENABLED: 702 ASSERT(tp->ftt_ids != NULL); 703 idp = &tp->ftt_ids; 704 break; 705 706 case DTFTP_RETURN: 707 case DTFTP_POST_OFFSETS: 708 ASSERT(tp->ftt_retids != NULL); 709 idp = &tp->ftt_retids; 710 break; 711 712 default: 713 ASSERT(0); 714 } 715 716 while ((idp)->fti_probe != probe) { 717* idp = &(idp)->fti_next; 718* ASSERT(idp != NULL); 719* } 720 721 id = idp; 722* idp = id->fti_next; 723* membar_producer(); 724 725 ASSERT(id->fti_probe == probe); 726 727 /* 728 * If there are other registered enablings of this tracepoint, we're 729 * all done, but if this was the last probe assocated with this 730 * this tracepoint, we need to remove and free it. 731 / 732* if (tp->ftt_ids != NULL \|\| tp->ftt_retids != NULL) { 733 734 /* 735 * If the current probe's tracepoint is in use, swap it 736 * for an unused tracepoint. 737 / 738* if (tp == probe->ftp_tps[index].fit_tp) { 739 fasttrap_probe_t tmp_probe; 740* fasttrap_tracepoint_t *tmp_tp; 741* uint_t tmp_index; 742 743 if (tp->ftt_ids != NULL) { 744 tmp_probe = tp->ftt_ids->fti_probe; 745 /* LINTED - alignment / 746* tmp_index = FASTTRAP_ID_INDEX(tp->ftt_ids); 747 tmp_tp = &tmp_probe->ftp_tps[tmp_index].fit_tp; 748 } else { 749 tmp_probe = tp->ftt_retids->fti_probe; 750 /* LINTED - alignment / 751* tmp_index = FASTTRAP_ID_INDEX(tp->ftt_retids); 752 tmp_tp = &tmp_probe->ftp_tps[tmp_index].fit_tp; 753 } 754 755 ASSERT(tmp_tp != NULL); 756* ASSERT(tmp_tp != probe->ftp_tps[index].fit_tp); 757* ASSERT((tmp_tp)->ftt_ids == NULL); 758* ASSERT((tmp_tp)->ftt_retids == NULL); 759* 760 probe->ftp_tps[index].fit_tp = tmp_tp; 761* tmp_tp = tp; 762* } 763 764 mutex_exit(&bucket->ftb_mtx); 765 766 /* 767 * Tag the modified probe with the generation in which it was 768 * changed. 769 / 770* probe->ftp_gen = fasttrap_mod_gen; 771 return; 772 } 773 774 mutex_exit(&bucket->ftb_mtx); 775 776 /* 777 * We can't safely remove the tracepoint from the set of active 778 * tracepoints until we've actually removed the fasttrap instruction 779 * from the process's text. We can, however, operate on this 780 * tracepoint secure in the knowledge that no other thread is going to 781 * be looking at it since we hold P_PR_LOCK on the process if it's 782 * live or we hold the provider lock on the process if it's dead and 783 * gone. 784 / 785* 786 /* 787 * We only need to remove the actual instruction if we're looking 788 * at an existing process 789 / 790* if (p != NULL) { 791 /* 792 * If we fail to restore the instruction we need to kill 793 * this process since it's in a completely unrecoverable 794 * state. 795 / 796* if (fasttrap_tracepoint_remove(p, tp) != 0) 797 fasttrap_sigtrap(p, NULL, pc); 798 799 /* 800 * Decrement the count of the number of tracepoints active 801 * in the victim process. 802 / 803* ASSERT(p->p_proc_flag & P_PR_LOCK); 804 p->p_dtrace_count--; 805 } 806 807 /* 808 * Remove the probe from the hash table of active tracepoints. 809 / 810* mutex_enter(&bucket->ftb_mtx); 811 pp = (fasttrap_tracepoint_t *)&bucket->ftb_data; 812* ASSERT(pp != NULL); 813* while (pp != tp) { 814* pp = &(pp)->ftt_next; 815* ASSERT(pp != NULL); 816* } 817 818 pp = tp->ftt_next; 819* membar_producer(); 820 821 mutex_exit(&bucket->ftb_mtx); 822 823 /* 824 * Tag the modified probe with the generation in which it was changed. 825 / 826* probe->ftp_gen = fasttrap_mod_gen; 827} 828 829static void 830fasttrap_enable_callbacks(void) 831{ 832 /* 833 * We don't have to play the rw lock game here because we're 834 * providing something rather than taking something away -- 835 * we can be sure that no threads have tried to follow this 836 * function pointer yet. 837 / 838* mutex_enter(&fasttrap_count_mtx); 839 if (fasttrap_pid_count == 0) { 840 ASSERT(dtrace_pid_probe_ptr == NULL); 841 ASSERT(dtrace_return_probe_ptr == NULL); 842 dtrace_pid_probe_ptr = &fasttrap_pid_probe; 843 dtrace_return_probe_ptr = &fasttrap_return_probe; 844 } 845 ASSERT(dtrace_pid_probe_ptr == &fasttrap_pid_probe); 846 ASSERT(dtrace_return_probe_ptr == &fasttrap_return_probe); 847 fasttrap_pid_count++; 848 mutex_exit(&fasttrap_count_mtx); 849} 850 851static void 852fasttrap_disable_callbacks(void) 853{ 854 ASSERT(MUTEX_HELD(&cpu_lock)); 855 856 mutex_enter(&fasttrap_count_mtx); 857 ASSERT(fasttrap_pid_count > 0); 858 fasttrap_pid_count--; 859 if (fasttrap_pid_count == 0) { 860 cpu_t cur, cpu = CPU; 861 862 for (cur = cpu->cpu_next_onln; cur != cpu; 863 cur = cur->cpu_next_onln) { 864 rw_enter(&cur->cpu_ft_lock, RW_WRITER); 865 } 866 867 dtrace_pid_probe_ptr = NULL; 868 dtrace_return_probe_ptr = NULL; 869 870 for (cur = cpu->cpu_next_onln; cur != cpu; 871 cur = cur->cpu_next_onln) { 872 rw_exit(&cur->cpu_ft_lock); 873 } 874 } 875 mutex_exit(&fasttrap_count_mtx); 876} 877 878/ARGSUSED/ 879static void 880fasttrap_pid_enable(void arg, dtrace_id_t id, void parg) 881{ 882 fasttrap_probe_t probe = parg; 883* proc_t p; 884* int i, rc; 885 886 ASSERT(probe != NULL); 887 ASSERT(!probe->ftp_enabled); 888 ASSERT(id == probe->ftp_id); 889 ASSERT(MUTEX_HELD(&cpu_lock)); 890 891 /* 892 * Increment the count of enabled probes on this probe's provider; 893 * the provider can't go away while the probe still exists. We 894 * must increment this even if we aren't able to properly enable 895 * this probe. 896 / 897* mutex_enter(&probe->ftp_prov->ftp_mtx); 898 probe->ftp_prov->ftp_rcount++; 899 mutex_exit(&probe->ftp_prov->ftp_mtx); 900 901 /* 902 * If this probe's provider is retired (meaning it was valid in a 903 * previously exec'ed incarnation of this address space), bail out. The 904 * provider can't go away while we're in this code path. 905 / 906* if (probe->ftp_prov->ftp_retired) 907 return; 908 909 /* 910 * If we can't find the process, it may be that we're in the context of 911 * a fork in which the traced process is being born and we're copying 912 * USDT probes. Otherwise, the process is gone so bail. 913 / 914* if ((p = sprlock(probe->ftp_pid)) == NULL) { 915 if ((curproc->p_flag & SFORKING) == 0) 916 return; 917 918 mutex_enter(&pidlock); 919 p = prfind(probe->ftp_pid); 920 921 /* 922 * Confirm that curproc is indeed forking the process in which 923 * we're trying to enable probes. 924 / 925* ASSERT(p != NULL); 926 ASSERT(p->p_parent == curproc); 927 ASSERT(p->p_stat == SIDL); 928 929 mutex_enter(&p->p_lock); 930 mutex_exit(&pidlock); 931 932 sprlock_proc(p); 933 } 934 935 ASSERT(!(p->p_flag & SVFORK)); 936 mutex_exit(&p->p_lock); 937 938 /* 939 * We have to enable the trap entry point before any user threads have 940 * the chance to execute the trap instruction we're about to place 941 * in their process's text. 942 / 943* fasttrap_enable_callbacks(); 944 945 /* 946 * Enable all the tracepoints and add this probe's id to each 947 * tracepoint's list of active probes. 948 / 949* for (i = 0; i < probe->ftp_ntps; i++) { 950 if ((rc = fasttrap_tracepoint_enable(p, probe, i)) != 0) { 951 /* 952 * If enabling the tracepoint failed completely, 953 * we don't have to disable it; if the failure 954 * was only partial we must disable it. 955 / 956* if (rc == FASTTRAP_ENABLE_FAIL) 957 i--; 958 else 959 ASSERT(rc == FASTTRAP_ENABLE_PARTIAL); 960 961 /* 962 * Back up and pull out all the tracepoints we've 963 * created so far for this probe. 964 / 965* while (i >= 0) { 966 fasttrap_tracepoint_disable(p, probe, i); 967 i--; 968 } 969 970 mutex_enter(&p->p_lock); 971 sprunlock(p); 972 973 /* 974 * Since we're not actually enabling this probe, 975 * drop our reference on the trap table entry. 976 / 977* fasttrap_disable_callbacks(); 978 return; 979 } 980 } 981 982 mutex_enter(&p->p_lock); 983 sprunlock(p); 984 985 probe->ftp_enabled = 1; 986} 987 988/ARGSUSED/ 989static void 990fasttrap_pid_disable(void arg, dtrace_id_t id, void parg) 991{ 992 fasttrap_probe_t probe = parg; 993* fasttrap_provider_t provider = probe->ftp_prov; 994* proc_t p; 995* int i, whack = 0; 996 997 ASSERT(id == probe->ftp_id); 998 999 /* 1000 * We won't be able to acquire a /proc-esque lock on the process 1001 * iff the process is dead and gone. In this case, we rely on the 1002 * provider lock as a point of mutual exclusion to prevent other 1003 * DTrace consumers from disabling this probe. 1004 / 1005* if ((p = sprlock(probe->ftp_pid)) != NULL) { 1006 ASSERT(!(p->p_flag & SVFORK)); 1007 mutex_exit(&p->p_lock); 1008 } 1009 1010 mutex_enter(&provider->ftp_mtx); 1011 1012 /* 1013 * Disable all the associated tracepoints (for fully enabled probes). 1014 / 1015* if (probe->ftp_enabled) { 1016 for (i = 0; i < probe->ftp_ntps; i++) { 1017 fasttrap_tracepoint_disable(p, probe, i); 1018 } 1019 } 1020 1021 ASSERT(provider->ftp_rcount > 0); 1022 provider->ftp_rcount--; 1023 1024 if (p != NULL) { 1025 /* 1026 * Even though we may not be able to remove it entirely, we 1027 * mark this retired provider to get a chance to remove some 1028 * of the associated probes. 1029 / 1030* if (provider->ftp_retired && !provider->ftp_marked) 1031 whack = provider->ftp_marked = 1; 1032 mutex_exit(&provider->ftp_mtx); 1033 1034 mutex_enter(&p->p_lock); 1035 sprunlock(p); 1036 } else { 1037 /* 1038 * If the process is dead, we're just waiting for the 1039 * last probe to be disabled to be able to free it. 1040 / 1041* if (provider->ftp_rcount == 0 && !provider->ftp_marked) 1042 whack = provider->ftp_marked = 1; 1043 mutex_exit(&provider->ftp_mtx); 1044 } 1045 1046 if (whack) 1047 fasttrap_pid_cleanup(); 1048 1049 if (!probe->ftp_enabled) 1050 return; 1051 1052 probe->ftp_enabled = 0; 1053 1054 ASSERT(MUTEX_HELD(&cpu_lock)); 1055 fasttrap_disable_callbacks(); 1056} 1057 1058/ARGSUSED/ 1059static void 1060fasttrap_pid_getargdesc(void arg, dtrace_id_t id, void parg, 1061 dtrace_argdesc_t desc) 1062{ 1063* fasttrap_probe_t probe = parg; 1064* char str; 1065* int i, ndx; 1066 1067 desc->dtargd_native[0] = '\0'; 1068 desc->dtargd_xlate[0] = '\0'; 1069 1070 if (probe->ftp_prov->ftp_retired != 0 \|\| 1071 desc->dtargd_ndx >= probe->ftp_nargs) { 1072 desc->dtargd_ndx = DTRACE_ARGNONE; 1073 return; 1074 } 1075 1076 ndx = (probe->ftp_argmap != NULL) ? 1077 probe->ftp_argmap[desc->dtargd_ndx] : desc->dtargd_ndx; 1078 1079 str = probe->ftp_ntypes; 1080 for (i = 0; i < ndx; i++) { 1081 str += strlen(str) + 1; 1082 } 1083 1084 ASSERT(strlen(str + 1) < sizeof (desc->dtargd_native)); 1085 (void) strcpy(desc->dtargd_native, str); 1086 1087 if (probe->ftp_xtypes == NULL) 1088 return; 1089 1090 str = probe->ftp_xtypes; 1091 for (i = 0; i < desc->dtargd_ndx; i++) { 1092 str += strlen(str) + 1; 1093 } 1094 1095 ASSERT(strlen(str + 1) < sizeof (desc->dtargd_xlate)); 1096 (void) strcpy(desc->dtargd_xlate, str); 1097} 1098 1099/ARGSUSED/ 1100static void 1101fasttrap_pid_destroy(void arg, dtrace_id_t id, void parg) 1102{ 1103 fasttrap_probe_t probe = parg; 1104* int i; 1105 size_t size; 1106 1107 ASSERT(probe != NULL); 1108 ASSERT(!probe->ftp_enabled); 1109 ASSERT(fasttrap_total >= probe->ftp_ntps); 1110 1111 atomic_add_32(&fasttrap_total, -probe->ftp_ntps); 1112 size = offsetof(fasttrap_probe_t, ftp_tps[probe->ftp_ntps]); 1113 1114 if (probe->ftp_gen + 1 >= fasttrap_mod_gen) 1115 fasttrap_mod_barrier(probe->ftp_gen); 1116 1117 for (i = 0; i < probe->ftp_ntps; i++) { 1118 kmem_free(probe->ftp_tps[i].fit_tp, 1119 sizeof (fasttrap_tracepoint_t)); 1120 } 1121 1122 kmem_free(probe, size); 1123} 1124 1125 1126static const dtrace_pattr_t pid_attr = { 1127{ DTRACE_STABILITY_EVOLVING, DTRACE_STABILITY_EVOLVING, DTRACE_CLASS_ISA }, 1128{ DTRACE_STABILITY_PRIVATE, DTRACE_STABILITY_PRIVATE, DTRACE_CLASS_UNKNOWN }, 1129{ DTRACE_STABILITY_PRIVATE, DTRACE_STABILITY_PRIVATE, DTRACE_CLASS_UNKNOWN }, 1130{ DTRACE_STABILITY_EVOLVING, DTRACE_STABILITY_EVOLVING, DTRACE_CLASS_ISA }, 1131{ DTRACE_STABILITY_PRIVATE, DTRACE_STABILITY_PRIVATE, DTRACE_CLASS_UNKNOWN }, 1132}; 1133 1134static dtrace_pops_t pid_pops = { 1135 fasttrap_pid_provide, 1136 NULL, 1137 fasttrap_pid_enable, 1138 fasttrap_pid_disable, 1139 NULL, 1140 NULL, 1141 fasttrap_pid_getargdesc, 1142 fasttrap_pid_getarg, 1143 NULL, 1144 fasttrap_pid_destroy 1145}; 1146 1147static dtrace_pops_t usdt_pops = { 1148 fasttrap_pid_provide, 1149 NULL, 1150 fasttrap_pid_enable, 1151 fasttrap_pid_disable, 1152 NULL, 1153 NULL, 1154 fasttrap_pid_getargdesc, 1155 fasttrap_usdt_getarg, 1156 NULL, 1157 fasttrap_pid_destroy 1158}; 1159 1160static fasttrap_proc_t * 1161fasttrap_proc_lookup(pid_t pid) 1162{ 1163 fasttrap_bucket_t bucket; 1164* fasttrap_proc_t fprc, new_fprc; 1165 1166 bucket = &fasttrap_procs.fth_table[FASTTRAP_PROCS_INDEX(pid)]; 1167 mutex_enter(&bucket->ftb_mtx); 1168 1169 for (fprc = bucket->ftb_data; fprc != NULL; fprc = fprc->ftpc_next) { 1170 if (fprc->ftpc_pid == pid && fprc->ftpc_acount != 0) { 1171 mutex_enter(&fprc->ftpc_mtx); 1172 mutex_exit(&bucket->ftb_mtx); 1173 fprc->ftpc_rcount++; 1174 atomic_add_64(&fprc->ftpc_acount, 1); 1175 ASSERT(fprc->ftpc_acount <= fprc->ftpc_rcount); 1176 mutex_exit(&fprc->ftpc_mtx); 1177 1178 return (fprc); 1179 } 1180 } 1181 1182 /* 1183 * Drop the bucket lock so we don't try to perform a sleeping 1184 * allocation under it. 1185 / 1186* mutex_exit(&bucket->ftb_mtx); 1187 1188 new_fprc = kmem_zalloc(sizeof (fasttrap_proc_t), KM_SLEEP); 1189 new_fprc->ftpc_pid = pid; 1190 new_fprc->ftpc_rcount = 1; 1191 new_fprc->ftpc_acount = 1; 1192 1193 mutex_enter(&bucket->ftb_mtx); 1194 1195 /* 1196 * Take another lap through the list to make sure a proc hasn't 1197 * been created for this pid while we weren't under the bucket lock. 1198 / 1199* for (fprc = bucket->ftb_data; fprc != NULL; fprc = fprc->ftpc_next) { 1200 if (fprc->ftpc_pid == pid && fprc->ftpc_acount != 0) { 1201 mutex_enter(&fprc->ftpc_mtx); 1202 mutex_exit(&bucket->ftb_mtx); 1203 fprc->ftpc_rcount++; 1204 atomic_add_64(&fprc->ftpc_acount, 1); 1205 ASSERT(fprc->ftpc_acount <= fprc->ftpc_rcount); 1206 mutex_exit(&fprc->ftpc_mtx); 1207 1208 kmem_free(new_fprc, sizeof (fasttrap_proc_t)); 1209 1210 return (fprc); 1211 } 1212 } 1213 1214 new_fprc->ftpc_next = bucket->ftb_data; 1215 bucket->ftb_data = new_fprc; 1216 1217 mutex_exit(&bucket->ftb_mtx); 1218 1219 return (new_fprc); 1220} 1221 1222static void 1223fasttrap_proc_release(fasttrap_proc_t proc) 1224{ 1225* fasttrap_bucket_t bucket; 1226* fasttrap_proc_t fprc, fprcp; 1227* pid_t pid = proc->ftpc_pid; 1228 1229 mutex_enter(&proc->ftpc_mtx); 1230 1231 ASSERT(proc->ftpc_rcount != 0); 1232 ASSERT(proc->ftpc_acount <= proc->ftpc_rcount); 1233 1234 if (--proc->ftpc_rcount != 0) { 1235 mutex_exit(&proc->ftpc_mtx); 1236 return; 1237 } 1238 1239 mutex_exit(&proc->ftpc_mtx); 1240 1241 /* 1242 * There should definitely be no live providers associated with this 1243 * process at this point. 1244 / 1245* ASSERT(proc->ftpc_acount == 0); 1246 1247 bucket = &fasttrap_procs.fth_table[FASTTRAP_PROCS_INDEX(pid)]; 1248 mutex_enter(&bucket->ftb_mtx); 1249 1250 fprcp = (fasttrap_proc_t *)&bucket->ftb_data; 1251* while ((fprc = fprcp) != NULL) { 1252* if (fprc == proc) 1253 break; 1254 1255 fprcp = &fprc->ftpc_next; 1256 } 1257 1258 /* 1259 * Something strange has happened if we can't find the proc. 1260 / 1261* ASSERT(fprc != NULL); 1262 1263 fprcp = fprc->ftpc_next; 1264* 1265 mutex_exit(&bucket->ftb_mtx); 1266 1267 kmem_free(fprc, sizeof (fasttrap_proc_t)); 1268} 1269 1270/* 1271 * Lookup a fasttrap-managed provider based on its name and associated pid. 1272 * If the pattr argument is non-NULL, this function instantiates the provider 1273 * if it doesn't exist otherwise it returns NULL. The provider is returned 1274 * with its lock held. 1275 / 1276static fasttrap_provider_t 1277fasttrap_provider_lookup(pid_t pid, const char name, 1278* const dtrace_pattr_t pattr) 1279{ 1280* fasttrap_provider_t fp, new_fp = NULL; 1281 fasttrap_bucket_t bucket; 1282* char provname[DTRACE_PROVNAMELEN]; 1283 proc_t p; 1284* cred_t cred; 1285* 1286 ASSERT(strlen(name) < sizeof (fp->ftp_name)); 1287 ASSERT(pattr != NULL); 1288 1289 bucket = &fasttrap_provs.fth_table[FASTTRAP_PROVS_INDEX(pid, name)]; 1290 mutex_enter(&bucket->ftb_mtx); 1291 1292 /* 1293 * Take a lap through the list and return the match if we find it. 1294 / 1295* for (fp = bucket->ftb_data; fp != NULL; fp = fp->ftp_next) { 1296 if (fp->ftp_pid == pid && strcmp(fp->ftp_name, name) == 0 && 1297 !fp->ftp_retired) { 1298 mutex_enter(&fp->ftp_mtx); 1299 mutex_exit(&bucket->ftb_mtx); 1300 return (fp); 1301 } 1302 } 1303 1304 /* 1305 * Drop the bucket lock so we don't try to perform a sleeping 1306 * allocation under it. 1307 / 1308* mutex_exit(&bucket->ftb_mtx); 1309 1310 /* 1311 * Make sure the process exists, isn't a child created as the result 1312 * of a vfork(2), and isn't a zombie (but may be in fork). 1313 / 1314* mutex_enter(&pidlock); 1315 if ((p = prfind(pid)) == NULL) { 1316 mutex_exit(&pidlock); 1317 return (NULL); 1318 } 1319 mutex_enter(&p->p_lock); 1320 mutex_exit(&pidlock); 1321 if (p->p_flag & (SVFORK \| SEXITING)) { 1322 mutex_exit(&p->p_lock); 1323 return (NULL); 1324 } 1325 1326 /* 1327 * Increment p_dtrace_probes so that the process knows to inform us 1328 * when it exits or execs. fasttrap_provider_free() decrements this 1329 * when we're done with this provider. 1330 / 1331* p->p_dtrace_probes++; 1332 1333 /* 1334 * Grab the credentials for this process so we have 1335 * something to pass to dtrace_register(). 1336 / 1337* mutex_enter(&p->p_crlock); 1338 crhold(p->p_cred); 1339 cred = p->p_cred; 1340 mutex_exit(&p->p_crlock); 1341 mutex_exit(&p->p_lock); 1342 1343 new_fp = kmem_zalloc(sizeof (fasttrap_provider_t), KM_SLEEP); 1344 new_fp->ftp_pid = pid; 1345 new_fp->ftp_proc = fasttrap_proc_lookup(pid); 1346 1347 ASSERT(new_fp->ftp_proc != NULL); 1348 1349 mutex_enter(&bucket->ftb_mtx); 1350 1351 /* 1352 * Take another lap through the list to make sure a provider hasn't 1353 * been created for this pid while we weren't under the bucket lock. 1354 / 1355* for (fp = bucket->ftb_data; fp != NULL; fp = fp->ftp_next) { 1356 if (fp->ftp_pid == pid && strcmp(fp->ftp_name, name) == 0 && 1357 !fp->ftp_retired) { 1358 mutex_enter(&fp->ftp_mtx); 1359 mutex_exit(&bucket->ftb_mtx); 1360 fasttrap_provider_free(new_fp); 1361 crfree(cred); 1362 return (fp); 1363 } 1364 } 1365 1366 (void) strcpy(new_fp->ftp_name, name); 1367 1368 /* 1369 * Fail and return NULL if either the provider name is too long 1370 * or we fail to register this new provider with the DTrace 1371 * framework. Note that this is the only place we ever construct 1372 * the full provider name -- we keep it in pieces in the provider 1373 * structure. 1374 / 1375* if (snprintf(provname, sizeof (provname), "%s%u", name, (uint_t)pid) >= 1376 sizeof (provname) \|\| 1377 dtrace_register(provname, pattr, 1378 DTRACE_PRIV_PROC \| DTRACE_PRIV_OWNER \| DTRACE_PRIV_ZONEOWNER, cred, 1379 pattr == &pid_attr ? &pid_pops : &usdt_pops, new_fp, 1380 &new_fp->ftp_provid) != 0) { 1381 mutex_exit(&bucket->ftb_mtx); 1382 fasttrap_provider_free(new_fp); 1383 crfree(cred); 1384 return (NULL); 1385 } 1386 1387 new_fp->ftp_next = bucket->ftb_data; 1388 bucket->ftb_data = new_fp; 1389 1390 mutex_enter(&new_fp->ftp_mtx); 1391 mutex_exit(&bucket->ftb_mtx); 1392 1393 crfree(cred); 1394 return (new_fp); 1395} 1396 1397static void 1398fasttrap_provider_free(fasttrap_provider_t provider) 1399{ 1400* pid_t pid = provider->ftp_pid; 1401 proc_t p; 1402* 1403 /* 1404 * There need to be no associated enabled probes, no consumers 1405 * creating probes, and no meta providers referencing this provider. 1406 / 1407* ASSERT(provider->ftp_rcount == 0); 1408 ASSERT(provider->ftp_ccount == 0); 1409 ASSERT(provider->ftp_mcount == 0); 1410 1411 /* 1412 * If this provider hasn't been retired, we need to explicitly drop the 1413 * count of active providers on the associated process structure. 1414 / 1415* if (!provider->ftp_retired) { 1416 atomic_add_64(&provider->ftp_proc->ftpc_acount, -1); 1417 ASSERT(provider->ftp_proc->ftpc_acount < 1418 provider->ftp_proc->ftpc_rcount); 1419 } 1420 1421 fasttrap_proc_release(provider->ftp_proc); 1422 1423 kmem_free(provider, sizeof (fasttrap_provider_t)); 1424 1425 /* 1426 * Decrement p_dtrace_probes on the process whose provider we're 1427 * freeing. We don't have to worry about clobbering somone else's 1428 * modifications to it because we have locked the bucket that 1429 * corresponds to this process's hash chain in the provider hash 1430 * table. Don't sweat it if we can't find the process. 1431 / 1432* mutex_enter(&pidlock); 1433 if ((p = prfind(pid)) == NULL) { 1434 mutex_exit(&pidlock); 1435 return; 1436 } 1437 1438 mutex_enter(&p->p_lock); 1439 mutex_exit(&pidlock); 1440 1441 p->p_dtrace_probes--; 1442 mutex_exit(&p->p_lock); 1443} 1444 1445static void 1446fasttrap_provider_retire(pid_t pid, const char name, int mprov) 1447{ 1448* fasttrap_provider_t fp; 1449* fasttrap_bucket_t bucket; 1450* dtrace_provider_id_t provid; 1451 1452 ASSERT(strlen(name) < sizeof (fp->ftp_name)); 1453 1454 bucket = &fasttrap_provs.fth_table[FASTTRAP_PROVS_INDEX(pid, name)]; 1455 mutex_enter(&bucket->ftb_mtx); 1456 1457 for (fp = bucket->ftb_data; fp != NULL; fp = fp->ftp_next) { 1458 if (fp->ftp_pid == pid && strcmp(fp->ftp_name, name) == 0 && 1459 !fp->ftp_retired) 1460 break; 1461 } 1462 1463 if (fp == NULL) { 1464 mutex_exit(&bucket->ftb_mtx); 1465 return; 1466 } 1467 1468 mutex_enter(&fp->ftp_mtx); 1469 ASSERT(!mprov \|\| fp->ftp_mcount > 0); 1470 if (mprov && --fp->ftp_mcount != 0) { 1471 mutex_exit(&fp->ftp_mtx); 1472 mutex_exit(&bucket->ftb_mtx); 1473 return; 1474 } 1475 1476 /* 1477 * Mark the provider to be removed in our post-processing step, mark it 1478 * retired, and drop the active count on its proc. Marking it indicates 1479 * that we should try to remove it; setting the retired flag indicates 1480 * that we're done with this provider; dropping the active the proc 1481 * releases our hold, and when this reaches zero (as it will during 1482 * exit or exec) the proc and associated providers become defunct. 1483 * 1484 * We obviously need to take the bucket lock before the provider lock 1485 * to perform the lookup, but we need to drop the provider lock 1486 * before calling into the DTrace framework since we acquire the 1487 * provider lock in callbacks invoked from the DTrace framework. The 1488 * bucket lock therefore protects the integrity of the provider hash 1489 * table. 1490 / 1491* atomic_add_64(&fp->ftp_proc->ftpc_acount, -1); 1492 ASSERT(fp->ftp_proc->ftpc_acount < fp->ftp_proc->ftpc_rcount); 1493 1494 fp->ftp_retired = 1; 1495 fp->ftp_marked = 1; 1496 provid = fp->ftp_provid; 1497 mutex_exit(&fp->ftp_mtx); 1498 1499 /* 1500 * We don't have to worry about invalidating the same provider twice 1501 * since fasttrap_provider_lookup() will ignore provider that have 1502 * been marked as retired. 1503 / 1504* dtrace_invalidate(provid); 1505 1506 mutex_exit(&bucket->ftb_mtx); 1507 1508 fasttrap_pid_cleanup(); 1509} 1510 1511static int 1512fasttrap_uint32_cmp(const void ap, const void bp) 1513{ 1514 return ((const uint32_t )ap - (const uint32_t )bp); 1515} 1516 1517static int 1518fasttrap_uint64_cmp(const void ap, const void bp) 1519{ 1520 return ((const uint64_t )ap - (const uint64_t )bp); 1521} 1522 1523static int 1524fasttrap_add_probe(fasttrap_probe_spec_t pdata) 1525{ 1526* fasttrap_provider_t provider; 1527* fasttrap_probe_t pp; 1528* fasttrap_tracepoint_t tp; 1529* char name; 1530* int i, aframes, whack; 1531 1532 /* 1533 * There needs to be at least one desired trace point. 1534 / 1535* if (pdata->ftps_noffs == 0) 1536 return (EINVAL); 1537 1538 switch (pdata->ftps_type) { 1539 case DTFTP_ENTRY: 1540 name = "entry"; 1541 aframes = FASTTRAP_ENTRY_AFRAMES; 1542 break; 1543 case DTFTP_RETURN: 1544 name = "return"; 1545 aframes = FASTTRAP_RETURN_AFRAMES; 1546 break; 1547 case DTFTP_OFFSETS: 1548 name = NULL; 1549 break; 1550 default: 1551 return (EINVAL); 1552 } 1553 1554 if ((provider = fasttrap_provider_lookup(pdata->ftps_pid, 1555 FASTTRAP_PID_NAME, &pid_attr)) == NULL) 1556 return (ESRCH); 1557 1558 /* 1559 * Increment this reference count to indicate that a consumer is 1560 * actively adding a new probe associated with this provider. This 1561 * prevents the provider from being deleted -- we'll need to check 1562 * for pending deletions when we drop this reference count. 1563 / 1564* provider->ftp_ccount++; 1565 mutex_exit(&provider->ftp_mtx); 1566 1567 /* 1568 * Grab the creation lock to ensure consistency between calls to 1569 * dtrace_probe_lookup() and dtrace_probe_create() in the face of 1570 * other threads creating probes. We must drop the provider lock 1571 * before taking this lock to avoid a three-way deadlock with the 1572 * DTrace framework. 1573 / 1574* mutex_enter(&provider->ftp_cmtx); 1575 1576 if (name == NULL) { 1577 for (i = 0; i < pdata->ftps_noffs; i++) { 1578 char name_str[17]; 1579 1580 (void) sprintf(name_str, "%llx", 1581 (unsigned long long)pdata->ftps_offs[i]); 1582 1583 if (dtrace_probe_lookup(provider->ftp_provid, 1584 pdata->ftps_mod, pdata->ftps_func, name_str) != 0) 1585 continue; 1586 1587 atomic_add_32(&fasttrap_total, 1); 1588 1589 if (fasttrap_total > fasttrap_max) { 1590 atomic_add_32(&fasttrap_total, -1); 1591 goto no_mem; 1592 } 1593 1594 pp = kmem_zalloc(sizeof (fasttrap_probe_t), KM_SLEEP); 1595 1596 pp->ftp_prov = provider; 1597 pp->ftp_faddr = pdata->ftps_pc; 1598 pp->ftp_fsize = pdata->ftps_size; 1599 pp->ftp_pid = pdata->ftps_pid; 1600 pp->ftp_ntps = 1; 1601 1602 tp = kmem_zalloc(sizeof (fasttrap_tracepoint_t), 1603 KM_SLEEP); 1604 1605 tp->ftt_proc = provider->ftp_proc; 1606 tp->ftt_pc = pdata->ftps_offs[i] + pdata->ftps_pc; 1607 tp->ftt_pid = pdata->ftps_pid; 1608 1609 pp->ftp_tps[0].fit_tp = tp; 1610 pp->ftp_tps[0].fit_id.fti_probe = pp; 1611 pp->ftp_tps[0].fit_id.fti_ptype = pdata->ftps_type; 1612 1613 pp->ftp_id = dtrace_probe_create(provider->ftp_provid, 1614 pdata->ftps_mod, pdata->ftps_func, name_str, 1615 FASTTRAP_OFFSET_AFRAMES, pp); 1616 } 1617 1618 } else if (dtrace_probe_lookup(provider->ftp_provid, pdata->ftps_mod, 1619 pdata->ftps_func, name) == 0) { 1620 atomic_add_32(&fasttrap_total, pdata->ftps_noffs); 1621 1622 if (fasttrap_total > fasttrap_max) { 1623 atomic_add_32(&fasttrap_total, -pdata->ftps_noffs); 1624 goto no_mem; 1625 } 1626 1627 /* 1628 * Make sure all tracepoint program counter values are unique. 1629 * We later assume that each probe has exactly one tracepoint 1630 * for a given pc. 1631 / 1632* qsort(pdata->ftps_offs, pdata->ftps_noffs, 1633 sizeof (uint64_t), fasttrap_uint64_cmp); 1634 for (i = 1; i < pdata->ftps_noffs; i++) { 1635 if (pdata->ftps_offs[i] > pdata->ftps_offs[i - 1]) 1636 continue; 1637 1638 atomic_add_32(&fasttrap_total, -pdata->ftps_noffs); 1639 goto no_mem; 1640 } 1641 1642 ASSERT(pdata->ftps_noffs > 0); 1643 pp = kmem_zalloc(offsetof(fasttrap_probe_t, 1644 ftp_tps[pdata->ftps_noffs]), KM_SLEEP); 1645 1646 pp->ftp_prov = provider; 1647 pp->ftp_faddr = pdata->ftps_pc; 1648 pp->ftp_fsize = pdata->ftps_size; 1649 pp->ftp_pid = pdata->ftps_pid; 1650 pp->ftp_ntps = pdata->ftps_noffs; 1651 1652 for (i = 0; i < pdata->ftps_noffs; i++) { 1653 tp = kmem_zalloc(sizeof (fasttrap_tracepoint_t), 1654 KM_SLEEP); 1655 1656 tp->ftt_proc = provider->ftp_proc; 1657 tp->ftt_pc = pdata->ftps_offs[i] + pdata->ftps_pc; 1658 tp->ftt_pid = pdata->ftps_pid; 1659 1660 pp->ftp_tps[i].fit_tp = tp; 1661 pp->ftp_tps[i].fit_id.fti_probe = pp; 1662 pp->ftp_tps[i].fit_id.fti_ptype = pdata->ftps_type; 1663 } 1664 1665 pp->ftp_id = dtrace_probe_create(provider->ftp_provid, 1666 pdata->ftps_mod, pdata->ftps_func, name, aframes, pp); 1667 } 1668 1669 mutex_exit(&provider->ftp_cmtx); 1670 1671 /* 1672 * We know that the provider is still valid since we incremented the 1673 * creation reference count. If someone tried to clean up this provider 1674 * while we were using it (e.g. because the process called exec(2) or 1675 * exit(2)), take note of that and try to clean it up now. 1676 / 1677* mutex_enter(&provider->ftp_mtx); 1678 provider->ftp_ccount--; 1679 whack = provider->ftp_retired; 1680 mutex_exit(&provider->ftp_mtx); 1681 1682 if (whack) 1683 fasttrap_pid_cleanup(); 1684 1685 return (0); 1686 1687no_mem: 1688 /* 1689 * If we've exhausted the allowable resources, we'll try to remove 1690 * this provider to free some up. This is to cover the case where 1691 * the user has accidentally created many more probes than was 1692 * intended (e.g. pid123:::). 1693 / 1694* mutex_exit(&provider->ftp_cmtx); 1695 mutex_enter(&provider->ftp_mtx); 1696 provider->ftp_ccount--; 1697 provider->ftp_marked = 1; 1698 mutex_exit(&provider->ftp_mtx); 1699 1700 fasttrap_pid_cleanup(); 1701 1702 return (ENOMEM); 1703} 1704 1705/ARGSUSED/ 1706static void * 1707fasttrap_meta_provide(void arg, dtrace_helper_provdesc_t dhpv, pid_t pid) 1708{ 1709 fasttrap_provider_t provider; 1710* 1711 /* 1712 * A 32-bit unsigned integer (like a pid for example) can be 1713 * expressed in 10 or fewer decimal digits. Make sure that we'll 1714 * have enough space for the provider name. 1715 / 1716* if (strlen(dhpv->dthpv_provname) + 10 >= 1717 sizeof (provider->ftp_name)) { 1718 cmn_err(CE_WARN, "failed to instantiate provider %s: " 1719 "name too long to accomodate pid", dhpv->dthpv_provname); 1720 return (NULL); 1721 } 1722 1723 /* 1724 * Don't let folks spoof the true pid provider. 1725 / 1726* if (strcmp(dhpv->dthpv_provname, FASTTRAP_PID_NAME) == 0) { 1727 cmn_err(CE_WARN, "failed to instantiate provider %s: " 1728 "%s is an invalid name", dhpv->dthpv_provname, 1729 FASTTRAP_PID_NAME); 1730 return (NULL); 1731 } 1732 1733 /* 1734 * The highest stability class that fasttrap supports is ISA; cap 1735 * the stability of the new provider accordingly. 1736 / 1737* if (dhpv->dthpv_pattr.dtpa_provider.dtat_class > DTRACE_CLASS_ISA) 1738 dhpv->dthpv_pattr.dtpa_provider.dtat_class = DTRACE_CLASS_ISA; 1739 if (dhpv->dthpv_pattr.dtpa_mod.dtat_class > DTRACE_CLASS_ISA) 1740 dhpv->dthpv_pattr.dtpa_mod.dtat_class = DTRACE_CLASS_ISA; 1741 if (dhpv->dthpv_pattr.dtpa_func.dtat_class > DTRACE_CLASS_ISA) 1742 dhpv->dthpv_pattr.dtpa_func.dtat_class = DTRACE_CLASS_ISA; 1743 if (dhpv->dthpv_pattr.dtpa_name.dtat_class > DTRACE_CLASS_ISA) 1744 dhpv->dthpv_pattr.dtpa_name.dtat_class = DTRACE_CLASS_ISA; 1745 if (dhpv->dthpv_pattr.dtpa_args.dtat_class > DTRACE_CLASS_ISA) 1746 dhpv->dthpv_pattr.dtpa_args.dtat_class = DTRACE_CLASS_ISA; 1747 1748 if ((provider = fasttrap_provider_lookup(pid, dhpv->dthpv_provname, 1749 &dhpv->dthpv_pattr)) == NULL) { 1750 cmn_err(CE_WARN, "failed to instantiate provider %s for " 1751 "process %u", dhpv->dthpv_provname, (uint_t)pid); 1752 return (NULL); 1753 } 1754 1755 /* 1756 * Up the meta provider count so this provider isn't removed until 1757 * the meta provider has been told to remove it. 1758 / 1759* provider->ftp_mcount++; 1760 1761 mutex_exit(&provider->ftp_mtx); 1762 1763 return (provider); 1764} 1765 1766/ARGSUSED/ 1767static void 1768fasttrap_meta_create_probe(void arg, void parg, 1769 dtrace_helper_probedesc_t dhpb) 1770{ 1771* fasttrap_provider_t provider = parg; 1772* fasttrap_probe_t pp; 1773* fasttrap_tracepoint_t tp; 1774* int i, j; 1775 uint32_t ntps; 1776 1777 /* 1778 * Since the meta provider count is non-zero we don't have to worry 1779 * about this provider disappearing. 1780 / 1781* ASSERT(provider->ftp_mcount > 0); 1782 1783 /* 1784 * The offsets must be unique. 1785 / 1786* qsort(dhpb->dthpb_offs, dhpb->dthpb_noffs, sizeof (uint32_t), 1787 fasttrap_uint32_cmp); 1788 for (i = 1; i < dhpb->dthpb_noffs; i++) { 1789 if (dhpb->dthpb_base + dhpb->dthpb_offs[i] <= 1790 dhpb->dthpb_base + dhpb->dthpb_offs[i - 1]) 1791 return; 1792 } 1793 1794 qsort(dhpb->dthpb_enoffs, dhpb->dthpb_nenoffs, sizeof (uint32_t), 1795 fasttrap_uint32_cmp); 1796 for (i = 1; i < dhpb->dthpb_nenoffs; i++) { 1797 if (dhpb->dthpb_base + dhpb->dthpb_enoffs[i] <= 1798 dhpb->dthpb_base + dhpb->dthpb_enoffs[i - 1]) 1799 return; 1800 } 1801 1802 /* 1803 * Grab the creation lock to ensure consistency between calls to 1804 * dtrace_probe_lookup() and dtrace_probe_create() in the face of 1805 * other threads creating probes. 1806 / 1807* mutex_enter(&provider->ftp_cmtx); 1808 1809 if (dtrace_probe_lookup(provider->ftp_provid, dhpb->dthpb_mod, 1810 dhpb->dthpb_func, dhpb->dthpb_name) != 0) { 1811 mutex_exit(&provider->ftp_cmtx); 1812 return; 1813 } 1814 1815 ntps = dhpb->dthpb_noffs + dhpb->dthpb_nenoffs; 1816 ASSERT(ntps > 0); 1817 1818 atomic_add_32(&fasttrap_total, ntps); 1819 1820 if (fasttrap_total > fasttrap_max) { 1821 atomic_add_32(&fasttrap_total, -ntps); 1822 mutex_exit(&provider->ftp_cmtx); 1823 return; 1824 } 1825 1826 pp = kmem_zalloc(offsetof(fasttrap_probe_t, ftp_tps[ntps]), KM_SLEEP); 1827 1828 pp->ftp_prov = provider; 1829 pp->ftp_pid = provider->ftp_pid; 1830 pp->ftp_ntps = ntps; 1831 pp->ftp_nargs = dhpb->dthpb_xargc; 1832 pp->ftp_xtypes = dhpb->dthpb_xtypes; 1833 pp->ftp_ntypes = dhpb->dthpb_ntypes; 1834 1835 /* 1836 * First create a tracepoint for each actual point of interest. 1837 / 1838* for (i = 0; i < dhpb->dthpb_noffs; i++) { 1839 tp = kmem_zalloc(sizeof (fasttrap_tracepoint_t), KM_SLEEP); 1840 1841 tp->ftt_proc = provider->ftp_proc; 1842 tp->ftt_pc = dhpb->dthpb_base + dhpb->dthpb_offs[i]; 1843 tp->ftt_pid = provider->ftp_pid; 1844 1845 pp->ftp_tps[i].fit_tp = tp; 1846 pp->ftp_tps[i].fit_id.fti_probe = pp; 1847#ifdef __sparc 1848 pp->ftp_tps[i].fit_id.fti_ptype = DTFTP_POST_OFFSETS; 1849#else 1850 pp->ftp_tps[i].fit_id.fti_ptype = DTFTP_OFFSETS; 1851#endif 1852 } 1853 1854 /* 1855 * Then create a tracepoint for each is-enabled point. 1856 / 1857* for (j = 0; i < ntps; i++, j++) { 1858 tp = kmem_zalloc(sizeof (fasttrap_tracepoint_t), KM_SLEEP); 1859 1860 tp->ftt_proc = provider->ftp_proc; 1861 tp->ftt_pc = dhpb->dthpb_base + dhpb->dthpb_enoffs[j]; 1862 tp->ftt_pid = provider->ftp_pid; 1863 1864 pp->ftp_tps[i].fit_tp = tp; 1865 pp->ftp_tps[i].fit_id.fti_probe = pp; 1866 pp->ftp_tps[i].fit_id.fti_ptype = DTFTP_IS_ENABLED; 1867 } 1868 1869 /* 1870 * If the arguments are shuffled around we set the argument remapping 1871 * table. Later, when the probe fires, we only remap the arguments 1872 * if the table is non-NULL. 1873 / 1874* for (i = 0; i < dhpb->dthpb_xargc; i++) { 1875 if (dhpb->dthpb_args[i] != i) { 1876 pp->ftp_argmap = dhpb->dthpb_args; 1877 break; 1878 } 1879 } 1880 1881 /* 1882 * The probe is fully constructed -- register it with DTrace. 1883 / 1884* pp->ftp_id = dtrace_probe_create(provider->ftp_provid, dhpb->dthpb_mod, 1885 dhpb->dthpb_func, dhpb->dthpb_name, FASTTRAP_OFFSET_AFRAMES, pp); 1886 1887 mutex_exit(&provider->ftp_cmtx); 1888} 1889 1890/ARGSUSED/ 1891static void 1892fasttrap_meta_remove(void arg, dtrace_helper_provdesc_t dhpv, pid_t pid) 1893{ 1894 /* 1895 * Clean up the USDT provider. There may be active consumers of the 1896 * provider busy adding probes, no damage will actually befall the 1897 * provider until that count has dropped to zero. This just puts 1898 * the provider on death row. 1899 / 1900* fasttrap_provider_retire(pid, dhpv->dthpv_provname, 1); 1901} 1902 1903static dtrace_mops_t fasttrap_mops = { 1904 fasttrap_meta_create_probe, 1905 fasttrap_meta_provide, 1906 fasttrap_meta_remove 1907}; 1908 1909/ARGSUSED/ 1910static int 1911fasttrap_open(dev_t devp, int flag, int otyp, cred_t cred_p) 1912{ 1913 return (0); 1914} 1915 1916/ARGSUSED/ 1917static int 1918fasttrap_ioctl(dev_t dev, int cmd, intptr_t arg, int md, cred_t cr, int rv) 1919{ 1920 if (!dtrace_attached()) 1921 return (EAGAIN); 1922 1923 if (cmd == FASTTRAPIOC_MAKEPROBE) { 1924 fasttrap_probe_spec_t uprobe = (void )arg; 1925 fasttrap_probe_spec_t probe; 1926* uint64_t noffs; 1927 size_t size; 1928 int ret; 1929 char c; 1930* 1931 if (copyin(&uprobe->ftps_noffs, &noffs, 1932 sizeof (uprobe->ftps_noffs))) 1933 return (EFAULT); 1934 1935 /* 1936 * Probes must have at least one tracepoint. 1937 / 1938* if (noffs == 0) 1939 return (EINVAL); 1940 1941 size = sizeof (fasttrap_probe_spec_t) + 1942 sizeof (probe->ftps_offs[0]) * (noffs - 1); 1943 1944 if (size > 1024 * 1024) 1945 return (ENOMEM); 1946 1947 probe = kmem_alloc(size, KM_SLEEP); 1948 1949 if (copyin(uprobe, probe, size) != 0) { 1950 kmem_free(probe, size); 1951 return (EFAULT); 1952 } 1953 1954 /* 1955 * Verify that the function and module strings contain no 1956 * funny characters. 1957 / 1958* for (c = &probe->ftps_func[0]; c != '\0'; c++) { 1959* if (c < 0x20 \|\| 0x7f <= c) { 1960 ret = EINVAL; 1961 goto err; 1962 } 1963 } 1964 1965 for (c = &probe->ftps_mod[0]; c != '\0'; c++) { 1966* if (c < 0x20 \|\| 0x7f <= c) { 1967 ret = EINVAL; 1968 goto err; 1969 } 1970 } 1971 1972 if (!PRIV_POLICY_CHOICE(cr, PRIV_ALL, B_FALSE)) { 1973 proc_t p; 1974* pid_t pid = probe->ftps_pid; 1975 1976 mutex_enter(&pidlock); 1977 /* 1978 * Report an error if the process doesn't exist 1979 * or is actively being birthed. 1980 / 1981* if ((p = prfind(pid)) == NULL \|\| p->p_stat == SIDL) { 1982 mutex_exit(&pidlock); 1983 return (ESRCH); 1984 } 1985 mutex_enter(&p->p_lock); 1986 mutex_exit(&pidlock); 1987 1988 if ((ret = priv_proc_cred_perm(cr, p, NULL, 1989 VREAD \| VWRITE)) != 0) { 1990 mutex_exit(&p->p_lock); 1991 return (ret); 1992 } 1993 1994 mutex_exit(&p->p_lock); 1995 } 1996 1997 ret = fasttrap_add_probe(probe); 1998err: 1999 kmem_free(probe, size); 2000 2001 return (ret); 2002 2003 } else if (cmd == FASTTRAPIOC_GETINSTR) { 2004 fasttrap_instr_query_t instr; 2005 fasttrap_tracepoint_t tp; 2006* uint_t index; 2007 int ret; 2008 2009 if (copyin((void )arg, &instr, sizeof (instr)) != 0) 2010* return (EFAULT); 2011 2012 if (!PRIV_POLICY_CHOICE(cr, PRIV_ALL, B_FALSE)) { 2013 proc_t p; 2014* pid_t pid = instr.ftiq_pid; 2015 2016 mutex_enter(&pidlock); 2017 /* 2018 * Report an error if the process doesn't exist 2019 * or is actively being birthed. 2020 / 2021* if ((p = prfind(pid)) == NULL \|\| p->p_stat == SIDL) { 2022 mutex_exit(&pidlock); 2023 return (ESRCH); 2024 } 2025 mutex_enter(&p->p_lock); 2026 mutex_exit(&pidlock); 2027 2028 if ((ret = priv_proc_cred_perm(cr, p, NULL, 2029 VREAD)) != 0) { 2030 mutex_exit(&p->p_lock); 2031 return (ret); 2032 } 2033 2034 mutex_exit(&p->p_lock); 2035 } 2036 2037 index = FASTTRAP_TPOINTS_INDEX(instr.ftiq_pid, instr.ftiq_pc); 2038 2039 mutex_enter(&fasttrap_tpoints.fth_table[index].ftb_mtx); 2040 tp = fasttrap_tpoints.fth_table[index].ftb_data; 2041 while (tp != NULL) { 2042 if (instr.ftiq_pid == tp->ftt_pid && 2043 instr.ftiq_pc == tp->ftt_pc && 2044 tp->ftt_proc->ftpc_acount != 0) 2045 break; 2046
2047 /* 2048 * The count of active providers can only be 2049 * decremented (i.e. to zero) during exec, exit, and 2050 * removal of a meta provider so it should be 2051 * impossible to drop the count during this operation(). 2052 / 2053* ASSERT(tp->ftt_proc->ftpc_acount != 0);
2054 tp = tp->ftt_next; 2055 } 2056 2057 if (tp == NULL) { 2058 mutex_exit(&fasttrap_tpoints.fth_table[index].ftb_mtx); 2059 return (ENOENT); 2060 } 2061 2062 bcopy(&tp->ftt_instr, &instr.ftiq_instr, 2063 sizeof (instr.ftiq_instr)); 2064 mutex_exit(&fasttrap_tpoints.fth_table[index].ftb_mtx); 2065 2066 if (copyout(&instr, (void )arg, sizeof (instr)) != 0) 2067* return (EFAULT); 2068 2069 return (0); 2070 } 2071 2072 return (EINVAL); 2073} 2074 2075static struct cb_ops fasttrap_cb_ops = { 2076 fasttrap_open, /* open / 2077* nodev, /* close / 2078* nulldev, /* strategy / 2079* nulldev, /* print / 2080* nodev, /* dump / 2081* nodev, /* read / 2082* nodev, /* write / 2083* fasttrap_ioctl, /* ioctl / 2084* nodev, /* devmap / 2085* nodev, /* mmap / 2086* nodev, /* segmap / 2087* nochpoll, /* poll / 2088* ddi_prop_op, /* cb_prop_op / 2089* 0, /* streamtab / 2090* D_NEW \| D_MP /* Driver compatibility flag / 2091}; 2092* 2093/ARGSUSED/ 2094static int 2095fasttrap_info(dev_info_t dip, ddi_info_cmd_t infocmd, void arg, void *result) 2096{ 2097* int error; 2098 2099 switch (infocmd) { 2100 case DDI_INFO_DEVT2DEVINFO: 2101 result = (void )fasttrap_devi; 2102 error = DDI_SUCCESS; 2103 break; 2104 case DDI_INFO_DEVT2INSTANCE: 2105 result = (void )0; 2106 error = DDI_SUCCESS; 2107 break; 2108 default: 2109 error = DDI_FAILURE; 2110 } 2111 return (error); 2112} 2113 2114static int 2115fasttrap_attach(dev_info_t devi, ddi_attach_cmd_t cmd) 2116{ 2117* ulong_t nent; 2118 2119 switch (cmd) { 2120 case DDI_ATTACH: 2121 break; 2122 case DDI_RESUME: 2123 return (DDI_SUCCESS); 2124 default: 2125 return (DDI_FAILURE); 2126 } 2127 2128 if (ddi_create_minor_node(devi, "fasttrap", S_IFCHR, 0, 2129 DDI_PSEUDO, NULL) == DDI_FAILURE) { 2130 ddi_remove_minor_node(devi, NULL); 2131 return (DDI_FAILURE); 2132 } 2133 2134 ddi_report_dev(devi); 2135 fasttrap_devi = devi; 2136 2137 /* 2138 * Install our hooks into fork(2), exec(2), and exit(2). 2139 / 2140* dtrace_fasttrap_fork_ptr = &fasttrap_fork; 2141 dtrace_fasttrap_exit_ptr = &fasttrap_exec_exit; 2142 dtrace_fasttrap_exec_ptr = &fasttrap_exec_exit; 2143 2144 fasttrap_max = ddi_getprop(DDI_DEV_T_ANY, devi, DDI_PROP_DONTPASS, 2145 "fasttrap-max-probes", FASTTRAP_MAX_DEFAULT); 2146 fasttrap_total = 0; 2147 2148 /* 2149 * Conjure up the tracepoints hashtable... 2150 / 2151* nent = ddi_getprop(DDI_DEV_T_ANY, devi, DDI_PROP_DONTPASS, 2152 "fasttrap-hash-size", FASTTRAP_TPOINTS_DEFAULT_SIZE); 2153 2154 if (nent == 0 \|\| nent > 0x1000000) 2155 nent = FASTTRAP_TPOINTS_DEFAULT_SIZE; 2156 2157 if ((nent & (nent - 1)) == 0) 2158 fasttrap_tpoints.fth_nent = nent; 2159 else 2160 fasttrap_tpoints.fth_nent = 1 << fasttrap_highbit(nent); 2161 ASSERT(fasttrap_tpoints.fth_nent > 0); 2162 fasttrap_tpoints.fth_mask = fasttrap_tpoints.fth_nent - 1; 2163 fasttrap_tpoints.fth_table = kmem_zalloc(fasttrap_tpoints.fth_nent * 2164 sizeof (fasttrap_bucket_t), KM_SLEEP); 2165 2166 /* 2167 * ... and the providers hash table... 2168 / 2169* nent = FASTTRAP_PROVIDERS_DEFAULT_SIZE; 2170 if ((nent & (nent - 1)) == 0) 2171 fasttrap_provs.fth_nent = nent; 2172 else 2173 fasttrap_provs.fth_nent = 1 << fasttrap_highbit(nent); 2174 ASSERT(fasttrap_provs.fth_nent > 0); 2175 fasttrap_provs.fth_mask = fasttrap_provs.fth_nent - 1; 2176 fasttrap_provs.fth_table = kmem_zalloc(fasttrap_provs.fth_nent * 2177 sizeof (fasttrap_bucket_t), KM_SLEEP); 2178 2179 /* 2180 * ... and the procs hash table. 2181 / 2182* nent = FASTTRAP_PROCS_DEFAULT_SIZE; 2183 if ((nent & (nent - 1)) == 0) 2184 fasttrap_procs.fth_nent = nent; 2185 else 2186 fasttrap_procs.fth_nent = 1 << fasttrap_highbit(nent); 2187 ASSERT(fasttrap_procs.fth_nent > 0); 2188 fasttrap_procs.fth_mask = fasttrap_procs.fth_nent - 1; 2189 fasttrap_procs.fth_table = kmem_zalloc(fasttrap_procs.fth_nent * 2190 sizeof (fasttrap_bucket_t), KM_SLEEP); 2191 2192 (void) dtrace_meta_register("fasttrap", &fasttrap_mops, NULL, 2193 &fasttrap_meta_id); 2194 2195 return (DDI_SUCCESS); 2196} 2197 2198static int 2199fasttrap_detach(dev_info_t devi, ddi_detach_cmd_t cmd) 2200{ 2201* int i, fail = 0; 2202 timeout_id_t tmp; 2203 2204 switch (cmd) { 2205 case DDI_DETACH: 2206 break; 2207 case DDI_SUSPEND: 2208 return (DDI_SUCCESS); 2209 default: 2210 return (DDI_FAILURE); 2211 } 2212 2213 /* 2214 * Unregister the meta-provider to make sure no new fasttrap- 2215 * managed providers come along while we're trying to close up 2216 * shop. If we fail to detach, we'll need to re-register as a 2217 * meta-provider. We can fail to unregister as a meta-provider 2218 * if providers we manage still exist. 2219 / 2220* if (fasttrap_meta_id != DTRACE_METAPROVNONE && 2221 dtrace_meta_unregister(fasttrap_meta_id) != 0) 2222 return (DDI_FAILURE); 2223 2224 /* 2225 * Prevent any new timeouts from running by setting fasttrap_timeout 2226 * to a non-zero value, and wait for the current timeout to complete. 2227 / 2228* mutex_enter(&fasttrap_cleanup_mtx); 2229 fasttrap_cleanup_work = 0; 2230 2231 while (fasttrap_timeout != (timeout_id_t)1) { 2232 tmp = fasttrap_timeout; 2233 fasttrap_timeout = (timeout_id_t)1; 2234 2235 if (tmp != 0) { 2236 mutex_exit(&fasttrap_cleanup_mtx); 2237 (void) untimeout(tmp); 2238 mutex_enter(&fasttrap_cleanup_mtx); 2239 } 2240 } 2241 2242 fasttrap_cleanup_work = 0; 2243 mutex_exit(&fasttrap_cleanup_mtx); 2244 2245 /* 2246 * Iterate over all of our providers. If there's still a process 2247 * that corresponds to that pid, fail to detach. 2248 / 2249* for (i = 0; i < fasttrap_provs.fth_nent; i++) { 2250 fasttrap_provider_t *fpp, fp; 2251 fasttrap_bucket_t bucket = &fasttrap_provs.fth_table[i]; 2252* 2253 mutex_enter(&bucket->ftb_mtx); 2254 fpp = (fasttrap_provider_t *)&bucket->ftb_data; 2255* while ((fp = fpp) != NULL) { 2256* /* 2257 * Acquire and release the lock as a simple way of 2258 * waiting for any other consumer to finish with 2259 * this provider. A thread must first acquire the 2260 * bucket lock so there's no chance of another thread 2261 * blocking on the provider's lock. 2262 / 2263* mutex_enter(&fp->ftp_mtx); 2264 mutex_exit(&fp->ftp_mtx); 2265 2266 if (dtrace_unregister(fp->ftp_provid) != 0) { 2267 fail = 1; 2268 fpp = &fp->ftp_next; 2269 } else { 2270 fpp = fp->ftp_next; 2271* fasttrap_provider_free(fp); 2272 } 2273 } 2274 2275 mutex_exit(&bucket->ftb_mtx); 2276 } 2277 2278 if (fail) { 2279 uint_t work; 2280 /* 2281 * If we're failing to detach, we need to unblock timeouts 2282 * and start a new timeout if any work has accumulated while 2283 * we've been unsuccessfully trying to detach. 2284 / 2285* mutex_enter(&fasttrap_cleanup_mtx); 2286 fasttrap_timeout = 0; 2287 work = fasttrap_cleanup_work; 2288 mutex_exit(&fasttrap_cleanup_mtx); 2289 2290 if (work) 2291 fasttrap_pid_cleanup(); 2292 2293 (void) dtrace_meta_register("fasttrap", &fasttrap_mops, NULL, 2294 &fasttrap_meta_id); 2295 2296 return (DDI_FAILURE); 2297 } 2298 2299#ifdef DEBUG 2300 mutex_enter(&fasttrap_count_mtx); 2301 ASSERT(fasttrap_pid_count == 0); 2302 mutex_exit(&fasttrap_count_mtx); 2303#endif 2304 2305 kmem_free(fasttrap_tpoints.fth_table, 2306 fasttrap_tpoints.fth_nent * sizeof (fasttrap_bucket_t)); 2307 fasttrap_tpoints.fth_nent = 0; 2308 2309 kmem_free(fasttrap_provs.fth_table, 2310 fasttrap_provs.fth_nent * sizeof (fasttrap_bucket_t)); 2311 fasttrap_provs.fth_nent = 0; 2312 2313 kmem_free(fasttrap_procs.fth_table, 2314 fasttrap_procs.fth_nent * sizeof (fasttrap_bucket_t)); 2315 fasttrap_procs.fth_nent = 0; 2316 2317 /* 2318 * We know there are no tracepoints in any process anywhere in 2319 * the system so there is no process which has its p_dtrace_count 2320 * greater than zero, therefore we know that no thread can actively 2321 * be executing code in fasttrap_fork(). Similarly for p_dtrace_probes 2322 * and fasttrap_exec() and fasttrap_exit(). 2323 / 2324* ASSERT(dtrace_fasttrap_fork_ptr == &fasttrap_fork); 2325 dtrace_fasttrap_fork_ptr = NULL; 2326 2327 ASSERT(dtrace_fasttrap_exec_ptr == &fasttrap_exec_exit); 2328 dtrace_fasttrap_exec_ptr = NULL; 2329 2330 ASSERT(dtrace_fasttrap_exit_ptr == &fasttrap_exec_exit); 2331 dtrace_fasttrap_exit_ptr = NULL; 2332 2333 ddi_remove_minor_node(devi, NULL); 2334 2335 return (DDI_SUCCESS); 2336} 2337 2338static struct dev_ops fasttrap_ops = { 2339 DEVO_REV, /* devo_rev / 2340* 0, /* refcnt / 2341* fasttrap_info, /* get_dev_info / 2342* nulldev, /* identify / 2343* nulldev, /* probe / 2344* fasttrap_attach, /* attach / 2345* fasttrap_detach, /* detach / 2346* nodev, /* reset / 2347* &fasttrap_cb_ops, /* driver operations / 2348* NULL, /* bus operations / 2349* nodev /* dev power / 2350}; 2351* 2352/* 2353 * Module linkage information for the kernel. 2354 / 2355static struct modldrv modldrv = { 2356* &mod_driverops, /* module type (this is a pseudo driver) / 2357* "Fasttrap Tracing", /* name of module / 2358* &fasttrap_ops, /* driver ops / 2359}; 2360* 2361static struct modlinkage modlinkage = { 2362 MODREV_1, 2363 (void )&modldrv, 2364* NULL 2365}; 2366 2367int 2368_init(void) 2369{ 2370 return (mod_install(&modlinkage)); 2371} 2372 2373int 2374_info(struct modinfo modinfop) 2375{ 2376* return (mod_info(&modlinkage, modinfop)); 2377} 2378 2379int 2380_fini(void) 2381{ 2382 return (mod_remove(&modlinkage)); 2383}	2047 tp = tp->ftt_next; 2048 } 2049 2050 if (tp == NULL) { 2051 mutex_exit(&fasttrap_tpoints.fth_table[index].ftb_mtx); 2052 return (ENOENT); 2053 } 2054 2055 bcopy(&tp->ftt_instr, &instr.ftiq_instr, 2056 sizeof (instr.ftiq_instr)); 2057 mutex_exit(&fasttrap_tpoints.fth_table[index].ftb_mtx); 2058 2059 if (copyout(&instr, (void )arg, sizeof (instr)) != 0) 2060* return (EFAULT); 2061 2062 return (0); 2063 } 2064 2065 return (EINVAL); 2066} 2067 2068static struct cb_ops fasttrap_cb_ops = { 2069 fasttrap_open, /* open / 2070* nodev, /* close / 2071* nulldev, /* strategy / 2072* nulldev, /* print / 2073* nodev, /* dump / 2074* nodev, /* read / 2075* nodev, /* write / 2076* fasttrap_ioctl, /* ioctl / 2077* nodev, /* devmap / 2078* nodev, /* mmap / 2079* nodev, /* segmap / 2080* nochpoll, /* poll / 2081* ddi_prop_op, /* cb_prop_op / 2082* 0, /* streamtab / 2083* D_NEW \| D_MP /* Driver compatibility flag / 2084}; 2085* 2086/ARGSUSED/ 2087static int 2088fasttrap_info(dev_info_t dip, ddi_info_cmd_t infocmd, void arg, void *result) 2089{ 2090* int error; 2091 2092 switch (infocmd) { 2093 case DDI_INFO_DEVT2DEVINFO: 2094 result = (void )fasttrap_devi; 2095 error = DDI_SUCCESS; 2096 break; 2097 case DDI_INFO_DEVT2INSTANCE: 2098 result = (void )0; 2099 error = DDI_SUCCESS; 2100 break; 2101 default: 2102 error = DDI_FAILURE; 2103 } 2104 return (error); 2105} 2106 2107static int 2108fasttrap_attach(dev_info_t devi, ddi_attach_cmd_t cmd) 2109{ 2110* ulong_t nent; 2111 2112 switch (cmd) { 2113 case DDI_ATTACH: 2114 break; 2115 case DDI_RESUME: 2116 return (DDI_SUCCESS); 2117 default: 2118 return (DDI_FAILURE); 2119 } 2120 2121 if (ddi_create_minor_node(devi, "fasttrap", S_IFCHR, 0, 2122 DDI_PSEUDO, NULL) == DDI_FAILURE) { 2123 ddi_remove_minor_node(devi, NULL); 2124 return (DDI_FAILURE); 2125 } 2126 2127 ddi_report_dev(devi); 2128 fasttrap_devi = devi; 2129 2130 /* 2131 * Install our hooks into fork(2), exec(2), and exit(2). 2132 / 2133* dtrace_fasttrap_fork_ptr = &fasttrap_fork; 2134 dtrace_fasttrap_exit_ptr = &fasttrap_exec_exit; 2135 dtrace_fasttrap_exec_ptr = &fasttrap_exec_exit; 2136 2137 fasttrap_max = ddi_getprop(DDI_DEV_T_ANY, devi, DDI_PROP_DONTPASS, 2138 "fasttrap-max-probes", FASTTRAP_MAX_DEFAULT); 2139 fasttrap_total = 0; 2140 2141 /* 2142 * Conjure up the tracepoints hashtable... 2143 / 2144* nent = ddi_getprop(DDI_DEV_T_ANY, devi, DDI_PROP_DONTPASS, 2145 "fasttrap-hash-size", FASTTRAP_TPOINTS_DEFAULT_SIZE); 2146 2147 if (nent == 0 \|\| nent > 0x1000000) 2148 nent = FASTTRAP_TPOINTS_DEFAULT_SIZE; 2149 2150 if ((nent & (nent - 1)) == 0) 2151 fasttrap_tpoints.fth_nent = nent; 2152 else 2153 fasttrap_tpoints.fth_nent = 1 << fasttrap_highbit(nent); 2154 ASSERT(fasttrap_tpoints.fth_nent > 0); 2155 fasttrap_tpoints.fth_mask = fasttrap_tpoints.fth_nent - 1; 2156 fasttrap_tpoints.fth_table = kmem_zalloc(fasttrap_tpoints.fth_nent * 2157 sizeof (fasttrap_bucket_t), KM_SLEEP); 2158 2159 /* 2160 * ... and the providers hash table... 2161 / 2162* nent = FASTTRAP_PROVIDERS_DEFAULT_SIZE; 2163 if ((nent & (nent - 1)) == 0) 2164 fasttrap_provs.fth_nent = nent; 2165 else 2166 fasttrap_provs.fth_nent = 1 << fasttrap_highbit(nent); 2167 ASSERT(fasttrap_provs.fth_nent > 0); 2168 fasttrap_provs.fth_mask = fasttrap_provs.fth_nent - 1; 2169 fasttrap_provs.fth_table = kmem_zalloc(fasttrap_provs.fth_nent * 2170 sizeof (fasttrap_bucket_t), KM_SLEEP); 2171 2172 /* 2173 * ... and the procs hash table. 2174 / 2175* nent = FASTTRAP_PROCS_DEFAULT_SIZE; 2176 if ((nent & (nent - 1)) == 0) 2177 fasttrap_procs.fth_nent = nent; 2178 else 2179 fasttrap_procs.fth_nent = 1 << fasttrap_highbit(nent); 2180 ASSERT(fasttrap_procs.fth_nent > 0); 2181 fasttrap_procs.fth_mask = fasttrap_procs.fth_nent - 1; 2182 fasttrap_procs.fth_table = kmem_zalloc(fasttrap_procs.fth_nent * 2183 sizeof (fasttrap_bucket_t), KM_SLEEP); 2184 2185 (void) dtrace_meta_register("fasttrap", &fasttrap_mops, NULL, 2186 &fasttrap_meta_id); 2187 2188 return (DDI_SUCCESS); 2189} 2190 2191static int 2192fasttrap_detach(dev_info_t devi, ddi_detach_cmd_t cmd) 2193{ 2194* int i, fail = 0; 2195 timeout_id_t tmp; 2196 2197 switch (cmd) { 2198 case DDI_DETACH: 2199 break; 2200 case DDI_SUSPEND: 2201 return (DDI_SUCCESS); 2202 default: 2203 return (DDI_FAILURE); 2204 } 2205 2206 /* 2207 * Unregister the meta-provider to make sure no new fasttrap- 2208 * managed providers come along while we're trying to close up 2209 * shop. If we fail to detach, we'll need to re-register as a 2210 * meta-provider. We can fail to unregister as a meta-provider 2211 * if providers we manage still exist. 2212 / 2213* if (fasttrap_meta_id != DTRACE_METAPROVNONE && 2214 dtrace_meta_unregister(fasttrap_meta_id) != 0) 2215 return (DDI_FAILURE); 2216 2217 /* 2218 * Prevent any new timeouts from running by setting fasttrap_timeout 2219 * to a non-zero value, and wait for the current timeout to complete. 2220 / 2221* mutex_enter(&fasttrap_cleanup_mtx); 2222 fasttrap_cleanup_work = 0; 2223 2224 while (fasttrap_timeout != (timeout_id_t)1) { 2225 tmp = fasttrap_timeout; 2226 fasttrap_timeout = (timeout_id_t)1; 2227 2228 if (tmp != 0) { 2229 mutex_exit(&fasttrap_cleanup_mtx); 2230 (void) untimeout(tmp); 2231 mutex_enter(&fasttrap_cleanup_mtx); 2232 } 2233 } 2234 2235 fasttrap_cleanup_work = 0; 2236 mutex_exit(&fasttrap_cleanup_mtx); 2237 2238 /* 2239 * Iterate over all of our providers. If there's still a process 2240 * that corresponds to that pid, fail to detach. 2241 / 2242* for (i = 0; i < fasttrap_provs.fth_nent; i++) { 2243 fasttrap_provider_t *fpp, fp; 2244 fasttrap_bucket_t bucket = &fasttrap_provs.fth_table[i]; 2245* 2246 mutex_enter(&bucket->ftb_mtx); 2247 fpp = (fasttrap_provider_t *)&bucket->ftb_data; 2248* while ((fp = fpp) != NULL) { 2249* /* 2250 * Acquire and release the lock as a simple way of 2251 * waiting for any other consumer to finish with 2252 * this provider. A thread must first acquire the 2253 * bucket lock so there's no chance of another thread 2254 * blocking on the provider's lock. 2255 / 2256* mutex_enter(&fp->ftp_mtx); 2257 mutex_exit(&fp->ftp_mtx); 2258 2259 if (dtrace_unregister(fp->ftp_provid) != 0) { 2260 fail = 1; 2261 fpp = &fp->ftp_next; 2262 } else { 2263 fpp = fp->ftp_next; 2264* fasttrap_provider_free(fp); 2265 } 2266 } 2267 2268 mutex_exit(&bucket->ftb_mtx); 2269 } 2270 2271 if (fail) { 2272 uint_t work; 2273 /* 2274 * If we're failing to detach, we need to unblock timeouts 2275 * and start a new timeout if any work has accumulated while 2276 * we've been unsuccessfully trying to detach. 2277 / 2278* mutex_enter(&fasttrap_cleanup_mtx); 2279 fasttrap_timeout = 0; 2280 work = fasttrap_cleanup_work; 2281 mutex_exit(&fasttrap_cleanup_mtx); 2282 2283 if (work) 2284 fasttrap_pid_cleanup(); 2285 2286 (void) dtrace_meta_register("fasttrap", &fasttrap_mops, NULL, 2287 &fasttrap_meta_id); 2288 2289 return (DDI_FAILURE); 2290 } 2291 2292#ifdef DEBUG 2293 mutex_enter(&fasttrap_count_mtx); 2294 ASSERT(fasttrap_pid_count == 0); 2295 mutex_exit(&fasttrap_count_mtx); 2296#endif 2297 2298 kmem_free(fasttrap_tpoints.fth_table, 2299 fasttrap_tpoints.fth_nent * sizeof (fasttrap_bucket_t)); 2300 fasttrap_tpoints.fth_nent = 0; 2301 2302 kmem_free(fasttrap_provs.fth_table, 2303 fasttrap_provs.fth_nent * sizeof (fasttrap_bucket_t)); 2304 fasttrap_provs.fth_nent = 0; 2305 2306 kmem_free(fasttrap_procs.fth_table, 2307 fasttrap_procs.fth_nent * sizeof (fasttrap_bucket_t)); 2308 fasttrap_procs.fth_nent = 0; 2309 2310 /* 2311 * We know there are no tracepoints in any process anywhere in 2312 * the system so there is no process which has its p_dtrace_count 2313 * greater than zero, therefore we know that no thread can actively 2314 * be executing code in fasttrap_fork(). Similarly for p_dtrace_probes 2315 * and fasttrap_exec() and fasttrap_exit(). 2316 / 2317* ASSERT(dtrace_fasttrap_fork_ptr == &fasttrap_fork); 2318 dtrace_fasttrap_fork_ptr = NULL; 2319 2320 ASSERT(dtrace_fasttrap_exec_ptr == &fasttrap_exec_exit); 2321 dtrace_fasttrap_exec_ptr = NULL; 2322 2323 ASSERT(dtrace_fasttrap_exit_ptr == &fasttrap_exec_exit); 2324 dtrace_fasttrap_exit_ptr = NULL; 2325 2326 ddi_remove_minor_node(devi, NULL); 2327 2328 return (DDI_SUCCESS); 2329} 2330 2331static struct dev_ops fasttrap_ops = { 2332 DEVO_REV, /* devo_rev / 2333* 0, /* refcnt / 2334* fasttrap_info, /* get_dev_info / 2335* nulldev, /* identify / 2336* nulldev, /* probe / 2337* fasttrap_attach, /* attach / 2338* fasttrap_detach, /* detach / 2339* nodev, /* reset / 2340* &fasttrap_cb_ops, /* driver operations / 2341* NULL, /* bus operations / 2342* nodev /* dev power / 2343}; 2344* 2345/* 2346 * Module linkage information for the kernel. 2347 / 2348static struct modldrv modldrv = { 2349* &mod_driverops, /* module type (this is a pseudo driver) / 2350* "Fasttrap Tracing", /* name of module / 2351* &fasttrap_ops, /* driver ops / 2352}; 2353* 2354static struct modlinkage modlinkage = { 2355 MODREV_1, 2356 (void )&modldrv, 2357* NULL 2358}; 2359 2360int 2361_init(void) 2362{ 2363 return (mod_install(&modlinkage)); 2364} 2365 2366int 2367_info(struct modinfo modinfop) 2368{ 2369* return (mod_info(&modlinkage, modinfop)); 2370} 2371 2372int 2373_fini(void) 2374{ 2375 return (mod_remove(&modlinkage)); 2376}