24 */ 25 26/* 27 * Copyright 2008 Sun Microsystems, Inc. All rights reserved. 28 * Use is subject to license terms. 29 */ 30 31#if defined(sun) 32#pragma ident "%Z%%M% %I% %E% SMI" 33#endif 34 35#include <sys/atomic.h> 36#include <sys/errno.h> 37#include <sys/stat.h> 38#include <sys/modctl.h> 39#include <sys/conf.h> 40#include <sys/systm.h> 41#if defined(sun) 42#include <sys/ddi.h> 43#endif 44#include <sys/sunddi.h> 45#include <sys/cpuvar.h> 46#include <sys/kmem.h> 47#if defined(sun) 48#include <sys/strsubr.h> 49#endif 50#include <sys/fasttrap.h> 51#include <sys/fasttrap_impl.h> 52#include <sys/fasttrap_isa.h> 53#include <sys/dtrace.h> 54#include <sys/dtrace_impl.h> 55#include <sys/sysmacros.h> 56#include <sys/proc.h> 57#include <sys/policy.h> 58#if defined(sun) 59#include <util/qsort.h> 60#endif 61#include <sys/mutex.h> 62#include <sys/kernel.h> 63#if !defined(sun) 64#include <sys/user.h> 65#include <sys/dtrace_bsd.h> 66#include <cddl/dev/dtrace/dtrace_cddl.h> 67#endif 68 69/* 70 * User-Land Trap-Based Tracing 71 * ---------------------------- 72 * 73 * The fasttrap provider allows DTrace consumers to instrument any user-level 74 * instruction to gather data; this includes probes with semantic 75 * signifigance like entry and return as well as simple offsets into the 76 * function. While the specific techniques used are very ISA specific, the 77 * methodology is generalizable to any architecture. 78 * 79 * 80 * The General Methodology 81 * ----------------------- 82 * 83 * With the primary goal of tracing every user-land instruction and the 84 * limitation that we can't trust user space so don't want to rely on much 85 * information there, we begin by replacing the instructions we want to trace 86 * with trap instructions. Each instruction we overwrite is saved into a hash 87 * table keyed by process ID and pc address. When we enter the kernel due to 88 * this trap instruction, we need the effects of the replaced instruction to 89 * appear to have occurred before we proceed with the user thread's 90 * execution. 91 * 92 * Each user level thread is represented by a ulwp_t structure which is 93 * always easily accessible through a register. The most basic way to produce 94 * the effects of the instruction we replaced is to copy that instruction out 95 * to a bit of scratch space reserved in the user thread's ulwp_t structure 96 * (a sort of kernel-private thread local storage), set the PC to that 97 * scratch space and single step. When we reenter the kernel after single 98 * stepping the instruction we must then adjust the PC to point to what would 99 * normally be the next instruction. Of course, special care must be taken 100 * for branches and jumps, but these represent such a small fraction of any 101 * instruction set that writing the code to emulate these in the kernel is 102 * not too difficult. 103 * 104 * Return probes may require several tracepoints to trace every return site, 105 * and, conversely, each tracepoint may activate several probes (the entry 106 * and offset 0 probes, for example). To solve this muliplexing problem, 107 * tracepoints contain lists of probes to activate and probes contain lists 108 * of tracepoints to enable. If a probe is activated, it adds its ID to 109 * existing tracepoints or creates new ones as necessary. 110 * 111 * Most probes are activated _before_ the instruction is executed, but return 112 * probes are activated _after_ the effects of the last instruction of the 113 * function are visible. Return probes must be fired _after_ we have 114 * single-stepped the instruction whereas all other probes are fired 115 * beforehand. 116 * 117 * 118 * Lock Ordering 119 * ------------- 120 * 121 * The lock ordering below -- both internally and with respect to the DTrace 122 * framework -- is a little tricky and bears some explanation. Each provider 123 * has a lock (ftp_mtx) that protects its members including reference counts 124 * for enabled probes (ftp_rcount), consumers actively creating probes 125 * (ftp_ccount) and USDT consumers (ftp_mcount); all three prevent a provider 126 * from being freed. A provider is looked up by taking the bucket lock for the 127 * provider hash table, and is returned with its lock held. The provider lock 128 * may be taken in functions invoked by the DTrace framework, but may not be 129 * held while calling functions in the DTrace framework. 130 * 131 * To ensure consistency over multiple calls to the DTrace framework, the 132 * creation lock (ftp_cmtx) should be held. Naturally, the creation lock may 133 * not be taken when holding the provider lock as that would create a cyclic 134 * lock ordering. In situations where one would naturally take the provider 135 * lock and then the creation lock, we instead up a reference count to prevent 136 * the provider from disappearing, drop the provider lock, and acquire the 137 * creation lock. 138 * 139 * Briefly: 140 * bucket lock before provider lock 141 * DTrace before provider lock 142 * creation lock before DTrace 143 * never hold the provider lock and creation lock simultaneously 144 */ 145 146static d_open_t fasttrap_open; 147static d_ioctl_t fasttrap_ioctl; 148 149static struct cdevsw fasttrap_cdevsw = { 150 .d_version = D_VERSION, 151 .d_open = fasttrap_open, 152 .d_ioctl = fasttrap_ioctl, 153 .d_name = "fasttrap", 154}; 155static struct cdev *fasttrap_cdev; 156static dtrace_meta_provider_id_t fasttrap_meta_id; 157 158static struct proc *fasttrap_cleanup_proc; 159static struct mtx fasttrap_cleanup_mtx; 160static uint_t fasttrap_cleanup_work, fasttrap_cleanup_drain, fasttrap_cleanup_cv; 161 162/* 163 * Generation count on modifications to the global tracepoint lookup table. 164 */ 165static volatile uint64_t fasttrap_mod_gen; 166 167/* 168 * When the fasttrap provider is loaded, fasttrap_max is set to either 169 * FASTTRAP_MAX_DEFAULT or the value for fasttrap-max-probes in the 170 * fasttrap.conf file. Each time a probe is created, fasttrap_total is 171 * incremented by the number of tracepoints that may be associated with that 172 * probe; fasttrap_total is capped at fasttrap_max. 173 */ 174#define FASTTRAP_MAX_DEFAULT 250000 175static uint32_t fasttrap_max; 176static uint32_t fasttrap_total; 177 178/* 179 * Copyright (c) 2011, Joyent, Inc. All rights reserved. 180 */ 181 182#define FASTTRAP_TPOINTS_DEFAULT_SIZE 0x4000 183#define FASTTRAP_PROVIDERS_DEFAULT_SIZE 0x100 184#define FASTTRAP_PROCS_DEFAULT_SIZE 0x100 185 186#define FASTTRAP_PID_NAME "pid" 187 188fasttrap_hash_t fasttrap_tpoints; 189static fasttrap_hash_t fasttrap_provs; 190static fasttrap_hash_t fasttrap_procs; 191 192static uint64_t fasttrap_pid_count; /* pid ref count */ 193static kmutex_t fasttrap_count_mtx; /* lock on ref count */ 194 195#define FASTTRAP_ENABLE_FAIL 1 196#define FASTTRAP_ENABLE_PARTIAL 2 197 198static int fasttrap_tracepoint_enable(proc_t *, fasttrap_probe_t *, uint_t); 199static void fasttrap_tracepoint_disable(proc_t *, fasttrap_probe_t *, uint_t); 200 201static fasttrap_provider_t *fasttrap_provider_lookup(pid_t, const char *, 202 const dtrace_pattr_t *); 203static void fasttrap_provider_retire(pid_t, const char *, int); 204static void fasttrap_provider_free(fasttrap_provider_t *); 205 206static fasttrap_proc_t *fasttrap_proc_lookup(pid_t); 207static void fasttrap_proc_release(fasttrap_proc_t *); 208 209#define FASTTRAP_PROVS_INDEX(pid, name) \ 210 ((fasttrap_hash_str(name) + (pid)) & fasttrap_provs.fth_mask) 211 212#define FASTTRAP_PROCS_INDEX(pid) ((pid) & fasttrap_procs.fth_mask) 213 214#if !defined(sun) 215static kmutex_t fasttrap_cpuc_pid_lock[MAXCPU]; 216#endif 217 218static int 219fasttrap_highbit(ulong_t i) 220{ 221 int h = 1; 222 223 if (i == 0) 224 return (0); 225#ifdef _LP64 226 if (i & 0xffffffff00000000ul) { 227 h += 32; i >>= 32; 228 } 229#endif 230 if (i & 0xffff0000) { 231 h += 16; i >>= 16; 232 } 233 if (i & 0xff00) { 234 h += 8; i >>= 8; 235 } 236 if (i & 0xf0) { 237 h += 4; i >>= 4; 238 } 239 if (i & 0xc) { 240 h += 2; i >>= 2; 241 } 242 if (i & 0x2) { 243 h += 1; 244 } 245 return (h); 246} 247 248static uint_t 249fasttrap_hash_str(const char *p) 250{ 251 unsigned int g; 252 uint_t hval = 0; 253 254 while (*p) { 255 hval = (hval << 4) + *p++; 256 if ((g = (hval & 0xf0000000)) != 0) 257 hval ^= g >> 24; 258 hval &= ~g; 259 } 260 return (hval); 261} 262 263void 264fasttrap_sigtrap(proc_t *p, kthread_t *t, uintptr_t pc) 265{ 266#if defined(sun) 267 sigqueue_t *sqp = kmem_zalloc(sizeof (sigqueue_t), KM_SLEEP); 268 269 sqp->sq_info.si_signo = SIGTRAP; 270 sqp->sq_info.si_code = TRAP_DTRACE; 271 sqp->sq_info.si_addr = (caddr_t)pc; 272 273 mutex_enter(&p->p_lock); 274 sigaddqa(p, t, sqp); 275 mutex_exit(&p->p_lock); 276 277 if (t != NULL) 278 aston(t); 279#else 280 ksiginfo_t *ksi = kmem_zalloc(sizeof (ksiginfo_t), KM_SLEEP); 281 282 ksiginfo_init(ksi); 283 ksi->ksi_signo = SIGTRAP; 284 ksi->ksi_code = TRAP_DTRACE; 285 ksi->ksi_addr = (caddr_t)pc; 286 PROC_LOCK(p); 287 (void) tdksignal(t, SIGTRAP, ksi); 288 PROC_UNLOCK(p); 289#endif 290} 291 292/* 293 * This function ensures that no threads are actively using the memory 294 * associated with probes that were formerly live. 295 */ 296static void 297fasttrap_mod_barrier(uint64_t gen) 298{ 299 int i; 300 301 if (gen < fasttrap_mod_gen) 302 return; 303 304 fasttrap_mod_gen++; 305 306 CPU_FOREACH(i) { 307 mutex_enter(&fasttrap_cpuc_pid_lock[i]); 308 mutex_exit(&fasttrap_cpuc_pid_lock[i]); 309 } 310} 311 312/* 313 * This function performs asynchronous cleanup of fasttrap providers. The 314 * Solaris implementation of this mechanism use a timeout that's activated in 315 * fasttrap_pid_cleanup(), but this doesn't work in FreeBSD: one may sleep while 316 * holding the DTrace mutexes, but it is unsafe to sleep in a callout handler. 317 * Thus we use a dedicated process to perform the cleanup when requested. 318 */ 319/*ARGSUSED*/ 320static void 321fasttrap_pid_cleanup_cb(void *data) 322{ 323 fasttrap_provider_t **fpp, *fp; 324 fasttrap_bucket_t *bucket; 325 dtrace_provider_id_t provid; 326 int i, later = 0, rval; 327 328 mtx_lock(&fasttrap_cleanup_mtx); 329 while (!fasttrap_cleanup_drain || later > 0) { 330 fasttrap_cleanup_work = 0; 331 mtx_unlock(&fasttrap_cleanup_mtx); 332 333 later = 0; 334 335 /* 336 * Iterate over all the providers trying to remove the marked 337 * ones. If a provider is marked but not retired, we just 338 * have to take a crack at removing it -- it's no big deal if 339 * we can't. 340 */ 341 for (i = 0; i < fasttrap_provs.fth_nent; i++) { 342 bucket = &fasttrap_provs.fth_table[i]; 343 mutex_enter(&bucket->ftb_mtx); 344 fpp = (fasttrap_provider_t **)&bucket->ftb_data; 345 346 while ((fp = *fpp) != NULL) { 347 if (!fp->ftp_marked) { 348 fpp = &fp->ftp_next; 349 continue; 350 } 351 352 mutex_enter(&fp->ftp_mtx); 353 354 /* 355 * If this provider has consumers actively 356 * creating probes (ftp_ccount) or is a USDT 357 * provider (ftp_mcount), we can't unregister 358 * or even condense. 359 */ 360 if (fp->ftp_ccount != 0 || 361 fp->ftp_mcount != 0) { 362 mutex_exit(&fp->ftp_mtx); 363 fp->ftp_marked = 0; 364 continue; 365 } 366 367 if (!fp->ftp_retired || fp->ftp_rcount != 0) 368 fp->ftp_marked = 0; 369 370 mutex_exit(&fp->ftp_mtx); 371 372 /* 373 * If we successfully unregister this 374 * provider we can remove it from the hash 375 * chain and free the memory. If our attempt 376 * to unregister fails and this is a retired 377 * provider, increment our flag to try again 378 * pretty soon. If we've consumed more than 379 * half of our total permitted number of 380 * probes call dtrace_condense() to try to 381 * clean out the unenabled probes. 382 */ 383 provid = fp->ftp_provid; 384 if ((rval = dtrace_unregister(provid)) != 0) { 385 if (fasttrap_total > fasttrap_max / 2) 386 (void) dtrace_condense(provid); 387 388 if (rval == EAGAIN) 389 fp->ftp_marked = 1; 390 391 later += fp->ftp_marked; 392 fpp = &fp->ftp_next; 393 } else { 394 *fpp = fp->ftp_next; 395 fasttrap_provider_free(fp); 396 } 397 } 398 mutex_exit(&bucket->ftb_mtx); 399 } 400 mtx_lock(&fasttrap_cleanup_mtx); 401 402 /* 403 * If we were unable to retire a provider, try again after a 404 * second. This situation can occur in certain circumstances 405 * where providers cannot be unregistered even though they have 406 * no probes enabled because of an execution of dtrace -l or 407 * something similar. 408 */ 409 if (later > 0 || fasttrap_cleanup_work || 410 fasttrap_cleanup_drain) { 411 mtx_unlock(&fasttrap_cleanup_mtx); 412 pause("ftclean", hz); 413 mtx_lock(&fasttrap_cleanup_mtx); 414 } else 415 mtx_sleep(&fasttrap_cleanup_cv, &fasttrap_cleanup_mtx, 416 0, "ftcl", 0); 417 } 418 419 /* 420 * Wake up the thread in fasttrap_unload() now that we're done. 421 */ 422 wakeup(&fasttrap_cleanup_drain); 423 mtx_unlock(&fasttrap_cleanup_mtx); 424 425 kthread_exit(); 426} 427 428/* 429 * Activates the asynchronous cleanup mechanism. 430 */ 431static void 432fasttrap_pid_cleanup(void) 433{ 434 435 mtx_lock(&fasttrap_cleanup_mtx); 436 if (!fasttrap_cleanup_work) { 437 fasttrap_cleanup_work = 1; 438 wakeup(&fasttrap_cleanup_cv); 439 } 440 mtx_unlock(&fasttrap_cleanup_mtx); 441} 442 443/* 444 * This is called from cfork() via dtrace_fasttrap_fork(). The child 445 * process's address space is (roughly) a copy of the parent process's so 446 * we have to remove all the instrumentation we had previously enabled in the 447 * parent. 448 */ 449static void 450fasttrap_fork(proc_t *p, proc_t *cp) 451{ 452 pid_t ppid = p->p_pid; 453 int i; 454 455#if defined(sun) 456 ASSERT(curproc == p); 457 ASSERT(p->p_proc_flag & P_PR_LOCK); 458#else 459 PROC_LOCK_ASSERT(p, MA_OWNED); 460#endif 461#if defined(sun) 462 ASSERT(p->p_dtrace_count > 0); 463#else 464 if (p->p_dtrace_helpers) { 465 /* 466 * dtrace_helpers_duplicate() allocates memory. 467 */ 468 _PHOLD(cp); 469 PROC_UNLOCK(p); 470 PROC_UNLOCK(cp); 471 dtrace_helpers_duplicate(p, cp); 472 PROC_LOCK(cp); 473 PROC_LOCK(p); 474 _PRELE(cp); 475 } 476 /* 477 * This check is purposely here instead of in kern_fork.c because, 478 * for legal resons, we cannot include the dtrace_cddl.h header 479 * inside kern_fork.c and insert if-clause there. 480 */ 481 if (p->p_dtrace_count == 0) 482 return; 483#endif 484 ASSERT(cp->p_dtrace_count == 0); 485 486 /* 487 * This would be simpler and faster if we maintained per-process 488 * hash tables of enabled tracepoints. It could, however, potentially 489 * slow down execution of a tracepoint since we'd need to go 490 * through two levels of indirection. In the future, we should 491 * consider either maintaining per-process ancillary lists of 492 * enabled tracepoints or hanging a pointer to a per-process hash 493 * table of enabled tracepoints off the proc structure. 494 */ 495 496 /* 497 * We don't have to worry about the child process disappearing 498 * because we're in fork(). 499 */ 500#if defined(sun) 501 mtx_lock_spin(&cp->p_slock); 502 sprlock_proc(cp); 503 mtx_unlock_spin(&cp->p_slock); 504#else 505 _PHOLD(cp); 506#endif 507 508 /* 509 * Iterate over every tracepoint looking for ones that belong to the 510 * parent process, and remove each from the child process. 511 */ 512 for (i = 0; i < fasttrap_tpoints.fth_nent; i++) { 513 fasttrap_tracepoint_t *tp; 514 fasttrap_bucket_t *bucket = &fasttrap_tpoints.fth_table[i]; 515 516 mutex_enter(&bucket->ftb_mtx); 517 for (tp = bucket->ftb_data; tp != NULL; tp = tp->ftt_next) { 518 if (tp->ftt_pid == ppid && 519 tp->ftt_proc->ftpc_acount != 0) { 520 int ret = fasttrap_tracepoint_remove(cp, tp); 521 ASSERT(ret == 0); 522 523 /* 524 * The count of active providers can only be 525 * decremented (i.e. to zero) during exec, 526 * exit, and removal of a meta provider so it 527 * should be impossible to drop the count 528 * mid-fork. 529 */ 530 ASSERT(tp->ftt_proc->ftpc_acount != 0); 531 } 532 } 533 mutex_exit(&bucket->ftb_mtx); 534 } 535 536#if defined(sun) 537 mutex_enter(&cp->p_lock); 538 sprunlock(cp); 539#else 540 _PRELE(cp); 541#endif 542} 543 544/* 545 * This is called from proc_exit() or from exec_common() if p_dtrace_probes 546 * is set on the proc structure to indicate that there is a pid provider 547 * associated with this process. 548 */ 549static void 550fasttrap_exec_exit(proc_t *p) 551{ 552#if defined(sun) 553 ASSERT(p == curproc); 554#endif 555 PROC_LOCK_ASSERT(p, MA_OWNED); 556 _PHOLD(p); 557 PROC_UNLOCK(p); 558 559 /* 560 * We clean up the pid provider for this process here; user-land 561 * static probes are handled by the meta-provider remove entry point. 562 */ 563 fasttrap_provider_retire(p->p_pid, FASTTRAP_PID_NAME, 0); 564#if !defined(sun) 565 if (p->p_dtrace_helpers) 566 dtrace_helpers_destroy(p); 567#endif 568 PROC_LOCK(p); 569 _PRELE(p); 570} 571 572 573/*ARGSUSED*/ 574static void 575fasttrap_pid_provide(void *arg, dtrace_probedesc_t *desc) 576{ 577 /* 578 * There are no "default" pid probes. 579 */ 580} 581 582static int 583fasttrap_tracepoint_enable(proc_t *p, fasttrap_probe_t *probe, uint_t index) 584{ 585 fasttrap_tracepoint_t *tp, *new_tp = NULL; 586 fasttrap_bucket_t *bucket; 587 fasttrap_id_t *id; 588 pid_t pid; 589 uintptr_t pc; 590 591 ASSERT(index < probe->ftp_ntps); 592 593 pid = probe->ftp_pid; 594 pc = probe->ftp_tps[index].fit_tp->ftt_pc; 595 id = &probe->ftp_tps[index].fit_id; 596 597 ASSERT(probe->ftp_tps[index].fit_tp->ftt_pid == pid); 598 599#if defined(sun) 600 ASSERT(!(p->p_flag & SVFORK)); 601#endif 602 603 /* 604 * Before we make any modifications, make sure we've imposed a barrier 605 * on the generation in which this probe was last modified. 606 */ 607 fasttrap_mod_barrier(probe->ftp_gen); 608 609 bucket = &fasttrap_tpoints.fth_table[FASTTRAP_TPOINTS_INDEX(pid, pc)]; 610 611 /* 612 * If the tracepoint has already been enabled, just add our id to the 613 * list of interested probes. This may be our second time through 614 * this path in which case we'll have constructed the tracepoint we'd 615 * like to install. If we can't find a match, and have an allocated 616 * tracepoint ready to go, enable that one now. 617 * 618 * A tracepoint whose process is defunct is also considered defunct. 619 */ 620again: 621 mutex_enter(&bucket->ftb_mtx); 622 for (tp = bucket->ftb_data; tp != NULL; tp = tp->ftt_next) { 623 /* 624 * Note that it's safe to access the active count on the 625 * associated proc structure because we know that at least one 626 * provider (this one) will still be around throughout this 627 * operation. 628 */ 629 if (tp->ftt_pid != pid || tp->ftt_pc != pc || 630 tp->ftt_proc->ftpc_acount == 0) 631 continue; 632 633 /* 634 * Now that we've found a matching tracepoint, it would be 635 * a decent idea to confirm that the tracepoint is still 636 * enabled and the trap instruction hasn't been overwritten. 637 * Since this is a little hairy, we'll punt for now. 638 */ 639 640 /* 641 * This can't be the first interested probe. We don't have 642 * to worry about another thread being in the midst of 643 * deleting this tracepoint (which would be the only valid 644 * reason for a tracepoint to have no interested probes) 645 * since we're holding P_PR_LOCK for this process. 646 */ 647 ASSERT(tp->ftt_ids != NULL || tp->ftt_retids != NULL); 648 649 switch (id->fti_ptype) { 650 case DTFTP_ENTRY: 651 case DTFTP_OFFSETS: 652 case DTFTP_IS_ENABLED: 653 id->fti_next = tp->ftt_ids; 654 membar_producer(); 655 tp->ftt_ids = id; 656 membar_producer(); 657 break; 658 659 case DTFTP_RETURN: 660 case DTFTP_POST_OFFSETS: 661 id->fti_next = tp->ftt_retids; 662 membar_producer(); 663 tp->ftt_retids = id; 664 membar_producer(); 665 break; 666 667 default: 668 ASSERT(0); 669 } 670 671 mutex_exit(&bucket->ftb_mtx); 672 673 if (new_tp != NULL) { 674 new_tp->ftt_ids = NULL; 675 new_tp->ftt_retids = NULL; 676 } 677 678 return (0); 679 } 680 681 /* 682 * If we have a good tracepoint ready to go, install it now while 683 * we have the lock held and no one can screw with us. 684 */ 685 if (new_tp != NULL) { 686 int rc = 0; 687 688 new_tp->ftt_next = bucket->ftb_data; 689 membar_producer(); 690 bucket->ftb_data = new_tp; 691 membar_producer(); 692 mutex_exit(&bucket->ftb_mtx); 693 694 /* 695 * Activate the tracepoint in the ISA-specific manner. 696 * If this fails, we need to report the failure, but 697 * indicate that this tracepoint must still be disabled 698 * by calling fasttrap_tracepoint_disable(). 699 */ 700 if (fasttrap_tracepoint_install(p, new_tp) != 0) 701 rc = FASTTRAP_ENABLE_PARTIAL; 702 703 /* 704 * Increment the count of the number of tracepoints active in 705 * the victim process. 706 */ 707#if defined(sun) 708 ASSERT(p->p_proc_flag & P_PR_LOCK); 709#endif 710 p->p_dtrace_count++; 711 712 return (rc); 713 } 714 715 mutex_exit(&bucket->ftb_mtx); 716 717 /* 718 * Initialize the tracepoint that's been preallocated with the probe. 719 */ 720 new_tp = probe->ftp_tps[index].fit_tp; 721 722 ASSERT(new_tp->ftt_pid == pid); 723 ASSERT(new_tp->ftt_pc == pc); 724 ASSERT(new_tp->ftt_proc == probe->ftp_prov->ftp_proc); 725 ASSERT(new_tp->ftt_ids == NULL); 726 ASSERT(new_tp->ftt_retids == NULL); 727 728 switch (id->fti_ptype) { 729 case DTFTP_ENTRY: 730 case DTFTP_OFFSETS: 731 case DTFTP_IS_ENABLED: 732 id->fti_next = NULL; 733 new_tp->ftt_ids = id; 734 break; 735 736 case DTFTP_RETURN: 737 case DTFTP_POST_OFFSETS: 738 id->fti_next = NULL; 739 new_tp->ftt_retids = id; 740 break; 741 742 default: 743 ASSERT(0); 744 } 745 746 /* 747 * If the ISA-dependent initialization goes to plan, go back to the 748 * beginning and try to install this freshly made tracepoint. 749 */ 750 if (fasttrap_tracepoint_init(p, new_tp, pc, id->fti_ptype) == 0) 751 goto again; 752 753 new_tp->ftt_ids = NULL; 754 new_tp->ftt_retids = NULL; 755 756 return (FASTTRAP_ENABLE_FAIL); 757} 758 759static void 760fasttrap_tracepoint_disable(proc_t *p, fasttrap_probe_t *probe, uint_t index) 761{ 762 fasttrap_bucket_t *bucket; 763 fasttrap_provider_t *provider = probe->ftp_prov; 764 fasttrap_tracepoint_t **pp, *tp; 765 fasttrap_id_t *id, **idp = NULL; 766 pid_t pid; 767 uintptr_t pc; 768 769 ASSERT(index < probe->ftp_ntps); 770 771 pid = probe->ftp_pid; 772 pc = probe->ftp_tps[index].fit_tp->ftt_pc; 773 id = &probe->ftp_tps[index].fit_id; 774 775 ASSERT(probe->ftp_tps[index].fit_tp->ftt_pid == pid); 776 777 /* 778 * Find the tracepoint and make sure that our id is one of the 779 * ones registered with it. 780 */ 781 bucket = &fasttrap_tpoints.fth_table[FASTTRAP_TPOINTS_INDEX(pid, pc)]; 782 mutex_enter(&bucket->ftb_mtx); 783 for (tp = bucket->ftb_data; tp != NULL; tp = tp->ftt_next) { 784 if (tp->ftt_pid == pid && tp->ftt_pc == pc && 785 tp->ftt_proc == provider->ftp_proc) 786 break; 787 } 788 789 /* 790 * If we somehow lost this tracepoint, we're in a world of hurt. 791 */ 792 ASSERT(tp != NULL); 793 794 switch (id->fti_ptype) { 795 case DTFTP_ENTRY: 796 case DTFTP_OFFSETS: 797 case DTFTP_IS_ENABLED: 798 ASSERT(tp->ftt_ids != NULL); 799 idp = &tp->ftt_ids; 800 break; 801 802 case DTFTP_RETURN: 803 case DTFTP_POST_OFFSETS: 804 ASSERT(tp->ftt_retids != NULL); 805 idp = &tp->ftt_retids; 806 break; 807 808 default: 809 ASSERT(0); 810 } 811 812 while ((*idp)->fti_probe != probe) { 813 idp = &(*idp)->fti_next; 814 ASSERT(*idp != NULL); 815 } 816 817 id = *idp; 818 *idp = id->fti_next; 819 membar_producer(); 820 821 ASSERT(id->fti_probe == probe); 822 823 /* 824 * If there are other registered enablings of this tracepoint, we're 825 * all done, but if this was the last probe assocated with this 826 * this tracepoint, we need to remove and free it. 827 */ 828 if (tp->ftt_ids != NULL || tp->ftt_retids != NULL) { 829 830 /* 831 * If the current probe's tracepoint is in use, swap it 832 * for an unused tracepoint. 833 */ 834 if (tp == probe->ftp_tps[index].fit_tp) { 835 fasttrap_probe_t *tmp_probe; 836 fasttrap_tracepoint_t **tmp_tp; 837 uint_t tmp_index; 838 839 if (tp->ftt_ids != NULL) { 840 tmp_probe = tp->ftt_ids->fti_probe; 841 /* LINTED - alignment */ 842 tmp_index = FASTTRAP_ID_INDEX(tp->ftt_ids); 843 tmp_tp = &tmp_probe->ftp_tps[tmp_index].fit_tp; 844 } else { 845 tmp_probe = tp->ftt_retids->fti_probe; 846 /* LINTED - alignment */ 847 tmp_index = FASTTRAP_ID_INDEX(tp->ftt_retids); 848 tmp_tp = &tmp_probe->ftp_tps[tmp_index].fit_tp; 849 } 850 851 ASSERT(*tmp_tp != NULL); 852 ASSERT(*tmp_tp != probe->ftp_tps[index].fit_tp); 853 ASSERT((*tmp_tp)->ftt_ids == NULL); 854 ASSERT((*tmp_tp)->ftt_retids == NULL); 855 856 probe->ftp_tps[index].fit_tp = *tmp_tp; 857 *tmp_tp = tp; 858 } 859 860 mutex_exit(&bucket->ftb_mtx); 861 862 /* 863 * Tag the modified probe with the generation in which it was 864 * changed. 865 */ 866 probe->ftp_gen = fasttrap_mod_gen; 867 return; 868 } 869 870 mutex_exit(&bucket->ftb_mtx); 871 872 /* 873 * We can't safely remove the tracepoint from the set of active 874 * tracepoints until we've actually removed the fasttrap instruction 875 * from the process's text. We can, however, operate on this 876 * tracepoint secure in the knowledge that no other thread is going to 877 * be looking at it since we hold P_PR_LOCK on the process if it's 878 * live or we hold the provider lock on the process if it's dead and 879 * gone. 880 */ 881 882 /* 883 * We only need to remove the actual instruction if we're looking 884 * at an existing process 885 */ 886 if (p != NULL) { 887 /* 888 * If we fail to restore the instruction we need to kill 889 * this process since it's in a completely unrecoverable 890 * state. 891 */ 892 if (fasttrap_tracepoint_remove(p, tp) != 0) 893 fasttrap_sigtrap(p, NULL, pc); 894 895 /* 896 * Decrement the count of the number of tracepoints active 897 * in the victim process. 898 */ 899#if defined(sun) 900 ASSERT(p->p_proc_flag & P_PR_LOCK); 901#endif 902 p->p_dtrace_count--; 903 } 904 905 /* 906 * Remove the probe from the hash table of active tracepoints. 907 */ 908 mutex_enter(&bucket->ftb_mtx); 909 pp = (fasttrap_tracepoint_t **)&bucket->ftb_data; 910 ASSERT(*pp != NULL); 911 while (*pp != tp) { 912 pp = &(*pp)->ftt_next; 913 ASSERT(*pp != NULL); 914 } 915 916 *pp = tp->ftt_next; 917 membar_producer(); 918 919 mutex_exit(&bucket->ftb_mtx); 920 921 /* 922 * Tag the modified probe with the generation in which it was changed. 923 */ 924 probe->ftp_gen = fasttrap_mod_gen; 925} 926 927static void 928fasttrap_enable_callbacks(void) 929{ 930 /* 931 * We don't have to play the rw lock game here because we're 932 * providing something rather than taking something away -- 933 * we can be sure that no threads have tried to follow this 934 * function pointer yet. 935 */ 936 mutex_enter(&fasttrap_count_mtx); 937 if (fasttrap_pid_count == 0) { 938 ASSERT(dtrace_pid_probe_ptr == NULL); 939 ASSERT(dtrace_return_probe_ptr == NULL); 940 dtrace_pid_probe_ptr = &fasttrap_pid_probe; 941 dtrace_return_probe_ptr = &fasttrap_return_probe; 942 } 943 ASSERT(dtrace_pid_probe_ptr == &fasttrap_pid_probe); 944 ASSERT(dtrace_return_probe_ptr == &fasttrap_return_probe); 945 fasttrap_pid_count++; 946 mutex_exit(&fasttrap_count_mtx); 947} 948 949static void 950fasttrap_disable_callbacks(void) 951{ 952#if defined(sun) 953 ASSERT(MUTEX_HELD(&cpu_lock)); 954#endif 955 956 957 mutex_enter(&fasttrap_count_mtx); 958 ASSERT(fasttrap_pid_count > 0); 959 fasttrap_pid_count--; 960 if (fasttrap_pid_count == 0) { 961#if defined(sun) 962 cpu_t *cur, *cpu = CPU; 963 964 for (cur = cpu->cpu_next_onln; cur != cpu; 965 cur = cur->cpu_next_onln) { 966 rw_enter(&cur->cpu_ft_lock, RW_WRITER); 967 } 968#endif 969 dtrace_pid_probe_ptr = NULL; 970 dtrace_return_probe_ptr = NULL; 971#if defined(sun) 972 for (cur = cpu->cpu_next_onln; cur != cpu; 973 cur = cur->cpu_next_onln) { 974 rw_exit(&cur->cpu_ft_lock); 975 } 976#endif 977 } 978 mutex_exit(&fasttrap_count_mtx); 979} 980 981/*ARGSUSED*/ 982static void 983fasttrap_pid_enable(void *arg, dtrace_id_t id, void *parg) 984{ 985 fasttrap_probe_t *probe = parg; 986 proc_t *p = NULL; 987 int i, rc; 988 989 ASSERT(probe != NULL); 990 ASSERT(!probe->ftp_enabled); 991 ASSERT(id == probe->ftp_id); 992#if defined(sun) 993 ASSERT(MUTEX_HELD(&cpu_lock)); 994#endif 995 996 /* 997 * Increment the count of enabled probes on this probe's provider; 998 * the provider can't go away while the probe still exists. We 999 * must increment this even if we aren't able to properly enable 1000 * this probe. 1001 */ 1002 mutex_enter(&probe->ftp_prov->ftp_mtx); 1003 probe->ftp_prov->ftp_rcount++; 1004 mutex_exit(&probe->ftp_prov->ftp_mtx); 1005 1006 /* 1007 * If this probe's provider is retired (meaning it was valid in a 1008 * previously exec'ed incarnation of this address space), bail out. The 1009 * provider can't go away while we're in this code path. 1010 */ 1011 if (probe->ftp_prov->ftp_retired) 1012 return; 1013 1014 /* 1015 * If we can't find the process, it may be that we're in the context of 1016 * a fork in which the traced process is being born and we're copying 1017 * USDT probes. Otherwise, the process is gone so bail. 1018 */ 1019#if defined(sun) 1020 if ((p = sprlock(probe->ftp_pid)) == NULL) { 1021 if ((curproc->p_flag & SFORKING) == 0) 1022 return; 1023 1024 mutex_enter(&pidlock); 1025 p = prfind(probe->ftp_pid); 1026 1027 /* 1028 * Confirm that curproc is indeed forking the process in which 1029 * we're trying to enable probes. 1030 */ 1031 ASSERT(p != NULL); 1032 ASSERT(p->p_parent == curproc); 1033 ASSERT(p->p_stat == SIDL); 1034 1035 mutex_enter(&p->p_lock); 1036 mutex_exit(&pidlock); 1037 1038 sprlock_proc(p); 1039 } 1040 1041 ASSERT(!(p->p_flag & SVFORK)); 1042 mutex_exit(&p->p_lock); 1043#else 1044 if ((p = pfind(probe->ftp_pid)) == NULL) 1045 return; 1046#endif 1047 1048 /* 1049 * We have to enable the trap entry point before any user threads have 1050 * the chance to execute the trap instruction we're about to place 1051 * in their process's text. 1052 */ 1053#ifdef __FreeBSD__ 1054 /* 1055 * pfind() returns a locked process. 1056 */ 1057 _PHOLD(p); 1058 PROC_UNLOCK(p); 1059#endif 1060 fasttrap_enable_callbacks(); 1061 1062 /* 1063 * Enable all the tracepoints and add this probe's id to each 1064 * tracepoint's list of active probes. 1065 */ 1066 for (i = 0; i < probe->ftp_ntps; i++) { 1067 if ((rc = fasttrap_tracepoint_enable(p, probe, i)) != 0) { 1068 /* 1069 * If enabling the tracepoint failed completely, 1070 * we don't have to disable it; if the failure 1071 * was only partial we must disable it. 1072 */ 1073 if (rc == FASTTRAP_ENABLE_FAIL) 1074 i--; 1075 else 1076 ASSERT(rc == FASTTRAP_ENABLE_PARTIAL); 1077 1078 /* 1079 * Back up and pull out all the tracepoints we've 1080 * created so far for this probe. 1081 */ 1082 while (i >= 0) { 1083 fasttrap_tracepoint_disable(p, probe, i); 1084 i--; 1085 } 1086 1087#if defined(sun) 1088 mutex_enter(&p->p_lock); 1089 sprunlock(p); 1090#else 1091 PRELE(p); 1092#endif 1093 1094 /* 1095 * Since we're not actually enabling this probe, 1096 * drop our reference on the trap table entry. 1097 */ 1098 fasttrap_disable_callbacks(); 1099 return; 1100 } 1101 } 1102#if defined(sun) 1103 mutex_enter(&p->p_lock); 1104 sprunlock(p); 1105#else 1106 PRELE(p); 1107#endif 1108 1109 probe->ftp_enabled = 1; 1110} 1111 1112/*ARGSUSED*/ 1113static void 1114fasttrap_pid_disable(void *arg, dtrace_id_t id, void *parg) 1115{ 1116 fasttrap_probe_t *probe = parg; 1117 fasttrap_provider_t *provider = probe->ftp_prov; 1118 proc_t *p; 1119 int i, whack = 0; 1120 1121 ASSERT(id == probe->ftp_id); 1122 1123 mutex_enter(&provider->ftp_mtx); 1124 1125 /* 1126 * We won't be able to acquire a /proc-esque lock on the process 1127 * iff the process is dead and gone. In this case, we rely on the 1128 * provider lock as a point of mutual exclusion to prevent other 1129 * DTrace consumers from disabling this probe. 1130 */ 1131 if ((p = pfind(probe->ftp_pid)) != NULL) { 1132#ifdef __FreeBSD__ 1133 _PHOLD(p); 1134 PROC_UNLOCK(p); 1135#endif 1136 } 1137 1138 /* 1139 * Disable all the associated tracepoints (for fully enabled probes). 1140 */ 1141 if (probe->ftp_enabled) { 1142 for (i = 0; i < probe->ftp_ntps; i++) { 1143 fasttrap_tracepoint_disable(p, probe, i); 1144 } 1145 } 1146 1147 ASSERT(provider->ftp_rcount > 0); 1148 provider->ftp_rcount--; 1149 1150 if (p != NULL) { 1151 /* 1152 * Even though we may not be able to remove it entirely, we 1153 * mark this retired provider to get a chance to remove some 1154 * of the associated probes. 1155 */ 1156 if (provider->ftp_retired && !provider->ftp_marked) 1157 whack = provider->ftp_marked = 1; 1158 mutex_exit(&provider->ftp_mtx); 1159 } else { 1160 /* 1161 * If the process is dead, we're just waiting for the 1162 * last probe to be disabled to be able to free it. 1163 */ 1164 if (provider->ftp_rcount == 0 && !provider->ftp_marked) 1165 whack = provider->ftp_marked = 1; 1166 mutex_exit(&provider->ftp_mtx); 1167 } 1168 1169 if (whack) 1170 fasttrap_pid_cleanup(); 1171 1172#ifdef __FreeBSD__ 1173 if (p != NULL) 1174 PRELE(p); 1175#endif 1176 if (!probe->ftp_enabled) 1177 return; 1178 1179 probe->ftp_enabled = 0; 1180 1181#if defined(sun) 1182 ASSERT(MUTEX_HELD(&cpu_lock)); 1183#endif 1184 fasttrap_disable_callbacks(); 1185} 1186 1187/*ARGSUSED*/ 1188static void 1189fasttrap_pid_getargdesc(void *arg, dtrace_id_t id, void *parg, 1190 dtrace_argdesc_t *desc) 1191{ 1192 fasttrap_probe_t *probe = parg; 1193 char *str; 1194 int i, ndx; 1195 1196 desc->dtargd_native[0] = '\0'; 1197 desc->dtargd_xlate[0] = '\0'; 1198 1199 if (probe->ftp_prov->ftp_retired != 0 || 1200 desc->dtargd_ndx >= probe->ftp_nargs) { 1201 desc->dtargd_ndx = DTRACE_ARGNONE; 1202 return; 1203 } 1204 1205 ndx = (probe->ftp_argmap != NULL) ? 1206 probe->ftp_argmap[desc->dtargd_ndx] : desc->dtargd_ndx; 1207 1208 str = probe->ftp_ntypes; 1209 for (i = 0; i < ndx; i++) { 1210 str += strlen(str) + 1; 1211 } 1212 1213 ASSERT(strlen(str + 1) < sizeof (desc->dtargd_native)); 1214 (void) strcpy(desc->dtargd_native, str); 1215 1216 if (probe->ftp_xtypes == NULL) 1217 return; 1218 1219 str = probe->ftp_xtypes; 1220 for (i = 0; i < desc->dtargd_ndx; i++) { 1221 str += strlen(str) + 1; 1222 } 1223 1224 ASSERT(strlen(str + 1) < sizeof (desc->dtargd_xlate)); 1225 (void) strcpy(desc->dtargd_xlate, str); 1226} 1227 1228/*ARGSUSED*/ 1229static void 1230fasttrap_pid_destroy(void *arg, dtrace_id_t id, void *parg) 1231{ 1232 fasttrap_probe_t *probe = parg; 1233 int i; 1234 size_t size; 1235 1236 ASSERT(probe != NULL); 1237 ASSERT(!probe->ftp_enabled); 1238 ASSERT(fasttrap_total >= probe->ftp_ntps); 1239 1240 atomic_add_32(&fasttrap_total, -probe->ftp_ntps); 1241 size = offsetof(fasttrap_probe_t, ftp_tps[probe->ftp_ntps]); 1242 1243 if (probe->ftp_gen + 1 >= fasttrap_mod_gen) 1244 fasttrap_mod_barrier(probe->ftp_gen); 1245 1246 for (i = 0; i < probe->ftp_ntps; i++) { 1247 kmem_free(probe->ftp_tps[i].fit_tp, 1248 sizeof (fasttrap_tracepoint_t)); 1249 } 1250 1251 kmem_free(probe, size); 1252} 1253 1254 1255static const dtrace_pattr_t pid_attr = { 1256{ DTRACE_STABILITY_EVOLVING, DTRACE_STABILITY_EVOLVING, DTRACE_CLASS_ISA }, 1257{ DTRACE_STABILITY_PRIVATE, DTRACE_STABILITY_PRIVATE, DTRACE_CLASS_UNKNOWN }, 1258{ DTRACE_STABILITY_PRIVATE, DTRACE_STABILITY_PRIVATE, DTRACE_CLASS_UNKNOWN }, 1259{ DTRACE_STABILITY_EVOLVING, DTRACE_STABILITY_EVOLVING, DTRACE_CLASS_ISA }, 1260{ DTRACE_STABILITY_PRIVATE, DTRACE_STABILITY_PRIVATE, DTRACE_CLASS_UNKNOWN }, 1261}; 1262 1263static dtrace_pops_t pid_pops = { 1264 fasttrap_pid_provide, 1265 NULL, 1266 fasttrap_pid_enable, 1267 fasttrap_pid_disable, 1268 NULL, 1269 NULL, 1270 fasttrap_pid_getargdesc, 1271 fasttrap_pid_getarg, 1272 NULL, 1273 fasttrap_pid_destroy 1274}; 1275 1276static dtrace_pops_t usdt_pops = { 1277 fasttrap_pid_provide, 1278 NULL, 1279 fasttrap_pid_enable, 1280 fasttrap_pid_disable, 1281 NULL, 1282 NULL, 1283 fasttrap_pid_getargdesc, 1284 fasttrap_usdt_getarg, 1285 NULL, 1286 fasttrap_pid_destroy 1287}; 1288 1289static fasttrap_proc_t * 1290fasttrap_proc_lookup(pid_t pid) 1291{ 1292 fasttrap_bucket_t *bucket; 1293 fasttrap_proc_t *fprc, *new_fprc; 1294 1295 1296 bucket = &fasttrap_procs.fth_table[FASTTRAP_PROCS_INDEX(pid)]; 1297 mutex_enter(&bucket->ftb_mtx); 1298 1299 for (fprc = bucket->ftb_data; fprc != NULL; fprc = fprc->ftpc_next) { 1300 if (fprc->ftpc_pid == pid && fprc->ftpc_acount != 0) { 1301 mutex_enter(&fprc->ftpc_mtx); 1302 mutex_exit(&bucket->ftb_mtx); 1303 fprc->ftpc_rcount++; 1304 atomic_add_64(&fprc->ftpc_acount, 1); 1305 ASSERT(fprc->ftpc_acount <= fprc->ftpc_rcount); 1306 mutex_exit(&fprc->ftpc_mtx); 1307 1308 return (fprc); 1309 } 1310 } 1311 1312 /* 1313 * Drop the bucket lock so we don't try to perform a sleeping 1314 * allocation under it. 1315 */ 1316 mutex_exit(&bucket->ftb_mtx); 1317 1318 new_fprc = kmem_zalloc(sizeof (fasttrap_proc_t), KM_SLEEP); 1319 new_fprc->ftpc_pid = pid; 1320 new_fprc->ftpc_rcount = 1; 1321 new_fprc->ftpc_acount = 1; 1322#if !defined(sun) 1323 mutex_init(&new_fprc->ftpc_mtx, "fasttrap proc mtx", MUTEX_DEFAULT, 1324 NULL); 1325#endif 1326 1327 mutex_enter(&bucket->ftb_mtx); 1328 1329 /* 1330 * Take another lap through the list to make sure a proc hasn't 1331 * been created for this pid while we weren't under the bucket lock. 1332 */ 1333 for (fprc = bucket->ftb_data; fprc != NULL; fprc = fprc->ftpc_next) { 1334 if (fprc->ftpc_pid == pid && fprc->ftpc_acount != 0) { 1335 mutex_enter(&fprc->ftpc_mtx); 1336 mutex_exit(&bucket->ftb_mtx); 1337 fprc->ftpc_rcount++; 1338 atomic_add_64(&fprc->ftpc_acount, 1); 1339 ASSERT(fprc->ftpc_acount <= fprc->ftpc_rcount); 1340 mutex_exit(&fprc->ftpc_mtx); 1341 1342 kmem_free(new_fprc, sizeof (fasttrap_proc_t)); 1343 1344 return (fprc); 1345 } 1346 } 1347 1348 new_fprc->ftpc_next = bucket->ftb_data; 1349 bucket->ftb_data = new_fprc; 1350 1351 mutex_exit(&bucket->ftb_mtx); 1352 1353 return (new_fprc); 1354} 1355 1356static void 1357fasttrap_proc_release(fasttrap_proc_t *proc) 1358{ 1359 fasttrap_bucket_t *bucket; 1360 fasttrap_proc_t *fprc, **fprcp; 1361 pid_t pid = proc->ftpc_pid; 1362 1363 mutex_enter(&proc->ftpc_mtx); 1364 1365 ASSERT(proc->ftpc_rcount != 0); 1366 ASSERT(proc->ftpc_acount <= proc->ftpc_rcount); 1367 1368 if (--proc->ftpc_rcount != 0) { 1369 mutex_exit(&proc->ftpc_mtx); 1370 return; 1371 } 1372 1373 mutex_exit(&proc->ftpc_mtx); 1374 1375 /* 1376 * There should definitely be no live providers associated with this 1377 * process at this point. 1378 */ 1379 ASSERT(proc->ftpc_acount == 0); 1380 1381 bucket = &fasttrap_procs.fth_table[FASTTRAP_PROCS_INDEX(pid)]; 1382 mutex_enter(&bucket->ftb_mtx); 1383 1384 fprcp = (fasttrap_proc_t **)&bucket->ftb_data; 1385 while ((fprc = *fprcp) != NULL) { 1386 if (fprc == proc) 1387 break; 1388 1389 fprcp = &fprc->ftpc_next; 1390 } 1391 1392 /* 1393 * Something strange has happened if we can't find the proc. 1394 */ 1395 ASSERT(fprc != NULL); 1396 1397 *fprcp = fprc->ftpc_next; 1398 1399 mutex_exit(&bucket->ftb_mtx); 1400 1401 kmem_free(fprc, sizeof (fasttrap_proc_t)); 1402} 1403 1404/* 1405 * Lookup a fasttrap-managed provider based on its name and associated pid. 1406 * If the pattr argument is non-NULL, this function instantiates the provider 1407 * if it doesn't exist otherwise it returns NULL. The provider is returned 1408 * with its lock held. 1409 */ 1410static fasttrap_provider_t * 1411fasttrap_provider_lookup(pid_t pid, const char *name, 1412 const dtrace_pattr_t *pattr) 1413{ 1414 fasttrap_provider_t *fp, *new_fp = NULL; 1415 fasttrap_bucket_t *bucket; 1416 char provname[DTRACE_PROVNAMELEN]; 1417 proc_t *p; 1418 cred_t *cred; 1419 1420 ASSERT(strlen(name) < sizeof (fp->ftp_name)); 1421 ASSERT(pattr != NULL); 1422 1423 bucket = &fasttrap_provs.fth_table[FASTTRAP_PROVS_INDEX(pid, name)]; 1424 mutex_enter(&bucket->ftb_mtx); 1425 1426 /* 1427 * Take a lap through the list and return the match if we find it. 1428 */ 1429 for (fp = bucket->ftb_data; fp != NULL; fp = fp->ftp_next) { 1430 if (fp->ftp_pid == pid && strcmp(fp->ftp_name, name) == 0 && 1431 !fp->ftp_retired) { 1432 mutex_enter(&fp->ftp_mtx); 1433 mutex_exit(&bucket->ftb_mtx); 1434 return (fp); 1435 } 1436 } 1437 1438 /* 1439 * Drop the bucket lock so we don't try to perform a sleeping 1440 * allocation under it. 1441 */ 1442 mutex_exit(&bucket->ftb_mtx); 1443 1444 /* 1445 * Make sure the process exists, isn't a child created as the result 1446 * of a vfork(2), and isn't a zombie (but may be in fork). 1447 */ 1448 if ((p = pfind(pid)) == NULL) 1449 return (NULL); 1450 1451 /* 1452 * Increment p_dtrace_probes so that the process knows to inform us 1453 * when it exits or execs. fasttrap_provider_free() decrements this 1454 * when we're done with this provider. 1455 */ 1456 p->p_dtrace_probes++; 1457 1458 /* 1459 * Grab the credentials for this process so we have 1460 * something to pass to dtrace_register(). 1461 */ 1462 PROC_LOCK_ASSERT(p, MA_OWNED); 1463 crhold(p->p_ucred); 1464 cred = p->p_ucred; 1465 PROC_UNLOCK(p); 1466 1467 new_fp = kmem_zalloc(sizeof (fasttrap_provider_t), KM_SLEEP); 1468 new_fp->ftp_pid = pid; 1469 new_fp->ftp_proc = fasttrap_proc_lookup(pid); 1470#if !defined(sun) 1471 mutex_init(&new_fp->ftp_mtx, "provider mtx", MUTEX_DEFAULT, NULL); 1472 mutex_init(&new_fp->ftp_cmtx, "lock on creating", MUTEX_DEFAULT, NULL); 1473#endif 1474 1475 ASSERT(new_fp->ftp_proc != NULL); 1476 1477 mutex_enter(&bucket->ftb_mtx); 1478 1479 /* 1480 * Take another lap through the list to make sure a provider hasn't 1481 * been created for this pid while we weren't under the bucket lock. 1482 */ 1483 for (fp = bucket->ftb_data; fp != NULL; fp = fp->ftp_next) { 1484 if (fp->ftp_pid == pid && strcmp(fp->ftp_name, name) == 0 && 1485 !fp->ftp_retired) { 1486 mutex_enter(&fp->ftp_mtx); 1487 mutex_exit(&bucket->ftb_mtx); 1488 fasttrap_provider_free(new_fp); 1489 crfree(cred); 1490 return (fp); 1491 } 1492 } 1493 1494 (void) strcpy(new_fp->ftp_name, name); 1495 1496 /* 1497 * Fail and return NULL if either the provider name is too long 1498 * or we fail to register this new provider with the DTrace 1499 * framework. Note that this is the only place we ever construct 1500 * the full provider name -- we keep it in pieces in the provider 1501 * structure. 1502 */ 1503 if (snprintf(provname, sizeof (provname), "%s%u", name, (uint_t)pid) >= 1504 sizeof (provname) || 1505 dtrace_register(provname, pattr, 1506 DTRACE_PRIV_PROC | DTRACE_PRIV_OWNER | DTRACE_PRIV_ZONEOWNER, cred, 1507 pattr == &pid_attr ? &pid_pops : &usdt_pops, new_fp, 1508 &new_fp->ftp_provid) != 0) { 1509 mutex_exit(&bucket->ftb_mtx); 1510 fasttrap_provider_free(new_fp); 1511 crfree(cred); 1512 return (NULL); 1513 } 1514 1515 new_fp->ftp_next = bucket->ftb_data; 1516 bucket->ftb_data = new_fp; 1517 1518 mutex_enter(&new_fp->ftp_mtx); 1519 mutex_exit(&bucket->ftb_mtx); 1520 1521 crfree(cred); 1522 return (new_fp); 1523} 1524 1525static void 1526fasttrap_provider_free(fasttrap_provider_t *provider) 1527{ 1528 pid_t pid = provider->ftp_pid; 1529 proc_t *p; 1530 1531 /* 1532 * There need to be no associated enabled probes, no consumers 1533 * creating probes, and no meta providers referencing this provider. 1534 */ 1535 ASSERT(provider->ftp_rcount == 0); 1536 ASSERT(provider->ftp_ccount == 0); 1537 ASSERT(provider->ftp_mcount == 0); 1538 1539 /* 1540 * If this provider hasn't been retired, we need to explicitly drop the 1541 * count of active providers on the associated process structure. 1542 */ 1543 if (!provider->ftp_retired) { 1544 atomic_add_64(&provider->ftp_proc->ftpc_acount, -1); 1545 ASSERT(provider->ftp_proc->ftpc_acount < 1546 provider->ftp_proc->ftpc_rcount); 1547 } 1548 1549 fasttrap_proc_release(provider->ftp_proc); 1550 1551#if !defined(sun) 1552 mutex_destroy(&provider->ftp_mtx); 1553 mutex_destroy(&provider->ftp_cmtx); 1554#endif 1555 kmem_free(provider, sizeof (fasttrap_provider_t)); 1556 1557 /* 1558 * Decrement p_dtrace_probes on the process whose provider we're 1559 * freeing. We don't have to worry about clobbering somone else's 1560 * modifications to it because we have locked the bucket that 1561 * corresponds to this process's hash chain in the provider hash 1562 * table. Don't sweat it if we can't find the process. 1563 */ 1564 if ((p = pfind(pid)) == NULL) { 1565 return; 1566 } 1567 1568 p->p_dtrace_probes--; 1569#if !defined(sun) 1570 PROC_UNLOCK(p); 1571#endif 1572} 1573 1574static void 1575fasttrap_provider_retire(pid_t pid, const char *name, int mprov) 1576{ 1577 fasttrap_provider_t *fp; 1578 fasttrap_bucket_t *bucket; 1579 dtrace_provider_id_t provid; 1580 1581 ASSERT(strlen(name) < sizeof (fp->ftp_name)); 1582 1583 bucket = &fasttrap_provs.fth_table[FASTTRAP_PROVS_INDEX(pid, name)]; 1584 mutex_enter(&bucket->ftb_mtx); 1585 1586 for (fp = bucket->ftb_data; fp != NULL; fp = fp->ftp_next) { 1587 if (fp->ftp_pid == pid && strcmp(fp->ftp_name, name) == 0 && 1588 !fp->ftp_retired) 1589 break; 1590 } 1591 1592 if (fp == NULL) { 1593 mutex_exit(&bucket->ftb_mtx); 1594 return; 1595 } 1596 1597 mutex_enter(&fp->ftp_mtx); 1598 ASSERT(!mprov || fp->ftp_mcount > 0); 1599 if (mprov && --fp->ftp_mcount != 0) { 1600 mutex_exit(&fp->ftp_mtx); 1601 mutex_exit(&bucket->ftb_mtx); 1602 return; 1603 } 1604 1605 /* 1606 * Mark the provider to be removed in our post-processing step, mark it 1607 * retired, and drop the active count on its proc. Marking it indicates 1608 * that we should try to remove it; setting the retired flag indicates 1609 * that we're done with this provider; dropping the active the proc 1610 * releases our hold, and when this reaches zero (as it will during 1611 * exit or exec) the proc and associated providers become defunct. 1612 * 1613 * We obviously need to take the bucket lock before the provider lock 1614 * to perform the lookup, but we need to drop the provider lock 1615 * before calling into the DTrace framework since we acquire the 1616 * provider lock in callbacks invoked from the DTrace framework. The 1617 * bucket lock therefore protects the integrity of the provider hash 1618 * table. 1619 */ 1620 atomic_add_64(&fp->ftp_proc->ftpc_acount, -1); 1621 ASSERT(fp->ftp_proc->ftpc_acount < fp->ftp_proc->ftpc_rcount); 1622 1623 fp->ftp_retired = 1; 1624 fp->ftp_marked = 1; 1625 provid = fp->ftp_provid; 1626 mutex_exit(&fp->ftp_mtx); 1627 1628 /* 1629 * We don't have to worry about invalidating the same provider twice 1630 * since fasttrap_provider_lookup() will ignore provider that have 1631 * been marked as retired. 1632 */ 1633 dtrace_invalidate(provid); 1634 1635 mutex_exit(&bucket->ftb_mtx); 1636 1637 fasttrap_pid_cleanup(); 1638} 1639 1640static int 1641fasttrap_uint32_cmp(const void *ap, const void *bp) 1642{ 1643 return (*(const uint32_t *)ap - *(const uint32_t *)bp); 1644} 1645 1646static int 1647fasttrap_uint64_cmp(const void *ap, const void *bp) 1648{ 1649 return (*(const uint64_t *)ap - *(const uint64_t *)bp); 1650} 1651 1652static int 1653fasttrap_add_probe(fasttrap_probe_spec_t *pdata) 1654{ 1655 fasttrap_provider_t *provider; 1656 fasttrap_probe_t *pp; 1657 fasttrap_tracepoint_t *tp; 1658 char *name; 1659 int i, aframes = 0, whack; 1660 1661 /* 1662 * There needs to be at least one desired trace point. 1663 */ 1664 if (pdata->ftps_noffs == 0) 1665 return (EINVAL); 1666 1667 switch (pdata->ftps_type) { 1668 case DTFTP_ENTRY: 1669 name = "entry"; 1670 aframes = FASTTRAP_ENTRY_AFRAMES; 1671 break; 1672 case DTFTP_RETURN: 1673 name = "return"; 1674 aframes = FASTTRAP_RETURN_AFRAMES; 1675 break; 1676 case DTFTP_OFFSETS: 1677 name = NULL; 1678 break; 1679 default: 1680 return (EINVAL); 1681 } 1682 1683 if ((provider = fasttrap_provider_lookup(pdata->ftps_pid, 1684 FASTTRAP_PID_NAME, &pid_attr)) == NULL) 1685 return (ESRCH); 1686 1687 /* 1688 * Increment this reference count to indicate that a consumer is 1689 * actively adding a new probe associated with this provider. This 1690 * prevents the provider from being deleted -- we'll need to check 1691 * for pending deletions when we drop this reference count. 1692 */ 1693 provider->ftp_ccount++; 1694 mutex_exit(&provider->ftp_mtx); 1695 1696 /* 1697 * Grab the creation lock to ensure consistency between calls to 1698 * dtrace_probe_lookup() and dtrace_probe_create() in the face of 1699 * other threads creating probes. We must drop the provider lock 1700 * before taking this lock to avoid a three-way deadlock with the 1701 * DTrace framework. 1702 */ 1703 mutex_enter(&provider->ftp_cmtx); 1704 1705 if (name == NULL) { 1706 for (i = 0; i < pdata->ftps_noffs; i++) { 1707 char name_str[17]; 1708 1709 (void) sprintf(name_str, "%llx", 1710 (unsigned long long)pdata->ftps_offs[i]); 1711 1712 if (dtrace_probe_lookup(provider->ftp_provid, 1713 pdata->ftps_mod, pdata->ftps_func, name_str) != 0) 1714 continue; 1715 1716 atomic_add_32(&fasttrap_total, 1); 1717 1718 if (fasttrap_total > fasttrap_max) { 1719 atomic_add_32(&fasttrap_total, -1); 1720 goto no_mem; 1721 } 1722 1723 pp = kmem_zalloc(sizeof (fasttrap_probe_t), KM_SLEEP); 1724 1725 pp->ftp_prov = provider; 1726 pp->ftp_faddr = pdata->ftps_pc; 1727 pp->ftp_fsize = pdata->ftps_size; 1728 pp->ftp_pid = pdata->ftps_pid; 1729 pp->ftp_ntps = 1; 1730 1731 tp = kmem_zalloc(sizeof (fasttrap_tracepoint_t), 1732 KM_SLEEP); 1733 1734 tp->ftt_proc = provider->ftp_proc; 1735 tp->ftt_pc = pdata->ftps_offs[i] + pdata->ftps_pc; 1736 tp->ftt_pid = pdata->ftps_pid; 1737 1738 pp->ftp_tps[0].fit_tp = tp; 1739 pp->ftp_tps[0].fit_id.fti_probe = pp; 1740 pp->ftp_tps[0].fit_id.fti_ptype = pdata->ftps_type; 1741 1742 pp->ftp_id = dtrace_probe_create(provider->ftp_provid, 1743 pdata->ftps_mod, pdata->ftps_func, name_str, 1744 FASTTRAP_OFFSET_AFRAMES, pp); 1745 } 1746 1747 } else if (dtrace_probe_lookup(provider->ftp_provid, pdata->ftps_mod, 1748 pdata->ftps_func, name) == 0) { 1749 atomic_add_32(&fasttrap_total, pdata->ftps_noffs); 1750 1751 if (fasttrap_total > fasttrap_max) { 1752 atomic_add_32(&fasttrap_total, -pdata->ftps_noffs); 1753 goto no_mem; 1754 } 1755 1756 /* 1757 * Make sure all tracepoint program counter values are unique. 1758 * We later assume that each probe has exactly one tracepoint 1759 * for a given pc. 1760 */ 1761 qsort(pdata->ftps_offs, pdata->ftps_noffs, 1762 sizeof (uint64_t), fasttrap_uint64_cmp); 1763 for (i = 1; i < pdata->ftps_noffs; i++) { 1764 if (pdata->ftps_offs[i] > pdata->ftps_offs[i - 1]) 1765 continue; 1766 1767 atomic_add_32(&fasttrap_total, -pdata->ftps_noffs); 1768 goto no_mem; 1769 } 1770 1771 ASSERT(pdata->ftps_noffs > 0); 1772 pp = kmem_zalloc(offsetof(fasttrap_probe_t, 1773 ftp_tps[pdata->ftps_noffs]), KM_SLEEP); 1774 1775 pp->ftp_prov = provider; 1776 pp->ftp_faddr = pdata->ftps_pc; 1777 pp->ftp_fsize = pdata->ftps_size; 1778 pp->ftp_pid = pdata->ftps_pid; 1779 pp->ftp_ntps = pdata->ftps_noffs; 1780 1781 for (i = 0; i < pdata->ftps_noffs; i++) { 1782 tp = kmem_zalloc(sizeof (fasttrap_tracepoint_t), 1783 KM_SLEEP); 1784 1785 tp->ftt_proc = provider->ftp_proc; 1786 tp->ftt_pc = pdata->ftps_offs[i] + pdata->ftps_pc; 1787 tp->ftt_pid = pdata->ftps_pid; 1788 1789 pp->ftp_tps[i].fit_tp = tp; 1790 pp->ftp_tps[i].fit_id.fti_probe = pp; 1791 pp->ftp_tps[i].fit_id.fti_ptype = pdata->ftps_type; 1792 } 1793 1794 pp->ftp_id = dtrace_probe_create(provider->ftp_provid, 1795 pdata->ftps_mod, pdata->ftps_func, name, aframes, pp); 1796 } 1797 1798 mutex_exit(&provider->ftp_cmtx); 1799 1800 /* 1801 * We know that the provider is still valid since we incremented the 1802 * creation reference count. If someone tried to clean up this provider 1803 * while we were using it (e.g. because the process called exec(2) or 1804 * exit(2)), take note of that and try to clean it up now. 1805 */ 1806 mutex_enter(&provider->ftp_mtx); 1807 provider->ftp_ccount--; 1808 whack = provider->ftp_retired; 1809 mutex_exit(&provider->ftp_mtx); 1810 1811 if (whack) 1812 fasttrap_pid_cleanup(); 1813 1814 return (0); 1815 1816no_mem: 1817 /* 1818 * If we've exhausted the allowable resources, we'll try to remove 1819 * this provider to free some up. This is to cover the case where 1820 * the user has accidentally created many more probes than was 1821 * intended (e.g. pid123:::). 1822 */ 1823 mutex_exit(&provider->ftp_cmtx); 1824 mutex_enter(&provider->ftp_mtx); 1825 provider->ftp_ccount--; 1826 provider->ftp_marked = 1; 1827 mutex_exit(&provider->ftp_mtx); 1828 1829 fasttrap_pid_cleanup(); 1830 1831 return (ENOMEM); 1832} 1833 1834/*ARGSUSED*/ 1835static void * 1836fasttrap_meta_provide(void *arg, dtrace_helper_provdesc_t *dhpv, pid_t pid) 1837{ 1838 fasttrap_provider_t *provider; 1839 1840 /* 1841 * A 32-bit unsigned integer (like a pid for example) can be 1842 * expressed in 10 or fewer decimal digits. Make sure that we'll 1843 * have enough space for the provider name. 1844 */ 1845 if (strlen(dhpv->dthpv_provname) + 10 >= 1846 sizeof (provider->ftp_name)) { 1847 printf("failed to instantiate provider %s: " 1848 "name too long to accomodate pid", dhpv->dthpv_provname); 1849 return (NULL); 1850 } 1851 1852 /* 1853 * Don't let folks spoof the true pid provider. 1854 */ 1855 if (strcmp(dhpv->dthpv_provname, FASTTRAP_PID_NAME) == 0) { 1856 printf("failed to instantiate provider %s: " 1857 "%s is an invalid name", dhpv->dthpv_provname, 1858 FASTTRAP_PID_NAME); 1859 return (NULL); 1860 } 1861 1862 /* 1863 * The highest stability class that fasttrap supports is ISA; cap 1864 * the stability of the new provider accordingly. 1865 */ 1866 if (dhpv->dthpv_pattr.dtpa_provider.dtat_class > DTRACE_CLASS_ISA) 1867 dhpv->dthpv_pattr.dtpa_provider.dtat_class = DTRACE_CLASS_ISA; 1868 if (dhpv->dthpv_pattr.dtpa_mod.dtat_class > DTRACE_CLASS_ISA) 1869 dhpv->dthpv_pattr.dtpa_mod.dtat_class = DTRACE_CLASS_ISA; 1870 if (dhpv->dthpv_pattr.dtpa_func.dtat_class > DTRACE_CLASS_ISA) 1871 dhpv->dthpv_pattr.dtpa_func.dtat_class = DTRACE_CLASS_ISA; 1872 if (dhpv->dthpv_pattr.dtpa_name.dtat_class > DTRACE_CLASS_ISA) 1873 dhpv->dthpv_pattr.dtpa_name.dtat_class = DTRACE_CLASS_ISA; 1874 if (dhpv->dthpv_pattr.dtpa_args.dtat_class > DTRACE_CLASS_ISA) 1875 dhpv->dthpv_pattr.dtpa_args.dtat_class = DTRACE_CLASS_ISA; 1876 1877 if ((provider = fasttrap_provider_lookup(pid, dhpv->dthpv_provname, 1878 &dhpv->dthpv_pattr)) == NULL) { 1879 printf("failed to instantiate provider %s for " 1880 "process %u", dhpv->dthpv_provname, (uint_t)pid); 1881 return (NULL); 1882 } 1883 1884 /* 1885 * Up the meta provider count so this provider isn't removed until 1886 * the meta provider has been told to remove it. 1887 */ 1888 provider->ftp_mcount++; 1889 1890 mutex_exit(&provider->ftp_mtx); 1891 1892 return (provider); 1893} 1894 1895/*ARGSUSED*/ 1896static void 1897fasttrap_meta_create_probe(void *arg, void *parg, 1898 dtrace_helper_probedesc_t *dhpb) 1899{ 1900 fasttrap_provider_t *provider = parg; 1901 fasttrap_probe_t *pp; 1902 fasttrap_tracepoint_t *tp; 1903 int i, j; 1904 uint32_t ntps; 1905 1906 /* 1907 * Since the meta provider count is non-zero we don't have to worry 1908 * about this provider disappearing. 1909 */ 1910 ASSERT(provider->ftp_mcount > 0); 1911 1912 /* 1913 * The offsets must be unique. 1914 */ 1915 qsort(dhpb->dthpb_offs, dhpb->dthpb_noffs, sizeof (uint32_t), 1916 fasttrap_uint32_cmp); 1917 for (i = 1; i < dhpb->dthpb_noffs; i++) { 1918 if (dhpb->dthpb_base + dhpb->dthpb_offs[i] <= 1919 dhpb->dthpb_base + dhpb->dthpb_offs[i - 1]) 1920 return; 1921 } 1922 1923 qsort(dhpb->dthpb_enoffs, dhpb->dthpb_nenoffs, sizeof (uint32_t), 1924 fasttrap_uint32_cmp); 1925 for (i = 1; i < dhpb->dthpb_nenoffs; i++) { 1926 if (dhpb->dthpb_base + dhpb->dthpb_enoffs[i] <= 1927 dhpb->dthpb_base + dhpb->dthpb_enoffs[i - 1]) 1928 return; 1929 } 1930 1931 /* 1932 * Grab the creation lock to ensure consistency between calls to 1933 * dtrace_probe_lookup() and dtrace_probe_create() in the face of 1934 * other threads creating probes. 1935 */ 1936 mutex_enter(&provider->ftp_cmtx); 1937 1938 if (dtrace_probe_lookup(provider->ftp_provid, dhpb->dthpb_mod, 1939 dhpb->dthpb_func, dhpb->dthpb_name) != 0) { 1940 mutex_exit(&provider->ftp_cmtx); 1941 return; 1942 } 1943 1944 ntps = dhpb->dthpb_noffs + dhpb->dthpb_nenoffs; 1945 ASSERT(ntps > 0); 1946 1947 atomic_add_32(&fasttrap_total, ntps); 1948 1949 if (fasttrap_total > fasttrap_max) { 1950 atomic_add_32(&fasttrap_total, -ntps); 1951 mutex_exit(&provider->ftp_cmtx); 1952 return; 1953 } 1954 1955 pp = kmem_zalloc(offsetof(fasttrap_probe_t, ftp_tps[ntps]), KM_SLEEP); 1956 1957 pp->ftp_prov = provider; 1958 pp->ftp_pid = provider->ftp_pid; 1959 pp->ftp_ntps = ntps; 1960 pp->ftp_nargs = dhpb->dthpb_xargc; 1961 pp->ftp_xtypes = dhpb->dthpb_xtypes; 1962 pp->ftp_ntypes = dhpb->dthpb_ntypes; 1963 1964 /* 1965 * First create a tracepoint for each actual point of interest. 1966 */ 1967 for (i = 0; i < dhpb->dthpb_noffs; i++) { 1968 tp = kmem_zalloc(sizeof (fasttrap_tracepoint_t), KM_SLEEP); 1969 1970 tp->ftt_proc = provider->ftp_proc; 1971 tp->ftt_pc = dhpb->dthpb_base + dhpb->dthpb_offs[i]; 1972 tp->ftt_pid = provider->ftp_pid; 1973 1974 pp->ftp_tps[i].fit_tp = tp; 1975 pp->ftp_tps[i].fit_id.fti_probe = pp; 1976#ifdef __sparc 1977 pp->ftp_tps[i].fit_id.fti_ptype = DTFTP_POST_OFFSETS; 1978#else 1979 pp->ftp_tps[i].fit_id.fti_ptype = DTFTP_OFFSETS; 1980#endif 1981 } 1982 1983 /* 1984 * Then create a tracepoint for each is-enabled point. 1985 */ 1986 for (j = 0; i < ntps; i++, j++) { 1987 tp = kmem_zalloc(sizeof (fasttrap_tracepoint_t), KM_SLEEP); 1988 1989 tp->ftt_proc = provider->ftp_proc; 1990 tp->ftt_pc = dhpb->dthpb_base + dhpb->dthpb_enoffs[j]; 1991 tp->ftt_pid = provider->ftp_pid; 1992 1993 pp->ftp_tps[i].fit_tp = tp; 1994 pp->ftp_tps[i].fit_id.fti_probe = pp; 1995 pp->ftp_tps[i].fit_id.fti_ptype = DTFTP_IS_ENABLED; 1996 } 1997 1998 /* 1999 * If the arguments are shuffled around we set the argument remapping 2000 * table. Later, when the probe fires, we only remap the arguments 2001 * if the table is non-NULL. 2002 */ 2003 for (i = 0; i < dhpb->dthpb_xargc; i++) { 2004 if (dhpb->dthpb_args[i] != i) { 2005 pp->ftp_argmap = dhpb->dthpb_args; 2006 break; 2007 } 2008 } 2009 2010 /* 2011 * The probe is fully constructed -- register it with DTrace. 2012 */ 2013 pp->ftp_id = dtrace_probe_create(provider->ftp_provid, dhpb->dthpb_mod, 2014 dhpb->dthpb_func, dhpb->dthpb_name, FASTTRAP_OFFSET_AFRAMES, pp); 2015 2016 mutex_exit(&provider->ftp_cmtx); 2017} 2018 2019/*ARGSUSED*/ 2020static void 2021fasttrap_meta_remove(void *arg, dtrace_helper_provdesc_t *dhpv, pid_t pid) 2022{ 2023 /* 2024 * Clean up the USDT provider. There may be active consumers of the 2025 * provider busy adding probes, no damage will actually befall the 2026 * provider until that count has dropped to zero. This just puts 2027 * the provider on death row. 2028 */ 2029 fasttrap_provider_retire(pid, dhpv->dthpv_provname, 1); 2030} 2031 2032static dtrace_mops_t fasttrap_mops = { 2033 fasttrap_meta_create_probe, 2034 fasttrap_meta_provide, 2035 fasttrap_meta_remove 2036}; 2037 2038/*ARGSUSED*/ 2039static int 2040fasttrap_open(struct cdev *dev __unused, int oflags __unused, 2041 int devtype __unused, struct thread *td __unused) 2042{ 2043 return (0); 2044} 2045 2046/*ARGSUSED*/ 2047static int 2048fasttrap_ioctl(struct cdev *dev, u_long cmd, caddr_t arg, int fflag, 2049 struct thread *td) 2050{ 2051#ifdef notyet 2052 struct kinfo_proc kp; 2053 const cred_t *cr = td->td_ucred; 2054#endif 2055 if (!dtrace_attached()) 2056 return (EAGAIN); 2057 2058 if (cmd == FASTTRAPIOC_MAKEPROBE) { 2059 fasttrap_probe_spec_t *uprobe = (void *)arg; 2060 fasttrap_probe_spec_t *probe; 2061 uint64_t noffs; 2062 size_t size; 2063 int ret; 2064 char *c; 2065 2066#if defined(sun) 2067 if (copyin(&uprobe->ftps_noffs, &noffs, 2068 sizeof (uprobe->ftps_noffs))) 2069 return (EFAULT); 2070#else 2071 noffs = uprobe->ftps_noffs; 2072#endif 2073 2074 /* 2075 * Probes must have at least one tracepoint. 2076 */ 2077 if (noffs == 0) 2078 return (EINVAL); 2079 2080 size = sizeof (fasttrap_probe_spec_t) + 2081 sizeof (probe->ftps_offs[0]) * (noffs - 1); 2082 2083 if (size > 1024 * 1024) 2084 return (ENOMEM); 2085 2086 probe = kmem_alloc(size, KM_SLEEP); 2087 2088#if defined(sun) 2089 if (copyin(uprobe, probe, size) != 0) { 2090 kmem_free(probe, size); 2091 return (EFAULT); 2092 } 2093#else 2094 memcpy(probe, uprobe, sizeof(*probe)); 2095 if (noffs > 1 && copyin(uprobe + 1, probe + 1, size) != 0) { 2096 kmem_free(probe, size); 2097 return (EFAULT); 2098 } 2099#endif 2100 2101 2102 /* 2103 * Verify that the function and module strings contain no 2104 * funny characters. 2105 */ 2106 for (c = &probe->ftps_func[0]; *c != '\0'; c++) { 2107 if (*c < 0x20 || 0x7f <= *c) { 2108 ret = EINVAL; 2109 goto err; 2110 } 2111 } 2112 2113 for (c = &probe->ftps_mod[0]; *c != '\0'; c++) { 2114 if (*c < 0x20 || 0x7f <= *c) { 2115 ret = EINVAL; 2116 goto err; 2117 } 2118 } 2119 2120#ifdef notyet 2121 if (!PRIV_POLICY_CHOICE(cr, PRIV_ALL, B_FALSE)) { 2122 proc_t *p; 2123 pid_t pid = probe->ftps_pid; 2124 2125#if defined(sun) 2126 mutex_enter(&pidlock); 2127#endif 2128 /* 2129 * Report an error if the process doesn't exist 2130 * or is actively being birthed. 2131 */ 2132 p = pfind(pid); 2133 if (p) 2134 fill_kinfo_proc(p, &kp); 2135 if (p == NULL || kp.ki_stat == SIDL) { 2136#if defined(sun) 2137 mutex_exit(&pidlock); 2138#endif 2139 return (ESRCH); 2140 } 2141#if defined(sun) 2142 mutex_enter(&p->p_lock); 2143 mutex_exit(&pidlock); 2144#else 2145 PROC_LOCK_ASSERT(p, MA_OWNED); 2146#endif 2147 2148#ifdef notyet 2149 if ((ret = priv_proc_cred_perm(cr, p, NULL, 2150 VREAD | VWRITE)) != 0) { 2151#if defined(sun) 2152 mutex_exit(&p->p_lock); 2153#else 2154 PROC_UNLOCK(p); 2155#endif 2156 return (ret); 2157 } 2158#endif /* notyet */ 2159#if defined(sun) 2160 mutex_exit(&p->p_lock); 2161#else 2162 PROC_UNLOCK(p); 2163#endif 2164 } 2165#endif /* notyet */ 2166 2167 ret = fasttrap_add_probe(probe); 2168err: 2169 kmem_free(probe, size); 2170 2171 return (ret); 2172 2173 } else if (cmd == FASTTRAPIOC_GETINSTR) { 2174 fasttrap_instr_query_t instr; 2175 fasttrap_tracepoint_t *tp; 2176 uint_t index; 2177#if defined(sun) 2178 int ret; 2179#endif 2180 2181#if defined(sun) 2182 if (copyin((void *)arg, &instr, sizeof (instr)) != 0) 2183 return (EFAULT); 2184#endif 2185 2186#ifdef notyet 2187 if (!PRIV_POLICY_CHOICE(cr, PRIV_ALL, B_FALSE)) { 2188 proc_t *p; 2189 pid_t pid = instr.ftiq_pid; 2190 2191#if defined(sun) 2192 mutex_enter(&pidlock); 2193#endif 2194 /* 2195 * Report an error if the process doesn't exist 2196 * or is actively being birthed. 2197 */ 2198 p = pfind(pid); 2199 if (p) 2200 fill_kinfo_proc(p, &kp); 2201 if (p == NULL || kp.ki_stat == SIDL) { 2202#if defined(sun) 2203 mutex_exit(&pidlock); 2204#endif 2205 return (ESRCH); 2206 } 2207#if defined(sun) 2208 mutex_enter(&p->p_lock); 2209 mutex_exit(&pidlock); 2210#else 2211 PROC_LOCK_ASSERT(p, MA_OWNED); 2212#endif 2213 2214#ifdef notyet 2215 if ((ret = priv_proc_cred_perm(cr, p, NULL, 2216 VREAD)) != 0) { 2217#if defined(sun) 2218 mutex_exit(&p->p_lock); 2219#else 2220 PROC_UNLOCK(p); 2221#endif 2222 return (ret); 2223 } 2224#endif /* notyet */ 2225 2226#if defined(sun) 2227 mutex_exit(&p->p_lock); 2228#else 2229 PROC_UNLOCK(p); 2230#endif 2231 } 2232#endif /* notyet */ 2233 2234 index = FASTTRAP_TPOINTS_INDEX(instr.ftiq_pid, instr.ftiq_pc); 2235 2236 mutex_enter(&fasttrap_tpoints.fth_table[index].ftb_mtx); 2237 tp = fasttrap_tpoints.fth_table[index].ftb_data; 2238 while (tp != NULL) { 2239 if (instr.ftiq_pid == tp->ftt_pid && 2240 instr.ftiq_pc == tp->ftt_pc && 2241 tp->ftt_proc->ftpc_acount != 0) 2242 break; 2243 2244 tp = tp->ftt_next; 2245 } 2246 2247 if (tp == NULL) { 2248 mutex_exit(&fasttrap_tpoints.fth_table[index].ftb_mtx); 2249 return (ENOENT); 2250 } 2251 2252 bcopy(&tp->ftt_instr, &instr.ftiq_instr, 2253 sizeof (instr.ftiq_instr)); 2254 mutex_exit(&fasttrap_tpoints.fth_table[index].ftb_mtx); 2255 2256 if (copyout(&instr, (void *)arg, sizeof (instr)) != 0) 2257 return (EFAULT); 2258 2259 return (0); 2260 } 2261 2262 return (EINVAL); 2263} 2264 2265static int 2266fasttrap_load(void) 2267{ 2268 ulong_t nent; 2269 int i, ret; 2270 2271 /* Create the /dev/dtrace/fasttrap entry. */ 2272 fasttrap_cdev = make_dev(&fasttrap_cdevsw, 0, UID_ROOT, GID_WHEEL, 0600, 2273 "dtrace/fasttrap"); 2274 2275 mtx_init(&fasttrap_cleanup_mtx, "fasttrap clean", "dtrace", MTX_DEF); 2276 mutex_init(&fasttrap_count_mtx, "fasttrap count mtx", MUTEX_DEFAULT, 2277 NULL); 2278 2279 ret = kproc_create(fasttrap_pid_cleanup_cb, NULL, 2280 &fasttrap_cleanup_proc, 0, 0, "ftcleanup"); 2281 if (ret != 0) { 2282 destroy_dev(fasttrap_cdev); 2283 return (ret); 2284 } 2285 2286 /* 2287 * Install our hooks into fork(2), exec(2), and exit(2). 2288 */ 2289 dtrace_fasttrap_fork = &fasttrap_fork; 2290 dtrace_fasttrap_exit = &fasttrap_exec_exit; 2291 dtrace_fasttrap_exec = &fasttrap_exec_exit; 2292 2293#if defined(sun) 2294 fasttrap_max = ddi_getprop(DDI_DEV_T_ANY, devi, DDI_PROP_DONTPASS, 2295 "fasttrap-max-probes", FASTTRAP_MAX_DEFAULT); 2296#else 2297 fasttrap_max = FASTTRAP_MAX_DEFAULT; 2298#endif 2299 fasttrap_total = 0; 2300 2301 /* 2302 * Conjure up the tracepoints hashtable... 2303 */ 2304#if defined(sun) 2305 nent = ddi_getprop(DDI_DEV_T_ANY, devi, DDI_PROP_DONTPASS, 2306 "fasttrap-hash-size", FASTTRAP_TPOINTS_DEFAULT_SIZE); 2307#else 2308 nent = FASTTRAP_TPOINTS_DEFAULT_SIZE; 2309#endif 2310 2311 if (nent == 0 || nent > 0x1000000) 2312 nent = FASTTRAP_TPOINTS_DEFAULT_SIZE; 2313 2314 if ((nent & (nent - 1)) == 0) 2315 fasttrap_tpoints.fth_nent = nent; 2316 else 2317 fasttrap_tpoints.fth_nent = 1 << fasttrap_highbit(nent); 2318 ASSERT(fasttrap_tpoints.fth_nent > 0); 2319 fasttrap_tpoints.fth_mask = fasttrap_tpoints.fth_nent - 1; 2320 fasttrap_tpoints.fth_table = kmem_zalloc(fasttrap_tpoints.fth_nent * 2321 sizeof (fasttrap_bucket_t), KM_SLEEP); 2322#if !defined(sun) 2323 for (i = 0; i < fasttrap_tpoints.fth_nent; i++) 2324 mutex_init(&fasttrap_tpoints.fth_table[i].ftb_mtx, 2325 "tracepoints bucket mtx", MUTEX_DEFAULT, NULL); 2326#endif 2327 2328 /* 2329 * ... and the providers hash table... 2330 */ 2331 nent = FASTTRAP_PROVIDERS_DEFAULT_SIZE; 2332 if ((nent & (nent - 1)) == 0) 2333 fasttrap_provs.fth_nent = nent; 2334 else 2335 fasttrap_provs.fth_nent = 1 << fasttrap_highbit(nent); 2336 ASSERT(fasttrap_provs.fth_nent > 0); 2337 fasttrap_provs.fth_mask = fasttrap_provs.fth_nent - 1; 2338 fasttrap_provs.fth_table = kmem_zalloc(fasttrap_provs.fth_nent * 2339 sizeof (fasttrap_bucket_t), KM_SLEEP); 2340#if !defined(sun) 2341 for (i = 0; i < fasttrap_provs.fth_nent; i++) 2342 mutex_init(&fasttrap_provs.fth_table[i].ftb_mtx, 2343 "providers bucket mtx", MUTEX_DEFAULT, NULL); 2344#endif 2345 2346 /* 2347 * ... and the procs hash table. 2348 */ 2349 nent = FASTTRAP_PROCS_DEFAULT_SIZE; 2350 if ((nent & (nent - 1)) == 0) 2351 fasttrap_procs.fth_nent = nent; 2352 else 2353 fasttrap_procs.fth_nent = 1 << fasttrap_highbit(nent); 2354 ASSERT(fasttrap_procs.fth_nent > 0); 2355 fasttrap_procs.fth_mask = fasttrap_procs.fth_nent - 1; 2356 fasttrap_procs.fth_table = kmem_zalloc(fasttrap_procs.fth_nent * 2357 sizeof (fasttrap_bucket_t), KM_SLEEP); 2358#if !defined(sun) 2359 for (i = 0; i < fasttrap_procs.fth_nent; i++) 2360 mutex_init(&fasttrap_procs.fth_table[i].ftb_mtx, 2361 "processes bucket mtx", MUTEX_DEFAULT, NULL); 2362 2363 CPU_FOREACH(i) { 2364 mutex_init(&fasttrap_cpuc_pid_lock[i], "fasttrap barrier", 2365 MUTEX_DEFAULT, NULL); 2366 } 2367#endif 2368 2369 (void) dtrace_meta_register("fasttrap", &fasttrap_mops, NULL, 2370 &fasttrap_meta_id); 2371 2372 return (0); 2373} 2374 2375static int 2376fasttrap_unload(void) 2377{ 2378 int i, fail = 0; 2379 2380 /* 2381 * Unregister the meta-provider to make sure no new fasttrap- 2382 * managed providers come along while we're trying to close up 2383 * shop. If we fail to detach, we'll need to re-register as a 2384 * meta-provider. We can fail to unregister as a meta-provider 2385 * if providers we manage still exist. 2386 */ 2387 if (fasttrap_meta_id != DTRACE_METAPROVNONE && 2388 dtrace_meta_unregister(fasttrap_meta_id) != 0) 2389 return (-1); 2390 2391 /* 2392 * Iterate over all of our providers. If there's still a process 2393 * that corresponds to that pid, fail to detach. 2394 */ 2395 for (i = 0; i < fasttrap_provs.fth_nent; i++) { 2396 fasttrap_provider_t **fpp, *fp; 2397 fasttrap_bucket_t *bucket = &fasttrap_provs.fth_table[i]; 2398 2399 mutex_enter(&bucket->ftb_mtx); 2400 fpp = (fasttrap_provider_t **)&bucket->ftb_data; 2401 while ((fp = *fpp) != NULL) { 2402 /* 2403 * Acquire and release the lock as a simple way of 2404 * waiting for any other consumer to finish with 2405 * this provider. A thread must first acquire the 2406 * bucket lock so there's no chance of another thread 2407 * blocking on the provider's lock. 2408 */ 2409 mutex_enter(&fp->ftp_mtx); 2410 mutex_exit(&fp->ftp_mtx); 2411 2412 if (dtrace_unregister(fp->ftp_provid) != 0) { 2413 fail = 1; 2414 fpp = &fp->ftp_next; 2415 } else { 2416 *fpp = fp->ftp_next; 2417 fasttrap_provider_free(fp); 2418 } 2419 } 2420 2421 mutex_exit(&bucket->ftb_mtx); 2422 } 2423 2424 if (fail) { 2425 (void) dtrace_meta_register("fasttrap", &fasttrap_mops, NULL, 2426 &fasttrap_meta_id); 2427 2428 return (-1); 2429 } 2430 2431 mtx_lock(&fasttrap_cleanup_mtx); 2432 fasttrap_cleanup_drain = 1; 2433 /* Wait for the cleanup thread to finish up and signal us. */ 2434 wakeup(&fasttrap_cleanup_cv); 2435 mtx_sleep(&fasttrap_cleanup_drain, &fasttrap_cleanup_mtx, 0, "ftcld", 2436 0); 2437 fasttrap_cleanup_proc = NULL;
| 24 */ 25 26/* 27 * Copyright 2008 Sun Microsystems, Inc. All rights reserved. 28 * Use is subject to license terms. 29 */ 30 31#if defined(sun) 32#pragma ident "%Z%%M% %I% %E% SMI" 33#endif 34 35#include <sys/atomic.h> 36#include <sys/errno.h> 37#include <sys/stat.h> 38#include <sys/modctl.h> 39#include <sys/conf.h> 40#include <sys/systm.h> 41#if defined(sun) 42#include <sys/ddi.h> 43#endif 44#include <sys/sunddi.h> 45#include <sys/cpuvar.h> 46#include <sys/kmem.h> 47#if defined(sun) 48#include <sys/strsubr.h> 49#endif 50#include <sys/fasttrap.h> 51#include <sys/fasttrap_impl.h> 52#include <sys/fasttrap_isa.h> 53#include <sys/dtrace.h> 54#include <sys/dtrace_impl.h> 55#include <sys/sysmacros.h> 56#include <sys/proc.h> 57#include <sys/policy.h> 58#if defined(sun) 59#include <util/qsort.h> 60#endif 61#include <sys/mutex.h> 62#include <sys/kernel.h> 63#if !defined(sun) 64#include <sys/user.h> 65#include <sys/dtrace_bsd.h> 66#include <cddl/dev/dtrace/dtrace_cddl.h> 67#endif 68 69/* 70 * User-Land Trap-Based Tracing 71 * ---------------------------- 72 * 73 * The fasttrap provider allows DTrace consumers to instrument any user-level 74 * instruction to gather data; this includes probes with semantic 75 * signifigance like entry and return as well as simple offsets into the 76 * function. While the specific techniques used are very ISA specific, the 77 * methodology is generalizable to any architecture. 78 * 79 * 80 * The General Methodology 81 * ----------------------- 82 * 83 * With the primary goal of tracing every user-land instruction and the 84 * limitation that we can't trust user space so don't want to rely on much 85 * information there, we begin by replacing the instructions we want to trace 86 * with trap instructions. Each instruction we overwrite is saved into a hash 87 * table keyed by process ID and pc address. When we enter the kernel due to 88 * this trap instruction, we need the effects of the replaced instruction to 89 * appear to have occurred before we proceed with the user thread's 90 * execution. 91 * 92 * Each user level thread is represented by a ulwp_t structure which is 93 * always easily accessible through a register. The most basic way to produce 94 * the effects of the instruction we replaced is to copy that instruction out 95 * to a bit of scratch space reserved in the user thread's ulwp_t structure 96 * (a sort of kernel-private thread local storage), set the PC to that 97 * scratch space and single step. When we reenter the kernel after single 98 * stepping the instruction we must then adjust the PC to point to what would 99 * normally be the next instruction. Of course, special care must be taken 100 * for branches and jumps, but these represent such a small fraction of any 101 * instruction set that writing the code to emulate these in the kernel is 102 * not too difficult. 103 * 104 * Return probes may require several tracepoints to trace every return site, 105 * and, conversely, each tracepoint may activate several probes (the entry 106 * and offset 0 probes, for example). To solve this muliplexing problem, 107 * tracepoints contain lists of probes to activate and probes contain lists 108 * of tracepoints to enable. If a probe is activated, it adds its ID to 109 * existing tracepoints or creates new ones as necessary. 110 * 111 * Most probes are activated _before_ the instruction is executed, but return 112 * probes are activated _after_ the effects of the last instruction of the 113 * function are visible. Return probes must be fired _after_ we have 114 * single-stepped the instruction whereas all other probes are fired 115 * beforehand. 116 * 117 * 118 * Lock Ordering 119 * ------------- 120 * 121 * The lock ordering below -- both internally and with respect to the DTrace 122 * framework -- is a little tricky and bears some explanation. Each provider 123 * has a lock (ftp_mtx) that protects its members including reference counts 124 * for enabled probes (ftp_rcount), consumers actively creating probes 125 * (ftp_ccount) and USDT consumers (ftp_mcount); all three prevent a provider 126 * from being freed. A provider is looked up by taking the bucket lock for the 127 * provider hash table, and is returned with its lock held. The provider lock 128 * may be taken in functions invoked by the DTrace framework, but may not be 129 * held while calling functions in the DTrace framework. 130 * 131 * To ensure consistency over multiple calls to the DTrace framework, the 132 * creation lock (ftp_cmtx) should be held. Naturally, the creation lock may 133 * not be taken when holding the provider lock as that would create a cyclic 134 * lock ordering. In situations where one would naturally take the provider 135 * lock and then the creation lock, we instead up a reference count to prevent 136 * the provider from disappearing, drop the provider lock, and acquire the 137 * creation lock. 138 * 139 * Briefly: 140 * bucket lock before provider lock 141 * DTrace before provider lock 142 * creation lock before DTrace 143 * never hold the provider lock and creation lock simultaneously 144 */ 145 146static d_open_t fasttrap_open; 147static d_ioctl_t fasttrap_ioctl; 148 149static struct cdevsw fasttrap_cdevsw = { 150 .d_version = D_VERSION, 151 .d_open = fasttrap_open, 152 .d_ioctl = fasttrap_ioctl, 153 .d_name = "fasttrap", 154}; 155static struct cdev *fasttrap_cdev; 156static dtrace_meta_provider_id_t fasttrap_meta_id; 157 158static struct proc *fasttrap_cleanup_proc; 159static struct mtx fasttrap_cleanup_mtx; 160static uint_t fasttrap_cleanup_work, fasttrap_cleanup_drain, fasttrap_cleanup_cv; 161 162/* 163 * Generation count on modifications to the global tracepoint lookup table. 164 */ 165static volatile uint64_t fasttrap_mod_gen; 166 167/* 168 * When the fasttrap provider is loaded, fasttrap_max is set to either 169 * FASTTRAP_MAX_DEFAULT or the value for fasttrap-max-probes in the 170 * fasttrap.conf file. Each time a probe is created, fasttrap_total is 171 * incremented by the number of tracepoints that may be associated with that 172 * probe; fasttrap_total is capped at fasttrap_max. 173 */ 174#define FASTTRAP_MAX_DEFAULT 250000 175static uint32_t fasttrap_max; 176static uint32_t fasttrap_total; 177 178/* 179 * Copyright (c) 2011, Joyent, Inc. All rights reserved. 180 */ 181 182#define FASTTRAP_TPOINTS_DEFAULT_SIZE 0x4000 183#define FASTTRAP_PROVIDERS_DEFAULT_SIZE 0x100 184#define FASTTRAP_PROCS_DEFAULT_SIZE 0x100 185 186#define FASTTRAP_PID_NAME "pid" 187 188fasttrap_hash_t fasttrap_tpoints; 189static fasttrap_hash_t fasttrap_provs; 190static fasttrap_hash_t fasttrap_procs; 191 192static uint64_t fasttrap_pid_count; /* pid ref count */ 193static kmutex_t fasttrap_count_mtx; /* lock on ref count */ 194 195#define FASTTRAP_ENABLE_FAIL 1 196#define FASTTRAP_ENABLE_PARTIAL 2 197 198static int fasttrap_tracepoint_enable(proc_t *, fasttrap_probe_t *, uint_t); 199static void fasttrap_tracepoint_disable(proc_t *, fasttrap_probe_t *, uint_t); 200 201static fasttrap_provider_t *fasttrap_provider_lookup(pid_t, const char *, 202 const dtrace_pattr_t *); 203static void fasttrap_provider_retire(pid_t, const char *, int); 204static void fasttrap_provider_free(fasttrap_provider_t *); 205 206static fasttrap_proc_t *fasttrap_proc_lookup(pid_t); 207static void fasttrap_proc_release(fasttrap_proc_t *); 208 209#define FASTTRAP_PROVS_INDEX(pid, name) \ 210 ((fasttrap_hash_str(name) + (pid)) & fasttrap_provs.fth_mask) 211 212#define FASTTRAP_PROCS_INDEX(pid) ((pid) & fasttrap_procs.fth_mask) 213 214#if !defined(sun) 215static kmutex_t fasttrap_cpuc_pid_lock[MAXCPU]; 216#endif 217 218static int 219fasttrap_highbit(ulong_t i) 220{ 221 int h = 1; 222 223 if (i == 0) 224 return (0); 225#ifdef _LP64 226 if (i & 0xffffffff00000000ul) { 227 h += 32; i >>= 32; 228 } 229#endif 230 if (i & 0xffff0000) { 231 h += 16; i >>= 16; 232 } 233 if (i & 0xff00) { 234 h += 8; i >>= 8; 235 } 236 if (i & 0xf0) { 237 h += 4; i >>= 4; 238 } 239 if (i & 0xc) { 240 h += 2; i >>= 2; 241 } 242 if (i & 0x2) { 243 h += 1; 244 } 245 return (h); 246} 247 248static uint_t 249fasttrap_hash_str(const char *p) 250{ 251 unsigned int g; 252 uint_t hval = 0; 253 254 while (*p) { 255 hval = (hval << 4) + *p++; 256 if ((g = (hval & 0xf0000000)) != 0) 257 hval ^= g >> 24; 258 hval &= ~g; 259 } 260 return (hval); 261} 262 263void 264fasttrap_sigtrap(proc_t *p, kthread_t *t, uintptr_t pc) 265{ 266#if defined(sun) 267 sigqueue_t *sqp = kmem_zalloc(sizeof (sigqueue_t), KM_SLEEP); 268 269 sqp->sq_info.si_signo = SIGTRAP; 270 sqp->sq_info.si_code = TRAP_DTRACE; 271 sqp->sq_info.si_addr = (caddr_t)pc; 272 273 mutex_enter(&p->p_lock); 274 sigaddqa(p, t, sqp); 275 mutex_exit(&p->p_lock); 276 277 if (t != NULL) 278 aston(t); 279#else 280 ksiginfo_t *ksi = kmem_zalloc(sizeof (ksiginfo_t), KM_SLEEP); 281 282 ksiginfo_init(ksi); 283 ksi->ksi_signo = SIGTRAP; 284 ksi->ksi_code = TRAP_DTRACE; 285 ksi->ksi_addr = (caddr_t)pc; 286 PROC_LOCK(p); 287 (void) tdksignal(t, SIGTRAP, ksi); 288 PROC_UNLOCK(p); 289#endif 290} 291 292/* 293 * This function ensures that no threads are actively using the memory 294 * associated with probes that were formerly live. 295 */ 296static void 297fasttrap_mod_barrier(uint64_t gen) 298{ 299 int i; 300 301 if (gen < fasttrap_mod_gen) 302 return; 303 304 fasttrap_mod_gen++; 305 306 CPU_FOREACH(i) { 307 mutex_enter(&fasttrap_cpuc_pid_lock[i]); 308 mutex_exit(&fasttrap_cpuc_pid_lock[i]); 309 } 310} 311 312/* 313 * This function performs asynchronous cleanup of fasttrap providers. The 314 * Solaris implementation of this mechanism use a timeout that's activated in 315 * fasttrap_pid_cleanup(), but this doesn't work in FreeBSD: one may sleep while 316 * holding the DTrace mutexes, but it is unsafe to sleep in a callout handler. 317 * Thus we use a dedicated process to perform the cleanup when requested. 318 */ 319/*ARGSUSED*/ 320static void 321fasttrap_pid_cleanup_cb(void *data) 322{ 323 fasttrap_provider_t **fpp, *fp; 324 fasttrap_bucket_t *bucket; 325 dtrace_provider_id_t provid; 326 int i, later = 0, rval; 327 328 mtx_lock(&fasttrap_cleanup_mtx); 329 while (!fasttrap_cleanup_drain || later > 0) { 330 fasttrap_cleanup_work = 0; 331 mtx_unlock(&fasttrap_cleanup_mtx); 332 333 later = 0; 334 335 /* 336 * Iterate over all the providers trying to remove the marked 337 * ones. If a provider is marked but not retired, we just 338 * have to take a crack at removing it -- it's no big deal if 339 * we can't. 340 */ 341 for (i = 0; i < fasttrap_provs.fth_nent; i++) { 342 bucket = &fasttrap_provs.fth_table[i]; 343 mutex_enter(&bucket->ftb_mtx); 344 fpp = (fasttrap_provider_t **)&bucket->ftb_data; 345 346 while ((fp = *fpp) != NULL) { 347 if (!fp->ftp_marked) { 348 fpp = &fp->ftp_next; 349 continue; 350 } 351 352 mutex_enter(&fp->ftp_mtx); 353 354 /* 355 * If this provider has consumers actively 356 * creating probes (ftp_ccount) or is a USDT 357 * provider (ftp_mcount), we can't unregister 358 * or even condense. 359 */ 360 if (fp->ftp_ccount != 0 || 361 fp->ftp_mcount != 0) { 362 mutex_exit(&fp->ftp_mtx); 363 fp->ftp_marked = 0; 364 continue; 365 } 366 367 if (!fp->ftp_retired || fp->ftp_rcount != 0) 368 fp->ftp_marked = 0; 369 370 mutex_exit(&fp->ftp_mtx); 371 372 /* 373 * If we successfully unregister this 374 * provider we can remove it from the hash 375 * chain and free the memory. If our attempt 376 * to unregister fails and this is a retired 377 * provider, increment our flag to try again 378 * pretty soon. If we've consumed more than 379 * half of our total permitted number of 380 * probes call dtrace_condense() to try to 381 * clean out the unenabled probes. 382 */ 383 provid = fp->ftp_provid; 384 if ((rval = dtrace_unregister(provid)) != 0) { 385 if (fasttrap_total > fasttrap_max / 2) 386 (void) dtrace_condense(provid); 387 388 if (rval == EAGAIN) 389 fp->ftp_marked = 1; 390 391 later += fp->ftp_marked; 392 fpp = &fp->ftp_next; 393 } else { 394 *fpp = fp->ftp_next; 395 fasttrap_provider_free(fp); 396 } 397 } 398 mutex_exit(&bucket->ftb_mtx); 399 } 400 mtx_lock(&fasttrap_cleanup_mtx); 401 402 /* 403 * If we were unable to retire a provider, try again after a 404 * second. This situation can occur in certain circumstances 405 * where providers cannot be unregistered even though they have 406 * no probes enabled because of an execution of dtrace -l or 407 * something similar. 408 */ 409 if (later > 0 || fasttrap_cleanup_work || 410 fasttrap_cleanup_drain) { 411 mtx_unlock(&fasttrap_cleanup_mtx); 412 pause("ftclean", hz); 413 mtx_lock(&fasttrap_cleanup_mtx); 414 } else 415 mtx_sleep(&fasttrap_cleanup_cv, &fasttrap_cleanup_mtx, 416 0, "ftcl", 0); 417 } 418 419 /* 420 * Wake up the thread in fasttrap_unload() now that we're done. 421 */ 422 wakeup(&fasttrap_cleanup_drain); 423 mtx_unlock(&fasttrap_cleanup_mtx); 424 425 kthread_exit(); 426} 427 428/* 429 * Activates the asynchronous cleanup mechanism. 430 */ 431static void 432fasttrap_pid_cleanup(void) 433{ 434 435 mtx_lock(&fasttrap_cleanup_mtx); 436 if (!fasttrap_cleanup_work) { 437 fasttrap_cleanup_work = 1; 438 wakeup(&fasttrap_cleanup_cv); 439 } 440 mtx_unlock(&fasttrap_cleanup_mtx); 441} 442 443/* 444 * This is called from cfork() via dtrace_fasttrap_fork(). The child 445 * process's address space is (roughly) a copy of the parent process's so 446 * we have to remove all the instrumentation we had previously enabled in the 447 * parent. 448 */ 449static void 450fasttrap_fork(proc_t *p, proc_t *cp) 451{ 452 pid_t ppid = p->p_pid; 453 int i; 454 455#if defined(sun) 456 ASSERT(curproc == p); 457 ASSERT(p->p_proc_flag & P_PR_LOCK); 458#else 459 PROC_LOCK_ASSERT(p, MA_OWNED); 460#endif 461#if defined(sun) 462 ASSERT(p->p_dtrace_count > 0); 463#else 464 if (p->p_dtrace_helpers) { 465 /* 466 * dtrace_helpers_duplicate() allocates memory. 467 */ 468 _PHOLD(cp); 469 PROC_UNLOCK(p); 470 PROC_UNLOCK(cp); 471 dtrace_helpers_duplicate(p, cp); 472 PROC_LOCK(cp); 473 PROC_LOCK(p); 474 _PRELE(cp); 475 } 476 /* 477 * This check is purposely here instead of in kern_fork.c because, 478 * for legal resons, we cannot include the dtrace_cddl.h header 479 * inside kern_fork.c and insert if-clause there. 480 */ 481 if (p->p_dtrace_count == 0) 482 return; 483#endif 484 ASSERT(cp->p_dtrace_count == 0); 485 486 /* 487 * This would be simpler and faster if we maintained per-process 488 * hash tables of enabled tracepoints. It could, however, potentially 489 * slow down execution of a tracepoint since we'd need to go 490 * through two levels of indirection. In the future, we should 491 * consider either maintaining per-process ancillary lists of 492 * enabled tracepoints or hanging a pointer to a per-process hash 493 * table of enabled tracepoints off the proc structure. 494 */ 495 496 /* 497 * We don't have to worry about the child process disappearing 498 * because we're in fork(). 499 */ 500#if defined(sun) 501 mtx_lock_spin(&cp->p_slock); 502 sprlock_proc(cp); 503 mtx_unlock_spin(&cp->p_slock); 504#else 505 _PHOLD(cp); 506#endif 507 508 /* 509 * Iterate over every tracepoint looking for ones that belong to the 510 * parent process, and remove each from the child process. 511 */ 512 for (i = 0; i < fasttrap_tpoints.fth_nent; i++) { 513 fasttrap_tracepoint_t *tp; 514 fasttrap_bucket_t *bucket = &fasttrap_tpoints.fth_table[i]; 515 516 mutex_enter(&bucket->ftb_mtx); 517 for (tp = bucket->ftb_data; tp != NULL; tp = tp->ftt_next) { 518 if (tp->ftt_pid == ppid && 519 tp->ftt_proc->ftpc_acount != 0) { 520 int ret = fasttrap_tracepoint_remove(cp, tp); 521 ASSERT(ret == 0); 522 523 /* 524 * The count of active providers can only be 525 * decremented (i.e. to zero) during exec, 526 * exit, and removal of a meta provider so it 527 * should be impossible to drop the count 528 * mid-fork. 529 */ 530 ASSERT(tp->ftt_proc->ftpc_acount != 0); 531 } 532 } 533 mutex_exit(&bucket->ftb_mtx); 534 } 535 536#if defined(sun) 537 mutex_enter(&cp->p_lock); 538 sprunlock(cp); 539#else 540 _PRELE(cp); 541#endif 542} 543 544/* 545 * This is called from proc_exit() or from exec_common() if p_dtrace_probes 546 * is set on the proc structure to indicate that there is a pid provider 547 * associated with this process. 548 */ 549static void 550fasttrap_exec_exit(proc_t *p) 551{ 552#if defined(sun) 553 ASSERT(p == curproc); 554#endif 555 PROC_LOCK_ASSERT(p, MA_OWNED); 556 _PHOLD(p); 557 PROC_UNLOCK(p); 558 559 /* 560 * We clean up the pid provider for this process here; user-land 561 * static probes are handled by the meta-provider remove entry point. 562 */ 563 fasttrap_provider_retire(p->p_pid, FASTTRAP_PID_NAME, 0); 564#if !defined(sun) 565 if (p->p_dtrace_helpers) 566 dtrace_helpers_destroy(p); 567#endif 568 PROC_LOCK(p); 569 _PRELE(p); 570} 571 572 573/*ARGSUSED*/ 574static void 575fasttrap_pid_provide(void *arg, dtrace_probedesc_t *desc) 576{ 577 /* 578 * There are no "default" pid probes. 579 */ 580} 581 582static int 583fasttrap_tracepoint_enable(proc_t *p, fasttrap_probe_t *probe, uint_t index) 584{ 585 fasttrap_tracepoint_t *tp, *new_tp = NULL; 586 fasttrap_bucket_t *bucket; 587 fasttrap_id_t *id; 588 pid_t pid; 589 uintptr_t pc; 590 591 ASSERT(index < probe->ftp_ntps); 592 593 pid = probe->ftp_pid; 594 pc = probe->ftp_tps[index].fit_tp->ftt_pc; 595 id = &probe->ftp_tps[index].fit_id; 596 597 ASSERT(probe->ftp_tps[index].fit_tp->ftt_pid == pid); 598 599#if defined(sun) 600 ASSERT(!(p->p_flag & SVFORK)); 601#endif 602 603 /* 604 * Before we make any modifications, make sure we've imposed a barrier 605 * on the generation in which this probe was last modified. 606 */ 607 fasttrap_mod_barrier(probe->ftp_gen); 608 609 bucket = &fasttrap_tpoints.fth_table[FASTTRAP_TPOINTS_INDEX(pid, pc)]; 610 611 /* 612 * If the tracepoint has already been enabled, just add our id to the 613 * list of interested probes. This may be our second time through 614 * this path in which case we'll have constructed the tracepoint we'd 615 * like to install. If we can't find a match, and have an allocated 616 * tracepoint ready to go, enable that one now. 617 * 618 * A tracepoint whose process is defunct is also considered defunct. 619 */ 620again: 621 mutex_enter(&bucket->ftb_mtx); 622 for (tp = bucket->ftb_data; tp != NULL; tp = tp->ftt_next) { 623 /* 624 * Note that it's safe to access the active count on the 625 * associated proc structure because we know that at least one 626 * provider (this one) will still be around throughout this 627 * operation. 628 */ 629 if (tp->ftt_pid != pid || tp->ftt_pc != pc || 630 tp->ftt_proc->ftpc_acount == 0) 631 continue; 632 633 /* 634 * Now that we've found a matching tracepoint, it would be 635 * a decent idea to confirm that the tracepoint is still 636 * enabled and the trap instruction hasn't been overwritten. 637 * Since this is a little hairy, we'll punt for now. 638 */ 639 640 /* 641 * This can't be the first interested probe. We don't have 642 * to worry about another thread being in the midst of 643 * deleting this tracepoint (which would be the only valid 644 * reason for a tracepoint to have no interested probes) 645 * since we're holding P_PR_LOCK for this process. 646 */ 647 ASSERT(tp->ftt_ids != NULL || tp->ftt_retids != NULL); 648 649 switch (id->fti_ptype) { 650 case DTFTP_ENTRY: 651 case DTFTP_OFFSETS: 652 case DTFTP_IS_ENABLED: 653 id->fti_next = tp->ftt_ids; 654 membar_producer(); 655 tp->ftt_ids = id; 656 membar_producer(); 657 break; 658 659 case DTFTP_RETURN: 660 case DTFTP_POST_OFFSETS: 661 id->fti_next = tp->ftt_retids; 662 membar_producer(); 663 tp->ftt_retids = id; 664 membar_producer(); 665 break; 666 667 default: 668 ASSERT(0); 669 } 670 671 mutex_exit(&bucket->ftb_mtx); 672 673 if (new_tp != NULL) { 674 new_tp->ftt_ids = NULL; 675 new_tp->ftt_retids = NULL; 676 } 677 678 return (0); 679 } 680 681 /* 682 * If we have a good tracepoint ready to go, install it now while 683 * we have the lock held and no one can screw with us. 684 */ 685 if (new_tp != NULL) { 686 int rc = 0; 687 688 new_tp->ftt_next = bucket->ftb_data; 689 membar_producer(); 690 bucket->ftb_data = new_tp; 691 membar_producer(); 692 mutex_exit(&bucket->ftb_mtx); 693 694 /* 695 * Activate the tracepoint in the ISA-specific manner. 696 * If this fails, we need to report the failure, but 697 * indicate that this tracepoint must still be disabled 698 * by calling fasttrap_tracepoint_disable(). 699 */ 700 if (fasttrap_tracepoint_install(p, new_tp) != 0) 701 rc = FASTTRAP_ENABLE_PARTIAL; 702 703 /* 704 * Increment the count of the number of tracepoints active in 705 * the victim process. 706 */ 707#if defined(sun) 708 ASSERT(p->p_proc_flag & P_PR_LOCK); 709#endif 710 p->p_dtrace_count++; 711 712 return (rc); 713 } 714 715 mutex_exit(&bucket->ftb_mtx); 716 717 /* 718 * Initialize the tracepoint that's been preallocated with the probe. 719 */ 720 new_tp = probe->ftp_tps[index].fit_tp; 721 722 ASSERT(new_tp->ftt_pid == pid); 723 ASSERT(new_tp->ftt_pc == pc); 724 ASSERT(new_tp->ftt_proc == probe->ftp_prov->ftp_proc); 725 ASSERT(new_tp->ftt_ids == NULL); 726 ASSERT(new_tp->ftt_retids == NULL); 727 728 switch (id->fti_ptype) { 729 case DTFTP_ENTRY: 730 case DTFTP_OFFSETS: 731 case DTFTP_IS_ENABLED: 732 id->fti_next = NULL; 733 new_tp->ftt_ids = id; 734 break; 735 736 case DTFTP_RETURN: 737 case DTFTP_POST_OFFSETS: 738 id->fti_next = NULL; 739 new_tp->ftt_retids = id; 740 break; 741 742 default: 743 ASSERT(0); 744 } 745 746 /* 747 * If the ISA-dependent initialization goes to plan, go back to the 748 * beginning and try to install this freshly made tracepoint. 749 */ 750 if (fasttrap_tracepoint_init(p, new_tp, pc, id->fti_ptype) == 0) 751 goto again; 752 753 new_tp->ftt_ids = NULL; 754 new_tp->ftt_retids = NULL; 755 756 return (FASTTRAP_ENABLE_FAIL); 757} 758 759static void 760fasttrap_tracepoint_disable(proc_t *p, fasttrap_probe_t *probe, uint_t index) 761{ 762 fasttrap_bucket_t *bucket; 763 fasttrap_provider_t *provider = probe->ftp_prov; 764 fasttrap_tracepoint_t **pp, *tp; 765 fasttrap_id_t *id, **idp = NULL; 766 pid_t pid; 767 uintptr_t pc; 768 769 ASSERT(index < probe->ftp_ntps); 770 771 pid = probe->ftp_pid; 772 pc = probe->ftp_tps[index].fit_tp->ftt_pc; 773 id = &probe->ftp_tps[index].fit_id; 774 775 ASSERT(probe->ftp_tps[index].fit_tp->ftt_pid == pid); 776 777 /* 778 * Find the tracepoint and make sure that our id is one of the 779 * ones registered with it. 780 */ 781 bucket = &fasttrap_tpoints.fth_table[FASTTRAP_TPOINTS_INDEX(pid, pc)]; 782 mutex_enter(&bucket->ftb_mtx); 783 for (tp = bucket->ftb_data; tp != NULL; tp = tp->ftt_next) { 784 if (tp->ftt_pid == pid && tp->ftt_pc == pc && 785 tp->ftt_proc == provider->ftp_proc) 786 break; 787 } 788 789 /* 790 * If we somehow lost this tracepoint, we're in a world of hurt. 791 */ 792 ASSERT(tp != NULL); 793 794 switch (id->fti_ptype) { 795 case DTFTP_ENTRY: 796 case DTFTP_OFFSETS: 797 case DTFTP_IS_ENABLED: 798 ASSERT(tp->ftt_ids != NULL); 799 idp = &tp->ftt_ids; 800 break; 801 802 case DTFTP_RETURN: 803 case DTFTP_POST_OFFSETS: 804 ASSERT(tp->ftt_retids != NULL); 805 idp = &tp->ftt_retids; 806 break; 807 808 default: 809 ASSERT(0); 810 } 811 812 while ((*idp)->fti_probe != probe) { 813 idp = &(*idp)->fti_next; 814 ASSERT(*idp != NULL); 815 } 816 817 id = *idp; 818 *idp = id->fti_next; 819 membar_producer(); 820 821 ASSERT(id->fti_probe == probe); 822 823 /* 824 * If there are other registered enablings of this tracepoint, we're 825 * all done, but if this was the last probe assocated with this 826 * this tracepoint, we need to remove and free it. 827 */ 828 if (tp->ftt_ids != NULL || tp->ftt_retids != NULL) { 829 830 /* 831 * If the current probe's tracepoint is in use, swap it 832 * for an unused tracepoint. 833 */ 834 if (tp == probe->ftp_tps[index].fit_tp) { 835 fasttrap_probe_t *tmp_probe; 836 fasttrap_tracepoint_t **tmp_tp; 837 uint_t tmp_index; 838 839 if (tp->ftt_ids != NULL) { 840 tmp_probe = tp->ftt_ids->fti_probe; 841 /* LINTED - alignment */ 842 tmp_index = FASTTRAP_ID_INDEX(tp->ftt_ids); 843 tmp_tp = &tmp_probe->ftp_tps[tmp_index].fit_tp; 844 } else { 845 tmp_probe = tp->ftt_retids->fti_probe; 846 /* LINTED - alignment */ 847 tmp_index = FASTTRAP_ID_INDEX(tp->ftt_retids); 848 tmp_tp = &tmp_probe->ftp_tps[tmp_index].fit_tp; 849 } 850 851 ASSERT(*tmp_tp != NULL); 852 ASSERT(*tmp_tp != probe->ftp_tps[index].fit_tp); 853 ASSERT((*tmp_tp)->ftt_ids == NULL); 854 ASSERT((*tmp_tp)->ftt_retids == NULL); 855 856 probe->ftp_tps[index].fit_tp = *tmp_tp; 857 *tmp_tp = tp; 858 } 859 860 mutex_exit(&bucket->ftb_mtx); 861 862 /* 863 * Tag the modified probe with the generation in which it was 864 * changed. 865 */ 866 probe->ftp_gen = fasttrap_mod_gen; 867 return; 868 } 869 870 mutex_exit(&bucket->ftb_mtx); 871 872 /* 873 * We can't safely remove the tracepoint from the set of active 874 * tracepoints until we've actually removed the fasttrap instruction 875 * from the process's text. We can, however, operate on this 876 * tracepoint secure in the knowledge that no other thread is going to 877 * be looking at it since we hold P_PR_LOCK on the process if it's 878 * live or we hold the provider lock on the process if it's dead and 879 * gone. 880 */ 881 882 /* 883 * We only need to remove the actual instruction if we're looking 884 * at an existing process 885 */ 886 if (p != NULL) { 887 /* 888 * If we fail to restore the instruction we need to kill 889 * this process since it's in a completely unrecoverable 890 * state. 891 */ 892 if (fasttrap_tracepoint_remove(p, tp) != 0) 893 fasttrap_sigtrap(p, NULL, pc); 894 895 /* 896 * Decrement the count of the number of tracepoints active 897 * in the victim process. 898 */ 899#if defined(sun) 900 ASSERT(p->p_proc_flag & P_PR_LOCK); 901#endif 902 p->p_dtrace_count--; 903 } 904 905 /* 906 * Remove the probe from the hash table of active tracepoints. 907 */ 908 mutex_enter(&bucket->ftb_mtx); 909 pp = (fasttrap_tracepoint_t **)&bucket->ftb_data; 910 ASSERT(*pp != NULL); 911 while (*pp != tp) { 912 pp = &(*pp)->ftt_next; 913 ASSERT(*pp != NULL); 914 } 915 916 *pp = tp->ftt_next; 917 membar_producer(); 918 919 mutex_exit(&bucket->ftb_mtx); 920 921 /* 922 * Tag the modified probe with the generation in which it was changed. 923 */ 924 probe->ftp_gen = fasttrap_mod_gen; 925} 926 927static void 928fasttrap_enable_callbacks(void) 929{ 930 /* 931 * We don't have to play the rw lock game here because we're 932 * providing something rather than taking something away -- 933 * we can be sure that no threads have tried to follow this 934 * function pointer yet. 935 */ 936 mutex_enter(&fasttrap_count_mtx); 937 if (fasttrap_pid_count == 0) { 938 ASSERT(dtrace_pid_probe_ptr == NULL); 939 ASSERT(dtrace_return_probe_ptr == NULL); 940 dtrace_pid_probe_ptr = &fasttrap_pid_probe; 941 dtrace_return_probe_ptr = &fasttrap_return_probe; 942 } 943 ASSERT(dtrace_pid_probe_ptr == &fasttrap_pid_probe); 944 ASSERT(dtrace_return_probe_ptr == &fasttrap_return_probe); 945 fasttrap_pid_count++; 946 mutex_exit(&fasttrap_count_mtx); 947} 948 949static void 950fasttrap_disable_callbacks(void) 951{ 952#if defined(sun) 953 ASSERT(MUTEX_HELD(&cpu_lock)); 954#endif 955 956 957 mutex_enter(&fasttrap_count_mtx); 958 ASSERT(fasttrap_pid_count > 0); 959 fasttrap_pid_count--; 960 if (fasttrap_pid_count == 0) { 961#if defined(sun) 962 cpu_t *cur, *cpu = CPU; 963 964 for (cur = cpu->cpu_next_onln; cur != cpu; 965 cur = cur->cpu_next_onln) { 966 rw_enter(&cur->cpu_ft_lock, RW_WRITER); 967 } 968#endif 969 dtrace_pid_probe_ptr = NULL; 970 dtrace_return_probe_ptr = NULL; 971#if defined(sun) 972 for (cur = cpu->cpu_next_onln; cur != cpu; 973 cur = cur->cpu_next_onln) { 974 rw_exit(&cur->cpu_ft_lock); 975 } 976#endif 977 } 978 mutex_exit(&fasttrap_count_mtx); 979} 980 981/*ARGSUSED*/ 982static void 983fasttrap_pid_enable(void *arg, dtrace_id_t id, void *parg) 984{ 985 fasttrap_probe_t *probe = parg; 986 proc_t *p = NULL; 987 int i, rc; 988 989 ASSERT(probe != NULL); 990 ASSERT(!probe->ftp_enabled); 991 ASSERT(id == probe->ftp_id); 992#if defined(sun) 993 ASSERT(MUTEX_HELD(&cpu_lock)); 994#endif 995 996 /* 997 * Increment the count of enabled probes on this probe's provider; 998 * the provider can't go away while the probe still exists. We 999 * must increment this even if we aren't able to properly enable 1000 * this probe. 1001 */ 1002 mutex_enter(&probe->ftp_prov->ftp_mtx); 1003 probe->ftp_prov->ftp_rcount++; 1004 mutex_exit(&probe->ftp_prov->ftp_mtx); 1005 1006 /* 1007 * If this probe's provider is retired (meaning it was valid in a 1008 * previously exec'ed incarnation of this address space), bail out. The 1009 * provider can't go away while we're in this code path. 1010 */ 1011 if (probe->ftp_prov->ftp_retired) 1012 return; 1013 1014 /* 1015 * If we can't find the process, it may be that we're in the context of 1016 * a fork in which the traced process is being born and we're copying 1017 * USDT probes. Otherwise, the process is gone so bail. 1018 */ 1019#if defined(sun) 1020 if ((p = sprlock(probe->ftp_pid)) == NULL) { 1021 if ((curproc->p_flag & SFORKING) == 0) 1022 return; 1023 1024 mutex_enter(&pidlock); 1025 p = prfind(probe->ftp_pid); 1026 1027 /* 1028 * Confirm that curproc is indeed forking the process in which 1029 * we're trying to enable probes. 1030 */ 1031 ASSERT(p != NULL); 1032 ASSERT(p->p_parent == curproc); 1033 ASSERT(p->p_stat == SIDL); 1034 1035 mutex_enter(&p->p_lock); 1036 mutex_exit(&pidlock); 1037 1038 sprlock_proc(p); 1039 } 1040 1041 ASSERT(!(p->p_flag & SVFORK)); 1042 mutex_exit(&p->p_lock); 1043#else 1044 if ((p = pfind(probe->ftp_pid)) == NULL) 1045 return; 1046#endif 1047 1048 /* 1049 * We have to enable the trap entry point before any user threads have 1050 * the chance to execute the trap instruction we're about to place 1051 * in their process's text. 1052 */ 1053#ifdef __FreeBSD__ 1054 /* 1055 * pfind() returns a locked process. 1056 */ 1057 _PHOLD(p); 1058 PROC_UNLOCK(p); 1059#endif 1060 fasttrap_enable_callbacks(); 1061 1062 /* 1063 * Enable all the tracepoints and add this probe's id to each 1064 * tracepoint's list of active probes. 1065 */ 1066 for (i = 0; i < probe->ftp_ntps; i++) { 1067 if ((rc = fasttrap_tracepoint_enable(p, probe, i)) != 0) { 1068 /* 1069 * If enabling the tracepoint failed completely, 1070 * we don't have to disable it; if the failure 1071 * was only partial we must disable it. 1072 */ 1073 if (rc == FASTTRAP_ENABLE_FAIL) 1074 i--; 1075 else 1076 ASSERT(rc == FASTTRAP_ENABLE_PARTIAL); 1077 1078 /* 1079 * Back up and pull out all the tracepoints we've 1080 * created so far for this probe. 1081 */ 1082 while (i >= 0) { 1083 fasttrap_tracepoint_disable(p, probe, i); 1084 i--; 1085 } 1086 1087#if defined(sun) 1088 mutex_enter(&p->p_lock); 1089 sprunlock(p); 1090#else 1091 PRELE(p); 1092#endif 1093 1094 /* 1095 * Since we're not actually enabling this probe, 1096 * drop our reference on the trap table entry. 1097 */ 1098 fasttrap_disable_callbacks(); 1099 return; 1100 } 1101 } 1102#if defined(sun) 1103 mutex_enter(&p->p_lock); 1104 sprunlock(p); 1105#else 1106 PRELE(p); 1107#endif 1108 1109 probe->ftp_enabled = 1; 1110} 1111 1112/*ARGSUSED*/ 1113static void 1114fasttrap_pid_disable(void *arg, dtrace_id_t id, void *parg) 1115{ 1116 fasttrap_probe_t *probe = parg; 1117 fasttrap_provider_t *provider = probe->ftp_prov; 1118 proc_t *p; 1119 int i, whack = 0; 1120 1121 ASSERT(id == probe->ftp_id); 1122 1123 mutex_enter(&provider->ftp_mtx); 1124 1125 /* 1126 * We won't be able to acquire a /proc-esque lock on the process 1127 * iff the process is dead and gone. In this case, we rely on the 1128 * provider lock as a point of mutual exclusion to prevent other 1129 * DTrace consumers from disabling this probe. 1130 */ 1131 if ((p = pfind(probe->ftp_pid)) != NULL) { 1132#ifdef __FreeBSD__ 1133 _PHOLD(p); 1134 PROC_UNLOCK(p); 1135#endif 1136 } 1137 1138 /* 1139 * Disable all the associated tracepoints (for fully enabled probes). 1140 */ 1141 if (probe->ftp_enabled) { 1142 for (i = 0; i < probe->ftp_ntps; i++) { 1143 fasttrap_tracepoint_disable(p, probe, i); 1144 } 1145 } 1146 1147 ASSERT(provider->ftp_rcount > 0); 1148 provider->ftp_rcount--; 1149 1150 if (p != NULL) { 1151 /* 1152 * Even though we may not be able to remove it entirely, we 1153 * mark this retired provider to get a chance to remove some 1154 * of the associated probes. 1155 */ 1156 if (provider->ftp_retired && !provider->ftp_marked) 1157 whack = provider->ftp_marked = 1; 1158 mutex_exit(&provider->ftp_mtx); 1159 } else { 1160 /* 1161 * If the process is dead, we're just waiting for the 1162 * last probe to be disabled to be able to free it. 1163 */ 1164 if (provider->ftp_rcount == 0 && !provider->ftp_marked) 1165 whack = provider->ftp_marked = 1; 1166 mutex_exit(&provider->ftp_mtx); 1167 } 1168 1169 if (whack) 1170 fasttrap_pid_cleanup(); 1171 1172#ifdef __FreeBSD__ 1173 if (p != NULL) 1174 PRELE(p); 1175#endif 1176 if (!probe->ftp_enabled) 1177 return; 1178 1179 probe->ftp_enabled = 0; 1180 1181#if defined(sun) 1182 ASSERT(MUTEX_HELD(&cpu_lock)); 1183#endif 1184 fasttrap_disable_callbacks(); 1185} 1186 1187/*ARGSUSED*/ 1188static void 1189fasttrap_pid_getargdesc(void *arg, dtrace_id_t id, void *parg, 1190 dtrace_argdesc_t *desc) 1191{ 1192 fasttrap_probe_t *probe = parg; 1193 char *str; 1194 int i, ndx; 1195 1196 desc->dtargd_native[0] = '\0'; 1197 desc->dtargd_xlate[0] = '\0'; 1198 1199 if (probe->ftp_prov->ftp_retired != 0 || 1200 desc->dtargd_ndx >= probe->ftp_nargs) { 1201 desc->dtargd_ndx = DTRACE_ARGNONE; 1202 return; 1203 } 1204 1205 ndx = (probe->ftp_argmap != NULL) ? 1206 probe->ftp_argmap[desc->dtargd_ndx] : desc->dtargd_ndx; 1207 1208 str = probe->ftp_ntypes; 1209 for (i = 0; i < ndx; i++) { 1210 str += strlen(str) + 1; 1211 } 1212 1213 ASSERT(strlen(str + 1) < sizeof (desc->dtargd_native)); 1214 (void) strcpy(desc->dtargd_native, str); 1215 1216 if (probe->ftp_xtypes == NULL) 1217 return; 1218 1219 str = probe->ftp_xtypes; 1220 for (i = 0; i < desc->dtargd_ndx; i++) { 1221 str += strlen(str) + 1; 1222 } 1223 1224 ASSERT(strlen(str + 1) < sizeof (desc->dtargd_xlate)); 1225 (void) strcpy(desc->dtargd_xlate, str); 1226} 1227 1228/*ARGSUSED*/ 1229static void 1230fasttrap_pid_destroy(void *arg, dtrace_id_t id, void *parg) 1231{ 1232 fasttrap_probe_t *probe = parg; 1233 int i; 1234 size_t size; 1235 1236 ASSERT(probe != NULL); 1237 ASSERT(!probe->ftp_enabled); 1238 ASSERT(fasttrap_total >= probe->ftp_ntps); 1239 1240 atomic_add_32(&fasttrap_total, -probe->ftp_ntps); 1241 size = offsetof(fasttrap_probe_t, ftp_tps[probe->ftp_ntps]); 1242 1243 if (probe->ftp_gen + 1 >= fasttrap_mod_gen) 1244 fasttrap_mod_barrier(probe->ftp_gen); 1245 1246 for (i = 0; i < probe->ftp_ntps; i++) { 1247 kmem_free(probe->ftp_tps[i].fit_tp, 1248 sizeof (fasttrap_tracepoint_t)); 1249 } 1250 1251 kmem_free(probe, size); 1252} 1253 1254 1255static const dtrace_pattr_t pid_attr = { 1256{ DTRACE_STABILITY_EVOLVING, DTRACE_STABILITY_EVOLVING, DTRACE_CLASS_ISA }, 1257{ DTRACE_STABILITY_PRIVATE, DTRACE_STABILITY_PRIVATE, DTRACE_CLASS_UNKNOWN }, 1258{ DTRACE_STABILITY_PRIVATE, DTRACE_STABILITY_PRIVATE, DTRACE_CLASS_UNKNOWN }, 1259{ DTRACE_STABILITY_EVOLVING, DTRACE_STABILITY_EVOLVING, DTRACE_CLASS_ISA }, 1260{ DTRACE_STABILITY_PRIVATE, DTRACE_STABILITY_PRIVATE, DTRACE_CLASS_UNKNOWN }, 1261}; 1262 1263static dtrace_pops_t pid_pops = { 1264 fasttrap_pid_provide, 1265 NULL, 1266 fasttrap_pid_enable, 1267 fasttrap_pid_disable, 1268 NULL, 1269 NULL, 1270 fasttrap_pid_getargdesc, 1271 fasttrap_pid_getarg, 1272 NULL, 1273 fasttrap_pid_destroy 1274}; 1275 1276static dtrace_pops_t usdt_pops = { 1277 fasttrap_pid_provide, 1278 NULL, 1279 fasttrap_pid_enable, 1280 fasttrap_pid_disable, 1281 NULL, 1282 NULL, 1283 fasttrap_pid_getargdesc, 1284 fasttrap_usdt_getarg, 1285 NULL, 1286 fasttrap_pid_destroy 1287}; 1288 1289static fasttrap_proc_t * 1290fasttrap_proc_lookup(pid_t pid) 1291{ 1292 fasttrap_bucket_t *bucket; 1293 fasttrap_proc_t *fprc, *new_fprc; 1294 1295 1296 bucket = &fasttrap_procs.fth_table[FASTTRAP_PROCS_INDEX(pid)]; 1297 mutex_enter(&bucket->ftb_mtx); 1298 1299 for (fprc = bucket->ftb_data; fprc != NULL; fprc = fprc->ftpc_next) { 1300 if (fprc->ftpc_pid == pid && fprc->ftpc_acount != 0) { 1301 mutex_enter(&fprc->ftpc_mtx); 1302 mutex_exit(&bucket->ftb_mtx); 1303 fprc->ftpc_rcount++; 1304 atomic_add_64(&fprc->ftpc_acount, 1); 1305 ASSERT(fprc->ftpc_acount <= fprc->ftpc_rcount); 1306 mutex_exit(&fprc->ftpc_mtx); 1307 1308 return (fprc); 1309 } 1310 } 1311 1312 /* 1313 * Drop the bucket lock so we don't try to perform a sleeping 1314 * allocation under it. 1315 */ 1316 mutex_exit(&bucket->ftb_mtx); 1317 1318 new_fprc = kmem_zalloc(sizeof (fasttrap_proc_t), KM_SLEEP); 1319 new_fprc->ftpc_pid = pid; 1320 new_fprc->ftpc_rcount = 1; 1321 new_fprc->ftpc_acount = 1; 1322#if !defined(sun) 1323 mutex_init(&new_fprc->ftpc_mtx, "fasttrap proc mtx", MUTEX_DEFAULT, 1324 NULL); 1325#endif 1326 1327 mutex_enter(&bucket->ftb_mtx); 1328 1329 /* 1330 * Take another lap through the list to make sure a proc hasn't 1331 * been created for this pid while we weren't under the bucket lock. 1332 */ 1333 for (fprc = bucket->ftb_data; fprc != NULL; fprc = fprc->ftpc_next) { 1334 if (fprc->ftpc_pid == pid && fprc->ftpc_acount != 0) { 1335 mutex_enter(&fprc->ftpc_mtx); 1336 mutex_exit(&bucket->ftb_mtx); 1337 fprc->ftpc_rcount++; 1338 atomic_add_64(&fprc->ftpc_acount, 1); 1339 ASSERT(fprc->ftpc_acount <= fprc->ftpc_rcount); 1340 mutex_exit(&fprc->ftpc_mtx); 1341 1342 kmem_free(new_fprc, sizeof (fasttrap_proc_t)); 1343 1344 return (fprc); 1345 } 1346 } 1347 1348 new_fprc->ftpc_next = bucket->ftb_data; 1349 bucket->ftb_data = new_fprc; 1350 1351 mutex_exit(&bucket->ftb_mtx); 1352 1353 return (new_fprc); 1354} 1355 1356static void 1357fasttrap_proc_release(fasttrap_proc_t *proc) 1358{ 1359 fasttrap_bucket_t *bucket; 1360 fasttrap_proc_t *fprc, **fprcp; 1361 pid_t pid = proc->ftpc_pid; 1362 1363 mutex_enter(&proc->ftpc_mtx); 1364 1365 ASSERT(proc->ftpc_rcount != 0); 1366 ASSERT(proc->ftpc_acount <= proc->ftpc_rcount); 1367 1368 if (--proc->ftpc_rcount != 0) { 1369 mutex_exit(&proc->ftpc_mtx); 1370 return; 1371 } 1372 1373 mutex_exit(&proc->ftpc_mtx); 1374 1375 /* 1376 * There should definitely be no live providers associated with this 1377 * process at this point. 1378 */ 1379 ASSERT(proc->ftpc_acount == 0); 1380 1381 bucket = &fasttrap_procs.fth_table[FASTTRAP_PROCS_INDEX(pid)]; 1382 mutex_enter(&bucket->ftb_mtx); 1383 1384 fprcp = (fasttrap_proc_t **)&bucket->ftb_data; 1385 while ((fprc = *fprcp) != NULL) { 1386 if (fprc == proc) 1387 break; 1388 1389 fprcp = &fprc->ftpc_next; 1390 } 1391 1392 /* 1393 * Something strange has happened if we can't find the proc. 1394 */ 1395 ASSERT(fprc != NULL); 1396 1397 *fprcp = fprc->ftpc_next; 1398 1399 mutex_exit(&bucket->ftb_mtx); 1400 1401 kmem_free(fprc, sizeof (fasttrap_proc_t)); 1402} 1403 1404/* 1405 * Lookup a fasttrap-managed provider based on its name and associated pid. 1406 * If the pattr argument is non-NULL, this function instantiates the provider 1407 * if it doesn't exist otherwise it returns NULL. The provider is returned 1408 * with its lock held. 1409 */ 1410static fasttrap_provider_t * 1411fasttrap_provider_lookup(pid_t pid, const char *name, 1412 const dtrace_pattr_t *pattr) 1413{ 1414 fasttrap_provider_t *fp, *new_fp = NULL; 1415 fasttrap_bucket_t *bucket; 1416 char provname[DTRACE_PROVNAMELEN]; 1417 proc_t *p; 1418 cred_t *cred; 1419 1420 ASSERT(strlen(name) < sizeof (fp->ftp_name)); 1421 ASSERT(pattr != NULL); 1422 1423 bucket = &fasttrap_provs.fth_table[FASTTRAP_PROVS_INDEX(pid, name)]; 1424 mutex_enter(&bucket->ftb_mtx); 1425 1426 /* 1427 * Take a lap through the list and return the match if we find it. 1428 */ 1429 for (fp = bucket->ftb_data; fp != NULL; fp = fp->ftp_next) { 1430 if (fp->ftp_pid == pid && strcmp(fp->ftp_name, name) == 0 && 1431 !fp->ftp_retired) { 1432 mutex_enter(&fp->ftp_mtx); 1433 mutex_exit(&bucket->ftb_mtx); 1434 return (fp); 1435 } 1436 } 1437 1438 /* 1439 * Drop the bucket lock so we don't try to perform a sleeping 1440 * allocation under it. 1441 */ 1442 mutex_exit(&bucket->ftb_mtx); 1443 1444 /* 1445 * Make sure the process exists, isn't a child created as the result 1446 * of a vfork(2), and isn't a zombie (but may be in fork). 1447 */ 1448 if ((p = pfind(pid)) == NULL) 1449 return (NULL); 1450 1451 /* 1452 * Increment p_dtrace_probes so that the process knows to inform us 1453 * when it exits or execs. fasttrap_provider_free() decrements this 1454 * when we're done with this provider. 1455 */ 1456 p->p_dtrace_probes++; 1457 1458 /* 1459 * Grab the credentials for this process so we have 1460 * something to pass to dtrace_register(). 1461 */ 1462 PROC_LOCK_ASSERT(p, MA_OWNED); 1463 crhold(p->p_ucred); 1464 cred = p->p_ucred; 1465 PROC_UNLOCK(p); 1466 1467 new_fp = kmem_zalloc(sizeof (fasttrap_provider_t), KM_SLEEP); 1468 new_fp->ftp_pid = pid; 1469 new_fp->ftp_proc = fasttrap_proc_lookup(pid); 1470#if !defined(sun) 1471 mutex_init(&new_fp->ftp_mtx, "provider mtx", MUTEX_DEFAULT, NULL); 1472 mutex_init(&new_fp->ftp_cmtx, "lock on creating", MUTEX_DEFAULT, NULL); 1473#endif 1474 1475 ASSERT(new_fp->ftp_proc != NULL); 1476 1477 mutex_enter(&bucket->ftb_mtx); 1478 1479 /* 1480 * Take another lap through the list to make sure a provider hasn't 1481 * been created for this pid while we weren't under the bucket lock. 1482 */ 1483 for (fp = bucket->ftb_data; fp != NULL; fp = fp->ftp_next) { 1484 if (fp->ftp_pid == pid && strcmp(fp->ftp_name, name) == 0 && 1485 !fp->ftp_retired) { 1486 mutex_enter(&fp->ftp_mtx); 1487 mutex_exit(&bucket->ftb_mtx); 1488 fasttrap_provider_free(new_fp); 1489 crfree(cred); 1490 return (fp); 1491 } 1492 } 1493 1494 (void) strcpy(new_fp->ftp_name, name); 1495 1496 /* 1497 * Fail and return NULL if either the provider name is too long 1498 * or we fail to register this new provider with the DTrace 1499 * framework. Note that this is the only place we ever construct 1500 * the full provider name -- we keep it in pieces in the provider 1501 * structure. 1502 */ 1503 if (snprintf(provname, sizeof (provname), "%s%u", name, (uint_t)pid) >= 1504 sizeof (provname) || 1505 dtrace_register(provname, pattr, 1506 DTRACE_PRIV_PROC | DTRACE_PRIV_OWNER | DTRACE_PRIV_ZONEOWNER, cred, 1507 pattr == &pid_attr ? &pid_pops : &usdt_pops, new_fp, 1508 &new_fp->ftp_provid) != 0) { 1509 mutex_exit(&bucket->ftb_mtx); 1510 fasttrap_provider_free(new_fp); 1511 crfree(cred); 1512 return (NULL); 1513 } 1514 1515 new_fp->ftp_next = bucket->ftb_data; 1516 bucket->ftb_data = new_fp; 1517 1518 mutex_enter(&new_fp->ftp_mtx); 1519 mutex_exit(&bucket->ftb_mtx); 1520 1521 crfree(cred); 1522 return (new_fp); 1523} 1524 1525static void 1526fasttrap_provider_free(fasttrap_provider_t *provider) 1527{ 1528 pid_t pid = provider->ftp_pid; 1529 proc_t *p; 1530 1531 /* 1532 * There need to be no associated enabled probes, no consumers 1533 * creating probes, and no meta providers referencing this provider. 1534 */ 1535 ASSERT(provider->ftp_rcount == 0); 1536 ASSERT(provider->ftp_ccount == 0); 1537 ASSERT(provider->ftp_mcount == 0); 1538 1539 /* 1540 * If this provider hasn't been retired, we need to explicitly drop the 1541 * count of active providers on the associated process structure. 1542 */ 1543 if (!provider->ftp_retired) { 1544 atomic_add_64(&provider->ftp_proc->ftpc_acount, -1); 1545 ASSERT(provider->ftp_proc->ftpc_acount < 1546 provider->ftp_proc->ftpc_rcount); 1547 } 1548 1549 fasttrap_proc_release(provider->ftp_proc); 1550 1551#if !defined(sun) 1552 mutex_destroy(&provider->ftp_mtx); 1553 mutex_destroy(&provider->ftp_cmtx); 1554#endif 1555 kmem_free(provider, sizeof (fasttrap_provider_t)); 1556 1557 /* 1558 * Decrement p_dtrace_probes on the process whose provider we're 1559 * freeing. We don't have to worry about clobbering somone else's 1560 * modifications to it because we have locked the bucket that 1561 * corresponds to this process's hash chain in the provider hash 1562 * table. Don't sweat it if we can't find the process. 1563 */ 1564 if ((p = pfind(pid)) == NULL) { 1565 return; 1566 } 1567 1568 p->p_dtrace_probes--; 1569#if !defined(sun) 1570 PROC_UNLOCK(p); 1571#endif 1572} 1573 1574static void 1575fasttrap_provider_retire(pid_t pid, const char *name, int mprov) 1576{ 1577 fasttrap_provider_t *fp; 1578 fasttrap_bucket_t *bucket; 1579 dtrace_provider_id_t provid; 1580 1581 ASSERT(strlen(name) < sizeof (fp->ftp_name)); 1582 1583 bucket = &fasttrap_provs.fth_table[FASTTRAP_PROVS_INDEX(pid, name)]; 1584 mutex_enter(&bucket->ftb_mtx); 1585 1586 for (fp = bucket->ftb_data; fp != NULL; fp = fp->ftp_next) { 1587 if (fp->ftp_pid == pid && strcmp(fp->ftp_name, name) == 0 && 1588 !fp->ftp_retired) 1589 break; 1590 } 1591 1592 if (fp == NULL) { 1593 mutex_exit(&bucket->ftb_mtx); 1594 return; 1595 } 1596 1597 mutex_enter(&fp->ftp_mtx); 1598 ASSERT(!mprov || fp->ftp_mcount > 0); 1599 if (mprov && --fp->ftp_mcount != 0) { 1600 mutex_exit(&fp->ftp_mtx); 1601 mutex_exit(&bucket->ftb_mtx); 1602 return; 1603 } 1604 1605 /* 1606 * Mark the provider to be removed in our post-processing step, mark it 1607 * retired, and drop the active count on its proc. Marking it indicates 1608 * that we should try to remove it; setting the retired flag indicates 1609 * that we're done with this provider; dropping the active the proc 1610 * releases our hold, and when this reaches zero (as it will during 1611 * exit or exec) the proc and associated providers become defunct. 1612 * 1613 * We obviously need to take the bucket lock before the provider lock 1614 * to perform the lookup, but we need to drop the provider lock 1615 * before calling into the DTrace framework since we acquire the 1616 * provider lock in callbacks invoked from the DTrace framework. The 1617 * bucket lock therefore protects the integrity of the provider hash 1618 * table. 1619 */ 1620 atomic_add_64(&fp->ftp_proc->ftpc_acount, -1); 1621 ASSERT(fp->ftp_proc->ftpc_acount < fp->ftp_proc->ftpc_rcount); 1622 1623 fp->ftp_retired = 1; 1624 fp->ftp_marked = 1; 1625 provid = fp->ftp_provid; 1626 mutex_exit(&fp->ftp_mtx); 1627 1628 /* 1629 * We don't have to worry about invalidating the same provider twice 1630 * since fasttrap_provider_lookup() will ignore provider that have 1631 * been marked as retired. 1632 */ 1633 dtrace_invalidate(provid); 1634 1635 mutex_exit(&bucket->ftb_mtx); 1636 1637 fasttrap_pid_cleanup(); 1638} 1639 1640static int 1641fasttrap_uint32_cmp(const void *ap, const void *bp) 1642{ 1643 return (*(const uint32_t *)ap - *(const uint32_t *)bp); 1644} 1645 1646static int 1647fasttrap_uint64_cmp(const void *ap, const void *bp) 1648{ 1649 return (*(const uint64_t *)ap - *(const uint64_t *)bp); 1650} 1651 1652static int 1653fasttrap_add_probe(fasttrap_probe_spec_t *pdata) 1654{ 1655 fasttrap_provider_t *provider; 1656 fasttrap_probe_t *pp; 1657 fasttrap_tracepoint_t *tp; 1658 char *name; 1659 int i, aframes = 0, whack; 1660 1661 /* 1662 * There needs to be at least one desired trace point. 1663 */ 1664 if (pdata->ftps_noffs == 0) 1665 return (EINVAL); 1666 1667 switch (pdata->ftps_type) { 1668 case DTFTP_ENTRY: 1669 name = "entry"; 1670 aframes = FASTTRAP_ENTRY_AFRAMES; 1671 break; 1672 case DTFTP_RETURN: 1673 name = "return"; 1674 aframes = FASTTRAP_RETURN_AFRAMES; 1675 break; 1676 case DTFTP_OFFSETS: 1677 name = NULL; 1678 break; 1679 default: 1680 return (EINVAL); 1681 } 1682 1683 if ((provider = fasttrap_provider_lookup(pdata->ftps_pid, 1684 FASTTRAP_PID_NAME, &pid_attr)) == NULL) 1685 return (ESRCH); 1686 1687 /* 1688 * Increment this reference count to indicate that a consumer is 1689 * actively adding a new probe associated with this provider. This 1690 * prevents the provider from being deleted -- we'll need to check 1691 * for pending deletions when we drop this reference count. 1692 */ 1693 provider->ftp_ccount++; 1694 mutex_exit(&provider->ftp_mtx); 1695 1696 /* 1697 * Grab the creation lock to ensure consistency between calls to 1698 * dtrace_probe_lookup() and dtrace_probe_create() in the face of 1699 * other threads creating probes. We must drop the provider lock 1700 * before taking this lock to avoid a three-way deadlock with the 1701 * DTrace framework. 1702 */ 1703 mutex_enter(&provider->ftp_cmtx); 1704 1705 if (name == NULL) { 1706 for (i = 0; i < pdata->ftps_noffs; i++) { 1707 char name_str[17]; 1708 1709 (void) sprintf(name_str, "%llx", 1710 (unsigned long long)pdata->ftps_offs[i]); 1711 1712 if (dtrace_probe_lookup(provider->ftp_provid, 1713 pdata->ftps_mod, pdata->ftps_func, name_str) != 0) 1714 continue; 1715 1716 atomic_add_32(&fasttrap_total, 1); 1717 1718 if (fasttrap_total > fasttrap_max) { 1719 atomic_add_32(&fasttrap_total, -1); 1720 goto no_mem; 1721 } 1722 1723 pp = kmem_zalloc(sizeof (fasttrap_probe_t), KM_SLEEP); 1724 1725 pp->ftp_prov = provider; 1726 pp->ftp_faddr = pdata->ftps_pc; 1727 pp->ftp_fsize = pdata->ftps_size; 1728 pp->ftp_pid = pdata->ftps_pid; 1729 pp->ftp_ntps = 1; 1730 1731 tp = kmem_zalloc(sizeof (fasttrap_tracepoint_t), 1732 KM_SLEEP); 1733 1734 tp->ftt_proc = provider->ftp_proc; 1735 tp->ftt_pc = pdata->ftps_offs[i] + pdata->ftps_pc; 1736 tp->ftt_pid = pdata->ftps_pid; 1737 1738 pp->ftp_tps[0].fit_tp = tp; 1739 pp->ftp_tps[0].fit_id.fti_probe = pp; 1740 pp->ftp_tps[0].fit_id.fti_ptype = pdata->ftps_type; 1741 1742 pp->ftp_id = dtrace_probe_create(provider->ftp_provid, 1743 pdata->ftps_mod, pdata->ftps_func, name_str, 1744 FASTTRAP_OFFSET_AFRAMES, pp); 1745 } 1746 1747 } else if (dtrace_probe_lookup(provider->ftp_provid, pdata->ftps_mod, 1748 pdata->ftps_func, name) == 0) { 1749 atomic_add_32(&fasttrap_total, pdata->ftps_noffs); 1750 1751 if (fasttrap_total > fasttrap_max) { 1752 atomic_add_32(&fasttrap_total, -pdata->ftps_noffs); 1753 goto no_mem; 1754 } 1755 1756 /* 1757 * Make sure all tracepoint program counter values are unique. 1758 * We later assume that each probe has exactly one tracepoint 1759 * for a given pc. 1760 */ 1761 qsort(pdata->ftps_offs, pdata->ftps_noffs, 1762 sizeof (uint64_t), fasttrap_uint64_cmp); 1763 for (i = 1; i < pdata->ftps_noffs; i++) { 1764 if (pdata->ftps_offs[i] > pdata->ftps_offs[i - 1]) 1765 continue; 1766 1767 atomic_add_32(&fasttrap_total, -pdata->ftps_noffs); 1768 goto no_mem; 1769 } 1770 1771 ASSERT(pdata->ftps_noffs > 0); 1772 pp = kmem_zalloc(offsetof(fasttrap_probe_t, 1773 ftp_tps[pdata->ftps_noffs]), KM_SLEEP); 1774 1775 pp->ftp_prov = provider; 1776 pp->ftp_faddr = pdata->ftps_pc; 1777 pp->ftp_fsize = pdata->ftps_size; 1778 pp->ftp_pid = pdata->ftps_pid; 1779 pp->ftp_ntps = pdata->ftps_noffs; 1780 1781 for (i = 0; i < pdata->ftps_noffs; i++) { 1782 tp = kmem_zalloc(sizeof (fasttrap_tracepoint_t), 1783 KM_SLEEP); 1784 1785 tp->ftt_proc = provider->ftp_proc; 1786 tp->ftt_pc = pdata->ftps_offs[i] + pdata->ftps_pc; 1787 tp->ftt_pid = pdata->ftps_pid; 1788 1789 pp->ftp_tps[i].fit_tp = tp; 1790 pp->ftp_tps[i].fit_id.fti_probe = pp; 1791 pp->ftp_tps[i].fit_id.fti_ptype = pdata->ftps_type; 1792 } 1793 1794 pp->ftp_id = dtrace_probe_create(provider->ftp_provid, 1795 pdata->ftps_mod, pdata->ftps_func, name, aframes, pp); 1796 } 1797 1798 mutex_exit(&provider->ftp_cmtx); 1799 1800 /* 1801 * We know that the provider is still valid since we incremented the 1802 * creation reference count. If someone tried to clean up this provider 1803 * while we were using it (e.g. because the process called exec(2) or 1804 * exit(2)), take note of that and try to clean it up now. 1805 */ 1806 mutex_enter(&provider->ftp_mtx); 1807 provider->ftp_ccount--; 1808 whack = provider->ftp_retired; 1809 mutex_exit(&provider->ftp_mtx); 1810 1811 if (whack) 1812 fasttrap_pid_cleanup(); 1813 1814 return (0); 1815 1816no_mem: 1817 /* 1818 * If we've exhausted the allowable resources, we'll try to remove 1819 * this provider to free some up. This is to cover the case where 1820 * the user has accidentally created many more probes than was 1821 * intended (e.g. pid123:::). 1822 */ 1823 mutex_exit(&provider->ftp_cmtx); 1824 mutex_enter(&provider->ftp_mtx); 1825 provider->ftp_ccount--; 1826 provider->ftp_marked = 1; 1827 mutex_exit(&provider->ftp_mtx); 1828 1829 fasttrap_pid_cleanup(); 1830 1831 return (ENOMEM); 1832} 1833 1834/*ARGSUSED*/ 1835static void * 1836fasttrap_meta_provide(void *arg, dtrace_helper_provdesc_t *dhpv, pid_t pid) 1837{ 1838 fasttrap_provider_t *provider; 1839 1840 /* 1841 * A 32-bit unsigned integer (like a pid for example) can be 1842 * expressed in 10 or fewer decimal digits. Make sure that we'll 1843 * have enough space for the provider name. 1844 */ 1845 if (strlen(dhpv->dthpv_provname) + 10 >= 1846 sizeof (provider->ftp_name)) { 1847 printf("failed to instantiate provider %s: " 1848 "name too long to accomodate pid", dhpv->dthpv_provname); 1849 return (NULL); 1850 } 1851 1852 /* 1853 * Don't let folks spoof the true pid provider. 1854 */ 1855 if (strcmp(dhpv->dthpv_provname, FASTTRAP_PID_NAME) == 0) { 1856 printf("failed to instantiate provider %s: " 1857 "%s is an invalid name", dhpv->dthpv_provname, 1858 FASTTRAP_PID_NAME); 1859 return (NULL); 1860 } 1861 1862 /* 1863 * The highest stability class that fasttrap supports is ISA; cap 1864 * the stability of the new provider accordingly. 1865 */ 1866 if (dhpv->dthpv_pattr.dtpa_provider.dtat_class > DTRACE_CLASS_ISA) 1867 dhpv->dthpv_pattr.dtpa_provider.dtat_class = DTRACE_CLASS_ISA; 1868 if (dhpv->dthpv_pattr.dtpa_mod.dtat_class > DTRACE_CLASS_ISA) 1869 dhpv->dthpv_pattr.dtpa_mod.dtat_class = DTRACE_CLASS_ISA; 1870 if (dhpv->dthpv_pattr.dtpa_func.dtat_class > DTRACE_CLASS_ISA) 1871 dhpv->dthpv_pattr.dtpa_func.dtat_class = DTRACE_CLASS_ISA; 1872 if (dhpv->dthpv_pattr.dtpa_name.dtat_class > DTRACE_CLASS_ISA) 1873 dhpv->dthpv_pattr.dtpa_name.dtat_class = DTRACE_CLASS_ISA; 1874 if (dhpv->dthpv_pattr.dtpa_args.dtat_class > DTRACE_CLASS_ISA) 1875 dhpv->dthpv_pattr.dtpa_args.dtat_class = DTRACE_CLASS_ISA; 1876 1877 if ((provider = fasttrap_provider_lookup(pid, dhpv->dthpv_provname, 1878 &dhpv->dthpv_pattr)) == NULL) { 1879 printf("failed to instantiate provider %s for " 1880 "process %u", dhpv->dthpv_provname, (uint_t)pid); 1881 return (NULL); 1882 } 1883 1884 /* 1885 * Up the meta provider count so this provider isn't removed until 1886 * the meta provider has been told to remove it. 1887 */ 1888 provider->ftp_mcount++; 1889 1890 mutex_exit(&provider->ftp_mtx); 1891 1892 return (provider); 1893} 1894 1895/*ARGSUSED*/ 1896static void 1897fasttrap_meta_create_probe(void *arg, void *parg, 1898 dtrace_helper_probedesc_t *dhpb) 1899{ 1900 fasttrap_provider_t *provider = parg; 1901 fasttrap_probe_t *pp; 1902 fasttrap_tracepoint_t *tp; 1903 int i, j; 1904 uint32_t ntps; 1905 1906 /* 1907 * Since the meta provider count is non-zero we don't have to worry 1908 * about this provider disappearing. 1909 */ 1910 ASSERT(provider->ftp_mcount > 0); 1911 1912 /* 1913 * The offsets must be unique. 1914 */ 1915 qsort(dhpb->dthpb_offs, dhpb->dthpb_noffs, sizeof (uint32_t), 1916 fasttrap_uint32_cmp); 1917 for (i = 1; i < dhpb->dthpb_noffs; i++) { 1918 if (dhpb->dthpb_base + dhpb->dthpb_offs[i] <= 1919 dhpb->dthpb_base + dhpb->dthpb_offs[i - 1]) 1920 return; 1921 } 1922 1923 qsort(dhpb->dthpb_enoffs, dhpb->dthpb_nenoffs, sizeof (uint32_t), 1924 fasttrap_uint32_cmp); 1925 for (i = 1; i < dhpb->dthpb_nenoffs; i++) { 1926 if (dhpb->dthpb_base + dhpb->dthpb_enoffs[i] <= 1927 dhpb->dthpb_base + dhpb->dthpb_enoffs[i - 1]) 1928 return; 1929 } 1930 1931 /* 1932 * Grab the creation lock to ensure consistency between calls to 1933 * dtrace_probe_lookup() and dtrace_probe_create() in the face of 1934 * other threads creating probes. 1935 */ 1936 mutex_enter(&provider->ftp_cmtx); 1937 1938 if (dtrace_probe_lookup(provider->ftp_provid, dhpb->dthpb_mod, 1939 dhpb->dthpb_func, dhpb->dthpb_name) != 0) { 1940 mutex_exit(&provider->ftp_cmtx); 1941 return; 1942 } 1943 1944 ntps = dhpb->dthpb_noffs + dhpb->dthpb_nenoffs; 1945 ASSERT(ntps > 0); 1946 1947 atomic_add_32(&fasttrap_total, ntps); 1948 1949 if (fasttrap_total > fasttrap_max) { 1950 atomic_add_32(&fasttrap_total, -ntps); 1951 mutex_exit(&provider->ftp_cmtx); 1952 return; 1953 } 1954 1955 pp = kmem_zalloc(offsetof(fasttrap_probe_t, ftp_tps[ntps]), KM_SLEEP); 1956 1957 pp->ftp_prov = provider; 1958 pp->ftp_pid = provider->ftp_pid; 1959 pp->ftp_ntps = ntps; 1960 pp->ftp_nargs = dhpb->dthpb_xargc; 1961 pp->ftp_xtypes = dhpb->dthpb_xtypes; 1962 pp->ftp_ntypes = dhpb->dthpb_ntypes; 1963 1964 /* 1965 * First create a tracepoint for each actual point of interest. 1966 */ 1967 for (i = 0; i < dhpb->dthpb_noffs; i++) { 1968 tp = kmem_zalloc(sizeof (fasttrap_tracepoint_t), KM_SLEEP); 1969 1970 tp->ftt_proc = provider->ftp_proc; 1971 tp->ftt_pc = dhpb->dthpb_base + dhpb->dthpb_offs[i]; 1972 tp->ftt_pid = provider->ftp_pid; 1973 1974 pp->ftp_tps[i].fit_tp = tp; 1975 pp->ftp_tps[i].fit_id.fti_probe = pp; 1976#ifdef __sparc 1977 pp->ftp_tps[i].fit_id.fti_ptype = DTFTP_POST_OFFSETS; 1978#else 1979 pp->ftp_tps[i].fit_id.fti_ptype = DTFTP_OFFSETS; 1980#endif 1981 } 1982 1983 /* 1984 * Then create a tracepoint for each is-enabled point. 1985 */ 1986 for (j = 0; i < ntps; i++, j++) { 1987 tp = kmem_zalloc(sizeof (fasttrap_tracepoint_t), KM_SLEEP); 1988 1989 tp->ftt_proc = provider->ftp_proc; 1990 tp->ftt_pc = dhpb->dthpb_base + dhpb->dthpb_enoffs[j]; 1991 tp->ftt_pid = provider->ftp_pid; 1992 1993 pp->ftp_tps[i].fit_tp = tp; 1994 pp->ftp_tps[i].fit_id.fti_probe = pp; 1995 pp->ftp_tps[i].fit_id.fti_ptype = DTFTP_IS_ENABLED; 1996 } 1997 1998 /* 1999 * If the arguments are shuffled around we set the argument remapping 2000 * table. Later, when the probe fires, we only remap the arguments 2001 * if the table is non-NULL. 2002 */ 2003 for (i = 0; i < dhpb->dthpb_xargc; i++) { 2004 if (dhpb->dthpb_args[i] != i) { 2005 pp->ftp_argmap = dhpb->dthpb_args; 2006 break; 2007 } 2008 } 2009 2010 /* 2011 * The probe is fully constructed -- register it with DTrace. 2012 */ 2013 pp->ftp_id = dtrace_probe_create(provider->ftp_provid, dhpb->dthpb_mod, 2014 dhpb->dthpb_func, dhpb->dthpb_name, FASTTRAP_OFFSET_AFRAMES, pp); 2015 2016 mutex_exit(&provider->ftp_cmtx); 2017} 2018 2019/*ARGSUSED*/ 2020static void 2021fasttrap_meta_remove(void *arg, dtrace_helper_provdesc_t *dhpv, pid_t pid) 2022{ 2023 /* 2024 * Clean up the USDT provider. There may be active consumers of the 2025 * provider busy adding probes, no damage will actually befall the 2026 * provider until that count has dropped to zero. This just puts 2027 * the provider on death row. 2028 */ 2029 fasttrap_provider_retire(pid, dhpv->dthpv_provname, 1); 2030} 2031 2032static dtrace_mops_t fasttrap_mops = { 2033 fasttrap_meta_create_probe, 2034 fasttrap_meta_provide, 2035 fasttrap_meta_remove 2036}; 2037 2038/*ARGSUSED*/ 2039static int 2040fasttrap_open(struct cdev *dev __unused, int oflags __unused, 2041 int devtype __unused, struct thread *td __unused) 2042{ 2043 return (0); 2044} 2045 2046/*ARGSUSED*/ 2047static int 2048fasttrap_ioctl(struct cdev *dev, u_long cmd, caddr_t arg, int fflag, 2049 struct thread *td) 2050{ 2051#ifdef notyet 2052 struct kinfo_proc kp; 2053 const cred_t *cr = td->td_ucred; 2054#endif 2055 if (!dtrace_attached()) 2056 return (EAGAIN); 2057 2058 if (cmd == FASTTRAPIOC_MAKEPROBE) { 2059 fasttrap_probe_spec_t *uprobe = (void *)arg; 2060 fasttrap_probe_spec_t *probe; 2061 uint64_t noffs; 2062 size_t size; 2063 int ret; 2064 char *c; 2065 2066#if defined(sun) 2067 if (copyin(&uprobe->ftps_noffs, &noffs, 2068 sizeof (uprobe->ftps_noffs))) 2069 return (EFAULT); 2070#else 2071 noffs = uprobe->ftps_noffs; 2072#endif 2073 2074 /* 2075 * Probes must have at least one tracepoint. 2076 */ 2077 if (noffs == 0) 2078 return (EINVAL); 2079 2080 size = sizeof (fasttrap_probe_spec_t) + 2081 sizeof (probe->ftps_offs[0]) * (noffs - 1); 2082 2083 if (size > 1024 * 1024) 2084 return (ENOMEM); 2085 2086 probe = kmem_alloc(size, KM_SLEEP); 2087 2088#if defined(sun) 2089 if (copyin(uprobe, probe, size) != 0) { 2090 kmem_free(probe, size); 2091 return (EFAULT); 2092 } 2093#else 2094 memcpy(probe, uprobe, sizeof(*probe)); 2095 if (noffs > 1 && copyin(uprobe + 1, probe + 1, size) != 0) { 2096 kmem_free(probe, size); 2097 return (EFAULT); 2098 } 2099#endif 2100 2101 2102 /* 2103 * Verify that the function and module strings contain no 2104 * funny characters. 2105 */ 2106 for (c = &probe->ftps_func[0]; *c != '\0'; c++) { 2107 if (*c < 0x20 || 0x7f <= *c) { 2108 ret = EINVAL; 2109 goto err; 2110 } 2111 } 2112 2113 for (c = &probe->ftps_mod[0]; *c != '\0'; c++) { 2114 if (*c < 0x20 || 0x7f <= *c) { 2115 ret = EINVAL; 2116 goto err; 2117 } 2118 } 2119 2120#ifdef notyet 2121 if (!PRIV_POLICY_CHOICE(cr, PRIV_ALL, B_FALSE)) { 2122 proc_t *p; 2123 pid_t pid = probe->ftps_pid; 2124 2125#if defined(sun) 2126 mutex_enter(&pidlock); 2127#endif 2128 /* 2129 * Report an error if the process doesn't exist 2130 * or is actively being birthed. 2131 */ 2132 p = pfind(pid); 2133 if (p) 2134 fill_kinfo_proc(p, &kp); 2135 if (p == NULL || kp.ki_stat == SIDL) { 2136#if defined(sun) 2137 mutex_exit(&pidlock); 2138#endif 2139 return (ESRCH); 2140 } 2141#if defined(sun) 2142 mutex_enter(&p->p_lock); 2143 mutex_exit(&pidlock); 2144#else 2145 PROC_LOCK_ASSERT(p, MA_OWNED); 2146#endif 2147 2148#ifdef notyet 2149 if ((ret = priv_proc_cred_perm(cr, p, NULL, 2150 VREAD | VWRITE)) != 0) { 2151#if defined(sun) 2152 mutex_exit(&p->p_lock); 2153#else 2154 PROC_UNLOCK(p); 2155#endif 2156 return (ret); 2157 } 2158#endif /* notyet */ 2159#if defined(sun) 2160 mutex_exit(&p->p_lock); 2161#else 2162 PROC_UNLOCK(p); 2163#endif 2164 } 2165#endif /* notyet */ 2166 2167 ret = fasttrap_add_probe(probe); 2168err: 2169 kmem_free(probe, size); 2170 2171 return (ret); 2172 2173 } else if (cmd == FASTTRAPIOC_GETINSTR) { 2174 fasttrap_instr_query_t instr; 2175 fasttrap_tracepoint_t *tp; 2176 uint_t index; 2177#if defined(sun) 2178 int ret; 2179#endif 2180 2181#if defined(sun) 2182 if (copyin((void *)arg, &instr, sizeof (instr)) != 0) 2183 return (EFAULT); 2184#endif 2185 2186#ifdef notyet 2187 if (!PRIV_POLICY_CHOICE(cr, PRIV_ALL, B_FALSE)) { 2188 proc_t *p; 2189 pid_t pid = instr.ftiq_pid; 2190 2191#if defined(sun) 2192 mutex_enter(&pidlock); 2193#endif 2194 /* 2195 * Report an error if the process doesn't exist 2196 * or is actively being birthed. 2197 */ 2198 p = pfind(pid); 2199 if (p) 2200 fill_kinfo_proc(p, &kp); 2201 if (p == NULL || kp.ki_stat == SIDL) { 2202#if defined(sun) 2203 mutex_exit(&pidlock); 2204#endif 2205 return (ESRCH); 2206 } 2207#if defined(sun) 2208 mutex_enter(&p->p_lock); 2209 mutex_exit(&pidlock); 2210#else 2211 PROC_LOCK_ASSERT(p, MA_OWNED); 2212#endif 2213 2214#ifdef notyet 2215 if ((ret = priv_proc_cred_perm(cr, p, NULL, 2216 VREAD)) != 0) { 2217#if defined(sun) 2218 mutex_exit(&p->p_lock); 2219#else 2220 PROC_UNLOCK(p); 2221#endif 2222 return (ret); 2223 } 2224#endif /* notyet */ 2225 2226#if defined(sun) 2227 mutex_exit(&p->p_lock); 2228#else 2229 PROC_UNLOCK(p); 2230#endif 2231 } 2232#endif /* notyet */ 2233 2234 index = FASTTRAP_TPOINTS_INDEX(instr.ftiq_pid, instr.ftiq_pc); 2235 2236 mutex_enter(&fasttrap_tpoints.fth_table[index].ftb_mtx); 2237 tp = fasttrap_tpoints.fth_table[index].ftb_data; 2238 while (tp != NULL) { 2239 if (instr.ftiq_pid == tp->ftt_pid && 2240 instr.ftiq_pc == tp->ftt_pc && 2241 tp->ftt_proc->ftpc_acount != 0) 2242 break; 2243 2244 tp = tp->ftt_next; 2245 } 2246 2247 if (tp == NULL) { 2248 mutex_exit(&fasttrap_tpoints.fth_table[index].ftb_mtx); 2249 return (ENOENT); 2250 } 2251 2252 bcopy(&tp->ftt_instr, &instr.ftiq_instr, 2253 sizeof (instr.ftiq_instr)); 2254 mutex_exit(&fasttrap_tpoints.fth_table[index].ftb_mtx); 2255 2256 if (copyout(&instr, (void *)arg, sizeof (instr)) != 0) 2257 return (EFAULT); 2258 2259 return (0); 2260 } 2261 2262 return (EINVAL); 2263} 2264 2265static int 2266fasttrap_load(void) 2267{ 2268 ulong_t nent; 2269 int i, ret; 2270 2271 /* Create the /dev/dtrace/fasttrap entry. */ 2272 fasttrap_cdev = make_dev(&fasttrap_cdevsw, 0, UID_ROOT, GID_WHEEL, 0600, 2273 "dtrace/fasttrap"); 2274 2275 mtx_init(&fasttrap_cleanup_mtx, "fasttrap clean", "dtrace", MTX_DEF); 2276 mutex_init(&fasttrap_count_mtx, "fasttrap count mtx", MUTEX_DEFAULT, 2277 NULL); 2278 2279 ret = kproc_create(fasttrap_pid_cleanup_cb, NULL, 2280 &fasttrap_cleanup_proc, 0, 0, "ftcleanup"); 2281 if (ret != 0) { 2282 destroy_dev(fasttrap_cdev); 2283 return (ret); 2284 } 2285 2286 /* 2287 * Install our hooks into fork(2), exec(2), and exit(2). 2288 */ 2289 dtrace_fasttrap_fork = &fasttrap_fork; 2290 dtrace_fasttrap_exit = &fasttrap_exec_exit; 2291 dtrace_fasttrap_exec = &fasttrap_exec_exit; 2292 2293#if defined(sun) 2294 fasttrap_max = ddi_getprop(DDI_DEV_T_ANY, devi, DDI_PROP_DONTPASS, 2295 "fasttrap-max-probes", FASTTRAP_MAX_DEFAULT); 2296#else 2297 fasttrap_max = FASTTRAP_MAX_DEFAULT; 2298#endif 2299 fasttrap_total = 0; 2300 2301 /* 2302 * Conjure up the tracepoints hashtable... 2303 */ 2304#if defined(sun) 2305 nent = ddi_getprop(DDI_DEV_T_ANY, devi, DDI_PROP_DONTPASS, 2306 "fasttrap-hash-size", FASTTRAP_TPOINTS_DEFAULT_SIZE); 2307#else 2308 nent = FASTTRAP_TPOINTS_DEFAULT_SIZE; 2309#endif 2310 2311 if (nent == 0 || nent > 0x1000000) 2312 nent = FASTTRAP_TPOINTS_DEFAULT_SIZE; 2313 2314 if ((nent & (nent - 1)) == 0) 2315 fasttrap_tpoints.fth_nent = nent; 2316 else 2317 fasttrap_tpoints.fth_nent = 1 << fasttrap_highbit(nent); 2318 ASSERT(fasttrap_tpoints.fth_nent > 0); 2319 fasttrap_tpoints.fth_mask = fasttrap_tpoints.fth_nent - 1; 2320 fasttrap_tpoints.fth_table = kmem_zalloc(fasttrap_tpoints.fth_nent * 2321 sizeof (fasttrap_bucket_t), KM_SLEEP); 2322#if !defined(sun) 2323 for (i = 0; i < fasttrap_tpoints.fth_nent; i++) 2324 mutex_init(&fasttrap_tpoints.fth_table[i].ftb_mtx, 2325 "tracepoints bucket mtx", MUTEX_DEFAULT, NULL); 2326#endif 2327 2328 /* 2329 * ... and the providers hash table... 2330 */ 2331 nent = FASTTRAP_PROVIDERS_DEFAULT_SIZE; 2332 if ((nent & (nent - 1)) == 0) 2333 fasttrap_provs.fth_nent = nent; 2334 else 2335 fasttrap_provs.fth_nent = 1 << fasttrap_highbit(nent); 2336 ASSERT(fasttrap_provs.fth_nent > 0); 2337 fasttrap_provs.fth_mask = fasttrap_provs.fth_nent - 1; 2338 fasttrap_provs.fth_table = kmem_zalloc(fasttrap_provs.fth_nent * 2339 sizeof (fasttrap_bucket_t), KM_SLEEP); 2340#if !defined(sun) 2341 for (i = 0; i < fasttrap_provs.fth_nent; i++) 2342 mutex_init(&fasttrap_provs.fth_table[i].ftb_mtx, 2343 "providers bucket mtx", MUTEX_DEFAULT, NULL); 2344#endif 2345 2346 /* 2347 * ... and the procs hash table. 2348 */ 2349 nent = FASTTRAP_PROCS_DEFAULT_SIZE; 2350 if ((nent & (nent - 1)) == 0) 2351 fasttrap_procs.fth_nent = nent; 2352 else 2353 fasttrap_procs.fth_nent = 1 << fasttrap_highbit(nent); 2354 ASSERT(fasttrap_procs.fth_nent > 0); 2355 fasttrap_procs.fth_mask = fasttrap_procs.fth_nent - 1; 2356 fasttrap_procs.fth_table = kmem_zalloc(fasttrap_procs.fth_nent * 2357 sizeof (fasttrap_bucket_t), KM_SLEEP); 2358#if !defined(sun) 2359 for (i = 0; i < fasttrap_procs.fth_nent; i++) 2360 mutex_init(&fasttrap_procs.fth_table[i].ftb_mtx, 2361 "processes bucket mtx", MUTEX_DEFAULT, NULL); 2362 2363 CPU_FOREACH(i) { 2364 mutex_init(&fasttrap_cpuc_pid_lock[i], "fasttrap barrier", 2365 MUTEX_DEFAULT, NULL); 2366 } 2367#endif 2368 2369 (void) dtrace_meta_register("fasttrap", &fasttrap_mops, NULL, 2370 &fasttrap_meta_id); 2371 2372 return (0); 2373} 2374 2375static int 2376fasttrap_unload(void) 2377{ 2378 int i, fail = 0; 2379 2380 /* 2381 * Unregister the meta-provider to make sure no new fasttrap- 2382 * managed providers come along while we're trying to close up 2383 * shop. If we fail to detach, we'll need to re-register as a 2384 * meta-provider. We can fail to unregister as a meta-provider 2385 * if providers we manage still exist. 2386 */ 2387 if (fasttrap_meta_id != DTRACE_METAPROVNONE && 2388 dtrace_meta_unregister(fasttrap_meta_id) != 0) 2389 return (-1); 2390 2391 /* 2392 * Iterate over all of our providers. If there's still a process 2393 * that corresponds to that pid, fail to detach. 2394 */ 2395 for (i = 0; i < fasttrap_provs.fth_nent; i++) { 2396 fasttrap_provider_t **fpp, *fp; 2397 fasttrap_bucket_t *bucket = &fasttrap_provs.fth_table[i]; 2398 2399 mutex_enter(&bucket->ftb_mtx); 2400 fpp = (fasttrap_provider_t **)&bucket->ftb_data; 2401 while ((fp = *fpp) != NULL) { 2402 /* 2403 * Acquire and release the lock as a simple way of 2404 * waiting for any other consumer to finish with 2405 * this provider. A thread must first acquire the 2406 * bucket lock so there's no chance of another thread 2407 * blocking on the provider's lock. 2408 */ 2409 mutex_enter(&fp->ftp_mtx); 2410 mutex_exit(&fp->ftp_mtx); 2411 2412 if (dtrace_unregister(fp->ftp_provid) != 0) { 2413 fail = 1; 2414 fpp = &fp->ftp_next; 2415 } else { 2416 *fpp = fp->ftp_next; 2417 fasttrap_provider_free(fp); 2418 } 2419 } 2420 2421 mutex_exit(&bucket->ftb_mtx); 2422 } 2423 2424 if (fail) { 2425 (void) dtrace_meta_register("fasttrap", &fasttrap_mops, NULL, 2426 &fasttrap_meta_id); 2427 2428 return (-1); 2429 } 2430 2431 mtx_lock(&fasttrap_cleanup_mtx); 2432 fasttrap_cleanup_drain = 1; 2433 /* Wait for the cleanup thread to finish up and signal us. */ 2434 wakeup(&fasttrap_cleanup_cv); 2435 mtx_sleep(&fasttrap_cleanup_drain, &fasttrap_cleanup_mtx, 0, "ftcld", 2436 0); 2437 fasttrap_cleanup_proc = NULL;
|