Deleted Added
sdiff udiff text old ( 258311 ) new ( 264434 )
full compact
fasttrap.c (258311) fasttrap.c (264434)
1/*
2 * CDDL HEADER START
3 *
4 * The contents of this file are subject to the terms of the
5 * Common Development and Distribution License (the "License").
6 * You may not use this file except in compliance with the License.
7 *
8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE

--- 6 unchanged lines hidden (view full) ---

15 * If applicable, add the following below this CDDL HEADER, with the
16 * fields enclosed by brackets "[]" replaced with your own identifying
17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 *
19 * CDDL HEADER END
20 *
21 * Portions Copyright 2010 The FreeBSD Foundation
22 *
1/*
2 * CDDL HEADER START
3 *
4 * The contents of this file are subject to the terms of the
5 * Common Development and Distribution License (the "License").
6 * You may not use this file except in compliance with the License.
7 *
8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE

--- 6 unchanged lines hidden (view full) ---

15 * If applicable, add the following below this CDDL HEADER, with the
16 * fields enclosed by brackets "[]" replaced with your own identifying
17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 *
19 * CDDL HEADER END
20 *
21 * Portions Copyright 2010 The FreeBSD Foundation
22 *
23 * $FreeBSD: head/sys/cddl/contrib/opensolaris/uts/common/dtrace/fasttrap.c 258311 2013-11-18 16:51:56Z asomers $
23 * $FreeBSD: head/sys/cddl/contrib/opensolaris/uts/common/dtrace/fasttrap.c 264434 2014-04-14 00:22:42Z markj $
24 */
25
26/*
27 * Copyright 2008 Sun Microsystems, Inc. All rights reserved.
28 * Use is subject to license terms.
29 */
30
31#if defined(sun)

--- 24 unchanged lines hidden (view full) ---

56#include <sys/proc.h>
57#include <sys/policy.h>
58#if defined(sun)
59#include <util/qsort.h>
60#endif
61#include <sys/mutex.h>
62#include <sys/kernel.h>
63#if !defined(sun)
24 */
25
26/*
27 * Copyright 2008 Sun Microsystems, Inc. All rights reserved.
28 * Use is subject to license terms.
29 */
30
31#if defined(sun)

--- 24 unchanged lines hidden (view full) ---

56#include <sys/proc.h>
57#include <sys/policy.h>
58#if defined(sun)
59#include <util/qsort.h>
60#endif
61#include <sys/mutex.h>
62#include <sys/kernel.h>
63#if !defined(sun)
64#include <sys/user.h>
65#include <sys/dtrace_bsd.h>
64#include <sys/dtrace_bsd.h>
65#include <sys/eventhandler.h>
66#include <sys/user.h>
67#include <vm/vm.h>
68#include <vm/pmap.h>
69#include <vm/vm_map.h>
70#include <vm/vm_param.h>
66#include <cddl/dev/dtrace/dtrace_cddl.h>
67#endif
68
69/*
70 * User-Land Trap-Based Tracing
71 * ----------------------------
72 *
73 * The fasttrap provider allows DTrace consumers to instrument any user-level

--- 127 unchanged lines hidden (view full) ---

201static fasttrap_provider_t *fasttrap_provider_lookup(pid_t, const char *,
202 const dtrace_pattr_t *);
203static void fasttrap_provider_retire(pid_t, const char *, int);
204static void fasttrap_provider_free(fasttrap_provider_t *);
205
206static fasttrap_proc_t *fasttrap_proc_lookup(pid_t);
207static void fasttrap_proc_release(fasttrap_proc_t *);
208
71#include <cddl/dev/dtrace/dtrace_cddl.h>
72#endif
73
74/*
75 * User-Land Trap-Based Tracing
76 * ----------------------------
77 *
78 * The fasttrap provider allows DTrace consumers to instrument any user-level

--- 127 unchanged lines hidden (view full) ---

206static fasttrap_provider_t *fasttrap_provider_lookup(pid_t, const char *,
207 const dtrace_pattr_t *);
208static void fasttrap_provider_retire(pid_t, const char *, int);
209static void fasttrap_provider_free(fasttrap_provider_t *);
210
211static fasttrap_proc_t *fasttrap_proc_lookup(pid_t);
212static void fasttrap_proc_release(fasttrap_proc_t *);
213
214#if !defined(sun)
215static void fasttrap_thread_dtor(void *, struct thread *);
216#endif
217
209#define FASTTRAP_PROVS_INDEX(pid, name) \
210 ((fasttrap_hash_str(name) + (pid)) & fasttrap_provs.fth_mask)
211
212#define FASTTRAP_PROCS_INDEX(pid) ((pid) & fasttrap_procs.fth_mask)
213
214#if !defined(sun)
215static kmutex_t fasttrap_cpuc_pid_lock[MAXCPU];
218#define FASTTRAP_PROVS_INDEX(pid, name) \
219 ((fasttrap_hash_str(name) + (pid)) & fasttrap_provs.fth_mask)
220
221#define FASTTRAP_PROCS_INDEX(pid) ((pid) & fasttrap_procs.fth_mask)
222
223#if !defined(sun)
224static kmutex_t fasttrap_cpuc_pid_lock[MAXCPU];
225static eventhandler_tag fasttrap_thread_dtor_tag;
216#endif
217
218static int
219fasttrap_highbit(ulong_t i)
220{
221 int h = 1;
222
223 if (i == 0)

--- 60 unchanged lines hidden (view full) ---

284 ksi->ksi_code = TRAP_DTRACE;
285 ksi->ksi_addr = (caddr_t)pc;
286 PROC_LOCK(p);
287 (void) tdksignal(t, SIGTRAP, ksi);
288 PROC_UNLOCK(p);
289#endif
290}
291
226#endif
227
228static int
229fasttrap_highbit(ulong_t i)
230{
231 int h = 1;
232
233 if (i == 0)

--- 60 unchanged lines hidden (view full) ---

294 ksi->ksi_code = TRAP_DTRACE;
295 ksi->ksi_addr = (caddr_t)pc;
296 PROC_LOCK(p);
297 (void) tdksignal(t, SIGTRAP, ksi);
298 PROC_UNLOCK(p);
299#endif
300}
301
302#if !defined(sun)
292/*
303/*
304 * Obtain a chunk of scratch space in the address space of the target process.
305 */
306fasttrap_scrspace_t *
307fasttrap_scraddr(struct thread *td, fasttrap_proc_t *fprc)
308{
309 fasttrap_scrblock_t *scrblk;
310 fasttrap_scrspace_t *scrspc;
311 struct proc *p;
312 vm_offset_t addr;
313 int error, i;
314
315 scrspc = NULL;
316 if (td->t_dtrace_sscr != NULL) {
317 /* If the thread already has scratch space, we're done. */
318 scrspc = (fasttrap_scrspace_t *)td->t_dtrace_sscr;
319 return (scrspc);
320 }
321
322 p = td->td_proc;
323
324 mutex_enter(&fprc->ftpc_mtx);
325 if (LIST_EMPTY(&fprc->ftpc_fscr)) {
326 /*
327 * No scratch space is available, so we'll map a new scratch
328 * space block into the traced process' address space.
329 */
330 addr = 0;
331 error = vm_map_find(&p->p_vmspace->vm_map, NULL, 0, &addr,
332 FASTTRAP_SCRBLOCK_SIZE, 0, VMFS_ANY_SPACE, VM_PROT_ALL,
333 VM_PROT_ALL, 0);
334 if (error != KERN_SUCCESS)
335 goto done;
336
337 scrblk = malloc(sizeof(*scrblk), M_SOLARIS, M_WAITOK);
338 scrblk->ftsb_addr = addr;
339 LIST_INSERT_HEAD(&fprc->ftpc_scrblks, scrblk, ftsb_next);
340
341 /*
342 * Carve the block up into chunks and put them on the free list.
343 */
344 for (i = 0;
345 i < FASTTRAP_SCRBLOCK_SIZE / FASTTRAP_SCRSPACE_SIZE; i++) {
346 scrspc = malloc(sizeof(*scrspc), M_SOLARIS, M_WAITOK);
347 scrspc->ftss_addr = addr +
348 i * FASTTRAP_SCRSPACE_SIZE;
349 LIST_INSERT_HEAD(&fprc->ftpc_fscr, scrspc,
350 ftss_next);
351 }
352 }
353
354 /*
355 * Take the first scratch chunk off the free list, put it on the
356 * allocated list, and return its address.
357 */
358 scrspc = LIST_FIRST(&fprc->ftpc_fscr);
359 LIST_REMOVE(scrspc, ftss_next);
360 LIST_INSERT_HEAD(&fprc->ftpc_ascr, scrspc, ftss_next);
361
362 /*
363 * This scratch space is reserved for use by td until the thread exits.
364 */
365 td->t_dtrace_sscr = scrspc;
366
367done:
368 mutex_exit(&fprc->ftpc_mtx);
369
370 return (scrspc);
371}
372
373/*
374 * Return any allocated per-thread scratch space chunks back to the process'
375 * free list.
376 */
377static void
378fasttrap_thread_dtor(void *arg __unused, struct thread *td)
379{
380 fasttrap_bucket_t *bucket;
381 fasttrap_proc_t *fprc;
382 fasttrap_scrspace_t *scrspc;
383 pid_t pid;
384
385 if (td->t_dtrace_sscr == NULL)
386 return;
387
388 pid = td->td_proc->p_pid;
389 bucket = &fasttrap_procs.fth_table[FASTTRAP_PROCS_INDEX(pid)];
390 fprc = NULL;
391
392 /* Look up the fasttrap process handle for this process. */
393 mutex_enter(&bucket->ftb_mtx);
394 for (fprc = bucket->ftb_data; fprc != NULL; fprc = fprc->ftpc_next) {
395 if (fprc->ftpc_pid == pid) {
396 mutex_enter(&fprc->ftpc_mtx);
397 mutex_exit(&bucket->ftb_mtx);
398 break;
399 }
400 }
401 if (fprc == NULL) {
402 mutex_exit(&bucket->ftb_mtx);
403 return;
404 }
405
406 scrspc = (fasttrap_scrspace_t *)td->t_dtrace_sscr;
407 LIST_REMOVE(scrspc, ftss_next);
408 LIST_INSERT_HEAD(&fprc->ftpc_fscr, scrspc, ftss_next);
409
410 mutex_exit(&fprc->ftpc_mtx);
411}
412#endif
413
414/*
293 * This function ensures that no threads are actively using the memory
294 * associated with probes that were formerly live.
295 */
296static void
297fasttrap_mod_barrier(uint64_t gen)
298{
299 int i;
300

--- 143 unchanged lines hidden (view full) ---

444 * This is called from cfork() via dtrace_fasttrap_fork(). The child
445 * process's address space is (roughly) a copy of the parent process's so
446 * we have to remove all the instrumentation we had previously enabled in the
447 * parent.
448 */
449static void
450fasttrap_fork(proc_t *p, proc_t *cp)
451{
415 * This function ensures that no threads are actively using the memory
416 * associated with probes that were formerly live.
417 */
418static void
419fasttrap_mod_barrier(uint64_t gen)
420{
421 int i;
422

--- 143 unchanged lines hidden (view full) ---

566 * This is called from cfork() via dtrace_fasttrap_fork(). The child
567 * process's address space is (roughly) a copy of the parent process's so
568 * we have to remove all the instrumentation we had previously enabled in the
569 * parent.
570 */
571static void
572fasttrap_fork(proc_t *p, proc_t *cp)
573{
574#if !defined(sun)
575 fasttrap_scrblock_t *scrblk;
576 fasttrap_proc_t *fprc = NULL;
577#endif
452 pid_t ppid = p->p_pid;
453 int i;
454
455#if defined(sun)
456 ASSERT(curproc == p);
457 ASSERT(p->p_proc_flag & P_PR_LOCK);
458#else
459 PROC_LOCK_ASSERT(p, MA_OWNED);

--- 69 unchanged lines hidden (view full) ---

529 /*
530 * The count of active providers can only be
531 * decremented (i.e. to zero) during exec,
532 * exit, and removal of a meta provider so it
533 * should be impossible to drop the count
534 * mid-fork.
535 */
536 ASSERT(tp->ftt_proc->ftpc_acount != 0);
578 pid_t ppid = p->p_pid;
579 int i;
580
581#if defined(sun)
582 ASSERT(curproc == p);
583 ASSERT(p->p_proc_flag & P_PR_LOCK);
584#else
585 PROC_LOCK_ASSERT(p, MA_OWNED);

--- 69 unchanged lines hidden (view full) ---

655 /*
656 * The count of active providers can only be
657 * decremented (i.e. to zero) during exec,
658 * exit, and removal of a meta provider so it
659 * should be impossible to drop the count
660 * mid-fork.
661 */
662 ASSERT(tp->ftt_proc->ftpc_acount != 0);
663#if !defined(sun)
664 fprc = tp->ftt_proc;
665#endif
537 }
538 }
539 mutex_exit(&bucket->ftb_mtx);
666 }
667 }
668 mutex_exit(&bucket->ftb_mtx);
669
670#if !defined(sun)
671 /*
672 * Unmap any scratch space inherited from the parent's address
673 * space.
674 */
675 if (fprc != NULL) {
676 mutex_enter(&fprc->ftpc_mtx);
677 LIST_FOREACH(scrblk, &fprc->ftpc_scrblks, ftsb_next) {
678 vm_map_remove(&cp->p_vmspace->vm_map,
679 scrblk->ftsb_addr,
680 scrblk->ftsb_addr + FASTTRAP_SCRBLOCK_SIZE);
681 }
682 mutex_exit(&fprc->ftpc_mtx);
683 }
684#endif
540 }
541
542#if defined(sun)
543 mutex_enter(&cp->p_lock);
544 sprunlock(cp);
545#else
546 PROC_LOCK(p);
547 PROC_LOCK(cp);

--- 4 unchanged lines hidden (view full) ---

552/*
553 * This is called from proc_exit() or from exec_common() if p_dtrace_probes
554 * is set on the proc structure to indicate that there is a pid provider
555 * associated with this process.
556 */
557static void
558fasttrap_exec_exit(proc_t *p)
559{
685 }
686
687#if defined(sun)
688 mutex_enter(&cp->p_lock);
689 sprunlock(cp);
690#else
691 PROC_LOCK(p);
692 PROC_LOCK(cp);

--- 4 unchanged lines hidden (view full) ---

697/*
698 * This is called from proc_exit() or from exec_common() if p_dtrace_probes
699 * is set on the proc structure to indicate that there is a pid provider
700 * associated with this process.
701 */
702static void
703fasttrap_exec_exit(proc_t *p)
704{
705#if !defined(sun)
706 struct thread *td;
707#endif
708
560#if defined(sun)
561 ASSERT(p == curproc);
709#if defined(sun)
710 ASSERT(p == curproc);
562#endif
711#else
563 PROC_LOCK_ASSERT(p, MA_OWNED);
564 _PHOLD(p);
712 PROC_LOCK_ASSERT(p, MA_OWNED);
713 _PHOLD(p);
714 /*
715 * Since struct threads may be recycled, we cannot rely on t_dtrace_sscr
716 * fields to be zeroed by kdtrace_thread_ctor. Thus we must zero it
717 * ourselves when a process exits.
718 */
719 FOREACH_THREAD_IN_PROC(p, td)
720 td->t_dtrace_sscr = NULL;
565 PROC_UNLOCK(p);
721 PROC_UNLOCK(p);
722#endif
566
567 /*
568 * We clean up the pid provider for this process here; user-land
569 * static probes are handled by the meta-provider remove entry point.
570 */
571 fasttrap_provider_retire(p->p_pid, FASTTRAP_PID_NAME, 0);
572#if !defined(sun)
573 if (p->p_dtrace_helpers)
574 dtrace_helpers_destroy(p);
723
724 /*
725 * We clean up the pid provider for this process here; user-land
726 * static probes are handled by the meta-provider remove entry point.
727 */
728 fasttrap_provider_retire(p->p_pid, FASTTRAP_PID_NAME, 0);
729#if !defined(sun)
730 if (p->p_dtrace_helpers)
731 dtrace_helpers_destroy(p);
575#endif
576 PROC_LOCK(p);
577 _PRELE(p);
732 PROC_LOCK(p);
733 _PRELE(p);
734#endif
578}
579
580
581/*ARGSUSED*/
582static void
583fasttrap_pid_provide(void *arg, dtrace_probedesc_t *desc)
584{
585 /*

--- 776 unchanged lines hidden (view full) ---

1362}
1363
1364static void
1365fasttrap_proc_release(fasttrap_proc_t *proc)
1366{
1367 fasttrap_bucket_t *bucket;
1368 fasttrap_proc_t *fprc, **fprcp;
1369 pid_t pid = proc->ftpc_pid;
735}
736
737
738/*ARGSUSED*/
739static void
740fasttrap_pid_provide(void *arg, dtrace_probedesc_t *desc)
741{
742 /*

--- 776 unchanged lines hidden (view full) ---

1519}
1520
1521static void
1522fasttrap_proc_release(fasttrap_proc_t *proc)
1523{
1524 fasttrap_bucket_t *bucket;
1525 fasttrap_proc_t *fprc, **fprcp;
1526 pid_t pid = proc->ftpc_pid;
1527#if !defined(sun)
1528 fasttrap_scrblock_t *scrblk, *scrblktmp;
1529 fasttrap_scrspace_t *scrspc, *scrspctmp;
1530 struct proc *p;
1531 struct thread *td;
1532#endif
1370
1371 mutex_enter(&proc->ftpc_mtx);
1372
1373 ASSERT(proc->ftpc_rcount != 0);
1374 ASSERT(proc->ftpc_acount <= proc->ftpc_rcount);
1375
1376 if (--proc->ftpc_rcount != 0) {
1377 mutex_exit(&proc->ftpc_mtx);
1378 return;
1379 }
1380
1533
1534 mutex_enter(&proc->ftpc_mtx);
1535
1536 ASSERT(proc->ftpc_rcount != 0);
1537 ASSERT(proc->ftpc_acount <= proc->ftpc_rcount);
1538
1539 if (--proc->ftpc_rcount != 0) {
1540 mutex_exit(&proc->ftpc_mtx);
1541 return;
1542 }
1543
1544#if !defined(sun)
1545 /*
1546 * Free all structures used to manage per-thread scratch space.
1547 */
1548 LIST_FOREACH_SAFE(scrblk, &proc->ftpc_scrblks, ftsb_next,
1549 scrblktmp) {
1550 LIST_REMOVE(scrblk, ftsb_next);
1551 free(scrblk, M_SOLARIS);
1552 }
1553 LIST_FOREACH_SAFE(scrspc, &proc->ftpc_fscr, ftss_next, scrspctmp) {
1554 LIST_REMOVE(scrspc, ftss_next);
1555 free(scrspc, M_SOLARIS);
1556 }
1557 LIST_FOREACH_SAFE(scrspc, &proc->ftpc_ascr, ftss_next, scrspctmp) {
1558 LIST_REMOVE(scrspc, ftss_next);
1559 free(scrspc, M_SOLARIS);
1560 }
1561
1562 if ((p = pfind(pid)) != NULL) {
1563 FOREACH_THREAD_IN_PROC(p, td)
1564 td->t_dtrace_sscr = NULL;
1565 PROC_UNLOCK(p);
1566 }
1567#endif
1568
1381 mutex_exit(&proc->ftpc_mtx);
1382
1383 /*
1384 * There should definitely be no live providers associated with this
1385 * process at this point.
1386 */
1387 ASSERT(proc->ftpc_acount == 0);
1388

--- 969 unchanged lines hidden (view full) ---

2358 for (i = 0; i < fasttrap_procs.fth_nent; i++)
2359 mutex_init(&fasttrap_procs.fth_table[i].ftb_mtx,
2360 "processes bucket mtx", MUTEX_DEFAULT, NULL);
2361
2362 CPU_FOREACH(i) {
2363 mutex_init(&fasttrap_cpuc_pid_lock[i], "fasttrap barrier",
2364 MUTEX_DEFAULT, NULL);
2365 }
1569 mutex_exit(&proc->ftpc_mtx);
1570
1571 /*
1572 * There should definitely be no live providers associated with this
1573 * process at this point.
1574 */
1575 ASSERT(proc->ftpc_acount == 0);
1576

--- 969 unchanged lines hidden (view full) ---

2546 for (i = 0; i < fasttrap_procs.fth_nent; i++)
2547 mutex_init(&fasttrap_procs.fth_table[i].ftb_mtx,
2548 "processes bucket mtx", MUTEX_DEFAULT, NULL);
2549
2550 CPU_FOREACH(i) {
2551 mutex_init(&fasttrap_cpuc_pid_lock[i], "fasttrap barrier",
2552 MUTEX_DEFAULT, NULL);
2553 }
2554
2555 /*
2556 * This event handler must run before kdtrace_thread_dtor() since it
2557 * accesses the thread's struct kdtrace_thread.
2558 */
2559 fasttrap_thread_dtor_tag = EVENTHANDLER_REGISTER(thread_dtor,
2560 fasttrap_thread_dtor, NULL, EVENTHANDLER_PRI_FIRST);
2366#endif
2367
2368 /*
2369 * Install our hooks into fork(2), exec(2), and exit(2).
2370 */
2371 dtrace_fasttrap_fork = &fasttrap_fork;
2372 dtrace_fasttrap_exit = &fasttrap_exec_exit;
2373 dtrace_fasttrap_exec = &fasttrap_exec_exit;

--- 85 unchanged lines hidden (view full) ---

2459
2460#ifdef DEBUG
2461 mutex_enter(&fasttrap_count_mtx);
2462 ASSERT(fasttrap_pid_count == 0);
2463 mutex_exit(&fasttrap_count_mtx);
2464#endif
2465
2466#if !defined(sun)
2561#endif
2562
2563 /*
2564 * Install our hooks into fork(2), exec(2), and exit(2).
2565 */
2566 dtrace_fasttrap_fork = &fasttrap_fork;
2567 dtrace_fasttrap_exit = &fasttrap_exec_exit;
2568 dtrace_fasttrap_exec = &fasttrap_exec_exit;

--- 85 unchanged lines hidden (view full) ---

2654
2655#ifdef DEBUG
2656 mutex_enter(&fasttrap_count_mtx);
2657 ASSERT(fasttrap_pid_count == 0);
2658 mutex_exit(&fasttrap_count_mtx);
2659#endif
2660
2661#if !defined(sun)
2662 EVENTHANDLER_DEREGISTER(thread_dtor, fasttrap_thread_dtor_tag);
2663
2467 for (i = 0; i < fasttrap_tpoints.fth_nent; i++)
2468 mutex_destroy(&fasttrap_tpoints.fth_table[i].ftb_mtx);
2469 for (i = 0; i < fasttrap_provs.fth_nent; i++)
2470 mutex_destroy(&fasttrap_provs.fth_table[i].ftb_mtx);
2471 for (i = 0; i < fasttrap_procs.fth_nent; i++)
2472 mutex_destroy(&fasttrap_procs.fth_table[i].ftb_mtx);
2473#endif
2474 kmem_free(fasttrap_tpoints.fth_table,

--- 54 unchanged lines hidden --- 		2664 for (i = 0; i < fasttrap_tpoints.fth_nent; i++)
2665 mutex_destroy(&fasttrap_tpoints.fth_table[i].ftb_mtx);
2666 for (i = 0; i < fasttrap_provs.fth_nent; i++)
2667 mutex_destroy(&fasttrap_provs.fth_table[i].ftb_mtx);
2668 for (i = 0; i < fasttrap_procs.fth_nent; i++)
2669 mutex_destroy(&fasttrap_procs.fth_table[i].ftb_mtx);
2670#endif
2671 kmem_free(fasttrap_tpoints.fth_table,

--- 54 unchanged lines hidden ---