Deleted Added
sdiff udiff text old ( 185363 ) new ( 186037 )
full compact
hwpmc_mod.c (185363) hwpmc_mod.c (186037)
1/*-
2 * Copyright (c) 2003-2008 Joseph Koshy
3 * Copyright (c) 2007 The FreeBSD Foundation
4 * All rights reserved.
5 *
6 * Portions of this software were developed by A. Joseph Koshy under
7 * sponsorship from the FreeBSD Foundation and Google, Inc.
8 *

--- 16 unchanged lines hidden (view full) ---

25 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
26 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
27 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
28 * SUCH DAMAGE.
29 *
30 */
31
32#include <sys/cdefs.h>
1/*-
2 * Copyright (c) 2003-2008 Joseph Koshy
3 * Copyright (c) 2007 The FreeBSD Foundation
4 * All rights reserved.
5 *
6 * Portions of this software were developed by A. Joseph Koshy under
7 * sponsorship from the FreeBSD Foundation and Google, Inc.
8 *

--- 16 unchanged lines hidden (view full) ---

25 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
26 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
27 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
28 * SUCH DAMAGE.
29 *
30 */
31
32#include <sys/cdefs.h>
33__FBSDID("$FreeBSD: head/sys/dev/hwpmc/hwpmc_mod.c 185363 2008-11-27 09:00:47Z jkoshy $");
33__FBSDID("$FreeBSD: head/sys/dev/hwpmc/hwpmc_mod.c 186037 2008-12-13 13:07:12Z jkoshy $");
34
35#include <sys/param.h>
36#include <sys/eventhandler.h>
37#include <sys/jail.h>
38#include <sys/kernel.h>
39#include <sys/kthread.h>
40#include <sys/limits.h>
41#include <sys/lock.h>

--- 1816 unchanged lines hidden (view full) ---

1858 sx_assert(&pmc_sx, SX_LOCKED);
1859 pmc_process_munmap(td, (struct pmckern_map_out *) arg);
1860 break;
1861
1862 case PMC_FN_USER_CALLCHAIN:
1863 /*
1864 * Record a call chain.
1865 */
34
35#include <sys/param.h>
36#include <sys/eventhandler.h>
37#include <sys/jail.h>
38#include <sys/kernel.h>
39#include <sys/kthread.h>
40#include <sys/limits.h>
41#include <sys/lock.h>

--- 1816 unchanged lines hidden (view full) ---

1858 sx_assert(&pmc_sx, SX_LOCKED);
1859 pmc_process_munmap(td, (struct pmckern_map_out *) arg);
1860 break;
1861
1862 case PMC_FN_USER_CALLCHAIN:
1863 /*
1864 * Record a call chain.
1865 */
1866 KASSERT(td == curthread, ("[pmc,%d] td != curthread",
1867 __LINE__));
1866 pmc_capture_user_callchain(PCPU_GET(cpuid),
1867 (struct trapframe *) arg);
1868 pmc_capture_user_callchain(PCPU_GET(cpuid),
1869 (struct trapframe *) arg);
1870 td->td_pflags &= ~TDP_CALLCHAIN;
1868 break;
1869
1870 default:
1871#ifdef DEBUG
1872 KASSERT(0, ("[pmc,%d] unknown hook %d\n", __LINE__, function));
1873#endif
1874 break;
1875

--- 1913 unchanged lines hidden (view full) ---

3789
3790/*
3791 * Mark the thread as needing callchain capture and post an AST. The
3792 * actual callchain capture will be done in a context where it is safe
3793 * to take page faults.
3794 */
3795
3796static void
1871 break;
1872
1873 default:
1874#ifdef DEBUG
1875 KASSERT(0, ("[pmc,%d] unknown hook %d\n", __LINE__, function));
1876#endif
1877 break;
1878

--- 1913 unchanged lines hidden (view full) ---

3792
3793/*
3794 * Mark the thread as needing callchain capture and post an AST. The
3795 * actual callchain capture will be done in a context where it is safe
3796 * to take page faults.
3797 */
3798
3799static void
3797pmc_post_callchain_ast(void)
3800pmc_post_callchain_callback(void)
3798{
3799 struct thread *td;
3800
3801 td = curthread;
3802
3801{
3802 struct thread *td;
3803
3804 td = curthread;
3805
3806 KASSERT((td->td_pflags & TDP_CALLCHAIN) == 0,
3807 ("[pmc,%d] thread %p already marked for callchain capture",
3808 __LINE__, (void *) td));
3809
3803 /*
3810 /*
3804 * Mark this thread as needing processing in ast().
3805 * td->td_pflags will be safe to touch as the process was in
3806 * user space when it was interrupted.
3811 * Mark this thread as needing callchain capture.
3812 * `td->td_pflags' will be safe to touch because this thread
3813 * was in user space when it was interrupted.
3807 */
3808 td->td_pflags |= TDP_CALLCHAIN;
3809
3810 /*
3814 */
3815 td->td_pflags |= TDP_CALLCHAIN;
3816
3817 /*
3811 * Again, since we've entered this function directly from
3812 * userland, `td' is guaranteed to be not locked by this CPU,
3813 * so its safe to try acquire the thread lock even though we
3814 * are executing in an NMI context. We need to acquire this
3815 * lock before touching `td_flags' because other CPUs may be
3816 * in the process of touching this field.
3818 * Don't let this thread migrate between CPUs until callchain
3819 * capture completes.
3817 */
3820 */
3818 thread_lock(td);
3819 td->td_flags |= TDF_ASTPENDING;
3820 thread_unlock(td);
3821 sched_pin();
3821
3822 return;
3823}
3824
3825/*
3826 * Interrupt processing.
3827 *
3828 * Find a free slot in the per-cpu array of samples and capture the

--- 35 unchanged lines hidden (view full) ---

3864
3865
3866 /* Fill in entry. */
3867 PMCDBG(SAM,INT,1,"cpu=%d pm=%p tf=%p um=%d wr=%d rd=%d", cpu, pm,
3868 (void *) tf, inuserspace,
3869 (int) (psb->ps_write - psb->ps_samples),
3870 (int) (psb->ps_read - psb->ps_samples));
3871
3822
3823 return;
3824}
3825
3826/*
3827 * Interrupt processing.
3828 *
3829 * Find a free slot in the per-cpu array of samples and capture the

--- 35 unchanged lines hidden (view full) ---

3865
3866
3867 /* Fill in entry. */
3868 PMCDBG(SAM,INT,1,"cpu=%d pm=%p tf=%p um=%d wr=%d rd=%d", cpu, pm,
3869 (void *) tf, inuserspace,
3870 (int) (psb->ps_write - psb->ps_samples),
3871 (int) (psb->ps_read - psb->ps_samples));
3872
3873 KASSERT(pm->pm_runcount >= 0,
3874 ("[pmc,%d] pm=%p runcount %d", __LINE__, (void *) pm,
3875 pm->pm_runcount));
3876
3872 atomic_add_rel_32(&pm->pm_runcount, 1); /* hold onto PMC */
3873 ps->ps_pmc = pm;
3874 if ((td = curthread) && td->td_proc)
3875 ps->ps_pid = td->td_proc->p_pid;
3876 else
3877 ps->ps_pid = -1;
3878 ps->ps_cpu = cpu;
3877 atomic_add_rel_32(&pm->pm_runcount, 1); /* hold onto PMC */
3878 ps->ps_pmc = pm;
3879 if ((td = curthread) && td->td_proc)
3880 ps->ps_pid = td->td_proc->p_pid;
3881 else
3882 ps->ps_pid = -1;
3883 ps->ps_cpu = cpu;
3884 ps->ps_td = td;
3879 ps->ps_flags = inuserspace ? PMC_CC_F_USERSPACE : 0;
3880
3881 callchaindepth = (pm->pm_flags & PMC_F_CALLCHAIN) ?
3882 pmc_callchaindepth : 1;
3883
3884 if (callchaindepth == 1)
3885 ps->ps_pc[0] = PMC_TRAPFRAME_TO_PC(tf);
3886 else {
3887 /*
3888 * Kernel stack traversals can be done immediately,
3889 * while we defer to an AST for user space traversals.
3890 */
3891 if (!inuserspace)
3892 callchaindepth =
3893 pmc_save_kernel_callchain(ps->ps_pc,
3894 callchaindepth, tf);
3895 else {
3885 ps->ps_flags = inuserspace ? PMC_CC_F_USERSPACE : 0;
3886
3887 callchaindepth = (pm->pm_flags & PMC_F_CALLCHAIN) ?
3888 pmc_callchaindepth : 1;
3889
3890 if (callchaindepth == 1)
3891 ps->ps_pc[0] = PMC_TRAPFRAME_TO_PC(tf);
3892 else {
3893 /*
3894 * Kernel stack traversals can be done immediately,
3895 * while we defer to an AST for user space traversals.
3896 */
3897 if (!inuserspace)
3898 callchaindepth =
3899 pmc_save_kernel_callchain(ps->ps_pc,
3900 callchaindepth, tf);
3901 else {
3896 pmc_post_callchain_ast();
3902 pmc_post_callchain_callback();
3897 callchaindepth = PMC_SAMPLE_INUSE;
3898 }
3899 }
3900
3901 ps->ps_nsamples = callchaindepth; /* mark entry as in use */
3902
3903 /* increment write pointer, modulo ring buffer size */
3904 ps++;

--- 15 unchanged lines hidden (view full) ---

3920 * rescheduled.
3921 */
3922
3923static void
3924pmc_capture_user_callchain(int cpu, struct trapframe *tf)
3925{
3926 int i;
3927 struct pmc *pm;
3903 callchaindepth = PMC_SAMPLE_INUSE;
3904 }
3905 }
3906
3907 ps->ps_nsamples = callchaindepth; /* mark entry as in use */
3908
3909 /* increment write pointer, modulo ring buffer size */
3910 ps++;

--- 15 unchanged lines hidden (view full) ---

3926 * rescheduled.
3927 */
3928
3929static void
3930pmc_capture_user_callchain(int cpu, struct trapframe *tf)
3931{
3932 int i;
3933 struct pmc *pm;
3934 struct thread *td;
3928 struct pmc_sample *ps;
3929 struct pmc_samplebuffer *psb;
3935 struct pmc_sample *ps;
3936 struct pmc_samplebuffer *psb;
3937#ifdef INVARIANTS
3938 int ncallchains;
3939#endif
3930
3940
3941 sched_unpin(); /* Can migrate safely now. */
3942
3931 psb = pmc_pcpu[cpu]->pc_sb;
3943 psb = pmc_pcpu[cpu]->pc_sb;
3944 td = curthread;
3932
3945
3946 KASSERT(td->td_pflags & TDP_CALLCHAIN,
3947 ("[pmc,%d] Retrieving callchain for thread that doesn't want it",
3948 __LINE__));
3949
3950#ifdef INVARIANTS
3951 ncallchains = 0;
3952#endif
3953
3933 /*
3934 * Iterate through all deferred callchain requests.
3935 */
3936
3954 /*
3955 * Iterate through all deferred callchain requests.
3956 */
3957
3937 for (i = 0; i < pmc_nsamples; i++) {
3958 ps = psb->ps_samples;
3959 for (i = 0; i < pmc_nsamples; i++, ps++) {
3938
3960
3939 ps = &psb->ps_samples[i];
3940 if (ps->ps_nsamples != PMC_SAMPLE_INUSE)
3941 continue;
3961 if (ps->ps_nsamples != PMC_SAMPLE_INUSE)
3962 continue;
3963 if (ps->ps_td != td)
3964 continue;
3942
3965
3966 KASSERT(ps->ps_cpu == cpu,
3967 ("[pmc,%d] cpu mismatch ps_cpu=%d pcpu=%d", __LINE__,
3968 ps->ps_cpu, PCPU_GET(cpuid)));
3969
3943 pm = ps->ps_pmc;
3944
3945 KASSERT(pm->pm_flags & PMC_F_CALLCHAIN,
3946 ("[pmc,%d] Retrieving callchain for PMC that doesn't "
3947 "want it", __LINE__));
3948
3970 pm = ps->ps_pmc;
3971
3972 KASSERT(pm->pm_flags & PMC_F_CALLCHAIN,
3973 ("[pmc,%d] Retrieving callchain for PMC that doesn't "
3974 "want it", __LINE__));
3975
3976 KASSERT(pm->pm_runcount > 0,
3977 ("[pmc,%d] runcount %d", __LINE__, pm->pm_runcount));
3978
3949 /*
3950 * Retrieve the callchain and mark the sample buffer
3951 * as 'processable' by the timer tick sweep code.
3952 */
3953 ps->ps_nsamples = pmc_save_user_callchain(ps->ps_pc,
3954 pmc_callchaindepth, tf);
3979 /*
3980 * Retrieve the callchain and mark the sample buffer
3981 * as 'processable' by the timer tick sweep code.
3982 */
3983 ps->ps_nsamples = pmc_save_user_callchain(ps->ps_pc,
3984 pmc_callchaindepth, tf);
3985
3986#ifdef INVARIANTS
3987 ncallchains++;
3988#endif
3989
3955 }
3956
3990 }
3991
3992 KASSERT(ncallchains > 0,
3993 ("[pmc,%d] cpu %d didn't find a sample to collect", __LINE__,
3994 cpu));
3995
3957 return;
3958}
3959
3960
3961/*
3962 * Process saved PC samples.
3963 */
3964

--- 21 unchanged lines hidden (view full) ---

3986 break;
3987 if (ps->ps_nsamples == PMC_SAMPLE_INUSE) {
3988 /* Need a rescan at a later time. */
3989 atomic_set_rel_int(&pmc_cpumask, (1 << cpu));
3990 break;
3991 }
3992
3993 pm = ps->ps_pmc;
3996 return;
3997}
3998
3999
4000/*
4001 * Process saved PC samples.
4002 */
4003

--- 21 unchanged lines hidden (view full) ---

4025 break;
4026 if (ps->ps_nsamples == PMC_SAMPLE_INUSE) {
4027 /* Need a rescan at a later time. */
4028 atomic_set_rel_int(&pmc_cpumask, (1 << cpu));
4029 break;
4030 }
4031
4032 pm = ps->ps_pmc;
4033
4034 KASSERT(pm->pm_runcount > 0,
4035 ("[pmc,%d] pm=%p runcount %d", __LINE__, (void *) pm,
4036 pm->pm_runcount));
4037
3994 po = pm->pm_owner;
3995
3996 KASSERT(PMC_IS_SAMPLING_MODE(PMC_TO_MODE(pm)),
3997 ("[pmc,%d] pmc=%p non-sampling mode=%d", __LINE__,
3998 pm, PMC_TO_MODE(pm)));
3999
4000 /* Ignore PMCs that have been switched off */
4001 if (pm->pm_state != PMC_STATE_RUNNING)

--- 736 unchanged lines hidden --- 		4038 po = pm->pm_owner;
4039
4040 KASSERT(PMC_IS_SAMPLING_MODE(PMC_TO_MODE(pm)),
4041 ("[pmc,%d] pmc=%p non-sampling mode=%d", __LINE__,
4042 pm, PMC_TO_MODE(pm)));
4043
4044 /* Ignore PMCs that have been switched off */
4045 if (pm->pm_state != PMC_STATE_RUNNING)

--- 736 unchanged lines hidden ---