1145256Sjkoshy/*- 2174395Sjkoshy * Copyright (c) 2003-2007, Joseph Koshy 3174395Sjkoshy * Copyright (c) 2007 The FreeBSD Foundation 4145256Sjkoshy * All rights reserved. 5145256Sjkoshy * 6174395Sjkoshy * Portions of this software were developed by A. Joseph Koshy under 7174395Sjkoshy * sponsorship from the FreeBSD Foundation and Google, Inc. 8174395Sjkoshy * 9145256Sjkoshy * Redistribution and use in source and binary forms, with or without 10145256Sjkoshy * modification, are permitted provided that the following conditions 11145256Sjkoshy * are met: 12145256Sjkoshy * 1. Redistributions of source code must retain the above copyright 13145256Sjkoshy * notice, this list of conditions and the following disclaimer. 14145256Sjkoshy * 2. Redistributions in binary form must reproduce the above copyright 15145256Sjkoshy * notice, this list of conditions and the following disclaimer in the 16145256Sjkoshy * documentation and/or other materials provided with the distribution. 17145256Sjkoshy * 18145256Sjkoshy * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 19145256Sjkoshy * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 20145256Sjkoshy * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 21145256Sjkoshy * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 22145256Sjkoshy * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 23145256Sjkoshy * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 24145256Sjkoshy * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 25145256Sjkoshy * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 26145256Sjkoshy * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 27145256Sjkoshy * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 28145256Sjkoshy * SUCH DAMAGE. 29145256Sjkoshy * 30145256Sjkoshy * $FreeBSD$ 31145256Sjkoshy */ 32145256Sjkoshy 33145256Sjkoshy/* 34145256Sjkoshy * PMC interface used by the base kernel. 35145256Sjkoshy */ 36145256Sjkoshy 37145256Sjkoshy#ifndef _SYS_PMCKERN_H_ 38145256Sjkoshy#define _SYS_PMCKERN_H_ 39145256Sjkoshy 40145256Sjkoshy#include <sys/param.h> 41233628Sfabient#include <sys/systm.h> 42145256Sjkoshy#include <sys/kernel.h> 43145256Sjkoshy#include <sys/lock.h> 44145256Sjkoshy#include <sys/proc.h> 45145256Sjkoshy#include <sys/sx.h> 46233628Sfabient#include <sys/pmc.h> 47145256Sjkoshy 48233628Sfabient#include <machine/cpufunc.h> 49233628Sfabient 50146799Sjkoshy#define PMC_FN_PROCESS_EXEC 1 51146799Sjkoshy#define PMC_FN_CSW_IN 2 52146799Sjkoshy#define PMC_FN_CSW_OUT 3 53146799Sjkoshy#define PMC_FN_DO_SAMPLES 4 54254813Smarkj#define PMC_FN_UNUSED1 5 55254813Smarkj#define PMC_FN_UNUSED2 6 56157144Sjkoshy#define PMC_FN_MMAP 7 57157144Sjkoshy#define PMC_FN_MUNMAP 8 58174395Sjkoshy#define PMC_FN_USER_CALLCHAIN 9 59233628Sfabient#define PMC_FN_USER_CALLCHAIN_SOFT 10 60233628Sfabient#define PMC_FN_SOFT_SAMPLING 11 61145256Sjkoshy 62233628Sfabient#define PMC_HR 0 /* Hardware ring buffer */ 63233628Sfabient#define PMC_SR 1 /* Software ring buffer */ 64233628Sfabient 65147708Sjkoshystruct pmckern_procexec { 66147708Sjkoshy int pm_credentialschanged; 67157144Sjkoshy uintfptr_t pm_entryaddr; 68147708Sjkoshy}; 69147708Sjkoshy 70157144Sjkoshystruct pmckern_map_in { 71157144Sjkoshy void *pm_file; /* filename or vnode pointer */ 72157144Sjkoshy uintfptr_t pm_address; /* address object is loaded at */ 73157144Sjkoshy}; 74157144Sjkoshy 75157144Sjkoshystruct pmckern_map_out { 76157144Sjkoshy uintfptr_t pm_address; /* start address of region */ 77157144Sjkoshy size_t pm_size; /* size of unmapped region */ 78157144Sjkoshy}; 79157144Sjkoshy 80233628Sfabientstruct pmckern_soft { 81233628Sfabient enum pmc_event pm_ev; 82233628Sfabient int pm_cpu; 83233628Sfabient struct trapframe *pm_tf; 84233628Sfabient}; 85233628Sfabient 86233628Sfabient/* 87233628Sfabient * Soft PMC. 88233628Sfabient */ 89233628Sfabient 90247836Sfabient#define PMC_SOFT_DEFINE_EX(prov, mod, func, name, alloc, release) \ 91233628Sfabient struct pmc_soft pmc_##prov##_##mod##_##func##_##name = \ 92247836Sfabient { 0, alloc, release, { #prov "_" #mod "_" #func "." #name, 0 } }; \ 93233628Sfabient SYSINIT(pmc_##prov##_##mod##_##func##_##name##_init, SI_SUB_KDTRACE, \ 94233628Sfabient SI_ORDER_SECOND + 1, pmc_soft_ev_register, \ 95233628Sfabient &pmc_##prov##_##mod##_##func##_##name ); \ 96233628Sfabient SYSUNINIT(pmc_##prov##_##mod##_##func##_##name##_uninit, \ 97233628Sfabient SI_SUB_KDTRACE, SI_ORDER_SECOND + 1, pmc_soft_ev_deregister, \ 98233628Sfabient &pmc_##prov##_##mod##_##func##_##name ) 99233628Sfabient 100247836Sfabient#define PMC_SOFT_DEFINE(prov, mod, func, name) \ 101247836Sfabient PMC_SOFT_DEFINE_EX(prov, mod, func, name, NULL, NULL) 102247836Sfabient 103233628Sfabient#define PMC_SOFT_DECLARE(prov, mod, func, name) \ 104233628Sfabient extern struct pmc_soft pmc_##prov##_##mod##_##func##_##name 105233628Sfabient 106233628Sfabient/* 107233628Sfabient * PMC_SOFT_CALL can be used anywhere in the kernel. 108233628Sfabient * Require md defined PMC_FAKE_TRAPFRAME. 109233628Sfabient */ 110233628Sfabient#ifdef PMC_FAKE_TRAPFRAME 111233628Sfabient#define PMC_SOFT_CALL(pr, mo, fu, na) \ 112233628Sfabientdo { \ 113233628Sfabient if (pmc_##pr##_##mo##_##fu##_##na.ps_running) { \ 114233628Sfabient struct pmckern_soft ks; \ 115233628Sfabient register_t intr; \ 116233628Sfabient intr = intr_disable(); \ 117233628Sfabient PMC_FAKE_TRAPFRAME(&pmc_tf[curcpu]); \ 118233628Sfabient ks.pm_ev = pmc_##pr##_##mo##_##fu##_##na.ps_ev.pm_ev_code; \ 119233628Sfabient ks.pm_cpu = PCPU_GET(cpuid); \ 120233628Sfabient ks.pm_tf = &pmc_tf[curcpu]; \ 121233628Sfabient PMC_CALL_HOOK_UNLOCKED(curthread, \ 122233628Sfabient PMC_FN_SOFT_SAMPLING, (void *) &ks); \ 123233628Sfabient intr_restore(intr); \ 124233628Sfabient } \ 125233628Sfabient} while (0) 126233628Sfabient#else 127233628Sfabient#define PMC_SOFT_CALL(pr, mo, fu, na) \ 128233628Sfabientdo { \ 129233628Sfabient} while (0) 130233628Sfabient#endif 131233628Sfabient 132233628Sfabient/* 133233628Sfabient * PMC_SOFT_CALL_TF need to be used carefully. 134233628Sfabient * Userland capture will be done during AST processing. 135233628Sfabient */ 136233628Sfabient#define PMC_SOFT_CALL_TF(pr, mo, fu, na, tf) \ 137233628Sfabientdo { \ 138233628Sfabient if (pmc_##pr##_##mo##_##fu##_##na.ps_running) { \ 139233628Sfabient struct pmckern_soft ks; \ 140233628Sfabient register_t intr; \ 141233628Sfabient intr = intr_disable(); \ 142233628Sfabient ks.pm_ev = pmc_##pr##_##mo##_##fu##_##na.ps_ev.pm_ev_code; \ 143233628Sfabient ks.pm_cpu = PCPU_GET(cpuid); \ 144233628Sfabient ks.pm_tf = tf; \ 145233628Sfabient PMC_CALL_HOOK_UNLOCKED(curthread, \ 146233628Sfabient PMC_FN_SOFT_SAMPLING, (void *) &ks); \ 147233628Sfabient intr_restore(intr); \ 148233628Sfabient } \ 149233628Sfabient} while (0) 150233628Sfabient 151233628Sfabientstruct pmc_soft { 152233628Sfabient int ps_running; 153247836Sfabient void (*ps_alloc)(void); 154247836Sfabient void (*ps_release)(void); 155233628Sfabient struct pmc_dyn_event_descr ps_ev; 156233628Sfabient}; 157233628Sfabient 158145256Sjkoshy/* hook */ 159145256Sjkoshyextern int (*pmc_hook)(struct thread *_td, int _function, void *_arg); 160174395Sjkoshyextern int (*pmc_intr)(int _cpu, struct trapframe *_frame); 161145256Sjkoshy 162145256Sjkoshy/* SX lock protecting the hook */ 163145256Sjkoshyextern struct sx pmc_sx; 164145256Sjkoshy 165146799Sjkoshy/* Per-cpu flags indicating availability of sampling data */ 166222813Sattilioextern volatile cpuset_t pmc_cpumask; 167146799Sjkoshy 168147191Sjkoshy/* Count of system-wide sampling PMCs in existence */ 169147191Sjkoshyextern volatile int pmc_ss_count; 170147191Sjkoshy 171148562Sjkoshy/* kernel version number */ 172148562Sjkoshyextern const int pmc_kernel_version; 173148562Sjkoshy 174233628Sfabient/* PMC soft per cpu trapframe */ 175233628Sfabientextern struct trapframe pmc_tf[MAXCPU]; 176233628Sfabient 177146799Sjkoshy/* Hook invocation; for use within the kernel */ 178145256Sjkoshy#define PMC_CALL_HOOK(t, cmd, arg) \ 179145256Sjkoshydo { \ 180145256Sjkoshy sx_slock(&pmc_sx); \ 181145256Sjkoshy if (pmc_hook != NULL) \ 182145256Sjkoshy (pmc_hook)((t), (cmd), (arg)); \ 183145256Sjkoshy sx_sunlock(&pmc_sx); \ 184145256Sjkoshy} while (0) 185145256Sjkoshy 186146799Sjkoshy/* Hook invocation that needs an exclusive lock */ 187145256Sjkoshy#define PMC_CALL_HOOK_X(t, cmd, arg) \ 188145256Sjkoshydo { \ 189145256Sjkoshy sx_xlock(&pmc_sx); \ 190145256Sjkoshy if (pmc_hook != NULL) \ 191145256Sjkoshy (pmc_hook)((t), (cmd), (arg)); \ 192145256Sjkoshy sx_xunlock(&pmc_sx); \ 193145256Sjkoshy} while (0) 194145256Sjkoshy 195146799Sjkoshy/* 196146799Sjkoshy * Some hook invocations (e.g., from context switch and clock handling 197146799Sjkoshy * code) need to be lock-free. 198146799Sjkoshy */ 199146799Sjkoshy#define PMC_CALL_HOOK_UNLOCKED(t, cmd, arg) \ 200145256Sjkoshydo { \ 201145256Sjkoshy if (pmc_hook != NULL) \ 202146799Sjkoshy (pmc_hook)((t), (cmd), (arg)); \ 203145256Sjkoshy} while (0) 204145256Sjkoshy 205146799Sjkoshy#define PMC_SWITCH_CONTEXT(t,cmd) PMC_CALL_HOOK_UNLOCKED(t,cmd,NULL) 206145256Sjkoshy 207146799Sjkoshy/* Check if a process is using HWPMCs.*/ 208145256Sjkoshy#define PMC_PROC_IS_USING_PMCS(p) \ 209239591Sjimharris (__predict_false(p->p_flag & P_HWPMC)) 210145256Sjkoshy 211233628Sfabient/* Check if a thread have pending user capture. */ 212233628Sfabient#define PMC_IS_PENDING_CALLCHAIN(p) \ 213233628Sfabient (__predict_false((p)->td_pflags & TDP_CALLCHAIN)) 214233628Sfabient 215147191Sjkoshy#define PMC_SYSTEM_SAMPLING_ACTIVE() (pmc_ss_count > 0) 216147191Sjkoshy 217146799Sjkoshy/* Check if a CPU has recorded samples. */ 218222813Sattilio#define PMC_CPU_HAS_SAMPLES(C) (__predict_false(CPU_ISSET(C, &pmc_cpumask))) 219146799Sjkoshy 220183266Sjkoshy/* 221183266Sjkoshy * Helper functions. 222183266Sjkoshy */ 223183266Sjkoshyint pmc_cpu_is_disabled(int _cpu); /* deprecated */ 224183266Sjkoshyint pmc_cpu_is_active(int _cpu); 225183266Sjkoshyint pmc_cpu_is_present(int _cpu); 226183266Sjkoshyint pmc_cpu_is_primary(int _cpu); 227183266Sjkoshyunsigned int pmc_cpu_max(void); 228145256Sjkoshy 229183266Sjkoshy#ifdef INVARIANTS 230183266Sjkoshyint pmc_cpu_max_active(void); 231183266Sjkoshy#endif 232183266Sjkoshy 233233628Sfabient/* 234233628Sfabient * Soft events functions. 235233628Sfabient */ 236233628Sfabientvoid pmc_soft_ev_register(struct pmc_soft *ps); 237233628Sfabientvoid pmc_soft_ev_deregister(struct pmc_soft *ps); 238233628Sfabientstruct pmc_soft *pmc_soft_ev_acquire(enum pmc_event ev); 239233628Sfabientvoid pmc_soft_ev_release(struct pmc_soft *ps); 240233628Sfabient 241145256Sjkoshy#endif /* _SYS_PMCKERN_H_ */ 242