hwpmc_mod.c (170307) | hwpmc_mod.c (174395) |
---|---|
1/*- | 1/*- |
2 * Copyright (c) 2003-2006 Joseph Koshy | 2 * Copyright (c) 2003-2007 Joseph Koshy 3 * Copyright (c) 2007 The FreeBSD Foundation |
3 * All rights reserved. 4 * | 4 * All rights reserved. 5 * |
6 * Portions of this software were developed by A. Joseph Koshy under 7 * sponsorship from the FreeBSD Foundation and Google, Inc. 8 * |
|
5 * Redistribution and use in source and binary forms, with or without 6 * modification, are permitted provided that the following conditions 7 * are met: 8 * 1. Redistributions of source code must retain the above copyright 9 * notice, this list of conditions and the following disclaimer. 10 * 2. Redistributions in binary form must reproduce the above copyright 11 * notice, this list of conditions and the following disclaimer in the 12 * documentation and/or other materials provided with the distribution. --- 8 unchanged lines hidden (view full) --- 21 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 22 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 23 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 24 * SUCH DAMAGE. 25 * 26 */ 27 28#include <sys/cdefs.h> | 9 * Redistribution and use in source and binary forms, with or without 10 * modification, are permitted provided that the following conditions 11 * are met: 12 * 1. Redistributions of source code must retain the above copyright 13 * notice, this list of conditions and the following disclaimer. 14 * 2. Redistributions in binary form must reproduce the above copyright 15 * notice, this list of conditions and the following disclaimer in the 16 * documentation and/or other materials provided with the distribution. --- 8 unchanged lines hidden (view full) --- 25 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 26 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 27 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 28 * SUCH DAMAGE. 29 * 30 */ 31 32#include <sys/cdefs.h> |
29__FBSDID("$FreeBSD: head/sys/dev/hwpmc/hwpmc_mod.c 170307 2007-06-05 00:00:57Z jeff $"); | 33__FBSDID("$FreeBSD: head/sys/dev/hwpmc/hwpmc_mod.c 174395 2007-12-07 08:20:17Z jkoshy $"); |
30 31#include <sys/param.h> 32#include <sys/eventhandler.h> 33#include <sys/jail.h> 34#include <sys/kernel.h> 35#include <sys/kthread.h> 36#include <sys/limits.h> 37#include <sys/lock.h> --- 123 unchanged lines hidden (view full) --- 161static int load(struct module *module, int cmd, void *arg); 162static int pmc_attach_process(struct proc *p, struct pmc *pm); 163static struct pmc *pmc_allocate_pmc_descriptor(void); 164static struct pmc_owner *pmc_allocate_owner_descriptor(struct proc *p); 165static int pmc_attach_one_process(struct proc *p, struct pmc *pm); 166static int pmc_can_allocate_rowindex(struct proc *p, unsigned int ri, 167 int cpu); 168static int pmc_can_attach(struct pmc *pm, struct proc *p); | 34 35#include <sys/param.h> 36#include <sys/eventhandler.h> 37#include <sys/jail.h> 38#include <sys/kernel.h> 39#include <sys/kthread.h> 40#include <sys/limits.h> 41#include <sys/lock.h> --- 123 unchanged lines hidden (view full) --- 165static int load(struct module *module, int cmd, void *arg); 166static int pmc_attach_process(struct proc *p, struct pmc *pm); 167static struct pmc *pmc_allocate_pmc_descriptor(void); 168static struct pmc_owner *pmc_allocate_owner_descriptor(struct proc *p); 169static int pmc_attach_one_process(struct proc *p, struct pmc *pm); 170static int pmc_can_allocate_rowindex(struct proc *p, unsigned int ri, 171 int cpu); 172static int pmc_can_attach(struct pmc *pm, struct proc *p); |
173static void pmc_capture_user_callchain(int cpu, struct trapframe *tf); |
|
169static void pmc_cleanup(void); 170static int pmc_detach_process(struct proc *p, struct pmc *pm); 171static int pmc_detach_one_process(struct proc *p, struct pmc *pm, 172 int flags); 173static void pmc_destroy_owner_descriptor(struct pmc_owner *po); 174static struct pmc_owner *pmc_find_owner_descriptor(struct proc *p); 175static int pmc_find_pmc(pmc_id_t pmcid, struct pmc **pm); 176static struct pmc *pmc_find_pmc_descriptor_in_process(struct pmc_owner *po, 177 pmc_id_t pmc); 178static struct pmc_process *pmc_find_process_descriptor(struct proc *p, 179 uint32_t mode); 180static void pmc_force_context_switch(void); 181static void pmc_link_target_process(struct pmc *pm, 182 struct pmc_process *pp); | 174static void pmc_cleanup(void); 175static int pmc_detach_process(struct proc *p, struct pmc *pm); 176static int pmc_detach_one_process(struct proc *p, struct pmc *pm, 177 int flags); 178static void pmc_destroy_owner_descriptor(struct pmc_owner *po); 179static struct pmc_owner *pmc_find_owner_descriptor(struct proc *p); 180static int pmc_find_pmc(pmc_id_t pmcid, struct pmc **pm); 181static struct pmc *pmc_find_pmc_descriptor_in_process(struct pmc_owner *po, 182 pmc_id_t pmc); 183static struct pmc_process *pmc_find_process_descriptor(struct proc *p, 184 uint32_t mode); 185static void pmc_force_context_switch(void); 186static void pmc_link_target_process(struct pmc *pm, 187 struct pmc_process *pp); |
188static void pmc_log_all_process_mappings(struct pmc_owner *po); 189static void pmc_log_kernel_mappings(struct pmc *pm); 190static void pmc_log_process_mappings(struct pmc_owner *po, struct proc *p); |
|
183static void pmc_maybe_remove_owner(struct pmc_owner *po); 184static void pmc_process_csw_in(struct thread *td); 185static void pmc_process_csw_out(struct thread *td); 186static void pmc_process_exit(void *arg, struct proc *p); 187static void pmc_process_fork(void *arg, struct proc *p1, 188 struct proc *p2, int n); 189static void pmc_process_samples(int cpu); 190static void pmc_release_pmc_descriptor(struct pmc *pmc); --- 9 unchanged lines hidden (view full) --- 200 struct pmc_process *pp); 201 202/* 203 * Kernel tunables and sysctl(8) interface. 204 */ 205 206SYSCTL_NODE(_kern, OID_AUTO, hwpmc, CTLFLAG_RW, 0, "HWPMC parameters"); 207 | 191static void pmc_maybe_remove_owner(struct pmc_owner *po); 192static void pmc_process_csw_in(struct thread *td); 193static void pmc_process_csw_out(struct thread *td); 194static void pmc_process_exit(void *arg, struct proc *p); 195static void pmc_process_fork(void *arg, struct proc *p1, 196 struct proc *p2, int n); 197static void pmc_process_samples(int cpu); 198static void pmc_release_pmc_descriptor(struct pmc *pmc); --- 9 unchanged lines hidden (view full) --- 208 struct pmc_process *pp); 209 210/* 211 * Kernel tunables and sysctl(8) interface. 212 */ 213 214SYSCTL_NODE(_kern, OID_AUTO, hwpmc, CTLFLAG_RW, 0, "HWPMC parameters"); 215 |
216static int pmc_callchaindepth = PMC_CALLCHAIN_DEPTH; 217TUNABLE_INT(PMC_SYSCTL_NAME_PREFIX "callchaindepth", &pmc_callchaindepth); 218SYSCTL_INT(_kern_hwpmc, OID_AUTO, callchaindepth, CTLFLAG_TUN|CTLFLAG_RD, 219 &pmc_callchaindepth, 0, "depth of call chain records"); 220 |
|
208#ifdef DEBUG 209struct pmc_debugflags pmc_debugflags = PMC_DEBUG_DEFAULT_FLAGS; 210char pmc_debugstr[PMC_DEBUG_STRSIZE]; 211TUNABLE_STR(PMC_SYSCTL_NAME_PREFIX "debugflags", pmc_debugstr, 212 sizeof(pmc_debugstr)); 213SYSCTL_PROC(_kern_hwpmc, OID_AUTO, debugflags, 214 CTLTYPE_STRING|CTLFLAG_RW|CTLFLAG_TUN, 215 0, 0, pmc_debugflags_sysctl_handler, "A", "debug flags"); --- 5 unchanged lines hidden (view full) --- 221 */ 222 223static int pmc_hashsize = PMC_HASH_SIZE; 224TUNABLE_INT(PMC_SYSCTL_NAME_PREFIX "hashsize", &pmc_hashsize); 225SYSCTL_INT(_kern_hwpmc, OID_AUTO, hashsize, CTLFLAG_TUN|CTLFLAG_RD, 226 &pmc_hashsize, 0, "rows in hash tables"); 227 228/* | 221#ifdef DEBUG 222struct pmc_debugflags pmc_debugflags = PMC_DEBUG_DEFAULT_FLAGS; 223char pmc_debugstr[PMC_DEBUG_STRSIZE]; 224TUNABLE_STR(PMC_SYSCTL_NAME_PREFIX "debugflags", pmc_debugstr, 225 sizeof(pmc_debugstr)); 226SYSCTL_PROC(_kern_hwpmc, OID_AUTO, debugflags, 227 CTLTYPE_STRING|CTLFLAG_RW|CTLFLAG_TUN, 228 0, 0, pmc_debugflags_sysctl_handler, "A", "debug flags"); --- 5 unchanged lines hidden (view full) --- 234 */ 235 236static int pmc_hashsize = PMC_HASH_SIZE; 237TUNABLE_INT(PMC_SYSCTL_NAME_PREFIX "hashsize", &pmc_hashsize); 238SYSCTL_INT(_kern_hwpmc, OID_AUTO, hashsize, CTLFLAG_TUN|CTLFLAG_RD, 239 &pmc_hashsize, 0, "rows in hash tables"); 240 241/* |
229 * kern.hwpmc.nsamples --- number of PC samples per CPU | 242 * kern.hwpmc.nsamples --- number of PC samples/callchain stacks per CPU |
230 */ 231 232static int pmc_nsamples = PMC_NSAMPLES; 233TUNABLE_INT(PMC_SYSCTL_NAME_PREFIX "nsamples", &pmc_nsamples); 234SYSCTL_INT(_kern_hwpmc, OID_AUTO, nsamples, CTLFLAG_TUN|CTLFLAG_RD, 235 &pmc_nsamples, 0, "number of PC samples per CPU"); 236 | 243 */ 244 245static int pmc_nsamples = PMC_NSAMPLES; 246TUNABLE_INT(PMC_SYSCTL_NAME_PREFIX "nsamples", &pmc_nsamples); 247SYSCTL_INT(_kern_hwpmc, OID_AUTO, nsamples, CTLFLAG_TUN|CTLFLAG_RD, 248 &pmc_nsamples, 0, "number of PC samples per CPU"); 249 |
250 |
|
237/* 238 * kern.hwpmc.mtxpoolsize -- number of mutexes in the mutex pool. 239 */ 240 241static int pmc_mtxpool_size = PMC_MTXPOOL_SIZE; 242TUNABLE_INT(PMC_SYSCTL_NAME_PREFIX "mtxpoolsize", &pmc_mtxpool_size); 243SYSCTL_INT(_kern_hwpmc, OID_AUTO, mtxpoolsize, CTLFLAG_TUN|CTLFLAG_RD, 244 &pmc_mtxpool_size, 0, "size of spin mutex pool"); --- 707 unchanged lines hidden (view full) --- 952 pm->pm_flags |= PMC_F_ATTACH_DONE; /* mark as attached */ 953 954 /* issue an attach event to a configured log file */ 955 if (pm->pm_owner->po_flags & PMC_PO_OWNS_LOGFILE) { 956 pmc_getfilename(p->p_textvp, &fullpath, &freepath); 957 pmclog_process_pmcattach(pm, p->p_pid, fullpath); 958 if (freepath) 959 FREE(freepath, M_TEMP); | 251/* 252 * kern.hwpmc.mtxpoolsize -- number of mutexes in the mutex pool. 253 */ 254 255static int pmc_mtxpool_size = PMC_MTXPOOL_SIZE; 256TUNABLE_INT(PMC_SYSCTL_NAME_PREFIX "mtxpoolsize", &pmc_mtxpool_size); 257SYSCTL_INT(_kern_hwpmc, OID_AUTO, mtxpoolsize, CTLFLAG_TUN|CTLFLAG_RD, 258 &pmc_mtxpool_size, 0, "size of spin mutex pool"); --- 707 unchanged lines hidden (view full) --- 966 pm->pm_flags |= PMC_F_ATTACH_DONE; /* mark as attached */ 967 968 /* issue an attach event to a configured log file */ 969 if (pm->pm_owner->po_flags & PMC_PO_OWNS_LOGFILE) { 970 pmc_getfilename(p->p_textvp, &fullpath, &freepath); 971 pmclog_process_pmcattach(pm, p->p_pid, fullpath); 972 if (freepath) 973 FREE(freepath, M_TEMP); |
974 if (PMC_IS_SAMPLING_MODE(PMC_TO_MODE(pm))) 975 pmc_log_process_mappings(pm->pm_owner, p); |
|
960 } 961 /* mark process as using HWPMCs */ 962 PROC_LOCK(p); 963 p->p_flag |= P_HWPMC; 964 PROC_UNLOCK(p); 965 966 return 0; 967} --- 476 unchanged lines hidden (view full) --- 1444 struct pmc_owner *po; 1445 1446 sx_assert(&pmc_sx, SX_LOCKED); 1447 1448 LIST_FOREACH(po, &pmc_ss_owners, po_ssnext) 1449 if (po->po_flags & PMC_PO_OWNS_LOGFILE) 1450 pmclog_process_map_out(po, (pid_t) -1, 1451 pkm->pm_address, pkm->pm_address + pkm->pm_size); | 976 } 977 /* mark process as using HWPMCs */ 978 PROC_LOCK(p); 979 p->p_flag |= P_HWPMC; 980 PROC_UNLOCK(p); 981 982 return 0; 983} --- 476 unchanged lines hidden (view full) --- 1460 struct pmc_owner *po; 1461 1462 sx_assert(&pmc_sx, SX_LOCKED); 1463 1464 LIST_FOREACH(po, &pmc_ss_owners, po_ssnext) 1465 if (po->po_flags & PMC_PO_OWNS_LOGFILE) 1466 pmclog_process_map_out(po, (pid_t) -1, 1467 pkm->pm_address, pkm->pm_address + pkm->pm_size); |
1452 | 1468 |
1453 /* 1454 * TODO: Notify owners of process-sampling PMCs. 1455 */ 1456} 1457 1458/* 1459 * A mapping change for a process. 1460 */ --- 62 unchanged lines hidden (view full) --- 1523 for (ri = 0; ri < md->pmd_npmc; ri++) 1524 if ((pm = pp->pp_pmcs[ri].pp_pmc) != NULL && 1525 PMC_IS_SAMPLING_MODE(PMC_TO_MODE(pm))) 1526 pmclog_process_map_out(pm->pm_owner, pid, 1527 pkm->pm_address, pkm->pm_address + pkm->pm_size); 1528} 1529 1530/* | 1469 /* 1470 * TODO: Notify owners of process-sampling PMCs. 1471 */ 1472} 1473 1474/* 1475 * A mapping change for a process. 1476 */ --- 62 unchanged lines hidden (view full) --- 1539 for (ri = 0; ri < md->pmd_npmc; ri++) 1540 if ((pm = pp->pp_pmcs[ri].pp_pmc) != NULL && 1541 PMC_IS_SAMPLING_MODE(PMC_TO_MODE(pm))) 1542 pmclog_process_map_out(pm->pm_owner, pid, 1543 pkm->pm_address, pkm->pm_address + pkm->pm_size); 1544} 1545 1546/* |
1547 * Log mapping information about the kernel. 1548 */ 1549 1550static void 1551pmc_log_kernel_mappings(struct pmc *pm) 1552{ 1553 struct pmc_owner *po; 1554 struct pmckern_map_in *km, *kmbase; 1555 1556 sx_assert(&pmc_sx, SX_LOCKED); 1557 KASSERT(PMC_IS_SAMPLING_MODE(PMC_TO_MODE(pm)), 1558 ("[pmc,%d] non-sampling PMC (%p) desires mapping information", 1559 __LINE__, (void *) pm)); 1560 1561 po = pm->pm_owner; 1562 1563 if (po->po_flags & PMC_PO_INITIAL_MAPPINGS_DONE) 1564 return; 1565 1566 /* 1567 * Log the current set of kernel modules. 1568 */ 1569 kmbase = linker_hwpmc_list_objects(); 1570 for (km = kmbase; km->pm_file != NULL; km++) { 1571 PMCDBG(LOG,REG,1,"%s %p", (char *) km->pm_file, 1572 (void *) km->pm_address); 1573 pmclog_process_map_in(po, (pid_t) -1, km->pm_address, 1574 km->pm_file); 1575 } 1576 FREE(kmbase, M_LINKER); 1577 1578 po->po_flags |= PMC_PO_INITIAL_MAPPINGS_DONE; 1579} 1580 1581/* 1582 * Log the mappings for a single process. 1583 */ 1584 1585static void 1586pmc_log_process_mappings(struct pmc_owner *po, struct proc *p) 1587{ 1588} 1589 1590/* 1591 * Log mappings for all processes in the system. 1592 */ 1593 1594static void 1595pmc_log_all_process_mappings(struct pmc_owner *po) 1596{ 1597 struct proc *p, *top; 1598 1599 sx_assert(&pmc_sx, SX_XLOCKED); 1600 1601 if ((p = pfind(1)) == NULL) 1602 panic("[pmc,%d] Cannot find init", __LINE__); 1603 1604 PROC_UNLOCK(p); 1605 1606 sx_slock(&proctree_lock); 1607 1608 top = p; 1609 1610 for (;;) { 1611 pmc_log_process_mappings(po, p); 1612 if (!LIST_EMPTY(&p->p_children)) 1613 p = LIST_FIRST(&p->p_children); 1614 else for (;;) { 1615 if (p == top) 1616 goto done; 1617 if (LIST_NEXT(p, p_sibling)) { 1618 p = LIST_NEXT(p, p_sibling); 1619 break; 1620 } 1621 p = p->p_pptr; 1622 } 1623 } 1624 done: 1625 sx_sunlock(&proctree_lock); 1626} 1627 1628/* |
|
1531 * The 'hook' invoked from the kernel proper 1532 */ 1533 1534 1535#ifdef DEBUG 1536const char *pmc_hooknames[] = { 1537 /* these strings correspond to PMC_FN_* in <sys/pmckern.h> */ 1538 "", 1539 "EXEC", 1540 "CSW-IN", 1541 "CSW-OUT", 1542 "SAMPLE", 1543 "KLDLOAD", 1544 "KLDUNLOAD", 1545 "MMAP", | 1629 * The 'hook' invoked from the kernel proper 1630 */ 1631 1632 1633#ifdef DEBUG 1634const char *pmc_hooknames[] = { 1635 /* these strings correspond to PMC_FN_* in <sys/pmckern.h> */ 1636 "", 1637 "EXEC", 1638 "CSW-IN", 1639 "CSW-OUT", 1640 "SAMPLE", 1641 "KLDLOAD", 1642 "KLDUNLOAD", 1643 "MMAP", |
1546 "MUNMAP" | 1644 "MUNMAP", 1645 "CALLCHAIN" |
1547}; 1548#endif 1549 1550static int 1551pmc_hook_handler(struct thread *td, int function, void *arg) 1552{ 1553 1554 PMCDBG(MOD,PMH,1, "hook td=%p func=%d \"%s\" arg=%p", td, function, --- 166 unchanged lines hidden (view full) --- 1721 pmc_process_mmap(td, (struct pmckern_map_in *) arg); 1722 break; 1723 1724 case PMC_FN_MUNMAP: 1725 sx_assert(&pmc_sx, SX_LOCKED); 1726 pmc_process_munmap(td, (struct pmckern_map_out *) arg); 1727 break; 1728 | 1646}; 1647#endif 1648 1649static int 1650pmc_hook_handler(struct thread *td, int function, void *arg) 1651{ 1652 1653 PMCDBG(MOD,PMH,1, "hook td=%p func=%d \"%s\" arg=%p", td, function, --- 166 unchanged lines hidden (view full) --- 1820 pmc_process_mmap(td, (struct pmckern_map_in *) arg); 1821 break; 1822 1823 case PMC_FN_MUNMAP: 1824 sx_assert(&pmc_sx, SX_LOCKED); 1825 pmc_process_munmap(td, (struct pmckern_map_out *) arg); 1826 break; 1827 |
1828 case PMC_FN_USER_CALLCHAIN: 1829 /* 1830 * Record a call chain. 1831 */ 1832 pmc_capture_user_callchain(PCPU_GET(cpuid), 1833 (struct trapframe *) arg); 1834 break; 1835 |
|
1729 default: 1730#ifdef DEBUG 1731 KASSERT(0, ("[pmc,%d] unknown hook %d\n", __LINE__, function)); 1732#endif 1733 break; 1734 1735 } 1736 --- 579 unchanged lines hidden (view full) --- 2316 mode = PMC_TO_MODE(pm); 2317 ri = PMC_TO_ROWINDEX(pm); 2318 error = 0; 2319 2320 PMCDBG(PMC,OPS,1, "start pmc=%p mode=%d ri=%d", pm, mode, ri); 2321 2322 po = pm->pm_owner; 2323 | 1836 default: 1837#ifdef DEBUG 1838 KASSERT(0, ("[pmc,%d] unknown hook %d\n", __LINE__, function)); 1839#endif 1840 break; 1841 1842 } 1843 --- 579 unchanged lines hidden (view full) --- 2423 mode = PMC_TO_MODE(pm); 2424 ri = PMC_TO_ROWINDEX(pm); 2425 error = 0; 2426 2427 PMCDBG(PMC,OPS,1, "start pmc=%p mode=%d ri=%d", pm, mode, ri); 2428 2429 po = pm->pm_owner; 2430 |
2431 /* 2432 * Disallow PMCSTART if a logfile is required but has not been 2433 * configured yet. 2434 */ 2435 if ((pm->pm_flags & PMC_F_NEEDS_LOGFILE) && 2436 (po->po_flags & PMC_PO_OWNS_LOGFILE) == 0) 2437 return EDOOFUS; /* programming error */ 2438 2439 /* 2440 * If this is a sampling mode PMC, log mapping information for 2441 * the kernel modules that are currently loaded. 2442 */ 2443 if (PMC_IS_SAMPLING_MODE(PMC_TO_MODE(pm))) 2444 pmc_log_kernel_mappings(pm); 2445 |
|
2324 if (PMC_IS_VIRTUAL_MODE(mode)) { 2325 2326 /* 2327 * If a PMCATTACH has never been done on this PMC, 2328 * attach it to its owner process. 2329 */ 2330 2331 if (LIST_EMPTY(&pm->pm_targets)) 2332 error = (pm->pm_flags & PMC_F_ATTACH_DONE) ? ESRCH : 2333 pmc_attach_process(po->po_owner, pm); 2334 2335 /* | 2446 if (PMC_IS_VIRTUAL_MODE(mode)) { 2447 2448 /* 2449 * If a PMCATTACH has never been done on this PMC, 2450 * attach it to its owner process. 2451 */ 2452 2453 if (LIST_EMPTY(&pm->pm_targets)) 2454 error = (pm->pm_flags & PMC_F_ATTACH_DONE) ? ESRCH : 2455 pmc_attach_process(po->po_owner, pm); 2456 2457 /* |
2336 * Disallow PMCSTART if a logfile is required but has not 2337 * been configured yet. 2338 */ 2339 2340 if (error == 0 && (pm->pm_flags & PMC_F_NEEDS_LOGFILE) && 2341 (po->po_flags & PMC_PO_OWNS_LOGFILE) == 0) 2342 error = EDOOFUS; 2343 2344 /* | |
2345 * If the PMC is attached to its owner, then force a context 2346 * switch to ensure that the MD state gets set correctly. 2347 */ 2348 2349 if (error == 0) { 2350 pm->pm_state = PMC_STATE_RUNNING; 2351 if (pm->pm_flags & PMC_F_ATTACHED_TO_OWNER) 2352 pmc_force_context_switch(); 2353 } 2354 2355 return error; 2356 } 2357 2358 2359 /* 2360 * A system-wide PMC. | 2458 * If the PMC is attached to its owner, then force a context 2459 * switch to ensure that the MD state gets set correctly. 2460 */ 2461 2462 if (error == 0) { 2463 pm->pm_state = PMC_STATE_RUNNING; 2464 if (pm->pm_flags & PMC_F_ATTACHED_TO_OWNER) 2465 pmc_force_context_switch(); 2466 } 2467 2468 return error; 2469 } 2470 2471 2472 /* 2473 * A system-wide PMC. |
2361 */ 2362 2363 if ((pm->pm_flags & PMC_F_NEEDS_LOGFILE) && 2364 (po->po_flags & PMC_PO_OWNS_LOGFILE) == 0) 2365 return EDOOFUS; /* programming error */ 2366 2367 /* | 2474 * |
2368 * Add the owner to the global list if this is a system-wide 2369 * sampling PMC. 2370 */ 2371 2372 if (mode == PMC_MODE_SS) { 2373 if (po->po_sscount == 0) { 2374 LIST_INSERT_HEAD(&pmc_ss_owners, po, po_ssnext); 2375 atomic_add_rel_int(&pmc_ss_count, 1); 2376 PMCDBG(PMC,OPS,1, "po=%p in global list", po); 2377 } 2378 po->po_sscount++; 2379 } 2380 | 2475 * Add the owner to the global list if this is a system-wide 2476 * sampling PMC. 2477 */ 2478 2479 if (mode == PMC_MODE_SS) { 2480 if (po->po_sscount == 0) { 2481 LIST_INSERT_HEAD(&pmc_ss_owners, po, po_ssnext); 2482 atomic_add_rel_int(&pmc_ss_count, 1); 2483 PMCDBG(PMC,OPS,1, "po=%p in global list", po); 2484 } 2485 po->po_sscount++; 2486 } 2487 |
2381 /* TODO: dump system wide process mappings to the log? */ | 2488 /* Log mapping information for all processes in the system. */ 2489 pmc_log_all_process_mappings(po); |
2382 2383 /* 2384 * Move to the CPU associated with this 2385 * PMC, and start the hardware. 2386 */ 2387 2388 pmc_save_cpu_binding(&pb); 2389 --- 159 unchanged lines hidden (view full) --- 2549 * XXX This OP will be reworked. 2550 */ 2551 2552 case PMC_OP_CONFIGURELOG: 2553 { 2554 struct proc *p; 2555 struct pmc *pm; 2556 struct pmc_owner *po; | 2490 2491 /* 2492 * Move to the CPU associated with this 2493 * PMC, and start the hardware. 2494 */ 2495 2496 pmc_save_cpu_binding(&pb); 2497 --- 159 unchanged lines hidden (view full) --- 2657 * XXX This OP will be reworked. 2658 */ 2659 2660 case PMC_OP_CONFIGURELOG: 2661 { 2662 struct proc *p; 2663 struct pmc *pm; 2664 struct pmc_owner *po; |
2557 struct pmckern_map_in *km, *kmbase; | |
2558 struct pmc_op_configurelog cl; 2559 2560 sx_assert(&pmc_sx, SX_XLOCKED); 2561 2562 if ((error = copyin(arg, &cl, sizeof(cl))) != 0) 2563 break; 2564 2565 /* mark this process as owning a log file */ --- 22 unchanged lines hidden (view full) --- 2588 pmc_stop(pm); 2589 error = pmclog_deconfigure_log(po); 2590 } 2591 } else 2592 error = EINVAL; 2593 2594 if (error) 2595 break; | 2665 struct pmc_op_configurelog cl; 2666 2667 sx_assert(&pmc_sx, SX_XLOCKED); 2668 2669 if ((error = copyin(arg, &cl, sizeof(cl))) != 0) 2670 break; 2671 2672 /* mark this process as owning a log file */ --- 22 unchanged lines hidden (view full) --- 2695 pmc_stop(pm); 2696 error = pmclog_deconfigure_log(po); 2697 } 2698 } else 2699 error = EINVAL; 2700 2701 if (error) 2702 break; |
2596 2597 /* 2598 * Log the current set of kernel modules. 2599 */ 2600 kmbase = linker_hwpmc_list_objects(); 2601 for (km = kmbase; km->pm_file != NULL; km++) { 2602 PMCDBG(LOG,REG,1,"%s %p", (char *) km->pm_file, 2603 (void *) km->pm_address); 2604 pmclog_process_map_in(po, (pid_t) -1, km->pm_address, 2605 km->pm_file); 2606 } 2607 FREE(kmbase, M_LINKER); | |
2608 } 2609 break; 2610 2611 2612 /* 2613 * Flush a log file. 2614 */ 2615 --- 324 unchanged lines hidden (view full) --- 2940 if (error) 2941 break; 2942 2943 /* 2944 * Look for valid values for 'pm_flags' 2945 */ 2946 2947 if ((pa.pm_flags & ~(PMC_F_DESCENDANTS | PMC_F_LOG_PROCCSW | | 2703 } 2704 break; 2705 2706 2707 /* 2708 * Flush a log file. 2709 */ 2710 --- 324 unchanged lines hidden (view full) --- 3035 if (error) 3036 break; 3037 3038 /* 3039 * Look for valid values for 'pm_flags' 3040 */ 3041 3042 if ((pa.pm_flags & ~(PMC_F_DESCENDANTS | PMC_F_LOG_PROCCSW | |
2948 PMC_F_LOG_PROCEXIT)) != 0) { | 3043 PMC_F_LOG_PROCEXIT | PMC_F_CALLCHAIN)) != 0) { |
2949 error = EINVAL; 2950 break; 2951 } 2952 2953 /* process logging options are not allowed for system PMCs */ 2954 if (PMC_IS_SYSTEM_MODE(mode) && (pa.pm_flags & 2955 (PMC_F_LOG_PROCCSW | PMC_F_LOG_PROCEXIT))) { 2956 error = EINVAL; --- 671 unchanged lines hidden (view full) --- 3628} 3629 3630/* 3631 * Helper functions 3632 */ 3633 3634 3635/* | 3044 error = EINVAL; 3045 break; 3046 } 3047 3048 /* process logging options are not allowed for system PMCs */ 3049 if (PMC_IS_SYSTEM_MODE(mode) && (pa.pm_flags & 3050 (PMC_F_LOG_PROCCSW | PMC_F_LOG_PROCEXIT))) { 3051 error = EINVAL; --- 671 unchanged lines hidden (view full) --- 3723} 3724 3725/* 3726 * Helper functions 3727 */ 3728 3729 3730/* |
3731 * Mark the thread as needing callchain capture and post an AST. The 3732 * actual callchain capture will be done in a context where it is safe 3733 * to take page faults. 3734 */ 3735 3736static void 3737pmc_post_callchain_ast(void) 3738{ 3739 struct thread *td; 3740 3741 td = curthread; 3742 3743 /* 3744 * Mark this thread as needing processing in ast(). 3745 * td->td_pflags will be safe to touch as the process was in 3746 * user space when it was interrupted. 3747 */ 3748 td->td_pflags |= TDP_CALLCHAIN; 3749 3750 /* 3751 * Again, since we've entered this function directly from 3752 * userland, `td' is guaranteed to be not locked by this CPU, 3753 * so its safe to try acquire the thread lock even though we 3754 * are executing in an NMI context. We need to acquire this 3755 * lock before touching `td_flags' because other CPUs may be 3756 * in the process of touching this field. 3757 */ 3758 thread_lock(td); 3759 td->td_flags |= TDF_ASTPENDING; 3760 thread_unlock(td); 3761 3762 return; 3763} 3764 3765/* |
|
3636 * Interrupt processing. 3637 * | 3766 * Interrupt processing. 3767 * |
3638 * Find a free slot in the per-cpu array of PC samples and write the 3639 * current (PMC,PID,PC) triple to it. If an event was successfully 3640 * added, a bit is set in mask 'pmc_cpumask' denoting that the 3641 * DO_SAMPLES hook needs to be invoked from the clock handler. | 3768 * Find a free slot in the per-cpu array of samples and capture the 3769 * current callchain there. If a sample was successfully added, a bit 3770 * is set in mask 'pmc_cpumask' denoting that the DO_SAMPLES hook 3771 * needs to be invoked from the clock handler. |
3642 * 3643 * This function is meant to be called from an NMI handler. It cannot 3644 * use any of the locking primitives supplied by the OS. 3645 */ 3646 3647int | 3772 * 3773 * This function is meant to be called from an NMI handler. It cannot 3774 * use any of the locking primitives supplied by the OS. 3775 */ 3776 3777int |
3648pmc_process_interrupt(int cpu, struct pmc *pm, uintfptr_t pc, int usermode) | 3778pmc_process_interrupt(int cpu, struct pmc *pm, struct trapframe *tf, 3779 int inuserspace) |
3649{ | 3780{ |
3650 int error, ri; | 3781 int error, callchaindepth; |
3651 struct thread *td; 3652 struct pmc_sample *ps; 3653 struct pmc_samplebuffer *psb; 3654 3655 error = 0; | 3782 struct thread *td; 3783 struct pmc_sample *ps; 3784 struct pmc_samplebuffer *psb; 3785 3786 error = 0; |
3656 ri = PMC_TO_ROWINDEX(pm); | |
3657 | 3787 |
3788 /* 3789 * Allocate space for a sample buffer. 3790 */ |
|
3658 psb = pmc_pcpu[cpu]->pc_sb; 3659 3660 ps = psb->ps_write; | 3791 psb = pmc_pcpu[cpu]->pc_sb; 3792 3793 ps = psb->ps_write; |
3661 if (ps->ps_pc) { /* in use, reader hasn't caught up */ | 3794 if (ps->ps_nsamples) { /* in use, reader hasn't caught up */ |
3662 pm->pm_stalled = 1; 3663 atomic_add_int(&pmc_stats.pm_intr_bufferfull, 1); | 3795 pm->pm_stalled = 1; 3796 atomic_add_int(&pmc_stats.pm_intr_bufferfull, 1); |
3664 PMCDBG(SAM,INT,1,"(spc) cpu=%d pm=%p pc=%jx um=%d wr=%d rd=%d", 3665 cpu, pm, (uint64_t) pc, usermode, | 3797 PMCDBG(SAM,INT,1,"(spc) cpu=%d pm=%p tf=%p um=%d wr=%d rd=%d", 3798 cpu, pm, (void *) tf, inuserspace, |
3666 (int) (psb->ps_write - psb->ps_samples), 3667 (int) (psb->ps_read - psb->ps_samples)); 3668 error = ENOMEM; 3669 goto done; 3670 } 3671 | 3799 (int) (psb->ps_write - psb->ps_samples), 3800 (int) (psb->ps_read - psb->ps_samples)); 3801 error = ENOMEM; 3802 goto done; 3803 } 3804 |
3672 /* fill in entry */ 3673 PMCDBG(SAM,INT,1,"cpu=%d pm=%p pc=%jx um=%d wr=%d rd=%d", cpu, pm, 3674 (uint64_t) pc, usermode, | 3805 3806 /* Fill in entry. */ 3807 PMCDBG(SAM,INT,1,"cpu=%d pm=%p tf=%p um=%d wr=%d rd=%d", cpu, pm, 3808 (void *) tf, inuserspace, |
3675 (int) (psb->ps_write - psb->ps_samples), 3676 (int) (psb->ps_read - psb->ps_samples)); 3677 | 3809 (int) (psb->ps_write - psb->ps_samples), 3810 (int) (psb->ps_read - psb->ps_samples)); 3811 |
3678 atomic_add_rel_32(&pm->pm_runcount, 1); /* hold onto PMC */ | 3812 atomic_add_rel_32(&pm->pm_runcount, 1); /* hold onto PMC */ |
3679 ps->ps_pmc = pm; 3680 if ((td = curthread) && td->td_proc) 3681 ps->ps_pid = td->td_proc->p_pid; 3682 else 3683 ps->ps_pid = -1; | 3813 ps->ps_pmc = pm; 3814 if ((td = curthread) && td->td_proc) 3815 ps->ps_pid = td->td_proc->p_pid; 3816 else 3817 ps->ps_pid = -1; |
3684 ps->ps_usermode = usermode; 3685 ps->ps_pc = pc; /* mark entry as in use */ | 3818 ps->ps_cpu = cpu; 3819 ps->ps_flags = inuserspace ? PMC_CC_F_USERSPACE : 0; |
3686 | 3820 |
3821 callchaindepth = (pm->pm_flags & PMC_F_CALLCHAIN) ? 3822 pmc_callchaindepth : 1; 3823 3824 if (callchaindepth == 1) 3825 ps->ps_pc[0] = PMC_TRAPFRAME_TO_PC(tf); 3826 else { 3827 /* 3828 * Kernel stack traversals can be done immediately, 3829 * while we defer to an AST for user space traversals. 3830 */ 3831 if (!inuserspace) 3832 callchaindepth = 3833 pmc_save_kernel_callchain(ps->ps_pc, 3834 callchaindepth, tf); 3835 else { 3836 pmc_post_callchain_ast(); 3837 callchaindepth = PMC_SAMPLE_INUSE; 3838 } 3839 } 3840 3841 ps->ps_nsamples = callchaindepth; /* mark entry as in use */ 3842 |
|
3687 /* increment write pointer, modulo ring buffer size */ 3688 ps++; 3689 if (ps == psb->ps_fence) 3690 psb->ps_write = psb->ps_samples; 3691 else 3692 psb->ps_write = ps; 3693 3694 done: 3695 /* mark CPU as needing processing */ 3696 atomic_set_rel_int(&pmc_cpumask, (1 << cpu)); 3697 | 3843 /* increment write pointer, modulo ring buffer size */ 3844 ps++; 3845 if (ps == psb->ps_fence) 3846 psb->ps_write = psb->ps_samples; 3847 else 3848 psb->ps_write = ps; 3849 3850 done: 3851 /* mark CPU as needing processing */ 3852 atomic_set_rel_int(&pmc_cpumask, (1 << cpu)); 3853 |
3698 return error; | 3854 return (error); |
3699} 3700 | 3855} 3856 |
3857/* 3858 * Capture a user call chain. This function will be called from ast() 3859 * before control returns to userland and before the process gets 3860 * rescheduled. 3861 */ |
|
3701 | 3862 |
3863static void 3864pmc_capture_user_callchain(int cpu, struct trapframe *tf) 3865{ 3866 int i; 3867 struct pmc *pm; 3868 struct pmc_sample *ps; 3869 struct pmc_samplebuffer *psb; 3870 3871 psb = pmc_pcpu[cpu]->pc_sb; 3872 3873 /* 3874 * Iterate through all deferred callchain requests. 3875 */ 3876 3877 for (i = 0; i < pmc_nsamples; i++) { 3878 3879 ps = &psb->ps_samples[i]; 3880 if (ps->ps_nsamples != PMC_SAMPLE_INUSE) 3881 continue; 3882 3883 pm = ps->ps_pmc; 3884 3885 KASSERT(pm->pm_flags & PMC_F_CALLCHAIN, 3886 ("[pmc,%d] Retrieving callchain for PMC that doesn't " 3887 "want it", __LINE__)); 3888 3889 /* 3890 * Retrieve the callchain and mark the sample buffer 3891 * as 'processable' by the timer tick sweep code. 3892 */ 3893 ps->ps_nsamples = pmc_save_user_callchain(ps->ps_pc, 3894 pmc_callchaindepth, tf); 3895 } 3896 3897 return; 3898} 3899 3900 |
|
3702/* 3703 * Process saved PC samples. 3704 */ 3705 3706static void 3707pmc_process_samples(int cpu) 3708{ 3709 int n, ri; --- 7 unchanged lines hidden (view full) --- 3717 ("[pmc,%d] not on the correct CPU pcpu=%d cpu=%d", __LINE__, 3718 PCPU_GET(cpuid), cpu)); 3719 3720 psb = pmc_pcpu[cpu]->pc_sb; 3721 3722 for (n = 0; n < pmc_nsamples; n++) { /* bound on #iterations */ 3723 3724 ps = psb->ps_read; | 3901/* 3902 * Process saved PC samples. 3903 */ 3904 3905static void 3906pmc_process_samples(int cpu) 3907{ 3908 int n, ri; --- 7 unchanged lines hidden (view full) --- 3916 ("[pmc,%d] not on the correct CPU pcpu=%d cpu=%d", __LINE__, 3917 PCPU_GET(cpuid), cpu)); 3918 3919 psb = pmc_pcpu[cpu]->pc_sb; 3920 3921 for (n = 0; n < pmc_nsamples; n++) { /* bound on #iterations */ 3922 3923 ps = psb->ps_read; |
3725 if (ps->ps_pc == (uintfptr_t) 0) /* no data */ | 3924 if (ps->ps_nsamples == PMC_SAMPLE_FREE) |
3726 break; | 3925 break; |
3926 if (ps->ps_nsamples == PMC_SAMPLE_INUSE) { 3927 /* Need a rescan at a later time. */ 3928 atomic_set_rel_int(&pmc_cpumask, (1 << cpu)); 3929 break; 3930 } |
|
3727 3728 pm = ps->ps_pmc; 3729 po = pm->pm_owner; 3730 3731 KASSERT(PMC_IS_SAMPLING_MODE(PMC_TO_MODE(pm)), 3732 ("[pmc,%d] pmc=%p non-sampling mode=%d", __LINE__, 3733 pm, PMC_TO_MODE(pm))); 3734 3735 /* Ignore PMCs that have been switched off */ 3736 if (pm->pm_state != PMC_STATE_RUNNING) 3737 goto entrydone; 3738 | 3931 3932 pm = ps->ps_pmc; 3933 po = pm->pm_owner; 3934 3935 KASSERT(PMC_IS_SAMPLING_MODE(PMC_TO_MODE(pm)), 3936 ("[pmc,%d] pmc=%p non-sampling mode=%d", __LINE__, 3937 pm, PMC_TO_MODE(pm))); 3938 3939 /* Ignore PMCs that have been switched off */ 3940 if (pm->pm_state != PMC_STATE_RUNNING) 3941 goto entrydone; 3942 |
3739 PMCDBG(SAM,OPS,1,"cpu=%d pm=%p pc=%jx um=%d wr=%d rd=%d", cpu, 3740 pm, (uint64_t) ps->ps_pc, ps->ps_usermode, | 3943 PMCDBG(SAM,OPS,1,"cpu=%d pm=%p n=%d fl=%x wr=%d rd=%d", cpu, 3944 pm, ps->ps_nsamples, ps->ps_flags, |
3741 (int) (psb->ps_write - psb->ps_samples), 3742 (int) (psb->ps_read - psb->ps_samples)); 3743 3744 /* 3745 * If this is a process-mode PMC that is attached to 3746 * its owner, and if the PC is in user mode, update 3747 * profiling statistics like timer-based profiling 3748 * would have done. 3749 */ 3750 if (pm->pm_flags & PMC_F_ATTACHED_TO_OWNER) { | 3945 (int) (psb->ps_write - psb->ps_samples), 3946 (int) (psb->ps_read - psb->ps_samples)); 3947 3948 /* 3949 * If this is a process-mode PMC that is attached to 3950 * its owner, and if the PC is in user mode, update 3951 * profiling statistics like timer-based profiling 3952 * would have done. 3953 */ 3954 if (pm->pm_flags & PMC_F_ATTACHED_TO_OWNER) { |
3751 if (ps->ps_usermode) { | 3955 if (ps->ps_flags & PMC_CC_F_USERSPACE) { |
3752 td = FIRST_THREAD_IN_PROC(po->po_owner); | 3956 td = FIRST_THREAD_IN_PROC(po->po_owner); |
3753 addupc_intr(td, ps->ps_pc, 1); | 3957 addupc_intr(td, ps->ps_pc[0], 1); |
3754 } 3755 goto entrydone; 3756 } 3757 3758 /* 3759 * Otherwise, this is either a sampling mode PMC that 3760 * is attached to a different process than its owner, 3761 * or a system-wide sampling PMC. Dispatch a log 3762 * entry to the PMC's owner process. 3763 */ 3764 | 3958 } 3959 goto entrydone; 3960 } 3961 3962 /* 3963 * Otherwise, this is either a sampling mode PMC that 3964 * is attached to a different process than its owner, 3965 * or a system-wide sampling PMC. Dispatch a log 3966 * entry to the PMC's owner process. 3967 */ 3968 |
3765 pmclog_process_pcsample(pm, ps); | 3969 pmclog_process_callchain(pm, ps); |
3766 3767 entrydone: | 3970 3971 entrydone: |
3768 ps->ps_pc = (uintfptr_t) 0; /* mark entry as free */ | 3972 ps->ps_nsamples = 0; /* mark entry as free */ |
3769 atomic_subtract_rel_32(&pm->pm_runcount, 1); 3770 3771 /* increment read pointer, modulo sample size */ 3772 if (++ps == psb->ps_fence) 3773 psb->ps_read = psb->ps_samples; 3774 else 3775 psb->ps_read = ps; 3776 } --- 305 unchanged lines hidden (view full) --- 4082 __PMC_CLASSES() 4083}; 4084 4085static int 4086pmc_initialize(void) 4087{ 4088 int cpu, error, n; 4089 struct pmc_binding pb; | 3973 atomic_subtract_rel_32(&pm->pm_runcount, 1); 3974 3975 /* increment read pointer, modulo sample size */ 3976 if (++ps == psb->ps_fence) 3977 psb->ps_read = psb->ps_samples; 3978 else 3979 psb->ps_read = ps; 3980 } --- 305 unchanged lines hidden (view full) --- 4286 __PMC_CLASSES() 4287}; 4288 4289static int 4290pmc_initialize(void) 4291{ 4292 int cpu, error, n; 4293 struct pmc_binding pb; |
4294 struct pmc_sample *ps; |
|
4090 struct pmc_samplebuffer *sb; 4091 4092 md = NULL; 4093 error = 0; 4094 4095#ifdef DEBUG 4096 /* parse debug flags first */ 4097 if (TUNABLE_STR_FETCH(PMC_SYSCTL_NAME_PREFIX "debugflags", --- 16 unchanged lines hidden (view full) --- 4114 return EPROGMISMATCH; 4115 } 4116 4117 /* 4118 * check sysctl parameters 4119 */ 4120 4121 if (pmc_hashsize <= 0) { | 4295 struct pmc_samplebuffer *sb; 4296 4297 md = NULL; 4298 error = 0; 4299 4300#ifdef DEBUG 4301 /* parse debug flags first */ 4302 if (TUNABLE_STR_FETCH(PMC_SYSCTL_NAME_PREFIX "debugflags", --- 16 unchanged lines hidden (view full) --- 4319 return EPROGMISMATCH; 4320 } 4321 4322 /* 4323 * check sysctl parameters 4324 */ 4325 4326 if (pmc_hashsize <= 0) { |
4122 (void) printf("hwpmc: tunable hashsize=%d must be greater " 4123 "than zero.\n", pmc_hashsize); | 4327 (void) printf("hwpmc: tunable \"hashsize\"=%d must be " 4328 "greater than zero.\n", pmc_hashsize); |
4124 pmc_hashsize = PMC_HASH_SIZE; 4125 } 4126 4127 if (pmc_nsamples <= 0 || pmc_nsamples > 65535) { | 4329 pmc_hashsize = PMC_HASH_SIZE; 4330 } 4331 4332 if (pmc_nsamples <= 0 || pmc_nsamples > 65535) { |
4128 (void) printf("hwpmc: tunable nsamples=%d out of range.\n", 4129 pmc_nsamples); | 4333 (void) printf("hwpmc: tunable \"nsamples\"=%d out of " 4334 "range.\n", pmc_nsamples); |
4130 pmc_nsamples = PMC_NSAMPLES; 4131 } 4132 | 4335 pmc_nsamples = PMC_NSAMPLES; 4336 } 4337 |
4338 if (pmc_callchaindepth <= 0 || 4339 pmc_callchaindepth > PMC_CALLCHAIN_DEPTH_MAX) { 4340 (void) printf("hwpmc: tunable \"callchaindepth\"=%d out of " 4341 "range.\n", pmc_callchaindepth); 4342 pmc_callchaindepth = PMC_CALLCHAIN_DEPTH; 4343 } 4344 |
|
4133 md = pmc_md_initialize(); 4134 4135 if (md == NULL || md->pmd_init == NULL) 4136 return ENOSYS; 4137 4138 /* allocate space for the per-cpu array */ 4139 MALLOC(pmc_pcpu, struct pmc_cpu **, mp_ncpus * sizeof(struct pmc_cpu *), 4140 M_PMC, M_WAITOK|M_ZERO); --- 25 unchanged lines hidden (view full) --- 4166 pmc_nsamples * sizeof(struct pmc_sample), M_PMC, 4167 M_WAITOK|M_ZERO); 4168 4169 sb->ps_read = sb->ps_write = sb->ps_samples; 4170 sb->ps_fence = sb->ps_samples + pmc_nsamples; 4171 KASSERT(pmc_pcpu[cpu] != NULL, 4172 ("[pmc,%d] cpu=%d Null per-cpu data", __LINE__, cpu)); 4173 | 4345 md = pmc_md_initialize(); 4346 4347 if (md == NULL || md->pmd_init == NULL) 4348 return ENOSYS; 4349 4350 /* allocate space for the per-cpu array */ 4351 MALLOC(pmc_pcpu, struct pmc_cpu **, mp_ncpus * sizeof(struct pmc_cpu *), 4352 M_PMC, M_WAITOK|M_ZERO); --- 25 unchanged lines hidden (view full) --- 4378 pmc_nsamples * sizeof(struct pmc_sample), M_PMC, 4379 M_WAITOK|M_ZERO); 4380 4381 sb->ps_read = sb->ps_write = sb->ps_samples; 4382 sb->ps_fence = sb->ps_samples + pmc_nsamples; 4383 KASSERT(pmc_pcpu[cpu] != NULL, 4384 ("[pmc,%d] cpu=%d Null per-cpu data", __LINE__, cpu)); 4385 |
4386 MALLOC(sb->ps_callchains, uintptr_t *, 4387 pmc_callchaindepth * pmc_nsamples * sizeof(uintptr_t), 4388 M_PMC, M_WAITOK|M_ZERO); 4389 4390 for (n = 0, ps = sb->ps_samples; n < pmc_nsamples; n++, ps++) 4391 ps->ps_pc = sb->ps_callchains + 4392 (n * pmc_callchaindepth); 4393 |
|
4174 pmc_pcpu[cpu]->pc_sb = sb; 4175 } 4176 4177 /* allocate space for the row disposition array */ 4178 pmc_pmcdisp = malloc(sizeof(enum pmc_mode) * md->pmd_npmc, 4179 M_PMC, M_WAITOK|M_ZERO); 4180 4181 KASSERT(pmc_pmcdisp != NULL, --- 140 unchanged lines hidden (view full) --- 4322 4323 /* free the per-cpu sample buffers */ 4324 for (cpu = 0; cpu < mp_ncpus; cpu++) { 4325 if (pmc_cpu_is_disabled(cpu)) 4326 continue; 4327 KASSERT(pmc_pcpu[cpu]->pc_sb != NULL, 4328 ("[pmc,%d] Null cpu sample buffer cpu=%d", __LINE__, 4329 cpu)); | 4394 pmc_pcpu[cpu]->pc_sb = sb; 4395 } 4396 4397 /* allocate space for the row disposition array */ 4398 pmc_pmcdisp = malloc(sizeof(enum pmc_mode) * md->pmd_npmc, 4399 M_PMC, M_WAITOK|M_ZERO); 4400 4401 KASSERT(pmc_pmcdisp != NULL, --- 140 unchanged lines hidden (view full) --- 4542 4543 /* free the per-cpu sample buffers */ 4544 for (cpu = 0; cpu < mp_ncpus; cpu++) { 4545 if (pmc_cpu_is_disabled(cpu)) 4546 continue; 4547 KASSERT(pmc_pcpu[cpu]->pc_sb != NULL, 4548 ("[pmc,%d] Null cpu sample buffer cpu=%d", __LINE__, 4549 cpu)); |
4550 FREE(pmc_pcpu[cpu]->pc_sb->ps_callchains, M_PMC); |
|
4330 FREE(pmc_pcpu[cpu]->pc_sb, M_PMC); 4331 pmc_pcpu[cpu]->pc_sb = NULL; 4332 } 4333 4334 /* do processor dependent cleanup */ 4335 PMCDBG(MOD,INI,3, "%s", "md cleanup"); 4336 if (md) { 4337 pmc_save_cpu_binding(&pb); --- 69 unchanged lines hidden --- | 4551 FREE(pmc_pcpu[cpu]->pc_sb, M_PMC); 4552 pmc_pcpu[cpu]->pc_sb = NULL; 4553 } 4554 4555 /* do processor dependent cleanup */ 4556 PMCDBG(MOD,INI,3, "%s", "md cleanup"); 4557 if (md) { 4558 pmc_save_cpu_binding(&pb); --- 69 unchanged lines hidden --- |