Deleted Added
sdiff udiff text old ( 170307 ) new ( 174395 )
full compact
hwpmc_mod.c (170307) hwpmc_mod.c (174395)
1/*-
1/*-
2 * Copyright (c) 2003-2006 Joseph Koshy
2 * Copyright (c) 2003-2007 Joseph Koshy
3 * Copyright (c) 2007 The FreeBSD Foundation
3 * All rights reserved.
4 *
4 * All rights reserved.
5 *
6 * Portions of this software were developed by A. Joseph Koshy under
7 * sponsorship from the FreeBSD Foundation and Google, Inc.
8 *
5 * Redistribution and use in source and binary forms, with or without
6 * modification, are permitted provided that the following conditions
7 * are met:
8 * 1. Redistributions of source code must retain the above copyright
9 * notice, this list of conditions and the following disclaimer.
10 * 2. Redistributions in binary form must reproduce the above copyright
11 * notice, this list of conditions and the following disclaimer in the
12 * documentation and/or other materials provided with the distribution.

--- 8 unchanged lines hidden (view full) ---

21 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
22 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
23 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
24 * SUCH DAMAGE.
25 *
26 */
27
28#include <sys/cdefs.h>
9 * Redistribution and use in source and binary forms, with or without
10 * modification, are permitted provided that the following conditions
11 * are met:
12 * 1. Redistributions of source code must retain the above copyright
13 * notice, this list of conditions and the following disclaimer.
14 * 2. Redistributions in binary form must reproduce the above copyright
15 * notice, this list of conditions and the following disclaimer in the
16 * documentation and/or other materials provided with the distribution.

--- 8 unchanged lines hidden (view full) ---

25 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
26 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
27 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
28 * SUCH DAMAGE.
29 *
30 */
31
32#include <sys/cdefs.h>
29__FBSDID("$FreeBSD: head/sys/dev/hwpmc/hwpmc_mod.c 170307 2007-06-05 00:00:57Z jeff $");
33__FBSDID("$FreeBSD: head/sys/dev/hwpmc/hwpmc_mod.c 174395 2007-12-07 08:20:17Z jkoshy $");
30
31#include <sys/param.h>
32#include <sys/eventhandler.h>
33#include <sys/jail.h>
34#include <sys/kernel.h>
35#include <sys/kthread.h>
36#include <sys/limits.h>
37#include <sys/lock.h>

--- 123 unchanged lines hidden (view full) ---

161static int load(struct module *module, int cmd, void *arg);
162static int pmc_attach_process(struct proc *p, struct pmc *pm);
163static struct pmc *pmc_allocate_pmc_descriptor(void);
164static struct pmc_owner *pmc_allocate_owner_descriptor(struct proc *p);
165static int pmc_attach_one_process(struct proc *p, struct pmc *pm);
166static int pmc_can_allocate_rowindex(struct proc *p, unsigned int ri,
167 int cpu);
168static int pmc_can_attach(struct pmc *pm, struct proc *p);
34
35#include <sys/param.h>
36#include <sys/eventhandler.h>
37#include <sys/jail.h>
38#include <sys/kernel.h>
39#include <sys/kthread.h>
40#include <sys/limits.h>
41#include <sys/lock.h>

--- 123 unchanged lines hidden (view full) ---

165static int load(struct module *module, int cmd, void *arg);
166static int pmc_attach_process(struct proc *p, struct pmc *pm);
167static struct pmc *pmc_allocate_pmc_descriptor(void);
168static struct pmc_owner *pmc_allocate_owner_descriptor(struct proc *p);
169static int pmc_attach_one_process(struct proc *p, struct pmc *pm);
170static int pmc_can_allocate_rowindex(struct proc *p, unsigned int ri,
171 int cpu);
172static int pmc_can_attach(struct pmc *pm, struct proc *p);
173static void pmc_capture_user_callchain(int cpu, struct trapframe *tf);
169static void pmc_cleanup(void);
170static int pmc_detach_process(struct proc *p, struct pmc *pm);
171static int pmc_detach_one_process(struct proc *p, struct pmc *pm,
172 int flags);
173static void pmc_destroy_owner_descriptor(struct pmc_owner *po);
174static struct pmc_owner *pmc_find_owner_descriptor(struct proc *p);
175static int pmc_find_pmc(pmc_id_t pmcid, struct pmc **pm);
176static struct pmc *pmc_find_pmc_descriptor_in_process(struct pmc_owner *po,
177 pmc_id_t pmc);
178static struct pmc_process *pmc_find_process_descriptor(struct proc *p,
179 uint32_t mode);
180static void pmc_force_context_switch(void);
181static void pmc_link_target_process(struct pmc *pm,
182 struct pmc_process *pp);
174static void pmc_cleanup(void);
175static int pmc_detach_process(struct proc *p, struct pmc *pm);
176static int pmc_detach_one_process(struct proc *p, struct pmc *pm,
177 int flags);
178static void pmc_destroy_owner_descriptor(struct pmc_owner *po);
179static struct pmc_owner *pmc_find_owner_descriptor(struct proc *p);
180static int pmc_find_pmc(pmc_id_t pmcid, struct pmc **pm);
181static struct pmc *pmc_find_pmc_descriptor_in_process(struct pmc_owner *po,
182 pmc_id_t pmc);
183static struct pmc_process *pmc_find_process_descriptor(struct proc *p,
184 uint32_t mode);
185static void pmc_force_context_switch(void);
186static void pmc_link_target_process(struct pmc *pm,
187 struct pmc_process *pp);
188static void pmc_log_all_process_mappings(struct pmc_owner *po);
189static void pmc_log_kernel_mappings(struct pmc *pm);
190static void pmc_log_process_mappings(struct pmc_owner *po, struct proc *p);
183static void pmc_maybe_remove_owner(struct pmc_owner *po);
184static void pmc_process_csw_in(struct thread *td);
185static void pmc_process_csw_out(struct thread *td);
186static void pmc_process_exit(void *arg, struct proc *p);
187static void pmc_process_fork(void *arg, struct proc *p1,
188 struct proc *p2, int n);
189static void pmc_process_samples(int cpu);
190static void pmc_release_pmc_descriptor(struct pmc *pmc);

--- 9 unchanged lines hidden (view full) ---

200 struct pmc_process *pp);
201
202/*
203 * Kernel tunables and sysctl(8) interface.
204 */
205
206SYSCTL_NODE(_kern, OID_AUTO, hwpmc, CTLFLAG_RW, 0, "HWPMC parameters");
207
191static void pmc_maybe_remove_owner(struct pmc_owner *po);
192static void pmc_process_csw_in(struct thread *td);
193static void pmc_process_csw_out(struct thread *td);
194static void pmc_process_exit(void *arg, struct proc *p);
195static void pmc_process_fork(void *arg, struct proc *p1,
196 struct proc *p2, int n);
197static void pmc_process_samples(int cpu);
198static void pmc_release_pmc_descriptor(struct pmc *pmc);

--- 9 unchanged lines hidden (view full) ---

208 struct pmc_process *pp);
209
210/*
211 * Kernel tunables and sysctl(8) interface.
212 */
213
214SYSCTL_NODE(_kern, OID_AUTO, hwpmc, CTLFLAG_RW, 0, "HWPMC parameters");
215
216static int pmc_callchaindepth = PMC_CALLCHAIN_DEPTH;
217TUNABLE_INT(PMC_SYSCTL_NAME_PREFIX "callchaindepth", &pmc_callchaindepth);
218SYSCTL_INT(_kern_hwpmc, OID_AUTO, callchaindepth, CTLFLAG_TUN|CTLFLAG_RD,
219 &pmc_callchaindepth, 0, "depth of call chain records");
220
208#ifdef DEBUG
209struct pmc_debugflags pmc_debugflags = PMC_DEBUG_DEFAULT_FLAGS;
210char pmc_debugstr[PMC_DEBUG_STRSIZE];
211TUNABLE_STR(PMC_SYSCTL_NAME_PREFIX "debugflags", pmc_debugstr,
212 sizeof(pmc_debugstr));
213SYSCTL_PROC(_kern_hwpmc, OID_AUTO, debugflags,
214 CTLTYPE_STRING|CTLFLAG_RW|CTLFLAG_TUN,
215 0, 0, pmc_debugflags_sysctl_handler, "A", "debug flags");

--- 5 unchanged lines hidden (view full) ---

221 */
222
223static int pmc_hashsize = PMC_HASH_SIZE;
224TUNABLE_INT(PMC_SYSCTL_NAME_PREFIX "hashsize", &pmc_hashsize);
225SYSCTL_INT(_kern_hwpmc, OID_AUTO, hashsize, CTLFLAG_TUN|CTLFLAG_RD,
226 &pmc_hashsize, 0, "rows in hash tables");
227
228/*
221#ifdef DEBUG
222struct pmc_debugflags pmc_debugflags = PMC_DEBUG_DEFAULT_FLAGS;
223char pmc_debugstr[PMC_DEBUG_STRSIZE];
224TUNABLE_STR(PMC_SYSCTL_NAME_PREFIX "debugflags", pmc_debugstr,
225 sizeof(pmc_debugstr));
226SYSCTL_PROC(_kern_hwpmc, OID_AUTO, debugflags,
227 CTLTYPE_STRING|CTLFLAG_RW|CTLFLAG_TUN,
228 0, 0, pmc_debugflags_sysctl_handler, "A", "debug flags");

--- 5 unchanged lines hidden (view full) ---

234 */
235
236static int pmc_hashsize = PMC_HASH_SIZE;
237TUNABLE_INT(PMC_SYSCTL_NAME_PREFIX "hashsize", &pmc_hashsize);
238SYSCTL_INT(_kern_hwpmc, OID_AUTO, hashsize, CTLFLAG_TUN|CTLFLAG_RD,
239 &pmc_hashsize, 0, "rows in hash tables");
240
241/*
229 * kern.hwpmc.nsamples --- number of PC samples per CPU
242 * kern.hwpmc.nsamples --- number of PC samples/callchain stacks per CPU
230 */
231
232static int pmc_nsamples = PMC_NSAMPLES;
233TUNABLE_INT(PMC_SYSCTL_NAME_PREFIX "nsamples", &pmc_nsamples);
234SYSCTL_INT(_kern_hwpmc, OID_AUTO, nsamples, CTLFLAG_TUN|CTLFLAG_RD,
235 &pmc_nsamples, 0, "number of PC samples per CPU");
236
243 */
244
245static int pmc_nsamples = PMC_NSAMPLES;
246TUNABLE_INT(PMC_SYSCTL_NAME_PREFIX "nsamples", &pmc_nsamples);
247SYSCTL_INT(_kern_hwpmc, OID_AUTO, nsamples, CTLFLAG_TUN|CTLFLAG_RD,
248 &pmc_nsamples, 0, "number of PC samples per CPU");
249
250
237/*
238 * kern.hwpmc.mtxpoolsize -- number of mutexes in the mutex pool.
239 */
240
241static int pmc_mtxpool_size = PMC_MTXPOOL_SIZE;
242TUNABLE_INT(PMC_SYSCTL_NAME_PREFIX "mtxpoolsize", &pmc_mtxpool_size);
243SYSCTL_INT(_kern_hwpmc, OID_AUTO, mtxpoolsize, CTLFLAG_TUN|CTLFLAG_RD,
244 &pmc_mtxpool_size, 0, "size of spin mutex pool");

--- 707 unchanged lines hidden (view full) ---

952 pm->pm_flags |= PMC_F_ATTACH_DONE; /* mark as attached */
953
954 /* issue an attach event to a configured log file */
955 if (pm->pm_owner->po_flags & PMC_PO_OWNS_LOGFILE) {
956 pmc_getfilename(p->p_textvp, &fullpath, &freepath);
957 pmclog_process_pmcattach(pm, p->p_pid, fullpath);
958 if (freepath)
959 FREE(freepath, M_TEMP);
251/*
252 * kern.hwpmc.mtxpoolsize -- number of mutexes in the mutex pool.
253 */
254
255static int pmc_mtxpool_size = PMC_MTXPOOL_SIZE;
256TUNABLE_INT(PMC_SYSCTL_NAME_PREFIX "mtxpoolsize", &pmc_mtxpool_size);
257SYSCTL_INT(_kern_hwpmc, OID_AUTO, mtxpoolsize, CTLFLAG_TUN|CTLFLAG_RD,
258 &pmc_mtxpool_size, 0, "size of spin mutex pool");

--- 707 unchanged lines hidden (view full) ---

966 pm->pm_flags |= PMC_F_ATTACH_DONE; /* mark as attached */
967
968 /* issue an attach event to a configured log file */
969 if (pm->pm_owner->po_flags & PMC_PO_OWNS_LOGFILE) {
970 pmc_getfilename(p->p_textvp, &fullpath, &freepath);
971 pmclog_process_pmcattach(pm, p->p_pid, fullpath);
972 if (freepath)
973 FREE(freepath, M_TEMP);
974 if (PMC_IS_SAMPLING_MODE(PMC_TO_MODE(pm)))
975 pmc_log_process_mappings(pm->pm_owner, p);
960 }
961 /* mark process as using HWPMCs */
962 PROC_LOCK(p);
963 p->p_flag |= P_HWPMC;
964 PROC_UNLOCK(p);
965
966 return 0;
967}

--- 476 unchanged lines hidden (view full) ---

1444 struct pmc_owner *po;
1445
1446 sx_assert(&pmc_sx, SX_LOCKED);
1447
1448 LIST_FOREACH(po, &pmc_ss_owners, po_ssnext)
1449 if (po->po_flags & PMC_PO_OWNS_LOGFILE)
1450 pmclog_process_map_out(po, (pid_t) -1,
1451 pkm->pm_address, pkm->pm_address + pkm->pm_size);
976 }
977 /* mark process as using HWPMCs */
978 PROC_LOCK(p);
979 p->p_flag |= P_HWPMC;
980 PROC_UNLOCK(p);
981
982 return 0;
983}

--- 476 unchanged lines hidden (view full) ---

1460 struct pmc_owner *po;
1461
1462 sx_assert(&pmc_sx, SX_LOCKED);
1463
1464 LIST_FOREACH(po, &pmc_ss_owners, po_ssnext)
1465 if (po->po_flags & PMC_PO_OWNS_LOGFILE)
1466 pmclog_process_map_out(po, (pid_t) -1,
1467 pkm->pm_address, pkm->pm_address + pkm->pm_size);
1452
1468
1453 /*
1454 * TODO: Notify owners of process-sampling PMCs.
1455 */
1456}
1457
1458/*
1459 * A mapping change for a process.
1460 */

--- 62 unchanged lines hidden (view full) ---

1523 for (ri = 0; ri < md->pmd_npmc; ri++)
1524 if ((pm = pp->pp_pmcs[ri].pp_pmc) != NULL &&
1525 PMC_IS_SAMPLING_MODE(PMC_TO_MODE(pm)))
1526 pmclog_process_map_out(pm->pm_owner, pid,
1527 pkm->pm_address, pkm->pm_address + pkm->pm_size);
1528}
1529
1530/*
1469 /*
1470 * TODO: Notify owners of process-sampling PMCs.
1471 */
1472}
1473
1474/*
1475 * A mapping change for a process.
1476 */

--- 62 unchanged lines hidden (view full) ---

1539 for (ri = 0; ri < md->pmd_npmc; ri++)
1540 if ((pm = pp->pp_pmcs[ri].pp_pmc) != NULL &&
1541 PMC_IS_SAMPLING_MODE(PMC_TO_MODE(pm)))
1542 pmclog_process_map_out(pm->pm_owner, pid,
1543 pkm->pm_address, pkm->pm_address + pkm->pm_size);
1544}
1545
1546/*
1547 * Log mapping information about the kernel.
1548 */
1549
1550static void
1551pmc_log_kernel_mappings(struct pmc *pm)
1552{
1553 struct pmc_owner *po;
1554 struct pmckern_map_in *km, *kmbase;
1555
1556 sx_assert(&pmc_sx, SX_LOCKED);
1557 KASSERT(PMC_IS_SAMPLING_MODE(PMC_TO_MODE(pm)),
1558 ("[pmc,%d] non-sampling PMC (%p) desires mapping information",
1559 __LINE__, (void *) pm));
1560
1561 po = pm->pm_owner;
1562
1563 if (po->po_flags & PMC_PO_INITIAL_MAPPINGS_DONE)
1564 return;
1565
1566 /*
1567 * Log the current set of kernel modules.
1568 */
1569 kmbase = linker_hwpmc_list_objects();
1570 for (km = kmbase; km->pm_file != NULL; km++) {
1571 PMCDBG(LOG,REG,1,"%s %p", (char *) km->pm_file,
1572 (void *) km->pm_address);
1573 pmclog_process_map_in(po, (pid_t) -1, km->pm_address,
1574 km->pm_file);
1575 }
1576 FREE(kmbase, M_LINKER);
1577
1578 po->po_flags |= PMC_PO_INITIAL_MAPPINGS_DONE;
1579}
1580
1581/*
1582 * Log the mappings for a single process.
1583 */
1584
1585static void
1586pmc_log_process_mappings(struct pmc_owner *po, struct proc *p)
1587{
1588}
1589
1590/*
1591 * Log mappings for all processes in the system.
1592 */
1593
1594static void
1595pmc_log_all_process_mappings(struct pmc_owner *po)
1596{
1597 struct proc *p, *top;
1598
1599 sx_assert(&pmc_sx, SX_XLOCKED);
1600
1601 if ((p = pfind(1)) == NULL)
1602 panic("[pmc,%d] Cannot find init", __LINE__);
1603
1604 PROC_UNLOCK(p);
1605
1606 sx_slock(&proctree_lock);
1607
1608 top = p;
1609
1610 for (;;) {
1611 pmc_log_process_mappings(po, p);
1612 if (!LIST_EMPTY(&p->p_children))
1613 p = LIST_FIRST(&p->p_children);
1614 else for (;;) {
1615 if (p == top)
1616 goto done;
1617 if (LIST_NEXT(p, p_sibling)) {
1618 p = LIST_NEXT(p, p_sibling);
1619 break;
1620 }
1621 p = p->p_pptr;
1622 }
1623 }
1624 done:
1625 sx_sunlock(&proctree_lock);
1626}
1627
1628/*
1531 * The 'hook' invoked from the kernel proper
1532 */
1533
1534
1535#ifdef DEBUG
1536const char *pmc_hooknames[] = {
1537 /* these strings correspond to PMC_FN_* in <sys/pmckern.h> */
1538 "",
1539 "EXEC",
1540 "CSW-IN",
1541 "CSW-OUT",
1542 "SAMPLE",
1543 "KLDLOAD",
1544 "KLDUNLOAD",
1545 "MMAP",
1629 * The 'hook' invoked from the kernel proper
1630 */
1631
1632
1633#ifdef DEBUG
1634const char *pmc_hooknames[] = {
1635 /* these strings correspond to PMC_FN_* in <sys/pmckern.h> */
1636 "",
1637 "EXEC",
1638 "CSW-IN",
1639 "CSW-OUT",
1640 "SAMPLE",
1641 "KLDLOAD",
1642 "KLDUNLOAD",
1643 "MMAP",
1546 "MUNMAP"
1644 "MUNMAP",
1645 "CALLCHAIN"
1547};
1548#endif
1549
1550static int
1551pmc_hook_handler(struct thread *td, int function, void *arg)
1552{
1553
1554 PMCDBG(MOD,PMH,1, "hook td=%p func=%d \"%s\" arg=%p", td, function,

--- 166 unchanged lines hidden (view full) ---

1721 pmc_process_mmap(td, (struct pmckern_map_in *) arg);
1722 break;
1723
1724 case PMC_FN_MUNMAP:
1725 sx_assert(&pmc_sx, SX_LOCKED);
1726 pmc_process_munmap(td, (struct pmckern_map_out *) arg);
1727 break;
1728
1646};
1647#endif
1648
1649static int
1650pmc_hook_handler(struct thread *td, int function, void *arg)
1651{
1652
1653 PMCDBG(MOD,PMH,1, "hook td=%p func=%d \"%s\" arg=%p", td, function,

--- 166 unchanged lines hidden (view full) ---

1820 pmc_process_mmap(td, (struct pmckern_map_in *) arg);
1821 break;
1822
1823 case PMC_FN_MUNMAP:
1824 sx_assert(&pmc_sx, SX_LOCKED);
1825 pmc_process_munmap(td, (struct pmckern_map_out *) arg);
1826 break;
1827
1828 case PMC_FN_USER_CALLCHAIN:
1829 /*
1830 * Record a call chain.
1831 */
1832 pmc_capture_user_callchain(PCPU_GET(cpuid),
1833 (struct trapframe *) arg);
1834 break;
1835
1729 default:
1730#ifdef DEBUG
1731 KASSERT(0, ("[pmc,%d] unknown hook %d\n", __LINE__, function));
1732#endif
1733 break;
1734
1735 }
1736

--- 579 unchanged lines hidden (view full) ---

2316 mode = PMC_TO_MODE(pm);
2317 ri = PMC_TO_ROWINDEX(pm);
2318 error = 0;
2319
2320 PMCDBG(PMC,OPS,1, "start pmc=%p mode=%d ri=%d", pm, mode, ri);
2321
2322 po = pm->pm_owner;
2323
1836 default:
1837#ifdef DEBUG
1838 KASSERT(0, ("[pmc,%d] unknown hook %d\n", __LINE__, function));
1839#endif
1840 break;
1841
1842 }
1843

--- 579 unchanged lines hidden (view full) ---

2423 mode = PMC_TO_MODE(pm);
2424 ri = PMC_TO_ROWINDEX(pm);
2425 error = 0;
2426
2427 PMCDBG(PMC,OPS,1, "start pmc=%p mode=%d ri=%d", pm, mode, ri);
2428
2429 po = pm->pm_owner;
2430
2431 /*
2432 * Disallow PMCSTART if a logfile is required but has not been
2433 * configured yet.
2434 */
2435 if ((pm->pm_flags & PMC_F_NEEDS_LOGFILE) &&
2436 (po->po_flags & PMC_PO_OWNS_LOGFILE) == 0)
2437 return EDOOFUS; /* programming error */
2438
2439 /*
2440 * If this is a sampling mode PMC, log mapping information for
2441 * the kernel modules that are currently loaded.
2442 */
2443 if (PMC_IS_SAMPLING_MODE(PMC_TO_MODE(pm)))
2444 pmc_log_kernel_mappings(pm);
2445
2324 if (PMC_IS_VIRTUAL_MODE(mode)) {
2325
2326 /*
2327 * If a PMCATTACH has never been done on this PMC,
2328 * attach it to its owner process.
2329 */
2330
2331 if (LIST_EMPTY(&pm->pm_targets))
2332 error = (pm->pm_flags & PMC_F_ATTACH_DONE) ? ESRCH :
2333 pmc_attach_process(po->po_owner, pm);
2334
2335 /*
2446 if (PMC_IS_VIRTUAL_MODE(mode)) {
2447
2448 /*
2449 * If a PMCATTACH has never been done on this PMC,
2450 * attach it to its owner process.
2451 */
2452
2453 if (LIST_EMPTY(&pm->pm_targets))
2454 error = (pm->pm_flags & PMC_F_ATTACH_DONE) ? ESRCH :
2455 pmc_attach_process(po->po_owner, pm);
2456
2457 /*
2336 * Disallow PMCSTART if a logfile is required but has not
2337 * been configured yet.
2338 */
2339
2340 if (error == 0 && (pm->pm_flags & PMC_F_NEEDS_LOGFILE) &&
2341 (po->po_flags & PMC_PO_OWNS_LOGFILE) == 0)
2342 error = EDOOFUS;
2343
2344 /*
2345 * If the PMC is attached to its owner, then force a context
2346 * switch to ensure that the MD state gets set correctly.
2347 */
2348
2349 if (error == 0) {
2350 pm->pm_state = PMC_STATE_RUNNING;
2351 if (pm->pm_flags & PMC_F_ATTACHED_TO_OWNER)
2352 pmc_force_context_switch();
2353 }
2354
2355 return error;
2356 }
2357
2358
2359 /*
2360 * A system-wide PMC.
2458 * If the PMC is attached to its owner, then force a context
2459 * switch to ensure that the MD state gets set correctly.
2460 */
2461
2462 if (error == 0) {
2463 pm->pm_state = PMC_STATE_RUNNING;
2464 if (pm->pm_flags & PMC_F_ATTACHED_TO_OWNER)
2465 pmc_force_context_switch();
2466 }
2467
2468 return error;
2469 }
2470
2471
2472 /*
2473 * A system-wide PMC.
2361 */
2362
2363 if ((pm->pm_flags & PMC_F_NEEDS_LOGFILE) &&
2364 (po->po_flags & PMC_PO_OWNS_LOGFILE) == 0)
2365 return EDOOFUS; /* programming error */
2366
2367 /*
2474 *
2368 * Add the owner to the global list if this is a system-wide
2369 * sampling PMC.
2370 */
2371
2372 if (mode == PMC_MODE_SS) {
2373 if (po->po_sscount == 0) {
2374 LIST_INSERT_HEAD(&pmc_ss_owners, po, po_ssnext);
2375 atomic_add_rel_int(&pmc_ss_count, 1);
2376 PMCDBG(PMC,OPS,1, "po=%p in global list", po);
2377 }
2378 po->po_sscount++;
2379 }
2380
2475 * Add the owner to the global list if this is a system-wide
2476 * sampling PMC.
2477 */
2478
2479 if (mode == PMC_MODE_SS) {
2480 if (po->po_sscount == 0) {
2481 LIST_INSERT_HEAD(&pmc_ss_owners, po, po_ssnext);
2482 atomic_add_rel_int(&pmc_ss_count, 1);
2483 PMCDBG(PMC,OPS,1, "po=%p in global list", po);
2484 }
2485 po->po_sscount++;
2486 }
2487
2381 /* TODO: dump system wide process mappings to the log? */
2488 /* Log mapping information for all processes in the system. */
2489 pmc_log_all_process_mappings(po);
2382
2383 /*
2384 * Move to the CPU associated with this
2385 * PMC, and start the hardware.
2386 */
2387
2388 pmc_save_cpu_binding(&pb);
2389

--- 159 unchanged lines hidden (view full) ---

2549 * XXX This OP will be reworked.
2550 */
2551
2552 case PMC_OP_CONFIGURELOG:
2553 {
2554 struct proc *p;
2555 struct pmc *pm;
2556 struct pmc_owner *po;
2490
2491 /*
2492 * Move to the CPU associated with this
2493 * PMC, and start the hardware.
2494 */
2495
2496 pmc_save_cpu_binding(&pb);
2497

--- 159 unchanged lines hidden (view full) ---

2657 * XXX This OP will be reworked.
2658 */
2659
2660 case PMC_OP_CONFIGURELOG:
2661 {
2662 struct proc *p;
2663 struct pmc *pm;
2664 struct pmc_owner *po;
2557 struct pmckern_map_in *km, *kmbase;
2558 struct pmc_op_configurelog cl;
2559
2560 sx_assert(&pmc_sx, SX_XLOCKED);
2561
2562 if ((error = copyin(arg, &cl, sizeof(cl))) != 0)
2563 break;
2564
2565 /* mark this process as owning a log file */

--- 22 unchanged lines hidden (view full) ---

2588 pmc_stop(pm);
2589 error = pmclog_deconfigure_log(po);
2590 }
2591 } else
2592 error = EINVAL;
2593
2594 if (error)
2595 break;
2665 struct pmc_op_configurelog cl;
2666
2667 sx_assert(&pmc_sx, SX_XLOCKED);
2668
2669 if ((error = copyin(arg, &cl, sizeof(cl))) != 0)
2670 break;
2671
2672 /* mark this process as owning a log file */

--- 22 unchanged lines hidden (view full) ---

2695 pmc_stop(pm);
2696 error = pmclog_deconfigure_log(po);
2697 }
2698 } else
2699 error = EINVAL;
2700
2701 if (error)
2702 break;
2596
2597 /*
2598 * Log the current set of kernel modules.
2599 */
2600 kmbase = linker_hwpmc_list_objects();
2601 for (km = kmbase; km->pm_file != NULL; km++) {
2602 PMCDBG(LOG,REG,1,"%s %p", (char *) km->pm_file,
2603 (void *) km->pm_address);
2604 pmclog_process_map_in(po, (pid_t) -1, km->pm_address,
2605 km->pm_file);
2606 }
2607 FREE(kmbase, M_LINKER);
2608 }
2609 break;
2610
2611
2612 /*
2613 * Flush a log file.
2614 */
2615

--- 324 unchanged lines hidden (view full) ---

2940 if (error)
2941 break;
2942
2943 /*
2944 * Look for valid values for 'pm_flags'
2945 */
2946
2947 if ((pa.pm_flags & ~(PMC_F_DESCENDANTS | PMC_F_LOG_PROCCSW |
2703 }
2704 break;
2705
2706
2707 /*
2708 * Flush a log file.
2709 */
2710

--- 324 unchanged lines hidden (view full) ---

3035 if (error)
3036 break;
3037
3038 /*
3039 * Look for valid values for 'pm_flags'
3040 */
3041
3042 if ((pa.pm_flags & ~(PMC_F_DESCENDANTS | PMC_F_LOG_PROCCSW |
2948 PMC_F_LOG_PROCEXIT)) != 0) {
3043 PMC_F_LOG_PROCEXIT | PMC_F_CALLCHAIN)) != 0) {
2949 error = EINVAL;
2950 break;
2951 }
2952
2953 /* process logging options are not allowed for system PMCs */
2954 if (PMC_IS_SYSTEM_MODE(mode) && (pa.pm_flags &
2955 (PMC_F_LOG_PROCCSW | PMC_F_LOG_PROCEXIT))) {
2956 error = EINVAL;

--- 671 unchanged lines hidden (view full) ---

3628}
3629
3630/*
3631 * Helper functions
3632 */
3633
3634
3635/*
3044 error = EINVAL;
3045 break;
3046 }
3047
3048 /* process logging options are not allowed for system PMCs */
3049 if (PMC_IS_SYSTEM_MODE(mode) && (pa.pm_flags &
3050 (PMC_F_LOG_PROCCSW | PMC_F_LOG_PROCEXIT))) {
3051 error = EINVAL;

--- 671 unchanged lines hidden (view full) ---

3723}
3724
3725/*
3726 * Helper functions
3727 */
3728
3729
3730/*
3731 * Mark the thread as needing callchain capture and post an AST. The
3732 * actual callchain capture will be done in a context where it is safe
3733 * to take page faults.
3734 */
3735
3736static void
3737pmc_post_callchain_ast(void)
3738{
3739 struct thread *td;
3740
3741 td = curthread;
3742
3743 /*
3744 * Mark this thread as needing processing in ast().
3745 * td->td_pflags will be safe to touch as the process was in
3746 * user space when it was interrupted.
3747 */
3748 td->td_pflags |= TDP_CALLCHAIN;
3749
3750 /*
3751 * Again, since we've entered this function directly from
3752 * userland, `td' is guaranteed to be not locked by this CPU,
3753 * so its safe to try acquire the thread lock even though we
3754 * are executing in an NMI context. We need to acquire this
3755 * lock before touching `td_flags' because other CPUs may be
3756 * in the process of touching this field.
3757 */
3758 thread_lock(td);
3759 td->td_flags |= TDF_ASTPENDING;
3760 thread_unlock(td);
3761
3762 return;
3763}
3764
3765/*
3636 * Interrupt processing.
3637 *
3766 * Interrupt processing.
3767 *
3638 * Find a free slot in the per-cpu array of PC samples and write the
3639 * current (PMC,PID,PC) triple to it. If an event was successfully
3640 * added, a bit is set in mask 'pmc_cpumask' denoting that the
3641 * DO_SAMPLES hook needs to be invoked from the clock handler.
3768 * Find a free slot in the per-cpu array of samples and capture the
3769 * current callchain there. If a sample was successfully added, a bit
3770 * is set in mask 'pmc_cpumask' denoting that the DO_SAMPLES hook
3771 * needs to be invoked from the clock handler.
3642 *
3643 * This function is meant to be called from an NMI handler. It cannot
3644 * use any of the locking primitives supplied by the OS.
3645 */
3646
3647int
3772 *
3773 * This function is meant to be called from an NMI handler. It cannot
3774 * use any of the locking primitives supplied by the OS.
3775 */
3776
3777int
3648pmc_process_interrupt(int cpu, struct pmc *pm, uintfptr_t pc, int usermode)
3778pmc_process_interrupt(int cpu, struct pmc *pm, struct trapframe *tf,
3779 int inuserspace)
3649{
3780{
3650 int error, ri;
3781 int error, callchaindepth;
3651 struct thread *td;
3652 struct pmc_sample *ps;
3653 struct pmc_samplebuffer *psb;
3654
3655 error = 0;
3782 struct thread *td;
3783 struct pmc_sample *ps;
3784 struct pmc_samplebuffer *psb;
3785
3786 error = 0;
3656 ri = PMC_TO_ROWINDEX(pm);
3657
3787
3788 /*
3789 * Allocate space for a sample buffer.
3790 */
3658 psb = pmc_pcpu[cpu]->pc_sb;
3659
3660 ps = psb->ps_write;
3791 psb = pmc_pcpu[cpu]->pc_sb;
3792
3793 ps = psb->ps_write;
3661 if (ps->ps_pc) { /* in use, reader hasn't caught up */
3794 if (ps->ps_nsamples) { /* in use, reader hasn't caught up */
3662 pm->pm_stalled = 1;
3663 atomic_add_int(&pmc_stats.pm_intr_bufferfull, 1);
3795 pm->pm_stalled = 1;
3796 atomic_add_int(&pmc_stats.pm_intr_bufferfull, 1);
3664 PMCDBG(SAM,INT,1,"(spc) cpu=%d pm=%p pc=%jx um=%d wr=%d rd=%d",
3665 cpu, pm, (uint64_t) pc, usermode,
3797 PMCDBG(SAM,INT,1,"(spc) cpu=%d pm=%p tf=%p um=%d wr=%d rd=%d",
3798 cpu, pm, (void *) tf, inuserspace,
3666 (int) (psb->ps_write - psb->ps_samples),
3667 (int) (psb->ps_read - psb->ps_samples));
3668 error = ENOMEM;
3669 goto done;
3670 }
3671
3799 (int) (psb->ps_write - psb->ps_samples),
3800 (int) (psb->ps_read - psb->ps_samples));
3801 error = ENOMEM;
3802 goto done;
3803 }
3804
3672 /* fill in entry */
3673 PMCDBG(SAM,INT,1,"cpu=%d pm=%p pc=%jx um=%d wr=%d rd=%d", cpu, pm,
3674 (uint64_t) pc, usermode,
3805
3806 /* Fill in entry. */
3807 PMCDBG(SAM,INT,1,"cpu=%d pm=%p tf=%p um=%d wr=%d rd=%d", cpu, pm,
3808 (void *) tf, inuserspace,
3675 (int) (psb->ps_write - psb->ps_samples),
3676 (int) (psb->ps_read - psb->ps_samples));
3677
3809 (int) (psb->ps_write - psb->ps_samples),
3810 (int) (psb->ps_read - psb->ps_samples));
3811
3678 atomic_add_rel_32(&pm->pm_runcount, 1); /* hold onto PMC */
3812 atomic_add_rel_32(&pm->pm_runcount, 1); /* hold onto PMC */
3679 ps->ps_pmc = pm;
3680 if ((td = curthread) && td->td_proc)
3681 ps->ps_pid = td->td_proc->p_pid;
3682 else
3683 ps->ps_pid = -1;
3813 ps->ps_pmc = pm;
3814 if ((td = curthread) && td->td_proc)
3815 ps->ps_pid = td->td_proc->p_pid;
3816 else
3817 ps->ps_pid = -1;
3684 ps->ps_usermode = usermode;
3685 ps->ps_pc = pc; /* mark entry as in use */
3818 ps->ps_cpu = cpu;
3819 ps->ps_flags = inuserspace ? PMC_CC_F_USERSPACE : 0;
3686
3820
3821 callchaindepth = (pm->pm_flags & PMC_F_CALLCHAIN) ?
3822 pmc_callchaindepth : 1;
3823
3824 if (callchaindepth == 1)
3825 ps->ps_pc[0] = PMC_TRAPFRAME_TO_PC(tf);
3826 else {
3827 /*
3828 * Kernel stack traversals can be done immediately,
3829 * while we defer to an AST for user space traversals.
3830 */
3831 if (!inuserspace)
3832 callchaindepth =
3833 pmc_save_kernel_callchain(ps->ps_pc,
3834 callchaindepth, tf);
3835 else {
3836 pmc_post_callchain_ast();
3837 callchaindepth = PMC_SAMPLE_INUSE;
3838 }
3839 }
3840
3841 ps->ps_nsamples = callchaindepth; /* mark entry as in use */
3842
3687 /* increment write pointer, modulo ring buffer size */
3688 ps++;
3689 if (ps == psb->ps_fence)
3690 psb->ps_write = psb->ps_samples;
3691 else
3692 psb->ps_write = ps;
3693
3694 done:
3695 /* mark CPU as needing processing */
3696 atomic_set_rel_int(&pmc_cpumask, (1 << cpu));
3697
3843 /* increment write pointer, modulo ring buffer size */
3844 ps++;
3845 if (ps == psb->ps_fence)
3846 psb->ps_write = psb->ps_samples;
3847 else
3848 psb->ps_write = ps;
3849
3850 done:
3851 /* mark CPU as needing processing */
3852 atomic_set_rel_int(&pmc_cpumask, (1 << cpu));
3853
3698 return error;
3854 return (error);
3699}
3700
3855}
3856
3857/*
3858 * Capture a user call chain. This function will be called from ast()
3859 * before control returns to userland and before the process gets
3860 * rescheduled.
3861 */
3701
3862
3863static void
3864pmc_capture_user_callchain(int cpu, struct trapframe *tf)
3865{
3866 int i;
3867 struct pmc *pm;
3868 struct pmc_sample *ps;
3869 struct pmc_samplebuffer *psb;
3870
3871 psb = pmc_pcpu[cpu]->pc_sb;
3872
3873 /*
3874 * Iterate through all deferred callchain requests.
3875 */
3876
3877 for (i = 0; i < pmc_nsamples; i++) {
3878
3879 ps = &psb->ps_samples[i];
3880 if (ps->ps_nsamples != PMC_SAMPLE_INUSE)
3881 continue;
3882
3883 pm = ps->ps_pmc;
3884
3885 KASSERT(pm->pm_flags & PMC_F_CALLCHAIN,
3886 ("[pmc,%d] Retrieving callchain for PMC that doesn't "
3887 "want it", __LINE__));
3888
3889 /*
3890 * Retrieve the callchain and mark the sample buffer
3891 * as 'processable' by the timer tick sweep code.
3892 */
3893 ps->ps_nsamples = pmc_save_user_callchain(ps->ps_pc,
3894 pmc_callchaindepth, tf);
3895 }
3896
3897 return;
3898}
3899
3900
3702/*
3703 * Process saved PC samples.
3704 */
3705
3706static void
3707pmc_process_samples(int cpu)
3708{
3709 int n, ri;

--- 7 unchanged lines hidden (view full) ---

3717 ("[pmc,%d] not on the correct CPU pcpu=%d cpu=%d", __LINE__,
3718 PCPU_GET(cpuid), cpu));
3719
3720 psb = pmc_pcpu[cpu]->pc_sb;
3721
3722 for (n = 0; n < pmc_nsamples; n++) { /* bound on #iterations */
3723
3724 ps = psb->ps_read;
3901/*
3902 * Process saved PC samples.
3903 */
3904
3905static void
3906pmc_process_samples(int cpu)
3907{
3908 int n, ri;

--- 7 unchanged lines hidden (view full) ---

3916 ("[pmc,%d] not on the correct CPU pcpu=%d cpu=%d", __LINE__,
3917 PCPU_GET(cpuid), cpu));
3918
3919 psb = pmc_pcpu[cpu]->pc_sb;
3920
3921 for (n = 0; n < pmc_nsamples; n++) { /* bound on #iterations */
3922
3923 ps = psb->ps_read;
3725 if (ps->ps_pc == (uintfptr_t) 0) /* no data */
3924 if (ps->ps_nsamples == PMC_SAMPLE_FREE)
3726 break;
3925 break;
3926 if (ps->ps_nsamples == PMC_SAMPLE_INUSE) {
3927 /* Need a rescan at a later time. */
3928 atomic_set_rel_int(&pmc_cpumask, (1 << cpu));
3929 break;
3930 }
3727
3728 pm = ps->ps_pmc;
3729 po = pm->pm_owner;
3730
3731 KASSERT(PMC_IS_SAMPLING_MODE(PMC_TO_MODE(pm)),
3732 ("[pmc,%d] pmc=%p non-sampling mode=%d", __LINE__,
3733 pm, PMC_TO_MODE(pm)));
3734
3735 /* Ignore PMCs that have been switched off */
3736 if (pm->pm_state != PMC_STATE_RUNNING)
3737 goto entrydone;
3738
3931
3932 pm = ps->ps_pmc;
3933 po = pm->pm_owner;
3934
3935 KASSERT(PMC_IS_SAMPLING_MODE(PMC_TO_MODE(pm)),
3936 ("[pmc,%d] pmc=%p non-sampling mode=%d", __LINE__,
3937 pm, PMC_TO_MODE(pm)));
3938
3939 /* Ignore PMCs that have been switched off */
3940 if (pm->pm_state != PMC_STATE_RUNNING)
3941 goto entrydone;
3942
3739 PMCDBG(SAM,OPS,1,"cpu=%d pm=%p pc=%jx um=%d wr=%d rd=%d", cpu,
3740 pm, (uint64_t) ps->ps_pc, ps->ps_usermode,
3943 PMCDBG(SAM,OPS,1,"cpu=%d pm=%p n=%d fl=%x wr=%d rd=%d", cpu,
3944 pm, ps->ps_nsamples, ps->ps_flags,
3741 (int) (psb->ps_write - psb->ps_samples),
3742 (int) (psb->ps_read - psb->ps_samples));
3743
3744 /*
3745 * If this is a process-mode PMC that is attached to
3746 * its owner, and if the PC is in user mode, update
3747 * profiling statistics like timer-based profiling
3748 * would have done.
3749 */
3750 if (pm->pm_flags & PMC_F_ATTACHED_TO_OWNER) {
3945 (int) (psb->ps_write - psb->ps_samples),
3946 (int) (psb->ps_read - psb->ps_samples));
3947
3948 /*
3949 * If this is a process-mode PMC that is attached to
3950 * its owner, and if the PC is in user mode, update
3951 * profiling statistics like timer-based profiling
3952 * would have done.
3953 */
3954 if (pm->pm_flags & PMC_F_ATTACHED_TO_OWNER) {
3751 if (ps->ps_usermode) {
3955 if (ps->ps_flags & PMC_CC_F_USERSPACE) {
3752 td = FIRST_THREAD_IN_PROC(po->po_owner);
3956 td = FIRST_THREAD_IN_PROC(po->po_owner);
3753 addupc_intr(td, ps->ps_pc, 1);
3957 addupc_intr(td, ps->ps_pc[0], 1);
3754 }
3755 goto entrydone;
3756 }
3757
3758 /*
3759 * Otherwise, this is either a sampling mode PMC that
3760 * is attached to a different process than its owner,
3761 * or a system-wide sampling PMC. Dispatch a log
3762 * entry to the PMC's owner process.
3763 */
3764
3958 }
3959 goto entrydone;
3960 }
3961
3962 /*
3963 * Otherwise, this is either a sampling mode PMC that
3964 * is attached to a different process than its owner,
3965 * or a system-wide sampling PMC. Dispatch a log
3966 * entry to the PMC's owner process.
3967 */
3968
3765 pmclog_process_pcsample(pm, ps);
3969 pmclog_process_callchain(pm, ps);
3766
3767 entrydone:
3970
3971 entrydone:
3768 ps->ps_pc = (uintfptr_t) 0; /* mark entry as free */
3972 ps->ps_nsamples = 0; /* mark entry as free */
3769 atomic_subtract_rel_32(&pm->pm_runcount, 1);
3770
3771 /* increment read pointer, modulo sample size */
3772 if (++ps == psb->ps_fence)
3773 psb->ps_read = psb->ps_samples;
3774 else
3775 psb->ps_read = ps;
3776 }

--- 305 unchanged lines hidden (view full) ---

4082 __PMC_CLASSES()
4083};
4084
4085static int
4086pmc_initialize(void)
4087{
4088 int cpu, error, n;
4089 struct pmc_binding pb;
3973 atomic_subtract_rel_32(&pm->pm_runcount, 1);
3974
3975 /* increment read pointer, modulo sample size */
3976 if (++ps == psb->ps_fence)
3977 psb->ps_read = psb->ps_samples;
3978 else
3979 psb->ps_read = ps;
3980 }

--- 305 unchanged lines hidden (view full) ---

4286 __PMC_CLASSES()
4287};
4288
4289static int
4290pmc_initialize(void)
4291{
4292 int cpu, error, n;
4293 struct pmc_binding pb;
4294 struct pmc_sample *ps;
4090 struct pmc_samplebuffer *sb;
4091
4092 md = NULL;
4093 error = 0;
4094
4095#ifdef DEBUG
4096 /* parse debug flags first */
4097 if (TUNABLE_STR_FETCH(PMC_SYSCTL_NAME_PREFIX "debugflags",

--- 16 unchanged lines hidden (view full) ---

4114 return EPROGMISMATCH;
4115 }
4116
4117 /*
4118 * check sysctl parameters
4119 */
4120
4121 if (pmc_hashsize <= 0) {
4295 struct pmc_samplebuffer *sb;
4296
4297 md = NULL;
4298 error = 0;
4299
4300#ifdef DEBUG
4301 /* parse debug flags first */
4302 if (TUNABLE_STR_FETCH(PMC_SYSCTL_NAME_PREFIX "debugflags",

--- 16 unchanged lines hidden (view full) ---

4319 return EPROGMISMATCH;
4320 }
4321
4322 /*
4323 * check sysctl parameters
4324 */
4325
4326 if (pmc_hashsize <= 0) {
4122 (void) printf("hwpmc: tunable hashsize=%d must be greater "
4123 "than zero.\n", pmc_hashsize);
4327 (void) printf("hwpmc: tunable \"hashsize\"=%d must be "
4328 "greater than zero.\n", pmc_hashsize);
4124 pmc_hashsize = PMC_HASH_SIZE;
4125 }
4126
4127 if (pmc_nsamples <= 0 || pmc_nsamples > 65535) {
4329 pmc_hashsize = PMC_HASH_SIZE;
4330 }
4331
4332 if (pmc_nsamples <= 0 || pmc_nsamples > 65535) {
4128 (void) printf("hwpmc: tunable nsamples=%d out of range.\n",
4129 pmc_nsamples);
4333 (void) printf("hwpmc: tunable \"nsamples\"=%d out of "
4334 "range.\n", pmc_nsamples);
4130 pmc_nsamples = PMC_NSAMPLES;
4131 }
4132
4335 pmc_nsamples = PMC_NSAMPLES;
4336 }
4337
4338 if (pmc_callchaindepth <= 0 ||
4339 pmc_callchaindepth > PMC_CALLCHAIN_DEPTH_MAX) {
4340 (void) printf("hwpmc: tunable \"callchaindepth\"=%d out of "
4341 "range.\n", pmc_callchaindepth);
4342 pmc_callchaindepth = PMC_CALLCHAIN_DEPTH;
4343 }
4344
4133 md = pmc_md_initialize();
4134
4135 if (md == NULL || md->pmd_init == NULL)
4136 return ENOSYS;
4137
4138 /* allocate space for the per-cpu array */
4139 MALLOC(pmc_pcpu, struct pmc_cpu **, mp_ncpus * sizeof(struct pmc_cpu *),
4140 M_PMC, M_WAITOK|M_ZERO);

--- 25 unchanged lines hidden (view full) ---

4166 pmc_nsamples * sizeof(struct pmc_sample), M_PMC,
4167 M_WAITOK|M_ZERO);
4168
4169 sb->ps_read = sb->ps_write = sb->ps_samples;
4170 sb->ps_fence = sb->ps_samples + pmc_nsamples;
4171 KASSERT(pmc_pcpu[cpu] != NULL,
4172 ("[pmc,%d] cpu=%d Null per-cpu data", __LINE__, cpu));
4173
4345 md = pmc_md_initialize();
4346
4347 if (md == NULL || md->pmd_init == NULL)
4348 return ENOSYS;
4349
4350 /* allocate space for the per-cpu array */
4351 MALLOC(pmc_pcpu, struct pmc_cpu **, mp_ncpus * sizeof(struct pmc_cpu *),
4352 M_PMC, M_WAITOK|M_ZERO);

--- 25 unchanged lines hidden (view full) ---

4378 pmc_nsamples * sizeof(struct pmc_sample), M_PMC,
4379 M_WAITOK|M_ZERO);
4380
4381 sb->ps_read = sb->ps_write = sb->ps_samples;
4382 sb->ps_fence = sb->ps_samples + pmc_nsamples;
4383 KASSERT(pmc_pcpu[cpu] != NULL,
4384 ("[pmc,%d] cpu=%d Null per-cpu data", __LINE__, cpu));
4385
4386 MALLOC(sb->ps_callchains, uintptr_t *,
4387 pmc_callchaindepth * pmc_nsamples * sizeof(uintptr_t),
4388 M_PMC, M_WAITOK|M_ZERO);
4389
4390 for (n = 0, ps = sb->ps_samples; n < pmc_nsamples; n++, ps++)
4391 ps->ps_pc = sb->ps_callchains +
4392 (n * pmc_callchaindepth);
4393
4174 pmc_pcpu[cpu]->pc_sb = sb;
4175 }
4176
4177 /* allocate space for the row disposition array */
4178 pmc_pmcdisp = malloc(sizeof(enum pmc_mode) * md->pmd_npmc,
4179 M_PMC, M_WAITOK|M_ZERO);
4180
4181 KASSERT(pmc_pmcdisp != NULL,

--- 140 unchanged lines hidden (view full) ---

4322
4323 /* free the per-cpu sample buffers */
4324 for (cpu = 0; cpu < mp_ncpus; cpu++) {
4325 if (pmc_cpu_is_disabled(cpu))
4326 continue;
4327 KASSERT(pmc_pcpu[cpu]->pc_sb != NULL,
4328 ("[pmc,%d] Null cpu sample buffer cpu=%d", __LINE__,
4329 cpu));
4394 pmc_pcpu[cpu]->pc_sb = sb;
4395 }
4396
4397 /* allocate space for the row disposition array */
4398 pmc_pmcdisp = malloc(sizeof(enum pmc_mode) * md->pmd_npmc,
4399 M_PMC, M_WAITOK|M_ZERO);
4400
4401 KASSERT(pmc_pmcdisp != NULL,

--- 140 unchanged lines hidden (view full) ---

4542
4543 /* free the per-cpu sample buffers */
4544 for (cpu = 0; cpu < mp_ncpus; cpu++) {
4545 if (pmc_cpu_is_disabled(cpu))
4546 continue;
4547 KASSERT(pmc_pcpu[cpu]->pc_sb != NULL,
4548 ("[pmc,%d] Null cpu sample buffer cpu=%d", __LINE__,
4549 cpu));
4550 FREE(pmc_pcpu[cpu]->pc_sb->ps_callchains, M_PMC);
4330 FREE(pmc_pcpu[cpu]->pc_sb, M_PMC);
4331 pmc_pcpu[cpu]->pc_sb = NULL;
4332 }
4333
4334 /* do processor dependent cleanup */
4335 PMCDBG(MOD,INI,3, "%s", "md cleanup");
4336 if (md) {
4337 pmc_save_cpu_binding(&pb);

--- 69 unchanged lines hidden --- 		4551 FREE(pmc_pcpu[cpu]->pc_sb, M_PMC);
4552 pmc_pcpu[cpu]->pc_sb = NULL;
4553 }
4554
4555 /* do processor dependent cleanup */
4556 PMCDBG(MOD,INI,3, "%s", "md cleanup");
4557 if (md) {
4558 pmc_save_cpu_binding(&pb);

--- 69 unchanged lines hidden ---