1/*-
2 * Copyright (c) 2003-2005 Joseph Koshy
3 * All rights reserved.
4 *
5 * Redistribution and use in source and binary forms, with or without
6 * modification, are permitted provided that the following conditions
7 * are met:
8 * 1. Redistributions of source code must retain the above copyright
9 * notice, this list of conditions and the following disclaimer.
10 * 2. Redistributions in binary form must reproduce the above copyright
11 * notice, this list of conditions and the following disclaimer in the
12 * documentation and/or other materials provided with the distribution.
13 *
14 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
15 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
16 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
17 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
18 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
19 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
20 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
21 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
22 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
23 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
24 * SUCH DAMAGE.
25 *
26 */
27
28#include <sys/cdefs.h>
29__FBSDID("$FreeBSD: head/sys/dev/hwpmc/hwpmc_mod.c 148562 2005-07-30 09:02:42Z jkoshy $");
30
31#include <sys/param.h>
32#include <sys/eventhandler.h>
33#include <sys/jail.h>
34#include <sys/kernel.h>
35#include <sys/kthread.h>
36#include <sys/limits.h>
37#include <sys/lock.h>
38#include <sys/malloc.h>
39#include <sys/module.h>
40#include <sys/mutex.h>
41#include <sys/pmc.h>
42#include <sys/pmckern.h>
43#include <sys/pmclog.h>
44#include <sys/proc.h>
45#include <sys/queue.h>
46#include <sys/resourcevar.h>
47#include <sys/sched.h>
48#include <sys/signalvar.h>
49#include <sys/smp.h>
50#include <sys/sx.h>
51#include <sys/sysctl.h>
52#include <sys/sysent.h>
53#include <sys/systm.h>
54#include <sys/vnode.h>
55
56#include <machine/atomic.h>
57#include <machine/md_var.h>
58
59/*
60 * Types
61 */
62
63enum pmc_flags {
64 PMC_FLAG_NONE = 0x00, /* do nothing */
65 PMC_FLAG_REMOVE = 0x01, /* atomically remove entry from hash */
66 PMC_FLAG_ALLOCATE = 0x02, /* add entry to hash if not found */
67};
68
69/*
70 * The offset in sysent where the syscall is allocated.
71 */
72
73static int pmc_syscall_num = NO_SYSCALL;
74struct pmc_cpu **pmc_pcpu; /* per-cpu state */
75pmc_value_t *pmc_pcpu_saved; /* saved PMC values: CSW handling */
76
77#define PMC_PCPU_SAVED(C,R) pmc_pcpu_saved[(R) + md->pmd_npmc*(C)]
78
79struct mtx_pool *pmc_mtxpool;
80static int *pmc_pmcdisp; /* PMC row dispositions */
81
82#define PMC_ROW_DISP_IS_FREE(R) (pmc_pmcdisp[(R)] == 0)
83#define PMC_ROW_DISP_IS_THREAD(R) (pmc_pmcdisp[(R)] > 0)
84#define PMC_ROW_DISP_IS_STANDALONE(R) (pmc_pmcdisp[(R)] < 0)
85
86#define PMC_MARK_ROW_FREE(R) do { \
87 pmc_pmcdisp[(R)] = 0; \
88} while (0)
89
90#define PMC_MARK_ROW_STANDALONE(R) do { \
91 KASSERT(pmc_pmcdisp[(R)] <= 0, ("[pmc,%d] row disposition error", \
92 __LINE__)); \
93 atomic_add_int(&pmc_pmcdisp[(R)], -1); \
94 KASSERT(pmc_pmcdisp[(R)] >= (-mp_ncpus), ("[pmc,%d] row " \
95 "disposition error", __LINE__)); \
96} while (0)
97
98#define PMC_UNMARK_ROW_STANDALONE(R) do { \
99 atomic_add_int(&pmc_pmcdisp[(R)], 1); \
100 KASSERT(pmc_pmcdisp[(R)] <= 0, ("[pmc,%d] row disposition error", \
101 __LINE__)); \
102} while (0)
103
104#define PMC_MARK_ROW_THREAD(R) do { \
105 KASSERT(pmc_pmcdisp[(R)] >= 0, ("[pmc,%d] row disposition error", \
106 __LINE__)); \
107 atomic_add_int(&pmc_pmcdisp[(R)], 1); \
108} while (0)
109
110#define PMC_UNMARK_ROW_THREAD(R) do { \
111 atomic_add_int(&pmc_pmcdisp[(R)], -1); \
112 KASSERT(pmc_pmcdisp[(R)] >= 0, ("[pmc,%d] row disposition error", \
113 __LINE__)); \
114} while (0)
115
116
117/* various event handlers */
118static eventhandler_tag pmc_exit_tag, pmc_fork_tag;
119
120/* Module statistics */
121struct pmc_op_getdriverstats pmc_stats;
122
123/* Machine/processor dependent operations */
124struct pmc_mdep *md;
125
126/*
127 * Hash tables mapping owner processes and target threads to PMCs.
128 */
129
130struct mtx pmc_processhash_mtx; /* spin mutex */
131static u_long pmc_processhashmask;
132static LIST_HEAD(pmc_processhash, pmc_process) *pmc_processhash;
133
134/*
135 * Hash table of PMC owner descriptors. This table is protected by
136 * the shared PMC "sx" lock.
137 */
138
139static u_long pmc_ownerhashmask;
140static LIST_HEAD(pmc_ownerhash, pmc_owner) *pmc_ownerhash;
141
142/*
143 * List of PMC owners with system-wide sampling PMCs.
144 */
145
146static LIST_HEAD(, pmc_owner) pmc_ss_owners;
147
148
149/*
150 * Prototypes
151 */
152
153#if DEBUG
154static int pmc_debugflags_sysctl_handler(SYSCTL_HANDLER_ARGS);
155static int pmc_debugflags_parse(char *newstr, char *fence);
156#endif
157
158static int load(struct module *module, int cmd, void *arg);
159static int pmc_attach_process(struct proc *p, struct pmc *pm);
160static struct pmc *pmc_allocate_pmc_descriptor(void);
161static struct pmc_owner *pmc_allocate_owner_descriptor(struct proc *p);
162static int pmc_attach_one_process(struct proc *p, struct pmc *pm);
163static int pmc_can_allocate_rowindex(struct proc *p, unsigned int ri,
164 int cpu);
165static int pmc_can_attach(struct pmc *pm, struct proc *p);
166static void pmc_cleanup(void);
167static int pmc_detach_process(struct proc *p, struct pmc *pm);
168static int pmc_detach_one_process(struct proc *p, struct pmc *pm,
169 int flags);
170static void pmc_destroy_owner_descriptor(struct pmc_owner *po);
171static struct pmc_owner *pmc_find_owner_descriptor(struct proc *p);
172static int pmc_find_pmc(pmc_id_t pmcid, struct pmc **pm);
173static struct pmc *pmc_find_pmc_descriptor_in_process(struct pmc_owner *po,
174 pmc_id_t pmc);
175static struct pmc_process *pmc_find_process_descriptor(struct proc *p,
176 uint32_t mode);
177static void pmc_force_context_switch(void);
178static void pmc_link_target_process(struct pmc *pm,
179 struct pmc_process *pp);
180static void pmc_maybe_remove_owner(struct pmc_owner *po);
181static void pmc_process_csw_in(struct thread *td);
182static void pmc_process_csw_out(struct thread *td);
183static void pmc_process_exit(void *arg, struct proc *p);
184static void pmc_process_fork(void *arg, struct proc *p1,
185 struct proc *p2, int n);
186static void pmc_process_samples(int cpu);
187static void pmc_release_pmc_descriptor(struct pmc *pmc);
188static void pmc_remove_owner(struct pmc_owner *po);
189static void pmc_remove_process_descriptor(struct pmc_process *pp);
190static void pmc_restore_cpu_binding(struct pmc_binding *pb);
191static void pmc_save_cpu_binding(struct pmc_binding *pb);
192static void pmc_select_cpu(int cpu);
193static int pmc_start(struct pmc *pm);
194static int pmc_stop(struct pmc *pm);
195static int pmc_syscall_handler(struct thread *td, void *syscall_args);
196static void pmc_unlink_target_process(struct pmc *pmc,
197 struct pmc_process *pp);
198
199/*
200 * Kernel tunables and sysctl(8) interface.
201 */
202
203SYSCTL_NODE(_kern, OID_AUTO, hwpmc, CTLFLAG_RW, 0, "HWPMC parameters");
204
205#if DEBUG
206struct pmc_debugflags pmc_debugflags = PMC_DEBUG_DEFAULT_FLAGS;
207char pmc_debugstr[PMC_DEBUG_STRSIZE];
208TUNABLE_STR(PMC_SYSCTL_NAME_PREFIX "debugflags", pmc_debugstr,
209 sizeof(pmc_debugstr));
210SYSCTL_PROC(_kern_hwpmc, OID_AUTO, debugflags,
211 CTLTYPE_STRING|CTLFLAG_RW|CTLFLAG_TUN,
212 0, 0, pmc_debugflags_sysctl_handler, "A", "debug flags");
213#endif
214
215/*
216 * kern.hwpmc.hashrows -- determines the number of rows in the
217 * of the hash table used to look up threads
218 */
219
220static int pmc_hashsize = PMC_HASH_SIZE;
221TUNABLE_INT(PMC_SYSCTL_NAME_PREFIX "hashsize", &pmc_hashsize);
222SYSCTL_INT(_kern_hwpmc, OID_AUTO, hashsize, CTLFLAG_TUN|CTLFLAG_RD,
223 &pmc_hashsize, 0, "rows in hash tables");
224
225/*
226 * kern.hwpmc.nsamples --- number of PC samples per CPU
227 */
228
229static int pmc_nsamples = PMC_NSAMPLES;
230TUNABLE_INT(PMC_SYSCTL_NAME_PREFIX "nsamples", &pmc_nsamples);
231SYSCTL_INT(_kern_hwpmc, OID_AUTO, nsamples, CTLFLAG_TUN|CTLFLAG_RD,
232 &pmc_nsamples, 0, "number of PC samples per CPU");
233
234/*
235 * kern.hwpmc.mtxpoolsize -- number of mutexes in the mutex pool.
236 */
237
238static int pmc_mtxpool_size = PMC_MTXPOOL_SIZE;
239TUNABLE_INT(PMC_SYSCTL_NAME_PREFIX "mtxpoolsize", &pmc_mtxpool_size);
240SYSCTL_INT(_kern_hwpmc, OID_AUTO, mtxpoolsize, CTLFLAG_TUN|CTLFLAG_RD,
241 &pmc_mtxpool_size, 0, "size of spin mutex pool");
242
243
244/*
245 * security.bsd.unprivileged_syspmcs -- allow non-root processes to
246 * allocate system-wide PMCs.
247 *
248 * Allowing unprivileged processes to allocate system PMCs is convenient
249 * if system-wide measurements need to be taken concurrently with other
250 * per-process measurements. This feature is turned off by default.
251 */
252
253SYSCTL_DECL(_security_bsd);
254
255static int pmc_unprivileged_syspmcs = 0;
256TUNABLE_INT("security.bsd.unprivileged_syspmcs", &pmc_unprivileged_syspmcs);
257SYSCTL_INT(_security_bsd, OID_AUTO, unprivileged_syspmcs, CTLFLAG_RW,
258 &pmc_unprivileged_syspmcs, 0,
259 "allow unprivileged process to allocate system PMCs");
260
261/*
262 * Hash function. Discard the lower 2 bits of the pointer since
263 * these are always zero for our uses. The hash multiplier is
264 * round((2^LONG_BIT) * ((sqrt(5)-1)/2)).
265 */
266
267#if LONG_BIT == 64
268#define _PMC_HM 11400714819323198486u
269#elif LONG_BIT == 32
270#define _PMC_HM 2654435769u
271#else
272#error Must know the size of 'long' to compile
273#endif
274
275#define PMC_HASH_PTR(P,M) ((((unsigned long) (P) >> 2) * _PMC_HM) & (M))
276
277/*
278 * Syscall structures
279 */
280
281/* The `sysent' for the new syscall */
282static struct sysent pmc_sysent = {
283 2, /* sy_narg */
284 pmc_syscall_handler /* sy_call */
285};
286
287static struct syscall_module_data pmc_syscall_mod = {
288 load,
289 NULL,
290 &pmc_syscall_num,
291 &pmc_sysent,
292 { 0, NULL }
293};
294
295static moduledata_t pmc_mod = {
296 PMC_MODULE_NAME,
297 syscall_module_handler,
298 &pmc_syscall_mod
299};
300
301DECLARE_MODULE(pmc, pmc_mod, SI_SUB_SMP, SI_ORDER_ANY);
302MODULE_VERSION(pmc, PMC_VERSION);
303
304#if DEBUG
305enum pmc_dbgparse_state {
306 PMCDS_WS, /* in whitespace */
307 PMCDS_MAJOR, /* seen a major keyword */
308 PMCDS_MINOR
309};
310
311static int
312pmc_debugflags_parse(char *newstr, char *fence)
313{
314 char c, *p, *q;
315 struct pmc_debugflags *tmpflags;
316 int error, found, *newbits, tmp;
317 size_t kwlen;
318
319 MALLOC(tmpflags, struct pmc_debugflags *, sizeof(*tmpflags),
320 M_PMC, M_WAITOK|M_ZERO);
321
322 p = newstr;
323 error = 0;
324
325 for (; p < fence && (c = *p); p++) {
326
327 /* skip white space */
328 if (c == ' ' || c == '\t')
329 continue;
330
331 /* look for a keyword followed by "=" */
332 for (q = p; p < fence && (c = *p) && c != '='; p++)
333 ;
334 if (c != '=') {
335 error = EINVAL;
336 goto done;
337 }
338
339 kwlen = p - q;
340 newbits = NULL;
341
342 /* lookup flag group name */
343#define DBG_SET_FLAG_MAJ(S,F) \
344 if (kwlen == sizeof(S)-1 && strncmp(q, S, kwlen) == 0) \
345 newbits = &tmpflags->pdb_ ## F;
346
347 DBG_SET_FLAG_MAJ("cpu", CPU);
348 DBG_SET_FLAG_MAJ("csw", CSW);
349 DBG_SET_FLAG_MAJ("logging", LOG);
350 DBG_SET_FLAG_MAJ("module", MOD);
351 DBG_SET_FLAG_MAJ("md", MDP);
352 DBG_SET_FLAG_MAJ("owner", OWN);
353 DBG_SET_FLAG_MAJ("pmc", PMC);
354 DBG_SET_FLAG_MAJ("process", PRC);
355 DBG_SET_FLAG_MAJ("sampling", SAM);
356
357 if (newbits == NULL) {
358 error = EINVAL;
359 goto done;
360 }
361
362 p++; /* skip the '=' */
363
364 /* Now parse the individual flags */
365 tmp = 0;
366 newflag:
367 for (q = p; p < fence && (c = *p); p++)
368 if (c == ' ' || c == '\t' || c == ',')
369 break;
370
371 /* p == fence or c == ws or c == "," or c == 0 */
372
373 if ((kwlen = p - q) == 0) {
374 *newbits = tmp;
375 continue;
376 }
377
378 found = 0;
379#define DBG_SET_FLAG_MIN(S,F) \
380 if (kwlen == sizeof(S)-1 && strncmp(q, S, kwlen) == 0) \
381 tmp |= found = (1 << PMC_DEBUG_MIN_ ## F)
382
383 /* a '*' denotes all possible flags in the group */
384 if (kwlen == 1 && *q == '*')
385 tmp = found = ~0;
386 /* look for individual flag names */
387 DBG_SET_FLAG_MIN("allocaterow", ALR);
388 DBG_SET_FLAG_MIN("allocate", ALL);
389 DBG_SET_FLAG_MIN("attach", ATT);
390 DBG_SET_FLAG_MIN("bind", BND);
391 DBG_SET_FLAG_MIN("config", CFG);
392 DBG_SET_FLAG_MIN("exec", EXC);
393 DBG_SET_FLAG_MIN("exit", EXT);
394 DBG_SET_FLAG_MIN("find", FND);
395 DBG_SET_FLAG_MIN("flush", FLS);
396 DBG_SET_FLAG_MIN("fork", FRK);
397 DBG_SET_FLAG_MIN("getbuf", GTB);
398 DBG_SET_FLAG_MIN("hook", PMH);
399 DBG_SET_FLAG_MIN("init", INI);
400 DBG_SET_FLAG_MIN("intr", INT);
401 DBG_SET_FLAG_MIN("linktarget", TLK);
402 DBG_SET_FLAG_MIN("mayberemove", OMR);
403 DBG_SET_FLAG_MIN("ops", OPS);
404 DBG_SET_FLAG_MIN("read", REA);
405 DBG_SET_FLAG_MIN("register", REG);
406 DBG_SET_FLAG_MIN("release", REL);
407 DBG_SET_FLAG_MIN("remove", ORM);
408 DBG_SET_FLAG_MIN("sample", SAM);
409 DBG_SET_FLAG_MIN("scheduleio", SIO);
410 DBG_SET_FLAG_MIN("select", SEL);
411 DBG_SET_FLAG_MIN("signal", SIG);
412 DBG_SET_FLAG_MIN("swi", SWI);
413 DBG_SET_FLAG_MIN("swo", SWO);
414 DBG_SET_FLAG_MIN("start", STA);
415 DBG_SET_FLAG_MIN("stop", STO);
416 DBG_SET_FLAG_MIN("syscall", PMS);
417 DBG_SET_FLAG_MIN("unlinktarget", TUL);
418 DBG_SET_FLAG_MIN("write", WRI);
419 if (found == 0) {
420 /* unrecognized flag name */
421 error = EINVAL;
422 goto done;
423 }
424
425 if (c == 0 || c == ' ' || c == '\t') { /* end of flag group */
426 *newbits = tmp;
427 continue;
428 }
429
430 p++;
431 goto newflag;
432 }
433
434 /* save the new flag set */
435 bcopy(tmpflags, &pmc_debugflags, sizeof(pmc_debugflags));
436
437 done:
438 FREE(tmpflags, M_PMC);
439 return error;
440}
441
442static int
443pmc_debugflags_sysctl_handler(SYSCTL_HANDLER_ARGS)
444{
445 char *fence, *newstr;
446 int error;
447 unsigned int n;
448
449 (void) arg1; (void) arg2; /* unused parameters */
450
451 n = sizeof(pmc_debugstr);
452 MALLOC(newstr, char *, n, M_PMC, M_ZERO|M_WAITOK);
453 (void) strlcpy(newstr, pmc_debugstr, n);
454
455 error = sysctl_handle_string(oidp, newstr, n, req);
456
457 /* if there is a new string, parse and copy it */
458 if (error == 0 && req->newptr != NULL) {
459 fence = newstr + (n < req->newlen ? n : req->newlen + 1);
460 if ((error = pmc_debugflags_parse(newstr, fence)) == 0)
461 (void) strlcpy(pmc_debugstr, newstr,
462 sizeof(pmc_debugstr));
463 }
464
465 FREE(newstr, M_PMC);
466
467 return error;
468}
469#endif
470
471/*
472 * Concurrency Control
473 *
474 * The driver manages the following data structures:
475 *
476 * - target process descriptors, one per target process
477 * - owner process descriptors (and attached lists), one per owner process
478 * - lookup hash tables for owner and target processes
479 * - PMC descriptors (and attached lists)
480 * - per-cpu hardware state
481 * - the 'hook' variable through which the kernel calls into
482 * this module
483 * - the machine hardware state (managed by the MD layer)
484 *
485 * These data structures are accessed from:
486 *
487 * - thread context-switch code
488 * - interrupt handlers (possibly on multiple cpus)
489 * - kernel threads on multiple cpus running on behalf of user
490 * processes doing system calls
491 * - this driver's private kernel threads
492 *
493 * = Locks and Locking strategy =
494 *
495 * The driver uses four locking strategies for its operation:
496 *
497 * - There is a 'global' SX lock "pmc_sx" that is used to protect
498 * the its 'meta-data'.
499 *
500 * Calls into the module (via syscall() or by the kernel) start with
501 * this lock being held in exclusive mode. Depending on the requested
502 * operation, the lock may be downgraded to 'shared' mode to allow
503 * more concurrent readers into the module.
504 *
505 * This SX lock is held in exclusive mode for any operations that
506 * modify the linkages between the driver's internal data structures.
507 *
508 * The 'pmc_hook' function pointer is also protected by this lock.
509 * It is only examined with the sx lock held in exclusive mode. The
510 * kernel module is allowed to be unloaded only with the sx lock
511 * held in exclusive mode. In normal syscall handling, after
512 * acquiring the pmc_sx lock we first check that 'pmc_hook' is
513 * non-null before proceeding. This prevents races between the
514 * thread unloading the module and other threads seeking to use the
515 * module.
516 *
517 * - Lookups of target process structures and owner process structures
518 * cannot use the global "pmc_sx" SX lock because these lookups need
519 * to happen during context switches and in other critical sections
520 * where sleeping is not allowed. We protect these lookup tables
521 * with their own private spin-mutexes, "pmc_processhash_mtx" and
522 * "pmc_ownerhash_mtx". These are 'leaf' mutexes, in that no other
523 * lock is acquired with these locks held.
524 *
525 * - Interrupt handlers work in a lock free manner. At interrupt
526 * time, handlers look at the PMC pointer (phw->phw_pmc) configured
527 * when the PMC was started. If this pointer is NULL, the interrupt
528 * is ignored after updating driver statistics. We ensure that this
529 * pointer is set (using an atomic operation if necessary) before the
530 * PMC hardware is started. Conversely, this pointer is unset atomically
531 * only after the PMC hardware is stopped.
532 *
533 * We ensure that everything needed for the operation of an
534 * interrupt handler is available without it needing to acquire any
535 * locks. We also ensure that a PMC's software state is destroyed only
536 * after the PMC is taken off hardware (on all CPUs).
537 *
538 * - Context-switch handling with process-private PMCs needs more
539 * care.
540 *
541 * A given process may be the target of multiple PMCs. For example,
542 * PMCATTACH and PMCDETACH may be requested by a process on one CPU
543 * while the target process is running on another. A PMC could also
544 * be getting released because its owner is exiting. We tackle
545 * these situations in the following manner:
546 *
547 * - each target process structure 'pmc_process' has an array
548 * of 'struct pmc *' pointers, one for each hardware PMC.
549 *
550 * - At context switch IN time, each "target" PMC in RUNNING state
551 * gets started on hardware and a pointer to each PMC is copied into
552 * the per-cpu phw array. The 'runcount' for the PMC is
553 * incremented.
554 *
555 * - At context switch OUT time, all process-virtual PMCs are stopped
556 * on hardware. The saved value is added to the PMCs value field
557 * only if the PMC is in a non-deleted state (the PMCs state could
558 * have changed during the current time slice).
559 *
560 * Note that since in-between a switch IN on a processor and a switch
561 * OUT, the PMC could have been released on another CPU. Therefore
562 * context switch OUT always looks at the hardware state to turn
563 * OFF PMCs and will update a PMC's saved value only if reachable
564 * from the target process record.
565 *
566 * - OP PMCRELEASE could be called on a PMC at any time (the PMC could
567 * be attached to many processes at the time of the call and could
568 * be active on multiple CPUs).
569 *
570 * We prevent further scheduling of the PMC by marking it as in
571 * state 'DELETED'. If the runcount of the PMC is non-zero then
572 * this PMC is currently running on a CPU somewhere. The thread
573 * doing the PMCRELEASE operation waits by repeatedly doing an
574 * tsleep() till the runcount comes to zero.
575 *
576 */
577
578/*
579 * save the cpu binding of the current kthread
580 */
581
582static void
583pmc_save_cpu_binding(struct pmc_binding *pb)
584{
585 PMCDBG(CPU,BND,2, "%s", "save-cpu");
586 mtx_lock_spin(&sched_lock);
587 pb->pb_bound = sched_is_bound(curthread);
588 pb->pb_cpu = curthread->td_oncpu;
589 mtx_unlock_spin(&sched_lock);
590 PMCDBG(CPU,BND,2, "save-cpu cpu=%d", pb->pb_cpu);
591}
592
593/*
594 * restore the cpu binding of the current thread
595 */
596
597static void
598pmc_restore_cpu_binding(struct pmc_binding *pb)
599{
600 PMCDBG(CPU,BND,2, "restore-cpu curcpu=%d restore=%d",
601 curthread->td_oncpu, pb->pb_cpu);
602 mtx_lock_spin(&sched_lock);
603 if (pb->pb_bound)
604 sched_bind(curthread, pb->pb_cpu);
605 else
606 sched_unbind(curthread);
607 mtx_unlock_spin(&sched_lock);
608 PMCDBG(CPU,BND,2, "%s", "restore-cpu done");
609}
610
611/*
612 * move execution over the specified cpu and bind it there.
613 */
614
615static void
616pmc_select_cpu(int cpu)
617{
618 KASSERT(cpu >= 0 && cpu < mp_ncpus,
619 ("[pmc,%d] bad cpu number %d", __LINE__, cpu));
620
621 /* never move to a disabled CPU */
622 KASSERT(pmc_cpu_is_disabled(cpu) == 0, ("[pmc,%d] selecting "
623 "disabled CPU %d", __LINE__, cpu));
624
625 PMCDBG(CPU,SEL,2, "select-cpu cpu=%d", cpu);
626 mtx_lock_spin(&sched_lock);
627 sched_bind(curthread, cpu);
628 mtx_unlock_spin(&sched_lock);
629
630 KASSERT(curthread->td_oncpu == cpu,
631 ("[pmc,%d] CPU not bound [cpu=%d, curr=%d]", __LINE__,
632 cpu, curthread->td_oncpu));
633
634 PMCDBG(CPU,SEL,2, "select-cpu cpu=%d ok", cpu);
635}
636
637/*
638 * Force a context switch.
639 *
640 * We do this by tsleep'ing for 1 tick -- invoking mi_switch() is not
641 * guaranteed to force a context switch.
642 */
643
644static void
645pmc_force_context_switch(void)
646{
647 u_char curpri;
648
649 mtx_lock_spin(&sched_lock);
650 curpri = curthread->td_priority;
651 mtx_unlock_spin(&sched_lock);
652
653 (void) tsleep((void *) pmc_force_context_switch, curpri,
654 "pmcctx", 1);
655
656}
657
658/*
659 * Get the file name for an executable. This is a simple wrapper
660 * around vn_fullpath(9).
661 */
662
663static void
664pmc_getfilename(struct vnode *v, char **fullpath, char **freepath)
665{
666 struct thread *td;
667
668 td = curthread;
669 *fullpath = "unknown";
670 *freepath = NULL;
671 vn_lock(v, LK_CANRECURSE | LK_EXCLUSIVE | LK_RETRY, td);
672 vn_fullpath(td, v, fullpath, freepath);
673 VOP_UNLOCK(v, 0, td);
674}
675
676/*
677 * remove an process owning PMCs
678 */
679
680void
681pmc_remove_owner(struct pmc_owner *po)
682{
683 struct pmc *pm, *tmp;
684
685 sx_assert(&pmc_sx, SX_XLOCKED);
686
687 PMCDBG(OWN,ORM,1, "remove-owner po=%p", po);
688
689 /* Remove descriptor from the owner hash table */
690 LIST_REMOVE(po, po_next);
691
692 /* release all owned PMC descriptors */
693 LIST_FOREACH_SAFE(pm, &po->po_pmcs, pm_next, tmp) {
694 PMCDBG(OWN,ORM,2, "pmc=%p", pm);
695 KASSERT(pm->pm_owner == po,
696 ("[pmc,%d] owner %p != po %p", __LINE__, pm->pm_owner, po));
697
698 pmc_release_pmc_descriptor(pm); /* will unlink from the list */
699 }
700
701 KASSERT(po->po_sscount == 0,
702 ("[pmc,%d] SS count not zero", __LINE__));
703 KASSERT(LIST_EMPTY(&po->po_pmcs),
704 ("[pmc,%d] PMC list not empty", __LINE__));
705
706 /* de-configure the log file if present */
707 if (po->po_flags & PMC_PO_OWNS_LOGFILE)
708 pmclog_deconfigure_log(po);
709}
710
711/*
712 * remove an owner process record if all conditions are met.
713 */
714
715static void
716pmc_maybe_remove_owner(struct pmc_owner *po)
717{
718
719 PMCDBG(OWN,OMR,1, "maybe-remove-owner po=%p", po);
720
721 /*
722 * Remove owner record if
723 * - this process does not own any PMCs
724 * - this process has not allocated a system-wide sampling buffer
725 */
726
727 if (LIST_EMPTY(&po->po_pmcs) &&
728 ((po->po_flags & PMC_PO_OWNS_LOGFILE) == 0)) {
729 pmc_remove_owner(po);
730 pmc_destroy_owner_descriptor(po);
731 }
732}
733
734/*
735 * Add an association between a target process and a PMC.
736 */
737
738static void
739pmc_link_target_process(struct pmc *pm, struct pmc_process *pp)
740{
741 int ri;
742 struct pmc_target *pt;
743
744 sx_assert(&pmc_sx, SX_XLOCKED);
745
746 KASSERT(pm != NULL && pp != NULL,
747 ("[pmc,%d] Null pm %p or pp %p", __LINE__, pm, pp));
748 KASSERT(PMC_IS_VIRTUAL_MODE(PMC_TO_MODE(pm)),
749 ("[pmc,%d] Attaching a non-process-virtual pmc=%p to pid=%d",
750 __LINE__, pm, pp->pp_proc->p_pid));
751 KASSERT(pp->pp_refcnt >= 0 && pp->pp_refcnt < ((int) md->pmd_npmc - 1),
752 ("[pmc,%d] Illegal reference count %d for process record %p",
753 __LINE__, pp->pp_refcnt, (void *) pp));
754
755 ri = PMC_TO_ROWINDEX(pm);
756
757 PMCDBG(PRC,TLK,1, "link-target pmc=%p ri=%d pmc-process=%p",
758 pm, ri, pp);
759
760#if DEBUG
761 LIST_FOREACH(pt, &pm->pm_targets, pt_next)
762 if (pt->pt_process == pp)
763 KASSERT(0, ("[pmc,%d] pp %p already in pmc %p targets",
764 __LINE__, pp, pm));
765#endif
766
767 MALLOC(pt, struct pmc_target *, sizeof(struct pmc_target),
768 M_PMC, M_ZERO|M_WAITOK);
769
770 pt->pt_process = pp;
771
772 LIST_INSERT_HEAD(&pm->pm_targets, pt, pt_next);
773
774 atomic_store_rel_ptr((uintptr_t *)&pp->pp_pmcs[ri].pp_pmc,
775 (uintptr_t)pm);
776
777 if (pm->pm_owner->po_owner == pp->pp_proc)
778 pm->pm_flags |= PMC_F_ATTACHED_TO_OWNER;
779
780 /*
781 * Initialize the per-process values at this row index.
782 */
783 pp->pp_pmcs[ri].pp_pmcval = PMC_TO_MODE(pm) == PMC_MODE_TS ?
784 pm->pm_sc.pm_reloadcount : 0;
785
786 pp->pp_refcnt++;
787
788}
789
790/*
791 * Removes the association between a target process and a PMC.
792 */
793
794static void
795pmc_unlink_target_process(struct pmc *pm, struct pmc_process *pp)
796{
797 int ri;
798 struct proc *p;
799 struct pmc_target *ptgt;
800
801 sx_assert(&pmc_sx, SX_XLOCKED);
802
803 KASSERT(pm != NULL && pp != NULL,
804 ("[pmc,%d] Null pm %p or pp %p", __LINE__, pm, pp));
805
806 KASSERT(pp->pp_refcnt >= 1 && pp->pp_refcnt < (int) md->pmd_npmc,
807 ("[pmc,%d] Illegal ref count %d on process record %p",
808 __LINE__, pp->pp_refcnt, (void *) pp));
809
810 ri = PMC_TO_ROWINDEX(pm);
811
812 PMCDBG(PRC,TUL,1, "unlink-target pmc=%p ri=%d pmc-process=%p",
813 pm, ri, pp);
814
815 KASSERT(pp->pp_pmcs[ri].pp_pmc == pm,
816 ("[pmc,%d] PMC ri %d mismatch pmc %p pp->[ri] %p", __LINE__,
817 ri, pm, pp->pp_pmcs[ri].pp_pmc));
818
819 pp->pp_pmcs[ri].pp_pmc = NULL;
820 pp->pp_pmcs[ri].pp_pmcval = (pmc_value_t) 0;
821
822 /* Remove owner-specific flags */
823 if (pm->pm_owner->po_owner == pp->pp_proc) {
824 pp->pp_flags &= ~PMC_PP_ENABLE_MSR_ACCESS;
825 pm->pm_flags &= ~PMC_F_ATTACHED_TO_OWNER;
826 }
827
828 pp->pp_refcnt--;
829
830 /* Remove the target process from the PMC structure */
831 LIST_FOREACH(ptgt, &pm->pm_targets, pt_next)
832 if (ptgt->pt_process == pp)
833 break;
834
835 KASSERT(ptgt != NULL, ("[pmc,%d] process %p (pp: %p) not found "
836 "in pmc %p", __LINE__, pp->pp_proc, pp, pm));
837
838 LIST_REMOVE(ptgt, pt_next);
839 FREE(ptgt, M_PMC);
840
841 /* if the PMC now lacks targets, send the owner a SIGIO */
842 if (LIST_EMPTY(&pm->pm_targets)) {
843 p = pm->pm_owner->po_owner;
844 PROC_LOCK(p);
845 psignal(p, SIGIO);
846 PROC_UNLOCK(p);
847
848 PMCDBG(PRC,SIG,2, "signalling proc=%p signal=%d", p,
849 SIGIO);
850 }
851}
852
853/*
854 * Check if PMC 'pm' may be attached to target process 't'.
855 */
856
857static int
858pmc_can_attach(struct pmc *pm, struct proc *t)
859{
860 struct proc *o; /* pmc owner */
861 struct ucred *oc, *tc; /* owner, target credentials */
862 int decline_attach, i;
863
864 /*
865 * A PMC's owner can always attach that PMC to itself.
866 */
867
868 if ((o = pm->pm_owner->po_owner) == t)
869 return 0;
870
871 PROC_LOCK(o);
872 oc = o->p_ucred;
873 crhold(oc);
874 PROC_UNLOCK(o);
875
876 PROC_LOCK(t);
877 tc = t->p_ucred;
878 crhold(tc);
879 PROC_UNLOCK(t);
880
881 /*
882 * The effective uid of the PMC owner should match at least one
883 * of the {effective,real,saved} uids of the target process.
884 */
885
886 decline_attach = oc->cr_uid != tc->cr_uid &&
887 oc->cr_uid != tc->cr_svuid &&
888 oc->cr_uid != tc->cr_ruid;
889
890 /*
891 * Every one of the target's group ids, must be in the owner's
892 * group list.
893 */
894 for (i = 0; !decline_attach && i < tc->cr_ngroups; i++)
895 decline_attach = !groupmember(tc->cr_groups[i], oc);
896
897 /* check the read and saved gids too */
898 if (decline_attach == 0)
899 decline_attach = !groupmember(tc->cr_rgid, oc) ||
900 !groupmember(tc->cr_svgid, oc);
901
902 crfree(tc);
903 crfree(oc);
904
905 return !decline_attach;
906}
907
908/*
909 * Attach a process to a PMC.
910 */
911
912static int
913pmc_attach_one_process(struct proc *p, struct pmc *pm)
914{
915 int ri;
916 char *fullpath, *freepath;
917 struct pmc_process *pp;
918
919 sx_assert(&pmc_sx, SX_XLOCKED);
920
921 PMCDBG(PRC,ATT,2, "attach-one pm=%p ri=%d proc=%p (%d, %s)", pm,
922 PMC_TO_ROWINDEX(pm), p, p->p_pid, p->p_comm);
923
924 /*
925 * Locate the process descriptor corresponding to process 'p',
926 * allocating space as needed.
927 *
928 * Verify that rowindex 'pm_rowindex' is free in the process
929 * descriptor.
930 *
931 * If not, allocate space for a descriptor and link the
932 * process descriptor and PMC.
933 */
934 ri = PMC_TO_ROWINDEX(pm);
935
936 if ((pp = pmc_find_process_descriptor(p, PMC_FLAG_ALLOCATE)) == NULL)
937 return ENOMEM;
938
939 if (pp->pp_pmcs[ri].pp_pmc == pm) /* already present at slot [ri] */
940 return EEXIST;
941
942 if (pp->pp_pmcs[ri].pp_pmc != NULL)
943 return EBUSY;
944
945 pmc_link_target_process(pm, pp);
946
947 if (PMC_IS_SAMPLING_MODE(PMC_TO_MODE(pm)) &&
948 (pm->pm_flags & PMC_F_ATTACHED_TO_OWNER) == 0)
949 pm->pm_flags |= PMC_F_NEEDS_LOGFILE;
950
951 pm->pm_flags |= PMC_F_ATTACH_DONE; /* mark as attached */
952
953 /* issue an attach event to a configured log file */
954 if (pm->pm_owner->po_flags & PMC_PO_OWNS_LOGFILE) {
955 pmc_getfilename(p->p_textvp, &fullpath, &freepath);
956 pmclog_process_pmcattach(pm, p->p_pid, fullpath);
957 if (freepath)
958 FREE(freepath, M_TEMP);
959 }
960 /* mark process as using HWPMCs */
961 PROC_LOCK(p);
962 p->p_flag |= P_HWPMC;
963 PROC_UNLOCK(p);
964
965 return 0;
966}
967
968/*
969 * Attach a process and optionally its children
970 */
971
972static int
973pmc_attach_process(struct proc *p, struct pmc *pm)
974{
975 int error;
976 struct proc *top;
977
978 sx_assert(&pmc_sx, SX_XLOCKED);
979
980 PMCDBG(PRC,ATT,1, "attach pm=%p ri=%d proc=%p (%d, %s)", pm,
981 PMC_TO_ROWINDEX(pm), p, p->p_pid, p->p_comm);
982
983
984 /*
985 * If this PMC successfully allowed a GETMSR operation
986 * in the past, disallow further ATTACHes.
987 */
988
989 if ((pm->pm_flags & PMC_PP_ENABLE_MSR_ACCESS) != 0)
990 return EPERM;
991
992 if ((pm->pm_flags & PMC_F_DESCENDANTS) == 0)
993 return pmc_attach_one_process(p, pm);
994
995 /*
996 * Traverse all child processes, attaching them to
997 * this PMC.
998 */
999
1000 sx_slock(&proctree_lock);
1001
1002 top = p;
1003
1004 for (;;) {
1005 if ((error = pmc_attach_one_process(p, pm)) != 0)
1006 break;
1007 if (!LIST_EMPTY(&p->p_children))
1008 p = LIST_FIRST(&p->p_children);
1009 else for (;;) {
1010 if (p == top)
1011 goto done;
1012 if (LIST_NEXT(p, p_sibling)) {
1013 p = LIST_NEXT(p, p_sibling);
1014 break;
1015 }
1016 p = p->p_pptr;
1017 }
1018 }
1019
1020 if (error)
1021 (void) pmc_detach_process(top, pm);
1022
1023 done:
1024 sx_sunlock(&proctree_lock);
1025 return error;
1026}
1027
1028/*
1029 * Detach a process from a PMC. If there are no other PMCs tracking
1030 * this process, remove the process structure from its hash table. If
1031 * 'flags' contains PMC_FLAG_REMOVE, then free the process structure.
1032 */
1033
1034static int
1035pmc_detach_one_process(struct proc *p, struct pmc *pm, int flags)
1036{
1037 int ri;
1038 struct pmc_process *pp;
1039
1040 sx_assert(&pmc_sx, SX_XLOCKED);
1041
1042 KASSERT(pm != NULL,
1043 ("[pmc,%d] null pm pointer", __LINE__));
1044
1045 ri = PMC_TO_ROWINDEX(pm);
1046
1047 PMCDBG(PRC,ATT,2, "detach-one pm=%p ri=%d proc=%p (%d, %s) flags=0x%x",
1048 pm, ri, p, p->p_pid, p->p_comm, flags);
1049
1050 if ((pp = pmc_find_process_descriptor(p, 0)) == NULL)
1051 return ESRCH;
1052
1053 if (pp->pp_pmcs[ri].pp_pmc != pm)
1054 return EINVAL;
1055
1056 pmc_unlink_target_process(pm, pp);
1057
1058 /* Issue a detach entry if a log file is configured */
1059 if (pm->pm_owner->po_flags & PMC_PO_OWNS_LOGFILE)
1060 pmclog_process_pmcdetach(pm, p->p_pid);
1061
1062 /*
1063 * If there are no PMCs targetting this process, we remove its
1064 * descriptor from the target hash table and unset the P_HWPMC
1065 * flag in the struct proc.
1066 */
1067 KASSERT(pp->pp_refcnt >= 0 && pp->pp_refcnt < (int) md->pmd_npmc,
1068 ("[pmc,%d] Illegal refcnt %d for process struct %p",
1069 __LINE__, pp->pp_refcnt, pp));
1070
1071 if (pp->pp_refcnt != 0) /* still a target of some PMC */
1072 return 0;
1073
1074 pmc_remove_process_descriptor(pp);
1075
1076 if (flags & PMC_FLAG_REMOVE)
1077 FREE(pp, M_PMC);
1078
1079 PROC_LOCK(p);
1080 p->p_flag &= ~P_HWPMC;
1081 PROC_UNLOCK(p);
1082
1083 return 0;
1084}
1085
1086/*
1087 * Detach a process and optionally its descendants from a PMC.
1088 */
1089
1090static int
1091pmc_detach_process(struct proc *p, struct pmc *pm)
1092{
1093 struct proc *top;
1094
1095 sx_assert(&pmc_sx, SX_XLOCKED);
1096
1097 PMCDBG(PRC,ATT,1, "detach pm=%p ri=%d proc=%p (%d, %s)", pm,
1098 PMC_TO_ROWINDEX(pm), p, p->p_pid, p->p_comm);
1099
1100 if ((pm->pm_flags & PMC_F_DESCENDANTS) == 0)
1101 return pmc_detach_one_process(p, pm, PMC_FLAG_REMOVE);
1102
1103 /*
1104 * Traverse all children, detaching them from this PMC. We
1105 * ignore errors since we could be detaching a PMC from a
1106 * partially attached proc tree.
1107 */
1108
1109 sx_slock(&proctree_lock);
1110
1111 top = p;
1112
1113 for (;;) {
1114 (void) pmc_detach_one_process(p, pm, PMC_FLAG_REMOVE);
1115
1116 if (!LIST_EMPTY(&p->p_children))
1117 p = LIST_FIRST(&p->p_children);
1118 else for (;;) {
1119 if (p == top)
1120 goto done;
1121 if (LIST_NEXT(p, p_sibling)) {
1122 p = LIST_NEXT(p, p_sibling);
1123 break;
1124 }
1125 p = p->p_pptr;
1126 }
1127 }
1128
1129 done:
1130 sx_sunlock(&proctree_lock);
1131
1132 if (LIST_EMPTY(&pm->pm_targets))
1133 pm->pm_flags &= ~PMC_F_ATTACH_DONE;
1134
1135 return 0;
1136}
1137
1138
1139/*
1140 * Thread context switch IN
1141 */
1142
1143static void
1144pmc_process_csw_in(struct thread *td)
1145{
1146 int cpu;
1147 unsigned int ri;
1148 struct pmc *pm;
1149 struct proc *p;
1150 struct pmc_cpu *pc;
1151 struct pmc_hw *phw;
1152 struct pmc_process *pp;
1153 pmc_value_t newvalue;
1154
1155 p = td->td_proc;
1156
1157 if ((pp = pmc_find_process_descriptor(p, PMC_FLAG_NONE)) == NULL)
1158 return;
1159
1160 KASSERT(pp->pp_proc == td->td_proc,
1161 ("[pmc,%d] not my thread state", __LINE__));
1162
1163 critical_enter(); /* no preemption from this point */
1164
1165 cpu = PCPU_GET(cpuid); /* td->td_oncpu is invalid */
1166
1167 PMCDBG(CSW,SWI,1, "cpu=%d proc=%p (%d, %s) pp=%p", cpu, p,
1168 p->p_pid, p->p_comm, pp);
1169
1170 KASSERT(cpu >= 0 && cpu < mp_ncpus,
1171 ("[pmc,%d] wierd CPU id %d", __LINE__, cpu));
1172
1173 pc = pmc_pcpu[cpu];
1174
1175 for (ri = 0; ri < md->pmd_npmc; ri++) {
1176
1177 if ((pm = pp->pp_pmcs[ri].pp_pmc) == NULL)
1178 continue;
1179
1180 KASSERT(PMC_IS_VIRTUAL_MODE(PMC_TO_MODE(pm)),
1181 ("[pmc,%d] Target PMC in non-virtual mode (%d)",
1182 __LINE__, PMC_TO_MODE(pm)));
1183
1184 KASSERT(PMC_TO_ROWINDEX(pm) == ri,
1185 ("[pmc,%d] Row index mismatch pmc %d != ri %d",
1186 __LINE__, PMC_TO_ROWINDEX(pm), ri));
1187
1188 /*
1189 * Only PMCs that are marked as 'RUNNING' need
1190 * be placed on hardware.
1191 */
1192
1193 if (pm->pm_state != PMC_STATE_RUNNING)
1194 continue;
1195
1196 /* increment PMC runcount */
1197 atomic_add_rel_32(&pm->pm_runcount, 1);
1198
1199 /* configure the HWPMC we are going to use. */
1200 md->pmd_config_pmc(cpu, ri, pm);
1201
1202 phw = pc->pc_hwpmcs[ri];
1203
1204 KASSERT(phw != NULL,
1205 ("[pmc,%d] null hw pointer", __LINE__));
1206
1207 KASSERT(phw->phw_pmc == pm,
1208 ("[pmc,%d] hw->pmc %p != pmc %p", __LINE__,
1209 phw->phw_pmc, pm));
1210
1211 /*
1212 * Write out saved value and start the PMC.
1213 *
1214 * Sampling PMCs use a per-process value, while
1215 * counting mode PMCs use a per-pmc value that is
1216 * inherited across descendants.
1217 */
1218 if (PMC_TO_MODE(pm) == PMC_MODE_TS) {
1219 mtx_pool_lock_spin(pmc_mtxpool, pm);
1220 newvalue = PMC_PCPU_SAVED(cpu,ri) =
1221 pp->pp_pmcs[ri].pp_pmcval;
1222 mtx_pool_unlock_spin(pmc_mtxpool, pm);
1223 } else {
1224 KASSERT(PMC_TO_MODE(pm) == PMC_MODE_TC,
1225 ("[pmc,%d] illegal mode=%d", __LINE__,
1226 PMC_TO_MODE(pm)));
1227 mtx_pool_lock_spin(pmc_mtxpool, pm);
1228 newvalue = PMC_PCPU_SAVED(cpu, ri) =
1229 pm->pm_gv.pm_savedvalue;
1230 mtx_pool_unlock_spin(pmc_mtxpool, pm);
1231 }
1232
1233 PMCDBG(CSW,SWI,1,"cpu=%d ri=%d new=%jd", cpu, ri, newvalue);
1234
1235 md->pmd_write_pmc(cpu, ri, newvalue);
1236 md->pmd_start_pmc(cpu, ri);
1237 }
1238
1239 /*
1240 * perform any other architecture/cpu dependent thread
1241 * switch-in actions.
1242 */
1243
1244 (void) (*md->pmd_switch_in)(pc, pp);
1245
1246 critical_exit();
1247
1248}
1249
1250/*
1251 * Thread context switch OUT.
1252 */
1253
1254static void
1255pmc_process_csw_out(struct thread *td)
1256{
1257 int cpu;
1258 enum pmc_mode mode;
1259 unsigned int ri;
1260 struct pmc *pm;
1261 struct proc *p;
1262 struct pmc_cpu *pc;
1263 struct pmc_process *pp;
1264 int64_t tmp;
1265 pmc_value_t newvalue;
1266
1267 /*
1268 * Locate our process descriptor; this may be NULL if
1269 * this process is exiting and we have already removed
1270 * the process from the target process table.
1271 *
1272 * Note that due to kernel preemption, multiple
1273 * context switches may happen while the process is
1274 * exiting.
1275 *
1276 * Note also that if the target process cannot be
1277 * found we still need to deconfigure any PMCs that
1278 * are currently running on hardware.
1279 */
1280
1281 p = td->td_proc;
1282 pp = pmc_find_process_descriptor(p, PMC_FLAG_NONE);
1283
1284 /*
1285 * save PMCs
1286 */
1287
1288 critical_enter();
1289
1290 cpu = PCPU_GET(cpuid); /* td->td_oncpu is invalid */
1291
1292 PMCDBG(CSW,SWO,1, "cpu=%d proc=%p (%d, %s) pp=%p", cpu, p,
1293 p->p_pid, p->p_comm, pp);
1294
1295 KASSERT(cpu >= 0 && cpu < mp_ncpus,
1296 ("[pmc,%d wierd CPU id %d", __LINE__, cpu));
1297
1298 pc = pmc_pcpu[cpu];
1299
1300 /*
1301 * When a PMC gets unlinked from a target PMC, it will
1302 * be removed from the target's pp_pmc[] array.
1303 *
1304 * However, on a MP system, the target could have been
1305 * executing on another CPU at the time of the unlink.
1306 * So, at context switch OUT time, we need to look at
1307 * the hardware to determine if a PMC is scheduled on
1308 * it.
1309 */
1310
1311 for (ri = 0; ri < md->pmd_npmc; ri++) {
1312
1313 pm = NULL;
1314 (void) (*md->pmd_get_config)(cpu, ri, &pm);
1315
1316 if (pm == NULL) /* nothing at this row index */
1317 continue;
1318
1319 mode = PMC_TO_MODE(pm);
1320 if (!PMC_IS_VIRTUAL_MODE(mode))
1321 continue; /* not a process virtual PMC */
1322
1323 KASSERT(PMC_TO_ROWINDEX(pm) == ri,
1324 ("[pmc,%d] ri mismatch pmc(%d) ri(%d)",
1325 __LINE__, PMC_TO_ROWINDEX(pm), ri));
1326
1327 /* Stop hardware if not already stopped */
1328 if (pm->pm_stalled == 0)
1329 md->pmd_stop_pmc(cpu, ri);
1330
1331 /* reduce this PMC's runcount */
1332 atomic_subtract_rel_32(&pm->pm_runcount, 1);
1333
1334 /*
1335 * If this PMC is associated with this process,
1336 * save the reading.
1337 */
1338
1339 if (pp != NULL && pp->pp_pmcs[ri].pp_pmc != NULL) {
1340
1341 KASSERT(pm == pp->pp_pmcs[ri].pp_pmc,
1342 ("[pmc,%d] pm %p != pp_pmcs[%d] %p", __LINE__,
1343 pm, ri, pp->pp_pmcs[ri].pp_pmc));
1344
1345 KASSERT(pp->pp_refcnt > 0,
1346 ("[pmc,%d] pp refcnt = %d", __LINE__,
1347 pp->pp_refcnt));
1348
1349 md->pmd_read_pmc(cpu, ri, &newvalue);
1350
1351 tmp = newvalue - PMC_PCPU_SAVED(cpu,ri);
1352
1353 PMCDBG(CSW,SWI,1,"cpu=%d ri=%d tmp=%jd", cpu, ri,
1354 tmp);
1355
1356 if (mode == PMC_MODE_TS) {
1357
1358 /*
1359 * For sampling process-virtual PMCs,
1360 * we expect the count to be
1361 * decreasing as the 'value'
1362 * programmed into the PMC is the
1363 * number of events to be seen till
1364 * the next sampling interrupt.
1365 */
1366 if (tmp < 0)
1367 tmp += pm->pm_sc.pm_reloadcount;
1368 mtx_pool_lock_spin(pmc_mtxpool, pm);
1369 pp->pp_pmcs[ri].pp_pmcval -= tmp;
1370 if ((int64_t) pp->pp_pmcs[ri].pp_pmcval < 0)
1371 pp->pp_pmcs[ri].pp_pmcval +=
1372 pm->pm_sc.pm_reloadcount;
1373 mtx_pool_unlock_spin(pmc_mtxpool, pm);
1374
1375 } else {
1376
1377 /*
1378 * For counting process-virtual PMCs,
1379 * we expect the count to be
1380 * increasing monotonically, modulo a 64
1381 * bit wraparound.
1382 */
1383 KASSERT((int64_t) tmp >= 0,
1384 ("[pmc,%d] negative increment cpu=%d "
1385 "ri=%d newvalue=%jx saved=%jx "
1386 "incr=%jx", __LINE__, cpu, ri,
1387 newvalue, PMC_PCPU_SAVED(cpu,ri), tmp));
1388
1389 mtx_pool_lock_spin(pmc_mtxpool, pm);
1390 pm->pm_gv.pm_savedvalue += tmp;
1391 pp->pp_pmcs[ri].pp_pmcval += tmp;
1392 mtx_pool_unlock_spin(pmc_mtxpool, pm);
1393
1394 if (pm->pm_flags & PMC_F_LOG_PROCCSW)
1395 pmclog_process_proccsw(pm, pp, tmp);
1396 }
1397 }
1398
1399 /* mark hardware as free */
1400 md->pmd_config_pmc(cpu, ri, NULL);
1401 }
1402
1403 /*
1404 * perform any other architecture/cpu dependent thread
1405 * switch out functions.
1406 */
1407
1408 (void) (*md->pmd_switch_out)(pc, pp);
1409
1410 critical_exit();
1411}
1412
1413/*
1414 * The 'hook' invoked from the kernel proper
1415 */
1416
1417
1418#if DEBUG
1419const char *pmc_hooknames[] = {
1420 "",
1421 "EXIT",
1422 "EXEC",
1423 "FORK",
1424 "CSW-IN",
1425 "CSW-OUT",
1426 "SAMPLE"
1427};
1428#endif
1429
1430static int
1431pmc_hook_handler(struct thread *td, int function, void *arg)
1432{
1433
1434 PMCDBG(MOD,PMH,1, "hook td=%p func=%d \"%s\" arg=%p", td, function,
1435 pmc_hooknames[function], arg);
1436
1437 switch (function)
1438 {
1439
1440 /*
1441 * Process exec()
1442 */
1443
1444 case PMC_FN_PROCESS_EXEC:
1445 {
1446 char *fullpath, *freepath;
1447 unsigned int ri;
1448 int is_using_hwpmcs;
1449 struct pmc *pm;
1450 struct proc *p;
1451 struct pmc_owner *po;
1452 struct pmc_process *pp;
1453 struct pmckern_procexec *pk;
1454
1455 sx_assert(&pmc_sx, SX_XLOCKED);
1456
1457 p = td->td_proc;
1458 pmc_getfilename(p->p_textvp, &fullpath, &freepath);
1459
1460 pk = (struct pmckern_procexec *) arg;
1461
1462 /* Inform owners of SS mode PMCs of the exec event. */
1463 LIST_FOREACH(po, &pmc_ss_owners, po_ssnext)
1464 if (po->po_flags & PMC_PO_OWNS_LOGFILE)
1465 pmclog_process_procexec(po, PMC_ID_INVALID,
1466 p->p_pid, pk->pm_entryaddr, fullpath);
1467
1468 PROC_LOCK(p);
1469 is_using_hwpmcs = p->p_flag & P_HWPMC;
1470 PROC_UNLOCK(p);
1471
1472 if (!is_using_hwpmcs) {
1473 if (freepath)
1474 FREE(freepath, M_TEMP);
1475 break;
1476 }
1477
1478 /*
1479 * PMCs are not inherited across an exec(): remove any
1480 * PMCs that this process is the owner of.
1481 */
1482
1483 if ((po = pmc_find_owner_descriptor(p)) != NULL) {
1484 pmc_remove_owner(po);
1485 pmc_destroy_owner_descriptor(po);
1486 }
1487
1488 /*
1489 * If this process is the target of a PMC, check if the new
1490 * credentials are compatible with the owner's permissions.
1491 */
1492
1493 if ((pp = pmc_find_process_descriptor(p, 0)) == NULL)
1494 break;
1495
1496 /*
1497 * Log the exec event to all monitoring owners. Skip
1498 * owners who have already recieved the event because
1499 * the have system sampling PMCs active.
1500 */
1501 for (ri = 0; ri < md->pmd_npmc; ri++)
1502 if ((pm = pp->pp_pmcs[ri].pp_pmc) != NULL) {
1503 po = pm->pm_owner;
1504 if (po->po_sscount == 0 &&
1505 po->po_flags & PMC_PO_OWNS_LOGFILE)
1506 pmclog_process_procexec(po, pm->pm_id,
1507 p->p_pid, pk->pm_entryaddr,
1508 fullpath);
1509 }
1510
1511 if (freepath)
1512 FREE(freepath, M_TEMP);
1513
1514
1515 PMCDBG(PRC,EXC,1, "exec proc=%p (%d, %s) cred-changed=%d",
1516 p, p->p_pid, p->p_comm, pk->pm_credentialschanged);
1517
1518 if (pk->pm_credentialschanged == 0) /* no change */
1519 break;
1520
1521 /*
1522 * If the newly exec()'ed process has a different credential
1523 * than before, allow it to be the target of a PMC only if
1524 * the PMC's owner has sufficient priviledge.
1525 */
1526
1527 for (ri = 0; ri < md->pmd_npmc; ri++)
1528 if ((pm = pp->pp_pmcs[ri].pp_pmc) != NULL)
1529 if (pmc_can_attach(pm, td->td_proc) != 0)
1530 pmc_detach_one_process(td->td_proc,
1531 pm, PMC_FLAG_NONE);
1532
1533 KASSERT(pp->pp_refcnt >= 0 && pp->pp_refcnt < (int) md->pmd_npmc,
1534 ("[pmc,%d] Illegal ref count %d on pp %p", __LINE__,
1535 pp->pp_refcnt, pp));
1536
1537 /*
1538 * If this process is no longer the target of any
1539 * PMCs, we can remove the process entry and free
1540 * up space.
1541 */
1542
1543 if (pp->pp_refcnt == 0) {
1544 pmc_remove_process_descriptor(pp);
1545 FREE(pp, M_PMC);
1546 break;
1547 }
1548
1549 }
1550 break;
1551
1552 case PMC_FN_CSW_IN:
1553 pmc_process_csw_in(td);
1554 break;
1555
1556 case PMC_FN_CSW_OUT:
1557 pmc_process_csw_out(td);
1558 break;
1559
1560 /*
1561 * Process accumulated PC samples.
1562 *
1563 * This function is expected to be called by hardclock() for
1564 * each CPU that has accumulated PC samples.
1565 *
1566 * This function is to be executed on the CPU whose samples
1567 * are being processed.
1568 */
1569 case PMC_FN_DO_SAMPLES:
1570
1571 /*
1572 * Clear the cpu specific bit in the CPU mask before
1573 * do the rest of the processing. If the NMI handler
1574 * gets invoked after the "atomic_clear_int()" call
1575 * below but before "pmc_process_samples()" gets
1576 * around to processing the interrupt, then we will
1577 * come back here at the next hardclock() tick (and
1578 * may find nothing to do if "pmc_process_samples()"
1579 * had already processed the interrupt). We don't
1580 * lose the interrupt sample.
1581 */
1582 atomic_clear_int(&pmc_cpumask, (1 << PCPU_GET(cpuid)));
1583 pmc_process_samples(PCPU_GET(cpuid));
1584 break;
1585
1586 default:
1587#if DEBUG
1588 KASSERT(0, ("[pmc,%d] unknown hook %d\n", __LINE__, function));
1589#endif
1590 break;
1591
1592 }
1593
1594 return 0;
1595}
1596
1597/*
1598 * allocate a 'struct pmc_owner' descriptor in the owner hash table.
1599 */
1600
1601static struct pmc_owner *
1602pmc_allocate_owner_descriptor(struct proc *p)
1603{
1604 uint32_t hindex;
1605 struct pmc_owner *po;
1606 struct pmc_ownerhash *poh;
1607
1608 hindex = PMC_HASH_PTR(p, pmc_ownerhashmask);
1609 poh = &pmc_ownerhash[hindex];
1610
1611 /* allocate space for N pointers and one descriptor struct */
1612 MALLOC(po, struct pmc_owner *, sizeof(struct pmc_owner),
1613 M_PMC, M_ZERO|M_WAITOK);
1614
1615 po->po_sscount = po->po_error = po->po_flags = 0;
1616 po->po_file = NULL;
1617 po->po_owner = p;
1618 po->po_kthread = NULL;
1619 LIST_INIT(&po->po_pmcs);
1620 LIST_INSERT_HEAD(poh, po, po_next); /* insert into hash table */
1621
1622 TAILQ_INIT(&po->po_logbuffers);
1623 mtx_init(&po->po_mtx, "pmc-owner-mtx", "pmc", MTX_SPIN);
1624
1625 PMCDBG(OWN,ALL,1, "allocate-owner proc=%p (%d, %s) pmc-owner=%p",
1626 p, p->p_pid, p->p_comm, po);
1627
1628 return po;
1629}
1630
1631static void
1632pmc_destroy_owner_descriptor(struct pmc_owner *po)
1633{
1634
1635 PMCDBG(OWN,REL,1, "destroy-owner po=%p proc=%p (%d, %s)",
1636 po, po->po_owner, po->po_owner->p_pid, po->po_owner->p_comm);
1637
1638 mtx_destroy(&po->po_mtx);
1639 FREE(po, M_PMC);
1640}
1641
1642/*
1643 * find the descriptor corresponding to process 'p', adding or removing it
1644 * as specified by 'mode'.
1645 */
1646
1647static struct pmc_process *
1648pmc_find_process_descriptor(struct proc *p, uint32_t mode)
1649{
1650 uint32_t hindex;
1651 struct pmc_process *pp, *ppnew;
1652 struct pmc_processhash *pph;
1653
1654 hindex = PMC_HASH_PTR(p, pmc_processhashmask);
1655 pph = &pmc_processhash[hindex];
1656
1657 ppnew = NULL;
1658
1659 /*
1660 * Pre-allocate memory in the FIND_ALLOCATE case since we
1661 * cannot call malloc(9) once we hold a spin lock.
1662 */
1663
1664 if (mode & PMC_FLAG_ALLOCATE) {
1665 /* allocate additional space for 'n' pmc pointers */
1666 MALLOC(ppnew, struct pmc_process *,
1667 sizeof(struct pmc_process) + md->pmd_npmc *
1668 sizeof(struct pmc_targetstate), M_PMC, M_ZERO|M_WAITOK);
1669 }
1670
1671 mtx_lock_spin(&pmc_processhash_mtx);
1672 LIST_FOREACH(pp, pph, pp_next)
1673 if (pp->pp_proc == p)
1674 break;
1675
1676 if ((mode & PMC_FLAG_REMOVE) && pp != NULL)
1677 LIST_REMOVE(pp, pp_next);
1678
1679 if ((mode & PMC_FLAG_ALLOCATE) && pp == NULL &&
1680 ppnew != NULL) {
1681 ppnew->pp_proc = p;
1682 LIST_INSERT_HEAD(pph, ppnew, pp_next);
1683 pp = ppnew;
1684 ppnew = NULL;
1685 }
1686 mtx_unlock_spin(&pmc_processhash_mtx);
1687
1688 if (pp != NULL && ppnew != NULL)
1689 FREE(ppnew, M_PMC);
1690
1691 return pp;
1692}
1693
1694/*
1695 * remove a process descriptor from the process hash table.
1696 */
1697
1698static void
1699pmc_remove_process_descriptor(struct pmc_process *pp)
1700{
1701 KASSERT(pp->pp_refcnt == 0,
1702 ("[pmc,%d] Removing process descriptor %p with count %d",
1703 __LINE__, pp, pp->pp_refcnt));
1704
1705 mtx_lock_spin(&pmc_processhash_mtx);
1706 LIST_REMOVE(pp, pp_next);
1707 mtx_unlock_spin(&pmc_processhash_mtx);
1708}
1709
1710
1711/*
1712 * find an owner descriptor corresponding to proc 'p'
1713 */
1714
1715static struct pmc_owner *
1716pmc_find_owner_descriptor(struct proc *p)
1717{
1718 uint32_t hindex;
1719 struct pmc_owner *po;
1720 struct pmc_ownerhash *poh;
1721
1722 hindex = PMC_HASH_PTR(p, pmc_ownerhashmask);
1723 poh = &pmc_ownerhash[hindex];
1724
1725 po = NULL;
1726 LIST_FOREACH(po, poh, po_next)
1727 if (po->po_owner == p)
1728 break;
1729
1730 PMCDBG(OWN,FND,1, "find-owner proc=%p (%d, %s) hindex=0x%x -> "
1731 "pmc-owner=%p", p, p->p_pid, p->p_comm, hindex, po);
1732
1733 return po;
1734}
1735
1736/*
1737 * pmc_allocate_pmc_descriptor
1738 *
1739 * Allocate a pmc descriptor and initialize its
1740 * fields.
1741 */
1742
1743static struct pmc *
1744pmc_allocate_pmc_descriptor(void)
1745{
1746 struct pmc *pmc;
1747
1748 MALLOC(pmc, struct pmc *, sizeof(struct pmc), M_PMC, M_ZERO|M_WAITOK);
1749
1750 if (pmc != NULL) {
1751 pmc->pm_owner = NULL;
1752 LIST_INIT(&pmc->pm_targets);
1753 }
1754
1755 PMCDBG(PMC,ALL,1, "allocate-pmc -> pmc=%p", pmc);
1756
1757 return pmc;
1758}
1759
1760/*
1761 * Destroy a pmc descriptor.
1762 */
1763
1764static void
1765pmc_destroy_pmc_descriptor(struct pmc *pm)
1766{
1767 (void) pm;
1768
1769#if DEBUG
1770 KASSERT(pm->pm_state == PMC_STATE_DELETED ||
1771 pm->pm_state == PMC_STATE_FREE,
1772 ("[pmc,%d] destroying non-deleted PMC", __LINE__));
1773 KASSERT(LIST_EMPTY(&pm->pm_targets),
1774 ("[pmc,%d] destroying pmc with targets", __LINE__));
1775 KASSERT(pm->pm_owner == NULL,
1776 ("[pmc,%d] destroying pmc attached to an owner", __LINE__));
1777 KASSERT(pm->pm_runcount == 0,
1778 ("[pmc,%d] pmc has non-zero run count %d", __LINE__,
1779 pm->pm_runcount));
1780#endif
1781}
1782
1783static void
1784pmc_wait_for_pmc_idle(struct pmc *pm)
1785{
1786#if DEBUG
1787 volatile int maxloop;
1788
1789 maxloop = 100 * mp_ncpus;
1790#endif
1791
1792 /*
1793 * Loop (with a forced context switch) till the PMC's runcount
1794 * comes down to zero.
1795 */
1796 while (atomic_load_acq_32(&pm->pm_runcount) > 0) {
1797#if DEBUG
1798 maxloop--;
1799 KASSERT(maxloop > 0,
1800 ("[pmc,%d] (ri%d, rc%d) waiting too long for "
1801 "pmc to be free", __LINE__,
1802 PMC_TO_ROWINDEX(pm), pm->pm_runcount));
1803#endif
1804 pmc_force_context_switch();
1805 }
1806}
1807
1808/*
1809 * This function does the following things:
1810 *
1811 * - detaches the PMC from hardware
1812 * - unlinks all target threads that were attached to it
1813 * - removes the PMC from its owner's list
1814 * - destroy's the PMC private mutex
1815 *
1816 * Once this function completes, the given pmc pointer can be safely
1817 * FREE'd by the caller.
1818 */
1819
1820static void
1821pmc_release_pmc_descriptor(struct pmc *pm)
1822{
1823 u_int ri, cpu;
1824 enum pmc_mode mode;
1825 struct pmc_hw *phw;
1826 struct pmc_owner *po;
1827 struct pmc_process *pp;
1828 struct pmc_target *ptgt, *tmp;
1829 struct pmc_binding pb;
1830
1831 sx_assert(&pmc_sx, SX_XLOCKED);
1832
1833 KASSERT(pm, ("[pmc,%d] null pmc", __LINE__));
1834
1835 ri = PMC_TO_ROWINDEX(pm);
1836 mode = PMC_TO_MODE(pm);
1837
1838 PMCDBG(PMC,REL,1, "release-pmc pmc=%p ri=%d mode=%d", pm, ri,
1839 mode);
1840
1841 /*
1842 * First, we take the PMC off hardware.
1843 */
1844 cpu = 0;
1845 if (PMC_IS_SYSTEM_MODE(mode)) {
1846
1847 /*
1848 * A system mode PMC runs on a specific CPU. Switch
1849 * to this CPU and turn hardware off.
1850 */
1851 pmc_save_cpu_binding(&pb);
1852
1853 cpu = PMC_TO_CPU(pm);
1854
1855 pmc_select_cpu(cpu);
1856
1857 /* switch off non-stalled CPUs */
1858 if (pm->pm_state == PMC_STATE_RUNNING &&
1859 pm->pm_stalled == 0) {
1860
1861 phw = pmc_pcpu[cpu]->pc_hwpmcs[ri];
1862
1863 KASSERT(phw->phw_pmc == pm,
1864 ("[pmc, %d] pmc ptr ri(%d) hw(%p) pm(%p)",
1865 __LINE__, ri, phw->phw_pmc, pm));
1866 PMCDBG(PMC,REL,2, "stopping cpu=%d ri=%d", cpu, ri);
1867
1868 critical_enter();
1869 md->pmd_stop_pmc(cpu, ri);
1870 critical_exit();
1871 }
1872
1873 PMCDBG(PMC,REL,2, "decfg cpu=%d ri=%d", cpu, ri);
1874
1875 critical_enter();
1876 md->pmd_config_pmc(cpu, ri, NULL);
1877 critical_exit();
1878
1879 /* adjust the global and process count of SS mode PMCs */
1880 if (mode == PMC_MODE_SS && pm->pm_state == PMC_STATE_RUNNING) {
1881 po = pm->pm_owner;
1882 po->po_sscount--;
1883 if (po->po_sscount == 0) {
1884 atomic_subtract_rel_int(&pmc_ss_count, 1);
1885 LIST_REMOVE(po, po_ssnext);
1886 }
1887 }
1888
1889 pm->pm_state = PMC_STATE_DELETED;
1890
1891 pmc_restore_cpu_binding(&pb);
1892
1893 /*
1894 * We could have references to this PMC structure in
1895 * the per-cpu sample queues. Wait for the queue to
1896 * drain.
1897 */
1898 pmc_wait_for_pmc_idle(pm);
1899
1900 } else if (PMC_IS_VIRTUAL_MODE(mode)) {
1901
1902 /*
1903 * A virtual PMC could be running on multiple CPUs at
1904 * a given instant.
1905 *
1906 * By marking its state as DELETED, we ensure that
1907 * this PMC is never further scheduled on hardware.
1908 *
1909 * Then we wait till all CPUs are done with this PMC.
1910 */
1911 pm->pm_state = PMC_STATE_DELETED;
1912
1913
1914 /* Wait for the PMCs runcount to come to zero. */
1915 pmc_wait_for_pmc_idle(pm);
1916
1917 /*
1918 * At this point the PMC is off all CPUs and cannot be
1919 * freshly scheduled onto a CPU. It is now safe to
1920 * unlink all targets from this PMC. If a
1921 * process-record's refcount falls to zero, we remove
1922 * it from the hash table. The module-wide SX lock
1923 * protects us from races.
1924 */
1925 LIST_FOREACH_SAFE(ptgt, &pm->pm_targets, pt_next, tmp) {
1926 pp = ptgt->pt_process;
1927 pmc_unlink_target_process(pm, pp); /* frees 'ptgt' */
1928
1929 PMCDBG(PMC,REL,3, "pp->refcnt=%d", pp->pp_refcnt);
1930
1931 /*
1932 * If the target process record shows that no
1933 * PMCs are attached to it, reclaim its space.
1934 */
1935
1936 if (pp->pp_refcnt == 0) {
1937 pmc_remove_process_descriptor(pp);
1938 FREE(pp, M_PMC);
1939 }
1940 }
1941
1942 cpu = curthread->td_oncpu; /* setup cpu for pmd_release() */
1943
1944 }
1945
1946 /*
1947 * Release any MD resources
1948 */
1949
1950 (void) md->pmd_release_pmc(cpu, ri, pm);
1951
1952 /*
1953 * Update row disposition
1954 */
1955
1956 if (PMC_IS_SYSTEM_MODE(PMC_TO_MODE(pm)))
1957 PMC_UNMARK_ROW_STANDALONE(ri);
1958 else
1959 PMC_UNMARK_ROW_THREAD(ri);
1960
1961 /* unlink from the owner's list */
1962 if (pm->pm_owner) {
1963 LIST_REMOVE(pm, pm_next);
1964 pm->pm_owner = NULL;
1965 }
1966
1967 pmc_destroy_pmc_descriptor(pm);
1968}
1969
1970/*
1971 * Register an owner and a pmc.
1972 */
1973
1974static int
1975pmc_register_owner(struct proc *p, struct pmc *pmc)
1976{
1977 struct pmc_owner *po;
1978
1979 sx_assert(&pmc_sx, SX_XLOCKED);
1980
1981 if ((po = pmc_find_owner_descriptor(p)) == NULL)
1982 if ((po = pmc_allocate_owner_descriptor(p)) == NULL)
1983 return ENOMEM;
1984
1985 KASSERT(pmc->pm_owner == NULL,
1986 ("[pmc,%d] attempting to own an initialized PMC", __LINE__));
1987 pmc->pm_owner = po;
1988
1989 LIST_INSERT_HEAD(&po->po_pmcs, pmc, pm_next);
1990
1991 PROC_LOCK(p);
1992 p->p_flag |= P_HWPMC;
1993 PROC_UNLOCK(p);
1994
1995 if (po->po_flags & PMC_PO_OWNS_LOGFILE)
1996 pmclog_process_pmcallocate(pmc);
1997
1998 PMCDBG(PMC,REG,1, "register-owner pmc-owner=%p pmc=%p",
1999 po, pmc);
2000
2001 return 0;
2002}
2003
2004/*
2005 * Return the current row disposition:
2006 * == 0 => FREE
2007 * > 0 => PROCESS MODE
2008 * < 0 => SYSTEM MODE
2009 */
2010
2011int
2012pmc_getrowdisp(int ri)
2013{
2014 return pmc_pmcdisp[ri];
2015}
2016
2017/*
2018 * Check if a PMC at row index 'ri' can be allocated to the current
2019 * process.
2020 *
2021 * Allocation can fail if:
2022 * - the current process is already being profiled by a PMC at index 'ri',
2023 * attached to it via OP_PMCATTACH.
2024 * - the current process has already allocated a PMC at index 'ri'
2025 * via OP_ALLOCATE.
2026 */
2027
2028static int
2029pmc_can_allocate_rowindex(struct proc *p, unsigned int ri, int cpu)
2030{
2031 enum pmc_mode mode;
2032 struct pmc *pm;
2033 struct pmc_owner *po;
2034 struct pmc_process *pp;
2035
2036 PMCDBG(PMC,ALR,1, "can-allocate-rowindex proc=%p (%d, %s) ri=%d "
2037 "cpu=%d", p, p->p_pid, p->p_comm, ri, cpu);
2038
2039 /*
2040 * We shouldn't have already allocated a process-mode PMC at
2041 * row index 'ri'.
2042 *
2043 * We shouldn't have allocated a system-wide PMC on the same
2044 * CPU and same RI.
2045 */
2046 if ((po = pmc_find_owner_descriptor(p)) != NULL)
2047 LIST_FOREACH(pm, &po->po_pmcs, pm_next) {
2048 if (PMC_TO_ROWINDEX(pm) == ri) {
2049 mode = PMC_TO_MODE(pm);
2050 if (PMC_IS_VIRTUAL_MODE(mode))
2051 return EEXIST;
2052 if (PMC_IS_SYSTEM_MODE(mode) &&
2053 (int) PMC_TO_CPU(pm) == cpu)
2054 return EEXIST;
2055 }
2056 }
2057
2058 /*
2059 * We also shouldn't be the target of any PMC at this index
2060 * since otherwise a PMC_ATTACH to ourselves will fail.
2061 */
2062 if ((pp = pmc_find_process_descriptor(p, 0)) != NULL)
2063 if (pp->pp_pmcs[ri].pp_pmc)
2064 return EEXIST;
2065
2066 PMCDBG(PMC,ALR,2, "can-allocate-rowindex proc=%p (%d, %s) ri=%d ok",
2067 p, p->p_pid, p->p_comm, ri);
2068
2069 return 0;
2070}
2071
2072/*
2073 * Check if a given PMC at row index 'ri' can be currently used in
2074 * mode 'mode'.
2075 */
2076
2077static int
2078pmc_can_allocate_row(int ri, enum pmc_mode mode)
2079{
2080 enum pmc_disp disp;
2081
2082 sx_assert(&pmc_sx, SX_XLOCKED);
2083
2084 PMCDBG(PMC,ALR,1, "can-allocate-row ri=%d mode=%d", ri, mode);
2085
2086 if (PMC_IS_SYSTEM_MODE(mode))
2087 disp = PMC_DISP_STANDALONE;
2088 else
2089 disp = PMC_DISP_THREAD;
2090
2091 /*
2092 * check disposition for PMC row 'ri':
2093 *
2094 * Expected disposition Row-disposition Result
2095 *
2096 * STANDALONE STANDALONE or FREE proceed
2097 * STANDALONE THREAD fail
2098 * THREAD THREAD or FREE proceed
2099 * THREAD STANDALONE fail
2100 */
2101
2102 if (!PMC_ROW_DISP_IS_FREE(ri) &&
2103 !(disp == PMC_DISP_THREAD && PMC_ROW_DISP_IS_THREAD(ri)) &&
2104 !(disp == PMC_DISP_STANDALONE && PMC_ROW_DISP_IS_STANDALONE(ri)))
2105 return EBUSY;
2106
2107 /*
2108 * All OK
2109 */
2110
2111 PMCDBG(PMC,ALR,2, "can-allocate-row ri=%d mode=%d ok", ri, mode);
2112
2113 return 0;
2114
2115}
2116
2117/*
2118 * Find a PMC descriptor with user handle 'pmcid' for thread 'td'.
2119 */
2120
2121static struct pmc *
2122pmc_find_pmc_descriptor_in_process(struct pmc_owner *po, pmc_id_t pmcid)
2123{
2124 struct pmc *pm;
2125
2126 KASSERT(PMC_ID_TO_ROWINDEX(pmcid) < md->pmd_npmc,
2127 ("[pmc,%d] Illegal pmc index %d (max %d)", __LINE__,
2128 PMC_ID_TO_ROWINDEX(pmcid), md->pmd_npmc));
2129
2130 LIST_FOREACH(pm, &po->po_pmcs, pm_next)
2131 if (pm->pm_id == pmcid)
2132 return pm;
2133
2134 return NULL;
2135}
2136
2137static int
2138pmc_find_pmc(pmc_id_t pmcid, struct pmc **pmc)
2139{
2140
2141 struct pmc *pm;
2142 struct pmc_owner *po;
2143
2144 PMCDBG(PMC,FND,1, "find-pmc id=%d", pmcid);
2145
2146 if ((po = pmc_find_owner_descriptor(curthread->td_proc)) == NULL)
2147 return ESRCH;
2148
2149 if ((pm = pmc_find_pmc_descriptor_in_process(po, pmcid)) == NULL)
2150 return EINVAL;
2151
2152 PMCDBG(PMC,FND,2, "find-pmc id=%d -> pmc=%p", pmcid, pm);
2153
2154 *pmc = pm;
2155 return 0;
2156}
2157
2158/*
2159 * Start a PMC.
2160 */
2161
2162static int
2163pmc_start(struct pmc *pm)
2164{
2165 int error, cpu, ri;
2166 enum pmc_mode mode;
2167 struct pmc_owner *po;
2168 struct pmc_binding pb;
2169
2170 KASSERT(pm != NULL,
2171 ("[pmc,%d] null pm", __LINE__));
2172
2173 mode = PMC_TO_MODE(pm);
2174 ri = PMC_TO_ROWINDEX(pm);
2175 error = 0;
2176
2177 PMCDBG(PMC,OPS,1, "start pmc=%p mode=%d ri=%d", pm, mode, ri);
2178
2179 po = pm->pm_owner;
2180
2181 if (PMC_IS_VIRTUAL_MODE(mode)) {
2182
2183 /*
2184 * If a PMCATTACH has never been done on this PMC,
2185 * attach it to its owner process.
2186 */
2187
2188 if (LIST_EMPTY(&pm->pm_targets))
2189 error = (pm->pm_flags & PMC_F_ATTACH_DONE) ? ESRCH :
2190 pmc_attach_process(po->po_owner, pm);
2191
2192 /*
2193 * Disallow PMCSTART if a logfile is required but has not
2194 * been configured yet.
2195 */
2196
2197 if (error == 0 && (pm->pm_flags & PMC_F_NEEDS_LOGFILE) &&
2198 (po->po_flags & PMC_PO_OWNS_LOGFILE) == 0)
2199 error = EDOOFUS;
2200
2201 /*
2202 * If the PMC is attached to its owner, then force a context
2203 * switch to ensure that the MD state gets set correctly.
2204 */
2205
2206 if (error == 0) {
2207 pm->pm_state = PMC_STATE_RUNNING;
2208 if (pm->pm_flags & PMC_F_ATTACHED_TO_OWNER)
2209 pmc_force_context_switch();
2210 }
2211
2212 return error;
2213 }
2214
2215
2216 /*
2217 * A system-wide PMC.
2218 */
2219
2220 if ((pm->pm_flags & PMC_F_NEEDS_LOGFILE) &&
2221 (po->po_flags & PMC_PO_OWNS_LOGFILE) == 0)
2222 return EDOOFUS; /* programming error */
2223
2224 /*
2225 * Add the owner to the global list if this is a system-wide
2226 * sampling PMC.
2227 */
2228
2229 if (mode == PMC_MODE_SS) {
2230 if (po->po_sscount == 0) {
2231 LIST_INSERT_HEAD(&pmc_ss_owners, po, po_ssnext);
2232 atomic_add_rel_int(&pmc_ss_count, 1);
2233 PMCDBG(PMC,OPS,1, "po=%p in global list", po);
2234 }
2235 po->po_sscount++;
2236 }
2237
2238 /*
2239 * Move to the CPU associated with this
2240 * PMC, and start the hardware.
2241 */
2242
2243 pmc_save_cpu_binding(&pb);
2244
2245 cpu = PMC_TO_CPU(pm);
2246
2247 if (pmc_cpu_is_disabled(cpu))
2248 return ENXIO;
2249
2250 pmc_select_cpu(cpu);
2251
2252 /*
2253 * global PMCs are configured at allocation time
2254 * so write out the initial value and start the PMC.
2255 */
2256
2257 pm->pm_state = PMC_STATE_RUNNING;
2258
2259 critical_enter();
2260 if ((error = md->pmd_write_pmc(cpu, ri,
2261 PMC_IS_SAMPLING_MODE(mode) ?
2262 pm->pm_sc.pm_reloadcount :
2263 pm->pm_sc.pm_initial)) == 0)
2264 error = md->pmd_start_pmc(cpu, ri);
2265 critical_exit();
2266
2267 pmc_restore_cpu_binding(&pb);
2268
2269 return error;
2270}
2271
2272/*
2273 * Stop a PMC.
2274 */
2275
2276static int
2277pmc_stop(struct pmc *pm)
2278{
2279 int cpu, error, ri;
2280 struct pmc_owner *po;
2281 struct pmc_binding pb;
2282
2283 KASSERT(pm != NULL, ("[pmc,%d] null pmc", __LINE__));
2284
2285 PMCDBG(PMC,OPS,1, "stop pmc=%p mode=%d ri=%d", pm,
2286 PMC_TO_MODE(pm), PMC_TO_ROWINDEX(pm));
2287
2288 pm->pm_state = PMC_STATE_STOPPED;
2289
2290 /*
2291 * If the PMC is a virtual mode one, changing the state to
2292 * non-RUNNING is enough to ensure that the PMC never gets
2293 * scheduled.
2294 *
2295 * If this PMC is current running on a CPU, then it will
2296 * handled correctly at the time its target process is context
2297 * switched out.
2298 */
2299
2300 if (PMC_IS_VIRTUAL_MODE(PMC_TO_MODE(pm)))
2301 return 0;
2302
2303 /*
2304 * A system-mode PMC. Move to the CPU associated with
2305 * this PMC, and stop the hardware. We update the
2306 * 'initial count' so that a subsequent PMCSTART will
2307 * resume counting from the current hardware count.
2308 */
2309
2310 pmc_save_cpu_binding(&pb);
2311
2312 cpu = PMC_TO_CPU(pm);
2313
2314 KASSERT(cpu >= 0 && cpu < mp_ncpus,
2315 ("[pmc,%d] illegal cpu=%d", __LINE__, cpu));
2316
2317 if (pmc_cpu_is_disabled(cpu))
2318 return ENXIO;
2319
2320 pmc_select_cpu(cpu);
2321
2322 ri = PMC_TO_ROWINDEX(pm);
2323
2324 critical_enter();
2325 if ((error = md->pmd_stop_pmc(cpu, ri)) == 0)
2326 error = md->pmd_read_pmc(cpu, ri, &pm->pm_sc.pm_initial);
2327 critical_exit();
2328
2329 pmc_restore_cpu_binding(&pb);
2330
2331 po = pm->pm_owner;
2332
2333 /* remove this owner from the global list of SS PMC owners */
2334 if (PMC_TO_MODE(pm) == PMC_MODE_SS) {
2335 po->po_sscount--;
2336 if (po->po_sscount == 0) {
2337 atomic_subtract_rel_int(&pmc_ss_count, 1);
2338 LIST_REMOVE(po, po_ssnext);
2339 PMCDBG(PMC,OPS,2,"po=%p removed from global list", po);
2340 }
2341 }
2342
2343 return error;
2344}
2345
2346
2347#if DEBUG
2348static const char *pmc_op_to_name[] = {
2349#undef __PMC_OP
2350#define __PMC_OP(N, D) #N ,
2351 __PMC_OPS()
2352 NULL
2353};
2354#endif
2355
2356/*
2357 * The syscall interface
2358 */
2359
2360#define PMC_GET_SX_XLOCK(...) do { \
2361 sx_xlock(&pmc_sx); \
2362 if (pmc_hook == NULL) { \
2363 sx_xunlock(&pmc_sx); \
2364 return __VA_ARGS__; \
2365 } \
2366} while (0)
2367
2368#define PMC_DOWNGRADE_SX() do { \
2369 sx_downgrade(&pmc_sx); \
2370 is_sx_downgraded = 1; \
2371} while (0)
2372
2373static int
2374pmc_syscall_handler(struct thread *td, void *syscall_args)
2375{
2376 int error, is_sx_downgraded, op;
2377 struct pmc_syscall_args *c;
2378 void *arg;
2379
2380 PMC_GET_SX_XLOCK(ENOSYS);
2381
2382 DROP_GIANT();
2383
2384 is_sx_downgraded = 0;
2385
2386 c = (struct pmc_syscall_args *) syscall_args;
2387
2388 op = c->pmop_code;
2389 arg = c->pmop_data;
2390
2391 PMCDBG(MOD,PMS,1, "syscall op=%d \"%s\" arg=%p", op,
2392 pmc_op_to_name[op], arg);
2393
2394 error = 0;
2395 atomic_add_int(&pmc_stats.pm_syscalls, 1);
2396
2397 switch(op)
2398 {
2399
2400
2401 /*
2402 * Configure a log file.
2403 *
2404 * XXX This OP will be reworked.
2405 */
2406
2407 case PMC_OP_CONFIGURELOG:
2408 {
2409 struct pmc_owner *po;
2410 struct pmc_op_configurelog cl;
2411 struct proc *p;
2412
2413 sx_assert(&pmc_sx, SX_XLOCKED);
2414
2415 if ((error = copyin(arg, &cl, sizeof(cl))) != 0)
2416 break;
2417
2418 /* mark this process as owning a log file */
2419 p = td->td_proc;
2420 if ((po = pmc_find_owner_descriptor(p)) == NULL)
2421 if ((po = pmc_allocate_owner_descriptor(p)) == NULL) {
2422 error = ENOMEM;
2423 break;
2424 }
2425
2426 /*
2427 * If a valid fd was passed in, try to configure that,
2428 * otherwise if 'fd' was less than zero and there was
2429 * a log file configured, flush its buffers and
2430 * de-configure it.
2431 */
2432 if (cl.pm_logfd >= 0)
2433 error = pmclog_configure_log(po, cl.pm_logfd);
2434 else if (po->po_flags & PMC_PO_OWNS_LOGFILE) {
2435 pmclog_process_closelog(po);
2436 error = pmclog_flush(po);
2437 if (error == 0)
2438 error = pmclog_deconfigure_log(po);
2439 } else
2440 error = EINVAL;
2441 }
2442 break;
2443
2444
2445 /*
2446 * Flush a log file.
2447 */
2448
2449 case PMC_OP_FLUSHLOG:
2450 {
2451 struct pmc_owner *po;
2452
2453 sx_assert(&pmc_sx, SX_XLOCKED);
2454
2455 if ((po = pmc_find_owner_descriptor(td->td_proc)) == NULL) {
2456 error = EINVAL;
2457 break;
2458 }
2459
2460 error = pmclog_flush(po);
2461 }
2462 break;
2463
2464 /*
2465 * Retrieve hardware configuration.
2466 */
2467
2468 case PMC_OP_GETCPUINFO: /* CPU information */
2469 {
2470 struct pmc_op_getcpuinfo gci;
2471
2472 gci.pm_cputype = md->pmd_cputype;
2473 gci.pm_ncpu = mp_ncpus;
2474 gci.pm_npmc = md->pmd_npmc;
2475 gci.pm_nclass = md->pmd_nclass;
2476 bcopy(md->pmd_classes, &gci.pm_classes,
2477 sizeof(gci.pm_classes));
2478 error = copyout(&gci, arg, sizeof(gci));
2479 }
2480 break;
2481
2482
2483 /*
2484 * Get module statistics
2485 */
2486
2487 case PMC_OP_GETDRIVERSTATS:
2488 {
2489 struct pmc_op_getdriverstats gms;
2490
2491 bcopy(&pmc_stats, &gms, sizeof(gms));
2492 error = copyout(&gms, arg, sizeof(gms));
2493 }
2494 break;
2495
2496
2497 /*
2498 * Retrieve module version number
2499 */
2500
2501 case PMC_OP_GETMODULEVERSION:
2502 {
2503 uint32_t cv, modv;
2504
2505 /* retrieve the client's idea of the ABI version */
2506 if ((error = copyin(arg, &cv, sizeof(uint32_t))) != 0)
2507 break;
2508 /* don't service clients newer than our driver */
2509 modv = PMC_VERSION;
2510 if ((cv & 0xFFFF0000) > (modv & 0xFFFF0000)) {
2511 error = EPROGMISMATCH;
2512 break;
2513 }
2514 error = copyout(&modv, arg, sizeof(int));
2515 }
2516 break;
2517
2518
2519 /*
2520 * Retrieve the state of all the PMCs on a given
2521 * CPU.
2522 */
2523
2524 case PMC_OP_GETPMCINFO:
2525 {
2526 uint32_t cpu, n, npmc;
2527 size_t pmcinfo_size;
2528 struct pmc *pm;
2529 struct pmc_info *p, *pmcinfo;
2530 struct pmc_op_getpmcinfo *gpi;
2531 struct pmc_owner *po;
2532 struct pmc_binding pb;
2533
2534 PMC_DOWNGRADE_SX();
2535
2536 gpi = (struct pmc_op_getpmcinfo *) arg;
2537
2538 if ((error = copyin(&gpi->pm_cpu, &cpu, sizeof(cpu))) != 0)
2539 break;
2540
2541 if (cpu >= (unsigned int) mp_ncpus) {
2542 error = EINVAL;
2543 break;
2544 }
2545
2546 if (pmc_cpu_is_disabled(cpu)) {
2547 error = ENXIO;
2548 break;
2549 }
2550
2551 /* switch to CPU 'cpu' */
2552 pmc_save_cpu_binding(&pb);
2553 pmc_select_cpu(cpu);
2554
2555 npmc = md->pmd_npmc;
2556
2557 pmcinfo_size = npmc * sizeof(struct pmc_info);
2558 MALLOC(pmcinfo, struct pmc_info *, pmcinfo_size, M_PMC,
2559 M_WAITOK);
2560
2561 p = pmcinfo;
2562
2563 for (n = 0; n < md->pmd_npmc; n++, p++) {
2564
2565 if ((error = md->pmd_describe(cpu, n, p, &pm)) != 0)
2566 break;
2567
2568 if (PMC_ROW_DISP_IS_STANDALONE(n))
2569 p->pm_rowdisp = PMC_DISP_STANDALONE;
2570 else if (PMC_ROW_DISP_IS_THREAD(n))
2571 p->pm_rowdisp = PMC_DISP_THREAD;
2572 else
2573 p->pm_rowdisp = PMC_DISP_FREE;
2574
2575 p->pm_ownerpid = -1;
2576
2577 if (pm == NULL) /* no PMC associated */
2578 continue;
2579
2580 po = pm->pm_owner;
2581
2582 KASSERT(po->po_owner != NULL,
2583 ("[pmc,%d] pmc_owner had a null proc pointer",
2584 __LINE__));
2585
2586 p->pm_ownerpid = po->po_owner->p_pid;
2587 p->pm_mode = PMC_TO_MODE(pm);
2588 p->pm_event = pm->pm_event;
2589 p->pm_flags = pm->pm_flags;
2590
2591 if (PMC_IS_SAMPLING_MODE(PMC_TO_MODE(pm)))
2592 p->pm_reloadcount =
2593 pm->pm_sc.pm_reloadcount;
2594 }
2595
2596 pmc_restore_cpu_binding(&pb);
2597
2598 /* now copy out the PMC info collected */
2599 if (error == 0)
2600 error = copyout(pmcinfo, &gpi->pm_pmcs, pmcinfo_size);
2601
2602 FREE(pmcinfo, M_PMC);
2603 }
2604 break;
2605
2606
2607 /*
2608 * Set the administrative state of a PMC. I.e. whether
2609 * the PMC is to be used or not.
2610 */
2611
2612 case PMC_OP_PMCADMIN:
2613 {
2614 int cpu, ri;
2615 enum pmc_state request;
2616 struct pmc_cpu *pc;
2617 struct pmc_hw *phw;
2618 struct pmc_op_pmcadmin pma;
2619 struct pmc_binding pb;
2620
2621 sx_assert(&pmc_sx, SX_XLOCKED);
2622
2623 KASSERT(td == curthread,
2624 ("[pmc,%d] td != curthread", __LINE__));
2625
2626 if (suser(td) || jailed(td->td_ucred)) {
2627 error = EPERM;
2628 break;
2629 }
2630
2631 if ((error = copyin(arg, &pma, sizeof(pma))) != 0)
2632 break;
2633
2634 cpu = pma.pm_cpu;
2635
2636 if (cpu < 0 || cpu >= mp_ncpus) {
2637 error = EINVAL;
2638 break;
2639 }
2640
2641 if (pmc_cpu_is_disabled(cpu)) {
2642 error = ENXIO;
2643 break;
2644 }
2645
2646 request = pma.pm_state;
2647
2648 if (request != PMC_STATE_DISABLED &&
2649 request != PMC_STATE_FREE) {
2650 error = EINVAL;
2651 break;
2652 }
2653
2654 ri = pma.pm_pmc; /* pmc id == row index */
2655 if (ri < 0 || ri >= (int) md->pmd_npmc) {
2656 error = EINVAL;
2657 break;
2658 }
2659
2660 /*
2661 * We can't disable a PMC with a row-index allocated
2662 * for process virtual PMCs.
2663 */
2664
2665 if (PMC_ROW_DISP_IS_THREAD(ri) &&
2666 request == PMC_STATE_DISABLED) {
2667 error = EBUSY;
2668 break;
2669 }
2670
2671 /*
2672 * otherwise, this PMC on this CPU is either free or
2673 * in system-wide mode.
2674 */
2675
2676 pmc_save_cpu_binding(&pb);
2677 pmc_select_cpu(cpu);
2678
2679 pc = pmc_pcpu[cpu];
2680 phw = pc->pc_hwpmcs[ri];
2681
2682 /*
2683 * XXX do we need some kind of 'forced' disable?
2684 */
2685
2686 if (phw->phw_pmc == NULL) {
2687 if (request == PMC_STATE_DISABLED &&
2688 (phw->phw_state & PMC_PHW_FLAG_IS_ENABLED)) {
2689 phw->phw_state &= ~PMC_PHW_FLAG_IS_ENABLED;
2690 PMC_MARK_ROW_STANDALONE(ri);
2691 } else if (request == PMC_STATE_FREE &&
2692 (phw->phw_state & PMC_PHW_FLAG_IS_ENABLED) == 0) {
2693 phw->phw_state |= PMC_PHW_FLAG_IS_ENABLED;
2694 PMC_UNMARK_ROW_STANDALONE(ri);
2695 }
2696 /* other cases are a no-op */
2697 } else
2698 error = EBUSY;
2699
2700 pmc_restore_cpu_binding(&pb);
2701 }
2702 break;
2703
2704
2705 /*
2706 * Allocate a PMC.
2707 */
2708
2709 case PMC_OP_PMCALLOCATE:
2710 {
2711 uint32_t caps;
2712 u_int cpu;
2713 int n;
2714 enum pmc_mode mode;
2715 struct pmc *pmc;
2716 struct pmc_hw *phw;
2717 struct pmc_op_pmcallocate pa;
2718 struct pmc_binding pb;
2719
2720 if ((error = copyin(arg, &pa, sizeof(pa))) != 0)
2721 break;
2722
2723 caps = pa.pm_caps;
2724 mode = pa.pm_mode;
2725 cpu = pa.pm_cpu;
2726
2727 if ((mode != PMC_MODE_SS && mode != PMC_MODE_SC &&
2728 mode != PMC_MODE_TS && mode != PMC_MODE_TC) ||
2729 (cpu != (u_int) PMC_CPU_ANY && cpu >= (u_int) mp_ncpus)) {
2730 error = EINVAL;
2731 break;
2732 }
2733
2734 /*
2735 * Virtual PMCs should only ask for a default CPU.
2736 * System mode PMCs need to specify a non-default CPU.
2737 */
2738
2739 if ((PMC_IS_VIRTUAL_MODE(mode) && cpu != (u_int) PMC_CPU_ANY) ||
2740 (PMC_IS_SYSTEM_MODE(mode) && cpu == (u_int) PMC_CPU_ANY)) {
2741 error = EINVAL;
2742 break;
2743 }
2744
2745 /*
2746 * Check that a disabled CPU is not being asked for.
2747 */
2748
2749 if (PMC_IS_SYSTEM_MODE(mode) && pmc_cpu_is_disabled(cpu)) {
2750 error = ENXIO;
2751 break;
2752 }
2753
2754 /*
2755 * Refuse an allocation for a system-wide PMC if this
2756 * process has been jailed, or if this process lacks
2757 * super-user credentials and the sysctl tunable
2758 * 'security.bsd.unprivileged_syspmcs' is zero.
2759 */
2760
2761 if (PMC_IS_SYSTEM_MODE(mode)) {
2762 if (jailed(curthread->td_ucred))
2763 error = EPERM;
2764 else if (suser(curthread) &&
2765 (pmc_unprivileged_syspmcs == 0))
2766 error = EPERM;
2767 }
2768
2769 if (error)
2770 break;
2771
2772 /*
2773 * Look for valid values for 'pm_flags'
2774 */
2775
2776 if ((pa.pm_flags & ~(PMC_F_DESCENDANTS | PMC_F_LOG_PROCCSW |
2777 PMC_F_LOG_PROCEXIT)) != 0) {
2778 error = EINVAL;
2779 break;
2780 }
2781
2782 /* process logging options are not allowed for system PMCs */
2783 if (PMC_IS_SYSTEM_MODE(mode) && (pa.pm_flags &
2784 (PMC_F_LOG_PROCCSW | PMC_F_LOG_PROCEXIT))) {
2785 error = EINVAL;
2786 break;
2787 }
2788
2789 /*
2790 * All sampling mode PMCs need to be able to interrupt the
2791 * CPU.
2792 */
2793
2794 if (PMC_IS_SAMPLING_MODE(mode))
2795 caps |= PMC_CAP_INTERRUPT;
2796
2797 PMCDBG(PMC,ALL,2, "event=%d caps=0x%x mode=%d cpu=%d",
2798 pa.pm_ev, caps, mode, cpu);
2799
2800 pmc = pmc_allocate_pmc_descriptor();
2801 pmc->pm_id = PMC_ID_MAKE_ID(cpu,pa.pm_mode,pa.pm_class,
2802 PMC_ID_INVALID);
2803 pmc->pm_event = pa.pm_ev;
2804 pmc->pm_state = PMC_STATE_FREE;
2805 pmc->pm_caps = caps;
2806 pmc->pm_flags = pa.pm_flags;
2807
2808 /* switch thread to CPU 'cpu' */
2809 pmc_save_cpu_binding(&pb);
2810
2811#define PMC_IS_SHAREABLE_PMC(cpu, n) \
2812 (pmc_pcpu[(cpu)]->pc_hwpmcs[(n)]->phw_state & \
2813 PMC_PHW_FLAG_IS_SHAREABLE)
2814#define PMC_IS_UNALLOCATED(cpu, n) \
2815 (pmc_pcpu[(cpu)]->pc_hwpmcs[(n)]->phw_pmc == NULL)
2816
2817 if (PMC_IS_SYSTEM_MODE(mode)) {
2818 pmc_select_cpu(cpu);
2819 for (n = 0; n < (int) md->pmd_npmc; n++)
2820 if (pmc_can_allocate_row(n, mode) == 0 &&
2821 pmc_can_allocate_rowindex(
2822 curthread->td_proc, n, cpu) == 0 &&
2823 (PMC_IS_UNALLOCATED(cpu, n) ||
2824 PMC_IS_SHAREABLE_PMC(cpu, n)) &&
2825 md->pmd_allocate_pmc(cpu, n, pmc,
2826 &pa) == 0)
2827 break;
2828 } else {
2829 /* Process virtual mode */
2830 for (n = 0; n < (int) md->pmd_npmc; n++) {
2831 if (pmc_can_allocate_row(n, mode) == 0 &&
2832 pmc_can_allocate_rowindex(
2833 curthread->td_proc, n,
2834 PMC_CPU_ANY) == 0 &&
2835 md->pmd_allocate_pmc(curthread->td_oncpu,
2836 n, pmc, &pa) == 0)
2837 break;
2838 }
2839 }
2840
2841#undef PMC_IS_UNALLOCATED
2842#undef PMC_IS_SHAREABLE_PMC
2843
2844 pmc_restore_cpu_binding(&pb);
2845
2846 if (n == (int) md->pmd_npmc) {
2847 pmc_destroy_pmc_descriptor(pmc);
2848 FREE(pmc, M_PMC);
2849 pmc = NULL;
2850 error = EINVAL;
2851 break;
2852 }
2853
2854 /* Fill in the correct value in the ID field */
2855 pmc->pm_id = PMC_ID_MAKE_ID(cpu,mode,pa.pm_class,n);
2856
2857 PMCDBG(PMC,ALL,2, "ev=%d class=%d mode=%d n=%d -> pmcid=%x",
2858 pmc->pm_event, pa.pm_class, mode, n, pmc->pm_id);
2859
2860 /* Process mode PMCs with logging enabled need log files */
2861 if (pmc->pm_flags & (PMC_F_LOG_PROCEXIT | PMC_F_LOG_PROCCSW))
2862 pmc->pm_flags |= PMC_F_NEEDS_LOGFILE;
2863
2864 /* All system mode sampling PMCs require a log file */
2865 if (PMC_IS_SAMPLING_MODE(mode) && PMC_IS_SYSTEM_MODE(mode))
2866 pmc->pm_flags |= PMC_F_NEEDS_LOGFILE;
2867
2868 /*
2869 * Configure global pmc's immediately
2870 */
2871
2872 if (PMC_IS_SYSTEM_MODE(PMC_TO_MODE(pmc))) {
2873
2874 pmc_save_cpu_binding(&pb);
2875 pmc_select_cpu(cpu);
2876
2877 phw = pmc_pcpu[cpu]->pc_hwpmcs[n];
2878
2879 if ((phw->phw_state & PMC_PHW_FLAG_IS_ENABLED) == 0 ||
2880 (error = md->pmd_config_pmc(cpu, n, pmc)) != 0) {
2881 (void) md->pmd_release_pmc(cpu, n, pmc);
2882 pmc_destroy_pmc_descriptor(pmc);
2883 FREE(pmc, M_PMC);
2884 pmc = NULL;
2885 pmc_restore_cpu_binding(&pb);
2886 error = EPERM;
2887 break;
2888 }
2889
2890 pmc_restore_cpu_binding(&pb);
2891 }
2892
2893 pmc->pm_state = PMC_STATE_ALLOCATED;
2894
2895 /*
2896 * mark row disposition
2897 */
2898
2899 if (PMC_IS_SYSTEM_MODE(mode))
2900 PMC_MARK_ROW_STANDALONE(n);
2901 else
2902 PMC_MARK_ROW_THREAD(n);
2903
2904 /*
2905 * Register this PMC with the current thread as its owner.
2906 */
2907
2908 if ((error =
2909 pmc_register_owner(curthread->td_proc, pmc)) != 0) {
2910 pmc_release_pmc_descriptor(pmc);
2911 FREE(pmc, M_PMC);
2912 pmc = NULL;
2913 break;
2914 }
2915
2916 /*
2917 * Return the allocated index.
2918 */
2919
2920 pa.pm_pmcid = pmc->pm_id;
2921
2922 error = copyout(&pa, arg, sizeof(pa));
2923 }
2924 break;
2925
2926
2927 /*
2928 * Attach a PMC to a process.
2929 */
2930
2931 case PMC_OP_PMCATTACH:
2932 {
2933 struct pmc *pm;
2934 struct proc *p;
2935 struct pmc_op_pmcattach a;
2936
2937 sx_assert(&pmc_sx, SX_XLOCKED);
2938
2939 if ((error = copyin(arg, &a, sizeof(a))) != 0)
2940 break;
2941
2942 if (a.pm_pid < 0) {
2943 error = EINVAL;
2944 break;
2945 } else if (a.pm_pid == 0)
2946 a.pm_pid = td->td_proc->p_pid;
2947
2948 if ((error = pmc_find_pmc(a.pm_pmc, &pm)) != 0)
2949 break;
2950
2951 if (PMC_IS_SYSTEM_MODE(PMC_TO_MODE(pm))) {
2952 error = EINVAL;
2953 break;
2954 }
2955
2956 /* PMCs may be (re)attached only when allocated or stopped */
2957 if (pm->pm_state == PMC_STATE_RUNNING) {
2958 error = EBUSY;
2959 break;
2960 } else if (pm->pm_state != PMC_STATE_ALLOCATED &&
2961 pm->pm_state != PMC_STATE_STOPPED) {
2962 error = EINVAL;
2963 break;
2964 }
2965
2966 /* lookup pid */
2967 if ((p = pfind(a.pm_pid)) == NULL) {
2968 error = ESRCH;
2969 break;
2970 }
2971
2972 /*
2973 * Ignore processes that are working on exiting.
2974 */
2975 if (p->p_flag & P_WEXIT) {
2976 error = ESRCH;
2977 PROC_UNLOCK(p); /* pfind() returns a locked process */
2978 break;
2979 }
2980
2981 /*
2982 * we are allowed to attach a PMC to a process if
2983 * we can debug it.
2984 */
2985 error = p_candebug(curthread, p);
2986
2987 PROC_UNLOCK(p);
2988
2989 if (error == 0)
2990 error = pmc_attach_process(p, pm);
2991 }
2992 break;
2993
2994
2995 /*
2996 * Detach an attached PMC from a process.
2997 */
2998
2999 case PMC_OP_PMCDETACH:
3000 {
3001 struct pmc *pm;
3002 struct proc *p;
3003 struct pmc_op_pmcattach a;
3004
3005 if ((error = copyin(arg, &a, sizeof(a))) != 0)
3006 break;
3007
3008 if (a.pm_pid < 0) {
3009 error = EINVAL;
3010 break;
3011 } else if (a.pm_pid == 0)
3012 a.pm_pid = td->td_proc->p_pid;
3013
3014 if ((error = pmc_find_pmc(a.pm_pmc, &pm)) != 0)
3015 break;
3016
3017 if ((p = pfind(a.pm_pid)) == NULL) {
3018 error = ESRCH;
3019 break;
3020 }
3021
3022 /*
3023 * Treat processes that are in the process of exiting
3024 * as if they were not present.
3025 */
3026
3027 if (p->p_flag & P_WEXIT)
3028 error = ESRCH;
3029
3030 PROC_UNLOCK(p); /* pfind() returns a locked process */
3031
3032 if (error == 0)
3033 error = pmc_detach_process(p, pm);
3034 }
3035 break;
3036
3037
3038 /*
3039 * Retrieve the MSR number associated with the counter
3040 * 'pmc_id'. This allows processes to directly use RDPMC
3041 * instructions to read their PMCs, without the overhead of a
3042 * system call.
3043 */
3044
3045 case PMC_OP_PMCGETMSR:
3046 {
3047 int ri;
3048 struct pmc *pm;
3049 struct pmc_target *pt;
3050 struct pmc_op_getmsr gm;
3051
3052 PMC_DOWNGRADE_SX();
3053
3054 /* CPU has no 'GETMSR' support */
3055 if (md->pmd_get_msr == NULL) {
3056 error = ENOSYS;
3057 break;
3058 }
3059
3060 if ((error = copyin(arg, &gm, sizeof(gm))) != 0)
3061 break;
3062
3063 if ((error = pmc_find_pmc(gm.pm_pmcid, &pm)) != 0)
3064 break;
3065
3066 /*
3067 * The allocated PMC has to be a process virtual PMC,
3068 * i.e., of type MODE_T[CS]. Global PMCs can only be
3069 * read using the PMCREAD operation since they may be
3070 * allocated on a different CPU than the one we could
3071 * be running on at the time of the RDPMC instruction.
3072 *
3073 * The GETMSR operation is not allowed for PMCs that
3074 * are inherited across processes.
3075 */
3076
3077 if (!PMC_IS_VIRTUAL_MODE(PMC_TO_MODE(pm)) ||
3078 (pm->pm_flags & PMC_F_DESCENDANTS)) {
3079 error = EINVAL;
3080 break;
3081 }
3082
3083 /*
3084 * It only makes sense to use a RDPMC (or its
3085 * equivalent instruction on non-x86 architectures) on
3086 * a process that has allocated and attached a PMC to
3087 * itself. Conversely the PMC is only allowed to have
3088 * one process attached to it -- its owner.
3089 */
3090
3091 if ((pt = LIST_FIRST(&pm->pm_targets)) == NULL ||
3092 LIST_NEXT(pt, pt_next) != NULL ||
3093 pt->pt_process->pp_proc != pm->pm_owner->po_owner) {
3094 error = EINVAL;
3095 break;
3096 }
3097
3098 ri = PMC_TO_ROWINDEX(pm);
3099
3100 if ((error = (*md->pmd_get_msr)(ri, &gm.pm_msr)) < 0)
3101 break;
3102
3103 if ((error = copyout(&gm, arg, sizeof(gm))) < 0)
3104 break;
3105
3106 /*
3107 * Mark our process as using MSRs. Update machine
3108 * state using a forced context switch.
3109 */
3110
3111 pt->pt_process->pp_flags |= PMC_PP_ENABLE_MSR_ACCESS;
3112 pmc_force_context_switch();
3113
3114 }
3115 break;
3116
3117 /*
3118 * Release an allocated PMC
3119 */
3120
3121 case PMC_OP_PMCRELEASE:
3122 {
3123 pmc_id_t pmcid;
3124 struct pmc *pm;
3125 struct pmc_owner *po;
3126 struct pmc_op_simple sp;
3127
3128 /*
3129 * Find PMC pointer for the named PMC.
3130 *
3131 * Use pmc_release_pmc_descriptor() to switch off the
3132 * PMC, remove all its target threads, and remove the
3133 * PMC from its owner's list.
3134 *
3135 * Remove the owner record if this is the last PMC
3136 * owned.
3137 *
3138 * Free up space.
3139 */
3140
3141 if ((error = copyin(arg, &sp, sizeof(sp))) != 0)
3142 break;
3143
3144 pmcid = sp.pm_pmcid;
3145
3146 if ((error = pmc_find_pmc(pmcid, &pm)) != 0)
3147 break;
3148
3149 po = pm->pm_owner;
3150 pmc_release_pmc_descriptor(pm);
3151 pmc_maybe_remove_owner(po);
3152
3153 FREE(pm, M_PMC);
3154 }
3155 break;
3156
3157
3158 /*
3159 * Read and/or write a PMC.
3160 */
3161
3162 case PMC_OP_PMCRW:
3163 {
3164 uint32_t cpu, ri;
3165 struct pmc *pm;
3166 struct pmc_op_pmcrw *pprw;
3167 struct pmc_op_pmcrw prw;
3168 struct pmc_binding pb;
3169 pmc_value_t oldvalue;
3170
3171 PMC_DOWNGRADE_SX();
3172
3173 if ((error = copyin(arg, &prw, sizeof(prw))) != 0)
3174 break;
3175
3176 ri = 0;
3177 PMCDBG(PMC,OPS,1, "rw id=%d flags=0x%x", prw.pm_pmcid,
3178 prw.pm_flags);
3179
3180 /* must have at least one flag set */
3181 if ((prw.pm_flags & (PMC_F_OLDVALUE|PMC_F_NEWVALUE)) == 0) {
3182 error = EINVAL;
3183 break;
3184 }
3185
3186 /* locate pmc descriptor */
3187 if ((error = pmc_find_pmc(prw.pm_pmcid, &pm)) != 0)
3188 break;
3189
3190 /* Can't read a PMC that hasn't been started. */
3191 if (pm->pm_state != PMC_STATE_ALLOCATED &&
3192 pm->pm_state != PMC_STATE_STOPPED &&
3193 pm->pm_state != PMC_STATE_RUNNING) {
3194 error = EINVAL;
3195 break;
3196 }
3197
3198 /* writing a new value is allowed only for 'STOPPED' pmcs */
3199 if (pm->pm_state == PMC_STATE_RUNNING &&
3200 (prw.pm_flags & PMC_F_NEWVALUE)) {
3201 error = EBUSY;
3202 break;
3203 }
3204
3205 if (PMC_IS_VIRTUAL_MODE(PMC_TO_MODE(pm))) {
3206
3207 /*
3208 * If this PMC is attached to its owner (i.e.,
3209 * the process requesting this operation) and
3210 * is running, then attempt to get an
3211 * upto-date reading from hardware for a READ.
3212 * Writes are only allowed when the PMC is
3213 * stopped, so only update the saved value
3214 * field.
3215 *
3216 * If the PMC is not running, or is not
3217 * attached to its owner, read/write to the
3218 * savedvalue field.
3219 */
3220
3221 ri = PMC_TO_ROWINDEX(pm);
3222
3223 mtx_pool_lock_spin(pmc_mtxpool, pm);
3224 cpu = curthread->td_oncpu;
3225
3226 if (prw.pm_flags & PMC_F_OLDVALUE) {
3227 if ((pm->pm_flags & PMC_F_ATTACHED_TO_OWNER) &&
3228 (pm->pm_state == PMC_STATE_RUNNING))
3229 error = (*md->pmd_read_pmc)(cpu, ri,
3230 &oldvalue);
3231 else
3232 oldvalue = pm->pm_gv.pm_savedvalue;
3233 }
3234 if (prw.pm_flags & PMC_F_NEWVALUE)
3235 pm->pm_gv.pm_savedvalue = prw.pm_value;
3236
3237 mtx_pool_unlock_spin(pmc_mtxpool, pm);
3238
3239 } else { /* System mode PMCs */
3240 cpu = PMC_TO_CPU(pm);
3241 ri = PMC_TO_ROWINDEX(pm);
3242
3243 if (pmc_cpu_is_disabled(cpu)) {
3244 error = ENXIO;
3245 break;
3246 }
3247
3248 /* move this thread to CPU 'cpu' */
3249 pmc_save_cpu_binding(&pb);
3250 pmc_select_cpu(cpu);
3251
3252 critical_enter();
3253 /* save old value */
3254 if (prw.pm_flags & PMC_F_OLDVALUE)
3255 if ((error = (*md->pmd_read_pmc)(cpu, ri,
3256 &oldvalue)))
3257 goto error;
3258 /* write out new value */
3259 if (prw.pm_flags & PMC_F_NEWVALUE)
3260 error = (*md->pmd_write_pmc)(cpu, ri,
3261 prw.pm_value);
3262 error:
3263 critical_exit();
3264 pmc_restore_cpu_binding(&pb);
3265 if (error)
3266 break;
3267 }
3268
3269 pprw = (struct pmc_op_pmcrw *) arg;
3270
3271#if DEBUG
3272 if (prw.pm_flags & PMC_F_NEWVALUE)
3273 PMCDBG(PMC,OPS,2, "rw id=%d new %jx -> old %jx",
3274 ri, prw.pm_value, oldvalue);
3275 else
3276 PMCDBG(PMC,OPS,2, "rw id=%d -> old %jx", ri, oldvalue);
3277#endif
3278
3279 /* return old value if requested */
3280 if (prw.pm_flags & PMC_F_OLDVALUE)
3281 if ((error = copyout(&oldvalue, &pprw->pm_value,
3282 sizeof(prw.pm_value))))
3283 break;
3284
3285 }
3286 break;
3287
3288
3289 /*
3290 * Set the sampling rate for a sampling mode PMC and the
3291 * initial count for a counting mode PMC.
3292 */
3293
3294 case PMC_OP_PMCSETCOUNT:
3295 {
3296 struct pmc *pm;
3297 struct pmc_op_pmcsetcount sc;
3298
3299 PMC_DOWNGRADE_SX();
3300
3301 if ((error = copyin(arg, &sc, sizeof(sc))) != 0)
3302 break;
3303
3304 if ((error = pmc_find_pmc(sc.pm_pmcid, &pm)) != 0)
3305 break;
3306
3307 if (pm->pm_state == PMC_STATE_RUNNING) {
3308 error = EBUSY;
3309 break;
3310 }
3311
3312 if (PMC_IS_SAMPLING_MODE(PMC_TO_MODE(pm)))
3313 pm->pm_sc.pm_reloadcount = sc.pm_count;
3314 else
3315 pm->pm_sc.pm_initial = sc.pm_count;
3316 }
3317 break;
3318
3319
3320 /*
3321 * Start a PMC.
3322 */
3323
3324 case PMC_OP_PMCSTART:
3325 {
3326 pmc_id_t pmcid;
3327 struct pmc *pm;
3328 struct pmc_op_simple sp;
3329
3330 sx_assert(&pmc_sx, SX_XLOCKED);
3331
3332 if ((error = copyin(arg, &sp, sizeof(sp))) != 0)
3333 break;
3334
3335 pmcid = sp.pm_pmcid;
3336
3337 if ((error = pmc_find_pmc(pmcid, &pm)) != 0)
3338 break;
3339
3340 KASSERT(pmcid == pm->pm_id,
3341 ("[pmc,%d] pmcid %x != id %x", __LINE__,
3342 pm->pm_id, pmcid));
3343
3344 if (pm->pm_state == PMC_STATE_RUNNING) /* already running */
3345 break;
3346 else if (pm->pm_state != PMC_STATE_STOPPED &&
3347 pm->pm_state != PMC_STATE_ALLOCATED) {
3348 error = EINVAL;
3349 break;
3350 }
3351
3352 error = pmc_start(pm);
3353 }
3354 break;
3355
3356
3357 /*
3358 * Stop a PMC.
3359 */
3360
3361 case PMC_OP_PMCSTOP:
3362 {
3363 pmc_id_t pmcid;
3364 struct pmc *pm;
3365 struct pmc_op_simple sp;
3366
3367 PMC_DOWNGRADE_SX();
3368
3369 if ((error = copyin(arg, &sp, sizeof(sp))) != 0)
3370 break;
3371
3372 pmcid = sp.pm_pmcid;
3373
3374 /*
3375 * Mark the PMC as inactive and invoke the MD stop
3376 * routines if needed.
3377 */
3378
3379 if ((error = pmc_find_pmc(pmcid, &pm)) != 0)
3380 break;
3381
3382 KASSERT(pmcid == pm->pm_id,
3383 ("[pmc,%d] pmc id %x != pmcid %x", __LINE__,
3384 pm->pm_id, pmcid));
3385
3386 if (pm->pm_state == PMC_STATE_STOPPED) /* already stopped */
3387 break;
3388 else if (pm->pm_state != PMC_STATE_RUNNING) {
3389 error = EINVAL;
3390 break;
3391 }
3392
3393 error = pmc_stop(pm);
3394 }
3395 break;
3396
3397
3398 /*
3399 * Write a user supplied value to the log file.
3400 */
3401
3402 case PMC_OP_WRITELOG:
3403 {
3404 struct pmc_op_writelog wl;
3405 struct pmc_owner *po;
3406
3407 PMC_DOWNGRADE_SX();
3408
3409 if ((error = copyin(arg, &wl, sizeof(wl))) != 0)
3410 break;
3411
3412 if ((po = pmc_find_owner_descriptor(td->td_proc)) == NULL) {
3413 error = EINVAL;
3414 break;
3415 }
3416
3417 if ((po->po_flags & PMC_PO_OWNS_LOGFILE) == 0) {
3418 error = EINVAL;
3419 break;
3420 }
3421
3422 error = pmclog_process_userlog(po, &wl);
3423 }
3424 break;
3425
3426
3427 default:
3428 error = EINVAL;
3429 break;
3430 }
3431
3432 if (is_sx_downgraded)
3433 sx_sunlock(&pmc_sx);
3434 else
3435 sx_xunlock(&pmc_sx);
3436
3437 if (error)
3438 atomic_add_int(&pmc_stats.pm_syscall_errors, 1);
3439
3440 PICKUP_GIANT();
3441
3442 return error;
3443}
3444
3445/*
3446 * Helper functions
3447 */
3448
3449
3450/*
3451 * Interrupt processing.
3452 *
3453 * Find a free slot in the per-cpu array of PC samples and write the
3454 * current (PMC,PID,PC) triple to it. If an event was successfully
3455 * added, a bit is set in mask 'pmc_cpumask' denoting that the
3456 * DO_SAMPLES hook needs to be invoked from the clock handler.
3457 *
3458 * This function is meant to be called from an NMI handler. It cannot
3459 * use any of the locking primitives supplied by the OS.
3460 */
3461
3462int
3463pmc_process_interrupt(int cpu, struct pmc *pm, uintfptr_t pc, int usermode)
3464{
3465 int error, ri;
3466 struct thread *td;
3467 struct pmc_sample *ps;
3468 struct pmc_samplebuffer *psb;
3469
3470 error = 0;
3471 ri = PMC_TO_ROWINDEX(pm);
3472
3473 psb = pmc_pcpu[cpu]->pc_sb;
3474
3475 ps = psb->ps_write;
3476 if (ps->ps_pc) { /* in use, reader hasn't caught up */
3477 pm->pm_stalled = 1;
3478 atomic_add_int(&pmc_stats.pm_intr_bufferfull, 1);
3479 PMCDBG(SAM,INT,1,"(spc) cpu=%d pm=%p pc=%jx um=%d wr=%d rd=%d",
3480 cpu, pm, (uint64_t) pc, usermode,
3481 (int) (psb->ps_write - psb->ps_samples),
3482 (int) (psb->ps_read - psb->ps_samples));
3483 error = ENOMEM;
3484 goto done;
3485 }
3486
3487 /* fill in entry */
3488 PMCDBG(SAM,INT,1,"cpu=%d pm=%p pc=%jx um=%d wr=%d rd=%d", cpu, pm,
3489 (uint64_t) pc, usermode,
3490 (int) (psb->ps_write - psb->ps_samples),
3491 (int) (psb->ps_read - psb->ps_samples));
3492
3493 atomic_add_rel_32(&pm->pm_runcount, 1); /* hold onto PMC */
3494 ps->ps_pmc = pm;
3495 if ((td = curthread) && td->td_proc)
3496 ps->ps_pid = td->td_proc->p_pid;
3497 else
3498 ps->ps_pid = -1;
3499 ps->ps_usermode = usermode;
3500 ps->ps_pc = pc; /* mark entry as in use */
3501
3502 /* increment write pointer, modulo ring buffer size */
3503 ps++;
3504 if (ps == psb->ps_fence)
3505 psb->ps_write = psb->ps_samples;
3506 else
3507 psb->ps_write = ps;
3508
3509 done:
3510 /* mark CPU as needing processing */
3511 atomic_set_rel_int(&pmc_cpumask, (1 << cpu));
3512
3513 return error;
3514}
3515
3516
3517/*
3518 * Process saved PC samples.
3519 */
3520
3521static void
3522pmc_process_samples(int cpu)
3523{
3524 int n, ri;
3525 struct pmc *pm;
3526 struct thread *td;
3527 struct pmc_owner *po;
3528 struct pmc_sample *ps;
3529 struct pmc_samplebuffer *psb;
3530
3531 KASSERT(PCPU_GET(cpuid) == cpu,
3532 ("[pmc,%d] not on the correct CPU pcpu=%d cpu=%d", __LINE__,
3533 PCPU_GET(cpuid), cpu));
3534
3535 psb = pmc_pcpu[cpu]->pc_sb;
3536
3537 for (n = 0; n < pmc_nsamples; n++) { /* bound on #iterations */
3538
3539 ps = psb->ps_read;
3540 if (ps->ps_pc == (uintfptr_t) 0) /* no data */
3541 break;
3542
3543 pm = ps->ps_pmc;
3544 po = pm->pm_owner;
3545
3546 KASSERT(PMC_IS_SAMPLING_MODE(PMC_TO_MODE(pm)),
3547 ("[pmc,%d] pmc=%p non-sampling mode=%d", __LINE__,
3548 pm, PMC_TO_MODE(pm)));
3549
3550 /* Ignore PMCs that have been switched off */
3551 if (pm->pm_state != PMC_STATE_RUNNING)
3552 goto entrydone;
3553
3554 PMCDBG(SAM,OPS,1,"cpu=%d pm=%p pc=%jx um=%d wr=%d rd=%d", cpu,
3555 pm, (uint64_t) ps->ps_pc, ps->ps_usermode,
3556 (int) (psb->ps_write - psb->ps_samples),
3557 (int) (psb->ps_read - psb->ps_samples));
3558
3559 /*
3560 * If this is a process-mode PMC that is attached to
3561 * its owner, and if the PC is in user mode, update
3562 * profiling statistics like timer-based profiling
3563 * would have done.
3564 */
3565 if (pm->pm_flags & PMC_F_ATTACHED_TO_OWNER) {
3566 if (ps->ps_usermode) {
3567 td = FIRST_THREAD_IN_PROC(po->po_owner);
3568 addupc_intr(td, ps->ps_pc, 1);
3569 }
3570 goto entrydone;
3571 }
3572
3573 /*
3574 * Otherwise, this is either a sampling mode PMC that
3575 * is attached to a different process than its owner,
3576 * or a system-wide sampling PMC. Dispatch a log
3577 * entry to the PMC's owner process.
3578 */
3579
3580 pmclog_process_pcsample(pm, ps);
3581
3582 entrydone:
3583 ps->ps_pc = (uintfptr_t) 0; /* mark entry as free */
3584 atomic_subtract_rel_32(&pm->pm_runcount, 1);
3585
3586 /* increment read pointer, modulo sample size */
3587 if (++ps == psb->ps_fence)
3588 psb->ps_read = psb->ps_samples;
3589 else
3590 psb->ps_read = ps;
3591 }
3592
3593 atomic_add_int(&pmc_stats.pm_log_sweeps, 1);
3594
3595 /* Do not re-enable stalled PMCs if we failed to process any samples */
3596 if (n == 0)
3597 return;
3598
3599 /*
3600 * Restart any stalled sampling PMCs on this CPU.
3601 *
3602 * If the NMI handler sets the pm_stalled field of a PMC after
3603 * the check below, we'll end up processing the stalled PMC at
3604 * the next hardclock tick.
3605 */
3606 for (n = 0; n < md->pmd_npmc; n++) {
3607 (void) (*md->pmd_get_config)(cpu,n,&pm);
3608 if (pm == NULL || /* !cfg'ed */
3609 pm->pm_state != PMC_STATE_RUNNING || /* !active */
3610 !PMC_IS_SAMPLING_MODE(PMC_TO_MODE(pm)) || /* !sampling */
3611 pm->pm_stalled == 0) /* !stalled */
3612 continue;
3613
3614 pm->pm_stalled = 0;
3615 ri = PMC_TO_ROWINDEX(pm);
3616 (*md->pmd_start_pmc)(cpu, ri);
3617 }
3618}
3619
3620/*
3621 * Event handlers.
3622 */
3623
3624/*
3625 * Handle a process exit.
3626 *
3627 * Remove this process from all hash tables. If this process
3628 * owned any PMCs, turn off those PMCs and deallocate them,
3629 * removing any associations with target processes.
3630 *
3631 * This function will be called by the last 'thread' of a
3632 * process.
3633 *
3634 * XXX This eventhandler gets called early in the exit process.
3635 * Consider using a 'hook' invocation from thread_exit() or equivalent
3636 * spot. Another negative is that kse_exit doesn't seem to call
3637 * exit1() [??].
3638 *
3639 */
3640
3641static void
3642pmc_process_exit(void *arg __unused, struct proc *p)
3643{
3644 int is_using_hwpmcs;
3645 int cpu;
3646 unsigned int ri;
3647 struct pmc *pm;
3648 struct pmc_process *pp;
3649 struct pmc_owner *po;
3650 pmc_value_t newvalue, tmp;
3651
3652 PROC_LOCK(p);
3653 is_using_hwpmcs = p->p_flag & P_HWPMC;
3654 PROC_UNLOCK(p);
3655
3656 /*
3657 * Log a sysexit event to all SS PMC owners.
3658 */
3659 LIST_FOREACH(po, &pmc_ss_owners, po_ssnext)
3660 if (po->po_flags & PMC_PO_OWNS_LOGFILE)
3661 pmclog_process_sysexit(po, p->p_pid);
3662
3663 if (!is_using_hwpmcs)
3664 return;
3665
3666 PMC_GET_SX_XLOCK();
3667 PMCDBG(PRC,EXT,1,"process-exit proc=%p (%d, %s)", p, p->p_pid,
3668 p->p_comm);
3669
3670 /*
3671 * Since this code is invoked by the last thread in an exiting
3672 * process, we would have context switched IN at some prior
3673 * point. However, with PREEMPTION, kernel mode context
3674 * switches may happen any time, so we want to disable a
3675 * context switch OUT till we get any PMCs targetting this
3676 * process off the hardware.
3677 *
3678 * We also need to atomically remove this process'
3679 * entry from our target process hash table, using
3680 * PMC_FLAG_REMOVE.
3681 */
3682 PMCDBG(PRC,EXT,1, "process-exit proc=%p (%d, %s)", p, p->p_pid,
3683 p->p_comm);
3684
3685 critical_enter(); /* no preemption */
3686
3687 cpu = curthread->td_oncpu;
3688
3689 if ((pp = pmc_find_process_descriptor(p,
3690 PMC_FLAG_REMOVE)) != NULL) {
3691
3692 PMCDBG(PRC,EXT,2,
3693 "process-exit proc=%p pmc-process=%p", p, pp);
3694
3695 /*
3696 * The exiting process could the target of
3697 * some PMCs which will be running on
3698 * currently executing CPU.
3699 *
3700 * We need to turn these PMCs off like we
3701 * would do at context switch OUT time.
3702 */
3703 for (ri = 0; ri < md->pmd_npmc; ri++) {
3704
3705 /*
3706 * Pick up the pmc pointer from hardware
3707 * state similar to the CSW_OUT code.
3708 */
3709 pm = NULL;
3710 (void) (*md->pmd_get_config)(cpu, ri, &pm);
3711
3712 PMCDBG(PRC,EXT,2, "ri=%d pm=%p", ri, pm);
3713
3714 if (pm == NULL ||
3715 !PMC_IS_VIRTUAL_MODE(PMC_TO_MODE(pm)))
3716 continue;
3717
3718 PMCDBG(PRC,EXT,2, "ppmcs[%d]=%p pm=%p "
3719 "state=%d", ri, pp->pp_pmcs[ri].pp_pmc,
3720 pm, pm->pm_state);
3721
3722 KASSERT(PMC_TO_ROWINDEX(pm) == ri,
3723 ("[pmc,%d] ri mismatch pmc(%d) ri(%d)",
3724 __LINE__, PMC_TO_ROWINDEX(pm), ri));
3725
3726 KASSERT(pm == pp->pp_pmcs[ri].pp_pmc,
3727 ("[pmc,%d] pm %p != pp_pmcs[%d] %p",
3728 __LINE__, pm, ri, pp->pp_pmcs[ri].pp_pmc));
3729
3730 (void) md->pmd_stop_pmc(cpu, ri);
3731
3732 KASSERT(pm->pm_runcount > 0,
3733 ("[pmc,%d] bad runcount ri %d rc %d",
3734 __LINE__, ri, pm->pm_runcount));
3735
3736 /* Stop hardware only if it is actually running */
3737 if (pm->pm_state == PMC_STATE_RUNNING &&
3738 pm->pm_stalled == 0) {
3739 md->pmd_read_pmc(cpu, ri, &newvalue);
3740 tmp = newvalue -
3741 PMC_PCPU_SAVED(cpu,ri);
3742
3743 mtx_pool_lock_spin(pmc_mtxpool, pm);
3744 pm->pm_gv.pm_savedvalue += tmp;
3745 pp->pp_pmcs[ri].pp_pmcval += tmp;
3746 mtx_pool_unlock_spin(pmc_mtxpool, pm);
3747 }
3748
3749 atomic_subtract_rel_32(&pm->pm_runcount,1);
3750
3751 KASSERT((int) pm->pm_runcount >= 0,
3752 ("[pmc,%d] runcount is %d", __LINE__, ri));
3753
3754 (void) md->pmd_config_pmc(cpu, ri, NULL);
3755 }
3756
3757 /*
3758 * Inform the MD layer of this pseudo "context switch
3759 * out"
3760 */
3761 (void) md->pmd_switch_out(pmc_pcpu[cpu], pp);
3762
3763 critical_exit(); /* ok to be pre-empted now */
3764
3765 /*
3766 * Unlink this process from the PMCs that are
3767 * targetting it. This will send a signal to
3768 * all PMC owner's whose PMCs are orphaned.
3769 *
3770 * Log PMC value at exit time if requested.
3771 */
3772 for (ri = 0; ri < md->pmd_npmc; ri++)
3773 if ((pm = pp->pp_pmcs[ri].pp_pmc) != NULL) {
3774 if (pm->pm_flags & PMC_F_NEEDS_LOGFILE &&
3775 PMC_IS_COUNTING_MODE(PMC_TO_MODE(pm)))
3776 pmclog_process_procexit(pm, pp);
3777 pmc_unlink_target_process(pm, pp);
3778 }
3779 FREE(pp, M_PMC);
3780
3781 } else
3782 critical_exit(); /* pp == NULL */
3783
3784
3785 /*
3786 * If the process owned PMCs, free them up and free up
3787 * memory.
3788 */
3789 if ((po = pmc_find_owner_descriptor(p)) != NULL) {
3790 pmc_remove_owner(po);
3791 pmc_destroy_owner_descriptor(po);
3792 }
3793
3794 sx_xunlock(&pmc_sx);
3795}
3796
3797/*
3798 * Handle a process fork.
3799 *
3800 * If the parent process 'p1' is under HWPMC monitoring, then copy
3801 * over any attached PMCs that have 'do_descendants' semantics.
3802 */
3803
3804static void
3805pmc_process_fork(void *arg __unused, struct proc *p1, struct proc *newproc,
3806 int flags)
3807{
3808 int is_using_hwpmcs;
3809 unsigned int ri;
3810 uint32_t do_descendants;
3811 struct pmc *pm;
3812 struct pmc_owner *po;
3813 struct pmc_process *ppnew, *ppold;
3814
3815 (void) flags; /* unused parameter */
3816
3817 PROC_LOCK(p1);
3818 is_using_hwpmcs = p1->p_flag & P_HWPMC;
3819 PROC_UNLOCK(p1);
3820
3821 /*
3822 * If there are system-wide sampling PMCs active, we need to
3823 * log all fork events to their owner's logs.
3824 */
3825
3826 LIST_FOREACH(po, &pmc_ss_owners, po_ssnext)
3827 if (po->po_flags & PMC_PO_OWNS_LOGFILE)
3828 pmclog_process_procfork(po, p1->p_pid, newproc->p_pid);
3829
3830 if (!is_using_hwpmcs)
3831 return;
3832
3833 PMC_GET_SX_XLOCK();
3834 PMCDBG(PMC,FRK,1, "process-fork proc=%p (%d, %s) -> %p", p1,
3835 p1->p_pid, p1->p_comm, newproc);
3836
3837 /*
3838 * If the parent process (curthread->td_proc) is a
3839 * target of any PMCs, look for PMCs that are to be
3840 * inherited, and link these into the new process
3841 * descriptor.
3842 */
3843 if ((ppold = pmc_find_process_descriptor(curthread->td_proc,
3844 PMC_FLAG_NONE)) == NULL)
3845 goto done; /* nothing to do */
3846
3847 do_descendants = 0;
3848 for (ri = 0; ri < md->pmd_npmc; ri++)
3849 if ((pm = ppold->pp_pmcs[ri].pp_pmc) != NULL)
3850 do_descendants |= pm->pm_flags & PMC_F_DESCENDANTS;
3851 if (do_descendants == 0) /* nothing to do */
3852 goto done;
3853
3854 /* allocate a descriptor for the new process */
3855 if ((ppnew = pmc_find_process_descriptor(newproc,
3856 PMC_FLAG_ALLOCATE)) == NULL)
3857 goto done;
3858
3859 /*
3860 * Run through all PMCs that were targeting the old process
3861 * and which specified F_DESCENDANTS and attach them to the
3862 * new process.
3863 *
3864 * Log the fork event to all owners of PMCs attached to this
3865 * process, if not already logged.
3866 */
3867 for (ri = 0; ri < md->pmd_npmc; ri++)
3868 if ((pm = ppold->pp_pmcs[ri].pp_pmc) != NULL &&
3869 (pm->pm_flags & PMC_F_DESCENDANTS)) {
3870 pmc_link_target_process(pm, ppnew);
3871 po = pm->pm_owner;
3872 if (po->po_sscount == 0 &&
3873 po->po_flags & PMC_PO_OWNS_LOGFILE)
3874 pmclog_process_procfork(po, p1->p_pid,
3875 newproc->p_pid);
3876 }
3877
3878 /*
3879 * Now mark the new process as being tracked by this driver.
3880 */
3881 PROC_LOCK(newproc);
3882 newproc->p_flag |= P_HWPMC;
3883 PROC_UNLOCK(newproc);
3884
3885 done:
3886 sx_xunlock(&pmc_sx);
3887}
3888
3889
3890/*
3891 * initialization
3892 */
3893
3894static const char *pmc_name_of_pmcclass[] = {
3895#undef __PMC_CLASS
3896#define __PMC_CLASS(N) #N ,
3897 __PMC_CLASSES()
3898};
3899
3900static int
3901pmc_initialize(void)
3902{
3903 int cpu, error, n;
3904 struct pmc_binding pb;
3905 struct pmc_samplebuffer *sb;
3906
3907 md = NULL;
3908 error = 0;
3909
3910#if DEBUG
3911 /* parse debug flags first */
3912 if (TUNABLE_STR_FETCH(PMC_SYSCTL_NAME_PREFIX "debugflags",
3913 pmc_debugstr, sizeof(pmc_debugstr)))
3914 pmc_debugflags_parse(pmc_debugstr,
3915 pmc_debugstr+strlen(pmc_debugstr));
3916#endif
3917
3918 PMCDBG(MOD,INI,0, "PMC Initialize (version %x)", PMC_VERSION);
3919
3920 /* check kernel version */
3921 if (pmc_kernel_version != PMC_VERSION) {
3922 if (pmc_kernel_version == 0)
3923 printf("hwpmc: this kernel has not been compiled with "
3924 "'options HWPMC_HOOKS'.\n");
3925 else
3926 printf("hwpmc: kernel version (0x%x) does not match "
3927 "module version (0x%x).\n", pmc_kernel_version,
3928 PMC_VERSION);
3929 return EPROGMISMATCH;
3930 }
3931
3932 /*
3933 * check sysctl parameters
3934 */
3935
3936 if (pmc_hashsize <= 0) {
3937 (void) printf("hwpmc: tunable hashsize=%d must be greater "
3938 "than zero.\n", pmc_hashsize);
3939 pmc_hashsize = PMC_HASH_SIZE;
3940 }
3941
3942 if (pmc_nsamples <= 0 || pmc_nsamples > 65535) {
3943 (void) printf("hwpmc: tunable nsamples=%d out of range.\n", pmc_nsamples);
3944 pmc_nsamples = PMC_NSAMPLES;
3945 }
3946
3947 md = pmc_md_initialize();
3948
3949 if (md == NULL || md->pmd_init == NULL)
3950 return ENOSYS;
3951
3952 /* allocate space for the per-cpu array */
3953 MALLOC(pmc_pcpu, struct pmc_cpu **, mp_ncpus * sizeof(struct pmc_cpu *),
3954 M_PMC, M_WAITOK|M_ZERO);
3955
3956 /* per-cpu 'saved values' for managing process-mode PMCs */
3957 MALLOC(pmc_pcpu_saved, pmc_value_t *,
3958 sizeof(pmc_value_t) * mp_ncpus * md->pmd_npmc, M_PMC, M_WAITOK);
3959
3960 /* perform cpu dependent initialization */
3961 pmc_save_cpu_binding(&pb);
3962 for (cpu = 0; cpu < mp_ncpus; cpu++) {
3963 if (pmc_cpu_is_disabled(cpu))
3964 continue;
3965 pmc_select_cpu(cpu);
3966 if ((error = md->pmd_init(cpu)) != 0)
3967 break;
3968 }
3969 pmc_restore_cpu_binding(&pb);
3970
3971 if (error != 0)
3972 return error;
3973
3974 /* allocate space for the sample array */
3975 for (cpu = 0; cpu < mp_ncpus; cpu++) {
3976 if (pmc_cpu_is_disabled(cpu))
3977 continue;
3978 MALLOC(sb, struct pmc_samplebuffer *,
3979 sizeof(struct pmc_samplebuffer) +
3980 pmc_nsamples * sizeof(struct pmc_sample), M_PMC,
3981 M_WAITOK|M_ZERO);
3982
3983 sb->ps_read = sb->ps_write = sb->ps_samples;
3984 sb->ps_fence = sb->ps_samples + pmc_nsamples
3985;
3986 KASSERT(pmc_pcpu[cpu] != NULL,
3987 ("[pmc,%d] cpu=%d Null per-cpu data", __LINE__, cpu));
3988
3989 pmc_pcpu[cpu]->pc_sb = sb;
3990 }
3991
3992 /* allocate space for the row disposition array */
3993 pmc_pmcdisp = malloc(sizeof(enum pmc_mode) * md->pmd_npmc,
3994 M_PMC, M_WAITOK|M_ZERO);
3995
3996 KASSERT(pmc_pmcdisp != NULL,
3997 ("[pmc,%d] pmcdisp allocation returned NULL", __LINE__));
3998
3999 /* mark all PMCs as available */
4000 for (n = 0; n < (int) md->pmd_npmc; n++)
4001 PMC_MARK_ROW_FREE(n);
4002
4003 /* allocate thread hash tables */
4004 pmc_ownerhash = hashinit(pmc_hashsize, M_PMC,
4005 &pmc_ownerhashmask);
4006
4007 pmc_processhash = hashinit(pmc_hashsize, M_PMC,
4008 &pmc_processhashmask);
4009 mtx_init(&pmc_processhash_mtx, "pmc-process-hash", "pmc", MTX_SPIN);
4010
4011 LIST_INIT(&pmc_ss_owners);
4012 pmc_ss_count = 0;
4013
4014 /* allocate a pool of spin mutexes */
4015 pmc_mtxpool = mtx_pool_create("pmc", pmc_mtxpool_size, MTX_SPIN);
4016
4017 PMCDBG(MOD,INI,1, "pmc_ownerhash=%p, mask=0x%lx "
4018 "targethash=%p mask=0x%lx", pmc_ownerhash, pmc_ownerhashmask,
4019 pmc_processhash, pmc_processhashmask);
4020
4021 /* register process {exit,fork,exec} handlers */
4022 pmc_exit_tag = EVENTHANDLER_REGISTER(process_exit,
4023 pmc_process_exit, NULL, EVENTHANDLER_PRI_ANY);
4024 pmc_fork_tag = EVENTHANDLER_REGISTER(process_fork,
4025 pmc_process_fork, NULL, EVENTHANDLER_PRI_ANY);
4026
4027 /* initialize logging */
4028 pmclog_initialize();
4029
4030 /* set hook functions */
4031 pmc_intr = md->pmd_intr;
4032 pmc_hook = pmc_hook_handler;
4033
4034 if (error == 0) {
4035 printf(PMC_MODULE_NAME ":");
4036 for (n = 0; n < (int) md->pmd_nclass; n++)
4037 printf(" %s(%d)",
4038 pmc_name_of_pmcclass[md->pmd_classes[n].pm_class],
4039 md->pmd_nclasspmcs[n]);
4040 printf("\n");
4041 }
4042
4043 return error;
4044}
4045
4046/* prepare to be unloaded */
4047static void
4048pmc_cleanup(void)
4049{
4050 int cpu;
4051 struct pmc_ownerhash *ph;
4052 struct pmc_owner *po, *tmp;
4053 struct pmc_binding pb;
4054#if DEBUG
4055 struct pmc_processhash *prh;
4056#endif
4057
4058 PMCDBG(MOD,INI,0, "%s", "cleanup");
4059
4060 /* switch off sampling */
4061 atomic_store_rel_int(&pmc_cpumask, 0);
4062 pmc_intr = NULL;
4063
4064 sx_xlock(&pmc_sx);
4065 if (pmc_hook == NULL) { /* being unloaded already */
4066 sx_xunlock(&pmc_sx);
4067 return;
4068 }
4069
4070 pmc_hook = NULL; /* prevent new threads from entering module */
4071
4072 /* deregister event handlers */
4073 EVENTHANDLER_DEREGISTER(process_fork, pmc_fork_tag);
4074 EVENTHANDLER_DEREGISTER(process_exit, pmc_exit_tag);
4075
4076 /* send SIGBUS to all owner threads, free up allocations */
4077 if (pmc_ownerhash)
4078 for (ph = pmc_ownerhash;
4079 ph <= &pmc_ownerhash[pmc_ownerhashmask];
4080 ph++) {
4081 LIST_FOREACH_SAFE(po, ph, po_next, tmp) {
4082 pmc_remove_owner(po);
4083
4084 /* send SIGBUS to owner processes */
4085 PMCDBG(MOD,INI,2, "cleanup signal proc=%p "
4086 "(%d, %s)", po->po_owner,
4087 po->po_owner->p_pid,
4088 po->po_owner->p_comm);
4089
4090 PROC_LOCK(po->po_owner);
4091 psignal(po->po_owner, SIGBUS);
4092 PROC_UNLOCK(po->po_owner);
4093
4094 pmc_destroy_owner_descriptor(po);
4095 }
4096 }
4097
4098 /* reclaim allocated data structures */
4099 if (pmc_mtxpool)
4100 mtx_pool_destroy(&pmc_mtxpool);
4101
4102 mtx_destroy(&pmc_processhash_mtx);
4103 if (pmc_processhash) {
4104#if DEBUG
4105 struct pmc_process *pp;
4106
4107 PMCDBG(MOD,INI,3, "%s", "destroy process hash");
4108 for (prh = pmc_processhash;
4109 prh <= &pmc_processhash[pmc_processhashmask];
4110 prh++)
4111 LIST_FOREACH(pp, prh, pp_next)
4112 PMCDBG(MOD,INI,3, "pid=%d", pp->pp_proc->p_pid);
4113#endif
4114
4115 hashdestroy(pmc_processhash, M_PMC, pmc_processhashmask);
4116 pmc_processhash = NULL;
4117 }
4118
4119 if (pmc_ownerhash) {
4120 PMCDBG(MOD,INI,3, "%s", "destroy owner hash");
4121 hashdestroy(pmc_ownerhash, M_PMC, pmc_ownerhashmask);
4122 pmc_ownerhash = NULL;
4123 }
4124
4125 KASSERT(LIST_EMPTY(&pmc_ss_owners),
4126 ("[pmc,%d] Global SS owner list not empty", __LINE__));
4127 KASSERT(pmc_ss_count == 0,
4128 ("[pmc,%d] Global SS count not empty", __LINE__));
4129
4130 /* do processor dependent cleanup */
4131 PMCDBG(MOD,INI,3, "%s", "md cleanup");
4132 if (md) {
4133 pmc_save_cpu_binding(&pb);
4134 for (cpu = 0; cpu < mp_ncpus; cpu++) {
4135 PMCDBG(MOD,INI,1,"pmc-cleanup cpu=%d pcs=%p",
4136 cpu, pmc_pcpu[cpu]);
4137 if (pmc_cpu_is_disabled(cpu))
4138 continue;
4139 pmc_select_cpu(cpu);
4140 if (pmc_pcpu[cpu])
4141 (void) md->pmd_cleanup(cpu);
4142 }
4143 FREE(md, M_PMC);
4144 md = NULL;
4145 pmc_restore_cpu_binding(&pb);
4146 }
4147
4148 /* deallocate per-cpu structures */
4149 FREE(pmc_pcpu, M_PMC);
4150 pmc_pcpu = NULL;
4151
4152 FREE(pmc_pcpu_saved, M_PMC);
4153 pmc_pcpu_saved = NULL;
4154
4155 if (pmc_pmcdisp) {
4156 FREE(pmc_pmcdisp, M_PMC);
4157 pmc_pmcdisp = NULL;
4158 }
4159
4160 pmclog_shutdown();
4161
4162 sx_xunlock(&pmc_sx); /* we are done */
4163}
4164
4165/*
4166 * The function called at load/unload.
4167 */
4168
4169static int
4170load (struct module *module __unused, int cmd, void *arg __unused)
4171{
4172 int error;
4173
4174 error = 0;
4175
4176 switch (cmd) {
4177 case MOD_LOAD :
4178 /* initialize the subsystem */
4179 error = pmc_initialize();
4180 if (error != 0)
4181 break;
4182 PMCDBG(MOD,INI,1, "syscall=%d ncpus=%d",
4183 pmc_syscall_num, mp_ncpus);
4184 break;
4185
4186
4187 case MOD_UNLOAD :
4188 case MOD_SHUTDOWN:
4189 pmc_cleanup();
4190 PMCDBG(MOD,INI,1, "%s", "unloaded");
4191 break;
4192
4193 default :
4194 error = EINVAL; /* XXX should panic(9) */
4195 break;
4196 }
4197
4198 return error;
4199}
4200
4201/* memory pool */
4202MALLOC_DEFINE(M_PMC, "pmc", "Memory space for the PMC module");