1/*-
2 * Copyright (c) 2012 Fabien Thomas
3 * All rights reserved.
4 *
5 * Redistribution and use in source and binary forms, with or without
6 * modification, are permitted provided that the following conditions
7 * are met:
8 * 1. Redistributions of source code must retain the above copyright
9 *    notice, this list of conditions and the following disclaimer.
10 * 2. Redistributions in binary form must reproduce the above copyright
11 *    notice, this list of conditions and the following disclaimer in the
12 *    documentation and/or other materials provided with the distribution.
13 *
14 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
15 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
16 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
17 * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
18 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
19 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
20 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
21 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
22 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
23 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
24 * SUCH DAMAGE.
25 */
26
27#include <sys/cdefs.h>
28__FBSDID("$FreeBSD$");
29
30#include <sys/param.h>
31#include <sys/pmc.h>
32#include <sys/pmckern.h>
33#include <sys/systm.h>
34#include <sys/mutex.h>
35
36#include <machine/cpu.h>
37#include <machine/cpufunc.h>
38
39#include "hwpmc_soft.h"
40
41/*
42 * Software PMC support.
43 */
44
45#define	SOFT_CAPS (PMC_CAP_READ | PMC_CAP_WRITE | PMC_CAP_INTERRUPT | \
46    PMC_CAP_USER | PMC_CAP_SYSTEM)
47
48struct soft_descr {
49	struct pmc_descr pm_descr;  /* "base class" */
50};
51
52static struct soft_descr soft_pmcdesc[SOFT_NPMCS] =
53{
54#define	SOFT_PMCDESCR(N)				\
55	{						\
56		.pm_descr =				\
57		{					\
58			.pd_name = #N,			\
59			.pd_class = PMC_CLASS_SOFT,	\
60			.pd_caps = SOFT_CAPS,		\
61			.pd_width = 64			\
62		},					\
63	}
64
65	SOFT_PMCDESCR(SOFT0),
66	SOFT_PMCDESCR(SOFT1),
67	SOFT_PMCDESCR(SOFT2),
68	SOFT_PMCDESCR(SOFT3),
69	SOFT_PMCDESCR(SOFT4),
70	SOFT_PMCDESCR(SOFT5),
71	SOFT_PMCDESCR(SOFT6),
72	SOFT_PMCDESCR(SOFT7),
73	SOFT_PMCDESCR(SOFT8),
74	SOFT_PMCDESCR(SOFT9),
75	SOFT_PMCDESCR(SOFT10),
76	SOFT_PMCDESCR(SOFT11),
77	SOFT_PMCDESCR(SOFT12),
78	SOFT_PMCDESCR(SOFT13),
79	SOFT_PMCDESCR(SOFT14),
80	SOFT_PMCDESCR(SOFT15)
81};
82
83/*
84 * Per-CPU data structure.
85 */
86
87struct soft_cpu {
88	struct pmc_hw	soft_hw[SOFT_NPMCS];
89	pmc_value_t	soft_values[SOFT_NPMCS];
90};
91
92
93static struct soft_cpu **soft_pcpu;
94
95static int
96soft_allocate_pmc(int cpu, int ri, struct pmc *pm,
97    const struct pmc_op_pmcallocate *a)
98{
99	enum pmc_event ev;
100	struct pmc_soft *ps;
101
102	(void) cpu;
103
104	KASSERT(cpu >= 0 && cpu < pmc_cpu_max(),
105	    ("[soft,%d] illegal CPU value %d", __LINE__, cpu));
106	KASSERT(ri >= 0 && ri < SOFT_NPMCS,
107	    ("[soft,%d] illegal row-index %d", __LINE__, ri));
108
109	if (a->pm_class != PMC_CLASS_SOFT)
110		return (EINVAL);
111
112	if ((pm->pm_caps & SOFT_CAPS) == 0)
113		return (EINVAL);
114
115	if ((pm->pm_caps & ~SOFT_CAPS) != 0)
116		return (EPERM);
117
118	ev = pm->pm_event;
119	if ((int)ev < PMC_EV_SOFT_FIRST || (int)ev > PMC_EV_SOFT_LAST)
120		return (EINVAL);
121
122	/* Check if event is registered. */
123	ps = pmc_soft_ev_acquire(ev);
124	if (ps == NULL)
125		return (EINVAL);
126	pmc_soft_ev_release(ps);
127	/* Module unload is protected by pmc SX lock. */
128	if (ps->ps_alloc != NULL)
129		ps->ps_alloc();
130
131	return (0);
132}
133
134static int
135soft_config_pmc(int cpu, int ri, struct pmc *pm)
136{
137	struct pmc_hw *phw;
138
139	PMCDBG3(MDP,CFG,1, "cpu=%d ri=%d pm=%p", cpu, ri, pm);
140
141	KASSERT(cpu >= 0 && cpu < pmc_cpu_max(),
142	    ("[soft,%d] illegal CPU value %d", __LINE__, cpu));
143	KASSERT(ri >= 0 && ri < SOFT_NPMCS,
144	    ("[soft,%d] illegal row-index %d", __LINE__, ri));
145
146	phw = &soft_pcpu[cpu]->soft_hw[ri];
147
148	KASSERT(pm == NULL || phw->phw_pmc == NULL,
149	    ("[soft,%d] pm=%p phw->pm=%p hwpmc not unconfigured", __LINE__,
150	    pm, phw->phw_pmc));
151
152	phw->phw_pmc = pm;
153
154	return (0);
155}
156
157static int
158soft_describe(int cpu, int ri, struct pmc_info *pi, struct pmc **ppmc)
159{
160	int error;
161	size_t copied;
162	const struct soft_descr *pd;
163	struct pmc_hw *phw;
164
165	KASSERT(cpu >= 0 && cpu < pmc_cpu_max(),
166	    ("[soft,%d] illegal CPU %d", __LINE__, cpu));
167	KASSERT(ri >= 0 && ri < SOFT_NPMCS,
168	    ("[soft,%d] illegal row-index %d", __LINE__, ri));
169
170	phw = &soft_pcpu[cpu]->soft_hw[ri];
171	pd  = &soft_pmcdesc[ri];
172
173	if ((error = copystr(pd->pm_descr.pd_name, pi->pm_name,
174	    PMC_NAME_MAX, &copied)) != 0)
175		return (error);
176
177	pi->pm_class = pd->pm_descr.pd_class;
178
179	if (phw->phw_state & PMC_PHW_FLAG_IS_ENABLED) {
180		pi->pm_enabled = TRUE;
181		*ppmc          = phw->phw_pmc;
182	} else {
183		pi->pm_enabled = FALSE;
184		*ppmc          = NULL;
185	}
186
187	return (0);
188}
189
190static int
191soft_get_config(int cpu, int ri, struct pmc **ppm)
192{
193	(void) ri;
194
195	KASSERT(cpu >= 0 && cpu < pmc_cpu_max(),
196	    ("[soft,%d] illegal CPU %d", __LINE__, cpu));
197	KASSERT(ri >= 0 && ri < SOFT_NPMCS,
198	    ("[soft,%d] illegal row-index %d", __LINE__, ri));
199
200	*ppm = soft_pcpu[cpu]->soft_hw[ri].phw_pmc;
201	return (0);
202}
203
204static int
205soft_pcpu_fini(struct pmc_mdep *md, int cpu)
206{
207	int ri;
208	struct pmc_cpu *pc;
209
210	KASSERT(cpu >= 0 && cpu < pmc_cpu_max(),
211	    ("[soft,%d] illegal cpu %d", __LINE__, cpu));
212	KASSERT(soft_pcpu[cpu] != NULL, ("[soft,%d] null pcpu", __LINE__));
213
214	free(soft_pcpu[cpu], M_PMC);
215	soft_pcpu[cpu] = NULL;
216
217	ri = md->pmd_classdep[PMC_CLASS_INDEX_SOFT].pcd_ri;
218
219	KASSERT(ri >= 0 && ri < SOFT_NPMCS,
220	    ("[soft,%d] ri=%d", __LINE__, ri));
221
222	pc = pmc_pcpu[cpu];
223	pc->pc_hwpmcs[ri] = NULL;
224
225	return (0);
226}
227
228static int
229soft_pcpu_init(struct pmc_mdep *md, int cpu)
230{
231	int first_ri, n;
232	struct pmc_cpu *pc;
233	struct soft_cpu *soft_pc;
234	struct pmc_hw *phw;
235
236
237	KASSERT(cpu >= 0 && cpu < pmc_cpu_max(),
238	    ("[soft,%d] illegal cpu %d", __LINE__, cpu));
239	KASSERT(soft_pcpu, ("[soft,%d] null pcpu", __LINE__));
240	KASSERT(soft_pcpu[cpu] == NULL, ("[soft,%d] non-null per-cpu",
241	    __LINE__));
242
243	soft_pc = malloc(sizeof(struct soft_cpu), M_PMC, M_WAITOK|M_ZERO);
244	pc = pmc_pcpu[cpu];
245
246	KASSERT(pc != NULL, ("[soft,%d] cpu %d null per-cpu", __LINE__, cpu));
247
248	soft_pcpu[cpu] = soft_pc;
249	phw = soft_pc->soft_hw;
250	first_ri = md->pmd_classdep[PMC_CLASS_INDEX_SOFT].pcd_ri;
251
252	for (n = 0; n < SOFT_NPMCS; n++, phw++) {
253		phw->phw_state = PMC_PHW_FLAG_IS_ENABLED |
254		    PMC_PHW_CPU_TO_STATE(cpu) | PMC_PHW_INDEX_TO_STATE(n);
255		phw->phw_pmc = NULL;
256		pc->pc_hwpmcs[n + first_ri] = phw;
257	}
258
259	return (0);
260}
261
262static int
263soft_read_pmc(int cpu, int ri, pmc_value_t *v)
264{
265	struct pmc *pm;
266	const struct pmc_hw *phw;
267
268	KASSERT(cpu >= 0 && cpu < pmc_cpu_max(),
269	    ("[soft,%d] illegal CPU value %d", __LINE__, cpu));
270	KASSERT(ri >= 0 && ri < SOFT_NPMCS,
271	    ("[soft,%d] illegal row-index %d", __LINE__, ri));
272
273	phw = &soft_pcpu[cpu]->soft_hw[ri];
274	pm  = phw->phw_pmc;
275
276	KASSERT(pm != NULL,
277	    ("[soft,%d] no owner for PHW [cpu%d,pmc%d]", __LINE__, cpu, ri));
278
279	PMCDBG1(MDP,REA,1,"soft-read id=%d", ri);
280
281	*v = soft_pcpu[cpu]->soft_values[ri];
282
283	return (0);
284}
285
286static int
287soft_write_pmc(int cpu, int ri, pmc_value_t v)
288{
289	struct pmc *pm;
290	const struct soft_descr *pd;
291
292	KASSERT(cpu >= 0 && cpu < pmc_cpu_max(),
293	    ("[soft,%d] illegal cpu value %d", __LINE__, cpu));
294	KASSERT(ri >= 0 && ri < SOFT_NPMCS,
295	    ("[soft,%d] illegal row-index %d", __LINE__, ri));
296
297	pm = soft_pcpu[cpu]->soft_hw[ri].phw_pmc;
298	pd = &soft_pmcdesc[ri];
299
300	KASSERT(pm,
301	    ("[soft,%d] cpu %d ri %d pmc not configured", __LINE__, cpu, ri));
302
303	PMCDBG3(MDP,WRI,1, "soft-write cpu=%d ri=%d v=%jx", cpu, ri, v);
304
305	soft_pcpu[cpu]->soft_values[ri] = v;
306
307	return (0);
308}
309
310static int
311soft_release_pmc(int cpu, int ri, struct pmc *pmc)
312{
313	struct pmc_hw *phw;
314	enum pmc_event ev;
315	struct pmc_soft *ps;
316
317	(void) pmc;
318
319	KASSERT(cpu >= 0 && cpu < pmc_cpu_max(),
320	    ("[soft,%d] illegal CPU value %d", __LINE__, cpu));
321	KASSERT(ri >= 0 && ri < SOFT_NPMCS,
322	    ("[soft,%d] illegal row-index %d", __LINE__, ri));
323
324	phw = &soft_pcpu[cpu]->soft_hw[ri];
325
326	KASSERT(phw->phw_pmc == NULL,
327	    ("[soft,%d] PHW pmc %p non-NULL", __LINE__, phw->phw_pmc));
328
329	ev = pmc->pm_event;
330
331	/* Check if event is registered. */
332	ps = pmc_soft_ev_acquire(ev);
333	KASSERT(ps != NULL,
334	    ("[soft,%d] unregistered event %d", __LINE__, ev));
335	pmc_soft_ev_release(ps);
336	/* Module unload is protected by pmc SX lock. */
337	if (ps->ps_release != NULL)
338		ps->ps_release();
339	return (0);
340}
341
342static int
343soft_start_pmc(int cpu, int ri)
344{
345	struct pmc *pm;
346	struct soft_cpu *pc;
347	struct pmc_soft *ps;
348
349	KASSERT(cpu >= 0 && cpu < pmc_cpu_max(),
350	    ("[soft,%d] illegal CPU value %d", __LINE__, cpu));
351	KASSERT(ri >= 0 && ri < SOFT_NPMCS,
352	    ("[soft,%d] illegal row-index %d", __LINE__, ri));
353
354	pc = soft_pcpu[cpu];
355	pm = pc->soft_hw[ri].phw_pmc;
356
357	KASSERT(pm,
358	    ("[soft,%d] cpu %d ri %d pmc not configured", __LINE__, cpu, ri));
359
360	ps = pmc_soft_ev_acquire(pm->pm_event);
361	if (ps == NULL)
362		return (EINVAL);
363	atomic_add_int(&ps->ps_running, 1);
364	pmc_soft_ev_release(ps);
365
366	return (0);
367}
368
369static int
370soft_stop_pmc(int cpu, int ri)
371{
372	struct pmc *pm;
373	struct soft_cpu *pc;
374	struct pmc_soft *ps;
375
376	KASSERT(cpu >= 0 && cpu < pmc_cpu_max(),
377	    ("[soft,%d] illegal CPU value %d", __LINE__, cpu));
378	KASSERT(ri >= 0 && ri < SOFT_NPMCS,
379	    ("[soft,%d] illegal row-index %d", __LINE__, ri));
380
381	pc = soft_pcpu[cpu];
382	pm = pc->soft_hw[ri].phw_pmc;
383
384	KASSERT(pm,
385	    ("[soft,%d] cpu %d ri %d pmc not configured", __LINE__, cpu, ri));
386
387	ps = pmc_soft_ev_acquire(pm->pm_event);
388	/* event unregistered ? */
389	if (ps != NULL) {
390		atomic_subtract_int(&ps->ps_running, 1);
391		pmc_soft_ev_release(ps);
392	}
393
394	return (0);
395}
396
397int
398pmc_soft_intr(struct pmckern_soft *ks)
399{
400	struct pmc *pm;
401	struct soft_cpu *pc;
402	int ri, processed, error, user_mode;
403
404	KASSERT(ks->pm_cpu >= 0 && ks->pm_cpu < pmc_cpu_max(),
405	    ("[soft,%d] CPU %d out of range", __LINE__, ks->pm_cpu));
406
407	processed = 0;
408	pc = soft_pcpu[ks->pm_cpu];
409
410	for (ri = 0; ri < SOFT_NPMCS; ri++) {
411
412		pm = pc->soft_hw[ri].phw_pmc;
413		if (pm == NULL ||
414		    pm->pm_state != PMC_STATE_RUNNING ||
415		    pm->pm_event != ks->pm_ev) {
416				continue;
417		}
418
419		processed = 1;
420		if (PMC_IS_SAMPLING_MODE(PMC_TO_MODE(pm))) {
421			if ((pc->soft_values[ri]--) <= 0)
422				pc->soft_values[ri] += pm->pm_sc.pm_reloadcount;
423			else
424				continue;
425			user_mode = TRAPF_USERMODE(ks->pm_tf);
426			error = pmc_process_interrupt(ks->pm_cpu, PMC_SR, pm,
427			    ks->pm_tf, user_mode);
428			if (error) {
429				soft_stop_pmc(ks->pm_cpu, ri);
430				continue;
431			}
432
433			if (user_mode) {
434				/* If in user mode setup AST to process
435				 * callchain out of interrupt context.
436				 */
437				curthread->td_flags |= TDF_ASTPENDING;
438			}
439		} else
440			pc->soft_values[ri]++;
441	}
442
443	atomic_add_int(processed ? &pmc_stats.pm_intr_processed :
444	    &pmc_stats.pm_intr_ignored, 1);
445
446	return (processed);
447}
448
449void
450pmc_soft_initialize(struct pmc_mdep *md)
451{
452	struct pmc_classdep *pcd;
453
454	/* Add SOFT PMCs. */
455	soft_pcpu = malloc(sizeof(struct soft_cpu *) * pmc_cpu_max(), M_PMC,
456	    M_ZERO|M_WAITOK);
457
458	pcd = &md->pmd_classdep[PMC_CLASS_INDEX_SOFT];
459
460	pcd->pcd_caps	= SOFT_CAPS;
461	pcd->pcd_class	= PMC_CLASS_SOFT;
462	pcd->pcd_num	= SOFT_NPMCS;
463	pcd->pcd_ri	= md->pmd_npmc;
464	pcd->pcd_width	= 64;
465
466	pcd->pcd_allocate_pmc = soft_allocate_pmc;
467	pcd->pcd_config_pmc   = soft_config_pmc;
468	pcd->pcd_describe     = soft_describe;
469	pcd->pcd_get_config   = soft_get_config;
470	pcd->pcd_get_msr      = NULL;
471	pcd->pcd_pcpu_init    = soft_pcpu_init;
472	pcd->pcd_pcpu_fini    = soft_pcpu_fini;
473	pcd->pcd_read_pmc     = soft_read_pmc;
474	pcd->pcd_write_pmc    = soft_write_pmc;
475	pcd->pcd_release_pmc  = soft_release_pmc;
476	pcd->pcd_start_pmc    = soft_start_pmc;
477	pcd->pcd_stop_pmc     = soft_stop_pmc;
478
479	md->pmd_npmc += SOFT_NPMCS;
480}
481
482void
483pmc_soft_finalize(struct pmc_mdep *md)
484{
485#ifdef	INVARIANTS
486	int i, ncpus;
487
488	ncpus = pmc_cpu_max();
489	for (i = 0; i < ncpus; i++)
490		KASSERT(soft_pcpu[i] == NULL, ("[soft,%d] non-null pcpu cpu %d",
491		    __LINE__, i));
492
493	KASSERT(md->pmd_classdep[PMC_CLASS_INDEX_SOFT].pcd_class ==
494	    PMC_CLASS_SOFT, ("[soft,%d] class mismatch", __LINE__));
495#endif
496	free(soft_pcpu, M_PMC);
497	soft_pcpu = NULL;
498}
499