1/*-
2 * SPDX-License-Identifier: BSD-2-Clause-FreeBSD
3 *
4 * Copyright (c) 2012 Fabien Thomas
5 * All rights reserved.
6 *
7 * Redistribution and use in source and binary forms, with or without
8 * modification, are permitted provided that the following conditions
9 * are met:
10 * 1. Redistributions of source code must retain the above copyright
11 *    notice, this list of conditions and the following disclaimer.
12 * 2. Redistributions in binary form must reproduce the above copyright
13 *    notice, this list of conditions and the following disclaimer in the
14 *    documentation and/or other materials provided with the distribution.
15 *
16 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
17 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
18 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
19 * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
20 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
21 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
22 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
23 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
24 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
25 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
26 * SUCH DAMAGE.
27 */
28
29#include <sys/cdefs.h>
30__FBSDID("$FreeBSD$");
31
32#include <sys/param.h>
33#include <sys/pmc.h>
34#include <sys/pmckern.h>
35#include <sys/systm.h>
36#include <sys/mutex.h>
37
38#include <machine/cpu.h>
39#include <machine/cpufunc.h>
40
41#include "hwpmc_soft.h"
42
43/*
44 * Software PMC support.
45 */
46
47#define	SOFT_CAPS (PMC_CAP_READ | PMC_CAP_WRITE | PMC_CAP_INTERRUPT | \
48    PMC_CAP_USER | PMC_CAP_SYSTEM)
49
50struct soft_descr {
51	struct pmc_descr pm_descr;  /* "base class" */
52};
53
54static struct soft_descr soft_pmcdesc[SOFT_NPMCS] =
55{
56#define	SOFT_PMCDESCR(N)				\
57	{						\
58		.pm_descr =				\
59		{					\
60			.pd_name = #N,			\
61			.pd_class = PMC_CLASS_SOFT,	\
62			.pd_caps = SOFT_CAPS,		\
63			.pd_width = 64			\
64		},					\
65	}
66
67	SOFT_PMCDESCR(SOFT0),
68	SOFT_PMCDESCR(SOFT1),
69	SOFT_PMCDESCR(SOFT2),
70	SOFT_PMCDESCR(SOFT3),
71	SOFT_PMCDESCR(SOFT4),
72	SOFT_PMCDESCR(SOFT5),
73	SOFT_PMCDESCR(SOFT6),
74	SOFT_PMCDESCR(SOFT7),
75	SOFT_PMCDESCR(SOFT8),
76	SOFT_PMCDESCR(SOFT9),
77	SOFT_PMCDESCR(SOFT10),
78	SOFT_PMCDESCR(SOFT11),
79	SOFT_PMCDESCR(SOFT12),
80	SOFT_PMCDESCR(SOFT13),
81	SOFT_PMCDESCR(SOFT14),
82	SOFT_PMCDESCR(SOFT15)
83};
84
85/*
86 * Per-CPU data structure.
87 */
88
89struct soft_cpu {
90	struct pmc_hw	soft_hw[SOFT_NPMCS];
91	pmc_value_t	soft_values[SOFT_NPMCS];
92};
93
94
95static struct soft_cpu **soft_pcpu;
96
97static int
98soft_allocate_pmc(int cpu, int ri, struct pmc *pm,
99    const struct pmc_op_pmcallocate *a)
100{
101	enum pmc_event ev;
102	struct pmc_soft *ps;
103
104	(void) cpu;
105
106	KASSERT(cpu >= 0 && cpu < pmc_cpu_max(),
107	    ("[soft,%d] illegal CPU value %d", __LINE__, cpu));
108	KASSERT(ri >= 0 && ri < SOFT_NPMCS,
109	    ("[soft,%d] illegal row-index %d", __LINE__, ri));
110
111	if (a->pm_class != PMC_CLASS_SOFT)
112		return (EINVAL);
113
114	if ((pm->pm_caps & SOFT_CAPS) == 0)
115		return (EINVAL);
116
117	if ((pm->pm_caps & ~SOFT_CAPS) != 0)
118		return (EPERM);
119
120	ev = pm->pm_event;
121	if ((int)ev < PMC_EV_SOFT_FIRST || (int)ev > PMC_EV_SOFT_LAST)
122		return (EINVAL);
123
124	/* Check if event is registered. */
125	ps = pmc_soft_ev_acquire(ev);
126	if (ps == NULL)
127		return (EINVAL);
128	pmc_soft_ev_release(ps);
129	/* Module unload is protected by pmc SX lock. */
130	if (ps->ps_alloc != NULL)
131		ps->ps_alloc();
132
133	return (0);
134}
135
136static int
137soft_config_pmc(int cpu, int ri, struct pmc *pm)
138{
139	struct pmc_hw *phw;
140
141	PMCDBG3(MDP,CFG,1, "cpu=%d ri=%d pm=%p", cpu, ri, pm);
142
143	KASSERT(cpu >= 0 && cpu < pmc_cpu_max(),
144	    ("[soft,%d] illegal CPU value %d", __LINE__, cpu));
145	KASSERT(ri >= 0 && ri < SOFT_NPMCS,
146	    ("[soft,%d] illegal row-index %d", __LINE__, ri));
147
148	phw = &soft_pcpu[cpu]->soft_hw[ri];
149
150	KASSERT(pm == NULL || phw->phw_pmc == NULL,
151	    ("[soft,%d] pm=%p phw->pm=%p hwpmc not unconfigured", __LINE__,
152	    pm, phw->phw_pmc));
153
154	phw->phw_pmc = pm;
155
156	return (0);
157}
158
159static int
160soft_describe(int cpu, int ri, struct pmc_info *pi, struct pmc **ppmc)
161{
162	int error;
163	size_t copied;
164	const struct soft_descr *pd;
165	struct pmc_hw *phw;
166
167	KASSERT(cpu >= 0 && cpu < pmc_cpu_max(),
168	    ("[soft,%d] illegal CPU %d", __LINE__, cpu));
169	KASSERT(ri >= 0 && ri < SOFT_NPMCS,
170	    ("[soft,%d] illegal row-index %d", __LINE__, ri));
171
172	phw = &soft_pcpu[cpu]->soft_hw[ri];
173	pd  = &soft_pmcdesc[ri];
174
175	if ((error = copystr(pd->pm_descr.pd_name, pi->pm_name,
176	    PMC_NAME_MAX, &copied)) != 0)
177		return (error);
178
179	pi->pm_class = pd->pm_descr.pd_class;
180
181	if (phw->phw_state & PMC_PHW_FLAG_IS_ENABLED) {
182		pi->pm_enabled = TRUE;
183		*ppmc          = phw->phw_pmc;
184	} else {
185		pi->pm_enabled = FALSE;
186		*ppmc          = NULL;
187	}
188
189	return (0);
190}
191
192static int
193soft_get_config(int cpu, int ri, struct pmc **ppm)
194{
195	(void) ri;
196
197	KASSERT(cpu >= 0 && cpu < pmc_cpu_max(),
198	    ("[soft,%d] illegal CPU %d", __LINE__, cpu));
199	KASSERT(ri >= 0 && ri < SOFT_NPMCS,
200	    ("[soft,%d] illegal row-index %d", __LINE__, ri));
201
202	*ppm = soft_pcpu[cpu]->soft_hw[ri].phw_pmc;
203	return (0);
204}
205
206static int
207soft_pcpu_fini(struct pmc_mdep *md, int cpu)
208{
209	int ri;
210	struct pmc_cpu *pc;
211
212	KASSERT(cpu >= 0 && cpu < pmc_cpu_max(),
213	    ("[soft,%d] illegal cpu %d", __LINE__, cpu));
214	KASSERT(soft_pcpu[cpu] != NULL, ("[soft,%d] null pcpu", __LINE__));
215
216	free(soft_pcpu[cpu], M_PMC);
217	soft_pcpu[cpu] = NULL;
218
219	ri = md->pmd_classdep[PMC_CLASS_INDEX_SOFT].pcd_ri;
220
221	KASSERT(ri >= 0 && ri < SOFT_NPMCS,
222	    ("[soft,%d] ri=%d", __LINE__, ri));
223
224	pc = pmc_pcpu[cpu];
225	pc->pc_hwpmcs[ri] = NULL;
226
227	return (0);
228}
229
230static int
231soft_pcpu_init(struct pmc_mdep *md, int cpu)
232{
233	int first_ri, n;
234	struct pmc_cpu *pc;
235	struct soft_cpu *soft_pc;
236	struct pmc_hw *phw;
237
238
239	KASSERT(cpu >= 0 && cpu < pmc_cpu_max(),
240	    ("[soft,%d] illegal cpu %d", __LINE__, cpu));
241	KASSERT(soft_pcpu, ("[soft,%d] null pcpu", __LINE__));
242	KASSERT(soft_pcpu[cpu] == NULL, ("[soft,%d] non-null per-cpu",
243	    __LINE__));
244
245	soft_pc = malloc(sizeof(struct soft_cpu), M_PMC, M_WAITOK|M_ZERO);
246	pc = pmc_pcpu[cpu];
247
248	KASSERT(pc != NULL, ("[soft,%d] cpu %d null per-cpu", __LINE__, cpu));
249
250	soft_pcpu[cpu] = soft_pc;
251	phw = soft_pc->soft_hw;
252	first_ri = md->pmd_classdep[PMC_CLASS_INDEX_SOFT].pcd_ri;
253
254	for (n = 0; n < SOFT_NPMCS; n++, phw++) {
255		phw->phw_state = PMC_PHW_FLAG_IS_ENABLED |
256		    PMC_PHW_CPU_TO_STATE(cpu) | PMC_PHW_INDEX_TO_STATE(n);
257		phw->phw_pmc = NULL;
258		pc->pc_hwpmcs[n + first_ri] = phw;
259	}
260
261	return (0);
262}
263
264static int
265soft_read_pmc(int cpu, int ri, pmc_value_t *v)
266{
267	struct pmc *pm;
268	const struct pmc_hw *phw;
269
270	KASSERT(cpu >= 0 && cpu < pmc_cpu_max(),
271	    ("[soft,%d] illegal CPU value %d", __LINE__, cpu));
272	KASSERT(ri >= 0 && ri < SOFT_NPMCS,
273	    ("[soft,%d] illegal row-index %d", __LINE__, ri));
274
275	phw = &soft_pcpu[cpu]->soft_hw[ri];
276	pm  = phw->phw_pmc;
277
278	KASSERT(pm != NULL,
279	    ("[soft,%d] no owner for PHW [cpu%d,pmc%d]", __LINE__, cpu, ri));
280
281	PMCDBG1(MDP,REA,1,"soft-read id=%d", ri);
282
283	*v = soft_pcpu[cpu]->soft_values[ri];
284
285	return (0);
286}
287
288static int
289soft_write_pmc(int cpu, int ri, pmc_value_t v)
290{
291	struct pmc *pm;
292	const struct soft_descr *pd;
293
294	KASSERT(cpu >= 0 && cpu < pmc_cpu_max(),
295	    ("[soft,%d] illegal cpu value %d", __LINE__, cpu));
296	KASSERT(ri >= 0 && ri < SOFT_NPMCS,
297	    ("[soft,%d] illegal row-index %d", __LINE__, ri));
298
299	pm = soft_pcpu[cpu]->soft_hw[ri].phw_pmc;
300	pd = &soft_pmcdesc[ri];
301
302	KASSERT(pm,
303	    ("[soft,%d] cpu %d ri %d pmc not configured", __LINE__, cpu, ri));
304
305	PMCDBG3(MDP,WRI,1, "soft-write cpu=%d ri=%d v=%jx", cpu, ri, v);
306
307	soft_pcpu[cpu]->soft_values[ri] = v;
308
309	return (0);
310}
311
312static int
313soft_release_pmc(int cpu, int ri, struct pmc *pmc)
314{
315	struct pmc_hw *phw;
316	enum pmc_event ev;
317	struct pmc_soft *ps;
318
319	(void) pmc;
320
321	KASSERT(cpu >= 0 && cpu < pmc_cpu_max(),
322	    ("[soft,%d] illegal CPU value %d", __LINE__, cpu));
323	KASSERT(ri >= 0 && ri < SOFT_NPMCS,
324	    ("[soft,%d] illegal row-index %d", __LINE__, ri));
325
326	phw = &soft_pcpu[cpu]->soft_hw[ri];
327
328	KASSERT(phw->phw_pmc == NULL,
329	    ("[soft,%d] PHW pmc %p non-NULL", __LINE__, phw->phw_pmc));
330
331	ev = pmc->pm_event;
332
333	/* Check if event is registered. */
334	ps = pmc_soft_ev_acquire(ev);
335	KASSERT(ps != NULL,
336	    ("[soft,%d] unregistered event %d", __LINE__, ev));
337	pmc_soft_ev_release(ps);
338	/* Module unload is protected by pmc SX lock. */
339	if (ps->ps_release != NULL)
340		ps->ps_release();
341	return (0);
342}
343
344static int
345soft_start_pmc(int cpu, int ri)
346{
347	struct pmc *pm;
348	struct soft_cpu *pc;
349	struct pmc_soft *ps;
350
351	KASSERT(cpu >= 0 && cpu < pmc_cpu_max(),
352	    ("[soft,%d] illegal CPU value %d", __LINE__, cpu));
353	KASSERT(ri >= 0 && ri < SOFT_NPMCS,
354	    ("[soft,%d] illegal row-index %d", __LINE__, ri));
355
356	pc = soft_pcpu[cpu];
357	pm = pc->soft_hw[ri].phw_pmc;
358
359	KASSERT(pm,
360	    ("[soft,%d] cpu %d ri %d pmc not configured", __LINE__, cpu, ri));
361
362	ps = pmc_soft_ev_acquire(pm->pm_event);
363	if (ps == NULL)
364		return (EINVAL);
365	atomic_add_int(&ps->ps_running, 1);
366	pmc_soft_ev_release(ps);
367
368	return (0);
369}
370
371static int
372soft_stop_pmc(int cpu, int ri)
373{
374	struct pmc *pm;
375	struct soft_cpu *pc;
376	struct pmc_soft *ps;
377
378	KASSERT(cpu >= 0 && cpu < pmc_cpu_max(),
379	    ("[soft,%d] illegal CPU value %d", __LINE__, cpu));
380	KASSERT(ri >= 0 && ri < SOFT_NPMCS,
381	    ("[soft,%d] illegal row-index %d", __LINE__, ri));
382
383	pc = soft_pcpu[cpu];
384	pm = pc->soft_hw[ri].phw_pmc;
385
386	KASSERT(pm,
387	    ("[soft,%d] cpu %d ri %d pmc not configured", __LINE__, cpu, ri));
388
389	ps = pmc_soft_ev_acquire(pm->pm_event);
390	/* event unregistered ? */
391	if (ps != NULL) {
392		atomic_subtract_int(&ps->ps_running, 1);
393		pmc_soft_ev_release(ps);
394	}
395
396	return (0);
397}
398
399int
400pmc_soft_intr(struct pmckern_soft *ks)
401{
402	struct pmc *pm;
403	struct soft_cpu *pc;
404	int ri, processed, error, user_mode;
405
406	KASSERT(ks->pm_cpu >= 0 && ks->pm_cpu < pmc_cpu_max(),
407	    ("[soft,%d] CPU %d out of range", __LINE__, ks->pm_cpu));
408
409	processed = 0;
410	pc = soft_pcpu[ks->pm_cpu];
411
412	for (ri = 0; ri < SOFT_NPMCS; ri++) {
413
414		pm = pc->soft_hw[ri].phw_pmc;
415		if (pm == NULL ||
416		    pm->pm_state != PMC_STATE_RUNNING ||
417		    pm->pm_event != ks->pm_ev) {
418				continue;
419		}
420
421		processed = 1;
422		if (PMC_IS_SAMPLING_MODE(PMC_TO_MODE(pm))) {
423			if ((pc->soft_values[ri]--) <= 0)
424				pc->soft_values[ri] += pm->pm_sc.pm_reloadcount;
425			else
426				continue;
427			user_mode = TRAPF_USERMODE(ks->pm_tf);
428			error = pmc_process_interrupt(PMC_SR, pm, ks->pm_tf);
429			if (error) {
430				soft_stop_pmc(ks->pm_cpu, ri);
431				continue;
432			}
433
434			if (user_mode) {
435				/* If in user mode setup AST to process
436				 * callchain out of interrupt context.
437				 */
438				curthread->td_flags |= TDF_ASTPENDING;
439			}
440		} else
441			pc->soft_values[ri]++;
442	}
443	if (processed)
444		counter_u64_add(pmc_stats.pm_intr_processed, 1);
445	else
446		counter_u64_add(pmc_stats.pm_intr_ignored, 1);
447
448	return (processed);
449}
450
451void
452pmc_soft_initialize(struct pmc_mdep *md)
453{
454	struct pmc_classdep *pcd;
455
456	/* Add SOFT PMCs. */
457	soft_pcpu = malloc(sizeof(struct soft_cpu *) * pmc_cpu_max(), M_PMC,
458	    M_ZERO|M_WAITOK);
459
460	pcd = &md->pmd_classdep[PMC_CLASS_INDEX_SOFT];
461
462	pcd->pcd_caps	= SOFT_CAPS;
463	pcd->pcd_class	= PMC_CLASS_SOFT;
464	pcd->pcd_num	= SOFT_NPMCS;
465	pcd->pcd_ri	= md->pmd_npmc;
466	pcd->pcd_width	= 64;
467
468	pcd->pcd_allocate_pmc = soft_allocate_pmc;
469	pcd->pcd_config_pmc   = soft_config_pmc;
470	pcd->pcd_describe     = soft_describe;
471	pcd->pcd_get_config   = soft_get_config;
472	pcd->pcd_get_msr      = NULL;
473	pcd->pcd_pcpu_init    = soft_pcpu_init;
474	pcd->pcd_pcpu_fini    = soft_pcpu_fini;
475	pcd->pcd_read_pmc     = soft_read_pmc;
476	pcd->pcd_write_pmc    = soft_write_pmc;
477	pcd->pcd_release_pmc  = soft_release_pmc;
478	pcd->pcd_start_pmc    = soft_start_pmc;
479	pcd->pcd_stop_pmc     = soft_stop_pmc;
480
481	md->pmd_npmc += SOFT_NPMCS;
482}
483
484void
485pmc_soft_finalize(struct pmc_mdep *md)
486{
487#ifdef	INVARIANTS
488	int i, ncpus;
489
490	ncpus = pmc_cpu_max();
491	for (i = 0; i < ncpus; i++)
492		KASSERT(soft_pcpu[i] == NULL, ("[soft,%d] non-null pcpu cpu %d",
493		    __LINE__, i));
494
495	KASSERT(md->pmd_classdep[PMC_CLASS_INDEX_SOFT].pcd_class ==
496	    PMC_CLASS_SOFT, ("[soft,%d] class mismatch", __LINE__));
497#endif
498	free(soft_pcpu, M_PMC);
499	soft_pcpu = NULL;
500}
501