1/*-
2 * SPDX-License-Identifier: BSD-2-Clause-FreeBSD
3 *
4 * Copyright (c) 2011,2013 Justin Hibbits
5 * Copyright (c) 2005, Joseph Koshy
6 * All rights reserved.
7 *
8 * Redistribution and use in source and binary forms, with or without
9 * modification, are permitted provided that the following conditions
10 * are met:
11 * 1. Redistributions of source code must retain the above copyright
12 *    notice, this list of conditions and the following disclaimer.
13 * 2. Redistributions in binary form must reproduce the above copyright
14 *    notice, this list of conditions and the following disclaimer in the
15 *    documentation and/or other materials provided with the distribution.
16 *
17 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
18 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
19 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
20 * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
21 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
22 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
23 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
24 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
25 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
26 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
27 * SUCH DAMAGE.
28 *
29 */
30
31#include <sys/cdefs.h>
32__FBSDID("$FreeBSD$");
33
34#include <sys/param.h>
35#include <sys/pmc.h>
36#include <sys/pmckern.h>
37#include <sys/sysent.h>
38#include <sys/syslog.h>
39#include <sys/systm.h>
40
41#include <machine/pmc_mdep.h>
42#include <machine/spr.h>
43#include <machine/pte.h>
44#include <machine/sr.h>
45#include <machine/cpu.h>
46#include <machine/stack.h>
47
48#include "hwpmc_powerpc.h"
49
50#ifdef __powerpc64__
51#define OFFSET 4 /* Account for the TOC reload slot */
52#else
53#define OFFSET 0
54#endif
55
56struct powerpc_cpu **powerpc_pcpu;
57struct pmc_ppc_event *ppc_event_codes;
58size_t ppc_event_codes_size;
59int ppc_event_first;
60int ppc_event_last;
61int ppc_max_pmcs;
62
63void (*powerpc_set_pmc)(int cpu, int ri, int config);
64pmc_value_t (*powerpc_pmcn_read)(unsigned int pmc);
65void (*powerpc_pmcn_write)(unsigned int pmc, uint32_t val);
66void (*powerpc_resume_pmc)(bool ie);
67
68
69int
70pmc_save_kernel_callchain(uintptr_t *cc, int maxsamples,
71    struct trapframe *tf)
72{
73	uintptr_t *osp, *sp;
74	uintptr_t pc;
75	int frames = 0;
76
77	cc[frames++] = PMC_TRAPFRAME_TO_PC(tf);
78	sp = (uintptr_t *)PMC_TRAPFRAME_TO_FP(tf);
79	osp = (uintptr_t *)PAGE_SIZE;
80
81	for (; frames < maxsamples; frames++) {
82		if (sp <= osp)
83			break;
84	    #ifdef __powerpc64__
85		pc = sp[2];
86	    #else
87		pc = sp[1];
88	    #endif
89		if ((pc & 3) || (pc < 0x100))
90			break;
91
92		/*
93		 * trapexit() and asttrapexit() are sentinels
94		 * for kernel stack tracing.
95		 * */
96		if (pc + OFFSET == (uintptr_t) &trapexit ||
97		    pc + OFFSET == (uintptr_t) &asttrapexit)
98			break;
99
100		cc[frames] = pc;
101		osp = sp;
102		sp = (uintptr_t *)*sp;
103	}
104	return (frames);
105}
106
107static int
108powerpc_switch_in(struct pmc_cpu *pc, struct pmc_process *pp)
109{
110
111	return (0);
112}
113
114static int
115powerpc_switch_out(struct pmc_cpu *pc, struct pmc_process *pp)
116{
117
118	return (0);
119}
120
121int
122powerpc_describe(int cpu, int ri, struct pmc_info *pi, struct pmc **ppmc)
123{
124	int error;
125	struct pmc_hw *phw;
126	char powerpc_name[PMC_NAME_MAX];
127
128	KASSERT(cpu >= 0 && cpu < pmc_cpu_max(),
129	    ("[powerpc,%d], illegal CPU %d", __LINE__, cpu));
130
131	phw = &powerpc_pcpu[cpu]->pc_ppcpmcs[ri];
132	snprintf(powerpc_name, sizeof(powerpc_name), "POWERPC-%d", ri);
133	if ((error = copystr(powerpc_name, pi->pm_name, PMC_NAME_MAX,
134	    NULL)) != 0)
135		return error;
136	pi->pm_class = powerpc_pcpu[cpu]->pc_class;
137	if (phw->phw_state & PMC_PHW_FLAG_IS_ENABLED) {
138		pi->pm_enabled = TRUE;
139		*ppmc          = phw->phw_pmc;
140	} else {
141		pi->pm_enabled = FALSE;
142		*ppmc	       = NULL;
143	}
144
145	return (0);
146}
147
148int
149powerpc_get_config(int cpu, int ri, struct pmc **ppm)
150{
151
152	*ppm = powerpc_pcpu[cpu]->pc_ppcpmcs[ri].phw_pmc;
153
154	return (0);
155}
156
157int
158powerpc_pcpu_init(struct pmc_mdep *md, int cpu)
159{
160	struct pmc_cpu *pc;
161	struct powerpc_cpu *pac;
162	struct pmc_hw  *phw;
163	int first_ri, i;
164
165	KASSERT(cpu >= 0 && cpu < pmc_cpu_max(),
166	    ("[powerpc,%d] wrong cpu number %d", __LINE__, cpu));
167	PMCDBG1(MDP,INI,1,"powerpc-init cpu=%d", cpu);
168
169	powerpc_pcpu[cpu] = pac = malloc(sizeof(struct powerpc_cpu), M_PMC,
170	    M_WAITOK|M_ZERO);
171	pac->pc_ppcpmcs = malloc(sizeof(struct pmc_hw) * ppc_max_pmcs,
172	    M_PMC, M_WAITOK|M_ZERO);
173	pac->pc_class =
174	    md->pmd_classdep[PMC_MDEP_CLASS_INDEX_POWERPC].pcd_class;
175
176	pc = pmc_pcpu[cpu];
177	first_ri = md->pmd_classdep[PMC_MDEP_CLASS_INDEX_POWERPC].pcd_ri;
178	KASSERT(pc != NULL, ("[powerpc,%d] NULL per-cpu pointer", __LINE__));
179
180	for (i = 0, phw = pac->pc_ppcpmcs; i < ppc_max_pmcs; i++, phw++) {
181		phw->phw_state = PMC_PHW_FLAG_IS_ENABLED |
182		    PMC_PHW_CPU_TO_STATE(cpu) | PMC_PHW_INDEX_TO_STATE(i);
183		phw->phw_pmc = NULL;
184		pc->pc_hwpmcs[i + first_ri] = phw;
185	}
186
187	return (0);
188}
189
190int
191powerpc_pcpu_fini(struct pmc_mdep *md, int cpu)
192{
193	PMCDBG1(MDP,INI,1,"powerpc-fini cpu=%d", cpu);
194
195	free(powerpc_pcpu[cpu]->pc_ppcpmcs, M_PMC);
196	free(powerpc_pcpu[cpu], M_PMC);
197
198	return (0);
199}
200
201int
202powerpc_allocate_pmc(int cpu, int ri, struct pmc *pm,
203    const struct pmc_op_pmcallocate *a)
204{
205	enum pmc_event pe;
206	uint32_t caps, config = 0, counter = 0;
207	int i;
208
209	KASSERT(cpu >= 0 && cpu < pmc_cpu_max(),
210	    ("[powerpc,%d] illegal CPU value %d", __LINE__, cpu));
211	KASSERT(ri >= 0 && ri < ppc_max_pmcs,
212	    ("[powerpc,%d] illegal row index %d", __LINE__, ri));
213
214	caps = a->pm_caps;
215
216	pe = a->pm_ev;
217
218	if (pe < ppc_event_first || pe > ppc_event_last)
219		return (EINVAL);
220
221	for (i = 0; i < ppc_event_codes_size; i++) {
222		if (ppc_event_codes[i].pe_event == pe) {
223			config = ppc_event_codes[i].pe_code;
224			counter =  ppc_event_codes[i].pe_flags;
225			break;
226		}
227	}
228	if (i == ppc_event_codes_size)
229		return (EINVAL);
230
231	if ((counter & (1 << ri)) == 0)
232		return (EINVAL);
233
234	if (caps & PMC_CAP_SYSTEM)
235		config |= POWERPC_PMC_KERNEL_ENABLE;
236	if (caps & PMC_CAP_USER)
237		config |= POWERPC_PMC_USER_ENABLE;
238	if ((caps & (PMC_CAP_USER | PMC_CAP_SYSTEM)) == 0)
239		config |= POWERPC_PMC_ENABLE;
240
241	pm->pm_md.pm_powerpc.pm_powerpc_evsel = config;
242
243	PMCDBG3(MDP,ALL,1,"powerpc-allocate cpu=%d ri=%d -> config=0x%x",
244	    cpu, ri, config);
245	return (0);
246}
247
248int
249powerpc_release_pmc(int cpu, int ri, struct pmc *pmc)
250{
251	struct pmc_hw *phw;
252
253	KASSERT(cpu >= 0 && cpu < pmc_cpu_max(),
254	    ("[powerpc,%d] illegal CPU value %d", __LINE__, cpu));
255	KASSERT(ri >= 0 && ri < ppc_max_pmcs,
256	    ("[powerpc,%d] illegal row-index %d", __LINE__, ri));
257
258	phw = &powerpc_pcpu[cpu]->pc_ppcpmcs[ri];
259	KASSERT(phw->phw_pmc == NULL,
260	    ("[powerpc,%d] PHW pmc %p non-NULL", __LINE__, phw->phw_pmc));
261
262	return (0);
263}
264
265int
266powerpc_start_pmc(int cpu, int ri)
267{
268	struct pmc *pm;
269
270	PMCDBG2(MDP,STA,1,"powerpc-start cpu=%d ri=%d", cpu, ri);
271	pm = powerpc_pcpu[cpu]->pc_ppcpmcs[ri].phw_pmc;
272	powerpc_set_pmc(cpu, ri, pm->pm_md.pm_powerpc.pm_powerpc_evsel);
273
274	return (0);
275}
276
277int
278powerpc_stop_pmc(int cpu, int ri)
279{
280	PMCDBG2(MDP,STO,1, "powerpc-stop cpu=%d ri=%d", cpu, ri);
281	powerpc_set_pmc(cpu, ri, PMCN_NONE);
282	return (0);
283}
284
285int
286powerpc_config_pmc(int cpu, int ri, struct pmc *pm)
287{
288	struct pmc_hw *phw;
289
290	PMCDBG3(MDP,CFG,1, "powerpc-config cpu=%d ri=%d pm=%p", cpu, ri, pm);
291
292	KASSERT(cpu >= 0 && cpu < pmc_cpu_max(),
293	    ("[powerpc,%d] illegal CPU value %d", __LINE__, cpu));
294	KASSERT(ri >= 0 && ri < ppc_max_pmcs,
295	    ("[powerpc,%d] illegal row-index %d", __LINE__, ri));
296
297	phw = &powerpc_pcpu[cpu]->pc_ppcpmcs[ri];
298
299	KASSERT(pm == NULL || phw->phw_pmc == NULL,
300	    ("[powerpc,%d] pm=%p phw->pm=%p hwpmc not unconfigured",
301	    __LINE__, pm, phw->phw_pmc));
302
303	phw->phw_pmc = pm;
304
305	return (0);
306}
307
308pmc_value_t
309powerpc_pmcn_read_default(unsigned int pmc)
310{
311	pmc_value_t val;
312
313	if (pmc > ppc_max_pmcs)
314		panic("Invalid PMC number: %d\n", pmc);
315
316	switch (pmc) {
317	case 0:
318		val = mfspr(SPR_PMC1);
319		break;
320	case 1:
321		val = mfspr(SPR_PMC2);
322		break;
323	case 2:
324		val = mfspr(SPR_PMC3);
325		break;
326	case 3:
327		val = mfspr(SPR_PMC4);
328		break;
329	case 4:
330		val = mfspr(SPR_PMC5);
331		break;
332	case 5:
333		val = mfspr(SPR_PMC6);
334		break;
335	case 6:
336		val = mfspr(SPR_PMC7);
337		break;
338	case 7:
339		val = mfspr(SPR_PMC8);
340		break;
341	}
342
343	return (val);
344}
345
346void
347powerpc_pmcn_write_default(unsigned int pmc, uint32_t val)
348{
349	if (pmc > ppc_max_pmcs)
350		panic("Invalid PMC number: %d\n", pmc);
351
352	switch (pmc) {
353	case 0:
354		mtspr(SPR_PMC1, val);
355		break;
356	case 1:
357		mtspr(SPR_PMC2, val);
358		break;
359	case 2:
360		mtspr(SPR_PMC3, val);
361		break;
362	case 3:
363		mtspr(SPR_PMC4, val);
364		break;
365	case 4:
366		mtspr(SPR_PMC5, val);
367		break;
368	case 5:
369		mtspr(SPR_PMC6, val);
370		break;
371	case 6:
372		mtspr(SPR_PMC7, val);
373		break;
374	case 7:
375		mtspr(SPR_PMC8, val);
376		break;
377	}
378}
379
380int
381powerpc_read_pmc(int cpu, int ri, pmc_value_t *v)
382{
383	struct pmc *pm;
384	pmc_value_t p, r, tmp;
385
386	KASSERT(cpu >= 0 && cpu < pmc_cpu_max(),
387	    ("[powerpc,%d] illegal CPU value %d", __LINE__, cpu));
388	KASSERT(ri >= 0 && ri < ppc_max_pmcs,
389	    ("[powerpc,%d] illegal row index %d", __LINE__, ri));
390
391	pm  = powerpc_pcpu[cpu]->pc_ppcpmcs[ri].phw_pmc;
392	KASSERT(pm,
393	    ("[core,%d] cpu %d ri %d pmc not configured", __LINE__, cpu,
394		ri));
395
396	/*
397	 * After an interrupt occurs because of a PMC overflow, the PMC value
398	 * is not always MAX_PMC_VALUE + 1, but may be a little above it.
399	 * This may mess up calculations and frustrate machine independent
400	 * layer expectations, such as that no value read should be greater
401	 * than reload count in sampling mode.
402	 * To avoid these issues, use MAX_PMC_VALUE as an upper limit.
403	 */
404	p = MIN(powerpc_pmcn_read(ri), POWERPC_MAX_PMC_VALUE);
405	r = pm->pm_sc.pm_reloadcount;
406
407	if (PMC_IS_SAMPLING_MODE(PMC_TO_MODE(pm))) {
408		/*
409		 * Special case 1: r is too big
410		 * This usually happens when a PMC write fails, the PMC is
411		 * stopped and then it is read.
412		 *
413		 * Special case 2: PMC was reseted or has a value
414		 * that should not be possible with current r.
415		 *
416		 * In the above cases, just return 0 instead of an arbitrary
417		 * value.
418		 */
419		if (r > POWERPC_MAX_PMC_VALUE || p + r <= POWERPC_MAX_PMC_VALUE)
420			tmp = 0;
421		else
422			tmp = POWERPC_PERFCTR_VALUE_TO_RELOAD_COUNT(p);
423	} else
424		tmp = p + (POWERPC_MAX_PMC_VALUE + 1) * PPC_OVERFLOWCNT(pm);
425
426	PMCDBG5(MDP,REA,1,"ppc-read cpu=%d ri=%d -> %jx (%jx,%jx)",
427	    cpu, ri, (uintmax_t)tmp, (uintmax_t)PPC_OVERFLOWCNT(pm),
428	    (uintmax_t)p);
429	*v = tmp;
430	return (0);
431}
432
433int
434powerpc_write_pmc(int cpu, int ri, pmc_value_t v)
435{
436	struct pmc *pm;
437	pmc_value_t vlo;
438
439	KASSERT(cpu >= 0 && cpu < pmc_cpu_max(),
440	    ("[powerpc,%d] illegal CPU value %d", __LINE__, cpu));
441	KASSERT(ri >= 0 && ri < ppc_max_pmcs,
442	    ("[powerpc,%d] illegal row-index %d", __LINE__, ri));
443
444	pm = powerpc_pcpu[cpu]->pc_ppcpmcs[ri].phw_pmc;
445
446	if (PMC_IS_COUNTING_MODE(PMC_TO_MODE(pm))) {
447		PPC_OVERFLOWCNT(pm) = v / (POWERPC_MAX_PMC_VALUE + 1);
448		vlo = v % (POWERPC_MAX_PMC_VALUE + 1);
449	} else if (v > POWERPC_MAX_PMC_VALUE) {
450		PMCDBG3(MDP,WRI,2,
451		    "powerpc-write cpu=%d ri=%d: PMC value is too big: %jx",
452		    cpu, ri, (uintmax_t)v);
453		return (EINVAL);
454	} else
455		vlo = POWERPC_RELOAD_COUNT_TO_PERFCTR_VALUE(v);
456
457	PMCDBG5(MDP,WRI,1,"powerpc-write cpu=%d ri=%d -> %jx (%jx,%jx)",
458	    cpu, ri, (uintmax_t)v, (uintmax_t)PPC_OVERFLOWCNT(pm),
459	    (uintmax_t)vlo);
460
461	powerpc_pmcn_write(ri, vlo);
462	return (0);
463}
464
465int
466powerpc_pmc_intr(struct trapframe *tf)
467{
468	struct pmc *pm;
469	struct powerpc_cpu *pc;
470	int cpu, error, i, retval;
471
472	cpu = curcpu;
473	KASSERT(cpu >= 0 && cpu < pmc_cpu_max(),
474	    ("[powerpc,%d] out of range CPU %d", __LINE__, cpu));
475
476	PMCDBG3(MDP,INT,1, "cpu=%d tf=%p um=%d", cpu, (void *) tf,
477	    TRAPF_USERMODE(tf));
478
479	retval = 0;
480	pc = powerpc_pcpu[cpu];
481
482	/*
483	 * Look for a running, sampling PMC which has overflowed
484	 * and which has a valid 'struct pmc' association.
485	 */
486	for (i = 0; i < ppc_max_pmcs; i++) {
487		if (!POWERPC_PMC_HAS_OVERFLOWED(i))
488			continue;
489		retval = 1;	/* Found an interrupting PMC. */
490
491		/*
492		 * Always clear the PMC, to make it stop interrupting.
493		 * If pm is available and in sampling mode, use reload
494		 * count, to make PMC read after stop correct.
495		 * Otherwise, just reset the PMC.
496		 */
497		if ((pm = pc->pc_ppcpmcs[i].phw_pmc) != NULL &&
498		    PMC_IS_SAMPLING_MODE(PMC_TO_MODE(pm))) {
499			if (pm->pm_state != PMC_STATE_RUNNING) {
500				powerpc_write_pmc(cpu, i,
501				    pm->pm_sc.pm_reloadcount);
502				continue;
503			}
504		} else {
505			if (pm != NULL) { /* !PMC_IS_SAMPLING_MODE */
506				PPC_OVERFLOWCNT(pm) = (PPC_OVERFLOWCNT(pm) +
507				    1) % PPC_OVERFLOWCNT_MAX;
508				PMCDBG3(MDP,INT,2,
509				    "cpu=%d ri=%d: overflowcnt=%d",
510				    cpu, i, PPC_OVERFLOWCNT(pm));
511			}
512
513			powerpc_pmcn_write(i, 0);
514			continue;
515		}
516
517		error = pmc_process_interrupt(PMC_HR, pm, tf);
518		if (error != 0) {
519			PMCDBG3(MDP,INT,3,
520			    "cpu=%d ri=%d: error %d processing interrupt",
521			    cpu, i, error);
522			powerpc_stop_pmc(cpu, i);
523		}
524
525		/* Reload sampling count */
526		powerpc_write_pmc(cpu, i, pm->pm_sc.pm_reloadcount);
527	}
528
529	if (retval)
530		counter_u64_add(pmc_stats.pm_intr_processed, 1);
531	else
532		counter_u64_add(pmc_stats.pm_intr_ignored, 1);
533
534	/*
535	 * Re-enable PERF exceptions if we were able to find the interrupt
536	 * source and handle it. Otherwise, it's better to disable PERF
537	 * interrupts, to avoid the risk of processing the same interrupt
538	 * forever.
539	 */
540	powerpc_resume_pmc(retval != 0);
541	if (retval == 0)
542		log(LOG_WARNING,
543		    "pmc_intr: couldn't find interrupting PMC on cpu %d - "
544		    "disabling PERF interrupts\n", cpu);
545
546	return (retval);
547}
548
549struct pmc_mdep *
550pmc_md_initialize()
551{
552	struct pmc_mdep *pmc_mdep;
553	int error;
554	uint16_t vers;
555
556	/*
557	 * Allocate space for pointers to PMC HW descriptors and for
558	 * the MDEP structure used by MI code.
559	 */
560	powerpc_pcpu = malloc(sizeof(struct powerpc_cpu *) * pmc_cpu_max(), M_PMC,
561			   M_WAITOK|M_ZERO);
562
563	/* Just one class */
564	pmc_mdep = pmc_mdep_alloc(1);
565
566	vers = mfpvr() >> 16;
567
568	pmc_mdep->pmd_switch_in  = powerpc_switch_in;
569	pmc_mdep->pmd_switch_out = powerpc_switch_out;
570
571	switch (vers) {
572	case MPC7447A:
573	case MPC7448:
574	case MPC7450:
575	case MPC7455:
576	case MPC7457:
577		error = pmc_mpc7xxx_initialize(pmc_mdep);
578		break;
579	case IBM970:
580	case IBM970FX:
581	case IBM970MP:
582		error = pmc_ppc970_initialize(pmc_mdep);
583		break;
584	case IBMPOWER8E:
585	case IBMPOWER8NVL:
586	case IBMPOWER8:
587	case IBMPOWER9:
588		error = pmc_power8_initialize(pmc_mdep);
589		break;
590	case FSL_E500v1:
591	case FSL_E500v2:
592	case FSL_E500mc:
593	case FSL_E5500:
594		error = pmc_e500_initialize(pmc_mdep);
595		break;
596	default:
597		error = -1;
598		break;
599	}
600
601	if (error != 0) {
602		pmc_mdep_free(pmc_mdep);
603		pmc_mdep = NULL;
604	}
605
606	return (pmc_mdep);
607}
608
609void
610pmc_md_finalize(struct pmc_mdep *md)
611{
612
613	free(powerpc_pcpu, M_PMC);
614	powerpc_pcpu = NULL;
615}
616
617int
618pmc_save_user_callchain(uintptr_t *cc, int maxsamples,
619    struct trapframe *tf)
620{
621	uintptr_t *osp, *sp;
622	int frames = 0;
623
624	cc[frames++] = PMC_TRAPFRAME_TO_PC(tf);
625	sp = (uintptr_t *)PMC_TRAPFRAME_TO_FP(tf);
626	osp = NULL;
627
628	for (; frames < maxsamples; frames++) {
629		if (sp <= osp)
630			break;
631		osp = sp;
632#ifdef __powerpc64__
633		/* Check if 32-bit mode. */
634		if (!(tf->srr1 & PSL_SF)) {
635			cc[frames] = fuword32((uint32_t *)sp + 1);
636			sp = (uintptr_t *)(uintptr_t)fuword32(sp);
637		} else {
638			cc[frames] = fuword(sp + 2);
639			sp = (uintptr_t *)fuword(sp);
640		}
641#else
642		cc[frames] = fuword32((uint32_t *)sp + 1);
643		sp = (uintptr_t *)fuword32(sp);
644#endif
645	}
646
647	return (frames);
648}
649