1// SPDX-License-Identifier: GPL-2.0-or-later
2/*
3 * Performance event support - Freescale Embedded Performance Monitor
4 *
5 * Copyright 2008-2009 Paul Mackerras, IBM Corporation.
6 * Copyright 2010 Freescale Semiconductor, Inc.
7 */
8#include <linux/kernel.h>
9#include <linux/sched.h>
10#include <linux/perf_event.h>
11#include <linux/percpu.h>
12#include <linux/hardirq.h>
13#include <asm/reg_fsl_emb.h>
14#include <asm/pmc.h>
15#include <asm/machdep.h>
16#include <asm/firmware.h>
17#include <asm/ptrace.h>
18
19struct cpu_hw_events {
20	int n_events;
21	int disabled;
22	u8  pmcs_enabled;
23	struct perf_event *event[MAX_HWEVENTS];
24};
25static DEFINE_PER_CPU(struct cpu_hw_events, cpu_hw_events);
26
27static struct fsl_emb_pmu *ppmu;
28
29/* Number of perf_events counting hardware events */
30static atomic_t num_events;
31/* Used to avoid races in calling reserve/release_pmc_hardware */
32static DEFINE_MUTEX(pmc_reserve_mutex);
33
34static void perf_event_interrupt(struct pt_regs *regs);
35
36/*
37 * Read one performance monitor counter (PMC).
38 */
39static unsigned long read_pmc(int idx)
40{
41	unsigned long val;
42
43	switch (idx) {
44	case 0:
45		val = mfpmr(PMRN_PMC0);
46		break;
47	case 1:
48		val = mfpmr(PMRN_PMC1);
49		break;
50	case 2:
51		val = mfpmr(PMRN_PMC2);
52		break;
53	case 3:
54		val = mfpmr(PMRN_PMC3);
55		break;
56	case 4:
57		val = mfpmr(PMRN_PMC4);
58		break;
59	case 5:
60		val = mfpmr(PMRN_PMC5);
61		break;
62	default:
63		printk(KERN_ERR "oops trying to read PMC%d\n", idx);
64		val = 0;
65	}
66	return val;
67}
68
69/*
70 * Write one PMC.
71 */
72static void write_pmc(int idx, unsigned long val)
73{
74	switch (idx) {
75	case 0:
76		mtpmr(PMRN_PMC0, val);
77		break;
78	case 1:
79		mtpmr(PMRN_PMC1, val);
80		break;
81	case 2:
82		mtpmr(PMRN_PMC2, val);
83		break;
84	case 3:
85		mtpmr(PMRN_PMC3, val);
86		break;
87	case 4:
88		mtpmr(PMRN_PMC4, val);
89		break;
90	case 5:
91		mtpmr(PMRN_PMC5, val);
92		break;
93	default:
94		printk(KERN_ERR "oops trying to write PMC%d\n", idx);
95	}
96
97	isync();
98}
99
100/*
101 * Write one local control A register
102 */
103static void write_pmlca(int idx, unsigned long val)
104{
105	switch (idx) {
106	case 0:
107		mtpmr(PMRN_PMLCA0, val);
108		break;
109	case 1:
110		mtpmr(PMRN_PMLCA1, val);
111		break;
112	case 2:
113		mtpmr(PMRN_PMLCA2, val);
114		break;
115	case 3:
116		mtpmr(PMRN_PMLCA3, val);
117		break;
118	case 4:
119		mtpmr(PMRN_PMLCA4, val);
120		break;
121	case 5:
122		mtpmr(PMRN_PMLCA5, val);
123		break;
124	default:
125		printk(KERN_ERR "oops trying to write PMLCA%d\n", idx);
126	}
127
128	isync();
129}
130
131/*
132 * Write one local control B register
133 */
134static void write_pmlcb(int idx, unsigned long val)
135{
136	switch (idx) {
137	case 0:
138		mtpmr(PMRN_PMLCB0, val);
139		break;
140	case 1:
141		mtpmr(PMRN_PMLCB1, val);
142		break;
143	case 2:
144		mtpmr(PMRN_PMLCB2, val);
145		break;
146	case 3:
147		mtpmr(PMRN_PMLCB3, val);
148		break;
149	case 4:
150		mtpmr(PMRN_PMLCB4, val);
151		break;
152	case 5:
153		mtpmr(PMRN_PMLCB5, val);
154		break;
155	default:
156		printk(KERN_ERR "oops trying to write PMLCB%d\n", idx);
157	}
158
159	isync();
160}
161
162static void fsl_emb_pmu_read(struct perf_event *event)
163{
164	s64 val, delta, prev;
165
166	if (event->hw.state & PERF_HES_STOPPED)
167		return;
168
169	/*
170	 * Performance monitor interrupts come even when interrupts
171	 * are soft-disabled, as long as interrupts are hard-enabled.
172	 * Therefore we treat them like NMIs.
173	 */
174	do {
175		prev = local64_read(&event->hw.prev_count);
176		barrier();
177		val = read_pmc(event->hw.idx);
178	} while (local64_cmpxchg(&event->hw.prev_count, prev, val) != prev);
179
180	/* The counters are only 32 bits wide */
181	delta = (val - prev) & 0xfffffffful;
182	local64_add(delta, &event->count);
183	local64_sub(delta, &event->hw.period_left);
184}
185
186/*
187 * Disable all events to prevent PMU interrupts and to allow
188 * events to be added or removed.
189 */
190static void fsl_emb_pmu_disable(struct pmu *pmu)
191{
192	struct cpu_hw_events *cpuhw;
193	unsigned long flags;
194
195	local_irq_save(flags);
196	cpuhw = this_cpu_ptr(&cpu_hw_events);
197
198	if (!cpuhw->disabled) {
199		cpuhw->disabled = 1;
200
201		/*
202		 * Check if we ever enabled the PMU on this cpu.
203		 */
204		if (!cpuhw->pmcs_enabled) {
205			ppc_enable_pmcs();
206			cpuhw->pmcs_enabled = 1;
207		}
208
209		if (atomic_read(&num_events)) {
210			/*
211			 * Set the 'freeze all counters' bit, and disable
212			 * interrupts.  The barrier is to make sure the
213			 * mtpmr has been executed and the PMU has frozen
214			 * the events before we return.
215			 */
216
217			mtpmr(PMRN_PMGC0, PMGC0_FAC);
218			isync();
219		}
220	}
221	local_irq_restore(flags);
222}
223
224/*
225 * Re-enable all events if disable == 0.
226 * If we were previously disabled and events were added, then
227 * put the new config on the PMU.
228 */
229static void fsl_emb_pmu_enable(struct pmu *pmu)
230{
231	struct cpu_hw_events *cpuhw;
232	unsigned long flags;
233
234	local_irq_save(flags);
235	cpuhw = this_cpu_ptr(&cpu_hw_events);
236	if (!cpuhw->disabled)
237		goto out;
238
239	cpuhw->disabled = 0;
240	ppc_set_pmu_inuse(cpuhw->n_events != 0);
241
242	if (cpuhw->n_events > 0) {
243		mtpmr(PMRN_PMGC0, PMGC0_PMIE | PMGC0_FCECE);
244		isync();
245	}
246
247 out:
248	local_irq_restore(flags);
249}
250
251static int collect_events(struct perf_event *group, int max_count,
252			  struct perf_event *ctrs[])
253{
254	int n = 0;
255	struct perf_event *event;
256
257	if (!is_software_event(group)) {
258		if (n >= max_count)
259			return -1;
260		ctrs[n] = group;
261		n++;
262	}
263	for_each_sibling_event(event, group) {
264		if (!is_software_event(event) &&
265		    event->state != PERF_EVENT_STATE_OFF) {
266			if (n >= max_count)
267				return -1;
268			ctrs[n] = event;
269			n++;
270		}
271	}
272	return n;
273}
274
275/* context locked on entry */
276static int fsl_emb_pmu_add(struct perf_event *event, int flags)
277{
278	struct cpu_hw_events *cpuhw;
279	int ret = -EAGAIN;
280	int num_counters = ppmu->n_counter;
281	u64 val;
282	int i;
283
284	perf_pmu_disable(event->pmu);
285	cpuhw = &get_cpu_var(cpu_hw_events);
286
287	if (event->hw.config & FSL_EMB_EVENT_RESTRICTED)
288		num_counters = ppmu->n_restricted;
289
290	/*
291	 * Allocate counters from top-down, so that restricted-capable
292	 * counters are kept free as long as possible.
293	 */
294	for (i = num_counters - 1; i >= 0; i--) {
295		if (cpuhw->event[i])
296			continue;
297
298		break;
299	}
300
301	if (i < 0)
302		goto out;
303
304	event->hw.idx = i;
305	cpuhw->event[i] = event;
306	++cpuhw->n_events;
307
308	val = 0;
309	if (event->hw.sample_period) {
310		s64 left = local64_read(&event->hw.period_left);
311		if (left < 0x80000000L)
312			val = 0x80000000L - left;
313	}
314	local64_set(&event->hw.prev_count, val);
315
316	if (unlikely(!(flags & PERF_EF_START))) {
317		event->hw.state = PERF_HES_STOPPED | PERF_HES_UPTODATE;
318		val = 0;
319	} else {
320		event->hw.state &= ~(PERF_HES_STOPPED | PERF_HES_UPTODATE);
321	}
322
323	write_pmc(i, val);
324	perf_event_update_userpage(event);
325
326	write_pmlcb(i, event->hw.config >> 32);
327	write_pmlca(i, event->hw.config_base);
328
329	ret = 0;
330 out:
331	put_cpu_var(cpu_hw_events);
332	perf_pmu_enable(event->pmu);
333	return ret;
334}
335
336/* context locked on entry */
337static void fsl_emb_pmu_del(struct perf_event *event, int flags)
338{
339	struct cpu_hw_events *cpuhw;
340	int i = event->hw.idx;
341
342	perf_pmu_disable(event->pmu);
343	if (i < 0)
344		goto out;
345
346	fsl_emb_pmu_read(event);
347
348	cpuhw = &get_cpu_var(cpu_hw_events);
349
350	WARN_ON(event != cpuhw->event[event->hw.idx]);
351
352	write_pmlca(i, 0);
353	write_pmlcb(i, 0);
354	write_pmc(i, 0);
355
356	cpuhw->event[i] = NULL;
357	event->hw.idx = -1;
358
359	/*
360	 * TODO: if at least one restricted event exists, and we
361	 * just freed up a non-restricted-capable counter, and
362	 * there is a restricted-capable counter occupied by
363	 * a non-restricted event, migrate that event to the
364	 * vacated counter.
365	 */
366
367	cpuhw->n_events--;
368
369 out:
370	perf_pmu_enable(event->pmu);
371	put_cpu_var(cpu_hw_events);
372}
373
374static void fsl_emb_pmu_start(struct perf_event *event, int ef_flags)
375{
376	unsigned long flags;
377	unsigned long val;
378	s64 left;
379
380	if (event->hw.idx < 0 || !event->hw.sample_period)
381		return;
382
383	if (!(event->hw.state & PERF_HES_STOPPED))
384		return;
385
386	if (ef_flags & PERF_EF_RELOAD)
387		WARN_ON_ONCE(!(event->hw.state & PERF_HES_UPTODATE));
388
389	local_irq_save(flags);
390	perf_pmu_disable(event->pmu);
391
392	event->hw.state = 0;
393	left = local64_read(&event->hw.period_left);
394	val = 0;
395	if (left < 0x80000000L)
396		val = 0x80000000L - left;
397	write_pmc(event->hw.idx, val);
398
399	perf_event_update_userpage(event);
400	perf_pmu_enable(event->pmu);
401	local_irq_restore(flags);
402}
403
404static void fsl_emb_pmu_stop(struct perf_event *event, int ef_flags)
405{
406	unsigned long flags;
407
408	if (event->hw.idx < 0 || !event->hw.sample_period)
409		return;
410
411	if (event->hw.state & PERF_HES_STOPPED)
412		return;
413
414	local_irq_save(flags);
415	perf_pmu_disable(event->pmu);
416
417	fsl_emb_pmu_read(event);
418	event->hw.state |= PERF_HES_STOPPED | PERF_HES_UPTODATE;
419	write_pmc(event->hw.idx, 0);
420
421	perf_event_update_userpage(event);
422	perf_pmu_enable(event->pmu);
423	local_irq_restore(flags);
424}
425
426/*
427 * Release the PMU if this is the last perf_event.
428 */
429static void hw_perf_event_destroy(struct perf_event *event)
430{
431	if (!atomic_add_unless(&num_events, -1, 1)) {
432		mutex_lock(&pmc_reserve_mutex);
433		if (atomic_dec_return(&num_events) == 0)
434			release_pmc_hardware();
435		mutex_unlock(&pmc_reserve_mutex);
436	}
437}
438
439/*
440 * Translate a generic cache event_id config to a raw event_id code.
441 */
442static int hw_perf_cache_event(u64 config, u64 *eventp)
443{
444	unsigned long type, op, result;
445	int ev;
446
447	if (!ppmu->cache_events)
448		return -EINVAL;
449
450	/* unpack config */
451	type = config & 0xff;
452	op = (config >> 8) & 0xff;
453	result = (config >> 16) & 0xff;
454
455	if (type >= PERF_COUNT_HW_CACHE_MAX ||
456	    op >= PERF_COUNT_HW_CACHE_OP_MAX ||
457	    result >= PERF_COUNT_HW_CACHE_RESULT_MAX)
458		return -EINVAL;
459
460	ev = (*ppmu->cache_events)[type][op][result];
461	if (ev == 0)
462		return -EOPNOTSUPP;
463	if (ev == -1)
464		return -EINVAL;
465	*eventp = ev;
466	return 0;
467}
468
469static int fsl_emb_pmu_event_init(struct perf_event *event)
470{
471	u64 ev;
472	struct perf_event *events[MAX_HWEVENTS];
473	int n;
474	int err;
475	int num_restricted;
476	int i;
477
478	if (ppmu->n_counter > MAX_HWEVENTS) {
479		WARN(1, "No. of perf counters (%d) is higher than max array size(%d)\n",
480			ppmu->n_counter, MAX_HWEVENTS);
481		ppmu->n_counter = MAX_HWEVENTS;
482	}
483
484	switch (event->attr.type) {
485	case PERF_TYPE_HARDWARE:
486		ev = event->attr.config;
487		if (ev >= ppmu->n_generic || ppmu->generic_events[ev] == 0)
488			return -EOPNOTSUPP;
489		ev = ppmu->generic_events[ev];
490		break;
491
492	case PERF_TYPE_HW_CACHE:
493		err = hw_perf_cache_event(event->attr.config, &ev);
494		if (err)
495			return err;
496		break;
497
498	case PERF_TYPE_RAW:
499		ev = event->attr.config;
500		break;
501
502	default:
503		return -ENOENT;
504	}
505
506	event->hw.config = ppmu->xlate_event(ev);
507	if (!(event->hw.config & FSL_EMB_EVENT_VALID))
508		return -EINVAL;
509
510	/*
511	 * If this is in a group, check if it can go on with all the
512	 * other hardware events in the group.  We assume the event
513	 * hasn't been linked into its leader's sibling list at this point.
514	 */
515	n = 0;
516	if (event->group_leader != event) {
517		n = collect_events(event->group_leader,
518		                   ppmu->n_counter - 1, events);
519		if (n < 0)
520			return -EINVAL;
521	}
522
523	if (event->hw.config & FSL_EMB_EVENT_RESTRICTED) {
524		num_restricted = 0;
525		for (i = 0; i < n; i++) {
526			if (events[i]->hw.config & FSL_EMB_EVENT_RESTRICTED)
527				num_restricted++;
528		}
529
530		if (num_restricted >= ppmu->n_restricted)
531			return -EINVAL;
532	}
533
534	event->hw.idx = -1;
535
536	event->hw.config_base = PMLCA_CE | PMLCA_FCM1 |
537	                        (u32)((ev << 16) & PMLCA_EVENT_MASK);
538
539	if (event->attr.exclude_user)
540		event->hw.config_base |= PMLCA_FCU;
541	if (event->attr.exclude_kernel)
542		event->hw.config_base |= PMLCA_FCS;
543	if (event->attr.exclude_idle)
544		return -ENOTSUPP;
545
546	event->hw.last_period = event->hw.sample_period;
547	local64_set(&event->hw.period_left, event->hw.last_period);
548
549	/*
550	 * See if we need to reserve the PMU.
551	 * If no events are currently in use, then we have to take a
552	 * mutex to ensure that we don't race with another task doing
553	 * reserve_pmc_hardware or release_pmc_hardware.
554	 */
555	err = 0;
556	if (!atomic_inc_not_zero(&num_events)) {
557		mutex_lock(&pmc_reserve_mutex);
558		if (atomic_read(&num_events) == 0 &&
559		    reserve_pmc_hardware(perf_event_interrupt))
560			err = -EBUSY;
561		else
562			atomic_inc(&num_events);
563		mutex_unlock(&pmc_reserve_mutex);
564
565		mtpmr(PMRN_PMGC0, PMGC0_FAC);
566		isync();
567	}
568	event->destroy = hw_perf_event_destroy;
569
570	return err;
571}
572
573static struct pmu fsl_emb_pmu = {
574	.pmu_enable	= fsl_emb_pmu_enable,
575	.pmu_disable	= fsl_emb_pmu_disable,
576	.event_init	= fsl_emb_pmu_event_init,
577	.add		= fsl_emb_pmu_add,
578	.del		= fsl_emb_pmu_del,
579	.start		= fsl_emb_pmu_start,
580	.stop		= fsl_emb_pmu_stop,
581	.read		= fsl_emb_pmu_read,
582};
583
584/*
585 * A counter has overflowed; update its count and record
586 * things if requested.  Note that interrupts are hard-disabled
587 * here so there is no possibility of being interrupted.
588 */
589static void record_and_restart(struct perf_event *event, unsigned long val,
590			       struct pt_regs *regs)
591{
592	u64 period = event->hw.sample_period;
593	s64 prev, delta, left;
594	int record = 0;
595
596	if (event->hw.state & PERF_HES_STOPPED) {
597		write_pmc(event->hw.idx, 0);
598		return;
599	}
600
601	/* we don't have to worry about interrupts here */
602	prev = local64_read(&event->hw.prev_count);
603	delta = (val - prev) & 0xfffffffful;
604	local64_add(delta, &event->count);
605
606	/*
607	 * See if the total period for this event has expired,
608	 * and update for the next period.
609	 */
610	val = 0;
611	left = local64_read(&event->hw.period_left) - delta;
612	if (period) {
613		if (left <= 0) {
614			left += period;
615			if (left <= 0)
616				left = period;
617			record = 1;
618			event->hw.last_period = event->hw.sample_period;
619		}
620		if (left < 0x80000000LL)
621			val = 0x80000000LL - left;
622	}
623
624	write_pmc(event->hw.idx, val);
625	local64_set(&event->hw.prev_count, val);
626	local64_set(&event->hw.period_left, left);
627	perf_event_update_userpage(event);
628
629	/*
630	 * Finally record data if requested.
631	 */
632	if (record) {
633		struct perf_sample_data data;
634
635		perf_sample_data_init(&data, 0, event->hw.last_period);
636
637		if (perf_event_overflow(event, &data, regs))
638			fsl_emb_pmu_stop(event, 0);
639	}
640}
641
642static void perf_event_interrupt(struct pt_regs *regs)
643{
644	int i;
645	struct cpu_hw_events *cpuhw = this_cpu_ptr(&cpu_hw_events);
646	struct perf_event *event;
647	unsigned long val;
648
649	for (i = 0; i < ppmu->n_counter; ++i) {
650		event = cpuhw->event[i];
651
652		val = read_pmc(i);
653		if ((int)val < 0) {
654			if (event) {
655				/* event has overflowed */
656				record_and_restart(event, val, regs);
657			} else {
658				/*
659				 * Disabled counter is negative,
660				 * reset it just in case.
661				 */
662				write_pmc(i, 0);
663			}
664		}
665	}
666
667	/* PMM will keep counters frozen until we return from the interrupt. */
668	mtmsr(mfmsr() | MSR_PMM);
669	mtpmr(PMRN_PMGC0, PMGC0_PMIE | PMGC0_FCECE);
670	isync();
671}
672
673static int fsl_emb_pmu_prepare_cpu(unsigned int cpu)
674{
675	struct cpu_hw_events *cpuhw = &per_cpu(cpu_hw_events, cpu);
676
677	memset(cpuhw, 0, sizeof(*cpuhw));
678
679	return 0;
680}
681
682int register_fsl_emb_pmu(struct fsl_emb_pmu *pmu)
683{
684	if (ppmu)
685		return -EBUSY;		/* something's already registered */
686
687	ppmu = pmu;
688	pr_info("%s performance monitor hardware support registered\n",
689		pmu->name);
690
691	perf_pmu_register(&fsl_emb_pmu, "cpu", PERF_TYPE_RAW);
692	cpuhp_setup_state(CPUHP_PERF_POWER, "perf/powerpc:prepare",
693			  fsl_emb_pmu_prepare_cpu, NULL);
694
695	return 0;
696}
697