1/* SPDX-License-Identifier: GPL-2.0 */
2/*
3 * Copyright (C) 1999 Cort Dougan <cort@cs.nmt.edu>
4 */
5#ifndef _ASM_POWERPC_HW_IRQ_H
6#define _ASM_POWERPC_HW_IRQ_H
7
8#ifdef __KERNEL__
9
10#include <linux/errno.h>
11#include <linux/compiler.h>
12#include <asm/ptrace.h>
13#include <asm/processor.h>
14
15#ifdef CONFIG_PPC64
16
17/*
18 * PACA flags in paca->irq_happened.
19 *
20 * This bits are set when interrupts occur while soft-disabled
21 * and allow a proper replay.
22 *
23 * The PACA_IRQ_HARD_DIS is set whenever we hard disable. It is almost
24 * always in synch with the MSR[EE] state, except:
25 * - A window in interrupt entry, where hardware disables MSR[EE] and that
26 *   must be "reconciled" with the soft mask state.
27 * - NMI interrupts that hit in awkward places, until they fix the state.
28 * - When local irqs are being enabled and state is being fixed up.
29 * - When returning from an interrupt there are some windows where this
30 *   can become out of synch, but gets fixed before the RFI or before
31 *   executing the next user instruction (see arch/powerpc/kernel/interrupt.c).
32 */
33#define PACA_IRQ_HARD_DIS	0x01
34#define PACA_IRQ_DBELL		0x02
35#define PACA_IRQ_EE		0x04
36#define PACA_IRQ_DEC		0x08 /* Or FIT */
37#define PACA_IRQ_HMI		0x10
38#define PACA_IRQ_PMI		0x20
39#define PACA_IRQ_REPLAYING	0x40
40
41/*
42 * Some soft-masked interrupts must be hard masked until they are replayed
43 * (e.g., because the soft-masked handler does not clear the exception).
44 * Interrupt replay itself must remain hard masked too.
45 */
46#ifdef CONFIG_PPC_BOOK3S
47#define PACA_IRQ_MUST_HARD_MASK	(PACA_IRQ_EE|PACA_IRQ_PMI|PACA_IRQ_REPLAYING)
48#else
49#define PACA_IRQ_MUST_HARD_MASK	(PACA_IRQ_EE|PACA_IRQ_REPLAYING)
50#endif
51
52#endif /* CONFIG_PPC64 */
53
54/*
55 * flags for paca->irq_soft_mask
56 */
57#define IRQS_ENABLED		0
58#define IRQS_DISABLED		1 /* local_irq_disable() interrupts */
59#define IRQS_PMI_DISABLED	2
60#define IRQS_ALL_DISABLED	(IRQS_DISABLED | IRQS_PMI_DISABLED)
61
62#ifndef __ASSEMBLY__
63
64static inline void __hard_irq_enable(void)
65{
66	if (IS_ENABLED(CONFIG_BOOKE_OR_40x))
67		wrtee(MSR_EE);
68	else if (IS_ENABLED(CONFIG_PPC_8xx))
69		wrtspr(SPRN_EIE);
70	else if (IS_ENABLED(CONFIG_PPC_BOOK3S_64))
71		__mtmsrd(MSR_EE | MSR_RI, 1);
72	else
73		mtmsr(mfmsr() | MSR_EE);
74}
75
76static inline void __hard_irq_disable(void)
77{
78	if (IS_ENABLED(CONFIG_BOOKE_OR_40x))
79		wrtee(0);
80	else if (IS_ENABLED(CONFIG_PPC_8xx))
81		wrtspr(SPRN_EID);
82	else if (IS_ENABLED(CONFIG_PPC_BOOK3S_64))
83		__mtmsrd(MSR_RI, 1);
84	else
85		mtmsr(mfmsr() & ~MSR_EE);
86}
87
88static inline void __hard_EE_RI_disable(void)
89{
90	if (IS_ENABLED(CONFIG_BOOKE_OR_40x))
91		wrtee(0);
92	else if (IS_ENABLED(CONFIG_PPC_8xx))
93		wrtspr(SPRN_NRI);
94	else if (IS_ENABLED(CONFIG_PPC_BOOK3S_64))
95		__mtmsrd(0, 1);
96	else
97		mtmsr(mfmsr() & ~(MSR_EE | MSR_RI));
98}
99
100static inline void __hard_RI_enable(void)
101{
102	if (IS_ENABLED(CONFIG_BOOKE_OR_40x))
103		return;
104
105	if (IS_ENABLED(CONFIG_PPC_8xx))
106		wrtspr(SPRN_EID);
107	else if (IS_ENABLED(CONFIG_PPC_BOOK3S_64))
108		__mtmsrd(MSR_RI, 1);
109	else
110		mtmsr(mfmsr() | MSR_RI);
111}
112
113#ifdef CONFIG_PPC64
114#include <asm/paca.h>
115
116static inline notrace unsigned long irq_soft_mask_return(void)
117{
118	unsigned long flags;
119
120	asm volatile(
121		"lbz %0,%1(13)"
122		: "=r" (flags)
123		: "i" (offsetof(struct paca_struct, irq_soft_mask)));
124
125	return flags;
126}
127
128/*
129 * The "memory" clobber acts as both a compiler barrier
130 * for the critical section and as a clobber because
131 * we changed paca->irq_soft_mask
132 */
133static inline notrace void irq_soft_mask_set(unsigned long mask)
134{
135	/*
136	 * The irq mask must always include the STD bit if any are set.
137	 *
138	 * and interrupts don't get replayed until the standard
139	 * interrupt (local_irq_disable()) is unmasked.
140	 *
141	 * Other masks must only provide additional masking beyond
142	 * the standard, and they are also not replayed until the
143	 * standard interrupt becomes unmasked.
144	 *
145	 * This could be changed, but it will require partial
146	 * unmasks to be replayed, among other things. For now, take
147	 * the simple approach.
148	 */
149	if (IS_ENABLED(CONFIG_PPC_IRQ_SOFT_MASK_DEBUG))
150		WARN_ON(mask && !(mask & IRQS_DISABLED));
151
152	asm volatile(
153		"stb %0,%1(13)"
154		:
155		: "r" (mask),
156		  "i" (offsetof(struct paca_struct, irq_soft_mask))
157		: "memory");
158}
159
160static inline notrace unsigned long irq_soft_mask_set_return(unsigned long mask)
161{
162	unsigned long flags = irq_soft_mask_return();
163
164	irq_soft_mask_set(mask);
165
166	return flags;
167}
168
169static inline notrace unsigned long irq_soft_mask_or_return(unsigned long mask)
170{
171	unsigned long flags = irq_soft_mask_return();
172
173	irq_soft_mask_set(flags | mask);
174
175	return flags;
176}
177
178static inline notrace unsigned long irq_soft_mask_andc_return(unsigned long mask)
179{
180	unsigned long flags = irq_soft_mask_return();
181
182	irq_soft_mask_set(flags & ~mask);
183
184	return flags;
185}
186
187static inline unsigned long arch_local_save_flags(void)
188{
189	return irq_soft_mask_return();
190}
191
192static inline void arch_local_irq_disable(void)
193{
194	irq_soft_mask_set(IRQS_DISABLED);
195}
196
197extern void arch_local_irq_restore(unsigned long);
198
199static inline void arch_local_irq_enable(void)
200{
201	arch_local_irq_restore(IRQS_ENABLED);
202}
203
204static inline unsigned long arch_local_irq_save(void)
205{
206	return irq_soft_mask_or_return(IRQS_DISABLED);
207}
208
209static inline bool arch_irqs_disabled_flags(unsigned long flags)
210{
211	return flags & IRQS_DISABLED;
212}
213
214static inline bool arch_irqs_disabled(void)
215{
216	return arch_irqs_disabled_flags(arch_local_save_flags());
217}
218
219static inline void set_pmi_irq_pending(void)
220{
221	/*
222	 * Invoked from PMU callback functions to set PMI bit in the paca.
223	 * This has to be called with irq's disabled (via hard_irq_disable()).
224	 */
225	if (IS_ENABLED(CONFIG_PPC_IRQ_SOFT_MASK_DEBUG))
226		WARN_ON_ONCE(mfmsr() & MSR_EE);
227
228	get_paca()->irq_happened |= PACA_IRQ_PMI;
229}
230
231static inline void clear_pmi_irq_pending(void)
232{
233	/*
234	 * Invoked from PMU callback functions to clear the pending PMI bit
235	 * in the paca.
236	 */
237	if (IS_ENABLED(CONFIG_PPC_IRQ_SOFT_MASK_DEBUG))
238		WARN_ON_ONCE(mfmsr() & MSR_EE);
239
240	get_paca()->irq_happened &= ~PACA_IRQ_PMI;
241}
242
243static inline bool pmi_irq_pending(void)
244{
245	/*
246	 * Invoked from PMU callback functions to check if there is a pending
247	 * PMI bit in the paca.
248	 */
249	if (get_paca()->irq_happened & PACA_IRQ_PMI)
250		return true;
251
252	return false;
253}
254
255#ifdef CONFIG_PPC_BOOK3S
256/*
257 * To support disabling and enabling of irq with PMI, set of
258 * new powerpc_local_irq_pmu_save() and powerpc_local_irq_restore()
259 * functions are added. These macros are implemented using generic
260 * linux local_irq_* code from include/linux/irqflags.h.
261 */
262#define raw_local_irq_pmu_save(flags)					\
263	do {								\
264		typecheck(unsigned long, flags);			\
265		flags = irq_soft_mask_or_return(IRQS_DISABLED |	\
266				IRQS_PMI_DISABLED);			\
267	} while(0)
268
269#define raw_local_irq_pmu_restore(flags)				\
270	do {								\
271		typecheck(unsigned long, flags);			\
272		arch_local_irq_restore(flags);				\
273	} while(0)
274
275#ifdef CONFIG_TRACE_IRQFLAGS
276#define powerpc_local_irq_pmu_save(flags)			\
277	 do {							\
278		raw_local_irq_pmu_save(flags);			\
279		if (!raw_irqs_disabled_flags(flags))		\
280			trace_hardirqs_off();			\
281	} while(0)
282#define powerpc_local_irq_pmu_restore(flags)			\
283	do {							\
284		if (!raw_irqs_disabled_flags(flags))		\
285			trace_hardirqs_on();			\
286		raw_local_irq_pmu_restore(flags);		\
287	} while(0)
288#else
289#define powerpc_local_irq_pmu_save(flags)			\
290	do {							\
291		raw_local_irq_pmu_save(flags);			\
292	} while(0)
293#define powerpc_local_irq_pmu_restore(flags)			\
294	do {							\
295		raw_local_irq_pmu_restore(flags);		\
296	} while (0)
297#endif  /* CONFIG_TRACE_IRQFLAGS */
298
299#endif /* CONFIG_PPC_BOOK3S */
300
301#define hard_irq_disable()	do {					\
302	unsigned long flags;						\
303	__hard_irq_disable();						\
304	flags = irq_soft_mask_set_return(IRQS_ALL_DISABLED);		\
305	local_paca->irq_happened |= PACA_IRQ_HARD_DIS;			\
306	if (!arch_irqs_disabled_flags(flags)) {				\
307		asm volatile("std%X0 %1,%0" : "=m" (local_paca->saved_r1) \
308					    : "r" (current_stack_pointer)); \
309		trace_hardirqs_off();					\
310	}								\
311} while(0)
312
313static inline bool __lazy_irq_pending(u8 irq_happened)
314{
315	return !!(irq_happened & ~PACA_IRQ_HARD_DIS);
316}
317
318/*
319 * Check if a lazy IRQ is pending. Should be called with IRQs hard disabled.
320 */
321static inline bool lazy_irq_pending(void)
322{
323	return __lazy_irq_pending(get_paca()->irq_happened);
324}
325
326/*
327 * Check if a lazy IRQ is pending, with no debugging checks.
328 * Should be called with IRQs hard disabled.
329 * For use in RI disabled code or other constrained situations.
330 */
331static inline bool lazy_irq_pending_nocheck(void)
332{
333	return __lazy_irq_pending(local_paca->irq_happened);
334}
335
336bool power_pmu_wants_prompt_pmi(void);
337
338/*
339 * This is called by asynchronous interrupts to check whether to
340 * conditionally re-enable hard interrupts after having cleared
341 * the source of the interrupt. They are kept disabled if there
342 * is a different soft-masked interrupt pending that requires hard
343 * masking.
344 */
345static inline bool should_hard_irq_enable(struct pt_regs *regs)
346{
347	if (IS_ENABLED(CONFIG_PPC_IRQ_SOFT_MASK_DEBUG)) {
348		WARN_ON(irq_soft_mask_return() != IRQS_ALL_DISABLED);
349		WARN_ON(!(get_paca()->irq_happened & PACA_IRQ_HARD_DIS));
350		WARN_ON(mfmsr() & MSR_EE);
351	}
352
353	if (!IS_ENABLED(CONFIG_PERF_EVENTS))
354		return false;
355	/*
356	 * If the PMU is not running, there is not much reason to enable
357	 * MSR[EE] in irq handlers because any interrupts would just be
358	 * soft-masked.
359	 *
360	 * TODO: Add test for 64e
361	 */
362	if (IS_ENABLED(CONFIG_PPC_BOOK3S_64)) {
363		if (!power_pmu_wants_prompt_pmi())
364			return false;
365		/*
366		 * If PMIs are disabled then IRQs should be disabled as well,
367		 * so we shouldn't see this condition, check for it just in
368		 * case because we are about to enable PMIs.
369		 */
370		if (WARN_ON_ONCE(regs->softe & IRQS_PMI_DISABLED))
371			return false;
372	}
373
374	if (get_paca()->irq_happened & PACA_IRQ_MUST_HARD_MASK)
375		return false;
376
377	return true;
378}
379
380/*
381 * Do the hard enabling, only call this if should_hard_irq_enable is true.
382 * This allows PMI interrupts to profile irq handlers.
383 */
384static inline void do_hard_irq_enable(void)
385{
386	/*
387	 * Asynch interrupts come in with IRQS_ALL_DISABLED,
388	 * PACA_IRQ_HARD_DIS, and MSR[EE]=0.
389	 */
390	if (IS_ENABLED(CONFIG_PPC_BOOK3S_64))
391		irq_soft_mask_andc_return(IRQS_PMI_DISABLED);
392	get_paca()->irq_happened &= ~PACA_IRQ_HARD_DIS;
393	__hard_irq_enable();
394}
395
396static inline bool arch_irq_disabled_regs(struct pt_regs *regs)
397{
398	return (regs->softe & IRQS_DISABLED);
399}
400
401extern bool prep_irq_for_idle(void);
402extern bool prep_irq_for_idle_irqsoff(void);
403extern void irq_set_pending_from_srr1(unsigned long srr1);
404
405#define fini_irq_for_idle_irqsoff() trace_hardirqs_off();
406
407extern void force_external_irq_replay(void);
408
409static inline void irq_soft_mask_regs_set_state(struct pt_regs *regs, unsigned long val)
410{
411	regs->softe = val;
412}
413#else /* CONFIG_PPC64 */
414
415static inline notrace unsigned long irq_soft_mask_return(void)
416{
417	return 0;
418}
419
420static inline unsigned long arch_local_save_flags(void)
421{
422	return mfmsr();
423}
424
425static inline void arch_local_irq_restore(unsigned long flags)
426{
427	if (IS_ENABLED(CONFIG_BOOKE))
428		wrtee(flags);
429	else
430		mtmsr(flags);
431}
432
433static inline unsigned long arch_local_irq_save(void)
434{
435	unsigned long flags = arch_local_save_flags();
436
437	if (IS_ENABLED(CONFIG_BOOKE))
438		wrtee(0);
439	else if (IS_ENABLED(CONFIG_PPC_8xx))
440		wrtspr(SPRN_EID);
441	else
442		mtmsr(flags & ~MSR_EE);
443
444	return flags;
445}
446
447static inline void arch_local_irq_disable(void)
448{
449	__hard_irq_disable();
450}
451
452static inline void arch_local_irq_enable(void)
453{
454	__hard_irq_enable();
455}
456
457static inline bool arch_irqs_disabled_flags(unsigned long flags)
458{
459	return (flags & MSR_EE) == 0;
460}
461
462static inline bool arch_irqs_disabled(void)
463{
464	return arch_irqs_disabled_flags(arch_local_save_flags());
465}
466
467#define hard_irq_disable()		arch_local_irq_disable()
468
469static inline bool arch_irq_disabled_regs(struct pt_regs *regs)
470{
471	return !(regs->msr & MSR_EE);
472}
473
474static __always_inline bool should_hard_irq_enable(struct pt_regs *regs)
475{
476	return false;
477}
478
479static inline void do_hard_irq_enable(void)
480{
481	BUILD_BUG();
482}
483
484static inline void clear_pmi_irq_pending(void) { }
485static inline void set_pmi_irq_pending(void) { }
486static inline bool pmi_irq_pending(void) { return false; }
487
488static inline void irq_soft_mask_regs_set_state(struct pt_regs *regs, unsigned long val)
489{
490}
491#endif /* CONFIG_PPC64 */
492
493static inline unsigned long mtmsr_isync_irqsafe(unsigned long msr)
494{
495#ifdef CONFIG_PPC64
496	if (arch_irqs_disabled()) {
497		/*
498		 * With soft-masking, MSR[EE] can change from 1 to 0
499		 * asynchronously when irqs are disabled, and we don't want to
500		 * set MSR[EE] back to 1 here if that has happened. A race-free
501		 * way to do this is ensure EE is already 0. Another way it
502		 * could be done is with a RESTART_TABLE handler, but that's
503		 * probably overkill here.
504		 */
505		msr &= ~MSR_EE;
506		mtmsr_isync(msr);
507		irq_soft_mask_set(IRQS_ALL_DISABLED);
508		local_paca->irq_happened |= PACA_IRQ_HARD_DIS;
509	} else
510#endif
511		mtmsr_isync(msr);
512
513	return msr;
514}
515
516
517#define ARCH_IRQ_INIT_FLAGS	IRQ_NOREQUEST
518
519#endif  /* __ASSEMBLY__ */
520#endif	/* __KERNEL__ */
521#endif	/* _ASM_POWERPC_HW_IRQ_H */
522