1// SPDX-License-Identifier: GPL-2.0-only
2/*
3 * Based on arch/arm/kernel/irq.c
4 *
5 * Copyright (C) 1992 Linus Torvalds
6 * Modifications for ARM processor Copyright (C) 1995-2000 Russell King.
7 * Support for Dynamic Tick Timer Copyright (C) 2004-2005 Nokia Corporation.
8 * Dynamic Tick Timer written by Tony Lindgren <tony@atomide.com> and
9 * Tuukka Tikkanen <tuukka.tikkanen@elektrobit.com>.
10 * Copyright (C) 2012 ARM Ltd.
11 */
12
13#include <linux/hardirq.h>
14#include <linux/init.h>
15#include <linux/irq.h>
16#include <linux/irqchip.h>
17#include <linux/kprobes.h>
18#include <linux/memory.h>
19#include <linux/scs.h>
20#include <linux/seq_file.h>
21#include <linux/smp.h>
22#include <linux/vmalloc.h>
23#include <asm/daifflags.h>
24#include <asm/exception.h>
25#include <asm/numa.h>
26#include <asm/softirq_stack.h>
27#include <asm/stacktrace.h>
28#include <asm/vmap_stack.h>
29
30/* Only access this in an NMI enter/exit */
31DEFINE_PER_CPU(struct nmi_ctx, nmi_contexts);
32
33DEFINE_PER_CPU(unsigned long *, irq_stack_ptr);
34
35
36DECLARE_PER_CPU(unsigned long *, irq_shadow_call_stack_ptr);
37
38#ifdef CONFIG_SHADOW_CALL_STACK
39DEFINE_PER_CPU(unsigned long *, irq_shadow_call_stack_ptr);
40#endif
41
42static void init_irq_scs(void)
43{
44	int cpu;
45
46	if (!scs_is_enabled())
47		return;
48
49	for_each_possible_cpu(cpu)
50		per_cpu(irq_shadow_call_stack_ptr, cpu) =
51			scs_alloc(early_cpu_to_node(cpu));
52}
53
54#ifdef CONFIG_VMAP_STACK
55static void __init init_irq_stacks(void)
56{
57	int cpu;
58	unsigned long *p;
59
60	for_each_possible_cpu(cpu) {
61		p = arch_alloc_vmap_stack(IRQ_STACK_SIZE, early_cpu_to_node(cpu));
62		per_cpu(irq_stack_ptr, cpu) = p;
63	}
64}
65#else
66/* irq stack only needs to be 16 byte aligned - not IRQ_STACK_SIZE aligned. */
67DEFINE_PER_CPU_ALIGNED(unsigned long [IRQ_STACK_SIZE/sizeof(long)], irq_stack);
68
69static void init_irq_stacks(void)
70{
71	int cpu;
72
73	for_each_possible_cpu(cpu)
74		per_cpu(irq_stack_ptr, cpu) = per_cpu(irq_stack, cpu);
75}
76#endif
77
78#ifndef CONFIG_PREEMPT_RT
79static void ____do_softirq(struct pt_regs *regs)
80{
81	__do_softirq();
82}
83
84void do_softirq_own_stack(void)
85{
86	call_on_irq_stack(NULL, ____do_softirq);
87}
88#endif
89
90static void default_handle_irq(struct pt_regs *regs)
91{
92	panic("IRQ taken without a root IRQ handler\n");
93}
94
95static void default_handle_fiq(struct pt_regs *regs)
96{
97	panic("FIQ taken without a root FIQ handler\n");
98}
99
100void (*handle_arch_irq)(struct pt_regs *) __ro_after_init = default_handle_irq;
101void (*handle_arch_fiq)(struct pt_regs *) __ro_after_init = default_handle_fiq;
102
103int __init set_handle_irq(void (*handle_irq)(struct pt_regs *))
104{
105	if (handle_arch_irq != default_handle_irq)
106		return -EBUSY;
107
108	handle_arch_irq = handle_irq;
109	pr_info("Root IRQ handler: %ps\n", handle_irq);
110	return 0;
111}
112
113int __init set_handle_fiq(void (*handle_fiq)(struct pt_regs *))
114{
115	if (handle_arch_fiq != default_handle_fiq)
116		return -EBUSY;
117
118	handle_arch_fiq = handle_fiq;
119	pr_info("Root FIQ handler: %ps\n", handle_fiq);
120	return 0;
121}
122
123void __init init_IRQ(void)
124{
125	init_irq_stacks();
126	init_irq_scs();
127	irqchip_init();
128
129	if (system_uses_irq_prio_masking()) {
130		/*
131		 * Now that we have a stack for our IRQ handler, set
132		 * the PMR/PSR pair to a consistent state.
133		 */
134		WARN_ON(read_sysreg(daif) & PSR_A_BIT);
135		local_daif_restore(DAIF_PROCCTX_NOIRQ);
136	}
137}
138