1/* SPDX-License-Identifier: GPL-2.0 */ 2#ifndef _ASM_X86_IRQ_STACK_H 3#define _ASM_X86_IRQ_STACK_H 4 5#include <linux/ptrace.h> 6#include <linux/objtool.h> 7 8#include <asm/processor.h> 9 10#ifdef CONFIG_X86_64 11 12/* 13 * Macro to inline switching to an interrupt stack and invoking function 14 * calls from there. The following rules apply: 15 * 16 * - Ordering: 17 * 18 * 1. Write the stack pointer into the top most place of the irq 19 * stack. This ensures that the various unwinders can link back to the 20 * original stack. 21 * 22 * 2. Switch the stack pointer to the top of the irq stack. 23 * 24 * 3. Invoke whatever needs to be done (@asm_call argument) 25 * 26 * 4. Pop the original stack pointer from the top of the irq stack 27 * which brings it back to the original stack where it left off. 28 * 29 * - Function invocation: 30 * 31 * To allow flexible usage of the macro, the actual function code including 32 * the store of the arguments in the call ABI registers is handed in via 33 * the @asm_call argument. 34 * 35 * - Local variables: 36 * 37 * @tos: 38 * The @tos variable holds a pointer to the top of the irq stack and 39 * _must_ be allocated in a non-callee saved register as this is a 40 * restriction coming from objtool. 41 * 42 * Note, that (tos) is both in input and output constraints to ensure 43 * that the compiler does not assume that R11 is left untouched in 44 * case this macro is used in some place where the per cpu interrupt 45 * stack pointer is used again afterwards 46 * 47 * - Function arguments: 48 * The function argument(s), if any, have to be defined in register 49 * variables at the place where this is invoked. Storing the 50 * argument(s) in the proper register(s) is part of the @asm_call 51 * 52 * - Constraints: 53 * 54 * The constraints have to be done very carefully because the compiler 55 * does not know about the assembly call. 56 * 57 * output: 58 * As documented already above the @tos variable is required to be in 59 * the output constraints to make the compiler aware that R11 cannot be 60 * reused after the asm() statement. 61 * 62 * For builds with CONFIG_UNWINDER_FRAME_POINTER, ASM_CALL_CONSTRAINT is 63 * required as well as this prevents certain creative GCC variants from 64 * misplacing the ASM code. 65 * 66 * input: 67 * - func: 68 * Immediate, which tells the compiler that the function is referenced. 69 * 70 * - tos: 71 * Register. The actual register is defined by the variable declaration. 72 * 73 * - function arguments: 74 * The constraints are handed in via the 'argconstr' argument list. They 75 * describe the register arguments which are used in @asm_call. 76 * 77 * clobbers: 78 * Function calls can clobber anything except the callee-saved 79 * registers. Tell the compiler. 80 */ 81#define call_on_stack(stack, func, asm_call, argconstr...) \ 82{ \ 83 register void *tos asm("r11"); \ 84 \ 85 tos = ((void *)(stack)); \ 86 \ 87 asm_inline volatile( \ 88 "movq %%rsp, (%[tos]) \n" \ 89 "movq %[tos], %%rsp \n" \ 90 \ 91 asm_call \ 92 \ 93 "popq %%rsp \n" \ 94 \ 95 : "+r" (tos), ASM_CALL_CONSTRAINT \ 96 : [__func] "i" (func), [tos] "r" (tos) argconstr \ 97 : "cc", "rax", "rcx", "rdx", "rsi", "rdi", "r8", "r9", "r10", \ 98 "memory" \ 99 ); \ 100} 101 102#define ASM_CALL_ARG0 \ 103 "call %P[__func] \n" \ 104 ASM_REACHABLE 105 106#define ASM_CALL_ARG1 \ 107 "movq %[arg1], %%rdi \n" \ 108 ASM_CALL_ARG0 109 110#define ASM_CALL_ARG2 \ 111 "movq %[arg2], %%rsi \n" \ 112 ASM_CALL_ARG1 113 114#define ASM_CALL_ARG3 \ 115 "movq %[arg3], %%rdx \n" \ 116 ASM_CALL_ARG2 117 118#define call_on_irqstack(func, asm_call, argconstr...) \ 119 call_on_stack(__this_cpu_read(pcpu_hot.hardirq_stack_ptr), \ 120 func, asm_call, argconstr) 121 122/* Macros to assert type correctness for run_*_on_irqstack macros */ 123#define assert_function_type(func, proto) \ 124 static_assert(__builtin_types_compatible_p(typeof(&func), proto)) 125 126#define assert_arg_type(arg, proto) \ 127 static_assert(__builtin_types_compatible_p(typeof(arg), proto)) 128 129/* 130 * Macro to invoke system vector and device interrupt C handlers. 131 */ 132#define call_on_irqstack_cond(func, regs, asm_call, constr, c_args...) \ 133{ \ 134 /* \ 135 * User mode entry and interrupt on the irq stack do not \ 136 * switch stacks. If from user mode the task stack is empty. \ 137 */ \ 138 if (user_mode(regs) || __this_cpu_read(pcpu_hot.hardirq_stack_inuse)) { \ 139 irq_enter_rcu(); \ 140 func(c_args); \ 141 irq_exit_rcu(); \ 142 } else { \ 143 /* \ 144 * Mark the irq stack inuse _before_ and unmark _after_ \ 145 * switching stacks. Interrupts are disabled in both \ 146 * places. Invoke the stack switch macro with the call \ 147 * sequence which matches the above direct invocation. \ 148 */ \ 149 __this_cpu_write(pcpu_hot.hardirq_stack_inuse, true); \ 150 call_on_irqstack(func, asm_call, constr); \ 151 __this_cpu_write(pcpu_hot.hardirq_stack_inuse, false); \ 152 } \ 153} 154 155/* 156 * Function call sequence for __call_on_irqstack() for system vectors. 157 * 158 * Note that irq_enter_rcu() and irq_exit_rcu() do not use the input 159 * mechanism because these functions are global and cannot be optimized out 160 * when compiling a particular source file which uses one of these macros. 161 * 162 * The argument (regs) does not need to be pushed or stashed in a callee 163 * saved register to be safe vs. the irq_enter_rcu() call because the 164 * clobbers already prevent the compiler from storing it in a callee 165 * clobbered register. As the compiler has to preserve @regs for the final 166 * call to idtentry_exit() anyway, it's likely that it does not cause extra 167 * effort for this asm magic. 168 */ 169#define ASM_CALL_SYSVEC \ 170 "call irq_enter_rcu \n" \ 171 ASM_CALL_ARG1 \ 172 "call irq_exit_rcu \n" 173 174#define SYSVEC_CONSTRAINTS , [arg1] "r" (regs) 175 176#define run_sysvec_on_irqstack_cond(func, regs) \ 177{ \ 178 assert_function_type(func, void (*)(struct pt_regs *)); \ 179 assert_arg_type(regs, struct pt_regs *); \ 180 \ 181 call_on_irqstack_cond(func, regs, ASM_CALL_SYSVEC, \ 182 SYSVEC_CONSTRAINTS, regs); \ 183} 184 185/* 186 * As in ASM_CALL_SYSVEC above the clobbers force the compiler to store 187 * @regs and @vector in callee saved registers. 188 */ 189#define ASM_CALL_IRQ \ 190 "call irq_enter_rcu \n" \ 191 ASM_CALL_ARG2 \ 192 "call irq_exit_rcu \n" 193 194#define IRQ_CONSTRAINTS , [arg1] "r" (regs), [arg2] "r" ((unsigned long)vector) 195 196#define run_irq_on_irqstack_cond(func, regs, vector) \ 197{ \ 198 assert_function_type(func, void (*)(struct pt_regs *, u32)); \ 199 assert_arg_type(regs, struct pt_regs *); \ 200 assert_arg_type(vector, u32); \ 201 \ 202 call_on_irqstack_cond(func, regs, ASM_CALL_IRQ, \ 203 IRQ_CONSTRAINTS, regs, vector); \ 204} 205 206#ifdef CONFIG_SOFTIRQ_ON_OWN_STACK 207/* 208 * Macro to invoke __do_softirq on the irq stack. This is only called from 209 * task context when bottom halves are about to be reenabled and soft 210 * interrupts are pending to be processed. The interrupt stack cannot be in 211 * use here. 212 */ 213#define do_softirq_own_stack() \ 214{ \ 215 __this_cpu_write(pcpu_hot.hardirq_stack_inuse, true); \ 216 call_on_irqstack(__do_softirq, ASM_CALL_ARG0); \ 217 __this_cpu_write(pcpu_hot.hardirq_stack_inuse, false); \ 218} 219 220#endif 221 222#else /* CONFIG_X86_64 */ 223/* System vector handlers always run on the stack they interrupted. */ 224#define run_sysvec_on_irqstack_cond(func, regs) \ 225{ \ 226 irq_enter_rcu(); \ 227 func(regs); \ 228 irq_exit_rcu(); \ 229} 230 231/* Switches to the irq stack within func() */ 232#define run_irq_on_irqstack_cond(func, regs, vector) \ 233{ \ 234 irq_enter_rcu(); \ 235 func(regs, vector); \ 236 irq_exit_rcu(); \ 237} 238 239#endif /* !CONFIG_X86_64 */ 240 241#endif 242