1// SPDX-License-Identifier: GPL-2.0 2/* 3 * preemptoff and irqoff tracepoints 4 * 5 * Copyright (C) Joel Fernandes (Google) <joel@joelfernandes.org> 6 */ 7 8#include <linux/kallsyms.h> 9#include <linux/uaccess.h> 10#include <linux/module.h> 11#include <linux/ftrace.h> 12#include <linux/kprobes.h> 13#include "trace.h" 14 15#define CREATE_TRACE_POINTS 16#include <trace/events/preemptirq.h> 17 18/* 19 * Use regular trace points on architectures that implement noinstr 20 * tooling: these calls will only happen with RCU enabled, which can 21 * use a regular tracepoint. 22 * 23 * On older architectures, use the rcuidle tracing methods (which 24 * aren't NMI-safe - so exclude NMI contexts): 25 */ 26#ifdef CONFIG_ARCH_WANTS_NO_INSTR 27#define trace(point) trace_##point 28#else 29#define trace(point) if (!in_nmi()) trace_##point##_rcuidle 30#endif 31 32#ifdef CONFIG_TRACE_IRQFLAGS 33/* Per-cpu variable to prevent redundant calls when IRQs already off */ 34static DEFINE_PER_CPU(int, tracing_irq_cpu); 35 36/* 37 * Like trace_hardirqs_on() but without the lockdep invocation. This is 38 * used in the low level entry code where the ordering vs. RCU is important 39 * and lockdep uses a staged approach which splits the lockdep hardirq 40 * tracking into a RCU on and a RCU off section. 41 */ 42void trace_hardirqs_on_prepare(void) 43{ 44 if (this_cpu_read(tracing_irq_cpu)) { 45 trace(irq_enable)(CALLER_ADDR0, CALLER_ADDR1); 46 tracer_hardirqs_on(CALLER_ADDR0, CALLER_ADDR1); 47 this_cpu_write(tracing_irq_cpu, 0); 48 } 49} 50EXPORT_SYMBOL(trace_hardirqs_on_prepare); 51NOKPROBE_SYMBOL(trace_hardirqs_on_prepare); 52 53void trace_hardirqs_on(void) 54{ 55 if (this_cpu_read(tracing_irq_cpu)) { 56 trace(irq_enable)(CALLER_ADDR0, CALLER_ADDR1); 57 tracer_hardirqs_on(CALLER_ADDR0, CALLER_ADDR1); 58 this_cpu_write(tracing_irq_cpu, 0); 59 } 60 61 lockdep_hardirqs_on_prepare(); 62 lockdep_hardirqs_on(CALLER_ADDR0); 63} 64EXPORT_SYMBOL(trace_hardirqs_on); 65NOKPROBE_SYMBOL(trace_hardirqs_on); 66 67/* 68 * Like trace_hardirqs_off() but without the lockdep invocation. This is 69 * used in the low level entry code where the ordering vs. RCU is important 70 * and lockdep uses a staged approach which splits the lockdep hardirq 71 * tracking into a RCU on and a RCU off section. 72 */ 73void trace_hardirqs_off_finish(void) 74{ 75 if (!this_cpu_read(tracing_irq_cpu)) { 76 this_cpu_write(tracing_irq_cpu, 1); 77 tracer_hardirqs_off(CALLER_ADDR0, CALLER_ADDR1); 78 trace(irq_disable)(CALLER_ADDR0, CALLER_ADDR1); 79 } 80 81} 82EXPORT_SYMBOL(trace_hardirqs_off_finish); 83NOKPROBE_SYMBOL(trace_hardirqs_off_finish); 84 85void trace_hardirqs_off(void) 86{ 87 lockdep_hardirqs_off(CALLER_ADDR0); 88 89 if (!this_cpu_read(tracing_irq_cpu)) { 90 this_cpu_write(tracing_irq_cpu, 1); 91 tracer_hardirqs_off(CALLER_ADDR0, CALLER_ADDR1); 92 trace(irq_disable)(CALLER_ADDR0, CALLER_ADDR1); 93 } 94} 95EXPORT_SYMBOL(trace_hardirqs_off); 96NOKPROBE_SYMBOL(trace_hardirqs_off); 97#endif /* CONFIG_TRACE_IRQFLAGS */ 98 99#ifdef CONFIG_TRACE_PREEMPT_TOGGLE 100 101void trace_preempt_on(unsigned long a0, unsigned long a1) 102{ 103 trace(preempt_enable)(a0, a1); 104 tracer_preempt_on(a0, a1); 105} 106 107void trace_preempt_off(unsigned long a0, unsigned long a1) 108{ 109 trace(preempt_disable)(a0, a1); 110 tracer_preempt_off(a0, a1); 111} 112#endif 113