1/* 2 * Code for replacing ftrace calls with jumps. 3 * 4 * Copyright (C) 2007-2008 Steven Rostedt <srostedt@redhat.com> 5 * Copyright (C) 2009, 2010 DSLab, Lanzhou University, China 6 * Author: Wu Zhangjin <wuzhangjin@gmail.com> 7 * 8 * Thanks goes to Steven Rostedt for writing the original x86 version. 9 */ 10 11#include <linux/uaccess.h> 12#include <linux/init.h> 13#include <linux/ftrace.h> 14 15#include <asm/asm.h> 16#include <asm/asm-offsets.h> 17#include <asm/cacheflush.h> 18#include <asm/uasm.h> 19 20 21static inline int in_module(unsigned long ip) 22{ 23 return ip & 0x40000000; 24} 25 26#ifdef CONFIG_DYNAMIC_FTRACE 27 28#define JAL 0x0c000000 /* jump & link: ip --> ra, jump to target */ 29#define ADDR_MASK 0x03ffffff /* op_code|addr : 31...26|25 ....0 */ 30#define JUMP_RANGE_MASK ((1UL << 28) - 1) 31 32#define INSN_B_1F_4 0x10000004 /* b 1f; offset = 4 */ 33#define INSN_B_1F_5 0x10000005 /* b 1f; offset = 5 */ 34#define INSN_NOP 0x00000000 /* nop */ 35#define INSN_JAL(addr) \ 36 ((unsigned int)(JAL | (((addr) >> 2) & ADDR_MASK))) 37 38static unsigned int insn_jal_ftrace_caller __read_mostly; 39static unsigned int insn_lui_v1_hi16_mcount __read_mostly; 40static unsigned int insn_j_ftrace_graph_caller __maybe_unused __read_mostly; 41 42static inline void ftrace_dyn_arch_init_insns(void) 43{ 44 u32 *buf; 45 unsigned int v1; 46 47 /* lui v1, hi16_mcount */ 48 v1 = 3; 49 buf = (u32 *)&insn_lui_v1_hi16_mcount; 50 UASM_i_LA_mostly(&buf, v1, MCOUNT_ADDR); 51 52 /* jal (ftrace_caller + 8), jump over the first two instruction */ 53 buf = (u32 *)&insn_jal_ftrace_caller; 54 uasm_i_jal(&buf, (FTRACE_ADDR + 8) & JUMP_RANGE_MASK); 55 56#ifdef CONFIG_FUNCTION_GRAPH_TRACER 57 /* j ftrace_graph_caller */ 58 buf = (u32 *)&insn_j_ftrace_graph_caller; 59 uasm_i_j(&buf, (unsigned long)ftrace_graph_caller & JUMP_RANGE_MASK); 60#endif 61} 62 63static int ftrace_modify_code(unsigned long ip, unsigned int new_code) 64{ 65 int faulted; 66 67 /* *(unsigned int *)ip = new_code; */ 68 safe_store_code(new_code, ip, faulted); 69 70 if (unlikely(faulted)) 71 return -EFAULT; 72 73 flush_icache_range(ip, ip + 8); 74 75 return 0; 76} 77 78int ftrace_make_nop(struct module *mod, 79 struct dyn_ftrace *rec, unsigned long addr) 80{ 81 unsigned int new; 82 unsigned long ip = rec->ip; 83 84 /* 85 * We have compiled module with -mlong-calls, but compiled the kernel 86 * without it, we need to cope with them respectively. 87 */ 88 if (in_module(ip)) { 89#if defined(KBUILD_MCOUNT_RA_ADDRESS) && defined(CONFIG_32BIT) 90 /* 91 * lui v1, hi_16bit_of_mcount --> b 1f (0x10000005) 92 * addiu v1, v1, low_16bit_of_mcount 93 * move at, ra 94 * move $12, ra_address 95 * jalr v1 96 * sub sp, sp, 8 97 * 1: offset = 5 instructions 98 */ 99 new = INSN_B_1F_5; 100#else 101 /* 102 * lui v1, hi_16bit_of_mcount --> b 1f (0x10000004) 103 * addiu v1, v1, low_16bit_of_mcount 104 * move at, ra 105 * jalr v1 106 * nop | move $12, ra_address | sub sp, sp, 8 107 * 1: offset = 4 instructions 108 */ 109 new = INSN_B_1F_4; 110#endif 111 } else { 112 /* 113 * move at, ra 114 * jal _mcount --> nop 115 */ 116 new = INSN_NOP; 117 } 118 return ftrace_modify_code(ip, new); 119} 120 121int ftrace_make_call(struct dyn_ftrace *rec, unsigned long addr) 122{ 123 unsigned int new; 124 unsigned long ip = rec->ip; 125 126 /* ip, module: 0xc0000000, kernel: 0x80000000 */ 127 new = in_module(ip) ? insn_lui_v1_hi16_mcount : insn_jal_ftrace_caller; 128 129 return ftrace_modify_code(ip, new); 130} 131 132#define FTRACE_CALL_IP ((unsigned long)(&ftrace_call)) 133 134int ftrace_update_ftrace_func(ftrace_func_t func) 135{ 136 unsigned int new; 137 138 new = INSN_JAL((unsigned long)func); 139 140 return ftrace_modify_code(FTRACE_CALL_IP, new); 141} 142 143int __init ftrace_dyn_arch_init(void *data) 144{ 145 /* Encode the instructions when booting */ 146 ftrace_dyn_arch_init_insns(); 147 148 /* Remove "b ftrace_stub" to ensure ftrace_caller() is executed */ 149 ftrace_modify_code(MCOUNT_ADDR, INSN_NOP); 150 151 /* The return code is retured via data */ 152 *(unsigned long *)data = 0; 153 154 return 0; 155} 156#endif /* CONFIG_DYNAMIC_FTRACE */ 157 158#ifdef CONFIG_FUNCTION_GRAPH_TRACER 159 160#ifdef CONFIG_DYNAMIC_FTRACE 161 162extern void ftrace_graph_call(void); 163#define FTRACE_GRAPH_CALL_IP ((unsigned long)(&ftrace_graph_call)) 164 165int ftrace_enable_ftrace_graph_caller(void) 166{ 167 return ftrace_modify_code(FTRACE_GRAPH_CALL_IP, 168 insn_j_ftrace_graph_caller); 169} 170 171int ftrace_disable_ftrace_graph_caller(void) 172{ 173 return ftrace_modify_code(FTRACE_GRAPH_CALL_IP, INSN_NOP); 174} 175 176#endif /* CONFIG_DYNAMIC_FTRACE */ 177 178#ifndef KBUILD_MCOUNT_RA_ADDRESS 179 180#define S_RA_SP (0xafbf << 16) /* s{d,w} ra, offset(sp) */ 181#define S_R_SP (0xafb0 << 16) /* s{d,w} R, offset(sp) */ 182#define OFFSET_MASK 0xffff /* stack offset range: 0 ~ PT_SIZE */ 183 184unsigned long ftrace_get_parent_addr(unsigned long self_addr, 185 unsigned long parent, 186 unsigned long parent_addr, 187 unsigned long fp) 188{ 189 unsigned long sp, ip, ra; 190 unsigned int code; 191 int faulted; 192 193 /* 194 * For module, move the ip from calling site of mcount to the 195 * instruction "lui v1, hi_16bit_of_mcount"(offset is 20), but for 196 * kernel, move to the instruction "move ra, at"(offset is 12) 197 */ 198 ip = self_addr - (in_module(self_addr) ? 20 : 12); 199 200 /* 201 * search the text until finding the non-store instruction or "s{d,w} 202 * ra, offset(sp)" instruction 203 */ 204 do { 205 ip -= 4; 206 207 /* get the code at "ip": code = *(unsigned int *)ip; */ 208 safe_load_code(code, ip, faulted); 209 210 if (unlikely(faulted)) 211 return 0; 212 /* 213 * If we hit the non-store instruction before finding where the 214 * ra is stored, then this is a leaf function and it does not 215 * store the ra on the stack 216 */ 217 if ((code & S_R_SP) != S_R_SP) 218 return parent_addr; 219 220 } while (((code & S_RA_SP) != S_RA_SP)); 221 222 sp = fp + (code & OFFSET_MASK); 223 224 /* ra = *(unsigned long *)sp; */ 225 safe_load_stack(ra, sp, faulted); 226 if (unlikely(faulted)) 227 return 0; 228 229 if (ra == parent) 230 return sp; 231 return 0; 232} 233 234#endif /* !KBUILD_MCOUNT_RA_ADDRESS */ 235 236/* 237 * Hook the return address and push it in the stack of return addrs 238 * in current thread info. 239 */ 240void prepare_ftrace_return(unsigned long *parent, unsigned long self_addr, 241 unsigned long fp) 242{ 243 unsigned long old; 244 struct ftrace_graph_ent trace; 245 unsigned long return_hooker = (unsigned long) 246 &return_to_handler; 247 int faulted; 248 249 if (unlikely(atomic_read(¤t->tracing_graph_pause))) 250 return; 251 252 /* 253 * "parent" is the stack address saved the return address of the caller 254 * of _mcount. 255 * 256 * if the gcc < 4.5, a leaf function does not save the return address 257 * in the stack address, so, we "emulate" one in _mcount's stack space, 258 * and hijack it directly, but for a non-leaf function, it save the 259 * return address to the its own stack space, we can not hijack it 260 * directly, but need to find the real stack address, 261 * ftrace_get_parent_addr() does it! 262 * 263 * if gcc>= 4.5, with the new -mmcount-ra-address option, for a 264 * non-leaf function, the location of the return address will be saved 265 * to $12 for us, and for a leaf function, only put a zero into $12. we 266 * do it in ftrace_graph_caller of mcount.S. 267 */ 268 269 /* old = *parent; */ 270 safe_load_stack(old, parent, faulted); 271 if (unlikely(faulted)) 272 goto out; 273#ifndef KBUILD_MCOUNT_RA_ADDRESS 274 parent = (unsigned long *)ftrace_get_parent_addr(self_addr, old, 275 (unsigned long)parent, fp); 276 /* 277 * If fails when getting the stack address of the non-leaf function's 278 * ra, stop function graph tracer and return 279 */ 280 if (parent == 0) 281 goto out; 282#endif 283 /* *parent = return_hooker; */ 284 safe_store_stack(return_hooker, parent, faulted); 285 if (unlikely(faulted)) 286 goto out; 287 288 if (ftrace_push_return_trace(old, self_addr, &trace.depth, fp) == 289 -EBUSY) { 290 *parent = old; 291 return; 292 } 293 294 trace.func = self_addr; 295 296 /* Only trace if the calling function expects to */ 297 if (!ftrace_graph_entry(&trace)) { 298 current->curr_ret_stack--; 299 *parent = old; 300 } 301 return; 302out: 303 ftrace_graph_stop(); 304 WARN_ON(1); 305} 306#endif /* CONFIG_FUNCTION_GRAPH_TRACER */ 307