1/* 2 * Kernel Probes (KProbes) 3 * 4 * This program is free software; you can redistribute it and/or modify 5 * it under the terms of the GNU General Public License as published by 6 * the Free Software Foundation; either version 2 of the License, or 7 * (at your option) any later version. 8 * 9 * This program is distributed in the hope that it will be useful, 10 * but WITHOUT ANY WARRANTY; without even the implied warranty of 11 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 * GNU General Public License for more details. 13 * 14 * You should have received a copy of the GNU General Public License 15 * along with this program; if not, write to the Free Software 16 * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. 17 * 18 * Copyright (C) IBM Corporation, 2002, 2004 19 * 20 * 2002-Oct Created by Vamsi Krishna S <vamsi_krishna@in.ibm.com> Kernel 21 * Probes initial implementation ( includes contributions from 22 * Rusty Russell). 23 * 2004-July Suparna Bhattacharya <suparna@in.ibm.com> added jumper probes 24 * interface to access function arguments. 25 * 2004-Nov Ananth N Mavinakayanahalli <ananth@in.ibm.com> kprobes port 26 * for PPC64 27 */ 28 29#include <linux/kprobes.h> 30#include <linux/ptrace.h> 31#include <linux/preempt.h> 32#include <linux/module.h> 33#include <linux/kdebug.h> 34#include <linux/slab.h> 35#include <asm/cacheflush.h> 36#include <asm/sstep.h> 37#include <asm/uaccess.h> 38#include <asm/system.h> 39 40#ifdef CONFIG_PPC_ADV_DEBUG_REGS 41#define MSR_SINGLESTEP (MSR_DE) 42#else 43#define MSR_SINGLESTEP (MSR_SE) 44#endif 45 46DEFINE_PER_CPU(struct kprobe *, current_kprobe) = NULL; 47DEFINE_PER_CPU(struct kprobe_ctlblk, kprobe_ctlblk); 48 49struct kretprobe_blackpoint kretprobe_blacklist[] = {{NULL, NULL}}; 50 51int __kprobes arch_prepare_kprobe(struct kprobe *p) 52{ 53 int ret = 0; 54 kprobe_opcode_t insn = *p->addr; 55 56 if ((unsigned long)p->addr & 0x03) { 57 printk("Attempt to register kprobe at an unaligned address\n"); 58 ret = -EINVAL; 59 } else if (IS_MTMSRD(insn) || IS_RFID(insn) || IS_RFI(insn)) { 60 printk("Cannot register a kprobe on rfi/rfid or mtmsr[d]\n"); 61 ret = -EINVAL; 62 } 63 64 /* insn must be on a special executable page on ppc64. This is 65 * not explicitly required on ppc32 (right now), but it doesn't hurt */ 66 if (!ret) { 67 p->ainsn.insn = get_insn_slot(); 68 if (!p->ainsn.insn) 69 ret = -ENOMEM; 70 } 71 72 if (!ret) { 73 memcpy(p->ainsn.insn, p->addr, 74 MAX_INSN_SIZE * sizeof(kprobe_opcode_t)); 75 p->opcode = *p->addr; 76 flush_icache_range((unsigned long)p->ainsn.insn, 77 (unsigned long)p->ainsn.insn + sizeof(kprobe_opcode_t)); 78 } 79 80 p->ainsn.boostable = 0; 81 return ret; 82} 83 84void __kprobes arch_arm_kprobe(struct kprobe *p) 85{ 86 *p->addr = BREAKPOINT_INSTRUCTION; 87 flush_icache_range((unsigned long) p->addr, 88 (unsigned long) p->addr + sizeof(kprobe_opcode_t)); 89} 90 91void __kprobes arch_disarm_kprobe(struct kprobe *p) 92{ 93 *p->addr = p->opcode; 94 flush_icache_range((unsigned long) p->addr, 95 (unsigned long) p->addr + sizeof(kprobe_opcode_t)); 96} 97 98void __kprobes arch_remove_kprobe(struct kprobe *p) 99{ 100 if (p->ainsn.insn) { 101 free_insn_slot(p->ainsn.insn, 0); 102 p->ainsn.insn = NULL; 103 } 104} 105 106static void __kprobes prepare_singlestep(struct kprobe *p, struct pt_regs *regs) 107{ 108 /* We turn off async exceptions to ensure that the single step will 109 * be for the instruction we have the kprobe on, if we dont its 110 * possible we'd get the single step reported for an exception handler 111 * like Decrementer or External Interrupt */ 112 regs->msr &= ~MSR_EE; 113 regs->msr |= MSR_SINGLESTEP; 114#ifdef CONFIG_PPC_ADV_DEBUG_REGS 115 regs->msr &= ~MSR_CE; 116 mtspr(SPRN_DBCR0, mfspr(SPRN_DBCR0) | DBCR0_IC | DBCR0_IDM); 117#ifdef CONFIG_PPC_47x 118 isync(); 119#endif 120#endif 121 122 /* 123 * On powerpc we should single step on the original 124 * instruction even if the probed insn is a trap 125 * variant as values in regs could play a part in 126 * if the trap is taken or not 127 */ 128 regs->nip = (unsigned long)p->ainsn.insn; 129} 130 131static void __kprobes save_previous_kprobe(struct kprobe_ctlblk *kcb) 132{ 133 kcb->prev_kprobe.kp = kprobe_running(); 134 kcb->prev_kprobe.status = kcb->kprobe_status; 135 kcb->prev_kprobe.saved_msr = kcb->kprobe_saved_msr; 136} 137 138static void __kprobes restore_previous_kprobe(struct kprobe_ctlblk *kcb) 139{ 140 __get_cpu_var(current_kprobe) = kcb->prev_kprobe.kp; 141 kcb->kprobe_status = kcb->prev_kprobe.status; 142 kcb->kprobe_saved_msr = kcb->prev_kprobe.saved_msr; 143} 144 145static void __kprobes set_current_kprobe(struct kprobe *p, struct pt_regs *regs, 146 struct kprobe_ctlblk *kcb) 147{ 148 __get_cpu_var(current_kprobe) = p; 149 kcb->kprobe_saved_msr = regs->msr; 150} 151 152void __kprobes arch_prepare_kretprobe(struct kretprobe_instance *ri, 153 struct pt_regs *regs) 154{ 155 ri->ret_addr = (kprobe_opcode_t *)regs->link; 156 157 /* Replace the return addr with trampoline addr */ 158 regs->link = (unsigned long)kretprobe_trampoline; 159} 160 161static int __kprobes kprobe_handler(struct pt_regs *regs) 162{ 163 struct kprobe *p; 164 int ret = 0; 165 unsigned int *addr = (unsigned int *)regs->nip; 166 struct kprobe_ctlblk *kcb; 167 168 /* 169 * We don't want to be preempted for the entire 170 * duration of kprobe processing 171 */ 172 preempt_disable(); 173 kcb = get_kprobe_ctlblk(); 174 175 /* Check we're not actually recursing */ 176 if (kprobe_running()) { 177 p = get_kprobe(addr); 178 if (p) { 179 kprobe_opcode_t insn = *p->ainsn.insn; 180 if (kcb->kprobe_status == KPROBE_HIT_SS && 181 is_trap(insn)) { 182 /* Turn off 'trace' bits */ 183 regs->msr &= ~MSR_SINGLESTEP; 184 regs->msr |= kcb->kprobe_saved_msr; 185 goto no_kprobe; 186 } 187 /* We have reentered the kprobe_handler(), since 188 * another probe was hit while within the handler. 189 * We here save the original kprobes variables and 190 * just single step on the instruction of the new probe 191 * without calling any user handlers. 192 */ 193 save_previous_kprobe(kcb); 194 set_current_kprobe(p, regs, kcb); 195 kcb->kprobe_saved_msr = regs->msr; 196 kprobes_inc_nmissed_count(p); 197 prepare_singlestep(p, regs); 198 kcb->kprobe_status = KPROBE_REENTER; 199 return 1; 200 } else { 201 if (*addr != BREAKPOINT_INSTRUCTION) { 202 /* If trap variant, then it belongs not to us */ 203 kprobe_opcode_t cur_insn = *addr; 204 if (is_trap(cur_insn)) 205 goto no_kprobe; 206 /* The breakpoint instruction was removed by 207 * another cpu right after we hit, no further 208 * handling of this interrupt is appropriate 209 */ 210 ret = 1; 211 goto no_kprobe; 212 } 213 p = __get_cpu_var(current_kprobe); 214 if (p->break_handler && p->break_handler(p, regs)) { 215 goto ss_probe; 216 } 217 } 218 goto no_kprobe; 219 } 220 221 p = get_kprobe(addr); 222 if (!p) { 223 if (*addr != BREAKPOINT_INSTRUCTION) { 224 /* 225 * PowerPC has multiple variants of the "trap" 226 * instruction. If the current instruction is a 227 * trap variant, it could belong to someone else 228 */ 229 kprobe_opcode_t cur_insn = *addr; 230 if (is_trap(cur_insn)) 231 goto no_kprobe; 232 /* 233 * The breakpoint instruction was removed right 234 * after we hit it. Another cpu has removed 235 * either a probepoint or a debugger breakpoint 236 * at this address. In either case, no further 237 * handling of this interrupt is appropriate. 238 */ 239 ret = 1; 240 } 241 /* Not one of ours: let kernel handle it */ 242 goto no_kprobe; 243 } 244 245 kcb->kprobe_status = KPROBE_HIT_ACTIVE; 246 set_current_kprobe(p, regs, kcb); 247 if (p->pre_handler && p->pre_handler(p, regs)) 248 /* handler has already set things up, so skip ss setup */ 249 return 1; 250 251ss_probe: 252 if (p->ainsn.boostable >= 0) { 253 unsigned int insn = *p->ainsn.insn; 254 255 /* regs->nip is also adjusted if emulate_step returns 1 */ 256 ret = emulate_step(regs, insn); 257 if (ret > 0) { 258 /* 259 * Once this instruction has been boosted 260 * successfully, set the boostable flag 261 */ 262 if (unlikely(p->ainsn.boostable == 0)) 263 p->ainsn.boostable = 1; 264 265 if (p->post_handler) 266 p->post_handler(p, regs, 0); 267 268 kcb->kprobe_status = KPROBE_HIT_SSDONE; 269 reset_current_kprobe(); 270 preempt_enable_no_resched(); 271 return 1; 272 } else if (ret < 0) { 273 /* 274 * We don't allow kprobes on mtmsr(d)/rfi(d), etc. 275 * So, we should never get here... but, its still 276 * good to catch them, just in case... 277 */ 278 printk("Can't step on instruction %x\n", insn); 279 BUG(); 280 } else if (ret == 0) 281 /* This instruction can't be boosted */ 282 p->ainsn.boostable = -1; 283 } 284 prepare_singlestep(p, regs); 285 kcb->kprobe_status = KPROBE_HIT_SS; 286 return 1; 287 288no_kprobe: 289 preempt_enable_no_resched(); 290 return ret; 291} 292 293/* 294 * Function return probe trampoline: 295 * - init_kprobes() establishes a probepoint here 296 * - When the probed function returns, this probe 297 * causes the handlers to fire 298 */ 299static void __used kretprobe_trampoline_holder(void) 300{ 301 asm volatile(".global kretprobe_trampoline\n" 302 "kretprobe_trampoline:\n" 303 "nop\n"); 304} 305 306/* 307 * Called when the probe at kretprobe trampoline is hit 308 */ 309static int __kprobes trampoline_probe_handler(struct kprobe *p, 310 struct pt_regs *regs) 311{ 312 struct kretprobe_instance *ri = NULL; 313 struct hlist_head *head, empty_rp; 314 struct hlist_node *node, *tmp; 315 unsigned long flags, orig_ret_address = 0; 316 unsigned long trampoline_address =(unsigned long)&kretprobe_trampoline; 317 318 INIT_HLIST_HEAD(&empty_rp); 319 kretprobe_hash_lock(current, &head, &flags); 320 321 /* 322 * It is possible to have multiple instances associated with a given 323 * task either because an multiple functions in the call path 324 * have a return probe installed on them, and/or more than one return 325 * return probe was registered for a target function. 326 * 327 * We can handle this because: 328 * - instances are always inserted at the head of the list 329 * - when multiple return probes are registered for the same 330 * function, the first instance's ret_addr will point to the 331 * real return address, and all the rest will point to 332 * kretprobe_trampoline 333 */ 334 hlist_for_each_entry_safe(ri, node, tmp, head, hlist) { 335 if (ri->task != current) 336 /* another task is sharing our hash bucket */ 337 continue; 338 339 if (ri->rp && ri->rp->handler) 340 ri->rp->handler(ri, regs); 341 342 orig_ret_address = (unsigned long)ri->ret_addr; 343 recycle_rp_inst(ri, &empty_rp); 344 345 if (orig_ret_address != trampoline_address) 346 /* 347 * This is the real return address. Any other 348 * instances associated with this task are for 349 * other calls deeper on the call stack 350 */ 351 break; 352 } 353 354 kretprobe_assert(ri, orig_ret_address, trampoline_address); 355 regs->nip = orig_ret_address; 356 357 reset_current_kprobe(); 358 kretprobe_hash_unlock(current, &flags); 359 preempt_enable_no_resched(); 360 361 hlist_for_each_entry_safe(ri, node, tmp, &empty_rp, hlist) { 362 hlist_del(&ri->hlist); 363 kfree(ri); 364 } 365 /* 366 * By returning a non-zero value, we are telling 367 * kprobe_handler() that we don't want the post_handler 368 * to run (and have re-enabled preemption) 369 */ 370 return 1; 371} 372 373/* 374 * Called after single-stepping. p->addr is the address of the 375 * instruction whose first byte has been replaced by the "breakpoint" 376 * instruction. To avoid the SMP problems that can occur when we 377 * temporarily put back the original opcode to single-step, we 378 * single-stepped a copy of the instruction. The address of this 379 * copy is p->ainsn.insn. 380 */ 381static int __kprobes post_kprobe_handler(struct pt_regs *regs) 382{ 383 struct kprobe *cur = kprobe_running(); 384 struct kprobe_ctlblk *kcb = get_kprobe_ctlblk(); 385 386 if (!cur) 387 return 0; 388 389 /* make sure we got here for instruction we have a kprobe on */ 390 if (((unsigned long)cur->ainsn.insn + 4) != regs->nip) 391 return 0; 392 393 if ((kcb->kprobe_status != KPROBE_REENTER) && cur->post_handler) { 394 kcb->kprobe_status = KPROBE_HIT_SSDONE; 395 cur->post_handler(cur, regs, 0); 396 } 397 398 /* Adjust nip to after the single-stepped instruction */ 399 regs->nip = (unsigned long)cur->addr + 4; 400 regs->msr |= kcb->kprobe_saved_msr; 401 402 /*Restore back the original saved kprobes variables and continue. */ 403 if (kcb->kprobe_status == KPROBE_REENTER) { 404 restore_previous_kprobe(kcb); 405 goto out; 406 } 407 reset_current_kprobe(); 408out: 409 preempt_enable_no_resched(); 410 411 /* 412 * if somebody else is singlestepping across a probe point, msr 413 * will have DE/SE set, in which case, continue the remaining processing 414 * of do_debug, as if this is not a probe hit. 415 */ 416 if (regs->msr & MSR_SINGLESTEP) 417 return 0; 418 419 return 1; 420} 421 422int __kprobes kprobe_fault_handler(struct pt_regs *regs, int trapnr) 423{ 424 struct kprobe *cur = kprobe_running(); 425 struct kprobe_ctlblk *kcb = get_kprobe_ctlblk(); 426 const struct exception_table_entry *entry; 427 428 switch(kcb->kprobe_status) { 429 case KPROBE_HIT_SS: 430 case KPROBE_REENTER: 431 /* 432 * We are here because the instruction being single 433 * stepped caused a page fault. We reset the current 434 * kprobe and the nip points back to the probe address 435 * and allow the page fault handler to continue as a 436 * normal page fault. 437 */ 438 regs->nip = (unsigned long)cur->addr; 439 regs->msr &= ~MSR_SINGLESTEP; /* Turn off 'trace' bits */ 440 regs->msr |= kcb->kprobe_saved_msr; 441 if (kcb->kprobe_status == KPROBE_REENTER) 442 restore_previous_kprobe(kcb); 443 else 444 reset_current_kprobe(); 445 preempt_enable_no_resched(); 446 break; 447 case KPROBE_HIT_ACTIVE: 448 case KPROBE_HIT_SSDONE: 449 /* 450 * We increment the nmissed count for accounting, 451 * we can also use npre/npostfault count for accouting 452 * these specific fault cases. 453 */ 454 kprobes_inc_nmissed_count(cur); 455 456 /* 457 * We come here because instructions in the pre/post 458 * handler caused the page_fault, this could happen 459 * if handler tries to access user space by 460 * copy_from_user(), get_user() etc. Let the 461 * user-specified handler try to fix it first. 462 */ 463 if (cur->fault_handler && cur->fault_handler(cur, regs, trapnr)) 464 return 1; 465 466 /* 467 * In case the user-specified fault handler returned 468 * zero, try to fix up. 469 */ 470 if ((entry = search_exception_tables(regs->nip)) != NULL) { 471 regs->nip = entry->fixup; 472 return 1; 473 } 474 475 /* 476 * fixup_exception() could not handle it, 477 * Let do_page_fault() fix it. 478 */ 479 break; 480 default: 481 break; 482 } 483 return 0; 484} 485 486/* 487 * Wrapper routine to for handling exceptions. 488 */ 489int __kprobes kprobe_exceptions_notify(struct notifier_block *self, 490 unsigned long val, void *data) 491{ 492 struct die_args *args = (struct die_args *)data; 493 int ret = NOTIFY_DONE; 494 495 if (args->regs && user_mode(args->regs)) 496 return ret; 497 498 switch (val) { 499 case DIE_BPT: 500 if (kprobe_handler(args->regs)) 501 ret = NOTIFY_STOP; 502 break; 503 case DIE_SSTEP: 504 if (post_kprobe_handler(args->regs)) 505 ret = NOTIFY_STOP; 506 break; 507 default: 508 break; 509 } 510 return ret; 511} 512 513#ifdef CONFIG_PPC64 514unsigned long arch_deref_entry_point(void *entry) 515{ 516 return ((func_descr_t *)entry)->entry; 517} 518#endif 519 520int __kprobes setjmp_pre_handler(struct kprobe *p, struct pt_regs *regs) 521{ 522 struct jprobe *jp = container_of(p, struct jprobe, kp); 523 struct kprobe_ctlblk *kcb = get_kprobe_ctlblk(); 524 525 memcpy(&kcb->jprobe_saved_regs, regs, sizeof(struct pt_regs)); 526 527 /* setup return addr to the jprobe handler routine */ 528 regs->nip = arch_deref_entry_point(jp->entry); 529#ifdef CONFIG_PPC64 530 regs->gpr[2] = (unsigned long)(((func_descr_t *)jp->entry)->toc); 531#endif 532 533 return 1; 534} 535 536void __used __kprobes jprobe_return(void) 537{ 538 asm volatile("trap" ::: "memory"); 539} 540 541static void __used __kprobes jprobe_return_end(void) 542{ 543}; 544 545int __kprobes longjmp_break_handler(struct kprobe *p, struct pt_regs *regs) 546{ 547 struct kprobe_ctlblk *kcb = get_kprobe_ctlblk(); 548 549 memcpy(regs, &kcb->jprobe_saved_regs, sizeof(struct pt_regs)); 550 preempt_enable_no_resched(); 551 return 1; 552} 553 554static struct kprobe trampoline_p = { 555 .addr = (kprobe_opcode_t *) &kretprobe_trampoline, 556 .pre_handler = trampoline_probe_handler 557}; 558 559int __init arch_init_kprobes(void) 560{ 561 return register_kprobe(&trampoline_p); 562} 563 564int __kprobes arch_trampoline_kprobe(struct kprobe *p) 565{ 566 if (p->addr == (kprobe_opcode_t *)&kretprobe_trampoline) 567 return 1; 568 569 return 0; 570} 571