1/* 2 * Kernel Probes (KProbes) 3 * kernel/kprobes.c 4 * 5 * This program is free software; you can redistribute it and/or modify 6 * it under the terms of the GNU General Public License as published by 7 * the Free Software Foundation; either version 2 of the License, or 8 * (at your option) any later version. 9 * 10 * This program is distributed in the hope that it will be useful, 11 * but WITHOUT ANY WARRANTY; without even the implied warranty of 12 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 13 * GNU General Public License for more details. 14 * 15 * You should have received a copy of the GNU General Public License 16 * along with this program; if not, write to the Free Software 17 * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. 18 * 19 * Copyright (C) IBM Corporation, 2002, 2004 20 * 21 * 2002-Oct Created by Vamsi Krishna S <vamsi_krishna@in.ibm.com> Kernel 22 * Probes initial implementation (includes suggestions from 23 * Rusty Russell). 24 * 2004-Aug Updated by Prasanna S Panchamukhi <prasanna@in.ibm.com> with 25 * hlists and exceptions notifier as suggested by Andi Kleen. 26 * 2004-July Suparna Bhattacharya <suparna@in.ibm.com> added jumper probes 27 * interface to access function arguments. 28 * 2004-Sep Prasanna S Panchamukhi <prasanna@in.ibm.com> Changed Kprobes 29 * exceptions notifier to be first on the priority list. 30 * 2005-May Hien Nguyen <hien@us.ibm.com>, Jim Keniston 31 * <jkenisto@us.ibm.com> and Prasanna S Panchamukhi 32 * <prasanna@in.ibm.com> added function-return probes. 33 */ 34#include <linux/kprobes.h> 35#include <linux/hash.h> 36#include <linux/init.h> 37#include <linux/slab.h> 38#include <linux/stddef.h> 39#include <linux/module.h> 40#include <linux/moduleloader.h> 41#include <linux/kallsyms.h> 42#include <linux/freezer.h> 43#include <linux/seq_file.h> 44#include <linux/debugfs.h> 45#include <linux/kdebug.h> 46 47#include <asm-generic/sections.h> 48#include <asm/cacheflush.h> 49#include <asm/errno.h> 50#include <asm/uaccess.h> 51 52#define KPROBE_HASH_BITS 6 53#define KPROBE_TABLE_SIZE (1 << KPROBE_HASH_BITS) 54 55 56/* 57 * Some oddball architectures like 64bit powerpc have function descriptors 58 * so this must be overridable. 59 */ 60#ifndef kprobe_lookup_name 61#define kprobe_lookup_name(name, addr) \ 62 addr = ((kprobe_opcode_t *)(kallsyms_lookup_name(name))) 63#endif 64 65static struct hlist_head kprobe_table[KPROBE_TABLE_SIZE]; 66static struct hlist_head kretprobe_inst_table[KPROBE_TABLE_SIZE]; 67static atomic_t kprobe_count; 68 69/* NOTE: change this value only with kprobe_mutex held */ 70static bool kprobe_enabled; 71 72DEFINE_MUTEX(kprobe_mutex); /* Protects kprobe_table */ 73DEFINE_SPINLOCK(kretprobe_lock); /* Protects kretprobe_inst_table */ 74static DEFINE_PER_CPU(struct kprobe *, kprobe_instance) = NULL; 75 76static struct notifier_block kprobe_page_fault_nb = { 77 .notifier_call = kprobe_exceptions_notify, 78 .priority = 0x7fffffff /* we need to notified first */ 79}; 80 81#ifdef __ARCH_WANT_KPROBES_INSN_SLOT 82/* 83 * kprobe->ainsn.insn points to the copy of the instruction to be 84 * single-stepped. x86_64, POWER4 and above have no-exec support and 85 * stepping on the instruction on a vmalloced/kmalloced/data page 86 * is a recipe for disaster 87 */ 88#define INSNS_PER_PAGE (PAGE_SIZE/(MAX_INSN_SIZE * sizeof(kprobe_opcode_t))) 89 90struct kprobe_insn_page { 91 struct hlist_node hlist; 92 kprobe_opcode_t *insns; /* Page of instruction slots */ 93 char slot_used[INSNS_PER_PAGE]; 94 int nused; 95 int ngarbage; 96}; 97 98enum kprobe_slot_state { 99 SLOT_CLEAN = 0, 100 SLOT_DIRTY = 1, 101 SLOT_USED = 2, 102}; 103 104static struct hlist_head kprobe_insn_pages; 105static int kprobe_garbage_slots; 106static int collect_garbage_slots(void); 107 108static int __kprobes check_safety(void) 109{ 110 int ret = 0; 111#if defined(CONFIG_PREEMPT) && defined(CONFIG_PM) 112 ret = freeze_processes(); 113 if (ret == 0) { 114 struct task_struct *p, *q; 115 do_each_thread(p, q) { 116 if (p != current && p->state == TASK_RUNNING && 117 p->pid != 0) { 118 printk("Check failed: %s is running\n",p->comm); 119 ret = -1; 120 goto loop_end; 121 } 122 } while_each_thread(p, q); 123 } 124loop_end: 125 thaw_processes(); 126#else 127 synchronize_sched(); 128#endif 129 return ret; 130} 131 132/** 133 * get_insn_slot() - Find a slot on an executable page for an instruction. 134 * We allocate an executable page if there's no room on existing ones. 135 */ 136kprobe_opcode_t __kprobes *get_insn_slot(void) 137{ 138 struct kprobe_insn_page *kip; 139 struct hlist_node *pos; 140 141 retry: 142 hlist_for_each_entry(kip, pos, &kprobe_insn_pages, hlist) { 143 if (kip->nused < INSNS_PER_PAGE) { 144 int i; 145 for (i = 0; i < INSNS_PER_PAGE; i++) { 146 if (kip->slot_used[i] == SLOT_CLEAN) { 147 kip->slot_used[i] = SLOT_USED; 148 kip->nused++; 149 return kip->insns + (i * MAX_INSN_SIZE); 150 } 151 } 152 /* Surprise! No unused slots. Fix kip->nused. */ 153 kip->nused = INSNS_PER_PAGE; 154 } 155 } 156 157 /* If there are any garbage slots, collect it and try again. */ 158 if (kprobe_garbage_slots && collect_garbage_slots() == 0) { 159 goto retry; 160 } 161 /* All out of space. Need to allocate a new page. Use slot 0. */ 162 kip = kmalloc(sizeof(struct kprobe_insn_page), GFP_KERNEL); 163 if (!kip) 164 return NULL; 165 166 /* 167 * Use module_alloc so this page is within +/- 2GB of where the 168 * kernel image and loaded module images reside. This is required 169 * so x86_64 can correctly handle the %rip-relative fixups. 170 */ 171 kip->insns = module_alloc(PAGE_SIZE); 172 if (!kip->insns) { 173 kfree(kip); 174 return NULL; 175 } 176 INIT_HLIST_NODE(&kip->hlist); 177 hlist_add_head(&kip->hlist, &kprobe_insn_pages); 178 memset(kip->slot_used, SLOT_CLEAN, INSNS_PER_PAGE); 179 kip->slot_used[0] = SLOT_USED; 180 kip->nused = 1; 181 kip->ngarbage = 0; 182 return kip->insns; 183} 184 185/* Return 1 if all garbages are collected, otherwise 0. */ 186static int __kprobes collect_one_slot(struct kprobe_insn_page *kip, int idx) 187{ 188 kip->slot_used[idx] = SLOT_CLEAN; 189 kip->nused--; 190 if (kip->nused == 0) { 191 /* 192 * Page is no longer in use. Free it unless 193 * it's the last one. We keep the last one 194 * so as not to have to set it up again the 195 * next time somebody inserts a probe. 196 */ 197 hlist_del(&kip->hlist); 198 if (hlist_empty(&kprobe_insn_pages)) { 199 INIT_HLIST_NODE(&kip->hlist); 200 hlist_add_head(&kip->hlist, 201 &kprobe_insn_pages); 202 } else { 203 module_free(NULL, kip->insns); 204 kfree(kip); 205 } 206 return 1; 207 } 208 return 0; 209} 210 211static int __kprobes collect_garbage_slots(void) 212{ 213 struct kprobe_insn_page *kip; 214 struct hlist_node *pos, *next; 215 216 /* Ensure no-one is preepmted on the garbages */ 217 if (check_safety() != 0) 218 return -EAGAIN; 219 220 hlist_for_each_entry_safe(kip, pos, next, &kprobe_insn_pages, hlist) { 221 int i; 222 if (kip->ngarbage == 0) 223 continue; 224 kip->ngarbage = 0; /* we will collect all garbages */ 225 for (i = 0; i < INSNS_PER_PAGE; i++) { 226 if (kip->slot_used[i] == SLOT_DIRTY && 227 collect_one_slot(kip, i)) 228 break; 229 } 230 } 231 kprobe_garbage_slots = 0; 232 return 0; 233} 234 235void __kprobes free_insn_slot(kprobe_opcode_t * slot, int dirty) 236{ 237 struct kprobe_insn_page *kip; 238 struct hlist_node *pos; 239 240 hlist_for_each_entry(kip, pos, &kprobe_insn_pages, hlist) { 241 if (kip->insns <= slot && 242 slot < kip->insns + (INSNS_PER_PAGE * MAX_INSN_SIZE)) { 243 int i = (slot - kip->insns) / MAX_INSN_SIZE; 244 if (dirty) { 245 kip->slot_used[i] = SLOT_DIRTY; 246 kip->ngarbage++; 247 } else { 248 collect_one_slot(kip, i); 249 } 250 break; 251 } 252 } 253 254 if (dirty && ++kprobe_garbage_slots > INSNS_PER_PAGE) 255 collect_garbage_slots(); 256} 257#endif 258 259/* We have preemption disabled.. so it is safe to use __ versions */ 260static inline void set_kprobe_instance(struct kprobe *kp) 261{ 262 __get_cpu_var(kprobe_instance) = kp; 263} 264 265static inline void reset_kprobe_instance(void) 266{ 267 __get_cpu_var(kprobe_instance) = NULL; 268} 269 270struct kprobe __kprobes *get_kprobe(void *addr) 271{ 272 struct hlist_head *head; 273 struct hlist_node *node; 274 struct kprobe *p; 275 276 head = &kprobe_table[hash_ptr(addr, KPROBE_HASH_BITS)]; 277 hlist_for_each_entry_rcu(p, node, head, hlist) { 278 if (p->addr == addr) 279 return p; 280 } 281 return NULL; 282} 283 284/* 285 * Aggregate handlers for multiple kprobes support - these handlers 286 * take care of invoking the individual kprobe handlers on p->list 287 */ 288static int __kprobes aggr_pre_handler(struct kprobe *p, struct pt_regs *regs) 289{ 290 struct kprobe *kp; 291 292 list_for_each_entry_rcu(kp, &p->list, list) { 293 if (kp->pre_handler) { 294 set_kprobe_instance(kp); 295 if (kp->pre_handler(kp, regs)) 296 return 1; 297 } 298 reset_kprobe_instance(); 299 } 300 return 0; 301} 302 303static void __kprobes aggr_post_handler(struct kprobe *p, struct pt_regs *regs, 304 unsigned long flags) 305{ 306 struct kprobe *kp; 307 308 list_for_each_entry_rcu(kp, &p->list, list) { 309 if (kp->post_handler) { 310 set_kprobe_instance(kp); 311 kp->post_handler(kp, regs, flags); 312 reset_kprobe_instance(); 313 } 314 } 315} 316 317static int __kprobes aggr_fault_handler(struct kprobe *p, struct pt_regs *regs, 318 int trapnr) 319{ 320 struct kprobe *cur = __get_cpu_var(kprobe_instance); 321 322 /* 323 * if we faulted "during" the execution of a user specified 324 * probe handler, invoke just that probe's fault handler 325 */ 326 if (cur && cur->fault_handler) { 327 if (cur->fault_handler(cur, regs, trapnr)) 328 return 1; 329 } 330 return 0; 331} 332 333static int __kprobes aggr_break_handler(struct kprobe *p, struct pt_regs *regs) 334{ 335 struct kprobe *cur = __get_cpu_var(kprobe_instance); 336 int ret = 0; 337 338 if (cur && cur->break_handler) { 339 if (cur->break_handler(cur, regs)) 340 ret = 1; 341 } 342 reset_kprobe_instance(); 343 return ret; 344} 345 346/* Walks the list and increments nmissed count for multiprobe case */ 347void __kprobes kprobes_inc_nmissed_count(struct kprobe *p) 348{ 349 struct kprobe *kp; 350 if (p->pre_handler != aggr_pre_handler) { 351 p->nmissed++; 352 } else { 353 list_for_each_entry_rcu(kp, &p->list, list) 354 kp->nmissed++; 355 } 356 return; 357} 358 359/* Called with kretprobe_lock held */ 360void __kprobes recycle_rp_inst(struct kretprobe_instance *ri, 361 struct hlist_head *head) 362{ 363 /* remove rp inst off the rprobe_inst_table */ 364 hlist_del(&ri->hlist); 365 if (ri->rp) { 366 /* remove rp inst off the used list */ 367 hlist_del(&ri->uflist); 368 /* put rp inst back onto the free list */ 369 INIT_HLIST_NODE(&ri->uflist); 370 hlist_add_head(&ri->uflist, &ri->rp->free_instances); 371 } else 372 /* Unregistering */ 373 hlist_add_head(&ri->hlist, head); 374} 375 376struct hlist_head __kprobes *kretprobe_inst_table_head(struct task_struct *tsk) 377{ 378 return &kretprobe_inst_table[hash_ptr(tsk, KPROBE_HASH_BITS)]; 379} 380 381/* 382 * This function is called from finish_task_switch when task tk becomes dead, 383 * so that we can recycle any function-return probe instances associated 384 * with this task. These left over instances represent probed functions 385 * that have been called but will never return. 386 */ 387void __kprobes kprobe_flush_task(struct task_struct *tk) 388{ 389 struct kretprobe_instance *ri; 390 struct hlist_head *head, empty_rp; 391 struct hlist_node *node, *tmp; 392 unsigned long flags = 0; 393 394 INIT_HLIST_HEAD(&empty_rp); 395 spin_lock_irqsave(&kretprobe_lock, flags); 396 head = kretprobe_inst_table_head(tk); 397 hlist_for_each_entry_safe(ri, node, tmp, head, hlist) { 398 if (ri->task == tk) 399 recycle_rp_inst(ri, &empty_rp); 400 } 401 spin_unlock_irqrestore(&kretprobe_lock, flags); 402 403 hlist_for_each_entry_safe(ri, node, tmp, &empty_rp, hlist) { 404 hlist_del(&ri->hlist); 405 kfree(ri); 406 } 407} 408 409static inline void free_rp_inst(struct kretprobe *rp) 410{ 411 struct kretprobe_instance *ri; 412 struct hlist_node *pos, *next; 413 414 hlist_for_each_entry_safe(ri, pos, next, &rp->free_instances, uflist) { 415 hlist_del(&ri->uflist); 416 kfree(ri); 417 } 418} 419 420/* 421 * Keep all fields in the kprobe consistent 422 */ 423static inline void copy_kprobe(struct kprobe *old_p, struct kprobe *p) 424{ 425 memcpy(&p->opcode, &old_p->opcode, sizeof(kprobe_opcode_t)); 426 memcpy(&p->ainsn, &old_p->ainsn, sizeof(struct arch_specific_insn)); 427} 428 429/* 430* Add the new probe to old_p->list. Fail if this is the 431* second jprobe at the address - two jprobes can't coexist 432*/ 433static int __kprobes add_new_kprobe(struct kprobe *old_p, struct kprobe *p) 434{ 435 if (p->break_handler) { 436 if (old_p->break_handler) 437 return -EEXIST; 438 list_add_tail_rcu(&p->list, &old_p->list); 439 old_p->break_handler = aggr_break_handler; 440 } else 441 list_add_rcu(&p->list, &old_p->list); 442 if (p->post_handler && !old_p->post_handler) 443 old_p->post_handler = aggr_post_handler; 444 return 0; 445} 446 447/* 448 * Fill in the required fields of the "manager kprobe". Replace the 449 * earlier kprobe in the hlist with the manager kprobe 450 */ 451static inline void add_aggr_kprobe(struct kprobe *ap, struct kprobe *p) 452{ 453 copy_kprobe(p, ap); 454 flush_insn_slot(ap); 455 ap->addr = p->addr; 456 ap->pre_handler = aggr_pre_handler; 457 ap->fault_handler = aggr_fault_handler; 458 if (p->post_handler) 459 ap->post_handler = aggr_post_handler; 460 if (p->break_handler) 461 ap->break_handler = aggr_break_handler; 462 463 INIT_LIST_HEAD(&ap->list); 464 list_add_rcu(&p->list, &ap->list); 465 466 hlist_replace_rcu(&p->hlist, &ap->hlist); 467} 468 469/* 470 * This is the second or subsequent kprobe at the address - handle 471 * the intricacies 472 */ 473static int __kprobes register_aggr_kprobe(struct kprobe *old_p, 474 struct kprobe *p) 475{ 476 int ret = 0; 477 struct kprobe *ap; 478 479 if (old_p->pre_handler == aggr_pre_handler) { 480 copy_kprobe(old_p, p); 481 ret = add_new_kprobe(old_p, p); 482 } else { 483 ap = kzalloc(sizeof(struct kprobe), GFP_KERNEL); 484 if (!ap) 485 return -ENOMEM; 486 add_aggr_kprobe(ap, old_p); 487 copy_kprobe(ap, p); 488 ret = add_new_kprobe(ap, p); 489 } 490 return ret; 491} 492 493static int __kprobes in_kprobes_functions(unsigned long addr) 494{ 495 if (addr >= (unsigned long)__kprobes_text_start && 496 addr < (unsigned long)__kprobes_text_end) 497 return -EINVAL; 498 return 0; 499} 500 501static int __kprobes __register_kprobe(struct kprobe *p, 502 unsigned long called_from) 503{ 504 int ret = 0; 505 struct kprobe *old_p; 506 struct module *probed_mod; 507 508 /* 509 * If we have a symbol_name argument look it up, 510 * and add it to the address. That way the addr 511 * field can either be global or relative to a symbol. 512 */ 513 if (p->symbol_name) { 514 if (p->addr) 515 return -EINVAL; 516 kprobe_lookup_name(p->symbol_name, p->addr); 517 } 518 519 if (!p->addr) 520 return -EINVAL; 521 p->addr = (kprobe_opcode_t *)(((char *)p->addr)+ p->offset); 522 523 if (!kernel_text_address((unsigned long) p->addr) || 524 in_kprobes_functions((unsigned long) p->addr)) 525 return -EINVAL; 526 527 p->mod_refcounted = 0; 528 529 /* 530 * Check if are we probing a module. 531 */ 532 probed_mod = module_text_address((unsigned long) p->addr); 533 if (probed_mod) { 534 struct module *calling_mod = module_text_address(called_from); 535 /* 536 * We must allow modules to probe themself and in this case 537 * avoid incrementing the module refcount, so as to allow 538 * unloading of self probing modules. 539 */ 540 if (calling_mod && calling_mod != probed_mod) { 541 if (unlikely(!try_module_get(probed_mod))) 542 return -EINVAL; 543 p->mod_refcounted = 1; 544 } else 545 probed_mod = NULL; 546 } 547 548 p->nmissed = 0; 549 mutex_lock(&kprobe_mutex); 550 old_p = get_kprobe(p->addr); 551 if (old_p) { 552 ret = register_aggr_kprobe(old_p, p); 553 if (!ret) 554 atomic_inc(&kprobe_count); 555 goto out; 556 } 557 558 ret = arch_prepare_kprobe(p); 559 if (ret) 560 goto out; 561 562 INIT_HLIST_NODE(&p->hlist); 563 hlist_add_head_rcu(&p->hlist, 564 &kprobe_table[hash_ptr(p->addr, KPROBE_HASH_BITS)]); 565 566 if (kprobe_enabled) { 567 if (atomic_add_return(1, &kprobe_count) == \ 568 (ARCH_INACTIVE_KPROBE_COUNT + 1)) 569 register_page_fault_notifier(&kprobe_page_fault_nb); 570 571 arch_arm_kprobe(p); 572 } 573out: 574 mutex_unlock(&kprobe_mutex); 575 576 if (ret && probed_mod) 577 module_put(probed_mod); 578 return ret; 579} 580 581int __kprobes register_kprobe(struct kprobe *p) 582{ 583 return __register_kprobe(p, (unsigned long)__builtin_return_address(0)); 584} 585 586void __kprobes unregister_kprobe(struct kprobe *p) 587{ 588 struct module *mod; 589 struct kprobe *old_p, *list_p; 590 int cleanup_p; 591 592 mutex_lock(&kprobe_mutex); 593 old_p = get_kprobe(p->addr); 594 if (unlikely(!old_p)) { 595 mutex_unlock(&kprobe_mutex); 596 return; 597 } 598 if (p != old_p) { 599 list_for_each_entry_rcu(list_p, &old_p->list, list) 600 if (list_p == p) 601 /* kprobe p is a valid probe */ 602 goto valid_p; 603 mutex_unlock(&kprobe_mutex); 604 return; 605 } 606valid_p: 607 if (old_p == p || 608 (old_p->pre_handler == aggr_pre_handler && 609 p->list.next == &old_p->list && p->list.prev == &old_p->list)) { 610 /* 611 * Only probe on the hash list. Disarm only if kprobes are 612 * enabled - otherwise, the breakpoint would already have 613 * been removed. We save on flushing icache. 614 */ 615 if (kprobe_enabled) 616 arch_disarm_kprobe(p); 617 hlist_del_rcu(&old_p->hlist); 618 cleanup_p = 1; 619 } else { 620 list_del_rcu(&p->list); 621 cleanup_p = 0; 622 } 623 624 mutex_unlock(&kprobe_mutex); 625 626 synchronize_sched(); 627 if (p->mod_refcounted) { 628 mod = module_text_address((unsigned long)p->addr); 629 if (mod) 630 module_put(mod); 631 } 632 633 if (cleanup_p) { 634 if (p != old_p) { 635 list_del_rcu(&p->list); 636 kfree(old_p); 637 } 638 arch_remove_kprobe(p); 639 } else { 640 mutex_lock(&kprobe_mutex); 641 if (p->break_handler) 642 old_p->break_handler = NULL; 643 if (p->post_handler){ 644 list_for_each_entry_rcu(list_p, &old_p->list, list){ 645 if (list_p->post_handler){ 646 cleanup_p = 2; 647 break; 648 } 649 } 650 if (cleanup_p == 0) 651 old_p->post_handler = NULL; 652 } 653 mutex_unlock(&kprobe_mutex); 654 } 655 656 /* Call unregister_page_fault_notifier() 657 * if no probes are active 658 */ 659 mutex_lock(&kprobe_mutex); 660 if (atomic_add_return(-1, &kprobe_count) == \ 661 ARCH_INACTIVE_KPROBE_COUNT) 662 unregister_page_fault_notifier(&kprobe_page_fault_nb); 663 mutex_unlock(&kprobe_mutex); 664 return; 665} 666 667static struct notifier_block kprobe_exceptions_nb = { 668 .notifier_call = kprobe_exceptions_notify, 669 .priority = 0x7fffffff /* we need to be notified first */ 670}; 671 672 673int __kprobes register_jprobe(struct jprobe *jp) 674{ 675 /* Todo: Verify probepoint is a function entry point */ 676 jp->kp.pre_handler = setjmp_pre_handler; 677 jp->kp.break_handler = longjmp_break_handler; 678 679 return __register_kprobe(&jp->kp, 680 (unsigned long)__builtin_return_address(0)); 681} 682 683void __kprobes unregister_jprobe(struct jprobe *jp) 684{ 685 unregister_kprobe(&jp->kp); 686} 687 688#ifdef ARCH_SUPPORTS_KRETPROBES 689 690/* 691 * This kprobe pre_handler is registered with every kretprobe. When probe 692 * hits it will set up the return probe. 693 */ 694static int __kprobes pre_handler_kretprobe(struct kprobe *p, 695 struct pt_regs *regs) 696{ 697 struct kretprobe *rp = container_of(p, struct kretprobe, kp); 698 unsigned long flags = 0; 699 700 /*TODO: consider to only swap the RA after the last pre_handler fired */ 701 spin_lock_irqsave(&kretprobe_lock, flags); 702 if (!hlist_empty(&rp->free_instances)) { 703 struct kretprobe_instance *ri; 704 705 ri = hlist_entry(rp->free_instances.first, 706 struct kretprobe_instance, uflist); 707 ri->rp = rp; 708 ri->task = current; 709 arch_prepare_kretprobe(ri, regs); 710 711 hlist_del(&ri->uflist); 712 hlist_add_head(&ri->uflist, &ri->rp->used_instances); 713 hlist_add_head(&ri->hlist, kretprobe_inst_table_head(ri->task)); 714 } else 715 rp->nmissed++; 716 spin_unlock_irqrestore(&kretprobe_lock, flags); 717 return 0; 718} 719 720int __kprobes register_kretprobe(struct kretprobe *rp) 721{ 722 int ret = 0; 723 struct kretprobe_instance *inst; 724 int i; 725 726 rp->kp.pre_handler = pre_handler_kretprobe; 727 rp->kp.post_handler = NULL; 728 rp->kp.fault_handler = NULL; 729 rp->kp.break_handler = NULL; 730 731 /* Pre-allocate memory for max kretprobe instances */ 732 if (rp->maxactive <= 0) { 733#ifdef CONFIG_PREEMPT 734 rp->maxactive = max(10, 2 * NR_CPUS); 735#else 736 rp->maxactive = NR_CPUS; 737#endif 738 } 739 INIT_HLIST_HEAD(&rp->used_instances); 740 INIT_HLIST_HEAD(&rp->free_instances); 741 for (i = 0; i < rp->maxactive; i++) { 742 inst = kmalloc(sizeof(struct kretprobe_instance), GFP_KERNEL); 743 if (inst == NULL) { 744 free_rp_inst(rp); 745 return -ENOMEM; 746 } 747 INIT_HLIST_NODE(&inst->uflist); 748 hlist_add_head(&inst->uflist, &rp->free_instances); 749 } 750 751 rp->nmissed = 0; 752 /* Establish function entry probe point */ 753 if ((ret = __register_kprobe(&rp->kp, 754 (unsigned long)__builtin_return_address(0))) != 0) 755 free_rp_inst(rp); 756 return ret; 757} 758 759#else /* ARCH_SUPPORTS_KRETPROBES */ 760 761int __kprobes register_kretprobe(struct kretprobe *rp) 762{ 763 return -ENOSYS; 764} 765 766static int __kprobes pre_handler_kretprobe(struct kprobe *p, 767 struct pt_regs *regs) 768{ 769 return 0; 770} 771 772#endif /* ARCH_SUPPORTS_KRETPROBES */ 773 774void __kprobes unregister_kretprobe(struct kretprobe *rp) 775{ 776 unsigned long flags; 777 struct kretprobe_instance *ri; 778 struct hlist_node *pos, *next; 779 780 unregister_kprobe(&rp->kp); 781 782 /* No race here */ 783 spin_lock_irqsave(&kretprobe_lock, flags); 784 hlist_for_each_entry_safe(ri, pos, next, &rp->used_instances, uflist) { 785 ri->rp = NULL; 786 hlist_del(&ri->uflist); 787 } 788 spin_unlock_irqrestore(&kretprobe_lock, flags); 789 free_rp_inst(rp); 790} 791 792static int __init init_kprobes(void) 793{ 794 int i, err = 0; 795 796 /* initialize all list heads */ 797 for (i = 0; i < KPROBE_TABLE_SIZE; i++) { 798 INIT_HLIST_HEAD(&kprobe_table[i]); 799 INIT_HLIST_HEAD(&kretprobe_inst_table[i]); 800 } 801 atomic_set(&kprobe_count, 0); 802 803 /* By default, kprobes are enabled */ 804 kprobe_enabled = true; 805 806 err = arch_init_kprobes(); 807 if (!err) 808 err = register_die_notifier(&kprobe_exceptions_nb); 809 810 return err; 811} 812 813#ifdef CONFIG_DEBUG_FS 814static void __kprobes report_probe(struct seq_file *pi, struct kprobe *p, 815 const char *sym, int offset,char *modname) 816{ 817 char *kprobe_type; 818 819 if (p->pre_handler == pre_handler_kretprobe) 820 kprobe_type = "r"; 821 else if (p->pre_handler == setjmp_pre_handler) 822 kprobe_type = "j"; 823 else 824 kprobe_type = "k"; 825 if (sym) 826 seq_printf(pi, "%p %s %s+0x%x %s\n", p->addr, kprobe_type, 827 sym, offset, (modname ? modname : " ")); 828 else 829 seq_printf(pi, "%p %s %p\n", p->addr, kprobe_type, p->addr); 830} 831 832static void __kprobes *kprobe_seq_start(struct seq_file *f, loff_t *pos) 833{ 834 return (*pos < KPROBE_TABLE_SIZE) ? pos : NULL; 835} 836 837static void __kprobes *kprobe_seq_next(struct seq_file *f, void *v, loff_t *pos) 838{ 839 (*pos)++; 840 if (*pos >= KPROBE_TABLE_SIZE) 841 return NULL; 842 return pos; 843} 844 845static void __kprobes kprobe_seq_stop(struct seq_file *f, void *v) 846{ 847 /* Nothing to do */ 848} 849 850static int __kprobes show_kprobe_addr(struct seq_file *pi, void *v) 851{ 852 struct hlist_head *head; 853 struct hlist_node *node; 854 struct kprobe *p, *kp; 855 const char *sym = NULL; 856 unsigned int i = *(loff_t *) v; 857 unsigned long offset = 0; 858 char *modname, namebuf[128]; 859 860 head = &kprobe_table[i]; 861 preempt_disable(); 862 hlist_for_each_entry_rcu(p, node, head, hlist) { 863 sym = kallsyms_lookup((unsigned long)p->addr, NULL, 864 &offset, &modname, namebuf); 865 if (p->pre_handler == aggr_pre_handler) { 866 list_for_each_entry_rcu(kp, &p->list, list) 867 report_probe(pi, kp, sym, offset, modname); 868 } else 869 report_probe(pi, p, sym, offset, modname); 870 } 871 preempt_enable(); 872 return 0; 873} 874 875static struct seq_operations kprobes_seq_ops = { 876 .start = kprobe_seq_start, 877 .next = kprobe_seq_next, 878 .stop = kprobe_seq_stop, 879 .show = show_kprobe_addr 880}; 881 882static int __kprobes kprobes_open(struct inode *inode, struct file *filp) 883{ 884 return seq_open(filp, &kprobes_seq_ops); 885} 886 887static struct file_operations debugfs_kprobes_operations = { 888 .open = kprobes_open, 889 .read = seq_read, 890 .llseek = seq_lseek, 891 .release = seq_release, 892}; 893 894static void __kprobes enable_all_kprobes(void) 895{ 896 struct hlist_head *head; 897 struct hlist_node *node; 898 struct kprobe *p; 899 unsigned int i; 900 901 mutex_lock(&kprobe_mutex); 902 903 /* If kprobes are already enabled, just return */ 904 if (kprobe_enabled) 905 goto already_enabled; 906 907 /* 908 * Re-register the page fault notifier only if there are any 909 * active probes at the time of enabling kprobes globally 910 */ 911 if (atomic_read(&kprobe_count) > ARCH_INACTIVE_KPROBE_COUNT) 912 register_page_fault_notifier(&kprobe_page_fault_nb); 913 914 for (i = 0; i < KPROBE_TABLE_SIZE; i++) { 915 head = &kprobe_table[i]; 916 hlist_for_each_entry_rcu(p, node, head, hlist) 917 arch_arm_kprobe(p); 918 } 919 920 kprobe_enabled = true; 921 printk(KERN_INFO "Kprobes globally enabled\n"); 922 923already_enabled: 924 mutex_unlock(&kprobe_mutex); 925 return; 926} 927 928static void __kprobes disable_all_kprobes(void) 929{ 930 struct hlist_head *head; 931 struct hlist_node *node; 932 struct kprobe *p; 933 unsigned int i; 934 935 mutex_lock(&kprobe_mutex); 936 937 /* If kprobes are already disabled, just return */ 938 if (!kprobe_enabled) 939 goto already_disabled; 940 941 kprobe_enabled = false; 942 printk(KERN_INFO "Kprobes globally disabled\n"); 943 for (i = 0; i < KPROBE_TABLE_SIZE; i++) { 944 head = &kprobe_table[i]; 945 hlist_for_each_entry_rcu(p, node, head, hlist) { 946 if (!arch_trampoline_kprobe(p)) 947 arch_disarm_kprobe(p); 948 } 949 } 950 951 mutex_unlock(&kprobe_mutex); 952 /* Allow all currently running kprobes to complete */ 953 synchronize_sched(); 954 955 mutex_lock(&kprobe_mutex); 956 /* Unconditionally unregister the page_fault notifier */ 957 unregister_page_fault_notifier(&kprobe_page_fault_nb); 958 959already_disabled: 960 mutex_unlock(&kprobe_mutex); 961 return; 962} 963 964static ssize_t read_enabled_file_bool(struct file *file, 965 char __user *user_buf, size_t count, loff_t *ppos) 966{ 967 char buf[3]; 968 969 if (kprobe_enabled) 970 buf[0] = '1'; 971 else 972 buf[0] = '0'; 973 buf[1] = '\n'; 974 buf[2] = 0x00; 975 return simple_read_from_buffer(user_buf, count, ppos, buf, 2); 976} 977 978static ssize_t write_enabled_file_bool(struct file *file, 979 const char __user *user_buf, size_t count, loff_t *ppos) 980{ 981 char buf[32]; 982 int buf_size; 983 984 buf_size = min(count, (sizeof(buf)-1)); 985 if (copy_from_user(buf, user_buf, buf_size)) 986 return -EFAULT; 987 988 switch (buf[0]) { 989 case 'y': 990 case 'Y': 991 case '1': 992 enable_all_kprobes(); 993 break; 994 case 'n': 995 case 'N': 996 case '0': 997 disable_all_kprobes(); 998 break; 999 } 1000 1001 return count; 1002} 1003 1004static struct file_operations fops_kp = { 1005 .read = read_enabled_file_bool, 1006 .write = write_enabled_file_bool, 1007}; 1008 1009static int __kprobes debugfs_kprobe_init(void) 1010{ 1011 struct dentry *dir, *file; 1012 unsigned int value = 1; 1013 1014 dir = debugfs_create_dir("kprobes", NULL); 1015 if (!dir) 1016 return -ENOMEM; 1017 1018 file = debugfs_create_file("list", 0444, dir, NULL, 1019 &debugfs_kprobes_operations); 1020 if (!file) { 1021 debugfs_remove(dir); 1022 return -ENOMEM; 1023 } 1024 1025 file = debugfs_create_file("enabled", 0600, dir, 1026 &value, &fops_kp); 1027 if (!file) { 1028 debugfs_remove(dir); 1029 return -ENOMEM; 1030 } 1031 1032 return 0; 1033} 1034 1035late_initcall(debugfs_kprobe_init); 1036#endif /* CONFIG_DEBUG_FS */ 1037 1038module_init(init_kprobes); 1039 1040EXPORT_SYMBOL_GPL(register_kprobe); 1041EXPORT_SYMBOL_GPL(unregister_kprobe); 1042EXPORT_SYMBOL_GPL(register_jprobe); 1043EXPORT_SYMBOL_GPL(unregister_jprobe); 1044EXPORT_SYMBOL_GPL(jprobe_return); 1045EXPORT_SYMBOL_GPL(register_kretprobe); 1046EXPORT_SYMBOL_GPL(unregister_kretprobe); 1047