1/* 2 * Infrastructure for profiling code inserted by 'gcc -pg'. 3 * 4 * Copyright (C) 2007-2008 Steven Rostedt <srostedt@redhat.com> 5 * Copyright (C) 2004-2008 Ingo Molnar <mingo@redhat.com> 6 * 7 * Originally ported from the -rt patch by: 8 * Copyright (C) 2007 Arnaldo Carvalho de Melo <acme@redhat.com> 9 * 10 * Based on code in the latency_tracer, that is: 11 * 12 * Copyright (C) 2004-2006 Ingo Molnar 13 * Copyright (C) 2004 William Lee Irwin III 14 */ 15 16#include <linux/stop_machine.h> 17#include <linux/clocksource.h> 18#include <linux/kallsyms.h> 19#include <linux/seq_file.h> 20#include <linux/suspend.h> 21#include <linux/debugfs.h> 22#include <linux/hardirq.h> 23#include <linux/kthread.h> 24#include <linux/uaccess.h> 25#include <linux/ftrace.h> 26#include <linux/sysctl.h> 27#include <linux/slab.h> 28#include <linux/ctype.h> 29#include <linux/list.h> 30#include <linux/hash.h> 31#include <linux/rcupdate.h> 32 33#include <trace/events/sched.h> 34 35#include <asm/ftrace.h> 36#include <asm/setup.h> 37 38#include "trace_output.h" 39#include "trace_stat.h" 40 41#define FTRACE_WARN_ON(cond) \ 42 do { \ 43 if (WARN_ON(cond)) \ 44 ftrace_kill(); \ 45 } while (0) 46 47#define FTRACE_WARN_ON_ONCE(cond) \ 48 do { \ 49 if (WARN_ON_ONCE(cond)) \ 50 ftrace_kill(); \ 51 } while (0) 52 53/* hash bits for specific function selection */ 54#define FTRACE_HASH_BITS 7 55#define FTRACE_FUNC_HASHSIZE (1 << FTRACE_HASH_BITS) 56 57/* ftrace_enabled is a method to turn ftrace on or off */ 58int ftrace_enabled __read_mostly; 59static int last_ftrace_enabled; 60 61/* Quick disabling of function tracer. */ 62int function_trace_stop; 63 64/* List for set_ftrace_pid's pids. */ 65LIST_HEAD(ftrace_pids); 66struct ftrace_pid { 67 struct list_head list; 68 struct pid *pid; 69}; 70 71/* 72 * ftrace_disabled is set when an anomaly is discovered. 73 * ftrace_disabled is much stronger than ftrace_enabled. 74 */ 75static int ftrace_disabled __read_mostly; 76 77static DEFINE_MUTEX(ftrace_lock); 78 79static struct ftrace_ops ftrace_list_end __read_mostly = 80{ 81 .func = ftrace_stub, 82}; 83 84static struct ftrace_ops *ftrace_list __read_mostly = &ftrace_list_end; 85ftrace_func_t ftrace_trace_function __read_mostly = ftrace_stub; 86ftrace_func_t __ftrace_trace_function __read_mostly = ftrace_stub; 87ftrace_func_t ftrace_pid_function __read_mostly = ftrace_stub; 88 89/* 90 * Traverse the ftrace_list, invoking all entries. The reason that we 91 * can use rcu_dereference_raw() is that elements removed from this list 92 * are simply leaked, so there is no need to interact with a grace-period 93 * mechanism. The rcu_dereference_raw() calls are needed to handle 94 * concurrent insertions into the ftrace_list. 95 * 96 * Silly Alpha and silly pointer-speculation compiler optimizations! 97 */ 98static void ftrace_list_func(unsigned long ip, unsigned long parent_ip) 99{ 100 struct ftrace_ops *op = rcu_dereference_raw(ftrace_list); /*see above*/ 101 102 while (op != &ftrace_list_end) { 103 op->func(ip, parent_ip); 104 op = rcu_dereference_raw(op->next); /*see above*/ 105 }; 106} 107 108static void ftrace_pid_func(unsigned long ip, unsigned long parent_ip) 109{ 110 if (!test_tsk_trace_trace(current)) 111 return; 112 113 ftrace_pid_function(ip, parent_ip); 114} 115 116static void set_ftrace_pid_function(ftrace_func_t func) 117{ 118 /* do not set ftrace_pid_function to itself! */ 119 if (func != ftrace_pid_func) 120 ftrace_pid_function = func; 121} 122 123/** 124 * clear_ftrace_function - reset the ftrace function 125 * 126 * This NULLs the ftrace function and in essence stops 127 * tracing. There may be lag 128 */ 129void clear_ftrace_function(void) 130{ 131 ftrace_trace_function = ftrace_stub; 132 __ftrace_trace_function = ftrace_stub; 133 ftrace_pid_function = ftrace_stub; 134} 135 136#ifndef CONFIG_HAVE_FUNCTION_TRACE_MCOUNT_TEST 137/* 138 * For those archs that do not test ftrace_trace_stop in their 139 * mcount call site, we need to do it from C. 140 */ 141static void ftrace_test_stop_func(unsigned long ip, unsigned long parent_ip) 142{ 143 if (function_trace_stop) 144 return; 145 146 __ftrace_trace_function(ip, parent_ip); 147} 148#endif 149 150static int __register_ftrace_function(struct ftrace_ops *ops) 151{ 152 ops->next = ftrace_list; 153 /* 154 * We are entering ops into the ftrace_list but another 155 * CPU might be walking that list. We need to make sure 156 * the ops->next pointer is valid before another CPU sees 157 * the ops pointer included into the ftrace_list. 158 */ 159 rcu_assign_pointer(ftrace_list, ops); 160 161 if (ftrace_enabled) { 162 ftrace_func_t func; 163 164 if (ops->next == &ftrace_list_end) 165 func = ops->func; 166 else 167 func = ftrace_list_func; 168 169 if (!list_empty(&ftrace_pids)) { 170 set_ftrace_pid_function(func); 171 func = ftrace_pid_func; 172 } 173 174 /* 175 * For one func, simply call it directly. 176 * For more than one func, call the chain. 177 */ 178#ifdef CONFIG_HAVE_FUNCTION_TRACE_MCOUNT_TEST 179 ftrace_trace_function = func; 180#else 181 __ftrace_trace_function = func; 182 ftrace_trace_function = ftrace_test_stop_func; 183#endif 184 } 185 186 return 0; 187} 188 189static int __unregister_ftrace_function(struct ftrace_ops *ops) 190{ 191 struct ftrace_ops **p; 192 193 /* 194 * If we are removing the last function, then simply point 195 * to the ftrace_stub. 196 */ 197 if (ftrace_list == ops && ops->next == &ftrace_list_end) { 198 ftrace_trace_function = ftrace_stub; 199 ftrace_list = &ftrace_list_end; 200 return 0; 201 } 202 203 for (p = &ftrace_list; *p != &ftrace_list_end; p = &(*p)->next) 204 if (*p == ops) 205 break; 206 207 if (*p != ops) 208 return -1; 209 210 *p = (*p)->next; 211 212 if (ftrace_enabled) { 213 /* If we only have one func left, then call that directly */ 214 if (ftrace_list->next == &ftrace_list_end) { 215 ftrace_func_t func = ftrace_list->func; 216 217 if (!list_empty(&ftrace_pids)) { 218 set_ftrace_pid_function(func); 219 func = ftrace_pid_func; 220 } 221#ifdef CONFIG_HAVE_FUNCTION_TRACE_MCOUNT_TEST 222 ftrace_trace_function = func; 223#else 224 __ftrace_trace_function = func; 225#endif 226 } 227 } 228 229 return 0; 230} 231 232static void ftrace_update_pid_func(void) 233{ 234 ftrace_func_t func; 235 236 if (ftrace_trace_function == ftrace_stub) 237 return; 238 239#ifdef CONFIG_HAVE_FUNCTION_TRACE_MCOUNT_TEST 240 func = ftrace_trace_function; 241#else 242 func = __ftrace_trace_function; 243#endif 244 245 if (!list_empty(&ftrace_pids)) { 246 set_ftrace_pid_function(func); 247 func = ftrace_pid_func; 248 } else { 249 if (func == ftrace_pid_func) 250 func = ftrace_pid_function; 251 } 252 253#ifdef CONFIG_HAVE_FUNCTION_TRACE_MCOUNT_TEST 254 ftrace_trace_function = func; 255#else 256 __ftrace_trace_function = func; 257#endif 258} 259 260#ifdef CONFIG_FUNCTION_PROFILER 261struct ftrace_profile { 262 struct hlist_node node; 263 unsigned long ip; 264 unsigned long counter; 265#ifdef CONFIG_FUNCTION_GRAPH_TRACER 266 unsigned long long time; 267 unsigned long long time_squared; 268#endif 269}; 270 271struct ftrace_profile_page { 272 struct ftrace_profile_page *next; 273 unsigned long index; 274 struct ftrace_profile records[]; 275}; 276 277struct ftrace_profile_stat { 278 atomic_t disabled; 279 struct hlist_head *hash; 280 struct ftrace_profile_page *pages; 281 struct ftrace_profile_page *start; 282 struct tracer_stat stat; 283}; 284 285#define PROFILE_RECORDS_SIZE \ 286 (PAGE_SIZE - offsetof(struct ftrace_profile_page, records)) 287 288#define PROFILES_PER_PAGE \ 289 (PROFILE_RECORDS_SIZE / sizeof(struct ftrace_profile)) 290 291static int ftrace_profile_bits __read_mostly; 292static int ftrace_profile_enabled __read_mostly; 293 294/* ftrace_profile_lock - synchronize the enable and disable of the profiler */ 295static DEFINE_MUTEX(ftrace_profile_lock); 296 297static DEFINE_PER_CPU(struct ftrace_profile_stat, ftrace_profile_stats); 298 299#define FTRACE_PROFILE_HASH_SIZE 1024 /* must be power of 2 */ 300 301static void * 302function_stat_next(void *v, int idx) 303{ 304 struct ftrace_profile *rec = v; 305 struct ftrace_profile_page *pg; 306 307 pg = (struct ftrace_profile_page *)((unsigned long)rec & PAGE_MASK); 308 309 again: 310 if (idx != 0) 311 rec++; 312 313 if ((void *)rec >= (void *)&pg->records[pg->index]) { 314 pg = pg->next; 315 if (!pg) 316 return NULL; 317 rec = &pg->records[0]; 318 if (!rec->counter) 319 goto again; 320 } 321 322 return rec; 323} 324 325static void *function_stat_start(struct tracer_stat *trace) 326{ 327 struct ftrace_profile_stat *stat = 328 container_of(trace, struct ftrace_profile_stat, stat); 329 330 if (!stat || !stat->start) 331 return NULL; 332 333 return function_stat_next(&stat->start->records[0], 0); 334} 335 336#ifdef CONFIG_FUNCTION_GRAPH_TRACER 337/* function graph compares on total time */ 338static int function_stat_cmp(void *p1, void *p2) 339{ 340 struct ftrace_profile *a = p1; 341 struct ftrace_profile *b = p2; 342 343 if (a->time < b->time) 344 return -1; 345 if (a->time > b->time) 346 return 1; 347 else 348 return 0; 349} 350#else 351/* not function graph compares against hits */ 352static int function_stat_cmp(void *p1, void *p2) 353{ 354 struct ftrace_profile *a = p1; 355 struct ftrace_profile *b = p2; 356 357 if (a->counter < b->counter) 358 return -1; 359 if (a->counter > b->counter) 360 return 1; 361 else 362 return 0; 363} 364#endif 365 366static int function_stat_headers(struct seq_file *m) 367{ 368#ifdef CONFIG_FUNCTION_GRAPH_TRACER 369 seq_printf(m, " Function " 370 "Hit Time Avg s^2\n" 371 " -------- " 372 "--- ---- --- ---\n"); 373#else 374 seq_printf(m, " Function Hit\n" 375 " -------- ---\n"); 376#endif 377 return 0; 378} 379 380static int function_stat_show(struct seq_file *m, void *v) 381{ 382 struct ftrace_profile *rec = v; 383 char str[KSYM_SYMBOL_LEN]; 384 int ret = 0; 385#ifdef CONFIG_FUNCTION_GRAPH_TRACER 386 static struct trace_seq s; 387 unsigned long long avg; 388 unsigned long long stddev; 389#endif 390 mutex_lock(&ftrace_profile_lock); 391 392 /* we raced with function_profile_reset() */ 393 if (unlikely(rec->counter == 0)) { 394 ret = -EBUSY; 395 goto out; 396 } 397 398 kallsyms_lookup(rec->ip, NULL, NULL, NULL, str); 399 seq_printf(m, " %-30.30s %10lu", str, rec->counter); 400 401#ifdef CONFIG_FUNCTION_GRAPH_TRACER 402 seq_printf(m, " "); 403 avg = rec->time; 404 do_div(avg, rec->counter); 405 406 /* Sample standard deviation (s^2) */ 407 if (rec->counter <= 1) 408 stddev = 0; 409 else { 410 stddev = rec->time_squared - rec->counter * avg * avg; 411 /* 412 * Divide only 1000 for ns^2 -> us^2 conversion. 413 * trace_print_graph_duration will divide 1000 again. 414 */ 415 do_div(stddev, (rec->counter - 1) * 1000); 416 } 417 418 trace_seq_init(&s); 419 trace_print_graph_duration(rec->time, &s); 420 trace_seq_puts(&s, " "); 421 trace_print_graph_duration(avg, &s); 422 trace_seq_puts(&s, " "); 423 trace_print_graph_duration(stddev, &s); 424 trace_print_seq(m, &s); 425#endif 426 seq_putc(m, '\n'); 427out: 428 mutex_unlock(&ftrace_profile_lock); 429 430 return ret; 431} 432 433static void ftrace_profile_reset(struct ftrace_profile_stat *stat) 434{ 435 struct ftrace_profile_page *pg; 436 437 pg = stat->pages = stat->start; 438 439 while (pg) { 440 memset(pg->records, 0, PROFILE_RECORDS_SIZE); 441 pg->index = 0; 442 pg = pg->next; 443 } 444 445 memset(stat->hash, 0, 446 FTRACE_PROFILE_HASH_SIZE * sizeof(struct hlist_head)); 447} 448 449int ftrace_profile_pages_init(struct ftrace_profile_stat *stat) 450{ 451 struct ftrace_profile_page *pg; 452 int functions; 453 int pages; 454 int i; 455 456 /* If we already allocated, do nothing */ 457 if (stat->pages) 458 return 0; 459 460 stat->pages = (void *)get_zeroed_page(GFP_KERNEL); 461 if (!stat->pages) 462 return -ENOMEM; 463 464#ifdef CONFIG_DYNAMIC_FTRACE 465 functions = ftrace_update_tot_cnt; 466#else 467 /* 468 * We do not know the number of functions that exist because 469 * dynamic tracing is what counts them. With past experience 470 * we have around 20K functions. That should be more than enough. 471 * It is highly unlikely we will execute every function in 472 * the kernel. 473 */ 474 functions = 20000; 475#endif 476 477 pg = stat->start = stat->pages; 478 479 pages = DIV_ROUND_UP(functions, PROFILES_PER_PAGE); 480 481 for (i = 0; i < pages; i++) { 482 pg->next = (void *)get_zeroed_page(GFP_KERNEL); 483 if (!pg->next) 484 goto out_free; 485 pg = pg->next; 486 } 487 488 return 0; 489 490 out_free: 491 pg = stat->start; 492 while (pg) { 493 unsigned long tmp = (unsigned long)pg; 494 495 pg = pg->next; 496 free_page(tmp); 497 } 498 499 free_page((unsigned long)stat->pages); 500 stat->pages = NULL; 501 stat->start = NULL; 502 503 return -ENOMEM; 504} 505 506static int ftrace_profile_init_cpu(int cpu) 507{ 508 struct ftrace_profile_stat *stat; 509 int size; 510 511 stat = &per_cpu(ftrace_profile_stats, cpu); 512 513 if (stat->hash) { 514 /* If the profile is already created, simply reset it */ 515 ftrace_profile_reset(stat); 516 return 0; 517 } 518 519 /* 520 * We are profiling all functions, but usually only a few thousand 521 * functions are hit. We'll make a hash of 1024 items. 522 */ 523 size = FTRACE_PROFILE_HASH_SIZE; 524 525 stat->hash = kzalloc(sizeof(struct hlist_head) * size, GFP_KERNEL); 526 527 if (!stat->hash) 528 return -ENOMEM; 529 530 if (!ftrace_profile_bits) { 531 size--; 532 533 for (; size; size >>= 1) 534 ftrace_profile_bits++; 535 } 536 537 /* Preallocate the function profiling pages */ 538 if (ftrace_profile_pages_init(stat) < 0) { 539 kfree(stat->hash); 540 stat->hash = NULL; 541 return -ENOMEM; 542 } 543 544 return 0; 545} 546 547static int ftrace_profile_init(void) 548{ 549 int cpu; 550 int ret = 0; 551 552 for_each_online_cpu(cpu) { 553 ret = ftrace_profile_init_cpu(cpu); 554 if (ret) 555 break; 556 } 557 558 return ret; 559} 560 561/* interrupts must be disabled */ 562static struct ftrace_profile * 563ftrace_find_profiled_func(struct ftrace_profile_stat *stat, unsigned long ip) 564{ 565 struct ftrace_profile *rec; 566 struct hlist_head *hhd; 567 struct hlist_node *n; 568 unsigned long key; 569 570 key = hash_long(ip, ftrace_profile_bits); 571 hhd = &stat->hash[key]; 572 573 if (hlist_empty(hhd)) 574 return NULL; 575 576 hlist_for_each_entry_rcu(rec, n, hhd, node) { 577 if (rec->ip == ip) 578 return rec; 579 } 580 581 return NULL; 582} 583 584static void ftrace_add_profile(struct ftrace_profile_stat *stat, 585 struct ftrace_profile *rec) 586{ 587 unsigned long key; 588 589 key = hash_long(rec->ip, ftrace_profile_bits); 590 hlist_add_head_rcu(&rec->node, &stat->hash[key]); 591} 592 593/* 594 * The memory is already allocated, this simply finds a new record to use. 595 */ 596static struct ftrace_profile * 597ftrace_profile_alloc(struct ftrace_profile_stat *stat, unsigned long ip) 598{ 599 struct ftrace_profile *rec = NULL; 600 601 /* prevent recursion (from NMIs) */ 602 if (atomic_inc_return(&stat->disabled) != 1) 603 goto out; 604 605 /* 606 * Try to find the function again since an NMI 607 * could have added it 608 */ 609 rec = ftrace_find_profiled_func(stat, ip); 610 if (rec) 611 goto out; 612 613 if (stat->pages->index == PROFILES_PER_PAGE) { 614 if (!stat->pages->next) 615 goto out; 616 stat->pages = stat->pages->next; 617 } 618 619 rec = &stat->pages->records[stat->pages->index++]; 620 rec->ip = ip; 621 ftrace_add_profile(stat, rec); 622 623 out: 624 atomic_dec(&stat->disabled); 625 626 return rec; 627} 628 629static void 630function_profile_call(unsigned long ip, unsigned long parent_ip) 631{ 632 struct ftrace_profile_stat *stat; 633 struct ftrace_profile *rec; 634 unsigned long flags; 635 636 if (!ftrace_profile_enabled) 637 return; 638 639 local_irq_save(flags); 640 641 stat = &__get_cpu_var(ftrace_profile_stats); 642 if (!stat->hash || !ftrace_profile_enabled) 643 goto out; 644 645 rec = ftrace_find_profiled_func(stat, ip); 646 if (!rec) { 647 rec = ftrace_profile_alloc(stat, ip); 648 if (!rec) 649 goto out; 650 } 651 652 rec->counter++; 653 out: 654 local_irq_restore(flags); 655} 656 657#ifdef CONFIG_FUNCTION_GRAPH_TRACER 658static int profile_graph_entry(struct ftrace_graph_ent *trace) 659{ 660 function_profile_call(trace->func, 0); 661 return 1; 662} 663 664static void profile_graph_return(struct ftrace_graph_ret *trace) 665{ 666 struct ftrace_profile_stat *stat; 667 unsigned long long calltime; 668 struct ftrace_profile *rec; 669 unsigned long flags; 670 671 local_irq_save(flags); 672 stat = &__get_cpu_var(ftrace_profile_stats); 673 if (!stat->hash || !ftrace_profile_enabled) 674 goto out; 675 676 /* If the calltime was zero'd ignore it */ 677 if (!trace->calltime) 678 goto out; 679 680 calltime = trace->rettime - trace->calltime; 681 682 if (!(trace_flags & TRACE_ITER_GRAPH_TIME)) { 683 int index; 684 685 index = trace->depth; 686 687 /* Append this call time to the parent time to subtract */ 688 if (index) 689 current->ret_stack[index - 1].subtime += calltime; 690 691 if (current->ret_stack[index].subtime < calltime) 692 calltime -= current->ret_stack[index].subtime; 693 else 694 calltime = 0; 695 } 696 697 rec = ftrace_find_profiled_func(stat, trace->func); 698 if (rec) { 699 rec->time += calltime; 700 rec->time_squared += calltime * calltime; 701 } 702 703 out: 704 local_irq_restore(flags); 705} 706 707static int register_ftrace_profiler(void) 708{ 709 return register_ftrace_graph(&profile_graph_return, 710 &profile_graph_entry); 711} 712 713static void unregister_ftrace_profiler(void) 714{ 715 unregister_ftrace_graph(); 716} 717#else 718static struct ftrace_ops ftrace_profile_ops __read_mostly = 719{ 720 .func = function_profile_call, 721}; 722 723static int register_ftrace_profiler(void) 724{ 725 return register_ftrace_function(&ftrace_profile_ops); 726} 727 728static void unregister_ftrace_profiler(void) 729{ 730 unregister_ftrace_function(&ftrace_profile_ops); 731} 732#endif /* CONFIG_FUNCTION_GRAPH_TRACER */ 733 734static ssize_t 735ftrace_profile_write(struct file *filp, const char __user *ubuf, 736 size_t cnt, loff_t *ppos) 737{ 738 unsigned long val; 739 char buf[64]; /* big enough to hold a number */ 740 int ret; 741 742 if (cnt >= sizeof(buf)) 743 return -EINVAL; 744 745 if (copy_from_user(&buf, ubuf, cnt)) 746 return -EFAULT; 747 748 buf[cnt] = 0; 749 750 ret = strict_strtoul(buf, 10, &val); 751 if (ret < 0) 752 return ret; 753 754 val = !!val; 755 756 mutex_lock(&ftrace_profile_lock); 757 if (ftrace_profile_enabled ^ val) { 758 if (val) { 759 ret = ftrace_profile_init(); 760 if (ret < 0) { 761 cnt = ret; 762 goto out; 763 } 764 765 ret = register_ftrace_profiler(); 766 if (ret < 0) { 767 cnt = ret; 768 goto out; 769 } 770 ftrace_profile_enabled = 1; 771 } else { 772 ftrace_profile_enabled = 0; 773 /* 774 * unregister_ftrace_profiler calls stop_machine 775 * so this acts like an synchronize_sched. 776 */ 777 unregister_ftrace_profiler(); 778 } 779 } 780 out: 781 mutex_unlock(&ftrace_profile_lock); 782 783 *ppos += cnt; 784 785 return cnt; 786} 787 788static ssize_t 789ftrace_profile_read(struct file *filp, char __user *ubuf, 790 size_t cnt, loff_t *ppos) 791{ 792 char buf[64]; /* big enough to hold a number */ 793 int r; 794 795 r = sprintf(buf, "%u\n", ftrace_profile_enabled); 796 return simple_read_from_buffer(ubuf, cnt, ppos, buf, r); 797} 798 799static const struct file_operations ftrace_profile_fops = { 800 .open = tracing_open_generic, 801 .read = ftrace_profile_read, 802 .write = ftrace_profile_write, 803}; 804 805/* used to initialize the real stat files */ 806static struct tracer_stat function_stats __initdata = { 807 .name = "functions", 808 .stat_start = function_stat_start, 809 .stat_next = function_stat_next, 810 .stat_cmp = function_stat_cmp, 811 .stat_headers = function_stat_headers, 812 .stat_show = function_stat_show 813}; 814 815static __init void ftrace_profile_debugfs(struct dentry *d_tracer) 816{ 817 struct ftrace_profile_stat *stat; 818 struct dentry *entry; 819 char *name; 820 int ret; 821 int cpu; 822 823 for_each_possible_cpu(cpu) { 824 stat = &per_cpu(ftrace_profile_stats, cpu); 825 826 /* allocate enough for function name + cpu number */ 827 name = kmalloc(32, GFP_KERNEL); 828 if (!name) { 829 /* 830 * The files created are permanent, if something happens 831 * we still do not free memory. 832 */ 833 WARN(1, 834 "Could not allocate stat file for cpu %d\n", 835 cpu); 836 return; 837 } 838 stat->stat = function_stats; 839 snprintf(name, 32, "function%d", cpu); 840 stat->stat.name = name; 841 ret = register_stat_tracer(&stat->stat); 842 if (ret) { 843 WARN(1, 844 "Could not register function stat for cpu %d\n", 845 cpu); 846 kfree(name); 847 return; 848 } 849 } 850 851 entry = debugfs_create_file("function_profile_enabled", 0644, 852 d_tracer, NULL, &ftrace_profile_fops); 853 if (!entry) 854 pr_warning("Could not create debugfs " 855 "'function_profile_enabled' entry\n"); 856} 857 858#else /* CONFIG_FUNCTION_PROFILER */ 859static __init void ftrace_profile_debugfs(struct dentry *d_tracer) 860{ 861} 862#endif /* CONFIG_FUNCTION_PROFILER */ 863 864static struct pid * const ftrace_swapper_pid = &init_struct_pid; 865 866#ifdef CONFIG_DYNAMIC_FTRACE 867 868#ifndef CONFIG_FTRACE_MCOUNT_RECORD 869# error Dynamic ftrace depends on MCOUNT_RECORD 870#endif 871 872static struct hlist_head ftrace_func_hash[FTRACE_FUNC_HASHSIZE] __read_mostly; 873 874struct ftrace_func_probe { 875 struct hlist_node node; 876 struct ftrace_probe_ops *ops; 877 unsigned long flags; 878 unsigned long ip; 879 void *data; 880 struct rcu_head rcu; 881}; 882 883enum { 884 FTRACE_ENABLE_CALLS = (1 << 0), 885 FTRACE_DISABLE_CALLS = (1 << 1), 886 FTRACE_UPDATE_TRACE_FUNC = (1 << 2), 887 FTRACE_ENABLE_MCOUNT = (1 << 3), 888 FTRACE_DISABLE_MCOUNT = (1 << 4), 889 FTRACE_START_FUNC_RET = (1 << 5), 890 FTRACE_STOP_FUNC_RET = (1 << 6), 891}; 892 893static int ftrace_filtered; 894 895static struct dyn_ftrace *ftrace_new_addrs; 896 897static DEFINE_MUTEX(ftrace_regex_lock); 898 899struct ftrace_page { 900 struct ftrace_page *next; 901 int index; 902 struct dyn_ftrace records[]; 903}; 904 905#define ENTRIES_PER_PAGE \ 906 ((PAGE_SIZE - sizeof(struct ftrace_page)) / sizeof(struct dyn_ftrace)) 907 908/* estimate from running different kernels */ 909#define NR_TO_INIT 10000 910 911static struct ftrace_page *ftrace_pages_start; 912static struct ftrace_page *ftrace_pages; 913 914static struct dyn_ftrace *ftrace_free_records; 915 916/* 917 * This is a double for. Do not use 'break' to break out of the loop, 918 * you must use a goto. 919 */ 920#define do_for_each_ftrace_rec(pg, rec) \ 921 for (pg = ftrace_pages_start; pg; pg = pg->next) { \ 922 int _____i; \ 923 for (_____i = 0; _____i < pg->index; _____i++) { \ 924 rec = &pg->records[_____i]; 925 926#define while_for_each_ftrace_rec() \ 927 } \ 928 } 929 930static void ftrace_free_rec(struct dyn_ftrace *rec) 931{ 932 rec->freelist = ftrace_free_records; 933 ftrace_free_records = rec; 934 rec->flags |= FTRACE_FL_FREE; 935} 936 937static struct dyn_ftrace *ftrace_alloc_dyn_node(unsigned long ip) 938{ 939 struct dyn_ftrace *rec; 940 941 /* First check for freed records */ 942 if (ftrace_free_records) { 943 rec = ftrace_free_records; 944 945 if (unlikely(!(rec->flags & FTRACE_FL_FREE))) { 946 FTRACE_WARN_ON_ONCE(1); 947 ftrace_free_records = NULL; 948 return NULL; 949 } 950 951 ftrace_free_records = rec->freelist; 952 memset(rec, 0, sizeof(*rec)); 953 return rec; 954 } 955 956 if (ftrace_pages->index == ENTRIES_PER_PAGE) { 957 if (!ftrace_pages->next) { 958 /* allocate another page */ 959 ftrace_pages->next = 960 (void *)get_zeroed_page(GFP_KERNEL); 961 if (!ftrace_pages->next) 962 return NULL; 963 } 964 ftrace_pages = ftrace_pages->next; 965 } 966 967 return &ftrace_pages->records[ftrace_pages->index++]; 968} 969 970static struct dyn_ftrace * 971ftrace_record_ip(unsigned long ip) 972{ 973 struct dyn_ftrace *rec; 974 975 if (ftrace_disabled) 976 return NULL; 977 978 rec = ftrace_alloc_dyn_node(ip); 979 if (!rec) 980 return NULL; 981 982 rec->ip = ip; 983 rec->newlist = ftrace_new_addrs; 984 ftrace_new_addrs = rec; 985 986 return rec; 987} 988 989static void print_ip_ins(const char *fmt, unsigned char *p) 990{ 991 int i; 992 993 printk(KERN_CONT "%s", fmt); 994 995 for (i = 0; i < MCOUNT_INSN_SIZE; i++) 996 printk(KERN_CONT "%s%02x", i ? ":" : "", p[i]); 997} 998 999static void ftrace_bug(int failed, unsigned long ip) 1000{ 1001 switch (failed) { 1002 case -EFAULT: 1003 FTRACE_WARN_ON_ONCE(1); 1004 pr_info("ftrace faulted on modifying "); 1005 print_ip_sym(ip); 1006 break; 1007 case -EINVAL: 1008 FTRACE_WARN_ON_ONCE(1); 1009 pr_info("ftrace failed to modify "); 1010 print_ip_sym(ip); 1011 print_ip_ins(" actual: ", (unsigned char *)ip); 1012 printk(KERN_CONT "\n"); 1013 break; 1014 case -EPERM: 1015 FTRACE_WARN_ON_ONCE(1); 1016 pr_info("ftrace faulted on writing "); 1017 print_ip_sym(ip); 1018 break; 1019 default: 1020 FTRACE_WARN_ON_ONCE(1); 1021 pr_info("ftrace faulted on unknown error "); 1022 print_ip_sym(ip); 1023 } 1024} 1025 1026 1027/* Return 1 if the address range is reserved for ftrace */ 1028int ftrace_text_reserved(void *start, void *end) 1029{ 1030 struct dyn_ftrace *rec; 1031 struct ftrace_page *pg; 1032 1033 do_for_each_ftrace_rec(pg, rec) { 1034 if (rec->ip <= (unsigned long)end && 1035 rec->ip + MCOUNT_INSN_SIZE > (unsigned long)start) 1036 return 1; 1037 } while_for_each_ftrace_rec(); 1038 return 0; 1039} 1040 1041 1042static int 1043__ftrace_replace_code(struct dyn_ftrace *rec, int enable) 1044{ 1045 unsigned long ftrace_addr; 1046 unsigned long flag = 0UL; 1047 1048 ftrace_addr = (unsigned long)FTRACE_ADDR; 1049 1050 /* 1051 * If this record is not to be traced or we want to disable it, 1052 * then disable it. 1053 * 1054 * If we want to enable it and filtering is off, then enable it. 1055 * 1056 * If we want to enable it and filtering is on, enable it only if 1057 * it's filtered 1058 */ 1059 if (enable && !(rec->flags & FTRACE_FL_NOTRACE)) { 1060 if (!ftrace_filtered || (rec->flags & FTRACE_FL_FILTER)) 1061 flag = FTRACE_FL_ENABLED; 1062 } 1063 1064 /* If the state of this record hasn't changed, then do nothing */ 1065 if ((rec->flags & FTRACE_FL_ENABLED) == flag) 1066 return 0; 1067 1068 if (flag) { 1069 rec->flags |= FTRACE_FL_ENABLED; 1070 return ftrace_make_call(rec, ftrace_addr); 1071 } 1072 1073 rec->flags &= ~FTRACE_FL_ENABLED; 1074 return ftrace_make_nop(NULL, rec, ftrace_addr); 1075} 1076 1077static void ftrace_replace_code(int enable) 1078{ 1079 struct dyn_ftrace *rec; 1080 struct ftrace_page *pg; 1081 int failed; 1082 1083 do_for_each_ftrace_rec(pg, rec) { 1084 /* 1085 * Skip over free records, records that have 1086 * failed and not converted. 1087 */ 1088 if (rec->flags & FTRACE_FL_FREE || 1089 rec->flags & FTRACE_FL_FAILED || 1090 !(rec->flags & FTRACE_FL_CONVERTED)) 1091 continue; 1092 1093 failed = __ftrace_replace_code(rec, enable); 1094 if (failed) { 1095 rec->flags |= FTRACE_FL_FAILED; 1096 ftrace_bug(failed, rec->ip); 1097 /* Stop processing */ 1098 return; 1099 } 1100 } while_for_each_ftrace_rec(); 1101} 1102 1103static int 1104ftrace_code_disable(struct module *mod, struct dyn_ftrace *rec) 1105{ 1106 unsigned long ip; 1107 int ret; 1108 1109 ip = rec->ip; 1110 1111 ret = ftrace_make_nop(mod, rec, MCOUNT_ADDR); 1112 if (ret) { 1113 ftrace_bug(ret, ip); 1114 rec->flags |= FTRACE_FL_FAILED; 1115 return 0; 1116 } 1117 return 1; 1118} 1119 1120/* 1121 * archs can override this function if they must do something 1122 * before the modifying code is performed. 1123 */ 1124int __weak ftrace_arch_code_modify_prepare(void) 1125{ 1126 return 0; 1127} 1128 1129/* 1130 * archs can override this function if they must do something 1131 * after the modifying code is performed. 1132 */ 1133int __weak ftrace_arch_code_modify_post_process(void) 1134{ 1135 return 0; 1136} 1137 1138static int __ftrace_modify_code(void *data) 1139{ 1140 int *command = data; 1141 1142 if (*command & FTRACE_ENABLE_CALLS) 1143 ftrace_replace_code(1); 1144 else if (*command & FTRACE_DISABLE_CALLS) 1145 ftrace_replace_code(0); 1146 1147 if (*command & FTRACE_UPDATE_TRACE_FUNC) 1148 ftrace_update_ftrace_func(ftrace_trace_function); 1149 1150 if (*command & FTRACE_START_FUNC_RET) 1151 ftrace_enable_ftrace_graph_caller(); 1152 else if (*command & FTRACE_STOP_FUNC_RET) 1153 ftrace_disable_ftrace_graph_caller(); 1154 1155 return 0; 1156} 1157 1158static void ftrace_run_update_code(int command) 1159{ 1160 int ret; 1161 1162 ret = ftrace_arch_code_modify_prepare(); 1163 FTRACE_WARN_ON(ret); 1164 if (ret) 1165 return; 1166 1167 stop_machine(__ftrace_modify_code, &command, NULL); 1168 1169 ret = ftrace_arch_code_modify_post_process(); 1170 FTRACE_WARN_ON(ret); 1171} 1172 1173static ftrace_func_t saved_ftrace_func; 1174static int ftrace_start_up; 1175 1176static void ftrace_startup_enable(int command) 1177{ 1178 if (saved_ftrace_func != ftrace_trace_function) { 1179 saved_ftrace_func = ftrace_trace_function; 1180 command |= FTRACE_UPDATE_TRACE_FUNC; 1181 } 1182 1183 if (!command || !ftrace_enabled) 1184 return; 1185 1186 ftrace_run_update_code(command); 1187} 1188 1189static void ftrace_startup(int command) 1190{ 1191 if (unlikely(ftrace_disabled)) 1192 return; 1193 1194 ftrace_start_up++; 1195 command |= FTRACE_ENABLE_CALLS; 1196 1197 ftrace_startup_enable(command); 1198} 1199 1200static void ftrace_shutdown(int command) 1201{ 1202 if (unlikely(ftrace_disabled)) 1203 return; 1204 1205 ftrace_start_up--; 1206 /* 1207 * Just warn in case of unbalance, no need to kill ftrace, it's not 1208 * critical but the ftrace_call callers may be never nopped again after 1209 * further ftrace uses. 1210 */ 1211 WARN_ON_ONCE(ftrace_start_up < 0); 1212 1213 if (!ftrace_start_up) 1214 command |= FTRACE_DISABLE_CALLS; 1215 1216 if (saved_ftrace_func != ftrace_trace_function) { 1217 saved_ftrace_func = ftrace_trace_function; 1218 command |= FTRACE_UPDATE_TRACE_FUNC; 1219 } 1220 1221 if (!command || !ftrace_enabled) 1222 return; 1223 1224 ftrace_run_update_code(command); 1225} 1226 1227static void ftrace_startup_sysctl(void) 1228{ 1229 int command = FTRACE_ENABLE_MCOUNT; 1230 1231 if (unlikely(ftrace_disabled)) 1232 return; 1233 1234 /* Force update next time */ 1235 saved_ftrace_func = NULL; 1236 /* ftrace_start_up is true if we want ftrace running */ 1237 if (ftrace_start_up) 1238 command |= FTRACE_ENABLE_CALLS; 1239 1240 ftrace_run_update_code(command); 1241} 1242 1243static void ftrace_shutdown_sysctl(void) 1244{ 1245 int command = FTRACE_DISABLE_MCOUNT; 1246 1247 if (unlikely(ftrace_disabled)) 1248 return; 1249 1250 /* ftrace_start_up is true if ftrace is running */ 1251 if (ftrace_start_up) 1252 command |= FTRACE_DISABLE_CALLS; 1253 1254 ftrace_run_update_code(command); 1255} 1256 1257static cycle_t ftrace_update_time; 1258static unsigned long ftrace_update_cnt; 1259unsigned long ftrace_update_tot_cnt; 1260 1261static int ftrace_update_code(struct module *mod) 1262{ 1263 struct dyn_ftrace *p; 1264 cycle_t start, stop; 1265 1266 start = ftrace_now(raw_smp_processor_id()); 1267 ftrace_update_cnt = 0; 1268 1269 while (ftrace_new_addrs) { 1270 1271 /* If something went wrong, bail without enabling anything */ 1272 if (unlikely(ftrace_disabled)) 1273 return -1; 1274 1275 p = ftrace_new_addrs; 1276 ftrace_new_addrs = p->newlist; 1277 p->flags = 0L; 1278 1279 /* 1280 * Do the initial record convertion from mcount jump 1281 * to the NOP instructions. 1282 */ 1283 if (!ftrace_code_disable(mod, p)) { 1284 ftrace_free_rec(p); 1285 continue; 1286 } 1287 1288 p->flags |= FTRACE_FL_CONVERTED; 1289 ftrace_update_cnt++; 1290 1291 /* 1292 * If the tracing is enabled, go ahead and enable the record. 1293 * 1294 * The reason not to enable the record immediatelly is the 1295 * inherent check of ftrace_make_nop/ftrace_make_call for 1296 * correct previous instructions. Making first the NOP 1297 * conversion puts the module to the correct state, thus 1298 * passing the ftrace_make_call check. 1299 */ 1300 if (ftrace_start_up) { 1301 int failed = __ftrace_replace_code(p, 1); 1302 if (failed) { 1303 ftrace_bug(failed, p->ip); 1304 ftrace_free_rec(p); 1305 } 1306 } 1307 } 1308 1309 stop = ftrace_now(raw_smp_processor_id()); 1310 ftrace_update_time = stop - start; 1311 ftrace_update_tot_cnt += ftrace_update_cnt; 1312 1313 return 0; 1314} 1315 1316static int __init ftrace_dyn_table_alloc(unsigned long num_to_init) 1317{ 1318 struct ftrace_page *pg; 1319 int cnt; 1320 int i; 1321 1322 /* allocate a few pages */ 1323 ftrace_pages_start = (void *)get_zeroed_page(GFP_KERNEL); 1324 if (!ftrace_pages_start) 1325 return -1; 1326 1327 /* 1328 * Allocate a few more pages. 1329 * 1330 * TODO: have some parser search vmlinux before 1331 * final linking to find all calls to ftrace. 1332 * Then we can: 1333 * a) know how many pages to allocate. 1334 * and/or 1335 * b) set up the table then. 1336 * 1337 * The dynamic code is still necessary for 1338 * modules. 1339 */ 1340 1341 pg = ftrace_pages = ftrace_pages_start; 1342 1343 cnt = num_to_init / ENTRIES_PER_PAGE; 1344 pr_info("ftrace: allocating %ld entries in %d pages\n", 1345 num_to_init, cnt + 1); 1346 1347 for (i = 0; i < cnt; i++) { 1348 pg->next = (void *)get_zeroed_page(GFP_KERNEL); 1349 1350 /* If we fail, we'll try later anyway */ 1351 if (!pg->next) 1352 break; 1353 1354 pg = pg->next; 1355 } 1356 1357 return 0; 1358} 1359 1360enum { 1361 FTRACE_ITER_FILTER = (1 << 0), 1362 FTRACE_ITER_NOTRACE = (1 << 1), 1363 FTRACE_ITER_FAILURES = (1 << 2), 1364 FTRACE_ITER_PRINTALL = (1 << 3), 1365 FTRACE_ITER_HASH = (1 << 4), 1366}; 1367 1368#define FTRACE_BUFF_MAX (KSYM_SYMBOL_LEN+4) /* room for wildcards */ 1369 1370struct ftrace_iterator { 1371 struct ftrace_page *pg; 1372 int hidx; 1373 int idx; 1374 unsigned flags; 1375 struct trace_parser parser; 1376}; 1377 1378static void * 1379t_hash_next(struct seq_file *m, void *v, loff_t *pos) 1380{ 1381 struct ftrace_iterator *iter = m->private; 1382 struct hlist_node *hnd = v; 1383 struct hlist_head *hhd; 1384 1385 WARN_ON(!(iter->flags & FTRACE_ITER_HASH)); 1386 1387 (*pos)++; 1388 1389 retry: 1390 if (iter->hidx >= FTRACE_FUNC_HASHSIZE) 1391 return NULL; 1392 1393 hhd = &ftrace_func_hash[iter->hidx]; 1394 1395 if (hlist_empty(hhd)) { 1396 iter->hidx++; 1397 hnd = NULL; 1398 goto retry; 1399 } 1400 1401 if (!hnd) 1402 hnd = hhd->first; 1403 else { 1404 hnd = hnd->next; 1405 if (!hnd) { 1406 iter->hidx++; 1407 goto retry; 1408 } 1409 } 1410 1411 return hnd; 1412} 1413 1414static void *t_hash_start(struct seq_file *m, loff_t *pos) 1415{ 1416 struct ftrace_iterator *iter = m->private; 1417 void *p = NULL; 1418 loff_t l; 1419 1420 if (!(iter->flags & FTRACE_ITER_HASH)) 1421 *pos = 0; 1422 1423 iter->flags |= FTRACE_ITER_HASH; 1424 1425 iter->hidx = 0; 1426 for (l = 0; l <= *pos; ) { 1427 p = t_hash_next(m, p, &l); 1428 if (!p) 1429 break; 1430 } 1431 return p; 1432} 1433 1434static int t_hash_show(struct seq_file *m, void *v) 1435{ 1436 struct ftrace_func_probe *rec; 1437 struct hlist_node *hnd = v; 1438 1439 rec = hlist_entry(hnd, struct ftrace_func_probe, node); 1440 1441 if (rec->ops->print) 1442 return rec->ops->print(m, rec->ip, rec->ops, rec->data); 1443 1444 seq_printf(m, "%ps:%ps", (void *)rec->ip, (void *)rec->ops->func); 1445 1446 if (rec->data) 1447 seq_printf(m, ":%p", rec->data); 1448 seq_putc(m, '\n'); 1449 1450 return 0; 1451} 1452 1453static void * 1454t_next(struct seq_file *m, void *v, loff_t *pos) 1455{ 1456 struct ftrace_iterator *iter = m->private; 1457 struct dyn_ftrace *rec = NULL; 1458 1459 if (iter->flags & FTRACE_ITER_HASH) 1460 return t_hash_next(m, v, pos); 1461 1462 (*pos)++; 1463 1464 if (iter->flags & FTRACE_ITER_PRINTALL) 1465 return NULL; 1466 1467 retry: 1468 if (iter->idx >= iter->pg->index) { 1469 if (iter->pg->next) { 1470 iter->pg = iter->pg->next; 1471 iter->idx = 0; 1472 goto retry; 1473 } 1474 } else { 1475 rec = &iter->pg->records[iter->idx++]; 1476 if ((rec->flags & FTRACE_FL_FREE) || 1477 1478 (!(iter->flags & FTRACE_ITER_FAILURES) && 1479 (rec->flags & FTRACE_FL_FAILED)) || 1480 1481 ((iter->flags & FTRACE_ITER_FAILURES) && 1482 !(rec->flags & FTRACE_FL_FAILED)) || 1483 1484 ((iter->flags & FTRACE_ITER_FILTER) && 1485 !(rec->flags & FTRACE_FL_FILTER)) || 1486 1487 ((iter->flags & FTRACE_ITER_NOTRACE) && 1488 !(rec->flags & FTRACE_FL_NOTRACE))) { 1489 rec = NULL; 1490 goto retry; 1491 } 1492 } 1493 1494 return rec; 1495} 1496 1497static void *t_start(struct seq_file *m, loff_t *pos) 1498{ 1499 struct ftrace_iterator *iter = m->private; 1500 void *p = NULL; 1501 loff_t l; 1502 1503 mutex_lock(&ftrace_lock); 1504 /* 1505 * For set_ftrace_filter reading, if we have the filter 1506 * off, we can short cut and just print out that all 1507 * functions are enabled. 1508 */ 1509 if (iter->flags & FTRACE_ITER_FILTER && !ftrace_filtered) { 1510 if (*pos > 0) 1511 return t_hash_start(m, pos); 1512 iter->flags |= FTRACE_ITER_PRINTALL; 1513 /* reset in case of seek/pread */ 1514 iter->flags &= ~FTRACE_ITER_HASH; 1515 return iter; 1516 } 1517 1518 if (iter->flags & FTRACE_ITER_HASH) 1519 return t_hash_start(m, pos); 1520 1521 iter->pg = ftrace_pages_start; 1522 iter->idx = 0; 1523 for (l = 0; l <= *pos; ) { 1524 p = t_next(m, p, &l); 1525 if (!p) 1526 break; 1527 } 1528 1529 if (!p && iter->flags & FTRACE_ITER_FILTER) 1530 return t_hash_start(m, pos); 1531 1532 return p; 1533} 1534 1535static void t_stop(struct seq_file *m, void *p) 1536{ 1537 mutex_unlock(&ftrace_lock); 1538} 1539 1540static int t_show(struct seq_file *m, void *v) 1541{ 1542 struct ftrace_iterator *iter = m->private; 1543 struct dyn_ftrace *rec = v; 1544 1545 if (iter->flags & FTRACE_ITER_HASH) 1546 return t_hash_show(m, v); 1547 1548 if (iter->flags & FTRACE_ITER_PRINTALL) { 1549 seq_printf(m, "#### all functions enabled ####\n"); 1550 return 0; 1551 } 1552 1553 if (!rec) 1554 return 0; 1555 1556 seq_printf(m, "%ps\n", (void *)rec->ip); 1557 1558 return 0; 1559} 1560 1561static const struct seq_operations show_ftrace_seq_ops = { 1562 .start = t_start, 1563 .next = t_next, 1564 .stop = t_stop, 1565 .show = t_show, 1566}; 1567 1568static int 1569ftrace_avail_open(struct inode *inode, struct file *file) 1570{ 1571 struct ftrace_iterator *iter; 1572 int ret; 1573 1574 if (unlikely(ftrace_disabled)) 1575 return -ENODEV; 1576 1577 iter = kzalloc(sizeof(*iter), GFP_KERNEL); 1578 if (!iter) 1579 return -ENOMEM; 1580 1581 iter->pg = ftrace_pages_start; 1582 1583 ret = seq_open(file, &show_ftrace_seq_ops); 1584 if (!ret) { 1585 struct seq_file *m = file->private_data; 1586 1587 m->private = iter; 1588 } else { 1589 kfree(iter); 1590 } 1591 1592 return ret; 1593} 1594 1595static int 1596ftrace_failures_open(struct inode *inode, struct file *file) 1597{ 1598 int ret; 1599 struct seq_file *m; 1600 struct ftrace_iterator *iter; 1601 1602 ret = ftrace_avail_open(inode, file); 1603 if (!ret) { 1604 m = (struct seq_file *)file->private_data; 1605 iter = (struct ftrace_iterator *)m->private; 1606 iter->flags = FTRACE_ITER_FAILURES; 1607 } 1608 1609 return ret; 1610} 1611 1612 1613static void ftrace_filter_reset(int enable) 1614{ 1615 struct ftrace_page *pg; 1616 struct dyn_ftrace *rec; 1617 unsigned long type = enable ? FTRACE_FL_FILTER : FTRACE_FL_NOTRACE; 1618 1619 mutex_lock(&ftrace_lock); 1620 if (enable) 1621 ftrace_filtered = 0; 1622 do_for_each_ftrace_rec(pg, rec) { 1623 if (rec->flags & FTRACE_FL_FAILED) 1624 continue; 1625 rec->flags &= ~type; 1626 } while_for_each_ftrace_rec(); 1627 mutex_unlock(&ftrace_lock); 1628} 1629 1630static int 1631ftrace_regex_open(struct inode *inode, struct file *file, int enable) 1632{ 1633 struct ftrace_iterator *iter; 1634 int ret = 0; 1635 1636 if (unlikely(ftrace_disabled)) 1637 return -ENODEV; 1638 1639 iter = kzalloc(sizeof(*iter), GFP_KERNEL); 1640 if (!iter) 1641 return -ENOMEM; 1642 1643 if (trace_parser_get_init(&iter->parser, FTRACE_BUFF_MAX)) { 1644 kfree(iter); 1645 return -ENOMEM; 1646 } 1647 1648 mutex_lock(&ftrace_regex_lock); 1649 if ((file->f_mode & FMODE_WRITE) && 1650 (file->f_flags & O_TRUNC)) 1651 ftrace_filter_reset(enable); 1652 1653 if (file->f_mode & FMODE_READ) { 1654 iter->pg = ftrace_pages_start; 1655 iter->flags = enable ? FTRACE_ITER_FILTER : 1656 FTRACE_ITER_NOTRACE; 1657 1658 ret = seq_open(file, &show_ftrace_seq_ops); 1659 if (!ret) { 1660 struct seq_file *m = file->private_data; 1661 m->private = iter; 1662 } else { 1663 trace_parser_put(&iter->parser); 1664 kfree(iter); 1665 } 1666 } else 1667 file->private_data = iter; 1668 mutex_unlock(&ftrace_regex_lock); 1669 1670 return ret; 1671} 1672 1673static int 1674ftrace_filter_open(struct inode *inode, struct file *file) 1675{ 1676 return ftrace_regex_open(inode, file, 1); 1677} 1678 1679static int 1680ftrace_notrace_open(struct inode *inode, struct file *file) 1681{ 1682 return ftrace_regex_open(inode, file, 0); 1683} 1684 1685static loff_t 1686ftrace_regex_lseek(struct file *file, loff_t offset, int origin) 1687{ 1688 loff_t ret; 1689 1690 if (file->f_mode & FMODE_READ) 1691 ret = seq_lseek(file, offset, origin); 1692 else 1693 file->f_pos = ret = 1; 1694 1695 return ret; 1696} 1697 1698static int ftrace_match(char *str, char *regex, int len, int type) 1699{ 1700 int matched = 0; 1701 int slen; 1702 1703 switch (type) { 1704 case MATCH_FULL: 1705 if (strcmp(str, regex) == 0) 1706 matched = 1; 1707 break; 1708 case MATCH_FRONT_ONLY: 1709 if (strncmp(str, regex, len) == 0) 1710 matched = 1; 1711 break; 1712 case MATCH_MIDDLE_ONLY: 1713 if (strstr(str, regex)) 1714 matched = 1; 1715 break; 1716 case MATCH_END_ONLY: 1717 slen = strlen(str); 1718 if (slen >= len && memcmp(str + slen - len, regex, len) == 0) 1719 matched = 1; 1720 break; 1721 } 1722 1723 return matched; 1724} 1725 1726static int 1727ftrace_match_record(struct dyn_ftrace *rec, char *regex, int len, int type) 1728{ 1729 char str[KSYM_SYMBOL_LEN]; 1730 1731 kallsyms_lookup(rec->ip, NULL, NULL, NULL, str); 1732 return ftrace_match(str, regex, len, type); 1733} 1734 1735static int ftrace_match_records(char *buff, int len, int enable) 1736{ 1737 unsigned int search_len; 1738 struct ftrace_page *pg; 1739 struct dyn_ftrace *rec; 1740 unsigned long flag; 1741 char *search; 1742 int type; 1743 int not; 1744 int found = 0; 1745 1746 flag = enable ? FTRACE_FL_FILTER : FTRACE_FL_NOTRACE; 1747 type = filter_parse_regex(buff, len, &search, ¬); 1748 1749 search_len = strlen(search); 1750 1751 mutex_lock(&ftrace_lock); 1752 do_for_each_ftrace_rec(pg, rec) { 1753 1754 if (rec->flags & FTRACE_FL_FAILED) 1755 continue; 1756 1757 if (ftrace_match_record(rec, search, search_len, type)) { 1758 if (not) 1759 rec->flags &= ~flag; 1760 else 1761 rec->flags |= flag; 1762 found = 1; 1763 } 1764 /* 1765 * Only enable filtering if we have a function that 1766 * is filtered on. 1767 */ 1768 if (enable && (rec->flags & FTRACE_FL_FILTER)) 1769 ftrace_filtered = 1; 1770 } while_for_each_ftrace_rec(); 1771 mutex_unlock(&ftrace_lock); 1772 1773 return found; 1774} 1775 1776static int 1777ftrace_match_module_record(struct dyn_ftrace *rec, char *mod, 1778 char *regex, int len, int type) 1779{ 1780 char str[KSYM_SYMBOL_LEN]; 1781 char *modname; 1782 1783 kallsyms_lookup(rec->ip, NULL, NULL, &modname, str); 1784 1785 if (!modname || strcmp(modname, mod)) 1786 return 0; 1787 1788 /* blank search means to match all funcs in the mod */ 1789 if (len) 1790 return ftrace_match(str, regex, len, type); 1791 else 1792 return 1; 1793} 1794 1795static int ftrace_match_module_records(char *buff, char *mod, int enable) 1796{ 1797 unsigned search_len = 0; 1798 struct ftrace_page *pg; 1799 struct dyn_ftrace *rec; 1800 int type = MATCH_FULL; 1801 char *search = buff; 1802 unsigned long flag; 1803 int not = 0; 1804 int found = 0; 1805 1806 flag = enable ? FTRACE_FL_FILTER : FTRACE_FL_NOTRACE; 1807 1808 /* blank or '*' mean the same */ 1809 if (strcmp(buff, "*") == 0) 1810 buff[0] = 0; 1811 1812 /* handle the case of 'dont filter this module' */ 1813 if (strcmp(buff, "!") == 0 || strcmp(buff, "!*") == 0) { 1814 buff[0] = 0; 1815 not = 1; 1816 } 1817 1818 if (strlen(buff)) { 1819 type = filter_parse_regex(buff, strlen(buff), &search, ¬); 1820 search_len = strlen(search); 1821 } 1822 1823 mutex_lock(&ftrace_lock); 1824 do_for_each_ftrace_rec(pg, rec) { 1825 1826 if (rec->flags & FTRACE_FL_FAILED) 1827 continue; 1828 1829 if (ftrace_match_module_record(rec, mod, 1830 search, search_len, type)) { 1831 if (not) 1832 rec->flags &= ~flag; 1833 else 1834 rec->flags |= flag; 1835 found = 1; 1836 } 1837 if (enable && (rec->flags & FTRACE_FL_FILTER)) 1838 ftrace_filtered = 1; 1839 1840 } while_for_each_ftrace_rec(); 1841 mutex_unlock(&ftrace_lock); 1842 1843 return found; 1844} 1845 1846/* 1847 * We register the module command as a template to show others how 1848 * to register the a command as well. 1849 */ 1850 1851static int 1852ftrace_mod_callback(char *func, char *cmd, char *param, int enable) 1853{ 1854 char *mod; 1855 1856 /* 1857 * cmd == 'mod' because we only registered this func 1858 * for the 'mod' ftrace_func_command. 1859 * But if you register one func with multiple commands, 1860 * you can tell which command was used by the cmd 1861 * parameter. 1862 */ 1863 1864 /* we must have a module name */ 1865 if (!param) 1866 return -EINVAL; 1867 1868 mod = strsep(¶m, ":"); 1869 if (!strlen(mod)) 1870 return -EINVAL; 1871 1872 if (ftrace_match_module_records(func, mod, enable)) 1873 return 0; 1874 return -EINVAL; 1875} 1876 1877static struct ftrace_func_command ftrace_mod_cmd = { 1878 .name = "mod", 1879 .func = ftrace_mod_callback, 1880}; 1881 1882static int __init ftrace_mod_cmd_init(void) 1883{ 1884 return register_ftrace_command(&ftrace_mod_cmd); 1885} 1886device_initcall(ftrace_mod_cmd_init); 1887 1888static void 1889function_trace_probe_call(unsigned long ip, unsigned long parent_ip) 1890{ 1891 struct ftrace_func_probe *entry; 1892 struct hlist_head *hhd; 1893 struct hlist_node *n; 1894 unsigned long key; 1895 1896 key = hash_long(ip, FTRACE_HASH_BITS); 1897 1898 hhd = &ftrace_func_hash[key]; 1899 1900 if (hlist_empty(hhd)) 1901 return; 1902 1903 /* 1904 * Disable preemption for these calls to prevent a RCU grace 1905 * period. This syncs the hash iteration and freeing of items 1906 * on the hash. rcu_read_lock is too dangerous here. 1907 */ 1908 preempt_disable_notrace(); 1909 hlist_for_each_entry_rcu(entry, n, hhd, node) { 1910 if (entry->ip == ip) 1911 entry->ops->func(ip, parent_ip, &entry->data); 1912 } 1913 preempt_enable_notrace(); 1914} 1915 1916static struct ftrace_ops trace_probe_ops __read_mostly = 1917{ 1918 .func = function_trace_probe_call, 1919}; 1920 1921static int ftrace_probe_registered; 1922 1923static void __enable_ftrace_function_probe(void) 1924{ 1925 int i; 1926 1927 if (ftrace_probe_registered) 1928 return; 1929 1930 for (i = 0; i < FTRACE_FUNC_HASHSIZE; i++) { 1931 struct hlist_head *hhd = &ftrace_func_hash[i]; 1932 if (hhd->first) 1933 break; 1934 } 1935 /* Nothing registered? */ 1936 if (i == FTRACE_FUNC_HASHSIZE) 1937 return; 1938 1939 __register_ftrace_function(&trace_probe_ops); 1940 ftrace_startup(0); 1941 ftrace_probe_registered = 1; 1942} 1943 1944static void __disable_ftrace_function_probe(void) 1945{ 1946 int i; 1947 1948 if (!ftrace_probe_registered) 1949 return; 1950 1951 for (i = 0; i < FTRACE_FUNC_HASHSIZE; i++) { 1952 struct hlist_head *hhd = &ftrace_func_hash[i]; 1953 if (hhd->first) 1954 return; 1955 } 1956 1957 /* no more funcs left */ 1958 __unregister_ftrace_function(&trace_probe_ops); 1959 ftrace_shutdown(0); 1960 ftrace_probe_registered = 0; 1961} 1962 1963 1964static void ftrace_free_entry_rcu(struct rcu_head *rhp) 1965{ 1966 struct ftrace_func_probe *entry = 1967 container_of(rhp, struct ftrace_func_probe, rcu); 1968 1969 if (entry->ops->free) 1970 entry->ops->free(&entry->data); 1971 kfree(entry); 1972} 1973 1974 1975int 1976register_ftrace_function_probe(char *glob, struct ftrace_probe_ops *ops, 1977 void *data) 1978{ 1979 struct ftrace_func_probe *entry; 1980 struct ftrace_page *pg; 1981 struct dyn_ftrace *rec; 1982 int type, len, not; 1983 unsigned long key; 1984 int count = 0; 1985 char *search; 1986 1987 type = filter_parse_regex(glob, strlen(glob), &search, ¬); 1988 len = strlen(search); 1989 1990 /* we do not support '!' for function probes */ 1991 if (WARN_ON(not)) 1992 return -EINVAL; 1993 1994 mutex_lock(&ftrace_lock); 1995 do_for_each_ftrace_rec(pg, rec) { 1996 1997 if (rec->flags & FTRACE_FL_FAILED) 1998 continue; 1999 2000 if (!ftrace_match_record(rec, search, len, type)) 2001 continue; 2002 2003 entry = kmalloc(sizeof(*entry), GFP_KERNEL); 2004 if (!entry) { 2005 /* If we did not process any, then return error */ 2006 if (!count) 2007 count = -ENOMEM; 2008 goto out_unlock; 2009 } 2010 2011 count++; 2012 2013 entry->data = data; 2014 2015 /* 2016 * The caller might want to do something special 2017 * for each function we find. We call the callback 2018 * to give the caller an opportunity to do so. 2019 */ 2020 if (ops->callback) { 2021 if (ops->callback(rec->ip, &entry->data) < 0) { 2022 /* caller does not like this func */ 2023 kfree(entry); 2024 continue; 2025 } 2026 } 2027 2028 entry->ops = ops; 2029 entry->ip = rec->ip; 2030 2031 key = hash_long(entry->ip, FTRACE_HASH_BITS); 2032 hlist_add_head_rcu(&entry->node, &ftrace_func_hash[key]); 2033 2034 } while_for_each_ftrace_rec(); 2035 __enable_ftrace_function_probe(); 2036 2037 out_unlock: 2038 mutex_unlock(&ftrace_lock); 2039 2040 return count; 2041} 2042 2043enum { 2044 PROBE_TEST_FUNC = 1, 2045 PROBE_TEST_DATA = 2 2046}; 2047 2048static void 2049__unregister_ftrace_function_probe(char *glob, struct ftrace_probe_ops *ops, 2050 void *data, int flags) 2051{ 2052 struct ftrace_func_probe *entry; 2053 struct hlist_node *n, *tmp; 2054 char str[KSYM_SYMBOL_LEN]; 2055 int type = MATCH_FULL; 2056 int i, len = 0; 2057 char *search; 2058 2059 if (glob && (strcmp(glob, "*") == 0 || !strlen(glob))) 2060 glob = NULL; 2061 else if (glob) { 2062 int not; 2063 2064 type = filter_parse_regex(glob, strlen(glob), &search, ¬); 2065 len = strlen(search); 2066 2067 /* we do not support '!' for function probes */ 2068 if (WARN_ON(not)) 2069 return; 2070 } 2071 2072 mutex_lock(&ftrace_lock); 2073 for (i = 0; i < FTRACE_FUNC_HASHSIZE; i++) { 2074 struct hlist_head *hhd = &ftrace_func_hash[i]; 2075 2076 hlist_for_each_entry_safe(entry, n, tmp, hhd, node) { 2077 2078 /* break up if statements for readability */ 2079 if ((flags & PROBE_TEST_FUNC) && entry->ops != ops) 2080 continue; 2081 2082 if ((flags & PROBE_TEST_DATA) && entry->data != data) 2083 continue; 2084 2085 /* do this last, since it is the most expensive */ 2086 if (glob) { 2087 kallsyms_lookup(entry->ip, NULL, NULL, 2088 NULL, str); 2089 if (!ftrace_match(str, glob, len, type)) 2090 continue; 2091 } 2092 2093 hlist_del(&entry->node); 2094 call_rcu(&entry->rcu, ftrace_free_entry_rcu); 2095 } 2096 } 2097 __disable_ftrace_function_probe(); 2098 mutex_unlock(&ftrace_lock); 2099} 2100 2101void 2102unregister_ftrace_function_probe(char *glob, struct ftrace_probe_ops *ops, 2103 void *data) 2104{ 2105 __unregister_ftrace_function_probe(glob, ops, data, 2106 PROBE_TEST_FUNC | PROBE_TEST_DATA); 2107} 2108 2109void 2110unregister_ftrace_function_probe_func(char *glob, struct ftrace_probe_ops *ops) 2111{ 2112 __unregister_ftrace_function_probe(glob, ops, NULL, PROBE_TEST_FUNC); 2113} 2114 2115void unregister_ftrace_function_probe_all(char *glob) 2116{ 2117 __unregister_ftrace_function_probe(glob, NULL, NULL, 0); 2118} 2119 2120static LIST_HEAD(ftrace_commands); 2121static DEFINE_MUTEX(ftrace_cmd_mutex); 2122 2123int register_ftrace_command(struct ftrace_func_command *cmd) 2124{ 2125 struct ftrace_func_command *p; 2126 int ret = 0; 2127 2128 mutex_lock(&ftrace_cmd_mutex); 2129 list_for_each_entry(p, &ftrace_commands, list) { 2130 if (strcmp(cmd->name, p->name) == 0) { 2131 ret = -EBUSY; 2132 goto out_unlock; 2133 } 2134 } 2135 list_add(&cmd->list, &ftrace_commands); 2136 out_unlock: 2137 mutex_unlock(&ftrace_cmd_mutex); 2138 2139 return ret; 2140} 2141 2142int unregister_ftrace_command(struct ftrace_func_command *cmd) 2143{ 2144 struct ftrace_func_command *p, *n; 2145 int ret = -ENODEV; 2146 2147 mutex_lock(&ftrace_cmd_mutex); 2148 list_for_each_entry_safe(p, n, &ftrace_commands, list) { 2149 if (strcmp(cmd->name, p->name) == 0) { 2150 ret = 0; 2151 list_del_init(&p->list); 2152 goto out_unlock; 2153 } 2154 } 2155 out_unlock: 2156 mutex_unlock(&ftrace_cmd_mutex); 2157 2158 return ret; 2159} 2160 2161static int ftrace_process_regex(char *buff, int len, int enable) 2162{ 2163 char *func, *command, *next = buff; 2164 struct ftrace_func_command *p; 2165 int ret = -EINVAL; 2166 2167 func = strsep(&next, ":"); 2168 2169 if (!next) { 2170 if (ftrace_match_records(func, len, enable)) 2171 return 0; 2172 return ret; 2173 } 2174 2175 /* command found */ 2176 2177 command = strsep(&next, ":"); 2178 2179 mutex_lock(&ftrace_cmd_mutex); 2180 list_for_each_entry(p, &ftrace_commands, list) { 2181 if (strcmp(p->name, command) == 0) { 2182 ret = p->func(func, command, next, enable); 2183 goto out_unlock; 2184 } 2185 } 2186 out_unlock: 2187 mutex_unlock(&ftrace_cmd_mutex); 2188 2189 return ret; 2190} 2191 2192static ssize_t 2193ftrace_regex_write(struct file *file, const char __user *ubuf, 2194 size_t cnt, loff_t *ppos, int enable) 2195{ 2196 struct ftrace_iterator *iter; 2197 struct trace_parser *parser; 2198 ssize_t ret, read; 2199 2200 if (!cnt) 2201 return 0; 2202 2203 mutex_lock(&ftrace_regex_lock); 2204 2205 if (file->f_mode & FMODE_READ) { 2206 struct seq_file *m = file->private_data; 2207 iter = m->private; 2208 } else 2209 iter = file->private_data; 2210 2211 parser = &iter->parser; 2212 read = trace_get_user(parser, ubuf, cnt, ppos); 2213 2214 if (read >= 0 && trace_parser_loaded(parser) && 2215 !trace_parser_cont(parser)) { 2216 ret = ftrace_process_regex(parser->buffer, 2217 parser->idx, enable); 2218 trace_parser_clear(parser); 2219 if (ret) 2220 goto out_unlock; 2221 } 2222 2223 ret = read; 2224out_unlock: 2225 mutex_unlock(&ftrace_regex_lock); 2226 2227 return ret; 2228} 2229 2230static ssize_t 2231ftrace_filter_write(struct file *file, const char __user *ubuf, 2232 size_t cnt, loff_t *ppos) 2233{ 2234 return ftrace_regex_write(file, ubuf, cnt, ppos, 1); 2235} 2236 2237static ssize_t 2238ftrace_notrace_write(struct file *file, const char __user *ubuf, 2239 size_t cnt, loff_t *ppos) 2240{ 2241 return ftrace_regex_write(file, ubuf, cnt, ppos, 0); 2242} 2243 2244static void 2245ftrace_set_regex(unsigned char *buf, int len, int reset, int enable) 2246{ 2247 if (unlikely(ftrace_disabled)) 2248 return; 2249 2250 mutex_lock(&ftrace_regex_lock); 2251 if (reset) 2252 ftrace_filter_reset(enable); 2253 if (buf) 2254 ftrace_match_records(buf, len, enable); 2255 mutex_unlock(&ftrace_regex_lock); 2256} 2257 2258/** 2259 * ftrace_set_filter - set a function to filter on in ftrace 2260 * @buf - the string that holds the function filter text. 2261 * @len - the length of the string. 2262 * @reset - non zero to reset all filters before applying this filter. 2263 * 2264 * Filters denote which functions should be enabled when tracing is enabled. 2265 * If @buf is NULL and reset is set, all functions will be enabled for tracing. 2266 */ 2267void ftrace_set_filter(unsigned char *buf, int len, int reset) 2268{ 2269 ftrace_set_regex(buf, len, reset, 1); 2270} 2271 2272/** 2273 * ftrace_set_notrace - set a function to not trace in ftrace 2274 * @buf - the string that holds the function notrace text. 2275 * @len - the length of the string. 2276 * @reset - non zero to reset all filters before applying this filter. 2277 * 2278 * Notrace Filters denote which functions should not be enabled when tracing 2279 * is enabled. If @buf is NULL and reset is set, all functions will be enabled 2280 * for tracing. 2281 */ 2282void ftrace_set_notrace(unsigned char *buf, int len, int reset) 2283{ 2284 ftrace_set_regex(buf, len, reset, 0); 2285} 2286 2287/* 2288 * command line interface to allow users to set filters on boot up. 2289 */ 2290#define FTRACE_FILTER_SIZE COMMAND_LINE_SIZE 2291static char ftrace_notrace_buf[FTRACE_FILTER_SIZE] __initdata; 2292static char ftrace_filter_buf[FTRACE_FILTER_SIZE] __initdata; 2293 2294static int __init set_ftrace_notrace(char *str) 2295{ 2296 strncpy(ftrace_notrace_buf, str, FTRACE_FILTER_SIZE); 2297 return 1; 2298} 2299__setup("ftrace_notrace=", set_ftrace_notrace); 2300 2301static int __init set_ftrace_filter(char *str) 2302{ 2303 strncpy(ftrace_filter_buf, str, FTRACE_FILTER_SIZE); 2304 return 1; 2305} 2306__setup("ftrace_filter=", set_ftrace_filter); 2307 2308#ifdef CONFIG_FUNCTION_GRAPH_TRACER 2309static char ftrace_graph_buf[FTRACE_FILTER_SIZE] __initdata; 2310static int ftrace_set_func(unsigned long *array, int *idx, char *buffer); 2311 2312static int __init set_graph_function(char *str) 2313{ 2314 strlcpy(ftrace_graph_buf, str, FTRACE_FILTER_SIZE); 2315 return 1; 2316} 2317__setup("ftrace_graph_filter=", set_graph_function); 2318 2319static void __init set_ftrace_early_graph(char *buf) 2320{ 2321 int ret; 2322 char *func; 2323 2324 while (buf) { 2325 func = strsep(&buf, ","); 2326 /* we allow only one expression at a time */ 2327 ret = ftrace_set_func(ftrace_graph_funcs, &ftrace_graph_count, 2328 func); 2329 if (ret) 2330 printk(KERN_DEBUG "ftrace: function %s not " 2331 "traceable\n", func); 2332 } 2333} 2334#endif /* CONFIG_FUNCTION_GRAPH_TRACER */ 2335 2336static void __init set_ftrace_early_filter(char *buf, int enable) 2337{ 2338 char *func; 2339 2340 while (buf) { 2341 func = strsep(&buf, ","); 2342 ftrace_set_regex(func, strlen(func), 0, enable); 2343 } 2344} 2345 2346static void __init set_ftrace_early_filters(void) 2347{ 2348 if (ftrace_filter_buf[0]) 2349 set_ftrace_early_filter(ftrace_filter_buf, 1); 2350 if (ftrace_notrace_buf[0]) 2351 set_ftrace_early_filter(ftrace_notrace_buf, 0); 2352#ifdef CONFIG_FUNCTION_GRAPH_TRACER 2353 if (ftrace_graph_buf[0]) 2354 set_ftrace_early_graph(ftrace_graph_buf); 2355#endif /* CONFIG_FUNCTION_GRAPH_TRACER */ 2356} 2357 2358static int 2359ftrace_regex_release(struct inode *inode, struct file *file, int enable) 2360{ 2361 struct seq_file *m = (struct seq_file *)file->private_data; 2362 struct ftrace_iterator *iter; 2363 struct trace_parser *parser; 2364 2365 mutex_lock(&ftrace_regex_lock); 2366 if (file->f_mode & FMODE_READ) { 2367 iter = m->private; 2368 2369 seq_release(inode, file); 2370 } else 2371 iter = file->private_data; 2372 2373 parser = &iter->parser; 2374 if (trace_parser_loaded(parser)) { 2375 parser->buffer[parser->idx] = 0; 2376 ftrace_match_records(parser->buffer, parser->idx, enable); 2377 } 2378 2379 mutex_lock(&ftrace_lock); 2380 if (ftrace_start_up && ftrace_enabled) 2381 ftrace_run_update_code(FTRACE_ENABLE_CALLS); 2382 mutex_unlock(&ftrace_lock); 2383 2384 trace_parser_put(parser); 2385 kfree(iter); 2386 2387 mutex_unlock(&ftrace_regex_lock); 2388 return 0; 2389} 2390 2391static int 2392ftrace_filter_release(struct inode *inode, struct file *file) 2393{ 2394 return ftrace_regex_release(inode, file, 1); 2395} 2396 2397static int 2398ftrace_notrace_release(struct inode *inode, struct file *file) 2399{ 2400 return ftrace_regex_release(inode, file, 0); 2401} 2402 2403static const struct file_operations ftrace_avail_fops = { 2404 .open = ftrace_avail_open, 2405 .read = seq_read, 2406 .llseek = seq_lseek, 2407 .release = seq_release_private, 2408}; 2409 2410static const struct file_operations ftrace_failures_fops = { 2411 .open = ftrace_failures_open, 2412 .read = seq_read, 2413 .llseek = seq_lseek, 2414 .release = seq_release_private, 2415}; 2416 2417static const struct file_operations ftrace_filter_fops = { 2418 .open = ftrace_filter_open, 2419 .read = seq_read, 2420 .write = ftrace_filter_write, 2421 .llseek = no_llseek, 2422 .release = ftrace_filter_release, 2423}; 2424 2425static const struct file_operations ftrace_notrace_fops = { 2426 .open = ftrace_notrace_open, 2427 .read = seq_read, 2428 .write = ftrace_notrace_write, 2429 .llseek = ftrace_regex_lseek, 2430 .release = ftrace_notrace_release, 2431}; 2432 2433#ifdef CONFIG_FUNCTION_GRAPH_TRACER 2434 2435static DEFINE_MUTEX(graph_lock); 2436 2437int ftrace_graph_count; 2438int ftrace_graph_filter_enabled; 2439unsigned long ftrace_graph_funcs[FTRACE_GRAPH_MAX_FUNCS] __read_mostly; 2440 2441static void * 2442__g_next(struct seq_file *m, loff_t *pos) 2443{ 2444 if (*pos >= ftrace_graph_count) 2445 return NULL; 2446 return &ftrace_graph_funcs[*pos]; 2447} 2448 2449static void * 2450g_next(struct seq_file *m, void *v, loff_t *pos) 2451{ 2452 (*pos)++; 2453 return __g_next(m, pos); 2454} 2455 2456static void *g_start(struct seq_file *m, loff_t *pos) 2457{ 2458 mutex_lock(&graph_lock); 2459 2460 /* Nothing, tell g_show to print all functions are enabled */ 2461 if (!ftrace_graph_filter_enabled && !*pos) 2462 return (void *)1; 2463 2464 return __g_next(m, pos); 2465} 2466 2467static void g_stop(struct seq_file *m, void *p) 2468{ 2469 mutex_unlock(&graph_lock); 2470} 2471 2472static int g_show(struct seq_file *m, void *v) 2473{ 2474 unsigned long *ptr = v; 2475 2476 if (!ptr) 2477 return 0; 2478 2479 if (ptr == (unsigned long *)1) { 2480 seq_printf(m, "#### all functions enabled ####\n"); 2481 return 0; 2482 } 2483 2484 seq_printf(m, "%ps\n", (void *)*ptr); 2485 2486 return 0; 2487} 2488 2489static const struct seq_operations ftrace_graph_seq_ops = { 2490 .start = g_start, 2491 .next = g_next, 2492 .stop = g_stop, 2493 .show = g_show, 2494}; 2495 2496static int 2497ftrace_graph_open(struct inode *inode, struct file *file) 2498{ 2499 int ret = 0; 2500 2501 if (unlikely(ftrace_disabled)) 2502 return -ENODEV; 2503 2504 mutex_lock(&graph_lock); 2505 if ((file->f_mode & FMODE_WRITE) && 2506 (file->f_flags & O_TRUNC)) { 2507 ftrace_graph_filter_enabled = 0; 2508 ftrace_graph_count = 0; 2509 memset(ftrace_graph_funcs, 0, sizeof(ftrace_graph_funcs)); 2510 } 2511 mutex_unlock(&graph_lock); 2512 2513 if (file->f_mode & FMODE_READ) 2514 ret = seq_open(file, &ftrace_graph_seq_ops); 2515 2516 return ret; 2517} 2518 2519static int 2520ftrace_graph_release(struct inode *inode, struct file *file) 2521{ 2522 if (file->f_mode & FMODE_READ) 2523 seq_release(inode, file); 2524 return 0; 2525} 2526 2527static int 2528ftrace_set_func(unsigned long *array, int *idx, char *buffer) 2529{ 2530 struct dyn_ftrace *rec; 2531 struct ftrace_page *pg; 2532 int search_len; 2533 int fail = 1; 2534 int type, not; 2535 char *search; 2536 bool exists; 2537 int i; 2538 2539 if (ftrace_disabled) 2540 return -ENODEV; 2541 2542 /* decode regex */ 2543 type = filter_parse_regex(buffer, strlen(buffer), &search, ¬); 2544 if (!not && *idx >= FTRACE_GRAPH_MAX_FUNCS) 2545 return -EBUSY; 2546 2547 search_len = strlen(search); 2548 2549 mutex_lock(&ftrace_lock); 2550 do_for_each_ftrace_rec(pg, rec) { 2551 2552 if (rec->flags & (FTRACE_FL_FAILED | FTRACE_FL_FREE)) 2553 continue; 2554 2555 if (ftrace_match_record(rec, search, search_len, type)) { 2556 /* if it is in the array */ 2557 exists = false; 2558 for (i = 0; i < *idx; i++) { 2559 if (array[i] == rec->ip) { 2560 exists = true; 2561 break; 2562 } 2563 } 2564 2565 if (!not) { 2566 fail = 0; 2567 if (!exists) { 2568 array[(*idx)++] = rec->ip; 2569 if (*idx >= FTRACE_GRAPH_MAX_FUNCS) 2570 goto out; 2571 } 2572 } else { 2573 if (exists) { 2574 array[i] = array[--(*idx)]; 2575 array[*idx] = 0; 2576 fail = 0; 2577 } 2578 } 2579 } 2580 } while_for_each_ftrace_rec(); 2581out: 2582 mutex_unlock(&ftrace_lock); 2583 2584 if (fail) 2585 return -EINVAL; 2586 2587 ftrace_graph_filter_enabled = 1; 2588 return 0; 2589} 2590 2591static ssize_t 2592ftrace_graph_write(struct file *file, const char __user *ubuf, 2593 size_t cnt, loff_t *ppos) 2594{ 2595 struct trace_parser parser; 2596 ssize_t read, ret; 2597 2598 if (!cnt) 2599 return 0; 2600 2601 mutex_lock(&graph_lock); 2602 2603 if (trace_parser_get_init(&parser, FTRACE_BUFF_MAX)) { 2604 ret = -ENOMEM; 2605 goto out_unlock; 2606 } 2607 2608 read = trace_get_user(&parser, ubuf, cnt, ppos); 2609 2610 if (read >= 0 && trace_parser_loaded((&parser))) { 2611 parser.buffer[parser.idx] = 0; 2612 2613 /* we allow only one expression at a time */ 2614 ret = ftrace_set_func(ftrace_graph_funcs, &ftrace_graph_count, 2615 parser.buffer); 2616 if (ret) 2617 goto out_free; 2618 } 2619 2620 ret = read; 2621 2622out_free: 2623 trace_parser_put(&parser); 2624out_unlock: 2625 mutex_unlock(&graph_lock); 2626 2627 return ret; 2628} 2629 2630static const struct file_operations ftrace_graph_fops = { 2631 .open = ftrace_graph_open, 2632 .read = seq_read, 2633 .write = ftrace_graph_write, 2634 .release = ftrace_graph_release, 2635}; 2636#endif /* CONFIG_FUNCTION_GRAPH_TRACER */ 2637 2638static __init int ftrace_init_dyn_debugfs(struct dentry *d_tracer) 2639{ 2640 2641 trace_create_file("available_filter_functions", 0444, 2642 d_tracer, NULL, &ftrace_avail_fops); 2643 2644 trace_create_file("failures", 0444, 2645 d_tracer, NULL, &ftrace_failures_fops); 2646 2647 trace_create_file("set_ftrace_filter", 0644, d_tracer, 2648 NULL, &ftrace_filter_fops); 2649 2650 trace_create_file("set_ftrace_notrace", 0644, d_tracer, 2651 NULL, &ftrace_notrace_fops); 2652 2653#ifdef CONFIG_FUNCTION_GRAPH_TRACER 2654 trace_create_file("set_graph_function", 0444, d_tracer, 2655 NULL, 2656 &ftrace_graph_fops); 2657#endif /* CONFIG_FUNCTION_GRAPH_TRACER */ 2658 2659 return 0; 2660} 2661 2662static int ftrace_process_locs(struct module *mod, 2663 unsigned long *start, 2664 unsigned long *end) 2665{ 2666 unsigned long *p; 2667 unsigned long addr; 2668 unsigned long flags; 2669 2670 mutex_lock(&ftrace_lock); 2671 p = start; 2672 while (p < end) { 2673 addr = ftrace_call_adjust(*p++); 2674 /* 2675 * Some architecture linkers will pad between 2676 * the different mcount_loc sections of different 2677 * object files to satisfy alignments. 2678 * Skip any NULL pointers. 2679 */ 2680 if (!addr) 2681 continue; 2682 ftrace_record_ip(addr); 2683 } 2684 2685 /* disable interrupts to prevent kstop machine */ 2686 local_irq_save(flags); 2687 ftrace_update_code(mod); 2688 local_irq_restore(flags); 2689 mutex_unlock(&ftrace_lock); 2690 2691 return 0; 2692} 2693 2694#ifdef CONFIG_MODULES 2695void ftrace_release_mod(struct module *mod) 2696{ 2697 struct dyn_ftrace *rec; 2698 struct ftrace_page *pg; 2699 2700 if (ftrace_disabled) 2701 return; 2702 2703 mutex_lock(&ftrace_lock); 2704 do_for_each_ftrace_rec(pg, rec) { 2705 if (within_module_core(rec->ip, mod)) { 2706 /* 2707 * rec->ip is changed in ftrace_free_rec() 2708 * It should not between s and e if record was freed. 2709 */ 2710 FTRACE_WARN_ON(rec->flags & FTRACE_FL_FREE); 2711 ftrace_free_rec(rec); 2712 } 2713 } while_for_each_ftrace_rec(); 2714 mutex_unlock(&ftrace_lock); 2715} 2716 2717static void ftrace_init_module(struct module *mod, 2718 unsigned long *start, unsigned long *end) 2719{ 2720 if (ftrace_disabled || start == end) 2721 return; 2722 ftrace_process_locs(mod, start, end); 2723} 2724 2725static int ftrace_module_notify(struct notifier_block *self, 2726 unsigned long val, void *data) 2727{ 2728 struct module *mod = data; 2729 2730 switch (val) { 2731 case MODULE_STATE_COMING: 2732 ftrace_init_module(mod, mod->ftrace_callsites, 2733 mod->ftrace_callsites + 2734 mod->num_ftrace_callsites); 2735 break; 2736 case MODULE_STATE_GOING: 2737 ftrace_release_mod(mod); 2738 break; 2739 } 2740 2741 return 0; 2742} 2743#else 2744static int ftrace_module_notify(struct notifier_block *self, 2745 unsigned long val, void *data) 2746{ 2747 return 0; 2748} 2749#endif /* CONFIG_MODULES */ 2750 2751struct notifier_block ftrace_module_nb = { 2752 .notifier_call = ftrace_module_notify, 2753 .priority = 0, 2754}; 2755 2756extern unsigned long __start_mcount_loc[]; 2757extern unsigned long __stop_mcount_loc[]; 2758 2759void __init ftrace_init(void) 2760{ 2761 unsigned long count, addr, flags; 2762 int ret; 2763 2764 /* Keep the ftrace pointer to the stub */ 2765 addr = (unsigned long)ftrace_stub; 2766 2767 local_irq_save(flags); 2768 ftrace_dyn_arch_init(&addr); 2769 local_irq_restore(flags); 2770 2771 /* ftrace_dyn_arch_init places the return code in addr */ 2772 if (addr) 2773 goto failed; 2774 2775 count = __stop_mcount_loc - __start_mcount_loc; 2776 2777 ret = ftrace_dyn_table_alloc(count); 2778 if (ret) 2779 goto failed; 2780 2781 last_ftrace_enabled = ftrace_enabled = 1; 2782 2783 ret = ftrace_process_locs(NULL, 2784 __start_mcount_loc, 2785 __stop_mcount_loc); 2786 2787 ret = register_module_notifier(&ftrace_module_nb); 2788 if (ret) 2789 pr_warning("Failed to register trace ftrace module notifier\n"); 2790 2791 set_ftrace_early_filters(); 2792 2793 return; 2794 failed: 2795 ftrace_disabled = 1; 2796} 2797 2798#else 2799 2800static int __init ftrace_nodyn_init(void) 2801{ 2802 ftrace_enabled = 1; 2803 return 0; 2804} 2805device_initcall(ftrace_nodyn_init); 2806 2807static inline int ftrace_init_dyn_debugfs(struct dentry *d_tracer) { return 0; } 2808static inline void ftrace_startup_enable(int command) { } 2809/* Keep as macros so we do not need to define the commands */ 2810# define ftrace_startup(command) do { } while (0) 2811# define ftrace_shutdown(command) do { } while (0) 2812# define ftrace_startup_sysctl() do { } while (0) 2813# define ftrace_shutdown_sysctl() do { } while (0) 2814#endif /* CONFIG_DYNAMIC_FTRACE */ 2815 2816static void clear_ftrace_swapper(void) 2817{ 2818 struct task_struct *p; 2819 int cpu; 2820 2821 get_online_cpus(); 2822 for_each_online_cpu(cpu) { 2823 p = idle_task(cpu); 2824 clear_tsk_trace_trace(p); 2825 } 2826 put_online_cpus(); 2827} 2828 2829static void set_ftrace_swapper(void) 2830{ 2831 struct task_struct *p; 2832 int cpu; 2833 2834 get_online_cpus(); 2835 for_each_online_cpu(cpu) { 2836 p = idle_task(cpu); 2837 set_tsk_trace_trace(p); 2838 } 2839 put_online_cpus(); 2840} 2841 2842static void clear_ftrace_pid(struct pid *pid) 2843{ 2844 struct task_struct *p; 2845 2846 rcu_read_lock(); 2847 do_each_pid_task(pid, PIDTYPE_PID, p) { 2848 clear_tsk_trace_trace(p); 2849 } while_each_pid_task(pid, PIDTYPE_PID, p); 2850 rcu_read_unlock(); 2851 2852 put_pid(pid); 2853} 2854 2855static void set_ftrace_pid(struct pid *pid) 2856{ 2857 struct task_struct *p; 2858 2859 rcu_read_lock(); 2860 do_each_pid_task(pid, PIDTYPE_PID, p) { 2861 set_tsk_trace_trace(p); 2862 } while_each_pid_task(pid, PIDTYPE_PID, p); 2863 rcu_read_unlock(); 2864} 2865 2866static void clear_ftrace_pid_task(struct pid *pid) 2867{ 2868 if (pid == ftrace_swapper_pid) 2869 clear_ftrace_swapper(); 2870 else 2871 clear_ftrace_pid(pid); 2872} 2873 2874static void set_ftrace_pid_task(struct pid *pid) 2875{ 2876 if (pid == ftrace_swapper_pid) 2877 set_ftrace_swapper(); 2878 else 2879 set_ftrace_pid(pid); 2880} 2881 2882static int ftrace_pid_add(int p) 2883{ 2884 struct pid *pid; 2885 struct ftrace_pid *fpid; 2886 int ret = -EINVAL; 2887 2888 mutex_lock(&ftrace_lock); 2889 2890 if (!p) 2891 pid = ftrace_swapper_pid; 2892 else 2893 pid = find_get_pid(p); 2894 2895 if (!pid) 2896 goto out; 2897 2898 ret = 0; 2899 2900 list_for_each_entry(fpid, &ftrace_pids, list) 2901 if (fpid->pid == pid) 2902 goto out_put; 2903 2904 ret = -ENOMEM; 2905 2906 fpid = kmalloc(sizeof(*fpid), GFP_KERNEL); 2907 if (!fpid) 2908 goto out_put; 2909 2910 list_add(&fpid->list, &ftrace_pids); 2911 fpid->pid = pid; 2912 2913 set_ftrace_pid_task(pid); 2914 2915 ftrace_update_pid_func(); 2916 ftrace_startup_enable(0); 2917 2918 mutex_unlock(&ftrace_lock); 2919 return 0; 2920 2921out_put: 2922 if (pid != ftrace_swapper_pid) 2923 put_pid(pid); 2924 2925out: 2926 mutex_unlock(&ftrace_lock); 2927 return ret; 2928} 2929 2930static void ftrace_pid_reset(void) 2931{ 2932 struct ftrace_pid *fpid, *safe; 2933 2934 mutex_lock(&ftrace_lock); 2935 list_for_each_entry_safe(fpid, safe, &ftrace_pids, list) { 2936 struct pid *pid = fpid->pid; 2937 2938 clear_ftrace_pid_task(pid); 2939 2940 list_del(&fpid->list); 2941 kfree(fpid); 2942 } 2943 2944 ftrace_update_pid_func(); 2945 ftrace_startup_enable(0); 2946 2947 mutex_unlock(&ftrace_lock); 2948} 2949 2950static void *fpid_start(struct seq_file *m, loff_t *pos) 2951{ 2952 mutex_lock(&ftrace_lock); 2953 2954 if (list_empty(&ftrace_pids) && (!*pos)) 2955 return (void *) 1; 2956 2957 return seq_list_start(&ftrace_pids, *pos); 2958} 2959 2960static void *fpid_next(struct seq_file *m, void *v, loff_t *pos) 2961{ 2962 if (v == (void *)1) 2963 return NULL; 2964 2965 return seq_list_next(v, &ftrace_pids, pos); 2966} 2967 2968static void fpid_stop(struct seq_file *m, void *p) 2969{ 2970 mutex_unlock(&ftrace_lock); 2971} 2972 2973static int fpid_show(struct seq_file *m, void *v) 2974{ 2975 const struct ftrace_pid *fpid = list_entry(v, struct ftrace_pid, list); 2976 2977 if (v == (void *)1) { 2978 seq_printf(m, "no pid\n"); 2979 return 0; 2980 } 2981 2982 if (fpid->pid == ftrace_swapper_pid) 2983 seq_printf(m, "swapper tasks\n"); 2984 else 2985 seq_printf(m, "%u\n", pid_vnr(fpid->pid)); 2986 2987 return 0; 2988} 2989 2990static const struct seq_operations ftrace_pid_sops = { 2991 .start = fpid_start, 2992 .next = fpid_next, 2993 .stop = fpid_stop, 2994 .show = fpid_show, 2995}; 2996 2997static int 2998ftrace_pid_open(struct inode *inode, struct file *file) 2999{ 3000 int ret = 0; 3001 3002 if ((file->f_mode & FMODE_WRITE) && 3003 (file->f_flags & O_TRUNC)) 3004 ftrace_pid_reset(); 3005 3006 if (file->f_mode & FMODE_READ) 3007 ret = seq_open(file, &ftrace_pid_sops); 3008 3009 return ret; 3010} 3011 3012static ssize_t 3013ftrace_pid_write(struct file *filp, const char __user *ubuf, 3014 size_t cnt, loff_t *ppos) 3015{ 3016 char buf[64], *tmp; 3017 long val; 3018 int ret; 3019 3020 if (cnt >= sizeof(buf)) 3021 return -EINVAL; 3022 3023 if (copy_from_user(&buf, ubuf, cnt)) 3024 return -EFAULT; 3025 3026 buf[cnt] = 0; 3027 3028 /* 3029 * Allow "echo > set_ftrace_pid" or "echo -n '' > set_ftrace_pid" 3030 * to clean the filter quietly. 3031 */ 3032 tmp = strstrip(buf); 3033 if (strlen(tmp) == 0) 3034 return 1; 3035 3036 ret = strict_strtol(tmp, 10, &val); 3037 if (ret < 0) 3038 return ret; 3039 3040 ret = ftrace_pid_add(val); 3041 3042 return ret ? ret : cnt; 3043} 3044 3045static int 3046ftrace_pid_release(struct inode *inode, struct file *file) 3047{ 3048 if (file->f_mode & FMODE_READ) 3049 seq_release(inode, file); 3050 3051 return 0; 3052} 3053 3054static const struct file_operations ftrace_pid_fops = { 3055 .open = ftrace_pid_open, 3056 .write = ftrace_pid_write, 3057 .read = seq_read, 3058 .llseek = seq_lseek, 3059 .release = ftrace_pid_release, 3060}; 3061 3062static __init int ftrace_init_debugfs(void) 3063{ 3064 struct dentry *d_tracer; 3065 3066 d_tracer = tracing_init_dentry(); 3067 if (!d_tracer) 3068 return 0; 3069 3070 ftrace_init_dyn_debugfs(d_tracer); 3071 3072 trace_create_file("set_ftrace_pid", 0644, d_tracer, 3073 NULL, &ftrace_pid_fops); 3074 3075 ftrace_profile_debugfs(d_tracer); 3076 3077 return 0; 3078} 3079fs_initcall(ftrace_init_debugfs); 3080 3081/** 3082 * ftrace_kill - kill ftrace 3083 * 3084 * This function should be used by panic code. It stops ftrace 3085 * but in a not so nice way. If you need to simply kill ftrace 3086 * from a non-atomic section, use ftrace_kill. 3087 */ 3088void ftrace_kill(void) 3089{ 3090 ftrace_disabled = 1; 3091 ftrace_enabled = 0; 3092 clear_ftrace_function(); 3093} 3094 3095/** 3096 * register_ftrace_function - register a function for profiling 3097 * @ops - ops structure that holds the function for profiling. 3098 * 3099 * Register a function to be called by all functions in the 3100 * kernel. 3101 * 3102 * Note: @ops->func and all the functions it calls must be labeled 3103 * with "notrace", otherwise it will go into a 3104 * recursive loop. 3105 */ 3106int register_ftrace_function(struct ftrace_ops *ops) 3107{ 3108 int ret; 3109 3110 if (unlikely(ftrace_disabled)) 3111 return -1; 3112 3113 mutex_lock(&ftrace_lock); 3114 3115 ret = __register_ftrace_function(ops); 3116 ftrace_startup(0); 3117 3118 mutex_unlock(&ftrace_lock); 3119 return ret; 3120} 3121 3122/** 3123 * unregister_ftrace_function - unregister a function for profiling. 3124 * @ops - ops structure that holds the function to unregister 3125 * 3126 * Unregister a function that was added to be called by ftrace profiling. 3127 */ 3128int unregister_ftrace_function(struct ftrace_ops *ops) 3129{ 3130 int ret; 3131 3132 mutex_lock(&ftrace_lock); 3133 ret = __unregister_ftrace_function(ops); 3134 ftrace_shutdown(0); 3135 mutex_unlock(&ftrace_lock); 3136 3137 return ret; 3138} 3139 3140int 3141ftrace_enable_sysctl(struct ctl_table *table, int write, 3142 void __user *buffer, size_t *lenp, 3143 loff_t *ppos) 3144{ 3145 int ret; 3146 3147 if (unlikely(ftrace_disabled)) 3148 return -ENODEV; 3149 3150 mutex_lock(&ftrace_lock); 3151 3152 ret = proc_dointvec(table, write, buffer, lenp, ppos); 3153 3154 if (ret || !write || (last_ftrace_enabled == !!ftrace_enabled)) 3155 goto out; 3156 3157 last_ftrace_enabled = !!ftrace_enabled; 3158 3159 if (ftrace_enabled) { 3160 3161 ftrace_startup_sysctl(); 3162 3163 /* we are starting ftrace again */ 3164 if (ftrace_list != &ftrace_list_end) { 3165 if (ftrace_list->next == &ftrace_list_end) 3166 ftrace_trace_function = ftrace_list->func; 3167 else 3168 ftrace_trace_function = ftrace_list_func; 3169 } 3170 3171 } else { 3172 /* stopping ftrace calls (just send to ftrace_stub) */ 3173 ftrace_trace_function = ftrace_stub; 3174 3175 ftrace_shutdown_sysctl(); 3176 } 3177 3178 out: 3179 mutex_unlock(&ftrace_lock); 3180 return ret; 3181} 3182 3183#ifdef CONFIG_FUNCTION_GRAPH_TRACER 3184 3185static int ftrace_graph_active; 3186static struct notifier_block ftrace_suspend_notifier; 3187 3188int ftrace_graph_entry_stub(struct ftrace_graph_ent *trace) 3189{ 3190 return 0; 3191} 3192 3193/* The callbacks that hook a function */ 3194trace_func_graph_ret_t ftrace_graph_return = 3195 (trace_func_graph_ret_t)ftrace_stub; 3196trace_func_graph_ent_t ftrace_graph_entry = ftrace_graph_entry_stub; 3197 3198/* Try to assign a return stack array on FTRACE_RETSTACK_ALLOC_SIZE tasks. */ 3199static int alloc_retstack_tasklist(struct ftrace_ret_stack **ret_stack_list) 3200{ 3201 int i; 3202 int ret = 0; 3203 unsigned long flags; 3204 int start = 0, end = FTRACE_RETSTACK_ALLOC_SIZE; 3205 struct task_struct *g, *t; 3206 3207 for (i = 0; i < FTRACE_RETSTACK_ALLOC_SIZE; i++) { 3208 ret_stack_list[i] = kmalloc(FTRACE_RETFUNC_DEPTH 3209 * sizeof(struct ftrace_ret_stack), 3210 GFP_KERNEL); 3211 if (!ret_stack_list[i]) { 3212 start = 0; 3213 end = i; 3214 ret = -ENOMEM; 3215 goto free; 3216 } 3217 } 3218 3219 read_lock_irqsave(&tasklist_lock, flags); 3220 do_each_thread(g, t) { 3221 if (start == end) { 3222 ret = -EAGAIN; 3223 goto unlock; 3224 } 3225 3226 if (t->ret_stack == NULL) { 3227 atomic_set(&t->tracing_graph_pause, 0); 3228 atomic_set(&t->trace_overrun, 0); 3229 t->curr_ret_stack = -1; 3230 /* Make sure the tasks see the -1 first: */ 3231 smp_wmb(); 3232 t->ret_stack = ret_stack_list[start++]; 3233 } 3234 } while_each_thread(g, t); 3235 3236unlock: 3237 read_unlock_irqrestore(&tasklist_lock, flags); 3238free: 3239 for (i = start; i < end; i++) 3240 kfree(ret_stack_list[i]); 3241 return ret; 3242} 3243 3244static void 3245ftrace_graph_probe_sched_switch(void *ignore, 3246 struct task_struct *prev, struct task_struct *next) 3247{ 3248 unsigned long long timestamp; 3249 int index; 3250 3251 /* 3252 * Does the user want to count the time a function was asleep. 3253 * If so, do not update the time stamps. 3254 */ 3255 if (trace_flags & TRACE_ITER_SLEEP_TIME) 3256 return; 3257 3258 timestamp = trace_clock_local(); 3259 3260 prev->ftrace_timestamp = timestamp; 3261 3262 /* only process tasks that we timestamped */ 3263 if (!next->ftrace_timestamp) 3264 return; 3265 3266 /* 3267 * Update all the counters in next to make up for the 3268 * time next was sleeping. 3269 */ 3270 timestamp -= next->ftrace_timestamp; 3271 3272 for (index = next->curr_ret_stack; index >= 0; index--) 3273 next->ret_stack[index].calltime += timestamp; 3274} 3275 3276/* Allocate a return stack for each task */ 3277static int start_graph_tracing(void) 3278{ 3279 struct ftrace_ret_stack **ret_stack_list; 3280 int ret, cpu; 3281 3282 ret_stack_list = kmalloc(FTRACE_RETSTACK_ALLOC_SIZE * 3283 sizeof(struct ftrace_ret_stack *), 3284 GFP_KERNEL); 3285 3286 if (!ret_stack_list) 3287 return -ENOMEM; 3288 3289 /* The cpu_boot init_task->ret_stack will never be freed */ 3290 for_each_online_cpu(cpu) { 3291 if (!idle_task(cpu)->ret_stack) 3292 ftrace_graph_init_task(idle_task(cpu)); 3293 } 3294 3295 do { 3296 ret = alloc_retstack_tasklist(ret_stack_list); 3297 } while (ret == -EAGAIN); 3298 3299 if (!ret) { 3300 ret = register_trace_sched_switch(ftrace_graph_probe_sched_switch, NULL); 3301 if (ret) 3302 pr_info("ftrace_graph: Couldn't activate tracepoint" 3303 " probe to kernel_sched_switch\n"); 3304 } 3305 3306 kfree(ret_stack_list); 3307 return ret; 3308} 3309 3310/* 3311 * Hibernation protection. 3312 * The state of the current task is too much unstable during 3313 * suspend/restore to disk. We want to protect against that. 3314 */ 3315static int 3316ftrace_suspend_notifier_call(struct notifier_block *bl, unsigned long state, 3317 void *unused) 3318{ 3319 switch (state) { 3320 case PM_HIBERNATION_PREPARE: 3321 pause_graph_tracing(); 3322 break; 3323 3324 case PM_POST_HIBERNATION: 3325 unpause_graph_tracing(); 3326 break; 3327 } 3328 return NOTIFY_DONE; 3329} 3330 3331int register_ftrace_graph(trace_func_graph_ret_t retfunc, 3332 trace_func_graph_ent_t entryfunc) 3333{ 3334 int ret = 0; 3335 3336 mutex_lock(&ftrace_lock); 3337 3338 /* we currently allow only one tracer registered at a time */ 3339 if (ftrace_graph_active) { 3340 ret = -EBUSY; 3341 goto out; 3342 } 3343 3344 ftrace_suspend_notifier.notifier_call = ftrace_suspend_notifier_call; 3345 register_pm_notifier(&ftrace_suspend_notifier); 3346 3347 ftrace_graph_active++; 3348 ret = start_graph_tracing(); 3349 if (ret) { 3350 ftrace_graph_active--; 3351 goto out; 3352 } 3353 3354 ftrace_graph_return = retfunc; 3355 ftrace_graph_entry = entryfunc; 3356 3357 ftrace_startup(FTRACE_START_FUNC_RET); 3358 3359out: 3360 mutex_unlock(&ftrace_lock); 3361 return ret; 3362} 3363 3364void unregister_ftrace_graph(void) 3365{ 3366 mutex_lock(&ftrace_lock); 3367 3368 if (unlikely(!ftrace_graph_active)) 3369 goto out; 3370 3371 ftrace_graph_active--; 3372 ftrace_graph_return = (trace_func_graph_ret_t)ftrace_stub; 3373 ftrace_graph_entry = ftrace_graph_entry_stub; 3374 ftrace_shutdown(FTRACE_STOP_FUNC_RET); 3375 unregister_pm_notifier(&ftrace_suspend_notifier); 3376 unregister_trace_sched_switch(ftrace_graph_probe_sched_switch, NULL); 3377 3378 out: 3379 mutex_unlock(&ftrace_lock); 3380} 3381 3382/* Allocate a return stack for newly created task */ 3383void ftrace_graph_init_task(struct task_struct *t) 3384{ 3385 /* Make sure we do not use the parent ret_stack */ 3386 t->ret_stack = NULL; 3387 t->curr_ret_stack = -1; 3388 3389 if (ftrace_graph_active) { 3390 struct ftrace_ret_stack *ret_stack; 3391 3392 ret_stack = kmalloc(FTRACE_RETFUNC_DEPTH 3393 * sizeof(struct ftrace_ret_stack), 3394 GFP_KERNEL); 3395 if (!ret_stack) 3396 return; 3397 atomic_set(&t->tracing_graph_pause, 0); 3398 atomic_set(&t->trace_overrun, 0); 3399 t->ftrace_timestamp = 0; 3400 /* make curr_ret_stack visable before we add the ret_stack */ 3401 smp_wmb(); 3402 t->ret_stack = ret_stack; 3403 } 3404} 3405 3406void ftrace_graph_exit_task(struct task_struct *t) 3407{ 3408 struct ftrace_ret_stack *ret_stack = t->ret_stack; 3409 3410 t->ret_stack = NULL; 3411 /* NULL must become visible to IRQs before we free it: */ 3412 barrier(); 3413 3414 kfree(ret_stack); 3415} 3416 3417void ftrace_graph_stop(void) 3418{ 3419 ftrace_stop(); 3420} 3421#endif 3422