1/* 2 * ring buffer based function tracer 3 * 4 * Copyright (C) 2007-2008 Steven Rostedt <srostedt@redhat.com> 5 * Copyright (C) 2008 Ingo Molnar <mingo@redhat.com> 6 * 7 * Originally taken from the RT patch by: 8 * Arnaldo Carvalho de Melo <acme@redhat.com> 9 * 10 * Based on code from the latency_tracer, that is: 11 * Copyright (C) 2004-2006 Ingo Molnar 12 * Copyright (C) 2004 William Lee Irwin III 13 */ 14#include <linux/ring_buffer.h> 15#include <generated/utsrelease.h> 16#include <linux/stacktrace.h> 17#include <linux/writeback.h> 18#include <linux/kallsyms.h> 19#include <linux/seq_file.h> 20#include <linux/smp_lock.h> 21#include <linux/notifier.h> 22#include <linux/irqflags.h> 23#include <linux/debugfs.h> 24#include <linux/pagemap.h> 25#include <linux/hardirq.h> 26#include <linux/linkage.h> 27#include <linux/uaccess.h> 28#include <linux/kprobes.h> 29#include <linux/ftrace.h> 30#include <linux/module.h> 31#include <linux/percpu.h> 32#include <linux/splice.h> 33#include <linux/kdebug.h> 34#include <linux/string.h> 35#include <linux/rwsem.h> 36#include <linux/slab.h> 37#include <linux/ctype.h> 38#include <linux/init.h> 39#include <linux/poll.h> 40#include <linux/fs.h> 41 42#include "trace.h" 43#include "trace_output.h" 44 45#define TRACE_BUFFER_FLAGS (RB_FL_OVERWRITE) 46 47/* 48 * On boot up, the ring buffer is set to the minimum size, so that 49 * we do not waste memory on systems that are not using tracing. 50 */ 51int ring_buffer_expanded; 52 53/* 54 * We need to change this state when a selftest is running. 55 * A selftest will lurk into the ring-buffer to count the 56 * entries inserted during the selftest although some concurrent 57 * insertions into the ring-buffer such as trace_printk could occurred 58 * at the same time, giving false positive or negative results. 59 */ 60static bool __read_mostly tracing_selftest_running; 61 62/* 63 * If a tracer is running, we do not want to run SELFTEST. 64 */ 65bool __read_mostly tracing_selftest_disabled; 66 67/* For tracers that don't implement custom flags */ 68static struct tracer_opt dummy_tracer_opt[] = { 69 { } 70}; 71 72static struct tracer_flags dummy_tracer_flags = { 73 .val = 0, 74 .opts = dummy_tracer_opt 75}; 76 77static int dummy_set_flag(u32 old_flags, u32 bit, int set) 78{ 79 return 0; 80} 81 82/* 83 * Kill all tracing for good (never come back). 84 * It is initialized to 1 but will turn to zero if the initialization 85 * of the tracer is successful. But that is the only place that sets 86 * this back to zero. 87 */ 88static int tracing_disabled = 1; 89 90DEFINE_PER_CPU(int, ftrace_cpu_disabled); 91 92static inline void ftrace_disable_cpu(void) 93{ 94 preempt_disable(); 95 __this_cpu_inc(ftrace_cpu_disabled); 96} 97 98static inline void ftrace_enable_cpu(void) 99{ 100 __this_cpu_dec(ftrace_cpu_disabled); 101 preempt_enable(); 102} 103 104cpumask_var_t __read_mostly tracing_buffer_mask; 105 106/* 107 * ftrace_dump_on_oops - variable to dump ftrace buffer on oops 108 * 109 * If there is an oops (or kernel panic) and the ftrace_dump_on_oops 110 * is set, then ftrace_dump is called. This will output the contents 111 * of the ftrace buffers to the console. This is very useful for 112 * capturing traces that lead to crashes and outputing it to a 113 * serial console. 114 * 115 * It is default off, but you can enable it with either specifying 116 * "ftrace_dump_on_oops" in the kernel command line, or setting 117 * /proc/sys/kernel/ftrace_dump_on_oops 118 * Set 1 if you want to dump buffers of all CPUs 119 * Set 2 if you want to dump the buffer of the CPU that triggered oops 120 */ 121 122enum ftrace_dump_mode ftrace_dump_on_oops; 123 124static int tracing_set_tracer(const char *buf); 125 126#define MAX_TRACER_SIZE 100 127static char bootup_tracer_buf[MAX_TRACER_SIZE] __initdata; 128static char *default_bootup_tracer; 129 130static int __init set_cmdline_ftrace(char *str) 131{ 132 strncpy(bootup_tracer_buf, str, MAX_TRACER_SIZE); 133 default_bootup_tracer = bootup_tracer_buf; 134 /* We are using ftrace early, expand it */ 135 ring_buffer_expanded = 1; 136 return 1; 137} 138__setup("ftrace=", set_cmdline_ftrace); 139 140static int __init set_ftrace_dump_on_oops(char *str) 141{ 142 if (*str++ != '=' || !*str) { 143 ftrace_dump_on_oops = DUMP_ALL; 144 return 1; 145 } 146 147 if (!strcmp("orig_cpu", str)) { 148 ftrace_dump_on_oops = DUMP_ORIG; 149 return 1; 150 } 151 152 return 0; 153} 154__setup("ftrace_dump_on_oops", set_ftrace_dump_on_oops); 155 156unsigned long long ns2usecs(cycle_t nsec) 157{ 158 nsec += 500; 159 do_div(nsec, 1000); 160 return nsec; 161} 162 163/* 164 * The global_trace is the descriptor that holds the tracing 165 * buffers for the live tracing. For each CPU, it contains 166 * a link list of pages that will store trace entries. The 167 * page descriptor of the pages in the memory is used to hold 168 * the link list by linking the lru item in the page descriptor 169 * to each of the pages in the buffer per CPU. 170 * 171 * For each active CPU there is a data field that holds the 172 * pages for the buffer for that CPU. Each CPU has the same number 173 * of pages allocated for its buffer. 174 */ 175static struct trace_array global_trace; 176 177static DEFINE_PER_CPU(struct trace_array_cpu, global_trace_cpu); 178 179int filter_current_check_discard(struct ring_buffer *buffer, 180 struct ftrace_event_call *call, void *rec, 181 struct ring_buffer_event *event) 182{ 183 return filter_check_discard(call, rec, buffer, event); 184} 185EXPORT_SYMBOL_GPL(filter_current_check_discard); 186 187cycle_t ftrace_now(int cpu) 188{ 189 u64 ts; 190 191 /* Early boot up does not have a buffer yet */ 192 if (!global_trace.buffer) 193 return trace_clock_local(); 194 195 ts = ring_buffer_time_stamp(global_trace.buffer, cpu); 196 ring_buffer_normalize_time_stamp(global_trace.buffer, cpu, &ts); 197 198 return ts; 199} 200 201/* 202 * The max_tr is used to snapshot the global_trace when a maximum 203 * latency is reached. Some tracers will use this to store a maximum 204 * trace while it continues examining live traces. 205 * 206 * The buffers for the max_tr are set up the same as the global_trace. 207 * When a snapshot is taken, the link list of the max_tr is swapped 208 * with the link list of the global_trace and the buffers are reset for 209 * the global_trace so the tracing can continue. 210 */ 211static struct trace_array max_tr; 212 213static DEFINE_PER_CPU(struct trace_array_cpu, max_tr_data); 214 215/* tracer_enabled is used to toggle activation of a tracer */ 216static int tracer_enabled = 1; 217 218/** 219 * tracing_is_enabled - return tracer_enabled status 220 * 221 * This function is used by other tracers to know the status 222 * of the tracer_enabled flag. Tracers may use this function 223 * to know if it should enable their features when starting 224 * up. See irqsoff tracer for an example (start_irqsoff_tracer). 225 */ 226int tracing_is_enabled(void) 227{ 228 return tracer_enabled; 229} 230 231/* 232 * trace_buf_size is the size in bytes that is allocated 233 * for a buffer. Note, the number of bytes is always rounded 234 * to page size. 235 * 236 * This number is purposely set to a low number of 16384. 237 * If the dump on oops happens, it will be much appreciated 238 * to not have to wait for all that output. Anyway this can be 239 * boot time and run time configurable. 240 */ 241#define TRACE_BUF_SIZE_DEFAULT 1441792UL /* 16384 * 88 (sizeof(entry)) */ 242 243static unsigned long trace_buf_size = TRACE_BUF_SIZE_DEFAULT; 244 245/* trace_types holds a link list of available tracers. */ 246static struct tracer *trace_types __read_mostly; 247 248/* current_trace points to the tracer that is currently active */ 249static struct tracer *current_trace __read_mostly; 250 251/* 252 * trace_types_lock is used to protect the trace_types list. 253 */ 254static DEFINE_MUTEX(trace_types_lock); 255 256/* 257 * serialize the access of the ring buffer 258 * 259 * ring buffer serializes readers, but it is low level protection. 260 * The validity of the events (which returns by ring_buffer_peek() ..etc) 261 * are not protected by ring buffer. 262 * 263 * The content of events may become garbage if we allow other process consumes 264 * these events concurrently: 265 * A) the page of the consumed events may become a normal page 266 * (not reader page) in ring buffer, and this page will be rewrited 267 * by events producer. 268 * B) The page of the consumed events may become a page for splice_read, 269 * and this page will be returned to system. 270 * 271 * These primitives allow multi process access to different cpu ring buffer 272 * concurrently. 273 * 274 * These primitives don't distinguish read-only and read-consume access. 275 * Multi read-only access are also serialized. 276 */ 277 278#ifdef CONFIG_SMP 279static DECLARE_RWSEM(all_cpu_access_lock); 280static DEFINE_PER_CPU(struct mutex, cpu_access_lock); 281 282static inline void trace_access_lock(int cpu) 283{ 284 if (cpu == TRACE_PIPE_ALL_CPU) { 285 /* gain it for accessing the whole ring buffer. */ 286 down_write(&all_cpu_access_lock); 287 } else { 288 /* gain it for accessing a cpu ring buffer. */ 289 290 /* Firstly block other trace_access_lock(TRACE_PIPE_ALL_CPU). */ 291 down_read(&all_cpu_access_lock); 292 293 /* Secondly block other access to this @cpu ring buffer. */ 294 mutex_lock(&per_cpu(cpu_access_lock, cpu)); 295 } 296} 297 298static inline void trace_access_unlock(int cpu) 299{ 300 if (cpu == TRACE_PIPE_ALL_CPU) { 301 up_write(&all_cpu_access_lock); 302 } else { 303 mutex_unlock(&per_cpu(cpu_access_lock, cpu)); 304 up_read(&all_cpu_access_lock); 305 } 306} 307 308static inline void trace_access_lock_init(void) 309{ 310 int cpu; 311 312 for_each_possible_cpu(cpu) 313 mutex_init(&per_cpu(cpu_access_lock, cpu)); 314} 315 316#else 317 318static DEFINE_MUTEX(access_lock); 319 320static inline void trace_access_lock(int cpu) 321{ 322 (void)cpu; 323 mutex_lock(&access_lock); 324} 325 326static inline void trace_access_unlock(int cpu) 327{ 328 (void)cpu; 329 mutex_unlock(&access_lock); 330} 331 332static inline void trace_access_lock_init(void) 333{ 334} 335 336#endif 337 338/* trace_wait is a waitqueue for tasks blocked on trace_poll */ 339static DECLARE_WAIT_QUEUE_HEAD(trace_wait); 340 341/* trace_flags holds trace_options default values */ 342unsigned long trace_flags = TRACE_ITER_PRINT_PARENT | TRACE_ITER_PRINTK | 343 TRACE_ITER_ANNOTATE | TRACE_ITER_CONTEXT_INFO | TRACE_ITER_SLEEP_TIME | 344 TRACE_ITER_GRAPH_TIME | TRACE_ITER_RECORD_CMD; 345 346static int trace_stop_count; 347static DEFINE_SPINLOCK(tracing_start_lock); 348 349/** 350 * trace_wake_up - wake up tasks waiting for trace input 351 * 352 * Simply wakes up any task that is blocked on the trace_wait 353 * queue. These is used with trace_poll for tasks polling the trace. 354 */ 355void trace_wake_up(void) 356{ 357 int cpu; 358 359 if (trace_flags & TRACE_ITER_BLOCK) 360 return; 361 /* 362 * The runqueue_is_locked() can fail, but this is the best we 363 * have for now: 364 */ 365 cpu = get_cpu(); 366 if (!runqueue_is_locked(cpu)) 367 wake_up(&trace_wait); 368 put_cpu(); 369} 370 371static int __init set_buf_size(char *str) 372{ 373 unsigned long buf_size; 374 375 if (!str) 376 return 0; 377 buf_size = memparse(str, &str); 378 /* nr_entries can not be zero */ 379 if (buf_size == 0) 380 return 0; 381 trace_buf_size = buf_size; 382 return 1; 383} 384__setup("trace_buf_size=", set_buf_size); 385 386static int __init set_tracing_thresh(char *str) 387{ 388 unsigned long threshhold; 389 int ret; 390 391 if (!str) 392 return 0; 393 ret = strict_strtoul(str, 0, &threshhold); 394 if (ret < 0) 395 return 0; 396 tracing_thresh = threshhold * 1000; 397 return 1; 398} 399__setup("tracing_thresh=", set_tracing_thresh); 400 401unsigned long nsecs_to_usecs(unsigned long nsecs) 402{ 403 return nsecs / 1000; 404} 405 406/* These must match the bit postions in trace_iterator_flags */ 407static const char *trace_options[] = { 408 "print-parent", 409 "sym-offset", 410 "sym-addr", 411 "verbose", 412 "raw", 413 "hex", 414 "bin", 415 "block", 416 "stacktrace", 417 "trace_printk", 418 "ftrace_preempt", 419 "branch", 420 "annotate", 421 "userstacktrace", 422 "sym-userobj", 423 "printk-msg-only", 424 "context-info", 425 "latency-format", 426 "sleep-time", 427 "graph-time", 428 "record-cmd", 429 NULL 430}; 431 432static struct { 433 u64 (*func)(void); 434 const char *name; 435} trace_clocks[] = { 436 { trace_clock_local, "local" }, 437 { trace_clock_global, "global" }, 438}; 439 440int trace_clock_id; 441 442/* 443 * trace_parser_get_init - gets the buffer for trace parser 444 */ 445int trace_parser_get_init(struct trace_parser *parser, int size) 446{ 447 memset(parser, 0, sizeof(*parser)); 448 449 parser->buffer = kmalloc(size, GFP_KERNEL); 450 if (!parser->buffer) 451 return 1; 452 453 parser->size = size; 454 return 0; 455} 456 457/* 458 * trace_parser_put - frees the buffer for trace parser 459 */ 460void trace_parser_put(struct trace_parser *parser) 461{ 462 kfree(parser->buffer); 463} 464 465/* 466 * trace_get_user - reads the user input string separated by space 467 * (matched by isspace(ch)) 468 * 469 * For each string found the 'struct trace_parser' is updated, 470 * and the function returns. 471 * 472 * Returns number of bytes read. 473 * 474 * See kernel/trace/trace.h for 'struct trace_parser' details. 475 */ 476int trace_get_user(struct trace_parser *parser, const char __user *ubuf, 477 size_t cnt, loff_t *ppos) 478{ 479 char ch; 480 size_t read = 0; 481 ssize_t ret; 482 483 if (!*ppos) 484 trace_parser_clear(parser); 485 486 ret = get_user(ch, ubuf++); 487 if (ret) 488 goto out; 489 490 read++; 491 cnt--; 492 493 /* 494 * The parser is not finished with the last write, 495 * continue reading the user input without skipping spaces. 496 */ 497 if (!parser->cont) { 498 /* skip white space */ 499 while (cnt && isspace(ch)) { 500 ret = get_user(ch, ubuf++); 501 if (ret) 502 goto out; 503 read++; 504 cnt--; 505 } 506 507 /* only spaces were written */ 508 if (isspace(ch)) { 509 *ppos += read; 510 ret = read; 511 goto out; 512 } 513 514 parser->idx = 0; 515 } 516 517 /* read the non-space input */ 518 while (cnt && !isspace(ch)) { 519 if (parser->idx < parser->size - 1) 520 parser->buffer[parser->idx++] = ch; 521 else { 522 ret = -EINVAL; 523 goto out; 524 } 525 ret = get_user(ch, ubuf++); 526 if (ret) 527 goto out; 528 read++; 529 cnt--; 530 } 531 532 /* We either got finished input or we have to wait for another call. */ 533 if (isspace(ch)) { 534 parser->buffer[parser->idx] = 0; 535 parser->cont = false; 536 } else { 537 parser->cont = true; 538 parser->buffer[parser->idx++] = ch; 539 } 540 541 *ppos += read; 542 ret = read; 543 544out: 545 return ret; 546} 547 548ssize_t trace_seq_to_user(struct trace_seq *s, char __user *ubuf, size_t cnt) 549{ 550 int len; 551 int ret; 552 553 if (!cnt) 554 return 0; 555 556 if (s->len <= s->readpos) 557 return -EBUSY; 558 559 len = s->len - s->readpos; 560 if (cnt > len) 561 cnt = len; 562 ret = copy_to_user(ubuf, s->buffer + s->readpos, cnt); 563 if (ret == cnt) 564 return -EFAULT; 565 566 cnt -= ret; 567 568 s->readpos += cnt; 569 return cnt; 570} 571 572static ssize_t trace_seq_to_buffer(struct trace_seq *s, void *buf, size_t cnt) 573{ 574 int len; 575 void *ret; 576 577 if (s->len <= s->readpos) 578 return -EBUSY; 579 580 len = s->len - s->readpos; 581 if (cnt > len) 582 cnt = len; 583 ret = memcpy(buf, s->buffer + s->readpos, cnt); 584 if (!ret) 585 return -EFAULT; 586 587 s->readpos += cnt; 588 return cnt; 589} 590 591/* 592 * ftrace_max_lock is used to protect the swapping of buffers 593 * when taking a max snapshot. The buffers themselves are 594 * protected by per_cpu spinlocks. But the action of the swap 595 * needs its own lock. 596 * 597 * This is defined as a arch_spinlock_t in order to help 598 * with performance when lockdep debugging is enabled. 599 * 600 * It is also used in other places outside the update_max_tr 601 * so it needs to be defined outside of the 602 * CONFIG_TRACER_MAX_TRACE. 603 */ 604static arch_spinlock_t ftrace_max_lock = 605 (arch_spinlock_t)__ARCH_SPIN_LOCK_UNLOCKED; 606 607unsigned long __read_mostly tracing_thresh; 608 609#ifdef CONFIG_TRACER_MAX_TRACE 610unsigned long __read_mostly tracing_max_latency; 611 612/* 613 * Copy the new maximum trace into the separate maximum-trace 614 * structure. (this way the maximum trace is permanently saved, 615 * for later retrieval via /sys/kernel/debug/tracing/latency_trace) 616 */ 617static void 618__update_max_tr(struct trace_array *tr, struct task_struct *tsk, int cpu) 619{ 620 struct trace_array_cpu *data = tr->data[cpu]; 621 struct trace_array_cpu *max_data; 622 623 max_tr.cpu = cpu; 624 max_tr.time_start = data->preempt_timestamp; 625 626 max_data = max_tr.data[cpu]; 627 max_data->saved_latency = tracing_max_latency; 628 max_data->critical_start = data->critical_start; 629 max_data->critical_end = data->critical_end; 630 631 memcpy(max_data->comm, tsk->comm, TASK_COMM_LEN); 632 max_data->pid = tsk->pid; 633 max_data->uid = task_uid(tsk); 634 max_data->nice = tsk->static_prio - 20 - MAX_RT_PRIO; 635 max_data->policy = tsk->policy; 636 max_data->rt_priority = tsk->rt_priority; 637 638 /* record this tasks comm */ 639 tracing_record_cmdline(tsk); 640} 641 642/** 643 * update_max_tr - snapshot all trace buffers from global_trace to max_tr 644 * @tr: tracer 645 * @tsk: the task with the latency 646 * @cpu: The cpu that initiated the trace. 647 * 648 * Flip the buffers between the @tr and the max_tr and record information 649 * about which task was the cause of this latency. 650 */ 651void 652update_max_tr(struct trace_array *tr, struct task_struct *tsk, int cpu) 653{ 654 struct ring_buffer *buf = tr->buffer; 655 656 if (trace_stop_count) 657 return; 658 659 WARN_ON_ONCE(!irqs_disabled()); 660 if (!current_trace->use_max_tr) { 661 WARN_ON_ONCE(1); 662 return; 663 } 664 arch_spin_lock(&ftrace_max_lock); 665 666 tr->buffer = max_tr.buffer; 667 max_tr.buffer = buf; 668 669 __update_max_tr(tr, tsk, cpu); 670 arch_spin_unlock(&ftrace_max_lock); 671} 672 673/** 674 * update_max_tr_single - only copy one trace over, and reset the rest 675 * @tr - tracer 676 * @tsk - task with the latency 677 * @cpu - the cpu of the buffer to copy. 678 * 679 * Flip the trace of a single CPU buffer between the @tr and the max_tr. 680 */ 681void 682update_max_tr_single(struct trace_array *tr, struct task_struct *tsk, int cpu) 683{ 684 int ret; 685 686 if (trace_stop_count) 687 return; 688 689 WARN_ON_ONCE(!irqs_disabled()); 690 if (!current_trace->use_max_tr) { 691 WARN_ON_ONCE(1); 692 return; 693 } 694 695 arch_spin_lock(&ftrace_max_lock); 696 697 ftrace_disable_cpu(); 698 699 ret = ring_buffer_swap_cpu(max_tr.buffer, tr->buffer, cpu); 700 701 if (ret == -EBUSY) { 702 /* 703 * We failed to swap the buffer due to a commit taking 704 * place on this CPU. We fail to record, but we reset 705 * the max trace buffer (no one writes directly to it) 706 * and flag that it failed. 707 */ 708 trace_array_printk(&max_tr, _THIS_IP_, 709 "Failed to swap buffers due to commit in progress\n"); 710 } 711 712 ftrace_enable_cpu(); 713 714 WARN_ON_ONCE(ret && ret != -EAGAIN && ret != -EBUSY); 715 716 __update_max_tr(tr, tsk, cpu); 717 arch_spin_unlock(&ftrace_max_lock); 718} 719#endif /* CONFIG_TRACER_MAX_TRACE */ 720 721/** 722 * register_tracer - register a tracer with the ftrace system. 723 * @type - the plugin for the tracer 724 * 725 * Register a new plugin tracer. 726 */ 727int register_tracer(struct tracer *type) 728__releases(kernel_lock) 729__acquires(kernel_lock) 730{ 731 struct tracer *t; 732 int ret = 0; 733 734 if (!type->name) { 735 pr_info("Tracer must have a name\n"); 736 return -1; 737 } 738 739 if (strlen(type->name) >= MAX_TRACER_SIZE) { 740 pr_info("Tracer has a name longer than %d\n", MAX_TRACER_SIZE); 741 return -1; 742 } 743 744 mutex_lock(&trace_types_lock); 745 746 tracing_selftest_running = true; 747 748 for (t = trace_types; t; t = t->next) { 749 if (strcmp(type->name, t->name) == 0) { 750 /* already found */ 751 pr_info("Tracer %s already registered\n", 752 type->name); 753 ret = -1; 754 goto out; 755 } 756 } 757 758 if (!type->set_flag) 759 type->set_flag = &dummy_set_flag; 760 if (!type->flags) 761 type->flags = &dummy_tracer_flags; 762 else 763 if (!type->flags->opts) 764 type->flags->opts = dummy_tracer_opt; 765 if (!type->wait_pipe) 766 type->wait_pipe = default_wait_pipe; 767 768 769#ifdef CONFIG_FTRACE_STARTUP_TEST 770 if (type->selftest && !tracing_selftest_disabled) { 771 struct tracer *saved_tracer = current_trace; 772 struct trace_array *tr = &global_trace; 773 774 /* 775 * Run a selftest on this tracer. 776 * Here we reset the trace buffer, and set the current 777 * tracer to be this tracer. The tracer can then run some 778 * internal tracing to verify that everything is in order. 779 * If we fail, we do not register this tracer. 780 */ 781 tracing_reset_online_cpus(tr); 782 783 current_trace = type; 784 /* the test is responsible for initializing and enabling */ 785 pr_info("Testing tracer %s: ", type->name); 786 ret = type->selftest(type, tr); 787 /* the test is responsible for resetting too */ 788 current_trace = saved_tracer; 789 if (ret) { 790 printk(KERN_CONT "FAILED!\n"); 791 goto out; 792 } 793 /* Only reset on passing, to avoid touching corrupted buffers */ 794 tracing_reset_online_cpus(tr); 795 796 printk(KERN_CONT "PASSED\n"); 797 } 798#endif 799 800 type->next = trace_types; 801 trace_types = type; 802 803 out: 804 tracing_selftest_running = false; 805 mutex_unlock(&trace_types_lock); 806 807 if (ret || !default_bootup_tracer) 808 goto out_unlock; 809 810 if (strncmp(default_bootup_tracer, type->name, MAX_TRACER_SIZE)) 811 goto out_unlock; 812 813 printk(KERN_INFO "Starting tracer '%s'\n", type->name); 814 /* Do we want this tracer to start on bootup? */ 815 tracing_set_tracer(type->name); 816 default_bootup_tracer = NULL; 817 /* disable other selftests, since this will break it. */ 818 tracing_selftest_disabled = 1; 819#ifdef CONFIG_FTRACE_STARTUP_TEST 820 printk(KERN_INFO "Disabling FTRACE selftests due to running tracer '%s'\n", 821 type->name); 822#endif 823 824 out_unlock: 825 return ret; 826} 827 828void unregister_tracer(struct tracer *type) 829{ 830 struct tracer **t; 831 832 mutex_lock(&trace_types_lock); 833 for (t = &trace_types; *t; t = &(*t)->next) { 834 if (*t == type) 835 goto found; 836 } 837 pr_info("Tracer %s not registered\n", type->name); 838 goto out; 839 840 found: 841 *t = (*t)->next; 842 843 if (type == current_trace && tracer_enabled) { 844 tracer_enabled = 0; 845 tracing_stop(); 846 if (current_trace->stop) 847 current_trace->stop(&global_trace); 848 current_trace = &nop_trace; 849 } 850out: 851 mutex_unlock(&trace_types_lock); 852} 853 854static void __tracing_reset(struct ring_buffer *buffer, int cpu) 855{ 856 ftrace_disable_cpu(); 857 ring_buffer_reset_cpu(buffer, cpu); 858 ftrace_enable_cpu(); 859} 860 861void tracing_reset(struct trace_array *tr, int cpu) 862{ 863 struct ring_buffer *buffer = tr->buffer; 864 865 ring_buffer_record_disable(buffer); 866 867 /* Make sure all commits have finished */ 868 synchronize_sched(); 869 __tracing_reset(buffer, cpu); 870 871 ring_buffer_record_enable(buffer); 872} 873 874void tracing_reset_online_cpus(struct trace_array *tr) 875{ 876 struct ring_buffer *buffer = tr->buffer; 877 int cpu; 878 879 ring_buffer_record_disable(buffer); 880 881 /* Make sure all commits have finished */ 882 synchronize_sched(); 883 884 tr->time_start = ftrace_now(tr->cpu); 885 886 for_each_online_cpu(cpu) 887 __tracing_reset(buffer, cpu); 888 889 ring_buffer_record_enable(buffer); 890} 891 892void tracing_reset_current(int cpu) 893{ 894 tracing_reset(&global_trace, cpu); 895} 896 897void tracing_reset_current_online_cpus(void) 898{ 899 tracing_reset_online_cpus(&global_trace); 900} 901 902#define SAVED_CMDLINES 128 903#define NO_CMDLINE_MAP UINT_MAX 904static unsigned map_pid_to_cmdline[PID_MAX_DEFAULT+1]; 905static unsigned map_cmdline_to_pid[SAVED_CMDLINES]; 906static char saved_cmdlines[SAVED_CMDLINES][TASK_COMM_LEN]; 907static int cmdline_idx; 908static arch_spinlock_t trace_cmdline_lock = __ARCH_SPIN_LOCK_UNLOCKED; 909 910/* temporary disable recording */ 911static atomic_t trace_record_cmdline_disabled __read_mostly; 912 913static void trace_init_cmdlines(void) 914{ 915 memset(&map_pid_to_cmdline, NO_CMDLINE_MAP, sizeof(map_pid_to_cmdline)); 916 memset(&map_cmdline_to_pid, NO_CMDLINE_MAP, sizeof(map_cmdline_to_pid)); 917 cmdline_idx = 0; 918} 919 920int is_tracing_stopped(void) 921{ 922 return trace_stop_count; 923} 924 925/** 926 * ftrace_off_permanent - disable all ftrace code permanently 927 * 928 * This should only be called when a serious anomally has 929 * been detected. This will turn off the function tracing, 930 * ring buffers, and other tracing utilites. It takes no 931 * locks and can be called from any context. 932 */ 933void ftrace_off_permanent(void) 934{ 935 tracing_disabled = 1; 936 ftrace_stop(); 937 tracing_off_permanent(); 938} 939 940/** 941 * tracing_start - quick start of the tracer 942 * 943 * If tracing is enabled but was stopped by tracing_stop, 944 * this will start the tracer back up. 945 */ 946void tracing_start(void) 947{ 948 struct ring_buffer *buffer; 949 unsigned long flags; 950 951 if (tracing_disabled) 952 return; 953 954 spin_lock_irqsave(&tracing_start_lock, flags); 955 if (--trace_stop_count) { 956 if (trace_stop_count < 0) { 957 /* Someone screwed up their debugging */ 958 WARN_ON_ONCE(1); 959 trace_stop_count = 0; 960 } 961 goto out; 962 } 963 964 /* Prevent the buffers from switching */ 965 arch_spin_lock(&ftrace_max_lock); 966 967 buffer = global_trace.buffer; 968 if (buffer) 969 ring_buffer_record_enable(buffer); 970 971 buffer = max_tr.buffer; 972 if (buffer) 973 ring_buffer_record_enable(buffer); 974 975 arch_spin_unlock(&ftrace_max_lock); 976 977 ftrace_start(); 978 out: 979 spin_unlock_irqrestore(&tracing_start_lock, flags); 980} 981 982/** 983 * tracing_stop - quick stop of the tracer 984 * 985 * Light weight way to stop tracing. Use in conjunction with 986 * tracing_start. 987 */ 988void tracing_stop(void) 989{ 990 struct ring_buffer *buffer; 991 unsigned long flags; 992 993 ftrace_stop(); 994 spin_lock_irqsave(&tracing_start_lock, flags); 995 if (trace_stop_count++) 996 goto out; 997 998 /* Prevent the buffers from switching */ 999 arch_spin_lock(&ftrace_max_lock); 1000 1001 buffer = global_trace.buffer; 1002 if (buffer) 1003 ring_buffer_record_disable(buffer); 1004 1005 buffer = max_tr.buffer; 1006 if (buffer) 1007 ring_buffer_record_disable(buffer); 1008 1009 arch_spin_unlock(&ftrace_max_lock); 1010 1011 out: 1012 spin_unlock_irqrestore(&tracing_start_lock, flags); 1013} 1014 1015void trace_stop_cmdline_recording(void); 1016 1017static void trace_save_cmdline(struct task_struct *tsk) 1018{ 1019 unsigned pid, idx; 1020 1021 if (!tsk->pid || unlikely(tsk->pid > PID_MAX_DEFAULT)) 1022 return; 1023 1024 /* 1025 * It's not the end of the world if we don't get 1026 * the lock, but we also don't want to spin 1027 * nor do we want to disable interrupts, 1028 * so if we miss here, then better luck next time. 1029 */ 1030 if (!arch_spin_trylock(&trace_cmdline_lock)) 1031 return; 1032 1033 idx = map_pid_to_cmdline[tsk->pid]; 1034 if (idx == NO_CMDLINE_MAP) { 1035 idx = (cmdline_idx + 1) % SAVED_CMDLINES; 1036 1037 /* 1038 * Check whether the cmdline buffer at idx has a pid 1039 * mapped. We are going to overwrite that entry so we 1040 * need to clear the map_pid_to_cmdline. Otherwise we 1041 * would read the new comm for the old pid. 1042 */ 1043 pid = map_cmdline_to_pid[idx]; 1044 if (pid != NO_CMDLINE_MAP) 1045 map_pid_to_cmdline[pid] = NO_CMDLINE_MAP; 1046 1047 map_cmdline_to_pid[idx] = tsk->pid; 1048 map_pid_to_cmdline[tsk->pid] = idx; 1049 1050 cmdline_idx = idx; 1051 } 1052 1053 memcpy(&saved_cmdlines[idx], tsk->comm, TASK_COMM_LEN); 1054 1055 arch_spin_unlock(&trace_cmdline_lock); 1056} 1057 1058void trace_find_cmdline(int pid, char comm[]) 1059{ 1060 unsigned map; 1061 1062 if (!pid) { 1063 strcpy(comm, "<idle>"); 1064 return; 1065 } 1066 1067 if (WARN_ON_ONCE(pid < 0)) { 1068 strcpy(comm, "<XXX>"); 1069 return; 1070 } 1071 1072 if (pid > PID_MAX_DEFAULT) { 1073 strcpy(comm, "<...>"); 1074 return; 1075 } 1076 1077 preempt_disable(); 1078 arch_spin_lock(&trace_cmdline_lock); 1079 map = map_pid_to_cmdline[pid]; 1080 if (map != NO_CMDLINE_MAP) 1081 strcpy(comm, saved_cmdlines[map]); 1082 else 1083 strcpy(comm, "<...>"); 1084 1085 arch_spin_unlock(&trace_cmdline_lock); 1086 preempt_enable(); 1087} 1088 1089void tracing_record_cmdline(struct task_struct *tsk) 1090{ 1091 if (atomic_read(&trace_record_cmdline_disabled) || !tracer_enabled || 1092 !tracing_is_on()) 1093 return; 1094 1095 trace_save_cmdline(tsk); 1096} 1097 1098void 1099tracing_generic_entry_update(struct trace_entry *entry, unsigned long flags, 1100 int pc) 1101{ 1102 struct task_struct *tsk = current; 1103 1104 entry->preempt_count = pc & 0xff; 1105 entry->pid = (tsk) ? tsk->pid : 0; 1106 entry->lock_depth = (tsk) ? tsk->lock_depth : 0; 1107 entry->flags = 1108#ifdef CONFIG_TRACE_IRQFLAGS_SUPPORT 1109 (irqs_disabled_flags(flags) ? TRACE_FLAG_IRQS_OFF : 0) | 1110#else 1111 TRACE_FLAG_IRQS_NOSUPPORT | 1112#endif 1113 ((pc & HARDIRQ_MASK) ? TRACE_FLAG_HARDIRQ : 0) | 1114 ((pc & SOFTIRQ_MASK) ? TRACE_FLAG_SOFTIRQ : 0) | 1115 (need_resched() ? TRACE_FLAG_NEED_RESCHED : 0); 1116} 1117EXPORT_SYMBOL_GPL(tracing_generic_entry_update); 1118 1119struct ring_buffer_event * 1120trace_buffer_lock_reserve(struct ring_buffer *buffer, 1121 int type, 1122 unsigned long len, 1123 unsigned long flags, int pc) 1124{ 1125 struct ring_buffer_event *event; 1126 1127 event = ring_buffer_lock_reserve(buffer, len); 1128 if (event != NULL) { 1129 struct trace_entry *ent = ring_buffer_event_data(event); 1130 1131 tracing_generic_entry_update(ent, flags, pc); 1132 ent->type = type; 1133 } 1134 1135 return event; 1136} 1137 1138static inline void 1139__trace_buffer_unlock_commit(struct ring_buffer *buffer, 1140 struct ring_buffer_event *event, 1141 unsigned long flags, int pc, 1142 int wake) 1143{ 1144 ring_buffer_unlock_commit(buffer, event); 1145 1146 ftrace_trace_stack(buffer, flags, 6, pc); 1147 ftrace_trace_userstack(buffer, flags, pc); 1148 1149 if (wake) 1150 trace_wake_up(); 1151} 1152 1153void trace_buffer_unlock_commit(struct ring_buffer *buffer, 1154 struct ring_buffer_event *event, 1155 unsigned long flags, int pc) 1156{ 1157 __trace_buffer_unlock_commit(buffer, event, flags, pc, 1); 1158} 1159 1160struct ring_buffer_event * 1161trace_current_buffer_lock_reserve(struct ring_buffer **current_rb, 1162 int type, unsigned long len, 1163 unsigned long flags, int pc) 1164{ 1165 *current_rb = global_trace.buffer; 1166 return trace_buffer_lock_reserve(*current_rb, 1167 type, len, flags, pc); 1168} 1169EXPORT_SYMBOL_GPL(trace_current_buffer_lock_reserve); 1170 1171void trace_current_buffer_unlock_commit(struct ring_buffer *buffer, 1172 struct ring_buffer_event *event, 1173 unsigned long flags, int pc) 1174{ 1175 __trace_buffer_unlock_commit(buffer, event, flags, pc, 1); 1176} 1177EXPORT_SYMBOL_GPL(trace_current_buffer_unlock_commit); 1178 1179void trace_nowake_buffer_unlock_commit(struct ring_buffer *buffer, 1180 struct ring_buffer_event *event, 1181 unsigned long flags, int pc) 1182{ 1183 __trace_buffer_unlock_commit(buffer, event, flags, pc, 0); 1184} 1185EXPORT_SYMBOL_GPL(trace_nowake_buffer_unlock_commit); 1186 1187void trace_current_buffer_discard_commit(struct ring_buffer *buffer, 1188 struct ring_buffer_event *event) 1189{ 1190 ring_buffer_discard_commit(buffer, event); 1191} 1192EXPORT_SYMBOL_GPL(trace_current_buffer_discard_commit); 1193 1194void 1195trace_function(struct trace_array *tr, 1196 unsigned long ip, unsigned long parent_ip, unsigned long flags, 1197 int pc) 1198{ 1199 struct ftrace_event_call *call = &event_function; 1200 struct ring_buffer *buffer = tr->buffer; 1201 struct ring_buffer_event *event; 1202 struct ftrace_entry *entry; 1203 1204 /* If we are reading the ring buffer, don't trace */ 1205 if (unlikely(__this_cpu_read(ftrace_cpu_disabled))) 1206 return; 1207 1208 event = trace_buffer_lock_reserve(buffer, TRACE_FN, sizeof(*entry), 1209 flags, pc); 1210 if (!event) 1211 return; 1212 entry = ring_buffer_event_data(event); 1213 entry->ip = ip; 1214 entry->parent_ip = parent_ip; 1215 1216 if (!filter_check_discard(call, entry, buffer, event)) 1217 ring_buffer_unlock_commit(buffer, event); 1218} 1219 1220void 1221ftrace(struct trace_array *tr, struct trace_array_cpu *data, 1222 unsigned long ip, unsigned long parent_ip, unsigned long flags, 1223 int pc) 1224{ 1225 if (likely(!atomic_read(&data->disabled))) 1226 trace_function(tr, ip, parent_ip, flags, pc); 1227} 1228 1229#ifdef CONFIG_STACKTRACE 1230static void __ftrace_trace_stack(struct ring_buffer *buffer, 1231 unsigned long flags, 1232 int skip, int pc) 1233{ 1234 struct ftrace_event_call *call = &event_kernel_stack; 1235 struct ring_buffer_event *event; 1236 struct stack_entry *entry; 1237 struct stack_trace trace; 1238 1239 event = trace_buffer_lock_reserve(buffer, TRACE_STACK, 1240 sizeof(*entry), flags, pc); 1241 if (!event) 1242 return; 1243 entry = ring_buffer_event_data(event); 1244 memset(&entry->caller, 0, sizeof(entry->caller)); 1245 1246 trace.nr_entries = 0; 1247 trace.max_entries = FTRACE_STACK_ENTRIES; 1248 trace.skip = skip; 1249 trace.entries = entry->caller; 1250 1251 save_stack_trace(&trace); 1252 if (!filter_check_discard(call, entry, buffer, event)) 1253 ring_buffer_unlock_commit(buffer, event); 1254} 1255 1256void ftrace_trace_stack(struct ring_buffer *buffer, unsigned long flags, 1257 int skip, int pc) 1258{ 1259 if (!(trace_flags & TRACE_ITER_STACKTRACE)) 1260 return; 1261 1262 __ftrace_trace_stack(buffer, flags, skip, pc); 1263} 1264 1265void __trace_stack(struct trace_array *tr, unsigned long flags, int skip, 1266 int pc) 1267{ 1268 __ftrace_trace_stack(tr->buffer, flags, skip, pc); 1269} 1270 1271/** 1272 * trace_dump_stack - record a stack back trace in the trace buffer 1273 */ 1274void trace_dump_stack(void) 1275{ 1276 unsigned long flags; 1277 1278 if (tracing_disabled || tracing_selftest_running) 1279 return; 1280 1281 local_save_flags(flags); 1282 1283 /* skipping 3 traces, seems to get us at the caller of this function */ 1284 __ftrace_trace_stack(global_trace.buffer, flags, 3, preempt_count()); 1285} 1286 1287void 1288ftrace_trace_userstack(struct ring_buffer *buffer, unsigned long flags, int pc) 1289{ 1290 struct ftrace_event_call *call = &event_user_stack; 1291 struct ring_buffer_event *event; 1292 struct userstack_entry *entry; 1293 struct stack_trace trace; 1294 1295 if (!(trace_flags & TRACE_ITER_USERSTACKTRACE)) 1296 return; 1297 1298 /* 1299 * NMIs can not handle page faults, even with fix ups. 1300 * The save user stack can (and often does) fault. 1301 */ 1302 if (unlikely(in_nmi())) 1303 return; 1304 1305 event = trace_buffer_lock_reserve(buffer, TRACE_USER_STACK, 1306 sizeof(*entry), flags, pc); 1307 if (!event) 1308 return; 1309 entry = ring_buffer_event_data(event); 1310 1311 entry->tgid = current->tgid; 1312 memset(&entry->caller, 0, sizeof(entry->caller)); 1313 1314 trace.nr_entries = 0; 1315 trace.max_entries = FTRACE_STACK_ENTRIES; 1316 trace.skip = 0; 1317 trace.entries = entry->caller; 1318 1319 save_stack_trace_user(&trace); 1320 if (!filter_check_discard(call, entry, buffer, event)) 1321 ring_buffer_unlock_commit(buffer, event); 1322} 1323 1324#ifdef UNUSED 1325static void __trace_userstack(struct trace_array *tr, unsigned long flags) 1326{ 1327 ftrace_trace_userstack(tr, flags, preempt_count()); 1328} 1329#endif /* UNUSED */ 1330 1331#endif /* CONFIG_STACKTRACE */ 1332 1333/** 1334 * trace_vbprintk - write binary msg to tracing buffer 1335 * 1336 */ 1337int trace_vbprintk(unsigned long ip, const char *fmt, va_list args) 1338{ 1339 static arch_spinlock_t trace_buf_lock = 1340 (arch_spinlock_t)__ARCH_SPIN_LOCK_UNLOCKED; 1341 static u32 trace_buf[TRACE_BUF_SIZE]; 1342 1343 struct ftrace_event_call *call = &event_bprint; 1344 struct ring_buffer_event *event; 1345 struct ring_buffer *buffer; 1346 struct trace_array *tr = &global_trace; 1347 struct trace_array_cpu *data; 1348 struct bprint_entry *entry; 1349 unsigned long flags; 1350 int disable; 1351 int cpu, len = 0, size, pc; 1352 1353 if (unlikely(tracing_selftest_running || tracing_disabled)) 1354 return 0; 1355 1356 /* Don't pollute graph traces with trace_vprintk internals */ 1357 pause_graph_tracing(); 1358 1359 pc = preempt_count(); 1360 preempt_disable_notrace(); 1361 cpu = raw_smp_processor_id(); 1362 data = tr->data[cpu]; 1363 1364 disable = atomic_inc_return(&data->disabled); 1365 if (unlikely(disable != 1)) 1366 goto out; 1367 1368 /* Lockdep uses trace_printk for lock tracing */ 1369 local_irq_save(flags); 1370 arch_spin_lock(&trace_buf_lock); 1371 len = vbin_printf(trace_buf, TRACE_BUF_SIZE, fmt, args); 1372 1373 if (len > TRACE_BUF_SIZE || len < 0) 1374 goto out_unlock; 1375 1376 size = sizeof(*entry) + sizeof(u32) * len; 1377 buffer = tr->buffer; 1378 event = trace_buffer_lock_reserve(buffer, TRACE_BPRINT, size, 1379 flags, pc); 1380 if (!event) 1381 goto out_unlock; 1382 entry = ring_buffer_event_data(event); 1383 entry->ip = ip; 1384 entry->fmt = fmt; 1385 1386 memcpy(entry->buf, trace_buf, sizeof(u32) * len); 1387 if (!filter_check_discard(call, entry, buffer, event)) { 1388 ring_buffer_unlock_commit(buffer, event); 1389 ftrace_trace_stack(buffer, flags, 6, pc); 1390 } 1391 1392out_unlock: 1393 arch_spin_unlock(&trace_buf_lock); 1394 local_irq_restore(flags); 1395 1396out: 1397 atomic_dec_return(&data->disabled); 1398 preempt_enable_notrace(); 1399 unpause_graph_tracing(); 1400 1401 return len; 1402} 1403EXPORT_SYMBOL_GPL(trace_vbprintk); 1404 1405int trace_array_printk(struct trace_array *tr, 1406 unsigned long ip, const char *fmt, ...) 1407{ 1408 int ret; 1409 va_list ap; 1410 1411 if (!(trace_flags & TRACE_ITER_PRINTK)) 1412 return 0; 1413 1414 va_start(ap, fmt); 1415 ret = trace_array_vprintk(tr, ip, fmt, ap); 1416 va_end(ap); 1417 return ret; 1418} 1419 1420int trace_array_vprintk(struct trace_array *tr, 1421 unsigned long ip, const char *fmt, va_list args) 1422{ 1423 static arch_spinlock_t trace_buf_lock = __ARCH_SPIN_LOCK_UNLOCKED; 1424 static char trace_buf[TRACE_BUF_SIZE]; 1425 1426 struct ftrace_event_call *call = &event_print; 1427 struct ring_buffer_event *event; 1428 struct ring_buffer *buffer; 1429 struct trace_array_cpu *data; 1430 int cpu, len = 0, size, pc; 1431 struct print_entry *entry; 1432 unsigned long irq_flags; 1433 int disable; 1434 1435 if (tracing_disabled || tracing_selftest_running) 1436 return 0; 1437 1438 pc = preempt_count(); 1439 preempt_disable_notrace(); 1440 cpu = raw_smp_processor_id(); 1441 data = tr->data[cpu]; 1442 1443 disable = atomic_inc_return(&data->disabled); 1444 if (unlikely(disable != 1)) 1445 goto out; 1446 1447 pause_graph_tracing(); 1448 raw_local_irq_save(irq_flags); 1449 arch_spin_lock(&trace_buf_lock); 1450 len = vsnprintf(trace_buf, TRACE_BUF_SIZE, fmt, args); 1451 1452 size = sizeof(*entry) + len + 1; 1453 buffer = tr->buffer; 1454 event = trace_buffer_lock_reserve(buffer, TRACE_PRINT, size, 1455 irq_flags, pc); 1456 if (!event) 1457 goto out_unlock; 1458 entry = ring_buffer_event_data(event); 1459 entry->ip = ip; 1460 1461 memcpy(&entry->buf, trace_buf, len); 1462 entry->buf[len] = '\0'; 1463 if (!filter_check_discard(call, entry, buffer, event)) { 1464 ring_buffer_unlock_commit(buffer, event); 1465 ftrace_trace_stack(buffer, irq_flags, 6, pc); 1466 } 1467 1468 out_unlock: 1469 arch_spin_unlock(&trace_buf_lock); 1470 raw_local_irq_restore(irq_flags); 1471 unpause_graph_tracing(); 1472 out: 1473 atomic_dec_return(&data->disabled); 1474 preempt_enable_notrace(); 1475 1476 return len; 1477} 1478 1479int trace_vprintk(unsigned long ip, const char *fmt, va_list args) 1480{ 1481 return trace_array_vprintk(&global_trace, ip, fmt, args); 1482} 1483EXPORT_SYMBOL_GPL(trace_vprintk); 1484 1485static void trace_iterator_increment(struct trace_iterator *iter) 1486{ 1487 /* Don't allow ftrace to trace into the ring buffers */ 1488 ftrace_disable_cpu(); 1489 1490 iter->idx++; 1491 if (iter->buffer_iter[iter->cpu]) 1492 ring_buffer_read(iter->buffer_iter[iter->cpu], NULL); 1493 1494 ftrace_enable_cpu(); 1495} 1496 1497static struct trace_entry * 1498peek_next_entry(struct trace_iterator *iter, int cpu, u64 *ts, 1499 unsigned long *lost_events) 1500{ 1501 struct ring_buffer_event *event; 1502 struct ring_buffer_iter *buf_iter = iter->buffer_iter[cpu]; 1503 1504 /* Don't allow ftrace to trace into the ring buffers */ 1505 ftrace_disable_cpu(); 1506 1507 if (buf_iter) 1508 event = ring_buffer_iter_peek(buf_iter, ts); 1509 else 1510 event = ring_buffer_peek(iter->tr->buffer, cpu, ts, 1511 lost_events); 1512 1513 ftrace_enable_cpu(); 1514 1515 return event ? ring_buffer_event_data(event) : NULL; 1516} 1517 1518static struct trace_entry * 1519__find_next_entry(struct trace_iterator *iter, int *ent_cpu, 1520 unsigned long *missing_events, u64 *ent_ts) 1521{ 1522 struct ring_buffer *buffer = iter->tr->buffer; 1523 struct trace_entry *ent, *next = NULL; 1524 unsigned long lost_events = 0, next_lost = 0; 1525 int cpu_file = iter->cpu_file; 1526 u64 next_ts = 0, ts; 1527 int next_cpu = -1; 1528 int cpu; 1529 1530 /* 1531 * If we are in a per_cpu trace file, don't bother by iterating over 1532 * all cpu and peek directly. 1533 */ 1534 if (cpu_file > TRACE_PIPE_ALL_CPU) { 1535 if (ring_buffer_empty_cpu(buffer, cpu_file)) 1536 return NULL; 1537 ent = peek_next_entry(iter, cpu_file, ent_ts, missing_events); 1538 if (ent_cpu) 1539 *ent_cpu = cpu_file; 1540 1541 return ent; 1542 } 1543 1544 for_each_tracing_cpu(cpu) { 1545 1546 if (ring_buffer_empty_cpu(buffer, cpu)) 1547 continue; 1548 1549 ent = peek_next_entry(iter, cpu, &ts, &lost_events); 1550 1551 /* 1552 * Pick the entry with the smallest timestamp: 1553 */ 1554 if (ent && (!next || ts < next_ts)) { 1555 next = ent; 1556 next_cpu = cpu; 1557 next_ts = ts; 1558 next_lost = lost_events; 1559 } 1560 } 1561 1562 if (ent_cpu) 1563 *ent_cpu = next_cpu; 1564 1565 if (ent_ts) 1566 *ent_ts = next_ts; 1567 1568 if (missing_events) 1569 *missing_events = next_lost; 1570 1571 return next; 1572} 1573 1574/* Find the next real entry, without updating the iterator itself */ 1575struct trace_entry *trace_find_next_entry(struct trace_iterator *iter, 1576 int *ent_cpu, u64 *ent_ts) 1577{ 1578 return __find_next_entry(iter, ent_cpu, NULL, ent_ts); 1579} 1580 1581/* Find the next real entry, and increment the iterator to the next entry */ 1582void *trace_find_next_entry_inc(struct trace_iterator *iter) 1583{ 1584 iter->ent = __find_next_entry(iter, &iter->cpu, 1585 &iter->lost_events, &iter->ts); 1586 1587 if (iter->ent) 1588 trace_iterator_increment(iter); 1589 1590 return iter->ent ? iter : NULL; 1591} 1592 1593static void trace_consume(struct trace_iterator *iter) 1594{ 1595 /* Don't allow ftrace to trace into the ring buffers */ 1596 ftrace_disable_cpu(); 1597 ring_buffer_consume(iter->tr->buffer, iter->cpu, &iter->ts, 1598 &iter->lost_events); 1599 ftrace_enable_cpu(); 1600} 1601 1602static void *s_next(struct seq_file *m, void *v, loff_t *pos) 1603{ 1604 struct trace_iterator *iter = m->private; 1605 int i = (int)*pos; 1606 void *ent; 1607 1608 WARN_ON_ONCE(iter->leftover); 1609 1610 (*pos)++; 1611 1612 /* can't go backwards */ 1613 if (iter->idx > i) 1614 return NULL; 1615 1616 if (iter->idx < 0) 1617 ent = trace_find_next_entry_inc(iter); 1618 else 1619 ent = iter; 1620 1621 while (ent && iter->idx < i) 1622 ent = trace_find_next_entry_inc(iter); 1623 1624 iter->pos = *pos; 1625 1626 return ent; 1627} 1628 1629void tracing_iter_reset(struct trace_iterator *iter, int cpu) 1630{ 1631 struct trace_array *tr = iter->tr; 1632 struct ring_buffer_event *event; 1633 struct ring_buffer_iter *buf_iter; 1634 unsigned long entries = 0; 1635 u64 ts; 1636 1637 tr->data[cpu]->skipped_entries = 0; 1638 1639 if (!iter->buffer_iter[cpu]) 1640 return; 1641 1642 buf_iter = iter->buffer_iter[cpu]; 1643 ring_buffer_iter_reset(buf_iter); 1644 1645 /* 1646 * We could have the case with the max latency tracers 1647 * that a reset never took place on a cpu. This is evident 1648 * by the timestamp being before the start of the buffer. 1649 */ 1650 while ((event = ring_buffer_iter_peek(buf_iter, &ts))) { 1651 if (ts >= iter->tr->time_start) 1652 break; 1653 entries++; 1654 ring_buffer_read(buf_iter, NULL); 1655 } 1656 1657 tr->data[cpu]->skipped_entries = entries; 1658} 1659 1660/* 1661 * The current tracer is copied to avoid a global locking 1662 * all around. 1663 */ 1664static void *s_start(struct seq_file *m, loff_t *pos) 1665{ 1666 struct trace_iterator *iter = m->private; 1667 static struct tracer *old_tracer; 1668 int cpu_file = iter->cpu_file; 1669 void *p = NULL; 1670 loff_t l = 0; 1671 int cpu; 1672 1673 /* copy the tracer to avoid using a global lock all around */ 1674 mutex_lock(&trace_types_lock); 1675 if (unlikely(old_tracer != current_trace && current_trace)) { 1676 old_tracer = current_trace; 1677 *iter->trace = *current_trace; 1678 } 1679 mutex_unlock(&trace_types_lock); 1680 1681 atomic_inc(&trace_record_cmdline_disabled); 1682 1683 if (*pos != iter->pos) { 1684 iter->ent = NULL; 1685 iter->cpu = 0; 1686 iter->idx = -1; 1687 1688 ftrace_disable_cpu(); 1689 1690 if (cpu_file == TRACE_PIPE_ALL_CPU) { 1691 for_each_tracing_cpu(cpu) 1692 tracing_iter_reset(iter, cpu); 1693 } else 1694 tracing_iter_reset(iter, cpu_file); 1695 1696 ftrace_enable_cpu(); 1697 1698 iter->leftover = 0; 1699 for (p = iter; p && l < *pos; p = s_next(m, p, &l)) 1700 ; 1701 1702 } else { 1703 /* 1704 * If we overflowed the seq_file before, then we want 1705 * to just reuse the trace_seq buffer again. 1706 */ 1707 if (iter->leftover) 1708 p = iter; 1709 else { 1710 l = *pos - 1; 1711 p = s_next(m, p, &l); 1712 } 1713 } 1714 1715 trace_event_read_lock(); 1716 trace_access_lock(cpu_file); 1717 return p; 1718} 1719 1720static void s_stop(struct seq_file *m, void *p) 1721{ 1722 struct trace_iterator *iter = m->private; 1723 1724 atomic_dec(&trace_record_cmdline_disabled); 1725 trace_access_unlock(iter->cpu_file); 1726 trace_event_read_unlock(); 1727} 1728 1729static void print_lat_help_header(struct seq_file *m) 1730{ 1731 seq_puts(m, "# _------=> CPU# \n"); 1732 seq_puts(m, "# / _-----=> irqs-off \n"); 1733 seq_puts(m, "# | / _----=> need-resched \n"); 1734 seq_puts(m, "# || / _---=> hardirq/softirq \n"); 1735 seq_puts(m, "# ||| / _--=> preempt-depth \n"); 1736 seq_puts(m, "# |||| /_--=> lock-depth \n"); 1737 seq_puts(m, "# |||||/ delay \n"); 1738 seq_puts(m, "# cmd pid |||||| time | caller \n"); 1739 seq_puts(m, "# \\ / |||||| \\ | / \n"); 1740} 1741 1742static void print_func_help_header(struct seq_file *m) 1743{ 1744 seq_puts(m, "# TASK-PID CPU# TIMESTAMP FUNCTION\n"); 1745 seq_puts(m, "# | | | | |\n"); 1746} 1747 1748 1749void 1750print_trace_header(struct seq_file *m, struct trace_iterator *iter) 1751{ 1752 unsigned long sym_flags = (trace_flags & TRACE_ITER_SYM_MASK); 1753 struct trace_array *tr = iter->tr; 1754 struct trace_array_cpu *data = tr->data[tr->cpu]; 1755 struct tracer *type = current_trace; 1756 unsigned long entries = 0; 1757 unsigned long total = 0; 1758 unsigned long count; 1759 const char *name = "preemption"; 1760 int cpu; 1761 1762 if (type) 1763 name = type->name; 1764 1765 1766 for_each_tracing_cpu(cpu) { 1767 count = ring_buffer_entries_cpu(tr->buffer, cpu); 1768 /* 1769 * If this buffer has skipped entries, then we hold all 1770 * entries for the trace and we need to ignore the 1771 * ones before the time stamp. 1772 */ 1773 if (tr->data[cpu]->skipped_entries) { 1774 count -= tr->data[cpu]->skipped_entries; 1775 /* total is the same as the entries */ 1776 total += count; 1777 } else 1778 total += count + 1779 ring_buffer_overrun_cpu(tr->buffer, cpu); 1780 entries += count; 1781 } 1782 1783 seq_printf(m, "# %s latency trace v1.1.5 on %s\n", 1784 name, UTS_RELEASE); 1785 seq_puts(m, "# -----------------------------------" 1786 "---------------------------------\n"); 1787 seq_printf(m, "# latency: %lu us, #%lu/%lu, CPU#%d |" 1788 " (M:%s VP:%d, KP:%d, SP:%d HP:%d", 1789 nsecs_to_usecs(data->saved_latency), 1790 entries, 1791 total, 1792 tr->cpu, 1793#if defined(CONFIG_PREEMPT_NONE) 1794 "server", 1795#elif defined(CONFIG_PREEMPT_VOLUNTARY) 1796 "desktop", 1797#elif defined(CONFIG_PREEMPT) 1798 "preempt", 1799#else 1800 "unknown", 1801#endif 1802 /* These are reserved for later use */ 1803 0, 0, 0, 0); 1804#ifdef CONFIG_SMP 1805 seq_printf(m, " #P:%d)\n", num_online_cpus()); 1806#else 1807 seq_puts(m, ")\n"); 1808#endif 1809 seq_puts(m, "# -----------------\n"); 1810 seq_printf(m, "# | task: %.16s-%d " 1811 "(uid:%d nice:%ld policy:%ld rt_prio:%ld)\n", 1812 data->comm, data->pid, data->uid, data->nice, 1813 data->policy, data->rt_priority); 1814 seq_puts(m, "# -----------------\n"); 1815 1816 if (data->critical_start) { 1817 seq_puts(m, "# => started at: "); 1818 seq_print_ip_sym(&iter->seq, data->critical_start, sym_flags); 1819 trace_print_seq(m, &iter->seq); 1820 seq_puts(m, "\n# => ended at: "); 1821 seq_print_ip_sym(&iter->seq, data->critical_end, sym_flags); 1822 trace_print_seq(m, &iter->seq); 1823 seq_puts(m, "\n#\n"); 1824 } 1825 1826 seq_puts(m, "#\n"); 1827} 1828 1829static void test_cpu_buff_start(struct trace_iterator *iter) 1830{ 1831 struct trace_seq *s = &iter->seq; 1832 1833 if (!(trace_flags & TRACE_ITER_ANNOTATE)) 1834 return; 1835 1836 if (!(iter->iter_flags & TRACE_FILE_ANNOTATE)) 1837 return; 1838 1839 if (cpumask_test_cpu(iter->cpu, iter->started)) 1840 return; 1841 1842 if (iter->tr->data[iter->cpu]->skipped_entries) 1843 return; 1844 1845 cpumask_set_cpu(iter->cpu, iter->started); 1846 1847 /* Don't print started cpu buffer for the first entry of the trace */ 1848 if (iter->idx > 1) 1849 trace_seq_printf(s, "##### CPU %u buffer started ####\n", 1850 iter->cpu); 1851} 1852 1853static enum print_line_t print_trace_fmt(struct trace_iterator *iter) 1854{ 1855 struct trace_seq *s = &iter->seq; 1856 unsigned long sym_flags = (trace_flags & TRACE_ITER_SYM_MASK); 1857 struct trace_entry *entry; 1858 struct trace_event *event; 1859 1860 entry = iter->ent; 1861 1862 test_cpu_buff_start(iter); 1863 1864 event = ftrace_find_event(entry->type); 1865 1866 if (trace_flags & TRACE_ITER_CONTEXT_INFO) { 1867 if (iter->iter_flags & TRACE_FILE_LAT_FMT) { 1868 if (!trace_print_lat_context(iter)) 1869 goto partial; 1870 } else { 1871 if (!trace_print_context(iter)) 1872 goto partial; 1873 } 1874 } 1875 1876 if (event) 1877 return event->funcs->trace(iter, sym_flags, event); 1878 1879 if (!trace_seq_printf(s, "Unknown type %d\n", entry->type)) 1880 goto partial; 1881 1882 return TRACE_TYPE_HANDLED; 1883partial: 1884 return TRACE_TYPE_PARTIAL_LINE; 1885} 1886 1887static enum print_line_t print_raw_fmt(struct trace_iterator *iter) 1888{ 1889 struct trace_seq *s = &iter->seq; 1890 struct trace_entry *entry; 1891 struct trace_event *event; 1892 1893 entry = iter->ent; 1894 1895 if (trace_flags & TRACE_ITER_CONTEXT_INFO) { 1896 if (!trace_seq_printf(s, "%d %d %llu ", 1897 entry->pid, iter->cpu, iter->ts)) 1898 goto partial; 1899 } 1900 1901 event = ftrace_find_event(entry->type); 1902 if (event) 1903 return event->funcs->raw(iter, 0, event); 1904 1905 if (!trace_seq_printf(s, "%d ?\n", entry->type)) 1906 goto partial; 1907 1908 return TRACE_TYPE_HANDLED; 1909partial: 1910 return TRACE_TYPE_PARTIAL_LINE; 1911} 1912 1913static enum print_line_t print_hex_fmt(struct trace_iterator *iter) 1914{ 1915 struct trace_seq *s = &iter->seq; 1916 unsigned char newline = '\n'; 1917 struct trace_entry *entry; 1918 struct trace_event *event; 1919 1920 entry = iter->ent; 1921 1922 if (trace_flags & TRACE_ITER_CONTEXT_INFO) { 1923 SEQ_PUT_HEX_FIELD_RET(s, entry->pid); 1924 SEQ_PUT_HEX_FIELD_RET(s, iter->cpu); 1925 SEQ_PUT_HEX_FIELD_RET(s, iter->ts); 1926 } 1927 1928 event = ftrace_find_event(entry->type); 1929 if (event) { 1930 enum print_line_t ret = event->funcs->hex(iter, 0, event); 1931 if (ret != TRACE_TYPE_HANDLED) 1932 return ret; 1933 } 1934 1935 SEQ_PUT_FIELD_RET(s, newline); 1936 1937 return TRACE_TYPE_HANDLED; 1938} 1939 1940static enum print_line_t print_bin_fmt(struct trace_iterator *iter) 1941{ 1942 struct trace_seq *s = &iter->seq; 1943 struct trace_entry *entry; 1944 struct trace_event *event; 1945 1946 entry = iter->ent; 1947 1948 if (trace_flags & TRACE_ITER_CONTEXT_INFO) { 1949 SEQ_PUT_FIELD_RET(s, entry->pid); 1950 SEQ_PUT_FIELD_RET(s, iter->cpu); 1951 SEQ_PUT_FIELD_RET(s, iter->ts); 1952 } 1953 1954 event = ftrace_find_event(entry->type); 1955 return event ? event->funcs->binary(iter, 0, event) : 1956 TRACE_TYPE_HANDLED; 1957} 1958 1959int trace_empty(struct trace_iterator *iter) 1960{ 1961 int cpu; 1962 1963 /* If we are looking at one CPU buffer, only check that one */ 1964 if (iter->cpu_file != TRACE_PIPE_ALL_CPU) { 1965 cpu = iter->cpu_file; 1966 if (iter->buffer_iter[cpu]) { 1967 if (!ring_buffer_iter_empty(iter->buffer_iter[cpu])) 1968 return 0; 1969 } else { 1970 if (!ring_buffer_empty_cpu(iter->tr->buffer, cpu)) 1971 return 0; 1972 } 1973 return 1; 1974 } 1975 1976 for_each_tracing_cpu(cpu) { 1977 if (iter->buffer_iter[cpu]) { 1978 if (!ring_buffer_iter_empty(iter->buffer_iter[cpu])) 1979 return 0; 1980 } else { 1981 if (!ring_buffer_empty_cpu(iter->tr->buffer, cpu)) 1982 return 0; 1983 } 1984 } 1985 1986 return 1; 1987} 1988 1989/* Called with trace_event_read_lock() held. */ 1990enum print_line_t print_trace_line(struct trace_iterator *iter) 1991{ 1992 enum print_line_t ret; 1993 1994 if (iter->lost_events) 1995 trace_seq_printf(&iter->seq, "CPU:%d [LOST %lu EVENTS]\n", 1996 iter->cpu, iter->lost_events); 1997 1998 if (iter->trace && iter->trace->print_line) { 1999 ret = iter->trace->print_line(iter); 2000 if (ret != TRACE_TYPE_UNHANDLED) 2001 return ret; 2002 } 2003 2004 if (iter->ent->type == TRACE_BPRINT && 2005 trace_flags & TRACE_ITER_PRINTK && 2006 trace_flags & TRACE_ITER_PRINTK_MSGONLY) 2007 return trace_print_bprintk_msg_only(iter); 2008 2009 if (iter->ent->type == TRACE_PRINT && 2010 trace_flags & TRACE_ITER_PRINTK && 2011 trace_flags & TRACE_ITER_PRINTK_MSGONLY) 2012 return trace_print_printk_msg_only(iter); 2013 2014 if (trace_flags & TRACE_ITER_BIN) 2015 return print_bin_fmt(iter); 2016 2017 if (trace_flags & TRACE_ITER_HEX) 2018 return print_hex_fmt(iter); 2019 2020 if (trace_flags & TRACE_ITER_RAW) 2021 return print_raw_fmt(iter); 2022 2023 return print_trace_fmt(iter); 2024} 2025 2026void trace_default_header(struct seq_file *m) 2027{ 2028 struct trace_iterator *iter = m->private; 2029 2030 if (iter->iter_flags & TRACE_FILE_LAT_FMT) { 2031 /* print nothing if the buffers are empty */ 2032 if (trace_empty(iter)) 2033 return; 2034 print_trace_header(m, iter); 2035 if (!(trace_flags & TRACE_ITER_VERBOSE)) 2036 print_lat_help_header(m); 2037 } else { 2038 if (!(trace_flags & TRACE_ITER_VERBOSE)) 2039 print_func_help_header(m); 2040 } 2041} 2042 2043static int s_show(struct seq_file *m, void *v) 2044{ 2045 struct trace_iterator *iter = v; 2046 int ret; 2047 2048 if (iter->ent == NULL) { 2049 if (iter->tr) { 2050 seq_printf(m, "# tracer: %s\n", iter->trace->name); 2051 seq_puts(m, "#\n"); 2052 } 2053 if (iter->trace && iter->trace->print_header) 2054 iter->trace->print_header(m); 2055 else 2056 trace_default_header(m); 2057 2058 } else if (iter->leftover) { 2059 /* 2060 * If we filled the seq_file buffer earlier, we 2061 * want to just show it now. 2062 */ 2063 ret = trace_print_seq(m, &iter->seq); 2064 2065 /* ret should this time be zero, but you never know */ 2066 iter->leftover = ret; 2067 2068 } else { 2069 print_trace_line(iter); 2070 ret = trace_print_seq(m, &iter->seq); 2071 /* 2072 * If we overflow the seq_file buffer, then it will 2073 * ask us for this data again at start up. 2074 * Use that instead. 2075 * ret is 0 if seq_file write succeeded. 2076 * -1 otherwise. 2077 */ 2078 iter->leftover = ret; 2079 } 2080 2081 return 0; 2082} 2083 2084static const struct seq_operations tracer_seq_ops = { 2085 .start = s_start, 2086 .next = s_next, 2087 .stop = s_stop, 2088 .show = s_show, 2089}; 2090 2091static struct trace_iterator * 2092__tracing_open(struct inode *inode, struct file *file) 2093{ 2094 long cpu_file = (long) inode->i_private; 2095 void *fail_ret = ERR_PTR(-ENOMEM); 2096 struct trace_iterator *iter; 2097 struct seq_file *m; 2098 int cpu, ret; 2099 2100 if (tracing_disabled) 2101 return ERR_PTR(-ENODEV); 2102 2103 iter = kzalloc(sizeof(*iter), GFP_KERNEL); 2104 if (!iter) 2105 return ERR_PTR(-ENOMEM); 2106 2107 /* 2108 * We make a copy of the current tracer to avoid concurrent 2109 * changes on it while we are reading. 2110 */ 2111 mutex_lock(&trace_types_lock); 2112 iter->trace = kzalloc(sizeof(*iter->trace), GFP_KERNEL); 2113 if (!iter->trace) 2114 goto fail; 2115 2116 if (current_trace) 2117 *iter->trace = *current_trace; 2118 2119 if (!zalloc_cpumask_var(&iter->started, GFP_KERNEL)) 2120 goto fail; 2121 2122 if (current_trace && current_trace->print_max) 2123 iter->tr = &max_tr; 2124 else 2125 iter->tr = &global_trace; 2126 iter->pos = -1; 2127 mutex_init(&iter->mutex); 2128 iter->cpu_file = cpu_file; 2129 2130 /* Notify the tracer early; before we stop tracing. */ 2131 if (iter->trace && iter->trace->open) 2132 iter->trace->open(iter); 2133 2134 /* Annotate start of buffers if we had overruns */ 2135 if (ring_buffer_overruns(iter->tr->buffer)) 2136 iter->iter_flags |= TRACE_FILE_ANNOTATE; 2137 2138 /* stop the trace while dumping */ 2139 tracing_stop(); 2140 2141 if (iter->cpu_file == TRACE_PIPE_ALL_CPU) { 2142 for_each_tracing_cpu(cpu) { 2143 iter->buffer_iter[cpu] = 2144 ring_buffer_read_prepare(iter->tr->buffer, cpu); 2145 } 2146 ring_buffer_read_prepare_sync(); 2147 for_each_tracing_cpu(cpu) { 2148 ring_buffer_read_start(iter->buffer_iter[cpu]); 2149 tracing_iter_reset(iter, cpu); 2150 } 2151 } else { 2152 cpu = iter->cpu_file; 2153 iter->buffer_iter[cpu] = 2154 ring_buffer_read_prepare(iter->tr->buffer, cpu); 2155 ring_buffer_read_prepare_sync(); 2156 ring_buffer_read_start(iter->buffer_iter[cpu]); 2157 tracing_iter_reset(iter, cpu); 2158 } 2159 2160 ret = seq_open(file, &tracer_seq_ops); 2161 if (ret < 0) { 2162 fail_ret = ERR_PTR(ret); 2163 goto fail_buffer; 2164 } 2165 2166 m = file->private_data; 2167 m->private = iter; 2168 2169 mutex_unlock(&trace_types_lock); 2170 2171 return iter; 2172 2173 fail_buffer: 2174 for_each_tracing_cpu(cpu) { 2175 if (iter->buffer_iter[cpu]) 2176 ring_buffer_read_finish(iter->buffer_iter[cpu]); 2177 } 2178 free_cpumask_var(iter->started); 2179 tracing_start(); 2180 fail: 2181 mutex_unlock(&trace_types_lock); 2182 kfree(iter->trace); 2183 kfree(iter); 2184 2185 return fail_ret; 2186} 2187 2188int tracing_open_generic(struct inode *inode, struct file *filp) 2189{ 2190 if (tracing_disabled) 2191 return -ENODEV; 2192 2193 filp->private_data = inode->i_private; 2194 return 0; 2195} 2196 2197static int tracing_release(struct inode *inode, struct file *file) 2198{ 2199 struct seq_file *m = (struct seq_file *)file->private_data; 2200 struct trace_iterator *iter; 2201 int cpu; 2202 2203 if (!(file->f_mode & FMODE_READ)) 2204 return 0; 2205 2206 iter = m->private; 2207 2208 mutex_lock(&trace_types_lock); 2209 for_each_tracing_cpu(cpu) { 2210 if (iter->buffer_iter[cpu]) 2211 ring_buffer_read_finish(iter->buffer_iter[cpu]); 2212 } 2213 2214 if (iter->trace && iter->trace->close) 2215 iter->trace->close(iter); 2216 2217 /* reenable tracing if it was previously enabled */ 2218 tracing_start(); 2219 mutex_unlock(&trace_types_lock); 2220 2221 seq_release(inode, file); 2222 mutex_destroy(&iter->mutex); 2223 free_cpumask_var(iter->started); 2224 kfree(iter->trace); 2225 kfree(iter); 2226 return 0; 2227} 2228 2229static int tracing_open(struct inode *inode, struct file *file) 2230{ 2231 struct trace_iterator *iter; 2232 int ret = 0; 2233 2234 /* If this file was open for write, then erase contents */ 2235 if ((file->f_mode & FMODE_WRITE) && 2236 (file->f_flags & O_TRUNC)) { 2237 long cpu = (long) inode->i_private; 2238 2239 if (cpu == TRACE_PIPE_ALL_CPU) 2240 tracing_reset_online_cpus(&global_trace); 2241 else 2242 tracing_reset(&global_trace, cpu); 2243 } 2244 2245 if (file->f_mode & FMODE_READ) { 2246 iter = __tracing_open(inode, file); 2247 if (IS_ERR(iter)) 2248 ret = PTR_ERR(iter); 2249 else if (trace_flags & TRACE_ITER_LATENCY_FMT) 2250 iter->iter_flags |= TRACE_FILE_LAT_FMT; 2251 } 2252 return ret; 2253} 2254 2255static void * 2256t_next(struct seq_file *m, void *v, loff_t *pos) 2257{ 2258 struct tracer *t = v; 2259 2260 (*pos)++; 2261 2262 if (t) 2263 t = t->next; 2264 2265 return t; 2266} 2267 2268static void *t_start(struct seq_file *m, loff_t *pos) 2269{ 2270 struct tracer *t; 2271 loff_t l = 0; 2272 2273 mutex_lock(&trace_types_lock); 2274 for (t = trace_types; t && l < *pos; t = t_next(m, t, &l)) 2275 ; 2276 2277 return t; 2278} 2279 2280static void t_stop(struct seq_file *m, void *p) 2281{ 2282 mutex_unlock(&trace_types_lock); 2283} 2284 2285static int t_show(struct seq_file *m, void *v) 2286{ 2287 struct tracer *t = v; 2288 2289 if (!t) 2290 return 0; 2291 2292 seq_printf(m, "%s", t->name); 2293 if (t->next) 2294 seq_putc(m, ' '); 2295 else 2296 seq_putc(m, '\n'); 2297 2298 return 0; 2299} 2300 2301static const struct seq_operations show_traces_seq_ops = { 2302 .start = t_start, 2303 .next = t_next, 2304 .stop = t_stop, 2305 .show = t_show, 2306}; 2307 2308static int show_traces_open(struct inode *inode, struct file *file) 2309{ 2310 if (tracing_disabled) 2311 return -ENODEV; 2312 2313 return seq_open(file, &show_traces_seq_ops); 2314} 2315 2316static ssize_t 2317tracing_write_stub(struct file *filp, const char __user *ubuf, 2318 size_t count, loff_t *ppos) 2319{ 2320 return count; 2321} 2322 2323static loff_t tracing_seek(struct file *file, loff_t offset, int origin) 2324{ 2325 if (file->f_mode & FMODE_READ) 2326 return seq_lseek(file, offset, origin); 2327 else 2328 return 0; 2329} 2330 2331static const struct file_operations tracing_fops = { 2332 .open = tracing_open, 2333 .read = seq_read, 2334 .write = tracing_write_stub, 2335 .llseek = tracing_seek, 2336 .release = tracing_release, 2337}; 2338 2339static const struct file_operations show_traces_fops = { 2340 .open = show_traces_open, 2341 .read = seq_read, 2342 .release = seq_release, 2343 .llseek = seq_lseek, 2344}; 2345 2346/* 2347 * Only trace on a CPU if the bitmask is set: 2348 */ 2349static cpumask_var_t tracing_cpumask; 2350 2351/* 2352 * The tracer itself will not take this lock, but still we want 2353 * to provide a consistent cpumask to user-space: 2354 */ 2355static DEFINE_MUTEX(tracing_cpumask_update_lock); 2356 2357/* 2358 * Temporary storage for the character representation of the 2359 * CPU bitmask (and one more byte for the newline): 2360 */ 2361static char mask_str[NR_CPUS + 1]; 2362 2363static ssize_t 2364tracing_cpumask_read(struct file *filp, char __user *ubuf, 2365 size_t count, loff_t *ppos) 2366{ 2367 int len; 2368 2369 mutex_lock(&tracing_cpumask_update_lock); 2370 2371 len = cpumask_scnprintf(mask_str, count, tracing_cpumask); 2372 if (count - len < 2) { 2373 count = -EINVAL; 2374 goto out_err; 2375 } 2376 len += sprintf(mask_str + len, "\n"); 2377 count = simple_read_from_buffer(ubuf, count, ppos, mask_str, NR_CPUS+1); 2378 2379out_err: 2380 mutex_unlock(&tracing_cpumask_update_lock); 2381 2382 return count; 2383} 2384 2385static ssize_t 2386tracing_cpumask_write(struct file *filp, const char __user *ubuf, 2387 size_t count, loff_t *ppos) 2388{ 2389 int err, cpu; 2390 cpumask_var_t tracing_cpumask_new; 2391 2392 if (!alloc_cpumask_var(&tracing_cpumask_new, GFP_KERNEL)) 2393 return -ENOMEM; 2394 2395 err = cpumask_parse_user(ubuf, count, tracing_cpumask_new); 2396 if (err) 2397 goto err_unlock; 2398 2399 mutex_lock(&tracing_cpumask_update_lock); 2400 2401 local_irq_disable(); 2402 arch_spin_lock(&ftrace_max_lock); 2403 for_each_tracing_cpu(cpu) { 2404 /* 2405 * Increase/decrease the disabled counter if we are 2406 * about to flip a bit in the cpumask: 2407 */ 2408 if (cpumask_test_cpu(cpu, tracing_cpumask) && 2409 !cpumask_test_cpu(cpu, tracing_cpumask_new)) { 2410 atomic_inc(&global_trace.data[cpu]->disabled); 2411 } 2412 if (!cpumask_test_cpu(cpu, tracing_cpumask) && 2413 cpumask_test_cpu(cpu, tracing_cpumask_new)) { 2414 atomic_dec(&global_trace.data[cpu]->disabled); 2415 } 2416 } 2417 arch_spin_unlock(&ftrace_max_lock); 2418 local_irq_enable(); 2419 2420 cpumask_copy(tracing_cpumask, tracing_cpumask_new); 2421 2422 mutex_unlock(&tracing_cpumask_update_lock); 2423 free_cpumask_var(tracing_cpumask_new); 2424 2425 return count; 2426 2427err_unlock: 2428 free_cpumask_var(tracing_cpumask_new); 2429 2430 return err; 2431} 2432 2433static const struct file_operations tracing_cpumask_fops = { 2434 .open = tracing_open_generic, 2435 .read = tracing_cpumask_read, 2436 .write = tracing_cpumask_write, 2437 .llseek = generic_file_llseek, 2438}; 2439 2440static int tracing_trace_options_show(struct seq_file *m, void *v) 2441{ 2442 struct tracer_opt *trace_opts; 2443 u32 tracer_flags; 2444 int i; 2445 2446 mutex_lock(&trace_types_lock); 2447 tracer_flags = current_trace->flags->val; 2448 trace_opts = current_trace->flags->opts; 2449 2450 for (i = 0; trace_options[i]; i++) { 2451 if (trace_flags & (1 << i)) 2452 seq_printf(m, "%s\n", trace_options[i]); 2453 else 2454 seq_printf(m, "no%s\n", trace_options[i]); 2455 } 2456 2457 for (i = 0; trace_opts[i].name; i++) { 2458 if (tracer_flags & trace_opts[i].bit) 2459 seq_printf(m, "%s\n", trace_opts[i].name); 2460 else 2461 seq_printf(m, "no%s\n", trace_opts[i].name); 2462 } 2463 mutex_unlock(&trace_types_lock); 2464 2465 return 0; 2466} 2467 2468static int __set_tracer_option(struct tracer *trace, 2469 struct tracer_flags *tracer_flags, 2470 struct tracer_opt *opts, int neg) 2471{ 2472 int ret; 2473 2474 ret = trace->set_flag(tracer_flags->val, opts->bit, !neg); 2475 if (ret) 2476 return ret; 2477 2478 if (neg) 2479 tracer_flags->val &= ~opts->bit; 2480 else 2481 tracer_flags->val |= opts->bit; 2482 return 0; 2483} 2484 2485/* Try to assign a tracer specific option */ 2486static int set_tracer_option(struct tracer *trace, char *cmp, int neg) 2487{ 2488 struct tracer_flags *tracer_flags = trace->flags; 2489 struct tracer_opt *opts = NULL; 2490 int i; 2491 2492 for (i = 0; tracer_flags->opts[i].name; i++) { 2493 opts = &tracer_flags->opts[i]; 2494 2495 if (strcmp(cmp, opts->name) == 0) 2496 return __set_tracer_option(trace, trace->flags, 2497 opts, neg); 2498 } 2499 2500 return -EINVAL; 2501} 2502 2503static void set_tracer_flags(unsigned int mask, int enabled) 2504{ 2505 /* do nothing if flag is already set */ 2506 if (!!(trace_flags & mask) == !!enabled) 2507 return; 2508 2509 if (enabled) 2510 trace_flags |= mask; 2511 else 2512 trace_flags &= ~mask; 2513 2514 if (mask == TRACE_ITER_RECORD_CMD) 2515 trace_event_enable_cmd_record(enabled); 2516} 2517 2518static ssize_t 2519tracing_trace_options_write(struct file *filp, const char __user *ubuf, 2520 size_t cnt, loff_t *ppos) 2521{ 2522 char buf[64]; 2523 char *cmp; 2524 int neg = 0; 2525 int ret; 2526 int i; 2527 2528 if (cnt >= sizeof(buf)) 2529 return -EINVAL; 2530 2531 if (copy_from_user(&buf, ubuf, cnt)) 2532 return -EFAULT; 2533 2534 buf[cnt] = 0; 2535 cmp = strstrip(buf); 2536 2537 if (strncmp(cmp, "no", 2) == 0) { 2538 neg = 1; 2539 cmp += 2; 2540 } 2541 2542 for (i = 0; trace_options[i]; i++) { 2543 if (strcmp(cmp, trace_options[i]) == 0) { 2544 set_tracer_flags(1 << i, !neg); 2545 break; 2546 } 2547 } 2548 2549 /* If no option could be set, test the specific tracer options */ 2550 if (!trace_options[i]) { 2551 mutex_lock(&trace_types_lock); 2552 ret = set_tracer_option(current_trace, cmp, neg); 2553 mutex_unlock(&trace_types_lock); 2554 if (ret) 2555 return ret; 2556 } 2557 2558 *ppos += cnt; 2559 2560 return cnt; 2561} 2562 2563static int tracing_trace_options_open(struct inode *inode, struct file *file) 2564{ 2565 if (tracing_disabled) 2566 return -ENODEV; 2567 return single_open(file, tracing_trace_options_show, NULL); 2568} 2569 2570static const struct file_operations tracing_iter_fops = { 2571 .open = tracing_trace_options_open, 2572 .read = seq_read, 2573 .llseek = seq_lseek, 2574 .release = single_release, 2575 .write = tracing_trace_options_write, 2576}; 2577 2578static const char readme_msg[] = 2579 "tracing mini-HOWTO:\n\n" 2580 "# mount -t debugfs nodev /sys/kernel/debug\n\n" 2581 "# cat /sys/kernel/debug/tracing/available_tracers\n" 2582 "wakeup preemptirqsoff preemptoff irqsoff function sched_switch nop\n\n" 2583 "# cat /sys/kernel/debug/tracing/current_tracer\n" 2584 "nop\n" 2585 "# echo sched_switch > /sys/kernel/debug/tracing/current_tracer\n" 2586 "# cat /sys/kernel/debug/tracing/current_tracer\n" 2587 "sched_switch\n" 2588 "# cat /sys/kernel/debug/tracing/trace_options\n" 2589 "noprint-parent nosym-offset nosym-addr noverbose\n" 2590 "# echo print-parent > /sys/kernel/debug/tracing/trace_options\n" 2591 "# echo 1 > /sys/kernel/debug/tracing/tracing_enabled\n" 2592 "# cat /sys/kernel/debug/tracing/trace > /tmp/trace.txt\n" 2593 "# echo 0 > /sys/kernel/debug/tracing/tracing_enabled\n" 2594; 2595 2596static ssize_t 2597tracing_readme_read(struct file *filp, char __user *ubuf, 2598 size_t cnt, loff_t *ppos) 2599{ 2600 return simple_read_from_buffer(ubuf, cnt, ppos, 2601 readme_msg, strlen(readme_msg)); 2602} 2603 2604static const struct file_operations tracing_readme_fops = { 2605 .open = tracing_open_generic, 2606 .read = tracing_readme_read, 2607 .llseek = generic_file_llseek, 2608}; 2609 2610static ssize_t 2611tracing_saved_cmdlines_read(struct file *file, char __user *ubuf, 2612 size_t cnt, loff_t *ppos) 2613{ 2614 char *buf_comm; 2615 char *file_buf; 2616 char *buf; 2617 int len = 0; 2618 int pid; 2619 int i; 2620 2621 file_buf = kmalloc(SAVED_CMDLINES*(16+TASK_COMM_LEN), GFP_KERNEL); 2622 if (!file_buf) 2623 return -ENOMEM; 2624 2625 buf_comm = kmalloc(TASK_COMM_LEN, GFP_KERNEL); 2626 if (!buf_comm) { 2627 kfree(file_buf); 2628 return -ENOMEM; 2629 } 2630 2631 buf = file_buf; 2632 2633 for (i = 0; i < SAVED_CMDLINES; i++) { 2634 int r; 2635 2636 pid = map_cmdline_to_pid[i]; 2637 if (pid == -1 || pid == NO_CMDLINE_MAP) 2638 continue; 2639 2640 trace_find_cmdline(pid, buf_comm); 2641 r = sprintf(buf, "%d %s\n", pid, buf_comm); 2642 buf += r; 2643 len += r; 2644 } 2645 2646 len = simple_read_from_buffer(ubuf, cnt, ppos, 2647 file_buf, len); 2648 2649 kfree(file_buf); 2650 kfree(buf_comm); 2651 2652 return len; 2653} 2654 2655static const struct file_operations tracing_saved_cmdlines_fops = { 2656 .open = tracing_open_generic, 2657 .read = tracing_saved_cmdlines_read, 2658 .llseek = generic_file_llseek, 2659}; 2660 2661static ssize_t 2662tracing_ctrl_read(struct file *filp, char __user *ubuf, 2663 size_t cnt, loff_t *ppos) 2664{ 2665 char buf[64]; 2666 int r; 2667 2668 r = sprintf(buf, "%u\n", tracer_enabled); 2669 return simple_read_from_buffer(ubuf, cnt, ppos, buf, r); 2670} 2671 2672static ssize_t 2673tracing_ctrl_write(struct file *filp, const char __user *ubuf, 2674 size_t cnt, loff_t *ppos) 2675{ 2676 struct trace_array *tr = filp->private_data; 2677 char buf[64]; 2678 unsigned long val; 2679 int ret; 2680 2681 if (cnt >= sizeof(buf)) 2682 return -EINVAL; 2683 2684 if (copy_from_user(&buf, ubuf, cnt)) 2685 return -EFAULT; 2686 2687 buf[cnt] = 0; 2688 2689 ret = strict_strtoul(buf, 10, &val); 2690 if (ret < 0) 2691 return ret; 2692 2693 val = !!val; 2694 2695 mutex_lock(&trace_types_lock); 2696 if (tracer_enabled ^ val) { 2697 if (val) { 2698 tracer_enabled = 1; 2699 if (current_trace->start) 2700 current_trace->start(tr); 2701 tracing_start(); 2702 } else { 2703 tracer_enabled = 0; 2704 tracing_stop(); 2705 if (current_trace->stop) 2706 current_trace->stop(tr); 2707 } 2708 } 2709 mutex_unlock(&trace_types_lock); 2710 2711 *ppos += cnt; 2712 2713 return cnt; 2714} 2715 2716static ssize_t 2717tracing_set_trace_read(struct file *filp, char __user *ubuf, 2718 size_t cnt, loff_t *ppos) 2719{ 2720 char buf[MAX_TRACER_SIZE+2]; 2721 int r; 2722 2723 mutex_lock(&trace_types_lock); 2724 if (current_trace) 2725 r = sprintf(buf, "%s\n", current_trace->name); 2726 else 2727 r = sprintf(buf, "\n"); 2728 mutex_unlock(&trace_types_lock); 2729 2730 return simple_read_from_buffer(ubuf, cnt, ppos, buf, r); 2731} 2732 2733int tracer_init(struct tracer *t, struct trace_array *tr) 2734{ 2735 tracing_reset_online_cpus(tr); 2736 return t->init(tr); 2737} 2738 2739static int tracing_resize_ring_buffer(unsigned long size) 2740{ 2741 int ret; 2742 2743 /* 2744 * If kernel or user changes the size of the ring buffer 2745 * we use the size that was given, and we can forget about 2746 * expanding it later. 2747 */ 2748 ring_buffer_expanded = 1; 2749 2750 ret = ring_buffer_resize(global_trace.buffer, size); 2751 if (ret < 0) 2752 return ret; 2753 2754 if (!current_trace->use_max_tr) 2755 goto out; 2756 2757 ret = ring_buffer_resize(max_tr.buffer, size); 2758 if (ret < 0) { 2759 int r; 2760 2761 r = ring_buffer_resize(global_trace.buffer, 2762 global_trace.entries); 2763 if (r < 0) { 2764 /* 2765 * AARGH! We are left with different 2766 * size max buffer!!!! 2767 * The max buffer is our "snapshot" buffer. 2768 * When a tracer needs a snapshot (one of the 2769 * latency tracers), it swaps the max buffer 2770 * with the saved snap shot. We succeeded to 2771 * update the size of the main buffer, but failed to 2772 * update the size of the max buffer. But when we tried 2773 * to reset the main buffer to the original size, we 2774 * failed there too. This is very unlikely to 2775 * happen, but if it does, warn and kill all 2776 * tracing. 2777 */ 2778 WARN_ON(1); 2779 tracing_disabled = 1; 2780 } 2781 return ret; 2782 } 2783 2784 max_tr.entries = size; 2785 out: 2786 global_trace.entries = size; 2787 2788 return ret; 2789} 2790 2791 2792/** 2793 * tracing_update_buffers - used by tracing facility to expand ring buffers 2794 * 2795 * To save on memory when the tracing is never used on a system with it 2796 * configured in. The ring buffers are set to a minimum size. But once 2797 * a user starts to use the tracing facility, then they need to grow 2798 * to their default size. 2799 * 2800 * This function is to be called when a tracer is about to be used. 2801 */ 2802int tracing_update_buffers(void) 2803{ 2804 int ret = 0; 2805 2806 mutex_lock(&trace_types_lock); 2807 if (!ring_buffer_expanded) 2808 ret = tracing_resize_ring_buffer(trace_buf_size); 2809 mutex_unlock(&trace_types_lock); 2810 2811 return ret; 2812} 2813 2814struct trace_option_dentry; 2815 2816static struct trace_option_dentry * 2817create_trace_option_files(struct tracer *tracer); 2818 2819static void 2820destroy_trace_option_files(struct trace_option_dentry *topts); 2821 2822static int tracing_set_tracer(const char *buf) 2823{ 2824 static struct trace_option_dentry *topts; 2825 struct trace_array *tr = &global_trace; 2826 struct tracer *t; 2827 int ret = 0; 2828 2829 mutex_lock(&trace_types_lock); 2830 2831 if (!ring_buffer_expanded) { 2832 ret = tracing_resize_ring_buffer(trace_buf_size); 2833 if (ret < 0) 2834 goto out; 2835 ret = 0; 2836 } 2837 2838 for (t = trace_types; t; t = t->next) { 2839 if (strcmp(t->name, buf) == 0) 2840 break; 2841 } 2842 if (!t) { 2843 ret = -EINVAL; 2844 goto out; 2845 } 2846 if (t == current_trace) 2847 goto out; 2848 2849 trace_branch_disable(); 2850 if (current_trace && current_trace->reset) 2851 current_trace->reset(tr); 2852 if (current_trace && current_trace->use_max_tr) { 2853 /* 2854 * We don't free the ring buffer. instead, resize it because 2855 * The max_tr ring buffer has some state (e.g. ring->clock) and 2856 * we want preserve it. 2857 */ 2858 ring_buffer_resize(max_tr.buffer, 1); 2859 max_tr.entries = 1; 2860 } 2861 destroy_trace_option_files(topts); 2862 2863 current_trace = t; 2864 2865 topts = create_trace_option_files(current_trace); 2866 if (current_trace->use_max_tr) { 2867 ret = ring_buffer_resize(max_tr.buffer, global_trace.entries); 2868 if (ret < 0) 2869 goto out; 2870 max_tr.entries = global_trace.entries; 2871 } 2872 2873 if (t->init) { 2874 ret = tracer_init(t, tr); 2875 if (ret) 2876 goto out; 2877 } 2878 2879 trace_branch_enable(tr); 2880 out: 2881 mutex_unlock(&trace_types_lock); 2882 2883 return ret; 2884} 2885 2886static ssize_t 2887tracing_set_trace_write(struct file *filp, const char __user *ubuf, 2888 size_t cnt, loff_t *ppos) 2889{ 2890 char buf[MAX_TRACER_SIZE+1]; 2891 int i; 2892 size_t ret; 2893 int err; 2894 2895 ret = cnt; 2896 2897 if (cnt > MAX_TRACER_SIZE) 2898 cnt = MAX_TRACER_SIZE; 2899 2900 if (copy_from_user(&buf, ubuf, cnt)) 2901 return -EFAULT; 2902 2903 buf[cnt] = 0; 2904 2905 /* strip ending whitespace. */ 2906 for (i = cnt - 1; i > 0 && isspace(buf[i]); i--) 2907 buf[i] = 0; 2908 2909 err = tracing_set_tracer(buf); 2910 if (err) 2911 return err; 2912 2913 *ppos += ret; 2914 2915 return ret; 2916} 2917 2918static ssize_t 2919tracing_max_lat_read(struct file *filp, char __user *ubuf, 2920 size_t cnt, loff_t *ppos) 2921{ 2922 unsigned long *ptr = filp->private_data; 2923 char buf[64]; 2924 int r; 2925 2926 r = snprintf(buf, sizeof(buf), "%ld\n", 2927 *ptr == (unsigned long)-1 ? -1 : nsecs_to_usecs(*ptr)); 2928 if (r > sizeof(buf)) 2929 r = sizeof(buf); 2930 return simple_read_from_buffer(ubuf, cnt, ppos, buf, r); 2931} 2932 2933static ssize_t 2934tracing_max_lat_write(struct file *filp, const char __user *ubuf, 2935 size_t cnt, loff_t *ppos) 2936{ 2937 unsigned long *ptr = filp->private_data; 2938 char buf[64]; 2939 unsigned long val; 2940 int ret; 2941 2942 if (cnt >= sizeof(buf)) 2943 return -EINVAL; 2944 2945 if (copy_from_user(&buf, ubuf, cnt)) 2946 return -EFAULT; 2947 2948 buf[cnt] = 0; 2949 2950 ret = strict_strtoul(buf, 10, &val); 2951 if (ret < 0) 2952 return ret; 2953 2954 *ptr = val * 1000; 2955 2956 return cnt; 2957} 2958 2959static int tracing_open_pipe(struct inode *inode, struct file *filp) 2960{ 2961 long cpu_file = (long) inode->i_private; 2962 struct trace_iterator *iter; 2963 int ret = 0; 2964 2965 if (tracing_disabled) 2966 return -ENODEV; 2967 2968 mutex_lock(&trace_types_lock); 2969 2970 /* create a buffer to store the information to pass to userspace */ 2971 iter = kzalloc(sizeof(*iter), GFP_KERNEL); 2972 if (!iter) { 2973 ret = -ENOMEM; 2974 goto out; 2975 } 2976 2977 /* 2978 * We make a copy of the current tracer to avoid concurrent 2979 * changes on it while we are reading. 2980 */ 2981 iter->trace = kmalloc(sizeof(*iter->trace), GFP_KERNEL); 2982 if (!iter->trace) { 2983 ret = -ENOMEM; 2984 goto fail; 2985 } 2986 if (current_trace) 2987 *iter->trace = *current_trace; 2988 2989 if (!alloc_cpumask_var(&iter->started, GFP_KERNEL)) { 2990 ret = -ENOMEM; 2991 goto fail; 2992 } 2993 2994 /* trace pipe does not show start of buffer */ 2995 cpumask_setall(iter->started); 2996 2997 if (trace_flags & TRACE_ITER_LATENCY_FMT) 2998 iter->iter_flags |= TRACE_FILE_LAT_FMT; 2999 3000 iter->cpu_file = cpu_file; 3001 iter->tr = &global_trace; 3002 mutex_init(&iter->mutex); 3003 filp->private_data = iter; 3004 3005 if (iter->trace->pipe_open) 3006 iter->trace->pipe_open(iter); 3007 3008 nonseekable_open(inode, filp); 3009out: 3010 mutex_unlock(&trace_types_lock); 3011 return ret; 3012 3013fail: 3014 kfree(iter->trace); 3015 kfree(iter); 3016 mutex_unlock(&trace_types_lock); 3017 return ret; 3018} 3019 3020static int tracing_release_pipe(struct inode *inode, struct file *file) 3021{ 3022 struct trace_iterator *iter = file->private_data; 3023 3024 mutex_lock(&trace_types_lock); 3025 3026 if (iter->trace->pipe_close) 3027 iter->trace->pipe_close(iter); 3028 3029 mutex_unlock(&trace_types_lock); 3030 3031 free_cpumask_var(iter->started); 3032 mutex_destroy(&iter->mutex); 3033 kfree(iter->trace); 3034 kfree(iter); 3035 3036 return 0; 3037} 3038 3039static unsigned int 3040tracing_poll_pipe(struct file *filp, poll_table *poll_table) 3041{ 3042 struct trace_iterator *iter = filp->private_data; 3043 3044 if (trace_flags & TRACE_ITER_BLOCK) { 3045 /* 3046 * Always select as readable when in blocking mode 3047 */ 3048 return POLLIN | POLLRDNORM; 3049 } else { 3050 if (!trace_empty(iter)) 3051 return POLLIN | POLLRDNORM; 3052 poll_wait(filp, &trace_wait, poll_table); 3053 if (!trace_empty(iter)) 3054 return POLLIN | POLLRDNORM; 3055 3056 return 0; 3057 } 3058} 3059 3060 3061void default_wait_pipe(struct trace_iterator *iter) 3062{ 3063 DEFINE_WAIT(wait); 3064 3065 prepare_to_wait(&trace_wait, &wait, TASK_INTERRUPTIBLE); 3066 3067 if (trace_empty(iter)) 3068 schedule(); 3069 3070 finish_wait(&trace_wait, &wait); 3071} 3072 3073/* 3074 * This is a make-shift waitqueue. 3075 * A tracer might use this callback on some rare cases: 3076 * 3077 * 1) the current tracer might hold the runqueue lock when it wakes up 3078 * a reader, hence a deadlock (sched, function, and function graph tracers) 3079 * 2) the function tracers, trace all functions, we don't want 3080 * the overhead of calling wake_up and friends 3081 * (and tracing them too) 3082 * 3083 * Anyway, this is really very primitive wakeup. 3084 */ 3085void poll_wait_pipe(struct trace_iterator *iter) 3086{ 3087 set_current_state(TASK_INTERRUPTIBLE); 3088 /* sleep for 100 msecs, and try again. */ 3089 schedule_timeout(HZ / 10); 3090} 3091 3092/* Must be called with trace_types_lock mutex held. */ 3093static int tracing_wait_pipe(struct file *filp) 3094{ 3095 struct trace_iterator *iter = filp->private_data; 3096 3097 while (trace_empty(iter)) { 3098 3099 if ((filp->f_flags & O_NONBLOCK)) { 3100 return -EAGAIN; 3101 } 3102 3103 mutex_unlock(&iter->mutex); 3104 3105 iter->trace->wait_pipe(iter); 3106 3107 mutex_lock(&iter->mutex); 3108 3109 if (signal_pending(current)) 3110 return -EINTR; 3111 3112 /* 3113 * We block until we read something and tracing is disabled. 3114 * We still block if tracing is disabled, but we have never 3115 * read anything. This allows a user to cat this file, and 3116 * then enable tracing. But after we have read something, 3117 * we give an EOF when tracing is again disabled. 3118 * 3119 * iter->pos will be 0 if we haven't read anything. 3120 */ 3121 if (!tracer_enabled && iter->pos) 3122 break; 3123 } 3124 3125 return 1; 3126} 3127 3128/* 3129 * Consumer reader. 3130 */ 3131static ssize_t 3132tracing_read_pipe(struct file *filp, char __user *ubuf, 3133 size_t cnt, loff_t *ppos) 3134{ 3135 struct trace_iterator *iter = filp->private_data; 3136 static struct tracer *old_tracer; 3137 ssize_t sret; 3138 3139 /* return any leftover data */ 3140 sret = trace_seq_to_user(&iter->seq, ubuf, cnt); 3141 if (sret != -EBUSY) 3142 return sret; 3143 3144 trace_seq_init(&iter->seq); 3145 3146 /* copy the tracer to avoid using a global lock all around */ 3147 mutex_lock(&trace_types_lock); 3148 if (unlikely(old_tracer != current_trace && current_trace)) { 3149 old_tracer = current_trace; 3150 *iter->trace = *current_trace; 3151 } 3152 mutex_unlock(&trace_types_lock); 3153 3154 /* 3155 * Avoid more than one consumer on a single file descriptor 3156 * This is just a matter of traces coherency, the ring buffer itself 3157 * is protected. 3158 */ 3159 mutex_lock(&iter->mutex); 3160 if (iter->trace->read) { 3161 sret = iter->trace->read(iter, filp, ubuf, cnt, ppos); 3162 if (sret) 3163 goto out; 3164 } 3165 3166waitagain: 3167 sret = tracing_wait_pipe(filp); 3168 if (sret <= 0) 3169 goto out; 3170 3171 /* stop when tracing is finished */ 3172 if (trace_empty(iter)) { 3173 sret = 0; 3174 goto out; 3175 } 3176 3177 if (cnt >= PAGE_SIZE) 3178 cnt = PAGE_SIZE - 1; 3179 3180 /* reset all but tr, trace, and overruns */ 3181 memset(&iter->seq, 0, 3182 sizeof(struct trace_iterator) - 3183 offsetof(struct trace_iterator, seq)); 3184 iter->pos = -1; 3185 3186 trace_event_read_lock(); 3187 trace_access_lock(iter->cpu_file); 3188 while (trace_find_next_entry_inc(iter) != NULL) { 3189 enum print_line_t ret; 3190 int len = iter->seq.len; 3191 3192 ret = print_trace_line(iter); 3193 if (ret == TRACE_TYPE_PARTIAL_LINE) { 3194 /* don't print partial lines */ 3195 iter->seq.len = len; 3196 break; 3197 } 3198 if (ret != TRACE_TYPE_NO_CONSUME) 3199 trace_consume(iter); 3200 3201 if (iter->seq.len >= cnt) 3202 break; 3203 } 3204 trace_access_unlock(iter->cpu_file); 3205 trace_event_read_unlock(); 3206 3207 /* Now copy what we have to the user */ 3208 sret = trace_seq_to_user(&iter->seq, ubuf, cnt); 3209 if (iter->seq.readpos >= iter->seq.len) 3210 trace_seq_init(&iter->seq); 3211 3212 /* 3213 * If there was nothing to send to user, inspite of consuming trace 3214 * entries, go back to wait for more entries. 3215 */ 3216 if (sret == -EBUSY) 3217 goto waitagain; 3218 3219out: 3220 mutex_unlock(&iter->mutex); 3221 3222 return sret; 3223} 3224 3225static void tracing_pipe_buf_release(struct pipe_inode_info *pipe, 3226 struct pipe_buffer *buf) 3227{ 3228 __free_page(buf->page); 3229} 3230 3231static void tracing_spd_release_pipe(struct splice_pipe_desc *spd, 3232 unsigned int idx) 3233{ 3234 __free_page(spd->pages[idx]); 3235} 3236 3237static const struct pipe_buf_operations tracing_pipe_buf_ops = { 3238 .can_merge = 0, 3239 .map = generic_pipe_buf_map, 3240 .unmap = generic_pipe_buf_unmap, 3241 .confirm = generic_pipe_buf_confirm, 3242 .release = tracing_pipe_buf_release, 3243 .steal = generic_pipe_buf_steal, 3244 .get = generic_pipe_buf_get, 3245}; 3246 3247static size_t 3248tracing_fill_pipe_page(size_t rem, struct trace_iterator *iter) 3249{ 3250 size_t count; 3251 int ret; 3252 3253 /* Seq buffer is page-sized, exactly what we need. */ 3254 for (;;) { 3255 count = iter->seq.len; 3256 ret = print_trace_line(iter); 3257 count = iter->seq.len - count; 3258 if (rem < count) { 3259 rem = 0; 3260 iter->seq.len -= count; 3261 break; 3262 } 3263 if (ret == TRACE_TYPE_PARTIAL_LINE) { 3264 iter->seq.len -= count; 3265 break; 3266 } 3267 3268 if (ret != TRACE_TYPE_NO_CONSUME) 3269 trace_consume(iter); 3270 rem -= count; 3271 if (!trace_find_next_entry_inc(iter)) { 3272 rem = 0; 3273 iter->ent = NULL; 3274 break; 3275 } 3276 } 3277 3278 return rem; 3279} 3280 3281static ssize_t tracing_splice_read_pipe(struct file *filp, 3282 loff_t *ppos, 3283 struct pipe_inode_info *pipe, 3284 size_t len, 3285 unsigned int flags) 3286{ 3287 struct page *pages_def[PIPE_DEF_BUFFERS]; 3288 struct partial_page partial_def[PIPE_DEF_BUFFERS]; 3289 struct trace_iterator *iter = filp->private_data; 3290 struct splice_pipe_desc spd = { 3291 .pages = pages_def, 3292 .partial = partial_def, 3293 .nr_pages = 0, /* This gets updated below. */ 3294 .flags = flags, 3295 .ops = &tracing_pipe_buf_ops, 3296 .spd_release = tracing_spd_release_pipe, 3297 }; 3298 static struct tracer *old_tracer; 3299 ssize_t ret; 3300 size_t rem; 3301 unsigned int i; 3302 3303 if (splice_grow_spd(pipe, &spd)) 3304 return -ENOMEM; 3305 3306 /* copy the tracer to avoid using a global lock all around */ 3307 mutex_lock(&trace_types_lock); 3308 if (unlikely(old_tracer != current_trace && current_trace)) { 3309 old_tracer = current_trace; 3310 *iter->trace = *current_trace; 3311 } 3312 mutex_unlock(&trace_types_lock); 3313 3314 mutex_lock(&iter->mutex); 3315 3316 if (iter->trace->splice_read) { 3317 ret = iter->trace->splice_read(iter, filp, 3318 ppos, pipe, len, flags); 3319 if (ret) 3320 goto out_err; 3321 } 3322 3323 ret = tracing_wait_pipe(filp); 3324 if (ret <= 0) 3325 goto out_err; 3326 3327 if (!iter->ent && !trace_find_next_entry_inc(iter)) { 3328 ret = -EFAULT; 3329 goto out_err; 3330 } 3331 3332 trace_event_read_lock(); 3333 trace_access_lock(iter->cpu_file); 3334 3335 /* Fill as many pages as possible. */ 3336 for (i = 0, rem = len; i < pipe->buffers && rem; i++) { 3337 spd.pages[i] = alloc_page(GFP_KERNEL); 3338 if (!spd.pages[i]) 3339 break; 3340 3341 rem = tracing_fill_pipe_page(rem, iter); 3342 3343 /* Copy the data into the page, so we can start over. */ 3344 ret = trace_seq_to_buffer(&iter->seq, 3345 page_address(spd.pages[i]), 3346 iter->seq.len); 3347 if (ret < 0) { 3348 __free_page(spd.pages[i]); 3349 break; 3350 } 3351 spd.partial[i].offset = 0; 3352 spd.partial[i].len = iter->seq.len; 3353 3354 trace_seq_init(&iter->seq); 3355 } 3356 3357 trace_access_unlock(iter->cpu_file); 3358 trace_event_read_unlock(); 3359 mutex_unlock(&iter->mutex); 3360 3361 spd.nr_pages = i; 3362 3363 ret = splice_to_pipe(pipe, &spd); 3364out: 3365 splice_shrink_spd(pipe, &spd); 3366 return ret; 3367 3368out_err: 3369 mutex_unlock(&iter->mutex); 3370 goto out; 3371} 3372 3373static ssize_t 3374tracing_entries_read(struct file *filp, char __user *ubuf, 3375 size_t cnt, loff_t *ppos) 3376{ 3377 struct trace_array *tr = filp->private_data; 3378 char buf[96]; 3379 int r; 3380 3381 mutex_lock(&trace_types_lock); 3382 if (!ring_buffer_expanded) 3383 r = sprintf(buf, "%lu (expanded: %lu)\n", 3384 tr->entries >> 10, 3385 trace_buf_size >> 10); 3386 else 3387 r = sprintf(buf, "%lu\n", tr->entries >> 10); 3388 mutex_unlock(&trace_types_lock); 3389 3390 return simple_read_from_buffer(ubuf, cnt, ppos, buf, r); 3391} 3392 3393static ssize_t 3394tracing_entries_write(struct file *filp, const char __user *ubuf, 3395 size_t cnt, loff_t *ppos) 3396{ 3397 unsigned long val; 3398 char buf[64]; 3399 int ret, cpu; 3400 3401 if (cnt >= sizeof(buf)) 3402 return -EINVAL; 3403 3404 if (copy_from_user(&buf, ubuf, cnt)) 3405 return -EFAULT; 3406 3407 buf[cnt] = 0; 3408 3409 ret = strict_strtoul(buf, 10, &val); 3410 if (ret < 0) 3411 return ret; 3412 3413 /* must have at least 1 entry */ 3414 if (!val) 3415 return -EINVAL; 3416 3417 mutex_lock(&trace_types_lock); 3418 3419 tracing_stop(); 3420 3421 /* disable all cpu buffers */ 3422 for_each_tracing_cpu(cpu) { 3423 if (global_trace.data[cpu]) 3424 atomic_inc(&global_trace.data[cpu]->disabled); 3425 if (max_tr.data[cpu]) 3426 atomic_inc(&max_tr.data[cpu]->disabled); 3427 } 3428 3429 /* value is in KB */ 3430 val <<= 10; 3431 3432 if (val != global_trace.entries) { 3433 ret = tracing_resize_ring_buffer(val); 3434 if (ret < 0) { 3435 cnt = ret; 3436 goto out; 3437 } 3438 } 3439 3440 *ppos += cnt; 3441 3442 /* If check pages failed, return ENOMEM */ 3443 if (tracing_disabled) 3444 cnt = -ENOMEM; 3445 out: 3446 for_each_tracing_cpu(cpu) { 3447 if (global_trace.data[cpu]) 3448 atomic_dec(&global_trace.data[cpu]->disabled); 3449 if (max_tr.data[cpu]) 3450 atomic_dec(&max_tr.data[cpu]->disabled); 3451 } 3452 3453 tracing_start(); 3454 mutex_unlock(&trace_types_lock); 3455 3456 return cnt; 3457} 3458 3459static int mark_printk(const char *fmt, ...) 3460{ 3461 int ret; 3462 va_list args; 3463 va_start(args, fmt); 3464 ret = trace_vprintk(0, fmt, args); 3465 va_end(args); 3466 return ret; 3467} 3468 3469static ssize_t 3470tracing_mark_write(struct file *filp, const char __user *ubuf, 3471 size_t cnt, loff_t *fpos) 3472{ 3473 char *buf; 3474 size_t written; 3475 3476 if (tracing_disabled) 3477 return -EINVAL; 3478 3479 if (cnt > TRACE_BUF_SIZE) 3480 cnt = TRACE_BUF_SIZE; 3481 3482 buf = kmalloc(cnt + 2, GFP_KERNEL); 3483 if (buf == NULL) 3484 return -ENOMEM; 3485 3486 if (copy_from_user(buf, ubuf, cnt)) { 3487 kfree(buf); 3488 return -EFAULT; 3489 } 3490 if (buf[cnt-1] != '\n') { 3491 buf[cnt] = '\n'; 3492 buf[cnt+1] = '\0'; 3493 } else 3494 buf[cnt] = '\0'; 3495 3496 written = mark_printk("%s", buf); 3497 kfree(buf); 3498 *fpos += written; 3499 3500 /* don't tell userspace we wrote more - it might confuse them */ 3501 if (written > cnt) 3502 written = cnt; 3503 3504 return written; 3505} 3506 3507static int tracing_clock_show(struct seq_file *m, void *v) 3508{ 3509 int i; 3510 3511 for (i = 0; i < ARRAY_SIZE(trace_clocks); i++) 3512 seq_printf(m, 3513 "%s%s%s%s", i ? " " : "", 3514 i == trace_clock_id ? "[" : "", trace_clocks[i].name, 3515 i == trace_clock_id ? "]" : ""); 3516 seq_putc(m, '\n'); 3517 3518 return 0; 3519} 3520 3521static ssize_t tracing_clock_write(struct file *filp, const char __user *ubuf, 3522 size_t cnt, loff_t *fpos) 3523{ 3524 char buf[64]; 3525 const char *clockstr; 3526 int i; 3527 3528 if (cnt >= sizeof(buf)) 3529 return -EINVAL; 3530 3531 if (copy_from_user(&buf, ubuf, cnt)) 3532 return -EFAULT; 3533 3534 buf[cnt] = 0; 3535 3536 clockstr = strstrip(buf); 3537 3538 for (i = 0; i < ARRAY_SIZE(trace_clocks); i++) { 3539 if (strcmp(trace_clocks[i].name, clockstr) == 0) 3540 break; 3541 } 3542 if (i == ARRAY_SIZE(trace_clocks)) 3543 return -EINVAL; 3544 3545 trace_clock_id = i; 3546 3547 mutex_lock(&trace_types_lock); 3548 3549 ring_buffer_set_clock(global_trace.buffer, trace_clocks[i].func); 3550 if (max_tr.buffer) 3551 ring_buffer_set_clock(max_tr.buffer, trace_clocks[i].func); 3552 3553 mutex_unlock(&trace_types_lock); 3554 3555 *fpos += cnt; 3556 3557 return cnt; 3558} 3559 3560static int tracing_clock_open(struct inode *inode, struct file *file) 3561{ 3562 if (tracing_disabled) 3563 return -ENODEV; 3564 return single_open(file, tracing_clock_show, NULL); 3565} 3566 3567static const struct file_operations tracing_max_lat_fops = { 3568 .open = tracing_open_generic, 3569 .read = tracing_max_lat_read, 3570 .write = tracing_max_lat_write, 3571 .llseek = generic_file_llseek, 3572}; 3573 3574static const struct file_operations tracing_ctrl_fops = { 3575 .open = tracing_open_generic, 3576 .read = tracing_ctrl_read, 3577 .write = tracing_ctrl_write, 3578 .llseek = generic_file_llseek, 3579}; 3580 3581static const struct file_operations set_tracer_fops = { 3582 .open = tracing_open_generic, 3583 .read = tracing_set_trace_read, 3584 .write = tracing_set_trace_write, 3585 .llseek = generic_file_llseek, 3586}; 3587 3588static const struct file_operations tracing_pipe_fops = { 3589 .open = tracing_open_pipe, 3590 .poll = tracing_poll_pipe, 3591 .read = tracing_read_pipe, 3592 .splice_read = tracing_splice_read_pipe, 3593 .release = tracing_release_pipe, 3594 .llseek = no_llseek, 3595}; 3596 3597static const struct file_operations tracing_entries_fops = { 3598 .open = tracing_open_generic, 3599 .read = tracing_entries_read, 3600 .write = tracing_entries_write, 3601 .llseek = generic_file_llseek, 3602}; 3603 3604static const struct file_operations tracing_mark_fops = { 3605 .open = tracing_open_generic, 3606 .write = tracing_mark_write, 3607 .llseek = generic_file_llseek, 3608}; 3609 3610static const struct file_operations trace_clock_fops = { 3611 .open = tracing_clock_open, 3612 .read = seq_read, 3613 .llseek = seq_lseek, 3614 .release = single_release, 3615 .write = tracing_clock_write, 3616}; 3617 3618struct ftrace_buffer_info { 3619 struct trace_array *tr; 3620 void *spare; 3621 int cpu; 3622 unsigned int read; 3623}; 3624 3625static int tracing_buffers_open(struct inode *inode, struct file *filp) 3626{ 3627 int cpu = (int)(long)inode->i_private; 3628 struct ftrace_buffer_info *info; 3629 3630 if (tracing_disabled) 3631 return -ENODEV; 3632 3633 info = kzalloc(sizeof(*info), GFP_KERNEL); 3634 if (!info) 3635 return -ENOMEM; 3636 3637 info->tr = &global_trace; 3638 info->cpu = cpu; 3639 info->spare = NULL; 3640 /* Force reading ring buffer for first read */ 3641 info->read = (unsigned int)-1; 3642 3643 filp->private_data = info; 3644 3645 return nonseekable_open(inode, filp); 3646} 3647 3648static ssize_t 3649tracing_buffers_read(struct file *filp, char __user *ubuf, 3650 size_t count, loff_t *ppos) 3651{ 3652 struct ftrace_buffer_info *info = filp->private_data; 3653 ssize_t ret; 3654 size_t size; 3655 3656 if (!count) 3657 return 0; 3658 3659 if (!info->spare) 3660 info->spare = ring_buffer_alloc_read_page(info->tr->buffer); 3661 if (!info->spare) 3662 return -ENOMEM; 3663 3664 /* Do we have previous read data to read? */ 3665 if (info->read < PAGE_SIZE) 3666 goto read; 3667 3668 info->read = 0; 3669 3670 trace_access_lock(info->cpu); 3671 ret = ring_buffer_read_page(info->tr->buffer, 3672 &info->spare, 3673 count, 3674 info->cpu, 0); 3675 trace_access_unlock(info->cpu); 3676 if (ret < 0) 3677 return 0; 3678 3679read: 3680 size = PAGE_SIZE - info->read; 3681 if (size > count) 3682 size = count; 3683 3684 ret = copy_to_user(ubuf, info->spare + info->read, size); 3685 if (ret == size) 3686 return -EFAULT; 3687 size -= ret; 3688 3689 *ppos += size; 3690 info->read += size; 3691 3692 return size; 3693} 3694 3695static int tracing_buffers_release(struct inode *inode, struct file *file) 3696{ 3697 struct ftrace_buffer_info *info = file->private_data; 3698 3699 if (info->spare) 3700 ring_buffer_free_read_page(info->tr->buffer, info->spare); 3701 kfree(info); 3702 3703 return 0; 3704} 3705 3706struct buffer_ref { 3707 struct ring_buffer *buffer; 3708 void *page; 3709 int ref; 3710}; 3711 3712static void buffer_pipe_buf_release(struct pipe_inode_info *pipe, 3713 struct pipe_buffer *buf) 3714{ 3715 struct buffer_ref *ref = (struct buffer_ref *)buf->private; 3716 3717 if (--ref->ref) 3718 return; 3719 3720 ring_buffer_free_read_page(ref->buffer, ref->page); 3721 kfree(ref); 3722 buf->private = 0; 3723} 3724 3725static int buffer_pipe_buf_steal(struct pipe_inode_info *pipe, 3726 struct pipe_buffer *buf) 3727{ 3728 return 1; 3729} 3730 3731static void buffer_pipe_buf_get(struct pipe_inode_info *pipe, 3732 struct pipe_buffer *buf) 3733{ 3734 struct buffer_ref *ref = (struct buffer_ref *)buf->private; 3735 3736 ref->ref++; 3737} 3738 3739/* Pipe buffer operations for a buffer. */ 3740static const struct pipe_buf_operations buffer_pipe_buf_ops = { 3741 .can_merge = 0, 3742 .map = generic_pipe_buf_map, 3743 .unmap = generic_pipe_buf_unmap, 3744 .confirm = generic_pipe_buf_confirm, 3745 .release = buffer_pipe_buf_release, 3746 .steal = buffer_pipe_buf_steal, 3747 .get = buffer_pipe_buf_get, 3748}; 3749 3750/* 3751 * Callback from splice_to_pipe(), if we need to release some pages 3752 * at the end of the spd in case we error'ed out in filling the pipe. 3753 */ 3754static void buffer_spd_release(struct splice_pipe_desc *spd, unsigned int i) 3755{ 3756 struct buffer_ref *ref = 3757 (struct buffer_ref *)spd->partial[i].private; 3758 3759 if (--ref->ref) 3760 return; 3761 3762 ring_buffer_free_read_page(ref->buffer, ref->page); 3763 kfree(ref); 3764 spd->partial[i].private = 0; 3765} 3766 3767static ssize_t 3768tracing_buffers_splice_read(struct file *file, loff_t *ppos, 3769 struct pipe_inode_info *pipe, size_t len, 3770 unsigned int flags) 3771{ 3772 struct ftrace_buffer_info *info = file->private_data; 3773 struct partial_page partial_def[PIPE_DEF_BUFFERS]; 3774 struct page *pages_def[PIPE_DEF_BUFFERS]; 3775 struct splice_pipe_desc spd = { 3776 .pages = pages_def, 3777 .partial = partial_def, 3778 .flags = flags, 3779 .ops = &buffer_pipe_buf_ops, 3780 .spd_release = buffer_spd_release, 3781 }; 3782 struct buffer_ref *ref; 3783 int entries, size, i; 3784 size_t ret; 3785 3786 if (splice_grow_spd(pipe, &spd)) 3787 return -ENOMEM; 3788 3789 if (*ppos & (PAGE_SIZE - 1)) { 3790 WARN_ONCE(1, "Ftrace: previous read must page-align\n"); 3791 ret = -EINVAL; 3792 goto out; 3793 } 3794 3795 if (len & (PAGE_SIZE - 1)) { 3796 WARN_ONCE(1, "Ftrace: splice_read should page-align\n"); 3797 if (len < PAGE_SIZE) { 3798 ret = -EINVAL; 3799 goto out; 3800 } 3801 len &= PAGE_MASK; 3802 } 3803 3804 trace_access_lock(info->cpu); 3805 entries = ring_buffer_entries_cpu(info->tr->buffer, info->cpu); 3806 3807 for (i = 0; i < pipe->buffers && len && entries; i++, len -= PAGE_SIZE) { 3808 struct page *page; 3809 int r; 3810 3811 ref = kzalloc(sizeof(*ref), GFP_KERNEL); 3812 if (!ref) 3813 break; 3814 3815 ref->ref = 1; 3816 ref->buffer = info->tr->buffer; 3817 ref->page = ring_buffer_alloc_read_page(ref->buffer); 3818 if (!ref->page) { 3819 kfree(ref); 3820 break; 3821 } 3822 3823 r = ring_buffer_read_page(ref->buffer, &ref->page, 3824 len, info->cpu, 1); 3825 if (r < 0) { 3826 ring_buffer_free_read_page(ref->buffer, 3827 ref->page); 3828 kfree(ref); 3829 break; 3830 } 3831 3832 /* 3833 * zero out any left over data, this is going to 3834 * user land. 3835 */ 3836 size = ring_buffer_page_len(ref->page); 3837 if (size < PAGE_SIZE) 3838 memset(ref->page + size, 0, PAGE_SIZE - size); 3839 3840 page = virt_to_page(ref->page); 3841 3842 spd.pages[i] = page; 3843 spd.partial[i].len = PAGE_SIZE; 3844 spd.partial[i].offset = 0; 3845 spd.partial[i].private = (unsigned long)ref; 3846 spd.nr_pages++; 3847 *ppos += PAGE_SIZE; 3848 3849 entries = ring_buffer_entries_cpu(info->tr->buffer, info->cpu); 3850 } 3851 3852 trace_access_unlock(info->cpu); 3853 spd.nr_pages = i; 3854 3855 /* did we read anything? */ 3856 if (!spd.nr_pages) { 3857 if (flags & SPLICE_F_NONBLOCK) 3858 ret = -EAGAIN; 3859 else 3860 ret = 0; 3861 /* TODO: block */ 3862 goto out; 3863 } 3864 3865 ret = splice_to_pipe(pipe, &spd); 3866 splice_shrink_spd(pipe, &spd); 3867out: 3868 return ret; 3869} 3870 3871static const struct file_operations tracing_buffers_fops = { 3872 .open = tracing_buffers_open, 3873 .read = tracing_buffers_read, 3874 .release = tracing_buffers_release, 3875 .splice_read = tracing_buffers_splice_read, 3876 .llseek = no_llseek, 3877}; 3878 3879static ssize_t 3880tracing_stats_read(struct file *filp, char __user *ubuf, 3881 size_t count, loff_t *ppos) 3882{ 3883 unsigned long cpu = (unsigned long)filp->private_data; 3884 struct trace_array *tr = &global_trace; 3885 struct trace_seq *s; 3886 unsigned long cnt; 3887 3888 s = kmalloc(sizeof(*s), GFP_KERNEL); 3889 if (!s) 3890 return -ENOMEM; 3891 3892 trace_seq_init(s); 3893 3894 cnt = ring_buffer_entries_cpu(tr->buffer, cpu); 3895 trace_seq_printf(s, "entries: %ld\n", cnt); 3896 3897 cnt = ring_buffer_overrun_cpu(tr->buffer, cpu); 3898 trace_seq_printf(s, "overrun: %ld\n", cnt); 3899 3900 cnt = ring_buffer_commit_overrun_cpu(tr->buffer, cpu); 3901 trace_seq_printf(s, "commit overrun: %ld\n", cnt); 3902 3903 count = simple_read_from_buffer(ubuf, count, ppos, s->buffer, s->len); 3904 3905 kfree(s); 3906 3907 return count; 3908} 3909 3910static const struct file_operations tracing_stats_fops = { 3911 .open = tracing_open_generic, 3912 .read = tracing_stats_read, 3913 .llseek = generic_file_llseek, 3914}; 3915 3916#ifdef CONFIG_DYNAMIC_FTRACE 3917 3918int __weak ftrace_arch_read_dyn_info(char *buf, int size) 3919{ 3920 return 0; 3921} 3922 3923static ssize_t 3924tracing_read_dyn_info(struct file *filp, char __user *ubuf, 3925 size_t cnt, loff_t *ppos) 3926{ 3927 static char ftrace_dyn_info_buffer[1024]; 3928 static DEFINE_MUTEX(dyn_info_mutex); 3929 unsigned long *p = filp->private_data; 3930 char *buf = ftrace_dyn_info_buffer; 3931 int size = ARRAY_SIZE(ftrace_dyn_info_buffer); 3932 int r; 3933 3934 mutex_lock(&dyn_info_mutex); 3935 r = sprintf(buf, "%ld ", *p); 3936 3937 r += ftrace_arch_read_dyn_info(buf+r, (size-1)-r); 3938 buf[r++] = '\n'; 3939 3940 r = simple_read_from_buffer(ubuf, cnt, ppos, buf, r); 3941 3942 mutex_unlock(&dyn_info_mutex); 3943 3944 return r; 3945} 3946 3947static const struct file_operations tracing_dyn_info_fops = { 3948 .open = tracing_open_generic, 3949 .read = tracing_read_dyn_info, 3950 .llseek = generic_file_llseek, 3951}; 3952#endif 3953 3954static struct dentry *d_tracer; 3955 3956struct dentry *tracing_init_dentry(void) 3957{ 3958 static int once; 3959 3960 if (d_tracer) 3961 return d_tracer; 3962 3963 if (!debugfs_initialized()) 3964 return NULL; 3965 3966 d_tracer = debugfs_create_dir("tracing", NULL); 3967 3968 if (!d_tracer && !once) { 3969 once = 1; 3970 pr_warning("Could not create debugfs directory 'tracing'\n"); 3971 return NULL; 3972 } 3973 3974 return d_tracer; 3975} 3976 3977static struct dentry *d_percpu; 3978 3979struct dentry *tracing_dentry_percpu(void) 3980{ 3981 static int once; 3982 struct dentry *d_tracer; 3983 3984 if (d_percpu) 3985 return d_percpu; 3986 3987 d_tracer = tracing_init_dentry(); 3988 3989 if (!d_tracer) 3990 return NULL; 3991 3992 d_percpu = debugfs_create_dir("per_cpu", d_tracer); 3993 3994 if (!d_percpu && !once) { 3995 once = 1; 3996 pr_warning("Could not create debugfs directory 'per_cpu'\n"); 3997 return NULL; 3998 } 3999 4000 return d_percpu; 4001} 4002 4003static void tracing_init_debugfs_percpu(long cpu) 4004{ 4005 struct dentry *d_percpu = tracing_dentry_percpu(); 4006 struct dentry *d_cpu; 4007 /* strlen(cpu) + MAX(log10(cpu)) + '\0' */ 4008 char cpu_dir[7]; 4009 4010 if (cpu > 999 || cpu < 0) 4011 return; 4012 4013 sprintf(cpu_dir, "cpu%ld", cpu); 4014 d_cpu = debugfs_create_dir(cpu_dir, d_percpu); 4015 if (!d_cpu) { 4016 pr_warning("Could not create debugfs '%s' entry\n", cpu_dir); 4017 return; 4018 } 4019 4020 /* per cpu trace_pipe */ 4021 trace_create_file("trace_pipe", 0444, d_cpu, 4022 (void *) cpu, &tracing_pipe_fops); 4023 4024 /* per cpu trace */ 4025 trace_create_file("trace", 0644, d_cpu, 4026 (void *) cpu, &tracing_fops); 4027 4028 trace_create_file("trace_pipe_raw", 0444, d_cpu, 4029 (void *) cpu, &tracing_buffers_fops); 4030 4031 trace_create_file("stats", 0444, d_cpu, 4032 (void *) cpu, &tracing_stats_fops); 4033} 4034 4035#ifdef CONFIG_FTRACE_SELFTEST 4036/* Let selftest have access to static functions in this file */ 4037#include "trace_selftest.c" 4038#endif 4039 4040struct trace_option_dentry { 4041 struct tracer_opt *opt; 4042 struct tracer_flags *flags; 4043 struct dentry *entry; 4044}; 4045 4046static ssize_t 4047trace_options_read(struct file *filp, char __user *ubuf, size_t cnt, 4048 loff_t *ppos) 4049{ 4050 struct trace_option_dentry *topt = filp->private_data; 4051 char *buf; 4052 4053 if (topt->flags->val & topt->opt->bit) 4054 buf = "1\n"; 4055 else 4056 buf = "0\n"; 4057 4058 return simple_read_from_buffer(ubuf, cnt, ppos, buf, 2); 4059} 4060 4061static ssize_t 4062trace_options_write(struct file *filp, const char __user *ubuf, size_t cnt, 4063 loff_t *ppos) 4064{ 4065 struct trace_option_dentry *topt = filp->private_data; 4066 unsigned long val; 4067 char buf[64]; 4068 int ret; 4069 4070 if (cnt >= sizeof(buf)) 4071 return -EINVAL; 4072 4073 if (copy_from_user(&buf, ubuf, cnt)) 4074 return -EFAULT; 4075 4076 buf[cnt] = 0; 4077 4078 ret = strict_strtoul(buf, 10, &val); 4079 if (ret < 0) 4080 return ret; 4081 4082 if (val != 0 && val != 1) 4083 return -EINVAL; 4084 4085 if (!!(topt->flags->val & topt->opt->bit) != val) { 4086 mutex_lock(&trace_types_lock); 4087 ret = __set_tracer_option(current_trace, topt->flags, 4088 topt->opt, !val); 4089 mutex_unlock(&trace_types_lock); 4090 if (ret) 4091 return ret; 4092 } 4093 4094 *ppos += cnt; 4095 4096 return cnt; 4097} 4098 4099 4100static const struct file_operations trace_options_fops = { 4101 .open = tracing_open_generic, 4102 .read = trace_options_read, 4103 .write = trace_options_write, 4104 .llseek = generic_file_llseek, 4105}; 4106 4107static ssize_t 4108trace_options_core_read(struct file *filp, char __user *ubuf, size_t cnt, 4109 loff_t *ppos) 4110{ 4111 long index = (long)filp->private_data; 4112 char *buf; 4113 4114 if (trace_flags & (1 << index)) 4115 buf = "1\n"; 4116 else 4117 buf = "0\n"; 4118 4119 return simple_read_from_buffer(ubuf, cnt, ppos, buf, 2); 4120} 4121 4122static ssize_t 4123trace_options_core_write(struct file *filp, const char __user *ubuf, size_t cnt, 4124 loff_t *ppos) 4125{ 4126 long index = (long)filp->private_data; 4127 char buf[64]; 4128 unsigned long val; 4129 int ret; 4130 4131 if (cnt >= sizeof(buf)) 4132 return -EINVAL; 4133 4134 if (copy_from_user(&buf, ubuf, cnt)) 4135 return -EFAULT; 4136 4137 buf[cnt] = 0; 4138 4139 ret = strict_strtoul(buf, 10, &val); 4140 if (ret < 0) 4141 return ret; 4142 4143 if (val != 0 && val != 1) 4144 return -EINVAL; 4145 set_tracer_flags(1 << index, val); 4146 4147 *ppos += cnt; 4148 4149 return cnt; 4150} 4151 4152static const struct file_operations trace_options_core_fops = { 4153 .open = tracing_open_generic, 4154 .read = trace_options_core_read, 4155 .write = trace_options_core_write, 4156 .llseek = generic_file_llseek, 4157}; 4158 4159struct dentry *trace_create_file(const char *name, 4160 mode_t mode, 4161 struct dentry *parent, 4162 void *data, 4163 const struct file_operations *fops) 4164{ 4165 struct dentry *ret; 4166 4167 ret = debugfs_create_file(name, mode, parent, data, fops); 4168 if (!ret) 4169 pr_warning("Could not create debugfs '%s' entry\n", name); 4170 4171 return ret; 4172} 4173 4174 4175static struct dentry *trace_options_init_dentry(void) 4176{ 4177 struct dentry *d_tracer; 4178 static struct dentry *t_options; 4179 4180 if (t_options) 4181 return t_options; 4182 4183 d_tracer = tracing_init_dentry(); 4184 if (!d_tracer) 4185 return NULL; 4186 4187 t_options = debugfs_create_dir("options", d_tracer); 4188 if (!t_options) { 4189 pr_warning("Could not create debugfs directory 'options'\n"); 4190 return NULL; 4191 } 4192 4193 return t_options; 4194} 4195 4196static void 4197create_trace_option_file(struct trace_option_dentry *topt, 4198 struct tracer_flags *flags, 4199 struct tracer_opt *opt) 4200{ 4201 struct dentry *t_options; 4202 4203 t_options = trace_options_init_dentry(); 4204 if (!t_options) 4205 return; 4206 4207 topt->flags = flags; 4208 topt->opt = opt; 4209 4210 topt->entry = trace_create_file(opt->name, 0644, t_options, topt, 4211 &trace_options_fops); 4212 4213} 4214 4215static struct trace_option_dentry * 4216create_trace_option_files(struct tracer *tracer) 4217{ 4218 struct trace_option_dentry *topts; 4219 struct tracer_flags *flags; 4220 struct tracer_opt *opts; 4221 int cnt; 4222 4223 if (!tracer) 4224 return NULL; 4225 4226 flags = tracer->flags; 4227 4228 if (!flags || !flags->opts) 4229 return NULL; 4230 4231 opts = flags->opts; 4232 4233 for (cnt = 0; opts[cnt].name; cnt++) 4234 ; 4235 4236 topts = kcalloc(cnt + 1, sizeof(*topts), GFP_KERNEL); 4237 if (!topts) 4238 return NULL; 4239 4240 for (cnt = 0; opts[cnt].name; cnt++) 4241 create_trace_option_file(&topts[cnt], flags, 4242 &opts[cnt]); 4243 4244 return topts; 4245} 4246 4247static void 4248destroy_trace_option_files(struct trace_option_dentry *topts) 4249{ 4250 int cnt; 4251 4252 if (!topts) 4253 return; 4254 4255 for (cnt = 0; topts[cnt].opt; cnt++) { 4256 if (topts[cnt].entry) 4257 debugfs_remove(topts[cnt].entry); 4258 } 4259 4260 kfree(topts); 4261} 4262 4263static struct dentry * 4264create_trace_option_core_file(const char *option, long index) 4265{ 4266 struct dentry *t_options; 4267 4268 t_options = trace_options_init_dentry(); 4269 if (!t_options) 4270 return NULL; 4271 4272 return trace_create_file(option, 0644, t_options, (void *)index, 4273 &trace_options_core_fops); 4274} 4275 4276static __init void create_trace_options_dir(void) 4277{ 4278 struct dentry *t_options; 4279 int i; 4280 4281 t_options = trace_options_init_dentry(); 4282 if (!t_options) 4283 return; 4284 4285 for (i = 0; trace_options[i]; i++) 4286 create_trace_option_core_file(trace_options[i], i); 4287} 4288 4289static __init int tracer_init_debugfs(void) 4290{ 4291 struct dentry *d_tracer; 4292 int cpu; 4293 4294 trace_access_lock_init(); 4295 4296 d_tracer = tracing_init_dentry(); 4297 4298 trace_create_file("tracing_enabled", 0644, d_tracer, 4299 &global_trace, &tracing_ctrl_fops); 4300 4301 trace_create_file("trace_options", 0644, d_tracer, 4302 NULL, &tracing_iter_fops); 4303 4304 trace_create_file("tracing_cpumask", 0644, d_tracer, 4305 NULL, &tracing_cpumask_fops); 4306 4307 trace_create_file("trace", 0644, d_tracer, 4308 (void *) TRACE_PIPE_ALL_CPU, &tracing_fops); 4309 4310 trace_create_file("available_tracers", 0444, d_tracer, 4311 &global_trace, &show_traces_fops); 4312 4313 trace_create_file("current_tracer", 0644, d_tracer, 4314 &global_trace, &set_tracer_fops); 4315 4316#ifdef CONFIG_TRACER_MAX_TRACE 4317 trace_create_file("tracing_max_latency", 0644, d_tracer, 4318 &tracing_max_latency, &tracing_max_lat_fops); 4319#endif 4320 4321 trace_create_file("tracing_thresh", 0644, d_tracer, 4322 &tracing_thresh, &tracing_max_lat_fops); 4323 4324 trace_create_file("README", 0444, d_tracer, 4325 NULL, &tracing_readme_fops); 4326 4327 trace_create_file("trace_pipe", 0444, d_tracer, 4328 (void *) TRACE_PIPE_ALL_CPU, &tracing_pipe_fops); 4329 4330 trace_create_file("buffer_size_kb", 0644, d_tracer, 4331 &global_trace, &tracing_entries_fops); 4332 4333 trace_create_file("trace_marker", 0220, d_tracer, 4334 NULL, &tracing_mark_fops); 4335 4336 trace_create_file("saved_cmdlines", 0444, d_tracer, 4337 NULL, &tracing_saved_cmdlines_fops); 4338 4339 trace_create_file("trace_clock", 0644, d_tracer, NULL, 4340 &trace_clock_fops); 4341 4342#ifdef CONFIG_DYNAMIC_FTRACE 4343 trace_create_file("dyn_ftrace_total_info", 0444, d_tracer, 4344 &ftrace_update_tot_cnt, &tracing_dyn_info_fops); 4345#endif 4346 4347 create_trace_options_dir(); 4348 4349 for_each_tracing_cpu(cpu) 4350 tracing_init_debugfs_percpu(cpu); 4351 4352 return 0; 4353} 4354 4355static int trace_panic_handler(struct notifier_block *this, 4356 unsigned long event, void *unused) 4357{ 4358 if (ftrace_dump_on_oops) 4359 ftrace_dump(ftrace_dump_on_oops); 4360 return NOTIFY_OK; 4361} 4362 4363static struct notifier_block trace_panic_notifier = { 4364 .notifier_call = trace_panic_handler, 4365 .next = NULL, 4366 .priority = 150 /* priority: INT_MAX >= x >= 0 */ 4367}; 4368 4369static int trace_die_handler(struct notifier_block *self, 4370 unsigned long val, 4371 void *data) 4372{ 4373 switch (val) { 4374 case DIE_OOPS: 4375 if (ftrace_dump_on_oops) 4376 ftrace_dump(ftrace_dump_on_oops); 4377 break; 4378 default: 4379 break; 4380 } 4381 return NOTIFY_OK; 4382} 4383 4384static struct notifier_block trace_die_notifier = { 4385 .notifier_call = trace_die_handler, 4386 .priority = 200 4387}; 4388 4389/* 4390 * printk is set to max of 1024, we really don't need it that big. 4391 * Nothing should be printing 1000 characters anyway. 4392 */ 4393#define TRACE_MAX_PRINT 1000 4394 4395/* 4396 * Define here KERN_TRACE so that we have one place to modify 4397 * it if we decide to change what log level the ftrace dump 4398 * should be at. 4399 */ 4400#define KERN_TRACE KERN_EMERG 4401 4402void 4403trace_printk_seq(struct trace_seq *s) 4404{ 4405 /* Probably should print a warning here. */ 4406 if (s->len >= 1000) 4407 s->len = 1000; 4408 4409 /* should be zero ended, but we are paranoid. */ 4410 s->buffer[s->len] = 0; 4411 4412 printk(KERN_TRACE "%s", s->buffer); 4413 4414 trace_seq_init(s); 4415} 4416 4417void trace_init_global_iter(struct trace_iterator *iter) 4418{ 4419 iter->tr = &global_trace; 4420 iter->trace = current_trace; 4421 iter->cpu_file = TRACE_PIPE_ALL_CPU; 4422} 4423 4424static void 4425__ftrace_dump(bool disable_tracing, enum ftrace_dump_mode oops_dump_mode) 4426{ 4427 static arch_spinlock_t ftrace_dump_lock = 4428 (arch_spinlock_t)__ARCH_SPIN_LOCK_UNLOCKED; 4429 /* use static because iter can be a bit big for the stack */ 4430 static struct trace_iterator iter; 4431 unsigned int old_userobj; 4432 static int dump_ran; 4433 unsigned long flags; 4434 int cnt = 0, cpu; 4435 4436 /* only one dump */ 4437 local_irq_save(flags); 4438 arch_spin_lock(&ftrace_dump_lock); 4439 if (dump_ran) 4440 goto out; 4441 4442 dump_ran = 1; 4443 4444 tracing_off(); 4445 4446 if (disable_tracing) 4447 ftrace_kill(); 4448 4449 trace_init_global_iter(&iter); 4450 4451 for_each_tracing_cpu(cpu) { 4452 atomic_inc(&iter.tr->data[cpu]->disabled); 4453 } 4454 4455 old_userobj = trace_flags & TRACE_ITER_SYM_USEROBJ; 4456 4457 /* don't look at user memory in panic mode */ 4458 trace_flags &= ~TRACE_ITER_SYM_USEROBJ; 4459 4460 /* Simulate the iterator */ 4461 iter.tr = &global_trace; 4462 iter.trace = current_trace; 4463 4464 switch (oops_dump_mode) { 4465 case DUMP_ALL: 4466 iter.cpu_file = TRACE_PIPE_ALL_CPU; 4467 break; 4468 case DUMP_ORIG: 4469 iter.cpu_file = raw_smp_processor_id(); 4470 break; 4471 case DUMP_NONE: 4472 goto out_enable; 4473 default: 4474 printk(KERN_TRACE "Bad dumping mode, switching to all CPUs dump\n"); 4475 iter.cpu_file = TRACE_PIPE_ALL_CPU; 4476 } 4477 4478 printk(KERN_TRACE "Dumping ftrace buffer:\n"); 4479 4480 /* 4481 * We need to stop all tracing on all CPUS to read the 4482 * the next buffer. This is a bit expensive, but is 4483 * not done often. We fill all what we can read, 4484 * and then release the locks again. 4485 */ 4486 4487 while (!trace_empty(&iter)) { 4488 4489 if (!cnt) 4490 printk(KERN_TRACE "---------------------------------\n"); 4491 4492 cnt++; 4493 4494 /* reset all but tr, trace, and overruns */ 4495 memset(&iter.seq, 0, 4496 sizeof(struct trace_iterator) - 4497 offsetof(struct trace_iterator, seq)); 4498 iter.iter_flags |= TRACE_FILE_LAT_FMT; 4499 iter.pos = -1; 4500 4501 if (trace_find_next_entry_inc(&iter) != NULL) { 4502 int ret; 4503 4504 ret = print_trace_line(&iter); 4505 if (ret != TRACE_TYPE_NO_CONSUME) 4506 trace_consume(&iter); 4507 } 4508 4509 trace_printk_seq(&iter.seq); 4510 } 4511 4512 if (!cnt) 4513 printk(KERN_TRACE " (ftrace buffer empty)\n"); 4514 else 4515 printk(KERN_TRACE "---------------------------------\n"); 4516 4517 out_enable: 4518 /* Re-enable tracing if requested */ 4519 if (!disable_tracing) { 4520 trace_flags |= old_userobj; 4521 4522 for_each_tracing_cpu(cpu) { 4523 atomic_dec(&iter.tr->data[cpu]->disabled); 4524 } 4525 tracing_on(); 4526 } 4527 4528 out: 4529 arch_spin_unlock(&ftrace_dump_lock); 4530 local_irq_restore(flags); 4531} 4532 4533/* By default: disable tracing after the dump */ 4534void ftrace_dump(enum ftrace_dump_mode oops_dump_mode) 4535{ 4536 __ftrace_dump(true, oops_dump_mode); 4537} 4538 4539__init static int tracer_alloc_buffers(void) 4540{ 4541 int ring_buf_size; 4542 int i; 4543 int ret = -ENOMEM; 4544 4545 if (!alloc_cpumask_var(&tracing_buffer_mask, GFP_KERNEL)) 4546 goto out; 4547 4548 if (!alloc_cpumask_var(&tracing_cpumask, GFP_KERNEL)) 4549 goto out_free_buffer_mask; 4550 4551 /* To save memory, keep the ring buffer size to its minimum */ 4552 if (ring_buffer_expanded) 4553 ring_buf_size = trace_buf_size; 4554 else 4555 ring_buf_size = 1; 4556 4557 cpumask_copy(tracing_buffer_mask, cpu_possible_mask); 4558 cpumask_copy(tracing_cpumask, cpu_all_mask); 4559 4560 /* TODO: make the number of buffers hot pluggable with CPUS */ 4561 global_trace.buffer = ring_buffer_alloc(ring_buf_size, 4562 TRACE_BUFFER_FLAGS); 4563 if (!global_trace.buffer) { 4564 printk(KERN_ERR "tracer: failed to allocate ring buffer!\n"); 4565 WARN_ON(1); 4566 goto out_free_cpumask; 4567 } 4568 global_trace.entries = ring_buffer_size(global_trace.buffer); 4569 4570 4571#ifdef CONFIG_TRACER_MAX_TRACE 4572 max_tr.buffer = ring_buffer_alloc(1, TRACE_BUFFER_FLAGS); 4573 if (!max_tr.buffer) { 4574 printk(KERN_ERR "tracer: failed to allocate max ring buffer!\n"); 4575 WARN_ON(1); 4576 ring_buffer_free(global_trace.buffer); 4577 goto out_free_cpumask; 4578 } 4579 max_tr.entries = 1; 4580#endif 4581 4582 /* Allocate the first page for all buffers */ 4583 for_each_tracing_cpu(i) { 4584 global_trace.data[i] = &per_cpu(global_trace_cpu, i); 4585 max_tr.data[i] = &per_cpu(max_tr_data, i); 4586 } 4587 4588 trace_init_cmdlines(); 4589 4590 register_tracer(&nop_trace); 4591 current_trace = &nop_trace; 4592 /* All seems OK, enable tracing */ 4593 tracing_disabled = 0; 4594 4595 atomic_notifier_chain_register(&panic_notifier_list, 4596 &trace_panic_notifier); 4597 4598 register_die_notifier(&trace_die_notifier); 4599 4600 return 0; 4601 4602out_free_cpumask: 4603 free_cpumask_var(tracing_cpumask); 4604out_free_buffer_mask: 4605 free_cpumask_var(tracing_buffer_mask); 4606out: 4607 return ret; 4608} 4609 4610__init static int clear_boot_tracer(void) 4611{ 4612 /* 4613 * The default tracer at boot buffer is an init section. 4614 * This function is called in lateinit. If we did not 4615 * find the boot tracer, then clear it out, to prevent 4616 * later registration from accessing the buffer that is 4617 * about to be freed. 4618 */ 4619 if (!default_bootup_tracer) 4620 return 0; 4621 4622 printk(KERN_INFO "ftrace bootup tracer '%s' not registered.\n", 4623 default_bootup_tracer); 4624 default_bootup_tracer = NULL; 4625 4626 return 0; 4627} 4628 4629early_initcall(tracer_alloc_buffers); 4630fs_initcall(tracer_init_debugfs); 4631late_initcall(clear_boot_tracer); 4632