1/** 2 * @file buffer_sync.c 3 * 4 * @remark Copyright 2002-2009 OProfile authors 5 * @remark Read the file COPYING 6 * 7 * @author John Levon <levon@movementarian.org> 8 * @author Barry Kasindorf 9 * @author Robert Richter <robert.richter@amd.com> 10 * 11 * This is the core of the buffer management. Each 12 * CPU buffer is processed and entered into the 13 * global event buffer. Such processing is necessary 14 * in several circumstances, mentioned below. 15 * 16 * The processing does the job of converting the 17 * transitory EIP value into a persistent dentry/offset 18 * value that the profiler can record at its leisure. 19 * 20 * See fs/dcookies.c for a description of the dentry/offset 21 * objects. 22 */ 23 24#include <linux/mm.h> 25#include <linux/workqueue.h> 26#include <linux/notifier.h> 27#include <linux/dcookies.h> 28#include <linux/profile.h> 29#include <linux/module.h> 30#include <linux/fs.h> 31#include <linux/oprofile.h> 32#include <linux/sched.h> 33#include <linux/gfp.h> 34 35#include "oprofile_stats.h" 36#include "event_buffer.h" 37#include "cpu_buffer.h" 38#include "buffer_sync.h" 39 40static LIST_HEAD(dying_tasks); 41static LIST_HEAD(dead_tasks); 42static cpumask_var_t marked_cpus; 43static DEFINE_SPINLOCK(task_mortuary); 44static void process_task_mortuary(void); 45 46/* Take ownership of the task struct and place it on the 47 * list for processing. Only after two full buffer syncs 48 * does the task eventually get freed, because by then 49 * we are sure we will not reference it again. 50 * Can be invoked from softirq via RCU callback due to 51 * call_rcu() of the task struct, hence the _irqsave. 52 */ 53static int 54task_free_notify(struct notifier_block *self, unsigned long val, void *data) 55{ 56 unsigned long flags; 57 struct task_struct *task = data; 58 spin_lock_irqsave(&task_mortuary, flags); 59 list_add(&task->tasks, &dying_tasks); 60 spin_unlock_irqrestore(&task_mortuary, flags); 61 return NOTIFY_OK; 62} 63 64 65/* The task is on its way out. A sync of the buffer means we can catch 66 * any remaining samples for this task. 67 */ 68static int 69task_exit_notify(struct notifier_block *self, unsigned long val, void *data) 70{ 71 /* To avoid latency problems, we only process the current CPU, 72 * hoping that most samples for the task are on this CPU 73 */ 74 sync_buffer(raw_smp_processor_id()); 75 return 0; 76} 77 78 79/* The task is about to try a do_munmap(). We peek at what it's going to 80 * do, and if it's an executable region, process the samples first, so 81 * we don't lose any. This does not have to be exact, it's a QoI issue 82 * only. 83 */ 84static int 85munmap_notify(struct notifier_block *self, unsigned long val, void *data) 86{ 87 unsigned long addr = (unsigned long)data; 88 struct mm_struct *mm = current->mm; 89 struct vm_area_struct *mpnt; 90 91 down_read(&mm->mmap_sem); 92 93 mpnt = find_vma(mm, addr); 94 if (mpnt && mpnt->vm_file && (mpnt->vm_flags & VM_EXEC)) { 95 up_read(&mm->mmap_sem); 96 /* To avoid latency problems, we only process the current CPU, 97 * hoping that most samples for the task are on this CPU 98 */ 99 sync_buffer(raw_smp_processor_id()); 100 return 0; 101 } 102 103 up_read(&mm->mmap_sem); 104 return 0; 105} 106 107 108/* We need to be told about new modules so we don't attribute to a previously 109 * loaded module, or drop the samples on the floor. 110 */ 111static int 112module_load_notify(struct notifier_block *self, unsigned long val, void *data) 113{ 114#ifdef CONFIG_MODULES 115 if (val != MODULE_STATE_COMING) 116 return 0; 117 118 mutex_lock(&buffer_mutex); 119 add_event_entry(ESCAPE_CODE); 120 add_event_entry(MODULE_LOADED_CODE); 121 mutex_unlock(&buffer_mutex); 122#endif 123 return 0; 124} 125 126 127static struct notifier_block task_free_nb = { 128 .notifier_call = task_free_notify, 129}; 130 131static struct notifier_block task_exit_nb = { 132 .notifier_call = task_exit_notify, 133}; 134 135static struct notifier_block munmap_nb = { 136 .notifier_call = munmap_notify, 137}; 138 139static struct notifier_block module_load_nb = { 140 .notifier_call = module_load_notify, 141}; 142 143int sync_start(void) 144{ 145 int err; 146 147 if (!zalloc_cpumask_var(&marked_cpus, GFP_KERNEL)) 148 return -ENOMEM; 149 150 mutex_lock(&buffer_mutex); 151 152 err = task_handoff_register(&task_free_nb); 153 if (err) 154 goto out1; 155 err = profile_event_register(PROFILE_TASK_EXIT, &task_exit_nb); 156 if (err) 157 goto out2; 158 err = profile_event_register(PROFILE_MUNMAP, &munmap_nb); 159 if (err) 160 goto out3; 161 err = register_module_notifier(&module_load_nb); 162 if (err) 163 goto out4; 164 165 start_cpu_work(); 166 167out: 168 mutex_unlock(&buffer_mutex); 169 return err; 170out4: 171 profile_event_unregister(PROFILE_MUNMAP, &munmap_nb); 172out3: 173 profile_event_unregister(PROFILE_TASK_EXIT, &task_exit_nb); 174out2: 175 task_handoff_unregister(&task_free_nb); 176out1: 177 free_cpumask_var(marked_cpus); 178 goto out; 179} 180 181 182void sync_stop(void) 183{ 184 /* flush buffers */ 185 mutex_lock(&buffer_mutex); 186 end_cpu_work(); 187 unregister_module_notifier(&module_load_nb); 188 profile_event_unregister(PROFILE_MUNMAP, &munmap_nb); 189 profile_event_unregister(PROFILE_TASK_EXIT, &task_exit_nb); 190 task_handoff_unregister(&task_free_nb); 191 mutex_unlock(&buffer_mutex); 192 flush_scheduled_work(); 193 194 /* make sure we don't leak task structs */ 195 process_task_mortuary(); 196 process_task_mortuary(); 197 198 free_cpumask_var(marked_cpus); 199} 200 201 202/* Optimisation. We can manage without taking the dcookie sem 203 * because we cannot reach this code without at least one 204 * dcookie user still being registered (namely, the reader 205 * of the event buffer). */ 206static inline unsigned long fast_get_dcookie(struct path *path) 207{ 208 unsigned long cookie; 209 210 if (path->dentry->d_flags & DCACHE_COOKIE) 211 return (unsigned long)path->dentry; 212 get_dcookie(path, &cookie); 213 return cookie; 214} 215 216 217/* Look up the dcookie for the task's first VM_EXECUTABLE mapping, 218 * which corresponds loosely to "application name". This is 219 * not strictly necessary but allows oprofile to associate 220 * shared-library samples with particular applications 221 */ 222static unsigned long get_exec_dcookie(struct mm_struct *mm) 223{ 224 unsigned long cookie = NO_COOKIE; 225 struct vm_area_struct *vma; 226 227 if (!mm) 228 goto out; 229 230 for (vma = mm->mmap; vma; vma = vma->vm_next) { 231 if (!vma->vm_file) 232 continue; 233 if (!(vma->vm_flags & VM_EXECUTABLE)) 234 continue; 235 cookie = fast_get_dcookie(&vma->vm_file->f_path); 236 break; 237 } 238 239out: 240 return cookie; 241} 242 243 244/* Convert the EIP value of a sample into a persistent dentry/offset 245 * pair that can then be added to the global event buffer. We make 246 * sure to do this lookup before a mm->mmap modification happens so 247 * we don't lose track. 248 */ 249static unsigned long 250lookup_dcookie(struct mm_struct *mm, unsigned long addr, off_t *offset) 251{ 252 unsigned long cookie = NO_COOKIE; 253 struct vm_area_struct *vma; 254 255 for (vma = find_vma(mm, addr); vma; vma = vma->vm_next) { 256 257 if (addr < vma->vm_start || addr >= vma->vm_end) 258 continue; 259 260 if (vma->vm_file) { 261 cookie = fast_get_dcookie(&vma->vm_file->f_path); 262 *offset = (vma->vm_pgoff << PAGE_SHIFT) + addr - 263 vma->vm_start; 264 } else { 265 /* must be an anonymous map */ 266 *offset = addr; 267 } 268 269 break; 270 } 271 272 if (!vma) 273 cookie = INVALID_COOKIE; 274 275 return cookie; 276} 277 278static unsigned long last_cookie = INVALID_COOKIE; 279 280static void add_cpu_switch(int i) 281{ 282 add_event_entry(ESCAPE_CODE); 283 add_event_entry(CPU_SWITCH_CODE); 284 add_event_entry(i); 285 last_cookie = INVALID_COOKIE; 286} 287 288static void add_kernel_ctx_switch(unsigned int in_kernel) 289{ 290 add_event_entry(ESCAPE_CODE); 291 if (in_kernel) 292 add_event_entry(KERNEL_ENTER_SWITCH_CODE); 293 else 294 add_event_entry(KERNEL_EXIT_SWITCH_CODE); 295} 296 297static void 298add_user_ctx_switch(struct task_struct const *task, unsigned long cookie) 299{ 300 add_event_entry(ESCAPE_CODE); 301 add_event_entry(CTX_SWITCH_CODE); 302 add_event_entry(task->pid); 303 add_event_entry(cookie); 304 /* Another code for daemon back-compat */ 305 add_event_entry(ESCAPE_CODE); 306 add_event_entry(CTX_TGID_CODE); 307 add_event_entry(task->tgid); 308} 309 310 311static void add_cookie_switch(unsigned long cookie) 312{ 313 add_event_entry(ESCAPE_CODE); 314 add_event_entry(COOKIE_SWITCH_CODE); 315 add_event_entry(cookie); 316} 317 318 319static void add_trace_begin(void) 320{ 321 add_event_entry(ESCAPE_CODE); 322 add_event_entry(TRACE_BEGIN_CODE); 323} 324 325static void add_data(struct op_entry *entry, struct mm_struct *mm) 326{ 327 unsigned long code, pc, val; 328 unsigned long cookie; 329 off_t offset; 330 331 if (!op_cpu_buffer_get_data(entry, &code)) 332 return; 333 if (!op_cpu_buffer_get_data(entry, &pc)) 334 return; 335 if (!op_cpu_buffer_get_size(entry)) 336 return; 337 338 if (mm) { 339 cookie = lookup_dcookie(mm, pc, &offset); 340 341 if (cookie == NO_COOKIE) 342 offset = pc; 343 if (cookie == INVALID_COOKIE) { 344 atomic_inc(&oprofile_stats.sample_lost_no_mapping); 345 offset = pc; 346 } 347 if (cookie != last_cookie) { 348 add_cookie_switch(cookie); 349 last_cookie = cookie; 350 } 351 } else 352 offset = pc; 353 354 add_event_entry(ESCAPE_CODE); 355 add_event_entry(code); 356 add_event_entry(offset); /* Offset from Dcookie */ 357 358 while (op_cpu_buffer_get_data(entry, &val)) 359 add_event_entry(val); 360} 361 362static inline void add_sample_entry(unsigned long offset, unsigned long event) 363{ 364 add_event_entry(offset); 365 add_event_entry(event); 366} 367 368 369/* 370 * Add a sample to the global event buffer. If possible the 371 * sample is converted into a persistent dentry/offset pair 372 * for later lookup from userspace. Return 0 on failure. 373 */ 374static int 375add_sample(struct mm_struct *mm, struct op_sample *s, int in_kernel) 376{ 377 unsigned long cookie; 378 off_t offset; 379 380 if (in_kernel) { 381 add_sample_entry(s->eip, s->event); 382 return 1; 383 } 384 385 /* add userspace sample */ 386 387 if (!mm) { 388 atomic_inc(&oprofile_stats.sample_lost_no_mm); 389 return 0; 390 } 391 392 cookie = lookup_dcookie(mm, s->eip, &offset); 393 394 if (cookie == INVALID_COOKIE) { 395 atomic_inc(&oprofile_stats.sample_lost_no_mapping); 396 return 0; 397 } 398 399 if (cookie != last_cookie) { 400 add_cookie_switch(cookie); 401 last_cookie = cookie; 402 } 403 404 add_sample_entry(offset, s->event); 405 406 return 1; 407} 408 409 410static void release_mm(struct mm_struct *mm) 411{ 412 if (!mm) 413 return; 414 up_read(&mm->mmap_sem); 415 mmput(mm); 416} 417 418 419static struct mm_struct *take_tasks_mm(struct task_struct *task) 420{ 421 struct mm_struct *mm = get_task_mm(task); 422 if (mm) 423 down_read(&mm->mmap_sem); 424 return mm; 425} 426 427 428static inline int is_code(unsigned long val) 429{ 430 return val == ESCAPE_CODE; 431} 432 433 434/* Move tasks along towards death. Any tasks on dead_tasks 435 * will definitely have no remaining references in any 436 * CPU buffers at this point, because we use two lists, 437 * and to have reached the list, it must have gone through 438 * one full sync already. 439 */ 440static void process_task_mortuary(void) 441{ 442 unsigned long flags; 443 LIST_HEAD(local_dead_tasks); 444 struct task_struct *task; 445 struct task_struct *ttask; 446 447 spin_lock_irqsave(&task_mortuary, flags); 448 449 list_splice_init(&dead_tasks, &local_dead_tasks); 450 list_splice_init(&dying_tasks, &dead_tasks); 451 452 spin_unlock_irqrestore(&task_mortuary, flags); 453 454 list_for_each_entry_safe(task, ttask, &local_dead_tasks, tasks) { 455 list_del(&task->tasks); 456 free_task(task); 457 } 458} 459 460 461static void mark_done(int cpu) 462{ 463 int i; 464 465 cpumask_set_cpu(cpu, marked_cpus); 466 467 for_each_online_cpu(i) { 468 if (!cpumask_test_cpu(i, marked_cpus)) 469 return; 470 } 471 472 /* All CPUs have been processed at least once, 473 * we can process the mortuary once 474 */ 475 process_task_mortuary(); 476 477 cpumask_clear(marked_cpus); 478} 479 480 481typedef enum { 482 sb_bt_ignore = -2, 483 sb_buffer_start, 484 sb_bt_start, 485 sb_sample_start, 486} sync_buffer_state; 487 488/* Sync one of the CPU's buffers into the global event buffer. 489 * Here we need to go through each batch of samples punctuated 490 * by context switch notes, taking the task's mmap_sem and doing 491 * lookup in task->mm->mmap to convert EIP into dcookie/offset 492 * value. 493 */ 494void sync_buffer(int cpu) 495{ 496 struct mm_struct *mm = NULL; 497 struct mm_struct *oldmm; 498 unsigned long val; 499 struct task_struct *new; 500 unsigned long cookie = 0; 501 int in_kernel = 1; 502 sync_buffer_state state = sb_buffer_start; 503 unsigned int i; 504 unsigned long available; 505 unsigned long flags; 506 struct op_entry entry; 507 struct op_sample *sample; 508 509 mutex_lock(&buffer_mutex); 510 511 add_cpu_switch(cpu); 512 513 op_cpu_buffer_reset(cpu); 514 available = op_cpu_buffer_entries(cpu); 515 516 for (i = 0; i < available; ++i) { 517 sample = op_cpu_buffer_read_entry(&entry, cpu); 518 if (!sample) 519 break; 520 521 if (is_code(sample->eip)) { 522 flags = sample->event; 523 if (flags & TRACE_BEGIN) { 524 state = sb_bt_start; 525 add_trace_begin(); 526 } 527 if (flags & KERNEL_CTX_SWITCH) { 528 /* kernel/userspace switch */ 529 in_kernel = flags & IS_KERNEL; 530 if (state == sb_buffer_start) 531 state = sb_sample_start; 532 add_kernel_ctx_switch(flags & IS_KERNEL); 533 } 534 if (flags & USER_CTX_SWITCH 535 && op_cpu_buffer_get_data(&entry, &val)) { 536 /* userspace context switch */ 537 new = (struct task_struct *)val; 538 oldmm = mm; 539 release_mm(oldmm); 540 mm = take_tasks_mm(new); 541 if (mm != oldmm) 542 cookie = get_exec_dcookie(mm); 543 add_user_ctx_switch(new, cookie); 544 } 545 if (op_cpu_buffer_get_size(&entry)) 546 add_data(&entry, mm); 547 continue; 548 } 549 550 if (state < sb_bt_start) 551 /* ignore sample */ 552 continue; 553 554 if (add_sample(mm, sample, in_kernel)) 555 continue; 556 557 /* ignore backtraces if failed to add a sample */ 558 if (state == sb_bt_start) { 559 state = sb_bt_ignore; 560 atomic_inc(&oprofile_stats.bt_lost_no_mapping); 561 } 562 } 563 release_mm(mm); 564 565 mark_done(cpu); 566 567 mutex_unlock(&buffer_mutex); 568} 569 570/* The function can be used to add a buffer worth of data directly to 571 * the kernel buffer. The buffer is assumed to be a circular buffer. 572 * Take the entries from index start and end at index end, wrapping 573 * at max_entries. 574 */ 575void oprofile_put_buff(unsigned long *buf, unsigned int start, 576 unsigned int stop, unsigned int max) 577{ 578 int i; 579 580 i = start; 581 582 mutex_lock(&buffer_mutex); 583 while (i != stop) { 584 add_event_entry(buf[i++]); 585 586 if (i >= max) 587 i = 0; 588 } 589 590 mutex_unlock(&buffer_mutex); 591} 592