1/* 2 * Cell Broadband Engine OProfile Support 3 * 4 * (C) Copyright IBM Corporation 2006 5 * 6 * Author: Maynard Johnson <maynardj@us.ibm.com> 7 * 8 * This program is free software; you can redistribute it and/or 9 * modify it under the terms of the GNU General Public License 10 * as published by the Free Software Foundation; either version 11 * 2 of the License, or (at your option) any later version. 12 */ 13 14/* The purpose of this file is to handle SPU event task switching 15 * and to record SPU context information into the OProfile 16 * event buffer. 17 * 18 * Additionally, the spu_sync_buffer function is provided as a helper 19 * for recoding actual SPU program counter samples to the event buffer. 20 */ 21#include <linux/dcookies.h> 22#include <linux/kref.h> 23#include <linux/mm.h> 24#include <linux/fs.h> 25#include <linux/module.h> 26#include <linux/notifier.h> 27#include <linux/numa.h> 28#include <linux/oprofile.h> 29#include <linux/slab.h> 30#include <linux/spinlock.h> 31#include "pr_util.h" 32 33#define RELEASE_ALL 9999 34 35static DEFINE_SPINLOCK(buffer_lock); 36static DEFINE_SPINLOCK(cache_lock); 37static int num_spu_nodes; 38int spu_prof_num_nodes; 39 40struct spu_buffer spu_buff[MAX_NUMNODES * SPUS_PER_NODE]; 41struct delayed_work spu_work; 42static unsigned max_spu_buff; 43 44static void spu_buff_add(unsigned long int value, int spu) 45{ 46 /* spu buff is a circular buffer. Add entries to the 47 * head. Head is the index to store the next value. 48 * The buffer is full when there is one available entry 49 * in the queue, i.e. head and tail can't be equal. 50 * That way we can tell the difference between the 51 * buffer being full versus empty. 52 * 53 * ASSUPTION: the buffer_lock is held when this function 54 * is called to lock the buffer, head and tail. 55 */ 56 int full = 1; 57 58 if (spu_buff[spu].head >= spu_buff[spu].tail) { 59 if ((spu_buff[spu].head - spu_buff[spu].tail) 60 < (max_spu_buff - 1)) 61 full = 0; 62 63 } else if (spu_buff[spu].tail > spu_buff[spu].head) { 64 if ((spu_buff[spu].tail - spu_buff[spu].head) 65 > 1) 66 full = 0; 67 } 68 69 if (!full) { 70 spu_buff[spu].buff[spu_buff[spu].head] = value; 71 spu_buff[spu].head++; 72 73 if (spu_buff[spu].head >= max_spu_buff) 74 spu_buff[spu].head = 0; 75 } else { 76 /* From the user's perspective make the SPU buffer 77 * size management/overflow look like we are using 78 * per cpu buffers. The user uses the same 79 * per cpu parameter to adjust the SPU buffer size. 80 * Increment the sample_lost_overflow to inform 81 * the user the buffer size needs to be increased. 82 */ 83 oprofile_cpu_buffer_inc_smpl_lost(); 84 } 85} 86 87/* This function copies the per SPU buffers to the 88 * OProfile kernel buffer. 89 */ 90void sync_spu_buff(void) 91{ 92 int spu; 93 unsigned long flags; 94 int curr_head; 95 96 for (spu = 0; spu < num_spu_nodes; spu++) { 97 /* In case there was an issue and the buffer didn't 98 * get created skip it. 99 */ 100 if (spu_buff[spu].buff == NULL) 101 continue; 102 103 /* Hold the lock to make sure the head/tail 104 * doesn't change while spu_buff_add() is 105 * deciding if the buffer is full or not. 106 * Being a little paranoid. 107 */ 108 spin_lock_irqsave(&buffer_lock, flags); 109 curr_head = spu_buff[spu].head; 110 spin_unlock_irqrestore(&buffer_lock, flags); 111 112 /* Transfer the current contents to the kernel buffer. 113 * data can still be added to the head of the buffer. 114 */ 115 oprofile_put_buff(spu_buff[spu].buff, 116 spu_buff[spu].tail, 117 curr_head, max_spu_buff); 118 119 spin_lock_irqsave(&buffer_lock, flags); 120 spu_buff[spu].tail = curr_head; 121 spin_unlock_irqrestore(&buffer_lock, flags); 122 } 123 124} 125 126static void wq_sync_spu_buff(struct work_struct *work) 127{ 128 /* move data from spu buffers to kernel buffer */ 129 sync_spu_buff(); 130 131 /* only reschedule if profiling is not done */ 132 if (spu_prof_running) 133 schedule_delayed_work(&spu_work, DEFAULT_TIMER_EXPIRE); 134} 135 136/* Container for caching information about an active SPU task. */ 137struct cached_info { 138 struct vma_to_fileoffset_map *map; 139 struct spu *the_spu; /* needed to access pointer to local_store */ 140 struct kref cache_ref; 141}; 142 143static struct cached_info *spu_info[MAX_NUMNODES * 8]; 144 145static void destroy_cached_info(struct kref *kref) 146{ 147 struct cached_info *info; 148 149 info = container_of(kref, struct cached_info, cache_ref); 150 vma_map_free(info->map); 151 kfree(info); 152 module_put(THIS_MODULE); 153} 154 155/* Return the cached_info for the passed SPU number. 156 * ATTENTION: Callers are responsible for obtaining the 157 * cache_lock if needed prior to invoking this function. 158 */ 159static struct cached_info *get_cached_info(struct spu *the_spu, int spu_num) 160{ 161 struct kref *ref; 162 struct cached_info *ret_info; 163 164 if (spu_num >= num_spu_nodes) { 165 printk(KERN_ERR "SPU_PROF: " 166 "%s, line %d: Invalid index %d into spu info cache\n", 167 __func__, __LINE__, spu_num); 168 ret_info = NULL; 169 goto out; 170 } 171 if (!spu_info[spu_num] && the_spu) { 172 ref = spu_get_profile_private_kref(the_spu->ctx); 173 if (ref) { 174 spu_info[spu_num] = container_of(ref, struct cached_info, cache_ref); 175 kref_get(&spu_info[spu_num]->cache_ref); 176 } 177 } 178 179 ret_info = spu_info[spu_num]; 180 out: 181 return ret_info; 182} 183 184 185/* Looks for cached info for the passed spu. If not found, the 186 * cached info is created for the passed spu. 187 * Returns 0 for success; otherwise, -1 for error. 188 */ 189static int 190prepare_cached_spu_info(struct spu *spu, unsigned long objectId) 191{ 192 unsigned long flags; 193 struct vma_to_fileoffset_map *new_map; 194 int retval = 0; 195 struct cached_info *info; 196 197 /* We won't bother getting cache_lock here since 198 * don't do anything with the cached_info that's returned. 199 */ 200 info = get_cached_info(spu, spu->number); 201 202 if (info) { 203 pr_debug("Found cached SPU info.\n"); 204 goto out; 205 } 206 207 /* Create cached_info and set spu_info[spu->number] to point to it. 208 * spu->number is a system-wide value, not a per-node value. 209 */ 210 info = kzalloc(sizeof(struct cached_info), GFP_KERNEL); 211 if (!info) { 212 printk(KERN_ERR "SPU_PROF: " 213 "%s, line %d: create vma_map failed\n", 214 __func__, __LINE__); 215 retval = -ENOMEM; 216 goto err_alloc; 217 } 218 new_map = create_vma_map(spu, objectId); 219 if (!new_map) { 220 printk(KERN_ERR "SPU_PROF: " 221 "%s, line %d: create vma_map failed\n", 222 __func__, __LINE__); 223 retval = -ENOMEM; 224 goto err_alloc; 225 } 226 227 pr_debug("Created vma_map\n"); 228 info->map = new_map; 229 info->the_spu = spu; 230 kref_init(&info->cache_ref); 231 spin_lock_irqsave(&cache_lock, flags); 232 spu_info[spu->number] = info; 233 /* Increment count before passing off ref to SPUFS. */ 234 kref_get(&info->cache_ref); 235 236 /* We increment the module refcount here since SPUFS is 237 * responsible for the final destruction of the cached_info, 238 * and it must be able to access the destroy_cached_info() 239 * function defined in the OProfile module. We decrement 240 * the module refcount in destroy_cached_info. 241 */ 242 try_module_get(THIS_MODULE); 243 spu_set_profile_private_kref(spu->ctx, &info->cache_ref, 244 destroy_cached_info); 245 spin_unlock_irqrestore(&cache_lock, flags); 246 goto out; 247 248err_alloc: 249 kfree(info); 250out: 251 return retval; 252} 253 254/* 255 * NOTE: The caller is responsible for locking the 256 * cache_lock prior to calling this function. 257 */ 258static int release_cached_info(int spu_index) 259{ 260 int index, end; 261 262 if (spu_index == RELEASE_ALL) { 263 end = num_spu_nodes; 264 index = 0; 265 } else { 266 if (spu_index >= num_spu_nodes) { 267 printk(KERN_ERR "SPU_PROF: " 268 "%s, line %d: " 269 "Invalid index %d into spu info cache\n", 270 __func__, __LINE__, spu_index); 271 goto out; 272 } 273 end = spu_index + 1; 274 index = spu_index; 275 } 276 for (; index < end; index++) { 277 if (spu_info[index]) { 278 kref_put(&spu_info[index]->cache_ref, 279 destroy_cached_info); 280 spu_info[index] = NULL; 281 } 282 } 283 284out: 285 return 0; 286} 287 288/* The source code for fast_get_dcookie was "borrowed" 289 * from drivers/oprofile/buffer_sync.c. 290 */ 291 292/* Optimisation. We can manage without taking the dcookie sem 293 * because we cannot reach this code without at least one 294 * dcookie user still being registered (namely, the reader 295 * of the event buffer). 296 */ 297static inline unsigned long fast_get_dcookie(struct path *path) 298{ 299 unsigned long cookie; 300 301 if (path->dentry->d_flags & DCACHE_COOKIE) 302 return (unsigned long)path->dentry; 303 get_dcookie(path, &cookie); 304 return cookie; 305} 306 307/* Look up the dcookie for the task's first VM_EXECUTABLE mapping, 308 * which corresponds loosely to "application name". Also, determine 309 * the offset for the SPU ELF object. If computed offset is 310 * non-zero, it implies an embedded SPU object; otherwise, it's a 311 * separate SPU binary, in which case we retrieve it's dcookie. 312 * For the embedded case, we must determine if SPU ELF is embedded 313 * in the executable application or another file (i.e., shared lib). 314 * If embedded in a shared lib, we must get the dcookie and return 315 * that to the caller. 316 */ 317static unsigned long 318get_exec_dcookie_and_offset(struct spu *spu, unsigned int *offsetp, 319 unsigned long *spu_bin_dcookie, 320 unsigned long spu_ref) 321{ 322 unsigned long app_cookie = 0; 323 unsigned int my_offset = 0; 324 struct file *app = NULL; 325 struct vm_area_struct *vma; 326 struct mm_struct *mm = spu->mm; 327 328 if (!mm) 329 goto out; 330 331 down_read(&mm->mmap_sem); 332 333 for (vma = mm->mmap; vma; vma = vma->vm_next) { 334 if (!vma->vm_file) 335 continue; 336 if (!(vma->vm_flags & VM_EXECUTABLE)) 337 continue; 338 app_cookie = fast_get_dcookie(&vma->vm_file->f_path); 339 pr_debug("got dcookie for %s\n", 340 vma->vm_file->f_dentry->d_name.name); 341 app = vma->vm_file; 342 break; 343 } 344 345 for (vma = mm->mmap; vma; vma = vma->vm_next) { 346 if (vma->vm_start > spu_ref || vma->vm_end <= spu_ref) 347 continue; 348 my_offset = spu_ref - vma->vm_start; 349 if (!vma->vm_file) 350 goto fail_no_image_cookie; 351 352 pr_debug("Found spu ELF at %X(object-id:%lx) for file %s\n", 353 my_offset, spu_ref, 354 vma->vm_file->f_dentry->d_name.name); 355 *offsetp = my_offset; 356 break; 357 } 358 359 *spu_bin_dcookie = fast_get_dcookie(&vma->vm_file->f_path); 360 pr_debug("got dcookie for %s\n", vma->vm_file->f_dentry->d_name.name); 361 362 up_read(&mm->mmap_sem); 363 364out: 365 return app_cookie; 366 367fail_no_image_cookie: 368 up_read(&mm->mmap_sem); 369 370 printk(KERN_ERR "SPU_PROF: " 371 "%s, line %d: Cannot find dcookie for SPU binary\n", 372 __func__, __LINE__); 373 goto out; 374} 375 376 377 378/* This function finds or creates cached context information for the 379 * passed SPU and records SPU context information into the OProfile 380 * event buffer. 381 */ 382static int process_context_switch(struct spu *spu, unsigned long objectId) 383{ 384 unsigned long flags; 385 int retval; 386 unsigned int offset = 0; 387 unsigned long spu_cookie = 0, app_dcookie; 388 389 retval = prepare_cached_spu_info(spu, objectId); 390 if (retval) 391 goto out; 392 393 /* Get dcookie first because a mutex_lock is taken in that 394 * code path, so interrupts must not be disabled. 395 */ 396 app_dcookie = get_exec_dcookie_and_offset(spu, &offset, &spu_cookie, objectId); 397 if (!app_dcookie || !spu_cookie) { 398 retval = -ENOENT; 399 goto out; 400 } 401 402 /* Record context info in event buffer */ 403 spin_lock_irqsave(&buffer_lock, flags); 404 spu_buff_add(ESCAPE_CODE, spu->number); 405 spu_buff_add(SPU_CTX_SWITCH_CODE, spu->number); 406 spu_buff_add(spu->number, spu->number); 407 spu_buff_add(spu->pid, spu->number); 408 spu_buff_add(spu->tgid, spu->number); 409 spu_buff_add(app_dcookie, spu->number); 410 spu_buff_add(spu_cookie, spu->number); 411 spu_buff_add(offset, spu->number); 412 413 /* Set flag to indicate SPU PC data can now be written out. If 414 * the SPU program counter data is seen before an SPU context 415 * record is seen, the postprocessing will fail. 416 */ 417 spu_buff[spu->number].ctx_sw_seen = 1; 418 419 spin_unlock_irqrestore(&buffer_lock, flags); 420 smp_wmb(); /* insure spu event buffer updates are written */ 421 /* don't want entries intermingled... */ 422out: 423 return retval; 424} 425 426/* 427 * This function is invoked on either a bind_context or unbind_context. 428 * If called for an unbind_context, the val arg is 0; otherwise, 429 * it is the object-id value for the spu context. 430 * The data arg is of type 'struct spu *'. 431 */ 432static int spu_active_notify(struct notifier_block *self, unsigned long val, 433 void *data) 434{ 435 int retval; 436 unsigned long flags; 437 struct spu *the_spu = data; 438 439 pr_debug("SPU event notification arrived\n"); 440 if (!val) { 441 spin_lock_irqsave(&cache_lock, flags); 442 retval = release_cached_info(the_spu->number); 443 spin_unlock_irqrestore(&cache_lock, flags); 444 } else { 445 retval = process_context_switch(the_spu, val); 446 } 447 return retval; 448} 449 450static struct notifier_block spu_active = { 451 .notifier_call = spu_active_notify, 452}; 453 454static int number_of_online_nodes(void) 455{ 456 u32 cpu; u32 tmp; 457 int nodes = 0; 458 for_each_online_cpu(cpu) { 459 tmp = cbe_cpu_to_node(cpu) + 1; 460 if (tmp > nodes) 461 nodes++; 462 } 463 return nodes; 464} 465 466static int oprofile_spu_buff_create(void) 467{ 468 int spu; 469 470 max_spu_buff = oprofile_get_cpu_buffer_size(); 471 472 for (spu = 0; spu < num_spu_nodes; spu++) { 473 /* create circular buffers to store the data in. 474 * use locks to manage accessing the buffers 475 */ 476 spu_buff[spu].head = 0; 477 spu_buff[spu].tail = 0; 478 479 /* 480 * Create a buffer for each SPU. Can't reliably 481 * create a single buffer for all spus due to not 482 * enough contiguous kernel memory. 483 */ 484 485 spu_buff[spu].buff = kzalloc((max_spu_buff 486 * sizeof(unsigned long)), 487 GFP_KERNEL); 488 489 if (!spu_buff[spu].buff) { 490 printk(KERN_ERR "SPU_PROF: " 491 "%s, line %d: oprofile_spu_buff_create " 492 "failed to allocate spu buffer %d.\n", 493 __func__, __LINE__, spu); 494 495 /* release the spu buffers that have been allocated */ 496 while (spu >= 0) { 497 kfree(spu_buff[spu].buff); 498 spu_buff[spu].buff = 0; 499 spu--; 500 } 501 return -ENOMEM; 502 } 503 } 504 return 0; 505} 506 507/* The main purpose of this function is to synchronize 508 * OProfile with SPUFS by registering to be notified of 509 * SPU task switches. 510 * 511 * NOTE: When profiling SPUs, we must ensure that only 512 * spu_sync_start is invoked and not the generic sync_start 513 * in drivers/oprofile/oprof.c. A return value of 514 * SKIP_GENERIC_SYNC or SYNC_START_ERROR will 515 * accomplish this. 516 */ 517int spu_sync_start(void) 518{ 519 int spu; 520 int ret = SKIP_GENERIC_SYNC; 521 int register_ret; 522 unsigned long flags = 0; 523 524 spu_prof_num_nodes = number_of_online_nodes(); 525 num_spu_nodes = spu_prof_num_nodes * 8; 526 INIT_DELAYED_WORK(&spu_work, wq_sync_spu_buff); 527 528 /* create buffer for storing the SPU data to put in 529 * the kernel buffer. 530 */ 531 ret = oprofile_spu_buff_create(); 532 if (ret) 533 goto out; 534 535 spin_lock_irqsave(&buffer_lock, flags); 536 for (spu = 0; spu < num_spu_nodes; spu++) { 537 spu_buff_add(ESCAPE_CODE, spu); 538 spu_buff_add(SPU_PROFILING_CODE, spu); 539 spu_buff_add(num_spu_nodes, spu); 540 } 541 spin_unlock_irqrestore(&buffer_lock, flags); 542 543 for (spu = 0; spu < num_spu_nodes; spu++) { 544 spu_buff[spu].ctx_sw_seen = 0; 545 spu_buff[spu].last_guard_val = 0; 546 } 547 548 /* Register for SPU events */ 549 register_ret = spu_switch_event_register(&spu_active); 550 if (register_ret) { 551 ret = SYNC_START_ERROR; 552 goto out; 553 } 554 555 pr_debug("spu_sync_start -- running.\n"); 556out: 557 return ret; 558} 559 560/* Record SPU program counter samples to the oprofile event buffer. */ 561void spu_sync_buffer(int spu_num, unsigned int *samples, 562 int num_samples) 563{ 564 unsigned long long file_offset; 565 unsigned long flags; 566 int i; 567 struct vma_to_fileoffset_map *map; 568 struct spu *the_spu; 569 unsigned long long spu_num_ll = spu_num; 570 unsigned long long spu_num_shifted = spu_num_ll << 32; 571 struct cached_info *c_info; 572 573 /* We need to obtain the cache_lock here because it's 574 * possible that after getting the cached_info, the SPU job 575 * corresponding to this cached_info may end, thus resulting 576 * in the destruction of the cached_info. 577 */ 578 spin_lock_irqsave(&cache_lock, flags); 579 c_info = get_cached_info(NULL, spu_num); 580 if (!c_info) { 581 /* This legitimately happens when the SPU task ends before all 582 * samples are recorded. 583 * No big deal -- so we just drop a few samples. 584 */ 585 pr_debug("SPU_PROF: No cached SPU contex " 586 "for SPU #%d. Dropping samples.\n", spu_num); 587 goto out; 588 } 589 590 map = c_info->map; 591 the_spu = c_info->the_spu; 592 spin_lock(&buffer_lock); 593 for (i = 0; i < num_samples; i++) { 594 unsigned int sample = *(samples+i); 595 int grd_val = 0; 596 file_offset = 0; 597 if (sample == 0) 598 continue; 599 file_offset = vma_map_lookup( map, sample, the_spu, &grd_val); 600 601 /* If overlays are used by this SPU application, the guard 602 * value is non-zero, indicating which overlay section is in 603 * use. We need to discard samples taken during the time 604 * period which an overlay occurs (i.e., guard value changes). 605 */ 606 if (grd_val && grd_val != spu_buff[spu_num].last_guard_val) { 607 spu_buff[spu_num].last_guard_val = grd_val; 608 /* Drop the rest of the samples. */ 609 break; 610 } 611 612 /* We must ensure that the SPU context switch has been written 613 * out before samples for the SPU. Otherwise, the SPU context 614 * information is not available and the postprocessing of the 615 * SPU PC will fail with no available anonymous map information. 616 */ 617 if (spu_buff[spu_num].ctx_sw_seen) 618 spu_buff_add((file_offset | spu_num_shifted), 619 spu_num); 620 } 621 spin_unlock(&buffer_lock); 622out: 623 spin_unlock_irqrestore(&cache_lock, flags); 624} 625 626 627int spu_sync_stop(void) 628{ 629 unsigned long flags = 0; 630 int ret; 631 int k; 632 633 ret = spu_switch_event_unregister(&spu_active); 634 635 if (ret) 636 printk(KERN_ERR "SPU_PROF: " 637 "%s, line %d: spu_switch_event_unregister " \ 638 "returned %d\n", 639 __func__, __LINE__, ret); 640 641 /* flush any remaining data in the per SPU buffers */ 642 sync_spu_buff(); 643 644 spin_lock_irqsave(&cache_lock, flags); 645 ret = release_cached_info(RELEASE_ALL); 646 spin_unlock_irqrestore(&cache_lock, flags); 647 648 /* remove scheduled work queue item rather then waiting 649 * for every queued entry to execute. Then flush pending 650 * system wide buffer to event buffer. 651 */ 652 cancel_delayed_work(&spu_work); 653 654 for (k = 0; k < num_spu_nodes; k++) { 655 spu_buff[k].ctx_sw_seen = 0; 656 657 /* 658 * spu_sys_buff will be null if there was a problem 659 * allocating the buffer. Only delete if it exists. 660 */ 661 kfree(spu_buff[k].buff); 662 spu_buff[k].buff = 0; 663 } 664 pr_debug("spu_sync_stop -- done.\n"); 665 return ret; 666} 667