1/* 2 * Copyright (C) 2008 Advanced Micro Devices, Inc. 3 * 4 * Author: Joerg Roedel <joerg.roedel@amd.com> 5 * 6 * This program is free software; you can redistribute it and/or modify it 7 * under the terms of the GNU General Public License version 2 as published 8 * by the Free Software Foundation. 9 * 10 * This program is distributed in the hope that it will be useful, 11 * but WITHOUT ANY WARRANTY; without even the implied warranty of 12 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 13 * GNU General Public License for more details. 14 * 15 * You should have received a copy of the GNU General Public License 16 * along with this program; if not, write to the Free Software 17 * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA 18 */ 19 20#include <linux/scatterlist.h> 21#include <linux/dma-mapping.h> 22#include <linux/stacktrace.h> 23#include <linux/dma-debug.h> 24#include <linux/spinlock.h> 25#include <linux/debugfs.h> 26#include <linux/uaccess.h> 27#include <linux/device.h> 28#include <linux/types.h> 29#include <linux/sched.h> 30#include <linux/ctype.h> 31#include <linux/list.h> 32#include <linux/slab.h> 33 34#include <asm/sections.h> 35 36#define HASH_SIZE 1024ULL 37#define HASH_FN_SHIFT 13 38#define HASH_FN_MASK (HASH_SIZE - 1) 39 40enum { 41 dma_debug_single, 42 dma_debug_page, 43 dma_debug_sg, 44 dma_debug_coherent, 45}; 46 47#define DMA_DEBUG_STACKTRACE_ENTRIES 5 48 49struct dma_debug_entry { 50 struct list_head list; 51 struct device *dev; 52 int type; 53 phys_addr_t paddr; 54 u64 dev_addr; 55 u64 size; 56 int direction; 57 int sg_call_ents; 58 int sg_mapped_ents; 59#ifdef CONFIG_STACKTRACE 60 struct stack_trace stacktrace; 61 unsigned long st_entries[DMA_DEBUG_STACKTRACE_ENTRIES]; 62#endif 63}; 64 65struct hash_bucket { 66 struct list_head list; 67 spinlock_t lock; 68} ____cacheline_aligned_in_smp; 69 70/* Hash list to save the allocated dma addresses */ 71static struct hash_bucket dma_entry_hash[HASH_SIZE]; 72/* List of pre-allocated dma_debug_entry's */ 73static LIST_HEAD(free_entries); 74/* Lock for the list above */ 75static DEFINE_SPINLOCK(free_entries_lock); 76 77/* Global disable flag - will be set in case of an error */ 78static bool global_disable __read_mostly; 79 80/* Global error count */ 81static u32 error_count; 82 83/* Global error show enable*/ 84static u32 show_all_errors __read_mostly; 85/* Number of errors to show */ 86static u32 show_num_errors = 1; 87 88static u32 num_free_entries; 89static u32 min_free_entries; 90static u32 nr_total_entries; 91 92/* number of preallocated entries requested by kernel cmdline */ 93static u32 req_entries; 94 95/* debugfs dentry's for the stuff above */ 96static struct dentry *dma_debug_dent __read_mostly; 97static struct dentry *global_disable_dent __read_mostly; 98static struct dentry *error_count_dent __read_mostly; 99static struct dentry *show_all_errors_dent __read_mostly; 100static struct dentry *show_num_errors_dent __read_mostly; 101static struct dentry *num_free_entries_dent __read_mostly; 102static struct dentry *min_free_entries_dent __read_mostly; 103static struct dentry *filter_dent __read_mostly; 104 105/* per-driver filter related state */ 106 107#define NAME_MAX_LEN 64 108 109static char current_driver_name[NAME_MAX_LEN] __read_mostly; 110static struct device_driver *current_driver __read_mostly; 111 112static DEFINE_RWLOCK(driver_name_lock); 113 114static const char *type2name[4] = { "single", "page", 115 "scather-gather", "coherent" }; 116 117static const char *dir2name[4] = { "DMA_BIDIRECTIONAL", "DMA_TO_DEVICE", 118 "DMA_FROM_DEVICE", "DMA_NONE" }; 119 120/* little merge helper - remove it after the merge window */ 121#ifndef BUS_NOTIFY_UNBOUND_DRIVER 122#define BUS_NOTIFY_UNBOUND_DRIVER 0x0005 123#endif 124 125/* 126 * The access to some variables in this macro is racy. We can't use atomic_t 127 * here because all these variables are exported to debugfs. Some of them even 128 * writeable. This is also the reason why a lock won't help much. But anyway, 129 * the races are no big deal. Here is why: 130 * 131 * error_count: the addition is racy, but the worst thing that can happen is 132 * that we don't count some errors 133 * show_num_errors: the subtraction is racy. Also no big deal because in 134 * worst case this will result in one warning more in the 135 * system log than the user configured. This variable is 136 * writeable via debugfs. 137 */ 138static inline void dump_entry_trace(struct dma_debug_entry *entry) 139{ 140#ifdef CONFIG_STACKTRACE 141 if (entry) { 142 pr_warning("Mapped at:\n"); 143 print_stack_trace(&entry->stacktrace, 0); 144 } 145#endif 146} 147 148static bool driver_filter(struct device *dev) 149{ 150 struct device_driver *drv; 151 unsigned long flags; 152 bool ret; 153 154 /* driver filter off */ 155 if (likely(!current_driver_name[0])) 156 return true; 157 158 /* driver filter on and initialized */ 159 if (current_driver && dev && dev->driver == current_driver) 160 return true; 161 162 /* driver filter on, but we can't filter on a NULL device... */ 163 if (!dev) 164 return false; 165 166 if (current_driver || !current_driver_name[0]) 167 return false; 168 169 /* driver filter on but not yet initialized */ 170 drv = get_driver(dev->driver); 171 if (!drv) 172 return false; 173 174 /* lock to protect against change of current_driver_name */ 175 read_lock_irqsave(&driver_name_lock, flags); 176 177 ret = false; 178 if (drv->name && 179 strncmp(current_driver_name, drv->name, NAME_MAX_LEN - 1) == 0) { 180 current_driver = drv; 181 ret = true; 182 } 183 184 read_unlock_irqrestore(&driver_name_lock, flags); 185 put_driver(drv); 186 187 return ret; 188} 189 190#define err_printk(dev, entry, format, arg...) do { \ 191 error_count += 1; \ 192 if (driver_filter(dev) && \ 193 (show_all_errors || show_num_errors > 0)) { \ 194 WARN(1, "%s %s: " format, \ 195 dev ? dev_driver_string(dev) : "NULL", \ 196 dev ? dev_name(dev) : "NULL", ## arg); \ 197 dump_entry_trace(entry); \ 198 } \ 199 if (!show_all_errors && show_num_errors > 0) \ 200 show_num_errors -= 1; \ 201 } while (0); 202 203/* 204 * Hash related functions 205 * 206 * Every DMA-API request is saved into a struct dma_debug_entry. To 207 * have quick access to these structs they are stored into a hash. 208 */ 209static int hash_fn(struct dma_debug_entry *entry) 210{ 211 /* 212 * Hash function is based on the dma address. 213 * We use bits 20-27 here as the index into the hash 214 */ 215 return (entry->dev_addr >> HASH_FN_SHIFT) & HASH_FN_MASK; 216} 217 218/* 219 * Request exclusive access to a hash bucket for a given dma_debug_entry. 220 */ 221static struct hash_bucket *get_hash_bucket(struct dma_debug_entry *entry, 222 unsigned long *flags) 223{ 224 int idx = hash_fn(entry); 225 unsigned long __flags; 226 227 spin_lock_irqsave(&dma_entry_hash[idx].lock, __flags); 228 *flags = __flags; 229 return &dma_entry_hash[idx]; 230} 231 232/* 233 * Give up exclusive access to the hash bucket 234 */ 235static void put_hash_bucket(struct hash_bucket *bucket, 236 unsigned long *flags) 237{ 238 unsigned long __flags = *flags; 239 240 spin_unlock_irqrestore(&bucket->lock, __flags); 241} 242 243/* 244 * Search a given entry in the hash bucket list 245 */ 246static struct dma_debug_entry *hash_bucket_find(struct hash_bucket *bucket, 247 struct dma_debug_entry *ref) 248{ 249 struct dma_debug_entry *entry, *ret = NULL; 250 int matches = 0, match_lvl, last_lvl = 0; 251 252 list_for_each_entry(entry, &bucket->list, list) { 253 if ((entry->dev_addr != ref->dev_addr) || 254 (entry->dev != ref->dev)) 255 continue; 256 257 /* 258 * Some drivers map the same physical address multiple 259 * times. Without a hardware IOMMU this results in the 260 * same device addresses being put into the dma-debug 261 * hash multiple times too. This can result in false 262 * positives being reported. Therefore we implement a 263 * best-fit algorithm here which returns the entry from 264 * the hash which fits best to the reference value 265 * instead of the first-fit. 266 */ 267 matches += 1; 268 match_lvl = 0; 269 entry->size == ref->size ? ++match_lvl : 0; 270 entry->type == ref->type ? ++match_lvl : 0; 271 entry->direction == ref->direction ? ++match_lvl : 0; 272 entry->sg_call_ents == ref->sg_call_ents ? ++match_lvl : 0; 273 274 if (match_lvl == 4) { 275 /* perfect-fit - return the result */ 276 return entry; 277 } else if (match_lvl > last_lvl) { 278 /* 279 * We found an entry that fits better then the 280 * previous one 281 */ 282 last_lvl = match_lvl; 283 ret = entry; 284 } 285 } 286 287 /* 288 * If we have multiple matches but no perfect-fit, just return 289 * NULL. 290 */ 291 ret = (matches == 1) ? ret : NULL; 292 293 return ret; 294} 295 296/* 297 * Add an entry to a hash bucket 298 */ 299static void hash_bucket_add(struct hash_bucket *bucket, 300 struct dma_debug_entry *entry) 301{ 302 list_add_tail(&entry->list, &bucket->list); 303} 304 305/* 306 * Remove entry from a hash bucket list 307 */ 308static void hash_bucket_del(struct dma_debug_entry *entry) 309{ 310 list_del(&entry->list); 311} 312 313/* 314 * Dump mapping entries for debugging purposes 315 */ 316void debug_dma_dump_mappings(struct device *dev) 317{ 318 int idx; 319 320 for (idx = 0; idx < HASH_SIZE; idx++) { 321 struct hash_bucket *bucket = &dma_entry_hash[idx]; 322 struct dma_debug_entry *entry; 323 unsigned long flags; 324 325 spin_lock_irqsave(&bucket->lock, flags); 326 327 list_for_each_entry(entry, &bucket->list, list) { 328 if (!dev || dev == entry->dev) { 329 dev_info(entry->dev, 330 "%s idx %d P=%Lx D=%Lx L=%Lx %s\n", 331 type2name[entry->type], idx, 332 (unsigned long long)entry->paddr, 333 entry->dev_addr, entry->size, 334 dir2name[entry->direction]); 335 } 336 } 337 338 spin_unlock_irqrestore(&bucket->lock, flags); 339 } 340} 341EXPORT_SYMBOL(debug_dma_dump_mappings); 342 343/* 344 * Wrapper function for adding an entry to the hash. 345 * This function takes care of locking itself. 346 */ 347static void add_dma_entry(struct dma_debug_entry *entry) 348{ 349 struct hash_bucket *bucket; 350 unsigned long flags; 351 352 bucket = get_hash_bucket(entry, &flags); 353 hash_bucket_add(bucket, entry); 354 put_hash_bucket(bucket, &flags); 355} 356 357static struct dma_debug_entry *__dma_entry_alloc(void) 358{ 359 struct dma_debug_entry *entry; 360 361 entry = list_entry(free_entries.next, struct dma_debug_entry, list); 362 list_del(&entry->list); 363 memset(entry, 0, sizeof(*entry)); 364 365 num_free_entries -= 1; 366 if (num_free_entries < min_free_entries) 367 min_free_entries = num_free_entries; 368 369 return entry; 370} 371 372/* struct dma_entry allocator 373 * 374 * The next two functions implement the allocator for 375 * struct dma_debug_entries. 376 */ 377static struct dma_debug_entry *dma_entry_alloc(void) 378{ 379 struct dma_debug_entry *entry = NULL; 380 unsigned long flags; 381 382 spin_lock_irqsave(&free_entries_lock, flags); 383 384 if (list_empty(&free_entries)) { 385 pr_err("DMA-API: debugging out of memory - disabling\n"); 386 global_disable = true; 387 goto out; 388 } 389 390 entry = __dma_entry_alloc(); 391 392#ifdef CONFIG_STACKTRACE 393 entry->stacktrace.max_entries = DMA_DEBUG_STACKTRACE_ENTRIES; 394 entry->stacktrace.entries = entry->st_entries; 395 entry->stacktrace.skip = 2; 396 save_stack_trace(&entry->stacktrace); 397#endif 398 399out: 400 spin_unlock_irqrestore(&free_entries_lock, flags); 401 402 return entry; 403} 404 405static void dma_entry_free(struct dma_debug_entry *entry) 406{ 407 unsigned long flags; 408 409 /* 410 * add to beginning of the list - this way the entries are 411 * more likely cache hot when they are reallocated. 412 */ 413 spin_lock_irqsave(&free_entries_lock, flags); 414 list_add(&entry->list, &free_entries); 415 num_free_entries += 1; 416 spin_unlock_irqrestore(&free_entries_lock, flags); 417} 418 419int dma_debug_resize_entries(u32 num_entries) 420{ 421 int i, delta, ret = 0; 422 unsigned long flags; 423 struct dma_debug_entry *entry; 424 LIST_HEAD(tmp); 425 426 spin_lock_irqsave(&free_entries_lock, flags); 427 428 if (nr_total_entries < num_entries) { 429 delta = num_entries - nr_total_entries; 430 431 spin_unlock_irqrestore(&free_entries_lock, flags); 432 433 for (i = 0; i < delta; i++) { 434 entry = kzalloc(sizeof(*entry), GFP_KERNEL); 435 if (!entry) 436 break; 437 438 list_add_tail(&entry->list, &tmp); 439 } 440 441 spin_lock_irqsave(&free_entries_lock, flags); 442 443 list_splice(&tmp, &free_entries); 444 nr_total_entries += i; 445 num_free_entries += i; 446 } else { 447 delta = nr_total_entries - num_entries; 448 449 for (i = 0; i < delta && !list_empty(&free_entries); i++) { 450 entry = __dma_entry_alloc(); 451 kfree(entry); 452 } 453 454 nr_total_entries -= i; 455 } 456 457 if (nr_total_entries != num_entries) 458 ret = 1; 459 460 spin_unlock_irqrestore(&free_entries_lock, flags); 461 462 return ret; 463} 464EXPORT_SYMBOL(dma_debug_resize_entries); 465 466/* 467 * DMA-API debugging init code 468 * 469 * The init code does two things: 470 * 1. Initialize core data structures 471 * 2. Preallocate a given number of dma_debug_entry structs 472 */ 473 474static int prealloc_memory(u32 num_entries) 475{ 476 struct dma_debug_entry *entry, *next_entry; 477 int i; 478 479 for (i = 0; i < num_entries; ++i) { 480 entry = kzalloc(sizeof(*entry), GFP_KERNEL); 481 if (!entry) 482 goto out_err; 483 484 list_add_tail(&entry->list, &free_entries); 485 } 486 487 num_free_entries = num_entries; 488 min_free_entries = num_entries; 489 490 pr_info("DMA-API: preallocated %d debug entries\n", num_entries); 491 492 return 0; 493 494out_err: 495 496 list_for_each_entry_safe(entry, next_entry, &free_entries, list) { 497 list_del(&entry->list); 498 kfree(entry); 499 } 500 501 return -ENOMEM; 502} 503 504static ssize_t filter_read(struct file *file, char __user *user_buf, 505 size_t count, loff_t *ppos) 506{ 507 char buf[NAME_MAX_LEN + 1]; 508 unsigned long flags; 509 int len; 510 511 if (!current_driver_name[0]) 512 return 0; 513 514 /* 515 * We can't copy to userspace directly because current_driver_name can 516 * only be read under the driver_name_lock with irqs disabled. So 517 * create a temporary copy first. 518 */ 519 read_lock_irqsave(&driver_name_lock, flags); 520 len = scnprintf(buf, NAME_MAX_LEN + 1, "%s\n", current_driver_name); 521 read_unlock_irqrestore(&driver_name_lock, flags); 522 523 return simple_read_from_buffer(user_buf, count, ppos, buf, len); 524} 525 526static ssize_t filter_write(struct file *file, const char __user *userbuf, 527 size_t count, loff_t *ppos) 528{ 529 char buf[NAME_MAX_LEN]; 530 unsigned long flags; 531 size_t len; 532 int i; 533 534 /* 535 * We can't copy from userspace directly. Access to 536 * current_driver_name is protected with a write_lock with irqs 537 * disabled. Since copy_from_user can fault and may sleep we 538 * need to copy to temporary buffer first 539 */ 540 len = min(count, (size_t)(NAME_MAX_LEN - 1)); 541 if (copy_from_user(buf, userbuf, len)) 542 return -EFAULT; 543 544 buf[len] = 0; 545 546 write_lock_irqsave(&driver_name_lock, flags); 547 548 /* 549 * Now handle the string we got from userspace very carefully. 550 * The rules are: 551 * - only use the first token we got 552 * - token delimiter is everything looking like a space 553 * character (' ', '\n', '\t' ...) 554 * 555 */ 556 if (!isalnum(buf[0])) { 557 /* 558 * If the first character userspace gave us is not 559 * alphanumerical then assume the filter should be 560 * switched off. 561 */ 562 if (current_driver_name[0]) 563 pr_info("DMA-API: switching off dma-debug driver filter\n"); 564 current_driver_name[0] = 0; 565 current_driver = NULL; 566 goto out_unlock; 567 } 568 569 /* 570 * Now parse out the first token and use it as the name for the 571 * driver to filter for. 572 */ 573 for (i = 0; i < NAME_MAX_LEN - 1; ++i) { 574 current_driver_name[i] = buf[i]; 575 if (isspace(buf[i]) || buf[i] == ' ' || buf[i] == 0) 576 break; 577 } 578 current_driver_name[i] = 0; 579 current_driver = NULL; 580 581 pr_info("DMA-API: enable driver filter for driver [%s]\n", 582 current_driver_name); 583 584out_unlock: 585 write_unlock_irqrestore(&driver_name_lock, flags); 586 587 return count; 588} 589 590static const struct file_operations filter_fops = { 591 .read = filter_read, 592 .write = filter_write, 593}; 594 595static int dma_debug_fs_init(void) 596{ 597 dma_debug_dent = debugfs_create_dir("dma-api", NULL); 598 if (!dma_debug_dent) { 599 pr_err("DMA-API: can not create debugfs directory\n"); 600 return -ENOMEM; 601 } 602 603 global_disable_dent = debugfs_create_bool("disabled", 0444, 604 dma_debug_dent, 605 (u32 *)&global_disable); 606 if (!global_disable_dent) 607 goto out_err; 608 609 error_count_dent = debugfs_create_u32("error_count", 0444, 610 dma_debug_dent, &error_count); 611 if (!error_count_dent) 612 goto out_err; 613 614 show_all_errors_dent = debugfs_create_u32("all_errors", 0644, 615 dma_debug_dent, 616 &show_all_errors); 617 if (!show_all_errors_dent) 618 goto out_err; 619 620 show_num_errors_dent = debugfs_create_u32("num_errors", 0644, 621 dma_debug_dent, 622 &show_num_errors); 623 if (!show_num_errors_dent) 624 goto out_err; 625 626 num_free_entries_dent = debugfs_create_u32("num_free_entries", 0444, 627 dma_debug_dent, 628 &num_free_entries); 629 if (!num_free_entries_dent) 630 goto out_err; 631 632 min_free_entries_dent = debugfs_create_u32("min_free_entries", 0444, 633 dma_debug_dent, 634 &min_free_entries); 635 if (!min_free_entries_dent) 636 goto out_err; 637 638 filter_dent = debugfs_create_file("driver_filter", 0644, 639 dma_debug_dent, NULL, &filter_fops); 640 if (!filter_dent) 641 goto out_err; 642 643 return 0; 644 645out_err: 646 debugfs_remove_recursive(dma_debug_dent); 647 648 return -ENOMEM; 649} 650 651static int device_dma_allocations(struct device *dev) 652{ 653 struct dma_debug_entry *entry; 654 unsigned long flags; 655 int count = 0, i; 656 657 local_irq_save(flags); 658 659 for (i = 0; i < HASH_SIZE; ++i) { 660 spin_lock(&dma_entry_hash[i].lock); 661 list_for_each_entry(entry, &dma_entry_hash[i].list, list) { 662 if (entry->dev == dev) 663 count += 1; 664 } 665 spin_unlock(&dma_entry_hash[i].lock); 666 } 667 668 local_irq_restore(flags); 669 670 return count; 671} 672 673static int dma_debug_device_change(struct notifier_block *nb, unsigned long action, void *data) 674{ 675 struct device *dev = data; 676 int count; 677 678 if (global_disable) 679 return 0; 680 681 switch (action) { 682 case BUS_NOTIFY_UNBOUND_DRIVER: 683 count = device_dma_allocations(dev); 684 if (count == 0) 685 break; 686 err_printk(dev, NULL, "DMA-API: device driver has pending " 687 "DMA allocations while released from device " 688 "[count=%d]\n", count); 689 break; 690 default: 691 break; 692 } 693 694 return 0; 695} 696 697void dma_debug_add_bus(struct bus_type *bus) 698{ 699 struct notifier_block *nb; 700 701 if (global_disable) 702 return; 703 704 nb = kzalloc(sizeof(struct notifier_block), GFP_KERNEL); 705 if (nb == NULL) { 706 pr_err("dma_debug_add_bus: out of memory\n"); 707 return; 708 } 709 710 nb->notifier_call = dma_debug_device_change; 711 712 bus_register_notifier(bus, nb); 713} 714 715/* 716 * Let the architectures decide how many entries should be preallocated. 717 */ 718void dma_debug_init(u32 num_entries) 719{ 720 int i; 721 722 if (global_disable) 723 return; 724 725 for (i = 0; i < HASH_SIZE; ++i) { 726 INIT_LIST_HEAD(&dma_entry_hash[i].list); 727 spin_lock_init(&dma_entry_hash[i].lock); 728 } 729 730 if (dma_debug_fs_init() != 0) { 731 pr_err("DMA-API: error creating debugfs entries - disabling\n"); 732 global_disable = true; 733 734 return; 735 } 736 737 if (req_entries) 738 num_entries = req_entries; 739 740 if (prealloc_memory(num_entries) != 0) { 741 pr_err("DMA-API: debugging out of memory error - disabled\n"); 742 global_disable = true; 743 744 return; 745 } 746 747 nr_total_entries = num_free_entries; 748 749 pr_info("DMA-API: debugging enabled by kernel config\n"); 750} 751 752static __init int dma_debug_cmdline(char *str) 753{ 754 if (!str) 755 return -EINVAL; 756 757 if (strncmp(str, "off", 3) == 0) { 758 pr_info("DMA-API: debugging disabled on kernel command line\n"); 759 global_disable = true; 760 } 761 762 return 0; 763} 764 765static __init int dma_debug_entries_cmdline(char *str) 766{ 767 int res; 768 769 if (!str) 770 return -EINVAL; 771 772 res = get_option(&str, &req_entries); 773 774 if (!res) 775 req_entries = 0; 776 777 return 0; 778} 779 780__setup("dma_debug=", dma_debug_cmdline); 781__setup("dma_debug_entries=", dma_debug_entries_cmdline); 782 783static void check_unmap(struct dma_debug_entry *ref) 784{ 785 struct dma_debug_entry *entry; 786 struct hash_bucket *bucket; 787 unsigned long flags; 788 789 if (dma_mapping_error(ref->dev, ref->dev_addr)) { 790 err_printk(ref->dev, NULL, "DMA-API: device driver tries " 791 "to free an invalid DMA memory address\n"); 792 return; 793 } 794 795 bucket = get_hash_bucket(ref, &flags); 796 entry = hash_bucket_find(bucket, ref); 797 798 if (!entry) { 799 err_printk(ref->dev, NULL, "DMA-API: device driver tries " 800 "to free DMA memory it has not allocated " 801 "[device address=0x%016llx] [size=%llu bytes]\n", 802 ref->dev_addr, ref->size); 803 goto out; 804 } 805 806 if (ref->size != entry->size) { 807 err_printk(ref->dev, entry, "DMA-API: device driver frees " 808 "DMA memory with different size " 809 "[device address=0x%016llx] [map size=%llu bytes] " 810 "[unmap size=%llu bytes]\n", 811 ref->dev_addr, entry->size, ref->size); 812 } 813 814 if (ref->type != entry->type) { 815 err_printk(ref->dev, entry, "DMA-API: device driver frees " 816 "DMA memory with wrong function " 817 "[device address=0x%016llx] [size=%llu bytes] " 818 "[mapped as %s] [unmapped as %s]\n", 819 ref->dev_addr, ref->size, 820 type2name[entry->type], type2name[ref->type]); 821 } else if ((entry->type == dma_debug_coherent) && 822 (ref->paddr != entry->paddr)) { 823 err_printk(ref->dev, entry, "DMA-API: device driver frees " 824 "DMA memory with different CPU address " 825 "[device address=0x%016llx] [size=%llu bytes] " 826 "[cpu alloc address=0x%016llx] " 827 "[cpu free address=0x%016llx]", 828 ref->dev_addr, ref->size, 829 (unsigned long long)entry->paddr, 830 (unsigned long long)ref->paddr); 831 } 832 833 if (ref->sg_call_ents && ref->type == dma_debug_sg && 834 ref->sg_call_ents != entry->sg_call_ents) { 835 err_printk(ref->dev, entry, "DMA-API: device driver frees " 836 "DMA sg list with different entry count " 837 "[map count=%d] [unmap count=%d]\n", 838 entry->sg_call_ents, ref->sg_call_ents); 839 } 840 841 /* 842 * This may be no bug in reality - but most implementations of the 843 * DMA API don't handle this properly, so check for it here 844 */ 845 if (ref->direction != entry->direction) { 846 err_printk(ref->dev, entry, "DMA-API: device driver frees " 847 "DMA memory with different direction " 848 "[device address=0x%016llx] [size=%llu bytes] " 849 "[mapped with %s] [unmapped with %s]\n", 850 ref->dev_addr, ref->size, 851 dir2name[entry->direction], 852 dir2name[ref->direction]); 853 } 854 855 hash_bucket_del(entry); 856 dma_entry_free(entry); 857 858out: 859 put_hash_bucket(bucket, &flags); 860} 861 862static void check_for_stack(struct device *dev, void *addr) 863{ 864 if (object_is_on_stack(addr)) 865 err_printk(dev, NULL, "DMA-API: device driver maps memory from" 866 "stack [addr=%p]\n", addr); 867} 868 869static inline bool overlap(void *addr, unsigned long len, void *start, void *end) 870{ 871 unsigned long a1 = (unsigned long)addr; 872 unsigned long b1 = a1 + len; 873 unsigned long a2 = (unsigned long)start; 874 unsigned long b2 = (unsigned long)end; 875 876 return !(b1 <= a2 || a1 >= b2); 877} 878 879static void check_for_illegal_area(struct device *dev, void *addr, unsigned long len) 880{ 881 if (overlap(addr, len, _text, _etext) || 882 overlap(addr, len, __start_rodata, __end_rodata)) 883 err_printk(dev, NULL, "DMA-API: device driver maps memory from kernel text or rodata [addr=%p] [len=%lu]\n", addr, len); 884} 885 886static void check_sync(struct device *dev, 887 struct dma_debug_entry *ref, 888 bool to_cpu) 889{ 890 struct dma_debug_entry *entry; 891 struct hash_bucket *bucket; 892 unsigned long flags; 893 894 bucket = get_hash_bucket(ref, &flags); 895 896 entry = hash_bucket_find(bucket, ref); 897 898 if (!entry) { 899 err_printk(dev, NULL, "DMA-API: device driver tries " 900 "to sync DMA memory it has not allocated " 901 "[device address=0x%016llx] [size=%llu bytes]\n", 902 (unsigned long long)ref->dev_addr, ref->size); 903 goto out; 904 } 905 906 if (ref->size > entry->size) { 907 err_printk(dev, entry, "DMA-API: device driver syncs" 908 " DMA memory outside allocated range " 909 "[device address=0x%016llx] " 910 "[allocation size=%llu bytes] " 911 "[sync offset+size=%llu]\n", 912 entry->dev_addr, entry->size, 913 ref->size); 914 } 915 916 if (entry->direction == DMA_BIDIRECTIONAL) 917 goto out; 918 919 if (ref->direction != entry->direction) { 920 err_printk(dev, entry, "DMA-API: device driver syncs " 921 "DMA memory with different direction " 922 "[device address=0x%016llx] [size=%llu bytes] " 923 "[mapped with %s] [synced with %s]\n", 924 (unsigned long long)ref->dev_addr, entry->size, 925 dir2name[entry->direction], 926 dir2name[ref->direction]); 927 } 928 929 if (to_cpu && !(entry->direction == DMA_FROM_DEVICE) && 930 !(ref->direction == DMA_TO_DEVICE)) 931 err_printk(dev, entry, "DMA-API: device driver syncs " 932 "device read-only DMA memory for cpu " 933 "[device address=0x%016llx] [size=%llu bytes] " 934 "[mapped with %s] [synced with %s]\n", 935 (unsigned long long)ref->dev_addr, entry->size, 936 dir2name[entry->direction], 937 dir2name[ref->direction]); 938 939 if (!to_cpu && !(entry->direction == DMA_TO_DEVICE) && 940 !(ref->direction == DMA_FROM_DEVICE)) 941 err_printk(dev, entry, "DMA-API: device driver syncs " 942 "device write-only DMA memory to device " 943 "[device address=0x%016llx] [size=%llu bytes] " 944 "[mapped with %s] [synced with %s]\n", 945 (unsigned long long)ref->dev_addr, entry->size, 946 dir2name[entry->direction], 947 dir2name[ref->direction]); 948 949out: 950 put_hash_bucket(bucket, &flags); 951} 952 953void debug_dma_map_page(struct device *dev, struct page *page, size_t offset, 954 size_t size, int direction, dma_addr_t dma_addr, 955 bool map_single) 956{ 957 struct dma_debug_entry *entry; 958 959 if (unlikely(global_disable)) 960 return; 961 962 if (unlikely(dma_mapping_error(dev, dma_addr))) 963 return; 964 965 entry = dma_entry_alloc(); 966 if (!entry) 967 return; 968 969 entry->dev = dev; 970 entry->type = dma_debug_page; 971 entry->paddr = page_to_phys(page) + offset; 972 entry->dev_addr = dma_addr; 973 entry->size = size; 974 entry->direction = direction; 975 976 if (map_single) 977 entry->type = dma_debug_single; 978 979 if (!PageHighMem(page)) { 980 void *addr = page_address(page) + offset; 981 982 check_for_stack(dev, addr); 983 check_for_illegal_area(dev, addr, size); 984 } 985 986 add_dma_entry(entry); 987} 988EXPORT_SYMBOL(debug_dma_map_page); 989 990void debug_dma_unmap_page(struct device *dev, dma_addr_t addr, 991 size_t size, int direction, bool map_single) 992{ 993 struct dma_debug_entry ref = { 994 .type = dma_debug_page, 995 .dev = dev, 996 .dev_addr = addr, 997 .size = size, 998 .direction = direction, 999 }; 1000 1001 if (unlikely(global_disable)) 1002 return; 1003 1004 if (map_single) 1005 ref.type = dma_debug_single; 1006 1007 check_unmap(&ref); 1008} 1009EXPORT_SYMBOL(debug_dma_unmap_page); 1010 1011void debug_dma_map_sg(struct device *dev, struct scatterlist *sg, 1012 int nents, int mapped_ents, int direction) 1013{ 1014 struct dma_debug_entry *entry; 1015 struct scatterlist *s; 1016 int i; 1017 1018 if (unlikely(global_disable)) 1019 return; 1020 1021 for_each_sg(sg, s, mapped_ents, i) { 1022 entry = dma_entry_alloc(); 1023 if (!entry) 1024 return; 1025 1026 entry->type = dma_debug_sg; 1027 entry->dev = dev; 1028 entry->paddr = sg_phys(s); 1029 entry->size = sg_dma_len(s); 1030 entry->dev_addr = sg_dma_address(s); 1031 entry->direction = direction; 1032 entry->sg_call_ents = nents; 1033 entry->sg_mapped_ents = mapped_ents; 1034 1035 if (!PageHighMem(sg_page(s))) { 1036 check_for_stack(dev, sg_virt(s)); 1037 check_for_illegal_area(dev, sg_virt(s), sg_dma_len(s)); 1038 } 1039 1040 add_dma_entry(entry); 1041 } 1042} 1043EXPORT_SYMBOL(debug_dma_map_sg); 1044 1045static int get_nr_mapped_entries(struct device *dev, 1046 struct dma_debug_entry *ref) 1047{ 1048 struct dma_debug_entry *entry; 1049 struct hash_bucket *bucket; 1050 unsigned long flags; 1051 int mapped_ents; 1052 1053 bucket = get_hash_bucket(ref, &flags); 1054 entry = hash_bucket_find(bucket, ref); 1055 mapped_ents = 0; 1056 1057 if (entry) 1058 mapped_ents = entry->sg_mapped_ents; 1059 put_hash_bucket(bucket, &flags); 1060 1061 return mapped_ents; 1062} 1063 1064void debug_dma_unmap_sg(struct device *dev, struct scatterlist *sglist, 1065 int nelems, int dir) 1066{ 1067 struct scatterlist *s; 1068 int mapped_ents = 0, i; 1069 1070 if (unlikely(global_disable)) 1071 return; 1072 1073 for_each_sg(sglist, s, nelems, i) { 1074 1075 struct dma_debug_entry ref = { 1076 .type = dma_debug_sg, 1077 .dev = dev, 1078 .paddr = sg_phys(s), 1079 .dev_addr = sg_dma_address(s), 1080 .size = sg_dma_len(s), 1081 .direction = dir, 1082 .sg_call_ents = nelems, 1083 }; 1084 1085 if (mapped_ents && i >= mapped_ents) 1086 break; 1087 1088 if (!i) 1089 mapped_ents = get_nr_mapped_entries(dev, &ref); 1090 1091 check_unmap(&ref); 1092 } 1093} 1094EXPORT_SYMBOL(debug_dma_unmap_sg); 1095 1096void debug_dma_alloc_coherent(struct device *dev, size_t size, 1097 dma_addr_t dma_addr, void *virt) 1098{ 1099 struct dma_debug_entry *entry; 1100 1101 if (unlikely(global_disable)) 1102 return; 1103 1104 if (unlikely(virt == NULL)) 1105 return; 1106 1107 entry = dma_entry_alloc(); 1108 if (!entry) 1109 return; 1110 1111 entry->type = dma_debug_coherent; 1112 entry->dev = dev; 1113 entry->paddr = virt_to_phys(virt); 1114 entry->size = size; 1115 entry->dev_addr = dma_addr; 1116 entry->direction = DMA_BIDIRECTIONAL; 1117 1118 add_dma_entry(entry); 1119} 1120EXPORT_SYMBOL(debug_dma_alloc_coherent); 1121 1122void debug_dma_free_coherent(struct device *dev, size_t size, 1123 void *virt, dma_addr_t addr) 1124{ 1125 struct dma_debug_entry ref = { 1126 .type = dma_debug_coherent, 1127 .dev = dev, 1128 .paddr = virt_to_phys(virt), 1129 .dev_addr = addr, 1130 .size = size, 1131 .direction = DMA_BIDIRECTIONAL, 1132 }; 1133 1134 if (unlikely(global_disable)) 1135 return; 1136 1137 check_unmap(&ref); 1138} 1139EXPORT_SYMBOL(debug_dma_free_coherent); 1140 1141void debug_dma_sync_single_for_cpu(struct device *dev, dma_addr_t dma_handle, 1142 size_t size, int direction) 1143{ 1144 struct dma_debug_entry ref; 1145 1146 if (unlikely(global_disable)) 1147 return; 1148 1149 ref.type = dma_debug_single; 1150 ref.dev = dev; 1151 ref.dev_addr = dma_handle; 1152 ref.size = size; 1153 ref.direction = direction; 1154 ref.sg_call_ents = 0; 1155 1156 check_sync(dev, &ref, true); 1157} 1158EXPORT_SYMBOL(debug_dma_sync_single_for_cpu); 1159 1160void debug_dma_sync_single_for_device(struct device *dev, 1161 dma_addr_t dma_handle, size_t size, 1162 int direction) 1163{ 1164 struct dma_debug_entry ref; 1165 1166 if (unlikely(global_disable)) 1167 return; 1168 1169 ref.type = dma_debug_single; 1170 ref.dev = dev; 1171 ref.dev_addr = dma_handle; 1172 ref.size = size; 1173 ref.direction = direction; 1174 ref.sg_call_ents = 0; 1175 1176 check_sync(dev, &ref, false); 1177} 1178EXPORT_SYMBOL(debug_dma_sync_single_for_device); 1179 1180void debug_dma_sync_single_range_for_cpu(struct device *dev, 1181 dma_addr_t dma_handle, 1182 unsigned long offset, size_t size, 1183 int direction) 1184{ 1185 struct dma_debug_entry ref; 1186 1187 if (unlikely(global_disable)) 1188 return; 1189 1190 ref.type = dma_debug_single; 1191 ref.dev = dev; 1192 ref.dev_addr = dma_handle; 1193 ref.size = offset + size; 1194 ref.direction = direction; 1195 ref.sg_call_ents = 0; 1196 1197 check_sync(dev, &ref, true); 1198} 1199EXPORT_SYMBOL(debug_dma_sync_single_range_for_cpu); 1200 1201void debug_dma_sync_single_range_for_device(struct device *dev, 1202 dma_addr_t dma_handle, 1203 unsigned long offset, 1204 size_t size, int direction) 1205{ 1206 struct dma_debug_entry ref; 1207 1208 if (unlikely(global_disable)) 1209 return; 1210 1211 ref.type = dma_debug_single; 1212 ref.dev = dev; 1213 ref.dev_addr = dma_handle; 1214 ref.size = offset + size; 1215 ref.direction = direction; 1216 ref.sg_call_ents = 0; 1217 1218 check_sync(dev, &ref, false); 1219} 1220EXPORT_SYMBOL(debug_dma_sync_single_range_for_device); 1221 1222void debug_dma_sync_sg_for_cpu(struct device *dev, struct scatterlist *sg, 1223 int nelems, int direction) 1224{ 1225 struct scatterlist *s; 1226 int mapped_ents = 0, i; 1227 1228 if (unlikely(global_disable)) 1229 return; 1230 1231 for_each_sg(sg, s, nelems, i) { 1232 1233 struct dma_debug_entry ref = { 1234 .type = dma_debug_sg, 1235 .dev = dev, 1236 .paddr = sg_phys(s), 1237 .dev_addr = sg_dma_address(s), 1238 .size = sg_dma_len(s), 1239 .direction = direction, 1240 .sg_call_ents = nelems, 1241 }; 1242 1243 if (!i) 1244 mapped_ents = get_nr_mapped_entries(dev, &ref); 1245 1246 if (i >= mapped_ents) 1247 break; 1248 1249 check_sync(dev, &ref, true); 1250 } 1251} 1252EXPORT_SYMBOL(debug_dma_sync_sg_for_cpu); 1253 1254void debug_dma_sync_sg_for_device(struct device *dev, struct scatterlist *sg, 1255 int nelems, int direction) 1256{ 1257 struct scatterlist *s; 1258 int mapped_ents = 0, i; 1259 1260 if (unlikely(global_disable)) 1261 return; 1262 1263 for_each_sg(sg, s, nelems, i) { 1264 1265 struct dma_debug_entry ref = { 1266 .type = dma_debug_sg, 1267 .dev = dev, 1268 .paddr = sg_phys(s), 1269 .dev_addr = sg_dma_address(s), 1270 .size = sg_dma_len(s), 1271 .direction = direction, 1272 .sg_call_ents = nelems, 1273 }; 1274 if (!i) 1275 mapped_ents = get_nr_mapped_entries(dev, &ref); 1276 1277 if (i >= mapped_ents) 1278 break; 1279 1280 check_sync(dev, &ref, false); 1281 } 1282} 1283EXPORT_SYMBOL(debug_dma_sync_sg_for_device); 1284 1285static int __init dma_debug_driver_setup(char *str) 1286{ 1287 int i; 1288 1289 for (i = 0; i < NAME_MAX_LEN - 1; ++i, ++str) { 1290 current_driver_name[i] = *str; 1291 if (*str == 0) 1292 break; 1293 } 1294 1295 if (current_driver_name[0]) 1296 pr_info("DMA-API: enable driver filter for driver [%s]\n", 1297 current_driver_name); 1298 1299 1300 return 1; 1301} 1302__setup("dma_debug_driver=", dma_debug_driver_setup); 1303