1// SPDX-License-Identifier: GPL-2.0 OR MIT 2/* 3 * Copyright 2020 Advanced Micro Devices, Inc. 4 * 5 * Permission is hereby granted, free of charge, to any person obtaining a 6 * copy of this software and associated documentation files (the "Software"), 7 * to deal in the Software without restriction, including without limitation 8 * the rights to use, copy, modify, merge, publish, distribute, sublicense, 9 * and/or sell copies of the Software, and to permit persons to whom the 10 * Software is furnished to do so, subject to the following conditions: 11 * 12 * The above copyright notice and this permission notice shall be included in 13 * all copies or substantial portions of the Software. 14 * 15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 18 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR 19 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, 20 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR 21 * OTHER DEALINGS IN THE SOFTWARE. 22 * 23 * Authors: Christian K��nig 24 */ 25 26/* Pooling of allocated pages is necessary because changing the caching 27 * attributes on x86 of the linear mapping requires a costly cross CPU TLB 28 * invalidate for those addresses. 29 * 30 * Additional to that allocations from the DMA coherent API are pooled as well 31 * cause they are rather slow compared to alloc_pages+map. 32 */ 33 34#include <linux/module.h> 35#include <linux/dma-mapping.h> 36#include <linux/debugfs.h> 37#include <linux/highmem.h> 38#include <linux/sched/mm.h> 39 40#ifdef CONFIG_X86 41#include <asm/set_memory.h> 42#endif 43 44#include <drm/ttm/ttm_pool.h> 45#include <drm/ttm/ttm_tt.h> 46#include <drm/ttm/ttm_bo.h> 47 48#include "ttm_module.h" 49 50/** 51 * struct ttm_pool_dma - Helper object for coherent DMA mappings 52 * 53 * @addr: original DMA address returned for the mapping 54 * @vaddr: original vaddr return for the mapping and order in the lower bits 55 */ 56struct ttm_pool_dma { 57 dma_addr_t addr; 58 unsigned long vaddr; 59}; 60 61static unsigned long page_pool_size; 62 63MODULE_PARM_DESC(page_pool_size, "Number of pages in the WC/UC/DMA pool"); 64module_param(page_pool_size, ulong, 0644); 65 66static atomic_long_t allocated_pages; 67 68static struct ttm_pool_type global_write_combined[NR_PAGE_ORDERS]; 69static struct ttm_pool_type global_uncached[NR_PAGE_ORDERS]; 70 71static struct ttm_pool_type global_dma32_write_combined[NR_PAGE_ORDERS]; 72static struct ttm_pool_type global_dma32_uncached[NR_PAGE_ORDERS]; 73 74static spinlock_t shrinker_lock; 75static struct list_head shrinker_list; 76static struct shrinker *mm_shrinker; 77static DECLARE_RWSEM(pool_shrink_rwsem); 78 79/* Allocate pages of size 1 << order with the given gfp_flags */ 80static struct page *ttm_pool_alloc_page(struct ttm_pool *pool, gfp_t gfp_flags, 81 unsigned int order) 82{ 83 unsigned long attr = DMA_ATTR_FORCE_CONTIGUOUS; 84 struct ttm_pool_dma *dma; 85 struct page *p; 86 void *vaddr; 87 88 /* Don't set the __GFP_COMP flag for higher order allocations. 89 * Mapping pages directly into an userspace process and calling 90 * put_page() on a TTM allocated page is illegal. 91 */ 92 if (order) 93 gfp_flags |= __GFP_NOMEMALLOC | __GFP_NORETRY | __GFP_NOWARN | 94 __GFP_KSWAPD_RECLAIM; 95 96 if (!pool->use_dma_alloc) { 97 p = alloc_pages_node(pool->nid, gfp_flags, order); 98 if (p) 99 p->private = order; 100 return p; 101 } 102 103 dma = kmalloc(sizeof(*dma), GFP_KERNEL); 104 if (!dma) 105 return NULL; 106 107 if (order) 108 attr |= DMA_ATTR_NO_WARN; 109 110 vaddr = dma_alloc_attrs(pool->dev, (1ULL << order) * PAGE_SIZE, 111 &dma->addr, gfp_flags, attr); 112 if (!vaddr) 113 goto error_free; 114 115 /* TODO: This is an illegal abuse of the DMA API, but we need to rework 116 * TTM page fault handling and extend the DMA API to clean this up. 117 */ 118 if (is_vmalloc_addr(vaddr)) 119 p = vmalloc_to_page(vaddr); 120 else 121 p = virt_to_page(vaddr); 122 123 dma->vaddr = (unsigned long)vaddr | order; 124 p->private = (unsigned long)dma; 125 return p; 126 127error_free: 128 kfree(dma); 129 return NULL; 130} 131 132/* Reset the caching and pages of size 1 << order */ 133static void ttm_pool_free_page(struct ttm_pool *pool, enum ttm_caching caching, 134 unsigned int order, struct page *p) 135{ 136 unsigned long attr = DMA_ATTR_FORCE_CONTIGUOUS; 137 struct ttm_pool_dma *dma; 138 void *vaddr; 139 140#ifdef CONFIG_X86 141 /* We don't care that set_pages_wb is inefficient here. This is only 142 * used when we have to shrink and CPU overhead is irrelevant then. 143 */ 144 if (caching != ttm_cached && !PageHighMem(p)) 145 set_pages_wb(p, 1 << order); 146#endif 147 148 if (!pool || !pool->use_dma_alloc) { 149 __free_pages(p, order); 150 return; 151 } 152 153 if (order) 154 attr |= DMA_ATTR_NO_WARN; 155 156 dma = (void *)p->private; 157 vaddr = (void *)(dma->vaddr & PAGE_MASK); 158 dma_free_attrs(pool->dev, (1UL << order) * PAGE_SIZE, vaddr, dma->addr, 159 attr); 160 kfree(dma); 161} 162 163/* Apply a new caching to an array of pages */ 164static int ttm_pool_apply_caching(struct page **first, struct page **last, 165 enum ttm_caching caching) 166{ 167#ifdef CONFIG_X86 168 unsigned int num_pages = last - first; 169 170 if (!num_pages) 171 return 0; 172 173 switch (caching) { 174 case ttm_cached: 175 break; 176 case ttm_write_combined: 177 return set_pages_array_wc(first, num_pages); 178 case ttm_uncached: 179 return set_pages_array_uc(first, num_pages); 180 } 181#endif 182 return 0; 183} 184 185/* Map pages of 1 << order size and fill the DMA address array */ 186static int ttm_pool_map(struct ttm_pool *pool, unsigned int order, 187 struct page *p, dma_addr_t **dma_addr) 188{ 189 dma_addr_t addr; 190 unsigned int i; 191 192 if (pool->use_dma_alloc) { 193 struct ttm_pool_dma *dma = (void *)p->private; 194 195 addr = dma->addr; 196 } else { 197 size_t size = (1ULL << order) * PAGE_SIZE; 198 199 addr = dma_map_page(pool->dev, p, 0, size, DMA_BIDIRECTIONAL); 200 if (dma_mapping_error(pool->dev, addr)) 201 return -EFAULT; 202 } 203 204 for (i = 1 << order; i ; --i) { 205 *(*dma_addr)++ = addr; 206 addr += PAGE_SIZE; 207 } 208 209 return 0; 210} 211 212/* Unmap pages of 1 << order size */ 213static void ttm_pool_unmap(struct ttm_pool *pool, dma_addr_t dma_addr, 214 unsigned int num_pages) 215{ 216 /* Unmapped while freeing the page */ 217 if (pool->use_dma_alloc) 218 return; 219 220 dma_unmap_page(pool->dev, dma_addr, (long)num_pages << PAGE_SHIFT, 221 DMA_BIDIRECTIONAL); 222} 223 224/* Give pages into a specific pool_type */ 225static void ttm_pool_type_give(struct ttm_pool_type *pt, struct page *p) 226{ 227 unsigned int i, num_pages = 1 << pt->order; 228 229 for (i = 0; i < num_pages; ++i) { 230 if (PageHighMem(p)) 231 clear_highpage(p + i); 232 else 233 clear_page(page_address(p + i)); 234 } 235 236 spin_lock(&pt->lock); 237 list_add(&p->lru, &pt->pages); 238 spin_unlock(&pt->lock); 239 atomic_long_add(1 << pt->order, &allocated_pages); 240} 241 242/* Take pages from a specific pool_type, return NULL when nothing available */ 243static struct page *ttm_pool_type_take(struct ttm_pool_type *pt) 244{ 245 struct page *p; 246 247 spin_lock(&pt->lock); 248 p = list_first_entry_or_null(&pt->pages, typeof(*p), lru); 249 if (p) { 250 atomic_long_sub(1 << pt->order, &allocated_pages); 251 list_del(&p->lru); 252 } 253 spin_unlock(&pt->lock); 254 255 return p; 256} 257 258/* Initialize and add a pool type to the global shrinker list */ 259static void ttm_pool_type_init(struct ttm_pool_type *pt, struct ttm_pool *pool, 260 enum ttm_caching caching, unsigned int order) 261{ 262 pt->pool = pool; 263 pt->caching = caching; 264 pt->order = order; 265 spin_lock_init(&pt->lock); 266 INIT_LIST_HEAD(&pt->pages); 267 268 spin_lock(&shrinker_lock); 269 list_add_tail(&pt->shrinker_list, &shrinker_list); 270 spin_unlock(&shrinker_lock); 271} 272 273/* Remove a pool_type from the global shrinker list and free all pages */ 274static void ttm_pool_type_fini(struct ttm_pool_type *pt) 275{ 276 struct page *p; 277 278 spin_lock(&shrinker_lock); 279 list_del(&pt->shrinker_list); 280 spin_unlock(&shrinker_lock); 281 282 while ((p = ttm_pool_type_take(pt))) 283 ttm_pool_free_page(pt->pool, pt->caching, pt->order, p); 284} 285 286/* Return the pool_type to use for the given caching and order */ 287static struct ttm_pool_type *ttm_pool_select_type(struct ttm_pool *pool, 288 enum ttm_caching caching, 289 unsigned int order) 290{ 291 if (pool->use_dma_alloc) 292 return &pool->caching[caching].orders[order]; 293 294#ifdef CONFIG_X86 295 switch (caching) { 296 case ttm_write_combined: 297 if (pool->nid != NUMA_NO_NODE) 298 return &pool->caching[caching].orders[order]; 299 300 if (pool->use_dma32) 301 return &global_dma32_write_combined[order]; 302 303 return &global_write_combined[order]; 304 case ttm_uncached: 305 if (pool->nid != NUMA_NO_NODE) 306 return &pool->caching[caching].orders[order]; 307 308 if (pool->use_dma32) 309 return &global_dma32_uncached[order]; 310 311 return &global_uncached[order]; 312 default: 313 break; 314 } 315#endif 316 317 return NULL; 318} 319 320/* Free pages using the global shrinker list */ 321static unsigned int ttm_pool_shrink(void) 322{ 323 struct ttm_pool_type *pt; 324 unsigned int num_pages; 325 struct page *p; 326 327 down_read(&pool_shrink_rwsem); 328 spin_lock(&shrinker_lock); 329 pt = list_first_entry(&shrinker_list, typeof(*pt), shrinker_list); 330 list_move_tail(&pt->shrinker_list, &shrinker_list); 331 spin_unlock(&shrinker_lock); 332 333 p = ttm_pool_type_take(pt); 334 if (p) { 335 ttm_pool_free_page(pt->pool, pt->caching, pt->order, p); 336 num_pages = 1 << pt->order; 337 } else { 338 num_pages = 0; 339 } 340 up_read(&pool_shrink_rwsem); 341 342 return num_pages; 343} 344 345/* Return the allocation order based for a page */ 346static unsigned int ttm_pool_page_order(struct ttm_pool *pool, struct page *p) 347{ 348 if (pool->use_dma_alloc) { 349 struct ttm_pool_dma *dma = (void *)p->private; 350 351 return dma->vaddr & ~PAGE_MASK; 352 } 353 354 return p->private; 355} 356 357/* Called when we got a page, either from a pool or newly allocated */ 358static int ttm_pool_page_allocated(struct ttm_pool *pool, unsigned int order, 359 struct page *p, dma_addr_t **dma_addr, 360 unsigned long *num_pages, 361 struct page ***pages) 362{ 363 unsigned int i; 364 int r; 365 366 if (*dma_addr) { 367 r = ttm_pool_map(pool, order, p, dma_addr); 368 if (r) 369 return r; 370 } 371 372 *num_pages -= 1 << order; 373 for (i = 1 << order; i; --i, ++(*pages), ++p) 374 **pages = p; 375 376 return 0; 377} 378 379/** 380 * ttm_pool_free_range() - Free a range of TTM pages 381 * @pool: The pool used for allocating. 382 * @tt: The struct ttm_tt holding the page pointers. 383 * @caching: The page caching mode used by the range. 384 * @start_page: index for first page to free. 385 * @end_page: index for last page to free + 1. 386 * 387 * During allocation the ttm_tt page-vector may be populated with ranges of 388 * pages with different attributes if allocation hit an error without being 389 * able to completely fulfill the allocation. This function can be used 390 * to free these individual ranges. 391 */ 392static void ttm_pool_free_range(struct ttm_pool *pool, struct ttm_tt *tt, 393 enum ttm_caching caching, 394 pgoff_t start_page, pgoff_t end_page) 395{ 396 struct page **pages = &tt->pages[start_page]; 397 unsigned int order; 398 pgoff_t i, nr; 399 400 for (i = start_page; i < end_page; i += nr, pages += nr) { 401 struct ttm_pool_type *pt = NULL; 402 403 order = ttm_pool_page_order(pool, *pages); 404 nr = (1UL << order); 405 if (tt->dma_address) 406 ttm_pool_unmap(pool, tt->dma_address[i], nr); 407 408 pt = ttm_pool_select_type(pool, caching, order); 409 if (pt) 410 ttm_pool_type_give(pt, *pages); 411 else 412 ttm_pool_free_page(pool, caching, order, *pages); 413 } 414} 415 416/** 417 * ttm_pool_alloc - Fill a ttm_tt object 418 * 419 * @pool: ttm_pool to use 420 * @tt: ttm_tt object to fill 421 * @ctx: operation context 422 * 423 * Fill the ttm_tt object with pages and also make sure to DMA map them when 424 * necessary. 425 * 426 * Returns: 0 on successe, negative error code otherwise. 427 */ 428int ttm_pool_alloc(struct ttm_pool *pool, struct ttm_tt *tt, 429 struct ttm_operation_ctx *ctx) 430{ 431 pgoff_t num_pages = tt->num_pages; 432 dma_addr_t *dma_addr = tt->dma_address; 433 struct page **caching = tt->pages; 434 struct page **pages = tt->pages; 435 enum ttm_caching page_caching; 436 gfp_t gfp_flags = GFP_USER; 437 pgoff_t caching_divide; 438 unsigned int order; 439 struct page *p; 440 int r; 441 442 WARN_ON(!num_pages || ttm_tt_is_populated(tt)); 443 WARN_ON(dma_addr && !pool->dev); 444 445 if (tt->page_flags & TTM_TT_FLAG_ZERO_ALLOC) 446 gfp_flags |= __GFP_ZERO; 447 448 if (ctx->gfp_retry_mayfail) 449 gfp_flags |= __GFP_RETRY_MAYFAIL; 450 451 if (pool->use_dma32) 452 gfp_flags |= GFP_DMA32; 453 else 454 gfp_flags |= GFP_HIGHUSER; 455 456 for (order = min_t(unsigned int, MAX_PAGE_ORDER, __fls(num_pages)); 457 num_pages; 458 order = min_t(unsigned int, order, __fls(num_pages))) { 459 struct ttm_pool_type *pt; 460 461 page_caching = tt->caching; 462 pt = ttm_pool_select_type(pool, tt->caching, order); 463 p = pt ? ttm_pool_type_take(pt) : NULL; 464 if (p) { 465 r = ttm_pool_apply_caching(caching, pages, 466 tt->caching); 467 if (r) 468 goto error_free_page; 469 470 caching = pages; 471 do { 472 r = ttm_pool_page_allocated(pool, order, p, 473 &dma_addr, 474 &num_pages, 475 &pages); 476 if (r) 477 goto error_free_page; 478 479 caching = pages; 480 if (num_pages < (1 << order)) 481 break; 482 483 p = ttm_pool_type_take(pt); 484 } while (p); 485 } 486 487 page_caching = ttm_cached; 488 while (num_pages >= (1 << order) && 489 (p = ttm_pool_alloc_page(pool, gfp_flags, order))) { 490 491 if (PageHighMem(p)) { 492 r = ttm_pool_apply_caching(caching, pages, 493 tt->caching); 494 if (r) 495 goto error_free_page; 496 caching = pages; 497 } 498 r = ttm_pool_page_allocated(pool, order, p, &dma_addr, 499 &num_pages, &pages); 500 if (r) 501 goto error_free_page; 502 if (PageHighMem(p)) 503 caching = pages; 504 } 505 506 if (!p) { 507 if (order) { 508 --order; 509 continue; 510 } 511 r = -ENOMEM; 512 goto error_free_all; 513 } 514 } 515 516 r = ttm_pool_apply_caching(caching, pages, tt->caching); 517 if (r) 518 goto error_free_all; 519 520 return 0; 521 522error_free_page: 523 ttm_pool_free_page(pool, page_caching, order, p); 524 525error_free_all: 526 num_pages = tt->num_pages - num_pages; 527 caching_divide = caching - tt->pages; 528 ttm_pool_free_range(pool, tt, tt->caching, 0, caching_divide); 529 ttm_pool_free_range(pool, tt, ttm_cached, caching_divide, num_pages); 530 531 return r; 532} 533EXPORT_SYMBOL(ttm_pool_alloc); 534 535/** 536 * ttm_pool_free - Free the backing pages from a ttm_tt object 537 * 538 * @pool: Pool to give pages back to. 539 * @tt: ttm_tt object to unpopulate 540 * 541 * Give the packing pages back to a pool or free them 542 */ 543void ttm_pool_free(struct ttm_pool *pool, struct ttm_tt *tt) 544{ 545 ttm_pool_free_range(pool, tt, tt->caching, 0, tt->num_pages); 546 547 while (atomic_long_read(&allocated_pages) > page_pool_size) 548 ttm_pool_shrink(); 549} 550EXPORT_SYMBOL(ttm_pool_free); 551 552/** 553 * ttm_pool_init - Initialize a pool 554 * 555 * @pool: the pool to initialize 556 * @dev: device for DMA allocations and mappings 557 * @nid: NUMA node to use for allocations 558 * @use_dma_alloc: true if coherent DMA alloc should be used 559 * @use_dma32: true if GFP_DMA32 should be used 560 * 561 * Initialize the pool and its pool types. 562 */ 563void ttm_pool_init(struct ttm_pool *pool, struct device *dev, 564 int nid, bool use_dma_alloc, bool use_dma32) 565{ 566 unsigned int i, j; 567 568 WARN_ON(!dev && use_dma_alloc); 569 570 pool->dev = dev; 571 pool->nid = nid; 572 pool->use_dma_alloc = use_dma_alloc; 573 pool->use_dma32 = use_dma32; 574 575 for (i = 0; i < TTM_NUM_CACHING_TYPES; ++i) { 576 for (j = 0; j < NR_PAGE_ORDERS; ++j) { 577 struct ttm_pool_type *pt; 578 579 /* Initialize only pool types which are actually used */ 580 pt = ttm_pool_select_type(pool, i, j); 581 if (pt != &pool->caching[i].orders[j]) 582 continue; 583 584 ttm_pool_type_init(pt, pool, i, j); 585 } 586 } 587} 588EXPORT_SYMBOL(ttm_pool_init); 589 590/** 591 * ttm_pool_synchronize_shrinkers - Wait for all running shrinkers to complete. 592 * 593 * This is useful to guarantee that all shrinker invocations have seen an 594 * update, before freeing memory, similar to rcu. 595 */ 596static void ttm_pool_synchronize_shrinkers(void) 597{ 598 down_write(&pool_shrink_rwsem); 599 up_write(&pool_shrink_rwsem); 600} 601 602/** 603 * ttm_pool_fini - Cleanup a pool 604 * 605 * @pool: the pool to clean up 606 * 607 * Free all pages in the pool and unregister the types from the global 608 * shrinker. 609 */ 610void ttm_pool_fini(struct ttm_pool *pool) 611{ 612 unsigned int i, j; 613 614 for (i = 0; i < TTM_NUM_CACHING_TYPES; ++i) { 615 for (j = 0; j < NR_PAGE_ORDERS; ++j) { 616 struct ttm_pool_type *pt; 617 618 pt = ttm_pool_select_type(pool, i, j); 619 if (pt != &pool->caching[i].orders[j]) 620 continue; 621 622 ttm_pool_type_fini(pt); 623 } 624 } 625 626 /* We removed the pool types from the LRU, but we need to also make sure 627 * that no shrinker is concurrently freeing pages from the pool. 628 */ 629 ttm_pool_synchronize_shrinkers(); 630} 631EXPORT_SYMBOL(ttm_pool_fini); 632 633/* As long as pages are available make sure to release at least one */ 634static unsigned long ttm_pool_shrinker_scan(struct shrinker *shrink, 635 struct shrink_control *sc) 636{ 637 unsigned long num_freed = 0; 638 639 do 640 num_freed += ttm_pool_shrink(); 641 while (!num_freed && atomic_long_read(&allocated_pages)); 642 643 return num_freed; 644} 645 646/* Return the number of pages available or SHRINK_EMPTY if we have none */ 647static unsigned long ttm_pool_shrinker_count(struct shrinker *shrink, 648 struct shrink_control *sc) 649{ 650 unsigned long num_pages = atomic_long_read(&allocated_pages); 651 652 return num_pages ? num_pages : SHRINK_EMPTY; 653} 654 655#ifdef CONFIG_DEBUG_FS 656/* Count the number of pages available in a pool_type */ 657static unsigned int ttm_pool_type_count(struct ttm_pool_type *pt) 658{ 659 unsigned int count = 0; 660 struct page *p; 661 662 spin_lock(&pt->lock); 663 /* Only used for debugfs, the overhead doesn't matter */ 664 list_for_each_entry(p, &pt->pages, lru) 665 ++count; 666 spin_unlock(&pt->lock); 667 668 return count; 669} 670 671/* Print a nice header for the order */ 672static void ttm_pool_debugfs_header(struct seq_file *m) 673{ 674 unsigned int i; 675 676 seq_puts(m, "\t "); 677 for (i = 0; i < NR_PAGE_ORDERS; ++i) 678 seq_printf(m, " ---%2u---", i); 679 seq_puts(m, "\n"); 680} 681 682/* Dump information about the different pool types */ 683static void ttm_pool_debugfs_orders(struct ttm_pool_type *pt, 684 struct seq_file *m) 685{ 686 unsigned int i; 687 688 for (i = 0; i < NR_PAGE_ORDERS; ++i) 689 seq_printf(m, " %8u", ttm_pool_type_count(&pt[i])); 690 seq_puts(m, "\n"); 691} 692 693/* Dump the total amount of allocated pages */ 694static void ttm_pool_debugfs_footer(struct seq_file *m) 695{ 696 seq_printf(m, "\ntotal\t: %8lu of %8lu\n", 697 atomic_long_read(&allocated_pages), page_pool_size); 698} 699 700/* Dump the information for the global pools */ 701static int ttm_pool_debugfs_globals_show(struct seq_file *m, void *data) 702{ 703 ttm_pool_debugfs_header(m); 704 705 spin_lock(&shrinker_lock); 706 seq_puts(m, "wc\t:"); 707 ttm_pool_debugfs_orders(global_write_combined, m); 708 seq_puts(m, "uc\t:"); 709 ttm_pool_debugfs_orders(global_uncached, m); 710 seq_puts(m, "wc 32\t:"); 711 ttm_pool_debugfs_orders(global_dma32_write_combined, m); 712 seq_puts(m, "uc 32\t:"); 713 ttm_pool_debugfs_orders(global_dma32_uncached, m); 714 spin_unlock(&shrinker_lock); 715 716 ttm_pool_debugfs_footer(m); 717 718 return 0; 719} 720DEFINE_SHOW_ATTRIBUTE(ttm_pool_debugfs_globals); 721 722/** 723 * ttm_pool_debugfs - Debugfs dump function for a pool 724 * 725 * @pool: the pool to dump the information for 726 * @m: seq_file to dump to 727 * 728 * Make a debugfs dump with the per pool and global information. 729 */ 730int ttm_pool_debugfs(struct ttm_pool *pool, struct seq_file *m) 731{ 732 unsigned int i; 733 734 if (!pool->use_dma_alloc) { 735 seq_puts(m, "unused\n"); 736 return 0; 737 } 738 739 ttm_pool_debugfs_header(m); 740 741 spin_lock(&shrinker_lock); 742 for (i = 0; i < TTM_NUM_CACHING_TYPES; ++i) { 743 seq_puts(m, "DMA "); 744 switch (i) { 745 case ttm_cached: 746 seq_puts(m, "\t:"); 747 break; 748 case ttm_write_combined: 749 seq_puts(m, "wc\t:"); 750 break; 751 case ttm_uncached: 752 seq_puts(m, "uc\t:"); 753 break; 754 } 755 ttm_pool_debugfs_orders(pool->caching[i].orders, m); 756 } 757 spin_unlock(&shrinker_lock); 758 759 ttm_pool_debugfs_footer(m); 760 return 0; 761} 762EXPORT_SYMBOL(ttm_pool_debugfs); 763 764/* Test the shrinker functions and dump the result */ 765static int ttm_pool_debugfs_shrink_show(struct seq_file *m, void *data) 766{ 767 struct shrink_control sc = { .gfp_mask = GFP_NOFS }; 768 769 fs_reclaim_acquire(GFP_KERNEL); 770 seq_printf(m, "%lu/%lu\n", ttm_pool_shrinker_count(mm_shrinker, &sc), 771 ttm_pool_shrinker_scan(mm_shrinker, &sc)); 772 fs_reclaim_release(GFP_KERNEL); 773 774 return 0; 775} 776DEFINE_SHOW_ATTRIBUTE(ttm_pool_debugfs_shrink); 777 778#endif 779 780/** 781 * ttm_pool_mgr_init - Initialize globals 782 * 783 * @num_pages: default number of pages 784 * 785 * Initialize the global locks and lists for the MM shrinker. 786 */ 787int ttm_pool_mgr_init(unsigned long num_pages) 788{ 789 unsigned int i; 790 791 if (!page_pool_size) 792 page_pool_size = num_pages; 793 794 spin_lock_init(&shrinker_lock); 795 INIT_LIST_HEAD(&shrinker_list); 796 797 for (i = 0; i < NR_PAGE_ORDERS; ++i) { 798 ttm_pool_type_init(&global_write_combined[i], NULL, 799 ttm_write_combined, i); 800 ttm_pool_type_init(&global_uncached[i], NULL, ttm_uncached, i); 801 802 ttm_pool_type_init(&global_dma32_write_combined[i], NULL, 803 ttm_write_combined, i); 804 ttm_pool_type_init(&global_dma32_uncached[i], NULL, 805 ttm_uncached, i); 806 } 807 808#ifdef CONFIG_DEBUG_FS 809 debugfs_create_file("page_pool", 0444, ttm_debugfs_root, NULL, 810 &ttm_pool_debugfs_globals_fops); 811 debugfs_create_file("page_pool_shrink", 0400, ttm_debugfs_root, NULL, 812 &ttm_pool_debugfs_shrink_fops); 813#endif 814 815 mm_shrinker = shrinker_alloc(0, "drm-ttm_pool"); 816 if (!mm_shrinker) 817 return -ENOMEM; 818 819 mm_shrinker->count_objects = ttm_pool_shrinker_count; 820 mm_shrinker->scan_objects = ttm_pool_shrinker_scan; 821 mm_shrinker->seeks = 1; 822 823 shrinker_register(mm_shrinker); 824 825 return 0; 826} 827 828/** 829 * ttm_pool_mgr_fini - Finalize globals 830 * 831 * Cleanup the global pools and unregister the MM shrinker. 832 */ 833void ttm_pool_mgr_fini(void) 834{ 835 unsigned int i; 836 837 for (i = 0; i < NR_PAGE_ORDERS; ++i) { 838 ttm_pool_type_fini(&global_write_combined[i]); 839 ttm_pool_type_fini(&global_uncached[i]); 840 841 ttm_pool_type_fini(&global_dma32_write_combined[i]); 842 ttm_pool_type_fini(&global_dma32_uncached[i]); 843 } 844 845 shrinker_free(mm_shrinker); 846 WARN_ON(!list_empty(&shrinker_list)); 847} 848