1/* 2 * Copyright (c) Red Hat Inc. 3 4 * Permission is hereby granted, free of charge, to any person obtaining a 5 * copy of this software and associated documentation files (the "Software"), 6 * to deal in the Software without restriction, including without limitation 7 * the rights to use, copy, modify, merge, publish, distribute, sub license, 8 * and/or sell copies of the Software, and to permit persons to whom the 9 * Software is furnished to do so, subject to the following conditions: 10 * 11 * The above copyright notice and this permission notice (including the 12 * next paragraph) shall be included in all copies or substantial portions 13 * of the Software. 14 * 15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL 18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER 21 * DEALINGS IN THE SOFTWARE. 22 * 23 * Authors: Dave Airlie <airlied@redhat.com> 24 * Jerome Glisse <jglisse@redhat.com> 25 * Pauli Nieminen <suokkos@gmail.com> 26 */ 27/* 28 * Copyright (c) 2013 The FreeBSD Foundation 29 * All rights reserved. 30 * 31 * Portions of this software were developed by Konstantin Belousov 32 * <kib@FreeBSD.org> under sponsorship from the FreeBSD Foundation. 33 */ 34 35/* simple list based uncached page pool 36 * - Pool collects resently freed pages for reuse 37 * - Use page->lru to keep a free list 38 * - doesn't track currently in use pages 39 */ 40 41#include <sys/cdefs.h> 42__FBSDID("$FreeBSD$"); 43 44#include <dev/drm2/drmP.h> 45#include <dev/drm2/ttm/ttm_bo_driver.h> 46#include <dev/drm2/ttm/ttm_page_alloc.h> 47#include <vm/vm_pageout.h> 48 49#define NUM_PAGES_TO_ALLOC (PAGE_SIZE/sizeof(vm_page_t)) 50#define SMALL_ALLOCATION 16 51#define FREE_ALL_PAGES (~0U) 52/* times are in msecs */ 53#define PAGE_FREE_INTERVAL 1000 54 55/** 56 * struct ttm_page_pool - Pool to reuse recently allocated uc/wc pages. 57 * 58 * @lock: Protects the shared pool from concurrnet access. Must be used with 59 * irqsave/irqrestore variants because pool allocator maybe called from 60 * delayed work. 61 * @fill_lock: Prevent concurrent calls to fill. 62 * @list: Pool of free uc/wc pages for fast reuse. 63 * @gfp_flags: Flags to pass for alloc_page. 64 * @npages: Number of pages in pool. 65 */ 66struct ttm_page_pool { 67 struct mtx lock; 68 bool fill_lock; 69 bool dma32; 70 struct pglist list; 71 int ttm_page_alloc_flags; 72 unsigned npages; 73 char *name; 74 unsigned long nfrees; 75 unsigned long nrefills; 76}; 77 78/** 79 * Limits for the pool. They are handled without locks because only place where 80 * they may change is in sysfs store. They won't have immediate effect anyway 81 * so forcing serialization to access them is pointless. 82 */ 83 84struct ttm_pool_opts { 85 unsigned alloc_size; 86 unsigned max_size; 87 unsigned small; 88}; 89 90#define NUM_POOLS 4 91 92/** 93 * struct ttm_pool_manager - Holds memory pools for fst allocation 94 * 95 * Manager is read only object for pool code so it doesn't need locking. 96 * 97 * @free_interval: minimum number of jiffies between freeing pages from pool. 98 * @page_alloc_inited: reference counting for pool allocation. 99 * @work: Work that is used to shrink the pool. Work is only run when there is 100 * some pages to free. 101 * @small_allocation: Limit in number of pages what is small allocation. 102 * 103 * @pools: All pool objects in use. 104 **/ 105struct ttm_pool_manager { 106 unsigned int kobj_ref; 107 eventhandler_tag lowmem_handler; 108 struct ttm_pool_opts options; 109 110 union { 111 struct ttm_page_pool u_pools[NUM_POOLS]; 112 struct _utag { 113 struct ttm_page_pool u_wc_pool; 114 struct ttm_page_pool u_uc_pool; 115 struct ttm_page_pool u_wc_pool_dma32; 116 struct ttm_page_pool u_uc_pool_dma32; 117 } _ut; 118 } _u; 119}; 120 121#define pools _u.u_pools 122#define wc_pool _u._ut.u_wc_pool 123#define uc_pool _u._ut.u_uc_pool 124#define wc_pool_dma32 _u._ut.u_wc_pool_dma32 125#define uc_pool_dma32 _u._ut.u_uc_pool_dma32 126 127MALLOC_DEFINE(M_TTM_POOLMGR, "ttm_poolmgr", "TTM Pool Manager"); 128 129static void 130ttm_vm_page_free(vm_page_t m) 131{ 132 133 KASSERT(m->object == NULL, ("ttm page %p is owned", m)); 134 KASSERT(m->wire_count == 1, ("ttm lost wire %p", m)); 135 KASSERT((m->flags & PG_FICTITIOUS) != 0, ("ttm lost fictitious %p", m)); 136 KASSERT((m->oflags & VPO_UNMANAGED) == 0, ("ttm got unmanaged %p", m)); 137 m->flags &= ~PG_FICTITIOUS; 138 m->oflags |= VPO_UNMANAGED; 139 vm_page_unwire(m, 0); 140 vm_page_free(m); 141} 142 143static vm_memattr_t 144ttm_caching_state_to_vm(enum ttm_caching_state cstate) 145{ 146 147 switch (cstate) { 148 case tt_uncached: 149 return (VM_MEMATTR_UNCACHEABLE); 150 case tt_wc: 151 return (VM_MEMATTR_WRITE_COMBINING); 152 case tt_cached: 153 return (VM_MEMATTR_WRITE_BACK); 154 } 155 panic("caching state %d\n", cstate); 156} 157 158static vm_page_t 159ttm_vm_page_alloc_dma32(int req, vm_memattr_t memattr) 160{ 161 vm_page_t p; 162 int tries; 163 164 for (tries = 0; ; tries++) { 165 p = vm_page_alloc_contig(NULL, 0, req, 1, 0, 0xffffffff, 166 PAGE_SIZE, 0, memattr); 167 if (p != NULL || tries > 2) 168 return (p); 169 170 /* 171 * Before growing the cache see if this is just a normal 172 * memory shortage. 173 */ 174 VM_WAIT; 175 vm_pageout_grow_cache(tries, 0, 0xffffffff); 176 } 177} 178 179static vm_page_t 180ttm_vm_page_alloc_any(int req, vm_memattr_t memattr) 181{ 182 vm_page_t p; 183 184 while (1) { 185 p = vm_page_alloc(NULL, 0, req); 186 if (p != NULL) 187 break; 188 VM_WAIT; 189 } 190 pmap_page_set_memattr(p, memattr); 191 return (p); 192} 193 194static vm_page_t 195ttm_vm_page_alloc(int flags, enum ttm_caching_state cstate) 196{ 197 vm_page_t p; 198 vm_memattr_t memattr; 199 int req; 200 201 memattr = ttm_caching_state_to_vm(cstate); 202 req = VM_ALLOC_NORMAL | VM_ALLOC_WIRED | VM_ALLOC_NOOBJ; 203 if ((flags & TTM_PAGE_FLAG_ZERO_ALLOC) != 0) 204 req |= VM_ALLOC_ZERO; 205 206 if ((flags & TTM_PAGE_FLAG_DMA32) != 0) 207 p = ttm_vm_page_alloc_dma32(req, memattr); 208 else 209 p = ttm_vm_page_alloc_any(req, memattr); 210 211 if (p != NULL) { 212 p->oflags &= ~VPO_UNMANAGED; 213 p->flags |= PG_FICTITIOUS; 214 } 215 return (p); 216} 217 218static void ttm_pool_kobj_release(struct ttm_pool_manager *m) 219{ 220 221 free(m, M_TTM_POOLMGR); 222} 223 224#if 0 225/* XXXKIB sysctl */ 226static ssize_t ttm_pool_store(struct ttm_pool_manager *m, 227 struct attribute *attr, const char *buffer, size_t size) 228{ 229 int chars; 230 unsigned val; 231 chars = sscanf(buffer, "%u", &val); 232 if (chars == 0) 233 return size; 234 235 /* Convert kb to number of pages */ 236 val = val / (PAGE_SIZE >> 10); 237 238 if (attr == &ttm_page_pool_max) 239 m->options.max_size = val; 240 else if (attr == &ttm_page_pool_small) 241 m->options.small = val; 242 else if (attr == &ttm_page_pool_alloc_size) { 243 if (val > NUM_PAGES_TO_ALLOC*8) { 244 pr_err("Setting allocation size to %lu is not allowed. Recommended size is %lu\n", 245 NUM_PAGES_TO_ALLOC*(PAGE_SIZE >> 7), 246 NUM_PAGES_TO_ALLOC*(PAGE_SIZE >> 10)); 247 return size; 248 } else if (val > NUM_PAGES_TO_ALLOC) { 249 pr_warn("Setting allocation size to larger than %lu is not recommended\n", 250 NUM_PAGES_TO_ALLOC*(PAGE_SIZE >> 10)); 251 } 252 m->options.alloc_size = val; 253 } 254 255 return size; 256} 257 258static ssize_t ttm_pool_show(struct ttm_pool_manager *m, 259 struct attribute *attr, char *buffer) 260{ 261 unsigned val = 0; 262 263 if (attr == &ttm_page_pool_max) 264 val = m->options.max_size; 265 else if (attr == &ttm_page_pool_small) 266 val = m->options.small; 267 else if (attr == &ttm_page_pool_alloc_size) 268 val = m->options.alloc_size; 269 270 val = val * (PAGE_SIZE >> 10); 271 272 return snprintf(buffer, PAGE_SIZE, "%u\n", val); 273} 274#endif 275 276static struct ttm_pool_manager *_manager; 277 278static int set_pages_array_wb(vm_page_t *pages, int addrinarray) 279{ 280#ifdef TTM_HAS_AGP 281 int i; 282 283 for (i = 0; i < addrinarray; i++) 284 pmap_page_set_memattr(pages[i], VM_MEMATTR_WRITE_BACK); 285#endif 286 return 0; 287} 288 289static int set_pages_array_wc(vm_page_t *pages, int addrinarray) 290{ 291#ifdef TTM_HAS_AGP 292 int i; 293 294 for (i = 0; i < addrinarray; i++) 295 pmap_page_set_memattr(pages[i], VM_MEMATTR_WRITE_COMBINING); 296#endif 297 return 0; 298} 299 300static int set_pages_array_uc(vm_page_t *pages, int addrinarray) 301{ 302#ifdef TTM_HAS_AGP 303 int i; 304 305 for (i = 0; i < addrinarray; i++) 306 pmap_page_set_memattr(pages[i], VM_MEMATTR_UNCACHEABLE); 307#endif 308 return 0; 309} 310 311/** 312 * Select the right pool or requested caching state and ttm flags. */ 313static struct ttm_page_pool *ttm_get_pool(int flags, 314 enum ttm_caching_state cstate) 315{ 316 int pool_index; 317 318 if (cstate == tt_cached) 319 return NULL; 320 321 if (cstate == tt_wc) 322 pool_index = 0x0; 323 else 324 pool_index = 0x1; 325 326 if (flags & TTM_PAGE_FLAG_DMA32) 327 pool_index |= 0x2; 328 329 return &_manager->pools[pool_index]; 330} 331 332/* set memory back to wb and free the pages. */ 333static void ttm_pages_put(vm_page_t *pages, unsigned npages) 334{ 335 unsigned i; 336 337 /* Our VM handles vm memattr automatically on the page free. */ 338 if (set_pages_array_wb(pages, npages)) 339 printf("[TTM] Failed to set %d pages to wb!\n", npages); 340 for (i = 0; i < npages; ++i) 341 ttm_vm_page_free(pages[i]); 342} 343 344static void ttm_pool_update_free_locked(struct ttm_page_pool *pool, 345 unsigned freed_pages) 346{ 347 pool->npages -= freed_pages; 348 pool->nfrees += freed_pages; 349} 350 351/** 352 * Free pages from pool. 353 * 354 * To prevent hogging the ttm_swap process we only free NUM_PAGES_TO_ALLOC 355 * number of pages in one go. 356 * 357 * @pool: to free the pages from 358 * @free_all: If set to true will free all pages in pool 359 **/ 360static int ttm_page_pool_free(struct ttm_page_pool *pool, unsigned nr_free) 361{ 362 vm_page_t p, p1; 363 vm_page_t *pages_to_free; 364 unsigned freed_pages = 0, 365 npages_to_free = nr_free; 366 unsigned i; 367 368 if (NUM_PAGES_TO_ALLOC < nr_free) 369 npages_to_free = NUM_PAGES_TO_ALLOC; 370 371 pages_to_free = malloc(npages_to_free * sizeof(vm_page_t), 372 M_TEMP, M_WAITOK | M_ZERO); 373 374restart: 375 mtx_lock(&pool->lock); 376 377 TAILQ_FOREACH_REVERSE_SAFE(p, &pool->list, pglist, plinks.q, p1) { 378 if (freed_pages >= npages_to_free) 379 break; 380 381 pages_to_free[freed_pages++] = p; 382 /* We can only remove NUM_PAGES_TO_ALLOC at a time. */ 383 if (freed_pages >= NUM_PAGES_TO_ALLOC) { 384 /* remove range of pages from the pool */ 385 for (i = 0; i < freed_pages; i++) 386 TAILQ_REMOVE(&pool->list, pages_to_free[i], plinks.q); 387 388 ttm_pool_update_free_locked(pool, freed_pages); 389 /** 390 * Because changing page caching is costly 391 * we unlock the pool to prevent stalling. 392 */ 393 mtx_unlock(&pool->lock); 394 395 ttm_pages_put(pages_to_free, freed_pages); 396 if (likely(nr_free != FREE_ALL_PAGES)) 397 nr_free -= freed_pages; 398 399 if (NUM_PAGES_TO_ALLOC >= nr_free) 400 npages_to_free = nr_free; 401 else 402 npages_to_free = NUM_PAGES_TO_ALLOC; 403 404 freed_pages = 0; 405 406 /* free all so restart the processing */ 407 if (nr_free) 408 goto restart; 409 410 /* Not allowed to fall through or break because 411 * following context is inside spinlock while we are 412 * outside here. 413 */ 414 goto out; 415 416 } 417 } 418 419 /* remove range of pages from the pool */ 420 if (freed_pages) { 421 for (i = 0; i < freed_pages; i++) 422 TAILQ_REMOVE(&pool->list, pages_to_free[i], plinks.q); 423 424 ttm_pool_update_free_locked(pool, freed_pages); 425 nr_free -= freed_pages; 426 } 427 428 mtx_unlock(&pool->lock); 429 430 if (freed_pages) 431 ttm_pages_put(pages_to_free, freed_pages); 432out: 433 free(pages_to_free, M_TEMP); 434 return nr_free; 435} 436 437/* Get good estimation how many pages are free in pools */ 438static int ttm_pool_get_num_unused_pages(void) 439{ 440 unsigned i; 441 int total = 0; 442 for (i = 0; i < NUM_POOLS; ++i) 443 total += _manager->pools[i].npages; 444 445 return total; 446} 447 448/** 449 * Callback for mm to request pool to reduce number of page held. 450 */ 451static int ttm_pool_mm_shrink(void *arg) 452{ 453 static unsigned int start_pool = 0; 454 unsigned i; 455 unsigned pool_offset = atomic_fetchadd_int(&start_pool, 1); 456 struct ttm_page_pool *pool; 457 int shrink_pages = 100; /* XXXKIB */ 458 459 pool_offset = pool_offset % NUM_POOLS; 460 /* select start pool in round robin fashion */ 461 for (i = 0; i < NUM_POOLS; ++i) { 462 unsigned nr_free = shrink_pages; 463 if (shrink_pages == 0) 464 break; 465 pool = &_manager->pools[(i + pool_offset)%NUM_POOLS]; 466 shrink_pages = ttm_page_pool_free(pool, nr_free); 467 } 468 /* return estimated number of unused pages in pool */ 469 return ttm_pool_get_num_unused_pages(); 470} 471 472static void ttm_pool_mm_shrink_init(struct ttm_pool_manager *manager) 473{ 474 475 manager->lowmem_handler = EVENTHANDLER_REGISTER(vm_lowmem, 476 ttm_pool_mm_shrink, manager, EVENTHANDLER_PRI_ANY); 477} 478 479static void ttm_pool_mm_shrink_fini(struct ttm_pool_manager *manager) 480{ 481 482 EVENTHANDLER_DEREGISTER(vm_lowmem, manager->lowmem_handler); 483} 484 485static int ttm_set_pages_caching(vm_page_t *pages, 486 enum ttm_caching_state cstate, unsigned cpages) 487{ 488 int r = 0; 489 /* Set page caching */ 490 switch (cstate) { 491 case tt_uncached: 492 r = set_pages_array_uc(pages, cpages); 493 if (r) 494 printf("[TTM] Failed to set %d pages to uc!\n", cpages); 495 break; 496 case tt_wc: 497 r = set_pages_array_wc(pages, cpages); 498 if (r) 499 printf("[TTM] Failed to set %d pages to wc!\n", cpages); 500 break; 501 default: 502 break; 503 } 504 return r; 505} 506 507/** 508 * Free pages the pages that failed to change the caching state. If there is 509 * any pages that have changed their caching state already put them to the 510 * pool. 511 */ 512static void ttm_handle_caching_state_failure(struct pglist *pages, 513 int ttm_flags, enum ttm_caching_state cstate, 514 vm_page_t *failed_pages, unsigned cpages) 515{ 516 unsigned i; 517 /* Failed pages have to be freed */ 518 for (i = 0; i < cpages; ++i) { 519 TAILQ_REMOVE(pages, failed_pages[i], plinks.q); 520 ttm_vm_page_free(failed_pages[i]); 521 } 522} 523 524/** 525 * Allocate new pages with correct caching. 526 * 527 * This function is reentrant if caller updates count depending on number of 528 * pages returned in pages array. 529 */ 530static int ttm_alloc_new_pages(struct pglist *pages, int ttm_alloc_flags, 531 int ttm_flags, enum ttm_caching_state cstate, unsigned count) 532{ 533 vm_page_t *caching_array; 534 vm_page_t p; 535 int r = 0; 536 unsigned i, cpages; 537 unsigned max_cpages = min(count, 538 (unsigned)(PAGE_SIZE/sizeof(vm_page_t))); 539 540 /* allocate array for page caching change */ 541 caching_array = malloc(max_cpages * sizeof(vm_page_t), M_TEMP, 542 M_WAITOK | M_ZERO); 543 544 for (i = 0, cpages = 0; i < count; ++i) { 545 p = ttm_vm_page_alloc(ttm_alloc_flags, cstate); 546 if (!p) { 547 printf("[TTM] Unable to get page %u\n", i); 548 549 /* store already allocated pages in the pool after 550 * setting the caching state */ 551 if (cpages) { 552 r = ttm_set_pages_caching(caching_array, 553 cstate, cpages); 554 if (r) 555 ttm_handle_caching_state_failure(pages, 556 ttm_flags, cstate, 557 caching_array, cpages); 558 } 559 r = -ENOMEM; 560 goto out; 561 } 562 563#ifdef CONFIG_HIGHMEM /* KIB: nop */ 564 /* gfp flags of highmem page should never be dma32 so we 565 * we should be fine in such case 566 */ 567 if (!PageHighMem(p)) 568#endif 569 { 570 caching_array[cpages++] = p; 571 if (cpages == max_cpages) { 572 573 r = ttm_set_pages_caching(caching_array, 574 cstate, cpages); 575 if (r) { 576 ttm_handle_caching_state_failure(pages, 577 ttm_flags, cstate, 578 caching_array, cpages); 579 goto out; 580 } 581 cpages = 0; 582 } 583 } 584 585 TAILQ_INSERT_HEAD(pages, p, plinks.q); 586 } 587 588 if (cpages) { 589 r = ttm_set_pages_caching(caching_array, cstate, cpages); 590 if (r) 591 ttm_handle_caching_state_failure(pages, 592 ttm_flags, cstate, 593 caching_array, cpages); 594 } 595out: 596 free(caching_array, M_TEMP); 597 598 return r; 599} 600 601/** 602 * Fill the given pool if there aren't enough pages and the requested number of 603 * pages is small. 604 */ 605static void ttm_page_pool_fill_locked(struct ttm_page_pool *pool, 606 int ttm_flags, enum ttm_caching_state cstate, unsigned count) 607{ 608 vm_page_t p; 609 int r; 610 unsigned cpages = 0; 611 /** 612 * Only allow one pool fill operation at a time. 613 * If pool doesn't have enough pages for the allocation new pages are 614 * allocated from outside of pool. 615 */ 616 if (pool->fill_lock) 617 return; 618 619 pool->fill_lock = true; 620 621 /* If allocation request is small and there are not enough 622 * pages in a pool we fill the pool up first. */ 623 if (count < _manager->options.small 624 && count > pool->npages) { 625 struct pglist new_pages; 626 unsigned alloc_size = _manager->options.alloc_size; 627 628 /** 629 * Can't change page caching if in irqsave context. We have to 630 * drop the pool->lock. 631 */ 632 mtx_unlock(&pool->lock); 633 634 TAILQ_INIT(&new_pages); 635 r = ttm_alloc_new_pages(&new_pages, pool->ttm_page_alloc_flags, 636 ttm_flags, cstate, alloc_size); 637 mtx_lock(&pool->lock); 638 639 if (!r) { 640 TAILQ_CONCAT(&pool->list, &new_pages, plinks.q); 641 ++pool->nrefills; 642 pool->npages += alloc_size; 643 } else { 644 printf("[TTM] Failed to fill pool (%p)\n", pool); 645 /* If we have any pages left put them to the pool. */ 646 TAILQ_FOREACH(p, &pool->list, plinks.q) { 647 ++cpages; 648 } 649 TAILQ_CONCAT(&pool->list, &new_pages, plinks.q); 650 pool->npages += cpages; 651 } 652 653 } 654 pool->fill_lock = false; 655} 656 657/** 658 * Cut 'count' number of pages from the pool and put them on the return list. 659 * 660 * @return count of pages still required to fulfill the request. 661 */ 662static unsigned ttm_page_pool_get_pages(struct ttm_page_pool *pool, 663 struct pglist *pages, 664 int ttm_flags, 665 enum ttm_caching_state cstate, 666 unsigned count) 667{ 668 vm_page_t p; 669 unsigned i; 670 671 mtx_lock(&pool->lock); 672 ttm_page_pool_fill_locked(pool, ttm_flags, cstate, count); 673 674 if (count >= pool->npages) { 675 /* take all pages from the pool */ 676 TAILQ_CONCAT(pages, &pool->list, plinks.q); 677 count -= pool->npages; 678 pool->npages = 0; 679 goto out; 680 } 681 for (i = 0; i < count; i++) { 682 p = TAILQ_FIRST(&pool->list); 683 TAILQ_REMOVE(&pool->list, p, plinks.q); 684 TAILQ_INSERT_TAIL(pages, p, plinks.q); 685 } 686 pool->npages -= count; 687 count = 0; 688out: 689 mtx_unlock(&pool->lock); 690 return count; 691} 692 693/* Put all pages in pages list to correct pool to wait for reuse */ 694static void ttm_put_pages(vm_page_t *pages, unsigned npages, int flags, 695 enum ttm_caching_state cstate) 696{ 697 struct ttm_page_pool *pool = ttm_get_pool(flags, cstate); 698 unsigned i; 699 700 if (pool == NULL) { 701 /* No pool for this memory type so free the pages */ 702 for (i = 0; i < npages; i++) { 703 if (pages[i]) { 704 ttm_vm_page_free(pages[i]); 705 pages[i] = NULL; 706 } 707 } 708 return; 709 } 710 711 mtx_lock(&pool->lock); 712 for (i = 0; i < npages; i++) { 713 if (pages[i]) { 714 TAILQ_INSERT_TAIL(&pool->list, pages[i], plinks.q); 715 pages[i] = NULL; 716 pool->npages++; 717 } 718 } 719 /* Check that we don't go over the pool limit */ 720 npages = 0; 721 if (pool->npages > _manager->options.max_size) { 722 npages = pool->npages - _manager->options.max_size; 723 /* free at least NUM_PAGES_TO_ALLOC number of pages 724 * to reduce calls to set_memory_wb */ 725 if (npages < NUM_PAGES_TO_ALLOC) 726 npages = NUM_PAGES_TO_ALLOC; 727 } 728 mtx_unlock(&pool->lock); 729 if (npages) 730 ttm_page_pool_free(pool, npages); 731} 732 733/* 734 * On success pages list will hold count number of correctly 735 * cached pages. 736 */ 737static int ttm_get_pages(vm_page_t *pages, unsigned npages, int flags, 738 enum ttm_caching_state cstate) 739{ 740 struct ttm_page_pool *pool = ttm_get_pool(flags, cstate); 741 struct pglist plist; 742 vm_page_t p = NULL; 743 int gfp_flags; 744 unsigned count; 745 int r; 746 747 /* No pool for cached pages */ 748 if (pool == NULL) { 749 for (r = 0; r < npages; ++r) { 750 p = ttm_vm_page_alloc(flags, cstate); 751 if (!p) { 752 printf("[TTM] Unable to allocate page\n"); 753 return -ENOMEM; 754 } 755 pages[r] = p; 756 } 757 return 0; 758 } 759 760 /* combine zero flag to pool flags */ 761 gfp_flags = flags | pool->ttm_page_alloc_flags; 762 763 /* First we take pages from the pool */ 764 TAILQ_INIT(&plist); 765 npages = ttm_page_pool_get_pages(pool, &plist, flags, cstate, npages); 766 count = 0; 767 TAILQ_FOREACH(p, &plist, plinks.q) { 768 pages[count++] = p; 769 } 770 771 /* clear the pages coming from the pool if requested */ 772 if (flags & TTM_PAGE_FLAG_ZERO_ALLOC) { 773 TAILQ_FOREACH(p, &plist, plinks.q) { 774 pmap_zero_page(p); 775 } 776 } 777 778 /* If pool didn't have enough pages allocate new one. */ 779 if (npages > 0) { 780 /* ttm_alloc_new_pages doesn't reference pool so we can run 781 * multiple requests in parallel. 782 **/ 783 TAILQ_INIT(&plist); 784 r = ttm_alloc_new_pages(&plist, gfp_flags, flags, cstate, 785 npages); 786 TAILQ_FOREACH(p, &plist, plinks.q) { 787 pages[count++] = p; 788 } 789 if (r) { 790 /* If there is any pages in the list put them back to 791 * the pool. */ 792 printf("[TTM] Failed to allocate extra pages for large request\n"); 793 ttm_put_pages(pages, count, flags, cstate); 794 return r; 795 } 796 } 797 798 return 0; 799} 800 801static void ttm_page_pool_init_locked(struct ttm_page_pool *pool, int flags, 802 char *name) 803{ 804 mtx_init(&pool->lock, "ttmpool", NULL, MTX_DEF); 805 pool->fill_lock = false; 806 TAILQ_INIT(&pool->list); 807 pool->npages = pool->nfrees = 0; 808 pool->ttm_page_alloc_flags = flags; 809 pool->name = name; 810} 811 812int ttm_page_alloc_init(struct ttm_mem_global *glob, unsigned max_pages) 813{ 814 815 if (_manager != NULL) 816 printf("[TTM] manager != NULL\n"); 817 printf("[TTM] Initializing pool allocator\n"); 818 819 _manager = malloc(sizeof(*_manager), M_TTM_POOLMGR, M_WAITOK | M_ZERO); 820 821 ttm_page_pool_init_locked(&_manager->wc_pool, 0, "wc"); 822 ttm_page_pool_init_locked(&_manager->uc_pool, 0, "uc"); 823 ttm_page_pool_init_locked(&_manager->wc_pool_dma32, 824 TTM_PAGE_FLAG_DMA32, "wc dma"); 825 ttm_page_pool_init_locked(&_manager->uc_pool_dma32, 826 TTM_PAGE_FLAG_DMA32, "uc dma"); 827 828 _manager->options.max_size = max_pages; 829 _manager->options.small = SMALL_ALLOCATION; 830 _manager->options.alloc_size = NUM_PAGES_TO_ALLOC; 831 832 refcount_init(&_manager->kobj_ref, 1); 833 ttm_pool_mm_shrink_init(_manager); 834 835 return 0; 836} 837 838void ttm_page_alloc_fini(void) 839{ 840 int i; 841 842 printf("[TTM] Finalizing pool allocator\n"); 843 ttm_pool_mm_shrink_fini(_manager); 844 845 for (i = 0; i < NUM_POOLS; ++i) 846 ttm_page_pool_free(&_manager->pools[i], FREE_ALL_PAGES); 847 848 if (refcount_release(&_manager->kobj_ref)) 849 ttm_pool_kobj_release(_manager); 850 _manager = NULL; 851} 852 853int ttm_pool_populate(struct ttm_tt *ttm) 854{ 855 struct ttm_mem_global *mem_glob = ttm->glob->mem_glob; 856 unsigned i; 857 int ret; 858 859 if (ttm->state != tt_unpopulated) 860 return 0; 861 862 for (i = 0; i < ttm->num_pages; ++i) { 863 ret = ttm_get_pages(&ttm->pages[i], 1, 864 ttm->page_flags, 865 ttm->caching_state); 866 if (ret != 0) { 867 ttm_pool_unpopulate(ttm); 868 return -ENOMEM; 869 } 870 871 ret = ttm_mem_global_alloc_page(mem_glob, ttm->pages[i], 872 false, false); 873 if (unlikely(ret != 0)) { 874 ttm_pool_unpopulate(ttm); 875 return -ENOMEM; 876 } 877 } 878 879 if (unlikely(ttm->page_flags & TTM_PAGE_FLAG_SWAPPED)) { 880 ret = ttm_tt_swapin(ttm); 881 if (unlikely(ret != 0)) { 882 ttm_pool_unpopulate(ttm); 883 return ret; 884 } 885 } 886 887 ttm->state = tt_unbound; 888 return 0; 889} 890 891void ttm_pool_unpopulate(struct ttm_tt *ttm) 892{ 893 unsigned i; 894 895 for (i = 0; i < ttm->num_pages; ++i) { 896 if (ttm->pages[i]) { 897 ttm_mem_global_free_page(ttm->glob->mem_glob, 898 ttm->pages[i]); 899 ttm_put_pages(&ttm->pages[i], 1, 900 ttm->page_flags, 901 ttm->caching_state); 902 } 903 } 904 ttm->state = tt_unpopulated; 905} 906 907#if 0 908/* XXXKIB sysctl */ 909int ttm_page_alloc_debugfs(struct seq_file *m, void *data) 910{ 911 struct ttm_page_pool *p; 912 unsigned i; 913 char *h[] = {"pool", "refills", "pages freed", "size"}; 914 if (!_manager) { 915 seq_printf(m, "No pool allocator running.\n"); 916 return 0; 917 } 918 seq_printf(m, "%6s %12s %13s %8s\n", 919 h[0], h[1], h[2], h[3]); 920 for (i = 0; i < NUM_POOLS; ++i) { 921 p = &_manager->pools[i]; 922 923 seq_printf(m, "%6s %12ld %13ld %8d\n", 924 p->name, p->nrefills, 925 p->nfrees, p->npages); 926 } 927 return 0; 928} 929#endif 930