1/* 2 * Copyright (c) Red Hat Inc. 3 4 * Permission is hereby granted, free of charge, to any person obtaining a 5 * copy of this software and associated documentation files (the "Software"), 6 * to deal in the Software without restriction, including without limitation 7 * the rights to use, copy, modify, merge, publish, distribute, sub license, 8 * and/or sell copies of the Software, and to permit persons to whom the 9 * Software is furnished to do so, subject to the following conditions: 10 * 11 * The above copyright notice and this permission notice (including the 12 * next paragraph) shall be included in all copies or substantial portions 13 * of the Software. 14 * 15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL 18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER 21 * DEALINGS IN THE SOFTWARE. 22 * 23 * Authors: Dave Airlie <airlied@redhat.com> 24 * Jerome Glisse <jglisse@redhat.com> 25 * Pauli Nieminen <suokkos@gmail.com> 26 */ 27/* 28 * Copyright (c) 2013 The FreeBSD Foundation 29 * All rights reserved. 30 * 31 * Portions of this software were developed by Konstantin Belousov 32 * <kib@FreeBSD.org> under sponsorship from the FreeBSD Foundation. 33 */ 34 35/* simple list based uncached page pool 36 * - Pool collects resently freed pages for reuse 37 * - Use page->lru to keep a free list 38 * - doesn't track currently in use pages 39 */ 40 41#include <sys/cdefs.h> 42__FBSDID("$FreeBSD: stable/11/sys/dev/drm2/ttm/ttm_page_alloc.c 318848 2017-05-25 01:17:07Z markj $"); 43 44#include <dev/drm2/drmP.h> 45#include <dev/drm2/ttm/ttm_bo_driver.h> 46#include <dev/drm2/ttm/ttm_page_alloc.h> 47#include <vm/vm_pageout.h> 48 49#define NUM_PAGES_TO_ALLOC (PAGE_SIZE/sizeof(vm_page_t)) 50#define SMALL_ALLOCATION 16 51#define FREE_ALL_PAGES (~0U) 52/* times are in msecs */ 53#define PAGE_FREE_INTERVAL 1000 54 55/** 56 * struct ttm_page_pool - Pool to reuse recently allocated uc/wc pages. 57 * 58 * @lock: Protects the shared pool from concurrnet access. Must be used with 59 * irqsave/irqrestore variants because pool allocator maybe called from 60 * delayed work. 61 * @fill_lock: Prevent concurrent calls to fill. 62 * @list: Pool of free uc/wc pages for fast reuse. 63 * @gfp_flags: Flags to pass for alloc_page. 64 * @npages: Number of pages in pool. 65 */ 66struct ttm_page_pool { 67 struct mtx lock; 68 bool fill_lock; 69 bool dma32; 70 struct pglist list; 71 int ttm_page_alloc_flags; 72 unsigned npages; 73 char *name; 74 unsigned long nfrees; 75 unsigned long nrefills; 76}; 77 78/** 79 * Limits for the pool. They are handled without locks because only place where 80 * they may change is in sysfs store. They won't have immediate effect anyway 81 * so forcing serialization to access them is pointless. 82 */ 83 84struct ttm_pool_opts { 85 unsigned alloc_size; 86 unsigned max_size; 87 unsigned small; 88}; 89 90#define NUM_POOLS 4 91 92/** 93 * struct ttm_pool_manager - Holds memory pools for fst allocation 94 * 95 * Manager is read only object for pool code so it doesn't need locking. 96 * 97 * @free_interval: minimum number of jiffies between freeing pages from pool. 98 * @page_alloc_inited: reference counting for pool allocation. 99 * @work: Work that is used to shrink the pool. Work is only run when there is 100 * some pages to free. 101 * @small_allocation: Limit in number of pages what is small allocation. 102 * 103 * @pools: All pool objects in use. 104 **/ 105struct ttm_pool_manager { 106 unsigned int kobj_ref; 107 eventhandler_tag lowmem_handler; 108 struct ttm_pool_opts options; 109 110 union { 111 struct ttm_page_pool u_pools[NUM_POOLS]; 112 struct _utag { 113 struct ttm_page_pool u_wc_pool; 114 struct ttm_page_pool u_uc_pool; 115 struct ttm_page_pool u_wc_pool_dma32; 116 struct ttm_page_pool u_uc_pool_dma32; 117 } _ut; 118 } _u; 119}; 120 121#define pools _u.u_pools 122#define wc_pool _u._ut.u_wc_pool 123#define uc_pool _u._ut.u_uc_pool 124#define wc_pool_dma32 _u._ut.u_wc_pool_dma32 125#define uc_pool_dma32 _u._ut.u_uc_pool_dma32 126 127MALLOC_DEFINE(M_TTM_POOLMGR, "ttm_poolmgr", "TTM Pool Manager"); 128 129static void 130ttm_vm_page_free(vm_page_t m) 131{ 132 133 KASSERT(m->object == NULL, ("ttm page %p is owned", m)); 134 KASSERT(m->wire_count == 1, ("ttm lost wire %p", m)); 135 KASSERT((m->flags & PG_FICTITIOUS) != 0, ("ttm lost fictitious %p", m)); 136 KASSERT((m->oflags & VPO_UNMANAGED) == 0, ("ttm got unmanaged %p", m)); 137 m->flags &= ~PG_FICTITIOUS; 138 m->oflags |= VPO_UNMANAGED; 139 vm_page_unwire(m, PQ_NONE); 140 vm_page_free(m); 141} 142 143static vm_memattr_t 144ttm_caching_state_to_vm(enum ttm_caching_state cstate) 145{ 146 147 switch (cstate) { 148 case tt_uncached: 149 return (VM_MEMATTR_UNCACHEABLE); 150 case tt_wc: 151 return (VM_MEMATTR_WRITE_COMBINING); 152 case tt_cached: 153 return (VM_MEMATTR_WRITE_BACK); 154 } 155 panic("caching state %d\n", cstate); 156} 157 158static vm_page_t 159ttm_vm_page_alloc_dma32(int req, vm_memattr_t memattr) 160{ 161 vm_page_t p; 162 int tries; 163 164 for (tries = 0; ; tries++) { 165 p = vm_page_alloc_contig(NULL, 0, req, 1, 0, 0xffffffff, 166 PAGE_SIZE, 0, memattr); 167 if (p != NULL || tries > 2) 168 return (p); 169 if (!vm_page_reclaim_contig(req, 1, 0, 0xffffffff, 170 PAGE_SIZE, 0)) 171 VM_WAIT; 172 } 173} 174 175static vm_page_t 176ttm_vm_page_alloc_any(int req, vm_memattr_t memattr) 177{ 178 vm_page_t p; 179 180 while (1) { 181 p = vm_page_alloc(NULL, 0, req); 182 if (p != NULL) 183 break; 184 VM_WAIT; 185 } 186 pmap_page_set_memattr(p, memattr); 187 return (p); 188} 189 190static vm_page_t 191ttm_vm_page_alloc(int flags, enum ttm_caching_state cstate) 192{ 193 vm_page_t p; 194 vm_memattr_t memattr; 195 int req; 196 197 memattr = ttm_caching_state_to_vm(cstate); 198 req = VM_ALLOC_NORMAL | VM_ALLOC_WIRED | VM_ALLOC_NOOBJ; 199 if ((flags & TTM_PAGE_FLAG_ZERO_ALLOC) != 0) 200 req |= VM_ALLOC_ZERO; 201 202 if ((flags & TTM_PAGE_FLAG_DMA32) != 0) 203 p = ttm_vm_page_alloc_dma32(req, memattr); 204 else 205 p = ttm_vm_page_alloc_any(req, memattr); 206 207 if (p != NULL) { 208 p->oflags &= ~VPO_UNMANAGED; 209 p->flags |= PG_FICTITIOUS; 210 } 211 return (p); 212} 213 214static void ttm_pool_kobj_release(struct ttm_pool_manager *m) 215{ 216 217 free(m, M_TTM_POOLMGR); 218} 219 220#if 0 221/* XXXKIB sysctl */ 222static ssize_t ttm_pool_store(struct ttm_pool_manager *m, 223 struct attribute *attr, const char *buffer, size_t size) 224{ 225 int chars; 226 unsigned val; 227 chars = sscanf(buffer, "%u", &val); 228 if (chars == 0) 229 return size; 230 231 /* Convert kb to number of pages */ 232 val = val / (PAGE_SIZE >> 10); 233 234 if (attr == &ttm_page_pool_max) 235 m->options.max_size = val; 236 else if (attr == &ttm_page_pool_small) 237 m->options.small = val; 238 else if (attr == &ttm_page_pool_alloc_size) { 239 if (val > NUM_PAGES_TO_ALLOC*8) { 240 pr_err("Setting allocation size to %lu is not allowed. Recommended size is %lu\n", 241 NUM_PAGES_TO_ALLOC*(PAGE_SIZE >> 7), 242 NUM_PAGES_TO_ALLOC*(PAGE_SIZE >> 10)); 243 return size; 244 } else if (val > NUM_PAGES_TO_ALLOC) { 245 pr_warn("Setting allocation size to larger than %lu is not recommended\n", 246 NUM_PAGES_TO_ALLOC*(PAGE_SIZE >> 10)); 247 } 248 m->options.alloc_size = val; 249 } 250 251 return size; 252} 253 254static ssize_t ttm_pool_show(struct ttm_pool_manager *m, 255 struct attribute *attr, char *buffer) 256{ 257 unsigned val = 0; 258 259 if (attr == &ttm_page_pool_max) 260 val = m->options.max_size; 261 else if (attr == &ttm_page_pool_small) 262 val = m->options.small; 263 else if (attr == &ttm_page_pool_alloc_size) 264 val = m->options.alloc_size; 265 266 val = val * (PAGE_SIZE >> 10); 267 268 return snprintf(buffer, PAGE_SIZE, "%u\n", val); 269} 270#endif 271 272static struct ttm_pool_manager *_manager; 273 274static int set_pages_array_wb(vm_page_t *pages, int addrinarray) 275{ 276#ifdef TTM_HAS_AGP 277 int i; 278 279 for (i = 0; i < addrinarray; i++) 280 pmap_page_set_memattr(pages[i], VM_MEMATTR_WRITE_BACK); 281#endif 282 return 0; 283} 284 285static int set_pages_array_wc(vm_page_t *pages, int addrinarray) 286{ 287#ifdef TTM_HAS_AGP 288 int i; 289 290 for (i = 0; i < addrinarray; i++) 291 pmap_page_set_memattr(pages[i], VM_MEMATTR_WRITE_COMBINING); 292#endif 293 return 0; 294} 295 296static int set_pages_array_uc(vm_page_t *pages, int addrinarray) 297{ 298#ifdef TTM_HAS_AGP 299 int i; 300 301 for (i = 0; i < addrinarray; i++) 302 pmap_page_set_memattr(pages[i], VM_MEMATTR_UNCACHEABLE); 303#endif 304 return 0; 305} 306 307/** 308 * Select the right pool or requested caching state and ttm flags. */ 309static struct ttm_page_pool *ttm_get_pool(int flags, 310 enum ttm_caching_state cstate) 311{ 312 int pool_index; 313 314 if (cstate == tt_cached) 315 return NULL; 316 317 if (cstate == tt_wc) 318 pool_index = 0x0; 319 else 320 pool_index = 0x1; 321 322 if (flags & TTM_PAGE_FLAG_DMA32) 323 pool_index |= 0x2; 324 325 return &_manager->pools[pool_index]; 326} 327 328/* set memory back to wb and free the pages. */ 329static void ttm_pages_put(vm_page_t *pages, unsigned npages) 330{ 331 unsigned i; 332 333 /* Our VM handles vm memattr automatically on the page free. */ 334 if (set_pages_array_wb(pages, npages)) 335 printf("[TTM] Failed to set %d pages to wb!\n", npages); 336 for (i = 0; i < npages; ++i) 337 ttm_vm_page_free(pages[i]); 338} 339 340static void ttm_pool_update_free_locked(struct ttm_page_pool *pool, 341 unsigned freed_pages) 342{ 343 pool->npages -= freed_pages; 344 pool->nfrees += freed_pages; 345} 346 347/** 348 * Free pages from pool. 349 * 350 * To prevent hogging the ttm_swap process we only free NUM_PAGES_TO_ALLOC 351 * number of pages in one go. 352 * 353 * @pool: to free the pages from 354 * @free_all: If set to true will free all pages in pool 355 **/ 356static int ttm_page_pool_free(struct ttm_page_pool *pool, unsigned nr_free) 357{ 358 vm_page_t p, p1; 359 vm_page_t *pages_to_free; 360 unsigned freed_pages = 0, 361 npages_to_free = nr_free; 362 unsigned i; 363 364 if (NUM_PAGES_TO_ALLOC < nr_free) 365 npages_to_free = NUM_PAGES_TO_ALLOC; 366 367 pages_to_free = malloc(npages_to_free * sizeof(vm_page_t), 368 M_TEMP, M_WAITOK | M_ZERO); 369 370restart: 371 mtx_lock(&pool->lock); 372 373 TAILQ_FOREACH_REVERSE_SAFE(p, &pool->list, pglist, plinks.q, p1) { 374 if (freed_pages >= npages_to_free) 375 break; 376 377 pages_to_free[freed_pages++] = p; 378 /* We can only remove NUM_PAGES_TO_ALLOC at a time. */ 379 if (freed_pages >= NUM_PAGES_TO_ALLOC) { 380 /* remove range of pages from the pool */ 381 for (i = 0; i < freed_pages; i++) 382 TAILQ_REMOVE(&pool->list, pages_to_free[i], plinks.q); 383 384 ttm_pool_update_free_locked(pool, freed_pages); 385 /** 386 * Because changing page caching is costly 387 * we unlock the pool to prevent stalling. 388 */ 389 mtx_unlock(&pool->lock); 390 391 ttm_pages_put(pages_to_free, freed_pages); 392 if (likely(nr_free != FREE_ALL_PAGES)) 393 nr_free -= freed_pages; 394 395 if (NUM_PAGES_TO_ALLOC >= nr_free) 396 npages_to_free = nr_free; 397 else 398 npages_to_free = NUM_PAGES_TO_ALLOC; 399 400 freed_pages = 0; 401 402 /* free all so restart the processing */ 403 if (nr_free) 404 goto restart; 405 406 /* Not allowed to fall through or break because 407 * following context is inside spinlock while we are 408 * outside here. 409 */ 410 goto out; 411 412 } 413 } 414 415 /* remove range of pages from the pool */ 416 if (freed_pages) { 417 for (i = 0; i < freed_pages; i++) 418 TAILQ_REMOVE(&pool->list, pages_to_free[i], plinks.q); 419 420 ttm_pool_update_free_locked(pool, freed_pages); 421 nr_free -= freed_pages; 422 } 423 424 mtx_unlock(&pool->lock); 425 426 if (freed_pages) 427 ttm_pages_put(pages_to_free, freed_pages); 428out: 429 free(pages_to_free, M_TEMP); 430 return nr_free; 431} 432 433/* Get good estimation how many pages are free in pools */ 434static int ttm_pool_get_num_unused_pages(void) 435{ 436 unsigned i; 437 int total = 0; 438 for (i = 0; i < NUM_POOLS; ++i) 439 total += _manager->pools[i].npages; 440 441 return total; 442} 443 444/** 445 * Callback for mm to request pool to reduce number of page held. 446 */ 447static int ttm_pool_mm_shrink(void *arg) 448{ 449 static unsigned int start_pool = 0; 450 unsigned i; 451 unsigned pool_offset = atomic_fetchadd_int(&start_pool, 1); 452 struct ttm_page_pool *pool; 453 int shrink_pages = 100; /* XXXKIB */ 454 455 pool_offset = pool_offset % NUM_POOLS; 456 /* select start pool in round robin fashion */ 457 for (i = 0; i < NUM_POOLS; ++i) { 458 unsigned nr_free = shrink_pages; 459 if (shrink_pages == 0) 460 break; 461 pool = &_manager->pools[(i + pool_offset)%NUM_POOLS]; 462 shrink_pages = ttm_page_pool_free(pool, nr_free); 463 } 464 /* return estimated number of unused pages in pool */ 465 return ttm_pool_get_num_unused_pages(); 466} 467 468static void ttm_pool_mm_shrink_init(struct ttm_pool_manager *manager) 469{ 470 471 manager->lowmem_handler = EVENTHANDLER_REGISTER(vm_lowmem, 472 ttm_pool_mm_shrink, manager, EVENTHANDLER_PRI_ANY); 473} 474 475static void ttm_pool_mm_shrink_fini(struct ttm_pool_manager *manager) 476{ 477 478 EVENTHANDLER_DEREGISTER(vm_lowmem, manager->lowmem_handler); 479} 480 481static int ttm_set_pages_caching(vm_page_t *pages, 482 enum ttm_caching_state cstate, unsigned cpages) 483{ 484 int r = 0; 485 /* Set page caching */ 486 switch (cstate) { 487 case tt_uncached: 488 r = set_pages_array_uc(pages, cpages); 489 if (r) 490 printf("[TTM] Failed to set %d pages to uc!\n", cpages); 491 break; 492 case tt_wc: 493 r = set_pages_array_wc(pages, cpages); 494 if (r) 495 printf("[TTM] Failed to set %d pages to wc!\n", cpages); 496 break; 497 default: 498 break; 499 } 500 return r; 501} 502 503/** 504 * Free pages the pages that failed to change the caching state. If there is 505 * any pages that have changed their caching state already put them to the 506 * pool. 507 */ 508static void ttm_handle_caching_state_failure(struct pglist *pages, 509 int ttm_flags, enum ttm_caching_state cstate, 510 vm_page_t *failed_pages, unsigned cpages) 511{ 512 unsigned i; 513 /* Failed pages have to be freed */ 514 for (i = 0; i < cpages; ++i) { 515 TAILQ_REMOVE(pages, failed_pages[i], plinks.q); 516 ttm_vm_page_free(failed_pages[i]); 517 } 518} 519 520/** 521 * Allocate new pages with correct caching. 522 * 523 * This function is reentrant if caller updates count depending on number of 524 * pages returned in pages array. 525 */ 526static int ttm_alloc_new_pages(struct pglist *pages, int ttm_alloc_flags, 527 int ttm_flags, enum ttm_caching_state cstate, unsigned count) 528{ 529 vm_page_t *caching_array; 530 vm_page_t p; 531 int r = 0; 532 unsigned i, cpages; 533 unsigned max_cpages = min(count, 534 (unsigned)(PAGE_SIZE/sizeof(vm_page_t))); 535 536 /* allocate array for page caching change */ 537 caching_array = malloc(max_cpages * sizeof(vm_page_t), M_TEMP, 538 M_WAITOK | M_ZERO); 539 540 for (i = 0, cpages = 0; i < count; ++i) { 541 p = ttm_vm_page_alloc(ttm_alloc_flags, cstate); 542 if (!p) { 543 printf("[TTM] Unable to get page %u\n", i); 544 545 /* store already allocated pages in the pool after 546 * setting the caching state */ 547 if (cpages) { 548 r = ttm_set_pages_caching(caching_array, 549 cstate, cpages); 550 if (r) 551 ttm_handle_caching_state_failure(pages, 552 ttm_flags, cstate, 553 caching_array, cpages); 554 } 555 r = -ENOMEM; 556 goto out; 557 } 558 559#ifdef CONFIG_HIGHMEM /* KIB: nop */ 560 /* gfp flags of highmem page should never be dma32 so we 561 * we should be fine in such case 562 */ 563 if (!PageHighMem(p)) 564#endif 565 { 566 caching_array[cpages++] = p; 567 if (cpages == max_cpages) { 568 569 r = ttm_set_pages_caching(caching_array, 570 cstate, cpages); 571 if (r) { 572 ttm_handle_caching_state_failure(pages, 573 ttm_flags, cstate, 574 caching_array, cpages); 575 goto out; 576 } 577 cpages = 0; 578 } 579 } 580 581 TAILQ_INSERT_HEAD(pages, p, plinks.q); 582 } 583 584 if (cpages) { 585 r = ttm_set_pages_caching(caching_array, cstate, cpages); 586 if (r) 587 ttm_handle_caching_state_failure(pages, 588 ttm_flags, cstate, 589 caching_array, cpages); 590 } 591out: 592 free(caching_array, M_TEMP); 593 594 return r; 595} 596 597/** 598 * Fill the given pool if there aren't enough pages and the requested number of 599 * pages is small. 600 */ 601static void ttm_page_pool_fill_locked(struct ttm_page_pool *pool, 602 int ttm_flags, enum ttm_caching_state cstate, unsigned count) 603{ 604 vm_page_t p; 605 int r; 606 unsigned cpages = 0; 607 /** 608 * Only allow one pool fill operation at a time. 609 * If pool doesn't have enough pages for the allocation new pages are 610 * allocated from outside of pool. 611 */ 612 if (pool->fill_lock) 613 return; 614 615 pool->fill_lock = true; 616 617 /* If allocation request is small and there are not enough 618 * pages in a pool we fill the pool up first. */ 619 if (count < _manager->options.small 620 && count > pool->npages) { 621 struct pglist new_pages; 622 unsigned alloc_size = _manager->options.alloc_size; 623 624 /** 625 * Can't change page caching if in irqsave context. We have to 626 * drop the pool->lock. 627 */ 628 mtx_unlock(&pool->lock); 629 630 TAILQ_INIT(&new_pages); 631 r = ttm_alloc_new_pages(&new_pages, pool->ttm_page_alloc_flags, 632 ttm_flags, cstate, alloc_size); 633 mtx_lock(&pool->lock); 634 635 if (!r) { 636 TAILQ_CONCAT(&pool->list, &new_pages, plinks.q); 637 ++pool->nrefills; 638 pool->npages += alloc_size; 639 } else { 640 printf("[TTM] Failed to fill pool (%p)\n", pool); 641 /* If we have any pages left put them to the pool. */ 642 TAILQ_FOREACH(p, &pool->list, plinks.q) { 643 ++cpages; 644 } 645 TAILQ_CONCAT(&pool->list, &new_pages, plinks.q); 646 pool->npages += cpages; 647 } 648 649 } 650 pool->fill_lock = false; 651} 652 653/** 654 * Cut 'count' number of pages from the pool and put them on the return list. 655 * 656 * @return count of pages still required to fulfill the request. 657 */ 658static unsigned ttm_page_pool_get_pages(struct ttm_page_pool *pool, 659 struct pglist *pages, 660 int ttm_flags, 661 enum ttm_caching_state cstate, 662 unsigned count) 663{ 664 vm_page_t p; 665 unsigned i; 666 667 mtx_lock(&pool->lock); 668 ttm_page_pool_fill_locked(pool, ttm_flags, cstate, count); 669 670 if (count >= pool->npages) { 671 /* take all pages from the pool */ 672 TAILQ_CONCAT(pages, &pool->list, plinks.q); 673 count -= pool->npages; 674 pool->npages = 0; 675 goto out; 676 } 677 for (i = 0; i < count; i++) { 678 p = TAILQ_FIRST(&pool->list); 679 TAILQ_REMOVE(&pool->list, p, plinks.q); 680 TAILQ_INSERT_TAIL(pages, p, plinks.q); 681 } 682 pool->npages -= count; 683 count = 0; 684out: 685 mtx_unlock(&pool->lock); 686 return count; 687} 688 689/* Put all pages in pages list to correct pool to wait for reuse */ 690static void ttm_put_pages(vm_page_t *pages, unsigned npages, int flags, 691 enum ttm_caching_state cstate) 692{ 693 struct ttm_page_pool *pool = ttm_get_pool(flags, cstate); 694 unsigned i; 695 696 if (pool == NULL) { 697 /* No pool for this memory type so free the pages */ 698 for (i = 0; i < npages; i++) { 699 if (pages[i]) { 700 ttm_vm_page_free(pages[i]); 701 pages[i] = NULL; 702 } 703 } 704 return; 705 } 706 707 mtx_lock(&pool->lock); 708 for (i = 0; i < npages; i++) { 709 if (pages[i]) { 710 TAILQ_INSERT_TAIL(&pool->list, pages[i], plinks.q); 711 pages[i] = NULL; 712 pool->npages++; 713 } 714 } 715 /* Check that we don't go over the pool limit */ 716 npages = 0; 717 if (pool->npages > _manager->options.max_size) { 718 npages = pool->npages - _manager->options.max_size; 719 /* free at least NUM_PAGES_TO_ALLOC number of pages 720 * to reduce calls to set_memory_wb */ 721 if (npages < NUM_PAGES_TO_ALLOC) 722 npages = NUM_PAGES_TO_ALLOC; 723 } 724 mtx_unlock(&pool->lock); 725 if (npages) 726 ttm_page_pool_free(pool, npages); 727} 728 729/* 730 * On success pages list will hold count number of correctly 731 * cached pages. 732 */ 733static int ttm_get_pages(vm_page_t *pages, unsigned npages, int flags, 734 enum ttm_caching_state cstate) 735{ 736 struct ttm_page_pool *pool = ttm_get_pool(flags, cstate); 737 struct pglist plist; 738 vm_page_t p = NULL; 739 int gfp_flags; 740 unsigned count; 741 int r; 742 743 /* No pool for cached pages */ 744 if (pool == NULL) { 745 for (r = 0; r < npages; ++r) { 746 p = ttm_vm_page_alloc(flags, cstate); 747 if (!p) { 748 printf("[TTM] Unable to allocate page\n"); 749 return -ENOMEM; 750 } 751 pages[r] = p; 752 } 753 return 0; 754 } 755 756 /* combine zero flag to pool flags */ 757 gfp_flags = flags | pool->ttm_page_alloc_flags; 758 759 /* First we take pages from the pool */ 760 TAILQ_INIT(&plist); 761 npages = ttm_page_pool_get_pages(pool, &plist, flags, cstate, npages); 762 count = 0; 763 TAILQ_FOREACH(p, &plist, plinks.q) { 764 pages[count++] = p; 765 } 766 767 /* clear the pages coming from the pool if requested */ 768 if (flags & TTM_PAGE_FLAG_ZERO_ALLOC) { 769 TAILQ_FOREACH(p, &plist, plinks.q) { 770 pmap_zero_page(p); 771 } 772 } 773 774 /* If pool didn't have enough pages allocate new one. */ 775 if (npages > 0) { 776 /* ttm_alloc_new_pages doesn't reference pool so we can run 777 * multiple requests in parallel. 778 **/ 779 TAILQ_INIT(&plist); 780 r = ttm_alloc_new_pages(&plist, gfp_flags, flags, cstate, 781 npages); 782 TAILQ_FOREACH(p, &plist, plinks.q) { 783 pages[count++] = p; 784 } 785 if (r) { 786 /* If there is any pages in the list put them back to 787 * the pool. */ 788 printf("[TTM] Failed to allocate extra pages for large request\n"); 789 ttm_put_pages(pages, count, flags, cstate); 790 return r; 791 } 792 } 793 794 return 0; 795} 796 797static void ttm_page_pool_init_locked(struct ttm_page_pool *pool, int flags, 798 char *name) 799{ 800 mtx_init(&pool->lock, "ttmpool", NULL, MTX_DEF); 801 pool->fill_lock = false; 802 TAILQ_INIT(&pool->list); 803 pool->npages = pool->nfrees = 0; 804 pool->ttm_page_alloc_flags = flags; 805 pool->name = name; 806} 807 808int ttm_page_alloc_init(struct ttm_mem_global *glob, unsigned max_pages) 809{ 810 811 if (_manager != NULL) 812 printf("[TTM] manager != NULL\n"); 813 printf("[TTM] Initializing pool allocator\n"); 814 815 _manager = malloc(sizeof(*_manager), M_TTM_POOLMGR, M_WAITOK | M_ZERO); 816 817 ttm_page_pool_init_locked(&_manager->wc_pool, 0, "wc"); 818 ttm_page_pool_init_locked(&_manager->uc_pool, 0, "uc"); 819 ttm_page_pool_init_locked(&_manager->wc_pool_dma32, 820 TTM_PAGE_FLAG_DMA32, "wc dma"); 821 ttm_page_pool_init_locked(&_manager->uc_pool_dma32, 822 TTM_PAGE_FLAG_DMA32, "uc dma"); 823 824 _manager->options.max_size = max_pages; 825 _manager->options.small = SMALL_ALLOCATION; 826 _manager->options.alloc_size = NUM_PAGES_TO_ALLOC; 827 828 refcount_init(&_manager->kobj_ref, 1); 829 ttm_pool_mm_shrink_init(_manager); 830 831 return 0; 832} 833 834void ttm_page_alloc_fini(void) 835{ 836 int i; 837 838 printf("[TTM] Finalizing pool allocator\n"); 839 ttm_pool_mm_shrink_fini(_manager); 840 841 for (i = 0; i < NUM_POOLS; ++i) 842 ttm_page_pool_free(&_manager->pools[i], FREE_ALL_PAGES); 843 844 if (refcount_release(&_manager->kobj_ref)) 845 ttm_pool_kobj_release(_manager); 846 _manager = NULL; 847} 848 849int ttm_pool_populate(struct ttm_tt *ttm) 850{ 851 struct ttm_mem_global *mem_glob = ttm->glob->mem_glob; 852 unsigned i; 853 int ret; 854 855 if (ttm->state != tt_unpopulated) 856 return 0; 857 858 for (i = 0; i < ttm->num_pages; ++i) { 859 ret = ttm_get_pages(&ttm->pages[i], 1, 860 ttm->page_flags, 861 ttm->caching_state); 862 if (ret != 0) { 863 ttm_pool_unpopulate(ttm); 864 return -ENOMEM; 865 } 866 867 ret = ttm_mem_global_alloc_page(mem_glob, ttm->pages[i], 868 false, false); 869 if (unlikely(ret != 0)) { 870 ttm_pool_unpopulate(ttm); 871 return -ENOMEM; 872 } 873 } 874 875 if (unlikely(ttm->page_flags & TTM_PAGE_FLAG_SWAPPED)) { 876 ret = ttm_tt_swapin(ttm); 877 if (unlikely(ret != 0)) { 878 ttm_pool_unpopulate(ttm); 879 return ret; 880 } 881 } 882 883 ttm->state = tt_unbound; 884 return 0; 885} 886 887void ttm_pool_unpopulate(struct ttm_tt *ttm) 888{ 889 unsigned i; 890 891 for (i = 0; i < ttm->num_pages; ++i) { 892 if (ttm->pages[i]) { 893 ttm_mem_global_free_page(ttm->glob->mem_glob, 894 ttm->pages[i]); 895 ttm_put_pages(&ttm->pages[i], 1, 896 ttm->page_flags, 897 ttm->caching_state); 898 } 899 } 900 ttm->state = tt_unpopulated; 901} 902 903#if 0 904/* XXXKIB sysctl */ 905int ttm_page_alloc_debugfs(struct seq_file *m, void *data) 906{ 907 struct ttm_page_pool *p; 908 unsigned i; 909 char *h[] = {"pool", "refills", "pages freed", "size"}; 910 if (!_manager) { 911 seq_printf(m, "No pool allocator running.\n"); 912 return 0; 913 } 914 seq_printf(m, "%6s %12s %13s %8s\n", 915 h[0], h[1], h[2], h[3]); 916 for (i = 0; i < NUM_POOLS; ++i) { 917 p = &_manager->pools[i]; 918 919 seq_printf(m, "%6s %12ld %13ld %8d\n", 920 p->name, p->nrefills, 921 p->nfrees, p->npages); 922 } 923 return 0; 924} 925#endif 926