1247835Skib/* 2247835Skib * Copyright (c) Red Hat Inc. 3247835Skib 4247835Skib * Permission is hereby granted, free of charge, to any person obtaining a 5247835Skib * copy of this software and associated documentation files (the "Software"), 6247835Skib * to deal in the Software without restriction, including without limitation 7247835Skib * the rights to use, copy, modify, merge, publish, distribute, sub license, 8247835Skib * and/or sell copies of the Software, and to permit persons to whom the 9247835Skib * Software is furnished to do so, subject to the following conditions: 10247835Skib * 11247835Skib * The above copyright notice and this permission notice (including the 12247835Skib * next paragraph) shall be included in all copies or substantial portions 13247835Skib * of the Software. 14247835Skib * 15247835Skib * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16247835Skib * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17247835Skib * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL 18247835Skib * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19247835Skib * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 20247835Skib * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER 21247835Skib * DEALINGS IN THE SOFTWARE. 22247835Skib * 23247835Skib * Authors: Dave Airlie <airlied@redhat.com> 24247835Skib * Jerome Glisse <jglisse@redhat.com> 25247835Skib * Pauli Nieminen <suokkos@gmail.com> 26247835Skib */ 27247835Skib/* 28247835Skib * Copyright (c) 2013 The FreeBSD Foundation 29247835Skib * All rights reserved. 30247835Skib * 31247835Skib * Portions of this software were developed by Konstantin Belousov 32247835Skib * <kib@FreeBSD.org> under sponsorship from the FreeBSD Foundation. 33247835Skib */ 34247835Skib 35247835Skib/* simple list based uncached page pool 36247835Skib * - Pool collects resently freed pages for reuse 37247835Skib * - Use page->lru to keep a free list 38247835Skib * - doesn't track currently in use pages 39247835Skib */ 40247835Skib 41247835Skib#include <sys/cdefs.h> 42247835Skib__FBSDID("$FreeBSD$"); 43247835Skib 44247835Skib#include <dev/drm2/drmP.h> 45247835Skib#include <dev/drm2/ttm/ttm_bo_driver.h> 46247835Skib#include <dev/drm2/ttm/ttm_page_alloc.h> 47247835Skib 48247835Skib#ifdef TTM_HAS_AGP 49247835Skib#include <asm/agp.h> 50247835Skib#endif 51247835Skib 52247835Skib#define NUM_PAGES_TO_ALLOC (PAGE_SIZE/sizeof(vm_page_t)) 53247835Skib#define SMALL_ALLOCATION 16 54247835Skib#define FREE_ALL_PAGES (~0U) 55247835Skib/* times are in msecs */ 56247835Skib#define PAGE_FREE_INTERVAL 1000 57247835Skib 58247835Skib/** 59247835Skib * struct ttm_page_pool - Pool to reuse recently allocated uc/wc pages. 60247835Skib * 61247835Skib * @lock: Protects the shared pool from concurrnet access. Must be used with 62247835Skib * irqsave/irqrestore variants because pool allocator maybe called from 63247835Skib * delayed work. 64247835Skib * @fill_lock: Prevent concurrent calls to fill. 65247835Skib * @list: Pool of free uc/wc pages for fast reuse. 66247835Skib * @gfp_flags: Flags to pass for alloc_page. 67247835Skib * @npages: Number of pages in pool. 68247835Skib */ 69247835Skibstruct ttm_page_pool { 70247835Skib struct mtx lock; 71247835Skib bool fill_lock; 72247835Skib bool dma32; 73247835Skib struct pglist list; 74247835Skib int ttm_page_alloc_flags; 75247835Skib unsigned npages; 76247835Skib char *name; 77247835Skib unsigned long nfrees; 78247835Skib unsigned long nrefills; 79247835Skib}; 80247835Skib 81247835Skib/** 82247835Skib * Limits for the pool. They are handled without locks because only place where 83247835Skib * they may change is in sysfs store. They won't have immediate effect anyway 84247835Skib * so forcing serialization to access them is pointless. 85247835Skib */ 86247835Skib 87247835Skibstruct ttm_pool_opts { 88247835Skib unsigned alloc_size; 89247835Skib unsigned max_size; 90247835Skib unsigned small; 91247835Skib}; 92247835Skib 93247835Skib#define NUM_POOLS 4 94247835Skib 95247835Skib/** 96247835Skib * struct ttm_pool_manager - Holds memory pools for fst allocation 97247835Skib * 98247835Skib * Manager is read only object for pool code so it doesn't need locking. 99247835Skib * 100247835Skib * @free_interval: minimum number of jiffies between freeing pages from pool. 101247835Skib * @page_alloc_inited: reference counting for pool allocation. 102247835Skib * @work: Work that is used to shrink the pool. Work is only run when there is 103247835Skib * some pages to free. 104247835Skib * @small_allocation: Limit in number of pages what is small allocation. 105247835Skib * 106247835Skib * @pools: All pool objects in use. 107247835Skib **/ 108247835Skibstruct ttm_pool_manager { 109247835Skib unsigned int kobj_ref; 110247835Skib eventhandler_tag lowmem_handler; 111247835Skib struct ttm_pool_opts options; 112247835Skib 113247835Skib union { 114247849Skib struct ttm_page_pool u_pools[NUM_POOLS]; 115247849Skib struct _utag { 116247849Skib struct ttm_page_pool u_wc_pool; 117247849Skib struct ttm_page_pool u_uc_pool; 118247849Skib struct ttm_page_pool u_wc_pool_dma32; 119247849Skib struct ttm_page_pool u_uc_pool_dma32; 120247849Skib } _ut; 121247849Skib } _u; 122247835Skib}; 123247835Skib 124247849Skib#define pools _u.u_pools 125247849Skib#define wc_pool _u._ut.u_wc_pool 126247849Skib#define uc_pool _u._ut.u_uc_pool 127247849Skib#define wc_pool_dma32 _u._ut.u_wc_pool_dma32 128247849Skib#define uc_pool_dma32 _u._ut.u_uc_pool_dma32 129247849Skib 130247835SkibMALLOC_DEFINE(M_TTM_POOLMGR, "ttm_poolmgr", "TTM Pool Manager"); 131247835Skib 132247835Skibstatic void 133247835Skibttm_vm_page_free(vm_page_t m) 134247835Skib{ 135247835Skib 136247835Skib KASSERT(m->object == NULL, ("ttm page %p is owned", m)); 137247835Skib KASSERT(m->wire_count == 1, ("ttm lost wire %p", m)); 138247835Skib KASSERT((m->flags & PG_FICTITIOUS) != 0, ("ttm lost fictitious %p", m)); 139247835Skib KASSERT((m->oflags & VPO_UNMANAGED) == 0, ("ttm got unmanaged %p", m)); 140247835Skib m->flags &= ~PG_FICTITIOUS; 141247835Skib m->oflags |= VPO_UNMANAGED; 142247835Skib vm_page_unwire(m, 0); 143247835Skib vm_page_free(m); 144247835Skib} 145247835Skib 146247835Skibstatic vm_memattr_t 147247835Skibttm_caching_state_to_vm(enum ttm_caching_state cstate) 148247835Skib{ 149247835Skib 150247835Skib switch (cstate) { 151247835Skib case tt_uncached: 152247835Skib return (VM_MEMATTR_UNCACHEABLE); 153247835Skib case tt_wc: 154247835Skib return (VM_MEMATTR_WRITE_COMBINING); 155247835Skib case tt_cached: 156247835Skib return (VM_MEMATTR_WRITE_BACK); 157247835Skib } 158247835Skib panic("caching state %d\n", cstate); 159247835Skib} 160247835Skib 161247835Skibstatic void ttm_pool_kobj_release(struct ttm_pool_manager *m) 162247835Skib{ 163247835Skib 164247835Skib free(m, M_TTM_POOLMGR); 165247835Skib} 166247835Skib 167247835Skib#if 0 168247835Skib/* XXXKIB sysctl */ 169247835Skibstatic ssize_t ttm_pool_store(struct ttm_pool_manager *m, 170247835Skib struct attribute *attr, const char *buffer, size_t size) 171247835Skib{ 172247835Skib int chars; 173247835Skib unsigned val; 174247835Skib chars = sscanf(buffer, "%u", &val); 175247835Skib if (chars == 0) 176247835Skib return size; 177247835Skib 178247835Skib /* Convert kb to number of pages */ 179247835Skib val = val / (PAGE_SIZE >> 10); 180247835Skib 181247835Skib if (attr == &ttm_page_pool_max) 182247835Skib m->options.max_size = val; 183247835Skib else if (attr == &ttm_page_pool_small) 184247835Skib m->options.small = val; 185247835Skib else if (attr == &ttm_page_pool_alloc_size) { 186247835Skib if (val > NUM_PAGES_TO_ALLOC*8) { 187247835Skib pr_err("Setting allocation size to %lu is not allowed. Recommended size is %lu\n", 188247835Skib NUM_PAGES_TO_ALLOC*(PAGE_SIZE >> 7), 189247835Skib NUM_PAGES_TO_ALLOC*(PAGE_SIZE >> 10)); 190247835Skib return size; 191247835Skib } else if (val > NUM_PAGES_TO_ALLOC) { 192247835Skib pr_warn("Setting allocation size to larger than %lu is not recommended\n", 193247835Skib NUM_PAGES_TO_ALLOC*(PAGE_SIZE >> 10)); 194247835Skib } 195247835Skib m->options.alloc_size = val; 196247835Skib } 197247835Skib 198247835Skib return size; 199247835Skib} 200247835Skib 201247835Skibstatic ssize_t ttm_pool_show(struct ttm_pool_manager *m, 202247835Skib struct attribute *attr, char *buffer) 203247835Skib{ 204247835Skib unsigned val = 0; 205247835Skib 206247835Skib if (attr == &ttm_page_pool_max) 207247835Skib val = m->options.max_size; 208247835Skib else if (attr == &ttm_page_pool_small) 209247835Skib val = m->options.small; 210247835Skib else if (attr == &ttm_page_pool_alloc_size) 211247835Skib val = m->options.alloc_size; 212247835Skib 213247835Skib val = val * (PAGE_SIZE >> 10); 214247835Skib 215247835Skib return snprintf(buffer, PAGE_SIZE, "%u\n", val); 216247835Skib} 217247835Skib#endif 218247835Skib 219247835Skibstatic struct ttm_pool_manager *_manager; 220247835Skib 221247835Skibstatic int set_pages_array_wb(vm_page_t *pages, int addrinarray) 222247835Skib{ 223247835Skib vm_page_t m; 224247835Skib int i; 225247835Skib 226247835Skib for (i = 0; i < addrinarray; i++) { 227247835Skib m = pages[i]; 228247835Skib#ifdef TTM_HAS_AGP 229247835Skib unmap_page_from_agp(m); 230247835Skib#endif 231247835Skib pmap_page_set_memattr(m, VM_MEMATTR_WRITE_BACK); 232247835Skib } 233247835Skib return 0; 234247835Skib} 235247835Skib 236247835Skibstatic int set_pages_array_wc(vm_page_t *pages, int addrinarray) 237247835Skib{ 238247835Skib vm_page_t m; 239247835Skib int i; 240247835Skib 241247835Skib for (i = 0; i < addrinarray; i++) { 242247835Skib m = pages[i]; 243247835Skib#ifdef TTM_HAS_AGP 244247835Skib map_page_into_agp(pages[i]); 245247835Skib#endif 246247835Skib pmap_page_set_memattr(m, VM_MEMATTR_WRITE_COMBINING); 247247835Skib } 248247835Skib return 0; 249247835Skib} 250247835Skib 251247835Skibstatic int set_pages_array_uc(vm_page_t *pages, int addrinarray) 252247835Skib{ 253247835Skib vm_page_t m; 254247835Skib int i; 255247835Skib 256247835Skib for (i = 0; i < addrinarray; i++) { 257247835Skib m = pages[i]; 258247835Skib#ifdef TTM_HAS_AGP 259247835Skib map_page_into_agp(pages[i]); 260247835Skib#endif 261247835Skib pmap_page_set_memattr(m, VM_MEMATTR_UNCACHEABLE); 262247835Skib } 263247835Skib return 0; 264247835Skib} 265247835Skib 266247835Skib/** 267247835Skib * Select the right pool or requested caching state and ttm flags. */ 268247835Skibstatic struct ttm_page_pool *ttm_get_pool(int flags, 269247835Skib enum ttm_caching_state cstate) 270247835Skib{ 271247835Skib int pool_index; 272247835Skib 273247835Skib if (cstate == tt_cached) 274247835Skib return NULL; 275247835Skib 276247835Skib if (cstate == tt_wc) 277247835Skib pool_index = 0x0; 278247835Skib else 279247835Skib pool_index = 0x1; 280247835Skib 281247835Skib if (flags & TTM_PAGE_FLAG_DMA32) 282247835Skib pool_index |= 0x2; 283247835Skib 284247835Skib return &_manager->pools[pool_index]; 285247835Skib} 286247835Skib 287247835Skib/* set memory back to wb and free the pages. */ 288247835Skibstatic void ttm_pages_put(vm_page_t *pages, unsigned npages) 289247835Skib{ 290247835Skib unsigned i; 291247835Skib 292247835Skib /* Our VM handles vm memattr automatically on the page free. */ 293247835Skib if (set_pages_array_wb(pages, npages)) 294247835Skib printf("[TTM] Failed to set %d pages to wb!\n", npages); 295247835Skib for (i = 0; i < npages; ++i) 296247835Skib ttm_vm_page_free(pages[i]); 297247835Skib} 298247835Skib 299247835Skibstatic void ttm_pool_update_free_locked(struct ttm_page_pool *pool, 300247835Skib unsigned freed_pages) 301247835Skib{ 302247835Skib pool->npages -= freed_pages; 303247835Skib pool->nfrees += freed_pages; 304247835Skib} 305247835Skib 306247835Skib/** 307247835Skib * Free pages from pool. 308247835Skib * 309247835Skib * To prevent hogging the ttm_swap process we only free NUM_PAGES_TO_ALLOC 310247835Skib * number of pages in one go. 311247835Skib * 312247835Skib * @pool: to free the pages from 313247835Skib * @free_all: If set to true will free all pages in pool 314247835Skib **/ 315247835Skibstatic int ttm_page_pool_free(struct ttm_page_pool *pool, unsigned nr_free) 316247835Skib{ 317247835Skib vm_page_t p, p1; 318247835Skib vm_page_t *pages_to_free; 319247835Skib unsigned freed_pages = 0, 320247835Skib npages_to_free = nr_free; 321254873Sdumbbell unsigned i; 322247835Skib 323247835Skib if (NUM_PAGES_TO_ALLOC < nr_free) 324247835Skib npages_to_free = NUM_PAGES_TO_ALLOC; 325247835Skib 326247835Skib pages_to_free = malloc(npages_to_free * sizeof(vm_page_t), 327247835Skib M_TEMP, M_WAITOK | M_ZERO); 328247835Skib 329247835Skibrestart: 330247835Skib mtx_lock(&pool->lock); 331247835Skib 332254182Skib TAILQ_FOREACH_REVERSE_SAFE(p, &pool->list, pglist, plinks.q, p1) { 333247835Skib if (freed_pages >= npages_to_free) 334247835Skib break; 335247835Skib 336247835Skib pages_to_free[freed_pages++] = p; 337247835Skib /* We can only remove NUM_PAGES_TO_ALLOC at a time. */ 338247835Skib if (freed_pages >= NUM_PAGES_TO_ALLOC) { 339247835Skib /* remove range of pages from the pool */ 340254873Sdumbbell for (i = 0; i < freed_pages; i++) 341254873Sdumbbell TAILQ_REMOVE(&pool->list, pages_to_free[i], plinks.q); 342247835Skib 343247835Skib ttm_pool_update_free_locked(pool, freed_pages); 344247835Skib /** 345247835Skib * Because changing page caching is costly 346247835Skib * we unlock the pool to prevent stalling. 347247835Skib */ 348247835Skib mtx_unlock(&pool->lock); 349247835Skib 350247835Skib ttm_pages_put(pages_to_free, freed_pages); 351247835Skib if (likely(nr_free != FREE_ALL_PAGES)) 352247835Skib nr_free -= freed_pages; 353247835Skib 354247835Skib if (NUM_PAGES_TO_ALLOC >= nr_free) 355247835Skib npages_to_free = nr_free; 356247835Skib else 357247835Skib npages_to_free = NUM_PAGES_TO_ALLOC; 358247835Skib 359247835Skib freed_pages = 0; 360247835Skib 361247835Skib /* free all so restart the processing */ 362247835Skib if (nr_free) 363247835Skib goto restart; 364247835Skib 365247835Skib /* Not allowed to fall through or break because 366247835Skib * following context is inside spinlock while we are 367247835Skib * outside here. 368247835Skib */ 369247835Skib goto out; 370247835Skib 371247835Skib } 372247835Skib } 373247835Skib 374247835Skib /* remove range of pages from the pool */ 375247835Skib if (freed_pages) { 376254873Sdumbbell for (i = 0; i < freed_pages; i++) 377254873Sdumbbell TAILQ_REMOVE(&pool->list, pages_to_free[i], plinks.q); 378247835Skib 379247835Skib ttm_pool_update_free_locked(pool, freed_pages); 380247835Skib nr_free -= freed_pages; 381247835Skib } 382247835Skib 383247835Skib mtx_unlock(&pool->lock); 384247835Skib 385247835Skib if (freed_pages) 386247835Skib ttm_pages_put(pages_to_free, freed_pages); 387247835Skibout: 388247835Skib free(pages_to_free, M_TEMP); 389247835Skib return nr_free; 390247835Skib} 391247835Skib 392247835Skib/* Get good estimation how many pages are free in pools */ 393247835Skibstatic int ttm_pool_get_num_unused_pages(void) 394247835Skib{ 395247835Skib unsigned i; 396247835Skib int total = 0; 397247835Skib for (i = 0; i < NUM_POOLS; ++i) 398247835Skib total += _manager->pools[i].npages; 399247835Skib 400247835Skib return total; 401247835Skib} 402247835Skib 403247835Skib/** 404247835Skib * Callback for mm to request pool to reduce number of page held. 405247835Skib */ 406247835Skibstatic int ttm_pool_mm_shrink(void *arg) 407247835Skib{ 408247835Skib static unsigned int start_pool = 0; 409247835Skib unsigned i; 410247835Skib unsigned pool_offset = atomic_fetchadd_int(&start_pool, 1); 411247835Skib struct ttm_page_pool *pool; 412247835Skib int shrink_pages = 100; /* XXXKIB */ 413247835Skib 414247835Skib pool_offset = pool_offset % NUM_POOLS; 415247835Skib /* select start pool in round robin fashion */ 416247835Skib for (i = 0; i < NUM_POOLS; ++i) { 417247835Skib unsigned nr_free = shrink_pages; 418247835Skib if (shrink_pages == 0) 419247835Skib break; 420247835Skib pool = &_manager->pools[(i + pool_offset)%NUM_POOLS]; 421247835Skib shrink_pages = ttm_page_pool_free(pool, nr_free); 422247835Skib } 423247835Skib /* return estimated number of unused pages in pool */ 424247835Skib return ttm_pool_get_num_unused_pages(); 425247835Skib} 426247835Skib 427247835Skibstatic void ttm_pool_mm_shrink_init(struct ttm_pool_manager *manager) 428247835Skib{ 429247835Skib 430247835Skib manager->lowmem_handler = EVENTHANDLER_REGISTER(vm_lowmem, 431247835Skib ttm_pool_mm_shrink, manager, EVENTHANDLER_PRI_ANY); 432247835Skib} 433247835Skib 434247835Skibstatic void ttm_pool_mm_shrink_fini(struct ttm_pool_manager *manager) 435247835Skib{ 436247835Skib 437247835Skib EVENTHANDLER_DEREGISTER(vm_lowmem, manager->lowmem_handler); 438247835Skib} 439247835Skib 440247835Skibstatic int ttm_set_pages_caching(vm_page_t *pages, 441247835Skib enum ttm_caching_state cstate, unsigned cpages) 442247835Skib{ 443247835Skib int r = 0; 444247835Skib /* Set page caching */ 445247835Skib switch (cstate) { 446247835Skib case tt_uncached: 447247835Skib r = set_pages_array_uc(pages, cpages); 448247835Skib if (r) 449247835Skib printf("[TTM] Failed to set %d pages to uc!\n", cpages); 450247835Skib break; 451247835Skib case tt_wc: 452247835Skib r = set_pages_array_wc(pages, cpages); 453247835Skib if (r) 454247835Skib printf("[TTM] Failed to set %d pages to wc!\n", cpages); 455247835Skib break; 456247835Skib default: 457247835Skib break; 458247835Skib } 459247835Skib return r; 460247835Skib} 461247835Skib 462247835Skib/** 463247835Skib * Free pages the pages that failed to change the caching state. If there is 464247835Skib * any pages that have changed their caching state already put them to the 465247835Skib * pool. 466247835Skib */ 467247835Skibstatic void ttm_handle_caching_state_failure(struct pglist *pages, 468247835Skib int ttm_flags, enum ttm_caching_state cstate, 469247835Skib vm_page_t *failed_pages, unsigned cpages) 470247835Skib{ 471247835Skib unsigned i; 472247835Skib /* Failed pages have to be freed */ 473247835Skib for (i = 0; i < cpages; ++i) { 474254182Skib TAILQ_REMOVE(pages, failed_pages[i], plinks.q); 475247835Skib ttm_vm_page_free(failed_pages[i]); 476247835Skib } 477247835Skib} 478247835Skib 479247835Skib/** 480247835Skib * Allocate new pages with correct caching. 481247835Skib * 482247835Skib * This function is reentrant if caller updates count depending on number of 483247835Skib * pages returned in pages array. 484247835Skib */ 485247835Skibstatic int ttm_alloc_new_pages(struct pglist *pages, int ttm_alloc_flags, 486247835Skib int ttm_flags, enum ttm_caching_state cstate, unsigned count) 487247835Skib{ 488247835Skib vm_page_t *caching_array; 489247835Skib vm_page_t p; 490247835Skib int r = 0; 491247835Skib unsigned i, cpages, aflags; 492247835Skib unsigned max_cpages = min(count, 493247835Skib (unsigned)(PAGE_SIZE/sizeof(vm_page_t))); 494247835Skib 495247835Skib aflags = VM_ALLOC_NORMAL | VM_ALLOC_WIRED | VM_ALLOC_NOOBJ | 496247835Skib ((ttm_alloc_flags & TTM_PAGE_FLAG_ZERO_ALLOC) != 0 ? 497247835Skib VM_ALLOC_ZERO : 0); 498247835Skib 499247835Skib /* allocate array for page caching change */ 500247835Skib caching_array = malloc(max_cpages * sizeof(vm_page_t), M_TEMP, 501247835Skib M_WAITOK | M_ZERO); 502247835Skib 503247835Skib for (i = 0, cpages = 0; i < count; ++i) { 504247835Skib p = vm_page_alloc_contig(NULL, 0, aflags, 1, 0, 505247835Skib (ttm_alloc_flags & TTM_PAGE_FLAG_DMA32) ? 0xffffffff : 506247835Skib VM_MAX_ADDRESS, PAGE_SIZE, 0, 507247835Skib ttm_caching_state_to_vm(cstate)); 508247835Skib if (!p) { 509247835Skib printf("[TTM] Unable to get page %u\n", i); 510247835Skib 511247835Skib /* store already allocated pages in the pool after 512247835Skib * setting the caching state */ 513247835Skib if (cpages) { 514247835Skib r = ttm_set_pages_caching(caching_array, 515247835Skib cstate, cpages); 516247835Skib if (r) 517247835Skib ttm_handle_caching_state_failure(pages, 518247835Skib ttm_flags, cstate, 519247835Skib caching_array, cpages); 520247835Skib } 521247835Skib r = -ENOMEM; 522247835Skib goto out; 523247835Skib } 524247835Skib p->oflags &= ~VPO_UNMANAGED; 525247835Skib p->flags |= PG_FICTITIOUS; 526247835Skib 527247835Skib#ifdef CONFIG_HIGHMEM /* KIB: nop */ 528247835Skib /* gfp flags of highmem page should never be dma32 so we 529247835Skib * we should be fine in such case 530247835Skib */ 531247835Skib if (!PageHighMem(p)) 532247835Skib#endif 533247835Skib { 534247835Skib caching_array[cpages++] = p; 535247835Skib if (cpages == max_cpages) { 536247835Skib 537247835Skib r = ttm_set_pages_caching(caching_array, 538247835Skib cstate, cpages); 539247835Skib if (r) { 540247835Skib ttm_handle_caching_state_failure(pages, 541247835Skib ttm_flags, cstate, 542247835Skib caching_array, cpages); 543247835Skib goto out; 544247835Skib } 545247835Skib cpages = 0; 546247835Skib } 547247835Skib } 548247835Skib 549254182Skib TAILQ_INSERT_HEAD(pages, p, plinks.q); 550247835Skib } 551247835Skib 552247835Skib if (cpages) { 553247835Skib r = ttm_set_pages_caching(caching_array, cstate, cpages); 554247835Skib if (r) 555247835Skib ttm_handle_caching_state_failure(pages, 556247835Skib ttm_flags, cstate, 557247835Skib caching_array, cpages); 558247835Skib } 559247835Skibout: 560247835Skib free(caching_array, M_TEMP); 561247835Skib 562247835Skib return r; 563247835Skib} 564247835Skib 565247835Skib/** 566247835Skib * Fill the given pool if there aren't enough pages and the requested number of 567247835Skib * pages is small. 568247835Skib */ 569247835Skibstatic void ttm_page_pool_fill_locked(struct ttm_page_pool *pool, 570247835Skib int ttm_flags, enum ttm_caching_state cstate, unsigned count) 571247835Skib{ 572247835Skib vm_page_t p; 573247835Skib int r; 574247835Skib unsigned cpages = 0; 575247835Skib /** 576247835Skib * Only allow one pool fill operation at a time. 577247835Skib * If pool doesn't have enough pages for the allocation new pages are 578247835Skib * allocated from outside of pool. 579247835Skib */ 580247835Skib if (pool->fill_lock) 581247835Skib return; 582247835Skib 583247835Skib pool->fill_lock = true; 584247835Skib 585247835Skib /* If allocation request is small and there are not enough 586247835Skib * pages in a pool we fill the pool up first. */ 587247835Skib if (count < _manager->options.small 588247835Skib && count > pool->npages) { 589247835Skib struct pglist new_pages; 590247835Skib unsigned alloc_size = _manager->options.alloc_size; 591247835Skib 592247835Skib /** 593247835Skib * Can't change page caching if in irqsave context. We have to 594247835Skib * drop the pool->lock. 595247835Skib */ 596247835Skib mtx_unlock(&pool->lock); 597247835Skib 598247835Skib TAILQ_INIT(&new_pages); 599247835Skib r = ttm_alloc_new_pages(&new_pages, pool->ttm_page_alloc_flags, 600247835Skib ttm_flags, cstate, alloc_size); 601247835Skib mtx_lock(&pool->lock); 602247835Skib 603247835Skib if (!r) { 604254182Skib TAILQ_CONCAT(&pool->list, &new_pages, plinks.q); 605247835Skib ++pool->nrefills; 606247835Skib pool->npages += alloc_size; 607247835Skib } else { 608247835Skib printf("[TTM] Failed to fill pool (%p)\n", pool); 609247835Skib /* If we have any pages left put them to the pool. */ 610254182Skib TAILQ_FOREACH(p, &pool->list, plinks.q) { 611247835Skib ++cpages; 612247835Skib } 613254182Skib TAILQ_CONCAT(&pool->list, &new_pages, plinks.q); 614247835Skib pool->npages += cpages; 615247835Skib } 616247835Skib 617247835Skib } 618247835Skib pool->fill_lock = false; 619247835Skib} 620247835Skib 621247835Skib/** 622247835Skib * Cut 'count' number of pages from the pool and put them on the return list. 623247835Skib * 624247835Skib * @return count of pages still required to fulfill the request. 625247835Skib */ 626247835Skibstatic unsigned ttm_page_pool_get_pages(struct ttm_page_pool *pool, 627247835Skib struct pglist *pages, 628247835Skib int ttm_flags, 629247835Skib enum ttm_caching_state cstate, 630247835Skib unsigned count) 631247835Skib{ 632247835Skib vm_page_t p; 633247835Skib unsigned i; 634247835Skib 635247835Skib mtx_lock(&pool->lock); 636247835Skib ttm_page_pool_fill_locked(pool, ttm_flags, cstate, count); 637247835Skib 638247835Skib if (count >= pool->npages) { 639247835Skib /* take all pages from the pool */ 640254182Skib TAILQ_CONCAT(pages, &pool->list, plinks.q); 641247835Skib count -= pool->npages; 642247835Skib pool->npages = 0; 643247835Skib goto out; 644247835Skib } 645247835Skib for (i = 0; i < count; i++) { 646247835Skib p = TAILQ_FIRST(&pool->list); 647254182Skib TAILQ_REMOVE(&pool->list, p, plinks.q); 648254182Skib TAILQ_INSERT_TAIL(pages, p, plinks.q); 649247835Skib } 650247835Skib pool->npages -= count; 651247835Skib count = 0; 652247835Skibout: 653247835Skib mtx_unlock(&pool->lock); 654247835Skib return count; 655247835Skib} 656247835Skib 657247835Skib/* Put all pages in pages list to correct pool to wait for reuse */ 658247835Skibstatic void ttm_put_pages(vm_page_t *pages, unsigned npages, int flags, 659247835Skib enum ttm_caching_state cstate) 660247835Skib{ 661247835Skib struct ttm_page_pool *pool = ttm_get_pool(flags, cstate); 662247835Skib unsigned i; 663247835Skib 664247835Skib if (pool == NULL) { 665247835Skib /* No pool for this memory type so free the pages */ 666247835Skib for (i = 0; i < npages; i++) { 667247835Skib if (pages[i]) { 668247835Skib ttm_vm_page_free(pages[i]); 669247835Skib pages[i] = NULL; 670247835Skib } 671247835Skib } 672247835Skib return; 673247835Skib } 674247835Skib 675247835Skib mtx_lock(&pool->lock); 676247835Skib for (i = 0; i < npages; i++) { 677247835Skib if (pages[i]) { 678254182Skib TAILQ_INSERT_TAIL(&pool->list, pages[i], plinks.q); 679247835Skib pages[i] = NULL; 680247835Skib pool->npages++; 681247835Skib } 682247835Skib } 683247835Skib /* Check that we don't go over the pool limit */ 684247835Skib npages = 0; 685247835Skib if (pool->npages > _manager->options.max_size) { 686247835Skib npages = pool->npages - _manager->options.max_size; 687247835Skib /* free at least NUM_PAGES_TO_ALLOC number of pages 688247835Skib * to reduce calls to set_memory_wb */ 689247835Skib if (npages < NUM_PAGES_TO_ALLOC) 690247835Skib npages = NUM_PAGES_TO_ALLOC; 691247835Skib } 692247835Skib mtx_unlock(&pool->lock); 693247835Skib if (npages) 694247835Skib ttm_page_pool_free(pool, npages); 695247835Skib} 696247835Skib 697247835Skib/* 698247835Skib * On success pages list will hold count number of correctly 699247835Skib * cached pages. 700247835Skib */ 701247835Skibstatic int ttm_get_pages(vm_page_t *pages, unsigned npages, int flags, 702247835Skib enum ttm_caching_state cstate) 703247835Skib{ 704247835Skib struct ttm_page_pool *pool = ttm_get_pool(flags, cstate); 705247835Skib struct pglist plist; 706247835Skib vm_page_t p = NULL; 707247835Skib int gfp_flags, aflags; 708247835Skib unsigned count; 709247835Skib int r; 710247835Skib 711247835Skib aflags = VM_ALLOC_NORMAL | VM_ALLOC_NOOBJ | VM_ALLOC_WIRED | 712247835Skib ((flags & TTM_PAGE_FLAG_ZERO_ALLOC) != 0 ? VM_ALLOC_ZERO : 0); 713247835Skib 714247835Skib /* No pool for cached pages */ 715247835Skib if (pool == NULL) { 716247835Skib for (r = 0; r < npages; ++r) { 717247835Skib p = vm_page_alloc_contig(NULL, 0, aflags, 1, 0, 718247835Skib (flags & TTM_PAGE_FLAG_DMA32) ? 0xffffffff : 719247835Skib VM_MAX_ADDRESS, PAGE_SIZE, 720247835Skib 0, ttm_caching_state_to_vm(cstate)); 721247835Skib if (!p) { 722247835Skib printf("[TTM] Unable to allocate page\n"); 723247835Skib return -ENOMEM; 724247835Skib } 725247835Skib p->oflags &= ~VPO_UNMANAGED; 726247835Skib p->flags |= PG_FICTITIOUS; 727247835Skib pages[r] = p; 728247835Skib } 729247835Skib return 0; 730247835Skib } 731247835Skib 732247835Skib /* combine zero flag to pool flags */ 733247835Skib gfp_flags = flags | pool->ttm_page_alloc_flags; 734247835Skib 735247835Skib /* First we take pages from the pool */ 736247835Skib TAILQ_INIT(&plist); 737247835Skib npages = ttm_page_pool_get_pages(pool, &plist, flags, cstate, npages); 738247835Skib count = 0; 739254182Skib TAILQ_FOREACH(p, &plist, plinks.q) { 740247835Skib pages[count++] = p; 741247835Skib } 742247835Skib 743247835Skib /* clear the pages coming from the pool if requested */ 744247835Skib if (flags & TTM_PAGE_FLAG_ZERO_ALLOC) { 745254182Skib TAILQ_FOREACH(p, &plist, plinks.q) { 746247835Skib pmap_zero_page(p); 747247835Skib } 748247835Skib } 749247835Skib 750247835Skib /* If pool didn't have enough pages allocate new one. */ 751247835Skib if (npages > 0) { 752247835Skib /* ttm_alloc_new_pages doesn't reference pool so we can run 753247835Skib * multiple requests in parallel. 754247835Skib **/ 755247835Skib TAILQ_INIT(&plist); 756247835Skib r = ttm_alloc_new_pages(&plist, gfp_flags, flags, cstate, 757247835Skib npages); 758254182Skib TAILQ_FOREACH(p, &plist, plinks.q) { 759247835Skib pages[count++] = p; 760247835Skib } 761247835Skib if (r) { 762247835Skib /* If there is any pages in the list put them back to 763247835Skib * the pool. */ 764247835Skib printf("[TTM] Failed to allocate extra pages for large request\n"); 765247835Skib ttm_put_pages(pages, count, flags, cstate); 766247835Skib return r; 767247835Skib } 768247835Skib } 769247835Skib 770247835Skib return 0; 771247835Skib} 772247835Skib 773247835Skibstatic void ttm_page_pool_init_locked(struct ttm_page_pool *pool, int flags, 774247835Skib char *name) 775247835Skib{ 776247835Skib mtx_init(&pool->lock, "ttmpool", NULL, MTX_DEF); 777247835Skib pool->fill_lock = false; 778247835Skib TAILQ_INIT(&pool->list); 779247835Skib pool->npages = pool->nfrees = 0; 780247835Skib pool->ttm_page_alloc_flags = flags; 781247835Skib pool->name = name; 782247835Skib} 783247835Skib 784247835Skibint ttm_page_alloc_init(struct ttm_mem_global *glob, unsigned max_pages) 785247835Skib{ 786247835Skib 787247835Skib if (_manager != NULL) 788247835Skib printf("[TTM] manager != NULL\n"); 789247835Skib printf("[TTM] Initializing pool allocator\n"); 790247835Skib 791247835Skib _manager = malloc(sizeof(*_manager), M_TTM_POOLMGR, M_WAITOK | M_ZERO); 792247835Skib 793247835Skib ttm_page_pool_init_locked(&_manager->wc_pool, 0, "wc"); 794247835Skib ttm_page_pool_init_locked(&_manager->uc_pool, 0, "uc"); 795247835Skib ttm_page_pool_init_locked(&_manager->wc_pool_dma32, 796247835Skib TTM_PAGE_FLAG_DMA32, "wc dma"); 797247835Skib ttm_page_pool_init_locked(&_manager->uc_pool_dma32, 798247835Skib TTM_PAGE_FLAG_DMA32, "uc dma"); 799247835Skib 800247835Skib _manager->options.max_size = max_pages; 801247835Skib _manager->options.small = SMALL_ALLOCATION; 802247835Skib _manager->options.alloc_size = NUM_PAGES_TO_ALLOC; 803247835Skib 804247835Skib refcount_init(&_manager->kobj_ref, 1); 805247835Skib ttm_pool_mm_shrink_init(_manager); 806247835Skib 807247835Skib return 0; 808247835Skib} 809247835Skib 810247835Skibvoid ttm_page_alloc_fini(void) 811247835Skib{ 812247835Skib int i; 813247835Skib 814247835Skib printf("[TTM] Finalizing pool allocator\n"); 815247835Skib ttm_pool_mm_shrink_fini(_manager); 816247835Skib 817247835Skib for (i = 0; i < NUM_POOLS; ++i) 818247835Skib ttm_page_pool_free(&_manager->pools[i], FREE_ALL_PAGES); 819247835Skib 820247835Skib if (refcount_release(&_manager->kobj_ref)) 821247835Skib ttm_pool_kobj_release(_manager); 822247835Skib _manager = NULL; 823247835Skib} 824247835Skib 825247835Skibint ttm_pool_populate(struct ttm_tt *ttm) 826247835Skib{ 827247835Skib struct ttm_mem_global *mem_glob = ttm->glob->mem_glob; 828247835Skib unsigned i; 829247835Skib int ret; 830247835Skib 831247835Skib if (ttm->state != tt_unpopulated) 832247835Skib return 0; 833247835Skib 834247835Skib for (i = 0; i < ttm->num_pages; ++i) { 835247835Skib ret = ttm_get_pages(&ttm->pages[i], 1, 836247835Skib ttm->page_flags, 837247835Skib ttm->caching_state); 838247835Skib if (ret != 0) { 839247835Skib ttm_pool_unpopulate(ttm); 840247835Skib return -ENOMEM; 841247835Skib } 842247835Skib 843247835Skib ret = ttm_mem_global_alloc_page(mem_glob, ttm->pages[i], 844247835Skib false, false); 845247835Skib if (unlikely(ret != 0)) { 846247835Skib ttm_pool_unpopulate(ttm); 847247835Skib return -ENOMEM; 848247835Skib } 849247835Skib } 850247835Skib 851247835Skib if (unlikely(ttm->page_flags & TTM_PAGE_FLAG_SWAPPED)) { 852247835Skib ret = ttm_tt_swapin(ttm); 853247835Skib if (unlikely(ret != 0)) { 854247835Skib ttm_pool_unpopulate(ttm); 855247835Skib return ret; 856247835Skib } 857247835Skib } 858247835Skib 859247835Skib ttm->state = tt_unbound; 860247835Skib return 0; 861247835Skib} 862247835Skib 863247835Skibvoid ttm_pool_unpopulate(struct ttm_tt *ttm) 864247835Skib{ 865247835Skib unsigned i; 866247835Skib 867247835Skib for (i = 0; i < ttm->num_pages; ++i) { 868247835Skib if (ttm->pages[i]) { 869247835Skib ttm_mem_global_free_page(ttm->glob->mem_glob, 870247835Skib ttm->pages[i]); 871247835Skib ttm_put_pages(&ttm->pages[i], 1, 872247835Skib ttm->page_flags, 873247835Skib ttm->caching_state); 874247835Skib } 875247835Skib } 876247835Skib ttm->state = tt_unpopulated; 877247835Skib} 878247835Skib 879247835Skib#if 0 880247835Skib/* XXXKIB sysctl */ 881247835Skibint ttm_page_alloc_debugfs(struct seq_file *m, void *data) 882247835Skib{ 883247835Skib struct ttm_page_pool *p; 884247835Skib unsigned i; 885247835Skib char *h[] = {"pool", "refills", "pages freed", "size"}; 886247835Skib if (!_manager) { 887247835Skib seq_printf(m, "No pool allocator running.\n"); 888247835Skib return 0; 889247835Skib } 890247835Skib seq_printf(m, "%6s %12s %13s %8s\n", 891247835Skib h[0], h[1], h[2], h[3]); 892247835Skib for (i = 0; i < NUM_POOLS; ++i) { 893247835Skib p = &_manager->pools[i]; 894247835Skib 895247835Skib seq_printf(m, "%6s %12ld %13ld %8d\n", 896247835Skib p->name, p->nrefills, 897247835Skib p->nfrees, p->npages); 898247835Skib } 899247835Skib return 0; 900247835Skib} 901247835Skib#endif 902