1247835Skib/************************************************************************** 2247835Skib * 3247835Skib * Copyright (c) 2006-2009 VMware, Inc., Palo Alto, CA., USA 4247835Skib * All Rights Reserved. 5247835Skib * 6247835Skib * Permission is hereby granted, free of charge, to any person obtaining a 7247835Skib * copy of this software and associated documentation files (the 8247835Skib * "Software"), to deal in the Software without restriction, including 9247835Skib * without limitation the rights to use, copy, modify, merge, publish, 10247835Skib * distribute, sub license, and/or sell copies of the Software, and to 11247835Skib * permit persons to whom the Software is furnished to do so, subject to 12247835Skib * the following conditions: 13247835Skib * 14247835Skib * The above copyright notice and this permission notice (including the 15247835Skib * next paragraph) shall be included in all copies or substantial portions 16247835Skib * of the Software. 17247835Skib * 18247835Skib * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 19247835Skib * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 20247835Skib * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL 21247835Skib * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM, 22247835Skib * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR 23247835Skib * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE 24247835Skib * USE OR OTHER DEALINGS IN THE SOFTWARE. 25247835Skib * 26247835Skib **************************************************************************/ 27247835Skib 28247835Skib#include <sys/cdefs.h> 29247835Skib__FBSDID("$FreeBSD$"); 30247835Skib 31247835Skib#include <dev/drm2/drmP.h> 32247835Skib#include <dev/drm2/ttm/ttm_memory.h> 33247835Skib#include <dev/drm2/ttm/ttm_module.h> 34247835Skib#include <dev/drm2/ttm/ttm_page_alloc.h> 35247835Skib 36247835Skib#define TTM_MEMORY_ALLOC_RETRIES 4 37247835Skib 38247835Skibstruct ttm_mem_zone { 39247835Skib u_int kobj_ref; 40247835Skib struct ttm_mem_global *glob; 41247835Skib const char *name; 42247835Skib uint64_t zone_mem; 43247835Skib uint64_t emer_mem; 44247835Skib uint64_t max_mem; 45247835Skib uint64_t swap_limit; 46247835Skib uint64_t used_mem; 47247835Skib}; 48247835Skib 49247835SkibMALLOC_DEFINE(M_TTM_ZONE, "ttm_zone", "TTM Zone"); 50247835Skib 51247835Skibstatic void ttm_mem_zone_kobj_release(struct ttm_mem_zone *zone) 52247835Skib{ 53247835Skib 54248663Sdumbbell printf("[TTM] Zone %7s: Used memory at exit: %llu kiB\n", 55247835Skib zone->name, (unsigned long long)zone->used_mem >> 10); 56247835Skib free(zone, M_TTM_ZONE); 57247835Skib} 58247835Skib 59247835Skib#if 0 60247835Skib/* XXXKIB sysctl */ 61247835Skibstatic ssize_t ttm_mem_zone_show(struct ttm_mem_zone *zone; 62247835Skib struct attribute *attr, 63247835Skib char *buffer) 64247835Skib{ 65247835Skib uint64_t val = 0; 66247835Skib 67247835Skib mtx_lock(&zone->glob->lock); 68247835Skib if (attr == &ttm_mem_sys) 69247835Skib val = zone->zone_mem; 70247835Skib else if (attr == &ttm_mem_emer) 71247835Skib val = zone->emer_mem; 72247835Skib else if (attr == &ttm_mem_max) 73247835Skib val = zone->max_mem; 74247835Skib else if (attr == &ttm_mem_swap) 75247835Skib val = zone->swap_limit; 76247835Skib else if (attr == &ttm_mem_used) 77247835Skib val = zone->used_mem; 78247835Skib mtx_unlock(&zone->glob->lock); 79247835Skib 80247835Skib return snprintf(buffer, PAGE_SIZE, "%llu\n", 81247835Skib (unsigned long long) val >> 10); 82247835Skib} 83247835Skib#endif 84247835Skib 85247835Skibstatic void ttm_check_swapping(struct ttm_mem_global *glob); 86247835Skib 87247835Skib#if 0 88247835Skib/* XXXKIB sysctl */ 89247835Skibstatic ssize_t ttm_mem_zone_store(struct ttm_mem_zone *zone, 90247835Skib struct attribute *attr, 91247835Skib const char *buffer, 92247835Skib size_t size) 93247835Skib{ 94247835Skib int chars; 95247835Skib unsigned long val; 96247835Skib uint64_t val64; 97247835Skib 98247835Skib chars = sscanf(buffer, "%lu", &val); 99247835Skib if (chars == 0) 100247835Skib return size; 101247835Skib 102247835Skib val64 = val; 103247835Skib val64 <<= 10; 104247835Skib 105247835Skib mtx_lock(&zone->glob->lock); 106247835Skib if (val64 > zone->zone_mem) 107247835Skib val64 = zone->zone_mem; 108247835Skib if (attr == &ttm_mem_emer) { 109247835Skib zone->emer_mem = val64; 110247835Skib if (zone->max_mem > val64) 111247835Skib zone->max_mem = val64; 112247835Skib } else if (attr == &ttm_mem_max) { 113247835Skib zone->max_mem = val64; 114247835Skib if (zone->emer_mem < val64) 115247835Skib zone->emer_mem = val64; 116247835Skib } else if (attr == &ttm_mem_swap) 117247835Skib zone->swap_limit = val64; 118247835Skib mtx_unlock(&zone->glob->lock); 119247835Skib 120247835Skib ttm_check_swapping(zone->glob); 121247835Skib 122247835Skib return size; 123247835Skib} 124247835Skib#endif 125247835Skib 126247835Skibstatic void ttm_mem_global_kobj_release(struct ttm_mem_global *glob) 127247835Skib{ 128247835Skib} 129247835Skib 130247835Skibstatic bool ttm_zones_above_swap_target(struct ttm_mem_global *glob, 131247835Skib bool from_wq, uint64_t extra) 132247835Skib{ 133247835Skib unsigned int i; 134247835Skib struct ttm_mem_zone *zone; 135247835Skib uint64_t target; 136247835Skib 137247835Skib for (i = 0; i < glob->num_zones; ++i) { 138247835Skib zone = glob->zones[i]; 139247835Skib 140247835Skib if (from_wq) 141247835Skib target = zone->swap_limit; 142247835Skib else if (priv_check(curthread, PRIV_VM_MLOCK) == 0) 143247835Skib target = zone->emer_mem; 144247835Skib else 145247835Skib target = zone->max_mem; 146247835Skib 147247835Skib target = (extra > target) ? 0ULL : target; 148247835Skib 149247835Skib if (zone->used_mem > target) 150247835Skib return true; 151247835Skib } 152247835Skib return false; 153247835Skib} 154247835Skib 155247835Skib/** 156247835Skib * At this point we only support a single shrink callback. 157247835Skib * Extend this if needed, perhaps using a linked list of callbacks. 158247835Skib * Note that this function is reentrant: 159247835Skib * many threads may try to swap out at any given time. 160247835Skib */ 161247835Skib 162247835Skibstatic void ttm_shrink(struct ttm_mem_global *glob, bool from_wq, 163247835Skib uint64_t extra) 164247835Skib{ 165247835Skib int ret; 166247835Skib struct ttm_mem_shrink *shrink; 167247835Skib 168247835Skib mtx_lock(&glob->lock); 169247835Skib if (glob->shrink == NULL) 170247835Skib goto out; 171247835Skib 172247835Skib while (ttm_zones_above_swap_target(glob, from_wq, extra)) { 173247835Skib shrink = glob->shrink; 174247835Skib mtx_unlock(&glob->lock); 175247835Skib ret = shrink->do_shrink(shrink); 176247835Skib mtx_lock(&glob->lock); 177247835Skib if (unlikely(ret != 0)) 178247835Skib goto out; 179247835Skib } 180247835Skibout: 181247835Skib mtx_unlock(&glob->lock); 182247835Skib} 183247835Skib 184247835Skib 185247835Skib 186247835Skibstatic void ttm_shrink_work(void *arg, int pending __unused) 187247835Skib{ 188247835Skib struct ttm_mem_global *glob = arg; 189247835Skib 190247835Skib ttm_shrink(glob, true, 0ULL); 191247835Skib} 192247835Skib 193247835Skibstatic int ttm_mem_init_kernel_zone(struct ttm_mem_global *glob, 194247835Skib uint64_t mem) 195247835Skib{ 196247835Skib struct ttm_mem_zone *zone; 197247835Skib 198247835Skib zone = malloc(sizeof(*zone), M_TTM_ZONE, M_WAITOK | M_ZERO); 199247835Skib 200247835Skib zone->name = "kernel"; 201247835Skib zone->zone_mem = mem; 202247835Skib zone->max_mem = mem >> 1; 203247835Skib zone->emer_mem = (mem >> 1) + (mem >> 2); 204247835Skib zone->swap_limit = zone->max_mem - (mem >> 3); 205247835Skib zone->used_mem = 0; 206247835Skib zone->glob = glob; 207247835Skib glob->zone_kernel = zone; 208247835Skib refcount_init(&zone->kobj_ref, 1); 209247835Skib glob->zones[glob->num_zones++] = zone; 210247835Skib return 0; 211247835Skib} 212247835Skib 213247835Skibstatic int ttm_mem_init_dma32_zone(struct ttm_mem_global *glob, 214247835Skib uint64_t mem) 215247835Skib{ 216247835Skib struct ttm_mem_zone *zone; 217247835Skib 218247835Skib zone = malloc(sizeof(*zone), M_TTM_ZONE, M_WAITOK | M_ZERO); 219247835Skib 220247835Skib /** 221247835Skib * No special dma32 zone needed. 222247835Skib */ 223247835Skib 224247835Skib if (mem <= ((uint64_t) 1ULL << 32)) { 225247835Skib free(zone, M_TTM_ZONE); 226247835Skib return 0; 227247835Skib } 228247835Skib 229247835Skib /* 230247835Skib * Limit max dma32 memory to 4GB for now 231247835Skib * until we can figure out how big this 232247835Skib * zone really is. 233247835Skib */ 234247835Skib 235247835Skib mem = ((uint64_t) 1ULL << 32); 236247835Skib zone->name = "dma32"; 237247835Skib zone->zone_mem = mem; 238247835Skib zone->max_mem = mem >> 1; 239247835Skib zone->emer_mem = (mem >> 1) + (mem >> 2); 240247835Skib zone->swap_limit = zone->max_mem - (mem >> 3); 241247835Skib zone->used_mem = 0; 242247835Skib zone->glob = glob; 243247835Skib glob->zone_dma32 = zone; 244247835Skib refcount_init(&zone->kobj_ref, 1); 245247835Skib glob->zones[glob->num_zones++] = zone; 246247835Skib return 0; 247247835Skib} 248247835Skib 249247835Skibint ttm_mem_global_init(struct ttm_mem_global *glob) 250247835Skib{ 251247835Skib u_int64_t mem; 252247835Skib int ret; 253247835Skib int i; 254247835Skib struct ttm_mem_zone *zone; 255247835Skib 256247835Skib mtx_init(&glob->lock, "ttmgz", NULL, MTX_DEF); 257247835Skib glob->swap_queue = taskqueue_create("ttm_swap", M_WAITOK, 258247835Skib taskqueue_thread_enqueue, &glob->swap_queue); 259247835Skib taskqueue_start_threads(&glob->swap_queue, 1, PVM, "ttm swap"); 260247835Skib TASK_INIT(&glob->work, 0, ttm_shrink_work, glob); 261247835Skib 262247835Skib refcount_init(&glob->kobj_ref, 1); 263247835Skib 264247835Skib mem = physmem * PAGE_SIZE; 265247835Skib 266247835Skib ret = ttm_mem_init_kernel_zone(glob, mem); 267247835Skib if (unlikely(ret != 0)) 268247835Skib goto out_no_zone; 269247835Skib ret = ttm_mem_init_dma32_zone(glob, mem); 270247835Skib if (unlikely(ret != 0)) 271247835Skib goto out_no_zone; 272247835Skib for (i = 0; i < glob->num_zones; ++i) { 273247835Skib zone = glob->zones[i]; 274247835Skib printf("[TTM] Zone %7s: Available graphics memory: %llu kiB\n", 275247835Skib zone->name, (unsigned long long)zone->max_mem >> 10); 276247835Skib } 277247835Skib ttm_page_alloc_init(glob, glob->zone_kernel->max_mem/(2*PAGE_SIZE)); 278247835Skib ttm_dma_page_alloc_init(glob, glob->zone_kernel->max_mem/(2*PAGE_SIZE)); 279247835Skib return 0; 280247835Skibout_no_zone: 281247835Skib ttm_mem_global_release(glob); 282247835Skib return ret; 283247835Skib} 284247835Skib 285247835Skibvoid ttm_mem_global_release(struct ttm_mem_global *glob) 286247835Skib{ 287247835Skib unsigned int i; 288247835Skib struct ttm_mem_zone *zone; 289247835Skib 290247835Skib /* let the page allocator first stop the shrink work. */ 291247835Skib ttm_page_alloc_fini(); 292247835Skib ttm_dma_page_alloc_fini(); 293247835Skib 294247835Skib taskqueue_drain(glob->swap_queue, &glob->work); 295247835Skib taskqueue_free(glob->swap_queue); 296247835Skib glob->swap_queue = NULL; 297247835Skib for (i = 0; i < glob->num_zones; ++i) { 298247835Skib zone = glob->zones[i]; 299247835Skib if (refcount_release(&zone->kobj_ref)) 300247835Skib ttm_mem_zone_kobj_release(zone); 301247835Skib } 302247835Skib if (refcount_release(&glob->kobj_ref)) 303247835Skib ttm_mem_global_kobj_release(glob); 304247835Skib} 305247835Skib 306247835Skibstatic void ttm_check_swapping(struct ttm_mem_global *glob) 307247835Skib{ 308247835Skib bool needs_swapping = false; 309247835Skib unsigned int i; 310247835Skib struct ttm_mem_zone *zone; 311247835Skib 312247835Skib mtx_lock(&glob->lock); 313247835Skib for (i = 0; i < glob->num_zones; ++i) { 314247835Skib zone = glob->zones[i]; 315247835Skib if (zone->used_mem > zone->swap_limit) { 316247835Skib needs_swapping = true; 317247835Skib break; 318247835Skib } 319247835Skib } 320247835Skib 321247835Skib mtx_unlock(&glob->lock); 322247835Skib 323247835Skib if (unlikely(needs_swapping)) 324247835Skib taskqueue_enqueue(glob->swap_queue, &glob->work); 325247835Skib 326247835Skib} 327247835Skib 328247835Skibstatic void ttm_mem_global_free_zone(struct ttm_mem_global *glob, 329247835Skib struct ttm_mem_zone *single_zone, 330247835Skib uint64_t amount) 331247835Skib{ 332247835Skib unsigned int i; 333247835Skib struct ttm_mem_zone *zone; 334247835Skib 335247835Skib mtx_lock(&glob->lock); 336247835Skib for (i = 0; i < glob->num_zones; ++i) { 337247835Skib zone = glob->zones[i]; 338247835Skib if (single_zone && zone != single_zone) 339247835Skib continue; 340247835Skib zone->used_mem -= amount; 341247835Skib } 342247835Skib mtx_unlock(&glob->lock); 343247835Skib} 344247835Skib 345247835Skibvoid ttm_mem_global_free(struct ttm_mem_global *glob, 346247835Skib uint64_t amount) 347247835Skib{ 348247835Skib return ttm_mem_global_free_zone(glob, NULL, amount); 349247835Skib} 350247835Skib 351247835Skibstatic int ttm_mem_global_reserve(struct ttm_mem_global *glob, 352247835Skib struct ttm_mem_zone *single_zone, 353247835Skib uint64_t amount, bool reserve) 354247835Skib{ 355247835Skib uint64_t limit; 356247835Skib int ret = -ENOMEM; 357247835Skib unsigned int i; 358247835Skib struct ttm_mem_zone *zone; 359247835Skib 360247835Skib mtx_lock(&glob->lock); 361247835Skib for (i = 0; i < glob->num_zones; ++i) { 362247835Skib zone = glob->zones[i]; 363247835Skib if (single_zone && zone != single_zone) 364247835Skib continue; 365247835Skib 366247835Skib limit = (priv_check(curthread, PRIV_VM_MLOCK) == 0) ? 367247835Skib zone->emer_mem : zone->max_mem; 368247835Skib 369247835Skib if (zone->used_mem > limit) 370247835Skib goto out_unlock; 371247835Skib } 372247835Skib 373247835Skib if (reserve) { 374247835Skib for (i = 0; i < glob->num_zones; ++i) { 375247835Skib zone = glob->zones[i]; 376247835Skib if (single_zone && zone != single_zone) 377247835Skib continue; 378247835Skib zone->used_mem += amount; 379247835Skib } 380247835Skib } 381247835Skib 382247835Skib ret = 0; 383247835Skibout_unlock: 384247835Skib mtx_unlock(&glob->lock); 385247835Skib ttm_check_swapping(glob); 386247835Skib 387247835Skib return ret; 388247835Skib} 389247835Skib 390247835Skib 391247835Skibstatic int ttm_mem_global_alloc_zone(struct ttm_mem_global *glob, 392247835Skib struct ttm_mem_zone *single_zone, 393247835Skib uint64_t memory, 394247835Skib bool no_wait, bool interruptible) 395247835Skib{ 396247835Skib int count = TTM_MEMORY_ALLOC_RETRIES; 397247835Skib 398247835Skib while (unlikely(ttm_mem_global_reserve(glob, 399247835Skib single_zone, 400247835Skib memory, true) 401247835Skib != 0)) { 402247835Skib if (no_wait) 403247835Skib return -ENOMEM; 404247835Skib if (unlikely(count-- == 0)) 405247835Skib return -ENOMEM; 406247835Skib ttm_shrink(glob, false, memory + (memory >> 2) + 16); 407247835Skib } 408247835Skib 409247835Skib return 0; 410247835Skib} 411247835Skib 412247835Skibint ttm_mem_global_alloc(struct ttm_mem_global *glob, uint64_t memory, 413247835Skib bool no_wait, bool interruptible) 414247835Skib{ 415247835Skib /** 416247835Skib * Normal allocations of kernel memory are registered in 417247835Skib * all zones. 418247835Skib */ 419247835Skib 420247835Skib return ttm_mem_global_alloc_zone(glob, NULL, memory, no_wait, 421247835Skib interruptible); 422247835Skib} 423247835Skib 424247835Skib#define page_to_pfn(pp) OFF_TO_IDX(VM_PAGE_TO_PHYS(pp)) 425247835Skib 426247835Skibint ttm_mem_global_alloc_page(struct ttm_mem_global *glob, 427247835Skib struct vm_page *page, 428247835Skib bool no_wait, bool interruptible) 429247835Skib{ 430247835Skib 431247835Skib struct ttm_mem_zone *zone = NULL; 432247835Skib 433247835Skib /** 434247835Skib * Page allocations may be registed in a single zone 435247835Skib * only if highmem or !dma32. 436247835Skib */ 437247835Skib 438247835Skib if (glob->zone_dma32 && page_to_pfn(page) > 0x00100000UL) 439247835Skib zone = glob->zone_kernel; 440247835Skib return ttm_mem_global_alloc_zone(glob, zone, PAGE_SIZE, no_wait, 441247835Skib interruptible); 442247835Skib} 443247835Skib 444247835Skibvoid ttm_mem_global_free_page(struct ttm_mem_global *glob, struct vm_page *page) 445247835Skib{ 446247835Skib struct ttm_mem_zone *zone = NULL; 447247835Skib 448247835Skib if (glob->zone_dma32 && page_to_pfn(page) > 0x00100000UL) 449247835Skib zone = glob->zone_kernel; 450247835Skib ttm_mem_global_free_zone(glob, zone, PAGE_SIZE); 451247835Skib} 452247835Skib 453247835Skib 454247835Skibsize_t ttm_round_pot(size_t size) 455247835Skib{ 456247835Skib if ((size & (size - 1)) == 0) 457247835Skib return size; 458247835Skib else if (size > PAGE_SIZE) 459247835Skib return PAGE_ALIGN(size); 460247835Skib else { 461247835Skib size_t tmp_size = 4; 462247835Skib 463247835Skib while (tmp_size < size) 464247835Skib tmp_size <<= 1; 465247835Skib 466247835Skib return tmp_size; 467247835Skib } 468247835Skib return 0; 469247835Skib} 470