1247835Skib/************************************************************************** 2247835Skib * 3247835Skib * Copyright (c) 2006-2009 VMware, Inc., Palo Alto, CA., USA 4247835Skib * All Rights Reserved. 5247835Skib * 6247835Skib * Permission is hereby granted, free of charge, to any person obtaining a 7247835Skib * copy of this software and associated documentation files (the 8247835Skib * "Software"), to deal in the Software without restriction, including 9247835Skib * without limitation the rights to use, copy, modify, merge, publish, 10247835Skib * distribute, sub license, and/or sell copies of the Software, and to 11247835Skib * permit persons to whom the Software is furnished to do so, subject to 12247835Skib * the following conditions: 13247835Skib * 14247835Skib * The above copyright notice and this permission notice (including the 15247835Skib * next paragraph) shall be included in all copies or substantial portions 16247835Skib * of the Software. 17247835Skib * 18247835Skib * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 19247835Skib * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 20247835Skib * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL 21247835Skib * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM, 22247835Skib * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR 23247835Skib * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE 24247835Skib * USE OR OTHER DEALINGS IN THE SOFTWARE. 25247835Skib * 26247835Skib **************************************************************************/ 27247835Skib/* 28247835Skib * Authors: Thomas Hellstrom <thellstrom-at-vmware-dot-com> 29247835Skib */ 30247835Skib/* 31247835Skib * Copyright (c) 2013 The FreeBSD Foundation 32247835Skib * All rights reserved. 33247835Skib * 34247835Skib * Portions of this software were developed by Konstantin Belousov 35247835Skib * <kib@FreeBSD.org> under sponsorship from the FreeBSD Foundation. 36247835Skib */ 37247835Skib 38247835Skib#include <sys/cdefs.h> 39247835Skib__FBSDID("$FreeBSD$"); 40247835Skib 41247835Skib#include <dev/drm2/drmP.h> 42247835Skib#include <dev/drm2/ttm/ttm_module.h> 43247835Skib#include <dev/drm2/ttm/ttm_bo_driver.h> 44247835Skib#include <dev/drm2/ttm/ttm_placement.h> 45247835Skib#include <dev/drm2/ttm/ttm_page_alloc.h> 46247835Skib 47247835SkibMALLOC_DEFINE(M_TTM_PD, "ttm_pd", "TTM Page Directories"); 48247835Skib 49247835Skib/** 50247835Skib * Allocates storage for pointers to the pages that back the ttm. 51247835Skib */ 52247835Skibstatic void ttm_tt_alloc_page_directory(struct ttm_tt *ttm) 53247835Skib{ 54247835Skib ttm->pages = malloc(ttm->num_pages * sizeof(void *), 55247835Skib M_TTM_PD, M_WAITOK | M_ZERO); 56247835Skib} 57247835Skib 58247835Skibstatic void ttm_dma_tt_alloc_page_directory(struct ttm_dma_tt *ttm) 59247835Skib{ 60247835Skib ttm->ttm.pages = malloc(ttm->ttm.num_pages * sizeof(void *), 61247835Skib M_TTM_PD, M_WAITOK | M_ZERO); 62247835Skib ttm->dma_address = malloc(ttm->ttm.num_pages * 63247835Skib sizeof(*ttm->dma_address), M_TTM_PD, M_WAITOK); 64247835Skib} 65247835Skib 66247835Skib#if defined(__i386__) || defined(__amd64__) 67247835Skibstatic inline int ttm_tt_set_page_caching(vm_page_t p, 68247835Skib enum ttm_caching_state c_old, 69247835Skib enum ttm_caching_state c_new) 70247835Skib{ 71247835Skib 72247835Skib /* XXXKIB our VM does not need this. */ 73247835Skib#if 0 74247835Skib if (c_old != tt_cached) { 75247835Skib /* p isn't in the default caching state, set it to 76247835Skib * writeback first to free its current memtype. */ 77247835Skib pmap_page_set_memattr(p, VM_MEMATTR_WRITE_BACK); 78247835Skib } 79247835Skib#endif 80247835Skib 81247835Skib if (c_new == tt_wc) 82247835Skib pmap_page_set_memattr(p, VM_MEMATTR_WRITE_COMBINING); 83247835Skib else if (c_new == tt_uncached) 84247835Skib pmap_page_set_memattr(p, VM_MEMATTR_UNCACHEABLE); 85247835Skib 86247835Skib return (0); 87247835Skib} 88247835Skib#else 89247835Skibstatic inline int ttm_tt_set_page_caching(vm_page_t p, 90247835Skib enum ttm_caching_state c_old, 91247835Skib enum ttm_caching_state c_new) 92247835Skib{ 93247835Skib return 0; 94247835Skib} 95247835Skib#endif 96247835Skib 97247835Skib/* 98247835Skib * Change caching policy for the linear kernel map 99247835Skib * for range of pages in a ttm. 100247835Skib */ 101247835Skib 102247835Skibstatic int ttm_tt_set_caching(struct ttm_tt *ttm, 103247835Skib enum ttm_caching_state c_state) 104247835Skib{ 105247835Skib int i, j; 106247835Skib vm_page_t cur_page; 107247835Skib int ret; 108247835Skib 109247835Skib if (ttm->caching_state == c_state) 110247835Skib return 0; 111247835Skib 112247835Skib if (ttm->state == tt_unpopulated) { 113247835Skib /* Change caching but don't populate */ 114247835Skib ttm->caching_state = c_state; 115247835Skib return 0; 116247835Skib } 117247835Skib 118247835Skib if (ttm->caching_state == tt_cached) 119247835Skib drm_clflush_pages(ttm->pages, ttm->num_pages); 120247835Skib 121247835Skib for (i = 0; i < ttm->num_pages; ++i) { 122247835Skib cur_page = ttm->pages[i]; 123247835Skib if (likely(cur_page != NULL)) { 124247835Skib ret = ttm_tt_set_page_caching(cur_page, 125247835Skib ttm->caching_state, 126247835Skib c_state); 127247835Skib if (unlikely(ret != 0)) 128247835Skib goto out_err; 129247835Skib } 130247835Skib } 131247835Skib 132247835Skib ttm->caching_state = c_state; 133247835Skib 134247835Skib return 0; 135247835Skib 136247835Skibout_err: 137247835Skib for (j = 0; j < i; ++j) { 138247835Skib cur_page = ttm->pages[j]; 139247835Skib if (cur_page != NULL) { 140247835Skib (void)ttm_tt_set_page_caching(cur_page, c_state, 141247835Skib ttm->caching_state); 142247835Skib } 143247835Skib } 144247835Skib 145247835Skib return ret; 146247835Skib} 147247835Skib 148247835Skibint ttm_tt_set_placement_caching(struct ttm_tt *ttm, uint32_t placement) 149247835Skib{ 150247835Skib enum ttm_caching_state state; 151247835Skib 152247835Skib if (placement & TTM_PL_FLAG_WC) 153247835Skib state = tt_wc; 154247835Skib else if (placement & TTM_PL_FLAG_UNCACHED) 155247835Skib state = tt_uncached; 156247835Skib else 157247835Skib state = tt_cached; 158247835Skib 159247835Skib return ttm_tt_set_caching(ttm, state); 160247835Skib} 161247835Skib 162247835Skibvoid ttm_tt_destroy(struct ttm_tt *ttm) 163247835Skib{ 164247835Skib if (unlikely(ttm == NULL)) 165247835Skib return; 166247835Skib 167247835Skib if (ttm->state == tt_bound) { 168247835Skib ttm_tt_unbind(ttm); 169247835Skib } 170247835Skib 171247835Skib if (likely(ttm->pages != NULL)) { 172247835Skib ttm->bdev->driver->ttm_tt_unpopulate(ttm); 173247835Skib } 174247835Skib 175247835Skib if (!(ttm->page_flags & TTM_PAGE_FLAG_PERSISTENT_SWAP) && 176247835Skib ttm->swap_storage) 177247835Skib vm_object_deallocate(ttm->swap_storage); 178247835Skib 179247835Skib ttm->swap_storage = NULL; 180247835Skib ttm->func->destroy(ttm); 181247835Skib} 182247835Skib 183247835Skibint ttm_tt_init(struct ttm_tt *ttm, struct ttm_bo_device *bdev, 184247835Skib unsigned long size, uint32_t page_flags, 185247835Skib vm_page_t dummy_read_page) 186247835Skib{ 187247835Skib ttm->bdev = bdev; 188247835Skib ttm->glob = bdev->glob; 189247835Skib ttm->num_pages = (size + PAGE_SIZE - 1) >> PAGE_SHIFT; 190247835Skib ttm->caching_state = tt_cached; 191247835Skib ttm->page_flags = page_flags; 192247835Skib ttm->dummy_read_page = dummy_read_page; 193247835Skib ttm->state = tt_unpopulated; 194247835Skib ttm->swap_storage = NULL; 195247835Skib 196247835Skib ttm_tt_alloc_page_directory(ttm); 197247835Skib if (!ttm->pages) { 198247835Skib ttm_tt_destroy(ttm); 199247835Skib printf("Failed allocating page table\n"); 200247835Skib return -ENOMEM; 201247835Skib } 202247835Skib return 0; 203247835Skib} 204247835Skib 205247835Skibvoid ttm_tt_fini(struct ttm_tt *ttm) 206247835Skib{ 207247835Skib free(ttm->pages, M_TTM_PD); 208247835Skib ttm->pages = NULL; 209247835Skib} 210247835Skib 211247835Skibint ttm_dma_tt_init(struct ttm_dma_tt *ttm_dma, struct ttm_bo_device *bdev, 212247835Skib unsigned long size, uint32_t page_flags, 213247835Skib vm_page_t dummy_read_page) 214247835Skib{ 215247835Skib struct ttm_tt *ttm = &ttm_dma->ttm; 216247835Skib 217247835Skib ttm->bdev = bdev; 218247835Skib ttm->glob = bdev->glob; 219247835Skib ttm->num_pages = (size + PAGE_SIZE - 1) >> PAGE_SHIFT; 220247835Skib ttm->caching_state = tt_cached; 221247835Skib ttm->page_flags = page_flags; 222247835Skib ttm->dummy_read_page = dummy_read_page; 223247835Skib ttm->state = tt_unpopulated; 224247835Skib ttm->swap_storage = NULL; 225247835Skib 226247835Skib INIT_LIST_HEAD(&ttm_dma->pages_list); 227247835Skib ttm_dma_tt_alloc_page_directory(ttm_dma); 228247835Skib if (!ttm->pages || !ttm_dma->dma_address) { 229247835Skib ttm_tt_destroy(ttm); 230247835Skib printf("Failed allocating page table\n"); 231247835Skib return -ENOMEM; 232247835Skib } 233247835Skib return 0; 234247835Skib} 235247835Skib 236247835Skibvoid ttm_dma_tt_fini(struct ttm_dma_tt *ttm_dma) 237247835Skib{ 238247835Skib struct ttm_tt *ttm = &ttm_dma->ttm; 239247835Skib 240247835Skib free(ttm->pages, M_TTM_PD); 241247835Skib ttm->pages = NULL; 242247835Skib free(ttm_dma->dma_address, M_TTM_PD); 243247835Skib ttm_dma->dma_address = NULL; 244247835Skib} 245247835Skib 246247835Skibvoid ttm_tt_unbind(struct ttm_tt *ttm) 247247835Skib{ 248247835Skib int ret; 249247835Skib 250247835Skib if (ttm->state == tt_bound) { 251247835Skib ret = ttm->func->unbind(ttm); 252247835Skib MPASS(ret == 0); 253247835Skib ttm->state = tt_unbound; 254247835Skib } 255247835Skib} 256247835Skib 257247835Skibint ttm_tt_bind(struct ttm_tt *ttm, struct ttm_mem_reg *bo_mem) 258247835Skib{ 259247835Skib int ret = 0; 260247835Skib 261247835Skib if (!ttm) 262247835Skib return -EINVAL; 263247835Skib 264247835Skib if (ttm->state == tt_bound) 265247835Skib return 0; 266247835Skib 267247835Skib ret = ttm->bdev->driver->ttm_tt_populate(ttm); 268247835Skib if (ret) 269247835Skib return ret; 270247835Skib 271247835Skib ret = ttm->func->bind(ttm, bo_mem); 272247835Skib if (unlikely(ret != 0)) 273247835Skib return ret; 274247835Skib 275247835Skib ttm->state = tt_bound; 276247835Skib 277247835Skib return 0; 278247835Skib} 279247835Skib 280247835Skibint ttm_tt_swapin(struct ttm_tt *ttm) 281247835Skib{ 282247835Skib vm_object_t obj; 283247835Skib vm_page_t from_page, to_page; 284247835Skib int i, ret, rv; 285247835Skib 286247835Skib obj = ttm->swap_storage; 287247835Skib 288248084Sattilio VM_OBJECT_WLOCK(obj); 289247835Skib vm_object_pip_add(obj, 1); 290247835Skib for (i = 0; i < ttm->num_pages; ++i) { 291254649Skib from_page = vm_page_grab(obj, i, VM_ALLOC_NORMAL); 292247835Skib if (from_page->valid != VM_PAGE_BITS_ALL) { 293247835Skib if (vm_pager_has_page(obj, i, NULL, NULL)) { 294292373Sglebius rv = vm_pager_get_pages(obj, &from_page, 1, 295292373Sglebius NULL, NULL); 296247835Skib if (rv != VM_PAGER_OK) { 297247835Skib vm_page_lock(from_page); 298247835Skib vm_page_free(from_page); 299247835Skib vm_page_unlock(from_page); 300247835Skib ret = -EIO; 301247835Skib goto err_ret; 302247835Skib } 303247835Skib } else 304247835Skib vm_page_zero_invalid(from_page, TRUE); 305247835Skib } 306254138Sattilio vm_page_xunbusy(from_page); 307247835Skib to_page = ttm->pages[i]; 308247835Skib if (unlikely(to_page == NULL)) { 309247835Skib ret = -ENOMEM; 310247835Skib goto err_ret; 311247835Skib } 312247835Skib pmap_copy_page(from_page, to_page); 313247835Skib } 314247835Skib vm_object_pip_wakeup(obj); 315248084Sattilio VM_OBJECT_WUNLOCK(obj); 316247835Skib 317247835Skib if (!(ttm->page_flags & TTM_PAGE_FLAG_PERSISTENT_SWAP)) 318247835Skib vm_object_deallocate(obj); 319247835Skib ttm->swap_storage = NULL; 320247835Skib ttm->page_flags &= ~TTM_PAGE_FLAG_SWAPPED; 321247835Skib return (0); 322247835Skib 323247835Skiberr_ret: 324247835Skib vm_object_pip_wakeup(obj); 325248084Sattilio VM_OBJECT_WUNLOCK(obj); 326247835Skib return (ret); 327247835Skib} 328247835Skib 329247835Skibint ttm_tt_swapout(struct ttm_tt *ttm, vm_object_t persistent_swap_storage) 330247835Skib{ 331247835Skib vm_object_t obj; 332247835Skib vm_page_t from_page, to_page; 333247835Skib int i; 334247835Skib 335247835Skib MPASS(ttm->state == tt_unbound || ttm->state == tt_unpopulated); 336247835Skib MPASS(ttm->caching_state == tt_cached); 337247835Skib 338247835Skib if (persistent_swap_storage == NULL) { 339247835Skib obj = vm_pager_allocate(OBJT_SWAP, NULL, 340247835Skib IDX_TO_OFF(ttm->num_pages), VM_PROT_DEFAULT, 0, 341247835Skib curthread->td_ucred); 342247835Skib if (obj == NULL) { 343247835Skib printf("[TTM] Failed allocating swap storage\n"); 344247835Skib return (-ENOMEM); 345247835Skib } 346247835Skib } else 347247835Skib obj = persistent_swap_storage; 348247835Skib 349248084Sattilio VM_OBJECT_WLOCK(obj); 350247835Skib vm_object_pip_add(obj, 1); 351247835Skib for (i = 0; i < ttm->num_pages; ++i) { 352247835Skib from_page = ttm->pages[i]; 353247835Skib if (unlikely(from_page == NULL)) 354247835Skib continue; 355254649Skib to_page = vm_page_grab(obj, i, VM_ALLOC_NORMAL); 356247835Skib pmap_copy_page(from_page, to_page); 357254875Sdumbbell to_page->valid = VM_PAGE_BITS_ALL; 358247835Skib vm_page_dirty(to_page); 359254138Sattilio vm_page_xunbusy(to_page); 360247835Skib } 361247835Skib vm_object_pip_wakeup(obj); 362248084Sattilio VM_OBJECT_WUNLOCK(obj); 363247835Skib 364247835Skib ttm->bdev->driver->ttm_tt_unpopulate(ttm); 365247835Skib ttm->swap_storage = obj; 366247835Skib ttm->page_flags |= TTM_PAGE_FLAG_SWAPPED; 367247835Skib if (persistent_swap_storage != NULL) 368247835Skib ttm->page_flags |= TTM_PAGE_FLAG_PERSISTENT_SWAP; 369247835Skib return (0); 370247835Skib} 371