1/************************************************************************** 2 * 3 * Copyright (c) 2007-2009 VMware, Inc., Palo Alto, CA., USA 4 * All Rights Reserved. 5 * 6 * Permission is hereby granted, free of charge, to any person obtaining a 7 * copy of this software and associated documentation files (the 8 * "Software"), to deal in the Software without restriction, including 9 * without limitation the rights to use, copy, modify, merge, publish, 10 * distribute, sub license, and/or sell copies of the Software, and to 11 * permit persons to whom the Software is furnished to do so, subject to 12 * the following conditions: 13 * 14 * The above copyright notice and this permission notice (including the 15 * next paragraph) shall be included in all copies or substantial portions 16 * of the Software. 17 * 18 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 19 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 20 * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL 21 * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM, 22 * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR 23 * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE 24 * USE OR OTHER DEALINGS IN THE SOFTWARE. 25 * 26 **************************************************************************/ 27/* 28 * Authors: Thomas Hellstrom <thellstrom-at-vmware-dot-com> 29 */ 30 31#include <sys/cdefs.h>
| 1/************************************************************************** 2 * 3 * Copyright (c) 2007-2009 VMware, Inc., Palo Alto, CA., USA 4 * All Rights Reserved. 5 * 6 * Permission is hereby granted, free of charge, to any person obtaining a 7 * copy of this software and associated documentation files (the 8 * "Software"), to deal in the Software without restriction, including 9 * without limitation the rights to use, copy, modify, merge, publish, 10 * distribute, sub license, and/or sell copies of the Software, and to 11 * permit persons to whom the Software is furnished to do so, subject to 12 * the following conditions: 13 * 14 * The above copyright notice and this permission notice (including the 15 * next paragraph) shall be included in all copies or substantial portions 16 * of the Software. 17 * 18 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 19 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 20 * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL 21 * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM, 22 * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR 23 * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE 24 * USE OR OTHER DEALINGS IN THE SOFTWARE. 25 * 26 **************************************************************************/ 27/* 28 * Authors: Thomas Hellstrom <thellstrom-at-vmware-dot-com> 29 */ 30 31#include <sys/cdefs.h>
|
32__FBSDID("$FreeBSD: head/sys/dev/drm2/ttm/ttm_bo_util.c 254025 2013-08-07 06:21:20Z jeff $");
| 32__FBSDID("$FreeBSD: head/sys/dev/drm2/ttm/ttm_bo_util.c 254866 2013-08-25 14:55:08Z dumbbell $");
|
33 34#include <dev/drm2/drmP.h> 35#include <dev/drm2/ttm/ttm_bo_driver.h> 36#include <dev/drm2/ttm/ttm_placement.h> 37#include <sys/sf_buf.h> 38 39void ttm_bo_free_old_node(struct ttm_buffer_object *bo) 40{ 41 ttm_bo_mem_put(bo, &bo->mem); 42} 43 44int ttm_bo_move_ttm(struct ttm_buffer_object *bo, 45 bool evict, 46 bool no_wait_gpu, struct ttm_mem_reg *new_mem) 47{ 48 struct ttm_tt *ttm = bo->ttm; 49 struct ttm_mem_reg *old_mem = &bo->mem; 50 int ret; 51 52 if (old_mem->mem_type != TTM_PL_SYSTEM) { 53 ttm_tt_unbind(ttm); 54 ttm_bo_free_old_node(bo); 55 ttm_flag_masked(&old_mem->placement, TTM_PL_FLAG_SYSTEM, 56 TTM_PL_MASK_MEM); 57 old_mem->mem_type = TTM_PL_SYSTEM; 58 } 59 60 ret = ttm_tt_set_placement_caching(ttm, new_mem->placement); 61 if (unlikely(ret != 0)) 62 return ret; 63 64 if (new_mem->mem_type != TTM_PL_SYSTEM) { 65 ret = ttm_tt_bind(ttm, new_mem); 66 if (unlikely(ret != 0)) 67 return ret; 68 } 69 70 *old_mem = *new_mem; 71 new_mem->mm_node = NULL; 72 73 return 0; 74} 75 76int ttm_mem_io_lock(struct ttm_mem_type_manager *man, bool interruptible) 77{ 78 if (likely(man->io_reserve_fastpath)) 79 return 0; 80 81 if (interruptible) { 82 if (sx_xlock_sig(&man->io_reserve_mutex)) 83 return (-EINTR); 84 else 85 return (0); 86 } 87 88 sx_xlock(&man->io_reserve_mutex); 89 return 0; 90} 91 92void ttm_mem_io_unlock(struct ttm_mem_type_manager *man) 93{ 94 if (likely(man->io_reserve_fastpath)) 95 return; 96 97 sx_xunlock(&man->io_reserve_mutex); 98} 99 100static int ttm_mem_io_evict(struct ttm_mem_type_manager *man) 101{ 102 struct ttm_buffer_object *bo; 103 104 if (!man->use_io_reserve_lru || list_empty(&man->io_reserve_lru)) 105 return -EAGAIN; 106 107 bo = list_first_entry(&man->io_reserve_lru, 108 struct ttm_buffer_object, 109 io_reserve_lru); 110 list_del_init(&bo->io_reserve_lru); 111 ttm_bo_unmap_virtual_locked(bo); 112 113 return 0; 114} 115 116static int ttm_mem_io_reserve(struct ttm_bo_device *bdev, 117 struct ttm_mem_reg *mem) 118{ 119 struct ttm_mem_type_manager *man = &bdev->man[mem->mem_type]; 120 int ret = 0; 121 122 if (!bdev->driver->io_mem_reserve) 123 return 0; 124 if (likely(man->io_reserve_fastpath)) 125 return bdev->driver->io_mem_reserve(bdev, mem); 126 127 if (bdev->driver->io_mem_reserve && 128 mem->bus.io_reserved_count++ == 0) { 129retry: 130 ret = bdev->driver->io_mem_reserve(bdev, mem); 131 if (ret == -EAGAIN) { 132 ret = ttm_mem_io_evict(man); 133 if (ret == 0) 134 goto retry; 135 } 136 } 137 return ret; 138} 139 140static void ttm_mem_io_free(struct ttm_bo_device *bdev, 141 struct ttm_mem_reg *mem) 142{ 143 struct ttm_mem_type_manager *man = &bdev->man[mem->mem_type]; 144 145 if (likely(man->io_reserve_fastpath)) 146 return; 147 148 if (bdev->driver->io_mem_reserve && 149 --mem->bus.io_reserved_count == 0 && 150 bdev->driver->io_mem_free) 151 bdev->driver->io_mem_free(bdev, mem); 152 153} 154 155int ttm_mem_io_reserve_vm(struct ttm_buffer_object *bo) 156{ 157 struct ttm_mem_reg *mem = &bo->mem; 158 int ret; 159 160 if (!mem->bus.io_reserved_vm) { 161 struct ttm_mem_type_manager *man = 162 &bo->bdev->man[mem->mem_type]; 163 164 ret = ttm_mem_io_reserve(bo->bdev, mem); 165 if (unlikely(ret != 0)) 166 return ret; 167 mem->bus.io_reserved_vm = true; 168 if (man->use_io_reserve_lru) 169 list_add_tail(&bo->io_reserve_lru, 170 &man->io_reserve_lru); 171 } 172 return 0; 173} 174 175void ttm_mem_io_free_vm(struct ttm_buffer_object *bo) 176{ 177 struct ttm_mem_reg *mem = &bo->mem; 178 179 if (mem->bus.io_reserved_vm) { 180 mem->bus.io_reserved_vm = false; 181 list_del_init(&bo->io_reserve_lru); 182 ttm_mem_io_free(bo->bdev, mem); 183 } 184} 185 186static 187int ttm_mem_reg_ioremap(struct ttm_bo_device *bdev, struct ttm_mem_reg *mem, 188 void **virtual) 189{ 190 struct ttm_mem_type_manager *man = &bdev->man[mem->mem_type]; 191 int ret; 192 void *addr; 193 194 *virtual = NULL; 195 (void) ttm_mem_io_lock(man, false); 196 ret = ttm_mem_io_reserve(bdev, mem); 197 ttm_mem_io_unlock(man); 198 if (ret || !mem->bus.is_iomem) 199 return ret; 200 201 if (mem->bus.addr) { 202 addr = mem->bus.addr; 203 } else { 204 addr = pmap_mapdev_attr(mem->bus.base + mem->bus.offset, 205 mem->bus.size, (mem->placement & TTM_PL_FLAG_WC) ? 206 VM_MEMATTR_WRITE_COMBINING : VM_MEMATTR_UNCACHEABLE); 207 if (!addr) { 208 (void) ttm_mem_io_lock(man, false); 209 ttm_mem_io_free(bdev, mem); 210 ttm_mem_io_unlock(man); 211 return -ENOMEM; 212 } 213 } 214 *virtual = addr; 215 return 0; 216} 217 218static 219void ttm_mem_reg_iounmap(struct ttm_bo_device *bdev, struct ttm_mem_reg *mem, 220 void *virtual) 221{ 222 struct ttm_mem_type_manager *man; 223 224 man = &bdev->man[mem->mem_type]; 225 226 if (virtual && mem->bus.addr == NULL) 227 pmap_unmapdev((vm_offset_t)virtual, mem->bus.size); 228 (void) ttm_mem_io_lock(man, false); 229 ttm_mem_io_free(bdev, mem); 230 ttm_mem_io_unlock(man); 231} 232 233static int ttm_copy_io_page(void *dst, void *src, unsigned long page) 234{ 235 uint32_t *dstP = 236 (uint32_t *) ((unsigned long)dst + (page << PAGE_SHIFT)); 237 uint32_t *srcP = 238 (uint32_t *) ((unsigned long)src + (page << PAGE_SHIFT)); 239 240 int i; 241 for (i = 0; i < PAGE_SIZE / sizeof(uint32_t); ++i) 242 /* iowrite32(ioread32(srcP++), dstP++); */ 243 *dstP++ = *srcP++; 244 return 0; 245} 246 247static int ttm_copy_io_ttm_page(struct ttm_tt *ttm, void *src, 248 unsigned long page, 249 vm_memattr_t prot) 250{ 251 vm_page_t d = ttm->pages[page]; 252 void *dst; 253 254 if (!d) 255 return -ENOMEM; 256 257 src = (void *)((unsigned long)src + (page << PAGE_SHIFT)); 258 259 /* XXXKIB can't sleep ? */ 260 dst = pmap_mapdev_attr(VM_PAGE_TO_PHYS(d), PAGE_SIZE, prot); 261 if (!dst) 262 return -ENOMEM; 263 264 memcpy(dst, src, PAGE_SIZE); 265 266 pmap_unmapdev((vm_offset_t)dst, PAGE_SIZE); 267 268 return 0; 269} 270 271static int ttm_copy_ttm_io_page(struct ttm_tt *ttm, void *dst, 272 unsigned long page, 273 vm_memattr_t prot) 274{ 275 vm_page_t s = ttm->pages[page]; 276 void *src; 277 278 if (!s) 279 return -ENOMEM; 280 281 dst = (void *)((unsigned long)dst + (page << PAGE_SHIFT)); 282 src = pmap_mapdev_attr(VM_PAGE_TO_PHYS(s), PAGE_SIZE, prot); 283 if (!src) 284 return -ENOMEM; 285 286 memcpy(dst, src, PAGE_SIZE); 287 288 pmap_unmapdev((vm_offset_t)src, PAGE_SIZE); 289 290 return 0; 291} 292 293int ttm_bo_move_memcpy(struct ttm_buffer_object *bo, 294 bool evict, bool no_wait_gpu, 295 struct ttm_mem_reg *new_mem) 296{ 297 struct ttm_bo_device *bdev = bo->bdev; 298 struct ttm_mem_type_manager *man = &bdev->man[new_mem->mem_type]; 299 struct ttm_tt *ttm = bo->ttm; 300 struct ttm_mem_reg *old_mem = &bo->mem; 301 struct ttm_mem_reg old_copy = *old_mem; 302 void *old_iomap; 303 void *new_iomap; 304 int ret; 305 unsigned long i; 306 unsigned long page; 307 unsigned long add = 0; 308 int dir; 309 310 ret = ttm_mem_reg_ioremap(bdev, old_mem, &old_iomap); 311 if (ret) 312 return ret; 313 ret = ttm_mem_reg_ioremap(bdev, new_mem, &new_iomap); 314 if (ret) 315 goto out; 316 317 if (old_iomap == NULL && new_iomap == NULL) 318 goto out2; 319 if (old_iomap == NULL && ttm == NULL) 320 goto out2; 321 322 if (ttm->state == tt_unpopulated) { 323 ret = ttm->bdev->driver->ttm_tt_populate(ttm);
| 33 34#include <dev/drm2/drmP.h> 35#include <dev/drm2/ttm/ttm_bo_driver.h> 36#include <dev/drm2/ttm/ttm_placement.h> 37#include <sys/sf_buf.h> 38 39void ttm_bo_free_old_node(struct ttm_buffer_object *bo) 40{ 41 ttm_bo_mem_put(bo, &bo->mem); 42} 43 44int ttm_bo_move_ttm(struct ttm_buffer_object *bo, 45 bool evict, 46 bool no_wait_gpu, struct ttm_mem_reg *new_mem) 47{ 48 struct ttm_tt *ttm = bo->ttm; 49 struct ttm_mem_reg *old_mem = &bo->mem; 50 int ret; 51 52 if (old_mem->mem_type != TTM_PL_SYSTEM) { 53 ttm_tt_unbind(ttm); 54 ttm_bo_free_old_node(bo); 55 ttm_flag_masked(&old_mem->placement, TTM_PL_FLAG_SYSTEM, 56 TTM_PL_MASK_MEM); 57 old_mem->mem_type = TTM_PL_SYSTEM; 58 } 59 60 ret = ttm_tt_set_placement_caching(ttm, new_mem->placement); 61 if (unlikely(ret != 0)) 62 return ret; 63 64 if (new_mem->mem_type != TTM_PL_SYSTEM) { 65 ret = ttm_tt_bind(ttm, new_mem); 66 if (unlikely(ret != 0)) 67 return ret; 68 } 69 70 *old_mem = *new_mem; 71 new_mem->mm_node = NULL; 72 73 return 0; 74} 75 76int ttm_mem_io_lock(struct ttm_mem_type_manager *man, bool interruptible) 77{ 78 if (likely(man->io_reserve_fastpath)) 79 return 0; 80 81 if (interruptible) { 82 if (sx_xlock_sig(&man->io_reserve_mutex)) 83 return (-EINTR); 84 else 85 return (0); 86 } 87 88 sx_xlock(&man->io_reserve_mutex); 89 return 0; 90} 91 92void ttm_mem_io_unlock(struct ttm_mem_type_manager *man) 93{ 94 if (likely(man->io_reserve_fastpath)) 95 return; 96 97 sx_xunlock(&man->io_reserve_mutex); 98} 99 100static int ttm_mem_io_evict(struct ttm_mem_type_manager *man) 101{ 102 struct ttm_buffer_object *bo; 103 104 if (!man->use_io_reserve_lru || list_empty(&man->io_reserve_lru)) 105 return -EAGAIN; 106 107 bo = list_first_entry(&man->io_reserve_lru, 108 struct ttm_buffer_object, 109 io_reserve_lru); 110 list_del_init(&bo->io_reserve_lru); 111 ttm_bo_unmap_virtual_locked(bo); 112 113 return 0; 114} 115 116static int ttm_mem_io_reserve(struct ttm_bo_device *bdev, 117 struct ttm_mem_reg *mem) 118{ 119 struct ttm_mem_type_manager *man = &bdev->man[mem->mem_type]; 120 int ret = 0; 121 122 if (!bdev->driver->io_mem_reserve) 123 return 0; 124 if (likely(man->io_reserve_fastpath)) 125 return bdev->driver->io_mem_reserve(bdev, mem); 126 127 if (bdev->driver->io_mem_reserve && 128 mem->bus.io_reserved_count++ == 0) { 129retry: 130 ret = bdev->driver->io_mem_reserve(bdev, mem); 131 if (ret == -EAGAIN) { 132 ret = ttm_mem_io_evict(man); 133 if (ret == 0) 134 goto retry; 135 } 136 } 137 return ret; 138} 139 140static void ttm_mem_io_free(struct ttm_bo_device *bdev, 141 struct ttm_mem_reg *mem) 142{ 143 struct ttm_mem_type_manager *man = &bdev->man[mem->mem_type]; 144 145 if (likely(man->io_reserve_fastpath)) 146 return; 147 148 if (bdev->driver->io_mem_reserve && 149 --mem->bus.io_reserved_count == 0 && 150 bdev->driver->io_mem_free) 151 bdev->driver->io_mem_free(bdev, mem); 152 153} 154 155int ttm_mem_io_reserve_vm(struct ttm_buffer_object *bo) 156{ 157 struct ttm_mem_reg *mem = &bo->mem; 158 int ret; 159 160 if (!mem->bus.io_reserved_vm) { 161 struct ttm_mem_type_manager *man = 162 &bo->bdev->man[mem->mem_type]; 163 164 ret = ttm_mem_io_reserve(bo->bdev, mem); 165 if (unlikely(ret != 0)) 166 return ret; 167 mem->bus.io_reserved_vm = true; 168 if (man->use_io_reserve_lru) 169 list_add_tail(&bo->io_reserve_lru, 170 &man->io_reserve_lru); 171 } 172 return 0; 173} 174 175void ttm_mem_io_free_vm(struct ttm_buffer_object *bo) 176{ 177 struct ttm_mem_reg *mem = &bo->mem; 178 179 if (mem->bus.io_reserved_vm) { 180 mem->bus.io_reserved_vm = false; 181 list_del_init(&bo->io_reserve_lru); 182 ttm_mem_io_free(bo->bdev, mem); 183 } 184} 185 186static 187int ttm_mem_reg_ioremap(struct ttm_bo_device *bdev, struct ttm_mem_reg *mem, 188 void **virtual) 189{ 190 struct ttm_mem_type_manager *man = &bdev->man[mem->mem_type]; 191 int ret; 192 void *addr; 193 194 *virtual = NULL; 195 (void) ttm_mem_io_lock(man, false); 196 ret = ttm_mem_io_reserve(bdev, mem); 197 ttm_mem_io_unlock(man); 198 if (ret || !mem->bus.is_iomem) 199 return ret; 200 201 if (mem->bus.addr) { 202 addr = mem->bus.addr; 203 } else { 204 addr = pmap_mapdev_attr(mem->bus.base + mem->bus.offset, 205 mem->bus.size, (mem->placement & TTM_PL_FLAG_WC) ? 206 VM_MEMATTR_WRITE_COMBINING : VM_MEMATTR_UNCACHEABLE); 207 if (!addr) { 208 (void) ttm_mem_io_lock(man, false); 209 ttm_mem_io_free(bdev, mem); 210 ttm_mem_io_unlock(man); 211 return -ENOMEM; 212 } 213 } 214 *virtual = addr; 215 return 0; 216} 217 218static 219void ttm_mem_reg_iounmap(struct ttm_bo_device *bdev, struct ttm_mem_reg *mem, 220 void *virtual) 221{ 222 struct ttm_mem_type_manager *man; 223 224 man = &bdev->man[mem->mem_type]; 225 226 if (virtual && mem->bus.addr == NULL) 227 pmap_unmapdev((vm_offset_t)virtual, mem->bus.size); 228 (void) ttm_mem_io_lock(man, false); 229 ttm_mem_io_free(bdev, mem); 230 ttm_mem_io_unlock(man); 231} 232 233static int ttm_copy_io_page(void *dst, void *src, unsigned long page) 234{ 235 uint32_t *dstP = 236 (uint32_t *) ((unsigned long)dst + (page << PAGE_SHIFT)); 237 uint32_t *srcP = 238 (uint32_t *) ((unsigned long)src + (page << PAGE_SHIFT)); 239 240 int i; 241 for (i = 0; i < PAGE_SIZE / sizeof(uint32_t); ++i) 242 /* iowrite32(ioread32(srcP++), dstP++); */ 243 *dstP++ = *srcP++; 244 return 0; 245} 246 247static int ttm_copy_io_ttm_page(struct ttm_tt *ttm, void *src, 248 unsigned long page, 249 vm_memattr_t prot) 250{ 251 vm_page_t d = ttm->pages[page]; 252 void *dst; 253 254 if (!d) 255 return -ENOMEM; 256 257 src = (void *)((unsigned long)src + (page << PAGE_SHIFT)); 258 259 /* XXXKIB can't sleep ? */ 260 dst = pmap_mapdev_attr(VM_PAGE_TO_PHYS(d), PAGE_SIZE, prot); 261 if (!dst) 262 return -ENOMEM; 263 264 memcpy(dst, src, PAGE_SIZE); 265 266 pmap_unmapdev((vm_offset_t)dst, PAGE_SIZE); 267 268 return 0; 269} 270 271static int ttm_copy_ttm_io_page(struct ttm_tt *ttm, void *dst, 272 unsigned long page, 273 vm_memattr_t prot) 274{ 275 vm_page_t s = ttm->pages[page]; 276 void *src; 277 278 if (!s) 279 return -ENOMEM; 280 281 dst = (void *)((unsigned long)dst + (page << PAGE_SHIFT)); 282 src = pmap_mapdev_attr(VM_PAGE_TO_PHYS(s), PAGE_SIZE, prot); 283 if (!src) 284 return -ENOMEM; 285 286 memcpy(dst, src, PAGE_SIZE); 287 288 pmap_unmapdev((vm_offset_t)src, PAGE_SIZE); 289 290 return 0; 291} 292 293int ttm_bo_move_memcpy(struct ttm_buffer_object *bo, 294 bool evict, bool no_wait_gpu, 295 struct ttm_mem_reg *new_mem) 296{ 297 struct ttm_bo_device *bdev = bo->bdev; 298 struct ttm_mem_type_manager *man = &bdev->man[new_mem->mem_type]; 299 struct ttm_tt *ttm = bo->ttm; 300 struct ttm_mem_reg *old_mem = &bo->mem; 301 struct ttm_mem_reg old_copy = *old_mem; 302 void *old_iomap; 303 void *new_iomap; 304 int ret; 305 unsigned long i; 306 unsigned long page; 307 unsigned long add = 0; 308 int dir; 309 310 ret = ttm_mem_reg_ioremap(bdev, old_mem, &old_iomap); 311 if (ret) 312 return ret; 313 ret = ttm_mem_reg_ioremap(bdev, new_mem, &new_iomap); 314 if (ret) 315 goto out; 316 317 if (old_iomap == NULL && new_iomap == NULL) 318 goto out2; 319 if (old_iomap == NULL && ttm == NULL) 320 goto out2; 321 322 if (ttm->state == tt_unpopulated) { 323 ret = ttm->bdev->driver->ttm_tt_populate(ttm);
|
324 if (ret)
| 324 if (ret) { 325 /* if we fail here don't nuke the mm node 326 * as the bo still owns it */ 327 old_copy.mm_node = NULL;
|
325 goto out1;
| 328 goto out1;
|
| 329 }
|
326 } 327 328 add = 0; 329 dir = 1; 330 331 if ((old_mem->mem_type == new_mem->mem_type) && 332 (new_mem->start < old_mem->start + old_mem->size)) { 333 dir = -1; 334 add = new_mem->num_pages - 1; 335 } 336 337 for (i = 0; i < new_mem->num_pages; ++i) { 338 page = i * dir + add; 339 if (old_iomap == NULL) { 340 vm_memattr_t prot = ttm_io_prot(old_mem->placement); 341 ret = ttm_copy_ttm_io_page(ttm, new_iomap, page, 342 prot); 343 } else if (new_iomap == NULL) { 344 vm_memattr_t prot = ttm_io_prot(new_mem->placement); 345 ret = ttm_copy_io_ttm_page(ttm, old_iomap, page, 346 prot); 347 } else 348 ret = ttm_copy_io_page(new_iomap, old_iomap, page);
| 330 } 331 332 add = 0; 333 dir = 1; 334 335 if ((old_mem->mem_type == new_mem->mem_type) && 336 (new_mem->start < old_mem->start + old_mem->size)) { 337 dir = -1; 338 add = new_mem->num_pages - 1; 339 } 340 341 for (i = 0; i < new_mem->num_pages; ++i) { 342 page = i * dir + add; 343 if (old_iomap == NULL) { 344 vm_memattr_t prot = ttm_io_prot(old_mem->placement); 345 ret = ttm_copy_ttm_io_page(ttm, new_iomap, page, 346 prot); 347 } else if (new_iomap == NULL) { 348 vm_memattr_t prot = ttm_io_prot(new_mem->placement); 349 ret = ttm_copy_io_ttm_page(ttm, old_iomap, page, 350 prot); 351 } else 352 ret = ttm_copy_io_page(new_iomap, old_iomap, page);
|
349 if (ret)
| 353 if (ret) { 354 /* failing here, means keep old copy as-is */ 355 old_copy.mm_node = NULL;
|
350 goto out1;
| 356 goto out1;
|
| 357 }
|
351 } 352 mb(); 353out2: 354 old_copy = *old_mem; 355 *old_mem = *new_mem; 356 new_mem->mm_node = NULL; 357 358 if ((man->flags & TTM_MEMTYPE_FLAG_FIXED) && (ttm != NULL)) { 359 ttm_tt_unbind(ttm); 360 ttm_tt_destroy(ttm); 361 bo->ttm = NULL; 362 } 363 364out1: 365 ttm_mem_reg_iounmap(bdev, old_mem, new_iomap); 366out: 367 ttm_mem_reg_iounmap(bdev, &old_copy, old_iomap); 368 ttm_bo_mem_put(bo, &old_copy); 369 return ret; 370} 371 372MALLOC_DEFINE(M_TTM_TRANSF_OBJ, "ttm_transf_obj", "TTM Transfer Objects"); 373 374static void ttm_transfered_destroy(struct ttm_buffer_object *bo) 375{ 376 free(bo, M_TTM_TRANSF_OBJ); 377} 378 379/** 380 * ttm_buffer_object_transfer 381 * 382 * @bo: A pointer to a struct ttm_buffer_object. 383 * @new_obj: A pointer to a pointer to a newly created ttm_buffer_object, 384 * holding the data of @bo with the old placement. 385 * 386 * This is a utility function that may be called after an accelerated move 387 * has been scheduled. A new buffer object is created as a placeholder for 388 * the old data while it's being copied. When that buffer object is idle, 389 * it can be destroyed, releasing the space of the old placement. 390 * Returns: 391 * !0: Failure. 392 */ 393 394static int 395ttm_buffer_object_transfer(struct ttm_buffer_object *bo, 396 void *sync_obj, struct ttm_buffer_object **new_obj) 397{ 398 struct ttm_buffer_object *fbo; 399 400 fbo = malloc(sizeof(*fbo), M_TTM_TRANSF_OBJ, M_ZERO | M_WAITOK); 401 *fbo = *bo; 402 403 /** 404 * Fix up members that we shouldn't copy directly: 405 * TODO: Explicit member copy would probably be better here. 406 */ 407 408 INIT_LIST_HEAD(&fbo->ddestroy); 409 INIT_LIST_HEAD(&fbo->lru); 410 INIT_LIST_HEAD(&fbo->swap); 411 INIT_LIST_HEAD(&fbo->io_reserve_lru); 412 fbo->vm_node = NULL; 413 atomic_set(&fbo->cpu_writers, 0); 414 415 fbo->sync_obj = sync_obj; 416 refcount_init(&fbo->list_kref, 1); 417 refcount_init(&fbo->kref, 1); 418 fbo->destroy = &ttm_transfered_destroy; 419 fbo->acc_size = 0; 420 421 *new_obj = fbo; 422 return 0; 423} 424 425vm_memattr_t 426ttm_io_prot(uint32_t caching_flags) 427{ 428#if defined(__i386__) || defined(__amd64__) 429 if (caching_flags & TTM_PL_FLAG_WC) 430 return (VM_MEMATTR_WRITE_COMBINING); 431 else 432 /* 433 * We do not support i386, look at the linux source 434 * for the reason of the comment. 435 */ 436 return (VM_MEMATTR_UNCACHEABLE); 437#else 438#error Port me 439#endif 440} 441 442static int ttm_bo_ioremap(struct ttm_buffer_object *bo, 443 unsigned long offset, 444 unsigned long size, 445 struct ttm_bo_kmap_obj *map) 446{ 447 struct ttm_mem_reg *mem = &bo->mem; 448 449 if (bo->mem.bus.addr) { 450 map->bo_kmap_type = ttm_bo_map_premapped; 451 map->virtual = (void *)(((u8 *)bo->mem.bus.addr) + offset); 452 } else { 453 map->bo_kmap_type = ttm_bo_map_iomap; 454 map->virtual = pmap_mapdev_attr(bo->mem.bus.base + 455 bo->mem.bus.offset + offset, size, 456 (mem->placement & TTM_PL_FLAG_WC) ? 457 VM_MEMATTR_WRITE_COMBINING : VM_MEMATTR_UNCACHEABLE); 458 map->size = size; 459 } 460 return (!map->virtual) ? -ENOMEM : 0; 461} 462 463static int ttm_bo_kmap_ttm(struct ttm_buffer_object *bo, 464 unsigned long start_page, 465 unsigned long num_pages, 466 struct ttm_bo_kmap_obj *map) 467{ 468 struct ttm_mem_reg *mem = &bo->mem; 469 vm_memattr_t prot; 470 struct ttm_tt *ttm = bo->ttm; 471 int i, ret; 472 473 MPASS(ttm != NULL); 474 475 if (ttm->state == tt_unpopulated) { 476 ret = ttm->bdev->driver->ttm_tt_populate(ttm); 477 if (ret) 478 return ret; 479 } 480 481 if (num_pages == 1 && (mem->placement & TTM_PL_FLAG_CACHED)) { 482 /* 483 * We're mapping a single page, and the desired 484 * page protection is consistent with the bo. 485 */ 486 487 map->bo_kmap_type = ttm_bo_map_kmap; 488 map->page = ttm->pages[start_page]; 489 map->sf = sf_buf_alloc(map->page, 0); 490 map->virtual = (void *)sf_buf_kva(map->sf); 491 } else { 492 /* 493 * We need to use vmap to get the desired page protection 494 * or to make the buffer object look contiguous. 495 */ 496 prot = (mem->placement & TTM_PL_FLAG_CACHED) ? 497 VM_MEMATTR_WRITE_COMBINING : 498 ttm_io_prot(mem->placement); 499 map->bo_kmap_type = ttm_bo_map_vmap; 500 map->num_pages = num_pages; 501 map->virtual = (void *)kva_alloc(num_pages * PAGE_SIZE); 502 if (map->virtual != NULL) { 503 for (i = 0; i < num_pages; i++) { 504 /* XXXKIB hack */ 505 pmap_page_set_memattr(ttm->pages[start_page + 506 i], prot); 507 } 508 pmap_qenter((vm_offset_t)map->virtual, 509 &ttm->pages[start_page], num_pages); 510 } 511 } 512 return (!map->virtual) ? -ENOMEM : 0; 513} 514 515int ttm_bo_kmap(struct ttm_buffer_object *bo, 516 unsigned long start_page, unsigned long num_pages, 517 struct ttm_bo_kmap_obj *map) 518{ 519 struct ttm_mem_type_manager *man = 520 &bo->bdev->man[bo->mem.mem_type]; 521 unsigned long offset, size; 522 int ret; 523 524 MPASS(list_empty(&bo->swap)); 525 map->virtual = NULL; 526 map->bo = bo; 527 if (num_pages > bo->num_pages) 528 return -EINVAL; 529 if (start_page > bo->num_pages) 530 return -EINVAL; 531#if 0 532 if (num_pages > 1 && !DRM_SUSER(DRM_CURPROC)) 533 return -EPERM; 534#endif 535 (void) ttm_mem_io_lock(man, false); 536 ret = ttm_mem_io_reserve(bo->bdev, &bo->mem); 537 ttm_mem_io_unlock(man); 538 if (ret) 539 return ret; 540 if (!bo->mem.bus.is_iomem) { 541 return ttm_bo_kmap_ttm(bo, start_page, num_pages, map); 542 } else { 543 offset = start_page << PAGE_SHIFT; 544 size = num_pages << PAGE_SHIFT; 545 return ttm_bo_ioremap(bo, offset, size, map); 546 } 547} 548 549void ttm_bo_kunmap(struct ttm_bo_kmap_obj *map) 550{ 551 struct ttm_buffer_object *bo = map->bo; 552 struct ttm_mem_type_manager *man = 553 &bo->bdev->man[bo->mem.mem_type]; 554 555 if (!map->virtual) 556 return; 557 switch (map->bo_kmap_type) { 558 case ttm_bo_map_iomap: 559 pmap_unmapdev((vm_offset_t)map->virtual, map->size); 560 break; 561 case ttm_bo_map_vmap: 562 pmap_qremove((vm_offset_t)(map->virtual), map->num_pages); 563 kva_free((vm_offset_t)map->virtual, 564 map->num_pages * PAGE_SIZE); 565 break; 566 case ttm_bo_map_kmap: 567 sf_buf_free(map->sf); 568 break; 569 case ttm_bo_map_premapped: 570 break; 571 default: 572 MPASS(0); 573 } 574 (void) ttm_mem_io_lock(man, false); 575 ttm_mem_io_free(map->bo->bdev, &map->bo->mem); 576 ttm_mem_io_unlock(man); 577 map->virtual = NULL; 578 map->page = NULL; 579 map->sf = NULL; 580} 581 582int ttm_bo_move_accel_cleanup(struct ttm_buffer_object *bo, 583 void *sync_obj, 584 bool evict, 585 bool no_wait_gpu, 586 struct ttm_mem_reg *new_mem) 587{ 588 struct ttm_bo_device *bdev = bo->bdev; 589 struct ttm_bo_driver *driver = bdev->driver; 590 struct ttm_mem_type_manager *man = &bdev->man[new_mem->mem_type]; 591 struct ttm_mem_reg *old_mem = &bo->mem; 592 int ret; 593 struct ttm_buffer_object *ghost_obj; 594 void *tmp_obj = NULL; 595 void *sync_obj_ref; 596 597 mtx_lock(&bdev->fence_lock); 598 if (bo->sync_obj) { 599 tmp_obj = bo->sync_obj; 600 bo->sync_obj = NULL; 601 } 602 bo->sync_obj = driver->sync_obj_ref(sync_obj); 603 if (evict) { 604 ret = ttm_bo_wait(bo, false, false, false); 605 mtx_unlock(&bdev->fence_lock); 606 if (tmp_obj) 607 driver->sync_obj_unref(&tmp_obj); 608 if (ret) 609 return ret; 610 611 if ((man->flags & TTM_MEMTYPE_FLAG_FIXED) && 612 (bo->ttm != NULL)) { 613 ttm_tt_unbind(bo->ttm); 614 ttm_tt_destroy(bo->ttm); 615 bo->ttm = NULL; 616 } 617 ttm_bo_free_old_node(bo); 618 } else { 619 /** 620 * This should help pipeline ordinary buffer moves. 621 * 622 * Hang old buffer memory on a new buffer object, 623 * and leave it to be released when the GPU 624 * operation has completed. 625 */ 626 627 set_bit(TTM_BO_PRIV_FLAG_MOVING, &bo->priv_flags); 628 629 sync_obj_ref = bo->bdev->driver->sync_obj_ref(bo->sync_obj); 630 mtx_unlock(&bdev->fence_lock); 631 /* ttm_buffer_object_transfer accesses bo->sync_obj */ 632 ret = ttm_buffer_object_transfer(bo, sync_obj_ref, &ghost_obj); 633 if (tmp_obj) 634 driver->sync_obj_unref(&tmp_obj); 635 636 if (ret) 637 return ret; 638 639 /** 640 * If we're not moving to fixed memory, the TTM object 641 * needs to stay alive. Otherwhise hang it on the ghost 642 * bo to be unbound and destroyed. 643 */ 644 645 if (!(man->flags & TTM_MEMTYPE_FLAG_FIXED)) 646 ghost_obj->ttm = NULL; 647 else 648 bo->ttm = NULL; 649 650 ttm_bo_unreserve(ghost_obj); 651 ttm_bo_unref(&ghost_obj); 652 } 653 654 *old_mem = *new_mem; 655 new_mem->mm_node = NULL; 656 657 return 0; 658}
| 358 } 359 mb(); 360out2: 361 old_copy = *old_mem; 362 *old_mem = *new_mem; 363 new_mem->mm_node = NULL; 364 365 if ((man->flags & TTM_MEMTYPE_FLAG_FIXED) && (ttm != NULL)) { 366 ttm_tt_unbind(ttm); 367 ttm_tt_destroy(ttm); 368 bo->ttm = NULL; 369 } 370 371out1: 372 ttm_mem_reg_iounmap(bdev, old_mem, new_iomap); 373out: 374 ttm_mem_reg_iounmap(bdev, &old_copy, old_iomap); 375 ttm_bo_mem_put(bo, &old_copy); 376 return ret; 377} 378 379MALLOC_DEFINE(M_TTM_TRANSF_OBJ, "ttm_transf_obj", "TTM Transfer Objects"); 380 381static void ttm_transfered_destroy(struct ttm_buffer_object *bo) 382{ 383 free(bo, M_TTM_TRANSF_OBJ); 384} 385 386/** 387 * ttm_buffer_object_transfer 388 * 389 * @bo: A pointer to a struct ttm_buffer_object. 390 * @new_obj: A pointer to a pointer to a newly created ttm_buffer_object, 391 * holding the data of @bo with the old placement. 392 * 393 * This is a utility function that may be called after an accelerated move 394 * has been scheduled. A new buffer object is created as a placeholder for 395 * the old data while it's being copied. When that buffer object is idle, 396 * it can be destroyed, releasing the space of the old placement. 397 * Returns: 398 * !0: Failure. 399 */ 400 401static int 402ttm_buffer_object_transfer(struct ttm_buffer_object *bo, 403 void *sync_obj, struct ttm_buffer_object **new_obj) 404{ 405 struct ttm_buffer_object *fbo; 406 407 fbo = malloc(sizeof(*fbo), M_TTM_TRANSF_OBJ, M_ZERO | M_WAITOK); 408 *fbo = *bo; 409 410 /** 411 * Fix up members that we shouldn't copy directly: 412 * TODO: Explicit member copy would probably be better here. 413 */ 414 415 INIT_LIST_HEAD(&fbo->ddestroy); 416 INIT_LIST_HEAD(&fbo->lru); 417 INIT_LIST_HEAD(&fbo->swap); 418 INIT_LIST_HEAD(&fbo->io_reserve_lru); 419 fbo->vm_node = NULL; 420 atomic_set(&fbo->cpu_writers, 0); 421 422 fbo->sync_obj = sync_obj; 423 refcount_init(&fbo->list_kref, 1); 424 refcount_init(&fbo->kref, 1); 425 fbo->destroy = &ttm_transfered_destroy; 426 fbo->acc_size = 0; 427 428 *new_obj = fbo; 429 return 0; 430} 431 432vm_memattr_t 433ttm_io_prot(uint32_t caching_flags) 434{ 435#if defined(__i386__) || defined(__amd64__) 436 if (caching_flags & TTM_PL_FLAG_WC) 437 return (VM_MEMATTR_WRITE_COMBINING); 438 else 439 /* 440 * We do not support i386, look at the linux source 441 * for the reason of the comment. 442 */ 443 return (VM_MEMATTR_UNCACHEABLE); 444#else 445#error Port me 446#endif 447} 448 449static int ttm_bo_ioremap(struct ttm_buffer_object *bo, 450 unsigned long offset, 451 unsigned long size, 452 struct ttm_bo_kmap_obj *map) 453{ 454 struct ttm_mem_reg *mem = &bo->mem; 455 456 if (bo->mem.bus.addr) { 457 map->bo_kmap_type = ttm_bo_map_premapped; 458 map->virtual = (void *)(((u8 *)bo->mem.bus.addr) + offset); 459 } else { 460 map->bo_kmap_type = ttm_bo_map_iomap; 461 map->virtual = pmap_mapdev_attr(bo->mem.bus.base + 462 bo->mem.bus.offset + offset, size, 463 (mem->placement & TTM_PL_FLAG_WC) ? 464 VM_MEMATTR_WRITE_COMBINING : VM_MEMATTR_UNCACHEABLE); 465 map->size = size; 466 } 467 return (!map->virtual) ? -ENOMEM : 0; 468} 469 470static int ttm_bo_kmap_ttm(struct ttm_buffer_object *bo, 471 unsigned long start_page, 472 unsigned long num_pages, 473 struct ttm_bo_kmap_obj *map) 474{ 475 struct ttm_mem_reg *mem = &bo->mem; 476 vm_memattr_t prot; 477 struct ttm_tt *ttm = bo->ttm; 478 int i, ret; 479 480 MPASS(ttm != NULL); 481 482 if (ttm->state == tt_unpopulated) { 483 ret = ttm->bdev->driver->ttm_tt_populate(ttm); 484 if (ret) 485 return ret; 486 } 487 488 if (num_pages == 1 && (mem->placement & TTM_PL_FLAG_CACHED)) { 489 /* 490 * We're mapping a single page, and the desired 491 * page protection is consistent with the bo. 492 */ 493 494 map->bo_kmap_type = ttm_bo_map_kmap; 495 map->page = ttm->pages[start_page]; 496 map->sf = sf_buf_alloc(map->page, 0); 497 map->virtual = (void *)sf_buf_kva(map->sf); 498 } else { 499 /* 500 * We need to use vmap to get the desired page protection 501 * or to make the buffer object look contiguous. 502 */ 503 prot = (mem->placement & TTM_PL_FLAG_CACHED) ? 504 VM_MEMATTR_WRITE_COMBINING : 505 ttm_io_prot(mem->placement); 506 map->bo_kmap_type = ttm_bo_map_vmap; 507 map->num_pages = num_pages; 508 map->virtual = (void *)kva_alloc(num_pages * PAGE_SIZE); 509 if (map->virtual != NULL) { 510 for (i = 0; i < num_pages; i++) { 511 /* XXXKIB hack */ 512 pmap_page_set_memattr(ttm->pages[start_page + 513 i], prot); 514 } 515 pmap_qenter((vm_offset_t)map->virtual, 516 &ttm->pages[start_page], num_pages); 517 } 518 } 519 return (!map->virtual) ? -ENOMEM : 0; 520} 521 522int ttm_bo_kmap(struct ttm_buffer_object *bo, 523 unsigned long start_page, unsigned long num_pages, 524 struct ttm_bo_kmap_obj *map) 525{ 526 struct ttm_mem_type_manager *man = 527 &bo->bdev->man[bo->mem.mem_type]; 528 unsigned long offset, size; 529 int ret; 530 531 MPASS(list_empty(&bo->swap)); 532 map->virtual = NULL; 533 map->bo = bo; 534 if (num_pages > bo->num_pages) 535 return -EINVAL; 536 if (start_page > bo->num_pages) 537 return -EINVAL; 538#if 0 539 if (num_pages > 1 && !DRM_SUSER(DRM_CURPROC)) 540 return -EPERM; 541#endif 542 (void) ttm_mem_io_lock(man, false); 543 ret = ttm_mem_io_reserve(bo->bdev, &bo->mem); 544 ttm_mem_io_unlock(man); 545 if (ret) 546 return ret; 547 if (!bo->mem.bus.is_iomem) { 548 return ttm_bo_kmap_ttm(bo, start_page, num_pages, map); 549 } else { 550 offset = start_page << PAGE_SHIFT; 551 size = num_pages << PAGE_SHIFT; 552 return ttm_bo_ioremap(bo, offset, size, map); 553 } 554} 555 556void ttm_bo_kunmap(struct ttm_bo_kmap_obj *map) 557{ 558 struct ttm_buffer_object *bo = map->bo; 559 struct ttm_mem_type_manager *man = 560 &bo->bdev->man[bo->mem.mem_type]; 561 562 if (!map->virtual) 563 return; 564 switch (map->bo_kmap_type) { 565 case ttm_bo_map_iomap: 566 pmap_unmapdev((vm_offset_t)map->virtual, map->size); 567 break; 568 case ttm_bo_map_vmap: 569 pmap_qremove((vm_offset_t)(map->virtual), map->num_pages); 570 kva_free((vm_offset_t)map->virtual, 571 map->num_pages * PAGE_SIZE); 572 break; 573 case ttm_bo_map_kmap: 574 sf_buf_free(map->sf); 575 break; 576 case ttm_bo_map_premapped: 577 break; 578 default: 579 MPASS(0); 580 } 581 (void) ttm_mem_io_lock(man, false); 582 ttm_mem_io_free(map->bo->bdev, &map->bo->mem); 583 ttm_mem_io_unlock(man); 584 map->virtual = NULL; 585 map->page = NULL; 586 map->sf = NULL; 587} 588 589int ttm_bo_move_accel_cleanup(struct ttm_buffer_object *bo, 590 void *sync_obj, 591 bool evict, 592 bool no_wait_gpu, 593 struct ttm_mem_reg *new_mem) 594{ 595 struct ttm_bo_device *bdev = bo->bdev; 596 struct ttm_bo_driver *driver = bdev->driver; 597 struct ttm_mem_type_manager *man = &bdev->man[new_mem->mem_type]; 598 struct ttm_mem_reg *old_mem = &bo->mem; 599 int ret; 600 struct ttm_buffer_object *ghost_obj; 601 void *tmp_obj = NULL; 602 void *sync_obj_ref; 603 604 mtx_lock(&bdev->fence_lock); 605 if (bo->sync_obj) { 606 tmp_obj = bo->sync_obj; 607 bo->sync_obj = NULL; 608 } 609 bo->sync_obj = driver->sync_obj_ref(sync_obj); 610 if (evict) { 611 ret = ttm_bo_wait(bo, false, false, false); 612 mtx_unlock(&bdev->fence_lock); 613 if (tmp_obj) 614 driver->sync_obj_unref(&tmp_obj); 615 if (ret) 616 return ret; 617 618 if ((man->flags & TTM_MEMTYPE_FLAG_FIXED) && 619 (bo->ttm != NULL)) { 620 ttm_tt_unbind(bo->ttm); 621 ttm_tt_destroy(bo->ttm); 622 bo->ttm = NULL; 623 } 624 ttm_bo_free_old_node(bo); 625 } else { 626 /** 627 * This should help pipeline ordinary buffer moves. 628 * 629 * Hang old buffer memory on a new buffer object, 630 * and leave it to be released when the GPU 631 * operation has completed. 632 */ 633 634 set_bit(TTM_BO_PRIV_FLAG_MOVING, &bo->priv_flags); 635 636 sync_obj_ref = bo->bdev->driver->sync_obj_ref(bo->sync_obj); 637 mtx_unlock(&bdev->fence_lock); 638 /* ttm_buffer_object_transfer accesses bo->sync_obj */ 639 ret = ttm_buffer_object_transfer(bo, sync_obj_ref, &ghost_obj); 640 if (tmp_obj) 641 driver->sync_obj_unref(&tmp_obj); 642 643 if (ret) 644 return ret; 645 646 /** 647 * If we're not moving to fixed memory, the TTM object 648 * needs to stay alive. Otherwhise hang it on the ghost 649 * bo to be unbound and destroyed. 650 */ 651 652 if (!(man->flags & TTM_MEMTYPE_FLAG_FIXED)) 653 ghost_obj->ttm = NULL; 654 else 655 bo->ttm = NULL; 656 657 ttm_bo_unreserve(ghost_obj); 658 ttm_bo_unref(&ghost_obj); 659 } 660 661 *old_mem = *new_mem; 662 new_mem->mm_node = NULL; 663 664 return 0; 665}
|