1/* 2 * Copyright (C) 2008 Ben Skeggs. 3 * All Rights Reserved. 4 * 5 * Permission is hereby granted, free of charge, to any person obtaining 6 * a copy of this software and associated documentation files (the 7 * "Software"), to deal in the Software without restriction, including 8 * without limitation the rights to use, copy, modify, merge, publish, 9 * distribute, sublicense, and/or sell copies of the Software, and to 10 * permit persons to whom the Software is furnished to do so, subject to 11 * the following conditions: 12 * 13 * The above copyright notice and this permission notice (including the 14 * next paragraph) shall be included in all copies or substantial 15 * portions of the Software. 16 * 17 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 18 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 19 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. 20 * IN NO EVENT SHALL THE COPYRIGHT OWNER(S) AND/OR ITS SUPPLIERS BE 21 * LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION 22 * OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION 23 * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 24 * 25 */ 26 27#include <drm/drm_gem_ttm_helper.h> 28 29#include "nouveau_drv.h" 30#include "nouveau_dma.h" 31#include "nouveau_fence.h" 32#include "nouveau_abi16.h" 33 34#include "nouveau_ttm.h" 35#include "nouveau_gem.h" 36#include "nouveau_mem.h" 37#include "nouveau_vmm.h" 38 39#include <nvif/class.h> 40#include <nvif/push206e.h> 41 42static vm_fault_t nouveau_ttm_fault(struct vm_fault *vmf) 43{ 44 struct vm_area_struct *vma = vmf->vma; 45 struct ttm_buffer_object *bo = vma->vm_private_data; 46 pgprot_t prot; 47 vm_fault_t ret; 48 49 ret = ttm_bo_vm_reserve(bo, vmf); 50 if (ret) 51 return ret; 52 53 ret = nouveau_ttm_fault_reserve_notify(bo); 54 if (ret) 55 goto error_unlock; 56 57 nouveau_bo_del_io_reserve_lru(bo); 58 prot = vm_get_page_prot(vma->vm_flags); 59 ret = ttm_bo_vm_fault_reserved(vmf, prot, TTM_BO_VM_NUM_PREFAULT); 60 nouveau_bo_add_io_reserve_lru(bo); 61 if (ret == VM_FAULT_RETRY && !(vmf->flags & FAULT_FLAG_RETRY_NOWAIT)) 62 return ret; 63 64error_unlock: 65 dma_resv_unlock(bo->base.resv); 66 return ret; 67} 68 69static const struct vm_operations_struct nouveau_ttm_vm_ops = { 70 .fault = nouveau_ttm_fault, 71 .open = ttm_bo_vm_open, 72 .close = ttm_bo_vm_close, 73 .access = ttm_bo_vm_access 74}; 75 76void 77nouveau_gem_object_del(struct drm_gem_object *gem) 78{ 79 struct nouveau_bo *nvbo = nouveau_gem_object(gem); 80 struct nouveau_drm *drm = nouveau_bdev(nvbo->bo.bdev); 81 struct device *dev = drm->dev->dev; 82 int ret; 83 84 ret = pm_runtime_get_sync(dev); 85 if (WARN_ON(ret < 0 && ret != -EACCES)) { 86 pm_runtime_put_autosuspend(dev); 87 return; 88 } 89 90 if (gem->import_attach) 91 drm_prime_gem_destroy(gem, nvbo->bo.sg); 92 93 ttm_bo_put(&nvbo->bo); 94 95 pm_runtime_mark_last_busy(dev); 96 pm_runtime_put_autosuspend(dev); 97} 98 99int 100nouveau_gem_object_open(struct drm_gem_object *gem, struct drm_file *file_priv) 101{ 102 struct nouveau_cli *cli = nouveau_cli(file_priv); 103 struct nouveau_bo *nvbo = nouveau_gem_object(gem); 104 struct nouveau_drm *drm = nouveau_bdev(nvbo->bo.bdev); 105 struct device *dev = drm->dev->dev; 106 struct nouveau_uvmm *uvmm = nouveau_cli_uvmm(cli); 107 struct nouveau_vmm *vmm = nouveau_cli_vmm(cli); 108 struct nouveau_vma *vma; 109 int ret; 110 111 if (vmm->vmm.object.oclass < NVIF_CLASS_VMM_NV50) 112 return 0; 113 114 if (nvbo->no_share && uvmm && 115 drm_gpuvm_resv(&uvmm->base) != nvbo->bo.base.resv) 116 return -EPERM; 117 118 ret = ttm_bo_reserve(&nvbo->bo, false, false, NULL); 119 if (ret) 120 return ret; 121 122 ret = pm_runtime_get_sync(dev); 123 if (ret < 0 && ret != -EACCES) { 124 pm_runtime_put_autosuspend(dev); 125 goto out; 126 } 127 128 /* only create a VMA on binding */ 129 if (!nouveau_cli_uvmm(cli)) 130 ret = nouveau_vma_new(nvbo, vmm, &vma); 131 else 132 ret = 0; 133 pm_runtime_mark_last_busy(dev); 134 pm_runtime_put_autosuspend(dev); 135out: 136 ttm_bo_unreserve(&nvbo->bo); 137 return ret; 138} 139 140struct nouveau_gem_object_unmap { 141 struct nouveau_cli_work work; 142 struct nouveau_vma *vma; 143}; 144 145static void 146nouveau_gem_object_delete(struct nouveau_vma *vma) 147{ 148 nouveau_fence_unref(&vma->fence); 149 nouveau_vma_del(&vma); 150} 151 152static void 153nouveau_gem_object_delete_work(struct nouveau_cli_work *w) 154{ 155 struct nouveau_gem_object_unmap *work = 156 container_of(w, typeof(*work), work); 157 nouveau_gem_object_delete(work->vma); 158 kfree(work); 159} 160 161static void 162nouveau_gem_object_unmap(struct nouveau_bo *nvbo, struct nouveau_vma *vma) 163{ 164 struct dma_fence *fence = vma->fence ? &vma->fence->base : NULL; 165 struct nouveau_gem_object_unmap *work; 166 167 list_del_init(&vma->head); 168 169 if (!fence) { 170 nouveau_gem_object_delete(vma); 171 return; 172 } 173 174 if (!(work = kmalloc(sizeof(*work), GFP_KERNEL))) { 175 WARN_ON(dma_fence_wait_timeout(fence, false, 2 * HZ) <= 0); 176 nouveau_gem_object_delete(vma); 177 return; 178 } 179 180 work->work.func = nouveau_gem_object_delete_work; 181 work->vma = vma; 182 nouveau_cli_work_queue(vma->vmm->cli, fence, &work->work); 183} 184 185void 186nouveau_gem_object_close(struct drm_gem_object *gem, struct drm_file *file_priv) 187{ 188 struct nouveau_cli *cli = nouveau_cli(file_priv); 189 struct nouveau_bo *nvbo = nouveau_gem_object(gem); 190 struct nouveau_drm *drm = nouveau_bdev(nvbo->bo.bdev); 191 struct device *dev = drm->dev->dev; 192 struct nouveau_vmm *vmm = nouveau_cli_vmm(cli); 193 struct nouveau_vma *vma; 194 int ret; 195 196 if (vmm->vmm.object.oclass < NVIF_CLASS_VMM_NV50) 197 return; 198 199 if (nouveau_cli_uvmm(cli)) 200 return; 201 202 ret = ttm_bo_reserve(&nvbo->bo, false, false, NULL); 203 if (ret) 204 return; 205 206 vma = nouveau_vma_find(nvbo, vmm); 207 if (vma) { 208 if (--vma->refs == 0) { 209 ret = pm_runtime_get_sync(dev); 210 if (!WARN_ON(ret < 0 && ret != -EACCES)) { 211 nouveau_gem_object_unmap(nvbo, vma); 212 pm_runtime_mark_last_busy(dev); 213 } 214 pm_runtime_put_autosuspend(dev); 215 } 216 } 217 ttm_bo_unreserve(&nvbo->bo); 218} 219 220const struct drm_gem_object_funcs nouveau_gem_object_funcs = { 221 .free = nouveau_gem_object_del, 222 .open = nouveau_gem_object_open, 223 .close = nouveau_gem_object_close, 224 .export = nouveau_gem_prime_export, 225 .pin = nouveau_gem_prime_pin, 226 .unpin = nouveau_gem_prime_unpin, 227 .get_sg_table = nouveau_gem_prime_get_sg_table, 228 .vmap = drm_gem_ttm_vmap, 229 .vunmap = drm_gem_ttm_vunmap, 230 .mmap = drm_gem_ttm_mmap, 231 .vm_ops = &nouveau_ttm_vm_ops, 232}; 233 234int 235nouveau_gem_new(struct nouveau_cli *cli, u64 size, int align, uint32_t domain, 236 uint32_t tile_mode, uint32_t tile_flags, 237 struct nouveau_bo **pnvbo) 238{ 239 struct nouveau_drm *drm = cli->drm; 240 struct nouveau_uvmm *uvmm = nouveau_cli_uvmm(cli); 241 struct dma_resv *resv = NULL; 242 struct nouveau_bo *nvbo; 243 int ret; 244 245 if (domain & NOUVEAU_GEM_DOMAIN_NO_SHARE) { 246 if (unlikely(!uvmm)) 247 return -EINVAL; 248 249 resv = drm_gpuvm_resv(&uvmm->base); 250 } 251 252 if (!(domain & (NOUVEAU_GEM_DOMAIN_VRAM | NOUVEAU_GEM_DOMAIN_GART))) 253 domain |= NOUVEAU_GEM_DOMAIN_CPU; 254 255 nvbo = nouveau_bo_alloc(cli, &size, &align, domain, tile_mode, 256 tile_flags, false); 257 if (IS_ERR(nvbo)) 258 return PTR_ERR(nvbo); 259 260 nvbo->bo.base.funcs = &nouveau_gem_object_funcs; 261 nvbo->no_share = domain & NOUVEAU_GEM_DOMAIN_NO_SHARE; 262 263 /* Initialize the embedded gem-object. We return a single gem-reference 264 * to the caller, instead of a normal nouveau_bo ttm reference. */ 265 ret = drm_gem_object_init(drm->dev, &nvbo->bo.base, size); 266 if (ret) { 267 drm_gem_object_release(&nvbo->bo.base); 268 kfree(nvbo); 269 return ret; 270 } 271 272 if (resv) 273 dma_resv_lock(resv, NULL); 274 275 ret = nouveau_bo_init(nvbo, size, align, domain, NULL, resv); 276 277 if (resv) 278 dma_resv_unlock(resv); 279 280 if (ret) 281 return ret; 282 283 /* we restrict allowed domains on nv50+ to only the types 284 * that were requested at creation time. not possibly on 285 * earlier chips without busting the ABI. 286 */ 287 nvbo->valid_domains = NOUVEAU_GEM_DOMAIN_VRAM | 288 NOUVEAU_GEM_DOMAIN_GART; 289 if (drm->client.device.info.family >= NV_DEVICE_INFO_V0_TESLA) 290 nvbo->valid_domains &= domain; 291 292 if (nvbo->no_share) { 293 nvbo->r_obj = drm_gpuvm_resv_obj(&uvmm->base); 294 drm_gem_object_get(nvbo->r_obj); 295 } 296 297 *pnvbo = nvbo; 298 return 0; 299} 300 301static int 302nouveau_gem_info(struct drm_file *file_priv, struct drm_gem_object *gem, 303 struct drm_nouveau_gem_info *rep) 304{ 305 struct nouveau_cli *cli = nouveau_cli(file_priv); 306 struct nouveau_bo *nvbo = nouveau_gem_object(gem); 307 struct nouveau_vmm *vmm = nouveau_cli_vmm(cli); 308 struct nouveau_vma *vma; 309 310 if (is_power_of_2(nvbo->valid_domains)) 311 rep->domain = nvbo->valid_domains; 312 else if (nvbo->bo.resource->mem_type == TTM_PL_TT) 313 rep->domain = NOUVEAU_GEM_DOMAIN_GART; 314 else 315 rep->domain = NOUVEAU_GEM_DOMAIN_VRAM; 316 rep->offset = nvbo->offset; 317 if (vmm->vmm.object.oclass >= NVIF_CLASS_VMM_NV50 && 318 !nouveau_cli_uvmm(cli)) { 319 vma = nouveau_vma_find(nvbo, vmm); 320 if (!vma) 321 return -EINVAL; 322 323 rep->offset = vma->addr; 324 } else 325 rep->offset = 0; 326 327 rep->size = nvbo->bo.base.size; 328 rep->map_handle = drm_vma_node_offset_addr(&nvbo->bo.base.vma_node); 329 rep->tile_mode = nvbo->mode; 330 rep->tile_flags = nvbo->contig ? 0 : NOUVEAU_GEM_TILE_NONCONTIG; 331 if (cli->device.info.family >= NV_DEVICE_INFO_V0_FERMI) 332 rep->tile_flags |= nvbo->kind << 8; 333 else 334 if (cli->device.info.family >= NV_DEVICE_INFO_V0_TESLA) 335 rep->tile_flags |= nvbo->kind << 8 | nvbo->comp << 16; 336 else 337 rep->tile_flags |= nvbo->zeta; 338 return 0; 339} 340 341int 342nouveau_gem_ioctl_new(struct drm_device *dev, void *data, 343 struct drm_file *file_priv) 344{ 345 struct nouveau_cli *cli = nouveau_cli(file_priv); 346 struct drm_nouveau_gem_new *req = data; 347 struct nouveau_bo *nvbo = NULL; 348 int ret = 0; 349 350 /* If uvmm wasn't initialized until now disable it completely to prevent 351 * userspace from mixing up UAPIs. 352 */ 353 nouveau_cli_disable_uvmm_noinit(cli); 354 355 ret = nouveau_gem_new(cli, req->info.size, req->align, 356 req->info.domain, req->info.tile_mode, 357 req->info.tile_flags, &nvbo); 358 if (ret) 359 return ret; 360 361 ret = drm_gem_handle_create(file_priv, &nvbo->bo.base, 362 &req->info.handle); 363 if (ret == 0) { 364 ret = nouveau_gem_info(file_priv, &nvbo->bo.base, &req->info); 365 if (ret) 366 drm_gem_handle_delete(file_priv, req->info.handle); 367 } 368 369 /* drop reference from allocate - handle holds it now */ 370 drm_gem_object_put(&nvbo->bo.base); 371 return ret; 372} 373 374static int 375nouveau_gem_set_domain(struct drm_gem_object *gem, uint32_t read_domains, 376 uint32_t write_domains, uint32_t valid_domains) 377{ 378 struct nouveau_bo *nvbo = nouveau_gem_object(gem); 379 struct ttm_buffer_object *bo = &nvbo->bo; 380 uint32_t domains = valid_domains & nvbo->valid_domains & 381 (write_domains ? write_domains : read_domains); 382 uint32_t pref_domains = 0; 383 384 if (!domains) 385 return -EINVAL; 386 387 valid_domains &= ~(NOUVEAU_GEM_DOMAIN_VRAM | NOUVEAU_GEM_DOMAIN_GART); 388 389 if ((domains & NOUVEAU_GEM_DOMAIN_VRAM) && 390 bo->resource->mem_type == TTM_PL_VRAM) 391 pref_domains |= NOUVEAU_GEM_DOMAIN_VRAM; 392 393 else if ((domains & NOUVEAU_GEM_DOMAIN_GART) && 394 bo->resource->mem_type == TTM_PL_TT) 395 pref_domains |= NOUVEAU_GEM_DOMAIN_GART; 396 397 else if (domains & NOUVEAU_GEM_DOMAIN_VRAM) 398 pref_domains |= NOUVEAU_GEM_DOMAIN_VRAM; 399 400 else 401 pref_domains |= NOUVEAU_GEM_DOMAIN_GART; 402 403 nouveau_bo_placement_set(nvbo, pref_domains, valid_domains); 404 405 return 0; 406} 407 408struct validate_op { 409 struct list_head list; 410 struct ww_acquire_ctx ticket; 411}; 412 413static void 414validate_fini_no_ticket(struct validate_op *op, struct nouveau_channel *chan, 415 struct nouveau_fence *fence, 416 struct drm_nouveau_gem_pushbuf_bo *pbbo) 417{ 418 struct nouveau_bo *nvbo; 419 struct drm_nouveau_gem_pushbuf_bo *b; 420 421 while (!list_empty(&op->list)) { 422 nvbo = list_entry(op->list.next, struct nouveau_bo, entry); 423 b = &pbbo[nvbo->pbbo_index]; 424 425 if (likely(fence)) { 426 nouveau_bo_fence(nvbo, fence, !!b->write_domains); 427 428 if (chan->vmm->vmm.object.oclass >= NVIF_CLASS_VMM_NV50) { 429 struct nouveau_vma *vma = 430 (void *)(unsigned long)b->user_priv; 431 nouveau_fence_unref(&vma->fence); 432 dma_fence_get(&fence->base); 433 vma->fence = fence; 434 } 435 } 436 437 if (unlikely(nvbo->validate_mapped)) { 438 ttm_bo_kunmap(&nvbo->kmap); 439 nvbo->validate_mapped = false; 440 } 441 442 list_del(&nvbo->entry); 443 nvbo->reserved_by = NULL; 444 ttm_bo_unreserve(&nvbo->bo); 445 drm_gem_object_put(&nvbo->bo.base); 446 } 447} 448 449static void 450validate_fini(struct validate_op *op, struct nouveau_channel *chan, 451 struct nouveau_fence *fence, 452 struct drm_nouveau_gem_pushbuf_bo *pbbo) 453{ 454 validate_fini_no_ticket(op, chan, fence, pbbo); 455 ww_acquire_fini(&op->ticket); 456} 457 458static int 459validate_init(struct nouveau_channel *chan, struct drm_file *file_priv, 460 struct drm_nouveau_gem_pushbuf_bo *pbbo, 461 int nr_buffers, struct validate_op *op) 462{ 463 struct nouveau_cli *cli = nouveau_cli(file_priv); 464 int trycnt = 0; 465 int ret = -EINVAL, i; 466 struct nouveau_bo *res_bo = NULL; 467 LIST_HEAD(gart_list); 468 LIST_HEAD(vram_list); 469 LIST_HEAD(both_list); 470 471 ww_acquire_init(&op->ticket, &reservation_ww_class); 472retry: 473 if (++trycnt > 100000) { 474 NV_PRINTK(err, cli, "%s failed and gave up.\n", __func__); 475 return -EINVAL; 476 } 477 478 for (i = 0; i < nr_buffers; i++) { 479 struct drm_nouveau_gem_pushbuf_bo *b = &pbbo[i]; 480 struct drm_gem_object *gem; 481 struct nouveau_bo *nvbo; 482 483 gem = drm_gem_object_lookup(file_priv, b->handle); 484 if (!gem) { 485 NV_PRINTK(err, cli, "Unknown handle 0x%08x\n", b->handle); 486 ret = -ENOENT; 487 break; 488 } 489 nvbo = nouveau_gem_object(gem); 490 if (nvbo == res_bo) { 491 res_bo = NULL; 492 drm_gem_object_put(gem); 493 continue; 494 } 495 496 if (nvbo->reserved_by && nvbo->reserved_by == file_priv) { 497 NV_PRINTK(err, cli, "multiple instances of buffer %d on " 498 "validation list\n", b->handle); 499 drm_gem_object_put(gem); 500 ret = -EINVAL; 501 break; 502 } 503 504 ret = ttm_bo_reserve(&nvbo->bo, true, false, &op->ticket); 505 if (ret) { 506 list_splice_tail_init(&vram_list, &op->list); 507 list_splice_tail_init(&gart_list, &op->list); 508 list_splice_tail_init(&both_list, &op->list); 509 validate_fini_no_ticket(op, chan, NULL, NULL); 510 if (unlikely(ret == -EDEADLK)) { 511 ret = ttm_bo_reserve_slowpath(&nvbo->bo, true, 512 &op->ticket); 513 if (!ret) 514 res_bo = nvbo; 515 } 516 if (unlikely(ret)) { 517 if (ret != -ERESTARTSYS) 518 NV_PRINTK(err, cli, "fail reserve\n"); 519 break; 520 } 521 } 522 523 if (chan->vmm->vmm.object.oclass >= NVIF_CLASS_VMM_NV50) { 524 struct nouveau_vmm *vmm = chan->vmm; 525 struct nouveau_vma *vma = nouveau_vma_find(nvbo, vmm); 526 if (!vma) { 527 NV_PRINTK(err, cli, "vma not found!\n"); 528 ret = -EINVAL; 529 break; 530 } 531 532 b->user_priv = (uint64_t)(unsigned long)vma; 533 } else { 534 b->user_priv = (uint64_t)(unsigned long)nvbo; 535 } 536 537 nvbo->reserved_by = file_priv; 538 nvbo->pbbo_index = i; 539 if ((b->valid_domains & NOUVEAU_GEM_DOMAIN_VRAM) && 540 (b->valid_domains & NOUVEAU_GEM_DOMAIN_GART)) 541 list_add_tail(&nvbo->entry, &both_list); 542 else 543 if (b->valid_domains & NOUVEAU_GEM_DOMAIN_VRAM) 544 list_add_tail(&nvbo->entry, &vram_list); 545 else 546 if (b->valid_domains & NOUVEAU_GEM_DOMAIN_GART) 547 list_add_tail(&nvbo->entry, &gart_list); 548 else { 549 NV_PRINTK(err, cli, "invalid valid domains: 0x%08x\n", 550 b->valid_domains); 551 list_add_tail(&nvbo->entry, &both_list); 552 ret = -EINVAL; 553 break; 554 } 555 if (nvbo == res_bo) 556 goto retry; 557 } 558 559 ww_acquire_done(&op->ticket); 560 list_splice_tail(&vram_list, &op->list); 561 list_splice_tail(&gart_list, &op->list); 562 list_splice_tail(&both_list, &op->list); 563 if (ret) 564 validate_fini(op, chan, NULL, NULL); 565 return ret; 566 567} 568 569static int 570validate_list(struct nouveau_channel *chan, struct nouveau_cli *cli, 571 struct list_head *list, struct drm_nouveau_gem_pushbuf_bo *pbbo) 572{ 573 struct nouveau_drm *drm = chan->drm; 574 struct nouveau_bo *nvbo; 575 int ret, relocs = 0; 576 577 list_for_each_entry(nvbo, list, entry) { 578 struct drm_nouveau_gem_pushbuf_bo *b = &pbbo[nvbo->pbbo_index]; 579 580 ret = nouveau_gem_set_domain(&nvbo->bo.base, b->read_domains, 581 b->write_domains, 582 b->valid_domains); 583 if (unlikely(ret)) { 584 NV_PRINTK(err, cli, "fail set_domain\n"); 585 return ret; 586 } 587 588 ret = nouveau_bo_validate(nvbo, true, false); 589 if (unlikely(ret)) { 590 if (ret != -ERESTARTSYS) 591 NV_PRINTK(err, cli, "fail ttm_validate\n"); 592 return ret; 593 } 594 595 ret = nouveau_fence_sync(nvbo, chan, !!b->write_domains, true); 596 if (unlikely(ret)) { 597 if (ret != -ERESTARTSYS) 598 NV_PRINTK(err, cli, "fail post-validate sync\n"); 599 return ret; 600 } 601 602 if (drm->client.device.info.family < NV_DEVICE_INFO_V0_TESLA) { 603 if (nvbo->offset == b->presumed.offset && 604 ((nvbo->bo.resource->mem_type == TTM_PL_VRAM && 605 b->presumed.domain & NOUVEAU_GEM_DOMAIN_VRAM) || 606 (nvbo->bo.resource->mem_type == TTM_PL_TT && 607 b->presumed.domain & NOUVEAU_GEM_DOMAIN_GART))) 608 continue; 609 610 if (nvbo->bo.resource->mem_type == TTM_PL_TT) 611 b->presumed.domain = NOUVEAU_GEM_DOMAIN_GART; 612 else 613 b->presumed.domain = NOUVEAU_GEM_DOMAIN_VRAM; 614 b->presumed.offset = nvbo->offset; 615 b->presumed.valid = 0; 616 relocs++; 617 } 618 } 619 620 return relocs; 621} 622 623static int 624nouveau_gem_pushbuf_validate(struct nouveau_channel *chan, 625 struct drm_file *file_priv, 626 struct drm_nouveau_gem_pushbuf_bo *pbbo, 627 int nr_buffers, 628 struct validate_op *op, bool *apply_relocs) 629{ 630 struct nouveau_cli *cli = nouveau_cli(file_priv); 631 int ret; 632 633 INIT_LIST_HEAD(&op->list); 634 635 if (nr_buffers == 0) 636 return 0; 637 638 ret = validate_init(chan, file_priv, pbbo, nr_buffers, op); 639 if (unlikely(ret)) { 640 if (ret != -ERESTARTSYS) 641 NV_PRINTK(err, cli, "validate_init\n"); 642 return ret; 643 } 644 645 ret = validate_list(chan, cli, &op->list, pbbo); 646 if (unlikely(ret < 0)) { 647 if (ret != -ERESTARTSYS) 648 NV_PRINTK(err, cli, "validating bo list\n"); 649 validate_fini(op, chan, NULL, NULL); 650 return ret; 651 } else if (ret > 0) { 652 *apply_relocs = true; 653 } 654 655 return 0; 656} 657 658static int 659nouveau_gem_pushbuf_reloc_apply(struct nouveau_cli *cli, 660 struct drm_nouveau_gem_pushbuf *req, 661 struct drm_nouveau_gem_pushbuf_reloc *reloc, 662 struct drm_nouveau_gem_pushbuf_bo *bo) 663{ 664 int ret = 0; 665 unsigned i; 666 667 for (i = 0; i < req->nr_relocs; i++) { 668 struct drm_nouveau_gem_pushbuf_reloc *r = &reloc[i]; 669 struct drm_nouveau_gem_pushbuf_bo *b; 670 struct nouveau_bo *nvbo; 671 uint32_t data; 672 long lret; 673 674 if (unlikely(r->bo_index >= req->nr_buffers)) { 675 NV_PRINTK(err, cli, "reloc bo index invalid\n"); 676 ret = -EINVAL; 677 break; 678 } 679 680 b = &bo[r->bo_index]; 681 if (b->presumed.valid) 682 continue; 683 684 if (unlikely(r->reloc_bo_index >= req->nr_buffers)) { 685 NV_PRINTK(err, cli, "reloc container bo index invalid\n"); 686 ret = -EINVAL; 687 break; 688 } 689 nvbo = (void *)(unsigned long)bo[r->reloc_bo_index].user_priv; 690 691 if (unlikely(r->reloc_bo_offset + 4 > 692 nvbo->bo.base.size)) { 693 NV_PRINTK(err, cli, "reloc outside of bo\n"); 694 ret = -EINVAL; 695 break; 696 } 697 698 if (!nvbo->kmap.virtual) { 699 ret = ttm_bo_kmap(&nvbo->bo, 0, PFN_UP(nvbo->bo.base.size), 700 &nvbo->kmap); 701 if (ret) { 702 NV_PRINTK(err, cli, "failed kmap for reloc\n"); 703 break; 704 } 705 nvbo->validate_mapped = true; 706 } 707 708 if (r->flags & NOUVEAU_GEM_RELOC_LOW) 709 data = b->presumed.offset + r->data; 710 else 711 if (r->flags & NOUVEAU_GEM_RELOC_HIGH) 712 data = (b->presumed.offset + r->data) >> 32; 713 else 714 data = r->data; 715 716 if (r->flags & NOUVEAU_GEM_RELOC_OR) { 717 if (b->presumed.domain == NOUVEAU_GEM_DOMAIN_GART) 718 data |= r->tor; 719 else 720 data |= r->vor; 721 } 722 723 lret = dma_resv_wait_timeout(nvbo->bo.base.resv, 724 DMA_RESV_USAGE_BOOKKEEP, 725 false, 15 * HZ); 726 if (!lret) 727 ret = -EBUSY; 728 else if (lret > 0) 729 ret = 0; 730 else 731 ret = lret; 732 733 if (ret) { 734 NV_PRINTK(err, cli, "reloc wait_idle failed: %d\n", 735 ret); 736 break; 737 } 738 739 nouveau_bo_wr32(nvbo, r->reloc_bo_offset >> 2, data); 740 } 741 742 return ret; 743} 744 745int 746nouveau_gem_ioctl_pushbuf(struct drm_device *dev, void *data, 747 struct drm_file *file_priv) 748{ 749 struct nouveau_abi16 *abi16 = nouveau_abi16_get(file_priv); 750 struct nouveau_cli *cli = nouveau_cli(file_priv); 751 struct nouveau_abi16_chan *temp; 752 struct nouveau_drm *drm = nouveau_drm(dev); 753 struct drm_nouveau_gem_pushbuf *req = data; 754 struct drm_nouveau_gem_pushbuf_push *push; 755 struct drm_nouveau_gem_pushbuf_reloc *reloc = NULL; 756 struct drm_nouveau_gem_pushbuf_bo *bo; 757 struct nouveau_channel *chan = NULL; 758 struct validate_op op; 759 struct nouveau_fence *fence = NULL; 760 int i, j, ret = 0; 761 bool do_reloc = false, sync = false; 762 763 if (unlikely(!abi16)) 764 return -ENOMEM; 765 766 if (unlikely(nouveau_cli_uvmm(cli))) 767 return nouveau_abi16_put(abi16, -ENOSYS); 768 769 list_for_each_entry(temp, &abi16->channels, head) { 770 if (temp->chan->chid == req->channel) { 771 chan = temp->chan; 772 break; 773 } 774 } 775 776 if (!chan) 777 return nouveau_abi16_put(abi16, -ENOENT); 778 if (unlikely(atomic_read(&chan->killed))) 779 return nouveau_abi16_put(abi16, -ENODEV); 780 781 sync = req->vram_available & NOUVEAU_GEM_PUSHBUF_SYNC; 782 783 req->vram_available = drm->gem.vram_available; 784 req->gart_available = drm->gem.gart_available; 785 if (unlikely(req->nr_push == 0)) 786 goto out_next; 787 788 if (unlikely(req->nr_push > NOUVEAU_GEM_MAX_PUSH)) { 789 NV_PRINTK(err, cli, "pushbuf push count exceeds limit: %d max %d\n", 790 req->nr_push, NOUVEAU_GEM_MAX_PUSH); 791 return nouveau_abi16_put(abi16, -EINVAL); 792 } 793 794 if (unlikely(req->nr_buffers > NOUVEAU_GEM_MAX_BUFFERS)) { 795 NV_PRINTK(err, cli, "pushbuf bo count exceeds limit: %d max %d\n", 796 req->nr_buffers, NOUVEAU_GEM_MAX_BUFFERS); 797 return nouveau_abi16_put(abi16, -EINVAL); 798 } 799 800 if (unlikely(req->nr_relocs > NOUVEAU_GEM_MAX_RELOCS)) { 801 NV_PRINTK(err, cli, "pushbuf reloc count exceeds limit: %d max %d\n", 802 req->nr_relocs, NOUVEAU_GEM_MAX_RELOCS); 803 return nouveau_abi16_put(abi16, -EINVAL); 804 } 805 806 push = u_memcpya(req->push, req->nr_push, sizeof(*push)); 807 if (IS_ERR(push)) 808 return nouveau_abi16_put(abi16, PTR_ERR(push)); 809 810 bo = u_memcpya(req->buffers, req->nr_buffers, sizeof(*bo)); 811 if (IS_ERR(bo)) { 812 u_free(push); 813 return nouveau_abi16_put(abi16, PTR_ERR(bo)); 814 } 815 816 /* Ensure all push buffers are on validate list */ 817 for (i = 0; i < req->nr_push; i++) { 818 if (push[i].bo_index >= req->nr_buffers) { 819 NV_PRINTK(err, cli, "push %d buffer not in list\n", i); 820 ret = -EINVAL; 821 goto out_prevalid; 822 } 823 } 824 825 /* Validate buffer list */ 826revalidate: 827 ret = nouveau_gem_pushbuf_validate(chan, file_priv, bo, 828 req->nr_buffers, &op, &do_reloc); 829 if (ret) { 830 if (ret != -ERESTARTSYS) 831 NV_PRINTK(err, cli, "validate: %d\n", ret); 832 goto out_prevalid; 833 } 834 835 /* Apply any relocations that are required */ 836 if (do_reloc) { 837 if (!reloc) { 838 validate_fini(&op, chan, NULL, bo); 839 reloc = u_memcpya(req->relocs, req->nr_relocs, sizeof(*reloc)); 840 if (IS_ERR(reloc)) { 841 ret = PTR_ERR(reloc); 842 goto out_prevalid; 843 } 844 845 goto revalidate; 846 } 847 848 ret = nouveau_gem_pushbuf_reloc_apply(cli, req, reloc, bo); 849 if (ret) { 850 NV_PRINTK(err, cli, "reloc apply: %d\n", ret); 851 goto out; 852 } 853 } 854 855 if (chan->dma.ib_max) { 856 ret = nouveau_dma_wait(chan, req->nr_push + 1, 16); 857 if (ret) { 858 NV_PRINTK(err, cli, "nv50cal_space: %d\n", ret); 859 goto out; 860 } 861 862 for (i = 0; i < req->nr_push; i++) { 863 struct nouveau_vma *vma = (void *)(unsigned long) 864 bo[push[i].bo_index].user_priv; 865 u64 addr = vma->addr + push[i].offset; 866 u32 length = push[i].length & ~NOUVEAU_GEM_PUSHBUF_NO_PREFETCH; 867 bool no_prefetch = push[i].length & NOUVEAU_GEM_PUSHBUF_NO_PREFETCH; 868 869 nv50_dma_push(chan, addr, length, no_prefetch); 870 } 871 } else 872 if (drm->client.device.info.chipset >= 0x25) { 873 ret = PUSH_WAIT(chan->chan.push, req->nr_push * 2); 874 if (ret) { 875 NV_PRINTK(err, cli, "cal_space: %d\n", ret); 876 goto out; 877 } 878 879 for (i = 0; i < req->nr_push; i++) { 880 struct nouveau_bo *nvbo = (void *)(unsigned long) 881 bo[push[i].bo_index].user_priv; 882 883 PUSH_CALL(chan->chan.push, nvbo->offset + push[i].offset); 884 PUSH_DATA(chan->chan.push, 0); 885 } 886 } else { 887 ret = PUSH_WAIT(chan->chan.push, req->nr_push * (2 + NOUVEAU_DMA_SKIPS)); 888 if (ret) { 889 NV_PRINTK(err, cli, "jmp_space: %d\n", ret); 890 goto out; 891 } 892 893 for (i = 0; i < req->nr_push; i++) { 894 struct nouveau_bo *nvbo = (void *)(unsigned long) 895 bo[push[i].bo_index].user_priv; 896 uint32_t cmd; 897 898 cmd = chan->push.addr + ((chan->dma.cur + 2) << 2); 899 cmd |= 0x20000000; 900 if (unlikely(cmd != req->suffix0)) { 901 if (!nvbo->kmap.virtual) { 902 ret = ttm_bo_kmap(&nvbo->bo, 0, 903 PFN_UP(nvbo->bo.base.size), 904 &nvbo->kmap); 905 if (ret) { 906 WIND_RING(chan); 907 goto out; 908 } 909 nvbo->validate_mapped = true; 910 } 911 912 nouveau_bo_wr32(nvbo, (push[i].offset + 913 push[i].length - 8) / 4, cmd); 914 } 915 916 PUSH_JUMP(chan->chan.push, nvbo->offset + push[i].offset); 917 PUSH_DATA(chan->chan.push, 0); 918 for (j = 0; j < NOUVEAU_DMA_SKIPS; j++) 919 PUSH_DATA(chan->chan.push, 0); 920 } 921 } 922 923 ret = nouveau_fence_new(&fence, chan); 924 if (ret) { 925 NV_PRINTK(err, cli, "error fencing pushbuf: %d\n", ret); 926 WIND_RING(chan); 927 goto out; 928 } 929 930 if (sync) { 931 if (!(ret = nouveau_fence_wait(fence, false, false))) { 932 if ((ret = dma_fence_get_status(&fence->base)) == 1) 933 ret = 0; 934 } 935 } 936 937out: 938 validate_fini(&op, chan, fence, bo); 939 nouveau_fence_unref(&fence); 940 941 if (do_reloc) { 942 struct drm_nouveau_gem_pushbuf_bo __user *upbbo = 943 u64_to_user_ptr(req->buffers); 944 945 for (i = 0; i < req->nr_buffers; i++) { 946 if (bo[i].presumed.valid) 947 continue; 948 949 if (copy_to_user(&upbbo[i].presumed, &bo[i].presumed, 950 sizeof(bo[i].presumed))) { 951 ret = -EFAULT; 952 break; 953 } 954 } 955 } 956out_prevalid: 957 if (!IS_ERR(reloc)) 958 u_free(reloc); 959 u_free(bo); 960 u_free(push); 961 962out_next: 963 if (chan->dma.ib_max) { 964 req->suffix0 = 0x00000000; 965 req->suffix1 = 0x00000000; 966 } else 967 if (drm->client.device.info.chipset >= 0x25) { 968 req->suffix0 = 0x00020000; 969 req->suffix1 = 0x00000000; 970 } else { 971 req->suffix0 = 0x20000000 | 972 (chan->push.addr + ((chan->dma.cur + 2) << 2)); 973 req->suffix1 = 0x00000000; 974 } 975 976 return nouveau_abi16_put(abi16, ret); 977} 978 979int 980nouveau_gem_ioctl_cpu_prep(struct drm_device *dev, void *data, 981 struct drm_file *file_priv) 982{ 983 struct drm_nouveau_gem_cpu_prep *req = data; 984 struct drm_gem_object *gem; 985 struct nouveau_bo *nvbo; 986 bool no_wait = !!(req->flags & NOUVEAU_GEM_CPU_PREP_NOWAIT); 987 bool write = !!(req->flags & NOUVEAU_GEM_CPU_PREP_WRITE); 988 long lret; 989 int ret; 990 991 gem = drm_gem_object_lookup(file_priv, req->handle); 992 if (!gem) 993 return -ENOENT; 994 nvbo = nouveau_gem_object(gem); 995 996 lret = dma_resv_wait_timeout(nvbo->bo.base.resv, 997 dma_resv_usage_rw(write), true, 998 no_wait ? 0 : 30 * HZ); 999 if (!lret) 1000 ret = -EBUSY; 1001 else if (lret > 0) 1002 ret = 0; 1003 else 1004 ret = lret; 1005 1006 nouveau_bo_sync_for_cpu(nvbo); 1007 drm_gem_object_put(gem); 1008 1009 return ret; 1010} 1011 1012int 1013nouveau_gem_ioctl_cpu_fini(struct drm_device *dev, void *data, 1014 struct drm_file *file_priv) 1015{ 1016 struct drm_nouveau_gem_cpu_fini *req = data; 1017 struct drm_gem_object *gem; 1018 struct nouveau_bo *nvbo; 1019 1020 gem = drm_gem_object_lookup(file_priv, req->handle); 1021 if (!gem) 1022 return -ENOENT; 1023 nvbo = nouveau_gem_object(gem); 1024 1025 nouveau_bo_sync_for_device(nvbo); 1026 drm_gem_object_put(gem); 1027 return 0; 1028} 1029 1030int 1031nouveau_gem_ioctl_info(struct drm_device *dev, void *data, 1032 struct drm_file *file_priv) 1033{ 1034 struct drm_nouveau_gem_info *req = data; 1035 struct drm_gem_object *gem; 1036 int ret; 1037 1038 gem = drm_gem_object_lookup(file_priv, req->handle); 1039 if (!gem) 1040 return -ENOENT; 1041 1042 ret = nouveau_gem_info(file_priv, gem, req); 1043 drm_gem_object_put(gem); 1044 return ret; 1045} 1046 1047