1/* 2 * Copyright (C) 2008 Ben Skeggs. 3 * All Rights Reserved. 4 * 5 * Permission is hereby granted, free of charge, to any person obtaining 6 * a copy of this software and associated documentation files (the 7 * "Software"), to deal in the Software without restriction, including 8 * without limitation the rights to use, copy, modify, merge, publish, 9 * distribute, sublicense, and/or sell copies of the Software, and to 10 * permit persons to whom the Software is furnished to do so, subject to 11 * the following conditions: 12 * 13 * The above copyright notice and this permission notice (including the 14 * next paragraph) shall be included in all copies or substantial 15 * portions of the Software. 16 * 17 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 18 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 19 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. 20 * IN NO EVENT SHALL THE COPYRIGHT OWNER(S) AND/OR ITS SUPPLIERS BE 21 * LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION 22 * OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION 23 * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 24 * 25 */ 26#include "drmP.h" 27#include "drm.h" 28 29#include "nouveau_drv.h" 30#include "nouveau_drm.h" 31#include "nouveau_dma.h" 32 33#define nouveau_gem_pushbuf_sync(chan) 0 34 35int 36nouveau_gem_object_new(struct drm_gem_object *gem) 37{ 38 return 0; 39} 40 41void 42nouveau_gem_object_del(struct drm_gem_object *gem) 43{ 44 struct nouveau_bo *nvbo = gem->driver_private; 45 struct ttm_buffer_object *bo = &nvbo->bo; 46 47 if (!nvbo) 48 return; 49 nvbo->gem = NULL; 50 51 if (unlikely(nvbo->cpu_filp)) 52 ttm_bo_synccpu_write_release(bo); 53 54 if (unlikely(nvbo->pin_refcnt)) { 55 nvbo->pin_refcnt = 1; 56 nouveau_bo_unpin(nvbo); 57 } 58 59 ttm_bo_unref(&bo); 60 61 drm_gem_object_release(gem); 62 kfree(gem); 63} 64 65int 66nouveau_gem_new(struct drm_device *dev, struct nouveau_channel *chan, 67 int size, int align, uint32_t flags, uint32_t tile_mode, 68 uint32_t tile_flags, bool no_vm, bool mappable, 69 struct nouveau_bo **pnvbo) 70{ 71 struct nouveau_bo *nvbo; 72 int ret; 73 74 ret = nouveau_bo_new(dev, chan, size, align, flags, tile_mode, 75 tile_flags, no_vm, mappable, pnvbo); 76 if (ret) 77 return ret; 78 nvbo = *pnvbo; 79 80 nvbo->gem = drm_gem_object_alloc(dev, nvbo->bo.mem.size); 81 if (!nvbo->gem) { 82 nouveau_bo_ref(NULL, pnvbo); 83 return -ENOMEM; 84 } 85 86 nvbo->bo.persistant_swap_storage = nvbo->gem->filp; 87 nvbo->gem->driver_private = nvbo; 88 return 0; 89} 90 91static int 92nouveau_gem_info(struct drm_gem_object *gem, struct drm_nouveau_gem_info *rep) 93{ 94 struct nouveau_bo *nvbo = nouveau_gem_object(gem); 95 96 if (nvbo->bo.mem.mem_type == TTM_PL_TT) 97 rep->domain = NOUVEAU_GEM_DOMAIN_GART; 98 else 99 rep->domain = NOUVEAU_GEM_DOMAIN_VRAM; 100 101 rep->size = nvbo->bo.mem.num_pages << PAGE_SHIFT; 102 rep->offset = nvbo->bo.offset; 103 rep->map_handle = nvbo->mappable ? nvbo->bo.addr_space_offset : 0; 104 rep->tile_mode = nvbo->tile_mode; 105 rep->tile_flags = nvbo->tile_flags; 106 return 0; 107} 108 109static bool 110nouveau_gem_tile_flags_valid(struct drm_device *dev, uint32_t tile_flags) { 111 switch (tile_flags) { 112 case 0x0000: 113 case 0x1800: 114 case 0x2800: 115 case 0x4800: 116 case 0x7000: 117 case 0x7400: 118 case 0x7a00: 119 case 0xe000: 120 break; 121 default: 122 NV_ERROR(dev, "bad page flags: 0x%08x\n", tile_flags); 123 return false; 124 } 125 126 return true; 127} 128 129int 130nouveau_gem_ioctl_new(struct drm_device *dev, void *data, 131 struct drm_file *file_priv) 132{ 133 struct drm_nouveau_private *dev_priv = dev->dev_private; 134 struct drm_nouveau_gem_new *req = data; 135 struct nouveau_bo *nvbo = NULL; 136 struct nouveau_channel *chan = NULL; 137 uint32_t flags = 0; 138 int ret = 0; 139 140 if (unlikely(dev_priv->ttm.bdev.dev_mapping == NULL)) 141 dev_priv->ttm.bdev.dev_mapping = dev_priv->dev->dev_mapping; 142 143 if (req->channel_hint) { 144 NOUVEAU_GET_USER_CHANNEL_WITH_RETURN(req->channel_hint, 145 file_priv, chan); 146 } 147 148 if (req->info.domain & NOUVEAU_GEM_DOMAIN_VRAM) 149 flags |= TTM_PL_FLAG_VRAM; 150 if (req->info.domain & NOUVEAU_GEM_DOMAIN_GART) 151 flags |= TTM_PL_FLAG_TT; 152 if (!flags || req->info.domain & NOUVEAU_GEM_DOMAIN_CPU) 153 flags |= TTM_PL_FLAG_SYSTEM; 154 155 if (!nouveau_gem_tile_flags_valid(dev, req->info.tile_flags)) 156 return -EINVAL; 157 158 ret = nouveau_gem_new(dev, chan, req->info.size, req->align, flags, 159 req->info.tile_mode, req->info.tile_flags, false, 160 (req->info.domain & NOUVEAU_GEM_DOMAIN_MAPPABLE), 161 &nvbo); 162 if (ret) 163 return ret; 164 165 ret = nouveau_gem_info(nvbo->gem, &req->info); 166 if (ret) 167 goto out; 168 169 ret = drm_gem_handle_create(file_priv, nvbo->gem, &req->info.handle); 170 /* drop reference from allocate - handle holds it now */ 171 drm_gem_object_unreference_unlocked(nvbo->gem); 172out: 173 return ret; 174} 175 176static int 177nouveau_gem_set_domain(struct drm_gem_object *gem, uint32_t read_domains, 178 uint32_t write_domains, uint32_t valid_domains) 179{ 180 struct nouveau_bo *nvbo = gem->driver_private; 181 struct ttm_buffer_object *bo = &nvbo->bo; 182 uint32_t domains = valid_domains & 183 (write_domains ? write_domains : read_domains); 184 uint32_t pref_flags = 0, valid_flags = 0; 185 186 if (!domains) 187 return -EINVAL; 188 189 if (valid_domains & NOUVEAU_GEM_DOMAIN_VRAM) 190 valid_flags |= TTM_PL_FLAG_VRAM; 191 192 if (valid_domains & NOUVEAU_GEM_DOMAIN_GART) 193 valid_flags |= TTM_PL_FLAG_TT; 194 195 if ((domains & NOUVEAU_GEM_DOMAIN_VRAM) && 196 bo->mem.mem_type == TTM_PL_VRAM) 197 pref_flags |= TTM_PL_FLAG_VRAM; 198 199 else if ((domains & NOUVEAU_GEM_DOMAIN_GART) && 200 bo->mem.mem_type == TTM_PL_TT) 201 pref_flags |= TTM_PL_FLAG_TT; 202 203 else if (domains & NOUVEAU_GEM_DOMAIN_VRAM) 204 pref_flags |= TTM_PL_FLAG_VRAM; 205 206 else 207 pref_flags |= TTM_PL_FLAG_TT; 208 209 nouveau_bo_placement_set(nvbo, pref_flags, valid_flags); 210 211 return 0; 212} 213 214struct validate_op { 215 struct list_head vram_list; 216 struct list_head gart_list; 217 struct list_head both_list; 218}; 219 220static void 221validate_fini_list(struct list_head *list, struct nouveau_fence *fence) 222{ 223 struct list_head *entry, *tmp; 224 struct nouveau_bo *nvbo; 225 226 list_for_each_safe(entry, tmp, list) { 227 nvbo = list_entry(entry, struct nouveau_bo, entry); 228 if (likely(fence)) { 229 struct nouveau_fence *prev_fence; 230 231 spin_lock(&nvbo->bo.lock); 232 prev_fence = nvbo->bo.sync_obj; 233 nvbo->bo.sync_obj = nouveau_fence_ref(fence); 234 spin_unlock(&nvbo->bo.lock); 235 nouveau_fence_unref((void *)&prev_fence); 236 } 237 238 if (unlikely(nvbo->validate_mapped)) { 239 ttm_bo_kunmap(&nvbo->kmap); 240 nvbo->validate_mapped = false; 241 } 242 243 list_del(&nvbo->entry); 244 nvbo->reserved_by = NULL; 245 ttm_bo_unreserve(&nvbo->bo); 246 drm_gem_object_unreference_unlocked(nvbo->gem); 247 } 248} 249 250static void 251validate_fini(struct validate_op *op, struct nouveau_fence* fence) 252{ 253 validate_fini_list(&op->vram_list, fence); 254 validate_fini_list(&op->gart_list, fence); 255 validate_fini_list(&op->both_list, fence); 256} 257 258static int 259validate_init(struct nouveau_channel *chan, struct drm_file *file_priv, 260 struct drm_nouveau_gem_pushbuf_bo *pbbo, 261 int nr_buffers, struct validate_op *op) 262{ 263 struct drm_device *dev = chan->dev; 264 struct drm_nouveau_private *dev_priv = dev->dev_private; 265 uint32_t sequence; 266 int trycnt = 0; 267 int ret, i; 268 269 sequence = atomic_add_return(1, &dev_priv->ttm.validate_sequence); 270retry: 271 if (++trycnt > 100000) { 272 NV_ERROR(dev, "%s failed and gave up.\n", __func__); 273 return -EINVAL; 274 } 275 276 for (i = 0; i < nr_buffers; i++) { 277 struct drm_nouveau_gem_pushbuf_bo *b = &pbbo[i]; 278 struct drm_gem_object *gem; 279 struct nouveau_bo *nvbo; 280 281 gem = drm_gem_object_lookup(dev, file_priv, b->handle); 282 if (!gem) { 283 NV_ERROR(dev, "Unknown handle 0x%08x\n", b->handle); 284 validate_fini(op, NULL); 285 return -ENOENT; 286 } 287 nvbo = gem->driver_private; 288 289 if (nvbo->reserved_by && nvbo->reserved_by == file_priv) { 290 NV_ERROR(dev, "multiple instances of buffer %d on " 291 "validation list\n", b->handle); 292 validate_fini(op, NULL); 293 return -EINVAL; 294 } 295 296 ret = ttm_bo_reserve(&nvbo->bo, false, false, true, sequence); 297 if (ret) { 298 validate_fini(op, NULL); 299 if (ret == -EAGAIN) 300 ret = ttm_bo_wait_unreserved(&nvbo->bo, false); 301 drm_gem_object_unreference_unlocked(gem); 302 if (ret) { 303 NV_ERROR(dev, "fail reserve\n"); 304 return ret; 305 } 306 goto retry; 307 } 308 309 b->user_priv = (uint64_t)(unsigned long)nvbo; 310 nvbo->reserved_by = file_priv; 311 nvbo->pbbo_index = i; 312 if ((b->valid_domains & NOUVEAU_GEM_DOMAIN_VRAM) && 313 (b->valid_domains & NOUVEAU_GEM_DOMAIN_GART)) 314 list_add_tail(&nvbo->entry, &op->both_list); 315 else 316 if (b->valid_domains & NOUVEAU_GEM_DOMAIN_VRAM) 317 list_add_tail(&nvbo->entry, &op->vram_list); 318 else 319 if (b->valid_domains & NOUVEAU_GEM_DOMAIN_GART) 320 list_add_tail(&nvbo->entry, &op->gart_list); 321 else { 322 NV_ERROR(dev, "invalid valid domains: 0x%08x\n", 323 b->valid_domains); 324 list_add_tail(&nvbo->entry, &op->both_list); 325 validate_fini(op, NULL); 326 return -EINVAL; 327 } 328 329 if (unlikely(atomic_read(&nvbo->bo.cpu_writers) > 0)) { 330 validate_fini(op, NULL); 331 332 if (nvbo->cpu_filp == file_priv) { 333 NV_ERROR(dev, "bo %p mapped by process trying " 334 "to validate it!\n", nvbo); 335 return -EINVAL; 336 } 337 338 mutex_unlock(&drm_global_mutex); 339 ret = ttm_bo_wait_cpu(&nvbo->bo, false); 340 mutex_lock(&drm_global_mutex); 341 if (ret) { 342 NV_ERROR(dev, "fail wait_cpu\n"); 343 return ret; 344 } 345 goto retry; 346 } 347 } 348 349 return 0; 350} 351 352static int 353validate_list(struct nouveau_channel *chan, struct list_head *list, 354 struct drm_nouveau_gem_pushbuf_bo *pbbo, uint64_t user_pbbo_ptr) 355{ 356 struct drm_nouveau_gem_pushbuf_bo __user *upbbo = 357 (void __force __user *)(uintptr_t)user_pbbo_ptr; 358 struct drm_device *dev = chan->dev; 359 struct nouveau_bo *nvbo; 360 int ret, relocs = 0; 361 362 list_for_each_entry(nvbo, list, entry) { 363 struct drm_nouveau_gem_pushbuf_bo *b = &pbbo[nvbo->pbbo_index]; 364 365 ret = nouveau_bo_sync_gpu(nvbo, chan); 366 if (unlikely(ret)) { 367 NV_ERROR(dev, "fail pre-validate sync\n"); 368 return ret; 369 } 370 371 ret = nouveau_gem_set_domain(nvbo->gem, b->read_domains, 372 b->write_domains, 373 b->valid_domains); 374 if (unlikely(ret)) { 375 NV_ERROR(dev, "fail set_domain\n"); 376 return ret; 377 } 378 379 nvbo->channel = (b->read_domains & (1 << 31)) ? NULL : chan; 380 ret = ttm_bo_validate(&nvbo->bo, &nvbo->placement, 381 false, false, false); 382 nvbo->channel = NULL; 383 if (unlikely(ret)) { 384 NV_ERROR(dev, "fail ttm_validate\n"); 385 return ret; 386 } 387 388 ret = nouveau_bo_sync_gpu(nvbo, chan); 389 if (unlikely(ret)) { 390 NV_ERROR(dev, "fail post-validate sync\n"); 391 return ret; 392 } 393 394 if (nvbo->bo.offset == b->presumed.offset && 395 ((nvbo->bo.mem.mem_type == TTM_PL_VRAM && 396 b->presumed.domain & NOUVEAU_GEM_DOMAIN_VRAM) || 397 (nvbo->bo.mem.mem_type == TTM_PL_TT && 398 b->presumed.domain & NOUVEAU_GEM_DOMAIN_GART))) 399 continue; 400 401 if (nvbo->bo.mem.mem_type == TTM_PL_TT) 402 b->presumed.domain = NOUVEAU_GEM_DOMAIN_GART; 403 else 404 b->presumed.domain = NOUVEAU_GEM_DOMAIN_VRAM; 405 b->presumed.offset = nvbo->bo.offset; 406 b->presumed.valid = 0; 407 relocs++; 408 409 if (DRM_COPY_TO_USER(&upbbo[nvbo->pbbo_index].presumed, 410 &b->presumed, sizeof(b->presumed))) 411 return -EFAULT; 412 } 413 414 return relocs; 415} 416 417static int 418nouveau_gem_pushbuf_validate(struct nouveau_channel *chan, 419 struct drm_file *file_priv, 420 struct drm_nouveau_gem_pushbuf_bo *pbbo, 421 uint64_t user_buffers, int nr_buffers, 422 struct validate_op *op, int *apply_relocs) 423{ 424 struct drm_device *dev = chan->dev; 425 int ret, relocs = 0; 426 427 INIT_LIST_HEAD(&op->vram_list); 428 INIT_LIST_HEAD(&op->gart_list); 429 INIT_LIST_HEAD(&op->both_list); 430 431 if (nr_buffers == 0) 432 return 0; 433 434 ret = validate_init(chan, file_priv, pbbo, nr_buffers, op); 435 if (unlikely(ret)) { 436 NV_ERROR(dev, "validate_init\n"); 437 return ret; 438 } 439 440 ret = validate_list(chan, &op->vram_list, pbbo, user_buffers); 441 if (unlikely(ret < 0)) { 442 NV_ERROR(dev, "validate vram_list\n"); 443 validate_fini(op, NULL); 444 return ret; 445 } 446 relocs += ret; 447 448 ret = validate_list(chan, &op->gart_list, pbbo, user_buffers); 449 if (unlikely(ret < 0)) { 450 NV_ERROR(dev, "validate gart_list\n"); 451 validate_fini(op, NULL); 452 return ret; 453 } 454 relocs += ret; 455 456 ret = validate_list(chan, &op->both_list, pbbo, user_buffers); 457 if (unlikely(ret < 0)) { 458 NV_ERROR(dev, "validate both_list\n"); 459 validate_fini(op, NULL); 460 return ret; 461 } 462 relocs += ret; 463 464 *apply_relocs = relocs; 465 return 0; 466} 467 468static inline void * 469u_memcpya(uint64_t user, unsigned nmemb, unsigned size) 470{ 471 void *mem; 472 void __user *userptr = (void __force __user *)(uintptr_t)user; 473 474 mem = kmalloc(nmemb * size, GFP_KERNEL); 475 if (!mem) 476 return ERR_PTR(-ENOMEM); 477 478 if (DRM_COPY_FROM_USER(mem, userptr, nmemb * size)) { 479 kfree(mem); 480 return ERR_PTR(-EFAULT); 481 } 482 483 return mem; 484} 485 486static int 487nouveau_gem_pushbuf_reloc_apply(struct drm_device *dev, 488 struct drm_nouveau_gem_pushbuf *req, 489 struct drm_nouveau_gem_pushbuf_bo *bo) 490{ 491 struct drm_nouveau_gem_pushbuf_reloc *reloc = NULL; 492 int ret = 0; 493 unsigned i; 494 495 reloc = u_memcpya(req->relocs, req->nr_relocs, sizeof(*reloc)); 496 if (IS_ERR(reloc)) 497 return PTR_ERR(reloc); 498 499 for (i = 0; i < req->nr_relocs; i++) { 500 struct drm_nouveau_gem_pushbuf_reloc *r = &reloc[i]; 501 struct drm_nouveau_gem_pushbuf_bo *b; 502 struct nouveau_bo *nvbo; 503 uint32_t data; 504 505 if (unlikely(r->bo_index > req->nr_buffers)) { 506 NV_ERROR(dev, "reloc bo index invalid\n"); 507 ret = -EINVAL; 508 break; 509 } 510 511 b = &bo[r->bo_index]; 512 if (b->presumed.valid) 513 continue; 514 515 if (unlikely(r->reloc_bo_index > req->nr_buffers)) { 516 NV_ERROR(dev, "reloc container bo index invalid\n"); 517 ret = -EINVAL; 518 break; 519 } 520 nvbo = (void *)(unsigned long)bo[r->reloc_bo_index].user_priv; 521 522 if (unlikely(r->reloc_bo_offset + 4 > 523 nvbo->bo.mem.num_pages << PAGE_SHIFT)) { 524 NV_ERROR(dev, "reloc outside of bo\n"); 525 ret = -EINVAL; 526 break; 527 } 528 529 if (!nvbo->kmap.virtual) { 530 ret = ttm_bo_kmap(&nvbo->bo, 0, nvbo->bo.mem.num_pages, 531 &nvbo->kmap); 532 if (ret) { 533 NV_ERROR(dev, "failed kmap for reloc\n"); 534 break; 535 } 536 nvbo->validate_mapped = true; 537 } 538 539 if (r->flags & NOUVEAU_GEM_RELOC_LOW) 540 data = b->presumed.offset + r->data; 541 else 542 if (r->flags & NOUVEAU_GEM_RELOC_HIGH) 543 data = (b->presumed.offset + r->data) >> 32; 544 else 545 data = r->data; 546 547 if (r->flags & NOUVEAU_GEM_RELOC_OR) { 548 if (b->presumed.domain == NOUVEAU_GEM_DOMAIN_GART) 549 data |= r->tor; 550 else 551 data |= r->vor; 552 } 553 554 spin_lock(&nvbo->bo.lock); 555 ret = ttm_bo_wait(&nvbo->bo, false, false, false); 556 spin_unlock(&nvbo->bo.lock); 557 if (ret) { 558 NV_ERROR(dev, "reloc wait_idle failed: %d\n", ret); 559 break; 560 } 561 562 nouveau_bo_wr32(nvbo, r->reloc_bo_offset >> 2, data); 563 } 564 565 kfree(reloc); 566 return ret; 567} 568 569int 570nouveau_gem_ioctl_pushbuf(struct drm_device *dev, void *data, 571 struct drm_file *file_priv) 572{ 573 struct drm_nouveau_private *dev_priv = dev->dev_private; 574 struct drm_nouveau_gem_pushbuf *req = data; 575 struct drm_nouveau_gem_pushbuf_push *push; 576 struct drm_nouveau_gem_pushbuf_bo *bo; 577 struct nouveau_channel *chan; 578 struct validate_op op; 579 struct nouveau_fence *fence = NULL; 580 int i, j, ret = 0, do_reloc = 0; 581 582 NOUVEAU_GET_USER_CHANNEL_WITH_RETURN(req->channel, file_priv, chan); 583 584 req->vram_available = dev_priv->fb_aper_free; 585 req->gart_available = dev_priv->gart_info.aper_free; 586 if (unlikely(req->nr_push == 0)) 587 goto out_next; 588 589 if (unlikely(req->nr_push > NOUVEAU_GEM_MAX_PUSH)) { 590 NV_ERROR(dev, "pushbuf push count exceeds limit: %d max %d\n", 591 req->nr_push, NOUVEAU_GEM_MAX_PUSH); 592 return -EINVAL; 593 } 594 595 if (unlikely(req->nr_buffers > NOUVEAU_GEM_MAX_BUFFERS)) { 596 NV_ERROR(dev, "pushbuf bo count exceeds limit: %d max %d\n", 597 req->nr_buffers, NOUVEAU_GEM_MAX_BUFFERS); 598 return -EINVAL; 599 } 600 601 if (unlikely(req->nr_relocs > NOUVEAU_GEM_MAX_RELOCS)) { 602 NV_ERROR(dev, "pushbuf reloc count exceeds limit: %d max %d\n", 603 req->nr_relocs, NOUVEAU_GEM_MAX_RELOCS); 604 return -EINVAL; 605 } 606 607 push = u_memcpya(req->push, req->nr_push, sizeof(*push)); 608 if (IS_ERR(push)) 609 return PTR_ERR(push); 610 611 bo = u_memcpya(req->buffers, req->nr_buffers, sizeof(*bo)); 612 if (IS_ERR(bo)) { 613 kfree(push); 614 return PTR_ERR(bo); 615 } 616 617 /* Mark push buffers as being used on PFIFO, the validation code 618 * will then make sure that if the pushbuf bo moves, that they 619 * happen on the kernel channel, which will in turn cause a sync 620 * to happen before we try and submit the push buffer. 621 */ 622 for (i = 0; i < req->nr_push; i++) { 623 if (push[i].bo_index >= req->nr_buffers) { 624 NV_ERROR(dev, "push %d buffer not in list\n", i); 625 ret = -EINVAL; 626 goto out; 627 } 628 629 bo[push[i].bo_index].read_domains |= (1 << 31); 630 } 631 632 /* Validate buffer list */ 633 ret = nouveau_gem_pushbuf_validate(chan, file_priv, bo, req->buffers, 634 req->nr_buffers, &op, &do_reloc); 635 if (ret) { 636 NV_ERROR(dev, "validate: %d\n", ret); 637 goto out; 638 } 639 640 /* Apply any relocations that are required */ 641 if (do_reloc) { 642 ret = nouveau_gem_pushbuf_reloc_apply(dev, req, bo); 643 if (ret) { 644 NV_ERROR(dev, "reloc apply: %d\n", ret); 645 goto out; 646 } 647 } 648 649 if (chan->dma.ib_max) { 650 ret = nouveau_dma_wait(chan, req->nr_push + 1, 6); 651 if (ret) { 652 NV_INFO(dev, "nv50cal_space: %d\n", ret); 653 goto out; 654 } 655 656 for (i = 0; i < req->nr_push; i++) { 657 struct nouveau_bo *nvbo = (void *)(unsigned long) 658 bo[push[i].bo_index].user_priv; 659 660 nv50_dma_push(chan, nvbo, push[i].offset, 661 push[i].length); 662 } 663 } else 664 if (dev_priv->chipset >= 0x25) { 665 ret = RING_SPACE(chan, req->nr_push * 2); 666 if (ret) { 667 NV_ERROR(dev, "cal_space: %d\n", ret); 668 goto out; 669 } 670 671 for (i = 0; i < req->nr_push; i++) { 672 struct nouveau_bo *nvbo = (void *)(unsigned long) 673 bo[push[i].bo_index].user_priv; 674 struct drm_mm_node *mem = nvbo->bo.mem.mm_node; 675 676 OUT_RING(chan, ((mem->start << PAGE_SHIFT) + 677 push[i].offset) | 2); 678 OUT_RING(chan, 0); 679 } 680 } else { 681 ret = RING_SPACE(chan, req->nr_push * (2 + NOUVEAU_DMA_SKIPS)); 682 if (ret) { 683 NV_ERROR(dev, "jmp_space: %d\n", ret); 684 goto out; 685 } 686 687 for (i = 0; i < req->nr_push; i++) { 688 struct nouveau_bo *nvbo = (void *)(unsigned long) 689 bo[push[i].bo_index].user_priv; 690 struct drm_mm_node *mem = nvbo->bo.mem.mm_node; 691 uint32_t cmd; 692 693 cmd = chan->pushbuf_base + ((chan->dma.cur + 2) << 2); 694 cmd |= 0x20000000; 695 if (unlikely(cmd != req->suffix0)) { 696 if (!nvbo->kmap.virtual) { 697 ret = ttm_bo_kmap(&nvbo->bo, 0, 698 nvbo->bo.mem. 699 num_pages, 700 &nvbo->kmap); 701 if (ret) { 702 WIND_RING(chan); 703 goto out; 704 } 705 nvbo->validate_mapped = true; 706 } 707 708 nouveau_bo_wr32(nvbo, (push[i].offset + 709 push[i].length - 8) / 4, cmd); 710 } 711 712 OUT_RING(chan, ((mem->start << PAGE_SHIFT) + 713 push[i].offset) | 0x20000000); 714 OUT_RING(chan, 0); 715 for (j = 0; j < NOUVEAU_DMA_SKIPS; j++) 716 OUT_RING(chan, 0); 717 } 718 } 719 720 ret = nouveau_fence_new(chan, &fence, true); 721 if (ret) { 722 NV_ERROR(dev, "error fencing pushbuf: %d\n", ret); 723 WIND_RING(chan); 724 goto out; 725 } 726 727out: 728 validate_fini(&op, fence); 729 nouveau_fence_unref((void**)&fence); 730 kfree(bo); 731 kfree(push); 732 733out_next: 734 if (chan->dma.ib_max) { 735 req->suffix0 = 0x00000000; 736 req->suffix1 = 0x00000000; 737 } else 738 if (dev_priv->chipset >= 0x25) { 739 req->suffix0 = 0x00020000; 740 req->suffix1 = 0x00000000; 741 } else { 742 req->suffix0 = 0x20000000 | 743 (chan->pushbuf_base + ((chan->dma.cur + 2) << 2)); 744 req->suffix1 = 0x00000000; 745 } 746 747 return ret; 748} 749 750static inline uint32_t 751domain_to_ttm(struct nouveau_bo *nvbo, uint32_t domain) 752{ 753 uint32_t flags = 0; 754 755 if (domain & NOUVEAU_GEM_DOMAIN_VRAM) 756 flags |= TTM_PL_FLAG_VRAM; 757 if (domain & NOUVEAU_GEM_DOMAIN_GART) 758 flags |= TTM_PL_FLAG_TT; 759 760 return flags; 761} 762 763int 764nouveau_gem_ioctl_cpu_prep(struct drm_device *dev, void *data, 765 struct drm_file *file_priv) 766{ 767 struct drm_nouveau_gem_cpu_prep *req = data; 768 struct drm_gem_object *gem; 769 struct nouveau_bo *nvbo; 770 bool no_wait = !!(req->flags & NOUVEAU_GEM_CPU_PREP_NOWAIT); 771 int ret = -EINVAL; 772 773 gem = drm_gem_object_lookup(dev, file_priv, req->handle); 774 if (!gem) 775 return -ENOENT; 776 nvbo = nouveau_gem_object(gem); 777 778 if (nvbo->cpu_filp) { 779 if (nvbo->cpu_filp == file_priv) 780 goto out; 781 782 ret = ttm_bo_wait_cpu(&nvbo->bo, no_wait); 783 if (ret) 784 goto out; 785 } 786 787 if (req->flags & NOUVEAU_GEM_CPU_PREP_NOBLOCK) { 788 spin_lock(&nvbo->bo.lock); 789 ret = ttm_bo_wait(&nvbo->bo, false, false, no_wait); 790 spin_unlock(&nvbo->bo.lock); 791 } else { 792 ret = ttm_bo_synccpu_write_grab(&nvbo->bo, no_wait); 793 if (ret == 0) 794 nvbo->cpu_filp = file_priv; 795 } 796 797out: 798 drm_gem_object_unreference_unlocked(gem); 799 return ret; 800} 801 802int 803nouveau_gem_ioctl_cpu_fini(struct drm_device *dev, void *data, 804 struct drm_file *file_priv) 805{ 806 struct drm_nouveau_gem_cpu_prep *req = data; 807 struct drm_gem_object *gem; 808 struct nouveau_bo *nvbo; 809 int ret = -EINVAL; 810 811 gem = drm_gem_object_lookup(dev, file_priv, req->handle); 812 if (!gem) 813 return -ENOENT; 814 nvbo = nouveau_gem_object(gem); 815 816 if (nvbo->cpu_filp != file_priv) 817 goto out; 818 nvbo->cpu_filp = NULL; 819 820 ttm_bo_synccpu_write_release(&nvbo->bo); 821 ret = 0; 822 823out: 824 drm_gem_object_unreference_unlocked(gem); 825 return ret; 826} 827 828int 829nouveau_gem_ioctl_info(struct drm_device *dev, void *data, 830 struct drm_file *file_priv) 831{ 832 struct drm_nouveau_gem_info *req = data; 833 struct drm_gem_object *gem; 834 int ret; 835 836 gem = drm_gem_object_lookup(dev, file_priv, req->handle); 837 if (!gem) 838 return -ENOENT; 839 840 ret = nouveau_gem_info(gem, req); 841 drm_gem_object_unreference_unlocked(gem); 842 return ret; 843} 844