1// SPDX-License-Identifier: GPL-2.0-only 2/* Copyright (c) 2020 NVIDIA Corporation */ 3 4#include <linux/dma-fence-array.h> 5#include <linux/dma-mapping.h> 6#include <linux/file.h> 7#include <linux/host1x.h> 8#include <linux/iommu.h> 9#include <linux/kref.h> 10#include <linux/list.h> 11#include <linux/nospec.h> 12#include <linux/pm_runtime.h> 13#include <linux/scatterlist.h> 14#include <linux/slab.h> 15#include <linux/sync_file.h> 16 17#include <drm/drm_drv.h> 18#include <drm/drm_file.h> 19#include <drm/drm_syncobj.h> 20 21#include "drm.h" 22#include "gem.h" 23#include "submit.h" 24#include "uapi.h" 25 26#define SUBMIT_ERR(context, fmt, ...) \ 27 dev_err_ratelimited(context->client->base.dev, \ 28 "%s: job submission failed: " fmt "\n", \ 29 current->comm, ##__VA_ARGS__) 30 31struct gather_bo { 32 struct host1x_bo base; 33 34 struct kref ref; 35 36 struct device *dev; 37 u32 *gather_data; 38 dma_addr_t gather_data_dma; 39 size_t gather_data_words; 40}; 41 42static struct host1x_bo *gather_bo_get(struct host1x_bo *host_bo) 43{ 44 struct gather_bo *bo = container_of(host_bo, struct gather_bo, base); 45 46 kref_get(&bo->ref); 47 48 return host_bo; 49} 50 51static void gather_bo_release(struct kref *ref) 52{ 53 struct gather_bo *bo = container_of(ref, struct gather_bo, ref); 54 55 dma_free_attrs(bo->dev, bo->gather_data_words * 4, bo->gather_data, bo->gather_data_dma, 56 0); 57 kfree(bo); 58} 59 60static void gather_bo_put(struct host1x_bo *host_bo) 61{ 62 struct gather_bo *bo = container_of(host_bo, struct gather_bo, base); 63 64 kref_put(&bo->ref, gather_bo_release); 65} 66 67static struct host1x_bo_mapping * 68gather_bo_pin(struct device *dev, struct host1x_bo *bo, enum dma_data_direction direction) 69{ 70 struct gather_bo *gather = container_of(bo, struct gather_bo, base); 71 struct host1x_bo_mapping *map; 72 int err; 73 74 map = kzalloc(sizeof(*map), GFP_KERNEL); 75 if (!map) 76 return ERR_PTR(-ENOMEM); 77 78 kref_init(&map->ref); 79 map->bo = host1x_bo_get(bo); 80 map->direction = direction; 81 map->dev = dev; 82 83 map->sgt = kzalloc(sizeof(*map->sgt), GFP_KERNEL); 84 if (!map->sgt) { 85 err = -ENOMEM; 86 goto free; 87 } 88 89 err = dma_get_sgtable(gather->dev, map->sgt, gather->gather_data, gather->gather_data_dma, 90 gather->gather_data_words * 4); 91 if (err) 92 goto free_sgt; 93 94 err = dma_map_sgtable(dev, map->sgt, direction, 0); 95 if (err) 96 goto free_sgt; 97 98 map->phys = sg_dma_address(map->sgt->sgl); 99 map->size = gather->gather_data_words * 4; 100 map->chunks = err; 101 102 return map; 103 104free_sgt: 105 sg_free_table(map->sgt); 106 kfree(map->sgt); 107free: 108 kfree(map); 109 return ERR_PTR(err); 110} 111 112static void gather_bo_unpin(struct host1x_bo_mapping *map) 113{ 114 if (!map) 115 return; 116 117 dma_unmap_sgtable(map->dev, map->sgt, map->direction, 0); 118 sg_free_table(map->sgt); 119 kfree(map->sgt); 120 host1x_bo_put(map->bo); 121 122 kfree(map); 123} 124 125static void *gather_bo_mmap(struct host1x_bo *host_bo) 126{ 127 struct gather_bo *bo = container_of(host_bo, struct gather_bo, base); 128 129 return bo->gather_data; 130} 131 132static void gather_bo_munmap(struct host1x_bo *host_bo, void *addr) 133{ 134} 135 136static const struct host1x_bo_ops gather_bo_ops = { 137 .get = gather_bo_get, 138 .put = gather_bo_put, 139 .pin = gather_bo_pin, 140 .unpin = gather_bo_unpin, 141 .mmap = gather_bo_mmap, 142 .munmap = gather_bo_munmap, 143}; 144 145static struct tegra_drm_mapping * 146tegra_drm_mapping_get(struct tegra_drm_context *context, u32 id) 147{ 148 struct tegra_drm_mapping *mapping; 149 150 xa_lock(&context->mappings); 151 152 mapping = xa_load(&context->mappings, id); 153 if (mapping) 154 kref_get(&mapping->ref); 155 156 xa_unlock(&context->mappings); 157 158 return mapping; 159} 160 161static void *alloc_copy_user_array(void __user *from, size_t count, size_t size) 162{ 163 size_t copy_len; 164 void *data; 165 166 if (check_mul_overflow(count, size, ©_len)) 167 return ERR_PTR(-EINVAL); 168 169 if (copy_len > 0x4000) 170 return ERR_PTR(-E2BIG); 171 172 data = vmemdup_user(from, copy_len); 173 if (IS_ERR(data)) 174 return ERR_CAST(data); 175 176 return data; 177} 178 179static int submit_copy_gather_data(struct gather_bo **pbo, struct device *dev, 180 struct tegra_drm_context *context, 181 struct drm_tegra_channel_submit *args) 182{ 183 struct gather_bo *bo; 184 size_t copy_len; 185 186 if (args->gather_data_words == 0) { 187 SUBMIT_ERR(context, "gather_data_words cannot be zero"); 188 return -EINVAL; 189 } 190 191 if (check_mul_overflow((size_t)args->gather_data_words, (size_t)4, ©_len)) { 192 SUBMIT_ERR(context, "gather_data_words is too large"); 193 return -EINVAL; 194 } 195 196 bo = kzalloc(sizeof(*bo), GFP_KERNEL); 197 if (!bo) { 198 SUBMIT_ERR(context, "failed to allocate memory for bo info"); 199 return -ENOMEM; 200 } 201 202 host1x_bo_init(&bo->base, &gather_bo_ops); 203 kref_init(&bo->ref); 204 bo->dev = dev; 205 206 bo->gather_data = dma_alloc_attrs(dev, copy_len, &bo->gather_data_dma, 207 GFP_KERNEL | __GFP_NOWARN, 0); 208 if (!bo->gather_data) { 209 SUBMIT_ERR(context, "failed to allocate memory for gather data"); 210 kfree(bo); 211 return -ENOMEM; 212 } 213 214 if (copy_from_user(bo->gather_data, u64_to_user_ptr(args->gather_data_ptr), copy_len)) { 215 SUBMIT_ERR(context, "failed to copy gather data from userspace"); 216 dma_free_attrs(dev, copy_len, bo->gather_data, bo->gather_data_dma, 0); 217 kfree(bo); 218 return -EFAULT; 219 } 220 221 bo->gather_data_words = args->gather_data_words; 222 223 *pbo = bo; 224 225 return 0; 226} 227 228static int submit_write_reloc(struct tegra_drm_context *context, struct gather_bo *bo, 229 struct drm_tegra_submit_buf *buf, struct tegra_drm_mapping *mapping) 230{ 231 /* TODO check that target_offset is within bounds */ 232 dma_addr_t iova = mapping->iova + buf->reloc.target_offset; 233 u32 written_ptr; 234 235#ifdef CONFIG_ARCH_DMA_ADDR_T_64BIT 236 if (buf->flags & DRM_TEGRA_SUBMIT_RELOC_SECTOR_LAYOUT) 237 iova |= BIT_ULL(39); 238#endif 239 240 written_ptr = iova >> buf->reloc.shift; 241 242 if (buf->reloc.gather_offset_words >= bo->gather_data_words) { 243 SUBMIT_ERR(context, 244 "relocation has too large gather offset (%u vs gather length %zu)", 245 buf->reloc.gather_offset_words, bo->gather_data_words); 246 return -EINVAL; 247 } 248 249 buf->reloc.gather_offset_words = array_index_nospec(buf->reloc.gather_offset_words, 250 bo->gather_data_words); 251 252 bo->gather_data[buf->reloc.gather_offset_words] = written_ptr; 253 254 return 0; 255} 256 257static int submit_process_bufs(struct tegra_drm_context *context, struct gather_bo *bo, 258 struct drm_tegra_channel_submit *args, 259 struct tegra_drm_submit_data *job_data) 260{ 261 struct tegra_drm_used_mapping *mappings; 262 struct drm_tegra_submit_buf *bufs; 263 int err; 264 u32 i; 265 266 bufs = alloc_copy_user_array(u64_to_user_ptr(args->bufs_ptr), args->num_bufs, 267 sizeof(*bufs)); 268 if (IS_ERR(bufs)) { 269 SUBMIT_ERR(context, "failed to copy bufs array from userspace"); 270 return PTR_ERR(bufs); 271 } 272 273 mappings = kcalloc(args->num_bufs, sizeof(*mappings), GFP_KERNEL); 274 if (!mappings) { 275 SUBMIT_ERR(context, "failed to allocate memory for mapping info"); 276 err = -ENOMEM; 277 goto done; 278 } 279 280 for (i = 0; i < args->num_bufs; i++) { 281 struct drm_tegra_submit_buf *buf = &bufs[i]; 282 struct tegra_drm_mapping *mapping; 283 284 if (buf->flags & ~DRM_TEGRA_SUBMIT_RELOC_SECTOR_LAYOUT) { 285 SUBMIT_ERR(context, "invalid flag specified for buffer"); 286 err = -EINVAL; 287 goto drop_refs; 288 } 289 290 mapping = tegra_drm_mapping_get(context, buf->mapping); 291 if (!mapping) { 292 SUBMIT_ERR(context, "invalid mapping ID '%u' for buffer", buf->mapping); 293 err = -EINVAL; 294 goto drop_refs; 295 } 296 297 err = submit_write_reloc(context, bo, buf, mapping); 298 if (err) { 299 tegra_drm_mapping_put(mapping); 300 goto drop_refs; 301 } 302 303 mappings[i].mapping = mapping; 304 mappings[i].flags = buf->flags; 305 } 306 307 job_data->used_mappings = mappings; 308 job_data->num_used_mappings = i; 309 310 err = 0; 311 312 goto done; 313 314drop_refs: 315 while (i--) 316 tegra_drm_mapping_put(mappings[i].mapping); 317 318 kfree(mappings); 319 job_data->used_mappings = NULL; 320 321done: 322 kvfree(bufs); 323 324 return err; 325} 326 327static int submit_get_syncpt(struct tegra_drm_context *context, struct host1x_job *job, 328 struct xarray *syncpoints, struct drm_tegra_channel_submit *args) 329{ 330 struct host1x_syncpt *sp; 331 332 if (args->syncpt.flags) { 333 SUBMIT_ERR(context, "invalid flag specified for syncpt"); 334 return -EINVAL; 335 } 336 337 /* Syncpt ref will be dropped on job release */ 338 sp = xa_load(syncpoints, args->syncpt.id); 339 if (!sp) { 340 SUBMIT_ERR(context, "syncpoint specified in syncpt was not allocated"); 341 return -EINVAL; 342 } 343 344 job->syncpt = host1x_syncpt_get(sp); 345 job->syncpt_incrs = args->syncpt.increments; 346 347 return 0; 348} 349 350static int submit_job_add_gather(struct host1x_job *job, struct tegra_drm_context *context, 351 struct drm_tegra_submit_cmd_gather_uptr *cmd, 352 struct gather_bo *bo, u32 *offset, 353 struct tegra_drm_submit_data *job_data, 354 u32 *class) 355{ 356 u32 next_offset; 357 358 if (cmd->reserved[0] || cmd->reserved[1] || cmd->reserved[2]) { 359 SUBMIT_ERR(context, "non-zero reserved field in GATHER_UPTR command"); 360 return -EINVAL; 361 } 362 363 /* Check for maximum gather size */ 364 if (cmd->words > 16383) { 365 SUBMIT_ERR(context, "too many words in GATHER_UPTR command"); 366 return -EINVAL; 367 } 368 369 if (check_add_overflow(*offset, cmd->words, &next_offset)) { 370 SUBMIT_ERR(context, "too many total words in job"); 371 return -EINVAL; 372 } 373 374 if (next_offset > bo->gather_data_words) { 375 SUBMIT_ERR(context, "GATHER_UPTR command overflows gather data"); 376 return -EINVAL; 377 } 378 379 if (tegra_drm_fw_validate(context->client, bo->gather_data, *offset, 380 cmd->words, job_data, class)) { 381 SUBMIT_ERR(context, "job was rejected by firewall"); 382 return -EINVAL; 383 } 384 385 host1x_job_add_gather(job, &bo->base, cmd->words, *offset * 4); 386 387 *offset = next_offset; 388 389 return 0; 390} 391 392static struct host1x_job * 393submit_create_job(struct tegra_drm_context *context, struct gather_bo *bo, 394 struct drm_tegra_channel_submit *args, struct tegra_drm_submit_data *job_data, 395 struct xarray *syncpoints) 396{ 397 struct drm_tegra_submit_cmd *cmds; 398 u32 i, gather_offset = 0, class; 399 struct host1x_job *job; 400 int err; 401 402 /* Set initial class for firewall. */ 403 class = context->client->base.class; 404 405 cmds = alloc_copy_user_array(u64_to_user_ptr(args->cmds_ptr), args->num_cmds, 406 sizeof(*cmds)); 407 if (IS_ERR(cmds)) { 408 SUBMIT_ERR(context, "failed to copy cmds array from userspace"); 409 return ERR_CAST(cmds); 410 } 411 412 job = host1x_job_alloc(context->channel, args->num_cmds, 0, true); 413 if (!job) { 414 SUBMIT_ERR(context, "failed to allocate memory for job"); 415 job = ERR_PTR(-ENOMEM); 416 goto done; 417 } 418 419 err = submit_get_syncpt(context, job, syncpoints, args); 420 if (err < 0) 421 goto free_job; 422 423 job->client = &context->client->base; 424 job->class = context->client->base.class; 425 job->serialize = true; 426 427 for (i = 0; i < args->num_cmds; i++) { 428 struct drm_tegra_submit_cmd *cmd = &cmds[i]; 429 430 if (cmd->flags) { 431 SUBMIT_ERR(context, "unknown flags given for cmd"); 432 err = -EINVAL; 433 goto free_job; 434 } 435 436 if (cmd->type == DRM_TEGRA_SUBMIT_CMD_GATHER_UPTR) { 437 err = submit_job_add_gather(job, context, &cmd->gather_uptr, bo, 438 &gather_offset, job_data, &class); 439 if (err) 440 goto free_job; 441 } else if (cmd->type == DRM_TEGRA_SUBMIT_CMD_WAIT_SYNCPT) { 442 if (cmd->wait_syncpt.reserved[0] || cmd->wait_syncpt.reserved[1]) { 443 SUBMIT_ERR(context, "non-zero reserved value"); 444 err = -EINVAL; 445 goto free_job; 446 } 447 448 host1x_job_add_wait(job, cmd->wait_syncpt.id, cmd->wait_syncpt.value, 449 false, class); 450 } else if (cmd->type == DRM_TEGRA_SUBMIT_CMD_WAIT_SYNCPT_RELATIVE) { 451 if (cmd->wait_syncpt.reserved[0] || cmd->wait_syncpt.reserved[1]) { 452 SUBMIT_ERR(context, "non-zero reserved value"); 453 err = -EINVAL; 454 goto free_job; 455 } 456 457 if (cmd->wait_syncpt.id != args->syncpt.id) { 458 SUBMIT_ERR(context, "syncpoint ID in CMD_WAIT_SYNCPT_RELATIVE is not used by the job"); 459 err = -EINVAL; 460 goto free_job; 461 } 462 463 host1x_job_add_wait(job, cmd->wait_syncpt.id, cmd->wait_syncpt.value, 464 true, class); 465 } else { 466 SUBMIT_ERR(context, "unknown cmd type"); 467 err = -EINVAL; 468 goto free_job; 469 } 470 } 471 472 if (gather_offset == 0) { 473 SUBMIT_ERR(context, "job must have at least one gather"); 474 err = -EINVAL; 475 goto free_job; 476 } 477 478 goto done; 479 480free_job: 481 host1x_job_put(job); 482 job = ERR_PTR(err); 483 484done: 485 kvfree(cmds); 486 487 return job; 488} 489 490static void release_job(struct host1x_job *job) 491{ 492 struct tegra_drm_client *client = container_of(job->client, struct tegra_drm_client, base); 493 struct tegra_drm_submit_data *job_data = job->user_data; 494 u32 i; 495 496 if (job->memory_context) 497 host1x_memory_context_put(job->memory_context); 498 499 for (i = 0; i < job_data->num_used_mappings; i++) 500 tegra_drm_mapping_put(job_data->used_mappings[i].mapping); 501 502 kfree(job_data->used_mappings); 503 kfree(job_data); 504 505 pm_runtime_mark_last_busy(client->base.dev); 506 pm_runtime_put_autosuspend(client->base.dev); 507} 508 509int tegra_drm_ioctl_channel_submit(struct drm_device *drm, void *data, 510 struct drm_file *file) 511{ 512 struct tegra_drm_file *fpriv = file->driver_priv; 513 struct drm_tegra_channel_submit *args = data; 514 struct tegra_drm_submit_data *job_data; 515 struct drm_syncobj *syncobj = NULL; 516 struct tegra_drm_context *context; 517 struct host1x_job *job; 518 struct gather_bo *bo; 519 u32 i; 520 int err; 521 522 mutex_lock(&fpriv->lock); 523 524 context = xa_load(&fpriv->contexts, args->context); 525 if (!context) { 526 mutex_unlock(&fpriv->lock); 527 pr_err_ratelimited("%s: %s: invalid channel context '%#x'", __func__, 528 current->comm, args->context); 529 return -EINVAL; 530 } 531 532 if (args->syncobj_in) { 533 struct dma_fence *fence; 534 535 err = drm_syncobj_find_fence(file, args->syncobj_in, 0, 0, &fence); 536 if (err) { 537 SUBMIT_ERR(context, "invalid syncobj_in '%#x'", args->syncobj_in); 538 goto unlock; 539 } 540 541 err = dma_fence_wait_timeout(fence, true, msecs_to_jiffies(10000)); 542 dma_fence_put(fence); 543 if (err) { 544 SUBMIT_ERR(context, "wait for syncobj_in timed out"); 545 goto unlock; 546 } 547 } 548 549 if (args->syncobj_out) { 550 syncobj = drm_syncobj_find(file, args->syncobj_out); 551 if (!syncobj) { 552 SUBMIT_ERR(context, "invalid syncobj_out '%#x'", args->syncobj_out); 553 err = -ENOENT; 554 goto unlock; 555 } 556 } 557 558 /* Allocate gather BO and copy gather words in. */ 559 err = submit_copy_gather_data(&bo, drm->dev, context, args); 560 if (err) 561 goto unlock; 562 563 job_data = kzalloc(sizeof(*job_data), GFP_KERNEL); 564 if (!job_data) { 565 SUBMIT_ERR(context, "failed to allocate memory for job data"); 566 err = -ENOMEM; 567 goto put_bo; 568 } 569 570 /* Get data buffer mappings and do relocation patching. */ 571 err = submit_process_bufs(context, bo, args, job_data); 572 if (err) 573 goto free_job_data; 574 575 /* Allocate host1x_job and add gathers and waits to it. */ 576 job = submit_create_job(context, bo, args, job_data, &fpriv->syncpoints); 577 if (IS_ERR(job)) { 578 err = PTR_ERR(job); 579 goto free_job_data; 580 } 581 582 /* Map gather data for Host1x. */ 583 err = host1x_job_pin(job, context->client->base.dev); 584 if (err) { 585 SUBMIT_ERR(context, "failed to pin job: %d", err); 586 goto put_job; 587 } 588 589 if (context->client->ops->get_streamid_offset) { 590 err = context->client->ops->get_streamid_offset( 591 context->client, &job->engine_streamid_offset); 592 if (err) { 593 SUBMIT_ERR(context, "failed to get streamid offset: %d", err); 594 goto unpin_job; 595 } 596 } 597 598 if (context->memory_context && context->client->ops->can_use_memory_ctx) { 599 bool supported; 600 601 err = context->client->ops->can_use_memory_ctx(context->client, &supported); 602 if (err) { 603 SUBMIT_ERR(context, "failed to detect if engine can use memory context: %d", err); 604 goto unpin_job; 605 } 606 607 if (supported) { 608 job->memory_context = context->memory_context; 609 host1x_memory_context_get(job->memory_context); 610 } 611 } else if (context->client->ops->get_streamid_offset) { 612 /* 613 * Job submission will need to temporarily change stream ID, 614 * so need to tell it what to change it back to. 615 */ 616 if (!tegra_dev_iommu_get_stream_id(context->client->base.dev, 617 &job->engine_fallback_streamid)) 618 job->engine_fallback_streamid = TEGRA_STREAM_ID_BYPASS; 619 } 620 621 /* Boot engine. */ 622 err = pm_runtime_resume_and_get(context->client->base.dev); 623 if (err < 0) { 624 SUBMIT_ERR(context, "could not power up engine: %d", err); 625 goto put_memory_context; 626 } 627 628 job->user_data = job_data; 629 job->release = release_job; 630 job->timeout = 10000; 631 632 /* 633 * job_data is now part of job reference counting, so don't release 634 * it from here. 635 */ 636 job_data = NULL; 637 638 /* Submit job to hardware. */ 639 err = host1x_job_submit(job); 640 if (err) { 641 SUBMIT_ERR(context, "host1x job submission failed: %d", err); 642 goto unpin_job; 643 } 644 645 /* Return postfences to userspace and add fences to DMA reservations. */ 646 args->syncpt.value = job->syncpt_end; 647 648 if (syncobj) { 649 struct dma_fence *fence = host1x_fence_create(job->syncpt, job->syncpt_end, true); 650 if (IS_ERR(fence)) { 651 err = PTR_ERR(fence); 652 SUBMIT_ERR(context, "failed to create postfence: %d", err); 653 } 654 655 drm_syncobj_replace_fence(syncobj, fence); 656 } 657 658 goto put_job; 659 660put_memory_context: 661 if (job->memory_context) 662 host1x_memory_context_put(job->memory_context); 663unpin_job: 664 host1x_job_unpin(job); 665put_job: 666 host1x_job_put(job); 667free_job_data: 668 if (job_data && job_data->used_mappings) { 669 for (i = 0; i < job_data->num_used_mappings; i++) 670 tegra_drm_mapping_put(job_data->used_mappings[i].mapping); 671 672 kfree(job_data->used_mappings); 673 } 674 675 kfree(job_data); 676put_bo: 677 gather_bo_put(&bo->base); 678unlock: 679 if (syncobj) 680 drm_syncobj_put(syncobj); 681 682 mutex_unlock(&fpriv->lock); 683 return err; 684} 685