1/* 2 * Copyright 2017 Red Hat 3 * Parts ported from amdgpu (fence wait code). 4 * Copyright 2016 Advanced Micro Devices, Inc. 5 * 6 * Permission is hereby granted, free of charge, to any person obtaining a 7 * copy of this software and associated documentation files (the "Software"), 8 * to deal in the Software without restriction, including without limitation 9 * the rights to use, copy, modify, merge, publish, distribute, sublicense, 10 * and/or sell copies of the Software, and to permit persons to whom the 11 * Software is furnished to do so, subject to the following conditions: 12 * 13 * The above copyright notice and this permission notice (including the next 14 * paragraph) shall be included in all copies or substantial portions of the 15 * Software. 16 * 17 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 18 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 19 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 20 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 21 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 22 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS 23 * IN THE SOFTWARE. 24 * 25 * Authors: 26 * 27 */ 28 29/** 30 * DOC: Overview 31 * 32 * DRM synchronisation objects (syncobj, see struct &drm_syncobj) provide a 33 * container for a synchronization primitive which can be used by userspace 34 * to explicitly synchronize GPU commands, can be shared between userspace 35 * processes, and can be shared between different DRM drivers. 36 * Their primary use-case is to implement Vulkan fences and semaphores. 37 * The syncobj userspace API provides ioctls for several operations: 38 * 39 * - Creation and destruction of syncobjs 40 * - Import and export of syncobjs to/from a syncobj file descriptor 41 * - Import and export a syncobj's underlying fence to/from a sync file 42 * - Reset a syncobj (set its fence to NULL) 43 * - Signal a syncobj (set a trivially signaled fence) 44 * - Wait for a syncobj's fence to appear and be signaled 45 * 46 * The syncobj userspace API also provides operations to manipulate a syncobj 47 * in terms of a timeline of struct &dma_fence_chain rather than a single 48 * struct &dma_fence, through the following operations: 49 * 50 * - Signal a given point on the timeline 51 * - Wait for a given point to appear and/or be signaled 52 * - Import and export from/to a given point of a timeline 53 * 54 * At it's core, a syncobj is simply a wrapper around a pointer to a struct 55 * &dma_fence which may be NULL. 56 * When a syncobj is first created, its pointer is either NULL or a pointer 57 * to an already signaled fence depending on whether the 58 * &DRM_SYNCOBJ_CREATE_SIGNALED flag is passed to 59 * &DRM_IOCTL_SYNCOBJ_CREATE. 60 * 61 * If the syncobj is considered as a binary (its state is either signaled or 62 * unsignaled) primitive, when GPU work is enqueued in a DRM driver to signal 63 * the syncobj, the syncobj's fence is replaced with a fence which will be 64 * signaled by the completion of that work. 65 * If the syncobj is considered as a timeline primitive, when GPU work is 66 * enqueued in a DRM driver to signal the a given point of the syncobj, a new 67 * struct &dma_fence_chain pointing to the DRM driver's fence and also 68 * pointing to the previous fence that was in the syncobj. The new struct 69 * &dma_fence_chain fence replace the syncobj's fence and will be signaled by 70 * completion of the DRM driver's work and also any work associated with the 71 * fence previously in the syncobj. 72 * 73 * When GPU work which waits on a syncobj is enqueued in a DRM driver, at the 74 * time the work is enqueued, it waits on the syncobj's fence before 75 * submitting the work to hardware. That fence is either : 76 * 77 * - The syncobj's current fence if the syncobj is considered as a binary 78 * primitive. 79 * - The struct &dma_fence associated with a given point if the syncobj is 80 * considered as a timeline primitive. 81 * 82 * If the syncobj's fence is NULL or not present in the syncobj's timeline, 83 * the enqueue operation is expected to fail. 84 * 85 * With binary syncobj, all manipulation of the syncobjs's fence happens in 86 * terms of the current fence at the time the ioctl is called by userspace 87 * regardless of whether that operation is an immediate host-side operation 88 * (signal or reset) or or an operation which is enqueued in some driver 89 * queue. &DRM_IOCTL_SYNCOBJ_RESET and &DRM_IOCTL_SYNCOBJ_SIGNAL can be used 90 * to manipulate a syncobj from the host by resetting its pointer to NULL or 91 * setting its pointer to a fence which is already signaled. 92 * 93 * With a timeline syncobj, all manipulation of the synobj's fence happens in 94 * terms of a u64 value referring to point in the timeline. See 95 * dma_fence_chain_find_seqno() to see how a given point is found in the 96 * timeline. 97 * 98 * Note that applications should be careful to always use timeline set of 99 * ioctl() when dealing with syncobj considered as timeline. Using a binary 100 * set of ioctl() with a syncobj considered as timeline could result incorrect 101 * synchronization. The use of binary syncobj is supported through the 102 * timeline set of ioctl() by using a point value of 0, this will reproduce 103 * the behavior of the binary set of ioctl() (for example replace the 104 * syncobj's fence when signaling). 105 * 106 * 107 * Host-side wait on syncobjs 108 * -------------------------- 109 * 110 * &DRM_IOCTL_SYNCOBJ_WAIT takes an array of syncobj handles and does a 111 * host-side wait on all of the syncobj fences simultaneously. 112 * If &DRM_SYNCOBJ_WAIT_FLAGS_WAIT_ALL is set, the wait ioctl will wait on 113 * all of the syncobj fences to be signaled before it returns. 114 * Otherwise, it returns once at least one syncobj fence has been signaled 115 * and the index of a signaled fence is written back to the client. 116 * 117 * Unlike the enqueued GPU work dependencies which fail if they see a NULL 118 * fence in a syncobj, if &DRM_SYNCOBJ_WAIT_FLAGS_WAIT_FOR_SUBMIT is set, 119 * the host-side wait will first wait for the syncobj to receive a non-NULL 120 * fence and then wait on that fence. 121 * If &DRM_SYNCOBJ_WAIT_FLAGS_WAIT_FOR_SUBMIT is not set and any one of the 122 * syncobjs in the array has a NULL fence, -EINVAL will be returned. 123 * Assuming the syncobj starts off with a NULL fence, this allows a client 124 * to do a host wait in one thread (or process) which waits on GPU work 125 * submitted in another thread (or process) without having to manually 126 * synchronize between the two. 127 * This requirement is inherited from the Vulkan fence API. 128 * 129 * If &DRM_SYNCOBJ_WAIT_FLAGS_WAIT_DEADLINE is set, the ioctl will also set 130 * a fence deadline hint on the backing fences before waiting, to provide the 131 * fence signaler with an appropriate sense of urgency. The deadline is 132 * specified as an absolute &CLOCK_MONOTONIC value in units of ns. 133 * 134 * Similarly, &DRM_IOCTL_SYNCOBJ_TIMELINE_WAIT takes an array of syncobj 135 * handles as well as an array of u64 points and does a host-side wait on all 136 * of syncobj fences at the given points simultaneously. 137 * 138 * &DRM_IOCTL_SYNCOBJ_TIMELINE_WAIT also adds the ability to wait for a given 139 * fence to materialize on the timeline without waiting for the fence to be 140 * signaled by using the &DRM_SYNCOBJ_WAIT_FLAGS_WAIT_AVAILABLE flag. This 141 * requirement is inherited from the wait-before-signal behavior required by 142 * the Vulkan timeline semaphore API. 143 * 144 * Alternatively, &DRM_IOCTL_SYNCOBJ_EVENTFD can be used to wait without 145 * blocking: an eventfd will be signaled when the syncobj is. This is useful to 146 * integrate the wait in an event loop. 147 * 148 * 149 * Import/export of syncobjs 150 * ------------------------- 151 * 152 * &DRM_IOCTL_SYNCOBJ_FD_TO_HANDLE and &DRM_IOCTL_SYNCOBJ_HANDLE_TO_FD 153 * provide two mechanisms for import/export of syncobjs. 154 * 155 * The first lets the client import or export an entire syncobj to a file 156 * descriptor. 157 * These fd's are opaque and have no other use case, except passing the 158 * syncobj between processes. 159 * All exported file descriptors and any syncobj handles created as a 160 * result of importing those file descriptors own a reference to the 161 * same underlying struct &drm_syncobj and the syncobj can be used 162 * persistently across all the processes with which it is shared. 163 * The syncobj is freed only once the last reference is dropped. 164 * Unlike dma-buf, importing a syncobj creates a new handle (with its own 165 * reference) for every import instead of de-duplicating. 166 * The primary use-case of this persistent import/export is for shared 167 * Vulkan fences and semaphores. 168 * 169 * The second import/export mechanism, which is indicated by 170 * &DRM_SYNCOBJ_FD_TO_HANDLE_FLAGS_IMPORT_SYNC_FILE or 171 * &DRM_SYNCOBJ_HANDLE_TO_FD_FLAGS_EXPORT_SYNC_FILE lets the client 172 * import/export the syncobj's current fence from/to a &sync_file. 173 * When a syncobj is exported to a sync file, that sync file wraps the 174 * sycnobj's fence at the time of export and any later signal or reset 175 * operations on the syncobj will not affect the exported sync file. 176 * When a sync file is imported into a syncobj, the syncobj's fence is set 177 * to the fence wrapped by that sync file. 178 * Because sync files are immutable, resetting or signaling the syncobj 179 * will not affect any sync files whose fences have been imported into the 180 * syncobj. 181 * 182 * 183 * Import/export of timeline points in timeline syncobjs 184 * ----------------------------------------------------- 185 * 186 * &DRM_IOCTL_SYNCOBJ_TRANSFER provides a mechanism to transfer a struct 187 * &dma_fence_chain of a syncobj at a given u64 point to another u64 point 188 * into another syncobj. 189 * 190 * Note that if you want to transfer a struct &dma_fence_chain from a given 191 * point on a timeline syncobj from/into a binary syncobj, you can use the 192 * point 0 to mean take/replace the fence in the syncobj. 193 */ 194 195#include <linux/anon_inodes.h> 196#include <linux/dma-fence-unwrap.h> 197#include <linux/eventfd.h> 198#include <linux/file.h> 199#include <linux/fs.h> 200#include <linux/sched/signal.h> 201#include <linux/sync_file.h> 202#include <linux/uaccess.h> 203 204#include <drm/drm.h> 205#include <drm/drm_drv.h> 206#include <drm/drm_file.h> 207#include <drm/drm_gem.h> 208#include <drm/drm_print.h> 209#include <drm/drm_syncobj.h> 210#include <drm/drm_utils.h> 211 212#include "drm_internal.h" 213 214struct syncobj_wait_entry { 215 struct list_head node; 216 struct task_struct *task; 217 struct dma_fence *fence; 218 struct dma_fence_cb fence_cb; 219 u64 point; 220}; 221 222static void syncobj_wait_syncobj_func(struct drm_syncobj *syncobj, 223 struct syncobj_wait_entry *wait); 224 225struct syncobj_eventfd_entry { 226 struct list_head node; 227 struct dma_fence *fence; 228 struct dma_fence_cb fence_cb; 229 struct drm_syncobj *syncobj; 230 struct eventfd_ctx *ev_fd_ctx; 231 u64 point; 232 u32 flags; 233}; 234 235static void 236syncobj_eventfd_entry_func(struct drm_syncobj *syncobj, 237 struct syncobj_eventfd_entry *entry); 238 239/** 240 * drm_syncobj_find - lookup and reference a sync object. 241 * @file_private: drm file private pointer 242 * @handle: sync object handle to lookup. 243 * 244 * Returns a reference to the syncobj pointed to by handle or NULL. The 245 * reference must be released by calling drm_syncobj_put(). 246 */ 247struct drm_syncobj *drm_syncobj_find(struct drm_file *file_private, 248 u32 handle) 249{ 250 struct drm_syncobj *syncobj; 251 252 spin_lock(&file_private->syncobj_table_lock); 253 254 /* Check if we currently have a reference on the object */ 255 syncobj = idr_find(&file_private->syncobj_idr, handle); 256 if (syncobj) 257 drm_syncobj_get(syncobj); 258 259 spin_unlock(&file_private->syncobj_table_lock); 260 261 return syncobj; 262} 263EXPORT_SYMBOL(drm_syncobj_find); 264 265static void drm_syncobj_fence_add_wait(struct drm_syncobj *syncobj, 266 struct syncobj_wait_entry *wait) 267{ 268 struct dma_fence *fence; 269 270 if (wait->fence) 271 return; 272 273 spin_lock(&syncobj->lock); 274 /* We've already tried once to get a fence and failed. Now that we 275 * have the lock, try one more time just to be sure we don't add a 276 * callback when a fence has already been set. 277 */ 278 fence = dma_fence_get(rcu_dereference_protected(syncobj->fence, 1)); 279 if (!fence || dma_fence_chain_find_seqno(&fence, wait->point)) { 280 dma_fence_put(fence); 281 list_add_tail(&wait->node, &syncobj->cb_list); 282 } else if (!fence) { 283 wait->fence = dma_fence_get_stub(); 284 } else { 285 wait->fence = fence; 286 } 287 spin_unlock(&syncobj->lock); 288} 289 290static void drm_syncobj_remove_wait(struct drm_syncobj *syncobj, 291 struct syncobj_wait_entry *wait) 292{ 293 if (!wait->node.next) 294 return; 295 296 spin_lock(&syncobj->lock); 297 list_del_init(&wait->node); 298 spin_unlock(&syncobj->lock); 299} 300 301static void 302syncobj_eventfd_entry_free(struct syncobj_eventfd_entry *entry) 303{ 304 eventfd_ctx_put(entry->ev_fd_ctx); 305 dma_fence_put(entry->fence); 306 /* This happens either inside the syncobj lock, or after the node has 307 * already been removed from the list. 308 */ 309 list_del(&entry->node); 310 kfree(entry); 311} 312 313static void 314drm_syncobj_add_eventfd(struct drm_syncobj *syncobj, 315 struct syncobj_eventfd_entry *entry) 316{ 317 spin_lock(&syncobj->lock); 318 list_add_tail(&entry->node, &syncobj->ev_fd_list); 319 syncobj_eventfd_entry_func(syncobj, entry); 320 spin_unlock(&syncobj->lock); 321} 322 323/** 324 * drm_syncobj_add_point - add new timeline point to the syncobj 325 * @syncobj: sync object to add timeline point do 326 * @chain: chain node to use to add the point 327 * @fence: fence to encapsulate in the chain node 328 * @point: sequence number to use for the point 329 * 330 * Add the chain node as new timeline point to the syncobj. 331 */ 332void drm_syncobj_add_point(struct drm_syncobj *syncobj, 333 struct dma_fence_chain *chain, 334 struct dma_fence *fence, 335 uint64_t point) 336{ 337 struct syncobj_wait_entry *wait_cur, *wait_tmp; 338 struct syncobj_eventfd_entry *ev_fd_cur, *ev_fd_tmp; 339 struct dma_fence *prev; 340 341 dma_fence_get(fence); 342 343 spin_lock(&syncobj->lock); 344 345 prev = drm_syncobj_fence_get(syncobj); 346 /* You are adding an unorder point to timeline, which could cause payload returned from query_ioctl is 0! */ 347 if (prev && prev->seqno >= point) 348 DRM_DEBUG("You are adding an unorder point to timeline!\n"); 349 dma_fence_chain_init(chain, prev, fence, point); 350 rcu_assign_pointer(syncobj->fence, &chain->base); 351 352 list_for_each_entry_safe(wait_cur, wait_tmp, &syncobj->cb_list, node) 353 syncobj_wait_syncobj_func(syncobj, wait_cur); 354 list_for_each_entry_safe(ev_fd_cur, ev_fd_tmp, &syncobj->ev_fd_list, node) 355 syncobj_eventfd_entry_func(syncobj, ev_fd_cur); 356 spin_unlock(&syncobj->lock); 357 358 /* Walk the chain once to trigger garbage collection */ 359 dma_fence_chain_for_each(fence, prev); 360 dma_fence_put(prev); 361} 362EXPORT_SYMBOL(drm_syncobj_add_point); 363 364/** 365 * drm_syncobj_replace_fence - replace fence in a sync object. 366 * @syncobj: Sync object to replace fence in 367 * @fence: fence to install in sync file. 368 * 369 * This replaces the fence on a sync object. 370 */ 371void drm_syncobj_replace_fence(struct drm_syncobj *syncobj, 372 struct dma_fence *fence) 373{ 374 struct dma_fence *old_fence; 375 struct syncobj_wait_entry *wait_cur, *wait_tmp; 376 struct syncobj_eventfd_entry *ev_fd_cur, *ev_fd_tmp; 377 378 if (fence) 379 dma_fence_get(fence); 380 381 spin_lock(&syncobj->lock); 382 383 old_fence = rcu_dereference_protected(syncobj->fence, 384 lockdep_is_held(&syncobj->lock)); 385 rcu_assign_pointer(syncobj->fence, fence); 386 387 if (fence != old_fence) { 388 list_for_each_entry_safe(wait_cur, wait_tmp, &syncobj->cb_list, node) 389 syncobj_wait_syncobj_func(syncobj, wait_cur); 390 list_for_each_entry_safe(ev_fd_cur, ev_fd_tmp, &syncobj->ev_fd_list, node) 391 syncobj_eventfd_entry_func(syncobj, ev_fd_cur); 392 } 393 394 spin_unlock(&syncobj->lock); 395 396 dma_fence_put(old_fence); 397} 398EXPORT_SYMBOL(drm_syncobj_replace_fence); 399 400/** 401 * drm_syncobj_assign_null_handle - assign a stub fence to the sync object 402 * @syncobj: sync object to assign the fence on 403 * 404 * Assign a already signaled stub fence to the sync object. 405 */ 406static int drm_syncobj_assign_null_handle(struct drm_syncobj *syncobj) 407{ 408 struct dma_fence *fence = dma_fence_allocate_private_stub(ktime_get()); 409 410 if (!fence) 411 return -ENOMEM; 412 413 drm_syncobj_replace_fence(syncobj, fence); 414 dma_fence_put(fence); 415 return 0; 416} 417 418/* 5s default for wait submission */ 419#define DRM_SYNCOBJ_WAIT_FOR_SUBMIT_TIMEOUT 5000000000ULL 420/** 421 * drm_syncobj_find_fence - lookup and reference the fence in a sync object 422 * @file_private: drm file private pointer 423 * @handle: sync object handle to lookup. 424 * @point: timeline point 425 * @flags: DRM_SYNCOBJ_WAIT_FLAGS_WAIT_FOR_SUBMIT or not 426 * @fence: out parameter for the fence 427 * 428 * This is just a convenience function that combines drm_syncobj_find() and 429 * drm_syncobj_fence_get(). 430 * 431 * Returns 0 on success or a negative error value on failure. On success @fence 432 * contains a reference to the fence, which must be released by calling 433 * dma_fence_put(). 434 */ 435int drm_syncobj_find_fence(struct drm_file *file_private, 436 u32 handle, u64 point, u64 flags, 437 struct dma_fence **fence) 438{ 439 struct drm_syncobj *syncobj = drm_syncobj_find(file_private, handle); 440 struct syncobj_wait_entry wait; 441 u64 timeout = nsecs_to_jiffies64(DRM_SYNCOBJ_WAIT_FOR_SUBMIT_TIMEOUT); 442 int ret; 443 444 if (flags & ~DRM_SYNCOBJ_WAIT_FLAGS_WAIT_FOR_SUBMIT) 445 return -EINVAL; 446 447 if (!syncobj) 448 return -ENOENT; 449 450 /* Waiting for userspace with locks help is illegal cause that can 451 * trivial deadlock with page faults for example. Make lockdep complain 452 * about it early on. 453 */ 454 if (flags & DRM_SYNCOBJ_WAIT_FLAGS_WAIT_FOR_SUBMIT) { 455 might_sleep(); 456 lockdep_assert_none_held_once(); 457 } 458 459 *fence = drm_syncobj_fence_get(syncobj); 460 461 if (*fence) { 462 ret = dma_fence_chain_find_seqno(fence, point); 463 if (!ret) { 464 /* If the requested seqno is already signaled 465 * drm_syncobj_find_fence may return a NULL 466 * fence. To make sure the recipient gets 467 * signalled, use a new fence instead. 468 */ 469 if (!*fence) 470 *fence = dma_fence_get_stub(); 471 472 goto out; 473 } 474 dma_fence_put(*fence); 475 } else { 476 ret = -EINVAL; 477 } 478 479 if (!(flags & DRM_SYNCOBJ_WAIT_FLAGS_WAIT_FOR_SUBMIT)) 480 goto out; 481 482 memset(&wait, 0, sizeof(wait)); 483 wait.task = current; 484 wait.point = point; 485 drm_syncobj_fence_add_wait(syncobj, &wait); 486 487 do { 488 set_current_state(TASK_INTERRUPTIBLE); 489 if (wait.fence) { 490 ret = 0; 491 break; 492 } 493 if (timeout == 0) { 494 ret = -ETIME; 495 break; 496 } 497 498 if (signal_pending(current)) { 499 ret = -ERESTARTSYS; 500 break; 501 } 502 503 timeout = schedule_timeout(timeout); 504 } while (1); 505 506 __set_current_state(TASK_RUNNING); 507 *fence = wait.fence; 508 509 if (wait.node.next) 510 drm_syncobj_remove_wait(syncobj, &wait); 511 512out: 513 drm_syncobj_put(syncobj); 514 515 return ret; 516} 517EXPORT_SYMBOL(drm_syncobj_find_fence); 518 519/** 520 * drm_syncobj_free - free a sync object. 521 * @kref: kref to free. 522 * 523 * Only to be called from kref_put in drm_syncobj_put. 524 */ 525void drm_syncobj_free(struct kref *kref) 526{ 527 struct drm_syncobj *syncobj = container_of(kref, 528 struct drm_syncobj, 529 refcount); 530 struct syncobj_eventfd_entry *ev_fd_cur, *ev_fd_tmp; 531 532 drm_syncobj_replace_fence(syncobj, NULL); 533 534 list_for_each_entry_safe(ev_fd_cur, ev_fd_tmp, &syncobj->ev_fd_list, node) 535 syncobj_eventfd_entry_free(ev_fd_cur); 536 537 kfree(syncobj); 538} 539EXPORT_SYMBOL(drm_syncobj_free); 540 541/** 542 * drm_syncobj_create - create a new syncobj 543 * @out_syncobj: returned syncobj 544 * @flags: DRM_SYNCOBJ_* flags 545 * @fence: if non-NULL, the syncobj will represent this fence 546 * 547 * This is the first function to create a sync object. After creating, drivers 548 * probably want to make it available to userspace, either through 549 * drm_syncobj_get_handle() or drm_syncobj_get_fd(). 550 * 551 * Returns 0 on success or a negative error value on failure. 552 */ 553int drm_syncobj_create(struct drm_syncobj **out_syncobj, uint32_t flags, 554 struct dma_fence *fence) 555{ 556 int ret; 557 struct drm_syncobj *syncobj; 558 559 syncobj = kzalloc(sizeof(struct drm_syncobj), GFP_KERNEL); 560 if (!syncobj) 561 return -ENOMEM; 562 563 kref_init(&syncobj->refcount); 564 INIT_LIST_HEAD(&syncobj->cb_list); 565 INIT_LIST_HEAD(&syncobj->ev_fd_list); 566 spin_lock_init(&syncobj->lock); 567 568 if (flags & DRM_SYNCOBJ_CREATE_SIGNALED) { 569 ret = drm_syncobj_assign_null_handle(syncobj); 570 if (ret < 0) { 571 drm_syncobj_put(syncobj); 572 return ret; 573 } 574 } 575 576 if (fence) 577 drm_syncobj_replace_fence(syncobj, fence); 578 579 *out_syncobj = syncobj; 580 return 0; 581} 582EXPORT_SYMBOL(drm_syncobj_create); 583 584/** 585 * drm_syncobj_get_handle - get a handle from a syncobj 586 * @file_private: drm file private pointer 587 * @syncobj: Sync object to export 588 * @handle: out parameter with the new handle 589 * 590 * Exports a sync object created with drm_syncobj_create() as a handle on 591 * @file_private to userspace. 592 * 593 * Returns 0 on success or a negative error value on failure. 594 */ 595int drm_syncobj_get_handle(struct drm_file *file_private, 596 struct drm_syncobj *syncobj, u32 *handle) 597{ 598 int ret; 599 600 /* take a reference to put in the idr */ 601 drm_syncobj_get(syncobj); 602 603 idr_preload(GFP_KERNEL); 604 spin_lock(&file_private->syncobj_table_lock); 605 ret = idr_alloc(&file_private->syncobj_idr, syncobj, 1, 0, GFP_NOWAIT); 606 spin_unlock(&file_private->syncobj_table_lock); 607 608 idr_preload_end(); 609 610 if (ret < 0) { 611 drm_syncobj_put(syncobj); 612 return ret; 613 } 614 615 *handle = ret; 616 return 0; 617} 618EXPORT_SYMBOL(drm_syncobj_get_handle); 619 620static int drm_syncobj_create_as_handle(struct drm_file *file_private, 621 u32 *handle, uint32_t flags) 622{ 623 int ret; 624 struct drm_syncobj *syncobj; 625 626 ret = drm_syncobj_create(&syncobj, flags, NULL); 627 if (ret) 628 return ret; 629 630 ret = drm_syncobj_get_handle(file_private, syncobj, handle); 631 drm_syncobj_put(syncobj); 632 return ret; 633} 634 635static int drm_syncobj_destroy(struct drm_file *file_private, 636 u32 handle) 637{ 638 struct drm_syncobj *syncobj; 639 640 spin_lock(&file_private->syncobj_table_lock); 641 syncobj = idr_remove(&file_private->syncobj_idr, handle); 642 spin_unlock(&file_private->syncobj_table_lock); 643 644 if (!syncobj) 645 return -EINVAL; 646 647 drm_syncobj_put(syncobj); 648 return 0; 649} 650 651static int drm_syncobj_file_release(struct inode *inode, struct file *file) 652{ 653 struct drm_syncobj *syncobj = file->private_data; 654 655 drm_syncobj_put(syncobj); 656 return 0; 657} 658 659static const struct file_operations drm_syncobj_file_fops = { 660 .release = drm_syncobj_file_release, 661}; 662 663/** 664 * drm_syncobj_get_fd - get a file descriptor from a syncobj 665 * @syncobj: Sync object to export 666 * @p_fd: out parameter with the new file descriptor 667 * 668 * Exports a sync object created with drm_syncobj_create() as a file descriptor. 669 * 670 * Returns 0 on success or a negative error value on failure. 671 */ 672int drm_syncobj_get_fd(struct drm_syncobj *syncobj, int *p_fd) 673{ 674 struct file *file; 675 int fd; 676 677 fd = get_unused_fd_flags(O_CLOEXEC); 678 if (fd < 0) 679 return fd; 680 681 file = anon_inode_getfile("syncobj_file", 682 &drm_syncobj_file_fops, 683 syncobj, 0); 684 if (IS_ERR(file)) { 685 put_unused_fd(fd); 686 return PTR_ERR(file); 687 } 688 689 drm_syncobj_get(syncobj); 690 fd_install(fd, file); 691 692 *p_fd = fd; 693 return 0; 694} 695EXPORT_SYMBOL(drm_syncobj_get_fd); 696 697static int drm_syncobj_handle_to_fd(struct drm_file *file_private, 698 u32 handle, int *p_fd) 699{ 700 struct drm_syncobj *syncobj = drm_syncobj_find(file_private, handle); 701 int ret; 702 703 if (!syncobj) 704 return -EINVAL; 705 706 ret = drm_syncobj_get_fd(syncobj, p_fd); 707 drm_syncobj_put(syncobj); 708 return ret; 709} 710 711static int drm_syncobj_fd_to_handle(struct drm_file *file_private, 712 int fd, u32 *handle) 713{ 714 struct drm_syncobj *syncobj; 715 struct fd f = fdget(fd); 716 int ret; 717 718 if (!f.file) 719 return -EINVAL; 720 721 if (f.file->f_op != &drm_syncobj_file_fops) { 722 fdput(f); 723 return -EINVAL; 724 } 725 726 /* take a reference to put in the idr */ 727 syncobj = f.file->private_data; 728 drm_syncobj_get(syncobj); 729 730 idr_preload(GFP_KERNEL); 731 spin_lock(&file_private->syncobj_table_lock); 732 ret = idr_alloc(&file_private->syncobj_idr, syncobj, 1, 0, GFP_NOWAIT); 733 spin_unlock(&file_private->syncobj_table_lock); 734 idr_preload_end(); 735 736 if (ret > 0) { 737 *handle = ret; 738 ret = 0; 739 } else 740 drm_syncobj_put(syncobj); 741 742 fdput(f); 743 return ret; 744} 745 746static int drm_syncobj_import_sync_file_fence(struct drm_file *file_private, 747 int fd, int handle) 748{ 749 struct dma_fence *fence = sync_file_get_fence(fd); 750 struct drm_syncobj *syncobj; 751 752 if (!fence) 753 return -EINVAL; 754 755 syncobj = drm_syncobj_find(file_private, handle); 756 if (!syncobj) { 757 dma_fence_put(fence); 758 return -ENOENT; 759 } 760 761 drm_syncobj_replace_fence(syncobj, fence); 762 dma_fence_put(fence); 763 drm_syncobj_put(syncobj); 764 return 0; 765} 766 767static int drm_syncobj_export_sync_file(struct drm_file *file_private, 768 int handle, int *p_fd) 769{ 770 int ret; 771 struct dma_fence *fence; 772 struct sync_file *sync_file; 773 int fd = get_unused_fd_flags(O_CLOEXEC); 774 775 if (fd < 0) 776 return fd; 777 778 ret = drm_syncobj_find_fence(file_private, handle, 0, 0, &fence); 779 if (ret) 780 goto err_put_fd; 781 782 sync_file = sync_file_create(fence); 783 784 dma_fence_put(fence); 785 786 if (!sync_file) { 787 ret = -EINVAL; 788 goto err_put_fd; 789 } 790 791 fd_install(fd, sync_file->file); 792 793 *p_fd = fd; 794 return 0; 795err_put_fd: 796 put_unused_fd(fd); 797 return ret; 798} 799/** 800 * drm_syncobj_open - initializes syncobj file-private structures at devnode open time 801 * @file_private: drm file-private structure to set up 802 * 803 * Called at device open time, sets up the structure for handling refcounting 804 * of sync objects. 805 */ 806void 807drm_syncobj_open(struct drm_file *file_private) 808{ 809 idr_init_base(&file_private->syncobj_idr, 1); 810 spin_lock_init(&file_private->syncobj_table_lock); 811} 812 813static int 814drm_syncobj_release_handle(int id, void *ptr, void *data) 815{ 816 struct drm_syncobj *syncobj = ptr; 817 818 drm_syncobj_put(syncobj); 819 return 0; 820} 821 822/** 823 * drm_syncobj_release - release file-private sync object resources 824 * @file_private: drm file-private structure to clean up 825 * 826 * Called at close time when the filp is going away. 827 * 828 * Releases any remaining references on objects by this filp. 829 */ 830void 831drm_syncobj_release(struct drm_file *file_private) 832{ 833 idr_for_each(&file_private->syncobj_idr, 834 &drm_syncobj_release_handle, file_private); 835 idr_destroy(&file_private->syncobj_idr); 836} 837 838int 839drm_syncobj_create_ioctl(struct drm_device *dev, void *data, 840 struct drm_file *file_private) 841{ 842 struct drm_syncobj_create *args = data; 843 844 if (!drm_core_check_feature(dev, DRIVER_SYNCOBJ)) 845 return -EOPNOTSUPP; 846 847 /* no valid flags yet */ 848 if (args->flags & ~DRM_SYNCOBJ_CREATE_SIGNALED) 849 return -EINVAL; 850 851 return drm_syncobj_create_as_handle(file_private, 852 &args->handle, args->flags); 853} 854 855int 856drm_syncobj_destroy_ioctl(struct drm_device *dev, void *data, 857 struct drm_file *file_private) 858{ 859 struct drm_syncobj_destroy *args = data; 860 861 if (!drm_core_check_feature(dev, DRIVER_SYNCOBJ)) 862 return -EOPNOTSUPP; 863 864 /* make sure padding is empty */ 865 if (args->pad) 866 return -EINVAL; 867 return drm_syncobj_destroy(file_private, args->handle); 868} 869 870int 871drm_syncobj_handle_to_fd_ioctl(struct drm_device *dev, void *data, 872 struct drm_file *file_private) 873{ 874 struct drm_syncobj_handle *args = data; 875 876 if (!drm_core_check_feature(dev, DRIVER_SYNCOBJ)) 877 return -EOPNOTSUPP; 878 879 if (args->pad) 880 return -EINVAL; 881 882 if (args->flags != 0 && 883 args->flags != DRM_SYNCOBJ_HANDLE_TO_FD_FLAGS_EXPORT_SYNC_FILE) 884 return -EINVAL; 885 886 if (args->flags & DRM_SYNCOBJ_HANDLE_TO_FD_FLAGS_EXPORT_SYNC_FILE) 887 return drm_syncobj_export_sync_file(file_private, args->handle, 888 &args->fd); 889 890 return drm_syncobj_handle_to_fd(file_private, args->handle, 891 &args->fd); 892} 893 894int 895drm_syncobj_fd_to_handle_ioctl(struct drm_device *dev, void *data, 896 struct drm_file *file_private) 897{ 898 struct drm_syncobj_handle *args = data; 899 900 if (!drm_core_check_feature(dev, DRIVER_SYNCOBJ)) 901 return -EOPNOTSUPP; 902 903 if (args->pad) 904 return -EINVAL; 905 906 if (args->flags != 0 && 907 args->flags != DRM_SYNCOBJ_FD_TO_HANDLE_FLAGS_IMPORT_SYNC_FILE) 908 return -EINVAL; 909 910 if (args->flags & DRM_SYNCOBJ_FD_TO_HANDLE_FLAGS_IMPORT_SYNC_FILE) 911 return drm_syncobj_import_sync_file_fence(file_private, 912 args->fd, 913 args->handle); 914 915 return drm_syncobj_fd_to_handle(file_private, args->fd, 916 &args->handle); 917} 918 919static int drm_syncobj_transfer_to_timeline(struct drm_file *file_private, 920 struct drm_syncobj_transfer *args) 921{ 922 struct drm_syncobj *timeline_syncobj = NULL; 923 struct dma_fence *fence, *tmp; 924 struct dma_fence_chain *chain; 925 int ret; 926 927 timeline_syncobj = drm_syncobj_find(file_private, args->dst_handle); 928 if (!timeline_syncobj) { 929 return -ENOENT; 930 } 931 ret = drm_syncobj_find_fence(file_private, args->src_handle, 932 args->src_point, args->flags, 933 &tmp); 934 if (ret) 935 goto err_put_timeline; 936 937 fence = dma_fence_unwrap_merge(tmp); 938 dma_fence_put(tmp); 939 if (!fence) { 940 ret = -ENOMEM; 941 goto err_put_timeline; 942 } 943 944 chain = dma_fence_chain_alloc(); 945 if (!chain) { 946 ret = -ENOMEM; 947 goto err_free_fence; 948 } 949 950 drm_syncobj_add_point(timeline_syncobj, chain, fence, args->dst_point); 951err_free_fence: 952 dma_fence_put(fence); 953err_put_timeline: 954 drm_syncobj_put(timeline_syncobj); 955 956 return ret; 957} 958 959static int 960drm_syncobj_transfer_to_binary(struct drm_file *file_private, 961 struct drm_syncobj_transfer *args) 962{ 963 struct drm_syncobj *binary_syncobj = NULL; 964 struct dma_fence *fence; 965 int ret; 966 967 binary_syncobj = drm_syncobj_find(file_private, args->dst_handle); 968 if (!binary_syncobj) 969 return -ENOENT; 970 ret = drm_syncobj_find_fence(file_private, args->src_handle, 971 args->src_point, args->flags, &fence); 972 if (ret) 973 goto err; 974 drm_syncobj_replace_fence(binary_syncobj, fence); 975 dma_fence_put(fence); 976err: 977 drm_syncobj_put(binary_syncobj); 978 979 return ret; 980} 981int 982drm_syncobj_transfer_ioctl(struct drm_device *dev, void *data, 983 struct drm_file *file_private) 984{ 985 struct drm_syncobj_transfer *args = data; 986 int ret; 987 988 if (!drm_core_check_feature(dev, DRIVER_SYNCOBJ_TIMELINE)) 989 return -EOPNOTSUPP; 990 991 if (args->pad) 992 return -EINVAL; 993 994 if (args->dst_point) 995 ret = drm_syncobj_transfer_to_timeline(file_private, args); 996 else 997 ret = drm_syncobj_transfer_to_binary(file_private, args); 998 999 return ret; 1000} 1001 1002static void syncobj_wait_fence_func(struct dma_fence *fence, 1003 struct dma_fence_cb *cb) 1004{ 1005 struct syncobj_wait_entry *wait = 1006 container_of(cb, struct syncobj_wait_entry, fence_cb); 1007 1008 wake_up_process(wait->task); 1009} 1010 1011static void syncobj_wait_syncobj_func(struct drm_syncobj *syncobj, 1012 struct syncobj_wait_entry *wait) 1013{ 1014 struct dma_fence *fence; 1015 1016 /* This happens inside the syncobj lock */ 1017 fence = rcu_dereference_protected(syncobj->fence, 1018 lockdep_is_held(&syncobj->lock)); 1019 dma_fence_get(fence); 1020 if (!fence || dma_fence_chain_find_seqno(&fence, wait->point)) { 1021 dma_fence_put(fence); 1022 return; 1023 } else if (!fence) { 1024 wait->fence = dma_fence_get_stub(); 1025 } else { 1026 wait->fence = fence; 1027 } 1028 1029 wake_up_process(wait->task); 1030 list_del_init(&wait->node); 1031} 1032 1033static signed long drm_syncobj_array_wait_timeout(struct drm_syncobj **syncobjs, 1034 void __user *user_points, 1035 uint32_t count, 1036 uint32_t flags, 1037 signed long timeout, 1038 uint32_t *idx, 1039 ktime_t *deadline) 1040{ 1041 struct syncobj_wait_entry *entries; 1042 struct dma_fence *fence; 1043 uint64_t *points; 1044 uint32_t signaled_count, i; 1045 1046 if (flags & (DRM_SYNCOBJ_WAIT_FLAGS_WAIT_FOR_SUBMIT | 1047 DRM_SYNCOBJ_WAIT_FLAGS_WAIT_AVAILABLE)) { 1048 might_sleep(); 1049 lockdep_assert_none_held_once(); 1050 } 1051 1052 points = kmalloc_array(count, sizeof(*points), GFP_KERNEL); 1053 if (points == NULL) 1054 return -ENOMEM; 1055 1056 if (!user_points) { 1057 memset(points, 0, count * sizeof(uint64_t)); 1058 1059 } else if (copy_from_user(points, user_points, 1060 sizeof(uint64_t) * count)) { 1061 timeout = -EFAULT; 1062 goto err_free_points; 1063 } 1064 1065 entries = kcalloc(count, sizeof(*entries), GFP_KERNEL); 1066 if (!entries) { 1067 timeout = -ENOMEM; 1068 goto err_free_points; 1069 } 1070 /* Walk the list of sync objects and initialize entries. We do 1071 * this up-front so that we can properly return -EINVAL if there is 1072 * a syncobj with a missing fence and then never have the chance of 1073 * returning -EINVAL again. 1074 */ 1075 signaled_count = 0; 1076 for (i = 0; i < count; ++i) { 1077 struct dma_fence *fence; 1078 1079 entries[i].task = current; 1080 entries[i].point = points[i]; 1081 fence = drm_syncobj_fence_get(syncobjs[i]); 1082 if (!fence || dma_fence_chain_find_seqno(&fence, points[i])) { 1083 dma_fence_put(fence); 1084 if (flags & (DRM_SYNCOBJ_WAIT_FLAGS_WAIT_FOR_SUBMIT | 1085 DRM_SYNCOBJ_WAIT_FLAGS_WAIT_AVAILABLE)) { 1086 continue; 1087 } else { 1088 timeout = -EINVAL; 1089 goto cleanup_entries; 1090 } 1091 } 1092 1093 if (fence) 1094 entries[i].fence = fence; 1095 else 1096 entries[i].fence = dma_fence_get_stub(); 1097 1098 if ((flags & DRM_SYNCOBJ_WAIT_FLAGS_WAIT_AVAILABLE) || 1099 dma_fence_is_signaled(entries[i].fence)) { 1100 if (signaled_count == 0 && idx) 1101 *idx = i; 1102 signaled_count++; 1103 } 1104 } 1105 1106 if (signaled_count == count || 1107 (signaled_count > 0 && 1108 !(flags & DRM_SYNCOBJ_WAIT_FLAGS_WAIT_ALL))) 1109 goto cleanup_entries; 1110 1111 /* There's a very annoying laxness in the dma_fence API here, in 1112 * that backends are not required to automatically report when a 1113 * fence is signaled prior to fence->ops->enable_signaling() being 1114 * called. So here if we fail to match signaled_count, we need to 1115 * fallthough and try a 0 timeout wait! 1116 */ 1117 1118 if (flags & (DRM_SYNCOBJ_WAIT_FLAGS_WAIT_FOR_SUBMIT | 1119 DRM_SYNCOBJ_WAIT_FLAGS_WAIT_AVAILABLE)) { 1120 for (i = 0; i < count; ++i) 1121 drm_syncobj_fence_add_wait(syncobjs[i], &entries[i]); 1122 } 1123 1124 if (deadline) { 1125 for (i = 0; i < count; ++i) { 1126 fence = entries[i].fence; 1127 if (!fence) 1128 continue; 1129 dma_fence_set_deadline(fence, *deadline); 1130 } 1131 } 1132 1133 do { 1134 set_current_state(TASK_INTERRUPTIBLE); 1135 1136 signaled_count = 0; 1137 for (i = 0; i < count; ++i) { 1138 fence = entries[i].fence; 1139 if (!fence) 1140 continue; 1141 1142 if ((flags & DRM_SYNCOBJ_WAIT_FLAGS_WAIT_AVAILABLE) || 1143 dma_fence_is_signaled(fence) || 1144 (!entries[i].fence_cb.func && 1145 dma_fence_add_callback(fence, 1146 &entries[i].fence_cb, 1147 syncobj_wait_fence_func))) { 1148 /* The fence has been signaled */ 1149 if (flags & DRM_SYNCOBJ_WAIT_FLAGS_WAIT_ALL) { 1150 signaled_count++; 1151 } else { 1152 if (idx) 1153 *idx = i; 1154 goto done_waiting; 1155 } 1156 } 1157 } 1158 1159 if (signaled_count == count) 1160 goto done_waiting; 1161 1162 if (timeout == 0) { 1163 timeout = -ETIME; 1164 goto done_waiting; 1165 } 1166 1167 if (signal_pending(current)) { 1168 timeout = -ERESTARTSYS; 1169 goto done_waiting; 1170 } 1171 1172 timeout = schedule_timeout(timeout); 1173 } while (1); 1174 1175done_waiting: 1176 __set_current_state(TASK_RUNNING); 1177 1178cleanup_entries: 1179 for (i = 0; i < count; ++i) { 1180 drm_syncobj_remove_wait(syncobjs[i], &entries[i]); 1181 if (entries[i].fence_cb.func) 1182 dma_fence_remove_callback(entries[i].fence, 1183 &entries[i].fence_cb); 1184 dma_fence_put(entries[i].fence); 1185 } 1186 kfree(entries); 1187 1188err_free_points: 1189 kfree(points); 1190 1191 return timeout; 1192} 1193 1194/** 1195 * drm_timeout_abs_to_jiffies - calculate jiffies timeout from absolute value 1196 * 1197 * @timeout_nsec: timeout nsec component in ns, 0 for poll 1198 * 1199 * Calculate the timeout in jiffies from an absolute time in sec/nsec. 1200 */ 1201signed long drm_timeout_abs_to_jiffies(int64_t timeout_nsec) 1202{ 1203 ktime_t abs_timeout, now; 1204 u64 timeout_ns, timeout_jiffies64; 1205 1206 /* make 0 timeout means poll - absolute 0 doesn't seem valid */ 1207 if (timeout_nsec == 0) 1208 return 0; 1209 1210 abs_timeout = ns_to_ktime(timeout_nsec); 1211 now = ktime_get(); 1212 1213 if (!ktime_after(abs_timeout, now)) 1214 return 0; 1215 1216 timeout_ns = ktime_to_ns(ktime_sub(abs_timeout, now)); 1217 1218 timeout_jiffies64 = nsecs_to_jiffies64(timeout_ns); 1219 /* clamp timeout to avoid infinite timeout */ 1220 if (timeout_jiffies64 >= MAX_SCHEDULE_TIMEOUT - 1) 1221 return MAX_SCHEDULE_TIMEOUT - 1; 1222 1223 return timeout_jiffies64 + 1; 1224} 1225EXPORT_SYMBOL(drm_timeout_abs_to_jiffies); 1226 1227static int drm_syncobj_array_wait(struct drm_device *dev, 1228 struct drm_file *file_private, 1229 struct drm_syncobj_wait *wait, 1230 struct drm_syncobj_timeline_wait *timeline_wait, 1231 struct drm_syncobj **syncobjs, bool timeline, 1232 ktime_t *deadline) 1233{ 1234 signed long timeout = 0; 1235 uint32_t first = ~0; 1236 1237 if (!timeline) { 1238 timeout = drm_timeout_abs_to_jiffies(wait->timeout_nsec); 1239 timeout = drm_syncobj_array_wait_timeout(syncobjs, 1240 NULL, 1241 wait->count_handles, 1242 wait->flags, 1243 timeout, &first, 1244 deadline); 1245 if (timeout < 0) 1246 return timeout; 1247 wait->first_signaled = first; 1248 } else { 1249 timeout = drm_timeout_abs_to_jiffies(timeline_wait->timeout_nsec); 1250 timeout = drm_syncobj_array_wait_timeout(syncobjs, 1251 u64_to_user_ptr(timeline_wait->points), 1252 timeline_wait->count_handles, 1253 timeline_wait->flags, 1254 timeout, &first, 1255 deadline); 1256 if (timeout < 0) 1257 return timeout; 1258 timeline_wait->first_signaled = first; 1259 } 1260 return 0; 1261} 1262 1263static int drm_syncobj_array_find(struct drm_file *file_private, 1264 void __user *user_handles, 1265 uint32_t count_handles, 1266 struct drm_syncobj ***syncobjs_out) 1267{ 1268 uint32_t i, *handles; 1269 struct drm_syncobj **syncobjs; 1270 int ret; 1271 1272 handles = kmalloc_array(count_handles, sizeof(*handles), GFP_KERNEL); 1273 if (handles == NULL) 1274 return -ENOMEM; 1275 1276 if (copy_from_user(handles, user_handles, 1277 sizeof(uint32_t) * count_handles)) { 1278 ret = -EFAULT; 1279 goto err_free_handles; 1280 } 1281 1282 syncobjs = kmalloc_array(count_handles, sizeof(*syncobjs), GFP_KERNEL); 1283 if (syncobjs == NULL) { 1284 ret = -ENOMEM; 1285 goto err_free_handles; 1286 } 1287 1288 for (i = 0; i < count_handles; i++) { 1289 syncobjs[i] = drm_syncobj_find(file_private, handles[i]); 1290 if (!syncobjs[i]) { 1291 ret = -ENOENT; 1292 goto err_put_syncobjs; 1293 } 1294 } 1295 1296 kfree(handles); 1297 *syncobjs_out = syncobjs; 1298 return 0; 1299 1300err_put_syncobjs: 1301 while (i-- > 0) 1302 drm_syncobj_put(syncobjs[i]); 1303 kfree(syncobjs); 1304err_free_handles: 1305 kfree(handles); 1306 1307 return ret; 1308} 1309 1310static void drm_syncobj_array_free(struct drm_syncobj **syncobjs, 1311 uint32_t count) 1312{ 1313 uint32_t i; 1314 1315 for (i = 0; i < count; i++) 1316 drm_syncobj_put(syncobjs[i]); 1317 kfree(syncobjs); 1318} 1319 1320int 1321drm_syncobj_wait_ioctl(struct drm_device *dev, void *data, 1322 struct drm_file *file_private) 1323{ 1324 struct drm_syncobj_wait *args = data; 1325 struct drm_syncobj **syncobjs; 1326 unsigned int possible_flags; 1327 ktime_t t, *tp = NULL; 1328 int ret = 0; 1329 1330 if (!drm_core_check_feature(dev, DRIVER_SYNCOBJ)) 1331 return -EOPNOTSUPP; 1332 1333 possible_flags = DRM_SYNCOBJ_WAIT_FLAGS_WAIT_ALL | 1334 DRM_SYNCOBJ_WAIT_FLAGS_WAIT_FOR_SUBMIT | 1335 DRM_SYNCOBJ_WAIT_FLAGS_WAIT_DEADLINE; 1336 1337 if (args->flags & ~possible_flags) 1338 return -EINVAL; 1339 1340 if (args->count_handles == 0) 1341 return 0; 1342 1343 ret = drm_syncobj_array_find(file_private, 1344 u64_to_user_ptr(args->handles), 1345 args->count_handles, 1346 &syncobjs); 1347 if (ret < 0) 1348 return ret; 1349 1350 if (args->flags & DRM_SYNCOBJ_WAIT_FLAGS_WAIT_DEADLINE) { 1351 t = ns_to_ktime(args->deadline_nsec); 1352 tp = &t; 1353 } 1354 1355 ret = drm_syncobj_array_wait(dev, file_private, 1356 args, NULL, syncobjs, false, tp); 1357 1358 drm_syncobj_array_free(syncobjs, args->count_handles); 1359 1360 return ret; 1361} 1362 1363int 1364drm_syncobj_timeline_wait_ioctl(struct drm_device *dev, void *data, 1365 struct drm_file *file_private) 1366{ 1367 struct drm_syncobj_timeline_wait *args = data; 1368 struct drm_syncobj **syncobjs; 1369 unsigned int possible_flags; 1370 ktime_t t, *tp = NULL; 1371 int ret = 0; 1372 1373 if (!drm_core_check_feature(dev, DRIVER_SYNCOBJ_TIMELINE)) 1374 return -EOPNOTSUPP; 1375 1376 possible_flags = DRM_SYNCOBJ_WAIT_FLAGS_WAIT_ALL | 1377 DRM_SYNCOBJ_WAIT_FLAGS_WAIT_FOR_SUBMIT | 1378 DRM_SYNCOBJ_WAIT_FLAGS_WAIT_AVAILABLE | 1379 DRM_SYNCOBJ_WAIT_FLAGS_WAIT_DEADLINE; 1380 1381 if (args->flags & ~possible_flags) 1382 return -EINVAL; 1383 1384 if (args->count_handles == 0) 1385 return 0; 1386 1387 ret = drm_syncobj_array_find(file_private, 1388 u64_to_user_ptr(args->handles), 1389 args->count_handles, 1390 &syncobjs); 1391 if (ret < 0) 1392 return ret; 1393 1394 if (args->flags & DRM_SYNCOBJ_WAIT_FLAGS_WAIT_DEADLINE) { 1395 t = ns_to_ktime(args->deadline_nsec); 1396 tp = &t; 1397 } 1398 1399 ret = drm_syncobj_array_wait(dev, file_private, 1400 NULL, args, syncobjs, true, tp); 1401 1402 drm_syncobj_array_free(syncobjs, args->count_handles); 1403 1404 return ret; 1405} 1406 1407static void syncobj_eventfd_entry_fence_func(struct dma_fence *fence, 1408 struct dma_fence_cb *cb) 1409{ 1410 struct syncobj_eventfd_entry *entry = 1411 container_of(cb, struct syncobj_eventfd_entry, fence_cb); 1412 1413 eventfd_signal(entry->ev_fd_ctx); 1414 syncobj_eventfd_entry_free(entry); 1415} 1416 1417static void 1418syncobj_eventfd_entry_func(struct drm_syncobj *syncobj, 1419 struct syncobj_eventfd_entry *entry) 1420{ 1421 int ret; 1422 struct dma_fence *fence; 1423 1424 /* This happens inside the syncobj lock */ 1425 fence = dma_fence_get(rcu_dereference_protected(syncobj->fence, 1)); 1426 if (!fence) 1427 return; 1428 1429 ret = dma_fence_chain_find_seqno(&fence, entry->point); 1430 if (ret != 0) { 1431 /* The given seqno has not been submitted yet. */ 1432 dma_fence_put(fence); 1433 return; 1434 } else if (!fence) { 1435 /* If dma_fence_chain_find_seqno returns 0 but sets the fence 1436 * to NULL, it implies that the given seqno is signaled and a 1437 * later seqno has already been submitted. Assign a stub fence 1438 * so that the eventfd still gets signaled below. 1439 */ 1440 fence = dma_fence_get_stub(); 1441 } 1442 1443 list_del_init(&entry->node); 1444 entry->fence = fence; 1445 1446 if (entry->flags & DRM_SYNCOBJ_WAIT_FLAGS_WAIT_AVAILABLE) { 1447 eventfd_signal(entry->ev_fd_ctx); 1448 syncobj_eventfd_entry_free(entry); 1449 } else { 1450 ret = dma_fence_add_callback(fence, &entry->fence_cb, 1451 syncobj_eventfd_entry_fence_func); 1452 if (ret == -ENOENT) { 1453 eventfd_signal(entry->ev_fd_ctx); 1454 syncobj_eventfd_entry_free(entry); 1455 } 1456 } 1457} 1458 1459int 1460drm_syncobj_eventfd_ioctl(struct drm_device *dev, void *data, 1461 struct drm_file *file_private) 1462{ 1463 struct drm_syncobj_eventfd *args = data; 1464 struct drm_syncobj *syncobj; 1465 struct eventfd_ctx *ev_fd_ctx; 1466 struct syncobj_eventfd_entry *entry; 1467 1468 if (!drm_core_check_feature(dev, DRIVER_SYNCOBJ_TIMELINE)) 1469 return -EOPNOTSUPP; 1470 1471 if (args->flags & ~DRM_SYNCOBJ_WAIT_FLAGS_WAIT_AVAILABLE) 1472 return -EINVAL; 1473 1474 if (args->pad) 1475 return -EINVAL; 1476 1477 syncobj = drm_syncobj_find(file_private, args->handle); 1478 if (!syncobj) 1479 return -ENOENT; 1480 1481 ev_fd_ctx = eventfd_ctx_fdget(args->fd); 1482 if (IS_ERR(ev_fd_ctx)) 1483 return PTR_ERR(ev_fd_ctx); 1484 1485 entry = kzalloc(sizeof(*entry), GFP_KERNEL); 1486 if (!entry) { 1487 eventfd_ctx_put(ev_fd_ctx); 1488 return -ENOMEM; 1489 } 1490 entry->syncobj = syncobj; 1491 entry->ev_fd_ctx = ev_fd_ctx; 1492 entry->point = args->point; 1493 entry->flags = args->flags; 1494 1495 drm_syncobj_add_eventfd(syncobj, entry); 1496 drm_syncobj_put(syncobj); 1497 1498 return 0; 1499} 1500 1501int 1502drm_syncobj_reset_ioctl(struct drm_device *dev, void *data, 1503 struct drm_file *file_private) 1504{ 1505 struct drm_syncobj_array *args = data; 1506 struct drm_syncobj **syncobjs; 1507 uint32_t i; 1508 int ret; 1509 1510 if (!drm_core_check_feature(dev, DRIVER_SYNCOBJ)) 1511 return -EOPNOTSUPP; 1512 1513 if (args->pad != 0) 1514 return -EINVAL; 1515 1516 if (args->count_handles == 0) 1517 return -EINVAL; 1518 1519 ret = drm_syncobj_array_find(file_private, 1520 u64_to_user_ptr(args->handles), 1521 args->count_handles, 1522 &syncobjs); 1523 if (ret < 0) 1524 return ret; 1525 1526 for (i = 0; i < args->count_handles; i++) 1527 drm_syncobj_replace_fence(syncobjs[i], NULL); 1528 1529 drm_syncobj_array_free(syncobjs, args->count_handles); 1530 1531 return 0; 1532} 1533 1534int 1535drm_syncobj_signal_ioctl(struct drm_device *dev, void *data, 1536 struct drm_file *file_private) 1537{ 1538 struct drm_syncobj_array *args = data; 1539 struct drm_syncobj **syncobjs; 1540 uint32_t i; 1541 int ret; 1542 1543 if (!drm_core_check_feature(dev, DRIVER_SYNCOBJ)) 1544 return -EOPNOTSUPP; 1545 1546 if (args->pad != 0) 1547 return -EINVAL; 1548 1549 if (args->count_handles == 0) 1550 return -EINVAL; 1551 1552 ret = drm_syncobj_array_find(file_private, 1553 u64_to_user_ptr(args->handles), 1554 args->count_handles, 1555 &syncobjs); 1556 if (ret < 0) 1557 return ret; 1558 1559 for (i = 0; i < args->count_handles; i++) { 1560 ret = drm_syncobj_assign_null_handle(syncobjs[i]); 1561 if (ret < 0) 1562 break; 1563 } 1564 1565 drm_syncobj_array_free(syncobjs, args->count_handles); 1566 1567 return ret; 1568} 1569 1570int 1571drm_syncobj_timeline_signal_ioctl(struct drm_device *dev, void *data, 1572 struct drm_file *file_private) 1573{ 1574 struct drm_syncobj_timeline_array *args = data; 1575 struct drm_syncobj **syncobjs; 1576 struct dma_fence_chain **chains; 1577 uint64_t *points; 1578 uint32_t i, j; 1579 int ret; 1580 1581 if (!drm_core_check_feature(dev, DRIVER_SYNCOBJ_TIMELINE)) 1582 return -EOPNOTSUPP; 1583 1584 if (args->flags != 0) 1585 return -EINVAL; 1586 1587 if (args->count_handles == 0) 1588 return -EINVAL; 1589 1590 ret = drm_syncobj_array_find(file_private, 1591 u64_to_user_ptr(args->handles), 1592 args->count_handles, 1593 &syncobjs); 1594 if (ret < 0) 1595 return ret; 1596 1597 points = kmalloc_array(args->count_handles, sizeof(*points), 1598 GFP_KERNEL); 1599 if (!points) { 1600 ret = -ENOMEM; 1601 goto out; 1602 } 1603 if (!u64_to_user_ptr(args->points)) { 1604 memset(points, 0, args->count_handles * sizeof(uint64_t)); 1605 } else if (copy_from_user(points, u64_to_user_ptr(args->points), 1606 sizeof(uint64_t) * args->count_handles)) { 1607 ret = -EFAULT; 1608 goto err_points; 1609 } 1610 1611 chains = kmalloc_array(args->count_handles, sizeof(void *), GFP_KERNEL); 1612 if (!chains) { 1613 ret = -ENOMEM; 1614 goto err_points; 1615 } 1616 for (i = 0; i < args->count_handles; i++) { 1617 chains[i] = dma_fence_chain_alloc(); 1618 if (!chains[i]) { 1619 for (j = 0; j < i; j++) 1620 dma_fence_chain_free(chains[j]); 1621 ret = -ENOMEM; 1622 goto err_chains; 1623 } 1624 } 1625 1626 for (i = 0; i < args->count_handles; i++) { 1627 struct dma_fence *fence = dma_fence_get_stub(); 1628 1629 drm_syncobj_add_point(syncobjs[i], chains[i], 1630 fence, points[i]); 1631 dma_fence_put(fence); 1632 } 1633err_chains: 1634 kfree(chains); 1635err_points: 1636 kfree(points); 1637out: 1638 drm_syncobj_array_free(syncobjs, args->count_handles); 1639 1640 return ret; 1641} 1642 1643int drm_syncobj_query_ioctl(struct drm_device *dev, void *data, 1644 struct drm_file *file_private) 1645{ 1646 struct drm_syncobj_timeline_array *args = data; 1647 struct drm_syncobj **syncobjs; 1648 uint64_t __user *points = u64_to_user_ptr(args->points); 1649 uint32_t i; 1650 int ret; 1651 1652 if (!drm_core_check_feature(dev, DRIVER_SYNCOBJ_TIMELINE)) 1653 return -EOPNOTSUPP; 1654 1655 if (args->flags & ~DRM_SYNCOBJ_QUERY_FLAGS_LAST_SUBMITTED) 1656 return -EINVAL; 1657 1658 if (args->count_handles == 0) 1659 return -EINVAL; 1660 1661 ret = drm_syncobj_array_find(file_private, 1662 u64_to_user_ptr(args->handles), 1663 args->count_handles, 1664 &syncobjs); 1665 if (ret < 0) 1666 return ret; 1667 1668 for (i = 0; i < args->count_handles; i++) { 1669 struct dma_fence_chain *chain; 1670 struct dma_fence *fence; 1671 uint64_t point; 1672 1673 fence = drm_syncobj_fence_get(syncobjs[i]); 1674 chain = to_dma_fence_chain(fence); 1675 if (chain) { 1676 struct dma_fence *iter, *last_signaled = 1677 dma_fence_get(fence); 1678 1679 if (args->flags & 1680 DRM_SYNCOBJ_QUERY_FLAGS_LAST_SUBMITTED) { 1681 point = fence->seqno; 1682 } else { 1683 dma_fence_chain_for_each(iter, fence) { 1684 if (iter->context != fence->context) { 1685 dma_fence_put(iter); 1686 /* It is most likely that timeline has 1687 * unorder points. */ 1688 break; 1689 } 1690 dma_fence_put(last_signaled); 1691 last_signaled = dma_fence_get(iter); 1692 } 1693 point = dma_fence_is_signaled(last_signaled) ? 1694 last_signaled->seqno : 1695 to_dma_fence_chain(last_signaled)->prev_seqno; 1696 } 1697 dma_fence_put(last_signaled); 1698 } else { 1699 point = 0; 1700 } 1701 dma_fence_put(fence); 1702 ret = copy_to_user(&points[i], &point, sizeof(uint64_t)); 1703 ret = ret ? -EFAULT : 0; 1704 if (ret) 1705 break; 1706 } 1707 drm_syncobj_array_free(syncobjs, args->count_handles); 1708 1709 return ret; 1710} 1711