1/* 2 * drm_irq.c IRQ and vblank support 3 * 4 * \author Rickard E. (Rik) Faith <faith@valinux.com> 5 * \author Gareth Hughes <gareth@valinux.com> 6 * 7 * Permission is hereby granted, free of charge, to any person obtaining a 8 * copy of this software and associated documentation files (the "Software"), 9 * to deal in the Software without restriction, including without limitation 10 * the rights to use, copy, modify, merge, publish, distribute, sublicense, 11 * and/or sell copies of the Software, and to permit persons to whom the 12 * Software is furnished to do so, subject to the following conditions: 13 * 14 * The above copyright notice and this permission notice (including the next 15 * paragraph) shall be included in all copies or substantial portions of the 16 * Software. 17 * 18 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 19 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 20 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 21 * VA LINUX SYSTEMS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM, DAMAGES OR 22 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, 23 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR 24 * OTHER DEALINGS IN THE SOFTWARE. 25 */ 26 27#include <linux/export.h> 28#include <linux/kthread.h> 29#include <linux/moduleparam.h> 30 31#include <drm/drm_crtc.h> 32#include <drm/drm_drv.h> 33#include <drm/drm_framebuffer.h> 34#include <drm/drm_managed.h> 35#include <drm/drm_modeset_helper_vtables.h> 36#include <drm/drm_print.h> 37#include <drm/drm_vblank.h> 38 39#include "drm_internal.h" 40#include "drm_trace.h" 41 42/** 43 * DOC: vblank handling 44 * 45 * From the computer's perspective, every time the monitor displays 46 * a new frame the scanout engine has "scanned out" the display image 47 * from top to bottom, one row of pixels at a time. The current row 48 * of pixels is referred to as the current scanline. 49 * 50 * In addition to the display's visible area, there's usually a couple of 51 * extra scanlines which aren't actually displayed on the screen. 52 * These extra scanlines don't contain image data and are occasionally used 53 * for features like audio and infoframes. The region made up of these 54 * scanlines is referred to as the vertical blanking region, or vblank for 55 * short. 56 * 57 * For historical reference, the vertical blanking period was designed to 58 * give the electron gun (on CRTs) enough time to move back to the top of 59 * the screen to start scanning out the next frame. Similar for horizontal 60 * blanking periods. They were designed to give the electron gun enough 61 * time to move back to the other side of the screen to start scanning the 62 * next scanline. 63 * 64 * :: 65 * 66 * 67 * physical ��� ������������������������������������������������������������������������������������������������������������������������ 68 * top of | | 69 * display | | 70 * | New frame | 71 * | | 72 * |������������������������������������������������������������������������������������������������������������������������| 73 * |~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~| ��� Scanline, 74 * |������������������������������������������������������������������������������������������������������������������������| updates the 75 * | | frame as it 76 * | | travels down 77 * | | ("scan out") 78 * | Old frame | 79 * | | 80 * | | 81 * | | 82 * | | physical 83 * | | bottom of 84 * vertical |������������������������������������������������������������������������������������������������������������������������| ��� display 85 * blanking ���xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx��� 86 * region ��� ���xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx��� 87 * ���xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx��� 88 * start of ��� ������������������������������������������������������������������������������������������������������������������������ 89 * new frame 90 * 91 * "Physical top of display" is the reference point for the high-precision/ 92 * corrected timestamp. 93 * 94 * On a lot of display hardware, programming needs to take effect during the 95 * vertical blanking period so that settings like gamma, the image buffer 96 * buffer to be scanned out, etc. can safely be changed without showing 97 * any visual artifacts on the screen. In some unforgiving hardware, some of 98 * this programming has to both start and end in the same vblank. To help 99 * with the timing of the hardware programming, an interrupt is usually 100 * available to notify the driver when it can start the updating of registers. 101 * The interrupt is in this context named the vblank interrupt. 102 * 103 * The vblank interrupt may be fired at different points depending on the 104 * hardware. Some hardware implementations will fire the interrupt when the 105 * new frame start, other implementations will fire the interrupt at different 106 * points in time. 107 * 108 * Vertical blanking plays a major role in graphics rendering. To achieve 109 * tear-free display, users must synchronize page flips and/or rendering to 110 * vertical blanking. The DRM API offers ioctls to perform page flips 111 * synchronized to vertical blanking and wait for vertical blanking. 112 * 113 * The DRM core handles most of the vertical blanking management logic, which 114 * involves filtering out spurious interrupts, keeping race-free blanking 115 * counters, coping with counter wrap-around and resets and keeping use counts. 116 * It relies on the driver to generate vertical blanking interrupts and 117 * optionally provide a hardware vertical blanking counter. 118 * 119 * Drivers must initialize the vertical blanking handling core with a call to 120 * drm_vblank_init(). Minimally, a driver needs to implement 121 * &drm_crtc_funcs.enable_vblank and &drm_crtc_funcs.disable_vblank plus call 122 * drm_crtc_handle_vblank() in its vblank interrupt handler for working vblank 123 * support. 124 * 125 * Vertical blanking interrupts can be enabled by the DRM core or by drivers 126 * themselves (for instance to handle page flipping operations). The DRM core 127 * maintains a vertical blanking use count to ensure that the interrupts are not 128 * disabled while a user still needs them. To increment the use count, drivers 129 * call drm_crtc_vblank_get() and release the vblank reference again with 130 * drm_crtc_vblank_put(). In between these two calls vblank interrupts are 131 * guaranteed to be enabled. 132 * 133 * On many hardware disabling the vblank interrupt cannot be done in a race-free 134 * manner, see &drm_driver.vblank_disable_immediate and 135 * &drm_driver.max_vblank_count. In that case the vblank core only disables the 136 * vblanks after a timer has expired, which can be configured through the 137 * ``vblankoffdelay`` module parameter. 138 * 139 * Drivers for hardware without support for vertical-blanking interrupts 140 * must not call drm_vblank_init(). For such drivers, atomic helpers will 141 * automatically generate fake vblank events as part of the display update. 142 * This functionality also can be controlled by the driver by enabling and 143 * disabling struct drm_crtc_state.no_vblank. 144 */ 145 146/* Retry timestamp calculation up to 3 times to satisfy 147 * drm_timestamp_precision before giving up. 148 */ 149#define DRM_TIMESTAMP_MAXRETRIES 3 150 151/* Threshold in nanoseconds for detection of redundant 152 * vblank irq in drm_handle_vblank(). 1 msec should be ok. 153 */ 154#define DRM_REDUNDANT_VBLIRQ_THRESH_NS 1000000 155 156static bool 157drm_get_last_vbltimestamp(struct drm_device *dev, unsigned int pipe, 158 ktime_t *tvblank, bool in_vblank_irq); 159 160static unsigned int drm_timestamp_precision = 20; /* Default to 20 usecs. */ 161 162static int drm_vblank_offdelay = 5000; /* Default to 5000 msecs. */ 163 164module_param_named(vblankoffdelay, drm_vblank_offdelay, int, 0600); 165module_param_named(timestamp_precision_usec, drm_timestamp_precision, int, 0600); 166MODULE_PARM_DESC(vblankoffdelay, "Delay until vblank irq auto-disable [msecs] (0: never disable, <0: disable immediately)"); 167MODULE_PARM_DESC(timestamp_precision_usec, "Max. error on timestamps [usecs]"); 168 169static void store_vblank(struct drm_device *dev, unsigned int pipe, 170 u32 vblank_count_inc, 171 ktime_t t_vblank, u32 last) 172{ 173 struct drm_vblank_crtc *vblank = &dev->vblank[pipe]; 174 175 assert_spin_locked(&dev->vblank_time_lock); 176 177 vblank->last = last; 178 179 write_seqlock(&vblank->seqlock); 180 vblank->time = t_vblank; 181 atomic64_add(vblank_count_inc, &vblank->count); 182 write_sequnlock(&vblank->seqlock); 183} 184 185static u32 drm_max_vblank_count(struct drm_device *dev, unsigned int pipe) 186{ 187 struct drm_vblank_crtc *vblank = &dev->vblank[pipe]; 188 189 return vblank->max_vblank_count ?: dev->max_vblank_count; 190} 191 192/* 193 * "No hw counter" fallback implementation of .get_vblank_counter() hook, 194 * if there is no usable hardware frame counter available. 195 */ 196static u32 drm_vblank_no_hw_counter(struct drm_device *dev, unsigned int pipe) 197{ 198 drm_WARN_ON_ONCE(dev, drm_max_vblank_count(dev, pipe) != 0); 199 return 0; 200} 201 202static u32 __get_vblank_counter(struct drm_device *dev, unsigned int pipe) 203{ 204 if (drm_core_check_feature(dev, DRIVER_MODESET)) { 205 struct drm_crtc *crtc = drm_crtc_from_index(dev, pipe); 206 207 if (drm_WARN_ON(dev, !crtc)) 208 return 0; 209 210 if (crtc->funcs->get_vblank_counter) 211 return crtc->funcs->get_vblank_counter(crtc); 212 } 213 214 return drm_vblank_no_hw_counter(dev, pipe); 215} 216 217/* 218 * Reset the stored timestamp for the current vblank count to correspond 219 * to the last vblank occurred. 220 * 221 * Only to be called from drm_crtc_vblank_on(). 222 * 223 * Note: caller must hold &drm_device.vbl_lock since this reads & writes 224 * device vblank fields. 225 */ 226static void drm_reset_vblank_timestamp(struct drm_device *dev, unsigned int pipe) 227{ 228 u32 cur_vblank; 229 bool rc; 230 ktime_t t_vblank; 231 int count = DRM_TIMESTAMP_MAXRETRIES; 232 233 spin_lock(&dev->vblank_time_lock); 234 235 /* 236 * sample the current counter to avoid random jumps 237 * when drm_vblank_enable() applies the diff 238 */ 239 do { 240 cur_vblank = __get_vblank_counter(dev, pipe); 241 rc = drm_get_last_vbltimestamp(dev, pipe, &t_vblank, false); 242 } while (cur_vblank != __get_vblank_counter(dev, pipe) && --count > 0); 243 244 /* 245 * Only reinitialize corresponding vblank timestamp if high-precision query 246 * available and didn't fail. Otherwise reinitialize delayed at next vblank 247 * interrupt and assign 0 for now, to mark the vblanktimestamp as invalid. 248 */ 249 if (!rc) 250 t_vblank = 0; 251 252 /* 253 * +1 to make sure user will never see the same 254 * vblank counter value before and after a modeset 255 */ 256 store_vblank(dev, pipe, 1, t_vblank, cur_vblank); 257 258 spin_unlock(&dev->vblank_time_lock); 259} 260 261/* 262 * Call back into the driver to update the appropriate vblank counter 263 * (specified by @pipe). Deal with wraparound, if it occurred, and 264 * update the last read value so we can deal with wraparound on the next 265 * call if necessary. 266 * 267 * Only necessary when going from off->on, to account for frames we 268 * didn't get an interrupt for. 269 * 270 * Note: caller must hold &drm_device.vbl_lock since this reads & writes 271 * device vblank fields. 272 */ 273static void drm_update_vblank_count(struct drm_device *dev, unsigned int pipe, 274 bool in_vblank_irq) 275{ 276 struct drm_vblank_crtc *vblank = &dev->vblank[pipe]; 277 u32 cur_vblank, diff; 278 bool rc; 279 ktime_t t_vblank; 280 int count = DRM_TIMESTAMP_MAXRETRIES; 281 int framedur_ns = vblank->framedur_ns; 282 u32 max_vblank_count = drm_max_vblank_count(dev, pipe); 283 284 /* 285 * Interrupts were disabled prior to this call, so deal with counter 286 * wrap if needed. 287 * NOTE! It's possible we lost a full dev->max_vblank_count + 1 events 288 * here if the register is small or we had vblank interrupts off for 289 * a long time. 290 * 291 * We repeat the hardware vblank counter & timestamp query until 292 * we get consistent results. This to prevent races between gpu 293 * updating its hardware counter while we are retrieving the 294 * corresponding vblank timestamp. 295 */ 296 do { 297 cur_vblank = __get_vblank_counter(dev, pipe); 298 rc = drm_get_last_vbltimestamp(dev, pipe, &t_vblank, in_vblank_irq); 299 } while (cur_vblank != __get_vblank_counter(dev, pipe) && --count > 0); 300 301 if (max_vblank_count) { 302 /* trust the hw counter when it's around */ 303 diff = (cur_vblank - vblank->last) & max_vblank_count; 304 } else if (rc && framedur_ns) { 305 u64 diff_ns = ktime_to_ns(ktime_sub(t_vblank, vblank->time)); 306 307 /* 308 * Figure out how many vblanks we've missed based 309 * on the difference in the timestamps and the 310 * frame/field duration. 311 */ 312 313 drm_dbg_vbl(dev, "crtc %u: Calculating number of vblanks." 314 " diff_ns = %lld, framedur_ns = %d)\n", 315 pipe, (long long)diff_ns, framedur_ns); 316 317 diff = DIV_ROUND_CLOSEST_ULL(diff_ns, framedur_ns); 318 319 if (diff == 0 && in_vblank_irq) 320 drm_dbg_vbl(dev, "crtc %u: Redundant vblirq ignored\n", 321 pipe); 322 } else { 323 /* some kind of default for drivers w/o accurate vbl timestamping */ 324 diff = in_vblank_irq ? 1 : 0; 325 } 326 327 /* 328 * Within a drm_vblank_pre_modeset - drm_vblank_post_modeset 329 * interval? If so then vblank irqs keep running and it will likely 330 * happen that the hardware vblank counter is not trustworthy as it 331 * might reset at some point in that interval and vblank timestamps 332 * are not trustworthy either in that interval. Iow. this can result 333 * in a bogus diff >> 1 which must be avoided as it would cause 334 * random large forward jumps of the software vblank counter. 335 */ 336 if (diff > 1 && (vblank->inmodeset & 0x2)) { 337 drm_dbg_vbl(dev, 338 "clamping vblank bump to 1 on crtc %u: diffr=%u" 339 " due to pre-modeset.\n", pipe, diff); 340 diff = 1; 341 } 342 343 drm_dbg_vbl(dev, "updating vblank count on crtc %u:" 344 " current=%llu, diff=%u, hw=%u hw_last=%u\n", 345 pipe, (unsigned long long)atomic64_read(&vblank->count), 346 diff, cur_vblank, vblank->last); 347 348 if (diff == 0) { 349 drm_WARN_ON_ONCE(dev, cur_vblank != vblank->last); 350 return; 351 } 352 353 /* 354 * Only reinitialize corresponding vblank timestamp if high-precision query 355 * available and didn't fail, or we were called from the vblank interrupt. 356 * Otherwise reinitialize delayed at next vblank interrupt and assign 0 357 * for now, to mark the vblanktimestamp as invalid. 358 */ 359 if (!rc && !in_vblank_irq) 360 t_vblank = 0; 361 362 store_vblank(dev, pipe, diff, t_vblank, cur_vblank); 363} 364 365u64 drm_vblank_count(struct drm_device *dev, unsigned int pipe) 366{ 367 struct drm_vblank_crtc *vblank = &dev->vblank[pipe]; 368 u64 count; 369 370 if (drm_WARN_ON(dev, pipe >= dev->num_crtcs)) 371 return 0; 372 373 count = atomic64_read(&vblank->count); 374 375 /* 376 * This read barrier corresponds to the implicit write barrier of the 377 * write seqlock in store_vblank(). Note that this is the only place 378 * where we need an explicit barrier, since all other access goes 379 * through drm_vblank_count_and_time(), which already has the required 380 * read barrier curtesy of the read seqlock. 381 */ 382 smp_rmb(); 383 384 return count; 385} 386 387/** 388 * drm_crtc_accurate_vblank_count - retrieve the master vblank counter 389 * @crtc: which counter to retrieve 390 * 391 * This function is similar to drm_crtc_vblank_count() but this function 392 * interpolates to handle a race with vblank interrupts using the high precision 393 * timestamping support. 394 * 395 * This is mostly useful for hardware that can obtain the scanout position, but 396 * doesn't have a hardware frame counter. 397 */ 398u64 drm_crtc_accurate_vblank_count(struct drm_crtc *crtc) 399{ 400 struct drm_device *dev = crtc->dev; 401 unsigned int pipe = drm_crtc_index(crtc); 402 u64 vblank; 403 unsigned long flags; 404 405 drm_WARN_ONCE(dev, drm_debug_enabled(DRM_UT_VBL) && 406 !crtc->funcs->get_vblank_timestamp, 407 "This function requires support for accurate vblank timestamps."); 408 409 spin_lock_irqsave(&dev->vblank_time_lock, flags); 410 411 drm_update_vblank_count(dev, pipe, false); 412 vblank = drm_vblank_count(dev, pipe); 413 414 spin_unlock_irqrestore(&dev->vblank_time_lock, flags); 415 416 return vblank; 417} 418EXPORT_SYMBOL(drm_crtc_accurate_vblank_count); 419 420static void __disable_vblank(struct drm_device *dev, unsigned int pipe) 421{ 422 if (drm_core_check_feature(dev, DRIVER_MODESET)) { 423 struct drm_crtc *crtc = drm_crtc_from_index(dev, pipe); 424 425 if (drm_WARN_ON(dev, !crtc)) 426 return; 427 428 if (crtc->funcs->disable_vblank) 429 crtc->funcs->disable_vblank(crtc); 430 } 431} 432 433/* 434 * Disable vblank irq's on crtc, make sure that last vblank count 435 * of hardware and corresponding consistent software vblank counter 436 * are preserved, even if there are any spurious vblank irq's after 437 * disable. 438 */ 439void drm_vblank_disable_and_save(struct drm_device *dev, unsigned int pipe) 440{ 441 struct drm_vblank_crtc *vblank = &dev->vblank[pipe]; 442 unsigned long irqflags; 443 444 assert_spin_locked(&dev->vbl_lock); 445 446 /* Prevent vblank irq processing while disabling vblank irqs, 447 * so no updates of timestamps or count can happen after we've 448 * disabled. Needed to prevent races in case of delayed irq's. 449 */ 450 spin_lock_irqsave(&dev->vblank_time_lock, irqflags); 451 452 /* 453 * Update vblank count and disable vblank interrupts only if the 454 * interrupts were enabled. This avoids calling the ->disable_vblank() 455 * operation in atomic context with the hardware potentially runtime 456 * suspended. 457 */ 458 if (!vblank->enabled) 459 goto out; 460 461 /* 462 * Update the count and timestamp to maintain the 463 * appearance that the counter has been ticking all along until 464 * this time. This makes the count account for the entire time 465 * between drm_crtc_vblank_on() and drm_crtc_vblank_off(). 466 */ 467 drm_update_vblank_count(dev, pipe, false); 468 __disable_vblank(dev, pipe); 469 vblank->enabled = false; 470 471out: 472 spin_unlock_irqrestore(&dev->vblank_time_lock, irqflags); 473} 474 475static void vblank_disable_fn(struct timer_list *t) 476{ 477 struct drm_vblank_crtc *vblank = from_timer(vblank, t, disable_timer); 478 struct drm_device *dev = vblank->dev; 479 unsigned int pipe = vblank->pipe; 480 unsigned long irqflags; 481 482 spin_lock_irqsave(&dev->vbl_lock, irqflags); 483 if (atomic_read(&vblank->refcount) == 0 && vblank->enabled) { 484 drm_dbg_core(dev, "disabling vblank on crtc %u\n", pipe); 485 drm_vblank_disable_and_save(dev, pipe); 486 } 487 spin_unlock_irqrestore(&dev->vbl_lock, irqflags); 488} 489 490static void drm_vblank_init_release(struct drm_device *dev, void *ptr) 491{ 492 struct drm_vblank_crtc *vblank = ptr; 493 494 drm_WARN_ON(dev, READ_ONCE(vblank->enabled) && 495 drm_core_check_feature(dev, DRIVER_MODESET)); 496 497 drm_vblank_destroy_worker(vblank); 498 del_timer_sync(&vblank->disable_timer); 499} 500 501/** 502 * drm_vblank_init - initialize vblank support 503 * @dev: DRM device 504 * @num_crtcs: number of CRTCs supported by @dev 505 * 506 * This function initializes vblank support for @num_crtcs display pipelines. 507 * Cleanup is handled automatically through a cleanup function added with 508 * drmm_add_action_or_reset(). 509 * 510 * Returns: 511 * Zero on success or a negative error code on failure. 512 */ 513int drm_vblank_init(struct drm_device *dev, unsigned int num_crtcs) 514{ 515 int ret; 516 unsigned int i; 517 518 spin_lock_init(&dev->vbl_lock); 519 spin_lock_init(&dev->vblank_time_lock); 520 521 dev->vblank = drmm_kcalloc(dev, num_crtcs, sizeof(*dev->vblank), GFP_KERNEL); 522 if (!dev->vblank) 523 return -ENOMEM; 524 525 dev->num_crtcs = num_crtcs; 526 527 for (i = 0; i < num_crtcs; i++) { 528 struct drm_vblank_crtc *vblank = &dev->vblank[i]; 529 530 vblank->dev = dev; 531 vblank->pipe = i; 532 init_waitqueue_head(&vblank->queue); 533 timer_setup(&vblank->disable_timer, vblank_disable_fn, 0); 534 seqlock_init(&vblank->seqlock); 535 536 ret = drmm_add_action_or_reset(dev, drm_vblank_init_release, 537 vblank); 538 if (ret) 539 return ret; 540 541 ret = drm_vblank_worker_init(vblank); 542 if (ret) 543 return ret; 544 } 545 546 return 0; 547} 548EXPORT_SYMBOL(drm_vblank_init); 549 550/** 551 * drm_dev_has_vblank - test if vblanking has been initialized for 552 * a device 553 * @dev: the device 554 * 555 * Drivers may call this function to test if vblank support is 556 * initialized for a device. For most hardware this means that vblanking 557 * can also be enabled. 558 * 559 * Atomic helpers use this function to initialize 560 * &drm_crtc_state.no_vblank. See also drm_atomic_helper_check_modeset(). 561 * 562 * Returns: 563 * True if vblanking has been initialized for the given device, false 564 * otherwise. 565 */ 566bool drm_dev_has_vblank(const struct drm_device *dev) 567{ 568 return dev->num_crtcs != 0; 569} 570EXPORT_SYMBOL(drm_dev_has_vblank); 571 572/** 573 * drm_crtc_vblank_waitqueue - get vblank waitqueue for the CRTC 574 * @crtc: which CRTC's vblank waitqueue to retrieve 575 * 576 * This function returns a pointer to the vblank waitqueue for the CRTC. 577 * Drivers can use this to implement vblank waits using wait_event() and related 578 * functions. 579 */ 580wait_queue_head_t *drm_crtc_vblank_waitqueue(struct drm_crtc *crtc) 581{ 582 return &crtc->dev->vblank[drm_crtc_index(crtc)].queue; 583} 584EXPORT_SYMBOL(drm_crtc_vblank_waitqueue); 585 586 587/** 588 * drm_calc_timestamping_constants - calculate vblank timestamp constants 589 * @crtc: drm_crtc whose timestamp constants should be updated. 590 * @mode: display mode containing the scanout timings 591 * 592 * Calculate and store various constants which are later needed by vblank and 593 * swap-completion timestamping, e.g, by 594 * drm_crtc_vblank_helper_get_vblank_timestamp(). They are derived from 595 * CRTC's true scanout timing, so they take things like panel scaling or 596 * other adjustments into account. 597 */ 598void drm_calc_timestamping_constants(struct drm_crtc *crtc, 599 const struct drm_display_mode *mode) 600{ 601 struct drm_device *dev = crtc->dev; 602 unsigned int pipe = drm_crtc_index(crtc); 603 struct drm_vblank_crtc *vblank = &dev->vblank[pipe]; 604 int linedur_ns = 0, framedur_ns = 0; 605 int dotclock = mode->crtc_clock; 606 607 if (!drm_dev_has_vblank(dev)) 608 return; 609 610 if (drm_WARN_ON(dev, pipe >= dev->num_crtcs)) 611 return; 612 613 /* Valid dotclock? */ 614 if (dotclock > 0) { 615 int frame_size = mode->crtc_htotal * mode->crtc_vtotal; 616 617 /* 618 * Convert scanline length in pixels and video 619 * dot clock to line duration and frame duration 620 * in nanoseconds: 621 */ 622 linedur_ns = div_u64((u64) mode->crtc_htotal * 1000000, dotclock); 623 framedur_ns = div_u64((u64) frame_size * 1000000, dotclock); 624 625 /* 626 * Fields of interlaced scanout modes are only half a frame duration. 627 */ 628 if (mode->flags & DRM_MODE_FLAG_INTERLACE) 629 framedur_ns /= 2; 630 } else { 631 drm_err(dev, "crtc %u: Can't calculate constants, dotclock = 0!\n", 632 crtc->base.id); 633 } 634 635 vblank->linedur_ns = linedur_ns; 636 vblank->framedur_ns = framedur_ns; 637 drm_mode_copy(&vblank->hwmode, mode); 638 639 drm_dbg_core(dev, 640 "crtc %u: hwmode: htotal %d, vtotal %d, vdisplay %d\n", 641 crtc->base.id, mode->crtc_htotal, 642 mode->crtc_vtotal, mode->crtc_vdisplay); 643 drm_dbg_core(dev, "crtc %u: clock %d kHz framedur %d linedur %d\n", 644 crtc->base.id, dotclock, framedur_ns, linedur_ns); 645} 646EXPORT_SYMBOL(drm_calc_timestamping_constants); 647 648/** 649 * drm_crtc_vblank_helper_get_vblank_timestamp_internal - precise vblank 650 * timestamp helper 651 * @crtc: CRTC whose vblank timestamp to retrieve 652 * @max_error: Desired maximum allowable error in timestamps (nanosecs) 653 * On return contains true maximum error of timestamp 654 * @vblank_time: Pointer to time which should receive the timestamp 655 * @in_vblank_irq: 656 * True when called from drm_crtc_handle_vblank(). Some drivers 657 * need to apply some workarounds for gpu-specific vblank irq quirks 658 * if flag is set. 659 * @get_scanout_position: 660 * Callback function to retrieve the scanout position. See 661 * @struct drm_crtc_helper_funcs.get_scanout_position. 662 * 663 * Implements calculation of exact vblank timestamps from given drm_display_mode 664 * timings and current video scanout position of a CRTC. 665 * 666 * The current implementation only handles standard video modes. For double scan 667 * and interlaced modes the driver is supposed to adjust the hardware mode 668 * (taken from &drm_crtc_state.adjusted mode for atomic modeset drivers) to 669 * match the scanout position reported. 670 * 671 * Note that atomic drivers must call drm_calc_timestamping_constants() before 672 * enabling a CRTC. The atomic helpers already take care of that in 673 * drm_atomic_helper_calc_timestamping_constants(). 674 * 675 * Returns: 676 * 677 * Returns true on success, and false on failure, i.e. when no accurate 678 * timestamp could be acquired. 679 */ 680bool 681drm_crtc_vblank_helper_get_vblank_timestamp_internal( 682 struct drm_crtc *crtc, int *max_error, ktime_t *vblank_time, 683 bool in_vblank_irq, 684 drm_vblank_get_scanout_position_func get_scanout_position) 685{ 686 struct drm_device *dev = crtc->dev; 687 unsigned int pipe = crtc->index; 688 struct drm_vblank_crtc *vblank = &dev->vblank[pipe]; 689 struct timespec64 ts_etime, ts_vblank_time; 690 ktime_t stime, etime; 691 bool vbl_status; 692 const struct drm_display_mode *mode; 693 int vpos, hpos, i; 694 int delta_ns, duration_ns; 695 696 if (pipe >= dev->num_crtcs) { 697 drm_err(dev, "Invalid crtc %u\n", pipe); 698 return false; 699 } 700 701 /* Scanout position query not supported? Should not happen. */ 702 if (!get_scanout_position) { 703 drm_err(dev, "Called from CRTC w/o get_scanout_position()!?\n"); 704 return false; 705 } 706 707 if (drm_drv_uses_atomic_modeset(dev)) 708 mode = &vblank->hwmode; 709 else 710 mode = &crtc->hwmode; 711 712 /* If mode timing undefined, just return as no-op: 713 * Happens during initial modesetting of a crtc. 714 */ 715 if (mode->crtc_clock == 0) { 716 drm_dbg_core(dev, "crtc %u: Noop due to uninitialized mode.\n", 717 pipe); 718 drm_WARN_ON_ONCE(dev, drm_drv_uses_atomic_modeset(dev)); 719 return false; 720 } 721 722 /* Get current scanout position with system timestamp. 723 * Repeat query up to DRM_TIMESTAMP_MAXRETRIES times 724 * if single query takes longer than max_error nanoseconds. 725 * 726 * This guarantees a tight bound on maximum error if 727 * code gets preempted or delayed for some reason. 728 */ 729 for (i = 0; i < DRM_TIMESTAMP_MAXRETRIES; i++) { 730 /* 731 * Get vertical and horizontal scanout position vpos, hpos, 732 * and bounding timestamps stime, etime, pre/post query. 733 */ 734 vbl_status = get_scanout_position(crtc, in_vblank_irq, 735 &vpos, &hpos, 736 &stime, &etime, 737 mode); 738 739 /* Return as no-op if scanout query unsupported or failed. */ 740 if (!vbl_status) { 741 drm_dbg_core(dev, 742 "crtc %u : scanoutpos query failed.\n", 743 pipe); 744 return false; 745 } 746 747 /* Compute uncertainty in timestamp of scanout position query. */ 748 duration_ns = ktime_to_ns(etime) - ktime_to_ns(stime); 749 750 /* Accept result with < max_error nsecs timing uncertainty. */ 751 if (duration_ns <= *max_error) 752 break; 753 } 754 755 /* Noisy system timing? */ 756 if (i == DRM_TIMESTAMP_MAXRETRIES) { 757 drm_dbg_core(dev, 758 "crtc %u: Noisy timestamp %d us > %d us [%d reps].\n", 759 pipe, duration_ns / 1000, *max_error / 1000, i); 760 } 761 762 /* Return upper bound of timestamp precision error. */ 763 *max_error = duration_ns; 764 765 /* Convert scanout position into elapsed time at raw_time query 766 * since start of scanout at first display scanline. delta_ns 767 * can be negative if start of scanout hasn't happened yet. 768 */ 769 delta_ns = div_s64(1000000LL * (vpos * mode->crtc_htotal + hpos), 770 mode->crtc_clock); 771 772 /* Subtract time delta from raw timestamp to get final 773 * vblank_time timestamp for end of vblank. 774 */ 775 *vblank_time = ktime_sub_ns(etime, delta_ns); 776 777 if (!drm_debug_enabled(DRM_UT_VBL)) 778 return true; 779 780 ts_etime = ktime_to_timespec64(etime); 781 ts_vblank_time = ktime_to_timespec64(*vblank_time); 782 783 drm_dbg_vbl(dev, 784 "crtc %u : v p(%d,%d)@ %lld.%06ld -> %lld.%06ld [e %d us, %d rep]\n", 785 pipe, hpos, vpos, 786 (u64)ts_etime.tv_sec, ts_etime.tv_nsec / 1000, 787 (u64)ts_vblank_time.tv_sec, ts_vblank_time.tv_nsec / 1000, 788 duration_ns / 1000, i); 789 790 return true; 791} 792EXPORT_SYMBOL(drm_crtc_vblank_helper_get_vblank_timestamp_internal); 793 794/** 795 * drm_crtc_vblank_helper_get_vblank_timestamp - precise vblank timestamp 796 * helper 797 * @crtc: CRTC whose vblank timestamp to retrieve 798 * @max_error: Desired maximum allowable error in timestamps (nanosecs) 799 * On return contains true maximum error of timestamp 800 * @vblank_time: Pointer to time which should receive the timestamp 801 * @in_vblank_irq: 802 * True when called from drm_crtc_handle_vblank(). Some drivers 803 * need to apply some workarounds for gpu-specific vblank irq quirks 804 * if flag is set. 805 * 806 * Implements calculation of exact vblank timestamps from given drm_display_mode 807 * timings and current video scanout position of a CRTC. This can be directly 808 * used as the &drm_crtc_funcs.get_vblank_timestamp implementation of a kms 809 * driver if &drm_crtc_helper_funcs.get_scanout_position is implemented. 810 * 811 * The current implementation only handles standard video modes. For double scan 812 * and interlaced modes the driver is supposed to adjust the hardware mode 813 * (taken from &drm_crtc_state.adjusted mode for atomic modeset drivers) to 814 * match the scanout position reported. 815 * 816 * Note that atomic drivers must call drm_calc_timestamping_constants() before 817 * enabling a CRTC. The atomic helpers already take care of that in 818 * drm_atomic_helper_calc_timestamping_constants(). 819 * 820 * Returns: 821 * 822 * Returns true on success, and false on failure, i.e. when no accurate 823 * timestamp could be acquired. 824 */ 825bool drm_crtc_vblank_helper_get_vblank_timestamp(struct drm_crtc *crtc, 826 int *max_error, 827 ktime_t *vblank_time, 828 bool in_vblank_irq) 829{ 830 return drm_crtc_vblank_helper_get_vblank_timestamp_internal( 831 crtc, max_error, vblank_time, in_vblank_irq, 832 crtc->helper_private->get_scanout_position); 833} 834EXPORT_SYMBOL(drm_crtc_vblank_helper_get_vblank_timestamp); 835 836/** 837 * drm_crtc_get_last_vbltimestamp - retrieve raw timestamp for the most 838 * recent vblank interval 839 * @crtc: CRTC whose vblank timestamp to retrieve 840 * @tvblank: Pointer to target time which should receive the timestamp 841 * @in_vblank_irq: 842 * True when called from drm_crtc_handle_vblank(). Some drivers 843 * need to apply some workarounds for gpu-specific vblank irq quirks 844 * if flag is set. 845 * 846 * Fetches the system timestamp corresponding to the time of the most recent 847 * vblank interval on specified CRTC. May call into kms-driver to 848 * compute the timestamp with a high-precision GPU specific method. 849 * 850 * Returns zero if timestamp originates from uncorrected do_gettimeofday() 851 * call, i.e., it isn't very precisely locked to the true vblank. 852 * 853 * Returns: 854 * True if timestamp is considered to be very precise, false otherwise. 855 */ 856static bool 857drm_crtc_get_last_vbltimestamp(struct drm_crtc *crtc, ktime_t *tvblank, 858 bool in_vblank_irq) 859{ 860 bool ret = false; 861 862 /* Define requested maximum error on timestamps (nanoseconds). */ 863 int max_error = (int) drm_timestamp_precision * 1000; 864 865 /* Query driver if possible and precision timestamping enabled. */ 866 if (crtc && crtc->funcs->get_vblank_timestamp && max_error > 0) { 867 ret = crtc->funcs->get_vblank_timestamp(crtc, &max_error, 868 tvblank, in_vblank_irq); 869 } 870 871 /* GPU high precision timestamp query unsupported or failed. 872 * Return current monotonic/gettimeofday timestamp as best estimate. 873 */ 874 if (!ret) 875 *tvblank = ktime_get(); 876 877 return ret; 878} 879 880static bool 881drm_get_last_vbltimestamp(struct drm_device *dev, unsigned int pipe, 882 ktime_t *tvblank, bool in_vblank_irq) 883{ 884 struct drm_crtc *crtc = drm_crtc_from_index(dev, pipe); 885 886 return drm_crtc_get_last_vbltimestamp(crtc, tvblank, in_vblank_irq); 887} 888 889/** 890 * drm_crtc_vblank_count - retrieve "cooked" vblank counter value 891 * @crtc: which counter to retrieve 892 * 893 * Fetches the "cooked" vblank count value that represents the number of 894 * vblank events since the system was booted, including lost events due to 895 * modesetting activity. Note that this timer isn't correct against a racing 896 * vblank interrupt (since it only reports the software vblank counter), see 897 * drm_crtc_accurate_vblank_count() for such use-cases. 898 * 899 * Note that for a given vblank counter value drm_crtc_handle_vblank() 900 * and drm_crtc_vblank_count() or drm_crtc_vblank_count_and_time() 901 * provide a barrier: Any writes done before calling 902 * drm_crtc_handle_vblank() will be visible to callers of the later 903 * functions, if the vblank count is the same or a later one. 904 * 905 * See also &drm_vblank_crtc.count. 906 * 907 * Returns: 908 * The software vblank counter. 909 */ 910u64 drm_crtc_vblank_count(struct drm_crtc *crtc) 911{ 912 return drm_vblank_count(crtc->dev, drm_crtc_index(crtc)); 913} 914EXPORT_SYMBOL(drm_crtc_vblank_count); 915 916/** 917 * drm_vblank_count_and_time - retrieve "cooked" vblank counter value and the 918 * system timestamp corresponding to that vblank counter value. 919 * @dev: DRM device 920 * @pipe: index of CRTC whose counter to retrieve 921 * @vblanktime: Pointer to ktime_t to receive the vblank timestamp. 922 * 923 * Fetches the "cooked" vblank count value that represents the number of 924 * vblank events since the system was booted, including lost events due to 925 * modesetting activity. Returns corresponding system timestamp of the time 926 * of the vblank interval that corresponds to the current vblank counter value. 927 * 928 * This is the legacy version of drm_crtc_vblank_count_and_time(). 929 */ 930static u64 drm_vblank_count_and_time(struct drm_device *dev, unsigned int pipe, 931 ktime_t *vblanktime) 932{ 933 struct drm_vblank_crtc *vblank = &dev->vblank[pipe]; 934 u64 vblank_count; 935 unsigned int seq; 936 937 if (drm_WARN_ON(dev, pipe >= dev->num_crtcs)) { 938 *vblanktime = 0; 939 return 0; 940 } 941 942 do { 943 seq = read_seqbegin(&vblank->seqlock); 944 vblank_count = atomic64_read(&vblank->count); 945 *vblanktime = vblank->time; 946 } while (read_seqretry(&vblank->seqlock, seq)); 947 948 return vblank_count; 949} 950 951/** 952 * drm_crtc_vblank_count_and_time - retrieve "cooked" vblank counter value 953 * and the system timestamp corresponding to that vblank counter value 954 * @crtc: which counter to retrieve 955 * @vblanktime: Pointer to time to receive the vblank timestamp. 956 * 957 * Fetches the "cooked" vblank count value that represents the number of 958 * vblank events since the system was booted, including lost events due to 959 * modesetting activity. Returns corresponding system timestamp of the time 960 * of the vblank interval that corresponds to the current vblank counter value. 961 * 962 * Note that for a given vblank counter value drm_crtc_handle_vblank() 963 * and drm_crtc_vblank_count() or drm_crtc_vblank_count_and_time() 964 * provide a barrier: Any writes done before calling 965 * drm_crtc_handle_vblank() will be visible to callers of the later 966 * functions, if the vblank count is the same or a later one. 967 * 968 * See also &drm_vblank_crtc.count. 969 */ 970u64 drm_crtc_vblank_count_and_time(struct drm_crtc *crtc, 971 ktime_t *vblanktime) 972{ 973 return drm_vblank_count_and_time(crtc->dev, drm_crtc_index(crtc), 974 vblanktime); 975} 976EXPORT_SYMBOL(drm_crtc_vblank_count_and_time); 977 978/** 979 * drm_crtc_next_vblank_start - calculate the time of the next vblank 980 * @crtc: the crtc for which to calculate next vblank time 981 * @vblanktime: pointer to time to receive the next vblank timestamp. 982 * 983 * Calculate the expected time of the start of the next vblank period, 984 * based on time of previous vblank and frame duration 985 */ 986int drm_crtc_next_vblank_start(struct drm_crtc *crtc, ktime_t *vblanktime) 987{ 988 unsigned int pipe = drm_crtc_index(crtc); 989 struct drm_vblank_crtc *vblank; 990 struct drm_display_mode *mode; 991 u64 vblank_start; 992 993 if (!drm_dev_has_vblank(crtc->dev)) 994 return -EINVAL; 995 996 vblank = &crtc->dev->vblank[pipe]; 997 mode = &vblank->hwmode; 998 999 if (!vblank->framedur_ns || !vblank->linedur_ns) 1000 return -EINVAL; 1001 1002 if (!drm_crtc_get_last_vbltimestamp(crtc, vblanktime, false)) 1003 return -EINVAL; 1004 1005 vblank_start = DIV_ROUND_DOWN_ULL( 1006 (u64)vblank->framedur_ns * mode->crtc_vblank_start, 1007 mode->crtc_vtotal); 1008 *vblanktime = ktime_add(*vblanktime, ns_to_ktime(vblank_start)); 1009 1010 return 0; 1011} 1012EXPORT_SYMBOL(drm_crtc_next_vblank_start); 1013 1014static void send_vblank_event(struct drm_device *dev, 1015 struct drm_pending_vblank_event *e, 1016 u64 seq, ktime_t now) 1017{ 1018 struct timespec64 tv; 1019 1020 switch (e->event.base.type) { 1021 case DRM_EVENT_VBLANK: 1022 case DRM_EVENT_FLIP_COMPLETE: 1023 tv = ktime_to_timespec64(now); 1024 e->event.vbl.sequence = seq; 1025 /* 1026 * e->event is a user space structure, with hardcoded unsigned 1027 * 32-bit seconds/microseconds. This is safe as we always use 1028 * monotonic timestamps since linux-4.15 1029 */ 1030 e->event.vbl.tv_sec = tv.tv_sec; 1031 e->event.vbl.tv_usec = tv.tv_nsec / 1000; 1032 break; 1033 case DRM_EVENT_CRTC_SEQUENCE: 1034 if (seq) 1035 e->event.seq.sequence = seq; 1036 e->event.seq.time_ns = ktime_to_ns(now); 1037 break; 1038 } 1039 trace_drm_vblank_event_delivered(e->base.file_priv, e->pipe, seq); 1040 /* 1041 * Use the same timestamp for any associated fence signal to avoid 1042 * mismatch in timestamps for vsync & fence events triggered by the 1043 * same HW event. Frameworks like SurfaceFlinger in Android expects the 1044 * retire-fence timestamp to match exactly with HW vsync as it uses it 1045 * for its software vsync modeling. 1046 */ 1047 drm_send_event_timestamp_locked(dev, &e->base, now); 1048} 1049 1050/** 1051 * drm_crtc_arm_vblank_event - arm vblank event after pageflip 1052 * @crtc: the source CRTC of the vblank event 1053 * @e: the event to send 1054 * 1055 * A lot of drivers need to generate vblank events for the very next vblank 1056 * interrupt. For example when the page flip interrupt happens when the page 1057 * flip gets armed, but not when it actually executes within the next vblank 1058 * period. This helper function implements exactly the required vblank arming 1059 * behaviour. 1060 * 1061 * NOTE: Drivers using this to send out the &drm_crtc_state.event as part of an 1062 * atomic commit must ensure that the next vblank happens at exactly the same 1063 * time as the atomic commit is committed to the hardware. This function itself 1064 * does **not** protect against the next vblank interrupt racing with either this 1065 * function call or the atomic commit operation. A possible sequence could be: 1066 * 1067 * 1. Driver commits new hardware state into vblank-synchronized registers. 1068 * 2. A vblank happens, committing the hardware state. Also the corresponding 1069 * vblank interrupt is fired off and fully processed by the interrupt 1070 * handler. 1071 * 3. The atomic commit operation proceeds to call drm_crtc_arm_vblank_event(). 1072 * 4. The event is only send out for the next vblank, which is wrong. 1073 * 1074 * An equivalent race can happen when the driver calls 1075 * drm_crtc_arm_vblank_event() before writing out the new hardware state. 1076 * 1077 * The only way to make this work safely is to prevent the vblank from firing 1078 * (and the hardware from committing anything else) until the entire atomic 1079 * commit sequence has run to completion. If the hardware does not have such a 1080 * feature (e.g. using a "go" bit), then it is unsafe to use this functions. 1081 * Instead drivers need to manually send out the event from their interrupt 1082 * handler by calling drm_crtc_send_vblank_event() and make sure that there's no 1083 * possible race with the hardware committing the atomic update. 1084 * 1085 * Caller must hold a vblank reference for the event @e acquired by a 1086 * drm_crtc_vblank_get(), which will be dropped when the next vblank arrives. 1087 */ 1088void drm_crtc_arm_vblank_event(struct drm_crtc *crtc, 1089 struct drm_pending_vblank_event *e) 1090{ 1091 struct drm_device *dev = crtc->dev; 1092 unsigned int pipe = drm_crtc_index(crtc); 1093 1094 assert_spin_locked(&dev->event_lock); 1095 1096 e->pipe = pipe; 1097 e->sequence = drm_crtc_accurate_vblank_count(crtc) + 1; 1098 list_add_tail(&e->base.link, &dev->vblank_event_list); 1099} 1100EXPORT_SYMBOL(drm_crtc_arm_vblank_event); 1101 1102/** 1103 * drm_crtc_send_vblank_event - helper to send vblank event after pageflip 1104 * @crtc: the source CRTC of the vblank event 1105 * @e: the event to send 1106 * 1107 * Updates sequence # and timestamp on event for the most recently processed 1108 * vblank, and sends it to userspace. Caller must hold event lock. 1109 * 1110 * See drm_crtc_arm_vblank_event() for a helper which can be used in certain 1111 * situation, especially to send out events for atomic commit operations. 1112 */ 1113void drm_crtc_send_vblank_event(struct drm_crtc *crtc, 1114 struct drm_pending_vblank_event *e) 1115{ 1116 struct drm_device *dev = crtc->dev; 1117 u64 seq; 1118 unsigned int pipe = drm_crtc_index(crtc); 1119 ktime_t now; 1120 1121 if (drm_dev_has_vblank(dev)) { 1122 seq = drm_vblank_count_and_time(dev, pipe, &now); 1123 } else { 1124 seq = 0; 1125 1126 now = ktime_get(); 1127 } 1128 e->pipe = pipe; 1129 send_vblank_event(dev, e, seq, now); 1130} 1131EXPORT_SYMBOL(drm_crtc_send_vblank_event); 1132 1133static int __enable_vblank(struct drm_device *dev, unsigned int pipe) 1134{ 1135 if (drm_core_check_feature(dev, DRIVER_MODESET)) { 1136 struct drm_crtc *crtc = drm_crtc_from_index(dev, pipe); 1137 1138 if (drm_WARN_ON(dev, !crtc)) 1139 return 0; 1140 1141 if (crtc->funcs->enable_vblank) 1142 return crtc->funcs->enable_vblank(crtc); 1143 } 1144 1145 return -EINVAL; 1146} 1147 1148static int drm_vblank_enable(struct drm_device *dev, unsigned int pipe) 1149{ 1150 struct drm_vblank_crtc *vblank = &dev->vblank[pipe]; 1151 int ret = 0; 1152 1153 assert_spin_locked(&dev->vbl_lock); 1154 1155 spin_lock(&dev->vblank_time_lock); 1156 1157 if (!vblank->enabled) { 1158 /* 1159 * Enable vblank irqs under vblank_time_lock protection. 1160 * All vblank count & timestamp updates are held off 1161 * until we are done reinitializing master counter and 1162 * timestamps. Filtercode in drm_handle_vblank() will 1163 * prevent double-accounting of same vblank interval. 1164 */ 1165 ret = __enable_vblank(dev, pipe); 1166 drm_dbg_core(dev, "enabling vblank on crtc %u, ret: %d\n", 1167 pipe, ret); 1168 if (ret) { 1169 atomic_dec(&vblank->refcount); 1170 } else { 1171 drm_update_vblank_count(dev, pipe, 0); 1172 /* drm_update_vblank_count() includes a wmb so we just 1173 * need to ensure that the compiler emits the write 1174 * to mark the vblank as enabled after the call 1175 * to drm_update_vblank_count(). 1176 */ 1177 WRITE_ONCE(vblank->enabled, true); 1178 } 1179 } 1180 1181 spin_unlock(&dev->vblank_time_lock); 1182 1183 return ret; 1184} 1185 1186int drm_vblank_get(struct drm_device *dev, unsigned int pipe) 1187{ 1188 struct drm_vblank_crtc *vblank = &dev->vblank[pipe]; 1189 unsigned long irqflags; 1190 int ret = 0; 1191 1192 if (!drm_dev_has_vblank(dev)) 1193 return -EINVAL; 1194 1195 if (drm_WARN_ON(dev, pipe >= dev->num_crtcs)) 1196 return -EINVAL; 1197 1198 spin_lock_irqsave(&dev->vbl_lock, irqflags); 1199 /* Going from 0->1 means we have to enable interrupts again */ 1200 if (atomic_add_return(1, &vblank->refcount) == 1) { 1201 ret = drm_vblank_enable(dev, pipe); 1202 } else { 1203 if (!vblank->enabled) { 1204 atomic_dec(&vblank->refcount); 1205 ret = -EINVAL; 1206 } 1207 } 1208 spin_unlock_irqrestore(&dev->vbl_lock, irqflags); 1209 1210 return ret; 1211} 1212 1213/** 1214 * drm_crtc_vblank_get - get a reference count on vblank events 1215 * @crtc: which CRTC to own 1216 * 1217 * Acquire a reference count on vblank events to avoid having them disabled 1218 * while in use. 1219 * 1220 * Returns: 1221 * Zero on success or a negative error code on failure. 1222 */ 1223int drm_crtc_vblank_get(struct drm_crtc *crtc) 1224{ 1225 return drm_vblank_get(crtc->dev, drm_crtc_index(crtc)); 1226} 1227EXPORT_SYMBOL(drm_crtc_vblank_get); 1228 1229void drm_vblank_put(struct drm_device *dev, unsigned int pipe) 1230{ 1231 struct drm_vblank_crtc *vblank = &dev->vblank[pipe]; 1232 1233 if (drm_WARN_ON(dev, pipe >= dev->num_crtcs)) 1234 return; 1235 1236 if (drm_WARN_ON(dev, atomic_read(&vblank->refcount) == 0)) 1237 return; 1238 1239 /* Last user schedules interrupt disable */ 1240 if (atomic_dec_and_test(&vblank->refcount)) { 1241 if (drm_vblank_offdelay == 0) 1242 return; 1243 else if (drm_vblank_offdelay < 0) 1244 vblank_disable_fn(&vblank->disable_timer); 1245 else if (!dev->vblank_disable_immediate) 1246 mod_timer(&vblank->disable_timer, 1247 jiffies + ((drm_vblank_offdelay * HZ)/1000)); 1248 } 1249} 1250 1251/** 1252 * drm_crtc_vblank_put - give up ownership of vblank events 1253 * @crtc: which counter to give up 1254 * 1255 * Release ownership of a given vblank counter, turning off interrupts 1256 * if possible. Disable interrupts after drm_vblank_offdelay milliseconds. 1257 */ 1258void drm_crtc_vblank_put(struct drm_crtc *crtc) 1259{ 1260 drm_vblank_put(crtc->dev, drm_crtc_index(crtc)); 1261} 1262EXPORT_SYMBOL(drm_crtc_vblank_put); 1263 1264/** 1265 * drm_wait_one_vblank - wait for one vblank 1266 * @dev: DRM device 1267 * @pipe: CRTC index 1268 * 1269 * This waits for one vblank to pass on @pipe, using the irq driver interfaces. 1270 * It is a failure to call this when the vblank irq for @pipe is disabled, e.g. 1271 * due to lack of driver support or because the crtc is off. 1272 * 1273 * This is the legacy version of drm_crtc_wait_one_vblank(). 1274 */ 1275void drm_wait_one_vblank(struct drm_device *dev, unsigned int pipe) 1276{ 1277 struct drm_vblank_crtc *vblank = &dev->vblank[pipe]; 1278 int ret; 1279 u64 last; 1280 1281 if (drm_WARN_ON(dev, pipe >= dev->num_crtcs)) 1282 return; 1283 1284 ret = drm_vblank_get(dev, pipe); 1285 if (drm_WARN(dev, ret, "vblank not available on crtc %i, ret=%i\n", 1286 pipe, ret)) 1287 return; 1288 1289 last = drm_vblank_count(dev, pipe); 1290 1291 ret = wait_event_timeout(vblank->queue, 1292 last != drm_vblank_count(dev, pipe), 1293 msecs_to_jiffies(100)); 1294 1295 drm_WARN(dev, ret == 0, "vblank wait timed out on crtc %i\n", pipe); 1296 1297 drm_vblank_put(dev, pipe); 1298} 1299EXPORT_SYMBOL(drm_wait_one_vblank); 1300 1301/** 1302 * drm_crtc_wait_one_vblank - wait for one vblank 1303 * @crtc: DRM crtc 1304 * 1305 * This waits for one vblank to pass on @crtc, using the irq driver interfaces. 1306 * It is a failure to call this when the vblank irq for @crtc is disabled, e.g. 1307 * due to lack of driver support or because the crtc is off. 1308 */ 1309void drm_crtc_wait_one_vblank(struct drm_crtc *crtc) 1310{ 1311 drm_wait_one_vblank(crtc->dev, drm_crtc_index(crtc)); 1312} 1313EXPORT_SYMBOL(drm_crtc_wait_one_vblank); 1314 1315/** 1316 * drm_crtc_vblank_off - disable vblank events on a CRTC 1317 * @crtc: CRTC in question 1318 * 1319 * Drivers can use this function to shut down the vblank interrupt handling when 1320 * disabling a crtc. This function ensures that the latest vblank frame count is 1321 * stored so that drm_vblank_on can restore it again. 1322 * 1323 * Drivers must use this function when the hardware vblank counter can get 1324 * reset, e.g. when suspending or disabling the @crtc in general. 1325 */ 1326void drm_crtc_vblank_off(struct drm_crtc *crtc) 1327{ 1328 struct drm_device *dev = crtc->dev; 1329 unsigned int pipe = drm_crtc_index(crtc); 1330 struct drm_vblank_crtc *vblank = &dev->vblank[pipe]; 1331 struct drm_pending_vblank_event *e, *t; 1332 ktime_t now; 1333 u64 seq; 1334 1335 if (drm_WARN_ON(dev, pipe >= dev->num_crtcs)) 1336 return; 1337 1338 /* 1339 * Grab event_lock early to prevent vblank work from being scheduled 1340 * while we're in the middle of shutting down vblank interrupts 1341 */ 1342 spin_lock_irq(&dev->event_lock); 1343 1344 spin_lock(&dev->vbl_lock); 1345 drm_dbg_vbl(dev, "crtc %d, vblank enabled %d, inmodeset %d\n", 1346 pipe, vblank->enabled, vblank->inmodeset); 1347 1348 /* Avoid redundant vblank disables without previous 1349 * drm_crtc_vblank_on(). */ 1350 if (drm_core_check_feature(dev, DRIVER_ATOMIC) || !vblank->inmodeset) 1351 drm_vblank_disable_and_save(dev, pipe); 1352 1353 wake_up(&vblank->queue); 1354 1355 /* 1356 * Prevent subsequent drm_vblank_get() from re-enabling 1357 * the vblank interrupt by bumping the refcount. 1358 */ 1359 if (!vblank->inmodeset) { 1360 atomic_inc(&vblank->refcount); 1361 vblank->inmodeset = 1; 1362 } 1363 spin_unlock(&dev->vbl_lock); 1364 1365 /* Send any queued vblank events, lest the natives grow disquiet */ 1366 seq = drm_vblank_count_and_time(dev, pipe, &now); 1367 1368 list_for_each_entry_safe(e, t, &dev->vblank_event_list, base.link) { 1369 if (e->pipe != pipe) 1370 continue; 1371 drm_dbg_core(dev, "Sending premature vblank event on disable: " 1372 "wanted %llu, current %llu\n", 1373 e->sequence, seq); 1374 list_del(&e->base.link); 1375 drm_vblank_put(dev, pipe); 1376 send_vblank_event(dev, e, seq, now); 1377 } 1378 1379 /* Cancel any leftover pending vblank work */ 1380 drm_vblank_cancel_pending_works(vblank); 1381 1382 spin_unlock_irq(&dev->event_lock); 1383 1384 /* Will be reset by the modeset helpers when re-enabling the crtc by 1385 * calling drm_calc_timestamping_constants(). */ 1386 vblank->hwmode.crtc_clock = 0; 1387 1388 /* Wait for any vblank work that's still executing to finish */ 1389 drm_vblank_flush_worker(vblank); 1390} 1391EXPORT_SYMBOL(drm_crtc_vblank_off); 1392 1393/** 1394 * drm_crtc_vblank_reset - reset vblank state to off on a CRTC 1395 * @crtc: CRTC in question 1396 * 1397 * Drivers can use this function to reset the vblank state to off at load time. 1398 * Drivers should use this together with the drm_crtc_vblank_off() and 1399 * drm_crtc_vblank_on() functions. The difference compared to 1400 * drm_crtc_vblank_off() is that this function doesn't save the vblank counter 1401 * and hence doesn't need to call any driver hooks. 1402 * 1403 * This is useful for recovering driver state e.g. on driver load, or on resume. 1404 */ 1405void drm_crtc_vblank_reset(struct drm_crtc *crtc) 1406{ 1407 struct drm_device *dev = crtc->dev; 1408 unsigned int pipe = drm_crtc_index(crtc); 1409 struct drm_vblank_crtc *vblank = &dev->vblank[pipe]; 1410 1411 spin_lock_irq(&dev->vbl_lock); 1412 /* 1413 * Prevent subsequent drm_vblank_get() from enabling the vblank 1414 * interrupt by bumping the refcount. 1415 */ 1416 if (!vblank->inmodeset) { 1417 atomic_inc(&vblank->refcount); 1418 vblank->inmodeset = 1; 1419 } 1420 spin_unlock_irq(&dev->vbl_lock); 1421 1422 drm_WARN_ON(dev, !list_empty(&dev->vblank_event_list)); 1423 drm_WARN_ON(dev, !list_empty(&vblank->pending_work)); 1424} 1425EXPORT_SYMBOL(drm_crtc_vblank_reset); 1426 1427/** 1428 * drm_crtc_set_max_vblank_count - configure the hw max vblank counter value 1429 * @crtc: CRTC in question 1430 * @max_vblank_count: max hardware vblank counter value 1431 * 1432 * Update the maximum hardware vblank counter value for @crtc 1433 * at runtime. Useful for hardware where the operation of the 1434 * hardware vblank counter depends on the currently active 1435 * display configuration. 1436 * 1437 * For example, if the hardware vblank counter does not work 1438 * when a specific connector is active the maximum can be set 1439 * to zero. And when that specific connector isn't active the 1440 * maximum can again be set to the appropriate non-zero value. 1441 * 1442 * If used, must be called before drm_vblank_on(). 1443 */ 1444void drm_crtc_set_max_vblank_count(struct drm_crtc *crtc, 1445 u32 max_vblank_count) 1446{ 1447 struct drm_device *dev = crtc->dev; 1448 unsigned int pipe = drm_crtc_index(crtc); 1449 struct drm_vblank_crtc *vblank = &dev->vblank[pipe]; 1450 1451 drm_WARN_ON(dev, dev->max_vblank_count); 1452 drm_WARN_ON(dev, !READ_ONCE(vblank->inmodeset)); 1453 1454 vblank->max_vblank_count = max_vblank_count; 1455} 1456EXPORT_SYMBOL(drm_crtc_set_max_vblank_count); 1457 1458/** 1459 * drm_crtc_vblank_on - enable vblank events on a CRTC 1460 * @crtc: CRTC in question 1461 * 1462 * This functions restores the vblank interrupt state captured with 1463 * drm_crtc_vblank_off() again and is generally called when enabling @crtc. Note 1464 * that calls to drm_crtc_vblank_on() and drm_crtc_vblank_off() can be 1465 * unbalanced and so can also be unconditionally called in driver load code to 1466 * reflect the current hardware state of the crtc. 1467 */ 1468void drm_crtc_vblank_on(struct drm_crtc *crtc) 1469{ 1470 struct drm_device *dev = crtc->dev; 1471 unsigned int pipe = drm_crtc_index(crtc); 1472 struct drm_vblank_crtc *vblank = &dev->vblank[pipe]; 1473 1474 if (drm_WARN_ON(dev, pipe >= dev->num_crtcs)) 1475 return; 1476 1477 spin_lock_irq(&dev->vbl_lock); 1478 drm_dbg_vbl(dev, "crtc %d, vblank enabled %d, inmodeset %d\n", 1479 pipe, vblank->enabled, vblank->inmodeset); 1480 1481 /* Drop our private "prevent drm_vblank_get" refcount */ 1482 if (vblank->inmodeset) { 1483 atomic_dec(&vblank->refcount); 1484 vblank->inmodeset = 0; 1485 } 1486 1487 drm_reset_vblank_timestamp(dev, pipe); 1488 1489 /* 1490 * re-enable interrupts if there are users left, or the 1491 * user wishes vblank interrupts to be enabled all the time. 1492 */ 1493 if (atomic_read(&vblank->refcount) != 0 || drm_vblank_offdelay == 0) 1494 drm_WARN_ON(dev, drm_vblank_enable(dev, pipe)); 1495 spin_unlock_irq(&dev->vbl_lock); 1496} 1497EXPORT_SYMBOL(drm_crtc_vblank_on); 1498 1499static void drm_vblank_restore(struct drm_device *dev, unsigned int pipe) 1500{ 1501 ktime_t t_vblank; 1502 struct drm_vblank_crtc *vblank; 1503 int framedur_ns; 1504 u64 diff_ns; 1505 u32 cur_vblank, diff = 1; 1506 int count = DRM_TIMESTAMP_MAXRETRIES; 1507 u32 max_vblank_count = drm_max_vblank_count(dev, pipe); 1508 1509 if (drm_WARN_ON(dev, pipe >= dev->num_crtcs)) 1510 return; 1511 1512 assert_spin_locked(&dev->vbl_lock); 1513 assert_spin_locked(&dev->vblank_time_lock); 1514 1515 vblank = &dev->vblank[pipe]; 1516 drm_WARN_ONCE(dev, 1517 drm_debug_enabled(DRM_UT_VBL) && !vblank->framedur_ns, 1518 "Cannot compute missed vblanks without frame duration\n"); 1519 framedur_ns = vblank->framedur_ns; 1520 1521 do { 1522 cur_vblank = __get_vblank_counter(dev, pipe); 1523 drm_get_last_vbltimestamp(dev, pipe, &t_vblank, false); 1524 } while (cur_vblank != __get_vblank_counter(dev, pipe) && --count > 0); 1525 1526 diff_ns = ktime_to_ns(ktime_sub(t_vblank, vblank->time)); 1527 if (framedur_ns) 1528 diff = DIV_ROUND_CLOSEST_ULL(diff_ns, framedur_ns); 1529 1530 1531 drm_dbg_vbl(dev, 1532 "missed %d vblanks in %lld ns, frame duration=%d ns, hw_diff=%d\n", 1533 diff, diff_ns, framedur_ns, cur_vblank - vblank->last); 1534 vblank->last = (cur_vblank - diff) & max_vblank_count; 1535} 1536 1537/** 1538 * drm_crtc_vblank_restore - estimate missed vblanks and update vblank count. 1539 * @crtc: CRTC in question 1540 * 1541 * Power manamement features can cause frame counter resets between vblank 1542 * disable and enable. Drivers can use this function in their 1543 * &drm_crtc_funcs.enable_vblank implementation to estimate missed vblanks since 1544 * the last &drm_crtc_funcs.disable_vblank using timestamps and update the 1545 * vblank counter. 1546 * 1547 * Note that drivers must have race-free high-precision timestamping support, 1548 * i.e. &drm_crtc_funcs.get_vblank_timestamp must be hooked up and 1549 * &drm_driver.vblank_disable_immediate must be set to indicate the 1550 * time-stamping functions are race-free against vblank hardware counter 1551 * increments. 1552 */ 1553void drm_crtc_vblank_restore(struct drm_crtc *crtc) 1554{ 1555 WARN_ON_ONCE(!crtc->funcs->get_vblank_timestamp); 1556 WARN_ON_ONCE(!crtc->dev->vblank_disable_immediate); 1557 1558 drm_vblank_restore(crtc->dev, drm_crtc_index(crtc)); 1559} 1560EXPORT_SYMBOL(drm_crtc_vblank_restore); 1561 1562static int drm_queue_vblank_event(struct drm_device *dev, unsigned int pipe, 1563 u64 req_seq, 1564 union drm_wait_vblank *vblwait, 1565 struct drm_file *file_priv) 1566{ 1567 struct drm_vblank_crtc *vblank = &dev->vblank[pipe]; 1568 struct drm_pending_vblank_event *e; 1569 ktime_t now; 1570 u64 seq; 1571 int ret; 1572 1573 e = kzalloc(sizeof(*e), GFP_KERNEL); 1574 if (e == NULL) { 1575 ret = -ENOMEM; 1576 goto err_put; 1577 } 1578 1579 e->pipe = pipe; 1580 e->event.base.type = DRM_EVENT_VBLANK; 1581 e->event.base.length = sizeof(e->event.vbl); 1582 e->event.vbl.user_data = vblwait->request.signal; 1583 e->event.vbl.crtc_id = 0; 1584 if (drm_core_check_feature(dev, DRIVER_MODESET)) { 1585 struct drm_crtc *crtc = drm_crtc_from_index(dev, pipe); 1586 1587 if (crtc) 1588 e->event.vbl.crtc_id = crtc->base.id; 1589 } 1590 1591 spin_lock_irq(&dev->event_lock); 1592 1593 /* 1594 * drm_crtc_vblank_off() might have been called after we called 1595 * drm_vblank_get(). drm_crtc_vblank_off() holds event_lock around the 1596 * vblank disable, so no need for further locking. The reference from 1597 * drm_vblank_get() protects against vblank disable from another source. 1598 */ 1599 if (!READ_ONCE(vblank->enabled)) { 1600 ret = -EINVAL; 1601 goto err_unlock; 1602 } 1603 1604 ret = drm_event_reserve_init_locked(dev, file_priv, &e->base, 1605 &e->event.base); 1606 1607 if (ret) 1608 goto err_unlock; 1609 1610 seq = drm_vblank_count_and_time(dev, pipe, &now); 1611 1612 drm_dbg_core(dev, "event on vblank count %llu, current %llu, crtc %u\n", 1613 req_seq, seq, pipe); 1614 1615 trace_drm_vblank_event_queued(file_priv, pipe, req_seq); 1616 1617 e->sequence = req_seq; 1618 if (drm_vblank_passed(seq, req_seq)) { 1619 drm_vblank_put(dev, pipe); 1620 send_vblank_event(dev, e, seq, now); 1621 vblwait->reply.sequence = seq; 1622 } else { 1623 /* drm_handle_vblank_events will call drm_vblank_put */ 1624 list_add_tail(&e->base.link, &dev->vblank_event_list); 1625 vblwait->reply.sequence = req_seq; 1626 } 1627 1628 spin_unlock_irq(&dev->event_lock); 1629 1630 return 0; 1631 1632err_unlock: 1633 spin_unlock_irq(&dev->event_lock); 1634 kfree(e); 1635err_put: 1636 drm_vblank_put(dev, pipe); 1637 return ret; 1638} 1639 1640static bool drm_wait_vblank_is_query(union drm_wait_vblank *vblwait) 1641{ 1642 if (vblwait->request.sequence) 1643 return false; 1644 1645 return _DRM_VBLANK_RELATIVE == 1646 (vblwait->request.type & (_DRM_VBLANK_TYPES_MASK | 1647 _DRM_VBLANK_EVENT | 1648 _DRM_VBLANK_NEXTONMISS)); 1649} 1650 1651/* 1652 * Widen a 32-bit param to 64-bits. 1653 * 1654 * \param narrow 32-bit value (missing upper 32 bits) 1655 * \param near 64-bit value that should be 'close' to near 1656 * 1657 * This function returns a 64-bit value using the lower 32-bits from 1658 * 'narrow' and constructing the upper 32-bits so that the result is 1659 * as close as possible to 'near'. 1660 */ 1661 1662static u64 widen_32_to_64(u32 narrow, u64 near) 1663{ 1664 return near + (s32) (narrow - near); 1665} 1666 1667static void drm_wait_vblank_reply(struct drm_device *dev, unsigned int pipe, 1668 struct drm_wait_vblank_reply *reply) 1669{ 1670 ktime_t now; 1671 struct timespec64 ts; 1672 1673 /* 1674 * drm_wait_vblank_reply is a UAPI structure that uses 'long' 1675 * to store the seconds. This is safe as we always use monotonic 1676 * timestamps since linux-4.15. 1677 */ 1678 reply->sequence = drm_vblank_count_and_time(dev, pipe, &now); 1679 ts = ktime_to_timespec64(now); 1680 reply->tval_sec = (u32)ts.tv_sec; 1681 reply->tval_usec = ts.tv_nsec / 1000; 1682} 1683 1684static bool drm_wait_vblank_supported(struct drm_device *dev) 1685{ 1686 return drm_dev_has_vblank(dev); 1687} 1688 1689int drm_wait_vblank_ioctl(struct drm_device *dev, void *data, 1690 struct drm_file *file_priv) 1691{ 1692 struct drm_crtc *crtc; 1693 struct drm_vblank_crtc *vblank; 1694 union drm_wait_vblank *vblwait = data; 1695 int ret; 1696 u64 req_seq, seq; 1697 unsigned int pipe_index; 1698 unsigned int flags, pipe, high_pipe; 1699 1700 if (!drm_wait_vblank_supported(dev)) 1701 return -EOPNOTSUPP; 1702 1703 if (vblwait->request.type & _DRM_VBLANK_SIGNAL) 1704 return -EINVAL; 1705 1706 if (vblwait->request.type & 1707 ~(_DRM_VBLANK_TYPES_MASK | _DRM_VBLANK_FLAGS_MASK | 1708 _DRM_VBLANK_HIGH_CRTC_MASK)) { 1709 drm_dbg_core(dev, 1710 "Unsupported type value 0x%x, supported mask 0x%x\n", 1711 vblwait->request.type, 1712 (_DRM_VBLANK_TYPES_MASK | _DRM_VBLANK_FLAGS_MASK | 1713 _DRM_VBLANK_HIGH_CRTC_MASK)); 1714 return -EINVAL; 1715 } 1716 1717 flags = vblwait->request.type & _DRM_VBLANK_FLAGS_MASK; 1718 high_pipe = (vblwait->request.type & _DRM_VBLANK_HIGH_CRTC_MASK); 1719 if (high_pipe) 1720 pipe_index = high_pipe >> _DRM_VBLANK_HIGH_CRTC_SHIFT; 1721 else 1722 pipe_index = flags & _DRM_VBLANK_SECONDARY ? 1 : 0; 1723 1724 /* Convert lease-relative crtc index into global crtc index */ 1725 if (drm_core_check_feature(dev, DRIVER_MODESET)) { 1726 pipe = 0; 1727 drm_for_each_crtc(crtc, dev) { 1728 if (drm_lease_held(file_priv, crtc->base.id)) { 1729 if (pipe_index == 0) 1730 break; 1731 pipe_index--; 1732 } 1733 pipe++; 1734 } 1735 } else { 1736 pipe = pipe_index; 1737 } 1738 1739 if (pipe >= dev->num_crtcs) 1740 return -EINVAL; 1741 1742 vblank = &dev->vblank[pipe]; 1743 1744 /* If the counter is currently enabled and accurate, short-circuit 1745 * queries to return the cached timestamp of the last vblank. 1746 */ 1747 if (dev->vblank_disable_immediate && 1748 drm_wait_vblank_is_query(vblwait) && 1749 READ_ONCE(vblank->enabled)) { 1750 drm_wait_vblank_reply(dev, pipe, &vblwait->reply); 1751 return 0; 1752 } 1753 1754 ret = drm_vblank_get(dev, pipe); 1755 if (ret) { 1756 drm_dbg_core(dev, 1757 "crtc %d failed to acquire vblank counter, %d\n", 1758 pipe, ret); 1759 return ret; 1760 } 1761 seq = drm_vblank_count(dev, pipe); 1762 1763 switch (vblwait->request.type & _DRM_VBLANK_TYPES_MASK) { 1764 case _DRM_VBLANK_RELATIVE: 1765 req_seq = seq + vblwait->request.sequence; 1766 vblwait->request.sequence = req_seq; 1767 vblwait->request.type &= ~_DRM_VBLANK_RELATIVE; 1768 break; 1769 case _DRM_VBLANK_ABSOLUTE: 1770 req_seq = widen_32_to_64(vblwait->request.sequence, seq); 1771 break; 1772 default: 1773 ret = -EINVAL; 1774 goto done; 1775 } 1776 1777 if ((flags & _DRM_VBLANK_NEXTONMISS) && 1778 drm_vblank_passed(seq, req_seq)) { 1779 req_seq = seq + 1; 1780 vblwait->request.type &= ~_DRM_VBLANK_NEXTONMISS; 1781 vblwait->request.sequence = req_seq; 1782 } 1783 1784 if (flags & _DRM_VBLANK_EVENT) { 1785 /* must hold on to the vblank ref until the event fires 1786 * drm_vblank_put will be called asynchronously 1787 */ 1788 return drm_queue_vblank_event(dev, pipe, req_seq, vblwait, file_priv); 1789 } 1790 1791 if (req_seq != seq) { 1792 int wait; 1793 1794 drm_dbg_core(dev, "waiting on vblank count %llu, crtc %u\n", 1795 req_seq, pipe); 1796 wait = wait_event_interruptible_timeout(vblank->queue, 1797 drm_vblank_passed(drm_vblank_count(dev, pipe), req_seq) || 1798 !READ_ONCE(vblank->enabled), 1799 msecs_to_jiffies(3000)); 1800 1801 switch (wait) { 1802 case 0: 1803 /* timeout */ 1804 ret = -EBUSY; 1805 break; 1806 case -ERESTARTSYS: 1807 /* interrupted by signal */ 1808 ret = -EINTR; 1809 break; 1810 default: 1811 ret = 0; 1812 break; 1813 } 1814 } 1815 1816 if (ret != -EINTR) { 1817 drm_wait_vblank_reply(dev, pipe, &vblwait->reply); 1818 1819 drm_dbg_core(dev, "crtc %d returning %u to client\n", 1820 pipe, vblwait->reply.sequence); 1821 } else { 1822 drm_dbg_core(dev, "crtc %d vblank wait interrupted by signal\n", 1823 pipe); 1824 } 1825 1826done: 1827 drm_vblank_put(dev, pipe); 1828 return ret; 1829} 1830 1831static void drm_handle_vblank_events(struct drm_device *dev, unsigned int pipe) 1832{ 1833 struct drm_crtc *crtc = drm_crtc_from_index(dev, pipe); 1834 bool high_prec = false; 1835 struct drm_pending_vblank_event *e, *t; 1836 ktime_t now; 1837 u64 seq; 1838 1839 assert_spin_locked(&dev->event_lock); 1840 1841 seq = drm_vblank_count_and_time(dev, pipe, &now); 1842 1843 list_for_each_entry_safe(e, t, &dev->vblank_event_list, base.link) { 1844 if (e->pipe != pipe) 1845 continue; 1846 if (!drm_vblank_passed(seq, e->sequence)) 1847 continue; 1848 1849 drm_dbg_core(dev, "vblank event on %llu, current %llu\n", 1850 e->sequence, seq); 1851 1852 list_del(&e->base.link); 1853 drm_vblank_put(dev, pipe); 1854 send_vblank_event(dev, e, seq, now); 1855 } 1856 1857 if (crtc && crtc->funcs->get_vblank_timestamp) 1858 high_prec = true; 1859 1860 trace_drm_vblank_event(pipe, seq, now, high_prec); 1861} 1862 1863/** 1864 * drm_handle_vblank - handle a vblank event 1865 * @dev: DRM device 1866 * @pipe: index of CRTC where this event occurred 1867 * 1868 * Drivers should call this routine in their vblank interrupt handlers to 1869 * update the vblank counter and send any signals that may be pending. 1870 * 1871 * This is the legacy version of drm_crtc_handle_vblank(). 1872 */ 1873bool drm_handle_vblank(struct drm_device *dev, unsigned int pipe) 1874{ 1875 struct drm_vblank_crtc *vblank = &dev->vblank[pipe]; 1876 unsigned long irqflags; 1877 bool disable_irq; 1878 1879 if (drm_WARN_ON_ONCE(dev, !drm_dev_has_vblank(dev))) 1880 return false; 1881 1882 if (drm_WARN_ON(dev, pipe >= dev->num_crtcs)) 1883 return false; 1884 1885 spin_lock_irqsave(&dev->event_lock, irqflags); 1886 1887 /* Need timestamp lock to prevent concurrent execution with 1888 * vblank enable/disable, as this would cause inconsistent 1889 * or corrupted timestamps and vblank counts. 1890 */ 1891 spin_lock(&dev->vblank_time_lock); 1892 1893 /* Vblank irq handling disabled. Nothing to do. */ 1894 if (!vblank->enabled) { 1895 spin_unlock(&dev->vblank_time_lock); 1896 spin_unlock_irqrestore(&dev->event_lock, irqflags); 1897 return false; 1898 } 1899 1900 drm_update_vblank_count(dev, pipe, true); 1901 1902 spin_unlock(&dev->vblank_time_lock); 1903 1904 wake_up(&vblank->queue); 1905 1906 /* With instant-off, we defer disabling the interrupt until after 1907 * we finish processing the following vblank after all events have 1908 * been signaled. The disable has to be last (after 1909 * drm_handle_vblank_events) so that the timestamp is always accurate. 1910 */ 1911 disable_irq = (dev->vblank_disable_immediate && 1912 drm_vblank_offdelay > 0 && 1913 !atomic_read(&vblank->refcount)); 1914 1915 drm_handle_vblank_events(dev, pipe); 1916 drm_handle_vblank_works(vblank); 1917 1918 spin_unlock_irqrestore(&dev->event_lock, irqflags); 1919 1920 if (disable_irq) 1921 vblank_disable_fn(&vblank->disable_timer); 1922 1923 return true; 1924} 1925EXPORT_SYMBOL(drm_handle_vblank); 1926 1927/** 1928 * drm_crtc_handle_vblank - handle a vblank event 1929 * @crtc: where this event occurred 1930 * 1931 * Drivers should call this routine in their vblank interrupt handlers to 1932 * update the vblank counter and send any signals that may be pending. 1933 * 1934 * This is the native KMS version of drm_handle_vblank(). 1935 * 1936 * Note that for a given vblank counter value drm_crtc_handle_vblank() 1937 * and drm_crtc_vblank_count() or drm_crtc_vblank_count_and_time() 1938 * provide a barrier: Any writes done before calling 1939 * drm_crtc_handle_vblank() will be visible to callers of the later 1940 * functions, if the vblank count is the same or a later one. 1941 * 1942 * See also &drm_vblank_crtc.count. 1943 * 1944 * Returns: 1945 * True if the event was successfully handled, false on failure. 1946 */ 1947bool drm_crtc_handle_vblank(struct drm_crtc *crtc) 1948{ 1949 return drm_handle_vblank(crtc->dev, drm_crtc_index(crtc)); 1950} 1951EXPORT_SYMBOL(drm_crtc_handle_vblank); 1952 1953/* 1954 * Get crtc VBLANK count. 1955 * 1956 * \param dev DRM device 1957 * \param data user argument, pointing to a drm_crtc_get_sequence structure. 1958 * \param file_priv drm file private for the user's open file descriptor 1959 */ 1960 1961int drm_crtc_get_sequence_ioctl(struct drm_device *dev, void *data, 1962 struct drm_file *file_priv) 1963{ 1964 struct drm_crtc *crtc; 1965 struct drm_vblank_crtc *vblank; 1966 int pipe; 1967 struct drm_crtc_get_sequence *get_seq = data; 1968 ktime_t now; 1969 bool vblank_enabled; 1970 int ret; 1971 1972 if (!drm_core_check_feature(dev, DRIVER_MODESET)) 1973 return -EOPNOTSUPP; 1974 1975 if (!drm_dev_has_vblank(dev)) 1976 return -EOPNOTSUPP; 1977 1978 crtc = drm_crtc_find(dev, file_priv, get_seq->crtc_id); 1979 if (!crtc) 1980 return -ENOENT; 1981 1982 pipe = drm_crtc_index(crtc); 1983 1984 vblank = &dev->vblank[pipe]; 1985 vblank_enabled = dev->vblank_disable_immediate && READ_ONCE(vblank->enabled); 1986 1987 if (!vblank_enabled) { 1988 ret = drm_crtc_vblank_get(crtc); 1989 if (ret) { 1990 drm_dbg_core(dev, 1991 "crtc %d failed to acquire vblank counter, %d\n", 1992 pipe, ret); 1993 return ret; 1994 } 1995 } 1996 drm_modeset_lock(&crtc->mutex, NULL); 1997 if (crtc->state) 1998 get_seq->active = crtc->state->enable; 1999 else 2000 get_seq->active = crtc->enabled; 2001 drm_modeset_unlock(&crtc->mutex); 2002 get_seq->sequence = drm_vblank_count_and_time(dev, pipe, &now); 2003 get_seq->sequence_ns = ktime_to_ns(now); 2004 if (!vblank_enabled) 2005 drm_crtc_vblank_put(crtc); 2006 return 0; 2007} 2008 2009/* 2010 * Queue a event for VBLANK sequence 2011 * 2012 * \param dev DRM device 2013 * \param data user argument, pointing to a drm_crtc_queue_sequence structure. 2014 * \param file_priv drm file private for the user's open file descriptor 2015 */ 2016 2017int drm_crtc_queue_sequence_ioctl(struct drm_device *dev, void *data, 2018 struct drm_file *file_priv) 2019{ 2020 struct drm_crtc *crtc; 2021 struct drm_vblank_crtc *vblank; 2022 int pipe; 2023 struct drm_crtc_queue_sequence *queue_seq = data; 2024 ktime_t now; 2025 struct drm_pending_vblank_event *e; 2026 u32 flags; 2027 u64 seq; 2028 u64 req_seq; 2029 int ret; 2030 2031 if (!drm_core_check_feature(dev, DRIVER_MODESET)) 2032 return -EOPNOTSUPP; 2033 2034 if (!drm_dev_has_vblank(dev)) 2035 return -EOPNOTSUPP; 2036 2037 crtc = drm_crtc_find(dev, file_priv, queue_seq->crtc_id); 2038 if (!crtc) 2039 return -ENOENT; 2040 2041 flags = queue_seq->flags; 2042 /* Check valid flag bits */ 2043 if (flags & ~(DRM_CRTC_SEQUENCE_RELATIVE| 2044 DRM_CRTC_SEQUENCE_NEXT_ON_MISS)) 2045 return -EINVAL; 2046 2047 pipe = drm_crtc_index(crtc); 2048 2049 vblank = &dev->vblank[pipe]; 2050 2051 e = kzalloc(sizeof(*e), GFP_KERNEL); 2052 if (e == NULL) 2053 return -ENOMEM; 2054 2055 ret = drm_crtc_vblank_get(crtc); 2056 if (ret) { 2057 drm_dbg_core(dev, 2058 "crtc %d failed to acquire vblank counter, %d\n", 2059 pipe, ret); 2060 goto err_free; 2061 } 2062 2063 seq = drm_vblank_count_and_time(dev, pipe, &now); 2064 req_seq = queue_seq->sequence; 2065 2066 if (flags & DRM_CRTC_SEQUENCE_RELATIVE) 2067 req_seq += seq; 2068 2069 if ((flags & DRM_CRTC_SEQUENCE_NEXT_ON_MISS) && drm_vblank_passed(seq, req_seq)) 2070 req_seq = seq + 1; 2071 2072 e->pipe = pipe; 2073 e->event.base.type = DRM_EVENT_CRTC_SEQUENCE; 2074 e->event.base.length = sizeof(e->event.seq); 2075 e->event.seq.user_data = queue_seq->user_data; 2076 2077 spin_lock_irq(&dev->event_lock); 2078 2079 /* 2080 * drm_crtc_vblank_off() might have been called after we called 2081 * drm_crtc_vblank_get(). drm_crtc_vblank_off() holds event_lock around the 2082 * vblank disable, so no need for further locking. The reference from 2083 * drm_crtc_vblank_get() protects against vblank disable from another source. 2084 */ 2085 if (!READ_ONCE(vblank->enabled)) { 2086 ret = -EINVAL; 2087 goto err_unlock; 2088 } 2089 2090 ret = drm_event_reserve_init_locked(dev, file_priv, &e->base, 2091 &e->event.base); 2092 2093 if (ret) 2094 goto err_unlock; 2095 2096 e->sequence = req_seq; 2097 2098 if (drm_vblank_passed(seq, req_seq)) { 2099 drm_crtc_vblank_put(crtc); 2100 send_vblank_event(dev, e, seq, now); 2101 queue_seq->sequence = seq; 2102 } else { 2103 /* drm_handle_vblank_events will call drm_vblank_put */ 2104 list_add_tail(&e->base.link, &dev->vblank_event_list); 2105 queue_seq->sequence = req_seq; 2106 } 2107 2108 spin_unlock_irq(&dev->event_lock); 2109 return 0; 2110 2111err_unlock: 2112 spin_unlock_irq(&dev->event_lock); 2113 drm_crtc_vblank_put(crtc); 2114err_free: 2115 kfree(e); 2116 return ret; 2117} 2118 2119