drm_vblank.c revision 1.6
1/* $NetBSD: drm_vblank.c,v 1.6 2021/12/19 11:08:02 riastradh Exp $ */ 2 3/* 4 * drm_irq.c IRQ and vblank support 5 * 6 * \author Rickard E. (Rik) Faith <faith@valinux.com> 7 * \author Gareth Hughes <gareth@valinux.com> 8 * 9 * Permission is hereby granted, free of charge, to any person obtaining a 10 * copy of this software and associated documentation files (the "Software"), 11 * to deal in the Software without restriction, including without limitation 12 * the rights to use, copy, modify, merge, publish, distribute, sublicense, 13 * and/or sell copies of the Software, and to permit persons to whom the 14 * Software is furnished to do so, subject to the following conditions: 15 * 16 * The above copyright notice and this permission notice (including the next 17 * paragraph) shall be included in all copies or substantial portions of the 18 * Software. 19 * 20 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 21 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 22 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 23 * VA LINUX SYSTEMS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM, DAMAGES OR 24 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, 25 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR 26 * OTHER DEALINGS IN THE SOFTWARE. 27 */ 28 29#include <sys/cdefs.h> 30__KERNEL_RCSID(0, "$NetBSD: drm_vblank.c,v 1.6 2021/12/19 11:08:02 riastradh Exp $"); 31 32#include <linux/export.h> 33#include <linux/moduleparam.h> 34#include <linux/math64.h> 35 36#include <drm/drm_crtc.h> 37#include <drm/drm_drv.h> 38#include <drm/drm_framebuffer.h> 39#include <drm/drm_print.h> 40#include <drm/drm_vblank.h> 41 42#include "drm_internal.h" 43#include "drm_trace.h" 44 45/** 46 * DOC: vblank handling 47 * 48 * Vertical blanking plays a major role in graphics rendering. To achieve 49 * tear-free display, users must synchronize page flips and/or rendering to 50 * vertical blanking. The DRM API offers ioctls to perform page flips 51 * synchronized to vertical blanking and wait for vertical blanking. 52 * 53 * The DRM core handles most of the vertical blanking management logic, which 54 * involves filtering out spurious interrupts, keeping race-free blanking 55 * counters, coping with counter wrap-around and resets and keeping use counts. 56 * It relies on the driver to generate vertical blanking interrupts and 57 * optionally provide a hardware vertical blanking counter. 58 * 59 * Drivers must initialize the vertical blanking handling core with a call to 60 * drm_vblank_init(). Minimally, a driver needs to implement 61 * &drm_crtc_funcs.enable_vblank and &drm_crtc_funcs.disable_vblank plus call 62 * drm_crtc_handle_vblank() in its vblank interrupt handler for working vblank 63 * support. 64 * 65 * Vertical blanking interrupts can be enabled by the DRM core or by drivers 66 * themselves (for instance to handle page flipping operations). The DRM core 67 * maintains a vertical blanking use count to ensure that the interrupts are not 68 * disabled while a user still needs them. To increment the use count, drivers 69 * call drm_crtc_vblank_get() and release the vblank reference again with 70 * drm_crtc_vblank_put(). In between these two calls vblank interrupts are 71 * guaranteed to be enabled. 72 * 73 * On many hardware disabling the vblank interrupt cannot be done in a race-free 74 * manner, see &drm_driver.vblank_disable_immediate and 75 * &drm_driver.max_vblank_count. In that case the vblank core only disables the 76 * vblanks after a timer has expired, which can be configured through the 77 * ``vblankoffdelay`` module parameter. 78 */ 79 80/* Retry timestamp calculation up to 3 times to satisfy 81 * drm_timestamp_precision before giving up. 82 */ 83#define DRM_TIMESTAMP_MAXRETRIES 3 84 85/* Threshold in nanoseconds for detection of redundant 86 * vblank irq in drm_handle_vblank(). 1 msec should be ok. 87 */ 88#define DRM_REDUNDANT_VBLIRQ_THRESH_NS 1000000 89 90static bool 91drm_get_last_vbltimestamp(struct drm_device *dev, unsigned int pipe, 92 ktime_t *tvblank, bool in_vblank_irq); 93 94static unsigned int drm_timestamp_precision = 20; /* Default to 20 usecs. */ 95 96static int drm_vblank_offdelay = 5000; /* Default to 5000 msecs. */ 97 98module_param_named(vblankoffdelay, drm_vblank_offdelay, int, 0600); 99module_param_named(timestamp_precision_usec, drm_timestamp_precision, int, 0600); 100MODULE_PARM_DESC(vblankoffdelay, "Delay until vblank irq auto-disable [msecs] (0: never disable, <0: disable immediately)"); 101MODULE_PARM_DESC(timestamp_precision_usec, "Max. error on timestamps [usecs]"); 102 103static void store_vblank(struct drm_device *dev, unsigned int pipe, 104 u32 vblank_count_inc, 105 ktime_t t_vblank, u32 last) 106{ 107 struct drm_vblank_crtc *vblank = &dev->vblank[pipe]; 108 109 assert_spin_locked(&dev->vblank_time_lock); 110 111 vblank->last = last; 112 113 write_seqlock(&vblank->seqlock); 114 vblank->time = t_vblank; 115 atomic64_add(vblank_count_inc, &vblank->count); 116 write_sequnlock(&vblank->seqlock); 117} 118 119static u32 drm_max_vblank_count(struct drm_device *dev, unsigned int pipe) 120{ 121 struct drm_vblank_crtc *vblank = &dev->vblank[pipe]; 122 123 return vblank->max_vblank_count ?: dev->max_vblank_count; 124} 125 126/* 127 * "No hw counter" fallback implementation of .get_vblank_counter() hook, 128 * if there is no useable hardware frame counter available. 129 */ 130static u32 drm_vblank_no_hw_counter(struct drm_device *dev, unsigned int pipe) 131{ 132 WARN_ON_ONCE(drm_max_vblank_count(dev, pipe) != 0); 133 return 0; 134} 135 136static u32 __get_vblank_counter(struct drm_device *dev, unsigned int pipe) 137{ 138 if (drm_core_check_feature(dev, DRIVER_MODESET)) { 139 struct drm_crtc *crtc = drm_crtc_from_index(dev, pipe); 140 141 if (WARN_ON(!crtc)) 142 return 0; 143 144 if (crtc->funcs->get_vblank_counter) 145 return crtc->funcs->get_vblank_counter(crtc); 146 } 147 148 if (dev->driver->get_vblank_counter) 149 return dev->driver->get_vblank_counter(dev, pipe); 150 151 return drm_vblank_no_hw_counter(dev, pipe); 152} 153 154/* 155 * Reset the stored timestamp for the current vblank count to correspond 156 * to the last vblank occurred. 157 * 158 * Only to be called from drm_crtc_vblank_on(). 159 * 160 * Note: caller must hold &drm_device.vbl_lock since this reads & writes 161 * device vblank fields. 162 */ 163static void drm_reset_vblank_timestamp(struct drm_device *dev, unsigned int pipe) 164{ 165 u32 cur_vblank; 166 bool rc; 167 ktime_t t_vblank; 168 int count = DRM_TIMESTAMP_MAXRETRIES; 169 170 assert_spin_locked(&dev->vbl_lock); 171 172 spin_lock(&dev->vblank_time_lock); 173 174 /* 175 * sample the current counter to avoid random jumps 176 * when drm_vblank_enable() applies the diff 177 */ 178 do { 179 cur_vblank = __get_vblank_counter(dev, pipe); 180 rc = drm_get_last_vbltimestamp(dev, pipe, &t_vblank, false); 181 } while (cur_vblank != __get_vblank_counter(dev, pipe) && --count > 0); 182 183 /* 184 * Only reinitialize corresponding vblank timestamp if high-precision query 185 * available and didn't fail. Otherwise reinitialize delayed at next vblank 186 * interrupt and assign 0 for now, to mark the vblanktimestamp as invalid. 187 */ 188 if (!rc) 189 t_vblank = 0; 190 191 /* 192 * +1 to make sure user will never see the same 193 * vblank counter value before and after a modeset 194 */ 195 store_vblank(dev, pipe, 1, t_vblank, cur_vblank); 196 197 spin_unlock(&dev->vblank_time_lock); 198} 199 200/* 201 * Call back into the driver to update the appropriate vblank counter 202 * (specified by @pipe). Deal with wraparound, if it occurred, and 203 * update the last read value so we can deal with wraparound on the next 204 * call if necessary. 205 * 206 * Only necessary when going from off->on, to account for frames we 207 * didn't get an interrupt for. 208 * 209 * Note: caller must hold &drm_device.vbl_lock since this reads & writes 210 * device vblank fields. 211 */ 212static void drm_update_vblank_count(struct drm_device *dev, unsigned int pipe, 213 bool in_vblank_irq) 214{ 215 struct drm_vblank_crtc *vblank = &dev->vblank[pipe]; 216 u32 cur_vblank, diff; 217 bool rc; 218 ktime_t t_vblank; 219 int count = DRM_TIMESTAMP_MAXRETRIES; 220 int framedur_ns = vblank->framedur_ns; 221 u32 max_vblank_count = drm_max_vblank_count(dev, pipe); 222 223 assert_spin_locked(&dev->vbl_lock); 224 225 /* 226 * Interrupts were disabled prior to this call, so deal with counter 227 * wrap if needed. 228 * NOTE! It's possible we lost a full dev->max_vblank_count + 1 events 229 * here if the register is small or we had vblank interrupts off for 230 * a long time. 231 * 232 * We repeat the hardware vblank counter & timestamp query until 233 * we get consistent results. This to prevent races between gpu 234 * updating its hardware counter while we are retrieving the 235 * corresponding vblank timestamp. 236 */ 237 do { 238 cur_vblank = __get_vblank_counter(dev, pipe); 239 rc = drm_get_last_vbltimestamp(dev, pipe, &t_vblank, in_vblank_irq); 240 } while (cur_vblank != __get_vblank_counter(dev, pipe) && --count > 0); 241 242 if (max_vblank_count) { 243 /* trust the hw counter when it's around */ 244 diff = (cur_vblank - vblank->last) & max_vblank_count; 245 } else if (rc && framedur_ns) { 246 u64 diff_ns = ktime_to_ns(ktime_sub(t_vblank, vblank->time)); 247 248 /* 249 * Figure out how many vblanks we've missed based 250 * on the difference in the timestamps and the 251 * frame/field duration. 252 */ 253 254 DRM_DEBUG_VBL("crtc %u: Calculating number of vblanks." 255 " diff_ns = %lld, framedur_ns = %d)\n", 256 pipe, (long long) diff_ns, framedur_ns); 257 258 diff = DIV_ROUND_CLOSEST_ULL(diff_ns, framedur_ns); 259 260 if (diff == 0 && in_vblank_irq) 261 DRM_DEBUG_VBL("crtc %u: Redundant vblirq ignored\n", 262 pipe); 263 } else { 264 /* some kind of default for drivers w/o accurate vbl timestamping */ 265 diff = in_vblank_irq ? 1 : 0; 266 } 267 268 /* 269 * Within a drm_vblank_pre_modeset - drm_vblank_post_modeset 270 * interval? If so then vblank irqs keep running and it will likely 271 * happen that the hardware vblank counter is not trustworthy as it 272 * might reset at some point in that interval and vblank timestamps 273 * are not trustworthy either in that interval. Iow. this can result 274 * in a bogus diff >> 1 which must be avoided as it would cause 275 * random large forward jumps of the software vblank counter. 276 */ 277 if (diff > 1 && (vblank->inmodeset & 0x2)) { 278 DRM_DEBUG_VBL("clamping vblank bump to 1 on crtc %u: diffr=%u" 279 " due to pre-modeset.\n", pipe, diff); 280 diff = 1; 281 } 282 283 DRM_DEBUG_VBL("updating vblank count on crtc %u:" 284 " current=%"PRIu64", diff=%u, hw=%u hw_last=%u\n", 285 pipe, atomic64_read(&vblank->count), diff, 286 cur_vblank, vblank->last); 287 288 if (diff == 0) { 289 WARN_ON_ONCE(cur_vblank != vblank->last); 290 return; 291 } 292 293 /* 294 * Only reinitialize corresponding vblank timestamp if high-precision query 295 * available and didn't fail, or we were called from the vblank interrupt. 296 * Otherwise reinitialize delayed at next vblank interrupt and assign 0 297 * for now, to mark the vblanktimestamp as invalid. 298 */ 299 if (!rc && !in_vblank_irq) 300 t_vblank = 0; 301 302 store_vblank(dev, pipe, diff, t_vblank, cur_vblank); 303} 304 305static u64 drm_vblank_count(struct drm_device *dev, unsigned int pipe) 306{ 307 struct drm_vblank_crtc *vblank = &dev->vblank[pipe]; 308 u64 count; 309 310 if (WARN_ON(pipe >= dev->num_crtcs)) 311 return 0; 312 313 count = atomic64_read(&vblank->count); 314 315 /* 316 * This read barrier corresponds to the implicit write barrier of the 317 * write seqlock in store_vblank(). Note that this is the only place 318 * where we need an explicit barrier, since all other access goes 319 * through drm_vblank_count_and_time(), which already has the required 320 * read barrier curtesy of the read seqlock. 321 */ 322 smp_rmb(); 323 324 return count; 325} 326 327/** 328 * drm_crtc_accurate_vblank_count - retrieve the master vblank counter 329 * @crtc: which counter to retrieve 330 * 331 * This function is similar to drm_crtc_vblank_count() but this function 332 * interpolates to handle a race with vblank interrupts using the high precision 333 * timestamping support. 334 * 335 * This is mostly useful for hardware that can obtain the scanout position, but 336 * doesn't have a hardware frame counter. 337 */ 338u64 drm_crtc_accurate_vblank_count(struct drm_crtc *crtc) 339{ 340 struct drm_device *dev = crtc->dev; 341 unsigned int pipe = drm_crtc_index(crtc); 342 u64 vblank; 343 unsigned long flags; 344 345 WARN_ONCE(drm_debug_enabled(DRM_UT_VBL) && !dev->driver->get_vblank_timestamp, 346 "This function requires support for accurate vblank timestamps."); 347 348 spin_lock_irqsave(&dev->vblank_time_lock, flags); 349 350 drm_update_vblank_count(dev, pipe, false); 351 vblank = drm_vblank_count(dev, pipe); 352 353 spin_unlock_irqrestore(&dev->vblank_time_lock, flags); 354 355 return vblank; 356} 357EXPORT_SYMBOL(drm_crtc_accurate_vblank_count); 358 359static void __disable_vblank(struct drm_device *dev, unsigned int pipe) 360{ 361 if (drm_core_check_feature(dev, DRIVER_MODESET)) { 362 struct drm_crtc *crtc = drm_crtc_from_index(dev, pipe); 363 364 if (WARN_ON(!crtc)) 365 return; 366 367 if (crtc->funcs->disable_vblank) { 368 crtc->funcs->disable_vblank(crtc); 369 return; 370 } 371 } 372 373 dev->driver->disable_vblank(dev, pipe); 374} 375 376/* 377 * Disable vblank irq's on crtc, make sure that last vblank count 378 * of hardware and corresponding consistent software vblank counter 379 * are preserved, even if there are any spurious vblank irq's after 380 * disable. 381 */ 382void drm_vblank_disable_and_save(struct drm_device *dev, unsigned int pipe) 383{ 384 struct drm_vblank_crtc *vblank = &dev->vblank[pipe]; 385 unsigned long irqflags; 386 387 assert_spin_locked(&dev->vbl_lock); 388 389 /* Prevent vblank irq processing while disabling vblank irqs, 390 * so no updates of timestamps or count can happen after we've 391 * disabled. Needed to prevent races in case of delayed irq's. 392 */ 393 spin_lock_irqsave(&dev->vblank_time_lock, irqflags); 394 395 /* 396 * Update vblank count and disable vblank interrupts only if the 397 * interrupts were enabled. This avoids calling the ->disable_vblank() 398 * operation in atomic context with the hardware potentially runtime 399 * suspended. 400 */ 401 if (!vblank->enabled) 402 goto out; 403 404 /* 405 * Update the count and timestamp to maintain the 406 * appearance that the counter has been ticking all along until 407 * this time. This makes the count account for the entire time 408 * between drm_crtc_vblank_on() and drm_crtc_vblank_off(). 409 */ 410 drm_update_vblank_count(dev, pipe, false); 411 __disable_vblank(dev, pipe); 412 vblank->enabled = false; 413 414out: 415 spin_unlock_irqrestore(&dev->vblank_time_lock, irqflags); 416} 417 418static void vblank_disable_fn(struct timer_list *t) 419{ 420 struct drm_vblank_crtc *vblank = from_timer(vblank, t, disable_timer); 421 struct drm_device *dev = vblank->dev; 422 unsigned int pipe = vblank->pipe; 423 unsigned long irqflags; 424 425 spin_lock_irqsave(&dev->vbl_lock, irqflags); 426 if (atomic_read(&vblank->refcount) == 0 && vblank->enabled) { 427 DRM_DEBUG("disabling vblank on crtc %u\n", pipe); 428 drm_vblank_disable_and_save(dev, pipe); 429 } 430 spin_unlock_irqrestore(&dev->vbl_lock, irqflags); 431} 432 433void drm_vblank_cleanup(struct drm_device *dev) 434{ 435 unsigned int pipe; 436 437 /* Bail if the driver didn't call drm_vblank_init() */ 438 if (dev->num_crtcs == 0) 439 return; 440 441 for (pipe = 0; pipe < dev->num_crtcs; pipe++) { 442 struct drm_vblank_crtc *vblank = &dev->vblank[pipe]; 443 444 WARN_ON(READ_ONCE(vblank->enabled) && 445 drm_core_check_feature(dev, DRIVER_MODESET)); 446 447 del_timer_sync(&vblank->disable_timer); 448#ifdef __NetBSD__ 449 teardown_timer(&vblank->disable_timer); 450 seqlock_destroy(&vblank->seqlock); 451#endif 452 } 453 454 kfree(dev->vblank); 455 456 dev->num_crtcs = 0; 457} 458 459/** 460 * drm_vblank_init - initialize vblank support 461 * @dev: DRM device 462 * @num_crtcs: number of CRTCs supported by @dev 463 * 464 * This function initializes vblank support for @num_crtcs display pipelines. 465 * Cleanup is handled by the DRM core, or through calling drm_dev_fini() for 466 * drivers with a &drm_driver.release callback. 467 * 468 * Returns: 469 * Zero on success or a negative error code on failure. 470 */ 471int drm_vblank_init(struct drm_device *dev, unsigned int num_crtcs) 472{ 473 int ret = -ENOMEM; 474 unsigned int i; 475 476 spin_lock_init(&dev->vbl_lock); 477 spin_lock_init(&dev->vblank_time_lock); 478 479 dev->num_crtcs = num_crtcs; 480 481 dev->vblank = kcalloc(num_crtcs, sizeof(*dev->vblank), GFP_KERNEL); 482 if (!dev->vblank) 483 goto err; 484 485 for (i = 0; i < num_crtcs; i++) { 486 struct drm_vblank_crtc *vblank = &dev->vblank[i]; 487 488 vblank->dev = dev; 489 vblank->pipe = i; 490#ifdef __NetBSD__ 491 DRM_INIT_WAITQUEUE(&vblank->queue, "drmvblnq"); 492#else 493 init_waitqueue_head(&vblank->queue); 494#endif 495 timer_setup(&vblank->disable_timer, vblank_disable_fn, 0); 496 seqlock_init(&vblank->seqlock); 497 } 498 499 DRM_INFO("Supports vblank timestamp caching Rev 2 (21.10.2013).\n"); 500 501 /* Driver specific high-precision vblank timestamping supported? */ 502 if (dev->driver->get_vblank_timestamp) 503 DRM_INFO("Driver supports precise vblank timestamp query.\n"); 504 else 505 DRM_INFO("No driver support for vblank timestamp query.\n"); 506 507 /* Must have precise timestamping for reliable vblank instant disable */ 508 if (dev->vblank_disable_immediate && !dev->driver->get_vblank_timestamp) { 509 dev->vblank_disable_immediate = false; 510 DRM_INFO("Setting vblank_disable_immediate to false because " 511 "get_vblank_timestamp == NULL\n"); 512 } 513 514 return 0; 515 516err: 517 dev->num_crtcs = 0; 518 return ret; 519} 520EXPORT_SYMBOL(drm_vblank_init); 521 522/** 523 * drm_crtc_vblank_waitqueue - get vblank waitqueue for the CRTC 524 * @crtc: which CRTC's vblank waitqueue to retrieve 525 * 526 * This function returns a pointer to the vblank waitqueue for the CRTC. 527 * Drivers can use this to implement vblank waits using wait_event() and related 528 * functions. 529 */ 530#ifdef __NetBSD__ 531drm_waitqueue_t *drm_crtc_vblank_waitqueue(struct drm_crtc *crtc) 532#else 533wait_queue_head_t *drm_crtc_vblank_waitqueue(struct drm_crtc *crtc) 534#endif 535{ 536 return &crtc->dev->vblank[drm_crtc_index(crtc)].queue; 537} 538EXPORT_SYMBOL(drm_crtc_vblank_waitqueue); 539 540 541/** 542 * drm_calc_timestamping_constants - calculate vblank timestamp constants 543 * @crtc: drm_crtc whose timestamp constants should be updated. 544 * @mode: display mode containing the scanout timings 545 * 546 * Calculate and store various constants which are later needed by vblank and 547 * swap-completion timestamping, e.g, by 548 * drm_calc_vbltimestamp_from_scanoutpos(). They are derived from CRTC's true 549 * scanout timing, so they take things like panel scaling or other adjustments 550 * into account. 551 */ 552void drm_calc_timestamping_constants(struct drm_crtc *crtc, 553 const struct drm_display_mode *mode) 554{ 555 struct drm_device *dev = crtc->dev; 556 unsigned int pipe = drm_crtc_index(crtc); 557 struct drm_vblank_crtc *vblank = &dev->vblank[pipe]; 558 int linedur_ns = 0, framedur_ns = 0; 559 int dotclock = mode->crtc_clock; 560 561 if (!dev->num_crtcs) 562 return; 563 564 if (WARN_ON(pipe >= dev->num_crtcs)) 565 return; 566 567 /* Valid dotclock? */ 568 if (dotclock > 0) { 569 int frame_size = mode->crtc_htotal * mode->crtc_vtotal; 570 571 /* 572 * Convert scanline length in pixels and video 573 * dot clock to line duration and frame duration 574 * in nanoseconds: 575 */ 576 linedur_ns = div_u64((u64) mode->crtc_htotal * 1000000, dotclock); 577 framedur_ns = div_u64((u64) frame_size * 1000000, dotclock); 578 579 /* 580 * Fields of interlaced scanout modes are only half a frame duration. 581 */ 582 if (mode->flags & DRM_MODE_FLAG_INTERLACE) 583 framedur_ns /= 2; 584 } else 585 DRM_ERROR("crtc %u: Can't calculate constants, dotclock = 0!\n", 586 crtc->base.id); 587 588 vblank->linedur_ns = linedur_ns; 589 vblank->framedur_ns = framedur_ns; 590 vblank->hwmode = *mode; 591 592 DRM_DEBUG("crtc %u: hwmode: htotal %d, vtotal %d, vdisplay %d\n", 593 crtc->base.id, mode->crtc_htotal, 594 mode->crtc_vtotal, mode->crtc_vdisplay); 595 DRM_DEBUG("crtc %u: clock %d kHz framedur %d linedur %d\n", 596 crtc->base.id, dotclock, framedur_ns, linedur_ns); 597} 598EXPORT_SYMBOL(drm_calc_timestamping_constants); 599 600/** 601 * drm_calc_vbltimestamp_from_scanoutpos - precise vblank timestamp helper 602 * @dev: DRM device 603 * @pipe: index of CRTC whose vblank timestamp to retrieve 604 * @max_error: Desired maximum allowable error in timestamps (nanosecs) 605 * On return contains true maximum error of timestamp 606 * @vblank_time: Pointer to time which should receive the timestamp 607 * @in_vblank_irq: 608 * True when called from drm_crtc_handle_vblank(). Some drivers 609 * need to apply some workarounds for gpu-specific vblank irq quirks 610 * if flag is set. 611 * 612 * Implements calculation of exact vblank timestamps from given drm_display_mode 613 * timings and current video scanout position of a CRTC. This can be directly 614 * used as the &drm_driver.get_vblank_timestamp implementation of a kms driver 615 * if &drm_driver.get_scanout_position is implemented. 616 * 617 * The current implementation only handles standard video modes. For double scan 618 * and interlaced modes the driver is supposed to adjust the hardware mode 619 * (taken from &drm_crtc_state.adjusted mode for atomic modeset drivers) to 620 * match the scanout position reported. 621 * 622 * Note that atomic drivers must call drm_calc_timestamping_constants() before 623 * enabling a CRTC. The atomic helpers already take care of that in 624 * drm_atomic_helper_update_legacy_modeset_state(). 625 * 626 * Returns: 627 * 628 * Returns true on success, and false on failure, i.e. when no accurate 629 * timestamp could be acquired. 630 */ 631bool drm_calc_vbltimestamp_from_scanoutpos(struct drm_device *dev, 632 unsigned int pipe, 633 int *max_error, 634 ktime_t *vblank_time, 635 bool in_vblank_irq) 636{ 637 struct timespec64 ts_etime, ts_vblank_time; 638 ktime_t stime, etime; 639 bool vbl_status; 640 struct drm_crtc *crtc; 641 const struct drm_display_mode *mode; 642 struct drm_vblank_crtc *vblank = &dev->vblank[pipe]; 643 int vpos, hpos, i; 644 int delta_ns, duration_ns; 645 646 if (!drm_core_check_feature(dev, DRIVER_MODESET)) 647 return false; 648 649 crtc = drm_crtc_from_index(dev, pipe); 650 651 if (pipe >= dev->num_crtcs || !crtc) { 652 DRM_ERROR("Invalid crtc %u\n", pipe); 653 return false; 654 } 655 656 /* Scanout position query not supported? Should not happen. */ 657 if (!dev->driver->get_scanout_position) { 658 DRM_ERROR("Called from driver w/o get_scanout_position()!?\n"); 659 return false; 660 } 661 662 if (drm_drv_uses_atomic_modeset(dev)) 663 mode = &vblank->hwmode; 664 else 665 mode = &crtc->hwmode; 666 667 /* If mode timing undefined, just return as no-op: 668 * Happens during initial modesetting of a crtc. 669 */ 670 if (mode->crtc_clock == 0) { 671 DRM_DEBUG("crtc %u: Noop due to uninitialized mode.\n", pipe); 672 WARN_ON_ONCE(drm_drv_uses_atomic_modeset(dev)); 673 674 return false; 675 } 676 677 /* Get current scanout position with system timestamp. 678 * Repeat query up to DRM_TIMESTAMP_MAXRETRIES times 679 * if single query takes longer than max_error nanoseconds. 680 * 681 * This guarantees a tight bound on maximum error if 682 * code gets preempted or delayed for some reason. 683 */ 684 for (i = 0; i < DRM_TIMESTAMP_MAXRETRIES; i++) { 685 /* 686 * Get vertical and horizontal scanout position vpos, hpos, 687 * and bounding timestamps stime, etime, pre/post query. 688 */ 689 vbl_status = dev->driver->get_scanout_position(dev, pipe, 690 in_vblank_irq, 691 &vpos, &hpos, 692 &stime, &etime, 693 mode); 694 695 /* Return as no-op if scanout query unsupported or failed. */ 696 if (!vbl_status) { 697 DRM_DEBUG("crtc %u : scanoutpos query failed.\n", 698 pipe); 699 return false; 700 } 701 702 /* Compute uncertainty in timestamp of scanout position query. */ 703 duration_ns = ktime_to_ns(etime) - ktime_to_ns(stime); 704 705 /* Accept result with < max_error nsecs timing uncertainty. */ 706 if (duration_ns <= *max_error) 707 break; 708 } 709 710 /* Noisy system timing? */ 711 if (i == DRM_TIMESTAMP_MAXRETRIES) { 712 DRM_DEBUG("crtc %u: Noisy timestamp %d us > %d us [%d reps].\n", 713 pipe, duration_ns/1000, *max_error/1000, i); 714 } 715 716 /* Return upper bound of timestamp precision error. */ 717 *max_error = duration_ns; 718 719 /* Convert scanout position into elapsed time at raw_time query 720 * since start of scanout at first display scanline. delta_ns 721 * can be negative if start of scanout hasn't happened yet. 722 */ 723 delta_ns = div_s64(1000000LL * (vpos * mode->crtc_htotal + hpos), 724 mode->crtc_clock); 725 726 /* Subtract time delta from raw timestamp to get final 727 * vblank_time timestamp for end of vblank. 728 */ 729 *vblank_time = ktime_sub_ns(etime, delta_ns); 730 731 if (!drm_debug_enabled(DRM_UT_VBL)) 732 return true; 733 734 ts_etime = ktime_to_timespec64(etime); 735 ts_vblank_time = ktime_to_timespec64(*vblank_time); 736 737 DRM_DEBUG_VBL("crtc %u : v p(%d,%d)@ %"PRId64".%06ld -> %"PRId64".%06ld [e %d us, %d rep]\n", 738 pipe, hpos, vpos, 739 (u64)ts_etime.tv_sec, ts_etime.tv_nsec / 1000, 740 (u64)ts_vblank_time.tv_sec, ts_vblank_time.tv_nsec / 1000, 741 duration_ns / 1000, i); 742 743 return true; 744} 745EXPORT_SYMBOL(drm_calc_vbltimestamp_from_scanoutpos); 746 747/** 748 * drm_get_last_vbltimestamp - retrieve raw timestamp for the most recent 749 * vblank interval 750 * @dev: DRM device 751 * @pipe: index of CRTC whose vblank timestamp to retrieve 752 * @tvblank: Pointer to target time which should receive the timestamp 753 * @in_vblank_irq: 754 * True when called from drm_crtc_handle_vblank(). Some drivers 755 * need to apply some workarounds for gpu-specific vblank irq quirks 756 * if flag is set. 757 * 758 * Fetches the system timestamp corresponding to the time of the most recent 759 * vblank interval on specified CRTC. May call into kms-driver to 760 * compute the timestamp with a high-precision GPU specific method. 761 * 762 * Returns zero if timestamp originates from uncorrected do_gettimeofday() 763 * call, i.e., it isn't very precisely locked to the true vblank. 764 * 765 * Returns: 766 * True if timestamp is considered to be very precise, false otherwise. 767 */ 768static bool 769drm_get_last_vbltimestamp(struct drm_device *dev, unsigned int pipe, 770 ktime_t *tvblank, bool in_vblank_irq) 771{ 772 bool ret = false; 773 774 /* Define requested maximum error on timestamps (nanoseconds). */ 775 int max_error = (int) drm_timestamp_precision * 1000; 776 777 /* Query driver if possible and precision timestamping enabled. */ 778 if (dev->driver->get_vblank_timestamp && (max_error > 0)) 779 ret = dev->driver->get_vblank_timestamp(dev, pipe, &max_error, 780 tvblank, in_vblank_irq); 781 782 /* GPU high precision timestamp query unsupported or failed. 783 * Return current monotonic/gettimeofday timestamp as best estimate. 784 */ 785 if (!ret) 786 *tvblank = ktime_get(); 787 788 return ret; 789} 790 791/** 792 * drm_crtc_vblank_count - retrieve "cooked" vblank counter value 793 * @crtc: which counter to retrieve 794 * 795 * Fetches the "cooked" vblank count value that represents the number of 796 * vblank events since the system was booted, including lost events due to 797 * modesetting activity. Note that this timer isn't correct against a racing 798 * vblank interrupt (since it only reports the software vblank counter), see 799 * drm_crtc_accurate_vblank_count() for such use-cases. 800 * 801 * Note that for a given vblank counter value drm_crtc_handle_vblank() 802 * and drm_crtc_vblank_count() or drm_crtc_vblank_count_and_time() 803 * provide a barrier: Any writes done before calling 804 * drm_crtc_handle_vblank() will be visible to callers of the later 805 * functions, iff the vblank count is the same or a later one. 806 * 807 * See also &drm_vblank_crtc.count. 808 * 809 * Returns: 810 * The software vblank counter. 811 */ 812u64 drm_crtc_vblank_count(struct drm_crtc *crtc) 813{ 814 return drm_vblank_count(crtc->dev, drm_crtc_index(crtc)); 815} 816EXPORT_SYMBOL(drm_crtc_vblank_count); 817 818/** 819 * drm_vblank_count_and_time - retrieve "cooked" vblank counter value and the 820 * system timestamp corresponding to that vblank counter value. 821 * @dev: DRM device 822 * @pipe: index of CRTC whose counter to retrieve 823 * @vblanktime: Pointer to ktime_t to receive the vblank timestamp. 824 * 825 * Fetches the "cooked" vblank count value that represents the number of 826 * vblank events since the system was booted, including lost events due to 827 * modesetting activity. Returns corresponding system timestamp of the time 828 * of the vblank interval that corresponds to the current vblank counter value. 829 * 830 * This is the legacy version of drm_crtc_vblank_count_and_time(). 831 */ 832static u64 drm_vblank_count_and_time(struct drm_device *dev, unsigned int pipe, 833 ktime_t *vblanktime) 834{ 835 struct drm_vblank_crtc *vblank = &dev->vblank[pipe]; 836 u64 vblank_count; 837 unsigned int seq; 838 839 if (WARN_ON(pipe >= dev->num_crtcs)) { 840 *vblanktime = 0; 841 return 0; 842 } 843 844 do { 845 seq = read_seqbegin(&vblank->seqlock); 846 vblank_count = atomic64_read(&vblank->count); 847 *vblanktime = vblank->time; 848 } while (read_seqretry(&vblank->seqlock, seq)); 849 850 return vblank_count; 851} 852 853/** 854 * drm_crtc_vblank_count_and_time - retrieve "cooked" vblank counter value 855 * and the system timestamp corresponding to that vblank counter value 856 * @crtc: which counter to retrieve 857 * @vblanktime: Pointer to time to receive the vblank timestamp. 858 * 859 * Fetches the "cooked" vblank count value that represents the number of 860 * vblank events since the system was booted, including lost events due to 861 * modesetting activity. Returns corresponding system timestamp of the time 862 * of the vblank interval that corresponds to the current vblank counter value. 863 * 864 * Note that for a given vblank counter value drm_crtc_handle_vblank() 865 * and drm_crtc_vblank_count() or drm_crtc_vblank_count_and_time() 866 * provide a barrier: Any writes done before calling 867 * drm_crtc_handle_vblank() will be visible to callers of the later 868 * functions, iff the vblank count is the same or a later one. 869 * 870 * See also &drm_vblank_crtc.count. 871 */ 872u64 drm_crtc_vblank_count_and_time(struct drm_crtc *crtc, 873 ktime_t *vblanktime) 874{ 875 return drm_vblank_count_and_time(crtc->dev, drm_crtc_index(crtc), 876 vblanktime); 877} 878EXPORT_SYMBOL(drm_crtc_vblank_count_and_time); 879 880static void send_vblank_event(struct drm_device *dev, 881 struct drm_pending_vblank_event *e, 882 u64 seq, ktime_t now) 883{ 884 struct timespec64 tv; 885 886 switch (e->event.base.type) { 887 case DRM_EVENT_VBLANK: 888 case DRM_EVENT_FLIP_COMPLETE: 889 tv = ktime_to_timespec64(now); 890 e->event.vbl.sequence = seq; 891 /* 892 * e->event is a user space structure, with hardcoded unsigned 893 * 32-bit seconds/microseconds. This is safe as we always use 894 * monotonic timestamps since linux-4.15 895 */ 896 e->event.vbl.tv_sec = tv.tv_sec; 897 e->event.vbl.tv_usec = tv.tv_nsec / 1000; 898 break; 899 case DRM_EVENT_CRTC_SEQUENCE: 900 if (seq) 901 e->event.seq.sequence = seq; 902 e->event.seq.time_ns = ktime_to_ns(now); 903 break; 904 } 905 trace_drm_vblank_event_delivered(e->base.file_priv, e->pipe, seq); 906 drm_send_event_locked(dev, &e->base); 907} 908 909/** 910 * drm_crtc_arm_vblank_event - arm vblank event after pageflip 911 * @crtc: the source CRTC of the vblank event 912 * @e: the event to send 913 * 914 * A lot of drivers need to generate vblank events for the very next vblank 915 * interrupt. For example when the page flip interrupt happens when the page 916 * flip gets armed, but not when it actually executes within the next vblank 917 * period. This helper function implements exactly the required vblank arming 918 * behaviour. 919 * 920 * NOTE: Drivers using this to send out the &drm_crtc_state.event as part of an 921 * atomic commit must ensure that the next vblank happens at exactly the same 922 * time as the atomic commit is committed to the hardware. This function itself 923 * does **not** protect against the next vblank interrupt racing with either this 924 * function call or the atomic commit operation. A possible sequence could be: 925 * 926 * 1. Driver commits new hardware state into vblank-synchronized registers. 927 * 2. A vblank happens, committing the hardware state. Also the corresponding 928 * vblank interrupt is fired off and fully processed by the interrupt 929 * handler. 930 * 3. The atomic commit operation proceeds to call drm_crtc_arm_vblank_event(). 931 * 4. The event is only send out for the next vblank, which is wrong. 932 * 933 * An equivalent race can happen when the driver calls 934 * drm_crtc_arm_vblank_event() before writing out the new hardware state. 935 * 936 * The only way to make this work safely is to prevent the vblank from firing 937 * (and the hardware from committing anything else) until the entire atomic 938 * commit sequence has run to completion. If the hardware does not have such a 939 * feature (e.g. using a "go" bit), then it is unsafe to use this functions. 940 * Instead drivers need to manually send out the event from their interrupt 941 * handler by calling drm_crtc_send_vblank_event() and make sure that there's no 942 * possible race with the hardware committing the atomic update. 943 * 944 * Caller must hold a vblank reference for the event @e acquired by a 945 * drm_crtc_vblank_get(), which will be dropped when the next vblank arrives. 946 */ 947void drm_crtc_arm_vblank_event(struct drm_crtc *crtc, 948 struct drm_pending_vblank_event *e) 949{ 950 struct drm_device *dev = crtc->dev; 951 unsigned int pipe = drm_crtc_index(crtc); 952 953 assert_spin_locked(&dev->event_lock); 954 955 e->pipe = pipe; 956 e->sequence = drm_crtc_accurate_vblank_count(crtc) + 1; 957 list_add_tail(&e->base.link, &dev->vblank_event_list); 958} 959EXPORT_SYMBOL(drm_crtc_arm_vblank_event); 960 961/** 962 * drm_crtc_send_vblank_event - helper to send vblank event after pageflip 963 * @crtc: the source CRTC of the vblank event 964 * @e: the event to send 965 * 966 * Updates sequence # and timestamp on event for the most recently processed 967 * vblank, and sends it to userspace. Caller must hold event lock. 968 * 969 * See drm_crtc_arm_vblank_event() for a helper which can be used in certain 970 * situation, especially to send out events for atomic commit operations. 971 */ 972void drm_crtc_send_vblank_event(struct drm_crtc *crtc, 973 struct drm_pending_vblank_event *e) 974{ 975 struct drm_device *dev = crtc->dev; 976 u64 seq; 977 unsigned int pipe = drm_crtc_index(crtc); 978 ktime_t now; 979 980 if (dev->num_crtcs > 0) { 981 seq = drm_vblank_count_and_time(dev, pipe, &now); 982 } else { 983 seq = 0; 984 985 now = ktime_get(); 986 } 987 e->pipe = pipe; 988 send_vblank_event(dev, e, seq, now); 989} 990EXPORT_SYMBOL(drm_crtc_send_vblank_event); 991 992static int __enable_vblank(struct drm_device *dev, unsigned int pipe) 993{ 994 if (drm_core_check_feature(dev, DRIVER_MODESET)) { 995 struct drm_crtc *crtc = drm_crtc_from_index(dev, pipe); 996 997 if (WARN_ON(!crtc)) 998 return 0; 999 1000 if (crtc->funcs->enable_vblank) 1001 return crtc->funcs->enable_vblank(crtc); 1002 } 1003 1004 return dev->driver->enable_vblank(dev, pipe); 1005} 1006 1007static int drm_vblank_enable(struct drm_device *dev, unsigned int pipe) 1008{ 1009 struct drm_vblank_crtc *vblank = &dev->vblank[pipe]; 1010 int ret = 0; 1011 1012 assert_spin_locked(&dev->vbl_lock); 1013 1014 spin_lock(&dev->vblank_time_lock); 1015 1016 if (!vblank->enabled) { 1017 /* 1018 * Enable vblank irqs under vblank_time_lock protection. 1019 * All vblank count & timestamp updates are held off 1020 * until we are done reinitializing master counter and 1021 * timestamps. Filtercode in drm_handle_vblank() will 1022 * prevent double-accounting of same vblank interval. 1023 */ 1024 ret = __enable_vblank(dev, pipe); 1025 DRM_DEBUG("enabling vblank on crtc %u, ret: %d\n", pipe, ret); 1026 if (ret) { 1027 atomic_dec(&vblank->refcount); 1028 } else { 1029 drm_update_vblank_count(dev, pipe, 0); 1030 /* drm_update_vblank_count() includes a wmb so we just 1031 * need to ensure that the compiler emits the write 1032 * to mark the vblank as enabled after the call 1033 * to drm_update_vblank_count(). 1034 */ 1035 WRITE_ONCE(vblank->enabled, true); 1036 } 1037 } 1038 1039 spin_unlock(&dev->vblank_time_lock); 1040 1041 return ret; 1042} 1043 1044static int drm_vblank_get(struct drm_device *dev, unsigned int pipe) 1045{ 1046 struct drm_vblank_crtc *vblank = &dev->vblank[pipe]; 1047 unsigned long irqflags; 1048 int ret = 0; 1049 1050 if (!dev->num_crtcs) 1051 return -EINVAL; 1052 1053 if (WARN_ON(pipe >= dev->num_crtcs)) 1054 return -EINVAL; 1055 1056 spin_lock_irqsave(&dev->vbl_lock, irqflags); 1057 /* Going from 0->1 means we have to enable interrupts again */ 1058 if (atomic_add_return(1, &vblank->refcount) == 1) { 1059 ret = drm_vblank_enable(dev, pipe); 1060 } else { 1061 if (!vblank->enabled) { 1062 atomic_dec(&vblank->refcount); 1063 ret = -EINVAL; 1064 } 1065 } 1066 spin_unlock_irqrestore(&dev->vbl_lock, irqflags); 1067 1068 return ret; 1069} 1070 1071/** 1072 * drm_crtc_vblank_get - get a reference count on vblank events 1073 * @crtc: which CRTC to own 1074 * 1075 * Acquire a reference count on vblank events to avoid having them disabled 1076 * while in use. 1077 * 1078 * Returns: 1079 * Zero on success or a negative error code on failure. 1080 */ 1081int drm_crtc_vblank_get(struct drm_crtc *crtc) 1082{ 1083 return drm_vblank_get(crtc->dev, drm_crtc_index(crtc)); 1084} 1085EXPORT_SYMBOL(drm_crtc_vblank_get); 1086 1087static void drm_vblank_put(struct drm_device *dev, unsigned int pipe) 1088{ 1089 struct drm_vblank_crtc *vblank = &dev->vblank[pipe]; 1090 1091 if (WARN_ON(pipe >= dev->num_crtcs)) 1092 return; 1093 1094 if (WARN_ON(atomic_read(&vblank->refcount) == 0)) 1095 return; 1096 1097 /* Last user schedules interrupt disable */ 1098 if (atomic_dec_and_test(&vblank->refcount)) { 1099 if (drm_vblank_offdelay == 0) 1100 return; 1101 else if (drm_vblank_offdelay < 0) 1102 vblank_disable_fn(&vblank->disable_timer); 1103 else if (!dev->vblank_disable_immediate) 1104 mod_timer(&vblank->disable_timer, 1105 jiffies + ((drm_vblank_offdelay * HZ)/1000)); 1106 } 1107} 1108 1109/** 1110 * drm_crtc_vblank_put - give up ownership of vblank events 1111 * @crtc: which counter to give up 1112 * 1113 * Release ownership of a given vblank counter, turning off interrupts 1114 * if possible. Disable interrupts after drm_vblank_offdelay milliseconds. 1115 */ 1116void drm_crtc_vblank_put(struct drm_crtc *crtc) 1117{ 1118 drm_vblank_put(crtc->dev, drm_crtc_index(crtc)); 1119} 1120EXPORT_SYMBOL(drm_crtc_vblank_put); 1121 1122/** 1123 * drm_wait_one_vblank - wait for one vblank 1124 * @dev: DRM device 1125 * @pipe: CRTC index 1126 * 1127 * This waits for one vblank to pass on @pipe, using the irq driver interfaces. 1128 * It is a failure to call this when the vblank irq for @pipe is disabled, e.g. 1129 * due to lack of driver support or because the crtc is off. 1130 * 1131 * This is the legacy version of drm_crtc_wait_one_vblank(). 1132 */ 1133void drm_wait_one_vblank(struct drm_device *dev, unsigned int pipe) 1134{ 1135 struct drm_vblank_crtc *vblank = &dev->vblank[pipe]; 1136 int ret; 1137 u64 last; 1138 1139 if (WARN_ON(pipe >= dev->num_crtcs)) 1140 return; 1141 1142 ret = drm_vblank_get(dev, pipe); 1143 if (WARN(ret, "vblank not available on crtc %i, ret=%i\n", pipe, ret)) 1144 return; 1145 1146#ifdef __NetBSD__ 1147 spin_lock(&dev->vbl_lock); 1148 last = drm_vblank_count(dev, pipe); 1149 DRM_SPIN_TIMED_WAIT_UNTIL(ret, &vblank->queue, &dev->vbl_lock, 1150 msecs_to_jiffies(100), 1151 last != drm_vblank_count(dev, pipe)); 1152 spin_unlock(&dev->vbl_lock); 1153#else 1154 last = drm_vblank_count(dev, pipe); 1155 1156 ret = wait_event_timeout(vblank->queue, 1157 last != drm_vblank_count(dev, pipe), 1158 msecs_to_jiffies(100)); 1159#endif 1160 1161 WARN(ret == 0, "vblank wait timed out on crtc %i\n", pipe); 1162 1163 drm_vblank_put(dev, pipe); 1164} 1165EXPORT_SYMBOL(drm_wait_one_vblank); 1166 1167/** 1168 * drm_crtc_wait_one_vblank - wait for one vblank 1169 * @crtc: DRM crtc 1170 * 1171 * This waits for one vblank to pass on @crtc, using the irq driver interfaces. 1172 * It is a failure to call this when the vblank irq for @crtc is disabled, e.g. 1173 * due to lack of driver support or because the crtc is off. 1174 */ 1175void drm_crtc_wait_one_vblank(struct drm_crtc *crtc) 1176{ 1177 drm_wait_one_vblank(crtc->dev, drm_crtc_index(crtc)); 1178} 1179EXPORT_SYMBOL(drm_crtc_wait_one_vblank); 1180 1181/** 1182 * drm_crtc_vblank_off - disable vblank events on a CRTC 1183 * @crtc: CRTC in question 1184 * 1185 * Drivers can use this function to shut down the vblank interrupt handling when 1186 * disabling a crtc. This function ensures that the latest vblank frame count is 1187 * stored so that drm_vblank_on can restore it again. 1188 * 1189 * Drivers must use this function when the hardware vblank counter can get 1190 * reset, e.g. when suspending or disabling the @crtc in general. 1191 */ 1192void drm_crtc_vblank_off(struct drm_crtc *crtc) 1193{ 1194 struct drm_device *dev = crtc->dev; 1195 unsigned int pipe = drm_crtc_index(crtc); 1196 struct drm_vblank_crtc *vblank = &dev->vblank[pipe]; 1197 struct drm_pending_vblank_event *e, *t; 1198 1199 ktime_t now; 1200 unsigned long irqflags; 1201 u64 seq; 1202 1203 if (WARN_ON(pipe >= dev->num_crtcs)) 1204 return; 1205 1206 spin_lock_irqsave(&dev->event_lock, irqflags); 1207 1208 spin_lock(&dev->vbl_lock); 1209 DRM_DEBUG_VBL("crtc %d, vblank enabled %d, inmodeset %d\n", 1210 pipe, vblank->enabled, vblank->inmodeset); 1211 1212 /* Avoid redundant vblank disables without previous 1213 * drm_crtc_vblank_on(). */ 1214 if (drm_core_check_feature(dev, DRIVER_ATOMIC) || !vblank->inmodeset) 1215 drm_vblank_disable_and_save(dev, pipe); 1216 1217#ifdef __NetBSD__ 1218 DRM_SPIN_WAKEUP_ONE(&vblank->queue, &dev->vbl_lock); 1219#else 1220 wake_up(&vblank->queue); 1221#endif 1222 1223 /* 1224 * Prevent subsequent drm_vblank_get() from re-enabling 1225 * the vblank interrupt by bumping the refcount. 1226 */ 1227 if (!vblank->inmodeset) { 1228 atomic_inc(&vblank->refcount); 1229 vblank->inmodeset = 1; 1230 } 1231 spin_unlock(&dev->vbl_lock); 1232 1233 /* Send any queued vblank events, lest the natives grow disquiet */ 1234 seq = drm_vblank_count_and_time(dev, pipe, &now); 1235 1236 list_for_each_entry_safe(e, t, &dev->vblank_event_list, base.link) { 1237 if (e->pipe != pipe) 1238 continue; 1239 DRM_DEBUG("Sending premature vblank event on disable: " 1240 "wanted %"PRIu64", current %"PRIu64"\n", 1241 e->sequence, seq); 1242 list_del(&e->base.link); 1243 drm_vblank_put(dev, pipe); 1244 send_vblank_event(dev, e, seq, now); 1245 } 1246 spin_unlock_irqrestore(&dev->event_lock, irqflags); 1247 1248 /* Will be reset by the modeset helpers when re-enabling the crtc by 1249 * calling drm_calc_timestamping_constants(). */ 1250 vblank->hwmode.crtc_clock = 0; 1251} 1252EXPORT_SYMBOL(drm_crtc_vblank_off); 1253 1254/** 1255 * drm_crtc_vblank_reset - reset vblank state to off on a CRTC 1256 * @crtc: CRTC in question 1257 * 1258 * Drivers can use this function to reset the vblank state to off at load time. 1259 * Drivers should use this together with the drm_crtc_vblank_off() and 1260 * drm_crtc_vblank_on() functions. The difference compared to 1261 * drm_crtc_vblank_off() is that this function doesn't save the vblank counter 1262 * and hence doesn't need to call any driver hooks. 1263 * 1264 * This is useful for recovering driver state e.g. on driver load, or on resume. 1265 */ 1266void drm_crtc_vblank_reset(struct drm_crtc *crtc) 1267{ 1268 struct drm_device *dev = crtc->dev; 1269 unsigned long irqflags; 1270 unsigned int pipe = drm_crtc_index(crtc); 1271 struct drm_vblank_crtc *vblank = &dev->vblank[pipe]; 1272 1273 spin_lock_irqsave(&dev->vbl_lock, irqflags); 1274 /* 1275 * Prevent subsequent drm_vblank_get() from enabling the vblank 1276 * interrupt by bumping the refcount. 1277 */ 1278 if (!vblank->inmodeset) { 1279 atomic_inc(&vblank->refcount); 1280 vblank->inmodeset = 1; 1281 } 1282 spin_unlock_irqrestore(&dev->vbl_lock, irqflags); 1283 1284 WARN_ON(!list_empty(&dev->vblank_event_list)); 1285} 1286EXPORT_SYMBOL(drm_crtc_vblank_reset); 1287 1288/** 1289 * drm_crtc_set_max_vblank_count - configure the hw max vblank counter value 1290 * @crtc: CRTC in question 1291 * @max_vblank_count: max hardware vblank counter value 1292 * 1293 * Update the maximum hardware vblank counter value for @crtc 1294 * at runtime. Useful for hardware where the operation of the 1295 * hardware vblank counter depends on the currently active 1296 * display configuration. 1297 * 1298 * For example, if the hardware vblank counter does not work 1299 * when a specific connector is active the maximum can be set 1300 * to zero. And when that specific connector isn't active the 1301 * maximum can again be set to the appropriate non-zero value. 1302 * 1303 * If used, must be called before drm_vblank_on(). 1304 */ 1305void drm_crtc_set_max_vblank_count(struct drm_crtc *crtc, 1306 u32 max_vblank_count) 1307{ 1308 struct drm_device *dev = crtc->dev; 1309 unsigned int pipe = drm_crtc_index(crtc); 1310 struct drm_vblank_crtc *vblank = &dev->vblank[pipe]; 1311 1312 WARN_ON(dev->max_vblank_count); 1313 WARN_ON(!READ_ONCE(vblank->inmodeset)); 1314 1315 vblank->max_vblank_count = max_vblank_count; 1316} 1317EXPORT_SYMBOL(drm_crtc_set_max_vblank_count); 1318 1319/** 1320 * drm_crtc_vblank_on - enable vblank events on a CRTC 1321 * @crtc: CRTC in question 1322 * 1323 * This functions restores the vblank interrupt state captured with 1324 * drm_crtc_vblank_off() again and is generally called when enabling @crtc. Note 1325 * that calls to drm_crtc_vblank_on() and drm_crtc_vblank_off() can be 1326 * unbalanced and so can also be unconditionally called in driver load code to 1327 * reflect the current hardware state of the crtc. 1328 */ 1329void drm_crtc_vblank_on(struct drm_crtc *crtc) 1330{ 1331 struct drm_device *dev = crtc->dev; 1332 unsigned int pipe = drm_crtc_index(crtc); 1333 struct drm_vblank_crtc *vblank = &dev->vblank[pipe]; 1334 unsigned long irqflags; 1335 1336 if (WARN_ON(pipe >= dev->num_crtcs)) 1337 return; 1338 1339 spin_lock_irqsave(&dev->vbl_lock, irqflags); 1340 DRM_DEBUG_VBL("crtc %d, vblank enabled %d, inmodeset %d\n", 1341 pipe, vblank->enabled, vblank->inmodeset); 1342 1343 /* Drop our private "prevent drm_vblank_get" refcount */ 1344 if (vblank->inmodeset) { 1345 atomic_dec(&vblank->refcount); 1346 vblank->inmodeset = 0; 1347 } 1348 1349 drm_reset_vblank_timestamp(dev, pipe); 1350 1351 /* 1352 * re-enable interrupts if there are users left, or the 1353 * user wishes vblank interrupts to be enabled all the time. 1354 */ 1355 if (atomic_read(&vblank->refcount) != 0 || drm_vblank_offdelay == 0) 1356 WARN_ON(drm_vblank_enable(dev, pipe)); 1357 spin_unlock_irqrestore(&dev->vbl_lock, irqflags); 1358} 1359EXPORT_SYMBOL(drm_crtc_vblank_on); 1360 1361/** 1362 * drm_vblank_restore - estimate missed vblanks and update vblank count. 1363 * @dev: DRM device 1364 * @pipe: CRTC index 1365 * 1366 * Power manamement features can cause frame counter resets between vblank 1367 * disable and enable. Drivers can use this function in their 1368 * &drm_crtc_funcs.enable_vblank implementation to estimate missed vblanks since 1369 * the last &drm_crtc_funcs.disable_vblank using timestamps and update the 1370 * vblank counter. 1371 * 1372 * This function is the legacy version of drm_crtc_vblank_restore(). 1373 */ 1374void drm_vblank_restore(struct drm_device *dev, unsigned int pipe) 1375{ 1376 ktime_t t_vblank; 1377 struct drm_vblank_crtc *vblank; 1378 int framedur_ns; 1379 u64 diff_ns; 1380 u32 cur_vblank, diff = 1; 1381 int count = DRM_TIMESTAMP_MAXRETRIES; 1382 1383 if (WARN_ON(pipe >= dev->num_crtcs)) 1384 return; 1385 1386 assert_spin_locked(&dev->vbl_lock); 1387 assert_spin_locked(&dev->vblank_time_lock); 1388 1389 vblank = &dev->vblank[pipe]; 1390 WARN_ONCE(drm_debug_enabled(DRM_UT_VBL) && !vblank->framedur_ns, 1391 "Cannot compute missed vblanks without frame duration\n"); 1392 framedur_ns = vblank->framedur_ns; 1393 1394 do { 1395 cur_vblank = __get_vblank_counter(dev, pipe); 1396 drm_get_last_vbltimestamp(dev, pipe, &t_vblank, false); 1397 } while (cur_vblank != __get_vblank_counter(dev, pipe) && --count > 0); 1398 1399 diff_ns = ktime_to_ns(ktime_sub(t_vblank, vblank->time)); 1400 if (framedur_ns) 1401 diff = DIV_ROUND_CLOSEST_ULL(diff_ns, framedur_ns); 1402 1403 1404 DRM_DEBUG_VBL("missed %d vblanks in %"PRId64" ns, frame duration=%d ns, hw_diff=%d\n", 1405 diff, diff_ns, framedur_ns, cur_vblank - vblank->last); 1406 store_vblank(dev, pipe, diff, t_vblank, cur_vblank); 1407} 1408EXPORT_SYMBOL(drm_vblank_restore); 1409 1410/** 1411 * drm_crtc_vblank_restore - estimate missed vblanks and update vblank count. 1412 * @crtc: CRTC in question 1413 * 1414 * Power manamement features can cause frame counter resets between vblank 1415 * disable and enable. Drivers can use this function in their 1416 * &drm_crtc_funcs.enable_vblank implementation to estimate missed vblanks since 1417 * the last &drm_crtc_funcs.disable_vblank using timestamps and update the 1418 * vblank counter. 1419 */ 1420void drm_crtc_vblank_restore(struct drm_crtc *crtc) 1421{ 1422 drm_vblank_restore(crtc->dev, drm_crtc_index(crtc)); 1423} 1424EXPORT_SYMBOL(drm_crtc_vblank_restore); 1425 1426static void drm_legacy_vblank_pre_modeset(struct drm_device *dev, 1427 unsigned int pipe) 1428{ 1429 struct drm_vblank_crtc *vblank = &dev->vblank[pipe]; 1430 1431 /* vblank is not initialized (IRQ not installed ?), or has been freed */ 1432 if (!dev->num_crtcs) 1433 return; 1434 1435 if (WARN_ON(pipe >= dev->num_crtcs)) 1436 return; 1437 1438 /* 1439 * To avoid all the problems that might happen if interrupts 1440 * were enabled/disabled around or between these calls, we just 1441 * have the kernel take a reference on the CRTC (just once though 1442 * to avoid corrupting the count if multiple, mismatch calls occur), 1443 * so that interrupts remain enabled in the interim. 1444 */ 1445 if (!vblank->inmodeset) { 1446 vblank->inmodeset = 0x1; 1447 if (drm_vblank_get(dev, pipe) == 0) 1448 vblank->inmodeset |= 0x2; 1449 } 1450} 1451 1452static void drm_legacy_vblank_post_modeset(struct drm_device *dev, 1453 unsigned int pipe) 1454{ 1455 struct drm_vblank_crtc *vblank = &dev->vblank[pipe]; 1456 unsigned long irqflags; 1457 1458 /* vblank is not initialized (IRQ not installed ?), or has been freed */ 1459 if (!dev->num_crtcs) 1460 return; 1461 1462 if (WARN_ON(pipe >= dev->num_crtcs)) 1463 return; 1464 1465 if (vblank->inmodeset) { 1466 spin_lock_irqsave(&dev->vbl_lock, irqflags); 1467 drm_reset_vblank_timestamp(dev, pipe); 1468 spin_unlock_irqrestore(&dev->vbl_lock, irqflags); 1469 1470 if (vblank->inmodeset & 0x2) 1471 drm_vblank_put(dev, pipe); 1472 1473 vblank->inmodeset = 0; 1474 } 1475} 1476 1477int drm_legacy_modeset_ctl_ioctl(struct drm_device *dev, void *data, 1478 struct drm_file *file_priv) 1479{ 1480 struct drm_modeset_ctl *modeset = data; 1481 unsigned int pipe; 1482 1483 /* If drm_vblank_init() hasn't been called yet, just no-op */ 1484 if (!dev->num_crtcs) 1485 return 0; 1486 1487 /* KMS drivers handle this internally */ 1488 if (!drm_core_check_feature(dev, DRIVER_LEGACY)) 1489 return 0; 1490 1491 pipe = modeset->crtc; 1492 if (pipe >= dev->num_crtcs) 1493 return -EINVAL; 1494 1495 switch (modeset->cmd) { 1496 case _DRM_PRE_MODESET: 1497 drm_legacy_vblank_pre_modeset(dev, pipe); 1498 break; 1499 case _DRM_POST_MODESET: 1500 drm_legacy_vblank_post_modeset(dev, pipe); 1501 break; 1502 default: 1503 return -EINVAL; 1504 } 1505 1506 return 0; 1507} 1508 1509static inline bool vblank_passed(u64 seq, u64 ref) 1510{ 1511 return (seq - ref) <= (1 << 23); 1512} 1513 1514static int drm_queue_vblank_event(struct drm_device *dev, unsigned int pipe, 1515 u64 req_seq, 1516 union drm_wait_vblank *vblwait, 1517 struct drm_file *file_priv) 1518{ 1519 struct drm_vblank_crtc *vblank = &dev->vblank[pipe]; 1520 struct drm_pending_vblank_event *e; 1521 ktime_t now; 1522 unsigned long flags; 1523 u64 seq; 1524 int ret; 1525 1526 e = kzalloc(sizeof(*e), GFP_KERNEL); 1527 if (e == NULL) { 1528 ret = -ENOMEM; 1529 goto err_put; 1530 } 1531 1532 e->pipe = pipe; 1533 e->event.base.type = DRM_EVENT_VBLANK; 1534 e->event.base.length = sizeof(e->event.vbl); 1535 e->event.vbl.user_data = vblwait->request.signal; 1536 e->event.vbl.crtc_id = 0; 1537 if (drm_core_check_feature(dev, DRIVER_MODESET)) { 1538 struct drm_crtc *crtc = drm_crtc_from_index(dev, pipe); 1539 if (crtc) 1540 e->event.vbl.crtc_id = crtc->base.id; 1541 } 1542 1543 spin_lock_irqsave(&dev->event_lock, flags); 1544 1545 /* 1546 * drm_crtc_vblank_off() might have been called after we called 1547 * drm_vblank_get(). drm_crtc_vblank_off() holds event_lock around the 1548 * vblank disable, so no need for further locking. The reference from 1549 * drm_vblank_get() protects against vblank disable from another source. 1550 */ 1551 if (!READ_ONCE(vblank->enabled)) { 1552 ret = -EINVAL; 1553 goto err_unlock; 1554 } 1555 1556 ret = drm_event_reserve_init_locked(dev, file_priv, &e->base, 1557 &e->event.base); 1558 1559 if (ret) 1560 goto err_unlock; 1561 1562 seq = drm_vblank_count_and_time(dev, pipe, &now); 1563 1564 DRM_DEBUG("event on vblank count %"PRIu64", current %"PRIu64", crtc %u\n", 1565 req_seq, seq, pipe); 1566 1567 trace_drm_vblank_event_queued(file_priv, pipe, req_seq); 1568 1569 e->sequence = req_seq; 1570 if (vblank_passed(seq, req_seq)) { 1571 drm_vblank_put(dev, pipe); 1572 send_vblank_event(dev, e, seq, now); 1573 vblwait->reply.sequence = seq; 1574 } else { 1575 /* drm_handle_vblank_events will call drm_vblank_put */ 1576 list_add_tail(&e->base.link, &dev->vblank_event_list); 1577 vblwait->reply.sequence = req_seq; 1578 } 1579 1580 spin_unlock_irqrestore(&dev->event_lock, flags); 1581 1582 return 0; 1583 1584err_unlock: 1585 spin_unlock_irqrestore(&dev->event_lock, flags); 1586 kfree(e); 1587err_put: 1588 drm_vblank_put(dev, pipe); 1589 return ret; 1590} 1591 1592static bool drm_wait_vblank_is_query(union drm_wait_vblank *vblwait) 1593{ 1594 if (vblwait->request.sequence) 1595 return false; 1596 1597 return _DRM_VBLANK_RELATIVE == 1598 (vblwait->request.type & (_DRM_VBLANK_TYPES_MASK | 1599 _DRM_VBLANK_EVENT | 1600 _DRM_VBLANK_NEXTONMISS)); 1601} 1602 1603/* 1604 * Widen a 32-bit param to 64-bits. 1605 * 1606 * \param narrow 32-bit value (missing upper 32 bits) 1607 * \param near 64-bit value that should be 'close' to near 1608 * 1609 * This function returns a 64-bit value using the lower 32-bits from 1610 * 'narrow' and constructing the upper 32-bits so that the result is 1611 * as close as possible to 'near'. 1612 */ 1613 1614static u64 widen_32_to_64(u32 narrow, u64 near) 1615{ 1616 return near + (s32) (narrow - near); 1617} 1618 1619static void drm_wait_vblank_reply(struct drm_device *dev, unsigned int pipe, 1620 struct drm_wait_vblank_reply *reply) 1621{ 1622 ktime_t now; 1623 struct timespec64 ts; 1624 1625 /* 1626 * drm_wait_vblank_reply is a UAPI structure that uses 'long' 1627 * to store the seconds. This is safe as we always use monotonic 1628 * timestamps since linux-4.15. 1629 */ 1630 reply->sequence = drm_vblank_count_and_time(dev, pipe, &now); 1631 ts = ktime_to_timespec64(now); 1632 reply->tval_sec = (u32)ts.tv_sec; 1633 reply->tval_usec = ts.tv_nsec / 1000; 1634} 1635 1636int drm_wait_vblank_ioctl(struct drm_device *dev, void *data, 1637 struct drm_file *file_priv) 1638{ 1639 struct drm_crtc *crtc; 1640 struct drm_vblank_crtc *vblank; 1641 union drm_wait_vblank *vblwait = data; 1642 int ret; 1643 u64 req_seq, seq; 1644 unsigned int pipe_index; 1645 unsigned int flags, pipe, high_pipe; 1646 1647 if (!dev->irq_enabled) 1648 return -EOPNOTSUPP; 1649 1650 if (vblwait->request.type & _DRM_VBLANK_SIGNAL) 1651 return -EINVAL; 1652 1653 if (vblwait->request.type & 1654 ~(_DRM_VBLANK_TYPES_MASK | _DRM_VBLANK_FLAGS_MASK | 1655 _DRM_VBLANK_HIGH_CRTC_MASK)) { 1656 DRM_DEBUG("Unsupported type value 0x%x, supported mask 0x%x\n", 1657 vblwait->request.type, 1658 (_DRM_VBLANK_TYPES_MASK | _DRM_VBLANK_FLAGS_MASK | 1659 _DRM_VBLANK_HIGH_CRTC_MASK)); 1660 return -EINVAL; 1661 } 1662 1663 flags = vblwait->request.type & _DRM_VBLANK_FLAGS_MASK; 1664 high_pipe = (vblwait->request.type & _DRM_VBLANK_HIGH_CRTC_MASK); 1665 if (high_pipe) 1666 pipe_index = high_pipe >> _DRM_VBLANK_HIGH_CRTC_SHIFT; 1667 else 1668 pipe_index = flags & _DRM_VBLANK_SECONDARY ? 1 : 0; 1669 1670 /* Convert lease-relative crtc index into global crtc index */ 1671 if (drm_core_check_feature(dev, DRIVER_MODESET)) { 1672 pipe = 0; 1673 drm_for_each_crtc(crtc, dev) { 1674 if (drm_lease_held(file_priv, crtc->base.id)) { 1675 if (pipe_index == 0) 1676 break; 1677 pipe_index--; 1678 } 1679 pipe++; 1680 } 1681 } else { 1682 pipe = pipe_index; 1683 } 1684 1685 if (pipe >= dev->num_crtcs) 1686 return -EINVAL; 1687 1688 vblank = &dev->vblank[pipe]; 1689 1690 /* If the counter is currently enabled and accurate, short-circuit 1691 * queries to return the cached timestamp of the last vblank. 1692 */ 1693 if (dev->vblank_disable_immediate && 1694 drm_wait_vblank_is_query(vblwait) && 1695 READ_ONCE(vblank->enabled)) { 1696 drm_wait_vblank_reply(dev, pipe, &vblwait->reply); 1697 return 0; 1698 } 1699 1700 ret = drm_vblank_get(dev, pipe); 1701 if (ret) { 1702 DRM_DEBUG("crtc %d failed to acquire vblank counter, %d\n", pipe, ret); 1703 return ret; 1704 } 1705 seq = drm_vblank_count(dev, pipe); 1706 1707 switch (vblwait->request.type & _DRM_VBLANK_TYPES_MASK) { 1708 case _DRM_VBLANK_RELATIVE: 1709 req_seq = seq + vblwait->request.sequence; 1710 vblwait->request.sequence = req_seq; 1711 vblwait->request.type &= ~_DRM_VBLANK_RELATIVE; 1712 break; 1713 case _DRM_VBLANK_ABSOLUTE: 1714 req_seq = widen_32_to_64(vblwait->request.sequence, seq); 1715 break; 1716 default: 1717 ret = -EINVAL; 1718 goto done; 1719 } 1720 1721 if ((flags & _DRM_VBLANK_NEXTONMISS) && 1722 vblank_passed(seq, req_seq)) { 1723 req_seq = seq + 1; 1724 vblwait->request.type &= ~_DRM_VBLANK_NEXTONMISS; 1725 vblwait->request.sequence = req_seq; 1726 } 1727 1728 if (flags & _DRM_VBLANK_EVENT) { 1729 /* must hold on to the vblank ref until the event fires 1730 * drm_vblank_put will be called asynchronously 1731 */ 1732 return drm_queue_vblank_event(dev, pipe, req_seq, vblwait, file_priv); 1733 } 1734 1735 if (req_seq != seq) { 1736 int wait; 1737 1738 DRM_DEBUG("waiting on vblank count %"PRIu64", crtc %u\n", 1739 req_seq, pipe); 1740#ifdef __NetBSD__ 1741 DRM_SPIN_TIMED_WAIT_UNTIL(wait, &vblank->queue, 1742 &dev->vbl_lock, msecs_to_jiffies(3000), 1743 vblank_passed(drm_vblank_count(dev, pipe), req_seq) || 1744 !READ_ONCE(vblank->enabled)); 1745#else 1746 wait = wait_event_interruptible_timeout(vblank->queue, 1747 vblank_passed(drm_vblank_count(dev, pipe), req_seq) || 1748 !READ_ONCE(vblank->enabled), 1749 msecs_to_jiffies(3000)); 1750#endif 1751 1752 switch (wait) { 1753 case 0: 1754 /* timeout */ 1755 ret = -EBUSY; 1756 break; 1757 case -ERESTARTSYS: 1758 /* interrupted by signal */ 1759 ret = -EINTR; 1760 break; 1761 default: 1762 ret = 0; 1763 break; 1764 } 1765 } 1766 1767 if (ret != -EINTR) { 1768 drm_wait_vblank_reply(dev, pipe, &vblwait->reply); 1769 1770 DRM_DEBUG("crtc %d returning %u to client\n", 1771 pipe, vblwait->reply.sequence); 1772 } else { 1773 DRM_DEBUG("crtc %d vblank wait interrupted by signal\n", pipe); 1774 } 1775 1776done: 1777 drm_vblank_put(dev, pipe); 1778 return ret; 1779} 1780 1781static void drm_handle_vblank_events(struct drm_device *dev, unsigned int pipe) 1782{ 1783 struct drm_pending_vblank_event *e, *t; 1784 ktime_t now; 1785 u64 seq; 1786 1787 assert_spin_locked(&dev->event_lock); 1788 1789 seq = drm_vblank_count_and_time(dev, pipe, &now); 1790 1791 list_for_each_entry_safe(e, t, &dev->vblank_event_list, base.link) { 1792 if (e->pipe != pipe) 1793 continue; 1794 if (!vblank_passed(seq, e->sequence)) 1795 continue; 1796 1797 DRM_DEBUG("vblank event on %"PRIu64", current %"PRIu64"\n", 1798 e->sequence, seq); 1799 1800 list_del(&e->base.link); 1801 drm_vblank_put(dev, pipe); 1802 send_vblank_event(dev, e, seq, now); 1803 } 1804 1805 trace_drm_vblank_event(pipe, seq, now, 1806 dev->driver->get_vblank_timestamp != NULL); 1807} 1808 1809/** 1810 * drm_handle_vblank - handle a vblank event 1811 * @dev: DRM device 1812 * @pipe: index of CRTC where this event occurred 1813 * 1814 * Drivers should call this routine in their vblank interrupt handlers to 1815 * update the vblank counter and send any signals that may be pending. 1816 * 1817 * This is the legacy version of drm_crtc_handle_vblank(). 1818 */ 1819bool drm_handle_vblank(struct drm_device *dev, unsigned int pipe) 1820{ 1821 struct drm_vblank_crtc *vblank = &dev->vblank[pipe]; 1822 unsigned long irqflags; 1823 bool disable_irq; 1824 1825 if (WARN_ON_ONCE(!dev->num_crtcs)) 1826 return false; 1827 1828 if (WARN_ON(pipe >= dev->num_crtcs)) 1829 return false; 1830 1831 spin_lock_irqsave(&dev->event_lock, irqflags); 1832 1833 /* Need timestamp lock to prevent concurrent execution with 1834 * vblank enable/disable, as this would cause inconsistent 1835 * or corrupted timestamps and vblank counts. 1836 */ 1837 spin_lock(&dev->vblank_time_lock); 1838 1839 spin_lock(&dev->vbl_lock); 1840 1841 /* Vblank irq handling disabled. Nothing to do. */ 1842 if (!vblank->enabled) { 1843 spin_unlock(&dev->vbl_lock); 1844 spin_unlock(&dev->vblank_time_lock); 1845 spin_unlock_irqrestore(&dev->event_lock, irqflags); 1846 return false; 1847 } 1848 1849 drm_update_vblank_count(dev, pipe, true); 1850 1851 spin_unlock(&dev->vblank_time_lock); 1852 1853#ifdef __NetBSD__ 1854 DRM_SPIN_WAKEUP_ONE(&vblank->queue, &dev->vbl_lock); 1855#else 1856 wake_up(&vblank->queue); 1857#endif 1858 1859 /* With instant-off, we defer disabling the interrupt until after 1860 * we finish processing the following vblank after all events have 1861 * been signaled. The disable has to be last (after 1862 * drm_handle_vblank_events) so that the timestamp is always accurate. 1863 */ 1864 disable_irq = (dev->vblank_disable_immediate && 1865 drm_vblank_offdelay > 0 && 1866 !atomic_read(&vblank->refcount)); 1867 1868 spin_unlock(&dev->vbl_lock); 1869 1870 drm_handle_vblank_events(dev, pipe); 1871 1872 spin_unlock_irqrestore(&dev->event_lock, irqflags); 1873 1874 if (disable_irq) 1875 vblank_disable_fn(&vblank->disable_timer); 1876 1877 return true; 1878} 1879EXPORT_SYMBOL(drm_handle_vblank); 1880 1881/** 1882 * drm_crtc_handle_vblank - handle a vblank event 1883 * @crtc: where this event occurred 1884 * 1885 * Drivers should call this routine in their vblank interrupt handlers to 1886 * update the vblank counter and send any signals that may be pending. 1887 * 1888 * This is the native KMS version of drm_handle_vblank(). 1889 * 1890 * Note that for a given vblank counter value drm_crtc_handle_vblank() 1891 * and drm_crtc_vblank_count() or drm_crtc_vblank_count_and_time() 1892 * provide a barrier: Any writes done before calling 1893 * drm_crtc_handle_vblank() will be visible to callers of the later 1894 * functions, iff the vblank count is the same or a later one. 1895 * 1896 * See also &drm_vblank_crtc.count. 1897 * 1898 * Returns: 1899 * True if the event was successfully handled, false on failure. 1900 */ 1901bool drm_crtc_handle_vblank(struct drm_crtc *crtc) 1902{ 1903 return drm_handle_vblank(crtc->dev, drm_crtc_index(crtc)); 1904} 1905EXPORT_SYMBOL(drm_crtc_handle_vblank); 1906 1907/* 1908 * Get crtc VBLANK count. 1909 * 1910 * \param dev DRM device 1911 * \param data user arguement, pointing to a drm_crtc_get_sequence structure. 1912 * \param file_priv drm file private for the user's open file descriptor 1913 */ 1914 1915int drm_crtc_get_sequence_ioctl(struct drm_device *dev, void *data, 1916 struct drm_file *file_priv) 1917{ 1918 struct drm_crtc *crtc; 1919 struct drm_vblank_crtc *vblank; 1920 int pipe; 1921 struct drm_crtc_get_sequence *get_seq = data; 1922 ktime_t now; 1923 bool vblank_enabled; 1924 int ret; 1925 1926 if (!drm_core_check_feature(dev, DRIVER_MODESET)) 1927 return -EOPNOTSUPP; 1928 1929 if (!dev->irq_enabled) 1930 return -EOPNOTSUPP; 1931 1932 crtc = drm_crtc_find(dev, file_priv, get_seq->crtc_id); 1933 if (!crtc) 1934 return -ENOENT; 1935 1936 pipe = drm_crtc_index(crtc); 1937 1938 vblank = &dev->vblank[pipe]; 1939 vblank_enabled = dev->vblank_disable_immediate && READ_ONCE(vblank->enabled); 1940 1941 if (!vblank_enabled) { 1942 ret = drm_crtc_vblank_get(crtc); 1943 if (ret) { 1944 DRM_DEBUG("crtc %d failed to acquire vblank counter, %d\n", pipe, ret); 1945 return ret; 1946 } 1947 } 1948 drm_modeset_lock(&crtc->mutex, NULL); 1949 if (crtc->state) 1950 get_seq->active = crtc->state->enable; 1951 else 1952 get_seq->active = crtc->enabled; 1953 drm_modeset_unlock(&crtc->mutex); 1954 get_seq->sequence = drm_vblank_count_and_time(dev, pipe, &now); 1955 get_seq->sequence_ns = ktime_to_ns(now); 1956 if (!vblank_enabled) 1957 drm_crtc_vblank_put(crtc); 1958 return 0; 1959} 1960 1961/* 1962 * Queue a event for VBLANK sequence 1963 * 1964 * \param dev DRM device 1965 * \param data user arguement, pointing to a drm_crtc_queue_sequence structure. 1966 * \param file_priv drm file private for the user's open file descriptor 1967 */ 1968 1969int drm_crtc_queue_sequence_ioctl(struct drm_device *dev, void *data, 1970 struct drm_file *file_priv) 1971{ 1972 struct drm_crtc *crtc; 1973 struct drm_vblank_crtc *vblank; 1974 int pipe; 1975 struct drm_crtc_queue_sequence *queue_seq = data; 1976 ktime_t now; 1977 struct drm_pending_vblank_event *e; 1978 u32 flags; 1979 u64 seq; 1980 u64 req_seq; 1981 int ret; 1982 unsigned long spin_flags; 1983 1984 if (!drm_core_check_feature(dev, DRIVER_MODESET)) 1985 return -EOPNOTSUPP; 1986 1987 if (!dev->irq_enabled) 1988 return -EOPNOTSUPP; 1989 1990 crtc = drm_crtc_find(dev, file_priv, queue_seq->crtc_id); 1991 if (!crtc) 1992 return -ENOENT; 1993 1994 flags = queue_seq->flags; 1995 /* Check valid flag bits */ 1996 if (flags & ~(DRM_CRTC_SEQUENCE_RELATIVE| 1997 DRM_CRTC_SEQUENCE_NEXT_ON_MISS)) 1998 return -EINVAL; 1999 2000 pipe = drm_crtc_index(crtc); 2001 2002 vblank = &dev->vblank[pipe]; 2003 2004 e = kzalloc(sizeof(*e), GFP_KERNEL); 2005 if (e == NULL) 2006 return -ENOMEM; 2007 2008 ret = drm_crtc_vblank_get(crtc); 2009 if (ret) { 2010 DRM_DEBUG("crtc %d failed to acquire vblank counter, %d\n", pipe, ret); 2011 goto err_free; 2012 } 2013 2014 seq = drm_vblank_count_and_time(dev, pipe, &now); 2015 req_seq = queue_seq->sequence; 2016 2017 if (flags & DRM_CRTC_SEQUENCE_RELATIVE) 2018 req_seq += seq; 2019 2020 if ((flags & DRM_CRTC_SEQUENCE_NEXT_ON_MISS) && vblank_passed(seq, req_seq)) 2021 req_seq = seq + 1; 2022 2023 e->pipe = pipe; 2024 e->event.base.type = DRM_EVENT_CRTC_SEQUENCE; 2025 e->event.base.length = sizeof(e->event.seq); 2026 e->event.seq.user_data = queue_seq->user_data; 2027 2028 spin_lock_irqsave(&dev->event_lock, spin_flags); 2029 2030 /* 2031 * drm_crtc_vblank_off() might have been called after we called 2032 * drm_crtc_vblank_get(). drm_crtc_vblank_off() holds event_lock around the 2033 * vblank disable, so no need for further locking. The reference from 2034 * drm_crtc_vblank_get() protects against vblank disable from another source. 2035 */ 2036 if (!READ_ONCE(vblank->enabled)) { 2037 ret = -EINVAL; 2038 goto err_unlock; 2039 } 2040 2041 ret = drm_event_reserve_init_locked(dev, file_priv, &e->base, 2042 &e->event.base); 2043 2044 if (ret) 2045 goto err_unlock; 2046 2047 e->sequence = req_seq; 2048 2049 if (vblank_passed(seq, req_seq)) { 2050 drm_crtc_vblank_put(crtc); 2051 send_vblank_event(dev, e, seq, now); 2052 queue_seq->sequence = seq; 2053 } else { 2054 /* drm_handle_vblank_events will call drm_vblank_put */ 2055 list_add_tail(&e->base.link, &dev->vblank_event_list); 2056 queue_seq->sequence = req_seq; 2057 } 2058 2059 spin_unlock_irqrestore(&dev->event_lock, spin_flags); 2060 return 0; 2061 2062err_unlock: 2063 spin_unlock_irqrestore(&dev->event_lock, spin_flags); 2064 drm_crtc_vblank_put(crtc); 2065err_free: 2066 kfree(e); 2067 return ret; 2068} 2069