1/* i915_irq.c -- IRQ support for the I915 -*- linux-c -*- 2 */ 3/*- 4 * Copyright 2003 Tungsten Graphics, Inc., Cedar Park, Texas. 5 * All Rights Reserved. 6 * 7 * Permission is hereby granted, free of charge, to any person obtaining a 8 * copy of this software and associated documentation files (the 9 * "Software"), to deal in the Software without restriction, including 10 * without limitation the rights to use, copy, modify, merge, publish, 11 * distribute, sub license, and/or sell copies of the Software, and to 12 * permit persons to whom the Software is furnished to do so, subject to 13 * the following conditions: 14 * 15 * The above copyright notice and this permission notice (including the 16 * next paragraph) shall be included in all copies or substantial portions 17 * of the Software. 18 * 19 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS 20 * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 21 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. 22 * IN NO EVENT SHALL TUNGSTEN GRAPHICS AND/OR ITS SUPPLIERS BE LIABLE FOR 23 * ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, 24 * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE 25 * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 26 * 27 */ 28 29#include <sys/cdefs.h> 30__FBSDID("$FreeBSD$"); 31 32#include "dev/drm/drmP.h" 33#include "dev/drm/drm.h" 34#include "dev/drm/i915_drm.h" 35#include "dev/drm/i915_drv.h" 36 37#define MAX_NOPID ((u32)~0) 38 39/** 40 * Interrupts that are always left unmasked. 41 * 42 * Since pipe events are edge-triggered from the PIPESTAT register to IIR, 43 * we leave them always unmasked in IMR and then control enabling them through 44 * PIPESTAT alone. 45 */ 46#define I915_INTERRUPT_ENABLE_FIX (I915_DISPLAY_PIPE_A_EVENT_INTERRUPT | \ 47 I915_DISPLAY_PIPE_B_EVENT_INTERRUPT) 48 49/** Interrupts that we mask and unmask at runtime. */ 50#define I915_INTERRUPT_ENABLE_VAR (I915_USER_INTERRUPT) 51 52/** These are all of the interrupts used by the driver */ 53#define I915_INTERRUPT_ENABLE_MASK (I915_INTERRUPT_ENABLE_FIX | \ 54 I915_INTERRUPT_ENABLE_VAR) 55 56#define DRM_I915_VBLANK_PIPE_ALL (DRM_I915_VBLANK_PIPE_A | \ 57 DRM_I915_VBLANK_PIPE_B) 58 59static inline void 60i915_enable_irq(drm_i915_private_t *dev_priv, u32 mask) 61{ 62 mask &= I915_INTERRUPT_ENABLE_VAR; 63 if ((dev_priv->irq_mask_reg & mask) != 0) { 64 dev_priv->irq_mask_reg &= ~mask; 65 I915_WRITE(IMR, dev_priv->irq_mask_reg); 66 (void) I915_READ(IMR); 67 } 68} 69 70static inline void 71i915_disable_irq(drm_i915_private_t *dev_priv, u32 mask) 72{ 73 mask &= I915_INTERRUPT_ENABLE_VAR; 74 if ((dev_priv->irq_mask_reg & mask) != mask) { 75 dev_priv->irq_mask_reg |= mask; 76 I915_WRITE(IMR, dev_priv->irq_mask_reg); 77 (void) I915_READ(IMR); 78 } 79} 80 81static inline u32 82i915_pipestat(int pipe) 83{ 84 if (pipe == 0) 85 return PIPEASTAT; 86 if (pipe == 1) 87 return PIPEBSTAT; 88 return -EINVAL; 89} 90 91void 92i915_enable_pipestat(drm_i915_private_t *dev_priv, int pipe, u32 mask) 93{ 94 if ((dev_priv->pipestat[pipe] & mask) != mask) { 95 u32 reg = i915_pipestat(pipe); 96 97 dev_priv->pipestat[pipe] |= mask; 98 /* Enable the interrupt, clear any pending status */ 99 I915_WRITE(reg, dev_priv->pipestat[pipe] | (mask >> 16)); 100 (void) I915_READ(reg); 101 } 102} 103 104void 105i915_disable_pipestat(drm_i915_private_t *dev_priv, int pipe, u32 mask) 106{ 107 if ((dev_priv->pipestat[pipe] & mask) != 0) { 108 u32 reg = i915_pipestat(pipe); 109 110 dev_priv->pipestat[pipe] &= ~mask; 111 I915_WRITE(reg, dev_priv->pipestat[pipe]); 112 (void) I915_READ(reg); 113 } 114} 115 116/** 117 * i915_pipe_enabled - check if a pipe is enabled 118 * @dev: DRM device 119 * @pipe: pipe to check 120 * 121 * Reading certain registers when the pipe is disabled can hang the chip. 122 * Use this routine to make sure the PLL is running and the pipe is active 123 * before reading such registers if unsure. 124 */ 125static int 126i915_pipe_enabled(struct drm_device *dev, int pipe) 127{ 128 drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private; 129 unsigned long pipeconf = pipe ? PIPEBCONF : PIPEACONF; 130 131 if (I915_READ(pipeconf) & PIPEACONF_ENABLE) 132 return 1; 133 134 return 0; 135} 136 137/* Called from drm generic code, passed a 'crtc', which 138 * we use as a pipe index 139 */ 140u32 i915_get_vblank_counter(struct drm_device *dev, int pipe) 141{ 142 drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private; 143 unsigned long high_frame; 144 unsigned long low_frame; 145 u32 high1, high2, low, count; 146 147 high_frame = pipe ? PIPEBFRAMEHIGH : PIPEAFRAMEHIGH; 148 low_frame = pipe ? PIPEBFRAMEPIXEL : PIPEAFRAMEPIXEL; 149 150 if (!i915_pipe_enabled(dev, pipe)) { 151 DRM_DEBUG("trying to get vblank count for disabled pipe %d\n", pipe); 152 return 0; 153 } 154 155 /* 156 * High & low register fields aren't synchronized, so make sure 157 * we get a low value that's stable across two reads of the high 158 * register. 159 */ 160 do { 161 high1 = ((I915_READ(high_frame) & PIPE_FRAME_HIGH_MASK) >> 162 PIPE_FRAME_HIGH_SHIFT); 163 low = ((I915_READ(low_frame) & PIPE_FRAME_LOW_MASK) >> 164 PIPE_FRAME_LOW_SHIFT); 165 high2 = ((I915_READ(high_frame) & PIPE_FRAME_HIGH_MASK) >> 166 PIPE_FRAME_HIGH_SHIFT); 167 } while (high1 != high2); 168 169 count = (high1 << 8) | low; 170 171 return count; 172} 173 174u32 g45_get_vblank_counter(struct drm_device *dev, int pipe) 175{ 176 drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private; 177 int reg = pipe ? PIPEB_FRMCOUNT_GM45 : PIPEA_FRMCOUNT_GM45; 178 179 if (!i915_pipe_enabled(dev, pipe)) { 180 DRM_DEBUG("trying to get vblank count for disabled pipe %d\n", pipe); 181 return 0; 182 } 183 184 return I915_READ(reg); 185} 186 187irqreturn_t i915_driver_irq_handler(DRM_IRQ_ARGS) 188{ 189 struct drm_device *dev = (struct drm_device *) arg; 190 drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private; 191 u32 iir, new_iir; 192 u32 pipea_stats, pipeb_stats; 193 u32 vblank_status; 194 u32 vblank_enable; 195 int irq_received; 196 197 iir = I915_READ(IIR); 198 199 if (IS_I965G(dev)) { 200 vblank_status = PIPE_START_VBLANK_INTERRUPT_STATUS; 201 vblank_enable = PIPE_START_VBLANK_INTERRUPT_ENABLE; 202 } else { 203 vblank_status = I915_VBLANK_INTERRUPT_STATUS; 204 vblank_enable = I915_VBLANK_INTERRUPT_ENABLE; 205 } 206 207 for (;;) { 208 irq_received = iir != 0; 209 210 /* Can't rely on pipestat interrupt bit in iir as it might 211 * have been cleared after the pipestat interrupt was received. 212 * It doesn't set the bit in iir again, but it still produces 213 * interrupts (for non-MSI). 214 */ 215 DRM_SPINLOCK(&dev_priv->user_irq_lock); 216 pipea_stats = I915_READ(PIPEASTAT); 217 pipeb_stats = I915_READ(PIPEBSTAT); 218 219 /* 220 * Clear the PIPE(A|B)STAT regs before the IIR 221 */ 222 if (pipea_stats & 0x8000ffff) { 223 I915_WRITE(PIPEASTAT, pipea_stats); 224 irq_received = 1; 225 } 226 227 if (pipeb_stats & 0x8000ffff) { 228 I915_WRITE(PIPEBSTAT, pipeb_stats); 229 irq_received = 1; 230 } 231 DRM_SPINUNLOCK(&dev_priv->user_irq_lock); 232 233 if (!irq_received) 234 break; 235 236 I915_WRITE(IIR, iir); 237 new_iir = I915_READ(IIR); /* Flush posted writes */ 238 239 if (dev_priv->sarea_priv) 240 dev_priv->sarea_priv->last_dispatch = 241 READ_BREADCRUMB(dev_priv); 242 243 if (iir & I915_USER_INTERRUPT) { 244 DRM_WAKEUP(&dev_priv->irq_queue); 245 } 246 247 if (pipea_stats & vblank_status) 248 drm_handle_vblank(dev, 0); 249 250 if (pipeb_stats & vblank_status) 251 drm_handle_vblank(dev, 1); 252 253 /* With MSI, interrupts are only generated when iir 254 * transitions from zero to nonzero. If another bit got 255 * set while we were handling the existing iir bits, then 256 * we would never get another interrupt. 257 * 258 * This is fine on non-MSI as well, as if we hit this path 259 * we avoid exiting the interrupt handler only to generate 260 * another one. 261 * 262 * Note that for MSI this could cause a stray interrupt report 263 * if an interrupt landed in the time between writing IIR and 264 * the posting read. This should be rare enough to never 265 * trigger the 99% of 100,000 interrupts test for disabling 266 * stray interrupts. 267 */ 268 iir = new_iir; 269 } 270} 271 272static int i915_emit_irq(struct drm_device * dev) 273{ 274 drm_i915_private_t *dev_priv = dev->dev_private; 275 RING_LOCALS; 276 277 i915_kernel_lost_context(dev); 278 279 if (++dev_priv->counter > 0x7FFFFFFFUL) 280 dev_priv->counter = 0; 281 if (dev_priv->sarea_priv) 282 dev_priv->sarea_priv->last_enqueue = dev_priv->counter; 283 284 DRM_DEBUG("emitting: %d\n", dev_priv->counter); 285 286 BEGIN_LP_RING(4); 287 OUT_RING(MI_STORE_DWORD_INDEX); 288 OUT_RING(I915_BREADCRUMB_INDEX << MI_STORE_DWORD_INDEX_SHIFT); 289 OUT_RING(dev_priv->counter); 290 OUT_RING(MI_USER_INTERRUPT); 291 ADVANCE_LP_RING(); 292 293 return dev_priv->counter; 294} 295 296void i915_user_irq_get(struct drm_device *dev) 297{ 298 drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private; 299 300 if (dev->irq_enabled == 0) 301 return; 302 303 DRM_DEBUG("\n"); 304 DRM_SPINLOCK(&dev_priv->user_irq_lock); 305 if (++dev_priv->user_irq_refcount == 1) 306 i915_enable_irq(dev_priv, I915_USER_INTERRUPT); 307 DRM_SPINUNLOCK(&dev_priv->user_irq_lock); 308} 309 310void i915_user_irq_put(struct drm_device *dev) 311{ 312 drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private; 313 314 if (dev->irq_enabled == 0) 315 return; 316 317 DRM_SPINLOCK(&dev_priv->user_irq_lock); 318 KASSERT(dev_priv->user_irq_refcount > 0, ("invalid refcount")); 319 if (--dev_priv->user_irq_refcount == 0) 320 i915_disable_irq(dev_priv, I915_USER_INTERRUPT); 321 DRM_SPINUNLOCK(&dev_priv->user_irq_lock); 322} 323 324static int i915_wait_irq(struct drm_device * dev, int irq_nr) 325{ 326 drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private; 327 int ret = 0; 328 329 if (READ_BREADCRUMB(dev_priv) >= irq_nr) { 330 if (dev_priv->sarea_priv) { 331 dev_priv->sarea_priv->last_dispatch = 332 READ_BREADCRUMB(dev_priv); 333 } 334 return 0; 335 } 336 337 if (dev_priv->sarea_priv) 338 dev_priv->sarea_priv->perf_boxes |= I915_BOX_WAIT; 339 340 DRM_DEBUG("irq_nr=%d breadcrumb=%d\n", irq_nr, 341 READ_BREADCRUMB(dev_priv)); 342 343 i915_user_irq_get(dev); 344 DRM_WAIT_ON(ret, dev_priv->irq_queue, 3 * DRM_HZ, 345 READ_BREADCRUMB(dev_priv) >= irq_nr); 346 i915_user_irq_put(dev); 347 348 if (ret == -ERESTART) 349 DRM_DEBUG("restarting syscall\n"); 350 351 if (ret == -EBUSY) { 352 DRM_ERROR("EBUSY -- rec: %d emitted: %d\n", 353 READ_BREADCRUMB(dev_priv), (int)dev_priv->counter); 354 } 355 356 return ret; 357} 358 359/* Needs the lock as it touches the ring. 360 */ 361int i915_irq_emit(struct drm_device *dev, void *data, 362 struct drm_file *file_priv) 363{ 364 drm_i915_private_t *dev_priv = dev->dev_private; 365 drm_i915_irq_emit_t *emit = data; 366 int result; 367 368 if (!dev_priv) { 369 DRM_ERROR("called with no initialization\n"); 370 return -EINVAL; 371 } 372 373 RING_LOCK_TEST_WITH_RETURN(dev, file_priv); 374 375 result = i915_emit_irq(dev); 376 377 if (DRM_COPY_TO_USER(emit->irq_seq, &result, sizeof(int))) { 378 DRM_ERROR("copy_to_user\n"); 379 return -EFAULT; 380 } 381 382 return 0; 383} 384 385/* Doesn't need the hardware lock. 386 */ 387int i915_irq_wait(struct drm_device *dev, void *data, 388 struct drm_file *file_priv) 389{ 390 drm_i915_private_t *dev_priv = dev->dev_private; 391 drm_i915_irq_wait_t *irqwait = data; 392 393 if (!dev_priv) { 394 DRM_ERROR("called with no initialization\n"); 395 return -EINVAL; 396 } 397 398 return i915_wait_irq(dev, irqwait->irq_seq); 399} 400 401/* Called from drm generic code, passed 'crtc' which 402 * we use as a pipe index 403 */ 404int i915_enable_vblank(struct drm_device *dev, int pipe) 405{ 406 drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private; 407 408 if (!i915_pipe_enabled(dev, pipe)) 409 return -EINVAL; 410 411 DRM_SPINLOCK(&dev_priv->user_irq_lock); 412 if (IS_I965G(dev)) 413 i915_enable_pipestat(dev_priv, pipe, 414 PIPE_START_VBLANK_INTERRUPT_ENABLE); 415 else 416 i915_enable_pipestat(dev_priv, pipe, 417 PIPE_VBLANK_INTERRUPT_ENABLE); 418 DRM_SPINUNLOCK(&dev_priv->user_irq_lock); 419 return 0; 420} 421 422/* Called from drm generic code, passed 'crtc' which 423 * we use as a pipe index 424 */ 425void i915_disable_vblank(struct drm_device *dev, int pipe) 426{ 427 drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private; 428 429 DRM_SPINLOCK(&dev_priv->user_irq_lock); 430 i915_disable_pipestat(dev_priv, pipe, 431 PIPE_VBLANK_INTERRUPT_ENABLE | 432 PIPE_START_VBLANK_INTERRUPT_ENABLE); 433 DRM_SPINUNLOCK(&dev_priv->user_irq_lock); 434} 435 436/* Set the vblank monitor pipe 437 */ 438int i915_vblank_pipe_set(struct drm_device *dev, void *data, 439 struct drm_file *file_priv) 440{ 441 drm_i915_private_t *dev_priv = dev->dev_private; 442 443 if (!dev_priv) { 444 DRM_ERROR("called with no initialization\n"); 445 return -EINVAL; 446 } 447 448 return 0; 449} 450 451int i915_vblank_pipe_get(struct drm_device *dev, void *data, 452 struct drm_file *file_priv) 453{ 454 drm_i915_private_t *dev_priv = dev->dev_private; 455 drm_i915_vblank_pipe_t *pipe = data; 456 457 if (!dev_priv) { 458 DRM_ERROR("called with no initialization\n"); 459 return -EINVAL; 460 } 461 462 pipe->pipe = DRM_I915_VBLANK_PIPE_A | DRM_I915_VBLANK_PIPE_B; 463 464 return 0; 465} 466 467/** 468 * Schedule buffer swap at given vertical blank. 469 */ 470int i915_vblank_swap(struct drm_device *dev, void *data, 471 struct drm_file *file_priv) 472{ 473 /* The delayed swap mechanism was fundamentally racy, and has been 474 * removed. The model was that the client requested a delayed flip/swap 475 * from the kernel, then waited for vblank before continuing to perform 476 * rendering. The problem was that the kernel might wake the client 477 * up before it dispatched the vblank swap (since the lock has to be 478 * held while touching the ringbuffer), in which case the client would 479 * clear and start the next frame before the swap occurred, and 480 * flicker would occur in addition to likely missing the vblank. 481 * 482 * In the absence of this ioctl, userland falls back to a correct path 483 * of waiting for a vblank, then dispatching the swap on its own. 484 * Context switching to userland and back is plenty fast enough for 485 * meeting the requirements of vblank swapping. 486 */ 487 return -EINVAL; 488} 489 490/* drm_dma.h hooks 491*/ 492void i915_driver_irq_preinstall(struct drm_device * dev) 493{ 494 drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private; 495 496 I915_WRITE(HWSTAM, 0xeffe); 497 I915_WRITE(PIPEASTAT, 0); 498 I915_WRITE(PIPEBSTAT, 0); 499 I915_WRITE(IMR, 0xffffffff); 500 I915_WRITE(IER, 0x0); 501 (void) I915_READ(IER); 502} 503 504int i915_driver_irq_postinstall(struct drm_device *dev) 505{ 506 drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private; 507 508 dev_priv->vblank_pipe = DRM_I915_VBLANK_PIPE_A | DRM_I915_VBLANK_PIPE_B; 509 510 /* Unmask the interrupts that we always want on. */ 511 dev_priv->irq_mask_reg = ~I915_INTERRUPT_ENABLE_FIX; 512 513 /* Disable pipe interrupt enables, clear pending pipe status */ 514 I915_WRITE(PIPEASTAT, I915_READ(PIPEASTAT) & 0x8000ffff); 515 I915_WRITE(PIPEBSTAT, I915_READ(PIPEBSTAT) & 0x8000ffff); 516 517 /* Clear pending interrupt status */ 518 I915_WRITE(IIR, I915_READ(IIR)); 519 520 I915_WRITE(IER, I915_INTERRUPT_ENABLE_MASK); 521 I915_WRITE(IMR, dev_priv->irq_mask_reg); 522 I915_WRITE(PIPEASTAT, dev_priv->pipestat[0] | 523 (dev_priv->pipestat[0] >> 16)); 524 I915_WRITE(PIPEBSTAT, dev_priv->pipestat[1] | 525 (dev_priv->pipestat[1] >> 16)); 526 (void) I915_READ(IER); 527 528 return 0; 529} 530 531void i915_driver_irq_uninstall(struct drm_device * dev) 532{ 533 drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private; 534 535 if (!dev_priv) 536 return; 537 538 dev_priv->vblank_pipe = 0; 539 540 I915_WRITE(HWSTAM, 0xffffffff); 541 I915_WRITE(PIPEASTAT, 0); 542 I915_WRITE(PIPEBSTAT, 0); 543 I915_WRITE(IMR, 0xffffffff); 544 I915_WRITE(IER, 0x0); 545 546 I915_WRITE(PIPEASTAT, I915_READ(PIPEASTAT) & 0x8000ffff); 547 I915_WRITE(PIPEBSTAT, I915_READ(PIPEBSTAT) & 0x8000ffff); 548 I915_WRITE(IIR, I915_READ(IIR)); 549} 550