via_dmablit.c revision 1.3
1/* via_dmablit.c -- PCI DMA BitBlt support for the VIA Unichrome/Pro 2 * 3 * Copyright (C) 2005 Thomas Hellstrom, All Rights Reserved. 4 * 5 * Permission is hereby granted, free of charge, to any person obtaining a 6 * copy of this software and associated documentation files (the "Software"), 7 * to deal in the Software without restriction, including without limitation 8 * the rights to use, copy, modify, merge, publish, distribute, sub license, 9 * and/or sell copies of the Software, and to permit persons to whom the 10 * Software is furnished to do so, subject to the following conditions: 11 * 12 * The above copyright notice and this permission notice (including the 13 * next paragraph) shall be included in all copies or substantial portions 14 * of the Software. 15 * 16 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 18 * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL 19 * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM, 20 * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR 21 * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE 22 * USE OR OTHER DEALINGS IN THE SOFTWARE. 23 * 24 * Authors: 25 * Thomas Hellstrom. 26 * Partially based on code obtained from Digeo Inc. 27 */ 28 29 30/* 31 * Unmaps the DMA mappings. 32 * FIXME: Is this a NoOp on x86? Also 33 * FIXME: What happens if this one is called and a pending blit has previously done 34 * the same DMA mappings? 35 */ 36 37#include <drm/drmP.h> 38#include <drm/via_drm.h> 39#include "via_drv.h" 40#include "via_dmablit.h" 41 42#include <linux/pagemap.h> 43#include <linux/slab.h> 44#include <linux/timer.h> 45 46#define VIA_PGDN(x) (((unsigned long)(x)) & PAGE_MASK) 47#define VIA_PGOFF(x) (((unsigned long)(x)) & ~PAGE_MASK) 48#define VIA_PFN(x) ((unsigned long)(x) >> PAGE_SHIFT) 49 50typedef struct _drm_via_descriptor { 51 uint32_t mem_addr; 52 uint32_t dev_addr; 53 uint32_t size; 54 uint32_t next; 55} drm_via_descriptor_t; 56 57 58/* 59 * Unmap a DMA mapping. 60 */ 61 62 63 64static void 65via_unmap_blit_from_device(struct drm_device *dev, struct pci_dev *pdev, 66 drm_via_sg_info_t *vsg) 67{ 68#ifdef __NetBSD__ 69 bus_dmamap_unload(dev->dmat, vsg->dmamap); 70#else 71 int num_desc = vsg->num_desc; 72 unsigned cur_descriptor_page = num_desc / vsg->descriptors_per_page; 73 unsigned descriptor_this_page = num_desc % vsg->descriptors_per_page; 74 drm_via_descriptor_t *desc_ptr = vsg->desc_pages[cur_descriptor_page] + 75 descriptor_this_page; 76 dma_addr_t next = vsg->chain_start; 77 78 while (num_desc--) { 79 if (descriptor_this_page-- == 0) { 80 cur_descriptor_page--; 81 descriptor_this_page = vsg->descriptors_per_page - 1; 82 desc_ptr = vsg->desc_pages[cur_descriptor_page] + 83 descriptor_this_page; 84 } 85 dma_unmap_single(&pdev->dev, next, sizeof(*desc_ptr), DMA_TO_DEVICE); 86 dma_unmap_page(&pdev->dev, desc_ptr->mem_addr, desc_ptr->size, vsg->direction); 87 next = (dma_addr_t) desc_ptr->next; 88 desc_ptr--; 89 } 90#endif 91} 92 93/* 94 * If mode = 0, count how many descriptors are needed. 95 * If mode = 1, Map the DMA pages for the device, put together and map also the descriptors. 96 * Descriptors are run in reverse order by the hardware because we are not allowed to update the 97 * 'next' field without syncing calls when the descriptor is already mapped. 98 */ 99 100static void 101via_map_blit_for_device(struct pci_dev *pdev, 102 const drm_via_dmablit_t *xfer, 103 drm_via_sg_info_t *vsg, 104 int mode) 105{ 106 unsigned cur_descriptor_page = 0; 107 unsigned num_descriptors_this_page = 0; 108 unsigned char *mem_addr = xfer->mem_addr; 109 unsigned char *cur_mem; 110#ifndef __NetBSD__ 111 unsigned char *first_addr = (unsigned char *)VIA_PGDN(mem_addr); 112#endif 113 uint32_t fb_addr = xfer->fb_addr; 114 uint32_t cur_fb; 115 unsigned long line_len; 116 unsigned remaining_len; 117 int num_desc = 0; 118 int cur_line; 119 dma_addr_t next = 0 | VIA_DMA_DPR_EC; 120 drm_via_descriptor_t *desc_ptr = NULL; 121 122 if (mode == 1) 123 desc_ptr = vsg->desc_pages[cur_descriptor_page]; 124 125 for (cur_line = 0; cur_line < xfer->num_lines; ++cur_line) { 126 127 line_len = xfer->line_length; 128 cur_fb = fb_addr; 129 cur_mem = mem_addr; 130 131 while (line_len > 0) { 132 133 remaining_len = min(PAGE_SIZE-VIA_PGOFF(cur_mem), line_len); 134 line_len -= remaining_len; 135 136 if (mode == 1) { 137#ifdef __NetBSD__ 138 const bus_dma_segment_t *const seg = 139 &vsg->dmamap->dm_segs[atop(cur_mem)]; 140 desc_ptr->mem_addr = 141 seg->ds_addr + trunc_page((vaddr_t)cur_mem); 142#else 143 desc_ptr->mem_addr = 144 dma_map_page(&pdev->dev, 145 vsg->pages[VIA_PFN(cur_mem) - 146 VIA_PFN(first_addr)], 147 VIA_PGOFF(cur_mem), remaining_len, 148 vsg->direction); 149#endif 150 desc_ptr->dev_addr = cur_fb; 151 152 desc_ptr->size = remaining_len; 153 desc_ptr->next = (uint32_t) next; 154#ifdef __NetBSD__ 155 next = vsg->desc_dmamap 156 ->dm_segs[cur_descriptor_page].ds_addr 157 + num_descriptors_this_page; 158#else 159 next = dma_map_single(&pdev->dev, desc_ptr, sizeof(*desc_ptr), 160 DMA_TO_DEVICE); 161#endif 162 desc_ptr++; 163 if (++num_descriptors_this_page >= vsg->descriptors_per_page) { 164 num_descriptors_this_page = 0; 165 desc_ptr = vsg->desc_pages[++cur_descriptor_page]; 166 } 167 } 168 169 num_desc++; 170 cur_mem += remaining_len; 171 cur_fb += remaining_len; 172 } 173 174 mem_addr += xfer->mem_stride; 175 fb_addr += xfer->fb_stride; 176 } 177 178 if (mode == 1) { 179 vsg->chain_start = next; 180 vsg->state = dr_via_device_mapped; 181 } 182 vsg->num_desc = num_desc; 183} 184 185/* 186 * Function that frees up all resources for a blit. It is usable even if the 187 * blit info has only been partially built as long as the status enum is consistent 188 * with the actual status of the used resources. 189 */ 190 191 192static void 193via_free_sg_info(struct drm_device *dev, struct pci_dev *pdev, 194 drm_via_sg_info_t *vsg) 195{ 196#ifndef __NetBSD__ 197 struct page *page; 198 int i; 199#endif 200 201 switch (vsg->state) { 202 case dr_via_device_mapped: 203 via_unmap_blit_from_device(dev, pdev, vsg); 204 case dr_via_desc_pages_alloc: 205#ifdef __NetBSD__ 206 bus_dmamap_unload(dev->dmat, vsg->desc_dmamap); 207 bus_dmamap_destroy(dev->dmat, vsg->desc_dmamap); 208 bus_dmamem_unmap(dev->dmat, vsg->desc_kva, 209 vsg->num_desc_pages << PAGE_SHIFT); 210 bus_dmamem_free(dev->dmat, vsg->desc_segs, vsg->num_desc_segs); 211 kfree(vsg->desc_segs); 212#else 213 for (i = 0; i < vsg->num_desc_pages; ++i) { 214 if (vsg->desc_pages[i] != NULL) 215 free_page((unsigned long)vsg->desc_pages[i]); 216 } 217#endif 218 kfree(vsg->desc_pages); 219 case dr_via_pages_locked: 220#ifdef __NetBSD__ 221 /* Make sure any completed transfer is synced. */ 222 bus_dmamap_sync(dev->dmat, vsg->dmamap, 0, 223 vsg->num_pages << PAGE_SHIFT, 224 (vsg->direction == DMA_FROM_DEVICE? 225 BUS_DMASYNC_POSTREAD : BUS_DMASYNC_POSTWRITE)); 226#else 227 for (i = 0; i < vsg->num_pages; ++i) { 228 if (NULL != (page = vsg->pages[i])) { 229 if (!PageReserved(page) && (DMA_FROM_DEVICE == vsg->direction)) 230 SetPageDirty(page); 231 page_cache_release(page); 232 } 233 } 234#endif 235 case dr_via_pages_alloc: 236#ifdef __NetBSD__ 237 bus_dmamap_destroy(dev->dmat, vsg->dmamap); 238#else 239 vfree(vsg->pages); 240#endif 241 default: 242 vsg->state = dr_via_sg_init; 243 } 244 vsg->free_on_sequence = 0; 245} 246 247/* 248 * Fire a blit engine. 249 */ 250 251static void 252via_fire_dmablit(struct drm_device *dev, drm_via_sg_info_t *vsg, int engine) 253{ 254 drm_via_private_t *dev_priv = (drm_via_private_t *)dev->dev_private; 255 256 VIA_WRITE(VIA_PCI_DMA_MAR0 + engine*0x10, 0); 257 VIA_WRITE(VIA_PCI_DMA_DAR0 + engine*0x10, 0); 258 VIA_WRITE(VIA_PCI_DMA_CSR0 + engine*0x04, VIA_DMA_CSR_DD | VIA_DMA_CSR_TD | 259 VIA_DMA_CSR_DE); 260 VIA_WRITE(VIA_PCI_DMA_MR0 + engine*0x04, VIA_DMA_MR_CM | VIA_DMA_MR_TDIE); 261 VIA_WRITE(VIA_PCI_DMA_BCR0 + engine*0x10, 0); 262 VIA_WRITE(VIA_PCI_DMA_DPR0 + engine*0x10, vsg->chain_start); 263 wmb(); 264 VIA_WRITE(VIA_PCI_DMA_CSR0 + engine*0x04, VIA_DMA_CSR_DE | VIA_DMA_CSR_TS); 265 VIA_READ(VIA_PCI_DMA_CSR0 + engine*0x04); 266} 267 268/* 269 * Obtain a page pointer array and lock all pages into system memory. A segmentation violation will 270 * occur here if the calling user does not have access to the submitted address. 271 */ 272 273static int 274via_lock_all_dma_pages(struct drm_device *dev, drm_via_sg_info_t *vsg, 275 drm_via_dmablit_t *xfer) 276{ 277 int ret; 278#ifdef __NetBSD__ 279 const bus_size_t nbytes = roundup2(xfer->num_lines * xfer->mem_stride, 280 PAGE_SIZE); 281 const bus_size_t npages = nbytes >> PAGE_SHIFT; 282 struct iovec iov = { 283 .iov_base = xfer->mem_addr, 284 .iov_len = nbytes, 285 }; 286 struct uio uio = { 287 .uio_iov = &iov, 288 .uio_iovcnt = 1, 289 .uio_offset = 0, 290 .uio_resid = nbytes, 291 .uio_rw = xfer->to_fb ? UIO_WRITE : UIO_READ, 292 .uio_vmspace = curproc->p_vmspace, 293 }; 294 295 /* 296 * XXX Lock out anyone else from doing this? Add a 297 * dr_via_pages_loading state? Just rely on the giant lock? 298 */ 299 /* XXX errno NetBSD->Linux */ 300 ret = -bus_dmamap_create(dev->dmat, nbytes, npages, nbytes, PAGE_SIZE, 301 BUS_DMA_WAITOK, &vsg->dmamap); 302 if (ret) { 303 DRM_ERROR("bus_dmamap_create failed: %d\n", ret); 304 return ret; 305 } 306 ret = -bus_dmamap_load_uio(dev->dmat, vsg->dmamap, &uio, 307 BUS_DMA_WAITOK | (xfer->to_fb? BUS_DMA_WRITE : BUS_DMA_READ)); 308 if (ret) { 309 DRM_ERROR("bus_dmamap_load failed: %d\n", ret); 310 bus_dmamap_destroy(dev->dmat, vsg->dmamap); 311 return ret; 312 } 313 vsg->num_pages = npages; 314#else 315 unsigned long first_pfn = VIA_PFN(xfer->mem_addr); 316 vsg->num_pages = VIA_PFN(xfer->mem_addr + (xfer->num_lines * xfer->mem_stride - 1)) - 317 first_pfn + 1; 318 319 vsg->pages = vzalloc(sizeof(struct page *) * vsg->num_pages); 320 if (NULL == vsg->pages) 321 return -ENOMEM; 322 down_read(¤t->mm->mmap_sem); 323 ret = get_user_pages(current, current->mm, 324 (unsigned long)xfer->mem_addr, 325 vsg->num_pages, 326 (vsg->direction == DMA_FROM_DEVICE), 327 0, vsg->pages, NULL); 328 329 up_read(¤t->mm->mmap_sem); 330 if (ret != vsg->num_pages) { 331 if (ret < 0) 332 return ret; 333 vsg->state = dr_via_pages_locked; 334 return -EINVAL; 335 } 336#endif 337 vsg->state = dr_via_pages_locked; 338 DRM_DEBUG("DMA pages locked\n"); 339 return 0; 340} 341 342/* 343 * Allocate DMA capable memory for the blit descriptor chain, and an array that keeps track of the 344 * pages we allocate. We don't want to use kmalloc for the descriptor chain because it may be 345 * quite large for some blits, and pages don't need to be contingous. 346 */ 347 348static int 349via_alloc_desc_pages(struct drm_device *dev, drm_via_sg_info_t *vsg) 350{ 351 int i; 352#ifdef __NetBSD__ 353 int ret; 354#endif 355 356 vsg->descriptors_per_page = PAGE_SIZE / sizeof(drm_via_descriptor_t); 357 vsg->num_desc_pages = (vsg->num_desc + vsg->descriptors_per_page - 1) / 358 vsg->descriptors_per_page; 359 360 if (NULL == (vsg->desc_pages = kcalloc(vsg->num_desc_pages, sizeof(void *), GFP_KERNEL))) 361 return -ENOMEM; 362 363#ifdef __NetBSD__ 364 vsg->desc_segs = kcalloc(vsg->num_desc_pages, sizeof(*vsg->desc_segs), 365 GFP_KERNEL); 366 if (vsg->desc_segs == NULL) { 367 kfree(vsg->desc_pages); 368 return -ENOMEM; 369 } 370 /* XXX errno NetBSD->Linux */ 371 ret = -bus_dmamem_alloc(dev->dmat, vsg->num_desc_pages << PAGE_SHIFT, 372 PAGE_SIZE, 0, vsg->desc_segs, vsg->num_pages, &vsg->num_desc_segs, 373 BUS_DMA_WAITOK); 374 if (ret) { 375 kfree(vsg->desc_segs); 376 kfree(vsg->desc_pages); 377 return -ENOMEM; 378 } 379 /* XXX No nice way to scatter/gather map bus_dmamem. */ 380 /* XXX errno NetBSD->Linux */ 381 ret = -bus_dmamem_map(dev->dmat, vsg->desc_segs, vsg->num_desc_segs, 382 vsg->num_desc_pages << PAGE_SHIFT, &vsg->desc_kva, BUS_DMA_WAITOK); 383 if (ret) { 384 bus_dmamem_free(dev->dmat, vsg->desc_segs, vsg->num_desc_segs); 385 kfree(vsg->desc_segs); 386 kfree(vsg->desc_pages); 387 return -ENOMEM; 388 } 389 /* XXX errno NetBSD->Linux */ 390 ret = -bus_dmamap_create(dev->dmat, vsg->num_desc_pages << PAGE_SHIFT, 391 vsg->num_desc_pages, PAGE_SIZE, 0, BUS_DMA_WAITOK, 392 &vsg->desc_dmamap); 393 if (ret) { 394 bus_dmamem_unmap(dev->dmat, vsg->desc_kva, 395 vsg->num_desc_pages << PAGE_SHIFT); 396 bus_dmamem_free(dev->dmat, vsg->desc_segs, vsg->num_desc_segs); 397 kfree(vsg->desc_segs); 398 kfree(vsg->desc_pages); 399 return -ENOMEM; 400 } 401 ret = -bus_dmamap_load(dev->dmat, vsg->desc_dmamap, vsg->desc_kva, 402 vsg->num_desc_pages << PAGE_SHIFT, NULL, BUS_DMA_WAITOK); 403 if (ret) { 404 bus_dmamap_destroy(dev->dmat, vsg->desc_dmamap); 405 bus_dmamem_unmap(dev->dmat, vsg->desc_kva, 406 vsg->num_desc_pages << PAGE_SHIFT); 407 bus_dmamem_free(dev->dmat, vsg->desc_segs, vsg->num_desc_segs); 408 kfree(vsg->desc_segs); 409 kfree(vsg->desc_pages); 410 return -ENOMEM; 411 } 412 for (i = 0; i < vsg->num_desc_pages; i++) 413 vsg->desc_pages[i] = (void *) 414 ((char *)vsg->desc_kva + (i * PAGE_SIZE)); 415 vsg->state = dr_via_desc_pages_alloc; 416#else 417 vsg->state = dr_via_desc_pages_alloc; 418 for (i = 0; i < vsg->num_desc_pages; ++i) { 419 if (NULL == (vsg->desc_pages[i] = 420 (drm_via_descriptor_t *) __get_free_page(GFP_KERNEL))) 421 return -ENOMEM; 422 } 423#endif 424 DRM_DEBUG("Allocated %d pages for %d descriptors.\n", vsg->num_desc_pages, 425 vsg->num_desc); 426 return 0; 427} 428 429static void 430via_abort_dmablit(struct drm_device *dev, int engine) 431{ 432 drm_via_private_t *dev_priv = (drm_via_private_t *)dev->dev_private; 433 434 VIA_WRITE(VIA_PCI_DMA_CSR0 + engine*0x04, VIA_DMA_CSR_TA); 435} 436 437static void 438via_dmablit_engine_off(struct drm_device *dev, int engine) 439{ 440 drm_via_private_t *dev_priv = (drm_via_private_t *)dev->dev_private; 441 442 VIA_WRITE(VIA_PCI_DMA_CSR0 + engine*0x04, VIA_DMA_CSR_TD | VIA_DMA_CSR_DD); 443} 444 445 446 447/* 448 * The dmablit part of the IRQ handler. Trying to do only reasonably fast things here. 449 * The rest, like unmapping and freeing memory for done blits is done in a separate workqueue 450 * task. Basically the task of the interrupt handler is to submit a new blit to the engine, while 451 * the workqueue task takes care of processing associated with the old blit. 452 */ 453 454void 455via_dmablit_handler(struct drm_device *dev, int engine, int from_irq) 456{ 457 drm_via_private_t *dev_priv = (drm_via_private_t *)dev->dev_private; 458 drm_via_blitq_t *blitq = dev_priv->blit_queues + engine; 459 int cur; 460 int done_transfer; 461 unsigned long irqsave = 0; 462 uint32_t status = 0; 463 464 DRM_DEBUG("DMA blit handler called. engine = %d, from_irq = %d, blitq = 0x%lx\n", 465 engine, from_irq, (unsigned long) blitq); 466 467 if (from_irq) 468 spin_lock(&blitq->blit_lock); 469 else 470 spin_lock_irqsave(&blitq->blit_lock, irqsave); 471 472 done_transfer = blitq->is_active && 473 ((status = VIA_READ(VIA_PCI_DMA_CSR0 + engine*0x04)) & VIA_DMA_CSR_TD); 474 done_transfer = done_transfer || (blitq->aborting && !(status & VIA_DMA_CSR_DE)); 475 476 cur = blitq->cur; 477 if (done_transfer) { 478 479 blitq->blits[cur]->aborted = blitq->aborting; 480 blitq->done_blit_handle++; 481#ifdef __NetBSD__ 482 DRM_SPIN_WAKEUP_ALL(&blitq->blit_queue[cur], 483 &blitq->blit_lock); 484#else 485 wake_up(blitq->blit_queue + cur); 486#endif 487 488 cur++; 489 if (cur >= VIA_NUM_BLIT_SLOTS) 490 cur = 0; 491 blitq->cur = cur; 492 493 /* 494 * Clear transfer done flag. 495 */ 496 497 VIA_WRITE(VIA_PCI_DMA_CSR0 + engine*0x04, VIA_DMA_CSR_TD); 498 499 blitq->is_active = 0; 500 blitq->aborting = 0; 501 schedule_work(&blitq->wq); 502 503 } else if (blitq->is_active && time_after_eq(jiffies, blitq->end)) { 504 505 /* 506 * Abort transfer after one second. 507 */ 508 509 via_abort_dmablit(dev, engine); 510 blitq->aborting = 1; 511 blitq->end = jiffies + DRM_HZ; 512 } 513 514 if (!blitq->is_active) { 515 if (blitq->num_outstanding) { 516 via_fire_dmablit(dev, blitq->blits[cur], engine); 517 blitq->is_active = 1; 518 blitq->cur = cur; 519 blitq->num_outstanding--; 520 blitq->end = jiffies + DRM_HZ; 521 if (!timer_pending(&blitq->poll_timer)) 522 mod_timer(&blitq->poll_timer, jiffies + 1); 523 } else { 524 if (timer_pending(&blitq->poll_timer)) 525 del_timer(&blitq->poll_timer); 526 via_dmablit_engine_off(dev, engine); 527 } 528 } 529 530 if (from_irq) 531 spin_unlock(&blitq->blit_lock); 532 else 533 spin_unlock_irqrestore(&blitq->blit_lock, irqsave); 534} 535 536 537 538/* 539 * Check whether this blit is still active, performing necessary locking. 540 */ 541 542static int 543#ifdef __NetBSD__ 544via_dmablit_active(drm_via_blitq_t *blitq, int engine, uint32_t handle, drm_waitqueue_t **queue) 545#else 546via_dmablit_active(drm_via_blitq_t *blitq, int engine, uint32_t handle, wait_queue_head_t **queue) 547#endif 548{ 549#ifndef __NetBSD__ 550 unsigned long irqsave; 551#endif 552 uint32_t slot; 553 int active; 554 555#ifndef __NetBSD__ 556 spin_lock_irqsave(&blitq->blit_lock, irqsave); 557#endif 558 559 /* 560 * Allow for handle wraparounds. 561 */ 562 563 active = ((blitq->done_blit_handle - handle) > (1 << 23)) && 564 ((blitq->cur_blit_handle - handle) <= (1 << 23)); 565 566 if (queue && active) { 567 slot = handle - blitq->done_blit_handle + blitq->cur - 1; 568 if (slot >= VIA_NUM_BLIT_SLOTS) 569 slot -= VIA_NUM_BLIT_SLOTS; 570 *queue = blitq->blit_queue + slot; 571 } 572 573#ifndef __NetBSD__ 574 spin_unlock_irqrestore(&blitq->blit_lock, irqsave); 575#endif 576 577 return active; 578} 579 580/* 581 * Sync. Wait for at least three seconds for the blit to be performed. 582 */ 583 584static int 585via_dmablit_sync(struct drm_device *dev, uint32_t handle, int engine) 586{ 587 588 drm_via_private_t *dev_priv = (drm_via_private_t *)dev->dev_private; 589 drm_via_blitq_t *blitq = dev_priv->blit_queues + engine; 590#ifdef __NetBSD__ 591 drm_waitqueue_t *queue; 592#else 593 wait_queue_head_t *queue; 594#endif 595 int ret = 0; 596 597#ifdef __NetBSD__ 598 spin_lock(&blitq->blit_lock); 599 if (via_dmablit_active(blitq, engine, handle, &queue)) { 600 DRM_SPIN_TIMED_WAIT_UNTIL(ret, queue, &blitq->blit_lock, 601 3*DRM_HZ, 602 !via_dmablit_active(blitq, engine, handle, NULL)); 603 if (ret < 0) /* Failure: return negative error as is. */ 604 ; 605 else if (ret == 0) /* Timed out: return -EBUSY like Linux. */ 606 ret = -EBUSY; 607 else /* Succeeded (ret > 0): return 0. */ 608 ret = 0; 609 } 610 spin_unlock(&blitq->blit_lock); 611#else 612 if (via_dmablit_active(blitq, engine, handle, &queue)) { 613 DRM_WAIT_ON(ret, *queue, 3 * HZ, 614 !via_dmablit_active(blitq, engine, handle, NULL)); 615 } 616#endif 617 DRM_DEBUG("DMA blit sync handle 0x%x engine %d returned %d\n", 618 handle, engine, ret); 619 620 return ret; 621} 622 623 624/* 625 * A timer that regularly polls the blit engine in cases where we don't have interrupts: 626 * a) Broken hardware (typically those that don't have any video capture facility). 627 * b) Blit abort. The hardware doesn't send an interrupt when a blit is aborted. 628 * The timer and hardware IRQ's can and do work in parallel. If the hardware has 629 * irqs, it will shorten the latency somewhat. 630 */ 631 632 633 634static void 635via_dmablit_timer(unsigned long data) 636{ 637 drm_via_blitq_t *blitq = (drm_via_blitq_t *) data; 638 struct drm_device *dev = blitq->dev; 639 int engine = (int) 640 (blitq - ((drm_via_private_t *)dev->dev_private)->blit_queues); 641 642 DRM_DEBUG("Polling timer called for engine %d, jiffies %lu\n", engine, 643 (unsigned long) jiffies); 644 645 via_dmablit_handler(dev, engine, 0); 646 647 if (!timer_pending(&blitq->poll_timer)) { 648 mod_timer(&blitq->poll_timer, jiffies + 1); 649 650 /* 651 * Rerun handler to delete timer if engines are off, and 652 * to shorten abort latency. This is a little nasty. 653 */ 654 655 via_dmablit_handler(dev, engine, 0); 656 657 } 658} 659 660 661 662 663/* 664 * Workqueue task that frees data and mappings associated with a blit. 665 * Also wakes up waiting processes. Each of these tasks handles one 666 * blit engine only and may not be called on each interrupt. 667 */ 668 669 670static void 671via_dmablit_workqueue(struct work_struct *work) 672{ 673 drm_via_blitq_t *blitq = container_of(work, drm_via_blitq_t, wq); 674 struct drm_device *dev = blitq->dev; 675 unsigned long irqsave; 676 drm_via_sg_info_t *cur_sg; 677 int cur_released; 678 679 680 DRM_DEBUG("Workqueue task called for blit engine %ld\n", (unsigned long) 681 (blitq - ((drm_via_private_t *)dev->dev_private)->blit_queues)); 682 683 spin_lock_irqsave(&blitq->blit_lock, irqsave); 684 685 while (blitq->serviced != blitq->cur) { 686 687 cur_released = blitq->serviced++; 688 689 DRM_DEBUG("Releasing blit slot %d\n", cur_released); 690 691 if (blitq->serviced >= VIA_NUM_BLIT_SLOTS) 692 blitq->serviced = 0; 693 694 cur_sg = blitq->blits[cur_released]; 695 blitq->num_free++; 696 697#ifdef __NetBSD__ 698 DRM_SPIN_WAKEUP_ONE(&blitq->busy_queue, &blitq->blit_lock); 699#endif 700 701 spin_unlock_irqrestore(&blitq->blit_lock, irqsave); 702 703#ifndef __NetBSD__ 704 wake_up(&blitq->busy_queue); 705#endif 706 707#ifdef __NetBSD__ 708 /* Transfer completed. Sync it. */ 709 bus_dmamap_sync(dev->dmat, cur_sg->dmamap, 0, 710 cur_sg->num_pages << PAGE_SHIFT, 711 (cur_sg->direction == DMA_FROM_DEVICE 712 ? BUS_DMASYNC_POSTREAD 713 : BUS_DMASYNC_POSTWRITE)); 714#endif 715 via_free_sg_info(dev, dev->pdev, cur_sg); 716 kfree(cur_sg); 717 718 spin_lock_irqsave(&blitq->blit_lock, irqsave); 719 } 720 721 spin_unlock_irqrestore(&blitq->blit_lock, irqsave); 722} 723 724 725/* 726 * Init all blit engines. Currently we use two, but some hardware have 4. 727 */ 728 729 730void 731via_init_dmablit(struct drm_device *dev) 732{ 733 int i, j; 734 drm_via_private_t *dev_priv = (drm_via_private_t *)dev->dev_private; 735 drm_via_blitq_t *blitq; 736 737 pci_set_master(dev->pdev); 738 739 for (i = 0; i < VIA_NUM_BLIT_ENGINES; ++i) { 740 blitq = dev_priv->blit_queues + i; 741 blitq->dev = dev; 742 blitq->cur_blit_handle = 0; 743 blitq->done_blit_handle = 0; 744 blitq->head = 0; 745 blitq->cur = 0; 746 blitq->serviced = 0; 747 blitq->num_free = VIA_NUM_BLIT_SLOTS - 1; 748 blitq->num_outstanding = 0; 749 blitq->is_active = 0; 750 blitq->aborting = 0; 751 spin_lock_init(&blitq->blit_lock); 752#ifdef __NetBSD__ 753 for (j = 0; j < VIA_NUM_BLIT_SLOTS; ++j) 754 DRM_INIT_WAITQUEUE(blitq->blit_queue + j, "viablt"); 755 DRM_INIT_WAITQUEUE(&blitq->busy_queue, "viabusy"); 756#else 757 for (j = 0; j < VIA_NUM_BLIT_SLOTS; ++j) 758 init_waitqueue_head(blitq->blit_queue + j); 759 init_waitqueue_head(&blitq->busy_queue); 760#endif 761 INIT_WORK(&blitq->wq, via_dmablit_workqueue); 762 setup_timer(&blitq->poll_timer, via_dmablit_timer, 763 (unsigned long)blitq); 764 } 765} 766 767/* 768 * Build all info and do all mappings required for a blit. 769 */ 770 771 772static int 773via_build_sg_info(struct drm_device *dev, drm_via_sg_info_t *vsg, drm_via_dmablit_t *xfer) 774{ 775 int draw = xfer->to_fb; 776 int ret = 0; 777 778 vsg->direction = (draw) ? DMA_TO_DEVICE : DMA_FROM_DEVICE; 779 780 vsg->state = dr_via_sg_init; 781 782 if (xfer->num_lines <= 0 || xfer->line_length <= 0) { 783 DRM_ERROR("Zero size bitblt.\n"); 784 return -EINVAL; 785 } 786 787 /* 788 * Below check is a driver limitation, not a hardware one. We 789 * don't want to lock unused pages, and don't want to incoporate the 790 * extra logic of avoiding them. Make sure there are no. 791 * (Not a big limitation anyway.) 792 */ 793 794 if ((xfer->mem_stride - xfer->line_length) > 2*PAGE_SIZE) { 795 DRM_ERROR("Too large system memory stride. Stride: %d, " 796 "Length: %d\n", xfer->mem_stride, xfer->line_length); 797 return -EINVAL; 798 } 799 800 if ((xfer->mem_stride == xfer->line_length) && 801 (xfer->fb_stride == xfer->line_length)) { 802 xfer->mem_stride *= xfer->num_lines; 803 xfer->line_length = xfer->mem_stride; 804 xfer->fb_stride = xfer->mem_stride; 805 xfer->num_lines = 1; 806 } 807 808 /* 809 * Don't lock an arbitrary large number of pages, since that causes a 810 * DOS security hole. 811 */ 812 813 if (xfer->num_lines > 2048 || (xfer->num_lines*xfer->mem_stride > (2048*2048*4))) { 814 DRM_ERROR("Too large PCI DMA bitblt.\n"); 815 return -EINVAL; 816 } 817 818 /* 819 * we allow a negative fb stride to allow flipping of images in 820 * transfer. 821 */ 822 823 if (xfer->mem_stride < xfer->line_length || 824 abs(xfer->fb_stride) < xfer->line_length) { 825 DRM_ERROR("Invalid frame-buffer / memory stride.\n"); 826 return -EINVAL; 827 } 828 829 /* 830 * A hardware bug seems to be worked around if system memory addresses start on 831 * 16 byte boundaries. This seems a bit restrictive however. VIA is contacted 832 * about this. Meanwhile, impose the following restrictions: 833 */ 834 835#ifdef VIA_BUGFREE 836 if ((((unsigned long)xfer->mem_addr & 3) != ((unsigned long)xfer->fb_addr & 3)) || 837 ((xfer->num_lines > 1) && ((xfer->mem_stride & 3) != (xfer->fb_stride & 3)))) { 838 DRM_ERROR("Invalid DRM bitblt alignment.\n"); 839 return -EINVAL; 840 } 841#else 842 if ((((unsigned long)xfer->mem_addr & 15) || 843 ((unsigned long)xfer->fb_addr & 3)) || 844 ((xfer->num_lines > 1) && 845 ((xfer->mem_stride & 15) || (xfer->fb_stride & 3)))) { 846 DRM_ERROR("Invalid DRM bitblt alignment.\n"); 847 return -EINVAL; 848 } 849#endif 850 851 if (0 != (ret = via_lock_all_dma_pages(dev, vsg, xfer))) { 852 DRM_ERROR("Could not lock DMA pages.\n"); 853 via_free_sg_info(dev, dev->pdev, vsg); 854 return ret; 855 } 856 857 via_map_blit_for_device(dev->pdev, xfer, vsg, 0); 858 if (0 != (ret = via_alloc_desc_pages(dev, vsg))) { 859 DRM_ERROR("Could not allocate DMA descriptor pages.\n"); 860 via_free_sg_info(dev, dev->pdev, vsg); 861 return ret; 862 } 863 via_map_blit_for_device(dev->pdev, xfer, vsg, 1); 864 865 return 0; 866} 867 868 869/* 870 * Reserve one free slot in the blit queue. Will wait for one second for one 871 * to become available. Otherwise -EBUSY is returned. 872 */ 873 874static int 875via_dmablit_grab_slot(drm_via_blitq_t *blitq, int engine) 876{ 877 int ret = 0; 878 unsigned long irqsave; 879 880 DRM_DEBUG("Num free is %d\n", blitq->num_free); 881 spin_lock_irqsave(&blitq->blit_lock, irqsave); 882 while (blitq->num_free == 0) { 883#ifdef __NetBSD__ 884 DRM_SPIN_TIMED_WAIT_UNTIL(ret, &blitq->busy_queue, 885 &blitq->blit_lock, DRM_HZ, 886 blitq->num_free > 0); 887 if (ret < 0) /* Failure: return negative error as is. */ 888 ; 889 else if (ret == 0) /* Timed out: return -EBUSY like Linux. */ 890 ret = -EBUSY; 891 else /* Success (ret > 0): return 0. */ 892 ret = 0; 893 /* Map -EINTR to -EAGAIN. */ 894 if (ret == -EINTR) 895 ret = -EAGAIN; 896 /* Bail on failure. */ 897 if (ret) { 898 spin_unlock_irqrestore(&blitq->blit_lock, irqsave); 899 return ret; 900 } 901#else 902 spin_unlock_irqrestore(&blitq->blit_lock, irqsave); 903 904 DRM_WAIT_ON(ret, blitq->busy_queue, HZ, blitq->num_free > 0); 905 if (ret) 906 return (-EINTR == ret) ? -EAGAIN : ret; 907 908 spin_lock_irqsave(&blitq->blit_lock, irqsave); 909#endif 910 } 911 912 blitq->num_free--; 913 spin_unlock_irqrestore(&blitq->blit_lock, irqsave); 914 915 return 0; 916} 917 918/* 919 * Hand back a free slot if we changed our mind. 920 */ 921 922static void 923via_dmablit_release_slot(drm_via_blitq_t *blitq) 924{ 925 unsigned long irqsave; 926 927 spin_lock_irqsave(&blitq->blit_lock, irqsave); 928 blitq->num_free++; 929#ifdef __NetBSD__ 930 DRM_SPIN_WAKEUP_ONE(&blitq->busy_queue, &blitq->blit_lock); 931#endif 932 spin_unlock_irqrestore(&blitq->blit_lock, irqsave); 933#ifndef __NetBSD__ 934 wake_up(&blitq->busy_queue); 935#endif 936} 937 938/* 939 * Grab a free slot. Build blit info and queue a blit. 940 */ 941 942 943static int 944via_dmablit(struct drm_device *dev, drm_via_dmablit_t *xfer) 945{ 946 drm_via_private_t *dev_priv = (drm_via_private_t *)dev->dev_private; 947 drm_via_sg_info_t *vsg; 948 drm_via_blitq_t *blitq; 949 int ret; 950 int engine; 951 unsigned long irqsave; 952 953 if (dev_priv == NULL) { 954 DRM_ERROR("Called without initialization.\n"); 955 return -EINVAL; 956 } 957 958 engine = (xfer->to_fb) ? 0 : 1; 959 blitq = dev_priv->blit_queues + engine; 960 if (0 != (ret = via_dmablit_grab_slot(blitq, engine))) 961 return ret; 962 if (NULL == (vsg = kmalloc(sizeof(*vsg), GFP_KERNEL))) { 963 via_dmablit_release_slot(blitq); 964 return -ENOMEM; 965 } 966 if (0 != (ret = via_build_sg_info(dev, vsg, xfer))) { 967 via_dmablit_release_slot(blitq); 968 kfree(vsg); 969 return ret; 970 } 971#ifdef __NetBSD__ 972 /* Prepare to begin a DMA transfer. */ 973 bus_dmamap_sync(dev->dmat, vsg->dmamap, 0, 974 vsg->num_pages << PAGE_SHIFT, 975 (vsg->direction == DMA_FROM_DEVICE 976 ? BUS_DMASYNC_PREREAD 977 : BUS_DMASYNC_PREWRITE)); 978#endif 979 spin_lock_irqsave(&blitq->blit_lock, irqsave); 980 981 blitq->blits[blitq->head++] = vsg; 982 if (blitq->head >= VIA_NUM_BLIT_SLOTS) 983 blitq->head = 0; 984 blitq->num_outstanding++; 985 xfer->sync.sync_handle = ++blitq->cur_blit_handle; 986 987 spin_unlock_irqrestore(&blitq->blit_lock, irqsave); 988 xfer->sync.engine = engine; 989 990 via_dmablit_handler(dev, engine, 0); 991 992 return 0; 993} 994 995/* 996 * Sync on a previously submitted blit. Note that the X server use signals extensively, and 997 * that there is a very big probability that this IOCTL will be interrupted by a signal. In that 998 * case it returns with -EAGAIN for the signal to be delivered. 999 * The caller should then reissue the IOCTL. This is similar to what is being done for drmGetLock(). 1000 */ 1001 1002int 1003via_dma_blit_sync(struct drm_device *dev, void *data, struct drm_file *file_priv) 1004{ 1005 drm_via_blitsync_t *sync = data; 1006 int err; 1007 1008 if (sync->engine >= VIA_NUM_BLIT_ENGINES) 1009 return -EINVAL; 1010 1011 err = via_dmablit_sync(dev, sync->sync_handle, sync->engine); 1012 1013 if (-EINTR == err) 1014 err = -EAGAIN; 1015 1016 return err; 1017} 1018 1019 1020/* 1021 * Queue a blit and hand back a handle to be used for sync. This IOCTL may be interrupted by a signal 1022 * while waiting for a free slot in the blit queue. In that case it returns with -EAGAIN and should 1023 * be reissued. See the above IOCTL code. 1024 */ 1025 1026int 1027via_dma_blit(struct drm_device *dev, void *data, struct drm_file *file_priv) 1028{ 1029 drm_via_dmablit_t *xfer = data; 1030 int err; 1031 1032 err = via_dmablit(dev, xfer); 1033 1034 return err; 1035} 1036