1/* 2 * Copyright(c) 2004 - 2006 Intel Corporation. All rights reserved. 3 * 4 * This program is free software; you can redistribute it and/or modify it 5 * under the terms of the GNU General Public License as published by the Free 6 * Software Foundation; either version 2 of the License, or (at your option) 7 * any later version. 8 * 9 * This program is distributed in the hope that it will be useful, but WITHOUT 10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for 12 * more details. 13 * 14 * You should have received a copy of the GNU General Public License along with 15 * this program; if not, write to the Free Software Foundation, Inc., 59 16 * Temple Place - Suite 330, Boston, MA 02111-1307, USA. 17 * 18 * The full GNU General Public License is included in this distribution in the 19 * file called COPYING. 20 */ 21 22/* 23 * This driver supports an Intel I/OAT DMA engine, which does asynchronous 24 * copy operations. 25 */ 26 27#include <linux/init.h> 28#include <linux/module.h> 29#include <linux/pci.h> 30#include <linux/interrupt.h> 31#include <linux/dmaengine.h> 32#include <linux/delay.h> 33#include <linux/dma-mapping.h> 34#include "ioatdma.h" 35#include "ioatdma_io.h" 36#include "ioatdma_registers.h" 37#include "ioatdma_hw.h" 38 39#define to_ioat_chan(chan) container_of(chan, struct ioat_dma_chan, common) 40#define to_ioat_device(dev) container_of(dev, struct ioat_device, common) 41#define to_ioat_desc(lh) container_of(lh, struct ioat_desc_sw, node) 42 43/* internal functions */ 44static int __devinit ioat_probe(struct pci_dev *pdev, const struct pci_device_id *ent); 45static void __devexit ioat_remove(struct pci_dev *pdev); 46 47static int enumerate_dma_channels(struct ioat_device *device) 48{ 49 u8 xfercap_scale; 50 u32 xfercap; 51 int i; 52 struct ioat_dma_chan *ioat_chan; 53 54 device->common.chancnt = ioatdma_read8(device, IOAT_CHANCNT_OFFSET); 55 xfercap_scale = ioatdma_read8(device, IOAT_XFERCAP_OFFSET); 56 xfercap = (xfercap_scale == 0 ? -1 : (1UL << xfercap_scale)); 57 58 for (i = 0; i < device->common.chancnt; i++) { 59 ioat_chan = kzalloc(sizeof(*ioat_chan), GFP_KERNEL); 60 if (!ioat_chan) { 61 device->common.chancnt = i; 62 break; 63 } 64 65 ioat_chan->device = device; 66 ioat_chan->reg_base = device->reg_base + (0x80 * (i + 1)); 67 ioat_chan->xfercap = xfercap; 68 spin_lock_init(&ioat_chan->cleanup_lock); 69 spin_lock_init(&ioat_chan->desc_lock); 70 INIT_LIST_HEAD(&ioat_chan->free_desc); 71 INIT_LIST_HEAD(&ioat_chan->used_desc); 72 /* This should be made common somewhere in dmaengine.c */ 73 ioat_chan->common.device = &device->common; 74 ioat_chan->common.client = NULL; 75 list_add_tail(&ioat_chan->common.device_node, 76 &device->common.channels); 77 } 78 return device->common.chancnt; 79} 80 81static struct ioat_desc_sw *ioat_dma_alloc_descriptor( 82 struct ioat_dma_chan *ioat_chan, 83 gfp_t flags) 84{ 85 struct ioat_dma_descriptor *desc; 86 struct ioat_desc_sw *desc_sw; 87 struct ioat_device *ioat_device; 88 dma_addr_t phys; 89 90 ioat_device = to_ioat_device(ioat_chan->common.device); 91 desc = pci_pool_alloc(ioat_device->dma_pool, flags, &phys); 92 if (unlikely(!desc)) 93 return NULL; 94 95 desc_sw = kzalloc(sizeof(*desc_sw), flags); 96 if (unlikely(!desc_sw)) { 97 pci_pool_free(ioat_device->dma_pool, desc, phys); 98 return NULL; 99 } 100 101 memset(desc, 0, sizeof(*desc)); 102 desc_sw->hw = desc; 103 desc_sw->phys = phys; 104 105 return desc_sw; 106} 107 108#define INITIAL_IOAT_DESC_COUNT 128 109 110static void ioat_start_null_desc(struct ioat_dma_chan *ioat_chan); 111 112/* returns the actual number of allocated descriptors */ 113static int ioat_dma_alloc_chan_resources(struct dma_chan *chan) 114{ 115 struct ioat_dma_chan *ioat_chan = to_ioat_chan(chan); 116 struct ioat_desc_sw *desc = NULL; 117 u16 chanctrl; 118 u32 chanerr; 119 int i; 120 LIST_HEAD(tmp_list); 121 122 /* 123 * In-use bit automatically set by reading chanctrl 124 * If 0, we got it, if 1, someone else did 125 */ 126 chanctrl = ioatdma_chan_read16(ioat_chan, IOAT_CHANCTRL_OFFSET); 127 if (chanctrl & IOAT_CHANCTRL_CHANNEL_IN_USE) 128 return -EBUSY; 129 130 /* Setup register to interrupt and write completion status on error */ 131 chanctrl = IOAT_CHANCTRL_CHANNEL_IN_USE | 132 IOAT_CHANCTRL_ERR_INT_EN | 133 IOAT_CHANCTRL_ANY_ERR_ABORT_EN | 134 IOAT_CHANCTRL_ERR_COMPLETION_EN; 135 ioatdma_chan_write16(ioat_chan, IOAT_CHANCTRL_OFFSET, chanctrl); 136 137 chanerr = ioatdma_chan_read32(ioat_chan, IOAT_CHANERR_OFFSET); 138 if (chanerr) { 139 printk("IOAT: CHANERR = %x, clearing\n", chanerr); 140 ioatdma_chan_write32(ioat_chan, IOAT_CHANERR_OFFSET, chanerr); 141 } 142 143 /* Allocate descriptors */ 144 for (i = 0; i < INITIAL_IOAT_DESC_COUNT; i++) { 145 desc = ioat_dma_alloc_descriptor(ioat_chan, GFP_KERNEL); 146 if (!desc) { 147 printk(KERN_ERR "IOAT: Only %d initial descriptors\n", i); 148 break; 149 } 150 list_add_tail(&desc->node, &tmp_list); 151 } 152 spin_lock_bh(&ioat_chan->desc_lock); 153 list_splice(&tmp_list, &ioat_chan->free_desc); 154 spin_unlock_bh(&ioat_chan->desc_lock); 155 156 /* allocate a completion writeback area */ 157 /* doing 2 32bit writes to mmio since 1 64b write doesn't work */ 158 ioat_chan->completion_virt = 159 pci_pool_alloc(ioat_chan->device->completion_pool, 160 GFP_KERNEL, 161 &ioat_chan->completion_addr); 162 memset(ioat_chan->completion_virt, 0, 163 sizeof(*ioat_chan->completion_virt)); 164 ioatdma_chan_write32(ioat_chan, IOAT_CHANCMP_OFFSET_LOW, 165 ((u64) ioat_chan->completion_addr) & 0x00000000FFFFFFFF); 166 ioatdma_chan_write32(ioat_chan, IOAT_CHANCMP_OFFSET_HIGH, 167 ((u64) ioat_chan->completion_addr) >> 32); 168 169 ioat_start_null_desc(ioat_chan); 170 return i; 171} 172 173static void ioat_dma_memcpy_cleanup(struct ioat_dma_chan *ioat_chan); 174 175static void ioat_dma_free_chan_resources(struct dma_chan *chan) 176{ 177 struct ioat_dma_chan *ioat_chan = to_ioat_chan(chan); 178 struct ioat_device *ioat_device = to_ioat_device(chan->device); 179 struct ioat_desc_sw *desc, *_desc; 180 u16 chanctrl; 181 int in_use_descs = 0; 182 183 ioat_dma_memcpy_cleanup(ioat_chan); 184 185 ioatdma_chan_write8(ioat_chan, IOAT_CHANCMD_OFFSET, IOAT_CHANCMD_RESET); 186 187 spin_lock_bh(&ioat_chan->desc_lock); 188 list_for_each_entry_safe(desc, _desc, &ioat_chan->used_desc, node) { 189 in_use_descs++; 190 list_del(&desc->node); 191 pci_pool_free(ioat_device->dma_pool, desc->hw, desc->phys); 192 kfree(desc); 193 } 194 list_for_each_entry_safe(desc, _desc, &ioat_chan->free_desc, node) { 195 list_del(&desc->node); 196 pci_pool_free(ioat_device->dma_pool, desc->hw, desc->phys); 197 kfree(desc); 198 } 199 spin_unlock_bh(&ioat_chan->desc_lock); 200 201 pci_pool_free(ioat_device->completion_pool, 202 ioat_chan->completion_virt, 203 ioat_chan->completion_addr); 204 205 /* one is ok since we left it on there on purpose */ 206 if (in_use_descs > 1) 207 printk(KERN_ERR "IOAT: Freeing %d in use descriptors!\n", 208 in_use_descs - 1); 209 210 ioat_chan->last_completion = ioat_chan->completion_addr = 0; 211 212 /* Tell hw the chan is free */ 213 chanctrl = ioatdma_chan_read16(ioat_chan, IOAT_CHANCTRL_OFFSET); 214 chanctrl &= ~IOAT_CHANCTRL_CHANNEL_IN_USE; 215 ioatdma_chan_write16(ioat_chan, IOAT_CHANCTRL_OFFSET, chanctrl); 216} 217 218/** 219 * do_ioat_dma_memcpy - actual function that initiates a IOAT DMA transaction 220 * @ioat_chan: IOAT DMA channel handle 221 * @dest: DMA destination address 222 * @src: DMA source address 223 * @len: transaction length in bytes 224 */ 225 226static dma_cookie_t do_ioat_dma_memcpy(struct ioat_dma_chan *ioat_chan, 227 dma_addr_t dest, 228 dma_addr_t src, 229 size_t len) 230{ 231 struct ioat_desc_sw *first; 232 struct ioat_desc_sw *prev; 233 struct ioat_desc_sw *new; 234 dma_cookie_t cookie; 235 LIST_HEAD(new_chain); 236 u32 copy; 237 size_t orig_len; 238 dma_addr_t orig_src, orig_dst; 239 unsigned int desc_count = 0; 240 unsigned int append = 0; 241 242 if (!ioat_chan || !dest || !src) 243 return -EFAULT; 244 245 if (!len) 246 return ioat_chan->common.cookie; 247 248 orig_len = len; 249 orig_src = src; 250 orig_dst = dest; 251 252 first = NULL; 253 prev = NULL; 254 255 spin_lock_bh(&ioat_chan->desc_lock); 256 257 while (len) { 258 if (!list_empty(&ioat_chan->free_desc)) { 259 new = to_ioat_desc(ioat_chan->free_desc.next); 260 list_del(&new->node); 261 } else { 262 /* try to get another desc */ 263 new = ioat_dma_alloc_descriptor(ioat_chan, GFP_ATOMIC); 264 /* will this ever happen? */ 265 /* TODO add upper limit on these */ 266 BUG_ON(!new); 267 } 268 269 copy = min((u32) len, ioat_chan->xfercap); 270 271 new->hw->size = copy; 272 new->hw->ctl = 0; 273 new->hw->src_addr = src; 274 new->hw->dst_addr = dest; 275 new->cookie = 0; 276 277 /* chain together the physical address list for the HW */ 278 if (!first) 279 first = new; 280 else 281 prev->hw->next = (u64) new->phys; 282 283 prev = new; 284 285 len -= copy; 286 dest += copy; 287 src += copy; 288 289 list_add_tail(&new->node, &new_chain); 290 desc_count++; 291 } 292 new->hw->ctl = IOAT_DMA_DESCRIPTOR_CTL_CP_STS; 293 new->hw->next = 0; 294 295 /* cookie incr and addition to used_list must be atomic */ 296 297 cookie = ioat_chan->common.cookie; 298 cookie++; 299 if (cookie < 0) 300 cookie = 1; 301 ioat_chan->common.cookie = new->cookie = cookie; 302 303 pci_unmap_addr_set(new, src, orig_src); 304 pci_unmap_addr_set(new, dst, orig_dst); 305 pci_unmap_len_set(new, src_len, orig_len); 306 pci_unmap_len_set(new, dst_len, orig_len); 307 308 /* write address into NextDescriptor field of last desc in chain */ 309 to_ioat_desc(ioat_chan->used_desc.prev)->hw->next = first->phys; 310 list_splice_init(&new_chain, ioat_chan->used_desc.prev); 311 312 ioat_chan->pending += desc_count; 313 if (ioat_chan->pending >= 20) { 314 append = 1; 315 ioat_chan->pending = 0; 316 } 317 318 spin_unlock_bh(&ioat_chan->desc_lock); 319 320 if (append) 321 ioatdma_chan_write8(ioat_chan, 322 IOAT_CHANCMD_OFFSET, 323 IOAT_CHANCMD_APPEND); 324 return cookie; 325} 326 327/** 328 * ioat_dma_memcpy_buf_to_buf - wrapper that takes src & dest bufs 329 * @chan: IOAT DMA channel handle 330 * @dest: DMA destination address 331 * @src: DMA source address 332 * @len: transaction length in bytes 333 */ 334 335static dma_cookie_t ioat_dma_memcpy_buf_to_buf(struct dma_chan *chan, 336 void *dest, 337 void *src, 338 size_t len) 339{ 340 dma_addr_t dest_addr; 341 dma_addr_t src_addr; 342 struct ioat_dma_chan *ioat_chan = to_ioat_chan(chan); 343 344 dest_addr = pci_map_single(ioat_chan->device->pdev, 345 dest, len, PCI_DMA_FROMDEVICE); 346 src_addr = pci_map_single(ioat_chan->device->pdev, 347 src, len, PCI_DMA_TODEVICE); 348 349 return do_ioat_dma_memcpy(ioat_chan, dest_addr, src_addr, len); 350} 351 352/** 353 * ioat_dma_memcpy_buf_to_pg - wrapper, copying from a buf to a page 354 * @chan: IOAT DMA channel handle 355 * @page: pointer to the page to copy to 356 * @offset: offset into that page 357 * @src: DMA source address 358 * @len: transaction length in bytes 359 */ 360 361static dma_cookie_t ioat_dma_memcpy_buf_to_pg(struct dma_chan *chan, 362 struct page *page, 363 unsigned int offset, 364 void *src, 365 size_t len) 366{ 367 dma_addr_t dest_addr; 368 dma_addr_t src_addr; 369 struct ioat_dma_chan *ioat_chan = to_ioat_chan(chan); 370 371 dest_addr = pci_map_page(ioat_chan->device->pdev, 372 page, offset, len, PCI_DMA_FROMDEVICE); 373 src_addr = pci_map_single(ioat_chan->device->pdev, 374 src, len, PCI_DMA_TODEVICE); 375 376 return do_ioat_dma_memcpy(ioat_chan, dest_addr, src_addr, len); 377} 378 379/** 380 * ioat_dma_memcpy_pg_to_pg - wrapper, copying between two pages 381 * @chan: IOAT DMA channel handle 382 * @dest_pg: pointer to the page to copy to 383 * @dest_off: offset into that page 384 * @src_pg: pointer to the page to copy from 385 * @src_off: offset into that page 386 * @len: transaction length in bytes. This is guaranteed not to make a copy 387 * across a page boundary. 388 */ 389 390static dma_cookie_t ioat_dma_memcpy_pg_to_pg(struct dma_chan *chan, 391 struct page *dest_pg, 392 unsigned int dest_off, 393 struct page *src_pg, 394 unsigned int src_off, 395 size_t len) 396{ 397 dma_addr_t dest_addr; 398 dma_addr_t src_addr; 399 struct ioat_dma_chan *ioat_chan = to_ioat_chan(chan); 400 401 dest_addr = pci_map_page(ioat_chan->device->pdev, 402 dest_pg, dest_off, len, PCI_DMA_FROMDEVICE); 403 src_addr = pci_map_page(ioat_chan->device->pdev, 404 src_pg, src_off, len, PCI_DMA_TODEVICE); 405 406 return do_ioat_dma_memcpy(ioat_chan, dest_addr, src_addr, len); 407} 408 409/** 410 * ioat_dma_memcpy_issue_pending - push potentially unrecognized appended descriptors to hw 411 * @chan: DMA channel handle 412 */ 413 414static void ioat_dma_memcpy_issue_pending(struct dma_chan *chan) 415{ 416 struct ioat_dma_chan *ioat_chan = to_ioat_chan(chan); 417 418 if (ioat_chan->pending != 0) { 419 ioat_chan->pending = 0; 420 ioatdma_chan_write8(ioat_chan, 421 IOAT_CHANCMD_OFFSET, 422 IOAT_CHANCMD_APPEND); 423 } 424} 425 426static void ioat_dma_memcpy_cleanup(struct ioat_dma_chan *chan) 427{ 428 unsigned long phys_complete; 429 struct ioat_desc_sw *desc, *_desc; 430 dma_cookie_t cookie = 0; 431 432 prefetch(chan->completion_virt); 433 434 if (!spin_trylock(&chan->cleanup_lock)) 435 return; 436 437 /* The completion writeback can happen at any time, 438 so reads by the driver need to be atomic operations 439 The descriptor physical addresses are limited to 32-bits 440 when the CPU can only do a 32-bit mov */ 441 442#if (BITS_PER_LONG == 64) 443 phys_complete = 444 chan->completion_virt->full & IOAT_CHANSTS_COMPLETED_DESCRIPTOR_ADDR; 445#else 446 phys_complete = chan->completion_virt->low & IOAT_LOW_COMPLETION_MASK; 447#endif 448 449 if ((chan->completion_virt->full & IOAT_CHANSTS_DMA_TRANSFER_STATUS) == 450 IOAT_CHANSTS_DMA_TRANSFER_STATUS_HALTED) { 451 printk("IOAT: Channel halted, chanerr = %x\n", 452 ioatdma_chan_read32(chan, IOAT_CHANERR_OFFSET)); 453 454 /* TODO do something to salvage the situation */ 455 } 456 457 if (phys_complete == chan->last_completion) { 458 spin_unlock(&chan->cleanup_lock); 459 return; 460 } 461 462 spin_lock_bh(&chan->desc_lock); 463 list_for_each_entry_safe(desc, _desc, &chan->used_desc, node) { 464 465 /* 466 * Incoming DMA requests may use multiple descriptors, due to 467 * exceeding xfercap, perhaps. If so, only the last one will 468 * have a cookie, and require unmapping. 469 */ 470 if (desc->cookie) { 471 cookie = desc->cookie; 472 473 /* yes we are unmapping both _page and _single alloc'd 474 regions with unmap_page. Is this *really* that bad? 475 */ 476 pci_unmap_page(chan->device->pdev, 477 pci_unmap_addr(desc, dst), 478 pci_unmap_len(desc, dst_len), 479 PCI_DMA_FROMDEVICE); 480 pci_unmap_page(chan->device->pdev, 481 pci_unmap_addr(desc, src), 482 pci_unmap_len(desc, src_len), 483 PCI_DMA_TODEVICE); 484 } 485 486 if (desc->phys != phys_complete) { 487 /* a completed entry, but not the last, so cleanup */ 488 list_del(&desc->node); 489 list_add_tail(&desc->node, &chan->free_desc); 490 } else { 491 /* last used desc. Do not remove, so we can append from 492 it, but don't look at it next time, either */ 493 desc->cookie = 0; 494 495 /* TODO check status bits? */ 496 break; 497 } 498 } 499 500 spin_unlock_bh(&chan->desc_lock); 501 502 chan->last_completion = phys_complete; 503 if (cookie != 0) 504 chan->completed_cookie = cookie; 505 506 spin_unlock(&chan->cleanup_lock); 507} 508 509/** 510 * ioat_dma_is_complete - poll the status of a IOAT DMA transaction 511 * @chan: IOAT DMA channel handle 512 * @cookie: DMA transaction identifier 513 * @done: if not %NULL, updated with last completed transaction 514 * @used: if not %NULL, updated with last used transaction 515 */ 516 517static enum dma_status ioat_dma_is_complete(struct dma_chan *chan, 518 dma_cookie_t cookie, 519 dma_cookie_t *done, 520 dma_cookie_t *used) 521{ 522 struct ioat_dma_chan *ioat_chan = to_ioat_chan(chan); 523 dma_cookie_t last_used; 524 dma_cookie_t last_complete; 525 enum dma_status ret; 526 527 last_used = chan->cookie; 528 last_complete = ioat_chan->completed_cookie; 529 530 if (done) 531 *done= last_complete; 532 if (used) 533 *used = last_used; 534 535 ret = dma_async_is_complete(cookie, last_complete, last_used); 536 if (ret == DMA_SUCCESS) 537 return ret; 538 539 ioat_dma_memcpy_cleanup(ioat_chan); 540 541 last_used = chan->cookie; 542 last_complete = ioat_chan->completed_cookie; 543 544 if (done) 545 *done= last_complete; 546 if (used) 547 *used = last_used; 548 549 return dma_async_is_complete(cookie, last_complete, last_used); 550} 551 552/* PCI API */ 553 554static struct pci_device_id ioat_pci_tbl[] = { 555 { PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_IOAT) }, 556 { 0, } 557}; 558 559static struct pci_driver ioat_pci_driver = { 560 .name = "ioatdma", 561 .id_table = ioat_pci_tbl, 562 .probe = ioat_probe, 563 .remove = __devexit_p(ioat_remove), 564}; 565 566static irqreturn_t ioat_do_interrupt(int irq, void *data) 567{ 568 struct ioat_device *instance = data; 569 unsigned long attnstatus; 570 u8 intrctrl; 571 572 intrctrl = ioatdma_read8(instance, IOAT_INTRCTRL_OFFSET); 573 574 if (!(intrctrl & IOAT_INTRCTRL_MASTER_INT_EN)) 575 return IRQ_NONE; 576 577 if (!(intrctrl & IOAT_INTRCTRL_INT_STATUS)) { 578 ioatdma_write8(instance, IOAT_INTRCTRL_OFFSET, intrctrl); 579 return IRQ_NONE; 580 } 581 582 attnstatus = ioatdma_read32(instance, IOAT_ATTNSTATUS_OFFSET); 583 584 printk(KERN_ERR "ioatdma error: interrupt! status %lx\n", attnstatus); 585 586 ioatdma_write8(instance, IOAT_INTRCTRL_OFFSET, intrctrl); 587 return IRQ_HANDLED; 588} 589 590static void ioat_start_null_desc(struct ioat_dma_chan *ioat_chan) 591{ 592 struct ioat_desc_sw *desc; 593 594 spin_lock_bh(&ioat_chan->desc_lock); 595 596 if (!list_empty(&ioat_chan->free_desc)) { 597 desc = to_ioat_desc(ioat_chan->free_desc.next); 598 list_del(&desc->node); 599 } else { 600 /* try to get another desc */ 601 spin_unlock_bh(&ioat_chan->desc_lock); 602 desc = ioat_dma_alloc_descriptor(ioat_chan, GFP_KERNEL); 603 spin_lock_bh(&ioat_chan->desc_lock); 604 /* will this ever happen? */ 605 BUG_ON(!desc); 606 } 607 608 desc->hw->ctl = IOAT_DMA_DESCRIPTOR_NUL; 609 desc->hw->next = 0; 610 611 list_add_tail(&desc->node, &ioat_chan->used_desc); 612 spin_unlock_bh(&ioat_chan->desc_lock); 613 614#if (BITS_PER_LONG == 64) 615 ioatdma_chan_write64(ioat_chan, IOAT_CHAINADDR_OFFSET, desc->phys); 616#else 617 ioatdma_chan_write32(ioat_chan, 618 IOAT_CHAINADDR_OFFSET_LOW, 619 (u32) desc->phys); 620 ioatdma_chan_write32(ioat_chan, IOAT_CHAINADDR_OFFSET_HIGH, 0); 621#endif 622 ioatdma_chan_write8(ioat_chan, IOAT_CHANCMD_OFFSET, IOAT_CHANCMD_START); 623} 624 625/* 626 * Perform a IOAT transaction to verify the HW works. 627 */ 628#define IOAT_TEST_SIZE 2000 629 630static int ioat_self_test(struct ioat_device *device) 631{ 632 int i; 633 u8 *src; 634 u8 *dest; 635 struct dma_chan *dma_chan; 636 dma_cookie_t cookie; 637 int err = 0; 638 639 src = kzalloc(sizeof(u8) * IOAT_TEST_SIZE, GFP_KERNEL); 640 if (!src) 641 return -ENOMEM; 642 dest = kzalloc(sizeof(u8) * IOAT_TEST_SIZE, GFP_KERNEL); 643 if (!dest) { 644 kfree(src); 645 return -ENOMEM; 646 } 647 648 /* Fill in src buffer */ 649 for (i = 0; i < IOAT_TEST_SIZE; i++) 650 src[i] = (u8)i; 651 652 /* Start copy, using first DMA channel */ 653 dma_chan = container_of(device->common.channels.next, 654 struct dma_chan, 655 device_node); 656 if (ioat_dma_alloc_chan_resources(dma_chan) < 1) { 657 err = -ENODEV; 658 goto out; 659 } 660 661 cookie = ioat_dma_memcpy_buf_to_buf(dma_chan, dest, src, IOAT_TEST_SIZE); 662 ioat_dma_memcpy_issue_pending(dma_chan); 663 msleep(1); 664 665 if (ioat_dma_is_complete(dma_chan, cookie, NULL, NULL) != DMA_SUCCESS) { 666 printk(KERN_ERR "ioatdma: Self-test copy timed out, disabling\n"); 667 err = -ENODEV; 668 goto free_resources; 669 } 670 if (memcmp(src, dest, IOAT_TEST_SIZE)) { 671 printk(KERN_ERR "ioatdma: Self-test copy failed compare, disabling\n"); 672 err = -ENODEV; 673 goto free_resources; 674 } 675 676free_resources: 677 ioat_dma_free_chan_resources(dma_chan); 678out: 679 kfree(src); 680 kfree(dest); 681 return err; 682} 683 684static int __devinit ioat_probe(struct pci_dev *pdev, 685 const struct pci_device_id *ent) 686{ 687 int err; 688 unsigned long mmio_start, mmio_len; 689 void __iomem *reg_base; 690 struct ioat_device *device; 691 692 err = pci_enable_device(pdev); 693 if (err) 694 goto err_enable_device; 695 696 err = pci_set_dma_mask(pdev, DMA_64BIT_MASK); 697 if (err) 698 err = pci_set_dma_mask(pdev, DMA_32BIT_MASK); 699 if (err) 700 goto err_set_dma_mask; 701 702 err = pci_request_regions(pdev, ioat_pci_driver.name); 703 if (err) 704 goto err_request_regions; 705 706 mmio_start = pci_resource_start(pdev, 0); 707 mmio_len = pci_resource_len(pdev, 0); 708 709 reg_base = ioremap(mmio_start, mmio_len); 710 if (!reg_base) { 711 err = -ENOMEM; 712 goto err_ioremap; 713 } 714 715 device = kzalloc(sizeof(*device), GFP_KERNEL); 716 if (!device) { 717 err = -ENOMEM; 718 goto err_kzalloc; 719 } 720 721 /* DMA coherent memory pool for DMA descriptor allocations */ 722 device->dma_pool = pci_pool_create("dma_desc_pool", pdev, 723 sizeof(struct ioat_dma_descriptor), 64, 0); 724 if (!device->dma_pool) { 725 err = -ENOMEM; 726 goto err_dma_pool; 727 } 728 729 device->completion_pool = pci_pool_create("completion_pool", pdev, sizeof(u64), SMP_CACHE_BYTES, SMP_CACHE_BYTES); 730 if (!device->completion_pool) { 731 err = -ENOMEM; 732 goto err_completion_pool; 733 } 734 735 device->pdev = pdev; 736 pci_set_drvdata(pdev, device); 737#ifdef CONFIG_PCI_MSI 738 if (pci_enable_msi(pdev) == 0) { 739 device->msi = 1; 740 } else { 741 device->msi = 0; 742 } 743#endif 744 err = request_irq(pdev->irq, &ioat_do_interrupt, IRQF_SHARED, "ioat", 745 device); 746 if (err) 747 goto err_irq; 748 749 device->reg_base = reg_base; 750 751 ioatdma_write8(device, IOAT_INTRCTRL_OFFSET, IOAT_INTRCTRL_MASTER_INT_EN); 752 pci_set_master(pdev); 753 754 INIT_LIST_HEAD(&device->common.channels); 755 enumerate_dma_channels(device); 756 757 device->common.device_alloc_chan_resources = ioat_dma_alloc_chan_resources; 758 device->common.device_free_chan_resources = ioat_dma_free_chan_resources; 759 device->common.device_memcpy_buf_to_buf = ioat_dma_memcpy_buf_to_buf; 760 device->common.device_memcpy_buf_to_pg = ioat_dma_memcpy_buf_to_pg; 761 device->common.device_memcpy_pg_to_pg = ioat_dma_memcpy_pg_to_pg; 762 device->common.device_memcpy_complete = ioat_dma_is_complete; 763 device->common.device_memcpy_issue_pending = ioat_dma_memcpy_issue_pending; 764 printk(KERN_INFO "Intel(R) I/OAT DMA Engine found, %d channels\n", 765 device->common.chancnt); 766 767 err = ioat_self_test(device); 768 if (err) 769 goto err_self_test; 770 771 dma_async_device_register(&device->common); 772 773 return 0; 774 775err_self_test: 776err_irq: 777 pci_pool_destroy(device->completion_pool); 778err_completion_pool: 779 pci_pool_destroy(device->dma_pool); 780err_dma_pool: 781 kfree(device); 782err_kzalloc: 783 iounmap(reg_base); 784err_ioremap: 785 pci_release_regions(pdev); 786err_request_regions: 787err_set_dma_mask: 788 pci_disable_device(pdev); 789err_enable_device: 790 return err; 791} 792 793static void __devexit ioat_remove(struct pci_dev *pdev) 794{ 795 struct ioat_device *device; 796 struct dma_chan *chan, *_chan; 797 struct ioat_dma_chan *ioat_chan; 798 799 device = pci_get_drvdata(pdev); 800 dma_async_device_unregister(&device->common); 801 802 free_irq(device->pdev->irq, device); 803#ifdef CONFIG_PCI_MSI 804 if (device->msi) 805 pci_disable_msi(device->pdev); 806#endif 807 pci_pool_destroy(device->dma_pool); 808 pci_pool_destroy(device->completion_pool); 809 iounmap(device->reg_base); 810 pci_release_regions(pdev); 811 pci_disable_device(pdev); 812 list_for_each_entry_safe(chan, _chan, &device->common.channels, device_node) { 813 ioat_chan = to_ioat_chan(chan); 814 list_del(&chan->device_node); 815 kfree(ioat_chan); 816 } 817 kfree(device); 818} 819 820/* MODULE API */ 821MODULE_VERSION("1.7"); 822MODULE_LICENSE("GPL"); 823MODULE_AUTHOR("Intel Corporation"); 824 825static int __init ioat_init_module(void) 826{ 827 /* it's currently unsafe to unload this module */ 828 /* if forced, worst case is that rmmod hangs */ 829 __unsafe(THIS_MODULE); 830 831 return pci_register_driver(&ioat_pci_driver); 832} 833 834module_init(ioat_init_module); 835 836static void __exit ioat_exit_module(void) 837{ 838 pci_unregister_driver(&ioat_pci_driver); 839} 840 841module_exit(ioat_exit_module); 842