1/* 2 * Intel I/OAT DMA Linux driver 3 * Copyright(c) 2004 - 2009 Intel Corporation. 4 * 5 * This program is free software; you can redistribute it and/or modify it 6 * under the terms and conditions of the GNU General Public License, 7 * version 2, as published by the Free Software Foundation. 8 * 9 * This program is distributed in the hope that it will be useful, but WITHOUT 10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for 12 * more details. 13 * 14 * You should have received a copy of the GNU General Public License along with 15 * this program; if not, write to the Free Software Foundation, Inc., 16 * 51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA. 17 * 18 * The full GNU General Public License is included in this distribution in 19 * the file called "COPYING". 20 * 21 */ 22 23/* 24 * This driver supports an Intel I/OAT DMA engine (versions >= 2), which 25 * does asynchronous data movement and checksumming operations. 26 */ 27 28#include <linux/init.h> 29#include <linux/module.h> 30#include <linux/slab.h> 31#include <linux/pci.h> 32#include <linux/interrupt.h> 33#include <linux/dmaengine.h> 34#include <linux/delay.h> 35#include <linux/dma-mapping.h> 36#include <linux/workqueue.h> 37#include <linux/i7300_idle.h> 38#include "dma.h" 39#include "dma_v2.h" 40#include "registers.h" 41#include "hw.h" 42 43int ioat_ring_alloc_order = 8; 44module_param(ioat_ring_alloc_order, int, 0644); 45MODULE_PARM_DESC(ioat_ring_alloc_order, 46 "ioat2+: allocate 2^n descriptors per channel" 47 " (default: 8 max: 16)"); 48static int ioat_ring_max_alloc_order = IOAT_MAX_ORDER; 49module_param(ioat_ring_max_alloc_order, int, 0644); 50MODULE_PARM_DESC(ioat_ring_max_alloc_order, 51 "ioat2+: upper limit for ring size (default: 16)"); 52 53void __ioat2_issue_pending(struct ioat2_dma_chan *ioat) 54{ 55 struct ioat_chan_common *chan = &ioat->base; 56 57 ioat->dmacount += ioat2_ring_pending(ioat); 58 ioat->issued = ioat->head; 59 writew(ioat->dmacount, chan->reg_base + IOAT_CHAN_DMACOUNT_OFFSET); 60 dev_dbg(to_dev(chan), 61 "%s: head: %#x tail: %#x issued: %#x count: %#x\n", 62 __func__, ioat->head, ioat->tail, ioat->issued, ioat->dmacount); 63} 64 65void ioat2_issue_pending(struct dma_chan *c) 66{ 67 struct ioat2_dma_chan *ioat = to_ioat2_chan(c); 68 69 if (ioat2_ring_pending(ioat)) { 70 spin_lock_bh(&ioat->prep_lock); 71 __ioat2_issue_pending(ioat); 72 spin_unlock_bh(&ioat->prep_lock); 73 } 74} 75 76/** 77 * ioat2_update_pending - log pending descriptors 78 * @ioat: ioat2+ channel 79 * 80 * Check if the number of unsubmitted descriptors has exceeded the 81 * watermark. Called with prep_lock held 82 */ 83static void ioat2_update_pending(struct ioat2_dma_chan *ioat) 84{ 85 if (ioat2_ring_pending(ioat) > ioat_pending_level) 86 __ioat2_issue_pending(ioat); 87} 88 89static void __ioat2_start_null_desc(struct ioat2_dma_chan *ioat) 90{ 91 struct ioat_ring_ent *desc; 92 struct ioat_dma_descriptor *hw; 93 94 if (ioat2_ring_space(ioat) < 1) { 95 dev_err(to_dev(&ioat->base), 96 "Unable to start null desc - ring full\n"); 97 return; 98 } 99 100 dev_dbg(to_dev(&ioat->base), "%s: head: %#x tail: %#x issued: %#x\n", 101 __func__, ioat->head, ioat->tail, ioat->issued); 102 desc = ioat2_get_ring_ent(ioat, ioat->head); 103 104 hw = desc->hw; 105 hw->ctl = 0; 106 hw->ctl_f.null = 1; 107 hw->ctl_f.int_en = 1; 108 hw->ctl_f.compl_write = 1; 109 /* set size to non-zero value (channel returns error when size is 0) */ 110 hw->size = NULL_DESC_BUFFER_SIZE; 111 hw->src_addr = 0; 112 hw->dst_addr = 0; 113 async_tx_ack(&desc->txd); 114 ioat2_set_chainaddr(ioat, desc->txd.phys); 115 dump_desc_dbg(ioat, desc); 116 wmb(); 117 ioat->head += 1; 118 __ioat2_issue_pending(ioat); 119} 120 121static void ioat2_start_null_desc(struct ioat2_dma_chan *ioat) 122{ 123 spin_lock_bh(&ioat->prep_lock); 124 __ioat2_start_null_desc(ioat); 125 spin_unlock_bh(&ioat->prep_lock); 126} 127 128static void __cleanup(struct ioat2_dma_chan *ioat, unsigned long phys_complete) 129{ 130 struct ioat_chan_common *chan = &ioat->base; 131 struct dma_async_tx_descriptor *tx; 132 struct ioat_ring_ent *desc; 133 bool seen_current = false; 134 u16 active; 135 int idx = ioat->tail, i; 136 137 dev_dbg(to_dev(chan), "%s: head: %#x tail: %#x issued: %#x\n", 138 __func__, ioat->head, ioat->tail, ioat->issued); 139 140 active = ioat2_ring_active(ioat); 141 for (i = 0; i < active && !seen_current; i++) { 142 smp_read_barrier_depends(); 143 prefetch(ioat2_get_ring_ent(ioat, idx + i + 1)); 144 desc = ioat2_get_ring_ent(ioat, idx + i); 145 tx = &desc->txd; 146 dump_desc_dbg(ioat, desc); 147 if (tx->cookie) { 148 ioat_dma_unmap(chan, tx->flags, desc->len, desc->hw); 149 chan->completed_cookie = tx->cookie; 150 tx->cookie = 0; 151 if (tx->callback) { 152 tx->callback(tx->callback_param); 153 tx->callback = NULL; 154 } 155 } 156 157 if (tx->phys == phys_complete) 158 seen_current = true; 159 } 160 smp_mb(); /* finish all descriptor reads before incrementing tail */ 161 ioat->tail = idx + i; 162 BUG_ON(active && !seen_current); /* no active descs have written a completion? */ 163 164 chan->last_completion = phys_complete; 165 if (active - i == 0) { 166 dev_dbg(to_dev(chan), "%s: cancel completion timeout\n", 167 __func__); 168 clear_bit(IOAT_COMPLETION_PENDING, &chan->state); 169 mod_timer(&chan->timer, jiffies + IDLE_TIMEOUT); 170 } 171} 172 173/** 174 * ioat2_cleanup - clean finished descriptors (advance tail pointer) 175 * @chan: ioat channel to be cleaned up 176 */ 177static void ioat2_cleanup(struct ioat2_dma_chan *ioat) 178{ 179 struct ioat_chan_common *chan = &ioat->base; 180 unsigned long phys_complete; 181 182 spin_lock_bh(&chan->cleanup_lock); 183 if (ioat_cleanup_preamble(chan, &phys_complete)) 184 __cleanup(ioat, phys_complete); 185 spin_unlock_bh(&chan->cleanup_lock); 186} 187 188void ioat2_cleanup_event(unsigned long data) 189{ 190 struct ioat2_dma_chan *ioat = to_ioat2_chan((void *) data); 191 192 ioat2_cleanup(ioat); 193 writew(IOAT_CHANCTRL_RUN, ioat->base.reg_base + IOAT_CHANCTRL_OFFSET); 194} 195 196void __ioat2_restart_chan(struct ioat2_dma_chan *ioat) 197{ 198 struct ioat_chan_common *chan = &ioat->base; 199 200 /* set the tail to be re-issued */ 201 ioat->issued = ioat->tail; 202 ioat->dmacount = 0; 203 set_bit(IOAT_COMPLETION_PENDING, &chan->state); 204 mod_timer(&chan->timer, jiffies + COMPLETION_TIMEOUT); 205 206 dev_dbg(to_dev(chan), 207 "%s: head: %#x tail: %#x issued: %#x count: %#x\n", 208 __func__, ioat->head, ioat->tail, ioat->issued, ioat->dmacount); 209 210 if (ioat2_ring_pending(ioat)) { 211 struct ioat_ring_ent *desc; 212 213 desc = ioat2_get_ring_ent(ioat, ioat->tail); 214 ioat2_set_chainaddr(ioat, desc->txd.phys); 215 __ioat2_issue_pending(ioat); 216 } else 217 __ioat2_start_null_desc(ioat); 218} 219 220int ioat2_quiesce(struct ioat_chan_common *chan, unsigned long tmo) 221{ 222 unsigned long end = jiffies + tmo; 223 int err = 0; 224 u32 status; 225 226 status = ioat_chansts(chan); 227 if (is_ioat_active(status) || is_ioat_idle(status)) 228 ioat_suspend(chan); 229 while (is_ioat_active(status) || is_ioat_idle(status)) { 230 if (tmo && time_after(jiffies, end)) { 231 err = -ETIMEDOUT; 232 break; 233 } 234 status = ioat_chansts(chan); 235 cpu_relax(); 236 } 237 238 return err; 239} 240 241int ioat2_reset_sync(struct ioat_chan_common *chan, unsigned long tmo) 242{ 243 unsigned long end = jiffies + tmo; 244 int err = 0; 245 246 ioat_reset(chan); 247 while (ioat_reset_pending(chan)) { 248 if (end && time_after(jiffies, end)) { 249 err = -ETIMEDOUT; 250 break; 251 } 252 cpu_relax(); 253 } 254 255 return err; 256} 257 258static void ioat2_restart_channel(struct ioat2_dma_chan *ioat) 259{ 260 struct ioat_chan_common *chan = &ioat->base; 261 unsigned long phys_complete; 262 263 ioat2_quiesce(chan, 0); 264 if (ioat_cleanup_preamble(chan, &phys_complete)) 265 __cleanup(ioat, phys_complete); 266 267 __ioat2_restart_chan(ioat); 268} 269 270void ioat2_timer_event(unsigned long data) 271{ 272 struct ioat2_dma_chan *ioat = to_ioat2_chan((void *) data); 273 struct ioat_chan_common *chan = &ioat->base; 274 275 if (test_bit(IOAT_COMPLETION_PENDING, &chan->state)) { 276 unsigned long phys_complete; 277 u64 status; 278 279 status = ioat_chansts(chan); 280 281 /* when halted due to errors check for channel 282 * programming errors before advancing the completion state 283 */ 284 if (is_ioat_halted(status)) { 285 u32 chanerr; 286 287 chanerr = readl(chan->reg_base + IOAT_CHANERR_OFFSET); 288 dev_err(to_dev(chan), "%s: Channel halted (%x)\n", 289 __func__, chanerr); 290 if (test_bit(IOAT_RUN, &chan->state)) 291 BUG_ON(is_ioat_bug(chanerr)); 292 else /* we never got off the ground */ 293 return; 294 } 295 296 /* if we haven't made progress and we have already 297 * acknowledged a pending completion once, then be more 298 * forceful with a restart 299 */ 300 spin_lock_bh(&chan->cleanup_lock); 301 if (ioat_cleanup_preamble(chan, &phys_complete)) { 302 __cleanup(ioat, phys_complete); 303 } else if (test_bit(IOAT_COMPLETION_ACK, &chan->state)) { 304 spin_lock_bh(&ioat->prep_lock); 305 ioat2_restart_channel(ioat); 306 spin_unlock_bh(&ioat->prep_lock); 307 } else { 308 set_bit(IOAT_COMPLETION_ACK, &chan->state); 309 mod_timer(&chan->timer, jiffies + COMPLETION_TIMEOUT); 310 } 311 spin_unlock_bh(&chan->cleanup_lock); 312 } else { 313 u16 active; 314 315 /* if the ring is idle, empty, and oversized try to step 316 * down the size 317 */ 318 spin_lock_bh(&chan->cleanup_lock); 319 spin_lock_bh(&ioat->prep_lock); 320 active = ioat2_ring_active(ioat); 321 if (active == 0 && ioat->alloc_order > ioat_get_alloc_order()) 322 reshape_ring(ioat, ioat->alloc_order-1); 323 spin_unlock_bh(&ioat->prep_lock); 324 spin_unlock_bh(&chan->cleanup_lock); 325 326 /* keep shrinking until we get back to our minimum 327 * default size 328 */ 329 if (ioat->alloc_order > ioat_get_alloc_order()) 330 mod_timer(&chan->timer, jiffies + IDLE_TIMEOUT); 331 } 332} 333 334static int ioat2_reset_hw(struct ioat_chan_common *chan) 335{ 336 /* throw away whatever the channel was doing and get it initialized */ 337 u32 chanerr; 338 339 ioat2_quiesce(chan, msecs_to_jiffies(100)); 340 341 chanerr = readl(chan->reg_base + IOAT_CHANERR_OFFSET); 342 writel(chanerr, chan->reg_base + IOAT_CHANERR_OFFSET); 343 344 return ioat2_reset_sync(chan, msecs_to_jiffies(200)); 345} 346 347/** 348 * ioat2_enumerate_channels - find and initialize the device's channels 349 * @device: the device to be enumerated 350 */ 351int ioat2_enumerate_channels(struct ioatdma_device *device) 352{ 353 struct ioat2_dma_chan *ioat; 354 struct device *dev = &device->pdev->dev; 355 struct dma_device *dma = &device->common; 356 u8 xfercap_log; 357 int i; 358 359 INIT_LIST_HEAD(&dma->channels); 360 dma->chancnt = readb(device->reg_base + IOAT_CHANCNT_OFFSET); 361 dma->chancnt &= 0x1f; /* bits [4:0] valid */ 362 if (dma->chancnt > ARRAY_SIZE(device->idx)) { 363 dev_warn(dev, "(%d) exceeds max supported channels (%zu)\n", 364 dma->chancnt, ARRAY_SIZE(device->idx)); 365 dma->chancnt = ARRAY_SIZE(device->idx); 366 } 367 xfercap_log = readb(device->reg_base + IOAT_XFERCAP_OFFSET); 368 xfercap_log &= 0x1f; /* bits [4:0] valid */ 369 if (xfercap_log == 0) 370 return 0; 371 dev_dbg(dev, "%s: xfercap = %d\n", __func__, 1 << xfercap_log); 372 373#ifdef CONFIG_I7300_IDLE_IOAT_CHANNEL 374 if (i7300_idle_platform_probe(NULL, NULL, 1) == 0) 375 dma->chancnt--; 376#endif 377 for (i = 0; i < dma->chancnt; i++) { 378 ioat = devm_kzalloc(dev, sizeof(*ioat), GFP_KERNEL); 379 if (!ioat) 380 break; 381 382 ioat_init_channel(device, &ioat->base, i); 383 ioat->xfercap_log = xfercap_log; 384 spin_lock_init(&ioat->prep_lock); 385 if (device->reset_hw(&ioat->base)) { 386 i = 0; 387 break; 388 } 389 } 390 dma->chancnt = i; 391 return i; 392} 393 394static dma_cookie_t ioat2_tx_submit_unlock(struct dma_async_tx_descriptor *tx) 395{ 396 struct dma_chan *c = tx->chan; 397 struct ioat2_dma_chan *ioat = to_ioat2_chan(c); 398 struct ioat_chan_common *chan = &ioat->base; 399 dma_cookie_t cookie = c->cookie; 400 401 cookie++; 402 if (cookie < 0) 403 cookie = 1; 404 tx->cookie = cookie; 405 c->cookie = cookie; 406 dev_dbg(to_dev(&ioat->base), "%s: cookie: %d\n", __func__, cookie); 407 408 if (!test_and_set_bit(IOAT_COMPLETION_PENDING, &chan->state)) 409 mod_timer(&chan->timer, jiffies + COMPLETION_TIMEOUT); 410 411 /* make descriptor updates visible before advancing ioat->head, 412 * this is purposefully not smp_wmb() since we are also 413 * publishing the descriptor updates to a dma device 414 */ 415 wmb(); 416 417 ioat->head += ioat->produce; 418 419 ioat2_update_pending(ioat); 420 spin_unlock_bh(&ioat->prep_lock); 421 422 return cookie; 423} 424 425static struct ioat_ring_ent *ioat2_alloc_ring_ent(struct dma_chan *chan, gfp_t flags) 426{ 427 struct ioat_dma_descriptor *hw; 428 struct ioat_ring_ent *desc; 429 struct ioatdma_device *dma; 430 dma_addr_t phys; 431 432 dma = to_ioatdma_device(chan->device); 433 hw = pci_pool_alloc(dma->dma_pool, flags, &phys); 434 if (!hw) 435 return NULL; 436 memset(hw, 0, sizeof(*hw)); 437 438 desc = kmem_cache_alloc(ioat2_cache, flags); 439 if (!desc) { 440 pci_pool_free(dma->dma_pool, hw, phys); 441 return NULL; 442 } 443 memset(desc, 0, sizeof(*desc)); 444 445 dma_async_tx_descriptor_init(&desc->txd, chan); 446 desc->txd.tx_submit = ioat2_tx_submit_unlock; 447 desc->hw = hw; 448 desc->txd.phys = phys; 449 return desc; 450} 451 452static void ioat2_free_ring_ent(struct ioat_ring_ent *desc, struct dma_chan *chan) 453{ 454 struct ioatdma_device *dma; 455 456 dma = to_ioatdma_device(chan->device); 457 pci_pool_free(dma->dma_pool, desc->hw, desc->txd.phys); 458 kmem_cache_free(ioat2_cache, desc); 459} 460 461static struct ioat_ring_ent **ioat2_alloc_ring(struct dma_chan *c, int order, gfp_t flags) 462{ 463 struct ioat_ring_ent **ring; 464 int descs = 1 << order; 465 int i; 466 467 if (order > ioat_get_max_alloc_order()) 468 return NULL; 469 470 /* allocate the array to hold the software ring */ 471 ring = kcalloc(descs, sizeof(*ring), flags); 472 if (!ring) 473 return NULL; 474 for (i = 0; i < descs; i++) { 475 ring[i] = ioat2_alloc_ring_ent(c, flags); 476 if (!ring[i]) { 477 while (i--) 478 ioat2_free_ring_ent(ring[i], c); 479 kfree(ring); 480 return NULL; 481 } 482 set_desc_id(ring[i], i); 483 } 484 485 /* link descs */ 486 for (i = 0; i < descs-1; i++) { 487 struct ioat_ring_ent *next = ring[i+1]; 488 struct ioat_dma_descriptor *hw = ring[i]->hw; 489 490 hw->next = next->txd.phys; 491 } 492 ring[i]->hw->next = ring[0]->txd.phys; 493 494 return ring; 495} 496 497void ioat2_free_chan_resources(struct dma_chan *c); 498 499/* ioat2_alloc_chan_resources - allocate/initialize ioat2 descriptor ring 500 * @chan: channel to be initialized 501 */ 502int ioat2_alloc_chan_resources(struct dma_chan *c) 503{ 504 struct ioat2_dma_chan *ioat = to_ioat2_chan(c); 505 struct ioat_chan_common *chan = &ioat->base; 506 struct ioat_ring_ent **ring; 507 u64 status; 508 int order; 509 510 /* have we already been set up? */ 511 if (ioat->ring) 512 return 1 << ioat->alloc_order; 513 514 /* Setup register to interrupt and write completion status on error */ 515 writew(IOAT_CHANCTRL_RUN, chan->reg_base + IOAT_CHANCTRL_OFFSET); 516 517 /* allocate a completion writeback area */ 518 /* doing 2 32bit writes to mmio since 1 64b write doesn't work */ 519 chan->completion = pci_pool_alloc(chan->device->completion_pool, 520 GFP_KERNEL, &chan->completion_dma); 521 if (!chan->completion) 522 return -ENOMEM; 523 524 memset(chan->completion, 0, sizeof(*chan->completion)); 525 writel(((u64) chan->completion_dma) & 0x00000000FFFFFFFF, 526 chan->reg_base + IOAT_CHANCMP_OFFSET_LOW); 527 writel(((u64) chan->completion_dma) >> 32, 528 chan->reg_base + IOAT_CHANCMP_OFFSET_HIGH); 529 530 order = ioat_get_alloc_order(); 531 ring = ioat2_alloc_ring(c, order, GFP_KERNEL); 532 if (!ring) 533 return -ENOMEM; 534 535 spin_lock_bh(&chan->cleanup_lock); 536 spin_lock_bh(&ioat->prep_lock); 537 ioat->ring = ring; 538 ioat->head = 0; 539 ioat->issued = 0; 540 ioat->tail = 0; 541 ioat->alloc_order = order; 542 spin_unlock_bh(&ioat->prep_lock); 543 spin_unlock_bh(&chan->cleanup_lock); 544 545 tasklet_enable(&chan->cleanup_task); 546 ioat2_start_null_desc(ioat); 547 548 /* check that we got off the ground */ 549 udelay(5); 550 status = ioat_chansts(chan); 551 if (is_ioat_active(status) || is_ioat_idle(status)) { 552 set_bit(IOAT_RUN, &chan->state); 553 return 1 << ioat->alloc_order; 554 } else { 555 u32 chanerr = readl(chan->reg_base + IOAT_CHANERR_OFFSET); 556 557 dev_WARN(to_dev(chan), 558 "failed to start channel chanerr: %#x\n", chanerr); 559 ioat2_free_chan_resources(c); 560 return -EFAULT; 561 } 562} 563 564bool reshape_ring(struct ioat2_dma_chan *ioat, int order) 565{ 566 /* reshape differs from normal ring allocation in that we want 567 * to allocate a new software ring while only 568 * extending/truncating the hardware ring 569 */ 570 struct ioat_chan_common *chan = &ioat->base; 571 struct dma_chan *c = &chan->common; 572 const u16 curr_size = ioat2_ring_size(ioat); 573 const u16 active = ioat2_ring_active(ioat); 574 const u16 new_size = 1 << order; 575 struct ioat_ring_ent **ring; 576 u16 i; 577 578 if (order > ioat_get_max_alloc_order()) 579 return false; 580 581 /* double check that we have at least 1 free descriptor */ 582 if (active == curr_size) 583 return false; 584 585 /* when shrinking, verify that we can hold the current active 586 * set in the new ring 587 */ 588 if (active >= new_size) 589 return false; 590 591 /* allocate the array to hold the software ring */ 592 ring = kcalloc(new_size, sizeof(*ring), GFP_NOWAIT); 593 if (!ring) 594 return false; 595 596 /* allocate/trim descriptors as needed */ 597 if (new_size > curr_size) { 598 /* copy current descriptors to the new ring */ 599 for (i = 0; i < curr_size; i++) { 600 u16 curr_idx = (ioat->tail+i) & (curr_size-1); 601 u16 new_idx = (ioat->tail+i) & (new_size-1); 602 603 ring[new_idx] = ioat->ring[curr_idx]; 604 set_desc_id(ring[new_idx], new_idx); 605 } 606 607 /* add new descriptors to the ring */ 608 for (i = curr_size; i < new_size; i++) { 609 u16 new_idx = (ioat->tail+i) & (new_size-1); 610 611 ring[new_idx] = ioat2_alloc_ring_ent(c, GFP_NOWAIT); 612 if (!ring[new_idx]) { 613 while (i--) { 614 u16 new_idx = (ioat->tail+i) & (new_size-1); 615 616 ioat2_free_ring_ent(ring[new_idx], c); 617 } 618 kfree(ring); 619 return false; 620 } 621 set_desc_id(ring[new_idx], new_idx); 622 } 623 624 /* hw link new descriptors */ 625 for (i = curr_size-1; i < new_size; i++) { 626 u16 new_idx = (ioat->tail+i) & (new_size-1); 627 struct ioat_ring_ent *next = ring[(new_idx+1) & (new_size-1)]; 628 struct ioat_dma_descriptor *hw = ring[new_idx]->hw; 629 630 hw->next = next->txd.phys; 631 } 632 } else { 633 struct ioat_dma_descriptor *hw; 634 struct ioat_ring_ent *next; 635 636 /* copy current descriptors to the new ring, dropping the 637 * removed descriptors 638 */ 639 for (i = 0; i < new_size; i++) { 640 u16 curr_idx = (ioat->tail+i) & (curr_size-1); 641 u16 new_idx = (ioat->tail+i) & (new_size-1); 642 643 ring[new_idx] = ioat->ring[curr_idx]; 644 set_desc_id(ring[new_idx], new_idx); 645 } 646 647 /* free deleted descriptors */ 648 for (i = new_size; i < curr_size; i++) { 649 struct ioat_ring_ent *ent; 650 651 ent = ioat2_get_ring_ent(ioat, ioat->tail+i); 652 ioat2_free_ring_ent(ent, c); 653 } 654 655 /* fix up hardware ring */ 656 hw = ring[(ioat->tail+new_size-1) & (new_size-1)]->hw; 657 next = ring[(ioat->tail+new_size) & (new_size-1)]; 658 hw->next = next->txd.phys; 659 } 660 661 dev_dbg(to_dev(chan), "%s: allocated %d descriptors\n", 662 __func__, new_size); 663 664 kfree(ioat->ring); 665 ioat->ring = ring; 666 ioat->alloc_order = order; 667 668 return true; 669} 670 671/** 672 * ioat2_check_space_lock - verify space and grab ring producer lock 673 * @ioat: ioat2,3 channel (ring) to operate on 674 * @num_descs: allocation length 675 */ 676int ioat2_check_space_lock(struct ioat2_dma_chan *ioat, int num_descs) 677{ 678 struct ioat_chan_common *chan = &ioat->base; 679 bool retry; 680 681 retry: 682 spin_lock_bh(&ioat->prep_lock); 683 /* never allow the last descriptor to be consumed, we need at 684 * least one free at all times to allow for on-the-fly ring 685 * resizing. 686 */ 687 if (likely(ioat2_ring_space(ioat) > num_descs)) { 688 dev_dbg(to_dev(chan), "%s: num_descs: %d (%x:%x:%x)\n", 689 __func__, num_descs, ioat->head, ioat->tail, ioat->issued); 690 ioat->produce = num_descs; 691 return 0; /* with ioat->prep_lock held */ 692 } 693 retry = test_and_set_bit(IOAT_RESHAPE_PENDING, &chan->state); 694 spin_unlock_bh(&ioat->prep_lock); 695 696 /* is another cpu already trying to expand the ring? */ 697 if (retry) 698 goto retry; 699 700 spin_lock_bh(&chan->cleanup_lock); 701 spin_lock_bh(&ioat->prep_lock); 702 retry = reshape_ring(ioat, ioat->alloc_order + 1); 703 clear_bit(IOAT_RESHAPE_PENDING, &chan->state); 704 spin_unlock_bh(&ioat->prep_lock); 705 spin_unlock_bh(&chan->cleanup_lock); 706 707 /* if we were able to expand the ring retry the allocation */ 708 if (retry) 709 goto retry; 710 711 if (printk_ratelimit()) 712 dev_dbg(to_dev(chan), "%s: ring full! num_descs: %d (%x:%x:%x)\n", 713 __func__, num_descs, ioat->head, ioat->tail, ioat->issued); 714 715 /* progress reclaim in the allocation failure case we may be 716 * called under bh_disabled so we need to trigger the timer 717 * event directly 718 */ 719 if (jiffies > chan->timer.expires && timer_pending(&chan->timer)) { 720 struct ioatdma_device *device = chan->device; 721 722 mod_timer(&chan->timer, jiffies + COMPLETION_TIMEOUT); 723 device->timer_fn((unsigned long) &chan->common); 724 } 725 726 return -ENOMEM; 727} 728 729struct dma_async_tx_descriptor * 730ioat2_dma_prep_memcpy_lock(struct dma_chan *c, dma_addr_t dma_dest, 731 dma_addr_t dma_src, size_t len, unsigned long flags) 732{ 733 struct ioat2_dma_chan *ioat = to_ioat2_chan(c); 734 struct ioat_dma_descriptor *hw; 735 struct ioat_ring_ent *desc; 736 dma_addr_t dst = dma_dest; 737 dma_addr_t src = dma_src; 738 size_t total_len = len; 739 int num_descs, idx, i; 740 741 num_descs = ioat2_xferlen_to_descs(ioat, len); 742 if (likely(num_descs) && ioat2_check_space_lock(ioat, num_descs) == 0) 743 idx = ioat->head; 744 else 745 return NULL; 746 i = 0; 747 do { 748 size_t copy = min_t(size_t, len, 1 << ioat->xfercap_log); 749 750 desc = ioat2_get_ring_ent(ioat, idx + i); 751 hw = desc->hw; 752 753 hw->size = copy; 754 hw->ctl = 0; 755 hw->src_addr = src; 756 hw->dst_addr = dst; 757 758 len -= copy; 759 dst += copy; 760 src += copy; 761 dump_desc_dbg(ioat, desc); 762 } while (++i < num_descs); 763 764 desc->txd.flags = flags; 765 desc->len = total_len; 766 hw->ctl_f.int_en = !!(flags & DMA_PREP_INTERRUPT); 767 hw->ctl_f.fence = !!(flags & DMA_PREP_FENCE); 768 hw->ctl_f.compl_write = 1; 769 dump_desc_dbg(ioat, desc); 770 /* we leave the channel locked to ensure in order submission */ 771 772 return &desc->txd; 773} 774 775/** 776 * ioat2_free_chan_resources - release all the descriptors 777 * @chan: the channel to be cleaned 778 */ 779void ioat2_free_chan_resources(struct dma_chan *c) 780{ 781 struct ioat2_dma_chan *ioat = to_ioat2_chan(c); 782 struct ioat_chan_common *chan = &ioat->base; 783 struct ioatdma_device *device = chan->device; 784 struct ioat_ring_ent *desc; 785 const u16 total_descs = 1 << ioat->alloc_order; 786 int descs; 787 int i; 788 789 /* Before freeing channel resources first check 790 * if they have been previously allocated for this channel. 791 */ 792 if (!ioat->ring) 793 return; 794 795 tasklet_disable(&chan->cleanup_task); 796 del_timer_sync(&chan->timer); 797 device->cleanup_fn((unsigned long) c); 798 device->reset_hw(chan); 799 clear_bit(IOAT_RUN, &chan->state); 800 801 spin_lock_bh(&chan->cleanup_lock); 802 spin_lock_bh(&ioat->prep_lock); 803 descs = ioat2_ring_space(ioat); 804 dev_dbg(to_dev(chan), "freeing %d idle descriptors\n", descs); 805 for (i = 0; i < descs; i++) { 806 desc = ioat2_get_ring_ent(ioat, ioat->head + i); 807 ioat2_free_ring_ent(desc, c); 808 } 809 810 if (descs < total_descs) 811 dev_err(to_dev(chan), "Freeing %d in use descriptors!\n", 812 total_descs - descs); 813 814 for (i = 0; i < total_descs - descs; i++) { 815 desc = ioat2_get_ring_ent(ioat, ioat->tail + i); 816 dump_desc_dbg(ioat, desc); 817 ioat2_free_ring_ent(desc, c); 818 } 819 820 kfree(ioat->ring); 821 ioat->ring = NULL; 822 ioat->alloc_order = 0; 823 pci_pool_free(device->completion_pool, chan->completion, 824 chan->completion_dma); 825 spin_unlock_bh(&ioat->prep_lock); 826 spin_unlock_bh(&chan->cleanup_lock); 827 828 chan->last_completion = 0; 829 chan->completion_dma = 0; 830 ioat->dmacount = 0; 831} 832 833static ssize_t ring_size_show(struct dma_chan *c, char *page) 834{ 835 struct ioat2_dma_chan *ioat = to_ioat2_chan(c); 836 837 return sprintf(page, "%d\n", (1 << ioat->alloc_order) & ~1); 838} 839static struct ioat_sysfs_entry ring_size_attr = __ATTR_RO(ring_size); 840 841static ssize_t ring_active_show(struct dma_chan *c, char *page) 842{ 843 struct ioat2_dma_chan *ioat = to_ioat2_chan(c); 844 845 /* ...taken outside the lock, no need to be precise */ 846 return sprintf(page, "%d\n", ioat2_ring_active(ioat)); 847} 848static struct ioat_sysfs_entry ring_active_attr = __ATTR_RO(ring_active); 849 850static struct attribute *ioat2_attrs[] = { 851 &ring_size_attr.attr, 852 &ring_active_attr.attr, 853 &ioat_cap_attr.attr, 854 &ioat_version_attr.attr, 855 NULL, 856}; 857 858struct kobj_type ioat2_ktype = { 859 .sysfs_ops = &ioat_sysfs_ops, 860 .default_attrs = ioat2_attrs, 861}; 862 863int __devinit ioat2_dma_probe(struct ioatdma_device *device, int dca) 864{ 865 struct pci_dev *pdev = device->pdev; 866 struct dma_device *dma; 867 struct dma_chan *c; 868 struct ioat_chan_common *chan; 869 int err; 870 871 device->enumerate_channels = ioat2_enumerate_channels; 872 device->reset_hw = ioat2_reset_hw; 873 device->cleanup_fn = ioat2_cleanup_event; 874 device->timer_fn = ioat2_timer_event; 875 device->self_test = ioat_dma_self_test; 876 dma = &device->common; 877 dma->device_prep_dma_memcpy = ioat2_dma_prep_memcpy_lock; 878 dma->device_issue_pending = ioat2_issue_pending; 879 dma->device_alloc_chan_resources = ioat2_alloc_chan_resources; 880 dma->device_free_chan_resources = ioat2_free_chan_resources; 881 dma->device_tx_status = ioat_dma_tx_status; 882 883 err = ioat_probe(device); 884 if (err) 885 return err; 886 ioat_set_tcp_copy_break(2048); 887 888 list_for_each_entry(c, &dma->channels, device_node) { 889 chan = to_chan_common(c); 890 writel(IOAT_DCACTRL_CMPL_WRITE_ENABLE | IOAT_DMA_DCA_ANY_CPU, 891 chan->reg_base + IOAT_DCACTRL_OFFSET); 892 } 893 894 err = ioat_register(device); 895 if (err) 896 return err; 897 898 ioat_kobject_add(device, &ioat2_ktype); 899 900 if (dca) 901 device->dca = ioat2_dca_init(pdev, device->reg_base); 902 903 return err; 904} 905