1// SPDX-License-Identifier: GPL-2.0-only 2/* 3 * offload engine driver for the Marvell XOR engine 4 * Copyright (C) 2007, 2008, Marvell International Ltd. 5 */ 6 7#include <linux/init.h> 8#include <linux/slab.h> 9#include <linux/delay.h> 10#include <linux/dma-mapping.h> 11#include <linux/spinlock.h> 12#include <linux/interrupt.h> 13#include <linux/platform_device.h> 14#include <linux/property.h> 15#include <linux/memory.h> 16#include <linux/clk.h> 17#include <linux/of.h> 18#include <linux/of_irq.h> 19#include <linux/irqdomain.h> 20#include <linux/cpumask.h> 21#include <linux/platform_data/dma-mv_xor.h> 22 23#include "dmaengine.h" 24#include "mv_xor.h" 25 26enum mv_xor_type { 27 XOR_ORION, 28 XOR_ARMADA_38X, 29 XOR_ARMADA_37XX, 30}; 31 32enum mv_xor_mode { 33 XOR_MODE_IN_REG, 34 XOR_MODE_IN_DESC, 35}; 36 37static void mv_xor_issue_pending(struct dma_chan *chan); 38 39#define to_mv_xor_chan(chan) \ 40 container_of(chan, struct mv_xor_chan, dmachan) 41 42#define to_mv_xor_slot(tx) \ 43 container_of(tx, struct mv_xor_desc_slot, async_tx) 44 45#define mv_chan_to_devp(chan) \ 46 ((chan)->dmadev.dev) 47 48static void mv_desc_init(struct mv_xor_desc_slot *desc, 49 dma_addr_t addr, u32 byte_count, 50 enum dma_ctrl_flags flags) 51{ 52 struct mv_xor_desc *hw_desc = desc->hw_desc; 53 54 hw_desc->status = XOR_DESC_DMA_OWNED; 55 hw_desc->phy_next_desc = 0; 56 /* Enable end-of-descriptor interrupts only for DMA_PREP_INTERRUPT */ 57 hw_desc->desc_command = (flags & DMA_PREP_INTERRUPT) ? 58 XOR_DESC_EOD_INT_EN : 0; 59 hw_desc->phy_dest_addr = addr; 60 hw_desc->byte_count = byte_count; 61} 62 63static void mv_desc_set_mode(struct mv_xor_desc_slot *desc) 64{ 65 struct mv_xor_desc *hw_desc = desc->hw_desc; 66 67 switch (desc->type) { 68 case DMA_XOR: 69 case DMA_INTERRUPT: 70 hw_desc->desc_command |= XOR_DESC_OPERATION_XOR; 71 break; 72 case DMA_MEMCPY: 73 hw_desc->desc_command |= XOR_DESC_OPERATION_MEMCPY; 74 break; 75 default: 76 BUG(); 77 return; 78 } 79} 80 81static void mv_desc_set_next_desc(struct mv_xor_desc_slot *desc, 82 u32 next_desc_addr) 83{ 84 struct mv_xor_desc *hw_desc = desc->hw_desc; 85 BUG_ON(hw_desc->phy_next_desc); 86 hw_desc->phy_next_desc = next_desc_addr; 87} 88 89static void mv_desc_set_src_addr(struct mv_xor_desc_slot *desc, 90 int index, dma_addr_t addr) 91{ 92 struct mv_xor_desc *hw_desc = desc->hw_desc; 93 hw_desc->phy_src_addr[mv_phy_src_idx(index)] = addr; 94 if (desc->type == DMA_XOR) 95 hw_desc->desc_command |= (1 << index); 96} 97 98static u32 mv_chan_get_current_desc(struct mv_xor_chan *chan) 99{ 100 return readl_relaxed(XOR_CURR_DESC(chan)); 101} 102 103static void mv_chan_set_next_descriptor(struct mv_xor_chan *chan, 104 u32 next_desc_addr) 105{ 106 writel_relaxed(next_desc_addr, XOR_NEXT_DESC(chan)); 107} 108 109static void mv_chan_unmask_interrupts(struct mv_xor_chan *chan) 110{ 111 u32 val = readl_relaxed(XOR_INTR_MASK(chan)); 112 val |= XOR_INTR_MASK_VALUE << (chan->idx * 16); 113 writel_relaxed(val, XOR_INTR_MASK(chan)); 114} 115 116static u32 mv_chan_get_intr_cause(struct mv_xor_chan *chan) 117{ 118 u32 intr_cause = readl_relaxed(XOR_INTR_CAUSE(chan)); 119 intr_cause = (intr_cause >> (chan->idx * 16)) & 0xFFFF; 120 return intr_cause; 121} 122 123static void mv_chan_clear_eoc_cause(struct mv_xor_chan *chan) 124{ 125 u32 val; 126 127 val = XOR_INT_END_OF_DESC | XOR_INT_END_OF_CHAIN | XOR_INT_STOPPED; 128 val = ~(val << (chan->idx * 16)); 129 dev_dbg(mv_chan_to_devp(chan), "%s, val 0x%08x\n", __func__, val); 130 writel_relaxed(val, XOR_INTR_CAUSE(chan)); 131} 132 133static void mv_chan_clear_err_status(struct mv_xor_chan *chan) 134{ 135 u32 val = 0xFFFF0000 >> (chan->idx * 16); 136 writel_relaxed(val, XOR_INTR_CAUSE(chan)); 137} 138 139static void mv_chan_set_mode(struct mv_xor_chan *chan, 140 u32 op_mode) 141{ 142 u32 config = readl_relaxed(XOR_CONFIG(chan)); 143 144 config &= ~0x7; 145 config |= op_mode; 146 147#if defined(__BIG_ENDIAN) 148 config |= XOR_DESCRIPTOR_SWAP; 149#else 150 config &= ~XOR_DESCRIPTOR_SWAP; 151#endif 152 153 writel_relaxed(config, XOR_CONFIG(chan)); 154} 155 156static void mv_chan_activate(struct mv_xor_chan *chan) 157{ 158 dev_dbg(mv_chan_to_devp(chan), " activate chan.\n"); 159 160 /* writel ensures all descriptors are flushed before activation */ 161 writel(BIT(0), XOR_ACTIVATION(chan)); 162} 163 164static char mv_chan_is_busy(struct mv_xor_chan *chan) 165{ 166 u32 state = readl_relaxed(XOR_ACTIVATION(chan)); 167 168 state = (state >> 4) & 0x3; 169 170 return (state == 1) ? 1 : 0; 171} 172 173/* 174 * mv_chan_start_new_chain - program the engine to operate on new 175 * chain headed by sw_desc 176 * Caller must hold &mv_chan->lock while calling this function 177 */ 178static void mv_chan_start_new_chain(struct mv_xor_chan *mv_chan, 179 struct mv_xor_desc_slot *sw_desc) 180{ 181 dev_dbg(mv_chan_to_devp(mv_chan), "%s %d: sw_desc %p\n", 182 __func__, __LINE__, sw_desc); 183 184 /* set the hardware chain */ 185 mv_chan_set_next_descriptor(mv_chan, sw_desc->async_tx.phys); 186 187 mv_chan->pending++; 188 mv_xor_issue_pending(&mv_chan->dmachan); 189} 190 191static dma_cookie_t 192mv_desc_run_tx_complete_actions(struct mv_xor_desc_slot *desc, 193 struct mv_xor_chan *mv_chan, 194 dma_cookie_t cookie) 195{ 196 BUG_ON(desc->async_tx.cookie < 0); 197 198 if (desc->async_tx.cookie > 0) { 199 cookie = desc->async_tx.cookie; 200 201 dma_descriptor_unmap(&desc->async_tx); 202 /* call the callback (must not sleep or submit new 203 * operations to this channel) 204 */ 205 dmaengine_desc_get_callback_invoke(&desc->async_tx, NULL); 206 } 207 208 /* run dependent operations */ 209 dma_run_dependencies(&desc->async_tx); 210 211 return cookie; 212} 213 214static int 215mv_chan_clean_completed_slots(struct mv_xor_chan *mv_chan) 216{ 217 struct mv_xor_desc_slot *iter, *_iter; 218 219 dev_dbg(mv_chan_to_devp(mv_chan), "%s %d\n", __func__, __LINE__); 220 list_for_each_entry_safe(iter, _iter, &mv_chan->completed_slots, 221 node) { 222 223 if (async_tx_test_ack(&iter->async_tx)) { 224 list_move_tail(&iter->node, &mv_chan->free_slots); 225 if (!list_empty(&iter->sg_tx_list)) { 226 list_splice_tail_init(&iter->sg_tx_list, 227 &mv_chan->free_slots); 228 } 229 } 230 } 231 return 0; 232} 233 234static int 235mv_desc_clean_slot(struct mv_xor_desc_slot *desc, 236 struct mv_xor_chan *mv_chan) 237{ 238 dev_dbg(mv_chan_to_devp(mv_chan), "%s %d: desc %p flags %d\n", 239 __func__, __LINE__, desc, desc->async_tx.flags); 240 241 /* the client is allowed to attach dependent operations 242 * until 'ack' is set 243 */ 244 if (!async_tx_test_ack(&desc->async_tx)) { 245 /* move this slot to the completed_slots */ 246 list_move_tail(&desc->node, &mv_chan->completed_slots); 247 if (!list_empty(&desc->sg_tx_list)) { 248 list_splice_tail_init(&desc->sg_tx_list, 249 &mv_chan->completed_slots); 250 } 251 } else { 252 list_move_tail(&desc->node, &mv_chan->free_slots); 253 if (!list_empty(&desc->sg_tx_list)) { 254 list_splice_tail_init(&desc->sg_tx_list, 255 &mv_chan->free_slots); 256 } 257 } 258 259 return 0; 260} 261 262/* This function must be called with the mv_xor_chan spinlock held */ 263static void mv_chan_slot_cleanup(struct mv_xor_chan *mv_chan) 264{ 265 struct mv_xor_desc_slot *iter, *_iter; 266 dma_cookie_t cookie = 0; 267 int busy = mv_chan_is_busy(mv_chan); 268 u32 current_desc = mv_chan_get_current_desc(mv_chan); 269 int current_cleaned = 0; 270 struct mv_xor_desc *hw_desc; 271 272 dev_dbg(mv_chan_to_devp(mv_chan), "%s %d\n", __func__, __LINE__); 273 dev_dbg(mv_chan_to_devp(mv_chan), "current_desc %x\n", current_desc); 274 mv_chan_clean_completed_slots(mv_chan); 275 276 /* free completed slots from the chain starting with 277 * the oldest descriptor 278 */ 279 280 list_for_each_entry_safe(iter, _iter, &mv_chan->chain, 281 node) { 282 283 /* clean finished descriptors */ 284 hw_desc = iter->hw_desc; 285 if (hw_desc->status & XOR_DESC_SUCCESS) { 286 cookie = mv_desc_run_tx_complete_actions(iter, mv_chan, 287 cookie); 288 289 /* done processing desc, clean slot */ 290 mv_desc_clean_slot(iter, mv_chan); 291 292 /* break if we did cleaned the current */ 293 if (iter->async_tx.phys == current_desc) { 294 current_cleaned = 1; 295 break; 296 } 297 } else { 298 if (iter->async_tx.phys == current_desc) { 299 current_cleaned = 0; 300 break; 301 } 302 } 303 } 304 305 if ((busy == 0) && !list_empty(&mv_chan->chain)) { 306 if (current_cleaned) { 307 /* 308 * current descriptor cleaned and removed, run 309 * from list head 310 */ 311 iter = list_entry(mv_chan->chain.next, 312 struct mv_xor_desc_slot, 313 node); 314 mv_chan_start_new_chain(mv_chan, iter); 315 } else { 316 if (!list_is_last(&iter->node, &mv_chan->chain)) { 317 /* 318 * descriptors are still waiting after 319 * current, trigger them 320 */ 321 iter = list_entry(iter->node.next, 322 struct mv_xor_desc_slot, 323 node); 324 mv_chan_start_new_chain(mv_chan, iter); 325 } else { 326 /* 327 * some descriptors are still waiting 328 * to be cleaned 329 */ 330 tasklet_schedule(&mv_chan->irq_tasklet); 331 } 332 } 333 } 334 335 if (cookie > 0) 336 mv_chan->dmachan.completed_cookie = cookie; 337} 338 339static void mv_xor_tasklet(struct tasklet_struct *t) 340{ 341 struct mv_xor_chan *chan = from_tasklet(chan, t, irq_tasklet); 342 343 spin_lock(&chan->lock); 344 mv_chan_slot_cleanup(chan); 345 spin_unlock(&chan->lock); 346} 347 348static struct mv_xor_desc_slot * 349mv_chan_alloc_slot(struct mv_xor_chan *mv_chan) 350{ 351 struct mv_xor_desc_slot *iter; 352 353 spin_lock_bh(&mv_chan->lock); 354 355 if (!list_empty(&mv_chan->free_slots)) { 356 iter = list_first_entry(&mv_chan->free_slots, 357 struct mv_xor_desc_slot, 358 node); 359 360 list_move_tail(&iter->node, &mv_chan->allocated_slots); 361 362 spin_unlock_bh(&mv_chan->lock); 363 364 /* pre-ack descriptor */ 365 async_tx_ack(&iter->async_tx); 366 iter->async_tx.cookie = -EBUSY; 367 368 return iter; 369 370 } 371 372 spin_unlock_bh(&mv_chan->lock); 373 374 /* try to free some slots if the allocation fails */ 375 tasklet_schedule(&mv_chan->irq_tasklet); 376 377 return NULL; 378} 379 380/************************ DMA engine API functions ****************************/ 381static dma_cookie_t 382mv_xor_tx_submit(struct dma_async_tx_descriptor *tx) 383{ 384 struct mv_xor_desc_slot *sw_desc = to_mv_xor_slot(tx); 385 struct mv_xor_chan *mv_chan = to_mv_xor_chan(tx->chan); 386 struct mv_xor_desc_slot *old_chain_tail; 387 dma_cookie_t cookie; 388 int new_hw_chain = 1; 389 390 dev_dbg(mv_chan_to_devp(mv_chan), 391 "%s sw_desc %p: async_tx %p\n", 392 __func__, sw_desc, &sw_desc->async_tx); 393 394 spin_lock_bh(&mv_chan->lock); 395 cookie = dma_cookie_assign(tx); 396 397 if (list_empty(&mv_chan->chain)) 398 list_move_tail(&sw_desc->node, &mv_chan->chain); 399 else { 400 new_hw_chain = 0; 401 402 old_chain_tail = list_entry(mv_chan->chain.prev, 403 struct mv_xor_desc_slot, 404 node); 405 list_move_tail(&sw_desc->node, &mv_chan->chain); 406 407 dev_dbg(mv_chan_to_devp(mv_chan), "Append to last desc %pa\n", 408 &old_chain_tail->async_tx.phys); 409 410 /* fix up the hardware chain */ 411 mv_desc_set_next_desc(old_chain_tail, sw_desc->async_tx.phys); 412 413 /* if the channel is not busy */ 414 if (!mv_chan_is_busy(mv_chan)) { 415 u32 current_desc = mv_chan_get_current_desc(mv_chan); 416 /* 417 * and the curren desc is the end of the chain before 418 * the append, then we need to start the channel 419 */ 420 if (current_desc == old_chain_tail->async_tx.phys) 421 new_hw_chain = 1; 422 } 423 } 424 425 if (new_hw_chain) 426 mv_chan_start_new_chain(mv_chan, sw_desc); 427 428 spin_unlock_bh(&mv_chan->lock); 429 430 return cookie; 431} 432 433/* returns the number of allocated descriptors */ 434static int mv_xor_alloc_chan_resources(struct dma_chan *chan) 435{ 436 void *virt_desc; 437 dma_addr_t dma_desc; 438 int idx; 439 struct mv_xor_chan *mv_chan = to_mv_xor_chan(chan); 440 struct mv_xor_desc_slot *slot = NULL; 441 int num_descs_in_pool = MV_XOR_POOL_SIZE/MV_XOR_SLOT_SIZE; 442 443 /* Allocate descriptor slots */ 444 idx = mv_chan->slots_allocated; 445 while (idx < num_descs_in_pool) { 446 slot = kzalloc(sizeof(*slot), GFP_KERNEL); 447 if (!slot) { 448 dev_info(mv_chan_to_devp(mv_chan), 449 "channel only initialized %d descriptor slots", 450 idx); 451 break; 452 } 453 virt_desc = mv_chan->dma_desc_pool_virt; 454 slot->hw_desc = virt_desc + idx * MV_XOR_SLOT_SIZE; 455 456 dma_async_tx_descriptor_init(&slot->async_tx, chan); 457 slot->async_tx.tx_submit = mv_xor_tx_submit; 458 INIT_LIST_HEAD(&slot->node); 459 INIT_LIST_HEAD(&slot->sg_tx_list); 460 dma_desc = mv_chan->dma_desc_pool; 461 slot->async_tx.phys = dma_desc + idx * MV_XOR_SLOT_SIZE; 462 slot->idx = idx++; 463 464 spin_lock_bh(&mv_chan->lock); 465 mv_chan->slots_allocated = idx; 466 list_add_tail(&slot->node, &mv_chan->free_slots); 467 spin_unlock_bh(&mv_chan->lock); 468 } 469 470 dev_dbg(mv_chan_to_devp(mv_chan), 471 "allocated %d descriptor slots\n", 472 mv_chan->slots_allocated); 473 474 return mv_chan->slots_allocated ? : -ENOMEM; 475} 476 477/* 478 * Check if source or destination is an PCIe/IO address (non-SDRAM) and add 479 * a new MBus window if necessary. Use a cache for these check so that 480 * the MMIO mapped registers don't have to be accessed for this check 481 * to speed up this process. 482 */ 483static int mv_xor_add_io_win(struct mv_xor_chan *mv_chan, u32 addr) 484{ 485 struct mv_xor_device *xordev = mv_chan->xordev; 486 void __iomem *base = mv_chan->mmr_high_base; 487 u32 win_enable; 488 u32 size; 489 u8 target, attr; 490 int ret; 491 int i; 492 493 /* Nothing needs to get done for the Armada 3700 */ 494 if (xordev->xor_type == XOR_ARMADA_37XX) 495 return 0; 496 497 /* 498 * Loop over the cached windows to check, if the requested area 499 * is already mapped. If this the case, nothing needs to be done 500 * and we can return. 501 */ 502 for (i = 0; i < WINDOW_COUNT; i++) { 503 if (addr >= xordev->win_start[i] && 504 addr <= xordev->win_end[i]) { 505 /* Window is already mapped */ 506 return 0; 507 } 508 } 509 510 /* 511 * The window is not mapped, so we need to create the new mapping 512 */ 513 514 /* If no IO window is found that addr has to be located in SDRAM */ 515 ret = mvebu_mbus_get_io_win_info(addr, &size, &target, &attr); 516 if (ret < 0) 517 return 0; 518 519 /* 520 * Mask the base addr 'addr' according to 'size' read back from the 521 * MBus window. Otherwise we might end up with an address located 522 * somewhere in the middle of this area here. 523 */ 524 size -= 1; 525 addr &= ~size; 526 527 /* 528 * Reading one of both enabled register is enough, as they are always 529 * programmed to the identical values 530 */ 531 win_enable = readl(base + WINDOW_BAR_ENABLE(0)); 532 533 /* Set 'i' to the first free window to write the new values to */ 534 i = ffs(~win_enable) - 1; 535 if (i >= WINDOW_COUNT) 536 return -ENOMEM; 537 538 writel((addr & 0xffff0000) | (attr << 8) | target, 539 base + WINDOW_BASE(i)); 540 writel(size & 0xffff0000, base + WINDOW_SIZE(i)); 541 542 /* Fill the caching variables for later use */ 543 xordev->win_start[i] = addr; 544 xordev->win_end[i] = addr + size; 545 546 win_enable |= (1 << i); 547 win_enable |= 3 << (16 + (2 * i)); 548 writel(win_enable, base + WINDOW_BAR_ENABLE(0)); 549 writel(win_enable, base + WINDOW_BAR_ENABLE(1)); 550 551 return 0; 552} 553 554static struct dma_async_tx_descriptor * 555mv_xor_prep_dma_xor(struct dma_chan *chan, dma_addr_t dest, dma_addr_t *src, 556 unsigned int src_cnt, size_t len, unsigned long flags) 557{ 558 struct mv_xor_chan *mv_chan = to_mv_xor_chan(chan); 559 struct mv_xor_desc_slot *sw_desc; 560 int ret; 561 562 if (unlikely(len < MV_XOR_MIN_BYTE_COUNT)) 563 return NULL; 564 565 BUG_ON(len > MV_XOR_MAX_BYTE_COUNT); 566 567 dev_dbg(mv_chan_to_devp(mv_chan), 568 "%s src_cnt: %d len: %zu dest %pad flags: %ld\n", 569 __func__, src_cnt, len, &dest, flags); 570 571 /* Check if a new window needs to get added for 'dest' */ 572 ret = mv_xor_add_io_win(mv_chan, dest); 573 if (ret) 574 return NULL; 575 576 sw_desc = mv_chan_alloc_slot(mv_chan); 577 if (sw_desc) { 578 sw_desc->type = DMA_XOR; 579 sw_desc->async_tx.flags = flags; 580 mv_desc_init(sw_desc, dest, len, flags); 581 if (mv_chan->op_in_desc == XOR_MODE_IN_DESC) 582 mv_desc_set_mode(sw_desc); 583 while (src_cnt--) { 584 /* Check if a new window needs to get added for 'src' */ 585 ret = mv_xor_add_io_win(mv_chan, src[src_cnt]); 586 if (ret) 587 return NULL; 588 mv_desc_set_src_addr(sw_desc, src_cnt, src[src_cnt]); 589 } 590 } 591 592 dev_dbg(mv_chan_to_devp(mv_chan), 593 "%s sw_desc %p async_tx %p \n", 594 __func__, sw_desc, &sw_desc->async_tx); 595 return sw_desc ? &sw_desc->async_tx : NULL; 596} 597 598static struct dma_async_tx_descriptor * 599mv_xor_prep_dma_memcpy(struct dma_chan *chan, dma_addr_t dest, dma_addr_t src, 600 size_t len, unsigned long flags) 601{ 602 /* 603 * A MEMCPY operation is identical to an XOR operation with only 604 * a single source address. 605 */ 606 return mv_xor_prep_dma_xor(chan, dest, &src, 1, len, flags); 607} 608 609static struct dma_async_tx_descriptor * 610mv_xor_prep_dma_interrupt(struct dma_chan *chan, unsigned long flags) 611{ 612 struct mv_xor_chan *mv_chan = to_mv_xor_chan(chan); 613 dma_addr_t src, dest; 614 size_t len; 615 616 src = mv_chan->dummy_src_addr; 617 dest = mv_chan->dummy_dst_addr; 618 len = MV_XOR_MIN_BYTE_COUNT; 619 620 /* 621 * We implement the DMA_INTERRUPT operation as a minimum sized 622 * XOR operation with a single dummy source address. 623 */ 624 return mv_xor_prep_dma_xor(chan, dest, &src, 1, len, flags); 625} 626 627static void mv_xor_free_chan_resources(struct dma_chan *chan) 628{ 629 struct mv_xor_chan *mv_chan = to_mv_xor_chan(chan); 630 struct mv_xor_desc_slot *iter, *_iter; 631 int in_use_descs = 0; 632 633 spin_lock_bh(&mv_chan->lock); 634 635 mv_chan_slot_cleanup(mv_chan); 636 637 list_for_each_entry_safe(iter, _iter, &mv_chan->chain, 638 node) { 639 in_use_descs++; 640 list_move_tail(&iter->node, &mv_chan->free_slots); 641 } 642 list_for_each_entry_safe(iter, _iter, &mv_chan->completed_slots, 643 node) { 644 in_use_descs++; 645 list_move_tail(&iter->node, &mv_chan->free_slots); 646 } 647 list_for_each_entry_safe(iter, _iter, &mv_chan->allocated_slots, 648 node) { 649 in_use_descs++; 650 list_move_tail(&iter->node, &mv_chan->free_slots); 651 } 652 list_for_each_entry_safe_reverse( 653 iter, _iter, &mv_chan->free_slots, node) { 654 list_del(&iter->node); 655 kfree(iter); 656 mv_chan->slots_allocated--; 657 } 658 659 dev_dbg(mv_chan_to_devp(mv_chan), "%s slots_allocated %d\n", 660 __func__, mv_chan->slots_allocated); 661 spin_unlock_bh(&mv_chan->lock); 662 663 if (in_use_descs) 664 dev_err(mv_chan_to_devp(mv_chan), 665 "freeing %d in use descriptors!\n", in_use_descs); 666} 667 668/** 669 * mv_xor_status - poll the status of an XOR transaction 670 * @chan: XOR channel handle 671 * @cookie: XOR transaction identifier 672 * @txstate: XOR transactions state holder (or NULL) 673 */ 674static enum dma_status mv_xor_status(struct dma_chan *chan, 675 dma_cookie_t cookie, 676 struct dma_tx_state *txstate) 677{ 678 struct mv_xor_chan *mv_chan = to_mv_xor_chan(chan); 679 enum dma_status ret; 680 681 ret = dma_cookie_status(chan, cookie, txstate); 682 if (ret == DMA_COMPLETE) 683 return ret; 684 685 spin_lock_bh(&mv_chan->lock); 686 mv_chan_slot_cleanup(mv_chan); 687 spin_unlock_bh(&mv_chan->lock); 688 689 return dma_cookie_status(chan, cookie, txstate); 690} 691 692static void mv_chan_dump_regs(struct mv_xor_chan *chan) 693{ 694 u32 val; 695 696 val = readl_relaxed(XOR_CONFIG(chan)); 697 dev_err(mv_chan_to_devp(chan), "config 0x%08x\n", val); 698 699 val = readl_relaxed(XOR_ACTIVATION(chan)); 700 dev_err(mv_chan_to_devp(chan), "activation 0x%08x\n", val); 701 702 val = readl_relaxed(XOR_INTR_CAUSE(chan)); 703 dev_err(mv_chan_to_devp(chan), "intr cause 0x%08x\n", val); 704 705 val = readl_relaxed(XOR_INTR_MASK(chan)); 706 dev_err(mv_chan_to_devp(chan), "intr mask 0x%08x\n", val); 707 708 val = readl_relaxed(XOR_ERROR_CAUSE(chan)); 709 dev_err(mv_chan_to_devp(chan), "error cause 0x%08x\n", val); 710 711 val = readl_relaxed(XOR_ERROR_ADDR(chan)); 712 dev_err(mv_chan_to_devp(chan), "error addr 0x%08x\n", val); 713} 714 715static void mv_chan_err_interrupt_handler(struct mv_xor_chan *chan, 716 u32 intr_cause) 717{ 718 if (intr_cause & XOR_INT_ERR_DECODE) { 719 dev_dbg(mv_chan_to_devp(chan), "ignoring address decode error\n"); 720 return; 721 } 722 723 dev_err(mv_chan_to_devp(chan), "error on chan %d. intr cause 0x%08x\n", 724 chan->idx, intr_cause); 725 726 mv_chan_dump_regs(chan); 727 WARN_ON(1); 728} 729 730static irqreturn_t mv_xor_interrupt_handler(int irq, void *data) 731{ 732 struct mv_xor_chan *chan = data; 733 u32 intr_cause = mv_chan_get_intr_cause(chan); 734 735 dev_dbg(mv_chan_to_devp(chan), "intr cause %x\n", intr_cause); 736 737 if (intr_cause & XOR_INTR_ERRORS) 738 mv_chan_err_interrupt_handler(chan, intr_cause); 739 740 tasklet_schedule(&chan->irq_tasklet); 741 742 mv_chan_clear_eoc_cause(chan); 743 744 return IRQ_HANDLED; 745} 746 747static void mv_xor_issue_pending(struct dma_chan *chan) 748{ 749 struct mv_xor_chan *mv_chan = to_mv_xor_chan(chan); 750 751 if (mv_chan->pending >= MV_XOR_THRESHOLD) { 752 mv_chan->pending = 0; 753 mv_chan_activate(mv_chan); 754 } 755} 756 757/* 758 * Perform a transaction to verify the HW works. 759 */ 760 761static int mv_chan_memcpy_self_test(struct mv_xor_chan *mv_chan) 762{ 763 int i, ret; 764 void *src, *dest; 765 dma_addr_t src_dma, dest_dma; 766 struct dma_chan *dma_chan; 767 dma_cookie_t cookie; 768 struct dma_async_tx_descriptor *tx; 769 struct dmaengine_unmap_data *unmap; 770 int err = 0; 771 772 src = kmalloc(PAGE_SIZE, GFP_KERNEL); 773 if (!src) 774 return -ENOMEM; 775 776 dest = kzalloc(PAGE_SIZE, GFP_KERNEL); 777 if (!dest) { 778 kfree(src); 779 return -ENOMEM; 780 } 781 782 /* Fill in src buffer */ 783 for (i = 0; i < PAGE_SIZE; i++) 784 ((u8 *) src)[i] = (u8)i; 785 786 dma_chan = &mv_chan->dmachan; 787 if (mv_xor_alloc_chan_resources(dma_chan) < 1) { 788 err = -ENODEV; 789 goto out; 790 } 791 792 unmap = dmaengine_get_unmap_data(dma_chan->device->dev, 2, GFP_KERNEL); 793 if (!unmap) { 794 err = -ENOMEM; 795 goto free_resources; 796 } 797 798 src_dma = dma_map_page(dma_chan->device->dev, virt_to_page(src), 799 offset_in_page(src), PAGE_SIZE, 800 DMA_TO_DEVICE); 801 unmap->addr[0] = src_dma; 802 803 ret = dma_mapping_error(dma_chan->device->dev, src_dma); 804 if (ret) { 805 err = -ENOMEM; 806 goto free_resources; 807 } 808 unmap->to_cnt = 1; 809 810 dest_dma = dma_map_page(dma_chan->device->dev, virt_to_page(dest), 811 offset_in_page(dest), PAGE_SIZE, 812 DMA_FROM_DEVICE); 813 unmap->addr[1] = dest_dma; 814 815 ret = dma_mapping_error(dma_chan->device->dev, dest_dma); 816 if (ret) { 817 err = -ENOMEM; 818 goto free_resources; 819 } 820 unmap->from_cnt = 1; 821 unmap->len = PAGE_SIZE; 822 823 tx = mv_xor_prep_dma_memcpy(dma_chan, dest_dma, src_dma, 824 PAGE_SIZE, 0); 825 if (!tx) { 826 dev_err(dma_chan->device->dev, 827 "Self-test cannot prepare operation, disabling\n"); 828 err = -ENODEV; 829 goto free_resources; 830 } 831 832 cookie = mv_xor_tx_submit(tx); 833 if (dma_submit_error(cookie)) { 834 dev_err(dma_chan->device->dev, 835 "Self-test submit error, disabling\n"); 836 err = -ENODEV; 837 goto free_resources; 838 } 839 840 mv_xor_issue_pending(dma_chan); 841 async_tx_ack(tx); 842 msleep(1); 843 844 if (mv_xor_status(dma_chan, cookie, NULL) != 845 DMA_COMPLETE) { 846 dev_err(dma_chan->device->dev, 847 "Self-test copy timed out, disabling\n"); 848 err = -ENODEV; 849 goto free_resources; 850 } 851 852 dma_sync_single_for_cpu(dma_chan->device->dev, dest_dma, 853 PAGE_SIZE, DMA_FROM_DEVICE); 854 if (memcmp(src, dest, PAGE_SIZE)) { 855 dev_err(dma_chan->device->dev, 856 "Self-test copy failed compare, disabling\n"); 857 err = -ENODEV; 858 goto free_resources; 859 } 860 861free_resources: 862 dmaengine_unmap_put(unmap); 863 mv_xor_free_chan_resources(dma_chan); 864out: 865 kfree(src); 866 kfree(dest); 867 return err; 868} 869 870#define MV_XOR_NUM_SRC_TEST 4 /* must be <= 15 */ 871static int 872mv_chan_xor_self_test(struct mv_xor_chan *mv_chan) 873{ 874 int i, src_idx, ret; 875 struct page *dest; 876 struct page *xor_srcs[MV_XOR_NUM_SRC_TEST]; 877 dma_addr_t dma_srcs[MV_XOR_NUM_SRC_TEST]; 878 dma_addr_t dest_dma; 879 struct dma_async_tx_descriptor *tx; 880 struct dmaengine_unmap_data *unmap; 881 struct dma_chan *dma_chan; 882 dma_cookie_t cookie; 883 u8 cmp_byte = 0; 884 u32 cmp_word; 885 int err = 0; 886 int src_count = MV_XOR_NUM_SRC_TEST; 887 888 for (src_idx = 0; src_idx < src_count; src_idx++) { 889 xor_srcs[src_idx] = alloc_page(GFP_KERNEL); 890 if (!xor_srcs[src_idx]) { 891 while (src_idx--) 892 __free_page(xor_srcs[src_idx]); 893 return -ENOMEM; 894 } 895 } 896 897 dest = alloc_page(GFP_KERNEL); 898 if (!dest) { 899 while (src_idx--) 900 __free_page(xor_srcs[src_idx]); 901 return -ENOMEM; 902 } 903 904 /* Fill in src buffers */ 905 for (src_idx = 0; src_idx < src_count; src_idx++) { 906 u8 *ptr = page_address(xor_srcs[src_idx]); 907 for (i = 0; i < PAGE_SIZE; i++) 908 ptr[i] = (1 << src_idx); 909 } 910 911 for (src_idx = 0; src_idx < src_count; src_idx++) 912 cmp_byte ^= (u8) (1 << src_idx); 913 914 cmp_word = (cmp_byte << 24) | (cmp_byte << 16) | 915 (cmp_byte << 8) | cmp_byte; 916 917 memset(page_address(dest), 0, PAGE_SIZE); 918 919 dma_chan = &mv_chan->dmachan; 920 if (mv_xor_alloc_chan_resources(dma_chan) < 1) { 921 err = -ENODEV; 922 goto out; 923 } 924 925 unmap = dmaengine_get_unmap_data(dma_chan->device->dev, src_count + 1, 926 GFP_KERNEL); 927 if (!unmap) { 928 err = -ENOMEM; 929 goto free_resources; 930 } 931 932 /* test xor */ 933 for (i = 0; i < src_count; i++) { 934 unmap->addr[i] = dma_map_page(dma_chan->device->dev, xor_srcs[i], 935 0, PAGE_SIZE, DMA_TO_DEVICE); 936 dma_srcs[i] = unmap->addr[i]; 937 ret = dma_mapping_error(dma_chan->device->dev, unmap->addr[i]); 938 if (ret) { 939 err = -ENOMEM; 940 goto free_resources; 941 } 942 unmap->to_cnt++; 943 } 944 945 unmap->addr[src_count] = dma_map_page(dma_chan->device->dev, dest, 0, PAGE_SIZE, 946 DMA_FROM_DEVICE); 947 dest_dma = unmap->addr[src_count]; 948 ret = dma_mapping_error(dma_chan->device->dev, unmap->addr[src_count]); 949 if (ret) { 950 err = -ENOMEM; 951 goto free_resources; 952 } 953 unmap->from_cnt = 1; 954 unmap->len = PAGE_SIZE; 955 956 tx = mv_xor_prep_dma_xor(dma_chan, dest_dma, dma_srcs, 957 src_count, PAGE_SIZE, 0); 958 if (!tx) { 959 dev_err(dma_chan->device->dev, 960 "Self-test cannot prepare operation, disabling\n"); 961 err = -ENODEV; 962 goto free_resources; 963 } 964 965 cookie = mv_xor_tx_submit(tx); 966 if (dma_submit_error(cookie)) { 967 dev_err(dma_chan->device->dev, 968 "Self-test submit error, disabling\n"); 969 err = -ENODEV; 970 goto free_resources; 971 } 972 973 mv_xor_issue_pending(dma_chan); 974 async_tx_ack(tx); 975 msleep(8); 976 977 if (mv_xor_status(dma_chan, cookie, NULL) != 978 DMA_COMPLETE) { 979 dev_err(dma_chan->device->dev, 980 "Self-test xor timed out, disabling\n"); 981 err = -ENODEV; 982 goto free_resources; 983 } 984 985 dma_sync_single_for_cpu(dma_chan->device->dev, dest_dma, 986 PAGE_SIZE, DMA_FROM_DEVICE); 987 for (i = 0; i < (PAGE_SIZE / sizeof(u32)); i++) { 988 u32 *ptr = page_address(dest); 989 if (ptr[i] != cmp_word) { 990 dev_err(dma_chan->device->dev, 991 "Self-test xor failed compare, disabling. index %d, data %x, expected %x\n", 992 i, ptr[i], cmp_word); 993 err = -ENODEV; 994 goto free_resources; 995 } 996 } 997 998free_resources: 999 dmaengine_unmap_put(unmap); 1000 mv_xor_free_chan_resources(dma_chan); 1001out: 1002 src_idx = src_count; 1003 while (src_idx--) 1004 __free_page(xor_srcs[src_idx]); 1005 __free_page(dest); 1006 return err; 1007} 1008 1009static int mv_xor_channel_remove(struct mv_xor_chan *mv_chan) 1010{ 1011 struct dma_chan *chan, *_chan; 1012 struct device *dev = mv_chan->dmadev.dev; 1013 1014 dma_async_device_unregister(&mv_chan->dmadev); 1015 1016 dma_free_coherent(dev, MV_XOR_POOL_SIZE, 1017 mv_chan->dma_desc_pool_virt, mv_chan->dma_desc_pool); 1018 dma_unmap_single(dev, mv_chan->dummy_src_addr, 1019 MV_XOR_MIN_BYTE_COUNT, DMA_FROM_DEVICE); 1020 dma_unmap_single(dev, mv_chan->dummy_dst_addr, 1021 MV_XOR_MIN_BYTE_COUNT, DMA_TO_DEVICE); 1022 1023 list_for_each_entry_safe(chan, _chan, &mv_chan->dmadev.channels, 1024 device_node) { 1025 list_del(&chan->device_node); 1026 } 1027 1028 free_irq(mv_chan->irq, mv_chan); 1029 1030 return 0; 1031} 1032 1033static struct mv_xor_chan * 1034mv_xor_channel_add(struct mv_xor_device *xordev, 1035 struct platform_device *pdev, 1036 int idx, dma_cap_mask_t cap_mask, int irq) 1037{ 1038 int ret = 0; 1039 struct mv_xor_chan *mv_chan; 1040 struct dma_device *dma_dev; 1041 1042 mv_chan = devm_kzalloc(&pdev->dev, sizeof(*mv_chan), GFP_KERNEL); 1043 if (!mv_chan) 1044 return ERR_PTR(-ENOMEM); 1045 1046 mv_chan->idx = idx; 1047 mv_chan->irq = irq; 1048 if (xordev->xor_type == XOR_ORION) 1049 mv_chan->op_in_desc = XOR_MODE_IN_REG; 1050 else 1051 mv_chan->op_in_desc = XOR_MODE_IN_DESC; 1052 1053 dma_dev = &mv_chan->dmadev; 1054 dma_dev->dev = &pdev->dev; 1055 mv_chan->xordev = xordev; 1056 1057 /* 1058 * These source and destination dummy buffers are used to implement 1059 * a DMA_INTERRUPT operation as a minimum-sized XOR operation. 1060 * Hence, we only need to map the buffers at initialization-time. 1061 */ 1062 mv_chan->dummy_src_addr = dma_map_single(dma_dev->dev, 1063 mv_chan->dummy_src, MV_XOR_MIN_BYTE_COUNT, DMA_FROM_DEVICE); 1064 mv_chan->dummy_dst_addr = dma_map_single(dma_dev->dev, 1065 mv_chan->dummy_dst, MV_XOR_MIN_BYTE_COUNT, DMA_TO_DEVICE); 1066 1067 /* allocate coherent memory for hardware descriptors 1068 * note: writecombine gives slightly better performance, but 1069 * requires that we explicitly flush the writes 1070 */ 1071 mv_chan->dma_desc_pool_virt = 1072 dma_alloc_wc(&pdev->dev, MV_XOR_POOL_SIZE, &mv_chan->dma_desc_pool, 1073 GFP_KERNEL); 1074 if (!mv_chan->dma_desc_pool_virt) 1075 return ERR_PTR(-ENOMEM); 1076 1077 /* discover transaction capabilites from the platform data */ 1078 dma_dev->cap_mask = cap_mask; 1079 1080 INIT_LIST_HEAD(&dma_dev->channels); 1081 1082 /* set base routines */ 1083 dma_dev->device_alloc_chan_resources = mv_xor_alloc_chan_resources; 1084 dma_dev->device_free_chan_resources = mv_xor_free_chan_resources; 1085 dma_dev->device_tx_status = mv_xor_status; 1086 dma_dev->device_issue_pending = mv_xor_issue_pending; 1087 1088 /* set prep routines based on capability */ 1089 if (dma_has_cap(DMA_INTERRUPT, dma_dev->cap_mask)) 1090 dma_dev->device_prep_dma_interrupt = mv_xor_prep_dma_interrupt; 1091 if (dma_has_cap(DMA_MEMCPY, dma_dev->cap_mask)) 1092 dma_dev->device_prep_dma_memcpy = mv_xor_prep_dma_memcpy; 1093 if (dma_has_cap(DMA_XOR, dma_dev->cap_mask)) { 1094 dma_dev->max_xor = 8; 1095 dma_dev->device_prep_dma_xor = mv_xor_prep_dma_xor; 1096 } 1097 1098 mv_chan->mmr_base = xordev->xor_base; 1099 mv_chan->mmr_high_base = xordev->xor_high_base; 1100 tasklet_setup(&mv_chan->irq_tasklet, mv_xor_tasklet); 1101 1102 /* clear errors before enabling interrupts */ 1103 mv_chan_clear_err_status(mv_chan); 1104 1105 ret = request_irq(mv_chan->irq, mv_xor_interrupt_handler, 1106 0, dev_name(&pdev->dev), mv_chan); 1107 if (ret) 1108 goto err_free_dma; 1109 1110 mv_chan_unmask_interrupts(mv_chan); 1111 1112 if (mv_chan->op_in_desc == XOR_MODE_IN_DESC) 1113 mv_chan_set_mode(mv_chan, XOR_OPERATION_MODE_IN_DESC); 1114 else 1115 mv_chan_set_mode(mv_chan, XOR_OPERATION_MODE_XOR); 1116 1117 spin_lock_init(&mv_chan->lock); 1118 INIT_LIST_HEAD(&mv_chan->chain); 1119 INIT_LIST_HEAD(&mv_chan->completed_slots); 1120 INIT_LIST_HEAD(&mv_chan->free_slots); 1121 INIT_LIST_HEAD(&mv_chan->allocated_slots); 1122 mv_chan->dmachan.device = dma_dev; 1123 dma_cookie_init(&mv_chan->dmachan); 1124 1125 list_add_tail(&mv_chan->dmachan.device_node, &dma_dev->channels); 1126 1127 if (dma_has_cap(DMA_MEMCPY, dma_dev->cap_mask)) { 1128 ret = mv_chan_memcpy_self_test(mv_chan); 1129 dev_dbg(&pdev->dev, "memcpy self test returned %d\n", ret); 1130 if (ret) 1131 goto err_free_irq; 1132 } 1133 1134 if (dma_has_cap(DMA_XOR, dma_dev->cap_mask)) { 1135 ret = mv_chan_xor_self_test(mv_chan); 1136 dev_dbg(&pdev->dev, "xor self test returned %d\n", ret); 1137 if (ret) 1138 goto err_free_irq; 1139 } 1140 1141 dev_info(&pdev->dev, "Marvell XOR (%s): ( %s%s%s)\n", 1142 mv_chan->op_in_desc ? "Descriptor Mode" : "Registers Mode", 1143 dma_has_cap(DMA_XOR, dma_dev->cap_mask) ? "xor " : "", 1144 dma_has_cap(DMA_MEMCPY, dma_dev->cap_mask) ? "cpy " : "", 1145 dma_has_cap(DMA_INTERRUPT, dma_dev->cap_mask) ? "intr " : ""); 1146 1147 ret = dma_async_device_register(dma_dev); 1148 if (ret) 1149 goto err_free_irq; 1150 1151 return mv_chan; 1152 1153err_free_irq: 1154 free_irq(mv_chan->irq, mv_chan); 1155err_free_dma: 1156 dma_free_coherent(&pdev->dev, MV_XOR_POOL_SIZE, 1157 mv_chan->dma_desc_pool_virt, mv_chan->dma_desc_pool); 1158 return ERR_PTR(ret); 1159} 1160 1161static void 1162mv_xor_conf_mbus_windows(struct mv_xor_device *xordev, 1163 const struct mbus_dram_target_info *dram) 1164{ 1165 void __iomem *base = xordev->xor_high_base; 1166 u32 win_enable = 0; 1167 int i; 1168 1169 for (i = 0; i < 8; i++) { 1170 writel(0, base + WINDOW_BASE(i)); 1171 writel(0, base + WINDOW_SIZE(i)); 1172 if (i < 4) 1173 writel(0, base + WINDOW_REMAP_HIGH(i)); 1174 } 1175 1176 for (i = 0; i < dram->num_cs; i++) { 1177 const struct mbus_dram_window *cs = dram->cs + i; 1178 1179 writel((cs->base & 0xffff0000) | 1180 (cs->mbus_attr << 8) | 1181 dram->mbus_dram_target_id, base + WINDOW_BASE(i)); 1182 writel((cs->size - 1) & 0xffff0000, base + WINDOW_SIZE(i)); 1183 1184 /* Fill the caching variables for later use */ 1185 xordev->win_start[i] = cs->base; 1186 xordev->win_end[i] = cs->base + cs->size - 1; 1187 1188 win_enable |= (1 << i); 1189 win_enable |= 3 << (16 + (2 * i)); 1190 } 1191 1192 writel(win_enable, base + WINDOW_BAR_ENABLE(0)); 1193 writel(win_enable, base + WINDOW_BAR_ENABLE(1)); 1194 writel(0, base + WINDOW_OVERRIDE_CTRL(0)); 1195 writel(0, base + WINDOW_OVERRIDE_CTRL(1)); 1196} 1197 1198static void 1199mv_xor_conf_mbus_windows_a3700(struct mv_xor_device *xordev) 1200{ 1201 void __iomem *base = xordev->xor_high_base; 1202 u32 win_enable = 0; 1203 int i; 1204 1205 for (i = 0; i < 8; i++) { 1206 writel(0, base + WINDOW_BASE(i)); 1207 writel(0, base + WINDOW_SIZE(i)); 1208 if (i < 4) 1209 writel(0, base + WINDOW_REMAP_HIGH(i)); 1210 } 1211 /* 1212 * For Armada3700 open default 4GB Mbus window. The dram 1213 * related configuration are done at AXIS level. 1214 */ 1215 writel(0xffff0000, base + WINDOW_SIZE(0)); 1216 win_enable |= 1; 1217 win_enable |= 3 << 16; 1218 1219 writel(win_enable, base + WINDOW_BAR_ENABLE(0)); 1220 writel(win_enable, base + WINDOW_BAR_ENABLE(1)); 1221 writel(0, base + WINDOW_OVERRIDE_CTRL(0)); 1222 writel(0, base + WINDOW_OVERRIDE_CTRL(1)); 1223} 1224 1225/* 1226 * Since this XOR driver is basically used only for RAID5, we don't 1227 * need to care about synchronizing ->suspend with DMA activity, 1228 * because the DMA engine will naturally be quiet due to the block 1229 * devices being suspended. 1230 */ 1231static int mv_xor_suspend(struct platform_device *pdev, pm_message_t state) 1232{ 1233 struct mv_xor_device *xordev = platform_get_drvdata(pdev); 1234 int i; 1235 1236 for (i = 0; i < MV_XOR_MAX_CHANNELS; i++) { 1237 struct mv_xor_chan *mv_chan = xordev->channels[i]; 1238 1239 if (!mv_chan) 1240 continue; 1241 1242 mv_chan->saved_config_reg = 1243 readl_relaxed(XOR_CONFIG(mv_chan)); 1244 mv_chan->saved_int_mask_reg = 1245 readl_relaxed(XOR_INTR_MASK(mv_chan)); 1246 } 1247 1248 return 0; 1249} 1250 1251static int mv_xor_resume(struct platform_device *dev) 1252{ 1253 struct mv_xor_device *xordev = platform_get_drvdata(dev); 1254 const struct mbus_dram_target_info *dram; 1255 int i; 1256 1257 for (i = 0; i < MV_XOR_MAX_CHANNELS; i++) { 1258 struct mv_xor_chan *mv_chan = xordev->channels[i]; 1259 1260 if (!mv_chan) 1261 continue; 1262 1263 writel_relaxed(mv_chan->saved_config_reg, 1264 XOR_CONFIG(mv_chan)); 1265 writel_relaxed(mv_chan->saved_int_mask_reg, 1266 XOR_INTR_MASK(mv_chan)); 1267 } 1268 1269 if (xordev->xor_type == XOR_ARMADA_37XX) { 1270 mv_xor_conf_mbus_windows_a3700(xordev); 1271 return 0; 1272 } 1273 1274 dram = mv_mbus_dram_info(); 1275 if (dram) 1276 mv_xor_conf_mbus_windows(xordev, dram); 1277 1278 return 0; 1279} 1280 1281static const struct of_device_id mv_xor_dt_ids[] = { 1282 { .compatible = "marvell,orion-xor", .data = (void *)XOR_ORION }, 1283 { .compatible = "marvell,armada-380-xor", .data = (void *)XOR_ARMADA_38X }, 1284 { .compatible = "marvell,armada-3700-xor", .data = (void *)XOR_ARMADA_37XX }, 1285 {}, 1286}; 1287 1288static unsigned int mv_xor_engine_count; 1289 1290static int mv_xor_probe(struct platform_device *pdev) 1291{ 1292 const struct mbus_dram_target_info *dram; 1293 struct mv_xor_device *xordev; 1294 struct mv_xor_platform_data *pdata = dev_get_platdata(&pdev->dev); 1295 struct resource *res; 1296 unsigned int max_engines, max_channels; 1297 int i, ret; 1298 1299 dev_notice(&pdev->dev, "Marvell shared XOR driver\n"); 1300 1301 xordev = devm_kzalloc(&pdev->dev, sizeof(*xordev), GFP_KERNEL); 1302 if (!xordev) 1303 return -ENOMEM; 1304 1305 res = platform_get_resource(pdev, IORESOURCE_MEM, 0); 1306 if (!res) 1307 return -ENODEV; 1308 1309 xordev->xor_base = devm_ioremap(&pdev->dev, res->start, 1310 resource_size(res)); 1311 if (!xordev->xor_base) 1312 return -EBUSY; 1313 1314 res = platform_get_resource(pdev, IORESOURCE_MEM, 1); 1315 if (!res) 1316 return -ENODEV; 1317 1318 xordev->xor_high_base = devm_ioremap(&pdev->dev, res->start, 1319 resource_size(res)); 1320 if (!xordev->xor_high_base) 1321 return -EBUSY; 1322 1323 platform_set_drvdata(pdev, xordev); 1324 1325 1326 /* 1327 * We need to know which type of XOR device we use before 1328 * setting up. In non-dt case it can only be the legacy one. 1329 */ 1330 xordev->xor_type = XOR_ORION; 1331 if (pdev->dev.of_node) 1332 xordev->xor_type = (uintptr_t)device_get_match_data(&pdev->dev); 1333 1334 /* 1335 * (Re-)program MBUS remapping windows if we are asked to. 1336 */ 1337 if (xordev->xor_type == XOR_ARMADA_37XX) { 1338 mv_xor_conf_mbus_windows_a3700(xordev); 1339 } else { 1340 dram = mv_mbus_dram_info(); 1341 if (dram) 1342 mv_xor_conf_mbus_windows(xordev, dram); 1343 } 1344 1345 /* Not all platforms can gate the clock, so it is not 1346 * an error if the clock does not exists. 1347 */ 1348 xordev->clk = clk_get(&pdev->dev, NULL); 1349 if (!IS_ERR(xordev->clk)) 1350 clk_prepare_enable(xordev->clk); 1351 1352 /* 1353 * We don't want to have more than one channel per CPU in 1354 * order for async_tx to perform well. So we limit the number 1355 * of engines and channels so that we take into account this 1356 * constraint. Note that we also want to use channels from 1357 * separate engines when possible. For dual-CPU Armada 3700 1358 * SoC with single XOR engine allow using its both channels. 1359 */ 1360 max_engines = num_present_cpus(); 1361 if (xordev->xor_type == XOR_ARMADA_37XX) 1362 max_channels = num_present_cpus(); 1363 else 1364 max_channels = min_t(unsigned int, 1365 MV_XOR_MAX_CHANNELS, 1366 DIV_ROUND_UP(num_present_cpus(), 2)); 1367 1368 if (mv_xor_engine_count >= max_engines) 1369 return 0; 1370 1371 if (pdev->dev.of_node) { 1372 struct device_node *np; 1373 int i = 0; 1374 1375 for_each_child_of_node(pdev->dev.of_node, np) { 1376 struct mv_xor_chan *chan; 1377 dma_cap_mask_t cap_mask; 1378 int irq; 1379 1380 if (i >= max_channels) 1381 continue; 1382 1383 dma_cap_zero(cap_mask); 1384 dma_cap_set(DMA_MEMCPY, cap_mask); 1385 dma_cap_set(DMA_XOR, cap_mask); 1386 dma_cap_set(DMA_INTERRUPT, cap_mask); 1387 1388 irq = irq_of_parse_and_map(np, 0); 1389 if (!irq) { 1390 ret = -ENODEV; 1391 goto err_channel_add; 1392 } 1393 1394 chan = mv_xor_channel_add(xordev, pdev, i, 1395 cap_mask, irq); 1396 if (IS_ERR(chan)) { 1397 ret = PTR_ERR(chan); 1398 irq_dispose_mapping(irq); 1399 goto err_channel_add; 1400 } 1401 1402 xordev->channels[i] = chan; 1403 i++; 1404 } 1405 } else if (pdata && pdata->channels) { 1406 for (i = 0; i < max_channels; i++) { 1407 struct mv_xor_channel_data *cd; 1408 struct mv_xor_chan *chan; 1409 int irq; 1410 1411 cd = &pdata->channels[i]; 1412 irq = platform_get_irq(pdev, i); 1413 if (irq < 0) { 1414 ret = irq; 1415 goto err_channel_add; 1416 } 1417 1418 chan = mv_xor_channel_add(xordev, pdev, i, 1419 cd->cap_mask, irq); 1420 if (IS_ERR(chan)) { 1421 ret = PTR_ERR(chan); 1422 goto err_channel_add; 1423 } 1424 1425 xordev->channels[i] = chan; 1426 } 1427 } 1428 1429 return 0; 1430 1431err_channel_add: 1432 for (i = 0; i < MV_XOR_MAX_CHANNELS; i++) 1433 if (xordev->channels[i]) { 1434 mv_xor_channel_remove(xordev->channels[i]); 1435 if (pdev->dev.of_node) 1436 irq_dispose_mapping(xordev->channels[i]->irq); 1437 } 1438 1439 if (!IS_ERR(xordev->clk)) { 1440 clk_disable_unprepare(xordev->clk); 1441 clk_put(xordev->clk); 1442 } 1443 1444 return ret; 1445} 1446 1447static struct platform_driver mv_xor_driver = { 1448 .probe = mv_xor_probe, 1449 .suspend = mv_xor_suspend, 1450 .resume = mv_xor_resume, 1451 .driver = { 1452 .name = MV_XOR_NAME, 1453 .of_match_table = mv_xor_dt_ids, 1454 }, 1455}; 1456 1457builtin_platform_driver(mv_xor_driver); 1458 1459/* 1460MODULE_AUTHOR("Saeed Bishara <saeed@marvell.com>"); 1461MODULE_DESCRIPTION("DMA engine driver for Marvell's XOR engine"); 1462MODULE_LICENSE("GPL"); 1463*/ 1464