1/* 2 * drivers/dma/fsl_raid.c 3 * 4 * Freescale RAID Engine device driver 5 * 6 * Author: 7 * Harninder Rai <harninder.rai@freescale.com> 8 * Naveen Burmi <naveenburmi@freescale.com> 9 * 10 * Rewrite: 11 * Xuelin Shi <xuelin.shi@freescale.com> 12 * 13 * Copyright (c) 2010-2014 Freescale Semiconductor, Inc. 14 * 15 * Redistribution and use in source and binary forms, with or without 16 * modification, are permitted provided that the following conditions are met: 17 * * Redistributions of source code must retain the above copyright 18 * notice, this list of conditions and the following disclaimer. 19 * * Redistributions in binary form must reproduce the above copyright 20 * notice, this list of conditions and the following disclaimer in the 21 * documentation and/or other materials provided with the distribution. 22 * * Neither the name of Freescale Semiconductor nor the 23 * names of its contributors may be used to endorse or promote products 24 * derived from this software without specific prior written permission. 25 * 26 * ALTERNATIVELY, this software may be distributed under the terms of the 27 * GNU General Public License ("GPL") as published by the Free Software 28 * Foundation, either version 2 of that License or (at your option) any 29 * later version. 30 * 31 * THIS SOFTWARE IS PROVIDED BY Freescale Semiconductor ``AS IS'' AND ANY 32 * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 33 * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 34 * DISCLAIMED. IN NO EVENT SHALL Freescale Semiconductor BE LIABLE FOR ANY 35 * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES 36 * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; 37 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND 38 * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 39 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS 40 * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 41 * 42 * Theory of operation: 43 * 44 * General capabilities: 45 * RAID Engine (RE) block is capable of offloading XOR, memcpy and P/Q 46 * calculations required in RAID5 and RAID6 operations. RE driver 47 * registers with Linux's ASYNC layer as dma driver. RE hardware 48 * maintains strict ordering of the requests through chained 49 * command queueing. 50 * 51 * Data flow: 52 * Software RAID layer of Linux (MD layer) maintains RAID partitions, 53 * strips, stripes etc. It sends requests to the underlying ASYNC layer 54 * which further passes it to RE driver. ASYNC layer decides which request 55 * goes to which job ring of RE hardware. For every request processed by 56 * RAID Engine, driver gets an interrupt unless coalescing is set. The 57 * per job ring interrupt handler checks the status register for errors, 58 * clears the interrupt and leave the post interrupt processing to the irq 59 * thread. 60 */ 61#include <linux/interrupt.h> 62#include <linux/module.h> 63#include <linux/of.h> 64#include <linux/of_irq.h> 65#include <linux/of_platform.h> 66#include <linux/platform_device.h> 67#include <linux/dma-mapping.h> 68#include <linux/dmapool.h> 69#include <linux/dmaengine.h> 70#include <linux/io.h> 71#include <linux/spinlock.h> 72#include <linux/slab.h> 73 74#include "dmaengine.h" 75#include "fsl_raid.h" 76 77#define FSL_RE_MAX_XOR_SRCS 16 78#define FSL_RE_MAX_PQ_SRCS 16 79#define FSL_RE_MIN_DESCS 256 80#define FSL_RE_MAX_DESCS (4 * FSL_RE_MIN_DESCS) 81#define FSL_RE_FRAME_FORMAT 0x1 82#define FSL_RE_MAX_DATA_LEN (1024*1024) 83 84#define to_fsl_re_dma_desc(tx) container_of(tx, struct fsl_re_desc, async_tx) 85 86/* Add descriptors into per chan software queue - submit_q */ 87static dma_cookie_t fsl_re_tx_submit(struct dma_async_tx_descriptor *tx) 88{ 89 struct fsl_re_desc *desc; 90 struct fsl_re_chan *re_chan; 91 dma_cookie_t cookie; 92 unsigned long flags; 93 94 desc = to_fsl_re_dma_desc(tx); 95 re_chan = container_of(tx->chan, struct fsl_re_chan, chan); 96 97 spin_lock_irqsave(&re_chan->desc_lock, flags); 98 cookie = dma_cookie_assign(tx); 99 list_add_tail(&desc->node, &re_chan->submit_q); 100 spin_unlock_irqrestore(&re_chan->desc_lock, flags); 101 102 return cookie; 103} 104 105/* Copy descriptor from per chan software queue into hardware job ring */ 106static void fsl_re_issue_pending(struct dma_chan *chan) 107{ 108 struct fsl_re_chan *re_chan; 109 int avail; 110 struct fsl_re_desc *desc, *_desc; 111 unsigned long flags; 112 113 re_chan = container_of(chan, struct fsl_re_chan, chan); 114 115 spin_lock_irqsave(&re_chan->desc_lock, flags); 116 avail = FSL_RE_SLOT_AVAIL( 117 in_be32(&re_chan->jrregs->inbring_slot_avail)); 118 119 list_for_each_entry_safe(desc, _desc, &re_chan->submit_q, node) { 120 if (!avail) 121 break; 122 123 list_move_tail(&desc->node, &re_chan->active_q); 124 125 memcpy(&re_chan->inb_ring_virt_addr[re_chan->inb_count], 126 &desc->hwdesc, sizeof(struct fsl_re_hw_desc)); 127 128 re_chan->inb_count = (re_chan->inb_count + 1) & 129 FSL_RE_RING_SIZE_MASK; 130 out_be32(&re_chan->jrregs->inbring_add_job, FSL_RE_ADD_JOB(1)); 131 avail--; 132 } 133 spin_unlock_irqrestore(&re_chan->desc_lock, flags); 134} 135 136static void fsl_re_desc_done(struct fsl_re_desc *desc) 137{ 138 dma_cookie_complete(&desc->async_tx); 139 dma_descriptor_unmap(&desc->async_tx); 140 dmaengine_desc_get_callback_invoke(&desc->async_tx, NULL); 141} 142 143static void fsl_re_cleanup_descs(struct fsl_re_chan *re_chan) 144{ 145 struct fsl_re_desc *desc, *_desc; 146 unsigned long flags; 147 148 spin_lock_irqsave(&re_chan->desc_lock, flags); 149 list_for_each_entry_safe(desc, _desc, &re_chan->ack_q, node) { 150 if (async_tx_test_ack(&desc->async_tx)) 151 list_move_tail(&desc->node, &re_chan->free_q); 152 } 153 spin_unlock_irqrestore(&re_chan->desc_lock, flags); 154 155 fsl_re_issue_pending(&re_chan->chan); 156} 157 158static void fsl_re_dequeue(struct tasklet_struct *t) 159{ 160 struct fsl_re_chan *re_chan = from_tasklet(re_chan, t, irqtask); 161 struct fsl_re_desc *desc, *_desc; 162 struct fsl_re_hw_desc *hwdesc; 163 unsigned long flags; 164 unsigned int count, oub_count; 165 int found; 166 167 fsl_re_cleanup_descs(re_chan); 168 169 spin_lock_irqsave(&re_chan->desc_lock, flags); 170 count = FSL_RE_SLOT_FULL(in_be32(&re_chan->jrregs->oubring_slot_full)); 171 while (count--) { 172 found = 0; 173 hwdesc = &re_chan->oub_ring_virt_addr[re_chan->oub_count]; 174 list_for_each_entry_safe(desc, _desc, &re_chan->active_q, 175 node) { 176 /* compare the hw dma addr to find the completed */ 177 if (desc->hwdesc.lbea32 == hwdesc->lbea32 && 178 desc->hwdesc.addr_low == hwdesc->addr_low) { 179 found = 1; 180 break; 181 } 182 } 183 184 if (found) { 185 fsl_re_desc_done(desc); 186 list_move_tail(&desc->node, &re_chan->ack_q); 187 } else { 188 dev_err(re_chan->dev, 189 "found hwdesc not in sw queue, discard it\n"); 190 } 191 192 oub_count = (re_chan->oub_count + 1) & FSL_RE_RING_SIZE_MASK; 193 re_chan->oub_count = oub_count; 194 195 out_be32(&re_chan->jrregs->oubring_job_rmvd, 196 FSL_RE_RMVD_JOB(1)); 197 } 198 spin_unlock_irqrestore(&re_chan->desc_lock, flags); 199} 200 201/* Per Job Ring interrupt handler */ 202static irqreturn_t fsl_re_isr(int irq, void *data) 203{ 204 struct fsl_re_chan *re_chan; 205 u32 irqstate, status; 206 207 re_chan = dev_get_drvdata((struct device *)data); 208 209 irqstate = in_be32(&re_chan->jrregs->jr_interrupt_status); 210 if (!irqstate) 211 return IRQ_NONE; 212 213 /* 214 * There's no way in upper layer (read MD layer) to recover from 215 * error conditions except restart everything. In long term we 216 * need to do something more than just crashing 217 */ 218 if (irqstate & FSL_RE_ERROR) { 219 status = in_be32(&re_chan->jrregs->jr_status); 220 dev_err(re_chan->dev, "chan error irqstate: %x, status: %x\n", 221 irqstate, status); 222 } 223 224 /* Clear interrupt */ 225 out_be32(&re_chan->jrregs->jr_interrupt_status, FSL_RE_CLR_INTR); 226 227 tasklet_schedule(&re_chan->irqtask); 228 229 return IRQ_HANDLED; 230} 231 232static enum dma_status fsl_re_tx_status(struct dma_chan *chan, 233 dma_cookie_t cookie, 234 struct dma_tx_state *txstate) 235{ 236 return dma_cookie_status(chan, cookie, txstate); 237} 238 239static void fill_cfd_frame(struct fsl_re_cmpnd_frame *cf, u8 index, 240 size_t length, dma_addr_t addr, bool final) 241{ 242 u32 efrl = length & FSL_RE_CF_LENGTH_MASK; 243 244 efrl |= final << FSL_RE_CF_FINAL_SHIFT; 245 cf[index].efrl32 = efrl; 246 cf[index].addr_high = upper_32_bits(addr); 247 cf[index].addr_low = lower_32_bits(addr); 248} 249 250static struct fsl_re_desc *fsl_re_init_desc(struct fsl_re_chan *re_chan, 251 struct fsl_re_desc *desc, 252 void *cf, dma_addr_t paddr) 253{ 254 desc->re_chan = re_chan; 255 desc->async_tx.tx_submit = fsl_re_tx_submit; 256 dma_async_tx_descriptor_init(&desc->async_tx, &re_chan->chan); 257 INIT_LIST_HEAD(&desc->node); 258 259 desc->hwdesc.fmt32 = FSL_RE_FRAME_FORMAT << FSL_RE_HWDESC_FMT_SHIFT; 260 desc->hwdesc.lbea32 = upper_32_bits(paddr); 261 desc->hwdesc.addr_low = lower_32_bits(paddr); 262 desc->cf_addr = cf; 263 desc->cf_paddr = paddr; 264 265 desc->cdb_addr = (void *)(cf + FSL_RE_CF_DESC_SIZE); 266 desc->cdb_paddr = paddr + FSL_RE_CF_DESC_SIZE; 267 268 return desc; 269} 270 271static struct fsl_re_desc *fsl_re_chan_alloc_desc(struct fsl_re_chan *re_chan, 272 unsigned long flags) 273{ 274 struct fsl_re_desc *desc = NULL; 275 void *cf; 276 dma_addr_t paddr; 277 unsigned long lock_flag; 278 279 fsl_re_cleanup_descs(re_chan); 280 281 spin_lock_irqsave(&re_chan->desc_lock, lock_flag); 282 if (!list_empty(&re_chan->free_q)) { 283 /* take one desc from free_q */ 284 desc = list_first_entry(&re_chan->free_q, 285 struct fsl_re_desc, node); 286 list_del(&desc->node); 287 288 desc->async_tx.flags = flags; 289 } 290 spin_unlock_irqrestore(&re_chan->desc_lock, lock_flag); 291 292 if (!desc) { 293 desc = kzalloc(sizeof(*desc), GFP_NOWAIT); 294 if (!desc) 295 return NULL; 296 297 cf = dma_pool_alloc(re_chan->re_dev->cf_desc_pool, GFP_NOWAIT, 298 &paddr); 299 if (!cf) { 300 kfree(desc); 301 return NULL; 302 } 303 304 desc = fsl_re_init_desc(re_chan, desc, cf, paddr); 305 desc->async_tx.flags = flags; 306 307 spin_lock_irqsave(&re_chan->desc_lock, lock_flag); 308 re_chan->alloc_count++; 309 spin_unlock_irqrestore(&re_chan->desc_lock, lock_flag); 310 } 311 312 return desc; 313} 314 315static struct dma_async_tx_descriptor *fsl_re_prep_dma_genq( 316 struct dma_chan *chan, dma_addr_t dest, dma_addr_t *src, 317 unsigned int src_cnt, const unsigned char *scf, size_t len, 318 unsigned long flags) 319{ 320 struct fsl_re_chan *re_chan; 321 struct fsl_re_desc *desc; 322 struct fsl_re_xor_cdb *xor; 323 struct fsl_re_cmpnd_frame *cf; 324 u32 cdb; 325 unsigned int i, j; 326 unsigned int save_src_cnt = src_cnt; 327 int cont_q = 0; 328 329 re_chan = container_of(chan, struct fsl_re_chan, chan); 330 if (len > FSL_RE_MAX_DATA_LEN) { 331 dev_err(re_chan->dev, "genq tx length %zu, max length %d\n", 332 len, FSL_RE_MAX_DATA_LEN); 333 return NULL; 334 } 335 336 desc = fsl_re_chan_alloc_desc(re_chan, flags); 337 if (desc <= 0) 338 return NULL; 339 340 if (scf && (flags & DMA_PREP_CONTINUE)) { 341 cont_q = 1; 342 src_cnt += 1; 343 } 344 345 /* Filling xor CDB */ 346 cdb = FSL_RE_XOR_OPCODE << FSL_RE_CDB_OPCODE_SHIFT; 347 cdb |= (src_cnt - 1) << FSL_RE_CDB_NRCS_SHIFT; 348 cdb |= FSL_RE_BLOCK_SIZE << FSL_RE_CDB_BLKSIZE_SHIFT; 349 cdb |= FSL_RE_INTR_ON_ERROR << FSL_RE_CDB_ERROR_SHIFT; 350 cdb |= FSL_RE_DATA_DEP << FSL_RE_CDB_DEPEND_SHIFT; 351 xor = desc->cdb_addr; 352 xor->cdb32 = cdb; 353 354 if (scf) { 355 /* compute q = src0*coef0^src1*coef1^..., * is GF(8) mult */ 356 for (i = 0; i < save_src_cnt; i++) 357 xor->gfm[i] = scf[i]; 358 if (cont_q) 359 xor->gfm[i++] = 1; 360 } else { 361 /* compute P, that is XOR all srcs */ 362 for (i = 0; i < src_cnt; i++) 363 xor->gfm[i] = 1; 364 } 365 366 /* Filling frame 0 of compound frame descriptor with CDB */ 367 cf = desc->cf_addr; 368 fill_cfd_frame(cf, 0, sizeof(*xor), desc->cdb_paddr, 0); 369 370 /* Fill CFD's 1st frame with dest buffer */ 371 fill_cfd_frame(cf, 1, len, dest, 0); 372 373 /* Fill CFD's rest of the frames with source buffers */ 374 for (i = 2, j = 0; j < save_src_cnt; i++, j++) 375 fill_cfd_frame(cf, i, len, src[j], 0); 376 377 if (cont_q) 378 fill_cfd_frame(cf, i++, len, dest, 0); 379 380 /* Setting the final bit in the last source buffer frame in CFD */ 381 cf[i - 1].efrl32 |= 1 << FSL_RE_CF_FINAL_SHIFT; 382 383 return &desc->async_tx; 384} 385 386/* 387 * Prep function for P parity calculation.In RAID Engine terminology, 388 * XOR calculation is called GenQ calculation done through GenQ command 389 */ 390static struct dma_async_tx_descriptor *fsl_re_prep_dma_xor( 391 struct dma_chan *chan, dma_addr_t dest, dma_addr_t *src, 392 unsigned int src_cnt, size_t len, unsigned long flags) 393{ 394 /* NULL let genq take all coef as 1 */ 395 return fsl_re_prep_dma_genq(chan, dest, src, src_cnt, NULL, len, flags); 396} 397 398/* 399 * Prep function for P/Q parity calculation.In RAID Engine terminology, 400 * P/Q calculation is called GenQQ done through GenQQ command 401 */ 402static struct dma_async_tx_descriptor *fsl_re_prep_dma_pq( 403 struct dma_chan *chan, dma_addr_t *dest, dma_addr_t *src, 404 unsigned int src_cnt, const unsigned char *scf, size_t len, 405 unsigned long flags) 406{ 407 struct fsl_re_chan *re_chan; 408 struct fsl_re_desc *desc; 409 struct fsl_re_pq_cdb *pq; 410 struct fsl_re_cmpnd_frame *cf; 411 u32 cdb; 412 u8 *p; 413 int gfmq_len, i, j; 414 unsigned int save_src_cnt = src_cnt; 415 416 re_chan = container_of(chan, struct fsl_re_chan, chan); 417 if (len > FSL_RE_MAX_DATA_LEN) { 418 dev_err(re_chan->dev, "pq tx length is %zu, max length is %d\n", 419 len, FSL_RE_MAX_DATA_LEN); 420 return NULL; 421 } 422 423 /* 424 * RE requires at least 2 sources, if given only one source, we pass the 425 * second source same as the first one. 426 * With only one source, generating P is meaningless, only generate Q. 427 */ 428 if (src_cnt == 1) { 429 struct dma_async_tx_descriptor *tx; 430 dma_addr_t dma_src[2]; 431 unsigned char coef[2]; 432 433 dma_src[0] = *src; 434 coef[0] = *scf; 435 dma_src[1] = *src; 436 coef[1] = 0; 437 tx = fsl_re_prep_dma_genq(chan, dest[1], dma_src, 2, coef, len, 438 flags); 439 if (tx) 440 desc = to_fsl_re_dma_desc(tx); 441 442 return tx; 443 } 444 445 /* 446 * During RAID6 array creation, Linux's MD layer gets P and Q 447 * calculated separately in two steps. But our RAID Engine has 448 * the capability to calculate both P and Q with a single command 449 * Hence to merge well with MD layer, we need to provide a hook 450 * here and call re_jq_prep_dma_genq() function 451 */ 452 453 if (flags & DMA_PREP_PQ_DISABLE_P) 454 return fsl_re_prep_dma_genq(chan, dest[1], src, src_cnt, 455 scf, len, flags); 456 457 if (flags & DMA_PREP_CONTINUE) 458 src_cnt += 3; 459 460 desc = fsl_re_chan_alloc_desc(re_chan, flags); 461 if (desc <= 0) 462 return NULL; 463 464 /* Filling GenQQ CDB */ 465 cdb = FSL_RE_PQ_OPCODE << FSL_RE_CDB_OPCODE_SHIFT; 466 cdb |= (src_cnt - 1) << FSL_RE_CDB_NRCS_SHIFT; 467 cdb |= FSL_RE_BLOCK_SIZE << FSL_RE_CDB_BLKSIZE_SHIFT; 468 cdb |= FSL_RE_BUFFER_OUTPUT << FSL_RE_CDB_BUFFER_SHIFT; 469 cdb |= FSL_RE_DATA_DEP << FSL_RE_CDB_DEPEND_SHIFT; 470 471 pq = desc->cdb_addr; 472 pq->cdb32 = cdb; 473 474 p = pq->gfm_q1; 475 /* Init gfm_q1[] */ 476 for (i = 0; i < src_cnt; i++) 477 p[i] = 1; 478 479 /* Align gfm[] to 32bit */ 480 gfmq_len = ALIGN(src_cnt, 4); 481 482 /* Init gfm_q2[] */ 483 p += gfmq_len; 484 for (i = 0; i < src_cnt; i++) 485 p[i] = scf[i]; 486 487 /* Filling frame 0 of compound frame descriptor with CDB */ 488 cf = desc->cf_addr; 489 fill_cfd_frame(cf, 0, sizeof(struct fsl_re_pq_cdb), desc->cdb_paddr, 0); 490 491 /* Fill CFD's 1st & 2nd frame with dest buffers */ 492 for (i = 1, j = 0; i < 3; i++, j++) 493 fill_cfd_frame(cf, i, len, dest[j], 0); 494 495 /* Fill CFD's rest of the frames with source buffers */ 496 for (i = 3, j = 0; j < save_src_cnt; i++, j++) 497 fill_cfd_frame(cf, i, len, src[j], 0); 498 499 /* PQ computation continuation */ 500 if (flags & DMA_PREP_CONTINUE) { 501 if (src_cnt - save_src_cnt == 3) { 502 p[save_src_cnt] = 0; 503 p[save_src_cnt + 1] = 0; 504 p[save_src_cnt + 2] = 1; 505 fill_cfd_frame(cf, i++, len, dest[0], 0); 506 fill_cfd_frame(cf, i++, len, dest[1], 0); 507 fill_cfd_frame(cf, i++, len, dest[1], 0); 508 } else { 509 dev_err(re_chan->dev, "PQ tx continuation error!\n"); 510 return NULL; 511 } 512 } 513 514 /* Setting the final bit in the last source buffer frame in CFD */ 515 cf[i - 1].efrl32 |= 1 << FSL_RE_CF_FINAL_SHIFT; 516 517 return &desc->async_tx; 518} 519 520/* 521 * Prep function for memcpy. In RAID Engine, memcpy is done through MOVE 522 * command. Logic of this function will need to be modified once multipage 523 * support is added in Linux's MD/ASYNC Layer 524 */ 525static struct dma_async_tx_descriptor *fsl_re_prep_dma_memcpy( 526 struct dma_chan *chan, dma_addr_t dest, dma_addr_t src, 527 size_t len, unsigned long flags) 528{ 529 struct fsl_re_chan *re_chan; 530 struct fsl_re_desc *desc; 531 size_t length; 532 struct fsl_re_cmpnd_frame *cf; 533 struct fsl_re_move_cdb *move; 534 u32 cdb; 535 536 re_chan = container_of(chan, struct fsl_re_chan, chan); 537 538 if (len > FSL_RE_MAX_DATA_LEN) { 539 dev_err(re_chan->dev, "cp tx length is %zu, max length is %d\n", 540 len, FSL_RE_MAX_DATA_LEN); 541 return NULL; 542 } 543 544 desc = fsl_re_chan_alloc_desc(re_chan, flags); 545 if (desc <= 0) 546 return NULL; 547 548 /* Filling move CDB */ 549 cdb = FSL_RE_MOVE_OPCODE << FSL_RE_CDB_OPCODE_SHIFT; 550 cdb |= FSL_RE_BLOCK_SIZE << FSL_RE_CDB_BLKSIZE_SHIFT; 551 cdb |= FSL_RE_INTR_ON_ERROR << FSL_RE_CDB_ERROR_SHIFT; 552 cdb |= FSL_RE_DATA_DEP << FSL_RE_CDB_DEPEND_SHIFT; 553 554 move = desc->cdb_addr; 555 move->cdb32 = cdb; 556 557 /* Filling frame 0 of CFD with move CDB */ 558 cf = desc->cf_addr; 559 fill_cfd_frame(cf, 0, sizeof(*move), desc->cdb_paddr, 0); 560 561 length = min_t(size_t, len, FSL_RE_MAX_DATA_LEN); 562 563 /* Fill CFD's 1st frame with dest buffer */ 564 fill_cfd_frame(cf, 1, length, dest, 0); 565 566 /* Fill CFD's 2nd frame with src buffer */ 567 fill_cfd_frame(cf, 2, length, src, 1); 568 569 return &desc->async_tx; 570} 571 572static int fsl_re_alloc_chan_resources(struct dma_chan *chan) 573{ 574 struct fsl_re_chan *re_chan; 575 struct fsl_re_desc *desc; 576 void *cf; 577 dma_addr_t paddr; 578 int i; 579 580 re_chan = container_of(chan, struct fsl_re_chan, chan); 581 for (i = 0; i < FSL_RE_MIN_DESCS; i++) { 582 desc = kzalloc(sizeof(*desc), GFP_KERNEL); 583 if (!desc) 584 break; 585 586 cf = dma_pool_alloc(re_chan->re_dev->cf_desc_pool, GFP_KERNEL, 587 &paddr); 588 if (!cf) { 589 kfree(desc); 590 break; 591 } 592 593 INIT_LIST_HEAD(&desc->node); 594 fsl_re_init_desc(re_chan, desc, cf, paddr); 595 596 list_add_tail(&desc->node, &re_chan->free_q); 597 re_chan->alloc_count++; 598 } 599 return re_chan->alloc_count; 600} 601 602static void fsl_re_free_chan_resources(struct dma_chan *chan) 603{ 604 struct fsl_re_chan *re_chan; 605 struct fsl_re_desc *desc; 606 607 re_chan = container_of(chan, struct fsl_re_chan, chan); 608 while (re_chan->alloc_count--) { 609 desc = list_first_entry(&re_chan->free_q, 610 struct fsl_re_desc, 611 node); 612 613 list_del(&desc->node); 614 dma_pool_free(re_chan->re_dev->cf_desc_pool, desc->cf_addr, 615 desc->cf_paddr); 616 kfree(desc); 617 } 618 619 if (!list_empty(&re_chan->free_q)) 620 dev_err(re_chan->dev, "chan resource cannot be cleaned!\n"); 621} 622 623static int fsl_re_chan_probe(struct platform_device *ofdev, 624 struct device_node *np, u8 q, u32 off) 625{ 626 struct device *dev, *chandev; 627 struct fsl_re_drv_private *re_priv; 628 struct fsl_re_chan *chan; 629 struct dma_device *dma_dev; 630 u32 ptr; 631 u32 status; 632 int ret = 0, rc; 633 struct platform_device *chan_ofdev; 634 635 dev = &ofdev->dev; 636 re_priv = dev_get_drvdata(dev); 637 dma_dev = &re_priv->dma_dev; 638 639 chan = devm_kzalloc(dev, sizeof(*chan), GFP_KERNEL); 640 if (!chan) 641 return -ENOMEM; 642 643 /* create platform device for chan node */ 644 chan_ofdev = of_platform_device_create(np, NULL, dev); 645 if (!chan_ofdev) { 646 dev_err(dev, "Not able to create ofdev for jr %d\n", q); 647 ret = -EINVAL; 648 goto err_free; 649 } 650 651 /* read reg property from dts */ 652 rc = of_property_read_u32(np, "reg", &ptr); 653 if (rc) { 654 dev_err(dev, "Reg property not found in jr %d\n", q); 655 ret = -ENODEV; 656 goto err_free; 657 } 658 659 chan->jrregs = (struct fsl_re_chan_cfg *)((u8 *)re_priv->re_regs + 660 off + ptr); 661 662 /* read irq property from dts */ 663 chan->irq = irq_of_parse_and_map(np, 0); 664 if (!chan->irq) { 665 dev_err(dev, "No IRQ defined for JR %d\n", q); 666 ret = -ENODEV; 667 goto err_free; 668 } 669 670 snprintf(chan->name, sizeof(chan->name), "re_jr%02d", q); 671 672 chandev = &chan_ofdev->dev; 673 tasklet_setup(&chan->irqtask, fsl_re_dequeue); 674 675 ret = request_irq(chan->irq, fsl_re_isr, 0, chan->name, chandev); 676 if (ret) { 677 dev_err(dev, "Unable to register interrupt for JR %d\n", q); 678 ret = -EINVAL; 679 goto err_free; 680 } 681 682 re_priv->re_jrs[q] = chan; 683 chan->chan.device = dma_dev; 684 chan->chan.private = chan; 685 chan->dev = chandev; 686 chan->re_dev = re_priv; 687 688 spin_lock_init(&chan->desc_lock); 689 INIT_LIST_HEAD(&chan->ack_q); 690 INIT_LIST_HEAD(&chan->active_q); 691 INIT_LIST_HEAD(&chan->submit_q); 692 INIT_LIST_HEAD(&chan->free_q); 693 694 chan->inb_ring_virt_addr = dma_pool_alloc(chan->re_dev->hw_desc_pool, 695 GFP_KERNEL, &chan->inb_phys_addr); 696 if (!chan->inb_ring_virt_addr) { 697 dev_err(dev, "No dma memory for inb_ring_virt_addr\n"); 698 ret = -ENOMEM; 699 goto err_free; 700 } 701 702 chan->oub_ring_virt_addr = dma_pool_alloc(chan->re_dev->hw_desc_pool, 703 GFP_KERNEL, &chan->oub_phys_addr); 704 if (!chan->oub_ring_virt_addr) { 705 dev_err(dev, "No dma memory for oub_ring_virt_addr\n"); 706 ret = -ENOMEM; 707 goto err_free_1; 708 } 709 710 /* Program the Inbound/Outbound ring base addresses and size */ 711 out_be32(&chan->jrregs->inbring_base_h, 712 chan->inb_phys_addr & FSL_RE_ADDR_BIT_MASK); 713 out_be32(&chan->jrregs->oubring_base_h, 714 chan->oub_phys_addr & FSL_RE_ADDR_BIT_MASK); 715 out_be32(&chan->jrregs->inbring_base_l, 716 chan->inb_phys_addr >> FSL_RE_ADDR_BIT_SHIFT); 717 out_be32(&chan->jrregs->oubring_base_l, 718 chan->oub_phys_addr >> FSL_RE_ADDR_BIT_SHIFT); 719 out_be32(&chan->jrregs->inbring_size, 720 FSL_RE_RING_SIZE << FSL_RE_RING_SIZE_SHIFT); 721 out_be32(&chan->jrregs->oubring_size, 722 FSL_RE_RING_SIZE << FSL_RE_RING_SIZE_SHIFT); 723 724 /* Read LIODN value from u-boot */ 725 status = in_be32(&chan->jrregs->jr_config_1) & FSL_RE_REG_LIODN_MASK; 726 727 /* Program the CFG reg */ 728 out_be32(&chan->jrregs->jr_config_1, 729 FSL_RE_CFG1_CBSI | FSL_RE_CFG1_CBS0 | status); 730 731 dev_set_drvdata(chandev, chan); 732 733 /* Enable RE/CHAN */ 734 out_be32(&chan->jrregs->jr_command, FSL_RE_ENABLE); 735 736 return 0; 737 738err_free_1: 739 dma_pool_free(chan->re_dev->hw_desc_pool, chan->inb_ring_virt_addr, 740 chan->inb_phys_addr); 741err_free: 742 return ret; 743} 744 745/* Probe function for RAID Engine */ 746static int fsl_re_probe(struct platform_device *ofdev) 747{ 748 struct fsl_re_drv_private *re_priv; 749 struct device_node *np; 750 struct device_node *child; 751 u32 off; 752 u8 ridx = 0; 753 struct dma_device *dma_dev; 754 struct resource *res; 755 int rc; 756 struct device *dev = &ofdev->dev; 757 758 re_priv = devm_kzalloc(dev, sizeof(*re_priv), GFP_KERNEL); 759 if (!re_priv) 760 return -ENOMEM; 761 762 res = platform_get_resource(ofdev, IORESOURCE_MEM, 0); 763 if (!res) 764 return -ENODEV; 765 766 /* IOMAP the entire RAID Engine region */ 767 re_priv->re_regs = devm_ioremap(dev, res->start, resource_size(res)); 768 if (!re_priv->re_regs) 769 return -EBUSY; 770 771 /* Program the RE mode */ 772 out_be32(&re_priv->re_regs->global_config, FSL_RE_NON_DPAA_MODE); 773 774 /* Program Galois Field polynomial */ 775 out_be32(&re_priv->re_regs->galois_field_config, FSL_RE_GFM_POLY); 776 777 dev_info(dev, "version %x, mode %x, gfp %x\n", 778 in_be32(&re_priv->re_regs->re_version_id), 779 in_be32(&re_priv->re_regs->global_config), 780 in_be32(&re_priv->re_regs->galois_field_config)); 781 782 dma_dev = &re_priv->dma_dev; 783 dma_dev->dev = dev; 784 INIT_LIST_HEAD(&dma_dev->channels); 785 dma_set_mask(dev, DMA_BIT_MASK(40)); 786 787 dma_dev->device_alloc_chan_resources = fsl_re_alloc_chan_resources; 788 dma_dev->device_tx_status = fsl_re_tx_status; 789 dma_dev->device_issue_pending = fsl_re_issue_pending; 790 791 dma_dev->max_xor = FSL_RE_MAX_XOR_SRCS; 792 dma_dev->device_prep_dma_xor = fsl_re_prep_dma_xor; 793 dma_cap_set(DMA_XOR, dma_dev->cap_mask); 794 795 dma_dev->max_pq = FSL_RE_MAX_PQ_SRCS; 796 dma_dev->device_prep_dma_pq = fsl_re_prep_dma_pq; 797 dma_cap_set(DMA_PQ, dma_dev->cap_mask); 798 799 dma_dev->device_prep_dma_memcpy = fsl_re_prep_dma_memcpy; 800 dma_cap_set(DMA_MEMCPY, dma_dev->cap_mask); 801 802 dma_dev->device_free_chan_resources = fsl_re_free_chan_resources; 803 804 re_priv->total_chans = 0; 805 806 re_priv->cf_desc_pool = dmam_pool_create("fsl_re_cf_desc_pool", dev, 807 FSL_RE_CF_CDB_SIZE, 808 FSL_RE_CF_CDB_ALIGN, 0); 809 810 if (!re_priv->cf_desc_pool) { 811 dev_err(dev, "No memory for fsl re_cf desc pool\n"); 812 return -ENOMEM; 813 } 814 815 re_priv->hw_desc_pool = dmam_pool_create("fsl_re_hw_desc_pool", dev, 816 sizeof(struct fsl_re_hw_desc) * FSL_RE_RING_SIZE, 817 FSL_RE_FRAME_ALIGN, 0); 818 if (!re_priv->hw_desc_pool) { 819 dev_err(dev, "No memory for fsl re_hw desc pool\n"); 820 return -ENOMEM; 821 } 822 823 dev_set_drvdata(dev, re_priv); 824 825 /* Parse Device tree to find out the total number of JQs present */ 826 for_each_compatible_node(np, NULL, "fsl,raideng-v1.0-job-queue") { 827 rc = of_property_read_u32(np, "reg", &off); 828 if (rc) { 829 dev_err(dev, "Reg property not found in JQ node\n"); 830 of_node_put(np); 831 return -ENODEV; 832 } 833 /* Find out the Job Rings present under each JQ */ 834 for_each_child_of_node(np, child) { 835 rc = of_device_is_compatible(child, 836 "fsl,raideng-v1.0-job-ring"); 837 if (rc) { 838 fsl_re_chan_probe(ofdev, child, ridx++, off); 839 re_priv->total_chans++; 840 } 841 } 842 } 843 844 dma_async_device_register(dma_dev); 845 846 return 0; 847} 848 849static void fsl_re_remove_chan(struct fsl_re_chan *chan) 850{ 851 tasklet_kill(&chan->irqtask); 852 853 dma_pool_free(chan->re_dev->hw_desc_pool, chan->inb_ring_virt_addr, 854 chan->inb_phys_addr); 855 856 dma_pool_free(chan->re_dev->hw_desc_pool, chan->oub_ring_virt_addr, 857 chan->oub_phys_addr); 858} 859 860static void fsl_re_remove(struct platform_device *ofdev) 861{ 862 struct fsl_re_drv_private *re_priv; 863 struct device *dev; 864 int i; 865 866 dev = &ofdev->dev; 867 re_priv = dev_get_drvdata(dev); 868 869 /* Cleanup chan related memory areas */ 870 for (i = 0; i < re_priv->total_chans; i++) 871 fsl_re_remove_chan(re_priv->re_jrs[i]); 872 873 /* Unregister the driver */ 874 dma_async_device_unregister(&re_priv->dma_dev); 875} 876 877static const struct of_device_id fsl_re_ids[] = { 878 { .compatible = "fsl,raideng-v1.0", }, 879 {} 880}; 881MODULE_DEVICE_TABLE(of, fsl_re_ids); 882 883static struct platform_driver fsl_re_driver = { 884 .driver = { 885 .name = "fsl-raideng", 886 .of_match_table = fsl_re_ids, 887 }, 888 .probe = fsl_re_probe, 889 .remove_new = fsl_re_remove, 890}; 891 892module_platform_driver(fsl_re_driver); 893 894MODULE_AUTHOR("Harninder Rai <harninder.rai@freescale.com>"); 895MODULE_LICENSE("GPL v2"); 896MODULE_DESCRIPTION("Freescale RAID Engine Device Driver"); 897