1/* 2 * DMA Engine test module 3 * 4 * Copyright (C) 2007 Atmel Corporation 5 * 6 * This program is free software; you can redistribute it and/or modify 7 * it under the terms of the GNU General Public License version 2 as 8 * published by the Free Software Foundation. 9 */ 10#include <linux/delay.h> 11#include <linux/dmaengine.h> 12#include <linux/init.h> 13#include <linux/kthread.h> 14#include <linux/module.h> 15#include <linux/moduleparam.h> 16#include <linux/random.h> 17#include <linux/slab.h> 18#include <linux/wait.h> 19 20static unsigned int test_buf_size = 16384; 21module_param(test_buf_size, uint, S_IRUGO); 22MODULE_PARM_DESC(test_buf_size, "Size of the memcpy test buffer"); 23 24static char test_channel[20]; 25module_param_string(channel, test_channel, sizeof(test_channel), S_IRUGO); 26MODULE_PARM_DESC(channel, "Bus ID of the channel to test (default: any)"); 27 28static char test_device[20]; 29module_param_string(device, test_device, sizeof(test_device), S_IRUGO); 30MODULE_PARM_DESC(device, "Bus ID of the DMA Engine to test (default: any)"); 31 32static unsigned int threads_per_chan = 1; 33module_param(threads_per_chan, uint, S_IRUGO); 34MODULE_PARM_DESC(threads_per_chan, 35 "Number of threads to start per channel (default: 1)"); 36 37static unsigned int max_channels; 38module_param(max_channels, uint, S_IRUGO); 39MODULE_PARM_DESC(max_channels, 40 "Maximum number of channels to use (default: all)"); 41 42static unsigned int iterations; 43module_param(iterations, uint, S_IRUGO); 44MODULE_PARM_DESC(iterations, 45 "Iterations before stopping test (default: infinite)"); 46 47static unsigned int xor_sources = 3; 48module_param(xor_sources, uint, S_IRUGO); 49MODULE_PARM_DESC(xor_sources, 50 "Number of xor source buffers (default: 3)"); 51 52static unsigned int pq_sources = 3; 53module_param(pq_sources, uint, S_IRUGO); 54MODULE_PARM_DESC(pq_sources, 55 "Number of p+q source buffers (default: 3)"); 56 57/* 58 * Initialization patterns. All bytes in the source buffer has bit 7 59 * set, all bytes in the destination buffer has bit 7 cleared. 60 * 61 * Bit 6 is set for all bytes which are to be copied by the DMA 62 * engine. Bit 5 is set for all bytes which are to be overwritten by 63 * the DMA engine. 64 * 65 * The remaining bits are the inverse of a counter which increments by 66 * one for each byte address. 67 */ 68#define PATTERN_SRC 0x80 69#define PATTERN_DST 0x00 70#define PATTERN_COPY 0x40 71#define PATTERN_OVERWRITE 0x20 72#define PATTERN_COUNT_MASK 0x1f 73 74struct dmatest_thread { 75 struct list_head node; 76 struct task_struct *task; 77 struct dma_chan *chan; 78 u8 **srcs; 79 u8 **dsts; 80 enum dma_transaction_type type; 81}; 82 83struct dmatest_chan { 84 struct list_head node; 85 struct dma_chan *chan; 86 struct list_head threads; 87}; 88 89/* 90 * These are protected by dma_list_mutex since they're only used by 91 * the DMA filter function callback 92 */ 93static LIST_HEAD(dmatest_channels); 94static unsigned int nr_channels; 95 96static bool dmatest_match_channel(struct dma_chan *chan) 97{ 98 if (test_channel[0] == '\0') 99 return true; 100 return strcmp(dma_chan_name(chan), test_channel) == 0; 101} 102 103static bool dmatest_match_device(struct dma_device *device) 104{ 105 if (test_device[0] == '\0') 106 return true; 107 return strcmp(dev_name(device->dev), test_device) == 0; 108} 109 110static unsigned long dmatest_random(void) 111{ 112 unsigned long buf; 113 114 get_random_bytes(&buf, sizeof(buf)); 115 return buf; 116} 117 118static void dmatest_init_srcs(u8 **bufs, unsigned int start, unsigned int len) 119{ 120 unsigned int i; 121 u8 *buf; 122 123 for (; (buf = *bufs); bufs++) { 124 for (i = 0; i < start; i++) 125 buf[i] = PATTERN_SRC | (~i & PATTERN_COUNT_MASK); 126 for ( ; i < start + len; i++) 127 buf[i] = PATTERN_SRC | PATTERN_COPY 128 | (~i & PATTERN_COUNT_MASK); 129 for ( ; i < test_buf_size; i++) 130 buf[i] = PATTERN_SRC | (~i & PATTERN_COUNT_MASK); 131 buf++; 132 } 133} 134 135static void dmatest_init_dsts(u8 **bufs, unsigned int start, unsigned int len) 136{ 137 unsigned int i; 138 u8 *buf; 139 140 for (; (buf = *bufs); bufs++) { 141 for (i = 0; i < start; i++) 142 buf[i] = PATTERN_DST | (~i & PATTERN_COUNT_MASK); 143 for ( ; i < start + len; i++) 144 buf[i] = PATTERN_DST | PATTERN_OVERWRITE 145 | (~i & PATTERN_COUNT_MASK); 146 for ( ; i < test_buf_size; i++) 147 buf[i] = PATTERN_DST | (~i & PATTERN_COUNT_MASK); 148 } 149} 150 151static void dmatest_mismatch(u8 actual, u8 pattern, unsigned int index, 152 unsigned int counter, bool is_srcbuf) 153{ 154 u8 diff = actual ^ pattern; 155 u8 expected = pattern | (~counter & PATTERN_COUNT_MASK); 156 const char *thread_name = current->comm; 157 158 if (is_srcbuf) 159 pr_warning("%s: srcbuf[0x%x] overwritten!" 160 " Expected %02x, got %02x\n", 161 thread_name, index, expected, actual); 162 else if ((pattern & PATTERN_COPY) 163 && (diff & (PATTERN_COPY | PATTERN_OVERWRITE))) 164 pr_warning("%s: dstbuf[0x%x] not copied!" 165 " Expected %02x, got %02x\n", 166 thread_name, index, expected, actual); 167 else if (diff & PATTERN_SRC) 168 pr_warning("%s: dstbuf[0x%x] was copied!" 169 " Expected %02x, got %02x\n", 170 thread_name, index, expected, actual); 171 else 172 pr_warning("%s: dstbuf[0x%x] mismatch!" 173 " Expected %02x, got %02x\n", 174 thread_name, index, expected, actual); 175} 176 177static unsigned int dmatest_verify(u8 **bufs, unsigned int start, 178 unsigned int end, unsigned int counter, u8 pattern, 179 bool is_srcbuf) 180{ 181 unsigned int i; 182 unsigned int error_count = 0; 183 u8 actual; 184 u8 expected; 185 u8 *buf; 186 unsigned int counter_orig = counter; 187 188 for (; (buf = *bufs); bufs++) { 189 counter = counter_orig; 190 for (i = start; i < end; i++) { 191 actual = buf[i]; 192 expected = pattern | (~counter & PATTERN_COUNT_MASK); 193 if (actual != expected) { 194 if (error_count < 32) 195 dmatest_mismatch(actual, pattern, i, 196 counter, is_srcbuf); 197 error_count++; 198 } 199 counter++; 200 } 201 } 202 203 if (error_count > 32) 204 pr_warning("%s: %u errors suppressed\n", 205 current->comm, error_count - 32); 206 207 return error_count; 208} 209 210static void dmatest_callback(void *completion) 211{ 212 complete(completion); 213} 214 215/* 216 * This function repeatedly tests DMA transfers of various lengths and 217 * offsets for a given operation type until it is told to exit by 218 * kthread_stop(). There may be multiple threads running this function 219 * in parallel for a single channel, and there may be multiple channels 220 * being tested in parallel. 221 * 222 * Before each test, the source and destination buffer is initialized 223 * with a known pattern. This pattern is different depending on 224 * whether it's in an area which is supposed to be copied or 225 * overwritten, and different in the source and destination buffers. 226 * So if the DMA engine doesn't copy exactly what we tell it to copy, 227 * we'll notice. 228 */ 229static int dmatest_func(void *data) 230{ 231 struct dmatest_thread *thread = data; 232 struct dma_chan *chan; 233 const char *thread_name; 234 unsigned int src_off, dst_off, len; 235 unsigned int error_count; 236 unsigned int failed_tests = 0; 237 unsigned int total_tests = 0; 238 dma_cookie_t cookie; 239 enum dma_status status; 240 enum dma_ctrl_flags flags; 241 u8 pq_coefs[pq_sources + 1]; 242 int ret; 243 int src_cnt; 244 int dst_cnt; 245 int i; 246 247 thread_name = current->comm; 248 249 ret = -ENOMEM; 250 251 smp_rmb(); 252 chan = thread->chan; 253 if (thread->type == DMA_MEMCPY) 254 src_cnt = dst_cnt = 1; 255 else if (thread->type == DMA_XOR) { 256 src_cnt = xor_sources | 1; /* force odd to ensure dst = src */ 257 dst_cnt = 1; 258 } else if (thread->type == DMA_PQ) { 259 src_cnt = pq_sources | 1; /* force odd to ensure dst = src */ 260 dst_cnt = 2; 261 for (i = 0; i < src_cnt; i++) 262 pq_coefs[i] = 1; 263 } else 264 goto err_srcs; 265 266 thread->srcs = kcalloc(src_cnt+1, sizeof(u8 *), GFP_KERNEL); 267 if (!thread->srcs) 268 goto err_srcs; 269 for (i = 0; i < src_cnt; i++) { 270 thread->srcs[i] = kmalloc(test_buf_size, GFP_KERNEL); 271 if (!thread->srcs[i]) 272 goto err_srcbuf; 273 } 274 thread->srcs[i] = NULL; 275 276 thread->dsts = kcalloc(dst_cnt+1, sizeof(u8 *), GFP_KERNEL); 277 if (!thread->dsts) 278 goto err_dsts; 279 for (i = 0; i < dst_cnt; i++) { 280 thread->dsts[i] = kmalloc(test_buf_size, GFP_KERNEL); 281 if (!thread->dsts[i]) 282 goto err_dstbuf; 283 } 284 thread->dsts[i] = NULL; 285 286 set_user_nice(current, 10); 287 288 flags = DMA_CTRL_ACK | DMA_COMPL_SKIP_DEST_UNMAP | DMA_PREP_INTERRUPT; 289 290 while (!kthread_should_stop() 291 && !(iterations && total_tests >= iterations)) { 292 struct dma_device *dev = chan->device; 293 struct dma_async_tx_descriptor *tx = NULL; 294 dma_addr_t dma_srcs[src_cnt]; 295 dma_addr_t dma_dsts[dst_cnt]; 296 struct completion cmp; 297 unsigned long tmo = msecs_to_jiffies(3000); 298 u8 align = 0; 299 300 total_tests++; 301 302 /* honor alignment restrictions */ 303 if (thread->type == DMA_MEMCPY) 304 align = dev->copy_align; 305 else if (thread->type == DMA_XOR) 306 align = dev->xor_align; 307 else if (thread->type == DMA_PQ) 308 align = dev->pq_align; 309 310 if (1 << align > test_buf_size) { 311 pr_err("%u-byte buffer too small for %d-byte alignment\n", 312 test_buf_size, 1 << align); 313 break; 314 } 315 316 len = dmatest_random() % test_buf_size + 1; 317 len = (len >> align) << align; 318 if (!len) 319 len = 1 << align; 320 src_off = dmatest_random() % (test_buf_size - len + 1); 321 dst_off = dmatest_random() % (test_buf_size - len + 1); 322 323 src_off = (src_off >> align) << align; 324 dst_off = (dst_off >> align) << align; 325 326 dmatest_init_srcs(thread->srcs, src_off, len); 327 dmatest_init_dsts(thread->dsts, dst_off, len); 328 329 for (i = 0; i < src_cnt; i++) { 330 u8 *buf = thread->srcs[i] + src_off; 331 332 dma_srcs[i] = dma_map_single(dev->dev, buf, len, 333 DMA_TO_DEVICE); 334 } 335 /* map with DMA_BIDIRECTIONAL to force writeback/invalidate */ 336 for (i = 0; i < dst_cnt; i++) { 337 dma_dsts[i] = dma_map_single(dev->dev, thread->dsts[i], 338 test_buf_size, 339 DMA_BIDIRECTIONAL); 340 } 341 342 343 if (thread->type == DMA_MEMCPY) 344 tx = dev->device_prep_dma_memcpy(chan, 345 dma_dsts[0] + dst_off, 346 dma_srcs[0], len, 347 flags); 348 else if (thread->type == DMA_XOR) 349 tx = dev->device_prep_dma_xor(chan, 350 dma_dsts[0] + dst_off, 351 dma_srcs, src_cnt, 352 len, flags); 353 else if (thread->type == DMA_PQ) { 354 dma_addr_t dma_pq[dst_cnt]; 355 356 for (i = 0; i < dst_cnt; i++) 357 dma_pq[i] = dma_dsts[i] + dst_off; 358 tx = dev->device_prep_dma_pq(chan, dma_pq, dma_srcs, 359 src_cnt, pq_coefs, 360 len, flags); 361 } 362 363 if (!tx) { 364 for (i = 0; i < src_cnt; i++) 365 dma_unmap_single(dev->dev, dma_srcs[i], len, 366 DMA_TO_DEVICE); 367 for (i = 0; i < dst_cnt; i++) 368 dma_unmap_single(dev->dev, dma_dsts[i], 369 test_buf_size, 370 DMA_BIDIRECTIONAL); 371 pr_warning("%s: #%u: prep error with src_off=0x%x " 372 "dst_off=0x%x len=0x%x\n", 373 thread_name, total_tests - 1, 374 src_off, dst_off, len); 375 msleep(100); 376 failed_tests++; 377 continue; 378 } 379 380 init_completion(&cmp); 381 tx->callback = dmatest_callback; 382 tx->callback_param = &cmp; 383 cookie = tx->tx_submit(tx); 384 385 if (dma_submit_error(cookie)) { 386 pr_warning("%s: #%u: submit error %d with src_off=0x%x " 387 "dst_off=0x%x len=0x%x\n", 388 thread_name, total_tests - 1, cookie, 389 src_off, dst_off, len); 390 msleep(100); 391 failed_tests++; 392 continue; 393 } 394 dma_async_issue_pending(chan); 395 396 tmo = wait_for_completion_timeout(&cmp, tmo); 397 status = dma_async_is_tx_complete(chan, cookie, NULL, NULL); 398 399 if (tmo == 0) { 400 pr_warning("%s: #%u: test timed out\n", 401 thread_name, total_tests - 1); 402 failed_tests++; 403 continue; 404 } else if (status != DMA_SUCCESS) { 405 pr_warning("%s: #%u: got completion callback," 406 " but status is \'%s\'\n", 407 thread_name, total_tests - 1, 408 status == DMA_ERROR ? "error" : "in progress"); 409 failed_tests++; 410 continue; 411 } 412 413 /* Unmap by myself (see DMA_COMPL_SKIP_DEST_UNMAP above) */ 414 for (i = 0; i < dst_cnt; i++) 415 dma_unmap_single(dev->dev, dma_dsts[i], test_buf_size, 416 DMA_BIDIRECTIONAL); 417 418 error_count = 0; 419 420 pr_debug("%s: verifying source buffer...\n", thread_name); 421 error_count += dmatest_verify(thread->srcs, 0, src_off, 422 0, PATTERN_SRC, true); 423 error_count += dmatest_verify(thread->srcs, src_off, 424 src_off + len, src_off, 425 PATTERN_SRC | PATTERN_COPY, true); 426 error_count += dmatest_verify(thread->srcs, src_off + len, 427 test_buf_size, src_off + len, 428 PATTERN_SRC, true); 429 430 pr_debug("%s: verifying dest buffer...\n", 431 thread->task->comm); 432 error_count += dmatest_verify(thread->dsts, 0, dst_off, 433 0, PATTERN_DST, false); 434 error_count += dmatest_verify(thread->dsts, dst_off, 435 dst_off + len, src_off, 436 PATTERN_SRC | PATTERN_COPY, false); 437 error_count += dmatest_verify(thread->dsts, dst_off + len, 438 test_buf_size, dst_off + len, 439 PATTERN_DST, false); 440 441 if (error_count) { 442 pr_warning("%s: #%u: %u errors with " 443 "src_off=0x%x dst_off=0x%x len=0x%x\n", 444 thread_name, total_tests - 1, error_count, 445 src_off, dst_off, len); 446 failed_tests++; 447 } else { 448 pr_debug("%s: #%u: No errors with " 449 "src_off=0x%x dst_off=0x%x len=0x%x\n", 450 thread_name, total_tests - 1, 451 src_off, dst_off, len); 452 } 453 } 454 455 ret = 0; 456 for (i = 0; thread->dsts[i]; i++) 457 kfree(thread->dsts[i]); 458err_dstbuf: 459 kfree(thread->dsts); 460err_dsts: 461 for (i = 0; thread->srcs[i]; i++) 462 kfree(thread->srcs[i]); 463err_srcbuf: 464 kfree(thread->srcs); 465err_srcs: 466 pr_notice("%s: terminating after %u tests, %u failures (status %d)\n", 467 thread_name, total_tests, failed_tests, ret); 468 469 if (iterations > 0) 470 while (!kthread_should_stop()) { 471 DECLARE_WAIT_QUEUE_HEAD_ONSTACK(wait_dmatest_exit); 472 interruptible_sleep_on(&wait_dmatest_exit); 473 } 474 475 return ret; 476} 477 478static void dmatest_cleanup_channel(struct dmatest_chan *dtc) 479{ 480 struct dmatest_thread *thread; 481 struct dmatest_thread *_thread; 482 int ret; 483 484 list_for_each_entry_safe(thread, _thread, &dtc->threads, node) { 485 ret = kthread_stop(thread->task); 486 pr_debug("dmatest: thread %s exited with status %d\n", 487 thread->task->comm, ret); 488 list_del(&thread->node); 489 kfree(thread); 490 } 491 kfree(dtc); 492} 493 494static int dmatest_add_threads(struct dmatest_chan *dtc, enum dma_transaction_type type) 495{ 496 struct dmatest_thread *thread; 497 struct dma_chan *chan = dtc->chan; 498 char *op; 499 unsigned int i; 500 501 if (type == DMA_MEMCPY) 502 op = "copy"; 503 else if (type == DMA_XOR) 504 op = "xor"; 505 else if (type == DMA_PQ) 506 op = "pq"; 507 else 508 return -EINVAL; 509 510 for (i = 0; i < threads_per_chan; i++) { 511 thread = kzalloc(sizeof(struct dmatest_thread), GFP_KERNEL); 512 if (!thread) { 513 pr_warning("dmatest: No memory for %s-%s%u\n", 514 dma_chan_name(chan), op, i); 515 516 break; 517 } 518 thread->chan = dtc->chan; 519 thread->type = type; 520 smp_wmb(); 521 thread->task = kthread_run(dmatest_func, thread, "%s-%s%u", 522 dma_chan_name(chan), op, i); 523 if (IS_ERR(thread->task)) { 524 pr_warning("dmatest: Failed to run thread %s-%s%u\n", 525 dma_chan_name(chan), op, i); 526 kfree(thread); 527 break; 528 } 529 530 /* srcbuf and dstbuf are allocated by the thread itself */ 531 532 list_add_tail(&thread->node, &dtc->threads); 533 } 534 535 return i; 536} 537 538static int dmatest_add_channel(struct dma_chan *chan) 539{ 540 struct dmatest_chan *dtc; 541 struct dma_device *dma_dev = chan->device; 542 unsigned int thread_count = 0; 543 int cnt; 544 545 dtc = kmalloc(sizeof(struct dmatest_chan), GFP_KERNEL); 546 if (!dtc) { 547 pr_warning("dmatest: No memory for %s\n", dma_chan_name(chan)); 548 return -ENOMEM; 549 } 550 551 dtc->chan = chan; 552 INIT_LIST_HEAD(&dtc->threads); 553 554 if (dma_has_cap(DMA_MEMCPY, dma_dev->cap_mask)) { 555 cnt = dmatest_add_threads(dtc, DMA_MEMCPY); 556 thread_count += cnt > 0 ? cnt : 0; 557 } 558 if (dma_has_cap(DMA_XOR, dma_dev->cap_mask)) { 559 cnt = dmatest_add_threads(dtc, DMA_XOR); 560 thread_count += cnt > 0 ? cnt : 0; 561 } 562 if (dma_has_cap(DMA_PQ, dma_dev->cap_mask)) { 563 cnt = dmatest_add_threads(dtc, DMA_PQ); 564 thread_count += cnt > 0 ?: 0; 565 } 566 567 pr_info("dmatest: Started %u threads using %s\n", 568 thread_count, dma_chan_name(chan)); 569 570 list_add_tail(&dtc->node, &dmatest_channels); 571 nr_channels++; 572 573 return 0; 574} 575 576static bool filter(struct dma_chan *chan, void *param) 577{ 578 if (!dmatest_match_channel(chan) || !dmatest_match_device(chan->device)) 579 return false; 580 else 581 return true; 582} 583 584static int __init dmatest_init(void) 585{ 586 dma_cap_mask_t mask; 587 struct dma_chan *chan; 588 int err = 0; 589 590 dma_cap_zero(mask); 591 dma_cap_set(DMA_MEMCPY, mask); 592 for (;;) { 593 chan = dma_request_channel(mask, filter, NULL); 594 if (chan) { 595 err = dmatest_add_channel(chan); 596 if (err) { 597 dma_release_channel(chan); 598 break; /* add_channel failed, punt */ 599 } 600 } else 601 break; /* no more channels available */ 602 if (max_channels && nr_channels >= max_channels) 603 break; /* we have all we need */ 604 } 605 606 return err; 607} 608/* when compiled-in wait for drivers to load first */ 609late_initcall(dmatest_init); 610 611static void __exit dmatest_exit(void) 612{ 613 struct dmatest_chan *dtc, *_dtc; 614 struct dma_chan *chan; 615 616 list_for_each_entry_safe(dtc, _dtc, &dmatest_channels, node) { 617 list_del(&dtc->node); 618 chan = dtc->chan; 619 dmatest_cleanup_channel(dtc); 620 pr_debug("dmatest: dropped channel %s\n", 621 dma_chan_name(chan)); 622 dma_release_channel(chan); 623 } 624} 625module_exit(dmatest_exit); 626 627MODULE_AUTHOR("Haavard Skinnemoen <hskinnemoen@atmel.com>"); 628MODULE_LICENSE("GPL v2"); 629