1/* $FreeBSD: head/sys/dev/usb2/core/usb2_busdma.c 184824 2008-11-10 20:54:31Z thompsa $ */
|
1/* $FreeBSD: head/sys/dev/usb2/core/usb2_busdma.c 185087 2008-11-19 08:56:35Z alfred $ */ |
2/*- 3 * Copyright (c) 2008 Hans Petter Selasky. All rights reserved. 4 * 5 * Redistribution and use in source and binary forms, with or without 6 * modification, are permitted provided that the following conditions 7 * are met: 8 * 1. Redistributions of source code must retain the above copyright 9 * notice, this list of conditions and the following disclaimer. 10 * 2. Redistributions in binary form must reproduce the above copyright 11 * notice, this list of conditions and the following disclaimer in the 12 * documentation and/or other materials provided with the distribution. 13 * 14 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 15 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 16 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 17 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 18 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 19 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 20 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 21 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 22 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 23 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 24 * SUCH DAMAGE. 25 */ 26 27#include <dev/usb2/include/usb2_mfunc.h> 28#include <dev/usb2/include/usb2_error.h> 29#include <dev/usb2/include/usb2_standard.h> 30#include <dev/usb2/include/usb2_defs.h> 31 32#include <dev/usb2/core/usb2_core.h> 33#include <dev/usb2/core/usb2_busdma.h> 34#include <dev/usb2/core/usb2_process.h> 35#include <dev/usb2/core/usb2_transfer.h> 36#include <dev/usb2/core/usb2_device.h> 37#include <dev/usb2/core/usb2_util.h> 38 39#include <dev/usb2/controller/usb2_controller.h> 40#include <dev/usb2/controller/usb2_bus.h> 41 42static void usb2_dma_tag_create(struct usb2_dma_tag *udt, uint32_t size, uint32_t align); 43static void usb2_dma_tag_destroy(struct usb2_dma_tag *udt); 44 45#ifdef __FreeBSD__ 46static void usb2_dma_lock_cb(void *arg, bus_dma_lock_op_t op); 47static int32_t usb2_m_copy_in_cb(void *arg, void *src, uint32_t count); 48static void usb2_pc_alloc_mem_cb(void *arg, bus_dma_segment_t *segs, int nseg, int error); 49static void usb2_pc_load_mem_cb(void *arg, bus_dma_segment_t *segs, int nseg, int error); 50static void usb2_pc_common_mem_cb(void *arg, bus_dma_segment_t *segs, int nseg, int error, uint8_t isload); 51 52#endif 53 54#ifdef __NetBSD__ 55static int32_t usb2_m_copy_in_cb(void *arg, caddr_t src, uint32_t count); 56static void usb2_pc_common_mem_cb(struct usb2_page_cache *pc, bus_dma_segment_t *segs, int nseg, int error, uint8_t isload); 57 58#endif 59 60/*------------------------------------------------------------------------* 61 * usb2_get_page - lookup DMA-able memory for the given offset 62 * 63 * NOTE: Only call this function when the "page_cache" structure has 64 * been properly initialized ! 65 *------------------------------------------------------------------------*/ 66void 67usb2_get_page(struct usb2_page_cache *pc, uint32_t offset, 68 struct usb2_page_search *res) 69{ 70 struct usb2_page *page; 71 72 if (pc->page_start) { 73 74 /* Case 1 - something has been loaded into DMA */ 75 76 if (pc->buffer) { 77 78 /* Case 1a - Kernel Virtual Address */ 79 80 res->buffer = USB_ADD_BYTES(pc->buffer, offset); 81 } 82 offset += pc->page_offset_buf; 83 84 /* compute destination page */ 85 86 page = pc->page_start; 87 88 if (pc->ismultiseg) { 89 90 page += (offset / USB_PAGE_SIZE); 91 92 offset %= USB_PAGE_SIZE; 93 94 res->length = USB_PAGE_SIZE - offset; 95 res->physaddr = page->physaddr + offset; 96 } else { 97 res->length = 0 - 1; 98 res->physaddr = page->physaddr + offset; 99 } 100 if (!pc->buffer) { 101 102 /* Case 1b - Non Kernel Virtual Address */ 103 104 res->buffer = USB_ADD_BYTES(page->buffer, offset); 105 } 106 } else { 107 108 /* Case 2 - Plain PIO */ 109 110 res->buffer = USB_ADD_BYTES(pc->buffer, offset); 111 res->length = 0 - 1; 112 res->physaddr = 0; 113 } 114 return; 115} 116 117/*------------------------------------------------------------------------* 118 * usb2_copy_in - copy directly to DMA-able memory 119 *------------------------------------------------------------------------*/ 120void 121usb2_copy_in(struct usb2_page_cache *cache, uint32_t offset, 122 const void *ptr, uint32_t len) 123{ 124 struct usb2_page_search buf_res; 125 126 while (len != 0) { 127 128 usb2_get_page(cache, offset, &buf_res); 129 130 if (buf_res.length > len) { 131 buf_res.length = len; 132 } 133 bcopy(ptr, buf_res.buffer, buf_res.length); 134 135 offset += buf_res.length; 136 len -= buf_res.length; 137 ptr = USB_ADD_BYTES(ptr, buf_res.length); 138 } 139 return; 140} 141 142/*------------------------------------------------------------------------* 143 * usb2_copy_in_user - copy directly to DMA-able memory from userland 144 * 145 * Return values: 146 * 0: Success 147 * Else: Failure 148 *------------------------------------------------------------------------*/ 149int 150usb2_copy_in_user(struct usb2_page_cache *cache, uint32_t offset, 151 const void *ptr, uint32_t len) 152{ 153 struct usb2_page_search buf_res; 154 int error; 155 156 while (len != 0) { 157 158 usb2_get_page(cache, offset, &buf_res); 159 160 if (buf_res.length > len) { 161 buf_res.length = len; 162 } 163 error = copyin(ptr, buf_res.buffer, buf_res.length); 164 if (error) 165 return (error); 166 167 offset += buf_res.length; 168 len -= buf_res.length; 169 ptr = USB_ADD_BYTES(ptr, buf_res.length); 170 } 171 return (0); /* success */ 172} 173 174/*------------------------------------------------------------------------* 175 * usb2_m_copy_in - copy a mbuf chain directly into DMA-able memory 176 *------------------------------------------------------------------------*/ 177struct usb2_m_copy_in_arg { 178 struct usb2_page_cache *cache; 179 uint32_t dst_offset; 180}; 181 182static int32_t 183#ifdef __FreeBSD__ 184usb2_m_copy_in_cb(void *arg, void *src, uint32_t count) 185#else 186usb2_m_copy_in_cb(void *arg, caddr_t src, uint32_t count) 187#endif 188{ 189 register struct usb2_m_copy_in_arg *ua = arg; 190 191 usb2_copy_in(ua->cache, ua->dst_offset, src, count); 192 ua->dst_offset += count; 193 return (0); 194} 195 196void 197usb2_m_copy_in(struct usb2_page_cache *cache, uint32_t dst_offset, 198 struct mbuf *m, uint32_t src_offset, uint32_t src_len) 199{ 200 struct usb2_m_copy_in_arg arg = {cache, dst_offset}; 201 register int error; 202 203 error = m_apply(m, src_offset, src_len, &usb2_m_copy_in_cb, &arg); 204 return; 205} 206 207/*------------------------------------------------------------------------* 208 * usb2_uiomove - factored out code 209 *------------------------------------------------------------------------*/ 210int 211usb2_uiomove(struct usb2_page_cache *pc, struct uio *uio, 212 uint32_t pc_offset, uint32_t len) 213{ 214 struct usb2_page_search res; 215 int error = 0; 216 217 while (len != 0) { 218 219 usb2_get_page(pc, pc_offset, &res); 220 221 if (res.length > len) { 222 res.length = len; 223 } 224 /* 225 * "uiomove()" can sleep so one needs to make a wrapper, 226 * exiting the mutex and checking things 227 */ 228 error = uiomove(res.buffer, res.length, uio); 229 230 if (error) { 231 break; 232 } 233 pc_offset += res.length; 234 len -= res.length; 235 } 236 return (error); 237} 238 239/*------------------------------------------------------------------------* 240 * usb2_copy_out - copy directly from DMA-able memory 241 *------------------------------------------------------------------------*/ 242void 243usb2_copy_out(struct usb2_page_cache *cache, uint32_t offset, 244 void *ptr, uint32_t len) 245{ 246 struct usb2_page_search res; 247 248 while (len != 0) { 249 250 usb2_get_page(cache, offset, &res); 251 252 if (res.length > len) { 253 res.length = len; 254 } 255 bcopy(res.buffer, ptr, res.length); 256 257 offset += res.length; 258 len -= res.length; 259 ptr = USB_ADD_BYTES(ptr, res.length); 260 } 261 return; 262} 263 264/*------------------------------------------------------------------------* 265 * usb2_copy_out_user - copy directly from DMA-able memory to userland 266 * 267 * Return values: 268 * 0: Success 269 * Else: Failure 270 *------------------------------------------------------------------------*/ 271int 272usb2_copy_out_user(struct usb2_page_cache *cache, uint32_t offset, 273 void *ptr, uint32_t len) 274{ 275 struct usb2_page_search res; 276 int error; 277 278 while (len != 0) { 279 280 usb2_get_page(cache, offset, &res); 281 282 if (res.length > len) { 283 res.length = len; 284 } 285 error = copyout(res.buffer, ptr, res.length); 286 if (error) 287 return (error); 288 289 offset += res.length; 290 len -= res.length; 291 ptr = USB_ADD_BYTES(ptr, res.length); 292 } 293 return (0); /* success */ 294} 295 296/*------------------------------------------------------------------------* 297 * usb2_bzero - zero DMA-able memory 298 *------------------------------------------------------------------------*/ 299void 300usb2_bzero(struct usb2_page_cache *cache, uint32_t offset, uint32_t len) 301{ 302 struct usb2_page_search res; 303 304 while (len != 0) { 305 306 usb2_get_page(cache, offset, &res); 307 308 if (res.length > len) { 309 res.length = len; 310 } 311 bzero(res.buffer, res.length); 312 313 offset += res.length; 314 len -= res.length; 315 } 316 return; 317} 318 319 320#ifdef __FreeBSD__ 321 322/*------------------------------------------------------------------------* 323 * usb2_dma_lock_cb - dummy callback 324 *------------------------------------------------------------------------*/ 325static void 326usb2_dma_lock_cb(void *arg, bus_dma_lock_op_t op) 327{ 328 /* we use "mtx_owned()" instead of this function */ 329 return; 330} 331 332/*------------------------------------------------------------------------* 333 * usb2_dma_tag_create - allocate a DMA tag 334 * 335 * NOTE: If the "align" parameter has a value of 1 the DMA-tag will 336 * allow multi-segment mappings. Else all mappings are single-segment. 337 *------------------------------------------------------------------------*/ 338static void 339usb2_dma_tag_create(struct usb2_dma_tag *udt, 340 uint32_t size, uint32_t align) 341{ 342 bus_dma_tag_t tag; 343 344 if (bus_dma_tag_create 345 ( /* parent */ udt->tag_parent->tag, 346 /* alignment */ align, 347 /* boundary */ USB_PAGE_SIZE, 348 /* lowaddr */ (2ULL << (udt->tag_parent->dma_bits - 1)) - 1, 349 /* highaddr */ BUS_SPACE_MAXADDR, 350 /* filter */ NULL, 351 /* filterarg */ NULL, 352 /* maxsize */ size, 353 /* nsegments */ (align == 1) ? 354 (2 + (size / USB_PAGE_SIZE)) : 1, 355 /* maxsegsz */ (align == 1) ? 356 USB_PAGE_SIZE : size, 357 /* flags */ 0, 358 /* lockfn */ &usb2_dma_lock_cb, 359 /* lockarg */ NULL, 360 &tag)) { 361 tag = NULL; 362 } 363 udt->tag = tag; 364 return; 365} 366 367/*------------------------------------------------------------------------* 368 * usb2_dma_tag_free - free a DMA tag 369 *------------------------------------------------------------------------*/ 370static void 371usb2_dma_tag_destroy(struct usb2_dma_tag *udt) 372{ 373 bus_dma_tag_destroy(udt->tag); 374 return; 375} 376 377/*------------------------------------------------------------------------* 378 * usb2_pc_alloc_mem_cb - BUS-DMA callback function 379 *------------------------------------------------------------------------*/ 380static void 381usb2_pc_alloc_mem_cb(void *arg, bus_dma_segment_t *segs, 382 int nseg, int error) 383{ 384 usb2_pc_common_mem_cb(arg, segs, nseg, error, 0); 385 return; 386} 387 388/*------------------------------------------------------------------------* 389 * usb2_pc_load_mem_cb - BUS-DMA callback function 390 *------------------------------------------------------------------------*/ 391static void 392usb2_pc_load_mem_cb(void *arg, bus_dma_segment_t *segs, 393 int nseg, int error) 394{ 395 usb2_pc_common_mem_cb(arg, segs, nseg, error, 1); 396 return; 397} 398 399/*------------------------------------------------------------------------* 400 * usb2_pc_common_mem_cb - BUS-DMA callback function 401 *------------------------------------------------------------------------*/ 402static void 403usb2_pc_common_mem_cb(void *arg, bus_dma_segment_t *segs, 404 int nseg, int error, uint8_t isload) 405{ 406 struct usb2_dma_parent_tag *uptag; 407 struct usb2_page_cache *pc; 408 struct usb2_page *pg; 409 uint32_t rem; 410 uint8_t owned; 411 412 pc = arg; 413 uptag = pc->tag_parent; 414 415 /* 416 * XXX There is sometimes recursive locking here. 417 * XXX We should try to find a better solution. 418 * XXX Until further the "owned" variable does 419 * XXX the trick. 420 */ 421 422 if (error) { 423 goto done; 424 } 425 pg = pc->page_start; 426 pg->physaddr = segs->ds_addr & ~(USB_PAGE_SIZE - 1); 427 rem = segs->ds_addr & (USB_PAGE_SIZE - 1); 428 pc->page_offset_buf = rem; 429 pc->page_offset_end += rem; 430 nseg--; 431 432 while (nseg > 0) { 433 nseg--; 434 segs++; 435 pg++; 436 pg->physaddr = segs->ds_addr & ~(USB_PAGE_SIZE - 1); 437 } 438 439done: 440 owned = mtx_owned(uptag->mtx); 441 if (!owned) 442 mtx_lock(uptag->mtx); 443 444 uptag->dma_error = (error ? 1 : 0); 445 if (isload) { 446 (uptag->func) (uptag); 447 } else { 448 usb2_cv_broadcast(uptag->cv); 449 } 450 if (!owned) 451 mtx_unlock(uptag->mtx); 452 return; 453} 454 455/*------------------------------------------------------------------------* 456 * usb2_pc_alloc_mem - allocate DMA'able memory 457 * 458 * Returns: 459 * 0: Success 460 * Else: Failure 461 *------------------------------------------------------------------------*/ 462uint8_t 463usb2_pc_alloc_mem(struct usb2_page_cache *pc, struct usb2_page *pg, 464 uint32_t size, uint32_t align) 465{ 466 struct usb2_dma_parent_tag *uptag; 467 struct usb2_dma_tag *utag; 468 bus_dmamap_t map; 469 void *ptr; 470 int err; 471 472 uptag = pc->tag_parent; 473 474 if (align != 1) { 475 /* 476 * The alignment must be greater or equal to the 477 * "size" else the object can be split between two 478 * memory pages and we get a problem! 479 */ 480 while (align < size) { 481 align *= 2; 482 if (align == 0) { 483 goto error; 484 } 485 } 486#if 1 487 /* 488 * XXX BUS-DMA workaround - FIXME later: 489 * 490 * We assume that that the aligment at this point of 491 * the code is greater than or equal to the size and 492 * less than two times the size, so that if we double 493 * the size, the size will be greater than the 494 * alignment. 495 * 496 * The bus-dma system has a check for "alignment" 497 * being less than "size". If that check fails we end 498 * up using contigmalloc which is page based even for 499 * small allocations. Try to avoid that to save 500 * memory, hence we sometimes to a large number of 501 * small allocations! 502 */ 503 if (size <= (USB_PAGE_SIZE / 2)) { 504 size *= 2; 505 } 506#endif 507 } 508 /* get the correct DMA tag */ 509 utag = usb2_dma_tag_find(uptag, size, align); 510 if (utag == NULL) { 511 goto error; 512 } 513 /* allocate memory */ 514 if (bus_dmamem_alloc( 515 utag->tag, &ptr, (BUS_DMA_WAITOK | BUS_DMA_COHERENT), &map)) { 516 goto error; 517 } 518 /* setup page cache */ 519 pc->buffer = ptr; 520 pc->page_start = pg; 521 pc->page_offset_buf = 0; 522 pc->page_offset_end = size; 523 pc->map = map; 524 pc->tag = utag->tag; 525 pc->ismultiseg = (align == 1); 526 527 mtx_lock(uptag->mtx); 528 529 /* load memory into DMA */ 530 err = bus_dmamap_load( 531 utag->tag, map, ptr, size, &usb2_pc_alloc_mem_cb, 532 pc, (BUS_DMA_WAITOK | BUS_DMA_COHERENT)); 533 534 if (err == EINPROGRESS) { 535 usb2_cv_wait(uptag->cv, uptag->mtx); 536 err = 0; 537 } 538 mtx_unlock(uptag->mtx); 539 540 if (err || uptag->dma_error) { 541 bus_dmamem_free(utag->tag, ptr, map); 542 goto error; 543 } 544 bzero(ptr, size); 545 546 usb2_pc_cpu_flush(pc); 547 548 return (0); 549 550error: 551 /* reset most of the page cache */ 552 pc->buffer = NULL; 553 pc->page_start = NULL; 554 pc->page_offset_buf = 0; 555 pc->page_offset_end = 0; 556 pc->map = NULL; 557 pc->tag = NULL; 558 return (1); 559} 560 561/*------------------------------------------------------------------------* 562 * usb2_pc_free_mem - free DMA memory 563 * 564 * This function is NULL safe. 565 *------------------------------------------------------------------------*/ 566void 567usb2_pc_free_mem(struct usb2_page_cache *pc) 568{ 569 if (pc && pc->buffer) { 570 571 bus_dmamap_unload(pc->tag, pc->map); 572 573 bus_dmamem_free(pc->tag, pc->buffer, pc->map); 574 575 pc->buffer = NULL; 576 } 577 return; 578} 579 580/*------------------------------------------------------------------------* 581 * usb2_pc_load_mem - load virtual memory into DMA 582 * 583 * Return values: 584 * 0: Success 585 * Else: Error 586 *------------------------------------------------------------------------*/ 587uint8_t 588usb2_pc_load_mem(struct usb2_page_cache *pc, uint32_t size, uint8_t sync) 589{ 590 /* setup page cache */ 591 pc->page_offset_buf = 0; 592 pc->page_offset_end = size; 593 pc->ismultiseg = 1; 594 595 mtx_assert(pc->tag_parent->mtx, MA_OWNED); 596 597 if (size > 0) { 598 if (sync) { 599 struct usb2_dma_parent_tag *uptag; 600 int err; 601 602 uptag = pc->tag_parent; 603 604 /*
|
605 * We have to unload the previous loaded DMA 606 * pages before trying to load a new one! 607 */ 608 bus_dmamap_unload(pc->tag, pc->map); 609 610 /* |
611 * Try to load memory into DMA. 612 */ 613 err = bus_dmamap_load( 614 pc->tag, pc->map, pc->buffer, size, 615 &usb2_pc_alloc_mem_cb, pc, BUS_DMA_WAITOK); 616 if (err == EINPROGRESS) { 617 usb2_cv_wait(uptag->cv, uptag->mtx); 618 err = 0; 619 } 620 if (err || uptag->dma_error) { 621 return (1); 622 } 623 } else { 624 625 /*
|
626 * We have to unload the previous loaded DMA 627 * pages before trying to load a new one! 628 */ 629 bus_dmamap_unload(pc->tag, pc->map); 630 631 /* |
632 * Try to load memory into DMA. The callback 633 * will be called in all cases: 634 */ 635 if (bus_dmamap_load( 636 pc->tag, pc->map, pc->buffer, size, 637 &usb2_pc_load_mem_cb, pc, BUS_DMA_WAITOK)) { 638 } 639 } 640 } else { 641 if (!sync) { 642 /* 643 * Call callback so that refcount is decremented 644 * properly: 645 */ 646 pc->tag_parent->dma_error = 0; 647 (pc->tag_parent->func) (pc->tag_parent); 648 } 649 } 650 return (0); 651} 652 653/*------------------------------------------------------------------------* 654 * usb2_pc_cpu_invalidate - invalidate CPU cache 655 *------------------------------------------------------------------------*/ 656void 657usb2_pc_cpu_invalidate(struct usb2_page_cache *pc) 658{
|
659 if (pc->page_offset_end == pc->page_offset_buf) { 660 /* nothing has been loaded into this page cache! */ 661 return; 662 } |
663 bus_dmamap_sync(pc->tag, pc->map, 664 BUS_DMASYNC_POSTWRITE | BUS_DMASYNC_POSTREAD); 665 return; 666} 667 668/*------------------------------------------------------------------------* 669 * usb2_pc_cpu_flush - flush CPU cache 670 *------------------------------------------------------------------------*/ 671void 672usb2_pc_cpu_flush(struct usb2_page_cache *pc) 673{
|
674 if (pc->page_offset_end == pc->page_offset_buf) { 675 /* nothing has been loaded into this page cache! */ 676 return; 677 } |
678 bus_dmamap_sync(pc->tag, pc->map, 679 BUS_DMASYNC_PREWRITE | BUS_DMASYNC_PREREAD); 680 return; 681} 682 683/*------------------------------------------------------------------------* 684 * usb2_pc_dmamap_create - create a DMA map 685 * 686 * Returns: 687 * 0: Success 688 * Else: Failure 689 *------------------------------------------------------------------------*/ 690uint8_t 691usb2_pc_dmamap_create(struct usb2_page_cache *pc, uint32_t size) 692{ 693 struct usb2_xfer_root *info; 694 struct usb2_dma_tag *utag; 695 696 /* get info */ 697 info = pc->tag_parent->info; 698 699 /* sanity check */ 700 if (info == NULL) { 701 goto error; 702 } 703 utag = usb2_dma_tag_find(pc->tag_parent, size, 1); 704 if (utag == NULL) { 705 goto error; 706 } 707 /* create DMA map */ 708 if (bus_dmamap_create(utag->tag, 0, &pc->map)) { 709 goto error; 710 } 711 pc->tag = utag->tag; 712 return 0; /* success */ 713 714error: 715 pc->map = NULL; 716 pc->tag = NULL; 717 return 1; /* failure */ 718} 719 720/*------------------------------------------------------------------------* 721 * usb2_pc_dmamap_destroy 722 * 723 * This function is NULL safe. 724 *------------------------------------------------------------------------*/ 725void 726usb2_pc_dmamap_destroy(struct usb2_page_cache *pc) 727{ 728 if (pc && pc->tag) { 729 bus_dmamap_destroy(pc->tag, pc->map); 730 pc->tag = NULL; 731 pc->map = NULL; 732 } 733 return; 734} 735 736#endif 737 738#ifdef __NetBSD__ 739 740/*------------------------------------------------------------------------* 741 * usb2_dma_tag_create - allocate a DMA tag 742 * 743 * NOTE: If the "align" parameter has a value of 1 the DMA-tag will 744 * allow multi-segment mappings. Else all mappings are single-segment. 745 *------------------------------------------------------------------------*/ 746static void 747usb2_dma_tag_create(struct usb2_dma_tag *udt, 748 uint32_t size, uint32_t align) 749{ 750 uint32_t nseg; 751 752 if (align == 1) { 753 nseg = (2 + (size / USB_PAGE_SIZE)); 754 } else { 755 nseg = 1; 756 } 757 758 udt->p_seg = malloc(nseg * sizeof(*(udt->p_seg)), 759 M_USB, M_WAITOK | M_ZERO); 760 761 if (udt->p_seg == NULL) { 762 return; 763 } 764 udt->tag = udt->tag_parent->tag; 765 udt->n_seg = nseg; 766 return; 767} 768 769/*------------------------------------------------------------------------* 770 * usb2_dma_tag_free - free a DMA tag 771 *------------------------------------------------------------------------*/ 772static void 773usb2_dma_tag_destroy(struct usb2_dma_tag *udt) 774{ 775 free(udt->p_seg, M_USB); 776 return; 777} 778 779/*------------------------------------------------------------------------* 780 * usb2_pc_common_mem_cb - BUS-DMA callback function 781 *------------------------------------------------------------------------*/ 782static void 783usb2_pc_common_mem_cb(struct usb2_page_cache *pc, bus_dma_segment_t *segs, 784 int nseg, int error, uint8_t isload, uint8_t dolock) 785{ 786 struct usb2_dma_parent_tag *uptag; 787 struct usb2_page *pg; 788 uint32_t rem; 789 uint8_t ext_seg; /* extend last segment */ 790 791 uptag = pc->tag_parent; 792 793 if (error) { 794 goto done; 795 } 796 pg = pc->page_start; 797 pg->physaddr = segs->ds_addr & ~(USB_PAGE_SIZE - 1); 798 rem = segs->ds_addr & (USB_PAGE_SIZE - 1); 799 pc->page_offset_buf = rem; 800 pc->page_offset_end += rem; 801 if (nseg < ((pc->page_offset_end + 802 (USB_PAGE_SIZE - 1)) / USB_PAGE_SIZE)) { 803 ext_seg = 1; 804 } else { 805 ext_seg = 0; 806 } 807 nseg--; 808 809 while (nseg > 0) { 810 nseg--; 811 segs++; 812 pg++; 813 pg->physaddr = segs->ds_addr & ~(USB_PAGE_SIZE - 1); 814 } 815 816 /* 817 * XXX The segments we get from BUS-DMA are not aligned, 818 * XXX so we need to extend the last segment if we are 819 * XXX unaligned and cross the segment boundary! 820 */ 821 if (ext_seg && pc->ismultiseg) { 822 (pg + 1)->physaddr = pg->physaddr + USB_PAGE_SIZE; 823 } 824done: 825 if (dolock) 826 mtx_lock(uptag->mtx); 827 828 uptag->dma_error = (error ? 1 : 0); 829 if (isload) { 830 (uptag->func) (uptag); 831 } 832 if (dolock) 833 mtx_unlock(uptag->mtx); 834 return; 835} 836 837/*------------------------------------------------------------------------* 838 * usb2_pc_alloc_mem - allocate DMA'able memory 839 * 840 * Returns: 841 * 0: Success 842 * Else: Failure 843 *------------------------------------------------------------------------*/ 844uint8_t 845usb2_pc_alloc_mem(struct usb2_page_cache *pc, struct usb2_page *pg, 846 uint32_t size, uint32_t align) 847{ 848 struct usb2_dma_parent_tag *uptag; 849 struct usb2_dma_tag *utag; 850 caddr_t ptr = NULL; 851 bus_dmamap_t map; 852 int seg_count; 853 854 uptag = pc->tag_parent; 855 856 if (align != 1) { 857 /* 858 * The alignment must be greater or equal to the 859 * "size" else the object can be split between two 860 * memory pages and we get a problem! 861 */ 862 while (align < size) { 863 align *= 2; 864 if (align == 0) { 865 goto done_5; 866 } 867 } 868 } 869 /* get the correct DMA tag */ 870 utag = usb2_dma_tag_find(pc->tag_parent, size, align); 871 if (utag == NULL) { 872 goto done_5; 873 } 874 if (bus_dmamem_alloc(utag->tag, size, align, 0, utag->p_seg, 875 utag->n_seg, &seg_count, BUS_DMA_WAITOK)) { 876 goto done_4; 877 } 878 if (bus_dmamem_map(utag->tag, utag->p_seg, seg_count, size, 879 &ptr, BUS_DMA_WAITOK | BUS_DMA_COHERENT)) { 880 goto done_3; 881 } 882 if (bus_dmamap_create(utag->tag, size, utag->n_seg, (align == 1) ? 883 USB_PAGE_SIZE : size, 0, BUS_DMA_WAITOK, &map)) { 884 goto done_2; 885 } 886 if (bus_dmamap_load(utag->tag, map, ptr, size, NULL, 887 BUS_DMA_WAITOK)) { 888 goto done_1; 889 } 890 pc->p_seg = malloc(seg_count * sizeof(*(pc->p_seg)), 891 M_USB, M_WAITOK | M_ZERO); 892 if (pc->p_seg == NULL) { 893 goto done_0; 894 } 895 /* store number if actual segments used */ 896 pc->n_seg = seg_count; 897 898 /* make a copy of the segments */ 899 bcopy(utag->p_seg, pc->p_seg, 900 seg_count * sizeof(*(pc->p_seg))); 901 902 /* setup page cache */ 903 pc->buffer = ptr; 904 pc->page_start = pg; 905 pc->page_offset_buf = 0; 906 pc->page_offset_end = size; 907 pc->map = map; 908 pc->tag = utag->tag; 909 pc->ismultiseg = (align == 1); 910 911 usb2_pc_common_mem_cb(pc, utag->p_seg, seg_count, 0, 0, 1); 912 913 bzero(ptr, size); 914 915 usb2_pc_cpu_flush(pc); 916 917 return (0); 918 919done_0: 920 bus_dmamap_unload(utag->tag, map); 921done_1: 922 bus_dmamap_destroy(utag->tag, map); 923done_2: 924 bus_dmamem_unmap(utag->tag, ptr, size); 925done_3: 926 bus_dmamem_free(utag->tag, utag->p_seg, seg_count); 927done_4: 928 /* utag is destroyed later */ 929done_5: 930 /* reset most of the page cache */ 931 pc->buffer = NULL; 932 pc->page_start = NULL; 933 pc->page_offset_buf = 0; 934 pc->page_offset_end = 0; 935 pc->map = NULL; 936 pc->tag = NULL; 937 pc->n_seg = 0; 938 pc->p_seg = NULL; 939 return (1); 940} 941 942/*------------------------------------------------------------------------* 943 * usb2_pc_free_mem - free DMA memory 944 * 945 * This function is NULL safe. 946 *------------------------------------------------------------------------*/ 947void 948usb2_pc_free_mem(struct usb2_page_cache *pc) 949{ 950 if (pc && pc->buffer) { 951 bus_dmamap_unload(pc->tag, pc->map); 952 bus_dmamap_destroy(pc->tag, pc->map); 953 bus_dmamem_unmap(pc->tag, pc->buffer, 954 pc->page_offset_end - pc->page_offset_buf); 955 bus_dmamem_free(pc->tag, pc->p_seg, pc->n_seg); 956 free(pc->p_seg, M_USB); 957 pc->buffer = NULL; 958 } 959 return; 960} 961 962/*------------------------------------------------------------------------* 963 * usb2_pc_load_mem - load virtual memory into DMA 964 * 965 * Return values: 966 * 0: Success 967 * Else: Error 968 *------------------------------------------------------------------------*/ 969uint8_t 970usb2_pc_load_mem(struct usb2_page_cache *pc, uint32_t size, uint8_t sync) 971{ 972 int error; 973 974 /* setup page cache */ 975 pc->page_offset_buf = 0; 976 pc->page_offset_end = size; 977 pc->ismultiseg = 1; 978 979 if (size > 0) { 980
|
981 /* 982 * We have to unload the previous loaded DMA 983 * pages before trying to load a new one! 984 */ 985 bus_dmamap_unload(pc->tag, pc->map); 986 |
987 /* try to load memory into DMA using using no wait option */ 988 if (bus_dmamap_load(pc->tag, pc->map, pc->buffer, 989 size, NULL, BUS_DMA_NOWAIT)) { 990 error = ENOMEM; 991 } else { 992 error = 0; 993 } 994 995 usb2_pc_common_mem_cb(pc, pc->map->dm_segs, 996 pc->map->dm_nsegs, error, !sync); 997 998 if (error) { 999 return (1); 1000 } 1001 } else { 1002 if (!sync) { 1003 /* 1004 * Call callback so that refcount is decremented 1005 * properly: 1006 */ 1007 pc->tag_parent->dma_error = 0; 1008 (pc->tag_parent->func) (pc->tag_parent); 1009 } 1010 } 1011 return (0); 1012} 1013 1014/*------------------------------------------------------------------------* 1015 * usb2_pc_cpu_invalidate - invalidate CPU cache 1016 *------------------------------------------------------------------------*/ 1017void 1018usb2_pc_cpu_invalidate(struct usb2_page_cache *pc) 1019{ 1020 uint32_t len; 1021 1022 len = pc->page_offset_end - pc->page_offset_buf; 1023
|
1024 if (len == 0) { 1025 /* nothing has been loaded into this page cache */ 1026 return; 1027 } |
1028 bus_dmamap_sync(pc->tag, pc->map, 0, len, 1029 BUS_DMASYNC_POSTWRITE | BUS_DMASYNC_POSTREAD); 1030 return; 1031} 1032 1033/*------------------------------------------------------------------------* 1034 * usb2_pc_cpu_flush - flush CPU cache 1035 *------------------------------------------------------------------------*/ 1036void 1037usb2_pc_cpu_flush(struct usb2_page_cache *pc) 1038{ 1039 uint32_t len; 1040 1041 len = pc->page_offset_end - pc->page_offset_buf; 1042
|
1043 if (len == 0) { 1044 /* nothing has been loaded into this page cache */ 1045 return; 1046 } |
1047 bus_dmamap_sync(pc->tag, pc->map, 0, len, 1048 BUS_DMASYNC_PREWRITE | BUS_DMASYNC_PREREAD); 1049 return; 1050} 1051 1052/*------------------------------------------------------------------------* 1053 * usb2_pc_dmamap_create - create a DMA map 1054 * 1055 * Returns: 1056 * 0: Success 1057 * Else: Failure 1058 *------------------------------------------------------------------------*/ 1059uint8_t 1060usb2_pc_dmamap_create(struct usb2_page_cache *pc, uint32_t size) 1061{ 1062 struct usb2_xfer_root *info; 1063 struct usb2_dma_tag *utag; 1064 1065 /* get info */ 1066 info = pc->tag_parent->info; 1067 1068 /* sanity check */ 1069 if (info == NULL) { 1070 goto error; 1071 } 1072 utag = usb2_dma_tag_find(pc->tag_parent, size, 1); 1073 if (utag == NULL) { 1074 goto error; 1075 } 1076 if (bus_dmamap_create(utag->tag, size, utag->n_seg, 1077 USB_PAGE_SIZE, 0, BUS_DMA_WAITOK, &pc->map)) { 1078 goto error; 1079 } 1080 pc->tag = utag->tag; 1081 pc->p_seg = utag->p_seg; 1082 pc->n_seg = utag->n_seg; 1083 return 0; /* success */ 1084 1085error: 1086 pc->map = NULL; 1087 pc->tag = NULL; 1088 pc->p_seg = NULL; 1089 pc->n_seg = 0; 1090 return 1; /* failure */ 1091} 1092 1093/*------------------------------------------------------------------------* 1094 * usb2_pc_dmamap_destroy 1095 * 1096 * This function is NULL safe. 1097 *------------------------------------------------------------------------*/ 1098void 1099usb2_pc_dmamap_destroy(struct usb2_page_cache *pc) 1100{ 1101 if (pc && pc->tag) { 1102 bus_dmamap_destroy(pc->tag, pc->map); 1103 pc->tag = NULL; 1104 pc->map = NULL; 1105 } 1106 return; 1107} 1108 1109#endif 1110 1111/*------------------------------------------------------------------------* 1112 * usb2_dma_tag_find - factored out code 1113 *------------------------------------------------------------------------*/ 1114struct usb2_dma_tag * 1115usb2_dma_tag_find(struct usb2_dma_parent_tag *udpt, 1116 uint32_t size, uint32_t align) 1117{ 1118 struct usb2_dma_tag *udt; 1119 uint8_t nudt; 1120 1121 USB_ASSERT(align > 0, ("Invalid parameter align = 0!\n")); 1122 USB_ASSERT(size > 0, ("Invalid parameter size = 0!\n")); 1123 1124 udt = udpt->utag_first; 1125 nudt = udpt->utag_max; 1126 1127 while (nudt--) { 1128 1129 if (udt->align == 0) { 1130 usb2_dma_tag_create(udt, size, align); 1131 if (udt->tag == NULL) { 1132 return (NULL); 1133 } 1134 udt->align = align; 1135 udt->size = size; 1136 return (udt); 1137 } 1138 if ((udt->align == align) && (udt->size == size)) { 1139 return (udt); 1140 } 1141 udt++; 1142 } 1143 return (NULL); 1144} 1145 1146/*------------------------------------------------------------------------* 1147 * usb2_dma_tag_setup - initialise USB DMA tags 1148 *------------------------------------------------------------------------*/ 1149void 1150usb2_dma_tag_setup(struct usb2_dma_parent_tag *udpt, 1151 struct usb2_dma_tag *udt, bus_dma_tag_t dmat, 1152 struct mtx *mtx, usb2_dma_callback_t *func, 1153 struct usb2_xfer_root *info, uint8_t ndmabits, 1154 uint8_t nudt) 1155{ 1156 bzero(udpt, sizeof(*udpt)); 1157 1158 /* sanity checking */ 1159 if ((nudt == 0) || 1160 (ndmabits == 0) || 1161 (mtx == NULL)) { 1162 /* something is corrupt */ 1163 return; 1164 } 1165#ifdef __FreeBSD__ 1166 /* initialise condition variable */ 1167 usb2_cv_init(udpt->cv, "USB DMA CV"); 1168#endif 1169 1170 /* store some information */ 1171 udpt->mtx = mtx; 1172 udpt->info = info; 1173 udpt->func = func; 1174 udpt->tag = dmat; 1175 udpt->utag_first = udt; 1176 udpt->utag_max = nudt; 1177 udpt->dma_bits = ndmabits; 1178 1179 while (nudt--) { 1180 bzero(udt, sizeof(*udt)); 1181 udt->tag_parent = udpt; 1182 udt++; 1183 } 1184 return; 1185} 1186 1187/*------------------------------------------------------------------------* 1188 * usb2_bus_tag_unsetup - factored out code 1189 *------------------------------------------------------------------------*/ 1190void 1191usb2_dma_tag_unsetup(struct usb2_dma_parent_tag *udpt) 1192{ 1193 struct usb2_dma_tag *udt; 1194 uint8_t nudt; 1195 1196 udt = udpt->utag_first; 1197 nudt = udpt->utag_max; 1198 1199 while (nudt--) { 1200 1201 if (udt->align) { 1202 /* destroy the USB DMA tag */ 1203 usb2_dma_tag_destroy(udt); 1204 udt->align = 0; 1205 } 1206 udt++; 1207 } 1208 1209 if (udpt->utag_max) { 1210#ifdef __FreeBSD__ 1211 /* destroy the condition variable */ 1212 usb2_cv_destroy(udpt->cv); 1213#endif 1214 } 1215 return; 1216} 1217 1218/*------------------------------------------------------------------------* 1219 * usb2_bdma_work_loop 1220 * 1221 * This function handles loading of virtual buffers into DMA and is 1222 * only called when "dma_refcount" is zero. 1223 *------------------------------------------------------------------------*/ 1224void 1225usb2_bdma_work_loop(struct usb2_xfer_queue *pq) 1226{ 1227 struct usb2_xfer_root *info; 1228 struct usb2_xfer *xfer; 1229 uint32_t nframes; 1230 1231 xfer = pq->curr; 1232 info = xfer->usb2_root; 1233 1234 mtx_assert(info->priv_mtx, MA_OWNED); 1235 1236 if (xfer->error) { 1237 /* some error happened */ 1238 USB_BUS_LOCK(xfer->udev->bus); 1239 usb2_transfer_done(xfer, 0); 1240 USB_BUS_UNLOCK(xfer->udev->bus); 1241 return; 1242 } 1243 if (!xfer->flags_int.bdma_setup) { 1244 struct usb2_page *pg; 1245 uint32_t frlength_0; 1246 uint8_t isread; 1247 1248 xfer->flags_int.bdma_setup = 1; 1249 1250 /* reset BUS-DMA load state */ 1251 1252 info->dma_error = 0; 1253 1254 if (xfer->flags_int.isochronous_xfr) { 1255 /* only one frame buffer */ 1256 nframes = 1; 1257 frlength_0 = xfer->sumlen; 1258 } else { 1259 /* can be multiple frame buffers */ 1260 nframes = xfer->nframes; 1261 frlength_0 = xfer->frlengths[0]; 1262 } 1263 1264 /* 1265 * Set DMA direction first. This is needed to 1266 * select the correct cache invalidate and cache 1267 * flush operations. 1268 */ 1269 isread = USB_GET_DATA_ISREAD(xfer); 1270 pg = xfer->dma_page_ptr; 1271 1272 if (xfer->flags_int.control_xfr && 1273 xfer->flags_int.control_hdr) { 1274 /* special case */ 1275 if (xfer->flags_int.usb2_mode == USB_MODE_DEVICE) { 1276 /* The device controller writes to memory */ 1277 xfer->frbuffers[0].isread = 1; 1278 } else { 1279 /* The host controller reads from memory */ 1280 xfer->frbuffers[0].isread = 0; 1281 } 1282 } else { 1283 /* default case */ 1284 xfer->frbuffers[0].isread = isread; 1285 } 1286 1287 /* 1288 * Setup the "page_start" pointer which points to an array of 1289 * USB pages where information about the physical address of a 1290 * page will be stored. Also initialise the "isread" field of 1291 * the USB page caches. 1292 */ 1293 xfer->frbuffers[0].page_start = pg; 1294 1295 info->dma_nframes = nframes; 1296 info->dma_currframe = 0; 1297 info->dma_frlength_0 = frlength_0; 1298 1299 pg += (frlength_0 / USB_PAGE_SIZE); 1300 pg += 2; 1301 1302 while (--nframes > 0) { 1303 xfer->frbuffers[nframes].isread = isread; 1304 xfer->frbuffers[nframes].page_start = pg; 1305 1306 pg += (xfer->frlengths[nframes] / USB_PAGE_SIZE); 1307 pg += 2; 1308 } 1309 1310 } 1311 if (info->dma_error) { 1312 USB_BUS_LOCK(xfer->udev->bus); 1313 usb2_transfer_done(xfer, USB_ERR_DMA_LOAD_FAILED); 1314 USB_BUS_UNLOCK(xfer->udev->bus); 1315 return; 1316 } 1317 if (info->dma_currframe != info->dma_nframes) { 1318 1319 if (info->dma_currframe == 0) { 1320 /* special case */ 1321 usb2_pc_load_mem(xfer->frbuffers, 1322 info->dma_frlength_0, 0); 1323 } else { 1324 /* default case */ 1325 nframes = info->dma_currframe; 1326 usb2_pc_load_mem(xfer->frbuffers + nframes, 1327 xfer->frlengths[nframes], 0); 1328 } 1329 1330 /* advance frame index */ 1331 info->dma_currframe++; 1332 1333 return; 1334 } 1335 /* go ahead */ 1336 usb2_bdma_pre_sync(xfer); 1337 1338 /* start loading next USB transfer, if any */ 1339 usb2_command_wrapper(pq, NULL); 1340 1341 /* finally start the hardware */ 1342 usb2_pipe_enter(xfer); 1343 1344 return; 1345} 1346 1347/*------------------------------------------------------------------------* 1348 * usb2_bdma_done_event 1349 * 1350 * This function is called when the BUS-DMA has loaded virtual memory 1351 * into DMA, if any. 1352 *------------------------------------------------------------------------*/ 1353void 1354usb2_bdma_done_event(struct usb2_dma_parent_tag *udpt) 1355{ 1356 struct usb2_xfer_root *info; 1357 1358 info = udpt->info; 1359 1360 mtx_assert(info->priv_mtx, MA_OWNED); 1361 1362 /* copy error */ 1363 info->dma_error = udpt->dma_error; 1364 1365 /* enter workloop again */ 1366 usb2_command_wrapper(&info->dma_q, 1367 info->dma_q.curr); 1368 return; 1369} 1370 1371/*------------------------------------------------------------------------* 1372 * usb2_bdma_pre_sync 1373 * 1374 * This function handles DMA synchronisation that must be done before 1375 * an USB transfer is started. 1376 *------------------------------------------------------------------------*/ 1377void 1378usb2_bdma_pre_sync(struct usb2_xfer *xfer) 1379{ 1380 struct usb2_page_cache *pc; 1381 uint32_t nframes; 1382 1383 if (xfer->flags_int.isochronous_xfr) { 1384 /* only one frame buffer */ 1385 nframes = 1; 1386 } else { 1387 /* can be multiple frame buffers */ 1388 nframes = xfer->nframes; 1389 } 1390 1391 pc = xfer->frbuffers; 1392 1393 while (nframes--) { 1394
|
1361 if (pc->page_offset_buf != pc->page_offset_end) {
1362 if (pc->isread) {
1363 usb2_pc_cpu_invalidate(pc);
1364 } else {
1365 usb2_pc_cpu_flush(pc);
1366 }
|
1395 if (pc->isread) { 1396 usb2_pc_cpu_invalidate(pc); 1397 } else { 1398 usb2_pc_cpu_flush(pc); |
1399 } 1400 pc++; 1401 } 1402 1403 return; 1404} 1405 1406/*------------------------------------------------------------------------* 1407 * usb2_bdma_post_sync 1408 * 1409 * This function handles DMA synchronisation that must be done after 1410 * an USB transfer is complete. 1411 *------------------------------------------------------------------------*/ 1412void 1413usb2_bdma_post_sync(struct usb2_xfer *xfer) 1414{ 1415 struct usb2_page_cache *pc; 1416 uint32_t nframes; 1417 1418 if (xfer->flags_int.isochronous_xfr) { 1419 /* only one frame buffer */ 1420 nframes = 1; 1421 } else { 1422 /* can be multiple frame buffers */ 1423 nframes = xfer->nframes; 1424 } 1425 1426 pc = xfer->frbuffers; 1427 1428 while (nframes--) {
|
1397
1398 if (pc->page_offset_buf != pc->page_offset_end) {
1399 if (pc->isread) {
1400 usb2_pc_cpu_invalidate(pc);
1401 }
|
1429 if (pc->isread) { 1430 usb2_pc_cpu_invalidate(pc); |
1431 } 1432 pc++; 1433 } 1434 return; 1435}
|