1/* $FreeBSD: head/sys/dev/usb2/core/usb2_busdma.c 184824 2008-11-10 20:54:31Z thompsa $ */
|
| 1/* $FreeBSD: head/sys/dev/usb2/core/usb2_busdma.c 185087 2008-11-19 08:56:35Z alfred $ */ |
2/*-
3 * Copyright (c) 2008 Hans Petter Selasky. All rights reserved.
4 *
5 * Redistribution and use in source and binary forms, with or without
6 * modification, are permitted provided that the following conditions
7 * are met:
8 * 1. Redistributions of source code must retain the above copyright
9 * notice, this list of conditions and the following disclaimer.
10 * 2. Redistributions in binary form must reproduce the above copyright
11 * notice, this list of conditions and the following disclaimer in the
12 * documentation and/or other materials provided with the distribution.
13 *
14 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
15 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
16 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
17 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
18 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
19 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
20 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
21 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
22 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
23 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
24 * SUCH DAMAGE.
25 */
26
27#include <dev/usb2/include/usb2_mfunc.h>
28#include <dev/usb2/include/usb2_error.h>
29#include <dev/usb2/include/usb2_standard.h>
30#include <dev/usb2/include/usb2_defs.h>
31
32#include <dev/usb2/core/usb2_core.h>
33#include <dev/usb2/core/usb2_busdma.h>
34#include <dev/usb2/core/usb2_process.h>
35#include <dev/usb2/core/usb2_transfer.h>
36#include <dev/usb2/core/usb2_device.h>
37#include <dev/usb2/core/usb2_util.h>
38
39#include <dev/usb2/controller/usb2_controller.h>
40#include <dev/usb2/controller/usb2_bus.h>
41
42static void usb2_dma_tag_create(struct usb2_dma_tag *udt, uint32_t size, uint32_t align);
43static void usb2_dma_tag_destroy(struct usb2_dma_tag *udt);
44
45#ifdef __FreeBSD__
46static void usb2_dma_lock_cb(void *arg, bus_dma_lock_op_t op);
47static int32_t usb2_m_copy_in_cb(void *arg, void *src, uint32_t count);
48static void usb2_pc_alloc_mem_cb(void *arg, bus_dma_segment_t *segs, int nseg, int error);
49static void usb2_pc_load_mem_cb(void *arg, bus_dma_segment_t *segs, int nseg, int error);
50static void usb2_pc_common_mem_cb(void *arg, bus_dma_segment_t *segs, int nseg, int error, uint8_t isload);
51
52#endif
53
54#ifdef __NetBSD__
55static int32_t usb2_m_copy_in_cb(void *arg, caddr_t src, uint32_t count);
56static void usb2_pc_common_mem_cb(struct usb2_page_cache *pc, bus_dma_segment_t *segs, int nseg, int error, uint8_t isload);
57
58#endif
59
60/*------------------------------------------------------------------------*
61 * usb2_get_page - lookup DMA-able memory for the given offset
62 *
63 * NOTE: Only call this function when the "page_cache" structure has
64 * been properly initialized !
65 *------------------------------------------------------------------------*/
66void
67usb2_get_page(struct usb2_page_cache *pc, uint32_t offset,
68 struct usb2_page_search *res)
69{
70 struct usb2_page *page;
71
72 if (pc->page_start) {
73
74 /* Case 1 - something has been loaded into DMA */
75
76 if (pc->buffer) {
77
78 /* Case 1a - Kernel Virtual Address */
79
80 res->buffer = USB_ADD_BYTES(pc->buffer, offset);
81 }
82 offset += pc->page_offset_buf;
83
84 /* compute destination page */
85
86 page = pc->page_start;
87
88 if (pc->ismultiseg) {
89
90 page += (offset / USB_PAGE_SIZE);
91
92 offset %= USB_PAGE_SIZE;
93
94 res->length = USB_PAGE_SIZE - offset;
95 res->physaddr = page->physaddr + offset;
96 } else {
97 res->length = 0 - 1;
98 res->physaddr = page->physaddr + offset;
99 }
100 if (!pc->buffer) {
101
102 /* Case 1b - Non Kernel Virtual Address */
103
104 res->buffer = USB_ADD_BYTES(page->buffer, offset);
105 }
106 } else {
107
108 /* Case 2 - Plain PIO */
109
110 res->buffer = USB_ADD_BYTES(pc->buffer, offset);
111 res->length = 0 - 1;
112 res->physaddr = 0;
113 }
114 return;
115}
116
117/*------------------------------------------------------------------------*
118 * usb2_copy_in - copy directly to DMA-able memory
119 *------------------------------------------------------------------------*/
120void
121usb2_copy_in(struct usb2_page_cache *cache, uint32_t offset,
122 const void *ptr, uint32_t len)
123{
124 struct usb2_page_search buf_res;
125
126 while (len != 0) {
127
128 usb2_get_page(cache, offset, &buf_res);
129
130 if (buf_res.length > len) {
131 buf_res.length = len;
132 }
133 bcopy(ptr, buf_res.buffer, buf_res.length);
134
135 offset += buf_res.length;
136 len -= buf_res.length;
137 ptr = USB_ADD_BYTES(ptr, buf_res.length);
138 }
139 return;
140}
141
142/*------------------------------------------------------------------------*
143 * usb2_copy_in_user - copy directly to DMA-able memory from userland
144 *
145 * Return values:
146 * 0: Success
147 * Else: Failure
148 *------------------------------------------------------------------------*/
149int
150usb2_copy_in_user(struct usb2_page_cache *cache, uint32_t offset,
151 const void *ptr, uint32_t len)
152{
153 struct usb2_page_search buf_res;
154 int error;
155
156 while (len != 0) {
157
158 usb2_get_page(cache, offset, &buf_res);
159
160 if (buf_res.length > len) {
161 buf_res.length = len;
162 }
163 error = copyin(ptr, buf_res.buffer, buf_res.length);
164 if (error)
165 return (error);
166
167 offset += buf_res.length;
168 len -= buf_res.length;
169 ptr = USB_ADD_BYTES(ptr, buf_res.length);
170 }
171 return (0); /* success */
172}
173
174/*------------------------------------------------------------------------*
175 * usb2_m_copy_in - copy a mbuf chain directly into DMA-able memory
176 *------------------------------------------------------------------------*/
177struct usb2_m_copy_in_arg {
178 struct usb2_page_cache *cache;
179 uint32_t dst_offset;
180};
181
182static int32_t
183#ifdef __FreeBSD__
184usb2_m_copy_in_cb(void *arg, void *src, uint32_t count)
185#else
186usb2_m_copy_in_cb(void *arg, caddr_t src, uint32_t count)
187#endif
188{
189 register struct usb2_m_copy_in_arg *ua = arg;
190
191 usb2_copy_in(ua->cache, ua->dst_offset, src, count);
192 ua->dst_offset += count;
193 return (0);
194}
195
196void
197usb2_m_copy_in(struct usb2_page_cache *cache, uint32_t dst_offset,
198 struct mbuf *m, uint32_t src_offset, uint32_t src_len)
199{
200 struct usb2_m_copy_in_arg arg = {cache, dst_offset};
201 register int error;
202
203 error = m_apply(m, src_offset, src_len, &usb2_m_copy_in_cb, &arg);
204 return;
205}
206
207/*------------------------------------------------------------------------*
208 * usb2_uiomove - factored out code
209 *------------------------------------------------------------------------*/
210int
211usb2_uiomove(struct usb2_page_cache *pc, struct uio *uio,
212 uint32_t pc_offset, uint32_t len)
213{
214 struct usb2_page_search res;
215 int error = 0;
216
217 while (len != 0) {
218
219 usb2_get_page(pc, pc_offset, &res);
220
221 if (res.length > len) {
222 res.length = len;
223 }
224 /*
225 * "uiomove()" can sleep so one needs to make a wrapper,
226 * exiting the mutex and checking things
227 */
228 error = uiomove(res.buffer, res.length, uio);
229
230 if (error) {
231 break;
232 }
233 pc_offset += res.length;
234 len -= res.length;
235 }
236 return (error);
237}
238
239/*------------------------------------------------------------------------*
240 * usb2_copy_out - copy directly from DMA-able memory
241 *------------------------------------------------------------------------*/
242void
243usb2_copy_out(struct usb2_page_cache *cache, uint32_t offset,
244 void *ptr, uint32_t len)
245{
246 struct usb2_page_search res;
247
248 while (len != 0) {
249
250 usb2_get_page(cache, offset, &res);
251
252 if (res.length > len) {
253 res.length = len;
254 }
255 bcopy(res.buffer, ptr, res.length);
256
257 offset += res.length;
258 len -= res.length;
259 ptr = USB_ADD_BYTES(ptr, res.length);
260 }
261 return;
262}
263
264/*------------------------------------------------------------------------*
265 * usb2_copy_out_user - copy directly from DMA-able memory to userland
266 *
267 * Return values:
268 * 0: Success
269 * Else: Failure
270 *------------------------------------------------------------------------*/
271int
272usb2_copy_out_user(struct usb2_page_cache *cache, uint32_t offset,
273 void *ptr, uint32_t len)
274{
275 struct usb2_page_search res;
276 int error;
277
278 while (len != 0) {
279
280 usb2_get_page(cache, offset, &res);
281
282 if (res.length > len) {
283 res.length = len;
284 }
285 error = copyout(res.buffer, ptr, res.length);
286 if (error)
287 return (error);
288
289 offset += res.length;
290 len -= res.length;
291 ptr = USB_ADD_BYTES(ptr, res.length);
292 }
293 return (0); /* success */
294}
295
296/*------------------------------------------------------------------------*
297 * usb2_bzero - zero DMA-able memory
298 *------------------------------------------------------------------------*/
299void
300usb2_bzero(struct usb2_page_cache *cache, uint32_t offset, uint32_t len)
301{
302 struct usb2_page_search res;
303
304 while (len != 0) {
305
306 usb2_get_page(cache, offset, &res);
307
308 if (res.length > len) {
309 res.length = len;
310 }
311 bzero(res.buffer, res.length);
312
313 offset += res.length;
314 len -= res.length;
315 }
316 return;
317}
318
319
320#ifdef __FreeBSD__
321
322/*------------------------------------------------------------------------*
323 * usb2_dma_lock_cb - dummy callback
324 *------------------------------------------------------------------------*/
325static void
326usb2_dma_lock_cb(void *arg, bus_dma_lock_op_t op)
327{
328 /* we use "mtx_owned()" instead of this function */
329 return;
330}
331
332/*------------------------------------------------------------------------*
333 * usb2_dma_tag_create - allocate a DMA tag
334 *
335 * NOTE: If the "align" parameter has a value of 1 the DMA-tag will
336 * allow multi-segment mappings. Else all mappings are single-segment.
337 *------------------------------------------------------------------------*/
338static void
339usb2_dma_tag_create(struct usb2_dma_tag *udt,
340 uint32_t size, uint32_t align)
341{
342 bus_dma_tag_t tag;
343
344 if (bus_dma_tag_create
345 ( /* parent */ udt->tag_parent->tag,
346 /* alignment */ align,
347 /* boundary */ USB_PAGE_SIZE,
348 /* lowaddr */ (2ULL << (udt->tag_parent->dma_bits - 1)) - 1,
349 /* highaddr */ BUS_SPACE_MAXADDR,
350 /* filter */ NULL,
351 /* filterarg */ NULL,
352 /* maxsize */ size,
353 /* nsegments */ (align == 1) ?
354 (2 + (size / USB_PAGE_SIZE)) : 1,
355 /* maxsegsz */ (align == 1) ?
356 USB_PAGE_SIZE : size,
357 /* flags */ 0,
358 /* lockfn */ &usb2_dma_lock_cb,
359 /* lockarg */ NULL,
360 &tag)) {
361 tag = NULL;
362 }
363 udt->tag = tag;
364 return;
365}
366
367/*------------------------------------------------------------------------*
368 * usb2_dma_tag_free - free a DMA tag
369 *------------------------------------------------------------------------*/
370static void
371usb2_dma_tag_destroy(struct usb2_dma_tag *udt)
372{
373 bus_dma_tag_destroy(udt->tag);
374 return;
375}
376
377/*------------------------------------------------------------------------*
378 * usb2_pc_alloc_mem_cb - BUS-DMA callback function
379 *------------------------------------------------------------------------*/
380static void
381usb2_pc_alloc_mem_cb(void *arg, bus_dma_segment_t *segs,
382 int nseg, int error)
383{
384 usb2_pc_common_mem_cb(arg, segs, nseg, error, 0);
385 return;
386}
387
388/*------------------------------------------------------------------------*
389 * usb2_pc_load_mem_cb - BUS-DMA callback function
390 *------------------------------------------------------------------------*/
391static void
392usb2_pc_load_mem_cb(void *arg, bus_dma_segment_t *segs,
393 int nseg, int error)
394{
395 usb2_pc_common_mem_cb(arg, segs, nseg, error, 1);
396 return;
397}
398
399/*------------------------------------------------------------------------*
400 * usb2_pc_common_mem_cb - BUS-DMA callback function
401 *------------------------------------------------------------------------*/
402static void
403usb2_pc_common_mem_cb(void *arg, bus_dma_segment_t *segs,
404 int nseg, int error, uint8_t isload)
405{
406 struct usb2_dma_parent_tag *uptag;
407 struct usb2_page_cache *pc;
408 struct usb2_page *pg;
409 uint32_t rem;
410 uint8_t owned;
411
412 pc = arg;
413 uptag = pc->tag_parent;
414
415 /*
416 * XXX There is sometimes recursive locking here.
417 * XXX We should try to find a better solution.
418 * XXX Until further the "owned" variable does
419 * XXX the trick.
420 */
421
422 if (error) {
423 goto done;
424 }
425 pg = pc->page_start;
426 pg->physaddr = segs->ds_addr & ~(USB_PAGE_SIZE - 1);
427 rem = segs->ds_addr & (USB_PAGE_SIZE - 1);
428 pc->page_offset_buf = rem;
429 pc->page_offset_end += rem;
430 nseg--;
431
432 while (nseg > 0) {
433 nseg--;
434 segs++;
435 pg++;
436 pg->physaddr = segs->ds_addr & ~(USB_PAGE_SIZE - 1);
437 }
438
439done:
440 owned = mtx_owned(uptag->mtx);
441 if (!owned)
442 mtx_lock(uptag->mtx);
443
444 uptag->dma_error = (error ? 1 : 0);
445 if (isload) {
446 (uptag->func) (uptag);
447 } else {
448 usb2_cv_broadcast(uptag->cv);
449 }
450 if (!owned)
451 mtx_unlock(uptag->mtx);
452 return;
453}
454
455/*------------------------------------------------------------------------*
456 * usb2_pc_alloc_mem - allocate DMA'able memory
457 *
458 * Returns:
459 * 0: Success
460 * Else: Failure
461 *------------------------------------------------------------------------*/
462uint8_t
463usb2_pc_alloc_mem(struct usb2_page_cache *pc, struct usb2_page *pg,
464 uint32_t size, uint32_t align)
465{
466 struct usb2_dma_parent_tag *uptag;
467 struct usb2_dma_tag *utag;
468 bus_dmamap_t map;
469 void *ptr;
470 int err;
471
472 uptag = pc->tag_parent;
473
474 if (align != 1) {
475 /*
476 * The alignment must be greater or equal to the
477 * "size" else the object can be split between two
478 * memory pages and we get a problem!
479 */
480 while (align < size) {
481 align *= 2;
482 if (align == 0) {
483 goto error;
484 }
485 }
486#if 1
487 /*
488 * XXX BUS-DMA workaround - FIXME later:
489 *
490 * We assume that that the aligment at this point of
491 * the code is greater than or equal to the size and
492 * less than two times the size, so that if we double
493 * the size, the size will be greater than the
494 * alignment.
495 *
496 * The bus-dma system has a check for "alignment"
497 * being less than "size". If that check fails we end
498 * up using contigmalloc which is page based even for
499 * small allocations. Try to avoid that to save
500 * memory, hence we sometimes to a large number of
501 * small allocations!
502 */
503 if (size <= (USB_PAGE_SIZE / 2)) {
504 size *= 2;
505 }
506#endif
507 }
508 /* get the correct DMA tag */
509 utag = usb2_dma_tag_find(uptag, size, align);
510 if (utag == NULL) {
511 goto error;
512 }
513 /* allocate memory */
514 if (bus_dmamem_alloc(
515 utag->tag, &ptr, (BUS_DMA_WAITOK | BUS_DMA_COHERENT), &map)) {
516 goto error;
517 }
518 /* setup page cache */
519 pc->buffer = ptr;
520 pc->page_start = pg;
521 pc->page_offset_buf = 0;
522 pc->page_offset_end = size;
523 pc->map = map;
524 pc->tag = utag->tag;
525 pc->ismultiseg = (align == 1);
526
527 mtx_lock(uptag->mtx);
528
529 /* load memory into DMA */
530 err = bus_dmamap_load(
531 utag->tag, map, ptr, size, &usb2_pc_alloc_mem_cb,
532 pc, (BUS_DMA_WAITOK | BUS_DMA_COHERENT));
533
534 if (err == EINPROGRESS) {
535 usb2_cv_wait(uptag->cv, uptag->mtx);
536 err = 0;
537 }
538 mtx_unlock(uptag->mtx);
539
540 if (err || uptag->dma_error) {
541 bus_dmamem_free(utag->tag, ptr, map);
542 goto error;
543 }
544 bzero(ptr, size);
545
546 usb2_pc_cpu_flush(pc);
547
548 return (0);
549
550error:
551 /* reset most of the page cache */
552 pc->buffer = NULL;
553 pc->page_start = NULL;
554 pc->page_offset_buf = 0;
555 pc->page_offset_end = 0;
556 pc->map = NULL;
557 pc->tag = NULL;
558 return (1);
559}
560
561/*------------------------------------------------------------------------*
562 * usb2_pc_free_mem - free DMA memory
563 *
564 * This function is NULL safe.
565 *------------------------------------------------------------------------*/
566void
567usb2_pc_free_mem(struct usb2_page_cache *pc)
568{
569 if (pc && pc->buffer) {
570
571 bus_dmamap_unload(pc->tag, pc->map);
572
573 bus_dmamem_free(pc->tag, pc->buffer, pc->map);
574
575 pc->buffer = NULL;
576 }
577 return;
578}
579
580/*------------------------------------------------------------------------*
581 * usb2_pc_load_mem - load virtual memory into DMA
582 *
583 * Return values:
584 * 0: Success
585 * Else: Error
586 *------------------------------------------------------------------------*/
587uint8_t
588usb2_pc_load_mem(struct usb2_page_cache *pc, uint32_t size, uint8_t sync)
589{
590 /* setup page cache */
591 pc->page_offset_buf = 0;
592 pc->page_offset_end = size;
593 pc->ismultiseg = 1;
594
595 mtx_assert(pc->tag_parent->mtx, MA_OWNED);
596
597 if (size > 0) {
598 if (sync) {
599 struct usb2_dma_parent_tag *uptag;
600 int err;
601
602 uptag = pc->tag_parent;
603
604 /*
|
605 * We have to unload the previous loaded DMA
606 * pages before trying to load a new one!
607 */
608 bus_dmamap_unload(pc->tag, pc->map);
609
610 /* |
611 * Try to load memory into DMA.
612 */
613 err = bus_dmamap_load(
614 pc->tag, pc->map, pc->buffer, size,
615 &usb2_pc_alloc_mem_cb, pc, BUS_DMA_WAITOK);
616 if (err == EINPROGRESS) {
617 usb2_cv_wait(uptag->cv, uptag->mtx);
618 err = 0;
619 }
620 if (err || uptag->dma_error) {
621 return (1);
622 }
623 } else {
624
625 /*
|
626 * We have to unload the previous loaded DMA
627 * pages before trying to load a new one!
628 */
629 bus_dmamap_unload(pc->tag, pc->map);
630
631 /* |
632 * Try to load memory into DMA. The callback
633 * will be called in all cases:
634 */
635 if (bus_dmamap_load(
636 pc->tag, pc->map, pc->buffer, size,
637 &usb2_pc_load_mem_cb, pc, BUS_DMA_WAITOK)) {
638 }
639 }
640 } else {
641 if (!sync) {
642 /*
643 * Call callback so that refcount is decremented
644 * properly:
645 */
646 pc->tag_parent->dma_error = 0;
647 (pc->tag_parent->func) (pc->tag_parent);
648 }
649 }
650 return (0);
651}
652
653/*------------------------------------------------------------------------*
654 * usb2_pc_cpu_invalidate - invalidate CPU cache
655 *------------------------------------------------------------------------*/
656void
657usb2_pc_cpu_invalidate(struct usb2_page_cache *pc)
658{
|
659 if (pc->page_offset_end == pc->page_offset_buf) {
660 /* nothing has been loaded into this page cache! */
661 return;
662 } |
663 bus_dmamap_sync(pc->tag, pc->map,
664 BUS_DMASYNC_POSTWRITE | BUS_DMASYNC_POSTREAD);
665 return;
666}
667
668/*------------------------------------------------------------------------*
669 * usb2_pc_cpu_flush - flush CPU cache
670 *------------------------------------------------------------------------*/
671void
672usb2_pc_cpu_flush(struct usb2_page_cache *pc)
673{
|
674 if (pc->page_offset_end == pc->page_offset_buf) {
675 /* nothing has been loaded into this page cache! */
676 return;
677 } |
678 bus_dmamap_sync(pc->tag, pc->map,
679 BUS_DMASYNC_PREWRITE | BUS_DMASYNC_PREREAD);
680 return;
681}
682
683/*------------------------------------------------------------------------*
684 * usb2_pc_dmamap_create - create a DMA map
685 *
686 * Returns:
687 * 0: Success
688 * Else: Failure
689 *------------------------------------------------------------------------*/
690uint8_t
691usb2_pc_dmamap_create(struct usb2_page_cache *pc, uint32_t size)
692{
693 struct usb2_xfer_root *info;
694 struct usb2_dma_tag *utag;
695
696 /* get info */
697 info = pc->tag_parent->info;
698
699 /* sanity check */
700 if (info == NULL) {
701 goto error;
702 }
703 utag = usb2_dma_tag_find(pc->tag_parent, size, 1);
704 if (utag == NULL) {
705 goto error;
706 }
707 /* create DMA map */
708 if (bus_dmamap_create(utag->tag, 0, &pc->map)) {
709 goto error;
710 }
711 pc->tag = utag->tag;
712 return 0; /* success */
713
714error:
715 pc->map = NULL;
716 pc->tag = NULL;
717 return 1; /* failure */
718}
719
720/*------------------------------------------------------------------------*
721 * usb2_pc_dmamap_destroy
722 *
723 * This function is NULL safe.
724 *------------------------------------------------------------------------*/
725void
726usb2_pc_dmamap_destroy(struct usb2_page_cache *pc)
727{
728 if (pc && pc->tag) {
729 bus_dmamap_destroy(pc->tag, pc->map);
730 pc->tag = NULL;
731 pc->map = NULL;
732 }
733 return;
734}
735
736#endif
737
738#ifdef __NetBSD__
739
740/*------------------------------------------------------------------------*
741 * usb2_dma_tag_create - allocate a DMA tag
742 *
743 * NOTE: If the "align" parameter has a value of 1 the DMA-tag will
744 * allow multi-segment mappings. Else all mappings are single-segment.
745 *------------------------------------------------------------------------*/
746static void
747usb2_dma_tag_create(struct usb2_dma_tag *udt,
748 uint32_t size, uint32_t align)
749{
750 uint32_t nseg;
751
752 if (align == 1) {
753 nseg = (2 + (size / USB_PAGE_SIZE));
754 } else {
755 nseg = 1;
756 }
757
758 udt->p_seg = malloc(nseg * sizeof(*(udt->p_seg)),
759 M_USB, M_WAITOK | M_ZERO);
760
761 if (udt->p_seg == NULL) {
762 return;
763 }
764 udt->tag = udt->tag_parent->tag;
765 udt->n_seg = nseg;
766 return;
767}
768
769/*------------------------------------------------------------------------*
770 * usb2_dma_tag_free - free a DMA tag
771 *------------------------------------------------------------------------*/
772static void
773usb2_dma_tag_destroy(struct usb2_dma_tag *udt)
774{
775 free(udt->p_seg, M_USB);
776 return;
777}
778
779/*------------------------------------------------------------------------*
780 * usb2_pc_common_mem_cb - BUS-DMA callback function
781 *------------------------------------------------------------------------*/
782static void
783usb2_pc_common_mem_cb(struct usb2_page_cache *pc, bus_dma_segment_t *segs,
784 int nseg, int error, uint8_t isload, uint8_t dolock)
785{
786 struct usb2_dma_parent_tag *uptag;
787 struct usb2_page *pg;
788 uint32_t rem;
789 uint8_t ext_seg; /* extend last segment */
790
791 uptag = pc->tag_parent;
792
793 if (error) {
794 goto done;
795 }
796 pg = pc->page_start;
797 pg->physaddr = segs->ds_addr & ~(USB_PAGE_SIZE - 1);
798 rem = segs->ds_addr & (USB_PAGE_SIZE - 1);
799 pc->page_offset_buf = rem;
800 pc->page_offset_end += rem;
801 if (nseg < ((pc->page_offset_end +
802 (USB_PAGE_SIZE - 1)) / USB_PAGE_SIZE)) {
803 ext_seg = 1;
804 } else {
805 ext_seg = 0;
806 }
807 nseg--;
808
809 while (nseg > 0) {
810 nseg--;
811 segs++;
812 pg++;
813 pg->physaddr = segs->ds_addr & ~(USB_PAGE_SIZE - 1);
814 }
815
816 /*
817 * XXX The segments we get from BUS-DMA are not aligned,
818 * XXX so we need to extend the last segment if we are
819 * XXX unaligned and cross the segment boundary!
820 */
821 if (ext_seg && pc->ismultiseg) {
822 (pg + 1)->physaddr = pg->physaddr + USB_PAGE_SIZE;
823 }
824done:
825 if (dolock)
826 mtx_lock(uptag->mtx);
827
828 uptag->dma_error = (error ? 1 : 0);
829 if (isload) {
830 (uptag->func) (uptag);
831 }
832 if (dolock)
833 mtx_unlock(uptag->mtx);
834 return;
835}
836
837/*------------------------------------------------------------------------*
838 * usb2_pc_alloc_mem - allocate DMA'able memory
839 *
840 * Returns:
841 * 0: Success
842 * Else: Failure
843 *------------------------------------------------------------------------*/
844uint8_t
845usb2_pc_alloc_mem(struct usb2_page_cache *pc, struct usb2_page *pg,
846 uint32_t size, uint32_t align)
847{
848 struct usb2_dma_parent_tag *uptag;
849 struct usb2_dma_tag *utag;
850 caddr_t ptr = NULL;
851 bus_dmamap_t map;
852 int seg_count;
853
854 uptag = pc->tag_parent;
855
856 if (align != 1) {
857 /*
858 * The alignment must be greater or equal to the
859 * "size" else the object can be split between two
860 * memory pages and we get a problem!
861 */
862 while (align < size) {
863 align *= 2;
864 if (align == 0) {
865 goto done_5;
866 }
867 }
868 }
869 /* get the correct DMA tag */
870 utag = usb2_dma_tag_find(pc->tag_parent, size, align);
871 if (utag == NULL) {
872 goto done_5;
873 }
874 if (bus_dmamem_alloc(utag->tag, size, align, 0, utag->p_seg,
875 utag->n_seg, &seg_count, BUS_DMA_WAITOK)) {
876 goto done_4;
877 }
878 if (bus_dmamem_map(utag->tag, utag->p_seg, seg_count, size,
879 &ptr, BUS_DMA_WAITOK | BUS_DMA_COHERENT)) {
880 goto done_3;
881 }
882 if (bus_dmamap_create(utag->tag, size, utag->n_seg, (align == 1) ?
883 USB_PAGE_SIZE : size, 0, BUS_DMA_WAITOK, &map)) {
884 goto done_2;
885 }
886 if (bus_dmamap_load(utag->tag, map, ptr, size, NULL,
887 BUS_DMA_WAITOK)) {
888 goto done_1;
889 }
890 pc->p_seg = malloc(seg_count * sizeof(*(pc->p_seg)),
891 M_USB, M_WAITOK | M_ZERO);
892 if (pc->p_seg == NULL) {
893 goto done_0;
894 }
895 /* store number if actual segments used */
896 pc->n_seg = seg_count;
897
898 /* make a copy of the segments */
899 bcopy(utag->p_seg, pc->p_seg,
900 seg_count * sizeof(*(pc->p_seg)));
901
902 /* setup page cache */
903 pc->buffer = ptr;
904 pc->page_start = pg;
905 pc->page_offset_buf = 0;
906 pc->page_offset_end = size;
907 pc->map = map;
908 pc->tag = utag->tag;
909 pc->ismultiseg = (align == 1);
910
911 usb2_pc_common_mem_cb(pc, utag->p_seg, seg_count, 0, 0, 1);
912
913 bzero(ptr, size);
914
915 usb2_pc_cpu_flush(pc);
916
917 return (0);
918
919done_0:
920 bus_dmamap_unload(utag->tag, map);
921done_1:
922 bus_dmamap_destroy(utag->tag, map);
923done_2:
924 bus_dmamem_unmap(utag->tag, ptr, size);
925done_3:
926 bus_dmamem_free(utag->tag, utag->p_seg, seg_count);
927done_4:
928 /* utag is destroyed later */
929done_5:
930 /* reset most of the page cache */
931 pc->buffer = NULL;
932 pc->page_start = NULL;
933 pc->page_offset_buf = 0;
934 pc->page_offset_end = 0;
935 pc->map = NULL;
936 pc->tag = NULL;
937 pc->n_seg = 0;
938 pc->p_seg = NULL;
939 return (1);
940}
941
942/*------------------------------------------------------------------------*
943 * usb2_pc_free_mem - free DMA memory
944 *
945 * This function is NULL safe.
946 *------------------------------------------------------------------------*/
947void
948usb2_pc_free_mem(struct usb2_page_cache *pc)
949{
950 if (pc && pc->buffer) {
951 bus_dmamap_unload(pc->tag, pc->map);
952 bus_dmamap_destroy(pc->tag, pc->map);
953 bus_dmamem_unmap(pc->tag, pc->buffer,
954 pc->page_offset_end - pc->page_offset_buf);
955 bus_dmamem_free(pc->tag, pc->p_seg, pc->n_seg);
956 free(pc->p_seg, M_USB);
957 pc->buffer = NULL;
958 }
959 return;
960}
961
962/*------------------------------------------------------------------------*
963 * usb2_pc_load_mem - load virtual memory into DMA
964 *
965 * Return values:
966 * 0: Success
967 * Else: Error
968 *------------------------------------------------------------------------*/
969uint8_t
970usb2_pc_load_mem(struct usb2_page_cache *pc, uint32_t size, uint8_t sync)
971{
972 int error;
973
974 /* setup page cache */
975 pc->page_offset_buf = 0;
976 pc->page_offset_end = size;
977 pc->ismultiseg = 1;
978
979 if (size > 0) {
980
|
981 /*
982 * We have to unload the previous loaded DMA
983 * pages before trying to load a new one!
984 */
985 bus_dmamap_unload(pc->tag, pc->map);
986 |
987 /* try to load memory into DMA using using no wait option */
988 if (bus_dmamap_load(pc->tag, pc->map, pc->buffer,
989 size, NULL, BUS_DMA_NOWAIT)) {
990 error = ENOMEM;
991 } else {
992 error = 0;
993 }
994
995 usb2_pc_common_mem_cb(pc, pc->map->dm_segs,
996 pc->map->dm_nsegs, error, !sync);
997
998 if (error) {
999 return (1);
1000 }
1001 } else {
1002 if (!sync) {
1003 /*
1004 * Call callback so that refcount is decremented
1005 * properly:
1006 */
1007 pc->tag_parent->dma_error = 0;
1008 (pc->tag_parent->func) (pc->tag_parent);
1009 }
1010 }
1011 return (0);
1012}
1013
1014/*------------------------------------------------------------------------*
1015 * usb2_pc_cpu_invalidate - invalidate CPU cache
1016 *------------------------------------------------------------------------*/
1017void
1018usb2_pc_cpu_invalidate(struct usb2_page_cache *pc)
1019{
1020 uint32_t len;
1021
1022 len = pc->page_offset_end - pc->page_offset_buf;
1023
|
1024 if (len == 0) {
1025 /* nothing has been loaded into this page cache */
1026 return;
1027 } |
1028 bus_dmamap_sync(pc->tag, pc->map, 0, len,
1029 BUS_DMASYNC_POSTWRITE | BUS_DMASYNC_POSTREAD);
1030 return;
1031}
1032
1033/*------------------------------------------------------------------------*
1034 * usb2_pc_cpu_flush - flush CPU cache
1035 *------------------------------------------------------------------------*/
1036void
1037usb2_pc_cpu_flush(struct usb2_page_cache *pc)
1038{
1039 uint32_t len;
1040
1041 len = pc->page_offset_end - pc->page_offset_buf;
1042
|
1043 if (len == 0) {
1044 /* nothing has been loaded into this page cache */
1045 return;
1046 } |
1047 bus_dmamap_sync(pc->tag, pc->map, 0, len,
1048 BUS_DMASYNC_PREWRITE | BUS_DMASYNC_PREREAD);
1049 return;
1050}
1051
1052/*------------------------------------------------------------------------*
1053 * usb2_pc_dmamap_create - create a DMA map
1054 *
1055 * Returns:
1056 * 0: Success
1057 * Else: Failure
1058 *------------------------------------------------------------------------*/
1059uint8_t
1060usb2_pc_dmamap_create(struct usb2_page_cache *pc, uint32_t size)
1061{
1062 struct usb2_xfer_root *info;
1063 struct usb2_dma_tag *utag;
1064
1065 /* get info */
1066 info = pc->tag_parent->info;
1067
1068 /* sanity check */
1069 if (info == NULL) {
1070 goto error;
1071 }
1072 utag = usb2_dma_tag_find(pc->tag_parent, size, 1);
1073 if (utag == NULL) {
1074 goto error;
1075 }
1076 if (bus_dmamap_create(utag->tag, size, utag->n_seg,
1077 USB_PAGE_SIZE, 0, BUS_DMA_WAITOK, &pc->map)) {
1078 goto error;
1079 }
1080 pc->tag = utag->tag;
1081 pc->p_seg = utag->p_seg;
1082 pc->n_seg = utag->n_seg;
1083 return 0; /* success */
1084
1085error:
1086 pc->map = NULL;
1087 pc->tag = NULL;
1088 pc->p_seg = NULL;
1089 pc->n_seg = 0;
1090 return 1; /* failure */
1091}
1092
1093/*------------------------------------------------------------------------*
1094 * usb2_pc_dmamap_destroy
1095 *
1096 * This function is NULL safe.
1097 *------------------------------------------------------------------------*/
1098void
1099usb2_pc_dmamap_destroy(struct usb2_page_cache *pc)
1100{
1101 if (pc && pc->tag) {
1102 bus_dmamap_destroy(pc->tag, pc->map);
1103 pc->tag = NULL;
1104 pc->map = NULL;
1105 }
1106 return;
1107}
1108
1109#endif
1110
1111/*------------------------------------------------------------------------*
1112 * usb2_dma_tag_find - factored out code
1113 *------------------------------------------------------------------------*/
1114struct usb2_dma_tag *
1115usb2_dma_tag_find(struct usb2_dma_parent_tag *udpt,
1116 uint32_t size, uint32_t align)
1117{
1118 struct usb2_dma_tag *udt;
1119 uint8_t nudt;
1120
1121 USB_ASSERT(align > 0, ("Invalid parameter align = 0!\n"));
1122 USB_ASSERT(size > 0, ("Invalid parameter size = 0!\n"));
1123
1124 udt = udpt->utag_first;
1125 nudt = udpt->utag_max;
1126
1127 while (nudt--) {
1128
1129 if (udt->align == 0) {
1130 usb2_dma_tag_create(udt, size, align);
1131 if (udt->tag == NULL) {
1132 return (NULL);
1133 }
1134 udt->align = align;
1135 udt->size = size;
1136 return (udt);
1137 }
1138 if ((udt->align == align) && (udt->size == size)) {
1139 return (udt);
1140 }
1141 udt++;
1142 }
1143 return (NULL);
1144}
1145
1146/*------------------------------------------------------------------------*
1147 * usb2_dma_tag_setup - initialise USB DMA tags
1148 *------------------------------------------------------------------------*/
1149void
1150usb2_dma_tag_setup(struct usb2_dma_parent_tag *udpt,
1151 struct usb2_dma_tag *udt, bus_dma_tag_t dmat,
1152 struct mtx *mtx, usb2_dma_callback_t *func,
1153 struct usb2_xfer_root *info, uint8_t ndmabits,
1154 uint8_t nudt)
1155{
1156 bzero(udpt, sizeof(*udpt));
1157
1158 /* sanity checking */
1159 if ((nudt == 0) ||
1160 (ndmabits == 0) ||
1161 (mtx == NULL)) {
1162 /* something is corrupt */
1163 return;
1164 }
1165#ifdef __FreeBSD__
1166 /* initialise condition variable */
1167 usb2_cv_init(udpt->cv, "USB DMA CV");
1168#endif
1169
1170 /* store some information */
1171 udpt->mtx = mtx;
1172 udpt->info = info;
1173 udpt->func = func;
1174 udpt->tag = dmat;
1175 udpt->utag_first = udt;
1176 udpt->utag_max = nudt;
1177 udpt->dma_bits = ndmabits;
1178
1179 while (nudt--) {
1180 bzero(udt, sizeof(*udt));
1181 udt->tag_parent = udpt;
1182 udt++;
1183 }
1184 return;
1185}
1186
1187/*------------------------------------------------------------------------*
1188 * usb2_bus_tag_unsetup - factored out code
1189 *------------------------------------------------------------------------*/
1190void
1191usb2_dma_tag_unsetup(struct usb2_dma_parent_tag *udpt)
1192{
1193 struct usb2_dma_tag *udt;
1194 uint8_t nudt;
1195
1196 udt = udpt->utag_first;
1197 nudt = udpt->utag_max;
1198
1199 while (nudt--) {
1200
1201 if (udt->align) {
1202 /* destroy the USB DMA tag */
1203 usb2_dma_tag_destroy(udt);
1204 udt->align = 0;
1205 }
1206 udt++;
1207 }
1208
1209 if (udpt->utag_max) {
1210#ifdef __FreeBSD__
1211 /* destroy the condition variable */
1212 usb2_cv_destroy(udpt->cv);
1213#endif
1214 }
1215 return;
1216}
1217
1218/*------------------------------------------------------------------------*
1219 * usb2_bdma_work_loop
1220 *
1221 * This function handles loading of virtual buffers into DMA and is
1222 * only called when "dma_refcount" is zero.
1223 *------------------------------------------------------------------------*/
1224void
1225usb2_bdma_work_loop(struct usb2_xfer_queue *pq)
1226{
1227 struct usb2_xfer_root *info;
1228 struct usb2_xfer *xfer;
1229 uint32_t nframes;
1230
1231 xfer = pq->curr;
1232 info = xfer->usb2_root;
1233
1234 mtx_assert(info->priv_mtx, MA_OWNED);
1235
1236 if (xfer->error) {
1237 /* some error happened */
1238 USB_BUS_LOCK(xfer->udev->bus);
1239 usb2_transfer_done(xfer, 0);
1240 USB_BUS_UNLOCK(xfer->udev->bus);
1241 return;
1242 }
1243 if (!xfer->flags_int.bdma_setup) {
1244 struct usb2_page *pg;
1245 uint32_t frlength_0;
1246 uint8_t isread;
1247
1248 xfer->flags_int.bdma_setup = 1;
1249
1250 /* reset BUS-DMA load state */
1251
1252 info->dma_error = 0;
1253
1254 if (xfer->flags_int.isochronous_xfr) {
1255 /* only one frame buffer */
1256 nframes = 1;
1257 frlength_0 = xfer->sumlen;
1258 } else {
1259 /* can be multiple frame buffers */
1260 nframes = xfer->nframes;
1261 frlength_0 = xfer->frlengths[0];
1262 }
1263
1264 /*
1265 * Set DMA direction first. This is needed to
1266 * select the correct cache invalidate and cache
1267 * flush operations.
1268 */
1269 isread = USB_GET_DATA_ISREAD(xfer);
1270 pg = xfer->dma_page_ptr;
1271
1272 if (xfer->flags_int.control_xfr &&
1273 xfer->flags_int.control_hdr) {
1274 /* special case */
1275 if (xfer->flags_int.usb2_mode == USB_MODE_DEVICE) {
1276 /* The device controller writes to memory */
1277 xfer->frbuffers[0].isread = 1;
1278 } else {
1279 /* The host controller reads from memory */
1280 xfer->frbuffers[0].isread = 0;
1281 }
1282 } else {
1283 /* default case */
1284 xfer->frbuffers[0].isread = isread;
1285 }
1286
1287 /*
1288 * Setup the "page_start" pointer which points to an array of
1289 * USB pages where information about the physical address of a
1290 * page will be stored. Also initialise the "isread" field of
1291 * the USB page caches.
1292 */
1293 xfer->frbuffers[0].page_start = pg;
1294
1295 info->dma_nframes = nframes;
1296 info->dma_currframe = 0;
1297 info->dma_frlength_0 = frlength_0;
1298
1299 pg += (frlength_0 / USB_PAGE_SIZE);
1300 pg += 2;
1301
1302 while (--nframes > 0) {
1303 xfer->frbuffers[nframes].isread = isread;
1304 xfer->frbuffers[nframes].page_start = pg;
1305
1306 pg += (xfer->frlengths[nframes] / USB_PAGE_SIZE);
1307 pg += 2;
1308 }
1309
1310 }
1311 if (info->dma_error) {
1312 USB_BUS_LOCK(xfer->udev->bus);
1313 usb2_transfer_done(xfer, USB_ERR_DMA_LOAD_FAILED);
1314 USB_BUS_UNLOCK(xfer->udev->bus);
1315 return;
1316 }
1317 if (info->dma_currframe != info->dma_nframes) {
1318
1319 if (info->dma_currframe == 0) {
1320 /* special case */
1321 usb2_pc_load_mem(xfer->frbuffers,
1322 info->dma_frlength_0, 0);
1323 } else {
1324 /* default case */
1325 nframes = info->dma_currframe;
1326 usb2_pc_load_mem(xfer->frbuffers + nframes,
1327 xfer->frlengths[nframes], 0);
1328 }
1329
1330 /* advance frame index */
1331 info->dma_currframe++;
1332
1333 return;
1334 }
1335 /* go ahead */
1336 usb2_bdma_pre_sync(xfer);
1337
1338 /* start loading next USB transfer, if any */
1339 usb2_command_wrapper(pq, NULL);
1340
1341 /* finally start the hardware */
1342 usb2_pipe_enter(xfer);
1343
1344 return;
1345}
1346
1347/*------------------------------------------------------------------------*
1348 * usb2_bdma_done_event
1349 *
1350 * This function is called when the BUS-DMA has loaded virtual memory
1351 * into DMA, if any.
1352 *------------------------------------------------------------------------*/
1353void
1354usb2_bdma_done_event(struct usb2_dma_parent_tag *udpt)
1355{
1356 struct usb2_xfer_root *info;
1357
1358 info = udpt->info;
1359
1360 mtx_assert(info->priv_mtx, MA_OWNED);
1361
1362 /* copy error */
1363 info->dma_error = udpt->dma_error;
1364
1365 /* enter workloop again */
1366 usb2_command_wrapper(&info->dma_q,
1367 info->dma_q.curr);
1368 return;
1369}
1370
1371/*------------------------------------------------------------------------*
1372 * usb2_bdma_pre_sync
1373 *
1374 * This function handles DMA synchronisation that must be done before
1375 * an USB transfer is started.
1376 *------------------------------------------------------------------------*/
1377void
1378usb2_bdma_pre_sync(struct usb2_xfer *xfer)
1379{
1380 struct usb2_page_cache *pc;
1381 uint32_t nframes;
1382
1383 if (xfer->flags_int.isochronous_xfr) {
1384 /* only one frame buffer */
1385 nframes = 1;
1386 } else {
1387 /* can be multiple frame buffers */
1388 nframes = xfer->nframes;
1389 }
1390
1391 pc = xfer->frbuffers;
1392
1393 while (nframes--) {
1394
|
1361 if (pc->page_offset_buf != pc->page_offset_end) {
1362 if (pc->isread) {
1363 usb2_pc_cpu_invalidate(pc);
1364 } else {
1365 usb2_pc_cpu_flush(pc);
1366 }
|
1395 if (pc->isread) {
1396 usb2_pc_cpu_invalidate(pc);
1397 } else {
1398 usb2_pc_cpu_flush(pc); |
1399 }
1400 pc++;
1401 }
1402
1403 return;
1404}
1405
1406/*------------------------------------------------------------------------*
1407 * usb2_bdma_post_sync
1408 *
1409 * This function handles DMA synchronisation that must be done after
1410 * an USB transfer is complete.
1411 *------------------------------------------------------------------------*/
1412void
1413usb2_bdma_post_sync(struct usb2_xfer *xfer)
1414{
1415 struct usb2_page_cache *pc;
1416 uint32_t nframes;
1417
1418 if (xfer->flags_int.isochronous_xfr) {
1419 /* only one frame buffer */
1420 nframes = 1;
1421 } else {
1422 /* can be multiple frame buffers */
1423 nframes = xfer->nframes;
1424 }
1425
1426 pc = xfer->frbuffers;
1427
1428 while (nframes--) {
|
1397
1398 if (pc->page_offset_buf != pc->page_offset_end) {
1399 if (pc->isread) {
1400 usb2_pc_cpu_invalidate(pc);
1401 }
|
1429 if (pc->isread) {
1430 usb2_pc_cpu_invalidate(pc); |
1431 }
1432 pc++;
1433 }
1434 return;
1435}
|