36
37static void usb2_dma_tag_create(struct usb2_dma_tag *udt, uint32_t size, uint32_t align);
38static void usb2_dma_tag_destroy(struct usb2_dma_tag *udt);
39
40#ifdef __FreeBSD__
41static void usb2_dma_lock_cb(void *arg, bus_dma_lock_op_t op);
42static int32_t usb2_m_copy_in_cb(void *arg, void *src, uint32_t count);
43static void usb2_pc_alloc_mem_cb(void *arg, bus_dma_segment_t *segs, int nseg, int error);
44static void usb2_pc_load_mem_cb(void *arg, bus_dma_segment_t *segs, int nseg, int error);
45static void usb2_pc_common_mem_cb(void *arg, bus_dma_segment_t *segs, int nseg, int error, uint8_t isload);
46
47#endif
48
49#ifdef __NetBSD__
50static int32_t usb2_m_copy_in_cb(void *arg, caddr_t src, uint32_t count);
51static void usb2_pc_common_mem_cb(struct usb2_page_cache *pc, bus_dma_segment_t *segs, int nseg, int error, uint8_t isload);
52
53#endif
54
55/*------------------------------------------------------------------------*
56 * usb2_get_page - lookup DMA-able memory for the given offset
57 *
58 * NOTE: Only call this function when the "page_cache" structure has
59 * been properly initialized !
60 *------------------------------------------------------------------------*/
61void
62usb2_get_page(struct usb2_page_cache *pc, uint32_t offset,
63 struct usb2_page_search *res)
64{
65 struct usb2_page *page;
66
67 if (pc->page_start) {
68
69 /* Case 1 - something has been loaded into DMA */
70
71 if (pc->buffer) {
72
73 /* Case 1a - Kernel Virtual Address */
74
75 res->buffer = USB_ADD_BYTES(pc->buffer, offset);
76 }
77 offset += pc->page_offset_buf;
78
79 /* compute destination page */
80
81 page = pc->page_start;
82
83 if (pc->ismultiseg) {
84
85 page += (offset / USB_PAGE_SIZE);
86
87 offset %= USB_PAGE_SIZE;
88
89 res->length = USB_PAGE_SIZE - offset;
90 res->physaddr = page->physaddr + offset;
91 } else {
92 res->length = 0 - 1;
93 res->physaddr = page->physaddr + offset;
94 }
95 if (!pc->buffer) {
96
97 /* Case 1b - Non Kernel Virtual Address */
98
99 res->buffer = USB_ADD_BYTES(page->buffer, offset);
100 }
101 } else {
102
103 /* Case 2 - Plain PIO */
104
105 res->buffer = USB_ADD_BYTES(pc->buffer, offset);
106 res->length = 0 - 1;
107 res->physaddr = 0;
108 }
109 return;
110}
111
112/*------------------------------------------------------------------------*
113 * usb2_copy_in - copy directly to DMA-able memory
114 *------------------------------------------------------------------------*/
115void
116usb2_copy_in(struct usb2_page_cache *cache, uint32_t offset,
117 const void *ptr, uint32_t len)
118{
119 struct usb2_page_search buf_res;
120
121 while (len != 0) {
122
123 usb2_get_page(cache, offset, &buf_res);
124
125 if (buf_res.length > len) {
126 buf_res.length = len;
127 }
128 bcopy(ptr, buf_res.buffer, buf_res.length);
129
130 offset += buf_res.length;
131 len -= buf_res.length;
132 ptr = USB_ADD_BYTES(ptr, buf_res.length);
133 }
134 return;
135}
136
137/*------------------------------------------------------------------------*
138 * usb2_copy_in_user - copy directly to DMA-able memory from userland
139 *
140 * Return values:
141 * 0: Success
142 * Else: Failure
143 *------------------------------------------------------------------------*/
144int
145usb2_copy_in_user(struct usb2_page_cache *cache, uint32_t offset,
146 const void *ptr, uint32_t len)
147{
148 struct usb2_page_search buf_res;
149 int error;
150
151 while (len != 0) {
152
153 usb2_get_page(cache, offset, &buf_res);
154
155 if (buf_res.length > len) {
156 buf_res.length = len;
157 }
158 error = copyin(ptr, buf_res.buffer, buf_res.length);
159 if (error)
160 return (error);
161
162 offset += buf_res.length;
163 len -= buf_res.length;
164 ptr = USB_ADD_BYTES(ptr, buf_res.length);
165 }
166 return (0); /* success */
167}
168
169/*------------------------------------------------------------------------*
170 * usb2_m_copy_in - copy a mbuf chain directly into DMA-able memory
171 *------------------------------------------------------------------------*/
172struct usb2_m_copy_in_arg {
173 struct usb2_page_cache *cache;
174 uint32_t dst_offset;
175};
176
177static int32_t
178#ifdef __FreeBSD__
179usb2_m_copy_in_cb(void *arg, void *src, uint32_t count)
180#else
181usb2_m_copy_in_cb(void *arg, caddr_t src, uint32_t count)
182#endif
183{
184 register struct usb2_m_copy_in_arg *ua = arg;
185
186 usb2_copy_in(ua->cache, ua->dst_offset, src, count);
187 ua->dst_offset += count;
188 return (0);
189}
190
191void
192usb2_m_copy_in(struct usb2_page_cache *cache, uint32_t dst_offset,
193 struct mbuf *m, uint32_t src_offset, uint32_t src_len)
194{
195 struct usb2_m_copy_in_arg arg = {cache, dst_offset};
196 register int error;
197
198 error = m_apply(m, src_offset, src_len, &usb2_m_copy_in_cb, &arg);
199 return;
200}
201
202/*------------------------------------------------------------------------*
203 * usb2_uiomove - factored out code
204 *------------------------------------------------------------------------*/
205int
206usb2_uiomove(struct usb2_page_cache *pc, struct uio *uio,
207 uint32_t pc_offset, uint32_t len)
208{
209 struct usb2_page_search res;
210 int error = 0;
211
212 while (len != 0) {
213
214 usb2_get_page(pc, pc_offset, &res);
215
216 if (res.length > len) {
217 res.length = len;
218 }
219 /*
220 * "uiomove()" can sleep so one needs to make a wrapper,
221 * exiting the mutex and checking things
222 */
223 error = uiomove(res.buffer, res.length, uio);
224
225 if (error) {
226 break;
227 }
228 pc_offset += res.length;
229 len -= res.length;
230 }
231 return (error);
232}
233
234/*------------------------------------------------------------------------*
235 * usb2_copy_out - copy directly from DMA-able memory
236 *------------------------------------------------------------------------*/
237void
238usb2_copy_out(struct usb2_page_cache *cache, uint32_t offset,
239 void *ptr, uint32_t len)
240{
241 struct usb2_page_search res;
242
243 while (len != 0) {
244
245 usb2_get_page(cache, offset, &res);
246
247 if (res.length > len) {
248 res.length = len;
249 }
250 bcopy(res.buffer, ptr, res.length);
251
252 offset += res.length;
253 len -= res.length;
254 ptr = USB_ADD_BYTES(ptr, res.length);
255 }
256 return;
257}
258
259/*------------------------------------------------------------------------*
260 * usb2_copy_out_user - copy directly from DMA-able memory to userland
261 *
262 * Return values:
263 * 0: Success
264 * Else: Failure
265 *------------------------------------------------------------------------*/
266int
267usb2_copy_out_user(struct usb2_page_cache *cache, uint32_t offset,
268 void *ptr, uint32_t len)
269{
270 struct usb2_page_search res;
271 int error;
272
273 while (len != 0) {
274
275 usb2_get_page(cache, offset, &res);
276
277 if (res.length > len) {
278 res.length = len;
279 }
280 error = copyout(res.buffer, ptr, res.length);
281 if (error)
282 return (error);
283
284 offset += res.length;
285 len -= res.length;
286 ptr = USB_ADD_BYTES(ptr, res.length);
287 }
288 return (0); /* success */
289}
290
291/*------------------------------------------------------------------------*
292 * usb2_bzero - zero DMA-able memory
293 *------------------------------------------------------------------------*/
294void
295usb2_bzero(struct usb2_page_cache *cache, uint32_t offset, uint32_t len)
296{
297 struct usb2_page_search res;
298
299 while (len != 0) {
300
301 usb2_get_page(cache, offset, &res);
302
303 if (res.length > len) {
304 res.length = len;
305 }
306 bzero(res.buffer, res.length);
307
308 offset += res.length;
309 len -= res.length;
310 }
311 return;
312}
313
314
315#ifdef __FreeBSD__
316
317/*------------------------------------------------------------------------*
318 * usb2_dma_lock_cb - dummy callback
319 *------------------------------------------------------------------------*/
320static void
321usb2_dma_lock_cb(void *arg, bus_dma_lock_op_t op)
322{
323 /* we use "mtx_owned()" instead of this function */
324 return;
325}
326
327/*------------------------------------------------------------------------*
328 * usb2_dma_tag_create - allocate a DMA tag
329 *
330 * NOTE: If the "align" parameter has a value of 1 the DMA-tag will
331 * allow multi-segment mappings. Else all mappings are single-segment.
332 *------------------------------------------------------------------------*/
333static void
334usb2_dma_tag_create(struct usb2_dma_tag *udt,
335 uint32_t size, uint32_t align)
336{
337 bus_dma_tag_t tag;
338
339 if (bus_dma_tag_create
340 ( /* parent */ udt->tag_parent->tag,
341 /* alignment */ align,
342 /* boundary */ USB_PAGE_SIZE,
343 /* lowaddr */ (2ULL << (udt->tag_parent->dma_bits - 1)) - 1,
344 /* highaddr */ BUS_SPACE_MAXADDR,
345 /* filter */ NULL,
346 /* filterarg */ NULL,
347 /* maxsize */ size,
348 /* nsegments */ (align == 1) ?
349 (2 + (size / USB_PAGE_SIZE)) : 1,
350 /* maxsegsz */ (align == 1) ?
351 USB_PAGE_SIZE : size,
352 /* flags */ 0,
353 /* lockfn */ &usb2_dma_lock_cb,
354 /* lockarg */ NULL,
355 &tag)) {
356 tag = NULL;
357 }
358 udt->tag = tag;
359 return;
360}
361
362/*------------------------------------------------------------------------*
363 * usb2_dma_tag_free - free a DMA tag
364 *------------------------------------------------------------------------*/
365static void
366usb2_dma_tag_destroy(struct usb2_dma_tag *udt)
367{
368 bus_dma_tag_destroy(udt->tag);
369 return;
370}
371
372/*------------------------------------------------------------------------*
373 * usb2_pc_alloc_mem_cb - BUS-DMA callback function
374 *------------------------------------------------------------------------*/
375static void
376usb2_pc_alloc_mem_cb(void *arg, bus_dma_segment_t *segs,
377 int nseg, int error)
378{
379 usb2_pc_common_mem_cb(arg, segs, nseg, error, 0);
380 return;
381}
382
383/*------------------------------------------------------------------------*
384 * usb2_pc_load_mem_cb - BUS-DMA callback function
385 *------------------------------------------------------------------------*/
386static void
387usb2_pc_load_mem_cb(void *arg, bus_dma_segment_t *segs,
388 int nseg, int error)
389{
390 usb2_pc_common_mem_cb(arg, segs, nseg, error, 1);
391 return;
392}
393
394/*------------------------------------------------------------------------*
395 * usb2_pc_common_mem_cb - BUS-DMA callback function
396 *------------------------------------------------------------------------*/
397static void
398usb2_pc_common_mem_cb(void *arg, bus_dma_segment_t *segs,
399 int nseg, int error, uint8_t isload)
400{
401 struct usb2_dma_parent_tag *uptag;
402 struct usb2_page_cache *pc;
403 struct usb2_page *pg;
404 uint32_t rem;
405 uint8_t owned;
406
407 pc = arg;
408 uptag = pc->tag_parent;
409
410 /*
411 * XXX There is sometimes recursive locking here.
412 * XXX We should try to find a better solution.
413 * XXX Until further the "owned" variable does
414 * XXX the trick.
415 */
416
417 if (error) {
418 goto done;
419 }
420 pg = pc->page_start;
421 pg->physaddr = segs->ds_addr & ~(USB_PAGE_SIZE - 1);
422 rem = segs->ds_addr & (USB_PAGE_SIZE - 1);
423 pc->page_offset_buf = rem;
424 pc->page_offset_end += rem;
425 nseg--;
426
427 while (nseg > 0) {
428 nseg--;
429 segs++;
430 pg++;
431 pg->physaddr = segs->ds_addr & ~(USB_PAGE_SIZE - 1);
432 }
433
434done:
435 owned = mtx_owned(uptag->mtx);
436 if (!owned)
437 mtx_lock(uptag->mtx);
438
439 uptag->dma_error = (error ? 1 : 0);
440 if (isload) {
441 (uptag->func) (uptag);
442 } else {
443 usb2_cv_broadcast(uptag->cv);
444 }
445 if (!owned)
446 mtx_unlock(uptag->mtx);
447 return;
448}
449
450/*------------------------------------------------------------------------*
451 * usb2_pc_alloc_mem - allocate DMA'able memory
452 *
453 * Returns:
454 * 0: Success
455 * Else: Failure
456 *------------------------------------------------------------------------*/
457uint8_t
458usb2_pc_alloc_mem(struct usb2_page_cache *pc, struct usb2_page *pg,
459 uint32_t size, uint32_t align)
460{
461 struct usb2_dma_parent_tag *uptag;
462 struct usb2_dma_tag *utag;
463 bus_dmamap_t map;
464 void *ptr;
465 int err;
466
467 uptag = pc->tag_parent;
468
469 if (align != 1) {
470 /*
471 * The alignment must be greater or equal to the
472 * "size" else the object can be split between two
473 * memory pages and we get a problem!
474 */
475 while (align < size) {
476 align *= 2;
477 if (align == 0) {
478 goto error;
479 }
480 }
481#if 1
482 /*
483 * XXX BUS-DMA workaround - FIXME later:
484 *
485 * We assume that that the aligment at this point of
486 * the code is greater than or equal to the size and
487 * less than two times the size, so that if we double
488 * the size, the size will be greater than the
489 * alignment.
490 *
491 * The bus-dma system has a check for "alignment"
492 * being less than "size". If that check fails we end
493 * up using contigmalloc which is page based even for
494 * small allocations. Try to avoid that to save
495 * memory, hence we sometimes to a large number of
496 * small allocations!
497 */
498 if (size <= (USB_PAGE_SIZE / 2)) {
499 size *= 2;
500 }
501#endif
502 }
503 /* get the correct DMA tag */
504 utag = usb2_dma_tag_find(uptag, size, align);
505 if (utag == NULL) {
506 goto error;
507 }
508 /* allocate memory */
509 if (bus_dmamem_alloc(
510 utag->tag, &ptr, (BUS_DMA_WAITOK | BUS_DMA_COHERENT), &map)) {
511 goto error;
512 }
513 /* setup page cache */
514 pc->buffer = ptr;
515 pc->page_start = pg;
516 pc->page_offset_buf = 0;
517 pc->page_offset_end = size;
518 pc->map = map;
519 pc->tag = utag->tag;
520 pc->ismultiseg = (align == 1);
521
522 mtx_lock(uptag->mtx);
523
524 /* load memory into DMA */
525 err = bus_dmamap_load(
526 utag->tag, map, ptr, size, &usb2_pc_alloc_mem_cb,
527 pc, (BUS_DMA_WAITOK | BUS_DMA_COHERENT));
528
529 if (err == EINPROGRESS) {
530 usb2_cv_wait(uptag->cv, uptag->mtx);
531 err = 0;
532 }
533 mtx_unlock(uptag->mtx);
534
535 if (err || uptag->dma_error) {
536 bus_dmamem_free(utag->tag, ptr, map);
537 goto error;
538 }
539 bzero(ptr, size);
540
541 usb2_pc_cpu_flush(pc);
542
543 return (0);
544
545error:
546 /* reset most of the page cache */
547 pc->buffer = NULL;
548 pc->page_start = NULL;
549 pc->page_offset_buf = 0;
550 pc->page_offset_end = 0;
551 pc->map = NULL;
552 pc->tag = NULL;
553 return (1);
554}
555
556/*------------------------------------------------------------------------*
557 * usb2_pc_free_mem - free DMA memory
558 *
559 * This function is NULL safe.
560 *------------------------------------------------------------------------*/
561void
562usb2_pc_free_mem(struct usb2_page_cache *pc)
563{
564 if (pc && pc->buffer) {
565
566 bus_dmamap_unload(pc->tag, pc->map);
567
568 bus_dmamem_free(pc->tag, pc->buffer, pc->map);
569
570 pc->buffer = NULL;
571 }
572 return;
573}
574
575/*------------------------------------------------------------------------*
576 * usb2_pc_load_mem - load virtual memory into DMA
577 *
578 * Return values:
579 * 0: Success
580 * Else: Error
581 *------------------------------------------------------------------------*/
582uint8_t
583usb2_pc_load_mem(struct usb2_page_cache *pc, uint32_t size, uint8_t sync)
584{
585 /* setup page cache */
586 pc->page_offset_buf = 0;
587 pc->page_offset_end = size;
588 pc->ismultiseg = 1;
589
590 mtx_assert(pc->tag_parent->mtx, MA_OWNED);
591
592 if (size > 0) {
593 if (sync) {
594 struct usb2_dma_parent_tag *uptag;
595 int err;
596
597 uptag = pc->tag_parent;
598
599 /*
600 * Try to load memory into DMA.
601 */
602 err = bus_dmamap_load(
603 pc->tag, pc->map, pc->buffer, size,
604 &usb2_pc_alloc_mem_cb, pc, BUS_DMA_WAITOK);
605 if (err == EINPROGRESS) {
606 usb2_cv_wait(uptag->cv, uptag->mtx);
607 err = 0;
608 }
609 if (err || uptag->dma_error) {
610 return (1);
611 }
612 } else {
613
614 /*
615 * Try to load memory into DMA. The callback
616 * will be called in all cases:
617 */
618 if (bus_dmamap_load(
619 pc->tag, pc->map, pc->buffer, size,
620 &usb2_pc_load_mem_cb, pc, BUS_DMA_WAITOK)) {
621 }
622 }
623 } else {
624 if (!sync) {
625 /*
626 * Call callback so that refcount is decremented
627 * properly:
628 */
629 pc->tag_parent->dma_error = 0;
630 (pc->tag_parent->func) (pc->tag_parent);
631 }
632 }
633 return (0);
634}
635
636/*------------------------------------------------------------------------*
637 * usb2_pc_cpu_invalidate - invalidate CPU cache
638 *------------------------------------------------------------------------*/
639void
640usb2_pc_cpu_invalidate(struct usb2_page_cache *pc)
641{
642 bus_dmamap_sync(pc->tag, pc->map,
643 BUS_DMASYNC_POSTWRITE | BUS_DMASYNC_POSTREAD);
644 return;
645}
646
647/*------------------------------------------------------------------------*
648 * usb2_pc_cpu_flush - flush CPU cache
649 *------------------------------------------------------------------------*/
650void
651usb2_pc_cpu_flush(struct usb2_page_cache *pc)
652{
653 bus_dmamap_sync(pc->tag, pc->map,
654 BUS_DMASYNC_PREWRITE | BUS_DMASYNC_PREREAD);
655 return;
656}
657
658/*------------------------------------------------------------------------*
659 * usb2_pc_dmamap_create - create a DMA map
660 *
661 * Returns:
662 * 0: Success
663 * Else: Failure
664 *------------------------------------------------------------------------*/
665uint8_t
666usb2_pc_dmamap_create(struct usb2_page_cache *pc, uint32_t size)
667{
668 struct usb2_xfer_root *info;
669 struct usb2_dma_tag *utag;
670
671 /* get info */
672 info = pc->tag_parent->info;
673
674 /* sanity check */
675 if (info == NULL) {
676 goto error;
677 }
678 utag = usb2_dma_tag_find(pc->tag_parent, size, 1);
679 if (utag == NULL) {
680 goto error;
681 }
682 /* create DMA map */
683 if (bus_dmamap_create(utag->tag, 0, &pc->map)) {
684 goto error;
685 }
686 pc->tag = utag->tag;
687 return 0; /* success */
688
689error:
690 pc->map = NULL;
691 pc->tag = NULL;
692 return 1; /* failure */
693}
694
695/*------------------------------------------------------------------------*
696 * usb2_pc_dmamap_destroy
697 *
698 * This function is NULL safe.
699 *------------------------------------------------------------------------*/
700void
701usb2_pc_dmamap_destroy(struct usb2_page_cache *pc)
702{
703 if (pc && pc->tag) {
704 bus_dmamap_destroy(pc->tag, pc->map);
705 pc->tag = NULL;
706 pc->map = NULL;
707 }
708 return;
709}
710
711#endif
712
713#ifdef __NetBSD__
714
715/*------------------------------------------------------------------------*
716 * usb2_dma_tag_create - allocate a DMA tag
717 *
718 * NOTE: If the "align" parameter has a value of 1 the DMA-tag will
719 * allow multi-segment mappings. Else all mappings are single-segment.
720 *------------------------------------------------------------------------*/
721static void
722usb2_dma_tag_create(struct usb2_dma_tag *udt,
723 uint32_t size, uint32_t align)
724{
725 uint32_t nseg;
726
727 if (align == 1) {
728 nseg = (2 + (size / USB_PAGE_SIZE));
729 } else {
730 nseg = 1;
731 }
732
733 udt->p_seg = malloc(nseg * sizeof(*(udt->p_seg)),
734 M_USB, M_WAITOK | M_ZERO);
735
736 if (udt->p_seg == NULL) {
737 return;
738 }
739 udt->tag = udt->tag_parent->tag;
740 udt->n_seg = nseg;
741 return;
742}
743
744/*------------------------------------------------------------------------*
745 * usb2_dma_tag_free - free a DMA tag
746 *------------------------------------------------------------------------*/
747static void
748usb2_dma_tag_destroy(struct usb2_dma_tag *udt)
749{
750 free(udt->p_seg, M_USB);
751 return;
752}
753
754/*------------------------------------------------------------------------*
755 * usb2_pc_common_mem_cb - BUS-DMA callback function
756 *------------------------------------------------------------------------*/
757static void
758usb2_pc_common_mem_cb(struct usb2_page_cache *pc, bus_dma_segment_t *segs,
759 int nseg, int error, uint8_t isload, uint8_t dolock)
760{
761 struct usb2_dma_parent_tag *uptag;
762 struct usb2_page *pg;
763 uint32_t rem;
764 uint8_t ext_seg; /* extend last segment */
765
766 uptag = pc->tag_parent;
767
768 if (error) {
769 goto done;
770 }
771 pg = pc->page_start;
772 pg->physaddr = segs->ds_addr & ~(USB_PAGE_SIZE - 1);
773 rem = segs->ds_addr & (USB_PAGE_SIZE - 1);
774 pc->page_offset_buf = rem;
775 pc->page_offset_end += rem;
776 if (nseg < ((pc->page_offset_end +
777 (USB_PAGE_SIZE - 1)) / USB_PAGE_SIZE)) {
778 ext_seg = 1;
779 } else {
780 ext_seg = 0;
781 }
782 nseg--;
783
784 while (nseg > 0) {
785 nseg--;
786 segs++;
787 pg++;
788 pg->physaddr = segs->ds_addr & ~(USB_PAGE_SIZE - 1);
789 }
790
791 /*
792 * XXX The segments we get from BUS-DMA are not aligned,
793 * XXX so we need to extend the last segment if we are
794 * XXX unaligned and cross the segment boundary!
795 */
796 if (ext_seg && pc->ismultiseg) {
797 (pg + 1)->physaddr = pg->physaddr + USB_PAGE_SIZE;
798 }
799done:
800 if (dolock)
801 mtx_lock(uptag->mtx);
802
803 uptag->dma_error = (error ? 1 : 0);
804 if (isload) {
805 (uptag->func) (uptag);
806 }
807 if (dolock)
808 mtx_unlock(uptag->mtx);
809 return;
810}
811
812/*------------------------------------------------------------------------*
813 * usb2_pc_alloc_mem - allocate DMA'able memory
814 *
815 * Returns:
816 * 0: Success
817 * Else: Failure
818 *------------------------------------------------------------------------*/
819uint8_t
820usb2_pc_alloc_mem(struct usb2_page_cache *pc, struct usb2_page *pg,
821 uint32_t size, uint32_t align)
822{
823 struct usb2_dma_parent_tag *uptag;
824 struct usb2_dma_tag *utag;
825 caddr_t ptr = NULL;
826 bus_dmamap_t map;
827 int seg_count;
828
829 uptag = pc->tag_parent;
830
831 if (align != 1) {
832 /*
833 * The alignment must be greater or equal to the
834 * "size" else the object can be split between two
835 * memory pages and we get a problem!
836 */
837 while (align < size) {
838 align *= 2;
839 if (align == 0) {
840 goto done_5;
841 }
842 }
843 }
844 /* get the correct DMA tag */
845 utag = usb2_dma_tag_find(pc->tag_parent, size, align);
846 if (utag == NULL) {
847 goto done_5;
848 }
849 if (bus_dmamem_alloc(utag->tag, size, align, 0, utag->p_seg,
850 utag->n_seg, &seg_count, BUS_DMA_WAITOK)) {
851 goto done_4;
852 }
853 if (bus_dmamem_map(utag->tag, utag->p_seg, seg_count, size,
854 &ptr, BUS_DMA_WAITOK | BUS_DMA_COHERENT)) {
855 goto done_3;
856 }
857 if (bus_dmamap_create(utag->tag, size, utag->n_seg, (align == 1) ?
858 USB_PAGE_SIZE : size, 0, BUS_DMA_WAITOK, &map)) {
859 goto done_2;
860 }
861 if (bus_dmamap_load(utag->tag, map, ptr, size, NULL,
862 BUS_DMA_WAITOK)) {
863 goto done_1;
864 }
865 pc->p_seg = malloc(seg_count * sizeof(*(pc->p_seg)),
866 M_USB, M_WAITOK | M_ZERO);
867 if (pc->p_seg == NULL) {
868 goto done_0;
869 }
870 /* store number if actual segments used */
871 pc->n_seg = seg_count;
872
873 /* make a copy of the segments */
874 bcopy(utag->p_seg, pc->p_seg,
875 seg_count * sizeof(*(pc->p_seg)));
876
877 /* setup page cache */
878 pc->buffer = ptr;
879 pc->page_start = pg;
880 pc->page_offset_buf = 0;
881 pc->page_offset_end = size;
882 pc->map = map;
883 pc->tag = utag->tag;
884 pc->ismultiseg = (align == 1);
885
886 usb2_pc_common_mem_cb(pc, utag->p_seg, seg_count, 0, 0, 1);
887
888 bzero(ptr, size);
889
890 usb2_pc_cpu_flush(pc);
891
892 return (0);
893
894done_0:
895 bus_dmamap_unload(utag->tag, map);
896done_1:
897 bus_dmamap_destroy(utag->tag, map);
898done_2:
899 bus_dmamem_unmap(utag->tag, ptr, size);
900done_3:
901 bus_dmamem_free(utag->tag, utag->p_seg, seg_count);
902done_4:
903 /* utag is destroyed later */
904done_5:
905 /* reset most of the page cache */
906 pc->buffer = NULL;
907 pc->page_start = NULL;
908 pc->page_offset_buf = 0;
909 pc->page_offset_end = 0;
910 pc->map = NULL;
911 pc->tag = NULL;
912 pc->n_seg = 0;
913 pc->p_seg = NULL;
914 return (1);
915}
916
917/*------------------------------------------------------------------------*
918 * usb2_pc_free_mem - free DMA memory
919 *
920 * This function is NULL safe.
921 *------------------------------------------------------------------------*/
922void
923usb2_pc_free_mem(struct usb2_page_cache *pc)
924{
925 if (pc && pc->buffer) {
926 bus_dmamap_unload(pc->tag, pc->map);
927 bus_dmamap_destroy(pc->tag, pc->map);
928 bus_dmamem_unmap(pc->tag, pc->buffer,
929 pc->page_offset_end - pc->page_offset_buf);
930 bus_dmamem_free(pc->tag, pc->p_seg, pc->n_seg);
931 free(pc->p_seg, M_USB);
932 pc->buffer = NULL;
933 }
934 return;
935}
936
937/*------------------------------------------------------------------------*
938 * usb2_pc_load_mem - load virtual memory into DMA
939 *
940 * Return values:
941 * 0: Success
942 * Else: Error
943 *------------------------------------------------------------------------*/
944uint8_t
945usb2_pc_load_mem(struct usb2_page_cache *pc, uint32_t size, uint8_t sync)
946{
947 int error;
948
949 /* setup page cache */
950 pc->page_offset_buf = 0;
951 pc->page_offset_end = size;
952 pc->ismultiseg = 1;
953
954 if (size > 0) {
955
956 /* try to load memory into DMA using using no wait option */
957 if (bus_dmamap_load(pc->tag, pc->map, pc->buffer,
958 size, NULL, BUS_DMA_NOWAIT)) {
959 error = ENOMEM;
960 } else {
961 error = 0;
962 }
963
964 usb2_pc_common_mem_cb(pc, pc->map->dm_segs,
965 pc->map->dm_nsegs, error, !sync);
966
967 if (error) {
968 return (1);
969 }
970 } else {
971 if (!sync) {
972 /*
973 * Call callback so that refcount is decremented
974 * properly:
975 */
976 pc->tag_parent->dma_error = 0;
977 (pc->tag_parent->func) (pc->tag_parent);
978 }
979 }
980 return (0);
981}
982
983/*------------------------------------------------------------------------*
984 * usb2_pc_cpu_invalidate - invalidate CPU cache
985 *------------------------------------------------------------------------*/
986void
987usb2_pc_cpu_invalidate(struct usb2_page_cache *pc)
988{
989 uint32_t len;
990
991 len = pc->page_offset_end - pc->page_offset_buf;
992
993 bus_dmamap_sync(pc->tag, pc->map, 0, len,
994 BUS_DMASYNC_POSTWRITE | BUS_DMASYNC_POSTREAD);
995 return;
996}
997
998/*------------------------------------------------------------------------*
999 * usb2_pc_cpu_flush - flush CPU cache
1000 *------------------------------------------------------------------------*/
1001void
1002usb2_pc_cpu_flush(struct usb2_page_cache *pc)
1003{
1004 uint32_t len;
1005
1006 len = pc->page_offset_end - pc->page_offset_buf;
1007
1008 bus_dmamap_sync(pc->tag, pc->map, 0, len,
1009 BUS_DMASYNC_PREWRITE | BUS_DMASYNC_PREREAD);
1010 return;
1011}
1012
1013/*------------------------------------------------------------------------*
1014 * usb2_pc_dmamap_create - create a DMA map
1015 *
1016 * Returns:
1017 * 0: Success
1018 * Else: Failure
1019 *------------------------------------------------------------------------*/
1020uint8_t
1021usb2_pc_dmamap_create(struct usb2_page_cache *pc, uint32_t size)
1022{
1023 struct usb2_xfer_root *info;
1024 struct usb2_dma_tag *utag;
1025
1026 /* get info */
1027 info = pc->tag_parent->info;
1028
1029 /* sanity check */
1030 if (info == NULL) {
1031 goto error;
1032 }
1033 utag = usb2_dma_tag_find(pc->tag_parent, size, 1);
1034 if (utag == NULL) {
1035 goto error;
1036 }
1037 if (bus_dmamap_create(utag->tag, size, utag->n_seg,
1038 USB_PAGE_SIZE, 0, BUS_DMA_WAITOK, &pc->map)) {
1039 goto error;
1040 }
1041 pc->tag = utag->tag;
1042 pc->p_seg = utag->p_seg;
1043 pc->n_seg = utag->n_seg;
1044 return 0; /* success */
1045
1046error:
1047 pc->map = NULL;
1048 pc->tag = NULL;
1049 pc->p_seg = NULL;
1050 pc->n_seg = 0;
1051 return 1; /* failure */
1052}
1053
1054/*------------------------------------------------------------------------*
1055 * usb2_pc_dmamap_destroy
1056 *
1057 * This function is NULL safe.
1058 *------------------------------------------------------------------------*/
1059void
1060usb2_pc_dmamap_destroy(struct usb2_page_cache *pc)
1061{
1062 if (pc && pc->tag) {
1063 bus_dmamap_destroy(pc->tag, pc->map);
1064 pc->tag = NULL;
1065 pc->map = NULL;
1066 }
1067 return;
1068}
1069
1070#endif
1071
1072/*------------------------------------------------------------------------*
1073 * usb2_dma_tag_find - factored out code
1074 *------------------------------------------------------------------------*/
1075struct usb2_dma_tag *
1076usb2_dma_tag_find(struct usb2_dma_parent_tag *udpt,
1077 uint32_t size, uint32_t align)
1078{
1079 struct usb2_dma_tag *udt;
1080 uint8_t nudt;
1081
1082 USB_ASSERT(align > 0, ("Invalid parameter align = 0!\n"));
1083 USB_ASSERT(size > 0, ("Invalid parameter size = 0!\n"));
1084
1085 udt = udpt->utag_first;
1086 nudt = udpt->utag_max;
1087
1088 while (nudt--) {
1089
1090 if (udt->align == 0) {
1091 usb2_dma_tag_create(udt, size, align);
1092 if (udt->tag == NULL) {
1093 return (NULL);
1094 }
1095 udt->align = align;
1096 udt->size = size;
1097 return (udt);
1098 }
1099 if ((udt->align == align) && (udt->size == size)) {
1100 return (udt);
1101 }
1102 udt++;
1103 }
1104 return (NULL);
1105}
1106
1107/*------------------------------------------------------------------------*
1108 * usb2_dma_tag_setup - initialise USB DMA tags
1109 *------------------------------------------------------------------------*/
1110void
1111usb2_dma_tag_setup(struct usb2_dma_parent_tag *udpt,
1112 struct usb2_dma_tag *udt, bus_dma_tag_t dmat,
1113 struct mtx *mtx, usb2_dma_callback_t *func,
1114 struct usb2_xfer_root *info, uint8_t ndmabits,
1115 uint8_t nudt)
1116{
1117 bzero(udpt, sizeof(*udpt));
1118
1119 /* sanity checking */
1120 if ((nudt == 0) ||
1121 (ndmabits == 0) ||
1122 (mtx == NULL)) {
1123 /* something is corrupt */
1124 return;
1125 }
1126#ifdef __FreeBSD__
1127 /* initialise condition variable */
1128 usb2_cv_init(udpt->cv, "USB DMA CV");
1129#endif
1130
1131 /* store some information */
1132 udpt->mtx = mtx;
1133 udpt->info = info;
1134 udpt->func = func;
1135 udpt->tag = dmat;
1136 udpt->utag_first = udt;
1137 udpt->utag_max = nudt;
1138 udpt->dma_bits = ndmabits;
1139
1140 while (nudt--) {
1141 bzero(udt, sizeof(*udt));
1142 udt->tag_parent = udpt;
1143 udt++;
1144 }
1145 return;
1146}
1147
1148/*------------------------------------------------------------------------*
1149 * usb2_bus_tag_unsetup - factored out code
1150 *------------------------------------------------------------------------*/
1151void
1152usb2_dma_tag_unsetup(struct usb2_dma_parent_tag *udpt)
1153{
1154 struct usb2_dma_tag *udt;
1155 uint8_t nudt;
1156
1157 udt = udpt->utag_first;
1158 nudt = udpt->utag_max;
1159
1160 while (nudt--) {
1161
1162 if (udt->align) {
1163 /* destroy the USB DMA tag */
1164 usb2_dma_tag_destroy(udt);
1165 udt->align = 0;
1166 }
1167 udt++;
1168 }
1169
1170 if (udpt->utag_max) {
1171#ifdef __FreeBSD__
1172 /* destroy the condition variable */
1173 usb2_cv_destroy(udpt->cv);
1174#endif
1175 }
1176 return;
1177}
1178
1179/*------------------------------------------------------------------------*
1180 * usb2_bdma_work_loop
1181 *
1182 * This function handles loading of virtual buffers into DMA and is
1183 * only called when "dma_refcount" is zero.
1184 *------------------------------------------------------------------------*/
1185void
1186usb2_bdma_work_loop(struct usb2_xfer_queue *pq)
1187{
1188 struct usb2_xfer_root *info;
1189 struct usb2_xfer *xfer;
1190 uint32_t nframes;
1191
1192 xfer = pq->curr;
1193 info = xfer->usb2_root;
1194
1195 mtx_assert(info->priv_mtx, MA_OWNED);
1196
1197 if (xfer->error) {
1198 /* some error happened */
| 41
42static void usb2_dma_tag_create(struct usb2_dma_tag *udt, uint32_t size, uint32_t align);
43static void usb2_dma_tag_destroy(struct usb2_dma_tag *udt);
44
45#ifdef __FreeBSD__
46static void usb2_dma_lock_cb(void *arg, bus_dma_lock_op_t op);
47static int32_t usb2_m_copy_in_cb(void *arg, void *src, uint32_t count);
48static void usb2_pc_alloc_mem_cb(void *arg, bus_dma_segment_t *segs, int nseg, int error);
49static void usb2_pc_load_mem_cb(void *arg, bus_dma_segment_t *segs, int nseg, int error);
50static void usb2_pc_common_mem_cb(void *arg, bus_dma_segment_t *segs, int nseg, int error, uint8_t isload);
51
52#endif
53
54#ifdef __NetBSD__
55static int32_t usb2_m_copy_in_cb(void *arg, caddr_t src, uint32_t count);
56static void usb2_pc_common_mem_cb(struct usb2_page_cache *pc, bus_dma_segment_t *segs, int nseg, int error, uint8_t isload);
57
58#endif
59
60/*------------------------------------------------------------------------*
61 * usb2_get_page - lookup DMA-able memory for the given offset
62 *
63 * NOTE: Only call this function when the "page_cache" structure has
64 * been properly initialized !
65 *------------------------------------------------------------------------*/
66void
67usb2_get_page(struct usb2_page_cache *pc, uint32_t offset,
68 struct usb2_page_search *res)
69{
70 struct usb2_page *page;
71
72 if (pc->page_start) {
73
74 /* Case 1 - something has been loaded into DMA */
75
76 if (pc->buffer) {
77
78 /* Case 1a - Kernel Virtual Address */
79
80 res->buffer = USB_ADD_BYTES(pc->buffer, offset);
81 }
82 offset += pc->page_offset_buf;
83
84 /* compute destination page */
85
86 page = pc->page_start;
87
88 if (pc->ismultiseg) {
89
90 page += (offset / USB_PAGE_SIZE);
91
92 offset %= USB_PAGE_SIZE;
93
94 res->length = USB_PAGE_SIZE - offset;
95 res->physaddr = page->physaddr + offset;
96 } else {
97 res->length = 0 - 1;
98 res->physaddr = page->physaddr + offset;
99 }
100 if (!pc->buffer) {
101
102 /* Case 1b - Non Kernel Virtual Address */
103
104 res->buffer = USB_ADD_BYTES(page->buffer, offset);
105 }
106 } else {
107
108 /* Case 2 - Plain PIO */
109
110 res->buffer = USB_ADD_BYTES(pc->buffer, offset);
111 res->length = 0 - 1;
112 res->physaddr = 0;
113 }
114 return;
115}
116
117/*------------------------------------------------------------------------*
118 * usb2_copy_in - copy directly to DMA-able memory
119 *------------------------------------------------------------------------*/
120void
121usb2_copy_in(struct usb2_page_cache *cache, uint32_t offset,
122 const void *ptr, uint32_t len)
123{
124 struct usb2_page_search buf_res;
125
126 while (len != 0) {
127
128 usb2_get_page(cache, offset, &buf_res);
129
130 if (buf_res.length > len) {
131 buf_res.length = len;
132 }
133 bcopy(ptr, buf_res.buffer, buf_res.length);
134
135 offset += buf_res.length;
136 len -= buf_res.length;
137 ptr = USB_ADD_BYTES(ptr, buf_res.length);
138 }
139 return;
140}
141
142/*------------------------------------------------------------------------*
143 * usb2_copy_in_user - copy directly to DMA-able memory from userland
144 *
145 * Return values:
146 * 0: Success
147 * Else: Failure
148 *------------------------------------------------------------------------*/
149int
150usb2_copy_in_user(struct usb2_page_cache *cache, uint32_t offset,
151 const void *ptr, uint32_t len)
152{
153 struct usb2_page_search buf_res;
154 int error;
155
156 while (len != 0) {
157
158 usb2_get_page(cache, offset, &buf_res);
159
160 if (buf_res.length > len) {
161 buf_res.length = len;
162 }
163 error = copyin(ptr, buf_res.buffer, buf_res.length);
164 if (error)
165 return (error);
166
167 offset += buf_res.length;
168 len -= buf_res.length;
169 ptr = USB_ADD_BYTES(ptr, buf_res.length);
170 }
171 return (0); /* success */
172}
173
174/*------------------------------------------------------------------------*
175 * usb2_m_copy_in - copy a mbuf chain directly into DMA-able memory
176 *------------------------------------------------------------------------*/
177struct usb2_m_copy_in_arg {
178 struct usb2_page_cache *cache;
179 uint32_t dst_offset;
180};
181
182static int32_t
183#ifdef __FreeBSD__
184usb2_m_copy_in_cb(void *arg, void *src, uint32_t count)
185#else
186usb2_m_copy_in_cb(void *arg, caddr_t src, uint32_t count)
187#endif
188{
189 register struct usb2_m_copy_in_arg *ua = arg;
190
191 usb2_copy_in(ua->cache, ua->dst_offset, src, count);
192 ua->dst_offset += count;
193 return (0);
194}
195
196void
197usb2_m_copy_in(struct usb2_page_cache *cache, uint32_t dst_offset,
198 struct mbuf *m, uint32_t src_offset, uint32_t src_len)
199{
200 struct usb2_m_copy_in_arg arg = {cache, dst_offset};
201 register int error;
202
203 error = m_apply(m, src_offset, src_len, &usb2_m_copy_in_cb, &arg);
204 return;
205}
206
207/*------------------------------------------------------------------------*
208 * usb2_uiomove - factored out code
209 *------------------------------------------------------------------------*/
210int
211usb2_uiomove(struct usb2_page_cache *pc, struct uio *uio,
212 uint32_t pc_offset, uint32_t len)
213{
214 struct usb2_page_search res;
215 int error = 0;
216
217 while (len != 0) {
218
219 usb2_get_page(pc, pc_offset, &res);
220
221 if (res.length > len) {
222 res.length = len;
223 }
224 /*
225 * "uiomove()" can sleep so one needs to make a wrapper,
226 * exiting the mutex and checking things
227 */
228 error = uiomove(res.buffer, res.length, uio);
229
230 if (error) {
231 break;
232 }
233 pc_offset += res.length;
234 len -= res.length;
235 }
236 return (error);
237}
238
239/*------------------------------------------------------------------------*
240 * usb2_copy_out - copy directly from DMA-able memory
241 *------------------------------------------------------------------------*/
242void
243usb2_copy_out(struct usb2_page_cache *cache, uint32_t offset,
244 void *ptr, uint32_t len)
245{
246 struct usb2_page_search res;
247
248 while (len != 0) {
249
250 usb2_get_page(cache, offset, &res);
251
252 if (res.length > len) {
253 res.length = len;
254 }
255 bcopy(res.buffer, ptr, res.length);
256
257 offset += res.length;
258 len -= res.length;
259 ptr = USB_ADD_BYTES(ptr, res.length);
260 }
261 return;
262}
263
264/*------------------------------------------------------------------------*
265 * usb2_copy_out_user - copy directly from DMA-able memory to userland
266 *
267 * Return values:
268 * 0: Success
269 * Else: Failure
270 *------------------------------------------------------------------------*/
271int
272usb2_copy_out_user(struct usb2_page_cache *cache, uint32_t offset,
273 void *ptr, uint32_t len)
274{
275 struct usb2_page_search res;
276 int error;
277
278 while (len != 0) {
279
280 usb2_get_page(cache, offset, &res);
281
282 if (res.length > len) {
283 res.length = len;
284 }
285 error = copyout(res.buffer, ptr, res.length);
286 if (error)
287 return (error);
288
289 offset += res.length;
290 len -= res.length;
291 ptr = USB_ADD_BYTES(ptr, res.length);
292 }
293 return (0); /* success */
294}
295
296/*------------------------------------------------------------------------*
297 * usb2_bzero - zero DMA-able memory
298 *------------------------------------------------------------------------*/
299void
300usb2_bzero(struct usb2_page_cache *cache, uint32_t offset, uint32_t len)
301{
302 struct usb2_page_search res;
303
304 while (len != 0) {
305
306 usb2_get_page(cache, offset, &res);
307
308 if (res.length > len) {
309 res.length = len;
310 }
311 bzero(res.buffer, res.length);
312
313 offset += res.length;
314 len -= res.length;
315 }
316 return;
317}
318
319
320#ifdef __FreeBSD__
321
322/*------------------------------------------------------------------------*
323 * usb2_dma_lock_cb - dummy callback
324 *------------------------------------------------------------------------*/
325static void
326usb2_dma_lock_cb(void *arg, bus_dma_lock_op_t op)
327{
328 /* we use "mtx_owned()" instead of this function */
329 return;
330}
331
332/*------------------------------------------------------------------------*
333 * usb2_dma_tag_create - allocate a DMA tag
334 *
335 * NOTE: If the "align" parameter has a value of 1 the DMA-tag will
336 * allow multi-segment mappings. Else all mappings are single-segment.
337 *------------------------------------------------------------------------*/
338static void
339usb2_dma_tag_create(struct usb2_dma_tag *udt,
340 uint32_t size, uint32_t align)
341{
342 bus_dma_tag_t tag;
343
344 if (bus_dma_tag_create
345 ( /* parent */ udt->tag_parent->tag,
346 /* alignment */ align,
347 /* boundary */ USB_PAGE_SIZE,
348 /* lowaddr */ (2ULL << (udt->tag_parent->dma_bits - 1)) - 1,
349 /* highaddr */ BUS_SPACE_MAXADDR,
350 /* filter */ NULL,
351 /* filterarg */ NULL,
352 /* maxsize */ size,
353 /* nsegments */ (align == 1) ?
354 (2 + (size / USB_PAGE_SIZE)) : 1,
355 /* maxsegsz */ (align == 1) ?
356 USB_PAGE_SIZE : size,
357 /* flags */ 0,
358 /* lockfn */ &usb2_dma_lock_cb,
359 /* lockarg */ NULL,
360 &tag)) {
361 tag = NULL;
362 }
363 udt->tag = tag;
364 return;
365}
366
367/*------------------------------------------------------------------------*
368 * usb2_dma_tag_free - free a DMA tag
369 *------------------------------------------------------------------------*/
370static void
371usb2_dma_tag_destroy(struct usb2_dma_tag *udt)
372{
373 bus_dma_tag_destroy(udt->tag);
374 return;
375}
376
377/*------------------------------------------------------------------------*
378 * usb2_pc_alloc_mem_cb - BUS-DMA callback function
379 *------------------------------------------------------------------------*/
380static void
381usb2_pc_alloc_mem_cb(void *arg, bus_dma_segment_t *segs,
382 int nseg, int error)
383{
384 usb2_pc_common_mem_cb(arg, segs, nseg, error, 0);
385 return;
386}
387
388/*------------------------------------------------------------------------*
389 * usb2_pc_load_mem_cb - BUS-DMA callback function
390 *------------------------------------------------------------------------*/
391static void
392usb2_pc_load_mem_cb(void *arg, bus_dma_segment_t *segs,
393 int nseg, int error)
394{
395 usb2_pc_common_mem_cb(arg, segs, nseg, error, 1);
396 return;
397}
398
399/*------------------------------------------------------------------------*
400 * usb2_pc_common_mem_cb - BUS-DMA callback function
401 *------------------------------------------------------------------------*/
402static void
403usb2_pc_common_mem_cb(void *arg, bus_dma_segment_t *segs,
404 int nseg, int error, uint8_t isload)
405{
406 struct usb2_dma_parent_tag *uptag;
407 struct usb2_page_cache *pc;
408 struct usb2_page *pg;
409 uint32_t rem;
410 uint8_t owned;
411
412 pc = arg;
413 uptag = pc->tag_parent;
414
415 /*
416 * XXX There is sometimes recursive locking here.
417 * XXX We should try to find a better solution.
418 * XXX Until further the "owned" variable does
419 * XXX the trick.
420 */
421
422 if (error) {
423 goto done;
424 }
425 pg = pc->page_start;
426 pg->physaddr = segs->ds_addr & ~(USB_PAGE_SIZE - 1);
427 rem = segs->ds_addr & (USB_PAGE_SIZE - 1);
428 pc->page_offset_buf = rem;
429 pc->page_offset_end += rem;
430 nseg--;
431
432 while (nseg > 0) {
433 nseg--;
434 segs++;
435 pg++;
436 pg->physaddr = segs->ds_addr & ~(USB_PAGE_SIZE - 1);
437 }
438
439done:
440 owned = mtx_owned(uptag->mtx);
441 if (!owned)
442 mtx_lock(uptag->mtx);
443
444 uptag->dma_error = (error ? 1 : 0);
445 if (isload) {
446 (uptag->func) (uptag);
447 } else {
448 usb2_cv_broadcast(uptag->cv);
449 }
450 if (!owned)
451 mtx_unlock(uptag->mtx);
452 return;
453}
454
455/*------------------------------------------------------------------------*
456 * usb2_pc_alloc_mem - allocate DMA'able memory
457 *
458 * Returns:
459 * 0: Success
460 * Else: Failure
461 *------------------------------------------------------------------------*/
462uint8_t
463usb2_pc_alloc_mem(struct usb2_page_cache *pc, struct usb2_page *pg,
464 uint32_t size, uint32_t align)
465{
466 struct usb2_dma_parent_tag *uptag;
467 struct usb2_dma_tag *utag;
468 bus_dmamap_t map;
469 void *ptr;
470 int err;
471
472 uptag = pc->tag_parent;
473
474 if (align != 1) {
475 /*
476 * The alignment must be greater or equal to the
477 * "size" else the object can be split between two
478 * memory pages and we get a problem!
479 */
480 while (align < size) {
481 align *= 2;
482 if (align == 0) {
483 goto error;
484 }
485 }
486#if 1
487 /*
488 * XXX BUS-DMA workaround - FIXME later:
489 *
490 * We assume that that the aligment at this point of
491 * the code is greater than or equal to the size and
492 * less than two times the size, so that if we double
493 * the size, the size will be greater than the
494 * alignment.
495 *
496 * The bus-dma system has a check for "alignment"
497 * being less than "size". If that check fails we end
498 * up using contigmalloc which is page based even for
499 * small allocations. Try to avoid that to save
500 * memory, hence we sometimes to a large number of
501 * small allocations!
502 */
503 if (size <= (USB_PAGE_SIZE / 2)) {
504 size *= 2;
505 }
506#endif
507 }
508 /* get the correct DMA tag */
509 utag = usb2_dma_tag_find(uptag, size, align);
510 if (utag == NULL) {
511 goto error;
512 }
513 /* allocate memory */
514 if (bus_dmamem_alloc(
515 utag->tag, &ptr, (BUS_DMA_WAITOK | BUS_DMA_COHERENT), &map)) {
516 goto error;
517 }
518 /* setup page cache */
519 pc->buffer = ptr;
520 pc->page_start = pg;
521 pc->page_offset_buf = 0;
522 pc->page_offset_end = size;
523 pc->map = map;
524 pc->tag = utag->tag;
525 pc->ismultiseg = (align == 1);
526
527 mtx_lock(uptag->mtx);
528
529 /* load memory into DMA */
530 err = bus_dmamap_load(
531 utag->tag, map, ptr, size, &usb2_pc_alloc_mem_cb,
532 pc, (BUS_DMA_WAITOK | BUS_DMA_COHERENT));
533
534 if (err == EINPROGRESS) {
535 usb2_cv_wait(uptag->cv, uptag->mtx);
536 err = 0;
537 }
538 mtx_unlock(uptag->mtx);
539
540 if (err || uptag->dma_error) {
541 bus_dmamem_free(utag->tag, ptr, map);
542 goto error;
543 }
544 bzero(ptr, size);
545
546 usb2_pc_cpu_flush(pc);
547
548 return (0);
549
550error:
551 /* reset most of the page cache */
552 pc->buffer = NULL;
553 pc->page_start = NULL;
554 pc->page_offset_buf = 0;
555 pc->page_offset_end = 0;
556 pc->map = NULL;
557 pc->tag = NULL;
558 return (1);
559}
560
561/*------------------------------------------------------------------------*
562 * usb2_pc_free_mem - free DMA memory
563 *
564 * This function is NULL safe.
565 *------------------------------------------------------------------------*/
566void
567usb2_pc_free_mem(struct usb2_page_cache *pc)
568{
569 if (pc && pc->buffer) {
570
571 bus_dmamap_unload(pc->tag, pc->map);
572
573 bus_dmamem_free(pc->tag, pc->buffer, pc->map);
574
575 pc->buffer = NULL;
576 }
577 return;
578}
579
580/*------------------------------------------------------------------------*
581 * usb2_pc_load_mem - load virtual memory into DMA
582 *
583 * Return values:
584 * 0: Success
585 * Else: Error
586 *------------------------------------------------------------------------*/
587uint8_t
588usb2_pc_load_mem(struct usb2_page_cache *pc, uint32_t size, uint8_t sync)
589{
590 /* setup page cache */
591 pc->page_offset_buf = 0;
592 pc->page_offset_end = size;
593 pc->ismultiseg = 1;
594
595 mtx_assert(pc->tag_parent->mtx, MA_OWNED);
596
597 if (size > 0) {
598 if (sync) {
599 struct usb2_dma_parent_tag *uptag;
600 int err;
601
602 uptag = pc->tag_parent;
603
604 /*
605 * Try to load memory into DMA.
606 */
607 err = bus_dmamap_load(
608 pc->tag, pc->map, pc->buffer, size,
609 &usb2_pc_alloc_mem_cb, pc, BUS_DMA_WAITOK);
610 if (err == EINPROGRESS) {
611 usb2_cv_wait(uptag->cv, uptag->mtx);
612 err = 0;
613 }
614 if (err || uptag->dma_error) {
615 return (1);
616 }
617 } else {
618
619 /*
620 * Try to load memory into DMA. The callback
621 * will be called in all cases:
622 */
623 if (bus_dmamap_load(
624 pc->tag, pc->map, pc->buffer, size,
625 &usb2_pc_load_mem_cb, pc, BUS_DMA_WAITOK)) {
626 }
627 }
628 } else {
629 if (!sync) {
630 /*
631 * Call callback so that refcount is decremented
632 * properly:
633 */
634 pc->tag_parent->dma_error = 0;
635 (pc->tag_parent->func) (pc->tag_parent);
636 }
637 }
638 return (0);
639}
640
641/*------------------------------------------------------------------------*
642 * usb2_pc_cpu_invalidate - invalidate CPU cache
643 *------------------------------------------------------------------------*/
644void
645usb2_pc_cpu_invalidate(struct usb2_page_cache *pc)
646{
647 bus_dmamap_sync(pc->tag, pc->map,
648 BUS_DMASYNC_POSTWRITE | BUS_DMASYNC_POSTREAD);
649 return;
650}
651
652/*------------------------------------------------------------------------*
653 * usb2_pc_cpu_flush - flush CPU cache
654 *------------------------------------------------------------------------*/
655void
656usb2_pc_cpu_flush(struct usb2_page_cache *pc)
657{
658 bus_dmamap_sync(pc->tag, pc->map,
659 BUS_DMASYNC_PREWRITE | BUS_DMASYNC_PREREAD);
660 return;
661}
662
663/*------------------------------------------------------------------------*
664 * usb2_pc_dmamap_create - create a DMA map
665 *
666 * Returns:
667 * 0: Success
668 * Else: Failure
669 *------------------------------------------------------------------------*/
670uint8_t
671usb2_pc_dmamap_create(struct usb2_page_cache *pc, uint32_t size)
672{
673 struct usb2_xfer_root *info;
674 struct usb2_dma_tag *utag;
675
676 /* get info */
677 info = pc->tag_parent->info;
678
679 /* sanity check */
680 if (info == NULL) {
681 goto error;
682 }
683 utag = usb2_dma_tag_find(pc->tag_parent, size, 1);
684 if (utag == NULL) {
685 goto error;
686 }
687 /* create DMA map */
688 if (bus_dmamap_create(utag->tag, 0, &pc->map)) {
689 goto error;
690 }
691 pc->tag = utag->tag;
692 return 0; /* success */
693
694error:
695 pc->map = NULL;
696 pc->tag = NULL;
697 return 1; /* failure */
698}
699
700/*------------------------------------------------------------------------*
701 * usb2_pc_dmamap_destroy
702 *
703 * This function is NULL safe.
704 *------------------------------------------------------------------------*/
705void
706usb2_pc_dmamap_destroy(struct usb2_page_cache *pc)
707{
708 if (pc && pc->tag) {
709 bus_dmamap_destroy(pc->tag, pc->map);
710 pc->tag = NULL;
711 pc->map = NULL;
712 }
713 return;
714}
715
716#endif
717
718#ifdef __NetBSD__
719
720/*------------------------------------------------------------------------*
721 * usb2_dma_tag_create - allocate a DMA tag
722 *
723 * NOTE: If the "align" parameter has a value of 1 the DMA-tag will
724 * allow multi-segment mappings. Else all mappings are single-segment.
725 *------------------------------------------------------------------------*/
726static void
727usb2_dma_tag_create(struct usb2_dma_tag *udt,
728 uint32_t size, uint32_t align)
729{
730 uint32_t nseg;
731
732 if (align == 1) {
733 nseg = (2 + (size / USB_PAGE_SIZE));
734 } else {
735 nseg = 1;
736 }
737
738 udt->p_seg = malloc(nseg * sizeof(*(udt->p_seg)),
739 M_USB, M_WAITOK | M_ZERO);
740
741 if (udt->p_seg == NULL) {
742 return;
743 }
744 udt->tag = udt->tag_parent->tag;
745 udt->n_seg = nseg;
746 return;
747}
748
749/*------------------------------------------------------------------------*
750 * usb2_dma_tag_free - free a DMA tag
751 *------------------------------------------------------------------------*/
752static void
753usb2_dma_tag_destroy(struct usb2_dma_tag *udt)
754{
755 free(udt->p_seg, M_USB);
756 return;
757}
758
759/*------------------------------------------------------------------------*
760 * usb2_pc_common_mem_cb - BUS-DMA callback function
761 *------------------------------------------------------------------------*/
762static void
763usb2_pc_common_mem_cb(struct usb2_page_cache *pc, bus_dma_segment_t *segs,
764 int nseg, int error, uint8_t isload, uint8_t dolock)
765{
766 struct usb2_dma_parent_tag *uptag;
767 struct usb2_page *pg;
768 uint32_t rem;
769 uint8_t ext_seg; /* extend last segment */
770
771 uptag = pc->tag_parent;
772
773 if (error) {
774 goto done;
775 }
776 pg = pc->page_start;
777 pg->physaddr = segs->ds_addr & ~(USB_PAGE_SIZE - 1);
778 rem = segs->ds_addr & (USB_PAGE_SIZE - 1);
779 pc->page_offset_buf = rem;
780 pc->page_offset_end += rem;
781 if (nseg < ((pc->page_offset_end +
782 (USB_PAGE_SIZE - 1)) / USB_PAGE_SIZE)) {
783 ext_seg = 1;
784 } else {
785 ext_seg = 0;
786 }
787 nseg--;
788
789 while (nseg > 0) {
790 nseg--;
791 segs++;
792 pg++;
793 pg->physaddr = segs->ds_addr & ~(USB_PAGE_SIZE - 1);
794 }
795
796 /*
797 * XXX The segments we get from BUS-DMA are not aligned,
798 * XXX so we need to extend the last segment if we are
799 * XXX unaligned and cross the segment boundary!
800 */
801 if (ext_seg && pc->ismultiseg) {
802 (pg + 1)->physaddr = pg->physaddr + USB_PAGE_SIZE;
803 }
804done:
805 if (dolock)
806 mtx_lock(uptag->mtx);
807
808 uptag->dma_error = (error ? 1 : 0);
809 if (isload) {
810 (uptag->func) (uptag);
811 }
812 if (dolock)
813 mtx_unlock(uptag->mtx);
814 return;
815}
816
817/*------------------------------------------------------------------------*
818 * usb2_pc_alloc_mem - allocate DMA'able memory
819 *
820 * Returns:
821 * 0: Success
822 * Else: Failure
823 *------------------------------------------------------------------------*/
824uint8_t
825usb2_pc_alloc_mem(struct usb2_page_cache *pc, struct usb2_page *pg,
826 uint32_t size, uint32_t align)
827{
828 struct usb2_dma_parent_tag *uptag;
829 struct usb2_dma_tag *utag;
830 caddr_t ptr = NULL;
831 bus_dmamap_t map;
832 int seg_count;
833
834 uptag = pc->tag_parent;
835
836 if (align != 1) {
837 /*
838 * The alignment must be greater or equal to the
839 * "size" else the object can be split between two
840 * memory pages and we get a problem!
841 */
842 while (align < size) {
843 align *= 2;
844 if (align == 0) {
845 goto done_5;
846 }
847 }
848 }
849 /* get the correct DMA tag */
850 utag = usb2_dma_tag_find(pc->tag_parent, size, align);
851 if (utag == NULL) {
852 goto done_5;
853 }
854 if (bus_dmamem_alloc(utag->tag, size, align, 0, utag->p_seg,
855 utag->n_seg, &seg_count, BUS_DMA_WAITOK)) {
856 goto done_4;
857 }
858 if (bus_dmamem_map(utag->tag, utag->p_seg, seg_count, size,
859 &ptr, BUS_DMA_WAITOK | BUS_DMA_COHERENT)) {
860 goto done_3;
861 }
862 if (bus_dmamap_create(utag->tag, size, utag->n_seg, (align == 1) ?
863 USB_PAGE_SIZE : size, 0, BUS_DMA_WAITOK, &map)) {
864 goto done_2;
865 }
866 if (bus_dmamap_load(utag->tag, map, ptr, size, NULL,
867 BUS_DMA_WAITOK)) {
868 goto done_1;
869 }
870 pc->p_seg = malloc(seg_count * sizeof(*(pc->p_seg)),
871 M_USB, M_WAITOK | M_ZERO);
872 if (pc->p_seg == NULL) {
873 goto done_0;
874 }
875 /* store number if actual segments used */
876 pc->n_seg = seg_count;
877
878 /* make a copy of the segments */
879 bcopy(utag->p_seg, pc->p_seg,
880 seg_count * sizeof(*(pc->p_seg)));
881
882 /* setup page cache */
883 pc->buffer = ptr;
884 pc->page_start = pg;
885 pc->page_offset_buf = 0;
886 pc->page_offset_end = size;
887 pc->map = map;
888 pc->tag = utag->tag;
889 pc->ismultiseg = (align == 1);
890
891 usb2_pc_common_mem_cb(pc, utag->p_seg, seg_count, 0, 0, 1);
892
893 bzero(ptr, size);
894
895 usb2_pc_cpu_flush(pc);
896
897 return (0);
898
899done_0:
900 bus_dmamap_unload(utag->tag, map);
901done_1:
902 bus_dmamap_destroy(utag->tag, map);
903done_2:
904 bus_dmamem_unmap(utag->tag, ptr, size);
905done_3:
906 bus_dmamem_free(utag->tag, utag->p_seg, seg_count);
907done_4:
908 /* utag is destroyed later */
909done_5:
910 /* reset most of the page cache */
911 pc->buffer = NULL;
912 pc->page_start = NULL;
913 pc->page_offset_buf = 0;
914 pc->page_offset_end = 0;
915 pc->map = NULL;
916 pc->tag = NULL;
917 pc->n_seg = 0;
918 pc->p_seg = NULL;
919 return (1);
920}
921
922/*------------------------------------------------------------------------*
923 * usb2_pc_free_mem - free DMA memory
924 *
925 * This function is NULL safe.
926 *------------------------------------------------------------------------*/
927void
928usb2_pc_free_mem(struct usb2_page_cache *pc)
929{
930 if (pc && pc->buffer) {
931 bus_dmamap_unload(pc->tag, pc->map);
932 bus_dmamap_destroy(pc->tag, pc->map);
933 bus_dmamem_unmap(pc->tag, pc->buffer,
934 pc->page_offset_end - pc->page_offset_buf);
935 bus_dmamem_free(pc->tag, pc->p_seg, pc->n_seg);
936 free(pc->p_seg, M_USB);
937 pc->buffer = NULL;
938 }
939 return;
940}
941
942/*------------------------------------------------------------------------*
943 * usb2_pc_load_mem - load virtual memory into DMA
944 *
945 * Return values:
946 * 0: Success
947 * Else: Error
948 *------------------------------------------------------------------------*/
949uint8_t
950usb2_pc_load_mem(struct usb2_page_cache *pc, uint32_t size, uint8_t sync)
951{
952 int error;
953
954 /* setup page cache */
955 pc->page_offset_buf = 0;
956 pc->page_offset_end = size;
957 pc->ismultiseg = 1;
958
959 if (size > 0) {
960
961 /* try to load memory into DMA using using no wait option */
962 if (bus_dmamap_load(pc->tag, pc->map, pc->buffer,
963 size, NULL, BUS_DMA_NOWAIT)) {
964 error = ENOMEM;
965 } else {
966 error = 0;
967 }
968
969 usb2_pc_common_mem_cb(pc, pc->map->dm_segs,
970 pc->map->dm_nsegs, error, !sync);
971
972 if (error) {
973 return (1);
974 }
975 } else {
976 if (!sync) {
977 /*
978 * Call callback so that refcount is decremented
979 * properly:
980 */
981 pc->tag_parent->dma_error = 0;
982 (pc->tag_parent->func) (pc->tag_parent);
983 }
984 }
985 return (0);
986}
987
988/*------------------------------------------------------------------------*
989 * usb2_pc_cpu_invalidate - invalidate CPU cache
990 *------------------------------------------------------------------------*/
991void
992usb2_pc_cpu_invalidate(struct usb2_page_cache *pc)
993{
994 uint32_t len;
995
996 len = pc->page_offset_end - pc->page_offset_buf;
997
998 bus_dmamap_sync(pc->tag, pc->map, 0, len,
999 BUS_DMASYNC_POSTWRITE | BUS_DMASYNC_POSTREAD);
1000 return;
1001}
1002
1003/*------------------------------------------------------------------------*
1004 * usb2_pc_cpu_flush - flush CPU cache
1005 *------------------------------------------------------------------------*/
1006void
1007usb2_pc_cpu_flush(struct usb2_page_cache *pc)
1008{
1009 uint32_t len;
1010
1011 len = pc->page_offset_end - pc->page_offset_buf;
1012
1013 bus_dmamap_sync(pc->tag, pc->map, 0, len,
1014 BUS_DMASYNC_PREWRITE | BUS_DMASYNC_PREREAD);
1015 return;
1016}
1017
1018/*------------------------------------------------------------------------*
1019 * usb2_pc_dmamap_create - create a DMA map
1020 *
1021 * Returns:
1022 * 0: Success
1023 * Else: Failure
1024 *------------------------------------------------------------------------*/
1025uint8_t
1026usb2_pc_dmamap_create(struct usb2_page_cache *pc, uint32_t size)
1027{
1028 struct usb2_xfer_root *info;
1029 struct usb2_dma_tag *utag;
1030
1031 /* get info */
1032 info = pc->tag_parent->info;
1033
1034 /* sanity check */
1035 if (info == NULL) {
1036 goto error;
1037 }
1038 utag = usb2_dma_tag_find(pc->tag_parent, size, 1);
1039 if (utag == NULL) {
1040 goto error;
1041 }
1042 if (bus_dmamap_create(utag->tag, size, utag->n_seg,
1043 USB_PAGE_SIZE, 0, BUS_DMA_WAITOK, &pc->map)) {
1044 goto error;
1045 }
1046 pc->tag = utag->tag;
1047 pc->p_seg = utag->p_seg;
1048 pc->n_seg = utag->n_seg;
1049 return 0; /* success */
1050
1051error:
1052 pc->map = NULL;
1053 pc->tag = NULL;
1054 pc->p_seg = NULL;
1055 pc->n_seg = 0;
1056 return 1; /* failure */
1057}
1058
1059/*------------------------------------------------------------------------*
1060 * usb2_pc_dmamap_destroy
1061 *
1062 * This function is NULL safe.
1063 *------------------------------------------------------------------------*/
1064void
1065usb2_pc_dmamap_destroy(struct usb2_page_cache *pc)
1066{
1067 if (pc && pc->tag) {
1068 bus_dmamap_destroy(pc->tag, pc->map);
1069 pc->tag = NULL;
1070 pc->map = NULL;
1071 }
1072 return;
1073}
1074
1075#endif
1076
1077/*------------------------------------------------------------------------*
1078 * usb2_dma_tag_find - factored out code
1079 *------------------------------------------------------------------------*/
1080struct usb2_dma_tag *
1081usb2_dma_tag_find(struct usb2_dma_parent_tag *udpt,
1082 uint32_t size, uint32_t align)
1083{
1084 struct usb2_dma_tag *udt;
1085 uint8_t nudt;
1086
1087 USB_ASSERT(align > 0, ("Invalid parameter align = 0!\n"));
1088 USB_ASSERT(size > 0, ("Invalid parameter size = 0!\n"));
1089
1090 udt = udpt->utag_first;
1091 nudt = udpt->utag_max;
1092
1093 while (nudt--) {
1094
1095 if (udt->align == 0) {
1096 usb2_dma_tag_create(udt, size, align);
1097 if (udt->tag == NULL) {
1098 return (NULL);
1099 }
1100 udt->align = align;
1101 udt->size = size;
1102 return (udt);
1103 }
1104 if ((udt->align == align) && (udt->size == size)) {
1105 return (udt);
1106 }
1107 udt++;
1108 }
1109 return (NULL);
1110}
1111
1112/*------------------------------------------------------------------------*
1113 * usb2_dma_tag_setup - initialise USB DMA tags
1114 *------------------------------------------------------------------------*/
1115void
1116usb2_dma_tag_setup(struct usb2_dma_parent_tag *udpt,
1117 struct usb2_dma_tag *udt, bus_dma_tag_t dmat,
1118 struct mtx *mtx, usb2_dma_callback_t *func,
1119 struct usb2_xfer_root *info, uint8_t ndmabits,
1120 uint8_t nudt)
1121{
1122 bzero(udpt, sizeof(*udpt));
1123
1124 /* sanity checking */
1125 if ((nudt == 0) ||
1126 (ndmabits == 0) ||
1127 (mtx == NULL)) {
1128 /* something is corrupt */
1129 return;
1130 }
1131#ifdef __FreeBSD__
1132 /* initialise condition variable */
1133 usb2_cv_init(udpt->cv, "USB DMA CV");
1134#endif
1135
1136 /* store some information */
1137 udpt->mtx = mtx;
1138 udpt->info = info;
1139 udpt->func = func;
1140 udpt->tag = dmat;
1141 udpt->utag_first = udt;
1142 udpt->utag_max = nudt;
1143 udpt->dma_bits = ndmabits;
1144
1145 while (nudt--) {
1146 bzero(udt, sizeof(*udt));
1147 udt->tag_parent = udpt;
1148 udt++;
1149 }
1150 return;
1151}
1152
1153/*------------------------------------------------------------------------*
1154 * usb2_bus_tag_unsetup - factored out code
1155 *------------------------------------------------------------------------*/
1156void
1157usb2_dma_tag_unsetup(struct usb2_dma_parent_tag *udpt)
1158{
1159 struct usb2_dma_tag *udt;
1160 uint8_t nudt;
1161
1162 udt = udpt->utag_first;
1163 nudt = udpt->utag_max;
1164
1165 while (nudt--) {
1166
1167 if (udt->align) {
1168 /* destroy the USB DMA tag */
1169 usb2_dma_tag_destroy(udt);
1170 udt->align = 0;
1171 }
1172 udt++;
1173 }
1174
1175 if (udpt->utag_max) {
1176#ifdef __FreeBSD__
1177 /* destroy the condition variable */
1178 usb2_cv_destroy(udpt->cv);
1179#endif
1180 }
1181 return;
1182}
1183
1184/*------------------------------------------------------------------------*
1185 * usb2_bdma_work_loop
1186 *
1187 * This function handles loading of virtual buffers into DMA and is
1188 * only called when "dma_refcount" is zero.
1189 *------------------------------------------------------------------------*/
1190void
1191usb2_bdma_work_loop(struct usb2_xfer_queue *pq)
1192{
1193 struct usb2_xfer_root *info;
1194 struct usb2_xfer *xfer;
1195 uint32_t nframes;
1196
1197 xfer = pq->curr;
1198 info = xfer->usb2_root;
1199
1200 mtx_assert(info->priv_mtx, MA_OWNED);
1201
1202 if (xfer->error) {
1203 /* some error happened */
|