1/* $FreeBSD: head/sys/dev/usb2/core/usb2_busdma.c 184824 2008-11-10 20:54:31Z thompsa $ */
| 1/* $FreeBSD: head/sys/dev/usb2/core/usb2_busdma.c 185087 2008-11-19 08:56:35Z alfred $ */
|
2/*-
3 * Copyright (c) 2008 Hans Petter Selasky. All rights reserved.
4 *
5 * Redistribution and use in source and binary forms, with or without
6 * modification, are permitted provided that the following conditions
7 * are met:
8 * 1. Redistributions of source code must retain the above copyright
9 * notice, this list of conditions and the following disclaimer.
10 * 2. Redistributions in binary form must reproduce the above copyright
11 * notice, this list of conditions and the following disclaimer in the
12 * documentation and/or other materials provided with the distribution.
13 *
14 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
15 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
16 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
17 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
18 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
19 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
20 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
21 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
22 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
23 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
24 * SUCH DAMAGE.
25 */
26
27#include <dev/usb2/include/usb2_mfunc.h>
28#include <dev/usb2/include/usb2_error.h>
29#include <dev/usb2/include/usb2_standard.h>
30#include <dev/usb2/include/usb2_defs.h>
31
32#include <dev/usb2/core/usb2_core.h>
33#include <dev/usb2/core/usb2_busdma.h>
34#include <dev/usb2/core/usb2_process.h>
35#include <dev/usb2/core/usb2_transfer.h>
36#include <dev/usb2/core/usb2_device.h>
37#include <dev/usb2/core/usb2_util.h>
38
39#include <dev/usb2/controller/usb2_controller.h>
40#include <dev/usb2/controller/usb2_bus.h>
41
42static void usb2_dma_tag_create(struct usb2_dma_tag *udt, uint32_t size, uint32_t align);
43static void usb2_dma_tag_destroy(struct usb2_dma_tag *udt);
44
45#ifdef __FreeBSD__
46static void usb2_dma_lock_cb(void *arg, bus_dma_lock_op_t op);
47static int32_t usb2_m_copy_in_cb(void *arg, void *src, uint32_t count);
48static void usb2_pc_alloc_mem_cb(void *arg, bus_dma_segment_t *segs, int nseg, int error);
49static void usb2_pc_load_mem_cb(void *arg, bus_dma_segment_t *segs, int nseg, int error);
50static void usb2_pc_common_mem_cb(void *arg, bus_dma_segment_t *segs, int nseg, int error, uint8_t isload);
51
52#endif
53
54#ifdef __NetBSD__
55static int32_t usb2_m_copy_in_cb(void *arg, caddr_t src, uint32_t count);
56static void usb2_pc_common_mem_cb(struct usb2_page_cache *pc, bus_dma_segment_t *segs, int nseg, int error, uint8_t isload);
57
58#endif
59
60/*------------------------------------------------------------------------*
61 * usb2_get_page - lookup DMA-able memory for the given offset
62 *
63 * NOTE: Only call this function when the "page_cache" structure has
64 * been properly initialized !
65 *------------------------------------------------------------------------*/
66void
67usb2_get_page(struct usb2_page_cache *pc, uint32_t offset,
68 struct usb2_page_search *res)
69{
70 struct usb2_page *page;
71
72 if (pc->page_start) {
73
74 /* Case 1 - something has been loaded into DMA */
75
76 if (pc->buffer) {
77
78 /* Case 1a - Kernel Virtual Address */
79
80 res->buffer = USB_ADD_BYTES(pc->buffer, offset);
81 }
82 offset += pc->page_offset_buf;
83
84 /* compute destination page */
85
86 page = pc->page_start;
87
88 if (pc->ismultiseg) {
89
90 page += (offset / USB_PAGE_SIZE);
91
92 offset %= USB_PAGE_SIZE;
93
94 res->length = USB_PAGE_SIZE - offset;
95 res->physaddr = page->physaddr + offset;
96 } else {
97 res->length = 0 - 1;
98 res->physaddr = page->physaddr + offset;
99 }
100 if (!pc->buffer) {
101
102 /* Case 1b - Non Kernel Virtual Address */
103
104 res->buffer = USB_ADD_BYTES(page->buffer, offset);
105 }
106 } else {
107
108 /* Case 2 - Plain PIO */
109
110 res->buffer = USB_ADD_BYTES(pc->buffer, offset);
111 res->length = 0 - 1;
112 res->physaddr = 0;
113 }
114 return;
115}
116
117/*------------------------------------------------------------------------*
118 * usb2_copy_in - copy directly to DMA-able memory
119 *------------------------------------------------------------------------*/
120void
121usb2_copy_in(struct usb2_page_cache *cache, uint32_t offset,
122 const void *ptr, uint32_t len)
123{
124 struct usb2_page_search buf_res;
125
126 while (len != 0) {
127
128 usb2_get_page(cache, offset, &buf_res);
129
130 if (buf_res.length > len) {
131 buf_res.length = len;
132 }
133 bcopy(ptr, buf_res.buffer, buf_res.length);
134
135 offset += buf_res.length;
136 len -= buf_res.length;
137 ptr = USB_ADD_BYTES(ptr, buf_res.length);
138 }
139 return;
140}
141
142/*------------------------------------------------------------------------*
143 * usb2_copy_in_user - copy directly to DMA-able memory from userland
144 *
145 * Return values:
146 * 0: Success
147 * Else: Failure
148 *------------------------------------------------------------------------*/
149int
150usb2_copy_in_user(struct usb2_page_cache *cache, uint32_t offset,
151 const void *ptr, uint32_t len)
152{
153 struct usb2_page_search buf_res;
154 int error;
155
156 while (len != 0) {
157
158 usb2_get_page(cache, offset, &buf_res);
159
160 if (buf_res.length > len) {
161 buf_res.length = len;
162 }
163 error = copyin(ptr, buf_res.buffer, buf_res.length);
164 if (error)
165 return (error);
166
167 offset += buf_res.length;
168 len -= buf_res.length;
169 ptr = USB_ADD_BYTES(ptr, buf_res.length);
170 }
171 return (0); /* success */
172}
173
174/*------------------------------------------------------------------------*
175 * usb2_m_copy_in - copy a mbuf chain directly into DMA-able memory
176 *------------------------------------------------------------------------*/
177struct usb2_m_copy_in_arg {
178 struct usb2_page_cache *cache;
179 uint32_t dst_offset;
180};
181
182static int32_t
183#ifdef __FreeBSD__
184usb2_m_copy_in_cb(void *arg, void *src, uint32_t count)
185#else
186usb2_m_copy_in_cb(void *arg, caddr_t src, uint32_t count)
187#endif
188{
189 register struct usb2_m_copy_in_arg *ua = arg;
190
191 usb2_copy_in(ua->cache, ua->dst_offset, src, count);
192 ua->dst_offset += count;
193 return (0);
194}
195
196void
197usb2_m_copy_in(struct usb2_page_cache *cache, uint32_t dst_offset,
198 struct mbuf *m, uint32_t src_offset, uint32_t src_len)
199{
200 struct usb2_m_copy_in_arg arg = {cache, dst_offset};
201 register int error;
202
203 error = m_apply(m, src_offset, src_len, &usb2_m_copy_in_cb, &arg);
204 return;
205}
206
207/*------------------------------------------------------------------------*
208 * usb2_uiomove - factored out code
209 *------------------------------------------------------------------------*/
210int
211usb2_uiomove(struct usb2_page_cache *pc, struct uio *uio,
212 uint32_t pc_offset, uint32_t len)
213{
214 struct usb2_page_search res;
215 int error = 0;
216
217 while (len != 0) {
218
219 usb2_get_page(pc, pc_offset, &res);
220
221 if (res.length > len) {
222 res.length = len;
223 }
224 /*
225 * "uiomove()" can sleep so one needs to make a wrapper,
226 * exiting the mutex and checking things
227 */
228 error = uiomove(res.buffer, res.length, uio);
229
230 if (error) {
231 break;
232 }
233 pc_offset += res.length;
234 len -= res.length;
235 }
236 return (error);
237}
238
239/*------------------------------------------------------------------------*
240 * usb2_copy_out - copy directly from DMA-able memory
241 *------------------------------------------------------------------------*/
242void
243usb2_copy_out(struct usb2_page_cache *cache, uint32_t offset,
244 void *ptr, uint32_t len)
245{
246 struct usb2_page_search res;
247
248 while (len != 0) {
249
250 usb2_get_page(cache, offset, &res);
251
252 if (res.length > len) {
253 res.length = len;
254 }
255 bcopy(res.buffer, ptr, res.length);
256
257 offset += res.length;
258 len -= res.length;
259 ptr = USB_ADD_BYTES(ptr, res.length);
260 }
261 return;
262}
263
264/*------------------------------------------------------------------------*
265 * usb2_copy_out_user - copy directly from DMA-able memory to userland
266 *
267 * Return values:
268 * 0: Success
269 * Else: Failure
270 *------------------------------------------------------------------------*/
271int
272usb2_copy_out_user(struct usb2_page_cache *cache, uint32_t offset,
273 void *ptr, uint32_t len)
274{
275 struct usb2_page_search res;
276 int error;
277
278 while (len != 0) {
279
280 usb2_get_page(cache, offset, &res);
281
282 if (res.length > len) {
283 res.length = len;
284 }
285 error = copyout(res.buffer, ptr, res.length);
286 if (error)
287 return (error);
288
289 offset += res.length;
290 len -= res.length;
291 ptr = USB_ADD_BYTES(ptr, res.length);
292 }
293 return (0); /* success */
294}
295
296/*------------------------------------------------------------------------*
297 * usb2_bzero - zero DMA-able memory
298 *------------------------------------------------------------------------*/
299void
300usb2_bzero(struct usb2_page_cache *cache, uint32_t offset, uint32_t len)
301{
302 struct usb2_page_search res;
303
304 while (len != 0) {
305
306 usb2_get_page(cache, offset, &res);
307
308 if (res.length > len) {
309 res.length = len;
310 }
311 bzero(res.buffer, res.length);
312
313 offset += res.length;
314 len -= res.length;
315 }
316 return;
317}
318
319
320#ifdef __FreeBSD__
321
322/*------------------------------------------------------------------------*
323 * usb2_dma_lock_cb - dummy callback
324 *------------------------------------------------------------------------*/
325static void
326usb2_dma_lock_cb(void *arg, bus_dma_lock_op_t op)
327{
328 /* we use "mtx_owned()" instead of this function */
329 return;
330}
331
332/*------------------------------------------------------------------------*
333 * usb2_dma_tag_create - allocate a DMA tag
334 *
335 * NOTE: If the "align" parameter has a value of 1 the DMA-tag will
336 * allow multi-segment mappings. Else all mappings are single-segment.
337 *------------------------------------------------------------------------*/
338static void
339usb2_dma_tag_create(struct usb2_dma_tag *udt,
340 uint32_t size, uint32_t align)
341{
342 bus_dma_tag_t tag;
343
344 if (bus_dma_tag_create
345 ( /* parent */ udt->tag_parent->tag,
346 /* alignment */ align,
347 /* boundary */ USB_PAGE_SIZE,
348 /* lowaddr */ (2ULL << (udt->tag_parent->dma_bits - 1)) - 1,
349 /* highaddr */ BUS_SPACE_MAXADDR,
350 /* filter */ NULL,
351 /* filterarg */ NULL,
352 /* maxsize */ size,
353 /* nsegments */ (align == 1) ?
354 (2 + (size / USB_PAGE_SIZE)) : 1,
355 /* maxsegsz */ (align == 1) ?
356 USB_PAGE_SIZE : size,
357 /* flags */ 0,
358 /* lockfn */ &usb2_dma_lock_cb,
359 /* lockarg */ NULL,
360 &tag)) {
361 tag = NULL;
362 }
363 udt->tag = tag;
364 return;
365}
366
367/*------------------------------------------------------------------------*
368 * usb2_dma_tag_free - free a DMA tag
369 *------------------------------------------------------------------------*/
370static void
371usb2_dma_tag_destroy(struct usb2_dma_tag *udt)
372{
373 bus_dma_tag_destroy(udt->tag);
374 return;
375}
376
377/*------------------------------------------------------------------------*
378 * usb2_pc_alloc_mem_cb - BUS-DMA callback function
379 *------------------------------------------------------------------------*/
380static void
381usb2_pc_alloc_mem_cb(void *arg, bus_dma_segment_t *segs,
382 int nseg, int error)
383{
384 usb2_pc_common_mem_cb(arg, segs, nseg, error, 0);
385 return;
386}
387
388/*------------------------------------------------------------------------*
389 * usb2_pc_load_mem_cb - BUS-DMA callback function
390 *------------------------------------------------------------------------*/
391static void
392usb2_pc_load_mem_cb(void *arg, bus_dma_segment_t *segs,
393 int nseg, int error)
394{
395 usb2_pc_common_mem_cb(arg, segs, nseg, error, 1);
396 return;
397}
398
399/*------------------------------------------------------------------------*
400 * usb2_pc_common_mem_cb - BUS-DMA callback function
401 *------------------------------------------------------------------------*/
402static void
403usb2_pc_common_mem_cb(void *arg, bus_dma_segment_t *segs,
404 int nseg, int error, uint8_t isload)
405{
406 struct usb2_dma_parent_tag *uptag;
407 struct usb2_page_cache *pc;
408 struct usb2_page *pg;
409 uint32_t rem;
410 uint8_t owned;
411
412 pc = arg;
413 uptag = pc->tag_parent;
414
415 /*
416 * XXX There is sometimes recursive locking here.
417 * XXX We should try to find a better solution.
418 * XXX Until further the "owned" variable does
419 * XXX the trick.
420 */
421
422 if (error) {
423 goto done;
424 }
425 pg = pc->page_start;
426 pg->physaddr = segs->ds_addr & ~(USB_PAGE_SIZE - 1);
427 rem = segs->ds_addr & (USB_PAGE_SIZE - 1);
428 pc->page_offset_buf = rem;
429 pc->page_offset_end += rem;
430 nseg--;
431
432 while (nseg > 0) {
433 nseg--;
434 segs++;
435 pg++;
436 pg->physaddr = segs->ds_addr & ~(USB_PAGE_SIZE - 1);
437 }
438
439done:
440 owned = mtx_owned(uptag->mtx);
441 if (!owned)
442 mtx_lock(uptag->mtx);
443
444 uptag->dma_error = (error ? 1 : 0);
445 if (isload) {
446 (uptag->func) (uptag);
447 } else {
448 usb2_cv_broadcast(uptag->cv);
449 }
450 if (!owned)
451 mtx_unlock(uptag->mtx);
452 return;
453}
454
455/*------------------------------------------------------------------------*
456 * usb2_pc_alloc_mem - allocate DMA'able memory
457 *
458 * Returns:
459 * 0: Success
460 * Else: Failure
461 *------------------------------------------------------------------------*/
462uint8_t
463usb2_pc_alloc_mem(struct usb2_page_cache *pc, struct usb2_page *pg,
464 uint32_t size, uint32_t align)
465{
466 struct usb2_dma_parent_tag *uptag;
467 struct usb2_dma_tag *utag;
468 bus_dmamap_t map;
469 void *ptr;
470 int err;
471
472 uptag = pc->tag_parent;
473
474 if (align != 1) {
475 /*
476 * The alignment must be greater or equal to the
477 * "size" else the object can be split between two
478 * memory pages and we get a problem!
479 */
480 while (align < size) {
481 align *= 2;
482 if (align == 0) {
483 goto error;
484 }
485 }
486#if 1
487 /*
488 * XXX BUS-DMA workaround - FIXME later:
489 *
490 * We assume that that the aligment at this point of
491 * the code is greater than or equal to the size and
492 * less than two times the size, so that if we double
493 * the size, the size will be greater than the
494 * alignment.
495 *
496 * The bus-dma system has a check for "alignment"
497 * being less than "size". If that check fails we end
498 * up using contigmalloc which is page based even for
499 * small allocations. Try to avoid that to save
500 * memory, hence we sometimes to a large number of
501 * small allocations!
502 */
503 if (size <= (USB_PAGE_SIZE / 2)) {
504 size *= 2;
505 }
506#endif
507 }
508 /* get the correct DMA tag */
509 utag = usb2_dma_tag_find(uptag, size, align);
510 if (utag == NULL) {
511 goto error;
512 }
513 /* allocate memory */
514 if (bus_dmamem_alloc(
515 utag->tag, &ptr, (BUS_DMA_WAITOK | BUS_DMA_COHERENT), &map)) {
516 goto error;
517 }
518 /* setup page cache */
519 pc->buffer = ptr;
520 pc->page_start = pg;
521 pc->page_offset_buf = 0;
522 pc->page_offset_end = size;
523 pc->map = map;
524 pc->tag = utag->tag;
525 pc->ismultiseg = (align == 1);
526
527 mtx_lock(uptag->mtx);
528
529 /* load memory into DMA */
530 err = bus_dmamap_load(
531 utag->tag, map, ptr, size, &usb2_pc_alloc_mem_cb,
532 pc, (BUS_DMA_WAITOK | BUS_DMA_COHERENT));
533
534 if (err == EINPROGRESS) {
535 usb2_cv_wait(uptag->cv, uptag->mtx);
536 err = 0;
537 }
538 mtx_unlock(uptag->mtx);
539
540 if (err || uptag->dma_error) {
541 bus_dmamem_free(utag->tag, ptr, map);
542 goto error;
543 }
544 bzero(ptr, size);
545
546 usb2_pc_cpu_flush(pc);
547
548 return (0);
549
550error:
551 /* reset most of the page cache */
552 pc->buffer = NULL;
553 pc->page_start = NULL;
554 pc->page_offset_buf = 0;
555 pc->page_offset_end = 0;
556 pc->map = NULL;
557 pc->tag = NULL;
558 return (1);
559}
560
561/*------------------------------------------------------------------------*
562 * usb2_pc_free_mem - free DMA memory
563 *
564 * This function is NULL safe.
565 *------------------------------------------------------------------------*/
566void
567usb2_pc_free_mem(struct usb2_page_cache *pc)
568{
569 if (pc && pc->buffer) {
570
571 bus_dmamap_unload(pc->tag, pc->map);
572
573 bus_dmamem_free(pc->tag, pc->buffer, pc->map);
574
575 pc->buffer = NULL;
576 }
577 return;
578}
579
580/*------------------------------------------------------------------------*
581 * usb2_pc_load_mem - load virtual memory into DMA
582 *
583 * Return values:
584 * 0: Success
585 * Else: Error
586 *------------------------------------------------------------------------*/
587uint8_t
588usb2_pc_load_mem(struct usb2_page_cache *pc, uint32_t size, uint8_t sync)
589{
590 /* setup page cache */
591 pc->page_offset_buf = 0;
592 pc->page_offset_end = size;
593 pc->ismultiseg = 1;
594
595 mtx_assert(pc->tag_parent->mtx, MA_OWNED);
596
597 if (size > 0) {
598 if (sync) {
599 struct usb2_dma_parent_tag *uptag;
600 int err;
601
602 uptag = pc->tag_parent;
603
604 /*
| 2/*-
3 * Copyright (c) 2008 Hans Petter Selasky. All rights reserved.
4 *
5 * Redistribution and use in source and binary forms, with or without
6 * modification, are permitted provided that the following conditions
7 * are met:
8 * 1. Redistributions of source code must retain the above copyright
9 * notice, this list of conditions and the following disclaimer.
10 * 2. Redistributions in binary form must reproduce the above copyright
11 * notice, this list of conditions and the following disclaimer in the
12 * documentation and/or other materials provided with the distribution.
13 *
14 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
15 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
16 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
17 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
18 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
19 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
20 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
21 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
22 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
23 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
24 * SUCH DAMAGE.
25 */
26
27#include <dev/usb2/include/usb2_mfunc.h>
28#include <dev/usb2/include/usb2_error.h>
29#include <dev/usb2/include/usb2_standard.h>
30#include <dev/usb2/include/usb2_defs.h>
31
32#include <dev/usb2/core/usb2_core.h>
33#include <dev/usb2/core/usb2_busdma.h>
34#include <dev/usb2/core/usb2_process.h>
35#include <dev/usb2/core/usb2_transfer.h>
36#include <dev/usb2/core/usb2_device.h>
37#include <dev/usb2/core/usb2_util.h>
38
39#include <dev/usb2/controller/usb2_controller.h>
40#include <dev/usb2/controller/usb2_bus.h>
41
42static void usb2_dma_tag_create(struct usb2_dma_tag *udt, uint32_t size, uint32_t align);
43static void usb2_dma_tag_destroy(struct usb2_dma_tag *udt);
44
45#ifdef __FreeBSD__
46static void usb2_dma_lock_cb(void *arg, bus_dma_lock_op_t op);
47static int32_t usb2_m_copy_in_cb(void *arg, void *src, uint32_t count);
48static void usb2_pc_alloc_mem_cb(void *arg, bus_dma_segment_t *segs, int nseg, int error);
49static void usb2_pc_load_mem_cb(void *arg, bus_dma_segment_t *segs, int nseg, int error);
50static void usb2_pc_common_mem_cb(void *arg, bus_dma_segment_t *segs, int nseg, int error, uint8_t isload);
51
52#endif
53
54#ifdef __NetBSD__
55static int32_t usb2_m_copy_in_cb(void *arg, caddr_t src, uint32_t count);
56static void usb2_pc_common_mem_cb(struct usb2_page_cache *pc, bus_dma_segment_t *segs, int nseg, int error, uint8_t isload);
57
58#endif
59
60/*------------------------------------------------------------------------*
61 * usb2_get_page - lookup DMA-able memory for the given offset
62 *
63 * NOTE: Only call this function when the "page_cache" structure has
64 * been properly initialized !
65 *------------------------------------------------------------------------*/
66void
67usb2_get_page(struct usb2_page_cache *pc, uint32_t offset,
68 struct usb2_page_search *res)
69{
70 struct usb2_page *page;
71
72 if (pc->page_start) {
73
74 /* Case 1 - something has been loaded into DMA */
75
76 if (pc->buffer) {
77
78 /* Case 1a - Kernel Virtual Address */
79
80 res->buffer = USB_ADD_BYTES(pc->buffer, offset);
81 }
82 offset += pc->page_offset_buf;
83
84 /* compute destination page */
85
86 page = pc->page_start;
87
88 if (pc->ismultiseg) {
89
90 page += (offset / USB_PAGE_SIZE);
91
92 offset %= USB_PAGE_SIZE;
93
94 res->length = USB_PAGE_SIZE - offset;
95 res->physaddr = page->physaddr + offset;
96 } else {
97 res->length = 0 - 1;
98 res->physaddr = page->physaddr + offset;
99 }
100 if (!pc->buffer) {
101
102 /* Case 1b - Non Kernel Virtual Address */
103
104 res->buffer = USB_ADD_BYTES(page->buffer, offset);
105 }
106 } else {
107
108 /* Case 2 - Plain PIO */
109
110 res->buffer = USB_ADD_BYTES(pc->buffer, offset);
111 res->length = 0 - 1;
112 res->physaddr = 0;
113 }
114 return;
115}
116
117/*------------------------------------------------------------------------*
118 * usb2_copy_in - copy directly to DMA-able memory
119 *------------------------------------------------------------------------*/
120void
121usb2_copy_in(struct usb2_page_cache *cache, uint32_t offset,
122 const void *ptr, uint32_t len)
123{
124 struct usb2_page_search buf_res;
125
126 while (len != 0) {
127
128 usb2_get_page(cache, offset, &buf_res);
129
130 if (buf_res.length > len) {
131 buf_res.length = len;
132 }
133 bcopy(ptr, buf_res.buffer, buf_res.length);
134
135 offset += buf_res.length;
136 len -= buf_res.length;
137 ptr = USB_ADD_BYTES(ptr, buf_res.length);
138 }
139 return;
140}
141
142/*------------------------------------------------------------------------*
143 * usb2_copy_in_user - copy directly to DMA-able memory from userland
144 *
145 * Return values:
146 * 0: Success
147 * Else: Failure
148 *------------------------------------------------------------------------*/
149int
150usb2_copy_in_user(struct usb2_page_cache *cache, uint32_t offset,
151 const void *ptr, uint32_t len)
152{
153 struct usb2_page_search buf_res;
154 int error;
155
156 while (len != 0) {
157
158 usb2_get_page(cache, offset, &buf_res);
159
160 if (buf_res.length > len) {
161 buf_res.length = len;
162 }
163 error = copyin(ptr, buf_res.buffer, buf_res.length);
164 if (error)
165 return (error);
166
167 offset += buf_res.length;
168 len -= buf_res.length;
169 ptr = USB_ADD_BYTES(ptr, buf_res.length);
170 }
171 return (0); /* success */
172}
173
174/*------------------------------------------------------------------------*
175 * usb2_m_copy_in - copy a mbuf chain directly into DMA-able memory
176 *------------------------------------------------------------------------*/
177struct usb2_m_copy_in_arg {
178 struct usb2_page_cache *cache;
179 uint32_t dst_offset;
180};
181
182static int32_t
183#ifdef __FreeBSD__
184usb2_m_copy_in_cb(void *arg, void *src, uint32_t count)
185#else
186usb2_m_copy_in_cb(void *arg, caddr_t src, uint32_t count)
187#endif
188{
189 register struct usb2_m_copy_in_arg *ua = arg;
190
191 usb2_copy_in(ua->cache, ua->dst_offset, src, count);
192 ua->dst_offset += count;
193 return (0);
194}
195
196void
197usb2_m_copy_in(struct usb2_page_cache *cache, uint32_t dst_offset,
198 struct mbuf *m, uint32_t src_offset, uint32_t src_len)
199{
200 struct usb2_m_copy_in_arg arg = {cache, dst_offset};
201 register int error;
202
203 error = m_apply(m, src_offset, src_len, &usb2_m_copy_in_cb, &arg);
204 return;
205}
206
207/*------------------------------------------------------------------------*
208 * usb2_uiomove - factored out code
209 *------------------------------------------------------------------------*/
210int
211usb2_uiomove(struct usb2_page_cache *pc, struct uio *uio,
212 uint32_t pc_offset, uint32_t len)
213{
214 struct usb2_page_search res;
215 int error = 0;
216
217 while (len != 0) {
218
219 usb2_get_page(pc, pc_offset, &res);
220
221 if (res.length > len) {
222 res.length = len;
223 }
224 /*
225 * "uiomove()" can sleep so one needs to make a wrapper,
226 * exiting the mutex and checking things
227 */
228 error = uiomove(res.buffer, res.length, uio);
229
230 if (error) {
231 break;
232 }
233 pc_offset += res.length;
234 len -= res.length;
235 }
236 return (error);
237}
238
239/*------------------------------------------------------------------------*
240 * usb2_copy_out - copy directly from DMA-able memory
241 *------------------------------------------------------------------------*/
242void
243usb2_copy_out(struct usb2_page_cache *cache, uint32_t offset,
244 void *ptr, uint32_t len)
245{
246 struct usb2_page_search res;
247
248 while (len != 0) {
249
250 usb2_get_page(cache, offset, &res);
251
252 if (res.length > len) {
253 res.length = len;
254 }
255 bcopy(res.buffer, ptr, res.length);
256
257 offset += res.length;
258 len -= res.length;
259 ptr = USB_ADD_BYTES(ptr, res.length);
260 }
261 return;
262}
263
264/*------------------------------------------------------------------------*
265 * usb2_copy_out_user - copy directly from DMA-able memory to userland
266 *
267 * Return values:
268 * 0: Success
269 * Else: Failure
270 *------------------------------------------------------------------------*/
271int
272usb2_copy_out_user(struct usb2_page_cache *cache, uint32_t offset,
273 void *ptr, uint32_t len)
274{
275 struct usb2_page_search res;
276 int error;
277
278 while (len != 0) {
279
280 usb2_get_page(cache, offset, &res);
281
282 if (res.length > len) {
283 res.length = len;
284 }
285 error = copyout(res.buffer, ptr, res.length);
286 if (error)
287 return (error);
288
289 offset += res.length;
290 len -= res.length;
291 ptr = USB_ADD_BYTES(ptr, res.length);
292 }
293 return (0); /* success */
294}
295
296/*------------------------------------------------------------------------*
297 * usb2_bzero - zero DMA-able memory
298 *------------------------------------------------------------------------*/
299void
300usb2_bzero(struct usb2_page_cache *cache, uint32_t offset, uint32_t len)
301{
302 struct usb2_page_search res;
303
304 while (len != 0) {
305
306 usb2_get_page(cache, offset, &res);
307
308 if (res.length > len) {
309 res.length = len;
310 }
311 bzero(res.buffer, res.length);
312
313 offset += res.length;
314 len -= res.length;
315 }
316 return;
317}
318
319
320#ifdef __FreeBSD__
321
322/*------------------------------------------------------------------------*
323 * usb2_dma_lock_cb - dummy callback
324 *------------------------------------------------------------------------*/
325static void
326usb2_dma_lock_cb(void *arg, bus_dma_lock_op_t op)
327{
328 /* we use "mtx_owned()" instead of this function */
329 return;
330}
331
332/*------------------------------------------------------------------------*
333 * usb2_dma_tag_create - allocate a DMA tag
334 *
335 * NOTE: If the "align" parameter has a value of 1 the DMA-tag will
336 * allow multi-segment mappings. Else all mappings are single-segment.
337 *------------------------------------------------------------------------*/
338static void
339usb2_dma_tag_create(struct usb2_dma_tag *udt,
340 uint32_t size, uint32_t align)
341{
342 bus_dma_tag_t tag;
343
344 if (bus_dma_tag_create
345 ( /* parent */ udt->tag_parent->tag,
346 /* alignment */ align,
347 /* boundary */ USB_PAGE_SIZE,
348 /* lowaddr */ (2ULL << (udt->tag_parent->dma_bits - 1)) - 1,
349 /* highaddr */ BUS_SPACE_MAXADDR,
350 /* filter */ NULL,
351 /* filterarg */ NULL,
352 /* maxsize */ size,
353 /* nsegments */ (align == 1) ?
354 (2 + (size / USB_PAGE_SIZE)) : 1,
355 /* maxsegsz */ (align == 1) ?
356 USB_PAGE_SIZE : size,
357 /* flags */ 0,
358 /* lockfn */ &usb2_dma_lock_cb,
359 /* lockarg */ NULL,
360 &tag)) {
361 tag = NULL;
362 }
363 udt->tag = tag;
364 return;
365}
366
367/*------------------------------------------------------------------------*
368 * usb2_dma_tag_free - free a DMA tag
369 *------------------------------------------------------------------------*/
370static void
371usb2_dma_tag_destroy(struct usb2_dma_tag *udt)
372{
373 bus_dma_tag_destroy(udt->tag);
374 return;
375}
376
377/*------------------------------------------------------------------------*
378 * usb2_pc_alloc_mem_cb - BUS-DMA callback function
379 *------------------------------------------------------------------------*/
380static void
381usb2_pc_alloc_mem_cb(void *arg, bus_dma_segment_t *segs,
382 int nseg, int error)
383{
384 usb2_pc_common_mem_cb(arg, segs, nseg, error, 0);
385 return;
386}
387
388/*------------------------------------------------------------------------*
389 * usb2_pc_load_mem_cb - BUS-DMA callback function
390 *------------------------------------------------------------------------*/
391static void
392usb2_pc_load_mem_cb(void *arg, bus_dma_segment_t *segs,
393 int nseg, int error)
394{
395 usb2_pc_common_mem_cb(arg, segs, nseg, error, 1);
396 return;
397}
398
399/*------------------------------------------------------------------------*
400 * usb2_pc_common_mem_cb - BUS-DMA callback function
401 *------------------------------------------------------------------------*/
402static void
403usb2_pc_common_mem_cb(void *arg, bus_dma_segment_t *segs,
404 int nseg, int error, uint8_t isload)
405{
406 struct usb2_dma_parent_tag *uptag;
407 struct usb2_page_cache *pc;
408 struct usb2_page *pg;
409 uint32_t rem;
410 uint8_t owned;
411
412 pc = arg;
413 uptag = pc->tag_parent;
414
415 /*
416 * XXX There is sometimes recursive locking here.
417 * XXX We should try to find a better solution.
418 * XXX Until further the "owned" variable does
419 * XXX the trick.
420 */
421
422 if (error) {
423 goto done;
424 }
425 pg = pc->page_start;
426 pg->physaddr = segs->ds_addr & ~(USB_PAGE_SIZE - 1);
427 rem = segs->ds_addr & (USB_PAGE_SIZE - 1);
428 pc->page_offset_buf = rem;
429 pc->page_offset_end += rem;
430 nseg--;
431
432 while (nseg > 0) {
433 nseg--;
434 segs++;
435 pg++;
436 pg->physaddr = segs->ds_addr & ~(USB_PAGE_SIZE - 1);
437 }
438
439done:
440 owned = mtx_owned(uptag->mtx);
441 if (!owned)
442 mtx_lock(uptag->mtx);
443
444 uptag->dma_error = (error ? 1 : 0);
445 if (isload) {
446 (uptag->func) (uptag);
447 } else {
448 usb2_cv_broadcast(uptag->cv);
449 }
450 if (!owned)
451 mtx_unlock(uptag->mtx);
452 return;
453}
454
455/*------------------------------------------------------------------------*
456 * usb2_pc_alloc_mem - allocate DMA'able memory
457 *
458 * Returns:
459 * 0: Success
460 * Else: Failure
461 *------------------------------------------------------------------------*/
462uint8_t
463usb2_pc_alloc_mem(struct usb2_page_cache *pc, struct usb2_page *pg,
464 uint32_t size, uint32_t align)
465{
466 struct usb2_dma_parent_tag *uptag;
467 struct usb2_dma_tag *utag;
468 bus_dmamap_t map;
469 void *ptr;
470 int err;
471
472 uptag = pc->tag_parent;
473
474 if (align != 1) {
475 /*
476 * The alignment must be greater or equal to the
477 * "size" else the object can be split between two
478 * memory pages and we get a problem!
479 */
480 while (align < size) {
481 align *= 2;
482 if (align == 0) {
483 goto error;
484 }
485 }
486#if 1
487 /*
488 * XXX BUS-DMA workaround - FIXME later:
489 *
490 * We assume that that the aligment at this point of
491 * the code is greater than or equal to the size and
492 * less than two times the size, so that if we double
493 * the size, the size will be greater than the
494 * alignment.
495 *
496 * The bus-dma system has a check for "alignment"
497 * being less than "size". If that check fails we end
498 * up using contigmalloc which is page based even for
499 * small allocations. Try to avoid that to save
500 * memory, hence we sometimes to a large number of
501 * small allocations!
502 */
503 if (size <= (USB_PAGE_SIZE / 2)) {
504 size *= 2;
505 }
506#endif
507 }
508 /* get the correct DMA tag */
509 utag = usb2_dma_tag_find(uptag, size, align);
510 if (utag == NULL) {
511 goto error;
512 }
513 /* allocate memory */
514 if (bus_dmamem_alloc(
515 utag->tag, &ptr, (BUS_DMA_WAITOK | BUS_DMA_COHERENT), &map)) {
516 goto error;
517 }
518 /* setup page cache */
519 pc->buffer = ptr;
520 pc->page_start = pg;
521 pc->page_offset_buf = 0;
522 pc->page_offset_end = size;
523 pc->map = map;
524 pc->tag = utag->tag;
525 pc->ismultiseg = (align == 1);
526
527 mtx_lock(uptag->mtx);
528
529 /* load memory into DMA */
530 err = bus_dmamap_load(
531 utag->tag, map, ptr, size, &usb2_pc_alloc_mem_cb,
532 pc, (BUS_DMA_WAITOK | BUS_DMA_COHERENT));
533
534 if (err == EINPROGRESS) {
535 usb2_cv_wait(uptag->cv, uptag->mtx);
536 err = 0;
537 }
538 mtx_unlock(uptag->mtx);
539
540 if (err || uptag->dma_error) {
541 bus_dmamem_free(utag->tag, ptr, map);
542 goto error;
543 }
544 bzero(ptr, size);
545
546 usb2_pc_cpu_flush(pc);
547
548 return (0);
549
550error:
551 /* reset most of the page cache */
552 pc->buffer = NULL;
553 pc->page_start = NULL;
554 pc->page_offset_buf = 0;
555 pc->page_offset_end = 0;
556 pc->map = NULL;
557 pc->tag = NULL;
558 return (1);
559}
560
561/*------------------------------------------------------------------------*
562 * usb2_pc_free_mem - free DMA memory
563 *
564 * This function is NULL safe.
565 *------------------------------------------------------------------------*/
566void
567usb2_pc_free_mem(struct usb2_page_cache *pc)
568{
569 if (pc && pc->buffer) {
570
571 bus_dmamap_unload(pc->tag, pc->map);
572
573 bus_dmamem_free(pc->tag, pc->buffer, pc->map);
574
575 pc->buffer = NULL;
576 }
577 return;
578}
579
580/*------------------------------------------------------------------------*
581 * usb2_pc_load_mem - load virtual memory into DMA
582 *
583 * Return values:
584 * 0: Success
585 * Else: Error
586 *------------------------------------------------------------------------*/
587uint8_t
588usb2_pc_load_mem(struct usb2_page_cache *pc, uint32_t size, uint8_t sync)
589{
590 /* setup page cache */
591 pc->page_offset_buf = 0;
592 pc->page_offset_end = size;
593 pc->ismultiseg = 1;
594
595 mtx_assert(pc->tag_parent->mtx, MA_OWNED);
596
597 if (size > 0) {
598 if (sync) {
599 struct usb2_dma_parent_tag *uptag;
600 int err;
601
602 uptag = pc->tag_parent;
603
604 /*
|
| 605 * We have to unload the previous loaded DMA
606 * pages before trying to load a new one!
607 */
608 bus_dmamap_unload(pc->tag, pc->map);
609
610 /*
|
605 * Try to load memory into DMA.
606 */
607 err = bus_dmamap_load(
608 pc->tag, pc->map, pc->buffer, size,
609 &usb2_pc_alloc_mem_cb, pc, BUS_DMA_WAITOK);
610 if (err == EINPROGRESS) {
611 usb2_cv_wait(uptag->cv, uptag->mtx);
612 err = 0;
613 }
614 if (err || uptag->dma_error) {
615 return (1);
616 }
617 } else {
618
619 /*
| 611 * Try to load memory into DMA.
612 */
613 err = bus_dmamap_load(
614 pc->tag, pc->map, pc->buffer, size,
615 &usb2_pc_alloc_mem_cb, pc, BUS_DMA_WAITOK);
616 if (err == EINPROGRESS) {
617 usb2_cv_wait(uptag->cv, uptag->mtx);
618 err = 0;
619 }
620 if (err || uptag->dma_error) {
621 return (1);
622 }
623 } else {
624
625 /*
|
| 626 * We have to unload the previous loaded DMA
627 * pages before trying to load a new one!
628 */
629 bus_dmamap_unload(pc->tag, pc->map);
630
631 /*
|
620 * Try to load memory into DMA. The callback
621 * will be called in all cases:
622 */
623 if (bus_dmamap_load(
624 pc->tag, pc->map, pc->buffer, size,
625 &usb2_pc_load_mem_cb, pc, BUS_DMA_WAITOK)) {
626 }
627 }
628 } else {
629 if (!sync) {
630 /*
631 * Call callback so that refcount is decremented
632 * properly:
633 */
634 pc->tag_parent->dma_error = 0;
635 (pc->tag_parent->func) (pc->tag_parent);
636 }
637 }
638 return (0);
639}
640
641/*------------------------------------------------------------------------*
642 * usb2_pc_cpu_invalidate - invalidate CPU cache
643 *------------------------------------------------------------------------*/
644void
645usb2_pc_cpu_invalidate(struct usb2_page_cache *pc)
646{
| 632 * Try to load memory into DMA. The callback
633 * will be called in all cases:
634 */
635 if (bus_dmamap_load(
636 pc->tag, pc->map, pc->buffer, size,
637 &usb2_pc_load_mem_cb, pc, BUS_DMA_WAITOK)) {
638 }
639 }
640 } else {
641 if (!sync) {
642 /*
643 * Call callback so that refcount is decremented
644 * properly:
645 */
646 pc->tag_parent->dma_error = 0;
647 (pc->tag_parent->func) (pc->tag_parent);
648 }
649 }
650 return (0);
651}
652
653/*------------------------------------------------------------------------*
654 * usb2_pc_cpu_invalidate - invalidate CPU cache
655 *------------------------------------------------------------------------*/
656void
657usb2_pc_cpu_invalidate(struct usb2_page_cache *pc)
658{
|
| 659 if (pc->page_offset_end == pc->page_offset_buf) {
660 /* nothing has been loaded into this page cache! */
661 return;
662 }
|
647 bus_dmamap_sync(pc->tag, pc->map,
648 BUS_DMASYNC_POSTWRITE | BUS_DMASYNC_POSTREAD);
649 return;
650}
651
652/*------------------------------------------------------------------------*
653 * usb2_pc_cpu_flush - flush CPU cache
654 *------------------------------------------------------------------------*/
655void
656usb2_pc_cpu_flush(struct usb2_page_cache *pc)
657{
| 663 bus_dmamap_sync(pc->tag, pc->map,
664 BUS_DMASYNC_POSTWRITE | BUS_DMASYNC_POSTREAD);
665 return;
666}
667
668/*------------------------------------------------------------------------*
669 * usb2_pc_cpu_flush - flush CPU cache
670 *------------------------------------------------------------------------*/
671void
672usb2_pc_cpu_flush(struct usb2_page_cache *pc)
673{
|
| 674 if (pc->page_offset_end == pc->page_offset_buf) {
675 /* nothing has been loaded into this page cache! */
676 return;
677 }
|
658 bus_dmamap_sync(pc->tag, pc->map,
659 BUS_DMASYNC_PREWRITE | BUS_DMASYNC_PREREAD);
660 return;
661}
662
663/*------------------------------------------------------------------------*
664 * usb2_pc_dmamap_create - create a DMA map
665 *
666 * Returns:
667 * 0: Success
668 * Else: Failure
669 *------------------------------------------------------------------------*/
670uint8_t
671usb2_pc_dmamap_create(struct usb2_page_cache *pc, uint32_t size)
672{
673 struct usb2_xfer_root *info;
674 struct usb2_dma_tag *utag;
675
676 /* get info */
677 info = pc->tag_parent->info;
678
679 /* sanity check */
680 if (info == NULL) {
681 goto error;
682 }
683 utag = usb2_dma_tag_find(pc->tag_parent, size, 1);
684 if (utag == NULL) {
685 goto error;
686 }
687 /* create DMA map */
688 if (bus_dmamap_create(utag->tag, 0, &pc->map)) {
689 goto error;
690 }
691 pc->tag = utag->tag;
692 return 0; /* success */
693
694error:
695 pc->map = NULL;
696 pc->tag = NULL;
697 return 1; /* failure */
698}
699
700/*------------------------------------------------------------------------*
701 * usb2_pc_dmamap_destroy
702 *
703 * This function is NULL safe.
704 *------------------------------------------------------------------------*/
705void
706usb2_pc_dmamap_destroy(struct usb2_page_cache *pc)
707{
708 if (pc && pc->tag) {
709 bus_dmamap_destroy(pc->tag, pc->map);
710 pc->tag = NULL;
711 pc->map = NULL;
712 }
713 return;
714}
715
716#endif
717
718#ifdef __NetBSD__
719
720/*------------------------------------------------------------------------*
721 * usb2_dma_tag_create - allocate a DMA tag
722 *
723 * NOTE: If the "align" parameter has a value of 1 the DMA-tag will
724 * allow multi-segment mappings. Else all mappings are single-segment.
725 *------------------------------------------------------------------------*/
726static void
727usb2_dma_tag_create(struct usb2_dma_tag *udt,
728 uint32_t size, uint32_t align)
729{
730 uint32_t nseg;
731
732 if (align == 1) {
733 nseg = (2 + (size / USB_PAGE_SIZE));
734 } else {
735 nseg = 1;
736 }
737
738 udt->p_seg = malloc(nseg * sizeof(*(udt->p_seg)),
739 M_USB, M_WAITOK | M_ZERO);
740
741 if (udt->p_seg == NULL) {
742 return;
743 }
744 udt->tag = udt->tag_parent->tag;
745 udt->n_seg = nseg;
746 return;
747}
748
749/*------------------------------------------------------------------------*
750 * usb2_dma_tag_free - free a DMA tag
751 *------------------------------------------------------------------------*/
752static void
753usb2_dma_tag_destroy(struct usb2_dma_tag *udt)
754{
755 free(udt->p_seg, M_USB);
756 return;
757}
758
759/*------------------------------------------------------------------------*
760 * usb2_pc_common_mem_cb - BUS-DMA callback function
761 *------------------------------------------------------------------------*/
762static void
763usb2_pc_common_mem_cb(struct usb2_page_cache *pc, bus_dma_segment_t *segs,
764 int nseg, int error, uint8_t isload, uint8_t dolock)
765{
766 struct usb2_dma_parent_tag *uptag;
767 struct usb2_page *pg;
768 uint32_t rem;
769 uint8_t ext_seg; /* extend last segment */
770
771 uptag = pc->tag_parent;
772
773 if (error) {
774 goto done;
775 }
776 pg = pc->page_start;
777 pg->physaddr = segs->ds_addr & ~(USB_PAGE_SIZE - 1);
778 rem = segs->ds_addr & (USB_PAGE_SIZE - 1);
779 pc->page_offset_buf = rem;
780 pc->page_offset_end += rem;
781 if (nseg < ((pc->page_offset_end +
782 (USB_PAGE_SIZE - 1)) / USB_PAGE_SIZE)) {
783 ext_seg = 1;
784 } else {
785 ext_seg = 0;
786 }
787 nseg--;
788
789 while (nseg > 0) {
790 nseg--;
791 segs++;
792 pg++;
793 pg->physaddr = segs->ds_addr & ~(USB_PAGE_SIZE - 1);
794 }
795
796 /*
797 * XXX The segments we get from BUS-DMA are not aligned,
798 * XXX so we need to extend the last segment if we are
799 * XXX unaligned and cross the segment boundary!
800 */
801 if (ext_seg && pc->ismultiseg) {
802 (pg + 1)->physaddr = pg->physaddr + USB_PAGE_SIZE;
803 }
804done:
805 if (dolock)
806 mtx_lock(uptag->mtx);
807
808 uptag->dma_error = (error ? 1 : 0);
809 if (isload) {
810 (uptag->func) (uptag);
811 }
812 if (dolock)
813 mtx_unlock(uptag->mtx);
814 return;
815}
816
817/*------------------------------------------------------------------------*
818 * usb2_pc_alloc_mem - allocate DMA'able memory
819 *
820 * Returns:
821 * 0: Success
822 * Else: Failure
823 *------------------------------------------------------------------------*/
824uint8_t
825usb2_pc_alloc_mem(struct usb2_page_cache *pc, struct usb2_page *pg,
826 uint32_t size, uint32_t align)
827{
828 struct usb2_dma_parent_tag *uptag;
829 struct usb2_dma_tag *utag;
830 caddr_t ptr = NULL;
831 bus_dmamap_t map;
832 int seg_count;
833
834 uptag = pc->tag_parent;
835
836 if (align != 1) {
837 /*
838 * The alignment must be greater or equal to the
839 * "size" else the object can be split between two
840 * memory pages and we get a problem!
841 */
842 while (align < size) {
843 align *= 2;
844 if (align == 0) {
845 goto done_5;
846 }
847 }
848 }
849 /* get the correct DMA tag */
850 utag = usb2_dma_tag_find(pc->tag_parent, size, align);
851 if (utag == NULL) {
852 goto done_5;
853 }
854 if (bus_dmamem_alloc(utag->tag, size, align, 0, utag->p_seg,
855 utag->n_seg, &seg_count, BUS_DMA_WAITOK)) {
856 goto done_4;
857 }
858 if (bus_dmamem_map(utag->tag, utag->p_seg, seg_count, size,
859 &ptr, BUS_DMA_WAITOK | BUS_DMA_COHERENT)) {
860 goto done_3;
861 }
862 if (bus_dmamap_create(utag->tag, size, utag->n_seg, (align == 1) ?
863 USB_PAGE_SIZE : size, 0, BUS_DMA_WAITOK, &map)) {
864 goto done_2;
865 }
866 if (bus_dmamap_load(utag->tag, map, ptr, size, NULL,
867 BUS_DMA_WAITOK)) {
868 goto done_1;
869 }
870 pc->p_seg = malloc(seg_count * sizeof(*(pc->p_seg)),
871 M_USB, M_WAITOK | M_ZERO);
872 if (pc->p_seg == NULL) {
873 goto done_0;
874 }
875 /* store number if actual segments used */
876 pc->n_seg = seg_count;
877
878 /* make a copy of the segments */
879 bcopy(utag->p_seg, pc->p_seg,
880 seg_count * sizeof(*(pc->p_seg)));
881
882 /* setup page cache */
883 pc->buffer = ptr;
884 pc->page_start = pg;
885 pc->page_offset_buf = 0;
886 pc->page_offset_end = size;
887 pc->map = map;
888 pc->tag = utag->tag;
889 pc->ismultiseg = (align == 1);
890
891 usb2_pc_common_mem_cb(pc, utag->p_seg, seg_count, 0, 0, 1);
892
893 bzero(ptr, size);
894
895 usb2_pc_cpu_flush(pc);
896
897 return (0);
898
899done_0:
900 bus_dmamap_unload(utag->tag, map);
901done_1:
902 bus_dmamap_destroy(utag->tag, map);
903done_2:
904 bus_dmamem_unmap(utag->tag, ptr, size);
905done_3:
906 bus_dmamem_free(utag->tag, utag->p_seg, seg_count);
907done_4:
908 /* utag is destroyed later */
909done_5:
910 /* reset most of the page cache */
911 pc->buffer = NULL;
912 pc->page_start = NULL;
913 pc->page_offset_buf = 0;
914 pc->page_offset_end = 0;
915 pc->map = NULL;
916 pc->tag = NULL;
917 pc->n_seg = 0;
918 pc->p_seg = NULL;
919 return (1);
920}
921
922/*------------------------------------------------------------------------*
923 * usb2_pc_free_mem - free DMA memory
924 *
925 * This function is NULL safe.
926 *------------------------------------------------------------------------*/
927void
928usb2_pc_free_mem(struct usb2_page_cache *pc)
929{
930 if (pc && pc->buffer) {
931 bus_dmamap_unload(pc->tag, pc->map);
932 bus_dmamap_destroy(pc->tag, pc->map);
933 bus_dmamem_unmap(pc->tag, pc->buffer,
934 pc->page_offset_end - pc->page_offset_buf);
935 bus_dmamem_free(pc->tag, pc->p_seg, pc->n_seg);
936 free(pc->p_seg, M_USB);
937 pc->buffer = NULL;
938 }
939 return;
940}
941
942/*------------------------------------------------------------------------*
943 * usb2_pc_load_mem - load virtual memory into DMA
944 *
945 * Return values:
946 * 0: Success
947 * Else: Error
948 *------------------------------------------------------------------------*/
949uint8_t
950usb2_pc_load_mem(struct usb2_page_cache *pc, uint32_t size, uint8_t sync)
951{
952 int error;
953
954 /* setup page cache */
955 pc->page_offset_buf = 0;
956 pc->page_offset_end = size;
957 pc->ismultiseg = 1;
958
959 if (size > 0) {
960
| 678 bus_dmamap_sync(pc->tag, pc->map,
679 BUS_DMASYNC_PREWRITE | BUS_DMASYNC_PREREAD);
680 return;
681}
682
683/*------------------------------------------------------------------------*
684 * usb2_pc_dmamap_create - create a DMA map
685 *
686 * Returns:
687 * 0: Success
688 * Else: Failure
689 *------------------------------------------------------------------------*/
690uint8_t
691usb2_pc_dmamap_create(struct usb2_page_cache *pc, uint32_t size)
692{
693 struct usb2_xfer_root *info;
694 struct usb2_dma_tag *utag;
695
696 /* get info */
697 info = pc->tag_parent->info;
698
699 /* sanity check */
700 if (info == NULL) {
701 goto error;
702 }
703 utag = usb2_dma_tag_find(pc->tag_parent, size, 1);
704 if (utag == NULL) {
705 goto error;
706 }
707 /* create DMA map */
708 if (bus_dmamap_create(utag->tag, 0, &pc->map)) {
709 goto error;
710 }
711 pc->tag = utag->tag;
712 return 0; /* success */
713
714error:
715 pc->map = NULL;
716 pc->tag = NULL;
717 return 1; /* failure */
718}
719
720/*------------------------------------------------------------------------*
721 * usb2_pc_dmamap_destroy
722 *
723 * This function is NULL safe.
724 *------------------------------------------------------------------------*/
725void
726usb2_pc_dmamap_destroy(struct usb2_page_cache *pc)
727{
728 if (pc && pc->tag) {
729 bus_dmamap_destroy(pc->tag, pc->map);
730 pc->tag = NULL;
731 pc->map = NULL;
732 }
733 return;
734}
735
736#endif
737
738#ifdef __NetBSD__
739
740/*------------------------------------------------------------------------*
741 * usb2_dma_tag_create - allocate a DMA tag
742 *
743 * NOTE: If the "align" parameter has a value of 1 the DMA-tag will
744 * allow multi-segment mappings. Else all mappings are single-segment.
745 *------------------------------------------------------------------------*/
746static void
747usb2_dma_tag_create(struct usb2_dma_tag *udt,
748 uint32_t size, uint32_t align)
749{
750 uint32_t nseg;
751
752 if (align == 1) {
753 nseg = (2 + (size / USB_PAGE_SIZE));
754 } else {
755 nseg = 1;
756 }
757
758 udt->p_seg = malloc(nseg * sizeof(*(udt->p_seg)),
759 M_USB, M_WAITOK | M_ZERO);
760
761 if (udt->p_seg == NULL) {
762 return;
763 }
764 udt->tag = udt->tag_parent->tag;
765 udt->n_seg = nseg;
766 return;
767}
768
769/*------------------------------------------------------------------------*
770 * usb2_dma_tag_free - free a DMA tag
771 *------------------------------------------------------------------------*/
772static void
773usb2_dma_tag_destroy(struct usb2_dma_tag *udt)
774{
775 free(udt->p_seg, M_USB);
776 return;
777}
778
779/*------------------------------------------------------------------------*
780 * usb2_pc_common_mem_cb - BUS-DMA callback function
781 *------------------------------------------------------------------------*/
782static void
783usb2_pc_common_mem_cb(struct usb2_page_cache *pc, bus_dma_segment_t *segs,
784 int nseg, int error, uint8_t isload, uint8_t dolock)
785{
786 struct usb2_dma_parent_tag *uptag;
787 struct usb2_page *pg;
788 uint32_t rem;
789 uint8_t ext_seg; /* extend last segment */
790
791 uptag = pc->tag_parent;
792
793 if (error) {
794 goto done;
795 }
796 pg = pc->page_start;
797 pg->physaddr = segs->ds_addr & ~(USB_PAGE_SIZE - 1);
798 rem = segs->ds_addr & (USB_PAGE_SIZE - 1);
799 pc->page_offset_buf = rem;
800 pc->page_offset_end += rem;
801 if (nseg < ((pc->page_offset_end +
802 (USB_PAGE_SIZE - 1)) / USB_PAGE_SIZE)) {
803 ext_seg = 1;
804 } else {
805 ext_seg = 0;
806 }
807 nseg--;
808
809 while (nseg > 0) {
810 nseg--;
811 segs++;
812 pg++;
813 pg->physaddr = segs->ds_addr & ~(USB_PAGE_SIZE - 1);
814 }
815
816 /*
817 * XXX The segments we get from BUS-DMA are not aligned,
818 * XXX so we need to extend the last segment if we are
819 * XXX unaligned and cross the segment boundary!
820 */
821 if (ext_seg && pc->ismultiseg) {
822 (pg + 1)->physaddr = pg->physaddr + USB_PAGE_SIZE;
823 }
824done:
825 if (dolock)
826 mtx_lock(uptag->mtx);
827
828 uptag->dma_error = (error ? 1 : 0);
829 if (isload) {
830 (uptag->func) (uptag);
831 }
832 if (dolock)
833 mtx_unlock(uptag->mtx);
834 return;
835}
836
837/*------------------------------------------------------------------------*
838 * usb2_pc_alloc_mem - allocate DMA'able memory
839 *
840 * Returns:
841 * 0: Success
842 * Else: Failure
843 *------------------------------------------------------------------------*/
844uint8_t
845usb2_pc_alloc_mem(struct usb2_page_cache *pc, struct usb2_page *pg,
846 uint32_t size, uint32_t align)
847{
848 struct usb2_dma_parent_tag *uptag;
849 struct usb2_dma_tag *utag;
850 caddr_t ptr = NULL;
851 bus_dmamap_t map;
852 int seg_count;
853
854 uptag = pc->tag_parent;
855
856 if (align != 1) {
857 /*
858 * The alignment must be greater or equal to the
859 * "size" else the object can be split between two
860 * memory pages and we get a problem!
861 */
862 while (align < size) {
863 align *= 2;
864 if (align == 0) {
865 goto done_5;
866 }
867 }
868 }
869 /* get the correct DMA tag */
870 utag = usb2_dma_tag_find(pc->tag_parent, size, align);
871 if (utag == NULL) {
872 goto done_5;
873 }
874 if (bus_dmamem_alloc(utag->tag, size, align, 0, utag->p_seg,
875 utag->n_seg, &seg_count, BUS_DMA_WAITOK)) {
876 goto done_4;
877 }
878 if (bus_dmamem_map(utag->tag, utag->p_seg, seg_count, size,
879 &ptr, BUS_DMA_WAITOK | BUS_DMA_COHERENT)) {
880 goto done_3;
881 }
882 if (bus_dmamap_create(utag->tag, size, utag->n_seg, (align == 1) ?
883 USB_PAGE_SIZE : size, 0, BUS_DMA_WAITOK, &map)) {
884 goto done_2;
885 }
886 if (bus_dmamap_load(utag->tag, map, ptr, size, NULL,
887 BUS_DMA_WAITOK)) {
888 goto done_1;
889 }
890 pc->p_seg = malloc(seg_count * sizeof(*(pc->p_seg)),
891 M_USB, M_WAITOK | M_ZERO);
892 if (pc->p_seg == NULL) {
893 goto done_0;
894 }
895 /* store number if actual segments used */
896 pc->n_seg = seg_count;
897
898 /* make a copy of the segments */
899 bcopy(utag->p_seg, pc->p_seg,
900 seg_count * sizeof(*(pc->p_seg)));
901
902 /* setup page cache */
903 pc->buffer = ptr;
904 pc->page_start = pg;
905 pc->page_offset_buf = 0;
906 pc->page_offset_end = size;
907 pc->map = map;
908 pc->tag = utag->tag;
909 pc->ismultiseg = (align == 1);
910
911 usb2_pc_common_mem_cb(pc, utag->p_seg, seg_count, 0, 0, 1);
912
913 bzero(ptr, size);
914
915 usb2_pc_cpu_flush(pc);
916
917 return (0);
918
919done_0:
920 bus_dmamap_unload(utag->tag, map);
921done_1:
922 bus_dmamap_destroy(utag->tag, map);
923done_2:
924 bus_dmamem_unmap(utag->tag, ptr, size);
925done_3:
926 bus_dmamem_free(utag->tag, utag->p_seg, seg_count);
927done_4:
928 /* utag is destroyed later */
929done_5:
930 /* reset most of the page cache */
931 pc->buffer = NULL;
932 pc->page_start = NULL;
933 pc->page_offset_buf = 0;
934 pc->page_offset_end = 0;
935 pc->map = NULL;
936 pc->tag = NULL;
937 pc->n_seg = 0;
938 pc->p_seg = NULL;
939 return (1);
940}
941
942/*------------------------------------------------------------------------*
943 * usb2_pc_free_mem - free DMA memory
944 *
945 * This function is NULL safe.
946 *------------------------------------------------------------------------*/
947void
948usb2_pc_free_mem(struct usb2_page_cache *pc)
949{
950 if (pc && pc->buffer) {
951 bus_dmamap_unload(pc->tag, pc->map);
952 bus_dmamap_destroy(pc->tag, pc->map);
953 bus_dmamem_unmap(pc->tag, pc->buffer,
954 pc->page_offset_end - pc->page_offset_buf);
955 bus_dmamem_free(pc->tag, pc->p_seg, pc->n_seg);
956 free(pc->p_seg, M_USB);
957 pc->buffer = NULL;
958 }
959 return;
960}
961
962/*------------------------------------------------------------------------*
963 * usb2_pc_load_mem - load virtual memory into DMA
964 *
965 * Return values:
966 * 0: Success
967 * Else: Error
968 *------------------------------------------------------------------------*/
969uint8_t
970usb2_pc_load_mem(struct usb2_page_cache *pc, uint32_t size, uint8_t sync)
971{
972 int error;
973
974 /* setup page cache */
975 pc->page_offset_buf = 0;
976 pc->page_offset_end = size;
977 pc->ismultiseg = 1;
978
979 if (size > 0) {
980
|
| 981 /*
982 * We have to unload the previous loaded DMA
983 * pages before trying to load a new one!
984 */
985 bus_dmamap_unload(pc->tag, pc->map);
986
|
961 /* try to load memory into DMA using using no wait option */
962 if (bus_dmamap_load(pc->tag, pc->map, pc->buffer,
963 size, NULL, BUS_DMA_NOWAIT)) {
964 error = ENOMEM;
965 } else {
966 error = 0;
967 }
968
969 usb2_pc_common_mem_cb(pc, pc->map->dm_segs,
970 pc->map->dm_nsegs, error, !sync);
971
972 if (error) {
973 return (1);
974 }
975 } else {
976 if (!sync) {
977 /*
978 * Call callback so that refcount is decremented
979 * properly:
980 */
981 pc->tag_parent->dma_error = 0;
982 (pc->tag_parent->func) (pc->tag_parent);
983 }
984 }
985 return (0);
986}
987
988/*------------------------------------------------------------------------*
989 * usb2_pc_cpu_invalidate - invalidate CPU cache
990 *------------------------------------------------------------------------*/
991void
992usb2_pc_cpu_invalidate(struct usb2_page_cache *pc)
993{
994 uint32_t len;
995
996 len = pc->page_offset_end - pc->page_offset_buf;
997
| 987 /* try to load memory into DMA using using no wait option */
988 if (bus_dmamap_load(pc->tag, pc->map, pc->buffer,
989 size, NULL, BUS_DMA_NOWAIT)) {
990 error = ENOMEM;
991 } else {
992 error = 0;
993 }
994
995 usb2_pc_common_mem_cb(pc, pc->map->dm_segs,
996 pc->map->dm_nsegs, error, !sync);
997
998 if (error) {
999 return (1);
1000 }
1001 } else {
1002 if (!sync) {
1003 /*
1004 * Call callback so that refcount is decremented
1005 * properly:
1006 */
1007 pc->tag_parent->dma_error = 0;
1008 (pc->tag_parent->func) (pc->tag_parent);
1009 }
1010 }
1011 return (0);
1012}
1013
1014/*------------------------------------------------------------------------*
1015 * usb2_pc_cpu_invalidate - invalidate CPU cache
1016 *------------------------------------------------------------------------*/
1017void
1018usb2_pc_cpu_invalidate(struct usb2_page_cache *pc)
1019{
1020 uint32_t len;
1021
1022 len = pc->page_offset_end - pc->page_offset_buf;
1023
|
| 1024 if (len == 0) {
1025 /* nothing has been loaded into this page cache */
1026 return;
1027 }
|
998 bus_dmamap_sync(pc->tag, pc->map, 0, len,
999 BUS_DMASYNC_POSTWRITE | BUS_DMASYNC_POSTREAD);
1000 return;
1001}
1002
1003/*------------------------------------------------------------------------*
1004 * usb2_pc_cpu_flush - flush CPU cache
1005 *------------------------------------------------------------------------*/
1006void
1007usb2_pc_cpu_flush(struct usb2_page_cache *pc)
1008{
1009 uint32_t len;
1010
1011 len = pc->page_offset_end - pc->page_offset_buf;
1012
| 1028 bus_dmamap_sync(pc->tag, pc->map, 0, len,
1029 BUS_DMASYNC_POSTWRITE | BUS_DMASYNC_POSTREAD);
1030 return;
1031}
1032
1033/*------------------------------------------------------------------------*
1034 * usb2_pc_cpu_flush - flush CPU cache
1035 *------------------------------------------------------------------------*/
1036void
1037usb2_pc_cpu_flush(struct usb2_page_cache *pc)
1038{
1039 uint32_t len;
1040
1041 len = pc->page_offset_end - pc->page_offset_buf;
1042
|
| 1043 if (len == 0) {
1044 /* nothing has been loaded into this page cache */
1045 return;
1046 }
|
1013 bus_dmamap_sync(pc->tag, pc->map, 0, len,
1014 BUS_DMASYNC_PREWRITE | BUS_DMASYNC_PREREAD);
1015 return;
1016}
1017
1018/*------------------------------------------------------------------------*
1019 * usb2_pc_dmamap_create - create a DMA map
1020 *
1021 * Returns:
1022 * 0: Success
1023 * Else: Failure
1024 *------------------------------------------------------------------------*/
1025uint8_t
1026usb2_pc_dmamap_create(struct usb2_page_cache *pc, uint32_t size)
1027{
1028 struct usb2_xfer_root *info;
1029 struct usb2_dma_tag *utag;
1030
1031 /* get info */
1032 info = pc->tag_parent->info;
1033
1034 /* sanity check */
1035 if (info == NULL) {
1036 goto error;
1037 }
1038 utag = usb2_dma_tag_find(pc->tag_parent, size, 1);
1039 if (utag == NULL) {
1040 goto error;
1041 }
1042 if (bus_dmamap_create(utag->tag, size, utag->n_seg,
1043 USB_PAGE_SIZE, 0, BUS_DMA_WAITOK, &pc->map)) {
1044 goto error;
1045 }
1046 pc->tag = utag->tag;
1047 pc->p_seg = utag->p_seg;
1048 pc->n_seg = utag->n_seg;
1049 return 0; /* success */
1050
1051error:
1052 pc->map = NULL;
1053 pc->tag = NULL;
1054 pc->p_seg = NULL;
1055 pc->n_seg = 0;
1056 return 1; /* failure */
1057}
1058
1059/*------------------------------------------------------------------------*
1060 * usb2_pc_dmamap_destroy
1061 *
1062 * This function is NULL safe.
1063 *------------------------------------------------------------------------*/
1064void
1065usb2_pc_dmamap_destroy(struct usb2_page_cache *pc)
1066{
1067 if (pc && pc->tag) {
1068 bus_dmamap_destroy(pc->tag, pc->map);
1069 pc->tag = NULL;
1070 pc->map = NULL;
1071 }
1072 return;
1073}
1074
1075#endif
1076
1077/*------------------------------------------------------------------------*
1078 * usb2_dma_tag_find - factored out code
1079 *------------------------------------------------------------------------*/
1080struct usb2_dma_tag *
1081usb2_dma_tag_find(struct usb2_dma_parent_tag *udpt,
1082 uint32_t size, uint32_t align)
1083{
1084 struct usb2_dma_tag *udt;
1085 uint8_t nudt;
1086
1087 USB_ASSERT(align > 0, ("Invalid parameter align = 0!\n"));
1088 USB_ASSERT(size > 0, ("Invalid parameter size = 0!\n"));
1089
1090 udt = udpt->utag_first;
1091 nudt = udpt->utag_max;
1092
1093 while (nudt--) {
1094
1095 if (udt->align == 0) {
1096 usb2_dma_tag_create(udt, size, align);
1097 if (udt->tag == NULL) {
1098 return (NULL);
1099 }
1100 udt->align = align;
1101 udt->size = size;
1102 return (udt);
1103 }
1104 if ((udt->align == align) && (udt->size == size)) {
1105 return (udt);
1106 }
1107 udt++;
1108 }
1109 return (NULL);
1110}
1111
1112/*------------------------------------------------------------------------*
1113 * usb2_dma_tag_setup - initialise USB DMA tags
1114 *------------------------------------------------------------------------*/
1115void
1116usb2_dma_tag_setup(struct usb2_dma_parent_tag *udpt,
1117 struct usb2_dma_tag *udt, bus_dma_tag_t dmat,
1118 struct mtx *mtx, usb2_dma_callback_t *func,
1119 struct usb2_xfer_root *info, uint8_t ndmabits,
1120 uint8_t nudt)
1121{
1122 bzero(udpt, sizeof(*udpt));
1123
1124 /* sanity checking */
1125 if ((nudt == 0) ||
1126 (ndmabits == 0) ||
1127 (mtx == NULL)) {
1128 /* something is corrupt */
1129 return;
1130 }
1131#ifdef __FreeBSD__
1132 /* initialise condition variable */
1133 usb2_cv_init(udpt->cv, "USB DMA CV");
1134#endif
1135
1136 /* store some information */
1137 udpt->mtx = mtx;
1138 udpt->info = info;
1139 udpt->func = func;
1140 udpt->tag = dmat;
1141 udpt->utag_first = udt;
1142 udpt->utag_max = nudt;
1143 udpt->dma_bits = ndmabits;
1144
1145 while (nudt--) {
1146 bzero(udt, sizeof(*udt));
1147 udt->tag_parent = udpt;
1148 udt++;
1149 }
1150 return;
1151}
1152
1153/*------------------------------------------------------------------------*
1154 * usb2_bus_tag_unsetup - factored out code
1155 *------------------------------------------------------------------------*/
1156void
1157usb2_dma_tag_unsetup(struct usb2_dma_parent_tag *udpt)
1158{
1159 struct usb2_dma_tag *udt;
1160 uint8_t nudt;
1161
1162 udt = udpt->utag_first;
1163 nudt = udpt->utag_max;
1164
1165 while (nudt--) {
1166
1167 if (udt->align) {
1168 /* destroy the USB DMA tag */
1169 usb2_dma_tag_destroy(udt);
1170 udt->align = 0;
1171 }
1172 udt++;
1173 }
1174
1175 if (udpt->utag_max) {
1176#ifdef __FreeBSD__
1177 /* destroy the condition variable */
1178 usb2_cv_destroy(udpt->cv);
1179#endif
1180 }
1181 return;
1182}
1183
1184/*------------------------------------------------------------------------*
1185 * usb2_bdma_work_loop
1186 *
1187 * This function handles loading of virtual buffers into DMA and is
1188 * only called when "dma_refcount" is zero.
1189 *------------------------------------------------------------------------*/
1190void
1191usb2_bdma_work_loop(struct usb2_xfer_queue *pq)
1192{
1193 struct usb2_xfer_root *info;
1194 struct usb2_xfer *xfer;
1195 uint32_t nframes;
1196
1197 xfer = pq->curr;
1198 info = xfer->usb2_root;
1199
1200 mtx_assert(info->priv_mtx, MA_OWNED);
1201
1202 if (xfer->error) {
1203 /* some error happened */
1204 USB_BUS_LOCK(xfer->udev->bus);
1205 usb2_transfer_done(xfer, 0);
1206 USB_BUS_UNLOCK(xfer->udev->bus);
1207 return;
1208 }
1209 if (!xfer->flags_int.bdma_setup) {
1210 struct usb2_page *pg;
1211 uint32_t frlength_0;
1212 uint8_t isread;
1213
1214 xfer->flags_int.bdma_setup = 1;
1215
1216 /* reset BUS-DMA load state */
1217
1218 info->dma_error = 0;
1219
1220 if (xfer->flags_int.isochronous_xfr) {
1221 /* only one frame buffer */
1222 nframes = 1;
1223 frlength_0 = xfer->sumlen;
1224 } else {
1225 /* can be multiple frame buffers */
1226 nframes = xfer->nframes;
1227 frlength_0 = xfer->frlengths[0];
1228 }
1229
1230 /*
1231 * Set DMA direction first. This is needed to
1232 * select the correct cache invalidate and cache
1233 * flush operations.
1234 */
1235 isread = USB_GET_DATA_ISREAD(xfer);
1236 pg = xfer->dma_page_ptr;
1237
1238 if (xfer->flags_int.control_xfr &&
1239 xfer->flags_int.control_hdr) {
1240 /* special case */
1241 if (xfer->flags_int.usb2_mode == USB_MODE_DEVICE) {
1242 /* The device controller writes to memory */
1243 xfer->frbuffers[0].isread = 1;
1244 } else {
1245 /* The host controller reads from memory */
1246 xfer->frbuffers[0].isread = 0;
1247 }
1248 } else {
1249 /* default case */
1250 xfer->frbuffers[0].isread = isread;
1251 }
1252
1253 /*
1254 * Setup the "page_start" pointer which points to an array of
1255 * USB pages where information about the physical address of a
1256 * page will be stored. Also initialise the "isread" field of
1257 * the USB page caches.
1258 */
1259 xfer->frbuffers[0].page_start = pg;
1260
1261 info->dma_nframes = nframes;
1262 info->dma_currframe = 0;
1263 info->dma_frlength_0 = frlength_0;
1264
1265 pg += (frlength_0 / USB_PAGE_SIZE);
1266 pg += 2;
1267
1268 while (--nframes > 0) {
1269 xfer->frbuffers[nframes].isread = isread;
1270 xfer->frbuffers[nframes].page_start = pg;
1271
1272 pg += (xfer->frlengths[nframes] / USB_PAGE_SIZE);
1273 pg += 2;
1274 }
1275
1276 }
1277 if (info->dma_error) {
1278 USB_BUS_LOCK(xfer->udev->bus);
1279 usb2_transfer_done(xfer, USB_ERR_DMA_LOAD_FAILED);
1280 USB_BUS_UNLOCK(xfer->udev->bus);
1281 return;
1282 }
1283 if (info->dma_currframe != info->dma_nframes) {
1284
1285 if (info->dma_currframe == 0) {
1286 /* special case */
1287 usb2_pc_load_mem(xfer->frbuffers,
1288 info->dma_frlength_0, 0);
1289 } else {
1290 /* default case */
1291 nframes = info->dma_currframe;
1292 usb2_pc_load_mem(xfer->frbuffers + nframes,
1293 xfer->frlengths[nframes], 0);
1294 }
1295
1296 /* advance frame index */
1297 info->dma_currframe++;
1298
1299 return;
1300 }
1301 /* go ahead */
1302 usb2_bdma_pre_sync(xfer);
1303
1304 /* start loading next USB transfer, if any */
1305 usb2_command_wrapper(pq, NULL);
1306
1307 /* finally start the hardware */
1308 usb2_pipe_enter(xfer);
1309
1310 return;
1311}
1312
1313/*------------------------------------------------------------------------*
1314 * usb2_bdma_done_event
1315 *
1316 * This function is called when the BUS-DMA has loaded virtual memory
1317 * into DMA, if any.
1318 *------------------------------------------------------------------------*/
1319void
1320usb2_bdma_done_event(struct usb2_dma_parent_tag *udpt)
1321{
1322 struct usb2_xfer_root *info;
1323
1324 info = udpt->info;
1325
1326 mtx_assert(info->priv_mtx, MA_OWNED);
1327
1328 /* copy error */
1329 info->dma_error = udpt->dma_error;
1330
1331 /* enter workloop again */
1332 usb2_command_wrapper(&info->dma_q,
1333 info->dma_q.curr);
1334 return;
1335}
1336
1337/*------------------------------------------------------------------------*
1338 * usb2_bdma_pre_sync
1339 *
1340 * This function handles DMA synchronisation that must be done before
1341 * an USB transfer is started.
1342 *------------------------------------------------------------------------*/
1343void
1344usb2_bdma_pre_sync(struct usb2_xfer *xfer)
1345{
1346 struct usb2_page_cache *pc;
1347 uint32_t nframes;
1348
1349 if (xfer->flags_int.isochronous_xfr) {
1350 /* only one frame buffer */
1351 nframes = 1;
1352 } else {
1353 /* can be multiple frame buffers */
1354 nframes = xfer->nframes;
1355 }
1356
1357 pc = xfer->frbuffers;
1358
1359 while (nframes--) {
1360
| 1047 bus_dmamap_sync(pc->tag, pc->map, 0, len,
1048 BUS_DMASYNC_PREWRITE | BUS_DMASYNC_PREREAD);
1049 return;
1050}
1051
1052/*------------------------------------------------------------------------*
1053 * usb2_pc_dmamap_create - create a DMA map
1054 *
1055 * Returns:
1056 * 0: Success
1057 * Else: Failure
1058 *------------------------------------------------------------------------*/
1059uint8_t
1060usb2_pc_dmamap_create(struct usb2_page_cache *pc, uint32_t size)
1061{
1062 struct usb2_xfer_root *info;
1063 struct usb2_dma_tag *utag;
1064
1065 /* get info */
1066 info = pc->tag_parent->info;
1067
1068 /* sanity check */
1069 if (info == NULL) {
1070 goto error;
1071 }
1072 utag = usb2_dma_tag_find(pc->tag_parent, size, 1);
1073 if (utag == NULL) {
1074 goto error;
1075 }
1076 if (bus_dmamap_create(utag->tag, size, utag->n_seg,
1077 USB_PAGE_SIZE, 0, BUS_DMA_WAITOK, &pc->map)) {
1078 goto error;
1079 }
1080 pc->tag = utag->tag;
1081 pc->p_seg = utag->p_seg;
1082 pc->n_seg = utag->n_seg;
1083 return 0; /* success */
1084
1085error:
1086 pc->map = NULL;
1087 pc->tag = NULL;
1088 pc->p_seg = NULL;
1089 pc->n_seg = 0;
1090 return 1; /* failure */
1091}
1092
1093/*------------------------------------------------------------------------*
1094 * usb2_pc_dmamap_destroy
1095 *
1096 * This function is NULL safe.
1097 *------------------------------------------------------------------------*/
1098void
1099usb2_pc_dmamap_destroy(struct usb2_page_cache *pc)
1100{
1101 if (pc && pc->tag) {
1102 bus_dmamap_destroy(pc->tag, pc->map);
1103 pc->tag = NULL;
1104 pc->map = NULL;
1105 }
1106 return;
1107}
1108
1109#endif
1110
1111/*------------------------------------------------------------------------*
1112 * usb2_dma_tag_find - factored out code
1113 *------------------------------------------------------------------------*/
1114struct usb2_dma_tag *
1115usb2_dma_tag_find(struct usb2_dma_parent_tag *udpt,
1116 uint32_t size, uint32_t align)
1117{
1118 struct usb2_dma_tag *udt;
1119 uint8_t nudt;
1120
1121 USB_ASSERT(align > 0, ("Invalid parameter align = 0!\n"));
1122 USB_ASSERT(size > 0, ("Invalid parameter size = 0!\n"));
1123
1124 udt = udpt->utag_first;
1125 nudt = udpt->utag_max;
1126
1127 while (nudt--) {
1128
1129 if (udt->align == 0) {
1130 usb2_dma_tag_create(udt, size, align);
1131 if (udt->tag == NULL) {
1132 return (NULL);
1133 }
1134 udt->align = align;
1135 udt->size = size;
1136 return (udt);
1137 }
1138 if ((udt->align == align) && (udt->size == size)) {
1139 return (udt);
1140 }
1141 udt++;
1142 }
1143 return (NULL);
1144}
1145
1146/*------------------------------------------------------------------------*
1147 * usb2_dma_tag_setup - initialise USB DMA tags
1148 *------------------------------------------------------------------------*/
1149void
1150usb2_dma_tag_setup(struct usb2_dma_parent_tag *udpt,
1151 struct usb2_dma_tag *udt, bus_dma_tag_t dmat,
1152 struct mtx *mtx, usb2_dma_callback_t *func,
1153 struct usb2_xfer_root *info, uint8_t ndmabits,
1154 uint8_t nudt)
1155{
1156 bzero(udpt, sizeof(*udpt));
1157
1158 /* sanity checking */
1159 if ((nudt == 0) ||
1160 (ndmabits == 0) ||
1161 (mtx == NULL)) {
1162 /* something is corrupt */
1163 return;
1164 }
1165#ifdef __FreeBSD__
1166 /* initialise condition variable */
1167 usb2_cv_init(udpt->cv, "USB DMA CV");
1168#endif
1169
1170 /* store some information */
1171 udpt->mtx = mtx;
1172 udpt->info = info;
1173 udpt->func = func;
1174 udpt->tag = dmat;
1175 udpt->utag_first = udt;
1176 udpt->utag_max = nudt;
1177 udpt->dma_bits = ndmabits;
1178
1179 while (nudt--) {
1180 bzero(udt, sizeof(*udt));
1181 udt->tag_parent = udpt;
1182 udt++;
1183 }
1184 return;
1185}
1186
1187/*------------------------------------------------------------------------*
1188 * usb2_bus_tag_unsetup - factored out code
1189 *------------------------------------------------------------------------*/
1190void
1191usb2_dma_tag_unsetup(struct usb2_dma_parent_tag *udpt)
1192{
1193 struct usb2_dma_tag *udt;
1194 uint8_t nudt;
1195
1196 udt = udpt->utag_first;
1197 nudt = udpt->utag_max;
1198
1199 while (nudt--) {
1200
1201 if (udt->align) {
1202 /* destroy the USB DMA tag */
1203 usb2_dma_tag_destroy(udt);
1204 udt->align = 0;
1205 }
1206 udt++;
1207 }
1208
1209 if (udpt->utag_max) {
1210#ifdef __FreeBSD__
1211 /* destroy the condition variable */
1212 usb2_cv_destroy(udpt->cv);
1213#endif
1214 }
1215 return;
1216}
1217
1218/*------------------------------------------------------------------------*
1219 * usb2_bdma_work_loop
1220 *
1221 * This function handles loading of virtual buffers into DMA and is
1222 * only called when "dma_refcount" is zero.
1223 *------------------------------------------------------------------------*/
1224void
1225usb2_bdma_work_loop(struct usb2_xfer_queue *pq)
1226{
1227 struct usb2_xfer_root *info;
1228 struct usb2_xfer *xfer;
1229 uint32_t nframes;
1230
1231 xfer = pq->curr;
1232 info = xfer->usb2_root;
1233
1234 mtx_assert(info->priv_mtx, MA_OWNED);
1235
1236 if (xfer->error) {
1237 /* some error happened */
1238 USB_BUS_LOCK(xfer->udev->bus);
1239 usb2_transfer_done(xfer, 0);
1240 USB_BUS_UNLOCK(xfer->udev->bus);
1241 return;
1242 }
1243 if (!xfer->flags_int.bdma_setup) {
1244 struct usb2_page *pg;
1245 uint32_t frlength_0;
1246 uint8_t isread;
1247
1248 xfer->flags_int.bdma_setup = 1;
1249
1250 /* reset BUS-DMA load state */
1251
1252 info->dma_error = 0;
1253
1254 if (xfer->flags_int.isochronous_xfr) {
1255 /* only one frame buffer */
1256 nframes = 1;
1257 frlength_0 = xfer->sumlen;
1258 } else {
1259 /* can be multiple frame buffers */
1260 nframes = xfer->nframes;
1261 frlength_0 = xfer->frlengths[0];
1262 }
1263
1264 /*
1265 * Set DMA direction first. This is needed to
1266 * select the correct cache invalidate and cache
1267 * flush operations.
1268 */
1269 isread = USB_GET_DATA_ISREAD(xfer);
1270 pg = xfer->dma_page_ptr;
1271
1272 if (xfer->flags_int.control_xfr &&
1273 xfer->flags_int.control_hdr) {
1274 /* special case */
1275 if (xfer->flags_int.usb2_mode == USB_MODE_DEVICE) {
1276 /* The device controller writes to memory */
1277 xfer->frbuffers[0].isread = 1;
1278 } else {
1279 /* The host controller reads from memory */
1280 xfer->frbuffers[0].isread = 0;
1281 }
1282 } else {
1283 /* default case */
1284 xfer->frbuffers[0].isread = isread;
1285 }
1286
1287 /*
1288 * Setup the "page_start" pointer which points to an array of
1289 * USB pages where information about the physical address of a
1290 * page will be stored. Also initialise the "isread" field of
1291 * the USB page caches.
1292 */
1293 xfer->frbuffers[0].page_start = pg;
1294
1295 info->dma_nframes = nframes;
1296 info->dma_currframe = 0;
1297 info->dma_frlength_0 = frlength_0;
1298
1299 pg += (frlength_0 / USB_PAGE_SIZE);
1300 pg += 2;
1301
1302 while (--nframes > 0) {
1303 xfer->frbuffers[nframes].isread = isread;
1304 xfer->frbuffers[nframes].page_start = pg;
1305
1306 pg += (xfer->frlengths[nframes] / USB_PAGE_SIZE);
1307 pg += 2;
1308 }
1309
1310 }
1311 if (info->dma_error) {
1312 USB_BUS_LOCK(xfer->udev->bus);
1313 usb2_transfer_done(xfer, USB_ERR_DMA_LOAD_FAILED);
1314 USB_BUS_UNLOCK(xfer->udev->bus);
1315 return;
1316 }
1317 if (info->dma_currframe != info->dma_nframes) {
1318
1319 if (info->dma_currframe == 0) {
1320 /* special case */
1321 usb2_pc_load_mem(xfer->frbuffers,
1322 info->dma_frlength_0, 0);
1323 } else {
1324 /* default case */
1325 nframes = info->dma_currframe;
1326 usb2_pc_load_mem(xfer->frbuffers + nframes,
1327 xfer->frlengths[nframes], 0);
1328 }
1329
1330 /* advance frame index */
1331 info->dma_currframe++;
1332
1333 return;
1334 }
1335 /* go ahead */
1336 usb2_bdma_pre_sync(xfer);
1337
1338 /* start loading next USB transfer, if any */
1339 usb2_command_wrapper(pq, NULL);
1340
1341 /* finally start the hardware */
1342 usb2_pipe_enter(xfer);
1343
1344 return;
1345}
1346
1347/*------------------------------------------------------------------------*
1348 * usb2_bdma_done_event
1349 *
1350 * This function is called when the BUS-DMA has loaded virtual memory
1351 * into DMA, if any.
1352 *------------------------------------------------------------------------*/
1353void
1354usb2_bdma_done_event(struct usb2_dma_parent_tag *udpt)
1355{
1356 struct usb2_xfer_root *info;
1357
1358 info = udpt->info;
1359
1360 mtx_assert(info->priv_mtx, MA_OWNED);
1361
1362 /* copy error */
1363 info->dma_error = udpt->dma_error;
1364
1365 /* enter workloop again */
1366 usb2_command_wrapper(&info->dma_q,
1367 info->dma_q.curr);
1368 return;
1369}
1370
1371/*------------------------------------------------------------------------*
1372 * usb2_bdma_pre_sync
1373 *
1374 * This function handles DMA synchronisation that must be done before
1375 * an USB transfer is started.
1376 *------------------------------------------------------------------------*/
1377void
1378usb2_bdma_pre_sync(struct usb2_xfer *xfer)
1379{
1380 struct usb2_page_cache *pc;
1381 uint32_t nframes;
1382
1383 if (xfer->flags_int.isochronous_xfr) {
1384 /* only one frame buffer */
1385 nframes = 1;
1386 } else {
1387 /* can be multiple frame buffers */
1388 nframes = xfer->nframes;
1389 }
1390
1391 pc = xfer->frbuffers;
1392
1393 while (nframes--) {
1394
|
1361 if (pc->page_offset_buf != pc->page_offset_end) {
1362 if (pc->isread) {
1363 usb2_pc_cpu_invalidate(pc);
1364 } else {
1365 usb2_pc_cpu_flush(pc);
1366 }
| 1395 if (pc->isread) {
1396 usb2_pc_cpu_invalidate(pc);
1397 } else {
1398 usb2_pc_cpu_flush(pc);
|
1367 }
1368 pc++;
1369 }
1370
1371 return;
1372}
1373
1374/*------------------------------------------------------------------------*
1375 * usb2_bdma_post_sync
1376 *
1377 * This function handles DMA synchronisation that must be done after
1378 * an USB transfer is complete.
1379 *------------------------------------------------------------------------*/
1380void
1381usb2_bdma_post_sync(struct usb2_xfer *xfer)
1382{
1383 struct usb2_page_cache *pc;
1384 uint32_t nframes;
1385
1386 if (xfer->flags_int.isochronous_xfr) {
1387 /* only one frame buffer */
1388 nframes = 1;
1389 } else {
1390 /* can be multiple frame buffers */
1391 nframes = xfer->nframes;
1392 }
1393
1394 pc = xfer->frbuffers;
1395
1396 while (nframes--) {
| 1399 }
1400 pc++;
1401 }
1402
1403 return;
1404}
1405
1406/*------------------------------------------------------------------------*
1407 * usb2_bdma_post_sync
1408 *
1409 * This function handles DMA synchronisation that must be done after
1410 * an USB transfer is complete.
1411 *------------------------------------------------------------------------*/
1412void
1413usb2_bdma_post_sync(struct usb2_xfer *xfer)
1414{
1415 struct usb2_page_cache *pc;
1416 uint32_t nframes;
1417
1418 if (xfer->flags_int.isochronous_xfr) {
1419 /* only one frame buffer */
1420 nframes = 1;
1421 } else {
1422 /* can be multiple frame buffers */
1423 nframes = xfer->nframes;
1424 }
1425
1426 pc = xfer->frbuffers;
1427
1428 while (nframes--) {
|
1397
1398 if (pc->page_offset_buf != pc->page_offset_end) {
1399 if (pc->isread) {
1400 usb2_pc_cpu_invalidate(pc);
1401 }
| 1429 if (pc->isread) {
1430 usb2_pc_cpu_invalidate(pc);
|
1402 }
1403 pc++;
1404 }
1405 return;
1406}
| 1431 }
1432 pc++;
1433 }
1434 return;
1435}
|