2/*-
3 * Copyright (c) 2008 Hans Petter Selasky. All rights reserved.
4 *
5 * Redistribution and use in source and binary forms, with or without
6 * modification, are permitted provided that the following conditions
7 * are met:
8 * 1. Redistributions of source code must retain the above copyright
9 * notice, this list of conditions and the following disclaimer.
10 * 2. Redistributions in binary form must reproduce the above copyright
11 * notice, this list of conditions and the following disclaimer in the
12 * documentation and/or other materials provided with the distribution.
13 *
14 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
15 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
16 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
17 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
18 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
19 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
20 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
21 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
22 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
23 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
24 * SUCH DAMAGE.
25 */
26
27#include <dev/usb/usb_mfunc.h>
28#include <dev/usb/usb_error.h>
29#include <dev/usb/usb.h>
30
31#define USB_DEBUG_VAR usb2_debug
32
33#include <dev/usb/usb_core.h>
34#include <dev/usb/usb_busdma.h>
35#include <dev/usb/usb_process.h>
36#include <dev/usb/usb_transfer.h>
37#include <dev/usb/usb_device.h>
38#include <dev/usb/usb_util.h>
39#include <dev/usb/usb_debug.h>
40
41#include <dev/usb/usb_controller.h>
42#include <dev/usb/usb_bus.h>
43
44#if USB_HAVE_BUSDMA
45static void usb2_dma_tag_create(struct usb_dma_tag *, usb_size_t, usb_size_t);
46static void usb2_dma_tag_destroy(struct usb_dma_tag *);
47static void usb2_dma_lock_cb(void *, bus_dma_lock_op_t);
48static void usb2_pc_alloc_mem_cb(void *, bus_dma_segment_t *, int, int);
49static void usb2_pc_load_mem_cb(void *, bus_dma_segment_t *, int, int);
50static void usb2_pc_common_mem_cb(void *, bus_dma_segment_t *, int, int,
51 uint8_t);
52#endif
53
54/*------------------------------------------------------------------------*
55 * usb2_get_page - lookup DMA-able memory for the given offset
56 *
57 * NOTE: Only call this function when the "page_cache" structure has
58 * been properly initialized !
59 *------------------------------------------------------------------------*/
60void
61usb2_get_page(struct usb_page_cache *pc, usb_frlength_t offset,
62 struct usb_page_search *res)
63{
64 struct usb_page *page;
65
66#if USB_HAVE_BUSDMA
67 if (pc->page_start) {
68
69 /* Case 1 - something has been loaded into DMA */
70
71 if (pc->buffer) {
72
73 /* Case 1a - Kernel Virtual Address */
74
75 res->buffer = USB_ADD_BYTES(pc->buffer, offset);
76 }
77 offset += pc->page_offset_buf;
78
79 /* compute destination page */
80
81 page = pc->page_start;
82
83 if (pc->ismultiseg) {
84
85 page += (offset / USB_PAGE_SIZE);
86
87 offset %= USB_PAGE_SIZE;
88
89 res->length = USB_PAGE_SIZE - offset;
90 res->physaddr = page->physaddr + offset;
91 } else {
92 res->length = 0 - 1;
93 res->physaddr = page->physaddr + offset;
94 }
95 if (!pc->buffer) {
96
97 /* Case 1b - Non Kernel Virtual Address */
98
99 res->buffer = USB_ADD_BYTES(page->buffer, offset);
100 }
101 return;
102 }
103#endif
104 /* Case 2 - Plain PIO */
105
106 res->buffer = USB_ADD_BYTES(pc->buffer, offset);
107 res->length = 0 - 1;
108#if USB_HAVE_BUSDMA
109 res->physaddr = 0;
110#endif
111}
112
113/*------------------------------------------------------------------------*
114 * usb2_copy_in - copy directly to DMA-able memory
115 *------------------------------------------------------------------------*/
116void
117usb2_copy_in(struct usb_page_cache *cache, usb_frlength_t offset,
118 const void *ptr, usb_frlength_t len)
119{
120 struct usb_page_search buf_res;
121
122 while (len != 0) {
123
124 usb2_get_page(cache, offset, &buf_res);
125
126 if (buf_res.length > len) {
127 buf_res.length = len;
128 }
129 bcopy(ptr, buf_res.buffer, buf_res.length);
130
131 offset += buf_res.length;
132 len -= buf_res.length;
133 ptr = USB_ADD_BYTES(ptr, buf_res.length);
134 }
135}
136
137/*------------------------------------------------------------------------*
138 * usb2_copy_in_user - copy directly to DMA-able memory from userland
139 *
140 * Return values:
141 * 0: Success
142 * Else: Failure
143 *------------------------------------------------------------------------*/
144#if USB_HAVE_USER_IO
145int
146usb2_copy_in_user(struct usb_page_cache *cache, usb_frlength_t offset,
147 const void *ptr, usb_frlength_t len)
148{
149 struct usb_page_search buf_res;
150 int error;
151
152 while (len != 0) {
153
154 usb2_get_page(cache, offset, &buf_res);
155
156 if (buf_res.length > len) {
157 buf_res.length = len;
158 }
159 error = copyin(ptr, buf_res.buffer, buf_res.length);
160 if (error)
161 return (error);
162
163 offset += buf_res.length;
164 len -= buf_res.length;
165 ptr = USB_ADD_BYTES(ptr, buf_res.length);
166 }
167 return (0); /* success */
168}
169#endif
170
171/*------------------------------------------------------------------------*
172 * usb2_m_copy_in - copy a mbuf chain directly into DMA-able memory
173 *------------------------------------------------------------------------*/
174#if USB_HAVE_MBUF
175struct usb2_m_copy_in_arg {
176 struct usb_page_cache *cache;
177 usb_frlength_t dst_offset;
178};
179
180static int
181usb2_m_copy_in_cb(void *arg, void *src, uint32_t count)
182{
183 register struct usb2_m_copy_in_arg *ua = arg;
184
185 usb2_copy_in(ua->cache, ua->dst_offset, src, count);
186 ua->dst_offset += count;
187 return (0);
188}
189
190void
191usb2_m_copy_in(struct usb_page_cache *cache, usb_frlength_t dst_offset,
192 struct mbuf *m, usb_size_t src_offset, usb_frlength_t src_len)
193{
194 struct usb2_m_copy_in_arg arg = {cache, dst_offset};
195 int error;
196
197 error = m_apply(m, src_offset, src_len, &usb2_m_copy_in_cb, &arg);
198}
199#endif
200
201/*------------------------------------------------------------------------*
202 * usb2_uiomove - factored out code
203 *------------------------------------------------------------------------*/
204#if USB_HAVE_USER_IO
205int
206usb2_uiomove(struct usb_page_cache *pc, struct uio *uio,
207 usb_frlength_t pc_offset, usb_frlength_t len)
208{
209 struct usb_page_search res;
210 int error = 0;
211
212 while (len != 0) {
213
214 usb2_get_page(pc, pc_offset, &res);
215
216 if (res.length > len) {
217 res.length = len;
218 }
219 /*
220 * "uiomove()" can sleep so one needs to make a wrapper,
221 * exiting the mutex and checking things
222 */
223 error = uiomove(res.buffer, res.length, uio);
224
225 if (error) {
226 break;
227 }
228 pc_offset += res.length;
229 len -= res.length;
230 }
231 return (error);
232}
233#endif
234
235/*------------------------------------------------------------------------*
236 * usb2_copy_out - copy directly from DMA-able memory
237 *------------------------------------------------------------------------*/
238void
239usb2_copy_out(struct usb_page_cache *cache, usb_frlength_t offset,
240 void *ptr, usb_frlength_t len)
241{
242 struct usb_page_search res;
243
244 while (len != 0) {
245
246 usb2_get_page(cache, offset, &res);
247
248 if (res.length > len) {
249 res.length = len;
250 }
251 bcopy(res.buffer, ptr, res.length);
252
253 offset += res.length;
254 len -= res.length;
255 ptr = USB_ADD_BYTES(ptr, res.length);
256 }
257}
258
259/*------------------------------------------------------------------------*
260 * usb2_copy_out_user - copy directly from DMA-able memory to userland
261 *
262 * Return values:
263 * 0: Success
264 * Else: Failure
265 *------------------------------------------------------------------------*/
266#if USB_HAVE_USER_IO
267int
268usb2_copy_out_user(struct usb_page_cache *cache, usb_frlength_t offset,
269 void *ptr, usb_frlength_t len)
270{
271 struct usb_page_search res;
272 int error;
273
274 while (len != 0) {
275
276 usb2_get_page(cache, offset, &res);
277
278 if (res.length > len) {
279 res.length = len;
280 }
281 error = copyout(res.buffer, ptr, res.length);
282 if (error)
283 return (error);
284
285 offset += res.length;
286 len -= res.length;
287 ptr = USB_ADD_BYTES(ptr, res.length);
288 }
289 return (0); /* success */
290}
291#endif
292
293/*------------------------------------------------------------------------*
294 * usb2_bzero - zero DMA-able memory
295 *------------------------------------------------------------------------*/
296void
297usb2_bzero(struct usb_page_cache *cache, usb_frlength_t offset,
298 usb_frlength_t len)
299{
300 struct usb_page_search res;
301
302 while (len != 0) {
303
304 usb2_get_page(cache, offset, &res);
305
306 if (res.length > len) {
307 res.length = len;
308 }
309 bzero(res.buffer, res.length);
310
311 offset += res.length;
312 len -= res.length;
313 }
314}
315
316#if USB_HAVE_BUSDMA
317
318/*------------------------------------------------------------------------*
319 * usb2_dma_lock_cb - dummy callback
320 *------------------------------------------------------------------------*/
321static void
322usb2_dma_lock_cb(void *arg, bus_dma_lock_op_t op)
323{
324 /* we use "mtx_owned()" instead of this function */
325}
326
327/*------------------------------------------------------------------------*
328 * usb2_dma_tag_create - allocate a DMA tag
329 *
330 * NOTE: If the "align" parameter has a value of 1 the DMA-tag will
331 * allow multi-segment mappings. Else all mappings are single-segment.
332 *------------------------------------------------------------------------*/
333static void
334usb2_dma_tag_create(struct usb_dma_tag *udt,
335 usb_size_t size, usb_size_t align)
336{
337 bus_dma_tag_t tag;
338
339 if (bus_dma_tag_create
340 ( /* parent */ udt->tag_parent->tag,
341 /* alignment */ align,
342 /* boundary */ USB_PAGE_SIZE,
343 /* lowaddr */ (2ULL << (udt->tag_parent->dma_bits - 1)) - 1,
344 /* highaddr */ BUS_SPACE_MAXADDR,
345 /* filter */ NULL,
346 /* filterarg */ NULL,
347 /* maxsize */ size,
348 /* nsegments */ (align == 1) ?
349 (2 + (size / USB_PAGE_SIZE)) : 1,
350 /* maxsegsz */ (align == 1) ?
351 USB_PAGE_SIZE : size,
352 /* flags */ BUS_DMA_KEEP_PG_OFFSET,
353 /* lockfn */ &usb2_dma_lock_cb,
354 /* lockarg */ NULL,
355 &tag)) {
356 tag = NULL;
357 }
358 udt->tag = tag;
359}
360
361/*------------------------------------------------------------------------*
362 * usb2_dma_tag_free - free a DMA tag
363 *------------------------------------------------------------------------*/
364static void
365usb2_dma_tag_destroy(struct usb_dma_tag *udt)
366{
367 bus_dma_tag_destroy(udt->tag);
368}
369
370/*------------------------------------------------------------------------*
371 * usb2_pc_alloc_mem_cb - BUS-DMA callback function
372 *------------------------------------------------------------------------*/
373static void
374usb2_pc_alloc_mem_cb(void *arg, bus_dma_segment_t *segs,
375 int nseg, int error)
376{
377 usb2_pc_common_mem_cb(arg, segs, nseg, error, 0);
378}
379
380/*------------------------------------------------------------------------*
381 * usb2_pc_load_mem_cb - BUS-DMA callback function
382 *------------------------------------------------------------------------*/
383static void
384usb2_pc_load_mem_cb(void *arg, bus_dma_segment_t *segs,
385 int nseg, int error)
386{
387 usb2_pc_common_mem_cb(arg, segs, nseg, error, 1);
388}
389
390/*------------------------------------------------------------------------*
391 * usb2_pc_common_mem_cb - BUS-DMA callback function
392 *------------------------------------------------------------------------*/
393static void
394usb2_pc_common_mem_cb(void *arg, bus_dma_segment_t *segs,
395 int nseg, int error, uint8_t isload)
396{
397 struct usb_dma_parent_tag *uptag;
398 struct usb_page_cache *pc;
399 struct usb_page *pg;
400 usb_size_t rem;
401 uint8_t owned;
402
403 pc = arg;
404 uptag = pc->tag_parent;
405
406 /*
407 * XXX There is sometimes recursive locking here.
408 * XXX We should try to find a better solution.
409 * XXX Until further the "owned" variable does
410 * XXX the trick.
411 */
412
413 if (error) {
414 goto done;
415 }
416 pg = pc->page_start;
417 pg->physaddr = segs->ds_addr & ~(USB_PAGE_SIZE - 1);
418 rem = segs->ds_addr & (USB_PAGE_SIZE - 1);
419 pc->page_offset_buf = rem;
420 pc->page_offset_end += rem;
421 nseg--;
422#if (USB_DEBUG != 0)
423 if (rem != (USB_P2U(pc->buffer) & (USB_PAGE_SIZE - 1))) {
424 /*
425 * This check verifies that the physical address is correct:
426 */
427 DPRINTFN(0, "Page offset was not preserved!\n");
428 error = 1;
429 goto done;
430 }
431#endif
432 while (nseg > 0) {
433 nseg--;
434 segs++;
435 pg++;
436 pg->physaddr = segs->ds_addr & ~(USB_PAGE_SIZE - 1);
437 }
438
439done:
440 owned = mtx_owned(uptag->mtx);
441 if (!owned)
442 mtx_lock(uptag->mtx);
443
444 uptag->dma_error = (error ? 1 : 0);
445 if (isload) {
446 (uptag->func) (uptag);
447 } else {
| 2/*-
3 * Copyright (c) 2008 Hans Petter Selasky. All rights reserved.
4 *
5 * Redistribution and use in source and binary forms, with or without
6 * modification, are permitted provided that the following conditions
7 * are met:
8 * 1. Redistributions of source code must retain the above copyright
9 * notice, this list of conditions and the following disclaimer.
10 * 2. Redistributions in binary form must reproduce the above copyright
11 * notice, this list of conditions and the following disclaimer in the
12 * documentation and/or other materials provided with the distribution.
13 *
14 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
15 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
16 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
17 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
18 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
19 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
20 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
21 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
22 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
23 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
24 * SUCH DAMAGE.
25 */
26
27#include <dev/usb/usb_mfunc.h>
28#include <dev/usb/usb_error.h>
29#include <dev/usb/usb.h>
30
31#define USB_DEBUG_VAR usb2_debug
32
33#include <dev/usb/usb_core.h>
34#include <dev/usb/usb_busdma.h>
35#include <dev/usb/usb_process.h>
36#include <dev/usb/usb_transfer.h>
37#include <dev/usb/usb_device.h>
38#include <dev/usb/usb_util.h>
39#include <dev/usb/usb_debug.h>
40
41#include <dev/usb/usb_controller.h>
42#include <dev/usb/usb_bus.h>
43
44#if USB_HAVE_BUSDMA
45static void usb2_dma_tag_create(struct usb_dma_tag *, usb_size_t, usb_size_t);
46static void usb2_dma_tag_destroy(struct usb_dma_tag *);
47static void usb2_dma_lock_cb(void *, bus_dma_lock_op_t);
48static void usb2_pc_alloc_mem_cb(void *, bus_dma_segment_t *, int, int);
49static void usb2_pc_load_mem_cb(void *, bus_dma_segment_t *, int, int);
50static void usb2_pc_common_mem_cb(void *, bus_dma_segment_t *, int, int,
51 uint8_t);
52#endif
53
54/*------------------------------------------------------------------------*
55 * usb2_get_page - lookup DMA-able memory for the given offset
56 *
57 * NOTE: Only call this function when the "page_cache" structure has
58 * been properly initialized !
59 *------------------------------------------------------------------------*/
60void
61usb2_get_page(struct usb_page_cache *pc, usb_frlength_t offset,
62 struct usb_page_search *res)
63{
64 struct usb_page *page;
65
66#if USB_HAVE_BUSDMA
67 if (pc->page_start) {
68
69 /* Case 1 - something has been loaded into DMA */
70
71 if (pc->buffer) {
72
73 /* Case 1a - Kernel Virtual Address */
74
75 res->buffer = USB_ADD_BYTES(pc->buffer, offset);
76 }
77 offset += pc->page_offset_buf;
78
79 /* compute destination page */
80
81 page = pc->page_start;
82
83 if (pc->ismultiseg) {
84
85 page += (offset / USB_PAGE_SIZE);
86
87 offset %= USB_PAGE_SIZE;
88
89 res->length = USB_PAGE_SIZE - offset;
90 res->physaddr = page->physaddr + offset;
91 } else {
92 res->length = 0 - 1;
93 res->physaddr = page->physaddr + offset;
94 }
95 if (!pc->buffer) {
96
97 /* Case 1b - Non Kernel Virtual Address */
98
99 res->buffer = USB_ADD_BYTES(page->buffer, offset);
100 }
101 return;
102 }
103#endif
104 /* Case 2 - Plain PIO */
105
106 res->buffer = USB_ADD_BYTES(pc->buffer, offset);
107 res->length = 0 - 1;
108#if USB_HAVE_BUSDMA
109 res->physaddr = 0;
110#endif
111}
112
113/*------------------------------------------------------------------------*
114 * usb2_copy_in - copy directly to DMA-able memory
115 *------------------------------------------------------------------------*/
116void
117usb2_copy_in(struct usb_page_cache *cache, usb_frlength_t offset,
118 const void *ptr, usb_frlength_t len)
119{
120 struct usb_page_search buf_res;
121
122 while (len != 0) {
123
124 usb2_get_page(cache, offset, &buf_res);
125
126 if (buf_res.length > len) {
127 buf_res.length = len;
128 }
129 bcopy(ptr, buf_res.buffer, buf_res.length);
130
131 offset += buf_res.length;
132 len -= buf_res.length;
133 ptr = USB_ADD_BYTES(ptr, buf_res.length);
134 }
135}
136
137/*------------------------------------------------------------------------*
138 * usb2_copy_in_user - copy directly to DMA-able memory from userland
139 *
140 * Return values:
141 * 0: Success
142 * Else: Failure
143 *------------------------------------------------------------------------*/
144#if USB_HAVE_USER_IO
145int
146usb2_copy_in_user(struct usb_page_cache *cache, usb_frlength_t offset,
147 const void *ptr, usb_frlength_t len)
148{
149 struct usb_page_search buf_res;
150 int error;
151
152 while (len != 0) {
153
154 usb2_get_page(cache, offset, &buf_res);
155
156 if (buf_res.length > len) {
157 buf_res.length = len;
158 }
159 error = copyin(ptr, buf_res.buffer, buf_res.length);
160 if (error)
161 return (error);
162
163 offset += buf_res.length;
164 len -= buf_res.length;
165 ptr = USB_ADD_BYTES(ptr, buf_res.length);
166 }
167 return (0); /* success */
168}
169#endif
170
171/*------------------------------------------------------------------------*
172 * usb2_m_copy_in - copy a mbuf chain directly into DMA-able memory
173 *------------------------------------------------------------------------*/
174#if USB_HAVE_MBUF
175struct usb2_m_copy_in_arg {
176 struct usb_page_cache *cache;
177 usb_frlength_t dst_offset;
178};
179
180static int
181usb2_m_copy_in_cb(void *arg, void *src, uint32_t count)
182{
183 register struct usb2_m_copy_in_arg *ua = arg;
184
185 usb2_copy_in(ua->cache, ua->dst_offset, src, count);
186 ua->dst_offset += count;
187 return (0);
188}
189
190void
191usb2_m_copy_in(struct usb_page_cache *cache, usb_frlength_t dst_offset,
192 struct mbuf *m, usb_size_t src_offset, usb_frlength_t src_len)
193{
194 struct usb2_m_copy_in_arg arg = {cache, dst_offset};
195 int error;
196
197 error = m_apply(m, src_offset, src_len, &usb2_m_copy_in_cb, &arg);
198}
199#endif
200
201/*------------------------------------------------------------------------*
202 * usb2_uiomove - factored out code
203 *------------------------------------------------------------------------*/
204#if USB_HAVE_USER_IO
205int
206usb2_uiomove(struct usb_page_cache *pc, struct uio *uio,
207 usb_frlength_t pc_offset, usb_frlength_t len)
208{
209 struct usb_page_search res;
210 int error = 0;
211
212 while (len != 0) {
213
214 usb2_get_page(pc, pc_offset, &res);
215
216 if (res.length > len) {
217 res.length = len;
218 }
219 /*
220 * "uiomove()" can sleep so one needs to make a wrapper,
221 * exiting the mutex and checking things
222 */
223 error = uiomove(res.buffer, res.length, uio);
224
225 if (error) {
226 break;
227 }
228 pc_offset += res.length;
229 len -= res.length;
230 }
231 return (error);
232}
233#endif
234
235/*------------------------------------------------------------------------*
236 * usb2_copy_out - copy directly from DMA-able memory
237 *------------------------------------------------------------------------*/
238void
239usb2_copy_out(struct usb_page_cache *cache, usb_frlength_t offset,
240 void *ptr, usb_frlength_t len)
241{
242 struct usb_page_search res;
243
244 while (len != 0) {
245
246 usb2_get_page(cache, offset, &res);
247
248 if (res.length > len) {
249 res.length = len;
250 }
251 bcopy(res.buffer, ptr, res.length);
252
253 offset += res.length;
254 len -= res.length;
255 ptr = USB_ADD_BYTES(ptr, res.length);
256 }
257}
258
259/*------------------------------------------------------------------------*
260 * usb2_copy_out_user - copy directly from DMA-able memory to userland
261 *
262 * Return values:
263 * 0: Success
264 * Else: Failure
265 *------------------------------------------------------------------------*/
266#if USB_HAVE_USER_IO
267int
268usb2_copy_out_user(struct usb_page_cache *cache, usb_frlength_t offset,
269 void *ptr, usb_frlength_t len)
270{
271 struct usb_page_search res;
272 int error;
273
274 while (len != 0) {
275
276 usb2_get_page(cache, offset, &res);
277
278 if (res.length > len) {
279 res.length = len;
280 }
281 error = copyout(res.buffer, ptr, res.length);
282 if (error)
283 return (error);
284
285 offset += res.length;
286 len -= res.length;
287 ptr = USB_ADD_BYTES(ptr, res.length);
288 }
289 return (0); /* success */
290}
291#endif
292
293/*------------------------------------------------------------------------*
294 * usb2_bzero - zero DMA-able memory
295 *------------------------------------------------------------------------*/
296void
297usb2_bzero(struct usb_page_cache *cache, usb_frlength_t offset,
298 usb_frlength_t len)
299{
300 struct usb_page_search res;
301
302 while (len != 0) {
303
304 usb2_get_page(cache, offset, &res);
305
306 if (res.length > len) {
307 res.length = len;
308 }
309 bzero(res.buffer, res.length);
310
311 offset += res.length;
312 len -= res.length;
313 }
314}
315
316#if USB_HAVE_BUSDMA
317
318/*------------------------------------------------------------------------*
319 * usb2_dma_lock_cb - dummy callback
320 *------------------------------------------------------------------------*/
321static void
322usb2_dma_lock_cb(void *arg, bus_dma_lock_op_t op)
323{
324 /* we use "mtx_owned()" instead of this function */
325}
326
327/*------------------------------------------------------------------------*
328 * usb2_dma_tag_create - allocate a DMA tag
329 *
330 * NOTE: If the "align" parameter has a value of 1 the DMA-tag will
331 * allow multi-segment mappings. Else all mappings are single-segment.
332 *------------------------------------------------------------------------*/
333static void
334usb2_dma_tag_create(struct usb_dma_tag *udt,
335 usb_size_t size, usb_size_t align)
336{
337 bus_dma_tag_t tag;
338
339 if (bus_dma_tag_create
340 ( /* parent */ udt->tag_parent->tag,
341 /* alignment */ align,
342 /* boundary */ USB_PAGE_SIZE,
343 /* lowaddr */ (2ULL << (udt->tag_parent->dma_bits - 1)) - 1,
344 /* highaddr */ BUS_SPACE_MAXADDR,
345 /* filter */ NULL,
346 /* filterarg */ NULL,
347 /* maxsize */ size,
348 /* nsegments */ (align == 1) ?
349 (2 + (size / USB_PAGE_SIZE)) : 1,
350 /* maxsegsz */ (align == 1) ?
351 USB_PAGE_SIZE : size,
352 /* flags */ BUS_DMA_KEEP_PG_OFFSET,
353 /* lockfn */ &usb2_dma_lock_cb,
354 /* lockarg */ NULL,
355 &tag)) {
356 tag = NULL;
357 }
358 udt->tag = tag;
359}
360
361/*------------------------------------------------------------------------*
362 * usb2_dma_tag_free - free a DMA tag
363 *------------------------------------------------------------------------*/
364static void
365usb2_dma_tag_destroy(struct usb_dma_tag *udt)
366{
367 bus_dma_tag_destroy(udt->tag);
368}
369
370/*------------------------------------------------------------------------*
371 * usb2_pc_alloc_mem_cb - BUS-DMA callback function
372 *------------------------------------------------------------------------*/
373static void
374usb2_pc_alloc_mem_cb(void *arg, bus_dma_segment_t *segs,
375 int nseg, int error)
376{
377 usb2_pc_common_mem_cb(arg, segs, nseg, error, 0);
378}
379
380/*------------------------------------------------------------------------*
381 * usb2_pc_load_mem_cb - BUS-DMA callback function
382 *------------------------------------------------------------------------*/
383static void
384usb2_pc_load_mem_cb(void *arg, bus_dma_segment_t *segs,
385 int nseg, int error)
386{
387 usb2_pc_common_mem_cb(arg, segs, nseg, error, 1);
388}
389
390/*------------------------------------------------------------------------*
391 * usb2_pc_common_mem_cb - BUS-DMA callback function
392 *------------------------------------------------------------------------*/
393static void
394usb2_pc_common_mem_cb(void *arg, bus_dma_segment_t *segs,
395 int nseg, int error, uint8_t isload)
396{
397 struct usb_dma_parent_tag *uptag;
398 struct usb_page_cache *pc;
399 struct usb_page *pg;
400 usb_size_t rem;
401 uint8_t owned;
402
403 pc = arg;
404 uptag = pc->tag_parent;
405
406 /*
407 * XXX There is sometimes recursive locking here.
408 * XXX We should try to find a better solution.
409 * XXX Until further the "owned" variable does
410 * XXX the trick.
411 */
412
413 if (error) {
414 goto done;
415 }
416 pg = pc->page_start;
417 pg->physaddr = segs->ds_addr & ~(USB_PAGE_SIZE - 1);
418 rem = segs->ds_addr & (USB_PAGE_SIZE - 1);
419 pc->page_offset_buf = rem;
420 pc->page_offset_end += rem;
421 nseg--;
422#if (USB_DEBUG != 0)
423 if (rem != (USB_P2U(pc->buffer) & (USB_PAGE_SIZE - 1))) {
424 /*
425 * This check verifies that the physical address is correct:
426 */
427 DPRINTFN(0, "Page offset was not preserved!\n");
428 error = 1;
429 goto done;
430 }
431#endif
432 while (nseg > 0) {
433 nseg--;
434 segs++;
435 pg++;
436 pg->physaddr = segs->ds_addr & ~(USB_PAGE_SIZE - 1);
437 }
438
439done:
440 owned = mtx_owned(uptag->mtx);
441 if (!owned)
442 mtx_lock(uptag->mtx);
443
444 uptag->dma_error = (error ? 1 : 0);
445 if (isload) {
446 (uptag->func) (uptag);
447 } else {
|
616 err = 0;
617 }
618 if (err || uptag->dma_error) {
619 return (1);
620 }
621 } else {
622
623 /*
624 * We have to unload the previous loaded DMA
625 * pages before trying to load a new one!
626 */
627 bus_dmamap_unload(pc->tag, pc->map);
628
629 /*
630 * Try to load memory into DMA. The callback
631 * will be called in all cases:
632 */
633 if (bus_dmamap_load(
634 pc->tag, pc->map, pc->buffer, size,
635 &usb2_pc_load_mem_cb, pc, BUS_DMA_WAITOK)) {
636 }
637 }
638 } else {
639 if (!sync) {
640 /*
641 * Call callback so that refcount is decremented
642 * properly:
643 */
644 pc->tag_parent->dma_error = 0;
645 (pc->tag_parent->func) (pc->tag_parent);
646 }
647 }
648 return (0);
649}
650
651/*------------------------------------------------------------------------*
652 * usb2_pc_cpu_invalidate - invalidate CPU cache
653 *------------------------------------------------------------------------*/
654void
655usb2_pc_cpu_invalidate(struct usb_page_cache *pc)
656{
657 if (pc->page_offset_end == pc->page_offset_buf) {
658 /* nothing has been loaded into this page cache! */
659 return;
660 }
661 bus_dmamap_sync(pc->tag, pc->map,
662 BUS_DMASYNC_POSTWRITE | BUS_DMASYNC_POSTREAD);
663}
664
665/*------------------------------------------------------------------------*
666 * usb2_pc_cpu_flush - flush CPU cache
667 *------------------------------------------------------------------------*/
668void
669usb2_pc_cpu_flush(struct usb_page_cache *pc)
670{
671 if (pc->page_offset_end == pc->page_offset_buf) {
672 /* nothing has been loaded into this page cache! */
673 return;
674 }
675 bus_dmamap_sync(pc->tag, pc->map,
676 BUS_DMASYNC_PREWRITE | BUS_DMASYNC_PREREAD);
677}
678
679/*------------------------------------------------------------------------*
680 * usb2_pc_dmamap_create - create a DMA map
681 *
682 * Returns:
683 * 0: Success
684 * Else: Failure
685 *------------------------------------------------------------------------*/
686uint8_t
687usb2_pc_dmamap_create(struct usb_page_cache *pc, usb_size_t size)
688{
689 struct usb_xfer_root *info;
690 struct usb_dma_tag *utag;
691
692 /* get info */
693 info = USB_DMATAG_TO_XROOT(pc->tag_parent);
694
695 /* sanity check */
696 if (info == NULL) {
697 goto error;
698 }
699 utag = usb2_dma_tag_find(pc->tag_parent, size, 1);
700 if (utag == NULL) {
701 goto error;
702 }
703 /* create DMA map */
704 if (bus_dmamap_create(utag->tag, 0, &pc->map)) {
705 goto error;
706 }
707 pc->tag = utag->tag;
708 return 0; /* success */
709
710error:
711 pc->map = NULL;
712 pc->tag = NULL;
713 return 1; /* failure */
714}
715
716/*------------------------------------------------------------------------*
717 * usb2_pc_dmamap_destroy
718 *
719 * This function is NULL safe.
720 *------------------------------------------------------------------------*/
721void
722usb2_pc_dmamap_destroy(struct usb_page_cache *pc)
723{
724 if (pc && pc->tag) {
725 bus_dmamap_destroy(pc->tag, pc->map);
726 pc->tag = NULL;
727 pc->map = NULL;
728 }
729}
730
731/*------------------------------------------------------------------------*
732 * usb2_dma_tag_find - factored out code
733 *------------------------------------------------------------------------*/
734struct usb_dma_tag *
735usb2_dma_tag_find(struct usb_dma_parent_tag *udpt,
736 usb_size_t size, usb_size_t align)
737{
738 struct usb_dma_tag *udt;
739 uint8_t nudt;
740
741 USB_ASSERT(align > 0, ("Invalid parameter align = 0!\n"));
742 USB_ASSERT(size > 0, ("Invalid parameter size = 0!\n"));
743
744 udt = udpt->utag_first;
745 nudt = udpt->utag_max;
746
747 while (nudt--) {
748
749 if (udt->align == 0) {
750 usb2_dma_tag_create(udt, size, align);
751 if (udt->tag == NULL) {
752 return (NULL);
753 }
754 udt->align = align;
755 udt->size = size;
756 return (udt);
757 }
758 if ((udt->align == align) && (udt->size == size)) {
759 return (udt);
760 }
761 udt++;
762 }
763 return (NULL);
764}
765
766/*------------------------------------------------------------------------*
767 * usb2_dma_tag_setup - initialise USB DMA tags
768 *------------------------------------------------------------------------*/
769void
770usb2_dma_tag_setup(struct usb_dma_parent_tag *udpt,
771 struct usb_dma_tag *udt, bus_dma_tag_t dmat,
772 struct mtx *mtx, usb_dma_callback_t *func,
773 uint8_t ndmabits, uint8_t nudt)
774{
775 bzero(udpt, sizeof(*udpt));
776
777 /* sanity checking */
778 if ((nudt == 0) ||
779 (ndmabits == 0) ||
780 (mtx == NULL)) {
781 /* something is corrupt */
782 return;
783 }
784 /* initialise condition variable */
| 616 err = 0;
617 }
618 if (err || uptag->dma_error) {
619 return (1);
620 }
621 } else {
622
623 /*
624 * We have to unload the previous loaded DMA
625 * pages before trying to load a new one!
626 */
627 bus_dmamap_unload(pc->tag, pc->map);
628
629 /*
630 * Try to load memory into DMA. The callback
631 * will be called in all cases:
632 */
633 if (bus_dmamap_load(
634 pc->tag, pc->map, pc->buffer, size,
635 &usb2_pc_load_mem_cb, pc, BUS_DMA_WAITOK)) {
636 }
637 }
638 } else {
639 if (!sync) {
640 /*
641 * Call callback so that refcount is decremented
642 * properly:
643 */
644 pc->tag_parent->dma_error = 0;
645 (pc->tag_parent->func) (pc->tag_parent);
646 }
647 }
648 return (0);
649}
650
651/*------------------------------------------------------------------------*
652 * usb2_pc_cpu_invalidate - invalidate CPU cache
653 *------------------------------------------------------------------------*/
654void
655usb2_pc_cpu_invalidate(struct usb_page_cache *pc)
656{
657 if (pc->page_offset_end == pc->page_offset_buf) {
658 /* nothing has been loaded into this page cache! */
659 return;
660 }
661 bus_dmamap_sync(pc->tag, pc->map,
662 BUS_DMASYNC_POSTWRITE | BUS_DMASYNC_POSTREAD);
663}
664
665/*------------------------------------------------------------------------*
666 * usb2_pc_cpu_flush - flush CPU cache
667 *------------------------------------------------------------------------*/
668void
669usb2_pc_cpu_flush(struct usb_page_cache *pc)
670{
671 if (pc->page_offset_end == pc->page_offset_buf) {
672 /* nothing has been loaded into this page cache! */
673 return;
674 }
675 bus_dmamap_sync(pc->tag, pc->map,
676 BUS_DMASYNC_PREWRITE | BUS_DMASYNC_PREREAD);
677}
678
679/*------------------------------------------------------------------------*
680 * usb2_pc_dmamap_create - create a DMA map
681 *
682 * Returns:
683 * 0: Success
684 * Else: Failure
685 *------------------------------------------------------------------------*/
686uint8_t
687usb2_pc_dmamap_create(struct usb_page_cache *pc, usb_size_t size)
688{
689 struct usb_xfer_root *info;
690 struct usb_dma_tag *utag;
691
692 /* get info */
693 info = USB_DMATAG_TO_XROOT(pc->tag_parent);
694
695 /* sanity check */
696 if (info == NULL) {
697 goto error;
698 }
699 utag = usb2_dma_tag_find(pc->tag_parent, size, 1);
700 if (utag == NULL) {
701 goto error;
702 }
703 /* create DMA map */
704 if (bus_dmamap_create(utag->tag, 0, &pc->map)) {
705 goto error;
706 }
707 pc->tag = utag->tag;
708 return 0; /* success */
709
710error:
711 pc->map = NULL;
712 pc->tag = NULL;
713 return 1; /* failure */
714}
715
716/*------------------------------------------------------------------------*
717 * usb2_pc_dmamap_destroy
718 *
719 * This function is NULL safe.
720 *------------------------------------------------------------------------*/
721void
722usb2_pc_dmamap_destroy(struct usb_page_cache *pc)
723{
724 if (pc && pc->tag) {
725 bus_dmamap_destroy(pc->tag, pc->map);
726 pc->tag = NULL;
727 pc->map = NULL;
728 }
729}
730
731/*------------------------------------------------------------------------*
732 * usb2_dma_tag_find - factored out code
733 *------------------------------------------------------------------------*/
734struct usb_dma_tag *
735usb2_dma_tag_find(struct usb_dma_parent_tag *udpt,
736 usb_size_t size, usb_size_t align)
737{
738 struct usb_dma_tag *udt;
739 uint8_t nudt;
740
741 USB_ASSERT(align > 0, ("Invalid parameter align = 0!\n"));
742 USB_ASSERT(size > 0, ("Invalid parameter size = 0!\n"));
743
744 udt = udpt->utag_first;
745 nudt = udpt->utag_max;
746
747 while (nudt--) {
748
749 if (udt->align == 0) {
750 usb2_dma_tag_create(udt, size, align);
751 if (udt->tag == NULL) {
752 return (NULL);
753 }
754 udt->align = align;
755 udt->size = size;
756 return (udt);
757 }
758 if ((udt->align == align) && (udt->size == size)) {
759 return (udt);
760 }
761 udt++;
762 }
763 return (NULL);
764}
765
766/*------------------------------------------------------------------------*
767 * usb2_dma_tag_setup - initialise USB DMA tags
768 *------------------------------------------------------------------------*/
769void
770usb2_dma_tag_setup(struct usb_dma_parent_tag *udpt,
771 struct usb_dma_tag *udt, bus_dma_tag_t dmat,
772 struct mtx *mtx, usb_dma_callback_t *func,
773 uint8_t ndmabits, uint8_t nudt)
774{
775 bzero(udpt, sizeof(*udpt));
776
777 /* sanity checking */
778 if ((nudt == 0) ||
779 (ndmabits == 0) ||
780 (mtx == NULL)) {
781 /* something is corrupt */
782 return;
783 }
784 /* initialise condition variable */
|