1/*-
2 * SPDX-License-Identifier: BSD-2-Clause
3 *
4 * Copyright (c) 2008 Hans Petter Selasky. All rights reserved.
5 *
6 * Redistribution and use in source and binary forms, with or without
7 * modification, are permitted provided that the following conditions
8 * are met:
9 * 1. Redistributions of source code must retain the above copyright
10 *    notice, this list of conditions and the following disclaimer.
11 * 2. Redistributions in binary form must reproduce the above copyright
12 *    notice, this list of conditions and the following disclaimer in the
13 *    documentation and/or other materials provided with the distribution.
14 *
15 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
16 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
17 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
18 * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
19 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
20 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
21 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
22 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
23 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
24 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
25 * SUCH DAMAGE.
26 */
27
28#ifdef USB_GLOBAL_INCLUDE_FILE
29#include USB_GLOBAL_INCLUDE_FILE
30#else
31#include <sys/stdint.h>
32#include <sys/stddef.h>
33#include <sys/param.h>
34#include <sys/queue.h>
35#include <sys/types.h>
36#include <sys/systm.h>
37#include <sys/kernel.h>
38#include <sys/bus.h>
39#include <sys/module.h>
40#include <sys/lock.h>
41#include <sys/mutex.h>
42#include <sys/condvar.h>
43#include <sys/sysctl.h>
44#include <sys/sx.h>
45#include <sys/unistd.h>
46#include <sys/callout.h>
47#include <sys/malloc.h>
48#include <sys/priv.h>
49
50#include <dev/usb/usb.h>
51#include <dev/usb/usbdi.h>
52#include <dev/usb/usbdi_util.h>
53
54#define	USB_DEBUG_VAR usb_debug
55
56#include <dev/usb/usb_core.h>
57#include <dev/usb/usb_busdma.h>
58#include <dev/usb/usb_process.h>
59#include <dev/usb/usb_transfer.h>
60#include <dev/usb/usb_device.h>
61#include <dev/usb/usb_util.h>
62#include <dev/usb/usb_debug.h>
63
64#include <dev/usb/usb_controller.h>
65#include <dev/usb/usb_bus.h>
66#endif			/* USB_GLOBAL_INCLUDE_FILE */
67
68#if USB_HAVE_BUSDMA
69static void	usb_dma_tag_create(struct usb_dma_tag *, usb_size_t, usb_size_t);
70static void	usb_dma_tag_destroy(struct usb_dma_tag *);
71static void	usb_dma_lock_cb(void *, bus_dma_lock_op_t);
72static void	usb_pc_alloc_mem_cb(void *, bus_dma_segment_t *, int, int);
73static void	usb_pc_load_mem_cb(void *, bus_dma_segment_t *, int, int);
74static void	usb_pc_common_mem_cb(void *, bus_dma_segment_t *, int, int,
75		    uint8_t);
76#endif
77
78/*------------------------------------------------------------------------*
79 *  usbd_get_page - lookup DMA-able memory for the given offset
80 *
81 * NOTE: Only call this function when the "page_cache" structure has
82 * been properly initialized !
83 *------------------------------------------------------------------------*/
84void
85usbd_get_page(struct usb_page_cache *pc, usb_frlength_t offset,
86    struct usb_page_search *res)
87{
88#if USB_HAVE_BUSDMA
89	struct usb_page *page;
90
91	if (pc->page_start) {
92		/* Case 1 - something has been loaded into DMA */
93
94		if (pc->buffer) {
95			/* Case 1a - Kernel Virtual Address */
96
97			res->buffer = USB_ADD_BYTES(pc->buffer, offset);
98		}
99		offset += pc->page_offset_buf;
100
101		/* compute destination page */
102
103		page = pc->page_start;
104
105		if (pc->ismultiseg) {
106			page += (offset / USB_PAGE_SIZE);
107
108			offset %= USB_PAGE_SIZE;
109
110			res->length = USB_PAGE_SIZE - offset;
111			res->physaddr = page->physaddr + offset;
112		} else {
113			res->length = (usb_size_t)-1;
114			res->physaddr = page->physaddr + offset;
115		}
116		if (!pc->buffer) {
117			/* Case 1b - Non Kernel Virtual Address */
118
119			res->buffer = USB_ADD_BYTES(page->buffer, offset);
120		}
121		return;
122	}
123#endif
124	/* Case 2 - Plain PIO */
125
126	res->buffer = USB_ADD_BYTES(pc->buffer, offset);
127	res->length = (usb_size_t)-1;
128#if USB_HAVE_BUSDMA
129	res->physaddr = 0;
130#endif
131}
132
133/*------------------------------------------------------------------------*
134 *  usb_pc_buffer_is_aligned - verify alignment
135 *
136 * This function is used to check if a page cache buffer is properly
137 * aligned to reduce the use of bounce buffers in PIO mode.
138 *------------------------------------------------------------------------*/
139uint8_t
140usb_pc_buffer_is_aligned(struct usb_page_cache *pc, usb_frlength_t offset,
141    usb_frlength_t len, usb_frlength_t mask)
142{
143	struct usb_page_search buf_res;
144
145	while (len != 0) {
146		usbd_get_page(pc, offset, &buf_res);
147
148		if (buf_res.length > len)
149			buf_res.length = len;
150		if (USB_P2U(buf_res.buffer) & mask)
151			return (0);
152		if (buf_res.length & mask)
153			return (0);
154
155		offset += buf_res.length;
156		len -= buf_res.length;
157	}
158	return (1);
159}
160
161/*------------------------------------------------------------------------*
162 *  usbd_copy_in - copy directly to DMA-able memory
163 *------------------------------------------------------------------------*/
164void
165usbd_copy_in(struct usb_page_cache *cache, usb_frlength_t offset,
166    const void *ptr, usb_frlength_t len)
167{
168	struct usb_page_search buf_res;
169
170	while (len != 0) {
171		usbd_get_page(cache, offset, &buf_res);
172
173		if (buf_res.length > len) {
174			buf_res.length = len;
175		}
176		memcpy(buf_res.buffer, ptr, buf_res.length);
177
178		offset += buf_res.length;
179		len -= buf_res.length;
180		ptr = USB_ADD_BYTES(ptr, buf_res.length);
181	}
182}
183
184/*------------------------------------------------------------------------*
185 *  usbd_copy_in_user - copy directly to DMA-able memory from userland
186 *
187 * Return values:
188 *    0: Success
189 * Else: Failure
190 *------------------------------------------------------------------------*/
191#if USB_HAVE_USER_IO
192int
193usbd_copy_in_user(struct usb_page_cache *cache, usb_frlength_t offset,
194    const void *ptr, usb_frlength_t len)
195{
196	struct usb_page_search buf_res;
197	int error;
198
199	while (len != 0) {
200		usbd_get_page(cache, offset, &buf_res);
201
202		if (buf_res.length > len) {
203			buf_res.length = len;
204		}
205		error = copyin(ptr, buf_res.buffer, buf_res.length);
206		if (error)
207			return (error);
208
209		offset += buf_res.length;
210		len -= buf_res.length;
211		ptr = USB_ADD_BYTES(ptr, buf_res.length);
212	}
213	return (0);			/* success */
214}
215#endif
216
217/*------------------------------------------------------------------------*
218 *  usbd_m_copy_in - copy a mbuf chain directly into DMA-able memory
219 *------------------------------------------------------------------------*/
220#if USB_HAVE_MBUF
221struct usb_m_copy_in_arg {
222	struct usb_page_cache *cache;
223	usb_frlength_t dst_offset;
224};
225
226static int
227usbd_m_copy_in_cb(void *arg, void *src, uint32_t count)
228{
229	struct usb_m_copy_in_arg *ua = arg;
230
231	usbd_copy_in(ua->cache, ua->dst_offset, src, count);
232	ua->dst_offset += count;
233	return (0);
234}
235
236void
237usbd_m_copy_in(struct usb_page_cache *cache, usb_frlength_t dst_offset,
238    struct mbuf *m, usb_size_t src_offset, usb_frlength_t src_len)
239{
240	struct usb_m_copy_in_arg arg = {cache, dst_offset};
241	(void) m_apply(m, src_offset, src_len, &usbd_m_copy_in_cb, &arg);
242}
243#endif
244
245/*------------------------------------------------------------------------*
246 *  usb_uiomove - factored out code
247 *------------------------------------------------------------------------*/
248#if USB_HAVE_USER_IO
249int
250usb_uiomove(struct usb_page_cache *pc, struct uio *uio,
251    usb_frlength_t pc_offset, usb_frlength_t len)
252{
253	struct usb_page_search res;
254	int error = 0;
255
256	while (len != 0) {
257		usbd_get_page(pc, pc_offset, &res);
258
259		if (res.length > len) {
260			res.length = len;
261		}
262		/*
263		 * "uiomove()" can sleep so one needs to make a wrapper,
264		 * exiting the mutex and checking things
265		 */
266		error = uiomove(res.buffer, res.length, uio);
267
268		if (error) {
269			break;
270		}
271		pc_offset += res.length;
272		len -= res.length;
273	}
274	return (error);
275}
276#endif
277
278/*------------------------------------------------------------------------*
279 *  usbd_copy_out - copy directly from DMA-able memory
280 *------------------------------------------------------------------------*/
281void
282usbd_copy_out(struct usb_page_cache *cache, usb_frlength_t offset,
283    void *ptr, usb_frlength_t len)
284{
285	struct usb_page_search res;
286
287	while (len != 0) {
288		usbd_get_page(cache, offset, &res);
289
290		if (res.length > len) {
291			res.length = len;
292		}
293		memcpy(ptr, res.buffer, res.length);
294
295		offset += res.length;
296		len -= res.length;
297		ptr = USB_ADD_BYTES(ptr, res.length);
298	}
299}
300
301/*------------------------------------------------------------------------*
302 *  usbd_copy_out_user - copy directly from DMA-able memory to userland
303 *
304 * Return values:
305 *    0: Success
306 * Else: Failure
307 *------------------------------------------------------------------------*/
308#if USB_HAVE_USER_IO
309int
310usbd_copy_out_user(struct usb_page_cache *cache, usb_frlength_t offset,
311    void *ptr, usb_frlength_t len)
312{
313	struct usb_page_search res;
314	int error;
315
316	while (len != 0) {
317		usbd_get_page(cache, offset, &res);
318
319		if (res.length > len) {
320			res.length = len;
321		}
322		error = copyout(res.buffer, ptr, res.length);
323		if (error)
324			return (error);
325
326		offset += res.length;
327		len -= res.length;
328		ptr = USB_ADD_BYTES(ptr, res.length);
329	}
330	return (0);			/* success */
331}
332#endif
333
334/*------------------------------------------------------------------------*
335 *  usbd_frame_zero - zero DMA-able memory
336 *------------------------------------------------------------------------*/
337void
338usbd_frame_zero(struct usb_page_cache *cache, usb_frlength_t offset,
339    usb_frlength_t len)
340{
341	struct usb_page_search res;
342
343	while (len != 0) {
344		usbd_get_page(cache, offset, &res);
345
346		if (res.length > len) {
347			res.length = len;
348		}
349		memset(res.buffer, 0, res.length);
350
351		offset += res.length;
352		len -= res.length;
353	}
354}
355
356#if USB_HAVE_BUSDMA
357
358/*------------------------------------------------------------------------*
359 *	usb_dma_lock_cb - dummy callback
360 *------------------------------------------------------------------------*/
361static void
362usb_dma_lock_cb(void *arg, bus_dma_lock_op_t op)
363{
364	/* we use "mtx_owned()" instead of this function */
365}
366
367/*------------------------------------------------------------------------*
368 *	usb_dma_tag_create - allocate a DMA tag
369 *
370 * NOTE: If the "align" parameter has a value of 1 the DMA-tag will
371 * allow multi-segment mappings. Else all mappings are single-segment.
372 *------------------------------------------------------------------------*/
373static void
374usb_dma_tag_create(struct usb_dma_tag *udt,
375    usb_size_t size, usb_size_t align)
376{
377	bus_dma_tag_t tag;
378
379	if (bus_dma_tag_create
380	    ( /* parent    */ udt->tag_parent->tag,
381	     /* alignment */ align,
382	     /* boundary  */ 0,
383	     /* lowaddr   */ (2ULL << (udt->tag_parent->dma_bits - 1)) - 1,
384	     /* highaddr  */ BUS_SPACE_MAXADDR,
385	     /* filter    */ NULL,
386	     /* filterarg */ NULL,
387	     /* maxsize   */ size,
388	     /* nsegments */ (align == 1 && size > 1) ?
389	    (2 + (size / USB_PAGE_SIZE)) : 1,
390	     /* maxsegsz  */ (align == 1 && size > USB_PAGE_SIZE) ?
391	    USB_PAGE_SIZE : size,
392	     /* flags     */ BUS_DMA_KEEP_PG_OFFSET,
393	     /* lockfn    */ &usb_dma_lock_cb,
394	     /* lockarg   */ NULL,
395	    &tag)) {
396		tag = NULL;
397	}
398	udt->tag = tag;
399}
400
401/*------------------------------------------------------------------------*
402 *	usb_dma_tag_free - free a DMA tag
403 *------------------------------------------------------------------------*/
404static void
405usb_dma_tag_destroy(struct usb_dma_tag *udt)
406{
407	bus_dma_tag_destroy(udt->tag);
408}
409
410/*------------------------------------------------------------------------*
411 *	usb_pc_alloc_mem_cb - BUS-DMA callback function
412 *------------------------------------------------------------------------*/
413static void
414usb_pc_alloc_mem_cb(void *arg, bus_dma_segment_t *segs,
415    int nseg, int error)
416{
417	usb_pc_common_mem_cb(arg, segs, nseg, error, 0);
418}
419
420/*------------------------------------------------------------------------*
421 *	usb_pc_load_mem_cb - BUS-DMA callback function
422 *------------------------------------------------------------------------*/
423static void
424usb_pc_load_mem_cb(void *arg, bus_dma_segment_t *segs,
425    int nseg, int error)
426{
427	usb_pc_common_mem_cb(arg, segs, nseg, error, 1);
428}
429
430/*------------------------------------------------------------------------*
431 *	usb_pc_common_mem_cb - BUS-DMA callback function
432 *------------------------------------------------------------------------*/
433static void
434usb_pc_common_mem_cb(void *arg, bus_dma_segment_t *segs,
435    int nseg, int error, uint8_t isload)
436{
437	struct usb_dma_parent_tag *uptag;
438	struct usb_page_cache *pc;
439	struct usb_page *pg;
440	usb_size_t rem;
441	bus_size_t off;
442	uint8_t owned;
443
444	pc = arg;
445	uptag = pc->tag_parent;
446
447	/*
448	 * XXX There is sometimes recursive locking here.
449	 * XXX We should try to find a better solution.
450	 * XXX Until further the "owned" variable does
451	 * XXX the trick.
452	 */
453
454	if (error) {
455		goto done;
456	}
457
458	off = 0;
459	pg = pc->page_start;
460	pg->physaddr = rounddown2(segs->ds_addr, USB_PAGE_SIZE);
461	rem = segs->ds_addr & (USB_PAGE_SIZE - 1);
462	pc->page_offset_buf = rem;
463	pc->page_offset_end += rem;
464#ifdef USB_DEBUG
465	if (nseg > 1) {
466		int x;
467
468		for (x = 0; x != nseg - 1; x++) {
469			if (((segs[x].ds_addr + segs[x].ds_len) & (USB_PAGE_SIZE - 1)) ==
470			    ((segs[x + 1].ds_addr & (USB_PAGE_SIZE - 1))))
471				continue;
472			/*
473			 * This check verifies there is no page offset
474			 * hole between any of the segments. See the
475			 * BUS_DMA_KEEP_PG_OFFSET flag.
476			 */
477			DPRINTFN(0, "Page offset was not preserved\n");
478			error = 1;
479			goto done;
480		}
481	}
482#endif
483	while (pc->ismultiseg) {
484		off += USB_PAGE_SIZE;
485		if (off >= (segs->ds_len + rem)) {
486			/* page crossing */
487			nseg--;
488			segs++;
489			off = 0;
490			rem = 0;
491			if (nseg == 0)
492				break;
493		}
494		pg++;
495		pg->physaddr = rounddown2(segs->ds_addr + off, USB_PAGE_SIZE);
496	}
497
498done:
499	owned = mtx_owned(uptag->mtx);
500	if (!owned)
501		USB_MTX_LOCK(uptag->mtx);
502
503	uptag->dma_error = (error ? 1 : 0);
504	if (isload) {
505		(uptag->func) (uptag);
506	} else {
507		cv_broadcast(uptag->cv);
508	}
509	if (!owned)
510		USB_MTX_UNLOCK(uptag->mtx);
511}
512
513/*------------------------------------------------------------------------*
514 *	usb_pc_alloc_mem - allocate DMA'able memory
515 *
516 * Returns:
517 *    0: Success
518 * Else: Failure
519 *------------------------------------------------------------------------*/
520uint8_t
521usb_pc_alloc_mem(struct usb_page_cache *pc, struct usb_page *pg,
522    usb_size_t size, usb_size_t align)
523{
524	struct usb_dma_parent_tag *uptag;
525	struct usb_dma_tag *utag;
526	bus_dmamap_t map;
527	void *ptr;
528	int err;
529
530	uptag = pc->tag_parent;
531
532	if (align != 1) {
533		/*
534	         * The alignment must be greater or equal to the
535	         * "size" else the object can be split between two
536	         * memory pages and we get a problem!
537	         */
538		while (align < size) {
539			align *= 2;
540			if (align == 0) {
541				goto error;
542			}
543		}
544#if 1
545		/*
546		 * XXX BUS-DMA workaround - FIXME later:
547		 *
548		 * We assume that the alignment at this point of
549		 * the code is greater than or equal to the size and
550		 * less than two times the size, so that if we double
551		 * the size, the size will be greater than the
552		 * alignment.
553		 *
554		 * The bus-dma system has a check for "alignment"
555		 * being less than "size". If that check fails we end
556		 * up using contigmalloc which is page based even for
557		 * small allocations. Try to avoid that to save
558		 * memory, hence we sometimes to a large number of
559		 * small allocations!
560		 */
561		if (size <= (USB_PAGE_SIZE / 2)) {
562			size *= 2;
563		}
564#endif
565	}
566	/* get the correct DMA tag */
567	utag = usb_dma_tag_find(uptag, size, align);
568	if (utag == NULL) {
569		goto error;
570	}
571	/* allocate memory */
572	if (bus_dmamem_alloc(
573	    utag->tag, &ptr, (BUS_DMA_WAITOK | BUS_DMA_COHERENT), &map)) {
574		goto error;
575	}
576	/* setup page cache */
577	pc->buffer = ptr;
578	pc->page_start = pg;
579	pc->page_offset_buf = 0;
580	pc->page_offset_end = size;
581	pc->map = map;
582	pc->tag = utag->tag;
583	pc->ismultiseg = (align == 1);
584
585	USB_MTX_LOCK(uptag->mtx);
586
587	/* load memory into DMA */
588	err = bus_dmamap_load(
589	    utag->tag, map, ptr, size, &usb_pc_alloc_mem_cb,
590	    pc, (BUS_DMA_WAITOK | BUS_DMA_COHERENT));
591
592	if (err == EINPROGRESS) {
593		cv_wait(uptag->cv, uptag->mtx);
594		err = 0;
595	}
596	USB_MTX_UNLOCK(uptag->mtx);
597
598	if (err || uptag->dma_error) {
599		bus_dmamem_free(utag->tag, ptr, map);
600		goto error;
601	}
602	pc->isloaded = 1;
603	memset(ptr, 0, size);
604
605	usb_pc_cpu_flush(pc);
606
607	return (0);
608
609error:
610	/* reset most of the page cache */
611	pc->buffer = NULL;
612	pc->page_start = NULL;
613	pc->page_offset_buf = 0;
614	pc->page_offset_end = 0;
615	pc->isloaded = 0;
616	pc->map = NULL;
617	pc->tag = NULL;
618	return (1);
619}
620
621/*------------------------------------------------------------------------*
622 *	usb_pc_free_mem - free DMA memory
623 *
624 * This function is NULL safe.
625 *------------------------------------------------------------------------*/
626void
627usb_pc_free_mem(struct usb_page_cache *pc)
628{
629	if (pc && pc->buffer) {
630		if (pc->isloaded)
631			bus_dmamap_unload(pc->tag, pc->map);
632
633		bus_dmamem_free(pc->tag, pc->buffer, pc->map);
634
635		pc->buffer = NULL;
636		pc->isloaded = 0;
637	}
638}
639
640/*------------------------------------------------------------------------*
641 *	usb_pc_load_mem - load virtual memory into DMA
642 *
643 * Return values:
644 * 0: Success
645 * Else: Error
646 *------------------------------------------------------------------------*/
647uint8_t
648usb_pc_load_mem(struct usb_page_cache *pc, usb_size_t size, uint8_t sync)
649{
650	/* setup page cache */
651	pc->page_offset_buf = 0;
652	pc->page_offset_end = size;
653	pc->ismultiseg = 1;
654
655	USB_MTX_ASSERT(pc->tag_parent->mtx, MA_OWNED);
656
657	if (size > 0) {
658		if (sync) {
659			struct usb_dma_parent_tag *uptag;
660			int err;
661
662			uptag = pc->tag_parent;
663
664			/*
665			 * We have to unload the previous loaded DMA
666			 * pages before trying to load a new one!
667			 */
668			if (pc->isloaded)
669				bus_dmamap_unload(pc->tag, pc->map);
670
671			/*
672			 * Try to load memory into DMA.
673			 */
674			err = bus_dmamap_load(
675			    pc->tag, pc->map, pc->buffer, size,
676			    &usb_pc_alloc_mem_cb, pc, BUS_DMA_WAITOK);
677			if (err == EINPROGRESS) {
678				cv_wait(uptag->cv, uptag->mtx);
679				err = 0;
680			}
681			if (err || uptag->dma_error) {
682				pc->isloaded = 0;
683				return (1);
684			}
685		} else {
686			/*
687			 * We have to unload the previous loaded DMA
688			 * pages before trying to load a new one!
689			 */
690			if (pc->isloaded)
691				bus_dmamap_unload(pc->tag, pc->map);
692
693			/*
694			 * Try to load memory into DMA. The callback
695			 * will be called in all cases:
696			 */
697			if (bus_dmamap_load(
698			    pc->tag, pc->map, pc->buffer, size,
699			    &usb_pc_load_mem_cb, pc, BUS_DMA_WAITOK)) {
700			}
701		}
702		pc->isloaded = 1;
703	} else {
704		if (!sync) {
705			/*
706			 * Call callback so that refcount is decremented
707			 * properly:
708			 */
709			pc->tag_parent->dma_error = 0;
710			(pc->tag_parent->func) (pc->tag_parent);
711		}
712	}
713	return (0);
714}
715
716/*------------------------------------------------------------------------*
717 *	usb_pc_cpu_invalidate - invalidate CPU cache
718 *------------------------------------------------------------------------*/
719void
720usb_pc_cpu_invalidate(struct usb_page_cache *pc)
721{
722	if (pc->page_offset_end == pc->page_offset_buf) {
723		/* nothing has been loaded into this page cache! */
724		return;
725	}
726
727	/*
728	 * TODO: We currently do XXX_POSTREAD and XXX_PREREAD at the
729	 * same time, but in the future we should try to isolate the
730	 * different cases to optimise the code. --HPS
731	 */
732	bus_dmamap_sync(pc->tag, pc->map, BUS_DMASYNC_POSTREAD);
733	bus_dmamap_sync(pc->tag, pc->map, BUS_DMASYNC_PREREAD);
734}
735
736/*------------------------------------------------------------------------*
737 *	usb_pc_cpu_flush - flush CPU cache
738 *------------------------------------------------------------------------*/
739void
740usb_pc_cpu_flush(struct usb_page_cache *pc)
741{
742	if (pc->page_offset_end == pc->page_offset_buf) {
743		/* nothing has been loaded into this page cache! */
744		return;
745	}
746	bus_dmamap_sync(pc->tag, pc->map, BUS_DMASYNC_PREWRITE);
747}
748
749/*------------------------------------------------------------------------*
750 *	usb_pc_dmamap_create - create a DMA map
751 *
752 * Returns:
753 *    0: Success
754 * Else: Failure
755 *------------------------------------------------------------------------*/
756uint8_t
757usb_pc_dmamap_create(struct usb_page_cache *pc, usb_size_t size)
758{
759	struct usb_xfer_root *info;
760	struct usb_dma_tag *utag;
761
762	/* get info */
763	info = USB_DMATAG_TO_XROOT(pc->tag_parent);
764
765	/* sanity check */
766	if (info == NULL) {
767		goto error;
768	}
769	utag = usb_dma_tag_find(pc->tag_parent, size, 1);
770	if (utag == NULL) {
771		goto error;
772	}
773	/* create DMA map */
774	if (bus_dmamap_create(utag->tag, 0, &pc->map)) {
775		goto error;
776	}
777	pc->tag = utag->tag;
778	return 0;			/* success */
779
780error:
781	pc->map = NULL;
782	pc->tag = NULL;
783	return 1;			/* failure */
784}
785
786/*------------------------------------------------------------------------*
787 *	usb_pc_dmamap_destroy
788 *
789 * This function is NULL safe.
790 *------------------------------------------------------------------------*/
791void
792usb_pc_dmamap_destroy(struct usb_page_cache *pc)
793{
794	if (pc && pc->tag) {
795		if (pc->isloaded)
796			bus_dmamap_unload(pc->tag, pc->map);
797		bus_dmamap_destroy(pc->tag, pc->map);
798		pc->tag = NULL;
799		pc->map = NULL;
800	}
801}
802
803/*------------------------------------------------------------------------*
804 *	usb_dma_tag_find - factored out code
805 *------------------------------------------------------------------------*/
806struct usb_dma_tag *
807usb_dma_tag_find(struct usb_dma_parent_tag *udpt,
808    usb_size_t size, usb_size_t align)
809{
810	struct usb_dma_tag *udt;
811	uint8_t nudt;
812
813	USB_ASSERT(align > 0, ("Invalid parameter align = 0\n"));
814	USB_ASSERT(size > 0, ("Invalid parameter size = 0\n"));
815
816	udt = udpt->utag_first;
817	nudt = udpt->utag_max;
818
819	while (nudt--) {
820		if (udt->align == 0) {
821			usb_dma_tag_create(udt, size, align);
822			if (udt->tag == NULL) {
823				return (NULL);
824			}
825			udt->align = align;
826			udt->size = size;
827			return (udt);
828		}
829		if ((udt->align == align) && (udt->size == size)) {
830			return (udt);
831		}
832		udt++;
833	}
834	return (NULL);
835}
836
837/*------------------------------------------------------------------------*
838 *	usb_dma_tag_setup - initialise USB DMA tags
839 *------------------------------------------------------------------------*/
840void
841usb_dma_tag_setup(struct usb_dma_parent_tag *udpt,
842    struct usb_dma_tag *udt, bus_dma_tag_t dmat,
843    struct mtx *mtx, usb_dma_callback_t *func,
844    uint8_t ndmabits, uint8_t nudt)
845{
846	memset(udpt, 0, sizeof(*udpt));
847
848	/* sanity checking */
849	if ((nudt == 0) ||
850	    (ndmabits == 0) ||
851	    (mtx == NULL)) {
852		/* something is corrupt */
853		return;
854	}
855	/* initialise condition variable */
856	cv_init(udpt->cv, "USB DMA CV");
857
858	/* store some information */
859	udpt->mtx = mtx;
860	udpt->func = func;
861	udpt->tag = dmat;
862	udpt->utag_first = udt;
863	udpt->utag_max = nudt;
864	udpt->dma_bits = ndmabits;
865
866	while (nudt--) {
867		memset(udt, 0, sizeof(*udt));
868		udt->tag_parent = udpt;
869		udt++;
870	}
871}
872
873/*------------------------------------------------------------------------*
874 *	usb_bus_tag_unsetup - factored out code
875 *------------------------------------------------------------------------*/
876void
877usb_dma_tag_unsetup(struct usb_dma_parent_tag *udpt)
878{
879	struct usb_dma_tag *udt;
880	uint8_t nudt;
881
882	udt = udpt->utag_first;
883	nudt = udpt->utag_max;
884
885	while (nudt--) {
886		if (udt->align) {
887			/* destroy the USB DMA tag */
888			usb_dma_tag_destroy(udt);
889			udt->align = 0;
890		}
891		udt++;
892	}
893
894	if (udpt->utag_max) {
895		/* destroy the condition variable */
896		cv_destroy(udpt->cv);
897	}
898}
899
900/*------------------------------------------------------------------------*
901 *	usb_bdma_work_loop
902 *
903 * This function handles loading of virtual buffers into DMA and is
904 * only called when "dma_refcount" is zero.
905 *------------------------------------------------------------------------*/
906void
907usb_bdma_work_loop(struct usb_xfer_queue *pq)
908{
909	struct usb_xfer_root *info;
910	struct usb_xfer *xfer;
911	usb_frcount_t nframes;
912
913	xfer = pq->curr;
914	info = xfer->xroot;
915
916	USB_MTX_ASSERT(info->xfer_mtx, MA_OWNED);
917
918	if (xfer->error) {
919		/* some error happened */
920		USB_BUS_LOCK(info->bus);
921		usbd_transfer_done(xfer, 0);
922		USB_BUS_UNLOCK(info->bus);
923		return;
924	}
925	if (!xfer->flags_int.bdma_setup) {
926		struct usb_page *pg;
927		usb_frlength_t frlength_0;
928		uint8_t isread;
929
930		xfer->flags_int.bdma_setup = 1;
931
932		/* reset BUS-DMA load state */
933
934		info->dma_error = 0;
935
936		if (xfer->flags_int.isochronous_xfr) {
937			/* only one frame buffer */
938			nframes = 1;
939			frlength_0 = xfer->sumlen;
940		} else {
941			/* can be multiple frame buffers */
942			nframes = xfer->nframes;
943			frlength_0 = xfer->frlengths[0];
944		}
945
946		/*
947		 * Set DMA direction first. This is needed to
948		 * select the correct cache invalidate and cache
949		 * flush operations.
950		 */
951		isread = USB_GET_DATA_ISREAD(xfer);
952		pg = xfer->dma_page_ptr;
953
954		if (xfer->flags_int.control_xfr &&
955		    xfer->flags_int.control_hdr) {
956			/* special case */
957			if (xfer->flags_int.usb_mode == USB_MODE_DEVICE) {
958				/* The device controller writes to memory */
959				xfer->frbuffers[0].isread = 1;
960			} else {
961				/* The host controller reads from memory */
962				xfer->frbuffers[0].isread = 0;
963			}
964		} else {
965			/* default case */
966			xfer->frbuffers[0].isread = isread;
967		}
968
969		/*
970		 * Setup the "page_start" pointer which points to an array of
971		 * USB pages where information about the physical address of a
972		 * page will be stored. Also initialise the "isread" field of
973		 * the USB page caches.
974		 */
975		xfer->frbuffers[0].page_start = pg;
976
977		info->dma_nframes = nframes;
978		info->dma_currframe = 0;
979		info->dma_frlength_0 = frlength_0;
980
981		pg += (frlength_0 / USB_PAGE_SIZE);
982		pg += 2;
983
984		while (--nframes > 0) {
985			xfer->frbuffers[nframes].isread = isread;
986			xfer->frbuffers[nframes].page_start = pg;
987
988			pg += (xfer->frlengths[nframes] / USB_PAGE_SIZE);
989			pg += 2;
990		}
991	}
992	if (info->dma_error) {
993		USB_BUS_LOCK(info->bus);
994		usbd_transfer_done(xfer, USB_ERR_DMA_LOAD_FAILED);
995		USB_BUS_UNLOCK(info->bus);
996		return;
997	}
998	if (info->dma_currframe != info->dma_nframes) {
999		if (info->dma_currframe == 0) {
1000			/* special case */
1001			usb_pc_load_mem(xfer->frbuffers,
1002			    info->dma_frlength_0, 0);
1003		} else {
1004			/* default case */
1005			nframes = info->dma_currframe;
1006			usb_pc_load_mem(xfer->frbuffers + nframes,
1007			    xfer->frlengths[nframes], 0);
1008		}
1009
1010		/* advance frame index */
1011		info->dma_currframe++;
1012
1013		return;
1014	}
1015	/* go ahead */
1016	usb_bdma_pre_sync(xfer);
1017
1018	/* start loading next USB transfer, if any */
1019	usb_command_wrapper(pq, NULL);
1020
1021	/* finally start the hardware */
1022	usbd_pipe_enter(xfer);
1023}
1024
1025/*------------------------------------------------------------------------*
1026 *	usb_bdma_done_event
1027 *
1028 * This function is called when the BUS-DMA has loaded virtual memory
1029 * into DMA, if any.
1030 *------------------------------------------------------------------------*/
1031void
1032usb_bdma_done_event(struct usb_dma_parent_tag *udpt)
1033{
1034	struct usb_xfer_root *info;
1035
1036	info = USB_DMATAG_TO_XROOT(udpt);
1037
1038	USB_MTX_ASSERT(info->xfer_mtx, MA_OWNED);
1039
1040	/* copy error */
1041	info->dma_error = udpt->dma_error;
1042
1043	/* enter workloop again */
1044	usb_command_wrapper(&info->dma_q,
1045	    info->dma_q.curr);
1046}
1047
1048/*------------------------------------------------------------------------*
1049 *	usb_bdma_pre_sync
1050 *
1051 * This function handles DMA synchronisation that must be done before
1052 * an USB transfer is started.
1053 *------------------------------------------------------------------------*/
1054void
1055usb_bdma_pre_sync(struct usb_xfer *xfer)
1056{
1057	struct usb_page_cache *pc;
1058	usb_frcount_t nframes;
1059
1060	if (xfer->flags_int.isochronous_xfr) {
1061		/* only one frame buffer */
1062		nframes = 1;
1063	} else {
1064		/* can be multiple frame buffers */
1065		nframes = xfer->nframes;
1066	}
1067
1068	pc = xfer->frbuffers;
1069
1070	while (nframes--) {
1071		if (pc->isread) {
1072			usb_pc_cpu_invalidate(pc);
1073		} else {
1074			usb_pc_cpu_flush(pc);
1075		}
1076		pc++;
1077	}
1078}
1079
1080/*------------------------------------------------------------------------*
1081 *	usb_bdma_post_sync
1082 *
1083 * This function handles DMA synchronisation that must be done after
1084 * an USB transfer is complete.
1085 *------------------------------------------------------------------------*/
1086void
1087usb_bdma_post_sync(struct usb_xfer *xfer)
1088{
1089	struct usb_page_cache *pc;
1090	usb_frcount_t nframes;
1091
1092	if (xfer->flags_int.isochronous_xfr) {
1093		/* only one frame buffer */
1094		nframes = 1;
1095	} else {
1096		/* can be multiple frame buffers */
1097		nframes = xfer->nframes;
1098	}
1099
1100	pc = xfer->frbuffers;
1101
1102	while (nframes--) {
1103		if (pc->isread) {
1104			usb_pc_cpu_invalidate(pc);
1105		}
1106		pc++;
1107	}
1108}
1109
1110#endif
1111