1/* $FreeBSD$ */
2/*-
3 * Copyright (c) 2008 Hans Petter Selasky. All rights reserved.
4 *
5 * Redistribution and use in source and binary forms, with or without
6 * modification, are permitted provided that the following conditions
7 * are met:
8 * 1. Redistributions of source code must retain the above copyright
9 *    notice, this list of conditions and the following disclaimer.
10 * 2. Redistributions in binary form must reproduce the above copyright
11 *    notice, this list of conditions and the following disclaimer in the
12 *    documentation and/or other materials provided with the distribution.
13 *
14 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
15 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
16 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
17 * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
18 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
19 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
20 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
21 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
22 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
23 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
24 * SUCH DAMAGE.
25 */
26
27#ifdef USB_GLOBAL_INCLUDE_FILE
28#include USB_GLOBAL_INCLUDE_FILE
29#else
30#include <sys/stdint.h>
31#include <sys/stddef.h>
32#include <sys/param.h>
33#include <sys/queue.h>
34#include <sys/types.h>
35#include <sys/systm.h>
36#include <sys/kernel.h>
37#include <sys/bus.h>
38#include <sys/module.h>
39#include <sys/lock.h>
40#include <sys/mutex.h>
41#include <sys/condvar.h>
42#include <sys/sysctl.h>
43#include <sys/sx.h>
44#include <sys/unistd.h>
45#include <sys/callout.h>
46#include <sys/malloc.h>
47#include <sys/priv.h>
48#include <sys/proc.h>
49
50#include <dev/usb/usb.h>
51#include <dev/usb/usbdi.h>
52#include <dev/usb/usbdi_util.h>
53
54#define	USB_DEBUG_VAR usb_debug
55
56#include <dev/usb/usb_core.h>
57#include <dev/usb/usb_busdma.h>
58#include <dev/usb/usb_process.h>
59#include <dev/usb/usb_transfer.h>
60#include <dev/usb/usb_device.h>
61#include <dev/usb/usb_debug.h>
62#include <dev/usb/usb_util.h>
63
64#include <dev/usb/usb_controller.h>
65#include <dev/usb/usb_bus.h>
66#include <dev/usb/usb_pf.h>
67#endif			/* USB_GLOBAL_INCLUDE_FILE */
68
69struct usb_std_packet_size {
70	struct {
71		uint16_t min;		/* inclusive */
72		uint16_t max;		/* inclusive */
73	}	range;
74
75	uint16_t fixed[4];
76};
77
78static usb_callback_t usb_request_callback;
79
80static const struct usb_config usb_control_ep_cfg[USB_CTRL_XFER_MAX] = {
81
82	/* This transfer is used for generic control endpoint transfers */
83
84	[0] = {
85		.type = UE_CONTROL,
86		.endpoint = 0x00,	/* Control endpoint */
87		.direction = UE_DIR_ANY,
88		.bufsize = USB_EP0_BUFSIZE,	/* bytes */
89		.flags = {.proxy_buffer = 1,},
90		.callback = &usb_request_callback,
91		.usb_mode = USB_MODE_DUAL,	/* both modes */
92	},
93
94	/* This transfer is used for generic clear stall only */
95
96	[1] = {
97		.type = UE_CONTROL,
98		.endpoint = 0x00,	/* Control pipe */
99		.direction = UE_DIR_ANY,
100		.bufsize = sizeof(struct usb_device_request),
101		.callback = &usb_do_clear_stall_callback,
102		.timeout = 1000,	/* 1 second */
103		.interval = 50,	/* 50ms */
104		.usb_mode = USB_MODE_HOST,
105	},
106};
107
108/* function prototypes */
109
110static void	usbd_update_max_frame_size(struct usb_xfer *);
111static void	usbd_transfer_unsetup_sub(struct usb_xfer_root *, uint8_t);
112static void	usbd_control_transfer_init(struct usb_xfer *);
113static int	usbd_setup_ctrl_transfer(struct usb_xfer *);
114static void	usb_callback_proc(struct usb_proc_msg *);
115static void	usbd_callback_ss_done_defer(struct usb_xfer *);
116static void	usbd_callback_wrapper(struct usb_xfer_queue *);
117static void	usbd_transfer_start_cb(void *);
118static uint8_t	usbd_callback_wrapper_sub(struct usb_xfer *);
119static void	usbd_get_std_packet_size(struct usb_std_packet_size *ptr,
120		    uint8_t type, enum usb_dev_speed speed);
121
122/*------------------------------------------------------------------------*
123 *	usb_request_callback
124 *------------------------------------------------------------------------*/
125static void
126usb_request_callback(struct usb_xfer *xfer, usb_error_t error)
127{
128	if (xfer->flags_int.usb_mode == USB_MODE_DEVICE)
129		usb_handle_request_callback(xfer, error);
130	else
131		usbd_do_request_callback(xfer, error);
132}
133
134/*------------------------------------------------------------------------*
135 *	usbd_update_max_frame_size
136 *
137 * This function updates the maximum frame size, hence high speed USB
138 * can transfer multiple consecutive packets.
139 *------------------------------------------------------------------------*/
140static void
141usbd_update_max_frame_size(struct usb_xfer *xfer)
142{
143	/* compute maximum frame size */
144	/* this computation should not overflow 16-bit */
145	/* max = 15 * 1024 */
146
147	xfer->max_frame_size = xfer->max_packet_size * xfer->max_packet_count;
148}
149
150/*------------------------------------------------------------------------*
151 *	usbd_get_dma_delay
152 *
153 * The following function is called when we need to
154 * synchronize with DMA hardware.
155 *
156 * Returns:
157 *    0: no DMA delay required
158 * Else: milliseconds of DMA delay
159 *------------------------------------------------------------------------*/
160usb_timeout_t
161usbd_get_dma_delay(struct usb_device *udev)
162{
163	struct usb_bus_methods *mtod;
164	uint32_t temp;
165
166	mtod = udev->bus->methods;
167	temp = 0;
168
169	if (mtod->get_dma_delay) {
170		(mtod->get_dma_delay) (udev, &temp);
171		/*
172		 * Round up and convert to milliseconds. Note that we use
173		 * 1024 milliseconds per second. to save a division.
174		 */
175		temp += 0x3FF;
176		temp /= 0x400;
177	}
178	return (temp);
179}
180
181/*------------------------------------------------------------------------*
182 *	usbd_transfer_setup_sub_malloc
183 *
184 * This function will allocate one or more DMA'able memory chunks
185 * according to "size", "align" and "count" arguments. "ppc" is
186 * pointed to a linear array of USB page caches afterwards.
187 *
188 * If the "align" argument is equal to "1" a non-contiguous allocation
189 * can happen. Else if the "align" argument is greater than "1", the
190 * allocation will always be contiguous in memory.
191 *
192 * Returns:
193 *    0: Success
194 * Else: Failure
195 *------------------------------------------------------------------------*/
196#if USB_HAVE_BUSDMA
197uint8_t
198usbd_transfer_setup_sub_malloc(struct usb_setup_params *parm,
199    struct usb_page_cache **ppc, usb_size_t size, usb_size_t align,
200    usb_size_t count)
201{
202	struct usb_page_cache *pc;
203	struct usb_page *pg;
204	void *buf;
205	usb_size_t n_dma_pc;
206	usb_size_t n_dma_pg;
207	usb_size_t n_obj;
208	usb_size_t x;
209	usb_size_t y;
210	usb_size_t r;
211	usb_size_t z;
212
213	USB_ASSERT(align > 0, ("Invalid alignment, 0x%08x\n",
214	    align));
215	USB_ASSERT(size > 0, ("Invalid size = 0\n"));
216
217	if (count == 0) {
218		return (0);		/* nothing to allocate */
219	}
220	/*
221	 * Make sure that the size is aligned properly.
222	 */
223	size = -((-size) & (-align));
224
225	/*
226	 * Try multi-allocation chunks to reduce the number of DMA
227	 * allocations, hence DMA allocations are slow.
228	 */
229	if (align == 1) {
230		/* special case - non-cached multi page DMA memory */
231		n_dma_pc = count;
232		n_dma_pg = (2 + (size / USB_PAGE_SIZE));
233		n_obj = 1;
234	} else if (size >= USB_PAGE_SIZE) {
235		n_dma_pc = count;
236		n_dma_pg = 1;
237		n_obj = 1;
238	} else {
239		/* compute number of objects per page */
240		n_obj = (USB_PAGE_SIZE / size);
241		/*
242		 * Compute number of DMA chunks, rounded up
243		 * to nearest one:
244		 */
245		n_dma_pc = ((count + n_obj - 1) / n_obj);
246		n_dma_pg = 1;
247	}
248
249	/*
250	 * DMA memory is allocated once, but mapped twice. That's why
251	 * there is one list for auto-free and another list for
252	 * non-auto-free which only holds the mapping and not the
253	 * allocation.
254	 */
255	if (parm->buf == NULL) {
256		/* reserve memory (auto-free) */
257		parm->dma_page_ptr += n_dma_pc * n_dma_pg;
258		parm->dma_page_cache_ptr += n_dma_pc;
259
260		/* reserve memory (no-auto-free) */
261		parm->dma_page_ptr += count * n_dma_pg;
262		parm->xfer_page_cache_ptr += count;
263		return (0);
264	}
265	for (x = 0; x != n_dma_pc; x++) {
266		/* need to initialize the page cache */
267		parm->dma_page_cache_ptr[x].tag_parent =
268		    &parm->curr_xfer->xroot->dma_parent_tag;
269	}
270	for (x = 0; x != count; x++) {
271		/* need to initialize the page cache */
272		parm->xfer_page_cache_ptr[x].tag_parent =
273		    &parm->curr_xfer->xroot->dma_parent_tag;
274	}
275
276	if (ppc) {
277		*ppc = parm->xfer_page_cache_ptr;
278	}
279	r = count;			/* set remainder count */
280	z = n_obj * size;		/* set allocation size */
281	pc = parm->xfer_page_cache_ptr;
282	pg = parm->dma_page_ptr;
283
284	for (x = 0; x != n_dma_pc; x++) {
285
286		if (r < n_obj) {
287			/* compute last remainder */
288			z = r * size;
289			n_obj = r;
290		}
291		if (usb_pc_alloc_mem(parm->dma_page_cache_ptr,
292		    pg, z, align)) {
293			return (1);	/* failure */
294		}
295		/* Set beginning of current buffer */
296		buf = parm->dma_page_cache_ptr->buffer;
297		/* Make room for one DMA page cache and one page */
298		parm->dma_page_cache_ptr++;
299		pg += n_dma_pg;
300
301		for (y = 0; (y != n_obj); y++, r--, pc++, pg += n_dma_pg) {
302
303			/* Load sub-chunk into DMA */
304			if (usb_pc_dmamap_create(pc, size)) {
305				return (1);	/* failure */
306			}
307			pc->buffer = USB_ADD_BYTES(buf, y * size);
308			pc->page_start = pg;
309
310			mtx_lock(pc->tag_parent->mtx);
311			if (usb_pc_load_mem(pc, size, 1 /* synchronous */ )) {
312				mtx_unlock(pc->tag_parent->mtx);
313				return (1);	/* failure */
314			}
315			mtx_unlock(pc->tag_parent->mtx);
316		}
317	}
318
319	parm->xfer_page_cache_ptr = pc;
320	parm->dma_page_ptr = pg;
321	return (0);
322}
323#endif
324
325/*------------------------------------------------------------------------*
326 *	usbd_transfer_setup_sub - transfer setup subroutine
327 *
328 * This function must be called from the "xfer_setup" callback of the
329 * USB Host or Device controller driver when setting up an USB
330 * transfer. This function will setup correct packet sizes, buffer
331 * sizes, flags and more, that are stored in the "usb_xfer"
332 * structure.
333 *------------------------------------------------------------------------*/
334void
335usbd_transfer_setup_sub(struct usb_setup_params *parm)
336{
337	enum {
338		REQ_SIZE = 8,
339		MIN_PKT = 8,
340	};
341	struct usb_xfer *xfer = parm->curr_xfer;
342	const struct usb_config *setup = parm->curr_setup;
343	struct usb_endpoint_ss_comp_descriptor *ecomp;
344	struct usb_endpoint_descriptor *edesc;
345	struct usb_std_packet_size std_size;
346	usb_frcount_t n_frlengths;
347	usb_frcount_t n_frbuffers;
348	usb_frcount_t x;
349	uint16_t maxp_old;
350	uint8_t type;
351	uint8_t zmps;
352
353	/*
354	 * Sanity check. The following parameters must be initialized before
355	 * calling this function.
356	 */
357	if ((parm->hc_max_packet_size == 0) ||
358	    (parm->hc_max_packet_count == 0) ||
359	    (parm->hc_max_frame_size == 0)) {
360		parm->err = USB_ERR_INVAL;
361		goto done;
362	}
363	edesc = xfer->endpoint->edesc;
364	ecomp = xfer->endpoint->ecomp;
365
366	type = (edesc->bmAttributes & UE_XFERTYPE);
367
368	xfer->flags = setup->flags;
369	xfer->nframes = setup->frames;
370	xfer->timeout = setup->timeout;
371	xfer->callback = setup->callback;
372	xfer->interval = setup->interval;
373	xfer->endpointno = edesc->bEndpointAddress;
374	xfer->max_packet_size = UGETW(edesc->wMaxPacketSize);
375	xfer->max_packet_count = 1;
376	/* make a shadow copy: */
377	xfer->flags_int.usb_mode = parm->udev->flags.usb_mode;
378
379	parm->bufsize = setup->bufsize;
380
381	switch (parm->speed) {
382	case USB_SPEED_HIGH:
383		switch (type) {
384		case UE_ISOCHRONOUS:
385		case UE_INTERRUPT:
386			xfer->max_packet_count +=
387			    (xfer->max_packet_size >> 11) & 3;
388
389			/* check for invalid max packet count */
390			if (xfer->max_packet_count > 3)
391				xfer->max_packet_count = 3;
392			break;
393		default:
394			break;
395		}
396		xfer->max_packet_size &= 0x7FF;
397		break;
398	case USB_SPEED_SUPER:
399		xfer->max_packet_count += (xfer->max_packet_size >> 11) & 3;
400
401		if (ecomp != NULL)
402			xfer->max_packet_count += ecomp->bMaxBurst;
403
404		if ((xfer->max_packet_count == 0) ||
405		    (xfer->max_packet_count > 16))
406			xfer->max_packet_count = 16;
407
408		switch (type) {
409		case UE_CONTROL:
410			xfer->max_packet_count = 1;
411			break;
412		case UE_ISOCHRONOUS:
413			if (ecomp != NULL) {
414				uint8_t mult;
415
416				mult = UE_GET_SS_ISO_MULT(
417				    ecomp->bmAttributes) + 1;
418				if (mult > 3)
419					mult = 3;
420
421				xfer->max_packet_count *= mult;
422			}
423			break;
424		default:
425			break;
426		}
427		xfer->max_packet_size &= 0x7FF;
428		break;
429	default:
430		break;
431	}
432	/* range check "max_packet_count" */
433
434	if (xfer->max_packet_count > parm->hc_max_packet_count) {
435		xfer->max_packet_count = parm->hc_max_packet_count;
436	}
437
438	/* store max packet size value before filtering */
439
440	maxp_old = xfer->max_packet_size;
441
442	/* filter "wMaxPacketSize" according to HC capabilities */
443
444	if ((xfer->max_packet_size > parm->hc_max_packet_size) ||
445	    (xfer->max_packet_size == 0)) {
446		xfer->max_packet_size = parm->hc_max_packet_size;
447	}
448	/* filter "wMaxPacketSize" according to standard sizes */
449
450	usbd_get_std_packet_size(&std_size, type, parm->speed);
451
452	if (std_size.range.min || std_size.range.max) {
453
454		if (xfer->max_packet_size < std_size.range.min) {
455			xfer->max_packet_size = std_size.range.min;
456		}
457		if (xfer->max_packet_size > std_size.range.max) {
458			xfer->max_packet_size = std_size.range.max;
459		}
460	} else {
461
462		if (xfer->max_packet_size >= std_size.fixed[3]) {
463			xfer->max_packet_size = std_size.fixed[3];
464		} else if (xfer->max_packet_size >= std_size.fixed[2]) {
465			xfer->max_packet_size = std_size.fixed[2];
466		} else if (xfer->max_packet_size >= std_size.fixed[1]) {
467			xfer->max_packet_size = std_size.fixed[1];
468		} else {
469			/* only one possibility left */
470			xfer->max_packet_size = std_size.fixed[0];
471		}
472	}
473
474	/*
475	 * Check if the max packet size was outside its allowed range
476	 * and clamped to a valid value:
477	 */
478	if (maxp_old != xfer->max_packet_size)
479		xfer->flags_int.maxp_was_clamped = 1;
480
481	/* compute "max_frame_size" */
482
483	usbd_update_max_frame_size(xfer);
484
485	/* check interrupt interval and transfer pre-delay */
486
487	if (type == UE_ISOCHRONOUS) {
488
489		uint16_t frame_limit;
490
491		xfer->interval = 0;	/* not used, must be zero */
492		xfer->flags_int.isochronous_xfr = 1;	/* set flag */
493
494		if (xfer->timeout == 0) {
495			/*
496			 * set a default timeout in
497			 * case something goes wrong!
498			 */
499			xfer->timeout = 1000 / 4;
500		}
501		switch (parm->speed) {
502		case USB_SPEED_LOW:
503		case USB_SPEED_FULL:
504			frame_limit = USB_MAX_FS_ISOC_FRAMES_PER_XFER;
505			xfer->fps_shift = 0;
506			break;
507		default:
508			frame_limit = USB_MAX_HS_ISOC_FRAMES_PER_XFER;
509			xfer->fps_shift = edesc->bInterval;
510			if (xfer->fps_shift > 0)
511				xfer->fps_shift--;
512			if (xfer->fps_shift > 3)
513				xfer->fps_shift = 3;
514			if (xfer->flags.pre_scale_frames != 0)
515				xfer->nframes <<= (3 - xfer->fps_shift);
516			break;
517		}
518
519		if (xfer->nframes > frame_limit) {
520			/*
521			 * this is not going to work
522			 * cross hardware
523			 */
524			parm->err = USB_ERR_INVAL;
525			goto done;
526		}
527		if (xfer->nframes == 0) {
528			/*
529			 * this is not a valid value
530			 */
531			parm->err = USB_ERR_ZERO_NFRAMES;
532			goto done;
533		}
534	} else {
535
536		/*
537		 * If a value is specified use that else check the
538		 * endpoint descriptor!
539		 */
540		if (type == UE_INTERRUPT) {
541
542			uint32_t temp;
543
544			if (xfer->interval == 0) {
545
546				xfer->interval = edesc->bInterval;
547
548				switch (parm->speed) {
549				case USB_SPEED_LOW:
550				case USB_SPEED_FULL:
551					break;
552				default:
553					/* 125us -> 1ms */
554					if (xfer->interval < 4)
555						xfer->interval = 1;
556					else if (xfer->interval > 16)
557						xfer->interval = (1 << (16 - 4));
558					else
559						xfer->interval =
560						    (1 << (xfer->interval - 4));
561					break;
562				}
563			}
564
565			if (xfer->interval == 0) {
566				/*
567				 * One millisecond is the smallest
568				 * interval we support:
569				 */
570				xfer->interval = 1;
571			}
572
573			xfer->fps_shift = 0;
574			temp = 1;
575
576			while ((temp != 0) && (temp < xfer->interval)) {
577				xfer->fps_shift++;
578				temp *= 2;
579			}
580
581			switch (parm->speed) {
582			case USB_SPEED_LOW:
583			case USB_SPEED_FULL:
584				break;
585			default:
586				xfer->fps_shift += 3;
587				break;
588			}
589		}
590	}
591
592	/*
593	 * NOTE: we do not allow "max_packet_size" or "max_frame_size"
594	 * to be equal to zero when setting up USB transfers, hence
595	 * this leads to alot of extra code in the USB kernel.
596	 */
597
598	if ((xfer->max_frame_size == 0) ||
599	    (xfer->max_packet_size == 0)) {
600
601		zmps = 1;
602
603		if ((parm->bufsize <= MIN_PKT) &&
604		    (type != UE_CONTROL) &&
605		    (type != UE_BULK)) {
606
607			/* workaround */
608			xfer->max_packet_size = MIN_PKT;
609			xfer->max_packet_count = 1;
610			parm->bufsize = 0;	/* automatic setup length */
611			usbd_update_max_frame_size(xfer);
612
613		} else {
614			parm->err = USB_ERR_ZERO_MAXP;
615			goto done;
616		}
617
618	} else {
619		zmps = 0;
620	}
621
622	/*
623	 * check if we should setup a default
624	 * length:
625	 */
626
627	if (parm->bufsize == 0) {
628
629		parm->bufsize = xfer->max_frame_size;
630
631		if (type == UE_ISOCHRONOUS) {
632			parm->bufsize *= xfer->nframes;
633		}
634	}
635	/*
636	 * check if we are about to setup a proxy
637	 * type of buffer:
638	 */
639
640	if (xfer->flags.proxy_buffer) {
641
642		/* round bufsize up */
643
644		parm->bufsize += (xfer->max_frame_size - 1);
645
646		if (parm->bufsize < xfer->max_frame_size) {
647			/* length wrapped around */
648			parm->err = USB_ERR_INVAL;
649			goto done;
650		}
651		/* subtract remainder */
652
653		parm->bufsize -= (parm->bufsize % xfer->max_frame_size);
654
655		/* add length of USB device request structure, if any */
656
657		if (type == UE_CONTROL) {
658			parm->bufsize += REQ_SIZE;	/* SETUP message */
659		}
660	}
661	xfer->max_data_length = parm->bufsize;
662
663	/* Setup "n_frlengths" and "n_frbuffers" */
664
665	if (type == UE_ISOCHRONOUS) {
666		n_frlengths = xfer->nframes;
667		n_frbuffers = 1;
668	} else {
669
670		if (type == UE_CONTROL) {
671			xfer->flags_int.control_xfr = 1;
672			if (xfer->nframes == 0) {
673				if (parm->bufsize <= REQ_SIZE) {
674					/*
675					 * there will never be any data
676					 * stage
677					 */
678					xfer->nframes = 1;
679				} else {
680					xfer->nframes = 2;
681				}
682			}
683		} else {
684			if (xfer->nframes == 0) {
685				xfer->nframes = 1;
686			}
687		}
688
689		n_frlengths = xfer->nframes;
690		n_frbuffers = xfer->nframes;
691	}
692
693	/*
694	 * check if we have room for the
695	 * USB device request structure:
696	 */
697
698	if (type == UE_CONTROL) {
699
700		if (xfer->max_data_length < REQ_SIZE) {
701			/* length wrapped around or too small bufsize */
702			parm->err = USB_ERR_INVAL;
703			goto done;
704		}
705		xfer->max_data_length -= REQ_SIZE;
706	}
707	/*
708	 * Setup "frlengths" and shadow "frlengths" for keeping the
709	 * initial frame lengths when a USB transfer is complete. This
710	 * information is useful when computing isochronous offsets.
711	 */
712	xfer->frlengths = parm->xfer_length_ptr;
713	parm->xfer_length_ptr += 2 * n_frlengths;
714
715	/* setup "frbuffers" */
716	xfer->frbuffers = parm->xfer_page_cache_ptr;
717	parm->xfer_page_cache_ptr += n_frbuffers;
718
719	/* initialize max frame count */
720	xfer->max_frame_count = xfer->nframes;
721
722	/*
723	 * check if we need to setup
724	 * a local buffer:
725	 */
726
727	if (!xfer->flags.ext_buffer) {
728#if USB_HAVE_BUSDMA
729		struct usb_page_search page_info;
730		struct usb_page_cache *pc;
731
732		if (usbd_transfer_setup_sub_malloc(parm,
733		    &pc, parm->bufsize, 1, 1)) {
734			parm->err = USB_ERR_NOMEM;
735		} else if (parm->buf != NULL) {
736
737			usbd_get_page(pc, 0, &page_info);
738
739			xfer->local_buffer = page_info.buffer;
740
741			usbd_xfer_set_frame_offset(xfer, 0, 0);
742
743			if ((type == UE_CONTROL) && (n_frbuffers > 1)) {
744				usbd_xfer_set_frame_offset(xfer, REQ_SIZE, 1);
745			}
746		}
747#else
748		/* align data */
749		parm->size[0] += ((-parm->size[0]) & (USB_HOST_ALIGN - 1));
750
751		if (parm->buf != NULL) {
752			xfer->local_buffer =
753			    USB_ADD_BYTES(parm->buf, parm->size[0]);
754
755			usbd_xfer_set_frame_offset(xfer, 0, 0);
756
757			if ((type == UE_CONTROL) && (n_frbuffers > 1)) {
758				usbd_xfer_set_frame_offset(xfer, REQ_SIZE, 1);
759			}
760		}
761		parm->size[0] += parm->bufsize;
762
763		/* align data again */
764		parm->size[0] += ((-parm->size[0]) & (USB_HOST_ALIGN - 1));
765#endif
766	}
767	/*
768	 * Compute maximum buffer size
769	 */
770
771	if (parm->bufsize_max < parm->bufsize) {
772		parm->bufsize_max = parm->bufsize;
773	}
774#if USB_HAVE_BUSDMA
775	if (xfer->flags_int.bdma_enable) {
776		/*
777		 * Setup "dma_page_ptr".
778		 *
779		 * Proof for formula below:
780		 *
781		 * Assume there are three USB frames having length "a", "b" and
782		 * "c". These USB frames will at maximum need "z"
783		 * "usb_page" structures. "z" is given by:
784		 *
785		 * z = ((a / USB_PAGE_SIZE) + 2) + ((b / USB_PAGE_SIZE) + 2) +
786		 * ((c / USB_PAGE_SIZE) + 2);
787		 *
788		 * Constraining "a", "b" and "c" like this:
789		 *
790		 * (a + b + c) <= parm->bufsize
791		 *
792		 * We know that:
793		 *
794		 * z <= ((parm->bufsize / USB_PAGE_SIZE) + (3*2));
795		 *
796		 * Here is the general formula:
797		 */
798		xfer->dma_page_ptr = parm->dma_page_ptr;
799		parm->dma_page_ptr += (2 * n_frbuffers);
800		parm->dma_page_ptr += (parm->bufsize / USB_PAGE_SIZE);
801	}
802#endif
803	if (zmps) {
804		/* correct maximum data length */
805		xfer->max_data_length = 0;
806	}
807	/* subtract USB frame remainder from "hc_max_frame_size" */
808
809	xfer->max_hc_frame_size =
810	    (parm->hc_max_frame_size -
811	    (parm->hc_max_frame_size % xfer->max_frame_size));
812
813	if (xfer->max_hc_frame_size == 0) {
814		parm->err = USB_ERR_INVAL;
815		goto done;
816	}
817
818	/* initialize frame buffers */
819
820	if (parm->buf) {
821		for (x = 0; x != n_frbuffers; x++) {
822			xfer->frbuffers[x].tag_parent =
823			    &xfer->xroot->dma_parent_tag;
824#if USB_HAVE_BUSDMA
825			if (xfer->flags_int.bdma_enable &&
826			    (parm->bufsize_max > 0)) {
827
828				if (usb_pc_dmamap_create(
829				    xfer->frbuffers + x,
830				    parm->bufsize_max)) {
831					parm->err = USB_ERR_NOMEM;
832					goto done;
833				}
834			}
835#endif
836		}
837	}
838done:
839	if (parm->err) {
840		/*
841		 * Set some dummy values so that we avoid division by zero:
842		 */
843		xfer->max_hc_frame_size = 1;
844		xfer->max_frame_size = 1;
845		xfer->max_packet_size = 1;
846		xfer->max_data_length = 0;
847		xfer->nframes = 0;
848		xfer->max_frame_count = 0;
849	}
850}
851
852/*------------------------------------------------------------------------*
853 *	usbd_transfer_setup - setup an array of USB transfers
854 *
855 * NOTE: You must always call "usbd_transfer_unsetup" after calling
856 * "usbd_transfer_setup" if success was returned.
857 *
858 * The idea is that the USB device driver should pre-allocate all its
859 * transfers by one call to this function.
860 *
861 * Return values:
862 *    0: Success
863 * Else: Failure
864 *------------------------------------------------------------------------*/
865usb_error_t
866usbd_transfer_setup(struct usb_device *udev,
867    const uint8_t *ifaces, struct usb_xfer **ppxfer,
868    const struct usb_config *setup_start, uint16_t n_setup,
869    void *priv_sc, struct mtx *xfer_mtx)
870{
871	const struct usb_config *setup_end = setup_start + n_setup;
872	const struct usb_config *setup;
873	struct usb_setup_params *parm;
874	struct usb_endpoint *ep;
875	struct usb_xfer_root *info;
876	struct usb_xfer *xfer;
877	void *buf = NULL;
878	usb_error_t error = 0;
879	uint16_t n;
880	uint16_t refcount;
881	uint8_t do_unlock;
882
883	WITNESS_WARN(WARN_GIANTOK | WARN_SLEEPOK, NULL,
884	    "usbd_transfer_setup can sleep!");
885
886	/* do some checking first */
887
888	if (n_setup == 0) {
889		DPRINTFN(6, "setup array has zero length!\n");
890		return (USB_ERR_INVAL);
891	}
892	if (ifaces == 0) {
893		DPRINTFN(6, "ifaces array is NULL!\n");
894		return (USB_ERR_INVAL);
895	}
896	if (xfer_mtx == NULL) {
897		DPRINTFN(6, "using global lock\n");
898		xfer_mtx = &Giant;
899	}
900
901	/* more sanity checks */
902
903	for (setup = setup_start, n = 0;
904	    setup != setup_end; setup++, n++) {
905		if (setup->bufsize == (usb_frlength_t)-1) {
906			error = USB_ERR_BAD_BUFSIZE;
907			DPRINTF("invalid bufsize\n");
908		}
909		if (setup->callback == NULL) {
910			error = USB_ERR_NO_CALLBACK;
911			DPRINTF("no callback\n");
912		}
913		ppxfer[n] = NULL;
914	}
915
916	if (error)
917		return (error);
918
919	/* Protect scratch area */
920	do_unlock = usbd_enum_lock(udev);
921
922	refcount = 0;
923	info = NULL;
924
925	parm = &udev->scratch.xfer_setup[0].parm;
926	memset(parm, 0, sizeof(*parm));
927
928	parm->udev = udev;
929	parm->speed = usbd_get_speed(udev);
930	parm->hc_max_packet_count = 1;
931
932	if (parm->speed >= USB_SPEED_MAX) {
933		parm->err = USB_ERR_INVAL;
934		goto done;
935	}
936	/* setup all transfers */
937
938	while (1) {
939
940		if (buf) {
941			/*
942			 * Initialize the "usb_xfer_root" structure,
943			 * which is common for all our USB transfers.
944			 */
945			info = USB_ADD_BYTES(buf, 0);
946
947			info->memory_base = buf;
948			info->memory_size = parm->size[0];
949
950#if USB_HAVE_BUSDMA
951			info->dma_page_cache_start = USB_ADD_BYTES(buf, parm->size[4]);
952			info->dma_page_cache_end = USB_ADD_BYTES(buf, parm->size[5]);
953#endif
954			info->xfer_page_cache_start = USB_ADD_BYTES(buf, parm->size[5]);
955			info->xfer_page_cache_end = USB_ADD_BYTES(buf, parm->size[2]);
956
957			cv_init(&info->cv_drain, "WDRAIN");
958
959			info->xfer_mtx = xfer_mtx;
960#if USB_HAVE_BUSDMA
961			usb_dma_tag_setup(&info->dma_parent_tag,
962			    parm->dma_tag_p, udev->bus->dma_parent_tag[0].tag,
963			    xfer_mtx, &usb_bdma_done_event, 32, parm->dma_tag_max);
964#endif
965
966			info->bus = udev->bus;
967			info->udev = udev;
968
969			TAILQ_INIT(&info->done_q.head);
970			info->done_q.command = &usbd_callback_wrapper;
971#if USB_HAVE_BUSDMA
972			TAILQ_INIT(&info->dma_q.head);
973			info->dma_q.command = &usb_bdma_work_loop;
974#endif
975			info->done_m[0].hdr.pm_callback = &usb_callback_proc;
976			info->done_m[0].xroot = info;
977			info->done_m[1].hdr.pm_callback = &usb_callback_proc;
978			info->done_m[1].xroot = info;
979
980			/*
981			 * In device side mode control endpoint
982			 * requests need to run from a separate
983			 * context, else there is a chance of
984			 * deadlock!
985			 */
986			if (setup_start == usb_control_ep_cfg)
987				info->done_p =
988				    USB_BUS_CONTROL_XFER_PROC(udev->bus);
989			else if (xfer_mtx == &Giant)
990				info->done_p =
991				    USB_BUS_GIANT_PROC(udev->bus);
992			else
993				info->done_p =
994				    USB_BUS_NON_GIANT_PROC(udev->bus);
995		}
996		/* reset sizes */
997
998		parm->size[0] = 0;
999		parm->buf = buf;
1000		parm->size[0] += sizeof(info[0]);
1001
1002		for (setup = setup_start, n = 0;
1003		    setup != setup_end; setup++, n++) {
1004
1005			/* skip USB transfers without callbacks: */
1006			if (setup->callback == NULL) {
1007				continue;
1008			}
1009			/* see if there is a matching endpoint */
1010			ep = usbd_get_endpoint(udev,
1011			    ifaces[setup->if_index], setup);
1012
1013			/*
1014			 * Check that the USB PIPE is valid and that
1015			 * the endpoint mode is proper.
1016			 *
1017			 * Make sure we don't allocate a streams
1018			 * transfer when such a combination is not
1019			 * valid.
1020			 */
1021			if ((ep == NULL) || (ep->methods == NULL) ||
1022			    ((ep->ep_mode != USB_EP_MODE_STREAMS) &&
1023			    (ep->ep_mode != USB_EP_MODE_DEFAULT)) ||
1024			    (setup->stream_id != 0 &&
1025			    (setup->stream_id >= USB_MAX_EP_STREAMS ||
1026			    (ep->ep_mode != USB_EP_MODE_STREAMS)))) {
1027				if (setup->flags.no_pipe_ok)
1028					continue;
1029				if ((setup->usb_mode != USB_MODE_DUAL) &&
1030				    (setup->usb_mode != udev->flags.usb_mode))
1031					continue;
1032				parm->err = USB_ERR_NO_PIPE;
1033				goto done;
1034			}
1035
1036			/* align data properly */
1037			parm->size[0] += ((-parm->size[0]) & (USB_HOST_ALIGN - 1));
1038
1039			/* store current setup pointer */
1040			parm->curr_setup = setup;
1041
1042			if (buf) {
1043				/*
1044				 * Common initialization of the
1045				 * "usb_xfer" structure.
1046				 */
1047				xfer = USB_ADD_BYTES(buf, parm->size[0]);
1048				xfer->address = udev->address;
1049				xfer->priv_sc = priv_sc;
1050				xfer->xroot = info;
1051
1052				usb_callout_init_mtx(&xfer->timeout_handle,
1053				    &udev->bus->bus_mtx, 0);
1054			} else {
1055				/*
1056				 * Setup a dummy xfer, hence we are
1057				 * writing to the "usb_xfer"
1058				 * structure pointed to by "xfer"
1059				 * before we have allocated any
1060				 * memory:
1061				 */
1062				xfer = &udev->scratch.xfer_setup[0].dummy;
1063				memset(xfer, 0, sizeof(*xfer));
1064				refcount++;
1065			}
1066
1067			/* set transfer endpoint pointer */
1068			xfer->endpoint = ep;
1069
1070			/* set transfer stream ID */
1071			xfer->stream_id = setup->stream_id;
1072
1073			parm->size[0] += sizeof(xfer[0]);
1074			parm->methods = xfer->endpoint->methods;
1075			parm->curr_xfer = xfer;
1076
1077			/*
1078			 * Call the Host or Device controller transfer
1079			 * setup routine:
1080			 */
1081			(udev->bus->methods->xfer_setup) (parm);
1082
1083			/* check for error */
1084			if (parm->err)
1085				goto done;
1086
1087			if (buf) {
1088				/*
1089				 * Increment the endpoint refcount. This
1090				 * basically prevents setting a new
1091				 * configuration and alternate setting
1092				 * when USB transfers are in use on
1093				 * the given interface. Search the USB
1094				 * code for "endpoint->refcount_alloc" if you
1095				 * want more information.
1096				 */
1097				USB_BUS_LOCK(info->bus);
1098				if (xfer->endpoint->refcount_alloc >= USB_EP_REF_MAX)
1099					parm->err = USB_ERR_INVAL;
1100
1101				xfer->endpoint->refcount_alloc++;
1102
1103				if (xfer->endpoint->refcount_alloc == 0)
1104					panic("usbd_transfer_setup(): Refcount wrapped to zero\n");
1105				USB_BUS_UNLOCK(info->bus);
1106
1107				/*
1108				 * Whenever we set ppxfer[] then we
1109				 * also need to increment the
1110				 * "setup_refcount":
1111				 */
1112				info->setup_refcount++;
1113
1114				/*
1115				 * Transfer is successfully setup and
1116				 * can be used:
1117				 */
1118				ppxfer[n] = xfer;
1119			}
1120
1121			/* check for error */
1122			if (parm->err)
1123				goto done;
1124		}
1125
1126		if (buf != NULL || parm->err != 0)
1127			goto done;
1128
1129		/* if no transfers, nothing to do */
1130		if (refcount == 0)
1131			goto done;
1132
1133		/* align data properly */
1134		parm->size[0] += ((-parm->size[0]) & (USB_HOST_ALIGN - 1));
1135
1136		/* store offset temporarily */
1137		parm->size[1] = parm->size[0];
1138
1139		/*
1140		 * The number of DMA tags required depends on
1141		 * the number of endpoints. The current estimate
1142		 * for maximum number of DMA tags per endpoint
1143		 * is three:
1144		 * 1) for loading memory
1145		 * 2) for allocating memory
1146		 * 3) for fixing memory [UHCI]
1147		 */
1148		parm->dma_tag_max += 3 * MIN(n_setup, USB_EP_MAX);
1149
1150		/*
1151		 * DMA tags for QH, TD, Data and more.
1152		 */
1153		parm->dma_tag_max += 8;
1154
1155		parm->dma_tag_p += parm->dma_tag_max;
1156
1157		parm->size[0] += ((uint8_t *)parm->dma_tag_p) -
1158		    ((uint8_t *)0);
1159
1160		/* align data properly */
1161		parm->size[0] += ((-parm->size[0]) & (USB_HOST_ALIGN - 1));
1162
1163		/* store offset temporarily */
1164		parm->size[3] = parm->size[0];
1165
1166		parm->size[0] += ((uint8_t *)parm->dma_page_ptr) -
1167		    ((uint8_t *)0);
1168
1169		/* align data properly */
1170		parm->size[0] += ((-parm->size[0]) & (USB_HOST_ALIGN - 1));
1171
1172		/* store offset temporarily */
1173		parm->size[4] = parm->size[0];
1174
1175		parm->size[0] += ((uint8_t *)parm->dma_page_cache_ptr) -
1176		    ((uint8_t *)0);
1177
1178		/* store end offset temporarily */
1179		parm->size[5] = parm->size[0];
1180
1181		parm->size[0] += ((uint8_t *)parm->xfer_page_cache_ptr) -
1182		    ((uint8_t *)0);
1183
1184		/* store end offset temporarily */
1185
1186		parm->size[2] = parm->size[0];
1187
1188		/* align data properly */
1189		parm->size[0] += ((-parm->size[0]) & (USB_HOST_ALIGN - 1));
1190
1191		parm->size[6] = parm->size[0];
1192
1193		parm->size[0] += ((uint8_t *)parm->xfer_length_ptr) -
1194		    ((uint8_t *)0);
1195
1196		/* align data properly */
1197		parm->size[0] += ((-parm->size[0]) & (USB_HOST_ALIGN - 1));
1198
1199		/* allocate zeroed memory */
1200		buf = malloc(parm->size[0], M_USB, M_WAITOK | M_ZERO);
1201
1202		if (buf == NULL) {
1203			parm->err = USB_ERR_NOMEM;
1204			DPRINTFN(0, "cannot allocate memory block for "
1205			    "configuration (%d bytes)\n",
1206			    parm->size[0]);
1207			goto done;
1208		}
1209		parm->dma_tag_p = USB_ADD_BYTES(buf, parm->size[1]);
1210		parm->dma_page_ptr = USB_ADD_BYTES(buf, parm->size[3]);
1211		parm->dma_page_cache_ptr = USB_ADD_BYTES(buf, parm->size[4]);
1212		parm->xfer_page_cache_ptr = USB_ADD_BYTES(buf, parm->size[5]);
1213		parm->xfer_length_ptr = USB_ADD_BYTES(buf, parm->size[6]);
1214	}
1215
1216done:
1217	if (buf) {
1218		if (info->setup_refcount == 0) {
1219			/*
1220			 * "usbd_transfer_unsetup_sub" will unlock
1221			 * the bus mutex before returning !
1222			 */
1223			USB_BUS_LOCK(info->bus);
1224
1225			/* something went wrong */
1226			usbd_transfer_unsetup_sub(info, 0);
1227		}
1228	}
1229
1230	/* check if any errors happened */
1231	if (parm->err)
1232		usbd_transfer_unsetup(ppxfer, n_setup);
1233
1234	error = parm->err;
1235
1236	if (do_unlock)
1237		usbd_enum_unlock(udev);
1238
1239	return (error);
1240}
1241
1242/*------------------------------------------------------------------------*
1243 *	usbd_transfer_unsetup_sub - factored out code
1244 *------------------------------------------------------------------------*/
1245static void
1246usbd_transfer_unsetup_sub(struct usb_xfer_root *info, uint8_t needs_delay)
1247{
1248#if USB_HAVE_BUSDMA
1249	struct usb_page_cache *pc;
1250#endif
1251
1252	USB_BUS_LOCK_ASSERT(info->bus, MA_OWNED);
1253
1254	/* wait for any outstanding DMA operations */
1255
1256	if (needs_delay) {
1257		usb_timeout_t temp;
1258		temp = usbd_get_dma_delay(info->udev);
1259		if (temp != 0) {
1260			usb_pause_mtx(&info->bus->bus_mtx,
1261			    USB_MS_TO_TICKS(temp));
1262		}
1263	}
1264
1265	/* make sure that our done messages are not queued anywhere */
1266	usb_proc_mwait(info->done_p, &info->done_m[0], &info->done_m[1]);
1267
1268	USB_BUS_UNLOCK(info->bus);
1269
1270#if USB_HAVE_BUSDMA
1271	/* free DMA'able memory, if any */
1272	pc = info->dma_page_cache_start;
1273	while (pc != info->dma_page_cache_end) {
1274		usb_pc_free_mem(pc);
1275		pc++;
1276	}
1277
1278	/* free DMA maps in all "xfer->frbuffers" */
1279	pc = info->xfer_page_cache_start;
1280	while (pc != info->xfer_page_cache_end) {
1281		usb_pc_dmamap_destroy(pc);
1282		pc++;
1283	}
1284
1285	/* free all DMA tags */
1286	usb_dma_tag_unsetup(&info->dma_parent_tag);
1287#endif
1288
1289	cv_destroy(&info->cv_drain);
1290
1291	/*
1292	 * free the "memory_base" last, hence the "info" structure is
1293	 * contained within the "memory_base"!
1294	 */
1295	free(info->memory_base, M_USB);
1296}
1297
1298/*------------------------------------------------------------------------*
1299 *	usbd_transfer_unsetup - unsetup/free an array of USB transfers
1300 *
1301 * NOTE: All USB transfers in progress will get called back passing
1302 * the error code "USB_ERR_CANCELLED" before this function
1303 * returns.
1304 *------------------------------------------------------------------------*/
1305void
1306usbd_transfer_unsetup(struct usb_xfer **pxfer, uint16_t n_setup)
1307{
1308	struct usb_xfer *xfer;
1309	struct usb_xfer_root *info;
1310	uint8_t needs_delay = 0;
1311
1312	WITNESS_WARN(WARN_GIANTOK | WARN_SLEEPOK, NULL,
1313	    "usbd_transfer_unsetup can sleep!");
1314
1315	while (n_setup--) {
1316		xfer = pxfer[n_setup];
1317
1318		if (xfer == NULL)
1319			continue;
1320
1321		info = xfer->xroot;
1322
1323		USB_XFER_LOCK(xfer);
1324		USB_BUS_LOCK(info->bus);
1325
1326		/*
1327		 * HINT: when you start/stop a transfer, it might be a
1328		 * good idea to directly use the "pxfer[]" structure:
1329		 *
1330		 * usbd_transfer_start(sc->pxfer[0]);
1331		 * usbd_transfer_stop(sc->pxfer[0]);
1332		 *
1333		 * That way, if your code has many parts that will not
1334		 * stop running under the same lock, in other words
1335		 * "xfer_mtx", the usbd_transfer_start and
1336		 * usbd_transfer_stop functions will simply return
1337		 * when they detect a NULL pointer argument.
1338		 *
1339		 * To avoid any races we clear the "pxfer[]" pointer
1340		 * while holding the private mutex of the driver:
1341		 */
1342		pxfer[n_setup] = NULL;
1343
1344		USB_BUS_UNLOCK(info->bus);
1345		USB_XFER_UNLOCK(xfer);
1346
1347		usbd_transfer_drain(xfer);
1348
1349#if USB_HAVE_BUSDMA
1350		if (xfer->flags_int.bdma_enable)
1351			needs_delay = 1;
1352#endif
1353		/*
1354		 * NOTE: default endpoint does not have an
1355		 * interface, even if endpoint->iface_index == 0
1356		 */
1357		USB_BUS_LOCK(info->bus);
1358		xfer->endpoint->refcount_alloc--;
1359		USB_BUS_UNLOCK(info->bus);
1360
1361		usb_callout_drain(&xfer->timeout_handle);
1362
1363		USB_BUS_LOCK(info->bus);
1364
1365		USB_ASSERT(info->setup_refcount != 0, ("Invalid setup "
1366		    "reference count\n"));
1367
1368		info->setup_refcount--;
1369
1370		if (info->setup_refcount == 0) {
1371			usbd_transfer_unsetup_sub(info,
1372			    needs_delay);
1373		} else {
1374			USB_BUS_UNLOCK(info->bus);
1375		}
1376	}
1377}
1378
1379/*------------------------------------------------------------------------*
1380 *	usbd_control_transfer_init - factored out code
1381 *
1382 * In USB Device Mode we have to wait for the SETUP packet which
1383 * containst the "struct usb_device_request" structure, before we can
1384 * transfer any data. In USB Host Mode we already have the SETUP
1385 * packet at the moment the USB transfer is started. This leads us to
1386 * having to setup the USB transfer at two different places in
1387 * time. This function just contains factored out control transfer
1388 * initialisation code, so that we don't duplicate the code.
1389 *------------------------------------------------------------------------*/
1390static void
1391usbd_control_transfer_init(struct usb_xfer *xfer)
1392{
1393	struct usb_device_request req;
1394
1395	/* copy out the USB request header */
1396
1397	usbd_copy_out(xfer->frbuffers, 0, &req, sizeof(req));
1398
1399	/* setup remainder */
1400
1401	xfer->flags_int.control_rem = UGETW(req.wLength);
1402
1403	/* copy direction to endpoint variable */
1404
1405	xfer->endpointno &= ~(UE_DIR_IN | UE_DIR_OUT);
1406	xfer->endpointno |=
1407	    (req.bmRequestType & UT_READ) ? UE_DIR_IN : UE_DIR_OUT;
1408}
1409
1410/*------------------------------------------------------------------------*
1411 *	usbd_setup_ctrl_transfer
1412 *
1413 * This function handles initialisation of control transfers. Control
1414 * transfers are special in that regard that they can both transmit
1415 * and receive data.
1416 *
1417 * Return values:
1418 *    0: Success
1419 * Else: Failure
1420 *------------------------------------------------------------------------*/
1421static int
1422usbd_setup_ctrl_transfer(struct usb_xfer *xfer)
1423{
1424	usb_frlength_t len;
1425
1426	/* Check for control endpoint stall */
1427	if (xfer->flags.stall_pipe && xfer->flags_int.control_act) {
1428		/* the control transfer is no longer active */
1429		xfer->flags_int.control_stall = 1;
1430		xfer->flags_int.control_act = 0;
1431	} else {
1432		/* don't stall control transfer by default */
1433		xfer->flags_int.control_stall = 0;
1434	}
1435
1436	/* Check for invalid number of frames */
1437	if (xfer->nframes > 2) {
1438		/*
1439		 * If you need to split a control transfer, you
1440		 * have to do one part at a time. Only with
1441		 * non-control transfers you can do multiple
1442		 * parts a time.
1443		 */
1444		DPRINTFN(0, "Too many frames: %u\n",
1445		    (unsigned int)xfer->nframes);
1446		goto error;
1447	}
1448
1449	/*
1450         * Check if there is a control
1451         * transfer in progress:
1452         */
1453	if (xfer->flags_int.control_act) {
1454
1455		if (xfer->flags_int.control_hdr) {
1456
1457			/* clear send header flag */
1458
1459			xfer->flags_int.control_hdr = 0;
1460
1461			/* setup control transfer */
1462			if (xfer->flags_int.usb_mode == USB_MODE_DEVICE) {
1463				usbd_control_transfer_init(xfer);
1464			}
1465		}
1466		/* get data length */
1467
1468		len = xfer->sumlen;
1469
1470	} else {
1471
1472		/* the size of the SETUP structure is hardcoded ! */
1473
1474		if (xfer->frlengths[0] != sizeof(struct usb_device_request)) {
1475			DPRINTFN(0, "Wrong framelength %u != %zu\n",
1476			    xfer->frlengths[0], sizeof(struct
1477			    usb_device_request));
1478			goto error;
1479		}
1480		/* check USB mode */
1481		if (xfer->flags_int.usb_mode == USB_MODE_DEVICE) {
1482
1483			/* check number of frames */
1484			if (xfer->nframes != 1) {
1485				/*
1486			         * We need to receive the setup
1487			         * message first so that we know the
1488			         * data direction!
1489			         */
1490				DPRINTF("Misconfigured transfer\n");
1491				goto error;
1492			}
1493			/*
1494			 * Set a dummy "control_rem" value.  This
1495			 * variable will be overwritten later by a
1496			 * call to "usbd_control_transfer_init()" !
1497			 */
1498			xfer->flags_int.control_rem = 0xFFFF;
1499		} else {
1500
1501			/* setup "endpoint" and "control_rem" */
1502
1503			usbd_control_transfer_init(xfer);
1504		}
1505
1506		/* set transfer-header flag */
1507
1508		xfer->flags_int.control_hdr = 1;
1509
1510		/* get data length */
1511
1512		len = (xfer->sumlen - sizeof(struct usb_device_request));
1513	}
1514
1515	/* check if there is a length mismatch */
1516
1517	if (len > xfer->flags_int.control_rem) {
1518		DPRINTFN(0, "Length (%d) greater than "
1519		    "remaining length (%d)\n", len,
1520		    xfer->flags_int.control_rem);
1521		goto error;
1522	}
1523	/* check if we are doing a short transfer */
1524
1525	if (xfer->flags.force_short_xfer) {
1526		xfer->flags_int.control_rem = 0;
1527	} else {
1528		if ((len != xfer->max_data_length) &&
1529		    (len != xfer->flags_int.control_rem) &&
1530		    (xfer->nframes != 1)) {
1531			DPRINTFN(0, "Short control transfer without "
1532			    "force_short_xfer set\n");
1533			goto error;
1534		}
1535		xfer->flags_int.control_rem -= len;
1536	}
1537
1538	/* the status part is executed when "control_act" is 0 */
1539
1540	if ((xfer->flags_int.control_rem > 0) ||
1541	    (xfer->flags.manual_status)) {
1542		/* don't execute the STATUS stage yet */
1543		xfer->flags_int.control_act = 1;
1544
1545		/* sanity check */
1546		if ((!xfer->flags_int.control_hdr) &&
1547		    (xfer->nframes == 1)) {
1548			/*
1549		         * This is not a valid operation!
1550		         */
1551			DPRINTFN(0, "Invalid parameter "
1552			    "combination\n");
1553			goto error;
1554		}
1555	} else {
1556		/* time to execute the STATUS stage */
1557		xfer->flags_int.control_act = 0;
1558	}
1559	return (0);			/* success */
1560
1561error:
1562	return (1);			/* failure */
1563}
1564
1565/*------------------------------------------------------------------------*
1566 *	usbd_transfer_submit - start USB hardware for the given transfer
1567 *
1568 * This function should only be called from the USB callback.
1569 *------------------------------------------------------------------------*/
1570void
1571usbd_transfer_submit(struct usb_xfer *xfer)
1572{
1573	struct usb_xfer_root *info;
1574	struct usb_bus *bus;
1575	usb_frcount_t x;
1576
1577	info = xfer->xroot;
1578	bus = info->bus;
1579
1580	DPRINTF("xfer=%p, endpoint=%p, nframes=%d, dir=%s\n",
1581	    xfer, xfer->endpoint, xfer->nframes, USB_GET_DATA_ISREAD(xfer) ?
1582	    "read" : "write");
1583
1584#ifdef USB_DEBUG
1585	if (USB_DEBUG_VAR > 0) {
1586		USB_BUS_LOCK(bus);
1587
1588		usb_dump_endpoint(xfer->endpoint);
1589
1590		USB_BUS_UNLOCK(bus);
1591	}
1592#endif
1593
1594	USB_XFER_LOCK_ASSERT(xfer, MA_OWNED);
1595	USB_BUS_LOCK_ASSERT(bus, MA_NOTOWNED);
1596
1597	/* Only open the USB transfer once! */
1598	if (!xfer->flags_int.open) {
1599		xfer->flags_int.open = 1;
1600
1601		DPRINTF("open\n");
1602
1603		USB_BUS_LOCK(bus);
1604		(xfer->endpoint->methods->open) (xfer);
1605		USB_BUS_UNLOCK(bus);
1606	}
1607	/* set "transferring" flag */
1608	xfer->flags_int.transferring = 1;
1609
1610#if USB_HAVE_POWERD
1611	/* increment power reference */
1612	usbd_transfer_power_ref(xfer, 1);
1613#endif
1614	/*
1615	 * Check if the transfer is waiting on a queue, most
1616	 * frequently the "done_q":
1617	 */
1618	if (xfer->wait_queue) {
1619		USB_BUS_LOCK(bus);
1620		usbd_transfer_dequeue(xfer);
1621		USB_BUS_UNLOCK(bus);
1622	}
1623	/* clear "did_dma_delay" flag */
1624	xfer->flags_int.did_dma_delay = 0;
1625
1626	/* clear "did_close" flag */
1627	xfer->flags_int.did_close = 0;
1628
1629#if USB_HAVE_BUSDMA
1630	/* clear "bdma_setup" flag */
1631	xfer->flags_int.bdma_setup = 0;
1632#endif
1633	/* by default we cannot cancel any USB transfer immediately */
1634	xfer->flags_int.can_cancel_immed = 0;
1635
1636	/* clear lengths and frame counts by default */
1637	xfer->sumlen = 0;
1638	xfer->actlen = 0;
1639	xfer->aframes = 0;
1640
1641	/* clear any previous errors */
1642	xfer->error = 0;
1643
1644	/* Check if the device is still alive */
1645	if (info->udev->state < USB_STATE_POWERED) {
1646		USB_BUS_LOCK(bus);
1647		/*
1648		 * Must return cancelled error code else
1649		 * device drivers can hang.
1650		 */
1651		usbd_transfer_done(xfer, USB_ERR_CANCELLED);
1652		USB_BUS_UNLOCK(bus);
1653		return;
1654	}
1655
1656	/* sanity check */
1657	if (xfer->nframes == 0) {
1658		if (xfer->flags.stall_pipe) {
1659			/*
1660			 * Special case - want to stall without transferring
1661			 * any data:
1662			 */
1663			DPRINTF("xfer=%p nframes=0: stall "
1664			    "or clear stall!\n", xfer);
1665			USB_BUS_LOCK(bus);
1666			xfer->flags_int.can_cancel_immed = 1;
1667			/* start the transfer */
1668			usb_command_wrapper(&xfer->endpoint->
1669			    endpoint_q[xfer->stream_id], xfer);
1670			USB_BUS_UNLOCK(bus);
1671			return;
1672		}
1673		USB_BUS_LOCK(bus);
1674		usbd_transfer_done(xfer, USB_ERR_INVAL);
1675		USB_BUS_UNLOCK(bus);
1676		return;
1677	}
1678	/* compute some variables */
1679
1680	for (x = 0; x != xfer->nframes; x++) {
1681		/* make a copy of the frlenghts[] */
1682		xfer->frlengths[x + xfer->max_frame_count] = xfer->frlengths[x];
1683		/* compute total transfer length */
1684		xfer->sumlen += xfer->frlengths[x];
1685		if (xfer->sumlen < xfer->frlengths[x]) {
1686			/* length wrapped around */
1687			USB_BUS_LOCK(bus);
1688			usbd_transfer_done(xfer, USB_ERR_INVAL);
1689			USB_BUS_UNLOCK(bus);
1690			return;
1691		}
1692	}
1693
1694	/* clear some internal flags */
1695
1696	xfer->flags_int.short_xfer_ok = 0;
1697	xfer->flags_int.short_frames_ok = 0;
1698
1699	/* check if this is a control transfer */
1700
1701	if (xfer->flags_int.control_xfr) {
1702
1703		if (usbd_setup_ctrl_transfer(xfer)) {
1704			USB_BUS_LOCK(bus);
1705			usbd_transfer_done(xfer, USB_ERR_STALLED);
1706			USB_BUS_UNLOCK(bus);
1707			return;
1708		}
1709	}
1710	/*
1711	 * Setup filtered version of some transfer flags,
1712	 * in case of data read direction
1713	 */
1714	if (USB_GET_DATA_ISREAD(xfer)) {
1715
1716		if (xfer->flags.short_frames_ok) {
1717			xfer->flags_int.short_xfer_ok = 1;
1718			xfer->flags_int.short_frames_ok = 1;
1719		} else if (xfer->flags.short_xfer_ok) {
1720			xfer->flags_int.short_xfer_ok = 1;
1721
1722			/* check for control transfer */
1723			if (xfer->flags_int.control_xfr) {
1724				/*
1725				 * 1) Control transfers do not support
1726				 * reception of multiple short USB
1727				 * frames in host mode and device side
1728				 * mode, with exception of:
1729				 *
1730				 * 2) Due to sometimes buggy device
1731				 * side firmware we need to do a
1732				 * STATUS stage in case of short
1733				 * control transfers in USB host mode.
1734				 * The STATUS stage then becomes the
1735				 * "alt_next" to the DATA stage.
1736				 */
1737				xfer->flags_int.short_frames_ok = 1;
1738			}
1739		}
1740	}
1741	/*
1742	 * Check if BUS-DMA support is enabled and try to load virtual
1743	 * buffers into DMA, if any:
1744	 */
1745#if USB_HAVE_BUSDMA
1746	if (xfer->flags_int.bdma_enable) {
1747		/* insert the USB transfer last in the BUS-DMA queue */
1748		usb_command_wrapper(&xfer->xroot->dma_q, xfer);
1749		return;
1750	}
1751#endif
1752	/*
1753	 * Enter the USB transfer into the Host Controller or
1754	 * Device Controller schedule:
1755	 */
1756	usbd_pipe_enter(xfer);
1757}
1758
1759/*------------------------------------------------------------------------*
1760 *	usbd_pipe_enter - factored out code
1761 *------------------------------------------------------------------------*/
1762void
1763usbd_pipe_enter(struct usb_xfer *xfer)
1764{
1765	struct usb_endpoint *ep;
1766
1767	USB_XFER_LOCK_ASSERT(xfer, MA_OWNED);
1768
1769	USB_BUS_LOCK(xfer->xroot->bus);
1770
1771	ep = xfer->endpoint;
1772
1773	DPRINTF("enter\n");
1774
1775	/* the transfer can now be cancelled */
1776	xfer->flags_int.can_cancel_immed = 1;
1777
1778	/* enter the transfer */
1779	(ep->methods->enter) (xfer);
1780
1781	/* check for transfer error */
1782	if (xfer->error) {
1783		/* some error has happened */
1784		usbd_transfer_done(xfer, 0);
1785		USB_BUS_UNLOCK(xfer->xroot->bus);
1786		return;
1787	}
1788
1789	/* start the transfer */
1790	usb_command_wrapper(&ep->endpoint_q[xfer->stream_id], xfer);
1791	USB_BUS_UNLOCK(xfer->xroot->bus);
1792}
1793
1794/*------------------------------------------------------------------------*
1795 *	usbd_transfer_start - start an USB transfer
1796 *
1797 * NOTE: Calling this function more than one time will only
1798 *       result in a single transfer start, until the USB transfer
1799 *       completes.
1800 *------------------------------------------------------------------------*/
1801void
1802usbd_transfer_start(struct usb_xfer *xfer)
1803{
1804	if (xfer == NULL) {
1805		/* transfer is gone */
1806		return;
1807	}
1808	USB_XFER_LOCK_ASSERT(xfer, MA_OWNED);
1809
1810	/* mark the USB transfer started */
1811
1812	if (!xfer->flags_int.started) {
1813		/* lock the BUS lock to avoid races updating flags_int */
1814		USB_BUS_LOCK(xfer->xroot->bus);
1815		xfer->flags_int.started = 1;
1816		USB_BUS_UNLOCK(xfer->xroot->bus);
1817	}
1818	/* check if the USB transfer callback is already transferring */
1819
1820	if (xfer->flags_int.transferring) {
1821		return;
1822	}
1823	USB_BUS_LOCK(xfer->xroot->bus);
1824	/* call the USB transfer callback */
1825	usbd_callback_ss_done_defer(xfer);
1826	USB_BUS_UNLOCK(xfer->xroot->bus);
1827}
1828
1829/*------------------------------------------------------------------------*
1830 *	usbd_transfer_stop - stop an USB transfer
1831 *
1832 * NOTE: Calling this function more than one time will only
1833 *       result in a single transfer stop.
1834 * NOTE: When this function returns it is not safe to free nor
1835 *       reuse any DMA buffers. See "usbd_transfer_drain()".
1836 *------------------------------------------------------------------------*/
1837void
1838usbd_transfer_stop(struct usb_xfer *xfer)
1839{
1840	struct usb_endpoint *ep;
1841
1842	if (xfer == NULL) {
1843		/* transfer is gone */
1844		return;
1845	}
1846	USB_XFER_LOCK_ASSERT(xfer, MA_OWNED);
1847
1848	/* check if the USB transfer was ever opened */
1849
1850	if (!xfer->flags_int.open) {
1851		if (xfer->flags_int.started) {
1852			/* nothing to do except clearing the "started" flag */
1853			/* lock the BUS lock to avoid races updating flags_int */
1854			USB_BUS_LOCK(xfer->xroot->bus);
1855			xfer->flags_int.started = 0;
1856			USB_BUS_UNLOCK(xfer->xroot->bus);
1857		}
1858		return;
1859	}
1860	/* try to stop the current USB transfer */
1861
1862	USB_BUS_LOCK(xfer->xroot->bus);
1863	/* override any previous error */
1864	xfer->error = USB_ERR_CANCELLED;
1865
1866	/*
1867	 * Clear "open" and "started" when both private and USB lock
1868	 * is locked so that we don't get a race updating "flags_int"
1869	 */
1870	xfer->flags_int.open = 0;
1871	xfer->flags_int.started = 0;
1872
1873	/*
1874	 * Check if we can cancel the USB transfer immediately.
1875	 */
1876	if (xfer->flags_int.transferring) {
1877		if (xfer->flags_int.can_cancel_immed &&
1878		    (!xfer->flags_int.did_close)) {
1879			DPRINTF("close\n");
1880			/*
1881			 * The following will lead to an USB_ERR_CANCELLED
1882			 * error code being passed to the USB callback.
1883			 */
1884			(xfer->endpoint->methods->close) (xfer);
1885			/* only close once */
1886			xfer->flags_int.did_close = 1;
1887		} else {
1888			/* need to wait for the next done callback */
1889		}
1890	} else {
1891		DPRINTF("close\n");
1892
1893		/* close here and now */
1894		(xfer->endpoint->methods->close) (xfer);
1895
1896		/*
1897		 * Any additional DMA delay is done by
1898		 * "usbd_transfer_unsetup()".
1899		 */
1900
1901		/*
1902		 * Special case. Check if we need to restart a blocked
1903		 * endpoint.
1904		 */
1905		ep = xfer->endpoint;
1906
1907		/*
1908		 * If the current USB transfer is completing we need
1909		 * to start the next one:
1910		 */
1911		if (ep->endpoint_q[xfer->stream_id].curr == xfer) {
1912			usb_command_wrapper(
1913			    &ep->endpoint_q[xfer->stream_id], NULL);
1914		}
1915	}
1916
1917	USB_BUS_UNLOCK(xfer->xroot->bus);
1918}
1919
1920/*------------------------------------------------------------------------*
1921 *	usbd_transfer_pending
1922 *
1923 * This function will check if an USB transfer is pending which is a
1924 * little bit complicated!
1925 * Return values:
1926 * 0: Not pending
1927 * 1: Pending: The USB transfer will receive a callback in the future.
1928 *------------------------------------------------------------------------*/
1929uint8_t
1930usbd_transfer_pending(struct usb_xfer *xfer)
1931{
1932	struct usb_xfer_root *info;
1933	struct usb_xfer_queue *pq;
1934
1935	if (xfer == NULL) {
1936		/* transfer is gone */
1937		return (0);
1938	}
1939	USB_XFER_LOCK_ASSERT(xfer, MA_OWNED);
1940
1941	if (xfer->flags_int.transferring) {
1942		/* trivial case */
1943		return (1);
1944	}
1945	USB_BUS_LOCK(xfer->xroot->bus);
1946	if (xfer->wait_queue) {
1947		/* we are waiting on a queue somewhere */
1948		USB_BUS_UNLOCK(xfer->xroot->bus);
1949		return (1);
1950	}
1951	info = xfer->xroot;
1952	pq = &info->done_q;
1953
1954	if (pq->curr == xfer) {
1955		/* we are currently scheduled for callback */
1956		USB_BUS_UNLOCK(xfer->xroot->bus);
1957		return (1);
1958	}
1959	/* we are not pending */
1960	USB_BUS_UNLOCK(xfer->xroot->bus);
1961	return (0);
1962}
1963
1964/*------------------------------------------------------------------------*
1965 *	usbd_transfer_drain
1966 *
1967 * This function will stop the USB transfer and wait for any
1968 * additional BUS-DMA and HW-DMA operations to complete. Buffers that
1969 * are loaded into DMA can safely be freed or reused after that this
1970 * function has returned.
1971 *------------------------------------------------------------------------*/
1972void
1973usbd_transfer_drain(struct usb_xfer *xfer)
1974{
1975	WITNESS_WARN(WARN_GIANTOK | WARN_SLEEPOK, NULL,
1976	    "usbd_transfer_drain can sleep!");
1977
1978	if (xfer == NULL) {
1979		/* transfer is gone */
1980		return;
1981	}
1982	if (xfer->xroot->xfer_mtx != &Giant) {
1983		USB_XFER_LOCK_ASSERT(xfer, MA_NOTOWNED);
1984	}
1985	USB_XFER_LOCK(xfer);
1986
1987	usbd_transfer_stop(xfer);
1988
1989	while (usbd_transfer_pending(xfer) ||
1990	    xfer->flags_int.doing_callback) {
1991
1992		/*
1993		 * It is allowed that the callback can drop its
1994		 * transfer mutex. In that case checking only
1995		 * "usbd_transfer_pending()" is not enough to tell if
1996		 * the USB transfer is fully drained. We also need to
1997		 * check the internal "doing_callback" flag.
1998		 */
1999		xfer->flags_int.draining = 1;
2000
2001		/*
2002		 * Wait until the current outstanding USB
2003		 * transfer is complete !
2004		 */
2005		cv_wait(&xfer->xroot->cv_drain, xfer->xroot->xfer_mtx);
2006	}
2007	USB_XFER_UNLOCK(xfer);
2008}
2009
2010struct usb_page_cache *
2011usbd_xfer_get_frame(struct usb_xfer *xfer, usb_frcount_t frindex)
2012{
2013	KASSERT(frindex < xfer->max_frame_count, ("frame index overflow"));
2014
2015	return (&xfer->frbuffers[frindex]);
2016}
2017
2018void *
2019usbd_xfer_get_frame_buffer(struct usb_xfer *xfer, usb_frcount_t frindex)
2020{
2021	struct usb_page_search page_info;
2022
2023	KASSERT(frindex < xfer->max_frame_count, ("frame index overflow"));
2024
2025	usbd_get_page(&xfer->frbuffers[frindex], 0, &page_info);
2026	return (page_info.buffer);
2027}
2028
2029/*------------------------------------------------------------------------*
2030 *	usbd_xfer_get_fps_shift
2031 *
2032 * The following function is only useful for isochronous transfers. It
2033 * returns how many times the frame execution rate has been shifted
2034 * down.
2035 *
2036 * Return value:
2037 * Success: 0..3
2038 * Failure: 0
2039 *------------------------------------------------------------------------*/
2040uint8_t
2041usbd_xfer_get_fps_shift(struct usb_xfer *xfer)
2042{
2043	return (xfer->fps_shift);
2044}
2045
2046usb_frlength_t
2047usbd_xfer_frame_len(struct usb_xfer *xfer, usb_frcount_t frindex)
2048{
2049	KASSERT(frindex < xfer->max_frame_count, ("frame index overflow"));
2050
2051	return (xfer->frlengths[frindex]);
2052}
2053
2054/*------------------------------------------------------------------------*
2055 *	usbd_xfer_set_frame_data
2056 *
2057 * This function sets the pointer of the buffer that should
2058 * loaded directly into DMA for the given USB frame. Passing "ptr"
2059 * equal to NULL while the corresponding "frlength" is greater
2060 * than zero gives undefined results!
2061 *------------------------------------------------------------------------*/
2062void
2063usbd_xfer_set_frame_data(struct usb_xfer *xfer, usb_frcount_t frindex,
2064    void *ptr, usb_frlength_t len)
2065{
2066	KASSERT(frindex < xfer->max_frame_count, ("frame index overflow"));
2067
2068	/* set virtual address to load and length */
2069	xfer->frbuffers[frindex].buffer = ptr;
2070	usbd_xfer_set_frame_len(xfer, frindex, len);
2071}
2072
2073void
2074usbd_xfer_frame_data(struct usb_xfer *xfer, usb_frcount_t frindex,
2075    void **ptr, int *len)
2076{
2077	KASSERT(frindex < xfer->max_frame_count, ("frame index overflow"));
2078
2079	if (ptr != NULL)
2080		*ptr = xfer->frbuffers[frindex].buffer;
2081	if (len != NULL)
2082		*len = xfer->frlengths[frindex];
2083}
2084
2085/*------------------------------------------------------------------------*
2086 *	usbd_xfer_old_frame_length
2087 *
2088 * This function returns the framelength of the given frame at the
2089 * time the transfer was submitted. This function can be used to
2090 * compute the starting data pointer of the next isochronous frame
2091 * when an isochronous transfer has completed.
2092 *------------------------------------------------------------------------*/
2093usb_frlength_t
2094usbd_xfer_old_frame_length(struct usb_xfer *xfer, usb_frcount_t frindex)
2095{
2096	KASSERT(frindex < xfer->max_frame_count, ("frame index overflow"));
2097
2098	return (xfer->frlengths[frindex + xfer->max_frame_count]);
2099}
2100
2101void
2102usbd_xfer_status(struct usb_xfer *xfer, int *actlen, int *sumlen, int *aframes,
2103    int *nframes)
2104{
2105	if (actlen != NULL)
2106		*actlen = xfer->actlen;
2107	if (sumlen != NULL)
2108		*sumlen = xfer->sumlen;
2109	if (aframes != NULL)
2110		*aframes = xfer->aframes;
2111	if (nframes != NULL)
2112		*nframes = xfer->nframes;
2113}
2114
2115/*------------------------------------------------------------------------*
2116 *	usbd_xfer_set_frame_offset
2117 *
2118 * This function sets the frame data buffer offset relative to the beginning
2119 * of the USB DMA buffer allocated for this USB transfer.
2120 *------------------------------------------------------------------------*/
2121void
2122usbd_xfer_set_frame_offset(struct usb_xfer *xfer, usb_frlength_t offset,
2123    usb_frcount_t frindex)
2124{
2125	KASSERT(!xfer->flags.ext_buffer, ("Cannot offset data frame "
2126	    "when the USB buffer is external\n"));
2127	KASSERT(frindex < xfer->max_frame_count, ("frame index overflow"));
2128
2129	/* set virtual address to load */
2130	xfer->frbuffers[frindex].buffer =
2131	    USB_ADD_BYTES(xfer->local_buffer, offset);
2132}
2133
2134void
2135usbd_xfer_set_interval(struct usb_xfer *xfer, int i)
2136{
2137	xfer->interval = i;
2138}
2139
2140void
2141usbd_xfer_set_timeout(struct usb_xfer *xfer, int t)
2142{
2143	xfer->timeout = t;
2144}
2145
2146void
2147usbd_xfer_set_frames(struct usb_xfer *xfer, usb_frcount_t n)
2148{
2149	xfer->nframes = n;
2150}
2151
2152usb_frcount_t
2153usbd_xfer_max_frames(struct usb_xfer *xfer)
2154{
2155	return (xfer->max_frame_count);
2156}
2157
2158usb_frlength_t
2159usbd_xfer_max_len(struct usb_xfer *xfer)
2160{
2161	return (xfer->max_data_length);
2162}
2163
2164usb_frlength_t
2165usbd_xfer_max_framelen(struct usb_xfer *xfer)
2166{
2167	return (xfer->max_frame_size);
2168}
2169
2170void
2171usbd_xfer_set_frame_len(struct usb_xfer *xfer, usb_frcount_t frindex,
2172    usb_frlength_t len)
2173{
2174	KASSERT(frindex < xfer->max_frame_count, ("frame index overflow"));
2175
2176	xfer->frlengths[frindex] = len;
2177}
2178
2179/*------------------------------------------------------------------------*
2180 *	usb_callback_proc - factored out code
2181 *
2182 * This function performs USB callbacks.
2183 *------------------------------------------------------------------------*/
2184static void
2185usb_callback_proc(struct usb_proc_msg *_pm)
2186{
2187	struct usb_done_msg *pm = (void *)_pm;
2188	struct usb_xfer_root *info = pm->xroot;
2189
2190	/* Change locking order */
2191	USB_BUS_UNLOCK(info->bus);
2192
2193	/*
2194	 * We exploit the fact that the mutex is the same for all
2195	 * callbacks that will be called from this thread:
2196	 */
2197	mtx_lock(info->xfer_mtx);
2198	USB_BUS_LOCK(info->bus);
2199
2200	/* Continue where we lost track */
2201	usb_command_wrapper(&info->done_q,
2202	    info->done_q.curr);
2203
2204	mtx_unlock(info->xfer_mtx);
2205}
2206
2207/*------------------------------------------------------------------------*
2208 *	usbd_callback_ss_done_defer
2209 *
2210 * This function will defer the start, stop and done callback to the
2211 * correct thread.
2212 *------------------------------------------------------------------------*/
2213static void
2214usbd_callback_ss_done_defer(struct usb_xfer *xfer)
2215{
2216	struct usb_xfer_root *info = xfer->xroot;
2217	struct usb_xfer_queue *pq = &info->done_q;
2218
2219	USB_BUS_LOCK_ASSERT(xfer->xroot->bus, MA_OWNED);
2220
2221	if (pq->curr != xfer) {
2222		usbd_transfer_enqueue(pq, xfer);
2223	}
2224	if (!pq->recurse_1) {
2225
2226		/*
2227	         * We have to postpone the callback due to the fact we
2228	         * will have a Lock Order Reversal, LOR, if we try to
2229	         * proceed !
2230	         */
2231		if (usb_proc_msignal(info->done_p,
2232		    &info->done_m[0], &info->done_m[1])) {
2233			/* ignore */
2234		}
2235	} else {
2236		/* clear second recurse flag */
2237		pq->recurse_2 = 0;
2238	}
2239	return;
2240
2241}
2242
2243/*------------------------------------------------------------------------*
2244 *	usbd_callback_wrapper
2245 *
2246 * This is a wrapper for USB callbacks. This wrapper does some
2247 * auto-magic things like figuring out if we can call the callback
2248 * directly from the current context or if we need to wakeup the
2249 * interrupt process.
2250 *------------------------------------------------------------------------*/
2251static void
2252usbd_callback_wrapper(struct usb_xfer_queue *pq)
2253{
2254	struct usb_xfer *xfer = pq->curr;
2255	struct usb_xfer_root *info = xfer->xroot;
2256
2257	USB_BUS_LOCK_ASSERT(info->bus, MA_OWNED);
2258	if (!mtx_owned(info->xfer_mtx) && !SCHEDULER_STOPPED()) {
2259		/*
2260	       	 * Cases that end up here:
2261		 *
2262		 * 5) HW interrupt done callback or other source.
2263		 */
2264		DPRINTFN(3, "case 5\n");
2265
2266		/*
2267	         * We have to postpone the callback due to the fact we
2268	         * will have a Lock Order Reversal, LOR, if we try to
2269	         * proceed !
2270	         */
2271		if (usb_proc_msignal(info->done_p,
2272		    &info->done_m[0], &info->done_m[1])) {
2273			/* ignore */
2274		}
2275		return;
2276	}
2277	/*
2278	 * Cases that end up here:
2279	 *
2280	 * 1) We are starting a transfer
2281	 * 2) We are prematurely calling back a transfer
2282	 * 3) We are stopping a transfer
2283	 * 4) We are doing an ordinary callback
2284	 */
2285	DPRINTFN(3, "case 1-4\n");
2286	/* get next USB transfer in the queue */
2287	info->done_q.curr = NULL;
2288
2289	/* set flag in case of drain */
2290	xfer->flags_int.doing_callback = 1;
2291
2292	USB_BUS_UNLOCK(info->bus);
2293	USB_BUS_LOCK_ASSERT(info->bus, MA_NOTOWNED);
2294
2295	/* set correct USB state for callback */
2296	if (!xfer->flags_int.transferring) {
2297		xfer->usb_state = USB_ST_SETUP;
2298		if (!xfer->flags_int.started) {
2299			/* we got stopped before we even got started */
2300			USB_BUS_LOCK(info->bus);
2301			goto done;
2302		}
2303	} else {
2304
2305		if (usbd_callback_wrapper_sub(xfer)) {
2306			/* the callback has been deferred */
2307			USB_BUS_LOCK(info->bus);
2308			goto done;
2309		}
2310#if USB_HAVE_POWERD
2311		/* decrement power reference */
2312		usbd_transfer_power_ref(xfer, -1);
2313#endif
2314		xfer->flags_int.transferring = 0;
2315
2316		if (xfer->error) {
2317			xfer->usb_state = USB_ST_ERROR;
2318		} else {
2319			/* set transferred state */
2320			xfer->usb_state = USB_ST_TRANSFERRED;
2321#if USB_HAVE_BUSDMA
2322			/* sync DMA memory, if any */
2323			if (xfer->flags_int.bdma_enable &&
2324			    (!xfer->flags_int.bdma_no_post_sync)) {
2325				usb_bdma_post_sync(xfer);
2326			}
2327#endif
2328		}
2329	}
2330
2331#if USB_HAVE_PF
2332	if (xfer->usb_state != USB_ST_SETUP)
2333		usbpf_xfertap(xfer, USBPF_XFERTAP_DONE);
2334#endif
2335	/* call processing routine */
2336	(xfer->callback) (xfer, xfer->error);
2337
2338	/* pickup the USB mutex again */
2339	USB_BUS_LOCK(info->bus);
2340
2341	/*
2342	 * Check if we got started after that we got cancelled, but
2343	 * before we managed to do the callback.
2344	 */
2345	if ((!xfer->flags_int.open) &&
2346	    (xfer->flags_int.started) &&
2347	    (xfer->usb_state == USB_ST_ERROR)) {
2348		/* clear flag in case of drain */
2349		xfer->flags_int.doing_callback = 0;
2350		/* try to loop, but not recursivly */
2351		usb_command_wrapper(&info->done_q, xfer);
2352		return;
2353	}
2354
2355done:
2356	/* clear flag in case of drain */
2357	xfer->flags_int.doing_callback = 0;
2358
2359	/*
2360	 * Check if we are draining.
2361	 */
2362	if (xfer->flags_int.draining &&
2363	    (!xfer->flags_int.transferring)) {
2364		/* "usbd_transfer_drain()" is waiting for end of transfer */
2365		xfer->flags_int.draining = 0;
2366		cv_broadcast(&info->cv_drain);
2367	}
2368
2369	/* do the next callback, if any */
2370	usb_command_wrapper(&info->done_q,
2371	    info->done_q.curr);
2372}
2373
2374/*------------------------------------------------------------------------*
2375 *	usb_dma_delay_done_cb
2376 *
2377 * This function is called when the DMA delay has been exectuded, and
2378 * will make sure that the callback is called to complete the USB
2379 * transfer. This code path is ususally only used when there is an USB
2380 * error like USB_ERR_CANCELLED.
2381 *------------------------------------------------------------------------*/
2382void
2383usb_dma_delay_done_cb(struct usb_xfer *xfer)
2384{
2385	USB_BUS_LOCK_ASSERT(xfer->xroot->bus, MA_OWNED);
2386
2387	DPRINTFN(3, "Completed %p\n", xfer);
2388
2389	/* queue callback for execution, again */
2390	usbd_transfer_done(xfer, 0);
2391}
2392
2393/*------------------------------------------------------------------------*
2394 *	usbd_transfer_dequeue
2395 *
2396 *  - This function is used to remove an USB transfer from a USB
2397 *  transfer queue.
2398 *
2399 *  - This function can be called multiple times in a row.
2400 *------------------------------------------------------------------------*/
2401void
2402usbd_transfer_dequeue(struct usb_xfer *xfer)
2403{
2404	struct usb_xfer_queue *pq;
2405
2406	pq = xfer->wait_queue;
2407	if (pq) {
2408		TAILQ_REMOVE(&pq->head, xfer, wait_entry);
2409		xfer->wait_queue = NULL;
2410	}
2411}
2412
2413/*------------------------------------------------------------------------*
2414 *	usbd_transfer_enqueue
2415 *
2416 *  - This function is used to insert an USB transfer into a USB *
2417 *  transfer queue.
2418 *
2419 *  - This function can be called multiple times in a row.
2420 *------------------------------------------------------------------------*/
2421void
2422usbd_transfer_enqueue(struct usb_xfer_queue *pq, struct usb_xfer *xfer)
2423{
2424	/*
2425	 * Insert the USB transfer into the queue, if it is not
2426	 * already on a USB transfer queue:
2427	 */
2428	if (xfer->wait_queue == NULL) {
2429		xfer->wait_queue = pq;
2430		TAILQ_INSERT_TAIL(&pq->head, xfer, wait_entry);
2431	}
2432}
2433
2434/*------------------------------------------------------------------------*
2435 *	usbd_transfer_done
2436 *
2437 *  - This function is used to remove an USB transfer from the busdma,
2438 *  pipe or interrupt queue.
2439 *
2440 *  - This function is used to queue the USB transfer on the done
2441 *  queue.
2442 *
2443 *  - This function is used to stop any USB transfer timeouts.
2444 *------------------------------------------------------------------------*/
2445void
2446usbd_transfer_done(struct usb_xfer *xfer, usb_error_t error)
2447{
2448	struct usb_xfer_root *info = xfer->xroot;
2449
2450	USB_BUS_LOCK_ASSERT(info->bus, MA_OWNED);
2451
2452	DPRINTF("err=%s\n", usbd_errstr(error));
2453
2454	/*
2455	 * If we are not transferring then just return.
2456	 * This can happen during transfer cancel.
2457	 */
2458	if (!xfer->flags_int.transferring) {
2459		DPRINTF("not transferring\n");
2460		/* end of control transfer, if any */
2461		xfer->flags_int.control_act = 0;
2462		return;
2463	}
2464	/* only set transfer error, if not already set */
2465	if (xfer->error == USB_ERR_NORMAL_COMPLETION)
2466		xfer->error = error;
2467
2468	/* stop any callouts */
2469	usb_callout_stop(&xfer->timeout_handle);
2470
2471	/*
2472	 * If we are waiting on a queue, just remove the USB transfer
2473	 * from the queue, if any. We should have the required locks
2474	 * locked to do the remove when this function is called.
2475	 */
2476	usbd_transfer_dequeue(xfer);
2477
2478#if USB_HAVE_BUSDMA
2479	if (mtx_owned(info->xfer_mtx)) {
2480		struct usb_xfer_queue *pq;
2481
2482		/*
2483		 * If the private USB lock is not locked, then we assume
2484		 * that the BUS-DMA load stage has been passed:
2485		 */
2486		pq = &info->dma_q;
2487
2488		if (pq->curr == xfer) {
2489			/* start the next BUS-DMA load, if any */
2490			usb_command_wrapper(pq, NULL);
2491		}
2492	}
2493#endif
2494	/* keep some statistics */
2495	if (xfer->error) {
2496		info->bus->stats_err.uds_requests
2497		    [xfer->endpoint->edesc->bmAttributes & UE_XFERTYPE]++;
2498	} else {
2499		info->bus->stats_ok.uds_requests
2500		    [xfer->endpoint->edesc->bmAttributes & UE_XFERTYPE]++;
2501	}
2502
2503	/* call the USB transfer callback */
2504	usbd_callback_ss_done_defer(xfer);
2505}
2506
2507/*------------------------------------------------------------------------*
2508 *	usbd_transfer_start_cb
2509 *
2510 * This function is called to start the USB transfer when
2511 * "xfer->interval" is greater than zero, and and the endpoint type is
2512 * BULK or CONTROL.
2513 *------------------------------------------------------------------------*/
2514static void
2515usbd_transfer_start_cb(void *arg)
2516{
2517	struct usb_xfer *xfer = arg;
2518	struct usb_endpoint *ep = xfer->endpoint;
2519
2520	USB_BUS_LOCK_ASSERT(xfer->xroot->bus, MA_OWNED);
2521
2522	DPRINTF("start\n");
2523
2524#if USB_HAVE_PF
2525	usbpf_xfertap(xfer, USBPF_XFERTAP_SUBMIT);
2526#endif
2527
2528	/* the transfer can now be cancelled */
2529	xfer->flags_int.can_cancel_immed = 1;
2530
2531	/* start USB transfer, if no error */
2532	if (xfer->error == 0)
2533		(ep->methods->start) (xfer);
2534
2535	/* check for transfer error */
2536	if (xfer->error) {
2537		/* some error has happened */
2538		usbd_transfer_done(xfer, 0);
2539	}
2540}
2541
2542/*------------------------------------------------------------------------*
2543 *	usbd_xfer_set_stall
2544 *
2545 * This function is used to set the stall flag outside the
2546 * callback. This function is NULL safe.
2547 *------------------------------------------------------------------------*/
2548void
2549usbd_xfer_set_stall(struct usb_xfer *xfer)
2550{
2551	if (xfer == NULL) {
2552		/* tearing down */
2553		return;
2554	}
2555	USB_XFER_LOCK_ASSERT(xfer, MA_OWNED);
2556
2557	/* avoid any races by locking the USB mutex */
2558	USB_BUS_LOCK(xfer->xroot->bus);
2559	xfer->flags.stall_pipe = 1;
2560	USB_BUS_UNLOCK(xfer->xroot->bus);
2561}
2562
2563int
2564usbd_xfer_is_stalled(struct usb_xfer *xfer)
2565{
2566	return (xfer->endpoint->is_stalled);
2567}
2568
2569/*------------------------------------------------------------------------*
2570 *	usbd_transfer_clear_stall
2571 *
2572 * This function is used to clear the stall flag outside the
2573 * callback. This function is NULL safe.
2574 *------------------------------------------------------------------------*/
2575void
2576usbd_transfer_clear_stall(struct usb_xfer *xfer)
2577{
2578	if (xfer == NULL) {
2579		/* tearing down */
2580		return;
2581	}
2582	USB_XFER_LOCK_ASSERT(xfer, MA_OWNED);
2583
2584	/* avoid any races by locking the USB mutex */
2585	USB_BUS_LOCK(xfer->xroot->bus);
2586
2587	xfer->flags.stall_pipe = 0;
2588
2589	USB_BUS_UNLOCK(xfer->xroot->bus);
2590}
2591
2592/*------------------------------------------------------------------------*
2593 *	usbd_pipe_start
2594 *
2595 * This function is used to add an USB transfer to the pipe transfer list.
2596 *------------------------------------------------------------------------*/
2597void
2598usbd_pipe_start(struct usb_xfer_queue *pq)
2599{
2600	struct usb_endpoint *ep;
2601	struct usb_xfer *xfer;
2602	uint8_t type;
2603
2604	xfer = pq->curr;
2605	ep = xfer->endpoint;
2606
2607	USB_BUS_LOCK_ASSERT(xfer->xroot->bus, MA_OWNED);
2608
2609	/*
2610	 * If the endpoint is already stalled we do nothing !
2611	 */
2612	if (ep->is_stalled) {
2613		return;
2614	}
2615	/*
2616	 * Check if we are supposed to stall the endpoint:
2617	 */
2618	if (xfer->flags.stall_pipe) {
2619		struct usb_device *udev;
2620		struct usb_xfer_root *info;
2621
2622		/* clear stall command */
2623		xfer->flags.stall_pipe = 0;
2624
2625		/* get pointer to USB device */
2626		info = xfer->xroot;
2627		udev = info->udev;
2628
2629		/*
2630		 * Only stall BULK and INTERRUPT endpoints.
2631		 */
2632		type = (ep->edesc->bmAttributes & UE_XFERTYPE);
2633		if ((type == UE_BULK) ||
2634		    (type == UE_INTERRUPT)) {
2635			uint8_t did_stall;
2636
2637			did_stall = 1;
2638
2639			if (udev->flags.usb_mode == USB_MODE_DEVICE) {
2640				(udev->bus->methods->set_stall) (
2641				    udev, ep, &did_stall);
2642			} else if (udev->ctrl_xfer[1]) {
2643				info = udev->ctrl_xfer[1]->xroot;
2644				usb_proc_msignal(
2645				    USB_BUS_NON_GIANT_PROC(info->bus),
2646				    &udev->cs_msg[0], &udev->cs_msg[1]);
2647			} else {
2648				/* should not happen */
2649				DPRINTFN(0, "No stall handler\n");
2650			}
2651			/*
2652			 * Check if we should stall. Some USB hardware
2653			 * handles set- and clear-stall in hardware.
2654			 */
2655			if (did_stall) {
2656				/*
2657				 * The transfer will be continued when
2658				 * the clear-stall control endpoint
2659				 * message is received.
2660				 */
2661				ep->is_stalled = 1;
2662				return;
2663			}
2664		} else if (type == UE_ISOCHRONOUS) {
2665
2666			/*
2667			 * Make sure any FIFO overflow or other FIFO
2668			 * error conditions go away by resetting the
2669			 * endpoint FIFO through the clear stall
2670			 * method.
2671			 */
2672			if (udev->flags.usb_mode == USB_MODE_DEVICE) {
2673				(udev->bus->methods->clear_stall) (udev, ep);
2674			}
2675		}
2676	}
2677	/* Set or clear stall complete - special case */
2678	if (xfer->nframes == 0) {
2679		/* we are complete */
2680		xfer->aframes = 0;
2681		usbd_transfer_done(xfer, 0);
2682		return;
2683	}
2684	/*
2685	 * Handled cases:
2686	 *
2687	 * 1) Start the first transfer queued.
2688	 *
2689	 * 2) Re-start the current USB transfer.
2690	 */
2691	/*
2692	 * Check if there should be any
2693	 * pre transfer start delay:
2694	 */
2695	if (xfer->interval > 0) {
2696		type = (ep->edesc->bmAttributes & UE_XFERTYPE);
2697		if ((type == UE_BULK) ||
2698		    (type == UE_CONTROL)) {
2699			usbd_transfer_timeout_ms(xfer,
2700			    &usbd_transfer_start_cb,
2701			    xfer->interval);
2702			return;
2703		}
2704	}
2705	DPRINTF("start\n");
2706
2707#if USB_HAVE_PF
2708	usbpf_xfertap(xfer, USBPF_XFERTAP_SUBMIT);
2709#endif
2710	/* the transfer can now be cancelled */
2711	xfer->flags_int.can_cancel_immed = 1;
2712
2713	/* start USB transfer, if no error */
2714	if (xfer->error == 0)
2715		(ep->methods->start) (xfer);
2716
2717	/* check for transfer error */
2718	if (xfer->error) {
2719		/* some error has happened */
2720		usbd_transfer_done(xfer, 0);
2721	}
2722}
2723
2724/*------------------------------------------------------------------------*
2725 *	usbd_transfer_timeout_ms
2726 *
2727 * This function is used to setup a timeout on the given USB
2728 * transfer. If the timeout has been deferred the callback given by
2729 * "cb" will get called after "ms" milliseconds.
2730 *------------------------------------------------------------------------*/
2731void
2732usbd_transfer_timeout_ms(struct usb_xfer *xfer,
2733    void (*cb) (void *arg), usb_timeout_t ms)
2734{
2735	USB_BUS_LOCK_ASSERT(xfer->xroot->bus, MA_OWNED);
2736
2737	/* defer delay */
2738	usb_callout_reset(&xfer->timeout_handle,
2739	    USB_MS_TO_TICKS(ms) + USB_CALLOUT_ZERO_TICKS, cb, xfer);
2740}
2741
2742/*------------------------------------------------------------------------*
2743 *	usbd_callback_wrapper_sub
2744 *
2745 *  - This function will update variables in an USB transfer after
2746 *  that the USB transfer is complete.
2747 *
2748 *  - This function is used to start the next USB transfer on the
2749 *  ep transfer queue, if any.
2750 *
2751 * NOTE: In some special cases the USB transfer will not be removed from
2752 * the pipe queue, but remain first. To enforce USB transfer removal call
2753 * this function passing the error code "USB_ERR_CANCELLED".
2754 *
2755 * Return values:
2756 * 0: Success.
2757 * Else: The callback has been deferred.
2758 *------------------------------------------------------------------------*/
2759static uint8_t
2760usbd_callback_wrapper_sub(struct usb_xfer *xfer)
2761{
2762	struct usb_endpoint *ep;
2763	struct usb_bus *bus;
2764	usb_frcount_t x;
2765
2766	bus = xfer->xroot->bus;
2767
2768	if ((!xfer->flags_int.open) &&
2769	    (!xfer->flags_int.did_close)) {
2770		DPRINTF("close\n");
2771		USB_BUS_LOCK(bus);
2772		(xfer->endpoint->methods->close) (xfer);
2773		USB_BUS_UNLOCK(bus);
2774		/* only close once */
2775		xfer->flags_int.did_close = 1;
2776		return (1);		/* wait for new callback */
2777	}
2778	/*
2779	 * If we have a non-hardware induced error we
2780	 * need to do the DMA delay!
2781	 */
2782	if (xfer->error != 0 && !xfer->flags_int.did_dma_delay &&
2783	    (xfer->error == USB_ERR_CANCELLED ||
2784	    xfer->error == USB_ERR_TIMEOUT ||
2785	    bus->methods->start_dma_delay != NULL)) {
2786
2787		usb_timeout_t temp;
2788
2789		/* only delay once */
2790		xfer->flags_int.did_dma_delay = 1;
2791
2792		/* we can not cancel this delay */
2793		xfer->flags_int.can_cancel_immed = 0;
2794
2795		temp = usbd_get_dma_delay(xfer->xroot->udev);
2796
2797		DPRINTFN(3, "DMA delay, %u ms, "
2798		    "on %p\n", temp, xfer);
2799
2800		if (temp != 0) {
2801			USB_BUS_LOCK(bus);
2802			/*
2803			 * Some hardware solutions have dedicated
2804			 * events when it is safe to free DMA'ed
2805			 * memory. For the other hardware platforms we
2806			 * use a static delay.
2807			 */
2808			if (bus->methods->start_dma_delay != NULL) {
2809				(bus->methods->start_dma_delay) (xfer);
2810			} else {
2811				usbd_transfer_timeout_ms(xfer,
2812				    (void (*)(void *))&usb_dma_delay_done_cb,
2813				    temp);
2814			}
2815			USB_BUS_UNLOCK(bus);
2816			return (1);	/* wait for new callback */
2817		}
2818	}
2819	/* check actual number of frames */
2820	if (xfer->aframes > xfer->nframes) {
2821		if (xfer->error == 0) {
2822			panic("%s: actual number of frames, %d, is "
2823			    "greater than initial number of frames, %d\n",
2824			    __FUNCTION__, xfer->aframes, xfer->nframes);
2825		} else {
2826			/* just set some valid value */
2827			xfer->aframes = xfer->nframes;
2828		}
2829	}
2830	/* compute actual length */
2831	xfer->actlen = 0;
2832
2833	for (x = 0; x != xfer->aframes; x++) {
2834		xfer->actlen += xfer->frlengths[x];
2835	}
2836
2837	/*
2838	 * Frames that were not transferred get zero actual length in
2839	 * case the USB device driver does not check the actual number
2840	 * of frames transferred, "xfer->aframes":
2841	 */
2842	for (; x < xfer->nframes; x++) {
2843		usbd_xfer_set_frame_len(xfer, x, 0);
2844	}
2845
2846	/* check actual length */
2847	if (xfer->actlen > xfer->sumlen) {
2848		if (xfer->error == 0) {
2849			panic("%s: actual length, %d, is greater than "
2850			    "initial length, %d\n",
2851			    __FUNCTION__, xfer->actlen, xfer->sumlen);
2852		} else {
2853			/* just set some valid value */
2854			xfer->actlen = xfer->sumlen;
2855		}
2856	}
2857	DPRINTFN(1, "xfer=%p endpoint=%p sts=%d alen=%d, slen=%d, afrm=%d, nfrm=%d\n",
2858	    xfer, xfer->endpoint, xfer->error, xfer->actlen, xfer->sumlen,
2859	    xfer->aframes, xfer->nframes);
2860
2861	if (xfer->error) {
2862		/* end of control transfer, if any */
2863		xfer->flags_int.control_act = 0;
2864
2865#if USB_HAVE_TT_SUPPORT
2866		switch (xfer->error) {
2867		case USB_ERR_NORMAL_COMPLETION:
2868		case USB_ERR_SHORT_XFER:
2869		case USB_ERR_STALLED:
2870		case USB_ERR_CANCELLED:
2871			/* nothing to do */
2872			break;
2873		default:
2874			/* try to reset the TT, if any */
2875			USB_BUS_LOCK(bus);
2876			uhub_tt_buffer_reset_async_locked(xfer->xroot->udev, xfer->endpoint);
2877			USB_BUS_UNLOCK(bus);
2878			break;
2879		}
2880#endif
2881		/* check if we should block the execution queue */
2882		if ((xfer->error != USB_ERR_CANCELLED) &&
2883		    (xfer->flags.pipe_bof)) {
2884			DPRINTFN(2, "xfer=%p: Block On Failure "
2885			    "on endpoint=%p\n", xfer, xfer->endpoint);
2886			goto done;
2887		}
2888	} else {
2889		/* check for short transfers */
2890		if (xfer->actlen < xfer->sumlen) {
2891
2892			/* end of control transfer, if any */
2893			xfer->flags_int.control_act = 0;
2894
2895			if (!xfer->flags_int.short_xfer_ok) {
2896				xfer->error = USB_ERR_SHORT_XFER;
2897				if (xfer->flags.pipe_bof) {
2898					DPRINTFN(2, "xfer=%p: Block On Failure on "
2899					    "Short Transfer on endpoint %p.\n",
2900					    xfer, xfer->endpoint);
2901					goto done;
2902				}
2903			}
2904		} else {
2905			/*
2906			 * Check if we are in the middle of a
2907			 * control transfer:
2908			 */
2909			if (xfer->flags_int.control_act) {
2910				DPRINTFN(5, "xfer=%p: Control transfer "
2911				    "active on endpoint=%p\n", xfer, xfer->endpoint);
2912				goto done;
2913			}
2914		}
2915	}
2916
2917	ep = xfer->endpoint;
2918
2919	/*
2920	 * If the current USB transfer is completing we need to start the
2921	 * next one:
2922	 */
2923	USB_BUS_LOCK(bus);
2924	if (ep->endpoint_q[xfer->stream_id].curr == xfer) {
2925		usb_command_wrapper(&ep->endpoint_q[xfer->stream_id], NULL);
2926
2927		if (ep->endpoint_q[xfer->stream_id].curr != NULL ||
2928		    TAILQ_FIRST(&ep->endpoint_q[xfer->stream_id].head) != NULL) {
2929			/* there is another USB transfer waiting */
2930		} else {
2931			/* this is the last USB transfer */
2932			/* clear isochronous sync flag */
2933			xfer->endpoint->is_synced = 0;
2934		}
2935	}
2936	USB_BUS_UNLOCK(bus);
2937done:
2938	return (0);
2939}
2940
2941/*------------------------------------------------------------------------*
2942 *	usb_command_wrapper
2943 *
2944 * This function is used to execute commands non-recursivly on an USB
2945 * transfer.
2946 *------------------------------------------------------------------------*/
2947void
2948usb_command_wrapper(struct usb_xfer_queue *pq, struct usb_xfer *xfer)
2949{
2950	if (xfer) {
2951		/*
2952		 * If the transfer is not already processing,
2953		 * queue it!
2954		 */
2955		if (pq->curr != xfer) {
2956			usbd_transfer_enqueue(pq, xfer);
2957			if (pq->curr != NULL) {
2958				/* something is already processing */
2959				DPRINTFN(6, "busy %p\n", pq->curr);
2960				return;
2961			}
2962		}
2963	} else {
2964		/* Get next element in queue */
2965		pq->curr = NULL;
2966	}
2967
2968	if (!pq->recurse_1) {
2969
2970		do {
2971
2972			/* set both recurse flags */
2973			pq->recurse_1 = 1;
2974			pq->recurse_2 = 1;
2975
2976			if (pq->curr == NULL) {
2977				xfer = TAILQ_FIRST(&pq->head);
2978				if (xfer) {
2979					TAILQ_REMOVE(&pq->head, xfer,
2980					    wait_entry);
2981					xfer->wait_queue = NULL;
2982					pq->curr = xfer;
2983				} else {
2984					break;
2985				}
2986			}
2987			DPRINTFN(6, "cb %p (enter)\n", pq->curr);
2988			(pq->command) (pq);
2989			DPRINTFN(6, "cb %p (leave)\n", pq->curr);
2990
2991		} while (!pq->recurse_2);
2992
2993		/* clear first recurse flag */
2994		pq->recurse_1 = 0;
2995
2996	} else {
2997		/* clear second recurse flag */
2998		pq->recurse_2 = 0;
2999	}
3000}
3001
3002/*------------------------------------------------------------------------*
3003 *	usbd_ctrl_transfer_setup
3004 *
3005 * This function is used to setup the default USB control endpoint
3006 * transfer.
3007 *------------------------------------------------------------------------*/
3008void
3009usbd_ctrl_transfer_setup(struct usb_device *udev)
3010{
3011	struct usb_xfer *xfer;
3012	uint8_t no_resetup;
3013	uint8_t iface_index;
3014
3015	/* check for root HUB */
3016	if (udev->parent_hub == NULL)
3017		return;
3018repeat:
3019
3020	xfer = udev->ctrl_xfer[0];
3021	if (xfer) {
3022		USB_XFER_LOCK(xfer);
3023		no_resetup =
3024		    ((xfer->address == udev->address) &&
3025		    (udev->ctrl_ep_desc.wMaxPacketSize[0] ==
3026		    udev->ddesc.bMaxPacketSize));
3027		if (udev->flags.usb_mode == USB_MODE_DEVICE) {
3028			if (no_resetup) {
3029				/*
3030				 * NOTE: checking "xfer->address" and
3031				 * starting the USB transfer must be
3032				 * atomic!
3033				 */
3034				usbd_transfer_start(xfer);
3035			}
3036		}
3037		USB_XFER_UNLOCK(xfer);
3038	} else {
3039		no_resetup = 0;
3040	}
3041
3042	if (no_resetup) {
3043		/*
3044	         * All parameters are exactly the same like before.
3045	         * Just return.
3046	         */
3047		return;
3048	}
3049	/*
3050	 * Update wMaxPacketSize for the default control endpoint:
3051	 */
3052	udev->ctrl_ep_desc.wMaxPacketSize[0] =
3053	    udev->ddesc.bMaxPacketSize;
3054
3055	/*
3056	 * Unsetup any existing USB transfer:
3057	 */
3058	usbd_transfer_unsetup(udev->ctrl_xfer, USB_CTRL_XFER_MAX);
3059
3060	/*
3061	 * Reset clear stall error counter.
3062	 */
3063	udev->clear_stall_errors = 0;
3064
3065	/*
3066	 * Try to setup a new USB transfer for the
3067	 * default control endpoint:
3068	 */
3069	iface_index = 0;
3070	if (usbd_transfer_setup(udev, &iface_index,
3071	    udev->ctrl_xfer, usb_control_ep_cfg, USB_CTRL_XFER_MAX, NULL,
3072	    &udev->device_mtx)) {
3073		DPRINTFN(0, "could not setup default "
3074		    "USB transfer\n");
3075	} else {
3076		goto repeat;
3077	}
3078}
3079
3080/*------------------------------------------------------------------------*
3081 *	usbd_clear_data_toggle - factored out code
3082 *
3083 * NOTE: the intention of this function is not to reset the hardware
3084 * data toggle.
3085 *------------------------------------------------------------------------*/
3086void
3087usbd_clear_stall_locked(struct usb_device *udev, struct usb_endpoint *ep)
3088{
3089	USB_BUS_LOCK_ASSERT(udev->bus, MA_OWNED);
3090
3091	/* check that we have a valid case */
3092	if (udev->flags.usb_mode == USB_MODE_HOST &&
3093	    udev->parent_hub != NULL &&
3094	    udev->bus->methods->clear_stall != NULL &&
3095	    ep->methods != NULL) {
3096		(udev->bus->methods->clear_stall) (udev, ep);
3097	}
3098}
3099
3100/*------------------------------------------------------------------------*
3101 *	usbd_clear_data_toggle - factored out code
3102 *
3103 * NOTE: the intention of this function is not to reset the hardware
3104 * data toggle on the USB device side.
3105 *------------------------------------------------------------------------*/
3106void
3107usbd_clear_data_toggle(struct usb_device *udev, struct usb_endpoint *ep)
3108{
3109	DPRINTFN(5, "udev=%p endpoint=%p\n", udev, ep);
3110
3111	USB_BUS_LOCK(udev->bus);
3112	ep->toggle_next = 0;
3113	/* some hardware needs a callback to clear the data toggle */
3114	usbd_clear_stall_locked(udev, ep);
3115	USB_BUS_UNLOCK(udev->bus);
3116}
3117
3118/*------------------------------------------------------------------------*
3119 *	usbd_clear_stall_callback - factored out clear stall callback
3120 *
3121 * Input parameters:
3122 *  xfer1: Clear Stall Control Transfer
3123 *  xfer2: Stalled USB Transfer
3124 *
3125 * This function is NULL safe.
3126 *
3127 * Return values:
3128 *   0: In progress
3129 *   Else: Finished
3130 *
3131 * Clear stall config example:
3132 *
3133 * static const struct usb_config my_clearstall =  {
3134 *	.type = UE_CONTROL,
3135 *	.endpoint = 0,
3136 *	.direction = UE_DIR_ANY,
3137 *	.interval = 50, //50 milliseconds
3138 *	.bufsize = sizeof(struct usb_device_request),
3139 *	.timeout = 1000, //1.000 seconds
3140 *	.callback = &my_clear_stall_callback, // **
3141 *	.usb_mode = USB_MODE_HOST,
3142 * };
3143 *
3144 * ** "my_clear_stall_callback" calls "usbd_clear_stall_callback"
3145 * passing the correct parameters.
3146 *------------------------------------------------------------------------*/
3147uint8_t
3148usbd_clear_stall_callback(struct usb_xfer *xfer1,
3149    struct usb_xfer *xfer2)
3150{
3151	struct usb_device_request req;
3152
3153	if (xfer2 == NULL) {
3154		/* looks like we are tearing down */
3155		DPRINTF("NULL input parameter\n");
3156		return (0);
3157	}
3158	USB_XFER_LOCK_ASSERT(xfer1, MA_OWNED);
3159	USB_XFER_LOCK_ASSERT(xfer2, MA_OWNED);
3160
3161	switch (USB_GET_STATE(xfer1)) {
3162	case USB_ST_SETUP:
3163
3164		/*
3165		 * pre-clear the data toggle to DATA0 ("umass.c" and
3166		 * "ata-usb.c" depends on this)
3167		 */
3168
3169		usbd_clear_data_toggle(xfer2->xroot->udev, xfer2->endpoint);
3170
3171		/* setup a clear-stall packet */
3172
3173		req.bmRequestType = UT_WRITE_ENDPOINT;
3174		req.bRequest = UR_CLEAR_FEATURE;
3175		USETW(req.wValue, UF_ENDPOINT_HALT);
3176		req.wIndex[0] = xfer2->endpoint->edesc->bEndpointAddress;
3177		req.wIndex[1] = 0;
3178		USETW(req.wLength, 0);
3179
3180		/*
3181		 * "usbd_transfer_setup_sub()" will ensure that
3182		 * we have sufficient room in the buffer for
3183		 * the request structure!
3184		 */
3185
3186		/* copy in the transfer */
3187
3188		usbd_copy_in(xfer1->frbuffers, 0, &req, sizeof(req));
3189
3190		/* set length */
3191		xfer1->frlengths[0] = sizeof(req);
3192		xfer1->nframes = 1;
3193
3194		usbd_transfer_submit(xfer1);
3195		return (0);
3196
3197	case USB_ST_TRANSFERRED:
3198		break;
3199
3200	default:			/* Error */
3201		if (xfer1->error == USB_ERR_CANCELLED) {
3202			return (0);
3203		}
3204		break;
3205	}
3206	return (1);			/* Clear Stall Finished */
3207}
3208
3209/*------------------------------------------------------------------------*
3210 *	usbd_transfer_poll
3211 *
3212 * The following function gets called from the USB keyboard driver and
3213 * UMASS when the system has paniced.
3214 *
3215 * NOTE: It is currently not possible to resume normal operation on
3216 * the USB controller which has been polled, due to clearing of the
3217 * "up_dsleep" and "up_msleep" flags.
3218 *------------------------------------------------------------------------*/
3219void
3220usbd_transfer_poll(struct usb_xfer **ppxfer, uint16_t max)
3221{
3222	struct usb_xfer *xfer;
3223	struct usb_xfer_root *xroot;
3224	struct usb_device *udev;
3225	struct usb_proc_msg *pm;
3226	uint16_t n;
3227	uint16_t drop_bus;
3228	uint16_t drop_xfer;
3229
3230	for (n = 0; n != max; n++) {
3231		/* Extra checks to avoid panic */
3232		xfer = ppxfer[n];
3233		if (xfer == NULL)
3234			continue;	/* no USB transfer */
3235		xroot = xfer->xroot;
3236		if (xroot == NULL)
3237			continue;	/* no USB root */
3238		udev = xroot->udev;
3239		if (udev == NULL)
3240			continue;	/* no USB device */
3241		if (udev->bus == NULL)
3242			continue;	/* no BUS structure */
3243		if (udev->bus->methods == NULL)
3244			continue;	/* no BUS methods */
3245		if (udev->bus->methods->xfer_poll == NULL)
3246			continue;	/* no poll method */
3247
3248		/* make sure that the BUS mutex is not locked */
3249		drop_bus = 0;
3250		while (mtx_owned(&xroot->udev->bus->bus_mtx) && !SCHEDULER_STOPPED()) {
3251			mtx_unlock(&xroot->udev->bus->bus_mtx);
3252			drop_bus++;
3253		}
3254
3255		/* make sure that the transfer mutex is not locked */
3256		drop_xfer = 0;
3257		while (mtx_owned(xroot->xfer_mtx) && !SCHEDULER_STOPPED()) {
3258			mtx_unlock(xroot->xfer_mtx);
3259			drop_xfer++;
3260		}
3261
3262		/* Make sure cv_signal() and cv_broadcast() is not called */
3263		USB_BUS_CONTROL_XFER_PROC(udev->bus)->up_msleep = 0;
3264		USB_BUS_EXPLORE_PROC(udev->bus)->up_msleep = 0;
3265		USB_BUS_GIANT_PROC(udev->bus)->up_msleep = 0;
3266		USB_BUS_NON_GIANT_PROC(udev->bus)->up_msleep = 0;
3267
3268		/* poll USB hardware */
3269		(udev->bus->methods->xfer_poll) (udev->bus);
3270
3271		USB_BUS_LOCK(xroot->bus);
3272
3273		/* check for clear stall */
3274		if (udev->ctrl_xfer[1] != NULL) {
3275
3276			/* poll clear stall start */
3277			pm = &udev->cs_msg[0].hdr;
3278			(pm->pm_callback) (pm);
3279			/* poll clear stall done thread */
3280			pm = &udev->ctrl_xfer[1]->
3281			    xroot->done_m[0].hdr;
3282			(pm->pm_callback) (pm);
3283		}
3284
3285		/* poll done thread */
3286		pm = &xroot->done_m[0].hdr;
3287		(pm->pm_callback) (pm);
3288
3289		USB_BUS_UNLOCK(xroot->bus);
3290
3291		/* restore transfer mutex */
3292		while (drop_xfer--)
3293			mtx_lock(xroot->xfer_mtx);
3294
3295		/* restore BUS mutex */
3296		while (drop_bus--)
3297			mtx_lock(&xroot->udev->bus->bus_mtx);
3298	}
3299}
3300
3301static void
3302usbd_get_std_packet_size(struct usb_std_packet_size *ptr,
3303    uint8_t type, enum usb_dev_speed speed)
3304{
3305	static const uint16_t intr_range_max[USB_SPEED_MAX] = {
3306		[USB_SPEED_LOW] = 8,
3307		[USB_SPEED_FULL] = 64,
3308		[USB_SPEED_HIGH] = 1024,
3309		[USB_SPEED_VARIABLE] = 1024,
3310		[USB_SPEED_SUPER] = 1024,
3311	};
3312
3313	static const uint16_t isoc_range_max[USB_SPEED_MAX] = {
3314		[USB_SPEED_LOW] = 0,	/* invalid */
3315		[USB_SPEED_FULL] = 1023,
3316		[USB_SPEED_HIGH] = 1024,
3317		[USB_SPEED_VARIABLE] = 3584,
3318		[USB_SPEED_SUPER] = 1024,
3319	};
3320
3321	static const uint16_t control_min[USB_SPEED_MAX] = {
3322		[USB_SPEED_LOW] = 8,
3323		[USB_SPEED_FULL] = 8,
3324		[USB_SPEED_HIGH] = 64,
3325		[USB_SPEED_VARIABLE] = 512,
3326		[USB_SPEED_SUPER] = 512,
3327	};
3328
3329	static const uint16_t bulk_min[USB_SPEED_MAX] = {
3330		[USB_SPEED_LOW] = 8,
3331		[USB_SPEED_FULL] = 8,
3332		[USB_SPEED_HIGH] = 512,
3333		[USB_SPEED_VARIABLE] = 512,
3334		[USB_SPEED_SUPER] = 1024,
3335	};
3336
3337	uint16_t temp;
3338
3339	memset(ptr, 0, sizeof(*ptr));
3340
3341	switch (type) {
3342	case UE_INTERRUPT:
3343		ptr->range.max = intr_range_max[speed];
3344		break;
3345	case UE_ISOCHRONOUS:
3346		ptr->range.max = isoc_range_max[speed];
3347		break;
3348	default:
3349		if (type == UE_BULK)
3350			temp = bulk_min[speed];
3351		else /* UE_CONTROL */
3352			temp = control_min[speed];
3353
3354		/* default is fixed */
3355		ptr->fixed[0] = temp;
3356		ptr->fixed[1] = temp;
3357		ptr->fixed[2] = temp;
3358		ptr->fixed[3] = temp;
3359
3360		if (speed == USB_SPEED_FULL) {
3361			/* multiple sizes */
3362			ptr->fixed[1] = 16;
3363			ptr->fixed[2] = 32;
3364			ptr->fixed[3] = 64;
3365		}
3366		if ((speed == USB_SPEED_VARIABLE) &&
3367		    (type == UE_BULK)) {
3368			/* multiple sizes */
3369			ptr->fixed[2] = 1024;
3370			ptr->fixed[3] = 1536;
3371		}
3372		break;
3373	}
3374}
3375
3376void	*
3377usbd_xfer_softc(struct usb_xfer *xfer)
3378{
3379	return (xfer->priv_sc);
3380}
3381
3382void *
3383usbd_xfer_get_priv(struct usb_xfer *xfer)
3384{
3385	return (xfer->priv_fifo);
3386}
3387
3388void
3389usbd_xfer_set_priv(struct usb_xfer *xfer, void *ptr)
3390{
3391	xfer->priv_fifo = ptr;
3392}
3393
3394uint8_t
3395usbd_xfer_state(struct usb_xfer *xfer)
3396{
3397	return (xfer->usb_state);
3398}
3399
3400void
3401usbd_xfer_set_flag(struct usb_xfer *xfer, int flag)
3402{
3403	switch (flag) {
3404		case USB_FORCE_SHORT_XFER:
3405			xfer->flags.force_short_xfer = 1;
3406			break;
3407		case USB_SHORT_XFER_OK:
3408			xfer->flags.short_xfer_ok = 1;
3409			break;
3410		case USB_MULTI_SHORT_OK:
3411			xfer->flags.short_frames_ok = 1;
3412			break;
3413		case USB_MANUAL_STATUS:
3414			xfer->flags.manual_status = 1;
3415			break;
3416	}
3417}
3418
3419void
3420usbd_xfer_clr_flag(struct usb_xfer *xfer, int flag)
3421{
3422	switch (flag) {
3423		case USB_FORCE_SHORT_XFER:
3424			xfer->flags.force_short_xfer = 0;
3425			break;
3426		case USB_SHORT_XFER_OK:
3427			xfer->flags.short_xfer_ok = 0;
3428			break;
3429		case USB_MULTI_SHORT_OK:
3430			xfer->flags.short_frames_ok = 0;
3431			break;
3432		case USB_MANUAL_STATUS:
3433			xfer->flags.manual_status = 0;
3434			break;
3435	}
3436}
3437
3438/*
3439 * The following function returns in milliseconds when the isochronous
3440 * transfer was completed by the hardware. The returned value wraps
3441 * around 65536 milliseconds.
3442 */
3443uint16_t
3444usbd_xfer_get_timestamp(struct usb_xfer *xfer)
3445{
3446	return (xfer->isoc_time_complete);
3447}
3448
3449/*
3450 * The following function returns non-zero if the max packet size
3451 * field was clamped to a valid value. Else it returns zero.
3452 */
3453uint8_t
3454usbd_xfer_maxp_was_clamped(struct usb_xfer *xfer)
3455{
3456	return (xfer->flags_int.maxp_was_clamped);
3457}
3458