usb_generic.c revision 188942
1/* $FreeBSD: head/sys/dev/usb/usb_generic.c 188942 2009-02-23 18:31:00Z thompsa $ */
2/*-
3 * Copyright (c) 2008 Hans Petter Selasky. All rights reserved.
4 *
5 * Redistribution and use in source and binary forms, with or without
6 * modification, are permitted provided that the following conditions
7 * are met:
8 * 1. Redistributions of source code must retain the above copyright
9 *    notice, this list of conditions and the following disclaimer.
10 * 2. Redistributions in binary form must reproduce the above copyright
11 *    notice, this list of conditions and the following disclaimer in the
12 *    documentation and/or other materials provided with the distribution.
13 *
14 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
15 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
16 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
17 * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
18 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
19 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
20 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
21 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
22 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
23 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
24 * SUCH DAMAGE.
25 */
26
27#include <dev/usb/usb_defs.h>
28#include <dev/usb/usb_mfunc.h>
29#include <dev/usb/usb.h>
30#include <dev/usb/usb_ioctl.h>
31#include <dev/usb/usb_error.h>
32
33#define	USB_DEBUG_VAR ugen_debug
34
35#include <dev/usb/usb_core.h>
36#include <dev/usb/usb_mbuf.h>
37#include <dev/usb/usb_dev.h>
38#include <dev/usb/usb_process.h>
39#include <dev/usb/usb_device.h>
40#include <dev/usb/usb_debug.h>
41#include <dev/usb/usb_request.h>
42#include <dev/usb/usb_busdma.h>
43#include <dev/usb/usb_util.h>
44#include <dev/usb/usb_hub.h>
45#include <dev/usb/usb_generic.h>
46#include <dev/usb/usb_transfer.h>
47
48#include <dev/usb/usb_controller.h>
49#include <dev/usb/usb_bus.h>
50
51/* defines */
52
53#define	UGEN_BULK_FS_BUFFER_SIZE	(64*32)	/* bytes */
54#define	UGEN_BULK_HS_BUFFER_SIZE	(1024*32)	/* bytes */
55#define	UGEN_HW_FRAMES	50		/* number of milliseconds per transfer */
56
57/* function prototypes */
58
59static usb2_callback_t ugen_read_clear_stall_callback;
60static usb2_callback_t ugen_write_clear_stall_callback;
61static usb2_callback_t ugen_default_read_callback;
62static usb2_callback_t ugen_default_write_callback;
63static usb2_callback_t ugen_isoc_read_callback;
64static usb2_callback_t ugen_isoc_write_callback;
65static usb2_callback_t ugen_default_fs_callback;
66
67static usb2_fifo_open_t ugen_open;
68static usb2_fifo_close_t ugen_close;
69static usb2_fifo_ioctl_t ugen_ioctl;
70static usb2_fifo_ioctl_t ugen_ioctl_post;
71static usb2_fifo_cmd_t ugen_start_read;
72static usb2_fifo_cmd_t ugen_start_write;
73static usb2_fifo_cmd_t ugen_stop_io;
74
75static int	ugen_transfer_setup(struct usb2_fifo *,
76		     const struct usb2_config *, uint8_t);
77static int	ugen_open_pipe_write(struct usb2_fifo *);
78static int	ugen_open_pipe_read(struct usb2_fifo *);
79static int	ugen_set_config(struct usb2_fifo *, uint8_t);
80static int	ugen_set_interface(struct usb2_fifo *, uint8_t, uint8_t);
81static int	ugen_get_cdesc(struct usb2_fifo *, struct usb2_gen_descriptor *);
82static int	ugen_get_sdesc(struct usb2_fifo *, struct usb2_gen_descriptor *);
83static int	ugen_get_iface_driver(struct usb2_fifo *f, struct usb2_gen_descriptor *ugd);
84static int	usb2_gen_fill_deviceinfo(struct usb2_fifo *,
85		    struct usb2_device_info *);
86static int	ugen_re_enumerate(struct usb2_fifo *);
87static int	ugen_iface_ioctl(struct usb2_fifo *, u_long, void *, int);
88static uint8_t	ugen_fs_get_complete(struct usb2_fifo *, uint8_t *);
89static int ugen_fs_uninit(struct usb2_fifo *f);
90
91/* structures */
92
93struct usb2_fifo_methods usb2_ugen_methods = {
94	.f_open = &ugen_open,
95	.f_close = &ugen_close,
96	.f_ioctl = &ugen_ioctl,
97	.f_ioctl_post = &ugen_ioctl_post,
98	.f_start_read = &ugen_start_read,
99	.f_stop_read = &ugen_stop_io,
100	.f_start_write = &ugen_start_write,
101	.f_stop_write = &ugen_stop_io,
102};
103
104#if USB_DEBUG
105static int ugen_debug = 0;
106
107SYSCTL_NODE(_hw_usb2, OID_AUTO, ugen, CTLFLAG_RW, 0, "USB generic");
108SYSCTL_INT(_hw_usb2_ugen, OID_AUTO, debug, CTLFLAG_RW, &ugen_debug,
109    0, "Debug level");
110#endif
111
112
113/* prototypes */
114
115static int
116ugen_transfer_setup(struct usb2_fifo *f,
117    const struct usb2_config *setup, uint8_t n_setup)
118{
119	struct usb2_pipe *pipe = f->priv_sc0;
120	struct usb2_device *udev = f->udev;
121	uint8_t iface_index = pipe->iface_index;
122	int error;
123
124	mtx_unlock(f->priv_mtx);
125
126	/*
127	 * "usb2_transfer_setup()" can sleep so one needs to make a wrapper,
128	 * exiting the mutex and checking things
129	 */
130	error = usb2_transfer_setup(udev, &iface_index, f->xfer,
131	    setup, n_setup, f, f->priv_mtx);
132	if (error == 0) {
133
134		if (f->xfer[0]->nframes == 1) {
135			error = usb2_fifo_alloc_buffer(f,
136			    f->xfer[0]->max_data_length, 2);
137		} else {
138			error = usb2_fifo_alloc_buffer(f,
139			    f->xfer[0]->max_frame_size,
140			    2 * f->xfer[0]->nframes);
141		}
142		if (error) {
143			usb2_transfer_unsetup(f->xfer, n_setup);
144		}
145	}
146	mtx_lock(f->priv_mtx);
147
148	return (error);
149}
150
151static int
152ugen_open(struct usb2_fifo *f, int fflags, struct thread *td)
153{
154	struct usb2_pipe *pipe = f->priv_sc0;
155	struct usb2_endpoint_descriptor *ed = pipe->edesc;
156	uint8_t type;
157
158	DPRINTFN(6, "flag=0x%x\n", fflags);
159
160	mtx_lock(f->priv_mtx);
161	switch (usb2_get_speed(f->udev)) {
162	case USB_SPEED_LOW:
163	case USB_SPEED_FULL:
164		f->nframes = UGEN_HW_FRAMES;
165		f->bufsize = UGEN_BULK_FS_BUFFER_SIZE;
166		break;
167	default:
168		f->nframes = UGEN_HW_FRAMES * 8;
169		f->bufsize = UGEN_BULK_HS_BUFFER_SIZE;
170		break;
171	}
172
173	type = ed->bmAttributes & UE_XFERTYPE;
174	if (type == UE_INTERRUPT) {
175		f->bufsize = 0;		/* use "wMaxPacketSize" */
176	}
177	f->timeout = USB_NO_TIMEOUT;
178	f->flag_short = 0;
179	f->fifo_zlp = 0;
180	mtx_unlock(f->priv_mtx);
181
182	return (0);
183}
184
185static void
186ugen_close(struct usb2_fifo *f, int fflags, struct thread *td)
187{
188	DPRINTFN(6, "flag=0x%x\n", fflags);
189
190	/* cleanup */
191
192	mtx_lock(f->priv_mtx);
193	usb2_transfer_stop(f->xfer[0]);
194	usb2_transfer_stop(f->xfer[1]);
195	mtx_unlock(f->priv_mtx);
196
197	usb2_transfer_unsetup(f->xfer, 2);
198	usb2_fifo_free_buffer(f);
199
200	if (ugen_fs_uninit(f)) {
201		/* ignore any errors - we are closing */
202		DPRINTFN(6, "no FIFOs\n");
203	}
204}
205
206static int
207ugen_open_pipe_write(struct usb2_fifo *f)
208{
209	struct usb2_config usb2_config[2];
210	struct usb2_pipe *pipe = f->priv_sc0;
211	struct usb2_endpoint_descriptor *ed = pipe->edesc;
212
213	mtx_assert(f->priv_mtx, MA_OWNED);
214
215	if (f->xfer[0] || f->xfer[1]) {
216		/* transfers are already opened */
217		return (0);
218	}
219	bzero(usb2_config, sizeof(usb2_config));
220
221	usb2_config[1].type = UE_CONTROL;
222	usb2_config[1].endpoint = 0;
223	usb2_config[1].direction = UE_DIR_ANY;
224	usb2_config[1].mh.timeout = 1000;	/* 1 second */
225	usb2_config[1].mh.interval = 50;/* 50 milliseconds */
226	usb2_config[1].mh.bufsize = sizeof(struct usb2_device_request);
227	usb2_config[1].mh.callback = &ugen_write_clear_stall_callback;
228
229	usb2_config[0].type = ed->bmAttributes & UE_XFERTYPE;
230	usb2_config[0].endpoint = ed->bEndpointAddress & UE_ADDR;
231	usb2_config[0].direction = ed->bEndpointAddress & (UE_DIR_OUT | UE_DIR_IN);
232	usb2_config[0].mh.interval = USB_DEFAULT_INTERVAL;
233	usb2_config[0].mh.flags.proxy_buffer = 1;
234
235	switch (ed->bmAttributes & UE_XFERTYPE) {
236	case UE_INTERRUPT:
237	case UE_BULK:
238		if (f->flag_short) {
239			usb2_config[0].mh.flags.force_short_xfer = 1;
240		}
241		usb2_config[0].mh.callback = &ugen_default_write_callback;
242		usb2_config[0].mh.timeout = f->timeout;
243		usb2_config[0].mh.frames = 1;
244		usb2_config[0].mh.bufsize = f->bufsize;
245		usb2_config[0].md = usb2_config[0].mh;	/* symmetric config */
246		if (ugen_transfer_setup(f, usb2_config, 2)) {
247			return (EIO);
248		}
249		/* first transfer does not clear stall */
250		f->flag_stall = 0;
251		break;
252
253	case UE_ISOCHRONOUS:
254		usb2_config[0].mh.flags.short_xfer_ok = 1;
255		usb2_config[0].mh.bufsize = 0;	/* use default */
256		usb2_config[0].mh.frames = f->nframes;
257		usb2_config[0].mh.callback = &ugen_isoc_write_callback;
258		usb2_config[0].mh.timeout = 0;
259		usb2_config[0].md = usb2_config[0].mh;	/* symmetric config */
260
261		/* clone configuration */
262		usb2_config[1] = usb2_config[0];
263
264		if (ugen_transfer_setup(f, usb2_config, 2)) {
265			return (EIO);
266		}
267		break;
268	default:
269		return (EINVAL);
270	}
271	return (0);
272}
273
274static int
275ugen_open_pipe_read(struct usb2_fifo *f)
276{
277	struct usb2_config usb2_config[2];
278	struct usb2_pipe *pipe = f->priv_sc0;
279	struct usb2_endpoint_descriptor *ed = pipe->edesc;
280
281	mtx_assert(f->priv_mtx, MA_OWNED);
282
283	if (f->xfer[0] || f->xfer[1]) {
284		/* transfers are already opened */
285		return (0);
286	}
287	bzero(usb2_config, sizeof(usb2_config));
288
289	usb2_config[1].type = UE_CONTROL;
290	usb2_config[1].endpoint = 0;
291	usb2_config[1].direction = UE_DIR_ANY;
292	usb2_config[1].mh.timeout = 1000;	/* 1 second */
293	usb2_config[1].mh.interval = 50;/* 50 milliseconds */
294	usb2_config[1].mh.bufsize = sizeof(struct usb2_device_request);
295	usb2_config[1].mh.callback = &ugen_read_clear_stall_callback;
296
297	usb2_config[0].type = ed->bmAttributes & UE_XFERTYPE;
298	usb2_config[0].endpoint = ed->bEndpointAddress & UE_ADDR;
299	usb2_config[0].direction = UE_DIR_IN;
300	usb2_config[0].mh.interval = USB_DEFAULT_INTERVAL;
301	usb2_config[0].mh.flags.proxy_buffer = 1;
302
303	switch (ed->bmAttributes & UE_XFERTYPE) {
304	case UE_INTERRUPT:
305	case UE_BULK:
306		if (f->flag_short) {
307			usb2_config[0].mh.flags.short_xfer_ok = 1;
308		}
309		usb2_config[0].mh.timeout = f->timeout;
310		usb2_config[0].mh.frames = 1;
311		usb2_config[0].mh.callback = &ugen_default_read_callback;
312		usb2_config[0].mh.bufsize = f->bufsize;
313		usb2_config[0].md = usb2_config[0].mh;	/* symmetric config */
314
315		if (ugen_transfer_setup(f, usb2_config, 2)) {
316			return (EIO);
317		}
318		/* first transfer does not clear stall */
319		f->flag_stall = 0;
320		break;
321
322	case UE_ISOCHRONOUS:
323		usb2_config[0].mh.flags.short_xfer_ok = 1;
324		usb2_config[0].mh.bufsize = 0;	/* use default */
325		usb2_config[0].mh.frames = f->nframes;
326		usb2_config[0].mh.callback = &ugen_isoc_read_callback;
327		usb2_config[0].mh.timeout = 0;
328		usb2_config[0].md = usb2_config[0].mh;	/* symmetric config */
329
330		/* clone configuration */
331		usb2_config[1] = usb2_config[0];
332
333		if (ugen_transfer_setup(f, usb2_config, 2)) {
334			return (EIO);
335		}
336		break;
337
338	default:
339		return (EINVAL);
340	}
341	return (0);
342}
343
344static void
345ugen_start_read(struct usb2_fifo *f)
346{
347	/* check that pipes are open */
348	if (ugen_open_pipe_read(f)) {
349		/* signal error */
350		usb2_fifo_put_data_error(f);
351	}
352	/* start transfers */
353	usb2_transfer_start(f->xfer[0]);
354	usb2_transfer_start(f->xfer[1]);
355}
356
357static void
358ugen_start_write(struct usb2_fifo *f)
359{
360	/* check that pipes are open */
361	if (ugen_open_pipe_write(f)) {
362		/* signal error */
363		usb2_fifo_get_data_error(f);
364	}
365	/* start transfers */
366	usb2_transfer_start(f->xfer[0]);
367	usb2_transfer_start(f->xfer[1]);
368}
369
370static void
371ugen_stop_io(struct usb2_fifo *f)
372{
373	/* stop transfers */
374	usb2_transfer_stop(f->xfer[0]);
375	usb2_transfer_stop(f->xfer[1]);
376}
377
378static void
379ugen_default_read_callback(struct usb2_xfer *xfer)
380{
381	struct usb2_fifo *f = xfer->priv_sc;
382	struct usb2_mbuf *m;
383
384	DPRINTFN(4, "actlen=%u, aframes=%u\n", xfer->actlen, xfer->aframes);
385
386	switch (USB_GET_STATE(xfer)) {
387	case USB_ST_TRANSFERRED:
388		if (xfer->actlen == 0) {
389			if (f->fifo_zlp != 4) {
390				f->fifo_zlp++;
391			} else {
392				/*
393				 * Throttle a little bit we have multiple ZLPs
394				 * in a row!
395				 */
396				xfer->interval = 64;	/* ms */
397			}
398		} else {
399			/* clear throttle */
400			xfer->interval = 0;
401			f->fifo_zlp = 0;
402		}
403		usb2_fifo_put_data(f, xfer->frbuffers, 0,
404		    xfer->actlen, 1);
405
406	case USB_ST_SETUP:
407		if (f->flag_stall) {
408			usb2_transfer_start(f->xfer[1]);
409			break;
410		}
411		USB_IF_POLL(&f->free_q, m);
412		if (m) {
413			xfer->frlengths[0] = xfer->max_data_length;
414			usb2_start_hardware(xfer);
415		}
416		break;
417
418	default:			/* Error */
419		if (xfer->error != USB_ERR_CANCELLED) {
420			f->flag_stall = 1;
421			f->fifo_zlp = 0;
422			usb2_transfer_start(f->xfer[1]);
423		}
424		break;
425	}
426}
427
428static void
429ugen_default_write_callback(struct usb2_xfer *xfer)
430{
431	struct usb2_fifo *f = xfer->priv_sc;
432	uint32_t actlen;
433
434	DPRINTFN(4, "actlen=%u, aframes=%u\n", xfer->actlen, xfer->aframes);
435
436	switch (USB_GET_STATE(xfer)) {
437	case USB_ST_SETUP:
438	case USB_ST_TRANSFERRED:
439		/*
440		 * If writing is in stall, just jump to clear stall
441		 * callback and solve the situation.
442		 */
443		if (f->flag_stall) {
444			usb2_transfer_start(f->xfer[1]);
445			break;
446		}
447		/*
448		 * Write data, setup and perform hardware transfer.
449		 */
450		if (usb2_fifo_get_data(f, xfer->frbuffers, 0,
451		    xfer->max_data_length, &actlen, 0)) {
452			xfer->frlengths[0] = actlen;
453			usb2_start_hardware(xfer);
454		}
455		break;
456
457	default:			/* Error */
458		if (xfer->error != USB_ERR_CANCELLED) {
459			f->flag_stall = 1;
460			usb2_transfer_start(f->xfer[1]);
461		}
462		break;
463	}
464}
465
466static void
467ugen_read_clear_stall_callback(struct usb2_xfer *xfer)
468{
469	struct usb2_fifo *f = xfer->priv_sc;
470	struct usb2_xfer *xfer_other = f->xfer[0];
471
472	if (f->flag_stall == 0) {
473		/* nothing to do */
474		return;
475	}
476	if (usb2_clear_stall_callback(xfer, xfer_other)) {
477		DPRINTFN(5, "f=%p: stall cleared\n", f);
478		f->flag_stall = 0;
479		usb2_transfer_start(xfer_other);
480	}
481}
482
483static void
484ugen_write_clear_stall_callback(struct usb2_xfer *xfer)
485{
486	struct usb2_fifo *f = xfer->priv_sc;
487	struct usb2_xfer *xfer_other = f->xfer[0];
488
489	if (f->flag_stall == 0) {
490		/* nothing to do */
491		return;
492	}
493	if (usb2_clear_stall_callback(xfer, xfer_other)) {
494		DPRINTFN(5, "f=%p: stall cleared\n", f);
495		f->flag_stall = 0;
496		usb2_transfer_start(xfer_other);
497	}
498}
499
500static void
501ugen_isoc_read_callback(struct usb2_xfer *xfer)
502{
503	struct usb2_fifo *f = xfer->priv_sc;
504	uint32_t offset;
505	uint16_t n;
506
507	DPRINTFN(4, "actlen=%u, aframes=%u\n", xfer->actlen, xfer->aframes);
508
509	switch (USB_GET_STATE(xfer)) {
510	case USB_ST_TRANSFERRED:
511
512		DPRINTFN(6, "actlen=%d\n", xfer->actlen);
513
514		offset = 0;
515
516		for (n = 0; n != xfer->aframes; n++) {
517			usb2_fifo_put_data(f, xfer->frbuffers, offset,
518			    xfer->frlengths[n], 1);
519			offset += xfer->max_frame_size;
520		}
521
522	case USB_ST_SETUP:
523tr_setup:
524		for (n = 0; n != xfer->nframes; n++) {
525			/* setup size for next transfer */
526			xfer->frlengths[n] = xfer->max_frame_size;
527		}
528		usb2_start_hardware(xfer);
529		break;
530
531	default:			/* Error */
532		if (xfer->error == USB_ERR_CANCELLED) {
533			break;
534		}
535		goto tr_setup;
536	}
537}
538
539static void
540ugen_isoc_write_callback(struct usb2_xfer *xfer)
541{
542	struct usb2_fifo *f = xfer->priv_sc;
543	uint32_t actlen;
544	uint32_t offset;
545	uint16_t n;
546
547	DPRINTFN(4, "actlen=%u, aframes=%u\n", xfer->actlen, xfer->aframes);
548
549	switch (USB_GET_STATE(xfer)) {
550	case USB_ST_TRANSFERRED:
551	case USB_ST_SETUP:
552tr_setup:
553		offset = 0;
554		for (n = 0; n != xfer->nframes; n++) {
555			if (usb2_fifo_get_data(f, xfer->frbuffers, offset,
556			    xfer->max_frame_size, &actlen, 1)) {
557				xfer->frlengths[n] = actlen;
558				offset += actlen;
559			} else {
560				break;
561			}
562		}
563
564		for (; n != xfer->nframes; n++) {
565			/* fill in zero frames */
566			xfer->frlengths[n] = 0;
567		}
568		usb2_start_hardware(xfer);
569		break;
570
571	default:			/* Error */
572		if (xfer->error == USB_ERR_CANCELLED) {
573			break;
574		}
575		goto tr_setup;
576	}
577}
578
579static int
580ugen_set_config(struct usb2_fifo *f, uint8_t index)
581{
582	DPRINTFN(2, "index %u\n", index);
583
584	if (f->udev->flags.usb2_mode != USB_MODE_HOST) {
585		/* not possible in device side mode */
586		return (ENOTTY);
587	}
588	if (f->udev->curr_config_index == index) {
589		/* no change needed */
590		return (0);
591	}
592	/* make sure all FIFO's are gone */
593	/* else there can be a deadlock */
594	if (ugen_fs_uninit(f)) {
595		/* ignore any errors */
596		DPRINTFN(6, "no FIFOs\n");
597	}
598	/* change setting - will free generic FIFOs, if any */
599	if (usb2_set_config_index(f->udev, index)) {
600		return (EIO);
601	}
602	/* probe and attach */
603	if (usb2_probe_and_attach(f->udev, USB_IFACE_INDEX_ANY)) {
604		return (EIO);
605	}
606	return (0);
607}
608
609static int
610ugen_set_interface(struct usb2_fifo *f,
611    uint8_t iface_index, uint8_t alt_index)
612{
613	DPRINTFN(2, "%u, %u\n", iface_index, alt_index);
614
615	if (f->udev->flags.usb2_mode != USB_MODE_HOST) {
616		/* not possible in device side mode */
617		return (ENOTTY);
618	}
619	/* make sure all FIFO's are gone */
620	/* else there can be a deadlock */
621	if (ugen_fs_uninit(f)) {
622		/* ignore any errors */
623		DPRINTFN(6, "no FIFOs\n");
624	}
625	/* change setting - will free generic FIFOs, if any */
626	if (usb2_set_alt_interface_index(f->udev, iface_index, alt_index)) {
627		return (EIO);
628	}
629	/* probe and attach */
630	if (usb2_probe_and_attach(f->udev, iface_index)) {
631		return (EIO);
632	}
633	return (0);
634}
635
636/*------------------------------------------------------------------------*
637 *	ugen_get_cdesc
638 *
639 * This function will retrieve the complete configuration descriptor
640 * at the given index.
641 *------------------------------------------------------------------------*/
642static int
643ugen_get_cdesc(struct usb2_fifo *f, struct usb2_gen_descriptor *ugd)
644{
645	struct usb2_config_descriptor *cdesc;
646	struct usb2_device *udev = f->udev;
647	int error;
648	uint16_t len;
649	uint8_t free_data;
650
651	DPRINTFN(6, "\n");
652
653	if (ugd->ugd_data == NULL) {
654		/* userland pointer should not be zero */
655		return (EINVAL);
656	}
657	if ((ugd->ugd_config_index == USB_UNCONFIG_INDEX) ||
658	    (ugd->ugd_config_index == udev->curr_config_index)) {
659		cdesc = usb2_get_config_descriptor(udev);
660		if (cdesc == NULL) {
661			return (ENXIO);
662		}
663		free_data = 0;
664
665	} else {
666		if (usb2_req_get_config_desc_full(udev,
667		    &Giant, &cdesc, M_USBDEV,
668		    ugd->ugd_config_index)) {
669			return (ENXIO);
670		}
671		free_data = 1;
672	}
673
674	len = UGETW(cdesc->wTotalLength);
675	if (len > ugd->ugd_maxlen) {
676		len = ugd->ugd_maxlen;
677	}
678	DPRINTFN(6, "len=%u\n", len);
679
680	ugd->ugd_actlen = len;
681	ugd->ugd_offset = 0;
682
683	error = copyout(cdesc, ugd->ugd_data, len);
684
685	if (free_data) {
686		free(cdesc, M_USBDEV);
687	}
688	return (error);
689}
690
691static int
692ugen_get_sdesc(struct usb2_fifo *f, struct usb2_gen_descriptor *ugd)
693{
694	void *ptr = f->udev->bus->scratch[0].data;
695	uint16_t size = sizeof(f->udev->bus->scratch[0].data);
696	int error;
697
698	if (usb2_req_get_string_desc(f->udev, &Giant, ptr,
699	    size, ugd->ugd_lang_id, ugd->ugd_string_index)) {
700		error = EINVAL;
701	} else {
702
703		if (size > ((uint8_t *)ptr)[0]) {
704			size = ((uint8_t *)ptr)[0];
705		}
706		if (size > ugd->ugd_maxlen) {
707			size = ugd->ugd_maxlen;
708		}
709		ugd->ugd_actlen = size;
710		ugd->ugd_offset = 0;
711
712		error = copyout(ptr, ugd->ugd_data, size);
713	}
714	return (error);
715}
716
717/*------------------------------------------------------------------------*
718 *	ugen_get_iface_driver
719 *
720 * This function generates an USB interface description for userland.
721 *
722 * Returns:
723 *    0: Success
724 * Else: Failure
725 *------------------------------------------------------------------------*/
726static int
727ugen_get_iface_driver(struct usb2_fifo *f, struct usb2_gen_descriptor *ugd)
728{
729	struct usb2_device *udev = f->udev;
730	struct usb2_interface *iface;
731	const char *ptr;
732	const char *desc;
733	unsigned int len;
734	unsigned int maxlen;
735	char buf[128];
736	int error;
737
738	DPRINTFN(6, "\n");
739
740	if ((ugd->ugd_data == NULL) || (ugd->ugd_maxlen == 0)) {
741		/* userland pointer should not be zero */
742		return (EINVAL);
743	}
744
745	iface = usb2_get_iface(udev, ugd->ugd_iface_index);
746	if ((iface == NULL) || (iface->idesc == NULL)) {
747		/* invalid interface index */
748		return (EINVAL);
749	}
750
751	/* read out device nameunit string, if any */
752	if ((iface->subdev != NULL) &&
753	    device_is_attached(iface->subdev) &&
754	    (ptr = device_get_nameunit(iface->subdev)) &&
755	    (desc = device_get_desc(iface->subdev))) {
756
757		/* print description */
758		snprintf(buf, sizeof(buf), "%s: <%s>", ptr, desc);
759
760		/* range checks */
761		maxlen = ugd->ugd_maxlen - 1;
762		len = strlen(buf);
763		if (len > maxlen)
764			len = maxlen;
765
766		/* update actual length, including terminating zero */
767		ugd->ugd_actlen = len + 1;
768
769		/* copy out interface description */
770		error = copyout(buf, ugd->ugd_data, ugd->ugd_actlen);
771	} else {
772		/* zero length string is default */
773		error = copyout("", ugd->ugd_data, 1);
774	}
775	return (error);
776}
777
778/*------------------------------------------------------------------------*
779 *	usb2_gen_fill_deviceinfo
780 *
781 * This function dumps information about an USB device to the
782 * structure pointed to by the "di" argument.
783 *
784 * Returns:
785 *    0: Success
786 * Else: Failure
787 *------------------------------------------------------------------------*/
788static int
789usb2_gen_fill_deviceinfo(struct usb2_fifo *f, struct usb2_device_info *di)
790{
791	struct usb2_device *udev;
792	struct usb2_device *hub;
793
794	udev = f->udev;
795
796	bzero(di, sizeof(di[0]));
797
798	di->udi_bus = device_get_unit(udev->bus->bdev);
799	di->udi_addr = udev->address;
800	di->udi_index = udev->device_index;
801	strlcpy(di->udi_serial, udev->serial,
802	    sizeof(di->udi_serial));
803	strlcpy(di->udi_vendor, udev->manufacturer,
804	    sizeof(di->udi_vendor));
805	strlcpy(di->udi_product, udev->product,
806	    sizeof(di->udi_product));
807	usb2_printBCD(di->udi_release, sizeof(di->udi_release),
808	    UGETW(udev->ddesc.bcdDevice));
809	di->udi_vendorNo = UGETW(udev->ddesc.idVendor);
810	di->udi_productNo = UGETW(udev->ddesc.idProduct);
811	di->udi_releaseNo = UGETW(udev->ddesc.bcdDevice);
812	di->udi_class = udev->ddesc.bDeviceClass;
813	di->udi_subclass = udev->ddesc.bDeviceSubClass;
814	di->udi_protocol = udev->ddesc.bDeviceProtocol;
815	di->udi_config_no = udev->curr_config_no;
816	di->udi_config_index = udev->curr_config_index;
817	di->udi_power = udev->flags.self_powered ? 0 : udev->power;
818	di->udi_speed = udev->speed;
819	di->udi_mode = udev->flags.usb2_mode;
820	di->udi_power_mode = udev->power_mode;
821	if (udev->flags.suspended) {
822		di->udi_suspended = 1;
823	} else {
824		di->udi_suspended = 0;
825	}
826
827	hub = udev->parent_hub;
828	if (hub) {
829		di->udi_hubaddr = hub->address;
830		di->udi_hubindex = hub->device_index;
831		di->udi_hubport = udev->port_no;
832	}
833	return (0);
834}
835
836/*------------------------------------------------------------------------*
837 *	ugen_check_request
838 *
839 * Return values:
840 * 0: Access allowed
841 * Else: No access
842 *------------------------------------------------------------------------*/
843static int
844ugen_check_request(struct usb2_device *udev, struct usb2_device_request *req)
845{
846	struct usb2_pipe *pipe;
847	int error;
848
849	/*
850	 * Avoid requests that would damage the bus integrity:
851	 */
852	if (((req->bmRequestType == UT_WRITE_DEVICE) &&
853	    (req->bRequest == UR_SET_ADDRESS)) ||
854	    ((req->bmRequestType == UT_WRITE_DEVICE) &&
855	    (req->bRequest == UR_SET_CONFIG)) ||
856	    ((req->bmRequestType == UT_WRITE_INTERFACE) &&
857	    (req->bRequest == UR_SET_INTERFACE))) {
858		/*
859		 * These requests can be useful for testing USB drivers.
860		 */
861		error = priv_check(curthread, PRIV_DRIVER);
862		if (error) {
863			return (error);
864		}
865	}
866	/*
867	 * Special case - handle clearing of stall
868	 */
869	if (req->bmRequestType == UT_WRITE_ENDPOINT) {
870
871		pipe = usb2_get_pipe_by_addr(udev, req->wIndex[0]);
872		if (pipe == NULL) {
873			return (EINVAL);
874		}
875		if (usb2_check_thread_perm(udev, curthread, FREAD | FWRITE,
876		    pipe->iface_index, req->wIndex[0] & UE_ADDR)) {
877			return (EPERM);
878		}
879		if ((req->bRequest == UR_CLEAR_FEATURE) &&
880		    (UGETW(req->wValue) == UF_ENDPOINT_HALT)) {
881			usb2_clear_data_toggle(udev, pipe);
882		}
883	}
884	/* TODO: add more checks to verify the interface index */
885
886	return (0);
887}
888
889int
890ugen_do_request(struct usb2_fifo *f, struct usb2_ctl_request *ur)
891{
892	int error;
893	uint16_t len;
894	uint16_t actlen;
895
896	if (ugen_check_request(f->udev, &ur->ucr_request)) {
897		return (EPERM);
898	}
899	len = UGETW(ur->ucr_request.wLength);
900
901	/* check if "ucr_data" is valid */
902	if (len != 0) {
903		if (ur->ucr_data == NULL) {
904			return (EFAULT);
905		}
906	}
907	/* do the USB request */
908	error = usb2_do_request_flags
909	    (f->udev, NULL, &ur->ucr_request, ur->ucr_data,
910	    (ur->ucr_flags & USB_SHORT_XFER_OK) |
911	    USB_USER_DATA_PTR, &actlen,
912	    USB_DEFAULT_TIMEOUT);
913
914	ur->ucr_actlen = actlen;
915
916	if (error) {
917		error = EIO;
918	}
919	return (error);
920}
921
922/*------------------------------------------------------------------------
923 *	ugen_re_enumerate
924 *------------------------------------------------------------------------*/
925static int
926ugen_re_enumerate(struct usb2_fifo *f)
927{
928	struct usb2_device *udev = f->udev;
929	int error;
930
931	/*
932	 * This request can be useful for testing USB drivers:
933	 */
934	error = priv_check(curthread, PRIV_DRIVER);
935	if (error) {
936		return (error);
937	}
938	/* get the device unconfigured */
939	error = ugen_set_config(f, USB_UNCONFIG_INDEX);
940	if (error) {
941		return (error);
942	}
943	/* do a bus-reset */
944	mtx_lock(f->priv_mtx);
945	error = usb2_req_re_enumerate(udev, f->priv_mtx);
946	mtx_unlock(f->priv_mtx);
947
948	if (error) {
949		return (ENXIO);
950	}
951	/* restore configuration to index 0 */
952	error = ugen_set_config(f, 0);
953	if (error) {
954		return (error);
955	}
956	return (0);
957}
958
959int
960ugen_fs_uninit(struct usb2_fifo *f)
961{
962	if (f->fs_xfer == NULL) {
963		return (EINVAL);
964	}
965	usb2_transfer_unsetup(f->fs_xfer, f->fs_ep_max);
966	free(f->fs_xfer, M_USB);
967	f->fs_xfer = NULL;
968	f->fs_ep_max = 0;
969	f->fs_ep_ptr = NULL;
970	f->flag_iscomplete = 0;
971	usb2_fifo_free_buffer(f);
972	return (0);
973}
974
975static uint8_t
976ugen_fs_get_complete(struct usb2_fifo *f, uint8_t *pindex)
977{
978	struct usb2_mbuf *m;
979
980	USB_IF_DEQUEUE(&f->used_q, m);
981
982	if (m) {
983		*pindex = *((uint8_t *)(m->cur_data_ptr));
984
985		USB_IF_ENQUEUE(&f->free_q, m);
986
987		return (0);		/* success */
988	} else {
989
990		*pindex = 0;		/* fix compiler warning */
991
992		f->flag_iscomplete = 0;
993	}
994	return (1);			/* failure */
995}
996
997static void
998ugen_fs_set_complete(struct usb2_fifo *f, uint8_t index)
999{
1000	struct usb2_mbuf *m;
1001
1002	USB_IF_DEQUEUE(&f->free_q, m);
1003
1004	if (m == NULL) {
1005		/* can happen during close */
1006		DPRINTF("out of buffers\n");
1007		return;
1008	}
1009	USB_MBUF_RESET(m);
1010
1011	*((uint8_t *)(m->cur_data_ptr)) = index;
1012
1013	USB_IF_ENQUEUE(&f->used_q, m);
1014
1015	f->flag_iscomplete = 1;
1016
1017	usb2_fifo_wakeup(f);
1018}
1019
1020static int
1021ugen_fs_copy_in(struct usb2_fifo *f, uint8_t ep_index)
1022{
1023	struct usb2_device_request *req;
1024	struct usb2_xfer *xfer;
1025	struct usb2_fs_endpoint fs_ep;
1026	void *uaddr;			/* userland pointer */
1027	void *kaddr;
1028	uint32_t offset;
1029	uint32_t length;
1030	uint32_t n;
1031	uint32_t rem;
1032	int error;
1033	uint8_t isread;
1034
1035	if (ep_index >= f->fs_ep_max) {
1036		return (EINVAL);
1037	}
1038	xfer = f->fs_xfer[ep_index];
1039	if (xfer == NULL) {
1040		return (EINVAL);
1041	}
1042	mtx_lock(f->priv_mtx);
1043	if (usb2_transfer_pending(xfer)) {
1044		mtx_unlock(f->priv_mtx);
1045		return (EBUSY);		/* should not happen */
1046	}
1047	mtx_unlock(f->priv_mtx);
1048
1049	error = copyin(f->fs_ep_ptr +
1050	    ep_index, &fs_ep, sizeof(fs_ep));
1051	if (error) {
1052		return (error);
1053	}
1054	/* security checks */
1055
1056	if (fs_ep.nFrames > xfer->max_frame_count) {
1057		xfer->error = USB_ERR_INVAL;
1058		goto complete;
1059	}
1060	if (fs_ep.nFrames == 0) {
1061		xfer->error = USB_ERR_INVAL;
1062		goto complete;
1063	}
1064	error = copyin(fs_ep.ppBuffer,
1065	    &uaddr, sizeof(uaddr));
1066	if (error) {
1067		return (error);
1068	}
1069	/* reset first frame */
1070	usb2_set_frame_offset(xfer, 0, 0);
1071
1072	if (xfer->flags_int.control_xfr) {
1073
1074		req = xfer->frbuffers[0].buffer;
1075
1076		error = copyin(fs_ep.pLength,
1077		    &length, sizeof(length));
1078		if (error) {
1079			return (error);
1080		}
1081		if (length >= sizeof(*req)) {
1082			xfer->error = USB_ERR_INVAL;
1083			goto complete;
1084		}
1085		if (length != 0) {
1086			error = copyin(uaddr, req, length);
1087			if (error) {
1088				return (error);
1089			}
1090		}
1091		if (ugen_check_request(f->udev, req)) {
1092			xfer->error = USB_ERR_INVAL;
1093			goto complete;
1094		}
1095		xfer->frlengths[0] = length;
1096
1097		/* Host mode only ! */
1098		if ((req->bmRequestType &
1099		    (UT_READ | UT_WRITE)) == UT_READ) {
1100			isread = 1;
1101		} else {
1102			isread = 0;
1103		}
1104		n = 1;
1105		offset = sizeof(*req);
1106
1107	} else {
1108		/* Device and Host mode */
1109		if (USB_GET_DATA_ISREAD(xfer)) {
1110			isread = 1;
1111		} else {
1112			isread = 0;
1113		}
1114		n = 0;
1115		offset = 0;
1116	}
1117
1118	rem = xfer->max_data_length;
1119	xfer->nframes = fs_ep.nFrames;
1120	xfer->timeout = fs_ep.timeout;
1121	if (xfer->timeout > 65535) {
1122		xfer->timeout = 65535;
1123	}
1124	if (fs_ep.flags & USB_FS_FLAG_SINGLE_SHORT_OK)
1125		xfer->flags.short_xfer_ok = 1;
1126	else
1127		xfer->flags.short_xfer_ok = 0;
1128
1129	if (fs_ep.flags & USB_FS_FLAG_MULTI_SHORT_OK)
1130		xfer->flags.short_frames_ok = 1;
1131	else
1132		xfer->flags.short_frames_ok = 0;
1133
1134	if (fs_ep.flags & USB_FS_FLAG_FORCE_SHORT)
1135		xfer->flags.force_short_xfer = 1;
1136	else
1137		xfer->flags.force_short_xfer = 0;
1138
1139	if (fs_ep.flags & USB_FS_FLAG_CLEAR_STALL)
1140		xfer->flags.stall_pipe = 1;
1141	else
1142		xfer->flags.stall_pipe = 0;
1143
1144	for (; n != xfer->nframes; n++) {
1145
1146		error = copyin(fs_ep.pLength + n,
1147		    &length, sizeof(length));
1148		if (error) {
1149			break;
1150		}
1151		xfer->frlengths[n] = length;
1152
1153		if (length > rem) {
1154			xfer->error = USB_ERR_INVAL;
1155			goto complete;
1156		}
1157		rem -= length;
1158
1159		if (!isread) {
1160
1161			/* we need to know the source buffer */
1162			error = copyin(fs_ep.ppBuffer + n,
1163			    &uaddr, sizeof(uaddr));
1164			if (error) {
1165				break;
1166			}
1167			if (xfer->flags_int.isochronous_xfr) {
1168				/* get kernel buffer address */
1169				kaddr = xfer->frbuffers[0].buffer;
1170				kaddr = USB_ADD_BYTES(kaddr, offset);
1171			} else {
1172				/* set current frame offset */
1173				usb2_set_frame_offset(xfer, offset, n);
1174
1175				/* get kernel buffer address */
1176				kaddr = xfer->frbuffers[n].buffer;
1177			}
1178
1179			/* move data */
1180			error = copyin(uaddr, kaddr, length);
1181			if (error) {
1182				break;
1183			}
1184		}
1185		offset += length;
1186	}
1187	return (error);
1188
1189complete:
1190	mtx_lock(f->priv_mtx);
1191	ugen_fs_set_complete(f, ep_index);
1192	mtx_unlock(f->priv_mtx);
1193	return (0);
1194}
1195
1196static int
1197ugen_fs_copy_out(struct usb2_fifo *f, uint8_t ep_index)
1198{
1199	struct usb2_device_request *req;
1200	struct usb2_xfer *xfer;
1201	struct usb2_fs_endpoint fs_ep;
1202	struct usb2_fs_endpoint *fs_ep_uptr;	/* userland ptr */
1203	void *uaddr;			/* userland ptr */
1204	void *kaddr;
1205	uint32_t offset;
1206	uint32_t length;
1207	uint32_t temp;
1208	uint32_t n;
1209	uint32_t rem;
1210	int error;
1211	uint8_t isread;
1212
1213	if (ep_index >= f->fs_ep_max) {
1214		return (EINVAL);
1215	}
1216	xfer = f->fs_xfer[ep_index];
1217	if (xfer == NULL) {
1218		return (EINVAL);
1219	}
1220	mtx_lock(f->priv_mtx);
1221	if (usb2_transfer_pending(xfer)) {
1222		mtx_unlock(f->priv_mtx);
1223		return (EBUSY);		/* should not happen */
1224	}
1225	mtx_unlock(f->priv_mtx);
1226
1227	fs_ep_uptr = f->fs_ep_ptr + ep_index;
1228	error = copyin(fs_ep_uptr, &fs_ep, sizeof(fs_ep));
1229	if (error) {
1230		return (error);
1231	}
1232	fs_ep.status = xfer->error;
1233	fs_ep.aFrames = xfer->aframes;
1234	fs_ep.isoc_time_complete = xfer->isoc_time_complete;
1235	if (xfer->error) {
1236		goto complete;
1237	}
1238	if (xfer->flags_int.control_xfr) {
1239		req = xfer->frbuffers[0].buffer;
1240
1241		/* Host mode only ! */
1242		if ((req->bmRequestType & (UT_READ | UT_WRITE)) == UT_READ) {
1243			isread = 1;
1244		} else {
1245			isread = 0;
1246		}
1247		if (xfer->nframes == 0)
1248			n = 0;		/* should never happen */
1249		else
1250			n = 1;
1251	} else {
1252		/* Device and Host mode */
1253		if (USB_GET_DATA_ISREAD(xfer)) {
1254			isread = 1;
1255		} else {
1256			isread = 0;
1257		}
1258		n = 0;
1259	}
1260
1261	/* Update lengths and copy out data */
1262
1263	rem = xfer->max_data_length;
1264	offset = 0;
1265
1266	for (; n != xfer->nframes; n++) {
1267
1268		/* get initial length into "temp" */
1269		error = copyin(fs_ep.pLength + n,
1270		    &temp, sizeof(temp));
1271		if (error) {
1272			return (error);
1273		}
1274		if (temp > rem) {
1275			/* the userland length has been corrupted */
1276			DPRINTF("corrupt userland length "
1277			    "%u > %u\n", temp, rem);
1278			fs_ep.status = USB_ERR_INVAL;
1279			goto complete;
1280		}
1281		rem -= temp;
1282
1283		/* get actual transfer length */
1284		length = xfer->frlengths[n];
1285		if (length > temp) {
1286			/* data overflow */
1287			fs_ep.status = USB_ERR_INVAL;
1288			DPRINTF("data overflow %u > %u\n",
1289			    length, temp);
1290			goto complete;
1291		}
1292		if (isread) {
1293
1294			/* we need to know the destination buffer */
1295			error = copyin(fs_ep.ppBuffer + n,
1296			    &uaddr, sizeof(uaddr));
1297			if (error) {
1298				return (error);
1299			}
1300			if (xfer->flags_int.isochronous_xfr) {
1301				/* only one frame buffer */
1302				kaddr = USB_ADD_BYTES(
1303				    xfer->frbuffers[0].buffer, offset);
1304			} else {
1305				/* multiple frame buffers */
1306				kaddr = xfer->frbuffers[n].buffer;
1307			}
1308
1309			/* move data */
1310			error = copyout(kaddr, uaddr, length);
1311			if (error) {
1312				return (error);
1313			}
1314		}
1315		/*
1316		 * Update offset according to initial length, which is
1317		 * needed by isochronous transfers!
1318		 */
1319		offset += temp;
1320
1321		/* update length */
1322		error = copyout(&length,
1323		    fs_ep.pLength + n, sizeof(length));
1324		if (error) {
1325			return (error);
1326		}
1327	}
1328
1329complete:
1330	/* update "aFrames" */
1331	error = copyout(&fs_ep.aFrames, &fs_ep_uptr->aFrames,
1332	    sizeof(fs_ep.aFrames));
1333	if (error)
1334		goto done;
1335
1336	/* update "isoc_time_complete" */
1337	error = copyout(&fs_ep.isoc_time_complete,
1338	    &fs_ep_uptr->isoc_time_complete,
1339	    sizeof(fs_ep.isoc_time_complete));
1340	if (error)
1341		goto done;
1342	/* update "status" */
1343	error = copyout(&fs_ep.status, &fs_ep_uptr->status,
1344	    sizeof(fs_ep.status));
1345done:
1346	return (error);
1347}
1348
1349static uint8_t
1350ugen_fifo_in_use(struct usb2_fifo *f, int fflags)
1351{
1352	struct usb2_fifo *f_rx;
1353	struct usb2_fifo *f_tx;
1354
1355	f_rx = f->udev->fifo[(f->fifo_index & ~1) + USB_FIFO_RX];
1356	f_tx = f->udev->fifo[(f->fifo_index & ~1) + USB_FIFO_TX];
1357
1358	if ((fflags & FREAD) && f_rx &&
1359	    (f_rx->xfer[0] || f_rx->xfer[1])) {
1360		return (1);		/* RX FIFO in use */
1361	}
1362	if ((fflags & FWRITE) && f_tx &&
1363	    (f_tx->xfer[0] || f_tx->xfer[1])) {
1364		return (1);		/* TX FIFO in use */
1365	}
1366	return (0);			/* not in use */
1367}
1368
1369static int
1370ugen_ioctl(struct usb2_fifo *f, u_long cmd, void *addr, int fflags,
1371    struct thread *td)
1372{
1373	struct usb2_config usb2_config[1];
1374	struct usb2_device_request req;
1375	union {
1376		struct usb2_fs_complete *pcomp;
1377		struct usb2_fs_start *pstart;
1378		struct usb2_fs_stop *pstop;
1379		struct usb2_fs_open *popen;
1380		struct usb2_fs_close *pclose;
1381		struct usb2_fs_clear_stall_sync *pstall;
1382		void   *addr;
1383	}     u;
1384	struct usb2_pipe *pipe;
1385	struct usb2_endpoint_descriptor *ed;
1386	int error = 0;
1387	uint8_t iface_index;
1388	uint8_t isread;
1389	uint8_t ep_index;
1390
1391	u.addr = addr;
1392
1393	DPRINTFN(6, "cmd=0x%08lx\n", cmd);
1394
1395	switch (cmd) {
1396	case USB_FS_COMPLETE:
1397		mtx_lock(f->priv_mtx);
1398		error = ugen_fs_get_complete(f, &ep_index);
1399		mtx_unlock(f->priv_mtx);
1400
1401		if (error) {
1402			error = EBUSY;
1403			break;
1404		}
1405		u.pcomp->ep_index = ep_index;
1406		error = ugen_fs_copy_out(f, u.pcomp->ep_index);
1407		break;
1408
1409	case USB_FS_START:
1410		error = ugen_fs_copy_in(f, u.pstart->ep_index);
1411		if (error) {
1412			break;
1413		}
1414		mtx_lock(f->priv_mtx);
1415		usb2_transfer_start(f->fs_xfer[u.pstart->ep_index]);
1416		mtx_unlock(f->priv_mtx);
1417		break;
1418
1419	case USB_FS_STOP:
1420		if (u.pstop->ep_index >= f->fs_ep_max) {
1421			error = EINVAL;
1422			break;
1423		}
1424		mtx_lock(f->priv_mtx);
1425		usb2_transfer_stop(f->fs_xfer[u.pstop->ep_index]);
1426		mtx_unlock(f->priv_mtx);
1427		break;
1428
1429	case USB_FS_OPEN:
1430		if (u.popen->ep_index >= f->fs_ep_max) {
1431			error = EINVAL;
1432			break;
1433		}
1434		if (f->fs_xfer[u.popen->ep_index] != NULL) {
1435			error = EBUSY;
1436			break;
1437		}
1438		if (u.popen->max_bufsize > USB_FS_MAX_BUFSIZE) {
1439			u.popen->max_bufsize = USB_FS_MAX_BUFSIZE;
1440		}
1441		if (u.popen->max_frames > USB_FS_MAX_FRAMES) {
1442			u.popen->max_frames = USB_FS_MAX_FRAMES;
1443			break;
1444		}
1445		if (u.popen->max_frames == 0) {
1446			error = EINVAL;
1447			break;
1448		}
1449		pipe = usb2_get_pipe_by_addr(f->udev, u.popen->ep_no);
1450		if (pipe == NULL) {
1451			error = EINVAL;
1452			break;
1453		}
1454		ed = pipe->edesc;
1455		if (ed == NULL) {
1456			error = ENXIO;
1457			break;
1458		}
1459		iface_index = pipe->iface_index;
1460
1461		error = usb2_check_thread_perm(f->udev, curthread, fflags,
1462		    iface_index, u.popen->ep_no);
1463		if (error) {
1464			break;
1465		}
1466		bzero(usb2_config, sizeof(usb2_config));
1467
1468		usb2_config[0].type = ed->bmAttributes & UE_XFERTYPE;
1469		usb2_config[0].endpoint = ed->bEndpointAddress & UE_ADDR;
1470		usb2_config[0].direction = ed->bEndpointAddress & (UE_DIR_OUT | UE_DIR_IN);
1471		usb2_config[0].mh.interval = USB_DEFAULT_INTERVAL;
1472		usb2_config[0].mh.flags.proxy_buffer = 1;
1473		usb2_config[0].mh.callback = &ugen_default_fs_callback;
1474		usb2_config[0].mh.timeout = 0;	/* no timeout */
1475		usb2_config[0].mh.frames = u.popen->max_frames;
1476		usb2_config[0].mh.bufsize = u.popen->max_bufsize;
1477		usb2_config[0].md = usb2_config[0].mh;	/* symmetric config */
1478
1479		if (usb2_config[0].type == UE_CONTROL) {
1480			if (f->udev->flags.usb2_mode != USB_MODE_HOST) {
1481				error = EINVAL;
1482				break;
1483			}
1484		} else {
1485
1486			isread = ((usb2_config[0].endpoint &
1487			    (UE_DIR_IN | UE_DIR_OUT)) == UE_DIR_IN);
1488
1489			if (f->udev->flags.usb2_mode != USB_MODE_HOST) {
1490				isread = !isread;
1491			}
1492			/* check permissions */
1493			if (isread) {
1494				if (!(fflags & FREAD)) {
1495					error = EPERM;
1496					break;
1497				}
1498			} else {
1499				if (!(fflags & FWRITE)) {
1500					error = EPERM;
1501					break;
1502				}
1503			}
1504		}
1505		error = usb2_transfer_setup(f->udev, &iface_index,
1506		    f->fs_xfer + u.popen->ep_index, usb2_config, 1,
1507		    f, f->priv_mtx);
1508		if (error == 0) {
1509			/* update maximums */
1510			u.popen->max_packet_length =
1511			    f->fs_xfer[u.popen->ep_index]->max_frame_size;
1512			u.popen->max_bufsize =
1513			    f->fs_xfer[u.popen->ep_index]->max_data_length;
1514			f->fs_xfer[u.popen->ep_index]->priv_fifo =
1515			    ((uint8_t *)0) + u.popen->ep_index;
1516		} else {
1517			error = ENOMEM;
1518		}
1519		break;
1520
1521	case USB_FS_CLOSE:
1522		if (u.pclose->ep_index >= f->fs_ep_max) {
1523			error = EINVAL;
1524			break;
1525		}
1526		if (f->fs_xfer[u.pclose->ep_index] == NULL) {
1527			error = EINVAL;
1528			break;
1529		}
1530		usb2_transfer_unsetup(f->fs_xfer + u.pclose->ep_index, 1);
1531		break;
1532
1533	case USB_FS_CLEAR_STALL_SYNC:
1534		if (u.pstall->ep_index >= f->fs_ep_max) {
1535			error = EINVAL;
1536			break;
1537		}
1538		if (f->fs_xfer[u.pstall->ep_index] == NULL) {
1539			error = EINVAL;
1540			break;
1541		}
1542		if (f->udev->flags.usb2_mode != USB_MODE_HOST) {
1543			error = EINVAL;
1544			break;
1545		}
1546		mtx_lock(f->priv_mtx);
1547		error = usb2_transfer_pending(f->fs_xfer[u.pstall->ep_index]);
1548		mtx_unlock(f->priv_mtx);
1549
1550		if (error) {
1551			return (EBUSY);
1552		}
1553		pipe = f->fs_xfer[u.pstall->ep_index]->pipe;
1554
1555		/* setup a clear-stall packet */
1556		req.bmRequestType = UT_WRITE_ENDPOINT;
1557		req.bRequest = UR_CLEAR_FEATURE;
1558		USETW(req.wValue, UF_ENDPOINT_HALT);
1559		req.wIndex[0] = pipe->edesc->bEndpointAddress;
1560		req.wIndex[1] = 0;
1561		USETW(req.wLength, 0);
1562
1563		error = usb2_do_request(f->udev, NULL, &req, NULL);
1564		if (error == 0) {
1565			usb2_clear_data_toggle(f->udev, pipe);
1566		} else {
1567			error = ENXIO;
1568		}
1569		break;
1570
1571	default:
1572		error = ENOIOCTL;
1573		break;
1574	}
1575
1576	DPRINTFN(6, "error=%d\n", error);
1577
1578	return (error);
1579}
1580
1581static int
1582ugen_set_short_xfer(struct usb2_fifo *f, void *addr)
1583{
1584	uint8_t t;
1585
1586	if (*(int *)addr)
1587		t = 1;
1588	else
1589		t = 0;
1590
1591	if (f->flag_short == t) {
1592		/* same value like before - accept */
1593		return (0);
1594	}
1595	if (f->xfer[0] || f->xfer[1]) {
1596		/* cannot change this during transfer */
1597		return (EBUSY);
1598	}
1599	f->flag_short = t;
1600	return (0);
1601}
1602
1603static int
1604ugen_set_timeout(struct usb2_fifo *f, void *addr)
1605{
1606	f->timeout = *(int *)addr;
1607	if (f->timeout > 65535) {
1608		/* limit user input */
1609		f->timeout = 65535;
1610	}
1611	return (0);
1612}
1613
1614static int
1615ugen_get_frame_size(struct usb2_fifo *f, void *addr)
1616{
1617	if (f->xfer[0]) {
1618		*(int *)addr = f->xfer[0]->max_frame_size;
1619	} else {
1620		return (EINVAL);
1621	}
1622	return (0);
1623}
1624
1625static int
1626ugen_set_buffer_size(struct usb2_fifo *f, void *addr)
1627{
1628	uint32_t t;
1629
1630	if (*(int *)addr < 1024)
1631		t = 1024;
1632	else if (*(int *)addr < (256 * 1024))
1633		t = *(int *)addr;
1634	else
1635		t = 256 * 1024;
1636
1637	if (f->bufsize == t) {
1638		/* same value like before - accept */
1639		return (0);
1640	}
1641	if (f->xfer[0] || f->xfer[1]) {
1642		/* cannot change this during transfer */
1643		return (EBUSY);
1644	}
1645	f->bufsize = t;
1646	return (0);
1647}
1648
1649static int
1650ugen_get_buffer_size(struct usb2_fifo *f, void *addr)
1651{
1652	*(int *)addr = f->bufsize;
1653	return (0);
1654}
1655
1656static int
1657ugen_get_iface_desc(struct usb2_fifo *f,
1658    struct usb2_interface_descriptor *idesc)
1659{
1660	struct usb2_interface *iface;
1661
1662	iface = usb2_get_iface(f->udev, f->iface_index);
1663	if (iface && iface->idesc) {
1664		*idesc = *(iface->idesc);
1665	} else {
1666		return (EIO);
1667	}
1668	return (0);
1669}
1670
1671static int
1672ugen_get_endpoint_desc(struct usb2_fifo *f,
1673    struct usb2_endpoint_descriptor *ed)
1674{
1675	struct usb2_pipe *pipe;
1676
1677	pipe = f->priv_sc0;
1678
1679	if (pipe && pipe->edesc) {
1680		*ed = *pipe->edesc;
1681	} else {
1682		return (EINVAL);
1683	}
1684	return (0);
1685}
1686
1687static int
1688ugen_set_power_mode(struct usb2_fifo *f, int mode)
1689{
1690	struct usb2_device *udev = f->udev;
1691	int err;
1692	uint8_t old_mode;
1693
1694	if ((udev == NULL) ||
1695	    (udev->parent_hub == NULL)) {
1696		return (EINVAL);
1697	}
1698	err = priv_check(curthread, PRIV_ROOT);
1699	if (err)
1700		return (err);
1701
1702	/* get old power mode */
1703	old_mode = udev->power_mode;
1704
1705	/* if no change, then just return */
1706	if (old_mode == mode)
1707		return (0);
1708
1709	switch (mode) {
1710	case USB_POWER_MODE_OFF:
1711		/* get the device unconfigured */
1712		err = ugen_set_config(f, USB_UNCONFIG_INDEX);
1713		if (err) {
1714			DPRINTFN(0, "Could not unconfigure "
1715			    "device (ignored)\n");
1716		}
1717
1718		/* clear port enable */
1719		err = usb2_req_clear_port_feature(udev->parent_hub,
1720		    NULL, udev->port_no, UHF_PORT_ENABLE);
1721		break;
1722
1723	case USB_POWER_MODE_ON:
1724	case USB_POWER_MODE_SAVE:
1725		break;
1726
1727	case USB_POWER_MODE_RESUME:
1728		err = usb2_req_clear_port_feature(udev->parent_hub,
1729		    NULL, udev->port_no, UHF_PORT_SUSPEND);
1730		mode = USB_POWER_MODE_SAVE;
1731		break;
1732
1733	case USB_POWER_MODE_SUSPEND:
1734		err = usb2_req_set_port_feature(udev->parent_hub,
1735		    NULL, udev->port_no, UHF_PORT_SUSPEND);
1736		mode = USB_POWER_MODE_SAVE;
1737		break;
1738
1739	default:
1740		return (EINVAL);
1741	}
1742
1743	if (err)
1744		return (ENXIO);		/* I/O failure */
1745
1746	/* if we are powered off we need to re-enumerate first */
1747	if (old_mode == USB_POWER_MODE_OFF) {
1748		err = ugen_re_enumerate(f);
1749		if (err)
1750			return (err);
1751	}
1752
1753	/* set new power mode */
1754	usb2_set_power_mode(udev, mode);
1755
1756	return (0);			/* success */
1757}
1758
1759static int
1760ugen_get_power_mode(struct usb2_fifo *f)
1761{
1762	struct usb2_device *udev = f->udev;
1763
1764	if ((udev == NULL) ||
1765	    (udev->parent_hub == NULL)) {
1766		return (USB_POWER_MODE_ON);
1767	}
1768	return (udev->power_mode);
1769}
1770
1771static int
1772ugen_do_port_feature(struct usb2_fifo *f, uint8_t port_no,
1773    uint8_t set, uint16_t feature)
1774{
1775	struct usb2_device *udev = f->udev;
1776	struct usb2_hub *hub;
1777	int err;
1778
1779	err = priv_check(curthread, PRIV_ROOT);
1780	if (err) {
1781		return (err);
1782	}
1783	if (port_no == 0) {
1784		return (EINVAL);
1785	}
1786	if ((udev == NULL) ||
1787	    (udev->hub == NULL)) {
1788		return (EINVAL);
1789	}
1790	hub = udev->hub;
1791
1792	if (port_no > hub->nports) {
1793		return (EINVAL);
1794	}
1795	if (set)
1796		err = usb2_req_set_port_feature(udev,
1797		    NULL, port_no, feature);
1798	else
1799		err = usb2_req_clear_port_feature(udev,
1800		    NULL, port_no, feature);
1801
1802	if (err)
1803		return (ENXIO);		/* failure */
1804
1805	return (0);			/* success */
1806}
1807
1808static int
1809ugen_iface_ioctl(struct usb2_fifo *f, u_long cmd, void *addr, int fflags)
1810{
1811	struct usb2_fifo *f_rx;
1812	struct usb2_fifo *f_tx;
1813	int error = 0;
1814
1815	f_rx = f->udev->fifo[(f->fifo_index & ~1) + USB_FIFO_RX];
1816	f_tx = f->udev->fifo[(f->fifo_index & ~1) + USB_FIFO_TX];
1817
1818	switch (cmd) {
1819	case USB_SET_RX_SHORT_XFER:
1820		if (fflags & FREAD) {
1821			error = ugen_set_short_xfer(f_rx, addr);
1822		} else {
1823			error = EINVAL;
1824		}
1825		break;
1826
1827	case USB_SET_TX_FORCE_SHORT:
1828		if (fflags & FWRITE) {
1829			error = ugen_set_short_xfer(f_tx, addr);
1830		} else {
1831			error = EINVAL;
1832		}
1833		break;
1834
1835	case USB_SET_RX_TIMEOUT:
1836		if (fflags & FREAD) {
1837			error = ugen_set_timeout(f_rx, addr);
1838		} else {
1839			error = EINVAL;
1840		}
1841		break;
1842
1843	case USB_SET_TX_TIMEOUT:
1844		if (fflags & FWRITE) {
1845			error = ugen_set_timeout(f_tx, addr);
1846		} else {
1847			error = EINVAL;
1848		}
1849		break;
1850
1851	case USB_GET_RX_FRAME_SIZE:
1852		if (fflags & FREAD) {
1853			error = ugen_get_frame_size(f_rx, addr);
1854		} else {
1855			error = EINVAL;
1856		}
1857		break;
1858
1859	case USB_GET_TX_FRAME_SIZE:
1860		if (fflags & FWRITE) {
1861			error = ugen_get_frame_size(f_tx, addr);
1862		} else {
1863			error = EINVAL;
1864		}
1865		break;
1866
1867	case USB_SET_RX_BUFFER_SIZE:
1868		if (fflags & FREAD) {
1869			error = ugen_set_buffer_size(f_rx, addr);
1870		} else {
1871			error = EINVAL;
1872		}
1873		break;
1874
1875	case USB_SET_TX_BUFFER_SIZE:
1876		if (fflags & FWRITE) {
1877			error = ugen_set_buffer_size(f_tx, addr);
1878		} else {
1879			error = EINVAL;
1880		}
1881		break;
1882
1883	case USB_GET_RX_BUFFER_SIZE:
1884		if (fflags & FREAD) {
1885			error = ugen_get_buffer_size(f_rx, addr);
1886		} else {
1887			error = EINVAL;
1888		}
1889		break;
1890
1891	case USB_GET_TX_BUFFER_SIZE:
1892		if (fflags & FWRITE) {
1893			error = ugen_get_buffer_size(f_tx, addr);
1894		} else {
1895			error = EINVAL;
1896		}
1897		break;
1898
1899	case USB_GET_RX_INTERFACE_DESC:
1900		if (fflags & FREAD) {
1901			error = ugen_get_iface_desc(f_rx, addr);
1902		} else {
1903			error = EINVAL;
1904		}
1905		break;
1906
1907	case USB_GET_TX_INTERFACE_DESC:
1908		if (fflags & FWRITE) {
1909			error = ugen_get_iface_desc(f_tx, addr);
1910		} else {
1911			error = EINVAL;
1912		}
1913		break;
1914
1915	case USB_GET_RX_ENDPOINT_DESC:
1916		if (fflags & FREAD) {
1917			error = ugen_get_endpoint_desc(f_rx, addr);
1918		} else {
1919			error = EINVAL;
1920		}
1921		break;
1922
1923	case USB_GET_TX_ENDPOINT_DESC:
1924		if (fflags & FWRITE) {
1925			error = ugen_get_endpoint_desc(f_tx, addr);
1926		} else {
1927			error = EINVAL;
1928		}
1929		break;
1930
1931	case USB_SET_RX_STALL_FLAG:
1932		if ((fflags & FREAD) && (*(int *)addr)) {
1933			f_rx->flag_stall = 1;
1934		}
1935		break;
1936
1937	case USB_SET_TX_STALL_FLAG:
1938		if ((fflags & FWRITE) && (*(int *)addr)) {
1939			f_tx->flag_stall = 1;
1940		}
1941		break;
1942
1943	default:
1944		error = ENOIOCTL;
1945		break;
1946	}
1947	return (error);
1948}
1949
1950static int
1951ugen_ioctl_post(struct usb2_fifo *f, u_long cmd, void *addr, int fflags,
1952    struct thread *td)
1953{
1954	union {
1955		struct usb2_interface_descriptor *idesc;
1956		struct usb2_alt_interface *ai;
1957		struct usb2_device_descriptor *ddesc;
1958		struct usb2_config_descriptor *cdesc;
1959		struct usb2_device_stats *stat;
1960		struct usb2_fs_init *pinit;
1961		struct usb2_fs_uninit *puninit;
1962		uint32_t *ptime;
1963		void   *addr;
1964		int    *pint;
1965	}     u;
1966	struct usb2_device_descriptor *dtemp;
1967	struct usb2_config_descriptor *ctemp;
1968	struct usb2_interface *iface;
1969	int error = 0;
1970	uint8_t n;
1971
1972	u.addr = addr;
1973
1974	DPRINTFN(6, "cmd=0x%08lx\n", cmd);
1975
1976	switch (cmd) {
1977	case USB_DISCOVER:
1978		usb2_needs_explore_all();
1979		break;
1980
1981	case USB_SETDEBUG:
1982		if (!(fflags & FWRITE)) {
1983			error = EPERM;
1984			break;
1985		}
1986		usb2_debug = *(int *)addr;
1987		break;
1988
1989	case USB_GET_CONFIG:
1990		*(int *)addr = f->udev->curr_config_index;
1991		break;
1992
1993	case USB_SET_CONFIG:
1994		if (!(fflags & FWRITE)) {
1995			error = EPERM;
1996			break;
1997		}
1998		error = ugen_set_config(f, *(int *)addr);
1999		break;
2000
2001	case USB_GET_ALTINTERFACE:
2002		iface = usb2_get_iface(f->udev,
2003		    u.ai->uai_interface_index);
2004		if (iface && iface->idesc) {
2005			u.ai->uai_alt_index = iface->alt_index;
2006		} else {
2007			error = EINVAL;
2008		}
2009		break;
2010
2011	case USB_SET_ALTINTERFACE:
2012		if (!(fflags & FWRITE)) {
2013			error = EPERM;
2014			break;
2015		}
2016		error = ugen_set_interface(f,
2017		    u.ai->uai_interface_index, u.ai->uai_alt_index);
2018		break;
2019
2020	case USB_GET_DEVICE_DESC:
2021		dtemp = usb2_get_device_descriptor(f->udev);
2022		if (!dtemp) {
2023			error = EIO;
2024			break;
2025		}
2026		*u.ddesc = *dtemp;
2027		break;
2028
2029	case USB_GET_CONFIG_DESC:
2030		ctemp = usb2_get_config_descriptor(f->udev);
2031		if (!ctemp) {
2032			error = EIO;
2033			break;
2034		}
2035		*u.cdesc = *ctemp;
2036		break;
2037
2038	case USB_GET_FULL_DESC:
2039		error = ugen_get_cdesc(f, addr);
2040		break;
2041
2042	case USB_GET_STRING_DESC:
2043		error = ugen_get_sdesc(f, addr);
2044		break;
2045
2046	case USB_GET_IFACE_DRIVER:
2047		error = ugen_get_iface_driver(f, addr);
2048		break;
2049
2050	case USB_REQUEST:
2051	case USB_DO_REQUEST:
2052		if (!(fflags & FWRITE)) {
2053			error = EPERM;
2054			break;
2055		}
2056		error = ugen_do_request(f, addr);
2057		break;
2058
2059	case USB_DEVICEINFO:
2060	case USB_GET_DEVICEINFO:
2061		error = usb2_gen_fill_deviceinfo(f, addr);
2062		break;
2063
2064	case USB_DEVICESTATS:
2065		for (n = 0; n != 4; n++) {
2066
2067			u.stat->uds_requests_fail[n] =
2068			    f->udev->bus->stats_err.uds_requests[n];
2069
2070			u.stat->uds_requests_ok[n] =
2071			    f->udev->bus->stats_ok.uds_requests[n];
2072		}
2073		break;
2074
2075	case USB_DEVICEENUMERATE:
2076		error = ugen_re_enumerate(f);
2077		break;
2078
2079	case USB_GET_PLUGTIME:
2080		*u.ptime = f->udev->plugtime;
2081		break;
2082
2083	case USB_CLAIM_INTERFACE:
2084	case USB_RELEASE_INTERFACE:
2085		/* TODO */
2086		break;
2087
2088	case USB_IFACE_DRIVER_ACTIVE:
2089		/* TODO */
2090		*u.pint = 0;
2091		break;
2092
2093	case USB_IFACE_DRIVER_DETACH:
2094		/* TODO */
2095		error = priv_check(curthread, PRIV_DRIVER);
2096		if (error) {
2097			break;
2098		}
2099		error = EINVAL;
2100		break;
2101
2102	case USB_SET_POWER_MODE:
2103		error = ugen_set_power_mode(f, *u.pint);
2104		break;
2105
2106	case USB_GET_POWER_MODE:
2107		*u.pint = ugen_get_power_mode(f);
2108		break;
2109
2110	case USB_SET_PORT_ENABLE:
2111		error = ugen_do_port_feature(f,
2112		    *u.pint, 1, UHF_PORT_ENABLE);
2113		break;
2114
2115	case USB_SET_PORT_DISABLE:
2116		error = ugen_do_port_feature(f,
2117		    *u.pint, 0, UHF_PORT_ENABLE);
2118		break;
2119
2120	case USB_FS_INIT:
2121		/* verify input parameters */
2122		if (u.pinit->pEndpoints == NULL) {
2123			error = EINVAL;
2124			break;
2125		}
2126		if (u.pinit->ep_index_max > 127) {
2127			error = EINVAL;
2128			break;
2129		}
2130		if (u.pinit->ep_index_max == 0) {
2131			error = EINVAL;
2132			break;
2133		}
2134		if (f->fs_xfer != NULL) {
2135			error = EBUSY;
2136			break;
2137		}
2138		if (f->dev_ep_index != 0) {
2139			error = EINVAL;
2140			break;
2141		}
2142		if (ugen_fifo_in_use(f, fflags)) {
2143			error = EBUSY;
2144			break;
2145		}
2146		error = usb2_fifo_alloc_buffer(f, 1, u.pinit->ep_index_max);
2147		if (error) {
2148			break;
2149		}
2150		f->fs_xfer = malloc(sizeof(f->fs_xfer[0]) *
2151		    u.pinit->ep_index_max, M_USB, M_WAITOK | M_ZERO);
2152		if (f->fs_xfer == NULL) {
2153			usb2_fifo_free_buffer(f);
2154			error = ENOMEM;
2155			break;
2156		}
2157		f->fs_ep_max = u.pinit->ep_index_max;
2158		f->fs_ep_ptr = u.pinit->pEndpoints;
2159		break;
2160
2161	case USB_FS_UNINIT:
2162		if (u.puninit->dummy != 0) {
2163			error = EINVAL;
2164			break;
2165		}
2166		error = ugen_fs_uninit(f);
2167		break;
2168
2169	default:
2170		mtx_lock(f->priv_mtx);
2171		error = ugen_iface_ioctl(f, cmd, addr, fflags);
2172		mtx_unlock(f->priv_mtx);
2173		break;
2174	}
2175	DPRINTFN(6, "error=%d\n", error);
2176	return (error);
2177}
2178
2179static void
2180ugen_default_fs_callback(struct usb2_xfer *xfer)
2181{
2182	;				/* workaround for a bug in "indent" */
2183
2184	DPRINTF("st=%u alen=%u aframes=%u\n",
2185	    USB_GET_STATE(xfer), xfer->actlen, xfer->aframes);
2186
2187	switch (USB_GET_STATE(xfer)) {
2188	case USB_ST_SETUP:
2189		usb2_start_hardware(xfer);
2190		break;
2191	default:
2192		ugen_fs_set_complete(xfer->priv_sc, USB_P2U(xfer->priv_fifo));
2193		break;
2194	}
2195}
2196