1/* $FreeBSD: head/sys/dev/usb/controller/xhci.c 251249 2013-06-02 10:54:47Z hselasky $ */
| 1/* $FreeBSD: head/sys/dev/usb/controller/xhci.c 251251 2013-06-02 11:58:31Z hselasky $ */
|
2/*-
3 * Copyright (c) 2010 Hans Petter Selasky. All rights reserved.
4 *
5 * Redistribution and use in source and binary forms, with or without
6 * modification, are permitted provided that the following conditions
7 * are met:
8 * 1. Redistributions of source code must retain the above copyright
9 * notice, this list of conditions and the following disclaimer.
10 * 2. Redistributions in binary form must reproduce the above copyright
11 * notice, this list of conditions and the following disclaimer in the
12 * documentation and/or other materials provided with the distribution.
13 *
14 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
15 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
16 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
17 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
18 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
19 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
20 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
21 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
22 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
23 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
24 * SUCH DAMAGE.
25 */
26
27/*
28 * USB eXtensible Host Controller Interface, a.k.a. USB 3.0 controller.
29 *
30 * The XHCI 1.0 spec can be found at
31 * http://www.intel.com/technology/usb/download/xHCI_Specification_for_USB.pdf
32 * and the USB 3.0 spec at
33 * http://www.usb.org/developers/docs/usb_30_spec_060910.zip
34 */
35
36/*
37 * A few words about the design implementation: This driver emulates
38 * the concept about TDs which is found in EHCI specification. This
39 * way we achieve that the USB controller drivers look similar to
40 * eachother which makes it easier to understand the code.
41 */
42
43#ifdef USB_GLOBAL_INCLUDE_FILE
44#include USB_GLOBAL_INCLUDE_FILE
45#else
46#include <sys/stdint.h>
47#include <sys/stddef.h>
48#include <sys/param.h>
49#include <sys/queue.h>
50#include <sys/types.h>
51#include <sys/systm.h>
52#include <sys/kernel.h>
53#include <sys/bus.h>
54#include <sys/module.h>
55#include <sys/lock.h>
56#include <sys/mutex.h>
57#include <sys/condvar.h>
58#include <sys/sysctl.h>
59#include <sys/sx.h>
60#include <sys/unistd.h>
61#include <sys/callout.h>
62#include <sys/malloc.h>
63#include <sys/priv.h>
64
65#include <dev/usb/usb.h>
66#include <dev/usb/usbdi.h>
67
68#define USB_DEBUG_VAR xhcidebug
69
70#include <dev/usb/usb_core.h>
71#include <dev/usb/usb_debug.h>
72#include <dev/usb/usb_busdma.h>
73#include <dev/usb/usb_process.h>
74#include <dev/usb/usb_transfer.h>
75#include <dev/usb/usb_device.h>
76#include <dev/usb/usb_hub.h>
77#include <dev/usb/usb_util.h>
78
79#include <dev/usb/usb_controller.h>
80#include <dev/usb/usb_bus.h>
81#endif /* USB_GLOBAL_INCLUDE_FILE */
82
83#include <dev/usb/controller/xhci.h>
84#include <dev/usb/controller/xhcireg.h>
85
86#define XHCI_BUS2SC(bus) \
87 ((struct xhci_softc *)(((uint8_t *)(bus)) - \
88 ((uint8_t *)&(((struct xhci_softc *)0)->sc_bus))))
89
90#ifdef USB_DEBUG
91static int xhcidebug;
92static int xhciroute;
93
94static SYSCTL_NODE(_hw_usb, OID_AUTO, xhci, CTLFLAG_RW, 0, "USB XHCI");
95SYSCTL_INT(_hw_usb_xhci, OID_AUTO, debug, CTLFLAG_RW | CTLFLAG_TUN,
96 &xhcidebug, 0, "Debug level");
97TUNABLE_INT("hw.usb.xhci.debug", &xhcidebug);
98SYSCTL_INT(_hw_usb_xhci, OID_AUTO, xhci_port_route, CTLFLAG_RW | CTLFLAG_TUN,
99 &xhciroute, 0, "Routing bitmap for switching EHCI ports to XHCI controller");
100TUNABLE_INT("hw.usb.xhci.xhci_port_route", &xhciroute);
101#endif
102
103#define XHCI_INTR_ENDPT 1
104
105struct xhci_std_temp {
106 struct xhci_softc *sc;
107 struct usb_page_cache *pc;
108 struct xhci_td *td;
109 struct xhci_td *td_next;
110 uint32_t len;
111 uint32_t offset;
112 uint32_t max_packet_size;
113 uint32_t average;
114 uint16_t isoc_delta;
115 uint16_t isoc_frame;
116 uint8_t shortpkt;
117 uint8_t multishort;
118 uint8_t last_frame;
119 uint8_t trb_type;
120 uint8_t direction;
121 uint8_t tbc;
122 uint8_t tlbpc;
123 uint8_t step_td;
124 uint8_t do_isoc_sync;
125};
126
127static void xhci_do_poll(struct usb_bus *);
128static void xhci_device_done(struct usb_xfer *, usb_error_t);
129static void xhci_root_intr(struct xhci_softc *);
130static void xhci_free_device_ext(struct usb_device *);
131static struct xhci_endpoint_ext *xhci_get_endpoint_ext(struct usb_device *,
132 struct usb_endpoint_descriptor *);
133static usb_proc_callback_t xhci_configure_msg;
134static usb_error_t xhci_configure_device(struct usb_device *);
135static usb_error_t xhci_configure_endpoint(struct usb_device *,
136 struct usb_endpoint_descriptor *, struct xhci_endpoint_ext *,
137 uint16_t, uint8_t, uint8_t, uint8_t, uint16_t, uint16_t,
138 uint8_t);
139static usb_error_t xhci_configure_mask(struct usb_device *,
140 uint32_t, uint8_t);
141static usb_error_t xhci_cmd_evaluate_ctx(struct xhci_softc *,
142 uint64_t, uint8_t);
143static void xhci_endpoint_doorbell(struct usb_xfer *);
144static void xhci_ctx_set_le32(struct xhci_softc *sc, volatile uint32_t *ptr, uint32_t val);
145static uint32_t xhci_ctx_get_le32(struct xhci_softc *sc, volatile uint32_t *ptr);
146static void xhci_ctx_set_le64(struct xhci_softc *sc, volatile uint64_t *ptr, uint64_t val);
147#ifdef USB_DEBUG
148static uint64_t xhci_ctx_get_le64(struct xhci_softc *sc, volatile uint64_t *ptr);
149#endif
150
151extern struct usb_bus_methods xhci_bus_methods;
152
153#ifdef USB_DEBUG
154static void
155xhci_dump_trb(struct xhci_trb *trb)
156{
157 DPRINTFN(5, "trb = %p\n", trb);
158 DPRINTFN(5, "qwTrb0 = 0x%016llx\n", (long long)le64toh(trb->qwTrb0));
159 DPRINTFN(5, "dwTrb2 = 0x%08x\n", le32toh(trb->dwTrb2));
160 DPRINTFN(5, "dwTrb3 = 0x%08x\n", le32toh(trb->dwTrb3));
161}
162
163static void
164xhci_dump_endpoint(struct xhci_softc *sc, struct xhci_endp_ctx *pep)
165{
166 DPRINTFN(5, "pep = %p\n", pep);
167 DPRINTFN(5, "dwEpCtx0=0x%08x\n", xhci_ctx_get_le32(sc, &pep->dwEpCtx0));
168 DPRINTFN(5, "dwEpCtx1=0x%08x\n", xhci_ctx_get_le32(sc, &pep->dwEpCtx1));
169 DPRINTFN(5, "qwEpCtx2=0x%016llx\n", (long long)xhci_ctx_get_le64(sc, &pep->qwEpCtx2));
170 DPRINTFN(5, "dwEpCtx4=0x%08x\n", xhci_ctx_get_le32(sc, &pep->dwEpCtx4));
171 DPRINTFN(5, "dwEpCtx5=0x%08x\n", xhci_ctx_get_le32(sc, &pep->dwEpCtx5));
172 DPRINTFN(5, "dwEpCtx6=0x%08x\n", xhci_ctx_get_le32(sc, &pep->dwEpCtx6));
173 DPRINTFN(5, "dwEpCtx7=0x%08x\n", xhci_ctx_get_le32(sc, &pep->dwEpCtx7));
174}
175
176static void
177xhci_dump_device(struct xhci_softc *sc, struct xhci_slot_ctx *psl)
178{
179 DPRINTFN(5, "psl = %p\n", psl);
180 DPRINTFN(5, "dwSctx0=0x%08x\n", xhci_ctx_get_le32(sc, &psl->dwSctx0));
181 DPRINTFN(5, "dwSctx1=0x%08x\n", xhci_ctx_get_le32(sc, &psl->dwSctx1));
182 DPRINTFN(5, "dwSctx2=0x%08x\n", xhci_ctx_get_le32(sc, &psl->dwSctx2));
183 DPRINTFN(5, "dwSctx3=0x%08x\n", xhci_ctx_get_le32(sc, &psl->dwSctx3));
184}
185#endif
186
187uint32_t
188xhci_get_port_route(void)
189{
190#ifdef USB_DEBUG
191 return (0xFFFFFFFFU ^ ((uint32_t)xhciroute));
192#else
193 return (0xFFFFFFFFU);
194#endif
195}
196
197static void
198xhci_iterate_hw_softc(struct usb_bus *bus, usb_bus_mem_sub_cb_t *cb)
199{
200 struct xhci_softc *sc = XHCI_BUS2SC(bus);
201 uint8_t i;
202
203 cb(bus, &sc->sc_hw.root_pc, &sc->sc_hw.root_pg,
204 sizeof(struct xhci_hw_root), XHCI_PAGE_SIZE);
205
206 cb(bus, &sc->sc_hw.ctx_pc, &sc->sc_hw.ctx_pg,
207 sizeof(struct xhci_dev_ctx_addr), XHCI_PAGE_SIZE);
208
209 for (i = 0; i != XHCI_MAX_SCRATCHPADS; i++) {
210 cb(bus, &sc->sc_hw.scratch_pc[i], &sc->sc_hw.scratch_pg[i],
211 XHCI_PAGE_SIZE, XHCI_PAGE_SIZE);
212 }
213}
214
215static void
216xhci_ctx_set_le32(struct xhci_softc *sc, volatile uint32_t *ptr, uint32_t val)
217{
218 if (sc->sc_ctx_is_64_byte) {
219 uint32_t offset;
220 /* exploit the fact that our structures are XHCI_PAGE_SIZE aligned */
221 /* all contexts are initially 32-bytes */
222 offset = ((uintptr_t)ptr) & ((XHCI_PAGE_SIZE - 1) & ~(31U));
223 ptr = (volatile uint32_t *)(((volatile uint8_t *)ptr) + offset);
224 }
225 *ptr = htole32(val);
226}
227
228static uint32_t
229xhci_ctx_get_le32(struct xhci_softc *sc, volatile uint32_t *ptr)
230{
231 if (sc->sc_ctx_is_64_byte) {
232 uint32_t offset;
233 /* exploit the fact that our structures are XHCI_PAGE_SIZE aligned */
234 /* all contexts are initially 32-bytes */
235 offset = ((uintptr_t)ptr) & ((XHCI_PAGE_SIZE - 1) & ~(31U));
236 ptr = (volatile uint32_t *)(((volatile uint8_t *)ptr) + offset);
237 }
238 return (le32toh(*ptr));
239}
240
241static void
242xhci_ctx_set_le64(struct xhci_softc *sc, volatile uint64_t *ptr, uint64_t val)
243{
244 if (sc->sc_ctx_is_64_byte) {
245 uint32_t offset;
246 /* exploit the fact that our structures are XHCI_PAGE_SIZE aligned */
247 /* all contexts are initially 32-bytes */
248 offset = ((uintptr_t)ptr) & ((XHCI_PAGE_SIZE - 1) & ~(31U));
249 ptr = (volatile uint64_t *)(((volatile uint8_t *)ptr) + offset);
250 }
251 *ptr = htole64(val);
252}
253
254#ifdef USB_DEBUG
255static uint64_t
256xhci_ctx_get_le64(struct xhci_softc *sc, volatile uint64_t *ptr)
257{
258 if (sc->sc_ctx_is_64_byte) {
259 uint32_t offset;
260 /* exploit the fact that our structures are XHCI_PAGE_SIZE aligned */
261 /* all contexts are initially 32-bytes */
262 offset = ((uintptr_t)ptr) & ((XHCI_PAGE_SIZE - 1) & ~(31U));
263 ptr = (volatile uint64_t *)(((volatile uint8_t *)ptr) + offset);
264 }
265 return (le64toh(*ptr));
266}
267#endif
268
269usb_error_t
270xhci_start_controller(struct xhci_softc *sc)
271{
272 struct usb_page_search buf_res;
273 struct xhci_hw_root *phwr;
274 struct xhci_dev_ctx_addr *pdctxa;
275 uint64_t addr;
276 uint32_t temp;
277 uint16_t i;
278
279 DPRINTF("\n");
280
281 sc->sc_capa_off = 0;
282 sc->sc_oper_off = XREAD1(sc, capa, XHCI_CAPLENGTH);
283 sc->sc_runt_off = XREAD4(sc, capa, XHCI_RTSOFF) & ~0x1F;
284 sc->sc_door_off = XREAD4(sc, capa, XHCI_DBOFF) & ~0x3;
285
286 DPRINTF("CAPLENGTH=0x%x\n", sc->sc_oper_off);
287 DPRINTF("RUNTIMEOFFSET=0x%x\n", sc->sc_runt_off);
288 DPRINTF("DOOROFFSET=0x%x\n", sc->sc_door_off);
289
290 sc->sc_event_ccs = 1;
291 sc->sc_event_idx = 0;
292 sc->sc_command_ccs = 1;
293 sc->sc_command_idx = 0;
294
295 DPRINTF("xHCI version = 0x%04x\n", XREAD2(sc, capa, XHCI_HCIVERSION));
296
297 temp = XREAD4(sc, capa, XHCI_HCSPARAMS0);
298
299 DPRINTF("HCS0 = 0x%08x\n", temp);
300
301 if (XHCI_HCS0_CSZ(temp)) {
302 sc->sc_ctx_is_64_byte = 1;
303 device_printf(sc->sc_bus.parent, "64 byte context size.\n");
304 } else {
305 sc->sc_ctx_is_64_byte = 0;
306 device_printf(sc->sc_bus.parent, "32 byte context size.\n");
307 }
308
309 /* Reset controller */
310 XWRITE4(sc, oper, XHCI_USBCMD, XHCI_CMD_HCRST);
311
312 for (i = 0; i != 100; i++) {
313 usb_pause_mtx(NULL, hz / 100);
314 temp = XREAD4(sc, oper, XHCI_USBCMD) &
315 (XHCI_CMD_HCRST | XHCI_STS_CNR);
316 if (!temp)
317 break;
318 }
319
320 if (temp) {
321 device_printf(sc->sc_bus.parent, "Controller "
322 "reset timeout.\n");
323 return (USB_ERR_IOERROR);
324 }
325
326 if (!(XREAD4(sc, oper, XHCI_PAGESIZE) & XHCI_PAGESIZE_4K)) {
327 device_printf(sc->sc_bus.parent, "Controller does "
328 "not support 4K page size.\n");
329 return (USB_ERR_IOERROR);
330 }
331
332 temp = XREAD4(sc, capa, XHCI_HCSPARAMS1);
333
334 i = XHCI_HCS1_N_PORTS(temp);
335
336 if (i == 0) {
337 device_printf(sc->sc_bus.parent, "Invalid number "
338 "of ports: %u\n", i);
339 return (USB_ERR_IOERROR);
340 }
341
342 sc->sc_noport = i;
343 sc->sc_noslot = XHCI_HCS1_DEVSLOT_MAX(temp);
344
345 if (sc->sc_noslot > XHCI_MAX_DEVICES)
346 sc->sc_noslot = XHCI_MAX_DEVICES;
347
348 /* setup number of device slots */
349
350 DPRINTF("CONFIG=0x%08x -> 0x%08x\n",
351 XREAD4(sc, oper, XHCI_CONFIG), sc->sc_noslot);
352
353 XWRITE4(sc, oper, XHCI_CONFIG, sc->sc_noslot);
354
355 DPRINTF("Max slots: %u\n", sc->sc_noslot);
356
357 temp = XREAD4(sc, capa, XHCI_HCSPARAMS2);
358
359 sc->sc_noscratch = XHCI_HCS2_SPB_MAX(temp);
360
361 if (sc->sc_noscratch > XHCI_MAX_SCRATCHPADS) {
362 device_printf(sc->sc_bus.parent, "XHCI request "
363 "too many scratchpads\n");
364 return (USB_ERR_NOMEM);
365 }
366
367 DPRINTF("Max scratch: %u\n", sc->sc_noscratch);
368
369 temp = XREAD4(sc, capa, XHCI_HCSPARAMS3);
370
371 sc->sc_exit_lat_max = XHCI_HCS3_U1_DEL(temp) +
372 XHCI_HCS3_U2_DEL(temp) + 250 /* us */;
373
374 temp = XREAD4(sc, oper, XHCI_USBSTS);
375
376 /* clear interrupts */
377 XWRITE4(sc, oper, XHCI_USBSTS, temp);
378 /* disable all device notifications */
379 XWRITE4(sc, oper, XHCI_DNCTRL, 0);
380
381 /* setup device context base address */
382 usbd_get_page(&sc->sc_hw.ctx_pc, 0, &buf_res);
383 pdctxa = buf_res.buffer;
384 memset(pdctxa, 0, sizeof(*pdctxa));
385
386 addr = buf_res.physaddr;
387 addr += (uintptr_t)&((struct xhci_dev_ctx_addr *)0)->qwSpBufPtr[0];
388
389 /* slot 0 points to the table of scratchpad pointers */
390 pdctxa->qwBaaDevCtxAddr[0] = htole64(addr);
391
392 for (i = 0; i != sc->sc_noscratch; i++) {
393 struct usb_page_search buf_scp;
394 usbd_get_page(&sc->sc_hw.scratch_pc[i], 0, &buf_scp);
395 pdctxa->qwSpBufPtr[i] = htole64((uint64_t)buf_scp.physaddr);
396 }
397
398 addr = buf_res.physaddr;
399
400 XWRITE4(sc, oper, XHCI_DCBAAP_LO, (uint32_t)addr);
401 XWRITE4(sc, oper, XHCI_DCBAAP_HI, (uint32_t)(addr >> 32));
402 XWRITE4(sc, oper, XHCI_DCBAAP_LO, (uint32_t)addr);
403 XWRITE4(sc, oper, XHCI_DCBAAP_HI, (uint32_t)(addr >> 32));
404
405 /* Setup event table size */
406
407 temp = XREAD4(sc, capa, XHCI_HCSPARAMS2);
408
409 DPRINTF("HCS2=0x%08x\n", temp);
410
411 temp = XHCI_HCS2_ERST_MAX(temp);
412 temp = 1U << temp;
413 if (temp > XHCI_MAX_RSEG)
414 temp = XHCI_MAX_RSEG;
415
416 sc->sc_erst_max = temp;
417
418 DPRINTF("ERSTSZ=0x%08x -> 0x%08x\n",
419 XREAD4(sc, runt, XHCI_ERSTSZ(0)), temp);
420
421 XWRITE4(sc, runt, XHCI_ERSTSZ(0), XHCI_ERSTS_SET(temp));
422
423 /* Setup interrupt rate */
424 XWRITE4(sc, runt, XHCI_IMOD(0), XHCI_IMOD_DEFAULT);
425
426 usbd_get_page(&sc->sc_hw.root_pc, 0, &buf_res);
427
428 phwr = buf_res.buffer;
429 addr = buf_res.physaddr;
430 addr += (uintptr_t)&((struct xhci_hw_root *)0)->hwr_events[0];
431
432 /* reset hardware root structure */
433 memset(phwr, 0, sizeof(*phwr));
434
435 phwr->hwr_ring_seg[0].qwEvrsTablePtr = htole64(addr);
436 phwr->hwr_ring_seg[0].dwEvrsTableSize = htole32(XHCI_MAX_EVENTS);
437
438 DPRINTF("ERDP(0)=0x%016llx\n", (unsigned long long)addr);
439
440 XWRITE4(sc, runt, XHCI_ERDP_LO(0), (uint32_t)addr);
441 XWRITE4(sc, runt, XHCI_ERDP_HI(0), (uint32_t)(addr >> 32));
442
443 addr = (uint64_t)buf_res.physaddr;
444
445 DPRINTF("ERSTBA(0)=0x%016llx\n", (unsigned long long)addr);
446
447 XWRITE4(sc, runt, XHCI_ERSTBA_LO(0), (uint32_t)addr);
448 XWRITE4(sc, runt, XHCI_ERSTBA_HI(0), (uint32_t)(addr >> 32));
449
450 /* Setup interrupter registers */
451
452 temp = XREAD4(sc, runt, XHCI_IMAN(0));
453 temp |= XHCI_IMAN_INTR_ENA;
454 XWRITE4(sc, runt, XHCI_IMAN(0), temp);
455
456 /* setup command ring control base address */
457 addr = buf_res.physaddr;
458 addr += (uintptr_t)&((struct xhci_hw_root *)0)->hwr_commands[0];
459
460 DPRINTF("CRCR=0x%016llx\n", (unsigned long long)addr);
461
462 XWRITE4(sc, oper, XHCI_CRCR_LO, ((uint32_t)addr) | XHCI_CRCR_LO_RCS);
463 XWRITE4(sc, oper, XHCI_CRCR_HI, (uint32_t)(addr >> 32));
464
465 phwr->hwr_commands[XHCI_MAX_COMMANDS - 1].qwTrb0 = htole64(addr);
466
467 usb_bus_mem_flush_all(&sc->sc_bus, &xhci_iterate_hw_softc);
468
469 /* Go! */
470 XWRITE4(sc, oper, XHCI_USBCMD, XHCI_CMD_RS |
471 XHCI_CMD_INTE | XHCI_CMD_HSEE);
472
473 for (i = 0; i != 100; i++) {
474 usb_pause_mtx(NULL, hz / 100);
475 temp = XREAD4(sc, oper, XHCI_USBSTS) & XHCI_STS_HCH;
476 if (!temp)
477 break;
478 }
479 if (temp) {
480 XWRITE4(sc, oper, XHCI_USBCMD, 0);
481 device_printf(sc->sc_bus.parent, "Run timeout.\n");
482 return (USB_ERR_IOERROR);
483 }
484
485 /* catch any lost interrupts */
486 xhci_do_poll(&sc->sc_bus);
487
488 return (0);
489}
490
491usb_error_t
492xhci_halt_controller(struct xhci_softc *sc)
493{
494 uint32_t temp;
495 uint16_t i;
496
497 DPRINTF("\n");
498
499 sc->sc_capa_off = 0;
500 sc->sc_oper_off = XREAD1(sc, capa, XHCI_CAPLENGTH);
501 sc->sc_runt_off = XREAD4(sc, capa, XHCI_RTSOFF) & ~0xF;
502 sc->sc_door_off = XREAD4(sc, capa, XHCI_DBOFF) & ~0x3;
503
504 /* Halt controller */
505 XWRITE4(sc, oper, XHCI_USBCMD, 0);
506
507 for (i = 0; i != 100; i++) {
508 usb_pause_mtx(NULL, hz / 100);
509 temp = XREAD4(sc, oper, XHCI_USBSTS) & XHCI_STS_HCH;
510 if (temp)
511 break;
512 }
513
514 if (!temp) {
515 device_printf(sc->sc_bus.parent, "Controller halt timeout.\n");
516 return (USB_ERR_IOERROR);
517 }
518 return (0);
519}
520
521usb_error_t
522xhci_init(struct xhci_softc *sc, device_t self)
523{
524 /* initialise some bus fields */
525 sc->sc_bus.parent = self;
526
527 /* set the bus revision */
528 sc->sc_bus.usbrev = USB_REV_3_0;
529
530 /* set up the bus struct */
531 sc->sc_bus.methods = &xhci_bus_methods;
532
533 /* setup devices array */
534 sc->sc_bus.devices = sc->sc_devices;
535 sc->sc_bus.devices_max = XHCI_MAX_DEVICES;
536
537 /* setup command queue mutex and condition varible */
538 cv_init(&sc->sc_cmd_cv, "CMDQ");
539 sx_init(&sc->sc_cmd_sx, "CMDQ lock");
540
541 /* get all DMA memory */
542 if (usb_bus_mem_alloc_all(&sc->sc_bus,
543 USB_GET_DMA_TAG(self), &xhci_iterate_hw_softc)) {
544 return (ENOMEM);
545 }
546
547 sc->sc_config_msg[0].hdr.pm_callback = &xhci_configure_msg;
548 sc->sc_config_msg[0].bus = &sc->sc_bus;
549 sc->sc_config_msg[1].hdr.pm_callback = &xhci_configure_msg;
550 sc->sc_config_msg[1].bus = &sc->sc_bus;
551
552 return (0);
553}
554
555void
556xhci_uninit(struct xhci_softc *sc)
557{
558 /*
559 * NOTE: At this point the control transfer process is gone
560 * and "xhci_configure_msg" is no longer called. Consequently
561 * waiting for the configuration messages to complete is not
562 * needed.
563 */
564 usb_bus_mem_free_all(&sc->sc_bus, &xhci_iterate_hw_softc);
565
566 cv_destroy(&sc->sc_cmd_cv);
567 sx_destroy(&sc->sc_cmd_sx);
568}
569
570static void
571xhci_set_hw_power_sleep(struct usb_bus *bus, uint32_t state)
572{
573 struct xhci_softc *sc = XHCI_BUS2SC(bus);
574
575 switch (state) {
576 case USB_HW_POWER_SUSPEND:
577 DPRINTF("Stopping the XHCI\n");
578 xhci_halt_controller(sc);
579 break;
580 case USB_HW_POWER_SHUTDOWN:
581 DPRINTF("Stopping the XHCI\n");
582 xhci_halt_controller(sc);
583 break;
584 case USB_HW_POWER_RESUME:
585 DPRINTF("Starting the XHCI\n");
586 xhci_start_controller(sc);
587 break;
588 default:
589 break;
590 }
591}
592
593static usb_error_t
594xhci_generic_done_sub(struct usb_xfer *xfer)
595{
596 struct xhci_td *td;
597 struct xhci_td *td_alt_next;
598 uint32_t len;
599 uint8_t status;
600
601 td = xfer->td_transfer_cache;
602 td_alt_next = td->alt_next;
603
604 if (xfer->aframes != xfer->nframes)
605 usbd_xfer_set_frame_len(xfer, xfer->aframes, 0);
606
607 while (1) {
608
609 usb_pc_cpu_invalidate(td->page_cache);
610
611 status = td->status;
612 len = td->remainder;
613
614 DPRINTFN(4, "xfer=%p[%u/%u] rem=%u/%u status=%u\n",
615 xfer, (unsigned int)xfer->aframes,
616 (unsigned int)xfer->nframes,
617 (unsigned int)len, (unsigned int)td->len,
618 (unsigned int)status);
619
620 /*
621 * Verify the status length and
622 * add the length to "frlengths[]":
623 */
624 if (len > td->len) {
625 /* should not happen */
626 DPRINTF("Invalid status length, "
627 "0x%04x/0x%04x bytes\n", len, td->len);
628 status = XHCI_TRB_ERROR_LENGTH;
629 } else if (xfer->aframes != xfer->nframes) {
630 xfer->frlengths[xfer->aframes] += td->len - len;
631 }
632 /* Check for last transfer */
633 if (((void *)td) == xfer->td_transfer_last) {
634 td = NULL;
635 break;
636 }
637 /* Check for transfer error */
638 if (status != XHCI_TRB_ERROR_SHORT_PKT &&
639 status != XHCI_TRB_ERROR_SUCCESS) {
640 /* the transfer is finished */
641 td = NULL;
642 break;
643 }
644 /* Check for short transfer */
645 if (len > 0) {
646 if (xfer->flags_int.short_frames_ok ||
647 xfer->flags_int.isochronous_xfr ||
648 xfer->flags_int.control_xfr) {
649 /* follow alt next */
650 td = td->alt_next;
651 } else {
652 /* the transfer is finished */
653 td = NULL;
654 }
655 break;
656 }
657 td = td->obj_next;
658
659 if (td->alt_next != td_alt_next) {
660 /* this USB frame is complete */
661 break;
662 }
663 }
664
665 /* update transfer cache */
666
667 xfer->td_transfer_cache = td;
668
669 return ((status == XHCI_TRB_ERROR_STALL) ? USB_ERR_STALLED :
670 (status != XHCI_TRB_ERROR_SHORT_PKT &&
671 status != XHCI_TRB_ERROR_SUCCESS) ? USB_ERR_IOERROR :
672 USB_ERR_NORMAL_COMPLETION);
673}
674
675static void
676xhci_generic_done(struct usb_xfer *xfer)
677{
678 usb_error_t err = 0;
679
680 DPRINTFN(13, "xfer=%p endpoint=%p transfer done\n",
681 xfer, xfer->endpoint);
682
683 /* reset scanner */
684
685 xfer->td_transfer_cache = xfer->td_transfer_first;
686
687 if (xfer->flags_int.control_xfr) {
688
689 if (xfer->flags_int.control_hdr)
690 err = xhci_generic_done_sub(xfer);
691
692 xfer->aframes = 1;
693
694 if (xfer->td_transfer_cache == NULL)
695 goto done;
696 }
697
698 while (xfer->aframes != xfer->nframes) {
699
700 err = xhci_generic_done_sub(xfer);
701 xfer->aframes++;
702
703 if (xfer->td_transfer_cache == NULL)
704 goto done;
705 }
706
707 if (xfer->flags_int.control_xfr &&
708 !xfer->flags_int.control_act)
709 err = xhci_generic_done_sub(xfer);
710done:
711 /* transfer is complete */
712 xhci_device_done(xfer, err);
713}
714
715static void
716xhci_activate_transfer(struct usb_xfer *xfer)
717{
718 struct xhci_td *td;
719
720 td = xfer->td_transfer_cache;
721
722 usb_pc_cpu_invalidate(td->page_cache);
723
724 if (!(td->td_trb[0].dwTrb3 & htole32(XHCI_TRB_3_CYCLE_BIT))) {
725
726 /* activate the transfer */
727
728 td->td_trb[0].dwTrb3 |= htole32(XHCI_TRB_3_CYCLE_BIT);
729 usb_pc_cpu_flush(td->page_cache);
730
731 xhci_endpoint_doorbell(xfer);
732 }
733}
734
735static void
736xhci_skip_transfer(struct usb_xfer *xfer)
737{
738 struct xhci_td *td;
739 struct xhci_td *td_last;
740
741 td = xfer->td_transfer_cache;
742 td_last = xfer->td_transfer_last;
743
744 td = td->alt_next;
745
746 usb_pc_cpu_invalidate(td->page_cache);
747
748 if (!(td->td_trb[0].dwTrb3 & htole32(XHCI_TRB_3_CYCLE_BIT))) {
749
750 usb_pc_cpu_invalidate(td_last->page_cache);
751
752 /* copy LINK TRB to current waiting location */
753
754 td->td_trb[0].qwTrb0 = td_last->td_trb[td_last->ntrb].qwTrb0;
755 td->td_trb[0].dwTrb2 = td_last->td_trb[td_last->ntrb].dwTrb2;
756 usb_pc_cpu_flush(td->page_cache);
757
758 td->td_trb[0].dwTrb3 = td_last->td_trb[td_last->ntrb].dwTrb3;
759 usb_pc_cpu_flush(td->page_cache);
760
761 xhci_endpoint_doorbell(xfer);
762 }
763}
764
765/*------------------------------------------------------------------------*
766 * xhci_check_transfer
767 *------------------------------------------------------------------------*/
768static void
769xhci_check_transfer(struct xhci_softc *sc, struct xhci_trb *trb)
770{
771 struct xhci_endpoint_ext *pepext;
772 int64_t offset;
773 uint64_t td_event;
774 uint32_t temp;
775 uint32_t remainder;
776 uint16_t stream_id;
777 uint16_t i;
778 uint8_t status;
779 uint8_t halted;
780 uint8_t epno;
781 uint8_t index;
782
783 /* decode TRB */
784 td_event = le64toh(trb->qwTrb0);
785 temp = le32toh(trb->dwTrb2);
786
787 remainder = XHCI_TRB_2_REM_GET(temp);
788 status = XHCI_TRB_2_ERROR_GET(temp);
789 stream_id = XHCI_TRB_2_STREAM_GET(temp);
790
791 temp = le32toh(trb->dwTrb3);
792 epno = XHCI_TRB_3_EP_GET(temp);
793 index = XHCI_TRB_3_SLOT_GET(temp);
794
795 /* check if error means halted */
796 halted = (status != XHCI_TRB_ERROR_SHORT_PKT &&
797 status != XHCI_TRB_ERROR_SUCCESS);
798
799 DPRINTF("slot=%u epno=%u stream=%u remainder=%u status=%u\n",
800 index, epno, stream_id, remainder, status);
801
802 if (index > sc->sc_noslot) {
803 DPRINTF("Invalid slot.\n");
804 return;
805 }
806
807 if ((epno == 0) || (epno >= XHCI_MAX_ENDPOINTS)) {
808 DPRINTF("Invalid endpoint.\n");
809 return;
810 }
811
812 pepext = &sc->sc_hw.devs[index].endp[epno];
813
814 if (pepext->trb_ep_mode != USB_EP_MODE_STREAMS) {
815 stream_id = 0;
816 DPRINTF("stream_id=0\n");
817 } else if (stream_id >= XHCI_MAX_STREAMS) {
818 DPRINTF("Invalid stream ID.\n");
819 return;
820 }
821
822 /* try to find the USB transfer that generated the event */
823 for (i = 0; i != (XHCI_MAX_TRANSFERS - 1); i++) {
824 struct usb_xfer *xfer;
825 struct xhci_td *td;
826
827 xfer = pepext->xfer[i + (XHCI_MAX_TRANSFERS * stream_id)];
828 if (xfer == NULL)
829 continue;
830
831 td = xfer->td_transfer_cache;
832
833 DPRINTFN(5, "Checking if 0x%016llx == (0x%016llx .. 0x%016llx)\n",
834 (long long)td_event,
835 (long long)td->td_self,
836 (long long)td->td_self + sizeof(td->td_trb));
837
838 /*
839 * NOTE: Some XHCI implementations might not trigger
840 * an event on the last LINK TRB so we need to
841 * consider both the last and second last event
842 * address as conditions for a successful transfer.
843 *
844 * NOTE: We assume that the XHCI will only trigger one
845 * event per chain of TRBs.
846 */
847
848 offset = td_event - td->td_self;
849
850 if (offset >= 0 &&
851 offset < (int64_t)sizeof(td->td_trb)) {
852
853 usb_pc_cpu_invalidate(td->page_cache);
854
855 /* compute rest of remainder, if any */
856 for (i = (offset / 16) + 1; i < td->ntrb; i++) {
857 temp = le32toh(td->td_trb[i].dwTrb2);
858 remainder += XHCI_TRB_2_BYTES_GET(temp);
859 }
860
861 DPRINTFN(5, "New remainder: %u\n", remainder);
862
863 /* clear isochronous transfer errors */
864 if (xfer->flags_int.isochronous_xfr) {
865 if (halted) {
866 halted = 0;
867 status = XHCI_TRB_ERROR_SUCCESS;
868 remainder = td->len;
869 }
870 }
871
872 /* "td->remainder" is verified later */
873 td->remainder = remainder;
874 td->status = status;
875
876 usb_pc_cpu_flush(td->page_cache);
877
878 /*
879 * 1) Last transfer descriptor makes the
880 * transfer done
881 */
882 if (((void *)td) == xfer->td_transfer_last) {
883 DPRINTF("TD is last\n");
884 xhci_generic_done(xfer);
885 break;
886 }
887
888 /*
889 * 2) Any kind of error makes the transfer
890 * done
891 */
892 if (halted) {
893 DPRINTF("TD has I/O error\n");
894 xhci_generic_done(xfer);
895 break;
896 }
897
898 /*
899 * 3) If there is no alternate next transfer,
900 * a short packet also makes the transfer done
901 */
902 if (td->remainder > 0) {
903 if (td->alt_next == NULL) {
904 DPRINTF(
905 "short TD has no alternate next\n");
906 xhci_generic_done(xfer);
907 break;
908 }
909 DPRINTF("TD has short pkt\n");
910 if (xfer->flags_int.short_frames_ok ||
911 xfer->flags_int.isochronous_xfr ||
912 xfer->flags_int.control_xfr) {
913 /* follow the alt next */
914 xfer->td_transfer_cache = td->alt_next;
915 xhci_activate_transfer(xfer);
916 break;
917 }
918 xhci_skip_transfer(xfer);
919 xhci_generic_done(xfer);
920 break;
921 }
922
923 /*
924 * 4) Transfer complete - go to next TD
925 */
926 DPRINTF("Following next TD\n");
927 xfer->td_transfer_cache = td->obj_next;
928 xhci_activate_transfer(xfer);
929 break; /* there should only be one match */
930 }
931 }
932}
933
934static void
935xhci_check_command(struct xhci_softc *sc, struct xhci_trb *trb)
936{
937 if (sc->sc_cmd_addr == trb->qwTrb0) {
938 DPRINTF("Received command event\n");
939 sc->sc_cmd_result[0] = trb->dwTrb2;
940 sc->sc_cmd_result[1] = trb->dwTrb3;
941 cv_signal(&sc->sc_cmd_cv);
942 }
943}
944
945static void
946xhci_interrupt_poll(struct xhci_softc *sc)
947{
948 struct usb_page_search buf_res;
949 struct xhci_hw_root *phwr;
950 uint64_t addr;
951 uint32_t temp;
952 uint16_t i;
953 uint8_t event;
954 uint8_t j;
955 uint8_t k;
956 uint8_t t;
957
958 usbd_get_page(&sc->sc_hw.root_pc, 0, &buf_res);
959
960 phwr = buf_res.buffer;
961
962 /* Receive any events */
963
964 usb_pc_cpu_invalidate(&sc->sc_hw.root_pc);
965
966 i = sc->sc_event_idx;
967 j = sc->sc_event_ccs;
968 t = 2;
969
970 while (1) {
971
972 temp = le32toh(phwr->hwr_events[i].dwTrb3);
973
974 k = (temp & XHCI_TRB_3_CYCLE_BIT) ? 1 : 0;
975
976 if (j != k)
977 break;
978
979 event = XHCI_TRB_3_TYPE_GET(temp);
980
981 DPRINTFN(10, "event[%u] = %u (0x%016llx 0x%08lx 0x%08lx)\n",
982 i, event, (long long)le64toh(phwr->hwr_events[i].qwTrb0),
983 (long)le32toh(phwr->hwr_events[i].dwTrb2),
984 (long)le32toh(phwr->hwr_events[i].dwTrb3));
985
986 switch (event) {
987 case XHCI_TRB_EVENT_TRANSFER:
988 xhci_check_transfer(sc, &phwr->hwr_events[i]);
989 break;
990 case XHCI_TRB_EVENT_CMD_COMPLETE:
991 xhci_check_command(sc, &phwr->hwr_events[i]);
992 break;
993 default:
994 DPRINTF("Unhandled event = %u\n", event);
995 break;
996 }
997
998 i++;
999
1000 if (i == XHCI_MAX_EVENTS) {
1001 i = 0;
1002 j ^= 1;
1003
1004 /* check for timeout */
1005 if (!--t)
1006 break;
1007 }
1008 }
1009
1010 sc->sc_event_idx = i;
1011 sc->sc_event_ccs = j;
1012
1013 /*
1014 * NOTE: The Event Ring Dequeue Pointer Register is 64-bit
1015 * latched. That means to activate the register we need to
1016 * write both the low and high double word of the 64-bit
1017 * register.
1018 */
1019
1020 addr = (uint32_t)buf_res.physaddr;
1021 addr += (uintptr_t)&((struct xhci_hw_root *)0)->hwr_events[i];
1022
1023 /* try to clear busy bit */
1024 addr |= XHCI_ERDP_LO_BUSY;
1025
1026 XWRITE4(sc, runt, XHCI_ERDP_LO(0), (uint32_t)addr);
1027 XWRITE4(sc, runt, XHCI_ERDP_HI(0), (uint32_t)(addr >> 32));
1028}
1029
1030static usb_error_t
1031xhci_do_command(struct xhci_softc *sc, struct xhci_trb *trb,
1032 uint16_t timeout_ms)
1033{
1034 struct usb_page_search buf_res;
1035 struct xhci_hw_root *phwr;
1036 uint64_t addr;
1037 uint32_t temp;
1038 uint8_t i;
1039 uint8_t j;
1040 int err;
1041
1042 XHCI_CMD_ASSERT_LOCKED(sc);
1043
1044 /* get hardware root structure */
1045
1046 usbd_get_page(&sc->sc_hw.root_pc, 0, &buf_res);
1047
1048 phwr = buf_res.buffer;
1049
1050 /* Queue command */
1051
1052 USB_BUS_LOCK(&sc->sc_bus);
1053
1054 i = sc->sc_command_idx;
1055 j = sc->sc_command_ccs;
1056
1057 DPRINTFN(10, "command[%u] = %u (0x%016llx, 0x%08lx, 0x%08lx)\n",
1058 i, XHCI_TRB_3_TYPE_GET(le32toh(trb->dwTrb3)),
1059 (long long)le64toh(trb->qwTrb0),
1060 (long)le32toh(trb->dwTrb2),
1061 (long)le32toh(trb->dwTrb3));
1062
1063 phwr->hwr_commands[i].qwTrb0 = trb->qwTrb0;
1064 phwr->hwr_commands[i].dwTrb2 = trb->dwTrb2;
1065
1066 usb_pc_cpu_flush(&sc->sc_hw.root_pc);
1067
1068 temp = trb->dwTrb3;
1069
1070 if (j)
1071 temp |= htole32(XHCI_TRB_3_CYCLE_BIT);
1072 else
1073 temp &= ~htole32(XHCI_TRB_3_CYCLE_BIT);
1074
1075 temp &= ~htole32(XHCI_TRB_3_TC_BIT);
1076
1077 phwr->hwr_commands[i].dwTrb3 = temp;
1078
1079 usb_pc_cpu_flush(&sc->sc_hw.root_pc);
1080
1081 addr = buf_res.physaddr;
1082 addr += (uintptr_t)&((struct xhci_hw_root *)0)->hwr_commands[i];
1083
1084 sc->sc_cmd_addr = htole64(addr);
1085
1086 i++;
1087
1088 if (i == (XHCI_MAX_COMMANDS - 1)) {
1089
1090 if (j) {
1091 temp = htole32(XHCI_TRB_3_TC_BIT |
1092 XHCI_TRB_3_TYPE_SET(XHCI_TRB_TYPE_LINK) |
1093 XHCI_TRB_3_CYCLE_BIT);
1094 } else {
1095 temp = htole32(XHCI_TRB_3_TC_BIT |
1096 XHCI_TRB_3_TYPE_SET(XHCI_TRB_TYPE_LINK));
1097 }
1098
1099 phwr->hwr_commands[i].dwTrb3 = temp;
1100
1101 usb_pc_cpu_flush(&sc->sc_hw.root_pc);
1102
1103 i = 0;
1104 j ^= 1;
1105 }
1106
1107 sc->sc_command_idx = i;
1108 sc->sc_command_ccs = j;
1109
1110 XWRITE4(sc, door, XHCI_DOORBELL(0), 0);
1111
1112 err = cv_timedwait(&sc->sc_cmd_cv, &sc->sc_bus.bus_mtx,
1113 USB_MS_TO_TICKS(timeout_ms));
1114
1115 if (err) {
1116 DPRINTFN(0, "Command timeout!\n");
1117 err = USB_ERR_TIMEOUT;
1118 trb->dwTrb2 = 0;
1119 trb->dwTrb3 = 0;
1120 } else {
1121 temp = le32toh(sc->sc_cmd_result[0]);
1122 if (XHCI_TRB_2_ERROR_GET(temp) != XHCI_TRB_ERROR_SUCCESS)
1123 err = USB_ERR_IOERROR;
1124
1125 trb->dwTrb2 = sc->sc_cmd_result[0];
1126 trb->dwTrb3 = sc->sc_cmd_result[1];
1127 }
1128
1129 USB_BUS_UNLOCK(&sc->sc_bus);
1130
1131 return (err);
1132}
1133
1134#if 0
1135static usb_error_t
1136xhci_cmd_nop(struct xhci_softc *sc)
1137{
1138 struct xhci_trb trb;
1139 uint32_t temp;
1140
1141 DPRINTF("\n");
1142
1143 trb.qwTrb0 = 0;
1144 trb.dwTrb2 = 0;
1145 temp = XHCI_TRB_3_TYPE_SET(XHCI_TRB_TYPE_NOOP);
1146
1147 trb.dwTrb3 = htole32(temp);
1148
1149 return (xhci_do_command(sc, &trb, 100 /* ms */));
1150}
1151#endif
1152
1153static usb_error_t
1154xhci_cmd_enable_slot(struct xhci_softc *sc, uint8_t *pslot)
1155{
1156 struct xhci_trb trb;
1157 uint32_t temp;
1158 usb_error_t err;
1159
1160 DPRINTF("\n");
1161
1162 trb.qwTrb0 = 0;
1163 trb.dwTrb2 = 0;
1164 trb.dwTrb3 = htole32(XHCI_TRB_3_TYPE_SET(XHCI_TRB_TYPE_ENABLE_SLOT));
1165
1166 err = xhci_do_command(sc, &trb, 100 /* ms */);
1167 if (err)
1168 goto done;
1169
1170 temp = le32toh(trb.dwTrb3);
1171
1172 *pslot = XHCI_TRB_3_SLOT_GET(temp);
1173
1174done:
1175 return (err);
1176}
1177
1178static usb_error_t
1179xhci_cmd_disable_slot(struct xhci_softc *sc, uint8_t slot_id)
1180{
1181 struct xhci_trb trb;
1182 uint32_t temp;
1183
1184 DPRINTF("\n");
1185
1186 trb.qwTrb0 = 0;
1187 trb.dwTrb2 = 0;
1188 temp = XHCI_TRB_3_TYPE_SET(XHCI_TRB_TYPE_DISABLE_SLOT) |
1189 XHCI_TRB_3_SLOT_SET(slot_id);
1190
1191 trb.dwTrb3 = htole32(temp);
1192
1193 return (xhci_do_command(sc, &trb, 100 /* ms */));
1194}
1195
1196static usb_error_t
1197xhci_cmd_set_address(struct xhci_softc *sc, uint64_t input_ctx,
1198 uint8_t bsr, uint8_t slot_id)
1199{
1200 struct xhci_trb trb;
1201 uint32_t temp;
1202
1203 DPRINTF("\n");
1204
1205 trb.qwTrb0 = htole64(input_ctx);
1206 trb.dwTrb2 = 0;
1207 temp = XHCI_TRB_3_TYPE_SET(XHCI_TRB_TYPE_ADDRESS_DEVICE) |
1208 XHCI_TRB_3_SLOT_SET(slot_id);
1209
1210 if (bsr)
1211 temp |= XHCI_TRB_3_BSR_BIT;
1212
1213 trb.dwTrb3 = htole32(temp);
1214
1215 return (xhci_do_command(sc, &trb, 500 /* ms */));
1216}
1217
1218static usb_error_t
1219xhci_set_address(struct usb_device *udev, struct mtx *mtx, uint16_t address)
1220{
1221 struct usb_page_search buf_inp;
1222 struct usb_page_search buf_dev;
1223 struct xhci_softc *sc = XHCI_BUS2SC(udev->bus);
1224 struct xhci_hw_dev *hdev;
1225 struct xhci_dev_ctx *pdev;
1226 struct xhci_endpoint_ext *pepext;
1227 uint32_t temp;
1228 uint16_t mps;
1229 usb_error_t err;
1230 uint8_t index;
1231
1232 /* the root HUB case is not handled here */
1233 if (udev->parent_hub == NULL)
1234 return (USB_ERR_INVAL);
1235
1236 index = udev->controller_slot_id;
1237
1238 hdev = &sc->sc_hw.devs[index];
1239
1240 if (mtx != NULL)
1241 mtx_unlock(mtx);
1242
1243 XHCI_CMD_LOCK(sc);
1244
1245 switch (hdev->state) {
1246 case XHCI_ST_DEFAULT:
1247 case XHCI_ST_ENABLED:
1248
1249 hdev->state = XHCI_ST_ENABLED;
1250
1251 /* set configure mask to slot and EP0 */
1252 xhci_configure_mask(udev, 3, 0);
1253
1254 /* configure input slot context structure */
1255 err = xhci_configure_device(udev);
1256
1257 if (err != 0) {
1258 DPRINTF("Could not configure device\n");
1259 break;
1260 }
1261
1262 /* configure input endpoint context structure */
1263 switch (udev->speed) {
1264 case USB_SPEED_LOW:
1265 case USB_SPEED_FULL:
1266 mps = 8;
1267 break;
1268 case USB_SPEED_HIGH:
1269 mps = 64;
1270 break;
1271 default:
1272 mps = 512;
1273 break;
1274 }
1275
1276 pepext = xhci_get_endpoint_ext(udev,
1277 &udev->ctrl_ep_desc);
1278 err = xhci_configure_endpoint(udev,
1279 &udev->ctrl_ep_desc, pepext,
1280 0, 1, 1, 0, mps, mps, USB_EP_MODE_DEFAULT);
1281
1282 if (err != 0) {
1283 DPRINTF("Could not configure default endpoint\n");
1284 break;
1285 }
1286
1287 /* execute set address command */
1288 usbd_get_page(&hdev->input_pc, 0, &buf_inp);
1289
1290 err = xhci_cmd_set_address(sc, buf_inp.physaddr,
1291 (address == 0), index);
1292
1293 if (err != 0) {
1294 DPRINTF("Could not set address "
1295 "for slot %u.\n", index);
1296 if (address != 0)
1297 break;
1298 }
1299
1300 /* update device address to new value */
1301
1302 usbd_get_page(&hdev->device_pc, 0, &buf_dev);
1303 pdev = buf_dev.buffer;
1304 usb_pc_cpu_invalidate(&hdev->device_pc);
1305
1306 temp = xhci_ctx_get_le32(sc, &pdev->ctx_slot.dwSctx3);
1307 udev->address = XHCI_SCTX_3_DEV_ADDR_GET(temp);
1308
1309 /* update device state to new value */
1310
1311 if (address != 0)
1312 hdev->state = XHCI_ST_ADDRESSED;
1313 else
1314 hdev->state = XHCI_ST_DEFAULT;
1315 break;
1316
1317 default:
1318 DPRINTF("Wrong state for set address.\n");
1319 err = USB_ERR_IOERROR;
1320 break;
1321 }
1322 XHCI_CMD_UNLOCK(sc);
1323
1324 if (mtx != NULL)
1325 mtx_lock(mtx);
1326
1327 return (err);
1328}
1329
1330static usb_error_t
1331xhci_cmd_configure_ep(struct xhci_softc *sc, uint64_t input_ctx,
1332 uint8_t deconfigure, uint8_t slot_id)
1333{
1334 struct xhci_trb trb;
1335 uint32_t temp;
1336
1337 DPRINTF("\n");
1338
1339 trb.qwTrb0 = htole64(input_ctx);
1340 trb.dwTrb2 = 0;
1341 temp = XHCI_TRB_3_TYPE_SET(XHCI_TRB_TYPE_CONFIGURE_EP) |
1342 XHCI_TRB_3_SLOT_SET(slot_id);
1343
1344 if (deconfigure)
1345 temp |= XHCI_TRB_3_DCEP_BIT;
1346
1347 trb.dwTrb3 = htole32(temp);
1348
1349 return (xhci_do_command(sc, &trb, 100 /* ms */));
1350}
1351
1352static usb_error_t
1353xhci_cmd_evaluate_ctx(struct xhci_softc *sc, uint64_t input_ctx,
1354 uint8_t slot_id)
1355{
1356 struct xhci_trb trb;
1357 uint32_t temp;
1358
1359 DPRINTF("\n");
1360
1361 trb.qwTrb0 = htole64(input_ctx);
1362 trb.dwTrb2 = 0;
1363 temp = XHCI_TRB_3_TYPE_SET(XHCI_TRB_TYPE_EVALUATE_CTX) |
1364 XHCI_TRB_3_SLOT_SET(slot_id);
1365 trb.dwTrb3 = htole32(temp);
1366
1367 return (xhci_do_command(sc, &trb, 100 /* ms */));
1368}
1369
1370static usb_error_t
1371xhci_cmd_reset_ep(struct xhci_softc *sc, uint8_t preserve,
1372 uint8_t ep_id, uint8_t slot_id)
1373{
1374 struct xhci_trb trb;
1375 uint32_t temp;
1376
1377 DPRINTF("\n");
1378
1379 trb.qwTrb0 = 0;
1380 trb.dwTrb2 = 0;
1381 temp = XHCI_TRB_3_TYPE_SET(XHCI_TRB_TYPE_RESET_EP) |
1382 XHCI_TRB_3_SLOT_SET(slot_id) |
1383 XHCI_TRB_3_EP_SET(ep_id);
1384
1385 if (preserve)
1386 temp |= XHCI_TRB_3_PRSV_BIT;
1387
1388 trb.dwTrb3 = htole32(temp);
1389
1390 return (xhci_do_command(sc, &trb, 100 /* ms */));
1391}
1392
1393static usb_error_t
1394xhci_cmd_set_tr_dequeue_ptr(struct xhci_softc *sc, uint64_t dequeue_ptr,
1395 uint16_t stream_id, uint8_t ep_id, uint8_t slot_id)
1396{
1397 struct xhci_trb trb;
1398 uint32_t temp;
1399
1400 DPRINTF("\n");
1401
1402 trb.qwTrb0 = htole64(dequeue_ptr);
1403
1404 temp = XHCI_TRB_2_STREAM_SET(stream_id);
1405 trb.dwTrb2 = htole32(temp);
1406
1407 temp = XHCI_TRB_3_TYPE_SET(XHCI_TRB_TYPE_SET_TR_DEQUEUE) |
1408 XHCI_TRB_3_SLOT_SET(slot_id) |
1409 XHCI_TRB_3_EP_SET(ep_id);
1410 trb.dwTrb3 = htole32(temp);
1411
1412 return (xhci_do_command(sc, &trb, 100 /* ms */));
1413}
1414
1415static usb_error_t
1416xhci_cmd_stop_ep(struct xhci_softc *sc, uint8_t suspend,
1417 uint8_t ep_id, uint8_t slot_id)
1418{
1419 struct xhci_trb trb;
1420 uint32_t temp;
1421
1422 DPRINTF("\n");
1423
1424 trb.qwTrb0 = 0;
1425 trb.dwTrb2 = 0;
1426 temp = XHCI_TRB_3_TYPE_SET(XHCI_TRB_TYPE_STOP_EP) |
1427 XHCI_TRB_3_SLOT_SET(slot_id) |
1428 XHCI_TRB_3_EP_SET(ep_id);
1429
1430 if (suspend)
1431 temp |= XHCI_TRB_3_SUSP_EP_BIT;
1432
1433 trb.dwTrb3 = htole32(temp);
1434
1435 return (xhci_do_command(sc, &trb, 100 /* ms */));
1436}
1437
1438static usb_error_t
1439xhci_cmd_reset_dev(struct xhci_softc *sc, uint8_t slot_id)
1440{
1441 struct xhci_trb trb;
1442 uint32_t temp;
1443
1444 DPRINTF("\n");
1445
1446 trb.qwTrb0 = 0;
1447 trb.dwTrb2 = 0;
1448 temp = XHCI_TRB_3_TYPE_SET(XHCI_TRB_TYPE_RESET_DEVICE) |
1449 XHCI_TRB_3_SLOT_SET(slot_id);
1450
1451 trb.dwTrb3 = htole32(temp);
1452
1453 return (xhci_do_command(sc, &trb, 100 /* ms */));
1454}
1455
1456/*------------------------------------------------------------------------*
1457 * xhci_interrupt - XHCI interrupt handler
1458 *------------------------------------------------------------------------*/
1459void
1460xhci_interrupt(struct xhci_softc *sc)
1461{
1462 uint32_t status;
1463 uint32_t iman;
1464
1465 USB_BUS_LOCK(&sc->sc_bus);
1466
1467 status = XREAD4(sc, oper, XHCI_USBSTS);
1468 if (status == 0)
1469 goto done;
1470
1471 /* acknowledge interrupts */
1472
1473 XWRITE4(sc, oper, XHCI_USBSTS, status);
1474
1475 DPRINTFN(16, "real interrupt (status=0x%08x)\n", status);
1476
1477 if (status & XHCI_STS_EINT) {
1478
1479 /* acknowledge pending event */
1480 iman = XREAD4(sc, runt, XHCI_IMAN(0));
1481
1482 /* reset interrupt */
1483 XWRITE4(sc, runt, XHCI_IMAN(0), iman);
1484
1485 DPRINTFN(16, "real interrupt (iman=0x%08x)\n", iman);
1486
1487 /* check for event(s) */
1488 xhci_interrupt_poll(sc);
1489 }
1490
1491 if (status & (XHCI_STS_PCD | XHCI_STS_HCH |
1492 XHCI_STS_HSE | XHCI_STS_HCE)) {
1493
1494 if (status & XHCI_STS_PCD) {
1495 xhci_root_intr(sc);
1496 }
1497
1498 if (status & XHCI_STS_HCH) {
1499 printf("%s: host controller halted\n",
1500 __FUNCTION__);
1501 }
1502
1503 if (status & XHCI_STS_HSE) {
1504 printf("%s: host system error\n",
1505 __FUNCTION__);
1506 }
1507
1508 if (status & XHCI_STS_HCE) {
1509 printf("%s: host controller error\n",
1510 __FUNCTION__);
1511 }
1512 }
1513done:
1514 USB_BUS_UNLOCK(&sc->sc_bus);
1515}
1516
1517/*------------------------------------------------------------------------*
1518 * xhci_timeout - XHCI timeout handler
1519 *------------------------------------------------------------------------*/
1520static void
1521xhci_timeout(void *arg)
1522{
1523 struct usb_xfer *xfer = arg;
1524
1525 DPRINTF("xfer=%p\n", xfer);
1526
1527 USB_BUS_LOCK_ASSERT(xfer->xroot->bus, MA_OWNED);
1528
1529 /* transfer is transferred */
1530 xhci_device_done(xfer, USB_ERR_TIMEOUT);
1531}
1532
1533static void
1534xhci_do_poll(struct usb_bus *bus)
1535{
1536 struct xhci_softc *sc = XHCI_BUS2SC(bus);
1537
1538 USB_BUS_LOCK(&sc->sc_bus);
1539 xhci_interrupt_poll(sc);
1540 USB_BUS_UNLOCK(&sc->sc_bus);
1541}
1542
1543static void
1544xhci_setup_generic_chain_sub(struct xhci_std_temp *temp)
1545{
1546 struct usb_page_search buf_res;
1547 struct xhci_td *td;
| 2/*-
3 * Copyright (c) 2010 Hans Petter Selasky. All rights reserved.
4 *
5 * Redistribution and use in source and binary forms, with or without
6 * modification, are permitted provided that the following conditions
7 * are met:
8 * 1. Redistributions of source code must retain the above copyright
9 * notice, this list of conditions and the following disclaimer.
10 * 2. Redistributions in binary form must reproduce the above copyright
11 * notice, this list of conditions and the following disclaimer in the
12 * documentation and/or other materials provided with the distribution.
13 *
14 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
15 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
16 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
17 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
18 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
19 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
20 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
21 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
22 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
23 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
24 * SUCH DAMAGE.
25 */
26
27/*
28 * USB eXtensible Host Controller Interface, a.k.a. USB 3.0 controller.
29 *
30 * The XHCI 1.0 spec can be found at
31 * http://www.intel.com/technology/usb/download/xHCI_Specification_for_USB.pdf
32 * and the USB 3.0 spec at
33 * http://www.usb.org/developers/docs/usb_30_spec_060910.zip
34 */
35
36/*
37 * A few words about the design implementation: This driver emulates
38 * the concept about TDs which is found in EHCI specification. This
39 * way we achieve that the USB controller drivers look similar to
40 * eachother which makes it easier to understand the code.
41 */
42
43#ifdef USB_GLOBAL_INCLUDE_FILE
44#include USB_GLOBAL_INCLUDE_FILE
45#else
46#include <sys/stdint.h>
47#include <sys/stddef.h>
48#include <sys/param.h>
49#include <sys/queue.h>
50#include <sys/types.h>
51#include <sys/systm.h>
52#include <sys/kernel.h>
53#include <sys/bus.h>
54#include <sys/module.h>
55#include <sys/lock.h>
56#include <sys/mutex.h>
57#include <sys/condvar.h>
58#include <sys/sysctl.h>
59#include <sys/sx.h>
60#include <sys/unistd.h>
61#include <sys/callout.h>
62#include <sys/malloc.h>
63#include <sys/priv.h>
64
65#include <dev/usb/usb.h>
66#include <dev/usb/usbdi.h>
67
68#define USB_DEBUG_VAR xhcidebug
69
70#include <dev/usb/usb_core.h>
71#include <dev/usb/usb_debug.h>
72#include <dev/usb/usb_busdma.h>
73#include <dev/usb/usb_process.h>
74#include <dev/usb/usb_transfer.h>
75#include <dev/usb/usb_device.h>
76#include <dev/usb/usb_hub.h>
77#include <dev/usb/usb_util.h>
78
79#include <dev/usb/usb_controller.h>
80#include <dev/usb/usb_bus.h>
81#endif /* USB_GLOBAL_INCLUDE_FILE */
82
83#include <dev/usb/controller/xhci.h>
84#include <dev/usb/controller/xhcireg.h>
85
86#define XHCI_BUS2SC(bus) \
87 ((struct xhci_softc *)(((uint8_t *)(bus)) - \
88 ((uint8_t *)&(((struct xhci_softc *)0)->sc_bus))))
89
90#ifdef USB_DEBUG
91static int xhcidebug;
92static int xhciroute;
93
94static SYSCTL_NODE(_hw_usb, OID_AUTO, xhci, CTLFLAG_RW, 0, "USB XHCI");
95SYSCTL_INT(_hw_usb_xhci, OID_AUTO, debug, CTLFLAG_RW | CTLFLAG_TUN,
96 &xhcidebug, 0, "Debug level");
97TUNABLE_INT("hw.usb.xhci.debug", &xhcidebug);
98SYSCTL_INT(_hw_usb_xhci, OID_AUTO, xhci_port_route, CTLFLAG_RW | CTLFLAG_TUN,
99 &xhciroute, 0, "Routing bitmap for switching EHCI ports to XHCI controller");
100TUNABLE_INT("hw.usb.xhci.xhci_port_route", &xhciroute);
101#endif
102
103#define XHCI_INTR_ENDPT 1
104
105struct xhci_std_temp {
106 struct xhci_softc *sc;
107 struct usb_page_cache *pc;
108 struct xhci_td *td;
109 struct xhci_td *td_next;
110 uint32_t len;
111 uint32_t offset;
112 uint32_t max_packet_size;
113 uint32_t average;
114 uint16_t isoc_delta;
115 uint16_t isoc_frame;
116 uint8_t shortpkt;
117 uint8_t multishort;
118 uint8_t last_frame;
119 uint8_t trb_type;
120 uint8_t direction;
121 uint8_t tbc;
122 uint8_t tlbpc;
123 uint8_t step_td;
124 uint8_t do_isoc_sync;
125};
126
127static void xhci_do_poll(struct usb_bus *);
128static void xhci_device_done(struct usb_xfer *, usb_error_t);
129static void xhci_root_intr(struct xhci_softc *);
130static void xhci_free_device_ext(struct usb_device *);
131static struct xhci_endpoint_ext *xhci_get_endpoint_ext(struct usb_device *,
132 struct usb_endpoint_descriptor *);
133static usb_proc_callback_t xhci_configure_msg;
134static usb_error_t xhci_configure_device(struct usb_device *);
135static usb_error_t xhci_configure_endpoint(struct usb_device *,
136 struct usb_endpoint_descriptor *, struct xhci_endpoint_ext *,
137 uint16_t, uint8_t, uint8_t, uint8_t, uint16_t, uint16_t,
138 uint8_t);
139static usb_error_t xhci_configure_mask(struct usb_device *,
140 uint32_t, uint8_t);
141static usb_error_t xhci_cmd_evaluate_ctx(struct xhci_softc *,
142 uint64_t, uint8_t);
143static void xhci_endpoint_doorbell(struct usb_xfer *);
144static void xhci_ctx_set_le32(struct xhci_softc *sc, volatile uint32_t *ptr, uint32_t val);
145static uint32_t xhci_ctx_get_le32(struct xhci_softc *sc, volatile uint32_t *ptr);
146static void xhci_ctx_set_le64(struct xhci_softc *sc, volatile uint64_t *ptr, uint64_t val);
147#ifdef USB_DEBUG
148static uint64_t xhci_ctx_get_le64(struct xhci_softc *sc, volatile uint64_t *ptr);
149#endif
150
151extern struct usb_bus_methods xhci_bus_methods;
152
153#ifdef USB_DEBUG
154static void
155xhci_dump_trb(struct xhci_trb *trb)
156{
157 DPRINTFN(5, "trb = %p\n", trb);
158 DPRINTFN(5, "qwTrb0 = 0x%016llx\n", (long long)le64toh(trb->qwTrb0));
159 DPRINTFN(5, "dwTrb2 = 0x%08x\n", le32toh(trb->dwTrb2));
160 DPRINTFN(5, "dwTrb3 = 0x%08x\n", le32toh(trb->dwTrb3));
161}
162
163static void
164xhci_dump_endpoint(struct xhci_softc *sc, struct xhci_endp_ctx *pep)
165{
166 DPRINTFN(5, "pep = %p\n", pep);
167 DPRINTFN(5, "dwEpCtx0=0x%08x\n", xhci_ctx_get_le32(sc, &pep->dwEpCtx0));
168 DPRINTFN(5, "dwEpCtx1=0x%08x\n", xhci_ctx_get_le32(sc, &pep->dwEpCtx1));
169 DPRINTFN(5, "qwEpCtx2=0x%016llx\n", (long long)xhci_ctx_get_le64(sc, &pep->qwEpCtx2));
170 DPRINTFN(5, "dwEpCtx4=0x%08x\n", xhci_ctx_get_le32(sc, &pep->dwEpCtx4));
171 DPRINTFN(5, "dwEpCtx5=0x%08x\n", xhci_ctx_get_le32(sc, &pep->dwEpCtx5));
172 DPRINTFN(5, "dwEpCtx6=0x%08x\n", xhci_ctx_get_le32(sc, &pep->dwEpCtx6));
173 DPRINTFN(5, "dwEpCtx7=0x%08x\n", xhci_ctx_get_le32(sc, &pep->dwEpCtx7));
174}
175
176static void
177xhci_dump_device(struct xhci_softc *sc, struct xhci_slot_ctx *psl)
178{
179 DPRINTFN(5, "psl = %p\n", psl);
180 DPRINTFN(5, "dwSctx0=0x%08x\n", xhci_ctx_get_le32(sc, &psl->dwSctx0));
181 DPRINTFN(5, "dwSctx1=0x%08x\n", xhci_ctx_get_le32(sc, &psl->dwSctx1));
182 DPRINTFN(5, "dwSctx2=0x%08x\n", xhci_ctx_get_le32(sc, &psl->dwSctx2));
183 DPRINTFN(5, "dwSctx3=0x%08x\n", xhci_ctx_get_le32(sc, &psl->dwSctx3));
184}
185#endif
186
187uint32_t
188xhci_get_port_route(void)
189{
190#ifdef USB_DEBUG
191 return (0xFFFFFFFFU ^ ((uint32_t)xhciroute));
192#else
193 return (0xFFFFFFFFU);
194#endif
195}
196
197static void
198xhci_iterate_hw_softc(struct usb_bus *bus, usb_bus_mem_sub_cb_t *cb)
199{
200 struct xhci_softc *sc = XHCI_BUS2SC(bus);
201 uint8_t i;
202
203 cb(bus, &sc->sc_hw.root_pc, &sc->sc_hw.root_pg,
204 sizeof(struct xhci_hw_root), XHCI_PAGE_SIZE);
205
206 cb(bus, &sc->sc_hw.ctx_pc, &sc->sc_hw.ctx_pg,
207 sizeof(struct xhci_dev_ctx_addr), XHCI_PAGE_SIZE);
208
209 for (i = 0; i != XHCI_MAX_SCRATCHPADS; i++) {
210 cb(bus, &sc->sc_hw.scratch_pc[i], &sc->sc_hw.scratch_pg[i],
211 XHCI_PAGE_SIZE, XHCI_PAGE_SIZE);
212 }
213}
214
215static void
216xhci_ctx_set_le32(struct xhci_softc *sc, volatile uint32_t *ptr, uint32_t val)
217{
218 if (sc->sc_ctx_is_64_byte) {
219 uint32_t offset;
220 /* exploit the fact that our structures are XHCI_PAGE_SIZE aligned */
221 /* all contexts are initially 32-bytes */
222 offset = ((uintptr_t)ptr) & ((XHCI_PAGE_SIZE - 1) & ~(31U));
223 ptr = (volatile uint32_t *)(((volatile uint8_t *)ptr) + offset);
224 }
225 *ptr = htole32(val);
226}
227
228static uint32_t
229xhci_ctx_get_le32(struct xhci_softc *sc, volatile uint32_t *ptr)
230{
231 if (sc->sc_ctx_is_64_byte) {
232 uint32_t offset;
233 /* exploit the fact that our structures are XHCI_PAGE_SIZE aligned */
234 /* all contexts are initially 32-bytes */
235 offset = ((uintptr_t)ptr) & ((XHCI_PAGE_SIZE - 1) & ~(31U));
236 ptr = (volatile uint32_t *)(((volatile uint8_t *)ptr) + offset);
237 }
238 return (le32toh(*ptr));
239}
240
241static void
242xhci_ctx_set_le64(struct xhci_softc *sc, volatile uint64_t *ptr, uint64_t val)
243{
244 if (sc->sc_ctx_is_64_byte) {
245 uint32_t offset;
246 /* exploit the fact that our structures are XHCI_PAGE_SIZE aligned */
247 /* all contexts are initially 32-bytes */
248 offset = ((uintptr_t)ptr) & ((XHCI_PAGE_SIZE - 1) & ~(31U));
249 ptr = (volatile uint64_t *)(((volatile uint8_t *)ptr) + offset);
250 }
251 *ptr = htole64(val);
252}
253
254#ifdef USB_DEBUG
255static uint64_t
256xhci_ctx_get_le64(struct xhci_softc *sc, volatile uint64_t *ptr)
257{
258 if (sc->sc_ctx_is_64_byte) {
259 uint32_t offset;
260 /* exploit the fact that our structures are XHCI_PAGE_SIZE aligned */
261 /* all contexts are initially 32-bytes */
262 offset = ((uintptr_t)ptr) & ((XHCI_PAGE_SIZE - 1) & ~(31U));
263 ptr = (volatile uint64_t *)(((volatile uint8_t *)ptr) + offset);
264 }
265 return (le64toh(*ptr));
266}
267#endif
268
269usb_error_t
270xhci_start_controller(struct xhci_softc *sc)
271{
272 struct usb_page_search buf_res;
273 struct xhci_hw_root *phwr;
274 struct xhci_dev_ctx_addr *pdctxa;
275 uint64_t addr;
276 uint32_t temp;
277 uint16_t i;
278
279 DPRINTF("\n");
280
281 sc->sc_capa_off = 0;
282 sc->sc_oper_off = XREAD1(sc, capa, XHCI_CAPLENGTH);
283 sc->sc_runt_off = XREAD4(sc, capa, XHCI_RTSOFF) & ~0x1F;
284 sc->sc_door_off = XREAD4(sc, capa, XHCI_DBOFF) & ~0x3;
285
286 DPRINTF("CAPLENGTH=0x%x\n", sc->sc_oper_off);
287 DPRINTF("RUNTIMEOFFSET=0x%x\n", sc->sc_runt_off);
288 DPRINTF("DOOROFFSET=0x%x\n", sc->sc_door_off);
289
290 sc->sc_event_ccs = 1;
291 sc->sc_event_idx = 0;
292 sc->sc_command_ccs = 1;
293 sc->sc_command_idx = 0;
294
295 DPRINTF("xHCI version = 0x%04x\n", XREAD2(sc, capa, XHCI_HCIVERSION));
296
297 temp = XREAD4(sc, capa, XHCI_HCSPARAMS0);
298
299 DPRINTF("HCS0 = 0x%08x\n", temp);
300
301 if (XHCI_HCS0_CSZ(temp)) {
302 sc->sc_ctx_is_64_byte = 1;
303 device_printf(sc->sc_bus.parent, "64 byte context size.\n");
304 } else {
305 sc->sc_ctx_is_64_byte = 0;
306 device_printf(sc->sc_bus.parent, "32 byte context size.\n");
307 }
308
309 /* Reset controller */
310 XWRITE4(sc, oper, XHCI_USBCMD, XHCI_CMD_HCRST);
311
312 for (i = 0; i != 100; i++) {
313 usb_pause_mtx(NULL, hz / 100);
314 temp = XREAD4(sc, oper, XHCI_USBCMD) &
315 (XHCI_CMD_HCRST | XHCI_STS_CNR);
316 if (!temp)
317 break;
318 }
319
320 if (temp) {
321 device_printf(sc->sc_bus.parent, "Controller "
322 "reset timeout.\n");
323 return (USB_ERR_IOERROR);
324 }
325
326 if (!(XREAD4(sc, oper, XHCI_PAGESIZE) & XHCI_PAGESIZE_4K)) {
327 device_printf(sc->sc_bus.parent, "Controller does "
328 "not support 4K page size.\n");
329 return (USB_ERR_IOERROR);
330 }
331
332 temp = XREAD4(sc, capa, XHCI_HCSPARAMS1);
333
334 i = XHCI_HCS1_N_PORTS(temp);
335
336 if (i == 0) {
337 device_printf(sc->sc_bus.parent, "Invalid number "
338 "of ports: %u\n", i);
339 return (USB_ERR_IOERROR);
340 }
341
342 sc->sc_noport = i;
343 sc->sc_noslot = XHCI_HCS1_DEVSLOT_MAX(temp);
344
345 if (sc->sc_noslot > XHCI_MAX_DEVICES)
346 sc->sc_noslot = XHCI_MAX_DEVICES;
347
348 /* setup number of device slots */
349
350 DPRINTF("CONFIG=0x%08x -> 0x%08x\n",
351 XREAD4(sc, oper, XHCI_CONFIG), sc->sc_noslot);
352
353 XWRITE4(sc, oper, XHCI_CONFIG, sc->sc_noslot);
354
355 DPRINTF("Max slots: %u\n", sc->sc_noslot);
356
357 temp = XREAD4(sc, capa, XHCI_HCSPARAMS2);
358
359 sc->sc_noscratch = XHCI_HCS2_SPB_MAX(temp);
360
361 if (sc->sc_noscratch > XHCI_MAX_SCRATCHPADS) {
362 device_printf(sc->sc_bus.parent, "XHCI request "
363 "too many scratchpads\n");
364 return (USB_ERR_NOMEM);
365 }
366
367 DPRINTF("Max scratch: %u\n", sc->sc_noscratch);
368
369 temp = XREAD4(sc, capa, XHCI_HCSPARAMS3);
370
371 sc->sc_exit_lat_max = XHCI_HCS3_U1_DEL(temp) +
372 XHCI_HCS3_U2_DEL(temp) + 250 /* us */;
373
374 temp = XREAD4(sc, oper, XHCI_USBSTS);
375
376 /* clear interrupts */
377 XWRITE4(sc, oper, XHCI_USBSTS, temp);
378 /* disable all device notifications */
379 XWRITE4(sc, oper, XHCI_DNCTRL, 0);
380
381 /* setup device context base address */
382 usbd_get_page(&sc->sc_hw.ctx_pc, 0, &buf_res);
383 pdctxa = buf_res.buffer;
384 memset(pdctxa, 0, sizeof(*pdctxa));
385
386 addr = buf_res.physaddr;
387 addr += (uintptr_t)&((struct xhci_dev_ctx_addr *)0)->qwSpBufPtr[0];
388
389 /* slot 0 points to the table of scratchpad pointers */
390 pdctxa->qwBaaDevCtxAddr[0] = htole64(addr);
391
392 for (i = 0; i != sc->sc_noscratch; i++) {
393 struct usb_page_search buf_scp;
394 usbd_get_page(&sc->sc_hw.scratch_pc[i], 0, &buf_scp);
395 pdctxa->qwSpBufPtr[i] = htole64((uint64_t)buf_scp.physaddr);
396 }
397
398 addr = buf_res.physaddr;
399
400 XWRITE4(sc, oper, XHCI_DCBAAP_LO, (uint32_t)addr);
401 XWRITE4(sc, oper, XHCI_DCBAAP_HI, (uint32_t)(addr >> 32));
402 XWRITE4(sc, oper, XHCI_DCBAAP_LO, (uint32_t)addr);
403 XWRITE4(sc, oper, XHCI_DCBAAP_HI, (uint32_t)(addr >> 32));
404
405 /* Setup event table size */
406
407 temp = XREAD4(sc, capa, XHCI_HCSPARAMS2);
408
409 DPRINTF("HCS2=0x%08x\n", temp);
410
411 temp = XHCI_HCS2_ERST_MAX(temp);
412 temp = 1U << temp;
413 if (temp > XHCI_MAX_RSEG)
414 temp = XHCI_MAX_RSEG;
415
416 sc->sc_erst_max = temp;
417
418 DPRINTF("ERSTSZ=0x%08x -> 0x%08x\n",
419 XREAD4(sc, runt, XHCI_ERSTSZ(0)), temp);
420
421 XWRITE4(sc, runt, XHCI_ERSTSZ(0), XHCI_ERSTS_SET(temp));
422
423 /* Setup interrupt rate */
424 XWRITE4(sc, runt, XHCI_IMOD(0), XHCI_IMOD_DEFAULT);
425
426 usbd_get_page(&sc->sc_hw.root_pc, 0, &buf_res);
427
428 phwr = buf_res.buffer;
429 addr = buf_res.physaddr;
430 addr += (uintptr_t)&((struct xhci_hw_root *)0)->hwr_events[0];
431
432 /* reset hardware root structure */
433 memset(phwr, 0, sizeof(*phwr));
434
435 phwr->hwr_ring_seg[0].qwEvrsTablePtr = htole64(addr);
436 phwr->hwr_ring_seg[0].dwEvrsTableSize = htole32(XHCI_MAX_EVENTS);
437
438 DPRINTF("ERDP(0)=0x%016llx\n", (unsigned long long)addr);
439
440 XWRITE4(sc, runt, XHCI_ERDP_LO(0), (uint32_t)addr);
441 XWRITE4(sc, runt, XHCI_ERDP_HI(0), (uint32_t)(addr >> 32));
442
443 addr = (uint64_t)buf_res.physaddr;
444
445 DPRINTF("ERSTBA(0)=0x%016llx\n", (unsigned long long)addr);
446
447 XWRITE4(sc, runt, XHCI_ERSTBA_LO(0), (uint32_t)addr);
448 XWRITE4(sc, runt, XHCI_ERSTBA_HI(0), (uint32_t)(addr >> 32));
449
450 /* Setup interrupter registers */
451
452 temp = XREAD4(sc, runt, XHCI_IMAN(0));
453 temp |= XHCI_IMAN_INTR_ENA;
454 XWRITE4(sc, runt, XHCI_IMAN(0), temp);
455
456 /* setup command ring control base address */
457 addr = buf_res.physaddr;
458 addr += (uintptr_t)&((struct xhci_hw_root *)0)->hwr_commands[0];
459
460 DPRINTF("CRCR=0x%016llx\n", (unsigned long long)addr);
461
462 XWRITE4(sc, oper, XHCI_CRCR_LO, ((uint32_t)addr) | XHCI_CRCR_LO_RCS);
463 XWRITE4(sc, oper, XHCI_CRCR_HI, (uint32_t)(addr >> 32));
464
465 phwr->hwr_commands[XHCI_MAX_COMMANDS - 1].qwTrb0 = htole64(addr);
466
467 usb_bus_mem_flush_all(&sc->sc_bus, &xhci_iterate_hw_softc);
468
469 /* Go! */
470 XWRITE4(sc, oper, XHCI_USBCMD, XHCI_CMD_RS |
471 XHCI_CMD_INTE | XHCI_CMD_HSEE);
472
473 for (i = 0; i != 100; i++) {
474 usb_pause_mtx(NULL, hz / 100);
475 temp = XREAD4(sc, oper, XHCI_USBSTS) & XHCI_STS_HCH;
476 if (!temp)
477 break;
478 }
479 if (temp) {
480 XWRITE4(sc, oper, XHCI_USBCMD, 0);
481 device_printf(sc->sc_bus.parent, "Run timeout.\n");
482 return (USB_ERR_IOERROR);
483 }
484
485 /* catch any lost interrupts */
486 xhci_do_poll(&sc->sc_bus);
487
488 return (0);
489}
490
491usb_error_t
492xhci_halt_controller(struct xhci_softc *sc)
493{
494 uint32_t temp;
495 uint16_t i;
496
497 DPRINTF("\n");
498
499 sc->sc_capa_off = 0;
500 sc->sc_oper_off = XREAD1(sc, capa, XHCI_CAPLENGTH);
501 sc->sc_runt_off = XREAD4(sc, capa, XHCI_RTSOFF) & ~0xF;
502 sc->sc_door_off = XREAD4(sc, capa, XHCI_DBOFF) & ~0x3;
503
504 /* Halt controller */
505 XWRITE4(sc, oper, XHCI_USBCMD, 0);
506
507 for (i = 0; i != 100; i++) {
508 usb_pause_mtx(NULL, hz / 100);
509 temp = XREAD4(sc, oper, XHCI_USBSTS) & XHCI_STS_HCH;
510 if (temp)
511 break;
512 }
513
514 if (!temp) {
515 device_printf(sc->sc_bus.parent, "Controller halt timeout.\n");
516 return (USB_ERR_IOERROR);
517 }
518 return (0);
519}
520
521usb_error_t
522xhci_init(struct xhci_softc *sc, device_t self)
523{
524 /* initialise some bus fields */
525 sc->sc_bus.parent = self;
526
527 /* set the bus revision */
528 sc->sc_bus.usbrev = USB_REV_3_0;
529
530 /* set up the bus struct */
531 sc->sc_bus.methods = &xhci_bus_methods;
532
533 /* setup devices array */
534 sc->sc_bus.devices = sc->sc_devices;
535 sc->sc_bus.devices_max = XHCI_MAX_DEVICES;
536
537 /* setup command queue mutex and condition varible */
538 cv_init(&sc->sc_cmd_cv, "CMDQ");
539 sx_init(&sc->sc_cmd_sx, "CMDQ lock");
540
541 /* get all DMA memory */
542 if (usb_bus_mem_alloc_all(&sc->sc_bus,
543 USB_GET_DMA_TAG(self), &xhci_iterate_hw_softc)) {
544 return (ENOMEM);
545 }
546
547 sc->sc_config_msg[0].hdr.pm_callback = &xhci_configure_msg;
548 sc->sc_config_msg[0].bus = &sc->sc_bus;
549 sc->sc_config_msg[1].hdr.pm_callback = &xhci_configure_msg;
550 sc->sc_config_msg[1].bus = &sc->sc_bus;
551
552 return (0);
553}
554
555void
556xhci_uninit(struct xhci_softc *sc)
557{
558 /*
559 * NOTE: At this point the control transfer process is gone
560 * and "xhci_configure_msg" is no longer called. Consequently
561 * waiting for the configuration messages to complete is not
562 * needed.
563 */
564 usb_bus_mem_free_all(&sc->sc_bus, &xhci_iterate_hw_softc);
565
566 cv_destroy(&sc->sc_cmd_cv);
567 sx_destroy(&sc->sc_cmd_sx);
568}
569
570static void
571xhci_set_hw_power_sleep(struct usb_bus *bus, uint32_t state)
572{
573 struct xhci_softc *sc = XHCI_BUS2SC(bus);
574
575 switch (state) {
576 case USB_HW_POWER_SUSPEND:
577 DPRINTF("Stopping the XHCI\n");
578 xhci_halt_controller(sc);
579 break;
580 case USB_HW_POWER_SHUTDOWN:
581 DPRINTF("Stopping the XHCI\n");
582 xhci_halt_controller(sc);
583 break;
584 case USB_HW_POWER_RESUME:
585 DPRINTF("Starting the XHCI\n");
586 xhci_start_controller(sc);
587 break;
588 default:
589 break;
590 }
591}
592
593static usb_error_t
594xhci_generic_done_sub(struct usb_xfer *xfer)
595{
596 struct xhci_td *td;
597 struct xhci_td *td_alt_next;
598 uint32_t len;
599 uint8_t status;
600
601 td = xfer->td_transfer_cache;
602 td_alt_next = td->alt_next;
603
604 if (xfer->aframes != xfer->nframes)
605 usbd_xfer_set_frame_len(xfer, xfer->aframes, 0);
606
607 while (1) {
608
609 usb_pc_cpu_invalidate(td->page_cache);
610
611 status = td->status;
612 len = td->remainder;
613
614 DPRINTFN(4, "xfer=%p[%u/%u] rem=%u/%u status=%u\n",
615 xfer, (unsigned int)xfer->aframes,
616 (unsigned int)xfer->nframes,
617 (unsigned int)len, (unsigned int)td->len,
618 (unsigned int)status);
619
620 /*
621 * Verify the status length and
622 * add the length to "frlengths[]":
623 */
624 if (len > td->len) {
625 /* should not happen */
626 DPRINTF("Invalid status length, "
627 "0x%04x/0x%04x bytes\n", len, td->len);
628 status = XHCI_TRB_ERROR_LENGTH;
629 } else if (xfer->aframes != xfer->nframes) {
630 xfer->frlengths[xfer->aframes] += td->len - len;
631 }
632 /* Check for last transfer */
633 if (((void *)td) == xfer->td_transfer_last) {
634 td = NULL;
635 break;
636 }
637 /* Check for transfer error */
638 if (status != XHCI_TRB_ERROR_SHORT_PKT &&
639 status != XHCI_TRB_ERROR_SUCCESS) {
640 /* the transfer is finished */
641 td = NULL;
642 break;
643 }
644 /* Check for short transfer */
645 if (len > 0) {
646 if (xfer->flags_int.short_frames_ok ||
647 xfer->flags_int.isochronous_xfr ||
648 xfer->flags_int.control_xfr) {
649 /* follow alt next */
650 td = td->alt_next;
651 } else {
652 /* the transfer is finished */
653 td = NULL;
654 }
655 break;
656 }
657 td = td->obj_next;
658
659 if (td->alt_next != td_alt_next) {
660 /* this USB frame is complete */
661 break;
662 }
663 }
664
665 /* update transfer cache */
666
667 xfer->td_transfer_cache = td;
668
669 return ((status == XHCI_TRB_ERROR_STALL) ? USB_ERR_STALLED :
670 (status != XHCI_TRB_ERROR_SHORT_PKT &&
671 status != XHCI_TRB_ERROR_SUCCESS) ? USB_ERR_IOERROR :
672 USB_ERR_NORMAL_COMPLETION);
673}
674
675static void
676xhci_generic_done(struct usb_xfer *xfer)
677{
678 usb_error_t err = 0;
679
680 DPRINTFN(13, "xfer=%p endpoint=%p transfer done\n",
681 xfer, xfer->endpoint);
682
683 /* reset scanner */
684
685 xfer->td_transfer_cache = xfer->td_transfer_first;
686
687 if (xfer->flags_int.control_xfr) {
688
689 if (xfer->flags_int.control_hdr)
690 err = xhci_generic_done_sub(xfer);
691
692 xfer->aframes = 1;
693
694 if (xfer->td_transfer_cache == NULL)
695 goto done;
696 }
697
698 while (xfer->aframes != xfer->nframes) {
699
700 err = xhci_generic_done_sub(xfer);
701 xfer->aframes++;
702
703 if (xfer->td_transfer_cache == NULL)
704 goto done;
705 }
706
707 if (xfer->flags_int.control_xfr &&
708 !xfer->flags_int.control_act)
709 err = xhci_generic_done_sub(xfer);
710done:
711 /* transfer is complete */
712 xhci_device_done(xfer, err);
713}
714
715static void
716xhci_activate_transfer(struct usb_xfer *xfer)
717{
718 struct xhci_td *td;
719
720 td = xfer->td_transfer_cache;
721
722 usb_pc_cpu_invalidate(td->page_cache);
723
724 if (!(td->td_trb[0].dwTrb3 & htole32(XHCI_TRB_3_CYCLE_BIT))) {
725
726 /* activate the transfer */
727
728 td->td_trb[0].dwTrb3 |= htole32(XHCI_TRB_3_CYCLE_BIT);
729 usb_pc_cpu_flush(td->page_cache);
730
731 xhci_endpoint_doorbell(xfer);
732 }
733}
734
735static void
736xhci_skip_transfer(struct usb_xfer *xfer)
737{
738 struct xhci_td *td;
739 struct xhci_td *td_last;
740
741 td = xfer->td_transfer_cache;
742 td_last = xfer->td_transfer_last;
743
744 td = td->alt_next;
745
746 usb_pc_cpu_invalidate(td->page_cache);
747
748 if (!(td->td_trb[0].dwTrb3 & htole32(XHCI_TRB_3_CYCLE_BIT))) {
749
750 usb_pc_cpu_invalidate(td_last->page_cache);
751
752 /* copy LINK TRB to current waiting location */
753
754 td->td_trb[0].qwTrb0 = td_last->td_trb[td_last->ntrb].qwTrb0;
755 td->td_trb[0].dwTrb2 = td_last->td_trb[td_last->ntrb].dwTrb2;
756 usb_pc_cpu_flush(td->page_cache);
757
758 td->td_trb[0].dwTrb3 = td_last->td_trb[td_last->ntrb].dwTrb3;
759 usb_pc_cpu_flush(td->page_cache);
760
761 xhci_endpoint_doorbell(xfer);
762 }
763}
764
765/*------------------------------------------------------------------------*
766 * xhci_check_transfer
767 *------------------------------------------------------------------------*/
768static void
769xhci_check_transfer(struct xhci_softc *sc, struct xhci_trb *trb)
770{
771 struct xhci_endpoint_ext *pepext;
772 int64_t offset;
773 uint64_t td_event;
774 uint32_t temp;
775 uint32_t remainder;
776 uint16_t stream_id;
777 uint16_t i;
778 uint8_t status;
779 uint8_t halted;
780 uint8_t epno;
781 uint8_t index;
782
783 /* decode TRB */
784 td_event = le64toh(trb->qwTrb0);
785 temp = le32toh(trb->dwTrb2);
786
787 remainder = XHCI_TRB_2_REM_GET(temp);
788 status = XHCI_TRB_2_ERROR_GET(temp);
789 stream_id = XHCI_TRB_2_STREAM_GET(temp);
790
791 temp = le32toh(trb->dwTrb3);
792 epno = XHCI_TRB_3_EP_GET(temp);
793 index = XHCI_TRB_3_SLOT_GET(temp);
794
795 /* check if error means halted */
796 halted = (status != XHCI_TRB_ERROR_SHORT_PKT &&
797 status != XHCI_TRB_ERROR_SUCCESS);
798
799 DPRINTF("slot=%u epno=%u stream=%u remainder=%u status=%u\n",
800 index, epno, stream_id, remainder, status);
801
802 if (index > sc->sc_noslot) {
803 DPRINTF("Invalid slot.\n");
804 return;
805 }
806
807 if ((epno == 0) || (epno >= XHCI_MAX_ENDPOINTS)) {
808 DPRINTF("Invalid endpoint.\n");
809 return;
810 }
811
812 pepext = &sc->sc_hw.devs[index].endp[epno];
813
814 if (pepext->trb_ep_mode != USB_EP_MODE_STREAMS) {
815 stream_id = 0;
816 DPRINTF("stream_id=0\n");
817 } else if (stream_id >= XHCI_MAX_STREAMS) {
818 DPRINTF("Invalid stream ID.\n");
819 return;
820 }
821
822 /* try to find the USB transfer that generated the event */
823 for (i = 0; i != (XHCI_MAX_TRANSFERS - 1); i++) {
824 struct usb_xfer *xfer;
825 struct xhci_td *td;
826
827 xfer = pepext->xfer[i + (XHCI_MAX_TRANSFERS * stream_id)];
828 if (xfer == NULL)
829 continue;
830
831 td = xfer->td_transfer_cache;
832
833 DPRINTFN(5, "Checking if 0x%016llx == (0x%016llx .. 0x%016llx)\n",
834 (long long)td_event,
835 (long long)td->td_self,
836 (long long)td->td_self + sizeof(td->td_trb));
837
838 /*
839 * NOTE: Some XHCI implementations might not trigger
840 * an event on the last LINK TRB so we need to
841 * consider both the last and second last event
842 * address as conditions for a successful transfer.
843 *
844 * NOTE: We assume that the XHCI will only trigger one
845 * event per chain of TRBs.
846 */
847
848 offset = td_event - td->td_self;
849
850 if (offset >= 0 &&
851 offset < (int64_t)sizeof(td->td_trb)) {
852
853 usb_pc_cpu_invalidate(td->page_cache);
854
855 /* compute rest of remainder, if any */
856 for (i = (offset / 16) + 1; i < td->ntrb; i++) {
857 temp = le32toh(td->td_trb[i].dwTrb2);
858 remainder += XHCI_TRB_2_BYTES_GET(temp);
859 }
860
861 DPRINTFN(5, "New remainder: %u\n", remainder);
862
863 /* clear isochronous transfer errors */
864 if (xfer->flags_int.isochronous_xfr) {
865 if (halted) {
866 halted = 0;
867 status = XHCI_TRB_ERROR_SUCCESS;
868 remainder = td->len;
869 }
870 }
871
872 /* "td->remainder" is verified later */
873 td->remainder = remainder;
874 td->status = status;
875
876 usb_pc_cpu_flush(td->page_cache);
877
878 /*
879 * 1) Last transfer descriptor makes the
880 * transfer done
881 */
882 if (((void *)td) == xfer->td_transfer_last) {
883 DPRINTF("TD is last\n");
884 xhci_generic_done(xfer);
885 break;
886 }
887
888 /*
889 * 2) Any kind of error makes the transfer
890 * done
891 */
892 if (halted) {
893 DPRINTF("TD has I/O error\n");
894 xhci_generic_done(xfer);
895 break;
896 }
897
898 /*
899 * 3) If there is no alternate next transfer,
900 * a short packet also makes the transfer done
901 */
902 if (td->remainder > 0) {
903 if (td->alt_next == NULL) {
904 DPRINTF(
905 "short TD has no alternate next\n");
906 xhci_generic_done(xfer);
907 break;
908 }
909 DPRINTF("TD has short pkt\n");
910 if (xfer->flags_int.short_frames_ok ||
911 xfer->flags_int.isochronous_xfr ||
912 xfer->flags_int.control_xfr) {
913 /* follow the alt next */
914 xfer->td_transfer_cache = td->alt_next;
915 xhci_activate_transfer(xfer);
916 break;
917 }
918 xhci_skip_transfer(xfer);
919 xhci_generic_done(xfer);
920 break;
921 }
922
923 /*
924 * 4) Transfer complete - go to next TD
925 */
926 DPRINTF("Following next TD\n");
927 xfer->td_transfer_cache = td->obj_next;
928 xhci_activate_transfer(xfer);
929 break; /* there should only be one match */
930 }
931 }
932}
933
934static void
935xhci_check_command(struct xhci_softc *sc, struct xhci_trb *trb)
936{
937 if (sc->sc_cmd_addr == trb->qwTrb0) {
938 DPRINTF("Received command event\n");
939 sc->sc_cmd_result[0] = trb->dwTrb2;
940 sc->sc_cmd_result[1] = trb->dwTrb3;
941 cv_signal(&sc->sc_cmd_cv);
942 }
943}
944
945static void
946xhci_interrupt_poll(struct xhci_softc *sc)
947{
948 struct usb_page_search buf_res;
949 struct xhci_hw_root *phwr;
950 uint64_t addr;
951 uint32_t temp;
952 uint16_t i;
953 uint8_t event;
954 uint8_t j;
955 uint8_t k;
956 uint8_t t;
957
958 usbd_get_page(&sc->sc_hw.root_pc, 0, &buf_res);
959
960 phwr = buf_res.buffer;
961
962 /* Receive any events */
963
964 usb_pc_cpu_invalidate(&sc->sc_hw.root_pc);
965
966 i = sc->sc_event_idx;
967 j = sc->sc_event_ccs;
968 t = 2;
969
970 while (1) {
971
972 temp = le32toh(phwr->hwr_events[i].dwTrb3);
973
974 k = (temp & XHCI_TRB_3_CYCLE_BIT) ? 1 : 0;
975
976 if (j != k)
977 break;
978
979 event = XHCI_TRB_3_TYPE_GET(temp);
980
981 DPRINTFN(10, "event[%u] = %u (0x%016llx 0x%08lx 0x%08lx)\n",
982 i, event, (long long)le64toh(phwr->hwr_events[i].qwTrb0),
983 (long)le32toh(phwr->hwr_events[i].dwTrb2),
984 (long)le32toh(phwr->hwr_events[i].dwTrb3));
985
986 switch (event) {
987 case XHCI_TRB_EVENT_TRANSFER:
988 xhci_check_transfer(sc, &phwr->hwr_events[i]);
989 break;
990 case XHCI_TRB_EVENT_CMD_COMPLETE:
991 xhci_check_command(sc, &phwr->hwr_events[i]);
992 break;
993 default:
994 DPRINTF("Unhandled event = %u\n", event);
995 break;
996 }
997
998 i++;
999
1000 if (i == XHCI_MAX_EVENTS) {
1001 i = 0;
1002 j ^= 1;
1003
1004 /* check for timeout */
1005 if (!--t)
1006 break;
1007 }
1008 }
1009
1010 sc->sc_event_idx = i;
1011 sc->sc_event_ccs = j;
1012
1013 /*
1014 * NOTE: The Event Ring Dequeue Pointer Register is 64-bit
1015 * latched. That means to activate the register we need to
1016 * write both the low and high double word of the 64-bit
1017 * register.
1018 */
1019
1020 addr = (uint32_t)buf_res.physaddr;
1021 addr += (uintptr_t)&((struct xhci_hw_root *)0)->hwr_events[i];
1022
1023 /* try to clear busy bit */
1024 addr |= XHCI_ERDP_LO_BUSY;
1025
1026 XWRITE4(sc, runt, XHCI_ERDP_LO(0), (uint32_t)addr);
1027 XWRITE4(sc, runt, XHCI_ERDP_HI(0), (uint32_t)(addr >> 32));
1028}
1029
1030static usb_error_t
1031xhci_do_command(struct xhci_softc *sc, struct xhci_trb *trb,
1032 uint16_t timeout_ms)
1033{
1034 struct usb_page_search buf_res;
1035 struct xhci_hw_root *phwr;
1036 uint64_t addr;
1037 uint32_t temp;
1038 uint8_t i;
1039 uint8_t j;
1040 int err;
1041
1042 XHCI_CMD_ASSERT_LOCKED(sc);
1043
1044 /* get hardware root structure */
1045
1046 usbd_get_page(&sc->sc_hw.root_pc, 0, &buf_res);
1047
1048 phwr = buf_res.buffer;
1049
1050 /* Queue command */
1051
1052 USB_BUS_LOCK(&sc->sc_bus);
1053
1054 i = sc->sc_command_idx;
1055 j = sc->sc_command_ccs;
1056
1057 DPRINTFN(10, "command[%u] = %u (0x%016llx, 0x%08lx, 0x%08lx)\n",
1058 i, XHCI_TRB_3_TYPE_GET(le32toh(trb->dwTrb3)),
1059 (long long)le64toh(trb->qwTrb0),
1060 (long)le32toh(trb->dwTrb2),
1061 (long)le32toh(trb->dwTrb3));
1062
1063 phwr->hwr_commands[i].qwTrb0 = trb->qwTrb0;
1064 phwr->hwr_commands[i].dwTrb2 = trb->dwTrb2;
1065
1066 usb_pc_cpu_flush(&sc->sc_hw.root_pc);
1067
1068 temp = trb->dwTrb3;
1069
1070 if (j)
1071 temp |= htole32(XHCI_TRB_3_CYCLE_BIT);
1072 else
1073 temp &= ~htole32(XHCI_TRB_3_CYCLE_BIT);
1074
1075 temp &= ~htole32(XHCI_TRB_3_TC_BIT);
1076
1077 phwr->hwr_commands[i].dwTrb3 = temp;
1078
1079 usb_pc_cpu_flush(&sc->sc_hw.root_pc);
1080
1081 addr = buf_res.physaddr;
1082 addr += (uintptr_t)&((struct xhci_hw_root *)0)->hwr_commands[i];
1083
1084 sc->sc_cmd_addr = htole64(addr);
1085
1086 i++;
1087
1088 if (i == (XHCI_MAX_COMMANDS - 1)) {
1089
1090 if (j) {
1091 temp = htole32(XHCI_TRB_3_TC_BIT |
1092 XHCI_TRB_3_TYPE_SET(XHCI_TRB_TYPE_LINK) |
1093 XHCI_TRB_3_CYCLE_BIT);
1094 } else {
1095 temp = htole32(XHCI_TRB_3_TC_BIT |
1096 XHCI_TRB_3_TYPE_SET(XHCI_TRB_TYPE_LINK));
1097 }
1098
1099 phwr->hwr_commands[i].dwTrb3 = temp;
1100
1101 usb_pc_cpu_flush(&sc->sc_hw.root_pc);
1102
1103 i = 0;
1104 j ^= 1;
1105 }
1106
1107 sc->sc_command_idx = i;
1108 sc->sc_command_ccs = j;
1109
1110 XWRITE4(sc, door, XHCI_DOORBELL(0), 0);
1111
1112 err = cv_timedwait(&sc->sc_cmd_cv, &sc->sc_bus.bus_mtx,
1113 USB_MS_TO_TICKS(timeout_ms));
1114
1115 if (err) {
1116 DPRINTFN(0, "Command timeout!\n");
1117 err = USB_ERR_TIMEOUT;
1118 trb->dwTrb2 = 0;
1119 trb->dwTrb3 = 0;
1120 } else {
1121 temp = le32toh(sc->sc_cmd_result[0]);
1122 if (XHCI_TRB_2_ERROR_GET(temp) != XHCI_TRB_ERROR_SUCCESS)
1123 err = USB_ERR_IOERROR;
1124
1125 trb->dwTrb2 = sc->sc_cmd_result[0];
1126 trb->dwTrb3 = sc->sc_cmd_result[1];
1127 }
1128
1129 USB_BUS_UNLOCK(&sc->sc_bus);
1130
1131 return (err);
1132}
1133
1134#if 0
1135static usb_error_t
1136xhci_cmd_nop(struct xhci_softc *sc)
1137{
1138 struct xhci_trb trb;
1139 uint32_t temp;
1140
1141 DPRINTF("\n");
1142
1143 trb.qwTrb0 = 0;
1144 trb.dwTrb2 = 0;
1145 temp = XHCI_TRB_3_TYPE_SET(XHCI_TRB_TYPE_NOOP);
1146
1147 trb.dwTrb3 = htole32(temp);
1148
1149 return (xhci_do_command(sc, &trb, 100 /* ms */));
1150}
1151#endif
1152
1153static usb_error_t
1154xhci_cmd_enable_slot(struct xhci_softc *sc, uint8_t *pslot)
1155{
1156 struct xhci_trb trb;
1157 uint32_t temp;
1158 usb_error_t err;
1159
1160 DPRINTF("\n");
1161
1162 trb.qwTrb0 = 0;
1163 trb.dwTrb2 = 0;
1164 trb.dwTrb3 = htole32(XHCI_TRB_3_TYPE_SET(XHCI_TRB_TYPE_ENABLE_SLOT));
1165
1166 err = xhci_do_command(sc, &trb, 100 /* ms */);
1167 if (err)
1168 goto done;
1169
1170 temp = le32toh(trb.dwTrb3);
1171
1172 *pslot = XHCI_TRB_3_SLOT_GET(temp);
1173
1174done:
1175 return (err);
1176}
1177
1178static usb_error_t
1179xhci_cmd_disable_slot(struct xhci_softc *sc, uint8_t slot_id)
1180{
1181 struct xhci_trb trb;
1182 uint32_t temp;
1183
1184 DPRINTF("\n");
1185
1186 trb.qwTrb0 = 0;
1187 trb.dwTrb2 = 0;
1188 temp = XHCI_TRB_3_TYPE_SET(XHCI_TRB_TYPE_DISABLE_SLOT) |
1189 XHCI_TRB_3_SLOT_SET(slot_id);
1190
1191 trb.dwTrb3 = htole32(temp);
1192
1193 return (xhci_do_command(sc, &trb, 100 /* ms */));
1194}
1195
1196static usb_error_t
1197xhci_cmd_set_address(struct xhci_softc *sc, uint64_t input_ctx,
1198 uint8_t bsr, uint8_t slot_id)
1199{
1200 struct xhci_trb trb;
1201 uint32_t temp;
1202
1203 DPRINTF("\n");
1204
1205 trb.qwTrb0 = htole64(input_ctx);
1206 trb.dwTrb2 = 0;
1207 temp = XHCI_TRB_3_TYPE_SET(XHCI_TRB_TYPE_ADDRESS_DEVICE) |
1208 XHCI_TRB_3_SLOT_SET(slot_id);
1209
1210 if (bsr)
1211 temp |= XHCI_TRB_3_BSR_BIT;
1212
1213 trb.dwTrb3 = htole32(temp);
1214
1215 return (xhci_do_command(sc, &trb, 500 /* ms */));
1216}
1217
1218static usb_error_t
1219xhci_set_address(struct usb_device *udev, struct mtx *mtx, uint16_t address)
1220{
1221 struct usb_page_search buf_inp;
1222 struct usb_page_search buf_dev;
1223 struct xhci_softc *sc = XHCI_BUS2SC(udev->bus);
1224 struct xhci_hw_dev *hdev;
1225 struct xhci_dev_ctx *pdev;
1226 struct xhci_endpoint_ext *pepext;
1227 uint32_t temp;
1228 uint16_t mps;
1229 usb_error_t err;
1230 uint8_t index;
1231
1232 /* the root HUB case is not handled here */
1233 if (udev->parent_hub == NULL)
1234 return (USB_ERR_INVAL);
1235
1236 index = udev->controller_slot_id;
1237
1238 hdev = &sc->sc_hw.devs[index];
1239
1240 if (mtx != NULL)
1241 mtx_unlock(mtx);
1242
1243 XHCI_CMD_LOCK(sc);
1244
1245 switch (hdev->state) {
1246 case XHCI_ST_DEFAULT:
1247 case XHCI_ST_ENABLED:
1248
1249 hdev->state = XHCI_ST_ENABLED;
1250
1251 /* set configure mask to slot and EP0 */
1252 xhci_configure_mask(udev, 3, 0);
1253
1254 /* configure input slot context structure */
1255 err = xhci_configure_device(udev);
1256
1257 if (err != 0) {
1258 DPRINTF("Could not configure device\n");
1259 break;
1260 }
1261
1262 /* configure input endpoint context structure */
1263 switch (udev->speed) {
1264 case USB_SPEED_LOW:
1265 case USB_SPEED_FULL:
1266 mps = 8;
1267 break;
1268 case USB_SPEED_HIGH:
1269 mps = 64;
1270 break;
1271 default:
1272 mps = 512;
1273 break;
1274 }
1275
1276 pepext = xhci_get_endpoint_ext(udev,
1277 &udev->ctrl_ep_desc);
1278 err = xhci_configure_endpoint(udev,
1279 &udev->ctrl_ep_desc, pepext,
1280 0, 1, 1, 0, mps, mps, USB_EP_MODE_DEFAULT);
1281
1282 if (err != 0) {
1283 DPRINTF("Could not configure default endpoint\n");
1284 break;
1285 }
1286
1287 /* execute set address command */
1288 usbd_get_page(&hdev->input_pc, 0, &buf_inp);
1289
1290 err = xhci_cmd_set_address(sc, buf_inp.physaddr,
1291 (address == 0), index);
1292
1293 if (err != 0) {
1294 DPRINTF("Could not set address "
1295 "for slot %u.\n", index);
1296 if (address != 0)
1297 break;
1298 }
1299
1300 /* update device address to new value */
1301
1302 usbd_get_page(&hdev->device_pc, 0, &buf_dev);
1303 pdev = buf_dev.buffer;
1304 usb_pc_cpu_invalidate(&hdev->device_pc);
1305
1306 temp = xhci_ctx_get_le32(sc, &pdev->ctx_slot.dwSctx3);
1307 udev->address = XHCI_SCTX_3_DEV_ADDR_GET(temp);
1308
1309 /* update device state to new value */
1310
1311 if (address != 0)
1312 hdev->state = XHCI_ST_ADDRESSED;
1313 else
1314 hdev->state = XHCI_ST_DEFAULT;
1315 break;
1316
1317 default:
1318 DPRINTF("Wrong state for set address.\n");
1319 err = USB_ERR_IOERROR;
1320 break;
1321 }
1322 XHCI_CMD_UNLOCK(sc);
1323
1324 if (mtx != NULL)
1325 mtx_lock(mtx);
1326
1327 return (err);
1328}
1329
1330static usb_error_t
1331xhci_cmd_configure_ep(struct xhci_softc *sc, uint64_t input_ctx,
1332 uint8_t deconfigure, uint8_t slot_id)
1333{
1334 struct xhci_trb trb;
1335 uint32_t temp;
1336
1337 DPRINTF("\n");
1338
1339 trb.qwTrb0 = htole64(input_ctx);
1340 trb.dwTrb2 = 0;
1341 temp = XHCI_TRB_3_TYPE_SET(XHCI_TRB_TYPE_CONFIGURE_EP) |
1342 XHCI_TRB_3_SLOT_SET(slot_id);
1343
1344 if (deconfigure)
1345 temp |= XHCI_TRB_3_DCEP_BIT;
1346
1347 trb.dwTrb3 = htole32(temp);
1348
1349 return (xhci_do_command(sc, &trb, 100 /* ms */));
1350}
1351
1352static usb_error_t
1353xhci_cmd_evaluate_ctx(struct xhci_softc *sc, uint64_t input_ctx,
1354 uint8_t slot_id)
1355{
1356 struct xhci_trb trb;
1357 uint32_t temp;
1358
1359 DPRINTF("\n");
1360
1361 trb.qwTrb0 = htole64(input_ctx);
1362 trb.dwTrb2 = 0;
1363 temp = XHCI_TRB_3_TYPE_SET(XHCI_TRB_TYPE_EVALUATE_CTX) |
1364 XHCI_TRB_3_SLOT_SET(slot_id);
1365 trb.dwTrb3 = htole32(temp);
1366
1367 return (xhci_do_command(sc, &trb, 100 /* ms */));
1368}
1369
1370static usb_error_t
1371xhci_cmd_reset_ep(struct xhci_softc *sc, uint8_t preserve,
1372 uint8_t ep_id, uint8_t slot_id)
1373{
1374 struct xhci_trb trb;
1375 uint32_t temp;
1376
1377 DPRINTF("\n");
1378
1379 trb.qwTrb0 = 0;
1380 trb.dwTrb2 = 0;
1381 temp = XHCI_TRB_3_TYPE_SET(XHCI_TRB_TYPE_RESET_EP) |
1382 XHCI_TRB_3_SLOT_SET(slot_id) |
1383 XHCI_TRB_3_EP_SET(ep_id);
1384
1385 if (preserve)
1386 temp |= XHCI_TRB_3_PRSV_BIT;
1387
1388 trb.dwTrb3 = htole32(temp);
1389
1390 return (xhci_do_command(sc, &trb, 100 /* ms */));
1391}
1392
1393static usb_error_t
1394xhci_cmd_set_tr_dequeue_ptr(struct xhci_softc *sc, uint64_t dequeue_ptr,
1395 uint16_t stream_id, uint8_t ep_id, uint8_t slot_id)
1396{
1397 struct xhci_trb trb;
1398 uint32_t temp;
1399
1400 DPRINTF("\n");
1401
1402 trb.qwTrb0 = htole64(dequeue_ptr);
1403
1404 temp = XHCI_TRB_2_STREAM_SET(stream_id);
1405 trb.dwTrb2 = htole32(temp);
1406
1407 temp = XHCI_TRB_3_TYPE_SET(XHCI_TRB_TYPE_SET_TR_DEQUEUE) |
1408 XHCI_TRB_3_SLOT_SET(slot_id) |
1409 XHCI_TRB_3_EP_SET(ep_id);
1410 trb.dwTrb3 = htole32(temp);
1411
1412 return (xhci_do_command(sc, &trb, 100 /* ms */));
1413}
1414
1415static usb_error_t
1416xhci_cmd_stop_ep(struct xhci_softc *sc, uint8_t suspend,
1417 uint8_t ep_id, uint8_t slot_id)
1418{
1419 struct xhci_trb trb;
1420 uint32_t temp;
1421
1422 DPRINTF("\n");
1423
1424 trb.qwTrb0 = 0;
1425 trb.dwTrb2 = 0;
1426 temp = XHCI_TRB_3_TYPE_SET(XHCI_TRB_TYPE_STOP_EP) |
1427 XHCI_TRB_3_SLOT_SET(slot_id) |
1428 XHCI_TRB_3_EP_SET(ep_id);
1429
1430 if (suspend)
1431 temp |= XHCI_TRB_3_SUSP_EP_BIT;
1432
1433 trb.dwTrb3 = htole32(temp);
1434
1435 return (xhci_do_command(sc, &trb, 100 /* ms */));
1436}
1437
1438static usb_error_t
1439xhci_cmd_reset_dev(struct xhci_softc *sc, uint8_t slot_id)
1440{
1441 struct xhci_trb trb;
1442 uint32_t temp;
1443
1444 DPRINTF("\n");
1445
1446 trb.qwTrb0 = 0;
1447 trb.dwTrb2 = 0;
1448 temp = XHCI_TRB_3_TYPE_SET(XHCI_TRB_TYPE_RESET_DEVICE) |
1449 XHCI_TRB_3_SLOT_SET(slot_id);
1450
1451 trb.dwTrb3 = htole32(temp);
1452
1453 return (xhci_do_command(sc, &trb, 100 /* ms */));
1454}
1455
1456/*------------------------------------------------------------------------*
1457 * xhci_interrupt - XHCI interrupt handler
1458 *------------------------------------------------------------------------*/
1459void
1460xhci_interrupt(struct xhci_softc *sc)
1461{
1462 uint32_t status;
1463 uint32_t iman;
1464
1465 USB_BUS_LOCK(&sc->sc_bus);
1466
1467 status = XREAD4(sc, oper, XHCI_USBSTS);
1468 if (status == 0)
1469 goto done;
1470
1471 /* acknowledge interrupts */
1472
1473 XWRITE4(sc, oper, XHCI_USBSTS, status);
1474
1475 DPRINTFN(16, "real interrupt (status=0x%08x)\n", status);
1476
1477 if (status & XHCI_STS_EINT) {
1478
1479 /* acknowledge pending event */
1480 iman = XREAD4(sc, runt, XHCI_IMAN(0));
1481
1482 /* reset interrupt */
1483 XWRITE4(sc, runt, XHCI_IMAN(0), iman);
1484
1485 DPRINTFN(16, "real interrupt (iman=0x%08x)\n", iman);
1486
1487 /* check for event(s) */
1488 xhci_interrupt_poll(sc);
1489 }
1490
1491 if (status & (XHCI_STS_PCD | XHCI_STS_HCH |
1492 XHCI_STS_HSE | XHCI_STS_HCE)) {
1493
1494 if (status & XHCI_STS_PCD) {
1495 xhci_root_intr(sc);
1496 }
1497
1498 if (status & XHCI_STS_HCH) {
1499 printf("%s: host controller halted\n",
1500 __FUNCTION__);
1501 }
1502
1503 if (status & XHCI_STS_HSE) {
1504 printf("%s: host system error\n",
1505 __FUNCTION__);
1506 }
1507
1508 if (status & XHCI_STS_HCE) {
1509 printf("%s: host controller error\n",
1510 __FUNCTION__);
1511 }
1512 }
1513done:
1514 USB_BUS_UNLOCK(&sc->sc_bus);
1515}
1516
1517/*------------------------------------------------------------------------*
1518 * xhci_timeout - XHCI timeout handler
1519 *------------------------------------------------------------------------*/
1520static void
1521xhci_timeout(void *arg)
1522{
1523 struct usb_xfer *xfer = arg;
1524
1525 DPRINTF("xfer=%p\n", xfer);
1526
1527 USB_BUS_LOCK_ASSERT(xfer->xroot->bus, MA_OWNED);
1528
1529 /* transfer is transferred */
1530 xhci_device_done(xfer, USB_ERR_TIMEOUT);
1531}
1532
1533static void
1534xhci_do_poll(struct usb_bus *bus)
1535{
1536 struct xhci_softc *sc = XHCI_BUS2SC(bus);
1537
1538 USB_BUS_LOCK(&sc->sc_bus);
1539 xhci_interrupt_poll(sc);
1540 USB_BUS_UNLOCK(&sc->sc_bus);
1541}
1542
1543static void
1544xhci_setup_generic_chain_sub(struct xhci_std_temp *temp)
1545{
1546 struct usb_page_search buf_res;
1547 struct xhci_td *td;
|
| 1548 struct xhci_td *td_first;
|
1548 struct xhci_td *td_next;
1549 struct xhci_td *td_alt_next;
1550 uint32_t buf_offset;
1551 uint32_t average;
1552 uint32_t len_old;
1553 uint32_t dword;
1554 uint8_t shortpkt_old;
1555 uint8_t precompute;
1556 uint8_t x;
1557
1558 td_alt_next = NULL;
1559 buf_offset = 0;
1560 shortpkt_old = temp->shortpkt;
1561 len_old = temp->len;
1562 precompute = 1;
1563
1564restart:
1565
1566 td = temp->td;
| 1549 struct xhci_td *td_next;
1550 struct xhci_td *td_alt_next;
1551 uint32_t buf_offset;
1552 uint32_t average;
1553 uint32_t len_old;
1554 uint32_t dword;
1555 uint8_t shortpkt_old;
1556 uint8_t precompute;
1557 uint8_t x;
1558
1559 td_alt_next = NULL;
1560 buf_offset = 0;
1561 shortpkt_old = temp->shortpkt;
1562 len_old = temp->len;
1563 precompute = 1;
1564
1565restart:
1566
1567 td = temp->td;
|
1567 td_next = temp->td_next;
| 1568 td_next = td_first = temp->td_next;
|
1568
1569 while (1) {
1570
1571 if (temp->len == 0) {
1572
1573 if (temp->shortpkt)
1574 break;
1575
1576 /* send a Zero Length Packet, ZLP, last */
1577
1578 temp->shortpkt = 1;
1579 average = 0;
1580
1581 } else {
1582
1583 average = temp->average;
1584
1585 if (temp->len < average) {
1586 if (temp->len % temp->max_packet_size) {
1587 temp->shortpkt = 1;
1588 }
1589 average = temp->len;
1590 }
1591 }
1592
1593 if (td_next == NULL)
1594 panic("%s: out of XHCI transfer descriptors!", __FUNCTION__);
1595
1596 /* get next TD */
1597
1598 td = td_next;
1599 td_next = td->obj_next;
1600
1601 /* check if we are pre-computing */
1602
1603 if (precompute) {
1604
1605 /* update remaining length */
1606
1607 temp->len -= average;
1608
1609 continue;
1610 }
1611 /* fill out current TD */
1612
1613 td->len = average;
1614 td->remainder = 0;
1615 td->status = 0;
1616
1617 /* update remaining length */
1618
1619 temp->len -= average;
1620
1621 /* reset TRB index */
1622
1623 x = 0;
1624
1625 if (temp->trb_type == XHCI_TRB_TYPE_SETUP_STAGE) {
1626 /* immediate data */
1627
1628 if (average > 8)
1629 average = 8;
1630
1631 td->td_trb[0].qwTrb0 = 0;
1632
1633 usbd_copy_out(temp->pc, temp->offset + buf_offset,
1634 (uint8_t *)(uintptr_t)&td->td_trb[0].qwTrb0,
1635 average);
1636
1637 dword = XHCI_TRB_2_BYTES_SET(8) |
1638 XHCI_TRB_2_TDSZ_SET(0) |
1639 XHCI_TRB_2_IRQ_SET(0);
1640
1641 td->td_trb[0].dwTrb2 = htole32(dword);
1642
1643 dword = XHCI_TRB_3_TYPE_SET(XHCI_TRB_TYPE_SETUP_STAGE) |
1644 XHCI_TRB_3_IDT_BIT | XHCI_TRB_3_CYCLE_BIT;
1645
1646 /* check wLength */
1647 if (td->td_trb[0].qwTrb0 &
1648 htole64(XHCI_TRB_0_WLENGTH_MASK)) {
1649 if (td->td_trb[0].qwTrb0 & htole64(1))
1650 dword |= XHCI_TRB_3_TRT_IN;
1651 else
1652 dword |= XHCI_TRB_3_TRT_OUT;
1653 }
1654
1655 td->td_trb[0].dwTrb3 = htole32(dword);
1656#ifdef USB_DEBUG
1657 xhci_dump_trb(&td->td_trb[x]);
1658#endif
1659 x++;
1660
1661 } else do {
1662
1663 uint32_t npkt;
1664
1665 /* fill out buffer pointers */
1666
1667 if (average == 0) {
1668 npkt = 1;
1669 memset(&buf_res, 0, sizeof(buf_res));
1670 } else {
1671 usbd_get_page(temp->pc, temp->offset +
1672 buf_offset, &buf_res);
1673
1674 /* get length to end of page */
1675 if (buf_res.length > average)
1676 buf_res.length = average;
1677
1678 /* check for maximum length */
1679 if (buf_res.length > XHCI_TD_PAGE_SIZE)
1680 buf_res.length = XHCI_TD_PAGE_SIZE;
1681
1682 /* setup npkt */
1683 npkt = (average + temp->max_packet_size - 1) /
1684 temp->max_packet_size;
1685
1686 if (npkt > 31)
1687 npkt = 31;
1688 }
1689
1690 /* fill out TRB's */
1691 td->td_trb[x].qwTrb0 =
1692 htole64((uint64_t)buf_res.physaddr);
1693
1694 dword =
1695 XHCI_TRB_2_BYTES_SET(buf_res.length) |
1696 XHCI_TRB_2_TDSZ_SET(npkt) |
1697 XHCI_TRB_2_IRQ_SET(0);
1698
1699 td->td_trb[x].dwTrb2 = htole32(dword);
1700
| 1569
1570 while (1) {
1571
1572 if (temp->len == 0) {
1573
1574 if (temp->shortpkt)
1575 break;
1576
1577 /* send a Zero Length Packet, ZLP, last */
1578
1579 temp->shortpkt = 1;
1580 average = 0;
1581
1582 } else {
1583
1584 average = temp->average;
1585
1586 if (temp->len < average) {
1587 if (temp->len % temp->max_packet_size) {
1588 temp->shortpkt = 1;
1589 }
1590 average = temp->len;
1591 }
1592 }
1593
1594 if (td_next == NULL)
1595 panic("%s: out of XHCI transfer descriptors!", __FUNCTION__);
1596
1597 /* get next TD */
1598
1599 td = td_next;
1600 td_next = td->obj_next;
1601
1602 /* check if we are pre-computing */
1603
1604 if (precompute) {
1605
1606 /* update remaining length */
1607
1608 temp->len -= average;
1609
1610 continue;
1611 }
1612 /* fill out current TD */
1613
1614 td->len = average;
1615 td->remainder = 0;
1616 td->status = 0;
1617
1618 /* update remaining length */
1619
1620 temp->len -= average;
1621
1622 /* reset TRB index */
1623
1624 x = 0;
1625
1626 if (temp->trb_type == XHCI_TRB_TYPE_SETUP_STAGE) {
1627 /* immediate data */
1628
1629 if (average > 8)
1630 average = 8;
1631
1632 td->td_trb[0].qwTrb0 = 0;
1633
1634 usbd_copy_out(temp->pc, temp->offset + buf_offset,
1635 (uint8_t *)(uintptr_t)&td->td_trb[0].qwTrb0,
1636 average);
1637
1638 dword = XHCI_TRB_2_BYTES_SET(8) |
1639 XHCI_TRB_2_TDSZ_SET(0) |
1640 XHCI_TRB_2_IRQ_SET(0);
1641
1642 td->td_trb[0].dwTrb2 = htole32(dword);
1643
1644 dword = XHCI_TRB_3_TYPE_SET(XHCI_TRB_TYPE_SETUP_STAGE) |
1645 XHCI_TRB_3_IDT_BIT | XHCI_TRB_3_CYCLE_BIT;
1646
1647 /* check wLength */
1648 if (td->td_trb[0].qwTrb0 &
1649 htole64(XHCI_TRB_0_WLENGTH_MASK)) {
1650 if (td->td_trb[0].qwTrb0 & htole64(1))
1651 dword |= XHCI_TRB_3_TRT_IN;
1652 else
1653 dword |= XHCI_TRB_3_TRT_OUT;
1654 }
1655
1656 td->td_trb[0].dwTrb3 = htole32(dword);
1657#ifdef USB_DEBUG
1658 xhci_dump_trb(&td->td_trb[x]);
1659#endif
1660 x++;
1661
1662 } else do {
1663
1664 uint32_t npkt;
1665
1666 /* fill out buffer pointers */
1667
1668 if (average == 0) {
1669 npkt = 1;
1670 memset(&buf_res, 0, sizeof(buf_res));
1671 } else {
1672 usbd_get_page(temp->pc, temp->offset +
1673 buf_offset, &buf_res);
1674
1675 /* get length to end of page */
1676 if (buf_res.length > average)
1677 buf_res.length = average;
1678
1679 /* check for maximum length */
1680 if (buf_res.length > XHCI_TD_PAGE_SIZE)
1681 buf_res.length = XHCI_TD_PAGE_SIZE;
1682
1683 /* setup npkt */
1684 npkt = (average + temp->max_packet_size - 1) /
1685 temp->max_packet_size;
1686
1687 if (npkt > 31)
1688 npkt = 31;
1689 }
1690
1691 /* fill out TRB's */
1692 td->td_trb[x].qwTrb0 =
1693 htole64((uint64_t)buf_res.physaddr);
1694
1695 dword =
1696 XHCI_TRB_2_BYTES_SET(buf_res.length) |
1697 XHCI_TRB_2_TDSZ_SET(npkt) |
1698 XHCI_TRB_2_IRQ_SET(0);
1699
1700 td->td_trb[x].dwTrb2 = htole32(dword);
1701
|
| 1702 /* BEI: Interrupts are inhibited until EOT */
|
1701 dword = XHCI_TRB_3_CHAIN_BIT | XHCI_TRB_3_CYCLE_BIT |
| 1703 dword = XHCI_TRB_3_CHAIN_BIT | XHCI_TRB_3_CYCLE_BIT |
|
| 1704 XHCI_TRB_3_BEI_BIT |
|
1702 XHCI_TRB_3_TYPE_SET(temp->trb_type) |
1703 XHCI_TRB_3_TBC_SET(temp->tbc) |
1704 XHCI_TRB_3_TLBPC_SET(temp->tlbpc);
1705
1706 if (temp->trb_type == XHCI_TRB_TYPE_ISOCH) {
1707 if (temp->do_isoc_sync != 0) {
1708 temp->do_isoc_sync = 0;
1709 dword |= XHCI_TRB_3_FRID_SET(temp->isoc_frame / 8);
1710 } else {
1711 dword |= XHCI_TRB_3_ISO_SIA_BIT;
1712 }
1713 }
1714 if (temp->direction == UE_DIR_IN) {
1715 dword |= XHCI_TRB_3_DIR_IN;
1716
1717 /*
1718 * NOTE: Only the SETUP stage should
1719 * use the IDT bit. Else transactions
1720 * can be sent using the wrong data
1721 * toggle value.
1722 */
1723 if (temp->trb_type !=
1724 XHCI_TRB_TYPE_SETUP_STAGE &&
1725 temp->trb_type !=
1726 XHCI_TRB_TYPE_STATUS_STAGE)
1727 dword |= XHCI_TRB_3_ISP_BIT;
1728 }
1729
1730 td->td_trb[x].dwTrb3 = htole32(dword);
1731
1732 average -= buf_res.length;
1733 buf_offset += buf_res.length;
1734#ifdef USB_DEBUG
1735 xhci_dump_trb(&td->td_trb[x]);
1736#endif
1737 x++;
1738
1739 } while (average != 0);
1740
1741 td->td_trb[x-1].dwTrb3 |= htole32(XHCI_TRB_3_IOC_BIT);
1742
1743 /* store number of data TRB's */
1744
1745 td->ntrb = x;
1746
1747 DPRINTF("NTRB=%u\n", x);
1748
1749 /* fill out link TRB */
1750
1751 if (td_next != NULL) {
1752 /* link the current TD with the next one */
1753 td->td_trb[x].qwTrb0 = htole64((uint64_t)td_next->td_self);
1754 DPRINTF("LINK=0x%08llx\n", (long long)td_next->td_self);
1755 } else {
1756 /* this field will get updated later */
1757 DPRINTF("NOLINK\n");
1758 }
1759
1760 dword = XHCI_TRB_2_IRQ_SET(0);
1761
1762 td->td_trb[x].dwTrb2 = htole32(dword);
1763
| 1705 XHCI_TRB_3_TYPE_SET(temp->trb_type) |
1706 XHCI_TRB_3_TBC_SET(temp->tbc) |
1707 XHCI_TRB_3_TLBPC_SET(temp->tlbpc);
1708
1709 if (temp->trb_type == XHCI_TRB_TYPE_ISOCH) {
1710 if (temp->do_isoc_sync != 0) {
1711 temp->do_isoc_sync = 0;
1712 dword |= XHCI_TRB_3_FRID_SET(temp->isoc_frame / 8);
1713 } else {
1714 dword |= XHCI_TRB_3_ISO_SIA_BIT;
1715 }
1716 }
1717 if (temp->direction == UE_DIR_IN) {
1718 dword |= XHCI_TRB_3_DIR_IN;
1719
1720 /*
1721 * NOTE: Only the SETUP stage should
1722 * use the IDT bit. Else transactions
1723 * can be sent using the wrong data
1724 * toggle value.
1725 */
1726 if (temp->trb_type !=
1727 XHCI_TRB_TYPE_SETUP_STAGE &&
1728 temp->trb_type !=
1729 XHCI_TRB_TYPE_STATUS_STAGE)
1730 dword |= XHCI_TRB_3_ISP_BIT;
1731 }
1732
1733 td->td_trb[x].dwTrb3 = htole32(dword);
1734
1735 average -= buf_res.length;
1736 buf_offset += buf_res.length;
1737#ifdef USB_DEBUG
1738 xhci_dump_trb(&td->td_trb[x]);
1739#endif
1740 x++;
1741
1742 } while (average != 0);
1743
1744 td->td_trb[x-1].dwTrb3 |= htole32(XHCI_TRB_3_IOC_BIT);
1745
1746 /* store number of data TRB's */
1747
1748 td->ntrb = x;
1749
1750 DPRINTF("NTRB=%u\n", x);
1751
1752 /* fill out link TRB */
1753
1754 if (td_next != NULL) {
1755 /* link the current TD with the next one */
1756 td->td_trb[x].qwTrb0 = htole64((uint64_t)td_next->td_self);
1757 DPRINTF("LINK=0x%08llx\n", (long long)td_next->td_self);
1758 } else {
1759 /* this field will get updated later */
1760 DPRINTF("NOLINK\n");
1761 }
1762
1763 dword = XHCI_TRB_2_IRQ_SET(0);
1764
1765 td->td_trb[x].dwTrb2 = htole32(dword);
1766
|
| 1767 /* BEI: interrupts are inhibited until EOT */
|
1764 dword = XHCI_TRB_3_TYPE_SET(XHCI_TRB_TYPE_LINK) |
| 1768 dword = XHCI_TRB_3_TYPE_SET(XHCI_TRB_TYPE_LINK) |
|
1765 XHCI_TRB_3_CYCLE_BIT | XHCI_TRB_3_IOC_BIT;
| 1769 XHCI_TRB_3_CYCLE_BIT | XHCI_TRB_3_IOC_BIT |
1770 XHCI_TRB_3_BEI_BIT;
|
1766
1767 td->td_trb[x].dwTrb3 = htole32(dword);
1768
1769 td->alt_next = td_alt_next;
1770#ifdef USB_DEBUG
1771 xhci_dump_trb(&td->td_trb[x]);
1772#endif
1773 usb_pc_cpu_flush(td->page_cache);
1774 }
1775
1776 if (precompute) {
1777 precompute = 0;
1778
1779 /* setup alt next pointer, if any */
1780 if (temp->last_frame) {
1781 td_alt_next = NULL;
1782 } else {
1783 /* we use this field internally */
1784 td_alt_next = td_next;
1785 }
1786
1787 /* restore */
1788 temp->shortpkt = shortpkt_old;
1789 temp->len = len_old;
1790 goto restart;
1791 }
1792
| 1771
1772 td->td_trb[x].dwTrb3 = htole32(dword);
1773
1774 td->alt_next = td_alt_next;
1775#ifdef USB_DEBUG
1776 xhci_dump_trb(&td->td_trb[x]);
1777#endif
1778 usb_pc_cpu_flush(td->page_cache);
1779 }
1780
1781 if (precompute) {
1782 precompute = 0;
1783
1784 /* setup alt next pointer, if any */
1785 if (temp->last_frame) {
1786 td_alt_next = NULL;
1787 } else {
1788 /* we use this field internally */
1789 td_alt_next = td_next;
1790 }
1791
1792 /* restore */
1793 temp->shortpkt = shortpkt_old;
1794 temp->len = len_old;
1795 goto restart;
1796 }
1797
|
1793 /* remove cycle bit from first if we are stepping the TRBs */
1794 if (temp->step_td)
1795 td->td_trb[0].dwTrb3 &= ~htole32(XHCI_TRB_3_CYCLE_BIT);
| 1798 /* need to force an interrupt if we are stepping the TRBs */
1799 if ((temp->direction & UE_DIR_IN) != 0 && temp->multishort == 0) {
1800 /* remove cycle bit from first TRB if we are stepping them */
1801 if (temp->step_td)
1802 td_first->td_trb[0].dwTrb3 &= ~htole32(XHCI_TRB_3_CYCLE_BIT);
1803 /* make sure the last LINK event generates an interrupt */
1804 td->td_trb[td->ntrb].dwTrb3 &= ~htole32(XHCI_TRB_3_BEI_BIT);
1805 }
|
1796
1797 /* remove chain bit because this is the last TRB in the chain */
1798 td->td_trb[td->ntrb - 1].dwTrb2 &= ~htole32(XHCI_TRB_2_TDSZ_SET(15));
1799 td->td_trb[td->ntrb - 1].dwTrb3 &= ~htole32(XHCI_TRB_3_CHAIN_BIT);
1800
1801 usb_pc_cpu_flush(td->page_cache);
1802
1803 temp->td = td;
1804 temp->td_next = td_next;
1805}
1806
1807static void
1808xhci_setup_generic_chain(struct usb_xfer *xfer)
1809{
1810 struct xhci_std_temp temp;
1811 struct xhci_td *td;
1812 uint32_t x;
1813 uint32_t y;
1814 uint8_t mult;
1815
1816 temp.do_isoc_sync = 0;
1817 temp.step_td = 0;
1818 temp.tbc = 0;
1819 temp.tlbpc = 0;
1820 temp.average = xfer->max_hc_frame_size;
1821 temp.max_packet_size = xfer->max_packet_size;
1822 temp.sc = XHCI_BUS2SC(xfer->xroot->bus);
1823 temp.pc = NULL;
1824 temp.last_frame = 0;
1825 temp.offset = 0;
1826 temp.multishort = xfer->flags_int.isochronous_xfr ||
1827 xfer->flags_int.control_xfr ||
1828 xfer->flags_int.short_frames_ok;
1829
1830 /* toggle the DMA set we are using */
1831 xfer->flags_int.curr_dma_set ^= 1;
1832
1833 /* get next DMA set */
1834 td = xfer->td_start[xfer->flags_int.curr_dma_set];
1835
1836 temp.td = NULL;
1837 temp.td_next = td;
1838
1839 xfer->td_transfer_first = td;
1840 xfer->td_transfer_cache = td;
1841
1842 if (xfer->flags_int.isochronous_xfr) {
1843 uint8_t shift;
1844
1845 /* compute multiplier for ISOCHRONOUS transfers */
1846 mult = xfer->endpoint->ecomp ?
1847 UE_GET_SS_ISO_MULT(xfer->endpoint->ecomp->bmAttributes)
1848 : 0;
1849 /* check for USB 2.0 multiplier */
1850 if (mult == 0) {
1851 mult = (xfer->endpoint->edesc->
1852 wMaxPacketSize[1] >> 3) & 3;
1853 }
1854 /* range check */
1855 if (mult > 2)
1856 mult = 3;
1857 else
1858 mult++;
1859
1860 x = XREAD4(temp.sc, runt, XHCI_MFINDEX);
1861
1862 DPRINTF("MFINDEX=0x%08x\n", x);
1863
1864 switch (usbd_get_speed(xfer->xroot->udev)) {
1865 case USB_SPEED_FULL:
1866 shift = 3;
1867 temp.isoc_delta = 8; /* 1ms */
1868 x += temp.isoc_delta - 1;
1869 x &= ~(temp.isoc_delta - 1);
1870 break;
1871 default:
1872 shift = usbd_xfer_get_fps_shift(xfer);
1873 temp.isoc_delta = 1U << shift;
1874 x += temp.isoc_delta - 1;
1875 x &= ~(temp.isoc_delta - 1);
1876 /* simple frame load balancing */
1877 x += xfer->endpoint->usb_uframe;
1878 break;
1879 }
1880
1881 y = XHCI_MFINDEX_GET(x - xfer->endpoint->isoc_next);
1882
1883 if ((xfer->endpoint->is_synced == 0) ||
1884 (y < (xfer->nframes << shift)) ||
1885 (XHCI_MFINDEX_GET(-y) >= (128 * 8))) {
1886 /*
1887 * If there is data underflow or the pipe
1888 * queue is empty we schedule the transfer a
1889 * few frames ahead of the current frame
1890 * position. Else two isochronous transfers
1891 * might overlap.
1892 */
1893 xfer->endpoint->isoc_next = XHCI_MFINDEX_GET(x + (3 * 8));
1894 xfer->endpoint->is_synced = 1;
1895 temp.do_isoc_sync = 1;
1896
1897 DPRINTFN(3, "start next=%d\n", xfer->endpoint->isoc_next);
1898 }
1899
1900 /* compute isochronous completion time */
1901
1902 y = XHCI_MFINDEX_GET(xfer->endpoint->isoc_next - (x & ~7));
1903
1904 xfer->isoc_time_complete =
1905 usb_isoc_time_expand(&temp.sc->sc_bus, x / 8) +
1906 (y / 8) + (((xfer->nframes << shift) + 7) / 8);
1907
1908 x = 0;
1909 temp.isoc_frame = xfer->endpoint->isoc_next;
1910 temp.trb_type = XHCI_TRB_TYPE_ISOCH;
1911
1912 xfer->endpoint->isoc_next += xfer->nframes << shift;
1913
1914 } else if (xfer->flags_int.control_xfr) {
1915
1916 /* check if we should prepend a setup message */
1917
1918 if (xfer->flags_int.control_hdr) {
1919
1920 temp.len = xfer->frlengths[0];
1921 temp.pc = xfer->frbuffers + 0;
1922 temp.shortpkt = temp.len ? 1 : 0;
1923 temp.trb_type = XHCI_TRB_TYPE_SETUP_STAGE;
1924 temp.direction = 0;
1925
1926 /* check for last frame */
1927 if (xfer->nframes == 1) {
1928 /* no STATUS stage yet, SETUP is last */
1929 if (xfer->flags_int.control_act)
1930 temp.last_frame = 1;
1931 }
1932
1933 xhci_setup_generic_chain_sub(&temp);
1934 }
1935 x = 1;
1936 mult = 1;
1937 temp.isoc_delta = 0;
1938 temp.isoc_frame = 0;
1939 temp.trb_type = XHCI_TRB_TYPE_DATA_STAGE;
1940 } else {
1941 x = 0;
1942 mult = 1;
1943 temp.isoc_delta = 0;
1944 temp.isoc_frame = 0;
1945 temp.trb_type = XHCI_TRB_TYPE_NORMAL;
1946 }
1947
1948 if (x != xfer->nframes) {
1949 /* setup page_cache pointer */
1950 temp.pc = xfer->frbuffers + x;
1951 /* set endpoint direction */
1952 temp.direction = UE_GET_DIR(xfer->endpointno);
1953 }
1954
1955 while (x != xfer->nframes) {
1956
1957 /* DATA0 / DATA1 message */
1958
1959 temp.len = xfer->frlengths[x];
1960 temp.step_td = ((xfer->endpointno & UE_DIR_IN) &&
1961 x != 0 && temp.multishort == 0);
1962
1963 x++;
1964
1965 if (x == xfer->nframes) {
1966 if (xfer->flags_int.control_xfr) {
1967 /* no STATUS stage yet, DATA is last */
1968 if (xfer->flags_int.control_act)
1969 temp.last_frame = 1;
1970 } else {
1971 temp.last_frame = 1;
1972 }
1973 }
1974 if (temp.len == 0) {
1975
1976 /* make sure that we send an USB packet */
1977
1978 temp.shortpkt = 0;
1979
1980 temp.tbc = 0;
1981 temp.tlbpc = mult - 1;
1982
1983 } else if (xfer->flags_int.isochronous_xfr) {
1984
1985 uint8_t tdpc;
1986
1987 /*
1988 * Isochronous transfers don't have short
1989 * packet termination:
1990 */
1991
1992 temp.shortpkt = 1;
1993
1994 /* isochronous transfers have a transfer limit */
1995
1996 if (temp.len > xfer->max_frame_size)
1997 temp.len = xfer->max_frame_size;
1998
1999 /* compute TD packet count */
2000 tdpc = (temp.len + xfer->max_packet_size - 1) /
2001 xfer->max_packet_size;
2002
2003 temp.tbc = ((tdpc + mult - 1) / mult) - 1;
2004 temp.tlbpc = (tdpc % mult);
2005
2006 if (temp.tlbpc == 0)
2007 temp.tlbpc = mult - 1;
2008 else
2009 temp.tlbpc--;
2010 } else {
2011
2012 /* regular data transfer */
2013
2014 temp.shortpkt = xfer->flags.force_short_xfer ? 0 : 1;
2015 }
2016
2017 xhci_setup_generic_chain_sub(&temp);
2018
2019 if (xfer->flags_int.isochronous_xfr) {
2020 temp.offset += xfer->frlengths[x - 1];
2021 temp.isoc_frame += temp.isoc_delta;
2022 } else {
2023 /* get next Page Cache pointer */
2024 temp.pc = xfer->frbuffers + x;
2025 }
2026 }
2027
2028 /* check if we should append a status stage */
2029
2030 if (xfer->flags_int.control_xfr &&
2031 !xfer->flags_int.control_act) {
2032
2033 /*
2034 * Send a DATA1 message and invert the current
2035 * endpoint direction.
2036 */
2037 temp.step_td = (xfer->nframes != 0);
2038 temp.direction = UE_GET_DIR(xfer->endpointno) ^ UE_DIR_IN;
2039 temp.len = 0;
2040 temp.pc = NULL;
2041 temp.shortpkt = 0;
2042 temp.last_frame = 1;
2043 temp.trb_type = XHCI_TRB_TYPE_STATUS_STAGE;
2044
2045 xhci_setup_generic_chain_sub(&temp);
2046 }
2047
2048 td = temp.td;
2049
2050 /* must have at least one frame! */
2051
2052 xfer->td_transfer_last = td;
2053
2054 DPRINTF("first=%p last=%p\n", xfer->td_transfer_first, td);
2055}
2056
2057static void
2058xhci_set_slot_pointer(struct xhci_softc *sc, uint8_t index, uint64_t dev_addr)
2059{
2060 struct usb_page_search buf_res;
2061 struct xhci_dev_ctx_addr *pdctxa;
2062
2063 usbd_get_page(&sc->sc_hw.ctx_pc, 0, &buf_res);
2064
2065 pdctxa = buf_res.buffer;
2066
2067 DPRINTF("addr[%u]=0x%016llx\n", index, (long long)dev_addr);
2068
2069 pdctxa->qwBaaDevCtxAddr[index] = htole64(dev_addr);
2070
2071 usb_pc_cpu_flush(&sc->sc_hw.ctx_pc);
2072}
2073
2074static usb_error_t
2075xhci_configure_mask(struct usb_device *udev, uint32_t mask, uint8_t drop)
2076{
2077 struct xhci_softc *sc = XHCI_BUS2SC(udev->bus);
2078 struct usb_page_search buf_inp;
2079 struct xhci_input_dev_ctx *pinp;
2080 uint32_t temp;
2081 uint8_t index;
2082 uint8_t x;
2083
2084 index = udev->controller_slot_id;
2085
2086 usbd_get_page(&sc->sc_hw.devs[index].input_pc, 0, &buf_inp);
2087
2088 pinp = buf_inp.buffer;
2089
2090 if (drop) {
2091 mask &= XHCI_INCTX_NON_CTRL_MASK;
2092 xhci_ctx_set_le32(sc, &pinp->ctx_input.dwInCtx0, mask);
2093 xhci_ctx_set_le32(sc, &pinp->ctx_input.dwInCtx1, 0);
2094 } else {
2095 xhci_ctx_set_le32(sc, &pinp->ctx_input.dwInCtx0, 0);
2096 xhci_ctx_set_le32(sc, &pinp->ctx_input.dwInCtx1, mask);
2097
2098 /* find most significant set bit */
2099 for (x = 31; x != 1; x--) {
2100 if (mask & (1 << x))
2101 break;
2102 }
2103
2104 /* adjust */
2105 x--;
2106
2107 /* figure out maximum */
2108 if (x > sc->sc_hw.devs[index].context_num) {
2109 sc->sc_hw.devs[index].context_num = x;
2110 temp = xhci_ctx_get_le32(sc, &pinp->ctx_slot.dwSctx0);
2111 temp &= ~XHCI_SCTX_0_CTX_NUM_SET(31);
2112 temp |= XHCI_SCTX_0_CTX_NUM_SET(x + 1);
2113 xhci_ctx_set_le32(sc, &pinp->ctx_slot.dwSctx0, temp);
2114 }
2115 }
2116 return (0);
2117}
2118
2119static usb_error_t
2120xhci_configure_endpoint(struct usb_device *udev,
2121 struct usb_endpoint_descriptor *edesc, struct xhci_endpoint_ext *pepext,
2122 uint16_t interval, uint8_t max_packet_count,
2123 uint8_t mult, uint8_t fps_shift, uint16_t max_packet_size,
2124 uint16_t max_frame_size, uint8_t ep_mode)
2125{
2126 struct usb_page_search buf_inp;
2127 struct xhci_softc *sc = XHCI_BUS2SC(udev->bus);
2128 struct xhci_input_dev_ctx *pinp;
2129 uint64_t ring_addr = pepext->physaddr;
2130 uint32_t temp;
2131 uint8_t index;
2132 uint8_t epno;
2133 uint8_t type;
2134
2135 index = udev->controller_slot_id;
2136
2137 usbd_get_page(&sc->sc_hw.devs[index].input_pc, 0, &buf_inp);
2138
2139 pinp = buf_inp.buffer;
2140
2141 epno = edesc->bEndpointAddress;
2142 type = edesc->bmAttributes & UE_XFERTYPE;
2143
2144 if (type == UE_CONTROL)
2145 epno |= UE_DIR_IN;
2146
2147 epno = XHCI_EPNO2EPID(epno);
2148
2149 if (epno == 0)
2150 return (USB_ERR_NO_PIPE); /* invalid */
2151
2152 if (max_packet_count == 0)
2153 return (USB_ERR_BAD_BUFSIZE);
2154
2155 max_packet_count--;
2156
2157 if (mult == 0)
2158 return (USB_ERR_BAD_BUFSIZE);
2159
2160 /* store endpoint mode */
2161 pepext->trb_ep_mode = ep_mode;
2162 usb_pc_cpu_flush(pepext->page_cache);
2163
2164 if (ep_mode == USB_EP_MODE_STREAMS) {
2165 temp = XHCI_EPCTX_0_EPSTATE_SET(0) |
2166 XHCI_EPCTX_0_MAXP_STREAMS_SET(XHCI_MAX_STREAMS_LOG - 1) |
2167 XHCI_EPCTX_0_LSA_SET(1);
2168
2169 ring_addr += sizeof(struct xhci_trb) *
2170 XHCI_MAX_TRANSFERS * XHCI_MAX_STREAMS;
2171 } else {
2172 temp = XHCI_EPCTX_0_EPSTATE_SET(0) |
2173 XHCI_EPCTX_0_MAXP_STREAMS_SET(0) |
2174 XHCI_EPCTX_0_LSA_SET(0);
2175
2176 ring_addr |= XHCI_EPCTX_2_DCS_SET(1);
2177 }
2178
2179 switch (udev->speed) {
2180 case USB_SPEED_FULL:
2181 case USB_SPEED_LOW:
2182 /* 1ms -> 125us */
2183 fps_shift += 3;
2184 break;
2185 default:
2186 break;
2187 }
2188
2189 switch (type) {
2190 case UE_INTERRUPT:
2191 if (fps_shift > 3)
2192 fps_shift--;
2193 temp |= XHCI_EPCTX_0_IVAL_SET(fps_shift);
2194 break;
2195 case UE_ISOCHRONOUS:
2196 temp |= XHCI_EPCTX_0_IVAL_SET(fps_shift);
2197
2198 switch (udev->speed) {
2199 case USB_SPEED_SUPER:
2200 if (mult > 3)
2201 mult = 3;
2202 temp |= XHCI_EPCTX_0_MULT_SET(mult - 1);
2203 max_packet_count /= mult;
2204 break;
2205 default:
2206 break;
2207 }
2208 break;
2209 default:
2210 break;
2211 }
2212
2213 xhci_ctx_set_le32(sc, &pinp->ctx_ep[epno - 1].dwEpCtx0, temp);
2214
2215 temp =
2216 XHCI_EPCTX_1_HID_SET(0) |
2217 XHCI_EPCTX_1_MAXB_SET(max_packet_count) |
2218 XHCI_EPCTX_1_MAXP_SIZE_SET(max_packet_size);
2219
2220 if ((udev->parent_hs_hub != NULL) || (udev->address != 0)) {
2221 if (type != UE_ISOCHRONOUS)
2222 temp |= XHCI_EPCTX_1_CERR_SET(3);
2223 }
2224
2225 switch (type) {
2226 case UE_CONTROL:
2227 temp |= XHCI_EPCTX_1_EPTYPE_SET(4);
2228 break;
2229 case UE_ISOCHRONOUS:
2230 temp |= XHCI_EPCTX_1_EPTYPE_SET(1);
2231 break;
2232 case UE_BULK:
2233 temp |= XHCI_EPCTX_1_EPTYPE_SET(2);
2234 break;
2235 default:
2236 temp |= XHCI_EPCTX_1_EPTYPE_SET(3);
2237 break;
2238 }
2239
2240 /* check for IN direction */
2241 if (epno & 1)
2242 temp |= XHCI_EPCTX_1_EPTYPE_SET(4);
2243
2244 xhci_ctx_set_le32(sc, &pinp->ctx_ep[epno - 1].dwEpCtx1, temp);
2245 xhci_ctx_set_le64(sc, &pinp->ctx_ep[epno - 1].qwEpCtx2, ring_addr);
2246
2247 switch (edesc->bmAttributes & UE_XFERTYPE) {
2248 case UE_INTERRUPT:
2249 case UE_ISOCHRONOUS:
2250 temp = XHCI_EPCTX_4_MAX_ESIT_PAYLOAD_SET(max_frame_size) |
2251 XHCI_EPCTX_4_AVG_TRB_LEN_SET(MIN(XHCI_PAGE_SIZE,
2252 max_frame_size));
2253 break;
2254 case UE_CONTROL:
2255 temp = XHCI_EPCTX_4_AVG_TRB_LEN_SET(8);
2256 break;
2257 default:
2258 temp = XHCI_EPCTX_4_AVG_TRB_LEN_SET(XHCI_PAGE_SIZE);
2259 break;
2260 }
2261
2262 xhci_ctx_set_le32(sc, &pinp->ctx_ep[epno - 1].dwEpCtx4, temp);
2263
2264#ifdef USB_DEBUG
2265 xhci_dump_endpoint(sc, &pinp->ctx_ep[epno - 1]);
2266#endif
2267 usb_pc_cpu_flush(&sc->sc_hw.devs[index].input_pc);
2268
2269 return (0); /* success */
2270}
2271
2272static usb_error_t
2273xhci_configure_endpoint_by_xfer(struct usb_xfer *xfer)
2274{
2275 struct xhci_endpoint_ext *pepext;
2276 struct usb_endpoint_ss_comp_descriptor *ecomp;
2277 usb_stream_t x;
2278
2279 pepext = xhci_get_endpoint_ext(xfer->xroot->udev,
2280 xfer->endpoint->edesc);
2281
2282 ecomp = xfer->endpoint->ecomp;
2283
2284 for (x = 0; x != XHCI_MAX_STREAMS; x++) {
2285 uint64_t temp;
2286
2287 /* halt any transfers */
2288 pepext->trb[x * XHCI_MAX_TRANSFERS].dwTrb3 = 0;
2289
2290 /* compute start of TRB ring for stream "x" */
2291 temp = pepext->physaddr +
2292 (x * XHCI_MAX_TRANSFERS * sizeof(struct xhci_trb)) +
2293 XHCI_SCTX_0_SCT_SEC_TR_RING;
2294
2295 /* make tree structure */
2296 pepext->trb[(XHCI_MAX_TRANSFERS *
2297 XHCI_MAX_STREAMS) + x].qwTrb0 = htole64(temp);
2298
2299 /* reserved fields */
2300 pepext->trb[(XHCI_MAX_TRANSFERS *
2301 XHCI_MAX_STREAMS) + x].dwTrb2 = 0;
2302 pepext->trb[(XHCI_MAX_TRANSFERS *
2303 XHCI_MAX_STREAMS) + x].dwTrb3 = 0;
2304 }
2305 usb_pc_cpu_flush(pepext->page_cache);
2306
2307 return (xhci_configure_endpoint(xfer->xroot->udev,
2308 xfer->endpoint->edesc, pepext,
2309 xfer->interval, xfer->max_packet_count,
2310 (ecomp != NULL) ? UE_GET_SS_ISO_MULT(ecomp->bmAttributes) + 1 : 1,
2311 usbd_xfer_get_fps_shift(xfer), xfer->max_packet_size,
2312 xfer->max_frame_size, xfer->endpoint->ep_mode));
2313}
2314
2315static usb_error_t
2316xhci_configure_device(struct usb_device *udev)
2317{
2318 struct xhci_softc *sc = XHCI_BUS2SC(udev->bus);
2319 struct usb_page_search buf_inp;
2320 struct usb_page_cache *pcinp;
2321 struct xhci_input_dev_ctx *pinp;
2322 struct usb_device *hubdev;
2323 uint32_t temp;
2324 uint32_t route;
2325 uint32_t rh_port;
2326 uint8_t is_hub;
2327 uint8_t index;
2328 uint8_t depth;
2329
2330 index = udev->controller_slot_id;
2331
2332 DPRINTF("index=%u\n", index);
2333
2334 pcinp = &sc->sc_hw.devs[index].input_pc;
2335
2336 usbd_get_page(pcinp, 0, &buf_inp);
2337
2338 pinp = buf_inp.buffer;
2339
2340 rh_port = 0;
2341 route = 0;
2342
2343 /* figure out route string and root HUB port number */
2344
2345 for (hubdev = udev; hubdev != NULL; hubdev = hubdev->parent_hub) {
2346
2347 if (hubdev->parent_hub == NULL)
2348 break;
2349
2350 depth = hubdev->parent_hub->depth;
2351
2352 /*
2353 * NOTE: HS/FS/LS devices and the SS root HUB can have
2354 * more than 15 ports
2355 */
2356
2357 rh_port = hubdev->port_no;
2358
2359 if (depth == 0)
2360 break;
2361
2362 if (rh_port > 15)
2363 rh_port = 15;
2364
2365 if (depth < 6)
2366 route |= rh_port << (4 * (depth - 1));
2367 }
2368
2369 DPRINTF("Route=0x%08x\n", route);
2370
2371 temp = XHCI_SCTX_0_ROUTE_SET(route) |
2372 XHCI_SCTX_0_CTX_NUM_SET(
2373 sc->sc_hw.devs[index].context_num + 1);
2374
2375 switch (udev->speed) {
2376 case USB_SPEED_LOW:
2377 temp |= XHCI_SCTX_0_SPEED_SET(2);
2378 if (udev->parent_hs_hub != NULL &&
2379 udev->parent_hs_hub->ddesc.bDeviceProtocol ==
2380 UDPROTO_HSHUBMTT) {
2381 DPRINTF("Device inherits MTT\n");
2382 temp |= XHCI_SCTX_0_MTT_SET(1);
2383 }
2384 break;
2385 case USB_SPEED_HIGH:
2386 temp |= XHCI_SCTX_0_SPEED_SET(3);
2387 if (sc->sc_hw.devs[index].nports != 0 &&
2388 udev->ddesc.bDeviceProtocol == UDPROTO_HSHUBMTT) {
2389 DPRINTF("HUB supports MTT\n");
2390 temp |= XHCI_SCTX_0_MTT_SET(1);
2391 }
2392 break;
2393 case USB_SPEED_FULL:
2394 temp |= XHCI_SCTX_0_SPEED_SET(1);
2395 if (udev->parent_hs_hub != NULL &&
2396 udev->parent_hs_hub->ddesc.bDeviceProtocol ==
2397 UDPROTO_HSHUBMTT) {
2398 DPRINTF("Device inherits MTT\n");
2399 temp |= XHCI_SCTX_0_MTT_SET(1);
2400 }
2401 break;
2402 default:
2403 temp |= XHCI_SCTX_0_SPEED_SET(4);
2404 break;
2405 }
2406
2407 is_hub = sc->sc_hw.devs[index].nports != 0 &&
2408 (udev->speed == USB_SPEED_SUPER ||
2409 udev->speed == USB_SPEED_HIGH);
2410
2411 if (is_hub)
2412 temp |= XHCI_SCTX_0_HUB_SET(1);
2413
2414 xhci_ctx_set_le32(sc, &pinp->ctx_slot.dwSctx0, temp);
2415
2416 temp = XHCI_SCTX_1_RH_PORT_SET(rh_port);
2417
2418 if (is_hub) {
2419 temp |= XHCI_SCTX_1_NUM_PORTS_SET(
2420 sc->sc_hw.devs[index].nports);
2421 }
2422
2423 switch (udev->speed) {
2424 case USB_SPEED_SUPER:
2425 switch (sc->sc_hw.devs[index].state) {
2426 case XHCI_ST_ADDRESSED:
2427 case XHCI_ST_CONFIGURED:
2428 /* enable power save */
2429 temp |= XHCI_SCTX_1_MAX_EL_SET(sc->sc_exit_lat_max);
2430 break;
2431 default:
2432 /* disable power save */
2433 break;
2434 }
2435 break;
2436 default:
2437 break;
2438 }
2439
2440 xhci_ctx_set_le32(sc, &pinp->ctx_slot.dwSctx1, temp);
2441
2442 temp = XHCI_SCTX_2_IRQ_TARGET_SET(0);
2443
2444 if (is_hub) {
2445 temp |= XHCI_SCTX_2_TT_THINK_TIME_SET(
2446 sc->sc_hw.devs[index].tt);
2447 }
2448
2449 hubdev = udev->parent_hs_hub;
2450
2451 /* check if we should activate the transaction translator */
2452 switch (udev->speed) {
2453 case USB_SPEED_FULL:
2454 case USB_SPEED_LOW:
2455 if (hubdev != NULL) {
2456 temp |= XHCI_SCTX_2_TT_HUB_SID_SET(
2457 hubdev->controller_slot_id);
2458 temp |= XHCI_SCTX_2_TT_PORT_NUM_SET(
2459 udev->hs_port_no);
2460 }
2461 break;
2462 default:
2463 break;
2464 }
2465
2466 xhci_ctx_set_le32(sc, &pinp->ctx_slot.dwSctx2, temp);
2467
2468 temp = XHCI_SCTX_3_DEV_ADDR_SET(udev->address) |
2469 XHCI_SCTX_3_SLOT_STATE_SET(0);
2470
2471 xhci_ctx_set_le32(sc, &pinp->ctx_slot.dwSctx3, temp);
2472
2473#ifdef USB_DEBUG
2474 xhci_dump_device(sc, &pinp->ctx_slot);
2475#endif
2476 usb_pc_cpu_flush(pcinp);
2477
2478 return (0); /* success */
2479}
2480
2481static usb_error_t
2482xhci_alloc_device_ext(struct usb_device *udev)
2483{
2484 struct xhci_softc *sc = XHCI_BUS2SC(udev->bus);
2485 struct usb_page_search buf_dev;
2486 struct usb_page_search buf_ep;
2487 struct xhci_trb *trb;
2488 struct usb_page_cache *pc;
2489 struct usb_page *pg;
2490 uint64_t addr;
2491 uint8_t index;
2492 uint8_t i;
2493
2494 index = udev->controller_slot_id;
2495
2496 pc = &sc->sc_hw.devs[index].device_pc;
2497 pg = &sc->sc_hw.devs[index].device_pg;
2498
2499 /* need to initialize the page cache */
2500 pc->tag_parent = sc->sc_bus.dma_parent_tag;
2501
2502 if (usb_pc_alloc_mem(pc, pg, sc->sc_ctx_is_64_byte ?
2503 (2 * sizeof(struct xhci_dev_ctx)) :
2504 sizeof(struct xhci_dev_ctx), XHCI_PAGE_SIZE))
2505 goto error;
2506
2507 usbd_get_page(pc, 0, &buf_dev);
2508
2509 pc = &sc->sc_hw.devs[index].input_pc;
2510 pg = &sc->sc_hw.devs[index].input_pg;
2511
2512 /* need to initialize the page cache */
2513 pc->tag_parent = sc->sc_bus.dma_parent_tag;
2514
2515 if (usb_pc_alloc_mem(pc, pg, sc->sc_ctx_is_64_byte ?
2516 (2 * sizeof(struct xhci_input_dev_ctx)) :
2517 sizeof(struct xhci_input_dev_ctx), XHCI_PAGE_SIZE)) {
2518 goto error;
2519 }
2520
2521 pc = &sc->sc_hw.devs[index].endpoint_pc;
2522 pg = &sc->sc_hw.devs[index].endpoint_pg;
2523
2524 /* need to initialize the page cache */
2525 pc->tag_parent = sc->sc_bus.dma_parent_tag;
2526
2527 if (usb_pc_alloc_mem(pc, pg,
2528 sizeof(struct xhci_dev_endpoint_trbs), XHCI_PAGE_SIZE)) {
2529 goto error;
2530 }
2531
2532 /* initialise all endpoint LINK TRBs */
2533
2534 for (i = 0; i != XHCI_MAX_ENDPOINTS; i++) {
2535
2536 /* lookup endpoint TRB ring */
2537 usbd_get_page(pc, (uintptr_t)&
2538 ((struct xhci_dev_endpoint_trbs *)0)->trb[i][0], &buf_ep);
2539
2540 /* get TRB pointer */
2541 trb = buf_ep.buffer;
2542 trb += XHCI_MAX_TRANSFERS - 1;
2543
2544 /* get TRB start address */
2545 addr = buf_ep.physaddr;
2546
2547 /* create LINK TRB */
2548 trb->qwTrb0 = htole64(addr);
2549 trb->dwTrb2 = htole32(XHCI_TRB_2_IRQ_SET(0));
2550 trb->dwTrb3 = htole32(XHCI_TRB_3_CYCLE_BIT |
2551 XHCI_TRB_3_TYPE_SET(XHCI_TRB_TYPE_LINK));
2552 }
2553
2554 usb_pc_cpu_flush(pc);
2555
2556 xhci_set_slot_pointer(sc, index, buf_dev.physaddr);
2557
2558 return (0);
2559
2560error:
2561 xhci_free_device_ext(udev);
2562
2563 return (USB_ERR_NOMEM);
2564}
2565
2566static void
2567xhci_free_device_ext(struct usb_device *udev)
2568{
2569 struct xhci_softc *sc = XHCI_BUS2SC(udev->bus);
2570 uint8_t index;
2571
2572 index = udev->controller_slot_id;
2573 xhci_set_slot_pointer(sc, index, 0);
2574
2575 usb_pc_free_mem(&sc->sc_hw.devs[index].device_pc);
2576 usb_pc_free_mem(&sc->sc_hw.devs[index].input_pc);
2577 usb_pc_free_mem(&sc->sc_hw.devs[index].endpoint_pc);
2578}
2579
2580static struct xhci_endpoint_ext *
2581xhci_get_endpoint_ext(struct usb_device *udev, struct usb_endpoint_descriptor *edesc)
2582{
2583 struct xhci_softc *sc = XHCI_BUS2SC(udev->bus);
2584 struct xhci_endpoint_ext *pepext;
2585 struct usb_page_cache *pc;
2586 struct usb_page_search buf_ep;
2587 uint8_t epno;
2588 uint8_t index;
2589
2590 epno = edesc->bEndpointAddress;
2591 if ((edesc->bmAttributes & UE_XFERTYPE) == UE_CONTROL)
2592 epno |= UE_DIR_IN;
2593
2594 epno = XHCI_EPNO2EPID(epno);
2595
2596 index = udev->controller_slot_id;
2597
2598 pc = &sc->sc_hw.devs[index].endpoint_pc;
2599
2600 usbd_get_page(pc, (uintptr_t)&((struct xhci_dev_endpoint_trbs *)0)->
2601 trb[epno][0], &buf_ep);
2602
2603 pepext = &sc->sc_hw.devs[index].endp[epno];
2604 pepext->page_cache = pc;
2605 pepext->trb = buf_ep.buffer;
2606 pepext->physaddr = buf_ep.physaddr;
2607
2608 return (pepext);
2609}
2610
2611static void
2612xhci_endpoint_doorbell(struct usb_xfer *xfer)
2613{
2614 struct xhci_softc *sc = XHCI_BUS2SC(xfer->xroot->bus);
2615 uint8_t epno;
2616 uint8_t index;
2617
2618 epno = xfer->endpointno;
2619 if (xfer->flags_int.control_xfr)
2620 epno |= UE_DIR_IN;
2621
2622 epno = XHCI_EPNO2EPID(epno);
2623 index = xfer->xroot->udev->controller_slot_id;
2624
2625 if (xfer->xroot->udev->flags.self_suspended == 0) {
2626 XWRITE4(sc, door, XHCI_DOORBELL(index),
2627 epno | XHCI_DB_SID_SET(xfer->stream_id));
2628 }
2629}
2630
2631static void
2632xhci_transfer_remove(struct usb_xfer *xfer, usb_error_t error)
2633{
2634 struct xhci_endpoint_ext *pepext;
2635
2636 if (xfer->flags_int.bandwidth_reclaimed) {
2637 xfer->flags_int.bandwidth_reclaimed = 0;
2638
2639 pepext = xhci_get_endpoint_ext(xfer->xroot->udev,
2640 xfer->endpoint->edesc);
2641
2642 pepext->trb_used[xfer->stream_id]--;
2643
2644 pepext->xfer[xfer->qh_pos] = NULL;
2645
2646 if (error && pepext->trb_running != 0) {
2647 pepext->trb_halted = 1;
2648 pepext->trb_running = 0;
2649 }
2650 }
2651}
2652
2653static usb_error_t
2654xhci_transfer_insert(struct usb_xfer *xfer)
2655{
2656 struct xhci_td *td_first;
2657 struct xhci_td *td_last;
| 1806
1807 /* remove chain bit because this is the last TRB in the chain */
1808 td->td_trb[td->ntrb - 1].dwTrb2 &= ~htole32(XHCI_TRB_2_TDSZ_SET(15));
1809 td->td_trb[td->ntrb - 1].dwTrb3 &= ~htole32(XHCI_TRB_3_CHAIN_BIT);
1810
1811 usb_pc_cpu_flush(td->page_cache);
1812
1813 temp->td = td;
1814 temp->td_next = td_next;
1815}
1816
1817static void
1818xhci_setup_generic_chain(struct usb_xfer *xfer)
1819{
1820 struct xhci_std_temp temp;
1821 struct xhci_td *td;
1822 uint32_t x;
1823 uint32_t y;
1824 uint8_t mult;
1825
1826 temp.do_isoc_sync = 0;
1827 temp.step_td = 0;
1828 temp.tbc = 0;
1829 temp.tlbpc = 0;
1830 temp.average = xfer->max_hc_frame_size;
1831 temp.max_packet_size = xfer->max_packet_size;
1832 temp.sc = XHCI_BUS2SC(xfer->xroot->bus);
1833 temp.pc = NULL;
1834 temp.last_frame = 0;
1835 temp.offset = 0;
1836 temp.multishort = xfer->flags_int.isochronous_xfr ||
1837 xfer->flags_int.control_xfr ||
1838 xfer->flags_int.short_frames_ok;
1839
1840 /* toggle the DMA set we are using */
1841 xfer->flags_int.curr_dma_set ^= 1;
1842
1843 /* get next DMA set */
1844 td = xfer->td_start[xfer->flags_int.curr_dma_set];
1845
1846 temp.td = NULL;
1847 temp.td_next = td;
1848
1849 xfer->td_transfer_first = td;
1850 xfer->td_transfer_cache = td;
1851
1852 if (xfer->flags_int.isochronous_xfr) {
1853 uint8_t shift;
1854
1855 /* compute multiplier for ISOCHRONOUS transfers */
1856 mult = xfer->endpoint->ecomp ?
1857 UE_GET_SS_ISO_MULT(xfer->endpoint->ecomp->bmAttributes)
1858 : 0;
1859 /* check for USB 2.0 multiplier */
1860 if (mult == 0) {
1861 mult = (xfer->endpoint->edesc->
1862 wMaxPacketSize[1] >> 3) & 3;
1863 }
1864 /* range check */
1865 if (mult > 2)
1866 mult = 3;
1867 else
1868 mult++;
1869
1870 x = XREAD4(temp.sc, runt, XHCI_MFINDEX);
1871
1872 DPRINTF("MFINDEX=0x%08x\n", x);
1873
1874 switch (usbd_get_speed(xfer->xroot->udev)) {
1875 case USB_SPEED_FULL:
1876 shift = 3;
1877 temp.isoc_delta = 8; /* 1ms */
1878 x += temp.isoc_delta - 1;
1879 x &= ~(temp.isoc_delta - 1);
1880 break;
1881 default:
1882 shift = usbd_xfer_get_fps_shift(xfer);
1883 temp.isoc_delta = 1U << shift;
1884 x += temp.isoc_delta - 1;
1885 x &= ~(temp.isoc_delta - 1);
1886 /* simple frame load balancing */
1887 x += xfer->endpoint->usb_uframe;
1888 break;
1889 }
1890
1891 y = XHCI_MFINDEX_GET(x - xfer->endpoint->isoc_next);
1892
1893 if ((xfer->endpoint->is_synced == 0) ||
1894 (y < (xfer->nframes << shift)) ||
1895 (XHCI_MFINDEX_GET(-y) >= (128 * 8))) {
1896 /*
1897 * If there is data underflow or the pipe
1898 * queue is empty we schedule the transfer a
1899 * few frames ahead of the current frame
1900 * position. Else two isochronous transfers
1901 * might overlap.
1902 */
1903 xfer->endpoint->isoc_next = XHCI_MFINDEX_GET(x + (3 * 8));
1904 xfer->endpoint->is_synced = 1;
1905 temp.do_isoc_sync = 1;
1906
1907 DPRINTFN(3, "start next=%d\n", xfer->endpoint->isoc_next);
1908 }
1909
1910 /* compute isochronous completion time */
1911
1912 y = XHCI_MFINDEX_GET(xfer->endpoint->isoc_next - (x & ~7));
1913
1914 xfer->isoc_time_complete =
1915 usb_isoc_time_expand(&temp.sc->sc_bus, x / 8) +
1916 (y / 8) + (((xfer->nframes << shift) + 7) / 8);
1917
1918 x = 0;
1919 temp.isoc_frame = xfer->endpoint->isoc_next;
1920 temp.trb_type = XHCI_TRB_TYPE_ISOCH;
1921
1922 xfer->endpoint->isoc_next += xfer->nframes << shift;
1923
1924 } else if (xfer->flags_int.control_xfr) {
1925
1926 /* check if we should prepend a setup message */
1927
1928 if (xfer->flags_int.control_hdr) {
1929
1930 temp.len = xfer->frlengths[0];
1931 temp.pc = xfer->frbuffers + 0;
1932 temp.shortpkt = temp.len ? 1 : 0;
1933 temp.trb_type = XHCI_TRB_TYPE_SETUP_STAGE;
1934 temp.direction = 0;
1935
1936 /* check for last frame */
1937 if (xfer->nframes == 1) {
1938 /* no STATUS stage yet, SETUP is last */
1939 if (xfer->flags_int.control_act)
1940 temp.last_frame = 1;
1941 }
1942
1943 xhci_setup_generic_chain_sub(&temp);
1944 }
1945 x = 1;
1946 mult = 1;
1947 temp.isoc_delta = 0;
1948 temp.isoc_frame = 0;
1949 temp.trb_type = XHCI_TRB_TYPE_DATA_STAGE;
1950 } else {
1951 x = 0;
1952 mult = 1;
1953 temp.isoc_delta = 0;
1954 temp.isoc_frame = 0;
1955 temp.trb_type = XHCI_TRB_TYPE_NORMAL;
1956 }
1957
1958 if (x != xfer->nframes) {
1959 /* setup page_cache pointer */
1960 temp.pc = xfer->frbuffers + x;
1961 /* set endpoint direction */
1962 temp.direction = UE_GET_DIR(xfer->endpointno);
1963 }
1964
1965 while (x != xfer->nframes) {
1966
1967 /* DATA0 / DATA1 message */
1968
1969 temp.len = xfer->frlengths[x];
1970 temp.step_td = ((xfer->endpointno & UE_DIR_IN) &&
1971 x != 0 && temp.multishort == 0);
1972
1973 x++;
1974
1975 if (x == xfer->nframes) {
1976 if (xfer->flags_int.control_xfr) {
1977 /* no STATUS stage yet, DATA is last */
1978 if (xfer->flags_int.control_act)
1979 temp.last_frame = 1;
1980 } else {
1981 temp.last_frame = 1;
1982 }
1983 }
1984 if (temp.len == 0) {
1985
1986 /* make sure that we send an USB packet */
1987
1988 temp.shortpkt = 0;
1989
1990 temp.tbc = 0;
1991 temp.tlbpc = mult - 1;
1992
1993 } else if (xfer->flags_int.isochronous_xfr) {
1994
1995 uint8_t tdpc;
1996
1997 /*
1998 * Isochronous transfers don't have short
1999 * packet termination:
2000 */
2001
2002 temp.shortpkt = 1;
2003
2004 /* isochronous transfers have a transfer limit */
2005
2006 if (temp.len > xfer->max_frame_size)
2007 temp.len = xfer->max_frame_size;
2008
2009 /* compute TD packet count */
2010 tdpc = (temp.len + xfer->max_packet_size - 1) /
2011 xfer->max_packet_size;
2012
2013 temp.tbc = ((tdpc + mult - 1) / mult) - 1;
2014 temp.tlbpc = (tdpc % mult);
2015
2016 if (temp.tlbpc == 0)
2017 temp.tlbpc = mult - 1;
2018 else
2019 temp.tlbpc--;
2020 } else {
2021
2022 /* regular data transfer */
2023
2024 temp.shortpkt = xfer->flags.force_short_xfer ? 0 : 1;
2025 }
2026
2027 xhci_setup_generic_chain_sub(&temp);
2028
2029 if (xfer->flags_int.isochronous_xfr) {
2030 temp.offset += xfer->frlengths[x - 1];
2031 temp.isoc_frame += temp.isoc_delta;
2032 } else {
2033 /* get next Page Cache pointer */
2034 temp.pc = xfer->frbuffers + x;
2035 }
2036 }
2037
2038 /* check if we should append a status stage */
2039
2040 if (xfer->flags_int.control_xfr &&
2041 !xfer->flags_int.control_act) {
2042
2043 /*
2044 * Send a DATA1 message and invert the current
2045 * endpoint direction.
2046 */
2047 temp.step_td = (xfer->nframes != 0);
2048 temp.direction = UE_GET_DIR(xfer->endpointno) ^ UE_DIR_IN;
2049 temp.len = 0;
2050 temp.pc = NULL;
2051 temp.shortpkt = 0;
2052 temp.last_frame = 1;
2053 temp.trb_type = XHCI_TRB_TYPE_STATUS_STAGE;
2054
2055 xhci_setup_generic_chain_sub(&temp);
2056 }
2057
2058 td = temp.td;
2059
2060 /* must have at least one frame! */
2061
2062 xfer->td_transfer_last = td;
2063
2064 DPRINTF("first=%p last=%p\n", xfer->td_transfer_first, td);
2065}
2066
2067static void
2068xhci_set_slot_pointer(struct xhci_softc *sc, uint8_t index, uint64_t dev_addr)
2069{
2070 struct usb_page_search buf_res;
2071 struct xhci_dev_ctx_addr *pdctxa;
2072
2073 usbd_get_page(&sc->sc_hw.ctx_pc, 0, &buf_res);
2074
2075 pdctxa = buf_res.buffer;
2076
2077 DPRINTF("addr[%u]=0x%016llx\n", index, (long long)dev_addr);
2078
2079 pdctxa->qwBaaDevCtxAddr[index] = htole64(dev_addr);
2080
2081 usb_pc_cpu_flush(&sc->sc_hw.ctx_pc);
2082}
2083
2084static usb_error_t
2085xhci_configure_mask(struct usb_device *udev, uint32_t mask, uint8_t drop)
2086{
2087 struct xhci_softc *sc = XHCI_BUS2SC(udev->bus);
2088 struct usb_page_search buf_inp;
2089 struct xhci_input_dev_ctx *pinp;
2090 uint32_t temp;
2091 uint8_t index;
2092 uint8_t x;
2093
2094 index = udev->controller_slot_id;
2095
2096 usbd_get_page(&sc->sc_hw.devs[index].input_pc, 0, &buf_inp);
2097
2098 pinp = buf_inp.buffer;
2099
2100 if (drop) {
2101 mask &= XHCI_INCTX_NON_CTRL_MASK;
2102 xhci_ctx_set_le32(sc, &pinp->ctx_input.dwInCtx0, mask);
2103 xhci_ctx_set_le32(sc, &pinp->ctx_input.dwInCtx1, 0);
2104 } else {
2105 xhci_ctx_set_le32(sc, &pinp->ctx_input.dwInCtx0, 0);
2106 xhci_ctx_set_le32(sc, &pinp->ctx_input.dwInCtx1, mask);
2107
2108 /* find most significant set bit */
2109 for (x = 31; x != 1; x--) {
2110 if (mask & (1 << x))
2111 break;
2112 }
2113
2114 /* adjust */
2115 x--;
2116
2117 /* figure out maximum */
2118 if (x > sc->sc_hw.devs[index].context_num) {
2119 sc->sc_hw.devs[index].context_num = x;
2120 temp = xhci_ctx_get_le32(sc, &pinp->ctx_slot.dwSctx0);
2121 temp &= ~XHCI_SCTX_0_CTX_NUM_SET(31);
2122 temp |= XHCI_SCTX_0_CTX_NUM_SET(x + 1);
2123 xhci_ctx_set_le32(sc, &pinp->ctx_slot.dwSctx0, temp);
2124 }
2125 }
2126 return (0);
2127}
2128
2129static usb_error_t
2130xhci_configure_endpoint(struct usb_device *udev,
2131 struct usb_endpoint_descriptor *edesc, struct xhci_endpoint_ext *pepext,
2132 uint16_t interval, uint8_t max_packet_count,
2133 uint8_t mult, uint8_t fps_shift, uint16_t max_packet_size,
2134 uint16_t max_frame_size, uint8_t ep_mode)
2135{
2136 struct usb_page_search buf_inp;
2137 struct xhci_softc *sc = XHCI_BUS2SC(udev->bus);
2138 struct xhci_input_dev_ctx *pinp;
2139 uint64_t ring_addr = pepext->physaddr;
2140 uint32_t temp;
2141 uint8_t index;
2142 uint8_t epno;
2143 uint8_t type;
2144
2145 index = udev->controller_slot_id;
2146
2147 usbd_get_page(&sc->sc_hw.devs[index].input_pc, 0, &buf_inp);
2148
2149 pinp = buf_inp.buffer;
2150
2151 epno = edesc->bEndpointAddress;
2152 type = edesc->bmAttributes & UE_XFERTYPE;
2153
2154 if (type == UE_CONTROL)
2155 epno |= UE_DIR_IN;
2156
2157 epno = XHCI_EPNO2EPID(epno);
2158
2159 if (epno == 0)
2160 return (USB_ERR_NO_PIPE); /* invalid */
2161
2162 if (max_packet_count == 0)
2163 return (USB_ERR_BAD_BUFSIZE);
2164
2165 max_packet_count--;
2166
2167 if (mult == 0)
2168 return (USB_ERR_BAD_BUFSIZE);
2169
2170 /* store endpoint mode */
2171 pepext->trb_ep_mode = ep_mode;
2172 usb_pc_cpu_flush(pepext->page_cache);
2173
2174 if (ep_mode == USB_EP_MODE_STREAMS) {
2175 temp = XHCI_EPCTX_0_EPSTATE_SET(0) |
2176 XHCI_EPCTX_0_MAXP_STREAMS_SET(XHCI_MAX_STREAMS_LOG - 1) |
2177 XHCI_EPCTX_0_LSA_SET(1);
2178
2179 ring_addr += sizeof(struct xhci_trb) *
2180 XHCI_MAX_TRANSFERS * XHCI_MAX_STREAMS;
2181 } else {
2182 temp = XHCI_EPCTX_0_EPSTATE_SET(0) |
2183 XHCI_EPCTX_0_MAXP_STREAMS_SET(0) |
2184 XHCI_EPCTX_0_LSA_SET(0);
2185
2186 ring_addr |= XHCI_EPCTX_2_DCS_SET(1);
2187 }
2188
2189 switch (udev->speed) {
2190 case USB_SPEED_FULL:
2191 case USB_SPEED_LOW:
2192 /* 1ms -> 125us */
2193 fps_shift += 3;
2194 break;
2195 default:
2196 break;
2197 }
2198
2199 switch (type) {
2200 case UE_INTERRUPT:
2201 if (fps_shift > 3)
2202 fps_shift--;
2203 temp |= XHCI_EPCTX_0_IVAL_SET(fps_shift);
2204 break;
2205 case UE_ISOCHRONOUS:
2206 temp |= XHCI_EPCTX_0_IVAL_SET(fps_shift);
2207
2208 switch (udev->speed) {
2209 case USB_SPEED_SUPER:
2210 if (mult > 3)
2211 mult = 3;
2212 temp |= XHCI_EPCTX_0_MULT_SET(mult - 1);
2213 max_packet_count /= mult;
2214 break;
2215 default:
2216 break;
2217 }
2218 break;
2219 default:
2220 break;
2221 }
2222
2223 xhci_ctx_set_le32(sc, &pinp->ctx_ep[epno - 1].dwEpCtx0, temp);
2224
2225 temp =
2226 XHCI_EPCTX_1_HID_SET(0) |
2227 XHCI_EPCTX_1_MAXB_SET(max_packet_count) |
2228 XHCI_EPCTX_1_MAXP_SIZE_SET(max_packet_size);
2229
2230 if ((udev->parent_hs_hub != NULL) || (udev->address != 0)) {
2231 if (type != UE_ISOCHRONOUS)
2232 temp |= XHCI_EPCTX_1_CERR_SET(3);
2233 }
2234
2235 switch (type) {
2236 case UE_CONTROL:
2237 temp |= XHCI_EPCTX_1_EPTYPE_SET(4);
2238 break;
2239 case UE_ISOCHRONOUS:
2240 temp |= XHCI_EPCTX_1_EPTYPE_SET(1);
2241 break;
2242 case UE_BULK:
2243 temp |= XHCI_EPCTX_1_EPTYPE_SET(2);
2244 break;
2245 default:
2246 temp |= XHCI_EPCTX_1_EPTYPE_SET(3);
2247 break;
2248 }
2249
2250 /* check for IN direction */
2251 if (epno & 1)
2252 temp |= XHCI_EPCTX_1_EPTYPE_SET(4);
2253
2254 xhci_ctx_set_le32(sc, &pinp->ctx_ep[epno - 1].dwEpCtx1, temp);
2255 xhci_ctx_set_le64(sc, &pinp->ctx_ep[epno - 1].qwEpCtx2, ring_addr);
2256
2257 switch (edesc->bmAttributes & UE_XFERTYPE) {
2258 case UE_INTERRUPT:
2259 case UE_ISOCHRONOUS:
2260 temp = XHCI_EPCTX_4_MAX_ESIT_PAYLOAD_SET(max_frame_size) |
2261 XHCI_EPCTX_4_AVG_TRB_LEN_SET(MIN(XHCI_PAGE_SIZE,
2262 max_frame_size));
2263 break;
2264 case UE_CONTROL:
2265 temp = XHCI_EPCTX_4_AVG_TRB_LEN_SET(8);
2266 break;
2267 default:
2268 temp = XHCI_EPCTX_4_AVG_TRB_LEN_SET(XHCI_PAGE_SIZE);
2269 break;
2270 }
2271
2272 xhci_ctx_set_le32(sc, &pinp->ctx_ep[epno - 1].dwEpCtx4, temp);
2273
2274#ifdef USB_DEBUG
2275 xhci_dump_endpoint(sc, &pinp->ctx_ep[epno - 1]);
2276#endif
2277 usb_pc_cpu_flush(&sc->sc_hw.devs[index].input_pc);
2278
2279 return (0); /* success */
2280}
2281
2282static usb_error_t
2283xhci_configure_endpoint_by_xfer(struct usb_xfer *xfer)
2284{
2285 struct xhci_endpoint_ext *pepext;
2286 struct usb_endpoint_ss_comp_descriptor *ecomp;
2287 usb_stream_t x;
2288
2289 pepext = xhci_get_endpoint_ext(xfer->xroot->udev,
2290 xfer->endpoint->edesc);
2291
2292 ecomp = xfer->endpoint->ecomp;
2293
2294 for (x = 0; x != XHCI_MAX_STREAMS; x++) {
2295 uint64_t temp;
2296
2297 /* halt any transfers */
2298 pepext->trb[x * XHCI_MAX_TRANSFERS].dwTrb3 = 0;
2299
2300 /* compute start of TRB ring for stream "x" */
2301 temp = pepext->physaddr +
2302 (x * XHCI_MAX_TRANSFERS * sizeof(struct xhci_trb)) +
2303 XHCI_SCTX_0_SCT_SEC_TR_RING;
2304
2305 /* make tree structure */
2306 pepext->trb[(XHCI_MAX_TRANSFERS *
2307 XHCI_MAX_STREAMS) + x].qwTrb0 = htole64(temp);
2308
2309 /* reserved fields */
2310 pepext->trb[(XHCI_MAX_TRANSFERS *
2311 XHCI_MAX_STREAMS) + x].dwTrb2 = 0;
2312 pepext->trb[(XHCI_MAX_TRANSFERS *
2313 XHCI_MAX_STREAMS) + x].dwTrb3 = 0;
2314 }
2315 usb_pc_cpu_flush(pepext->page_cache);
2316
2317 return (xhci_configure_endpoint(xfer->xroot->udev,
2318 xfer->endpoint->edesc, pepext,
2319 xfer->interval, xfer->max_packet_count,
2320 (ecomp != NULL) ? UE_GET_SS_ISO_MULT(ecomp->bmAttributes) + 1 : 1,
2321 usbd_xfer_get_fps_shift(xfer), xfer->max_packet_size,
2322 xfer->max_frame_size, xfer->endpoint->ep_mode));
2323}
2324
2325static usb_error_t
2326xhci_configure_device(struct usb_device *udev)
2327{
2328 struct xhci_softc *sc = XHCI_BUS2SC(udev->bus);
2329 struct usb_page_search buf_inp;
2330 struct usb_page_cache *pcinp;
2331 struct xhci_input_dev_ctx *pinp;
2332 struct usb_device *hubdev;
2333 uint32_t temp;
2334 uint32_t route;
2335 uint32_t rh_port;
2336 uint8_t is_hub;
2337 uint8_t index;
2338 uint8_t depth;
2339
2340 index = udev->controller_slot_id;
2341
2342 DPRINTF("index=%u\n", index);
2343
2344 pcinp = &sc->sc_hw.devs[index].input_pc;
2345
2346 usbd_get_page(pcinp, 0, &buf_inp);
2347
2348 pinp = buf_inp.buffer;
2349
2350 rh_port = 0;
2351 route = 0;
2352
2353 /* figure out route string and root HUB port number */
2354
2355 for (hubdev = udev; hubdev != NULL; hubdev = hubdev->parent_hub) {
2356
2357 if (hubdev->parent_hub == NULL)
2358 break;
2359
2360 depth = hubdev->parent_hub->depth;
2361
2362 /*
2363 * NOTE: HS/FS/LS devices and the SS root HUB can have
2364 * more than 15 ports
2365 */
2366
2367 rh_port = hubdev->port_no;
2368
2369 if (depth == 0)
2370 break;
2371
2372 if (rh_port > 15)
2373 rh_port = 15;
2374
2375 if (depth < 6)
2376 route |= rh_port << (4 * (depth - 1));
2377 }
2378
2379 DPRINTF("Route=0x%08x\n", route);
2380
2381 temp = XHCI_SCTX_0_ROUTE_SET(route) |
2382 XHCI_SCTX_0_CTX_NUM_SET(
2383 sc->sc_hw.devs[index].context_num + 1);
2384
2385 switch (udev->speed) {
2386 case USB_SPEED_LOW:
2387 temp |= XHCI_SCTX_0_SPEED_SET(2);
2388 if (udev->parent_hs_hub != NULL &&
2389 udev->parent_hs_hub->ddesc.bDeviceProtocol ==
2390 UDPROTO_HSHUBMTT) {
2391 DPRINTF("Device inherits MTT\n");
2392 temp |= XHCI_SCTX_0_MTT_SET(1);
2393 }
2394 break;
2395 case USB_SPEED_HIGH:
2396 temp |= XHCI_SCTX_0_SPEED_SET(3);
2397 if (sc->sc_hw.devs[index].nports != 0 &&
2398 udev->ddesc.bDeviceProtocol == UDPROTO_HSHUBMTT) {
2399 DPRINTF("HUB supports MTT\n");
2400 temp |= XHCI_SCTX_0_MTT_SET(1);
2401 }
2402 break;
2403 case USB_SPEED_FULL:
2404 temp |= XHCI_SCTX_0_SPEED_SET(1);
2405 if (udev->parent_hs_hub != NULL &&
2406 udev->parent_hs_hub->ddesc.bDeviceProtocol ==
2407 UDPROTO_HSHUBMTT) {
2408 DPRINTF("Device inherits MTT\n");
2409 temp |= XHCI_SCTX_0_MTT_SET(1);
2410 }
2411 break;
2412 default:
2413 temp |= XHCI_SCTX_0_SPEED_SET(4);
2414 break;
2415 }
2416
2417 is_hub = sc->sc_hw.devs[index].nports != 0 &&
2418 (udev->speed == USB_SPEED_SUPER ||
2419 udev->speed == USB_SPEED_HIGH);
2420
2421 if (is_hub)
2422 temp |= XHCI_SCTX_0_HUB_SET(1);
2423
2424 xhci_ctx_set_le32(sc, &pinp->ctx_slot.dwSctx0, temp);
2425
2426 temp = XHCI_SCTX_1_RH_PORT_SET(rh_port);
2427
2428 if (is_hub) {
2429 temp |= XHCI_SCTX_1_NUM_PORTS_SET(
2430 sc->sc_hw.devs[index].nports);
2431 }
2432
2433 switch (udev->speed) {
2434 case USB_SPEED_SUPER:
2435 switch (sc->sc_hw.devs[index].state) {
2436 case XHCI_ST_ADDRESSED:
2437 case XHCI_ST_CONFIGURED:
2438 /* enable power save */
2439 temp |= XHCI_SCTX_1_MAX_EL_SET(sc->sc_exit_lat_max);
2440 break;
2441 default:
2442 /* disable power save */
2443 break;
2444 }
2445 break;
2446 default:
2447 break;
2448 }
2449
2450 xhci_ctx_set_le32(sc, &pinp->ctx_slot.dwSctx1, temp);
2451
2452 temp = XHCI_SCTX_2_IRQ_TARGET_SET(0);
2453
2454 if (is_hub) {
2455 temp |= XHCI_SCTX_2_TT_THINK_TIME_SET(
2456 sc->sc_hw.devs[index].tt);
2457 }
2458
2459 hubdev = udev->parent_hs_hub;
2460
2461 /* check if we should activate the transaction translator */
2462 switch (udev->speed) {
2463 case USB_SPEED_FULL:
2464 case USB_SPEED_LOW:
2465 if (hubdev != NULL) {
2466 temp |= XHCI_SCTX_2_TT_HUB_SID_SET(
2467 hubdev->controller_slot_id);
2468 temp |= XHCI_SCTX_2_TT_PORT_NUM_SET(
2469 udev->hs_port_no);
2470 }
2471 break;
2472 default:
2473 break;
2474 }
2475
2476 xhci_ctx_set_le32(sc, &pinp->ctx_slot.dwSctx2, temp);
2477
2478 temp = XHCI_SCTX_3_DEV_ADDR_SET(udev->address) |
2479 XHCI_SCTX_3_SLOT_STATE_SET(0);
2480
2481 xhci_ctx_set_le32(sc, &pinp->ctx_slot.dwSctx3, temp);
2482
2483#ifdef USB_DEBUG
2484 xhci_dump_device(sc, &pinp->ctx_slot);
2485#endif
2486 usb_pc_cpu_flush(pcinp);
2487
2488 return (0); /* success */
2489}
2490
2491static usb_error_t
2492xhci_alloc_device_ext(struct usb_device *udev)
2493{
2494 struct xhci_softc *sc = XHCI_BUS2SC(udev->bus);
2495 struct usb_page_search buf_dev;
2496 struct usb_page_search buf_ep;
2497 struct xhci_trb *trb;
2498 struct usb_page_cache *pc;
2499 struct usb_page *pg;
2500 uint64_t addr;
2501 uint8_t index;
2502 uint8_t i;
2503
2504 index = udev->controller_slot_id;
2505
2506 pc = &sc->sc_hw.devs[index].device_pc;
2507 pg = &sc->sc_hw.devs[index].device_pg;
2508
2509 /* need to initialize the page cache */
2510 pc->tag_parent = sc->sc_bus.dma_parent_tag;
2511
2512 if (usb_pc_alloc_mem(pc, pg, sc->sc_ctx_is_64_byte ?
2513 (2 * sizeof(struct xhci_dev_ctx)) :
2514 sizeof(struct xhci_dev_ctx), XHCI_PAGE_SIZE))
2515 goto error;
2516
2517 usbd_get_page(pc, 0, &buf_dev);
2518
2519 pc = &sc->sc_hw.devs[index].input_pc;
2520 pg = &sc->sc_hw.devs[index].input_pg;
2521
2522 /* need to initialize the page cache */
2523 pc->tag_parent = sc->sc_bus.dma_parent_tag;
2524
2525 if (usb_pc_alloc_mem(pc, pg, sc->sc_ctx_is_64_byte ?
2526 (2 * sizeof(struct xhci_input_dev_ctx)) :
2527 sizeof(struct xhci_input_dev_ctx), XHCI_PAGE_SIZE)) {
2528 goto error;
2529 }
2530
2531 pc = &sc->sc_hw.devs[index].endpoint_pc;
2532 pg = &sc->sc_hw.devs[index].endpoint_pg;
2533
2534 /* need to initialize the page cache */
2535 pc->tag_parent = sc->sc_bus.dma_parent_tag;
2536
2537 if (usb_pc_alloc_mem(pc, pg,
2538 sizeof(struct xhci_dev_endpoint_trbs), XHCI_PAGE_SIZE)) {
2539 goto error;
2540 }
2541
2542 /* initialise all endpoint LINK TRBs */
2543
2544 for (i = 0; i != XHCI_MAX_ENDPOINTS; i++) {
2545
2546 /* lookup endpoint TRB ring */
2547 usbd_get_page(pc, (uintptr_t)&
2548 ((struct xhci_dev_endpoint_trbs *)0)->trb[i][0], &buf_ep);
2549
2550 /* get TRB pointer */
2551 trb = buf_ep.buffer;
2552 trb += XHCI_MAX_TRANSFERS - 1;
2553
2554 /* get TRB start address */
2555 addr = buf_ep.physaddr;
2556
2557 /* create LINK TRB */
2558 trb->qwTrb0 = htole64(addr);
2559 trb->dwTrb2 = htole32(XHCI_TRB_2_IRQ_SET(0));
2560 trb->dwTrb3 = htole32(XHCI_TRB_3_CYCLE_BIT |
2561 XHCI_TRB_3_TYPE_SET(XHCI_TRB_TYPE_LINK));
2562 }
2563
2564 usb_pc_cpu_flush(pc);
2565
2566 xhci_set_slot_pointer(sc, index, buf_dev.physaddr);
2567
2568 return (0);
2569
2570error:
2571 xhci_free_device_ext(udev);
2572
2573 return (USB_ERR_NOMEM);
2574}
2575
2576static void
2577xhci_free_device_ext(struct usb_device *udev)
2578{
2579 struct xhci_softc *sc = XHCI_BUS2SC(udev->bus);
2580 uint8_t index;
2581
2582 index = udev->controller_slot_id;
2583 xhci_set_slot_pointer(sc, index, 0);
2584
2585 usb_pc_free_mem(&sc->sc_hw.devs[index].device_pc);
2586 usb_pc_free_mem(&sc->sc_hw.devs[index].input_pc);
2587 usb_pc_free_mem(&sc->sc_hw.devs[index].endpoint_pc);
2588}
2589
2590static struct xhci_endpoint_ext *
2591xhci_get_endpoint_ext(struct usb_device *udev, struct usb_endpoint_descriptor *edesc)
2592{
2593 struct xhci_softc *sc = XHCI_BUS2SC(udev->bus);
2594 struct xhci_endpoint_ext *pepext;
2595 struct usb_page_cache *pc;
2596 struct usb_page_search buf_ep;
2597 uint8_t epno;
2598 uint8_t index;
2599
2600 epno = edesc->bEndpointAddress;
2601 if ((edesc->bmAttributes & UE_XFERTYPE) == UE_CONTROL)
2602 epno |= UE_DIR_IN;
2603
2604 epno = XHCI_EPNO2EPID(epno);
2605
2606 index = udev->controller_slot_id;
2607
2608 pc = &sc->sc_hw.devs[index].endpoint_pc;
2609
2610 usbd_get_page(pc, (uintptr_t)&((struct xhci_dev_endpoint_trbs *)0)->
2611 trb[epno][0], &buf_ep);
2612
2613 pepext = &sc->sc_hw.devs[index].endp[epno];
2614 pepext->page_cache = pc;
2615 pepext->trb = buf_ep.buffer;
2616 pepext->physaddr = buf_ep.physaddr;
2617
2618 return (pepext);
2619}
2620
2621static void
2622xhci_endpoint_doorbell(struct usb_xfer *xfer)
2623{
2624 struct xhci_softc *sc = XHCI_BUS2SC(xfer->xroot->bus);
2625 uint8_t epno;
2626 uint8_t index;
2627
2628 epno = xfer->endpointno;
2629 if (xfer->flags_int.control_xfr)
2630 epno |= UE_DIR_IN;
2631
2632 epno = XHCI_EPNO2EPID(epno);
2633 index = xfer->xroot->udev->controller_slot_id;
2634
2635 if (xfer->xroot->udev->flags.self_suspended == 0) {
2636 XWRITE4(sc, door, XHCI_DOORBELL(index),
2637 epno | XHCI_DB_SID_SET(xfer->stream_id));
2638 }
2639}
2640
2641static void
2642xhci_transfer_remove(struct usb_xfer *xfer, usb_error_t error)
2643{
2644 struct xhci_endpoint_ext *pepext;
2645
2646 if (xfer->flags_int.bandwidth_reclaimed) {
2647 xfer->flags_int.bandwidth_reclaimed = 0;
2648
2649 pepext = xhci_get_endpoint_ext(xfer->xroot->udev,
2650 xfer->endpoint->edesc);
2651
2652 pepext->trb_used[xfer->stream_id]--;
2653
2654 pepext->xfer[xfer->qh_pos] = NULL;
2655
2656 if (error && pepext->trb_running != 0) {
2657 pepext->trb_halted = 1;
2658 pepext->trb_running = 0;
2659 }
2660 }
2661}
2662
2663static usb_error_t
2664xhci_transfer_insert(struct usb_xfer *xfer)
2665{
2666 struct xhci_td *td_first;
2667 struct xhci_td *td_last;
|
| 2668 struct xhci_trb *trb_link;
|
2658 struct xhci_endpoint_ext *pepext;
2659 uint64_t addr;
2660 usb_stream_t id;
2661 uint8_t i;
2662 uint8_t inext;
2663 uint8_t trb_limit;
2664
2665 DPRINTFN(8, "\n");
2666
2667 id = xfer->stream_id;
2668
2669 /* check if already inserted */
2670 if (xfer->flags_int.bandwidth_reclaimed) {
2671 DPRINTFN(8, "Already in schedule\n");
2672 return (0);
2673 }
2674
2675 pepext = xhci_get_endpoint_ext(xfer->xroot->udev,
2676 xfer->endpoint->edesc);
2677
2678 td_first = xfer->td_transfer_first;
2679 td_last = xfer->td_transfer_last;
2680 addr = pepext->physaddr;
2681
2682 switch (xfer->endpoint->edesc->bmAttributes & UE_XFERTYPE) {
2683 case UE_CONTROL:
2684 case UE_INTERRUPT:
2685 /* single buffered */
2686 trb_limit = 1;
2687 break;
2688 default:
2689 /* multi buffered */
2690 trb_limit = (XHCI_MAX_TRANSFERS - 2);
2691 break;
2692 }
2693
2694 if (pepext->trb_used[id] >= trb_limit) {
2695 DPRINTFN(8, "Too many TDs queued.\n");
2696 return (USB_ERR_NOMEM);
2697 }
2698
2699 /* check for stopped condition, after putting transfer on interrupt queue */
2700 if (pepext->trb_running == 0) {
2701 struct xhci_softc *sc = XHCI_BUS2SC(xfer->xroot->bus);
2702
2703 DPRINTFN(8, "Not running\n");
2704
2705 /* start configuration */
2706 (void)usb_proc_msignal(USB_BUS_CONTROL_XFER_PROC(&sc->sc_bus),
2707 &sc->sc_config_msg[0], &sc->sc_config_msg[1]);
2708 return (0);
2709 }
2710
2711 pepext->trb_used[id]++;
2712
2713 /* get current TRB index */
2714 i = pepext->trb_index[id];
2715
2716 /* get next TRB index */
2717 inext = (i + 1);
2718
2719 /* the last entry of the ring is a hardcoded link TRB */
2720 if (inext >= (XHCI_MAX_TRANSFERS - 1))
2721 inext = 0;
2722
2723 /* store next TRB index, before stream ID offset is added */
2724 pepext->trb_index[id] = inext;
2725
2726 /* offset for stream */
2727 i += id * XHCI_MAX_TRANSFERS;
2728 inext += id * XHCI_MAX_TRANSFERS;
2729
2730 /* compute terminating return address */
2731 addr += (inext * sizeof(struct xhci_trb));
2732
| 2669 struct xhci_endpoint_ext *pepext;
2670 uint64_t addr;
2671 usb_stream_t id;
2672 uint8_t i;
2673 uint8_t inext;
2674 uint8_t trb_limit;
2675
2676 DPRINTFN(8, "\n");
2677
2678 id = xfer->stream_id;
2679
2680 /* check if already inserted */
2681 if (xfer->flags_int.bandwidth_reclaimed) {
2682 DPRINTFN(8, "Already in schedule\n");
2683 return (0);
2684 }
2685
2686 pepext = xhci_get_endpoint_ext(xfer->xroot->udev,
2687 xfer->endpoint->edesc);
2688
2689 td_first = xfer->td_transfer_first;
2690 td_last = xfer->td_transfer_last;
2691 addr = pepext->physaddr;
2692
2693 switch (xfer->endpoint->edesc->bmAttributes & UE_XFERTYPE) {
2694 case UE_CONTROL:
2695 case UE_INTERRUPT:
2696 /* single buffered */
2697 trb_limit = 1;
2698 break;
2699 default:
2700 /* multi buffered */
2701 trb_limit = (XHCI_MAX_TRANSFERS - 2);
2702 break;
2703 }
2704
2705 if (pepext->trb_used[id] >= trb_limit) {
2706 DPRINTFN(8, "Too many TDs queued.\n");
2707 return (USB_ERR_NOMEM);
2708 }
2709
2710 /* check for stopped condition, after putting transfer on interrupt queue */
2711 if (pepext->trb_running == 0) {
2712 struct xhci_softc *sc = XHCI_BUS2SC(xfer->xroot->bus);
2713
2714 DPRINTFN(8, "Not running\n");
2715
2716 /* start configuration */
2717 (void)usb_proc_msignal(USB_BUS_CONTROL_XFER_PROC(&sc->sc_bus),
2718 &sc->sc_config_msg[0], &sc->sc_config_msg[1]);
2719 return (0);
2720 }
2721
2722 pepext->trb_used[id]++;
2723
2724 /* get current TRB index */
2725 i = pepext->trb_index[id];
2726
2727 /* get next TRB index */
2728 inext = (i + 1);
2729
2730 /* the last entry of the ring is a hardcoded link TRB */
2731 if (inext >= (XHCI_MAX_TRANSFERS - 1))
2732 inext = 0;
2733
2734 /* store next TRB index, before stream ID offset is added */
2735 pepext->trb_index[id] = inext;
2736
2737 /* offset for stream */
2738 i += id * XHCI_MAX_TRANSFERS;
2739 inext += id * XHCI_MAX_TRANSFERS;
2740
2741 /* compute terminating return address */
2742 addr += (inext * sizeof(struct xhci_trb));
2743
|
| 2744 /* compute link TRB pointer */
2745 trb_link = td_last->td_trb + td_last->ntrb;
2746
|
2733 /* update next pointer of last link TRB */
| 2747 /* update next pointer of last link TRB */
|
2734 td_last->td_trb[td_last->ntrb].qwTrb0 = htole64(addr);
2735 td_last->td_trb[td_last->ntrb].dwTrb2 = htole32(XHCI_TRB_2_IRQ_SET(0));
2736 td_last->td_trb[td_last->ntrb].dwTrb3 = htole32(XHCI_TRB_3_IOC_BIT |
2737 XHCI_TRB_3_CYCLE_BIT | XHCI_TRB_3_TYPE_SET(XHCI_TRB_TYPE_LINK));
| 2748 trb_link->qwTrb0 = htole64(addr);
2749 trb_link->dwTrb2 = htole32(XHCI_TRB_2_IRQ_SET(0));
2750 trb_link->dwTrb3 = htole32(XHCI_TRB_3_IOC_BIT |
2751 XHCI_TRB_3_CYCLE_BIT |
2752 XHCI_TRB_3_TYPE_SET(XHCI_TRB_TYPE_LINK));
|
2738
2739#ifdef USB_DEBUG
2740 xhci_dump_trb(&td_last->td_trb[td_last->ntrb]);
2741#endif
2742 usb_pc_cpu_flush(td_last->page_cache);
2743
2744 /* write ahead chain end marker */
2745
2746 pepext->trb[inext].qwTrb0 = 0;
2747 pepext->trb[inext].dwTrb2 = 0;
2748 pepext->trb[inext].dwTrb3 = 0;
2749
2750 /* update next pointer of link TRB */
2751
2752 pepext->trb[i].qwTrb0 = htole64((uint64_t)td_first->td_self);
2753 pepext->trb[i].dwTrb2 = htole32(XHCI_TRB_2_IRQ_SET(0));
2754
2755#ifdef USB_DEBUG
2756 xhci_dump_trb(&pepext->trb[i]);
2757#endif
2758 usb_pc_cpu_flush(pepext->page_cache);
2759
2760 /* toggle cycle bit which activates the transfer chain */
2761
2762 pepext->trb[i].dwTrb3 = htole32(XHCI_TRB_3_CYCLE_BIT |
2763 XHCI_TRB_3_TYPE_SET(XHCI_TRB_TYPE_LINK));
2764
2765 usb_pc_cpu_flush(pepext->page_cache);
2766
2767 DPRINTF("qh_pos = %u\n", i);
2768
2769 pepext->xfer[i] = xfer;
2770
2771 xfer->qh_pos = i;
2772
2773 xfer->flags_int.bandwidth_reclaimed = 1;
2774
2775 xhci_endpoint_doorbell(xfer);
2776
2777 return (0);
2778}
2779
2780static void
2781xhci_root_intr(struct xhci_softc *sc)
2782{
2783 uint16_t i;
2784
2785 USB_BUS_LOCK_ASSERT(&sc->sc_bus, MA_OWNED);
2786
2787 /* clear any old interrupt data */
2788 memset(sc->sc_hub_idata, 0, sizeof(sc->sc_hub_idata));
2789
2790 for (i = 1; i <= sc->sc_noport; i++) {
2791 /* pick out CHANGE bits from the status register */
2792 if (XREAD4(sc, oper, XHCI_PORTSC(i)) & (
2793 XHCI_PS_CSC | XHCI_PS_PEC |
2794 XHCI_PS_OCC | XHCI_PS_WRC |
2795 XHCI_PS_PRC | XHCI_PS_PLC |
2796 XHCI_PS_CEC)) {
2797 sc->sc_hub_idata[i / 8] |= 1 << (i % 8);
2798 DPRINTF("port %d changed\n", i);
2799 }
2800 }
2801 uhub_root_intr(&sc->sc_bus, sc->sc_hub_idata,
2802 sizeof(sc->sc_hub_idata));
2803}
2804
2805/*------------------------------------------------------------------------*
2806 * xhci_device_done - XHCI done handler
2807 *
2808 * NOTE: This function can be called two times in a row on
2809 * the same USB transfer. From close and from interrupt.
2810 *------------------------------------------------------------------------*/
2811static void
2812xhci_device_done(struct usb_xfer *xfer, usb_error_t error)
2813{
2814 DPRINTFN(2, "xfer=%p, endpoint=%p, error=%d\n",
2815 xfer, xfer->endpoint, error);
2816
2817 /* remove transfer from HW queue */
2818 xhci_transfer_remove(xfer, error);
2819
2820 /* dequeue transfer and start next transfer */
2821 usbd_transfer_done(xfer, error);
2822}
2823
2824/*------------------------------------------------------------------------*
2825 * XHCI data transfer support (generic type)
2826 *------------------------------------------------------------------------*/
2827static void
2828xhci_device_generic_open(struct usb_xfer *xfer)
2829{
2830 if (xfer->flags_int.isochronous_xfr) {
2831 switch (xfer->xroot->udev->speed) {
2832 case USB_SPEED_FULL:
2833 break;
2834 default:
2835 usb_hs_bandwidth_alloc(xfer);
2836 break;
2837 }
2838 }
2839}
2840
2841static void
2842xhci_device_generic_close(struct usb_xfer *xfer)
2843{
2844 DPRINTF("\n");
2845
2846 xhci_device_done(xfer, USB_ERR_CANCELLED);
2847
2848 if (xfer->flags_int.isochronous_xfr) {
2849 switch (xfer->xroot->udev->speed) {
2850 case USB_SPEED_FULL:
2851 break;
2852 default:
2853 usb_hs_bandwidth_free(xfer);
2854 break;
2855 }
2856 }
2857}
2858
2859static void
2860xhci_device_generic_multi_enter(struct usb_endpoint *ep,
2861 usb_stream_t stream_id, struct usb_xfer *enter_xfer)
2862{
2863 struct usb_xfer *xfer;
2864
2865 /* check if there is a current transfer */
2866 xfer = ep->endpoint_q[stream_id].curr;
2867 if (xfer == NULL)
2868 return;
2869
2870 /*
2871 * Check if the current transfer is started and then pickup
2872 * the next one, if any. Else wait for next start event due to
2873 * block on failure feature.
2874 */
2875 if (!xfer->flags_int.bandwidth_reclaimed)
2876 return;
2877
2878 xfer = TAILQ_FIRST(&ep->endpoint_q[stream_id].head);
2879 if (xfer == NULL) {
2880 /*
2881 * In case of enter we have to consider that the
2882 * transfer is queued by the USB core after the enter
2883 * method is called.
2884 */
2885 xfer = enter_xfer;
2886
2887 if (xfer == NULL)
2888 return;
2889 }
2890
2891 /* try to multi buffer */
2892 xhci_transfer_insert(xfer);
2893}
2894
2895static void
2896xhci_device_generic_enter(struct usb_xfer *xfer)
2897{
2898 DPRINTF("\n");
2899
2900 /* setup TD's and QH */
2901 xhci_setup_generic_chain(xfer);
2902
2903 xhci_device_generic_multi_enter(xfer->endpoint,
2904 xfer->stream_id, xfer);
2905}
2906
2907static void
2908xhci_device_generic_start(struct usb_xfer *xfer)
2909{
2910 DPRINTF("\n");
2911
2912 /* try to insert xfer on HW queue */
2913 xhci_transfer_insert(xfer);
2914
2915 /* try to multi buffer */
2916 xhci_device_generic_multi_enter(xfer->endpoint,
2917 xfer->stream_id, NULL);
2918
2919 /* add transfer last on interrupt queue */
2920 usbd_transfer_enqueue(&xfer->xroot->bus->intr_q, xfer);
2921
2922 /* start timeout, if any */
2923 if (xfer->timeout != 0)
2924 usbd_transfer_timeout_ms(xfer, &xhci_timeout, xfer->timeout);
2925}
2926
2927struct usb_pipe_methods xhci_device_generic_methods =
2928{
2929 .open = xhci_device_generic_open,
2930 .close = xhci_device_generic_close,
2931 .enter = xhci_device_generic_enter,
2932 .start = xhci_device_generic_start,
2933};
2934
2935/*------------------------------------------------------------------------*
2936 * xhci root HUB support
2937 *------------------------------------------------------------------------*
2938 * Simulate a hardware HUB by handling all the necessary requests.
2939 *------------------------------------------------------------------------*/
2940
2941#define HSETW(ptr, val) ptr = { (uint8_t)(val), (uint8_t)((val) >> 8) }
2942
2943static const
2944struct usb_device_descriptor xhci_devd =
2945{
2946 .bLength = sizeof(xhci_devd),
2947 .bDescriptorType = UDESC_DEVICE, /* type */
2948 HSETW(.bcdUSB, 0x0300), /* USB version */
2949 .bDeviceClass = UDCLASS_HUB, /* class */
2950 .bDeviceSubClass = UDSUBCLASS_HUB, /* subclass */
2951 .bDeviceProtocol = UDPROTO_SSHUB, /* protocol */
2952 .bMaxPacketSize = 9, /* max packet size */
2953 HSETW(.idVendor, 0x0000), /* vendor */
2954 HSETW(.idProduct, 0x0000), /* product */
2955 HSETW(.bcdDevice, 0x0100), /* device version */
2956 .iManufacturer = 1,
2957 .iProduct = 2,
2958 .iSerialNumber = 0,
2959 .bNumConfigurations = 1, /* # of configurations */
2960};
2961
2962static const
2963struct xhci_bos_desc xhci_bosd = {
2964 .bosd = {
2965 .bLength = sizeof(xhci_bosd.bosd),
2966 .bDescriptorType = UDESC_BOS,
2967 HSETW(.wTotalLength, sizeof(xhci_bosd)),
2968 .bNumDeviceCaps = 3,
2969 },
2970 .usb2extd = {
2971 .bLength = sizeof(xhci_bosd.usb2extd),
2972 .bDescriptorType = 1,
2973 .bDevCapabilityType = 2,
2974 .bmAttributes[0] = 2,
2975 },
2976 .usbdcd = {
2977 .bLength = sizeof(xhci_bosd.usbdcd),
2978 .bDescriptorType = UDESC_DEVICE_CAPABILITY,
2979 .bDevCapabilityType = 3,
2980 .bmAttributes = 0, /* XXX */
2981 HSETW(.wSpeedsSupported, 0x000C),
2982 .bFunctionalitySupport = 8,
2983 .bU1DevExitLat = 255, /* dummy - not used */
2984 .wU2DevExitLat = { 0x00, 0x08 },
2985 },
2986 .cidd = {
2987 .bLength = sizeof(xhci_bosd.cidd),
2988 .bDescriptorType = 1,
2989 .bDevCapabilityType = 4,
2990 .bReserved = 0,
2991 .bContainerID = 0, /* XXX */
2992 },
2993};
2994
2995static const
2996struct xhci_config_desc xhci_confd = {
2997 .confd = {
2998 .bLength = sizeof(xhci_confd.confd),
2999 .bDescriptorType = UDESC_CONFIG,
3000 .wTotalLength[0] = sizeof(xhci_confd),
3001 .bNumInterface = 1,
3002 .bConfigurationValue = 1,
3003 .iConfiguration = 0,
3004 .bmAttributes = UC_SELF_POWERED,
3005 .bMaxPower = 0 /* max power */
3006 },
3007 .ifcd = {
3008 .bLength = sizeof(xhci_confd.ifcd),
3009 .bDescriptorType = UDESC_INTERFACE,
3010 .bNumEndpoints = 1,
3011 .bInterfaceClass = UICLASS_HUB,
3012 .bInterfaceSubClass = UISUBCLASS_HUB,
3013 .bInterfaceProtocol = 0,
3014 },
3015 .endpd = {
3016 .bLength = sizeof(xhci_confd.endpd),
3017 .bDescriptorType = UDESC_ENDPOINT,
3018 .bEndpointAddress = UE_DIR_IN | XHCI_INTR_ENDPT,
3019 .bmAttributes = UE_INTERRUPT,
3020 .wMaxPacketSize[0] = 2, /* max 15 ports */
3021 .bInterval = 255,
3022 },
3023 .endpcd = {
3024 .bLength = sizeof(xhci_confd.endpcd),
3025 .bDescriptorType = UDESC_ENDPOINT_SS_COMP,
3026 .bMaxBurst = 0,
3027 .bmAttributes = 0,
3028 },
3029};
3030
3031static const
3032struct usb_hub_ss_descriptor xhci_hubd = {
3033 .bLength = sizeof(xhci_hubd),
3034 .bDescriptorType = UDESC_SS_HUB,
3035};
3036
3037static usb_error_t
3038xhci_roothub_exec(struct usb_device *udev,
3039 struct usb_device_request *req, const void **pptr, uint16_t *plength)
3040{
3041 struct xhci_softc *sc = XHCI_BUS2SC(udev->bus);
3042 const char *str_ptr;
3043 const void *ptr;
3044 uint32_t port;
3045 uint32_t v;
3046 uint16_t len;
3047 uint16_t i;
3048 uint16_t value;
3049 uint16_t index;
3050 uint8_t j;
3051 usb_error_t err;
3052
3053 USB_BUS_LOCK_ASSERT(&sc->sc_bus, MA_OWNED);
3054
3055 /* buffer reset */
3056 ptr = (const void *)&sc->sc_hub_desc;
3057 len = 0;
3058 err = 0;
3059
3060 value = UGETW(req->wValue);
3061 index = UGETW(req->wIndex);
3062
3063 DPRINTFN(3, "type=0x%02x request=0x%02x wLen=0x%04x "
3064 "wValue=0x%04x wIndex=0x%04x\n",
3065 req->bmRequestType, req->bRequest,
3066 UGETW(req->wLength), value, index);
3067
3068#define C(x,y) ((x) | ((y) << 8))
3069 switch (C(req->bRequest, req->bmRequestType)) {
3070 case C(UR_CLEAR_FEATURE, UT_WRITE_DEVICE):
3071 case C(UR_CLEAR_FEATURE, UT_WRITE_INTERFACE):
3072 case C(UR_CLEAR_FEATURE, UT_WRITE_ENDPOINT):
3073 /*
3074 * DEVICE_REMOTE_WAKEUP and ENDPOINT_HALT are no-ops
3075 * for the integrated root hub.
3076 */
3077 break;
3078 case C(UR_GET_CONFIG, UT_READ_DEVICE):
3079 len = 1;
3080 sc->sc_hub_desc.temp[0] = sc->sc_conf;
3081 break;
3082 case C(UR_GET_DESCRIPTOR, UT_READ_DEVICE):
3083 switch (value >> 8) {
3084 case UDESC_DEVICE:
3085 if ((value & 0xff) != 0) {
3086 err = USB_ERR_IOERROR;
3087 goto done;
3088 }
3089 len = sizeof(xhci_devd);
3090 ptr = (const void *)&xhci_devd;
3091 break;
3092
3093 case UDESC_BOS:
3094 if ((value & 0xff) != 0) {
3095 err = USB_ERR_IOERROR;
3096 goto done;
3097 }
3098 len = sizeof(xhci_bosd);
3099 ptr = (const void *)&xhci_bosd;
3100 break;
3101
3102 case UDESC_CONFIG:
3103 if ((value & 0xff) != 0) {
3104 err = USB_ERR_IOERROR;
3105 goto done;
3106 }
3107 len = sizeof(xhci_confd);
3108 ptr = (const void *)&xhci_confd;
3109 break;
3110
3111 case UDESC_STRING:
3112 switch (value & 0xff) {
3113 case 0: /* Language table */
3114 str_ptr = "\001";
3115 break;
3116
3117 case 1: /* Vendor */
3118 str_ptr = sc->sc_vendor;
3119 break;
3120
3121 case 2: /* Product */
3122 str_ptr = "XHCI root HUB";
3123 break;
3124
3125 default:
3126 str_ptr = "";
3127 break;
3128 }
3129
3130 len = usb_make_str_desc(
3131 sc->sc_hub_desc.temp,
3132 sizeof(sc->sc_hub_desc.temp),
3133 str_ptr);
3134 break;
3135
3136 default:
3137 err = USB_ERR_IOERROR;
3138 goto done;
3139 }
3140 break;
3141 case C(UR_GET_INTERFACE, UT_READ_INTERFACE):
3142 len = 1;
3143 sc->sc_hub_desc.temp[0] = 0;
3144 break;
3145 case C(UR_GET_STATUS, UT_READ_DEVICE):
3146 len = 2;
3147 USETW(sc->sc_hub_desc.stat.wStatus, UDS_SELF_POWERED);
3148 break;
3149 case C(UR_GET_STATUS, UT_READ_INTERFACE):
3150 case C(UR_GET_STATUS, UT_READ_ENDPOINT):
3151 len = 2;
3152 USETW(sc->sc_hub_desc.stat.wStatus, 0);
3153 break;
3154 case C(UR_SET_ADDRESS, UT_WRITE_DEVICE):
3155 if (value >= XHCI_MAX_DEVICES) {
3156 err = USB_ERR_IOERROR;
3157 goto done;
3158 }
3159 break;
3160 case C(UR_SET_CONFIG, UT_WRITE_DEVICE):
3161 if (value != 0 && value != 1) {
3162 err = USB_ERR_IOERROR;
3163 goto done;
3164 }
3165 sc->sc_conf = value;
3166 break;
3167 case C(UR_SET_DESCRIPTOR, UT_WRITE_DEVICE):
3168 break;
3169 case C(UR_SET_FEATURE, UT_WRITE_DEVICE):
3170 case C(UR_SET_FEATURE, UT_WRITE_INTERFACE):
3171 case C(UR_SET_FEATURE, UT_WRITE_ENDPOINT):
3172 err = USB_ERR_IOERROR;
3173 goto done;
3174 case C(UR_SET_INTERFACE, UT_WRITE_INTERFACE):
3175 break;
3176 case C(UR_SYNCH_FRAME, UT_WRITE_ENDPOINT):
3177 break;
3178 /* Hub requests */
3179 case C(UR_CLEAR_FEATURE, UT_WRITE_CLASS_DEVICE):
3180 break;
3181 case C(UR_CLEAR_FEATURE, UT_WRITE_CLASS_OTHER):
3182 DPRINTFN(9, "UR_CLEAR_PORT_FEATURE\n");
3183
3184 if ((index < 1) ||
3185 (index > sc->sc_noport)) {
3186 err = USB_ERR_IOERROR;
3187 goto done;
3188 }
3189 port = XHCI_PORTSC(index);
3190
3191 v = XREAD4(sc, oper, port);
3192 i = XHCI_PS_PLS_GET(v);
3193 v &= ~XHCI_PS_CLEAR;
3194
3195 switch (value) {
3196 case UHF_C_BH_PORT_RESET:
3197 XWRITE4(sc, oper, port, v | XHCI_PS_WRC);
3198 break;
3199 case UHF_C_PORT_CONFIG_ERROR:
3200 XWRITE4(sc, oper, port, v | XHCI_PS_CEC);
3201 break;
3202 case UHF_C_PORT_SUSPEND:
3203 case UHF_C_PORT_LINK_STATE:
3204 XWRITE4(sc, oper, port, v | XHCI_PS_PLC);
3205 break;
3206 case UHF_C_PORT_CONNECTION:
3207 XWRITE4(sc, oper, port, v | XHCI_PS_CSC);
3208 break;
3209 case UHF_C_PORT_ENABLE:
3210 XWRITE4(sc, oper, port, v | XHCI_PS_PEC);
3211 break;
3212 case UHF_C_PORT_OVER_CURRENT:
3213 XWRITE4(sc, oper, port, v | XHCI_PS_OCC);
3214 break;
3215 case UHF_C_PORT_RESET:
3216 XWRITE4(sc, oper, port, v | XHCI_PS_PRC);
3217 break;
3218 case UHF_PORT_ENABLE:
3219 XWRITE4(sc, oper, port, v | XHCI_PS_PED);
3220 break;
3221 case UHF_PORT_POWER:
3222 XWRITE4(sc, oper, port, v & ~XHCI_PS_PP);
3223 break;
3224 case UHF_PORT_INDICATOR:
3225 XWRITE4(sc, oper, port, v & ~XHCI_PS_PIC_SET(3));
3226 break;
3227 case UHF_PORT_SUSPEND:
3228
3229 /* U3 -> U15 */
3230 if (i == 3) {
3231 XWRITE4(sc, oper, port, v |
3232 XHCI_PS_PLS_SET(0xF) | XHCI_PS_LWS);
3233 }
3234
3235 /* wait 20ms for resume sequence to complete */
3236 usb_pause_mtx(&sc->sc_bus.bus_mtx, hz / 50);
3237
3238 /* U0 */
3239 XWRITE4(sc, oper, port, v |
3240 XHCI_PS_PLS_SET(0) | XHCI_PS_LWS);
3241 break;
3242 default:
3243 err = USB_ERR_IOERROR;
3244 goto done;
3245 }
3246 break;
3247
3248 case C(UR_GET_DESCRIPTOR, UT_READ_CLASS_DEVICE):
3249 if ((value & 0xff) != 0) {
3250 err = USB_ERR_IOERROR;
3251 goto done;
3252 }
3253
3254 v = XREAD4(sc, capa, XHCI_HCSPARAMS0);
3255
3256 sc->sc_hub_desc.hubd = xhci_hubd;
3257
3258 sc->sc_hub_desc.hubd.bNbrPorts = sc->sc_noport;
3259
3260 if (XHCI_HCS0_PPC(v))
3261 i = UHD_PWR_INDIVIDUAL;
3262 else
3263 i = UHD_PWR_GANGED;
3264
3265 if (XHCI_HCS0_PIND(v))
3266 i |= UHD_PORT_IND;
3267
3268 i |= UHD_OC_INDIVIDUAL;
3269
3270 USETW(sc->sc_hub_desc.hubd.wHubCharacteristics, i);
3271
3272 /* see XHCI section 5.4.9: */
3273 sc->sc_hub_desc.hubd.bPwrOn2PwrGood = 10;
3274
3275 for (j = 1; j <= sc->sc_noport; j++) {
3276
3277 v = XREAD4(sc, oper, XHCI_PORTSC(j));
3278 if (v & XHCI_PS_DR) {
3279 sc->sc_hub_desc.hubd.
3280 DeviceRemovable[j / 8] |= 1U << (j % 8);
3281 }
3282 }
3283 len = sc->sc_hub_desc.hubd.bLength;
3284 break;
3285
3286 case C(UR_GET_STATUS, UT_READ_CLASS_DEVICE):
3287 len = 16;
3288 memset(sc->sc_hub_desc.temp, 0, 16);
3289 break;
3290
3291 case C(UR_GET_STATUS, UT_READ_CLASS_OTHER):
3292 DPRINTFN(9, "UR_GET_STATUS i=%d\n", index);
3293
3294 if ((index < 1) ||
3295 (index > sc->sc_noport)) {
3296 err = USB_ERR_IOERROR;
3297 goto done;
3298 }
3299
3300 v = XREAD4(sc, oper, XHCI_PORTSC(index));
3301
3302 DPRINTFN(9, "port status=0x%08x\n", v);
3303
3304 i = UPS_PORT_LINK_STATE_SET(XHCI_PS_PLS_GET(v));
3305
3306 switch (XHCI_PS_SPEED_GET(v)) {
3307 case 3:
3308 i |= UPS_HIGH_SPEED;
3309 break;
3310 case 2:
3311 i |= UPS_LOW_SPEED;
3312 break;
3313 case 1:
3314 /* FULL speed */
3315 break;
3316 default:
3317 i |= UPS_OTHER_SPEED;
3318 break;
3319 }
3320
3321 if (v & XHCI_PS_CCS)
3322 i |= UPS_CURRENT_CONNECT_STATUS;
3323 if (v & XHCI_PS_PED)
3324 i |= UPS_PORT_ENABLED;
3325 if (v & XHCI_PS_OCA)
3326 i |= UPS_OVERCURRENT_INDICATOR;
3327 if (v & XHCI_PS_PR)
3328 i |= UPS_RESET;
3329 if (v & XHCI_PS_PP) {
3330 /*
3331 * The USB 3.0 RH is using the
3332 * USB 2.0's power bit
3333 */
3334 i |= UPS_PORT_POWER;
3335 }
3336 USETW(sc->sc_hub_desc.ps.wPortStatus, i);
3337
3338 i = 0;
3339 if (v & XHCI_PS_CSC)
3340 i |= UPS_C_CONNECT_STATUS;
3341 if (v & XHCI_PS_PEC)
3342 i |= UPS_C_PORT_ENABLED;
3343 if (v & XHCI_PS_OCC)
3344 i |= UPS_C_OVERCURRENT_INDICATOR;
3345 if (v & XHCI_PS_WRC)
3346 i |= UPS_C_BH_PORT_RESET;
3347 if (v & XHCI_PS_PRC)
3348 i |= UPS_C_PORT_RESET;
3349 if (v & XHCI_PS_PLC)
3350 i |= UPS_C_PORT_LINK_STATE;
3351 if (v & XHCI_PS_CEC)
3352 i |= UPS_C_PORT_CONFIG_ERROR;
3353
3354 USETW(sc->sc_hub_desc.ps.wPortChange, i);
3355 len = sizeof(sc->sc_hub_desc.ps);
3356 break;
3357
3358 case C(UR_SET_DESCRIPTOR, UT_WRITE_CLASS_DEVICE):
3359 err = USB_ERR_IOERROR;
3360 goto done;
3361
3362 case C(UR_SET_FEATURE, UT_WRITE_CLASS_DEVICE):
3363 break;
3364
3365 case C(UR_SET_FEATURE, UT_WRITE_CLASS_OTHER):
3366
3367 i = index >> 8;
3368 index &= 0x00FF;
3369
3370 if ((index < 1) ||
3371 (index > sc->sc_noport)) {
3372 err = USB_ERR_IOERROR;
3373 goto done;
3374 }
3375
3376 port = XHCI_PORTSC(index);
3377 v = XREAD4(sc, oper, port) & ~XHCI_PS_CLEAR;
3378
3379 switch (value) {
3380 case UHF_PORT_U1_TIMEOUT:
3381 if (XHCI_PS_SPEED_GET(v) != 4) {
3382 err = USB_ERR_IOERROR;
3383 goto done;
3384 }
3385 port = XHCI_PORTPMSC(index);
3386 v = XREAD4(sc, oper, port);
3387 v &= ~XHCI_PM3_U1TO_SET(0xFF);
3388 v |= XHCI_PM3_U1TO_SET(i);
3389 XWRITE4(sc, oper, port, v);
3390 break;
3391 case UHF_PORT_U2_TIMEOUT:
3392 if (XHCI_PS_SPEED_GET(v) != 4) {
3393 err = USB_ERR_IOERROR;
3394 goto done;
3395 }
3396 port = XHCI_PORTPMSC(index);
3397 v = XREAD4(sc, oper, port);
3398 v &= ~XHCI_PM3_U2TO_SET(0xFF);
3399 v |= XHCI_PM3_U2TO_SET(i);
3400 XWRITE4(sc, oper, port, v);
3401 break;
3402 case UHF_BH_PORT_RESET:
3403 XWRITE4(sc, oper, port, v | XHCI_PS_WPR);
3404 break;
3405 case UHF_PORT_LINK_STATE:
3406 XWRITE4(sc, oper, port, v |
3407 XHCI_PS_PLS_SET(i) | XHCI_PS_LWS);
3408 /* 4ms settle time */
3409 usb_pause_mtx(&sc->sc_bus.bus_mtx, hz / 250);
3410 break;
3411 case UHF_PORT_ENABLE:
3412 DPRINTFN(3, "set port enable %d\n", index);
3413 break;
3414 case UHF_PORT_SUSPEND:
3415 DPRINTFN(6, "suspend port %u (LPM=%u)\n", index, i);
3416 j = XHCI_PS_SPEED_GET(v);
3417 if ((j < 1) || (j > 3)) {
3418 /* non-supported speed */
3419 err = USB_ERR_IOERROR;
3420 goto done;
3421 }
3422 XWRITE4(sc, oper, port, v |
3423 XHCI_PS_PLS_SET(i ? 2 /* LPM */ : 3) | XHCI_PS_LWS);
3424 break;
3425 case UHF_PORT_RESET:
3426 DPRINTFN(6, "reset port %d\n", index);
3427 XWRITE4(sc, oper, port, v | XHCI_PS_PR);
3428 break;
3429 case UHF_PORT_POWER:
3430 DPRINTFN(3, "set port power %d\n", index);
3431 XWRITE4(sc, oper, port, v | XHCI_PS_PP);
3432 break;
3433 case UHF_PORT_TEST:
3434 DPRINTFN(3, "set port test %d\n", index);
3435 break;
3436 case UHF_PORT_INDICATOR:
3437 DPRINTFN(3, "set port indicator %d\n", index);
3438
3439 v &= ~XHCI_PS_PIC_SET(3);
3440 v |= XHCI_PS_PIC_SET(1);
3441
3442 XWRITE4(sc, oper, port, v);
3443 break;
3444 default:
3445 err = USB_ERR_IOERROR;
3446 goto done;
3447 }
3448 break;
3449
3450 case C(UR_CLEAR_TT_BUFFER, UT_WRITE_CLASS_OTHER):
3451 case C(UR_RESET_TT, UT_WRITE_CLASS_OTHER):
3452 case C(UR_GET_TT_STATE, UT_READ_CLASS_OTHER):
3453 case C(UR_STOP_TT, UT_WRITE_CLASS_OTHER):
3454 break;
3455 default:
3456 err = USB_ERR_IOERROR;
3457 goto done;
3458 }
3459done:
3460 *plength = len;
3461 *pptr = ptr;
3462 return (err);
3463}
3464
3465static void
3466xhci_xfer_setup(struct usb_setup_params *parm)
3467{
3468 struct usb_page_search page_info;
3469 struct usb_page_cache *pc;
3470 struct xhci_softc *sc;
3471 struct usb_xfer *xfer;
3472 void *last_obj;
3473 uint32_t ntd;
3474 uint32_t n;
3475
3476 sc = XHCI_BUS2SC(parm->udev->bus);
3477 xfer = parm->curr_xfer;
3478
3479 /*
3480 * The proof for the "ntd" formula is illustrated like this:
3481 *
3482 * +------------------------------------+
3483 * | |
3484 * | |remainder -> |
3485 * | +-----+---+ |
3486 * | | xxx | x | frm 0 |
3487 * | +-----+---++ |
3488 * | | xxx | xx | frm 1 |
3489 * | +-----+----+ |
3490 * | ... |
3491 * +------------------------------------+
3492 *
3493 * "xxx" means a completely full USB transfer descriptor
3494 *
3495 * "x" and "xx" means a short USB packet
3496 *
3497 * For the remainder of an USB transfer modulo
3498 * "max_data_length" we need two USB transfer descriptors.
3499 * One to transfer the remaining data and one to finalise with
3500 * a zero length packet in case the "force_short_xfer" flag is
3501 * set. We only need two USB transfer descriptors in the case
3502 * where the transfer length of the first one is a factor of
3503 * "max_frame_size". The rest of the needed USB transfer
3504 * descriptors is given by the buffer size divided by the
3505 * maximum data payload.
3506 */
3507 parm->hc_max_packet_size = 0x400;
3508 parm->hc_max_packet_count = 16 * 3;
3509 parm->hc_max_frame_size = XHCI_TD_PAYLOAD_MAX;
3510
3511 xfer->flags_int.bdma_enable = 1;
3512
3513 usbd_transfer_setup_sub(parm);
3514
3515 if (xfer->flags_int.isochronous_xfr) {
3516 ntd = ((1 * xfer->nframes)
3517 + (xfer->max_data_length / xfer->max_hc_frame_size));
3518 } else if (xfer->flags_int.control_xfr) {
3519 ntd = ((2 * xfer->nframes) + 1 /* STATUS */
3520 + (xfer->max_data_length / xfer->max_hc_frame_size));
3521 } else {
3522 ntd = ((2 * xfer->nframes)
3523 + (xfer->max_data_length / xfer->max_hc_frame_size));
3524 }
3525
3526alloc_dma_set:
3527
3528 if (parm->err)
3529 return;
3530
3531 /*
3532 * Allocate queue heads and transfer descriptors
3533 */
3534 last_obj = NULL;
3535
3536 if (usbd_transfer_setup_sub_malloc(
3537 parm, &pc, sizeof(struct xhci_td),
3538 XHCI_TD_ALIGN, ntd)) {
3539 parm->err = USB_ERR_NOMEM;
3540 return;
3541 }
3542 if (parm->buf) {
3543 for (n = 0; n != ntd; n++) {
3544 struct xhci_td *td;
3545
3546 usbd_get_page(pc + n, 0, &page_info);
3547
3548 td = page_info.buffer;
3549
3550 /* init TD */
3551 td->td_self = page_info.physaddr;
3552 td->obj_next = last_obj;
3553 td->page_cache = pc + n;
3554
3555 last_obj = td;
3556
3557 usb_pc_cpu_flush(pc + n);
3558 }
3559 }
3560 xfer->td_start[xfer->flags_int.curr_dma_set] = last_obj;
3561
3562 if (!xfer->flags_int.curr_dma_set) {
3563 xfer->flags_int.curr_dma_set = 1;
3564 goto alloc_dma_set;
3565 }
3566}
3567
3568static usb_error_t
3569xhci_configure_reset_endpoint(struct usb_xfer *xfer)
3570{
3571 struct xhci_softc *sc = XHCI_BUS2SC(xfer->xroot->bus);
3572 struct usb_page_search buf_inp;
3573 struct usb_device *udev;
3574 struct xhci_endpoint_ext *pepext;
3575 struct usb_endpoint_descriptor *edesc;
3576 struct usb_page_cache *pcinp;
3577 usb_error_t err;
3578 usb_stream_t stream_id;
3579 uint8_t index;
3580 uint8_t epno;
3581
3582 pepext = xhci_get_endpoint_ext(xfer->xroot->udev,
3583 xfer->endpoint->edesc);
3584
3585 udev = xfer->xroot->udev;
3586 index = udev->controller_slot_id;
3587
3588 pcinp = &sc->sc_hw.devs[index].input_pc;
3589
3590 usbd_get_page(pcinp, 0, &buf_inp);
3591
3592 edesc = xfer->endpoint->edesc;
3593
3594 epno = edesc->bEndpointAddress;
3595 stream_id = xfer->stream_id;
3596
3597 if ((edesc->bmAttributes & UE_XFERTYPE) == UE_CONTROL)
3598 epno |= UE_DIR_IN;
3599
3600 epno = XHCI_EPNO2EPID(epno);
3601
3602 if (epno == 0)
3603 return (USB_ERR_NO_PIPE); /* invalid */
3604
3605 XHCI_CMD_LOCK(sc);
3606
3607 /* configure endpoint */
3608
3609 err = xhci_configure_endpoint_by_xfer(xfer);
3610
3611 if (err != 0) {
3612 XHCI_CMD_UNLOCK(sc);
3613 return (err);
3614 }
3615
3616 /*
3617 * Get the endpoint into the stopped state according to the
3618 * endpoint context state diagram in the XHCI specification:
3619 */
3620
3621 err = xhci_cmd_stop_ep(sc, 0, epno, index);
3622
3623 if (err != 0)
3624 DPRINTF("Could not stop endpoint %u\n", epno);
3625
3626 err = xhci_cmd_reset_ep(sc, 0, epno, index);
3627
3628 if (err != 0)
3629 DPRINTF("Could not reset endpoint %u\n", epno);
3630
3631 err = xhci_cmd_set_tr_dequeue_ptr(sc,
3632 (pepext->physaddr + (stream_id * sizeof(struct xhci_trb) *
3633 XHCI_MAX_TRANSFERS)) | XHCI_EPCTX_2_DCS_SET(1),
3634 stream_id, epno, index);
3635
3636 if (err != 0)
3637 DPRINTF("Could not set dequeue ptr for endpoint %u\n", epno);
3638
3639 /*
3640 * Get the endpoint into the running state according to the
3641 * endpoint context state diagram in the XHCI specification:
3642 */
3643
3644 xhci_configure_mask(udev, (1U << epno) | 1U, 0);
3645
3646 err = xhci_cmd_evaluate_ctx(sc, buf_inp.physaddr, index);
3647
3648 if (err != 0)
3649 DPRINTF("Could not configure endpoint %u\n", epno);
3650
3651 err = xhci_cmd_configure_ep(sc, buf_inp.physaddr, 0, index);
3652
3653 if (err != 0)
3654 DPRINTF("Could not configure endpoint %u\n", epno);
3655
3656 XHCI_CMD_UNLOCK(sc);
3657
3658 return (0);
3659}
3660
3661static void
3662xhci_xfer_unsetup(struct usb_xfer *xfer)
3663{
3664 return;
3665}
3666
3667static void
3668xhci_start_dma_delay(struct usb_xfer *xfer)
3669{
3670 struct xhci_softc *sc = XHCI_BUS2SC(xfer->xroot->bus);
3671
3672 /* put transfer on interrupt queue (again) */
3673 usbd_transfer_enqueue(&sc->sc_bus.intr_q, xfer);
3674
3675 (void)usb_proc_msignal(USB_BUS_CONTROL_XFER_PROC(&sc->sc_bus),
3676 &sc->sc_config_msg[0], &sc->sc_config_msg[1]);
3677}
3678
3679static void
3680xhci_configure_msg(struct usb_proc_msg *pm)
3681{
3682 struct xhci_softc *sc;
3683 struct xhci_endpoint_ext *pepext;
3684 struct usb_xfer *xfer;
3685
3686 sc = XHCI_BUS2SC(((struct usb_bus_msg *)pm)->bus);
3687
3688restart:
3689 TAILQ_FOREACH(xfer, &sc->sc_bus.intr_q.head, wait_entry) {
3690
3691 pepext = xhci_get_endpoint_ext(xfer->xroot->udev,
3692 xfer->endpoint->edesc);
3693
3694 if ((pepext->trb_halted != 0) ||
3695 (pepext->trb_running == 0)) {
3696
3697 uint16_t i;
3698
3699 /* clear halted and running */
3700 pepext->trb_halted = 0;
3701 pepext->trb_running = 0;
3702
3703 /* nuke remaining buffered transfers */
3704
3705 for (i = 0; i != (XHCI_MAX_TRANSFERS *
3706 XHCI_MAX_STREAMS); i++) {
3707 /*
3708 * NOTE: We need to use the timeout
3709 * error code here else existing
3710 * isochronous clients can get
3711 * confused:
3712 */
3713 if (pepext->xfer[i] != NULL) {
3714 xhci_device_done(pepext->xfer[i],
3715 USB_ERR_TIMEOUT);
3716 }
3717 }
3718
3719 /*
3720 * NOTE: The USB transfer cannot vanish in
3721 * this state!
3722 */
3723
3724 USB_BUS_UNLOCK(&sc->sc_bus);
3725
3726 xhci_configure_reset_endpoint(xfer);
3727
3728 USB_BUS_LOCK(&sc->sc_bus);
3729
3730 /* check if halted is still cleared */
3731 if (pepext->trb_halted == 0) {
3732 pepext->trb_running = 1;
3733 memset(pepext->trb_index, 0,
3734 sizeof(pepext->trb_index));
3735 }
3736 goto restart;
3737 }
3738
3739 if (xfer->flags_int.did_dma_delay) {
3740
3741 /* remove transfer from interrupt queue (again) */
3742 usbd_transfer_dequeue(xfer);
3743
3744 /* we are finally done */
3745 usb_dma_delay_done_cb(xfer);
3746
3747 /* queue changed - restart */
3748 goto restart;
3749 }
3750 }
3751
3752 TAILQ_FOREACH(xfer, &sc->sc_bus.intr_q.head, wait_entry) {
3753
3754 /* try to insert xfer on HW queue */
3755 xhci_transfer_insert(xfer);
3756
3757 /* try to multi buffer */
3758 xhci_device_generic_multi_enter(xfer->endpoint,
3759 xfer->stream_id, NULL);
3760 }
3761}
3762
3763static void
3764xhci_ep_init(struct usb_device *udev, struct usb_endpoint_descriptor *edesc,
3765 struct usb_endpoint *ep)
3766{
3767 struct xhci_endpoint_ext *pepext;
3768
3769 DPRINTFN(2, "endpoint=%p, addr=%d, endpt=%d, mode=%d\n",
3770 ep, udev->address, edesc->bEndpointAddress, udev->flags.usb_mode);
3771
3772 if (udev->parent_hub == NULL) {
3773 /* root HUB has special endpoint handling */
3774 return;
3775 }
3776
3777 ep->methods = &xhci_device_generic_methods;
3778
3779 pepext = xhci_get_endpoint_ext(udev, edesc);
3780
3781 USB_BUS_LOCK(udev->bus);
3782 pepext->trb_halted = 1;
3783 pepext->trb_running = 0;
3784 USB_BUS_UNLOCK(udev->bus);
3785}
3786
3787static void
3788xhci_ep_uninit(struct usb_device *udev, struct usb_endpoint *ep)
3789{
3790
3791}
3792
3793static void
3794xhci_ep_clear_stall(struct usb_device *udev, struct usb_endpoint *ep)
3795{
3796 struct xhci_endpoint_ext *pepext;
3797
3798 DPRINTF("\n");
3799
3800 if (udev->flags.usb_mode != USB_MODE_HOST) {
3801 /* not supported */
3802 return;
3803 }
3804 if (udev->parent_hub == NULL) {
3805 /* root HUB has special endpoint handling */
3806 return;
3807 }
3808
3809 pepext = xhci_get_endpoint_ext(udev, ep->edesc);
3810
3811 USB_BUS_LOCK(udev->bus);
3812 pepext->trb_halted = 1;
3813 pepext->trb_running = 0;
3814 USB_BUS_UNLOCK(udev->bus);
3815}
3816
3817static usb_error_t
3818xhci_device_init(struct usb_device *udev)
3819{
3820 struct xhci_softc *sc = XHCI_BUS2SC(udev->bus);
3821 usb_error_t err;
3822 uint8_t temp;
3823
3824 /* no init for root HUB */
3825 if (udev->parent_hub == NULL)
3826 return (0);
3827
3828 XHCI_CMD_LOCK(sc);
3829
3830 /* set invalid default */
3831
3832 udev->controller_slot_id = sc->sc_noslot + 1;
3833
3834 /* try to get a new slot ID from the XHCI */
3835
3836 err = xhci_cmd_enable_slot(sc, &temp);
3837
3838 if (err) {
3839 XHCI_CMD_UNLOCK(sc);
3840 return (err);
3841 }
3842
3843 if (temp > sc->sc_noslot) {
3844 XHCI_CMD_UNLOCK(sc);
3845 return (USB_ERR_BAD_ADDRESS);
3846 }
3847
3848 if (sc->sc_hw.devs[temp].state != XHCI_ST_DISABLED) {
3849 DPRINTF("slot %u already allocated.\n", temp);
3850 XHCI_CMD_UNLOCK(sc);
3851 return (USB_ERR_BAD_ADDRESS);
3852 }
3853
3854 /* store slot ID for later reference */
3855
3856 udev->controller_slot_id = temp;
3857
3858 /* reset data structure */
3859
3860 memset(&sc->sc_hw.devs[temp], 0, sizeof(sc->sc_hw.devs[0]));
3861
3862 /* set mark slot allocated */
3863
3864 sc->sc_hw.devs[temp].state = XHCI_ST_ENABLED;
3865
3866 err = xhci_alloc_device_ext(udev);
3867
3868 XHCI_CMD_UNLOCK(sc);
3869
3870 /* get device into default state */
3871
3872 if (err == 0)
3873 err = xhci_set_address(udev, NULL, 0);
3874
3875 return (err);
3876}
3877
3878static void
3879xhci_device_uninit(struct usb_device *udev)
3880{
3881 struct xhci_softc *sc = XHCI_BUS2SC(udev->bus);
3882 uint8_t index;
3883
3884 /* no init for root HUB */
3885 if (udev->parent_hub == NULL)
3886 return;
3887
3888 XHCI_CMD_LOCK(sc);
3889
3890 index = udev->controller_slot_id;
3891
3892 if (index <= sc->sc_noslot) {
3893 xhci_cmd_disable_slot(sc, index);
3894 sc->sc_hw.devs[index].state = XHCI_ST_DISABLED;
3895
3896 /* free device extension */
3897 xhci_free_device_ext(udev);
3898 }
3899
3900 XHCI_CMD_UNLOCK(sc);
3901}
3902
3903static void
3904xhci_get_dma_delay(struct usb_device *udev, uint32_t *pus)
3905{
3906 /*
3907 * Wait until the hardware has finished any possible use of
3908 * the transfer descriptor(s)
3909 */
3910 *pus = 2048; /* microseconds */
3911}
3912
3913static void
3914xhci_device_resume(struct usb_device *udev)
3915{
3916 struct xhci_softc *sc = XHCI_BUS2SC(udev->bus);
3917 uint8_t index;
3918 uint8_t n;
3919 uint8_t p;
3920
3921 DPRINTF("\n");
3922
3923 /* check for root HUB */
3924 if (udev->parent_hub == NULL)
3925 return;
3926
3927 index = udev->controller_slot_id;
3928
3929 XHCI_CMD_LOCK(sc);
3930
3931 /* blindly resume all endpoints */
3932
3933 USB_BUS_LOCK(udev->bus);
3934
3935 for (n = 1; n != XHCI_MAX_ENDPOINTS; n++) {
3936 for (p = 0; p != XHCI_MAX_STREAMS; p++) {
3937 XWRITE4(sc, door, XHCI_DOORBELL(index),
3938 n | XHCI_DB_SID_SET(p));
3939 }
3940 }
3941
3942 USB_BUS_UNLOCK(udev->bus);
3943
3944 XHCI_CMD_UNLOCK(sc);
3945}
3946
3947static void
3948xhci_device_suspend(struct usb_device *udev)
3949{
3950 struct xhci_softc *sc = XHCI_BUS2SC(udev->bus);
3951 uint8_t index;
3952 uint8_t n;
3953 usb_error_t err;
3954
3955 DPRINTF("\n");
3956
3957 /* check for root HUB */
3958 if (udev->parent_hub == NULL)
3959 return;
3960
3961 index = udev->controller_slot_id;
3962
3963 XHCI_CMD_LOCK(sc);
3964
3965 /* blindly suspend all endpoints */
3966
3967 for (n = 1; n != XHCI_MAX_ENDPOINTS; n++) {
3968 err = xhci_cmd_stop_ep(sc, 1, n, index);
3969 if (err != 0) {
3970 DPRINTF("Failed to suspend endpoint "
3971 "%u on slot %u (ignored).\n", n, index);
3972 }
3973 }
3974
3975 XHCI_CMD_UNLOCK(sc);
3976}
3977
3978static void
3979xhci_set_hw_power(struct usb_bus *bus)
3980{
3981 DPRINTF("\n");
3982}
3983
3984static void
3985xhci_device_state_change(struct usb_device *udev)
3986{
3987 struct xhci_softc *sc = XHCI_BUS2SC(udev->bus);
3988 struct usb_page_search buf_inp;
3989 usb_error_t err;
3990 uint8_t index;
3991
3992 /* check for root HUB */
3993 if (udev->parent_hub == NULL)
3994 return;
3995
3996 index = udev->controller_slot_id;
3997
3998 DPRINTF("\n");
3999
4000 if (usb_get_device_state(udev) == USB_STATE_CONFIGURED) {
4001 err = uhub_query_info(udev, &sc->sc_hw.devs[index].nports,
4002 &sc->sc_hw.devs[index].tt);
4003 if (err != 0)
4004 sc->sc_hw.devs[index].nports = 0;
4005 }
4006
4007 XHCI_CMD_LOCK(sc);
4008
4009 switch (usb_get_device_state(udev)) {
4010 case USB_STATE_POWERED:
4011 if (sc->sc_hw.devs[index].state == XHCI_ST_DEFAULT)
4012 break;
4013
4014 /* set default state */
4015 sc->sc_hw.devs[index].state = XHCI_ST_DEFAULT;
4016
4017 /* reset number of contexts */
4018 sc->sc_hw.devs[index].context_num = 0;
4019
4020 err = xhci_cmd_reset_dev(sc, index);
4021
4022 if (err != 0) {
4023 DPRINTF("Device reset failed "
4024 "for slot %u.\n", index);
4025 }
4026 break;
4027
4028 case USB_STATE_ADDRESSED:
4029 if (sc->sc_hw.devs[index].state == XHCI_ST_ADDRESSED)
4030 break;
4031
4032 sc->sc_hw.devs[index].state = XHCI_ST_ADDRESSED;
4033
4034 err = xhci_cmd_configure_ep(sc, 0, 1, index);
4035
4036 if (err) {
4037 DPRINTF("Failed to deconfigure "
4038 "slot %u.\n", index);
4039 }
4040 break;
4041
4042 case USB_STATE_CONFIGURED:
4043 if (sc->sc_hw.devs[index].state == XHCI_ST_CONFIGURED)
4044 break;
4045
4046 /* set configured state */
4047 sc->sc_hw.devs[index].state = XHCI_ST_CONFIGURED;
4048
4049 /* reset number of contexts */
4050 sc->sc_hw.devs[index].context_num = 0;
4051
4052 usbd_get_page(&sc->sc_hw.devs[index].input_pc, 0, &buf_inp);
4053
4054 xhci_configure_mask(udev, 3, 0);
4055
4056 err = xhci_configure_device(udev);
4057 if (err != 0) {
4058 DPRINTF("Could not configure device "
4059 "at slot %u.\n", index);
4060 }
4061
4062 err = xhci_cmd_evaluate_ctx(sc, buf_inp.physaddr, index);
4063 if (err != 0) {
4064 DPRINTF("Could not evaluate device "
4065 "context at slot %u.\n", index);
4066 }
4067 break;
4068
4069 default:
4070 break;
4071 }
4072 XHCI_CMD_UNLOCK(sc);
4073}
4074
4075static usb_error_t
4076xhci_set_endpoint_mode(struct usb_device *udev, struct usb_endpoint *ep,
4077 uint8_t ep_mode)
4078{
4079 switch (ep_mode) {
4080 case USB_EP_MODE_DEFAULT:
4081 return (0);
4082 case USB_EP_MODE_STREAMS:
4083 if ((ep->edesc->bmAttributes & UE_XFERTYPE) != UE_BULK ||
4084 udev->speed != USB_SPEED_SUPER)
4085 return (USB_ERR_INVAL);
4086 return (0);
4087 default:
4088 return (USB_ERR_INVAL);
4089 }
4090}
4091
4092struct usb_bus_methods xhci_bus_methods = {
4093 .endpoint_init = xhci_ep_init,
4094 .endpoint_uninit = xhci_ep_uninit,
4095 .xfer_setup = xhci_xfer_setup,
4096 .xfer_unsetup = xhci_xfer_unsetup,
4097 .get_dma_delay = xhci_get_dma_delay,
4098 .device_init = xhci_device_init,
4099 .device_uninit = xhci_device_uninit,
4100 .device_resume = xhci_device_resume,
4101 .device_suspend = xhci_device_suspend,
4102 .set_hw_power = xhci_set_hw_power,
4103 .roothub_exec = xhci_roothub_exec,
4104 .xfer_poll = xhci_do_poll,
4105 .start_dma_delay = xhci_start_dma_delay,
4106 .set_address = xhci_set_address,
4107 .clear_stall = xhci_ep_clear_stall,
4108 .device_state_change = xhci_device_state_change,
4109 .set_hw_power_sleep = xhci_set_hw_power_sleep,
4110 .set_endpoint_mode = xhci_set_endpoint_mode,
4111};
| 2753
2754#ifdef USB_DEBUG
2755 xhci_dump_trb(&td_last->td_trb[td_last->ntrb]);
2756#endif
2757 usb_pc_cpu_flush(td_last->page_cache);
2758
2759 /* write ahead chain end marker */
2760
2761 pepext->trb[inext].qwTrb0 = 0;
2762 pepext->trb[inext].dwTrb2 = 0;
2763 pepext->trb[inext].dwTrb3 = 0;
2764
2765 /* update next pointer of link TRB */
2766
2767 pepext->trb[i].qwTrb0 = htole64((uint64_t)td_first->td_self);
2768 pepext->trb[i].dwTrb2 = htole32(XHCI_TRB_2_IRQ_SET(0));
2769
2770#ifdef USB_DEBUG
2771 xhci_dump_trb(&pepext->trb[i]);
2772#endif
2773 usb_pc_cpu_flush(pepext->page_cache);
2774
2775 /* toggle cycle bit which activates the transfer chain */
2776
2777 pepext->trb[i].dwTrb3 = htole32(XHCI_TRB_3_CYCLE_BIT |
2778 XHCI_TRB_3_TYPE_SET(XHCI_TRB_TYPE_LINK));
2779
2780 usb_pc_cpu_flush(pepext->page_cache);
2781
2782 DPRINTF("qh_pos = %u\n", i);
2783
2784 pepext->xfer[i] = xfer;
2785
2786 xfer->qh_pos = i;
2787
2788 xfer->flags_int.bandwidth_reclaimed = 1;
2789
2790 xhci_endpoint_doorbell(xfer);
2791
2792 return (0);
2793}
2794
2795static void
2796xhci_root_intr(struct xhci_softc *sc)
2797{
2798 uint16_t i;
2799
2800 USB_BUS_LOCK_ASSERT(&sc->sc_bus, MA_OWNED);
2801
2802 /* clear any old interrupt data */
2803 memset(sc->sc_hub_idata, 0, sizeof(sc->sc_hub_idata));
2804
2805 for (i = 1; i <= sc->sc_noport; i++) {
2806 /* pick out CHANGE bits from the status register */
2807 if (XREAD4(sc, oper, XHCI_PORTSC(i)) & (
2808 XHCI_PS_CSC | XHCI_PS_PEC |
2809 XHCI_PS_OCC | XHCI_PS_WRC |
2810 XHCI_PS_PRC | XHCI_PS_PLC |
2811 XHCI_PS_CEC)) {
2812 sc->sc_hub_idata[i / 8] |= 1 << (i % 8);
2813 DPRINTF("port %d changed\n", i);
2814 }
2815 }
2816 uhub_root_intr(&sc->sc_bus, sc->sc_hub_idata,
2817 sizeof(sc->sc_hub_idata));
2818}
2819
2820/*------------------------------------------------------------------------*
2821 * xhci_device_done - XHCI done handler
2822 *
2823 * NOTE: This function can be called two times in a row on
2824 * the same USB transfer. From close and from interrupt.
2825 *------------------------------------------------------------------------*/
2826static void
2827xhci_device_done(struct usb_xfer *xfer, usb_error_t error)
2828{
2829 DPRINTFN(2, "xfer=%p, endpoint=%p, error=%d\n",
2830 xfer, xfer->endpoint, error);
2831
2832 /* remove transfer from HW queue */
2833 xhci_transfer_remove(xfer, error);
2834
2835 /* dequeue transfer and start next transfer */
2836 usbd_transfer_done(xfer, error);
2837}
2838
2839/*------------------------------------------------------------------------*
2840 * XHCI data transfer support (generic type)
2841 *------------------------------------------------------------------------*/
2842static void
2843xhci_device_generic_open(struct usb_xfer *xfer)
2844{
2845 if (xfer->flags_int.isochronous_xfr) {
2846 switch (xfer->xroot->udev->speed) {
2847 case USB_SPEED_FULL:
2848 break;
2849 default:
2850 usb_hs_bandwidth_alloc(xfer);
2851 break;
2852 }
2853 }
2854}
2855
2856static void
2857xhci_device_generic_close(struct usb_xfer *xfer)
2858{
2859 DPRINTF("\n");
2860
2861 xhci_device_done(xfer, USB_ERR_CANCELLED);
2862
2863 if (xfer->flags_int.isochronous_xfr) {
2864 switch (xfer->xroot->udev->speed) {
2865 case USB_SPEED_FULL:
2866 break;
2867 default:
2868 usb_hs_bandwidth_free(xfer);
2869 break;
2870 }
2871 }
2872}
2873
2874static void
2875xhci_device_generic_multi_enter(struct usb_endpoint *ep,
2876 usb_stream_t stream_id, struct usb_xfer *enter_xfer)
2877{
2878 struct usb_xfer *xfer;
2879
2880 /* check if there is a current transfer */
2881 xfer = ep->endpoint_q[stream_id].curr;
2882 if (xfer == NULL)
2883 return;
2884
2885 /*
2886 * Check if the current transfer is started and then pickup
2887 * the next one, if any. Else wait for next start event due to
2888 * block on failure feature.
2889 */
2890 if (!xfer->flags_int.bandwidth_reclaimed)
2891 return;
2892
2893 xfer = TAILQ_FIRST(&ep->endpoint_q[stream_id].head);
2894 if (xfer == NULL) {
2895 /*
2896 * In case of enter we have to consider that the
2897 * transfer is queued by the USB core after the enter
2898 * method is called.
2899 */
2900 xfer = enter_xfer;
2901
2902 if (xfer == NULL)
2903 return;
2904 }
2905
2906 /* try to multi buffer */
2907 xhci_transfer_insert(xfer);
2908}
2909
2910static void
2911xhci_device_generic_enter(struct usb_xfer *xfer)
2912{
2913 DPRINTF("\n");
2914
2915 /* setup TD's and QH */
2916 xhci_setup_generic_chain(xfer);
2917
2918 xhci_device_generic_multi_enter(xfer->endpoint,
2919 xfer->stream_id, xfer);
2920}
2921
2922static void
2923xhci_device_generic_start(struct usb_xfer *xfer)
2924{
2925 DPRINTF("\n");
2926
2927 /* try to insert xfer on HW queue */
2928 xhci_transfer_insert(xfer);
2929
2930 /* try to multi buffer */
2931 xhci_device_generic_multi_enter(xfer->endpoint,
2932 xfer->stream_id, NULL);
2933
2934 /* add transfer last on interrupt queue */
2935 usbd_transfer_enqueue(&xfer->xroot->bus->intr_q, xfer);
2936
2937 /* start timeout, if any */
2938 if (xfer->timeout != 0)
2939 usbd_transfer_timeout_ms(xfer, &xhci_timeout, xfer->timeout);
2940}
2941
2942struct usb_pipe_methods xhci_device_generic_methods =
2943{
2944 .open = xhci_device_generic_open,
2945 .close = xhci_device_generic_close,
2946 .enter = xhci_device_generic_enter,
2947 .start = xhci_device_generic_start,
2948};
2949
2950/*------------------------------------------------------------------------*
2951 * xhci root HUB support
2952 *------------------------------------------------------------------------*
2953 * Simulate a hardware HUB by handling all the necessary requests.
2954 *------------------------------------------------------------------------*/
2955
2956#define HSETW(ptr, val) ptr = { (uint8_t)(val), (uint8_t)((val) >> 8) }
2957
2958static const
2959struct usb_device_descriptor xhci_devd =
2960{
2961 .bLength = sizeof(xhci_devd),
2962 .bDescriptorType = UDESC_DEVICE, /* type */
2963 HSETW(.bcdUSB, 0x0300), /* USB version */
2964 .bDeviceClass = UDCLASS_HUB, /* class */
2965 .bDeviceSubClass = UDSUBCLASS_HUB, /* subclass */
2966 .bDeviceProtocol = UDPROTO_SSHUB, /* protocol */
2967 .bMaxPacketSize = 9, /* max packet size */
2968 HSETW(.idVendor, 0x0000), /* vendor */
2969 HSETW(.idProduct, 0x0000), /* product */
2970 HSETW(.bcdDevice, 0x0100), /* device version */
2971 .iManufacturer = 1,
2972 .iProduct = 2,
2973 .iSerialNumber = 0,
2974 .bNumConfigurations = 1, /* # of configurations */
2975};
2976
2977static const
2978struct xhci_bos_desc xhci_bosd = {
2979 .bosd = {
2980 .bLength = sizeof(xhci_bosd.bosd),
2981 .bDescriptorType = UDESC_BOS,
2982 HSETW(.wTotalLength, sizeof(xhci_bosd)),
2983 .bNumDeviceCaps = 3,
2984 },
2985 .usb2extd = {
2986 .bLength = sizeof(xhci_bosd.usb2extd),
2987 .bDescriptorType = 1,
2988 .bDevCapabilityType = 2,
2989 .bmAttributes[0] = 2,
2990 },
2991 .usbdcd = {
2992 .bLength = sizeof(xhci_bosd.usbdcd),
2993 .bDescriptorType = UDESC_DEVICE_CAPABILITY,
2994 .bDevCapabilityType = 3,
2995 .bmAttributes = 0, /* XXX */
2996 HSETW(.wSpeedsSupported, 0x000C),
2997 .bFunctionalitySupport = 8,
2998 .bU1DevExitLat = 255, /* dummy - not used */
2999 .wU2DevExitLat = { 0x00, 0x08 },
3000 },
3001 .cidd = {
3002 .bLength = sizeof(xhci_bosd.cidd),
3003 .bDescriptorType = 1,
3004 .bDevCapabilityType = 4,
3005 .bReserved = 0,
3006 .bContainerID = 0, /* XXX */
3007 },
3008};
3009
3010static const
3011struct xhci_config_desc xhci_confd = {
3012 .confd = {
3013 .bLength = sizeof(xhci_confd.confd),
3014 .bDescriptorType = UDESC_CONFIG,
3015 .wTotalLength[0] = sizeof(xhci_confd),
3016 .bNumInterface = 1,
3017 .bConfigurationValue = 1,
3018 .iConfiguration = 0,
3019 .bmAttributes = UC_SELF_POWERED,
3020 .bMaxPower = 0 /* max power */
3021 },
3022 .ifcd = {
3023 .bLength = sizeof(xhci_confd.ifcd),
3024 .bDescriptorType = UDESC_INTERFACE,
3025 .bNumEndpoints = 1,
3026 .bInterfaceClass = UICLASS_HUB,
3027 .bInterfaceSubClass = UISUBCLASS_HUB,
3028 .bInterfaceProtocol = 0,
3029 },
3030 .endpd = {
3031 .bLength = sizeof(xhci_confd.endpd),
3032 .bDescriptorType = UDESC_ENDPOINT,
3033 .bEndpointAddress = UE_DIR_IN | XHCI_INTR_ENDPT,
3034 .bmAttributes = UE_INTERRUPT,
3035 .wMaxPacketSize[0] = 2, /* max 15 ports */
3036 .bInterval = 255,
3037 },
3038 .endpcd = {
3039 .bLength = sizeof(xhci_confd.endpcd),
3040 .bDescriptorType = UDESC_ENDPOINT_SS_COMP,
3041 .bMaxBurst = 0,
3042 .bmAttributes = 0,
3043 },
3044};
3045
3046static const
3047struct usb_hub_ss_descriptor xhci_hubd = {
3048 .bLength = sizeof(xhci_hubd),
3049 .bDescriptorType = UDESC_SS_HUB,
3050};
3051
3052static usb_error_t
3053xhci_roothub_exec(struct usb_device *udev,
3054 struct usb_device_request *req, const void **pptr, uint16_t *plength)
3055{
3056 struct xhci_softc *sc = XHCI_BUS2SC(udev->bus);
3057 const char *str_ptr;
3058 const void *ptr;
3059 uint32_t port;
3060 uint32_t v;
3061 uint16_t len;
3062 uint16_t i;
3063 uint16_t value;
3064 uint16_t index;
3065 uint8_t j;
3066 usb_error_t err;
3067
3068 USB_BUS_LOCK_ASSERT(&sc->sc_bus, MA_OWNED);
3069
3070 /* buffer reset */
3071 ptr = (const void *)&sc->sc_hub_desc;
3072 len = 0;
3073 err = 0;
3074
3075 value = UGETW(req->wValue);
3076 index = UGETW(req->wIndex);
3077
3078 DPRINTFN(3, "type=0x%02x request=0x%02x wLen=0x%04x "
3079 "wValue=0x%04x wIndex=0x%04x\n",
3080 req->bmRequestType, req->bRequest,
3081 UGETW(req->wLength), value, index);
3082
3083#define C(x,y) ((x) | ((y) << 8))
3084 switch (C(req->bRequest, req->bmRequestType)) {
3085 case C(UR_CLEAR_FEATURE, UT_WRITE_DEVICE):
3086 case C(UR_CLEAR_FEATURE, UT_WRITE_INTERFACE):
3087 case C(UR_CLEAR_FEATURE, UT_WRITE_ENDPOINT):
3088 /*
3089 * DEVICE_REMOTE_WAKEUP and ENDPOINT_HALT are no-ops
3090 * for the integrated root hub.
3091 */
3092 break;
3093 case C(UR_GET_CONFIG, UT_READ_DEVICE):
3094 len = 1;
3095 sc->sc_hub_desc.temp[0] = sc->sc_conf;
3096 break;
3097 case C(UR_GET_DESCRIPTOR, UT_READ_DEVICE):
3098 switch (value >> 8) {
3099 case UDESC_DEVICE:
3100 if ((value & 0xff) != 0) {
3101 err = USB_ERR_IOERROR;
3102 goto done;
3103 }
3104 len = sizeof(xhci_devd);
3105 ptr = (const void *)&xhci_devd;
3106 break;
3107
3108 case UDESC_BOS:
3109 if ((value & 0xff) != 0) {
3110 err = USB_ERR_IOERROR;
3111 goto done;
3112 }
3113 len = sizeof(xhci_bosd);
3114 ptr = (const void *)&xhci_bosd;
3115 break;
3116
3117 case UDESC_CONFIG:
3118 if ((value & 0xff) != 0) {
3119 err = USB_ERR_IOERROR;
3120 goto done;
3121 }
3122 len = sizeof(xhci_confd);
3123 ptr = (const void *)&xhci_confd;
3124 break;
3125
3126 case UDESC_STRING:
3127 switch (value & 0xff) {
3128 case 0: /* Language table */
3129 str_ptr = "\001";
3130 break;
3131
3132 case 1: /* Vendor */
3133 str_ptr = sc->sc_vendor;
3134 break;
3135
3136 case 2: /* Product */
3137 str_ptr = "XHCI root HUB";
3138 break;
3139
3140 default:
3141 str_ptr = "";
3142 break;
3143 }
3144
3145 len = usb_make_str_desc(
3146 sc->sc_hub_desc.temp,
3147 sizeof(sc->sc_hub_desc.temp),
3148 str_ptr);
3149 break;
3150
3151 default:
3152 err = USB_ERR_IOERROR;
3153 goto done;
3154 }
3155 break;
3156 case C(UR_GET_INTERFACE, UT_READ_INTERFACE):
3157 len = 1;
3158 sc->sc_hub_desc.temp[0] = 0;
3159 break;
3160 case C(UR_GET_STATUS, UT_READ_DEVICE):
3161 len = 2;
3162 USETW(sc->sc_hub_desc.stat.wStatus, UDS_SELF_POWERED);
3163 break;
3164 case C(UR_GET_STATUS, UT_READ_INTERFACE):
3165 case C(UR_GET_STATUS, UT_READ_ENDPOINT):
3166 len = 2;
3167 USETW(sc->sc_hub_desc.stat.wStatus, 0);
3168 break;
3169 case C(UR_SET_ADDRESS, UT_WRITE_DEVICE):
3170 if (value >= XHCI_MAX_DEVICES) {
3171 err = USB_ERR_IOERROR;
3172 goto done;
3173 }
3174 break;
3175 case C(UR_SET_CONFIG, UT_WRITE_DEVICE):
3176 if (value != 0 && value != 1) {
3177 err = USB_ERR_IOERROR;
3178 goto done;
3179 }
3180 sc->sc_conf = value;
3181 break;
3182 case C(UR_SET_DESCRIPTOR, UT_WRITE_DEVICE):
3183 break;
3184 case C(UR_SET_FEATURE, UT_WRITE_DEVICE):
3185 case C(UR_SET_FEATURE, UT_WRITE_INTERFACE):
3186 case C(UR_SET_FEATURE, UT_WRITE_ENDPOINT):
3187 err = USB_ERR_IOERROR;
3188 goto done;
3189 case C(UR_SET_INTERFACE, UT_WRITE_INTERFACE):
3190 break;
3191 case C(UR_SYNCH_FRAME, UT_WRITE_ENDPOINT):
3192 break;
3193 /* Hub requests */
3194 case C(UR_CLEAR_FEATURE, UT_WRITE_CLASS_DEVICE):
3195 break;
3196 case C(UR_CLEAR_FEATURE, UT_WRITE_CLASS_OTHER):
3197 DPRINTFN(9, "UR_CLEAR_PORT_FEATURE\n");
3198
3199 if ((index < 1) ||
3200 (index > sc->sc_noport)) {
3201 err = USB_ERR_IOERROR;
3202 goto done;
3203 }
3204 port = XHCI_PORTSC(index);
3205
3206 v = XREAD4(sc, oper, port);
3207 i = XHCI_PS_PLS_GET(v);
3208 v &= ~XHCI_PS_CLEAR;
3209
3210 switch (value) {
3211 case UHF_C_BH_PORT_RESET:
3212 XWRITE4(sc, oper, port, v | XHCI_PS_WRC);
3213 break;
3214 case UHF_C_PORT_CONFIG_ERROR:
3215 XWRITE4(sc, oper, port, v | XHCI_PS_CEC);
3216 break;
3217 case UHF_C_PORT_SUSPEND:
3218 case UHF_C_PORT_LINK_STATE:
3219 XWRITE4(sc, oper, port, v | XHCI_PS_PLC);
3220 break;
3221 case UHF_C_PORT_CONNECTION:
3222 XWRITE4(sc, oper, port, v | XHCI_PS_CSC);
3223 break;
3224 case UHF_C_PORT_ENABLE:
3225 XWRITE4(sc, oper, port, v | XHCI_PS_PEC);
3226 break;
3227 case UHF_C_PORT_OVER_CURRENT:
3228 XWRITE4(sc, oper, port, v | XHCI_PS_OCC);
3229 break;
3230 case UHF_C_PORT_RESET:
3231 XWRITE4(sc, oper, port, v | XHCI_PS_PRC);
3232 break;
3233 case UHF_PORT_ENABLE:
3234 XWRITE4(sc, oper, port, v | XHCI_PS_PED);
3235 break;
3236 case UHF_PORT_POWER:
3237 XWRITE4(sc, oper, port, v & ~XHCI_PS_PP);
3238 break;
3239 case UHF_PORT_INDICATOR:
3240 XWRITE4(sc, oper, port, v & ~XHCI_PS_PIC_SET(3));
3241 break;
3242 case UHF_PORT_SUSPEND:
3243
3244 /* U3 -> U15 */
3245 if (i == 3) {
3246 XWRITE4(sc, oper, port, v |
3247 XHCI_PS_PLS_SET(0xF) | XHCI_PS_LWS);
3248 }
3249
3250 /* wait 20ms for resume sequence to complete */
3251 usb_pause_mtx(&sc->sc_bus.bus_mtx, hz / 50);
3252
3253 /* U0 */
3254 XWRITE4(sc, oper, port, v |
3255 XHCI_PS_PLS_SET(0) | XHCI_PS_LWS);
3256 break;
3257 default:
3258 err = USB_ERR_IOERROR;
3259 goto done;
3260 }
3261 break;
3262
3263 case C(UR_GET_DESCRIPTOR, UT_READ_CLASS_DEVICE):
3264 if ((value & 0xff) != 0) {
3265 err = USB_ERR_IOERROR;
3266 goto done;
3267 }
3268
3269 v = XREAD4(sc, capa, XHCI_HCSPARAMS0);
3270
3271 sc->sc_hub_desc.hubd = xhci_hubd;
3272
3273 sc->sc_hub_desc.hubd.bNbrPorts = sc->sc_noport;
3274
3275 if (XHCI_HCS0_PPC(v))
3276 i = UHD_PWR_INDIVIDUAL;
3277 else
3278 i = UHD_PWR_GANGED;
3279
3280 if (XHCI_HCS0_PIND(v))
3281 i |= UHD_PORT_IND;
3282
3283 i |= UHD_OC_INDIVIDUAL;
3284
3285 USETW(sc->sc_hub_desc.hubd.wHubCharacteristics, i);
3286
3287 /* see XHCI section 5.4.9: */
3288 sc->sc_hub_desc.hubd.bPwrOn2PwrGood = 10;
3289
3290 for (j = 1; j <= sc->sc_noport; j++) {
3291
3292 v = XREAD4(sc, oper, XHCI_PORTSC(j));
3293 if (v & XHCI_PS_DR) {
3294 sc->sc_hub_desc.hubd.
3295 DeviceRemovable[j / 8] |= 1U << (j % 8);
3296 }
3297 }
3298 len = sc->sc_hub_desc.hubd.bLength;
3299 break;
3300
3301 case C(UR_GET_STATUS, UT_READ_CLASS_DEVICE):
3302 len = 16;
3303 memset(sc->sc_hub_desc.temp, 0, 16);
3304 break;
3305
3306 case C(UR_GET_STATUS, UT_READ_CLASS_OTHER):
3307 DPRINTFN(9, "UR_GET_STATUS i=%d\n", index);
3308
3309 if ((index < 1) ||
3310 (index > sc->sc_noport)) {
3311 err = USB_ERR_IOERROR;
3312 goto done;
3313 }
3314
3315 v = XREAD4(sc, oper, XHCI_PORTSC(index));
3316
3317 DPRINTFN(9, "port status=0x%08x\n", v);
3318
3319 i = UPS_PORT_LINK_STATE_SET(XHCI_PS_PLS_GET(v));
3320
3321 switch (XHCI_PS_SPEED_GET(v)) {
3322 case 3:
3323 i |= UPS_HIGH_SPEED;
3324 break;
3325 case 2:
3326 i |= UPS_LOW_SPEED;
3327 break;
3328 case 1:
3329 /* FULL speed */
3330 break;
3331 default:
3332 i |= UPS_OTHER_SPEED;
3333 break;
3334 }
3335
3336 if (v & XHCI_PS_CCS)
3337 i |= UPS_CURRENT_CONNECT_STATUS;
3338 if (v & XHCI_PS_PED)
3339 i |= UPS_PORT_ENABLED;
3340 if (v & XHCI_PS_OCA)
3341 i |= UPS_OVERCURRENT_INDICATOR;
3342 if (v & XHCI_PS_PR)
3343 i |= UPS_RESET;
3344 if (v & XHCI_PS_PP) {
3345 /*
3346 * The USB 3.0 RH is using the
3347 * USB 2.0's power bit
3348 */
3349 i |= UPS_PORT_POWER;
3350 }
3351 USETW(sc->sc_hub_desc.ps.wPortStatus, i);
3352
3353 i = 0;
3354 if (v & XHCI_PS_CSC)
3355 i |= UPS_C_CONNECT_STATUS;
3356 if (v & XHCI_PS_PEC)
3357 i |= UPS_C_PORT_ENABLED;
3358 if (v & XHCI_PS_OCC)
3359 i |= UPS_C_OVERCURRENT_INDICATOR;
3360 if (v & XHCI_PS_WRC)
3361 i |= UPS_C_BH_PORT_RESET;
3362 if (v & XHCI_PS_PRC)
3363 i |= UPS_C_PORT_RESET;
3364 if (v & XHCI_PS_PLC)
3365 i |= UPS_C_PORT_LINK_STATE;
3366 if (v & XHCI_PS_CEC)
3367 i |= UPS_C_PORT_CONFIG_ERROR;
3368
3369 USETW(sc->sc_hub_desc.ps.wPortChange, i);
3370 len = sizeof(sc->sc_hub_desc.ps);
3371 break;
3372
3373 case C(UR_SET_DESCRIPTOR, UT_WRITE_CLASS_DEVICE):
3374 err = USB_ERR_IOERROR;
3375 goto done;
3376
3377 case C(UR_SET_FEATURE, UT_WRITE_CLASS_DEVICE):
3378 break;
3379
3380 case C(UR_SET_FEATURE, UT_WRITE_CLASS_OTHER):
3381
3382 i = index >> 8;
3383 index &= 0x00FF;
3384
3385 if ((index < 1) ||
3386 (index > sc->sc_noport)) {
3387 err = USB_ERR_IOERROR;
3388 goto done;
3389 }
3390
3391 port = XHCI_PORTSC(index);
3392 v = XREAD4(sc, oper, port) & ~XHCI_PS_CLEAR;
3393
3394 switch (value) {
3395 case UHF_PORT_U1_TIMEOUT:
3396 if (XHCI_PS_SPEED_GET(v) != 4) {
3397 err = USB_ERR_IOERROR;
3398 goto done;
3399 }
3400 port = XHCI_PORTPMSC(index);
3401 v = XREAD4(sc, oper, port);
3402 v &= ~XHCI_PM3_U1TO_SET(0xFF);
3403 v |= XHCI_PM3_U1TO_SET(i);
3404 XWRITE4(sc, oper, port, v);
3405 break;
3406 case UHF_PORT_U2_TIMEOUT:
3407 if (XHCI_PS_SPEED_GET(v) != 4) {
3408 err = USB_ERR_IOERROR;
3409 goto done;
3410 }
3411 port = XHCI_PORTPMSC(index);
3412 v = XREAD4(sc, oper, port);
3413 v &= ~XHCI_PM3_U2TO_SET(0xFF);
3414 v |= XHCI_PM3_U2TO_SET(i);
3415 XWRITE4(sc, oper, port, v);
3416 break;
3417 case UHF_BH_PORT_RESET:
3418 XWRITE4(sc, oper, port, v | XHCI_PS_WPR);
3419 break;
3420 case UHF_PORT_LINK_STATE:
3421 XWRITE4(sc, oper, port, v |
3422 XHCI_PS_PLS_SET(i) | XHCI_PS_LWS);
3423 /* 4ms settle time */
3424 usb_pause_mtx(&sc->sc_bus.bus_mtx, hz / 250);
3425 break;
3426 case UHF_PORT_ENABLE:
3427 DPRINTFN(3, "set port enable %d\n", index);
3428 break;
3429 case UHF_PORT_SUSPEND:
3430 DPRINTFN(6, "suspend port %u (LPM=%u)\n", index, i);
3431 j = XHCI_PS_SPEED_GET(v);
3432 if ((j < 1) || (j > 3)) {
3433 /* non-supported speed */
3434 err = USB_ERR_IOERROR;
3435 goto done;
3436 }
3437 XWRITE4(sc, oper, port, v |
3438 XHCI_PS_PLS_SET(i ? 2 /* LPM */ : 3) | XHCI_PS_LWS);
3439 break;
3440 case UHF_PORT_RESET:
3441 DPRINTFN(6, "reset port %d\n", index);
3442 XWRITE4(sc, oper, port, v | XHCI_PS_PR);
3443 break;
3444 case UHF_PORT_POWER:
3445 DPRINTFN(3, "set port power %d\n", index);
3446 XWRITE4(sc, oper, port, v | XHCI_PS_PP);
3447 break;
3448 case UHF_PORT_TEST:
3449 DPRINTFN(3, "set port test %d\n", index);
3450 break;
3451 case UHF_PORT_INDICATOR:
3452 DPRINTFN(3, "set port indicator %d\n", index);
3453
3454 v &= ~XHCI_PS_PIC_SET(3);
3455 v |= XHCI_PS_PIC_SET(1);
3456
3457 XWRITE4(sc, oper, port, v);
3458 break;
3459 default:
3460 err = USB_ERR_IOERROR;
3461 goto done;
3462 }
3463 break;
3464
3465 case C(UR_CLEAR_TT_BUFFER, UT_WRITE_CLASS_OTHER):
3466 case C(UR_RESET_TT, UT_WRITE_CLASS_OTHER):
3467 case C(UR_GET_TT_STATE, UT_READ_CLASS_OTHER):
3468 case C(UR_STOP_TT, UT_WRITE_CLASS_OTHER):
3469 break;
3470 default:
3471 err = USB_ERR_IOERROR;
3472 goto done;
3473 }
3474done:
3475 *plength = len;
3476 *pptr = ptr;
3477 return (err);
3478}
3479
3480static void
3481xhci_xfer_setup(struct usb_setup_params *parm)
3482{
3483 struct usb_page_search page_info;
3484 struct usb_page_cache *pc;
3485 struct xhci_softc *sc;
3486 struct usb_xfer *xfer;
3487 void *last_obj;
3488 uint32_t ntd;
3489 uint32_t n;
3490
3491 sc = XHCI_BUS2SC(parm->udev->bus);
3492 xfer = parm->curr_xfer;
3493
3494 /*
3495 * The proof for the "ntd" formula is illustrated like this:
3496 *
3497 * +------------------------------------+
3498 * | |
3499 * | |remainder -> |
3500 * | +-----+---+ |
3501 * | | xxx | x | frm 0 |
3502 * | +-----+---++ |
3503 * | | xxx | xx | frm 1 |
3504 * | +-----+----+ |
3505 * | ... |
3506 * +------------------------------------+
3507 *
3508 * "xxx" means a completely full USB transfer descriptor
3509 *
3510 * "x" and "xx" means a short USB packet
3511 *
3512 * For the remainder of an USB transfer modulo
3513 * "max_data_length" we need two USB transfer descriptors.
3514 * One to transfer the remaining data and one to finalise with
3515 * a zero length packet in case the "force_short_xfer" flag is
3516 * set. We only need two USB transfer descriptors in the case
3517 * where the transfer length of the first one is a factor of
3518 * "max_frame_size". The rest of the needed USB transfer
3519 * descriptors is given by the buffer size divided by the
3520 * maximum data payload.
3521 */
3522 parm->hc_max_packet_size = 0x400;
3523 parm->hc_max_packet_count = 16 * 3;
3524 parm->hc_max_frame_size = XHCI_TD_PAYLOAD_MAX;
3525
3526 xfer->flags_int.bdma_enable = 1;
3527
3528 usbd_transfer_setup_sub(parm);
3529
3530 if (xfer->flags_int.isochronous_xfr) {
3531 ntd = ((1 * xfer->nframes)
3532 + (xfer->max_data_length / xfer->max_hc_frame_size));
3533 } else if (xfer->flags_int.control_xfr) {
3534 ntd = ((2 * xfer->nframes) + 1 /* STATUS */
3535 + (xfer->max_data_length / xfer->max_hc_frame_size));
3536 } else {
3537 ntd = ((2 * xfer->nframes)
3538 + (xfer->max_data_length / xfer->max_hc_frame_size));
3539 }
3540
3541alloc_dma_set:
3542
3543 if (parm->err)
3544 return;
3545
3546 /*
3547 * Allocate queue heads and transfer descriptors
3548 */
3549 last_obj = NULL;
3550
3551 if (usbd_transfer_setup_sub_malloc(
3552 parm, &pc, sizeof(struct xhci_td),
3553 XHCI_TD_ALIGN, ntd)) {
3554 parm->err = USB_ERR_NOMEM;
3555 return;
3556 }
3557 if (parm->buf) {
3558 for (n = 0; n != ntd; n++) {
3559 struct xhci_td *td;
3560
3561 usbd_get_page(pc + n, 0, &page_info);
3562
3563 td = page_info.buffer;
3564
3565 /* init TD */
3566 td->td_self = page_info.physaddr;
3567 td->obj_next = last_obj;
3568 td->page_cache = pc + n;
3569
3570 last_obj = td;
3571
3572 usb_pc_cpu_flush(pc + n);
3573 }
3574 }
3575 xfer->td_start[xfer->flags_int.curr_dma_set] = last_obj;
3576
3577 if (!xfer->flags_int.curr_dma_set) {
3578 xfer->flags_int.curr_dma_set = 1;
3579 goto alloc_dma_set;
3580 }
3581}
3582
3583static usb_error_t
3584xhci_configure_reset_endpoint(struct usb_xfer *xfer)
3585{
3586 struct xhci_softc *sc = XHCI_BUS2SC(xfer->xroot->bus);
3587 struct usb_page_search buf_inp;
3588 struct usb_device *udev;
3589 struct xhci_endpoint_ext *pepext;
3590 struct usb_endpoint_descriptor *edesc;
3591 struct usb_page_cache *pcinp;
3592 usb_error_t err;
3593 usb_stream_t stream_id;
3594 uint8_t index;
3595 uint8_t epno;
3596
3597 pepext = xhci_get_endpoint_ext(xfer->xroot->udev,
3598 xfer->endpoint->edesc);
3599
3600 udev = xfer->xroot->udev;
3601 index = udev->controller_slot_id;
3602
3603 pcinp = &sc->sc_hw.devs[index].input_pc;
3604
3605 usbd_get_page(pcinp, 0, &buf_inp);
3606
3607 edesc = xfer->endpoint->edesc;
3608
3609 epno = edesc->bEndpointAddress;
3610 stream_id = xfer->stream_id;
3611
3612 if ((edesc->bmAttributes & UE_XFERTYPE) == UE_CONTROL)
3613 epno |= UE_DIR_IN;
3614
3615 epno = XHCI_EPNO2EPID(epno);
3616
3617 if (epno == 0)
3618 return (USB_ERR_NO_PIPE); /* invalid */
3619
3620 XHCI_CMD_LOCK(sc);
3621
3622 /* configure endpoint */
3623
3624 err = xhci_configure_endpoint_by_xfer(xfer);
3625
3626 if (err != 0) {
3627 XHCI_CMD_UNLOCK(sc);
3628 return (err);
3629 }
3630
3631 /*
3632 * Get the endpoint into the stopped state according to the
3633 * endpoint context state diagram in the XHCI specification:
3634 */
3635
3636 err = xhci_cmd_stop_ep(sc, 0, epno, index);
3637
3638 if (err != 0)
3639 DPRINTF("Could not stop endpoint %u\n", epno);
3640
3641 err = xhci_cmd_reset_ep(sc, 0, epno, index);
3642
3643 if (err != 0)
3644 DPRINTF("Could not reset endpoint %u\n", epno);
3645
3646 err = xhci_cmd_set_tr_dequeue_ptr(sc,
3647 (pepext->physaddr + (stream_id * sizeof(struct xhci_trb) *
3648 XHCI_MAX_TRANSFERS)) | XHCI_EPCTX_2_DCS_SET(1),
3649 stream_id, epno, index);
3650
3651 if (err != 0)
3652 DPRINTF("Could not set dequeue ptr for endpoint %u\n", epno);
3653
3654 /*
3655 * Get the endpoint into the running state according to the
3656 * endpoint context state diagram in the XHCI specification:
3657 */
3658
3659 xhci_configure_mask(udev, (1U << epno) | 1U, 0);
3660
3661 err = xhci_cmd_evaluate_ctx(sc, buf_inp.physaddr, index);
3662
3663 if (err != 0)
3664 DPRINTF("Could not configure endpoint %u\n", epno);
3665
3666 err = xhci_cmd_configure_ep(sc, buf_inp.physaddr, 0, index);
3667
3668 if (err != 0)
3669 DPRINTF("Could not configure endpoint %u\n", epno);
3670
3671 XHCI_CMD_UNLOCK(sc);
3672
3673 return (0);
3674}
3675
3676static void
3677xhci_xfer_unsetup(struct usb_xfer *xfer)
3678{
3679 return;
3680}
3681
3682static void
3683xhci_start_dma_delay(struct usb_xfer *xfer)
3684{
3685 struct xhci_softc *sc = XHCI_BUS2SC(xfer->xroot->bus);
3686
3687 /* put transfer on interrupt queue (again) */
3688 usbd_transfer_enqueue(&sc->sc_bus.intr_q, xfer);
3689
3690 (void)usb_proc_msignal(USB_BUS_CONTROL_XFER_PROC(&sc->sc_bus),
3691 &sc->sc_config_msg[0], &sc->sc_config_msg[1]);
3692}
3693
3694static void
3695xhci_configure_msg(struct usb_proc_msg *pm)
3696{
3697 struct xhci_softc *sc;
3698 struct xhci_endpoint_ext *pepext;
3699 struct usb_xfer *xfer;
3700
3701 sc = XHCI_BUS2SC(((struct usb_bus_msg *)pm)->bus);
3702
3703restart:
3704 TAILQ_FOREACH(xfer, &sc->sc_bus.intr_q.head, wait_entry) {
3705
3706 pepext = xhci_get_endpoint_ext(xfer->xroot->udev,
3707 xfer->endpoint->edesc);
3708
3709 if ((pepext->trb_halted != 0) ||
3710 (pepext->trb_running == 0)) {
3711
3712 uint16_t i;
3713
3714 /* clear halted and running */
3715 pepext->trb_halted = 0;
3716 pepext->trb_running = 0;
3717
3718 /* nuke remaining buffered transfers */
3719
3720 for (i = 0; i != (XHCI_MAX_TRANSFERS *
3721 XHCI_MAX_STREAMS); i++) {
3722 /*
3723 * NOTE: We need to use the timeout
3724 * error code here else existing
3725 * isochronous clients can get
3726 * confused:
3727 */
3728 if (pepext->xfer[i] != NULL) {
3729 xhci_device_done(pepext->xfer[i],
3730 USB_ERR_TIMEOUT);
3731 }
3732 }
3733
3734 /*
3735 * NOTE: The USB transfer cannot vanish in
3736 * this state!
3737 */
3738
3739 USB_BUS_UNLOCK(&sc->sc_bus);
3740
3741 xhci_configure_reset_endpoint(xfer);
3742
3743 USB_BUS_LOCK(&sc->sc_bus);
3744
3745 /* check if halted is still cleared */
3746 if (pepext->trb_halted == 0) {
3747 pepext->trb_running = 1;
3748 memset(pepext->trb_index, 0,
3749 sizeof(pepext->trb_index));
3750 }
3751 goto restart;
3752 }
3753
3754 if (xfer->flags_int.did_dma_delay) {
3755
3756 /* remove transfer from interrupt queue (again) */
3757 usbd_transfer_dequeue(xfer);
3758
3759 /* we are finally done */
3760 usb_dma_delay_done_cb(xfer);
3761
3762 /* queue changed - restart */
3763 goto restart;
3764 }
3765 }
3766
3767 TAILQ_FOREACH(xfer, &sc->sc_bus.intr_q.head, wait_entry) {
3768
3769 /* try to insert xfer on HW queue */
3770 xhci_transfer_insert(xfer);
3771
3772 /* try to multi buffer */
3773 xhci_device_generic_multi_enter(xfer->endpoint,
3774 xfer->stream_id, NULL);
3775 }
3776}
3777
3778static void
3779xhci_ep_init(struct usb_device *udev, struct usb_endpoint_descriptor *edesc,
3780 struct usb_endpoint *ep)
3781{
3782 struct xhci_endpoint_ext *pepext;
3783
3784 DPRINTFN(2, "endpoint=%p, addr=%d, endpt=%d, mode=%d\n",
3785 ep, udev->address, edesc->bEndpointAddress, udev->flags.usb_mode);
3786
3787 if (udev->parent_hub == NULL) {
3788 /* root HUB has special endpoint handling */
3789 return;
3790 }
3791
3792 ep->methods = &xhci_device_generic_methods;
3793
3794 pepext = xhci_get_endpoint_ext(udev, edesc);
3795
3796 USB_BUS_LOCK(udev->bus);
3797 pepext->trb_halted = 1;
3798 pepext->trb_running = 0;
3799 USB_BUS_UNLOCK(udev->bus);
3800}
3801
3802static void
3803xhci_ep_uninit(struct usb_device *udev, struct usb_endpoint *ep)
3804{
3805
3806}
3807
3808static void
3809xhci_ep_clear_stall(struct usb_device *udev, struct usb_endpoint *ep)
3810{
3811 struct xhci_endpoint_ext *pepext;
3812
3813 DPRINTF("\n");
3814
3815 if (udev->flags.usb_mode != USB_MODE_HOST) {
3816 /* not supported */
3817 return;
3818 }
3819 if (udev->parent_hub == NULL) {
3820 /* root HUB has special endpoint handling */
3821 return;
3822 }
3823
3824 pepext = xhci_get_endpoint_ext(udev, ep->edesc);
3825
3826 USB_BUS_LOCK(udev->bus);
3827 pepext->trb_halted = 1;
3828 pepext->trb_running = 0;
3829 USB_BUS_UNLOCK(udev->bus);
3830}
3831
3832static usb_error_t
3833xhci_device_init(struct usb_device *udev)
3834{
3835 struct xhci_softc *sc = XHCI_BUS2SC(udev->bus);
3836 usb_error_t err;
3837 uint8_t temp;
3838
3839 /* no init for root HUB */
3840 if (udev->parent_hub == NULL)
3841 return (0);
3842
3843 XHCI_CMD_LOCK(sc);
3844
3845 /* set invalid default */
3846
3847 udev->controller_slot_id = sc->sc_noslot + 1;
3848
3849 /* try to get a new slot ID from the XHCI */
3850
3851 err = xhci_cmd_enable_slot(sc, &temp);
3852
3853 if (err) {
3854 XHCI_CMD_UNLOCK(sc);
3855 return (err);
3856 }
3857
3858 if (temp > sc->sc_noslot) {
3859 XHCI_CMD_UNLOCK(sc);
3860 return (USB_ERR_BAD_ADDRESS);
3861 }
3862
3863 if (sc->sc_hw.devs[temp].state != XHCI_ST_DISABLED) {
3864 DPRINTF("slot %u already allocated.\n", temp);
3865 XHCI_CMD_UNLOCK(sc);
3866 return (USB_ERR_BAD_ADDRESS);
3867 }
3868
3869 /* store slot ID for later reference */
3870
3871 udev->controller_slot_id = temp;
3872
3873 /* reset data structure */
3874
3875 memset(&sc->sc_hw.devs[temp], 0, sizeof(sc->sc_hw.devs[0]));
3876
3877 /* set mark slot allocated */
3878
3879 sc->sc_hw.devs[temp].state = XHCI_ST_ENABLED;
3880
3881 err = xhci_alloc_device_ext(udev);
3882
3883 XHCI_CMD_UNLOCK(sc);
3884
3885 /* get device into default state */
3886
3887 if (err == 0)
3888 err = xhci_set_address(udev, NULL, 0);
3889
3890 return (err);
3891}
3892
3893static void
3894xhci_device_uninit(struct usb_device *udev)
3895{
3896 struct xhci_softc *sc = XHCI_BUS2SC(udev->bus);
3897 uint8_t index;
3898
3899 /* no init for root HUB */
3900 if (udev->parent_hub == NULL)
3901 return;
3902
3903 XHCI_CMD_LOCK(sc);
3904
3905 index = udev->controller_slot_id;
3906
3907 if (index <= sc->sc_noslot) {
3908 xhci_cmd_disable_slot(sc, index);
3909 sc->sc_hw.devs[index].state = XHCI_ST_DISABLED;
3910
3911 /* free device extension */
3912 xhci_free_device_ext(udev);
3913 }
3914
3915 XHCI_CMD_UNLOCK(sc);
3916}
3917
3918static void
3919xhci_get_dma_delay(struct usb_device *udev, uint32_t *pus)
3920{
3921 /*
3922 * Wait until the hardware has finished any possible use of
3923 * the transfer descriptor(s)
3924 */
3925 *pus = 2048; /* microseconds */
3926}
3927
3928static void
3929xhci_device_resume(struct usb_device *udev)
3930{
3931 struct xhci_softc *sc = XHCI_BUS2SC(udev->bus);
3932 uint8_t index;
3933 uint8_t n;
3934 uint8_t p;
3935
3936 DPRINTF("\n");
3937
3938 /* check for root HUB */
3939 if (udev->parent_hub == NULL)
3940 return;
3941
3942 index = udev->controller_slot_id;
3943
3944 XHCI_CMD_LOCK(sc);
3945
3946 /* blindly resume all endpoints */
3947
3948 USB_BUS_LOCK(udev->bus);
3949
3950 for (n = 1; n != XHCI_MAX_ENDPOINTS; n++) {
3951 for (p = 0; p != XHCI_MAX_STREAMS; p++) {
3952 XWRITE4(sc, door, XHCI_DOORBELL(index),
3953 n | XHCI_DB_SID_SET(p));
3954 }
3955 }
3956
3957 USB_BUS_UNLOCK(udev->bus);
3958
3959 XHCI_CMD_UNLOCK(sc);
3960}
3961
3962static void
3963xhci_device_suspend(struct usb_device *udev)
3964{
3965 struct xhci_softc *sc = XHCI_BUS2SC(udev->bus);
3966 uint8_t index;
3967 uint8_t n;
3968 usb_error_t err;
3969
3970 DPRINTF("\n");
3971
3972 /* check for root HUB */
3973 if (udev->parent_hub == NULL)
3974 return;
3975
3976 index = udev->controller_slot_id;
3977
3978 XHCI_CMD_LOCK(sc);
3979
3980 /* blindly suspend all endpoints */
3981
3982 for (n = 1; n != XHCI_MAX_ENDPOINTS; n++) {
3983 err = xhci_cmd_stop_ep(sc, 1, n, index);
3984 if (err != 0) {
3985 DPRINTF("Failed to suspend endpoint "
3986 "%u on slot %u (ignored).\n", n, index);
3987 }
3988 }
3989
3990 XHCI_CMD_UNLOCK(sc);
3991}
3992
3993static void
3994xhci_set_hw_power(struct usb_bus *bus)
3995{
3996 DPRINTF("\n");
3997}
3998
3999static void
4000xhci_device_state_change(struct usb_device *udev)
4001{
4002 struct xhci_softc *sc = XHCI_BUS2SC(udev->bus);
4003 struct usb_page_search buf_inp;
4004 usb_error_t err;
4005 uint8_t index;
4006
4007 /* check for root HUB */
4008 if (udev->parent_hub == NULL)
4009 return;
4010
4011 index = udev->controller_slot_id;
4012
4013 DPRINTF("\n");
4014
4015 if (usb_get_device_state(udev) == USB_STATE_CONFIGURED) {
4016 err = uhub_query_info(udev, &sc->sc_hw.devs[index].nports,
4017 &sc->sc_hw.devs[index].tt);
4018 if (err != 0)
4019 sc->sc_hw.devs[index].nports = 0;
4020 }
4021
4022 XHCI_CMD_LOCK(sc);
4023
4024 switch (usb_get_device_state(udev)) {
4025 case USB_STATE_POWERED:
4026 if (sc->sc_hw.devs[index].state == XHCI_ST_DEFAULT)
4027 break;
4028
4029 /* set default state */
4030 sc->sc_hw.devs[index].state = XHCI_ST_DEFAULT;
4031
4032 /* reset number of contexts */
4033 sc->sc_hw.devs[index].context_num = 0;
4034
4035 err = xhci_cmd_reset_dev(sc, index);
4036
4037 if (err != 0) {
4038 DPRINTF("Device reset failed "
4039 "for slot %u.\n", index);
4040 }
4041 break;
4042
4043 case USB_STATE_ADDRESSED:
4044 if (sc->sc_hw.devs[index].state == XHCI_ST_ADDRESSED)
4045 break;
4046
4047 sc->sc_hw.devs[index].state = XHCI_ST_ADDRESSED;
4048
4049 err = xhci_cmd_configure_ep(sc, 0, 1, index);
4050
4051 if (err) {
4052 DPRINTF("Failed to deconfigure "
4053 "slot %u.\n", index);
4054 }
4055 break;
4056
4057 case USB_STATE_CONFIGURED:
4058 if (sc->sc_hw.devs[index].state == XHCI_ST_CONFIGURED)
4059 break;
4060
4061 /* set configured state */
4062 sc->sc_hw.devs[index].state = XHCI_ST_CONFIGURED;
4063
4064 /* reset number of contexts */
4065 sc->sc_hw.devs[index].context_num = 0;
4066
4067 usbd_get_page(&sc->sc_hw.devs[index].input_pc, 0, &buf_inp);
4068
4069 xhci_configure_mask(udev, 3, 0);
4070
4071 err = xhci_configure_device(udev);
4072 if (err != 0) {
4073 DPRINTF("Could not configure device "
4074 "at slot %u.\n", index);
4075 }
4076
4077 err = xhci_cmd_evaluate_ctx(sc, buf_inp.physaddr, index);
4078 if (err != 0) {
4079 DPRINTF("Could not evaluate device "
4080 "context at slot %u.\n", index);
4081 }
4082 break;
4083
4084 default:
4085 break;
4086 }
4087 XHCI_CMD_UNLOCK(sc);
4088}
4089
4090static usb_error_t
4091xhci_set_endpoint_mode(struct usb_device *udev, struct usb_endpoint *ep,
4092 uint8_t ep_mode)
4093{
4094 switch (ep_mode) {
4095 case USB_EP_MODE_DEFAULT:
4096 return (0);
4097 case USB_EP_MODE_STREAMS:
4098 if ((ep->edesc->bmAttributes & UE_XFERTYPE) != UE_BULK ||
4099 udev->speed != USB_SPEED_SUPER)
4100 return (USB_ERR_INVAL);
4101 return (0);
4102 default:
4103 return (USB_ERR_INVAL);
4104 }
4105}
4106
4107struct usb_bus_methods xhci_bus_methods = {
4108 .endpoint_init = xhci_ep_init,
4109 .endpoint_uninit = xhci_ep_uninit,
4110 .xfer_setup = xhci_xfer_setup,
4111 .xfer_unsetup = xhci_xfer_unsetup,
4112 .get_dma_delay = xhci_get_dma_delay,
4113 .device_init = xhci_device_init,
4114 .device_uninit = xhci_device_uninit,
4115 .device_resume = xhci_device_resume,
4116 .device_suspend = xhci_device_suspend,
4117 .set_hw_power = xhci_set_hw_power,
4118 .roothub_exec = xhci_roothub_exec,
4119 .xfer_poll = xhci_do_poll,
4120 .start_dma_delay = xhci_start_dma_delay,
4121 .set_address = xhci_set_address,
4122 .clear_stall = xhci_ep_clear_stall,
4123 .device_state_change = xhci_device_state_change,
4124 .set_hw_power_sleep = xhci_set_hw_power_sleep,
4125 .set_endpoint_mode = xhci_set_endpoint_mode,
4126};
|