1213379Shselasky/*- 2213379Shselasky * Copyright (c) 2010 Hans Petter Selasky. All rights reserved. 3213379Shselasky * 4213379Shselasky * Redistribution and use in source and binary forms, with or without 5213379Shselasky * modification, are permitted provided that the following conditions 6213379Shselasky * are met: 7213379Shselasky * 1. Redistributions of source code must retain the above copyright 8213379Shselasky * notice, this list of conditions and the following disclaimer. 9213379Shselasky * 2. Redistributions in binary form must reproduce the above copyright 10213379Shselasky * notice, this list of conditions and the following disclaimer in the 11213379Shselasky * documentation and/or other materials provided with the distribution. 12213379Shselasky * 13213379Shselasky * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 14213379Shselasky * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 15213379Shselasky * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 16213379Shselasky * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 17213379Shselasky * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 18213379Shselasky * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 19213379Shselasky * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 20213379Shselasky * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 21213379Shselasky * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 22213379Shselasky * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 23213379Shselasky * SUCH DAMAGE. 24213379Shselasky */ 25213379Shselasky 26213379Shselasky/* 27213379Shselasky * USB eXtensible Host Controller Interface, a.k.a. USB 3.0 controller. 28213379Shselasky * 29213379Shselasky * The XHCI 1.0 spec can be found at 30213379Shselasky * http://www.intel.com/technology/usb/download/xHCI_Specification_for_USB.pdf 31213379Shselasky * and the USB 3.0 spec at 32213379Shselasky * http://www.usb.org/developers/docs/usb_30_spec_060910.zip 33213379Shselasky */ 34213379Shselasky 35213379Shselasky/* 36213379Shselasky * A few words about the design implementation: This driver emulates 37213379Shselasky * the concept about TDs which is found in EHCI specification. This 38213379Shselasky * way we avoid too much diveration among USB drivers. 39213379Shselasky */ 40213379Shselasky 41213379Shselasky#include <sys/cdefs.h> 42213379Shselasky__FBSDID("$FreeBSD$"); 43213379Shselasky 44213379Shselasky#include <sys/stdint.h> 45213379Shselasky#include <sys/stddef.h> 46213379Shselasky#include <sys/param.h> 47213379Shselasky#include <sys/queue.h> 48213379Shselasky#include <sys/types.h> 49213379Shselasky#include <sys/systm.h> 50213379Shselasky#include <sys/kernel.h> 51213379Shselasky#include <sys/bus.h> 52213379Shselasky#include <sys/module.h> 53213379Shselasky#include <sys/lock.h> 54213379Shselasky#include <sys/mutex.h> 55213379Shselasky#include <sys/condvar.h> 56213379Shselasky#include <sys/sysctl.h> 57213379Shselasky#include <sys/sx.h> 58213379Shselasky#include <sys/unistd.h> 59213379Shselasky#include <sys/callout.h> 60213379Shselasky#include <sys/malloc.h> 61213379Shselasky#include <sys/priv.h> 62213379Shselasky 63213379Shselasky#include <dev/usb/usb.h> 64213379Shselasky#include <dev/usb/usbdi.h> 65213379Shselasky 66213379Shselasky#define USB_DEBUG_VAR xhcidebug 67213379Shselasky 68213379Shselasky#include <dev/usb/usb_core.h> 69213379Shselasky#include <dev/usb/usb_debug.h> 70213379Shselasky#include <dev/usb/usb_busdma.h> 71213379Shselasky#include <dev/usb/usb_process.h> 72213379Shselasky#include <dev/usb/usb_transfer.h> 73213379Shselasky#include <dev/usb/usb_device.h> 74213379Shselasky#include <dev/usb/usb_hub.h> 75213379Shselasky#include <dev/usb/usb_util.h> 76213379Shselasky 77213379Shselasky#include <dev/usb/usb_controller.h> 78213379Shselasky#include <dev/usb/usb_bus.h> 79213379Shselasky#include <dev/usb/controller/xhci.h> 80213379Shselasky#include <dev/usb/controller/xhcireg.h> 81213379Shselasky 82213379Shselasky#define XHCI_BUS2SC(bus) \ 83213379Shselasky ((struct xhci_softc *)(((uint8_t *)(bus)) - \ 84213379Shselasky ((uint8_t *)&(((struct xhci_softc *)0)->sc_bus)))) 85213379Shselasky 86213379Shselasky#ifdef USB_DEBUG 87242774Shselaskystatic int xhcidebug; 88242774Shselaskystatic int xhciroute; 89213379Shselasky 90248085Smariusstatic SYSCTL_NODE(_hw_usb, OID_AUTO, xhci, CTLFLAG_RW, 0, "USB XHCI"); 91242775ShselaskySYSCTL_INT(_hw_usb_xhci, OID_AUTO, debug, CTLFLAG_RW | CTLFLAG_TUN, 92213379Shselasky &xhcidebug, 0, "Debug level"); 93242775ShselaskyTUNABLE_INT("hw.usb.xhci.debug", &xhcidebug); 94242775ShselaskySYSCTL_INT(_hw_usb_xhci, OID_AUTO, xhci_port_route, CTLFLAG_RW | CTLFLAG_TUN, 95242774Shselasky &xhciroute, 0, "Routing bitmap for switching EHCI ports to XHCI controller"); 96242774ShselaskyTUNABLE_INT("hw.usb.xhci.xhci_port_route", &xhciroute); 97255965Shselasky#else 98255965Shselasky#define xhciroute 0 99213379Shselasky#endif 100213379Shselasky 101213379Shselasky#define XHCI_INTR_ENDPT 1 102213379Shselasky 103213379Shselaskystruct xhci_std_temp { 104213379Shselasky struct xhci_softc *sc; 105213379Shselasky struct usb_page_cache *pc; 106213379Shselasky struct xhci_td *td; 107213379Shselasky struct xhci_td *td_next; 108213379Shselasky uint32_t len; 109213379Shselasky uint32_t offset; 110213379Shselasky uint32_t max_packet_size; 111213379Shselasky uint32_t average; 112213379Shselasky uint16_t isoc_delta; 113213379Shselasky uint16_t isoc_frame; 114213379Shselasky uint8_t shortpkt; 115213379Shselasky uint8_t multishort; 116213379Shselasky uint8_t last_frame; 117213379Shselasky uint8_t trb_type; 118213379Shselasky uint8_t direction; 119213379Shselasky uint8_t tbc; 120213379Shselasky uint8_t tlbpc; 121213379Shselasky uint8_t step_td; 122235001Shselasky uint8_t do_isoc_sync; 123213379Shselasky}; 124213379Shselasky 125213379Shselaskystatic void xhci_do_poll(struct usb_bus *); 126213379Shselaskystatic void xhci_device_done(struct usb_xfer *, usb_error_t); 127213379Shselaskystatic void xhci_root_intr(struct xhci_softc *); 128213379Shselaskystatic void xhci_free_device_ext(struct usb_device *); 129213379Shselaskystatic struct xhci_endpoint_ext *xhci_get_endpoint_ext(struct usb_device *, 130213379Shselasky struct usb_endpoint_descriptor *); 131213379Shselaskystatic usb_proc_callback_t xhci_configure_msg; 132213379Shselaskystatic usb_error_t xhci_configure_device(struct usb_device *); 133213379Shselaskystatic usb_error_t xhci_configure_endpoint(struct usb_device *, 134213379Shselasky struct usb_endpoint_descriptor *, uint64_t, uint16_t, 135213379Shselasky uint8_t, uint8_t, uint8_t, uint16_t, uint16_t); 136213379Shselaskystatic usb_error_t xhci_configure_mask(struct usb_device *, 137213379Shselasky uint32_t, uint8_t); 138213379Shselaskystatic usb_error_t xhci_cmd_evaluate_ctx(struct xhci_softc *, 139213379Shselasky uint64_t, uint8_t); 140213379Shselaskystatic void xhci_endpoint_doorbell(struct usb_xfer *); 141217374Shselaskystatic void xhci_ctx_set_le32(struct xhci_softc *sc, volatile uint32_t *ptr, uint32_t val); 142217374Shselaskystatic uint32_t xhci_ctx_get_le32(struct xhci_softc *sc, volatile uint32_t *ptr); 143217374Shselaskystatic void xhci_ctx_set_le64(struct xhci_softc *sc, volatile uint64_t *ptr, uint64_t val); 144217374Shselasky#ifdef USB_DEBUG 145217374Shselaskystatic uint64_t xhci_ctx_get_le64(struct xhci_softc *sc, volatile uint64_t *ptr); 146217374Shselasky#endif 147213379Shselasky 148213379Shselaskyextern struct usb_bus_methods xhci_bus_methods; 149213379Shselasky 150213379Shselasky#ifdef USB_DEBUG 151213379Shselaskystatic void 152213379Shselaskyxhci_dump_trb(struct xhci_trb *trb) 153213379Shselasky{ 154213379Shselasky DPRINTFN(5, "trb = %p\n", trb); 155213379Shselasky DPRINTFN(5, "qwTrb0 = 0x%016llx\n", (long long)le64toh(trb->qwTrb0)); 156213379Shselasky DPRINTFN(5, "dwTrb2 = 0x%08x\n", le32toh(trb->dwTrb2)); 157213379Shselasky DPRINTFN(5, "dwTrb3 = 0x%08x\n", le32toh(trb->dwTrb3)); 158213379Shselasky} 159213379Shselasky 160213379Shselaskystatic void 161217374Shselaskyxhci_dump_endpoint(struct xhci_softc *sc, struct xhci_endp_ctx *pep) 162213379Shselasky{ 163213379Shselasky DPRINTFN(5, "pep = %p\n", pep); 164217374Shselasky DPRINTFN(5, "dwEpCtx0=0x%08x\n", xhci_ctx_get_le32(sc, &pep->dwEpCtx0)); 165217374Shselasky DPRINTFN(5, "dwEpCtx1=0x%08x\n", xhci_ctx_get_le32(sc, &pep->dwEpCtx1)); 166217374Shselasky DPRINTFN(5, "qwEpCtx2=0x%016llx\n", (long long)xhci_ctx_get_le64(sc, &pep->qwEpCtx2)); 167217374Shselasky DPRINTFN(5, "dwEpCtx4=0x%08x\n", xhci_ctx_get_le32(sc, &pep->dwEpCtx4)); 168217374Shselasky DPRINTFN(5, "dwEpCtx5=0x%08x\n", xhci_ctx_get_le32(sc, &pep->dwEpCtx5)); 169217374Shselasky DPRINTFN(5, "dwEpCtx6=0x%08x\n", xhci_ctx_get_le32(sc, &pep->dwEpCtx6)); 170217374Shselasky DPRINTFN(5, "dwEpCtx7=0x%08x\n", xhci_ctx_get_le32(sc, &pep->dwEpCtx7)); 171213379Shselasky} 172213379Shselasky 173213379Shselaskystatic void 174217374Shselaskyxhci_dump_device(struct xhci_softc *sc, struct xhci_slot_ctx *psl) 175213379Shselasky{ 176213379Shselasky DPRINTFN(5, "psl = %p\n", psl); 177217374Shselasky DPRINTFN(5, "dwSctx0=0x%08x\n", xhci_ctx_get_le32(sc, &psl->dwSctx0)); 178217374Shselasky DPRINTFN(5, "dwSctx1=0x%08x\n", xhci_ctx_get_le32(sc, &psl->dwSctx1)); 179217374Shselasky DPRINTFN(5, "dwSctx2=0x%08x\n", xhci_ctx_get_le32(sc, &psl->dwSctx2)); 180217374Shselasky DPRINTFN(5, "dwSctx3=0x%08x\n", xhci_ctx_get_le32(sc, &psl->dwSctx3)); 181213379Shselasky} 182213379Shselasky#endif 183213379Shselasky 184213379Shselaskystatic void 185213379Shselaskyxhci_iterate_hw_softc(struct usb_bus *bus, usb_bus_mem_sub_cb_t *cb) 186213379Shselasky{ 187213379Shselasky struct xhci_softc *sc = XHCI_BUS2SC(bus); 188213379Shselasky uint8_t i; 189213379Shselasky 190213379Shselasky cb(bus, &sc->sc_hw.root_pc, &sc->sc_hw.root_pg, 191213379Shselasky sizeof(struct xhci_hw_root), XHCI_PAGE_SIZE); 192213379Shselasky 193213379Shselasky cb(bus, &sc->sc_hw.ctx_pc, &sc->sc_hw.ctx_pg, 194213379Shselasky sizeof(struct xhci_dev_ctx_addr), XHCI_PAGE_SIZE); 195213379Shselasky 196213379Shselasky for (i = 0; i != XHCI_MAX_SCRATCHPADS; i++) { 197213379Shselasky cb(bus, &sc->sc_hw.scratch_pc[i], &sc->sc_hw.scratch_pg[i], 198213379Shselasky XHCI_PAGE_SIZE, XHCI_PAGE_SIZE); 199213379Shselasky } 200213379Shselasky} 201213379Shselasky 202217374Shselaskystatic void 203217374Shselaskyxhci_ctx_set_le32(struct xhci_softc *sc, volatile uint32_t *ptr, uint32_t val) 204217374Shselasky{ 205217374Shselasky if (sc->sc_ctx_is_64_byte) { 206217374Shselasky uint32_t offset; 207217374Shselasky /* exploit the fact that our structures are XHCI_PAGE_SIZE aligned */ 208217374Shselasky /* all contexts are initially 32-bytes */ 209217374Shselasky offset = ((uintptr_t)ptr) & ((XHCI_PAGE_SIZE - 1) & ~(31U)); 210217374Shselasky ptr = (volatile uint32_t *)(((volatile uint8_t *)ptr) + offset); 211217374Shselasky } 212217374Shselasky *ptr = htole32(val); 213217374Shselasky} 214217374Shselasky 215217374Shselaskystatic uint32_t 216217374Shselaskyxhci_ctx_get_le32(struct xhci_softc *sc, volatile uint32_t *ptr) 217217374Shselasky{ 218217374Shselasky if (sc->sc_ctx_is_64_byte) { 219217374Shselasky uint32_t offset; 220217374Shselasky /* exploit the fact that our structures are XHCI_PAGE_SIZE aligned */ 221217374Shselasky /* all contexts are initially 32-bytes */ 222217374Shselasky offset = ((uintptr_t)ptr) & ((XHCI_PAGE_SIZE - 1) & ~(31U)); 223217374Shselasky ptr = (volatile uint32_t *)(((volatile uint8_t *)ptr) + offset); 224217374Shselasky } 225217374Shselasky return (le32toh(*ptr)); 226217374Shselasky} 227217374Shselasky 228217374Shselaskystatic void 229217374Shselaskyxhci_ctx_set_le64(struct xhci_softc *sc, volatile uint64_t *ptr, uint64_t val) 230217374Shselasky{ 231217374Shselasky if (sc->sc_ctx_is_64_byte) { 232217374Shselasky uint32_t offset; 233217374Shselasky /* exploit the fact that our structures are XHCI_PAGE_SIZE aligned */ 234217374Shselasky /* all contexts are initially 32-bytes */ 235217374Shselasky offset = ((uintptr_t)ptr) & ((XHCI_PAGE_SIZE - 1) & ~(31U)); 236217374Shselasky ptr = (volatile uint64_t *)(((volatile uint8_t *)ptr) + offset); 237217374Shselasky } 238217374Shselasky *ptr = htole64(val); 239217374Shselasky} 240217374Shselasky 241217374Shselasky#ifdef USB_DEBUG 242217374Shselaskystatic uint64_t 243217374Shselaskyxhci_ctx_get_le64(struct xhci_softc *sc, volatile uint64_t *ptr) 244217374Shselasky{ 245217374Shselasky if (sc->sc_ctx_is_64_byte) { 246217374Shselasky uint32_t offset; 247217374Shselasky /* exploit the fact that our structures are XHCI_PAGE_SIZE aligned */ 248217374Shselasky /* all contexts are initially 32-bytes */ 249217374Shselasky offset = ((uintptr_t)ptr) & ((XHCI_PAGE_SIZE - 1) & ~(31U)); 250217374Shselasky ptr = (volatile uint64_t *)(((volatile uint8_t *)ptr) + offset); 251217374Shselasky } 252217374Shselasky return (le64toh(*ptr)); 253217374Shselasky} 254217374Shselasky#endif 255217374Shselasky 256259602Shselaskystatic int 257259602Shselaskyxhci_reset_command_queue_locked(struct xhci_softc *sc) 258259602Shselasky{ 259259602Shselasky struct usb_page_search buf_res; 260259602Shselasky struct xhci_hw_root *phwr; 261259602Shselasky uint64_t addr; 262259602Shselasky uint32_t temp; 263259602Shselasky 264259602Shselasky DPRINTF("\n"); 265259602Shselasky 266259602Shselasky temp = XREAD4(sc, oper, XHCI_CRCR_LO); 267259602Shselasky if (temp & XHCI_CRCR_LO_CRR) { 268259602Shselasky DPRINTF("Command ring running\n"); 269259602Shselasky temp &= ~(XHCI_CRCR_LO_CS | XHCI_CRCR_LO_CA); 270259602Shselasky 271259602Shselasky /* 272259602Shselasky * Try to abort the last command as per section 273259602Shselasky * 4.6.1.2 "Aborting a Command" of the XHCI 274259602Shselasky * specification: 275259602Shselasky */ 276259602Shselasky 277259602Shselasky /* stop and cancel */ 278259602Shselasky XWRITE4(sc, oper, XHCI_CRCR_LO, temp | XHCI_CRCR_LO_CS); 279259602Shselasky XWRITE4(sc, oper, XHCI_CRCR_HI, 0); 280259602Shselasky 281259602Shselasky XWRITE4(sc, oper, XHCI_CRCR_LO, temp | XHCI_CRCR_LO_CA); 282259602Shselasky XWRITE4(sc, oper, XHCI_CRCR_HI, 0); 283259602Shselasky 284259602Shselasky /* wait 250ms */ 285259602Shselasky usb_pause_mtx(&sc->sc_bus.bus_mtx, hz / 4); 286259602Shselasky 287259602Shselasky /* check if command ring is still running */ 288259602Shselasky temp = XREAD4(sc, oper, XHCI_CRCR_LO); 289259602Shselasky if (temp & XHCI_CRCR_LO_CRR) { 290259602Shselasky DPRINTF("Comand ring still running\n"); 291259602Shselasky return (USB_ERR_IOERROR); 292259602Shselasky } 293259602Shselasky } 294259602Shselasky 295259602Shselasky /* reset command ring */ 296259602Shselasky sc->sc_command_ccs = 1; 297259602Shselasky sc->sc_command_idx = 0; 298259602Shselasky 299259602Shselasky usbd_get_page(&sc->sc_hw.root_pc, 0, &buf_res); 300259602Shselasky 301259602Shselasky /* setup command ring control base address */ 302259602Shselasky addr = buf_res.physaddr; 303259602Shselasky phwr = buf_res.buffer; 304259602Shselasky addr += (uintptr_t)&((struct xhci_hw_root *)0)->hwr_commands[0]; 305259602Shselasky 306259602Shselasky DPRINTF("CRCR=0x%016llx\n", (unsigned long long)addr); 307259602Shselasky 308259602Shselasky memset(phwr->hwr_commands, 0, sizeof(phwr->hwr_commands)); 309259602Shselasky phwr->hwr_commands[XHCI_MAX_COMMANDS - 1].qwTrb0 = htole64(addr); 310259602Shselasky 311259602Shselasky usb_pc_cpu_flush(&sc->sc_hw.root_pc); 312259602Shselasky 313259602Shselasky XWRITE4(sc, oper, XHCI_CRCR_LO, ((uint32_t)addr) | XHCI_CRCR_LO_RCS); 314259602Shselasky XWRITE4(sc, oper, XHCI_CRCR_HI, (uint32_t)(addr >> 32)); 315259602Shselasky 316259602Shselasky return (0); 317259602Shselasky} 318259602Shselasky 319213379Shselaskyusb_error_t 320213379Shselaskyxhci_start_controller(struct xhci_softc *sc) 321213379Shselasky{ 322213379Shselasky struct usb_page_search buf_res; 323213379Shselasky struct xhci_hw_root *phwr; 324213379Shselasky struct xhci_dev_ctx_addr *pdctxa; 325213379Shselasky uint64_t addr; 326213379Shselasky uint32_t temp; 327213379Shselasky uint16_t i; 328213379Shselasky 329213379Shselasky DPRINTF("\n"); 330213379Shselasky 331213379Shselasky sc->sc_capa_off = 0; 332213379Shselasky sc->sc_oper_off = XREAD1(sc, capa, XHCI_CAPLENGTH); 333213379Shselasky sc->sc_runt_off = XREAD4(sc, capa, XHCI_RTSOFF) & ~0x1F; 334213379Shselasky sc->sc_door_off = XREAD4(sc, capa, XHCI_DBOFF) & ~0x3; 335213379Shselasky 336213379Shselasky DPRINTF("CAPLENGTH=0x%x\n", sc->sc_oper_off); 337213379Shselasky DPRINTF("RUNTIMEOFFSET=0x%x\n", sc->sc_runt_off); 338213379Shselasky DPRINTF("DOOROFFSET=0x%x\n", sc->sc_door_off); 339213379Shselasky 340213379Shselasky sc->sc_event_ccs = 1; 341213379Shselasky sc->sc_event_idx = 0; 342213379Shselasky sc->sc_command_ccs = 1; 343213379Shselasky sc->sc_command_idx = 0; 344213379Shselasky 345213379Shselasky DPRINTF("xHCI version = 0x%04x\n", XREAD2(sc, capa, XHCI_HCIVERSION)); 346213379Shselasky 347213379Shselasky temp = XREAD4(sc, capa, XHCI_HCSPARAMS0); 348213379Shselasky 349213379Shselasky DPRINTF("HCS0 = 0x%08x\n", temp); 350213379Shselasky 351213379Shselasky if (XHCI_HCS0_CSZ(temp)) { 352217374Shselasky sc->sc_ctx_is_64_byte = 1; 353217374Shselasky device_printf(sc->sc_bus.parent, "64 byte context size.\n"); 354217374Shselasky } else { 355217374Shselasky sc->sc_ctx_is_64_byte = 0; 356217374Shselasky device_printf(sc->sc_bus.parent, "32 byte context size.\n"); 357213379Shselasky } 358213379Shselasky 359213379Shselasky /* Reset controller */ 360213379Shselasky XWRITE4(sc, oper, XHCI_USBCMD, XHCI_CMD_HCRST); 361213379Shselasky 362213379Shselasky for (i = 0; i != 100; i++) { 363229084Shselasky usb_pause_mtx(NULL, hz / 100); 364260538Shselasky temp = (XREAD4(sc, oper, XHCI_USBCMD) & XHCI_CMD_HCRST) | 365260538Shselasky (XREAD4(sc, oper, XHCI_USBSTS) & XHCI_STS_CNR); 366213379Shselasky if (!temp) 367213379Shselasky break; 368213379Shselasky } 369213379Shselasky 370213379Shselasky if (temp) { 371213379Shselasky device_printf(sc->sc_bus.parent, "Controller " 372213379Shselasky "reset timeout.\n"); 373213379Shselasky return (USB_ERR_IOERROR); 374213379Shselasky } 375213379Shselasky 376213379Shselasky if (!(XREAD4(sc, oper, XHCI_PAGESIZE) & XHCI_PAGESIZE_4K)) { 377213379Shselasky device_printf(sc->sc_bus.parent, "Controller does " 378213379Shselasky "not support 4K page size.\n"); 379213379Shselasky return (USB_ERR_IOERROR); 380213379Shselasky } 381213379Shselasky 382213379Shselasky temp = XREAD4(sc, capa, XHCI_HCSPARAMS1); 383213379Shselasky 384213379Shselasky i = XHCI_HCS1_N_PORTS(temp); 385213379Shselasky 386213379Shselasky if (i == 0) { 387213379Shselasky device_printf(sc->sc_bus.parent, "Invalid number " 388213379Shselasky "of ports: %u\n", i); 389213379Shselasky return (USB_ERR_IOERROR); 390213379Shselasky } 391213379Shselasky 392213379Shselasky sc->sc_noport = i; 393213379Shselasky sc->sc_noslot = XHCI_HCS1_DEVSLOT_MAX(temp); 394213379Shselasky 395213379Shselasky if (sc->sc_noslot > XHCI_MAX_DEVICES) 396213379Shselasky sc->sc_noslot = XHCI_MAX_DEVICES; 397213379Shselasky 398213379Shselasky /* setup number of device slots */ 399213379Shselasky 400213379Shselasky DPRINTF("CONFIG=0x%08x -> 0x%08x\n", 401213379Shselasky XREAD4(sc, oper, XHCI_CONFIG), sc->sc_noslot); 402213379Shselasky 403213379Shselasky XWRITE4(sc, oper, XHCI_CONFIG, sc->sc_noslot); 404213379Shselasky 405213379Shselasky DPRINTF("Max slots: %u\n", sc->sc_noslot); 406213379Shselasky 407213379Shselasky temp = XREAD4(sc, capa, XHCI_HCSPARAMS2); 408213379Shselasky 409213379Shselasky sc->sc_noscratch = XHCI_HCS2_SPB_MAX(temp); 410213379Shselasky 411213379Shselasky if (sc->sc_noscratch > XHCI_MAX_SCRATCHPADS) { 412213379Shselasky device_printf(sc->sc_bus.parent, "XHCI request " 413213379Shselasky "too many scratchpads\n"); 414213379Shselasky return (USB_ERR_NOMEM); 415213379Shselasky } 416213379Shselasky 417213379Shselasky DPRINTF("Max scratch: %u\n", sc->sc_noscratch); 418213379Shselasky 419213379Shselasky temp = XREAD4(sc, capa, XHCI_HCSPARAMS3); 420213379Shselasky 421213379Shselasky sc->sc_exit_lat_max = XHCI_HCS3_U1_DEL(temp) + 422213379Shselasky XHCI_HCS3_U2_DEL(temp) + 250 /* us */; 423213379Shselasky 424213379Shselasky temp = XREAD4(sc, oper, XHCI_USBSTS); 425213379Shselasky 426213379Shselasky /* clear interrupts */ 427213379Shselasky XWRITE4(sc, oper, XHCI_USBSTS, temp); 428213379Shselasky /* disable all device notifications */ 429213379Shselasky XWRITE4(sc, oper, XHCI_DNCTRL, 0); 430213379Shselasky 431213379Shselasky /* setup device context base address */ 432213379Shselasky usbd_get_page(&sc->sc_hw.ctx_pc, 0, &buf_res); 433213379Shselasky pdctxa = buf_res.buffer; 434213379Shselasky memset(pdctxa, 0, sizeof(*pdctxa)); 435213379Shselasky 436213379Shselasky addr = buf_res.physaddr; 437213379Shselasky addr += (uintptr_t)&((struct xhci_dev_ctx_addr *)0)->qwSpBufPtr[0]; 438213379Shselasky 439213379Shselasky /* slot 0 points to the table of scratchpad pointers */ 440213379Shselasky pdctxa->qwBaaDevCtxAddr[0] = htole64(addr); 441213379Shselasky 442213379Shselasky for (i = 0; i != sc->sc_noscratch; i++) { 443213379Shselasky struct usb_page_search buf_scp; 444213379Shselasky usbd_get_page(&sc->sc_hw.scratch_pc[i], 0, &buf_scp); 445213379Shselasky pdctxa->qwSpBufPtr[i] = htole64((uint64_t)buf_scp.physaddr); 446213379Shselasky } 447213379Shselasky 448213379Shselasky addr = buf_res.physaddr; 449213379Shselasky 450213379Shselasky XWRITE4(sc, oper, XHCI_DCBAAP_LO, (uint32_t)addr); 451213379Shselasky XWRITE4(sc, oper, XHCI_DCBAAP_HI, (uint32_t)(addr >> 32)); 452213379Shselasky XWRITE4(sc, oper, XHCI_DCBAAP_LO, (uint32_t)addr); 453213379Shselasky XWRITE4(sc, oper, XHCI_DCBAAP_HI, (uint32_t)(addr >> 32)); 454213379Shselasky 455213379Shselasky /* Setup event table size */ 456213379Shselasky 457213379Shselasky temp = XREAD4(sc, capa, XHCI_HCSPARAMS2); 458213379Shselasky 459213379Shselasky DPRINTF("HCS2=0x%08x\n", temp); 460213379Shselasky 461213379Shselasky temp = XHCI_HCS2_ERST_MAX(temp); 462213379Shselasky temp = 1U << temp; 463213379Shselasky if (temp > XHCI_MAX_RSEG) 464213379Shselasky temp = XHCI_MAX_RSEG; 465213379Shselasky 466213379Shselasky sc->sc_erst_max = temp; 467213379Shselasky 468213379Shselasky DPRINTF("ERSTSZ=0x%08x -> 0x%08x\n", 469213379Shselasky XREAD4(sc, runt, XHCI_ERSTSZ(0)), temp); 470213379Shselasky 471213379Shselasky XWRITE4(sc, runt, XHCI_ERSTSZ(0), XHCI_ERSTS_SET(temp)); 472213379Shselasky 473265079Shselasky /* Check if we should use the default IMOD value */ 474265079Shselasky if (sc->sc_imod_default == 0) 475265079Shselasky sc->sc_imod_default = XHCI_IMOD_DEFAULT; 476265079Shselasky 477213379Shselasky /* Setup interrupt rate */ 478265079Shselasky XWRITE4(sc, runt, XHCI_IMOD(0), sc->sc_imod_default); 479213379Shselasky 480213379Shselasky usbd_get_page(&sc->sc_hw.root_pc, 0, &buf_res); 481213379Shselasky 482213379Shselasky phwr = buf_res.buffer; 483213379Shselasky addr = buf_res.physaddr; 484213379Shselasky addr += (uintptr_t)&((struct xhci_hw_root *)0)->hwr_events[0]; 485213379Shselasky 486213379Shselasky /* reset hardware root structure */ 487213379Shselasky memset(phwr, 0, sizeof(*phwr)); 488213379Shselasky 489213379Shselasky phwr->hwr_ring_seg[0].qwEvrsTablePtr = htole64(addr); 490213379Shselasky phwr->hwr_ring_seg[0].dwEvrsTableSize = htole32(XHCI_MAX_EVENTS); 491213379Shselasky 492213379Shselasky DPRINTF("ERDP(0)=0x%016llx\n", (unsigned long long)addr); 493213379Shselasky 494213379Shselasky XWRITE4(sc, runt, XHCI_ERDP_LO(0), (uint32_t)addr); 495213379Shselasky XWRITE4(sc, runt, XHCI_ERDP_HI(0), (uint32_t)(addr >> 32)); 496213379Shselasky 497213379Shselasky addr = (uint64_t)buf_res.physaddr; 498213379Shselasky 499213379Shselasky DPRINTF("ERSTBA(0)=0x%016llx\n", (unsigned long long)addr); 500213379Shselasky 501213379Shselasky XWRITE4(sc, runt, XHCI_ERSTBA_LO(0), (uint32_t)addr); 502213379Shselasky XWRITE4(sc, runt, XHCI_ERSTBA_HI(0), (uint32_t)(addr >> 32)); 503213379Shselasky 504213379Shselasky /* Setup interrupter registers */ 505213379Shselasky 506213379Shselasky temp = XREAD4(sc, runt, XHCI_IMAN(0)); 507213379Shselasky temp |= XHCI_IMAN_INTR_ENA; 508213379Shselasky XWRITE4(sc, runt, XHCI_IMAN(0), temp); 509213379Shselasky 510213379Shselasky /* setup command ring control base address */ 511213379Shselasky addr = buf_res.physaddr; 512213379Shselasky addr += (uintptr_t)&((struct xhci_hw_root *)0)->hwr_commands[0]; 513213379Shselasky 514213379Shselasky DPRINTF("CRCR=0x%016llx\n", (unsigned long long)addr); 515213379Shselasky 516213379Shselasky XWRITE4(sc, oper, XHCI_CRCR_LO, ((uint32_t)addr) | XHCI_CRCR_LO_RCS); 517213379Shselasky XWRITE4(sc, oper, XHCI_CRCR_HI, (uint32_t)(addr >> 32)); 518213379Shselasky 519213379Shselasky phwr->hwr_commands[XHCI_MAX_COMMANDS - 1].qwTrb0 = htole64(addr); 520213379Shselasky 521213379Shselasky usb_bus_mem_flush_all(&sc->sc_bus, &xhci_iterate_hw_softc); 522213379Shselasky 523213379Shselasky /* Go! */ 524213379Shselasky XWRITE4(sc, oper, XHCI_USBCMD, XHCI_CMD_RS | 525213379Shselasky XHCI_CMD_INTE | XHCI_CMD_HSEE); 526213379Shselasky 527213379Shselasky for (i = 0; i != 100; i++) { 528229084Shselasky usb_pause_mtx(NULL, hz / 100); 529213379Shselasky temp = XREAD4(sc, oper, XHCI_USBSTS) & XHCI_STS_HCH; 530213379Shselasky if (!temp) 531213379Shselasky break; 532213379Shselasky } 533213379Shselasky if (temp) { 534213379Shselasky XWRITE4(sc, oper, XHCI_USBCMD, 0); 535213379Shselasky device_printf(sc->sc_bus.parent, "Run timeout.\n"); 536213379Shselasky return (USB_ERR_IOERROR); 537213379Shselasky } 538213379Shselasky 539213379Shselasky /* catch any lost interrupts */ 540213379Shselasky xhci_do_poll(&sc->sc_bus); 541213379Shselasky 542255965Shselasky if (sc->sc_port_route != NULL) { 543255965Shselasky /* Route all ports to the XHCI by default */ 544255965Shselasky sc->sc_port_route(sc->sc_bus.parent, 545255965Shselasky ~xhciroute, xhciroute); 546255965Shselasky } 547213379Shselasky return (0); 548213379Shselasky} 549213379Shselasky 550213379Shselaskyusb_error_t 551213379Shselaskyxhci_halt_controller(struct xhci_softc *sc) 552213379Shselasky{ 553213379Shselasky uint32_t temp; 554213379Shselasky uint16_t i; 555213379Shselasky 556213379Shselasky DPRINTF("\n"); 557213379Shselasky 558213379Shselasky sc->sc_capa_off = 0; 559213379Shselasky sc->sc_oper_off = XREAD1(sc, capa, XHCI_CAPLENGTH); 560213379Shselasky sc->sc_runt_off = XREAD4(sc, capa, XHCI_RTSOFF) & ~0xF; 561213379Shselasky sc->sc_door_off = XREAD4(sc, capa, XHCI_DBOFF) & ~0x3; 562213379Shselasky 563213379Shselasky /* Halt controller */ 564213379Shselasky XWRITE4(sc, oper, XHCI_USBCMD, 0); 565213379Shselasky 566213379Shselasky for (i = 0; i != 100; i++) { 567229084Shselasky usb_pause_mtx(NULL, hz / 100); 568213379Shselasky temp = XREAD4(sc, oper, XHCI_USBSTS) & XHCI_STS_HCH; 569213379Shselasky if (temp) 570213379Shselasky break; 571213379Shselasky } 572213379Shselasky 573213379Shselasky if (!temp) { 574213379Shselasky device_printf(sc->sc_bus.parent, "Controller halt timeout.\n"); 575213379Shselasky return (USB_ERR_IOERROR); 576213379Shselasky } 577213379Shselasky return (0); 578213379Shselasky} 579213379Shselasky 580213379Shselaskyusb_error_t 581213379Shselaskyxhci_init(struct xhci_softc *sc, device_t self) 582213379Shselasky{ 583213379Shselasky /* initialise some bus fields */ 584213379Shselasky sc->sc_bus.parent = self; 585213379Shselasky 586213379Shselasky /* set the bus revision */ 587213379Shselasky sc->sc_bus.usbrev = USB_REV_3_0; 588213379Shselasky 589213379Shselasky /* set up the bus struct */ 590213379Shselasky sc->sc_bus.methods = &xhci_bus_methods; 591213379Shselasky 592213379Shselasky /* setup devices array */ 593213379Shselasky sc->sc_bus.devices = sc->sc_devices; 594213379Shselasky sc->sc_bus.devices_max = XHCI_MAX_DEVICES; 595213379Shselasky 596213379Shselasky /* setup command queue mutex and condition varible */ 597213379Shselasky cv_init(&sc->sc_cmd_cv, "CMDQ"); 598213379Shselasky sx_init(&sc->sc_cmd_sx, "CMDQ lock"); 599213379Shselasky 600213379Shselasky /* get all DMA memory */ 601213379Shselasky if (usb_bus_mem_alloc_all(&sc->sc_bus, 602213379Shselasky USB_GET_DMA_TAG(self), &xhci_iterate_hw_softc)) { 603213379Shselasky return (ENOMEM); 604213379Shselasky } 605213379Shselasky 606213379Shselasky sc->sc_config_msg[0].hdr.pm_callback = &xhci_configure_msg; 607213379Shselasky sc->sc_config_msg[0].bus = &sc->sc_bus; 608213379Shselasky sc->sc_config_msg[1].hdr.pm_callback = &xhci_configure_msg; 609213379Shselasky sc->sc_config_msg[1].bus = &sc->sc_bus; 610213379Shselasky 611213379Shselasky if (usb_proc_create(&sc->sc_config_proc, 612213379Shselasky &sc->sc_bus.bus_mtx, device_get_nameunit(self), USB_PRI_MED)) { 613213379Shselasky printf("WARNING: Creation of XHCI configure " 614213379Shselasky "callback process failed.\n"); 615213379Shselasky } 616213379Shselasky return (0); 617213379Shselasky} 618213379Shselasky 619213379Shselaskyvoid 620213379Shselaskyxhci_uninit(struct xhci_softc *sc) 621213379Shselasky{ 622213379Shselasky usb_proc_free(&sc->sc_config_proc); 623213379Shselasky 624213379Shselasky usb_bus_mem_free_all(&sc->sc_bus, &xhci_iterate_hw_softc); 625213379Shselasky 626213379Shselasky cv_destroy(&sc->sc_cmd_cv); 627213379Shselasky sx_destroy(&sc->sc_cmd_sx); 628213379Shselasky} 629213379Shselasky 630229096Shselaskystatic void 631229096Shselaskyxhci_set_hw_power_sleep(struct usb_bus *bus, uint32_t state) 632213379Shselasky{ 633229096Shselasky struct xhci_softc *sc = XHCI_BUS2SC(bus); 634213379Shselasky 635229096Shselasky switch (state) { 636229096Shselasky case USB_HW_POWER_SUSPEND: 637229096Shselasky DPRINTF("Stopping the XHCI\n"); 638229096Shselasky xhci_halt_controller(sc); 639229096Shselasky break; 640229096Shselasky case USB_HW_POWER_SHUTDOWN: 641229096Shselasky DPRINTF("Stopping the XHCI\n"); 642229096Shselasky xhci_halt_controller(sc); 643229096Shselasky break; 644229096Shselasky case USB_HW_POWER_RESUME: 645229096Shselasky DPRINTF("Starting the XHCI\n"); 646229096Shselasky xhci_start_controller(sc); 647229096Shselasky break; 648229096Shselasky default: 649229096Shselasky break; 650229096Shselasky } 651213379Shselasky} 652213379Shselasky 653213379Shselaskystatic usb_error_t 654213379Shselaskyxhci_generic_done_sub(struct usb_xfer *xfer) 655213379Shselasky{ 656213379Shselasky struct xhci_td *td; 657213379Shselasky struct xhci_td *td_alt_next; 658213379Shselasky uint32_t len; 659213379Shselasky uint8_t status; 660213379Shselasky 661213379Shselasky td = xfer->td_transfer_cache; 662213379Shselasky td_alt_next = td->alt_next; 663213379Shselasky 664213379Shselasky if (xfer->aframes != xfer->nframes) 665213379Shselasky usbd_xfer_set_frame_len(xfer, xfer->aframes, 0); 666213379Shselasky 667213379Shselasky while (1) { 668213379Shselasky 669213379Shselasky usb_pc_cpu_invalidate(td->page_cache); 670213379Shselasky 671213379Shselasky status = td->status; 672213379Shselasky len = td->remainder; 673213379Shselasky 674213379Shselasky DPRINTFN(4, "xfer=%p[%u/%u] rem=%u/%u status=%u\n", 675213379Shselasky xfer, (unsigned int)xfer->aframes, 676213379Shselasky (unsigned int)xfer->nframes, 677213379Shselasky (unsigned int)len, (unsigned int)td->len, 678213379Shselasky (unsigned int)status); 679213379Shselasky 680213379Shselasky /* 681213379Shselasky * Verify the status length and 682213379Shselasky * add the length to "frlengths[]": 683213379Shselasky */ 684213379Shselasky if (len > td->len) { 685213379Shselasky /* should not happen */ 686213379Shselasky DPRINTF("Invalid status length, " 687213379Shselasky "0x%04x/0x%04x bytes\n", len, td->len); 688213379Shselasky status = XHCI_TRB_ERROR_LENGTH; 689213379Shselasky } else if (xfer->aframes != xfer->nframes) { 690213379Shselasky xfer->frlengths[xfer->aframes] += td->len - len; 691213379Shselasky } 692213379Shselasky /* Check for last transfer */ 693213379Shselasky if (((void *)td) == xfer->td_transfer_last) { 694213379Shselasky td = NULL; 695213379Shselasky break; 696213379Shselasky } 697213379Shselasky /* Check for transfer error */ 698213379Shselasky if (status != XHCI_TRB_ERROR_SHORT_PKT && 699213379Shselasky status != XHCI_TRB_ERROR_SUCCESS) { 700213379Shselasky /* the transfer is finished */ 701213379Shselasky td = NULL; 702213379Shselasky break; 703213379Shselasky } 704213379Shselasky /* Check for short transfer */ 705213379Shselasky if (len > 0) { 706213379Shselasky if (xfer->flags_int.short_frames_ok || 707213379Shselasky xfer->flags_int.isochronous_xfr || 708213379Shselasky xfer->flags_int.control_xfr) { 709213379Shselasky /* follow alt next */ 710213379Shselasky td = td->alt_next; 711213379Shselasky } else { 712213379Shselasky /* the transfer is finished */ 713213379Shselasky td = NULL; 714213379Shselasky } 715213379Shselasky break; 716213379Shselasky } 717213379Shselasky td = td->obj_next; 718213379Shselasky 719213379Shselasky if (td->alt_next != td_alt_next) { 720213379Shselasky /* this USB frame is complete */ 721213379Shselasky break; 722213379Shselasky } 723213379Shselasky } 724213379Shselasky 725213379Shselasky /* update transfer cache */ 726213379Shselasky 727213379Shselasky xfer->td_transfer_cache = td; 728213379Shselasky 729213379Shselasky return ((status == XHCI_TRB_ERROR_STALL) ? USB_ERR_STALLED : 730213379Shselasky (status != XHCI_TRB_ERROR_SHORT_PKT && 731213379Shselasky status != XHCI_TRB_ERROR_SUCCESS) ? USB_ERR_IOERROR : 732213379Shselasky USB_ERR_NORMAL_COMPLETION); 733213379Shselasky} 734213379Shselasky 735213379Shselaskystatic void 736213379Shselaskyxhci_generic_done(struct usb_xfer *xfer) 737213379Shselasky{ 738213379Shselasky usb_error_t err = 0; 739213379Shselasky 740213379Shselasky DPRINTFN(13, "xfer=%p endpoint=%p transfer done\n", 741213379Shselasky xfer, xfer->endpoint); 742213379Shselasky 743213379Shselasky /* reset scanner */ 744213379Shselasky 745213379Shselasky xfer->td_transfer_cache = xfer->td_transfer_first; 746213379Shselasky 747213379Shselasky if (xfer->flags_int.control_xfr) { 748213379Shselasky 749213379Shselasky if (xfer->flags_int.control_hdr) 750213379Shselasky err = xhci_generic_done_sub(xfer); 751213379Shselasky 752213379Shselasky xfer->aframes = 1; 753213379Shselasky 754213379Shselasky if (xfer->td_transfer_cache == NULL) 755213379Shselasky goto done; 756213379Shselasky } 757213379Shselasky 758213379Shselasky while (xfer->aframes != xfer->nframes) { 759213379Shselasky 760213379Shselasky err = xhci_generic_done_sub(xfer); 761213379Shselasky xfer->aframes++; 762213379Shselasky 763213379Shselasky if (xfer->td_transfer_cache == NULL) 764213379Shselasky goto done; 765213379Shselasky } 766213379Shselasky 767213379Shselasky if (xfer->flags_int.control_xfr && 768213379Shselasky !xfer->flags_int.control_act) 769213379Shselasky err = xhci_generic_done_sub(xfer); 770213379Shselaskydone: 771213379Shselasky /* transfer is complete */ 772213379Shselasky xhci_device_done(xfer, err); 773213379Shselasky} 774213379Shselasky 775213379Shselaskystatic void 776213379Shselaskyxhci_activate_transfer(struct usb_xfer *xfer) 777213379Shselasky{ 778213379Shselasky struct xhci_td *td; 779213379Shselasky 780213379Shselasky td = xfer->td_transfer_cache; 781213379Shselasky 782213379Shselasky usb_pc_cpu_invalidate(td->page_cache); 783213379Shselasky 784213379Shselasky if (!(td->td_trb[0].dwTrb3 & htole32(XHCI_TRB_3_CYCLE_BIT))) { 785213379Shselasky 786213379Shselasky /* activate the transfer */ 787213379Shselasky 788213379Shselasky td->td_trb[0].dwTrb3 |= htole32(XHCI_TRB_3_CYCLE_BIT); 789213379Shselasky usb_pc_cpu_flush(td->page_cache); 790213379Shselasky 791213379Shselasky xhci_endpoint_doorbell(xfer); 792213379Shselasky } 793213379Shselasky} 794213379Shselasky 795213379Shselaskystatic void 796213379Shselaskyxhci_skip_transfer(struct usb_xfer *xfer) 797213379Shselasky{ 798213379Shselasky struct xhci_td *td; 799213379Shselasky struct xhci_td *td_last; 800213379Shselasky 801213379Shselasky td = xfer->td_transfer_cache; 802213379Shselasky td_last = xfer->td_transfer_last; 803213379Shselasky 804213379Shselasky td = td->alt_next; 805213379Shselasky 806213379Shselasky usb_pc_cpu_invalidate(td->page_cache); 807213379Shselasky 808213379Shselasky if (!(td->td_trb[0].dwTrb3 & htole32(XHCI_TRB_3_CYCLE_BIT))) { 809213379Shselasky 810213379Shselasky usb_pc_cpu_invalidate(td_last->page_cache); 811213379Shselasky 812213379Shselasky /* copy LINK TRB to current waiting location */ 813213379Shselasky 814213379Shselasky td->td_trb[0].qwTrb0 = td_last->td_trb[td_last->ntrb].qwTrb0; 815213379Shselasky td->td_trb[0].dwTrb2 = td_last->td_trb[td_last->ntrb].dwTrb2; 816213379Shselasky usb_pc_cpu_flush(td->page_cache); 817213379Shselasky 818213379Shselasky td->td_trb[0].dwTrb3 = td_last->td_trb[td_last->ntrb].dwTrb3; 819213379Shselasky usb_pc_cpu_flush(td->page_cache); 820213379Shselasky 821213379Shselasky xhci_endpoint_doorbell(xfer); 822213379Shselasky } 823213379Shselasky} 824213379Shselasky 825213379Shselasky/*------------------------------------------------------------------------* 826213379Shselasky * xhci_check_transfer 827213379Shselasky *------------------------------------------------------------------------*/ 828213379Shselaskystatic void 829213379Shselaskyxhci_check_transfer(struct xhci_softc *sc, struct xhci_trb *trb) 830213379Shselasky{ 831213379Shselasky int64_t offset; 832213379Shselasky uint64_t td_event; 833213379Shselasky uint32_t temp; 834213379Shselasky uint32_t remainder; 835213379Shselasky uint8_t status; 836213379Shselasky uint8_t halted; 837213379Shselasky uint8_t epno; 838213379Shselasky uint8_t index; 839213379Shselasky uint8_t i; 840213379Shselasky 841213379Shselasky /* decode TRB */ 842213379Shselasky td_event = le64toh(trb->qwTrb0); 843213379Shselasky temp = le32toh(trb->dwTrb2); 844213379Shselasky 845213379Shselasky remainder = XHCI_TRB_2_REM_GET(temp); 846213379Shselasky status = XHCI_TRB_2_ERROR_GET(temp); 847213379Shselasky 848213379Shselasky temp = le32toh(trb->dwTrb3); 849213379Shselasky epno = XHCI_TRB_3_EP_GET(temp); 850213379Shselasky index = XHCI_TRB_3_SLOT_GET(temp); 851213379Shselasky 852213379Shselasky /* check if error means halted */ 853213379Shselasky halted = (status != XHCI_TRB_ERROR_SHORT_PKT && 854213379Shselasky status != XHCI_TRB_ERROR_SUCCESS); 855213379Shselasky 856213379Shselasky DPRINTF("slot=%u epno=%u remainder=%u status=%u\n", 857213379Shselasky index, epno, remainder, status); 858213379Shselasky 859213379Shselasky if (index > sc->sc_noslot) { 860213379Shselasky DPRINTF("Invalid slot.\n"); 861213379Shselasky return; 862213379Shselasky } 863213379Shselasky 864213379Shselasky if ((epno == 0) || (epno >= XHCI_MAX_ENDPOINTS)) { 865213379Shselasky DPRINTF("Invalid endpoint.\n"); 866213379Shselasky return; 867213379Shselasky } 868213379Shselasky 869213379Shselasky /* try to find the USB transfer that generated the event */ 870213379Shselasky for (i = 0; i != (XHCI_MAX_TRANSFERS - 1); i++) { 871213379Shselasky struct usb_xfer *xfer; 872213379Shselasky struct xhci_td *td; 873213379Shselasky struct xhci_endpoint_ext *pepext; 874213379Shselasky 875213379Shselasky pepext = &sc->sc_hw.devs[index].endp[epno]; 876213379Shselasky 877213379Shselasky xfer = pepext->xfer[i]; 878213379Shselasky if (xfer == NULL) 879213379Shselasky continue; 880213379Shselasky 881213379Shselasky td = xfer->td_transfer_cache; 882213379Shselasky 883213379Shselasky DPRINTFN(5, "Checking if 0x%016llx == (0x%016llx .. 0x%016llx)\n", 884213379Shselasky (long long)td_event, 885213379Shselasky (long long)td->td_self, 886213379Shselasky (long long)td->td_self + sizeof(td->td_trb)); 887213379Shselasky 888213379Shselasky /* 889213379Shselasky * NOTE: Some XHCI implementations might not trigger 890213379Shselasky * an event on the last LINK TRB so we need to 891213379Shselasky * consider both the last and second last event 892213379Shselasky * address as conditions for a successful transfer. 893213379Shselasky * 894213379Shselasky * NOTE: We assume that the XHCI will only trigger one 895213379Shselasky * event per chain of TRBs. 896213379Shselasky */ 897213379Shselasky 898213379Shselasky offset = td_event - td->td_self; 899213379Shselasky 900213379Shselasky if (offset >= 0 && 901235000Shselasky offset < (int64_t)sizeof(td->td_trb)) { 902213379Shselasky 903213379Shselasky usb_pc_cpu_invalidate(td->page_cache); 904213379Shselasky 905213379Shselasky /* compute rest of remainder, if any */ 906213379Shselasky for (i = (offset / 16) + 1; i < td->ntrb; i++) { 907213379Shselasky temp = le32toh(td->td_trb[i].dwTrb2); 908213379Shselasky remainder += XHCI_TRB_2_BYTES_GET(temp); 909213379Shselasky } 910213379Shselasky 911213379Shselasky DPRINTFN(5, "New remainder: %u\n", remainder); 912213379Shselasky 913213379Shselasky /* clear isochronous transfer errors */ 914213379Shselasky if (xfer->flags_int.isochronous_xfr) { 915213379Shselasky if (halted) { 916213379Shselasky halted = 0; 917213379Shselasky status = XHCI_TRB_ERROR_SUCCESS; 918213379Shselasky remainder = td->len; 919213379Shselasky } 920213379Shselasky } 921213379Shselasky 922213379Shselasky /* "td->remainder" is verified later */ 923213379Shselasky td->remainder = remainder; 924213379Shselasky td->status = status; 925213379Shselasky 926213379Shselasky usb_pc_cpu_flush(td->page_cache); 927213379Shselasky 928213379Shselasky /* 929213379Shselasky * 1) Last transfer descriptor makes the 930213379Shselasky * transfer done 931213379Shselasky */ 932213379Shselasky if (((void *)td) == xfer->td_transfer_last) { 933213379Shselasky DPRINTF("TD is last\n"); 934213379Shselasky xhci_generic_done(xfer); 935213379Shselasky break; 936213379Shselasky } 937213379Shselasky 938213379Shselasky /* 939213379Shselasky * 2) Any kind of error makes the transfer 940213379Shselasky * done 941213379Shselasky */ 942213379Shselasky if (halted) { 943213379Shselasky DPRINTF("TD has I/O error\n"); 944213379Shselasky xhci_generic_done(xfer); 945213379Shselasky break; 946213379Shselasky } 947213379Shselasky 948213379Shselasky /* 949213379Shselasky * 3) If there is no alternate next transfer, 950213379Shselasky * a short packet also makes the transfer done 951213379Shselasky */ 952213379Shselasky if (td->remainder > 0) { 953246395Shselasky if (td->alt_next == NULL) { 954246395Shselasky DPRINTF( 955246395Shselasky "short TD has no alternate next\n"); 956246395Shselasky xhci_generic_done(xfer); 957246395Shselasky break; 958246395Shselasky } 959213379Shselasky DPRINTF("TD has short pkt\n"); 960213379Shselasky if (xfer->flags_int.short_frames_ok || 961213379Shselasky xfer->flags_int.isochronous_xfr || 962213379Shselasky xfer->flags_int.control_xfr) { 963213379Shselasky /* follow the alt next */ 964213379Shselasky xfer->td_transfer_cache = td->alt_next; 965213379Shselasky xhci_activate_transfer(xfer); 966213379Shselasky break; 967213379Shselasky } 968213379Shselasky xhci_skip_transfer(xfer); 969213379Shselasky xhci_generic_done(xfer); 970213379Shselasky break; 971213379Shselasky } 972213379Shselasky 973213379Shselasky /* 974213379Shselasky * 4) Transfer complete - go to next TD 975213379Shselasky */ 976213379Shselasky DPRINTF("Following next TD\n"); 977213379Shselasky xfer->td_transfer_cache = td->obj_next; 978213379Shselasky xhci_activate_transfer(xfer); 979213379Shselasky break; /* there should only be one match */ 980213379Shselasky } 981213379Shselasky } 982213379Shselasky} 983213379Shselasky 984255965Shselaskystatic int 985213379Shselaskyxhci_check_command(struct xhci_softc *sc, struct xhci_trb *trb) 986213379Shselasky{ 987213379Shselasky if (sc->sc_cmd_addr == trb->qwTrb0) { 988213379Shselasky DPRINTF("Received command event\n"); 989213379Shselasky sc->sc_cmd_result[0] = trb->dwTrb2; 990213379Shselasky sc->sc_cmd_result[1] = trb->dwTrb3; 991213379Shselasky cv_signal(&sc->sc_cmd_cv); 992255965Shselasky return (1); /* command match */ 993213379Shselasky } 994255965Shselasky return (0); 995213379Shselasky} 996213379Shselasky 997255965Shselaskystatic int 998213379Shselaskyxhci_interrupt_poll(struct xhci_softc *sc) 999213379Shselasky{ 1000213379Shselasky struct usb_page_search buf_res; 1001213379Shselasky struct xhci_hw_root *phwr; 1002213379Shselasky uint64_t addr; 1003213379Shselasky uint32_t temp; 1004255965Shselasky int retval = 0; 1005213379Shselasky uint16_t i; 1006213379Shselasky uint8_t event; 1007213379Shselasky uint8_t j; 1008213379Shselasky uint8_t k; 1009213379Shselasky uint8_t t; 1010213379Shselasky 1011213379Shselasky usbd_get_page(&sc->sc_hw.root_pc, 0, &buf_res); 1012213379Shselasky 1013213379Shselasky phwr = buf_res.buffer; 1014213379Shselasky 1015213379Shselasky /* Receive any events */ 1016213379Shselasky 1017213379Shselasky usb_pc_cpu_invalidate(&sc->sc_hw.root_pc); 1018213379Shselasky 1019213379Shselasky i = sc->sc_event_idx; 1020213379Shselasky j = sc->sc_event_ccs; 1021213379Shselasky t = 2; 1022213379Shselasky 1023213379Shselasky while (1) { 1024213379Shselasky 1025213379Shselasky temp = le32toh(phwr->hwr_events[i].dwTrb3); 1026213379Shselasky 1027213379Shselasky k = (temp & XHCI_TRB_3_CYCLE_BIT) ? 1 : 0; 1028213379Shselasky 1029213379Shselasky if (j != k) 1030213379Shselasky break; 1031213379Shselasky 1032213379Shselasky event = XHCI_TRB_3_TYPE_GET(temp); 1033213379Shselasky 1034213379Shselasky DPRINTFN(10, "event[%u] = %u (0x%016llx 0x%08lx 0x%08lx)\n", 1035213379Shselasky i, event, (long long)le64toh(phwr->hwr_events[i].qwTrb0), 1036213379Shselasky (long)le32toh(phwr->hwr_events[i].dwTrb2), 1037213379Shselasky (long)le32toh(phwr->hwr_events[i].dwTrb3)); 1038213379Shselasky 1039213379Shselasky switch (event) { 1040213379Shselasky case XHCI_TRB_EVENT_TRANSFER: 1041213379Shselasky xhci_check_transfer(sc, &phwr->hwr_events[i]); 1042213379Shselasky break; 1043213379Shselasky case XHCI_TRB_EVENT_CMD_COMPLETE: 1044255965Shselasky retval |= xhci_check_command(sc, &phwr->hwr_events[i]); 1045213379Shselasky break; 1046213379Shselasky default: 1047213379Shselasky DPRINTF("Unhandled event = %u\n", event); 1048213379Shselasky break; 1049213379Shselasky } 1050213379Shselasky 1051213379Shselasky i++; 1052213379Shselasky 1053213379Shselasky if (i == XHCI_MAX_EVENTS) { 1054213379Shselasky i = 0; 1055213379Shselasky j ^= 1; 1056213379Shselasky 1057213379Shselasky /* check for timeout */ 1058213379Shselasky if (!--t) 1059213379Shselasky break; 1060213379Shselasky } 1061213379Shselasky } 1062213379Shselasky 1063213379Shselasky sc->sc_event_idx = i; 1064213379Shselasky sc->sc_event_ccs = j; 1065213379Shselasky 1066213379Shselasky /* 1067213379Shselasky * NOTE: The Event Ring Dequeue Pointer Register is 64-bit 1068213379Shselasky * latched. That means to activate the register we need to 1069213379Shselasky * write both the low and high double word of the 64-bit 1070213379Shselasky * register. 1071213379Shselasky */ 1072213379Shselasky 1073213379Shselasky addr = (uint32_t)buf_res.physaddr; 1074213379Shselasky addr += (uintptr_t)&((struct xhci_hw_root *)0)->hwr_events[i]; 1075213379Shselasky 1076213379Shselasky /* try to clear busy bit */ 1077213379Shselasky addr |= XHCI_ERDP_LO_BUSY; 1078213379Shselasky 1079213379Shselasky XWRITE4(sc, runt, XHCI_ERDP_LO(0), (uint32_t)addr); 1080213379Shselasky XWRITE4(sc, runt, XHCI_ERDP_HI(0), (uint32_t)(addr >> 32)); 1081255965Shselasky 1082255965Shselasky return (retval); 1083213379Shselasky} 1084213379Shselasky 1085213379Shselaskystatic usb_error_t 1086213379Shselaskyxhci_do_command(struct xhci_softc *sc, struct xhci_trb *trb, 1087213379Shselasky uint16_t timeout_ms) 1088213379Shselasky{ 1089213379Shselasky struct usb_page_search buf_res; 1090213379Shselasky struct xhci_hw_root *phwr; 1091213379Shselasky uint64_t addr; 1092213379Shselasky uint32_t temp; 1093213379Shselasky uint8_t i; 1094213379Shselasky uint8_t j; 1095259602Shselasky uint8_t timeout = 0; 1096213379Shselasky int err; 1097213379Shselasky 1098213379Shselasky XHCI_CMD_ASSERT_LOCKED(sc); 1099213379Shselasky 1100213379Shselasky /* get hardware root structure */ 1101213379Shselasky 1102213379Shselasky usbd_get_page(&sc->sc_hw.root_pc, 0, &buf_res); 1103213379Shselasky 1104213379Shselasky phwr = buf_res.buffer; 1105213379Shselasky 1106213379Shselasky /* Queue command */ 1107213379Shselasky 1108213379Shselasky USB_BUS_LOCK(&sc->sc_bus); 1109259602Shselaskyretry: 1110213379Shselasky i = sc->sc_command_idx; 1111213379Shselasky j = sc->sc_command_ccs; 1112213379Shselasky 1113213379Shselasky DPRINTFN(10, "command[%u] = %u (0x%016llx, 0x%08lx, 0x%08lx)\n", 1114213379Shselasky i, XHCI_TRB_3_TYPE_GET(le32toh(trb->dwTrb3)), 1115213379Shselasky (long long)le64toh(trb->qwTrb0), 1116213379Shselasky (long)le32toh(trb->dwTrb2), 1117213379Shselasky (long)le32toh(trb->dwTrb3)); 1118213379Shselasky 1119213379Shselasky phwr->hwr_commands[i].qwTrb0 = trb->qwTrb0; 1120213379Shselasky phwr->hwr_commands[i].dwTrb2 = trb->dwTrb2; 1121213379Shselasky 1122213379Shselasky usb_pc_cpu_flush(&sc->sc_hw.root_pc); 1123213379Shselasky 1124213379Shselasky temp = trb->dwTrb3; 1125213379Shselasky 1126213379Shselasky if (j) 1127213379Shselasky temp |= htole32(XHCI_TRB_3_CYCLE_BIT); 1128213379Shselasky else 1129213379Shselasky temp &= ~htole32(XHCI_TRB_3_CYCLE_BIT); 1130213379Shselasky 1131213379Shselasky temp &= ~htole32(XHCI_TRB_3_TC_BIT); 1132213379Shselasky 1133213379Shselasky phwr->hwr_commands[i].dwTrb3 = temp; 1134213379Shselasky 1135213379Shselasky usb_pc_cpu_flush(&sc->sc_hw.root_pc); 1136213379Shselasky 1137213379Shselasky addr = buf_res.physaddr; 1138213379Shselasky addr += (uintptr_t)&((struct xhci_hw_root *)0)->hwr_commands[i]; 1139213379Shselasky 1140213379Shselasky sc->sc_cmd_addr = htole64(addr); 1141213379Shselasky 1142213379Shselasky i++; 1143213379Shselasky 1144213379Shselasky if (i == (XHCI_MAX_COMMANDS - 1)) { 1145213379Shselasky 1146213379Shselasky if (j) { 1147213379Shselasky temp = htole32(XHCI_TRB_3_TC_BIT | 1148213379Shselasky XHCI_TRB_3_TYPE_SET(XHCI_TRB_TYPE_LINK) | 1149213379Shselasky XHCI_TRB_3_CYCLE_BIT); 1150213379Shselasky } else { 1151213379Shselasky temp = htole32(XHCI_TRB_3_TC_BIT | 1152213379Shselasky XHCI_TRB_3_TYPE_SET(XHCI_TRB_TYPE_LINK)); 1153213379Shselasky } 1154213379Shselasky 1155213379Shselasky phwr->hwr_commands[i].dwTrb3 = temp; 1156213379Shselasky 1157213379Shselasky usb_pc_cpu_flush(&sc->sc_hw.root_pc); 1158213379Shselasky 1159213379Shselasky i = 0; 1160213379Shselasky j ^= 1; 1161213379Shselasky } 1162213379Shselasky 1163213379Shselasky sc->sc_command_idx = i; 1164213379Shselasky sc->sc_command_ccs = j; 1165213379Shselasky 1166213379Shselasky XWRITE4(sc, door, XHCI_DOORBELL(0), 0); 1167213379Shselasky 1168213379Shselasky err = cv_timedwait(&sc->sc_cmd_cv, &sc->sc_bus.bus_mtx, 1169213379Shselasky USB_MS_TO_TICKS(timeout_ms)); 1170213379Shselasky 1171255965Shselasky /* 1172255965Shselasky * In some error cases event interrupts are not generated. 1173255965Shselasky * Poll one time to see if the command has completed. 1174255965Shselasky */ 1175255965Shselasky if (err != 0 && xhci_interrupt_poll(sc) != 0) { 1176255965Shselasky DPRINTF("Command was completed when polling\n"); 1177255965Shselasky err = 0; 1178255965Shselasky } 1179255965Shselasky if (err != 0) { 1180259602Shselasky DPRINTF("Command timeout!\n"); 1181257107Shselasky /* 1182259602Shselasky * After some weeks of continuous operation, it has 1183259602Shselasky * been observed that the ASMedia Technology, ASM1042 1184259602Shselasky * SuperSpeed USB Host Controller can suddenly stop 1185259602Shselasky * accepting commands via the command queue. Try to 1186259602Shselasky * first reset the command queue. If that fails do a 1187259602Shselasky * host controller reset. 1188257107Shselasky */ 1189259602Shselasky if (timeout == 0 && 1190259602Shselasky xhci_reset_command_queue_locked(sc) == 0) { 1191264337Shselasky temp = le32toh(trb->dwTrb3); 1192264337Shselasky 1193264337Shselasky /* 1194264337Shselasky * Avoid infinite XHCI reset loops if the set 1195264337Shselasky * address command fails to respond due to a 1196264337Shselasky * non-enumerating device: 1197264337Shselasky */ 1198264337Shselasky if (XHCI_TRB_3_TYPE_GET(temp) == XHCI_TRB_TYPE_ADDRESS_DEVICE && 1199264337Shselasky (temp & XHCI_TRB_3_BSR_BIT) == 0) { 1200264337Shselasky DPRINTF("Set address timeout\n"); 1201264337Shselasky } else { 1202264337Shselasky timeout = 1; 1203264337Shselasky goto retry; 1204264337Shselasky } 1205259602Shselasky } else { 1206259602Shselasky DPRINTF("Controller reset!\n"); 1207259602Shselasky usb_bus_reset_async_locked(&sc->sc_bus); 1208257107Shselasky } 1209213379Shselasky err = USB_ERR_TIMEOUT; 1210213379Shselasky trb->dwTrb2 = 0; 1211213379Shselasky trb->dwTrb3 = 0; 1212213379Shselasky } else { 1213213379Shselasky temp = le32toh(sc->sc_cmd_result[0]); 1214213379Shselasky if (XHCI_TRB_2_ERROR_GET(temp) != XHCI_TRB_ERROR_SUCCESS) 1215213379Shselasky err = USB_ERR_IOERROR; 1216213379Shselasky 1217213379Shselasky trb->dwTrb2 = sc->sc_cmd_result[0]; 1218213379Shselasky trb->dwTrb3 = sc->sc_cmd_result[1]; 1219213379Shselasky } 1220213379Shselasky 1221213379Shselasky USB_BUS_UNLOCK(&sc->sc_bus); 1222213379Shselasky 1223213379Shselasky return (err); 1224213379Shselasky} 1225213379Shselasky 1226213379Shselasky#if 0 1227213379Shselaskystatic usb_error_t 1228213379Shselaskyxhci_cmd_nop(struct xhci_softc *sc) 1229213379Shselasky{ 1230213379Shselasky struct xhci_trb trb; 1231213379Shselasky uint32_t temp; 1232213379Shselasky 1233213379Shselasky DPRINTF("\n"); 1234213379Shselasky 1235213379Shselasky trb.qwTrb0 = 0; 1236213379Shselasky trb.dwTrb2 = 0; 1237213379Shselasky temp = XHCI_TRB_3_TYPE_SET(XHCI_TRB_TYPE_NOOP); 1238213379Shselasky 1239213379Shselasky trb.dwTrb3 = htole32(temp); 1240213379Shselasky 1241229084Shselasky return (xhci_do_command(sc, &trb, 100 /* ms */)); 1242213379Shselasky} 1243213379Shselasky#endif 1244213379Shselasky 1245213379Shselaskystatic usb_error_t 1246213379Shselaskyxhci_cmd_enable_slot(struct xhci_softc *sc, uint8_t *pslot) 1247213379Shselasky{ 1248213379Shselasky struct xhci_trb trb; 1249213379Shselasky uint32_t temp; 1250213379Shselasky usb_error_t err; 1251213379Shselasky 1252213379Shselasky DPRINTF("\n"); 1253213379Shselasky 1254213379Shselasky trb.qwTrb0 = 0; 1255213379Shselasky trb.dwTrb2 = 0; 1256213379Shselasky trb.dwTrb3 = htole32(XHCI_TRB_3_TYPE_SET(XHCI_TRB_TYPE_ENABLE_SLOT)); 1257213379Shselasky 1258229084Shselasky err = xhci_do_command(sc, &trb, 100 /* ms */); 1259213379Shselasky if (err) 1260213379Shselasky goto done; 1261213379Shselasky 1262213379Shselasky temp = le32toh(trb.dwTrb3); 1263213379Shselasky 1264213379Shselasky *pslot = XHCI_TRB_3_SLOT_GET(temp); 1265213379Shselasky 1266213379Shselaskydone: 1267213379Shselasky return (err); 1268213379Shselasky} 1269213379Shselasky 1270213379Shselaskystatic usb_error_t 1271213379Shselaskyxhci_cmd_disable_slot(struct xhci_softc *sc, uint8_t slot_id) 1272213379Shselasky{ 1273213379Shselasky struct xhci_trb trb; 1274213379Shselasky uint32_t temp; 1275213379Shselasky 1276213379Shselasky DPRINTF("\n"); 1277213379Shselasky 1278213379Shselasky trb.qwTrb0 = 0; 1279213379Shselasky trb.dwTrb2 = 0; 1280213379Shselasky temp = XHCI_TRB_3_TYPE_SET(XHCI_TRB_TYPE_DISABLE_SLOT) | 1281213379Shselasky XHCI_TRB_3_SLOT_SET(slot_id); 1282213379Shselasky 1283213379Shselasky trb.dwTrb3 = htole32(temp); 1284213379Shselasky 1285229084Shselasky return (xhci_do_command(sc, &trb, 100 /* ms */)); 1286213379Shselasky} 1287213379Shselasky 1288213379Shselaskystatic usb_error_t 1289213379Shselaskyxhci_cmd_set_address(struct xhci_softc *sc, uint64_t input_ctx, 1290213379Shselasky uint8_t bsr, uint8_t slot_id) 1291213379Shselasky{ 1292213379Shselasky struct xhci_trb trb; 1293213379Shselasky uint32_t temp; 1294213379Shselasky 1295213379Shselasky DPRINTF("\n"); 1296213379Shselasky 1297213379Shselasky trb.qwTrb0 = htole64(input_ctx); 1298213379Shselasky trb.dwTrb2 = 0; 1299213379Shselasky temp = XHCI_TRB_3_TYPE_SET(XHCI_TRB_TYPE_ADDRESS_DEVICE) | 1300213379Shselasky XHCI_TRB_3_SLOT_SET(slot_id); 1301213379Shselasky 1302213379Shselasky if (bsr) 1303213379Shselasky temp |= XHCI_TRB_3_BSR_BIT; 1304213379Shselasky 1305213379Shselasky trb.dwTrb3 = htole32(temp); 1306213379Shselasky 1307213379Shselasky return (xhci_do_command(sc, &trb, 500 /* ms */)); 1308213379Shselasky} 1309213379Shselasky 1310213379Shselaskystatic usb_error_t 1311213379Shselaskyxhci_set_address(struct usb_device *udev, struct mtx *mtx, uint16_t address) 1312213379Shselasky{ 1313213379Shselasky struct usb_page_search buf_inp; 1314213379Shselasky struct usb_page_search buf_dev; 1315213379Shselasky struct xhci_softc *sc = XHCI_BUS2SC(udev->bus); 1316213379Shselasky struct xhci_hw_dev *hdev; 1317213379Shselasky struct xhci_dev_ctx *pdev; 1318213379Shselasky struct xhci_endpoint_ext *pepext; 1319217374Shselasky uint32_t temp; 1320213379Shselasky uint16_t mps; 1321213379Shselasky usb_error_t err; 1322213379Shselasky uint8_t index; 1323213379Shselasky 1324213379Shselasky /* the root HUB case is not handled here */ 1325213379Shselasky if (udev->parent_hub == NULL) 1326213379Shselasky return (USB_ERR_INVAL); 1327213379Shselasky 1328213379Shselasky index = udev->controller_slot_id; 1329213379Shselasky 1330213379Shselasky hdev = &sc->sc_hw.devs[index]; 1331213379Shselasky 1332213379Shselasky if (mtx != NULL) 1333213379Shselasky mtx_unlock(mtx); 1334213379Shselasky 1335213379Shselasky XHCI_CMD_LOCK(sc); 1336213379Shselasky 1337213379Shselasky switch (hdev->state) { 1338213379Shselasky case XHCI_ST_DEFAULT: 1339213379Shselasky case XHCI_ST_ENABLED: 1340213379Shselasky 1341213379Shselasky hdev->state = XHCI_ST_ENABLED; 1342213379Shselasky 1343213379Shselasky /* set configure mask to slot and EP0 */ 1344213379Shselasky xhci_configure_mask(udev, 3, 0); 1345213379Shselasky 1346213379Shselasky /* configure input slot context structure */ 1347213379Shselasky err = xhci_configure_device(udev); 1348213379Shselasky 1349213379Shselasky if (err != 0) { 1350213379Shselasky DPRINTF("Could not configure device\n"); 1351213379Shselasky break; 1352213379Shselasky } 1353213379Shselasky 1354213379Shselasky /* configure input endpoint context structure */ 1355213379Shselasky switch (udev->speed) { 1356213379Shselasky case USB_SPEED_LOW: 1357213379Shselasky case USB_SPEED_FULL: 1358213379Shselasky mps = 8; 1359213379Shselasky break; 1360213379Shselasky case USB_SPEED_HIGH: 1361213379Shselasky mps = 64; 1362213379Shselasky break; 1363213379Shselasky default: 1364213379Shselasky mps = 512; 1365213379Shselasky break; 1366213379Shselasky } 1367213379Shselasky 1368213379Shselasky pepext = xhci_get_endpoint_ext(udev, 1369213379Shselasky &udev->ctrl_ep_desc); 1370213379Shselasky err = xhci_configure_endpoint(udev, 1371213379Shselasky &udev->ctrl_ep_desc, pepext->physaddr, 1372213379Shselasky 0, 1, 1, 0, mps, mps); 1373213379Shselasky 1374213379Shselasky if (err != 0) { 1375213379Shselasky DPRINTF("Could not configure default endpoint\n"); 1376213379Shselasky break; 1377213379Shselasky } 1378213379Shselasky 1379213379Shselasky /* execute set address command */ 1380213379Shselasky usbd_get_page(&hdev->input_pc, 0, &buf_inp); 1381213379Shselasky 1382213379Shselasky err = xhci_cmd_set_address(sc, buf_inp.physaddr, 1383213379Shselasky (address == 0), index); 1384213379Shselasky 1385213379Shselasky if (err != 0) { 1386255965Shselasky temp = le32toh(sc->sc_cmd_result[0]); 1387255965Shselasky if (address == 0 && sc->sc_port_route != NULL && 1388255965Shselasky XHCI_TRB_2_ERROR_GET(temp) == 1389255965Shselasky XHCI_TRB_ERROR_PARAMETER) { 1390255965Shselasky /* LynxPoint XHCI - ports are not switchable */ 1391255965Shselasky /* Un-route all ports from the XHCI */ 1392255965Shselasky sc->sc_port_route(sc->sc_bus.parent, 0, ~0); 1393255965Shselasky } 1394213379Shselasky DPRINTF("Could not set address " 1395213379Shselasky "for slot %u.\n", index); 1396213379Shselasky if (address != 0) 1397213379Shselasky break; 1398213379Shselasky } 1399213379Shselasky 1400213379Shselasky /* update device address to new value */ 1401213379Shselasky 1402213379Shselasky usbd_get_page(&hdev->device_pc, 0, &buf_dev); 1403213379Shselasky pdev = buf_dev.buffer; 1404213379Shselasky usb_pc_cpu_invalidate(&hdev->device_pc); 1405213379Shselasky 1406217374Shselasky temp = xhci_ctx_get_le32(sc, &pdev->ctx_slot.dwSctx3); 1407217374Shselasky udev->address = XHCI_SCTX_3_DEV_ADDR_GET(temp); 1408217374Shselasky 1409213379Shselasky /* update device state to new value */ 1410213379Shselasky 1411213379Shselasky if (address != 0) 1412213379Shselasky hdev->state = XHCI_ST_ADDRESSED; 1413213379Shselasky else 1414213379Shselasky hdev->state = XHCI_ST_DEFAULT; 1415213379Shselasky break; 1416213379Shselasky 1417213379Shselasky default: 1418213379Shselasky DPRINTF("Wrong state for set address.\n"); 1419213379Shselasky err = USB_ERR_IOERROR; 1420213379Shselasky break; 1421213379Shselasky } 1422213379Shselasky XHCI_CMD_UNLOCK(sc); 1423213379Shselasky 1424213379Shselasky if (mtx != NULL) 1425213379Shselasky mtx_lock(mtx); 1426213379Shselasky 1427213379Shselasky return (err); 1428213379Shselasky} 1429213379Shselasky 1430213379Shselaskystatic usb_error_t 1431213379Shselaskyxhci_cmd_configure_ep(struct xhci_softc *sc, uint64_t input_ctx, 1432213379Shselasky uint8_t deconfigure, uint8_t slot_id) 1433213379Shselasky{ 1434213379Shselasky struct xhci_trb trb; 1435213379Shselasky uint32_t temp; 1436213379Shselasky 1437213379Shselasky DPRINTF("\n"); 1438213379Shselasky 1439213379Shselasky trb.qwTrb0 = htole64(input_ctx); 1440213379Shselasky trb.dwTrb2 = 0; 1441213379Shselasky temp = XHCI_TRB_3_TYPE_SET(XHCI_TRB_TYPE_CONFIGURE_EP) | 1442213379Shselasky XHCI_TRB_3_SLOT_SET(slot_id); 1443213379Shselasky 1444213379Shselasky if (deconfigure) 1445213379Shselasky temp |= XHCI_TRB_3_DCEP_BIT; 1446213379Shselasky 1447213379Shselasky trb.dwTrb3 = htole32(temp); 1448213379Shselasky 1449229084Shselasky return (xhci_do_command(sc, &trb, 100 /* ms */)); 1450213379Shselasky} 1451213379Shselasky 1452213379Shselaskystatic usb_error_t 1453213379Shselaskyxhci_cmd_evaluate_ctx(struct xhci_softc *sc, uint64_t input_ctx, 1454213379Shselasky uint8_t slot_id) 1455213379Shselasky{ 1456213379Shselasky struct xhci_trb trb; 1457213379Shselasky uint32_t temp; 1458213379Shselasky 1459213379Shselasky DPRINTF("\n"); 1460213379Shselasky 1461213379Shselasky trb.qwTrb0 = htole64(input_ctx); 1462213379Shselasky trb.dwTrb2 = 0; 1463213379Shselasky temp = XHCI_TRB_3_TYPE_SET(XHCI_TRB_TYPE_EVALUATE_CTX) | 1464213379Shselasky XHCI_TRB_3_SLOT_SET(slot_id); 1465213379Shselasky trb.dwTrb3 = htole32(temp); 1466213379Shselasky 1467229084Shselasky return (xhci_do_command(sc, &trb, 100 /* ms */)); 1468213379Shselasky} 1469213379Shselasky 1470213379Shselaskystatic usb_error_t 1471213379Shselaskyxhci_cmd_reset_ep(struct xhci_softc *sc, uint8_t preserve, 1472213379Shselasky uint8_t ep_id, uint8_t slot_id) 1473213379Shselasky{ 1474213379Shselasky struct xhci_trb trb; 1475213379Shselasky uint32_t temp; 1476213379Shselasky 1477213379Shselasky DPRINTF("\n"); 1478213379Shselasky 1479213379Shselasky trb.qwTrb0 = 0; 1480213379Shselasky trb.dwTrb2 = 0; 1481213379Shselasky temp = XHCI_TRB_3_TYPE_SET(XHCI_TRB_TYPE_RESET_EP) | 1482213379Shselasky XHCI_TRB_3_SLOT_SET(slot_id) | 1483213379Shselasky XHCI_TRB_3_EP_SET(ep_id); 1484213379Shselasky 1485213379Shselasky if (preserve) 1486213379Shselasky temp |= XHCI_TRB_3_PRSV_BIT; 1487213379Shselasky 1488213379Shselasky trb.dwTrb3 = htole32(temp); 1489213379Shselasky 1490229084Shselasky return (xhci_do_command(sc, &trb, 100 /* ms */)); 1491213379Shselasky} 1492213379Shselasky 1493213379Shselaskystatic usb_error_t 1494213379Shselaskyxhci_cmd_set_tr_dequeue_ptr(struct xhci_softc *sc, uint64_t dequeue_ptr, 1495213379Shselasky uint16_t stream_id, uint8_t ep_id, uint8_t slot_id) 1496213379Shselasky{ 1497213379Shselasky struct xhci_trb trb; 1498213379Shselasky uint32_t temp; 1499213379Shselasky 1500213379Shselasky DPRINTF("\n"); 1501213379Shselasky 1502213379Shselasky trb.qwTrb0 = htole64(dequeue_ptr); 1503213379Shselasky 1504213379Shselasky temp = XHCI_TRB_2_STREAM_SET(stream_id); 1505213379Shselasky trb.dwTrb2 = htole32(temp); 1506213379Shselasky 1507213379Shselasky temp = XHCI_TRB_3_TYPE_SET(XHCI_TRB_TYPE_SET_TR_DEQUEUE) | 1508213379Shselasky XHCI_TRB_3_SLOT_SET(slot_id) | 1509213379Shselasky XHCI_TRB_3_EP_SET(ep_id); 1510213379Shselasky trb.dwTrb3 = htole32(temp); 1511213379Shselasky 1512229084Shselasky return (xhci_do_command(sc, &trb, 100 /* ms */)); 1513213379Shselasky} 1514213379Shselasky 1515213379Shselaskystatic usb_error_t 1516213379Shselaskyxhci_cmd_stop_ep(struct xhci_softc *sc, uint8_t suspend, 1517213379Shselasky uint8_t ep_id, uint8_t slot_id) 1518213379Shselasky{ 1519213379Shselasky struct xhci_trb trb; 1520213379Shselasky uint32_t temp; 1521213379Shselasky 1522213379Shselasky DPRINTF("\n"); 1523213379Shselasky 1524213379Shselasky trb.qwTrb0 = 0; 1525213379Shselasky trb.dwTrb2 = 0; 1526213379Shselasky temp = XHCI_TRB_3_TYPE_SET(XHCI_TRB_TYPE_STOP_EP) | 1527213379Shselasky XHCI_TRB_3_SLOT_SET(slot_id) | 1528213379Shselasky XHCI_TRB_3_EP_SET(ep_id); 1529213379Shselasky 1530213379Shselasky if (suspend) 1531213379Shselasky temp |= XHCI_TRB_3_SUSP_EP_BIT; 1532213379Shselasky 1533213379Shselasky trb.dwTrb3 = htole32(temp); 1534213379Shselasky 1535229084Shselasky return (xhci_do_command(sc, &trb, 100 /* ms */)); 1536213379Shselasky} 1537213379Shselasky 1538213379Shselaskystatic usb_error_t 1539213379Shselaskyxhci_cmd_reset_dev(struct xhci_softc *sc, uint8_t slot_id) 1540213379Shselasky{ 1541213379Shselasky struct xhci_trb trb; 1542213379Shselasky uint32_t temp; 1543213379Shselasky 1544213379Shselasky DPRINTF("\n"); 1545213379Shselasky 1546213379Shselasky trb.qwTrb0 = 0; 1547213379Shselasky trb.dwTrb2 = 0; 1548213379Shselasky temp = XHCI_TRB_3_TYPE_SET(XHCI_TRB_TYPE_RESET_DEVICE) | 1549213379Shselasky XHCI_TRB_3_SLOT_SET(slot_id); 1550213379Shselasky 1551213379Shselasky trb.dwTrb3 = htole32(temp); 1552213379Shselasky 1553229084Shselasky return (xhci_do_command(sc, &trb, 100 /* ms */)); 1554213379Shselasky} 1555213379Shselasky 1556213379Shselasky/*------------------------------------------------------------------------* 1557213379Shselasky * xhci_interrupt - XHCI interrupt handler 1558213379Shselasky *------------------------------------------------------------------------*/ 1559213379Shselaskyvoid 1560213379Shselaskyxhci_interrupt(struct xhci_softc *sc) 1561213379Shselasky{ 1562213379Shselasky uint32_t status; 1563261097Shselasky uint32_t temp; 1564213379Shselasky 1565213379Shselasky USB_BUS_LOCK(&sc->sc_bus); 1566213379Shselasky 1567213379Shselasky status = XREAD4(sc, oper, XHCI_USBSTS); 1568213379Shselasky 1569261097Shselasky /* acknowledge interrupts, if any */ 1570261097Shselasky if (status != 0) { 1571261097Shselasky XWRITE4(sc, oper, XHCI_USBSTS, status); 1572261097Shselasky DPRINTFN(16, "real interrupt (status=0x%08x)\n", status); 1573261097Shselasky } 1574213379Shselasky 1575261097Shselasky temp = XREAD4(sc, runt, XHCI_IMAN(0)); 1576213379Shselasky 1577261097Shselasky /* force clearing of pending interrupts */ 1578261097Shselasky if (temp & XHCI_IMAN_INTR_PEND) 1579261097Shselasky XWRITE4(sc, runt, XHCI_IMAN(0), temp); 1580245732Shselasky 1581261097Shselasky /* check for event(s) */ 1582261097Shselasky xhci_interrupt_poll(sc); 1583213379Shselasky 1584245732Shselasky if (status & (XHCI_STS_PCD | XHCI_STS_HCH | 1585245732Shselasky XHCI_STS_HSE | XHCI_STS_HCE)) { 1586213379Shselasky 1587213379Shselasky if (status & XHCI_STS_PCD) { 1588213379Shselasky xhci_root_intr(sc); 1589213379Shselasky } 1590213379Shselasky 1591213379Shselasky if (status & XHCI_STS_HCH) { 1592213379Shselasky printf("%s: host controller halted\n", 1593213379Shselasky __FUNCTION__); 1594213379Shselasky } 1595213379Shselasky 1596213379Shselasky if (status & XHCI_STS_HSE) { 1597213379Shselasky printf("%s: host system error\n", 1598213379Shselasky __FUNCTION__); 1599213379Shselasky } 1600213379Shselasky 1601213379Shselasky if (status & XHCI_STS_HCE) { 1602213379Shselasky printf("%s: host controller error\n", 1603213379Shselasky __FUNCTION__); 1604213379Shselasky } 1605213379Shselasky } 1606213379Shselasky USB_BUS_UNLOCK(&sc->sc_bus); 1607213379Shselasky} 1608213379Shselasky 1609213379Shselasky/*------------------------------------------------------------------------* 1610213379Shselasky * xhci_timeout - XHCI timeout handler 1611213379Shselasky *------------------------------------------------------------------------*/ 1612213379Shselaskystatic void 1613213379Shselaskyxhci_timeout(void *arg) 1614213379Shselasky{ 1615213379Shselasky struct usb_xfer *xfer = arg; 1616213379Shselasky 1617213379Shselasky DPRINTF("xfer=%p\n", xfer); 1618213379Shselasky 1619213379Shselasky USB_BUS_LOCK_ASSERT(xfer->xroot->bus, MA_OWNED); 1620213379Shselasky 1621213379Shselasky /* transfer is transferred */ 1622213379Shselasky xhci_device_done(xfer, USB_ERR_TIMEOUT); 1623213379Shselasky} 1624213379Shselasky 1625213379Shselaskystatic void 1626213379Shselaskyxhci_do_poll(struct usb_bus *bus) 1627213379Shselasky{ 1628213379Shselasky struct xhci_softc *sc = XHCI_BUS2SC(bus); 1629213379Shselasky 1630213379Shselasky USB_BUS_LOCK(&sc->sc_bus); 1631213379Shselasky xhci_interrupt_poll(sc); 1632213379Shselasky USB_BUS_UNLOCK(&sc->sc_bus); 1633213379Shselasky} 1634213379Shselasky 1635213379Shselaskystatic void 1636213379Shselaskyxhci_setup_generic_chain_sub(struct xhci_std_temp *temp) 1637213379Shselasky{ 1638213379Shselasky struct usb_page_search buf_res; 1639213379Shselasky struct xhci_td *td; 1640213379Shselasky struct xhci_td *td_next; 1641213379Shselasky struct xhci_td *td_alt_next; 1642251614Shselasky struct xhci_td *td_first; 1643213379Shselasky uint32_t buf_offset; 1644213379Shselasky uint32_t average; 1645213379Shselasky uint32_t len_old; 1646251614Shselasky uint32_t npkt_off; 1647213379Shselasky uint32_t dword; 1648213379Shselasky uint8_t shortpkt_old; 1649213379Shselasky uint8_t precompute; 1650213379Shselasky uint8_t x; 1651213379Shselasky 1652213379Shselasky td_alt_next = NULL; 1653213379Shselasky buf_offset = 0; 1654213379Shselasky shortpkt_old = temp->shortpkt; 1655213379Shselasky len_old = temp->len; 1656251614Shselasky npkt_off = 0; 1657213379Shselasky precompute = 1; 1658213379Shselasky 1659213379Shselaskyrestart: 1660213379Shselasky 1661213379Shselasky td = temp->td; 1662251614Shselasky td_next = td_first = temp->td_next; 1663213379Shselasky 1664213379Shselasky while (1) { 1665213379Shselasky 1666213379Shselasky if (temp->len == 0) { 1667213379Shselasky 1668213379Shselasky if (temp->shortpkt) 1669213379Shselasky break; 1670213379Shselasky 1671213379Shselasky /* send a Zero Length Packet, ZLP, last */ 1672213379Shselasky 1673213379Shselasky temp->shortpkt = 1; 1674213379Shselasky average = 0; 1675213379Shselasky 1676213379Shselasky } else { 1677213379Shselasky 1678213379Shselasky average = temp->average; 1679213379Shselasky 1680213379Shselasky if (temp->len < average) { 1681213379Shselasky if (temp->len % temp->max_packet_size) { 1682213379Shselasky temp->shortpkt = 1; 1683213379Shselasky } 1684213379Shselasky average = temp->len; 1685213379Shselasky } 1686213379Shselasky } 1687213379Shselasky 1688213379Shselasky if (td_next == NULL) 1689213379Shselasky panic("%s: out of XHCI transfer descriptors!", __FUNCTION__); 1690213379Shselasky 1691213379Shselasky /* get next TD */ 1692213379Shselasky 1693213379Shselasky td = td_next; 1694213379Shselasky td_next = td->obj_next; 1695213379Shselasky 1696213379Shselasky /* check if we are pre-computing */ 1697213379Shselasky 1698213379Shselasky if (precompute) { 1699213379Shselasky 1700213379Shselasky /* update remaining length */ 1701213379Shselasky 1702213379Shselasky temp->len -= average; 1703213379Shselasky 1704213379Shselasky continue; 1705213379Shselasky } 1706213379Shselasky /* fill out current TD */ 1707213379Shselasky 1708213379Shselasky td->len = average; 1709213379Shselasky td->remainder = 0; 1710213379Shselasky td->status = 0; 1711213379Shselasky 1712213379Shselasky /* update remaining length */ 1713213379Shselasky 1714213379Shselasky temp->len -= average; 1715213379Shselasky 1716213379Shselasky /* reset TRB index */ 1717213379Shselasky 1718213379Shselasky x = 0; 1719213379Shselasky 1720213379Shselasky if (temp->trb_type == XHCI_TRB_TYPE_SETUP_STAGE) { 1721213379Shselasky /* immediate data */ 1722213379Shselasky 1723213379Shselasky if (average > 8) 1724213379Shselasky average = 8; 1725213379Shselasky 1726213379Shselasky td->td_trb[0].qwTrb0 = 0; 1727213379Shselasky 1728213379Shselasky usbd_copy_out(temp->pc, temp->offset + buf_offset, 1729213379Shselasky (uint8_t *)(uintptr_t)&td->td_trb[0].qwTrb0, 1730213379Shselasky average); 1731213379Shselasky 1732213379Shselasky dword = XHCI_TRB_2_BYTES_SET(8) | 1733213379Shselasky XHCI_TRB_2_TDSZ_SET(0) | 1734213379Shselasky XHCI_TRB_2_IRQ_SET(0); 1735213379Shselasky 1736213379Shselasky td->td_trb[0].dwTrb2 = htole32(dword); 1737213379Shselasky 1738213379Shselasky dword = XHCI_TRB_3_TYPE_SET(XHCI_TRB_TYPE_SETUP_STAGE) | 1739213379Shselasky XHCI_TRB_3_IDT_BIT | XHCI_TRB_3_CYCLE_BIT; 1740213379Shselasky 1741213379Shselasky /* check wLength */ 1742213379Shselasky if (td->td_trb[0].qwTrb0 & 1743213379Shselasky htole64(XHCI_TRB_0_WLENGTH_MASK)) { 1744262371Shselasky if (td->td_trb[0].qwTrb0 & 1745262371Shselasky htole64(XHCI_TRB_0_DIR_IN_MASK)) 1746213379Shselasky dword |= XHCI_TRB_3_TRT_IN; 1747213379Shselasky else 1748213379Shselasky dword |= XHCI_TRB_3_TRT_OUT; 1749213379Shselasky } 1750213379Shselasky 1751213379Shselasky td->td_trb[0].dwTrb3 = htole32(dword); 1752213379Shselasky#ifdef USB_DEBUG 1753213379Shselasky xhci_dump_trb(&td->td_trb[x]); 1754213379Shselasky#endif 1755213379Shselasky x++; 1756213379Shselasky 1757213379Shselasky } else do { 1758213379Shselasky 1759213379Shselasky uint32_t npkt; 1760213379Shselasky 1761213379Shselasky /* fill out buffer pointers */ 1762213379Shselasky 1763213379Shselasky if (average == 0) { 1764213379Shselasky memset(&buf_res, 0, sizeof(buf_res)); 1765213379Shselasky } else { 1766213379Shselasky usbd_get_page(temp->pc, temp->offset + 1767213379Shselasky buf_offset, &buf_res); 1768213379Shselasky 1769213379Shselasky /* get length to end of page */ 1770213379Shselasky if (buf_res.length > average) 1771213379Shselasky buf_res.length = average; 1772213379Shselasky 1773213379Shselasky /* check for maximum length */ 1774213379Shselasky if (buf_res.length > XHCI_TD_PAGE_SIZE) 1775213379Shselasky buf_res.length = XHCI_TD_PAGE_SIZE; 1776213379Shselasky 1777251614Shselasky npkt_off += buf_res.length; 1778253665Shselasky } 1779251614Shselasky 1780253665Shselasky /* setup npkt */ 1781253665Shselasky npkt = (len_old - npkt_off + temp->max_packet_size - 1) / 1782253665Shselasky temp->max_packet_size; 1783213379Shselasky 1784253665Shselasky if (npkt == 0) 1785253665Shselasky npkt = 1; 1786253665Shselasky else if (npkt > 31) 1787253665Shselasky npkt = 31; 1788213379Shselasky 1789213379Shselasky /* fill out TRB's */ 1790213379Shselasky td->td_trb[x].qwTrb0 = 1791213379Shselasky htole64((uint64_t)buf_res.physaddr); 1792213379Shselasky 1793213379Shselasky dword = 1794213379Shselasky XHCI_TRB_2_BYTES_SET(buf_res.length) | 1795213379Shselasky XHCI_TRB_2_TDSZ_SET(npkt) | 1796213379Shselasky XHCI_TRB_2_IRQ_SET(0); 1797213379Shselasky 1798213379Shselasky td->td_trb[x].dwTrb2 = htole32(dword); 1799213379Shselasky 1800251614Shselasky switch (temp->trb_type) { 1801251614Shselasky case XHCI_TRB_TYPE_ISOCH: 1802251614Shselasky dword = XHCI_TRB_3_CHAIN_BIT | XHCI_TRB_3_CYCLE_BIT | 1803251614Shselasky XHCI_TRB_3_TBC_SET(temp->tbc) | 1804251614Shselasky XHCI_TRB_3_TLBPC_SET(temp->tlbpc); 1805251614Shselasky if (td != td_first) { 1806251614Shselasky dword |= XHCI_TRB_3_TYPE_SET(XHCI_TRB_TYPE_NORMAL); 1807251614Shselasky } else if (temp->do_isoc_sync != 0) { 1808251614Shselasky temp->do_isoc_sync = 0; 1809251614Shselasky /* wait until "isoc_frame" */ 1810251614Shselasky dword |= XHCI_TRB_3_TYPE_SET(XHCI_TRB_TYPE_ISOCH) | 1811251614Shselasky XHCI_TRB_3_FRID_SET(temp->isoc_frame / 8); 1812251614Shselasky } else { 1813251614Shselasky /* start data transfer at next interval */ 1814251614Shselasky dword |= XHCI_TRB_3_TYPE_SET(XHCI_TRB_TYPE_ISOCH) | 1815251614Shselasky XHCI_TRB_3_ISO_SIA_BIT; 1816251614Shselasky } 1817251614Shselasky if (temp->direction == UE_DIR_IN) 1818266666Shselasky dword |= XHCI_TRB_3_ISP_BIT; 1819251614Shselasky break; 1820251614Shselasky case XHCI_TRB_TYPE_DATA_STAGE: 1821251614Shselasky dword = XHCI_TRB_3_CHAIN_BIT | XHCI_TRB_3_CYCLE_BIT | 1822266666Shselasky XHCI_TRB_3_TYPE_SET(XHCI_TRB_TYPE_DATA_STAGE); 1823251614Shselasky if (temp->direction == UE_DIR_IN) 1824251614Shselasky dword |= XHCI_TRB_3_DIR_IN | XHCI_TRB_3_ISP_BIT; 1825251614Shselasky break; 1826251614Shselasky case XHCI_TRB_TYPE_STATUS_STAGE: 1827251614Shselasky dword = XHCI_TRB_3_CHAIN_BIT | XHCI_TRB_3_CYCLE_BIT | 1828266666Shselasky XHCI_TRB_3_TYPE_SET(XHCI_TRB_TYPE_STATUS_STAGE); 1829251614Shselasky if (temp->direction == UE_DIR_IN) 1830251614Shselasky dword |= XHCI_TRB_3_DIR_IN; 1831251614Shselasky break; 1832251614Shselasky default: /* XHCI_TRB_TYPE_NORMAL */ 1833251614Shselasky dword = XHCI_TRB_3_CHAIN_BIT | XHCI_TRB_3_CYCLE_BIT | 1834266666Shselasky XHCI_TRB_3_TYPE_SET(XHCI_TRB_TYPE_NORMAL); 1835251614Shselasky if (temp->direction == UE_DIR_IN) 1836266666Shselasky dword |= XHCI_TRB_3_ISP_BIT; 1837251614Shselasky break; 1838213379Shselasky } 1839213379Shselasky td->td_trb[x].dwTrb3 = htole32(dword); 1840213379Shselasky 1841213379Shselasky average -= buf_res.length; 1842213379Shselasky buf_offset += buf_res.length; 1843213379Shselasky#ifdef USB_DEBUG 1844213379Shselasky xhci_dump_trb(&td->td_trb[x]); 1845213379Shselasky#endif 1846213379Shselasky x++; 1847213379Shselasky 1848213379Shselasky } while (average != 0); 1849213379Shselasky 1850213379Shselasky td->td_trb[x-1].dwTrb3 |= htole32(XHCI_TRB_3_IOC_BIT); 1851213379Shselasky 1852213379Shselasky /* store number of data TRB's */ 1853213379Shselasky 1854213379Shselasky td->ntrb = x; 1855213379Shselasky 1856213379Shselasky DPRINTF("NTRB=%u\n", x); 1857213379Shselasky 1858213379Shselasky /* fill out link TRB */ 1859213379Shselasky 1860213379Shselasky if (td_next != NULL) { 1861213379Shselasky /* link the current TD with the next one */ 1862213379Shselasky td->td_trb[x].qwTrb0 = htole64((uint64_t)td_next->td_self); 1863213379Shselasky DPRINTF("LINK=0x%08llx\n", (long long)td_next->td_self); 1864213379Shselasky } else { 1865213379Shselasky /* this field will get updated later */ 1866213379Shselasky DPRINTF("NOLINK\n"); 1867213379Shselasky } 1868213379Shselasky 1869213379Shselasky dword = XHCI_TRB_2_IRQ_SET(0); 1870213379Shselasky 1871213379Shselasky td->td_trb[x].dwTrb2 = htole32(dword); 1872213379Shselasky 1873213379Shselasky dword = XHCI_TRB_3_TYPE_SET(XHCI_TRB_TYPE_LINK) | 1874259606Shselasky XHCI_TRB_3_CYCLE_BIT | XHCI_TRB_3_IOC_BIT | 1875259606Shselasky /* 1876259606Shselasky * CHAIN-BIT: Ensure that a multi-TRB IN-endpoint 1877259606Shselasky * frame only receives a single short packet event 1878259606Shselasky * by setting the CHAIN bit in the LINK field. In 1879259606Shselasky * addition some XHCI controllers have problems 1880259606Shselasky * sending a ZLP unless the CHAIN-BIT is set in 1881259606Shselasky * the LINK TRB. 1882259606Shselasky */ 1883259606Shselasky XHCI_TRB_3_CHAIN_BIT; 1884213379Shselasky 1885213379Shselasky td->td_trb[x].dwTrb3 = htole32(dword); 1886213379Shselasky 1887213379Shselasky td->alt_next = td_alt_next; 1888213379Shselasky#ifdef USB_DEBUG 1889213379Shselasky xhci_dump_trb(&td->td_trb[x]); 1890213379Shselasky#endif 1891213379Shselasky usb_pc_cpu_flush(td->page_cache); 1892213379Shselasky } 1893213379Shselasky 1894213379Shselasky if (precompute) { 1895213379Shselasky precompute = 0; 1896213379Shselasky 1897213379Shselasky /* setup alt next pointer, if any */ 1898213379Shselasky if (temp->last_frame) { 1899213379Shselasky td_alt_next = NULL; 1900213379Shselasky } else { 1901213379Shselasky /* we use this field internally */ 1902213379Shselasky td_alt_next = td_next; 1903213379Shselasky } 1904213379Shselasky 1905213379Shselasky /* restore */ 1906213379Shselasky temp->shortpkt = shortpkt_old; 1907213379Shselasky temp->len = len_old; 1908213379Shselasky goto restart; 1909213379Shselasky } 1910213379Shselasky 1911251614Shselasky /* 1912251614Shselasky * Remove cycle bit from the first TRB if we are 1913251614Shselasky * stepping them: 1914251614Shselasky */ 1915251614Shselasky if (temp->step_td != 0) { 1916251614Shselasky td_first->td_trb[0].dwTrb3 &= ~htole32(XHCI_TRB_3_CYCLE_BIT); 1917251614Shselasky usb_pc_cpu_flush(td_first->page_cache); 1918251614Shselasky } 1919213379Shselasky 1920253665Shselasky /* clear TD SIZE to zero, hence this is the last TRB */ 1921259606Shselasky /* remove chain bit because this is the last data TRB in the chain */ 1922213379Shselasky td->td_trb[td->ntrb - 1].dwTrb2 &= ~htole32(XHCI_TRB_2_TDSZ_SET(15)); 1923213379Shselasky td->td_trb[td->ntrb - 1].dwTrb3 &= ~htole32(XHCI_TRB_3_CHAIN_BIT); 1924259606Shselasky /* remove CHAIN-BIT from last LINK TRB */ 1925259606Shselasky td->td_trb[td->ntrb].dwTrb3 &= ~htole32(XHCI_TRB_3_CHAIN_BIT); 1926213379Shselasky 1927213379Shselasky usb_pc_cpu_flush(td->page_cache); 1928213379Shselasky 1929213379Shselasky temp->td = td; 1930213379Shselasky temp->td_next = td_next; 1931213379Shselasky} 1932213379Shselasky 1933213379Shselaskystatic void 1934213379Shselaskyxhci_setup_generic_chain(struct usb_xfer *xfer) 1935213379Shselasky{ 1936213379Shselasky struct xhci_std_temp temp; 1937213379Shselasky struct xhci_td *td; 1938213379Shselasky uint32_t x; 1939213379Shselasky uint32_t y; 1940213379Shselasky uint8_t mult; 1941213379Shselasky 1942235001Shselasky temp.do_isoc_sync = 0; 1943213379Shselasky temp.step_td = 0; 1944213379Shselasky temp.tbc = 0; 1945213379Shselasky temp.tlbpc = 0; 1946213379Shselasky temp.average = xfer->max_hc_frame_size; 1947213379Shselasky temp.max_packet_size = xfer->max_packet_size; 1948213379Shselasky temp.sc = XHCI_BUS2SC(xfer->xroot->bus); 1949213379Shselasky temp.pc = NULL; 1950213379Shselasky temp.last_frame = 0; 1951213379Shselasky temp.offset = 0; 1952213379Shselasky temp.multishort = xfer->flags_int.isochronous_xfr || 1953213379Shselasky xfer->flags_int.control_xfr || 1954213379Shselasky xfer->flags_int.short_frames_ok; 1955213379Shselasky 1956213379Shselasky /* toggle the DMA set we are using */ 1957213379Shselasky xfer->flags_int.curr_dma_set ^= 1; 1958213379Shselasky 1959213379Shselasky /* get next DMA set */ 1960213379Shselasky td = xfer->td_start[xfer->flags_int.curr_dma_set]; 1961213379Shselasky 1962213379Shselasky temp.td = NULL; 1963213379Shselasky temp.td_next = td; 1964213379Shselasky 1965213379Shselasky xfer->td_transfer_first = td; 1966213379Shselasky xfer->td_transfer_cache = td; 1967213379Shselasky 1968213379Shselasky if (xfer->flags_int.isochronous_xfr) { 1969213379Shselasky uint8_t shift; 1970213379Shselasky 1971213379Shselasky /* compute multiplier for ISOCHRONOUS transfers */ 1972213379Shselasky mult = xfer->endpoint->ecomp ? 1973213379Shselasky (xfer->endpoint->ecomp->bmAttributes & 3) : 0; 1974213379Shselasky /* check for USB 2.0 multiplier */ 1975213379Shselasky if (mult == 0) { 1976213379Shselasky mult = (xfer->endpoint->edesc-> 1977213379Shselasky wMaxPacketSize[1] >> 3) & 3; 1978213379Shselasky } 1979213379Shselasky /* range check */ 1980213379Shselasky if (mult > 2) 1981213379Shselasky mult = 3; 1982213379Shselasky else 1983213379Shselasky mult++; 1984213379Shselasky 1985213379Shselasky x = XREAD4(temp.sc, runt, XHCI_MFINDEX); 1986213379Shselasky 1987213379Shselasky DPRINTF("MFINDEX=0x%08x\n", x); 1988213379Shselasky 1989213379Shselasky switch (usbd_get_speed(xfer->xroot->udev)) { 1990213379Shselasky case USB_SPEED_FULL: 1991213379Shselasky shift = 3; 1992213379Shselasky temp.isoc_delta = 8; /* 1ms */ 1993213379Shselasky x += temp.isoc_delta - 1; 1994213379Shselasky x &= ~(temp.isoc_delta - 1); 1995213379Shselasky break; 1996213379Shselasky default: 1997213379Shselasky shift = usbd_xfer_get_fps_shift(xfer); 1998213379Shselasky temp.isoc_delta = 1U << shift; 1999213379Shselasky x += temp.isoc_delta - 1; 2000213379Shselasky x &= ~(temp.isoc_delta - 1); 2001213379Shselasky /* simple frame load balancing */ 2002213379Shselasky x += xfer->endpoint->usb_uframe; 2003213379Shselasky break; 2004213379Shselasky } 2005213379Shselasky 2006213379Shselasky y = XHCI_MFINDEX_GET(x - xfer->endpoint->isoc_next); 2007213379Shselasky 2008213379Shselasky if ((xfer->endpoint->is_synced == 0) || 2009213379Shselasky (y < (xfer->nframes << shift)) || 2010213379Shselasky (XHCI_MFINDEX_GET(-y) >= (128 * 8))) { 2011213379Shselasky /* 2012213379Shselasky * If there is data underflow or the pipe 2013213379Shselasky * queue is empty we schedule the transfer a 2014213379Shselasky * few frames ahead of the current frame 2015213379Shselasky * position. Else two isochronous transfers 2016213379Shselasky * might overlap. 2017213379Shselasky */ 2018213379Shselasky xfer->endpoint->isoc_next = XHCI_MFINDEX_GET(x + (3 * 8)); 2019213379Shselasky xfer->endpoint->is_synced = 1; 2020235001Shselasky temp.do_isoc_sync = 1; 2021235001Shselasky 2022213379Shselasky DPRINTFN(3, "start next=%d\n", xfer->endpoint->isoc_next); 2023213379Shselasky } 2024213379Shselasky 2025213379Shselasky /* compute isochronous completion time */ 2026213379Shselasky 2027213379Shselasky y = XHCI_MFINDEX_GET(xfer->endpoint->isoc_next - (x & ~7)); 2028213379Shselasky 2029213379Shselasky xfer->isoc_time_complete = 2030213379Shselasky usb_isoc_time_expand(&temp.sc->sc_bus, x / 8) + 2031213379Shselasky (y / 8) + (((xfer->nframes << shift) + 7) / 8); 2032213379Shselasky 2033213379Shselasky x = 0; 2034213379Shselasky temp.isoc_frame = xfer->endpoint->isoc_next; 2035213379Shselasky temp.trb_type = XHCI_TRB_TYPE_ISOCH; 2036213379Shselasky 2037213379Shselasky xfer->endpoint->isoc_next += xfer->nframes << shift; 2038213379Shselasky 2039213379Shselasky } else if (xfer->flags_int.control_xfr) { 2040213379Shselasky 2041213379Shselasky /* check if we should prepend a setup message */ 2042213379Shselasky 2043213379Shselasky if (xfer->flags_int.control_hdr) { 2044213379Shselasky 2045213379Shselasky temp.len = xfer->frlengths[0]; 2046213379Shselasky temp.pc = xfer->frbuffers + 0; 2047213379Shselasky temp.shortpkt = temp.len ? 1 : 0; 2048213379Shselasky temp.trb_type = XHCI_TRB_TYPE_SETUP_STAGE; 2049213379Shselasky temp.direction = 0; 2050213379Shselasky 2051213379Shselasky /* check for last frame */ 2052213379Shselasky if (xfer->nframes == 1) { 2053213379Shselasky /* no STATUS stage yet, SETUP is last */ 2054213379Shselasky if (xfer->flags_int.control_act) 2055213379Shselasky temp.last_frame = 1; 2056213379Shselasky } 2057213379Shselasky 2058213379Shselasky xhci_setup_generic_chain_sub(&temp); 2059213379Shselasky } 2060213379Shselasky x = 1; 2061213379Shselasky mult = 1; 2062213379Shselasky temp.isoc_delta = 0; 2063213379Shselasky temp.isoc_frame = 0; 2064213379Shselasky temp.trb_type = XHCI_TRB_TYPE_DATA_STAGE; 2065213379Shselasky } else { 2066213379Shselasky x = 0; 2067213379Shselasky mult = 1; 2068213379Shselasky temp.isoc_delta = 0; 2069213379Shselasky temp.isoc_frame = 0; 2070213379Shselasky temp.trb_type = XHCI_TRB_TYPE_NORMAL; 2071213379Shselasky } 2072213379Shselasky 2073213379Shselasky if (x != xfer->nframes) { 2074213379Shselasky /* setup page_cache pointer */ 2075213379Shselasky temp.pc = xfer->frbuffers + x; 2076213379Shselasky /* set endpoint direction */ 2077213379Shselasky temp.direction = UE_GET_DIR(xfer->endpointno); 2078213379Shselasky } 2079213379Shselasky 2080213379Shselasky while (x != xfer->nframes) { 2081213379Shselasky 2082213379Shselasky /* DATA0 / DATA1 message */ 2083213379Shselasky 2084213379Shselasky temp.len = xfer->frlengths[x]; 2085213379Shselasky temp.step_td = ((xfer->endpointno & UE_DIR_IN) && 2086213379Shselasky x != 0 && temp.multishort == 0); 2087213379Shselasky 2088213379Shselasky x++; 2089213379Shselasky 2090213379Shselasky if (x == xfer->nframes) { 2091213379Shselasky if (xfer->flags_int.control_xfr) { 2092213379Shselasky /* no STATUS stage yet, DATA is last */ 2093213379Shselasky if (xfer->flags_int.control_act) 2094213379Shselasky temp.last_frame = 1; 2095213379Shselasky } else { 2096213379Shselasky temp.last_frame = 1; 2097213379Shselasky } 2098213379Shselasky } 2099213379Shselasky if (temp.len == 0) { 2100213379Shselasky 2101213379Shselasky /* make sure that we send an USB packet */ 2102213379Shselasky 2103213379Shselasky temp.shortpkt = 0; 2104213379Shselasky 2105213379Shselasky temp.tbc = 0; 2106213379Shselasky temp.tlbpc = mult - 1; 2107213379Shselasky 2108213379Shselasky } else if (xfer->flags_int.isochronous_xfr) { 2109213379Shselasky 2110213379Shselasky uint8_t tdpc; 2111213379Shselasky 2112235001Shselasky /* 2113235001Shselasky * Isochronous transfers don't have short 2114235001Shselasky * packet termination: 2115235001Shselasky */ 2116213379Shselasky 2117213379Shselasky temp.shortpkt = 1; 2118213379Shselasky 2119213379Shselasky /* isochronous transfers have a transfer limit */ 2120213379Shselasky 2121213379Shselasky if (temp.len > xfer->max_frame_size) 2122213379Shselasky temp.len = xfer->max_frame_size; 2123213379Shselasky 2124213379Shselasky /* compute TD packet count */ 2125213379Shselasky tdpc = (temp.len + xfer->max_packet_size - 1) / 2126213379Shselasky xfer->max_packet_size; 2127213379Shselasky 2128213379Shselasky temp.tbc = ((tdpc + mult - 1) / mult) - 1; 2129213379Shselasky temp.tlbpc = (tdpc % mult); 2130213379Shselasky 2131213379Shselasky if (temp.tlbpc == 0) 2132213379Shselasky temp.tlbpc = mult - 1; 2133213379Shselasky else 2134213379Shselasky temp.tlbpc--; 2135213379Shselasky } else { 2136213379Shselasky 2137213379Shselasky /* regular data transfer */ 2138213379Shselasky 2139213379Shselasky temp.shortpkt = xfer->flags.force_short_xfer ? 0 : 1; 2140213379Shselasky } 2141213379Shselasky 2142213379Shselasky xhci_setup_generic_chain_sub(&temp); 2143213379Shselasky 2144213379Shselasky if (xfer->flags_int.isochronous_xfr) { 2145213379Shselasky temp.offset += xfer->frlengths[x - 1]; 2146213379Shselasky temp.isoc_frame += temp.isoc_delta; 2147213379Shselasky } else { 2148213379Shselasky /* get next Page Cache pointer */ 2149213379Shselasky temp.pc = xfer->frbuffers + x; 2150213379Shselasky } 2151213379Shselasky } 2152213379Shselasky 2153213379Shselasky /* check if we should append a status stage */ 2154213379Shselasky 2155213379Shselasky if (xfer->flags_int.control_xfr && 2156213379Shselasky !xfer->flags_int.control_act) { 2157213379Shselasky 2158213379Shselasky /* 2159213379Shselasky * Send a DATA1 message and invert the current 2160213379Shselasky * endpoint direction. 2161213379Shselasky */ 2162213379Shselasky temp.step_td = (xfer->nframes != 0); 2163213379Shselasky temp.direction = UE_GET_DIR(xfer->endpointno) ^ UE_DIR_IN; 2164213379Shselasky temp.len = 0; 2165213379Shselasky temp.pc = NULL; 2166213379Shselasky temp.shortpkt = 0; 2167213379Shselasky temp.last_frame = 1; 2168213379Shselasky temp.trb_type = XHCI_TRB_TYPE_STATUS_STAGE; 2169213379Shselasky 2170213379Shselasky xhci_setup_generic_chain_sub(&temp); 2171213379Shselasky } 2172213379Shselasky 2173213379Shselasky td = temp.td; 2174213379Shselasky 2175213379Shselasky /* must have at least one frame! */ 2176213379Shselasky 2177213379Shselasky xfer->td_transfer_last = td; 2178213379Shselasky 2179213379Shselasky DPRINTF("first=%p last=%p\n", xfer->td_transfer_first, td); 2180213379Shselasky} 2181213379Shselasky 2182213379Shselaskystatic void 2183213379Shselaskyxhci_set_slot_pointer(struct xhci_softc *sc, uint8_t index, uint64_t dev_addr) 2184213379Shselasky{ 2185213379Shselasky struct usb_page_search buf_res; 2186213379Shselasky struct xhci_dev_ctx_addr *pdctxa; 2187213379Shselasky 2188213379Shselasky usbd_get_page(&sc->sc_hw.ctx_pc, 0, &buf_res); 2189213379Shselasky 2190213379Shselasky pdctxa = buf_res.buffer; 2191213379Shselasky 2192213379Shselasky DPRINTF("addr[%u]=0x%016llx\n", index, (long long)dev_addr); 2193213379Shselasky 2194213379Shselasky pdctxa->qwBaaDevCtxAddr[index] = htole64(dev_addr); 2195213379Shselasky 2196213379Shselasky usb_pc_cpu_flush(&sc->sc_hw.ctx_pc); 2197213379Shselasky} 2198213379Shselasky 2199213379Shselaskystatic usb_error_t 2200213379Shselaskyxhci_configure_mask(struct usb_device *udev, uint32_t mask, uint8_t drop) 2201213379Shselasky{ 2202213379Shselasky struct xhci_softc *sc = XHCI_BUS2SC(udev->bus); 2203213379Shselasky struct usb_page_search buf_inp; 2204213379Shselasky struct xhci_input_dev_ctx *pinp; 2205245731Shselasky uint32_t temp; 2206213379Shselasky uint8_t index; 2207245731Shselasky uint8_t x; 2208213379Shselasky 2209213379Shselasky index = udev->controller_slot_id; 2210213379Shselasky 2211213379Shselasky usbd_get_page(&sc->sc_hw.devs[index].input_pc, 0, &buf_inp); 2212213379Shselasky 2213213379Shselasky pinp = buf_inp.buffer; 2214213379Shselasky 2215213379Shselasky if (drop) { 2216213379Shselasky mask &= XHCI_INCTX_NON_CTRL_MASK; 2217217374Shselasky xhci_ctx_set_le32(sc, &pinp->ctx_input.dwInCtx0, mask); 2218217374Shselasky xhci_ctx_set_le32(sc, &pinp->ctx_input.dwInCtx1, 0); 2219213379Shselasky } else { 2220217374Shselasky xhci_ctx_set_le32(sc, &pinp->ctx_input.dwInCtx0, 0); 2221217374Shselasky xhci_ctx_set_le32(sc, &pinp->ctx_input.dwInCtx1, mask); 2222245731Shselasky 2223245731Shselasky /* find most significant set bit */ 2224245731Shselasky for (x = 31; x != 1; x--) { 2225245731Shselasky if (mask & (1 << x)) 2226245731Shselasky break; 2227245731Shselasky } 2228245731Shselasky 2229245731Shselasky /* adjust */ 2230245731Shselasky x--; 2231245731Shselasky 2232245731Shselasky /* figure out maximum */ 2233245731Shselasky if (x > sc->sc_hw.devs[index].context_num) { 2234245731Shselasky sc->sc_hw.devs[index].context_num = x; 2235245731Shselasky temp = xhci_ctx_get_le32(sc, &pinp->ctx_slot.dwSctx0); 2236245731Shselasky temp &= ~XHCI_SCTX_0_CTX_NUM_SET(31); 2237245731Shselasky temp |= XHCI_SCTX_0_CTX_NUM_SET(x + 1); 2238245731Shselasky xhci_ctx_set_le32(sc, &pinp->ctx_slot.dwSctx0, temp); 2239245731Shselasky } 2240213379Shselasky } 2241213379Shselasky return (0); 2242213379Shselasky} 2243213379Shselasky 2244213379Shselaskystatic usb_error_t 2245213379Shselaskyxhci_configure_endpoint(struct usb_device *udev, 2246213379Shselasky struct usb_endpoint_descriptor *edesc, uint64_t ring_addr, 2247213379Shselasky uint16_t interval, uint8_t max_packet_count, uint8_t mult, 2248213379Shselasky uint8_t fps_shift, uint16_t max_packet_size, uint16_t max_frame_size) 2249213379Shselasky{ 2250213379Shselasky struct usb_page_search buf_inp; 2251213379Shselasky struct xhci_softc *sc = XHCI_BUS2SC(udev->bus); 2252213379Shselasky struct xhci_input_dev_ctx *pinp; 2253213379Shselasky uint32_t temp; 2254213379Shselasky uint8_t index; 2255213379Shselasky uint8_t epno; 2256213379Shselasky uint8_t type; 2257213379Shselasky 2258213379Shselasky index = udev->controller_slot_id; 2259213379Shselasky 2260213379Shselasky usbd_get_page(&sc->sc_hw.devs[index].input_pc, 0, &buf_inp); 2261213379Shselasky 2262213379Shselasky pinp = buf_inp.buffer; 2263213379Shselasky 2264213379Shselasky epno = edesc->bEndpointAddress; 2265213379Shselasky type = edesc->bmAttributes & UE_XFERTYPE; 2266213379Shselasky 2267213379Shselasky if (type == UE_CONTROL) 2268213379Shselasky epno |= UE_DIR_IN; 2269213379Shselasky 2270213379Shselasky epno = XHCI_EPNO2EPID(epno); 2271213379Shselasky 2272213379Shselasky if (epno == 0) 2273213379Shselasky return (USB_ERR_NO_PIPE); /* invalid */ 2274213379Shselasky 2275213379Shselasky if (max_packet_count == 0) 2276213379Shselasky return (USB_ERR_BAD_BUFSIZE); 2277213379Shselasky 2278213379Shselasky max_packet_count--; 2279213379Shselasky 2280213379Shselasky if (mult == 0) 2281213379Shselasky return (USB_ERR_BAD_BUFSIZE); 2282213379Shselasky 2283213379Shselasky temp = XHCI_EPCTX_0_EPSTATE_SET(0) | 2284213379Shselasky XHCI_EPCTX_0_MAXP_STREAMS_SET(0) | 2285213379Shselasky XHCI_EPCTX_0_LSA_SET(0); 2286213379Shselasky 2287213379Shselasky switch (udev->speed) { 2288213379Shselasky case USB_SPEED_FULL: 2289213379Shselasky case USB_SPEED_LOW: 2290213379Shselasky /* 1ms -> 125us */ 2291213379Shselasky fps_shift += 3; 2292213379Shselasky break; 2293213379Shselasky default: 2294213379Shselasky break; 2295213379Shselasky } 2296213379Shselasky 2297213379Shselasky switch (type) { 2298213379Shselasky case UE_INTERRUPT: 2299213379Shselasky if (fps_shift > 3) 2300213379Shselasky fps_shift--; 2301213379Shselasky temp |= XHCI_EPCTX_0_IVAL_SET(fps_shift); 2302213379Shselasky break; 2303213379Shselasky case UE_ISOCHRONOUS: 2304213379Shselasky temp |= XHCI_EPCTX_0_IVAL_SET(fps_shift); 2305213379Shselasky 2306213379Shselasky switch (udev->speed) { 2307213379Shselasky case USB_SPEED_SUPER: 2308213379Shselasky if (mult > 3) 2309213379Shselasky mult = 3; 2310213379Shselasky temp |= XHCI_EPCTX_0_MULT_SET(mult - 1); 2311213379Shselasky max_packet_count /= mult; 2312213379Shselasky break; 2313213379Shselasky default: 2314213379Shselasky break; 2315213379Shselasky } 2316213379Shselasky break; 2317213379Shselasky default: 2318213379Shselasky break; 2319213379Shselasky } 2320213379Shselasky 2321217374Shselasky xhci_ctx_set_le32(sc, &pinp->ctx_ep[epno - 1].dwEpCtx0, temp); 2322213379Shselasky 2323213379Shselasky temp = 2324213379Shselasky XHCI_EPCTX_1_HID_SET(0) | 2325213379Shselasky XHCI_EPCTX_1_MAXB_SET(max_packet_count) | 2326213379Shselasky XHCI_EPCTX_1_MAXP_SIZE_SET(max_packet_size); 2327213379Shselasky 2328213379Shselasky if ((udev->parent_hs_hub != NULL) || (udev->address != 0)) { 2329213379Shselasky if (type != UE_ISOCHRONOUS) 2330213379Shselasky temp |= XHCI_EPCTX_1_CERR_SET(3); 2331213379Shselasky } 2332213379Shselasky 2333213379Shselasky switch (type) { 2334213379Shselasky case UE_CONTROL: 2335213379Shselasky temp |= XHCI_EPCTX_1_EPTYPE_SET(4); 2336213379Shselasky break; 2337213379Shselasky case UE_ISOCHRONOUS: 2338213379Shselasky temp |= XHCI_EPCTX_1_EPTYPE_SET(1); 2339213379Shselasky break; 2340213379Shselasky case UE_BULK: 2341213379Shselasky temp |= XHCI_EPCTX_1_EPTYPE_SET(2); 2342213379Shselasky break; 2343213379Shselasky default: 2344213379Shselasky temp |= XHCI_EPCTX_1_EPTYPE_SET(3); 2345213379Shselasky break; 2346213379Shselasky } 2347213379Shselasky 2348213379Shselasky /* check for IN direction */ 2349213379Shselasky if (epno & 1) 2350213379Shselasky temp |= XHCI_EPCTX_1_EPTYPE_SET(4); 2351213379Shselasky 2352217374Shselasky xhci_ctx_set_le32(sc, &pinp->ctx_ep[epno - 1].dwEpCtx1, temp); 2353213379Shselasky 2354213379Shselasky ring_addr |= XHCI_EPCTX_2_DCS_SET(1); 2355213379Shselasky 2356217374Shselasky xhci_ctx_set_le64(sc, &pinp->ctx_ep[epno - 1].qwEpCtx2, ring_addr); 2357213379Shselasky 2358213379Shselasky switch (edesc->bmAttributes & UE_XFERTYPE) { 2359213379Shselasky case UE_INTERRUPT: 2360213379Shselasky case UE_ISOCHRONOUS: 2361213379Shselasky temp = XHCI_EPCTX_4_MAX_ESIT_PAYLOAD_SET(max_frame_size) | 2362213379Shselasky XHCI_EPCTX_4_AVG_TRB_LEN_SET(MIN(XHCI_PAGE_SIZE, 2363213379Shselasky max_frame_size)); 2364213379Shselasky break; 2365213379Shselasky case UE_CONTROL: 2366213379Shselasky temp = XHCI_EPCTX_4_AVG_TRB_LEN_SET(8); 2367213379Shselasky break; 2368213379Shselasky default: 2369213379Shselasky temp = XHCI_EPCTX_4_AVG_TRB_LEN_SET(XHCI_PAGE_SIZE); 2370213379Shselasky break; 2371213379Shselasky } 2372213379Shselasky 2373217374Shselasky xhci_ctx_set_le32(sc, &pinp->ctx_ep[epno - 1].dwEpCtx4, temp); 2374213379Shselasky 2375213379Shselasky#ifdef USB_DEBUG 2376217374Shselasky xhci_dump_endpoint(sc, &pinp->ctx_ep[epno - 1]); 2377213379Shselasky#endif 2378213379Shselasky usb_pc_cpu_flush(&sc->sc_hw.devs[index].input_pc); 2379213379Shselasky 2380213379Shselasky return (0); /* success */ 2381213379Shselasky} 2382213379Shselasky 2383213379Shselaskystatic usb_error_t 2384213379Shselaskyxhci_configure_endpoint_by_xfer(struct usb_xfer *xfer) 2385213379Shselasky{ 2386213379Shselasky struct xhci_endpoint_ext *pepext; 2387213379Shselasky struct usb_endpoint_ss_comp_descriptor *ecomp; 2388213379Shselasky 2389213379Shselasky pepext = xhci_get_endpoint_ext(xfer->xroot->udev, 2390213379Shselasky xfer->endpoint->edesc); 2391213379Shselasky 2392213379Shselasky ecomp = xfer->endpoint->ecomp; 2393213379Shselasky 2394213379Shselasky pepext->trb[0].dwTrb3 = 0; /* halt any transfers */ 2395213379Shselasky usb_pc_cpu_flush(pepext->page_cache); 2396213379Shselasky 2397213379Shselasky return (xhci_configure_endpoint(xfer->xroot->udev, 2398213379Shselasky xfer->endpoint->edesc, pepext->physaddr, 2399213379Shselasky xfer->interval, xfer->max_packet_count, 2400213379Shselasky (ecomp != NULL) ? (ecomp->bmAttributes & 3) + 1 : 1, 2401213379Shselasky usbd_xfer_get_fps_shift(xfer), xfer->max_packet_size, 2402213379Shselasky xfer->max_frame_size)); 2403213379Shselasky} 2404213379Shselasky 2405213379Shselaskystatic usb_error_t 2406213379Shselaskyxhci_configure_device(struct usb_device *udev) 2407213379Shselasky{ 2408213379Shselasky struct xhci_softc *sc = XHCI_BUS2SC(udev->bus); 2409213379Shselasky struct usb_page_search buf_inp; 2410213379Shselasky struct usb_page_cache *pcinp; 2411213379Shselasky struct xhci_input_dev_ctx *pinp; 2412213379Shselasky struct usb_device *hubdev; 2413213379Shselasky uint32_t temp; 2414213379Shselasky uint32_t route; 2415230302Shselasky uint32_t rh_port; 2416213379Shselasky uint8_t is_hub; 2417213379Shselasky uint8_t index; 2418230302Shselasky uint8_t depth; 2419213379Shselasky 2420213379Shselasky index = udev->controller_slot_id; 2421213379Shselasky 2422213379Shselasky DPRINTF("index=%u\n", index); 2423213379Shselasky 2424213379Shselasky pcinp = &sc->sc_hw.devs[index].input_pc; 2425213379Shselasky 2426213379Shselasky usbd_get_page(pcinp, 0, &buf_inp); 2427213379Shselasky 2428213379Shselasky pinp = buf_inp.buffer; 2429213379Shselasky 2430213379Shselasky rh_port = 0; 2431213379Shselasky route = 0; 2432213379Shselasky 2433213379Shselasky /* figure out route string and root HUB port number */ 2434213379Shselasky 2435213379Shselasky for (hubdev = udev; hubdev != NULL; hubdev = hubdev->parent_hub) { 2436213379Shselasky 2437213379Shselasky if (hubdev->parent_hub == NULL) 2438213379Shselasky break; 2439213379Shselasky 2440230302Shselasky depth = hubdev->parent_hub->depth; 2441230302Shselasky 2442213379Shselasky /* 2443213379Shselasky * NOTE: HS/FS/LS devices and the SS root HUB can have 2444213379Shselasky * more than 15 ports 2445213379Shselasky */ 2446213379Shselasky 2447213379Shselasky rh_port = hubdev->port_no; 2448213379Shselasky 2449230302Shselasky if (depth == 0) 2450213379Shselasky break; 2451213379Shselasky 2452230302Shselasky if (rh_port > 15) 2453230302Shselasky rh_port = 15; 2454213379Shselasky 2455230302Shselasky if (depth < 6) 2456230302Shselasky route |= rh_port << (4 * (depth - 1)); 2457213379Shselasky } 2458213379Shselasky 2459230302Shselasky DPRINTF("Route=0x%08x\n", route); 2460230302Shselasky 2461245731Shselasky temp = XHCI_SCTX_0_ROUTE_SET(route) | 2462245731Shselasky XHCI_SCTX_0_CTX_NUM_SET( 2463245731Shselasky sc->sc_hw.devs[index].context_num + 1); 2464213379Shselasky 2465213379Shselasky switch (udev->speed) { 2466213379Shselasky case USB_SPEED_LOW: 2467213379Shselasky temp |= XHCI_SCTX_0_SPEED_SET(2); 2468235001Shselasky if (udev->parent_hs_hub != NULL && 2469235001Shselasky udev->parent_hs_hub->ddesc.bDeviceProtocol == 2470235001Shselasky UDPROTO_HSHUBMTT) { 2471235001Shselasky DPRINTF("Device inherits MTT\n"); 2472235001Shselasky temp |= XHCI_SCTX_0_MTT_SET(1); 2473235001Shselasky } 2474213379Shselasky break; 2475213379Shselasky case USB_SPEED_HIGH: 2476213379Shselasky temp |= XHCI_SCTX_0_SPEED_SET(3); 2477235001Shselasky if (sc->sc_hw.devs[index].nports != 0 && 2478235001Shselasky udev->ddesc.bDeviceProtocol == UDPROTO_HSHUBMTT) { 2479235001Shselasky DPRINTF("HUB supports MTT\n"); 2480235001Shselasky temp |= XHCI_SCTX_0_MTT_SET(1); 2481235001Shselasky } 2482213379Shselasky break; 2483213379Shselasky case USB_SPEED_FULL: 2484213379Shselasky temp |= XHCI_SCTX_0_SPEED_SET(1); 2485235001Shselasky if (udev->parent_hs_hub != NULL && 2486235001Shselasky udev->parent_hs_hub->ddesc.bDeviceProtocol == 2487235001Shselasky UDPROTO_HSHUBMTT) { 2488235001Shselasky DPRINTF("Device inherits MTT\n"); 2489235001Shselasky temp |= XHCI_SCTX_0_MTT_SET(1); 2490235001Shselasky } 2491213379Shselasky break; 2492213379Shselasky default: 2493213379Shselasky temp |= XHCI_SCTX_0_SPEED_SET(4); 2494213379Shselasky break; 2495213379Shselasky } 2496213379Shselasky 2497213379Shselasky is_hub = sc->sc_hw.devs[index].nports != 0 && 2498213379Shselasky (udev->speed == USB_SPEED_SUPER || 2499213379Shselasky udev->speed == USB_SPEED_HIGH); 2500213379Shselasky 2501235001Shselasky if (is_hub) 2502213379Shselasky temp |= XHCI_SCTX_0_HUB_SET(1); 2503213379Shselasky 2504217374Shselasky xhci_ctx_set_le32(sc, &pinp->ctx_slot.dwSctx0, temp); 2505213379Shselasky 2506213379Shselasky temp = XHCI_SCTX_1_RH_PORT_SET(rh_port); 2507213379Shselasky 2508213379Shselasky if (is_hub) { 2509213379Shselasky temp |= XHCI_SCTX_1_NUM_PORTS_SET( 2510213379Shselasky sc->sc_hw.devs[index].nports); 2511213379Shselasky } 2512213379Shselasky 2513213379Shselasky switch (udev->speed) { 2514213379Shselasky case USB_SPEED_SUPER: 2515213379Shselasky switch (sc->sc_hw.devs[index].state) { 2516213379Shselasky case XHCI_ST_ADDRESSED: 2517213379Shselasky case XHCI_ST_CONFIGURED: 2518213379Shselasky /* enable power save */ 2519213379Shselasky temp |= XHCI_SCTX_1_MAX_EL_SET(sc->sc_exit_lat_max); 2520213379Shselasky break; 2521213379Shselasky default: 2522213379Shselasky /* disable power save */ 2523213379Shselasky break; 2524213379Shselasky } 2525213379Shselasky break; 2526213379Shselasky default: 2527213379Shselasky break; 2528213379Shselasky } 2529213379Shselasky 2530217374Shselasky xhci_ctx_set_le32(sc, &pinp->ctx_slot.dwSctx1, temp); 2531213379Shselasky 2532213379Shselasky temp = XHCI_SCTX_2_IRQ_TARGET_SET(0); 2533213379Shselasky 2534235001Shselasky if (is_hub) { 2535235001Shselasky temp |= XHCI_SCTX_2_TT_THINK_TIME_SET( 2536235001Shselasky sc->sc_hw.devs[index].tt); 2537235001Shselasky } 2538213379Shselasky 2539213379Shselasky hubdev = udev->parent_hs_hub; 2540213379Shselasky 2541213379Shselasky /* check if we should activate the transaction translator */ 2542213379Shselasky switch (udev->speed) { 2543213379Shselasky case USB_SPEED_FULL: 2544213379Shselasky case USB_SPEED_LOW: 2545213379Shselasky if (hubdev != NULL) { 2546213379Shselasky temp |= XHCI_SCTX_2_TT_HUB_SID_SET( 2547213379Shselasky hubdev->controller_slot_id); 2548213379Shselasky temp |= XHCI_SCTX_2_TT_PORT_NUM_SET( 2549213379Shselasky udev->hs_port_no); 2550213379Shselasky } 2551213379Shselasky break; 2552213379Shselasky default: 2553213379Shselasky break; 2554213379Shselasky } 2555213379Shselasky 2556217374Shselasky xhci_ctx_set_le32(sc, &pinp->ctx_slot.dwSctx2, temp); 2557213379Shselasky 2558261112Shselasky /* 2559261112Shselasky * These fields should be initialized to zero, according to 2560261112Shselasky * XHCI section 6.2.2 - slot context: 2561261112Shselasky */ 2562261112Shselasky temp = XHCI_SCTX_3_DEV_ADDR_SET(0) | 2563213379Shselasky XHCI_SCTX_3_SLOT_STATE_SET(0); 2564213379Shselasky 2565217374Shselasky xhci_ctx_set_le32(sc, &pinp->ctx_slot.dwSctx3, temp); 2566213379Shselasky 2567213379Shselasky#ifdef USB_DEBUG 2568217374Shselasky xhci_dump_device(sc, &pinp->ctx_slot); 2569213379Shselasky#endif 2570213379Shselasky usb_pc_cpu_flush(pcinp); 2571213379Shselasky 2572213379Shselasky return (0); /* success */ 2573213379Shselasky} 2574213379Shselasky 2575213379Shselaskystatic usb_error_t 2576213379Shselaskyxhci_alloc_device_ext(struct usb_device *udev) 2577213379Shselasky{ 2578213379Shselasky struct xhci_softc *sc = XHCI_BUS2SC(udev->bus); 2579213379Shselasky struct usb_page_search buf_dev; 2580213379Shselasky struct usb_page_search buf_ep; 2581213379Shselasky struct xhci_trb *trb; 2582213379Shselasky struct usb_page_cache *pc; 2583213379Shselasky struct usb_page *pg; 2584213379Shselasky uint64_t addr; 2585213379Shselasky uint8_t index; 2586213379Shselasky uint8_t i; 2587213379Shselasky 2588213379Shselasky index = udev->controller_slot_id; 2589213379Shselasky 2590213379Shselasky pc = &sc->sc_hw.devs[index].device_pc; 2591213379Shselasky pg = &sc->sc_hw.devs[index].device_pg; 2592213379Shselasky 2593213379Shselasky /* need to initialize the page cache */ 2594213379Shselasky pc->tag_parent = sc->sc_bus.dma_parent_tag; 2595213379Shselasky 2596217374Shselasky if (usb_pc_alloc_mem(pc, pg, sc->sc_ctx_is_64_byte ? 2597217374Shselasky (2 * sizeof(struct xhci_dev_ctx)) : 2598217374Shselasky sizeof(struct xhci_dev_ctx), XHCI_PAGE_SIZE)) 2599213379Shselasky goto error; 2600213379Shselasky 2601213379Shselasky usbd_get_page(pc, 0, &buf_dev); 2602213379Shselasky 2603213379Shselasky pc = &sc->sc_hw.devs[index].input_pc; 2604213379Shselasky pg = &sc->sc_hw.devs[index].input_pg; 2605213379Shselasky 2606213379Shselasky /* need to initialize the page cache */ 2607213379Shselasky pc->tag_parent = sc->sc_bus.dma_parent_tag; 2608213379Shselasky 2609217374Shselasky if (usb_pc_alloc_mem(pc, pg, sc->sc_ctx_is_64_byte ? 2610217374Shselasky (2 * sizeof(struct xhci_input_dev_ctx)) : 2611245731Shselasky sizeof(struct xhci_input_dev_ctx), XHCI_PAGE_SIZE)) { 2612213379Shselasky goto error; 2613245731Shselasky } 2614213379Shselasky 2615213379Shselasky pc = &sc->sc_hw.devs[index].endpoint_pc; 2616213379Shselasky pg = &sc->sc_hw.devs[index].endpoint_pg; 2617213379Shselasky 2618213379Shselasky /* need to initialize the page cache */ 2619213379Shselasky pc->tag_parent = sc->sc_bus.dma_parent_tag; 2620213379Shselasky 2621245731Shselasky if (usb_pc_alloc_mem(pc, pg, 2622245731Shselasky sizeof(struct xhci_dev_endpoint_trbs), XHCI_PAGE_SIZE)) { 2623213379Shselasky goto error; 2624245731Shselasky } 2625213379Shselasky 2626213379Shselasky /* initialise all endpoint LINK TRBs */ 2627213379Shselasky 2628213379Shselasky for (i = 0; i != XHCI_MAX_ENDPOINTS; i++) { 2629213379Shselasky 2630213379Shselasky /* lookup endpoint TRB ring */ 2631245731Shselasky usbd_get_page(pc, (uintptr_t)& 2632245731Shselasky ((struct xhci_dev_endpoint_trbs *)0)->trb[i][0], &buf_ep); 2633213379Shselasky 2634213379Shselasky /* get TRB pointer */ 2635213379Shselasky trb = buf_ep.buffer; 2636213379Shselasky trb += XHCI_MAX_TRANSFERS - 1; 2637213379Shselasky 2638213379Shselasky /* get TRB start address */ 2639213379Shselasky addr = buf_ep.physaddr; 2640213379Shselasky 2641213379Shselasky /* create LINK TRB */ 2642213379Shselasky trb->qwTrb0 = htole64(addr); 2643213379Shselasky trb->dwTrb2 = htole32(XHCI_TRB_2_IRQ_SET(0)); 2644213379Shselasky trb->dwTrb3 = htole32(XHCI_TRB_3_CYCLE_BIT | 2645213379Shselasky XHCI_TRB_3_TYPE_SET(XHCI_TRB_TYPE_LINK)); 2646213379Shselasky } 2647213379Shselasky 2648213379Shselasky usb_pc_cpu_flush(pc); 2649213379Shselasky 2650213379Shselasky xhci_set_slot_pointer(sc, index, buf_dev.physaddr); 2651213379Shselasky 2652213379Shselasky return (0); 2653213379Shselasky 2654213379Shselaskyerror: 2655213379Shselasky xhci_free_device_ext(udev); 2656213379Shselasky 2657213379Shselasky return (USB_ERR_NOMEM); 2658213379Shselasky} 2659213379Shselasky 2660213379Shselaskystatic void 2661213379Shselaskyxhci_free_device_ext(struct usb_device *udev) 2662213379Shselasky{ 2663213379Shselasky struct xhci_softc *sc = XHCI_BUS2SC(udev->bus); 2664213379Shselasky uint8_t index; 2665213379Shselasky 2666213379Shselasky index = udev->controller_slot_id; 2667213379Shselasky xhci_set_slot_pointer(sc, index, 0); 2668213379Shselasky 2669213379Shselasky usb_pc_free_mem(&sc->sc_hw.devs[index].device_pc); 2670213379Shselasky usb_pc_free_mem(&sc->sc_hw.devs[index].input_pc); 2671213379Shselasky usb_pc_free_mem(&sc->sc_hw.devs[index].endpoint_pc); 2672213379Shselasky} 2673213379Shselasky 2674213379Shselaskystatic struct xhci_endpoint_ext * 2675213379Shselaskyxhci_get_endpoint_ext(struct usb_device *udev, struct usb_endpoint_descriptor *edesc) 2676213379Shselasky{ 2677213379Shselasky struct xhci_softc *sc = XHCI_BUS2SC(udev->bus); 2678213379Shselasky struct xhci_endpoint_ext *pepext; 2679213379Shselasky struct usb_page_cache *pc; 2680213379Shselasky struct usb_page_search buf_ep; 2681213379Shselasky uint8_t epno; 2682213379Shselasky uint8_t index; 2683213379Shselasky 2684213379Shselasky epno = edesc->bEndpointAddress; 2685213379Shselasky if ((edesc->bmAttributes & UE_XFERTYPE) == UE_CONTROL) 2686213379Shselasky epno |= UE_DIR_IN; 2687213379Shselasky 2688213379Shselasky epno = XHCI_EPNO2EPID(epno); 2689213379Shselasky 2690213379Shselasky index = udev->controller_slot_id; 2691213379Shselasky 2692213379Shselasky pc = &sc->sc_hw.devs[index].endpoint_pc; 2693213379Shselasky 2694213379Shselasky usbd_get_page(pc, (uintptr_t)&((struct xhci_dev_endpoint_trbs *)0)->trb[epno][0], &buf_ep); 2695213379Shselasky 2696213379Shselasky pepext = &sc->sc_hw.devs[index].endp[epno]; 2697213379Shselasky pepext->page_cache = pc; 2698213379Shselasky pepext->trb = buf_ep.buffer; 2699213379Shselasky pepext->physaddr = buf_ep.physaddr; 2700213379Shselasky 2701213379Shselasky return (pepext); 2702213379Shselasky} 2703213379Shselasky 2704213379Shselaskystatic void 2705213379Shselaskyxhci_endpoint_doorbell(struct usb_xfer *xfer) 2706213379Shselasky{ 2707213379Shselasky struct xhci_softc *sc = XHCI_BUS2SC(xfer->xroot->bus); 2708213379Shselasky uint8_t epno; 2709213379Shselasky uint8_t index; 2710213379Shselasky 2711213379Shselasky epno = xfer->endpointno; 2712213379Shselasky if (xfer->flags_int.control_xfr) 2713213379Shselasky epno |= UE_DIR_IN; 2714213379Shselasky 2715213379Shselasky epno = XHCI_EPNO2EPID(epno); 2716213379Shselasky index = xfer->xroot->udev->controller_slot_id; 2717213379Shselasky 2718245731Shselasky if (xfer->xroot->udev->flags.self_suspended == 0) { 2719245731Shselasky XWRITE4(sc, door, XHCI_DOORBELL(index), 2720245731Shselasky epno | XHCI_DB_SID_SET(/*xfer->stream_id*/ 0)); 2721245731Shselasky } 2722213379Shselasky} 2723213379Shselasky 2724213379Shselaskystatic void 2725213379Shselaskyxhci_transfer_remove(struct usb_xfer *xfer, usb_error_t error) 2726213379Shselasky{ 2727213379Shselasky struct xhci_endpoint_ext *pepext; 2728213379Shselasky 2729213379Shselasky if (xfer->flags_int.bandwidth_reclaimed) { 2730213379Shselasky xfer->flags_int.bandwidth_reclaimed = 0; 2731213379Shselasky 2732213379Shselasky pepext = xhci_get_endpoint_ext(xfer->xroot->udev, 2733213379Shselasky xfer->endpoint->edesc); 2734213379Shselasky 2735213379Shselasky pepext->trb_used--; 2736213379Shselasky 2737213379Shselasky pepext->xfer[xfer->qh_pos] = NULL; 2738213379Shselasky 2739213379Shselasky if (error && pepext->trb_running != 0) { 2740213379Shselasky pepext->trb_halted = 1; 2741213379Shselasky pepext->trb_running = 0; 2742213379Shselasky } 2743213379Shselasky } 2744213379Shselasky} 2745213379Shselasky 2746213379Shselaskystatic usb_error_t 2747213379Shselaskyxhci_transfer_insert(struct usb_xfer *xfer) 2748213379Shselasky{ 2749213379Shselasky struct xhci_td *td_first; 2750213379Shselasky struct xhci_td *td_last; 2751251614Shselasky struct xhci_trb *trb_link; 2752213379Shselasky struct xhci_endpoint_ext *pepext; 2753213379Shselasky uint64_t addr; 2754213379Shselasky uint8_t i; 2755213379Shselasky uint8_t inext; 2756213379Shselasky uint8_t trb_limit; 2757213379Shselasky 2758213379Shselasky DPRINTFN(8, "\n"); 2759213379Shselasky 2760213379Shselasky /* check if already inserted */ 2761213379Shselasky if (xfer->flags_int.bandwidth_reclaimed) { 2762213379Shselasky DPRINTFN(8, "Already in schedule\n"); 2763213379Shselasky return (0); 2764213379Shselasky } 2765213379Shselasky 2766213379Shselasky pepext = xhci_get_endpoint_ext(xfer->xroot->udev, 2767213379Shselasky xfer->endpoint->edesc); 2768213379Shselasky 2769213379Shselasky td_first = xfer->td_transfer_first; 2770213379Shselasky td_last = xfer->td_transfer_last; 2771213379Shselasky addr = pepext->physaddr; 2772213379Shselasky 2773213379Shselasky switch (xfer->endpoint->edesc->bmAttributes & UE_XFERTYPE) { 2774213379Shselasky case UE_CONTROL: 2775213379Shselasky case UE_INTERRUPT: 2776213379Shselasky /* single buffered */ 2777213379Shselasky trb_limit = 1; 2778213379Shselasky break; 2779213379Shselasky default: 2780213379Shselasky /* multi buffered */ 2781213379Shselasky trb_limit = (XHCI_MAX_TRANSFERS - 2); 2782213379Shselasky break; 2783213379Shselasky } 2784213379Shselasky 2785213379Shselasky if (pepext->trb_used >= trb_limit) { 2786213379Shselasky DPRINTFN(8, "Too many TDs queued.\n"); 2787213379Shselasky return (USB_ERR_NOMEM); 2788213379Shselasky } 2789213379Shselasky 2790213379Shselasky /* check for stopped condition, after putting transfer on interrupt queue */ 2791213379Shselasky if (pepext->trb_running == 0) { 2792213379Shselasky struct xhci_softc *sc = XHCI_BUS2SC(xfer->xroot->bus); 2793213379Shselasky 2794213379Shselasky DPRINTFN(8, "Not running\n"); 2795213379Shselasky 2796213379Shselasky /* start configuration */ 2797213379Shselasky (void)usb_proc_msignal(&sc->sc_config_proc, 2798213379Shselasky &sc->sc_config_msg[0], &sc->sc_config_msg[1]); 2799213379Shselasky return (0); 2800213379Shselasky } 2801213379Shselasky 2802213379Shselasky pepext->trb_used++; 2803213379Shselasky 2804213379Shselasky /* get current TRB index */ 2805213379Shselasky i = pepext->trb_index; 2806213379Shselasky 2807213379Shselasky /* get next TRB index */ 2808213379Shselasky inext = (i + 1); 2809213379Shselasky 2810213379Shselasky /* the last entry of the ring is a hardcoded link TRB */ 2811213379Shselasky if (inext >= (XHCI_MAX_TRANSFERS - 1)) 2812213379Shselasky inext = 0; 2813213379Shselasky 2814213379Shselasky /* compute terminating return address */ 2815213379Shselasky addr += inext * sizeof(struct xhci_trb); 2816213379Shselasky 2817251614Shselasky /* compute link TRB pointer */ 2818251614Shselasky trb_link = td_last->td_trb + td_last->ntrb; 2819251614Shselasky 2820213379Shselasky /* update next pointer of last link TRB */ 2821251614Shselasky trb_link->qwTrb0 = htole64(addr); 2822251614Shselasky trb_link->dwTrb2 = htole32(XHCI_TRB_2_IRQ_SET(0)); 2823251614Shselasky trb_link->dwTrb3 = htole32(XHCI_TRB_3_IOC_BIT | 2824251614Shselasky XHCI_TRB_3_CYCLE_BIT | 2825251614Shselasky XHCI_TRB_3_TYPE_SET(XHCI_TRB_TYPE_LINK)); 2826213379Shselasky 2827213379Shselasky#ifdef USB_DEBUG 2828213379Shselasky xhci_dump_trb(&td_last->td_trb[td_last->ntrb]); 2829213379Shselasky#endif 2830213379Shselasky usb_pc_cpu_flush(td_last->page_cache); 2831213379Shselasky 2832213379Shselasky /* write ahead chain end marker */ 2833213379Shselasky 2834213379Shselasky pepext->trb[inext].qwTrb0 = 0; 2835213379Shselasky pepext->trb[inext].dwTrb2 = 0; 2836213379Shselasky pepext->trb[inext].dwTrb3 = 0; 2837213379Shselasky 2838213379Shselasky /* update next pointer of link TRB */ 2839213379Shselasky 2840213379Shselasky pepext->trb[i].qwTrb0 = htole64((uint64_t)td_first->td_self); 2841213379Shselasky pepext->trb[i].dwTrb2 = htole32(XHCI_TRB_2_IRQ_SET(0)); 2842213379Shselasky 2843213379Shselasky#ifdef USB_DEBUG 2844213379Shselasky xhci_dump_trb(&pepext->trb[i]); 2845213379Shselasky#endif 2846213379Shselasky usb_pc_cpu_flush(pepext->page_cache); 2847213379Shselasky 2848213379Shselasky /* toggle cycle bit which activates the transfer chain */ 2849213379Shselasky 2850213379Shselasky pepext->trb[i].dwTrb3 = htole32(XHCI_TRB_3_CYCLE_BIT | 2851213379Shselasky XHCI_TRB_3_TYPE_SET(XHCI_TRB_TYPE_LINK)); 2852213379Shselasky 2853213379Shselasky usb_pc_cpu_flush(pepext->page_cache); 2854213379Shselasky 2855213379Shselasky DPRINTF("qh_pos = %u\n", i); 2856213379Shselasky 2857213379Shselasky pepext->xfer[i] = xfer; 2858213379Shselasky 2859213379Shselasky xfer->qh_pos = i; 2860213379Shselasky 2861213379Shselasky xfer->flags_int.bandwidth_reclaimed = 1; 2862213379Shselasky 2863213379Shselasky pepext->trb_index = inext; 2864213379Shselasky 2865213379Shselasky xhci_endpoint_doorbell(xfer); 2866213379Shselasky 2867213379Shselasky return (0); 2868213379Shselasky} 2869213379Shselasky 2870213379Shselaskystatic void 2871213379Shselaskyxhci_root_intr(struct xhci_softc *sc) 2872213379Shselasky{ 2873213379Shselasky uint16_t i; 2874213379Shselasky 2875213379Shselasky USB_BUS_LOCK_ASSERT(&sc->sc_bus, MA_OWNED); 2876213379Shselasky 2877213379Shselasky /* clear any old interrupt data */ 2878213379Shselasky memset(sc->sc_hub_idata, 0, sizeof(sc->sc_hub_idata)); 2879213379Shselasky 2880213379Shselasky for (i = 1; i <= sc->sc_noport; i++) { 2881213379Shselasky /* pick out CHANGE bits from the status register */ 2882213379Shselasky if (XREAD4(sc, oper, XHCI_PORTSC(i)) & ( 2883213379Shselasky XHCI_PS_CSC | XHCI_PS_PEC | 2884213379Shselasky XHCI_PS_OCC | XHCI_PS_WRC | 2885213379Shselasky XHCI_PS_PRC | XHCI_PS_PLC | 2886213379Shselasky XHCI_PS_CEC)) { 2887213379Shselasky sc->sc_hub_idata[i / 8] |= 1 << (i % 8); 2888213379Shselasky DPRINTF("port %d changed\n", i); 2889213379Shselasky } 2890213379Shselasky } 2891213379Shselasky uhub_root_intr(&sc->sc_bus, sc->sc_hub_idata, 2892213379Shselasky sizeof(sc->sc_hub_idata)); 2893213379Shselasky} 2894213379Shselasky 2895213379Shselasky/*------------------------------------------------------------------------* 2896213379Shselasky * xhci_device_done - XHCI done handler 2897213379Shselasky * 2898213379Shselasky * NOTE: This function can be called two times in a row on 2899213379Shselasky * the same USB transfer. From close and from interrupt. 2900213379Shselasky *------------------------------------------------------------------------*/ 2901213379Shselaskystatic void 2902213379Shselaskyxhci_device_done(struct usb_xfer *xfer, usb_error_t error) 2903213379Shselasky{ 2904213379Shselasky DPRINTFN(2, "xfer=%p, endpoint=%p, error=%d\n", 2905213379Shselasky xfer, xfer->endpoint, error); 2906213379Shselasky 2907213379Shselasky /* remove transfer from HW queue */ 2908213379Shselasky xhci_transfer_remove(xfer, error); 2909213379Shselasky 2910213379Shselasky /* dequeue transfer and start next transfer */ 2911213379Shselasky usbd_transfer_done(xfer, error); 2912213379Shselasky} 2913213379Shselasky 2914213379Shselasky/*------------------------------------------------------------------------* 2915213379Shselasky * XHCI data transfer support (generic type) 2916213379Shselasky *------------------------------------------------------------------------*/ 2917213379Shselaskystatic void 2918213379Shselaskyxhci_device_generic_open(struct usb_xfer *xfer) 2919213379Shselasky{ 2920213379Shselasky if (xfer->flags_int.isochronous_xfr) { 2921213379Shselasky switch (xfer->xroot->udev->speed) { 2922213379Shselasky case USB_SPEED_FULL: 2923213379Shselasky break; 2924213379Shselasky default: 2925213379Shselasky usb_hs_bandwidth_alloc(xfer); 2926213379Shselasky break; 2927213379Shselasky } 2928213379Shselasky } 2929213379Shselasky} 2930213379Shselasky 2931213379Shselaskystatic void 2932213379Shselaskyxhci_device_generic_close(struct usb_xfer *xfer) 2933213379Shselasky{ 2934213379Shselasky DPRINTF("\n"); 2935213379Shselasky 2936213379Shselasky xhci_device_done(xfer, USB_ERR_CANCELLED); 2937213379Shselasky 2938213379Shselasky if (xfer->flags_int.isochronous_xfr) { 2939213379Shselasky switch (xfer->xroot->udev->speed) { 2940213379Shselasky case USB_SPEED_FULL: 2941213379Shselasky break; 2942213379Shselasky default: 2943213379Shselasky usb_hs_bandwidth_free(xfer); 2944213379Shselasky break; 2945213379Shselasky } 2946213379Shselasky } 2947213379Shselasky} 2948213379Shselasky 2949213379Shselaskystatic void 2950213379Shselaskyxhci_device_generic_multi_enter(struct usb_endpoint *ep, 2951213379Shselasky struct usb_xfer *enter_xfer) 2952213379Shselasky{ 2953213379Shselasky struct usb_xfer *xfer; 2954213379Shselasky 2955213379Shselasky /* check if there is a current transfer */ 2956213379Shselasky xfer = ep->endpoint_q.curr; 2957213379Shselasky if (xfer == NULL) 2958213379Shselasky return; 2959213379Shselasky 2960213379Shselasky /* 2961213379Shselasky * Check if the current transfer is started and then pickup 2962213379Shselasky * the next one, if any. Else wait for next start event due to 2963213379Shselasky * block on failure feature. 2964213379Shselasky */ 2965213379Shselasky if (!xfer->flags_int.bandwidth_reclaimed) 2966213379Shselasky return; 2967213379Shselasky 2968213379Shselasky xfer = TAILQ_FIRST(&ep->endpoint_q.head); 2969213379Shselasky if (xfer == NULL) { 2970213379Shselasky /* 2971213379Shselasky * In case of enter we have to consider that the 2972213379Shselasky * transfer is queued by the USB core after the enter 2973213379Shselasky * method is called. 2974213379Shselasky */ 2975213379Shselasky xfer = enter_xfer; 2976213379Shselasky 2977213379Shselasky if (xfer == NULL) 2978213379Shselasky return; 2979213379Shselasky } 2980213379Shselasky 2981213379Shselasky /* try to multi buffer */ 2982213379Shselasky xhci_transfer_insert(xfer); 2983213379Shselasky} 2984213379Shselasky 2985213379Shselaskystatic void 2986213379Shselaskyxhci_device_generic_enter(struct usb_xfer *xfer) 2987213379Shselasky{ 2988213379Shselasky DPRINTF("\n"); 2989213379Shselasky 2990213379Shselasky /* setup TD's and QH */ 2991213379Shselasky xhci_setup_generic_chain(xfer); 2992213379Shselasky 2993213379Shselasky xhci_device_generic_multi_enter(xfer->endpoint, xfer); 2994213379Shselasky} 2995213379Shselasky 2996213379Shselaskystatic void 2997213379Shselaskyxhci_device_generic_start(struct usb_xfer *xfer) 2998213379Shselasky{ 2999213379Shselasky DPRINTF("\n"); 3000213379Shselasky 3001213379Shselasky /* try to insert xfer on HW queue */ 3002213379Shselasky xhci_transfer_insert(xfer); 3003213379Shselasky 3004213379Shselasky /* try to multi buffer */ 3005213379Shselasky xhci_device_generic_multi_enter(xfer->endpoint, NULL); 3006213379Shselasky 3007213379Shselasky /* add transfer last on interrupt queue */ 3008213379Shselasky usbd_transfer_enqueue(&xfer->xroot->bus->intr_q, xfer); 3009213379Shselasky 3010213379Shselasky /* start timeout, if any */ 3011213379Shselasky if (xfer->timeout != 0) 3012213379Shselasky usbd_transfer_timeout_ms(xfer, &xhci_timeout, xfer->timeout); 3013213379Shselasky} 3014213379Shselasky 3015213379Shselaskystruct usb_pipe_methods xhci_device_generic_methods = 3016213379Shselasky{ 3017213379Shselasky .open = xhci_device_generic_open, 3018213379Shselasky .close = xhci_device_generic_close, 3019213379Shselasky .enter = xhci_device_generic_enter, 3020213379Shselasky .start = xhci_device_generic_start, 3021213379Shselasky}; 3022213379Shselasky 3023213379Shselasky/*------------------------------------------------------------------------* 3024213379Shselasky * xhci root HUB support 3025213379Shselasky *------------------------------------------------------------------------* 3026213379Shselasky * Simulate a hardware HUB by handling all the necessary requests. 3027213379Shselasky *------------------------------------------------------------------------*/ 3028213379Shselasky 3029235000Shselasky#define HSETW(ptr, val) ptr = { (uint8_t)(val), (uint8_t)((val) >> 8) } 3030213379Shselasky 3031213379Shselaskystatic const 3032213379Shselaskystruct usb_device_descriptor xhci_devd = 3033213379Shselasky{ 3034213379Shselasky .bLength = sizeof(xhci_devd), 3035213379Shselasky .bDescriptorType = UDESC_DEVICE, /* type */ 3036213379Shselasky HSETW(.bcdUSB, 0x0300), /* USB version */ 3037213379Shselasky .bDeviceClass = UDCLASS_HUB, /* class */ 3038213379Shselasky .bDeviceSubClass = UDSUBCLASS_HUB, /* subclass */ 3039213379Shselasky .bDeviceProtocol = UDPROTO_SSHUB, /* protocol */ 3040213379Shselasky .bMaxPacketSize = 9, /* max packet size */ 3041213379Shselasky HSETW(.idVendor, 0x0000), /* vendor */ 3042213379Shselasky HSETW(.idProduct, 0x0000), /* product */ 3043213379Shselasky HSETW(.bcdDevice, 0x0100), /* device version */ 3044213379Shselasky .iManufacturer = 1, 3045213379Shselasky .iProduct = 2, 3046213379Shselasky .iSerialNumber = 0, 3047213379Shselasky .bNumConfigurations = 1, /* # of configurations */ 3048213379Shselasky}; 3049213379Shselasky 3050213379Shselaskystatic const 3051213379Shselaskystruct xhci_bos_desc xhci_bosd = { 3052213379Shselasky .bosd = { 3053213379Shselasky .bLength = sizeof(xhci_bosd.bosd), 3054213379Shselasky .bDescriptorType = UDESC_BOS, 3055213379Shselasky HSETW(.wTotalLength, sizeof(xhci_bosd)), 3056213379Shselasky .bNumDeviceCaps = 3, 3057213379Shselasky }, 3058213379Shselasky .usb2extd = { 3059213379Shselasky .bLength = sizeof(xhci_bosd.usb2extd), 3060213379Shselasky .bDescriptorType = 1, 3061213379Shselasky .bDevCapabilityType = 2, 3062229084Shselasky .bmAttributes[0] = 2, 3063213379Shselasky }, 3064213379Shselasky .usbdcd = { 3065213379Shselasky .bLength = sizeof(xhci_bosd.usbdcd), 3066213379Shselasky .bDescriptorType = UDESC_DEVICE_CAPABILITY, 3067213379Shselasky .bDevCapabilityType = 3, 3068213379Shselasky .bmAttributes = 0, /* XXX */ 3069213379Shselasky HSETW(.wSpeedsSupported, 0x000C), 3070213379Shselasky .bFunctionalitySupport = 8, 3071213379Shselasky .bU1DevExitLat = 255, /* dummy - not used */ 3072235000Shselasky .wU2DevExitLat = { 0x00, 0x08 }, 3073213379Shselasky }, 3074213379Shselasky .cidd = { 3075213379Shselasky .bLength = sizeof(xhci_bosd.cidd), 3076213379Shselasky .bDescriptorType = 1, 3077213379Shselasky .bDevCapabilityType = 4, 3078213379Shselasky .bReserved = 0, 3079213379Shselasky .bContainerID = 0, /* XXX */ 3080213379Shselasky }, 3081213379Shselasky}; 3082213379Shselasky 3083213379Shselaskystatic const 3084213379Shselaskystruct xhci_config_desc xhci_confd = { 3085213379Shselasky .confd = { 3086213379Shselasky .bLength = sizeof(xhci_confd.confd), 3087213379Shselasky .bDescriptorType = UDESC_CONFIG, 3088213379Shselasky .wTotalLength[0] = sizeof(xhci_confd), 3089213379Shselasky .bNumInterface = 1, 3090213379Shselasky .bConfigurationValue = 1, 3091213379Shselasky .iConfiguration = 0, 3092213379Shselasky .bmAttributes = UC_SELF_POWERED, 3093213379Shselasky .bMaxPower = 0 /* max power */ 3094213379Shselasky }, 3095213379Shselasky .ifcd = { 3096213379Shselasky .bLength = sizeof(xhci_confd.ifcd), 3097213379Shselasky .bDescriptorType = UDESC_INTERFACE, 3098213379Shselasky .bNumEndpoints = 1, 3099213379Shselasky .bInterfaceClass = UICLASS_HUB, 3100213379Shselasky .bInterfaceSubClass = UISUBCLASS_HUB, 3101213379Shselasky .bInterfaceProtocol = 0, 3102213379Shselasky }, 3103213379Shselasky .endpd = { 3104213379Shselasky .bLength = sizeof(xhci_confd.endpd), 3105213379Shselasky .bDescriptorType = UDESC_ENDPOINT, 3106213379Shselasky .bEndpointAddress = UE_DIR_IN | XHCI_INTR_ENDPT, 3107213379Shselasky .bmAttributes = UE_INTERRUPT, 3108213379Shselasky .wMaxPacketSize[0] = 2, /* max 15 ports */ 3109213379Shselasky .bInterval = 255, 3110213379Shselasky }, 3111213379Shselasky .endpcd = { 3112213379Shselasky .bLength = sizeof(xhci_confd.endpcd), 3113213379Shselasky .bDescriptorType = UDESC_ENDPOINT_SS_COMP, 3114213379Shselasky .bMaxBurst = 0, 3115213379Shselasky .bmAttributes = 0, 3116213379Shselasky }, 3117213379Shselasky}; 3118213379Shselasky 3119213379Shselaskystatic const 3120213379Shselaskystruct usb_hub_ss_descriptor xhci_hubd = { 3121213379Shselasky .bLength = sizeof(xhci_hubd), 3122213379Shselasky .bDescriptorType = UDESC_SS_HUB, 3123213379Shselasky}; 3124213379Shselasky 3125213379Shselaskystatic usb_error_t 3126213379Shselaskyxhci_roothub_exec(struct usb_device *udev, 3127213379Shselasky struct usb_device_request *req, const void **pptr, uint16_t *plength) 3128213379Shselasky{ 3129213379Shselasky struct xhci_softc *sc = XHCI_BUS2SC(udev->bus); 3130213379Shselasky const char *str_ptr; 3131213379Shselasky const void *ptr; 3132213379Shselasky uint32_t port; 3133213379Shselasky uint32_t v; 3134213379Shselasky uint16_t len; 3135213379Shselasky uint16_t i; 3136213379Shselasky uint16_t value; 3137213379Shselasky uint16_t index; 3138213379Shselasky uint8_t j; 3139213379Shselasky usb_error_t err; 3140213379Shselasky 3141213379Shselasky USB_BUS_LOCK_ASSERT(&sc->sc_bus, MA_OWNED); 3142213379Shselasky 3143213379Shselasky /* buffer reset */ 3144213379Shselasky ptr = (const void *)&sc->sc_hub_desc; 3145213379Shselasky len = 0; 3146213379Shselasky err = 0; 3147213379Shselasky 3148213379Shselasky value = UGETW(req->wValue); 3149213379Shselasky index = UGETW(req->wIndex); 3150213379Shselasky 3151213379Shselasky DPRINTFN(3, "type=0x%02x request=0x%02x wLen=0x%04x " 3152213379Shselasky "wValue=0x%04x wIndex=0x%04x\n", 3153213379Shselasky req->bmRequestType, req->bRequest, 3154213379Shselasky UGETW(req->wLength), value, index); 3155213379Shselasky 3156213379Shselasky#define C(x,y) ((x) | ((y) << 8)) 3157213379Shselasky switch (C(req->bRequest, req->bmRequestType)) { 3158213379Shselasky case C(UR_CLEAR_FEATURE, UT_WRITE_DEVICE): 3159213379Shselasky case C(UR_CLEAR_FEATURE, UT_WRITE_INTERFACE): 3160213379Shselasky case C(UR_CLEAR_FEATURE, UT_WRITE_ENDPOINT): 3161213379Shselasky /* 3162213379Shselasky * DEVICE_REMOTE_WAKEUP and ENDPOINT_HALT are no-ops 3163213379Shselasky * for the integrated root hub. 3164213379Shselasky */ 3165213379Shselasky break; 3166213379Shselasky case C(UR_GET_CONFIG, UT_READ_DEVICE): 3167213379Shselasky len = 1; 3168213379Shselasky sc->sc_hub_desc.temp[0] = sc->sc_conf; 3169213379Shselasky break; 3170213379Shselasky case C(UR_GET_DESCRIPTOR, UT_READ_DEVICE): 3171213379Shselasky switch (value >> 8) { 3172213379Shselasky case UDESC_DEVICE: 3173213379Shselasky if ((value & 0xff) != 0) { 3174213379Shselasky err = USB_ERR_IOERROR; 3175213379Shselasky goto done; 3176213379Shselasky } 3177213379Shselasky len = sizeof(xhci_devd); 3178213379Shselasky ptr = (const void *)&xhci_devd; 3179213379Shselasky break; 3180213379Shselasky 3181213379Shselasky case UDESC_BOS: 3182213379Shselasky if ((value & 0xff) != 0) { 3183213379Shselasky err = USB_ERR_IOERROR; 3184213379Shselasky goto done; 3185213379Shselasky } 3186213379Shselasky len = sizeof(xhci_bosd); 3187213379Shselasky ptr = (const void *)&xhci_bosd; 3188213379Shselasky break; 3189213379Shselasky 3190213379Shselasky case UDESC_CONFIG: 3191213379Shselasky if ((value & 0xff) != 0) { 3192213379Shselasky err = USB_ERR_IOERROR; 3193213379Shselasky goto done; 3194213379Shselasky } 3195213379Shselasky len = sizeof(xhci_confd); 3196213379Shselasky ptr = (const void *)&xhci_confd; 3197213379Shselasky break; 3198213379Shselasky 3199213379Shselasky case UDESC_STRING: 3200213379Shselasky switch (value & 0xff) { 3201213379Shselasky case 0: /* Language table */ 3202213379Shselasky str_ptr = "\001"; 3203213379Shselasky break; 3204213379Shselasky 3205213379Shselasky case 1: /* Vendor */ 3206213379Shselasky str_ptr = sc->sc_vendor; 3207213379Shselasky break; 3208213379Shselasky 3209213379Shselasky case 2: /* Product */ 3210213379Shselasky str_ptr = "XHCI root HUB"; 3211213379Shselasky break; 3212213379Shselasky 3213213379Shselasky default: 3214213379Shselasky str_ptr = ""; 3215213379Shselasky break; 3216213379Shselasky } 3217213379Shselasky 3218213379Shselasky len = usb_make_str_desc( 3219213379Shselasky sc->sc_hub_desc.temp, 3220213379Shselasky sizeof(sc->sc_hub_desc.temp), 3221213379Shselasky str_ptr); 3222213379Shselasky break; 3223213379Shselasky 3224213379Shselasky default: 3225213379Shselasky err = USB_ERR_IOERROR; 3226213379Shselasky goto done; 3227213379Shselasky } 3228213379Shselasky break; 3229213379Shselasky case C(UR_GET_INTERFACE, UT_READ_INTERFACE): 3230213379Shselasky len = 1; 3231213379Shselasky sc->sc_hub_desc.temp[0] = 0; 3232213379Shselasky break; 3233213379Shselasky case C(UR_GET_STATUS, UT_READ_DEVICE): 3234213379Shselasky len = 2; 3235213379Shselasky USETW(sc->sc_hub_desc.stat.wStatus, UDS_SELF_POWERED); 3236213379Shselasky break; 3237213379Shselasky case C(UR_GET_STATUS, UT_READ_INTERFACE): 3238213379Shselasky case C(UR_GET_STATUS, UT_READ_ENDPOINT): 3239213379Shselasky len = 2; 3240213379Shselasky USETW(sc->sc_hub_desc.stat.wStatus, 0); 3241213379Shselasky break; 3242213379Shselasky case C(UR_SET_ADDRESS, UT_WRITE_DEVICE): 3243213379Shselasky if (value >= XHCI_MAX_DEVICES) { 3244213379Shselasky err = USB_ERR_IOERROR; 3245213379Shselasky goto done; 3246213379Shselasky } 3247213379Shselasky break; 3248213379Shselasky case C(UR_SET_CONFIG, UT_WRITE_DEVICE): 3249213379Shselasky if (value != 0 && value != 1) { 3250213379Shselasky err = USB_ERR_IOERROR; 3251213379Shselasky goto done; 3252213379Shselasky } 3253213379Shselasky sc->sc_conf = value; 3254213379Shselasky break; 3255213379Shselasky case C(UR_SET_DESCRIPTOR, UT_WRITE_DEVICE): 3256213379Shselasky break; 3257213379Shselasky case C(UR_SET_FEATURE, UT_WRITE_DEVICE): 3258213379Shselasky case C(UR_SET_FEATURE, UT_WRITE_INTERFACE): 3259213379Shselasky case C(UR_SET_FEATURE, UT_WRITE_ENDPOINT): 3260213379Shselasky err = USB_ERR_IOERROR; 3261213379Shselasky goto done; 3262213379Shselasky case C(UR_SET_INTERFACE, UT_WRITE_INTERFACE): 3263213379Shselasky break; 3264213379Shselasky case C(UR_SYNCH_FRAME, UT_WRITE_ENDPOINT): 3265213379Shselasky break; 3266213379Shselasky /* Hub requests */ 3267213379Shselasky case C(UR_CLEAR_FEATURE, UT_WRITE_CLASS_DEVICE): 3268213379Shselasky break; 3269213379Shselasky case C(UR_CLEAR_FEATURE, UT_WRITE_CLASS_OTHER): 3270213379Shselasky DPRINTFN(9, "UR_CLEAR_PORT_FEATURE\n"); 3271213379Shselasky 3272213379Shselasky if ((index < 1) || 3273213379Shselasky (index > sc->sc_noport)) { 3274213379Shselasky err = USB_ERR_IOERROR; 3275213379Shselasky goto done; 3276213379Shselasky } 3277213379Shselasky port = XHCI_PORTSC(index); 3278213379Shselasky 3279226904Shselasky v = XREAD4(sc, oper, port); 3280226904Shselasky i = XHCI_PS_PLS_GET(v); 3281226904Shselasky v &= ~XHCI_PS_CLEAR; 3282213379Shselasky 3283213379Shselasky switch (value) { 3284213379Shselasky case UHF_C_BH_PORT_RESET: 3285213379Shselasky XWRITE4(sc, oper, port, v | XHCI_PS_WRC); 3286213379Shselasky break; 3287213379Shselasky case UHF_C_PORT_CONFIG_ERROR: 3288213379Shselasky XWRITE4(sc, oper, port, v | XHCI_PS_CEC); 3289213379Shselasky break; 3290230302Shselasky case UHF_C_PORT_SUSPEND: 3291213379Shselasky case UHF_C_PORT_LINK_STATE: 3292213379Shselasky XWRITE4(sc, oper, port, v | XHCI_PS_PLC); 3293213379Shselasky break; 3294213379Shselasky case UHF_C_PORT_CONNECTION: 3295213379Shselasky XWRITE4(sc, oper, port, v | XHCI_PS_CSC); 3296213379Shselasky break; 3297213379Shselasky case UHF_C_PORT_ENABLE: 3298213379Shselasky XWRITE4(sc, oper, port, v | XHCI_PS_PEC); 3299213379Shselasky break; 3300213379Shselasky case UHF_C_PORT_OVER_CURRENT: 3301213379Shselasky XWRITE4(sc, oper, port, v | XHCI_PS_OCC); 3302213379Shselasky break; 3303213379Shselasky case UHF_C_PORT_RESET: 3304213379Shselasky XWRITE4(sc, oper, port, v | XHCI_PS_PRC); 3305213379Shselasky break; 3306213379Shselasky case UHF_PORT_ENABLE: 3307213379Shselasky XWRITE4(sc, oper, port, v | XHCI_PS_PED); 3308213379Shselasky break; 3309213379Shselasky case UHF_PORT_POWER: 3310213379Shselasky XWRITE4(sc, oper, port, v & ~XHCI_PS_PP); 3311213379Shselasky break; 3312213379Shselasky case UHF_PORT_INDICATOR: 3313213379Shselasky XWRITE4(sc, oper, port, v & ~XHCI_PS_PIC_SET(3)); 3314213379Shselasky break; 3315213379Shselasky case UHF_PORT_SUSPEND: 3316226904Shselasky 3317226904Shselasky /* U3 -> U15 */ 3318226904Shselasky if (i == 3) { 3319226904Shselasky XWRITE4(sc, oper, port, v | 3320226904Shselasky XHCI_PS_PLS_SET(0xF) | XHCI_PS_LWS); 3321226904Shselasky } 3322226904Shselasky 3323226904Shselasky /* wait 20ms for resume sequence to complete */ 3324226904Shselasky usb_pause_mtx(&sc->sc_bus.bus_mtx, hz / 50); 3325226904Shselasky 3326226904Shselasky /* U0 */ 3327213379Shselasky XWRITE4(sc, oper, port, v | 3328213379Shselasky XHCI_PS_PLS_SET(0) | XHCI_PS_LWS); 3329213379Shselasky break; 3330213379Shselasky default: 3331213379Shselasky err = USB_ERR_IOERROR; 3332213379Shselasky goto done; 3333213379Shselasky } 3334213379Shselasky break; 3335213379Shselasky 3336213379Shselasky case C(UR_GET_DESCRIPTOR, UT_READ_CLASS_DEVICE): 3337213379Shselasky if ((value & 0xff) != 0) { 3338213379Shselasky err = USB_ERR_IOERROR; 3339213379Shselasky goto done; 3340213379Shselasky } 3341213379Shselasky 3342213379Shselasky v = XREAD4(sc, capa, XHCI_HCSPARAMS0); 3343213379Shselasky 3344213379Shselasky sc->sc_hub_desc.hubd = xhci_hubd; 3345213379Shselasky 3346213379Shselasky sc->sc_hub_desc.hubd.bNbrPorts = sc->sc_noport; 3347213379Shselasky 3348213379Shselasky if (XHCI_HCS0_PPC(v)) 3349213379Shselasky i = UHD_PWR_INDIVIDUAL; 3350213379Shselasky else 3351213379Shselasky i = UHD_PWR_GANGED; 3352213379Shselasky 3353213379Shselasky if (XHCI_HCS0_PIND(v)) 3354213379Shselasky i |= UHD_PORT_IND; 3355213379Shselasky 3356213379Shselasky i |= UHD_OC_INDIVIDUAL; 3357213379Shselasky 3358213379Shselasky USETW(sc->sc_hub_desc.hubd.wHubCharacteristics, i); 3359213379Shselasky 3360213379Shselasky /* see XHCI section 5.4.9: */ 3361213379Shselasky sc->sc_hub_desc.hubd.bPwrOn2PwrGood = 10; 3362213379Shselasky 3363213379Shselasky for (j = 1; j <= sc->sc_noport; j++) { 3364213379Shselasky 3365213379Shselasky v = XREAD4(sc, oper, XHCI_PORTSC(j)); 3366213379Shselasky if (v & XHCI_PS_DR) { 3367213379Shselasky sc->sc_hub_desc.hubd. 3368213379Shselasky DeviceRemovable[j / 8] |= 1U << (j % 8); 3369213379Shselasky } 3370213379Shselasky } 3371213379Shselasky len = sc->sc_hub_desc.hubd.bLength; 3372213379Shselasky break; 3373213379Shselasky 3374213379Shselasky case C(UR_GET_STATUS, UT_READ_CLASS_DEVICE): 3375213379Shselasky len = 16; 3376213379Shselasky memset(sc->sc_hub_desc.temp, 0, 16); 3377213379Shselasky break; 3378213379Shselasky 3379213379Shselasky case C(UR_GET_STATUS, UT_READ_CLASS_OTHER): 3380213379Shselasky DPRINTFN(9, "UR_GET_STATUS i=%d\n", index); 3381213379Shselasky 3382213379Shselasky if ((index < 1) || 3383213379Shselasky (index > sc->sc_noport)) { 3384213379Shselasky err = USB_ERR_IOERROR; 3385213379Shselasky goto done; 3386213379Shselasky } 3387213379Shselasky 3388213379Shselasky v = XREAD4(sc, oper, XHCI_PORTSC(index)); 3389213379Shselasky 3390213379Shselasky DPRINTFN(9, "port status=0x%08x\n", v); 3391213379Shselasky 3392213379Shselasky i = UPS_PORT_LINK_STATE_SET(XHCI_PS_PLS_GET(v)); 3393213379Shselasky 3394213379Shselasky switch (XHCI_PS_SPEED_GET(v)) { 3395213379Shselasky case 3: 3396213379Shselasky i |= UPS_HIGH_SPEED; 3397213379Shselasky break; 3398213379Shselasky case 2: 3399213379Shselasky i |= UPS_LOW_SPEED; 3400213379Shselasky break; 3401213379Shselasky case 1: 3402213379Shselasky /* FULL speed */ 3403213379Shselasky break; 3404213379Shselasky default: 3405213379Shselasky i |= UPS_OTHER_SPEED; 3406213379Shselasky break; 3407213379Shselasky } 3408213379Shselasky 3409213379Shselasky if (v & XHCI_PS_CCS) 3410213379Shselasky i |= UPS_CURRENT_CONNECT_STATUS; 3411213379Shselasky if (v & XHCI_PS_PED) 3412213379Shselasky i |= UPS_PORT_ENABLED; 3413213379Shselasky if (v & XHCI_PS_OCA) 3414213379Shselasky i |= UPS_OVERCURRENT_INDICATOR; 3415213379Shselasky if (v & XHCI_PS_PR) 3416213379Shselasky i |= UPS_RESET; 3417230302Shselasky if (v & XHCI_PS_PP) { 3418230302Shselasky /* 3419230302Shselasky * The USB 3.0 RH is using the 3420230302Shselasky * USB 2.0's power bit 3421230302Shselasky */ 3422213379Shselasky i |= UPS_PORT_POWER; 3423230302Shselasky } 3424213379Shselasky USETW(sc->sc_hub_desc.ps.wPortStatus, i); 3425213379Shselasky 3426213379Shselasky i = 0; 3427213379Shselasky if (v & XHCI_PS_CSC) 3428213379Shselasky i |= UPS_C_CONNECT_STATUS; 3429213379Shselasky if (v & XHCI_PS_PEC) 3430213379Shselasky i |= UPS_C_PORT_ENABLED; 3431213379Shselasky if (v & XHCI_PS_OCC) 3432213379Shselasky i |= UPS_C_OVERCURRENT_INDICATOR; 3433213379Shselasky if (v & XHCI_PS_WRC) 3434213379Shselasky i |= UPS_C_BH_PORT_RESET; 3435213379Shselasky if (v & XHCI_PS_PRC) 3436213379Shselasky i |= UPS_C_PORT_RESET; 3437213379Shselasky if (v & XHCI_PS_PLC) 3438213379Shselasky i |= UPS_C_PORT_LINK_STATE; 3439213379Shselasky if (v & XHCI_PS_CEC) 3440213379Shselasky i |= UPS_C_PORT_CONFIG_ERROR; 3441213379Shselasky 3442213379Shselasky USETW(sc->sc_hub_desc.ps.wPortChange, i); 3443213379Shselasky len = sizeof(sc->sc_hub_desc.ps); 3444213379Shselasky break; 3445213379Shselasky 3446213379Shselasky case C(UR_SET_DESCRIPTOR, UT_WRITE_CLASS_DEVICE): 3447213379Shselasky err = USB_ERR_IOERROR; 3448213379Shselasky goto done; 3449213379Shselasky 3450213379Shselasky case C(UR_SET_FEATURE, UT_WRITE_CLASS_DEVICE): 3451213379Shselasky break; 3452213379Shselasky 3453213379Shselasky case C(UR_SET_FEATURE, UT_WRITE_CLASS_OTHER): 3454213379Shselasky 3455213379Shselasky i = index >> 8; 3456213379Shselasky index &= 0x00FF; 3457213379Shselasky 3458213379Shselasky if ((index < 1) || 3459213379Shselasky (index > sc->sc_noport)) { 3460213379Shselasky err = USB_ERR_IOERROR; 3461213379Shselasky goto done; 3462213379Shselasky } 3463213379Shselasky 3464213379Shselasky port = XHCI_PORTSC(index); 3465213379Shselasky v = XREAD4(sc, oper, port) & ~XHCI_PS_CLEAR; 3466213379Shselasky 3467213379Shselasky switch (value) { 3468213379Shselasky case UHF_PORT_U1_TIMEOUT: 3469213379Shselasky if (XHCI_PS_SPEED_GET(v) != 4) { 3470213379Shselasky err = USB_ERR_IOERROR; 3471213379Shselasky goto done; 3472213379Shselasky } 3473213379Shselasky port = XHCI_PORTPMSC(index); 3474213379Shselasky v = XREAD4(sc, oper, port); 3475213379Shselasky v &= ~XHCI_PM3_U1TO_SET(0xFF); 3476213379Shselasky v |= XHCI_PM3_U1TO_SET(i); 3477213379Shselasky XWRITE4(sc, oper, port, v); 3478213379Shselasky break; 3479213379Shselasky case UHF_PORT_U2_TIMEOUT: 3480213379Shselasky if (XHCI_PS_SPEED_GET(v) != 4) { 3481213379Shselasky err = USB_ERR_IOERROR; 3482213379Shselasky goto done; 3483213379Shselasky } 3484213379Shselasky port = XHCI_PORTPMSC(index); 3485213379Shselasky v = XREAD4(sc, oper, port); 3486213379Shselasky v &= ~XHCI_PM3_U2TO_SET(0xFF); 3487213379Shselasky v |= XHCI_PM3_U2TO_SET(i); 3488213379Shselasky XWRITE4(sc, oper, port, v); 3489213379Shselasky break; 3490213379Shselasky case UHF_BH_PORT_RESET: 3491213379Shselasky XWRITE4(sc, oper, port, v | XHCI_PS_WPR); 3492213379Shselasky break; 3493213379Shselasky case UHF_PORT_LINK_STATE: 3494213379Shselasky XWRITE4(sc, oper, port, v | 3495213379Shselasky XHCI_PS_PLS_SET(i) | XHCI_PS_LWS); 3496213379Shselasky /* 4ms settle time */ 3497213379Shselasky usb_pause_mtx(&sc->sc_bus.bus_mtx, hz / 250); 3498213379Shselasky break; 3499213379Shselasky case UHF_PORT_ENABLE: 3500213379Shselasky DPRINTFN(3, "set port enable %d\n", index); 3501213379Shselasky break; 3502213379Shselasky case UHF_PORT_SUSPEND: 3503213379Shselasky DPRINTFN(6, "suspend port %u (LPM=%u)\n", index, i); 3504213379Shselasky j = XHCI_PS_SPEED_GET(v); 3505213379Shselasky if ((j < 1) || (j > 3)) { 3506213379Shselasky /* non-supported speed */ 3507213379Shselasky err = USB_ERR_IOERROR; 3508213379Shselasky goto done; 3509213379Shselasky } 3510213379Shselasky XWRITE4(sc, oper, port, v | 3511213379Shselasky XHCI_PS_PLS_SET(i ? 2 /* LPM */ : 3) | XHCI_PS_LWS); 3512213379Shselasky break; 3513213379Shselasky case UHF_PORT_RESET: 3514213379Shselasky DPRINTFN(6, "reset port %d\n", index); 3515213379Shselasky XWRITE4(sc, oper, port, v | XHCI_PS_PR); 3516213379Shselasky break; 3517213379Shselasky case UHF_PORT_POWER: 3518213379Shselasky DPRINTFN(3, "set port power %d\n", index); 3519213379Shselasky XWRITE4(sc, oper, port, v | XHCI_PS_PP); 3520213379Shselasky break; 3521213379Shselasky case UHF_PORT_TEST: 3522213379Shselasky DPRINTFN(3, "set port test %d\n", index); 3523213379Shselasky break; 3524213379Shselasky case UHF_PORT_INDICATOR: 3525213379Shselasky DPRINTFN(3, "set port indicator %d\n", index); 3526213379Shselasky 3527213379Shselasky v &= ~XHCI_PS_PIC_SET(3); 3528213379Shselasky v |= XHCI_PS_PIC_SET(1); 3529213379Shselasky 3530213379Shselasky XWRITE4(sc, oper, port, v); 3531213379Shselasky break; 3532213379Shselasky default: 3533213379Shselasky err = USB_ERR_IOERROR; 3534213379Shselasky goto done; 3535213379Shselasky } 3536213379Shselasky break; 3537213379Shselasky 3538213379Shselasky case C(UR_CLEAR_TT_BUFFER, UT_WRITE_CLASS_OTHER): 3539213379Shselasky case C(UR_RESET_TT, UT_WRITE_CLASS_OTHER): 3540213379Shselasky case C(UR_GET_TT_STATE, UT_READ_CLASS_OTHER): 3541213379Shselasky case C(UR_STOP_TT, UT_WRITE_CLASS_OTHER): 3542213379Shselasky break; 3543213379Shselasky default: 3544213379Shselasky err = USB_ERR_IOERROR; 3545213379Shselasky goto done; 3546213379Shselasky } 3547213379Shselaskydone: 3548213379Shselasky *plength = len; 3549213379Shselasky *pptr = ptr; 3550213379Shselasky return (err); 3551213379Shselasky} 3552213379Shselasky 3553213379Shselaskystatic void 3554213379Shselaskyxhci_xfer_setup(struct usb_setup_params *parm) 3555213379Shselasky{ 3556213379Shselasky struct usb_page_search page_info; 3557213379Shselasky struct usb_page_cache *pc; 3558213379Shselasky struct xhci_softc *sc; 3559213379Shselasky struct usb_xfer *xfer; 3560213379Shselasky void *last_obj; 3561213379Shselasky uint32_t ntd; 3562213379Shselasky uint32_t n; 3563213379Shselasky 3564213379Shselasky sc = XHCI_BUS2SC(parm->udev->bus); 3565213379Shselasky xfer = parm->curr_xfer; 3566213379Shselasky 3567213379Shselasky /* 3568213379Shselasky * The proof for the "ntd" formula is illustrated like this: 3569213379Shselasky * 3570213379Shselasky * +------------------------------------+ 3571213379Shselasky * | | 3572213379Shselasky * | |remainder -> | 3573213379Shselasky * | +-----+---+ | 3574213379Shselasky * | | xxx | x | frm 0 | 3575213379Shselasky * | +-----+---++ | 3576213379Shselasky * | | xxx | xx | frm 1 | 3577213379Shselasky * | +-----+----+ | 3578213379Shselasky * | ... | 3579213379Shselasky * +------------------------------------+ 3580213379Shselasky * 3581213379Shselasky * "xxx" means a completely full USB transfer descriptor 3582213379Shselasky * 3583213379Shselasky * "x" and "xx" means a short USB packet 3584213379Shselasky * 3585213379Shselasky * For the remainder of an USB transfer modulo 3586213379Shselasky * "max_data_length" we need two USB transfer descriptors. 3587213379Shselasky * One to transfer the remaining data and one to finalise with 3588213379Shselasky * a zero length packet in case the "force_short_xfer" flag is 3589213379Shselasky * set. We only need two USB transfer descriptors in the case 3590213379Shselasky * where the transfer length of the first one is a factor of 3591213379Shselasky * "max_frame_size". The rest of the needed USB transfer 3592213379Shselasky * descriptors is given by the buffer size divided by the 3593213379Shselasky * maximum data payload. 3594213379Shselasky */ 3595213379Shselasky parm->hc_max_packet_size = 0x400; 3596213379Shselasky parm->hc_max_packet_count = 16 * 3; 3597213379Shselasky parm->hc_max_frame_size = XHCI_TD_PAYLOAD_MAX; 3598213379Shselasky 3599213379Shselasky xfer->flags_int.bdma_enable = 1; 3600213379Shselasky 3601213379Shselasky usbd_transfer_setup_sub(parm); 3602213379Shselasky 3603213379Shselasky if (xfer->flags_int.isochronous_xfr) { 3604213379Shselasky ntd = ((1 * xfer->nframes) 3605213379Shselasky + (xfer->max_data_length / xfer->max_hc_frame_size)); 3606213379Shselasky } else if (xfer->flags_int.control_xfr) { 3607213379Shselasky ntd = ((2 * xfer->nframes) + 1 /* STATUS */ 3608213379Shselasky + (xfer->max_data_length / xfer->max_hc_frame_size)); 3609213379Shselasky } else { 3610213379Shselasky ntd = ((2 * xfer->nframes) 3611213379Shselasky + (xfer->max_data_length / xfer->max_hc_frame_size)); 3612213379Shselasky } 3613213379Shselasky 3614213379Shselaskyalloc_dma_set: 3615213379Shselasky 3616213379Shselasky if (parm->err) 3617213379Shselasky return; 3618213379Shselasky 3619213379Shselasky /* 3620213379Shselasky * Allocate queue heads and transfer descriptors 3621213379Shselasky */ 3622213379Shselasky last_obj = NULL; 3623213379Shselasky 3624213379Shselasky if (usbd_transfer_setup_sub_malloc( 3625213379Shselasky parm, &pc, sizeof(struct xhci_td), 3626213379Shselasky XHCI_TD_ALIGN, ntd)) { 3627213379Shselasky parm->err = USB_ERR_NOMEM; 3628213379Shselasky return; 3629213379Shselasky } 3630213379Shselasky if (parm->buf) { 3631213379Shselasky for (n = 0; n != ntd; n++) { 3632213379Shselasky struct xhci_td *td; 3633213379Shselasky 3634213379Shselasky usbd_get_page(pc + n, 0, &page_info); 3635213379Shselasky 3636213379Shselasky td = page_info.buffer; 3637213379Shselasky 3638213379Shselasky /* init TD */ 3639213379Shselasky td->td_self = page_info.physaddr; 3640213379Shselasky td->obj_next = last_obj; 3641213379Shselasky td->page_cache = pc + n; 3642213379Shselasky 3643213379Shselasky last_obj = td; 3644213379Shselasky 3645213379Shselasky usb_pc_cpu_flush(pc + n); 3646213379Shselasky } 3647213379Shselasky } 3648213379Shselasky xfer->td_start[xfer->flags_int.curr_dma_set] = last_obj; 3649213379Shselasky 3650213379Shselasky if (!xfer->flags_int.curr_dma_set) { 3651213379Shselasky xfer->flags_int.curr_dma_set = 1; 3652213379Shselasky goto alloc_dma_set; 3653213379Shselasky } 3654213379Shselasky} 3655213379Shselasky 3656213379Shselaskystatic usb_error_t 3657213379Shselaskyxhci_configure_reset_endpoint(struct usb_xfer *xfer) 3658213379Shselasky{ 3659213379Shselasky struct xhci_softc *sc = XHCI_BUS2SC(xfer->xroot->bus); 3660213379Shselasky struct usb_page_search buf_inp; 3661213379Shselasky struct usb_device *udev; 3662213379Shselasky struct xhci_endpoint_ext *pepext; 3663213379Shselasky struct usb_endpoint_descriptor *edesc; 3664213379Shselasky struct usb_page_cache *pcinp; 3665213379Shselasky usb_error_t err; 3666213379Shselasky uint8_t index; 3667213379Shselasky uint8_t epno; 3668213379Shselasky 3669213379Shselasky pepext = xhci_get_endpoint_ext(xfer->xroot->udev, 3670213379Shselasky xfer->endpoint->edesc); 3671213379Shselasky 3672213379Shselasky udev = xfer->xroot->udev; 3673213379Shselasky index = udev->controller_slot_id; 3674213379Shselasky 3675213379Shselasky pcinp = &sc->sc_hw.devs[index].input_pc; 3676213379Shselasky 3677213379Shselasky usbd_get_page(pcinp, 0, &buf_inp); 3678213379Shselasky 3679213379Shselasky edesc = xfer->endpoint->edesc; 3680213379Shselasky 3681213379Shselasky epno = edesc->bEndpointAddress; 3682213379Shselasky 3683213379Shselasky if ((edesc->bmAttributes & UE_XFERTYPE) == UE_CONTROL) 3684213379Shselasky epno |= UE_DIR_IN; 3685213379Shselasky 3686213379Shselasky epno = XHCI_EPNO2EPID(epno); 3687213379Shselasky 3688213379Shselasky if (epno == 0) 3689213379Shselasky return (USB_ERR_NO_PIPE); /* invalid */ 3690213379Shselasky 3691213379Shselasky XHCI_CMD_LOCK(sc); 3692213379Shselasky 3693213379Shselasky /* configure endpoint */ 3694213379Shselasky 3695213379Shselasky err = xhci_configure_endpoint_by_xfer(xfer); 3696213379Shselasky 3697213379Shselasky if (err != 0) { 3698213379Shselasky XHCI_CMD_UNLOCK(sc); 3699213379Shselasky return (err); 3700213379Shselasky } 3701213379Shselasky 3702213379Shselasky /* 3703213379Shselasky * Get the endpoint into the stopped state according to the 3704213379Shselasky * endpoint context state diagram in the XHCI specification: 3705213379Shselasky */ 3706213379Shselasky 3707213379Shselasky err = xhci_cmd_stop_ep(sc, 0, epno, index); 3708213379Shselasky 3709213379Shselasky if (err != 0) 3710213379Shselasky DPRINTF("Could not stop endpoint %u\n", epno); 3711213379Shselasky 3712213379Shselasky err = xhci_cmd_reset_ep(sc, 0, epno, index); 3713213379Shselasky 3714213379Shselasky if (err != 0) 3715213379Shselasky DPRINTF("Could not reset endpoint %u\n", epno); 3716213379Shselasky 3717213379Shselasky err = xhci_cmd_set_tr_dequeue_ptr(sc, pepext->physaddr | 3718213379Shselasky XHCI_EPCTX_2_DCS_SET(1), 0, epno, index); 3719213379Shselasky 3720213379Shselasky if (err != 0) 3721213379Shselasky DPRINTF("Could not set dequeue ptr for endpoint %u\n", epno); 3722213379Shselasky 3723213379Shselasky /* 3724213379Shselasky * Get the endpoint into the running state according to the 3725213379Shselasky * endpoint context state diagram in the XHCI specification: 3726213379Shselasky */ 3727213379Shselasky 3728245731Shselasky xhci_configure_mask(udev, (1U << epno) | 1U, 0); 3729213379Shselasky 3730213379Shselasky err = xhci_cmd_evaluate_ctx(sc, buf_inp.physaddr, index); 3731213379Shselasky 3732213379Shselasky if (err != 0) 3733213379Shselasky DPRINTF("Could not configure endpoint %u\n", epno); 3734213379Shselasky 3735213379Shselasky err = xhci_cmd_configure_ep(sc, buf_inp.physaddr, 0, index); 3736213379Shselasky 3737213379Shselasky if (err != 0) 3738213379Shselasky DPRINTF("Could not configure endpoint %u\n", epno); 3739213379Shselasky 3740213379Shselasky XHCI_CMD_UNLOCK(sc); 3741213379Shselasky 3742213379Shselasky return (0); 3743213379Shselasky} 3744213379Shselasky 3745213379Shselaskystatic void 3746213379Shselaskyxhci_xfer_unsetup(struct usb_xfer *xfer) 3747213379Shselasky{ 3748213379Shselasky return; 3749213379Shselasky} 3750213379Shselasky 3751213379Shselaskystatic void 3752213379Shselaskyxhci_start_dma_delay(struct usb_xfer *xfer) 3753213379Shselasky{ 3754213379Shselasky struct xhci_softc *sc = XHCI_BUS2SC(xfer->xroot->bus); 3755213379Shselasky 3756213379Shselasky /* put transfer on interrupt queue (again) */ 3757213379Shselasky usbd_transfer_enqueue(&sc->sc_bus.intr_q, xfer); 3758213379Shselasky 3759213379Shselasky (void)usb_proc_msignal(&sc->sc_config_proc, 3760213379Shselasky &sc->sc_config_msg[0], &sc->sc_config_msg[1]); 3761213379Shselasky} 3762213379Shselasky 3763213379Shselaskystatic void 3764213379Shselaskyxhci_configure_msg(struct usb_proc_msg *pm) 3765213379Shselasky{ 3766213379Shselasky struct xhci_softc *sc; 3767213379Shselasky struct xhci_endpoint_ext *pepext; 3768213379Shselasky struct usb_xfer *xfer; 3769213379Shselasky 3770213379Shselasky sc = XHCI_BUS2SC(((struct usb_bus_msg *)pm)->bus); 3771213379Shselasky 3772213379Shselaskyrestart: 3773213379Shselasky TAILQ_FOREACH(xfer, &sc->sc_bus.intr_q.head, wait_entry) { 3774213379Shselasky 3775213379Shselasky pepext = xhci_get_endpoint_ext(xfer->xroot->udev, 3776213379Shselasky xfer->endpoint->edesc); 3777213379Shselasky 3778213379Shselasky if ((pepext->trb_halted != 0) || 3779213379Shselasky (pepext->trb_running == 0)) { 3780213379Shselasky 3781213379Shselasky uint8_t i; 3782213379Shselasky 3783213379Shselasky /* clear halted and running */ 3784213379Shselasky pepext->trb_halted = 0; 3785213379Shselasky pepext->trb_running = 0; 3786213379Shselasky 3787213379Shselasky /* nuke remaining buffered transfers */ 3788213379Shselasky 3789213379Shselasky for (i = 0; i != (XHCI_MAX_TRANSFERS - 1); i++) { 3790213379Shselasky /* 3791213379Shselasky * NOTE: We need to use the timeout 3792213379Shselasky * error code here else existing 3793213379Shselasky * isochronous clients can get 3794213379Shselasky * confused: 3795213379Shselasky */ 3796213379Shselasky if (pepext->xfer[i] != NULL) { 3797213379Shselasky xhci_device_done(pepext->xfer[i], 3798213379Shselasky USB_ERR_TIMEOUT); 3799213379Shselasky } 3800213379Shselasky } 3801213379Shselasky 3802213379Shselasky /* 3803213379Shselasky * NOTE: The USB transfer cannot vanish in 3804213379Shselasky * this state! 3805213379Shselasky */ 3806213379Shselasky 3807213379Shselasky USB_BUS_UNLOCK(&sc->sc_bus); 3808213379Shselasky 3809213379Shselasky xhci_configure_reset_endpoint(xfer); 3810213379Shselasky 3811213379Shselasky USB_BUS_LOCK(&sc->sc_bus); 3812213379Shselasky 3813213379Shselasky /* check if halted is still cleared */ 3814213379Shselasky if (pepext->trb_halted == 0) { 3815213379Shselasky pepext->trb_running = 1; 3816213379Shselasky pepext->trb_index = 0; 3817213379Shselasky } 3818213379Shselasky goto restart; 3819213379Shselasky } 3820213379Shselasky 3821213379Shselasky if (xfer->flags_int.did_dma_delay) { 3822213379Shselasky 3823213379Shselasky /* remove transfer from interrupt queue (again) */ 3824213379Shselasky usbd_transfer_dequeue(xfer); 3825213379Shselasky 3826213379Shselasky /* we are finally done */ 3827213379Shselasky usb_dma_delay_done_cb(xfer); 3828213379Shselasky 3829213379Shselasky /* queue changed - restart */ 3830213379Shselasky goto restart; 3831213379Shselasky } 3832213379Shselasky } 3833213379Shselasky 3834213379Shselasky TAILQ_FOREACH(xfer, &sc->sc_bus.intr_q.head, wait_entry) { 3835213379Shselasky 3836213379Shselasky /* try to insert xfer on HW queue */ 3837213379Shselasky xhci_transfer_insert(xfer); 3838213379Shselasky 3839213379Shselasky /* try to multi buffer */ 3840213379Shselasky xhci_device_generic_multi_enter(xfer->endpoint, NULL); 3841213379Shselasky } 3842213379Shselasky} 3843213379Shselasky 3844213379Shselaskystatic void 3845213379Shselaskyxhci_ep_init(struct usb_device *udev, struct usb_endpoint_descriptor *edesc, 3846213379Shselasky struct usb_endpoint *ep) 3847213379Shselasky{ 3848213379Shselasky struct xhci_endpoint_ext *pepext; 3849213379Shselasky 3850213379Shselasky DPRINTFN(2, "endpoint=%p, addr=%d, endpt=%d, mode=%d\n", 3851213379Shselasky ep, udev->address, edesc->bEndpointAddress, udev->flags.usb_mode); 3852213379Shselasky 3853213379Shselasky if (udev->parent_hub == NULL) { 3854213379Shselasky /* root HUB has special endpoint handling */ 3855213379Shselasky return; 3856213379Shselasky } 3857213379Shselasky 3858213379Shselasky ep->methods = &xhci_device_generic_methods; 3859213379Shselasky 3860213379Shselasky pepext = xhci_get_endpoint_ext(udev, edesc); 3861213379Shselasky 3862213379Shselasky USB_BUS_LOCK(udev->bus); 3863213379Shselasky pepext->trb_halted = 1; 3864213379Shselasky pepext->trb_running = 0; 3865213379Shselasky USB_BUS_UNLOCK(udev->bus); 3866213379Shselasky} 3867213379Shselasky 3868213379Shselaskystatic void 3869213379Shselaskyxhci_ep_uninit(struct usb_device *udev, struct usb_endpoint *ep) 3870213379Shselasky{ 3871213379Shselasky 3872213379Shselasky} 3873213379Shselasky 3874213379Shselaskystatic void 3875213379Shselaskyxhci_ep_clear_stall(struct usb_device *udev, struct usb_endpoint *ep) 3876213379Shselasky{ 3877213379Shselasky struct xhci_endpoint_ext *pepext; 3878213379Shselasky 3879213379Shselasky DPRINTF("\n"); 3880213379Shselasky 3881213379Shselasky if (udev->flags.usb_mode != USB_MODE_HOST) { 3882213379Shselasky /* not supported */ 3883213379Shselasky return; 3884213379Shselasky } 3885213379Shselasky if (udev->parent_hub == NULL) { 3886213379Shselasky /* root HUB has special endpoint handling */ 3887213379Shselasky return; 3888213379Shselasky } 3889213379Shselasky 3890213379Shselasky pepext = xhci_get_endpoint_ext(udev, ep->edesc); 3891213379Shselasky 3892213379Shselasky USB_BUS_LOCK(udev->bus); 3893213379Shselasky pepext->trb_halted = 1; 3894213379Shselasky pepext->trb_running = 0; 3895213379Shselasky USB_BUS_UNLOCK(udev->bus); 3896213379Shselasky} 3897213379Shselasky 3898213379Shselaskystatic usb_error_t 3899213379Shselaskyxhci_device_init(struct usb_device *udev) 3900213379Shselasky{ 3901213379Shselasky struct xhci_softc *sc = XHCI_BUS2SC(udev->bus); 3902213379Shselasky usb_error_t err; 3903213379Shselasky uint8_t temp; 3904213379Shselasky 3905213379Shselasky /* no init for root HUB */ 3906213379Shselasky if (udev->parent_hub == NULL) 3907213379Shselasky return (0); 3908213379Shselasky 3909213379Shselasky XHCI_CMD_LOCK(sc); 3910213379Shselasky 3911213379Shselasky /* set invalid default */ 3912213379Shselasky 3913213379Shselasky udev->controller_slot_id = sc->sc_noslot + 1; 3914213379Shselasky 3915213379Shselasky /* try to get a new slot ID from the XHCI */ 3916213379Shselasky 3917213379Shselasky err = xhci_cmd_enable_slot(sc, &temp); 3918213379Shselasky 3919213379Shselasky if (err) { 3920213379Shselasky XHCI_CMD_UNLOCK(sc); 3921213379Shselasky return (err); 3922213379Shselasky } 3923213379Shselasky 3924213379Shselasky if (temp > sc->sc_noslot) { 3925213379Shselasky XHCI_CMD_UNLOCK(sc); 3926213379Shselasky return (USB_ERR_BAD_ADDRESS); 3927213379Shselasky } 3928213379Shselasky 3929213379Shselasky if (sc->sc_hw.devs[temp].state != XHCI_ST_DISABLED) { 3930213379Shselasky DPRINTF("slot %u already allocated.\n", temp); 3931213379Shselasky XHCI_CMD_UNLOCK(sc); 3932213379Shselasky return (USB_ERR_BAD_ADDRESS); 3933213379Shselasky } 3934213379Shselasky 3935213379Shselasky /* store slot ID for later reference */ 3936213379Shselasky 3937213379Shselasky udev->controller_slot_id = temp; 3938213379Shselasky 3939213379Shselasky /* reset data structure */ 3940213379Shselasky 3941213379Shselasky memset(&sc->sc_hw.devs[temp], 0, sizeof(sc->sc_hw.devs[0])); 3942213379Shselasky 3943213379Shselasky /* set mark slot allocated */ 3944213379Shselasky 3945213379Shselasky sc->sc_hw.devs[temp].state = XHCI_ST_ENABLED; 3946213379Shselasky 3947213379Shselasky err = xhci_alloc_device_ext(udev); 3948213379Shselasky 3949213379Shselasky XHCI_CMD_UNLOCK(sc); 3950213379Shselasky 3951213379Shselasky /* get device into default state */ 3952213379Shselasky 3953213379Shselasky if (err == 0) 3954213379Shselasky err = xhci_set_address(udev, NULL, 0); 3955213379Shselasky 3956213379Shselasky return (err); 3957213379Shselasky} 3958213379Shselasky 3959213379Shselaskystatic void 3960213379Shselaskyxhci_device_uninit(struct usb_device *udev) 3961213379Shselasky{ 3962213379Shselasky struct xhci_softc *sc = XHCI_BUS2SC(udev->bus); 3963213379Shselasky uint8_t index; 3964213379Shselasky 3965213379Shselasky /* no init for root HUB */ 3966213379Shselasky if (udev->parent_hub == NULL) 3967213379Shselasky return; 3968213379Shselasky 3969213379Shselasky XHCI_CMD_LOCK(sc); 3970213379Shselasky 3971213379Shselasky index = udev->controller_slot_id; 3972213379Shselasky 3973213379Shselasky if (index <= sc->sc_noslot) { 3974213379Shselasky xhci_cmd_disable_slot(sc, index); 3975213379Shselasky sc->sc_hw.devs[index].state = XHCI_ST_DISABLED; 3976213379Shselasky 3977213379Shselasky /* free device extension */ 3978213379Shselasky xhci_free_device_ext(udev); 3979213379Shselasky } 3980213379Shselasky 3981213379Shselasky XHCI_CMD_UNLOCK(sc); 3982213379Shselasky} 3983213379Shselasky 3984213379Shselaskystatic void 3985213379Shselaskyxhci_get_dma_delay(struct usb_device *udev, uint32_t *pus) 3986213379Shselasky{ 3987213379Shselasky /* 3988213379Shselasky * Wait until the hardware has finished any possible use of 3989213379Shselasky * the transfer descriptor(s) 3990213379Shselasky */ 3991213379Shselasky *pus = 2048; /* microseconds */ 3992213379Shselasky} 3993213379Shselasky 3994213379Shselaskystatic void 3995213379Shselaskyxhci_device_resume(struct usb_device *udev) 3996213379Shselasky{ 3997213379Shselasky struct xhci_softc *sc = XHCI_BUS2SC(udev->bus); 3998213379Shselasky uint8_t index; 3999213379Shselasky uint8_t n; 4000245731Shselasky uint8_t p; 4001213379Shselasky 4002213379Shselasky DPRINTF("\n"); 4003213379Shselasky 4004213379Shselasky /* check for root HUB */ 4005213379Shselasky if (udev->parent_hub == NULL) 4006213379Shselasky return; 4007213379Shselasky 4008213379Shselasky index = udev->controller_slot_id; 4009213379Shselasky 4010213379Shselasky XHCI_CMD_LOCK(sc); 4011213379Shselasky 4012213379Shselasky /* blindly resume all endpoints */ 4013213379Shselasky 4014213379Shselasky USB_BUS_LOCK(udev->bus); 4015213379Shselasky 4016245731Shselasky for (n = 1; n != XHCI_MAX_ENDPOINTS; n++) { 4017245731Shselasky for (p = 0; p != 1 /*XHCI_MAX_STREAMS*/; p++) { 4018245731Shselasky XWRITE4(sc, door, XHCI_DOORBELL(index), 4019245731Shselasky n | XHCI_DB_SID_SET(p)); 4020245731Shselasky } 4021245731Shselasky } 4022213379Shselasky 4023213379Shselasky USB_BUS_UNLOCK(udev->bus); 4024213379Shselasky 4025213379Shselasky XHCI_CMD_UNLOCK(sc); 4026213379Shselasky} 4027213379Shselasky 4028213379Shselaskystatic void 4029213379Shselaskyxhci_device_suspend(struct usb_device *udev) 4030213379Shselasky{ 4031213379Shselasky struct xhci_softc *sc = XHCI_BUS2SC(udev->bus); 4032213379Shselasky uint8_t index; 4033213379Shselasky uint8_t n; 4034213379Shselasky usb_error_t err; 4035213379Shselasky 4036213379Shselasky DPRINTF("\n"); 4037213379Shselasky 4038213379Shselasky /* check for root HUB */ 4039213379Shselasky if (udev->parent_hub == NULL) 4040213379Shselasky return; 4041213379Shselasky 4042213379Shselasky index = udev->controller_slot_id; 4043213379Shselasky 4044213379Shselasky XHCI_CMD_LOCK(sc); 4045213379Shselasky 4046213379Shselasky /* blindly suspend all endpoints */ 4047213379Shselasky 4048213379Shselasky for (n = 1; n != XHCI_MAX_ENDPOINTS; n++) { 4049213379Shselasky err = xhci_cmd_stop_ep(sc, 1, n, index); 4050213379Shselasky if (err != 0) { 4051213379Shselasky DPRINTF("Failed to suspend endpoint " 4052213379Shselasky "%u on slot %u (ignored).\n", n, index); 4053213379Shselasky } 4054213379Shselasky } 4055213379Shselasky 4056213379Shselasky XHCI_CMD_UNLOCK(sc); 4057213379Shselasky} 4058213379Shselasky 4059213379Shselaskystatic void 4060213379Shselaskyxhci_set_hw_power(struct usb_bus *bus) 4061213379Shselasky{ 4062213379Shselasky DPRINTF("\n"); 4063213379Shselasky} 4064213379Shselasky 4065213379Shselaskystatic void 4066213379Shselaskyxhci_device_state_change(struct usb_device *udev) 4067213379Shselasky{ 4068213379Shselasky struct xhci_softc *sc = XHCI_BUS2SC(udev->bus); 4069213379Shselasky struct usb_page_search buf_inp; 4070213379Shselasky usb_error_t err; 4071213379Shselasky uint8_t index; 4072213379Shselasky 4073213379Shselasky /* check for root HUB */ 4074213379Shselasky if (udev->parent_hub == NULL) 4075213379Shselasky return; 4076213379Shselasky 4077213379Shselasky index = udev->controller_slot_id; 4078213379Shselasky 4079213379Shselasky DPRINTF("\n"); 4080213379Shselasky 4081213379Shselasky if (usb_get_device_state(udev) == USB_STATE_CONFIGURED) { 4082213379Shselasky err = uhub_query_info(udev, &sc->sc_hw.devs[index].nports, 4083213379Shselasky &sc->sc_hw.devs[index].tt); 4084213379Shselasky if (err != 0) 4085213379Shselasky sc->sc_hw.devs[index].nports = 0; 4086213379Shselasky } 4087213379Shselasky 4088213379Shselasky XHCI_CMD_LOCK(sc); 4089213379Shselasky 4090213379Shselasky switch (usb_get_device_state(udev)) { 4091213379Shselasky case USB_STATE_POWERED: 4092213379Shselasky if (sc->sc_hw.devs[index].state == XHCI_ST_DEFAULT) 4093213379Shselasky break; 4094213379Shselasky 4095245731Shselasky /* set default state */ 4096213379Shselasky sc->sc_hw.devs[index].state = XHCI_ST_DEFAULT; 4097213379Shselasky 4098245731Shselasky /* reset number of contexts */ 4099245731Shselasky sc->sc_hw.devs[index].context_num = 0; 4100245731Shselasky 4101213379Shselasky err = xhci_cmd_reset_dev(sc, index); 4102213379Shselasky 4103213379Shselasky if (err != 0) { 4104213379Shselasky DPRINTF("Device reset failed " 4105213379Shselasky "for slot %u.\n", index); 4106213379Shselasky } 4107213379Shselasky break; 4108213379Shselasky 4109213379Shselasky case USB_STATE_ADDRESSED: 4110213379Shselasky if (sc->sc_hw.devs[index].state == XHCI_ST_ADDRESSED) 4111213379Shselasky break; 4112213379Shselasky 4113213379Shselasky sc->sc_hw.devs[index].state = XHCI_ST_ADDRESSED; 4114213379Shselasky 4115213379Shselasky err = xhci_cmd_configure_ep(sc, 0, 1, index); 4116213379Shselasky 4117213379Shselasky if (err) { 4118213379Shselasky DPRINTF("Failed to deconfigure " 4119213379Shselasky "slot %u.\n", index); 4120213379Shselasky } 4121213379Shselasky break; 4122213379Shselasky 4123213379Shselasky case USB_STATE_CONFIGURED: 4124213379Shselasky if (sc->sc_hw.devs[index].state == XHCI_ST_CONFIGURED) 4125213379Shselasky break; 4126213379Shselasky 4127245731Shselasky /* set configured state */ 4128213379Shselasky sc->sc_hw.devs[index].state = XHCI_ST_CONFIGURED; 4129213379Shselasky 4130245731Shselasky /* reset number of contexts */ 4131245731Shselasky sc->sc_hw.devs[index].context_num = 0; 4132245731Shselasky 4133213379Shselasky usbd_get_page(&sc->sc_hw.devs[index].input_pc, 0, &buf_inp); 4134213379Shselasky 4135245731Shselasky xhci_configure_mask(udev, 3, 0); 4136213379Shselasky 4137213379Shselasky err = xhci_configure_device(udev); 4138213379Shselasky if (err != 0) { 4139213379Shselasky DPRINTF("Could not configure device " 4140213379Shselasky "at slot %u.\n", index); 4141213379Shselasky } 4142213379Shselasky 4143213379Shselasky err = xhci_cmd_evaluate_ctx(sc, buf_inp.physaddr, index); 4144213379Shselasky if (err != 0) { 4145213379Shselasky DPRINTF("Could not evaluate device " 4146213379Shselasky "context at slot %u.\n", index); 4147213379Shselasky } 4148213379Shselasky break; 4149213379Shselasky 4150213379Shselasky default: 4151213379Shselasky break; 4152213379Shselasky } 4153213379Shselasky XHCI_CMD_UNLOCK(sc); 4154213379Shselasky} 4155213379Shselasky 4156213379Shselaskystruct usb_bus_methods xhci_bus_methods = { 4157213379Shselasky .endpoint_init = xhci_ep_init, 4158213379Shselasky .endpoint_uninit = xhci_ep_uninit, 4159213379Shselasky .xfer_setup = xhci_xfer_setup, 4160213379Shselasky .xfer_unsetup = xhci_xfer_unsetup, 4161213379Shselasky .get_dma_delay = xhci_get_dma_delay, 4162213379Shselasky .device_init = xhci_device_init, 4163213379Shselasky .device_uninit = xhci_device_uninit, 4164213379Shselasky .device_resume = xhci_device_resume, 4165213379Shselasky .device_suspend = xhci_device_suspend, 4166213379Shselasky .set_hw_power = xhci_set_hw_power, 4167213379Shselasky .roothub_exec = xhci_roothub_exec, 4168213379Shselasky .xfer_poll = xhci_do_poll, 4169213379Shselasky .start_dma_delay = xhci_start_dma_delay, 4170213379Shselasky .set_address = xhci_set_address, 4171213379Shselasky .clear_stall = xhci_ep_clear_stall, 4172213379Shselasky .device_state_change = xhci_device_state_change, 4173229096Shselasky .set_hw_power_sleep = xhci_set_hw_power_sleep, 4174213379Shselasky}; 4175