1// Copyright 2018 The Fuchsia Authors. All rights reserved. 2// Use of this source code is governed by a BSD-style license that can be 3// found in the LICENSE file. 4 5#include <ddk/debug.h> 6 7#include "xdc.h" 8#include "xdc-transfer.h" 9 10// Reads a range of bits from an integer. 11#define READ_FIELD(i, start, bits) (((i) >> (start)) & ((1 << (bits)) - 1)) 12 13static void xdc_ring_doorbell(xdc_t* xdc, xdc_endpoint_t* ep) { 14 uint8_t doorbell_val = ep->direction == USB_DIR_IN ? DCDB_DB_EP_IN : DCDB_DB_EP_OUT; 15 XHCI_SET_BITS32(&xdc->debug_cap_regs->dcdb, DCDB_DB_START, DCDB_DB_BITS, doorbell_val); 16} 17 18// Stores the value of the Dequeue Pointer into out_dequeue. 19// Returns ZX_OK if successful, or ZX_ERR_BAD_STATE if the endpoint was not in the Stopped state. 20static zx_status_t xdc_get_dequeue_ptr_locked(xdc_t* xdc, xdc_endpoint_t* ep, 21 uint64_t* out_dequeue) __TA_REQUIRES(xdc->lock) { 22 if (ep->state != XDC_EP_STATE_STOPPED) { 23 zxlogf(ERROR, "tried to read dequeue pointer of %s EP while not stopped, state is: %d\n", 24 ep->name, ep->state); 25 return ZX_ERR_BAD_STATE; 26 } 27 xdc_context_data_t* ctx = xdc->context_data; 28 xhci_endpoint_context_t* epc = ep->direction == USB_DIR_OUT ? &ctx->out_epc : &ctx->in_epc; 29 30 uint64_t dequeue_ptr_hi = XHCI_READ32(&epc->tr_dequeue_hi); 31 uint32_t dequeue_ptr_lo = XHCI_READ32(&epc->epc2) & EP_CTX_TR_DEQUEUE_LO_MASK; 32 *out_dequeue = (dequeue_ptr_hi << 32 | dequeue_ptr_lo); 33 return ZX_OK; 34} 35 36// Returns ZX_OK if the request was scheduled successfully, or ZX_ERR_SHOULD_WAIT 37// if we ran out of TRBs. 38static zx_status_t xdc_schedule_transfer_locked(xdc_t* xdc, xdc_endpoint_t* ep, 39 usb_request_t* req) __TA_REQUIRES(xdc->lock) { 40 xhci_transfer_ring_t* ring = &ep->transfer_ring; 41 42 // Need to clean the cache for both IN and OUT transfers, invalidate only for IN. 43 if (ep->direction == USB_DIR_IN) { 44 usb_request_cache_flush_invalidate(req, 0, req->header.length); 45 } else { 46 usb_request_cache_flush(req, 0, req->header.length); 47 } 48 49 zx_status_t status = xhci_queue_data_trbs(ring, &ep->transfer_state, req, 50 0 /* interrupter */, false /* isochronous */); 51 if (status != ZX_OK) { 52 return status; 53 } 54 55 // If we get here, then we are ready to ring the doorbell. 56 // Save the ring position so we can update the ring dequeue ptr once the transfer completes. 57 req->context = (void *)ring->current; 58 xdc_ring_doorbell(xdc, ep); 59 60 return ZX_OK; 61} 62 63// Schedules any queued requests on the endpoint's transfer ring, until we fill our 64// transfer ring or have no more requests. 65void xdc_process_transactions_locked(xdc_t* xdc, xdc_endpoint_t* ep) __TA_REQUIRES(xdc->lock) { 66 while (1) { 67 if (xhci_transfer_ring_free_trbs(&ep->transfer_ring) == 0) { 68 // No available TRBs - need to wait for some to complete. 69 return; 70 } 71 72 if (!ep->current_req) { 73 // Start the next transaction in the queue. 74 usb_request_t* req = list_remove_head_type(&ep->queued_reqs, usb_request_t, node); 75 if (!req) { 76 // No requests waiting. 77 return; 78 } 79 xhci_transfer_state_init(&ep->transfer_state, req, 80 USB_ENDPOINT_BULK, EP_CTX_MAX_PACKET_SIZE); 81 list_add_tail(&ep->pending_reqs, &req->node); 82 ep->current_req = req; 83 } 84 85 usb_request_t* req = ep->current_req; 86 zx_status_t status = xdc_schedule_transfer_locked(xdc, ep, req); 87 if (status == ZX_ERR_SHOULD_WAIT) { 88 // No available TRBs - need to wait for some to complete. 89 return; 90 } else { 91 ep->current_req = NULL; 92 } 93 } 94} 95 96zx_status_t xdc_queue_transfer(xdc_t* xdc, usb_request_t* req, bool in, bool is_ctrl_msg) { 97 xdc_endpoint_t* ep = in ? &xdc->eps[IN_EP_IDX] : &xdc->eps[OUT_EP_IDX]; 98 99 mtx_lock(&xdc->lock); 100 101 // We should always queue control messages unless there is an unrecoverable error. 102 if (!is_ctrl_msg && (!xdc->configured || ep->state == XDC_EP_STATE_DEAD)) { 103 mtx_unlock(&xdc->lock); 104 return ZX_ERR_IO_NOT_PRESENT; 105 } 106 107 if (req->header.length > 0) { 108 zx_status_t status = usb_request_physmap(req, xdc->bti_handle); 109 if (status != ZX_OK) { 110 zxlogf(ERROR, "%s: usb_request_physmap failed: %d\n", __FUNCTION__, status); 111 mtx_unlock(&xdc->lock); 112 return status; 113 } 114 } 115 116 list_add_tail(&ep->queued_reqs, &req->node); 117 118 // We can still queue requests for later while waiting for the xdc device to be configured, 119 // or while the endpoint is halted. Before scheduling the TRBs however, we should wait 120 // for the device to be configured, and/or the halt is cleared by DbC and we've cleaned 121 // up the transfer ring. 122 if (xdc->configured && ep->state == XDC_EP_STATE_RUNNING) { 123 xdc_process_transactions_locked(xdc, ep); 124 } 125 126 mtx_unlock(&xdc->lock); 127 128 return ZX_OK; 129} 130 131bool xdc_has_free_trbs(xdc_t* xdc, bool in) { 132 mtx_lock(&xdc->lock); 133 134 xdc_endpoint_t* ep = in ? &xdc->eps[IN_EP_IDX] : &xdc->eps[OUT_EP_IDX]; 135 bool has_trbs = xhci_transfer_ring_free_trbs(&ep->transfer_ring) > 0; 136 137 mtx_unlock(&xdc->lock); 138 return has_trbs; 139} 140 141zx_status_t xdc_restart_transfer_ring_locked(xdc_t* xdc, xdc_endpoint_t* ep) { 142 // Once the DbC clears the halt flag for the endpoint, the address stored in the 143 // TR Dequeue Pointer field is the next TRB to be executed (see XHCI Spec 7.6.4.3). 144 // There seems to be no guarantee which TRB this will point to. 145 // 146 // The easiest way to deal with this is to convert all scheduled TRBs to NO-OPs, 147 // and reschedule pending requests. 148 149 uint64_t dequeue_ptr; 150 zx_status_t status = xdc_get_dequeue_ptr_locked(xdc, ep, &dequeue_ptr); 151 if (status != ZX_OK) { 152 return status; 153 } 154 xhci_transfer_ring_t* ring = &ep->transfer_ring; 155 xhci_trb_t* trb = xhci_transfer_ring_phys_to_trb(ring, dequeue_ptr); 156 if (!trb) { 157 zxlogf(ERROR, "no valid TRB corresponding to dequeue_ptr: %lu\n", dequeue_ptr); 158 return ZX_ERR_BAD_STATE; 159 } 160 161 // Reset our copy of the dequeue pointer. 162 xhci_set_dequeue_ptr(ring, trb); 163 164 // Convert all pending TRBs on the transfer ring into NO-OPs TRBs. 165 // ring->current is just after our last queued TRB. 166 xhci_trb_t* last_trb = NULL; 167 while (trb != ring->current) { 168 xhci_set_transfer_noop_trb(trb); 169 last_trb = trb; 170 trb = xhci_get_next_trb(ring, trb); 171 } 172 if (last_trb) { 173 // Set IOC (Interrupt on Completion) on the last NO-OP TRB, so we know 174 // when we can overwrite them in the transfer ring. 175 uint32_t control = XHCI_READ32(&last_trb->control); 176 XHCI_WRITE32(&last_trb->control, control | XFER_TRB_IOC); 177 } 178 // Restart the transfer ring. 179 xdc_ring_doorbell(xdc, ep); 180 ep->state = XDC_EP_STATE_RUNNING; 181 182 // Requeue and reschedule the requests. 183 usb_request_t* req; 184 while ((req = list_remove_tail_type(&ep->pending_reqs, usb_request_t, node)) != NULL) { 185 list_add_head(&ep->queued_reqs, &req->node); 186 } 187 xdc_process_transactions_locked(xdc, ep); 188 return ZX_OK; 189} 190 191void xdc_handle_transfer_event_locked(xdc_t* xdc, xdc_poll_state_t* poll_state, xhci_trb_t* trb) { 192 uint32_t control = XHCI_READ32(&trb->control); 193 uint32_t status = XHCI_READ32(&trb->status); 194 uint32_t ep_dev_ctx_idx = READ_FIELD(control, TRB_ENDPOINT_ID_START, TRB_ENDPOINT_ID_BITS); 195 uint8_t xdc_ep_idx = ep_dev_ctx_idx == EP_IN_DEV_CTX_IDX ? IN_EP_IDX : OUT_EP_IDX; 196 xdc_endpoint_t* ep = &xdc->eps[xdc_ep_idx]; 197 xhci_transfer_ring_t* ring = &ep->transfer_ring; 198 199 uint32_t cc = READ_FIELD(status, EVT_TRB_CC_START, EVT_TRB_CC_BITS); 200 uint32_t length = READ_FIELD(status, EVT_TRB_XFER_LENGTH_START, EVT_TRB_XFER_LENGTH_BITS); 201 usb_request_t* req = NULL; 202 bool error = false; 203 204 switch (cc) { 205 case TRB_CC_SUCCESS: 206 case TRB_CC_SHORT_PACKET: 207 break; 208 case TRB_CC_BABBLE_DETECTED_ERROR: 209 case TRB_CC_USB_TRANSACTION_ERROR: 210 case TRB_CC_TRB_ERROR: 211 case TRB_CC_STALL_ERROR: 212 zxlogf(ERROR, "xdc_handle_transfer_event: error condition code: %d\n", cc); 213 error = true; 214 break; 215 default: 216 zxlogf(ERROR, "xdc_handle_transfer_event: unexpected condition code %d\n", cc); 217 error = true; 218 break; 219 } 220 221 // Even though the main poll loop checks for changes in the halt registers, 222 // it's possible we missed the halt register being set if the halt was cleared fast enough. 223 if (error) { 224 if (ep->state == XDC_EP_STATE_RUNNING) { 225 xdc_endpoint_set_halt_locked(xdc, poll_state, ep); 226 } 227 ep->got_err_event = true; 228 // We're going to requeue the transfer when we restart the transfer ring, 229 // so nothing else to do. 230 return; 231 } 232 233 if (control & EVT_TRB_ED) { 234 // An Event Data TRB generated the completion event, so the TRB Pointer field 235 // will contain the usb request pointer we previously stored. 236 req = (usb_request_t *)trb_get_ptr(trb); 237 } else { 238 // Get the pointer to the TRB that generated the event. 239 trb = xhci_read_trb_ptr(ring, trb); 240 if (trb_get_type(trb) == TRB_TRANSFER_NOOP) { 241 // If it's the NO-OP TRB we queued when dealing with the halt condition, 242 // there won't be a corresponding usb request. 243 zxlogf(TRACE, "xdc_handle_transfer_event: got a NO-OP TRB\n"); 244 xhci_set_dequeue_ptr(ring, xhci_get_next_trb(ring, trb)); 245 xdc_process_transactions_locked(xdc, ep); 246 return; 247 } 248 249 // Look for the Event Data TRB which will have the usb request pointer. 250 for (uint i = 0; i < TRANSFER_RING_SIZE && trb; i++) { 251 if (trb_get_type(trb) == TRB_TRANSFER_EVENT_DATA) { 252 req = (usb_request_t *)trb_get_ptr(trb); 253 break; 254 } 255 trb = xhci_get_next_trb(ring, trb); 256 } 257 } 258 259 if (!req) { 260 zxlogf(ERROR, "xdc_handle_transfer_event: unable to find request to complete\n"); 261 return; 262 } 263 264 // Find the usb request in the pending list. 265 bool found_req = false; 266 usb_request_t* test; 267 list_for_every_entry(&ep->pending_reqs, test, usb_request_t, node) { 268 if (test == req) { 269 found_req = true; 270 break; 271 } 272 } 273 if (!found_req) { 274 zxlogf(ERROR, "xdc_handle_transfer_event: ignoring event for completed transfer\n"); 275 return; 276 } 277 // Remove request from pending_reqs. 278 list_delete(&req->node); 279 280 // Update our copy of the dequeue_ptr to the TRB following this transaction. 281 xhci_set_dequeue_ptr(ring, req->context); 282 xdc_process_transactions_locked(xdc, ep); 283 284 // Save the request to be completed later out of the lock. 285 req->response.status = ZX_OK; 286 req->response.actual = length; 287 list_add_tail(&poll_state->completed_reqs, &req->node); 288} 289