1// Copyright 2018 The Fuchsia Authors. All rights reserved.
2// Use of this source code is governed by a BSD-style license that can be
3// found in the LICENSE file.
4
5#include <ddk/debug.h>
6
7#include "xdc.h"
8#include "xdc-transfer.h"
9
10// Reads a range of bits from an integer.
11#define READ_FIELD(i, start, bits) (((i) >> (start)) & ((1 << (bits)) - 1))
12
13static void xdc_ring_doorbell(xdc_t* xdc, xdc_endpoint_t* ep) {
14    uint8_t doorbell_val = ep->direction == USB_DIR_IN ? DCDB_DB_EP_IN : DCDB_DB_EP_OUT;
15    XHCI_SET_BITS32(&xdc->debug_cap_regs->dcdb, DCDB_DB_START, DCDB_DB_BITS, doorbell_val);
16}
17
18// Stores the value of the Dequeue Pointer into out_dequeue.
19// Returns ZX_OK if successful, or ZX_ERR_BAD_STATE if the endpoint was not in the Stopped state.
20static zx_status_t xdc_get_dequeue_ptr_locked(xdc_t* xdc, xdc_endpoint_t* ep,
21                                              uint64_t* out_dequeue) __TA_REQUIRES(xdc->lock) {
22    if (ep->state != XDC_EP_STATE_STOPPED) {
23        zxlogf(ERROR, "tried to read dequeue pointer of %s EP while not stopped, state is: %d\n",
24               ep->name, ep->state);
25        return ZX_ERR_BAD_STATE;
26    }
27    xdc_context_data_t* ctx = xdc->context_data;
28    xhci_endpoint_context_t* epc = ep->direction == USB_DIR_OUT ? &ctx->out_epc : &ctx->in_epc;
29
30    uint64_t dequeue_ptr_hi = XHCI_READ32(&epc->tr_dequeue_hi);
31    uint32_t dequeue_ptr_lo = XHCI_READ32(&epc->epc2) & EP_CTX_TR_DEQUEUE_LO_MASK;
32    *out_dequeue = (dequeue_ptr_hi << 32 | dequeue_ptr_lo);
33    return ZX_OK;
34}
35
36// Returns ZX_OK if the request was scheduled successfully, or ZX_ERR_SHOULD_WAIT
37// if we ran out of TRBs.
38static zx_status_t xdc_schedule_transfer_locked(xdc_t* xdc, xdc_endpoint_t* ep,
39                                                usb_request_t* req) __TA_REQUIRES(xdc->lock) {
40    xhci_transfer_ring_t* ring = &ep->transfer_ring;
41
42    // Need to clean the cache for both IN and OUT transfers, invalidate only for IN.
43    if (ep->direction == USB_DIR_IN) {
44        usb_request_cache_flush_invalidate(req, 0, req->header.length);
45    } else {
46        usb_request_cache_flush(req, 0, req->header.length);
47    }
48
49    zx_status_t status = xhci_queue_data_trbs(ring, &ep->transfer_state, req,
50                                              0 /* interrupter */, false /* isochronous */);
51    if (status != ZX_OK) {
52        return status;
53    }
54
55    // If we get here, then we are ready to ring the doorbell.
56    // Save the ring position so we can update the ring dequeue ptr once the transfer completes.
57    req->context = (void *)ring->current;
58    xdc_ring_doorbell(xdc, ep);
59
60    return ZX_OK;
61}
62
63// Schedules any queued requests on the endpoint's transfer ring, until we fill our
64// transfer ring or have no more requests.
65void xdc_process_transactions_locked(xdc_t* xdc, xdc_endpoint_t* ep) __TA_REQUIRES(xdc->lock) {
66    while (1) {
67        if (xhci_transfer_ring_free_trbs(&ep->transfer_ring) == 0) {
68            // No available TRBs - need to wait for some to complete.
69            return;
70        }
71
72        if (!ep->current_req) {
73            // Start the next transaction in the queue.
74            usb_request_t* req = list_remove_head_type(&ep->queued_reqs, usb_request_t, node);
75            if (!req) {
76                // No requests waiting.
77                return;
78            }
79            xhci_transfer_state_init(&ep->transfer_state, req,
80                                     USB_ENDPOINT_BULK, EP_CTX_MAX_PACKET_SIZE);
81            list_add_tail(&ep->pending_reqs, &req->node);
82            ep->current_req = req;
83        }
84
85        usb_request_t* req = ep->current_req;
86        zx_status_t status = xdc_schedule_transfer_locked(xdc, ep, req);
87        if (status == ZX_ERR_SHOULD_WAIT) {
88            // No available TRBs - need to wait for some to complete.
89            return;
90        } else {
91            ep->current_req = NULL;
92        }
93    }
94}
95
96zx_status_t xdc_queue_transfer(xdc_t* xdc, usb_request_t* req, bool in, bool is_ctrl_msg) {
97    xdc_endpoint_t* ep = in ? &xdc->eps[IN_EP_IDX] : &xdc->eps[OUT_EP_IDX];
98
99    mtx_lock(&xdc->lock);
100
101    // We should always queue control messages unless there is an unrecoverable error.
102    if (!is_ctrl_msg && (!xdc->configured || ep->state == XDC_EP_STATE_DEAD)) {
103        mtx_unlock(&xdc->lock);
104        return ZX_ERR_IO_NOT_PRESENT;
105    }
106
107    if (req->header.length > 0) {
108        zx_status_t status = usb_request_physmap(req, xdc->bti_handle);
109        if (status != ZX_OK) {
110            zxlogf(ERROR, "%s: usb_request_physmap failed: %d\n", __FUNCTION__, status);
111            mtx_unlock(&xdc->lock);
112            return status;
113        }
114    }
115
116    list_add_tail(&ep->queued_reqs, &req->node);
117
118    // We can still queue requests for later while waiting for the xdc device to be configured,
119    // or while the endpoint is halted. Before scheduling the TRBs however, we should wait
120    // for the device to be configured, and/or the halt is cleared by DbC and we've cleaned
121    // up the transfer ring.
122    if (xdc->configured && ep->state == XDC_EP_STATE_RUNNING) {
123        xdc_process_transactions_locked(xdc, ep);
124    }
125
126    mtx_unlock(&xdc->lock);
127
128    return ZX_OK;
129}
130
131bool xdc_has_free_trbs(xdc_t* xdc, bool in) {
132    mtx_lock(&xdc->lock);
133
134    xdc_endpoint_t* ep = in ? &xdc->eps[IN_EP_IDX] : &xdc->eps[OUT_EP_IDX];
135    bool has_trbs = xhci_transfer_ring_free_trbs(&ep->transfer_ring) > 0;
136
137    mtx_unlock(&xdc->lock);
138    return has_trbs;
139}
140
141zx_status_t xdc_restart_transfer_ring_locked(xdc_t* xdc, xdc_endpoint_t* ep) {
142    // Once the DbC clears the halt flag for the endpoint, the address stored in the
143    // TR Dequeue Pointer field is the next TRB to be executed (see XHCI Spec 7.6.4.3).
144    // There seems to be no guarantee which TRB this will point to.
145    //
146    // The easiest way to deal with this is to convert all scheduled TRBs to NO-OPs,
147    // and reschedule pending requests.
148
149    uint64_t dequeue_ptr;
150    zx_status_t status = xdc_get_dequeue_ptr_locked(xdc, ep, &dequeue_ptr);
151    if (status != ZX_OK) {
152        return status;
153    }
154    xhci_transfer_ring_t* ring = &ep->transfer_ring;
155    xhci_trb_t* trb = xhci_transfer_ring_phys_to_trb(ring, dequeue_ptr);
156    if (!trb) {
157        zxlogf(ERROR, "no valid TRB corresponding to dequeue_ptr: %lu\n", dequeue_ptr);
158        return ZX_ERR_BAD_STATE;
159    }
160
161    // Reset our copy of the dequeue pointer.
162    xhci_set_dequeue_ptr(ring, trb);
163
164    // Convert all pending TRBs on the transfer ring into NO-OPs TRBs.
165    // ring->current is just after our last queued TRB.
166    xhci_trb_t* last_trb = NULL;
167    while (trb != ring->current) {
168        xhci_set_transfer_noop_trb(trb);
169        last_trb = trb;
170        trb = xhci_get_next_trb(ring, trb);
171    }
172    if (last_trb) {
173        // Set IOC (Interrupt on Completion) on the last NO-OP TRB, so we know
174        // when we can overwrite them in the transfer ring.
175        uint32_t control = XHCI_READ32(&last_trb->control);
176        XHCI_WRITE32(&last_trb->control, control | XFER_TRB_IOC);
177    }
178    // Restart the transfer ring.
179    xdc_ring_doorbell(xdc, ep);
180    ep->state = XDC_EP_STATE_RUNNING;
181
182    // Requeue and reschedule the requests.
183    usb_request_t* req;
184    while ((req = list_remove_tail_type(&ep->pending_reqs, usb_request_t, node)) != NULL) {
185        list_add_head(&ep->queued_reqs, &req->node);
186    }
187    xdc_process_transactions_locked(xdc, ep);
188    return ZX_OK;
189}
190
191void xdc_handle_transfer_event_locked(xdc_t* xdc, xdc_poll_state_t* poll_state, xhci_trb_t* trb) {
192    uint32_t control = XHCI_READ32(&trb->control);
193    uint32_t status = XHCI_READ32(&trb->status);
194    uint32_t ep_dev_ctx_idx = READ_FIELD(control, TRB_ENDPOINT_ID_START, TRB_ENDPOINT_ID_BITS);
195    uint8_t xdc_ep_idx = ep_dev_ctx_idx == EP_IN_DEV_CTX_IDX ? IN_EP_IDX : OUT_EP_IDX;
196    xdc_endpoint_t* ep = &xdc->eps[xdc_ep_idx];
197    xhci_transfer_ring_t* ring = &ep->transfer_ring;
198
199    uint32_t cc = READ_FIELD(status, EVT_TRB_CC_START, EVT_TRB_CC_BITS);
200    uint32_t length = READ_FIELD(status, EVT_TRB_XFER_LENGTH_START, EVT_TRB_XFER_LENGTH_BITS);
201    usb_request_t* req = NULL;
202    bool error = false;
203
204    switch (cc) {
205        case TRB_CC_SUCCESS:
206        case TRB_CC_SHORT_PACKET:
207            break;
208        case TRB_CC_BABBLE_DETECTED_ERROR:
209        case TRB_CC_USB_TRANSACTION_ERROR:
210        case TRB_CC_TRB_ERROR:
211        case TRB_CC_STALL_ERROR:
212            zxlogf(ERROR, "xdc_handle_transfer_event: error condition code: %d\n", cc);
213            error = true;
214            break;
215        default:
216            zxlogf(ERROR, "xdc_handle_transfer_event: unexpected condition code %d\n", cc);
217            error = true;
218            break;
219    }
220
221    // Even though the main poll loop checks for changes in the halt registers,
222    // it's possible we missed the halt register being set if the halt was cleared fast enough.
223    if (error) {
224        if (ep->state == XDC_EP_STATE_RUNNING) {
225             xdc_endpoint_set_halt_locked(xdc, poll_state, ep);
226        }
227        ep->got_err_event = true;
228        // We're going to requeue the transfer when we restart the transfer ring,
229        // so nothing else to do.
230        return;
231    }
232
233    if (control & EVT_TRB_ED) {
234        // An Event Data TRB generated the completion event, so the TRB Pointer field
235        // will contain the usb request pointer we previously stored.
236        req = (usb_request_t *)trb_get_ptr(trb);
237    } else {
238        // Get the pointer to the TRB that generated the event.
239        trb = xhci_read_trb_ptr(ring, trb);
240        if (trb_get_type(trb) == TRB_TRANSFER_NOOP) {
241            // If it's the NO-OP TRB we queued when dealing with the halt condition,
242            // there won't be a corresponding usb request.
243            zxlogf(TRACE, "xdc_handle_transfer_event: got a NO-OP TRB\n");
244            xhci_set_dequeue_ptr(ring, xhci_get_next_trb(ring, trb));
245            xdc_process_transactions_locked(xdc, ep);
246            return;
247        }
248
249        // Look for the Event Data TRB which will have the usb request pointer.
250        for (uint i = 0; i < TRANSFER_RING_SIZE && trb; i++) {
251            if (trb_get_type(trb) == TRB_TRANSFER_EVENT_DATA) {
252                req = (usb_request_t *)trb_get_ptr(trb);
253                break;
254            }
255            trb = xhci_get_next_trb(ring, trb);
256        }
257    }
258
259    if (!req) {
260        zxlogf(ERROR, "xdc_handle_transfer_event: unable to find request to complete\n");
261        return;
262    }
263
264    // Find the usb request in the pending list.
265    bool found_req = false;
266    usb_request_t* test;
267    list_for_every_entry(&ep->pending_reqs, test, usb_request_t, node) {
268        if (test == req) {
269            found_req = true;
270            break;
271        }
272    }
273    if (!found_req) {
274        zxlogf(ERROR, "xdc_handle_transfer_event: ignoring event for completed transfer\n");
275        return;
276    }
277    // Remove request from pending_reqs.
278    list_delete(&req->node);
279
280    // Update our copy of the dequeue_ptr to the TRB following this transaction.
281    xhci_set_dequeue_ptr(ring, req->context);
282    xdc_process_transactions_locked(xdc, ep);
283
284    // Save the request to be completed later out of the lock.
285    req->response.status = ZX_OK;
286    req->response.actual = length;
287    list_add_tail(&poll_state->completed_reqs, &req->node);
288}
289