1// SPDX-License-Identifier: GPL-2.0 2/* 3 * xHCI host controller driver 4 * 5 * Copyright (C) 2008 Intel Corp. 6 * 7 * Author: Sarah Sharp 8 * Some code borrowed from the Linux EHCI driver. 9 */ 10 11#include <linux/usb.h> 12#include <linux/overflow.h> 13#include <linux/pci.h> 14#include <linux/slab.h> 15#include <linux/dmapool.h> 16#include <linux/dma-mapping.h> 17 18#include "xhci.h" 19#include "xhci-trace.h" 20#include "xhci-debugfs.h" 21 22/* 23 * Allocates a generic ring segment from the ring pool, sets the dma address, 24 * initializes the segment to zero, and sets the private next pointer to NULL. 25 * 26 * Section 4.11.1.1: 27 * "All components of all Command and Transfer TRBs shall be initialized to '0'" 28 */ 29static struct xhci_segment *xhci_segment_alloc(struct xhci_hcd *xhci, 30 unsigned int cycle_state, 31 unsigned int max_packet, 32 unsigned int num, 33 gfp_t flags) 34{ 35 struct xhci_segment *seg; 36 dma_addr_t dma; 37 int i; 38 struct device *dev = xhci_to_hcd(xhci)->self.sysdev; 39 40 seg = kzalloc_node(sizeof(*seg), flags, dev_to_node(dev)); 41 if (!seg) 42 return NULL; 43 44 seg->trbs = dma_pool_zalloc(xhci->segment_pool, flags, &dma); 45 if (!seg->trbs) { 46 kfree(seg); 47 return NULL; 48 } 49 50 if (max_packet) { 51 seg->bounce_buf = kzalloc_node(max_packet, flags, 52 dev_to_node(dev)); 53 if (!seg->bounce_buf) { 54 dma_pool_free(xhci->segment_pool, seg->trbs, dma); 55 kfree(seg); 56 return NULL; 57 } 58 } 59 /* If the cycle state is 0, set the cycle bit to 1 for all the TRBs */ 60 if (cycle_state == 0) { 61 for (i = 0; i < TRBS_PER_SEGMENT; i++) 62 seg->trbs[i].link.control = cpu_to_le32(TRB_CYCLE); 63 } 64 seg->num = num; 65 seg->dma = dma; 66 seg->next = NULL; 67 68 return seg; 69} 70 71static void xhci_segment_free(struct xhci_hcd *xhci, struct xhci_segment *seg) 72{ 73 if (seg->trbs) { 74 dma_pool_free(xhci->segment_pool, seg->trbs, seg->dma); 75 seg->trbs = NULL; 76 } 77 kfree(seg->bounce_buf); 78 kfree(seg); 79} 80 81static void xhci_free_segments_for_ring(struct xhci_hcd *xhci, 82 struct xhci_segment *first) 83{ 84 struct xhci_segment *seg; 85 86 seg = first->next; 87 while (seg && seg != first) { 88 struct xhci_segment *next = seg->next; 89 xhci_segment_free(xhci, seg); 90 seg = next; 91 } 92 xhci_segment_free(xhci, first); 93} 94 95/* 96 * Make the prev segment point to the next segment. 97 * 98 * Change the last TRB in the prev segment to be a Link TRB which points to the 99 * DMA address of the next segment. The caller needs to set any Link TRB 100 * related flags, such as End TRB, Toggle Cycle, and no snoop. 101 */ 102static void xhci_link_segments(struct xhci_segment *prev, 103 struct xhci_segment *next, 104 enum xhci_ring_type type, bool chain_links) 105{ 106 u32 val; 107 108 if (!prev || !next) 109 return; 110 prev->next = next; 111 if (type != TYPE_EVENT) { 112 prev->trbs[TRBS_PER_SEGMENT-1].link.segment_ptr = 113 cpu_to_le64(next->dma); 114 115 /* Set the last TRB in the segment to have a TRB type ID of Link TRB */ 116 val = le32_to_cpu(prev->trbs[TRBS_PER_SEGMENT-1].link.control); 117 val &= ~TRB_TYPE_BITMASK; 118 val |= TRB_TYPE(TRB_LINK); 119 if (chain_links) 120 val |= TRB_CHAIN; 121 prev->trbs[TRBS_PER_SEGMENT-1].link.control = cpu_to_le32(val); 122 } 123} 124 125/* 126 * Link the ring to the new segments. 127 * Set Toggle Cycle for the new ring if needed. 128 */ 129static void xhci_link_rings(struct xhci_hcd *xhci, struct xhci_ring *ring, 130 struct xhci_segment *first, struct xhci_segment *last, 131 unsigned int num_segs) 132{ 133 struct xhci_segment *next, *seg; 134 bool chain_links; 135 136 if (!ring || !first || !last) 137 return; 138 139 /* Set chain bit for 0.95 hosts, and for isoc rings on AMD 0.96 host */ 140 chain_links = !!(xhci_link_trb_quirk(xhci) || 141 (ring->type == TYPE_ISOC && 142 (xhci->quirks & XHCI_AMD_0x96_HOST))); 143 144 next = ring->enq_seg->next; 145 xhci_link_segments(ring->enq_seg, first, ring->type, chain_links); 146 xhci_link_segments(last, next, ring->type, chain_links); 147 ring->num_segs += num_segs; 148 149 if (ring->enq_seg == ring->last_seg) { 150 if (ring->type != TYPE_EVENT) { 151 ring->last_seg->trbs[TRBS_PER_SEGMENT-1].link.control 152 &= ~cpu_to_le32(LINK_TOGGLE); 153 last->trbs[TRBS_PER_SEGMENT-1].link.control 154 |= cpu_to_le32(LINK_TOGGLE); 155 } 156 ring->last_seg = last; 157 } 158 159 for (seg = last; seg != ring->last_seg; seg = seg->next) 160 seg->next->num = seg->num + 1; 161} 162 163/* 164 * We need a radix tree for mapping physical addresses of TRBs to which stream 165 * ID they belong to. We need to do this because the host controller won't tell 166 * us which stream ring the TRB came from. We could store the stream ID in an 167 * event data TRB, but that doesn't help us for the cancellation case, since the 168 * endpoint may stop before it reaches that event data TRB. 169 * 170 * The radix tree maps the upper portion of the TRB DMA address to a ring 171 * segment that has the same upper portion of DMA addresses. For example, say I 172 * have segments of size 1KB, that are always 1KB aligned. A segment may 173 * start at 0x10c91000 and end at 0x10c913f0. If I use the upper 10 bits, the 174 * key to the stream ID is 0x43244. I can use the DMA address of the TRB to 175 * pass the radix tree a key to get the right stream ID: 176 * 177 * 0x10c90fff >> 10 = 0x43243 178 * 0x10c912c0 >> 10 = 0x43244 179 * 0x10c91400 >> 10 = 0x43245 180 * 181 * Obviously, only those TRBs with DMA addresses that are within the segment 182 * will make the radix tree return the stream ID for that ring. 183 * 184 * Caveats for the radix tree: 185 * 186 * The radix tree uses an unsigned long as a key pair. On 32-bit systems, an 187 * unsigned long will be 32-bits; on a 64-bit system an unsigned long will be 188 * 64-bits. Since we only request 32-bit DMA addresses, we can use that as the 189 * key on 32-bit or 64-bit systems (it would also be fine if we asked for 64-bit 190 * PCI DMA addresses on a 64-bit system). There might be a problem on 32-bit 191 * extended systems (where the DMA address can be bigger than 32-bits), 192 * if we allow the PCI dma mask to be bigger than 32-bits. So don't do that. 193 */ 194static int xhci_insert_segment_mapping(struct radix_tree_root *trb_address_map, 195 struct xhci_ring *ring, 196 struct xhci_segment *seg, 197 gfp_t mem_flags) 198{ 199 unsigned long key; 200 int ret; 201 202 key = (unsigned long)(seg->dma >> TRB_SEGMENT_SHIFT); 203 /* Skip any segments that were already added. */ 204 if (radix_tree_lookup(trb_address_map, key)) 205 return 0; 206 207 ret = radix_tree_maybe_preload(mem_flags); 208 if (ret) 209 return ret; 210 ret = radix_tree_insert(trb_address_map, 211 key, ring); 212 radix_tree_preload_end(); 213 return ret; 214} 215 216static void xhci_remove_segment_mapping(struct radix_tree_root *trb_address_map, 217 struct xhci_segment *seg) 218{ 219 unsigned long key; 220 221 key = (unsigned long)(seg->dma >> TRB_SEGMENT_SHIFT); 222 if (radix_tree_lookup(trb_address_map, key)) 223 radix_tree_delete(trb_address_map, key); 224} 225 226static int xhci_update_stream_segment_mapping( 227 struct radix_tree_root *trb_address_map, 228 struct xhci_ring *ring, 229 struct xhci_segment *first_seg, 230 struct xhci_segment *last_seg, 231 gfp_t mem_flags) 232{ 233 struct xhci_segment *seg; 234 struct xhci_segment *failed_seg; 235 int ret; 236 237 if (WARN_ON_ONCE(trb_address_map == NULL)) 238 return 0; 239 240 seg = first_seg; 241 do { 242 ret = xhci_insert_segment_mapping(trb_address_map, 243 ring, seg, mem_flags); 244 if (ret) 245 goto remove_streams; 246 if (seg == last_seg) 247 return 0; 248 seg = seg->next; 249 } while (seg != first_seg); 250 251 return 0; 252 253remove_streams: 254 failed_seg = seg; 255 seg = first_seg; 256 do { 257 xhci_remove_segment_mapping(trb_address_map, seg); 258 if (seg == failed_seg) 259 return ret; 260 seg = seg->next; 261 } while (seg != first_seg); 262 263 return ret; 264} 265 266static void xhci_remove_stream_mapping(struct xhci_ring *ring) 267{ 268 struct xhci_segment *seg; 269 270 if (WARN_ON_ONCE(ring->trb_address_map == NULL)) 271 return; 272 273 seg = ring->first_seg; 274 do { 275 xhci_remove_segment_mapping(ring->trb_address_map, seg); 276 seg = seg->next; 277 } while (seg != ring->first_seg); 278} 279 280static int xhci_update_stream_mapping(struct xhci_ring *ring, gfp_t mem_flags) 281{ 282 return xhci_update_stream_segment_mapping(ring->trb_address_map, ring, 283 ring->first_seg, ring->last_seg, mem_flags); 284} 285 286/* XXX: Do we need the hcd structure in all these functions? */ 287void xhci_ring_free(struct xhci_hcd *xhci, struct xhci_ring *ring) 288{ 289 if (!ring) 290 return; 291 292 trace_xhci_ring_free(ring); 293 294 if (ring->first_seg) { 295 if (ring->type == TYPE_STREAM) 296 xhci_remove_stream_mapping(ring); 297 xhci_free_segments_for_ring(xhci, ring->first_seg); 298 } 299 300 kfree(ring); 301} 302 303void xhci_initialize_ring_info(struct xhci_ring *ring, 304 unsigned int cycle_state) 305{ 306 /* The ring is empty, so the enqueue pointer == dequeue pointer */ 307 ring->enqueue = ring->first_seg->trbs; 308 ring->enq_seg = ring->first_seg; 309 ring->dequeue = ring->enqueue; 310 ring->deq_seg = ring->first_seg; 311 /* The ring is initialized to 0. The producer must write 1 to the cycle 312 * bit to handover ownership of the TRB, so PCS = 1. The consumer must 313 * compare CCS to the cycle bit to check ownership, so CCS = 1. 314 * 315 * New rings are initialized with cycle state equal to 1; if we are 316 * handling ring expansion, set the cycle state equal to the old ring. 317 */ 318 ring->cycle_state = cycle_state; 319 320 /* 321 * Each segment has a link TRB, and leave an extra TRB for SW 322 * accounting purpose 323 */ 324 ring->num_trbs_free = ring->num_segs * (TRBS_PER_SEGMENT - 1) - 1; 325} 326EXPORT_SYMBOL_GPL(xhci_initialize_ring_info); 327 328/* Allocate segments and link them for a ring */ 329static int xhci_alloc_segments_for_ring(struct xhci_hcd *xhci, 330 struct xhci_segment **first, struct xhci_segment **last, 331 unsigned int num_segs, unsigned int num, 332 unsigned int cycle_state, enum xhci_ring_type type, 333 unsigned int max_packet, gfp_t flags) 334{ 335 struct xhci_segment *prev; 336 bool chain_links; 337 338 /* Set chain bit for 0.95 hosts, and for isoc rings on AMD 0.96 host */ 339 chain_links = !!(xhci_link_trb_quirk(xhci) || 340 (type == TYPE_ISOC && 341 (xhci->quirks & XHCI_AMD_0x96_HOST))); 342 343 prev = xhci_segment_alloc(xhci, cycle_state, max_packet, num, flags); 344 if (!prev) 345 return -ENOMEM; 346 num++; 347 348 *first = prev; 349 while (num < num_segs) { 350 struct xhci_segment *next; 351 352 next = xhci_segment_alloc(xhci, cycle_state, max_packet, num, 353 flags); 354 if (!next) 355 goto free_segments; 356 357 xhci_link_segments(prev, next, type, chain_links); 358 prev = next; 359 num++; 360 } 361 xhci_link_segments(prev, *first, type, chain_links); 362 *last = prev; 363 364 return 0; 365 366free_segments: 367 xhci_free_segments_for_ring(xhci, *first); 368 return -ENOMEM; 369} 370 371/* 372 * Create a new ring with zero or more segments. 373 * 374 * Link each segment together into a ring. 375 * Set the end flag and the cycle toggle bit on the last segment. 376 * See section 4.9.1 and figures 15 and 16. 377 */ 378struct xhci_ring *xhci_ring_alloc(struct xhci_hcd *xhci, 379 unsigned int num_segs, unsigned int cycle_state, 380 enum xhci_ring_type type, unsigned int max_packet, gfp_t flags) 381{ 382 struct xhci_ring *ring; 383 int ret; 384 struct device *dev = xhci_to_hcd(xhci)->self.sysdev; 385 386 ring = kzalloc_node(sizeof(*ring), flags, dev_to_node(dev)); 387 if (!ring) 388 return NULL; 389 390 ring->num_segs = num_segs; 391 ring->bounce_buf_len = max_packet; 392 INIT_LIST_HEAD(&ring->td_list); 393 ring->type = type; 394 if (num_segs == 0) 395 return ring; 396 397 ret = xhci_alloc_segments_for_ring(xhci, &ring->first_seg, 398 &ring->last_seg, num_segs, 0, cycle_state, type, 399 max_packet, flags); 400 if (ret) 401 goto fail; 402 403 /* Only event ring does not use link TRB */ 404 if (type != TYPE_EVENT) { 405 /* See section 4.9.2.1 and 6.4.4.1 */ 406 ring->last_seg->trbs[TRBS_PER_SEGMENT - 1].link.control |= 407 cpu_to_le32(LINK_TOGGLE); 408 } 409 xhci_initialize_ring_info(ring, cycle_state); 410 trace_xhci_ring_alloc(ring); 411 return ring; 412 413fail: 414 kfree(ring); 415 return NULL; 416} 417 418void xhci_free_endpoint_ring(struct xhci_hcd *xhci, 419 struct xhci_virt_device *virt_dev, 420 unsigned int ep_index) 421{ 422 xhci_ring_free(xhci, virt_dev->eps[ep_index].ring); 423 virt_dev->eps[ep_index].ring = NULL; 424} 425 426/* 427 * Expand an existing ring. 428 * Allocate a new ring which has same segment numbers and link the two rings. 429 */ 430int xhci_ring_expansion(struct xhci_hcd *xhci, struct xhci_ring *ring, 431 unsigned int num_new_segs, gfp_t flags) 432{ 433 struct xhci_segment *first; 434 struct xhci_segment *last; 435 int ret; 436 437 ret = xhci_alloc_segments_for_ring(xhci, &first, &last, 438 num_new_segs, ring->enq_seg->num + 1, 439 ring->cycle_state, ring->type, 440 ring->bounce_buf_len, flags); 441 if (ret) 442 return -ENOMEM; 443 444 if (ring->type == TYPE_STREAM) { 445 ret = xhci_update_stream_segment_mapping(ring->trb_address_map, 446 ring, first, last, flags); 447 if (ret) 448 goto free_segments; 449 } 450 451 xhci_link_rings(xhci, ring, first, last, num_new_segs); 452 trace_xhci_ring_expansion(ring); 453 xhci_dbg_trace(xhci, trace_xhci_dbg_ring_expansion, 454 "ring expansion succeed, now has %d segments", 455 ring->num_segs); 456 457 return 0; 458 459free_segments: 460 xhci_free_segments_for_ring(xhci, first); 461 return ret; 462} 463 464struct xhci_container_ctx *xhci_alloc_container_ctx(struct xhci_hcd *xhci, 465 int type, gfp_t flags) 466{ 467 struct xhci_container_ctx *ctx; 468 struct device *dev = xhci_to_hcd(xhci)->self.sysdev; 469 470 if ((type != XHCI_CTX_TYPE_DEVICE) && (type != XHCI_CTX_TYPE_INPUT)) 471 return NULL; 472 473 ctx = kzalloc_node(sizeof(*ctx), flags, dev_to_node(dev)); 474 if (!ctx) 475 return NULL; 476 477 ctx->type = type; 478 ctx->size = HCC_64BYTE_CONTEXT(xhci->hcc_params) ? 2048 : 1024; 479 if (type == XHCI_CTX_TYPE_INPUT) 480 ctx->size += CTX_SIZE(xhci->hcc_params); 481 482 ctx->bytes = dma_pool_zalloc(xhci->device_pool, flags, &ctx->dma); 483 if (!ctx->bytes) { 484 kfree(ctx); 485 return NULL; 486 } 487 return ctx; 488} 489 490void xhci_free_container_ctx(struct xhci_hcd *xhci, 491 struct xhci_container_ctx *ctx) 492{ 493 if (!ctx) 494 return; 495 dma_pool_free(xhci->device_pool, ctx->bytes, ctx->dma); 496 kfree(ctx); 497} 498 499struct xhci_input_control_ctx *xhci_get_input_control_ctx( 500 struct xhci_container_ctx *ctx) 501{ 502 if (ctx->type != XHCI_CTX_TYPE_INPUT) 503 return NULL; 504 505 return (struct xhci_input_control_ctx *)ctx->bytes; 506} 507 508struct xhci_slot_ctx *xhci_get_slot_ctx(struct xhci_hcd *xhci, 509 struct xhci_container_ctx *ctx) 510{ 511 if (ctx->type == XHCI_CTX_TYPE_DEVICE) 512 return (struct xhci_slot_ctx *)ctx->bytes; 513 514 return (struct xhci_slot_ctx *) 515 (ctx->bytes + CTX_SIZE(xhci->hcc_params)); 516} 517 518struct xhci_ep_ctx *xhci_get_ep_ctx(struct xhci_hcd *xhci, 519 struct xhci_container_ctx *ctx, 520 unsigned int ep_index) 521{ 522 /* increment ep index by offset of start of ep ctx array */ 523 ep_index++; 524 if (ctx->type == XHCI_CTX_TYPE_INPUT) 525 ep_index++; 526 527 return (struct xhci_ep_ctx *) 528 (ctx->bytes + (ep_index * CTX_SIZE(xhci->hcc_params))); 529} 530EXPORT_SYMBOL_GPL(xhci_get_ep_ctx); 531 532/***************** Streams structures manipulation *************************/ 533 534static void xhci_free_stream_ctx(struct xhci_hcd *xhci, 535 unsigned int num_stream_ctxs, 536 struct xhci_stream_ctx *stream_ctx, dma_addr_t dma) 537{ 538 struct device *dev = xhci_to_hcd(xhci)->self.sysdev; 539 size_t size = sizeof(struct xhci_stream_ctx) * num_stream_ctxs; 540 541 if (size > MEDIUM_STREAM_ARRAY_SIZE) 542 dma_free_coherent(dev, size, stream_ctx, dma); 543 else if (size > SMALL_STREAM_ARRAY_SIZE) 544 dma_pool_free(xhci->medium_streams_pool, stream_ctx, dma); 545 else 546 dma_pool_free(xhci->small_streams_pool, stream_ctx, dma); 547} 548 549/* 550 * The stream context array for each endpoint with bulk streams enabled can 551 * vary in size, based on: 552 * - how many streams the endpoint supports, 553 * - the maximum primary stream array size the host controller supports, 554 * - and how many streams the device driver asks for. 555 * 556 * The stream context array must be a power of 2, and can be as small as 557 * 64 bytes or as large as 1MB. 558 */ 559static struct xhci_stream_ctx *xhci_alloc_stream_ctx(struct xhci_hcd *xhci, 560 unsigned int num_stream_ctxs, dma_addr_t *dma, 561 gfp_t mem_flags) 562{ 563 struct device *dev = xhci_to_hcd(xhci)->self.sysdev; 564 size_t size = size_mul(sizeof(struct xhci_stream_ctx), num_stream_ctxs); 565 566 if (size > MEDIUM_STREAM_ARRAY_SIZE) 567 return dma_alloc_coherent(dev, size, dma, mem_flags); 568 if (size > SMALL_STREAM_ARRAY_SIZE) 569 return dma_pool_zalloc(xhci->medium_streams_pool, mem_flags, dma); 570 else 571 return dma_pool_zalloc(xhci->small_streams_pool, mem_flags, dma); 572} 573 574struct xhci_ring *xhci_dma_to_transfer_ring( 575 struct xhci_virt_ep *ep, 576 u64 address) 577{ 578 if (ep->ep_state & EP_HAS_STREAMS) 579 return radix_tree_lookup(&ep->stream_info->trb_address_map, 580 address >> TRB_SEGMENT_SHIFT); 581 return ep->ring; 582} 583 584/* 585 * Change an endpoint's internal structure so it supports stream IDs. The 586 * number of requested streams includes stream 0, which cannot be used by device 587 * drivers. 588 * 589 * The number of stream contexts in the stream context array may be bigger than 590 * the number of streams the driver wants to use. This is because the number of 591 * stream context array entries must be a power of two. 592 */ 593struct xhci_stream_info *xhci_alloc_stream_info(struct xhci_hcd *xhci, 594 unsigned int num_stream_ctxs, 595 unsigned int num_streams, 596 unsigned int max_packet, gfp_t mem_flags) 597{ 598 struct xhci_stream_info *stream_info; 599 u32 cur_stream; 600 struct xhci_ring *cur_ring; 601 u64 addr; 602 int ret; 603 struct device *dev = xhci_to_hcd(xhci)->self.sysdev; 604 605 xhci_dbg(xhci, "Allocating %u streams and %u stream context array entries.\n", 606 num_streams, num_stream_ctxs); 607 if (xhci->cmd_ring_reserved_trbs == MAX_RSVD_CMD_TRBS) { 608 xhci_dbg(xhci, "Command ring has no reserved TRBs available\n"); 609 return NULL; 610 } 611 xhci->cmd_ring_reserved_trbs++; 612 613 stream_info = kzalloc_node(sizeof(*stream_info), mem_flags, 614 dev_to_node(dev)); 615 if (!stream_info) 616 goto cleanup_trbs; 617 618 stream_info->num_streams = num_streams; 619 stream_info->num_stream_ctxs = num_stream_ctxs; 620 621 /* Initialize the array of virtual pointers to stream rings. */ 622 stream_info->stream_rings = kcalloc_node( 623 num_streams, sizeof(struct xhci_ring *), mem_flags, 624 dev_to_node(dev)); 625 if (!stream_info->stream_rings) 626 goto cleanup_info; 627 628 /* Initialize the array of DMA addresses for stream rings for the HW. */ 629 stream_info->stream_ctx_array = xhci_alloc_stream_ctx(xhci, 630 num_stream_ctxs, &stream_info->ctx_array_dma, 631 mem_flags); 632 if (!stream_info->stream_ctx_array) 633 goto cleanup_ring_array; 634 635 /* Allocate everything needed to free the stream rings later */ 636 stream_info->free_streams_command = 637 xhci_alloc_command_with_ctx(xhci, true, mem_flags); 638 if (!stream_info->free_streams_command) 639 goto cleanup_ctx; 640 641 INIT_RADIX_TREE(&stream_info->trb_address_map, GFP_ATOMIC); 642 643 /* Allocate rings for all the streams that the driver will use, 644 * and add their segment DMA addresses to the radix tree. 645 * Stream 0 is reserved. 646 */ 647 648 for (cur_stream = 1; cur_stream < num_streams; cur_stream++) { 649 stream_info->stream_rings[cur_stream] = 650 xhci_ring_alloc(xhci, 2, 1, TYPE_STREAM, max_packet, 651 mem_flags); 652 cur_ring = stream_info->stream_rings[cur_stream]; 653 if (!cur_ring) 654 goto cleanup_rings; 655 cur_ring->stream_id = cur_stream; 656 cur_ring->trb_address_map = &stream_info->trb_address_map; 657 /* Set deq ptr, cycle bit, and stream context type */ 658 addr = cur_ring->first_seg->dma | 659 SCT_FOR_CTX(SCT_PRI_TR) | 660 cur_ring->cycle_state; 661 stream_info->stream_ctx_array[cur_stream].stream_ring = 662 cpu_to_le64(addr); 663 xhci_dbg(xhci, "Setting stream %d ring ptr to 0x%08llx\n", cur_stream, addr); 664 665 ret = xhci_update_stream_mapping(cur_ring, mem_flags); 666 if (ret) { 667 xhci_ring_free(xhci, cur_ring); 668 stream_info->stream_rings[cur_stream] = NULL; 669 goto cleanup_rings; 670 } 671 } 672 /* Leave the other unused stream ring pointers in the stream context 673 * array initialized to zero. This will cause the xHC to give us an 674 * error if the device asks for a stream ID we don't have setup (if it 675 * was any other way, the host controller would assume the ring is 676 * "empty" and wait forever for data to be queued to that stream ID). 677 */ 678 679 return stream_info; 680 681cleanup_rings: 682 for (cur_stream = 1; cur_stream < num_streams; cur_stream++) { 683 cur_ring = stream_info->stream_rings[cur_stream]; 684 if (cur_ring) { 685 xhci_ring_free(xhci, cur_ring); 686 stream_info->stream_rings[cur_stream] = NULL; 687 } 688 } 689 xhci_free_command(xhci, stream_info->free_streams_command); 690cleanup_ctx: 691 xhci_free_stream_ctx(xhci, 692 stream_info->num_stream_ctxs, 693 stream_info->stream_ctx_array, 694 stream_info->ctx_array_dma); 695cleanup_ring_array: 696 kfree(stream_info->stream_rings); 697cleanup_info: 698 kfree(stream_info); 699cleanup_trbs: 700 xhci->cmd_ring_reserved_trbs--; 701 return NULL; 702} 703/* 704 * Sets the MaxPStreams field and the Linear Stream Array field. 705 * Sets the dequeue pointer to the stream context array. 706 */ 707void xhci_setup_streams_ep_input_ctx(struct xhci_hcd *xhci, 708 struct xhci_ep_ctx *ep_ctx, 709 struct xhci_stream_info *stream_info) 710{ 711 u32 max_primary_streams; 712 /* MaxPStreams is the number of stream context array entries, not the 713 * number we're actually using. Must be in 2^(MaxPstreams + 1) format. 714 * fls(0) = 0, fls(0x1) = 1, fls(0x10) = 2, fls(0x100) = 3, etc. 715 */ 716 max_primary_streams = fls(stream_info->num_stream_ctxs) - 2; 717 xhci_dbg_trace(xhci, trace_xhci_dbg_context_change, 718 "Setting number of stream ctx array entries to %u", 719 1 << (max_primary_streams + 1)); 720 ep_ctx->ep_info &= cpu_to_le32(~EP_MAXPSTREAMS_MASK); 721 ep_ctx->ep_info |= cpu_to_le32(EP_MAXPSTREAMS(max_primary_streams) 722 | EP_HAS_LSA); 723 ep_ctx->deq = cpu_to_le64(stream_info->ctx_array_dma); 724} 725 726/* 727 * Sets the MaxPStreams field and the Linear Stream Array field to 0. 728 * Reinstalls the "normal" endpoint ring (at its previous dequeue mark, 729 * not at the beginning of the ring). 730 */ 731void xhci_setup_no_streams_ep_input_ctx(struct xhci_ep_ctx *ep_ctx, 732 struct xhci_virt_ep *ep) 733{ 734 dma_addr_t addr; 735 ep_ctx->ep_info &= cpu_to_le32(~(EP_MAXPSTREAMS_MASK | EP_HAS_LSA)); 736 addr = xhci_trb_virt_to_dma(ep->ring->deq_seg, ep->ring->dequeue); 737 ep_ctx->deq = cpu_to_le64(addr | ep->ring->cycle_state); 738} 739 740/* Frees all stream contexts associated with the endpoint, 741 * 742 * Caller should fix the endpoint context streams fields. 743 */ 744void xhci_free_stream_info(struct xhci_hcd *xhci, 745 struct xhci_stream_info *stream_info) 746{ 747 int cur_stream; 748 struct xhci_ring *cur_ring; 749 750 if (!stream_info) 751 return; 752 753 for (cur_stream = 1; cur_stream < stream_info->num_streams; 754 cur_stream++) { 755 cur_ring = stream_info->stream_rings[cur_stream]; 756 if (cur_ring) { 757 xhci_ring_free(xhci, cur_ring); 758 stream_info->stream_rings[cur_stream] = NULL; 759 } 760 } 761 xhci_free_command(xhci, stream_info->free_streams_command); 762 xhci->cmd_ring_reserved_trbs--; 763 if (stream_info->stream_ctx_array) 764 xhci_free_stream_ctx(xhci, 765 stream_info->num_stream_ctxs, 766 stream_info->stream_ctx_array, 767 stream_info->ctx_array_dma); 768 769 kfree(stream_info->stream_rings); 770 kfree(stream_info); 771} 772 773 774/***************** Device context manipulation *************************/ 775 776static void xhci_free_tt_info(struct xhci_hcd *xhci, 777 struct xhci_virt_device *virt_dev, 778 int slot_id) 779{ 780 struct list_head *tt_list_head; 781 struct xhci_tt_bw_info *tt_info, *next; 782 bool slot_found = false; 783 784 /* If the device never made it past the Set Address stage, 785 * it may not have the root hub port pointer set correctly. 786 */ 787 if (!virt_dev->rhub_port) { 788 xhci_dbg(xhci, "Bad rhub port.\n"); 789 return; 790 } 791 792 tt_list_head = &(xhci->rh_bw[virt_dev->rhub_port->hw_portnum].tts); 793 list_for_each_entry_safe(tt_info, next, tt_list_head, tt_list) { 794 /* Multi-TT hubs will have more than one entry */ 795 if (tt_info->slot_id == slot_id) { 796 slot_found = true; 797 list_del(&tt_info->tt_list); 798 kfree(tt_info); 799 } else if (slot_found) { 800 break; 801 } 802 } 803} 804 805int xhci_alloc_tt_info(struct xhci_hcd *xhci, 806 struct xhci_virt_device *virt_dev, 807 struct usb_device *hdev, 808 struct usb_tt *tt, gfp_t mem_flags) 809{ 810 struct xhci_tt_bw_info *tt_info; 811 unsigned int num_ports; 812 int i, j; 813 struct device *dev = xhci_to_hcd(xhci)->self.sysdev; 814 815 if (!tt->multi) 816 num_ports = 1; 817 else 818 num_ports = hdev->maxchild; 819 820 for (i = 0; i < num_ports; i++, tt_info++) { 821 struct xhci_interval_bw_table *bw_table; 822 823 tt_info = kzalloc_node(sizeof(*tt_info), mem_flags, 824 dev_to_node(dev)); 825 if (!tt_info) 826 goto free_tts; 827 INIT_LIST_HEAD(&tt_info->tt_list); 828 list_add(&tt_info->tt_list, 829 &xhci->rh_bw[virt_dev->rhub_port->hw_portnum].tts); 830 tt_info->slot_id = virt_dev->udev->slot_id; 831 if (tt->multi) 832 tt_info->ttport = i+1; 833 bw_table = &tt_info->bw_table; 834 for (j = 0; j < XHCI_MAX_INTERVAL; j++) 835 INIT_LIST_HEAD(&bw_table->interval_bw[j].endpoints); 836 } 837 return 0; 838 839free_tts: 840 xhci_free_tt_info(xhci, virt_dev, virt_dev->udev->slot_id); 841 return -ENOMEM; 842} 843 844 845/* All the xhci_tds in the ring's TD list should be freed at this point. 846 * Should be called with xhci->lock held if there is any chance the TT lists 847 * will be manipulated by the configure endpoint, allocate device, or update 848 * hub functions while this function is removing the TT entries from the list. 849 */ 850void xhci_free_virt_device(struct xhci_hcd *xhci, int slot_id) 851{ 852 struct xhci_virt_device *dev; 853 int i; 854 int old_active_eps = 0; 855 856 /* Slot ID 0 is reserved */ 857 if (slot_id == 0 || !xhci->devs[slot_id]) 858 return; 859 860 dev = xhci->devs[slot_id]; 861 862 xhci->dcbaa->dev_context_ptrs[slot_id] = 0; 863 if (!dev) 864 return; 865 866 trace_xhci_free_virt_device(dev); 867 868 if (dev->tt_info) 869 old_active_eps = dev->tt_info->active_eps; 870 871 for (i = 0; i < 31; i++) { 872 if (dev->eps[i].ring) 873 xhci_ring_free(xhci, dev->eps[i].ring); 874 if (dev->eps[i].stream_info) 875 xhci_free_stream_info(xhci, 876 dev->eps[i].stream_info); 877 /* 878 * Endpoints are normally deleted from the bandwidth list when 879 * endpoints are dropped, before device is freed. 880 * If host is dying or being removed then endpoints aren't 881 * dropped cleanly, so delete the endpoint from list here. 882 * Only applicable for hosts with software bandwidth checking. 883 */ 884 885 if (!list_empty(&dev->eps[i].bw_endpoint_list)) { 886 list_del_init(&dev->eps[i].bw_endpoint_list); 887 xhci_dbg(xhci, "Slot %u endpoint %u not removed from BW list!\n", 888 slot_id, i); 889 } 890 } 891 /* If this is a hub, free the TT(s) from the TT list */ 892 xhci_free_tt_info(xhci, dev, slot_id); 893 /* If necessary, update the number of active TTs on this root port */ 894 xhci_update_tt_active_eps(xhci, dev, old_active_eps); 895 896 if (dev->in_ctx) 897 xhci_free_container_ctx(xhci, dev->in_ctx); 898 if (dev->out_ctx) 899 xhci_free_container_ctx(xhci, dev->out_ctx); 900 901 if (dev->udev && dev->udev->slot_id) 902 dev->udev->slot_id = 0; 903 if (dev->rhub_port && dev->rhub_port->slot_id == slot_id) 904 dev->rhub_port->slot_id = 0; 905 kfree(xhci->devs[slot_id]); 906 xhci->devs[slot_id] = NULL; 907} 908 909/* 910 * Free a virt_device structure. 911 * If the virt_device added a tt_info (a hub) and has children pointing to 912 * that tt_info, then free the child first. Recursive. 913 * We can't rely on udev at this point to find child-parent relationships. 914 */ 915static void xhci_free_virt_devices_depth_first(struct xhci_hcd *xhci, int slot_id) 916{ 917 struct xhci_virt_device *vdev; 918 struct list_head *tt_list_head; 919 struct xhci_tt_bw_info *tt_info, *next; 920 int i; 921 922 vdev = xhci->devs[slot_id]; 923 if (!vdev) 924 return; 925 926 if (!vdev->rhub_port) { 927 xhci_dbg(xhci, "Bad rhub port.\n"); 928 goto out; 929 } 930 931 tt_list_head = &(xhci->rh_bw[vdev->rhub_port->hw_portnum].tts); 932 list_for_each_entry_safe(tt_info, next, tt_list_head, tt_list) { 933 /* is this a hub device that added a tt_info to the tts list */ 934 if (tt_info->slot_id == slot_id) { 935 /* are any devices using this tt_info? */ 936 for (i = 1; i < HCS_MAX_SLOTS(xhci->hcs_params1); i++) { 937 vdev = xhci->devs[i]; 938 if (vdev && (vdev->tt_info == tt_info)) 939 xhci_free_virt_devices_depth_first( 940 xhci, i); 941 } 942 } 943 } 944out: 945 /* we are now at a leaf device */ 946 xhci_debugfs_remove_slot(xhci, slot_id); 947 xhci_free_virt_device(xhci, slot_id); 948} 949 950int xhci_alloc_virt_device(struct xhci_hcd *xhci, int slot_id, 951 struct usb_device *udev, gfp_t flags) 952{ 953 struct xhci_virt_device *dev; 954 int i; 955 956 /* Slot ID 0 is reserved */ 957 if (slot_id == 0 || xhci->devs[slot_id]) { 958 xhci_warn(xhci, "Bad Slot ID %d\n", slot_id); 959 return 0; 960 } 961 962 dev = kzalloc(sizeof(*dev), flags); 963 if (!dev) 964 return 0; 965 966 dev->slot_id = slot_id; 967 968 /* Allocate the (output) device context that will be used in the HC. */ 969 dev->out_ctx = xhci_alloc_container_ctx(xhci, XHCI_CTX_TYPE_DEVICE, flags); 970 if (!dev->out_ctx) 971 goto fail; 972 973 xhci_dbg(xhci, "Slot %d output ctx = 0x%pad (dma)\n", slot_id, &dev->out_ctx->dma); 974 975 /* Allocate the (input) device context for address device command */ 976 dev->in_ctx = xhci_alloc_container_ctx(xhci, XHCI_CTX_TYPE_INPUT, flags); 977 if (!dev->in_ctx) 978 goto fail; 979 980 xhci_dbg(xhci, "Slot %d input ctx = 0x%pad (dma)\n", slot_id, &dev->in_ctx->dma); 981 982 /* Initialize the cancellation and bandwidth list for each ep */ 983 for (i = 0; i < 31; i++) { 984 dev->eps[i].ep_index = i; 985 dev->eps[i].vdev = dev; 986 dev->eps[i].xhci = xhci; 987 INIT_LIST_HEAD(&dev->eps[i].cancelled_td_list); 988 INIT_LIST_HEAD(&dev->eps[i].bw_endpoint_list); 989 } 990 991 /* Allocate endpoint 0 ring */ 992 dev->eps[0].ring = xhci_ring_alloc(xhci, 2, 1, TYPE_CTRL, 0, flags); 993 if (!dev->eps[0].ring) 994 goto fail; 995 996 dev->udev = udev; 997 998 /* Point to output device context in dcbaa. */ 999 xhci->dcbaa->dev_context_ptrs[slot_id] = cpu_to_le64(dev->out_ctx->dma); 1000 xhci_dbg(xhci, "Set slot id %d dcbaa entry %p to 0x%llx\n", 1001 slot_id, 1002 &xhci->dcbaa->dev_context_ptrs[slot_id], 1003 le64_to_cpu(xhci->dcbaa->dev_context_ptrs[slot_id])); 1004 1005 trace_xhci_alloc_virt_device(dev); 1006 1007 xhci->devs[slot_id] = dev; 1008 1009 return 1; 1010fail: 1011 1012 if (dev->in_ctx) 1013 xhci_free_container_ctx(xhci, dev->in_ctx); 1014 if (dev->out_ctx) 1015 xhci_free_container_ctx(xhci, dev->out_ctx); 1016 kfree(dev); 1017 1018 return 0; 1019} 1020 1021void xhci_copy_ep0_dequeue_into_input_ctx(struct xhci_hcd *xhci, 1022 struct usb_device *udev) 1023{ 1024 struct xhci_virt_device *virt_dev; 1025 struct xhci_ep_ctx *ep0_ctx; 1026 struct xhci_ring *ep_ring; 1027 1028 virt_dev = xhci->devs[udev->slot_id]; 1029 ep0_ctx = xhci_get_ep_ctx(xhci, virt_dev->in_ctx, 0); 1030 ep_ring = virt_dev->eps[0].ring; 1031 /* 1032 * FIXME we don't keep track of the dequeue pointer very well after a 1033 * Set TR dequeue pointer, so we're setting the dequeue pointer of the 1034 * host to our enqueue pointer. This should only be called after a 1035 * configured device has reset, so all control transfers should have 1036 * been completed or cancelled before the reset. 1037 */ 1038 ep0_ctx->deq = cpu_to_le64(xhci_trb_virt_to_dma(ep_ring->enq_seg, 1039 ep_ring->enqueue) 1040 | ep_ring->cycle_state); 1041} 1042 1043/* 1044 * The xHCI roothub may have ports of differing speeds in any order in the port 1045 * status registers. 1046 * 1047 * The xHCI hardware wants to know the roothub port that the USB device 1048 * is attached to (or the roothub port its ancestor hub is attached to). All we 1049 * know is the index of that port under either the USB 2.0 or the USB 3.0 1050 * roothub, but that doesn't give us the real index into the HW port status 1051 * registers. 1052 */ 1053static struct xhci_port *xhci_find_rhub_port(struct xhci_hcd *xhci, struct usb_device *udev) 1054{ 1055 struct usb_device *top_dev; 1056 struct xhci_hub *rhub; 1057 struct usb_hcd *hcd; 1058 1059 if (udev->speed >= USB_SPEED_SUPER) 1060 hcd = xhci_get_usb3_hcd(xhci); 1061 else 1062 hcd = xhci->main_hcd; 1063 1064 for (top_dev = udev; top_dev->parent && top_dev->parent->parent; 1065 top_dev = top_dev->parent) 1066 /* Found device below root hub */; 1067 1068 rhub = xhci_get_rhub(hcd); 1069 return rhub->ports[top_dev->portnum - 1]; 1070} 1071 1072/* Setup an xHCI virtual device for a Set Address command */ 1073int xhci_setup_addressable_virt_dev(struct xhci_hcd *xhci, struct usb_device *udev) 1074{ 1075 struct xhci_virt_device *dev; 1076 struct xhci_ep_ctx *ep0_ctx; 1077 struct xhci_slot_ctx *slot_ctx; 1078 u32 max_packets; 1079 1080 dev = xhci->devs[udev->slot_id]; 1081 /* Slot ID 0 is reserved */ 1082 if (udev->slot_id == 0 || !dev) { 1083 xhci_warn(xhci, "Slot ID %d is not assigned to this device\n", 1084 udev->slot_id); 1085 return -EINVAL; 1086 } 1087 ep0_ctx = xhci_get_ep_ctx(xhci, dev->in_ctx, 0); 1088 slot_ctx = xhci_get_slot_ctx(xhci, dev->in_ctx); 1089 1090 /* 3) Only the control endpoint is valid - one endpoint context */ 1091 slot_ctx->dev_info |= cpu_to_le32(LAST_CTX(1) | udev->route); 1092 switch (udev->speed) { 1093 case USB_SPEED_SUPER_PLUS: 1094 slot_ctx->dev_info |= cpu_to_le32(SLOT_SPEED_SSP); 1095 max_packets = MAX_PACKET(512); 1096 break; 1097 case USB_SPEED_SUPER: 1098 slot_ctx->dev_info |= cpu_to_le32(SLOT_SPEED_SS); 1099 max_packets = MAX_PACKET(512); 1100 break; 1101 case USB_SPEED_HIGH: 1102 slot_ctx->dev_info |= cpu_to_le32(SLOT_SPEED_HS); 1103 max_packets = MAX_PACKET(64); 1104 break; 1105 /* USB core guesses at a 64-byte max packet first for FS devices */ 1106 case USB_SPEED_FULL: 1107 slot_ctx->dev_info |= cpu_to_le32(SLOT_SPEED_FS); 1108 max_packets = MAX_PACKET(64); 1109 break; 1110 case USB_SPEED_LOW: 1111 slot_ctx->dev_info |= cpu_to_le32(SLOT_SPEED_LS); 1112 max_packets = MAX_PACKET(8); 1113 break; 1114 default: 1115 /* Speed was set earlier, this shouldn't happen. */ 1116 return -EINVAL; 1117 } 1118 /* Find the root hub port this device is under */ 1119 dev->rhub_port = xhci_find_rhub_port(xhci, udev); 1120 if (!dev->rhub_port) 1121 return -EINVAL; 1122 /* Slot ID is set to the device directly below the root hub */ 1123 if (!udev->parent->parent) 1124 dev->rhub_port->slot_id = udev->slot_id; 1125 slot_ctx->dev_info2 |= cpu_to_le32(ROOT_HUB_PORT(dev->rhub_port->hw_portnum + 1)); 1126 xhci_dbg(xhci, "Slot ID %d: HW portnum %d, hcd portnum %d\n", 1127 udev->slot_id, dev->rhub_port->hw_portnum, dev->rhub_port->hcd_portnum); 1128 1129 /* Find the right bandwidth table that this device will be a part of. 1130 * If this is a full speed device attached directly to a root port (or a 1131 * decendent of one), it counts as a primary bandwidth domain, not a 1132 * secondary bandwidth domain under a TT. An xhci_tt_info structure 1133 * will never be created for the HS root hub. 1134 */ 1135 if (!udev->tt || !udev->tt->hub->parent) { 1136 dev->bw_table = &xhci->rh_bw[dev->rhub_port->hw_portnum].bw_table; 1137 } else { 1138 struct xhci_root_port_bw_info *rh_bw; 1139 struct xhci_tt_bw_info *tt_bw; 1140 1141 rh_bw = &xhci->rh_bw[dev->rhub_port->hw_portnum]; 1142 /* Find the right TT. */ 1143 list_for_each_entry(tt_bw, &rh_bw->tts, tt_list) { 1144 if (tt_bw->slot_id != udev->tt->hub->slot_id) 1145 continue; 1146 1147 if (!dev->udev->tt->multi || 1148 (udev->tt->multi && 1149 tt_bw->ttport == dev->udev->ttport)) { 1150 dev->bw_table = &tt_bw->bw_table; 1151 dev->tt_info = tt_bw; 1152 break; 1153 } 1154 } 1155 if (!dev->tt_info) 1156 xhci_warn(xhci, "WARN: Didn't find a matching TT\n"); 1157 } 1158 1159 /* Is this a LS/FS device under an external HS hub? */ 1160 if (udev->tt && udev->tt->hub->parent) { 1161 slot_ctx->tt_info = cpu_to_le32(udev->tt->hub->slot_id | 1162 (udev->ttport << 8)); 1163 if (udev->tt->multi) 1164 slot_ctx->dev_info |= cpu_to_le32(DEV_MTT); 1165 } 1166 xhci_dbg(xhci, "udev->tt = %p\n", udev->tt); 1167 xhci_dbg(xhci, "udev->ttport = 0x%x\n", udev->ttport); 1168 1169 /* Step 4 - ring already allocated */ 1170 /* Step 5 */ 1171 ep0_ctx->ep_info2 = cpu_to_le32(EP_TYPE(CTRL_EP)); 1172 1173 /* EP 0 can handle "burst" sizes of 1, so Max Burst Size field is 0 */ 1174 ep0_ctx->ep_info2 |= cpu_to_le32(MAX_BURST(0) | ERROR_COUNT(3) | 1175 max_packets); 1176 1177 ep0_ctx->deq = cpu_to_le64(dev->eps[0].ring->first_seg->dma | 1178 dev->eps[0].ring->cycle_state); 1179 1180 trace_xhci_setup_addressable_virt_device(dev); 1181 1182 /* Steps 7 and 8 were done in xhci_alloc_virt_device() */ 1183 1184 return 0; 1185} 1186 1187/* 1188 * Convert interval expressed as 2^(bInterval - 1) == interval into 1189 * straight exponent value 2^n == interval. 1190 * 1191 */ 1192static unsigned int xhci_parse_exponent_interval(struct usb_device *udev, 1193 struct usb_host_endpoint *ep) 1194{ 1195 unsigned int interval; 1196 1197 interval = clamp_val(ep->desc.bInterval, 1, 16) - 1; 1198 if (interval != ep->desc.bInterval - 1) 1199 dev_warn(&udev->dev, 1200 "ep %#x - rounding interval to %d %sframes\n", 1201 ep->desc.bEndpointAddress, 1202 1 << interval, 1203 udev->speed == USB_SPEED_FULL ? "" : "micro"); 1204 1205 if (udev->speed == USB_SPEED_FULL) { 1206 /* 1207 * Full speed isoc endpoints specify interval in frames, 1208 * not microframes. We are using microframes everywhere, 1209 * so adjust accordingly. 1210 */ 1211 interval += 3; /* 1 frame = 2^3 uframes */ 1212 } 1213 1214 return interval; 1215} 1216 1217/* 1218 * Convert bInterval expressed in microframes (in 1-255 range) to exponent of 1219 * microframes, rounded down to nearest power of 2. 1220 */ 1221static unsigned int xhci_microframes_to_exponent(struct usb_device *udev, 1222 struct usb_host_endpoint *ep, unsigned int desc_interval, 1223 unsigned int min_exponent, unsigned int max_exponent) 1224{ 1225 unsigned int interval; 1226 1227 interval = fls(desc_interval) - 1; 1228 interval = clamp_val(interval, min_exponent, max_exponent); 1229 if ((1 << interval) != desc_interval) 1230 dev_dbg(&udev->dev, 1231 "ep %#x - rounding interval to %d microframes, ep desc says %d microframes\n", 1232 ep->desc.bEndpointAddress, 1233 1 << interval, 1234 desc_interval); 1235 1236 return interval; 1237} 1238 1239static unsigned int xhci_parse_microframe_interval(struct usb_device *udev, 1240 struct usb_host_endpoint *ep) 1241{ 1242 if (ep->desc.bInterval == 0) 1243 return 0; 1244 return xhci_microframes_to_exponent(udev, ep, 1245 ep->desc.bInterval, 0, 15); 1246} 1247 1248 1249static unsigned int xhci_parse_frame_interval(struct usb_device *udev, 1250 struct usb_host_endpoint *ep) 1251{ 1252 return xhci_microframes_to_exponent(udev, ep, 1253 ep->desc.bInterval * 8, 3, 10); 1254} 1255 1256/* Return the polling or NAK interval. 1257 * 1258 * The polling interval is expressed in "microframes". If xHCI's Interval field 1259 * is set to N, it will service the endpoint every 2^(Interval)*125us. 1260 * 1261 * The NAK interval is one NAK per 1 to 255 microframes, or no NAKs if interval 1262 * is set to 0. 1263 */ 1264static unsigned int xhci_get_endpoint_interval(struct usb_device *udev, 1265 struct usb_host_endpoint *ep) 1266{ 1267 unsigned int interval = 0; 1268 1269 switch (udev->speed) { 1270 case USB_SPEED_HIGH: 1271 /* Max NAK rate */ 1272 if (usb_endpoint_xfer_control(&ep->desc) || 1273 usb_endpoint_xfer_bulk(&ep->desc)) { 1274 interval = xhci_parse_microframe_interval(udev, ep); 1275 break; 1276 } 1277 fallthrough; /* SS and HS isoc/int have same decoding */ 1278 1279 case USB_SPEED_SUPER_PLUS: 1280 case USB_SPEED_SUPER: 1281 if (usb_endpoint_xfer_int(&ep->desc) || 1282 usb_endpoint_xfer_isoc(&ep->desc)) { 1283 interval = xhci_parse_exponent_interval(udev, ep); 1284 } 1285 break; 1286 1287 case USB_SPEED_FULL: 1288 if (usb_endpoint_xfer_isoc(&ep->desc)) { 1289 interval = xhci_parse_exponent_interval(udev, ep); 1290 break; 1291 } 1292 /* 1293 * Fall through for interrupt endpoint interval decoding 1294 * since it uses the same rules as low speed interrupt 1295 * endpoints. 1296 */ 1297 fallthrough; 1298 1299 case USB_SPEED_LOW: 1300 if (usb_endpoint_xfer_int(&ep->desc) || 1301 usb_endpoint_xfer_isoc(&ep->desc)) { 1302 1303 interval = xhci_parse_frame_interval(udev, ep); 1304 } 1305 break; 1306 1307 default: 1308 BUG(); 1309 } 1310 return interval; 1311} 1312 1313/* The "Mult" field in the endpoint context is only set for SuperSpeed isoc eps. 1314 * High speed endpoint descriptors can define "the number of additional 1315 * transaction opportunities per microframe", but that goes in the Max Burst 1316 * endpoint context field. 1317 */ 1318static u32 xhci_get_endpoint_mult(struct usb_device *udev, 1319 struct usb_host_endpoint *ep) 1320{ 1321 if (udev->speed < USB_SPEED_SUPER || 1322 !usb_endpoint_xfer_isoc(&ep->desc)) 1323 return 0; 1324 return ep->ss_ep_comp.bmAttributes; 1325} 1326 1327static u32 xhci_get_endpoint_max_burst(struct usb_device *udev, 1328 struct usb_host_endpoint *ep) 1329{ 1330 /* Super speed and Plus have max burst in ep companion desc */ 1331 if (udev->speed >= USB_SPEED_SUPER) 1332 return ep->ss_ep_comp.bMaxBurst; 1333 1334 if (udev->speed == USB_SPEED_HIGH && 1335 (usb_endpoint_xfer_isoc(&ep->desc) || 1336 usb_endpoint_xfer_int(&ep->desc))) 1337 return usb_endpoint_maxp_mult(&ep->desc) - 1; 1338 1339 return 0; 1340} 1341 1342static u32 xhci_get_endpoint_type(struct usb_host_endpoint *ep) 1343{ 1344 int in; 1345 1346 in = usb_endpoint_dir_in(&ep->desc); 1347 1348 switch (usb_endpoint_type(&ep->desc)) { 1349 case USB_ENDPOINT_XFER_CONTROL: 1350 return CTRL_EP; 1351 case USB_ENDPOINT_XFER_BULK: 1352 return in ? BULK_IN_EP : BULK_OUT_EP; 1353 case USB_ENDPOINT_XFER_ISOC: 1354 return in ? ISOC_IN_EP : ISOC_OUT_EP; 1355 case USB_ENDPOINT_XFER_INT: 1356 return in ? INT_IN_EP : INT_OUT_EP; 1357 } 1358 return 0; 1359} 1360 1361/* Return the maximum endpoint service interval time (ESIT) payload. 1362 * Basically, this is the maxpacket size, multiplied by the burst size 1363 * and mult size. 1364 */ 1365static u32 xhci_get_max_esit_payload(struct usb_device *udev, 1366 struct usb_host_endpoint *ep) 1367{ 1368 int max_burst; 1369 int max_packet; 1370 1371 /* Only applies for interrupt or isochronous endpoints */ 1372 if (usb_endpoint_xfer_control(&ep->desc) || 1373 usb_endpoint_xfer_bulk(&ep->desc)) 1374 return 0; 1375 1376 /* SuperSpeedPlus Isoc ep sending over 48k per esit */ 1377 if ((udev->speed >= USB_SPEED_SUPER_PLUS) && 1378 USB_SS_SSP_ISOC_COMP(ep->ss_ep_comp.bmAttributes)) 1379 return le32_to_cpu(ep->ssp_isoc_ep_comp.dwBytesPerInterval); 1380 1381 /* SuperSpeed or SuperSpeedPlus Isoc ep with less than 48k per esit */ 1382 if (udev->speed >= USB_SPEED_SUPER) 1383 return le16_to_cpu(ep->ss_ep_comp.wBytesPerInterval); 1384 1385 max_packet = usb_endpoint_maxp(&ep->desc); 1386 max_burst = usb_endpoint_maxp_mult(&ep->desc); 1387 /* A 0 in max burst means 1 transfer per ESIT */ 1388 return max_packet * max_burst; 1389} 1390 1391/* Set up an endpoint with one ring segment. Do not allocate stream rings. 1392 * Drivers will have to call usb_alloc_streams() to do that. 1393 */ 1394int xhci_endpoint_init(struct xhci_hcd *xhci, 1395 struct xhci_virt_device *virt_dev, 1396 struct usb_device *udev, 1397 struct usb_host_endpoint *ep, 1398 gfp_t mem_flags) 1399{ 1400 unsigned int ep_index; 1401 struct xhci_ep_ctx *ep_ctx; 1402 struct xhci_ring *ep_ring; 1403 unsigned int max_packet; 1404 enum xhci_ring_type ring_type; 1405 u32 max_esit_payload; 1406 u32 endpoint_type; 1407 unsigned int max_burst; 1408 unsigned int interval; 1409 unsigned int mult; 1410 unsigned int avg_trb_len; 1411 unsigned int err_count = 0; 1412 1413 ep_index = xhci_get_endpoint_index(&ep->desc); 1414 ep_ctx = xhci_get_ep_ctx(xhci, virt_dev->in_ctx, ep_index); 1415 1416 endpoint_type = xhci_get_endpoint_type(ep); 1417 if (!endpoint_type) 1418 return -EINVAL; 1419 1420 ring_type = usb_endpoint_type(&ep->desc); 1421 1422 /* 1423 * Get values to fill the endpoint context, mostly from ep descriptor. 1424 * The average TRB buffer lengt for bulk endpoints is unclear as we 1425 * have no clue on scatter gather list entry size. For Isoc and Int, 1426 * set it to max available. See xHCI 1.1 spec 4.14.1.1 for details. 1427 */ 1428 max_esit_payload = xhci_get_max_esit_payload(udev, ep); 1429 interval = xhci_get_endpoint_interval(udev, ep); 1430 1431 /* Periodic endpoint bInterval limit quirk */ 1432 if (usb_endpoint_xfer_int(&ep->desc) || 1433 usb_endpoint_xfer_isoc(&ep->desc)) { 1434 if ((xhci->quirks & XHCI_LIMIT_ENDPOINT_INTERVAL_7) && 1435 udev->speed >= USB_SPEED_HIGH && 1436 interval >= 7) { 1437 interval = 6; 1438 } 1439 } 1440 1441 mult = xhci_get_endpoint_mult(udev, ep); 1442 max_packet = usb_endpoint_maxp(&ep->desc); 1443 max_burst = xhci_get_endpoint_max_burst(udev, ep); 1444 avg_trb_len = max_esit_payload; 1445 1446 /* FIXME dig Mult and streams info out of ep companion desc */ 1447 1448 /* Allow 3 retries for everything but isoc, set CErr = 3 */ 1449 if (!usb_endpoint_xfer_isoc(&ep->desc)) 1450 err_count = 3; 1451 /* HS bulk max packet should be 512, FS bulk supports 8, 16, 32 or 64 */ 1452 if (usb_endpoint_xfer_bulk(&ep->desc)) { 1453 if (udev->speed == USB_SPEED_HIGH) 1454 max_packet = 512; 1455 if (udev->speed == USB_SPEED_FULL) { 1456 max_packet = rounddown_pow_of_two(max_packet); 1457 max_packet = clamp_val(max_packet, 8, 64); 1458 } 1459 } 1460 /* xHCI 1.0 and 1.1 indicates that ctrl ep avg TRB Length should be 8 */ 1461 if (usb_endpoint_xfer_control(&ep->desc) && xhci->hci_version >= 0x100) 1462 avg_trb_len = 8; 1463 /* xhci 1.1 with LEC support doesn't use mult field, use RsvdZ */ 1464 if ((xhci->hci_version > 0x100) && HCC2_LEC(xhci->hcc_params2)) 1465 mult = 0; 1466 1467 /* Set up the endpoint ring */ 1468 virt_dev->eps[ep_index].new_ring = 1469 xhci_ring_alloc(xhci, 2, 1, ring_type, max_packet, mem_flags); 1470 if (!virt_dev->eps[ep_index].new_ring) 1471 return -ENOMEM; 1472 1473 virt_dev->eps[ep_index].skip = false; 1474 ep_ring = virt_dev->eps[ep_index].new_ring; 1475 1476 /* Fill the endpoint context */ 1477 ep_ctx->ep_info = cpu_to_le32(EP_MAX_ESIT_PAYLOAD_HI(max_esit_payload) | 1478 EP_INTERVAL(interval) | 1479 EP_MULT(mult)); 1480 ep_ctx->ep_info2 = cpu_to_le32(EP_TYPE(endpoint_type) | 1481 MAX_PACKET(max_packet) | 1482 MAX_BURST(max_burst) | 1483 ERROR_COUNT(err_count)); 1484 ep_ctx->deq = cpu_to_le64(ep_ring->first_seg->dma | 1485 ep_ring->cycle_state); 1486 1487 ep_ctx->tx_info = cpu_to_le32(EP_MAX_ESIT_PAYLOAD_LO(max_esit_payload) | 1488 EP_AVG_TRB_LENGTH(avg_trb_len)); 1489 1490 return 0; 1491} 1492 1493void xhci_endpoint_zero(struct xhci_hcd *xhci, 1494 struct xhci_virt_device *virt_dev, 1495 struct usb_host_endpoint *ep) 1496{ 1497 unsigned int ep_index; 1498 struct xhci_ep_ctx *ep_ctx; 1499 1500 ep_index = xhci_get_endpoint_index(&ep->desc); 1501 ep_ctx = xhci_get_ep_ctx(xhci, virt_dev->in_ctx, ep_index); 1502 1503 ep_ctx->ep_info = 0; 1504 ep_ctx->ep_info2 = 0; 1505 ep_ctx->deq = 0; 1506 ep_ctx->tx_info = 0; 1507 /* Don't free the endpoint ring until the set interface or configuration 1508 * request succeeds. 1509 */ 1510} 1511 1512void xhci_clear_endpoint_bw_info(struct xhci_bw_info *bw_info) 1513{ 1514 bw_info->ep_interval = 0; 1515 bw_info->mult = 0; 1516 bw_info->num_packets = 0; 1517 bw_info->max_packet_size = 0; 1518 bw_info->type = 0; 1519 bw_info->max_esit_payload = 0; 1520} 1521 1522void xhci_update_bw_info(struct xhci_hcd *xhci, 1523 struct xhci_container_ctx *in_ctx, 1524 struct xhci_input_control_ctx *ctrl_ctx, 1525 struct xhci_virt_device *virt_dev) 1526{ 1527 struct xhci_bw_info *bw_info; 1528 struct xhci_ep_ctx *ep_ctx; 1529 unsigned int ep_type; 1530 int i; 1531 1532 for (i = 1; i < 31; i++) { 1533 bw_info = &virt_dev->eps[i].bw_info; 1534 1535 /* We can't tell what endpoint type is being dropped, but 1536 * unconditionally clearing the bandwidth info for non-periodic 1537 * endpoints should be harmless because the info will never be 1538 * set in the first place. 1539 */ 1540 if (!EP_IS_ADDED(ctrl_ctx, i) && EP_IS_DROPPED(ctrl_ctx, i)) { 1541 /* Dropped endpoint */ 1542 xhci_clear_endpoint_bw_info(bw_info); 1543 continue; 1544 } 1545 1546 if (EP_IS_ADDED(ctrl_ctx, i)) { 1547 ep_ctx = xhci_get_ep_ctx(xhci, in_ctx, i); 1548 ep_type = CTX_TO_EP_TYPE(le32_to_cpu(ep_ctx->ep_info2)); 1549 1550 /* Ignore non-periodic endpoints */ 1551 if (ep_type != ISOC_OUT_EP && ep_type != INT_OUT_EP && 1552 ep_type != ISOC_IN_EP && 1553 ep_type != INT_IN_EP) 1554 continue; 1555 1556 /* Added or changed endpoint */ 1557 bw_info->ep_interval = CTX_TO_EP_INTERVAL( 1558 le32_to_cpu(ep_ctx->ep_info)); 1559 /* Number of packets and mult are zero-based in the 1560 * input context, but we want one-based for the 1561 * interval table. 1562 */ 1563 bw_info->mult = CTX_TO_EP_MULT( 1564 le32_to_cpu(ep_ctx->ep_info)) + 1; 1565 bw_info->num_packets = CTX_TO_MAX_BURST( 1566 le32_to_cpu(ep_ctx->ep_info2)) + 1; 1567 bw_info->max_packet_size = MAX_PACKET_DECODED( 1568 le32_to_cpu(ep_ctx->ep_info2)); 1569 bw_info->type = ep_type; 1570 bw_info->max_esit_payload = CTX_TO_MAX_ESIT_PAYLOAD( 1571 le32_to_cpu(ep_ctx->tx_info)); 1572 } 1573 } 1574} 1575 1576/* Copy output xhci_ep_ctx to the input xhci_ep_ctx copy. 1577 * Useful when you want to change one particular aspect of the endpoint and then 1578 * issue a configure endpoint command. 1579 */ 1580void xhci_endpoint_copy(struct xhci_hcd *xhci, 1581 struct xhci_container_ctx *in_ctx, 1582 struct xhci_container_ctx *out_ctx, 1583 unsigned int ep_index) 1584{ 1585 struct xhci_ep_ctx *out_ep_ctx; 1586 struct xhci_ep_ctx *in_ep_ctx; 1587 1588 out_ep_ctx = xhci_get_ep_ctx(xhci, out_ctx, ep_index); 1589 in_ep_ctx = xhci_get_ep_ctx(xhci, in_ctx, ep_index); 1590 1591 in_ep_ctx->ep_info = out_ep_ctx->ep_info; 1592 in_ep_ctx->ep_info2 = out_ep_ctx->ep_info2; 1593 in_ep_ctx->deq = out_ep_ctx->deq; 1594 in_ep_ctx->tx_info = out_ep_ctx->tx_info; 1595 if (xhci->quirks & XHCI_MTK_HOST) { 1596 in_ep_ctx->reserved[0] = out_ep_ctx->reserved[0]; 1597 in_ep_ctx->reserved[1] = out_ep_ctx->reserved[1]; 1598 } 1599} 1600 1601/* Copy output xhci_slot_ctx to the input xhci_slot_ctx. 1602 * Useful when you want to change one particular aspect of the endpoint and then 1603 * issue a configure endpoint command. Only the context entries field matters, 1604 * but we'll copy the whole thing anyway. 1605 */ 1606void xhci_slot_copy(struct xhci_hcd *xhci, 1607 struct xhci_container_ctx *in_ctx, 1608 struct xhci_container_ctx *out_ctx) 1609{ 1610 struct xhci_slot_ctx *in_slot_ctx; 1611 struct xhci_slot_ctx *out_slot_ctx; 1612 1613 in_slot_ctx = xhci_get_slot_ctx(xhci, in_ctx); 1614 out_slot_ctx = xhci_get_slot_ctx(xhci, out_ctx); 1615 1616 in_slot_ctx->dev_info = out_slot_ctx->dev_info; 1617 in_slot_ctx->dev_info2 = out_slot_ctx->dev_info2; 1618 in_slot_ctx->tt_info = out_slot_ctx->tt_info; 1619 in_slot_ctx->dev_state = out_slot_ctx->dev_state; 1620} 1621 1622/* Set up the scratchpad buffer array and scratchpad buffers, if needed. */ 1623static int scratchpad_alloc(struct xhci_hcd *xhci, gfp_t flags) 1624{ 1625 int i; 1626 struct device *dev = xhci_to_hcd(xhci)->self.sysdev; 1627 int num_sp = HCS_MAX_SCRATCHPAD(xhci->hcs_params2); 1628 1629 xhci_dbg_trace(xhci, trace_xhci_dbg_init, 1630 "Allocating %d scratchpad buffers", num_sp); 1631 1632 if (!num_sp) 1633 return 0; 1634 1635 xhci->scratchpad = kzalloc_node(sizeof(*xhci->scratchpad), flags, 1636 dev_to_node(dev)); 1637 if (!xhci->scratchpad) 1638 goto fail_sp; 1639 1640 xhci->scratchpad->sp_array = dma_alloc_coherent(dev, 1641 size_mul(sizeof(u64), num_sp), 1642 &xhci->scratchpad->sp_dma, flags); 1643 if (!xhci->scratchpad->sp_array) 1644 goto fail_sp2; 1645 1646 xhci->scratchpad->sp_buffers = kcalloc_node(num_sp, sizeof(void *), 1647 flags, dev_to_node(dev)); 1648 if (!xhci->scratchpad->sp_buffers) 1649 goto fail_sp3; 1650 1651 xhci->dcbaa->dev_context_ptrs[0] = cpu_to_le64(xhci->scratchpad->sp_dma); 1652 for (i = 0; i < num_sp; i++) { 1653 dma_addr_t dma; 1654 void *buf = dma_alloc_coherent(dev, xhci->page_size, &dma, 1655 flags); 1656 if (!buf) 1657 goto fail_sp4; 1658 1659 xhci->scratchpad->sp_array[i] = dma; 1660 xhci->scratchpad->sp_buffers[i] = buf; 1661 } 1662 1663 return 0; 1664 1665 fail_sp4: 1666 while (i--) 1667 dma_free_coherent(dev, xhci->page_size, 1668 xhci->scratchpad->sp_buffers[i], 1669 xhci->scratchpad->sp_array[i]); 1670 1671 kfree(xhci->scratchpad->sp_buffers); 1672 1673 fail_sp3: 1674 dma_free_coherent(dev, num_sp * sizeof(u64), 1675 xhci->scratchpad->sp_array, 1676 xhci->scratchpad->sp_dma); 1677 1678 fail_sp2: 1679 kfree(xhci->scratchpad); 1680 xhci->scratchpad = NULL; 1681 1682 fail_sp: 1683 return -ENOMEM; 1684} 1685 1686static void scratchpad_free(struct xhci_hcd *xhci) 1687{ 1688 int num_sp; 1689 int i; 1690 struct device *dev = xhci_to_hcd(xhci)->self.sysdev; 1691 1692 if (!xhci->scratchpad) 1693 return; 1694 1695 num_sp = HCS_MAX_SCRATCHPAD(xhci->hcs_params2); 1696 1697 for (i = 0; i < num_sp; i++) { 1698 dma_free_coherent(dev, xhci->page_size, 1699 xhci->scratchpad->sp_buffers[i], 1700 xhci->scratchpad->sp_array[i]); 1701 } 1702 kfree(xhci->scratchpad->sp_buffers); 1703 dma_free_coherent(dev, num_sp * sizeof(u64), 1704 xhci->scratchpad->sp_array, 1705 xhci->scratchpad->sp_dma); 1706 kfree(xhci->scratchpad); 1707 xhci->scratchpad = NULL; 1708} 1709 1710struct xhci_command *xhci_alloc_command(struct xhci_hcd *xhci, 1711 bool allocate_completion, gfp_t mem_flags) 1712{ 1713 struct xhci_command *command; 1714 struct device *dev = xhci_to_hcd(xhci)->self.sysdev; 1715 1716 command = kzalloc_node(sizeof(*command), mem_flags, dev_to_node(dev)); 1717 if (!command) 1718 return NULL; 1719 1720 if (allocate_completion) { 1721 command->completion = 1722 kzalloc_node(sizeof(struct completion), mem_flags, 1723 dev_to_node(dev)); 1724 if (!command->completion) { 1725 kfree(command); 1726 return NULL; 1727 } 1728 init_completion(command->completion); 1729 } 1730 1731 command->status = 0; 1732 /* set default timeout to 5000 ms */ 1733 command->timeout_ms = XHCI_CMD_DEFAULT_TIMEOUT; 1734 INIT_LIST_HEAD(&command->cmd_list); 1735 return command; 1736} 1737 1738struct xhci_command *xhci_alloc_command_with_ctx(struct xhci_hcd *xhci, 1739 bool allocate_completion, gfp_t mem_flags) 1740{ 1741 struct xhci_command *command; 1742 1743 command = xhci_alloc_command(xhci, allocate_completion, mem_flags); 1744 if (!command) 1745 return NULL; 1746 1747 command->in_ctx = xhci_alloc_container_ctx(xhci, XHCI_CTX_TYPE_INPUT, 1748 mem_flags); 1749 if (!command->in_ctx) { 1750 kfree(command->completion); 1751 kfree(command); 1752 return NULL; 1753 } 1754 return command; 1755} 1756 1757void xhci_urb_free_priv(struct urb_priv *urb_priv) 1758{ 1759 kfree(urb_priv); 1760} 1761 1762void xhci_free_command(struct xhci_hcd *xhci, 1763 struct xhci_command *command) 1764{ 1765 xhci_free_container_ctx(xhci, 1766 command->in_ctx); 1767 kfree(command->completion); 1768 kfree(command); 1769} 1770 1771static int xhci_alloc_erst(struct xhci_hcd *xhci, 1772 struct xhci_ring *evt_ring, 1773 struct xhci_erst *erst, 1774 gfp_t flags) 1775{ 1776 size_t size; 1777 unsigned int val; 1778 struct xhci_segment *seg; 1779 struct xhci_erst_entry *entry; 1780 1781 size = size_mul(sizeof(struct xhci_erst_entry), evt_ring->num_segs); 1782 erst->entries = dma_alloc_coherent(xhci_to_hcd(xhci)->self.sysdev, 1783 size, &erst->erst_dma_addr, flags); 1784 if (!erst->entries) 1785 return -ENOMEM; 1786 1787 erst->num_entries = evt_ring->num_segs; 1788 1789 seg = evt_ring->first_seg; 1790 for (val = 0; val < evt_ring->num_segs; val++) { 1791 entry = &erst->entries[val]; 1792 entry->seg_addr = cpu_to_le64(seg->dma); 1793 entry->seg_size = cpu_to_le32(TRBS_PER_SEGMENT); 1794 entry->rsvd = 0; 1795 seg = seg->next; 1796 } 1797 1798 return 0; 1799} 1800 1801static void 1802xhci_remove_interrupter(struct xhci_hcd *xhci, struct xhci_interrupter *ir) 1803{ 1804 u32 tmp; 1805 1806 if (!ir) 1807 return; 1808 1809 /* 1810 * Clean out interrupter registers except ERSTBA. Clearing either the 1811 * low or high 32 bits of ERSTBA immediately causes the controller to 1812 * dereference the partially cleared 64 bit address, causing IOMMU error. 1813 */ 1814 if (ir->ir_set) { 1815 tmp = readl(&ir->ir_set->erst_size); 1816 tmp &= ERST_SIZE_MASK; 1817 writel(tmp, &ir->ir_set->erst_size); 1818 1819 xhci_write_64(xhci, ERST_EHB, &ir->ir_set->erst_dequeue); 1820 } 1821} 1822 1823static void 1824xhci_free_interrupter(struct xhci_hcd *xhci, struct xhci_interrupter *ir) 1825{ 1826 struct device *dev = xhci_to_hcd(xhci)->self.sysdev; 1827 size_t erst_size; 1828 1829 if (!ir) 1830 return; 1831 1832 erst_size = sizeof(struct xhci_erst_entry) * ir->erst.num_entries; 1833 if (ir->erst.entries) 1834 dma_free_coherent(dev, erst_size, 1835 ir->erst.entries, 1836 ir->erst.erst_dma_addr); 1837 ir->erst.entries = NULL; 1838 1839 /* free interrupter event ring */ 1840 if (ir->event_ring) 1841 xhci_ring_free(xhci, ir->event_ring); 1842 1843 ir->event_ring = NULL; 1844 1845 kfree(ir); 1846} 1847 1848void xhci_remove_secondary_interrupter(struct usb_hcd *hcd, struct xhci_interrupter *ir) 1849{ 1850 struct xhci_hcd *xhci = hcd_to_xhci(hcd); 1851 unsigned int intr_num; 1852 1853 spin_lock_irq(&xhci->lock); 1854 1855 /* interrupter 0 is primary interrupter, don't touch it */ 1856 if (!ir || !ir->intr_num || ir->intr_num >= xhci->max_interrupters) { 1857 xhci_dbg(xhci, "Invalid secondary interrupter, can't remove\n"); 1858 spin_unlock_irq(&xhci->lock); 1859 return; 1860 } 1861 1862 intr_num = ir->intr_num; 1863 1864 xhci_remove_interrupter(xhci, ir); 1865 xhci->interrupters[intr_num] = NULL; 1866 1867 spin_unlock_irq(&xhci->lock); 1868 1869 xhci_free_interrupter(xhci, ir); 1870} 1871EXPORT_SYMBOL_GPL(xhci_remove_secondary_interrupter); 1872 1873void xhci_mem_cleanup(struct xhci_hcd *xhci) 1874{ 1875 struct device *dev = xhci_to_hcd(xhci)->self.sysdev; 1876 int i, j, num_ports; 1877 1878 cancel_delayed_work_sync(&xhci->cmd_timer); 1879 1880 for (i = 0; i < xhci->max_interrupters; i++) { 1881 if (xhci->interrupters[i]) { 1882 xhci_remove_interrupter(xhci, xhci->interrupters[i]); 1883 xhci_free_interrupter(xhci, xhci->interrupters[i]); 1884 xhci->interrupters[i] = NULL; 1885 } 1886 } 1887 xhci_dbg_trace(xhci, trace_xhci_dbg_init, "Freed interrupters"); 1888 1889 if (xhci->cmd_ring) 1890 xhci_ring_free(xhci, xhci->cmd_ring); 1891 xhci->cmd_ring = NULL; 1892 xhci_dbg_trace(xhci, trace_xhci_dbg_init, "Freed command ring"); 1893 xhci_cleanup_command_queue(xhci); 1894 1895 num_ports = HCS_MAX_PORTS(xhci->hcs_params1); 1896 for (i = 0; i < num_ports && xhci->rh_bw; i++) { 1897 struct xhci_interval_bw_table *bwt = &xhci->rh_bw[i].bw_table; 1898 for (j = 0; j < XHCI_MAX_INTERVAL; j++) { 1899 struct list_head *ep = &bwt->interval_bw[j].endpoints; 1900 while (!list_empty(ep)) 1901 list_del_init(ep->next); 1902 } 1903 } 1904 1905 for (i = HCS_MAX_SLOTS(xhci->hcs_params1); i > 0; i--) 1906 xhci_free_virt_devices_depth_first(xhci, i); 1907 1908 dma_pool_destroy(xhci->segment_pool); 1909 xhci->segment_pool = NULL; 1910 xhci_dbg_trace(xhci, trace_xhci_dbg_init, "Freed segment pool"); 1911 1912 dma_pool_destroy(xhci->device_pool); 1913 xhci->device_pool = NULL; 1914 xhci_dbg_trace(xhci, trace_xhci_dbg_init, "Freed device context pool"); 1915 1916 dma_pool_destroy(xhci->small_streams_pool); 1917 xhci->small_streams_pool = NULL; 1918 xhci_dbg_trace(xhci, trace_xhci_dbg_init, 1919 "Freed small stream array pool"); 1920 1921 dma_pool_destroy(xhci->medium_streams_pool); 1922 xhci->medium_streams_pool = NULL; 1923 xhci_dbg_trace(xhci, trace_xhci_dbg_init, 1924 "Freed medium stream array pool"); 1925 1926 if (xhci->dcbaa) 1927 dma_free_coherent(dev, sizeof(*xhci->dcbaa), 1928 xhci->dcbaa, xhci->dcbaa->dma); 1929 xhci->dcbaa = NULL; 1930 1931 scratchpad_free(xhci); 1932 1933 if (!xhci->rh_bw) 1934 goto no_bw; 1935 1936 for (i = 0; i < num_ports; i++) { 1937 struct xhci_tt_bw_info *tt, *n; 1938 list_for_each_entry_safe(tt, n, &xhci->rh_bw[i].tts, tt_list) { 1939 list_del(&tt->tt_list); 1940 kfree(tt); 1941 } 1942 } 1943 1944no_bw: 1945 xhci->cmd_ring_reserved_trbs = 0; 1946 xhci->usb2_rhub.num_ports = 0; 1947 xhci->usb3_rhub.num_ports = 0; 1948 xhci->num_active_eps = 0; 1949 kfree(xhci->usb2_rhub.ports); 1950 kfree(xhci->usb3_rhub.ports); 1951 kfree(xhci->hw_ports); 1952 kfree(xhci->rh_bw); 1953 kfree(xhci->ext_caps); 1954 for (i = 0; i < xhci->num_port_caps; i++) 1955 kfree(xhci->port_caps[i].psi); 1956 kfree(xhci->port_caps); 1957 kfree(xhci->interrupters); 1958 xhci->num_port_caps = 0; 1959 1960 xhci->usb2_rhub.ports = NULL; 1961 xhci->usb3_rhub.ports = NULL; 1962 xhci->hw_ports = NULL; 1963 xhci->rh_bw = NULL; 1964 xhci->ext_caps = NULL; 1965 xhci->port_caps = NULL; 1966 xhci->interrupters = NULL; 1967 1968 xhci->page_size = 0; 1969 xhci->page_shift = 0; 1970 xhci->usb2_rhub.bus_state.bus_suspended = 0; 1971 xhci->usb3_rhub.bus_state.bus_suspended = 0; 1972} 1973 1974static void xhci_set_hc_event_deq(struct xhci_hcd *xhci, struct xhci_interrupter *ir) 1975{ 1976 dma_addr_t deq; 1977 1978 deq = xhci_trb_virt_to_dma(ir->event_ring->deq_seg, 1979 ir->event_ring->dequeue); 1980 if (!deq) 1981 xhci_warn(xhci, "WARN something wrong with SW event ring dequeue ptr.\n"); 1982 /* Update HC event ring dequeue pointer */ 1983 /* Don't clear the EHB bit (which is RW1C) because 1984 * there might be more events to service. 1985 */ 1986 xhci_dbg_trace(xhci, trace_xhci_dbg_init, 1987 "// Write event ring dequeue pointer, preserving EHB bit"); 1988 xhci_write_64(xhci, deq & ERST_PTR_MASK, &ir->ir_set->erst_dequeue); 1989} 1990 1991static void xhci_add_in_port(struct xhci_hcd *xhci, unsigned int num_ports, 1992 __le32 __iomem *addr, int max_caps) 1993{ 1994 u32 temp, port_offset, port_count; 1995 int i; 1996 u8 major_revision, minor_revision, tmp_minor_revision; 1997 struct xhci_hub *rhub; 1998 struct device *dev = xhci_to_hcd(xhci)->self.sysdev; 1999 struct xhci_port_cap *port_cap; 2000 2001 temp = readl(addr); 2002 major_revision = XHCI_EXT_PORT_MAJOR(temp); 2003 minor_revision = XHCI_EXT_PORT_MINOR(temp); 2004 2005 if (major_revision == 0x03) { 2006 rhub = &xhci->usb3_rhub; 2007 /* 2008 * Some hosts incorrectly use sub-minor version for minor 2009 * version (i.e. 0x02 instead of 0x20 for bcdUSB 0x320 and 0x01 2010 * for bcdUSB 0x310). Since there is no USB release with sub 2011 * minor version 0x301 to 0x309, we can assume that they are 2012 * incorrect and fix it here. 2013 */ 2014 if (minor_revision > 0x00 && minor_revision < 0x10) 2015 minor_revision <<= 4; 2016 /* 2017 * Some zhaoxin's xHCI controller that follow usb3.1 spec 2018 * but only support Gen1. 2019 */ 2020 if (xhci->quirks & XHCI_ZHAOXIN_HOST) { 2021 tmp_minor_revision = minor_revision; 2022 minor_revision = 0; 2023 } 2024 2025 } else if (major_revision <= 0x02) { 2026 rhub = &xhci->usb2_rhub; 2027 } else { 2028 xhci_warn(xhci, "Ignoring unknown port speed, Ext Cap %p, revision = 0x%x\n", 2029 addr, major_revision); 2030 /* Ignoring port protocol we can't understand. FIXME */ 2031 return; 2032 } 2033 2034 /* Port offset and count in the third dword, see section 7.2 */ 2035 temp = readl(addr + 2); 2036 port_offset = XHCI_EXT_PORT_OFF(temp); 2037 port_count = XHCI_EXT_PORT_COUNT(temp); 2038 xhci_dbg_trace(xhci, trace_xhci_dbg_init, 2039 "Ext Cap %p, port offset = %u, count = %u, revision = 0x%x", 2040 addr, port_offset, port_count, major_revision); 2041 /* Port count includes the current port offset */ 2042 if (port_offset == 0 || (port_offset + port_count - 1) > num_ports) 2043 /* WTF? "Valid values are ���1��� to MaxPorts" */ 2044 return; 2045 2046 port_cap = &xhci->port_caps[xhci->num_port_caps++]; 2047 if (xhci->num_port_caps > max_caps) 2048 return; 2049 2050 port_cap->psi_count = XHCI_EXT_PORT_PSIC(temp); 2051 2052 if (port_cap->psi_count) { 2053 port_cap->psi = kcalloc_node(port_cap->psi_count, 2054 sizeof(*port_cap->psi), 2055 GFP_KERNEL, dev_to_node(dev)); 2056 if (!port_cap->psi) 2057 port_cap->psi_count = 0; 2058 2059 port_cap->psi_uid_count++; 2060 for (i = 0; i < port_cap->psi_count; i++) { 2061 port_cap->psi[i] = readl(addr + 4 + i); 2062 2063 /* count unique ID values, two consecutive entries can 2064 * have the same ID if link is assymetric 2065 */ 2066 if (i && (XHCI_EXT_PORT_PSIV(port_cap->psi[i]) != 2067 XHCI_EXT_PORT_PSIV(port_cap->psi[i - 1]))) 2068 port_cap->psi_uid_count++; 2069 2070 if (xhci->quirks & XHCI_ZHAOXIN_HOST && 2071 major_revision == 0x03 && 2072 XHCI_EXT_PORT_PSIV(port_cap->psi[i]) >= 5) 2073 minor_revision = tmp_minor_revision; 2074 2075 xhci_dbg(xhci, "PSIV:%d PSIE:%d PLT:%d PFD:%d LP:%d PSIM:%d\n", 2076 XHCI_EXT_PORT_PSIV(port_cap->psi[i]), 2077 XHCI_EXT_PORT_PSIE(port_cap->psi[i]), 2078 XHCI_EXT_PORT_PLT(port_cap->psi[i]), 2079 XHCI_EXT_PORT_PFD(port_cap->psi[i]), 2080 XHCI_EXT_PORT_LP(port_cap->psi[i]), 2081 XHCI_EXT_PORT_PSIM(port_cap->psi[i])); 2082 } 2083 } 2084 2085 rhub->maj_rev = major_revision; 2086 2087 if (rhub->min_rev < minor_revision) 2088 rhub->min_rev = minor_revision; 2089 2090 port_cap->maj_rev = major_revision; 2091 port_cap->min_rev = minor_revision; 2092 2093 /* cache usb2 port capabilities */ 2094 if (major_revision < 0x03 && xhci->num_ext_caps < max_caps) 2095 xhci->ext_caps[xhci->num_ext_caps++] = temp; 2096 2097 if ((xhci->hci_version >= 0x100) && (major_revision != 0x03) && 2098 (temp & XHCI_HLC)) { 2099 xhci_dbg_trace(xhci, trace_xhci_dbg_init, 2100 "xHCI 1.0: support USB2 hardware lpm"); 2101 xhci->hw_lpm_support = 1; 2102 } 2103 2104 port_offset--; 2105 for (i = port_offset; i < (port_offset + port_count); i++) { 2106 struct xhci_port *hw_port = &xhci->hw_ports[i]; 2107 /* Duplicate entry. Ignore the port if the revisions differ. */ 2108 if (hw_port->rhub) { 2109 xhci_warn(xhci, "Duplicate port entry, Ext Cap %p, port %u\n", addr, i); 2110 xhci_warn(xhci, "Port was marked as USB %u, duplicated as USB %u\n", 2111 hw_port->rhub->maj_rev, major_revision); 2112 /* Only adjust the roothub port counts if we haven't 2113 * found a similar duplicate. 2114 */ 2115 if (hw_port->rhub != rhub && 2116 hw_port->hcd_portnum != DUPLICATE_ENTRY) { 2117 hw_port->rhub->num_ports--; 2118 hw_port->hcd_portnum = DUPLICATE_ENTRY; 2119 } 2120 continue; 2121 } 2122 hw_port->rhub = rhub; 2123 hw_port->port_cap = port_cap; 2124 rhub->num_ports++; 2125 } 2126 /* FIXME: Should we disable ports not in the Extended Capabilities? */ 2127} 2128 2129static void xhci_create_rhub_port_array(struct xhci_hcd *xhci, 2130 struct xhci_hub *rhub, gfp_t flags) 2131{ 2132 int port_index = 0; 2133 int i; 2134 struct device *dev = xhci_to_hcd(xhci)->self.sysdev; 2135 2136 if (!rhub->num_ports) 2137 return; 2138 rhub->ports = kcalloc_node(rhub->num_ports, sizeof(*rhub->ports), 2139 flags, dev_to_node(dev)); 2140 if (!rhub->ports) 2141 return; 2142 2143 for (i = 0; i < HCS_MAX_PORTS(xhci->hcs_params1); i++) { 2144 if (xhci->hw_ports[i].rhub != rhub || 2145 xhci->hw_ports[i].hcd_portnum == DUPLICATE_ENTRY) 2146 continue; 2147 xhci->hw_ports[i].hcd_portnum = port_index; 2148 rhub->ports[port_index] = &xhci->hw_ports[i]; 2149 port_index++; 2150 if (port_index == rhub->num_ports) 2151 break; 2152 } 2153} 2154 2155/* 2156 * Scan the Extended Capabilities for the "Supported Protocol Capabilities" that 2157 * specify what speeds each port is supposed to be. We can't count on the port 2158 * speed bits in the PORTSC register being correct until a device is connected, 2159 * but we need to set up the two fake roothubs with the correct number of USB 2160 * 3.0 and USB 2.0 ports at host controller initialization time. 2161 */ 2162static int xhci_setup_port_arrays(struct xhci_hcd *xhci, gfp_t flags) 2163{ 2164 void __iomem *base; 2165 u32 offset; 2166 unsigned int num_ports; 2167 int i, j; 2168 int cap_count = 0; 2169 u32 cap_start; 2170 struct device *dev = xhci_to_hcd(xhci)->self.sysdev; 2171 2172 num_ports = HCS_MAX_PORTS(xhci->hcs_params1); 2173 xhci->hw_ports = kcalloc_node(num_ports, sizeof(*xhci->hw_ports), 2174 flags, dev_to_node(dev)); 2175 if (!xhci->hw_ports) 2176 return -ENOMEM; 2177 2178 for (i = 0; i < num_ports; i++) { 2179 xhci->hw_ports[i].addr = &xhci->op_regs->port_status_base + 2180 NUM_PORT_REGS * i; 2181 xhci->hw_ports[i].hw_portnum = i; 2182 2183 init_completion(&xhci->hw_ports[i].rexit_done); 2184 init_completion(&xhci->hw_ports[i].u3exit_done); 2185 } 2186 2187 xhci->rh_bw = kcalloc_node(num_ports, sizeof(*xhci->rh_bw), flags, 2188 dev_to_node(dev)); 2189 if (!xhci->rh_bw) 2190 return -ENOMEM; 2191 for (i = 0; i < num_ports; i++) { 2192 struct xhci_interval_bw_table *bw_table; 2193 2194 INIT_LIST_HEAD(&xhci->rh_bw[i].tts); 2195 bw_table = &xhci->rh_bw[i].bw_table; 2196 for (j = 0; j < XHCI_MAX_INTERVAL; j++) 2197 INIT_LIST_HEAD(&bw_table->interval_bw[j].endpoints); 2198 } 2199 base = &xhci->cap_regs->hc_capbase; 2200 2201 cap_start = xhci_find_next_ext_cap(base, 0, XHCI_EXT_CAPS_PROTOCOL); 2202 if (!cap_start) { 2203 xhci_err(xhci, "No Extended Capability registers, unable to set up roothub\n"); 2204 return -ENODEV; 2205 } 2206 2207 offset = cap_start; 2208 /* count extended protocol capability entries for later caching */ 2209 while (offset) { 2210 cap_count++; 2211 offset = xhci_find_next_ext_cap(base, offset, 2212 XHCI_EXT_CAPS_PROTOCOL); 2213 } 2214 2215 xhci->ext_caps = kcalloc_node(cap_count, sizeof(*xhci->ext_caps), 2216 flags, dev_to_node(dev)); 2217 if (!xhci->ext_caps) 2218 return -ENOMEM; 2219 2220 xhci->port_caps = kcalloc_node(cap_count, sizeof(*xhci->port_caps), 2221 flags, dev_to_node(dev)); 2222 if (!xhci->port_caps) 2223 return -ENOMEM; 2224 2225 offset = cap_start; 2226 2227 while (offset) { 2228 xhci_add_in_port(xhci, num_ports, base + offset, cap_count); 2229 if (xhci->usb2_rhub.num_ports + xhci->usb3_rhub.num_ports == 2230 num_ports) 2231 break; 2232 offset = xhci_find_next_ext_cap(base, offset, 2233 XHCI_EXT_CAPS_PROTOCOL); 2234 } 2235 if (xhci->usb2_rhub.num_ports == 0 && xhci->usb3_rhub.num_ports == 0) { 2236 xhci_warn(xhci, "No ports on the roothubs?\n"); 2237 return -ENODEV; 2238 } 2239 xhci_dbg_trace(xhci, trace_xhci_dbg_init, 2240 "Found %u USB 2.0 ports and %u USB 3.0 ports.", 2241 xhci->usb2_rhub.num_ports, xhci->usb3_rhub.num_ports); 2242 2243 /* Place limits on the number of roothub ports so that the hub 2244 * descriptors aren't longer than the USB core will allocate. 2245 */ 2246 if (xhci->usb3_rhub.num_ports > USB_SS_MAXPORTS) { 2247 xhci_dbg_trace(xhci, trace_xhci_dbg_init, 2248 "Limiting USB 3.0 roothub ports to %u.", 2249 USB_SS_MAXPORTS); 2250 xhci->usb3_rhub.num_ports = USB_SS_MAXPORTS; 2251 } 2252 if (xhci->usb2_rhub.num_ports > USB_MAXCHILDREN) { 2253 xhci_dbg_trace(xhci, trace_xhci_dbg_init, 2254 "Limiting USB 2.0 roothub ports to %u.", 2255 USB_MAXCHILDREN); 2256 xhci->usb2_rhub.num_ports = USB_MAXCHILDREN; 2257 } 2258 2259 if (!xhci->usb2_rhub.num_ports) 2260 xhci_info(xhci, "USB2 root hub has no ports\n"); 2261 2262 if (!xhci->usb3_rhub.num_ports) 2263 xhci_info(xhci, "USB3 root hub has no ports\n"); 2264 2265 xhci_create_rhub_port_array(xhci, &xhci->usb2_rhub, flags); 2266 xhci_create_rhub_port_array(xhci, &xhci->usb3_rhub, flags); 2267 2268 return 0; 2269} 2270 2271static struct xhci_interrupter * 2272xhci_alloc_interrupter(struct xhci_hcd *xhci, int segs, gfp_t flags) 2273{ 2274 struct device *dev = xhci_to_hcd(xhci)->self.sysdev; 2275 struct xhci_interrupter *ir; 2276 unsigned int num_segs = segs; 2277 int ret; 2278 2279 ir = kzalloc_node(sizeof(*ir), flags, dev_to_node(dev)); 2280 if (!ir) 2281 return NULL; 2282 2283 /* number of ring segments should be greater than 0 */ 2284 if (segs <= 0) 2285 num_segs = min_t(unsigned int, 1 << HCS_ERST_MAX(xhci->hcs_params2), 2286 ERST_MAX_SEGS); 2287 2288 ir->event_ring = xhci_ring_alloc(xhci, num_segs, 1, TYPE_EVENT, 0, 2289 flags); 2290 if (!ir->event_ring) { 2291 xhci_warn(xhci, "Failed to allocate interrupter event ring\n"); 2292 kfree(ir); 2293 return NULL; 2294 } 2295 2296 ret = xhci_alloc_erst(xhci, ir->event_ring, &ir->erst, flags); 2297 if (ret) { 2298 xhci_warn(xhci, "Failed to allocate interrupter erst\n"); 2299 xhci_ring_free(xhci, ir->event_ring); 2300 kfree(ir); 2301 return NULL; 2302 } 2303 2304 return ir; 2305} 2306 2307static int 2308xhci_add_interrupter(struct xhci_hcd *xhci, struct xhci_interrupter *ir, 2309 unsigned int intr_num) 2310{ 2311 u64 erst_base; 2312 u32 erst_size; 2313 2314 if (intr_num >= xhci->max_interrupters) { 2315 xhci_warn(xhci, "Can't add interrupter %d, max interrupters %d\n", 2316 intr_num, xhci->max_interrupters); 2317 return -EINVAL; 2318 } 2319 2320 if (xhci->interrupters[intr_num]) { 2321 xhci_warn(xhci, "Interrupter %d\n already set up", intr_num); 2322 return -EINVAL; 2323 } 2324 2325 xhci->interrupters[intr_num] = ir; 2326 ir->intr_num = intr_num; 2327 ir->ir_set = &xhci->run_regs->ir_set[intr_num]; 2328 2329 /* set ERST count with the number of entries in the segment table */ 2330 erst_size = readl(&ir->ir_set->erst_size); 2331 erst_size &= ERST_SIZE_MASK; 2332 erst_size |= ir->event_ring->num_segs; 2333 writel(erst_size, &ir->ir_set->erst_size); 2334 2335 erst_base = xhci_read_64(xhci, &ir->ir_set->erst_base); 2336 erst_base &= ERST_BASE_RSVDP; 2337 erst_base |= ir->erst.erst_dma_addr & ~ERST_BASE_RSVDP; 2338 xhci_write_64(xhci, erst_base, &ir->ir_set->erst_base); 2339 2340 /* Set the event ring dequeue address of this interrupter */ 2341 xhci_set_hc_event_deq(xhci, ir); 2342 2343 return 0; 2344} 2345 2346struct xhci_interrupter * 2347xhci_create_secondary_interrupter(struct usb_hcd *hcd, int num_seg) 2348{ 2349 struct xhci_hcd *xhci = hcd_to_xhci(hcd); 2350 struct xhci_interrupter *ir; 2351 unsigned int i; 2352 int err = -ENOSPC; 2353 2354 if (!xhci->interrupters || xhci->max_interrupters <= 1) 2355 return NULL; 2356 2357 ir = xhci_alloc_interrupter(xhci, num_seg, GFP_KERNEL); 2358 if (!ir) 2359 return NULL; 2360 2361 spin_lock_irq(&xhci->lock); 2362 2363 /* Find available secondary interrupter, interrupter 0 is reserved for primary */ 2364 for (i = 1; i < xhci->max_interrupters; i++) { 2365 if (xhci->interrupters[i] == NULL) { 2366 err = xhci_add_interrupter(xhci, ir, i); 2367 break; 2368 } 2369 } 2370 2371 spin_unlock_irq(&xhci->lock); 2372 2373 if (err) { 2374 xhci_warn(xhci, "Failed to add secondary interrupter, max interrupters %d\n", 2375 xhci->max_interrupters); 2376 xhci_free_interrupter(xhci, ir); 2377 return NULL; 2378 } 2379 2380 xhci_dbg(xhci, "Add secondary interrupter %d, max interrupters %d\n", 2381 i, xhci->max_interrupters); 2382 2383 return ir; 2384} 2385EXPORT_SYMBOL_GPL(xhci_create_secondary_interrupter); 2386 2387int xhci_mem_init(struct xhci_hcd *xhci, gfp_t flags) 2388{ 2389 struct xhci_interrupter *ir; 2390 struct device *dev = xhci_to_hcd(xhci)->self.sysdev; 2391 dma_addr_t dma; 2392 unsigned int val, val2; 2393 u64 val_64; 2394 u32 page_size, temp; 2395 int i; 2396 2397 INIT_LIST_HEAD(&xhci->cmd_list); 2398 2399 /* init command timeout work */ 2400 INIT_DELAYED_WORK(&xhci->cmd_timer, xhci_handle_command_timeout); 2401 init_completion(&xhci->cmd_ring_stop_completion); 2402 2403 page_size = readl(&xhci->op_regs->page_size); 2404 xhci_dbg_trace(xhci, trace_xhci_dbg_init, 2405 "Supported page size register = 0x%x", page_size); 2406 i = ffs(page_size); 2407 if (i < 16) 2408 xhci_dbg_trace(xhci, trace_xhci_dbg_init, 2409 "Supported page size of %iK", (1 << (i+12)) / 1024); 2410 else 2411 xhci_warn(xhci, "WARN: no supported page size\n"); 2412 /* Use 4K pages, since that's common and the minimum the HC supports */ 2413 xhci->page_shift = 12; 2414 xhci->page_size = 1 << xhci->page_shift; 2415 xhci_dbg_trace(xhci, trace_xhci_dbg_init, 2416 "HCD page size set to %iK", xhci->page_size / 1024); 2417 2418 /* 2419 * Program the Number of Device Slots Enabled field in the CONFIG 2420 * register with the max value of slots the HC can handle. 2421 */ 2422 val = HCS_MAX_SLOTS(readl(&xhci->cap_regs->hcs_params1)); 2423 xhci_dbg_trace(xhci, trace_xhci_dbg_init, 2424 "// xHC can handle at most %d device slots.", val); 2425 val2 = readl(&xhci->op_regs->config_reg); 2426 val |= (val2 & ~HCS_SLOTS_MASK); 2427 xhci_dbg_trace(xhci, trace_xhci_dbg_init, 2428 "// Setting Max device slots reg = 0x%x.", val); 2429 writel(val, &xhci->op_regs->config_reg); 2430 2431 /* 2432 * xHCI section 5.4.6 - Device Context array must be 2433 * "physically contiguous and 64-byte (cache line) aligned". 2434 */ 2435 xhci->dcbaa = dma_alloc_coherent(dev, sizeof(*xhci->dcbaa), &dma, 2436 flags); 2437 if (!xhci->dcbaa) 2438 goto fail; 2439 xhci->dcbaa->dma = dma; 2440 xhci_dbg_trace(xhci, trace_xhci_dbg_init, 2441 "// Device context base array address = 0x%pad (DMA), %p (virt)", 2442 &xhci->dcbaa->dma, xhci->dcbaa); 2443 xhci_write_64(xhci, dma, &xhci->op_regs->dcbaa_ptr); 2444 2445 /* 2446 * Initialize the ring segment pool. The ring must be a contiguous 2447 * structure comprised of TRBs. The TRBs must be 16 byte aligned, 2448 * however, the command ring segment needs 64-byte aligned segments 2449 * and our use of dma addresses in the trb_address_map radix tree needs 2450 * TRB_SEGMENT_SIZE alignment, so we pick the greater alignment need. 2451 */ 2452 if (xhci->quirks & XHCI_ZHAOXIN_TRB_FETCH) 2453 xhci->segment_pool = dma_pool_create("xHCI ring segments", dev, 2454 TRB_SEGMENT_SIZE * 2, TRB_SEGMENT_SIZE * 2, xhci->page_size * 2); 2455 else 2456 xhci->segment_pool = dma_pool_create("xHCI ring segments", dev, 2457 TRB_SEGMENT_SIZE, TRB_SEGMENT_SIZE, xhci->page_size); 2458 2459 /* See Table 46 and Note on Figure 55 */ 2460 xhci->device_pool = dma_pool_create("xHCI input/output contexts", dev, 2461 2112, 64, xhci->page_size); 2462 if (!xhci->segment_pool || !xhci->device_pool) 2463 goto fail; 2464 2465 /* Linear stream context arrays don't have any boundary restrictions, 2466 * and only need to be 16-byte aligned. 2467 */ 2468 xhci->small_streams_pool = 2469 dma_pool_create("xHCI 256 byte stream ctx arrays", 2470 dev, SMALL_STREAM_ARRAY_SIZE, 16, 0); 2471 xhci->medium_streams_pool = 2472 dma_pool_create("xHCI 1KB stream ctx arrays", 2473 dev, MEDIUM_STREAM_ARRAY_SIZE, 16, 0); 2474 /* Any stream context array bigger than MEDIUM_STREAM_ARRAY_SIZE 2475 * will be allocated with dma_alloc_coherent() 2476 */ 2477 2478 if (!xhci->small_streams_pool || !xhci->medium_streams_pool) 2479 goto fail; 2480 2481 /* Set up the command ring to have one segments for now. */ 2482 xhci->cmd_ring = xhci_ring_alloc(xhci, 1, 1, TYPE_COMMAND, 0, flags); 2483 if (!xhci->cmd_ring) 2484 goto fail; 2485 xhci_dbg_trace(xhci, trace_xhci_dbg_init, 2486 "Allocated command ring at %p", xhci->cmd_ring); 2487 xhci_dbg_trace(xhci, trace_xhci_dbg_init, "First segment DMA is 0x%pad", 2488 &xhci->cmd_ring->first_seg->dma); 2489 2490 /* Set the address in the Command Ring Control register */ 2491 val_64 = xhci_read_64(xhci, &xhci->op_regs->cmd_ring); 2492 val_64 = (val_64 & (u64) CMD_RING_RSVD_BITS) | 2493 (xhci->cmd_ring->first_seg->dma & (u64) ~CMD_RING_RSVD_BITS) | 2494 xhci->cmd_ring->cycle_state; 2495 xhci_dbg_trace(xhci, trace_xhci_dbg_init, 2496 "// Setting command ring address to 0x%016llx", val_64); 2497 xhci_write_64(xhci, val_64, &xhci->op_regs->cmd_ring); 2498 2499 /* Reserve one command ring TRB for disabling LPM. 2500 * Since the USB core grabs the shared usb_bus bandwidth mutex before 2501 * disabling LPM, we only need to reserve one TRB for all devices. 2502 */ 2503 xhci->cmd_ring_reserved_trbs++; 2504 2505 val = readl(&xhci->cap_regs->db_off); 2506 val &= DBOFF_MASK; 2507 xhci_dbg_trace(xhci, trace_xhci_dbg_init, 2508 "// Doorbell array is located at offset 0x%x from cap regs base addr", 2509 val); 2510 xhci->dba = (void __iomem *) xhci->cap_regs + val; 2511 2512 /* Allocate and set up primary interrupter 0 with an event ring. */ 2513 xhci_dbg_trace(xhci, trace_xhci_dbg_init, 2514 "Allocating primary event ring"); 2515 xhci->interrupters = kcalloc_node(xhci->max_interrupters, sizeof(*xhci->interrupters), 2516 flags, dev_to_node(dev)); 2517 2518 ir = xhci_alloc_interrupter(xhci, 0, flags); 2519 if (!ir) 2520 goto fail; 2521 2522 if (xhci_add_interrupter(xhci, ir, 0)) 2523 goto fail; 2524 2525 ir->isoc_bei_interval = AVOID_BEI_INTERVAL_MAX; 2526 2527 /* 2528 * XXX: Might need to set the Interrupter Moderation Register to 2529 * something other than the default (~1ms minimum between interrupts). 2530 * See section 5.5.1.2. 2531 */ 2532 for (i = 0; i < MAX_HC_SLOTS; i++) 2533 xhci->devs[i] = NULL; 2534 2535 if (scratchpad_alloc(xhci, flags)) 2536 goto fail; 2537 if (xhci_setup_port_arrays(xhci, flags)) 2538 goto fail; 2539 2540 /* Enable USB 3.0 device notifications for function remote wake, which 2541 * is necessary for allowing USB 3.0 devices to do remote wakeup from 2542 * U3 (device suspend). 2543 */ 2544 temp = readl(&xhci->op_regs->dev_notification); 2545 temp &= ~DEV_NOTE_MASK; 2546 temp |= DEV_NOTE_FWAKE; 2547 writel(temp, &xhci->op_regs->dev_notification); 2548 2549 return 0; 2550 2551fail: 2552 xhci_halt(xhci); 2553 xhci_reset(xhci, XHCI_RESET_SHORT_USEC); 2554 xhci_mem_cleanup(xhci); 2555 return -ENOMEM; 2556} 2557